From acf86568a7e21176ba2cca15861da231bec6932a Mon Sep 17 00:00:00 2001 From: Anton Ivashkin Date: Thu, 8 Oct 2020 18:45:10 +0300 Subject: [PATCH 001/544] S3 zero copy replication proof of concept --- src/Disks/DiskCacheWrapper.cpp | 18 +- src/Disks/DiskCacheWrapper.h | 6 +- src/Disks/DiskDecorator.cpp | 12 +- src/Disks/DiskDecorator.h | 7 +- src/Disks/DiskLocal.cpp | 6 +- src/Disks/DiskLocal.h | 6 +- src/Disks/DiskMemory.cpp | 6 +- src/Disks/DiskMemory.h | 6 +- src/Disks/IDisk.h | 13 +- src/Disks/S3/DiskS3.cpp | 34 ++- src/Disks/S3/DiskS3.h | 8 +- src/Interpreters/InterserverIOHandler.h | 8 + src/Storages/MergeTree/DataPartsExchange.cpp | 238 +++++++++++++++++- src/Storages/MergeTree/DataPartsExchange.h | 39 ++- src/Storages/MergeTree/IMergeTreeDataPart.cpp | 20 +- src/Storages/MergeTree/IMergeTreeDataPart.h | 2 +- src/Storages/StorageReplicatedMergeTree.cpp | 34 ++- 17 files changed, 392 insertions(+), 71 deletions(-) diff --git a/src/Disks/DiskCacheWrapper.cpp b/src/Disks/DiskCacheWrapper.cpp index c60f69920f4..94b15920cee 100644 --- a/src/Disks/DiskCacheWrapper.cpp +++ b/src/Disks/DiskCacheWrapper.cpp @@ -198,11 +198,11 @@ DiskCacheWrapper::writeFile(const String & path, size_t buf_size, WriteMode mode buf_size); } -void DiskCacheWrapper::clearDirectory(const String & path) +void DiskCacheWrapper::clearDirectory(const String & path, bool keep_s3) { if (cache_disk->exists(path)) - cache_disk->clearDirectory(path); - DiskDecorator::clearDirectory(path); + cache_disk->clearDirectory(path, keep_s3); + DiskDecorator::clearDirectory(path, keep_s3); } void DiskCacheWrapper::moveDirectory(const String & from_path, const String & to_path) @@ -251,18 +251,18 @@ void DiskCacheWrapper::copyFile(const String & from_path, const String & to_path DiskDecorator::copyFile(from_path, to_path); } -void DiskCacheWrapper::remove(const String & path) +void DiskCacheWrapper::remove(const String & path, bool keep_s3) { if (cache_disk->exists(path)) - cache_disk->remove(path); - DiskDecorator::remove(path); + cache_disk->remove(path, keep_s3); + DiskDecorator::remove(path, keep_s3); } -void DiskCacheWrapper::removeRecursive(const String & path) +void DiskCacheWrapper::removeRecursive(const String & path, bool keep_s3) { if (cache_disk->exists(path)) - cache_disk->removeRecursive(path); - DiskDecorator::removeRecursive(path); + cache_disk->removeRecursive(path, keep_s3); + DiskDecorator::removeRecursive(path, keep_s3); } void DiskCacheWrapper::createHardLink(const String & src_path, const String & dst_path) diff --git a/src/Disks/DiskCacheWrapper.h b/src/Disks/DiskCacheWrapper.h index b0b373d900c..9fca4e02e34 100644 --- a/src/Disks/DiskCacheWrapper.h +++ b/src/Disks/DiskCacheWrapper.h @@ -28,7 +28,7 @@ public: std::function cache_file_predicate_); void createDirectory(const String & path) override; void createDirectories(const String & path) override; - void clearDirectory(const String & path) override; + void clearDirectory(const String & path, bool keep_s3 = false) override; void moveDirectory(const String & from_path, const String & to_path) override; void moveFile(const String & from_path, const String & to_path) override; void replaceFile(const String & from_path, const String & to_path) override; @@ -37,8 +37,8 @@ public: readFile(const String & path, size_t buf_size, size_t estimated_size, size_t aio_threshold, size_t mmap_threshold) const override; std::unique_ptr writeFile(const String & path, size_t buf_size, WriteMode mode, size_t estimated_size, size_t aio_threshold) override; - void remove(const String & path) override; - void removeRecursive(const String & path) override; + void remove(const String & path, bool keep_s3 = false) override; + void removeRecursive(const String & path, bool keep_s3 = false) override; void createHardLink(const String & src_path, const String & dst_path) override; ReservationPtr reserve(UInt64 bytes) override; diff --git a/src/Disks/DiskDecorator.cpp b/src/Disks/DiskDecorator.cpp index 7f2ea58d7cf..9d61141a162 100644 --- a/src/Disks/DiskDecorator.cpp +++ b/src/Disks/DiskDecorator.cpp @@ -73,9 +73,9 @@ void DiskDecorator::createDirectories(const String & path) delegate->createDirectories(path); } -void DiskDecorator::clearDirectory(const String & path) +void DiskDecorator::clearDirectory(const String & path, bool keep_s3) { - delegate->clearDirectory(path); + delegate->clearDirectory(path, keep_s3); } void DiskDecorator::moveDirectory(const String & from_path, const String & to_path) @@ -130,14 +130,14 @@ DiskDecorator::writeFile(const String & path, size_t buf_size, WriteMode mode, s return delegate->writeFile(path, buf_size, mode, estimated_size, aio_threshold); } -void DiskDecorator::remove(const String & path) +void DiskDecorator::remove(const String & path, bool keep_s3) { - delegate->remove(path); + delegate->remove(path, keep_s3); } -void DiskDecorator::removeRecursive(const String & path) +void DiskDecorator::removeRecursive(const String & path, bool keep_s3) { - delegate->removeRecursive(path); + delegate->removeRecursive(path, keep_s3); } void DiskDecorator::setLastModified(const String & path, const Poco::Timestamp & timestamp) diff --git a/src/Disks/DiskDecorator.h b/src/Disks/DiskDecorator.h index f1ddfff4952..f1fea043843 100644 --- a/src/Disks/DiskDecorator.h +++ b/src/Disks/DiskDecorator.h @@ -22,7 +22,7 @@ public: size_t getFileSize(const String & path) const override; void createDirectory(const String & path) override; void createDirectories(const String & path) override; - void clearDirectory(const String & path) override; + void clearDirectory(const String & path, bool keep_s3 = false) override; void moveDirectory(const String & from_path, const String & to_path) override; DiskDirectoryIteratorPtr iterateDirectory(const String & path) override; void createFile(const String & path) override; @@ -35,8 +35,8 @@ public: readFile(const String & path, size_t buf_size, size_t estimated_size, size_t aio_threshold, size_t mmap_threshold) const override; std::unique_ptr writeFile(const String & path, size_t buf_size, WriteMode mode, size_t estimated_size, size_t aio_threshold) override; - void remove(const String & path) override; - void removeRecursive(const String & path) override; + void remove(const String & path, bool keep_s3 = false) override; + void removeRecursive(const String & path, bool keep_s3 = false) override; void setLastModified(const String & path, const Poco::Timestamp & timestamp) override; Poco::Timestamp getLastModified(const String & path) override; void setReadOnly(const String & path) override; @@ -46,6 +46,7 @@ public: void close(int fd) const override; void sync(int fd) const override; const String getType() const override { return delegate->getType(); } + const String getUniqueId(const String & path) const override { return delegate->getUniqueId(path); } protected: DiskPtr delegate; diff --git a/src/Disks/DiskLocal.cpp b/src/Disks/DiskLocal.cpp index a09ab7c5ac5..ad85fdf4236 100644 --- a/src/Disks/DiskLocal.cpp +++ b/src/Disks/DiskLocal.cpp @@ -180,7 +180,7 @@ void DiskLocal::createDirectories(const String & path) Poco::File(disk_path + path).createDirectories(); } -void DiskLocal::clearDirectory(const String & path) +void DiskLocal::clearDirectory(const String & path, bool) { std::vector files; Poco::File(disk_path + path).list(files); @@ -236,12 +236,12 @@ DiskLocal::writeFile(const String & path, size_t buf_size, WriteMode mode, size_ return createWriteBufferFromFileBase(disk_path + path, estimated_size, aio_threshold, buf_size, flags); } -void DiskLocal::remove(const String & path) +void DiskLocal::remove(const String & path, bool) { Poco::File(disk_path + path).remove(false); } -void DiskLocal::removeRecursive(const String & path) +void DiskLocal::removeRecursive(const String & path, bool) { Poco::File(disk_path + path).remove(true); } diff --git a/src/Disks/DiskLocal.h b/src/Disks/DiskLocal.h index 762a8502faa..18e6d072874 100644 --- a/src/Disks/DiskLocal.h +++ b/src/Disks/DiskLocal.h @@ -55,7 +55,7 @@ public: void createDirectories(const String & path) override; - void clearDirectory(const String & path) override; + void clearDirectory(const String & path, bool keep_s3 = false) override; void moveDirectory(const String & from_path, const String & to_path) override; @@ -87,9 +87,9 @@ public: size_t estimated_size, size_t aio_threshold) override; - void remove(const String & path) override; + void remove(const String & path, bool keep_s3 = false) override; - void removeRecursive(const String & path) override; + void removeRecursive(const String & path, bool keep_s3 = false) override; void setLastModified(const String & path, const Poco::Timestamp & timestamp) override; diff --git a/src/Disks/DiskMemory.cpp b/src/Disks/DiskMemory.cpp index d185263d48c..fc375707feb 100644 --- a/src/Disks/DiskMemory.cpp +++ b/src/Disks/DiskMemory.cpp @@ -233,7 +233,7 @@ void DiskMemory::createDirectoriesImpl(const String & path) files.emplace(path, FileData{FileType::Directory}); } -void DiskMemory::clearDirectory(const String & path) +void DiskMemory::clearDirectory(const String & path, bool) { std::lock_guard lock(mutex); @@ -348,7 +348,7 @@ std::unique_ptr DiskMemory::writeFile(const String & pa return std::make_unique(this, path, mode, buf_size); } -void DiskMemory::remove(const String & path) +void DiskMemory::remove(const String & path, bool) { std::lock_guard lock(mutex); @@ -368,7 +368,7 @@ void DiskMemory::remove(const String & path) } } -void DiskMemory::removeRecursive(const String & path) +void DiskMemory::removeRecursive(const String & path, bool) { std::lock_guard lock(mutex); diff --git a/src/Disks/DiskMemory.h b/src/Disks/DiskMemory.h index 4d4b947098b..e75d9bff100 100644 --- a/src/Disks/DiskMemory.h +++ b/src/Disks/DiskMemory.h @@ -48,7 +48,7 @@ public: void createDirectories(const String & path) override; - void clearDirectory(const String & path) override; + void clearDirectory(const String & path, bool keep_s3 = false) override; void moveDirectory(const String & from_path, const String & to_path) override; @@ -78,9 +78,9 @@ public: size_t estimated_size, size_t aio_threshold) override; - void remove(const String & path) override; + void remove(const String & path, bool keep_s3 = false) override; - void removeRecursive(const String & path) override; + void removeRecursive(const String & path, bool keep_s3 = false) override; void setLastModified(const String &, const Poco::Timestamp &) override {} diff --git a/src/Disks/IDisk.h b/src/Disks/IDisk.h index 688c1dfad42..324384fade6 100644 --- a/src/Disks/IDisk.h +++ b/src/Disks/IDisk.h @@ -105,7 +105,7 @@ public: virtual void createDirectories(const String & path) = 0; /// Remove all files from the directory. Directories are not removed. - virtual void clearDirectory(const String & path) = 0; + virtual void clearDirectory(const String & path, bool keep_s3 = false) = 0; /// Move directory from `from_path` to `to_path`. virtual void moveDirectory(const String & from_path, const String & to_path) = 0; @@ -153,16 +153,16 @@ public: size_t aio_threshold = 0) = 0; /// Remove file or directory. Throws exception if file doesn't exists or if directory is not empty. - virtual void remove(const String & path) = 0; + virtual void remove(const String & path, bool keep_s3 = false) = 0; /// Remove file or directory with all children. Use with extra caution. Throws exception if file doesn't exists. - virtual void removeRecursive(const String & path) = 0; + virtual void removeRecursive(const String & path, bool keep_s3 = false) = 0; /// Remove file or directory if it exists. - void removeIfExists(const String & path) + void removeIfExists(const String & path, bool keep_s3 = false) { if (exists(path)) - remove(path); + remove(path, keep_s3); } /// Set last modified time to file or directory at `path`. @@ -195,6 +195,9 @@ public: /// Invoked when Global Context is shutdown. virtual void shutdown() { } + /// Return some uniq string for file, overrided for S3 + virtual const String getUniqueId(const String & path) const { return path; } + private: /// Returns executor to perform asynchronous operations. Executor & getExecutor() { return *executor; } diff --git a/src/Disks/S3/DiskS3.cpp b/src/Disks/S3/DiskS3.cpp index 6abb72efeb0..8b6c3c8465c 100644 --- a/src/Disks/S3/DiskS3.cpp +++ b/src/Disks/S3/DiskS3.cpp @@ -535,16 +535,25 @@ void DiskS3::createDirectories(const String & path) Poco::File(metadata_path + path).createDirectories(); } +const String DiskS3::getUniqueId(const String & path) const +{ + Metadata metadata(s3_root_path, metadata_path, path); + String id; + if (!metadata.s3_objects.empty()) + id = metadata.s3_objects[0].first; + return id; +} + DiskDirectoryIteratorPtr DiskS3::iterateDirectory(const String & path) { return std::make_unique(metadata_path + path, path); } -void DiskS3::clearDirectory(const String & path) +void DiskS3::clearDirectory(const String & path, bool keep_s3) { for (auto it{iterateDirectory(path)}; it->isValid(); it->next()) if (isFile(it->path())) - remove(it->path()); + remove(it->path(), keep_s3); } void DiskS3::moveFile(const String & from_path, const String & to_path) @@ -634,7 +643,7 @@ std::unique_ptr DiskS3::writeFile(const String & path, } } -void DiskS3::remove(const String & path) +void DiskS3::remove(const String & path, bool keep_s3) { LOG_DEBUG(&Poco::Logger::get("DiskS3"), "Remove file by path: {}", backQuote(metadata_path + path)); @@ -647,13 +656,16 @@ void DiskS3::remove(const String & path) if (metadata.ref_count == 0) { file.remove(); - for (const auto & [s3_object_path, _] : metadata.s3_objects) + if (!keep_s3) { - /// TODO: Make operation idempotent. Do not throw exception if key is already deleted. - Aws::S3::Model::DeleteObjectRequest request; - request.SetBucket(bucket); - request.SetKey(s3_root_path + s3_object_path); - throwIfError(client->DeleteObject(request)); + for (const auto & [s3_object_path, _] : metadata.s3_objects) + { + /// TODO: Make operation idempotent. Do not throw exception if key is already deleted. + Aws::S3::Model::DeleteObjectRequest request; + request.SetBucket(bucket); + request.SetKey(s3_root_path + s3_object_path); + throwIfError(client->DeleteObject(request)); + } } } else /// In other case decrement number of references, save metadata and delete file. @@ -667,7 +679,7 @@ void DiskS3::remove(const String & path) file.remove(); } -void DiskS3::removeRecursive(const String & path) +void DiskS3::removeRecursive(const String & path, bool keep_s3) { checkStackSize(); /// This is needed to prevent stack overflow in case of cyclic symlinks. @@ -679,7 +691,7 @@ void DiskS3::removeRecursive(const String & path) else { for (auto it{iterateDirectory(path)}; it->isValid(); it->next()) - removeRecursive(it->path()); + removeRecursive(it->path(), keep_s3); file.remove(); } } diff --git a/src/Disks/S3/DiskS3.h b/src/Disks/S3/DiskS3.h index 2d9c7f79865..48644dcccf0 100644 --- a/src/Disks/S3/DiskS3.h +++ b/src/Disks/S3/DiskS3.h @@ -58,7 +58,7 @@ public: void createDirectories(const String & path) override; - void clearDirectory(const String & path) override; + void clearDirectory(const String & path, bool keep_s3 = false) override; void moveDirectory(const String & from_path, const String & to_path) override { moveFile(from_path, to_path); } @@ -86,9 +86,9 @@ public: size_t estimated_size, size_t aio_threshold) override; - void remove(const String & path) override; + void remove(const String & path, bool keep_s3 = false) override; - void removeRecursive(const String & path) override; + void removeRecursive(const String & path, bool keep_s3 = false) override; void createHardLink(const String & src_path, const String & dst_path) override; @@ -108,6 +108,8 @@ public: void shutdown() override; + const String getUniqueId(const String & path) const override; + private: bool tryReserve(UInt64 bytes); diff --git a/src/Interpreters/InterserverIOHandler.h b/src/Interpreters/InterserverIOHandler.h index 6d62c9651ca..bcb0e8736f0 100644 --- a/src/Interpreters/InterserverIOHandler.h +++ b/src/Interpreters/InterserverIOHandler.h @@ -16,6 +16,12 @@ namespace Poco { namespace Net { class HTTPServerResponse; } } +namespace zkutil +{ + class ZooKeeper; + using ZooKeeperPtr = std::shared_ptr; +} + namespace DB { @@ -34,6 +40,8 @@ public: virtual void processQuery(const Poco::Net::HTMLForm & params, ReadBuffer & body, WriteBuffer & out, Poco::Net::HTTPServerResponse & response) = 0; virtual ~InterserverIOEndpoint() = default; + virtual void setZooKeeper(const zkutil::ZooKeeperPtr &zookeeper_, const String & zookeeper_path_, const String & replica_name_) = 0; + /// You need to stop the data transfer if blocker is activated. ActionBlocker blocker; std::shared_mutex rwlock; diff --git a/src/Storages/MergeTree/DataPartsExchange.cpp b/src/Storages/MergeTree/DataPartsExchange.cpp index f9fb157942a..d9a37a01585 100644 --- a/src/Storages/MergeTree/DataPartsExchange.cpp +++ b/src/Storages/MergeTree/DataPartsExchange.cpp @@ -3,11 +3,15 @@ #include #include #include +#include #include #include #include +#include #include #include +#include +#include #include #include #include @@ -34,6 +38,7 @@ namespace ErrorCodes extern const int INSECURE_PATH; extern const int CORRUPTED_DATA; extern const int LOGICAL_ERROR; + extern const int S3_ERROR; } namespace DataPartsExchange @@ -45,6 +50,7 @@ constexpr auto REPLICATION_PROTOCOL_VERSION_WITH_PARTS_SIZE = 1; constexpr auto REPLICATION_PROTOCOL_VERSION_WITH_PARTS_SIZE_AND_TTL_INFOS = 2; constexpr auto REPLICATION_PROTOCOL_VERSION_WITH_PARTS_TYPE = 3; constexpr auto REPLICATION_PROTOCOL_VERSION_WITH_PARTS_DEFAULT_COMPRESSION = 4; +constexpr auto REPLICATION_PROTOCOL_VERSION_WITH_PARTS_S3_COPY = 5; std::string getEndpointId(const std::string & node_id) @@ -85,7 +91,7 @@ void Service::processQuery(const Poco::Net::HTMLForm & params, ReadBuffer & /*bo } /// We pretend to work as older server version, to be sure that client will correctly process our version - response.addCookie({"server_protocol_version", toString(std::min(client_protocol_version, REPLICATION_PROTOCOL_VERSION_WITH_PARTS_DEFAULT_COMPRESSION))}); + response.addCookie({"server_protocol_version", toString(std::min(client_protocol_version, REPLICATION_PROTOCOL_VERSION_WITH_PARTS_S3_COPY))}); ++total_sends; SCOPE_EXIT({--total_sends;}); @@ -118,8 +124,30 @@ void Service::processQuery(const Poco::Net::HTMLForm & params, ReadBuffer & /*bo sendPartFromMemory(part, out); else { - bool send_default_compression_file = client_protocol_version >= REPLICATION_PROTOCOL_VERSION_WITH_PARTS_DEFAULT_COMPRESSION; - sendPartFromDisk(part, out, send_default_compression_file); + bool try_use_s3_copy = false; + + if (client_protocol_version >= REPLICATION_PROTOCOL_VERSION_WITH_PARTS_S3_COPY) + { /// if source and destination are in the same S3 storage we try to use S3 CopyObject request first + int send_s3_metadata = parse(params.get("send_s3_metadata", "0")); + if (send_s3_metadata == 1) + { + auto disk = part->volume->getDisk(); + if (disk->getType() == "s3") + { + try_use_s3_copy = true; + } + } + } + if (try_use_s3_copy) + { + response.addCookie({"send_s3_metadata", "1"}); + sendPartS3Metadata(part, out); + } + else + { + bool send_default_compression_file = client_protocol_version >= REPLICATION_PROTOCOL_VERSION_WITH_PARTS_DEFAULT_COMPRESSION; + sendPartFromDisk(part, out, send_default_compression_file); + } } } catch (const NetException &) @@ -199,6 +227,62 @@ void Service::sendPartFromDisk(const MergeTreeData::DataPartPtr & part, WriteBuf part->checksums.checkEqual(data_checksums, false); } +void Service::sendPartS3Metadata(const MergeTreeData::DataPartPtr & part, WriteBuffer & out) +{ + /// We'll take a list of files from the list of checksums. + MergeTreeData::DataPart::Checksums checksums = part->checksums; + /// Add files that are not in the checksum list. + auto file_names_without_checksums = part->getFileNamesWithoutChecksums(); + for (const auto & file_name : file_names_without_checksums) + checksums.files[file_name] = {}; + + auto disk = part->volume->getDisk(); + if (disk->getType() != "s3") + throw Exception("S3 disk is not S3 anymore", ErrorCodes::LOGICAL_ERROR); + + String id = disk->getUniqueId(part->getFullRelativePath() + "checksums.txt"); + + if (id.empty()) + throw Exception("Can't lock part on S3 storage", ErrorCodes::LOGICAL_ERROR); + + String zookeeper_node = zookeeper_path + "/zero_copy_s3/" + id + "/" + replica_name; + + LOG_TRACE(log, "Set zookeeper lock {}", id); + + zookeeper->createAncestors(zookeeper_node); + zookeeper->createIfNotExists(zookeeper_node, "lock"); + + writeBinary(checksums.files.size(), out); + for (const auto & it : checksums.files) + { + String file_name = it.first; + + String metadata_file = disk->getPath() + part->getFullRelativePath() + file_name; + + Poco::File metadata(metadata_file); + + if (!metadata.exists()) + throw Exception("S3 metadata '" + file_name + "' is not exists", ErrorCodes::LOGICAL_ERROR); + if (!metadata.isFile()) + throw Exception("S3 metadata '" + file_name + "' is not a file", ErrorCodes::LOGICAL_ERROR); + UInt64 file_size = metadata.getSize(); + + writeStringBinary(it.first, out); + writeBinary(file_size, out); + + auto file_in = createReadBufferFromFileBase(metadata_file, 0, 0, 0, DBMS_DEFAULT_BUFFER_SIZE); + HashingWriteBuffer hashing_out(out); + copyData(*file_in, hashing_out, blocker.getCounter()); + if (blocker.isCancelled()) + throw Exception("Transferring part to replica was cancelled", ErrorCodes::ABORTED); + + if (hashing_out.count() != file_size) + throw Exception("Unexpected size of file " + metadata_file, ErrorCodes::BAD_SIZE_OF_FILE_IN_DATA_PART); + + writePODBinary(hashing_out.getHash(), out); + } +} + MergeTreeData::DataPartPtr Service::findPart(const String & name) { /// It is important to include PreCommitted and Outdated parts here because remote replicas cannot reliably @@ -222,7 +306,8 @@ MergeTreeData::MutableDataPartPtr Fetcher::fetchPart( const String & password, const String & interserver_scheme, bool to_detached, - const String & tmp_prefix_) + const String & tmp_prefix_, + bool try_use_s3_copy) { if (blocker.isCancelled()) throw Exception("Fetching of part was cancelled", ErrorCodes::ABORTED); @@ -239,10 +324,29 @@ MergeTreeData::MutableDataPartPtr Fetcher::fetchPart( { {"endpoint", getEndpointId(replica_path)}, {"part", part_name}, - {"client_protocol_version", toString(REPLICATION_PROTOCOL_VERSION_WITH_PARTS_DEFAULT_COMPRESSION)}, + {"client_protocol_version", toString(REPLICATION_PROTOCOL_VERSION_WITH_PARTS_S3_COPY)}, {"compress", "false"} }); + ReservationPtr reservationS3; + + if (try_use_s3_copy) + { + /// TODO: Make a normal check for S3 Disk + reservationS3 = data.makeEmptyReservationOnLargestDisk(); + auto disk = reservationS3->getDisk(); + + if (disk->getType() != "s3") + { + try_use_s3_copy = false; + } + } + + if (try_use_s3_copy) + { + uri.addQueryParameter("send_s3_metadata", "1"); + } + Poco::Net::HTTPBasicCredentials creds{}; if (!user.empty()) { @@ -263,6 +367,40 @@ MergeTreeData::MutableDataPartPtr Fetcher::fetchPart( int server_protocol_version = parse(in.getResponseCookie("server_protocol_version", "0")); + int send_s3 = parse(in.getResponseCookie("send_s3_metadata", "0")); + + if (send_s3 == 1) + { + if (server_protocol_version < REPLICATION_PROTOCOL_VERSION_WITH_PARTS_S3_COPY) + throw Exception("Got 'send_s3_metadata' cookie with old protocol version", ErrorCodes::LOGICAL_ERROR); + if (!try_use_s3_copy) + throw Exception("Got 'send_s3_metadata' cookie when was not requested", ErrorCodes::LOGICAL_ERROR); + + size_t sum_files_size = 0; + readBinary(sum_files_size, in); + IMergeTreeDataPart::TTLInfos ttl_infos; + /// Skip ttl infos, not required for S3 metadata + String ttl_infos_string; + readBinary(ttl_infos_string, in); + String part_type = "Wide"; + readStringBinary(part_type, in); + if (part_type == "InMemory") + throw Exception("Got 'send_s3_metadata' cookie for in-memory partition", ErrorCodes::LOGICAL_ERROR); + + try + { + return downloadPartToS3(part_name, replica_path, to_detached, tmp_prefix_, sync, std::move(reservationS3), in); + } + catch(const Exception& e) + { + if (e.code() != ErrorCodes::S3_ERROR) + throw; + /// Try again but without S3 copy + return fetchPart(metadata_snapshot, part_name, replica_path, host, port, timeouts, + user, password, interserver_scheme, to_detached, tmp_prefix_, false); + } + } + ReservationPtr reservation; size_t sum_files_size = 0; if (server_protocol_version >= REPLICATION_PROTOCOL_VERSION_WITH_PARTS_SIZE) @@ -418,6 +556,96 @@ MergeTreeData::MutableDataPartPtr Fetcher::downloadPartToDisk( return new_data_part; } +MergeTreeData::MutableDataPartPtr Fetcher::downloadPartToS3( + const String & part_name, + const String & replica_path, + bool to_detached, + const String & tmp_prefix_, + bool ,//sync, + const ReservationPtr reservation, + PooledReadWriteBufferFromHTTP & in + ) +{ + auto disk = reservation->getDisk(); + if (disk->getType() != "s3") + throw Exception("S3 disk is not S3 anymore", ErrorCodes::LOGICAL_ERROR); + + static const String TMP_PREFIX = "tmp_fetch_"; + String tmp_prefix = tmp_prefix_.empty() ? TMP_PREFIX : tmp_prefix_; + + String part_relative_path = String(to_detached ? "detached/" : "") + tmp_prefix + part_name; + String part_download_path = data.getRelativeDataPath() + part_relative_path + "/"; + + if (disk->exists(part_download_path)) + throw Exception("Directory " + fullPath(disk, part_download_path) + " already exists.", ErrorCodes::DIRECTORY_ALREADY_EXISTS); + + CurrentMetrics::Increment metric_increment{CurrentMetrics::ReplicatedFetch}; + + disk->createDirectories(part_download_path); + + size_t files; + readBinary(files, in); + + auto volume = std::make_shared("volume_" + part_name, disk); + MergeTreeData::MutableDataPartPtr new_data_part = data.createPart(part_name, volume, part_relative_path); + + for (size_t i = 0; i < files; ++i) + { + String file_name; + UInt64 file_size; + + readStringBinary(file_name, in); + readBinary(file_size, in); + + String metadata_file = disk->getPath() + new_data_part->getFullRelativePath() + file_name; + + auto file_out = createWriteBufferFromFileBase(metadata_file, 0, 0, DBMS_DEFAULT_BUFFER_SIZE, -1); + + HashingWriteBuffer hashing_out(*file_out); + + copyData(in, hashing_out, file_size, blocker.getCounter()); + + if (blocker.isCancelled()) + { + /// NOTE The is_cancelled flag also makes sense to check every time you read over the network, + /// performing a poll with a not very large timeout. + /// And now we check it only between read chunks (in the `copyData` function). + throw Exception("Fetching of part was cancelled", ErrorCodes::ABORTED); + } + + MergeTreeDataPartChecksum::uint128 expected_hash; + readPODBinary(expected_hash, in); + + if (expected_hash != hashing_out.getHash()) + { + throw Exception("Checksum mismatch for file " + metadata_file + " transferred from " + replica_path, + ErrorCodes::CHECKSUM_DOESNT_MATCH); + } + } + + assertEOF(in); + + new_data_part->is_temp = true; + new_data_part->modification_time = time(nullptr); + new_data_part->loadColumnsChecksumsIndexes(true, false); + + + String id = disk->getUniqueId(new_data_part->getFullRelativePath() + "checksums.txt"); + + if (id.empty()) + throw Exception("Can't lock part on S3 storage", ErrorCodes::LOGICAL_ERROR); + + String zookeeper_node = zookeeper_path + "/zero_copy_s3/" + id + "/" + replica_name; + + LOG_TRACE(log, "Set zookeeper lock {}", id); + + zookeeper->createAncestors(zookeeper_node); + zookeeper->createIfNotExists(zookeeper_node, "lock"); + + + return new_data_part; +} + } } diff --git a/src/Storages/MergeTree/DataPartsExchange.h b/src/Storages/MergeTree/DataPartsExchange.h index 52a34a2239a..e2e7b2adf4f 100644 --- a/src/Storages/MergeTree/DataPartsExchange.h +++ b/src/Storages/MergeTree/DataPartsExchange.h @@ -9,6 +9,12 @@ #include +namespace zkutil +{ + class ZooKeeper; + using ZooKeeperPtr = std::shared_ptr; +} + namespace DB { @@ -29,16 +35,27 @@ public: std::string getId(const std::string & node_id) const override; void processQuery(const Poco::Net::HTMLForm & params, ReadBuffer & body, WriteBuffer & out, Poco::Net::HTTPServerResponse & response) override; + void setZooKeeper(const zkutil::ZooKeeperPtr & zookeeper_, const String & zookeeper_path_, const String & replica_name_) override + { + zookeeper = zookeeper_; + zookeeper_path = zookeeper_path_; + replica_name = replica_name_; + } + private: MergeTreeData::DataPartPtr findPart(const String & name); void sendPartFromMemory(const MergeTreeData::DataPartPtr & part, WriteBuffer & out); void sendPartFromDisk(const MergeTreeData::DataPartPtr & part, WriteBuffer & out, bool send_default_compression_file); + void sendPartS3Metadata(const MergeTreeData::DataPartPtr & part, WriteBuffer & out); private: /// StorageReplicatedMergeTree::shutdown() waits for all parts exchange handlers to finish, /// so Service will never access dangling reference to storage MergeTreeData & data; Poco::Logger * log; + zkutil::ZooKeeperPtr zookeeper; + String zookeeper_path; + String replica_name; }; /** Client for getting the parts from the table *MergeTree. @@ -63,11 +80,19 @@ public: const String & password, const String & interserver_scheme, bool to_detached = false, - const String & tmp_prefix_ = ""); + const String & tmp_prefix_ = "", + bool try_use_s3_copy = true); /// You need to stop the data transfer. ActionBlocker blocker; + void setZooKeeper(const zkutil::ZooKeeperPtr & zookeeper_, const String & zookeeper_path_, const String & replica_name_) + { + zookeeper = zookeeper_; + zookeeper_path = zookeeper_path_; + replica_name = replica_name_; + } + private: MergeTreeData::MutableDataPartPtr downloadPartToDisk( const String & part_name, @@ -84,8 +109,20 @@ private: ReservationPtr reservation, PooledReadWriteBufferFromHTTP & in); + MergeTreeData::MutableDataPartPtr downloadPartToS3( + const String & part_name, + const String & replica_path, + bool to_detached, + const String & tmp_prefix_, + bool sync, + const ReservationPtr reservation, + PooledReadWriteBufferFromHTTP & in); + MergeTreeData & data; Poco::Logger * log; + zkutil::ZooKeeperPtr zookeeper; + String zookeeper_path; + String replica_name; }; } diff --git a/src/Storages/MergeTree/IMergeTreeDataPart.cpp b/src/Storages/MergeTree/IMergeTreeDataPart.cpp index 486e444763d..23fe60b44e5 100644 --- a/src/Storages/MergeTree/IMergeTreeDataPart.cpp +++ b/src/Storages/MergeTree/IMergeTreeDataPart.cpp @@ -262,7 +262,7 @@ void IMergeTreeDataPart::removeIfNeeded() } } - remove(); + remove(false); if (state == State::DeleteOnDestroy) { @@ -809,7 +809,7 @@ void IMergeTreeDataPart::renameTo(const String & new_relative_path, bool remove_ } -void IMergeTreeDataPart::remove() const +void IMergeTreeDataPart::remove(bool keep_s3) const { if (!isStoredOnDisk()) return; @@ -839,7 +839,7 @@ void IMergeTreeDataPart::remove() const try { - volume->getDisk()->removeRecursive(to + "/"); + volume->getDisk()->removeRecursive(to + "/", keep_s3); } catch (...) { @@ -862,7 +862,7 @@ void IMergeTreeDataPart::remove() const if (checksums.empty()) { /// If the part is not completely written, we cannot use fast path by listing files. - volume->getDisk()->removeRecursive(to + "/"); + volume->getDisk()->removeRecursive(to + "/", keep_s3); } else { @@ -875,18 +875,18 @@ void IMergeTreeDataPart::remove() const # pragma GCC diagnostic ignored "-Wunused-variable" #endif for (const auto & [file, _] : checksums.files) - volume->getDisk()->remove(to + "/" + file); + volume->getDisk()->remove(to + "/" + file, keep_s3); #if !__clang__ # pragma GCC diagnostic pop #endif for (const auto & file : {"checksums.txt", "columns.txt"}) - volume->getDisk()->remove(to + "/" + file); + volume->getDisk()->remove(to + "/" + file, keep_s3); - volume->getDisk()->removeIfExists(to + "/" + DEFAULT_COMPRESSION_CODEC_FILE_NAME); - volume->getDisk()->removeIfExists(to + "/" + DELETE_ON_DESTROY_MARKER_FILE_NAME); + volume->getDisk()->removeIfExists(to + "/" + DEFAULT_COMPRESSION_CODEC_FILE_NAME, keep_s3); + volume->getDisk()->removeIfExists(to + "/" + DELETE_ON_DESTROY_MARKER_FILE_NAME, keep_s3); - volume->getDisk()->remove(to); + volume->getDisk()->remove(to, keep_s3); } catch (...) { @@ -894,7 +894,7 @@ void IMergeTreeDataPart::remove() const LOG_ERROR(storage.log, "Cannot quickly remove directory {} by removing files; fallback to recursive removal. Reason: {}", fullPath(volume->getDisk(), to), getCurrentExceptionMessage(false)); - volume->getDisk()->removeRecursive(to + "/"); + volume->getDisk()->removeRecursive(to + "/", keep_s3); } } } diff --git a/src/Storages/MergeTree/IMergeTreeDataPart.h b/src/Storages/MergeTree/IMergeTreeDataPart.h index 78daf6c9017..3e7b03b2903 100644 --- a/src/Storages/MergeTree/IMergeTreeDataPart.h +++ b/src/Storages/MergeTree/IMergeTreeDataPart.h @@ -124,7 +124,7 @@ public: /// Throws an exception if part is not stored in on-disk format. void assertOnDisk() const; - void remove() const; + void remove(bool keep_s3 = false) const; /// Initialize columns (from columns.txt if exists, or create from column files if not). /// Load checksums from checksums.txt if exists. Load index if required. diff --git a/src/Storages/StorageReplicatedMergeTree.cpp b/src/Storages/StorageReplicatedMergeTree.cpp index 9613bd5111d..dbbf8645d36 100644 --- a/src/Storages/StorageReplicatedMergeTree.cpp +++ b/src/Storages/StorageReplicatedMergeTree.cpp @@ -370,6 +370,8 @@ StorageReplicatedMergeTree::StorageReplicatedMergeTree( } createNewZooKeeperNodes(); + + fetcher.setZooKeeper(current_zookeeper, zookeeper_path, replica_name); } @@ -3364,6 +3366,7 @@ void StorageReplicatedMergeTree::startup() queue.initialize(getDataParts()); data_parts_exchange_endpoint = std::make_shared(*this); + data_parts_exchange_endpoint->setZooKeeper(tryGetZooKeeper(), zookeeper_path, replica_name); global_context.getInterserverIOHandler().addEndpoint(data_parts_exchange_endpoint->getId(replica_path), data_parts_exchange_endpoint); /// In this thread replica will be activated. @@ -5010,13 +5013,40 @@ void StorageReplicatedMergeTree::clearOldPartsAndRemoveFromZK() } parts.clear(); - auto remove_parts_from_filesystem = [log=log] (const DataPartsVector & parts_to_remove) + auto remove_parts_from_filesystem = [log=log,&zookeeper=zookeeper,&zookeeper_path=zookeeper_path,&replica_name=replica_name] (const DataPartsVector & parts_to_remove) { for (const auto & part : parts_to_remove) { try { - part->remove(); + bool keep_s3 = false; + + auto disk = part->volume->getDisk(); + + if (disk->getType() == "s3") + { + String id = disk->getUniqueId(part->getFullRelativePath() + "checksums.txt"); + + if (!id.empty()) + { + String zookeeper_part_node = zookeeper_path + "/zero_copy_s3/" + id; + String zookeeper_node = zookeeper_part_node + "/" + replica_name; + + LOG_TRACE(log, "Remove zookeeper lock for {}", id); + + zookeeper->remove(zookeeper_node); + + Strings children; + zookeeper->tryGetChildren(zookeeper_part_node, children); + if (!children.empty()) + { + LOG_TRACE(log, "Found zookeper locks for {}", id); + keep_s3 = true; + } + } + } + + part->remove(keep_s3); } catch (...) { From a4adb39b2576eb26accc8336a27d6ab9eca4e1b4 Mon Sep 17 00:00:00 2001 From: Anton Ivashkin Date: Thu, 8 Oct 2020 19:23:04 +0300 Subject: [PATCH 002/544] S3 zero copy replication proof of concept - description --- S3ZeroCopyReplication.md | 47 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 47 insertions(+) create mode 100644 S3ZeroCopyReplication.md diff --git a/S3ZeroCopyReplication.md b/S3ZeroCopyReplication.md new file mode 100644 index 00000000000..7e7709ff5a7 --- /dev/null +++ b/S3ZeroCopyReplication.md @@ -0,0 +1,47 @@ +# ClickHouse S3 Zero Copy Replication + +Говнокод просто для теста, не production-ready ни разу. + +[Коммит](https://github.com/ianton-ru/ClickHouse/commit/acf86568a7e21176ba2cca15861da231bec6932a) + +[Ветка](https://github.com/ianton-ru/ClickHouse/tree/s3_zero_copy_replication) + +## Как сделано + +При fetch-е парта при репликации в случае, если источник хранит, а приемник собирается хранить парт в S3, вместо данных пересылаются только метаданные S3, приемник кладет их локально себе +и испольузет общие с источником данные на S3. Для того, чтобы не удалить такие пошареные данные, делается пометка в ZooKeeper. + +Введена новая версия протокола REPLICATION_PROTOCOL_VERSION_WITH_PARTS_S3_COPY. В запросе новый параметр send_s3_metadata, если 1, то приемних просит у источника метаданные вместо данных, если это возможно. +Приемник в ответ отсылает куку send_s3_metadata=1 в случае, если идут метаданные. В остальных случаях отсылаются данные, как и прежде. + +Применик перед запросом смотрит, будет ли хранить данные в S3. Провеока сейчас кривая - запрашивается резервирование на диске с наибольшим доступным местом, а потом смотрится, не на S3 ли оно. +Если на S3, то отсылает в запросе send_s3_metadata=1. + +Источник при получении такого запроса смотрит, лежит ли парт на S3. Если да, то в Зукипере ставит метку по пути `<путь к данным таблицы>/zero_copy_s3/<некий ID парта>/`, +ставит в ответ куку send_s3_metadata=1 и вместо файлов с данными отсылает только файлы метаданных. + +Приемник при получении ответа с send_s3_metadata=1 создает только файлики с идентичными меаданными, которые в итоге будут ссылаться на те же ключи в S3, ставит в зукипере аналогичную метку, +только со своим ID реплики, и работает с этим. + +При желании удалить парт нода удаляет в Зукипере ключ `<путь к данным таблицы>/zero_copy_s3/<некий ID парта>/`, потом получает все подключи `<путь к данным таблицы>/zero_copy_s3/<некий ID парта>`. +Если список не пустой, то считает, что данные использует другая нода и удаляет только локальные метаданные, если пустой, то удаляет и данные в S3. + +## Костыли и недоработки, коих много + +* Никакой проверки, один и тот же S3 у нод или разный сейчас нет, если будет несколько разных S3, работать не будет. + +* В качестве ID парта берется имя первого S3-ключа от файла checksums.txt. + +* Не нашел удобного способа прокидывать в коде зукипер, прокинул хадркодом. + +* При удалении класс диска ничего не знает про парты, прокинул флаг, что надо оставлять данные в S3 параметром, это очень криво получилось. + +* Возможна гонка, если источник отошлет метаданные про парт и тут же решит его удалить до того, как приемник поставит в зукипер пометку. + +* В протоколе репликации обмен инфой через параметр запрос в одну сторону и куку в другую мне не нравится, хотя так сделан обмен версиями репликации. + +* При ошибке должно пытаться реплицироваться по старому, но хз, всегда ли сработает + +* Не будет обратной совместимости, если образуются такие шареные парты, откатиться на старую версию кликхауса не получится, иначе нода может удалить используемые другой данные. + +* И вообще From 9272ed06b427f017d1b95e0d20ff6132f5cb06a2 Mon Sep 17 00:00:00 2001 From: Anton Ivashkin Date: Fri, 9 Oct 2020 17:24:10 +0300 Subject: [PATCH 003/544] Move Zookeeper lock for S3 shared part in IMergeTreeDataPart --- src/Storages/MergeTree/DataPartsExchange.cpp | 27 +--------- src/Storages/MergeTree/IMergeTreeDataPart.cpp | 52 +++++++++++++++++++ src/Storages/MergeTree/IMergeTreeDataPart.h | 14 +++++ src/Storages/StorageReplicatedMergeTree.cpp | 28 +--------- 4 files changed, 69 insertions(+), 52 deletions(-) diff --git a/src/Storages/MergeTree/DataPartsExchange.cpp b/src/Storages/MergeTree/DataPartsExchange.cpp index 2708373d1a4..da5acdbefcd 100644 --- a/src/Storages/MergeTree/DataPartsExchange.cpp +++ b/src/Storages/MergeTree/DataPartsExchange.cpp @@ -7,7 +7,6 @@ #include #include #include -#include #include #include #include @@ -240,17 +239,7 @@ void Service::sendPartS3Metadata(const MergeTreeData::DataPartPtr & part, WriteB if (disk->getType() != "s3") throw Exception("S3 disk is not S3 anymore", ErrorCodes::LOGICAL_ERROR); - String id = disk->getUniqueId(part->getFullRelativePath() + "checksums.txt"); - - if (id.empty()) - throw Exception("Can't lock part on S3 storage", ErrorCodes::LOGICAL_ERROR); - - String zookeeper_node = zookeeper_path + "/zero_copy_s3/" + id + "/" + replica_name; - - LOG_TRACE(log, "Set zookeeper lock {}", id); - - zookeeper->createAncestors(zookeeper_node); - zookeeper->createIfNotExists(zookeeper_node, "lock"); + part->lockSharedData(zookeeper_path, replica_name, zookeeper); writeBinary(checksums.files.size(), out); for (const auto & it : checksums.files) @@ -629,19 +618,7 @@ MergeTreeData::MutableDataPartPtr Fetcher::downloadPartToS3( new_data_part->modification_time = time(nullptr); new_data_part->loadColumnsChecksumsIndexes(true, false); - - String id = disk->getUniqueId(new_data_part->getFullRelativePath() + "checksums.txt"); - - if (id.empty()) - throw Exception("Can't lock part on S3 storage", ErrorCodes::LOGICAL_ERROR); - - String zookeeper_node = zookeeper_path + "/zero_copy_s3/" + id + "/" + replica_name; - - LOG_TRACE(log, "Set zookeeper lock {}", id); - - zookeeper->createAncestors(zookeeper_node); - zookeeper->createIfNotExists(zookeeper_node, "lock"); - + new_data_part->lockSharedData(zookeeper_path, replica_name, zookeeper); return new_data_part; } diff --git a/src/Storages/MergeTree/IMergeTreeDataPart.cpp b/src/Storages/MergeTree/IMergeTreeDataPart.cpp index 40a6569cd46..786bc056702 100644 --- a/src/Storages/MergeTree/IMergeTreeDataPart.cpp +++ b/src/Storages/MergeTree/IMergeTreeDataPart.cpp @@ -12,6 +12,7 @@ #include #include #include +#include #include #include #include @@ -1079,6 +1080,56 @@ bool IMergeTreeDataPart::checkAllTTLCalculated(const StorageMetadataPtr & metada return true; } +void IMergeTreeDataPart::lockSharedData(const String & zookeeper_path, const String & replica_name, zkutil::ZooKeeperPtr zookeeper) const +{ + auto disk = volume->getDisk(); + + if (disk->getType() != "s3") + return; + + String id = disk->getUniqueId(getFullRelativePath() + "checksums.txt"); + + if (id.empty()) + throw Exception("Can't lock part on S3 storage", ErrorCodes::LOGICAL_ERROR); + + String zookeeper_node = zookeeper_path + "/zero_copy_s3/" + id + "/" + replica_name; + + LOG_TRACE(storage.log, "Set zookeeper lock {}", id); + + zookeeper->createAncestors(zookeeper_node); + zookeeper->createIfNotExists(zookeeper_node, "lock"); +} + +bool IMergeTreeDataPart::unlockSharedData(const String & zookeeper_path, const String & replica_name, zkutil::ZooKeeperPtr zookeeper) const +{ + auto disk = volume->getDisk(); + + if (disk->getType() != "s3") + return true; + + String id = disk->getUniqueId(getFullRelativePath() + "checksums.txt"); + + if (id.empty()) + return true; + + String zookeeper_part_node = zookeeper_path + "/zero_copy_s3/" + id; + String zookeeper_node = zookeeper_part_node + "/" + replica_name; + + LOG_TRACE(storage.log, "Remove zookeeper lock for {}", id); + + zookeeper->remove(zookeeper_node); + + Strings children; + zookeeper->tryGetChildren(zookeeper_part_node, children); + + if (!children.empty()) + { + LOG_TRACE(storage.log, "Found zookeper locks for {}", id); + } + + return children.empty(); +} + bool isCompactPart(const MergeTreeDataPartPtr & data_part) { return (data_part && data_part->getType() == MergeTreeDataPartType::COMPACT); @@ -1095,3 +1146,4 @@ bool isInMemoryPart(const MergeTreeDataPartPtr & data_part) } } + diff --git a/src/Storages/MergeTree/IMergeTreeDataPart.h b/src/Storages/MergeTree/IMergeTreeDataPart.h index 3e7b03b2903..d40ff40f157 100644 --- a/src/Storages/MergeTree/IMergeTreeDataPart.h +++ b/src/Storages/MergeTree/IMergeTreeDataPart.h @@ -22,6 +22,12 @@ #include +namespace zkutil +{ + class ZooKeeper; + using ZooKeeperPtr = std::shared_ptr; +} + namespace DB { @@ -349,6 +355,14 @@ public: /// part creation (using alter query with materialize_ttl setting). bool checkAllTTLCalculated(const StorageMetadataPtr & metadata_snapshot) const; + /// Lock part in zookeeper for use common S3 data in several nodes + void lockSharedData(const String & zookeeper_path, const String & replica_name, zkutil::ZooKeeperPtr zookeeper) const; + + /// Unlock common S3 data part in zookeeper + /// Return true if data unlocked + /// Return false if data is still used by another node + bool unlockSharedData(const String & zookeeper_path, const String & replica_name, zkutil::ZooKeeperPtr zookeeper) const; + protected: /// Total size of all columns, calculated once in calcuateColumnSizesOnDisk diff --git a/src/Storages/StorageReplicatedMergeTree.cpp b/src/Storages/StorageReplicatedMergeTree.cpp index b1c7c754637..6355894d59e 100644 --- a/src/Storages/StorageReplicatedMergeTree.cpp +++ b/src/Storages/StorageReplicatedMergeTree.cpp @@ -5111,33 +5111,7 @@ void StorageReplicatedMergeTree::clearOldPartsAndRemoveFromZK() { try { - bool keep_s3 = false; - - auto disk = part->volume->getDisk(); - - if (disk->getType() == "s3") - { - String id = disk->getUniqueId(part->getFullRelativePath() + "checksums.txt"); - - if (!id.empty()) - { - String zookeeper_part_node = zookeeper_path + "/zero_copy_s3/" + id; - String zookeeper_node = zookeeper_part_node + "/" + replica_name; - - LOG_TRACE(log, "Remove zookeeper lock for {}", id); - - zookeeper->remove(zookeeper_node); - - Strings children; - zookeeper->tryGetChildren(zookeeper_part_node, children); - if (!children.empty()) - { - LOG_TRACE(log, "Found zookeper locks for {}", id); - keep_s3 = true; - } - } - } - + bool keep_s3 = !part->unlockSharedData(zookeeper_path, replica_name, zookeeper); part->remove(keep_s3); } catch (...) From b877459cf78fbff327c3b75481220e39ea8ee9a6 Mon Sep 17 00:00:00 2001 From: Anton Ivashkin Date: Wed, 14 Oct 2020 18:05:59 +0300 Subject: [PATCH 004/544] Zero copy replication over S3: check s3 storage --- src/Disks/DiskDecorator.h | 1 + src/Disks/IDisk.h | 3 + src/Disks/S3/DiskS3.cpp | 38 +++++++ src/Disks/S3/DiskS3.h | 2 + src/Disks/StoragePolicy.cpp | 11 ++ src/Disks/StoragePolicy.h | 3 + src/Storages/MergeTree/DataPartsExchange.cpp | 106 +++++++++++++------ src/Storages/MergeTree/DataPartsExchange.h | 2 +- src/Storages/MergeTree/MergeTreeData.h | 2 + 9 files changed, 136 insertions(+), 32 deletions(-) diff --git a/src/Disks/DiskDecorator.h b/src/Disks/DiskDecorator.h index f1fea043843..86d842ce2cf 100644 --- a/src/Disks/DiskDecorator.h +++ b/src/Disks/DiskDecorator.h @@ -47,6 +47,7 @@ public: void sync(int fd) const override; const String getType() const override { return delegate->getType(); } const String getUniqueId(const String & path) const override { return delegate->getUniqueId(path); } + bool checkFile(const String & path) const override { return delegate->checkFile(path); } protected: DiskPtr delegate; diff --git a/src/Disks/IDisk.h b/src/Disks/IDisk.h index 324384fade6..96a2e5e4669 100644 --- a/src/Disks/IDisk.h +++ b/src/Disks/IDisk.h @@ -198,6 +198,9 @@ public: /// Return some uniq string for file, overrided for S3 virtual const String getUniqueId(const String & path) const { return path; } + /// Check file, overrided for S3 only + virtual bool checkFile(const String & path) const { return exists(path); } + private: /// Returns executor to perform asynchronous operations. Executor & getExecutor() { return *executor; } diff --git a/src/Disks/S3/DiskS3.cpp b/src/Disks/S3/DiskS3.cpp index 9cb3178350c..b563c84094a 100644 --- a/src/Disks/S3/DiskS3.cpp +++ b/src/Disks/S3/DiskS3.cpp @@ -20,6 +20,7 @@ #include #include #include +#include #include @@ -83,6 +84,16 @@ namespace } } + template + void throwIfError(const Aws::Utils::Outcome & response) + { + if (!response.IsSuccess()) + { + const auto & err = response.GetError(); + throw Exception(err.GetMessage(), static_cast(err.GetErrorType())); + } + } + /** * S3 metadata file layout: * Number of S3 objects, Total size of all S3 objects. @@ -835,4 +846,31 @@ void DiskS3::shutdown() client->DisableRequestProcessing(); } +bool DiskS3::checkFile(const String & path) const +{ + Metadata metadata(s3_root_path, metadata_path, path); + + /// empty s3_objects list for empty file + if (metadata.s3_objects.empty()) + return true; + + String object = metadata.s3_root_path + metadata.s3_objects[0].first; + + Aws::S3::Model::ListObjectsRequest request; + request.SetBucket(bucket); + request.SetPrefix(object); + auto resp = client->ListObjects(request); + throwIfError(resp); + Aws::Vector object_list = resp.GetResult().GetContents(); + + /// Should be only one object with name equal to prefix + if (object_list.size() != 1) + return false; + + if (object_list[0].GetKey() != object) + return false; + return true; +} + + } diff --git a/src/Disks/S3/DiskS3.h b/src/Disks/S3/DiskS3.h index 7808f5a8007..07348c53417 100644 --- a/src/Disks/S3/DiskS3.h +++ b/src/Disks/S3/DiskS3.h @@ -112,6 +112,8 @@ public: const String getUniqueId(const String & path) const override; + bool checkFile(const String & path) const override; + private: bool tryReserve(UInt64 bytes); diff --git a/src/Disks/StoragePolicy.cpp b/src/Disks/StoragePolicy.cpp index 1aa20301bc0..746438bc72c 100644 --- a/src/Disks/StoragePolicy.cpp +++ b/src/Disks/StoragePolicy.cpp @@ -123,6 +123,17 @@ Disks StoragePolicy::getDisks() const } +Disks StoragePolicy::getDisksByType(const String & type) const +{ + Disks res; + for (const auto & volume : volumes) + for (const auto & disk : volume->getDisks()) + if (disk->getType() == type) + res.push_back(disk); + return res; +} + + DiskPtr StoragePolicy::getAnyDisk() const { /// StoragePolicy must contain at least one Volume diff --git a/src/Disks/StoragePolicy.h b/src/Disks/StoragePolicy.h index 0e0795d8bf1..b42886afcb2 100644 --- a/src/Disks/StoragePolicy.h +++ b/src/Disks/StoragePolicy.h @@ -41,6 +41,9 @@ public: /// Returns disks ordered by volumes priority Disks getDisks() const; + /// Returns disks by type ordered by volumes priority + Disks getDisksByType(const String & type) const; + /// Returns any disk /// Used when it's not important, for example for /// mutations files diff --git a/src/Storages/MergeTree/DataPartsExchange.cpp b/src/Storages/MergeTree/DataPartsExchange.cpp index da5acdbefcd..678acc2d848 100644 --- a/src/Storages/MergeTree/DataPartsExchange.cpp +++ b/src/Storages/MergeTree/DataPartsExchange.cpp @@ -317,18 +317,13 @@ MergeTreeData::MutableDataPartPtr Fetcher::fetchPart( {"compress", "false"} }); - ReservationPtr reservationS3; + Disks disksS3; if (try_use_s3_copy) { - /// TODO: Make a normal check for S3 Disk - reservationS3 = data.makeEmptyReservationOnLargestDisk(); - auto disk = reservationS3->getDisk(); - - if (disk->getType() != "s3") - { + disksS3 = data.getDisksByType("s3"); + if (disksS3.empty()) try_use_s3_copy = false; - } } if (try_use_s3_copy) @@ -378,7 +373,7 @@ MergeTreeData::MutableDataPartPtr Fetcher::fetchPart( try { - return downloadPartToS3(part_name, replica_path, to_detached, tmp_prefix_, sync, std::move(reservationS3), in); + return downloadPartToS3(part_name, replica_path, to_detached, tmp_prefix_, sync, std::move(disksS3), in); } catch(const Exception& e) { @@ -551,13 +546,14 @@ MergeTreeData::MutableDataPartPtr Fetcher::downloadPartToS3( bool to_detached, const String & tmp_prefix_, bool ,//sync, - const ReservationPtr reservation, + const Disks & disksS3, PooledReadWriteBufferFromHTTP & in ) { - auto disk = reservation->getDisk(); - if (disk->getType() != "s3") - throw Exception("S3 disk is not S3 anymore", ErrorCodes::LOGICAL_ERROR); + if (disksS3.empty()) + throw Exception("No S3 disks anymore", ErrorCodes::LOGICAL_ERROR); + + auto disk = disksS3[0]; static const String TMP_PREFIX = "tmp_fetch_"; String tmp_prefix = tmp_prefix_.empty() ? TMP_PREFIX : tmp_prefix_; @@ -586,29 +582,77 @@ MergeTreeData::MutableDataPartPtr Fetcher::downloadPartToS3( readStringBinary(file_name, in); readBinary(file_size, in); - String metadata_file = disk->getPath() + new_data_part->getFullRelativePath() + file_name; + String data_path = new_data_part->getFullRelativePath() + file_name; + String metadata_file = fullPath(disk, data_path); - auto file_out = createWriteBufferFromFileBase(metadata_file, 0, 0, DBMS_DEFAULT_BUFFER_SIZE, -1); - - HashingWriteBuffer hashing_out(*file_out); - - copyData(in, hashing_out, file_size, blocker.getCounter()); - - if (blocker.isCancelled()) { - /// NOTE The is_cancelled flag also makes sense to check every time you read over the network, - /// performing a poll with a not very large timeout. - /// And now we check it only between read chunks (in the `copyData` function). - throw Exception("Fetching of part was cancelled", ErrorCodes::ABORTED); + auto file_out = createWriteBufferFromFileBase(metadata_file, 0, 0, DBMS_DEFAULT_BUFFER_SIZE, -1); + + HashingWriteBuffer hashing_out(*file_out); + + copyData(in, hashing_out, file_size, blocker.getCounter()); + + if (blocker.isCancelled()) + { + /// NOTE The is_cancelled flag also makes sense to check every time you read over the network, + /// performing a poll with a not very large timeout. + /// And now we check it only between read chunks (in the `copyData` function). + disk->removeRecursive(part_download_path, true); + throw Exception("Fetching of part was cancelled", ErrorCodes::ABORTED); + } + + MergeTreeDataPartChecksum::uint128 expected_hash; + readPODBinary(expected_hash, in); + + if (expected_hash != hashing_out.getHash()) + { + throw Exception("Checksum mismatch for file " + metadata_file + " transferred from " + replica_path, + ErrorCodes::CHECKSUM_DOESNT_MATCH); + } } - MergeTreeDataPartChecksum::uint128 expected_hash; - readPODBinary(expected_hash, in); + if (!i) + { /// Check access for first s3 object of first file + if (!disk->checkFile(data_path)) + { /// Wrong S3 disk + Poco::File metadata(metadata_file); - if (expected_hash != hashing_out.getHash()) - { - throw Exception("Checksum mismatch for file " + metadata_file + " transferred from " + replica_path, - ErrorCodes::CHECKSUM_DOESNT_MATCH); + size_t disk_id = 1; + while (true) + { + if (disk_id >= disksS3.size()) + { /// No more S3 disks + disk->removeRecursive(part_download_path, true); + /// After catch this exception replication continues with full data copy + throw Exception("Can't find S3 drive for shared data", ErrorCodes::S3_ERROR); + } + + /// Try next S3 disk + auto next_disk = disksS3[disk_id]; + + auto next_volume = std::make_shared("volume_" + part_name, next_disk); + MergeTreeData::MutableDataPartPtr next_new_data_part = data.createPart(part_name, next_volume, part_relative_path); + + next_disk->createDirectories(part_download_path); + + String next_data_path = next_new_data_part->getFullRelativePath() + file_name; + String next_metadata_file = fullPath(next_disk, next_data_path); + metadata.copyTo(next_metadata_file); + if (next_disk->checkFile(next_data_path)) + { /// Right disk found + disk->removeRecursive(part_download_path, true); + disk = next_disk; + volume = next_volume; + data_path = next_data_path; + new_data_part = next_new_data_part; + break; + } + + /// Wrong disk again + next_disk->removeRecursive(part_download_path, true); + ++disk_id; + } + } } } diff --git a/src/Storages/MergeTree/DataPartsExchange.h b/src/Storages/MergeTree/DataPartsExchange.h index e2e7b2adf4f..7e59e81d6dc 100644 --- a/src/Storages/MergeTree/DataPartsExchange.h +++ b/src/Storages/MergeTree/DataPartsExchange.h @@ -115,7 +115,7 @@ private: bool to_detached, const String & tmp_prefix_, bool sync, - const ReservationPtr reservation, + const Disks & disksS3, PooledReadWriteBufferFromHTTP & in); MergeTreeData & data; diff --git a/src/Storages/MergeTree/MergeTreeData.h b/src/Storages/MergeTree/MergeTreeData.h index 5c18661dad1..1b620b3bdae 100644 --- a/src/Storages/MergeTree/MergeTreeData.h +++ b/src/Storages/MergeTree/MergeTreeData.h @@ -651,6 +651,8 @@ public: /// Reserves 0 bytes ReservationPtr makeEmptyReservationOnLargestDisk() { return getStoragePolicy()->makeEmptyReservationOnLargestDisk(); } + Disks getDisksByType(const String & type) const { return getStoragePolicy()->getDisksByType(type); } + /// Return alter conversions for part which must be applied on fly. AlterConversions getAlterConversionsForPart(const MergeTreeDataPartPtr part) const; /// Returns destination disk or volume for the TTL rule according to current storage policy From 14a78f87b03141721ad5978793d22c3d8fc36baa Mon Sep 17 00:00:00 2001 From: Anton Ivashkin Date: Thu, 15 Oct 2020 18:23:20 +0300 Subject: [PATCH 005/544] Zero copy replication over S3: fetch instead of merge --- S3ZeroCopyReplication.md | 33 ++++++++++++------- src/Storages/MergeTree/IMergeTreeDataPart.cpp | 4 +-- src/Storages/StorageReplicatedMergeTree.cpp | 14 ++++++++ 3 files changed, 38 insertions(+), 13 deletions(-) diff --git a/S3ZeroCopyReplication.md b/S3ZeroCopyReplication.md index 7e7709ff5a7..0744460012a 100644 --- a/S3ZeroCopyReplication.md +++ b/S3ZeroCopyReplication.md @@ -2,8 +2,6 @@ Говнокод просто для теста, не production-ready ни разу. -[Коммит](https://github.com/ianton-ru/ClickHouse/commit/acf86568a7e21176ba2cca15861da231bec6932a) - [Ветка](https://github.com/ianton-ru/ClickHouse/tree/s3_zero_copy_replication) ## Как сделано @@ -14,21 +12,24 @@ Введена новая версия протокола REPLICATION_PROTOCOL_VERSION_WITH_PARTS_S3_COPY. В запросе новый параметр send_s3_metadata, если 1, то приемних просит у источника метаданные вместо данных, если это возможно. Приемник в ответ отсылает куку send_s3_metadata=1 в случае, если идут метаданные. В остальных случаях отсылаются данные, как и прежде. -Применик перед запросом смотрит, будет ли хранить данные в S3. Провеока сейчас кривая - запрашивается резервирование на диске с наибольшим доступным местом, а потом смотрится, не на S3 ли оно. -Если на S3, то отсылает в запросе send_s3_metadata=1. +Применик перед запросом смотрит, будет ли хранить данные в S3. Проверка сейчас кривая - если в сторадже есть S3, то считаем, что будет S3. +Если да S3, то отсылает в запросе send_s3_metadata=1. -Источник при получении такого запроса смотрит, лежит ли парт на S3. Если да, то в Зукипере ставит метку по пути `<путь к данным таблицы>/zero_copy_s3/<некий ID парта>/`, +Источник при получении такого запроса смотрит, лежит ли парт на S3. Если да, то в Зукипере ставит метку по пути `<путь к данным таблицы>/zero_copy_s3/shared/<некий ID парта>/`, ставит в ответ куку send_s3_metadata=1 и вместо файлов с данными отсылает только файлы метаданных. Приемник при получении ответа с send_s3_metadata=1 создает только файлики с идентичными меаданными, которые в итоге будут ссылаться на те же ключи в S3, ставит в зукипере аналогичную метку, -только со своим ID реплики, и работает с этим. +только со своим ID реплики, и работает с этим. Для первого фалйа из списка проверяет наличие первого ы3-объекта (просто наличие), если объект с таким именем найден, то все ок, если нет, то откат на старую версию. +(Сейчас есть еще код на случай наличия более одного диска S3, тогда перебирает все и если на каком-то файл найден, то использует его, но мы внутри команды MDB смотрим на такую конфигурацию как на странную. +Планируем ограничить функционал только случаем одного S3 диска.) -При желании удалить парт нода удаляет в Зукипере ключ `<путь к данным таблицы>/zero_copy_s3/<некий ID парта>/`, потом получает все подключи `<путь к данным таблицы>/zero_copy_s3/<некий ID парта>`. +При желании удалить парт нода удаляет в Зукипере ключ `<путь к данным таблицы>/zero_copy_s3/shared/<некий ID парта>/`, потом получает все подключи `<путь к данным таблицы>/zero_copy_s3/shared/<некий ID парта>`. Если список не пустой, то считает, что данные использует другая нода и удаляет только локальные метаданные, если пустой, то удаляет и данные в S3. -## Костыли и недоработки, коих много +При мерже если реузльтат будет на S3, нода ставит эфемерную метку в Zookeeper по пути `<путь к данным таблицы>/zero_copy_s3/merged/<имя нового парта>`. Если такая метка уже есть, то считает, что другая нода +уже помержила или мержит сейчас, и надо сделать fetch вместо мержа самой. -* Никакой проверки, один и тот же S3 у нод или разный сейчас нет, если будет несколько разных S3, работать не будет. +## Костыли и недоработки, коих много * В качестве ID парта берется имя первого S3-ключа от файла checksums.txt. @@ -40,8 +41,18 @@ * В протоколе репликации обмен инфой через параметр запрос в одну сторону и куку в другую мне не нравится, хотя так сделан обмен версиями репликации. -* При ошибке должно пытаться реплицироваться по старому, но хз, всегда ли сработает +* При ошибке должно пытаться реплицироваться по старому, но не уверен, всегда ли сработает * Не будет обратной совместимости, если образуются такие шареные парты, откатиться на старую версию кликхауса не получится, иначе нода может удалить используемые другой данные. -* И вообще +* Возможны все же дублирования партов. Пример - нода делает мерж, падает. Другая нода незавимо делает мерж, первая нода поднимается. В итоге есть две копии померженого парта. + +* ... много их. Честно. + +## TODO, чего еще вообще не делалось + +* Флаг в конфиге для включения функционала, по умолчанию будет выключен. + +* Для гибридного хранилища сделать проверку и fetch при переезде парта с локального диска в S3. + +* Тесты. diff --git a/src/Storages/MergeTree/IMergeTreeDataPart.cpp b/src/Storages/MergeTree/IMergeTreeDataPart.cpp index 786bc056702..d9098aec1dc 100644 --- a/src/Storages/MergeTree/IMergeTreeDataPart.cpp +++ b/src/Storages/MergeTree/IMergeTreeDataPart.cpp @@ -1092,7 +1092,7 @@ void IMergeTreeDataPart::lockSharedData(const String & zookeeper_path, const Str if (id.empty()) throw Exception("Can't lock part on S3 storage", ErrorCodes::LOGICAL_ERROR); - String zookeeper_node = zookeeper_path + "/zero_copy_s3/" + id + "/" + replica_name; + String zookeeper_node = zookeeper_path + "/zero_copy_s3/shared/" + id + "/" + replica_name; LOG_TRACE(storage.log, "Set zookeeper lock {}", id); @@ -1112,7 +1112,7 @@ bool IMergeTreeDataPart::unlockSharedData(const String & zookeeper_path, const S if (id.empty()) return true; - String zookeeper_part_node = zookeeper_path + "/zero_copy_s3/" + id; + String zookeeper_part_node = zookeeper_path + "/zero_copy_s3/shared/" + id; String zookeeper_node = zookeeper_part_node + "/" + replica_name; LOG_TRACE(storage.log, "Remove zookeeper lock for {}", id); diff --git a/src/Storages/StorageReplicatedMergeTree.cpp b/src/Storages/StorageReplicatedMergeTree.cpp index 6355894d59e..c8e8388028b 100644 --- a/src/Storages/StorageReplicatedMergeTree.cpp +++ b/src/Storages/StorageReplicatedMergeTree.cpp @@ -1436,6 +1436,20 @@ bool StorageReplicatedMergeTree::tryExecuteMerge(const LogEntry & entry) future_merged_part.updatePath(*this, reserved_space); future_merged_part.merge_type = entry.merge_type; + { + auto disk = reserved_space->getDisk(); + if (disk->getType() == "s3") + { + auto zookeeper = getZooKeeper(); + String zookeeper_node = zookeeper_path + "/zero_copy_s3/merged/" + entry.new_part_name; + zookeeper->createAncestors(zookeeper_node); + auto code = zookeeper->tryCreate(zookeeper_node, "lock", zkutil::CreateMode::Ephemeral); + /// Someone else created or started create this merge + if (code == Coordination::Error::ZNODEEXISTS) + return false; + } + } + auto table_id = getStorageID(); MergeList::EntryPtr merge_entry = global_context.getMergeList().insert(table_id.database_name, table_id.table_name, future_merged_part); From fb178ef2139d56775f60d5b9d6cd2401aee6dd8c Mon Sep 17 00:00:00 2001 From: Anton Ivashkin Date: Mon, 19 Oct 2020 15:20:45 +0300 Subject: [PATCH 006/544] Zero copy replication over S3: base tests --- S3ZeroCopyReplication.md | 2 +- .../test_s3_zero_copy_replication/__init__.py | 0 .../configs/config.d/s3.xml | 49 +++++++++++ .../test_s3_zero_copy_replication/test.py | 84 +++++++++++++++++++ 4 files changed, 134 insertions(+), 1 deletion(-) create mode 100644 tests/integration/test_s3_zero_copy_replication/__init__.py create mode 100644 tests/integration/test_s3_zero_copy_replication/configs/config.d/s3.xml create mode 100644 tests/integration/test_s3_zero_copy_replication/test.py diff --git a/S3ZeroCopyReplication.md b/S3ZeroCopyReplication.md index 0744460012a..1e152977753 100644 --- a/S3ZeroCopyReplication.md +++ b/S3ZeroCopyReplication.md @@ -1,6 +1,6 @@ # ClickHouse S3 Zero Copy Replication -Говнокод просто для теста, не production-ready ни разу. +Код просто для теста, не production-ready ни разу. [Ветка](https://github.com/ianton-ru/ClickHouse/tree/s3_zero_copy_replication) diff --git a/tests/integration/test_s3_zero_copy_replication/__init__.py b/tests/integration/test_s3_zero_copy_replication/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tests/integration/test_s3_zero_copy_replication/configs/config.d/s3.xml b/tests/integration/test_s3_zero_copy_replication/configs/config.d/s3.xml new file mode 100644 index 00000000000..24a3fb95c53 --- /dev/null +++ b/tests/integration/test_s3_zero_copy_replication/configs/config.d/s3.xml @@ -0,0 +1,49 @@ + + + + + + s3 + http://minio1:9001/root/data/ + minio + minio123 + + + + + +
+ s31 +
+
+
+
+
+ + + 0 + 2 + + + + + + + node1 + 9000 + + + + + node2 + 9000 + + + + + + + test_cluster + + +
diff --git a/tests/integration/test_s3_zero_copy_replication/test.py b/tests/integration/test_s3_zero_copy_replication/test.py new file mode 100644 index 00000000000..278559f73f1 --- /dev/null +++ b/tests/integration/test_s3_zero_copy_replication/test.py @@ -0,0 +1,84 @@ +import logging +import time + +import pytest +from helpers.cluster import ClickHouseCluster + +logging.getLogger().setLevel(logging.INFO) +logging.getLogger().addHandler(logging.StreamHandler()) + + +@pytest.fixture(scope="module") +def cluster(): + try: + cluster = ClickHouseCluster(__file__) + cluster.add_instance("node1", main_configs=["configs/config.d/s3.xml"], macros={'replica': '1'}, + with_minio=True, + with_zookeeper=True) + cluster.add_instance("node2", main_configs=["configs/config.d/s3.xml"], macros={'replica': '2'}, + with_minio=True, + with_zookeeper=True) + logging.info("Starting cluster...") + cluster.start() + logging.info("Cluster started") + + yield cluster + finally: + cluster.shutdown() + + +def get_large_objects_count(cluster, size=100): + minio = cluster.minio_client + counter = 0 + for obj in minio.list_objects(cluster.minio_bucket, 'data/'): + if obj.size >= size: + counter = counter + 1 + return counter + + +@pytest.mark.parametrize( + "policy", ["s3"] +) +def test_s3_zero_copy_replication(cluster, policy): + node1 = cluster.instances["node1"] + node2 = cluster.instances["node2"] + + node1.query( + """ + CREATE TABLE s3_test ON CLUSTER test_cluster (id UInt32, value String) + ENGINE=ReplicatedMergeTree('/clickhouse/tables/s3_test', '{}') + ORDER BY id + SETTINGS storage_policy='{}' + """ + .format('{replica}', policy) + ) + + node1.query("INSERT INTO s3_test VALUES (0,'data'),(1,'data')") + assert node1.query("SELECT * FROM s3_test order by id FORMAT Values") == "(0,'data'),(1,'data')" + assert node2.query("SELECT * FROM s3_test order by id FORMAT Values") == "(0,'data'),(1,'data')" + + # Based on version 20.x - should be only one file with size 100+ (checksums.txt), used by both nodes + assert get_large_objects_count(cluster) == 1 + + node2.query("INSERT INTO s3_test VALUES (2,'data'),(3,'data')") + assert node2.query("SELECT * FROM s3_test order by id FORMAT Values") == "(0,'data'),(1,'data'),(2,'data'),(3,'data')" + assert node1.query("SELECT * FROM s3_test order by id FORMAT Values") == "(0,'data'),(1,'data'),(2,'data'),(3,'data')" + + # Based on version 20.x - two parts + assert get_large_objects_count(cluster) == 2 + + node1.query("OPTIMIZE TABLE s3_test") + + time.sleep(1) + + # Based on version 20.x - after merge, two old parts and one merged + assert get_large_objects_count(cluster) == 3 + + time.sleep(60) + + # Based on version 20.x - after cleanup - only one merged part + assert get_large_objects_count(cluster) == 1 + + node1.query("DROP TABLE IF EXISTS s3_test NO DELAY") + node2.query("DROP TABLE IF EXISTS s3_test NO DELAY") + From 652c56e74e7fcb560c535f6695845c6b16ab32a4 Mon Sep 17 00:00:00 2001 From: Anton Ivashkin Date: Thu, 22 Oct 2020 12:32:05 +0300 Subject: [PATCH 007/544] Fix style, fix build --- src/Disks/DiskDecorator.h | 2 +- src/Disks/IDisk.h | 2 +- src/Disks/S3/DiskS3.cpp | 2 +- src/Disks/S3/DiskS3.h | 2 +- src/Storages/MergeTree/DataPartsExchange.cpp | 12 +++++------- src/Storages/MergeTree/DataPartsExchange.h | 7 +++---- src/Storages/MergeTree/IMergeTreeDataPart.cpp | 2 +- src/Storages/MergeTree/IMergeTreeDataPart.h | 2 +- 8 files changed, 14 insertions(+), 17 deletions(-) diff --git a/src/Disks/DiskDecorator.h b/src/Disks/DiskDecorator.h index d230d49b400..8dcdb64ead5 100644 --- a/src/Disks/DiskDecorator.h +++ b/src/Disks/DiskDecorator.h @@ -50,7 +50,7 @@ public: void close(int fd) const override; void sync(int fd) const override; const String getType() const override { return delegate->getType(); } - const String getUniqueId(const String & path) const override { return delegate->getUniqueId(path); } + String getUniqueId(const String & path) const override { return delegate->getUniqueId(path); } bool checkFile(const String & path) const override { return delegate->checkFile(path); } Executor & getExecutor() override; diff --git a/src/Disks/IDisk.h b/src/Disks/IDisk.h index 143b094fb38..63432bc226a 100644 --- a/src/Disks/IDisk.h +++ b/src/Disks/IDisk.h @@ -196,7 +196,7 @@ public: virtual void shutdown() { } /// Return some uniq string for file, overrided for S3 - virtual const String getUniqueId(const String & path) const { return path; } + virtual String getUniqueId(const String & path) const { return path; } /// Check file, overrided for S3 only virtual bool checkFile(const String & path) const { return exists(path); } diff --git a/src/Disks/S3/DiskS3.cpp b/src/Disks/S3/DiskS3.cpp index b563c84094a..8e5e230d9db 100644 --- a/src/Disks/S3/DiskS3.cpp +++ b/src/Disks/S3/DiskS3.cpp @@ -572,7 +572,7 @@ void DiskS3::createDirectories(const String & path) Poco::File(metadata_path + path).createDirectories(); } -const String DiskS3::getUniqueId(const String & path) const +String DiskS3::getUniqueId(const String & path) const { Metadata metadata(s3_root_path, metadata_path, path); String id; diff --git a/src/Disks/S3/DiskS3.h b/src/Disks/S3/DiskS3.h index 07348c53417..cc52722f973 100644 --- a/src/Disks/S3/DiskS3.h +++ b/src/Disks/S3/DiskS3.h @@ -110,7 +110,7 @@ public: void shutdown() override; - const String getUniqueId(const String & path) const override; + String getUniqueId(const String & path) const override; bool checkFile(const String & path) const override; diff --git a/src/Storages/MergeTree/DataPartsExchange.cpp b/src/Storages/MergeTree/DataPartsExchange.cpp index c6568340620..265d855ba31 100644 --- a/src/Storages/MergeTree/DataPartsExchange.cpp +++ b/src/Storages/MergeTree/DataPartsExchange.cpp @@ -3,7 +3,6 @@ #include #include #include -#include #include #include #include @@ -269,7 +268,7 @@ void Service::sendPartS3Metadata(const MergeTreeData::DataPartPtr & part, WriteB throw Exception("Unexpected size of file " + metadata_file, ErrorCodes::BAD_SIZE_OF_FILE_IN_DATA_PART); writePODBinary(hashing_out.getHash(), out); - } + } } MergeTreeData::DataPartPtr Service::findPart(const String & name) @@ -359,7 +358,7 @@ MergeTreeData::MutableDataPartPtr Fetcher::fetchPart( throw Exception("Got 'send_s3_metadata' cookie with old protocol version", ErrorCodes::LOGICAL_ERROR); if (!try_use_s3_copy) throw Exception("Got 'send_s3_metadata' cookie when was not requested", ErrorCodes::LOGICAL_ERROR); - + size_t sum_files_size = 0; readBinary(sum_files_size, in); IMergeTreeDataPart::TTLInfos ttl_infos; @@ -373,14 +372,14 @@ MergeTreeData::MutableDataPartPtr Fetcher::fetchPart( try { - return downloadPartToS3(part_name, replica_path, to_detached, tmp_prefix_, sync, std::move(disksS3), in); + return downloadPartToS3(part_name, replica_path, to_detached, tmp_prefix_, std::move(disksS3), in); } - catch(const Exception& e) + catch (const Exception & e) { if (e.code() != ErrorCodes::S3_ERROR) throw; /// Try again but without S3 copy - return fetchPart(metadata_snapshot, part_name, replica_path, host, port, timeouts, + return fetchPart(metadata_snapshot, part_name, replica_path, host, port, timeouts, user, password, interserver_scheme, to_detached, tmp_prefix_, false); } } @@ -545,7 +544,6 @@ MergeTreeData::MutableDataPartPtr Fetcher::downloadPartToS3( const String & replica_path, bool to_detached, const String & tmp_prefix_, - bool ,//sync, const Disks & disksS3, PooledReadWriteBufferFromHTTP & in ) diff --git a/src/Storages/MergeTree/DataPartsExchange.h b/src/Storages/MergeTree/DataPartsExchange.h index 7e59e81d6dc..ac591c2046a 100644 --- a/src/Storages/MergeTree/DataPartsExchange.h +++ b/src/Storages/MergeTree/DataPartsExchange.h @@ -35,8 +35,8 @@ public: std::string getId(const std::string & node_id) const override; void processQuery(const Poco::Net::HTMLForm & params, ReadBuffer & body, WriteBuffer & out, Poco::Net::HTTPServerResponse & response) override; - void setZooKeeper(const zkutil::ZooKeeperPtr & zookeeper_, const String & zookeeper_path_, const String & replica_name_) override - { + void setZooKeeper(const zkutil::ZooKeeperPtr & zookeeper_, const String & zookeeper_path_, const String & replica_name_) override + { zookeeper = zookeeper_; zookeeper_path = zookeeper_path_; replica_name = replica_name_; @@ -87,7 +87,7 @@ public: ActionBlocker blocker; void setZooKeeper(const zkutil::ZooKeeperPtr & zookeeper_, const String & zookeeper_path_, const String & replica_name_) - { + { zookeeper = zookeeper_; zookeeper_path = zookeeper_path_; replica_name = replica_name_; @@ -114,7 +114,6 @@ private: const String & replica_path, bool to_detached, const String & tmp_prefix_, - bool sync, const Disks & disksS3, PooledReadWriteBufferFromHTTP & in); diff --git a/src/Storages/MergeTree/IMergeTreeDataPart.cpp b/src/Storages/MergeTree/IMergeTreeDataPart.cpp index 93f424cf0d1..badfb32cf58 100644 --- a/src/Storages/MergeTree/IMergeTreeDataPart.cpp +++ b/src/Storages/MergeTree/IMergeTreeDataPart.cpp @@ -1106,7 +1106,7 @@ void IMergeTreeDataPart::lockSharedData(const String & zookeeper_path, const Str if (id.empty()) throw Exception("Can't lock part on S3 storage", ErrorCodes::LOGICAL_ERROR); - + String zookeeper_node = zookeeper_path + "/zero_copy_s3/shared/" + id + "/" + replica_name; LOG_TRACE(storage.log, "Set zookeeper lock {}", id); diff --git a/src/Storages/MergeTree/IMergeTreeDataPart.h b/src/Storages/MergeTree/IMergeTreeDataPart.h index 9213578c831..8d21f5856fc 100644 --- a/src/Storages/MergeTree/IMergeTreeDataPart.h +++ b/src/Storages/MergeTree/IMergeTreeDataPart.h @@ -361,7 +361,7 @@ public: /// Lock part in zookeeper for use common S3 data in several nodes void lockSharedData(const String & zookeeper_path, const String & replica_name, zkutil::ZooKeeperPtr zookeeper) const; - + /// Unlock common S3 data part in zookeeper /// Return true if data unlocked /// Return false if data is still used by another node From 1ffe0b1d03db9fedafe4918489b4ca5598553480 Mon Sep 17 00:00:00 2001 From: Anton Ivashkin Date: Fri, 23 Oct 2020 13:01:40 +0300 Subject: [PATCH 008/544] S3 zero copy replication: fix tests --- .../configs/config.d/storage_conf.xml | 1 + .../__init__.py | 0 .../configs/config.d/storage_conf.xml | 50 +++++++++ .../test.py | 105 ++++++++++++++++++ .../configs/config.d/s3.xml | 1 + .../test_s3_zero_copy_replication/test.py | 2 + 6 files changed, 159 insertions(+) create mode 100644 tests/integration/test_replicated_merge_tree_s3_zero_copy/__init__.py create mode 100644 tests/integration/test_replicated_merge_tree_s3_zero_copy/configs/config.d/storage_conf.xml create mode 100644 tests/integration/test_replicated_merge_tree_s3_zero_copy/test.py diff --git a/tests/integration/test_replicated_merge_tree_s3/configs/config.d/storage_conf.xml b/tests/integration/test_replicated_merge_tree_s3/configs/config.d/storage_conf.xml index 20b750ffff3..1f75a4efeae 100644 --- a/tests/integration/test_replicated_merge_tree_s3/configs/config.d/storage_conf.xml +++ b/tests/integration/test_replicated_merge_tree_s3/configs/config.d/storage_conf.xml @@ -21,6 +21,7 @@ 0 + 0 diff --git a/tests/integration/test_replicated_merge_tree_s3_zero_copy/__init__.py b/tests/integration/test_replicated_merge_tree_s3_zero_copy/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tests/integration/test_replicated_merge_tree_s3_zero_copy/configs/config.d/storage_conf.xml b/tests/integration/test_replicated_merge_tree_s3_zero_copy/configs/config.d/storage_conf.xml new file mode 100644 index 00000000000..d8c7f49fc49 --- /dev/null +++ b/tests/integration/test_replicated_merge_tree_s3_zero_copy/configs/config.d/storage_conf.xml @@ -0,0 +1,50 @@ + + + + + s3 + http://minio1:9001/root/data/ + minio + minio123 + + + + + +
+ s3 +
+
+
+
+
+ + + 0 + 1 + + + + + + + node1 + 9000 + + + node2 + 9000 + + + node3 + 9000 + + + + + + + 0 + + +
diff --git a/tests/integration/test_replicated_merge_tree_s3_zero_copy/test.py b/tests/integration/test_replicated_merge_tree_s3_zero_copy/test.py new file mode 100644 index 00000000000..793abc53566 --- /dev/null +++ b/tests/integration/test_replicated_merge_tree_s3_zero_copy/test.py @@ -0,0 +1,105 @@ +import logging +import random +import string + +import pytest +from helpers.cluster import ClickHouseCluster + +logging.getLogger().setLevel(logging.INFO) +logging.getLogger().addHandler(logging.StreamHandler()) + + +@pytest.fixture(scope="module") +def cluster(): + try: + cluster = ClickHouseCluster(__file__) + + cluster.add_instance("node1", main_configs=["configs/config.d/storage_conf.xml"], macros={'replica': '1'}, + with_minio=True, with_zookeeper=True) + cluster.add_instance("node2", main_configs=["configs/config.d/storage_conf.xml"], macros={'replica': '2'}, + with_zookeeper=True) + cluster.add_instance("node3", main_configs=["configs/config.d/storage_conf.xml"], macros={'replica': '3'}, + with_zookeeper=True) + + logging.info("Starting cluster...") + cluster.start() + logging.info("Cluster started") + + yield cluster + finally: + cluster.shutdown() + + +FILES_OVERHEAD = 1 +FILES_OVERHEAD_PER_COLUMN = 2 # Data and mark files +FILES_OVERHEAD_PER_PART_WIDE = FILES_OVERHEAD_PER_COLUMN * 3 + 2 + 6 + 1 +FILES_OVERHEAD_PER_PART_COMPACT = 10 + 1 + + +def random_string(length): + letters = string.ascii_letters + return ''.join(random.choice(letters) for i in range(length)) + + +def generate_values(date_str, count, sign=1): + data = [[date_str, sign * (i + 1), random_string(10)] for i in range(count)] + data.sort(key=lambda tup: tup[1]) + return ",".join(["('{}',{},'{}')".format(x, y, z) for x, y, z in data]) + + +def create_table(cluster, additional_settings=None): + create_table_statement = """ + CREATE TABLE s3_test ON CLUSTER cluster( + dt Date, + id Int64, + data String, + INDEX min_max (id) TYPE minmax GRANULARITY 3 + ) ENGINE=ReplicatedMergeTree() + PARTITION BY dt + ORDER BY (dt, id) + SETTINGS storage_policy='s3' + """ + if additional_settings: + create_table_statement += "," + create_table_statement += additional_settings + + list(cluster.instances.values())[0].query(create_table_statement) + + +@pytest.fixture(autouse=True) +def drop_table(cluster): + yield + for node in list(cluster.instances.values()): + node.query("DROP TABLE IF EXISTS s3_test") + + minio = cluster.minio_client + # Remove extra objects to prevent tests cascade failing + for obj in list(minio.list_objects(cluster.minio_bucket, 'data/')): + minio.remove_object(cluster.minio_bucket, obj.object_name) + +@pytest.mark.parametrize( + "min_rows_for_wide_part,files_per_part", + [ + (0, FILES_OVERHEAD_PER_PART_WIDE), + (8192, FILES_OVERHEAD_PER_PART_COMPACT) + ] +) +def test_insert_select_replicated(cluster, min_rows_for_wide_part, files_per_part): + create_table(cluster, additional_settings="min_rows_for_wide_part={}".format(min_rows_for_wide_part)) + + all_values = "" + for node_idx in range(1, 4): + node = cluster.instances["node" + str(node_idx)] + values = generate_values("2020-01-0" + str(node_idx), 4096) + node.query("INSERT INTO s3_test VALUES {}".format(values), settings={"insert_quorum": 3}) + if node_idx != 1: + all_values += "," + all_values += values + + for node_idx in range(1, 4): + node = cluster.instances["node" + str(node_idx)] + assert node.query("SELECT * FROM s3_test order by dt, id FORMAT Values", + settings={"select_sequential_consistency": 1}) == all_values + + minio = cluster.minio_client + assert len(list(minio.list_objects(cluster.minio_bucket, 'data/'))) == (3 * FILES_OVERHEAD) + (files_per_part * 3) diff --git a/tests/integration/test_s3_zero_copy_replication/configs/config.d/s3.xml b/tests/integration/test_s3_zero_copy_replication/configs/config.d/s3.xml index 24a3fb95c53..285ade3f727 100644 --- a/tests/integration/test_s3_zero_copy_replication/configs/config.d/s3.xml +++ b/tests/integration/test_s3_zero_copy_replication/configs/config.d/s3.xml @@ -23,6 +23,7 @@ 0 2 + 1 diff --git a/tests/integration/test_s3_zero_copy_replication/test.py b/tests/integration/test_s3_zero_copy_replication/test.py index 278559f73f1..88d038e357b 100644 --- a/tests/integration/test_s3_zero_copy_replication/test.py +++ b/tests/integration/test_s3_zero_copy_replication/test.py @@ -54,6 +54,7 @@ def test_s3_zero_copy_replication(cluster, policy): ) node1.query("INSERT INTO s3_test VALUES (0,'data'),(1,'data')") + time.sleep(1) assert node1.query("SELECT * FROM s3_test order by id FORMAT Values") == "(0,'data'),(1,'data')" assert node2.query("SELECT * FROM s3_test order by id FORMAT Values") == "(0,'data'),(1,'data')" @@ -61,6 +62,7 @@ def test_s3_zero_copy_replication(cluster, policy): assert get_large_objects_count(cluster) == 1 node2.query("INSERT INTO s3_test VALUES (2,'data'),(3,'data')") + time.sleep(1) assert node2.query("SELECT * FROM s3_test order by id FORMAT Values") == "(0,'data'),(1,'data'),(2,'data'),(3,'data')" assert node1.query("SELECT * FROM s3_test order by id FORMAT Values") == "(0,'data'),(1,'data'),(2,'data'),(3,'data')" From e3879afa69672d28686b591dc3b088d1bf451b7a Mon Sep 17 00:00:00 2001 From: Anton Ivashkin Date: Fri, 23 Oct 2020 15:01:50 +0300 Subject: [PATCH 009/544] S3 zero copy replication: fix virtual method default parameter --- src/Disks/DiskCacheWrapper.cpp | 32 +++++++++++++------ src/Disks/DiskCacheWrapper.h | 8 +++-- src/Disks/DiskDecorator.cpp | 22 +++++++++---- src/Disks/DiskDecorator.h | 8 +++-- src/Disks/DiskLocal.cpp | 6 ++-- src/Disks/DiskLocal.h | 6 ++-- src/Disks/DiskMemory.cpp | 6 ++-- src/Disks/DiskMemory.h | 6 ++-- src/Disks/IDisk.h | 19 ++++++++--- src/Disks/S3/DiskS3.cpp | 8 ++--- src/Disks/S3/DiskS3.h | 10 ++++-- src/Storages/MergeTree/DataPartsExchange.cpp | 32 +++++++++++-------- src/Storages/MergeTree/DataPartsExchange.h | 2 +- src/Storages/MergeTree/IMergeTreeDataPart.cpp | 16 +++++----- src/Storages/MergeTree/MergeTreeSettings.h | 1 + 15 files changed, 114 insertions(+), 68 deletions(-) diff --git a/src/Disks/DiskCacheWrapper.cpp b/src/Disks/DiskCacheWrapper.cpp index 79e615d3609..8e0f77eed6d 100644 --- a/src/Disks/DiskCacheWrapper.cpp +++ b/src/Disks/DiskCacheWrapper.cpp @@ -199,11 +199,11 @@ DiskCacheWrapper::writeFile(const String & path, size_t buf_size, WriteMode mode buf_size); } -void DiskCacheWrapper::clearDirectory(const String & path, bool keep_s3) +void DiskCacheWrapper::clearDirectory(const String & path) { if (cache_disk->exists(path)) - cache_disk->clearDirectory(path, keep_s3); - DiskDecorator::clearDirectory(path, keep_s3); + cache_disk->clearDirectory(path); + DiskDecorator::clearDirectory(path); } void DiskCacheWrapper::moveDirectory(const String & from_path, const String & to_path) @@ -252,18 +252,32 @@ void DiskCacheWrapper::copyFile(const String & from_path, const String & to_path DiskDecorator::copyFile(from_path, to_path); } -void DiskCacheWrapper::remove(const String & path, bool keep_s3) +void DiskCacheWrapper::remove(const String & path) { if (cache_disk->exists(path)) - cache_disk->remove(path, keep_s3); - DiskDecorator::remove(path, keep_s3); + cache_disk->remove(path); + DiskDecorator::remove(path); } -void DiskCacheWrapper::removeRecursive(const String & path, bool keep_s3) +void DiskCacheWrapper::removeRecursive(const String & path) { if (cache_disk->exists(path)) - cache_disk->removeRecursive(path, keep_s3); - DiskDecorator::removeRecursive(path, keep_s3); + cache_disk->removeRecursive(path); + DiskDecorator::removeRecursive(path); +} + +void DiskCacheWrapper::removeShared(const String & path, bool keep_s3) +{ + if (cache_disk->exists(path)) + cache_disk->removeShared(path, keep_s3); + DiskDecorator::removeShared(path, keep_s3); +} + +void DiskCacheWrapper::removeSharedRecursive(const String & path, bool keep_s3) +{ + if (cache_disk->exists(path)) + cache_disk->removeSharedRecursive(path, keep_s3); + DiskDecorator::removeSharedRecursive(path, keep_s3); } void DiskCacheWrapper::createHardLink(const String & src_path, const String & dst_path) diff --git a/src/Disks/DiskCacheWrapper.h b/src/Disks/DiskCacheWrapper.h index 9fca4e02e34..6722d5bd1a5 100644 --- a/src/Disks/DiskCacheWrapper.h +++ b/src/Disks/DiskCacheWrapper.h @@ -28,7 +28,7 @@ public: std::function cache_file_predicate_); void createDirectory(const String & path) override; void createDirectories(const String & path) override; - void clearDirectory(const String & path, bool keep_s3 = false) override; + void clearDirectory(const String & path) override; void moveDirectory(const String & from_path, const String & to_path) override; void moveFile(const String & from_path, const String & to_path) override; void replaceFile(const String & from_path, const String & to_path) override; @@ -37,8 +37,10 @@ public: readFile(const String & path, size_t buf_size, size_t estimated_size, size_t aio_threshold, size_t mmap_threshold) const override; std::unique_ptr writeFile(const String & path, size_t buf_size, WriteMode mode, size_t estimated_size, size_t aio_threshold) override; - void remove(const String & path, bool keep_s3 = false) override; - void removeRecursive(const String & path, bool keep_s3 = false) override; + void remove(const String & path) override; + void removeRecursive(const String & path) override; + void removeShared(const String & path, bool keep_s3) override; + void removeSharedRecursive(const String & path, bool keep_s3) override; void createHardLink(const String & src_path, const String & dst_path) override; ReservationPtr reserve(UInt64 bytes) override; diff --git a/src/Disks/DiskDecorator.cpp b/src/Disks/DiskDecorator.cpp index 4ad71a67f95..e7a5beeaff1 100644 --- a/src/Disks/DiskDecorator.cpp +++ b/src/Disks/DiskDecorator.cpp @@ -73,9 +73,9 @@ void DiskDecorator::createDirectories(const String & path) delegate->createDirectories(path); } -void DiskDecorator::clearDirectory(const String & path, bool keep_s3) +void DiskDecorator::clearDirectory(const String & path) { - delegate->clearDirectory(path, keep_s3); + delegate->clearDirectory(path); } void DiskDecorator::moveDirectory(const String & from_path, const String & to_path) @@ -130,14 +130,24 @@ DiskDecorator::writeFile(const String & path, size_t buf_size, WriteMode mode, s return delegate->writeFile(path, buf_size, mode, estimated_size, aio_threshold); } -void DiskDecorator::remove(const String & path, bool keep_s3) +void DiskDecorator::remove(const String & path) { - delegate->remove(path, keep_s3); + delegate->remove(path); } -void DiskDecorator::removeRecursive(const String & path, bool keep_s3) +void DiskDecorator::removeRecursive(const String & path) { - delegate->removeRecursive(path, keep_s3); + delegate->removeRecursive(path); +} + +void DiskDecorator::removeShared(const String & path, bool keep_s3) +{ + delegate->removeShared(path, keep_s3); +} + +void DiskDecorator::removeSharedRecursive(const String & path, bool keep_s3) +{ + delegate->removeSharedRecursive(path, keep_s3); } void DiskDecorator::setLastModified(const String & path, const Poco::Timestamp & timestamp) diff --git a/src/Disks/DiskDecorator.h b/src/Disks/DiskDecorator.h index 8dcdb64ead5..4bc7879ffd3 100644 --- a/src/Disks/DiskDecorator.h +++ b/src/Disks/DiskDecorator.h @@ -26,7 +26,7 @@ public: size_t getFileSize(const String & path) const override; void createDirectory(const String & path) override; void createDirectories(const String & path) override; - void clearDirectory(const String & path, bool keep_s3 = false) override; + void clearDirectory(const String & path) override; void moveDirectory(const String & from_path, const String & to_path) override; DiskDirectoryIteratorPtr iterateDirectory(const String & path) override; void createFile(const String & path) override; @@ -39,8 +39,10 @@ public: readFile(const String & path, size_t buf_size, size_t estimated_size, size_t aio_threshold, size_t mmap_threshold) const override; std::unique_ptr writeFile(const String & path, size_t buf_size, WriteMode mode, size_t estimated_size, size_t aio_threshold) override; - void remove(const String & path, bool keep_s3 = false) override; - void removeRecursive(const String & path, bool keep_s3 = false) override; + void remove(const String & path) override; + void removeRecursive(const String & path) override; + void removeShared(const String & path, bool keep_s3) override; + void removeSharedRecursive(const String & path, bool keep_s3) override; void setLastModified(const String & path, const Poco::Timestamp & timestamp) override; Poco::Timestamp getLastModified(const String & path) override; void setReadOnly(const String & path) override; diff --git a/src/Disks/DiskLocal.cpp b/src/Disks/DiskLocal.cpp index ad85fdf4236..a09ab7c5ac5 100644 --- a/src/Disks/DiskLocal.cpp +++ b/src/Disks/DiskLocal.cpp @@ -180,7 +180,7 @@ void DiskLocal::createDirectories(const String & path) Poco::File(disk_path + path).createDirectories(); } -void DiskLocal::clearDirectory(const String & path, bool) +void DiskLocal::clearDirectory(const String & path) { std::vector files; Poco::File(disk_path + path).list(files); @@ -236,12 +236,12 @@ DiskLocal::writeFile(const String & path, size_t buf_size, WriteMode mode, size_ return createWriteBufferFromFileBase(disk_path + path, estimated_size, aio_threshold, buf_size, flags); } -void DiskLocal::remove(const String & path, bool) +void DiskLocal::remove(const String & path) { Poco::File(disk_path + path).remove(false); } -void DiskLocal::removeRecursive(const String & path, bool) +void DiskLocal::removeRecursive(const String & path) { Poco::File(disk_path + path).remove(true); } diff --git a/src/Disks/DiskLocal.h b/src/Disks/DiskLocal.h index 18e6d072874..762a8502faa 100644 --- a/src/Disks/DiskLocal.h +++ b/src/Disks/DiskLocal.h @@ -55,7 +55,7 @@ public: void createDirectories(const String & path) override; - void clearDirectory(const String & path, bool keep_s3 = false) override; + void clearDirectory(const String & path) override; void moveDirectory(const String & from_path, const String & to_path) override; @@ -87,9 +87,9 @@ public: size_t estimated_size, size_t aio_threshold) override; - void remove(const String & path, bool keep_s3 = false) override; + void remove(const String & path) override; - void removeRecursive(const String & path, bool keep_s3 = false) override; + void removeRecursive(const String & path) override; void setLastModified(const String & path, const Poco::Timestamp & timestamp) override; diff --git a/src/Disks/DiskMemory.cpp b/src/Disks/DiskMemory.cpp index fc375707feb..d185263d48c 100644 --- a/src/Disks/DiskMemory.cpp +++ b/src/Disks/DiskMemory.cpp @@ -233,7 +233,7 @@ void DiskMemory::createDirectoriesImpl(const String & path) files.emplace(path, FileData{FileType::Directory}); } -void DiskMemory::clearDirectory(const String & path, bool) +void DiskMemory::clearDirectory(const String & path) { std::lock_guard lock(mutex); @@ -348,7 +348,7 @@ std::unique_ptr DiskMemory::writeFile(const String & pa return std::make_unique(this, path, mode, buf_size); } -void DiskMemory::remove(const String & path, bool) +void DiskMemory::remove(const String & path) { std::lock_guard lock(mutex); @@ -368,7 +368,7 @@ void DiskMemory::remove(const String & path, bool) } } -void DiskMemory::removeRecursive(const String & path, bool) +void DiskMemory::removeRecursive(const String & path) { std::lock_guard lock(mutex); diff --git a/src/Disks/DiskMemory.h b/src/Disks/DiskMemory.h index e75d9bff100..4d4b947098b 100644 --- a/src/Disks/DiskMemory.h +++ b/src/Disks/DiskMemory.h @@ -48,7 +48,7 @@ public: void createDirectories(const String & path) override; - void clearDirectory(const String & path, bool keep_s3 = false) override; + void clearDirectory(const String & path) override; void moveDirectory(const String & from_path, const String & to_path) override; @@ -78,9 +78,9 @@ public: size_t estimated_size, size_t aio_threshold) override; - void remove(const String & path, bool keep_s3 = false) override; + void remove(const String & path) override; - void removeRecursive(const String & path, bool keep_s3 = false) override; + void removeRecursive(const String & path) override; void setLastModified(const String &, const Poco::Timestamp &) override {} diff --git a/src/Disks/IDisk.h b/src/Disks/IDisk.h index 63432bc226a..915c6da5a21 100644 --- a/src/Disks/IDisk.h +++ b/src/Disks/IDisk.h @@ -105,7 +105,7 @@ public: virtual void createDirectories(const String & path) = 0; /// Remove all files from the directory. Directories are not removed. - virtual void clearDirectory(const String & path, bool keep_s3 = false) = 0; + virtual void clearDirectory(const String & path) = 0; /// Move directory from `from_path` to `to_path`. virtual void moveDirectory(const String & from_path, const String & to_path) = 0; @@ -153,18 +153,27 @@ public: size_t aio_threshold = 0) = 0; /// Remove file or directory. Throws exception if file doesn't exists or if directory is not empty. - virtual void remove(const String & path, bool keep_s3 = false) = 0; + virtual void remove(const String & path) = 0; /// Remove file or directory with all children. Use with extra caution. Throws exception if file doesn't exists. - virtual void removeRecursive(const String & path, bool keep_s3 = false) = 0; + virtual void removeRecursive(const String & path) = 0; /// Remove file or directory if it exists. - void removeIfExists(const String & path, bool keep_s3 = false) + void removeIfExists(const String & path) { if (exists(path)) - remove(path, keep_s3); + remove(path); } + /// Remove file or directory. Throws exception if file doesn't exists or if directory is not empty. + virtual void removeShared(const String & path, bool) { remove(path); } + + /// Remove file or directory with all children. Use with extra caution. Throws exception if file doesn't exists. + virtual void removeSharedRecursive(const String & path, bool) { removeRecursive(path); } + + /// Remove file or directory if it exists. + void removeSharedIfExists(const String & path, bool) { removeIfExists(path); } + /// Set last modified time to file or directory at `path`. virtual void setLastModified(const String & path, const Poco::Timestamp & timestamp) = 0; diff --git a/src/Disks/S3/DiskS3.cpp b/src/Disks/S3/DiskS3.cpp index 8e5e230d9db..7334a5b8a9b 100644 --- a/src/Disks/S3/DiskS3.cpp +++ b/src/Disks/S3/DiskS3.cpp @@ -586,11 +586,11 @@ DiskDirectoryIteratorPtr DiskS3::iterateDirectory(const String & path) return std::make_unique(metadata_path + path, path); } -void DiskS3::clearDirectory(const String & path, bool keep_s3) +void DiskS3::clearDirectory(const String & path) { for (auto it{iterateDirectory(path)}; it->isValid(); it->next()) if (isFile(it->path())) - remove(it->path(), keep_s3); + remove(it->path()); } void DiskS3::moveFile(const String & from_path, const String & to_path) @@ -744,7 +744,7 @@ void DiskS3::removeAws(const AwsS3KeyKeeper & keys) } } -void DiskS3::remove(const String & path, bool keep_s3) +void DiskS3::removeShared(const String & path, bool keep_s3) { AwsS3KeyKeeper keys; removeMeta(path, keys); @@ -752,7 +752,7 @@ void DiskS3::remove(const String & path, bool keep_s3) removeAws(keys); } -void DiskS3::removeRecursive(const String & path, bool keep_s3) +void DiskS3::removeSharedRecursive(const String & path, bool keep_s3) { AwsS3KeyKeeper keys; removeMetaRecursive(path, keys); diff --git a/src/Disks/S3/DiskS3.h b/src/Disks/S3/DiskS3.h index cc52722f973..80752fa8253 100644 --- a/src/Disks/S3/DiskS3.h +++ b/src/Disks/S3/DiskS3.h @@ -60,7 +60,7 @@ public: void createDirectories(const String & path) override; - void clearDirectory(const String & path, bool keep_s3 = false) override; + void clearDirectory(const String & path) override; void moveDirectory(const String & from_path, const String & to_path) override { moveFile(from_path, to_path); } @@ -88,9 +88,13 @@ public: size_t estimated_size, size_t aio_threshold) override; - void remove(const String & path, bool keep_s3 = false) override; + void remove(const String & path) override { removeShared(path, false); } - void removeRecursive(const String & path, bool keep_s3 = false) override; + void removeRecursive(const String & path) override { removeSharedRecursive(path, false); } + + void removeShared(const String & path, bool keep_s3) override; + + void removeSharedRecursive(const String & path, bool keep_s3) override; void createHardLink(const String & src_path, const String & dst_path) override; diff --git a/src/Storages/MergeTree/DataPartsExchange.cpp b/src/Storages/MergeTree/DataPartsExchange.cpp index 265d855ba31..d2bd3c21173 100644 --- a/src/Storages/MergeTree/DataPartsExchange.cpp +++ b/src/Storages/MergeTree/DataPartsExchange.cpp @@ -124,7 +124,8 @@ void Service::processQuery(const Poco::Net::HTMLForm & params, ReadBuffer & /*bo { bool try_use_s3_copy = false; - if (client_protocol_version >= REPLICATION_PROTOCOL_VERSION_WITH_PARTS_S3_COPY) + if (data_settings->allow_s3_zero_copy_replication + && client_protocol_version >= REPLICATION_PROTOCOL_VERSION_WITH_PARTS_S3_COPY) { /// if source and destination are in the same S3 storage we try to use S3 CopyObject request first int send_s3_metadata = parse(params.get("send_s3_metadata", "0")); if (send_s3_metadata == 1) @@ -316,12 +317,15 @@ MergeTreeData::MutableDataPartPtr Fetcher::fetchPart( {"compress", "false"} }); - Disks disksS3; + Disks disks_s3; + + if (!data_settings->allow_s3_zero_copy_replication) + try_use_s3_copy = false; if (try_use_s3_copy) { - disksS3 = data.getDisksByType("s3"); - if (disksS3.empty()) + disks_s3 = data.getDisksByType("s3"); + if (disks_s3.empty()) try_use_s3_copy = false; } @@ -372,7 +376,7 @@ MergeTreeData::MutableDataPartPtr Fetcher::fetchPart( try { - return downloadPartToS3(part_name, replica_path, to_detached, tmp_prefix_, std::move(disksS3), in); + return downloadPartToS3(part_name, replica_path, to_detached, tmp_prefix_, std::move(disks_s3), in); } catch (const Exception & e) { @@ -544,14 +548,14 @@ MergeTreeData::MutableDataPartPtr Fetcher::downloadPartToS3( const String & replica_path, bool to_detached, const String & tmp_prefix_, - const Disks & disksS3, + const Disks & disks_s3, PooledReadWriteBufferFromHTTP & in ) { - if (disksS3.empty()) + if (disks_s3.empty()) throw Exception("No S3 disks anymore", ErrorCodes::LOGICAL_ERROR); - auto disk = disksS3[0]; + auto disk = disks_s3[0]; static const String TMP_PREFIX = "tmp_fetch_"; String tmp_prefix = tmp_prefix_.empty() ? TMP_PREFIX : tmp_prefix_; @@ -595,7 +599,7 @@ MergeTreeData::MutableDataPartPtr Fetcher::downloadPartToS3( /// NOTE The is_cancelled flag also makes sense to check every time you read over the network, /// performing a poll with a not very large timeout. /// And now we check it only between read chunks (in the `copyData` function). - disk->removeRecursive(part_download_path, true); + disk->removeSharedRecursive(part_download_path, true); throw Exception("Fetching of part was cancelled", ErrorCodes::ABORTED); } @@ -618,15 +622,15 @@ MergeTreeData::MutableDataPartPtr Fetcher::downloadPartToS3( size_t disk_id = 1; while (true) { - if (disk_id >= disksS3.size()) + if (disk_id >= disks_s3.size()) { /// No more S3 disks - disk->removeRecursive(part_download_path, true); + disk->removeSharedRecursive(part_download_path, true); /// After catch this exception replication continues with full data copy throw Exception("Can't find S3 drive for shared data", ErrorCodes::S3_ERROR); } /// Try next S3 disk - auto next_disk = disksS3[disk_id]; + auto next_disk = disks_s3[disk_id]; auto next_volume = std::make_shared("volume_" + part_name, next_disk); MergeTreeData::MutableDataPartPtr next_new_data_part = data.createPart(part_name, next_volume, part_relative_path); @@ -638,7 +642,7 @@ MergeTreeData::MutableDataPartPtr Fetcher::downloadPartToS3( metadata.copyTo(next_metadata_file); if (next_disk->checkFile(next_data_path)) { /// Right disk found - disk->removeRecursive(part_download_path, true); + disk->removeSharedRecursive(part_download_path, true); disk = next_disk; volume = next_volume; data_path = next_data_path; @@ -647,7 +651,7 @@ MergeTreeData::MutableDataPartPtr Fetcher::downloadPartToS3( } /// Wrong disk again - next_disk->removeRecursive(part_download_path, true); + next_disk->removeSharedRecursive(part_download_path, true); ++disk_id; } } diff --git a/src/Storages/MergeTree/DataPartsExchange.h b/src/Storages/MergeTree/DataPartsExchange.h index ac591c2046a..91edc3ba6d4 100644 --- a/src/Storages/MergeTree/DataPartsExchange.h +++ b/src/Storages/MergeTree/DataPartsExchange.h @@ -114,7 +114,7 @@ private: const String & replica_path, bool to_detached, const String & tmp_prefix_, - const Disks & disksS3, + const Disks & disks_s3, PooledReadWriteBufferFromHTTP & in); MergeTreeData & data; diff --git a/src/Storages/MergeTree/IMergeTreeDataPart.cpp b/src/Storages/MergeTree/IMergeTreeDataPart.cpp index badfb32cf58..be2f88e74e5 100644 --- a/src/Storages/MergeTree/IMergeTreeDataPart.cpp +++ b/src/Storages/MergeTree/IMergeTreeDataPart.cpp @@ -851,7 +851,7 @@ void IMergeTreeDataPart::remove(bool keep_s3) const try { - volume->getDisk()->removeRecursive(to + "/", keep_s3); + volume->getDisk()->removeSharedRecursive(to + "/", keep_s3); } catch (...) { @@ -874,7 +874,7 @@ void IMergeTreeDataPart::remove(bool keep_s3) const if (checksums.empty()) { /// If the part is not completely written, we cannot use fast path by listing files. - volume->getDisk()->removeRecursive(to + "/", keep_s3); + volume->getDisk()->removeSharedRecursive(to + "/", keep_s3); } else { @@ -887,18 +887,18 @@ void IMergeTreeDataPart::remove(bool keep_s3) const # pragma GCC diagnostic ignored "-Wunused-variable" #endif for (const auto & [file, _] : checksums.files) - volume->getDisk()->remove(to + "/" + file, keep_s3); + volume->getDisk()->removeShared(to + "/" + file, keep_s3); #if !__clang__ # pragma GCC diagnostic pop #endif for (const auto & file : {"checksums.txt", "columns.txt"}) - volume->getDisk()->remove(to + "/" + file, keep_s3); + volume->getDisk()->removeShared(to + "/" + file, keep_s3); - volume->getDisk()->removeIfExists(to + "/" + DEFAULT_COMPRESSION_CODEC_FILE_NAME, keep_s3); - volume->getDisk()->removeIfExists(to + "/" + DELETE_ON_DESTROY_MARKER_FILE_NAME, keep_s3); + volume->getDisk()->removeSharedIfExists(to + "/" + DEFAULT_COMPRESSION_CODEC_FILE_NAME, keep_s3); + volume->getDisk()->removeSharedIfExists(to + "/" + DELETE_ON_DESTROY_MARKER_FILE_NAME, keep_s3); - volume->getDisk()->remove(to, keep_s3); + volume->getDisk()->removeShared(to, keep_s3); } catch (...) { @@ -906,7 +906,7 @@ void IMergeTreeDataPart::remove(bool keep_s3) const LOG_ERROR(storage.log, "Cannot quickly remove directory {} by removing files; fallback to recursive removal. Reason: {}", fullPath(volume->getDisk(), to), getCurrentExceptionMessage(false)); - volume->getDisk()->removeRecursive(to + "/", keep_s3); + volume->getDisk()->removeSharedRecursive(to + "/", keep_s3); } } } diff --git a/src/Storages/MergeTree/MergeTreeSettings.h b/src/Storages/MergeTree/MergeTreeSettings.h index 97bc73caf5b..d39d212c5fc 100644 --- a/src/Storages/MergeTree/MergeTreeSettings.h +++ b/src/Storages/MergeTree/MergeTreeSettings.h @@ -105,6 +105,7 @@ struct Settings; M(UInt64, concurrent_part_removal_threshold, 100, "Activate concurrent part removal (see 'max_part_removal_threads') only if the number of inactive data parts is at least this.", 0) \ M(String, storage_policy, "default", "Name of storage disk policy", 0) \ M(Bool, allow_nullable_key, false, "Allow Nullable types as primary keys.", 0) \ + M(Bool, allow_s3_zero_copy_replication, true, "Allow Zero-copy replication over S3", 0) \ \ /** Settings for testing purposes */ \ M(Bool, randomize_part_type, false, "For testing purposes only. Randomizes part type between wide and compact", 0) \ From 78021714f1cbbf54246d09383bdf2a4d06389fa3 Mon Sep 17 00:00:00 2001 From: Anton Ivashkin Date: Tue, 3 Nov 2020 11:58:26 +0300 Subject: [PATCH 010/544] S3 zero copy replication: more simple s3 check --- S3ZeroCopyReplication.md | 22 ++++--- src/Disks/DiskDecorator.h | 2 +- src/Disks/IDisk.h | 2 +- src/Disks/S3/DiskS3.cpp | 27 +++----- src/Disks/S3/DiskS3.h | 2 +- src/Storages/MergeTree/DataPartsExchange.cpp | 64 ++++++------------- src/Storages/MergeTree/IMergeTreeDataPart.cpp | 29 ++++----- src/Storages/MergeTree/IMergeTreeDataPart.h | 2 + 8 files changed, 60 insertions(+), 90 deletions(-) diff --git a/S3ZeroCopyReplication.md b/S3ZeroCopyReplication.md index 1e152977753..22c01caa90c 100644 --- a/S3ZeroCopyReplication.md +++ b/S3ZeroCopyReplication.md @@ -12,23 +12,26 @@ Введена новая версия протокола REPLICATION_PROTOCOL_VERSION_WITH_PARTS_S3_COPY. В запросе новый параметр send_s3_metadata, если 1, то приемних просит у источника метаданные вместо данных, если это возможно. Приемник в ответ отсылает куку send_s3_metadata=1 в случае, если идут метаданные. В остальных случаях отсылаются данные, как и прежде. +В новой версии протокола перед полем с количеством файлов добавлена еще одна строка. Абстрактно это некий ID, по которому ноды могу понять, с одним S3 они работают или с разными. +Практически сейчас это один имя первого объекта файла checksums.txt. Эта же строка используется в качестве ID парта в зукипере. + Применик перед запросом смотрит, будет ли хранить данные в S3. Проверка сейчас кривая - если в сторадже есть S3, то считаем, что будет S3. Если да S3, то отсылает в запросе send_s3_metadata=1. -Источник при получении такого запроса смотрит, лежит ли парт на S3. Если да, то в Зукипере ставит метку по пути `<путь к данным таблицы>/zero_copy_s3/shared/<некий ID парта>/`, +Источник при получении такого запроса смотрит, лежит ли парт на S3. Если да, то в Зукипере ставит метку по пути `<путь к данным таблицы>/zero_copy_s3/shared//`, ставит в ответ куку send_s3_metadata=1 и вместо файлов с данными отсылает только файлы метаданных. -Приемник при получении ответа с send_s3_metadata=1 создает только файлики с идентичными меаданными, которые в итоге будут ссылаться на те же ключи в S3, ставит в зукипере аналогичную метку, -только со своим ID реплики, и работает с этим. Для первого фалйа из списка проверяет наличие первого ы3-объекта (просто наличие), если объект с таким именем найден, то все ок, если нет, то откат на старую версию. -(Сейчас есть еще код на случай наличия более одного диска S3, тогда перебирает все и если на каком-то файл найден, то использует его, но мы внутри команды MDB смотрим на такую конфигурацию как на странную. -Планируем ограничить функционал только случаем одного S3 диска.) +Приемник при получении ответа с send_s3_metadata=1 проверяет доступность по переданному ключу (первый объект checksums.txt) создает только файлики с идентичными меаданными, которые в итоге будут ссылаться на те же ключи в S3, ставит в зукипере аналогичную метку, +только со своим ID реплики, и работает с этим. -При желании удалить парт нода удаляет в Зукипере ключ `<путь к данным таблицы>/zero_copy_s3/shared/<некий ID парта>/`, потом получает все подключи `<путь к данным таблицы>/zero_copy_s3/shared/<некий ID парта>`. +При желании удалить парт нода удаляет в Зукипере ключ `<путь к данным таблицы>/zero_copy_s3/shared//`, потом получает все подключи `<путь к данным таблицы>/zero_copy_s3/shared/`. Если список не пустой, то считает, что данные использует другая нода и удаляет только локальные метаданные, если пустой, то удаляет и данные в S3. -При мерже если реузльтат будет на S3, нода ставит эфемерную метку в Zookeeper по пути `<путь к данным таблицы>/zero_copy_s3/merged/<имя нового парта>`. Если такая метка уже есть, то считает, что другая нода +При мерже если реузльтат будет на S3, нода ставит эфемерную метку в Zookeeper по пути `<путь к данным таблицы>/zero_copy_s3/merged/<имя нового парта>` (!! НЕ !!). Если такая метка уже есть, то считает, что другая нода уже помержила или мержит сейчас, и надо сделать fetch вместо мержа самой. +В конфиг добавлен флаг, по которому включается функционал нового протокола репликации - merge_tree->allow_s3_zero_copy_replication. Сейчас стоит в true - это времеменно, чтобы все тесты сейчас проходили с включенным флагом, перед финальным мержем надо не забыть заменить на false. + ## Костыли и недоработки, коих много * В качестве ID парта берется имя первого S3-ключа от файла checksums.txt. @@ -47,12 +50,11 @@ * Возможны все же дублирования партов. Пример - нода делает мерж, падает. Другая нода незавимо делает мерж, первая нода поднимается. В итоге есть две копии померженого парта. +* Тесты пока только самые базовые. + * ... много их. Честно. ## TODO, чего еще вообще не делалось -* Флаг в конфиге для включения функционала, по умолчанию будет выключен. - * Для гибридного хранилища сделать проверку и fetch при переезде парта с локального диска в S3. -* Тесты. diff --git a/src/Disks/DiskDecorator.h b/src/Disks/DiskDecorator.h index 4bc7879ffd3..71d75b92ab6 100644 --- a/src/Disks/DiskDecorator.h +++ b/src/Disks/DiskDecorator.h @@ -53,7 +53,7 @@ public: void sync(int fd) const override; const String getType() const override { return delegate->getType(); } String getUniqueId(const String & path) const override { return delegate->getUniqueId(path); } - bool checkFile(const String & path) const override { return delegate->checkFile(path); } + bool checkUniqueId(const String & id) const override { return delegate->checkUniqueId(id); } Executor & getExecutor() override; protected: diff --git a/src/Disks/IDisk.h b/src/Disks/IDisk.h index 915c6da5a21..e05b52c4a78 100644 --- a/src/Disks/IDisk.h +++ b/src/Disks/IDisk.h @@ -208,7 +208,7 @@ public: virtual String getUniqueId(const String & path) const { return path; } /// Check file, overrided for S3 only - virtual bool checkFile(const String & path) const { return exists(path); } + virtual bool checkUniqueId(const String & id) const { return exists(id); } /// Returns executor to perform asynchronous operations. virtual Executor & getExecutor() { return *executor; } diff --git a/src/Disks/S3/DiskS3.cpp b/src/Disks/S3/DiskS3.cpp index 7334a5b8a9b..01221d7c1a2 100644 --- a/src/Disks/S3/DiskS3.cpp +++ b/src/Disks/S3/DiskS3.cpp @@ -577,7 +577,7 @@ String DiskS3::getUniqueId(const String & path) const Metadata metadata(s3_root_path, metadata_path, path); String id; if (!metadata.s3_objects.empty()) - id = metadata.s3_objects[0].first; + id = metadata.s3_root_path + metadata.s3_objects[0].first; return id; } @@ -846,30 +846,23 @@ void DiskS3::shutdown() client->DisableRequestProcessing(); } -bool DiskS3::checkFile(const String & path) const +bool DiskS3::checkUniqueId(const String & id) const { - Metadata metadata(s3_root_path, metadata_path, path); - - /// empty s3_objects list for empty file - if (metadata.s3_objects.empty()) - return true; - - String object = metadata.s3_root_path + metadata.s3_objects[0].first; - + /// Check that we have right s3 and have access rights + /// Actually interprets id as s3 object name and checks if it exists Aws::S3::Model::ListObjectsRequest request; request.SetBucket(bucket); - request.SetPrefix(object); + request.SetPrefix(id); auto resp = client->ListObjects(request); throwIfError(resp); Aws::Vector object_list = resp.GetResult().GetContents(); - /// Should be only one object with name equal to prefix - if (object_list.size() != 1) + if (object_list.size() < 1) return false; - - if (object_list[0].GetKey() != object) - return false; - return true; + for (const auto & object : object_list) + if (object.GetKey() == id) + return true; + return false; } diff --git a/src/Disks/S3/DiskS3.h b/src/Disks/S3/DiskS3.h index 80752fa8253..43cec7838eb 100644 --- a/src/Disks/S3/DiskS3.h +++ b/src/Disks/S3/DiskS3.h @@ -116,7 +116,7 @@ public: String getUniqueId(const String & path) const override; - bool checkFile(const String & path) const override; + bool checkUniqueId(const String & path) const override; private: bool tryReserve(UInt64 bytes); diff --git a/src/Storages/MergeTree/DataPartsExchange.cpp b/src/Storages/MergeTree/DataPartsExchange.cpp index d2bd3c21173..2a1da0e0eaf 100644 --- a/src/Storages/MergeTree/DataPartsExchange.cpp +++ b/src/Storages/MergeTree/DataPartsExchange.cpp @@ -241,6 +241,9 @@ void Service::sendPartS3Metadata(const MergeTreeData::DataPartPtr & part, WriteB part->lockSharedData(zookeeper_path, replica_name, zookeeper); + String part_id = part->getUniqueId(); + writeStringBinary(part_id, out); + writeBinary(checksums.files.size(), out); for (const auto & it : checksums.files) { @@ -555,7 +558,22 @@ MergeTreeData::MutableDataPartPtr Fetcher::downloadPartToS3( if (disks_s3.empty()) throw Exception("No S3 disks anymore", ErrorCodes::LOGICAL_ERROR); - auto disk = disks_s3[0]; + String part_id; + readStringBinary(part_id, in); + + DiskPtr disk = disks_s3[0]; + + for (const auto & disk_ : disks_s3) + { + if (disk_->checkUniqueId(part_id)) + { + disk = disk_; + break; + } + } + + if (!disk) + throw Exception("Can't find S3 disk", ErrorCodes::S3_ERROR); static const String TMP_PREFIX = "tmp_fetch_"; String tmp_prefix = tmp_prefix_.empty() ? TMP_PREFIX : tmp_prefix_; @@ -612,50 +630,6 @@ MergeTreeData::MutableDataPartPtr Fetcher::downloadPartToS3( ErrorCodes::CHECKSUM_DOESNT_MATCH); } } - - if (!i) - { /// Check access for first s3 object of first file - if (!disk->checkFile(data_path)) - { /// Wrong S3 disk - Poco::File metadata(metadata_file); - - size_t disk_id = 1; - while (true) - { - if (disk_id >= disks_s3.size()) - { /// No more S3 disks - disk->removeSharedRecursive(part_download_path, true); - /// After catch this exception replication continues with full data copy - throw Exception("Can't find S3 drive for shared data", ErrorCodes::S3_ERROR); - } - - /// Try next S3 disk - auto next_disk = disks_s3[disk_id]; - - auto next_volume = std::make_shared("volume_" + part_name, next_disk); - MergeTreeData::MutableDataPartPtr next_new_data_part = data.createPart(part_name, next_volume, part_relative_path); - - next_disk->createDirectories(part_download_path); - - String next_data_path = next_new_data_part->getFullRelativePath() + file_name; - String next_metadata_file = fullPath(next_disk, next_data_path); - metadata.copyTo(next_metadata_file); - if (next_disk->checkFile(next_data_path)) - { /// Right disk found - disk->removeSharedRecursive(part_download_path, true); - disk = next_disk; - volume = next_volume; - data_path = next_data_path; - new_data_part = next_new_data_part; - break; - } - - /// Wrong disk again - next_disk->removeSharedRecursive(part_download_path, true); - ++disk_id; - } - } - } } assertEOF(in); diff --git a/src/Storages/MergeTree/IMergeTreeDataPart.cpp b/src/Storages/MergeTree/IMergeTreeDataPart.cpp index be2f88e74e5..85c2d5e4ab4 100644 --- a/src/Storages/MergeTree/IMergeTreeDataPart.cpp +++ b/src/Storages/MergeTree/IMergeTreeDataPart.cpp @@ -1095,17 +1095,24 @@ bool IMergeTreeDataPart::checkAllTTLCalculated(const StorageMetadataPtr & metada return true; } -void IMergeTreeDataPart::lockSharedData(const String & zookeeper_path, const String & replica_name, zkutil::ZooKeeperPtr zookeeper) const +String IMergeTreeDataPart::getUniqueId() const { + String id; + auto disk = volume->getDisk(); - if (disk->getType() != "s3") - return; - - String id = disk->getUniqueId(getFullRelativePath() + "checksums.txt"); + if (disk->getType() == "s3") + id = disk->getUniqueId(getFullRelativePath() + "checksums.txt"); if (id.empty()) - throw Exception("Can't lock part on S3 storage", ErrorCodes::LOGICAL_ERROR); + throw Exception("Can't get unique S3 object", ErrorCodes::LOGICAL_ERROR); + + return id; +} + +void IMergeTreeDataPart::lockSharedData(const String & zookeeper_path, const String & replica_name, zkutil::ZooKeeperPtr zookeeper) const +{ + String id = getUniqueId(); String zookeeper_node = zookeeper_path + "/zero_copy_s3/shared/" + id + "/" + replica_name; @@ -1117,15 +1124,7 @@ void IMergeTreeDataPart::lockSharedData(const String & zookeeper_path, const Str bool IMergeTreeDataPart::unlockSharedData(const String & zookeeper_path, const String & replica_name, zkutil::ZooKeeperPtr zookeeper) const { - auto disk = volume->getDisk(); - - if (disk->getType() != "s3") - return true; - - String id = disk->getUniqueId(getFullRelativePath() + "checksums.txt"); - - if (id.empty()) - return true; + String id = getUniqueId(); String zookeeper_part_node = zookeeper_path + "/zero_copy_s3/shared/" + id; String zookeeper_node = zookeeper_part_node + "/" + replica_name; diff --git a/src/Storages/MergeTree/IMergeTreeDataPart.h b/src/Storages/MergeTree/IMergeTreeDataPart.h index 8d21f5856fc..f948cbaa18c 100644 --- a/src/Storages/MergeTree/IMergeTreeDataPart.h +++ b/src/Storages/MergeTree/IMergeTreeDataPart.h @@ -359,6 +359,8 @@ public: /// part creation (using alter query with materialize_ttl setting). bool checkAllTTLCalculated(const StorageMetadataPtr & metadata_snapshot) const; + String getUniqueId() const; + /// Lock part in zookeeper for use common S3 data in several nodes void lockSharedData(const String & zookeeper_path, const String & replica_name, zkutil::ZooKeeperPtr zookeeper) const; From eba98b04b0322f02139f7553c2fab61b84a514e8 Mon Sep 17 00:00:00 2001 From: Anton Ivashkin Date: Thu, 14 Jan 2021 19:26:56 +0300 Subject: [PATCH 011/544] Zero copy replication over S3: Hybrid storage support --- S3ZeroCopyReplication.md | 17 +- src/Storages/MergeTree/DataPartsExchange.cpp | 26 ++- src/Storages/MergeTree/DataPartsExchange.h | 3 +- src/Storages/MergeTree/IMergeTreeDataPart.cpp | 218 ++++++++++++++++-- src/Storages/MergeTree/IMergeTreeDataPart.h | 11 +- src/Storages/MergeTree/MergeTreeData.cpp | 5 +- src/Storages/MergeTree/MergeTreeData.h | 3 +- .../MergeTree/MergeTreeDataMergerMutator.cpp | 1 + .../MergeTree/MergedBlockOutputStream.cpp | 1 + .../MergeTree/ReplicatedMergeTreeLogEntry.h | 40 ++-- src/Storages/StorageReplicatedMergeTree.cpp | 176 ++++++++++---- src/Storages/StorageReplicatedMergeTree.h | 24 +- 12 files changed, 431 insertions(+), 94 deletions(-) diff --git a/S3ZeroCopyReplication.md b/S3ZeroCopyReplication.md index 22c01caa90c..bfb39addcd2 100644 --- a/S3ZeroCopyReplication.md +++ b/S3ZeroCopyReplication.md @@ -18,9 +18,14 @@ Применик перед запросом смотрит, будет ли хранить данные в S3. Проверка сейчас кривая - если в сторадже есть S3, то считаем, что будет S3. Если да S3, то отсылает в запросе send_s3_metadata=1. -Источник при получении такого запроса смотрит, лежит ли парт на S3. Если да, то в Зукипере ставит метку по пути `<путь к данным таблицы>/zero_copy_s3/shared//`, +Источник при получении такого запроса смотрит, лежит ли парт на S3. Если да, то в Зукипере ставит метку по пути `<путь к данным таблицы>/zero_copy_s3/shared/<имя парта>//<Путь парта>/`, ставит в ответ куку send_s3_metadata=1 и вместо файлов с данными отсылает только файлы метаданных. +Путь получился сложным, потому что требуется +* по имени парта получить, на каких репликах он уже есть на S3 (нужно для гибридного хранилища) +* по уникальному пути понимать, используелся ли эта копия парта другими репликами +* для павильного времени жизни лока различать лок основного варианта (all_0_0_0) от временного (tmp_fetch_all_0_0_0) + Приемник при получении ответа с send_s3_metadata=1 проверяет доступность по переданному ключу (первый объект checksums.txt) создает только файлики с идентичными меаданными, которые в итоге будут ссылаться на те же ключи в S3, ставит в зукипере аналогичную метку, только со своим ID реплики, и работает с этим. @@ -30,14 +35,14 @@ При мерже если реузльтат будет на S3, нода ставит эфемерную метку в Zookeeper по пути `<путь к данным таблицы>/zero_copy_s3/merged/<имя нового парта>` (!! НЕ !!). Если такая метка уже есть, то считает, что другая нода уже помержила или мержит сейчас, и надо сделать fetch вместо мержа самой. +В гибридном хранилище если парт переносится на S3, нода через ZK проверяет, нет был ли парт перенесен другой нодой, если был, то делает fetch (модифицированный по сравнению с обычным fetch'ем). + В конфиг добавлен флаг, по которому включается функционал нового протокола репликации - merge_tree->allow_s3_zero_copy_replication. Сейчас стоит в true - это времеменно, чтобы все тесты сейчас проходили с включенным флагом, перед финальным мержем надо не забыть заменить на false. ## Костыли и недоработки, коих много * В качестве ID парта берется имя первого S3-ключа от файла checksums.txt. -* Не нашел удобного способа прокидывать в коде зукипер, прокинул хадркодом. - * При удалении класс диска ничего не знает про парты, прокинул флаг, что надо оставлять данные в S3 параметром, это очень криво получилось. * Возможна гонка, если источник отошлет метаданные про парт и тут же решит его удалить до того, как приемник поставит в зукипер пометку. @@ -52,9 +57,5 @@ * Тесты пока только самые базовые. -* ... много их. Честно. - -## TODO, чего еще вообще не делалось - -* Для гибридного хранилища сделать проверку и fetch при переезде парта с локального диска в S3. +* Для гибридного хранилища если две ноды решают одновременно перенести парт на S3, обе проверяют, что его там еще нет и обе переносят. diff --git a/src/Storages/MergeTree/DataPartsExchange.cpp b/src/Storages/MergeTree/DataPartsExchange.cpp index f8f5bfb5a3b..884dd22c295 100644 --- a/src/Storages/MergeTree/DataPartsExchange.cpp +++ b/src/Storages/MergeTree/DataPartsExchange.cpp @@ -267,7 +267,7 @@ void Service::sendPartS3Metadata(const MergeTreeData::DataPartPtr & part, WriteB if (disk->getType() != "s3") throw Exception("S3 disk is not S3 anymore", ErrorCodes::LOGICAL_ERROR); - part->lockSharedData(zookeeper_path, replica_name, zookeeper); + part->lockSharedData(); String part_id = part->getUniqueId(); writeStringBinary(part_id, out); @@ -327,7 +327,8 @@ MergeTreeData::MutableDataPartPtr Fetcher::fetchPart( const String & interserver_scheme, bool to_detached, const String & tmp_prefix_, - bool try_use_s3_copy) + bool try_use_s3_copy, + const DiskPtr disk_s3) { if (blocker.isCancelled()) throw Exception("Fetching of part was cancelled", ErrorCodes::ABORTED); @@ -348,6 +349,9 @@ MergeTreeData::MutableDataPartPtr Fetcher::fetchPart( {"compress", "false"} }); + if (try_use_s3_copy && disk_s3 && disk_s3->getType() != "s3") + throw Exception("Try to fetch shared s3 part on non-s3 disk", ErrorCodes::LOGICAL_ERROR); + Disks disks_s3; if (!data_settings->allow_s3_zero_copy_replication) @@ -355,9 +359,15 @@ MergeTreeData::MutableDataPartPtr Fetcher::fetchPart( if (try_use_s3_copy) { - disks_s3 = data.getDisksByType("s3"); - if (disks_s3.empty()) - try_use_s3_copy = false; + if (disk_s3) + disks_s3.push_back(disk_s3); + else + { + disks_s3 = data.getDisksByType("s3"); + + if (disks_s3.empty()) + try_use_s3_copy = false; + } } if (try_use_s3_copy) @@ -405,6 +415,10 @@ MergeTreeData::MutableDataPartPtr Fetcher::fetchPart( if (part_type == "InMemory") throw Exception("Got 'send_s3_metadata' cookie for in-memory partition", ErrorCodes::LOGICAL_ERROR); + UUID part_uuid = UUIDHelpers::Nil; + if (server_protocol_version >= REPLICATION_PROTOCOL_VERSION_WITH_PARTS_UUID) + readUUIDText(part_uuid, in); + try { return downloadPartToS3(part_name, replica_path, to_detached, tmp_prefix_, std::move(disks_s3), in); @@ -680,7 +694,7 @@ MergeTreeData::MutableDataPartPtr Fetcher::downloadPartToS3( new_data_part->modification_time = time(nullptr); new_data_part->loadColumnsChecksumsIndexes(true, false); - new_data_part->lockSharedData(zookeeper_path, replica_name, zookeeper); + new_data_part->lockSharedData(); return new_data_part; } diff --git a/src/Storages/MergeTree/DataPartsExchange.h b/src/Storages/MergeTree/DataPartsExchange.h index c5bc891b550..f0297aa1d28 100644 --- a/src/Storages/MergeTree/DataPartsExchange.h +++ b/src/Storages/MergeTree/DataPartsExchange.h @@ -81,7 +81,8 @@ public: const String & interserver_scheme, bool to_detached = false, const String & tmp_prefix_ = "", - bool try_use_s3_copy = true); + bool try_use_s3_copy = true, + const DiskPtr disk_s3 = nullptr); /// You need to stop the data transfer. ActionBlocker blocker; diff --git a/src/Storages/MergeTree/IMergeTreeDataPart.cpp b/src/Storages/MergeTree/IMergeTreeDataPart.cpp index 5f017972a47..f4635208cda 100644 --- a/src/Storages/MergeTree/IMergeTreeDataPart.cpp +++ b/src/Storages/MergeTree/IMergeTreeDataPart.cpp @@ -9,6 +9,7 @@ #include #include #include +#include #include #include #include @@ -20,6 +21,7 @@ namespace DB { + namespace ErrorCodes { extern const int DIRECTORY_ALREADY_EXISTS; @@ -773,7 +775,8 @@ void IMergeTreeDataPart::loadColumns(bool require) { /// We can get list of columns only from columns.txt in compact parts. if (require || part_type == Type::COMPACT) - throw Exception("No columns.txt in part " + name, ErrorCodes::NO_FILE_IN_DATA_PART); + throw Exception("No columns.txt in part " + name + ", expected path " + path + " on drive " + volume->getDisk()->getName(), + ErrorCodes::NO_FILE_IN_DATA_PART); /// If there is no file with a list of columns, write it down. for (const NameAndTypePair & column : metadata_snapshot->getColumns().getAllPhysical()) @@ -855,7 +858,10 @@ void IMergeTreeDataPart::renameTo(const String & new_relative_path, bool remove_ volume->getDisk()->setLastModified(from, Poco::Timestamp::fromEpochTime(time(nullptr))); volume->getDisk()->moveFile(from, to); + String old_relative_path = relative_path; relative_path = new_relative_path; + lockSharedData(); + unlockSharedData(old_relative_path); } @@ -1010,7 +1016,15 @@ void IMergeTreeDataPart::makeCloneOnDisk(const DiskPtr & disk, const String & di } disk->createDirectories(path_to_clone); - volume->getDisk()->copy(getFullRelativePath(), disk, path_to_clone); + bool is_fetched = false; + + if (disk->getType() == "s3") + { + is_fetched = tryToFetchIfShared(disk, path_to_clone + "/" + name); + } + + if (!is_fetched) + volume->getDisk()->copy(getFullRelativePath(), disk, path_to_clone); volume->getDisk()->removeIfExists(path_to_clone + '/' + DELETE_ON_DESTROY_MARKER_FILE_NAME); } @@ -1148,38 +1162,212 @@ String IMergeTreeDataPart::getUniqueId() const return id; } -void IMergeTreeDataPart::lockSharedData(const String & zookeeper_path, const String & replica_name, zkutil::ZooKeeperPtr zookeeper) const +void IMergeTreeDataPart::lockSharedData() const { + if (!volume) + return; + DiskPtr disk = volume->getDisk(); + if (!disk) + return; + if (disk->getType() != "s3") + return; + + const StorageReplicatedMergeTree *replicated_storage = dynamic_cast(&storage); + if (!replicated_storage) + return; + + StorageReplicatedMergeTree::ZooKeeperAccessData zk = replicated_storage->getZooKeeperAccessData(); + if (!zk.zookeeper) + return; + String id = getUniqueId(); + boost::replace_all(id, "/", "_"); + String norm_path = relative_path; + boost::replace_all(norm_path, "/", "_"); - String zookeeper_node = zookeeper_path + "/zero_copy_s3/shared/" + id + "/" + replica_name; + String zookeeper_node = zk.zookeeper_path + "/zero_copy_s3/shared/" + name + "/" + id + "/" + norm_path + "/" + zk.replica_name; - LOG_TRACE(storage.log, "Set zookeeper lock {}", id); + LOG_TRACE(storage.log, "Set zookeeper lock {}", zookeeper_node); - zookeeper->createAncestors(zookeeper_node); - zookeeper->createIfNotExists(zookeeper_node, "lock"); + zk.zookeeper->createAncestors(zookeeper_node); + zk.zookeeper->createIfNotExists(zookeeper_node, "lock"); } -bool IMergeTreeDataPart::unlockSharedData(const String & zookeeper_path, const String & replica_name, zkutil::ZooKeeperPtr zookeeper) const +bool IMergeTreeDataPart::unlockSharedData() const { + return unlockSharedData(relative_path); +} + +bool IMergeTreeDataPart::unlockSharedData(const String & path) const +{ + if (!volume) + return true; + DiskPtr disk = volume->getDisk(); + if (!disk) + return true; + if (disk->getType() != "s3") + return true; + + const StorageReplicatedMergeTree *replicated_storage = dynamic_cast(&storage); + if (!replicated_storage) + return true; + + StorageReplicatedMergeTree::ZooKeeperAccessData zk = replicated_storage->getZooKeeperAccessData(); + if (!zk.zookeeper) + return true; + String id = getUniqueId(); + boost::replace_all(id, "/", "_"); + String norm_path = path; + boost::replace_all(norm_path, "/", "_"); - String zookeeper_part_node = zookeeper_path + "/zero_copy_s3/shared/" + id; - String zookeeper_node = zookeeper_part_node + "/" + replica_name; + String zookeeper_part_node = zk.zookeeper_path + "/zero_copy_s3/shared/" + name; + String zookeeper_part_uniq_node = zookeeper_part_node + "/" + id; + String zookeeper_part_path_node = zookeeper_part_uniq_node + "/" + norm_path; + String zookeeper_node = zookeeper_part_path_node + "/" + zk.replica_name; - LOG_TRACE(storage.log, "Remove zookeeper lock for {}", id); + LOG_TRACE(storage.log, "Remove zookeeper lock {}", zookeeper_node); - zookeeper->remove(zookeeper_node); + zk.zookeeper->tryRemove(zookeeper_node); Strings children; - zookeeper->tryGetChildren(zookeeper_part_node, children); + zk.zookeeper->tryGetChildren(zookeeper_part_path_node, children); + if (!children.empty()) + { + LOG_TRACE(storage.log, "Found zookeper locks for {}", zookeeper_part_path_node); + return false; + } + + zk.zookeeper->tryRemove(zookeeper_part_path_node); + + children.clear(); + zk.zookeeper->tryGetChildren(zookeeper_part_uniq_node, children); if (!children.empty()) { - LOG_TRACE(storage.log, "Found zookeper locks for {}", id); + LOG_TRACE(storage.log, "Found zookeper locks for {}", zookeeper_part_uniq_node); + return false; } - return children.empty(); + zk.zookeeper->tryRemove(zookeeper_part_uniq_node); + + /// Even when we have lock with same part name, but with different uniq, we can remove files on S3 + children.clear(); + zk.zookeeper->tryGetChildren(zookeeper_part_node, children); + if (children.empty()) + /// Cleanup after last uniq removing + zk.zookeeper->tryRemove(zookeeper_part_node); + + return true; +} + +String IMergeTreeDataPart::getSharedDataReplica( + const String & zookeeper_path, + zkutil::ZooKeeperPtr zookeeper, + const String & replica_name) const +{ + String norm_path = relative_path; + boost::replace_all(norm_path, "/", "_"); + String zookeeper_part_node = zookeeper_path + "/zero_copy_s3/shared/" + name; + + Strings ids; + zookeeper->tryGetChildren(zookeeper_part_node, ids); + + Strings replicas; + for (const auto & id : ids) + { + String zookeeper_part_uniq_node = zookeeper_part_node + "/" + id; + Strings paths; + zookeeper->tryGetChildren(zookeeper_part_uniq_node, paths); + for (const auto &path : paths) + { + String zookeeper_node = zookeeper_part_uniq_node + "/" + path; + Strings id_replicas; + zookeeper->tryGetChildren(zookeeper_node, id_replicas); + LOG_TRACE(storage.log, "Found zookeper replicas for {}: {}", zookeeper_node, id_replicas.size()); + replicas.insert(replicas.end(), id_replicas.begin(), id_replicas.end()); + } + } + + LOG_TRACE(storage.log, "Found zookeper replicas for part {}: {}", name, replicas.size()); + + String best_replica; + Strings active_replicas; + + /// TODO: Move best replica choose in common method (here is the same code as in StorageReplicatedMergeTree::fetchPartition) + + /// Leave only active replicas. + active_replicas.reserve(replicas.size()); + + for (const String & replica : replicas) + if ((replica != replica_name) && (zookeeper->exists(zookeeper_path + "/replicas/" + replica + "/is_active"))) + active_replicas.push_back(replica); + + LOG_TRACE(storage.log, "Found zookeper active replicas for part {}: {}", name, active_replicas.size()); + + if (active_replicas.empty()) + return best_replica; + + /** You must select the best (most relevant) replica. + * This is a replica with the maximum `log_pointer`, then with the minimum `queue` size. + * NOTE This is not exactly the best criteria. It does not make sense to download old partitions, + * and it would be nice to be able to choose the replica closest by network. + * NOTE Of course, there are data races here. You can solve it by retrying. + */ + Int64 max_log_pointer = -1; + UInt64 min_queue_size = std::numeric_limits::max(); + + for (const String & replica : active_replicas) + { + String current_replica_path = zookeeper_path + "/replicas/" + replica; + + String log_pointer_str = zookeeper->get(current_replica_path + "/log_pointer"); + Int64 log_pointer = log_pointer_str.empty() ? 0 : parse(log_pointer_str); + + Coordination::Stat stat; + zookeeper->get(current_replica_path + "/queue", &stat); + size_t queue_size = stat.numChildren; + + if (log_pointer > max_log_pointer + || (log_pointer == max_log_pointer && queue_size < min_queue_size)) + { + max_log_pointer = log_pointer; + min_queue_size = queue_size; + best_replica = replica; + } + } + + return best_replica; +} + +bool IMergeTreeDataPart::tryToFetchIfShared(const DiskPtr & disk, const String & path) const +{ + const StorageReplicatedMergeTree *replicated_storage = dynamic_cast(&storage); + if (!replicated_storage) + return false; + + StorageReplicatedMergeTree::ZooKeeperAccessData zk = replicated_storage->getZooKeeperAccessData(); + if (!zk.zookeeper) + return false; + + String replica = getSharedDataReplica(zk.zookeeper_path, zk.zookeeper, zk.replica_name); + + /// We can't fetch part when none replicas have this part on S3 + if (replica.empty()) + return false; + + ReplicatedMergeTreeLogEntry log_entry; + log_entry.type = ReplicatedMergeTreeLogEntry::FETCH_SHARED_PART; + log_entry.source_replica = replica; + log_entry.new_part_name = name;//part_name; + log_entry.create_time = 0;//part_create_time; + log_entry.disk = disk; + log_entry.path = path; + + /// TODO: !!! Fix const usage !!! + StorageReplicatedMergeTree *replicated_storage_nc = const_cast(replicated_storage); + + return replicated_storage_nc->executeFetchShared(log_entry); } bool isCompactPart(const MergeTreeDataPartPtr & data_part) diff --git a/src/Storages/MergeTree/IMergeTreeDataPart.h b/src/Storages/MergeTree/IMergeTreeDataPart.h index 746d95fe78e..cfe3d7da263 100644 --- a/src/Storages/MergeTree/IMergeTreeDataPart.h +++ b/src/Storages/MergeTree/IMergeTreeDataPart.h @@ -369,12 +369,13 @@ public: String getUniqueId() const; /// Lock part in zookeeper for use common S3 data in several nodes - void lockSharedData(const String & zookeeper_path, const String & replica_name, zkutil::ZooKeeperPtr zookeeper) const; + void lockSharedData() const; /// Unlock common S3 data part in zookeeper /// Return true if data unlocked /// Return false if data is still used by another node - bool unlockSharedData(const String & zookeeper_path, const String & replica_name, zkutil::ZooKeeperPtr zookeeper) const; + bool unlockSharedData() const; + bool unlockSharedData(const String & path) const; protected: @@ -439,6 +440,12 @@ private: /// Found column without specific compression and return codec /// for this column with default parameters. CompressionCodecPtr detectDefaultCompressionCodec() const; + + /// Fetch part only if some replica has it on shared storage like S3 + bool tryToFetchIfShared(const DiskPtr & disk, const String & path) const; + + /// Get best replica having this partition on S3 + String getSharedDataReplica(const String & zookeeper_path, zkutil::ZooKeeperPtr zookeeper, const String & replica_name) const; }; using MergeTreeDataPartState = IMergeTreeDataPart::State; diff --git a/src/Storages/MergeTree/MergeTreeData.cpp b/src/Storages/MergeTree/MergeTreeData.cpp index c3a599665bb..37f7187585c 100644 --- a/src/Storages/MergeTree/MergeTreeData.cpp +++ b/src/Storages/MergeTree/MergeTreeData.cpp @@ -1833,7 +1833,8 @@ MergeTreeData::DataPartsVector MergeTreeData::getActivePartsToReplace( const MergeTreePartInfo & new_part_info, const String & new_part_name, DataPartPtr & out_covering_part, - DataPartsLock & /* data_parts_lock */) const + DataPartsLock & /* data_parts_lock */, + bool allow_duplicate) const { /// Parts contained in the part are consecutive in data_parts, intersecting the insertion place for the part itself. auto it_middle = data_parts_by_state_and_info.lower_bound(DataPartStateAndInfo{DataPartState::Committed, new_part_info}); @@ -1867,7 +1868,7 @@ MergeTreeData::DataPartsVector MergeTreeData::getActivePartsToReplace( DataPartIteratorByStateAndInfo end = it_middle; while (end != committed_parts_range.end()) { - if ((*end)->info == new_part_info) + if ((*end)->info == new_part_info && !allow_duplicate) throw Exception("Unexpected duplicate part " + (*end)->getNameWithState() + ". It is a bug.", ErrorCodes::LOGICAL_ERROR); if (!new_part_info.contains((*end)->info)) diff --git a/src/Storages/MergeTree/MergeTreeData.h b/src/Storages/MergeTree/MergeTreeData.h index e65d486d46f..53902688f1f 100644 --- a/src/Storages/MergeTree/MergeTreeData.h +++ b/src/Storages/MergeTree/MergeTreeData.h @@ -870,7 +870,8 @@ protected: const MergeTreePartInfo & new_part_info, const String & new_part_name, DataPartPtr & out_covering_part, - DataPartsLock & data_parts_lock) const; + DataPartsLock & data_parts_lock, + bool allow_duplicate = false) const; /// Checks whether the column is in the primary key, possibly wrapped in a chain of functions with single argument. bool isPrimaryOrMinMaxKeyColumnPossiblyWrappedInFunctions(const ASTPtr & node, const StorageMetadataPtr & metadata_snapshot) const; diff --git a/src/Storages/MergeTree/MergeTreeDataMergerMutator.cpp b/src/Storages/MergeTree/MergeTreeDataMergerMutator.cpp index 1065b992396..807d1e9eed2 100644 --- a/src/Storages/MergeTree/MergeTreeDataMergerMutator.cpp +++ b/src/Storages/MergeTree/MergeTreeDataMergerMutator.cpp @@ -1876,6 +1876,7 @@ void MergeTreeDataMergerMutator::finalizeMutatedPart( MergeTreeData::DataPart::calculateTotalSizeOnDisk(new_data_part->volume->getDisk(), new_data_part->getFullRelativePath())); new_data_part->default_codec = codec; new_data_part->calculateColumnsSizesOnDisk(); + new_data_part->lockSharedData(); } diff --git a/src/Storages/MergeTree/MergedBlockOutputStream.cpp b/src/Storages/MergeTree/MergedBlockOutputStream.cpp index 00a4c37c60d..255526eca11 100644 --- a/src/Storages/MergeTree/MergedBlockOutputStream.cpp +++ b/src/Storages/MergeTree/MergedBlockOutputStream.cpp @@ -126,6 +126,7 @@ void MergedBlockOutputStream::writeSuffixAndFinalizePart( new_part->calculateColumnsSizesOnDisk(); if (default_codec != nullptr) new_part->default_codec = default_codec; + new_part->lockSharedData(); } void MergedBlockOutputStream::finalizePartOnDisk( diff --git a/src/Storages/MergeTree/ReplicatedMergeTreeLogEntry.h b/src/Storages/MergeTree/ReplicatedMergeTreeLogEntry.h index 4b384171dde..e9e3d15c5ff 100644 --- a/src/Storages/MergeTree/ReplicatedMergeTreeLogEntry.h +++ b/src/Storages/MergeTree/ReplicatedMergeTreeLogEntry.h @@ -6,6 +6,7 @@ #include #include #include +#include #include #include @@ -29,29 +30,31 @@ struct ReplicatedMergeTreeLogEntryData { enum Type { - EMPTY, /// Not used. - GET_PART, /// Get the part from another replica. - MERGE_PARTS, /// Merge the parts. - DROP_RANGE, /// Delete the parts in the specified partition in the specified number range. - CLEAR_COLUMN, /// NOTE: Deprecated. Drop specific column from specified partition. - CLEAR_INDEX, /// NOTE: Deprecated. Drop specific index from specified partition. - REPLACE_RANGE, /// Drop certain range of partitions and replace them by new ones - MUTATE_PART, /// Apply one or several mutations to the part. - ALTER_METADATA, /// Apply alter modification according to global /metadata and /columns paths + EMPTY, /// Not used. + GET_PART, /// Get the part from another replica. + MERGE_PARTS, /// Merge the parts. + DROP_RANGE, /// Delete the parts in the specified partition in the specified number range. + CLEAR_COLUMN, /// NOTE: Deprecated. Drop specific column from specified partition. + CLEAR_INDEX, /// NOTE: Deprecated. Drop specific index from specified partition. + REPLACE_RANGE, /// Drop certain range of partitions and replace them by new ones + MUTATE_PART, /// Apply one or several mutations to the part. + ALTER_METADATA, /// Apply alter modification according to global /metadata and /columns paths + FETCH_SHARED_PART, /// Get the part from other replica only if it on shared S3 storade }; static String typeToString(Type type) { switch (type) { - case ReplicatedMergeTreeLogEntryData::GET_PART: return "GET_PART"; - case ReplicatedMergeTreeLogEntryData::MERGE_PARTS: return "MERGE_PARTS"; - case ReplicatedMergeTreeLogEntryData::DROP_RANGE: return "DROP_RANGE"; - case ReplicatedMergeTreeLogEntryData::CLEAR_COLUMN: return "CLEAR_COLUMN"; - case ReplicatedMergeTreeLogEntryData::CLEAR_INDEX: return "CLEAR_INDEX"; - case ReplicatedMergeTreeLogEntryData::REPLACE_RANGE: return "REPLACE_RANGE"; - case ReplicatedMergeTreeLogEntryData::MUTATE_PART: return "MUTATE_PART"; - case ReplicatedMergeTreeLogEntryData::ALTER_METADATA: return "ALTER_METADATA"; + case ReplicatedMergeTreeLogEntryData::GET_PART: return "GET_PART"; + case ReplicatedMergeTreeLogEntryData::MERGE_PARTS: return "MERGE_PARTS"; + case ReplicatedMergeTreeLogEntryData::DROP_RANGE: return "DROP_RANGE"; + case ReplicatedMergeTreeLogEntryData::CLEAR_COLUMN: return "CLEAR_COLUMN"; + case ReplicatedMergeTreeLogEntryData::CLEAR_INDEX: return "CLEAR_INDEX"; + case ReplicatedMergeTreeLogEntryData::REPLACE_RANGE: return "REPLACE_RANGE"; + case ReplicatedMergeTreeLogEntryData::MUTATE_PART: return "MUTATE_PART"; + case ReplicatedMergeTreeLogEntryData::ALTER_METADATA: return "ALTER_METADATA"; + case ReplicatedMergeTreeLogEntryData::FETCH_SHARED_PART: return "FETCH_SHARED_PART"; default: throw Exception("Unknown log entry type: " + DB::toString(type), ErrorCodes::LOGICAL_ERROR); } @@ -191,6 +194,9 @@ struct ReplicatedMergeTreeLogEntry : public ReplicatedMergeTreeLogEntryData, std std::condition_variable execution_complete; /// Awake when currently_executing becomes false. static Ptr parse(const String & s, const Coordination::Stat & stat); + + DiskPtr disk; + String path; }; using ReplicatedMergeTreeLogEntryPtr = std::shared_ptr; diff --git a/src/Storages/StorageReplicatedMergeTree.cpp b/src/Storages/StorageReplicatedMergeTree.cpp index 977a485f758..59312737a39 100644 --- a/src/Storages/StorageReplicatedMergeTree.cpp +++ b/src/Storages/StorageReplicatedMergeTree.cpp @@ -1891,6 +1891,60 @@ bool StorageReplicatedMergeTree::executeFetch(LogEntry & entry) } +bool StorageReplicatedMergeTree::executeFetchShared(ReplicatedMergeTreeLogEntry & entry) +{ + if (entry.type != LogEntry::FETCH_SHARED_PART) + { + throw Exception("Wrong entry.type in executeFetchShared", ErrorCodes::LOGICAL_ERROR); + } + + if (entry.source_replica.empty()) + { + LOG_INFO(log, "No active replica has part {} on S3.", entry.new_part_name); + return false; + } + + const auto storage_settings_ptr = getSettings(); + auto metadata_snapshot = getInMemoryMetadataPtr(); + + static std::atomic_uint total_fetches {0}; + if (storage_settings_ptr->replicated_max_parallel_fetches && total_fetches >= storage_settings_ptr->replicated_max_parallel_fetches) + { + throw Exception("Too many total fetches from replicas, maximum: " + storage_settings_ptr->replicated_max_parallel_fetches.toString(), + ErrorCodes::TOO_MANY_FETCHES); + } + + ++total_fetches; + SCOPE_EXIT({--total_fetches;}); + + if (storage_settings_ptr->replicated_max_parallel_fetches_for_table + && current_table_fetches >= storage_settings_ptr->replicated_max_parallel_fetches_for_table) + { + throw Exception("Too many fetches from replicas for table, maximum: " + storage_settings_ptr->replicated_max_parallel_fetches_for_table.toString(), + ErrorCodes::TOO_MANY_FETCHES); + } + + ++current_table_fetches; + SCOPE_EXIT({--current_table_fetches;}); + + try + { + if (!fetchPart(entry.new_part_name, metadata_snapshot, zookeeper_path + "/replicas/" + entry.source_replica, false, entry.quorum, + nullptr, true, entry.disk, entry.path)) + return false; + } + catch (Exception & e) + { + if (e.code() == ErrorCodes::RECEIVED_ERROR_TOO_MANY_REQUESTS) + e.addMessage("Too busy replica. Will try later."); + tryLogCurrentException(log, __PRETTY_FUNCTION__); + throw; + } + + return true; +} + + void StorageReplicatedMergeTree::executeDropRange(const LogEntry & entry) { auto drop_range_info = MergeTreePartInfo::fromPartName(entry.new_part_name, format_version); @@ -3133,6 +3187,29 @@ String StorageReplicatedMergeTree::findReplicaHavingPart(const String & part_nam return {}; } +String StorageReplicatedMergeTree::findReplicaHavingSharedPart(const String & part_name, bool active) +{ + auto zookeeper = getZooKeeper(); + Strings replicas = zookeeper->getChildren(zookeeper_path + "/replicas"); + + /// Select replicas in uniformly random order. + std::shuffle(replicas.begin(), replicas.end(), thread_local_rng); + + for (const String & replica : replicas) + { + /// We don't interested in ourself. + if (replica == replica_name) + continue; + + if (checkReplicaHavePart(replica, part_name) && + (!active || zookeeper->exists(zookeeper_path + "/replicas/" + replica + "/is_active"))) + return replica; + + /// Obviously, replica could become inactive or even vanish after return from this method. + } + + return {}; +} String StorageReplicatedMergeTree::findReplicaHavingCoveringPart(LogEntry & entry, bool active) { @@ -3330,7 +3407,6 @@ void StorageReplicatedMergeTree::updateQuorum(const String & part_name, bool is_ } } - void StorageReplicatedMergeTree::cleanLastPartNode(const String & partition_id) { auto zookeeper = getZooKeeper(); @@ -3382,7 +3458,6 @@ void StorageReplicatedMergeTree::cleanLastPartNode(const String & partition_id) } } - bool StorageReplicatedMergeTree::partIsInsertingWithParallelQuorum(const MergeTreePartInfo & part_info) const { auto zookeeper = getZooKeeper(); @@ -3411,7 +3486,8 @@ bool StorageReplicatedMergeTree::partIsLastQuorumPart(const MergeTreePartInfo & } bool StorageReplicatedMergeTree::fetchPart(const String & part_name, const StorageMetadataPtr & metadata_snapshot, - const String & source_replica_path, bool to_detached, size_t quorum, zkutil::ZooKeeper::Ptr zookeeper_) + const String & source_replica_path, bool to_detached, size_t quorum, zkutil::ZooKeeper::Ptr zookeeper_, bool replace_exists, + DiskPtr replaced_disk, String replaced_part_path) { auto zookeeper = zookeeper_ ? zookeeper_ : getZooKeeper(); const auto part_info = MergeTreePartInfo::fromPartName(part_name, format_version); @@ -3461,6 +3537,8 @@ bool StorageReplicatedMergeTree::fetchPart(const String & part_name, const Stora }; DataPartPtr part_to_clone; + + if (!replace_exists) { /// If the desired part is a result of a part mutation, try to find the source part and compare /// its checksums to the checksums of the desired part. If they match, we can just clone the local part. @@ -3520,7 +3598,8 @@ bool StorageReplicatedMergeTree::fetchPart(const String & part_name, const Stora return fetcher.fetchPart( metadata_snapshot, part_name, source_replica_path, address.host, address.replication_port, - timeouts, user_password.first, user_password.second, interserver_scheme, to_detached); + timeouts, user_password.first, user_password.second, interserver_scheme, to_detached, "", true, + replace_exists ? replaced_disk : nullptr); }; } @@ -3530,46 +3609,56 @@ bool StorageReplicatedMergeTree::fetchPart(const String & part_name, const Stora if (!to_detached) { - Transaction transaction(*this); - renameTempPartAndReplace(part, nullptr, &transaction); - - /** NOTE - * Here, an error occurs if ALTER occurred with a change in the column type or column deletion, - * and the part on remote server has not yet been modified. - * After a while, one of the following attempts to make `fetchPart` succeed. - */ - replaced_parts = checkPartChecksumsAndCommit(transaction, part); - - /** If a quorum is tracked for this part, you must update it. - * If you do not have time, in case of losing the session, when you restart the server - see the `ReplicatedMergeTreeRestartingThread::updateQuorumIfWeHavePart` method. - */ - if (quorum) + if (replace_exists) { - /// Check if this quorum insert is parallel or not - if (zookeeper->exists(zookeeper_path + "/quorum/parallel/" + part_name)) - updateQuorum(part_name, true); - else if (zookeeper->exists(zookeeper_path + "/quorum/status")) - updateQuorum(part_name, false); + if (part->volume->getDisk()->getName() != replaced_disk->getName()) + throw Exception("Part " + part->name + " fetched on wrong disk " + part->volume->getDisk()->getName(), ErrorCodes::LOGICAL_ERROR); + replaced_disk->removeIfExists(replaced_part_path); + replaced_disk->moveDirectory(part->getFullRelativePath(), replaced_part_path); } - - /// merged parts that are still inserted with quorum. if it only contains one block, it hasn't been merged before - if (part_info.level != 0 || part_info.mutation != 0) + else { - Strings quorum_parts = zookeeper->getChildren(zookeeper_path + "/quorum/parallel"); - for (const String & quorum_part : quorum_parts) + Transaction transaction(*this); + renameTempPartAndReplace(part, nullptr, &transaction); + + /** NOTE + * Here, an error occurs if ALTER occurred with a change in the column type or column deletion, + * and the part on remote server has not yet been modified. + * After a while, one of the following attempts to make `fetchPart` succeed. + */ + replaced_parts = checkPartChecksumsAndCommit(transaction, part); + + /** If a quorum is tracked for this part, you must update it. + * If you do not have time, in case of losing the session, when you restart the server - see the `ReplicatedMergeTreeRestartingThread::updateQuorumIfWeHavePart` method. + */ + if (quorum) { - auto quorum_part_info = MergeTreePartInfo::fromPartName(quorum_part, format_version); - if (part_info.contains(quorum_part_info)) - updateQuorum(quorum_part, true); + /// Check if this quorum insert is parallel or not + if (zookeeper->exists(zookeeper_path + "/quorum/parallel/" + part_name)) + updateQuorum(part_name, true); + else if (zookeeper->exists(zookeeper_path + "/quorum/status")) + updateQuorum(part_name, false); } - } - merge_selecting_task->schedule(); + /// merged parts that are still inserted with quorum. if it only contains one block, it hasn't been merged before + if (part_info.level != 0 || part_info.mutation != 0) + { + Strings quorum_parts = zookeeper->getChildren(zookeeper_path + "/quorum/parallel"); + for (const String & quorum_part : quorum_parts) + { + auto quorum_part_info = MergeTreePartInfo::fromPartName(quorum_part, format_version); + if (part_info.contains(quorum_part_info)) + updateQuorum(quorum_part, true); + } + } - for (const auto & replaced_part : replaced_parts) - { - LOG_DEBUG(log, "Part {} is rendered obsolete by fetching part {}", replaced_part->name, part_name); - ProfileEvents::increment(ProfileEvents::ObsoleteReplicatedParts); + merge_selecting_task->schedule(); + + for (const auto & replaced_part : replaced_parts) + { + LOG_DEBUG(log, "Part {} is rendered obsolete by fetching part {}", replaced_part->name, part_name); + ProfileEvents::increment(ProfileEvents::ObsoleteReplicatedParts); + } } write_part_log({}); @@ -5315,13 +5404,13 @@ void StorageReplicatedMergeTree::clearOldPartsAndRemoveFromZK() } parts.clear(); - auto remove_parts_from_filesystem = [log=log,&zookeeper=zookeeper,&zookeeper_path=zookeeper_path,&replica_name=replica_name] (const DataPartsVector & parts_to_remove) + auto remove_parts_from_filesystem = [log=log] (const DataPartsVector & parts_to_remove) { for (const auto & part : parts_to_remove) { try { - bool keep_s3 = !part->unlockSharedData(zookeeper_path, replica_name, zookeeper); + bool keep_s3 = !part->unlockSharedData(); part->remove(keep_s3); } catch (...) @@ -6271,4 +6360,13 @@ void StorageReplicatedMergeTree::startBackgroundMovesIfNeeded() background_moves_executor.start(); } +StorageReplicatedMergeTree::ZooKeeperAccessData StorageReplicatedMergeTree::getZooKeeperAccessData() const +{ + ZooKeeperAccessData res; + res.zookeeper = tryGetZooKeeper(); + res.zookeeper_path = zookeeper_path; + res.replica_name = replica_name; + return res; +} + } diff --git a/src/Storages/StorageReplicatedMergeTree.h b/src/Storages/StorageReplicatedMergeTree.h index d396f32dcca..11dc475257e 100644 --- a/src/Storages/StorageReplicatedMergeTree.h +++ b/src/Storages/StorageReplicatedMergeTree.h @@ -211,6 +211,18 @@ public: /// is not overloaded bool canExecuteFetch(const ReplicatedMergeTreeLogEntry & entry, String & disable_reason) const; + struct ZooKeeperAccessData + { + zkutil::ZooKeeperPtr zookeeper; + String zookeeper_path; + String replica_name; + }; + + ZooKeeperAccessData getZooKeeperAccessData() const; + + /// Fetch part only when it stored on shared storage like S3 + bool executeFetchShared(ReplicatedMergeTreeLogEntry & entry); + private: /// Get a sequential consistent view of current parts. ReplicatedMergeTreeQuorumAddedParts::PartitionIdToMaxBlock getMaxAddedBlocks() const; @@ -369,8 +381,7 @@ private: String getChecksumsForZooKeeper(const MergeTreeDataPartChecksums & checksums) const; /// Accepts a PreComitted part, atomically checks its checksums with ones on other replicas and commit the part - DataPartsVector checkPartChecksumsAndCommit(Transaction & transaction, - const DataPartPtr & part); + DataPartsVector checkPartChecksumsAndCommit(Transaction & transaction, const DataPartPtr & part); bool partIsAssignedToBackgroundOperation(const DataPartPtr & part) const override; @@ -487,6 +498,10 @@ private: */ String findReplicaHavingPart(const String & part_name, bool active); + /** Returns a replica with part on shared storage like S3. + */ + String findReplicaHavingSharedPart(const String & part_name, bool active); + bool checkReplicaHavePart(const String & replica, const String & part_name); /** Find replica having specified part or any part that covers it. @@ -508,7 +523,10 @@ private: const String & replica_path, bool to_detached, size_t quorum, - zkutil::ZooKeeper::Ptr zookeeper_ = nullptr); + zkutil::ZooKeeper::Ptr zookeeper_ = nullptr, + bool replace_exists = false, + DiskPtr replaced_disk = nullptr, + String replaced_part_path = ""); /// Required only to avoid races between executeLogEntry and fetchPartition std::unordered_set currently_fetching_parts; From a597ed0ff02074198d01068f6cf8e9789005e759 Mon Sep 17 00:00:00 2001 From: Vladimir Chebotarev Date: Thu, 4 Feb 2021 09:21:05 +0300 Subject: [PATCH 012/544] Fixed open behavior of remote host filter in case when there is remote_url_allow_hosts section in configuration but no entries there. --- src/Common/RemoteHostFilter.cpp | 3 ++- src/Common/RemoteHostFilter.h | 1 + .../test_allowed_url_from_config/test.py | 20 +++++++++++++++---- 3 files changed, 19 insertions(+), 5 deletions(-) diff --git a/src/Common/RemoteHostFilter.cpp b/src/Common/RemoteHostFilter.cpp index fb6fc4e9bc3..6ea366314e1 100644 --- a/src/Common/RemoteHostFilter.cpp +++ b/src/Common/RemoteHostFilter.cpp @@ -42,6 +42,7 @@ void RemoteHostFilter::setValuesFromConfig(const Poco::Util::AbstractConfigurati else if (startsWith(key, "host")) primary_hosts.insert(config.getString("remote_url_allow_hosts." + key)); } + is_allow_by_default = false; } } @@ -58,6 +59,6 @@ bool RemoteHostFilter::checkForDirectEntry(const std::string & str) const } return true; } - return true; + return is_allow_by_default; } } diff --git a/src/Common/RemoteHostFilter.h b/src/Common/RemoteHostFilter.h index 48d9b2bda7c..a445471a411 100644 --- a/src/Common/RemoteHostFilter.h +++ b/src/Common/RemoteHostFilter.h @@ -24,6 +24,7 @@ public: void checkHostAndPort(const std::string & host, const std::string & port) const; /// Does the same as checkURL, but for host and port. private: + bool is_allow_by_default = true; std::unordered_set primary_hosts; /// Allowed primary () URL from config.xml std::vector regexp_hosts; /// Allowed regexp () URL from config.xml diff --git a/tests/integration/test_allowed_url_from_config/test.py b/tests/integration/test_allowed_url_from_config/test.py index 6442937c8f4..59c7c6e37e7 100644 --- a/tests/integration/test_allowed_url_from_config/test.py +++ b/tests/integration/test_allowed_url_from_config/test.py @@ -6,6 +6,7 @@ node1 = cluster.add_instance('node1', main_configs=['configs/config_with_hosts.x node2 = cluster.add_instance('node2', main_configs=['configs/config_with_only_primary_hosts.xml']) node3 = cluster.add_instance('node3', main_configs=['configs/config_with_only_regexp_hosts.xml']) node4 = cluster.add_instance('node4', main_configs=['configs/config_without_allowed_hosts.xml']) +node5 = cluster.add_instance('node5', main_configs=[]) # No `remote_url_allow_hosts` at all. node6 = cluster.add_instance('node6', main_configs=['configs/config_for_remote.xml']) node7 = cluster.add_instance('node7', main_configs=['configs/config_for_redirect.xml'], with_hdfs=True) @@ -51,10 +52,21 @@ def test_config_with_only_regexp_hosts(start_cluster): def test_config_without_allowed_hosts(start_cluster): - assert node4.query("CREATE TABLE table_test_4_1 (word String) Engine=URL('https://host:80', CSV)") == "" - assert node4.query("CREATE TABLE table_test_4_2 (word String) Engine=URL('https://host', HDFS)") == "" - assert node4.query("CREATE TABLE table_test_4_3 (word String) Engine=URL('https://yandex.ru', CSV)") == "" - assert node4.query("CREATE TABLE table_test_4_4 (word String) Engine=URL('ftp://something.com', S3)") == "" + assert "not allowed" in node4.query_and_get_error( + "CREATE TABLE table_test_4_1 (word String) Engine=URL('https://host:80', CSV)") + assert "not allowed" in node4.query_and_get_error( + "CREATE TABLE table_test_4_2 (word String) Engine=URL('https://host', HDFS)") + assert "not allowed" in node4.query_and_get_error( + "CREATE TABLE table_test_4_3 (word String) Engine=URL('https://yandex.ru', CSV)") + assert "not allowed" in node4.query_and_get_error( + "CREATE TABLE table_test_4_4 (word String) Engine=URL('ftp://something.com', S3)") + + +def test_config_without_allowed_hosts_section(start_cluster): + assert node5.query("CREATE TABLE table_test_4_1 (word String) Engine=URL('https://host:80', CSV)") == "" + assert node5.query("CREATE TABLE table_test_4_2 (word String) Engine=URL('https://host', HDFS)") == "" + assert node5.query("CREATE TABLE table_test_4_3 (word String) Engine=URL('https://yandex.ru', CSV)") == "" + assert node5.query("CREATE TABLE table_test_4_4 (word String) Engine=URL('ftp://something.com', S3)") == "" def test_table_function_remote(start_cluster): From f106d58c05d89fcbd73114d62c8993722ea072a2 Mon Sep 17 00:00:00 2001 From: Vladimir Chebotarev Date: Thu, 4 Feb 2021 10:41:47 +0300 Subject: [PATCH 013/544] Minor fix. --- programs/server/config.xml | 4 +-- .../test_allowed_url_from_config/test.py | 34 +++++++++---------- 2 files changed, 19 insertions(+), 19 deletions(-) diff --git a/programs/server/config.xml b/programs/server/config.xml index 849d3dc32ba..650a9a28ff2 100644 --- a/programs/server/config.xml +++ b/programs/server/config.xml @@ -568,7 +568,7 @@ - + - + From 341e7bc8482e99478a0e40ea1afa446ca15f9312 Mon Sep 17 00:00:00 2001 From: George Date: Tue, 16 Feb 2021 13:23:52 +0300 Subject: [PATCH 020/544] Fixed links --- docs/en/sql-reference/statements/detach.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/en/sql-reference/statements/detach.md b/docs/en/sql-reference/statements/detach.md index f3f8b053724..b2720acaaa5 100644 --- a/docs/en/sql-reference/statements/detach.md +++ b/docs/en/sql-reference/statements/detach.md @@ -15,7 +15,7 @@ DETACH TABLE [IF EXISTS] [db.]name [ON CLUSTER cluster] This does not delete the table’s data or metadata. On the next server launch, the server will read the metadata and find out about the table again. -Similarly, a “detached” table can be re-attached using the [ATTACH](../../sql-reference/statements/attach) query (with the exception of system tables, which do not have metadata stored for them). +Similarly, a “detached” table can be re-attached using the [ATTACH](../../sql-reference/statements/attach.md) query (with the exception of system tables, which do not have metadata stored for them). ## DETACH PERMAMENTLY {#detach-permamently} @@ -29,6 +29,6 @@ DETACH TABLE/VIEW [IF EXISTS] [db.]name PERMAMENTLY [ON CLUSTER cluster] This statement does not delete the table’s data or metadata. -Permamently detached table or view can be reattached with [ATTACH](../../sql-reference/statements/attach) query and can be shown with [SHOW CREATE TABLE](../../sql-reference/statements/show.md#show-create-table) query. +Permamently detached table or view can be reattached with [ATTACH](../../sql-reference/statements/attach.md) query and can be shown with [SHOW CREATE TABLE](../../sql-reference/statements/show.md#show-create-table) query. [Original article](https://clickhouse.tech/docs/en/sql-reference/statements/detach/) From 17d7a49106342536a0348c020ca92e1cafc52434 Mon Sep 17 00:00:00 2001 From: George Date: Tue, 16 Feb 2021 13:33:20 +0300 Subject: [PATCH 021/544] Fixed typos --- docs/en/sql-reference/statements/detach.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/en/sql-reference/statements/detach.md b/docs/en/sql-reference/statements/detach.md index b2720acaaa5..adb2df570d7 100644 --- a/docs/en/sql-reference/statements/detach.md +++ b/docs/en/sql-reference/statements/detach.md @@ -17,9 +17,9 @@ This does not delete the table’s data or metadata. On the next server launch, Similarly, a “detached” table can be re-attached using the [ATTACH](../../sql-reference/statements/attach.md) query (with the exception of system tables, which do not have metadata stored for them). -## DETACH PERMAMENTLY {#detach-permamently} +## DETACH PERMANENTLY {#detach-permanently} -Deletes information about `name` table or view from the server. Permamently detached tables won't automatically reappear after the server restart. +Deletes information about `name` table or view from the server. Permanently detached tables won't automatically reappear after the server restart. Syntax: @@ -29,6 +29,6 @@ DETACH TABLE/VIEW [IF EXISTS] [db.]name PERMAMENTLY [ON CLUSTER cluster] This statement does not delete the table’s data or metadata. -Permamently detached table or view can be reattached with [ATTACH](../../sql-reference/statements/attach.md) query and can be shown with [SHOW CREATE TABLE](../../sql-reference/statements/show.md#show-create-table) query. +Permanently detached table or view can be reattached with [ATTACH](../../sql-reference/statements/attach.md) query and can be shown with [SHOW CREATE TABLE](../../sql-reference/statements/show.md#show-create-table) query. [Original article](https://clickhouse.tech/docs/en/sql-reference/statements/detach/) From f1fa110d486ce59b072df3c98b1d9cbf50296868 Mon Sep 17 00:00:00 2001 From: Anton Popov Date: Thu, 18 Feb 2021 01:05:31 +0300 Subject: [PATCH 022/544] fix type map with integer keys --- src/DataTypes/DataTypeMap.cpp | 17 ++++++++- src/DataTypes/DataTypeMap.h | 2 ++ src/Functions/array/arrayElement.cpp | 9 +++-- src/Interpreters/convertFieldToType.cpp | 35 +++++++++++++++++++ src/Parsers/ExpressionElementParsers.cpp | 1 - src/Parsers/ExpressionElementParsers.h | 12 ------- .../Impl/ConstantExpressionTemplate.cpp | 10 +----- 7 files changed, 61 insertions(+), 25 deletions(-) diff --git a/src/DataTypes/DataTypeMap.cpp b/src/DataTypes/DataTypeMap.cpp index af2ed8805e8..246d781b097 100644 --- a/src/DataTypes/DataTypeMap.cpp +++ b/src/DataTypes/DataTypeMap.cpp @@ -29,6 +29,7 @@ namespace ErrorCodes { extern const int NUMBER_OF_ARGUMENTS_DOESNT_MATCH; extern const int CANNOT_READ_MAP_FROM_TEXT; + extern const int BAD_ARGUMENTS; } @@ -38,6 +39,8 @@ DataTypeMap::DataTypeMap(const DataTypes & elems_) key_type = elems_[0]; value_type = elems_[1]; + assertKeyType(); + nested = std::make_shared( std::make_shared(DataTypes{key_type, value_type}, Names{"keys", "values"})); } @@ -45,7 +48,19 @@ DataTypeMap::DataTypeMap(const DataTypes & elems_) DataTypeMap::DataTypeMap(const DataTypePtr & key_type_, const DataTypePtr & value_type_) : key_type(key_type_), value_type(value_type_) , nested(std::make_shared( - std::make_shared(DataTypes{key_type_, value_type_}, Names{"keys", "values"}))) {} + std::make_shared(DataTypes{key_type_, value_type_}, Names{"keys", "values"}))) +{ + assertKeyType(); +} + +void DataTypeMap::assertKeyType() const +{ + if (!key_type->isValueRepresentedByInteger() && !isStringOrFixedString(*key_type) && !WhichDataType(key_type).isNothing()) + throw Exception(ErrorCodes::BAD_ARGUMENTS, + "Type of Map key must be a type, that can be represented by integer or string," + " but {} given", key_type->getName()); +} + std::string DataTypeMap::doGetName() const { diff --git a/src/DataTypes/DataTypeMap.h b/src/DataTypes/DataTypeMap.h index ea495f05548..2a9173dceae 100644 --- a/src/DataTypes/DataTypeMap.h +++ b/src/DataTypes/DataTypeMap.h @@ -98,6 +98,8 @@ private: template void deserializeTextImpl(IColumn & column, ReadBuffer & istr, bool need_safe_get_int_key, Reader && reader) const; + + void assertKeyType() const; }; } diff --git a/src/Functions/array/arrayElement.cpp b/src/Functions/array/arrayElement.cpp index 7d053988cae..cca252216b3 100644 --- a/src/Functions/array/arrayElement.cpp +++ b/src/Functions/array/arrayElement.cpp @@ -872,7 +872,7 @@ bool FunctionArrayElement::matchKeyToIndexNumberConst( if (!data_numeric) return false; - if (index.getType() != Field::Types::UInt64 && index.getType() != Field::Types::Int64) + if (index.getType() != Field::Types::UInt64 && index.getType() != Field::Types::Int64 && index.getType() != Field::Types::Int128) return false; MatcherNumberConst matcher{data_numeric->getData(), get(index)}; @@ -910,6 +910,7 @@ bool FunctionArrayElement::matchKeyToIndex( || matchKeyToIndexNumber(data, offsets, arguments, matched_idxs) || matchKeyToIndexNumber(data, offsets, arguments, matched_idxs) || matchKeyToIndexNumber(data, offsets, arguments, matched_idxs) + || matchKeyToIndexNumber(data, offsets, arguments, matched_idxs) || matchKeyToIndexString(data, offsets, arguments, matched_idxs); } @@ -925,6 +926,7 @@ bool FunctionArrayElement::matchKeyToIndexConst( || matchKeyToIndexNumberConst(data, offsets, index, matched_idxs) || matchKeyToIndexNumberConst(data, offsets, index, matched_idxs) || matchKeyToIndexNumberConst(data, offsets, index, matched_idxs) + || matchKeyToIndexNumberConst(data, offsets, index, matched_idxs) || matchKeyToIndexStringConst(data, offsets, index, matched_idxs); } @@ -945,11 +947,14 @@ ColumnPtr FunctionArrayElement::executeMap( indices_column->reserve(input_rows_count); auto & indices_data = assert_cast &>(*indices_column).getData(); + std::cerr << "types: " << arguments[0].type->getName() << " " << arguments[1].type->getName() << "\n"; + std::cerr << "columns: " << arguments[0].column->dumpStructure() << " " << arguments[1].column->dumpStructure() << "\n"; + if (!isColumnConst(*arguments[1].column)) { if (input_rows_count > 0 && !matchKeyToIndex(keys_data, offsets, arguments, indices_data)) throw Exception(ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT, - "Illegal types of arguments: {}, {} for function ", + "Illegal types of arguments: {}, {} for function {}", arguments[0].type->getName(), arguments[1].type->getName(), getName()); } else diff --git a/src/Interpreters/convertFieldToType.cpp b/src/Interpreters/convertFieldToType.cpp index 73bf493fa65..5bde9c8ec1a 100644 --- a/src/Interpreters/convertFieldToType.cpp +++ b/src/Interpreters/convertFieldToType.cpp @@ -15,6 +15,7 @@ #include #include #include +#include #include #include @@ -276,6 +277,40 @@ Field convertFieldToTypeImpl(const Field & src, const IDataType & type, const ID return have_unconvertible_element ? Field(Null()) : Field(res); } } + else if (const DataTypeMap * type_map = typeid_cast(&type)) + { + if (src.getType() == Field::Types::Map) + { + const auto & src_map = src.get(); + + const auto & key_type = *type_map->getKeyType(); + const auto & value_type = *type_map->getValueType(); + + bool have_unconvertible_element = false; + Map res(src_map.size()); + + for (size_t i = 0; i < src_map.size(); ++i) + { + const auto & src_tuple = src_map[i].safeGet(); + assert(src_tuple.size() == 2); + Tuple res_tuple(2); + + res_tuple[0] = convertFieldToType(src_tuple[0], key_type); + res_tuple[1] = convertFieldToType(src_tuple[1], value_type); + + if ((res_tuple[0].isNull() && !key_type.isNullable()) + || (res_tuple[1].isNull() && !value_type.isNullable())) + { + // See the comment for Tuples above. + have_unconvertible_element = true; + } + + res[i] = std::move(res_tuple); + } + + return have_unconvertible_element ? Field(Null()) : Field(res); + } + } else if (const DataTypeAggregateFunction * agg_func_type = typeid_cast(&type)) { if (src.getType() != Field::Types::AggregateFunctionState) diff --git a/src/Parsers/ExpressionElementParsers.cpp b/src/Parsers/ExpressionElementParsers.cpp index e7cd85798b9..fcb9a55c260 100644 --- a/src/Parsers/ExpressionElementParsers.cpp +++ b/src/Parsers/ExpressionElementParsers.cpp @@ -1979,7 +1979,6 @@ bool ParserExpressionElement::parseImpl(Pos & pos, ASTPtr & node, Expected & exp { return ParserSubquery().parse(pos, node, expected) || ParserTupleOfLiterals().parse(pos, node, expected) - || ParserMapOfLiterals().parse(pos, node, expected) || ParserParenthesisExpression().parse(pos, node, expected) || ParserArrayOfLiterals().parse(pos, node, expected) || ParserArray().parse(pos, node, expected) diff --git a/src/Parsers/ExpressionElementParsers.h b/src/Parsers/ExpressionElementParsers.h index ba18fc2cddd..02bd5d896e4 100644 --- a/src/Parsers/ExpressionElementParsers.h +++ b/src/Parsers/ExpressionElementParsers.h @@ -324,18 +324,6 @@ protected: } }; -class ParserMapOfLiterals : public IParserBase -{ -public: - ParserCollectionOfLiterals map_parser{TokenType::OpeningCurlyBrace, TokenType::ClosingCurlyBrace}; -protected: - const char * getName() const override { return "map"; } - bool parseImpl(Pos & pos, ASTPtr & node, Expected & expected) override - { - return map_parser.parse(pos, node, expected); - } -}; - class ParserArrayOfLiterals : public IParserBase { public: diff --git a/src/Processors/Formats/Impl/ConstantExpressionTemplate.cpp b/src/Processors/Formats/Impl/ConstantExpressionTemplate.cpp index d7a65c2f15d..c00dcd7a579 100644 --- a/src/Processors/Formats/Impl/ConstantExpressionTemplate.cpp +++ b/src/Processors/Formats/Impl/ConstantExpressionTemplate.cpp @@ -203,12 +203,6 @@ private: if (not_null == array.end()) return true; } - else if (literal->value.getType() == Field::Types::Map) - { - const Map & map = literal->value.get(); - if (map.size() % 2) - return false; - } String column_name = "_dummy_" + std::to_string(replaced_literals.size()); replaced_literals.emplace_back(literal, column_name, force_nullable); @@ -481,14 +475,12 @@ bool ConstantExpressionTemplate::parseLiteralAndAssertType(ReadBuffer & istr, co /// TODO faster way to check types without using Parsers ParserArrayOfLiterals parser_array; ParserTupleOfLiterals parser_tuple; - ParserMapOfLiterals parser_map; Tokens tokens_number(istr.position(), istr.buffer().end()); IParser::Pos iterator(tokens_number, settings.max_parser_depth); Expected expected; ASTPtr ast; - if (!parser_array.parse(iterator, ast, expected) && !parser_tuple.parse(iterator, ast, expected) - && !parser_map.parse(iterator, ast, expected)) + if (!parser_array.parse(iterator, ast, expected) && !parser_tuple.parse(iterator, ast, expected)) return false; istr.position() = const_cast(iterator->begin); From afed8a8192dbc118292072fd1286cc9af226c4c1 Mon Sep 17 00:00:00 2001 From: Alexander Tokmakov Date: Fri, 19 Feb 2021 01:08:44 +0300 Subject: [PATCH 023/544] process stress test results --- docker/test/stress/run.sh | 45 +++++++++++++++++++++++++++++++++++++-- docker/test/stress/stress | 3 ++- 2 files changed, 45 insertions(+), 3 deletions(-) diff --git a/docker/test/stress/run.sh b/docker/test/stress/run.sh index 88a633ac488..1e46adca966 100755 --- a/docker/test/stress/run.sh +++ b/docker/test/stress/run.sh @@ -64,9 +64,50 @@ clickhouse-client --query "RENAME TABLE datasets.hits_v1 TO test.hits" clickhouse-client --query "RENAME TABLE datasets.visits_v1 TO test.visits" clickhouse-client --query "SHOW TABLES FROM test" -./stress --hung-check --output-folder test_output --skip-func-tests "$SKIP_TESTS_OPTION" && echo "OK" > /test_output/script_exit_code.txt || echo "FAIL" > /test_output/script_exit_code.txt +./stress --hung-check --output-folder test_output --skip-func-tests "$SKIP_TESTS_OPTION" \ + && echo -e 'Test script exit code\tOK' >> /test_output/test_results.tsv \ + || echo -e 'Test script failed\tFAIL' >> /test_output/test_results.tsv stop start -clickhouse-client --query "SELECT 'Server successfuly started'" > /test_output/alive_check.txt || echo 'Server failed to start' > /test_output/alive_check.txt +clickhouse-client --query "SELECT 'Server successfully started', 'OK'" >> /test_output/test_results.tsv \ + || echo -e 'Server failed to start\tFAIL' >> /test_output/test_results.tsv + +[ -f /var/log/clickhouse-server/clickhouse-server.log ] || echo -e "Server log does not exist\tFAIL" +[ -f /var/log/clickhouse-server/stderr.log ] || echo -e "Stderr log does not exist\tFAIL" + +# Print Fatal log messages to stdout +zgrep -Fa " " /var/log/clickhouse-server/clickhouse-server.log + +# Grep logs for sanitizer asserts, crashes and other critical errors + +# Sanitizer asserts +zgrep -Fa "==================" /var/log/clickhouse-server/stderr.log >> /test_output/tmp +zgrep -Fa "WARNING" /var/log/clickhouse-server/stderr.log >> /test_output/tmp +zgrep -Fav "ASan doesn't fully support makecontext/swapcontext functions" > /dev/null \ + && echo -e 'Sanitizer assert (in stderr.log)\tFAIL' >> /test_output/test_results.tsv \ + || echo -e 'No sanitizer asserts\tOK' >> /test_output/test_results.tsv +rm -f /test_output/tmp + +# Logical errors +zgrep -Fa "Code: 49, e.displayText() = DB::Exception:" /var/log/clickhouse-server/clickhouse-server.log > /dev/null \ + && echo -e 'Logical error thrown (see clickhouse-server.log)\tFAIL' >> /test_output/test_results.tsv \ + || echo -e 'No logical errors\tOK' >> /test_output/test_results.tsv + +# Crash +zgrep -Fa "########################################" /var/log/clickhouse-server/clickhouse-server.log > /dev/null \ + && echo -e 'Killed by signal (in clickhouse-server.log)\tFAIL' >> /test_output/test_results.tsv \ + || echo -e 'Not crashed\tOK' >> /test_output/test_results.tsv + +# It also checks for OOM or crash without stacktrace (printed by watchdog) +zgrep -Fa " " /var/log/clickhouse-server/clickhouse-server.log > /dev/null \ + && echo -e 'Fatal message in clickhouse-server.log\tFAIL' >> /test_output/test_results.tsv \ + || echo -e 'No fatal messages in clickhouse-server.log\tOK' >> /test_output/test_results.tsv + +zgrep -Fa "########################################" /test_output/* > /dev/null \ + && echo -e 'Killed by signal (output files)\tFAIL' >> /test_output/test_results.tsv + +# Write check result into check_status.tsv +clickhouse-local --structure "test String, res String" -q "SELECT 'failure', test FROM table WHERE res != 'OK' LIMIT 1" < /test_output/test_results.tsv > /test_output/check_status.tsv +[ -s /test_output/check_status.tsv ] || echo -e "success\tNo errors found" diff --git a/docker/test/stress/stress b/docker/test/stress/stress index d2ec86b4421..8fad49ba5ee 100755 --- a/docker/test/stress/stress +++ b/docker/test/stress/stress @@ -88,8 +88,9 @@ if __name__ == "__main__": logging.info("Checking if some queries hung") cmd = "{} {} {}".format(args.test_cmd, "--hung-check", "00001_select_1") res = call(cmd, shell=True, stderr=STDOUT) + hung_check_status = "Hung check\t{}\n".format('FAIL' if res else 'OK') + open(os.path.join(args.output_folder, "test_results.tsv"), 'w+').write() if res != 0: logging.info("Hung check failed with exit code {}".format(res)) - sys.exit(1) logging.info("Stress test finished") From 4493c39bf72448b125919b6bf4beb904038c1e73 Mon Sep 17 00:00:00 2001 From: Alexander Tokmakov Date: Fri, 19 Feb 2021 12:57:09 +0300 Subject: [PATCH 024/544] fix --- docker/test/stress/run.sh | 4 ++++ docker/test/stress/stress | 2 +- tests/clickhouse-test | 6 ++++-- .../0_stateless/01079_parallel_alter_modify_zookeeper.sh | 1 + 4 files changed, 10 insertions(+), 3 deletions(-) diff --git a/docker/test/stress/run.sh b/docker/test/stress/run.sh index 1e46adca966..963b204c4c0 100755 --- a/docker/test/stress/run.sh +++ b/docker/test/stress/run.sh @@ -108,6 +108,10 @@ zgrep -Fa " " /var/log/clickhouse-server/clickhouse-server.log > /dev/nu zgrep -Fa "########################################" /test_output/* > /dev/null \ && echo -e 'Killed by signal (output files)\tFAIL' >> /test_output/test_results.tsv +# Put logs into /test_output/ +pigz < /var/log/clickhouse-server/clickhouse-server.log > /test_output/clickhouse-server.log.gz +mv /var/log/clickhouse-server/stderr.log /test_output/ + # Write check result into check_status.tsv clickhouse-local --structure "test String, res String" -q "SELECT 'failure', test FROM table WHERE res != 'OK' LIMIT 1" < /test_output/test_results.tsv > /test_output/check_status.tsv [ -s /test_output/check_status.tsv ] || echo -e "success\tNo errors found" diff --git a/docker/test/stress/stress b/docker/test/stress/stress index 8fad49ba5ee..e0189072f7d 100755 --- a/docker/test/stress/stress +++ b/docker/test/stress/stress @@ -89,7 +89,7 @@ if __name__ == "__main__": cmd = "{} {} {}".format(args.test_cmd, "--hung-check", "00001_select_1") res = call(cmd, shell=True, stderr=STDOUT) hung_check_status = "Hung check\t{}\n".format('FAIL' if res else 'OK') - open(os.path.join(args.output_folder, "test_results.tsv"), 'w+').write() + open(os.path.join(args.output_folder, "test_results.tsv"), 'w+').write(hung_check_status) if res != 0: logging.info("Hung check failed with exit code {}".format(res)) diff --git a/tests/clickhouse-test b/tests/clickhouse-test index 74f5f07eb9d..fa8d2891224 100755 --- a/tests/clickhouse-test +++ b/tests/clickhouse-test @@ -229,13 +229,15 @@ def get_stacktraces_from_clickhouse(client): def get_server_pid(server_tcp_port): cmd = "lsof -i tcp:{port} -s tcp:LISTEN -Fp | awk '/^p[0-9]+$/{{print substr($0, 2)}}'".format(port=server_tcp_port) + output = None try: - output = subprocess.check_output(cmd, shell=True) + output = subprocess.check_output(cmd, shell=True, stderr=subprocess.STDOUT) if output: return int(output) else: return None # server dead - except Exception: + except Exception as e: + print("Cannot get server pid, got {}: {}", output, e) return None diff --git a/tests/queries/0_stateless/01079_parallel_alter_modify_zookeeper.sh b/tests/queries/0_stateless/01079_parallel_alter_modify_zookeeper.sh index 5b14c5a8543..0749dc14dfa 100755 --- a/tests/queries/0_stateless/01079_parallel_alter_modify_zookeeper.sh +++ b/tests/queries/0_stateless/01079_parallel_alter_modify_zookeeper.sh @@ -14,6 +14,7 @@ for i in $(seq $REPLICAS); do $CLICKHOUSE_CLIENT --query "CREATE TABLE concurrent_alter_mt_$i (key UInt64, value1 UInt64, value2 Int32) ENGINE = ReplicatedMergeTree('/clickhouse/tables/test_01079/concurrent_alter_mt', '$i') ORDER BY key SETTINGS max_replicated_mutations_in_queue=1000, number_of_free_entries_in_pool_to_execute_mutation=0,max_replicated_merges_in_queue=1000" done + $CLICKHOUSE_CLIENT --query "INSERT INTO concurrent_alter_mt_1 SELECT number, number + 10, number from numbers(10)" $CLICKHOUSE_CLIENT --query "INSERT INTO concurrent_alter_mt_1 SELECT number, number + 10, number from numbers(10, 40)" From 033f55f498ead26bcc0bd5d2efa9332bf6db8482 Mon Sep 17 00:00:00 2001 From: Alexander Tokmakov Date: Fri, 19 Feb 2021 17:38:20 +0300 Subject: [PATCH 025/544] fix --- docker/test/stress/stress | 5 ++- tests/clickhouse-test | 44 +++++++++++-------- .../01079_parallel_alter_modify_zookeeper.sh | 1 - 3 files changed, 28 insertions(+), 22 deletions(-) diff --git a/docker/test/stress/stress b/docker/test/stress/stress index e0189072f7d..666fd4cce50 100755 --- a/docker/test/stress/stress +++ b/docker/test/stress/stress @@ -88,9 +88,10 @@ if __name__ == "__main__": logging.info("Checking if some queries hung") cmd = "{} {} {}".format(args.test_cmd, "--hung-check", "00001_select_1") res = call(cmd, shell=True, stderr=STDOUT) - hung_check_status = "Hung check\t{}\n".format('FAIL' if res else 'OK') - open(os.path.join(args.output_folder, "test_results.tsv"), 'w+').write(hung_check_status) + hung_check_status = "No queries hung\tOK\n" if res != 0: logging.info("Hung check failed with exit code {}".format(res)) + hung_check_status = "Hung check failed\tFAIL\n" + open(os.path.join(args.output_folder, "test_results.tsv"), 'w+').write(hung_check_status) logging.info("Stress test finished") diff --git a/tests/clickhouse-test b/tests/clickhouse-test index fa8d2891224..2aca0504141 100755 --- a/tests/clickhouse-test +++ b/tests/clickhouse-test @@ -212,7 +212,8 @@ def get_stacktraces_from_gdb(server_pid): try: return subprocess.check_output(cmd, shell=True).decode('utf-8') except Exception as ex: - return "Error occured while receiving stack traces from gdb: {}".format(str(ex)) + print("Error occured while receiving stack traces from gdb: {}".format(str(ex))) + return None # collect server stacktraces from system.stack_trace table @@ -224,21 +225,24 @@ def get_stacktraces_from_clickhouse(client): "arrayMap(x -> demangle(addressToSymbol(x)), trace)), '\n') as trace " "FROM system.stack_trace format Vertical\"".format(client), shell=True).decode('utf-8') except Exception as ex: - return "Error occured while receiving stack traces from client: {}".format(str(ex)) + print("Error occured while receiving stack traces from client: {}".format(str(ex))) + return None def get_server_pid(server_tcp_port): - cmd = "lsof -i tcp:{port} -s tcp:LISTEN -Fp | awk '/^p[0-9]+$/{{print substr($0, 2)}}'".format(port=server_tcp_port) + # lsof does not work in stress tests for some reason + cmd_lsof = "lsof -i tcp:{port} -s tcp:LISTEN -Fp | awk '/^p[0-9]+$/{{print substr($0, 2)}}'".format(port=server_tcp_port) + cmd_pidof = "pidof -s clickhouse-server" + commands = [cmd_lsof, cmd_pidof] output = None - try: - output = subprocess.check_output(cmd, shell=True, stderr=subprocess.STDOUT) - if output: - return int(output) - else: - return None # server dead - except Exception as e: - print("Cannot get server pid, got {}: {}", output, e) - return None + for cmd in commands: + try: + output = subprocess.check_output(cmd, shell=True, stderr=subprocess.STDOUT) + if output: + return int(output) + except Exception as e: + print("Cannot get server pid with {}, got {}: {}", cmd, output, e) + return None # most likely server dead def colored(text, args, color=None, on_color=None, attrs=None): @@ -796,21 +800,23 @@ def main(args): clickhouse_tcp_port = os.getenv("CLICKHOUSE_PORT_TCP", '9000') server_pid = get_server_pid(clickhouse_tcp_port) + bt = None if server_pid: print("\nLocated ClickHouse server process {} listening at TCP port {}".format(server_pid, clickhouse_tcp_port)) - - # It does not work in Sandbox - #print("\nCollecting stacktraces from system.stacktraces table:") - #print(get_stacktraces_from_clickhouse(args.client)) - print("\nCollecting stacktraces from all running threads with gdb:") - print(get_stacktraces_from_gdb(server_pid)) - else: + bt = get_stacktraces_from_gdb(server_pid) + if bt is None: + print("\nCollecting stacktraces from system.stacktraces table:") + bt = get_stacktraces_from_clickhouse(args.client) + if bt is None: print( colored( "\nUnable to locate ClickHouse server process listening at TCP port {}. " "It must have crashed or exited prematurely!".format(clickhouse_tcp_port), args, "red", attrs=["bold"])) + else: + print(bt) + exit_code = 1 else: diff --git a/tests/queries/0_stateless/01079_parallel_alter_modify_zookeeper.sh b/tests/queries/0_stateless/01079_parallel_alter_modify_zookeeper.sh index 0749dc14dfa..5b14c5a8543 100755 --- a/tests/queries/0_stateless/01079_parallel_alter_modify_zookeeper.sh +++ b/tests/queries/0_stateless/01079_parallel_alter_modify_zookeeper.sh @@ -14,7 +14,6 @@ for i in $(seq $REPLICAS); do $CLICKHOUSE_CLIENT --query "CREATE TABLE concurrent_alter_mt_$i (key UInt64, value1 UInt64, value2 Int32) ENGINE = ReplicatedMergeTree('/clickhouse/tables/test_01079/concurrent_alter_mt', '$i') ORDER BY key SETTINGS max_replicated_mutations_in_queue=1000, number_of_free_entries_in_pool_to_execute_mutation=0,max_replicated_merges_in_queue=1000" done - $CLICKHOUSE_CLIENT --query "INSERT INTO concurrent_alter_mt_1 SELECT number, number + 10, number from numbers(10)" $CLICKHOUSE_CLIENT --query "INSERT INTO concurrent_alter_mt_1 SELECT number, number + 10, number from numbers(10, 40)" From f5fc082ed0e4b3d9f412f6ecfb5db0ce49469d3e Mon Sep 17 00:00:00 2001 From: tavplubix Date: Fri, 19 Feb 2021 22:39:42 +0300 Subject: [PATCH 026/544] Update run.sh --- docker/test/stress/run.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/test/stress/run.sh b/docker/test/stress/run.sh index 963b204c4c0..df58a23794e 100755 --- a/docker/test/stress/run.sh +++ b/docker/test/stress/run.sh @@ -114,4 +114,4 @@ mv /var/log/clickhouse-server/stderr.log /test_output/ # Write check result into check_status.tsv clickhouse-local --structure "test String, res String" -q "SELECT 'failure', test FROM table WHERE res != 'OK' LIMIT 1" < /test_output/test_results.tsv > /test_output/check_status.tsv -[ -s /test_output/check_status.tsv ] || echo -e "success\tNo errors found" +[ -s /test_output/check_status.tsv ] || echo -e "success\tNo errors found" > /test_output/check_status.tsv From 694d89ad81bcb4a551903097a0df042dd48639c7 Mon Sep 17 00:00:00 2001 From: Alexander Tokmakov Date: Sat, 20 Feb 2021 19:27:04 +0300 Subject: [PATCH 027/544] fix --- docker/test/stress/run.sh | 5 +++-- docker/test/stress/stress | 2 ++ src/Interpreters/DDLWorker.cpp | 2 +- 3 files changed, 6 insertions(+), 3 deletions(-) diff --git a/docker/test/stress/run.sh b/docker/test/stress/run.sh index 03c140d8a83..ee291e5b04d 100755 --- a/docker/test/stress/run.sh +++ b/docker/test/stress/run.sh @@ -53,10 +53,11 @@ handle SIGBUS stop print handle SIGABRT stop print continue thread apply all backtrace -continue +detach +quit " > script.gdb - gdb -batch -command script.gdb -p "$(cat /var/run/clickhouse-server/clickhouse-server.pid)" & + gdb -batch -command script.gdb -p "$(cat /var/run/clickhouse-server/clickhouse-server.pid)" >> /test_output/gdb.log & } configure diff --git a/docker/test/stress/stress b/docker/test/stress/stress index 666fd4cce50..c62692f8683 100755 --- a/docker/test/stress/stress +++ b/docker/test/stress/stress @@ -85,6 +85,8 @@ if __name__ == "__main__": logging.info("All processes finished") if args.hung_check: + logging.info("Will terminate gdb (if any)") + res = call("killall -TERM gdb", shell=True, stderr=STDOUT) logging.info("Checking if some queries hung") cmd = "{} {} {}".format(args.test_cmd, "--hung-check", "00001_select_1") res = call(cmd, shell=True, stderr=STDOUT) diff --git a/src/Interpreters/DDLWorker.cpp b/src/Interpreters/DDLWorker.cpp index fc460a5584c..63df919de22 100644 --- a/src/Interpreters/DDLWorker.cpp +++ b/src/Interpreters/DDLWorker.cpp @@ -719,7 +719,7 @@ void DDLWorker::processTask(DDLTask & task) String dummy; if (zookeeper->tryGet(active_node_path, dummy, nullptr, eph_node_disappeared)) { - constexpr int timeout_ms = 5000; + constexpr int timeout_ms = 30 * 1000; if (!eph_node_disappeared->tryWait(timeout_ms)) throw Exception(ErrorCodes::LOGICAL_ERROR, "Ephemeral node {} still exists, " "probably it's owned by someone else", active_node_path); From eaed15b5e1ee241274c1b2ae2582a4a67170d6e8 Mon Sep 17 00:00:00 2001 From: Alexander Tokmakov Date: Sat, 20 Feb 2021 23:04:24 +0300 Subject: [PATCH 028/544] process result of func and unit tests --- docker/test/stateful/run.sh | 5 + docker/test/stateless/Dockerfile | 1 + .../process_functional_tests_result.py | 117 ++++++++++++++++++ docker/test/stateless/run.sh | 4 + docker/test/stress/stress | 2 +- docker/test/unit/Dockerfile | 6 +- docker/test/unit/process_unit_tests_result.py | 96 ++++++++++++++ docker/test/unit/run.sh | 7 ++ tests/clickhouse-test | 5 +- 9 files changed, 238 insertions(+), 5 deletions(-) create mode 100755 docker/test/stateless/process_functional_tests_result.py create mode 100755 docker/test/unit/process_unit_tests_result.py create mode 100644 docker/test/unit/run.sh diff --git a/docker/test/stateful/run.sh b/docker/test/stateful/run.sh index f2fcefd604f..620fcef5b73 100755 --- a/docker/test/stateful/run.sh +++ b/docker/test/stateful/run.sh @@ -61,3 +61,8 @@ fi read -ra ADDITIONAL_OPTIONS <<< "${ADDITIONAL_OPTIONS:-}" clickhouse-test --testname --shard --zookeeper --no-stateless --hung-check --print-time "$SKIP_LIST_OPT" "${ADDITIONAL_OPTIONS[@]}" "$SKIP_TESTS_OPTION" 2>&1 | ts '%Y-%m-%d %H:%M:%S' | tee test_output/test_result.txt + +./process_functional_tests_result.py + +pigz < /var/log/clickhouse-server/clickhouse-server.log > /test_output/clickhouse-server.log.gz +mv /var/log/clickhouse-server/stderr.log /test_output/ diff --git a/docker/test/stateless/Dockerfile b/docker/test/stateless/Dockerfile index 2437415d17c..61d1b2f4849 100644 --- a/docker/test/stateless/Dockerfile +++ b/docker/test/stateless/Dockerfile @@ -46,4 +46,5 @@ ENV NUM_TRIES=1 ENV MAX_RUN_TIME=0 COPY run.sh / +COPY process_functional_tests_result.py / CMD ["/bin/bash", "/run.sh"] diff --git a/docker/test/stateless/process_functional_tests_result.py b/docker/test/stateless/process_functional_tests_result.py new file mode 100755 index 00000000000..e00243b3a5a --- /dev/null +++ b/docker/test/stateless/process_functional_tests_result.py @@ -0,0 +1,117 @@ +#!/usr/bin/env python3 + +import os +import logging +import argparse +import csv + +OK_SIGN = "[ OK " +FAIL_SING = "[ FAIL " +TIMEOUT_SING = "[ Timeout! " +UNKNOWN_SIGN = "[ UNKNOWN " +SKIPPED_SIGN = "[ SKIPPED " +HUNG_SIGN = "Found hung queries in processlist" + +def process_test_log(log_path): + total = 0 + skipped = 0 + unknown = 0 + failed = 0 + success = 0 + hung = False + test_results = [] + with open(log_path, 'r') as test_file: + for line in test_file: + line = line.strip() + if HUNG_SIGN in line: + hung = True + if any(sign in line for sign in (OK_SIGN, FAIL_SING, UNKNOWN_SIGN, SKIPPED_SIGN)): + test_name = line.split(' ')[2].split(':')[0] + + test_time = '' + try: + time_token = line.split(']')[1].strip().split()[0] + float(time_token) + test_time = time_token + except: + pass + + total += 1 + if TIMEOUT_SING in line: + failed += 1 + test_results.append((test_name, "Timeout", test_time)) + elif FAIL_SING in line: + failed += 1 + test_results.append((test_name, "FAIL", test_time)) + elif UNKNOWN_SIGN in line: + unknown += 1 + test_results.append((test_name, "FAIL", test_time)) + elif SKIPPED_SIGN in line: + skipped += 1 + test_results.append((test_name, "SKIPPED", test_time)) + else: + success += int(OK_SIGN in line) + test_results.append((test_name, "OK", test_time)) + return total, skipped, unknown, failed, success, hung, test_results + +def process_result(result_path): + test_results = [] + state = "success" + description = "" + files = os.listdir(result_path) + if files: + logging.info("Find files in result folder %s", ','.join(files)) + result_path = os.path.join(result_path, 'test_result.txt') + else: + result_path = None + description = "No output log" + state = "error" + + if result_path and os.path.exists(result_path): + total, skipped, unknown, failed, success, hung, test_results = process_test_log(result_path) + is_flacky_check = 1 < int(os.environ.get('NUM_TRIES', 1)) + # If no tests were run (success == 0) it indicates an error (e.g. server did not start or crashed immediately) + # But it's Ok for "flaky checks" - they can contain just one test for check which is marked as skipped. + if failed != 0 or unknown != 0 or (success == 0 and (not is_flacky_check)): + state = "failure" + + if hung: + description = "Some queries hung, " + state = "failure" + else: + description = "" + + description += "fail: {}, passed: {}".format(failed, success) + if skipped != 0: + description += ", skipped: {}".format(skipped) + if unknown != 0: + description += ", unknown: {}".format(unknown) + else: + state = "failure" + description = "Output log doesn't exist" + test_results = [] + + return state, description, test_results + + +def write_results(results_file, status_file, results, status): + with open(results_file, 'w') as f: + out = csv.writer(f, delimiter='\t') + out.writerows(results) + with open(status_file, 'w') as f: + out = csv.writer(f, delimiter='\t') + out.writerow(status) + +if __name__ == "__main__": + logging.basicConfig(level=logging.INFO, format='%(asctime)s %(message)s') + parser = argparse.ArgumentParser(description="ClickHouse script for parsing results of functional tests") + parser.add_argument("--in-results-dir", default='/test_output/') + parser.add_argument("--out-results-file", default='/test_output/test_results.tsv') + parser.add_argument("--out-status-file", default='/test_output/check_status.tsv') + args = parser.parse_args() + + state, description, test_results = process_result(args.in_results_dir) + logging.info("Result parsed") + status = (state, description) + write_results(args.out_results_file, args.out_status_file, test_results, status) + logging.info("Result written") diff --git a/docker/test/stateless/run.sh b/docker/test/stateless/run.sh index 575be721a54..36b49dac961 100755 --- a/docker/test/stateless/run.sh +++ b/docker/test/stateless/run.sh @@ -68,5 +68,9 @@ export -f run_tests timeout "$MAX_RUN_TIME" bash -c run_tests ||: +./process_functional_tests_result.py + +pigz < /var/log/clickhouse-server/clickhouse-server.log > /test_output/clickhouse-server.log.gz +mv /var/log/clickhouse-server/stderr.log /test_output/ tar -chf /test_output/text_log_dump.tar /var/lib/clickhouse/data/system/text_log ||: tar -chf /test_output/query_log_dump.tar /var/lib/clickhouse/data/system/query_log ||: diff --git a/docker/test/stress/stress b/docker/test/stress/stress index c62692f8683..34dbdcfe10d 100755 --- a/docker/test/stress/stress +++ b/docker/test/stress/stress @@ -86,7 +86,7 @@ if __name__ == "__main__": logging.info("All processes finished") if args.hung_check: logging.info("Will terminate gdb (if any)") - res = call("killall -TERM gdb", shell=True, stderr=STDOUT) + res = call("kill -TERM $(pidof clickhouse)", shell=True, stderr=STDOUT) logging.info("Checking if some queries hung") cmd = "{} {} {}".format(args.test_cmd, "--hung-check", "00001_select_1") res = call(cmd, shell=True, stderr=STDOUT) diff --git a/docker/test/unit/Dockerfile b/docker/test/unit/Dockerfile index f01ed613918..e2f4a691939 100644 --- a/docker/test/unit/Dockerfile +++ b/docker/test/unit/Dockerfile @@ -5,6 +5,6 @@ ENV TZ=Europe/Moscow RUN ln -snf /usr/share/zoneinfo/$TZ /etc/localtime && echo $TZ > /etc/timezone RUN apt-get install gdb -CMD service zookeeper start && sleep 7 && /usr/share/zookeeper/bin/zkCli.sh -server localhost:2181 -create create /clickhouse_test ''; \ - gdb -q -ex 'set print inferior-events off' -ex 'set confirm off' -ex 'set print thread-events off' -ex run -ex bt -ex quit --args ./unit_tests_dbms | tee test_output/test_result.txt - +COPY run.sh / +COPY process_unit_tests_result.py / +CMD ["/bin/bash", "/run.sh"] diff --git a/docker/test/unit/process_unit_tests_result.py b/docker/test/unit/process_unit_tests_result.py new file mode 100755 index 00000000000..070262e1f4b --- /dev/null +++ b/docker/test/unit/process_unit_tests_result.py @@ -0,0 +1,96 @@ +#!/usr/bin/env python3 + +import os +import logging +import argparse +import csv + +OK_SIGN = 'OK ]' +FAILED_SIGN = 'FAILED ]' +SEGFAULT = 'Segmentation fault' +SIGNAL = 'received signal SIG' +PASSED = 'PASSED' + +def get_test_name(line): + elements = reversed(line.split(' ')) + for element in elements: + if '(' not in element and ')' not in element: + return element + raise Exception("No test name in line '{}'".format(line)) + +def process_result(result_folder): + summary = [] + total_counter = 0 + failed_counter = 0 + result_log_path = '{}/test_result.txt'.format(result_folder) + if not os.path.exists(result_log_path): + logging.info("No output log on path %s", result_log_path) + return "exception", "No output log", [] + + status = "success" + description = "" + passed = False + with open(result_log_path, 'r') as test_result: + for line in test_result: + if OK_SIGN in line: + logging.info("Found ok line: '%s'", line) + test_name = get_test_name(line.strip()) + logging.info("Test name: '%s'", test_name) + summary.append((test_name, "OK")) + total_counter += 1 + elif FAILED_SIGN in line and 'listed below' not in line and 'ms)' in line: + logging.info("Found fail line: '%s'", line) + test_name = get_test_name(line.strip()) + logging.info("Test name: '%s'", test_name) + summary.append((test_name, "FAIL")) + total_counter += 1 + failed_counter += 1 + elif SEGFAULT in line: + logging.info("Found segfault line: '%s'", line) + status = "failure" + description += "Segmentation fault. " + break + elif SIGNAL in line: + logging.info("Received signal line: '%s'", line) + status = "failure" + description += "Exit on signal. " + break + elif PASSED in line: + logging.info("PASSED record found: '%s'", line) + passed = True + + if not passed: + status = "failure" + description += "PASSED record not found. " + + if failed_counter != 0: + status = "failure" + + if not description: + description += "fail: {}, passed: {}".format(failed_counter, total_counter - failed_counter) + + return status, description, summary + + +def write_results(results_file, status_file, results, status): + with open(results_file, 'w') as f: + out = csv.writer(f, delimiter='\t') + out.writerows(results) + with open(status_file, 'w') as f: + out = csv.writer(f, delimiter='\t') + out.writerow(status) + +if __name__ == "__main__": + logging.basicConfig(level=logging.INFO, format='%(asctime)s %(message)s') + parser = argparse.ArgumentParser(description="ClickHouse script for parsing results of functional tests") + parser.add_argument("--in-results-dir", default='/test_output/') + parser.add_argument("--out-results-file", default='/test_output/test_results.tsv') + parser.add_argument("--out-status-file", default='/test_output/check_status.tsv') + args = parser.parse_args() + + state, description, test_results = process_result(args.in_results_dir) + logging.info("Result parsed") + status = (state, description) + write_results(args.out_results_file, args.out_status_file, test_results, status) + logging.info("Result written") + diff --git a/docker/test/unit/run.sh b/docker/test/unit/run.sh new file mode 100644 index 00000000000..037fd8494cb --- /dev/null +++ b/docker/test/unit/run.sh @@ -0,0 +1,7 @@ +#!/bin/bash + +set -x + +service zookeeper start && sleep 7 && /usr/share/zookeeper/bin/zkCli.sh -server localhost:2181 -create create /clickhouse_test ''; +gdb -q -ex 'set print inferior-events off' -ex 'set confirm off' -ex 'set print thread-events off' -ex run -ex bt -ex quit --args ./unit_tests_dbms | tee test_output/test_result.txt +./process_unit_tests_result.py diff --git a/tests/clickhouse-test b/tests/clickhouse-test index 2aca0504141..4044d29a49e 100755 --- a/tests/clickhouse-test +++ b/tests/clickhouse-test @@ -208,8 +208,8 @@ def get_processlist(args): # collect server stacktraces using gdb def get_stacktraces_from_gdb(server_pid): - cmd = "gdb -batch -ex 'thread apply all backtrace' -p {}".format(server_pid) try: + cmd = "gdb -batch -ex 'thread apply all backtrace' -p {}".format(server_pid) return subprocess.check_output(cmd, shell=True).decode('utf-8') except Exception as ex: print("Error occured while receiving stack traces from gdb: {}".format(str(ex))) @@ -805,6 +805,9 @@ def main(args): print("\nLocated ClickHouse server process {} listening at TCP port {}".format(server_pid, clickhouse_tcp_port)) print("\nCollecting stacktraces from all running threads with gdb:") bt = get_stacktraces_from_gdb(server_pid) + if len(bt) < 1000: + print("Got suspiciously small stacktraces: ", bt) + bt = None if bt is None: print("\nCollecting stacktraces from system.stacktraces table:") bt = get_stacktraces_from_clickhouse(args.client) From 487b1623438315fce1b1bde4ff0611057de6e712 Mon Sep 17 00:00:00 2001 From: Alexander Tokmakov Date: Mon, 22 Feb 2021 16:53:43 +0300 Subject: [PATCH 029/544] fix --- docker/test/stateful/run.sh | 9 ++++++--- docker/test/stateless/run.sh | 9 ++++++--- docker/test/stress/run.sh | 3 +++ docker/test/stress/stress | 2 +- docker/test/unit/run.sh | 2 +- tests/clickhouse-test | 2 +- 6 files changed, 18 insertions(+), 9 deletions(-) diff --git a/docker/test/stateful/run.sh b/docker/test/stateful/run.sh index 620fcef5b73..ddd59a324dc 100755 --- a/docker/test/stateful/run.sh +++ b/docker/test/stateful/run.sh @@ -62,7 +62,10 @@ read -ra ADDITIONAL_OPTIONS <<< "${ADDITIONAL_OPTIONS:-}" clickhouse-test --testname --shard --zookeeper --no-stateless --hung-check --print-time "$SKIP_LIST_OPT" "${ADDITIONAL_OPTIONS[@]}" "$SKIP_TESTS_OPTION" 2>&1 | ts '%Y-%m-%d %H:%M:%S' | tee test_output/test_result.txt -./process_functional_tests_result.py +./process_functional_tests_result.py || echo -e "failure\tCannot parse results" > /test_output/check_status.tsv -pigz < /var/log/clickhouse-server/clickhouse-server.log > /test_output/clickhouse-server.log.gz -mv /var/log/clickhouse-server/stderr.log /test_output/ +pigz < /var/log/clickhouse-server/clickhouse-server.log > /test_output/clickhouse-server.log.gz ||: +mv /var/log/clickhouse-server/stderr.log /test_output/ ||: +if [[ -n "$WITH_COVERAGE" ]] && [[ "$WITH_COVERAGE" -eq 1 ]]; then + tar -chf /test_output/clickhouse_coverage.tar.gz /profraw ||: +fi diff --git a/docker/test/stateless/run.sh b/docker/test/stateless/run.sh index 36b49dac961..9dd4cec585c 100755 --- a/docker/test/stateless/run.sh +++ b/docker/test/stateless/run.sh @@ -68,9 +68,12 @@ export -f run_tests timeout "$MAX_RUN_TIME" bash -c run_tests ||: -./process_functional_tests_result.py +./process_functional_tests_result.py || echo -e "failure\tCannot parse results" > /test_output/check_status.tsv -pigz < /var/log/clickhouse-server/clickhouse-server.log > /test_output/clickhouse-server.log.gz -mv /var/log/clickhouse-server/stderr.log /test_output/ +pigz < /var/log/clickhouse-server/clickhouse-server.log > /test_output/clickhouse-server.log.gz ||: +mv /var/log/clickhouse-server/stderr.log /test_output/ ||: +if [[ -n "$WITH_COVERAGE" ]] && [[ "$WITH_COVERAGE" -eq 1 ]]; then + tar -chf /test_output/clickhouse_coverage.tar.gz /profraw ||: +fi tar -chf /test_output/text_log_dump.tar /var/lib/clickhouse/data/system/text_log ||: tar -chf /test_output/query_log_dump.tar /var/lib/clickhouse/data/system/query_log ||: diff --git a/docker/test/stress/run.sh b/docker/test/stress/run.sh index 6799d1461cb..dcc92e1ad5f 100755 --- a/docker/test/stress/run.sh +++ b/docker/test/stress/run.sh @@ -57,6 +57,9 @@ detach quit " > script.gdb + # FIXME Hung check may work incorrectly because of attached gdb + # 1. False positives are possible + # 2. We cannot attach another gdb to get stacktraces if some queries hung gdb -batch -command script.gdb -p "$(cat /var/run/clickhouse-server/clickhouse-server.pid)" >> /test_output/gdb.log & } diff --git a/docker/test/stress/stress b/docker/test/stress/stress index 34dbdcfe10d..3426a20f67b 100755 --- a/docker/test/stress/stress +++ b/docker/test/stress/stress @@ -86,7 +86,7 @@ if __name__ == "__main__": logging.info("All processes finished") if args.hung_check: logging.info("Will terminate gdb (if any)") - res = call("kill -TERM $(pidof clickhouse)", shell=True, stderr=STDOUT) + res = call("kill -TERM $(pidof gdb)", shell=True, stderr=STDOUT) logging.info("Checking if some queries hung") cmd = "{} {} {}".format(args.test_cmd, "--hung-check", "00001_select_1") res = call(cmd, shell=True, stderr=STDOUT) diff --git a/docker/test/unit/run.sh b/docker/test/unit/run.sh index 037fd8494cb..abc35fa40d2 100644 --- a/docker/test/unit/run.sh +++ b/docker/test/unit/run.sh @@ -4,4 +4,4 @@ set -x service zookeeper start && sleep 7 && /usr/share/zookeeper/bin/zkCli.sh -server localhost:2181 -create create /clickhouse_test ''; gdb -q -ex 'set print inferior-events off' -ex 'set confirm off' -ex 'set print thread-events off' -ex run -ex bt -ex quit --args ./unit_tests_dbms | tee test_output/test_result.txt -./process_unit_tests_result.py +./process_unit_tests_result.py || echo -e "failure\tCannot parse results" > /test_output/check_status.tsv diff --git a/tests/clickhouse-test b/tests/clickhouse-test index 4044d29a49e..aeae7024250 100755 --- a/tests/clickhouse-test +++ b/tests/clickhouse-test @@ -223,7 +223,7 @@ def get_stacktraces_from_clickhouse(client): return subprocess.check_output("{} --allow_introspection_functions=1 --query " "\"SELECT arrayStringConcat(arrayMap(x, y -> concat(x, ': ', y), arrayMap(x -> addressToLine(x), trace), " "arrayMap(x -> demangle(addressToSymbol(x)), trace)), '\n') as trace " - "FROM system.stack_trace format Vertical\"".format(client), shell=True).decode('utf-8') + "FROM system.stack_trace format Vertical\"".format(client), shell=True, stderr=subprocess.STDOUT).decode('utf-8') except Exception as ex: print("Error occured while receiving stack traces from client: {}".format(str(ex))) return None From 72b67c07c4cf7d26220e9375915065946947d625 Mon Sep 17 00:00:00 2001 From: tavplubix Date: Mon, 22 Feb 2021 17:43:06 +0300 Subject: [PATCH 030/544] trigger CI --- docker/test/stateless/process_functional_tests_result.py | 1 + 1 file changed, 1 insertion(+) diff --git a/docker/test/stateless/process_functional_tests_result.py b/docker/test/stateless/process_functional_tests_result.py index e00243b3a5a..27210ef9b80 100755 --- a/docker/test/stateless/process_functional_tests_result.py +++ b/docker/test/stateless/process_functional_tests_result.py @@ -102,6 +102,7 @@ def write_results(results_file, status_file, results, status): out = csv.writer(f, delimiter='\t') out.writerow(status) + if __name__ == "__main__": logging.basicConfig(level=logging.INFO, format='%(asctime)s %(message)s') parser = argparse.ArgumentParser(description="ClickHouse script for parsing results of functional tests") From d5ea345aa9df9c77c95f9e5952251884407ae663 Mon Sep 17 00:00:00 2001 From: Alexander Tokmakov Date: Tue, 23 Feb 2021 10:24:51 +0300 Subject: [PATCH 031/544] add integration tests runner --- tests/integration/ci-runner.py | 426 +++++++++++++++++++++++++++++++++ 1 file changed, 426 insertions(+) create mode 100755 tests/integration/ci-runner.py diff --git a/tests/integration/ci-runner.py b/tests/integration/ci-runner.py new file mode 100755 index 00000000000..4806129b691 --- /dev/null +++ b/tests/integration/ci-runner.py @@ -0,0 +1,426 @@ +#!/usr/bin/env python3 + +import logging +import subprocess +import os +import time +import shutil +import pipes +from collections import defaultdict +import requests +import random +import json +import csv + + +MAX_RETRY = 2 +SLEEP_BETWEEN_RETRIES = 5 +CLICKHOUSE_BINARY_PATH = "/usr/bin/clickhouse" +CLICKHOUSE_ODBC_BRIDGE_BINARY_PATH = "/usr/bin/clickhouse-odbc-bridge" + + +def _get_deselect_option(tests): + return ' '.join(['--deselect {}'.format(t) for t in tests]) + + +def parse_test_results_output(fname): + read = False + description_output = [] + with open(fname, 'r') as out: + for line in out: + if read and line.strip() and not line.startswith('=='): + description_output.append(line.strip()) + if 'short test summary info' in line: + read = True + return description_output + + +def get_counters(output): + counters = { + "ERROR": set([]), + "PASSED": set([]), + "FAILED": set([]), + } + + for line in output: + if '.py' in line: + line_arr = line.strip().split(' ') + state = line_arr[0] + test_name = ' '.join(line_arr[1:]) + if ' - ' in test_name: + test_name = test_name[:test_name.find(' - ')] + if state in counters: + counters[state].add(test_name) + else: + logging.info("Strange line %s", line) + else: + logging.info("Strange line %s") + return {k: list(v) for k, v in counters.items()} + + +def parse_test_times(fname): + read = False + description_output = [] + with open(fname, 'r') as out: + for line in out: + if read and '==' in line: + break + if read and line.strip(): + description_output.append(line.strip()) + if 'slowest durations' in line: + read = True + return description_output + + +def get_test_times(output): + result = defaultdict(float) + for line in output: + if '.py' in line: + line_arr = line.strip().split(' ') + test_time = line_arr[0] + test_name = ' '.join([elem for elem in line_arr[2:] if elem]) + if test_name not in result: + result[test_name] = 0.0 + result[test_name] += float(test_time[:-1]) + return result + + +def clear_ip_tables_and_restart_daemons(): + logging.info("Dump iptables after run %s", subprocess.check_output("iptables -L", shell=True)) + try: + logging.info("Killing all alive docker containers") + subprocess.check_output("docker kill $(docker ps -q)", shell=True) + except subprocess.CalledProcessError as err: + logging.info("docker kill excepted: " + str(err)) + + try: + logging.info("Removing all docker containers") + subprocess.check_output("docker rm $(docker ps -a -q) --force", shell=True) + except subprocess.CalledProcessError as err: + logging.info("docker rm excepted: " + str(err)) + + try: + logging.info("Stopping docker daemon") + subprocess.check_output("service docker stop", shell=True) + except subprocess.CalledProcessError as err: + logging.info("docker stop excepted: " + str(err)) + + try: + for i in range(200): + try: + logging.info("Restarting docker %s", i) + subprocess.check_output("service docker start", shell=True) + subprocess.check_output("docker ps", shell=True) + break + except subprocess.CalledProcessError as err: + time.sleep(0.5) + logging.info("Waiting docker to start, current %s", str(err)) + else: + raise Exception("Docker daemon doesn't responding") + except subprocess.CalledProcessError as err: + logging.info("Can't reload docker: " + str(err)) + + try: + for i in xrange(1000): + subprocess.check_call("iptables -D DOCKER-USER 1", shell=True) # when rules will be empty, it will raise exception + except: + logging.info("All iptables rules cleared") + + +class ClickhouseIntegrationTestsRunner: + + def __init__(self, result_path, image_versions, shuffle_groups): + self.result_path = result_path + self.image_versions = image_versions + self.shuffle_groups = shuffle_groups + + def path(self): + return self.result_path + + def should_skip_tests(self): + return [] + + def get_image_with_version(self, name): + if name in self.image_versions: + return name + ":" + self.image_versions[name] + logging.warn("Cannot find image %s in params list %s", name, self.image_versions) + if ':' not in name: + return name + ":latest" + return name + + def get_single_image_version(self): + name = self.get_images_names()[0] + if name in self.image_versions: + return self.image_versions[name] + logging.warn("Cannot find image %s in params list %s", name, self.image_versions) + return 'latest' + + def shuffle_test_groups(self): + return self.shuffle_groups != 0 + + @staticmethod + def get_images_names(): + return ["yandex/clickhouse-integration-tests-runner", "yandex/clickhouse-mysql-golang-client", + "yandex/clickhouse-mysql-java-client", "yandex/clickhouse-mysql-js-client", + "yandex/clickhouse-mysql-php-client", "yandex/clickhouse-postgresql-java-client", + "yandex/clickhouse-integration-test", "yandex/clickhouse-kerberos-kdc", + "yandex/clickhouse-integration-helper", ] + + + def _can_run_with(self, path, opt): + with open(path, 'r') as script: + for line in script: + if opt in line: + return True + return False + + def _install_clickhouse(self, debs_path): + for package in ('clickhouse-common-static_', 'clickhouse-server_', 'clickhouse-client', 'clickhouse-common-static-dbg_'): # order matters + logging.info("Installing package %s", package) + for f in os.listdir(debs_path): + if package in f: + full_path = os.path.join(debs_path, f) + logging.info("Package found in %s", full_path) + log_name = "install_" + f + ".log" + log_path = os.path.join(str(self.path()), log_name) + with open(log_path, 'w') as log: + cmd = "dpkg -i {}".format(full_path) + logging.info("Executing installation cmd %s", cmd) + retcode = subprocess.Popen(cmd, shell=True, stderr=log, stdout=log).wait() + if retcode == 0: + logging.info("Instsallation of %s successfull", full_path) + else: + raise Exception("Installation of %s failed", full_path) + break + else: + raise Exception("Package with {} not found".format(package)) + logging.info("Unstripping binary") + # logging.info("Unstring %s", subprocess.check_output("eu-unstrip /usr/bin/clickhouse {}".format(CLICKHOUSE_BINARY_PATH), shell=True)) + + logging.info("All packages installed") + os.chmod(CLICKHOUSE_BINARY_PATH, 0o777) + os.chmod(CLICKHOUSE_ODBC_BRIDGE_BINARY_PATH, 0o777) + result_path_bin = os.path.join(str(self.path()), "clickhouse") + result_path_bridge = os.path.join(str(self.path()), "clickhouse-odbc-bridge") + shutil.copy(CLICKHOUSE_BINARY_PATH, result_path_bin) + shutil.copy(CLICKHOUSE_ODBC_BRIDGE_BINARY_PATH, result_path_bridge) + return None, None + + def _compress_logs(self, path, result_path): + subprocess.check_call("tar czf {} -C {} .".format(result_path, path), shell=True) + + def _get_all_tests(self, repo_path): + image_cmd = self._get_runner_image_cmd(repo_path) + cmd = "cd {}/tests/integration && ./runner {} ' --setup-plan' | grep '::' | sed 's/ (fixtures used:.*//g' | sed 's/^ *//g' > all_tests.txt".format(repo_path, image_cmd) + logging.info("Getting all tests with cmd '%s'", cmd) + subprocess.check_call(cmd, shell=True) + + all_tests_file_path = "{}/tests/integration/all_tests.txt".format(repo_path) + if not os.path.isfile(all_tests_file_path) or os.path.getsize(all_tests_file_path) == 0: + raise Exception("There is something wrong with getting all tests list: file '{}' is empty or does not exist.".format(all_tests_file_path)) + + all_tests = [] + with open(all_tests_file_path, "r") as all_tests_file: + for line in all_tests_file: + all_tests.append(line.strip()) + return list(sorted(all_tests)) + + def group_test_by_file(self, tests): + result = {} + for test in tests: + test_file = test.split('::')[0] + if test_file not in result: + result[test_file] = [] + result[test_file].append(test) + return result + + def _update_counters(self, main_counters, current_counters): + for test in current_counters["PASSED"]: + if test not in main_counters["PASSED"]: + if test in main_counters["FAILED"]: + main_counters["FAILED"].remove(test) + if test in main_counters["ERROR"]: + main_counters["ERROR"].remove(test) + main_counters["PASSED"].append(test) + + for state in ("ERROR", "FAILED"): + for test in current_counters[state]: + if test in main_counters["PASSED"]: + continue + if test not in main_counters[state]: + main_counters[state].append(test) + + def _get_runner_image_cmd(self, repo_path): + image_cmd = '' + if self._can_run_with(os.path.join(repo_path, "tests/integration", "runner"), '--docker-image-version'): + for img in self.get_images_names(): + if img == "yandex/clickhouse-integration-tests-runner": + runner_version = self.get_single_image_version() + logging.info("Can run with custom docker image version %s", runner_version) + image_cmd += ' --docker-image-version={} '.format(runner_version) + else: + if self._can_run_with(os.path.join(repo_path, "tests/integration", "runner"), '--docker-compose-images-tags'): + image_cmd += '--docker-compose-images-tags={} '.format(self.get_image_with_version(img)) + else: + image_cmd = '' + logging.info("Cannot run with custom docker image version :(") + return image_cmd + + def run_test_group(self, repo_path, test_group, tests_in_group, num_tries): + image_cmd = self._get_runner_image_cmd(repo_path) + counters = { + "ERROR": [], + "PASSED": [], + "FAILED": [], + } + tests_times = defaultdict(float) + test_group_str = test_group.replace('/', '_').replace('.', '_') + + for i in range(num_tries): + logging.info("Running test group %s for the %s retry", test_group, i) + clear_ip_tables_and_restart_daemons() + + output_path = os.path.join(str(self.path()), "test_output_" + test_group_str + "_" + str(i) + ".log") + log_name = "integration_run_" + test_group_str + "_" + str(i) + ".txt" + log_path = os.path.join(str(self.path()), log_name) + logging.info("Will wait output inside %s", output_path) + + test_names = set([]) + for test_name in tests_in_group: + if test_name not in counters["PASSED"]: + if '[' in test_name: + test_names.add(test_name[:test_name.find('[')]) + else: + test_names.add(test_name) + + test_cmd = ' '.join([test for test in sorted(test_names)]) + cmd = "cd {}/tests/integration && ./runner {} '-ss {} -rfEp --color=no --durations=0 {}' | tee {}".format( + repo_path, image_cmd, test_cmd, _get_deselect_option(self.should_skip_tests()), output_path) + + with open(log_path, 'w') as log: + logging.info("Executing cmd: %s", cmd) + retcode = subprocess.Popen(cmd, shell=True, stderr=log, stdout=log).wait() + if retcode == 0: + logging.info("Run %s group successfully", test_group) + else: + logging.info("Some tests failed") + + if os.path.exists(output_path): + lines = parse_test_results_output(output_path) + new_counters = get_counters(lines) + times_lines = parse_test_times(output_path) + new_tests_times = get_test_times(times_lines) + self._update_counters(counters, new_counters) + for test_name, test_time in new_tests_times.items(): + tests_times[test_name] = test_time + if len(counters["PASSED"]) == len(tests_in_group): + logging.info("All tests from group %s passed", test_group) + break + if len(counters["PASSED"]) >= 0 and len(counters["FAILED"]) == 0 and len(counters["ERROR"]) == 0: + logging.info("Seems like all tests passed but some of them are skipped or deselected. Ignoring them and finishing group.") + break + else: + for test in tests_in_group: + if test not in counters["PASSED"] and test not in counters["ERROR"] and test not in counters["FAILED"]: + counters["ERROR"].append(test) + + return counters, tests_times, log_name, log_path + + def run_impl(self, commit, repo, pull_request, repo_path, build_path): + self._install_clickhouse(build_path) + logging.info("Dump iptables before run %s", subprocess.check_output("iptables -L", shell=True)) + all_tests = self._get_all_tests(repo_path) + logging.info("Found %s tests first 3 %s", len(all_tests), ' '.join(all_tests[:3])) + grouped_tests = self.group_test_by_file(all_tests) + logging.info("Found %s tests groups", len(grouped_tests)) + + counters = { + "ERROR": [], + "PASSED": [], + "FAILED": [], + } + tests_times = defaultdict(float) + + logs = [] + items_to_run = list(grouped_tests.items()) + + logging.info("Total test groups %s", len(items_to_run)) + if self.shuffle_test_groups(): + logging.info("Shuffling test groups") + random.shuffle(items_to_run) + + for group, tests in items_to_run: + logging.info("Running test group %s countaining %s tests", group, len(tests)) + group_counters, group_test_times, log_name, log_path = self.run_test_group(repo_path, group, tests, MAX_RETRY) + total_tests = 0 + for counter, value in group_counters.items(): + logging.info("Tests from group %s stats, %s count %s", group, counter, len(value)) + counters[counter] += value + logging.info("Totally have %s with status %s", len(counters[counter]), counter) + total_tests += len(counters[counter]) + logging.info("Totally finished tests %s/%s", total_tests, len(all_tests)) + + for test_name, test_time in group_test_times.items(): + tests_times[test_name] = test_time + logs.append(log_path) + if len(counters["FAILED"]) + len(counters["ERROR"]) >= 20: + logging.info("Collected more than 20 failed/error tests, stopping") + break + + logging.info("Finally all tests done, going to compress test dir") + test_logs = os.path.join(str(self.path()), "./test_dir.tar") + self._compress_logs("{}/tests/integration".format(repo_path), test_logs) + logging.info("Compression finished") + + if counters["FAILED"] or counters["ERROR"]: + logging.info("Overall status failure, because we have tests in FAILED or ERROR state") + result_state = "failure" + else: + logging.info("Overall success!") + result_state = "success" + + test_result = [] + for state in ("ERROR", "FAILED", "PASSED"): + if state == "PASSED": + text_state = "OK" + elif state == "FAILED": + text_state = "FAIL" + else: + text_state = state + test_result += [(c, text_state, str(tests_times[c])) for c in counters[state]] + + status_text = ', '.join([str(n).lower().replace('failed', 'fail') + ': ' + str(len(c)) for n, c in counters.items()]) + + if not counters or sum(len(counter) for counter in counters.values()) == 0: + status_text = "No tests found for some reason! It's a bug" + result_state = "failure" + + return result_state, status_text, test_result, [test_logs] + logs + +def write_results(results_file, status_file, results, status): + with open(results_file, 'w') as f: + out = csv.writer(f, delimiter='\t') + out.writerows(results) + with open(status_file, 'w') as f: + out = csv.writer(f, delimiter='\t') + out.writerow(status) + +if __name__ == "__main__": + logging.basicConfig(level=logging.INFO, format='%(asctime)s %(message)s') + + repo_path = os.environ.get("CLICKHOUSE_TESTS_REPO_PATH") + build_path = os.environ.get("CLICKHOUSE_TESTS_BUILD_PATH") + result_path = os.environ.get("CLICKHOUSE_TESTS_RESULT_PATH") + image_versions = os.environ.get("CLICKHOUSE_TESTS_IMAGE_VERSIONS", '{}') + shuffle_groups = int(os.environ.get("SHUFFLE_TEST_GROUPS", '0')) + + runner = ClickhouseIntegrationTestsRunner(result_path, json.loads(image_versions), shuffle_groups) + + logging.info("Running tests") + state, description, test_results, logs = runner.run_impl(None, None, None, repo_path, build_path) + logging.info("Tests finished") + status = (state, description) + out_results_file = os.path.join(str(runner.path()), "test_results.tsv") + out_status_file = os.path.join(str(runner.path()), "check_status.tsv") + write_results(out_results_file, out_status_file, test_results, status) + logging.info("Result written") From 7bcec2947f37ee42d3428e7903b265cda44b2791 Mon Sep 17 00:00:00 2001 From: Alexander Tokmakov Date: Tue, 23 Feb 2021 13:48:06 +0300 Subject: [PATCH 032/544] fix --- tests/integration/ci-runner.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/tests/integration/ci-runner.py b/tests/integration/ci-runner.py index 4806129b691..92a5d94d8d8 100755 --- a/tests/integration/ci-runner.py +++ b/tests/integration/ci-runner.py @@ -5,9 +5,7 @@ import subprocess import os import time import shutil -import pipes from collections import defaultdict -import requests import random import json import csv From 7a6c9ceeef8cd02cea5dfe6ace04291726e79a90 Mon Sep 17 00:00:00 2001 From: Alexander Tokmakov Date: Tue, 23 Feb 2021 15:25:28 +0300 Subject: [PATCH 033/544] fix --- tests/integration/ci-runner.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/tests/integration/ci-runner.py b/tests/integration/ci-runner.py index 92a5d94d8d8..d6128cdfbc6 100755 --- a/tests/integration/ci-runner.py +++ b/tests/integration/ci-runner.py @@ -135,6 +135,9 @@ class ClickhouseIntegrationTestsRunner: def path(self): return self.result_path + def base_path(self): + return os.path.join(str(self.result_path), '../') + def should_skip_tests(self): return [] @@ -198,8 +201,8 @@ class ClickhouseIntegrationTestsRunner: logging.info("All packages installed") os.chmod(CLICKHOUSE_BINARY_PATH, 0o777) os.chmod(CLICKHOUSE_ODBC_BRIDGE_BINARY_PATH, 0o777) - result_path_bin = os.path.join(str(self.path()), "clickhouse") - result_path_bridge = os.path.join(str(self.path()), "clickhouse-odbc-bridge") + result_path_bin = os.path.join(str(self.base_path()), "clickhouse") + result_path_bridge = os.path.join(str(self.base_path()), "clickhouse-odbc-bridge") shutil.copy(CLICKHOUSE_BINARY_PATH, result_path_bin) shutil.copy(CLICKHOUSE_ODBC_BRIDGE_BINARY_PATH, result_path_bridge) return None, None From f41e68d118379b52460ac39cc4abdf10e8db479d Mon Sep 17 00:00:00 2001 From: Alexander Tokmakov Date: Tue, 23 Feb 2021 23:15:11 +0300 Subject: [PATCH 034/544] fix style check --- tests/integration/ci-runner.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/tests/integration/ci-runner.py b/tests/integration/ci-runner.py index d6128cdfbc6..80a283b009e 100755 --- a/tests/integration/ci-runner.py +++ b/tests/integration/ci-runner.py @@ -120,7 +120,8 @@ def clear_ip_tables_and_restart_daemons(): try: for i in xrange(1000): - subprocess.check_call("iptables -D DOCKER-USER 1", shell=True) # when rules will be empty, it will raise exception + # when rules will be empty, it will raise exception + subprocess.check_call("iptables -D DOCKER-USER 1", shell=True) # STYLE_CHECK_ALLOW_SUBPROCESS_CHECK_CALL except: logging.info("All iptables rules cleared") @@ -208,13 +209,13 @@ class ClickhouseIntegrationTestsRunner: return None, None def _compress_logs(self, path, result_path): - subprocess.check_call("tar czf {} -C {} .".format(result_path, path), shell=True) + subprocess.check_call("tar czf {} -C {} .".format(result_path, path), shell=True) # STYLE_CHECK_ALLOW_SUBPROCESS_CHECK_CALL def _get_all_tests(self, repo_path): image_cmd = self._get_runner_image_cmd(repo_path) cmd = "cd {}/tests/integration && ./runner {} ' --setup-plan' | grep '::' | sed 's/ (fixtures used:.*//g' | sed 's/^ *//g' > all_tests.txt".format(repo_path, image_cmd) logging.info("Getting all tests with cmd '%s'", cmd) - subprocess.check_call(cmd, shell=True) + subprocess.check_call(cmd, shell=True) # STYLE_CHECK_ALLOW_SUBPROCESS_CHECK_CALL all_tests_file_path = "{}/tests/integration/all_tests.txt".format(repo_path) if not os.path.isfile(all_tests_file_path) or os.path.getsize(all_tests_file_path) == 0: From 2d03d330bcc400a0b61c8028b01587de072aa60e Mon Sep 17 00:00:00 2001 From: Vasily Nemkov Date: Fri, 17 Apr 2020 16:26:44 +0300 Subject: [PATCH 035/544] Extended range of DateTime64 to years 1925 - 2238 The Year 1925 is a starting point because most of the timezones switched to saner (mostly 15-minutes based) offsets somewhere during 1924 or before. And that significantly simplifies implementation. 2238 is to simplify arithmetics for sanitizing LUT index access; there are less than 0x1ffff days from 1925. * Extended DateLUTImpl internal LUT to 0x1ffff items, some of which represent negative (pre-1970) time values. As a collateral benefit, Date now correctly supports dates up to 2149 (instead of 2106). * Added a new strong typedef ExtendedDayNum, which represents dates pre-1970 and post 2149. * Functions that used to return DayNum now return ExtendedDayNum. * Refactored DateLUTImpl to untie DayNum from the dual role of being a value and an index (due to negative time). Index is now a different type LUTIndex with explicit conversion functions from DatNum, time_t, and ExtendedDayNum. * Updated DateLUTImpl to properly support values close to epoch start (1970-01-01 00:00), including negative ones. * Reduced resolution of DateLUTImpl::Values::time_at_offset_change to multiple of 15-minutes to allow storing 64-bits of time_t in DateLUTImpl::Value while keeping same size. * Minor performance updates to DateLUTImpl when building month LUT by skipping non-start-of-month days. * Fixed extractTimeZoneFromFunctionArguments to work correctly with DateTime64. * New unit-tests and stateless integration tests for both DateTime and DateTime64. --- base/common/DateLUT.h | 1 - base/common/DateLUTImpl.cpp | 42 +- base/common/DateLUTImpl.h | 683 +++++++++++------- base/common/DayNum.h | 5 + base/common/LocalDate.h | 3 +- base/common/strong_typedef.h | 1 + base/common/tests/CMakeLists.txt | 4 +- base/common/tests/gtest_DateLutImpl.cpp | 515 +++++++++++++ programs/client/Client.cpp | 2 +- src/Core/DecimalFunctions.h | 18 +- src/Core/MySQL/MySQLReplication.cpp | 8 +- src/Core/tests/gtest_DecimalFunctions.cpp | 2 +- src/DataStreams/MongoDBBlockInputStream.cpp | 4 +- src/DataTypes/DataTypeDateTime64.h | 61 -- src/Functions/CustomWeekTransforms.h | 24 +- src/Functions/DateTimeTransforms.h | 225 +++++- src/Functions/FunctionCustomWeekToSomething.h | 1 + .../FunctionDateOrDateTimeAddInterval.h | 143 ++-- .../FunctionDateOrDateTimeToSomething.h | 3 +- src/Functions/FunctionsConversion.h | 106 +-- src/Functions/TransformDateTime64.h | 92 +++ src/Functions/dateDiff.cpp | 4 +- .../extractTimeZoneFromFunctionArguments.cpp | 5 +- src/Functions/formatDateTime.cpp | 6 +- src/Functions/now64.cpp | 2 +- src/Functions/toStartOfInterval.cpp | 50 +- src/Functions/today.cpp | 2 +- src/IO/ReadHelpers.h | 10 +- src/IO/WriteHelpers.h | 16 +- src/IO/parseDateTimeBestEffort.cpp | 15 +- src/IO/parseDateTimeBestEffort.h | 1 + src/Interpreters/CrashLog.cpp | 2 +- src/Interpreters/MetricLog.cpp | 2 +- src/Interpreters/OpenTelemetrySpanLog.cpp | 2 +- src/Interpreters/PartLog.cpp | 2 +- src/Interpreters/QueryLog.cpp | 2 +- src/Interpreters/QueryThreadLog.cpp | 2 +- src/Interpreters/TextLog.cpp | 2 +- src/Interpreters/TraceLog.cpp | 2 +- src/Interpreters/convertFieldToType.cpp | 2 +- src/Storages/StorageReplicatedMergeTree.cpp | 2 +- src/Storages/tests/part_name.cpp | 2 +- .../00921_datetime64_compatibility.python | 5 +- .../00921_datetime64_compatibility.reference | 152 ---- .../00921_datetime64_compatibility.sh | 2 +- .../01252_weird_time_zone.reference | 14 +- .../0_stateless/01252_weird_time_zone.sql | 28 +- .../01440_to_date_monotonicity.reference | 2 +- .../01561_Date_and_DateTime64_comparision.sql | 6 +- ...1_date_overflow_as_partition_key.reference | 4 +- .../01631_date_overflow_as_partition_key.sql | 2 +- .../01691_DateTime64_clamp.reference | 23 + .../0_stateless/01691_DateTime64_clamp.sql | 7 + .../convert-month-partitioned-parts/main.cpp | 5 +- 54 files changed, 1585 insertions(+), 741 deletions(-) create mode 100644 base/common/tests/gtest_DateLutImpl.cpp create mode 100644 src/Functions/TransformDateTime64.h diff --git a/base/common/DateLUT.h b/base/common/DateLUT.h index 93c6cb403e2..378b4360f3b 100644 --- a/base/common/DateLUT.h +++ b/base/common/DateLUT.h @@ -32,7 +32,6 @@ public: return date_lut.getImplementation(time_zone); } - static void setDefaultTimezone(const std::string & time_zone) { auto & date_lut = getInstance(); diff --git a/base/common/DateLUTImpl.cpp b/base/common/DateLUTImpl.cpp index 50620e21b8f..906f88fa90f 100644 --- a/base/common/DateLUTImpl.cpp +++ b/base/common/DateLUTImpl.cpp @@ -46,19 +46,26 @@ DateLUTImpl::DateLUTImpl(const std::string & time_zone_) if (&inside_main) assert(inside_main); - size_t i = 0; - time_t start_of_day = 0; cctz::time_zone cctz_time_zone; if (!cctz::load_time_zone(time_zone, &cctz_time_zone)) throw Poco::Exception("Cannot load time zone " + time_zone_); - cctz::time_zone::absolute_lookup start_of_epoch_lookup = cctz_time_zone.lookup(std::chrono::system_clock::from_time_t(start_of_day)); - offset_at_start_of_epoch = start_of_epoch_lookup.offset; + const cctz::civil_day epoch{1970, 1, 1}; + const cctz::civil_day lut_start{DATE_LUT_MIN_YEAR, 1, 1}; + time_t start_of_day = std::chrono::system_clock::to_time_t(cctz_time_zone.lookup(lut_start).pre); + time_offset_epoch = cctz::convert(cctz::civil_second(lut_start), cctz_time_zone).time_since_epoch().count(); + + // Note validated this against all timezones in the system. + assert((epoch - lut_start) == daynum_offset_epoch); + + offset_at_start_of_epoch = cctz_time_zone.lookup(cctz_time_zone.lookup(epoch).pre).offset; + offset_at_start_of_lut = cctz_time_zone.lookup(cctz_time_zone.lookup(lut_start).pre).offset; offset_is_whole_number_of_hours_everytime = true; - cctz::civil_day date{1970, 1, 1}; + cctz::civil_day date = lut_start; + UInt32 i = 0; do { cctz::time_zone::civil_lookup lookup = cctz_time_zone.lookup(date); @@ -72,7 +79,7 @@ DateLUTImpl::DateLUTImpl(const std::string & time_zone_) values.day_of_week = getDayOfWeek(date); values.date = start_of_day; - assert(values.year >= DATE_LUT_MIN_YEAR && values.year <= DATE_LUT_MAX_YEAR); + assert(values.year >= DATE_LUT_MIN_YEAR && values.year <= DATE_LUT_MAX_YEAR + 1); assert(values.month >= 1 && values.month <= 12); assert(values.day_of_month >= 1 && values.day_of_month <= 31); assert(values.day_of_week >= 1 && values.day_of_week <= 7); @@ -85,10 +92,13 @@ DateLUTImpl::DateLUTImpl(const std::string & time_zone_) else values.days_in_month = i != 0 ? lut[i - 1].days_in_month : 31; - values.time_at_offset_change = 0; - values.amount_of_offset_change = 0; + values.time_at_offset_change_value = 0; + values.amount_of_offset_change_value = 0; - if (start_of_day % 3600) + // TODO: this partially ignores fractional pre-epoch offsets, which may cause incorrect toRelativeHourNum() results for some timezones, namelly Europe\Minsk + // when pre-May 2 1924 it had an offset of UTC+1:50, and after it was UTC+2h. + // https://www.timeanddate.com/time/zone/belarus/minsk?syear=1900 + if (start_of_day > 0 && start_of_day % 3600) offset_is_whole_number_of_hours_everytime = false; /// If UTC offset was changed in previous day. @@ -97,7 +107,7 @@ DateLUTImpl::DateLUTImpl(const std::string & time_zone_) auto amount_of_offset_change_at_prev_day = 86400 - (lut[i].date - lut[i - 1].date); if (amount_of_offset_change_at_prev_day) { - lut[i - 1].amount_of_offset_change = amount_of_offset_change_at_prev_day; + lut[i - 1].amount_of_offset_change_value = amount_of_offset_change_at_prev_day / Values::OffsetChangeFactor; const auto utc_offset_at_beginning_of_day = cctz_time_zone.lookup(std::chrono::system_clock::from_time_t(lut[i - 1].date)).offset; @@ -116,11 +126,11 @@ DateLUTImpl::DateLUTImpl(const std::string & time_zone_) time_at_offset_change += 900; } - lut[i - 1].time_at_offset_change = time_at_offset_change; + lut[i - 1].time_at_offset_change_value = time_at_offset_change / Values::OffsetChangeFactor; - /// We doesn't support cases when time change results in switching to previous day. - if (static_cast(lut[i - 1].time_at_offset_change) + static_cast(lut[i - 1].amount_of_offset_change) < 0) - lut[i - 1].time_at_offset_change = -lut[i - 1].amount_of_offset_change; + /// We don't support cases when time change results in switching to previous day. + if (static_cast(lut[i - 1].time_at_offset_change()) + static_cast(lut[i - 1].amount_of_offset_change()) < 0) + lut[i - 1].time_at_offset_change_value = -lut[i - 1].amount_of_offset_change_value; } } @@ -128,7 +138,9 @@ DateLUTImpl::DateLUTImpl(const std::string & time_zone_) ++date; ++i; } - while (start_of_day <= DATE_LUT_MAX && i <= DATE_LUT_MAX_DAY_NUM); + while (i < DATE_LUT_SIZE && lut[i - 1].year <= DATE_LUT_MAX_YEAR); + +// date_lut_max = start_of_day; /// Fill excessive part of lookup table. This is needed only to simplify handling of overflow cases. while (i < DATE_LUT_SIZE) diff --git a/base/common/DateLUTImpl.h b/base/common/DateLUTImpl.h index 064787fb64e..adfffb04681 100644 --- a/base/common/DateLUTImpl.h +++ b/base/common/DateLUTImpl.h @@ -11,9 +11,9 @@ #define DATE_LUT_MAX (0xFFFFFFFFU - 86400) #define DATE_LUT_MAX_DAY_NUM (0xFFFFFFFFU / 86400) /// Table size is bigger than DATE_LUT_MAX_DAY_NUM to fill all indices within UInt16 range: this allows to remove extra check. -#define DATE_LUT_SIZE 0x10000 -#define DATE_LUT_MIN_YEAR 1970 -#define DATE_LUT_MAX_YEAR 2106 /// Last supported year (incomplete) +#define DATE_LUT_SIZE 0x20000 +#define DATE_LUT_MIN_YEAR 1925 /// 1925 since wast majority of timezones changed to 15-minute aligned offsets somewhere in 1924 or earlier. +#define DATE_LUT_MAX_YEAR 2283 /// Last supported year (complete) #define DATE_LUT_YEARS (1 + DATE_LUT_MAX_YEAR - DATE_LUT_MIN_YEAR) /// Number of years in lookup table #if defined(__PPC__) @@ -45,14 +45,68 @@ public: DateLUTImpl(const DateLUTImpl &&) = delete; DateLUTImpl & operator=(const DateLUTImpl &&) = delete; + // Normalized and bound-checked index of element in lut, + // has to be a separate type to support overloading + // TODO: make sure that any arithmetic on LUTIndex actually results in valid LUTIndex. + STRONG_TYPEDEF(UInt32, LUTIndex) + template + friend inline LUTIndex operator+(const LUTIndex & index, const T v) + { + return LUTIndex{(index.toUnderType() + v) & date_lut_mask}; + } + template + friend inline LUTIndex operator+(const T v, const LUTIndex & index) + { + return LUTIndex{(v + index.toUnderType()) & date_lut_mask}; + } + friend inline LUTIndex operator+(const LUTIndex & index, const LUTIndex & v) + { + return LUTIndex{(index.toUnderType() + v.toUnderType()) & date_lut_mask}; + } + + template + friend inline LUTIndex operator-(const LUTIndex & index, const T v) + { + return LUTIndex{(index.toUnderType() - v) & date_lut_mask}; + } + template + friend inline LUTIndex operator-(const T v, const LUTIndex & index) + { + return LUTIndex{(v - index.toUnderType()) & date_lut_mask}; + } + friend inline LUTIndex operator-(const LUTIndex & index, const LUTIndex & v) + { + return LUTIndex{(index.toUnderType() - v.toUnderType()) & date_lut_mask}; + } + + template + friend inline LUTIndex operator*(const LUTIndex & index, const T v) + { + return LUTIndex{(index.toUnderType() * v) & date_lut_mask}; + } + template + friend inline LUTIndex operator*(const T v, const LUTIndex & index) + { + return LUTIndex{(v * index.toUnderType()) & date_lut_mask}; + } + + template + friend inline LUTIndex operator/(const LUTIndex & index, const T v) + { + return LUTIndex{(index.toUnderType() / v) & date_lut_mask}; + } + template + friend inline LUTIndex operator/(const T v, const LUTIndex & index) + { + return LUTIndex{(v / index.toUnderType()) & date_lut_mask}; + } + public: /// The order of fields matters for alignment and sizeof. struct Values { - /// Least significat 32 bits from time_t at beginning of the day. - /// If the unix timestamp of beginning of the day is negative (example: 1970-01-01 MSK, where time_t == -10800), then value will overflow. - /// Change to time_t; change constants above; and recompile the sources if you need to support time after 2105 year. - UInt32 date; + /// Least significat 64 bits from time_t at beginning of the day. + Int64 date; /// Properties of the day. UInt16 year; @@ -65,107 +119,175 @@ public: UInt8 days_in_month; /// For days, when offset from UTC was changed due to daylight saving time or permanent change, following values could be non zero. - Int16 amount_of_offset_change; /// Usually -3600 or 3600, but look at Lord Howe Island. - UInt32 time_at_offset_change; /// In seconds from beginning of the day. + Int8 amount_of_offset_change_value; /// Usually -3600 or 3600, but look at Lord Howe Island. multiply by OffsetChangeFactor + UInt8 time_at_offset_change_value; /// In seconds from beginning of the day. multiply by OffsetChangeFactor + + inline Int32 amount_of_offset_change() const + { + return static_cast(amount_of_offset_change_value) * OffsetChangeFactor; + } + + inline UInt32 time_at_offset_change() const + { + return static_cast(time_at_offset_change_value) * OffsetChangeFactor; + } + + /// Since most of the modern timezones have a DST change aligned to 15 minutes, to save as much space as possible inside Value, + /// we are dividing any offset change related value by this factor before setting it to Value, + /// hence it has to be explicitly multiplied back by this factor before being used. + static const UInt16 OffsetChangeFactor = 900; }; static_assert(sizeof(Values) == 16); private: - /// Lookup table is indexed by DayNum. + + // Mask is all-ones to allow efficient protection against overflow. + static const UInt32 date_lut_mask = 0x1ffff; + static_assert(date_lut_mask == DATE_LUT_SIZE - 1); + + const UInt32 daynum_offset_epoch = 16436; // offset to epoch in days (ExtendedDayNum) of the first day in LUT. + + /// Lookup table is indexed by LUTIndex. /// Day nums are the same in all time zones. 1970-01-01 is 0 and so on. /// Table is relatively large, so better not to place the object on stack. /// In comparison to std::vector, plain array is cheaper by one indirection. - Values lut[DATE_LUT_SIZE]; + Values lut[DATE_LUT_SIZE + 1]; - /// Year number after DATE_LUT_MIN_YEAR -> day num for start of year. - DayNum years_lut[DATE_LUT_YEARS]; + /// Year number after DATE_LUT_MIN_YEAR -> LUTIndex in lut for start of year. + LUTIndex years_lut[DATE_LUT_YEARS]; /// Year number after DATE_LUT_MIN_YEAR * month number starting at zero -> day num for first day of month - DayNum years_months_lut[DATE_LUT_YEARS * 12]; + LUTIndex years_months_lut[DATE_LUT_YEARS * 12]; /// UTC offset at beginning of the Unix epoch. The same as unix timestamp of 1970-01-01 00:00:00 local time. time_t offset_at_start_of_epoch; + time_t offset_at_start_of_lut; bool offset_is_whole_number_of_hours_everytime; + time_t time_offset_epoch; /// Time zone name. std::string time_zone; - - /// We can correctly process only timestamps that less DATE_LUT_MAX (i.e. up to 2105 year inclusively) - /// We don't care about overflow. - inline DayNum findIndex(time_t t) const + inline LUTIndex findIndex(time_t t) const { /// First guess. - DayNum guess(t / 86400); + const UInt32 guess = ((t / 86400) + daynum_offset_epoch) & date_lut_mask; /// UTC offset is from -12 to +14 in all known time zones. This requires checking only three indices. - - if ((guess == 0 || t >= lut[guess].date) && t < lut[DayNum(guess + 1)].date) - return guess; + if ((guess == daynum_offset_epoch || t >= lut[guess].date) && t < lut[UInt32(guess + 1)].date) + return LUTIndex{guess}; /// Time zones that have offset 0 from UTC do daylight saving time change (if any) towards increasing UTC offset (example: British Standard Time). - if (t >= lut[DayNum(guess + 1)].date) - return DayNum(guess + 1); + if (t >= lut[UInt32(guess + 1)].date) + return LUTIndex(guess + 1); - return DayNum(guess - 1); + if (lut[guess - 1].date <= t) + return LUTIndex(guess - 1); + return LUTIndex(guess - 2); } - inline const Values & find(time_t t) const + inline LUTIndex toLUTIndex(DayNum d) const { - return lut[findIndex(t)]; + return LUTIndex{(d + daynum_offset_epoch) & date_lut_mask}; + } + + inline LUTIndex toLUTIndex(ExtendedDayNum d) const + { + return LUTIndex{static_cast(d + daynum_offset_epoch) & date_lut_mask}; + } + + inline LUTIndex toLUTIndex(time_t t) const + { + return findIndex(t); + } + + inline LUTIndex toLUTIndex(LUTIndex i) const + { + return i; + } + +// template +// inline LUTIndex toLUTIndex(T t) const +// { +// return LUTIndex{static_cast(t) & date_lut_mask}; +// } + + template + inline const Values & find(V v) const + { + return lut[toLUTIndex(v)]; } public: const std::string & getTimeZone() const { return time_zone; } + // Methods only for unit-testing, it makes very little sense to use it from user code. + auto getOffsetAtStartOfEpoch() const { return offset_at_start_of_epoch; } + auto getOffsetIsWholNumberOfHoursEveryWhere() const { return offset_is_whole_number_of_hours_everytime; } + auto getTimeOffsetEpoch() const { return time_offset_epoch; } + auto getTimeOffsetAtStartOfLUT() const { return offset_at_start_of_lut; } + /// All functions below are thread-safe; arguments are not checked. - inline time_t toDate(time_t t) const { return find(t).date; } - inline unsigned toMonth(time_t t) const { return find(t).month; } - inline unsigned toQuarter(time_t t) const { return (find(t).month - 1) / 3 + 1; } - inline unsigned toYear(time_t t) const { return find(t).year; } - inline unsigned toDayOfWeek(time_t t) const { return find(t).day_of_week; } - inline unsigned toDayOfMonth(time_t t) const { return find(t).day_of_month; } + inline ExtendedDayNum toDayNum(ExtendedDayNum d) const + { + return d; + } + + template + inline ExtendedDayNum toDayNum(V v) const + { + return ExtendedDayNum{static_cast(toLUTIndex(v).toUnderType()) - daynum_offset_epoch}; + } /// Round down to start of monday. - inline time_t toFirstDayOfWeek(time_t t) const + template + inline time_t toFirstDayOfWeek(V v) const { - DayNum index = findIndex(t); - return lut[DayNum(index - (lut[index].day_of_week - 1))].date; + const auto i = toLUTIndex(v); + return lut[i - (lut[i].day_of_week - 1)].date; } - inline DayNum toFirstDayNumOfWeek(DayNum d) const + template + inline ExtendedDayNum toFirstDayNumOfWeek(V v) const { - return DayNum(d - (lut[d].day_of_week - 1)); - } - - inline DayNum toFirstDayNumOfWeek(time_t t) const - { - return toFirstDayNumOfWeek(toDayNum(t)); + const auto i = toLUTIndex(v); + return toDayNum(i - (lut[i].day_of_week - 1)); } /// Round down to start of month. - inline time_t toFirstDayOfMonth(time_t t) const + template + inline time_t toFirstDayOfMonth(V v) const { - DayNum index = findIndex(t); - return lut[index - (lut[index].day_of_month - 1)].date; + const auto i = toLUTIndex(v); + return lut[i - (lut[i].day_of_month - 1)].date; } - inline DayNum toFirstDayNumOfMonth(DayNum d) const + template + inline ExtendedDayNum toFirstDayNumOfMonth(V v) const { - return DayNum(d - (lut[d].day_of_month - 1)); + const auto i = toLUTIndex(v); + return toDayNum(i - (lut[i].day_of_month - 1)); } - inline DayNum toFirstDayNumOfMonth(time_t t) const - { - return toFirstDayNumOfMonth(toDayNum(t)); - } +// inline DayNum toFirstDayNumOfMonth(time_t t) const +// { +// return toFirstDayNumOfMonth(toDayNum(t)); +// } /// Round down to start of quarter. - inline DayNum toFirstDayNumOfQuarter(DayNum d) const + template + inline ExtendedDayNum toFirstDayNumOfQuarter(V v) const { - DayNum index = d; + return toDayNum(toFirstDayOfQuarterIndex(v)); + } + + template + inline LUTIndex toFirstDayOfQuarterIndex(V v) const + { + //return fromDayNum(toFirstDayNumOfQuarter(v)); + auto index = toLUTIndex(v); size_t month_inside_quarter = (lut[index].month - 1) % 3; index -= lut[index].day_of_month; @@ -175,17 +297,13 @@ public: --month_inside_quarter; } - return DayNum(index + 1); + return index + 1; } - inline DayNum toFirstDayNumOfQuarter(time_t t) const + template + inline time_t toFirstDayOfQuarter(V v) const { - return toFirstDayNumOfQuarter(toDayNum(t)); - } - - inline time_t toFirstDayOfQuarter(time_t t) const - { - return fromDayNum(toFirstDayNumOfQuarter(t)); + return toDate(toFirstDayOfQuarterIndex(v)); } /// Round down to start of year. @@ -194,48 +312,47 @@ public: return lut[years_lut[lut[findIndex(t)].year - DATE_LUT_MIN_YEAR]].date; } - inline DayNum toFirstDayNumOfYear(DayNum d) const + template + inline LUTIndex toFirstDayNumOfYearIndex(V v) const { - return years_lut[lut[d].year - DATE_LUT_MIN_YEAR]; + return years_lut[lut[toLUTIndex(v)].year - DATE_LUT_MIN_YEAR]; } - inline DayNum toFirstDayNumOfYear(time_t t) const + template + inline ExtendedDayNum toFirstDayNumOfYear(V v) const { - return toFirstDayNumOfYear(toDayNum(t)); + return toDayNum(toFirstDayNumOfYearIndex(v)); } inline time_t toFirstDayOfNextMonth(time_t t) const { - DayNum index = findIndex(t); + auto index = findIndex(t); index += 32 - lut[index].day_of_month; return lut[index - (lut[index].day_of_month - 1)].date; } inline time_t toFirstDayOfPrevMonth(time_t t) const { - DayNum index = findIndex(t); + auto index = findIndex(t); index -= lut[index].day_of_month; return lut[index - (lut[index].day_of_month - 1)].date; } - inline UInt8 daysInMonth(DayNum d) const + template + inline UInt8 daysInMonth(V v) const { - return lut[d].days_in_month; + const auto i = toLUTIndex(v); + return lut[i].days_in_month; } - inline UInt8 daysInMonth(time_t t) const - { - return find(t).days_in_month; - } - - inline UInt8 daysInMonth(UInt16 year, UInt8 month) const + inline UInt8 daysInMonth(Int16 year, UInt8 month) const { UInt16 idx = year - DATE_LUT_MIN_YEAR; if (unlikely(idx >= DATE_LUT_YEARS)) return 31; /// Implementation specific behaviour on overflow. /// 32 makes arithmetic more simple. - DayNum any_day_of_month = DayNum(years_lut[idx] + 32 * (month - 1)); + const auto any_day_of_month = years_lut[year - DATE_LUT_MIN_YEAR] + 32 * (month - 1); return lut[any_day_of_month].days_in_month; } @@ -243,37 +360,38 @@ public: */ inline time_t toDateAndShift(time_t t, Int32 days) const { - return lut[DayNum(findIndex(t) + days)].date; + return lut[findIndex(t) + days].date; } inline time_t toTime(time_t t) const { - DayNum index = findIndex(t); + auto index = findIndex(t); - if (unlikely(index == 0 || index > DATE_LUT_MAX_DAY_NUM)) + if (unlikely(index == daynum_offset_epoch || index > DATE_LUT_MAX_DAY_NUM)) return t + offset_at_start_of_epoch; time_t res = t - lut[index].date; - if (res >= lut[index].time_at_offset_change) - res += lut[index].amount_of_offset_change; + if (res >= lut[index].time_at_offset_change()) + res += lut[index].amount_of_offset_change(); return res - offset_at_start_of_epoch; /// Starting at 1970-01-01 00:00:00 local time. } inline unsigned toHour(time_t t) const { - DayNum index = findIndex(t); + auto index = findIndex(t); /// If it is overflow case, - /// then limit number of hours to avoid insane results like 1970-01-01 89:28:15 - if (unlikely(index == 0 || index > DATE_LUT_MAX_DAY_NUM)) + /// than limit number of hours to avoid insane results like 1970-01-01 89:28:15 + if (unlikely(index == daynum_offset_epoch || index > DATE_LUT_MAX_DAY_NUM)) return static_cast((t + offset_at_start_of_epoch) / 3600) % 24; time_t time = t - lut[index].date; - if (time >= lut[index].time_at_offset_change) - time += lut[index].amount_of_offset_change; + /// Data is cleaned to avoid possibility of underflow. + if (time >= lut[index].time_at_offset_change()) + time += lut[index].amount_of_offset_change(); unsigned res = time / 3600; return res <= 23 ? res : 0; @@ -286,24 +404,32 @@ public: */ inline time_t timezoneOffset(time_t t) const { - DayNum index = findIndex(t); + const auto index = findIndex(t); /// Calculate daylight saving offset first. /// Because the "amount_of_offset_change" in LUT entry only exists in the change day, it's costly to scan it from the very begin. /// but we can figure out all the accumulated offsets from 1970-01-01 to that day just by get the whole difference between lut[].date, /// and then, we can directly subtract multiple 86400s to get the real DST offsets for the leap seconds is not considered now. - time_t res = (lut[index].date - lut[0].date) % 86400; + time_t res = (lut[index].date - lut[daynum_offset_epoch].date) % 86400; /// As so far to know, the maximal DST offset couldn't be more than 2 hours, so after the modulo operation the remainder /// will sits between [-offset --> 0 --> offset] which respectively corresponds to moving clock forward or backward. res = res > 43200 ? (86400 - res) : (0 - res); /// Check if has a offset change during this day. Add the change when cross the line - if (lut[index].amount_of_offset_change != 0 && t >= lut[index].date + lut[index].time_at_offset_change) - res += lut[index].amount_of_offset_change; + if (lut[index].amount_of_offset_change() != 0 && t >= lut[index].date + lut[index].time_at_offset_change()) + res += lut[index].amount_of_offset_change(); return res + offset_at_start_of_epoch; } + static inline time_t toSecondsSinceTheDayStart(time_t t) + { + t %= 86400; + t = (t < 0 ? t + 86400 : t); + + return t; + } + /** Only for time zones with/when offset from UTC is multiple of five minutes. * This is true for all time zones: right now, all time zones have an offset that is multiple of 15 minutes. * @@ -314,13 +440,15 @@ public: * Also please note, that unix timestamp doesn't count "leap seconds": * each minute, with added or subtracted leap second, spans exactly 60 unix timestamps. */ - - inline unsigned toSecond(time_t t) const { return UInt32(t) % 60; } + inline unsigned toSecond(time_t t) const + { + return toSecondsSinceTheDayStart(t) % 60; + } inline unsigned toMinute(time_t t) const { if (offset_is_whole_number_of_hours_everytime) - return (UInt32(t) / 60) % 60; + return (toSecondsSinceTheDayStart(t) / 60) % 60; UInt32 date = find(t).date; return (UInt32(t) - date) / 60 % 60; @@ -348,80 +476,85 @@ public: * because the same calendar day starts/ends at different timestamps in different time zones) */ - inline DayNum toDayNum(time_t t) const { return findIndex(t); } - inline time_t fromDayNum(DayNum d) const { return lut[d].date; } +// inline DayNum toDayNum(time_t t) const { return DayNum{findIndex(t) - daynum_offset_epoch}; } +// inline ExtendedDayNum toExtendedDayNum(time_t t) const { return ExtendedDayNum{findIndex(t) - daynum_offset_epoch}; } + inline time_t fromDayNum(DayNum d) const { return lut[toLUTIndex(d)].date; } + inline time_t fromDayNum(ExtendedDayNum d) const { return lut[toLUTIndex(d)].date; } - inline time_t toDate(DayNum d) const { return lut[d].date; } - inline unsigned toMonth(DayNum d) const { return lut[d].month; } - inline unsigned toQuarter(DayNum d) const { return (lut[d].month - 1) / 3 + 1; } - inline unsigned toYear(DayNum d) const { return lut[d].year; } - inline unsigned toDayOfWeek(DayNum d) const { return lut[d].day_of_week; } - inline unsigned toDayOfMonth(DayNum d) const { return lut[d].day_of_month; } - inline unsigned toDayOfYear(DayNum d) const { return d + 1 - toFirstDayNumOfYear(d); } - - inline unsigned toDayOfYear(time_t t) const { return toDayOfYear(toDayNum(t)); } + template + inline time_t toDate(V v) const { return lut[toLUTIndex(v)].date; } + template + inline unsigned toMonth(V v) const { return lut[toLUTIndex(v)].month; } + template + inline unsigned toQuarter(V v) const { return (lut[toLUTIndex(v)].month - 1) / 3 + 1; } + template + inline Int16 toYear(V v) const { return lut[toLUTIndex(v)].year; } + template + inline unsigned toDayOfWeek(V v) const { return lut[toLUTIndex(v)].day_of_week; } + template + inline unsigned toDayOfMonth(V v) const { return lut[toLUTIndex(v)].day_of_month; } + template + inline unsigned toDayOfYear(V v) const + { + // TODO: different overload for ExtendedDayNum + const auto i = toLUTIndex(v); + return i + 1 - toFirstDayNumOfYearIndex(i); + } /// Number of week from some fixed moment in the past. Week begins at monday. /// (round down to monday and divide DayNum by 7; we made an assumption, /// that in domain of the function there was no weeks with any other number of days than 7) - inline unsigned toRelativeWeekNum(DayNum d) const + template + inline unsigned toRelativeWeekNum(V v) const { + const auto i = toLUTIndex(v); /// We add 8 to avoid underflow at beginning of unix epoch. - return (d + 8 - toDayOfWeek(d)) / 7; - } - - inline unsigned toRelativeWeekNum(time_t t) const - { - return toRelativeWeekNum(toDayNum(t)); + return toDayNum(i + 8 - toDayOfWeek(i)) / 7; } /// Get year that contains most of the current week. Week begins at monday. - inline unsigned toISOYear(DayNum d) const + template + inline unsigned toISOYear(V v) const { + const auto i = toLUTIndex(v); /// That's effectively the year of thursday of current week. - return toYear(DayNum(d + 4 - toDayOfWeek(d))); - } - - inline unsigned toISOYear(time_t t) const - { - return toISOYear(toDayNum(t)); + return toYear(toLUTIndex(i + 4 - toDayOfWeek(i))); } /// ISO year begins with a monday of the week that is contained more than by half in the corresponding calendar year. /// Example: ISO year 2019 begins at 2018-12-31. And ISO year 2017 begins at 2017-01-02. /// https://en.wikipedia.org/wiki/ISO_week_date - inline DayNum toFirstDayNumOfISOYear(DayNum d) const + template + inline LUTIndex toFirstDayNumOfISOYearIndex(V v) const { - auto iso_year = toISOYear(d); + const auto i = toLUTIndex(v); + auto iso_year = toISOYear(i); - DayNum first_day_of_year = years_lut[iso_year - DATE_LUT_MIN_YEAR]; + const auto first_day_of_year = years_lut[iso_year - DATE_LUT_MIN_YEAR]; auto first_day_of_week_of_year = lut[first_day_of_year].day_of_week; - return DayNum(first_day_of_week_of_year <= 4 + return LUTIndex{first_day_of_week_of_year <= 4 ? first_day_of_year + 1 - first_day_of_week_of_year - : first_day_of_year + 8 - first_day_of_week_of_year); + : first_day_of_year + 8 - first_day_of_week_of_year}; } - inline DayNum toFirstDayNumOfISOYear(time_t t) const + template + inline ExtendedDayNum toFirstDayNumOfISOYear(V v) const { - return toFirstDayNumOfISOYear(toDayNum(t)); + return toDayNum(toFirstDayNumOfISOYearIndex(v)); } inline time_t toFirstDayOfISOYear(time_t t) const { - return fromDayNum(toFirstDayNumOfISOYear(t)); + return lut[toFirstDayNumOfISOYearIndex(t)].date; } /// ISO 8601 week number. Week begins at monday. /// The week number 1 is the first week in year that contains 4 or more days (that's more than half). - inline unsigned toISOWeek(DayNum d) const + template + inline unsigned toISOWeek(V v) const { - return 1 + DayNum(toFirstDayNumOfWeek(d) - toFirstDayNumOfISOYear(d)) / 7; - } - - inline unsigned toISOWeek(time_t t) const - { - return toISOWeek(toDayNum(t)); + return 1 + (toFirstDayNumOfWeek(v) - toFirstDayNumOfISOYear(v)) / 7; } /* @@ -457,30 +590,33 @@ public: Otherwise it is the last week of the previous year, and the next week is week 1. */ - inline YearWeek toYearWeek(DayNum d, UInt8 week_mode) const + template + inline YearWeek toYearWeek(V v, UInt8 week_mode) const { - bool newyear_day_mode = week_mode & static_cast(WeekModeFlag::NEWYEAR_DAY); + const bool newyear_day_mode = week_mode & static_cast(WeekModeFlag::NEWYEAR_DAY); week_mode = check_week_mode(week_mode); - bool monday_first_mode = week_mode & static_cast(WeekModeFlag::MONDAY_FIRST); + const bool monday_first_mode = week_mode & static_cast(WeekModeFlag::MONDAY_FIRST); bool week_year_mode = week_mode & static_cast(WeekModeFlag::YEAR); - bool first_weekday_mode = week_mode & static_cast(WeekModeFlag::FIRST_WEEKDAY); + const bool first_weekday_mode = week_mode & static_cast(WeekModeFlag::FIRST_WEEKDAY); + + const auto i = toLUTIndex(v); // Calculate week number of WeekModeFlag::NEWYEAR_DAY mode if (newyear_day_mode) { - return toYearWeekOfNewyearMode(d, monday_first_mode); + return toYearWeekOfNewyearMode(i, monday_first_mode); } - YearWeek yw(toYear(d), 0); + YearWeek yw(toYear(i), 0); UInt16 days = 0; - UInt16 daynr = makeDayNum(yw.first, toMonth(d), toDayOfMonth(d)); - UInt16 first_daynr = makeDayNum(yw.first, 1, 1); + const auto daynr = makeDayNum(yw.first, toMonth(i), toDayOfMonth(i)); + auto first_daynr = makeDayNum(yw.first, 1, 1); // 0 for monday, 1 for tuesday ... // get weekday from first day in year. - UInt16 weekday = calc_weekday(DayNum(first_daynr), !monday_first_mode); + UInt16 weekday = calc_weekday(first_daynr, !monday_first_mode); - if (toMonth(d) == 1 && toDayOfMonth(d) <= static_cast(7 - weekday)) + if (toMonth(i) == 1 && toDayOfMonth(i) <= static_cast(7 - weekday)) { if (!week_year_mode && ((first_weekday_mode && weekday != 0) || (!first_weekday_mode && weekday >= 4))) return yw; @@ -511,30 +647,34 @@ public: /// Calculate week number of WeekModeFlag::NEWYEAR_DAY mode /// The week number 1 is the first week in year that contains January 1, - inline YearWeek toYearWeekOfNewyearMode(DayNum d, bool monday_first_mode) const + template + inline YearWeek toYearWeekOfNewyearMode(V v, bool monday_first_mode) const { YearWeek yw(0, 0); UInt16 offset_day = monday_first_mode ? 0U : 1U; + const auto i = LUTIndex(v); + // Checking the week across the year - yw.first = toYear(DayNum(d + 7 - toDayOfWeek(DayNum(d + offset_day)))); + yw.first = toYear(i + 7 - toDayOfWeek(i + offset_day)); - DayNum first_day = makeDayNum(yw.first, 1, 1); - DayNum this_day = d; + auto first_day = makeLUTIndex(yw.first, 1, 1); + auto this_day = i; + //TODO: do not perform calculations in terms of DayNum, since that would under/overflow for extended range. if (monday_first_mode) { // Rounds down a date to the nearest Monday. first_day = toFirstDayNumOfWeek(first_day); - this_day = toFirstDayNumOfWeek(d); + this_day = toFirstDayNumOfWeek(i); } else { // Rounds down a date to the nearest Sunday. if (toDayOfWeek(first_day) != 7) - first_day = DayNum(first_day - toDayOfWeek(first_day)); - if (toDayOfWeek(d) != 7) - this_day = DayNum(d - toDayOfWeek(d)); + first_day = ExtendedDayNum(first_day - toDayOfWeek(first_day)); + if (toDayOfWeek(i) != 7) + this_day = ExtendedDayNum(i - toDayOfWeek(i)); } yw.second = (this_day - first_day) / 7 + 1; return yw; @@ -543,16 +683,17 @@ public: /** * get first day of week with week_mode, return Sunday or Monday */ - inline DayNum toFirstDayNumOfWeek(DayNum d, UInt8 week_mode) const + template + inline ExtendedDayNum toFirstDayNumOfWeek(V v, UInt8 week_mode) const { bool monday_first_mode = week_mode & static_cast(WeekModeFlag::MONDAY_FIRST); if (monday_first_mode) { - return toFirstDayNumOfWeek(d); + return toFirstDayNumOfWeek(v); } else { - return (toDayOfWeek(d) != 7) ? DayNum(d - toDayOfWeek(d)) : d; + return (toDayOfWeek(v) != 7) ? ExtendedDayNum(v - toDayOfWeek(v)) : toDayNum(v); } } @@ -568,39 +709,35 @@ public: /** Calculate weekday from d. * Returns 0 for monday, 1 for tuesday... */ - inline unsigned calc_weekday(DayNum d, bool sunday_first_day_of_week) const + template + inline unsigned calc_weekday(V v, bool sunday_first_day_of_week) const { + const auto i = toLUTIndex(v); if (!sunday_first_day_of_week) - return toDayOfWeek(d) - 1; + return toDayOfWeek(i) - 1; else - return toDayOfWeek(DayNum(d + 1)) - 1; + return toDayOfWeek(i + 1) - 1; } /// Calculate days in one year. - inline unsigned calc_days_in_year(UInt16 year) const + inline unsigned calc_days_in_year(Int32 year) const { return ((year & 3) == 0 && (year % 100 || (year % 400 == 0 && year)) ? 366 : 365); } /// Number of month from some fixed moment in the past (year * 12 + month) - inline unsigned toRelativeMonthNum(DayNum d) const + template + inline unsigned toRelativeMonthNum(V v) const { - return lut[d].year * 12 + lut[d].month; + const auto i = toLUTIndex(v); + return lut[i].year * 12 + lut[i].month; } - inline unsigned toRelativeMonthNum(time_t t) const + template + inline unsigned toRelativeQuarterNum(V v) const { - return toRelativeMonthNum(toDayNum(t)); - } - - inline unsigned toRelativeQuarterNum(DayNum d) const - { - return lut[d].year * 4 + (lut[d].month - 1) / 3; - } - - inline unsigned toRelativeQuarterNum(time_t t) const - { - return toRelativeQuarterNum(toDayNum(t)); + const auto i = toLUTIndex(v); + return lut[i].year * 4 + (lut[i].month - 1) / 3; } /// We count all hour-length intervals, unrelated to offset changes. @@ -614,9 +751,10 @@ public: return (t + 86400 - offset_at_start_of_epoch) / 3600; } - inline time_t toRelativeHourNum(DayNum d) const + template + inline time_t toRelativeHourNum(V v) const { - return toRelativeHourNum(lut[d].date); + return toRelativeHourNum(lut[toLUTIndex(v)].date); } inline time_t toRelativeMinuteNum(time_t t) const @@ -624,48 +762,52 @@ public: return t / 60; } - inline time_t toRelativeMinuteNum(DayNum d) const + template + inline time_t toRelativeMinuteNum(V v) const { - return toRelativeMinuteNum(lut[d].date); + return toRelativeMinuteNum(lut[toLUTIndex(v)].date); } - inline DayNum toStartOfYearInterval(DayNum d, UInt64 years) const + template + inline ExtendedDayNum toStartOfYearInterval(V v, UInt64 years) const { if (years == 1) - return toFirstDayNumOfYear(d); - return years_lut[(lut[d].year - DATE_LUT_MIN_YEAR) / years * years]; + return toFirstDayNumOfYear(v); + + const auto i = toLUTIndex(v); + return toDayNum(years_lut[lut[i].year / years * years - DATE_LUT_MIN_YEAR]); } - inline DayNum toStartOfQuarterInterval(DayNum d, UInt64 quarters) const + inline ExtendedDayNum toStartOfQuarterInterval(ExtendedDayNum d, UInt64 quarters) const { if (quarters == 1) return toFirstDayNumOfQuarter(d); return toStartOfMonthInterval(d, quarters * 3); } - inline DayNum toStartOfMonthInterval(DayNum d, UInt64 months) const + inline ExtendedDayNum toStartOfMonthInterval(ExtendedDayNum d, UInt64 months) const { if (months == 1) return toFirstDayNumOfMonth(d); - const auto & date = lut[d]; + const auto & date = lut[toLUTIndex(d)]; UInt32 month_total_index = (date.year - DATE_LUT_MIN_YEAR) * 12 + date.month - 1; - return years_months_lut[month_total_index / months * months]; + return toDayNum(years_months_lut[month_total_index / months * months]); } - inline DayNum toStartOfWeekInterval(DayNum d, UInt64 weeks) const + inline ExtendedDayNum toStartOfWeekInterval(ExtendedDayNum d, UInt64 weeks) const { if (weeks == 1) return toFirstDayNumOfWeek(d); UInt64 days = weeks * 7; // January 1st 1970 was Thursday so we need this 4-days offset to make weeks start on Monday. - return DayNum(4 + (d - 4) / days * days); + return ExtendedDayNum(4 + (d - 4) / days * days); } - inline time_t toStartOfDayInterval(DayNum d, UInt64 days) const + inline time_t toStartOfDayInterval(ExtendedDayNum d, UInt64 days) const { if (days == 1) return toDate(d); - return lut[d / days * days].date; + return lut[toLUTIndex(ExtendedDayNum(d / days * days))].date; } inline time_t toStartOfHourInterval(time_t t, UInt64 hours) const @@ -694,33 +836,41 @@ public: return t / seconds * seconds; } - /// Create DayNum from year, month, day of month. - inline DayNum makeDayNum(UInt16 year, UInt8 month, UInt8 day_of_month) const + inline LUTIndex makeLUTIndex(Int16 year, UInt8 month, UInt8 day_of_month) const { if (unlikely(year < DATE_LUT_MIN_YEAR || year > DATE_LUT_MAX_YEAR || month < 1 || month > 12 || day_of_month < 1 || day_of_month > 31)) - return DayNum(0); // TODO (nemkov, DateTime64 phase 2): implement creating real date for year outside of LUT range. + return LUTIndex(0); - // The day after 2106-02-07 will not stored fully as struct Values, so just overflow it as 0 - if (unlikely(year == DATE_LUT_MAX_YEAR && (month > 2 || (month == 2 && day_of_month > 7)))) - return DayNum(0); - - return DayNum(years_months_lut[(year - DATE_LUT_MIN_YEAR) * 12 + month - 1] + day_of_month - 1); + return LUTIndex{years_months_lut[(year - DATE_LUT_MIN_YEAR) * 12 + month - 1] + day_of_month - 1}; } - inline time_t makeDate(UInt16 year, UInt8 month, UInt8 day_of_month) const + /// Create DayNum from year, month, day of month. + inline ExtendedDayNum makeDayNum(Int16 year, UInt8 month, UInt8 day_of_month) const { - return lut[makeDayNum(year, month, day_of_month)].date; + if (unlikely(year < DATE_LUT_MIN_YEAR || year > DATE_LUT_MAX_YEAR || month < 1 || month > 12 || day_of_month < 1 || day_of_month > 31)) + return ExtendedDayNum(0); + + // The day after 2283 are not stored fully as struct Values, so just overflow it as 0 + if (unlikely(year > DATE_LUT_MAX_YEAR)) + return ExtendedDayNum(0); + + return toDayNum(makeLUTIndex(year, month, day_of_month)); + } + + inline time_t makeDate(Int16 year, UInt8 month, UInt8 day_of_month) const + { + return lut[makeLUTIndex(year, month, day_of_month)].date; } /** Does not accept daylight saving time as argument: in case of ambiguity, it choose greater timestamp. */ - inline time_t makeDateTime(UInt16 year, UInt8 month, UInt8 day_of_month, UInt8 hour, UInt8 minute, UInt8 second) const + inline time_t makeDateTime(Int16 year, UInt8 month, UInt8 day_of_month, UInt8 hour, UInt8 minute, UInt8 second) const { - size_t index = makeDayNum(year, month, day_of_month); + size_t index = makeLUTIndex(year, month, day_of_month); UInt32 time_offset = hour * 3600 + minute * 60 + second; - if (time_offset >= lut[index].time_at_offset_change) - time_offset -= lut[index].amount_of_offset_change; + if (time_offset >= lut[index].time_at_offset_change()) + time_offset -= lut[index].amount_of_offset_change(); UInt32 res = lut[index].date + time_offset; @@ -730,30 +880,20 @@ public: return res; } - inline const Values & getValues(DayNum d) const { return lut[d]; } - inline const Values & getValues(time_t t) const { return lut[findIndex(t)]; } + template + inline const Values & getValues(V v) const { return lut[toLUTIndex(v)]; } - inline UInt32 toNumYYYYMM(time_t t) const + template + inline UInt32 toNumYYYYMM(V v) const { - const Values & values = find(t); + const Values & values = getValues(v); return values.year * 100 + values.month; } - inline UInt32 toNumYYYYMM(DayNum d) const + template + inline UInt32 toNumYYYYMMDD(V v) const { - const Values & values = lut[d]; - return values.year * 100 + values.month; - } - - inline UInt32 toNumYYYYMMDD(time_t t) const - { - const Values & values = find(t); - return values.year * 10000 + values.month * 100 + values.day_of_month; - } - - inline UInt32 toNumYYYYMMDD(DayNum d) const - { - const Values & values = lut[d]; + const Values & values = getValues(v); return values.year * 10000 + values.month * 100 + values.day_of_month; } @@ -762,7 +902,7 @@ public: return makeDate(num / 10000, num / 100 % 100, num % 100); } - inline DayNum YYYYMMDDToDayNum(UInt32 num) const + inline ExtendedDayNum YYYYMMDDToDayNum(UInt32 num) const { return makeDayNum(num / 10000, num / 100 % 100, num % 100); } @@ -796,13 +936,14 @@ public: inline NO_SANITIZE_UNDEFINED time_t addDays(time_t t, Int64 delta) const { - DayNum index = findIndex(t); + auto index = findIndex(t); time_t time_offset = toHour(t) * 3600 + toMinute(t) * 60 + toSecond(t); index += delta; + index &= date_lut_mask; - if (time_offset >= lut[index].time_at_offset_change) - time_offset -= lut[index].amount_of_offset_change; + if (time_offset >= lut[index].time_at_offset_change()) + time_offset -= lut[index].amount_of_offset_change(); return lut[index].date + time_offset; } @@ -812,7 +953,7 @@ public: return addDays(t, delta * 7); } - inline UInt8 saturateDayOfMonth(UInt16 year, UInt8 month, UInt8 day_of_month) const + inline UInt8 saturateDayOfMonth(Int16 year, UInt8 month, UInt8 day_of_month) const { if (likely(day_of_month <= 28)) return day_of_month; @@ -825,23 +966,10 @@ public: return day_of_month; } - /// If resulting month has less deys than source month, then saturation can happen. - /// Example: 31 Aug + 1 month = 30 Sep. - inline time_t addMonths(time_t t, Int64 delta) const + template + inline LUTIndex addMonthsIndex(V v, Int64 delta) const { - DayNum result_day = addMonths(toDayNum(t), delta); - - time_t time_offset = toHour(t) * 3600 + toMinute(t) * 60 + toSecond(t); - - if (time_offset >= lut[result_day].time_at_offset_change) - time_offset -= lut[result_day].amount_of_offset_change; - - return lut[result_day].date + time_offset; - } - - inline NO_SANITIZE_UNDEFINED DayNum addMonths(DayNum d, Int64 delta) const - { - const Values & values = lut[d]; + const Values & values = lut[toLUTIndex(v)]; Int64 month = static_cast(values.month) + delta; @@ -851,7 +979,7 @@ public: month = ((month - 1) % 12) + 1; auto day_of_month = saturateDayOfMonth(year, month, values.day_of_month); - return makeDayNum(year, month, day_of_month); + return makeLUTIndex(year, month, day_of_month); } else { @@ -859,36 +987,43 @@ public: month = 12 - (-month % 12); auto day_of_month = saturateDayOfMonth(year, month, values.day_of_month); - return makeDayNum(year, month, day_of_month); + return makeLUTIndex(year, month, day_of_month); } } - inline NO_SANITIZE_UNDEFINED time_t addQuarters(time_t t, Int64 delta) const + /// If resulting month has less deys than source month, then saturation can happen. + /// Example: 31 Aug + 1 month = 30 Sep. + inline time_t NO_SANITIZE_UNDEFINED addMonths(time_t t, Int64 delta) const { - return addMonths(t, delta * 3); - } - - inline NO_SANITIZE_UNDEFINED DayNum addQuarters(DayNum d, Int64 delta) const - { - return addMonths(d, delta * 3); - } - - /// Saturation can occur if 29 Feb is mapped to non-leap year. - inline NO_SANITIZE_UNDEFINED time_t addYears(time_t t, Int64 delta) const - { - DayNum result_day = addYears(toDayNum(t), delta); + const auto result_day = addMonthsIndex(t, delta); time_t time_offset = toHour(t) * 3600 + toMinute(t) * 60 + toSecond(t); - if (time_offset >= lut[result_day].time_at_offset_change) - time_offset -= lut[result_day].amount_of_offset_change; + if (time_offset >= lut[result_day].time_at_offset_change()) + time_offset -= lut[result_day].amount_of_offset_change(); return lut[result_day].date + time_offset; } - inline NO_SANITIZE_UNDEFINED DayNum addYears(DayNum d, Int64 delta) const + inline ExtendedDayNum NO_SANITIZE_UNDEFINED addMonths(ExtendedDayNum d, Int64 delta) const { - const Values & values = lut[d]; + return toDayNum(addMonthsIndex(d, delta)); + } + + inline time_t NO_SANITIZE_UNDEFINED addQuarters(time_t t, Int64 delta) const + { + return addMonths(t, delta * 3); + } + + inline ExtendedDayNum addQuarters(ExtendedDayNum d, Int64 delta) const + { + return addMonths(d, delta * 3); + } + + template + inline LUTIndex NO_SANITIZE_UNDEFINED addYearsIndex(V v, Int64 delta) const + { + const Values & values = lut[toLUTIndex(v)]; auto year = values.year + delta; auto month = values.month; @@ -898,13 +1033,31 @@ public: if (unlikely(day_of_month == 29 && month == 2)) day_of_month = saturateDayOfMonth(year, month, day_of_month); - return makeDayNum(year, month, day_of_month); + return makeLUTIndex(year, month, day_of_month); + } + + /// Saturation can occur if 29 Feb is mapped to non-leap year. + inline time_t addYears(time_t t, Int64 delta) const + { + auto result_day = addYearsIndex(t, delta); + + time_t time_offset = toHour(t) * 3600 + toMinute(t) * 60 + toSecond(t); + + if (time_offset >= lut[result_day].time_at_offset_change()) + time_offset -= lut[result_day].amount_of_offset_change(); + + return lut[result_day].date + time_offset; + } + + inline ExtendedDayNum addYears(ExtendedDayNum d, Int64 delta) const + { + return toDayNum(addYearsIndex(d, delta)); } inline std::string timeToString(time_t t) const { - const Values & values = find(t); + const Values & values = getValues(t); std::string s {"0000-00-00 00:00:00"}; @@ -933,7 +1086,7 @@ public: inline std::string dateToString(time_t t) const { - const Values & values = find(t); + const Values & values = getValues(t); std::string s {"0000-00-00"}; @@ -949,9 +1102,9 @@ public: return s; } - inline std::string dateToString(DayNum d) const + inline std::string dateToString(ExtendedDayNum d) const { - const Values & values = lut[d]; + const Values & values = getValues(d); std::string s {"0000-00-00"}; diff --git a/base/common/DayNum.h b/base/common/DayNum.h index a4ef0c43b69..5cf4d4635c8 100644 --- a/base/common/DayNum.h +++ b/base/common/DayNum.h @@ -7,3 +7,8 @@ * See DateLUTImpl for usage examples. */ STRONG_TYPEDEF(UInt16, DayNum) + +/** Represent number of days since 1970-01-01 but in extended range, + * for dates before 1970-01-01 and after 2105 + */ +STRONG_TYPEDEF(Int32, ExtendedDayNum) diff --git a/base/common/LocalDate.h b/base/common/LocalDate.h index e5ebe877bc5..7e1260c1385 100644 --- a/base/common/LocalDate.h +++ b/base/common/LocalDate.h @@ -105,7 +105,8 @@ public: DayNum getDayNum() const { - return DateLUT::instance().makeDayNum(m_year, m_month, m_day); + const auto & lut = DateLUT::instance(); + return DayNum(lut.makeDayNum(m_year, m_month, m_day).toUnderType()); } operator DayNum() const diff --git a/base/common/strong_typedef.h b/base/common/strong_typedef.h index d9850a25c37..77b83bfa6e5 100644 --- a/base/common/strong_typedef.h +++ b/base/common/strong_typedef.h @@ -12,6 +12,7 @@ private: T t; public: + using UnderlyingType = T; template ::type> explicit StrongTypedef(const T & t_) : t(t_) {} template ::type> diff --git a/base/common/tests/CMakeLists.txt b/base/common/tests/CMakeLists.txt index b7082ee9900..b335b302cb0 100644 --- a/base/common/tests/CMakeLists.txt +++ b/base/common/tests/CMakeLists.txt @@ -16,7 +16,9 @@ target_link_libraries (realloc-perf PRIVATE common) add_check(local_date_time_comparison) if(USE_GTEST) - add_executable(unit_tests_libcommon gtest_json_test.cpp gtest_strong_typedef.cpp gtest_find_symbols.cpp) + add_executable(unit_tests_libcommon gtest_json_test.cpp gtest_strong_typedef.cpp gtest_find_symbols.cpp gtest_DateLutImpl.cpp + ${CMAKE_BINARY_DIR}/src/Storages/System/StorageSystemTimeZones.generated.cpp + ) target_link_libraries(unit_tests_libcommon PRIVATE common ${GTEST_MAIN_LIBRARIES} ${GTEST_LIBRARIES}) add_check(unit_tests_libcommon) endif() diff --git a/base/common/tests/gtest_DateLutImpl.cpp b/base/common/tests/gtest_DateLutImpl.cpp new file mode 100644 index 00000000000..395e2eddb00 --- /dev/null +++ b/base/common/tests/gtest_DateLutImpl.cpp @@ -0,0 +1,515 @@ +#include +#include + +#include + +#include +#include + +/// For the expansion of gtest macros. +#if defined(__clang__) + #pragma clang diagnostic ignored "-Wused-but-marked-unused" +#endif + +// All timezones present at build time and embedded into CH binary. +extern const char * auto_time_zones[]; + +namespace +{ + +cctz::civil_day YYYYMMDDToDay(unsigned value) +{ + return cctz::civil_day( + value / 10000, // year + (value % 10000) / 100, // month + value % 100); // day +} + +cctz::civil_second YYYYMMDDHMMSSToSecond(std::uint64_t value) +{ + return cctz::civil_second( + value / 10000000000, + value / 100000000 % 100, + value / 1000000 % 100, + value / 10000 % 100, + value / 100 % 100, + value % 100); +} + + +std::vector allTimezones() +{ + std::vector result; + + auto timezone_name = auto_time_zones; + while (*timezone_name) + { + result.push_back(*timezone_name); + ++timezone_name; + } + + return result; +} + +struct FailuresCount +{ + size_t non_fatal = 0; + size_t fatal = 0; + size_t total = 0; +}; + +FailuresCount countFailures(const ::testing::TestResult & test_result) +{ + FailuresCount failures{0, 0, 0}; + const size_t count = test_result.total_part_count(); + for (size_t i = 0; i < count; ++i) + { + const auto & part = test_result.GetTestPartResult(i); + if (part.nonfatally_failed()) + { + ++failures.non_fatal; + ++failures.total; + } + if (part.fatally_failed()) + { + ++failures.fatal; + ++failures.total; + } + } + + return failures; +} + +} + +TEST(YYYYMMDDToDay, Test) +{ + std::cerr << YYYYMMDDHMMSSToSecond(19700101'00'00'00) << std::endl; +} + +TEST(DateLUTTest, TimeValuesInMiddleOfRange) +{ + const DateLUTImpl lut("Europe/Minsk"); + const time_t time = 1568650811; // 2019-09-16 19:20:11 (Monday) + + EXPECT_EQ(lut.getTimeZone(), "Europe/Minsk"); + EXPECT_EQ(lut.getOffsetAtStartOfEpoch(), 3600*3); // UTC-3 + + EXPECT_EQ(lut.toDate(time), 1568581200); + EXPECT_EQ(lut.toMonth(time), 9); + EXPECT_EQ(lut.toQuarter(time), 3); + EXPECT_EQ(lut.toYear(time), 2019); + EXPECT_EQ(lut.toDayOfMonth(time), 16); + + EXPECT_EQ(lut.toFirstDayOfWeek(time), 1568581200 /*time_t*/); + EXPECT_EQ(lut.toFirstDayNumOfWeek(time), DayNum(18155) /*DayNum*/); + EXPECT_EQ(lut.toFirstDayOfMonth(time), 1567285200 /*time_t*/); + EXPECT_EQ(lut.toFirstDayNumOfMonth(time), DayNum(18140) /*DayNum*/); + EXPECT_EQ(lut.toFirstDayNumOfQuarter(time), DayNum(18078) /*DayNum*/); + EXPECT_EQ(lut.toFirstDayOfQuarter(time), 1561928400 /*time_t*/); + EXPECT_EQ(lut.toFirstDayOfYear(time), 1546290000 /*time_t*/); + EXPECT_EQ(lut.toFirstDayNumOfYear(time), DayNum(17897) /*DayNum*/); + EXPECT_EQ(lut.toFirstDayOfNextMonth(time), 1569877200 /*time_t*/); + EXPECT_EQ(lut.toFirstDayOfPrevMonth(time), 1564606800 /*time_t*/); + EXPECT_EQ(lut.daysInMonth(time), 30 /*UInt8*/); + EXPECT_EQ(lut.toDateAndShift(time, 10), 1569445200 /*time_t*/); + EXPECT_EQ(lut.toTime(time), 58811 /*time_t*/); + EXPECT_EQ(lut.toHour(time), 19 /*unsigned*/); + EXPECT_EQ(lut.toSecond(time), 11 /*unsigned*/); + EXPECT_EQ(lut.toMinute(time), 20 /*unsigned*/); + EXPECT_EQ(lut.toStartOfMinute(time), 1568650800 /*time_t*/); + EXPECT_EQ(lut.toStartOfFiveMinute(time), 1568650800 /*time_t*/); + EXPECT_EQ(lut.toStartOfFifteenMinutes(time), 1568650500 /*time_t*/); + EXPECT_EQ(lut.toStartOfTenMinutes(time), 1568650800 /*time_t*/); + EXPECT_EQ(lut.toStartOfHour(time), 1568649600 /*time_t*/); + EXPECT_EQ(lut.toDayNum(time), DayNum(18155) /*DayNum*/); + EXPECT_EQ(lut.toDayOfYear(time), 259 /*unsigned*/); + EXPECT_EQ(lut.toRelativeWeekNum(time), 2594 /*unsigned*/); + EXPECT_EQ(lut.toISOYear(time), 2019 /*unsigned*/); + EXPECT_EQ(lut.toFirstDayNumOfISOYear(time), DayNum(17896) /*DayNum*/); + EXPECT_EQ(lut.toFirstDayOfISOYear(time), 1546203600 /*time_t*/); + EXPECT_EQ(lut.toISOWeek(time), 38 /*unsigned*/); + EXPECT_EQ(lut.toRelativeMonthNum(time), 24237 /*unsigned*/); + EXPECT_EQ(lut.toRelativeQuarterNum(time), 8078 /*unsigned*/); + EXPECT_EQ(lut.toRelativeHourNum(time), 435736 /*time_t*/); + EXPECT_EQ(lut.toRelativeMinuteNum(time), 26144180 /*time_t*/); + EXPECT_EQ(lut.toStartOfHourInterval(time, 5), 1568646000 /*time_t*/); + EXPECT_EQ(lut.toStartOfMinuteInterval(time, 6), 1568650680 /*time_t*/); + EXPECT_EQ(lut.toStartOfSecondInterval(time, 7), 1568650811 /*time_t*/); + EXPECT_EQ(lut.toNumYYYYMM(time), 201909 /*UInt32*/); + EXPECT_EQ(lut.toNumYYYYMMDD(time), 20190916 /*UInt32*/); + EXPECT_EQ(lut.toNumYYYYMMDDhhmmss(time), 20190916192011 /*UInt64*/); + EXPECT_EQ(lut.addDays(time, 100), 1577290811 /*time_t*/); + EXPECT_EQ(lut.addWeeks(time, 100), 1629130811 /*time_t*/); + EXPECT_EQ(lut.addMonths(time, 100), 1831652411 /*time_t*/); + EXPECT_EQ(lut.addQuarters(time, 100), 2357655611 /*time_t*/); + EXPECT_EQ(lut.addYears(time, 10), 1884270011 /*time_t*/); + EXPECT_EQ(lut.timeToString(time), "2019-09-16 19:20:11" /*std::string*/); + EXPECT_EQ(lut.dateToString(time), "2019-09-16" /*std::string*/); +} + + +TEST(DateLUTTest, TimeValuesAtLeftBoderOfRange) +{ + const DateLUTImpl lut("UTC"); + const time_t time = 0; // 1970-01-01 00:00:00 (Thursday) + + EXPECT_EQ(lut.getTimeZone(), "UTC"); + + EXPECT_EQ(lut.toDate(time), 0); + EXPECT_EQ(lut.toMonth(time), 1); + EXPECT_EQ(lut.toQuarter(time), 1); + EXPECT_EQ(lut.toYear(time), 1970); + EXPECT_EQ(lut.toDayOfMonth(time), 1); + + EXPECT_EQ(lut.toFirstDayOfWeek(time), -259200 /*time_t*/); // 1969-12-29 00:00:00 + EXPECT_EQ(lut.toFirstDayNumOfWeek(time), ExtendedDayNum(-3) /*DayNum*/); + EXPECT_EQ(lut.toFirstDayOfMonth(time), 0 /*time_t*/); + EXPECT_EQ(lut.toFirstDayNumOfMonth(time), DayNum(0) /*DayNum*/); + EXPECT_EQ(lut.toFirstDayNumOfQuarter(time), DayNum(0) /*DayNum*/); + EXPECT_EQ(lut.toFirstDayOfQuarter(time), 0 /*time_t*/); + EXPECT_EQ(lut.toFirstDayOfYear(time), 0 /*time_t*/); + EXPECT_EQ(lut.toFirstDayNumOfYear(time), DayNum(0) /*DayNum*/); + EXPECT_EQ(lut.toFirstDayOfNextMonth(time), 2678400 /*time_t*/); + EXPECT_EQ(lut.toFirstDayOfPrevMonth(time), -2678400 /*time_t*/); // 1969-12-01 00:00:00 + EXPECT_EQ(lut.daysInMonth(time), 31 /*UInt8*/); + EXPECT_EQ(lut.toDateAndShift(time, 10), 864000 /*time_t*/); + EXPECT_EQ(lut.toTime(time), 0 /*time_t*/); + EXPECT_EQ(lut.toHour(time), 0 /*unsigned*/); + EXPECT_EQ(lut.toSecond(time), 0 /*unsigned*/); + EXPECT_EQ(lut.toMinute(time), 0 /*unsigned*/); + EXPECT_EQ(lut.toStartOfMinute(time), 0 /*time_t*/); + EXPECT_EQ(lut.toStartOfFiveMinute(time), 0 /*time_t*/); + EXPECT_EQ(lut.toStartOfFifteenMinutes(time), 0 /*time_t*/); + EXPECT_EQ(lut.toStartOfTenMinutes(time), 0 /*time_t*/); + EXPECT_EQ(lut.toStartOfHour(time), 0 /*time_t*/); + EXPECT_EQ(lut.toDayNum(time), DayNum(0) /*DayNum*/); + EXPECT_EQ(lut.toDayOfYear(time), 1 /*unsigned*/); + EXPECT_EQ(lut.toRelativeWeekNum(time), 0 /*unsigned*/); + EXPECT_EQ(lut.toISOYear(time), 1970 /*unsigned*/); + EXPECT_EQ(lut.toFirstDayNumOfISOYear(time), ExtendedDayNum(-3) /*DayNum*/); + EXPECT_EQ(lut.toFirstDayOfISOYear(time), -259200 /*time_t*/); // 1969-12-29 00:00:00 + EXPECT_EQ(lut.toISOWeek(time), 1 /*unsigned*/); + EXPECT_EQ(lut.toRelativeMonthNum(time), 23641 /*unsigned*/); // ? + EXPECT_EQ(lut.toRelativeQuarterNum(time), 7880 /*unsigned*/); // ? + EXPECT_EQ(lut.toRelativeHourNum(time), 0 /*time_t*/); + EXPECT_EQ(lut.toRelativeMinuteNum(time), 0 /*time_t*/); + EXPECT_EQ(lut.toStartOfHourInterval(time, 5), 0 /*time_t*/); + EXPECT_EQ(lut.toStartOfMinuteInterval(time, 6), 0 /*time_t*/); + EXPECT_EQ(lut.toStartOfSecondInterval(time, 7), 0 /*time_t*/); + EXPECT_EQ(lut.toNumYYYYMM(time), 197001 /*UInt32*/); + EXPECT_EQ(lut.toNumYYYYMMDD(time), 19700101 /*UInt32*/); + EXPECT_EQ(lut.toNumYYYYMMDDhhmmss(time), 19700101000000 /*UInt64*/); + EXPECT_EQ(lut.addDays(time, 100), 8640000 /*time_t*/); + EXPECT_EQ(lut.addWeeks(time, 100), 60480000 /*time_t*/); + EXPECT_EQ(lut.addMonths(time, 100), 262828800 /*time_t*/); + EXPECT_EQ(lut.addQuarters(time, 100), 788918400 /*time_t*/); + EXPECT_EQ(lut.addYears(time, 10), 315532800 /*time_t*/); + EXPECT_EQ(lut.timeToString(time), "1970-01-01 00:00:00" /*std::string*/); + EXPECT_EQ(lut.dateToString(time), "1970-01-01" /*std::string*/); +} + +TEST(DateLUTTest, TimeValuesAtRightBoderOfRangeOfOLDLut) +{ + // Value is at the right border of the OLD (small) LUT, and provides meaningful values where OLD LUT would provide garbage. + const DateLUTImpl lut("UTC"); + + const time_t time = 4294343873; // 2106-01-31T01:17:53 (Sunday) + + EXPECT_EQ(lut.getTimeZone(), "UTC"); + + EXPECT_EQ(lut.toDate(time), 4294339200); + EXPECT_EQ(lut.toMonth(time), 1); + EXPECT_EQ(lut.toQuarter(time), 1); + EXPECT_EQ(lut.toYear(time), 2106); + EXPECT_EQ(lut.toDayOfMonth(time), 31); + + EXPECT_EQ(lut.toFirstDayOfWeek(time), 4293820800 /*time_t*/); + EXPECT_EQ(lut.toFirstDayNumOfWeek(time), DayNum(49697)); + EXPECT_EQ(lut.toFirstDayOfMonth(time), 4291747200 /*time_t*/); // 2016-01-01 + EXPECT_EQ(lut.toFirstDayNumOfMonth(time), DayNum(49673)); + EXPECT_EQ(lut.toFirstDayNumOfQuarter(time), DayNum(49673) /*DayNum*/); + EXPECT_EQ(lut.toFirstDayOfQuarter(time), 4291747200 /*time_t*/); + EXPECT_EQ(lut.toFirstDayOfYear(time), 4291747200 /*time_t*/); + EXPECT_EQ(lut.toFirstDayNumOfYear(time), DayNum(49673) /*DayNum*/); + EXPECT_EQ(lut.toFirstDayOfNextMonth(time), 4294425600 /*time_t*/); // 2106-02-01 + EXPECT_EQ(lut.toFirstDayOfPrevMonth(time), 4289068800 /*time_t*/); // 2105-12-01 + EXPECT_EQ(lut.daysInMonth(time), 31 /*UInt8*/); + EXPECT_EQ(lut.toDateAndShift(time, 10), 4295203200 /*time_t*/); // 2106-02-10 + EXPECT_EQ(lut.toTime(time), 4673 /*time_t*/); + EXPECT_EQ(lut.toHour(time), 1 /*unsigned*/); + EXPECT_EQ(lut.toMinute(time), 17 /*unsigned*/); + EXPECT_EQ(lut.toSecond(time), 53 /*unsigned*/); + EXPECT_EQ(lut.toStartOfMinute(time), 4294343820 /*time_t*/); + EXPECT_EQ(lut.toStartOfFiveMinute(time), 4294343700 /*time_t*/); + EXPECT_EQ(lut.toStartOfFifteenMinutes(time), 4294343700 /*time_t*/); + EXPECT_EQ(lut.toStartOfTenMinutes(time), 4294343400 /*time_t*/); + EXPECT_EQ(lut.toStartOfHour(time), 4294342800 /*time_t*/); + EXPECT_EQ(lut.toDayNum(time), DayNum(49703) /*DayNum*/); + EXPECT_EQ(lut.toDayOfYear(time), 31 /*unsigned*/); + EXPECT_EQ(lut.toRelativeWeekNum(time), 7100 /*unsigned*/); + EXPECT_EQ(lut.toISOYear(time), 2106 /*unsigned*/); + EXPECT_EQ(lut.toFirstDayNumOfISOYear(time), DayNum(49676) /*DayNum*/); // 2106-01-04 + EXPECT_EQ(lut.toFirstDayOfISOYear(time), 4292006400 /*time_t*/); + EXPECT_EQ(lut.toISOWeek(time), 4 /*unsigned*/); + EXPECT_EQ(lut.toRelativeMonthNum(time), 25273 /*unsigned*/); + EXPECT_EQ(lut.toRelativeQuarterNum(time), 8424 /*unsigned*/); + EXPECT_EQ(lut.toRelativeHourNum(time), 1192873 /*time_t*/); + EXPECT_EQ(lut.toRelativeMinuteNum(time), 71572397 /*time_t*/); + EXPECT_EQ(lut.toStartOfHourInterval(time, 5), 4294332000 /*time_t*/); + EXPECT_EQ(lut.toStartOfMinuteInterval(time, 6), 4294343520 /*time_t*/); + EXPECT_EQ(lut.toStartOfSecondInterval(time, 7), 4294343872 /*time_t*/); + EXPECT_EQ(lut.toNumYYYYMM(time), 210601 /*UInt32*/); + EXPECT_EQ(lut.toNumYYYYMMDD(time), 21060131 /*UInt32*/); + EXPECT_EQ(lut.toNumYYYYMMDDhhmmss(time), 21060131011753 /*UInt64*/); + EXPECT_EQ(lut.addDays(time, 100), 4302983873 /*time_t*/); + EXPECT_EQ(lut.addWeeks(time, 10), 4300391873 /*time_t*/); + EXPECT_EQ(lut.addMonths(time, 10), 4320523073 /*time_t*/); // 2106-11-30 01:17:53 + EXPECT_EQ(lut.addQuarters(time, 10), 4373140673 /*time_t*/); // 2108-07-31 01:17:53 + EXPECT_EQ(lut.addYears(time, 10), 4609876673 /*time_t*/); // 2116-01-31 01:17:53 + + EXPECT_EQ(lut.timeToString(time), "2106-01-31 01:17:53" /*std::string*/); + EXPECT_EQ(lut.dateToString(time), "2106-01-31" /*std::string*/); +} + + +class DateLUT_TimeZone : public ::testing::TestWithParam +{}; + +TEST_P(DateLUT_TimeZone, DISABLED_LoadAllTimeZones) +{ + // There are some assumptions and assertions about TZ data made in DateLUTImpl which are verified upon loading, + // to make sure that those assertions are true for all timezones we are going to load all of them one by one. + DateLUTImpl{GetParam()}; +} + +// Another long running test, shouldn't be run to often +TEST_P(DateLUT_TimeZone, VaidateTimeComponentsAroundEpoch) +{ + // Converting time around 1970-01-01 to hour-minute-seconds time components + // could be problematic. + const size_t max_failures_per_tz = 3; + const auto timezone_name = GetParam(); + + const auto * test_info = ::testing::UnitTest::GetInstance()->current_test_info(); + const auto lut = DateLUTImpl(timezone_name); + + for (time_t i = -856147870; i < 86400 * 10000; i += 11 * 13 * 17 * 19) + { + SCOPED_TRACE(::testing::Message() + << "\n\tTimezone: " << timezone_name + << "\n\ttimestamp: " << i + << "\n\t offset at start of epoch : " << lut.getOffsetAtStartOfEpoch() + << "\n\t offset_is_whole_number_of_hours_everytime : " << lut.getOffsetIsWholNumberOfHoursEveryWhere() + << "\n\t time_offset_epoch : " << lut.getTimeOffsetEpoch() + << "\n\t offset_at_start_of_lut : " << lut.getTimeOffsetAtStartOfLUT()); + + EXPECT_GE(24, lut.toHour(i)); + EXPECT_GT(60, lut.toMinute(i)); + EXPECT_GT(60, lut.toSecond(i)); + + const auto current_failures = countFailures(*test_info->result()); + if (current_failures.total > 0) + { + if (i < 0) + i = -1; + } + + if (current_failures.total >= max_failures_per_tz) + break; + } +} + +TEST_P(DateLUT_TimeZone, getTimeZone) +{ + const auto & lut = DateLUT::instance(GetParam()); + + EXPECT_EQ(GetParam(), lut.getTimeZone()); +} + +TEST_P(DateLUT_TimeZone, ZeroTime) +{ + const auto & lut = DateLUT::instance(GetParam()); + + EXPECT_EQ(0, lut.toDayNum(time_t{0})); + EXPECT_EQ(0, lut.toDayNum(DayNum{0})); + EXPECT_EQ(0, lut.toDayNum(ExtendedDayNum{0})); +} + +// Group of tests for timezones that have or had some time ago an offset which is not multiple of 15 minutes. +INSTANTIATE_TEST_SUITE_P(ExoticTimezones, + DateLUT_TimeZone, + ::testing::ValuesIn(std::initializer_list{ + "Africa/El_Aaiun", + "Pacific/Apia", + "Pacific/Enderbury", + "Pacific/Fakaofo", + "Pacific/Kiritimati", + }) +); + +INSTANTIATE_TEST_SUITE_P(DISABLED_AllTimeZones, + DateLUT_TimeZone, + ::testing::ValuesIn(allTimezones()) +); + +std::ostream & operator<<(std::ostream & ostr, const DateLUTImpl::Values & v) +{ + return ostr << "DateLUTImpl::Values{" + << "\n\t date : " << v.date + << "\n\t year : " << static_cast(v.year) + << "\n\t month : " << static_cast(v.month) + << "\n\t day : " << static_cast(v.day_of_month) + << "\n\t weekday : " << static_cast(v.day_of_week) + << "\n\t days in month : " << static_cast(v.days_in_month) + << "\n\t offset change : " << v.amount_of_offset_change() + << "\n\t offfset change at : " << v.time_at_offset_change() + << "\n}"; +} + +struct TimeRangeParam +{ + const cctz::civil_second begin; + const cctz::civil_second end; + const int step_in_seconds; +}; + +std::ostream & operator<<(std::ostream & ostr, const TimeRangeParam & param) +{ + const auto approximate_step = [](const int step) -> std::string + { + // Convert seconds to a string of seconds or fractional count of minutes/hours/days. + static const size_t multipliers[] = {1 /*seconds to seconds*/, 60 /*seconds to minutes*/, 60 /*minutes to hours*/, 24 /*hours to days*/, 0 /*terminator*/}; + static const char* names[] = {"s", "m", "h", "d", nullptr}; + double result = step; + size_t i = 0; + for (; i < sizeof(multipliers)/sizeof(multipliers[0]) && result > multipliers[i]; ++i) + result /= multipliers[i]; + + char buffer[256] = {'\0'}; + std::snprintf(buffer, sizeof(buffer), "%.1f%s", result, names[i - 1]); + return std::string{buffer}; + }; + + return ostr << param.begin << " : " << param.end << " step: " << param.step_in_seconds << "s (" << approximate_step(param.step_in_seconds) << ")"; +} + +class DateLUT_Timezone_TimeRange : public ::testing::TestWithParam> +{}; + +// refactored test from tests/date_lut3.cpp +TEST_P(DateLUT_Timezone_TimeRange, InRange) +{ + // for a time_t values in range [begin, end) to match with reference obtained from cctz: + // compare date and time components: year, month, day, hours, minutes, seconds, formatted time string. + const auto & [timezone_name, range_data] = GetParam(); + const auto & [begin, end, step] = range_data; + + const auto * test_info = ::testing::UnitTest::GetInstance()->current_test_info(); + static const size_t max_failures_per_case = 3; + cctz::time_zone tz; + ASSERT_TRUE(cctz::load_time_zone(timezone_name, &tz)); + + const auto & lut = DateLUT::instance(timezone_name); + const auto start = cctz::convert(begin, tz).time_since_epoch().count(); + const auto stop = cctz::convert(end, tz).time_since_epoch().count(); + + for (time_t expected_time_t = start; expected_time_t < stop; expected_time_t += step) + { + SCOPED_TRACE(expected_time_t); + + const auto tz_time = cctz::convert(std::chrono::system_clock::from_time_t(expected_time_t), tz); + + EXPECT_EQ(tz_time.year(), lut.toYear(expected_time_t)); + EXPECT_EQ(tz_time.month(), lut.toMonth(expected_time_t)); + EXPECT_EQ(tz_time.day(), lut.toDayOfMonth(expected_time_t)); + EXPECT_EQ(static_cast(cctz::get_weekday(tz_time)) + 1, lut.toDayOfWeek(expected_time_t)); // tm.tm_wday Sunday is 0, while for DateLUTImpl it is 7 + EXPECT_EQ(cctz::get_yearday(tz_time), lut.toDayOfYear(expected_time_t)); + EXPECT_EQ(tz_time.hour(), lut.toHour(expected_time_t)); + EXPECT_EQ(tz_time.minute(), lut.toMinute(expected_time_t)); + EXPECT_EQ(tz_time.second(), lut.toSecond(expected_time_t)); + + const auto time_string = cctz::format("%E4Y-%m-%d %H:%M:%S", std::chrono::system_clock::from_time_t(expected_time_t), tz); + EXPECT_EQ(time_string, lut.timeToString(expected_time_t)); + + // it makes sense to let test execute all checks above to simplify debugging, + // but once we've found a bad apple, no need to dig deeper. + if (countFailures(*test_info->result()).total >= max_failures_per_case) + break; + } +} + +/** Next tests are disabled due to following reasons: + * 1. They are huge and take enormous amount of time to run + * 2. Current implementation of DateLUTImpl is inprecise and some cases fail and it seems impractical to try to fix those. + * 3. Many failures (~300) were fixed while refactoring, about ~40 remain the same and 3 new introduced: + * "Asia/Gaza" + * "Pacific/Enderbury" + * "Pacific/Kiritimati" + * So it would be tricky to skip knonw failures to allow all unit tests to pass. + */ +INSTANTIATE_TEST_SUITE_P(DISABLED_AllTimezones_Year2010, + DateLUT_Timezone_TimeRange, + ::testing::Combine( + ::testing::ValuesIn(allTimezones()), + ::testing::ValuesIn(std::initializer_list{ + // Values from tests/date_lut3.cpp + {YYYYMMDDToDay(20101031), YYYYMMDDToDay(20101101), 15 * 60}, + {YYYYMMDDToDay(20100328), YYYYMMDDToDay(20100330), 15 * 60} + })) +); + +INSTANTIATE_TEST_SUITE_P(DISABLED_AllTimezones_Year1970_WHOLE, + DateLUT_Timezone_TimeRange, + ::testing::Combine( + ::testing::ValuesIn(allTimezones()), + ::testing::ValuesIn(std::initializer_list{ + // Values from tests/date_lut3.cpp + {YYYYMMDDToDay(19700101), YYYYMMDDToDay(19701231), 3191 /*53m 11s*/}, + })) +); + +INSTANTIATE_TEST_SUITE_P(DISABLED_AllTimezones_Year2010_WHOLE, + DateLUT_Timezone_TimeRange, + ::testing::Combine( + ::testing::ValuesIn(allTimezones()), + ::testing::ValuesIn(std::initializer_list{ + // Values from tests/date_lut3.cpp + {YYYYMMDDToDay(20100101), YYYYMMDDToDay(20101231), 3191 /*53m 11s*/}, + })) +); + +INSTANTIATE_TEST_SUITE_P(DISABLED_AllTimezones_Year2020_WHOLE, + DateLUT_Timezone_TimeRange, + ::testing::Combine( + ::testing::ValuesIn(allTimezones()), + ::testing::ValuesIn(std::initializer_list{ + // Values from tests/date_lut3.cpp + {YYYYMMDDToDay(20200101), YYYYMMDDToDay(20201231), 3191 /*53m 11s*/}, + })) +); + +INSTANTIATE_TEST_SUITE_P(DISABLED_AllTimezones_PreEpoch, + DateLUT_Timezone_TimeRange, + ::testing::Combine( + ::testing::ValuesIn(allTimezones()), + ::testing::ValuesIn(std::initializer_list{ + {YYYYMMDDToDay(19500101), YYYYMMDDToDay(19600101), 15 * 60}, + {YYYYMMDDToDay(19300101), YYYYMMDDToDay(19350101), 11 * 15 * 60} + })) +); + +INSTANTIATE_TEST_SUITE_P(DISABLED_AllTimezones_Year1970, + DateLUT_Timezone_TimeRange, + ::testing::Combine( + ::testing::ValuesIn(allTimezones()), + ::testing::ValuesIn(std::initializer_list{ + {YYYYMMDDToDay(19700101), YYYYMMDDToDay(19700201), 15 * 60}, + {YYYYMMDDToDay(19700101), YYYYMMDDToDay(19701231), 11 * 13 * 17} +// // 11 was chosen as a number which can't divide product of 2-combinarions of (7, 24, 60), +// // to reduce likelehood of hitting same hour/minute/second values for different days. +// // + 12 is just to make sure that last day is covered fully. +// {0, 0 + 11 * 3600 * 24 + 12, 11}, + })) +); + diff --git a/programs/client/Client.cpp b/programs/client/Client.cpp index 3c27908741c..63fa5d8a5c9 100644 --- a/programs/client/Client.cpp +++ b/programs/client/Client.cpp @@ -390,7 +390,7 @@ private: for (auto d : chineseNewYearIndicators) { /// Let's celebrate until Lantern Festival - if (d <= days && d + 25u >= days) + if (d <= days && d + 25 >= days) return true; else if (d > days) return false; diff --git a/src/Core/DecimalFunctions.h b/src/Core/DecimalFunctions.h index 355cf1d378a..2cd50ab8d08 100644 --- a/src/Core/DecimalFunctions.h +++ b/src/Core/DecimalFunctions.h @@ -50,9 +50,10 @@ inline auto scaleMultiplier(UInt32 scale) * whole - represents whole part of decimal, can be negative or positive. * fractional - for fractional part of decimal, always positive. */ -template +template struct DecimalComponents { + using T = typename DecimalType::NativeType; T whole; T fractional; }; @@ -106,6 +107,15 @@ inline DecimalType decimalFromComponentsWithMultiplier( return DecimalType(value); } +template +inline DecimalType decimalFromComponentsWithMultiplier( + const DecimalComponents & components, + typename DecimalType::NativeType scale_multiplier) +{ + return decimalFromComponentsWithMultiplier(components.whole, components.fractional, scale_multiplier); +} + + /** Make a decimal value from whole and fractional components with given scale. * * @see `decimalFromComponentsWithMultiplier` for details. @@ -126,7 +136,7 @@ inline DecimalType decimalFromComponents( */ template inline DecimalType decimalFromComponents( - const DecimalComponents & components, + const DecimalComponents & components, UInt32 scale) { return decimalFromComponents(components.whole, components.fractional, scale); @@ -136,7 +146,7 @@ inline DecimalType decimalFromComponents( * This is an optimization to reduce number of calls to scaleMultiplier on known scale. */ template -inline DecimalComponents splitWithScaleMultiplier( +inline DecimalComponents splitWithScaleMultiplier( const DecimalType & decimal, typename DecimalType::NativeType scale_multiplier) { @@ -151,7 +161,7 @@ inline DecimalComponents splitWithScaleMultipl /// Split decimal into components: whole and fractional part, @see `DecimalComponents` for details. template -inline DecimalComponents split(const DecimalType & decimal, UInt32 scale) +inline DecimalComponents split(const DecimalType & decimal, UInt32 scale) { if (scale == 0) { diff --git a/src/Core/MySQL/MySQLReplication.cpp b/src/Core/MySQL/MySQLReplication.cpp index 1b202c4edb4..3e9c5230955 100644 --- a/src/Core/MySQL/MySQLReplication.cpp +++ b/src/Core/MySQL/MySQLReplication.cpp @@ -420,8 +420,8 @@ namespace MySQLReplication UInt32 i24 = 0; payload.readStrict(reinterpret_cast(&i24), 3); - DayNum date_day_number = DateLUT::instance().makeDayNum( - static_cast((i24 >> 9) & 0x7fff), static_cast((i24 >> 5) & 0xf), static_cast(i24 & 0x1f)); + const DayNum date_day_number{DateLUT::instance().makeDayNum( + static_cast((i24 >> 9) & 0x7fff), static_cast((i24 >> 5) & 0xf), static_cast(i24 & 0x1f)).toUnderType()}; row.push_back(Field(date_day_number.toUnderType())); break; @@ -443,7 +443,7 @@ namespace MySQLReplication row.push_back(Field{UInt32(date_time)}); else { - DB::DecimalUtils::DecimalComponents components{ + DB::DecimalUtils::DecimalComponents components{ static_cast(date_time), 0}; components.fractional = fsp; @@ -462,7 +462,7 @@ namespace MySQLReplication row.push_back(Field{sec}); else { - DB::DecimalUtils::DecimalComponents components{ + DB::DecimalUtils::DecimalComponents components{ static_cast(sec), 0}; components.fractional = fsp; diff --git a/src/Core/tests/gtest_DecimalFunctions.cpp b/src/Core/tests/gtest_DecimalFunctions.cpp index be64661176b..1069a810d64 100644 --- a/src/Core/tests/gtest_DecimalFunctions.cpp +++ b/src/Core/tests/gtest_DecimalFunctions.cpp @@ -14,7 +14,7 @@ struct DecimalUtilsSplitAndCombineTestParam Decimal64 decimal_value; uint8_t scale; - DecimalUtils::DecimalComponents components; + DecimalUtils::DecimalComponents components; }; std::ostream & operator << (std::ostream & ostr, const DecimalUtilsSplitAndCombineTestParam & param) diff --git a/src/DataStreams/MongoDBBlockInputStream.cpp b/src/DataStreams/MongoDBBlockInputStream.cpp index 5463d95151b..e4ddcd09ede 100644 --- a/src/DataStreams/MongoDBBlockInputStream.cpp +++ b/src/DataStreams/MongoDBBlockInputStream.cpp @@ -270,8 +270,8 @@ namespace throw Exception{"Type mismatch, expected Timestamp, got type id = " + toString(value.type()) + " for column " + name, ErrorCodes::TYPE_MISMATCH}; - assert_cast(column).getData().push_back(UInt16{DateLUT::instance().toDayNum( - static_cast &>(value).value().epochTime())}); + assert_cast(column).getData().push_back(static_cast(DateLUT::instance().toDayNum( + static_cast &>(value).value().epochTime()))); break; } diff --git a/src/DataTypes/DataTypeDateTime64.h b/src/DataTypes/DataTypeDateTime64.h index 198c3739f58..ec3f2fde889 100644 --- a/src/DataTypes/DataTypeDateTime64.h +++ b/src/DataTypes/DataTypeDateTime64.h @@ -48,66 +48,5 @@ public: bool canBePromoted() const override { return false; } }; -/** Tansform-type wrapper for DateTime64, applies given Transform to DateTime64 value or only to a whole part of it. - * - * Depending on what overloads of Transform::execute() are available, when called with DateTime64 value, - * invokes Transform::execute() with: - * * whole part of DateTime64 value, discarding fractional part. - * * DateTime64 value and scale factor. - * - * Suitable Transfotm-types are commonly used in Date/DateTime manipulation functions, - * and should implement static (or const) function with following signatures: - * R execute(UInt32 whole_value, ... , const TimeZoneImpl &) - * OR - * R execute(DateTime64 value, Int64 scale_factor, ... , const TimeZoneImpl &) - * - * Where R and T could be arbitrary types. -*/ -template -class TransformDateTime64 : public Transform -{ -private: - // Detect if Transform::execute is const or static method - // with signature defined by template args (ignoring result type). - template - struct TransformHasExecuteOverload : std::false_type {}; - - template - struct TransformHasExecuteOverload().execute(std::declval()...))>, Args...> - : std::true_type {}; - - template - static constexpr bool TransformHasExecuteOverload_v = TransformHasExecuteOverload::value; - -public: - static constexpr auto name = Transform::name; - - using Transform::execute; - - // non-explicit constructor to allow creating from scale value (or with no scale at all), indispensable in some contexts. - TransformDateTime64(UInt32 scale_ = 0) - : scale_multiplier(DecimalUtils::scaleMultiplier(scale_)) - {} - - template - inline auto execute(const DateTime64 & t, Args && ... args) const - { - const auto transform = static_cast(this); - - if constexpr (TransformHasExecuteOverload_v) - { - return transform->execute(t, scale_multiplier, std::forward(args)...); - } - else - { - const auto components = DecimalUtils::splitWithScaleMultiplier(t, scale_multiplier); - return transform->execute(static_cast(components.whole), std::forward(args)...); - } - } - -private: - DateTime64::NativeType scale_multiplier = 1; -}; - } diff --git a/src/Functions/CustomWeekTransforms.h b/src/Functions/CustomWeekTransforms.h index afcbadc835c..28da546eb93 100644 --- a/src/Functions/CustomWeekTransforms.h +++ b/src/Functions/CustomWeekTransforms.h @@ -33,14 +33,21 @@ static inline UInt32 dateIsNotSupported(const char * name) /// This factor transformation will say that the function is monotone everywhere. struct ZeroTransform { - static inline UInt16 execute(UInt32, UInt8, const DateLUTImpl &) { return 0; } static inline UInt16 execute(UInt16, UInt8, const DateLUTImpl &) { return 0; } + static inline UInt16 execute(UInt32, UInt8, const DateLUTImpl &) { return 0; } + static inline UInt16 execute(Int64, UInt8, const DateLUTImpl &) { return 0; } }; struct ToWeekImpl { static constexpr auto name = "toWeek"; + static inline UInt8 execute(Int64 t, UInt8 week_mode, const DateLUTImpl & time_zone) + { + // TODO: ditch conversion to DayNum, since it doesn't support extended range. + YearWeek yw = time_zone.toYearWeek(time_zone.toDayNum(t), week_mode); + return yw.second; + } static inline UInt8 execute(UInt32 t, UInt8 week_mode, const DateLUTImpl & time_zone) { YearWeek yw = time_zone.toYearWeek(time_zone.toDayNum(t), week_mode); @@ -59,6 +66,13 @@ struct ToYearWeekImpl { static constexpr auto name = "toYearWeek"; + static inline UInt32 execute(Int64 t, UInt8 week_mode, const DateLUTImpl & time_zone) + { + // TODO: ditch toDayNum() + YearWeek yw = time_zone.toYearWeek(time_zone.toDayNum(t), week_mode | static_cast(WeekModeFlag::YEAR)); + return yw.first * 100 + yw.second; + } + static inline UInt32 execute(UInt32 t, UInt8 week_mode, const DateLUTImpl & time_zone) { YearWeek yw = time_zone.toYearWeek(time_zone.toDayNum(t), week_mode | static_cast(WeekModeFlag::YEAR)); @@ -77,13 +91,19 @@ struct ToStartOfWeekImpl { static constexpr auto name = "toStartOfWeek"; + static inline UInt16 execute(Int64 t, UInt8 week_mode, const DateLUTImpl & time_zone) + { + return time_zone.toFirstDayNumOfWeek(time_zone.toDayNum(t), week_mode); +// return time_zone.toFirstDayNumOfWeek(t, week_mode); + } static inline UInt16 execute(UInt32 t, UInt8 week_mode, const DateLUTImpl & time_zone) { return time_zone.toFirstDayNumOfWeek(time_zone.toDayNum(t), week_mode); +// return time_zone.toFirstDayNumOfWeek(t, week_mode); } static inline UInt16 execute(UInt16 d, UInt8 week_mode, const DateLUTImpl & time_zone) { - return time_zone.toFirstDayNumOfWeek(DayNum(d), week_mode); + return time_zone.toFirstDayNumOfWeek(ExtendedDayNum(d), week_mode); } using FactorTransform = ZeroTransform; diff --git a/src/Functions/DateTimeTransforms.h b/src/Functions/DateTimeTransforms.h index 333b397312d..c299b9c4169 100644 --- a/src/Functions/DateTimeTransforms.h +++ b/src/Functions/DateTimeTransforms.h @@ -3,6 +3,7 @@ #include #include #include +//#include #include #include #include @@ -33,14 +34,15 @@ namespace ErrorCodes * factor-transformation F is "round to the nearest month" (2015-02-03 -> 2015-02-01). */ -static inline UInt32 dateIsNotSupported(const char * name) -{ - throw Exception("Illegal type Date of argument for function " + std::string(name), ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT); -} + static inline UInt32 dateIsNotSupported(const char * name) + { + throw Exception("Illegal type Date of argument for function " + std::string(name), ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT); + } /// This factor transformation will say that the function is monotone everywhere. struct ZeroTransform { + static inline UInt16 execute(Int64, const DateLUTImpl &) { return 0; } static inline UInt16 execute(UInt32, const DateLUTImpl &) { return 0; } static inline UInt16 execute(UInt16, const DateLUTImpl &) { return 0; } }; @@ -49,6 +51,10 @@ struct ToDateImpl { static constexpr auto name = "toDate"; + static inline UInt16 execute(Int64 t, const DateLUTImpl & time_zone) + { + return UInt16(time_zone.toDayNum(t)); + } static inline UInt16 execute(UInt32 t, const DateLUTImpl & time_zone) { return UInt16(time_zone.toDayNum(t)); @@ -65,13 +71,18 @@ struct ToStartOfDayImpl { static constexpr auto name = "toStartOfDay"; + //TODO: right now it is hardcoded to produce DateTime only, needs fixing later. See date_and_time_type_details::ResultDataTypeMap for deduction of result type example. + static inline UInt32 execute(const DecimalUtils::DecimalComponents & t, const DateLUTImpl & time_zone) + { + return time_zone.toDate(static_cast(t.whole)); + } static inline UInt32 execute(UInt32 t, const DateLUTImpl & time_zone) { return time_zone.toDate(t); } static inline UInt32 execute(UInt16 d, const DateLUTImpl & time_zone) { - return time_zone.toDate(DayNum(d)); + return time_zone.toDate(ExtendedDayNum(d)); } using FactorTransform = ZeroTransform; @@ -81,13 +92,19 @@ struct ToMondayImpl { static constexpr auto name = "toMonday"; + static inline UInt16 execute(Int64 t, const DateLUTImpl & time_zone) + { + //return time_zone.toFirstDayNumOfWeek(time_zone.toDayNum(t)); + return time_zone.toFirstDayNumOfWeek(t); + } static inline UInt16 execute(UInt32 t, const DateLUTImpl & time_zone) { - return time_zone.toFirstDayNumOfWeek(time_zone.toDayNum(t)); + //return time_zone.toFirstDayNumOfWeek(time_zone.toDayNum(t)); + return time_zone.toFirstDayNumOfWeek(t); } static inline UInt16 execute(UInt16 d, const DateLUTImpl & time_zone) { - return time_zone.toFirstDayNumOfWeek(DayNum(d)); + return time_zone.toFirstDayNumOfWeek(ExtendedDayNum(d)); } using FactorTransform = ZeroTransform; @@ -97,13 +114,17 @@ struct ToStartOfMonthImpl { static constexpr auto name = "toStartOfMonth"; + static inline UInt16 execute(Int64 t, const DateLUTImpl & time_zone) + { + return time_zone.toFirstDayNumOfMonth(time_zone.toDayNum(t)); + } static inline UInt16 execute(UInt32 t, const DateLUTImpl & time_zone) { return time_zone.toFirstDayNumOfMonth(time_zone.toDayNum(t)); } static inline UInt16 execute(UInt16 d, const DateLUTImpl & time_zone) { - return time_zone.toFirstDayNumOfMonth(DayNum(d)); + return time_zone.toFirstDayNumOfMonth(ExtendedDayNum(d)); } using FactorTransform = ZeroTransform; @@ -113,13 +134,17 @@ struct ToStartOfQuarterImpl { static constexpr auto name = "toStartOfQuarter"; + static inline UInt16 execute(Int64 t, const DateLUTImpl & time_zone) + { + return time_zone.toFirstDayNumOfQuarter(time_zone.toDayNum(t)); + } static inline UInt16 execute(UInt32 t, const DateLUTImpl & time_zone) { return time_zone.toFirstDayNumOfQuarter(time_zone.toDayNum(t)); } static inline UInt16 execute(UInt16 d, const DateLUTImpl & time_zone) { - return time_zone.toFirstDayNumOfQuarter(DayNum(d)); + return time_zone.toFirstDayNumOfQuarter(ExtendedDayNum(d)); } using FactorTransform = ZeroTransform; @@ -129,13 +154,17 @@ struct ToStartOfYearImpl { static constexpr auto name = "toStartOfYear"; + static inline UInt16 execute(Int64 t, const DateLUTImpl & time_zone) + { + return time_zone.toFirstDayNumOfYear(time_zone.toDayNum(t)); + } static inline UInt16 execute(UInt32 t, const DateLUTImpl & time_zone) { return time_zone.toFirstDayNumOfYear(time_zone.toDayNum(t)); } static inline UInt16 execute(UInt16 d, const DateLUTImpl & time_zone) { - return time_zone.toFirstDayNumOfYear(DayNum(d)); + return time_zone.toFirstDayNumOfYear(ExtendedDayNum(d)); } using FactorTransform = ZeroTransform; @@ -144,9 +173,13 @@ struct ToStartOfYearImpl struct ToTimeImpl { + /// When transforming to time, the date will be equated to 1970-01-01. static constexpr auto name = "toTime"; - /// When transforming to time, the date will be equated to 1970-01-02. + static UInt32 execute(const DecimalUtils::DecimalComponents & t, const DateLUTImpl & time_zone) + { + return time_zone.toTime(t.whole) + 86400; + } static inline UInt32 execute(UInt32 t, const DateLUTImpl & time_zone) { return time_zone.toTime(t) + 86400; @@ -164,6 +197,10 @@ struct ToStartOfMinuteImpl { static constexpr auto name = "toStartOfMinute"; + static inline UInt32 execute(const DecimalUtils::DecimalComponents & t, const DateLUTImpl & time_zone) + { + return time_zone.toStartOfMinute(t.whole); + } static inline UInt32 execute(UInt32 t, const DateLUTImpl & time_zone) { return time_zone.toStartOfMinute(t); @@ -215,6 +252,10 @@ struct ToStartOfFiveMinuteImpl { static constexpr auto name = "toStartOfFiveMinute"; + static inline UInt32 execute(const DecimalUtils::DecimalComponents & t, const DateLUTImpl & time_zone) + { + return time_zone.toStartOfFiveMinute(t.whole); + } static inline UInt32 execute(UInt32 t, const DateLUTImpl & time_zone) { return time_zone.toStartOfFiveMinute(t); @@ -231,6 +272,10 @@ struct ToStartOfTenMinutesImpl { static constexpr auto name = "toStartOfTenMinutes"; + static inline UInt32 execute(const DecimalUtils::DecimalComponents & t, const DateLUTImpl & time_zone) + { + return time_zone.toStartOfTenMinutes(t.whole); + } static inline UInt32 execute(UInt32 t, const DateLUTImpl & time_zone) { return time_zone.toStartOfTenMinutes(t); @@ -247,6 +292,10 @@ struct ToStartOfFifteenMinutesImpl { static constexpr auto name = "toStartOfFifteenMinutes"; + static inline UInt32 execute(const DecimalUtils::DecimalComponents & t, const DateLUTImpl & time_zone) + { + return time_zone.toStartOfFifteenMinutes(t.whole); + } static inline UInt32 execute(UInt32 t, const DateLUTImpl & time_zone) { return time_zone.toStartOfFifteenMinutes(t); @@ -264,6 +313,12 @@ struct TimeSlotImpl { static constexpr auto name = "timeSlot"; + //static inline DecimalUtils::DecimalComponents execute(const DecimalUtils::DecimalComponents & t, const DateLUTImpl &) + static inline UInt32 execute(const DecimalUtils::DecimalComponents & t, const DateLUTImpl &) + { + return t.whole / 1800 * 1800; + } + static inline UInt32 execute(UInt32 t, const DateLUTImpl &) { return t / 1800 * 1800; @@ -281,6 +336,11 @@ struct ToStartOfHourImpl { static constexpr auto name = "toStartOfHour"; + static inline UInt32 execute(const DecimalUtils::DecimalComponents & t, const DateLUTImpl & time_zone) + { + return time_zone.toStartOfHour(t.whole); + } + static inline UInt32 execute(UInt32 t, const DateLUTImpl & time_zone) { return time_zone.toStartOfHour(t); @@ -298,13 +358,17 @@ struct ToYearImpl { static constexpr auto name = "toYear"; + static inline UInt16 execute(Int64 t, const DateLUTImpl & time_zone) + { + return time_zone.toYear(t); + } static inline UInt16 execute(UInt32 t, const DateLUTImpl & time_zone) { return time_zone.toYear(t); } static inline UInt16 execute(UInt16 d, const DateLUTImpl & time_zone) { - return time_zone.toYear(DayNum(d)); + return time_zone.toYear(ExtendedDayNum(d)); } using FactorTransform = ZeroTransform; @@ -314,13 +378,17 @@ struct ToQuarterImpl { static constexpr auto name = "toQuarter"; + static inline UInt8 execute(Int64 t, const DateLUTImpl & time_zone) + { + return time_zone.toQuarter(t); + } static inline UInt8 execute(UInt32 t, const DateLUTImpl & time_zone) { return time_zone.toQuarter(t); } static inline UInt8 execute(UInt16 d, const DateLUTImpl & time_zone) { - return time_zone.toQuarter(DayNum(d)); + return time_zone.toQuarter(ExtendedDayNum(d)); } using FactorTransform = ToStartOfYearImpl; @@ -330,13 +398,17 @@ struct ToMonthImpl { static constexpr auto name = "toMonth"; + static inline UInt8 execute(Int64 t, const DateLUTImpl & time_zone) + { + return time_zone.toMonth(t); + } static inline UInt8 execute(UInt32 t, const DateLUTImpl & time_zone) { return time_zone.toMonth(t); } static inline UInt8 execute(UInt16 d, const DateLUTImpl & time_zone) { - return time_zone.toMonth(DayNum(d)); + return time_zone.toMonth(ExtendedDayNum(d)); } using FactorTransform = ToStartOfYearImpl; @@ -346,13 +418,17 @@ struct ToDayOfMonthImpl { static constexpr auto name = "toDayOfMonth"; + static inline UInt8 execute(Int64 t, const DateLUTImpl & time_zone) + { + return time_zone.toDayOfMonth(t); + } static inline UInt8 execute(UInt32 t, const DateLUTImpl & time_zone) { return time_zone.toDayOfMonth(t); } static inline UInt8 execute(UInt16 d, const DateLUTImpl & time_zone) { - return time_zone.toDayOfMonth(DayNum(d)); + return time_zone.toDayOfMonth(ExtendedDayNum(d)); } using FactorTransform = ToStartOfMonthImpl; @@ -362,13 +438,17 @@ struct ToDayOfWeekImpl { static constexpr auto name = "toDayOfWeek"; + static inline UInt8 execute(Int64 t, const DateLUTImpl & time_zone) + { + return time_zone.toDayOfWeek(t); + } static inline UInt8 execute(UInt32 t, const DateLUTImpl & time_zone) { return time_zone.toDayOfWeek(t); } static inline UInt8 execute(UInt16 d, const DateLUTImpl & time_zone) { - return time_zone.toDayOfWeek(DayNum(d)); + return time_zone.toDayOfWeek(ExtendedDayNum(d)); } using FactorTransform = ToMondayImpl; @@ -378,13 +458,17 @@ struct ToDayOfYearImpl { static constexpr auto name = "toDayOfYear"; + static inline UInt16 execute(Int64 t, const DateLUTImpl & time_zone) + { + return time_zone.toDayOfYear(t); + } static inline UInt16 execute(UInt32 t, const DateLUTImpl & time_zone) { return time_zone.toDayOfYear(t); } static inline UInt16 execute(UInt16 d, const DateLUTImpl & time_zone) { - return time_zone.toDayOfYear(DayNum(d)); + return time_zone.toDayOfYear(ExtendedDayNum(d)); } using FactorTransform = ToStartOfYearImpl; @@ -394,6 +478,10 @@ struct ToHourImpl { static constexpr auto name = "toHour"; + static inline UInt8 execute(Int64 t, const DateLUTImpl & time_zone) + { + return time_zone.toHour(t); + } static inline UInt8 execute(UInt32 t, const DateLUTImpl & time_zone) { return time_zone.toHour(t); @@ -411,6 +499,11 @@ struct TimezoneOffsetImpl { static constexpr auto name = "timezoneOffset"; + static inline time_t execute(Int64 t, const DateLUTImpl & time_zone) + { + return time_zone.timezoneOffset(t); + } + static inline time_t execute(UInt32 t, const DateLUTImpl & time_zone) { return time_zone.timezoneOffset(t); @@ -428,6 +521,10 @@ struct ToMinuteImpl { static constexpr auto name = "toMinute"; + static inline UInt8 execute(Int64 t, const DateLUTImpl & time_zone) + { + return time_zone.toMinute(t); + } static inline UInt8 execute(UInt32 t, const DateLUTImpl & time_zone) { return time_zone.toMinute(t); @@ -444,6 +541,10 @@ struct ToSecondImpl { static constexpr auto name = "toSecond"; + static inline UInt8 execute(Int64 t, const DateLUTImpl & time_zone) + { + return time_zone.toSecond(t); + } static inline UInt8 execute(UInt32 t, const DateLUTImpl & time_zone) { return time_zone.toSecond(t); @@ -460,13 +561,17 @@ struct ToISOYearImpl { static constexpr auto name = "toISOYear"; + static inline UInt16 execute(Int64 t, const DateLUTImpl & time_zone) + { + return time_zone.toISOYear(time_zone.toDayNum(t)); + } static inline UInt16 execute(UInt32 t, const DateLUTImpl & time_zone) { return time_zone.toISOYear(time_zone.toDayNum(t)); } static inline UInt16 execute(UInt16 d, const DateLUTImpl & time_zone) { - return time_zone.toISOYear(DayNum(d)); + return time_zone.toISOYear(ExtendedDayNum(d)); } using FactorTransform = ZeroTransform; @@ -476,13 +581,17 @@ struct ToStartOfISOYearImpl { static constexpr auto name = "toStartOfISOYear"; + static inline UInt16 execute(Int64 t, const DateLUTImpl & time_zone) + { + return time_zone.toFirstDayNumOfISOYear(time_zone.toDayNum(t)); + } static inline UInt16 execute(UInt32 t, const DateLUTImpl & time_zone) { return time_zone.toFirstDayNumOfISOYear(time_zone.toDayNum(t)); } static inline UInt16 execute(UInt16 d, const DateLUTImpl & time_zone) { - return time_zone.toFirstDayNumOfISOYear(DayNum(d)); + return time_zone.toFirstDayNumOfISOYear(ExtendedDayNum(d)); } using FactorTransform = ZeroTransform; @@ -492,13 +601,17 @@ struct ToISOWeekImpl { static constexpr auto name = "toISOWeek"; + static inline UInt8 execute(Int64 t, const DateLUTImpl & time_zone) + { + return time_zone.toISOWeek(time_zone.toDayNum(t)); + } static inline UInt8 execute(UInt32 t, const DateLUTImpl & time_zone) { return time_zone.toISOWeek(time_zone.toDayNum(t)); } static inline UInt8 execute(UInt16 d, const DateLUTImpl & time_zone) { - return time_zone.toISOWeek(DayNum(d)); + return time_zone.toISOWeek(ExtendedDayNum(d)); } using FactorTransform = ToISOYearImpl; @@ -508,13 +621,17 @@ struct ToRelativeYearNumImpl { static constexpr auto name = "toRelativeYearNum"; - static inline UInt16 execute(UInt32 t, const DateLUTImpl & time_zone) + static inline UInt16 execute(Int64 t, const DateLUTImpl & time_zone) { return time_zone.toYear(t); } + static inline UInt16 execute(UInt32 t, const DateLUTImpl & time_zone) + { + return time_zone.toYear(static_cast(t)); + } static inline UInt16 execute(UInt16 d, const DateLUTImpl & time_zone) { - return time_zone.toYear(DayNum(d)); + return time_zone.toYear(ExtendedDayNum(d)); } using FactorTransform = ZeroTransform; @@ -524,13 +641,17 @@ struct ToRelativeQuarterNumImpl { static constexpr auto name = "toRelativeQuarterNum"; - static inline UInt16 execute(UInt32 t, const DateLUTImpl & time_zone) + static inline UInt16 execute(Int64 t, const DateLUTImpl & time_zone) { return time_zone.toRelativeQuarterNum(t); } + static inline UInt16 execute(UInt32 t, const DateLUTImpl & time_zone) + { + return time_zone.toRelativeQuarterNum(static_cast(t)); + } static inline UInt16 execute(UInt16 d, const DateLUTImpl & time_zone) { - return time_zone.toRelativeQuarterNum(DayNum(d)); + return time_zone.toRelativeQuarterNum(ExtendedDayNum(d)); } using FactorTransform = ZeroTransform; @@ -540,13 +661,17 @@ struct ToRelativeMonthNumImpl { static constexpr auto name = "toRelativeMonthNum"; - static inline UInt16 execute(UInt32 t, const DateLUTImpl & time_zone) + static inline UInt16 execute(Int64 t, const DateLUTImpl & time_zone) { return time_zone.toRelativeMonthNum(t); } + static inline UInt16 execute(UInt32 t, const DateLUTImpl & time_zone) + { + return time_zone.toRelativeMonthNum(static_cast(t)); + } static inline UInt16 execute(UInt16 d, const DateLUTImpl & time_zone) { - return time_zone.toRelativeMonthNum(DayNum(d)); + return time_zone.toRelativeMonthNum(ExtendedDayNum(d)); } using FactorTransform = ZeroTransform; @@ -556,13 +681,17 @@ struct ToRelativeWeekNumImpl { static constexpr auto name = "toRelativeWeekNum"; - static inline UInt16 execute(UInt32 t, const DateLUTImpl & time_zone) + static inline UInt16 execute(Int64 t, const DateLUTImpl & time_zone) { return time_zone.toRelativeWeekNum(t); } + static inline UInt16 execute(UInt32 t, const DateLUTImpl & time_zone) + { + return time_zone.toRelativeWeekNum(static_cast(t)); + } static inline UInt16 execute(UInt16 d, const DateLUTImpl & time_zone) { - return time_zone.toRelativeWeekNum(DayNum(d)); + return time_zone.toRelativeWeekNum(ExtendedDayNum(d)); } using FactorTransform = ZeroTransform; @@ -572,10 +701,14 @@ struct ToRelativeDayNumImpl { static constexpr auto name = "toRelativeDayNum"; - static inline UInt16 execute(UInt32 t, const DateLUTImpl & time_zone) + static inline UInt16 execute(Int64 t, const DateLUTImpl & time_zone) { return time_zone.toDayNum(t); } + static inline UInt16 execute(UInt32 t, const DateLUTImpl & time_zone) + { + return time_zone.toDayNum(static_cast(t)); + } static inline UInt16 execute(UInt16 d, const DateLUTImpl &) { return static_cast(d); @@ -589,13 +722,17 @@ struct ToRelativeHourNumImpl { static constexpr auto name = "toRelativeHourNum"; - static inline UInt32 execute(UInt32 t, const DateLUTImpl & time_zone) + static inline UInt32 execute(Int64 t, const DateLUTImpl & time_zone) { return time_zone.toRelativeHourNum(t); } + static inline UInt32 execute(UInt32 t, const DateLUTImpl & time_zone) + { + return time_zone.toRelativeHourNum(static_cast(t)); + } static inline UInt32 execute(UInt16 d, const DateLUTImpl & time_zone) { - return time_zone.toRelativeHourNum(DayNum(d)); + return time_zone.toRelativeHourNum(ExtendedDayNum(d)); } using FactorTransform = ZeroTransform; @@ -605,13 +742,17 @@ struct ToRelativeMinuteNumImpl { static constexpr auto name = "toRelativeMinuteNum"; - static inline UInt32 execute(UInt32 t, const DateLUTImpl & time_zone) + static inline UInt32 execute(Int64 t, const DateLUTImpl & time_zone) { return time_zone.toRelativeMinuteNum(t); } + static inline UInt32 execute(UInt32 t, const DateLUTImpl & time_zone) + { + return time_zone.toRelativeMinuteNum(static_cast(t)); + } static inline UInt32 execute(UInt16 d, const DateLUTImpl & time_zone) { - return time_zone.toRelativeMinuteNum(DayNum(d)); + return time_zone.toRelativeMinuteNum(ExtendedDayNum(d)); } using FactorTransform = ZeroTransform; @@ -621,13 +762,17 @@ struct ToRelativeSecondNumImpl { static constexpr auto name = "toRelativeSecondNum"; + static inline Int64 execute(Int64 t, const DateLUTImpl &) + { + return t; + } static inline UInt32 execute(UInt32 t, const DateLUTImpl &) { return t; } static inline UInt32 execute(UInt16 d, const DateLUTImpl & time_zone) { - return time_zone.fromDayNum(DayNum(d)); + return time_zone.fromDayNum(ExtendedDayNum(d)); } using FactorTransform = ZeroTransform; @@ -637,6 +782,10 @@ struct ToYYYYMMImpl { static constexpr auto name = "toYYYYMM"; + static inline UInt32 execute(Int64 t, const DateLUTImpl & time_zone) + { + return time_zone.toNumYYYYMM(t); + } static inline UInt32 execute(UInt32 t, const DateLUTImpl & time_zone) { return time_zone.toNumYYYYMM(t); @@ -653,6 +802,10 @@ struct ToYYYYMMDDImpl { static constexpr auto name = "toYYYYMMDD"; + static inline UInt32 execute(Int64 t, const DateLUTImpl & time_zone) + { + return time_zone.toNumYYYYMMDD(t); + } static inline UInt32 execute(UInt32 t, const DateLUTImpl & time_zone) { return time_zone.toNumYYYYMMDD(t); @@ -669,6 +822,10 @@ struct ToYYYYMMDDhhmmssImpl { static constexpr auto name = "toYYYYMMDDhhmmss"; + static inline UInt64 execute(Int64 t, const DateLUTImpl & time_zone) + { + return time_zone.toNumYYYYMMDDhhmmss(t); + } static inline UInt64 execute(UInt32 t, const DateLUTImpl & time_zone) { return time_zone.toNumYYYYMMDDhhmmss(t); diff --git a/src/Functions/FunctionCustomWeekToSomething.h b/src/Functions/FunctionCustomWeekToSomething.h index 8a343cffb95..5634ea11584 100644 --- a/src/Functions/FunctionCustomWeekToSomething.h +++ b/src/Functions/FunctionCustomWeekToSomething.h @@ -4,6 +4,7 @@ #include #include #include +#include #include #include diff --git a/src/Functions/FunctionDateOrDateTimeAddInterval.h b/src/Functions/FunctionDateOrDateTimeAddInterval.h index 5f964b899b4..2b0082f4334 100644 --- a/src/Functions/FunctionDateOrDateTimeAddInterval.h +++ b/src/Functions/FunctionDateOrDateTimeAddInterval.h @@ -11,6 +11,7 @@ #include #include #include +#include #include @@ -25,31 +26,6 @@ namespace ErrorCodes extern const int ILLEGAL_COLUMN; } -/// AddOnDateTime64DefaultImpl provides default implementation of add-X functionality for DateTime64. -/// -/// Default implementation is not to change fractional part, but only modify whole part as if it was DateTime. -/// That means large whole values (for scale less than 9) might not fit into UInt32-range, -/// and hence default implementation will produce incorrect results. -template -struct AddOnDateTime64DefaultImpl -{ - AddOnDateTime64DefaultImpl(UInt32 scale_ = 0) - : scale_multiplier(DecimalUtils::scaleMultiplier(scale_)) - {} - - // Default implementation for add/sub on DateTime64: do math on whole part (the same way as for DateTime), leave fractional as it is. - inline DateTime64 execute(const DateTime64 & t, Int64 delta, const DateLUTImpl & time_zone) const - { - const auto components = DecimalUtils::splitWithScaleMultiplier(t, scale_multiplier); - - const auto whole = static_cast(this)->execute(static_cast(components.whole), delta, time_zone); - return DecimalUtils::decimalFromComponentsWithMultiplier(static_cast(whole), components.fractional, scale_multiplier); - } - - UInt32 scale_multiplier = 1; -}; - - /// Type of first argument of 'execute' function overload defines what INPUT DataType it is used for. /// Return type defines what is the OUTPUT (return) type of the CH function. /// Corresponding types: @@ -60,14 +36,15 @@ struct AddOnDateTime64DefaultImpl /// - 'AddSecondsImpl::execute(UInt32, ...) -> UInt32' is available to the ClickHouse users as 'addSeconds(DateTime, ...) -> DateTime' /// - 'AddSecondsImpl::execute(UInt16, ...) -> UInt32' is available to the ClickHouse users as 'addSeconds(Date, ...) -> DateTime' -struct AddSecondsImpl : public AddOnDateTime64DefaultImpl +struct AddSecondsImpl { - using Base = AddOnDateTime64DefaultImpl; - using Base::Base; - using Base::execute; - static constexpr auto name = "addSeconds"; + static inline NO_SANITIZE_UNDEFINED DecimalUtils::DecimalComponents execute(DecimalUtils::DecimalComponents t, Int64 delta, const DateLUTImpl &) + { + return {t.whole + delta, t.fractional}; + } + static inline NO_SANITIZE_UNDEFINED UInt32 execute(UInt32 t, Int64 delta, const DateLUTImpl &) { return t + delta; @@ -75,18 +52,19 @@ struct AddSecondsImpl : public AddOnDateTime64DefaultImpl static inline NO_SANITIZE_UNDEFINED UInt32 execute(UInt16 d, Int64 delta, const DateLUTImpl & time_zone) { - return time_zone.fromDayNum(DayNum(d)) + delta; + return time_zone.fromDayNum(ExtendedDayNum(d)) + delta; } }; -struct AddMinutesImpl : public AddOnDateTime64DefaultImpl +struct AddMinutesImpl { - using Base = AddOnDateTime64DefaultImpl; - using Base::Base; - using Base::execute; - static constexpr auto name = "addMinutes"; + static inline NO_SANITIZE_UNDEFINED DecimalUtils::DecimalComponents execute(DecimalUtils::DecimalComponents t, Int64 delta, const DateLUTImpl &) + { + return {t.whole + delta * 60, t.fractional}; + } + static inline NO_SANITIZE_UNDEFINED UInt32 execute(UInt32 t, Int64 delta, const DateLUTImpl &) { return t + delta * 60; @@ -94,18 +72,18 @@ struct AddMinutesImpl : public AddOnDateTime64DefaultImpl static inline NO_SANITIZE_UNDEFINED UInt32 execute(UInt16 d, Int64 delta, const DateLUTImpl & time_zone) { - return time_zone.fromDayNum(DayNum(d)) + delta * 60; + return time_zone.fromDayNum(ExtendedDayNum(d)) + delta * 60; } }; -struct AddHoursImpl : public AddOnDateTime64DefaultImpl +struct AddHoursImpl { - using Base = AddOnDateTime64DefaultImpl; - using Base::Base; - using Base::execute; - static constexpr auto name = "addHours"; + static inline NO_SANITIZE_UNDEFINED DecimalUtils::DecimalComponents execute(DecimalUtils::DecimalComponents t, Int64 delta, const DateLUTImpl &) + { + return {t.whole + delta * 3600, t.fractional}; + } static inline NO_SANITIZE_UNDEFINED UInt32 execute(UInt32 t, Int64 delta, const DateLUTImpl &) { return t + delta * 3600; @@ -113,19 +91,20 @@ struct AddHoursImpl : public AddOnDateTime64DefaultImpl static inline NO_SANITIZE_UNDEFINED UInt32 execute(UInt16 d, Int64 delta, const DateLUTImpl & time_zone) { - return time_zone.fromDayNum(DayNum(d)) + delta * 3600; + return time_zone.fromDayNum(ExtendedDayNum(d)) + delta * 3600; } }; -struct AddDaysImpl : public AddOnDateTime64DefaultImpl +struct AddDaysImpl { - using Base = AddOnDateTime64DefaultImpl; - using Base::Base; - using Base::execute; - static constexpr auto name = "addDays"; - static inline UInt32 execute(UInt32 t, Int64 delta, const DateLUTImpl & time_zone) + static inline NO_SANITIZE_UNDEFINED DecimalUtils::DecimalComponents execute(DecimalUtils::DecimalComponents t, Int64 delta, const DateLUTImpl & time_zone) + { + return {time_zone.addDays(t.whole, delta), t.fractional}; + } + + static inline NO_SANITIZE_UNDEFINED UInt32 execute(UInt32 t, Int64 delta, const DateLUTImpl & time_zone) { return time_zone.addDays(t, delta); } @@ -136,14 +115,15 @@ struct AddDaysImpl : public AddOnDateTime64DefaultImpl } }; -struct AddWeeksImpl : public AddOnDateTime64DefaultImpl +struct AddWeeksImpl { - using Base = AddOnDateTime64DefaultImpl; - using Base::Base; - using Base::execute; - static constexpr auto name = "addWeeks"; + static inline NO_SANITIZE_UNDEFINED DecimalUtils::DecimalComponents execute(DecimalUtils::DecimalComponents t, Int64 delta, const DateLUTImpl & time_zone) + { + return {time_zone.addWeeks(t.whole, delta), t.fractional}; + } + static inline NO_SANITIZE_UNDEFINED UInt32 execute(UInt32 t, Int64 delta, const DateLUTImpl & time_zone) { return time_zone.addWeeks(t, delta); @@ -155,14 +135,15 @@ struct AddWeeksImpl : public AddOnDateTime64DefaultImpl } }; -struct AddMonthsImpl : public AddOnDateTime64DefaultImpl +struct AddMonthsImpl { - using Base = AddOnDateTime64DefaultImpl; - using Base::Base; - using Base::execute; - static constexpr auto name = "addMonths"; + static inline DecimalUtils::DecimalComponents execute(DecimalUtils::DecimalComponents t, Int64 delta, const DateLUTImpl & time_zone) + { + return {time_zone.addMonths(t.whole, delta), t.fractional}; + } + static inline UInt32 execute(UInt32 t, Int64 delta, const DateLUTImpl & time_zone) { return time_zone.addMonths(t, delta); @@ -170,18 +151,19 @@ struct AddMonthsImpl : public AddOnDateTime64DefaultImpl static inline UInt16 execute(UInt16 d, Int64 delta, const DateLUTImpl & time_zone) { - return time_zone.addMonths(DayNum(d), delta); + return time_zone.addMonths(ExtendedDayNum(d), delta); } }; -struct AddQuartersImpl : public AddOnDateTime64DefaultImpl +struct AddQuartersImpl { - using Base = AddOnDateTime64DefaultImpl; - using Base::Base; - using Base::execute; - static constexpr auto name = "addQuarters"; + static inline DecimalUtils::DecimalComponents execute(DecimalUtils::DecimalComponents t, Int64 delta, const DateLUTImpl & time_zone) + { + return {time_zone.addQuarters(t.whole, delta), t.fractional}; + } + static inline UInt32 execute(UInt32 t, Int64 delta, const DateLUTImpl & time_zone) { return time_zone.addQuarters(t, delta); @@ -189,18 +171,19 @@ struct AddQuartersImpl : public AddOnDateTime64DefaultImpl static inline UInt16 execute(UInt16 d, Int64 delta, const DateLUTImpl & time_zone) { - return time_zone.addQuarters(DayNum(d), delta); + return time_zone.addQuarters(ExtendedDayNum(d), delta); } }; -struct AddYearsImpl : public AddOnDateTime64DefaultImpl +struct AddYearsImpl { - using Base = AddOnDateTime64DefaultImpl; - using Base::Base; - using Base::execute; - static constexpr auto name = "addYears"; + static inline DecimalUtils::DecimalComponents execute(DecimalUtils::DecimalComponents t, Int64 delta, const DateLUTImpl & time_zone) + { + return {time_zone.addYears(t.whole, delta), t.fractional}; + } + static inline UInt32 execute(UInt32 t, Int64 delta, const DateLUTImpl & time_zone) { return time_zone.addYears(t, delta); @@ -208,7 +191,7 @@ struct AddYearsImpl : public AddOnDateTime64DefaultImpl static inline UInt16 execute(UInt16 d, Int64 delta, const DateLUTImpl & time_zone) { - return time_zone.addYears(DayNum(d), delta); + return time_zone.addYears(ExtendedDayNum(d), delta); } }; @@ -351,6 +334,7 @@ template <> struct ResultDataTypeMap { using ResultDataType = DataTy template <> struct ResultDataTypeMap { using ResultDataType = DataTypeDateTime; }; template <> struct ResultDataTypeMap { using ResultDataType = DataTypeDateTime; }; template <> struct ResultDataTypeMap { using ResultDataType = DataTypeDateTime64; }; +template <> struct ResultDataTypeMap { using ResultDataType = DataTypeDateTime64; }; } template @@ -417,10 +401,18 @@ public: } } + // TransformDateTime64 helps choosing correct overload of exec and does some transformations + // on input and output parameters to simplify support of DateTime64 in concrete Transform. + template + using TransformType = std::conditional_t< + std::is_same_v, + TransformDateTime64, + Transform>; + /// Helper templates to deduce return type based on argument type, since some overloads may promote or denote types, /// e.g. addSeconds(Date, 1) => DateTime template - using TransformExecuteReturnType = decltype(std::declval().execute(FieldType(), 0, std::declval())); + using TransformExecuteReturnType = decltype(std::declval>().execute(FieldType(), 0, std::declval())); // Deduces RETURN DataType from INPUT DataType, based on return type of Transform{}.execute(INPUT_TYPE, UInt64, DateLUTImpl). // e.g. for Transform-type that has execute()-overload with 'UInt16' input and 'UInt32' return, @@ -475,8 +467,9 @@ public: } else if (const auto * datetime64_type = assert_cast(from_type)) { - return DateTimeAddIntervalImpl, Transform>::execute( - Transform{datetime64_type->getScale()}, arguments, result_type); + using WrappedTransformType = TransformType; + return DateTimeAddIntervalImpl, WrappedTransformType>::execute( + WrappedTransformType{datetime64_type->getScale()}, arguments, result_type); } else throw Exception("Illegal type " + arguments[0].type->getName() + " of first argument of function " + getName(), diff --git a/src/Functions/FunctionDateOrDateTimeToSomething.h b/src/Functions/FunctionDateOrDateTimeToSomething.h index e0676f3dc0f..abe859e2f29 100644 --- a/src/Functions/FunctionDateOrDateTimeToSomething.h +++ b/src/Functions/FunctionDateOrDateTimeToSomething.h @@ -5,6 +5,7 @@ #include #include #include +#include #include @@ -107,6 +108,7 @@ public: else if (which.isDateTime64()) { const auto scale = static_cast(from_type)->getScale(); + const TransformDateTime64 transformer(scale); return DateTimeTransformImpl::execute(arguments, result_type, input_rows_count, transformer); } @@ -133,7 +135,6 @@ public: /// This method is called only if the function has one argument. Therefore, we do not care about the non-local time zone. const DateLUTImpl & date_lut = DateLUT::instance(); - if (left.isNull() || right.isNull()) return is_not_monotonic; diff --git a/src/Functions/FunctionsConversion.h b/src/Functions/FunctionsConversion.h index 2e2a4ce9cfa..62577b8b402 100644 --- a/src/Functions/FunctionsConversion.h +++ b/src/Functions/FunctionsConversion.h @@ -44,6 +44,7 @@ #include #include #include +#include #include #include #include @@ -310,10 +311,15 @@ struct ToDateTimeImpl return time_zone.fromDayNum(DayNum(d)); } - // no-op conversion from DateTime to DateTime, used in DateTime64 to DateTime conversion. - static inline UInt32 execute(UInt32 d, const DateLUTImpl & /*time_zone*/) + static inline UInt32 execute(UInt32 dt, const DateLUTImpl & /*time_zone*/) { - return d; + return dt; + } + + // TODO: return UInt32 ??? + static inline Int64 execute(Int64 dt64, const DateLUTImpl & /*time_zone*/) + { + return dt64; } }; @@ -329,6 +335,7 @@ struct ToDateTransform32Or64 static inline NO_SANITIZE_UNDEFINED ToType execute(const FromType & from, const DateLUTImpl & time_zone) { + // since converting to Date, no need in values outside of default LUT range. return (from < 0xFFFF) ? from : time_zone.toDayNum(std::min(time_t(from), time_t(0xFFFFFFFF))); @@ -342,6 +349,7 @@ struct ToDateTransform32Or64Signed static inline NO_SANITIZE_UNDEFINED ToType execute(const FromType & from, const DateLUTImpl & time_zone) { + // TODO: decide narrow or extended range based on FromType /// The function should be monotonic (better for query optimizations), so we saturate instead of overflow. if (from < 0) return 0; @@ -447,35 +455,8 @@ template struct ConvertImpl struct ConvertImpl : DateTimeTransformImpl> {}; - -/** Conversion of Date or DateTime to DateTime64: add zero sub-second part. - */ -struct ToDateTime64Transform -{ - static constexpr auto name = "toDateTime64"; - - const DateTime64::NativeType scale_multiplier = 1; - - ToDateTime64Transform(UInt32 scale = 0) - : scale_multiplier(DecimalUtils::scaleMultiplier(scale)) - {} - - inline DateTime64::NativeType execute(UInt16 d, const DateLUTImpl & time_zone) const - { - const auto dt = ToDateTimeImpl::execute(d, time_zone); - return execute(dt, time_zone); - } - - inline DateTime64::NativeType execute(UInt32 dt, const DateLUTImpl & /*time_zone*/) const - { - return DecimalUtils::decimalFromComponentsWithMultiplier(dt, 0, scale_multiplier); - } -}; - -template struct ConvertImpl - : DateTimeTransformImpl {}; -template struct ConvertImpl - : DateTimeTransformImpl {}; +const time_t LUT_MIN_TIME = -1420070400l; // 1925-01-01 UTC +const time_t LUT_MAX_TIME = 9877248000l; // 2282-12-31 UTC /** Conversion of numeric to DateTime64 */ @@ -493,7 +474,7 @@ struct ToDateTime64TransformUnsigned inline NO_SANITIZE_UNDEFINED DateTime64::NativeType execute(FromType from, const DateLUTImpl &) const { - from = std::min(time_t(from), time_t(0xFFFFFFFF)); + from = std::min(from, LUT_MAX_TIME); return DecimalUtils::decimalFromComponentsWithMultiplier(from, 0, scale_multiplier); } }; @@ -510,9 +491,8 @@ struct ToDateTime64TransformSigned inline NO_SANITIZE_UNDEFINED DateTime64::NativeType execute(FromType from, const DateLUTImpl &) const { - if (from < 0) - return 0; - from = std::min(time_t(from), time_t(0xFFFFFFFF)); + from = std::max(from, LUT_MIN_TIME); + from = std::min(from, LUT_MAX_TIME); return DecimalUtils::decimalFromComponentsWithMultiplier(from, 0, scale_multiplier); } }; @@ -551,6 +531,7 @@ template struct ConvertImpl struct ConvertImpl : DateTimeTransformImpl> {}; + /** Conversion of DateTime64 to Date or DateTime: discards fractional part. */ template @@ -571,10 +552,41 @@ struct FromDateTime64Transform } }; +/** Conversion of DateTime64 to Date or DateTime: discards fractional part. + */ template struct ConvertImpl - : DateTimeTransformImpl> {}; + : DateTimeTransformImpl> {}; template struct ConvertImpl - : DateTimeTransformImpl> {}; + : DateTimeTransformImpl> {}; + +struct ToDateTime64Transform +{ + static constexpr auto name = "toDateTime64"; + + const DateTime64::NativeType scale_multiplier = 1; + + ToDateTime64Transform(UInt32 scale = 0) + : scale_multiplier(DecimalUtils::scaleMultiplier(scale)) + {} + + inline DateTime64::NativeType execute(UInt16 d, const DateLUTImpl & time_zone) const + { + const auto dt = ToDateTimeImpl::execute(d, time_zone); + return execute(dt, time_zone); + } + + inline DateTime64::NativeType execute(UInt32 dt, const DateLUTImpl & /*time_zone*/) const + { + return DecimalUtils::decimalFromComponentsWithMultiplier(dt, 0, scale_multiplier); + } +}; + +/** Conversion of Date or DateTime to DateTime64: add zero sub-second part. + */ +template struct ConvertImpl + : DateTimeTransformImpl {}; +template struct ConvertImpl + : DateTimeTransformImpl {}; /** Transformation of numbers, dates, datetimes to strings: through formatting. @@ -658,7 +670,6 @@ struct ConvertImpl(*col_with_type_and_name.type); const DateLUTImpl * time_zone = nullptr; - /// For argument of DateTime type, second argument with time zone could be specified. if constexpr (std::is_same_v || std::is_same_v) time_zone = &extractTimeZoneFromFunctionArguments(arguments, 1, 0); @@ -754,6 +765,7 @@ inline void parseImpl(DataTypeDate::FieldType & x, ReadBuffer & rb x = tmp; } +// NOTE: no need of extra overload of DateTime64, since readDateTimeText64 has different signature and that case is explicitly handled in the calling code. template <> inline void parseImpl(DataTypeDateTime::FieldType & x, ReadBuffer & rb, const DateLUTImpl * time_zone) { @@ -762,6 +774,7 @@ inline void parseImpl(DataTypeDateTime::FieldType & x, ReadBuf x = tmp; } + template <> inline void parseImpl(DataTypeUUID::FieldType & x, ReadBuffer & rb, const DateLUTImpl *) { @@ -989,9 +1002,18 @@ struct ConvertThroughParsing } else if constexpr (parsing_mode == ConvertFromStringParsingMode::BestEffortUS) { - time_t res; - parseDateTimeBestEffortUS(res, read_buffer, *local_time_zone, *utc_time_zone); - vec_to[i] = res; + if constexpr (to_datetime64) + { + DateTime64 res = 0; + parseDateTime64BestEffortUS(res, vec_to.getScale(), read_buffer, *local_time_zone, *utc_time_zone); + vec_to[i] = res; + } + else + { + time_t res; + parseDateTimeBestEffortUS(res, read_buffer, *local_time_zone, *utc_time_zone); + vec_to[i] = res; + } } else { diff --git a/src/Functions/TransformDateTime64.h b/src/Functions/TransformDateTime64.h new file mode 100644 index 00000000000..0a5e36cd2bd --- /dev/null +++ b/src/Functions/TransformDateTime64.h @@ -0,0 +1,92 @@ +#pragma once + +#include +#include + +namespace DB +{ +/** Tansform-type wrapper for DateTime64, simplifies DateTime64 support for given Transform. + * + * Depending on what overloads of Transform::execute() are available, when called with DateTime64 value, + * invokes Transform::execute() with either: + * * whole part of DateTime64 value, discarding fractional part (1) + * * DateTime64 value and scale factor (2) + * * DateTime64 broken down to components, result of execute is then re-assembled back into DateTime64 value (3) + * + * Suitable Transfotm-types are commonly used in Date/DateTime manipulation functions, + * and should implement static (or const) function with following signatures: + * 1: + * R execute(Int64 whole_value, ... ) + * 2: + * R execute(DateTime64 value, Int64 scale_multiplier, ... ) + * 3: + * R execute(DecimalUtils::DecimalComponents components, ... ) + * + * Where R could be of arbitrary type, in case of (3) if R is DecimalUtils::DecimalComponents, result is re-assembed back into DateTime64. +*/ +template +class TransformDateTime64 +{ +private: + // Detect if Transform::execute is const or static method + // with signature defined by template args (ignoring result type). + template + struct TransformHasExecuteOverload : std::false_type {}; + + template + struct TransformHasExecuteOverload().execute(std::declval()...))>, Args...> + : std::true_type {}; + + template + static constexpr bool TransformHasExecuteOverload_v = TransformHasExecuteOverload::value; + +public: + static constexpr auto name = Transform::name; + + // non-explicit constructor to allow creating from scale value (or with no scale at all), indispensable in some contexts. + TransformDateTime64(UInt32 scale_ = 0) + : scale_multiplier(DecimalUtils::scaleMultiplier(scale_)) + {} + + template + inline auto execute(const DateTime64 & t, Args && ... args) const + { + if constexpr (TransformHasExecuteOverload_v) + { + return wrapped_transform.execute(t, scale_multiplier, std::forward(args)...); + } + else if constexpr (TransformHasExecuteOverload_v, Args...>) + { + auto components = DecimalUtils::splitWithScaleMultiplier(t, scale_multiplier); + + const auto result = wrapped_transform.execute(components, std::forward(args)...); + using ResultType = std::decay_t; + + if constexpr (std::is_same_v, ResultType>) + { + return DecimalUtils::decimalFromComponentsWithMultiplier(result, scale_multiplier); + } + else + { + return result; + } + } + else + { + const auto components = DecimalUtils::splitWithScaleMultiplier(t, scale_multiplier); + return wrapped_transform.execute(static_cast(components.whole), std::forward(args)...); + } + } + + template >>> + inline auto execute(const T & t, Args && ... args) const + { + return wrapped_transform.execute(t, std::forward(args)...); + } + +private: + DateTime64::NativeType scale_multiplier = 1; + Transform wrapped_transform = {}; +}; + +} diff --git a/src/Functions/dateDiff.cpp b/src/Functions/dateDiff.cpp index 54833eb359f..f660b92efc5 100644 --- a/src/Functions/dateDiff.cpp +++ b/src/Functions/dateDiff.cpp @@ -97,8 +97,8 @@ public: size_t rows = input_rows_count; auto res = ColumnInt64::create(rows); - const DateLUTImpl & timezone_x = extractTimeZoneFromFunctionArguments(arguments, 3, 1); - const DateLUTImpl & timezone_y = extractTimeZoneFromFunctionArguments(arguments, 3, 2); + const auto & timezone_x = extractTimeZoneFromFunctionArguments(arguments, 3, 1); + const auto & timezone_y = extractTimeZoneFromFunctionArguments(arguments, 3, 2); if (unit == "year" || unit == "yy" || unit == "yyyy") dispatchForColumns(x, y, timezone_x, timezone_y, res->getData()); diff --git a/src/Functions/extractTimeZoneFromFunctionArguments.cpp b/src/Functions/extractTimeZoneFromFunctionArguments.cpp index 9d6e54a599e..0ba08b3c612 100644 --- a/src/Functions/extractTimeZoneFromFunctionArguments.cpp +++ b/src/Functions/extractTimeZoneFromFunctionArguments.cpp @@ -66,10 +66,11 @@ const DateLUTImpl & extractTimeZoneFromFunctionArguments(const ColumnsWithTypeAn if (arguments.empty()) return DateLUT::instance(); + const auto & dt_arg = arguments[datetime_arg_num].type.get(); /// If time zone is attached to an argument of type DateTime. - if (const auto * type = checkAndGetDataType(arguments[datetime_arg_num].type.get())) + if (const auto * type = checkAndGetDataType(dt_arg)) return type->getTimeZone(); - if (const auto * type = checkAndGetDataType(arguments[datetime_arg_num].type.get())) + if (const auto * type = checkAndGetDataType(dt_arg)) return type->getTimeZone(); return DateLUT::instance(); diff --git a/src/Functions/formatDateTime.cpp b/src/Functions/formatDateTime.cpp index fd909ed6fce..5128f077c5a 100644 --- a/src/Functions/formatDateTime.cpp +++ b/src/Functions/formatDateTime.cpp @@ -46,9 +46,8 @@ template <> struct ActionValueTypeMap { using ActionValueTyp template <> struct ActionValueTypeMap { using ActionValueType = UInt32; }; template <> struct ActionValueTypeMap { using ActionValueType = UInt16; }; template <> struct ActionValueTypeMap { using ActionValueType = UInt32; }; -// TODO(vnemkov): once there is support for Int64 in LUT, make that Int64. // TODO(vnemkov): to add sub-second format instruction, make that DateTime64 and do some math in Action. -template <> struct ActionValueTypeMap { using ActionValueType = UInt32; }; +template <> struct ActionValueTypeMap { using ActionValueType = Int64; }; /** formatDateTime(time, 'pattern') @@ -434,7 +433,6 @@ public: time_zone_tmp = &DateLUT::instance(); const DateLUTImpl & time_zone = *time_zone_tmp; - const auto & vec = times->getData(); UInt32 scale [[maybe_unused]] = 0; @@ -519,6 +517,8 @@ public: { if constexpr (std::is_same_v) instructions.emplace_back(func, shift); + else if constexpr (std::is_same_v) + instructions.emplace_back(func, shift); else add_shift(shift); }; diff --git a/src/Functions/now64.cpp b/src/Functions/now64.cpp index feb821fde82..32c7a95de17 100644 --- a/src/Functions/now64.cpp +++ b/src/Functions/now64.cpp @@ -30,7 +30,7 @@ Field nowSubsecond(UInt32 scale) if (clock_gettime(CLOCK_REALTIME, &spec)) throwFromErrno("Cannot clock_gettime.", ErrorCodes::CANNOT_CLOCK_GETTIME); - DecimalUtils::DecimalComponents components{spec.tv_sec, spec.tv_nsec}; + DecimalUtils::DecimalComponents components{spec.tv_sec, spec.tv_nsec}; // clock_gettime produces subsecond part in nanoseconds, but decimalFromComponents fractional is scale-dependent. // Andjust fractional to scale, e.g. for 123456789 nanoseconds: diff --git a/src/Functions/toStartOfInterval.cpp b/src/Functions/toStartOfInterval.cpp index 6f5a52ca182..f194da166aa 100644 --- a/src/Functions/toStartOfInterval.cpp +++ b/src/Functions/toStartOfInterval.cpp @@ -7,6 +7,7 @@ #include #include #include +#include #include @@ -35,13 +36,18 @@ namespace static UInt16 execute(UInt16 d, UInt64 years, const DateLUTImpl & time_zone) { - return time_zone.toStartOfYearInterval(DayNum(d), years); + return time_zone.toStartOfYearInterval(ExtendedDayNum(d), years); } static UInt16 execute(UInt32 t, UInt64 years, const DateLUTImpl & time_zone) { return time_zone.toStartOfYearInterval(time_zone.toDayNum(t), years); } + + static UInt16 execute(Int64 t, UInt64 years, const DateLUTImpl & time_zone) + { + return time_zone.toStartOfYearInterval(time_zone.toDayNum(t), years); + } }; template <> @@ -51,13 +57,18 @@ namespace static UInt16 execute(UInt16 d, UInt64 quarters, const DateLUTImpl & time_zone) { - return time_zone.toStartOfQuarterInterval(DayNum(d), quarters); + return time_zone.toStartOfQuarterInterval(ExtendedDayNum(d), quarters); } static UInt16 execute(UInt32 t, UInt64 quarters, const DateLUTImpl & time_zone) { return time_zone.toStartOfQuarterInterval(time_zone.toDayNum(t), quarters); } + + static UInt16 execute(Int64 t, UInt64 quarters, const DateLUTImpl & time_zone) + { + return time_zone.toStartOfQuarterInterval(time_zone.toDayNum(t), quarters); + } }; template <> @@ -67,13 +78,18 @@ namespace static UInt16 execute(UInt16 d, UInt64 months, const DateLUTImpl & time_zone) { - return time_zone.toStartOfMonthInterval(DayNum(d), months); + return time_zone.toStartOfMonthInterval(ExtendedDayNum(d), months); } static UInt16 execute(UInt32 t, UInt64 months, const DateLUTImpl & time_zone) { return time_zone.toStartOfMonthInterval(time_zone.toDayNum(t), months); } + + static UInt16 execute(Int64 t, UInt64 months, const DateLUTImpl & time_zone) + { + return time_zone.toStartOfMonthInterval(time_zone.toDayNum(t), months); + } }; template <> @@ -83,13 +99,18 @@ namespace static UInt16 execute(UInt16 d, UInt64 weeks, const DateLUTImpl & time_zone) { - return time_zone.toStartOfWeekInterval(DayNum(d), weeks); + return time_zone.toStartOfWeekInterval(ExtendedDayNum(d), weeks); } static UInt16 execute(UInt32 t, UInt64 weeks, const DateLUTImpl & time_zone) { return time_zone.toStartOfWeekInterval(time_zone.toDayNum(t), weeks); } + + static UInt16 execute(Int64 t, UInt64 weeks, const DateLUTImpl & time_zone) + { + return time_zone.toStartOfWeekInterval(time_zone.toDayNum(t), weeks); + } }; template <> @@ -99,13 +120,18 @@ namespace static UInt32 execute(UInt16 d, UInt64 days, const DateLUTImpl & time_zone) { - return time_zone.toStartOfDayInterval(DayNum(d), days); + return time_zone.toStartOfDayInterval(ExtendedDayNum(d), days); } static UInt32 execute(UInt32 t, UInt64 days, const DateLUTImpl & time_zone) { return time_zone.toStartOfDayInterval(time_zone.toDayNum(t), days); } + + static UInt32 execute(Int64 t, UInt64 days, const DateLUTImpl & time_zone) + { + return time_zone.toStartOfDayInterval(time_zone.toDayNum(t), days); + } }; template <> @@ -114,8 +140,8 @@ namespace static constexpr auto name = function_name; static UInt32 execute(UInt16, UInt64, const DateLUTImpl &) { return dateIsNotSupported(function_name); } - static UInt32 execute(UInt32 t, UInt64 hours, const DateLUTImpl & time_zone) { return time_zone.toStartOfHourInterval(t, hours); } + static UInt32 execute(Int64 t, UInt64 hours, const DateLUTImpl & time_zone) { return time_zone.toStartOfHourInterval(t, hours); } }; template <> @@ -129,6 +155,11 @@ namespace { return time_zone.toStartOfMinuteInterval(t, minutes); } + + static UInt32 execute(Int64 t, UInt64 minutes, const DateLUTImpl & time_zone) + { + return time_zone.toStartOfMinuteInterval(t, minutes); + } }; template <> @@ -142,6 +173,11 @@ namespace { return time_zone.toStartOfSecondInterval(t, seconds); } + + static Int64 execute(Int64 t, UInt64 seconds, const DateLUTImpl & time_zone) + { + return time_zone.toStartOfSecondInterval(t, seconds); + } }; @@ -230,7 +266,7 @@ public: { const auto & time_column = arguments[0]; const auto & interval_column = arguments[1]; - const DateLUTImpl & time_zone = extractTimeZoneFromFunctionArguments(arguments, 2, 0); + const auto & time_zone = extractTimeZoneFromFunctionArguments(arguments, 2, 0); auto result_column = dispatchForColumns(time_column, interval_column, time_zone); return result_column; } diff --git a/src/Functions/today.cpp b/src/Functions/today.cpp index 65373058540..511af881d73 100644 --- a/src/Functions/today.cpp +++ b/src/Functions/today.cpp @@ -77,7 +77,7 @@ public: FunctionBaseImplPtr build(const ColumnsWithTypeAndName &, const DataTypePtr &) const override { - return std::make_unique(DateLUT::instance().toDayNum(time(nullptr))); + return std::make_unique(DayNum(DateLUT::instance().toDayNum(time(nullptr)).toUnderType())); } }; diff --git a/src/IO/ReadHelpers.h b/src/IO/ReadHelpers.h index e33de04f322..369237f329d 100644 --- a/src/IO/ReadHelpers.h +++ b/src/IO/ReadHelpers.h @@ -747,7 +747,7 @@ inline ReturnType readDateTimeTextImpl(DateTime64 & datetime64, UInt32 scale, Re return ReturnType(false); } - DB::DecimalUtils::DecimalComponents components{static_cast(whole), 0}; + DB::DecimalUtils::DecimalComponents components{static_cast(whole), 0}; if (!buf.eof() && *buf.position() == '.') { @@ -791,9 +791,9 @@ inline ReturnType readDateTimeTextImpl(DateTime64 & datetime64, UInt32 scale, Re return ReturnType(true); } -inline void readDateTimeText(time_t & datetime, ReadBuffer & buf, const DateLUTImpl & date_lut = DateLUT::instance()) +inline void readDateTimeText(time_t & datetime, ReadBuffer & buf, const DateLUTImpl & time_zone = DateLUT::instance()) { - readDateTimeTextImpl(datetime, buf, date_lut); + readDateTimeTextImpl(datetime, buf, time_zone); } inline void readDateTime64Text(DateTime64 & datetime64, UInt32 scale, ReadBuffer & buf, const DateLUTImpl & date_lut = DateLUT::instance()) @@ -801,9 +801,9 @@ inline void readDateTime64Text(DateTime64 & datetime64, UInt32 scale, ReadBuffer readDateTimeTextImpl(datetime64, scale, buf, date_lut); } -inline bool tryReadDateTimeText(time_t & datetime, ReadBuffer & buf, const DateLUTImpl & date_lut = DateLUT::instance()) +inline bool tryReadDateTimeText(time_t & datetime, ReadBuffer & buf, const DateLUTImpl & time_zone = DateLUT::instance()) { - return readDateTimeTextImpl(datetime, buf, date_lut); + return readDateTimeTextImpl(datetime, buf, time_zone); } inline bool tryReadDateTime64Text(DateTime64 & datetime64, UInt32 scale, ReadBuffer & buf, const DateLUTImpl & date_lut = DateLUT::instance()) diff --git a/src/IO/WriteHelpers.h b/src/IO/WriteHelpers.h index a382ae13cdd..72c6b69114f 100644 --- a/src/IO/WriteHelpers.h +++ b/src/IO/WriteHelpers.h @@ -819,12 +819,12 @@ inline void writeDateTimeText(const LocalDateTime & datetime, WriteBuffer & buf) /// In the format YYYY-MM-DD HH:MM:SS, according to the specified time zone. template -inline void writeDateTimeText(time_t datetime, WriteBuffer & buf, const DateLUTImpl & date_lut = DateLUT::instance()) +inline void writeDateTimeText(time_t datetime, WriteBuffer & buf, const DateLUTImpl & time_zone = DateLUT::instance()) { - const auto & values = date_lut.getValues(datetime); + const auto & values = time_zone.getValues(datetime); writeDateTimeText( LocalDateTime(values.year, values.month, values.day_of_month, - date_lut.toHour(datetime), date_lut.toMinute(datetime), date_lut.toSecond(datetime)), buf); + time_zone.toHour(datetime), time_zone.toMinute(datetime), time_zone.toSecond(datetime)), buf); } /// In the format YYYY-MM-DD HH:MM:SS.NNNNNNNNN, according to the specified time zone. @@ -849,9 +849,9 @@ inline void writeDateTimeText(DateTime64 datetime64, UInt32 scale, WriteBuffer & /// In the RFC 1123 format: "Tue, 03 Dec 2019 00:11:50 GMT". You must provide GMT DateLUT. /// This is needed for HTTP requests. -inline void writeDateTimeTextRFC1123(time_t datetime, WriteBuffer & buf, const DateLUTImpl & date_lut) +inline void writeDateTimeTextRFC1123(time_t datetime, WriteBuffer & buf, const DateLUTImpl & time_zone = DateLUT::instance()) { - const auto & values = date_lut.getValues(datetime); + const auto & values = time_zone.getValues(datetime); static const char week_days[3 * 8 + 1] = "XXX" "Mon" "Tue" "Wed" "Thu" "Fri" "Sat" "Sun"; static const char months[3 * 13 + 1] = "XXX" "Jan" "Feb" "Mar" "Apr" "May" "Jun" "Jul" "Aug" "Sep" "Oct" "Nov" "Dec"; @@ -865,11 +865,11 @@ inline void writeDateTimeTextRFC1123(time_t datetime, WriteBuffer & buf, const D buf.write(&digits100[values.year / 100 * 2], 2); buf.write(&digits100[values.year % 100 * 2], 2); buf.write(' '); - buf.write(&digits100[date_lut.toHour(datetime) * 2], 2); + buf.write(&digits100[time_zone.toHour(datetime) * 2], 2); buf.write(':'); - buf.write(&digits100[date_lut.toMinute(datetime) * 2], 2); + buf.write(&digits100[time_zone.toMinute(datetime) * 2], 2); buf.write(':'); - buf.write(&digits100[date_lut.toSecond(datetime) * 2], 2); + buf.write(&digits100[time_zone.toSecond(datetime) * 2], 2); buf.write(" GMT", 4); } diff --git a/src/IO/parseDateTimeBestEffort.cpp b/src/IO/parseDateTimeBestEffort.cpp index 47a298ede29..26745a8f138 100644 --- a/src/IO/parseDateTimeBestEffort.cpp +++ b/src/IO/parseDateTimeBestEffort.cpp @@ -600,7 +600,7 @@ ReturnType parseDateTimeBestEffortImpl( return ReturnType(true); } -template +template ReturnType parseDateTime64BestEffortImpl(DateTime64 & res, UInt32 scale, ReadBuffer & in, const DateLUTImpl & local_time_zone, const DateLUTImpl & utc_time_zone) { time_t whole; @@ -608,12 +608,12 @@ ReturnType parseDateTime64BestEffortImpl(DateTime64 & res, UInt32 scale, ReadBuf if constexpr (std::is_same_v) { - if (!parseDateTimeBestEffortImpl(whole, in, local_time_zone, utc_time_zone, &subsecond)) + if (!parseDateTimeBestEffortImpl(whole, in, local_time_zone, utc_time_zone, &subsecond)) return false; } else { - parseDateTimeBestEffortImpl(whole, in, local_time_zone, utc_time_zone, &subsecond); + parseDateTimeBestEffortImpl(whole, in, local_time_zone, utc_time_zone, &subsecond); } @@ -661,12 +661,17 @@ bool tryParseDateTimeBestEffortUS(time_t & res, ReadBuffer & in, const DateLUTIm void parseDateTime64BestEffort(DateTime64 & res, UInt32 scale, ReadBuffer & in, const DateLUTImpl & local_time_zone, const DateLUTImpl & utc_time_zone) { - return parseDateTime64BestEffortImpl(res, scale, in, local_time_zone, utc_time_zone); + return parseDateTime64BestEffortImpl(res, scale, in, local_time_zone, utc_time_zone); +} + +void parseDateTime64BestEffortUS(DateTime64 & res, UInt32 scale, ReadBuffer & in, const DateLUTImpl & local_time_zone, const DateLUTImpl & utc_time_zone) +{ + return parseDateTime64BestEffortImpl(res, scale, in, local_time_zone, utc_time_zone); } bool tryParseDateTime64BestEffort(DateTime64 & res, UInt32 scale, ReadBuffer & in, const DateLUTImpl & local_time_zone, const DateLUTImpl & utc_time_zone) { - return parseDateTime64BestEffortImpl(res, scale, in, local_time_zone, utc_time_zone); + return parseDateTime64BestEffortImpl(res, scale, in, local_time_zone, utc_time_zone); } } diff --git a/src/IO/parseDateTimeBestEffort.h b/src/IO/parseDateTimeBestEffort.h index 65e92cbee42..fe3da24a797 100644 --- a/src/IO/parseDateTimeBestEffort.h +++ b/src/IO/parseDateTimeBestEffort.h @@ -61,6 +61,7 @@ bool tryParseDateTimeBestEffort(time_t & res, ReadBuffer & in, const DateLUTImpl void parseDateTimeBestEffortUS(time_t & res, ReadBuffer & in, const DateLUTImpl & local_time_zone, const DateLUTImpl & utc_time_zone); bool tryParseDateTimeBestEffortUS(time_t & res, ReadBuffer & in, const DateLUTImpl & local_time_zone, const DateLUTImpl & utc_time_zone); void parseDateTime64BestEffort(DateTime64 & res, UInt32 scale, ReadBuffer & in, const DateLUTImpl & local_time_zone, const DateLUTImpl & utc_time_zone); +void parseDateTime64BestEffortUS(DateTime64 & res, UInt32 scale, ReadBuffer & in, const DateLUTImpl & local_time_zone, const DateLUTImpl & utc_time_zone); bool tryParseDateTime64BestEffort(DateTime64 & res, UInt32 scale, ReadBuffer & in, const DateLUTImpl & local_time_zone, const DateLUTImpl & utc_time_zone); } diff --git a/src/Interpreters/CrashLog.cpp b/src/Interpreters/CrashLog.cpp index bf81a2e8aba..5067acd4a5c 100644 --- a/src/Interpreters/CrashLog.cpp +++ b/src/Interpreters/CrashLog.cpp @@ -40,7 +40,7 @@ void CrashLogElement::appendToBlock(MutableColumns & columns) const { size_t i = 0; - columns[i++]->insert(DateLUT::instance().toDayNum(event_time)); + columns[i++]->insert(DateLUT::instance().toDayNum(event_time).toUnderType()); columns[i++]->insert(event_time); columns[i++]->insert(timestamp_ns); columns[i++]->insert(signal); diff --git a/src/Interpreters/MetricLog.cpp b/src/Interpreters/MetricLog.cpp index ce5d5793b87..fd1c120f18c 100644 --- a/src/Interpreters/MetricLog.cpp +++ b/src/Interpreters/MetricLog.cpp @@ -41,7 +41,7 @@ void MetricLogElement::appendToBlock(MutableColumns & columns) const { size_t column_idx = 0; - columns[column_idx++]->insert(DateLUT::instance().toDayNum(event_time)); + columns[column_idx++]->insert(DateLUT::instance().toDayNum(event_time).toUnderType()); columns[column_idx++]->insert(event_time); columns[column_idx++]->insert(event_time_microseconds); columns[column_idx++]->insert(milliseconds); diff --git a/src/Interpreters/OpenTelemetrySpanLog.cpp b/src/Interpreters/OpenTelemetrySpanLog.cpp index e1df145cf51..f9ae6518af0 100644 --- a/src/Interpreters/OpenTelemetrySpanLog.cpp +++ b/src/Interpreters/OpenTelemetrySpanLog.cpp @@ -49,7 +49,7 @@ void OpenTelemetrySpanLogElement::appendToBlock(MutableColumns & columns) const columns[i++]->insert(operation_name); columns[i++]->insert(start_time_us); columns[i++]->insert(finish_time_us); - columns[i++]->insert(DateLUT::instance().toDayNum(finish_time_us / 1000000)); + columns[i++]->insert(DateLUT::instance().toDayNum(finish_time_us / 1000000).toUnderType()); columns[i++]->insert(attribute_names); // The user might add some ints values, and we will have Int Field, and the // insert will fail because the column requires Strings. Convert the fields diff --git a/src/Interpreters/PartLog.cpp b/src/Interpreters/PartLog.cpp index 860666a0035..c180a4dd254 100644 --- a/src/Interpreters/PartLog.cpp +++ b/src/Interpreters/PartLog.cpp @@ -71,7 +71,7 @@ void PartLogElement::appendToBlock(MutableColumns & columns) const columns[i++]->insert(query_id); columns[i++]->insert(event_type); - columns[i++]->insert(DateLUT::instance().toDayNum(event_time)); + columns[i++]->insert(DateLUT::instance().toDayNum(event_time).toUnderType()); columns[i++]->insert(event_time); columns[i++]->insert(event_time_microseconds); columns[i++]->insert(duration_ms); diff --git a/src/Interpreters/QueryLog.cpp b/src/Interpreters/QueryLog.cpp index 82b957f895b..b6902468242 100644 --- a/src/Interpreters/QueryLog.cpp +++ b/src/Interpreters/QueryLog.cpp @@ -119,7 +119,7 @@ void QueryLogElement::appendToBlock(MutableColumns & columns) const size_t i = 0; columns[i++]->insert(type); - columns[i++]->insert(DateLUT::instance().toDayNum(event_time)); + columns[i++]->insert(DateLUT::instance().toDayNum(event_time).toUnderType()); columns[i++]->insert(event_time); columns[i++]->insert(event_time_microseconds); columns[i++]->insert(query_start_time); diff --git a/src/Interpreters/QueryThreadLog.cpp b/src/Interpreters/QueryThreadLog.cpp index f1cce1a3da9..31f1fddc87f 100644 --- a/src/Interpreters/QueryThreadLog.cpp +++ b/src/Interpreters/QueryThreadLog.cpp @@ -76,7 +76,7 @@ void QueryThreadLogElement::appendToBlock(MutableColumns & columns) const { size_t i = 0; - columns[i++]->insert(DateLUT::instance().toDayNum(event_time)); + columns[i++]->insert(DateLUT::instance().toDayNum(event_time).toUnderType()); columns[i++]->insert(event_time); columns[i++]->insert(event_time_microseconds); columns[i++]->insert(query_start_time); diff --git a/src/Interpreters/TextLog.cpp b/src/Interpreters/TextLog.cpp index f60b6acae6f..489bb302ad0 100644 --- a/src/Interpreters/TextLog.cpp +++ b/src/Interpreters/TextLog.cpp @@ -55,7 +55,7 @@ void TextLogElement::appendToBlock(MutableColumns & columns) const { size_t i = 0; - columns[i++]->insert(DateLUT::instance().toDayNum(event_time)); + columns[i++]->insert(DateLUT::instance().toDayNum(event_time).toUnderType()); columns[i++]->insert(event_time); columns[i++]->insert(event_time_microseconds); columns[i++]->insert(microseconds); diff --git a/src/Interpreters/TraceLog.cpp b/src/Interpreters/TraceLog.cpp index 40bcc0db445..fe7512f2f00 100644 --- a/src/Interpreters/TraceLog.cpp +++ b/src/Interpreters/TraceLog.cpp @@ -42,7 +42,7 @@ void TraceLogElement::appendToBlock(MutableColumns & columns) const { size_t i = 0; - columns[i++]->insert(DateLUT::instance().toDayNum(event_time)); + columns[i++]->insert(DateLUT::instance().toDayNum(event_time).toUnderType()); columns[i++]->insert(event_time); columns[i++]->insert(event_time_microseconds); columns[i++]->insert(timestamp_ns); diff --git a/src/Interpreters/convertFieldToType.cpp b/src/Interpreters/convertFieldToType.cpp index 1d93ef56dea..d47f64cb1dc 100644 --- a/src/Interpreters/convertFieldToType.cpp +++ b/src/Interpreters/convertFieldToType.cpp @@ -141,7 +141,7 @@ Field convertFieldToTypeImpl(const Field & src, const IDataType & type, const ID /// Conversion between Date and DateTime and vice versa. if (which_type.isDate() && which_from_type.isDateTime()) { - return static_cast(*from_type_hint).getTimeZone().toDayNum(src.get()); + return static_cast(static_cast(*from_type_hint).getTimeZone().toDayNum(src.get()).toUnderType()); } else if (which_type.isDateTime() && which_from_type.isDate()) { diff --git a/src/Storages/StorageReplicatedMergeTree.cpp b/src/Storages/StorageReplicatedMergeTree.cpp index f2c88cdedd9..8bf785afa0f 100644 --- a/src/Storages/StorageReplicatedMergeTree.cpp +++ b/src/Storages/StorageReplicatedMergeTree.cpp @@ -4369,7 +4369,7 @@ static String getPartNamePossiblyFake(MergeTreeDataFormatVersion format_version, /// The date range is all month long. const auto & lut = DateLUT::instance(); time_t start_time = lut.YYYYMMDDToDate(parse(part_info.partition_id + "01")); - DayNum left_date = lut.toDayNum(start_time); + DayNum left_date = DayNum{lut.toDayNum(start_time).toUnderType()}; DayNum right_date = DayNum(static_cast(left_date) + lut.daysInMonth(start_time) - 1); return part_info.getPartNameV0(left_date, right_date); } diff --git a/src/Storages/tests/part_name.cpp b/src/Storages/tests/part_name.cpp index 79c5578a8ca..aeadfd208cc 100644 --- a/src/Storages/tests/part_name.cpp +++ b/src/Storages/tests/part_name.cpp @@ -5,7 +5,7 @@ int main(int, char **) { - DayNum today = DateLUT::instance().toDayNum(time(nullptr)); + const DayNum today{DateLUT::instance().toDayNum(time(nullptr)).toUnderType()}; for (DayNum date = today; DayNum(date + 10) > today; --date) { diff --git a/tests/queries/0_stateless/00921_datetime64_compatibility.python b/tests/queries/0_stateless/00921_datetime64_compatibility.python index bf0ae8a72ac..c8b9620629d 100644 --- a/tests/queries/0_stateless/00921_datetime64_compatibility.python +++ b/tests/queries/0_stateless/00921_datetime64_compatibility.python @@ -86,8 +86,7 @@ CAST(N as DateTime64(9, 'Europe/Minsk')) formatDateTime(N, '%C %d %D %e %F %H %I %j %m %M %p %R %S %T %u %V %w %y %Y %%') """.splitlines() -# Expanded later to cartesian product of all arguments. -# NOTE: {N} to be turned into N after str.format() for keys (format string), but not for list of values! +# Expanded later to cartesian product of all arguments, using format string. extra_ops = [ # With same type: ( @@ -179,7 +178,7 @@ def escape_string(s): def execute_functions_for_types(functions, types): - # TODO: use string.Template here to allow lines that do not contain type, like: SELECT CAST(toDateTime64(1234567890), 'DateTime64') + # NOTE: use string.Template here to allow lines with missing keys, like type, e.g. SELECT CAST(toDateTime64(1234567890), 'DateTime64') for func in functions: print(("""SELECT 'SELECT {func}';""".format(func=escape_string(func)))) for dt in types: diff --git a/tests/queries/0_stateless/00921_datetime64_compatibility.reference b/tests/queries/0_stateless/00921_datetime64_compatibility.reference index 004f4f5e824..67413512e06 100644 --- a/tests/queries/0_stateless/00921_datetime64_compatibility.reference +++ b/tests/queries/0_stateless/00921_datetime64_compatibility.reference @@ -1,5 +1,4 @@ SELECT toTimeZone(N, \'UTC\') - Code: 43 "DateTime('UTC')","2019-09-16 16:20:11" "DateTime64(3, 'UTC')","2019-09-16 16:20:11.234" @@ -35,25 +34,21 @@ SELECT toDayOfWeek(N) "UInt8",1 ------------------------------------------ SELECT toHour(N) - Code: 43 "UInt8",19 "UInt8",19 ------------------------------------------ SELECT toMinute(N) - Code: 43 "UInt8",20 "UInt8",20 ------------------------------------------ SELECT toSecond(N) - Code: 43 "UInt8",11 "UInt8",11 ------------------------------------------ SELECT toUnixTimestamp(N) - Code: 44 "UInt32",1568650811 "UInt32",1568650811 @@ -94,31 +89,26 @@ SELECT toStartOfDay(N) "DateTime('Europe/Minsk')","2019-09-16 00:00:00" ------------------------------------------ SELECT toStartOfHour(N) - Code: 43 "DateTime('Europe/Minsk')","2019-09-16 19:00:00" "DateTime('Europe/Minsk')","2019-09-16 19:00:00" ------------------------------------------ SELECT toStartOfMinute(N) - Code: 43 "DateTime('Europe/Minsk')","2019-09-16 19:20:00" "DateTime('Europe/Minsk')","2019-09-16 19:20:00" ------------------------------------------ SELECT toStartOfFiveMinute(N) - Code: 43 "DateTime('Europe/Minsk')","2019-09-16 19:20:00" "DateTime('Europe/Minsk')","2019-09-16 19:20:00" ------------------------------------------ SELECT toStartOfTenMinutes(N) - Code: 43 "DateTime('Europe/Minsk')","2019-09-16 19:20:00" "DateTime('Europe/Minsk')","2019-09-16 19:20:00" ------------------------------------------ SELECT toStartOfFifteenMinutes(N) - Code: 43 "DateTime('Europe/Minsk')","2019-09-16 19:15:00" "DateTime('Europe/Minsk')","2019-09-16 19:15:00" @@ -139,7 +129,6 @@ SELECT toStartOfInterval(N, INTERVAL 1 day) "DateTime('Europe/Minsk')","2019-09-16 00:00:00" ------------------------------------------ SELECT toStartOfInterval(N, INTERVAL 15 minute) - Code: 43 "DateTime('Europe/Minsk')","2019-09-16 19:15:00" "DateTime('Europe/Minsk')","2019-09-16 19:15:00" @@ -160,13 +149,11 @@ SELECT date_trunc(\'day\', N) "DateTime('Europe/Minsk')","2019-09-16 00:00:00" ------------------------------------------ SELECT date_trunc(\'minute\', N) - Code: 43 "DateTime('Europe/Minsk')","2019-09-16 19:20:00" "DateTime('Europe/Minsk')","2019-09-16 19:20:00" ------------------------------------------ SELECT toTime(N) - Code: 43 "DateTime('Europe/Minsk')","1970-01-02 19:20:11" "DateTime('Europe/Minsk')","1970-01-02 19:20:11" @@ -232,7 +219,6 @@ SELECT toYearWeek(N) "UInt32",201937 ------------------------------------------ SELECT timeSlot(N) - Code: 43 "DateTime('Europe/Minsk')","2019-09-16 19:00:00" "DateTime('Europe/Minsk')","2019-09-16 19:00:00" @@ -375,15 +361,11 @@ SELECT formatDateTime(N, \'%C %d %D %e %F %H %I %j %m %M %p %R %S %T %u %V %w %y SELECT N - N "Int32",0 "Int32",0 - Code: 43 ------------------------------------------ SELECT N + N - Code: 43 - Code: 43 - Code: 43 ------------------------------------------ SELECT N != N @@ -417,47 +399,33 @@ SELECT N >= N "UInt8",1 ------------------------------------------ SELECT N - DT - Code: 43 "Int32",0 - Code: 43 ------------------------------------------ SELECT DT - N - Code: 43 "Int32",0 - Code: 43 ------------------------------------------ SELECT N - D "Int32",0 - Code: 43 - Code: 43 ------------------------------------------ SELECT D - N "Int32",0 - Code: 43 - Code: 43 ------------------------------------------ SELECT N - DT64 - Code: 43 - Code: 43 - Code: 43 ------------------------------------------ SELECT DT64 - N - Code: 43 - Code: 43 - Code: 43 ------------------------------------------ SELECT N != DT @@ -726,11 +694,8 @@ SELECT N - toUInt8(1) "DateTime64(3, 'Europe/Minsk')","2019-09-16 19:20:10.234" ------------------------------------------ SELECT toUInt8(1) - N - Code: 43 - Code: 43 - Code: 43 ------------------------------------------ SELECT N - toInt8(-1) @@ -739,11 +704,8 @@ SELECT N - toInt8(-1) "DateTime64(3, 'Europe/Minsk')","2019-09-16 19:20:12.234" ------------------------------------------ SELECT toInt8(-1) - N - Code: 43 - Code: 43 - Code: 43 ------------------------------------------ SELECT N - toUInt16(1) @@ -752,11 +714,8 @@ SELECT N - toUInt16(1) "DateTime64(3, 'Europe/Minsk')","2019-09-16 19:20:10.234" ------------------------------------------ SELECT toUInt16(1) - N - Code: 43 - Code: 43 - Code: 43 ------------------------------------------ SELECT N - toInt16(-1) @@ -765,11 +724,8 @@ SELECT N - toInt16(-1) "DateTime64(3, 'Europe/Minsk')","2019-09-16 19:20:12.234" ------------------------------------------ SELECT toInt16(-1) - N - Code: 43 - Code: 43 - Code: 43 ------------------------------------------ SELECT N - toUInt32(1) @@ -778,11 +734,8 @@ SELECT N - toUInt32(1) "DateTime64(3, 'Europe/Minsk')","2019-09-16 19:20:10.234" ------------------------------------------ SELECT toUInt32(1) - N - Code: 43 - Code: 43 - Code: 43 ------------------------------------------ SELECT N - toInt32(-1) @@ -791,11 +744,8 @@ SELECT N - toInt32(-1) "DateTime64(3, 'Europe/Minsk')","2019-09-16 19:20:12.234" ------------------------------------------ SELECT toInt32(-1) - N - Code: 43 - Code: 43 - Code: 43 ------------------------------------------ SELECT N - toUInt64(1) @@ -804,11 +754,8 @@ SELECT N - toUInt64(1) "DateTime64(3, 'Europe/Minsk')","2019-09-16 19:20:10.234" ------------------------------------------ SELECT toUInt64(1) - N - Code: 43 - Code: 43 - Code: 43 ------------------------------------------ SELECT N - toInt64(-1) @@ -817,585 +764,486 @@ SELECT N - toInt64(-1) "DateTime64(3, 'Europe/Minsk')","2019-09-16 19:20:12.234" ------------------------------------------ SELECT toInt64(-1) - N - Code: 43 - Code: 43 - Code: 43 ------------------------------------------ SELECT N == toUInt8(1) - Code: 43 "UInt8",0 "UInt8",0 ------------------------------------------ SELECT toUInt8(1) == N - Code: 43 "UInt8",0 "UInt8",0 ------------------------------------------ SELECT N == toInt8(-1) - Code: 43 "UInt8",0 "UInt8",0 ------------------------------------------ SELECT toInt8(-1) == N - Code: 43 "UInt8",0 "UInt8",0 ------------------------------------------ SELECT N == toUInt16(1) - Code: 43 "UInt8",0 "UInt8",0 ------------------------------------------ SELECT toUInt16(1) == N - Code: 43 "UInt8",0 "UInt8",0 ------------------------------------------ SELECT N == toInt16(-1) - Code: 43 "UInt8",0 "UInt8",0 ------------------------------------------ SELECT toInt16(-1) == N - Code: 43 "UInt8",0 "UInt8",0 ------------------------------------------ SELECT N == toUInt32(1) - Code: 43 "UInt8",0 "UInt8",0 ------------------------------------------ SELECT toUInt32(1) == N - Code: 43 "UInt8",0 "UInt8",0 ------------------------------------------ SELECT N == toInt32(-1) - Code: 43 "UInt8",0 "UInt8",0 ------------------------------------------ SELECT toInt32(-1) == N - Code: 43 "UInt8",0 "UInt8",0 ------------------------------------------ SELECT N == toUInt64(1) - Code: 43 "UInt8",0 "UInt8",0 ------------------------------------------ SELECT toUInt64(1) == N - Code: 43 "UInt8",0 "UInt8",0 ------------------------------------------ SELECT N == toInt64(-1) - Code: 43 "UInt8",0 "UInt8",0 ------------------------------------------ SELECT toInt64(-1) == N - Code: 43 "UInt8",0 "UInt8",0 ------------------------------------------ SELECT N != toUInt8(1) - Code: 43 "UInt8",1 "UInt8",1 ------------------------------------------ SELECT toUInt8(1) != N - Code: 43 "UInt8",1 "UInt8",1 ------------------------------------------ SELECT N != toInt8(-1) - Code: 43 "UInt8",1 "UInt8",1 ------------------------------------------ SELECT toInt8(-1) != N - Code: 43 "UInt8",1 "UInt8",1 ------------------------------------------ SELECT N != toUInt16(1) - Code: 43 "UInt8",1 "UInt8",1 ------------------------------------------ SELECT toUInt16(1) != N - Code: 43 "UInt8",1 "UInt8",1 ------------------------------------------ SELECT N != toInt16(-1) - Code: 43 "UInt8",1 "UInt8",1 ------------------------------------------ SELECT toInt16(-1) != N - Code: 43 "UInt8",1 "UInt8",1 ------------------------------------------ SELECT N != toUInt32(1) - Code: 43 "UInt8",1 "UInt8",1 ------------------------------------------ SELECT toUInt32(1) != N - Code: 43 "UInt8",1 "UInt8",1 ------------------------------------------ SELECT N != toInt32(-1) - Code: 43 "UInt8",1 "UInt8",1 ------------------------------------------ SELECT toInt32(-1) != N - Code: 43 "UInt8",1 "UInt8",1 ------------------------------------------ SELECT N != toUInt64(1) - Code: 43 "UInt8",1 "UInt8",1 ------------------------------------------ SELECT toUInt64(1) != N - Code: 43 "UInt8",1 "UInt8",1 ------------------------------------------ SELECT N != toInt64(-1) - Code: 43 "UInt8",1 "UInt8",1 ------------------------------------------ SELECT toInt64(-1) != N - Code: 43 "UInt8",1 "UInt8",1 ------------------------------------------ SELECT N < toUInt8(1) - Code: 43 "UInt8",0 "UInt8",0 ------------------------------------------ SELECT toUInt8(1) < N - Code: 43 "UInt8",1 "UInt8",1 ------------------------------------------ SELECT N < toInt8(-1) - Code: 43 "UInt8",0 "UInt8",0 ------------------------------------------ SELECT toInt8(-1) < N - Code: 43 "UInt8",1 "UInt8",1 ------------------------------------------ SELECT N < toUInt16(1) - Code: 43 "UInt8",0 "UInt8",0 ------------------------------------------ SELECT toUInt16(1) < N - Code: 43 "UInt8",1 "UInt8",1 ------------------------------------------ SELECT N < toInt16(-1) - Code: 43 "UInt8",0 "UInt8",0 ------------------------------------------ SELECT toInt16(-1) < N - Code: 43 "UInt8",1 "UInt8",1 ------------------------------------------ SELECT N < toUInt32(1) - Code: 43 "UInt8",0 "UInt8",0 ------------------------------------------ SELECT toUInt32(1) < N - Code: 43 "UInt8",1 "UInt8",1 ------------------------------------------ SELECT N < toInt32(-1) - Code: 43 "UInt8",0 "UInt8",0 ------------------------------------------ SELECT toInt32(-1) < N - Code: 43 "UInt8",1 "UInt8",1 ------------------------------------------ SELECT N < toUInt64(1) - Code: 43 "UInt8",0 "UInt8",0 ------------------------------------------ SELECT toUInt64(1) < N - Code: 43 "UInt8",1 "UInt8",1 ------------------------------------------ SELECT N < toInt64(-1) - Code: 43 "UInt8",0 "UInt8",0 ------------------------------------------ SELECT toInt64(-1) < N - Code: 43 "UInt8",1 "UInt8",1 ------------------------------------------ SELECT N <= toUInt8(1) - Code: 43 "UInt8",0 "UInt8",0 ------------------------------------------ SELECT toUInt8(1) <= N - Code: 43 "UInt8",1 "UInt8",1 ------------------------------------------ SELECT N <= toInt8(-1) - Code: 43 "UInt8",0 "UInt8",0 ------------------------------------------ SELECT toInt8(-1) <= N - Code: 43 "UInt8",1 "UInt8",1 ------------------------------------------ SELECT N <= toUInt16(1) - Code: 43 "UInt8",0 "UInt8",0 ------------------------------------------ SELECT toUInt16(1) <= N - Code: 43 "UInt8",1 "UInt8",1 ------------------------------------------ SELECT N <= toInt16(-1) - Code: 43 "UInt8",0 "UInt8",0 ------------------------------------------ SELECT toInt16(-1) <= N - Code: 43 "UInt8",1 "UInt8",1 ------------------------------------------ SELECT N <= toUInt32(1) - Code: 43 "UInt8",0 "UInt8",0 ------------------------------------------ SELECT toUInt32(1) <= N - Code: 43 "UInt8",1 "UInt8",1 ------------------------------------------ SELECT N <= toInt32(-1) - Code: 43 "UInt8",0 "UInt8",0 ------------------------------------------ SELECT toInt32(-1) <= N - Code: 43 "UInt8",1 "UInt8",1 ------------------------------------------ SELECT N <= toUInt64(1) - Code: 43 "UInt8",0 "UInt8",0 ------------------------------------------ SELECT toUInt64(1) <= N - Code: 43 "UInt8",1 "UInt8",1 ------------------------------------------ SELECT N <= toInt64(-1) - Code: 43 "UInt8",0 "UInt8",0 ------------------------------------------ SELECT toInt64(-1) <= N - Code: 43 "UInt8",1 "UInt8",1 ------------------------------------------ SELECT N > toUInt8(1) - Code: 43 "UInt8",1 "UInt8",1 ------------------------------------------ SELECT toUInt8(1) > N - Code: 43 "UInt8",0 "UInt8",0 ------------------------------------------ SELECT N > toInt8(-1) - Code: 43 "UInt8",1 "UInt8",1 ------------------------------------------ SELECT toInt8(-1) > N - Code: 43 "UInt8",0 "UInt8",0 ------------------------------------------ SELECT N > toUInt16(1) - Code: 43 "UInt8",1 "UInt8",1 ------------------------------------------ SELECT toUInt16(1) > N - Code: 43 "UInt8",0 "UInt8",0 ------------------------------------------ SELECT N > toInt16(-1) - Code: 43 "UInt8",1 "UInt8",1 ------------------------------------------ SELECT toInt16(-1) > N - Code: 43 "UInt8",0 "UInt8",0 ------------------------------------------ SELECT N > toUInt32(1) - Code: 43 "UInt8",1 "UInt8",1 ------------------------------------------ SELECT toUInt32(1) > N - Code: 43 "UInt8",0 "UInt8",0 ------------------------------------------ SELECT N > toInt32(-1) - Code: 43 "UInt8",1 "UInt8",1 ------------------------------------------ SELECT toInt32(-1) > N - Code: 43 "UInt8",0 "UInt8",0 ------------------------------------------ SELECT N > toUInt64(1) - Code: 43 "UInt8",1 "UInt8",1 ------------------------------------------ SELECT toUInt64(1) > N - Code: 43 "UInt8",0 "UInt8",0 ------------------------------------------ SELECT N > toInt64(-1) - Code: 43 "UInt8",1 "UInt8",1 ------------------------------------------ SELECT toInt64(-1) > N - Code: 43 "UInt8",0 "UInt8",0 ------------------------------------------ SELECT N >= toUInt8(1) - Code: 43 "UInt8",1 "UInt8",1 ------------------------------------------ SELECT toUInt8(1) >= N - Code: 43 "UInt8",0 "UInt8",0 ------------------------------------------ SELECT N >= toInt8(-1) - Code: 43 "UInt8",1 "UInt8",1 ------------------------------------------ SELECT toInt8(-1) >= N - Code: 43 "UInt8",0 "UInt8",0 ------------------------------------------ SELECT N >= toUInt16(1) - Code: 43 "UInt8",1 "UInt8",1 ------------------------------------------ SELECT toUInt16(1) >= N - Code: 43 "UInt8",0 "UInt8",0 ------------------------------------------ SELECT N >= toInt16(-1) - Code: 43 "UInt8",1 "UInt8",1 ------------------------------------------ SELECT toInt16(-1) >= N - Code: 43 "UInt8",0 "UInt8",0 ------------------------------------------ SELECT N >= toUInt32(1) - Code: 43 "UInt8",1 "UInt8",1 ------------------------------------------ SELECT toUInt32(1) >= N - Code: 43 "UInt8",0 "UInt8",0 ------------------------------------------ SELECT N >= toInt32(-1) - Code: 43 "UInt8",1 "UInt8",1 ------------------------------------------ SELECT toInt32(-1) >= N - Code: 43 "UInt8",0 "UInt8",0 ------------------------------------------ SELECT N >= toUInt64(1) - Code: 43 "UInt8",1 "UInt8",1 ------------------------------------------ SELECT toUInt64(1) >= N - Code: 43 "UInt8",0 "UInt8",0 ------------------------------------------ SELECT N >= toInt64(-1) - Code: 43 "UInt8",1 "UInt8",1 ------------------------------------------ SELECT toInt64(-1) >= N - Code: 43 "UInt8",0 "UInt8",0 diff --git a/tests/queries/0_stateless/00921_datetime64_compatibility.sh b/tests/queries/0_stateless/00921_datetime64_compatibility.sh index 1617e5b1f77..5f5034819e4 100755 --- a/tests/queries/0_stateless/00921_datetime64_compatibility.sh +++ b/tests/queries/0_stateless/00921_datetime64_compatibility.sh @@ -13,4 +13,4 @@ CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) python3 "${CURDIR}"/00921_datetime64_compatibility.python \ | ${CLICKHOUSE_CLIENT} --ignore-error -T -nm --calculate_text_stack_trace 0 --log-level 'error' 2>&1 \ - | sed 's/Received exception .*//g; s/^\(Code: [0-9]\+\).*$/\1/g' + | grep -v 'Received exception .*$' | sed 's/^\(Code: [0-9]\+\).*$/\1/g' diff --git a/tests/queries/0_stateless/01252_weird_time_zone.reference b/tests/queries/0_stateless/01252_weird_time_zone.reference index f2968d4efa6..90f5bf0e30d 100644 --- a/tests/queries/0_stateless/01252_weird_time_zone.reference +++ b/tests/queries/0_stateless/01252_weird_time_zone.reference @@ -1,7 +1,7 @@ -2020-01-02 03:04:05 2020-01-02 00:00:00 3 -2020-01-02 03:04:05 2020-01-02 00:00:00 3 -2020-01-02 03:04:05 2020-01-02 00:00:00 3 -2020-01-02 03:04:05 2020-01-02 00:00:00 3 -2020-01-02 03:04:05 2020-01-02 00:00:00 3 -2020-01-02 03:04:05 2020-01-02 00:00:00 3 -2020-01-02 03:04:05 2020-01-02 00:00:00 3 +Pacific/Kiritimati 2020-01-02 03:04:05 2020-01-02 00:00:00 3 +Africa/El_Aaiun 2020-01-02 03:04:05 2020-01-02 00:00:00 3 +Asia/Pyongyang 2020-01-02 03:04:05 2020-01-02 00:00:00 3 +Pacific/Kwajalein 2020-01-02 03:04:05 2020-01-02 00:00:00 3 +Pacific/Apia 2020-01-02 03:04:05 2020-01-02 00:00:00 3 +Pacific/Enderbury 2020-01-02 03:04:05 2020-01-02 00:00:00 3 +Pacific/Fakaofo 2020-01-02 03:04:05 2020-01-02 00:00:00 3 diff --git a/tests/queries/0_stateless/01252_weird_time_zone.sql b/tests/queries/0_stateless/01252_weird_time_zone.sql index 68ea903a797..c4919ca4fe0 100644 --- a/tests/queries/0_stateless/01252_weird_time_zone.sql +++ b/tests/queries/0_stateless/01252_weird_time_zone.sql @@ -1,15 +1,15 @@ -SELECT toDateTime('2020-01-02 03:04:05', 'Pacific/Kiritimati') AS x, toStartOfDay(x), toHour(x); -SELECT toDateTime('2020-01-02 03:04:05', 'Africa/El_Aaiun') AS x, toStartOfDay(x), toHour(x); -SELECT toDateTime('2020-01-02 03:04:05', 'Asia/Pyongyang') AS x, toStartOfDay(x), toHour(x); -SELECT toDateTime('2020-01-02 03:04:05', 'Pacific/Kwajalein') AS x, toStartOfDay(x), toHour(x); -SELECT toDateTime('2020-01-02 03:04:05', 'Pacific/Apia') AS x, toStartOfDay(x), toHour(x); -SELECT toDateTime('2020-01-02 03:04:05', 'Pacific/Enderbury') AS x, toStartOfDay(x), toHour(x); -SELECT toDateTime('2020-01-02 03:04:05', 'Pacific/Fakaofo') AS x, toStartOfDay(x), toHour(x); +SELECT 'Pacific/Kiritimati', toDateTime('2020-01-02 03:04:05', 'Pacific/Kiritimati') AS x, toStartOfDay(x), toHour(x); +SELECT 'Africa/El_Aaiun', toDateTime('2020-01-02 03:04:05', 'Africa/El_Aaiun') AS x, toStartOfDay(x), toHour(x); +SELECT 'Asia/Pyongyang', toDateTime('2020-01-02 03:04:05', 'Asia/Pyongyang') AS x, toStartOfDay(x), toHour(x); +SELECT 'Pacific/Kwajalein', toDateTime('2020-01-02 03:04:05', 'Pacific/Kwajalein') AS x, toStartOfDay(x), toHour(x); +SELECT 'Pacific/Apia', toDateTime('2020-01-02 03:04:05', 'Pacific/Apia') AS x, toStartOfDay(x), toHour(x); +SELECT 'Pacific/Enderbury', toDateTime('2020-01-02 03:04:05', 'Pacific/Enderbury') AS x, toStartOfDay(x), toHour(x); +SELECT 'Pacific/Fakaofo', toDateTime('2020-01-02 03:04:05', 'Pacific/Fakaofo') AS x, toStartOfDay(x), toHour(x); -SELECT toHour(toDateTime(rand(), 'Pacific/Kiritimati') AS t) AS h, t FROM numbers(1000000) WHERE h < 0 OR h > 23 ORDER BY h LIMIT 1 BY h; -SELECT toHour(toDateTime(rand(), 'Africa/El_Aaiun') AS t) AS h, t FROM numbers(1000000) WHERE h < 0 OR h > 23 ORDER BY h LIMIT 1 BY h; -SELECT toHour(toDateTime(rand(), 'Asia/Pyongyang') AS t) AS h, t FROM numbers(1000000) WHERE h < 0 OR h > 23 ORDER BY h LIMIT 1 BY h; -SELECT toHour(toDateTime(rand(), 'Pacific/Kwajalein') AS t) AS h, t FROM numbers(1000000) WHERE h < 0 OR h > 23 ORDER BY h LIMIT 1 BY h; -SELECT toHour(toDateTime(rand(), 'Pacific/Apia') AS t) AS h, t FROM numbers(1000000) WHERE h < 0 OR h > 23 ORDER BY h LIMIT 1 BY h; -SELECT toHour(toDateTime(rand(), 'Pacific/Enderbury') AS t) AS h, t FROM numbers(1000000) WHERE h < 0 OR h > 23 ORDER BY h LIMIT 1 BY h; -SELECT toHour(toDateTime(rand(), 'Pacific/Fakaofo') AS t) AS h, t FROM numbers(1000000) WHERE h < 0 OR h > 23 ORDER BY h LIMIT 1 BY h; +SELECT 'Pacific/Kiritimati', rand() as r, toHour(toDateTime(r, 'Pacific/Kiritimati') AS t) AS h, t, toTypeName(t) FROM numbers(1000000) WHERE h < 0 OR h > 23 ORDER BY h LIMIT 1 BY h; +SELECT 'Africa/El_Aaiun', rand() as r, toHour(toDateTime(r, 'Africa/El_Aaiun') AS t) AS h, t, toTypeName(t) FROM numbers(1000000) WHERE h < 0 OR h > 23 ORDER BY h LIMIT 1 BY h; +SELECT 'Asia/Pyongyang', rand() as r, toHour(toDateTime(r, 'Asia/Pyongyang') AS t) AS h, t, toTypeName(t) FROM numbers(1000000) WHERE h < 0 OR h > 23 ORDER BY h LIMIT 1 BY h; +SELECT 'Pacific/Kwajalein', rand() as r, toHour(toDateTime(r, 'Pacific/Kwajalein') AS t) AS h, t, toTypeName(t) FROM numbers(1000000) WHERE h < 0 OR h > 23 ORDER BY h LIMIT 1 BY h; +SELECT 'Pacific/Apia', rand() as r, toHour(toDateTime(r, 'Pacific/Apia') AS t) AS h, t, toTypeName(t) FROM numbers(1000000) WHERE h < 0 OR h > 23 ORDER BY h LIMIT 1 BY h; +SELECT 'Pacific/Enderbury', rand() as r, toHour(toDateTime(r, 'Pacific/Enderbury') AS t) AS h, t, toTypeName(t) FROM numbers(1000000) WHERE h < 0 OR h > 23 ORDER BY h LIMIT 1 BY h; +SELECT 'Pacific/Fakaofo', rand() as r, toHour(toDateTime(r, 'Pacific/Fakaofo') AS t) AS h, t, toTypeName(t) FROM numbers(1000000) WHERE h < 0 OR h > 23 ORDER BY h LIMIT 1 BY h; diff --git a/tests/queries/0_stateless/01440_to_date_monotonicity.reference b/tests/queries/0_stateless/01440_to_date_monotonicity.reference index 96732e5996c..74716fe6223 100644 --- a/tests/queries/0_stateless/01440_to_date_monotonicity.reference +++ b/tests/queries/0_stateless/01440_to_date_monotonicity.reference @@ -1,4 +1,4 @@ 0 -1970-01-01 2106-02-07 1970-04-11 1970-01-01 2106-02-07 +1970-01-01 2106-02-07 1970-04-11 1970-01-01 2149-06-06 1970-01-01 03:00:00 2106-02-07 09:28:15 1970-01-01 03:16:40 2000-01-01 13:12:12 diff --git a/tests/queries/0_stateless/01561_Date_and_DateTime64_comparision.sql b/tests/queries/0_stateless/01561_Date_and_DateTime64_comparision.sql index 7e75d871e07..a61bcff4db7 100644 --- a/tests/queries/0_stateless/01561_Date_and_DateTime64_comparision.sql +++ b/tests/queries/0_stateless/01561_Date_and_DateTime64_comparision.sql @@ -6,7 +6,7 @@ SELECT dt64 < d, toDate(dt64) < d, dt64 < toDateTime64(d, 1, 'UTC'), - + '<=', dt64 <= d, toDate(dt64) <= d, @@ -16,7 +16,7 @@ SELECT dt64 = d, toDate(dt64) = d, dt64 = toDateTime64(d, 1, 'UTC'), - + '>=', dt64 >= d, toDate(dt64) >= d, @@ -31,7 +31,7 @@ SELECT dt64 != d, toDate(dt64) != d, dt64 != toDateTime64(d, 1, 'UTC') -FROM +FROM ( WITH toDateTime('2019-09-16 19:20:11') as val SELECT diff --git a/tests/queries/0_stateless/01631_date_overflow_as_partition_key.reference b/tests/queries/0_stateless/01631_date_overflow_as_partition_key.reference index dbcd92da11c..62f620f3ba9 100644 --- a/tests/queries/0_stateless/01631_date_overflow_as_partition_key.reference +++ b/tests/queries/0_stateless/01631_date_overflow_as_partition_key.reference @@ -1,2 +1,2 @@ -1970-01-01 1 -1970-01-01 1 +2106-11-11 1 +2106-11-12 1 diff --git a/tests/queries/0_stateless/01631_date_overflow_as_partition_key.sql b/tests/queries/0_stateless/01631_date_overflow_as_partition_key.sql index f252e10806a..9a8d37084fb 100644 --- a/tests/queries/0_stateless/01631_date_overflow_as_partition_key.sql +++ b/tests/queries/0_stateless/01631_date_overflow_as_partition_key.sql @@ -6,6 +6,6 @@ insert into dt_overflow values('2106-11-11', 1); insert into dt_overflow values('2106-11-12', 1); -select * from dt_overflow; +select * from dt_overflow ORDER BY d; drop table if exists dt_overflow; diff --git a/tests/queries/0_stateless/01691_DateTime64_clamp.reference b/tests/queries/0_stateless/01691_DateTime64_clamp.reference index 3adc9a17e5c..da80de59e50 100644 --- a/tests/queries/0_stateless/01691_DateTime64_clamp.reference +++ b/tests/queries/0_stateless/01691_DateTime64_clamp.reference @@ -1,4 +1,5 @@ -- { echo } +<<<<<<< HEAD SELECT toTimeZone(toDateTime(-2, 2), 'Europe/Moscow'); 1970-01-01 03:00:00.00 SELECT toDateTime64(-2, 2, 'Europe/Moscow'); @@ -15,3 +16,25 @@ SELECT toDateTime64(-2., 2, 'Europe/Moscow'); SELECT toDateTime64(toFloat32(bitShiftLeft(toUInt64(1),33)), 2, 'Europe/Moscow'); 2106-02-07 09:00:00.00 SELECT toDateTime64(toFloat64(bitShiftLeft(toUInt64(1),33)), 2, 'Europe/Moscow') FORMAT Null; +======= +-- These values are within the extended range of DateTime64 [1925-01-01, 2284-01-01) +SELECT toDateTime(-2, 2); +1970-01-01 02:59:58.00 +SELECT toDateTime64(-2, 2); +1970-01-01 02:59:58.00 +SELECT CAST(-1 AS DateTime64); +1970-01-01 02:59:59.000 +SELECT CAST('2020-01-01 00:00:00.3' AS DateTime64); +2020-01-01 00:00:00.300 +SELECT toDateTime64(bitShiftLeft(toUInt64(1),33), 2); +2242-03-16 15:56:32.00 +-- These are outsize of extended range and hence clamped +SELECT toDateTime64(-1 * bitShiftLeft(toUInt64(1),35), 2); +1925-01-01 02:00:00.00 +SELECT CAST(-1 * bitShiftLeft(toUInt64(1),35) AS DateTime64); +1925-01-01 02:00:00.000 +SELECT CAST(bitShiftLeft(toUInt64(1),35) AS DateTime64); +2282-12-31 03:00:00.000 +SELECT toDateTime64(bitShiftLeft(toUInt64(1),35), 2); +2282-12-31 03:00:00.00 +>>>>>>> af31042451... Extended range of DateTime64 to years 1925 - 2238 diff --git a/tests/queries/0_stateless/01691_DateTime64_clamp.sql b/tests/queries/0_stateless/01691_DateTime64_clamp.sql index 92d5a33328f..958de4edada 100644 --- a/tests/queries/0_stateless/01691_DateTime64_clamp.sql +++ b/tests/queries/0_stateless/01691_DateTime64_clamp.sql @@ -1,4 +1,5 @@ -- { echo } +-- These values are within the extended range of DateTime64 [1925-01-01, 2284-01-01) SELECT toTimeZone(toDateTime(-2, 2), 'Europe/Moscow'); SELECT toDateTime64(-2, 2, 'Europe/Moscow'); SELECT CAST(-1 AS DateTime64(0, 'Europe/Moscow')); @@ -8,3 +9,9 @@ SELECT toTimeZone(toDateTime(-2., 2), 'Europe/Moscow'); SELECT toDateTime64(-2., 2, 'Europe/Moscow'); SELECT toDateTime64(toFloat32(bitShiftLeft(toUInt64(1),33)), 2, 'Europe/Moscow'); SELECT toDateTime64(toFloat64(bitShiftLeft(toUInt64(1),33)), 2, 'Europe/Moscow') FORMAT Null; + +-- These are outsize of extended range and hence clamped +SELECT toDateTime64(-1 * bitShiftLeft(toUInt64(1),35), 2); +SELECT CAST(-1 * bitShiftLeft(toUInt64(1),35) AS DateTime64); +SELECT CAST(bitShiftLeft(toUInt64(1),35) AS DateTime64); +SELECT toDateTime64(bitShiftLeft(toUInt64(1),35), 2); diff --git a/utils/convert-month-partitioned-parts/main.cpp b/utils/convert-month-partitioned-parts/main.cpp index 0a697937eb6..a6829d79726 100644 --- a/utils/convert-month-partitioned-parts/main.cpp +++ b/utils/convert-month-partitioned-parts/main.cpp @@ -47,8 +47,9 @@ void run(String part_path, String date_column, String dest_path) DayNum max_date; MergeTreePartInfo::parseMinMaxDatesFromPartName(old_part_name, min_date, max_date); - UInt32 yyyymm = DateLUT::instance().toNumYYYYMM(min_date); - if (yyyymm != DateLUT::instance().toNumYYYYMM(max_date)) + const auto & time_zone = DateLUT::instance(); + UInt32 yyyymm = time_zone.toNumYYYYMM(min_date); + if (yyyymm != time_zone.toNumYYYYMM(max_date)) throw Exception("Part " + old_part_name + " spans different months", ErrorCodes::BAD_DATA_PART_NAME); From 4fcc23ec9ae35eff445089858804dc92d465b499 Mon Sep 17 00:00:00 2001 From: Vasily Nemkov Date: Sun, 14 Feb 2021 13:00:40 +0200 Subject: [PATCH 036/544] Fixed build for GCC-10 --- base/common/DateLUTImpl.h | 2 +- src/Core/MySQL/MySQLReplication.cpp | 4 ++-- src/Storages/StorageReplicatedMergeTree.cpp | 2 +- src/Storages/tests/part_name.cpp | 2 +- 4 files changed, 5 insertions(+), 5 deletions(-) diff --git a/base/common/DateLUTImpl.h b/base/common/DateLUTImpl.h index adfffb04681..f10c62e9865 100644 --- a/base/common/DateLUTImpl.h +++ b/base/common/DateLUTImpl.h @@ -238,7 +238,7 @@ public: template inline ExtendedDayNum toDayNum(V v) const { - return ExtendedDayNum{static_cast(toLUTIndex(v).toUnderType()) - daynum_offset_epoch}; + return ExtendedDayNum{static_cast(toLUTIndex(v).toUnderType() - daynum_offset_epoch)}; } /// Round down to start of monday. diff --git a/src/Core/MySQL/MySQLReplication.cpp b/src/Core/MySQL/MySQLReplication.cpp index 3e9c5230955..4cb885d4c34 100644 --- a/src/Core/MySQL/MySQLReplication.cpp +++ b/src/Core/MySQL/MySQLReplication.cpp @@ -420,8 +420,8 @@ namespace MySQLReplication UInt32 i24 = 0; payload.readStrict(reinterpret_cast(&i24), 3); - const DayNum date_day_number{DateLUT::instance().makeDayNum( - static_cast((i24 >> 9) & 0x7fff), static_cast((i24 >> 5) & 0xf), static_cast(i24 & 0x1f)).toUnderType()}; + const DayNum date_day_number(DateLUT::instance().makeDayNum( + static_cast((i24 >> 9) & 0x7fff), static_cast((i24 >> 5) & 0xf), static_cast(i24 & 0x1f)).toUnderType()); row.push_back(Field(date_day_number.toUnderType())); break; diff --git a/src/Storages/StorageReplicatedMergeTree.cpp b/src/Storages/StorageReplicatedMergeTree.cpp index 8bf785afa0f..150f2aa577f 100644 --- a/src/Storages/StorageReplicatedMergeTree.cpp +++ b/src/Storages/StorageReplicatedMergeTree.cpp @@ -4369,7 +4369,7 @@ static String getPartNamePossiblyFake(MergeTreeDataFormatVersion format_version, /// The date range is all month long. const auto & lut = DateLUT::instance(); time_t start_time = lut.YYYYMMDDToDate(parse(part_info.partition_id + "01")); - DayNum left_date = DayNum{lut.toDayNum(start_time).toUnderType()}; + DayNum left_date = DayNum(lut.toDayNum(start_time).toUnderType()); DayNum right_date = DayNum(static_cast(left_date) + lut.daysInMonth(start_time) - 1); return part_info.getPartNameV0(left_date, right_date); } diff --git a/src/Storages/tests/part_name.cpp b/src/Storages/tests/part_name.cpp index aeadfd208cc..227e19cf17c 100644 --- a/src/Storages/tests/part_name.cpp +++ b/src/Storages/tests/part_name.cpp @@ -5,7 +5,7 @@ int main(int, char **) { - const DayNum today{DateLUT::instance().toDayNum(time(nullptr)).toUnderType()}; + const DayNum today(DateLUT::instance().toDayNum(time(nullptr)).toUnderType()); for (DayNum date = today; DayNum(date + 10) > today; --date) { From 7a53daaefdbfd760cee1099e8f706e5684aac948 Mon Sep 17 00:00:00 2001 From: Vasily Nemkov Date: Tue, 16 Feb 2021 12:41:08 +0200 Subject: [PATCH 037/544] Fixed issues reported by PVS-Studio and Clang11/GCC10 --- base/common/DateLUTImpl.h | 4 --- base/common/tests/gtest_DateLutImpl.cpp | 48 +++++++------------------ 2 files changed, 12 insertions(+), 40 deletions(-) diff --git a/base/common/DateLUTImpl.h b/base/common/DateLUTImpl.h index f10c62e9865..898fb7bf843 100644 --- a/base/common/DateLUTImpl.h +++ b/base/common/DateLUTImpl.h @@ -850,10 +850,6 @@ public: if (unlikely(year < DATE_LUT_MIN_YEAR || year > DATE_LUT_MAX_YEAR || month < 1 || month > 12 || day_of_month < 1 || day_of_month > 31)) return ExtendedDayNum(0); - // The day after 2283 are not stored fully as struct Values, so just overflow it as 0 - if (unlikely(year > DATE_LUT_MAX_YEAR)) - return ExtendedDayNum(0); - return toDayNum(makeLUTIndex(year, month, day_of_month)); } diff --git a/base/common/tests/gtest_DateLutImpl.cpp b/base/common/tests/gtest_DateLutImpl.cpp index 395e2eddb00..2355c117328 100644 --- a/base/common/tests/gtest_DateLutImpl.cpp +++ b/base/common/tests/gtest_DateLutImpl.cpp @@ -25,18 +25,6 @@ cctz::civil_day YYYYMMDDToDay(unsigned value) value % 100); // day } -cctz::civil_second YYYYMMDDHMMSSToSecond(std::uint64_t value) -{ - return cctz::civil_second( - value / 10000000000, - value / 100000000 % 100, - value / 1000000 % 100, - value / 10000 % 100, - value / 100 % 100, - value % 100); -} - - std::vector allTimezones() { std::vector result; @@ -82,14 +70,17 @@ FailuresCount countFailures(const ::testing::TestResult & test_result) } -TEST(YYYYMMDDToDay, Test) +TEST(DateLUTTest, Test_makeDayNum) { - std::cerr << YYYYMMDDHMMSSToSecond(19700101'00'00'00) << std::endl; + const DateLUTImpl & lut = DateLUT::instance("UTC"); + EXPECT_EQ(0, lut.makeDayNum(2500, 12, 25)); + EXPECT_EQ(0, lut.makeDayNum(1924, 12, 31)); } + TEST(DateLUTTest, TimeValuesInMiddleOfRange) { - const DateLUTImpl lut("Europe/Minsk"); + const DateLUTImpl & lut = DateLUT::instance("Europe/Minsk"); const time_t time = 1568650811; // 2019-09-16 19:20:11 (Monday) EXPECT_EQ(lut.getTimeZone(), "Europe/Minsk"); @@ -151,7 +142,7 @@ TEST(DateLUTTest, TimeValuesInMiddleOfRange) TEST(DateLUTTest, TimeValuesAtLeftBoderOfRange) { - const DateLUTImpl lut("UTC"); + const DateLUTImpl & lut = DateLUT::instance("UTC"); const time_t time = 0; // 1970-01-01 00:00:00 (Thursday) EXPECT_EQ(lut.getTimeZone(), "UTC"); @@ -212,7 +203,7 @@ TEST(DateLUTTest, TimeValuesAtLeftBoderOfRange) TEST(DateLUTTest, TimeValuesAtRightBoderOfRangeOfOLDLut) { // Value is at the right border of the OLD (small) LUT, and provides meaningful values where OLD LUT would provide garbage. - const DateLUTImpl lut("UTC"); + const DateLUTImpl & lut = DateLUT::instance("UTC"); const time_t time = 4294343873; // 2106-01-31T01:17:53 (Sunday) @@ -276,11 +267,11 @@ TEST(DateLUTTest, TimeValuesAtRightBoderOfRangeOfOLDLut) class DateLUT_TimeZone : public ::testing::TestWithParam {}; -TEST_P(DateLUT_TimeZone, DISABLED_LoadAllTimeZones) +TEST_P(DateLUT_TimeZone, DISABLED_LoadLut) { // There are some assumptions and assertions about TZ data made in DateLUTImpl which are verified upon loading, // to make sure that those assertions are true for all timezones we are going to load all of them one by one. - DateLUTImpl{GetParam()}; + DateLUT::instance(GetParam()); } // Another long running test, shouldn't be run to often @@ -292,7 +283,7 @@ TEST_P(DateLUT_TimeZone, VaidateTimeComponentsAroundEpoch) const auto timezone_name = GetParam(); const auto * test_info = ::testing::UnitTest::GetInstance()->current_test_info(); - const auto lut = DateLUTImpl(timezone_name); + const DateLUTImpl & lut = DateLUT::instance(timezone_name); for (time_t i = -856147870; i < 86400 * 10000; i += 11 * 13 * 17 * 19) { @@ -376,22 +367,7 @@ struct TimeRangeParam std::ostream & operator<<(std::ostream & ostr, const TimeRangeParam & param) { - const auto approximate_step = [](const int step) -> std::string - { - // Convert seconds to a string of seconds or fractional count of minutes/hours/days. - static const size_t multipliers[] = {1 /*seconds to seconds*/, 60 /*seconds to minutes*/, 60 /*minutes to hours*/, 24 /*hours to days*/, 0 /*terminator*/}; - static const char* names[] = {"s", "m", "h", "d", nullptr}; - double result = step; - size_t i = 0; - for (; i < sizeof(multipliers)/sizeof(multipliers[0]) && result > multipliers[i]; ++i) - result /= multipliers[i]; - - char buffer[256] = {'\0'}; - std::snprintf(buffer, sizeof(buffer), "%.1f%s", result, names[i - 1]); - return std::string{buffer}; - }; - - return ostr << param.begin << " : " << param.end << " step: " << param.step_in_seconds << "s (" << approximate_step(param.step_in_seconds) << ")"; + return ostr << param.begin << " : " << param.end << " step: " << param.step_in_seconds << "s"; } class DateLUT_Timezone_TimeRange : public ::testing::TestWithParam> From d321c13cbf688f3fd526b1ec36ae296e8939ab2e Mon Sep 17 00:00:00 2001 From: Vasily Nemkov Date: Thu, 18 Feb 2021 00:33:34 +0200 Subject: [PATCH 038/544] Fixed special build --- base/common/DateLUTImpl.cpp | 2 +- base/common/tests/gtest_DateLutImpl.cpp | 36 ++++++++++++------------- 2 files changed, 19 insertions(+), 19 deletions(-) diff --git a/base/common/DateLUTImpl.cpp b/base/common/DateLUTImpl.cpp index 906f88fa90f..6f4fb3dd5fc 100644 --- a/base/common/DateLUTImpl.cpp +++ b/base/common/DateLUTImpl.cpp @@ -53,7 +53,7 @@ DateLUTImpl::DateLUTImpl(const std::string & time_zone_) const cctz::civil_day epoch{1970, 1, 1}; const cctz::civil_day lut_start{DATE_LUT_MIN_YEAR, 1, 1}; - time_t start_of_day = std::chrono::system_clock::to_time_t(cctz_time_zone.lookup(lut_start).pre); + time_t start_of_day; time_offset_epoch = cctz::convert(cctz::civil_second(lut_start), cctz_time_zone).time_since_epoch().count(); // Note validated this against all timezones in the system. diff --git a/base/common/tests/gtest_DateLutImpl.cpp b/base/common/tests/gtest_DateLutImpl.cpp index 2355c117328..9169d9e768f 100644 --- a/base/common/tests/gtest_DateLutImpl.cpp +++ b/base/common/tests/gtest_DateLutImpl.cpp @@ -29,7 +29,7 @@ std::vector allTimezones() { std::vector result; - auto timezone_name = auto_time_zones; + const auto * timezone_name = auto_time_zones; while (*timezone_name) { result.push_back(*timezone_name); @@ -70,7 +70,7 @@ FailuresCount countFailures(const ::testing::TestResult & test_result) } -TEST(DateLUTTest, Test_makeDayNum) +TEST(DateLUTTest, makeDayNumTest) { const DateLUTImpl & lut = DateLUT::instance("UTC"); EXPECT_EQ(0, lut.makeDayNum(2500, 12, 25)); @@ -264,10 +264,10 @@ TEST(DateLUTTest, TimeValuesAtRightBoderOfRangeOfOLDLut) } -class DateLUT_TimeZone : public ::testing::TestWithParam +class DateLUTWithTimeZone : public ::testing::TestWithParam {}; -TEST_P(DateLUT_TimeZone, DISABLED_LoadLut) +TEST_P(DateLUTWithTimeZone, DISABLED_LoadLut) { // There are some assumptions and assertions about TZ data made in DateLUTImpl which are verified upon loading, // to make sure that those assertions are true for all timezones we are going to load all of them one by one. @@ -275,12 +275,12 @@ TEST_P(DateLUT_TimeZone, DISABLED_LoadLut) } // Another long running test, shouldn't be run to often -TEST_P(DateLUT_TimeZone, VaidateTimeComponentsAroundEpoch) +TEST_P(DateLUTWithTimeZone, VaidateTimeComponentsAroundEpoch) { // Converting time around 1970-01-01 to hour-minute-seconds time components // could be problematic. const size_t max_failures_per_tz = 3; - const auto timezone_name = GetParam(); + const auto * timezone_name = GetParam(); const auto * test_info = ::testing::UnitTest::GetInstance()->current_test_info(); const DateLUTImpl & lut = DateLUT::instance(timezone_name); @@ -311,14 +311,14 @@ TEST_P(DateLUT_TimeZone, VaidateTimeComponentsAroundEpoch) } } -TEST_P(DateLUT_TimeZone, getTimeZone) +TEST_P(DateLUTWithTimeZone, getTimeZone) { const auto & lut = DateLUT::instance(GetParam()); EXPECT_EQ(GetParam(), lut.getTimeZone()); } -TEST_P(DateLUT_TimeZone, ZeroTime) +TEST_P(DateLUTWithTimeZone, ZeroTime) { const auto & lut = DateLUT::instance(GetParam()); @@ -329,7 +329,7 @@ TEST_P(DateLUT_TimeZone, ZeroTime) // Group of tests for timezones that have or had some time ago an offset which is not multiple of 15 minutes. INSTANTIATE_TEST_SUITE_P(ExoticTimezones, - DateLUT_TimeZone, + DateLUTWithTimeZone, ::testing::ValuesIn(std::initializer_list{ "Africa/El_Aaiun", "Pacific/Apia", @@ -340,7 +340,7 @@ INSTANTIATE_TEST_SUITE_P(ExoticTimezones, ); INSTANTIATE_TEST_SUITE_P(DISABLED_AllTimeZones, - DateLUT_TimeZone, + DateLUTWithTimeZone, ::testing::ValuesIn(allTimezones()) ); @@ -370,11 +370,11 @@ std::ostream & operator<<(std::ostream & ostr, const TimeRangeParam & param) return ostr << param.begin << " : " << param.end << " step: " << param.step_in_seconds << "s"; } -class DateLUT_Timezone_TimeRange : public ::testing::TestWithParam> +class DateLUTWithTimeZoneAndTimeRange : public ::testing::TestWithParam> {}; // refactored test from tests/date_lut3.cpp -TEST_P(DateLUT_Timezone_TimeRange, InRange) +TEST_P(DateLUTWithTimeZoneAndTimeRange, InRange) { // for a time_t values in range [begin, end) to match with reference obtained from cctz: // compare date and time components: year, month, day, hours, minutes, seconds, formatted time string. @@ -425,7 +425,7 @@ TEST_P(DateLUT_Timezone_TimeRange, InRange) * So it would be tricky to skip knonw failures to allow all unit tests to pass. */ INSTANTIATE_TEST_SUITE_P(DISABLED_AllTimezones_Year2010, - DateLUT_Timezone_TimeRange, + DateLUTWithTimeZoneAndTimeRange, ::testing::Combine( ::testing::ValuesIn(allTimezones()), ::testing::ValuesIn(std::initializer_list{ @@ -436,7 +436,7 @@ INSTANTIATE_TEST_SUITE_P(DISABLED_AllTimezones_Year2010, ); INSTANTIATE_TEST_SUITE_P(DISABLED_AllTimezones_Year1970_WHOLE, - DateLUT_Timezone_TimeRange, + DateLUTWithTimeZoneAndTimeRange, ::testing::Combine( ::testing::ValuesIn(allTimezones()), ::testing::ValuesIn(std::initializer_list{ @@ -446,7 +446,7 @@ INSTANTIATE_TEST_SUITE_P(DISABLED_AllTimezones_Year1970_WHOLE, ); INSTANTIATE_TEST_SUITE_P(DISABLED_AllTimezones_Year2010_WHOLE, - DateLUT_Timezone_TimeRange, + DateLUTWithTimeZoneAndTimeRange, ::testing::Combine( ::testing::ValuesIn(allTimezones()), ::testing::ValuesIn(std::initializer_list{ @@ -456,7 +456,7 @@ INSTANTIATE_TEST_SUITE_P(DISABLED_AllTimezones_Year2010_WHOLE, ); INSTANTIATE_TEST_SUITE_P(DISABLED_AllTimezones_Year2020_WHOLE, - DateLUT_Timezone_TimeRange, + DateLUTWithTimeZoneAndTimeRange, ::testing::Combine( ::testing::ValuesIn(allTimezones()), ::testing::ValuesIn(std::initializer_list{ @@ -466,7 +466,7 @@ INSTANTIATE_TEST_SUITE_P(DISABLED_AllTimezones_Year2020_WHOLE, ); INSTANTIATE_TEST_SUITE_P(DISABLED_AllTimezones_PreEpoch, - DateLUT_Timezone_TimeRange, + DateLUTWithTimeZoneAndTimeRange, ::testing::Combine( ::testing::ValuesIn(allTimezones()), ::testing::ValuesIn(std::initializer_list{ @@ -476,7 +476,7 @@ INSTANTIATE_TEST_SUITE_P(DISABLED_AllTimezones_PreEpoch, ); INSTANTIATE_TEST_SUITE_P(DISABLED_AllTimezones_Year1970, - DateLUT_Timezone_TimeRange, + DateLUTWithTimeZoneAndTimeRange, ::testing::Combine( ::testing::ValuesIn(allTimezones()), ::testing::ValuesIn(std::initializer_list{ From d568ba5ec7f966842075a66fc9ff554db8430701 Mon Sep 17 00:00:00 2001 From: Vasily Nemkov Date: Sun, 21 Feb 2021 12:04:29 +0200 Subject: [PATCH 039/544] Marked 00921_datetime64_compatibility as long test --- ...bility.python => 00921_datetime64_compatibility_long.python} | 0 ....reference => 00921_datetime64_compatibility_long.reference} | 0 ..._compatibility.sh => 00921_datetime64_compatibility_long.sh} | 2 +- 3 files changed, 1 insertion(+), 1 deletion(-) rename tests/queries/0_stateless/{00921_datetime64_compatibility.python => 00921_datetime64_compatibility_long.python} (100%) rename tests/queries/0_stateless/{00921_datetime64_compatibility.reference => 00921_datetime64_compatibility_long.reference} (100%) rename tests/queries/0_stateless/{00921_datetime64_compatibility.sh => 00921_datetime64_compatibility_long.sh} (91%) diff --git a/tests/queries/0_stateless/00921_datetime64_compatibility.python b/tests/queries/0_stateless/00921_datetime64_compatibility_long.python similarity index 100% rename from tests/queries/0_stateless/00921_datetime64_compatibility.python rename to tests/queries/0_stateless/00921_datetime64_compatibility_long.python diff --git a/tests/queries/0_stateless/00921_datetime64_compatibility.reference b/tests/queries/0_stateless/00921_datetime64_compatibility_long.reference similarity index 100% rename from tests/queries/0_stateless/00921_datetime64_compatibility.reference rename to tests/queries/0_stateless/00921_datetime64_compatibility_long.reference diff --git a/tests/queries/0_stateless/00921_datetime64_compatibility.sh b/tests/queries/0_stateless/00921_datetime64_compatibility_long.sh similarity index 91% rename from tests/queries/0_stateless/00921_datetime64_compatibility.sh rename to tests/queries/0_stateless/00921_datetime64_compatibility_long.sh index 5f5034819e4..52a29c19be1 100755 --- a/tests/queries/0_stateless/00921_datetime64_compatibility.sh +++ b/tests/queries/0_stateless/00921_datetime64_compatibility_long.sh @@ -11,6 +11,6 @@ CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) # ${CURDIR}/00921_datetime64_compatibility.python -python3 "${CURDIR}"/00921_datetime64_compatibility.python \ +python3 "${CURDIR}"/00921_datetime64_compatibility_long.python \ | ${CLICKHOUSE_CLIENT} --ignore-error -T -nm --calculate_text_stack_trace 0 --log-level 'error' 2>&1 \ | grep -v 'Received exception .*$' | sed 's/^\(Code: [0-9]\+\).*$/\1/g' From b8b916008c75e041c7d44c69d65fa0c1135d124d Mon Sep 17 00:00:00 2001 From: alexey-milovidov Date: Mon, 22 Feb 2021 20:50:17 +0300 Subject: [PATCH 040/544] Update DateLUTImpl.h --- base/common/DateLUTImpl.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/base/common/DateLUTImpl.h b/base/common/DateLUTImpl.h index 898fb7bf843..1c897080e3a 100644 --- a/base/common/DateLUTImpl.h +++ b/base/common/DateLUTImpl.h @@ -105,7 +105,7 @@ public: /// The order of fields matters for alignment and sizeof. struct Values { - /// Least significat 64 bits from time_t at beginning of the day. + /// time_t at beginning of the day. Int64 date; /// Properties of the day. From d5757c67e612b1846e71536a965fde7efda2ac2a Mon Sep 17 00:00:00 2001 From: alexey-milovidov Date: Mon, 22 Feb 2021 22:45:55 +0300 Subject: [PATCH 041/544] Update DateLUTImpl.h --- base/common/DateLUTImpl.h | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/base/common/DateLUTImpl.h b/base/common/DateLUTImpl.h index 1c897080e3a..5a12ad5dc13 100644 --- a/base/common/DateLUTImpl.h +++ b/base/common/DateLUTImpl.h @@ -119,8 +119,9 @@ public: UInt8 days_in_month; /// For days, when offset from UTC was changed due to daylight saving time or permanent change, following values could be non zero. - Int8 amount_of_offset_change_value; /// Usually -3600 or 3600, but look at Lord Howe Island. multiply by OffsetChangeFactor - UInt8 time_at_offset_change_value; /// In seconds from beginning of the day. multiply by OffsetChangeFactor + /// All in OffsetChangeFactor (15 minute) intervals. + Int8 amount_of_offset_change_value; /// Usually -4 or 4, but look at Lord Howe Island. Multiply by OffsetChangeFactor + UInt8 time_at_offset_change_value; /// In seconds from beginning of the day. Multiply by OffsetChangeFactor inline Int32 amount_of_offset_change() const { From 3632c1d879adaaa4ab49c5b77589e3fd48706629 Mon Sep 17 00:00:00 2001 From: Alexander Tokmakov Date: Wed, 24 Feb 2021 19:41:44 +0300 Subject: [PATCH 042/544] add flacky check runner --- docker/test/stress/stress | 24 ++++- tests/integration/ci-runner.py | 109 ++++++++++++++++++-- tests/integration/test_drop_replica/test.py | 1 + 3 files changed, 123 insertions(+), 11 deletions(-) diff --git a/docker/test/stress/stress b/docker/test/stress/stress index 3426a20f67b..54af335a440 100755 --- a/docker/test/stress/stress +++ b/docker/test/stress/stress @@ -55,6 +55,27 @@ def run_func_test(cmd, output_prefix, num_processes, skip_tests_option, global_t time.sleep(0.5) return pipes +def prepare_for_hung_check(): + # FIXME this function should not exist, but... + + # We attach gdb to clickhouse-server before running tests + # to print stacktraces of all crashes even if clickhouse cannot print it for some reason. + # However, it obstruct checking for hung queries. + logging.info("Will terminate gdb (if any)") + call("kill -TERM $(pidof gdb)", shell=True, stderr=STDOUT) + + # Some tests execute SYSTEM STOP MERGES or similar queries. + # It may cause some ALTERs to hang. + # Possibly we should fix tests and forbid to use such queries without specifying table. + call("clickhouse client -q 'SYSTEM START MERGES'", shell=True, stderr=STDOUT) + call("clickhouse client -q 'SYSTEM START DISTRIBUTED SENDS'", shell=True, stderr=STDOUT) + call("clickhouse client -q 'SYSTEM START TTL MERGES'", shell=True, stderr=STDOUT) + call("clickhouse client -q 'SYSTEM START MOVES'", shell=True, stderr=STDOUT) + call("clickhouse client -q 'SYSTEM START FETCHES'", shell=True, stderr=STDOUT) + call("clickhouse client -q 'SYSTEM START REPLICATED SENDS'", shell=True, stderr=STDOUT) + call("clickhouse client -q 'SYSTEM START REPLICATION QUEUES'", shell=True, stderr=STDOUT) + + time.sleep(30) if __name__ == "__main__": logging.basicConfig(level=logging.INFO, format='%(asctime)s %(message)s') @@ -85,8 +106,7 @@ if __name__ == "__main__": logging.info("All processes finished") if args.hung_check: - logging.info("Will terminate gdb (if any)") - res = call("kill -TERM $(pidof gdb)", shell=True, stderr=STDOUT) + prepare_for_hung_check() logging.info("Checking if some queries hung") cmd = "{} {} {}".format(args.test_cmd, "--hung-check", "00001_select_1") res = call(cmd, shell=True, stderr=STDOUT) diff --git a/tests/integration/ci-runner.py b/tests/integration/ci-runner.py index 80a283b009e..b72721e6404 100755 --- a/tests/integration/ci-runner.py +++ b/tests/integration/ci-runner.py @@ -16,6 +16,32 @@ SLEEP_BETWEEN_RETRIES = 5 CLICKHOUSE_BINARY_PATH = "/usr/bin/clickhouse" CLICKHOUSE_ODBC_BRIDGE_BINARY_PATH = "/usr/bin/clickhouse-odbc-bridge" +TRIES_COUNT = 10 +MAX_TIME_SECONDS = 3600 + + +def get_tests_to_run(pr_info): + result = set([]) + + if pr_info.changed_files is None: + return [] + + for fpath in pr_info.changed_files: + if 'tests/integration/test_' in fpath: + logging.info('File %s changed and seems like integration test', fpath) + result.add(fpath.split('/')[2]) + return list(result) + + +def filter_existing_tests(tests_to_run, repo_path): + result = [] + for relative_test_path in tests_to_run: + if os.path.exists(os.path.join(repo_path, 'tests/integration', relative_test_path)): + result.append(relative_test_path) + else: + logging.info("Skipping test %s, seems like it was removed", relative_test_path) + return result + def _get_deselect_option(tests): return ' '.join(['--deselect {}'.format(t) for t in tests]) @@ -128,10 +154,13 @@ def clear_ip_tables_and_restart_daemons(): class ClickhouseIntegrationTestsRunner: - def __init__(self, result_path, image_versions, shuffle_groups): + def __init__(self, result_path, params): self.result_path = result_path - self.image_versions = image_versions - self.shuffle_groups = shuffle_groups + self.params = params + + self.image_versions = self.params['docker_images_with_versions'] + self.shuffle_groups = self.params['shuffle_test_groups'] + self.flacky_check = 'flacky check' in self.params['context_name'] def path(self): return self.result_path @@ -328,7 +357,68 @@ class ClickhouseIntegrationTestsRunner: return counters, tests_times, log_name, log_path - def run_impl(self, commit, repo, pull_request, repo_path, build_path): + def run_flacky_check(self, repo_path, build_path): + pr_info = self.params['pr_info'] + + # pytest swears, if we require to run some tests which was renamed or deleted + tests_to_run = filter_existing_tests(get_tests_to_run(pr_info), repo_path) + if not tests_to_run: + logging.info("No tests to run found") + return 'success', 'Nothing to run', [('Nothing to run', 'OK')], '' + + self._install_clickhouse(build_path) + logging.info("Found '%s' tests to run", ' '.join(tests_to_run)) + result_state = "success" + description_prefix = "No flaky tests: " + start = time.time() + logging.info("Starting check with retries") + final_retry = 0 + log_paths = [] + for i in range(TRIES_COUNT): + final_retry += 1 + logging.info("Running tests for the %s time", i) + counters, tests_times, log_name, log_path = self.run_test_group(repo_path, "flaky", tests_to_run, 1) + log_paths.append(log_path) + if counters["FAILED"]: + logging.info("Found failed tests: %s", ' '.join(counters["FAILED"])) + description_prefix = "Flaky tests found: " + result_state = "failure" + break + if counters["ERROR"]: + description_prefix = "Flaky tests found: " + logging.info("Found error tests: %s", ' '.join(counters["ERROR"])) + result_state = "error" + break + logging.info("Try is OK, all tests passed, going to clear env") + clear_ip_tables_and_restart_daemons() + logging.info("And going to sleep for some time") + if time.time() - start > MAX_TIME_SECONDS: + logging.info("Timeout reached, going to finish flaky check") + break + time.sleep(5) + + logging.info("Finally all tests done, going to compress test dir") + test_logs = os.path.join(str(self.path()), "./test_dir.tar") + self._compress_logs("{}/tests/integration".format(repo_path), test_logs) + logging.info("Compression finished") + + test_result = [] + for state in ("ERROR", "FAILED", "PASSED"): + if state == "PASSED": + text_state = "OK" + elif state == "FAILED": + text_state = "FAIL" + else: + text_state = state + test_result += [(c + ' (✕' + str(final_retry) + ')', text_state, str(tests_times[c])) for c in counters[state]] + status_text = description_prefix + ', '.join([str(n).lower().replace('failed', 'fail') + ': ' + str(len(c)) for n, c in counters.items()]) + + return result_state, status_text, test_result, [test_logs] + log_paths + + def run_impl(self, repo_path, build_path): + if self.flacky_check: + return self.flacky_check(repo_path, build_path) + self._install_clickhouse(build_path) logging.info("Dump iptables before run %s", subprocess.check_output("iptables -L", shell=True)) all_tests = self._get_all_tests(repo_path) @@ -351,7 +441,7 @@ class ClickhouseIntegrationTestsRunner: logging.info("Shuffling test groups") random.shuffle(items_to_run) - for group, tests in items_to_run: + for group, tests in items_to_run[:10]: #FIXME logging.info("Running test group %s countaining %s tests", group, len(tests)) group_counters, group_test_times, log_name, log_path = self.run_test_group(repo_path, group, tests, MAX_RETRY) total_tests = 0 @@ -413,14 +503,15 @@ if __name__ == "__main__": repo_path = os.environ.get("CLICKHOUSE_TESTS_REPO_PATH") build_path = os.environ.get("CLICKHOUSE_TESTS_BUILD_PATH") result_path = os.environ.get("CLICKHOUSE_TESTS_RESULT_PATH") - image_versions = os.environ.get("CLICKHOUSE_TESTS_IMAGE_VERSIONS", '{}') - shuffle_groups = int(os.environ.get("SHUFFLE_TEST_GROUPS", '0')) + params_path = os.environ.get("CLICKHOUSE_TESTS_JSON_PARAMS_PATH") - runner = ClickhouseIntegrationTestsRunner(result_path, json.loads(image_versions), shuffle_groups) + params = json.loads(open(params_path, 'r').read()) + runner = ClickhouseIntegrationTestsRunner(result_path, params) logging.info("Running tests") - state, description, test_results, logs = runner.run_impl(None, None, None, repo_path, build_path) + state, description, test_results, logs = runner.run_impl(repo_path, build_path) logging.info("Tests finished") + status = (state, description) out_results_file = os.path.join(str(runner.path()), "test_results.tsv") out_status_file = os.path.join(str(runner.path()), "check_status.tsv") diff --git a/tests/integration/test_drop_replica/test.py b/tests/integration/test_drop_replica/test.py index f3af9dcb980..ac7e6954395 100644 --- a/tests/integration/test_drop_replica/test.py +++ b/tests/integration/test_drop_replica/test.py @@ -54,6 +54,7 @@ node_1_1 = cluster.add_instance('node_1_1', with_zookeeper=True, main_configs=[' node_1_2 = cluster.add_instance('node_1_2', with_zookeeper=True, main_configs=['configs/remote_servers.xml']) node_1_3 = cluster.add_instance('node_1_3', with_zookeeper=True, main_configs=['configs/remote_servers.xml']) +#FIXME it's just to run flacky check @pytest.fixture(scope="module") def start_cluster(): From ad5a03e8a4dd6f63db077b7fe181b3aa4a40307b Mon Sep 17 00:00:00 2001 From: Anton Popov Date: Wed, 24 Feb 2021 21:13:26 +0300 Subject: [PATCH 043/544] improve support of integer keys in type Map --- src/DataTypes/DataTypeMap.cpp | 2 +- src/DataTypes/DataTypeNumberBase.cpp | 4 ++-- src/Functions/array/arrayElement.cpp | 16 ++++++++++++---- src/Functions/map.cpp | 12 ++---------- 4 files changed, 17 insertions(+), 17 deletions(-) diff --git a/src/DataTypes/DataTypeMap.cpp b/src/DataTypes/DataTypeMap.cpp index 9d69c4ff66c..0fe479ae373 100644 --- a/src/DataTypes/DataTypeMap.cpp +++ b/src/DataTypes/DataTypeMap.cpp @@ -53,7 +53,7 @@ DataTypeMap::DataTypeMap(const DataTypePtr & key_type_, const DataTypePtr & valu assertKeyType(); } -void DataTypeMap::assertKeyType() const +void DataTypeMap::assertKeyType() const { if (!key_type->isValueRepresentedByInteger() && !isStringOrFixedString(*key_type) && !WhichDataType(key_type).isNothing()) throw Exception(ErrorCodes::BAD_ARGUMENTS, diff --git a/src/DataTypes/DataTypeNumberBase.cpp b/src/DataTypes/DataTypeNumberBase.cpp index ae3e6762d27..f6c06b97710 100644 --- a/src/DataTypes/DataTypeNumberBase.cpp +++ b/src/DataTypes/DataTypeNumberBase.cpp @@ -212,13 +212,13 @@ MutableColumnPtr DataTypeNumberBase::createColumn() const template bool DataTypeNumberBase::isValueRepresentedByInteger() const { - return std::is_integral_v; + return is_integer_v; } template bool DataTypeNumberBase::isValueRepresentedByUnsignedInteger() const { - return std::is_integral_v && is_unsigned_v; + return is_integer_v && is_unsigned_v; } diff --git a/src/Functions/array/arrayElement.cpp b/src/Functions/array/arrayElement.cpp index cca252216b3..a94a640958b 100644 --- a/src/Functions/array/arrayElement.cpp +++ b/src/Functions/array/arrayElement.cpp @@ -872,7 +872,12 @@ bool FunctionArrayElement::matchKeyToIndexNumberConst( if (!data_numeric) return false; - if (index.getType() != Field::Types::UInt64 && index.getType() != Field::Types::Int64 && index.getType() != Field::Types::Int128) + bool is_integer_field = Field::dispatch([](const auto & value) + { + return is_integer_v>; + }, index); + + if (!is_integer_field) return false; MatcherNumberConst matcher{data_numeric->getData(), get(index)}; @@ -911,6 +916,9 @@ bool FunctionArrayElement::matchKeyToIndex( || matchKeyToIndexNumber(data, offsets, arguments, matched_idxs) || matchKeyToIndexNumber(data, offsets, arguments, matched_idxs) || matchKeyToIndexNumber(data, offsets, arguments, matched_idxs) + || matchKeyToIndexNumber(data, offsets, arguments, matched_idxs) + || matchKeyToIndexNumber(data, offsets, arguments, matched_idxs) + || matchKeyToIndexNumber(data, offsets, arguments, matched_idxs) || matchKeyToIndexString(data, offsets, arguments, matched_idxs); } @@ -927,6 +935,9 @@ bool FunctionArrayElement::matchKeyToIndexConst( || matchKeyToIndexNumberConst(data, offsets, index, matched_idxs) || matchKeyToIndexNumberConst(data, offsets, index, matched_idxs) || matchKeyToIndexNumberConst(data, offsets, index, matched_idxs) + || matchKeyToIndexNumberConst(data, offsets, index, matched_idxs) + || matchKeyToIndexNumberConst(data, offsets, index, matched_idxs) + || matchKeyToIndexNumberConst(data, offsets, index, matched_idxs) || matchKeyToIndexStringConst(data, offsets, index, matched_idxs); } @@ -947,9 +958,6 @@ ColumnPtr FunctionArrayElement::executeMap( indices_column->reserve(input_rows_count); auto & indices_data = assert_cast &>(*indices_column).getData(); - std::cerr << "types: " << arguments[0].type->getName() << " " << arguments[1].type->getName() << "\n"; - std::cerr << "columns: " << arguments[0].column->dumpStructure() << " " << arguments[1].column->dumpStructure() << "\n"; - if (!isColumnConst(*arguments[1].column)) { if (input_rows_count > 0 && !matchKeyToIndex(keys_data, offsets, arguments, indices_data)) diff --git a/src/Functions/map.cpp b/src/Functions/map.cpp index 0c87f6f8e0d..7418691e61b 100644 --- a/src/Functions/map.cpp +++ b/src/Functions/map.cpp @@ -65,7 +65,8 @@ public: DataTypePtr getReturnTypeImpl(const DataTypes & arguments) const override { if (arguments.size() % 2 != 0) - throw Exception("Function " + getName() + " even number of arguments", ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH); + throw Exception(ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH, + "Function {} requires even number of arguments, but {} given", getName(), arguments.size()); DataTypes keys, values; for (size_t i = 0; i < arguments.size(); i += 2) @@ -84,9 +85,6 @@ public: { size_t num_elements = arguments.size(); - std::cerr << "map... input_rows_count: " << input_rows_count << "\n"; - std::cerr << "num_elements: " << num_elements << "\n"; - if (num_elements == 0) return result_type->createColumnConstWithDefaultValue(input_rows_count); @@ -102,13 +100,9 @@ public: const auto & arg = arguments[i]; const auto to_type = i % 2 == 0 ? key_type : value_type; - std::cerr << "to_type: " << to_type->getName() << ", arg: " << arg.column->dumpStructure() << "\n"; - ColumnPtr preprocessed_column = castColumn(arg, to_type); preprocessed_column = preprocessed_column->convertToFullColumnIfConst(); - std::cerr << "preprocessed_column: " << preprocessed_column->dumpStructure() << "\n"; - columns_holder[i] = std::move(preprocessed_column); column_ptrs[i] = columns_holder[i].get(); } @@ -140,8 +134,6 @@ public: auto nested_column = ColumnArray::create( ColumnTuple::create(Columns{std::move(keys_data), std::move(values_data)}), std::move(offsets)); - - std::cerr << "nested_column: " << nested_column->dumpStructure() << "\n"; return ColumnMap::create(nested_column); } From 94156986bf143b639f0b2e13d678d7f578a6ceb3 Mon Sep 17 00:00:00 2001 From: Anton Popov Date: Wed, 24 Feb 2021 21:15:42 +0300 Subject: [PATCH 044/544] add test for map with integer keys --- .../01720_type_map_and_casts.reference | 40 ++++++++++ .../0_stateless/01720_type_map_and_casts.sql | 79 +++++++++++++++++++ 2 files changed, 119 insertions(+) create mode 100644 tests/queries/0_stateless/01720_type_map_and_casts.reference create mode 100644 tests/queries/0_stateless/01720_type_map_and_casts.sql diff --git a/tests/queries/0_stateless/01720_type_map_and_casts.reference b/tests/queries/0_stateless/01720_type_map_and_casts.reference new file mode 100644 index 00000000000..1ceb4d78f81 --- /dev/null +++ b/tests/queries/0_stateless/01720_type_map_and_casts.reference @@ -0,0 +1,40 @@ +Map(Int8, Int8) +{127:1,0:1,-1:1} +{} +1 0 1 1 +0 0 0 0 +0 +1 +1 +0 +0 +0 +0 +0 +0 +0 +1 +Map(Int32, UInt16) +{-1:1,2147483647:2,-2147483648:3} +1 2 3 +0 +1 +0 +0 +0 +Map(Date, Int32) +{'2020-01-01':1,'2020-01-02':2,'1970-01-02':3} +1 2 0 +0 +3 +0 +Map(UUID, UInt16) +{'00001192-0000-4000-8000-000000000001':1,'00001192-0000-4000-7000-000000000001':2} +0 2 1 +Map(Int128, Int32) +{-1:'a',0:'b',1234567898765432193024000:'c',-1234567898765432193024000:'d'} +a b c d + +a +b + diff --git a/tests/queries/0_stateless/01720_type_map_and_casts.sql b/tests/queries/0_stateless/01720_type_map_and_casts.sql new file mode 100644 index 00000000000..2f333373dba --- /dev/null +++ b/tests/queries/0_stateless/01720_type_map_and_casts.sql @@ -0,0 +1,79 @@ +SET allow_experimental_map_type = 1; + +DROP TABLE IF EXISTS table_map_with_key_integer; + +CREATE TABLE table_map_with_key_integer (d DATE, m Map(Int8, Int8)) +ENGINE = MergeTree() ORDER BY d; + +INSERT INTO table_map_with_key_integer VALUES ('2020-01-01', map(127, 1, 0, 1, -1, 1)) ('2020-01-01', map()); + +SELECT 'Map(Int8, Int8)'; + +SELECT m FROM table_map_with_key_integer; +SELECT m[127], m[1], m[0], m[-1] FROM table_map_with_key_integer; +SELECT m[toInt8(number - 2)] FROM table_map_with_key_integer ARRAY JOIN range(5) AS number; + +SELECT count() FROM table_map_with_key_integer WHERE m = map(); + +DROP TABLE IF EXISTS table_map_with_key_integer; + +CREATE TABLE table_map_with_key_integer (d DATE, m Map(Int32, UInt16)) +ENGINE = MergeTree() ORDER BY d; + +INSERT INTO table_map_with_key_integer VALUES ('2020-01-01', map(-1, 1, 2147483647, 2, -2147483648, 3)); + +SELECT 'Map(Int32, UInt16)'; + +SELECT m FROM table_map_with_key_integer; +SELECT m[-1], m[2147483647], m[-2147483648] FROM table_map_with_key_integer; +SELECT m[toInt32(number - 2)] FROM table_map_with_key_integer ARRAY JOIN range(5) AS number; + +DROP TABLE IF EXISTS table_map_with_key_integer; + +CREATE TABLE table_map_with_key_integer (d DATE, m Map(Date, Int32)) +ENGINE = MergeTree() ORDER BY d; + +INSERT INTO table_map_with_key_integer VALUES ('2020-01-01', map('2020-01-01', 1, '2020-01-02', 2, '1970-01-02', 3)); + +SELECT 'Map(Date, Int32)'; + +SELECT m FROM table_map_with_key_integer; +SELECT m[toDate('2020-01-01')], m[toDate('2020-01-02')], m[toDate('2020-01-03')] FROM table_map_with_key_integer; +SELECT m[toDate(number)] FROM table_map_with_key_integer ARRAY JOIN range(3) AS number; + +DROP TABLE IF EXISTS table_map_with_key_integer; + +CREATE TABLE table_map_with_key_integer (d DATE, m Map(UUID, UInt16)) +ENGINE = MergeTree() ORDER BY d; + +INSERT INTO table_map_with_key_integer VALUES ('2020-01-01', map('00001192-0000-4000-8000-000000000001', 1, '00001192-0000-4000-7000-000000000001', 2)); + +SELECT 'Map(UUID, UInt16)'; + +SELECT m FROM table_map_with_key_integer; +SELECT + m[toUUID('00001192-0000-4000-6000-000000000001')], + m[toUUID('00001192-0000-4000-7000-000000000001')], + m[toUUID('00001192-0000-4000-8000-000000000001')] +FROM table_map_with_key_integer; + +DROP TABLE IF EXISTS table_map_with_key_integer; + +CREATE TABLE table_map_with_key_integer (d DATE, m Map(Int128, String)) +ENGINE = MergeTree() ORDER BY d; + + +INSERT INTO table_map_with_key_integer SELECT '2020-01-01', map(-1, 'a', 0, 'b', toInt128(1234567898765432123456789), 'c', toInt128(-1234567898765432123456789), 'd'); + +SELECT 'Map(Int128, Int32)'; + +SELECT m FROM table_map_with_key_integer; +SELECT m[toInt128(-1)], m[toInt128(0)], m[toInt128(1234567898765432123456789)], m[toInt128(-1234567898765432123456789)] FROM table_map_with_key_integer; +SELECT m[toInt128(number - 2)] FROM table_map_with_key_integer ARRAY JOIN range(4) AS number; + +DROP TABLE IF EXISTS table_map_with_key_integer; + + +CREATE TABLE table_map_with_key_integer (m Map(Float32, String)) ENGINE = MergeTree() ORDER BY tuple(); -- { serverError 36} +CREATE TABLE table_map_with_key_integer (m Map(Nullable(String), String)) ENGINE = MergeTree() ORDER BY tuple(); -- { serverError 36} +CREATE TABLE table_map_with_key_integer (m Map(Array(UInt32), String)) ENGINE = MergeTree() ORDER BY tuple(); -- { serverError 36} From 9811fce5dc21afdd05baf1a7d3404cf51fad81ca Mon Sep 17 00:00:00 2001 From: Anton Popov Date: Wed, 24 Feb 2021 21:54:18 +0300 Subject: [PATCH 045/544] Update convertFieldToType.cpp --- src/Interpreters/convertFieldToType.cpp | 1 - 1 file changed, 1 deletion(-) diff --git a/src/Interpreters/convertFieldToType.cpp b/src/Interpreters/convertFieldToType.cpp index d2b8dd732bd..1d93ef56dea 100644 --- a/src/Interpreters/convertFieldToType.cpp +++ b/src/Interpreters/convertFieldToType.cpp @@ -15,7 +15,6 @@ #include #include #include -#include #include #include From efd50a819b3552e587b49d2030de0099fb234b63 Mon Sep 17 00:00:00 2001 From: Alexander Tokmakov Date: Thu, 25 Feb 2021 02:09:09 +0300 Subject: [PATCH 046/544] fix --- tests/integration/ci-runner.py | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/tests/integration/ci-runner.py b/tests/integration/ci-runner.py index b72721e6404..a455df4f7c7 100755 --- a/tests/integration/ci-runner.py +++ b/tests/integration/ci-runner.py @@ -22,11 +22,12 @@ MAX_TIME_SECONDS = 3600 def get_tests_to_run(pr_info): result = set([]) + changed_files = pr_info['changed_files'] - if pr_info.changed_files is None: + if changed_files is None: return [] - for fpath in pr_info.changed_files: + for fpath in changed_files: if 'tests/integration/test_' in fpath: logging.info('File %s changed and seems like integration test', fpath) result.add(fpath.split('/')[2]) @@ -160,7 +161,7 @@ class ClickhouseIntegrationTestsRunner: self.image_versions = self.params['docker_images_with_versions'] self.shuffle_groups = self.params['shuffle_test_groups'] - self.flacky_check = 'flacky check' in self.params['context_name'] + self.flaky_check = 'flaky check' in self.params['context_name'] def path(self): return self.result_path @@ -344,6 +345,7 @@ class ClickhouseIntegrationTestsRunner: self._update_counters(counters, new_counters) for test_name, test_time in new_tests_times.items(): tests_times[test_name] = test_time + os.remove(output_path) if len(counters["PASSED"]) == len(tests_in_group): logging.info("All tests from group %s passed", test_group) break @@ -357,7 +359,7 @@ class ClickhouseIntegrationTestsRunner: return counters, tests_times, log_name, log_path - def run_flacky_check(self, repo_path, build_path): + def run_flaky_check(self, repo_path, build_path): pr_info = self.params['pr_info'] # pytest swears, if we require to run some tests which was renamed or deleted @@ -416,8 +418,8 @@ class ClickhouseIntegrationTestsRunner: return result_state, status_text, test_result, [test_logs] + log_paths def run_impl(self, repo_path, build_path): - if self.flacky_check: - return self.flacky_check(repo_path, build_path) + if self.flaky_check: + return self.flaky_check(repo_path, build_path) self._install_clickhouse(build_path) logging.info("Dump iptables before run %s", subprocess.check_output("iptables -L", shell=True)) From 65384f43d8ec2c915e573a812c04ff46f97cbfc7 Mon Sep 17 00:00:00 2001 From: Alexander Tokmakov Date: Thu, 25 Feb 2021 10:02:45 +0300 Subject: [PATCH 047/544] fix --- tests/integration/ci-runner.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/integration/ci-runner.py b/tests/integration/ci-runner.py index a455df4f7c7..09dff9b3853 100755 --- a/tests/integration/ci-runner.py +++ b/tests/integration/ci-runner.py @@ -419,7 +419,7 @@ class ClickhouseIntegrationTestsRunner: def run_impl(self, repo_path, build_path): if self.flaky_check: - return self.flaky_check(repo_path, build_path) + return self.run_flaky_check(repo_path, build_path) self._install_clickhouse(build_path) logging.info("Dump iptables before run %s", subprocess.check_output("iptables -L", shell=True)) @@ -443,7 +443,7 @@ class ClickhouseIntegrationTestsRunner: logging.info("Shuffling test groups") random.shuffle(items_to_run) - for group, tests in items_to_run[:10]: #FIXME + for group, tests in items_to_run: logging.info("Running test group %s countaining %s tests", group, len(tests)) group_counters, group_test_times, log_name, log_path = self.run_test_group(repo_path, group, tests, MAX_RETRY) total_tests = 0 From b331b03d3aad1255b340352e80e3b7e325af9531 Mon Sep 17 00:00:00 2001 From: Slach Date: Thu, 25 Feb 2021 16:07:09 +0500 Subject: [PATCH 048/544] fix indvalid links, add some PR re-created from scratch https://github.com/ClickHouse/ClickHouse/pull/21122 --- .../system-tables/replication_queue.md | 2 +- docs/en/sql-reference/operators/in.md | 2 +- docs/ru/operations/settings/settings.md | 11 +++++++--- .../system-tables/replication_queue.md | 2 +- docs/ru/sql-reference/operators/in.md | 22 +++++++++++++++++++ 5 files changed, 33 insertions(+), 6 deletions(-) diff --git a/docs/en/operations/system-tables/replication_queue.md b/docs/en/operations/system-tables/replication_queue.md index aa379caa46c..e2e606ef075 100644 --- a/docs/en/operations/system-tables/replication_queue.md +++ b/docs/en/operations/system-tables/replication_queue.md @@ -76,6 +76,6 @@ last_postpone_time: 1970-01-01 03:00:00 **See Also** -- [Managing ReplicatedMergeTree Tables](../../sql-reference/statements/system.md/#query-language-system-replicated) +- [Managing ReplicatedMergeTree Tables](../../sql-reference/statements/system.md#query-language-system-replicated) [Original article](https://clickhouse.tech/docs/en/operations/system_tables/replication_queue) diff --git a/docs/en/sql-reference/operators/in.md b/docs/en/sql-reference/operators/in.md index bfa8b3d1003..100d00356b1 100644 --- a/docs/en/sql-reference/operators/in.md +++ b/docs/en/sql-reference/operators/in.md @@ -203,7 +203,7 @@ It also makes sense to specify a local table in the `GLOBAL IN` clause, in case When max_parallel_replicas is greater than 1, distributed queries are further transformed. For example, the following: ```sql -SEELECT CounterID, count() FROM distributed_table_1 WHERE UserID IN (SELECT UserID FROM local_table_2 WHERE CounterID < 100) +SELECT CounterID, count() FROM distributed_table_1 WHERE UserID IN (SELECT UserID FROM local_table_2 WHERE CounterID < 100) SETTINGS max_parallel_replicas=3 ``` diff --git a/docs/ru/operations/settings/settings.md b/docs/ru/operations/settings/settings.md index 7322b6c9184..deda437e933 100644 --- a/docs/ru/operations/settings/settings.md +++ b/docs/ru/operations/settings/settings.md @@ -1086,9 +1086,14 @@ load_balancing = round_robin ## max_parallel_replicas {#settings-max_parallel_replicas} -Максимальное количество используемых реплик каждого шарда при выполнении запроса. -Для консистентности (чтобы получить разные части одного и того же разбиения), эта опция работает только при заданном ключе сэмплирования. -Отставание реплик не контролируется. +Максимальное кол-во реплик для каждого шарда во время исполениня запроса из distributed. В некоторых случаях, это может привести к более быстрому исполнению запроса за счет выполнения на большем кол-ве серверов. Эта настройка полезна только для реплицируемых таблиц созданных с использованием SAMPLING KEY выражения. Есть случаи когда производительность не улучшится или даже ухудшится: + +- позиция ключа семплирования в ключе партицирования не позволяет делать эффективные сканирования по диапозонам +- добавление семплирующего ключа к таблице, делает фильтрацию других колонок менее эффективной +- выражение используемое для вычисления ключа семплирования требует больших вычислительных затрат +- Распределение сетевых задержек внутри кластера имеет длинный хвост, так что запрос большего количества серверов может увеличить общую задержку запроса + +Кроме того, эта настройка может привести к некорректным результатам когда используются join или подзапросы и все таблицы не соответсвуют определенным условиям. Подробнее [Распределенные подзапросы и max_parallel_replicas](../../sql-reference/operators/in.md#max_parallel_replica-subqueries) for more details. ## compile {#compile} diff --git a/docs/ru/operations/system-tables/replication_queue.md b/docs/ru/operations/system-tables/replication_queue.md index 47f64aea55d..1ba4b2b8a36 100644 --- a/docs/ru/operations/system-tables/replication_queue.md +++ b/docs/ru/operations/system-tables/replication_queue.md @@ -76,6 +76,6 @@ last_postpone_time: 1970-01-01 03:00:00 **Смотрите также** -- [Управление таблицами ReplicatedMergeTree](../../sql-reference/statements/system.md/#query-language-system-replicated) +- [Управление таблицами ReplicatedMergeTree](../../sql-reference/statements/system.md#query-language-system-replicated) [Оригинальная статья](https://clickhouse.tech/docs/ru/operations/system_tables/replication_queue) diff --git a/docs/ru/sql-reference/operators/in.md b/docs/ru/sql-reference/operators/in.md index 4c1290df166..ee63d09fc66 100644 --- a/docs/ru/sql-reference/operators/in.md +++ b/docs/ru/sql-reference/operators/in.md @@ -197,3 +197,25 @@ SELECT uniq(UserID) FROM local_table WHERE CounterID = 101500 AND UserID GLOBAL 5. Если в GLOBAL IN есть частая необходимость, то спланируйте размещение кластера ClickHouse таким образом, чтобы в каждом дата-центре была хотя бы одна реплика каждого шарда, и среди них была быстрая сеть - чтобы запрос целиком можно было бы выполнить, передавая данные в пределах одного дата-центра. В секции `GLOBAL IN` также имеет смысл указывать локальную таблицу - в случае, если эта локальная таблица есть только на сервере-инициаторе запроса, и вы хотите воспользоваться данными из неё на удалённых серверах. + +### Распределенные подзапросы и max_parallel_replicas {#max_parallel_replica-subqueries} + +Когда настройка max_parallel_replicas больше чем 1, распределенные запросы преобразуются. Например, следующий запрос: + +```sql +SELECT CounterID, count() FROM distributed_table_1 WHERE UserID IN (SELECT UserID FROM local_table_2 WHERE CounterID < 100) +SETTINGS max_parallel_replicas=3 +``` + +преобразуются на каждом сервере в + +```sql +SELECT CounterID, count() FROM local_table_1 WHERE UserID IN (SELECT UserID FROM local_table_2 WHERE CounterID < 100) +SETTINGS parallel_replicas_count=3, parallel_replicas_offset=M +``` + +где M значение между 1 и 3 зависящее от того на какой реплике выполняется локальный запрос. Эти параметры влияют на каждую таблицу семейства MergeTree в запросе и имеют тот же эффект, что и применение `SAMPLE 1/3 OFFSET (M-1)/3` для каждой таблицы. + +Поэтому применение настройки max_parallel_replicas даст корректные результаты если обе таблицы имеют одинаковую схему репликации и семплированы по UserID выражению от UserID. В частности, если local_table_2 не имеет семплирующего ключа, будут получены неверные результаты. Тоже правило применяется для JOIN. + +Один из способов избежать этого, если local_table_2 не удовлетворяет требованиям, использовать `GLOBAL IN` или `GLOBAL JOIN`. From 64ea1f82989ad45555629759b6f395804b12c864 Mon Sep 17 00:00:00 2001 From: Nikolai Kochetov Date: Thu, 25 Feb 2021 18:51:01 +0300 Subject: [PATCH 049/544] Save packet keys. --- src/Common/ColumnsHashing.h | 8 ++++++ src/Interpreters/AggregationCommon.h | 40 ++++++++++++++++++++++++++++ 2 files changed, 48 insertions(+) diff --git a/src/Common/ColumnsHashing.h b/src/Common/ColumnsHashing.h index 1ac753fbae5..7bc84eb3429 100644 --- a/src/Common/ColumnsHashing.h +++ b/src/Common/ColumnsHashing.h @@ -484,6 +484,8 @@ struct HashMethodKeysFixed std::unique_ptr columns_data; #endif + PaddedPODArray prepared_keys; + HashMethodKeysFixed(const ColumnRawPtrs & key_columns, const Sizes & key_sizes_, const HashMethodContextPtr &) : Base(key_columns), key_sizes(std::move(key_sizes_)), keys_size(key_columns.size()) { @@ -505,6 +507,9 @@ struct HashMethodKeysFixed } } + if constexpr (!has_low_cardinality && !has_nullable_keys && sizeof(Key) <= 16) + packFixedBatch(keys_size, Base::getActualColumns(), key_sizes, prepared_keys); + #if defined(__SSSE3__) && !defined(MEMORY_SANITIZER) if constexpr (!has_low_cardinality && !has_nullable_keys && sizeof(Key) <= 16) { @@ -571,6 +576,9 @@ struct HashMethodKeysFixed return packFixed(row, keys_size, low_cardinality_keys.nested_columns, key_sizes, &low_cardinality_keys.positions, &low_cardinality_keys.position_sizes); + if (!prepared_keys.empty()) + return prepared_keys[row]; + #if defined(__SSSE3__) && !defined(MEMORY_SANITIZER) if constexpr (!has_low_cardinality && !has_nullable_keys && sizeof(Key) <= 16) return packFixedShuffle(columns_data.get(), keys_size, key_sizes.data(), row, masks.get()); diff --git a/src/Interpreters/AggregationCommon.h b/src/Interpreters/AggregationCommon.h index e896b0e14df..f05c3420f5f 100644 --- a/src/Interpreters/AggregationCommon.h +++ b/src/Interpreters/AggregationCommon.h @@ -70,6 +70,46 @@ constexpr auto getBitmapSize() } +template +void fillFixedBatch(size_t num_rows, const T * source, T * dest) +{ + for (size_t i = 0; i < num_rows; ++i) + { + *dest = *source; + ++source; + dest += step; + } +} + +template +void fillFixedBatch(size_t keys_size, const ColumnRawPtrs & key_columns, const Sizes & key_sizes, PaddedPODArray & out, size_t & offset) +{ + for (size_t i = 0; i < keys_size; ++i) + { + if (key_sizes[i] == sizeof(T)) + { + const auto * column = key_columns[i]; + size_t num_rows = column->size(); + out.resize(num_rows); + + const char * source = static_cast(column)->getRawDataBegin(); + T * dest = reinterpret_cast(reinterpret_cast(out.data()) + offset); + fillFixedBatch(num_rows, reinterpret_cast(source), dest); + offset += sizeof(T); + } + } +} + +template +void packFixedBatch(size_t keys_size, const ColumnRawPtrs & key_columns, const Sizes & key_sizes, PaddedPODArray & out) +{ + size_t offset = 0; + fillFixedBatch(keys_size, key_columns, key_sizes, out, offset); + fillFixedBatch(keys_size, key_columns, key_sizes, out, offset); + fillFixedBatch(keys_size, key_columns, key_sizes, out, offset); + fillFixedBatch(keys_size, key_columns, key_sizes, out, offset); +} + template using KeysNullMap = std::array()>; From fbbb0c4b69ce2f79cd17e896dcfedca6ffc9fc98 Mon Sep 17 00:00:00 2001 From: Alexander Tokmakov Date: Thu, 25 Feb 2021 19:11:43 +0300 Subject: [PATCH 050/544] fix --- docker/test/stress/run.sh | 2 +- tests/integration/ci-runner.py | 5 ++++- tests/integration/test_drop_replica/test.py | 2 -- tests/queries/0_stateless/00600_replace_running_query.sh | 5 ++++- 4 files changed, 9 insertions(+), 5 deletions(-) diff --git a/docker/test/stress/run.sh b/docker/test/stress/run.sh index dcc92e1ad5f..ca90d474df6 100755 --- a/docker/test/stress/run.sh +++ b/docker/test/stress/run.sh @@ -131,5 +131,5 @@ pigz < /var/log/clickhouse-server/clickhouse-server.log > /test_output/clickhous mv /var/log/clickhouse-server/stderr.log /test_output/ # Write check result into check_status.tsv -clickhouse-local --structure "test String, res String" -q "SELECT 'failure', test FROM table WHERE res != 'OK' LIMIT 1" < /test_output/test_results.tsv > /test_output/check_status.tsv +clickhouse-local --structure "test String, res String" -q "SELECT 'failure', test FROM table WHERE res != 'OK' order by (lower(test) like '%hung%') LIMIT 1" < /test_output/test_results.tsv > /test_output/check_status.tsv [ -s /test_output/check_status.tsv ] || echo -e "success\tNo errors found" > /test_output/check_status.tsv diff --git a/tests/integration/ci-runner.py b/tests/integration/ci-runner.py index 09dff9b3853..f758457ada0 100755 --- a/tests/integration/ci-runner.py +++ b/tests/integration/ci-runner.py @@ -483,12 +483,15 @@ class ClickhouseIntegrationTestsRunner: text_state = state test_result += [(c, text_state, str(tests_times[c])) for c in counters[state]] - status_text = ', '.join([str(n).lower().replace('failed', 'fail') + ': ' + str(len(c)) for n, c in counters.items()]) + status_text = "fail: {}, passed: {}, error: {}".format(len(counters['FAILED']), len(counters['PASSED']), len(counters['ERROR'])) if not counters or sum(len(counter) for counter in counters.values()) == 0: status_text = "No tests found for some reason! It's a bug" result_state = "failure" + if '(memory)' in self.params['context_name']: + result_state = "success" + return result_state, status_text, test_result, [test_logs] + logs def write_results(results_file, status_file, results, status): diff --git a/tests/integration/test_drop_replica/test.py b/tests/integration/test_drop_replica/test.py index ac7e6954395..7d7ad784166 100644 --- a/tests/integration/test_drop_replica/test.py +++ b/tests/integration/test_drop_replica/test.py @@ -49,12 +49,10 @@ def fill_nodes(nodes, shard): cluster = ClickHouseCluster(__file__) - node_1_1 = cluster.add_instance('node_1_1', with_zookeeper=True, main_configs=['configs/remote_servers.xml']) node_1_2 = cluster.add_instance('node_1_2', with_zookeeper=True, main_configs=['configs/remote_servers.xml']) node_1_3 = cluster.add_instance('node_1_3', with_zookeeper=True, main_configs=['configs/remote_servers.xml']) -#FIXME it's just to run flacky check @pytest.fixture(scope="module") def start_cluster(): diff --git a/tests/queries/0_stateless/00600_replace_running_query.sh b/tests/queries/0_stateless/00600_replace_running_query.sh index be5523e06ea..78ea4daf6bb 100755 --- a/tests/queries/0_stateless/00600_replace_running_query.sh +++ b/tests/queries/0_stateless/00600_replace_running_query.sh @@ -6,6 +6,8 @@ CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) # shellcheck source=../shell_config.sh . "$CURDIR"/../shell_config.sh +${CLICKHOUSE_CLIENT} -q "drop user if exists u_00600" +${CLICKHOUSE_CLIENT} -q "create user u_00600 settings max_execution_time=60, readonly=1" function wait_for_query_to_start() { @@ -22,7 +24,7 @@ $CLICKHOUSE_CURL -sS "$CLICKHOUSE_URL&query_id=hello&replace_running_query=1" -d # Wait for it to be replaced wait -${CLICKHOUSE_CLIENT_BINARY} --user=readonly --query_id=42 --query='SELECT 2, count() FROM system.numbers' 2>&1 | grep -cF 'was cancelled' & +${CLICKHOUSE_CLIENT_BINARY} --user=u_00600 --query_id=42 --query='SELECT 2, count() FROM system.numbers' 2>&1 | grep -cF 'was cancelled' & wait_for_query_to_start '42' # Trying to run another query with the same query_id @@ -39,3 +41,4 @@ wait_for_query_to_start '42' ${CLICKHOUSE_CLIENT} --query_id=42 --replace_running_query=1 --replace_running_query_max_wait_ms=500 --query='SELECT 43' 2>&1 | grep -F "can't be stopped" > /dev/null wait ${CLICKHOUSE_CLIENT} --query_id=42 --replace_running_query=1 --query='SELECT 44' +${CLICKHOUSE_CLIENT} -q "drop user u_00600" From 2bf533630c7a70232b1615e74cca9d8c699c7de0 Mon Sep 17 00:00:00 2001 From: Nikolai Kochetov Date: Thu, 25 Feb 2021 22:02:58 +0300 Subject: [PATCH 051/544] Fix tests. --- src/Interpreters/AggregationCommon.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Interpreters/AggregationCommon.h b/src/Interpreters/AggregationCommon.h index f05c3420f5f..8bdea536392 100644 --- a/src/Interpreters/AggregationCommon.h +++ b/src/Interpreters/AggregationCommon.h @@ -90,7 +90,7 @@ void fillFixedBatch(size_t keys_size, const ColumnRawPtrs & key_columns, const S { const auto * column = key_columns[i]; size_t num_rows = column->size(); - out.resize(num_rows); + out.resize_fill(num_rows); const char * source = static_cast(column)->getRawDataBegin(); T * dest = reinterpret_cast(reinterpret_cast(out.data()) + offset); From 5482a82342ccdafed90b154b193f5ff34c5d0a2d Mon Sep 17 00:00:00 2001 From: Anton Popov Date: Fri, 26 Feb 2021 00:34:35 +0300 Subject: [PATCH 052/544] skip test --- tests/queries/skip_list.json | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/queries/skip_list.json b/tests/queries/skip_list.json index 39ec8bac3cf..f7aad848260 100644 --- a/tests/queries/skip_list.json +++ b/tests/queries/skip_list.json @@ -111,6 +111,7 @@ "memory_tracking", "memory_usage", "live_view", + "01720_type_map_and_casts", "01413_alter_update_supertype", "01149_zookeeper_mutation_stuck_after_replace_partition", "00836_indices_alter_replicated_zookeeper", From ce365e2c19461cd590864d45fb1c36f79cfaec3b Mon Sep 17 00:00:00 2001 From: Nikolai Kochetov Date: Fri, 26 Feb 2021 11:52:50 +0300 Subject: [PATCH 053/544] Fix tests. --- src/Interpreters/Aggregator.cpp | 51 ++++++++++++++++++++++++++------- src/Interpreters/Aggregator.h | 46 +++++++++++++++++++++++++++-- 2 files changed, 84 insertions(+), 13 deletions(-) diff --git a/src/Interpreters/Aggregator.cpp b/src/Interpreters/Aggregator.cpp index abff6f21acf..911352c693c 100644 --- a/src/Interpreters/Aggregator.cpp +++ b/src/Interpreters/Aggregator.cpp @@ -1125,11 +1125,23 @@ void NO_INLINE Aggregator::convertToBlockImplFinal( } } - data.forEachValue([&](const auto & key, auto & mapped) + if constexpr (Method::fixed_keys) { - method.insertKeyIntoColumns(key, key_columns, key_sizes); - insertAggregatesIntoColumns(mapped, final_aggregate_columns, arena); - }); + auto new_key_columns = method.shuffleKeyColumns(key_columns, key_sizes); + data.forEachValue([&](const auto & key, auto & mapped) + { + method.insertKeyIntoColumns(key, new_key_columns.first, new_key_columns.second); + insertAggregatesIntoColumns(mapped, final_aggregate_columns, arena); + }); + } + else + { + data.forEachValue([&](const auto & key, auto & mapped) + { + method.insertKeyIntoColumns(key, key_columns, key_sizes); + insertAggregatesIntoColumns(mapped, final_aggregate_columns, arena); + }); + } } template @@ -1152,16 +1164,33 @@ void NO_INLINE Aggregator::convertToBlockImplNotFinal( } } - data.forEachValue([&](const auto & key, auto & mapped) + if constexpr (Method::fixed_keys) { - method.insertKeyIntoColumns(key, key_columns, key_sizes); + auto new_key_columns = method.shuffleKeyColumns(key_columns, key_sizes); + data.forEachValue([&](const auto & key, auto & mapped) + { + method.insertKeyIntoColumns(key, new_key_columns.first, new_key_columns.second); - /// reserved, so push_back does not throw exceptions - for (size_t i = 0; i < params.aggregates_size; ++i) - aggregate_columns[i]->push_back(mapped + offsets_of_aggregate_states[i]); + /// reserved, so push_back does not throw exceptions + for (size_t i = 0; i < params.aggregates_size; ++i) + aggregate_columns[i]->push_back(mapped + offsets_of_aggregate_states[i]); - mapped = nullptr; - }); + mapped = nullptr; + }); + } + else + { + data.forEachValue([&](const auto & key, auto & mapped) + { + method.insertKeyIntoColumns(key, key_columns, key_sizes); + + /// reserved, so push_back does not throw exceptions + for (size_t i = 0; i < params.aggregates_size; ++i) + aggregate_columns[i]->push_back(mapped + offsets_of_aggregate_states[i]); + + mapped = nullptr; + }); + } } diff --git a/src/Interpreters/Aggregator.h b/src/Interpreters/Aggregator.h index c5bcc1eb27f..abeb80171f4 100644 --- a/src/Interpreters/Aggregator.h +++ b/src/Interpreters/Aggregator.h @@ -195,6 +195,7 @@ struct AggregationMethodOneNumber /// Use optimization for low cardinality. static const bool low_cardinality_optimization = false; + static constexpr bool fixed_keys = false; // Insert the key from the hash table into columns. static void insertKeyIntoColumns(const Key & key, MutableColumns & key_columns, const Sizes & /*key_sizes*/) @@ -224,6 +225,7 @@ struct AggregationMethodString using State = ColumnsHashing::HashMethodString; static const bool low_cardinality_optimization = false; + static constexpr bool fixed_keys = false; static void insertKeyIntoColumns(const StringRef & key, MutableColumns & key_columns, const Sizes &) { @@ -250,6 +252,7 @@ struct AggregationMethodStringNoCache using State = ColumnsHashing::HashMethodString; static const bool low_cardinality_optimization = false; + static constexpr bool fixed_keys = false; static void insertKeyIntoColumns(const StringRef & key, MutableColumns & key_columns, const Sizes &) { @@ -276,6 +279,7 @@ struct AggregationMethodFixedString using State = ColumnsHashing::HashMethodFixedString; static const bool low_cardinality_optimization = false; + static constexpr bool fixed_keys = false; static void insertKeyIntoColumns(const StringRef & key, MutableColumns & key_columns, const Sizes &) { @@ -301,6 +305,7 @@ struct AggregationMethodFixedStringNoCache using State = ColumnsHashing::HashMethodFixedString; static const bool low_cardinality_optimization = false; + static constexpr bool fixed_keys = false; static void insertKeyIntoColumns(const StringRef & key, MutableColumns & key_columns, const Sizes &) { @@ -330,6 +335,7 @@ struct AggregationMethodSingleLowCardinalityColumn : public SingleColumnMethod using State = ColumnsHashing::HashMethodSingleLowCardinalityColumn; static const bool low_cardinality_optimization = true; + static constexpr bool fixed_keys = false; static void insertKeyIntoColumns(const Key & key, MutableColumns & key_columns_low_cardinality, const Sizes & /*key_sizes*/) @@ -374,8 +380,43 @@ struct AggregationMethodKeysFixed use_cache>; static const bool low_cardinality_optimization = false; + static constexpr bool fixed_keys = true; - static void insertKeyIntoColumns(const Key & key, MutableColumns & key_columns, const Sizes & key_sizes) + std::pair, Sizes> shuffleKeyColumns(MutableColumns & key_columns, const Sizes & key_sizes) + { + std::vector new_columns; + new_columns.reserve(key_columns.size()); + + if constexpr (!has_low_cardinality && !has_nullable_keys && sizeof(Key) <= 16) + { + Sizes new_sizes; + auto fill_size = [&](size_t size) + { + for (size_t i = 0; i < key_sizes.size(); ++i) + { + if (key_sizes[i] == size) + { + new_columns.push_back(key_columns[i].get()); + new_sizes.push_back(size); + } + } + }; + + fill_size(8); + fill_size(4); + fill_size(2); + fill_size(1); + + return {new_columns, new_sizes}; + } + + for (auto & column : key_columns) + new_columns.push_back(column.get()); + + return {new_columns, key_sizes}; + } + + static void insertKeyIntoColumns(const Key & key, std::vector & key_columns, const Sizes & key_sizes) { size_t keys_size = key_columns.size(); @@ -401,7 +442,7 @@ struct AggregationMethodKeysFixed } else { - observed_column = key_columns[i].get(); + observed_column = key_columns[i]; null_map = nullptr; } @@ -452,6 +493,7 @@ struct AggregationMethodSerialized using State = ColumnsHashing::HashMethodSerialized; static const bool low_cardinality_optimization = false; + static constexpr bool fixed_keys = false; static void insertKeyIntoColumns(const StringRef & key, MutableColumns & key_columns, const Sizes &) { From c891cf4557efbf10cc0312a716b756acb927d74c Mon Sep 17 00:00:00 2001 From: Anton Ivashkin Date: Fri, 26 Feb 2021 12:48:57 +0300 Subject: [PATCH 054/544] Fixes by review response --- src/Common/ErrorCodes.cpp | 1 + src/Disks/DiskType.h | 32 +++ src/Disks/IDisk.h | 35 +-- src/Disks/IStoragePolicy.h | 5 +- src/Disks/S3/DiskS3.h | 4 + src/Disks/StoragePolicy.cpp | 4 +- src/Disks/StoragePolicy.h | 2 +- src/Interpreters/InterserverIOHandler.h | 2 - src/Storages/MergeTree/DataPartsExchange.cpp | 13 +- src/Storages/MergeTree/DataPartsExchange.h | 20 -- src/Storages/MergeTree/IMergeTreeDataPart.cpp | 237 +--------------- src/Storages/MergeTree/IMergeTreeDataPart.h | 15 - src/Storages/MergeTree/MergeTreeData.h | 15 +- .../MergeTree/MergeTreeDataMergerMutator.cpp | 3 +- .../MergeTree/MergedBlockOutputStream.cpp | 2 +- src/Storages/StorageReplicatedMergeTree.cpp | 267 +++++++++++++++--- src/Storages/StorageReplicatedMergeTree.h | 28 +- .../configs/config.d/s3.xml | 2 +- .../test_s3_zero_copy_replication/test.py | 8 +- 19 files changed, 330 insertions(+), 365 deletions(-) create mode 100644 src/Disks/DiskType.h diff --git a/src/Common/ErrorCodes.cpp b/src/Common/ErrorCodes.cpp index 09e5945f2b5..8dd05615a19 100644 --- a/src/Common/ErrorCodes.cpp +++ b/src/Common/ErrorCodes.cpp @@ -534,6 +534,7 @@ M(565, TOO_MANY_PARTITIONS) \ M(566, CANNOT_RMDIR) \ M(567, DUPLICATED_PART_UUIDS) \ + M(568, INCORRECT_PART_TYPE) \ \ M(999, KEEPER_EXCEPTION) \ M(1000, POCO_EXCEPTION) \ diff --git a/src/Disks/DiskType.h b/src/Disks/DiskType.h new file mode 100644 index 00000000000..4e0ae226af4 --- /dev/null +++ b/src/Disks/DiskType.h @@ -0,0 +1,32 @@ +#pragma once + +#include + +namespace DB +{ + +struct DiskType +{ + enum class Type + { + Local, + RAM, + S3 + }; + static String toString(Type disk_type) + { + switch (disk_type) + { + case Type::Local: + return "local"; + case Type::RAM: + return "memory"; + case Type::S3: + return "s3"; + } + __builtin_unreachable(); + } +}; + +} + diff --git a/src/Disks/IDisk.h b/src/Disks/IDisk.h index ac9425a8b3a..44c4fe73d37 100644 --- a/src/Disks/IDisk.h +++ b/src/Disks/IDisk.h @@ -5,6 +5,7 @@ #include #include #include +#include #include #include @@ -57,29 +58,6 @@ public: using SpacePtr = std::shared_ptr; -struct DiskType -{ - enum class Type - { - Local, - RAM, - S3 - }; - static String toString(Type disk_type) - { - switch (disk_type) - { - case Type::Local: - return "local"; - case Type::RAM: - return "memory"; - case Type::S3: - return "s3"; - } - __builtin_unreachable(); - } -}; - /** * A guard, that should synchronize file's or directory's state * with storage device (e.g. fsync in POSIX) in its destructor. @@ -196,12 +174,18 @@ public: virtual void removeRecursive(const String & path) = 0; /// Remove file. Throws exception if file doesn't exists or if directory is not empty. + /// Differs from removeFile for S3 disks + /// Second bool param is a flag to remove (true) or keep (false) shared data on S3 virtual void removeSharedFile(const String & path, bool) { removeFile(path); } /// Remove file or directory with all children. Use with extra caution. Throws exception if file doesn't exists. + /// Differs from removeRecursive for S3 disks + /// Second bool param is a flag to remove (true) or keep (false) shared data on S3 virtual void removeSharedRecursive(const String & path, bool) { removeRecursive(path); } /// Remove file or directory if it exists. + /// Differs from removeFileIfExists for S3 disks + /// Second bool param is a flag to remove (true) or keep (false) shared data on S3 virtual void removeSharedFileIfExists(const String & path, bool) { removeFileIfExists(path); } /// Set last modified time to file or directory at `path`. @@ -226,9 +210,12 @@ public: virtual void shutdown() { } /// Return some uniq string for file, overrode for S3 + /// Required for distinguish different copies of the same part on S3 virtual String getUniqueId(const String & path) const { return path; } - /// Check file, overrode for S3 only + /// Check file exists and ClickHouse has an access to it + /// Overrode in DiskS3 + /// Required for S3 to ensure that replica has access to data wroten by other node virtual bool checkUniqueId(const String & id) const { return exists(id); } /// Returns executor to perform asynchronous operations. diff --git a/src/Disks/IStoragePolicy.h b/src/Disks/IStoragePolicy.h index 957021441b8..59cff3c85d5 100644 --- a/src/Disks/IStoragePolicy.h +++ b/src/Disks/IStoragePolicy.h @@ -1,4 +1,7 @@ #pragma once + +#include + #include #include #include @@ -36,7 +39,7 @@ public: /// mutations files virtual DiskPtr getAnyDisk() const = 0; virtual DiskPtr getDiskByName(const String & disk_name) const = 0; - virtual Disks getDisksByType(const String & type) const = 0; + virtual Disks getDisksByType(DiskType::Type type) const = 0; /// Get free space from most free disk virtual UInt64 getMaxUnreservedFreeSpace() const = 0; /// Reserves space on any volume with index > min_volume_index or returns nullptr diff --git a/src/Disks/S3/DiskS3.h b/src/Disks/S3/DiskS3.h index 4eef1512752..5d9effa16fa 100644 --- a/src/Disks/S3/DiskS3.h +++ b/src/Disks/S3/DiskS3.h @@ -118,8 +118,12 @@ public: void shutdown() override; + /// Return some uniq string for file + /// Required for distinguish different copies of the same part on S3 String getUniqueId(const String & path) const override; + /// Check file exists and ClickHouse has an access to it + /// Required for S3 to ensure that replica has access to data wroten by other node bool checkUniqueId(const String & id) const override; /// Actions performed after disk creation. diff --git a/src/Disks/StoragePolicy.cpp b/src/Disks/StoragePolicy.cpp index be40a5ae72d..cff2685ca24 100644 --- a/src/Disks/StoragePolicy.cpp +++ b/src/Disks/StoragePolicy.cpp @@ -159,12 +159,12 @@ Disks StoragePolicy::getDisks() const } -Disks StoragePolicy::getDisksByType(const String & type) const +Disks StoragePolicy::getDisksByType(DiskType::Type type) const { Disks res; for (const auto & volume : volumes) for (const auto & disk : volume->getDisks()) - if (DB::DiskType::toString(disk->getType()) == type) + if (disk->getType() == type) res.push_back(disk); return res; } diff --git a/src/Disks/StoragePolicy.h b/src/Disks/StoragePolicy.h index 7e72fcda8b1..71773e91f70 100644 --- a/src/Disks/StoragePolicy.h +++ b/src/Disks/StoragePolicy.h @@ -48,7 +48,7 @@ public: Disks getDisks() const override; /// Returns disks by type ordered by volumes priority - Disks getDisksByType(const String & type) const override; + Disks getDisksByType(DiskType::Type type) const override; /// Returns any disk /// Used when it's not important, for example for diff --git a/src/Interpreters/InterserverIOHandler.h b/src/Interpreters/InterserverIOHandler.h index bcb0e8736f0..f8b0f4d1247 100644 --- a/src/Interpreters/InterserverIOHandler.h +++ b/src/Interpreters/InterserverIOHandler.h @@ -40,8 +40,6 @@ public: virtual void processQuery(const Poco::Net::HTMLForm & params, ReadBuffer & body, WriteBuffer & out, Poco::Net::HTTPServerResponse & response) = 0; virtual ~InterserverIOEndpoint() = default; - virtual void setZooKeeper(const zkutil::ZooKeeperPtr &zookeeper_, const String & zookeeper_path_, const String & replica_name_) = 0; - /// You need to stop the data transfer if blocker is activated. ActionBlocker blocker; std::shared_mutex rwlock; diff --git a/src/Storages/MergeTree/DataPartsExchange.cpp b/src/Storages/MergeTree/DataPartsExchange.cpp index d031989bfcd..511cc82a870 100644 --- a/src/Storages/MergeTree/DataPartsExchange.cpp +++ b/src/Storages/MergeTree/DataPartsExchange.cpp @@ -35,6 +35,7 @@ namespace ErrorCodes extern const int CORRUPTED_DATA; extern const int LOGICAL_ERROR; extern const int S3_ERROR; + extern const int INCORRECT_PART_TYPE; } namespace DataPartsExchange @@ -265,7 +266,7 @@ void Service::sendPartS3Metadata(const MergeTreeData::DataPartPtr & part, WriteB if (disk->getType() != DB::DiskType::Type::S3) throw Exception("S3 disk is not S3 anymore", ErrorCodes::LOGICAL_ERROR); - part->lockSharedData(); + part->storage.lockSharedData(*part); String part_id = part->getUniqueId(); writeStringBinary(part_id, out); @@ -280,9 +281,9 @@ void Service::sendPartS3Metadata(const MergeTreeData::DataPartPtr & part, WriteB Poco::File metadata(metadata_file); if (!metadata.exists()) - throw Exception("S3 metadata '" + file_name + "' is not exists", ErrorCodes::LOGICAL_ERROR); + throw Exception("S3 metadata '" + file_name + "' is not exists", ErrorCodes::CORRUPTED_DATA); if (!metadata.isFile()) - throw Exception("S3 metadata '" + file_name + "' is not a file", ErrorCodes::LOGICAL_ERROR); + throw Exception("S3 metadata '" + file_name + "' is not a file", ErrorCodes::CORRUPTED_DATA); UInt64 file_size = metadata.getSize(); writeStringBinary(it.first, out); @@ -361,7 +362,7 @@ MergeTreeData::MutableDataPartPtr Fetcher::fetchPart( disks_s3.push_back(disk_s3); else { - disks_s3 = data.getDisksByType("s3"); + disks_s3 = data.getDisksByType(DiskType::Type::S3); if (disks_s3.empty()) try_use_s3_copy = false; @@ -411,7 +412,7 @@ MergeTreeData::MutableDataPartPtr Fetcher::fetchPart( String part_type = "Wide"; readStringBinary(part_type, in); if (part_type == "InMemory") - throw Exception("Got 'send_s3_metadata' cookie for in-memory partition", ErrorCodes::LOGICAL_ERROR); + throw Exception("Got 'send_s3_metadata' cookie for in-memory part", ErrorCodes::INCORRECT_PART_TYPE); UUID part_uuid = UUIDHelpers::Nil; if (server_protocol_version >= REPLICATION_PROTOCOL_VERSION_WITH_PARTS_UUID) @@ -692,7 +693,7 @@ MergeTreeData::MutableDataPartPtr Fetcher::downloadPartToS3( new_data_part->modification_time = time(nullptr); new_data_part->loadColumnsChecksumsIndexes(true, false); - new_data_part->lockSharedData(); + new_data_part->storage.lockSharedData(*new_data_part); return new_data_part; } diff --git a/src/Storages/MergeTree/DataPartsExchange.h b/src/Storages/MergeTree/DataPartsExchange.h index f0297aa1d28..1fcee1242e3 100644 --- a/src/Storages/MergeTree/DataPartsExchange.h +++ b/src/Storages/MergeTree/DataPartsExchange.h @@ -35,13 +35,6 @@ public: std::string getId(const std::string & node_id) const override; void processQuery(const Poco::Net::HTMLForm & params, ReadBuffer & body, WriteBuffer & out, Poco::Net::HTTPServerResponse & response) override; - void setZooKeeper(const zkutil::ZooKeeperPtr & zookeeper_, const String & zookeeper_path_, const String & replica_name_) override - { - zookeeper = zookeeper_; - zookeeper_path = zookeeper_path_; - replica_name = replica_name_; - } - private: MergeTreeData::DataPartPtr findPart(const String & name); void sendPartFromMemory(const MergeTreeData::DataPartPtr & part, WriteBuffer & out); @@ -53,9 +46,6 @@ private: /// so Service will never access dangling reference to storage MergeTreeData & data; Poco::Logger * log; - zkutil::ZooKeeperPtr zookeeper; - String zookeeper_path; - String replica_name; }; /** Client for getting the parts from the table *MergeTree. @@ -87,13 +77,6 @@ public: /// You need to stop the data transfer. ActionBlocker blocker; - void setZooKeeper(const zkutil::ZooKeeperPtr & zookeeper_, const String & zookeeper_path_, const String & replica_name_) - { - zookeeper = zookeeper_; - zookeeper_path = zookeeper_path_; - replica_name = replica_name_; - } - private: MergeTreeData::MutableDataPartPtr downloadPartToDisk( const String & part_name, @@ -121,9 +104,6 @@ private: MergeTreeData & data; Poco::Logger * log; - zkutil::ZooKeeperPtr zookeeper; - String zookeeper_path; - String replica_name; }; } diff --git a/src/Storages/MergeTree/IMergeTreeDataPart.cpp b/src/Storages/MergeTree/IMergeTreeDataPart.cpp index 5c35a8d0af3..03dbac0cb68 100644 --- a/src/Storages/MergeTree/IMergeTreeDataPart.cpp +++ b/src/Storages/MergeTree/IMergeTreeDataPart.cpp @@ -1010,8 +1010,8 @@ void IMergeTreeDataPart::renameTo(const String & new_relative_path, bool remove_ if (storage.getSettings()->fsync_part_directory) sync_guard = volume->getDisk()->getDirectorySyncGuard(to); - lockSharedData(); - unlockSharedData(old_relative_path); + storage.lockSharedData(*this); + storage.unlockSharedData(*this, old_relative_path); } @@ -1166,15 +1166,7 @@ void IMergeTreeDataPart::makeCloneOnDisk(const DiskPtr & disk, const String & di } disk->createDirectories(path_to_clone); - bool is_fetched = false; - - if (disk->getType() == DB::DiskType::Type::S3) - { - auto data_settings = storage.getSettings(); - if (data_settings->allow_s3_zero_copy_replication) - is_fetched = tryToFetchIfShared(disk, path_to_clone + "/" + name); - } - + bool is_fetched = storage.tryToFetchIfShared(*this, disk, path_to_clone + "/" + name); if (!is_fetched) volume->getDisk()->copy(getFullRelativePath(), disk, path_to_clone); volume->getDisk()->removeFileIfExists(path_to_clone + '/' + DELETE_ON_DESTROY_MARKER_FILE_NAME); @@ -1326,229 +1318,6 @@ String IMergeTreeDataPart::getUniqueId() const return id; } -void IMergeTreeDataPart::lockSharedData() const -{ - if (!volume) - return; - DiskPtr disk = volume->getDisk(); - if (!disk) - return; - if (disk->getType() != DB::DiskType::Type::S3) - return; - - const StorageReplicatedMergeTree *replicated_storage = dynamic_cast(&storage); - if (!replicated_storage) - return; - - StorageReplicatedMergeTree::ZooKeeperAccessData zk = replicated_storage->getZooKeeperAccessData(); - if (!zk.zookeeper) - return; - - String id = getUniqueId(); - boost::replace_all(id, "/", "_"); - String norm_path = relative_path; - boost::replace_all(norm_path, "/", "_"); - - String zookeeper_node = zk.zookeeper_path + "/zero_copy_s3/shared/" + name + "/" + id + "/" + norm_path + "/" + zk.replica_name; - - LOG_TRACE(storage.log, "Set zookeeper lock {}", zookeeper_node); - - /// In rare case other replica can remove path between createAncestors and createIfNotExists - /// So we make up to 5 attempts - for (int attempts = 5; attempts > 0; --attempts) - { - try - { - zk.zookeeper->createAncestors(zookeeper_node); - zk.zookeeper->createIfNotExists(zookeeper_node, "lock"); - break; - } - catch (const zkutil::KeeperException & e) - { - if (e.code == Coordination::Error::ZNONODE) - continue; - throw; - } - } -} - -bool IMergeTreeDataPart::unlockSharedData() const -{ - return unlockSharedData(relative_path); -} - -bool IMergeTreeDataPart::unlockSharedData(const String & path) const -{ - if (!volume) - return true; - DiskPtr disk = volume->getDisk(); - if (!disk) - return true; - if (disk->getType() != DB::DiskType::Type::S3) - return true; - - const StorageReplicatedMergeTree *replicated_storage = dynamic_cast(&storage); - if (!replicated_storage) - return true; - - StorageReplicatedMergeTree::ZooKeeperAccessData zk = replicated_storage->getZooKeeperAccessData(); - if (!zk.zookeeper) - return true; - - String id = getUniqueId(); - boost::replace_all(id, "/", "_"); - String norm_path = path; - boost::replace_all(norm_path, "/", "_"); - - String zookeeper_part_node = zk.zookeeper_path + "/zero_copy_s3/shared/" + name; - String zookeeper_part_uniq_node = zookeeper_part_node + "/" + id; - String zookeeper_part_path_node = zookeeper_part_uniq_node + "/" + norm_path; - String zookeeper_node = zookeeper_part_path_node + "/" + zk.replica_name; - - LOG_TRACE(storage.log, "Remove zookeeper lock {}", zookeeper_node); - - zk.zookeeper->tryRemove(zookeeper_node); - - Strings children; - zk.zookeeper->tryGetChildren(zookeeper_part_path_node, children); - if (!children.empty()) - { - LOG_TRACE(storage.log, "Found zookeper locks for {}", zookeeper_part_path_node); - return false; - } - - zk.zookeeper->tryRemove(zookeeper_part_path_node); - - children.clear(); - zk.zookeeper->tryGetChildren(zookeeper_part_uniq_node, children); - - if (!children.empty()) - { - LOG_TRACE(storage.log, "Found zookeper locks for {}", zookeeper_part_uniq_node); - return false; - } - - zk.zookeeper->tryRemove(zookeeper_part_uniq_node); - - /// Even when we have lock with same part name, but with different uniq, we can remove files on S3 - children.clear(); - zk.zookeeper->tryGetChildren(zookeeper_part_node, children); - if (children.empty()) - /// Cleanup after last uniq removing - zk.zookeeper->tryRemove(zookeeper_part_node); - - return true; -} - -String IMergeTreeDataPart::getSharedDataReplica( - const String & zookeeper_path, - zkutil::ZooKeeperPtr zookeeper, - const String & replica_name) const -{ - String norm_path = relative_path; - boost::replace_all(norm_path, "/", "_"); - String zookeeper_part_node = zookeeper_path + "/zero_copy_s3/shared/" + name; - - Strings ids; - zookeeper->tryGetChildren(zookeeper_part_node, ids); - - Strings replicas; - for (const auto & id : ids) - { - String zookeeper_part_uniq_node = zookeeper_part_node + "/" + id; - Strings paths; - zookeeper->tryGetChildren(zookeeper_part_uniq_node, paths); - for (const auto &path : paths) - { - String zookeeper_node = zookeeper_part_uniq_node + "/" + path; - Strings id_replicas; - zookeeper->tryGetChildren(zookeeper_node, id_replicas); - LOG_TRACE(storage.log, "Found zookeper replicas for {}: {}", zookeeper_node, id_replicas.size()); - replicas.insert(replicas.end(), id_replicas.begin(), id_replicas.end()); - } - } - - LOG_TRACE(storage.log, "Found zookeper replicas for part {}: {}", name, replicas.size()); - - String best_replica; - Strings active_replicas; - - /// TODO: Move best replica choose in common method (here is the same code as in StorageReplicatedMergeTree::fetchPartition) - - /// Leave only active replicas. - active_replicas.reserve(replicas.size()); - - for (const String & replica : replicas) - if ((replica != replica_name) && (zookeeper->exists(zookeeper_path + "/replicas/" + replica + "/is_active"))) - active_replicas.push_back(replica); - - LOG_TRACE(storage.log, "Found zookeper active replicas for part {}: {}", name, active_replicas.size()); - - if (active_replicas.empty()) - return best_replica; - - /** You must select the best (most relevant) replica. - * This is a replica with the maximum `log_pointer`, then with the minimum `queue` size. - * NOTE This is not exactly the best criteria. It does not make sense to download old partitions, - * and it would be nice to be able to choose the replica closest by network. - * NOTE Of course, there are data races here. You can solve it by retrying. - */ - Int64 max_log_pointer = -1; - UInt64 min_queue_size = std::numeric_limits::max(); - - for (const String & replica : active_replicas) - { - String current_replica_path = zookeeper_path + "/replicas/" + replica; - - String log_pointer_str = zookeeper->get(current_replica_path + "/log_pointer"); - Int64 log_pointer = log_pointer_str.empty() ? 0 : parse(log_pointer_str); - - Coordination::Stat stat; - zookeeper->get(current_replica_path + "/queue", &stat); - size_t queue_size = stat.numChildren; - - if (log_pointer > max_log_pointer - || (log_pointer == max_log_pointer && queue_size < min_queue_size)) - { - max_log_pointer = log_pointer; - min_queue_size = queue_size; - best_replica = replica; - } - } - - return best_replica; -} - -bool IMergeTreeDataPart::tryToFetchIfShared(const DiskPtr & disk, const String & path) const -{ - const StorageReplicatedMergeTree *replicated_storage = dynamic_cast(&storage); - if (!replicated_storage) - return false; - - StorageReplicatedMergeTree::ZooKeeperAccessData zk = replicated_storage->getZooKeeperAccessData(); - if (!zk.zookeeper) - return false; - - String replica = getSharedDataReplica(zk.zookeeper_path, zk.zookeeper, zk.replica_name); - - /// We can't fetch part when none replicas have this part on S3 - if (replica.empty()) - return false; - - ReplicatedMergeTreeLogEntry log_entry; - log_entry.type = ReplicatedMergeTreeLogEntry::FETCH_SHARED_PART; - log_entry.source_replica = replica; - log_entry.new_part_name = name;//part_name; - log_entry.create_time = 0;//part_create_time; - log_entry.disk = disk; - log_entry.path = path; - - /// TODO: Fix const usage - StorageReplicatedMergeTree *replicated_storage_nc = const_cast(replicated_storage); - - return replicated_storage_nc->executeFetchShared(log_entry); -} - bool isCompactPart(const MergeTreeDataPartPtr & data_part) { return (data_part && data_part->getType() == MergeTreeDataPartType::COMPACT); diff --git a/src/Storages/MergeTree/IMergeTreeDataPart.h b/src/Storages/MergeTree/IMergeTreeDataPart.h index 2c54cf4e096..f9b5f616f70 100644 --- a/src/Storages/MergeTree/IMergeTreeDataPart.h +++ b/src/Storages/MergeTree/IMergeTreeDataPart.h @@ -369,15 +369,6 @@ public: String getUniqueId() const; - /// Lock part in zookeeper for use common S3 data in several nodes - void lockSharedData() const; - - /// Unlock common S3 data part in zookeeper - /// Return true if data unlocked - /// Return false if data is still used by another node - bool unlockSharedData() const; - bool unlockSharedData(const String & path) const; - protected: /// Total size of all columns, calculated once in calcuateColumnSizesOnDisk @@ -442,12 +433,6 @@ private: /// for this column with default parameters. CompressionCodecPtr detectDefaultCompressionCodec() const; - /// Fetch part only if some replica has it on shared storage like S3 - bool tryToFetchIfShared(const DiskPtr & disk, const String & path) const; - - /// Get best replica having this partition on S3 - String getSharedDataReplica(const String & zookeeper_path, zkutil::ZooKeeperPtr zookeeper, const String & replica_name) const; - mutable State state{State::Temporary}; }; diff --git a/src/Storages/MergeTree/MergeTreeData.h b/src/Storages/MergeTree/MergeTreeData.h index c99ad621c88..916cdeaf692 100644 --- a/src/Storages/MergeTree/MergeTreeData.h +++ b/src/Storages/MergeTree/MergeTreeData.h @@ -662,7 +662,7 @@ public: /// Reserves 0 bytes ReservationPtr makeEmptyReservationOnLargestDisk() { return getStoragePolicy()->makeEmptyReservationOnLargestDisk(); } - Disks getDisksByType(const String & type) const { return getStoragePolicy()->getDisksByType(type); } + Disks getDisksByType(DiskType::Type type) const { return getStoragePolicy()->getDisksByType(type); } /// Return alter conversions for part which must be applied on fly. AlterConversions getAlterConversionsForPart(const MergeTreeDataPartPtr part) const; @@ -735,6 +735,19 @@ public: std::optional getDataMovingJob(); bool areBackgroundMovesNeeded() const; + /// Lock part in zookeeper for use common S3 data in several nodes + /// Overrided in StorageReplicatedMergeTree + virtual void lockSharedData(const IMergeTreeDataPart &) const {} + + /// Unlock common S3 data part in zookeeper + /// Overrided in StorageReplicatedMergeTree + virtual bool unlockSharedData(const IMergeTreeDataPart &) const { return true; } + virtual bool unlockSharedData(const IMergeTreeDataPart &, const String &) const { return true; } + + /// Fetch part only if some replica has it on shared storage like S3 + /// Overrided in StorageReplicatedMergeTree + virtual bool tryToFetchIfShared(const IMergeTreeDataPart &, const DiskPtr &, const String &) const { return false; } + protected: friend class IMergeTreeDataPart; diff --git a/src/Storages/MergeTree/MergeTreeDataMergerMutator.cpp b/src/Storages/MergeTree/MergeTreeDataMergerMutator.cpp index 1e870486314..a4b0e49e1a9 100644 --- a/src/Storages/MergeTree/MergeTreeDataMergerMutator.cpp +++ b/src/Storages/MergeTree/MergeTreeDataMergerMutator.cpp @@ -1894,10 +1894,9 @@ void MergeTreeDataMergerMutator::finalizeMutatedPart( MergeTreeData::DataPart::calculateTotalSizeOnDisk(new_data_part->volume->getDisk(), new_data_part->getFullRelativePath())); new_data_part->default_codec = codec; new_data_part->calculateColumnsSizesOnDisk(); - new_data_part->lockSharedData(); + new_data_part->storage.lockSharedData(*new_data_part); } - bool MergeTreeDataMergerMutator::checkOperationIsNotCanceled(const MergeListEntry & merge_entry) const { if (merges_blocker.isCancelled() || merge_entry->is_cancelled) diff --git a/src/Storages/MergeTree/MergedBlockOutputStream.cpp b/src/Storages/MergeTree/MergedBlockOutputStream.cpp index 945c047f30e..1b852622efc 100644 --- a/src/Storages/MergeTree/MergedBlockOutputStream.cpp +++ b/src/Storages/MergeTree/MergedBlockOutputStream.cpp @@ -91,7 +91,7 @@ void MergedBlockOutputStream::writeSuffixAndFinalizePart( new_part->calculateColumnsSizesOnDisk(); if (default_codec != nullptr) new_part->default_codec = default_codec; - new_part->lockSharedData(); + new_part->storage.lockSharedData(*new_part); } void MergedBlockOutputStream::finalizePartOnDisk( diff --git a/src/Storages/StorageReplicatedMergeTree.cpp b/src/Storages/StorageReplicatedMergeTree.cpp index 25d379e2960..ec2242291cd 100644 --- a/src/Storages/StorageReplicatedMergeTree.cpp +++ b/src/Storages/StorageReplicatedMergeTree.cpp @@ -421,8 +421,6 @@ StorageReplicatedMergeTree::StorageReplicatedMergeTree( } createNewZooKeeperNodes(); - - fetcher.setZooKeeper(current_zookeeper, zookeeper_path, replica_name); } @@ -1491,6 +1489,7 @@ bool StorageReplicatedMergeTree::tryExecuteMerge(const LogEntry & entry) future_merged_part.updatePath(*this, reserved_space); future_merged_part.merge_type = entry.merge_type; + if (storage_settings_ptr->allow_s3_zero_copy_replication) { auto disk = reserved_space->getDisk(); if (disk->getType() == DB::DiskType::Type::S3) @@ -3228,30 +3227,6 @@ String StorageReplicatedMergeTree::findReplicaHavingPart(const String & part_nam return {}; } -String StorageReplicatedMergeTree::findReplicaHavingSharedPart(const String & part_name, bool active) -{ - auto zookeeper = getZooKeeper(); - Strings replicas = zookeeper->getChildren(zookeeper_path + "/replicas"); - - /// Select replicas in uniformly random order. - std::shuffle(replicas.begin(), replicas.end(), thread_local_rng); - - for (const String & replica : replicas) - { - /// We don't interested in ourself. - if (replica == replica_name) - continue; - - if (checkReplicaHavePart(replica, part_name) && - (!active || zookeeper->exists(zookeeper_path + "/replicas/" + replica + "/is_active"))) - return replica; - - /// Obviously, replica could become inactive or even vanish after return from this method. - } - - return {}; -} - String StorageReplicatedMergeTree::findReplicaHavingCoveringPart(LogEntry & entry, bool active) { auto zookeeper = getZooKeeper(); @@ -3668,11 +3643,6 @@ bool StorageReplicatedMergeTree::fetchPart(const String & part_name, const Stora Transaction transaction(*this); renameTempPartAndReplace(part, nullptr, &transaction); - /** NOTE - * Here, an error occurs if ALTER occurred with a change in the column type or column deletion, - * and the part on remote server has not yet been modified. - * After a while, one of the following attempts to make `fetchPart` succeed. - */ replaced_parts = checkPartChecksumsAndCommit(transaction, part); /** If a quorum is tracked for this part, you must update it. @@ -3757,7 +3727,6 @@ void StorageReplicatedMergeTree::startup() InterserverIOEndpointPtr data_parts_exchange_ptr = std::make_shared(*this); [[maybe_unused]] auto prev_ptr = std::atomic_exchange(&data_parts_exchange_endpoint, data_parts_exchange_ptr); assert(prev_ptr == nullptr); - data_parts_exchange_ptr->setZooKeeper(tryGetZooKeeper(), zookeeper_path, replica_name); global_context.getInterserverIOHandler().addEndpoint(data_parts_exchange_ptr->getId(replica_path), data_parts_exchange_ptr); /// In this thread replica will be activated. @@ -5464,13 +5433,13 @@ void StorageReplicatedMergeTree::clearOldPartsAndRemoveFromZK() } parts.clear(); - auto remove_parts_from_filesystem = [log=log] (const DataPartsVector & parts_to_remove) + auto remove_parts_from_filesystem = [log=log, this] (const DataPartsVector & parts_to_remove) { for (const auto & part : parts_to_remove) { try { - bool keep_s3 = !part->unlockSharedData(); + bool keep_s3 = !this->unlockSharedData(*part); part->remove(keep_s3); } catch (...) @@ -6405,6 +6374,7 @@ CheckResults StorageReplicatedMergeTree::checkData(const ASTPtr & query, const C return results; } + bool StorageReplicatedMergeTree::canUseAdaptiveGranularity() const { const auto storage_settings_ptr = getSettings(); @@ -6419,19 +6389,236 @@ MutationCommands StorageReplicatedMergeTree::getFirstAlterMutationCommandsForPar return queue.getFirstAlterMutationCommandsForPart(part); } + void StorageReplicatedMergeTree::startBackgroundMovesIfNeeded() { if (areBackgroundMovesNeeded()) background_moves_executor.start(); } -StorageReplicatedMergeTree::ZooKeeperAccessData StorageReplicatedMergeTree::getZooKeeperAccessData() const + +void StorageReplicatedMergeTree::lockSharedData(const IMergeTreeDataPart & part) const { - ZooKeeperAccessData res; - res.zookeeper = tryGetZooKeeper(); - res.zookeeper_path = zookeeper_path; - res.replica_name = replica_name; - return res; + if (!part.volume) + return; + DiskPtr disk = part.volume->getDisk(); + if (!disk) + return; + if (disk->getType() != DB::DiskType::Type::S3) + return; + + zkutil::ZooKeeperPtr zookeeper = tryGetZooKeeper(); + if (!zookeeper) + return; + + String id = part.getUniqueId(); + boost::replace_all(id, "/", "_"); + String norm_path = part.relative_path; + boost::replace_all(norm_path, "/", "_"); + + String zookeeper_node = zookeeper_path + "/zero_copy_s3/shared/" + part.name + "/" + id + "/" + norm_path + "/" + replica_name; + + LOG_TRACE(log, "Set zookeeper lock {}", zookeeper_node); + + /// In rare case other replica can remove path between createAncestors and createIfNotExists + /// So we make up to 5 attempts + for (int attempts = 5; attempts > 0; --attempts) + { + try + { + zookeeper->createAncestors(zookeeper_node); + zookeeper->createIfNotExists(zookeeper_node, "lock"); + break; + } + catch (const zkutil::KeeperException & e) + { + if (e.code == Coordination::Error::ZNONODE) + continue; + throw; + } + } +} + + +bool StorageReplicatedMergeTree::unlockSharedData(const IMergeTreeDataPart & part) const +{ + return unlockSharedData(part, part.relative_path); +} + + +bool StorageReplicatedMergeTree::unlockSharedData(const IMergeTreeDataPart & part, const String & path) const +{ + if (!part.volume) + return true; + DiskPtr disk = part.volume->getDisk(); + if (!disk) + return true; + if (disk->getType() != DB::DiskType::Type::S3) + return true; + + zkutil::ZooKeeperPtr zookeeper = tryGetZooKeeper(); + if (!zookeeper) + return true; + + String id = part.getUniqueId(); + boost::replace_all(id, "/", "_"); + String norm_path = path; + boost::replace_all(norm_path, "/", "_"); + + String zookeeper_part_node = zookeeper_path + "/zero_copy_s3/shared/" + part.name; + String zookeeper_part_uniq_node = zookeeper_part_node + "/" + id; + String zookeeper_part_path_node = zookeeper_part_uniq_node + "/" + norm_path; + String zookeeper_node = zookeeper_part_path_node + "/" + replica_name; + + LOG_TRACE(log, "Remove zookeeper lock {}", zookeeper_node); + + zookeeper->tryRemove(zookeeper_node); + + Strings children; + zookeeper->tryGetChildren(zookeeper_part_path_node, children); + if (!children.empty()) + { + LOG_TRACE(log, "Found zookeper locks for {}", zookeeper_part_path_node); + return false; + } + + zookeeper->tryRemove(zookeeper_part_path_node); + + children.clear(); + zookeeper->tryGetChildren(zookeeper_part_uniq_node, children); + + if (!children.empty()) + { + LOG_TRACE(log, "Found zookeper locks for {}", zookeeper_part_uniq_node); + return false; + } + + zookeeper->tryRemove(zookeeper_part_uniq_node); + + /// Even when we have lock with same part name, but with different uniq, we can remove files on S3 + children.clear(); + zookeeper->tryGetChildren(zookeeper_part_node, children); + if (children.empty()) + /// Cleanup after last uniq removing + zookeeper->tryRemove(zookeeper_part_node); + + return true; +} + + +bool StorageReplicatedMergeTree::tryToFetchIfShared( + const IMergeTreeDataPart & part, + const DiskPtr & disk, + const String & path) const +{ + const auto data_settings = getSettings(); + if (!data_settings->allow_s3_zero_copy_replication) + return false; + + if (disk->getType() != DB::DiskType::Type::S3) + return false; + + String replica = getSharedDataReplica(part); + + /// We can't fetch part when none replicas have this part on S3 + if (replica.empty()) + return false; + + ReplicatedMergeTreeLogEntry log_entry; + log_entry.type = ReplicatedMergeTreeLogEntry::FETCH_SHARED_PART; + log_entry.source_replica = replica; + log_entry.new_part_name = part.name; + log_entry.create_time = 0; + log_entry.disk = disk; + log_entry.path = path; + + /// TODO: Fix const usage + StorageReplicatedMergeTree * replicated_storage_nc = const_cast(this); + + return replicated_storage_nc->executeFetchShared(log_entry); +} + + +String StorageReplicatedMergeTree::getSharedDataReplica( + const IMergeTreeDataPart & part) const +{ + String best_replica; + + zkutil::ZooKeeperPtr zookeeper = tryGetZooKeeper(); + if (!zookeeper) + return best_replica; + + String norm_path = part.relative_path; + boost::replace_all(norm_path, "/", "_"); + String zookeeper_part_node = zookeeper_path + "/zero_copy_s3/shared/" + part.name; + + Strings ids; + zookeeper->tryGetChildren(zookeeper_part_node, ids); + + Strings replicas; + for (const auto & id : ids) + { + String zookeeper_part_uniq_node = zookeeper_part_node + "/" + id; + Strings paths; + zookeeper->tryGetChildren(zookeeper_part_uniq_node, paths); + for (const auto & path : paths) + { + String zookeeper_node = zookeeper_part_uniq_node + "/" + path; + Strings id_replicas; + zookeeper->tryGetChildren(zookeeper_node, id_replicas); + LOG_TRACE(log, "Found zookeper replicas for {}: {}", zookeeper_node, id_replicas.size()); + replicas.insert(replicas.end(), id_replicas.begin(), id_replicas.end()); + } + } + + LOG_TRACE(log, "Found zookeper replicas for part {}: {}", part.name, replicas.size()); + + Strings active_replicas; + + /// TODO: Move best replica choose in common method (here is the same code as in StorageReplicatedMergeTree::fetchPartition) + + /// Leave only active replicas. + active_replicas.reserve(replicas.size()); + + for (const String & replica : replicas) + if ((replica != replica_name) && (zookeeper->exists(zookeeper_path + "/replicas/" + replica + "/is_active"))) + active_replicas.push_back(replica); + + LOG_TRACE(log, "Found zookeper active replicas for part {}: {}", part.name, active_replicas.size()); + + if (active_replicas.empty()) + return best_replica; + + /** You must select the best (most relevant) replica. + * This is a replica with the maximum `log_pointer`, then with the minimum `queue` size. + * NOTE This is not exactly the best criteria. It does not make sense to download old partitions, + * and it would be nice to be able to choose the replica closest by network. + * NOTE Of course, there are data races here. You can solve it by retrying. + */ + Int64 max_log_pointer = -1; + UInt64 min_queue_size = std::numeric_limits::max(); + + for (const String & replica : active_replicas) + { + String current_replica_path = zookeeper_path + "/replicas/" + replica; + + String log_pointer_str = zookeeper->get(current_replica_path + "/log_pointer"); + Int64 log_pointer = log_pointer_str.empty() ? 0 : parse(log_pointer_str); + + Coordination::Stat stat; + zookeeper->get(current_replica_path + "/queue", &stat); + size_t queue_size = stat.numChildren; + + if (log_pointer > max_log_pointer + || (log_pointer == max_log_pointer && queue_size < min_queue_size)) + { + max_log_pointer = log_pointer; + min_queue_size = queue_size; + best_replica = replica; + } + } + + return best_replica; } } diff --git a/src/Storages/StorageReplicatedMergeTree.h b/src/Storages/StorageReplicatedMergeTree.h index 177245d68af..32e5da0d04d 100644 --- a/src/Storages/StorageReplicatedMergeTree.h +++ b/src/Storages/StorageReplicatedMergeTree.h @@ -212,18 +212,24 @@ public: /// is not overloaded bool canExecuteFetch(const ReplicatedMergeTreeLogEntry & entry, String & disable_reason) const; - struct ZooKeeperAccessData - { - zkutil::ZooKeeperPtr zookeeper; - String zookeeper_path; - String replica_name; - }; - - ZooKeeperAccessData getZooKeeperAccessData() const; - /// Fetch part only when it stored on shared storage like S3 bool executeFetchShared(ReplicatedMergeTreeLogEntry & entry); + /// Lock part in zookeeper for use common S3 data in several nodes + void lockSharedData(const IMergeTreeDataPart & part) const override; + + /// Unlock common S3 data part in zookeeper + /// Return true if data unlocked + /// Return false if data is still used by another node + bool unlockSharedData(const IMergeTreeDataPart & part) const override; + bool unlockSharedData(const IMergeTreeDataPart & part, const String & path) const override; + + /// Fetch part only if some replica has it on shared storage like S3 + bool tryToFetchIfShared(const IMergeTreeDataPart & part, const DiskPtr & disk, const String & path) const override; + + /// Get best replica having this partition on S3 + String getSharedDataReplica(const IMergeTreeDataPart & part) const; + private: /// Get a sequential consistent view of current parts. ReplicatedMergeTreeQuorumAddedParts::PartitionIdToMaxBlock getMaxAddedBlocks() const; @@ -503,10 +509,6 @@ private: */ String findReplicaHavingPart(const String & part_name, bool active); - /** Returns a replica with part on shared storage like S3. - */ - String findReplicaHavingSharedPart(const String & part_name, bool active); - bool checkReplicaHavePart(const String & replica, const String & part_name); /** Find replica having specified part or any part that covers it. diff --git a/tests/integration/test_s3_zero_copy_replication/configs/config.d/s3.xml b/tests/integration/test_s3_zero_copy_replication/configs/config.d/s3.xml index 285ade3f727..7d8492ed68c 100644 --- a/tests/integration/test_s3_zero_copy_replication/configs/config.d/s3.xml +++ b/tests/integration/test_s3_zero_copy_replication/configs/config.d/s3.xml @@ -22,7 +22,7 @@ 0 - 2 + 1 1 diff --git a/tests/integration/test_s3_zero_copy_replication/test.py b/tests/integration/test_s3_zero_copy_replication/test.py index 88d038e357b..6a7336b9090 100644 --- a/tests/integration/test_s3_zero_copy_replication/test.py +++ b/tests/integration/test_s3_zero_copy_replication/test.py @@ -76,9 +76,13 @@ def test_s3_zero_copy_replication(cluster, policy): # Based on version 20.x - after merge, two old parts and one merged assert get_large_objects_count(cluster) == 3 - time.sleep(60) - # Based on version 20.x - after cleanup - only one merged part + countdown = 60 + while countdown > 0: + if get_large_objects_count(cluster) == 1: + break + time.sleep(1) + countdown -= 1 assert get_large_objects_count(cluster) == 1 node1.query("DROP TABLE IF EXISTS s3_test NO DELAY") From 545528917fd7700be0f6c582be970dbd23feeab5 Mon Sep 17 00:00:00 2001 From: Nikolai Kochetov Date: Fri, 26 Feb 2021 13:17:00 +0300 Subject: [PATCH 055/544] Fix tests. --- src/Common/ColumnsHashing.h | 10 +++++++- src/Interpreters/AggregationCommon.h | 1 + src/Interpreters/Aggregator.h | 35 +++++++++++++++++----------- 3 files changed, 32 insertions(+), 14 deletions(-) diff --git a/src/Common/ColumnsHashing.h b/src/Common/ColumnsHashing.h index 7bc84eb3429..37cb42a8bda 100644 --- a/src/Common/ColumnsHashing.h +++ b/src/Common/ColumnsHashing.h @@ -508,7 +508,15 @@ struct HashMethodKeysFixed } if constexpr (!has_low_cardinality && !has_nullable_keys && sizeof(Key) <= 16) - packFixedBatch(keys_size, Base::getActualColumns(), key_sizes, prepared_keys); + { + bool has_unsupported_sizes = false; + for (auto size : key_sizes) + if (size != 1 && size != 2 && size != 4 && size != 8 && size != 16) + has_unsupported_sizes = true; + + if (!has_unsupported_sizes) + packFixedBatch(keys_size, Base::getActualColumns(), key_sizes, prepared_keys); + } #if defined(__SSSE3__) && !defined(MEMORY_SANITIZER) if constexpr (!has_low_cardinality && !has_nullable_keys && sizeof(Key) <= 16) diff --git a/src/Interpreters/AggregationCommon.h b/src/Interpreters/AggregationCommon.h index 8bdea536392..0e9661eaef1 100644 --- a/src/Interpreters/AggregationCommon.h +++ b/src/Interpreters/AggregationCommon.h @@ -104,6 +104,7 @@ template void packFixedBatch(size_t keys_size, const ColumnRawPtrs & key_columns, const Sizes & key_sizes, PaddedPODArray & out) { size_t offset = 0; + fillFixedBatch(keys_size, key_columns, key_sizes, out, offset); fillFixedBatch(keys_size, key_columns, key_sizes, out, offset); fillFixedBatch(keys_size, key_columns, key_sizes, out, offset); fillFixedBatch(keys_size, key_columns, key_sizes, out, offset); diff --git a/src/Interpreters/Aggregator.h b/src/Interpreters/Aggregator.h index abeb80171f4..f08b9302082 100644 --- a/src/Interpreters/Aggregator.h +++ b/src/Interpreters/Aggregator.h @@ -389,25 +389,34 @@ struct AggregationMethodKeysFixed if constexpr (!has_low_cardinality && !has_nullable_keys && sizeof(Key) <= 16) { - Sizes new_sizes; - auto fill_size = [&](size_t size) + bool has_unsupported_sizes = false; + for (auto size : key_sizes) + if (size != 1 && size != 2 && size != 4 && size != 8 && size != 16) + has_unsupported_sizes = true; + + if (!has_unsupported_sizes) { - for (size_t i = 0; i < key_sizes.size(); ++i) + Sizes new_sizes; + auto fill_size = [&](size_t size) { - if (key_sizes[i] == size) + for (size_t i = 0; i < key_sizes.size(); ++i) { - new_columns.push_back(key_columns[i].get()); - new_sizes.push_back(size); + if (key_sizes[i] == size) + { + new_columns.push_back(key_columns[i].get()); + new_sizes.push_back(size); + } } - } - }; + }; - fill_size(8); - fill_size(4); - fill_size(2); - fill_size(1); + fill_size(16); + fill_size(8); + fill_size(4); + fill_size(2); + fill_size(1); - return {new_columns, new_sizes}; + return {new_columns, new_sizes}; + } } for (auto & column : key_columns) From 422e083c966238a28a29abd8b402f2a0f57c1787 Mon Sep 17 00:00:00 2001 From: Alexander Tokmakov Date: Fri, 26 Feb 2021 14:43:58 +0300 Subject: [PATCH 056/544] move more code --- docker/test/sqlancer/Dockerfile | 1 + .../test/sqlancer/process_sqlancer_result.py | 74 ++++++++++++++ docker/test/sqlancer/run.sh | 1 + docker/test/style/Dockerfile | 14 +-- .../test/style/process_style_check_result.py | 96 +++++++++++++++++++ docker/test/style/run.sh | 9 ++ docker/test/testflows/runner/Dockerfile | 3 +- .../runner/process_testflows_result.py | 67 +++++++++++++ docker/test/unit/process_unit_tests_result.py | 2 +- tests/queries/skip_list.json | 1 + 10 files changed, 255 insertions(+), 13 deletions(-) create mode 100755 docker/test/sqlancer/process_sqlancer_result.py create mode 100755 docker/test/style/process_style_check_result.py create mode 100755 docker/test/style/run.sh create mode 100755 docker/test/testflows/runner/process_testflows_result.py diff --git a/docker/test/sqlancer/Dockerfile b/docker/test/sqlancer/Dockerfile index 38a773e65ad..32012efb064 100644 --- a/docker/test/sqlancer/Dockerfile +++ b/docker/test/sqlancer/Dockerfile @@ -10,4 +10,5 @@ RUN mkdir /sqlancer && \ RUN cd /sqlancer/sqlancer-master && mvn package -DskipTests COPY run.sh / +COPY process_sqlancer_result.py / CMD ["/bin/bash", "/run.sh"] diff --git a/docker/test/sqlancer/process_sqlancer_result.py b/docker/test/sqlancer/process_sqlancer_result.py new file mode 100755 index 00000000000..411c1e18e19 --- /dev/null +++ b/docker/test/sqlancer/process_sqlancer_result.py @@ -0,0 +1,74 @@ +#!/usr/bin/env python3 + +import os +import logging +import argparse +import csv + + +def process_result(result_folder): + status = "success" + summary = [] + paths = [] + tests = ["TLPWhere", "TLPGroupBy", "TLPHaving", "TLPWhereGroupBy", "TLPDistinct", "TLPAggregate"] + + for test in tests: + err_path = '{}/{}.err'.format(result_folder, test) + out_path = '{}/{}.out'.format(result_folder, test) + if not os.path.exists(err_path): + logging.info("No output err on path %s", err_path) + summary.append((test, "SKIPPED")) + elif not os.path.exists(out_path): + logging.info("No output log on path %s", out_path) + else: + paths.append(err_path) + paths.append(out_path) + with open(err_path, 'r') as f: + if 'AssertionError' in f.read(): + summary.append((test, "FAIL")) + else: + summary.append((test, "OK")) + + logs_path = '{}/logs.tar.gz'.format(result_folder) + if not os.path.exists(logs_path): + logging.info("No logs tar on path %s", logs_path) + else: + paths.append(logs_path) + stdout_path = '{}/stdout.log'.format(result_folder) + if not os.path.exists(stdout_path): + logging.info("No stdout log on path %s", stdout_path) + else: + paths.append(stdout_path) + stderr_path = '{}/stderr.log'.format(result_folder) + if not os.path.exists(stderr_path): + logging.info("No stderr log on path %s", stderr_path) + else: + paths.append(stderr_path) + + description = "SQLancer test run. See report" + + return status, description, summary, paths + + +def write_results(results_file, status_file, results, status): + with open(results_file, 'w') as f: + out = csv.writer(f, delimiter='\t') + out.writerows(results) + with open(status_file, 'w') as f: + out = csv.writer(f, delimiter='\t') + out.writerow(status) + + +if __name__ == "__main__": + logging.basicConfig(level=logging.INFO, format='%(asctime)s %(message)s') + parser = argparse.ArgumentParser(description="ClickHouse script for parsing results of sqlancer test") + parser.add_argument("--in-results-dir", default='/test_output/') + parser.add_argument("--out-results-file", default='/test_output/test_results.tsv') + parser.add_argument("--out-status-file", default='/test_output/check_status.tsv') + args = parser.parse_args() + + state, description, test_results, logs = process_result(args.in_results_dir) + logging.info("Result parsed") + status = (state, description) + write_results(args.out_results_file, args.out_status_file, test_results, status) + logging.info("Result written") diff --git a/docker/test/sqlancer/run.sh b/docker/test/sqlancer/run.sh index ffe0afd98a8..13929c5c3e2 100755 --- a/docker/test/sqlancer/run.sh +++ b/docker/test/sqlancer/run.sh @@ -29,4 +29,5 @@ tail -n 1000 /var/log/clickhouse-server/stderr.log > /test_output/stderr.log tail -n 1000 /var/log/clickhouse-server/stdout.log > /test_output/stdout.log tail -n 1000 /var/log/clickhouse-server/clickhouse-server.log > /test_output/clickhouse-server.log +./process_sqlancer_result.py || echo -e "failure\tCannot parse results" > /test_output/check_status.tsv ls /test_output diff --git a/docker/test/style/Dockerfile b/docker/test/style/Dockerfile index e70f9e05679..86595a77a54 100644 --- a/docker/test/style/Dockerfile +++ b/docker/test/style/Dockerfile @@ -10,14 +10,6 @@ RUN apt-get update && env DEBIAN_FRONTEND=noninteractive apt-get install --yes \ yamllint \ && pip3 install codespell - -# For |& syntax -SHELL ["bash", "-c"] - -CMD cd /ClickHouse/utils/check-style && \ - ./check-style -n |& tee /test_output/style_output.txt && \ - ./check-typos |& tee /test_output/typos_output.txt && \ - ./check-whitespaces -n |& tee /test_output/whitespaces_output.txt && \ - ./check-duplicate-includes.sh |& tee /test_output/duplicate_output.txt && \ - ./shellcheck-run.sh |& tee /test_output/shellcheck_output.txt && \ - true +COPY run.sh / +COPY process_style_check_result.py / +CMD ["/bin/bash", "/run.sh"] diff --git a/docker/test/style/process_style_check_result.py b/docker/test/style/process_style_check_result.py new file mode 100755 index 00000000000..61b1e0f05c5 --- /dev/null +++ b/docker/test/style/process_style_check_result.py @@ -0,0 +1,96 @@ +#!/usr/bin/env python3 + +import os +import logging +import argparse +import csv + + +def process_result(result_folder): + status = "success" + description = "" + test_results = [] + + style_log_path = '{}/style_output.txt'.format(result_folder) + if not os.path.exists(style_log_path): + logging.info("No style check log on path %s", style_log_path) + return "exception", "No style check log", [] + elif os.stat(style_log_path).st_size != 0: + description += "Style check failed. " + test_results.append(("Style check", "FAIL")) + status = "failure" # Disabled for now + else: + test_results.append(("Style check", "OK")) + + typos_log_path = '{}/typos_output.txt'.format(result_folder) + if not os.path.exists(style_log_path): + logging.info("No typos check log on path %s", style_log_path) + return "exception", "No typos check log", [] + elif os.stat(typos_log_path).st_size != 0: + description += "Typos check failed. " + test_results.append(("Typos check", "FAIL")) + status = "failure" + else: + test_results.append(("Typos check", "OK")) + + whitespaces_log_path = '{}/whitespaces_output.txt'.format(result_folder) + if not os.path.exists(style_log_path): + logging.info("No whitespaces check log on path %s", style_log_path) + return "exception", "No whitespaces check log", [] + elif os.stat(whitespaces_log_path).st_size != 0: + description += "Whitespaces check failed. " + test_results.append(("Whitespaces check", "FAIL")) + status = "failure" + else: + test_results.append(("Whitespaces check", "OK")) + + duplicate_log_path = '{}/duplicate_output.txt'.format(result_folder) + if not os.path.exists(duplicate_log_path): + logging.info("No header duplicates check log on path %s", duplicate_log_path) + return "exception", "No header duplicates check log", [] + elif os.stat(duplicate_log_path).st_size != 0: + description += " Header duplicates check failed. " + test_results.append(("Header duplicates check", "FAIL")) + status = "failure" + else: + test_results.append(("Header duplicates check", "OK")) + + shellcheck_log_path = '{}/shellcheck_output.txt'.format(result_folder) + if not os.path.exists(shellcheck_log_path): + logging.info("No shellcheck log on path %s", shellcheck_log_path) + return "exception", "No shellcheck log", [] + elif os.stat(shellcheck_log_path).st_size != 0: + description += " Shellcheck check failed. " + test_results.append(("Shellcheck ", "FAIL")) + status = "failure" + else: + test_results.append(("Shellcheck", "OK")) + + if not description: + description += "Style check success" + + return status, description, test_results + + +def write_results(results_file, status_file, results, status): + with open(results_file, 'w') as f: + out = csv.writer(f, delimiter='\t') + out.writerows(results) + with open(status_file, 'w') as f: + out = csv.writer(f, delimiter='\t') + out.writerow(status) + + +if __name__ == "__main__": + logging.basicConfig(level=logging.INFO, format='%(asctime)s %(message)s') + parser = argparse.ArgumentParser(description="ClickHouse script for parsing results of style check") + parser.add_argument("--in-results-dir", default='/test_output/') + parser.add_argument("--out-results-file", default='/test_output/test_results.tsv') + parser.add_argument("--out-status-file", default='/test_output/check_status.tsv') + args = parser.parse_args() + + state, description, test_results = process_result(args.in_results_dir) + logging.info("Result parsed") + status = (state, description) + write_results(args.out_results_file, args.out_status_file, test_results, status) + logging.info("Result written") diff --git a/docker/test/style/run.sh b/docker/test/style/run.sh new file mode 100755 index 00000000000..b23f1e02449 --- /dev/null +++ b/docker/test/style/run.sh @@ -0,0 +1,9 @@ +#!/bin/bash + +cd /ClickHouse/utils/check-style +./check-style -n |& tee /test_output/style_output.txt +./check-typos |& tee /test_output/typos_output.txt +./check-whitespaces -n |& tee /test_output/whitespaces_output.txt +./check-duplicate-includes.sh |& tee /test_output/duplicate_output.txt +./shellcheck-run.sh |& tee /test_output/shellcheck_output.txt +./process_style_check_result.py || echo -e "failure\tCannot parse results" > /test_output/check_status.tsv diff --git a/docker/test/testflows/runner/Dockerfile b/docker/test/testflows/runner/Dockerfile index 4139fb9e044..afdcd37ad34 100644 --- a/docker/test/testflows/runner/Dockerfile +++ b/docker/test/testflows/runner/Dockerfile @@ -61,6 +61,7 @@ RUN set -eux; \ COPY modprobe.sh /usr/local/bin/modprobe COPY dockerd-entrypoint.sh /usr/local/bin/ +COPY process_testflows_result.py /usr/local/bin/ RUN set -x \ && addgroup --system dockremap \ @@ -72,5 +73,5 @@ RUN set -x \ VOLUME /var/lib/docker EXPOSE 2375 ENTRYPOINT ["dockerd-entrypoint.sh"] -CMD ["sh", "-c", "python3 regression.py --no-color -o classic --local --clickhouse-binary-path ${CLICKHOUSE_TESTS_SERVER_BIN_PATH} --log test.log ${TESTFLOWS_OPTS}; cat test.log | tfs report results --format json > results.json"] +CMD ["sh", "-c", "python3 regression.py --no-color -o classic --local --clickhouse-binary-path ${CLICKHOUSE_TESTS_SERVER_BIN_PATH} --log test.log ${TESTFLOWS_OPTS}; cat test.log | tfs report results --format json > results.json; /usr/local/bin/process_testflows_result.py || echo -e 'failure\tCannot parse results' > /test_output/check_status.tsv"] diff --git a/docker/test/testflows/runner/process_testflows_result.py b/docker/test/testflows/runner/process_testflows_result.py new file mode 100755 index 00000000000..a77cc2d8992 --- /dev/null +++ b/docker/test/testflows/runner/process_testflows_result.py @@ -0,0 +1,67 @@ +#!/usr/bin/env python3 + +import os +import logging +import argparse +import csv +import json + + +def process_result(result_folder): + json_path = os.path.join(result_folder, "results.json") + if not os.path.exists(json_path): + return "success", "No testflows in branch", None, [] + + test_binary_log = os.path.join(result_folder, "test.log") + with open(json_path) as source: + results = json.loads(source.read()) + + total_tests = 0 + total_ok = 0 + total_fail = 0 + total_other = 0 + test_results = [] + for test in results["tests"]: + test_name = test['test']['test_name'] + test_result = test['result']['result_type'].upper() + test_time = str(test['result']['message_rtime']) + total_tests += 1 + if test_result == "OK": + total_ok += 1 + elif test_result == "FAIL" or test_result == "ERROR": + total_fail += 1 + else: + total_other += 1 + + test_results.append((test_name, test_result, test_time)) + if total_fail != 0: + status = "failure" + else: + status = "success" + + description = "failed: {}, passed: {}, other: {}".format(total_fail, total_ok, total_other) + return status, description, test_results, [json_path, test_binary_log] + + +def write_results(results_file, status_file, results, status): + with open(results_file, 'w') as f: + out = csv.writer(f, delimiter='\t') + out.writerows(results) + with open(status_file, 'w') as f: + out = csv.writer(f, delimiter='\t') + out.writerow(status) + +if __name__ == "__main__": + logging.basicConfig(level=logging.INFO, format='%(asctime)s %(message)s') + parser = argparse.ArgumentParser(description="ClickHouse script for parsing results of Testflows tests") + parser.add_argument("--in-results-dir", default='/test_output/') + parser.add_argument("--out-results-file", default='/test_output/test_results.tsv') + parser.add_argument("--out-status-file", default='/test_output/check_status.tsv') + args = parser.parse_args() + + state, description, test_results, logs = process_result(args.in_results_dir) + logging.info("Result parsed") + status = (state, description) + write_results(args.out_results_file, args.out_status_file, test_results, status) + logging.info("Result written") + diff --git a/docker/test/unit/process_unit_tests_result.py b/docker/test/unit/process_unit_tests_result.py index 070262e1f4b..7219aa13b82 100755 --- a/docker/test/unit/process_unit_tests_result.py +++ b/docker/test/unit/process_unit_tests_result.py @@ -82,7 +82,7 @@ def write_results(results_file, status_file, results, status): if __name__ == "__main__": logging.basicConfig(level=logging.INFO, format='%(asctime)s %(message)s') - parser = argparse.ArgumentParser(description="ClickHouse script for parsing results of functional tests") + parser = argparse.ArgumentParser(description="ClickHouse script for parsing results of unit tests") parser.add_argument("--in-results-dir", default='/test_output/') parser.add_argument("--out-results-file", default='/test_output/test_results.tsv') parser.add_argument("--out-status-file", default='/test_output/check_status.tsv') diff --git a/tests/queries/skip_list.json b/tests/queries/skip_list.json index 475b87ae36a..e029b06818f 100644 --- a/tests/queries/skip_list.json +++ b/tests/queries/skip_list.json @@ -519,6 +519,7 @@ "00575_illegal_column_exception_when_drop_depen_column", "00599_create_view_with_subquery", "00604_show_create_database", + "00600_replace_running_query", "00612_http_max_query_size", "00619_union_highlite", "00620_optimize_on_nonleader_replica_zookeeper", From 5ee1ec5bede5ec4c5d7396daf263035911426136 Mon Sep 17 00:00:00 2001 From: Alexander Tokmakov Date: Fri, 26 Feb 2021 15:58:13 +0300 Subject: [PATCH 057/544] move more code --- docker/test/split_build_smoke_test/Dockerfile | 1 + .../process_split_build_smoke_test_result.py | 61 +++++++++++++++++++ docker/test/split_build_smoke_test/run.sh | 3 +- docker/test/sqlancer/run.sh | 2 +- docker/test/style/run.sh | 4 +- 5 files changed, 67 insertions(+), 4 deletions(-) create mode 100755 docker/test/split_build_smoke_test/process_split_build_smoke_test_result.py diff --git a/docker/test/split_build_smoke_test/Dockerfile b/docker/test/split_build_smoke_test/Dockerfile index c77db1c6c88..54a9eb17868 100644 --- a/docker/test/split_build_smoke_test/Dockerfile +++ b/docker/test/split_build_smoke_test/Dockerfile @@ -2,5 +2,6 @@ FROM yandex/clickhouse-binary-builder COPY run.sh /run.sh +COPY process_split_build_smoke_test_result.py / CMD /run.sh diff --git a/docker/test/split_build_smoke_test/process_split_build_smoke_test_result.py b/docker/test/split_build_smoke_test/process_split_build_smoke_test_result.py new file mode 100755 index 00000000000..58d6ba8c62a --- /dev/null +++ b/docker/test/split_build_smoke_test/process_split_build_smoke_test_result.py @@ -0,0 +1,61 @@ +#!/usr/bin/env python3 + +import os +import logging +import argparse +import csv + +RESULT_LOG_NAME = "run.log" + +def process_result(result_folder): + + status = "success" + description = 'Server started and responded' + summary = [("Smoke test", "OK")] + with open(os.path.join(result_folder, RESULT_LOG_NAME), 'r') as run_log: + lines = run_log.read().split('\n') + if not lines or lines[0].strip() != 'OK': + status = "failure" + logging.info("Lines is not ok: %s", str('\n'.join(lines))) + summary = [("Smoke test", "FAIL")] + description = 'Server failed to respond, see result in logs' + + result_logs = [] + server_log_path = os.path.join(result_folder, "clickhouse-server.log") + stderr_log_path = os.path.join(result_folder, "stderr.log") + client_stderr_log_path = os.path.join(result_folder, "clientstderr.log") + + if os.path.exists(server_log_path): + result_logs.append(server_log_path) + + if os.path.exists(stderr_log_path): + result_logs.append(stderr_log_path) + + if os.path.exists(client_stderr_log_path): + result_logs.append(client_stderr_log_path) + + return status, description, summary, result_logs + + +def write_results(results_file, status_file, results, status): + with open(results_file, 'w') as f: + out = csv.writer(f, delimiter='\t') + out.writerows(results) + with open(status_file, 'w') as f: + out = csv.writer(f, delimiter='\t') + out.writerow(status) + + +if __name__ == "__main__": + logging.basicConfig(level=logging.INFO, format='%(asctime)s %(message)s') + parser = argparse.ArgumentParser(description="ClickHouse script for parsing results of split build smoke test") + parser.add_argument("--in-results-dir", default='/test_output/') + parser.add_argument("--out-results-file", default='/test_output/test_results.tsv') + parser.add_argument("--out-status-file", default='/test_output/check_status.tsv') + args = parser.parse_args() + + state, description, test_results, logs = process_result(args.in_results_dir) + logging.info("Result parsed") + status = (state, description) + write_results(args.out_results_file, args.out_status_file, test_results, status) + logging.info("Result written") diff --git a/docker/test/split_build_smoke_test/run.sh b/docker/test/split_build_smoke_test/run.sh index eac9848030e..fa9a32b73ba 100755 --- a/docker/test/split_build_smoke_test/run.sh +++ b/docker/test/split_build_smoke_test/run.sh @@ -11,10 +11,11 @@ install_and_run_server() { run_client() { for i in {1..100}; do sleep 1 - LD_LIBRARY_PATH=/unpacked /unpacked/clickhouse-client --query "select 'OK'" 2>/var/log/clickhouse-server/clientstderr.log && break + LD_LIBRARY_PATH=/unpacked /unpacked/clickhouse-client --query "select 'OK'" > /test_output/run.log 2>/var/log/clickhouse-server/clientstderr.log && break [[ $i == 100 ]] && echo 'FAIL' done } install_and_run_server run_client +/process_split_build_smoke_test_result.py || echo -e "failure\tCannot parse results" > /test_output/check_status.tsv diff --git a/docker/test/sqlancer/run.sh b/docker/test/sqlancer/run.sh index 13929c5c3e2..20e82603567 100755 --- a/docker/test/sqlancer/run.sh +++ b/docker/test/sqlancer/run.sh @@ -29,5 +29,5 @@ tail -n 1000 /var/log/clickhouse-server/stderr.log > /test_output/stderr.log tail -n 1000 /var/log/clickhouse-server/stdout.log > /test_output/stdout.log tail -n 1000 /var/log/clickhouse-server/clickhouse-server.log > /test_output/clickhouse-server.log -./process_sqlancer_result.py || echo -e "failure\tCannot parse results" > /test_output/check_status.tsv +/process_sqlancer_result.py || echo -e "failure\tCannot parse results" > /test_output/check_status.tsv ls /test_output diff --git a/docker/test/style/run.sh b/docker/test/style/run.sh index b23f1e02449..424bfe71b15 100755 --- a/docker/test/style/run.sh +++ b/docker/test/style/run.sh @@ -1,9 +1,9 @@ #!/bin/bash -cd /ClickHouse/utils/check-style +cd /ClickHouse/utils/check-style || echo -e "failure\tRepo not found" > /test_output/check_status.tsv ./check-style -n |& tee /test_output/style_output.txt ./check-typos |& tee /test_output/typos_output.txt ./check-whitespaces -n |& tee /test_output/whitespaces_output.txt ./check-duplicate-includes.sh |& tee /test_output/duplicate_output.txt ./shellcheck-run.sh |& tee /test_output/shellcheck_output.txt -./process_style_check_result.py || echo -e "failure\tCannot parse results" > /test_output/check_status.tsv +/process_style_check_result.py || echo -e "failure\tCannot parse results" > /test_output/check_status.tsv From fb42dec355ab2863cd6d2b8b410f9a26e3340169 Mon Sep 17 00:00:00 2001 From: Alexander Tokmakov Date: Sat, 27 Feb 2021 01:20:49 +0300 Subject: [PATCH 058/544] fix --- docker/test/split_build_smoke_test/run.sh | 5 +++-- docker/test/testflows/runner/Dockerfile | 2 +- docker/test/testflows/runner/process_testflows_result.py | 6 +++--- tests/queries/skip_list.json | 1 + tests/testflows/runner | 8 ++++++++ 5 files changed, 16 insertions(+), 6 deletions(-) diff --git a/docker/test/split_build_smoke_test/run.sh b/docker/test/split_build_smoke_test/run.sh index fa9a32b73ba..b565d7a481e 100755 --- a/docker/test/split_build_smoke_test/run.sh +++ b/docker/test/split_build_smoke_test/run.sh @@ -5,17 +5,18 @@ set -x install_and_run_server() { mkdir /unpacked tar -xzf /package_folder/shared_build.tgz -C /unpacked --strip 1 - LD_LIBRARY_PATH=/unpacked /unpacked/clickhouse-server --config /unpacked/config/config.xml >/var/log/clickhouse-server/stderr.log 2>&1 & + LD_LIBRARY_PATH=/unpacked /unpacked/clickhouse-server --config /unpacked/config/config.xml >/test_output/stderr.log 2>&1 & } run_client() { for i in {1..100}; do sleep 1 - LD_LIBRARY_PATH=/unpacked /unpacked/clickhouse-client --query "select 'OK'" > /test_output/run.log 2>/var/log/clickhouse-server/clientstderr.log && break + LD_LIBRARY_PATH=/unpacked /unpacked/clickhouse-client --query "select 'OK'" > /test_output/run.log 2> /test_output/clientstderr.log && break [[ $i == 100 ]] && echo 'FAIL' done } install_and_run_server run_client +mv /var/log/clickhouse-server/clickhouse-server.log /test_output/clickhouse-server.log /process_split_build_smoke_test_result.py || echo -e "failure\tCannot parse results" > /test_output/check_status.tsv diff --git a/docker/test/testflows/runner/Dockerfile b/docker/test/testflows/runner/Dockerfile index afdcd37ad34..10014851a82 100644 --- a/docker/test/testflows/runner/Dockerfile +++ b/docker/test/testflows/runner/Dockerfile @@ -73,5 +73,5 @@ RUN set -x \ VOLUME /var/lib/docker EXPOSE 2375 ENTRYPOINT ["dockerd-entrypoint.sh"] -CMD ["sh", "-c", "python3 regression.py --no-color -o classic --local --clickhouse-binary-path ${CLICKHOUSE_TESTS_SERVER_BIN_PATH} --log test.log ${TESTFLOWS_OPTS}; cat test.log | tfs report results --format json > results.json; /usr/local/bin/process_testflows_result.py || echo -e 'failure\tCannot parse results' > /test_output/check_status.tsv"] +CMD ["sh", "-c", "python3 regression.py --no-color -o classic --local --clickhouse-binary-path ${CLICKHOUSE_TESTS_SERVER_BIN_PATH} --log test.log ${TESTFLOWS_OPTS}; cat test.log | tfs report results --format json > results.json; /usr/local/bin/process_testflows_result.py || echo -e 'failure\tCannot parse results' > check_status.tsv"] diff --git a/docker/test/testflows/runner/process_testflows_result.py b/docker/test/testflows/runner/process_testflows_result.py index a77cc2d8992..37d0b6a69d1 100755 --- a/docker/test/testflows/runner/process_testflows_result.py +++ b/docker/test/testflows/runner/process_testflows_result.py @@ -54,9 +54,9 @@ def write_results(results_file, status_file, results, status): if __name__ == "__main__": logging.basicConfig(level=logging.INFO, format='%(asctime)s %(message)s') parser = argparse.ArgumentParser(description="ClickHouse script for parsing results of Testflows tests") - parser.add_argument("--in-results-dir", default='/test_output/') - parser.add_argument("--out-results-file", default='/test_output/test_results.tsv') - parser.add_argument("--out-status-file", default='/test_output/check_status.tsv') + parser.add_argument("--in-results-dir", default='./') + parser.add_argument("--out-results-file", default='./test_results.tsv') + parser.add_argument("--out-status-file", default='./check_status.tsv') args = parser.parse_args() state, description, test_results, logs = process_result(args.in_results_dir) diff --git a/tests/queries/skip_list.json b/tests/queries/skip_list.json index e029b06818f..8115831d10e 100644 --- a/tests/queries/skip_list.json +++ b/tests/queries/skip_list.json @@ -275,6 +275,7 @@ "00534_functions_bad_arguments4", "00534_functions_bad_arguments9", "00564_temporary_table_management", + "00600_replace_running_query", "00626_replace_partition_from_table_zookeeper", "00652_replicated_mutations_zookeeper", "00687_top_and_offset", diff --git a/tests/testflows/runner b/tests/testflows/runner index 0acc3a25945..24c595e2e94 100755 --- a/tests/testflows/runner +++ b/tests/testflows/runner @@ -120,3 +120,11 @@ if __name__ == "__main__": print(("Running testflows container as: '" + cmd + "'.")) # testflows return non zero error code on failed tests subprocess.call(cmd, shell=True) + + result_path = os.environ.get("CLICKHOUSE_TESTS_RESULT_PATH", None) + if result_path is not None: + move_from = os.path.join(args.clickhouse_root, 'tests/testflows') + status = os.path.join(move_from, 'check_status') + results = os.path.join(move_from, 'test_results.tsv') + subprocess.call("mv {} {}".format(status, result_path), shell=True) + subprocess.call("mv {} {}".format(results, result_path), shell=True) From 0937127ea49c746ec6d8be93cce2f3c5a589ab8e Mon Sep 17 00:00:00 2001 From: Alexander Tokmakov Date: Sat, 27 Feb 2021 14:01:04 +0300 Subject: [PATCH 059/544] fix --- tests/testflows/runner | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/testflows/runner b/tests/testflows/runner index 24c595e2e94..213ff6e50d8 100755 --- a/tests/testflows/runner +++ b/tests/testflows/runner @@ -124,7 +124,7 @@ if __name__ == "__main__": result_path = os.environ.get("CLICKHOUSE_TESTS_RESULT_PATH", None) if result_path is not None: move_from = os.path.join(args.clickhouse_root, 'tests/testflows') - status = os.path.join(move_from, 'check_status') + status = os.path.join(move_from, 'check_status.tsv') results = os.path.join(move_from, 'test_results.tsv') subprocess.call("mv {} {}".format(status, result_path), shell=True) subprocess.call("mv {} {}".format(results, result_path), shell=True) From 3c11d444940e8b7c59d93a24366c9b2fb096ec2c Mon Sep 17 00:00:00 2001 From: Anton Ivashkin Date: Fri, 26 Feb 2021 14:06:24 +0300 Subject: [PATCH 060/544] Add description for getUniqueId method, fix typos --- src/Storages/MergeTree/IMergeTreeDataPart.h | 2 ++ src/Storages/MergeTree/MergeTreeData.h | 6 +++--- src/Storages/StorageReplicatedMergeTree.cpp | 6 ++++-- src/Storages/StorageReplicatedMergeTree.h | 3 +++ 4 files changed, 12 insertions(+), 5 deletions(-) diff --git a/src/Storages/MergeTree/IMergeTreeDataPart.h b/src/Storages/MergeTree/IMergeTreeDataPart.h index f9b5f616f70..83f8c672001 100644 --- a/src/Storages/MergeTree/IMergeTreeDataPart.h +++ b/src/Storages/MergeTree/IMergeTreeDataPart.h @@ -367,6 +367,8 @@ public: /// part creation (using alter query with materialize_ttl setting). bool checkAllTTLCalculated(const StorageMetadataPtr & metadata_snapshot) const; + /// Return some uniq string for file + /// Required for distinguish different copies of the same part on S3 String getUniqueId() const; protected: diff --git a/src/Storages/MergeTree/MergeTreeData.h b/src/Storages/MergeTree/MergeTreeData.h index 81e977d7ecc..3f41cc04217 100644 --- a/src/Storages/MergeTree/MergeTreeData.h +++ b/src/Storages/MergeTree/MergeTreeData.h @@ -739,16 +739,16 @@ public: bool areBackgroundMovesNeeded() const; /// Lock part in zookeeper for use common S3 data in several nodes - /// Overrided in StorageReplicatedMergeTree + /// Overridden in StorageReplicatedMergeTree virtual void lockSharedData(const IMergeTreeDataPart &) const {} /// Unlock common S3 data part in zookeeper - /// Overrided in StorageReplicatedMergeTree + /// Overridden in StorageReplicatedMergeTree virtual bool unlockSharedData(const IMergeTreeDataPart &) const { return true; } virtual bool unlockSharedData(const IMergeTreeDataPart &, const String &) const { return true; } /// Fetch part only if some replica has it on shared storage like S3 - /// Overrided in StorageReplicatedMergeTree + /// Overridden in StorageReplicatedMergeTree virtual bool tryToFetchIfShared(const IMergeTreeDataPart &, const DiskPtr &, const String &) const { return false; } protected: diff --git a/src/Storages/StorageReplicatedMergeTree.cpp b/src/Storages/StorageReplicatedMergeTree.cpp index b94d3dd3f89..8c5636d2aa0 100644 --- a/src/Storages/StorageReplicatedMergeTree.cpp +++ b/src/Storages/StorageReplicatedMergeTree.cpp @@ -142,6 +142,10 @@ static const auto MERGE_SELECTING_SLEEP_MS = 5 * 1000; static const auto MUTATIONS_FINALIZING_SLEEP_MS = 1 * 1000; static const auto MUTATIONS_FINALIZING_IDLE_SLEEP_MS = 5 * 1000; + +std::atomic_uint StorageReplicatedMergeTree::total_fetches {0}; + + void StorageReplicatedMergeTree::setZooKeeper() { std::lock_guard lock(current_zookeeper_mutex); @@ -1730,7 +1734,6 @@ bool StorageReplicatedMergeTree::executeFetch(LogEntry & entry) const auto storage_settings_ptr = getSettings(); auto metadata_snapshot = getInMemoryMetadataPtr(); - static std::atomic_uint total_fetches {0}; if (storage_settings_ptr->replicated_max_parallel_fetches && total_fetches >= storage_settings_ptr->replicated_max_parallel_fetches) { throw Exception("Too many total fetches from replicas, maximum: " + storage_settings_ptr->replicated_max_parallel_fetches.toString(), @@ -1934,7 +1937,6 @@ bool StorageReplicatedMergeTree::executeFetchShared(ReplicatedMergeTreeLogEntry const auto storage_settings_ptr = getSettings(); auto metadata_snapshot = getInMemoryMetadataPtr(); - static std::atomic_uint total_fetches {0}; if (storage_settings_ptr->replicated_max_parallel_fetches && total_fetches >= storage_settings_ptr->replicated_max_parallel_fetches) { throw Exception("Too many total fetches from replicas, maximum: " + storage_settings_ptr->replicated_max_parallel_fetches.toString(), diff --git a/src/Storages/StorageReplicatedMergeTree.h b/src/Storages/StorageReplicatedMergeTree.h index 70a83145da6..58bedfc0408 100644 --- a/src/Storages/StorageReplicatedMergeTree.h +++ b/src/Storages/StorageReplicatedMergeTree.h @@ -307,6 +307,9 @@ private: /// Event that is signalled (and is reset) by the restarting_thread when the ZooKeeper session expires. Poco::Event partial_shutdown_event {false}; /// Poco::Event::EVENT_MANUALRESET + /// Limiting parallel fetches per node + static std::atomic_uint total_fetches; + /// Limiting parallel fetches per one table std::atomic_uint current_table_fetches {0}; From 6af4d06dffb08842761d775caf04ac8fcca3d774 Mon Sep 17 00:00:00 2001 From: Vasily Nemkov Date: Tue, 23 Feb 2021 12:30:15 +0200 Subject: [PATCH 061/544] Added Grant, Revoke and System query_kind for system.query_log Also some tests to ensure that all queries are logged --- src/Interpreters/InterpreterGrantQuery.cpp | 10 ++ src/Interpreters/InterpreterGrantQuery.h | 1 + src/Interpreters/InterpreterSystemQuery.cpp | 5 + src/Interpreters/InterpreterSystemQuery.h | 2 + .../01702_system_query_log.reference | 92 +++++++++++ .../0_stateless/01702_system_query_log.sql | 148 ++++++++++++++++++ 6 files changed, 258 insertions(+) create mode 100644 tests/queries/0_stateless/01702_system_query_log.reference create mode 100644 tests/queries/0_stateless/01702_system_query_log.sql diff --git a/src/Interpreters/InterpreterGrantQuery.cpp b/src/Interpreters/InterpreterGrantQuery.cpp index dafe4d2e18c..034ebcec050 100644 --- a/src/Interpreters/InterpreterGrantQuery.cpp +++ b/src/Interpreters/InterpreterGrantQuery.cpp @@ -1,4 +1,5 @@ #include +#include #include #include #include @@ -209,4 +210,13 @@ void InterpreterGrantQuery::updateRoleFromQuery(Role & role, const ASTGrantQuery updateFromQueryImpl(role, query, roles_to_grant_or_revoke); } +void InterpreterGrantQuery::extendQueryLogElemImpl(QueryLogElement & elem, const ASTPtr & /*ast*/, const Context &) const +{ + auto & query = query_ptr->as(); + if (query.kind == Kind::GRANT) + elem.query_kind = "Grant"; + else if (query.kind == Kind::REVOKE) + elem.query_kind = "Revoke"; +} + } diff --git a/src/Interpreters/InterpreterGrantQuery.h b/src/Interpreters/InterpreterGrantQuery.h index 32810faecd2..645b33346c3 100644 --- a/src/Interpreters/InterpreterGrantQuery.h +++ b/src/Interpreters/InterpreterGrantQuery.h @@ -21,6 +21,7 @@ public: static void updateUserFromQuery(User & user, const ASTGrantQuery & query); static void updateRoleFromQuery(Role & role, const ASTGrantQuery & query); + void extendQueryLogElemImpl(QueryLogElement &, const ASTPtr &, const Context &) const override; private: ASTPtr query_ptr; diff --git a/src/Interpreters/InterpreterSystemQuery.cpp b/src/Interpreters/InterpreterSystemQuery.cpp index ece3209621b..53fb865be2d 100644 --- a/src/Interpreters/InterpreterSystemQuery.cpp +++ b/src/Interpreters/InterpreterSystemQuery.cpp @@ -754,4 +754,9 @@ AccessRightsElements InterpreterSystemQuery::getRequiredAccessForDDLOnCluster() return required_access; } +void InterpreterSystemQuery::extendQueryLogElemImpl(QueryLogElement & elem, const ASTPtr & /*ast*/, const Context &) const +{ + elem.query_kind = "System"; +} + } diff --git a/src/Interpreters/InterpreterSystemQuery.h b/src/Interpreters/InterpreterSystemQuery.h index 6fa0a432191..39f8bf69a6e 100644 --- a/src/Interpreters/InterpreterSystemQuery.h +++ b/src/Interpreters/InterpreterSystemQuery.h @@ -56,6 +56,8 @@ private: AccessRightsElements getRequiredAccessForDDLOnCluster() const; void startStopAction(StorageActionBlockType action_type, bool start); + + void extendQueryLogElemImpl(QueryLogElement &, const ASTPtr &, const Context &) const override; }; diff --git a/tests/queries/0_stateless/01702_system_query_log.reference b/tests/queries/0_stateless/01702_system_query_log.reference new file mode 100644 index 00000000000..7da0383dbb7 --- /dev/null +++ b/tests/queries/0_stateless/01702_system_query_log.reference @@ -0,0 +1,92 @@ +DROP queries and also a cleanup before the test +CREATE queries +SET queries +ALTER TABLE queries +SYSTEM queries +SHOW queries +GRANT queries +REVOKE queries +Misc queries +ACTUAL LOG CONTENT: +Select -- SET ROLE sqllt_role; -- tests are executed by user `default` which is defined in XML and is impossble to update.\n\nSELECT \'ALTER TABLE queries\'; +Select -- SYSTEM RELOAD DICTIONARY sqllt.dictionary; -- temporary out of order: Code: 210, Connection refused (localhost:9001) (version 21.3.1.1)\n-- DROP REPLICA\n-- haha, no\n-- SYSTEM KILL;\n-- SYSTEM SHUTDOWN;\n\n-- Since we don\'t really care about the actual output, suppress it with `FORMAT Null`.\nSELECT \'SHOW queries\'; +Select -- not done, seems to hard, so I\'ve skipped queries of ALTER-X, where X is:\n-- PARTITION\n-- ORDER BY\n-- SAMPLE BY\n-- INDEX\n-- CONSTRAINT\n-- TTL\n-- USER\n-- QUOTA\n-- ROLE\n-- ROW POLICY\n-- SETTINGS PROFILE\n\nSELECT \'SYSTEM queries\'; +Alter ALTER TABLE table ADD COLUMN new_col UInt32 DEFAULT 1; +Alter ALTER TABLE table CLEAR COLUMN new_col; +Alter ALTER TABLE table COMMENT COLUMN new_col \'dummy column with a comment\'; +Alter ALTER TABLE table DELETE WHERE i > 65535; +Alter ALTER TABLE table DROP COLUMN the_new_col; +Alter ALTER TABLE table MODIFY COLUMN new_col FixedString(12) DEFAULT \'Hello world!\'; +Alter ALTER TABLE table MODIFY COLUMN new_col REMOVE COMMENT; +Alter ALTER TABLE table RENAME COLUMN new_col TO the_new_col; +Alter ALTER TABLE table UPDATE i = i + 1 WHERE 1; +Create ATTACH TABLE sqllt.table; + CHECK TABLE sqllt.table FORMAT Null; +Create CREATE DATABASE sqllt; +Create CREATE DICTIONARY sqllt.dictionary (key UInt64, value UInt64) PRIMARY KEY key SOURCE(CLICKHOUSE(DB \'sqllt\' TABLE \'table\' HOST \'localhost\' PORT 9001)) LIFETIME(0) LAYOUT(FLAT()); + CREATE POLICY sqllt_policy ON sqllt.table, sqllt.view, sqllt.dictionary AS PERMISSIVE TO ALL; + CREATE POLICY sqllt_row_policy ON sqllt.table, sqllt.view, sqllt.dictionary AS PERMISSIVE TO ALL; + CREATE QUOTA sqllt_quota KEYED BY user_name TO sqllt_role; + CREATE ROLE sqllt_role; + CREATE SETTINGS PROFILE sqllt_settings_profile SETTINGS interactive_delay = 200000; +Create CREATE TABLE sqllt.table\n(\n i UInt8, s String\n)\nENGINE = MergeTree PARTITION BY tuple() ORDER BY tuple(); + CREATE USER sqllt_user IDENTIFIED WITH PLAINTEXT_PASSWORD BY \'password\'; +Create CREATE VIEW sqllt.view AS SELECT i, s FROM sqllt.table; + DESCRIBE TABLE sqllt.table FORMAT Null; +Drop DETACH TABLE sqllt.table; +Drop DROP DATABASE IF EXISTS sqllt; + DROP POLICY IF EXISTS sqllt_policy ON sqllt.table, sqllt.view, sqllt.dictionary; + DROP QUOTA IF EXISTS sqllt_quota; + DROP ROLE IF EXISTS sqllt_role; + DROP ROW POLICY IF EXISTS sqllt_row_policy ON sqllt.table, sqllt.view, sqllt.dictionary; + DROP SETTINGS PROFILE IF EXISTS sqllt_settings_profile; + DROP USER IF EXISTS sqllt_user; +Grant GRANT DROP ON sqllt.view TO sqllt_user; +Grant GRANT SELECT ON sqllt.table TO sqllt_user; +Grant GRANT sqllt_role TO sqllt_user; +Rename RENAME TABLE sqllt.table TO sqllt.table_new; +Rename RENAME TABLE sqllt.table_new TO sqllt.table; +Revoke REVOKE DROP ON sqllt.view FROM sqllt_user; +Revoke REVOKE SELECT ON sqllt.table FROM sqllt_user; +Select SELECT \'CREATE queries\'; +Select SELECT \'DROP queries and also a cleanup before the test\'; +Select SELECT \'GRANT queries\'; +Select SELECT \'Misc queries\'; +Select SELECT \'REVOKE queries\'; +Select SELECT \'SET queries\'; + SET DEFAULT ROLE sqllt_role TO sqllt_user; + SET log_profile_events=false; + SHOW CREATE DICTIONARY sqllt.dictionary FORMAT Null; + SHOW CREATE POLICY sqllt_policy FORMAT Null; + SHOW CREATE QUOTA sqllt_quota FORMAT Null; + SHOW CREATE ROLE sqllt_role FORMAT Null; + SHOW CREATE ROW POLICY sqllt_row_policy FORMAT Null; + SHOW CREATE SETTINGS PROFILE sqllt_settings_profile FORMAT Null; + SHOW CREATE TABLE sqllt.table FORMAT Null; + SHOW CREATE USER sqllt_user FORMAT Null; + SHOW DATABASES LIKE \'sqllt\' FORMAT Null; + SHOW DICTIONARIES FROM sqllt FORMAT Null; + SHOW GRANTS FOR sqllt_user FORMAT Null; + SHOW GRANTS FORMAT Null; + SHOW TABLES FROM sqllt FORMAT Null; +System SYSTEM DROP COMPILED EXPRESSION CACHE; +System SYSTEM DROP DNS CACHE; +System SYSTEM DROP MARK CACHE; +System SYSTEM DROP UNCOMPRESSED CACHE; +System SYSTEM FLUSH LOGS; +System SYSTEM RELOAD CONFIG; +System SYSTEM RELOAD DICTIONARIES; +System SYSTEM RELOAD EMBEDDED DICTIONARIES; +System SYSTEM START FETCHES; +System SYSTEM START MERGES; +System SYSTEM START MOVES; +System SYSTEM START REPLICATED SENDS; +System SYSTEM START TTL MERGES; +System SYSTEM STOP FETCHES; +System SYSTEM STOP MERGES; +System SYSTEM STOP MOVES; +System SYSTEM STOP REPLICATED SENDS; +System SYSTEM STOP TTL MERGES; +Drop TRUNCATE TABLE sqllt.table; + USE sqllt +DROP queries and also a cleanup after the test diff --git a/tests/queries/0_stateless/01702_system_query_log.sql b/tests/queries/0_stateless/01702_system_query_log.sql new file mode 100644 index 00000000000..9e14e600305 --- /dev/null +++ b/tests/queries/0_stateless/01702_system_query_log.sql @@ -0,0 +1,148 @@ +-- fire all kinds of queries and then check if those are present in the system.query_log +SET log_comment='system.query_log logging test'; + +SELECT 'DROP queries and also a cleanup before the test'; +DROP DATABASE IF EXISTS sqllt; +DROP USER IF EXISTS sqllt_user; +DROP ROLE IF EXISTS sqllt_role; +DROP POLICY IF EXISTS sqllt_policy ON sqllt.table, sqllt.view, sqllt.dictionary; +DROP ROW POLICY IF EXISTS sqllt_row_policy ON sqllt.table, sqllt.view, sqllt.dictionary; +DROP QUOTA IF EXISTS sqllt_quota; +DROP SETTINGS PROFILE IF EXISTS sqllt_settings_profile; + +SELECT 'CREATE queries'; +CREATE DATABASE sqllt; +USE sqllt; + +CREATE TABLE sqllt.table +( + i UInt8, s String +) +ENGINE = MergeTree PARTITION BY tuple() ORDER BY tuple(); + +CREATE VIEW sqllt.view AS SELECT i, s FROM sqllt.table; +CREATE DICTIONARY sqllt.dictionary (key UInt64, value UInt64) PRIMARY KEY key SOURCE(CLICKHOUSE(DB 'sqllt' TABLE 'table' HOST 'localhost' PORT 9001)) LIFETIME(0) LAYOUT(FLAT()); + +CREATE USER sqllt_user IDENTIFIED WITH PLAINTEXT_PASSWORD BY 'password'; +CREATE ROLE sqllt_role; + +CREATE POLICY sqllt_policy ON sqllt.table, sqllt.view, sqllt.dictionary AS PERMISSIVE TO ALL; +CREATE POLICY sqllt_row_policy ON sqllt.table, sqllt.view, sqllt.dictionary AS PERMISSIVE TO ALL; + +CREATE QUOTA sqllt_quota KEYED BY user_name TO sqllt_role; +CREATE SETTINGS PROFILE sqllt_settings_profile SETTINGS interactive_delay = 200000; + +GRANT sqllt_role TO sqllt_user; + + +SELECT 'SET queries'; +SET log_profile_events=false; +SET DEFAULT ROLE sqllt_role TO sqllt_user; +-- SET ROLE sqllt_role; -- tests are executed by user `default` which is defined in XML and is impossble to update. + +SELECT 'ALTER TABLE queries'; +ALTER TABLE table ADD COLUMN new_col UInt32 DEFAULT 1; +ALTER TABLE table COMMENT COLUMN new_col 'dummy column with a comment'; +ALTER TABLE table CLEAR COLUMN new_col; +ALTER TABLE table MODIFY COLUMN new_col FixedString(12) DEFAULT 'Hello world!'; +ALTER TABLE table MODIFY COLUMN new_col REMOVE COMMENT; +ALTER TABLE table RENAME COLUMN new_col TO the_new_col; +ALTER TABLE table DROP COLUMN the_new_col; +ALTER TABLE table UPDATE i = i + 1 WHERE 1; +ALTER TABLE table DELETE WHERE i > 65535; + +-- not done, seems to hard, so I've skipped queries of ALTER-X, where X is: +-- PARTITION +-- ORDER BY +-- SAMPLE BY +-- INDEX +-- CONSTRAINT +-- TTL +-- USER +-- QUOTA +-- ROLE +-- ROW POLICY +-- SETTINGS PROFILE + +SELECT 'SYSTEM queries'; +SYSTEM RELOAD EMBEDDED DICTIONARIES; +SYSTEM RELOAD DICTIONARIES; +SYSTEM DROP DNS CACHE; +SYSTEM DROP MARK CACHE; +SYSTEM DROP UNCOMPRESSED CACHE; +SYSTEM DROP COMPILED EXPRESSION CACHE; +SYSTEM FLUSH LOGS; +SYSTEM RELOAD CONFIG; +SYSTEM STOP MERGES; +SYSTEM START MERGES; +SYSTEM STOP TTL MERGES; +SYSTEM START TTL MERGES; +SYSTEM STOP MOVES; +SYSTEM START MOVES; +SYSTEM STOP FETCHES; +SYSTEM START FETCHES; +SYSTEM STOP REPLICATED SENDS; +SYSTEM START REPLICATED SENDS; + +-- SYSTEM RELOAD DICTIONARY sqllt.dictionary; -- temporary out of order: Code: 210, Connection refused (localhost:9001) (version 21.3.1.1) +-- DROP REPLICA +-- haha, no +-- SYSTEM KILL; +-- SYSTEM SHUTDOWN; + +-- Since we don't really care about the actual output, suppress it with `FORMAT Null`. +SELECT 'SHOW queries'; + +SHOW CREATE TABLE sqllt.table FORMAT Null; +SHOW CREATE DICTIONARY sqllt.dictionary FORMAT Null; +SHOW DATABASES LIKE 'sqllt' FORMAT Null; +SHOW TABLES FROM sqllt FORMAT Null; +SHOW DICTIONARIES FROM sqllt FORMAT Null; +SHOW GRANTS FORMAT Null; +SHOW GRANTS FOR sqllt_user FORMAT Null; +SHOW CREATE USER sqllt_user FORMAT Null; +SHOW CREATE ROLE sqllt_role FORMAT Null; +SHOW CREATE POLICY sqllt_policy FORMAT Null; +SHOW CREATE ROW POLICY sqllt_row_policy FORMAT Null; +SHOW CREATE QUOTA sqllt_quota FORMAT Null; +SHOW CREATE SETTINGS PROFILE sqllt_settings_profile FORMAT Null; + +SELECT 'GRANT queries'; +GRANT SELECT ON sqllt.table TO sqllt_user; +GRANT DROP ON sqllt.view TO sqllt_user; + +SELECT 'REVOKE queries'; +REVOKE SELECT ON sqllt.table FROM sqllt_user; +REVOKE DROP ON sqllt.view FROM sqllt_user; + +SELECT 'Misc queries'; +DESCRIBE TABLE sqllt.table FORMAT Null; + +CHECK TABLE sqllt.table FORMAT Null; +DETACH TABLE sqllt.table; +ATTACH TABLE sqllt.table; + +RENAME TABLE sqllt.table TO sqllt.table_new; +RENAME TABLE sqllt.table_new TO sqllt.table; +TRUNCATE TABLE sqllt.table; + +--------------------------------------------------------------------------------------------------- +-- Now get all logs related to this test +--------------------------------------------------------------------------------------------------- + +SYSTEM FLUSH LOGS; +SELECT 'ACTUAL LOG CONTENT:'; + +-- Try to filter out all possible previous junk events by excluding old log entries +SELECT query_kind, query FROM system.query_log WHERE log_comment LIKE '%system.query_log%' AND type == 'QueryStart' AND query_start_time >= now() - 5 ORDER BY query; + + +-- cleanup +SELECT 'DROP queries and also a cleanup after the test'; +DROP DATABASE IF EXISTS sqllt; +DROP USER IF EXISTS sqllt_user; +DROP ROLE IF EXISTS sqllt_role; +DROP POLICY IF EXISTS sqllt_policy ON sqllt.table, sqllt.view, sqllt.dictionary; +DROP ROW POLICY IF EXISTS sqllt_row_policy ON sqllt.table, sqllt.view, sqllt.dictionary; +DROP QUOTA IF EXISTS sqllt_quota; +DROP SETTINGS PROFILE IF EXISTS sqllt_settings_profile; From bfc5cb47cc5610f6e09a734469f39cd65102305c Mon Sep 17 00:00:00 2001 From: Vasily Nemkov Date: Mon, 1 Mar 2021 10:41:49 +0200 Subject: [PATCH 062/544] Fix tests --- .../0_stateless/01702_system_query_log.reference | 7 +++---- .../0_stateless/01702_system_query_log.sql | 16 ++++++++++------ 2 files changed, 13 insertions(+), 10 deletions(-) diff --git a/tests/queries/0_stateless/01702_system_query_log.reference b/tests/queries/0_stateless/01702_system_query_log.reference index 7da0383dbb7..763571fc946 100644 --- a/tests/queries/0_stateless/01702_system_query_log.reference +++ b/tests/queries/0_stateless/01702_system_query_log.reference @@ -8,15 +8,15 @@ GRANT queries REVOKE queries Misc queries ACTUAL LOG CONTENT: -Select -- SET ROLE sqllt_role; -- tests are executed by user `default` which is defined in XML and is impossble to update.\n\nSELECT \'ALTER TABLE queries\'; +Select -- SET ROLE sqllt_role; -- tests are executed by user `default` which is defined in XML and is impossible to update.\n\nSELECT \'ALTER TABLE queries\'; Select -- SYSTEM RELOAD DICTIONARY sqllt.dictionary; -- temporary out of order: Code: 210, Connection refused (localhost:9001) (version 21.3.1.1)\n-- DROP REPLICA\n-- haha, no\n-- SYSTEM KILL;\n-- SYSTEM SHUTDOWN;\n\n-- Since we don\'t really care about the actual output, suppress it with `FORMAT Null`.\nSELECT \'SHOW queries\'; Select -- not done, seems to hard, so I\'ve skipped queries of ALTER-X, where X is:\n-- PARTITION\n-- ORDER BY\n-- SAMPLE BY\n-- INDEX\n-- CONSTRAINT\n-- TTL\n-- USER\n-- QUOTA\n-- ROLE\n-- ROW POLICY\n-- SETTINGS PROFILE\n\nSELECT \'SYSTEM queries\'; -Alter ALTER TABLE table ADD COLUMN new_col UInt32 DEFAULT 1; +Alter ALTER TABLE table ADD COLUMN new_col UInt32 DEFAULT 123456789; Alter ALTER TABLE table CLEAR COLUMN new_col; Alter ALTER TABLE table COMMENT COLUMN new_col \'dummy column with a comment\'; Alter ALTER TABLE table DELETE WHERE i > 65535; Alter ALTER TABLE table DROP COLUMN the_new_col; -Alter ALTER TABLE table MODIFY COLUMN new_col FixedString(12) DEFAULT \'Hello world!\'; +Alter ALTER TABLE table MODIFY COLUMN new_col DateTime DEFAULT \'2015-05-18 07:40:13\'; Alter ALTER TABLE table MODIFY COLUMN new_col REMOVE COMMENT; Alter ALTER TABLE table RENAME COLUMN new_col TO the_new_col; Alter ALTER TABLE table UPDATE i = i + 1 WHERE 1; @@ -69,7 +69,6 @@ Select SELECT \'SET queries\'; SHOW GRANTS FOR sqllt_user FORMAT Null; SHOW GRANTS FORMAT Null; SHOW TABLES FROM sqllt FORMAT Null; -System SYSTEM DROP COMPILED EXPRESSION CACHE; System SYSTEM DROP DNS CACHE; System SYSTEM DROP MARK CACHE; System SYSTEM DROP UNCOMPRESSED CACHE; diff --git a/tests/queries/0_stateless/01702_system_query_log.sql b/tests/queries/0_stateless/01702_system_query_log.sql index 9e14e600305..bdf797865bf 100644 --- a/tests/queries/0_stateless/01702_system_query_log.sql +++ b/tests/queries/0_stateless/01702_system_query_log.sql @@ -38,13 +38,13 @@ GRANT sqllt_role TO sqllt_user; SELECT 'SET queries'; SET log_profile_events=false; SET DEFAULT ROLE sqllt_role TO sqllt_user; --- SET ROLE sqllt_role; -- tests are executed by user `default` which is defined in XML and is impossble to update. +-- SET ROLE sqllt_role; -- tests are executed by user `default` which is defined in XML and is impossible to update. SELECT 'ALTER TABLE queries'; -ALTER TABLE table ADD COLUMN new_col UInt32 DEFAULT 1; +ALTER TABLE table ADD COLUMN new_col UInt32 DEFAULT 123456789; ALTER TABLE table COMMENT COLUMN new_col 'dummy column with a comment'; ALTER TABLE table CLEAR COLUMN new_col; -ALTER TABLE table MODIFY COLUMN new_col FixedString(12) DEFAULT 'Hello world!'; +ALTER TABLE table MODIFY COLUMN new_col DateTime DEFAULT '2015-05-18 07:40:13'; ALTER TABLE table MODIFY COLUMN new_col REMOVE COMMENT; ALTER TABLE table RENAME COLUMN new_col TO the_new_col; ALTER TABLE table DROP COLUMN the_new_col; @@ -70,7 +70,6 @@ SYSTEM RELOAD DICTIONARIES; SYSTEM DROP DNS CACHE; SYSTEM DROP MARK CACHE; SYSTEM DROP UNCOMPRESSED CACHE; -SYSTEM DROP COMPILED EXPRESSION CACHE; SYSTEM FLUSH LOGS; SYSTEM RELOAD CONFIG; SYSTEM STOP MERGES; @@ -133,8 +132,13 @@ TRUNCATE TABLE sqllt.table; SYSTEM FLUSH LOGS; SELECT 'ACTUAL LOG CONTENT:'; --- Try to filter out all possible previous junk events by excluding old log entries -SELECT query_kind, query FROM system.query_log WHERE log_comment LIKE '%system.query_log%' AND type == 'QueryStart' AND query_start_time >= now() - 5 ORDER BY query; +-- Try to filter out all possible previous junk events by excluding old log entries, +SELECT query_kind, query FROM system.query_log +WHERE + log_comment LIKE '%system.query_log%' AND type == 'QueryStart' AND query_start_time >= now() - 5 + -- this one is to make stylecheck happy and to validate that CREATE\DROP queries of non-db-bound-objects, like USER\ROLS are logged properly. + AND (current_database == currentDatabase() OR current_database != currentDatabase()) +ORDER BY query; -- cleanup From d01cc968aae88f032eca3f4d0211ca55541f8f84 Mon Sep 17 00:00:00 2001 From: ikarishinjieva Date: Thu, 25 Feb 2021 17:57:00 +0800 Subject: [PATCH 063/544] fix #21170 --- src/Interpreters/InterpreterSelectWithUnionQuery.cpp | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/Interpreters/InterpreterSelectWithUnionQuery.cpp b/src/Interpreters/InterpreterSelectWithUnionQuery.cpp index b894db79c7b..5f2728804ca 100644 --- a/src/Interpreters/InterpreterSelectWithUnionQuery.cpp +++ b/src/Interpreters/InterpreterSelectWithUnionQuery.cpp @@ -53,9 +53,9 @@ struct CustomizeASTSelectWithUnionQueryNormalize auto & select_list = ast.list_of_selects->children; int i; + /// Rewrite UNION Mode for (i = union_modes.size() - 1; i >= 0; --i) { - /// Rewrite UNION Mode if (union_modes[i] == ASTSelectWithUnionQuery::Mode::Unspecified) { if (union_default_mode == UnionMode::ALL) @@ -67,7 +67,10 @@ struct CustomizeASTSelectWithUnionQueryNormalize "Expected ALL or DISTINCT in SelectWithUnion query, because setting (union_default_mode) is empty", DB::ErrorCodes::EXPECTED_ALL_OR_DISTINCT); } + } + for (i = union_modes.size() - 1; i >= 0; --i) + { if (union_modes[i] == ASTSelectWithUnionQuery::Mode::ALL) { if (auto * inner_union = select_list[i + 1]->as()) From d7bbffa410c2faddb3aff0393ac8cd7f76507e18 Mon Sep 17 00:00:00 2001 From: ikarishinjieva Date: Fri, 26 Feb 2021 13:38:59 +0800 Subject: [PATCH 064/544] fix #21170: use copy instead of move --- .../InterpreterSelectWithUnionQuery.cpp | 15 ++++++--------- 1 file changed, 6 insertions(+), 9 deletions(-) diff --git a/src/Interpreters/InterpreterSelectWithUnionQuery.cpp b/src/Interpreters/InterpreterSelectWithUnionQuery.cpp index 5f2728804ca..1d5b05ddd6b 100644 --- a/src/Interpreters/InterpreterSelectWithUnionQuery.cpp +++ b/src/Interpreters/InterpreterSelectWithUnionQuery.cpp @@ -43,7 +43,7 @@ struct CustomizeASTSelectWithUnionQueryNormalize return; } - selects.push_back(std::move(ast_select)); + selects.push_back(ast_select); } void visit(ASTSelectWithUnionQuery & ast, ASTPtr &) const @@ -53,9 +53,9 @@ struct CustomizeASTSelectWithUnionQueryNormalize auto & select_list = ast.list_of_selects->children; int i; - /// Rewrite UNION Mode for (i = union_modes.size() - 1; i >= 0; --i) { + /// Rewrite UNION Mode if (union_modes[i] == ASTSelectWithUnionQuery::Mode::Unspecified) { if (union_default_mode == UnionMode::ALL) @@ -67,10 +67,7 @@ struct CustomizeASTSelectWithUnionQueryNormalize "Expected ALL or DISTINCT in SelectWithUnion query, because setting (union_default_mode) is empty", DB::ErrorCodes::EXPECTED_ALL_OR_DISTINCT); } - } - for (i = union_modes.size() - 1; i >= 0; --i) - { if (union_modes[i] == ASTSelectWithUnionQuery::Mode::ALL) { if (auto * inner_union = select_list[i + 1]->as()) @@ -79,10 +76,10 @@ struct CustomizeASTSelectWithUnionQueryNormalize for (auto child = inner_union->list_of_selects->children.rbegin(); child != inner_union->list_of_selects->children.rend(); ++child) - selects.push_back(std::move(*child)); + selects.push_back(*child); } else - selects.push_back(std::move(select_list[i + 1])); + selects.push_back(select_list[i + 1]); } /// flatten all left nodes and current node to a UNION DISTINCT list else if (union_modes[i] == ASTSelectWithUnionQuery::Mode::DISTINCT) @@ -111,10 +108,10 @@ struct CustomizeASTSelectWithUnionQueryNormalize /// Inner_union is an UNION ALL list, just lift it up for (auto child = inner_union->list_of_selects->children.rbegin(); child != inner_union->list_of_selects->children.rend(); ++child) - selects.push_back(std::move(*child)); + selects.push_back(*child); } else - selects.push_back(std::move(select_list[0])); + selects.push_back(select_list[0]); } // reverse children list From 6fb68cd454919535b9009705c5ff26b79522caec Mon Sep 17 00:00:00 2001 From: ikarishinjieva Date: Mon, 1 Mar 2021 10:22:39 +0800 Subject: [PATCH 065/544] add functional test on fix #21170 --- tests/queries/0_stateless/01732_union_and_union_all.reference | 0 tests/queries/0_stateless/01732_union_and_union_all.sql | 1 + 2 files changed, 1 insertion(+) create mode 100644 tests/queries/0_stateless/01732_union_and_union_all.reference create mode 100644 tests/queries/0_stateless/01732_union_and_union_all.sql diff --git a/tests/queries/0_stateless/01732_union_and_union_all.reference b/tests/queries/0_stateless/01732_union_and_union_all.reference new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tests/queries/0_stateless/01732_union_and_union_all.sql b/tests/queries/0_stateless/01732_union_and_union_all.sql new file mode 100644 index 00000000000..2de6daa5bb9 --- /dev/null +++ b/tests/queries/0_stateless/01732_union_and_union_all.sql @@ -0,0 +1 @@ +select 1 UNION select 1 UNION ALL select 1; -- { serverError 558 } From f4b2cbc30f66371ab83eb0d59a7ecb55825c99f1 Mon Sep 17 00:00:00 2001 From: Vasily Nemkov Date: Sat, 27 Feb 2021 13:21:31 +0200 Subject: [PATCH 066/544] Fixed tests --- .../01691_DateTime64_clamp.reference | 23 ++++--------------- ..._toDateTime_from_string_clamping.reference | 8 +++---- 2 files changed, 9 insertions(+), 22 deletions(-) diff --git a/tests/queries/0_stateless/01691_DateTime64_clamp.reference b/tests/queries/0_stateless/01691_DateTime64_clamp.reference index da80de59e50..f29a9e2d1d5 100644 --- a/tests/queries/0_stateless/01691_DateTime64_clamp.reference +++ b/tests/queries/0_stateless/01691_DateTime64_clamp.reference @@ -1,11 +1,11 @@ -- { echo } -<<<<<<< HEAD +-- These values are within the extended range of DateTime64 [1925-01-01, 2284-01-01) SELECT toTimeZone(toDateTime(-2, 2), 'Europe/Moscow'); -1970-01-01 03:00:00.00 +1970-01-01 02:59:58.00 SELECT toDateTime64(-2, 2, 'Europe/Moscow'); -1970-01-01 03:00:00.00 +1970-01-01 02:59:58.00 SELECT CAST(-1 AS DateTime64(0, 'Europe/Moscow')); -1970-01-01 03:00:00 +1970-01-01 02:59:59 SELECT CAST('2020-01-01 00:00:00.3' AS DateTime64(0, 'Europe/Moscow')); 2020-01-01 00:00:00 SELECT toDateTime64(bitShiftLeft(toUInt64(1), 33), 2, 'Europe/Moscow') FORMAT Null; @@ -14,20 +14,8 @@ SELECT toTimeZone(toDateTime(-2., 2), 'Europe/Moscow'); SELECT toDateTime64(-2., 2, 'Europe/Moscow'); 1970-01-01 03:00:00.00 SELECT toDateTime64(toFloat32(bitShiftLeft(toUInt64(1),33)), 2, 'Europe/Moscow'); -2106-02-07 09:00:00.00 +2106-02-07 09:28:16.00 SELECT toDateTime64(toFloat64(bitShiftLeft(toUInt64(1),33)), 2, 'Europe/Moscow') FORMAT Null; -======= --- These values are within the extended range of DateTime64 [1925-01-01, 2284-01-01) -SELECT toDateTime(-2, 2); -1970-01-01 02:59:58.00 -SELECT toDateTime64(-2, 2); -1970-01-01 02:59:58.00 -SELECT CAST(-1 AS DateTime64); -1970-01-01 02:59:59.000 -SELECT CAST('2020-01-01 00:00:00.3' AS DateTime64); -2020-01-01 00:00:00.300 -SELECT toDateTime64(bitShiftLeft(toUInt64(1),33), 2); -2242-03-16 15:56:32.00 -- These are outsize of extended range and hence clamped SELECT toDateTime64(-1 * bitShiftLeft(toUInt64(1),35), 2); 1925-01-01 02:00:00.00 @@ -37,4 +25,3 @@ SELECT CAST(bitShiftLeft(toUInt64(1),35) AS DateTime64); 2282-12-31 03:00:00.000 SELECT toDateTime64(bitShiftLeft(toUInt64(1),35), 2); 2282-12-31 03:00:00.00 ->>>>>>> af31042451... Extended range of DateTime64 to years 1925 - 2238 diff --git a/tests/queries/0_stateless/01702_toDateTime_from_string_clamping.reference b/tests/queries/0_stateless/01702_toDateTime_from_string_clamping.reference index 228086615da..92639948fbc 100644 --- a/tests/queries/0_stateless/01702_toDateTime_from_string_clamping.reference +++ b/tests/queries/0_stateless/01702_toDateTime_from_string_clamping.reference @@ -1,9 +1,9 @@ -- { echo } SELECT toString(toDateTime('-922337203.6854775808', 1)); -2106-02-07 15:41:33.6 +1940-10-09 22:13:17.6 SELECT toString(toDateTime('9922337203.6854775808', 1)); -2104-12-30 00:50:11.6 +1925-07-26 00:46:43.6 SELECT toDateTime64(CAST('10000000000.1' AS Decimal64(1)), 1); -2106-02-07 20:50:08.1 +1928-01-11 00:46:40.1 SELECT toDateTime64(CAST('-10000000000.1' AS Decimal64(1)), 1); -2011-12-23 00:38:08.1 +2011-12-22 00:13:20.1 From 6e6ae1db5706e497a27e700772c835b032b9d3ad Mon Sep 17 00:00:00 2001 From: Vasily Nemkov Date: Tue, 2 Mar 2021 15:55:26 +0200 Subject: [PATCH 067/544] Nudge CI From 50c62c3b4cafdfbbbd965f1455f09d196cbcbb74 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Tue, 2 Mar 2021 22:09:37 +0300 Subject: [PATCH 068/544] Add memcpy implementation from @jart --- CMakeLists.txt | 3 +- base/common/CMakeLists.txt | 1 - base/common/tests/CMakeLists.txt | 2 +- base/glibc-compatibility/CMakeLists.txt | 17 +- .../glibc-compatibility/memcpy/CMakeLists.txt | 5 + base/glibc-compatibility/memcpy/memcpy.S | 151 ++++ base/glibc-compatibility/memcpy/memcpy.h | 28 + contrib/CMakeLists.txt | 1 - contrib/FastMemcpy/CMakeLists.txt | 28 - contrib/FastMemcpy/FastMemcpy.c | 220 ------ contrib/FastMemcpy/FastMemcpy.h | 694 ------------------ contrib/FastMemcpy/FastMemcpy_Avx.c | 171 ----- contrib/FastMemcpy/FastMemcpy_Avx.h | 492 ------------- contrib/FastMemcpy/LICENSE | 22 - contrib/FastMemcpy/README.md | 20 - contrib/FastMemcpy/memcpy_wrapper.c | 6 - 16 files changed, 193 insertions(+), 1668 deletions(-) create mode 100644 base/glibc-compatibility/memcpy/CMakeLists.txt create mode 100644 base/glibc-compatibility/memcpy/memcpy.S create mode 100644 base/glibc-compatibility/memcpy/memcpy.h delete mode 100644 contrib/FastMemcpy/CMakeLists.txt delete mode 100644 contrib/FastMemcpy/FastMemcpy.c delete mode 100644 contrib/FastMemcpy/FastMemcpy.h delete mode 100644 contrib/FastMemcpy/FastMemcpy_Avx.c delete mode 100644 contrib/FastMemcpy/FastMemcpy_Avx.h delete mode 100644 contrib/FastMemcpy/LICENSE delete mode 100644 contrib/FastMemcpy/README.md delete mode 100644 contrib/FastMemcpy/memcpy_wrapper.c diff --git a/CMakeLists.txt b/CMakeLists.txt index 9002f1df140..e622cbdebe6 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -155,7 +155,6 @@ option(ENABLE_TESTS "Provide unit_test_dbms target with Google.Test unit tests" if (OS_LINUX AND NOT UNBUNDLED AND MAKE_STATIC_LIBRARIES AND NOT SPLIT_SHARED_LIBRARIES AND CMAKE_VERSION VERSION_GREATER "3.9.0") # Only for Linux, x86_64. - # Implies ${ENABLE_FASTMEMCPY} option(GLIBC_COMPATIBILITY "Enable compatibility with older glibc libraries." ON) elseif(GLIBC_COMPATIBILITY) message (${RECONFIGURE_MESSAGE_LEVEL} "Glibc compatibility cannot be enabled in current configuration") @@ -536,7 +535,7 @@ macro (add_executable target) # explicitly acquire and interpose malloc symbols by clickhouse_malloc # if GLIBC_COMPATIBILITY is ON and ENABLE_THINLTO is on than provide memcpy symbol explicitly to neutrialize thinlto's libcall generation. if (GLIBC_COMPATIBILITY AND ENABLE_THINLTO) - _add_executable (${ARGV} $ $) + _add_executable (${ARGV} $ $) else () _add_executable (${ARGV} $) endif () diff --git a/base/common/CMakeLists.txt b/base/common/CMakeLists.txt index cea52b443dd..b4bf4f55466 100644 --- a/base/common/CMakeLists.txt +++ b/base/common/CMakeLists.txt @@ -74,7 +74,6 @@ target_link_libraries (common ${CITYHASH_LIBRARIES} boost::headers_only boost::system - FastMemcpy Poco::Net Poco::Net::SSL Poco::Util diff --git a/base/common/tests/CMakeLists.txt b/base/common/tests/CMakeLists.txt index b7082ee9900..6775d443fb6 100644 --- a/base/common/tests/CMakeLists.txt +++ b/base/common/tests/CMakeLists.txt @@ -11,7 +11,7 @@ set(PLATFORM_LIBS ${CMAKE_DL_LIBS}) target_link_libraries (date_lut2 PRIVATE common ${PLATFORM_LIBS}) target_link_libraries (date_lut3 PRIVATE common ${PLATFORM_LIBS}) target_link_libraries (date_lut_default_timezone PRIVATE common ${PLATFORM_LIBS}) -target_link_libraries (local_date_time_comparison PRIVATE common) +target_link_libraries (local_date_time_comparison PRIVATE common ${PLATFORM_LIBS}) target_link_libraries (realloc-perf PRIVATE common) add_check(local_date_time_comparison) diff --git a/base/glibc-compatibility/CMakeLists.txt b/base/glibc-compatibility/CMakeLists.txt index 684c6162941..1fc537ded24 100644 --- a/base/glibc-compatibility/CMakeLists.txt +++ b/base/glibc-compatibility/CMakeLists.txt @@ -1,5 +1,5 @@ if (GLIBC_COMPATIBILITY) - set (ENABLE_FASTMEMCPY ON) + add_subdirectory(memcpy) enable_language(ASM) include(CheckIncludeFile) @@ -27,13 +27,6 @@ if (GLIBC_COMPATIBILITY) list(APPEND glibc_compatibility_sources musl/getentropy.c) endif() - if (NOT ARCH_ARM) - # clickhouse_memcpy don't support ARCH_ARM, see https://github.com/ClickHouse/ClickHouse/issues/18951 - add_library (clickhouse_memcpy OBJECT - ${ClickHouse_SOURCE_DIR}/contrib/FastMemcpy/memcpy_wrapper.c - ) - endif() - # Need to omit frame pointers to match the performance of glibc set (CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -fomit-frame-pointer") @@ -51,15 +44,19 @@ if (GLIBC_COMPATIBILITY) target_compile_options(glibc-compatibility PRIVATE -fPIC) endif () - target_link_libraries(global-libs INTERFACE glibc-compatibility) + target_link_libraries(global-libs INTERFACE glibc-compatibility memcpy) install( - TARGETS glibc-compatibility + TARGETS glibc-compatibility memcpy EXPORT global ARCHIVE DESTINATION lib ) message (STATUS "Some symbols from glibc will be replaced for compatibility") + + if (ENABLE_TESTS) + add_subdirectory (tests) + endif () elseif (YANDEX_OFFICIAL_BUILD) message (WARNING "Option GLIBC_COMPATIBILITY must be turned on for production builds.") endif () diff --git a/base/glibc-compatibility/memcpy/CMakeLists.txt b/base/glibc-compatibility/memcpy/CMakeLists.txt new file mode 100644 index 00000000000..b51ebab1d50 --- /dev/null +++ b/base/glibc-compatibility/memcpy/CMakeLists.txt @@ -0,0 +1,5 @@ +enable_language(ASM) +add_library(memcpy STATIC memcpy.S) + +# We allow to include memcpy.h from user code for better inlining (less number of registers are clobbered?). +target_include_directories(memcpy PUBLIC $) diff --git a/base/glibc-compatibility/memcpy/memcpy.S b/base/glibc-compatibility/memcpy/memcpy.S new file mode 100644 index 00000000000..c1439c3c859 --- /dev/null +++ b/base/glibc-compatibility/memcpy/memcpy.S @@ -0,0 +1,151 @@ +/*-*- mode:unix-assembly; indent-tabs-mode:t; tab-width:8; coding:utf-8 -*-│ +│vi: set et ft=asm ts=8 tw=8 fenc=utf-8 :vi│ +╞══════════════════════════════════════════════════════════════════════════════╡ +│ Copyright 2020 Justine Alexandra Roberts Tunney │ +│ │ +│ Permission to use, copy, modify, and/or distribute this software for │ +│ any purpose with or without fee is hereby granted, provided that the │ +│ above copyright notice and this permission notice appear in all copies. │ +│ │ +│ THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL │ +│ WARRANTIES WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED │ +│ WARRANTIES OF MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE │ +│ AUTHOR BE LIABLE FOR ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL │ +│ DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR │ +│ PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER │ +│ TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR │ +│ PERFORMANCE OF THIS SOFTWARE. │ +╚─────────────────────────────────────────────────────────────────────────────*/ + +// Copies memory. +// +// DEST and SRC must not overlap, unless DEST≤SRC. +// +// @param rdi is dest +// @param rsi is src +// @param rdx is number of bytes +// @return original rdi copied to rax +// @mode long +// @asyncsignalsafe +memcpy: mov %rdi,%rax +// 𝑠𝑙𝑖𝑑𝑒 + .align 16 + .type memcpy,@function + .size memcpy,.-memcpy + .globl memcpy + +// Copies memory w/ minimal impact ABI. +// +// @param rdi is dest +// @param rsi is src +// @param rdx is number of bytes +// @clob flags,rcx,xmm3,xmm4 +// @mode long +MemCpy: mov $.Lmemcpytab.size,%ecx + cmp %rcx,%rdx + cmovb %rdx,%rcx + jmp *memcpytab(,%rcx,8) +.Lanchorpoint: +.L32r: cmp $1024,%rdx + jae .Lerms +.L32: vmovdqu -32(%rsi,%rdx),%ymm4 + mov $32,%rcx +0: add $32,%rcx + vmovdqu -64(%rsi,%rcx),%ymm3 + vmovdqu %ymm3,-64(%rdi,%rcx) + cmp %rcx,%rdx + ja 0b + vmovdqu %ymm4,-32(%rdi,%rdx) + vxorps %ymm4,%ymm4,%ymm4 + vxorps %ymm3,%ymm3,%ymm3 + jmp .L0 +.L16r: cmp $1024,%rdx + jae .Lerms +.L16: movdqu -16(%rsi,%rdx),%xmm4 + mov $16,%rcx +0: add $16,%rcx + movdqu -32(%rsi,%rcx),%xmm3 + movdqu %xmm3,-32(%rdi,%rcx) + cmp %rcx,%rdx + ja 0b + movdqu %xmm4,-16(%rdi,%rdx) + pxor %xmm4,%xmm4 + pxor %xmm3,%xmm3 + jmp .L0 +.L8: push %rbx + mov (%rsi),%rcx + mov -8(%rsi,%rdx),%rbx + mov %rcx,(%rdi) + mov %rbx,-8(%rdi,%rdx) +1: pop %rbx +.L0: ret +.L4: push %rbx + mov (%rsi),%ecx + mov -4(%rsi,%rdx),%ebx + mov %ecx,(%rdi) + mov %ebx,-4(%rdi,%rdx) + jmp 1b +.L3: push %rbx + mov (%rsi),%cx + mov -2(%rsi,%rdx),%bx + mov %cx,(%rdi) + mov %bx,-2(%rdi,%rdx) + jmp 1b +.L2: mov (%rsi),%cx + mov %cx,(%rdi) + jmp .L0 +.L1: mov (%rsi),%cl + mov %cl,(%rdi) + jmp .L0 +.Lerms: cmp $1024*1024,%rdx + ja .Lnts + push %rdi + push %rsi + mov %rdx,%rcx + rep movsb + pop %rsi + pop %rdi + jmp .L0 +.Lnts: movdqu (%rsi),%xmm3 + movdqu %xmm3,(%rdi) + lea 16(%rdi),%rcx + and $-16,%rcx + sub %rdi,%rcx + add %rcx,%rdi + add %rcx,%rsi + sub %rcx,%rdx + mov $16,%rcx +0: add $16,%rcx + movdqu -32(%rsi,%rcx),%xmm3 + movntdq %xmm3,-32(%rdi,%rcx) + cmp %rcx,%rdx + ja 0b + sfence + movdqu -16(%rsi,%rdx),%xmm3 + movdqu %xmm3,-16(%rdi,%rdx) + pxor %xmm3,%xmm3 + jmp .L0 + .type MemCpy,@function + .size MemCpy,.-MemCpy + .globl MemCpy + + .section .rodata + .align 8 +memcpytab: + .quad .L0 + .quad .L1 + .quad .L2 + .quad .L3 + .rept 4 + .quad .L4 + .endr + .rept 8 + .quad .L8 + .endr + .rept 16 + .quad .L16 + .endr + .equ .Lmemcpytab.size,(.-memcpytab)/8 + .quad .L32r # AVX + ERMS + .type memcpytab,@object + .previous diff --git a/base/glibc-compatibility/memcpy/memcpy.h b/base/glibc-compatibility/memcpy/memcpy.h new file mode 100644 index 00000000000..7e3069d6b79 --- /dev/null +++ b/base/glibc-compatibility/memcpy/memcpy.h @@ -0,0 +1,28 @@ +#ifndef MEMCPY_H_ +#define MEMCPY_H_ +#if !(__ASSEMBLER__ + __LINKER__ + 0) + +/// Note: this header can be included only after libc++ headers. +/// Note: you cannot write std::memcpy if you include this header. + +void *memcpy(void *, const void *, size_t); + +#ifdef __GNUC__ +#define __memcpy_isgoodsize(SIZE) \ + (__builtin_constant_p(SIZE) && ((SIZE) <= __BIGGEST_ALIGNMENT__ && \ + __builtin_popcountl((unsigned)(SIZE)) == 1)) +#define memcpy(DEST, SRC, SIZE) \ + (__memcpy_isgoodsize(SIZE) ? __builtin_memcpy(DEST, SRC, SIZE) : ({ \ + void *DeSt = (DEST); \ + const void *SrC = (SRC); \ + size_t SiZe = (SIZE); \ + asm("call\tMemCpy" \ + : "=m"(*(char(*)[SiZe])(DeSt)) \ + : "D"(DeSt), "S"(SrC), "d"(SiZe), "m"(*(const char(*)[SiZe])(SrC)) \ + : "xmm3", "xmm4", "rcx", "cc"); \ + DeSt; \ + })) +#endif + +#endif /* !(__ASSEMBLER__ + __LINKER__ + 0) */ +#endif /* MEMCPY_H_ */ diff --git a/contrib/CMakeLists.txt b/contrib/CMakeLists.txt index 20b4fad0437..3a635bda83a 100644 --- a/contrib/CMakeLists.txt +++ b/contrib/CMakeLists.txt @@ -37,7 +37,6 @@ add_subdirectory (boost-cmake) add_subdirectory (cctz-cmake) add_subdirectory (consistent-hashing) add_subdirectory (dragonbox-cmake) -add_subdirectory (FastMemcpy) add_subdirectory (hyperscan-cmake) add_subdirectory (jemalloc-cmake) add_subdirectory (libcpuid-cmake) diff --git a/contrib/FastMemcpy/CMakeLists.txt b/contrib/FastMemcpy/CMakeLists.txt deleted file mode 100644 index 8efe6d45dff..00000000000 --- a/contrib/FastMemcpy/CMakeLists.txt +++ /dev/null @@ -1,28 +0,0 @@ -option (ENABLE_FASTMEMCPY "Enable FastMemcpy library (only internal)" ${ENABLE_LIBRARIES}) - -if (NOT OS_LINUX OR ARCH_AARCH64) - set (ENABLE_FASTMEMCPY OFF) -endif () - -if (ENABLE_FASTMEMCPY) - set (LIBRARY_DIR ${ClickHouse_SOURCE_DIR}/contrib/FastMemcpy) - - set (SRCS - ${LIBRARY_DIR}/FastMemcpy.c - - memcpy_wrapper.c - ) - - add_library (FastMemcpy ${SRCS}) - target_include_directories (FastMemcpy PUBLIC ${LIBRARY_DIR}) - - target_compile_definitions(FastMemcpy PUBLIC USE_FASTMEMCPY=1) - - message (STATUS "Using FastMemcpy") -else () - add_library (FastMemcpy INTERFACE) - - target_compile_definitions(FastMemcpy INTERFACE USE_FASTMEMCPY=0) - - message (STATUS "Not using FastMemcpy") -endif () diff --git a/contrib/FastMemcpy/FastMemcpy.c b/contrib/FastMemcpy/FastMemcpy.c deleted file mode 100644 index 5021bcc7d16..00000000000 --- a/contrib/FastMemcpy/FastMemcpy.c +++ /dev/null @@ -1,220 +0,0 @@ -//===================================================================== -// -// FastMemcpy.c - skywind3000@163.com, 2015 -// -// feature: -// 50% speed up in avg. vs standard memcpy (tested in vc2012/gcc4.9) -// -//===================================================================== -#include -#include -#include -#include - -#if (defined(_WIN32) || defined(WIN32)) -#include -#include -#ifdef _MSC_VER -#pragma comment(lib, "winmm.lib") -#endif -#elif defined(__unix) -#include -#include -#else -#error it can only be compiled under windows or unix -#endif - -#include "FastMemcpy.h" - -unsigned int gettime() -{ - #if (defined(_WIN32) || defined(WIN32)) - return timeGetTime(); - #else - static struct timezone tz={ 0,0 }; - struct timeval time; - gettimeofday(&time,&tz); - return (time.tv_sec * 1000 + time.tv_usec / 1000); - #endif -} - -void sleepms(unsigned int millisec) -{ -#if defined(_WIN32) || defined(WIN32) - Sleep(millisec); -#else - usleep(millisec * 1000); -#endif -} - - -void benchmark(int dstalign, int srcalign, size_t size, int times) -{ - char *DATA1 = (char*)malloc(size + 64); - char *DATA2 = (char*)malloc(size + 64); - size_t LINEAR1 = ((size_t)DATA1); - size_t LINEAR2 = ((size_t)DATA2); - char *ALIGN1 = (char*)(((64 - (LINEAR1 & 63)) & 63) + LINEAR1); - char *ALIGN2 = (char*)(((64 - (LINEAR2 & 63)) & 63) + LINEAR2); - char *dst = (dstalign)? ALIGN1 : (ALIGN1 + 1); - char *src = (srcalign)? ALIGN2 : (ALIGN2 + 3); - unsigned int t1, t2; - int k; - - sleepms(100); - t1 = gettime(); - for (k = times; k > 0; k--) { - memcpy(dst, src, size); - } - t1 = gettime() - t1; - sleepms(100); - t2 = gettime(); - for (k = times; k > 0; k--) { - memcpy_fast(dst, src, size); - } - t2 = gettime() - t2; - - free(DATA1); - free(DATA2); - - printf("result(dst %s, src %s): memcpy_fast=%dms memcpy=%d ms\n", - dstalign? "aligned" : "unalign", - srcalign? "aligned" : "unalign", (int)t2, (int)t1); -} - - -void bench(int copysize, int times) -{ - printf("benchmark(size=%d bytes, times=%d):\n", copysize, times); - benchmark(1, 1, copysize, times); - benchmark(1, 0, copysize, times); - benchmark(0, 1, copysize, times); - benchmark(0, 0, copysize, times); - printf("\n"); -} - - -void random_bench(int maxsize, int times) -{ - static char A[11 * 1024 * 1024 + 2]; - static char B[11 * 1024 * 1024 + 2]; - static int random_offsets[0x10000]; - static int random_sizes[0x8000]; - unsigned int i, p1, p2; - unsigned int t1, t2; - for (i = 0; i < 0x10000; i++) { // generate random offsets - random_offsets[i] = rand() % (10 * 1024 * 1024 + 1); - } - for (i = 0; i < 0x8000; i++) { // generate random sizes - random_sizes[i] = 1 + rand() % maxsize; - } - sleepms(100); - t1 = gettime(); - for (p1 = 0, p2 = 0, i = 0; i < times; i++) { - int offset1 = random_offsets[(p1++) & 0xffff]; - int offset2 = random_offsets[(p1++) & 0xffff]; - int size = random_sizes[(p2++) & 0x7fff]; - memcpy(A + offset1, B + offset2, size); - } - t1 = gettime() - t1; - sleepms(100); - t2 = gettime(); - for (p1 = 0, p2 = 0, i = 0; i < times; i++) { - int offset1 = random_offsets[(p1++) & 0xffff]; - int offset2 = random_offsets[(p1++) & 0xffff]; - int size = random_sizes[(p2++) & 0x7fff]; - memcpy_fast(A + offset1, B + offset2, size); - } - t2 = gettime() - t2; - printf("benchmark random access:\n"); - printf("memcpy_fast=%dms memcpy=%dms\n\n", (int)t2, (int)t1); -} - - -#ifdef _MSC_VER -#pragma comment(lib, "winmm.lib") -#endif - -int main(void) -{ - bench(32, 0x1000000); - bench(64, 0x1000000); - bench(512, 0x800000); - bench(1024, 0x400000); - bench(4096, 0x80000); - bench(8192, 0x40000); - bench(1024 * 1024 * 1, 0x800); - bench(1024 * 1024 * 4, 0x200); - bench(1024 * 1024 * 8, 0x100); - - random_bench(2048, 8000000); - - return 0; -} - - - - -/* -benchmark(size=32 bytes, times=16777216): -result(dst aligned, src aligned): memcpy_fast=78ms memcpy=260 ms -result(dst aligned, src unalign): memcpy_fast=78ms memcpy=250 ms -result(dst unalign, src aligned): memcpy_fast=78ms memcpy=266 ms -result(dst unalign, src unalign): memcpy_fast=78ms memcpy=234 ms - -benchmark(size=64 bytes, times=16777216): -result(dst aligned, src aligned): memcpy_fast=109ms memcpy=281 ms -result(dst aligned, src unalign): memcpy_fast=109ms memcpy=328 ms -result(dst unalign, src aligned): memcpy_fast=109ms memcpy=343 ms -result(dst unalign, src unalign): memcpy_fast=93ms memcpy=344 ms - -benchmark(size=512 bytes, times=8388608): -result(dst aligned, src aligned): memcpy_fast=125ms memcpy=218 ms -result(dst aligned, src unalign): memcpy_fast=156ms memcpy=484 ms -result(dst unalign, src aligned): memcpy_fast=172ms memcpy=546 ms -result(dst unalign, src unalign): memcpy_fast=172ms memcpy=515 ms - -benchmark(size=1024 bytes, times=4194304): -result(dst aligned, src aligned): memcpy_fast=109ms memcpy=172 ms -result(dst aligned, src unalign): memcpy_fast=187ms memcpy=453 ms -result(dst unalign, src aligned): memcpy_fast=172ms memcpy=437 ms -result(dst unalign, src unalign): memcpy_fast=156ms memcpy=452 ms - -benchmark(size=4096 bytes, times=524288): -result(dst aligned, src aligned): memcpy_fast=62ms memcpy=78 ms -result(dst aligned, src unalign): memcpy_fast=109ms memcpy=202 ms -result(dst unalign, src aligned): memcpy_fast=94ms memcpy=203 ms -result(dst unalign, src unalign): memcpy_fast=110ms memcpy=218 ms - -benchmark(size=8192 bytes, times=262144): -result(dst aligned, src aligned): memcpy_fast=62ms memcpy=78 ms -result(dst aligned, src unalign): memcpy_fast=78ms memcpy=202 ms -result(dst unalign, src aligned): memcpy_fast=78ms memcpy=203 ms -result(dst unalign, src unalign): memcpy_fast=94ms memcpy=203 ms - -benchmark(size=1048576 bytes, times=2048): -result(dst aligned, src aligned): memcpy_fast=203ms memcpy=191 ms -result(dst aligned, src unalign): memcpy_fast=219ms memcpy=281 ms -result(dst unalign, src aligned): memcpy_fast=218ms memcpy=328 ms -result(dst unalign, src unalign): memcpy_fast=218ms memcpy=312 ms - -benchmark(size=4194304 bytes, times=512): -result(dst aligned, src aligned): memcpy_fast=312ms memcpy=406 ms -result(dst aligned, src unalign): memcpy_fast=296ms memcpy=421 ms -result(dst unalign, src aligned): memcpy_fast=312ms memcpy=468 ms -result(dst unalign, src unalign): memcpy_fast=297ms memcpy=452 ms - -benchmark(size=8388608 bytes, times=256): -result(dst aligned, src aligned): memcpy_fast=281ms memcpy=452 ms -result(dst aligned, src unalign): memcpy_fast=280ms memcpy=468 ms -result(dst unalign, src aligned): memcpy_fast=298ms memcpy=514 ms -result(dst unalign, src unalign): memcpy_fast=344ms memcpy=472 ms - -benchmark random access: -memcpy_fast=515ms memcpy=1014ms - -*/ - - - - diff --git a/contrib/FastMemcpy/FastMemcpy.h b/contrib/FastMemcpy/FastMemcpy.h deleted file mode 100644 index 5dcbfcf1656..00000000000 --- a/contrib/FastMemcpy/FastMemcpy.h +++ /dev/null @@ -1,694 +0,0 @@ -//===================================================================== -// -// FastMemcpy.c - skywind3000@163.com, 2015 -// -// feature: -// 50% speed up in avg. vs standard memcpy (tested in vc2012/gcc5.1) -// -//===================================================================== -#ifndef __FAST_MEMCPY_H__ -#define __FAST_MEMCPY_H__ - -#include -#include -#include - - -//--------------------------------------------------------------------- -// force inline for compilers -//--------------------------------------------------------------------- -#ifndef INLINE -#ifdef __GNUC__ -#if (__GNUC__ > 3) || ((__GNUC__ == 3) && (__GNUC_MINOR__ >= 1)) - #define INLINE __inline__ __attribute__((always_inline)) -#else - #define INLINE __inline__ -#endif -#elif defined(_MSC_VER) - #define INLINE __forceinline -#elif (defined(__BORLANDC__) || defined(__WATCOMC__)) - #define INLINE __inline -#else - #define INLINE -#endif -#endif - -typedef __attribute__((__aligned__(1))) uint16_t uint16_unaligned_t; -typedef __attribute__((__aligned__(1))) uint32_t uint32_unaligned_t; -typedef __attribute__((__aligned__(1))) uint64_t uint64_unaligned_t; - -//--------------------------------------------------------------------- -// fast copy for different sizes -//--------------------------------------------------------------------- -static INLINE void memcpy_sse2_16(void *dst, const void *src) { - __m128i m0 = _mm_loadu_si128(((const __m128i*)src) + 0); - _mm_storeu_si128(((__m128i*)dst) + 0, m0); -} - -static INLINE void memcpy_sse2_32(void *dst, const void *src) { - __m128i m0 = _mm_loadu_si128(((const __m128i*)src) + 0); - __m128i m1 = _mm_loadu_si128(((const __m128i*)src) + 1); - _mm_storeu_si128(((__m128i*)dst) + 0, m0); - _mm_storeu_si128(((__m128i*)dst) + 1, m1); -} - -static INLINE void memcpy_sse2_64(void *dst, const void *src) { - __m128i m0 = _mm_loadu_si128(((const __m128i*)src) + 0); - __m128i m1 = _mm_loadu_si128(((const __m128i*)src) + 1); - __m128i m2 = _mm_loadu_si128(((const __m128i*)src) + 2); - __m128i m3 = _mm_loadu_si128(((const __m128i*)src) + 3); - _mm_storeu_si128(((__m128i*)dst) + 0, m0); - _mm_storeu_si128(((__m128i*)dst) + 1, m1); - _mm_storeu_si128(((__m128i*)dst) + 2, m2); - _mm_storeu_si128(((__m128i*)dst) + 3, m3); -} - -static INLINE void memcpy_sse2_128(void *dst, const void *src) { - __m128i m0 = _mm_loadu_si128(((const __m128i*)src) + 0); - __m128i m1 = _mm_loadu_si128(((const __m128i*)src) + 1); - __m128i m2 = _mm_loadu_si128(((const __m128i*)src) + 2); - __m128i m3 = _mm_loadu_si128(((const __m128i*)src) + 3); - __m128i m4 = _mm_loadu_si128(((const __m128i*)src) + 4); - __m128i m5 = _mm_loadu_si128(((const __m128i*)src) + 5); - __m128i m6 = _mm_loadu_si128(((const __m128i*)src) + 6); - __m128i m7 = _mm_loadu_si128(((const __m128i*)src) + 7); - _mm_storeu_si128(((__m128i*)dst) + 0, m0); - _mm_storeu_si128(((__m128i*)dst) + 1, m1); - _mm_storeu_si128(((__m128i*)dst) + 2, m2); - _mm_storeu_si128(((__m128i*)dst) + 3, m3); - _mm_storeu_si128(((__m128i*)dst) + 4, m4); - _mm_storeu_si128(((__m128i*)dst) + 5, m5); - _mm_storeu_si128(((__m128i*)dst) + 6, m6); - _mm_storeu_si128(((__m128i*)dst) + 7, m7); -} - - -//--------------------------------------------------------------------- -// tiny memory copy with jump table optimized -//--------------------------------------------------------------------- -/// Attribute is used to avoid an error with undefined behaviour sanitizer -/// ../contrib/FastMemcpy/FastMemcpy.h:91:56: runtime error: applying zero offset to null pointer -/// Found by 01307_orc_output_format.sh, cause - ORCBlockInputFormat and external ORC library. -__attribute__((__no_sanitize__("undefined"))) static INLINE void *memcpy_tiny(void *dst, const void *src, size_t size) { - unsigned char *dd = ((unsigned char*)dst) + size; - const unsigned char *ss = ((const unsigned char*)src) + size; - - switch (size) { - case 64: - memcpy_sse2_64(dd - 64, ss - 64); - case 0: - break; - - case 65: - memcpy_sse2_64(dd - 65, ss - 65); - case 1: - dd[-1] = ss[-1]; - break; - - case 66: - memcpy_sse2_64(dd - 66, ss - 66); - case 2: - *((uint16_unaligned_t*)(dd - 2)) = *((uint16_unaligned_t*)(ss - 2)); - break; - - case 67: - memcpy_sse2_64(dd - 67, ss - 67); - case 3: - *((uint16_unaligned_t*)(dd - 3)) = *((uint16_unaligned_t*)(ss - 3)); - dd[-1] = ss[-1]; - break; - - case 68: - memcpy_sse2_64(dd - 68, ss - 68); - case 4: - *((uint32_unaligned_t*)(dd - 4)) = *((uint32_unaligned_t*)(ss - 4)); - break; - - case 69: - memcpy_sse2_64(dd - 69, ss - 69); - case 5: - *((uint32_unaligned_t*)(dd - 5)) = *((uint32_unaligned_t*)(ss - 5)); - dd[-1] = ss[-1]; - break; - - case 70: - memcpy_sse2_64(dd - 70, ss - 70); - case 6: - *((uint32_unaligned_t*)(dd - 6)) = *((uint32_unaligned_t*)(ss - 6)); - *((uint16_unaligned_t*)(dd - 2)) = *((uint16_unaligned_t*)(ss - 2)); - break; - - case 71: - memcpy_sse2_64(dd - 71, ss - 71); - case 7: - *((uint32_unaligned_t*)(dd - 7)) = *((uint32_unaligned_t*)(ss - 7)); - *((uint32_unaligned_t*)(dd - 4)) = *((uint32_unaligned_t*)(ss - 4)); - break; - - case 72: - memcpy_sse2_64(dd - 72, ss - 72); - case 8: - *((uint64_unaligned_t*)(dd - 8)) = *((uint64_unaligned_t*)(ss - 8)); - break; - - case 73: - memcpy_sse2_64(dd - 73, ss - 73); - case 9: - *((uint64_unaligned_t*)(dd - 9)) = *((uint64_unaligned_t*)(ss - 9)); - dd[-1] = ss[-1]; - break; - - case 74: - memcpy_sse2_64(dd - 74, ss - 74); - case 10: - *((uint64_unaligned_t*)(dd - 10)) = *((uint64_unaligned_t*)(ss - 10)); - *((uint16_unaligned_t*)(dd - 2)) = *((uint16_unaligned_t*)(ss - 2)); - break; - - case 75: - memcpy_sse2_64(dd - 75, ss - 75); - case 11: - *((uint64_unaligned_t*)(dd - 11)) = *((uint64_unaligned_t*)(ss - 11)); - *((uint32_unaligned_t*)(dd - 4)) = *((uint32_unaligned_t*)(ss - 4)); - break; - - case 76: - memcpy_sse2_64(dd - 76, ss - 76); - case 12: - *((uint64_unaligned_t*)(dd - 12)) = *((uint64_unaligned_t*)(ss - 12)); - *((uint32_unaligned_t*)(dd - 4)) = *((uint32_unaligned_t*)(ss - 4)); - break; - - case 77: - memcpy_sse2_64(dd - 77, ss - 77); - case 13: - *((uint64_unaligned_t*)(dd - 13)) = *((uint64_unaligned_t*)(ss - 13)); - *((uint32_unaligned_t*)(dd - 5)) = *((uint32_unaligned_t*)(ss - 5)); - dd[-1] = ss[-1]; - break; - - case 78: - memcpy_sse2_64(dd - 78, ss - 78); - case 14: - *((uint64_unaligned_t*)(dd - 14)) = *((uint64_unaligned_t*)(ss - 14)); - *((uint64_unaligned_t*)(dd - 8)) = *((uint64_unaligned_t*)(ss - 8)); - break; - - case 79: - memcpy_sse2_64(dd - 79, ss - 79); - case 15: - *((uint64_unaligned_t*)(dd - 15)) = *((uint64_unaligned_t*)(ss - 15)); - *((uint64_unaligned_t*)(dd - 8)) = *((uint64_unaligned_t*)(ss - 8)); - break; - - case 80: - memcpy_sse2_64(dd - 80, ss - 80); - case 16: - memcpy_sse2_16(dd - 16, ss - 16); - break; - - case 81: - memcpy_sse2_64(dd - 81, ss - 81); - case 17: - memcpy_sse2_16(dd - 17, ss - 17); - dd[-1] = ss[-1]; - break; - - case 82: - memcpy_sse2_64(dd - 82, ss - 82); - case 18: - memcpy_sse2_16(dd - 18, ss - 18); - *((uint16_unaligned_t*)(dd - 2)) = *((uint16_unaligned_t*)(ss - 2)); - break; - - case 83: - memcpy_sse2_64(dd - 83, ss - 83); - case 19: - memcpy_sse2_16(dd - 19, ss - 19); - *((uint16_unaligned_t*)(dd - 3)) = *((uint16_unaligned_t*)(ss - 3)); - dd[-1] = ss[-1]; - break; - - case 84: - memcpy_sse2_64(dd - 84, ss - 84); - case 20: - memcpy_sse2_16(dd - 20, ss - 20); - *((uint32_unaligned_t*)(dd - 4)) = *((uint32_unaligned_t*)(ss - 4)); - break; - - case 85: - memcpy_sse2_64(dd - 85, ss - 85); - case 21: - memcpy_sse2_16(dd - 21, ss - 21); - *((uint32_unaligned_t*)(dd - 5)) = *((uint32_unaligned_t*)(ss - 5)); - dd[-1] = ss[-1]; - break; - - case 86: - memcpy_sse2_64(dd - 86, ss - 86); - case 22: - memcpy_sse2_16(dd - 22, ss - 22); - *((uint32_unaligned_t*)(dd - 6)) = *((uint32_unaligned_t*)(ss - 6)); - *((uint16_unaligned_t*)(dd - 2)) = *((uint16_unaligned_t*)(ss - 2)); - break; - - case 87: - memcpy_sse2_64(dd - 87, ss - 87); - case 23: - memcpy_sse2_16(dd - 23, ss - 23); - *((uint32_unaligned_t*)(dd - 7)) = *((uint32_unaligned_t*)(ss - 7)); - *((uint32_unaligned_t*)(dd - 4)) = *((uint32_unaligned_t*)(ss - 4)); - break; - - case 88: - memcpy_sse2_64(dd - 88, ss - 88); - case 24: - memcpy_sse2_16(dd - 24, ss - 24); - memcpy_sse2_16(dd - 16, ss - 16); - break; - - case 89: - memcpy_sse2_64(dd - 89, ss - 89); - case 25: - memcpy_sse2_16(dd - 25, ss - 25); - memcpy_sse2_16(dd - 16, ss - 16); - break; - - case 90: - memcpy_sse2_64(dd - 90, ss - 90); - case 26: - memcpy_sse2_16(dd - 26, ss - 26); - memcpy_sse2_16(dd - 16, ss - 16); - break; - - case 91: - memcpy_sse2_64(dd - 91, ss - 91); - case 27: - memcpy_sse2_16(dd - 27, ss - 27); - memcpy_sse2_16(dd - 16, ss - 16); - break; - - case 92: - memcpy_sse2_64(dd - 92, ss - 92); - case 28: - memcpy_sse2_16(dd - 28, ss - 28); - memcpy_sse2_16(dd - 16, ss - 16); - break; - - case 93: - memcpy_sse2_64(dd - 93, ss - 93); - case 29: - memcpy_sse2_16(dd - 29, ss - 29); - memcpy_sse2_16(dd - 16, ss - 16); - break; - - case 94: - memcpy_sse2_64(dd - 94, ss - 94); - case 30: - memcpy_sse2_16(dd - 30, ss - 30); - memcpy_sse2_16(dd - 16, ss - 16); - break; - - case 95: - memcpy_sse2_64(dd - 95, ss - 95); - case 31: - memcpy_sse2_16(dd - 31, ss - 31); - memcpy_sse2_16(dd - 16, ss - 16); - break; - - case 96: - memcpy_sse2_64(dd - 96, ss - 96); - case 32: - memcpy_sse2_32(dd - 32, ss - 32); - break; - - case 97: - memcpy_sse2_64(dd - 97, ss - 97); - case 33: - memcpy_sse2_32(dd - 33, ss - 33); - dd[-1] = ss[-1]; - break; - - case 98: - memcpy_sse2_64(dd - 98, ss - 98); - case 34: - memcpy_sse2_32(dd - 34, ss - 34); - *((uint16_unaligned_t*)(dd - 2)) = *((uint16_unaligned_t*)(ss - 2)); - break; - - case 99: - memcpy_sse2_64(dd - 99, ss - 99); - case 35: - memcpy_sse2_32(dd - 35, ss - 35); - *((uint16_unaligned_t*)(dd - 3)) = *((uint16_unaligned_t*)(ss - 3)); - dd[-1] = ss[-1]; - break; - - case 100: - memcpy_sse2_64(dd - 100, ss - 100); - case 36: - memcpy_sse2_32(dd - 36, ss - 36); - *((uint32_unaligned_t*)(dd - 4)) = *((uint32_unaligned_t*)(ss - 4)); - break; - - case 101: - memcpy_sse2_64(dd - 101, ss - 101); - case 37: - memcpy_sse2_32(dd - 37, ss - 37); - *((uint32_unaligned_t*)(dd - 5)) = *((uint32_unaligned_t*)(ss - 5)); - dd[-1] = ss[-1]; - break; - - case 102: - memcpy_sse2_64(dd - 102, ss - 102); - case 38: - memcpy_sse2_32(dd - 38, ss - 38); - *((uint32_unaligned_t*)(dd - 6)) = *((uint32_unaligned_t*)(ss - 6)); - *((uint16_unaligned_t*)(dd - 2)) = *((uint16_unaligned_t*)(ss - 2)); - break; - - case 103: - memcpy_sse2_64(dd - 103, ss - 103); - case 39: - memcpy_sse2_32(dd - 39, ss - 39); - *((uint32_unaligned_t*)(dd - 7)) = *((uint32_unaligned_t*)(ss - 7)); - *((uint32_unaligned_t*)(dd - 4)) = *((uint32_unaligned_t*)(ss - 4)); - break; - - case 104: - memcpy_sse2_64(dd - 104, ss - 104); - case 40: - memcpy_sse2_32(dd - 40, ss - 40); - *((uint64_unaligned_t*)(dd - 8)) = *((uint64_unaligned_t*)(ss - 8)); - break; - - case 105: - memcpy_sse2_64(dd - 105, ss - 105); - case 41: - memcpy_sse2_32(dd - 41, ss - 41); - *((uint64_unaligned_t*)(dd - 9)) = *((uint64_unaligned_t*)(ss - 9)); - dd[-1] = ss[-1]; - break; - - case 106: - memcpy_sse2_64(dd - 106, ss - 106); - case 42: - memcpy_sse2_32(dd - 42, ss - 42); - *((uint64_unaligned_t*)(dd - 10)) = *((uint64_unaligned_t*)(ss - 10)); - *((uint16_unaligned_t*)(dd - 2)) = *((uint16_unaligned_t*)(ss - 2)); - break; - - case 107: - memcpy_sse2_64(dd - 107, ss - 107); - case 43: - memcpy_sse2_32(dd - 43, ss - 43); - *((uint64_unaligned_t*)(dd - 11)) = *((uint64_unaligned_t*)(ss - 11)); - *((uint32_unaligned_t*)(dd - 4)) = *((uint32_unaligned_t*)(ss - 4)); - break; - - case 108: - memcpy_sse2_64(dd - 108, ss - 108); - case 44: - memcpy_sse2_32(dd - 44, ss - 44); - *((uint64_unaligned_t*)(dd - 12)) = *((uint64_unaligned_t*)(ss - 12)); - *((uint32_unaligned_t*)(dd - 4)) = *((uint32_unaligned_t*)(ss - 4)); - break; - - case 109: - memcpy_sse2_64(dd - 109, ss - 109); - case 45: - memcpy_sse2_32(dd - 45, ss - 45); - *((uint64_unaligned_t*)(dd - 13)) = *((uint64_unaligned_t*)(ss - 13)); - *((uint32_unaligned_t*)(dd - 5)) = *((uint32_unaligned_t*)(ss - 5)); - dd[-1] = ss[-1]; - break; - - case 110: - memcpy_sse2_64(dd - 110, ss - 110); - case 46: - memcpy_sse2_32(dd - 46, ss - 46); - *((uint64_unaligned_t*)(dd - 14)) = *((uint64_unaligned_t*)(ss - 14)); - *((uint64_unaligned_t*)(dd - 8)) = *((uint64_unaligned_t*)(ss - 8)); - break; - - case 111: - memcpy_sse2_64(dd - 111, ss - 111); - case 47: - memcpy_sse2_32(dd - 47, ss - 47); - *((uint64_unaligned_t*)(dd - 15)) = *((uint64_unaligned_t*)(ss - 15)); - *((uint64_unaligned_t*)(dd - 8)) = *((uint64_unaligned_t*)(ss - 8)); - break; - - case 112: - memcpy_sse2_64(dd - 112, ss - 112); - case 48: - memcpy_sse2_32(dd - 48, ss - 48); - memcpy_sse2_16(dd - 16, ss - 16); - break; - - case 113: - memcpy_sse2_64(dd - 113, ss - 113); - case 49: - memcpy_sse2_32(dd - 49, ss - 49); - memcpy_sse2_16(dd - 17, ss - 17); - dd[-1] = ss[-1]; - break; - - case 114: - memcpy_sse2_64(dd - 114, ss - 114); - case 50: - memcpy_sse2_32(dd - 50, ss - 50); - memcpy_sse2_16(dd - 18, ss - 18); - *((uint16_unaligned_t*)(dd - 2)) = *((uint16_unaligned_t*)(ss - 2)); - break; - - case 115: - memcpy_sse2_64(dd - 115, ss - 115); - case 51: - memcpy_sse2_32(dd - 51, ss - 51); - memcpy_sse2_16(dd - 19, ss - 19); - *((uint16_unaligned_t*)(dd - 3)) = *((uint16_unaligned_t*)(ss - 3)); - dd[-1] = ss[-1]; - break; - - case 116: - memcpy_sse2_64(dd - 116, ss - 116); - case 52: - memcpy_sse2_32(dd - 52, ss - 52); - memcpy_sse2_16(dd - 20, ss - 20); - *((uint32_unaligned_t*)(dd - 4)) = *((uint32_unaligned_t*)(ss - 4)); - break; - - case 117: - memcpy_sse2_64(dd - 117, ss - 117); - case 53: - memcpy_sse2_32(dd - 53, ss - 53); - memcpy_sse2_16(dd - 21, ss - 21); - *((uint32_unaligned_t*)(dd - 5)) = *((uint32_unaligned_t*)(ss - 5)); - dd[-1] = ss[-1]; - break; - - case 118: - memcpy_sse2_64(dd - 118, ss - 118); - case 54: - memcpy_sse2_32(dd - 54, ss - 54); - memcpy_sse2_16(dd - 22, ss - 22); - *((uint32_unaligned_t*)(dd - 6)) = *((uint32_unaligned_t*)(ss - 6)); - *((uint16_unaligned_t*)(dd - 2)) = *((uint16_unaligned_t*)(ss - 2)); - break; - - case 119: - memcpy_sse2_64(dd - 119, ss - 119); - case 55: - memcpy_sse2_32(dd - 55, ss - 55); - memcpy_sse2_16(dd - 23, ss - 23); - *((uint32_unaligned_t*)(dd - 7)) = *((uint32_unaligned_t*)(ss - 7)); - *((uint32_unaligned_t*)(dd - 4)) = *((uint32_unaligned_t*)(ss - 4)); - break; - - case 120: - memcpy_sse2_64(dd - 120, ss - 120); - case 56: - memcpy_sse2_32(dd - 56, ss - 56); - memcpy_sse2_16(dd - 24, ss - 24); - memcpy_sse2_16(dd - 16, ss - 16); - break; - - case 121: - memcpy_sse2_64(dd - 121, ss - 121); - case 57: - memcpy_sse2_32(dd - 57, ss - 57); - memcpy_sse2_16(dd - 25, ss - 25); - memcpy_sse2_16(dd - 16, ss - 16); - break; - - case 122: - memcpy_sse2_64(dd - 122, ss - 122); - case 58: - memcpy_sse2_32(dd - 58, ss - 58); - memcpy_sse2_16(dd - 26, ss - 26); - memcpy_sse2_16(dd - 16, ss - 16); - break; - - case 123: - memcpy_sse2_64(dd - 123, ss - 123); - case 59: - memcpy_sse2_32(dd - 59, ss - 59); - memcpy_sse2_16(dd - 27, ss - 27); - memcpy_sse2_16(dd - 16, ss - 16); - break; - - case 124: - memcpy_sse2_64(dd - 124, ss - 124); - case 60: - memcpy_sse2_32(dd - 60, ss - 60); - memcpy_sse2_16(dd - 28, ss - 28); - memcpy_sse2_16(dd - 16, ss - 16); - break; - - case 125: - memcpy_sse2_64(dd - 125, ss - 125); - case 61: - memcpy_sse2_32(dd - 61, ss - 61); - memcpy_sse2_16(dd - 29, ss - 29); - memcpy_sse2_16(dd - 16, ss - 16); - break; - - case 126: - memcpy_sse2_64(dd - 126, ss - 126); - case 62: - memcpy_sse2_32(dd - 62, ss - 62); - memcpy_sse2_16(dd - 30, ss - 30); - memcpy_sse2_16(dd - 16, ss - 16); - break; - - case 127: - memcpy_sse2_64(dd - 127, ss - 127); - case 63: - memcpy_sse2_32(dd - 63, ss - 63); - memcpy_sse2_16(dd - 31, ss - 31); - memcpy_sse2_16(dd - 16, ss - 16); - break; - - case 128: - memcpy_sse2_128(dd - 128, ss - 128); - break; - } - - return dst; -} - - -//--------------------------------------------------------------------- -// main routine -//--------------------------------------------------------------------- -static void* memcpy_fast(void *destination, const void *source, size_t size) -{ - unsigned char *dst = (unsigned char*)destination; - const unsigned char *src = (const unsigned char*)source; - static size_t cachesize = 0x200000; // L2-cache size - size_t padding; - - // small memory copy - if (size <= 128) { - return memcpy_tiny(dst, src, size); - } - - // align destination to 16 bytes boundary - padding = (16 - (((size_t)dst) & 15)) & 15; - - if (padding > 0) { - __m128i head = _mm_loadu_si128((const __m128i*)src); - _mm_storeu_si128((__m128i*)dst, head); - dst += padding; - src += padding; - size -= padding; - } - - // medium size copy - if (size <= cachesize) { - __m128i c0, c1, c2, c3, c4, c5, c6, c7; - - for (; size >= 128; size -= 128) { - c0 = _mm_loadu_si128(((const __m128i*)src) + 0); - c1 = _mm_loadu_si128(((const __m128i*)src) + 1); - c2 = _mm_loadu_si128(((const __m128i*)src) + 2); - c3 = _mm_loadu_si128(((const __m128i*)src) + 3); - c4 = _mm_loadu_si128(((const __m128i*)src) + 4); - c5 = _mm_loadu_si128(((const __m128i*)src) + 5); - c6 = _mm_loadu_si128(((const __m128i*)src) + 6); - c7 = _mm_loadu_si128(((const __m128i*)src) + 7); - _mm_prefetch((const char*)(src + 256), _MM_HINT_NTA); - src += 128; - _mm_store_si128((((__m128i*)dst) + 0), c0); - _mm_store_si128((((__m128i*)dst) + 1), c1); - _mm_store_si128((((__m128i*)dst) + 2), c2); - _mm_store_si128((((__m128i*)dst) + 3), c3); - _mm_store_si128((((__m128i*)dst) + 4), c4); - _mm_store_si128((((__m128i*)dst) + 5), c5); - _mm_store_si128((((__m128i*)dst) + 6), c6); - _mm_store_si128((((__m128i*)dst) + 7), c7); - dst += 128; - } - } - else { // big memory copy - __m128i c0, c1, c2, c3, c4, c5, c6, c7; - - _mm_prefetch((const char*)(src), _MM_HINT_NTA); - - if ((((size_t)src) & 15) == 0) { // source aligned - for (; size >= 128; size -= 128) { - c0 = _mm_load_si128(((const __m128i*)src) + 0); - c1 = _mm_load_si128(((const __m128i*)src) + 1); - c2 = _mm_load_si128(((const __m128i*)src) + 2); - c3 = _mm_load_si128(((const __m128i*)src) + 3); - c4 = _mm_load_si128(((const __m128i*)src) + 4); - c5 = _mm_load_si128(((const __m128i*)src) + 5); - c6 = _mm_load_si128(((const __m128i*)src) + 6); - c7 = _mm_load_si128(((const __m128i*)src) + 7); - _mm_prefetch((const char*)(src + 256), _MM_HINT_NTA); - src += 128; - _mm_stream_si128((((__m128i*)dst) + 0), c0); - _mm_stream_si128((((__m128i*)dst) + 1), c1); - _mm_stream_si128((((__m128i*)dst) + 2), c2); - _mm_stream_si128((((__m128i*)dst) + 3), c3); - _mm_stream_si128((((__m128i*)dst) + 4), c4); - _mm_stream_si128((((__m128i*)dst) + 5), c5); - _mm_stream_si128((((__m128i*)dst) + 6), c6); - _mm_stream_si128((((__m128i*)dst) + 7), c7); - dst += 128; - } - } - else { // source unaligned - for (; size >= 128; size -= 128) { - c0 = _mm_loadu_si128(((const __m128i*)src) + 0); - c1 = _mm_loadu_si128(((const __m128i*)src) + 1); - c2 = _mm_loadu_si128(((const __m128i*)src) + 2); - c3 = _mm_loadu_si128(((const __m128i*)src) + 3); - c4 = _mm_loadu_si128(((const __m128i*)src) + 4); - c5 = _mm_loadu_si128(((const __m128i*)src) + 5); - c6 = _mm_loadu_si128(((const __m128i*)src) + 6); - c7 = _mm_loadu_si128(((const __m128i*)src) + 7); - _mm_prefetch((const char*)(src + 256), _MM_HINT_NTA); - src += 128; - _mm_stream_si128((((__m128i*)dst) + 0), c0); - _mm_stream_si128((((__m128i*)dst) + 1), c1); - _mm_stream_si128((((__m128i*)dst) + 2), c2); - _mm_stream_si128((((__m128i*)dst) + 3), c3); - _mm_stream_si128((((__m128i*)dst) + 4), c4); - _mm_stream_si128((((__m128i*)dst) + 5), c5); - _mm_stream_si128((((__m128i*)dst) + 6), c6); - _mm_stream_si128((((__m128i*)dst) + 7), c7); - dst += 128; - } - } - _mm_sfence(); - } - - memcpy_tiny(dst, src, size); - - return destination; -} - - -#endif diff --git a/contrib/FastMemcpy/FastMemcpy_Avx.c b/contrib/FastMemcpy/FastMemcpy_Avx.c deleted file mode 100644 index 6538c6b2126..00000000000 --- a/contrib/FastMemcpy/FastMemcpy_Avx.c +++ /dev/null @@ -1,171 +0,0 @@ -//===================================================================== -// -// FastMemcpy.c - skywind3000@163.com, 2015 -// -// feature: -// 50% speed up in avg. vs standard memcpy (tested in vc2012/gcc4.9) -// -//===================================================================== -#include -#include -#include -#include -#include - -#if (defined(_WIN32) || defined(WIN32)) -#include -#include -#ifdef _MSC_VER -#pragma comment(lib, "winmm.lib") -#endif -#elif defined(__unix) -#include -#include -#else -#error it can only be compiled under windows or unix -#endif - -#include "FastMemcpy_Avx.h" - - -unsigned int gettime() -{ - #if (defined(_WIN32) || defined(WIN32)) - return timeGetTime(); - #else - static struct timezone tz={ 0,0 }; - struct timeval time; - gettimeofday(&time,&tz); - return (time.tv_sec * 1000 + time.tv_usec / 1000); - #endif -} - -void sleepms(unsigned int millisec) -{ -#if defined(_WIN32) || defined(WIN32) - Sleep(millisec); -#else - usleep(millisec * 1000); -#endif -} - - - -void benchmark(int dstalign, int srcalign, size_t size, int times) -{ - char *DATA1 = (char*)malloc(size + 64); - char *DATA2 = (char*)malloc(size + 64); - size_t LINEAR1 = ((size_t)DATA1); - size_t LINEAR2 = ((size_t)DATA2); - char *ALIGN1 = (char*)(((64 - (LINEAR1 & 63)) & 63) + LINEAR1); - char *ALIGN2 = (char*)(((64 - (LINEAR2 & 63)) & 63) + LINEAR2); - char *dst = (dstalign)? ALIGN1 : (ALIGN1 + 1); - char *src = (srcalign)? ALIGN2 : (ALIGN2 + 3); - unsigned int t1, t2; - int k; - - sleepms(100); - t1 = gettime(); - for (k = times; k > 0; k--) { - memcpy(dst, src, size); - } - t1 = gettime() - t1; - sleepms(100); - t2 = gettime(); - for (k = times; k > 0; k--) { - memcpy_fast(dst, src, size); - } - t2 = gettime() - t2; - - free(DATA1); - free(DATA2); - - printf("result(dst %s, src %s): memcpy_fast=%dms memcpy=%d ms\n", - dstalign? "aligned" : "unalign", - srcalign? "aligned" : "unalign", (int)t2, (int)t1); -} - - -void bench(int copysize, int times) -{ - printf("benchmark(size=%d bytes, times=%d):\n", copysize, times); - benchmark(1, 1, copysize, times); - benchmark(1, 0, copysize, times); - benchmark(0, 1, copysize, times); - benchmark(0, 0, copysize, times); - printf("\n"); -} - - -void random_bench(int maxsize, int times) -{ - static char A[11 * 1024 * 1024 + 2]; - static char B[11 * 1024 * 1024 + 2]; - static int random_offsets[0x10000]; - static int random_sizes[0x8000]; - unsigned int i, p1, p2; - unsigned int t1, t2; - for (i = 0; i < 0x10000; i++) { // generate random offsets - random_offsets[i] = rand() % (10 * 1024 * 1024 + 1); - } - for (i = 0; i < 0x8000; i++) { // generate random sizes - random_sizes[i] = 1 + rand() % maxsize; - } - sleepms(100); - t1 = gettime(); - for (p1 = 0, p2 = 0, i = 0; i < times; i++) { - int offset1 = random_offsets[(p1++) & 0xffff]; - int offset2 = random_offsets[(p1++) & 0xffff]; - int size = random_sizes[(p2++) & 0x7fff]; - memcpy(A + offset1, B + offset2, size); - } - t1 = gettime() - t1; - sleepms(100); - t2 = gettime(); - for (p1 = 0, p2 = 0, i = 0; i < times; i++) { - int offset1 = random_offsets[(p1++) & 0xffff]; - int offset2 = random_offsets[(p1++) & 0xffff]; - int size = random_sizes[(p2++) & 0x7fff]; - memcpy_fast(A + offset1, B + offset2, size); - } - t2 = gettime() - t2; - printf("benchmark random access:\n"); - printf("memcpy_fast=%dms memcpy=%dms\n\n", (int)t2, (int)t1); -} - - -#ifdef _MSC_VER -#pragma comment(lib, "winmm.lib") -#endif - -int main(void) -{ -#if 1 - bench(32, 0x1000000); - bench(64, 0x1000000); - bench(512, 0x800000); - bench(1024, 0x400000); -#endif - bench(4096, 0x80000); - bench(8192, 0x40000); -#if 1 - bench(1024 * 1024 * 1, 0x800); - bench(1024 * 1024 * 4, 0x200); -#endif - bench(1024 * 1024 * 8, 0x100); - - random_bench(2048, 8000000); - - return 0; -} - - - - -/* - -*/ - - - - diff --git a/contrib/FastMemcpy/FastMemcpy_Avx.h b/contrib/FastMemcpy/FastMemcpy_Avx.h deleted file mode 100644 index 8ba064b0350..00000000000 --- a/contrib/FastMemcpy/FastMemcpy_Avx.h +++ /dev/null @@ -1,492 +0,0 @@ -//===================================================================== -// -// FastMemcpy.c - skywind3000@163.com, 2015 -// -// feature: -// 50% speed up in avg. vs standard memcpy (tested in vc2012/gcc5.1) -// -//===================================================================== -#ifndef __FAST_MEMCPY_H__ -#define __FAST_MEMCPY_H__ - -#include -#include -#include - - -//--------------------------------------------------------------------- -// force inline for compilers -//--------------------------------------------------------------------- -#ifndef INLINE -#ifdef __GNUC__ -#if (__GNUC__ > 3) || ((__GNUC__ == 3) && (__GNUC_MINOR__ >= 1)) - #define INLINE __inline__ __attribute__((always_inline)) -#else - #define INLINE __inline__ -#endif -#elif defined(_MSC_VER) - #define INLINE __forceinline -#elif (defined(__BORLANDC__) || defined(__WATCOMC__)) - #define INLINE __inline -#else - #define INLINE -#endif -#endif - - - -//--------------------------------------------------------------------- -// fast copy for different sizes -//--------------------------------------------------------------------- -static INLINE void memcpy_avx_16(void *dst, const void *src) { -#if 1 - __m128i m0 = _mm_loadu_si128(((const __m128i*)src) + 0); - _mm_storeu_si128(((__m128i*)dst) + 0, m0); -#else - *((uint64_t*)((char*)dst + 0)) = *((uint64_t*)((const char*)src + 0)); - *((uint64_t*)((char*)dst + 8)) = *((uint64_t*)((const char*)src + 8)); -#endif -} - -static INLINE void memcpy_avx_32(void *dst, const void *src) { - __m256i m0 = _mm256_loadu_si256(((const __m256i*)src) + 0); - _mm256_storeu_si256(((__m256i*)dst) + 0, m0); -} - -static INLINE void memcpy_avx_64(void *dst, const void *src) { - __m256i m0 = _mm256_loadu_si256(((const __m256i*)src) + 0); - __m256i m1 = _mm256_loadu_si256(((const __m256i*)src) + 1); - _mm256_storeu_si256(((__m256i*)dst) + 0, m0); - _mm256_storeu_si256(((__m256i*)dst) + 1, m1); -} - -static INLINE void memcpy_avx_128(void *dst, const void *src) { - __m256i m0 = _mm256_loadu_si256(((const __m256i*)src) + 0); - __m256i m1 = _mm256_loadu_si256(((const __m256i*)src) + 1); - __m256i m2 = _mm256_loadu_si256(((const __m256i*)src) + 2); - __m256i m3 = _mm256_loadu_si256(((const __m256i*)src) + 3); - _mm256_storeu_si256(((__m256i*)dst) + 0, m0); - _mm256_storeu_si256(((__m256i*)dst) + 1, m1); - _mm256_storeu_si256(((__m256i*)dst) + 2, m2); - _mm256_storeu_si256(((__m256i*)dst) + 3, m3); -} - -static INLINE void memcpy_avx_256(void *dst, const void *src) { - __m256i m0 = _mm256_loadu_si256(((const __m256i*)src) + 0); - __m256i m1 = _mm256_loadu_si256(((const __m256i*)src) + 1); - __m256i m2 = _mm256_loadu_si256(((const __m256i*)src) + 2); - __m256i m3 = _mm256_loadu_si256(((const __m256i*)src) + 3); - __m256i m4 = _mm256_loadu_si256(((const __m256i*)src) + 4); - __m256i m5 = _mm256_loadu_si256(((const __m256i*)src) + 5); - __m256i m6 = _mm256_loadu_si256(((const __m256i*)src) + 6); - __m256i m7 = _mm256_loadu_si256(((const __m256i*)src) + 7); - _mm256_storeu_si256(((__m256i*)dst) + 0, m0); - _mm256_storeu_si256(((__m256i*)dst) + 1, m1); - _mm256_storeu_si256(((__m256i*)dst) + 2, m2); - _mm256_storeu_si256(((__m256i*)dst) + 3, m3); - _mm256_storeu_si256(((__m256i*)dst) + 4, m4); - _mm256_storeu_si256(((__m256i*)dst) + 5, m5); - _mm256_storeu_si256(((__m256i*)dst) + 6, m6); - _mm256_storeu_si256(((__m256i*)dst) + 7, m7); -} - - -//--------------------------------------------------------------------- -// tiny memory copy with jump table optimized -//--------------------------------------------------------------------- -static INLINE void *memcpy_tiny(void *dst, const void *src, size_t size) { - unsigned char *dd = ((unsigned char*)dst) + size; - const unsigned char *ss = ((const unsigned char*)src) + size; - - switch (size) { - case 128: memcpy_avx_128(dd - 128, ss - 128); - case 0: break; - case 129: memcpy_avx_128(dd - 129, ss - 129); - case 1: dd[-1] = ss[-1]; break; - case 130: memcpy_avx_128(dd - 130, ss - 130); - case 2: *((uint16_t*)(dd - 2)) = *((uint16_t*)(ss - 2)); break; - case 131: memcpy_avx_128(dd - 131, ss - 131); - case 3: *((uint16_t*)(dd - 3)) = *((uint16_t*)(ss - 3)); dd[-1] = ss[-1]; break; - case 132: memcpy_avx_128(dd - 132, ss - 132); - case 4: *((uint32_t*)(dd - 4)) = *((uint32_t*)(ss - 4)); break; - case 133: memcpy_avx_128(dd - 133, ss - 133); - case 5: *((uint32_t*)(dd - 5)) = *((uint32_t*)(ss - 5)); dd[-1] = ss[-1]; break; - case 134: memcpy_avx_128(dd - 134, ss - 134); - case 6: *((uint32_t*)(dd - 6)) = *((uint32_t*)(ss - 6)); *((uint16_t*)(dd - 2)) = *((uint16_t*)(ss - 2)); break; - case 135: memcpy_avx_128(dd - 135, ss - 135); - case 7: *((uint32_t*)(dd - 7)) = *((uint32_t*)(ss - 7)); *((uint32_t*)(dd - 4)) = *((uint32_t*)(ss - 4)); break; - case 136: memcpy_avx_128(dd - 136, ss - 136); - case 8: *((uint64_t*)(dd - 8)) = *((uint64_t*)(ss - 8)); break; - case 137: memcpy_avx_128(dd - 137, ss - 137); - case 9: *((uint64_t*)(dd - 9)) = *((uint64_t*)(ss - 9)); dd[-1] = ss[-1]; break; - case 138: memcpy_avx_128(dd - 138, ss - 138); - case 10: *((uint64_t*)(dd - 10)) = *((uint64_t*)(ss - 10)); *((uint16_t*)(dd - 2)) = *((uint16_t*)(ss - 2)); break; - case 139: memcpy_avx_128(dd - 139, ss - 139); - case 11: *((uint64_t*)(dd - 11)) = *((uint64_t*)(ss - 11)); *((uint32_t*)(dd - 4)) = *((uint32_t*)(ss - 4)); break; - case 140: memcpy_avx_128(dd - 140, ss - 140); - case 12: *((uint64_t*)(dd - 12)) = *((uint64_t*)(ss - 12)); *((uint32_t*)(dd - 4)) = *((uint32_t*)(ss - 4)); break; - case 141: memcpy_avx_128(dd - 141, ss - 141); - case 13: *((uint64_t*)(dd - 13)) = *((uint64_t*)(ss - 13)); *((uint64_t*)(dd - 8)) = *((uint64_t*)(ss - 8)); break; - case 142: memcpy_avx_128(dd - 142, ss - 142); - case 14: *((uint64_t*)(dd - 14)) = *((uint64_t*)(ss - 14)); *((uint64_t*)(dd - 8)) = *((uint64_t*)(ss - 8)); break; - case 143: memcpy_avx_128(dd - 143, ss - 143); - case 15: *((uint64_t*)(dd - 15)) = *((uint64_t*)(ss - 15)); *((uint64_t*)(dd - 8)) = *((uint64_t*)(ss - 8)); break; - case 144: memcpy_avx_128(dd - 144, ss - 144); - case 16: memcpy_avx_16(dd - 16, ss - 16); break; - case 145: memcpy_avx_128(dd - 145, ss - 145); - case 17: memcpy_avx_16(dd - 17, ss - 17); dd[-1] = ss[-1]; break; - case 146: memcpy_avx_128(dd - 146, ss - 146); - case 18: memcpy_avx_16(dd - 18, ss - 18); *((uint16_t*)(dd - 2)) = *((uint16_t*)(ss - 2)); break; - case 147: memcpy_avx_128(dd - 147, ss - 147); - case 19: memcpy_avx_16(dd - 19, ss - 19); *((uint32_t*)(dd - 4)) = *((uint32_t*)(ss - 4)); break; - case 148: memcpy_avx_128(dd - 148, ss - 148); - case 20: memcpy_avx_16(dd - 20, ss - 20); *((uint32_t*)(dd - 4)) = *((uint32_t*)(ss - 4)); break; - case 149: memcpy_avx_128(dd - 149, ss - 149); - case 21: memcpy_avx_16(dd - 21, ss - 21); *((uint64_t*)(dd - 8)) = *((uint64_t*)(ss - 8)); break; - case 150: memcpy_avx_128(dd - 150, ss - 150); - case 22: memcpy_avx_16(dd - 22, ss - 22); *((uint64_t*)(dd - 8)) = *((uint64_t*)(ss - 8)); break; - case 151: memcpy_avx_128(dd - 151, ss - 151); - case 23: memcpy_avx_16(dd - 23, ss - 23); *((uint64_t*)(dd - 8)) = *((uint64_t*)(ss - 8)); break; - case 152: memcpy_avx_128(dd - 152, ss - 152); - case 24: memcpy_avx_16(dd - 24, ss - 24); *((uint64_t*)(dd - 8)) = *((uint64_t*)(ss - 8)); break; - case 153: memcpy_avx_128(dd - 153, ss - 153); - case 25: memcpy_avx_16(dd - 25, ss - 25); memcpy_avx_16(dd - 16, ss - 16); break; - case 154: memcpy_avx_128(dd - 154, ss - 154); - case 26: memcpy_avx_16(dd - 26, ss - 26); memcpy_avx_16(dd - 16, ss - 16); break; - case 155: memcpy_avx_128(dd - 155, ss - 155); - case 27: memcpy_avx_16(dd - 27, ss - 27); memcpy_avx_16(dd - 16, ss - 16); break; - case 156: memcpy_avx_128(dd - 156, ss - 156); - case 28: memcpy_avx_16(dd - 28, ss - 28); memcpy_avx_16(dd - 16, ss - 16); break; - case 157: memcpy_avx_128(dd - 157, ss - 157); - case 29: memcpy_avx_16(dd - 29, ss - 29); memcpy_avx_16(dd - 16, ss - 16); break; - case 158: memcpy_avx_128(dd - 158, ss - 158); - case 30: memcpy_avx_16(dd - 30, ss - 30); memcpy_avx_16(dd - 16, ss - 16); break; - case 159: memcpy_avx_128(dd - 159, ss - 159); - case 31: memcpy_avx_16(dd - 31, ss - 31); memcpy_avx_16(dd - 16, ss - 16); break; - case 160: memcpy_avx_128(dd - 160, ss - 160); - case 32: memcpy_avx_32(dd - 32, ss - 32); break; - case 161: memcpy_avx_128(dd - 161, ss - 161); - case 33: memcpy_avx_32(dd - 33, ss - 33); dd[-1] = ss[-1]; break; - case 162: memcpy_avx_128(dd - 162, ss - 162); - case 34: memcpy_avx_32(dd - 34, ss - 34); *((uint16_t*)(dd - 2)) = *((uint16_t*)(ss - 2)); break; - case 163: memcpy_avx_128(dd - 163, ss - 163); - case 35: memcpy_avx_32(dd - 35, ss - 35); *((uint32_t*)(dd - 4)) = *((uint32_t*)(ss - 4)); break; - case 164: memcpy_avx_128(dd - 164, ss - 164); - case 36: memcpy_avx_32(dd - 36, ss - 36); *((uint32_t*)(dd - 4)) = *((uint32_t*)(ss - 4)); break; - case 165: memcpy_avx_128(dd - 165, ss - 165); - case 37: memcpy_avx_32(dd - 37, ss - 37); *((uint64_t*)(dd - 8)) = *((uint64_t*)(ss - 8)); break; - case 166: memcpy_avx_128(dd - 166, ss - 166); - case 38: memcpy_avx_32(dd - 38, ss - 38); *((uint64_t*)(dd - 8)) = *((uint64_t*)(ss - 8)); break; - case 167: memcpy_avx_128(dd - 167, ss - 167); - case 39: memcpy_avx_32(dd - 39, ss - 39); *((uint64_t*)(dd - 8)) = *((uint64_t*)(ss - 8)); break; - case 168: memcpy_avx_128(dd - 168, ss - 168); - case 40: memcpy_avx_32(dd - 40, ss - 40); *((uint64_t*)(dd - 8)) = *((uint64_t*)(ss - 8)); break; - case 169: memcpy_avx_128(dd - 169, ss - 169); - case 41: memcpy_avx_32(dd - 41, ss - 41); memcpy_avx_16(dd - 16, ss - 16); break; - case 170: memcpy_avx_128(dd - 170, ss - 170); - case 42: memcpy_avx_32(dd - 42, ss - 42); memcpy_avx_16(dd - 16, ss - 16); break; - case 171: memcpy_avx_128(dd - 171, ss - 171); - case 43: memcpy_avx_32(dd - 43, ss - 43); memcpy_avx_16(dd - 16, ss - 16); break; - case 172: memcpy_avx_128(dd - 172, ss - 172); - case 44: memcpy_avx_32(dd - 44, ss - 44); memcpy_avx_16(dd - 16, ss - 16); break; - case 173: memcpy_avx_128(dd - 173, ss - 173); - case 45: memcpy_avx_32(dd - 45, ss - 45); memcpy_avx_16(dd - 16, ss - 16); break; - case 174: memcpy_avx_128(dd - 174, ss - 174); - case 46: memcpy_avx_32(dd - 46, ss - 46); memcpy_avx_16(dd - 16, ss - 16); break; - case 175: memcpy_avx_128(dd - 175, ss - 175); - case 47: memcpy_avx_32(dd - 47, ss - 47); memcpy_avx_16(dd - 16, ss - 16); break; - case 176: memcpy_avx_128(dd - 176, ss - 176); - case 48: memcpy_avx_32(dd - 48, ss - 48); memcpy_avx_16(dd - 16, ss - 16); break; - case 177: memcpy_avx_128(dd - 177, ss - 177); - case 49: memcpy_avx_32(dd - 49, ss - 49); memcpy_avx_32(dd - 32, ss - 32); break; - case 178: memcpy_avx_128(dd - 178, ss - 178); - case 50: memcpy_avx_32(dd - 50, ss - 50); memcpy_avx_32(dd - 32, ss - 32); break; - case 179: memcpy_avx_128(dd - 179, ss - 179); - case 51: memcpy_avx_32(dd - 51, ss - 51); memcpy_avx_32(dd - 32, ss - 32); break; - case 180: memcpy_avx_128(dd - 180, ss - 180); - case 52: memcpy_avx_32(dd - 52, ss - 52); memcpy_avx_32(dd - 32, ss - 32); break; - case 181: memcpy_avx_128(dd - 181, ss - 181); - case 53: memcpy_avx_32(dd - 53, ss - 53); memcpy_avx_32(dd - 32, ss - 32); break; - case 182: memcpy_avx_128(dd - 182, ss - 182); - case 54: memcpy_avx_32(dd - 54, ss - 54); memcpy_avx_32(dd - 32, ss - 32); break; - case 183: memcpy_avx_128(dd - 183, ss - 183); - case 55: memcpy_avx_32(dd - 55, ss - 55); memcpy_avx_32(dd - 32, ss - 32); break; - case 184: memcpy_avx_128(dd - 184, ss - 184); - case 56: memcpy_avx_32(dd - 56, ss - 56); memcpy_avx_32(dd - 32, ss - 32); break; - case 185: memcpy_avx_128(dd - 185, ss - 185); - case 57: memcpy_avx_32(dd - 57, ss - 57); memcpy_avx_32(dd - 32, ss - 32); break; - case 186: memcpy_avx_128(dd - 186, ss - 186); - case 58: memcpy_avx_32(dd - 58, ss - 58); memcpy_avx_32(dd - 32, ss - 32); break; - case 187: memcpy_avx_128(dd - 187, ss - 187); - case 59: memcpy_avx_32(dd - 59, ss - 59); memcpy_avx_32(dd - 32, ss - 32); break; - case 188: memcpy_avx_128(dd - 188, ss - 188); - case 60: memcpy_avx_32(dd - 60, ss - 60); memcpy_avx_32(dd - 32, ss - 32); break; - case 189: memcpy_avx_128(dd - 189, ss - 189); - case 61: memcpy_avx_32(dd - 61, ss - 61); memcpy_avx_32(dd - 32, ss - 32); break; - case 190: memcpy_avx_128(dd - 190, ss - 190); - case 62: memcpy_avx_32(dd - 62, ss - 62); memcpy_avx_32(dd - 32, ss - 32); break; - case 191: memcpy_avx_128(dd - 191, ss - 191); - case 63: memcpy_avx_32(dd - 63, ss - 63); memcpy_avx_32(dd - 32, ss - 32); break; - case 192: memcpy_avx_128(dd - 192, ss - 192); - case 64: memcpy_avx_64(dd - 64, ss - 64); break; - case 193: memcpy_avx_128(dd - 193, ss - 193); - case 65: memcpy_avx_64(dd - 65, ss - 65); dd[-1] = ss[-1]; break; - case 194: memcpy_avx_128(dd - 194, ss - 194); - case 66: memcpy_avx_64(dd - 66, ss - 66); *((uint16_t*)(dd - 2)) = *((uint16_t*)(ss - 2)); break; - case 195: memcpy_avx_128(dd - 195, ss - 195); - case 67: memcpy_avx_64(dd - 67, ss - 67); *((uint32_t*)(dd - 4)) = *((uint32_t*)(ss - 4)); break; - case 196: memcpy_avx_128(dd - 196, ss - 196); - case 68: memcpy_avx_64(dd - 68, ss - 68); *((uint32_t*)(dd - 4)) = *((uint32_t*)(ss - 4)); break; - case 197: memcpy_avx_128(dd - 197, ss - 197); - case 69: memcpy_avx_64(dd - 69, ss - 69); *((uint64_t*)(dd - 8)) = *((uint64_t*)(ss - 8)); break; - case 198: memcpy_avx_128(dd - 198, ss - 198); - case 70: memcpy_avx_64(dd - 70, ss - 70); *((uint64_t*)(dd - 8)) = *((uint64_t*)(ss - 8)); break; - case 199: memcpy_avx_128(dd - 199, ss - 199); - case 71: memcpy_avx_64(dd - 71, ss - 71); *((uint64_t*)(dd - 8)) = *((uint64_t*)(ss - 8)); break; - case 200: memcpy_avx_128(dd - 200, ss - 200); - case 72: memcpy_avx_64(dd - 72, ss - 72); *((uint64_t*)(dd - 8)) = *((uint64_t*)(ss - 8)); break; - case 201: memcpy_avx_128(dd - 201, ss - 201); - case 73: memcpy_avx_64(dd - 73, ss - 73); memcpy_avx_16(dd - 16, ss - 16); break; - case 202: memcpy_avx_128(dd - 202, ss - 202); - case 74: memcpy_avx_64(dd - 74, ss - 74); memcpy_avx_16(dd - 16, ss - 16); break; - case 203: memcpy_avx_128(dd - 203, ss - 203); - case 75: memcpy_avx_64(dd - 75, ss - 75); memcpy_avx_16(dd - 16, ss - 16); break; - case 204: memcpy_avx_128(dd - 204, ss - 204); - case 76: memcpy_avx_64(dd - 76, ss - 76); memcpy_avx_16(dd - 16, ss - 16); break; - case 205: memcpy_avx_128(dd - 205, ss - 205); - case 77: memcpy_avx_64(dd - 77, ss - 77); memcpy_avx_16(dd - 16, ss - 16); break; - case 206: memcpy_avx_128(dd - 206, ss - 206); - case 78: memcpy_avx_64(dd - 78, ss - 78); memcpy_avx_16(dd - 16, ss - 16); break; - case 207: memcpy_avx_128(dd - 207, ss - 207); - case 79: memcpy_avx_64(dd - 79, ss - 79); memcpy_avx_16(dd - 16, ss - 16); break; - case 208: memcpy_avx_128(dd - 208, ss - 208); - case 80: memcpy_avx_64(dd - 80, ss - 80); memcpy_avx_16(dd - 16, ss - 16); break; - case 209: memcpy_avx_128(dd - 209, ss - 209); - case 81: memcpy_avx_64(dd - 81, ss - 81); memcpy_avx_32(dd - 32, ss - 32); break; - case 210: memcpy_avx_128(dd - 210, ss - 210); - case 82: memcpy_avx_64(dd - 82, ss - 82); memcpy_avx_32(dd - 32, ss - 32); break; - case 211: memcpy_avx_128(dd - 211, ss - 211); - case 83: memcpy_avx_64(dd - 83, ss - 83); memcpy_avx_32(dd - 32, ss - 32); break; - case 212: memcpy_avx_128(dd - 212, ss - 212); - case 84: memcpy_avx_64(dd - 84, ss - 84); memcpy_avx_32(dd - 32, ss - 32); break; - case 213: memcpy_avx_128(dd - 213, ss - 213); - case 85: memcpy_avx_64(dd - 85, ss - 85); memcpy_avx_32(dd - 32, ss - 32); break; - case 214: memcpy_avx_128(dd - 214, ss - 214); - case 86: memcpy_avx_64(dd - 86, ss - 86); memcpy_avx_32(dd - 32, ss - 32); break; - case 215: memcpy_avx_128(dd - 215, ss - 215); - case 87: memcpy_avx_64(dd - 87, ss - 87); memcpy_avx_32(dd - 32, ss - 32); break; - case 216: memcpy_avx_128(dd - 216, ss - 216); - case 88: memcpy_avx_64(dd - 88, ss - 88); memcpy_avx_32(dd - 32, ss - 32); break; - case 217: memcpy_avx_128(dd - 217, ss - 217); - case 89: memcpy_avx_64(dd - 89, ss - 89); memcpy_avx_32(dd - 32, ss - 32); break; - case 218: memcpy_avx_128(dd - 218, ss - 218); - case 90: memcpy_avx_64(dd - 90, ss - 90); memcpy_avx_32(dd - 32, ss - 32); break; - case 219: memcpy_avx_128(dd - 219, ss - 219); - case 91: memcpy_avx_64(dd - 91, ss - 91); memcpy_avx_32(dd - 32, ss - 32); break; - case 220: memcpy_avx_128(dd - 220, ss - 220); - case 92: memcpy_avx_64(dd - 92, ss - 92); memcpy_avx_32(dd - 32, ss - 32); break; - case 221: memcpy_avx_128(dd - 221, ss - 221); - case 93: memcpy_avx_64(dd - 93, ss - 93); memcpy_avx_32(dd - 32, ss - 32); break; - case 222: memcpy_avx_128(dd - 222, ss - 222); - case 94: memcpy_avx_64(dd - 94, ss - 94); memcpy_avx_32(dd - 32, ss - 32); break; - case 223: memcpy_avx_128(dd - 223, ss - 223); - case 95: memcpy_avx_64(dd - 95, ss - 95); memcpy_avx_32(dd - 32, ss - 32); break; - case 224: memcpy_avx_128(dd - 224, ss - 224); - case 96: memcpy_avx_64(dd - 96, ss - 96); memcpy_avx_32(dd - 32, ss - 32); break; - case 225: memcpy_avx_128(dd - 225, ss - 225); - case 97: memcpy_avx_64(dd - 97, ss - 97); memcpy_avx_64(dd - 64, ss - 64); break; - case 226: memcpy_avx_128(dd - 226, ss - 226); - case 98: memcpy_avx_64(dd - 98, ss - 98); memcpy_avx_64(dd - 64, ss - 64); break; - case 227: memcpy_avx_128(dd - 227, ss - 227); - case 99: memcpy_avx_64(dd - 99, ss - 99); memcpy_avx_64(dd - 64, ss - 64); break; - case 228: memcpy_avx_128(dd - 228, ss - 228); - case 100: memcpy_avx_64(dd - 100, ss - 100); memcpy_avx_64(dd - 64, ss - 64); break; - case 229: memcpy_avx_128(dd - 229, ss - 229); - case 101: memcpy_avx_64(dd - 101, ss - 101); memcpy_avx_64(dd - 64, ss - 64); break; - case 230: memcpy_avx_128(dd - 230, ss - 230); - case 102: memcpy_avx_64(dd - 102, ss - 102); memcpy_avx_64(dd - 64, ss - 64); break; - case 231: memcpy_avx_128(dd - 231, ss - 231); - case 103: memcpy_avx_64(dd - 103, ss - 103); memcpy_avx_64(dd - 64, ss - 64); break; - case 232: memcpy_avx_128(dd - 232, ss - 232); - case 104: memcpy_avx_64(dd - 104, ss - 104); memcpy_avx_64(dd - 64, ss - 64); break; - case 233: memcpy_avx_128(dd - 233, ss - 233); - case 105: memcpy_avx_64(dd - 105, ss - 105); memcpy_avx_64(dd - 64, ss - 64); break; - case 234: memcpy_avx_128(dd - 234, ss - 234); - case 106: memcpy_avx_64(dd - 106, ss - 106); memcpy_avx_64(dd - 64, ss - 64); break; - case 235: memcpy_avx_128(dd - 235, ss - 235); - case 107: memcpy_avx_64(dd - 107, ss - 107); memcpy_avx_64(dd - 64, ss - 64); break; - case 236: memcpy_avx_128(dd - 236, ss - 236); - case 108: memcpy_avx_64(dd - 108, ss - 108); memcpy_avx_64(dd - 64, ss - 64); break; - case 237: memcpy_avx_128(dd - 237, ss - 237); - case 109: memcpy_avx_64(dd - 109, ss - 109); memcpy_avx_64(dd - 64, ss - 64); break; - case 238: memcpy_avx_128(dd - 238, ss - 238); - case 110: memcpy_avx_64(dd - 110, ss - 110); memcpy_avx_64(dd - 64, ss - 64); break; - case 239: memcpy_avx_128(dd - 239, ss - 239); - case 111: memcpy_avx_64(dd - 111, ss - 111); memcpy_avx_64(dd - 64, ss - 64); break; - case 240: memcpy_avx_128(dd - 240, ss - 240); - case 112: memcpy_avx_64(dd - 112, ss - 112); memcpy_avx_64(dd - 64, ss - 64); break; - case 241: memcpy_avx_128(dd - 241, ss - 241); - case 113: memcpy_avx_64(dd - 113, ss - 113); memcpy_avx_64(dd - 64, ss - 64); break; - case 242: memcpy_avx_128(dd - 242, ss - 242); - case 114: memcpy_avx_64(dd - 114, ss - 114); memcpy_avx_64(dd - 64, ss - 64); break; - case 243: memcpy_avx_128(dd - 243, ss - 243); - case 115: memcpy_avx_64(dd - 115, ss - 115); memcpy_avx_64(dd - 64, ss - 64); break; - case 244: memcpy_avx_128(dd - 244, ss - 244); - case 116: memcpy_avx_64(dd - 116, ss - 116); memcpy_avx_64(dd - 64, ss - 64); break; - case 245: memcpy_avx_128(dd - 245, ss - 245); - case 117: memcpy_avx_64(dd - 117, ss - 117); memcpy_avx_64(dd - 64, ss - 64); break; - case 246: memcpy_avx_128(dd - 246, ss - 246); - case 118: memcpy_avx_64(dd - 118, ss - 118); memcpy_avx_64(dd - 64, ss - 64); break; - case 247: memcpy_avx_128(dd - 247, ss - 247); - case 119: memcpy_avx_64(dd - 119, ss - 119); memcpy_avx_64(dd - 64, ss - 64); break; - case 248: memcpy_avx_128(dd - 248, ss - 248); - case 120: memcpy_avx_64(dd - 120, ss - 120); memcpy_avx_64(dd - 64, ss - 64); break; - case 249: memcpy_avx_128(dd - 249, ss - 249); - case 121: memcpy_avx_64(dd - 121, ss - 121); memcpy_avx_64(dd - 64, ss - 64); break; - case 250: memcpy_avx_128(dd - 250, ss - 250); - case 122: memcpy_avx_64(dd - 122, ss - 122); memcpy_avx_64(dd - 64, ss - 64); break; - case 251: memcpy_avx_128(dd - 251, ss - 251); - case 123: memcpy_avx_64(dd - 123, ss - 123); memcpy_avx_64(dd - 64, ss - 64); break; - case 252: memcpy_avx_128(dd - 252, ss - 252); - case 124: memcpy_avx_64(dd - 124, ss - 124); memcpy_avx_64(dd - 64, ss - 64); break; - case 253: memcpy_avx_128(dd - 253, ss - 253); - case 125: memcpy_avx_64(dd - 125, ss - 125); memcpy_avx_64(dd - 64, ss - 64); break; - case 254: memcpy_avx_128(dd - 254, ss - 254); - case 126: memcpy_avx_64(dd - 126, ss - 126); memcpy_avx_64(dd - 64, ss - 64); break; - case 255: memcpy_avx_128(dd - 255, ss - 255); - case 127: memcpy_avx_64(dd - 127, ss - 127); memcpy_avx_64(dd - 64, ss - 64); break; - case 256: memcpy_avx_256(dd - 256, ss - 256); break; - } - - return dst; -} - - -//--------------------------------------------------------------------- -// main routine -//--------------------------------------------------------------------- -static void* memcpy_fast(void *destination, const void *source, size_t size) -{ - unsigned char *dst = (unsigned char*)destination; - const unsigned char *src = (const unsigned char*)source; - static size_t cachesize = 0x200000; // L3-cache size - size_t padding; - - // small memory copy - if (size <= 256) { - memcpy_tiny(dst, src, size); - _mm256_zeroupper(); - return destination; - } - - // align destination to 16 bytes boundary - padding = (32 - (((size_t)dst) & 31)) & 31; - -#if 0 - if (padding > 0) { - __m256i head = _mm256_loadu_si256((const __m256i*)src); - _mm256_storeu_si256((__m256i*)dst, head); - dst += padding; - src += padding; - size -= padding; - } -#else - __m256i head = _mm256_loadu_si256((const __m256i*)src); - _mm256_storeu_si256((__m256i*)dst, head); - dst += padding; - src += padding; - size -= padding; -#endif - - // medium size copy - if (size <= cachesize) { - __m256i c0, c1, c2, c3, c4, c5, c6, c7; - - for (; size >= 256; size -= 256) { - c0 = _mm256_loadu_si256(((const __m256i*)src) + 0); - c1 = _mm256_loadu_si256(((const __m256i*)src) + 1); - c2 = _mm256_loadu_si256(((const __m256i*)src) + 2); - c3 = _mm256_loadu_si256(((const __m256i*)src) + 3); - c4 = _mm256_loadu_si256(((const __m256i*)src) + 4); - c5 = _mm256_loadu_si256(((const __m256i*)src) + 5); - c6 = _mm256_loadu_si256(((const __m256i*)src) + 6); - c7 = _mm256_loadu_si256(((const __m256i*)src) + 7); - _mm_prefetch((const char*)(src + 512), _MM_HINT_NTA); - src += 256; - _mm256_storeu_si256((((__m256i*)dst) + 0), c0); - _mm256_storeu_si256((((__m256i*)dst) + 1), c1); - _mm256_storeu_si256((((__m256i*)dst) + 2), c2); - _mm256_storeu_si256((((__m256i*)dst) + 3), c3); - _mm256_storeu_si256((((__m256i*)dst) + 4), c4); - _mm256_storeu_si256((((__m256i*)dst) + 5), c5); - _mm256_storeu_si256((((__m256i*)dst) + 6), c6); - _mm256_storeu_si256((((__m256i*)dst) + 7), c7); - dst += 256; - } - } - else { // big memory copy - __m256i c0, c1, c2, c3, c4, c5, c6, c7; - /* __m256i c0, c1, c2, c3, c4, c5, c6, c7; */ - - _mm_prefetch((const char*)(src), _MM_HINT_NTA); - - if ((((size_t)src) & 31) == 0) { // source aligned - for (; size >= 256; size -= 256) { - c0 = _mm256_load_si256(((const __m256i*)src) + 0); - c1 = _mm256_load_si256(((const __m256i*)src) + 1); - c2 = _mm256_load_si256(((const __m256i*)src) + 2); - c3 = _mm256_load_si256(((const __m256i*)src) + 3); - c4 = _mm256_load_si256(((const __m256i*)src) + 4); - c5 = _mm256_load_si256(((const __m256i*)src) + 5); - c6 = _mm256_load_si256(((const __m256i*)src) + 6); - c7 = _mm256_load_si256(((const __m256i*)src) + 7); - _mm_prefetch((const char*)(src + 512), _MM_HINT_NTA); - src += 256; - _mm256_stream_si256((((__m256i*)dst) + 0), c0); - _mm256_stream_si256((((__m256i*)dst) + 1), c1); - _mm256_stream_si256((((__m256i*)dst) + 2), c2); - _mm256_stream_si256((((__m256i*)dst) + 3), c3); - _mm256_stream_si256((((__m256i*)dst) + 4), c4); - _mm256_stream_si256((((__m256i*)dst) + 5), c5); - _mm256_stream_si256((((__m256i*)dst) + 6), c6); - _mm256_stream_si256((((__m256i*)dst) + 7), c7); - dst += 256; - } - } - else { // source unaligned - for (; size >= 256; size -= 256) { - c0 = _mm256_loadu_si256(((const __m256i*)src) + 0); - c1 = _mm256_loadu_si256(((const __m256i*)src) + 1); - c2 = _mm256_loadu_si256(((const __m256i*)src) + 2); - c3 = _mm256_loadu_si256(((const __m256i*)src) + 3); - c4 = _mm256_loadu_si256(((const __m256i*)src) + 4); - c5 = _mm256_loadu_si256(((const __m256i*)src) + 5); - c6 = _mm256_loadu_si256(((const __m256i*)src) + 6); - c7 = _mm256_loadu_si256(((const __m256i*)src) + 7); - _mm_prefetch((const char*)(src + 512), _MM_HINT_NTA); - src += 256; - _mm256_stream_si256((((__m256i*)dst) + 0), c0); - _mm256_stream_si256((((__m256i*)dst) + 1), c1); - _mm256_stream_si256((((__m256i*)dst) + 2), c2); - _mm256_stream_si256((((__m256i*)dst) + 3), c3); - _mm256_stream_si256((((__m256i*)dst) + 4), c4); - _mm256_stream_si256((((__m256i*)dst) + 5), c5); - _mm256_stream_si256((((__m256i*)dst) + 6), c6); - _mm256_stream_si256((((__m256i*)dst) + 7), c7); - dst += 256; - } - } - _mm_sfence(); - } - - memcpy_tiny(dst, src, size); - _mm256_zeroupper(); - - return destination; -} - - -#endif - - - diff --git a/contrib/FastMemcpy/LICENSE b/contrib/FastMemcpy/LICENSE deleted file mode 100644 index c449da6aa8a..00000000000 --- a/contrib/FastMemcpy/LICENSE +++ /dev/null @@ -1,22 +0,0 @@ -The MIT License (MIT) - -Copyright (c) 2015 Linwei - -Permission is hereby granted, free of charge, to any person obtaining a copy -of this software and associated documentation files (the "Software"), to deal -in the Software without restriction, including without limitation the rights -to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -copies of the Software, and to permit persons to whom the Software is -furnished to do so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in all -copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -SOFTWARE. - diff --git a/contrib/FastMemcpy/README.md b/contrib/FastMemcpy/README.md deleted file mode 100644 index e253f6bf5dd..00000000000 --- a/contrib/FastMemcpy/README.md +++ /dev/null @@ -1,20 +0,0 @@ -Internal implementation of `memcpy` function. - -It has the following advantages over `libc`-supplied implementation: -- it is linked statically, so the function is called directly, not through a `PLT` (procedure lookup table of shared library); -- it is linked statically, so the function can have position-dependent code; -- your binaries will not depend on `glibc`'s memcpy, that forces dependency on specific symbol version like `memcpy@@GLIBC_2.14` and consequently on specific version of `glibc` library; -- you can include `memcpy.h` directly and the function has the chance to be inlined, which is beneficial for small but unknown at compile time sizes of memory regions; -- this version of `memcpy` pretend to be faster (in our benchmarks, the difference is within few percents). - -Currently it uses the implementation from **Linwei** (skywind3000@163.com). -Look at https://www.zhihu.com/question/35172305 for discussion. - -Drawbacks: -- only use SSE 2, doesn't use wider (AVX, AVX 512) vector registers when available; -- no CPU dispatching; doesn't take into account actual cache size. - -Also worth to look at: -- simple implementation from Facebook: https://github.com/facebook/folly/blob/master/folly/memcpy.S -- implementation from Agner Fog: http://www.agner.org/optimize/ -- glibc source code. diff --git a/contrib/FastMemcpy/memcpy_wrapper.c b/contrib/FastMemcpy/memcpy_wrapper.c deleted file mode 100644 index 1f57345980a..00000000000 --- a/contrib/FastMemcpy/memcpy_wrapper.c +++ /dev/null @@ -1,6 +0,0 @@ -#include "FastMemcpy.h" - -void * memcpy(void * __restrict destination, const void * __restrict source, size_t size) -{ - return memcpy_fast(destination, source, size); -} From d1b3258ae7a01fdaa8daa15733351880d6e5f3d9 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Wed, 3 Mar 2021 00:16:34 +0300 Subject: [PATCH 069/544] Try without AVX --- base/glibc-compatibility/memcpy/memcpy.S | 15 +-------------- 1 file changed, 1 insertion(+), 14 deletions(-) diff --git a/base/glibc-compatibility/memcpy/memcpy.S b/base/glibc-compatibility/memcpy/memcpy.S index c1439c3c859..502fcc22b08 100644 --- a/base/glibc-compatibility/memcpy/memcpy.S +++ b/base/glibc-compatibility/memcpy/memcpy.S @@ -46,19 +46,6 @@ MemCpy: mov $.Lmemcpytab.size,%ecx cmovb %rdx,%rcx jmp *memcpytab(,%rcx,8) .Lanchorpoint: -.L32r: cmp $1024,%rdx - jae .Lerms -.L32: vmovdqu -32(%rsi,%rdx),%ymm4 - mov $32,%rcx -0: add $32,%rcx - vmovdqu -64(%rsi,%rcx),%ymm3 - vmovdqu %ymm3,-64(%rdi,%rcx) - cmp %rcx,%rdx - ja 0b - vmovdqu %ymm4,-32(%rdi,%rdx) - vxorps %ymm4,%ymm4,%ymm4 - vxorps %ymm3,%ymm3,%ymm3 - jmp .L0 .L16r: cmp $1024,%rdx jae .Lerms .L16: movdqu -16(%rsi,%rdx),%xmm4 @@ -146,6 +133,6 @@ memcpytab: .quad .L16 .endr .equ .Lmemcpytab.size,(.-memcpytab)/8 - .quad .L32r # AVX + ERMS + .quad .L16r # SSE + ERMS + NTS .type memcpytab,@object .previous From dbd91442e0e5fc8d574616b5ff26c9632bef9c81 Mon Sep 17 00:00:00 2001 From: romanzhukov Date: Wed, 3 Mar 2021 01:46:37 +0300 Subject: [PATCH 070/544] DOCSUP-7100: Add avg translation, argmax query fixes. --- .../aggregate-functions/reference/avg.md | 6 +- .../aggregate-functions/reference/argmax.md | 12 ++-- .../aggregate-functions/reference/avg.md | 61 ++++++++++++++++++- 3 files changed, 67 insertions(+), 12 deletions(-) diff --git a/docs/en/sql-reference/aggregate-functions/reference/avg.md b/docs/en/sql-reference/aggregate-functions/reference/avg.md index d53a47a36a3..196db2d6b63 100644 --- a/docs/en/sql-reference/aggregate-functions/reference/avg.md +++ b/docs/en/sql-reference/aggregate-functions/reference/avg.md @@ -23,8 +23,8 @@ avg(x) **Returned value** -- `NaN` if the supplied parameter is empty. -- Mean otherwise. +- `NaN` if the supplied parameter is empty. +- Mean otherwise. **Return type** is always [Float64](../../../sql-reference/data-types/float.md). @@ -33,7 +33,7 @@ avg(x) Query: ``` sql -SELECT avg(x) FROM values('x Int8', 0, 1, 2, 3, 4, 5) +SELECT avg(x) FROM values('x Int8', 0, 1, 2, 3, 4, 5); ``` Result: diff --git a/docs/ru/sql-reference/aggregate-functions/reference/argmax.md b/docs/ru/sql-reference/aggregate-functions/reference/argmax.md index f44e65831a9..dd2df23e1cd 100644 --- a/docs/ru/sql-reference/aggregate-functions/reference/argmax.md +++ b/docs/ru/sql-reference/aggregate-functions/reference/argmax.md @@ -27,13 +27,13 @@ argMax(tuple(arg, val)) **Возвращаемое значение** -- Значение `arg`, соответствующее максимальному значению `val`. +- значение `arg`, соответствующее максимальному значению `val`. Тип: соответствует типу `arg`. Если передан кортеж: -- Кортеж `(arg, val)` c максимальным значением `val` и соответствующим ему `arg`. +- кортеж `(arg, val)` c максимальным значением `val` и соответствующим ему `arg`. Тип: [Tuple](../../../sql-reference/data-types/tuple.md). @@ -52,15 +52,15 @@ argMax(tuple(arg, val)) Запрос: ``` sql -SELECT argMax(user, salary), argMax(tuple(user, salary)) FROM salary; +SELECT argMax(user, salary), argMax(tuple(user, salary), salary), argMax(tuple(user, salary)) FROM salary; ``` Результат: ``` text -┌─argMax(user, salary)─┬─argMax(tuple(user, salary))─┐ -│ director │ ('director',5000) │ -└──────────────────────┴─────────────────────────────┘ +┌─argMax(user, salary)─┬─argMax(tuple(user, salary), salary)─┬─argMax(tuple(user, salary))─┐ +│ director │ ('director',5000) │ ('director',5000) │ +└──────────────────────┴─────────────────────────────────────┴─────────────────────────────┘ ``` [Оригинальная статья](https://clickhouse.tech/docs/ru/sql-reference/aggregate-functions/reference/argmax/) diff --git a/docs/ru/sql-reference/aggregate-functions/reference/avg.md b/docs/ru/sql-reference/aggregate-functions/reference/avg.md index b0bee64ec66..da54254c427 100644 --- a/docs/ru/sql-reference/aggregate-functions/reference/avg.md +++ b/docs/ru/sql-reference/aggregate-functions/reference/avg.md @@ -4,8 +4,63 @@ toc_priority: 5 # avg {#agg_function-avg} -Вычисляет среднее. -Работает только для чисел. -Результат всегда Float64. +Вычисляет среднее арифметическое. + +**Синтаксис** + +``` sql +avg(x) +``` + +**Параметры** + +- `x` — входное значение в формате [Integer](../../../sql-reference/data-types/int-uint.md), [Float](../../../sql-reference/data-types/float.md) или [Decimal](../../../sql-reference/data-types/decimal.md). + +**Возвращаемое значение** + +- среднее арифметическое, в формате [Float64](../../../sql-reference/data-types/float.md). +- `NaN`, если не указано входное значение `x`. + +**Пример** + +Запрос: + +``` sql +SELECT avg(x) FROM values('x Int8', 0, 1, 2, 3, 4, 5); +``` + +Результат: + +``` text +┌─avg(x)─┐ +│ 2.5 │ +└────────┘ +``` + +**Пример** + +Создайте временную таблицу: + +Запрос: + +``` sql +CREATE table test (t UInt8) ENGINE = Memory; +``` + +Получите среднее арифметическое: + +Запрос: + +``` sql +SELECT avg(t) FROM test; +``` + +Результат: + +``` text +┌─avg(x)─┐ +│ nan │ +└────────┘ +``` [Оригинальная статья](https://clickhouse.tech/docs/en/sql-reference/aggregate-functions/reference/avg/) From 3d3639b99ab0d6637d25fbc79fd1a790fe95b83b Mon Sep 17 00:00:00 2001 From: romanzhukov Date: Wed, 3 Mar 2021 01:59:12 +0300 Subject: [PATCH 071/544] DOCSUP-7100: Add avg translation, argmax query fixes. --- .../aggregate-functions/reference/avg.md | 26 +++++++++++-------- .../aggregate-functions/reference/avg.md | 6 ++--- 2 files changed, 17 insertions(+), 15 deletions(-) diff --git a/docs/en/sql-reference/aggregate-functions/reference/avg.md b/docs/en/sql-reference/aggregate-functions/reference/avg.md index 196db2d6b63..b2dd9ae2cb4 100644 --- a/docs/en/sql-reference/aggregate-functions/reference/avg.md +++ b/docs/en/sql-reference/aggregate-functions/reference/avg.md @@ -14,19 +14,12 @@ avg(x) **Arguments** -- `x` — Values. - -`x` must be -[Integer](../../../sql-reference/data-types/int-uint.md), -[floating-point](../../../sql-reference/data-types/float.md), or -[Decimal](../../../sql-reference/data-types/decimal.md). +- `x` — input values, must be [Integer](../../../sql-reference/data-types/int-uint.md), [floating-point](../../../sql-reference/data-types/float.md), or [Decimal](../../../sql-reference/data-types/decimal.md). **Returned value** -- `NaN` if the supplied parameter is empty. -- Mean otherwise. - -**Return type** is always [Float64](../../../sql-reference/data-types/float.md). +- The arithmetic mean, always as [Float64](../../../sql-reference/data-types/float.md). +- `NaN` if the input parameter `x` is empty. **Example** @@ -46,11 +39,20 @@ Result: **Example** +Create a temp table: + Query: ``` sql CREATE table test (t UInt8) ENGINE = Memory; -SELECT avg(t) FROM test +``` + +Get the arithmetic mean: + +Query: + +``` +SELECT avg(t) FROM test; ``` Result: @@ -60,3 +62,5 @@ Result: │ nan │ └────────┘ ``` + +[Оригинальная статья](https://clickhouse.tech/docs/en/sql-reference/aggregate-functions/reference/avg/) diff --git a/docs/ru/sql-reference/aggregate-functions/reference/avg.md b/docs/ru/sql-reference/aggregate-functions/reference/avg.md index da54254c427..d193d3a7763 100644 --- a/docs/ru/sql-reference/aggregate-functions/reference/avg.md +++ b/docs/ru/sql-reference/aggregate-functions/reference/avg.md @@ -47,9 +47,7 @@ SELECT avg(x) FROM values('x Int8', 0, 1, 2, 3, 4, 5); CREATE table test (t UInt8) ENGINE = Memory; ``` -Получите среднее арифметическое: - -Запрос: +Выполните запрос: ``` sql SELECT avg(t) FROM test; @@ -63,4 +61,4 @@ SELECT avg(t) FROM test; └────────┘ ``` -[Оригинальная статья](https://clickhouse.tech/docs/en/sql-reference/aggregate-functions/reference/avg/) +[Оригинальная статья](https://clickhouse.tech/docs/ru/sql-reference/aggregate-functions/reference/avg/) From aecdadd02e7921ce5cd06614cd94ca5798b92d60 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Wed, 3 Mar 2021 02:33:17 +0300 Subject: [PATCH 072/544] Add missing files --- base/glibc-compatibility/tests/CMakeLists.txt | 2 ++ .../glibc-compatibility/tests/memcpy_test.cpp | 26 +++++++++++++++++++ 2 files changed, 28 insertions(+) create mode 100644 base/glibc-compatibility/tests/CMakeLists.txt create mode 100644 base/glibc-compatibility/tests/memcpy_test.cpp diff --git a/base/glibc-compatibility/tests/CMakeLists.txt b/base/glibc-compatibility/tests/CMakeLists.txt new file mode 100644 index 00000000000..f2978a86664 --- /dev/null +++ b/base/glibc-compatibility/tests/CMakeLists.txt @@ -0,0 +1,2 @@ +add_executable(memcpy_test memcpy_test.cpp) +target_link_libraries(memcpy_test common memcpy) diff --git a/base/glibc-compatibility/tests/memcpy_test.cpp b/base/glibc-compatibility/tests/memcpy_test.cpp new file mode 100644 index 00000000000..a1f782380b0 --- /dev/null +++ b/base/glibc-compatibility/tests/memcpy_test.cpp @@ -0,0 +1,26 @@ +#include +#include +#include + +__attribute__((__noinline__)) void memcpy_noinline(void * __restrict dst, const void * __restrict src, size_t size) +{ + memcpy(dst, src, size); +} + + +int main(int, char **) +{ + constexpr size_t buf_size = 100; + char buf[buf_size]{}; + memcpy_noinline(buf, "abc", 3); + + size_t bytes_to_copy = 3; + while (bytes_to_copy * 2 < buf_size) + { + memcpy_noinline(&buf[bytes_to_copy], buf, bytes_to_copy); + bytes_to_copy *= 2; + } + + std::cerr << buf << "\n"; + return 0; +} From 04672a4365409c3eedbce5d79b3f20692fa7f226 Mon Sep 17 00:00:00 2001 From: alexey-milovidov Date: Wed, 3 Mar 2021 22:29:26 +0300 Subject: [PATCH 073/544] Update DateLUTImpl.cpp --- base/common/DateLUTImpl.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/base/common/DateLUTImpl.cpp b/base/common/DateLUTImpl.cpp index 6f4fb3dd5fc..d9dde3a8d0d 100644 --- a/base/common/DateLUTImpl.cpp +++ b/base/common/DateLUTImpl.cpp @@ -56,7 +56,7 @@ DateLUTImpl::DateLUTImpl(const std::string & time_zone_) time_t start_of_day; time_offset_epoch = cctz::convert(cctz::civil_second(lut_start), cctz_time_zone).time_since_epoch().count(); - // Note validated this against all timezones in the system. + // Note: it's validated against all timezones in the system. assert((epoch - lut_start) == daynum_offset_epoch); offset_at_start_of_epoch = cctz_time_zone.lookup(cctz_time_zone.lookup(epoch).pre).offset; From e412bcb490c3519373fab7a69369be82bbf248bf Mon Sep 17 00:00:00 2001 From: alexey-milovidov Date: Wed, 3 Mar 2021 22:30:12 +0300 Subject: [PATCH 074/544] Update DateLUTImpl.cpp --- base/common/DateLUTImpl.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/base/common/DateLUTImpl.cpp b/base/common/DateLUTImpl.cpp index d9dde3a8d0d..563b744e073 100644 --- a/base/common/DateLUTImpl.cpp +++ b/base/common/DateLUTImpl.cpp @@ -95,7 +95,7 @@ DateLUTImpl::DateLUTImpl(const std::string & time_zone_) values.time_at_offset_change_value = 0; values.amount_of_offset_change_value = 0; - // TODO: this partially ignores fractional pre-epoch offsets, which may cause incorrect toRelativeHourNum() results for some timezones, namelly Europe\Minsk + // TODO: this partially ignores fractional pre-epoch offsets, which may cause incorrect toRelativeHourNum() results for some timezones, namelly Europe/Minsk // when pre-May 2 1924 it had an offset of UTC+1:50, and after it was UTC+2h. // https://www.timeanddate.com/time/zone/belarus/minsk?syear=1900 if (start_of_day > 0 && start_of_day % 3600) From 80c17d55376e29c3a73aa231aac78d341af03271 Mon Sep 17 00:00:00 2001 From: alexey-milovidov Date: Wed, 3 Mar 2021 22:41:10 +0300 Subject: [PATCH 075/544] Update DateLUTImpl.cpp --- base/common/DateLUTImpl.cpp | 2 -- 1 file changed, 2 deletions(-) diff --git a/base/common/DateLUTImpl.cpp b/base/common/DateLUTImpl.cpp index 563b744e073..68a3aa97e51 100644 --- a/base/common/DateLUTImpl.cpp +++ b/base/common/DateLUTImpl.cpp @@ -140,8 +140,6 @@ DateLUTImpl::DateLUTImpl(const std::string & time_zone_) } while (i < DATE_LUT_SIZE && lut[i - 1].year <= DATE_LUT_MAX_YEAR); -// date_lut_max = start_of_day; - /// Fill excessive part of lookup table. This is needed only to simplify handling of overflow cases. while (i < DATE_LUT_SIZE) { From 2f23f1b123517251b5661a971e7499f0e9fc99b7 Mon Sep 17 00:00:00 2001 From: alexey-milovidov Date: Wed, 3 Mar 2021 22:49:20 +0300 Subject: [PATCH 076/544] Update DateLUTImpl.h --- base/common/DateLUTImpl.h | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/base/common/DateLUTImpl.h b/base/common/DateLUTImpl.h index 5a12ad5dc13..72786e31cbc 100644 --- a/base/common/DateLUTImpl.h +++ b/base/common/DateLUTImpl.h @@ -136,18 +136,20 @@ public: /// Since most of the modern timezones have a DST change aligned to 15 minutes, to save as much space as possible inside Value, /// we are dividing any offset change related value by this factor before setting it to Value, /// hence it has to be explicitly multiplied back by this factor before being used. - static const UInt16 OffsetChangeFactor = 900; + static constexpr UInt16 OffsetChangeFactor = 900; }; static_assert(sizeof(Values) == 16); private: - // Mask is all-ones to allow efficient protection against overflow. - static const UInt32 date_lut_mask = 0x1ffff; + /// Mask is all-ones to allow efficient protection against overflow. + static constexpr UInt32 date_lut_mask = 0x1ffff; static_assert(date_lut_mask == DATE_LUT_SIZE - 1); - const UInt32 daynum_offset_epoch = 16436; // offset to epoch in days (ExtendedDayNum) of the first day in LUT. + /// Offset to epoch in days (ExtendedDayNum) of the first day in LUT. + static constexpr UInt32 daynum_offset_epoch = 16436; + static_assert(daynum_offset_epoch == (DATE_LUT_MIN_YEAR - 1970) * 365 + (1970 - DATE_LUT_MIN_YEAR / 4 * 4) / 4); /// Lookup table is indexed by LUTIndex. /// Day nums are the same in all time zones. 1970-01-01 is 0 and so on. From 0889e0da87ea511cc30669b6c2aec888052d00ee Mon Sep 17 00:00:00 2001 From: alexey-milovidov Date: Wed, 3 Mar 2021 22:52:07 +0300 Subject: [PATCH 077/544] Update DateLUTImpl.h --- base/common/DateLUTImpl.h | 1 + 1 file changed, 1 insertion(+) diff --git a/base/common/DateLUTImpl.h b/base/common/DateLUTImpl.h index 72786e31cbc..9b36c60eb5e 100644 --- a/base/common/DateLUTImpl.h +++ b/base/common/DateLUTImpl.h @@ -148,6 +148,7 @@ private: static_assert(date_lut_mask == DATE_LUT_SIZE - 1); /// Offset to epoch in days (ExtendedDayNum) of the first day in LUT. + /// "epoch" is the Unix Epoch (starts at unix timestamp zero) static constexpr UInt32 daynum_offset_epoch = 16436; static_assert(daynum_offset_epoch == (DATE_LUT_MIN_YEAR - 1970) * 365 + (1970 - DATE_LUT_MIN_YEAR / 4 * 4) / 4); From 3e19f4a00fc03d466b3648305e3b236653485e8b Mon Sep 17 00:00:00 2001 From: alexey-milovidov Date: Wed, 3 Mar 2021 23:09:59 +0300 Subject: [PATCH 078/544] Update DateLUTImpl.h --- base/common/DateLUTImpl.h | 1 + 1 file changed, 1 insertion(+) diff --git a/base/common/DateLUTImpl.h b/base/common/DateLUTImpl.h index 9b36c60eb5e..79bf1a9f127 100644 --- a/base/common/DateLUTImpl.h +++ b/base/common/DateLUTImpl.h @@ -166,6 +166,7 @@ private: /// UTC offset at beginning of the Unix epoch. The same as unix timestamp of 1970-01-01 00:00:00 local time. time_t offset_at_start_of_epoch; + /// UTC offset at the beginning of the first supported year. time_t offset_at_start_of_lut; bool offset_is_whole_number_of_hours_everytime; time_t time_offset_epoch; From cf0912ba02c8e89fd1d913318a9c117ab336a5df Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Wed, 3 Mar 2021 23:14:33 +0300 Subject: [PATCH 079/544] Remove unused field --- base/common/DateLUTImpl.cpp | 1 - base/common/DateLUTImpl.h | 2 -- base/common/tests/gtest_DateLutImpl.cpp | 1 - 3 files changed, 4 deletions(-) diff --git a/base/common/DateLUTImpl.cpp b/base/common/DateLUTImpl.cpp index 68a3aa97e51..bf180acb835 100644 --- a/base/common/DateLUTImpl.cpp +++ b/base/common/DateLUTImpl.cpp @@ -54,7 +54,6 @@ DateLUTImpl::DateLUTImpl(const std::string & time_zone_) const cctz::civil_day epoch{1970, 1, 1}; const cctz::civil_day lut_start{DATE_LUT_MIN_YEAR, 1, 1}; time_t start_of_day; - time_offset_epoch = cctz::convert(cctz::civil_second(lut_start), cctz_time_zone).time_since_epoch().count(); // Note: it's validated against all timezones in the system. assert((epoch - lut_start) == daynum_offset_epoch); diff --git a/base/common/DateLUTImpl.h b/base/common/DateLUTImpl.h index 79bf1a9f127..429db332b49 100644 --- a/base/common/DateLUTImpl.h +++ b/base/common/DateLUTImpl.h @@ -169,7 +169,6 @@ private: /// UTC offset at the beginning of the first supported year. time_t offset_at_start_of_lut; bool offset_is_whole_number_of_hours_everytime; - time_t time_offset_epoch; /// Time zone name. std::string time_zone; @@ -230,7 +229,6 @@ public: // Methods only for unit-testing, it makes very little sense to use it from user code. auto getOffsetAtStartOfEpoch() const { return offset_at_start_of_epoch; } auto getOffsetIsWholNumberOfHoursEveryWhere() const { return offset_is_whole_number_of_hours_everytime; } - auto getTimeOffsetEpoch() const { return time_offset_epoch; } auto getTimeOffsetAtStartOfLUT() const { return offset_at_start_of_lut; } /// All functions below are thread-safe; arguments are not checked. diff --git a/base/common/tests/gtest_DateLutImpl.cpp b/base/common/tests/gtest_DateLutImpl.cpp index 9169d9e768f..8cc4fbbbfb0 100644 --- a/base/common/tests/gtest_DateLutImpl.cpp +++ b/base/common/tests/gtest_DateLutImpl.cpp @@ -292,7 +292,6 @@ TEST_P(DateLUTWithTimeZone, VaidateTimeComponentsAroundEpoch) << "\n\ttimestamp: " << i << "\n\t offset at start of epoch : " << lut.getOffsetAtStartOfEpoch() << "\n\t offset_is_whole_number_of_hours_everytime : " << lut.getOffsetIsWholNumberOfHoursEveryWhere() - << "\n\t time_offset_epoch : " << lut.getTimeOffsetEpoch() << "\n\t offset_at_start_of_lut : " << lut.getTimeOffsetAtStartOfLUT()); EXPECT_GE(24, lut.toHour(i)); From aaef0c5ebd1246fa633371ee2d3635b2539bdcee Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Wed, 3 Mar 2021 23:19:26 +0300 Subject: [PATCH 080/544] Fix build --- base/common/DateLUTImpl.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/base/common/DateLUTImpl.h b/base/common/DateLUTImpl.h index 429db332b49..5275cc83abb 100644 --- a/base/common/DateLUTImpl.h +++ b/base/common/DateLUTImpl.h @@ -150,7 +150,7 @@ private: /// Offset to epoch in days (ExtendedDayNum) of the first day in LUT. /// "epoch" is the Unix Epoch (starts at unix timestamp zero) static constexpr UInt32 daynum_offset_epoch = 16436; - static_assert(daynum_offset_epoch == (DATE_LUT_MIN_YEAR - 1970) * 365 + (1970 - DATE_LUT_MIN_YEAR / 4 * 4) / 4); + static_assert(daynum_offset_epoch == (1970 - DATE_LUT_MIN_YEAR) * 365 + (1970 - DATE_LUT_MIN_YEAR / 4 * 4) / 4); /// Lookup table is indexed by LUTIndex. /// Day nums are the same in all time zones. 1970-01-01 is 0 and so on. From ed0099a11308719bc8319c9aac5707e092379f29 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Wed, 3 Mar 2021 23:24:56 +0300 Subject: [PATCH 081/544] Maybe unused condition --- base/common/DateLUTImpl.h | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/base/common/DateLUTImpl.h b/base/common/DateLUTImpl.h index 5275cc83abb..66df4744b72 100644 --- a/base/common/DateLUTImpl.h +++ b/base/common/DateLUTImpl.h @@ -179,10 +179,11 @@ private: const UInt32 guess = ((t / 86400) + daynum_offset_epoch) & date_lut_mask; /// UTC offset is from -12 to +14 in all known time zones. This requires checking only three indices. - if ((guess == daynum_offset_epoch || t >= lut[guess].date) && t < lut[UInt32(guess + 1)].date) + if (t >= lut[guess].date && t < lut[UInt32(guess + 1)].date) return LUTIndex{guess}; - /// Time zones that have offset 0 from UTC do daylight saving time change (if any) towards increasing UTC offset (example: British Standard Time). + /// Time zones that have offset 0 from UTC do daylight saving time change (if any) + /// towards increasing UTC offset (example: British Standard Time). if (t >= lut[UInt32(guess + 1)].date) return LUTIndex(guess + 1); From b1b3db09d765c008bdb053bdc2693d02f546055e Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Wed, 3 Mar 2021 23:34:15 +0300 Subject: [PATCH 082/544] Remove commented out code --- base/common/DateLUTImpl.h | 12 +----------- 1 file changed, 1 insertion(+), 11 deletions(-) diff --git a/base/common/DateLUTImpl.h b/base/common/DateLUTImpl.h index 66df4744b72..890ed3ae173 100644 --- a/base/common/DateLUTImpl.h +++ b/base/common/DateLUTImpl.h @@ -189,6 +189,7 @@ private: if (lut[guess - 1].date <= t) return LUTIndex(guess - 1); + return LUTIndex(guess - 2); } @@ -212,12 +213,6 @@ private: return i; } -// template -// inline LUTIndex toLUTIndex(T t) const -// { -// return LUTIndex{static_cast(t) & date_lut_mask}; -// } - template inline const Values & find(V v) const { @@ -275,11 +270,6 @@ public: return toDayNum(i - (lut[i].day_of_month - 1)); } -// inline DayNum toFirstDayNumOfMonth(time_t t) const -// { -// return toFirstDayNumOfMonth(toDayNum(t)); -// } - /// Round down to start of quarter. template inline ExtendedDayNum toFirstDayNumOfQuarter(V v) const From 6f5877abdfcf686913526c013ad09921c995d6df Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Wed, 3 Mar 2021 23:53:52 +0300 Subject: [PATCH 083/544] Fix build after merge with master --- base/common/DateLUTImpl.h | 221 +++++++++++++++++++------------------- 1 file changed, 111 insertions(+), 110 deletions(-) diff --git a/base/common/DateLUTImpl.h b/base/common/DateLUTImpl.h index 2fdf293cef4..740411b7113 100644 --- a/base/common/DateLUTImpl.h +++ b/base/common/DateLUTImpl.h @@ -213,8 +213,8 @@ private: return i; } - template - inline const Values & find(V v) const + template + inline const Values & find(DateOrTime v) const { return lut[toLUTIndex(v)]; } @@ -234,54 +234,53 @@ public: return d; } - template - inline ExtendedDayNum toDayNum(V v) const + template + inline ExtendedDayNum toDayNum(DateOrTime v) const { return ExtendedDayNum{static_cast(toLUTIndex(v).toUnderType() - daynum_offset_epoch)}; } /// Round down to start of monday. - template - inline time_t toFirstDayOfWeek(V v) const + template + inline time_t toFirstDayOfWeek(DateOrTime v) const { - const auto i = toLUTIndex(v); + const LUTIndex i = toLUTIndex(v); return lut[i - (lut[i].day_of_week - 1)].date; } - template - inline ExtendedDayNum toFirstDayNumOfWeek(V v) const + template + inline ExtendedDayNum toFirstDayNumOfWeek(DateOrTime v) const { - const auto i = toLUTIndex(v); + const LUTIndex i = toLUTIndex(v); return toDayNum(i - (lut[i].day_of_week - 1)); } /// Round down to start of month. - template - inline time_t toFirstDayOfMonth(V v) const + template + inline time_t toFirstDayOfMonth(DateOrTime v) const { - const auto i = toLUTIndex(v); + const LUTIndex i = toLUTIndex(v); return lut[i - (lut[i].day_of_month - 1)].date; } - template - inline ExtendedDayNum toFirstDayNumOfMonth(V v) const + template + inline ExtendedDayNum toFirstDayNumOfMonth(DateOrTime v) const { - const auto i = toLUTIndex(v); + const LUTIndex i = toLUTIndex(v); return toDayNum(i - (lut[i].day_of_month - 1)); } /// Round down to start of quarter. - template - inline ExtendedDayNum toFirstDayNumOfQuarter(V v) const + template + inline ExtendedDayNum toFirstDayNumOfQuarter(DateOrTime v) const { return toDayNum(toFirstDayOfQuarterIndex(v)); } - template - inline LUTIndex toFirstDayOfQuarterIndex(V v) const + template + inline LUTIndex toFirstDayOfQuarterIndex(DateOrTime v) const { - //return fromDayNum(toFirstDayNumOfQuarter(v)); - auto index = toLUTIndex(v); + LUTIndex index = toLUTIndex(v); size_t month_inside_quarter = (lut[index].month - 1) % 3; index -= lut[index].day_of_month; @@ -294,8 +293,8 @@ public: return index + 1; } - template - inline time_t toFirstDayOfQuarter(V v) const + template + inline time_t toFirstDayOfQuarter(DateOrTime v) const { return toDate(toFirstDayOfQuarterIndex(v)); } @@ -306,36 +305,36 @@ public: return lut[years_lut[lut[findIndex(t)].year - DATE_LUT_MIN_YEAR]].date; } - template - inline LUTIndex toFirstDayNumOfYearIndex(V v) const + template + inline LUTIndex toFirstDayNumOfYearIndex(DateOrTime v) const { return years_lut[lut[toLUTIndex(v)].year - DATE_LUT_MIN_YEAR]; } - template - inline ExtendedDayNum toFirstDayNumOfYear(V v) const + template + inline ExtendedDayNum toFirstDayNumOfYear(DateOrTime v) const { return toDayNum(toFirstDayNumOfYearIndex(v)); } inline time_t toFirstDayOfNextMonth(time_t t) const { - auto index = findIndex(t); + LUTIndex index = findIndex(t); index += 32 - lut[index].day_of_month; return lut[index - (lut[index].day_of_month - 1)].date; } inline time_t toFirstDayOfPrevMonth(time_t t) const { - auto index = findIndex(t); + LUTIndex index = findIndex(t); index -= lut[index].day_of_month; return lut[index - (lut[index].day_of_month - 1)].date; } - template - inline UInt8 daysInMonth(V v) const + template + inline UInt8 daysInMonth(DateOrTime value) const { - const auto i = toLUTIndex(v); + const LUTIndex i = toLUTIndex(value); return lut[i].days_in_month; } @@ -359,7 +358,7 @@ public: inline time_t toTime(time_t t) const { - auto index = findIndex(t); + const LUTIndex index = findIndex(t); if (unlikely(index == daynum_offset_epoch || index > DATE_LUT_MAX_DAY_NUM)) return t + offset_at_start_of_epoch; @@ -374,7 +373,7 @@ public: inline unsigned toHour(time_t t) const { - auto index = findIndex(t); + const LUTIndex index = findIndex(t); /// If it is overflow case, /// than limit number of hours to avoid insane results like 1970-01-01 89:28:15 @@ -398,7 +397,7 @@ public: */ inline time_t timezoneOffset(time_t t) const { - const auto index = findIndex(t); + const LUTIndex index = findIndex(t); /// Calculate daylight saving offset first. /// Because the "amount_of_offset_change" in LUT entry only exists in the change day, it's costly to scan it from the very begin. @@ -446,10 +445,10 @@ public: /// To consider the DST changing situation within this day. /// also make the special timezones with no whole hour offset such as 'Australia/Lord_Howe' been taken into account - DayNum index = findIndex(t); + LUTIndex index = findIndex(t); UInt32 res = t - lut[index].date; - if (lut[index].amount_of_offset_change != 0 && t >= lut[index].date + lut[index].time_at_offset_change) - res += lut[index].amount_of_offset_change; + if (lut[index].amount_of_offset_change() != 0 && t >= lut[index].date + lut[index].time_at_offset_change()) + res += lut[index].amount_of_offset_change(); return res / 60 % 60; } @@ -476,47 +475,51 @@ public: * because the same calendar day starts/ends at different timestamps in different time zones) */ -// inline DayNum toDayNum(time_t t) const { return DayNum{findIndex(t) - daynum_offset_epoch}; } -// inline ExtendedDayNum toExtendedDayNum(time_t t) const { return ExtendedDayNum{findIndex(t) - daynum_offset_epoch}; } inline time_t fromDayNum(DayNum d) const { return lut[toLUTIndex(d)].date; } inline time_t fromDayNum(ExtendedDayNum d) const { return lut[toLUTIndex(d)].date; } - template - inline time_t toDate(V v) const { return lut[toLUTIndex(v)].date; } - template - inline unsigned toMonth(V v) const { return lut[toLUTIndex(v)].month; } - template - inline unsigned toQuarter(V v) const { return (lut[toLUTIndex(v)].month - 1) / 3 + 1; } - template - inline Int16 toYear(V v) const { return lut[toLUTIndex(v)].year; } - template - inline unsigned toDayOfWeek(V v) const { return lut[toLUTIndex(v)].day_of_week; } - template - inline unsigned toDayOfMonth(V v) const { return lut[toLUTIndex(v)].day_of_month; } - template - inline unsigned toDayOfYear(V v) const + template + inline time_t toDate(DateOrTime v) const { return lut[toLUTIndex(v)].date; } + + template + inline unsigned toMonth(DateOrTime v) const { return lut[toLUTIndex(v)].month; } + + template + inline unsigned toQuarter(DateOrTime v) const { return (lut[toLUTIndex(v)].month - 1) / 3 + 1; } + + template + inline Int16 toYear(DateOrTime v) const { return lut[toLUTIndex(v)].year; } + + template + inline unsigned toDayOfWeek(DateOrTime v) const { return lut[toLUTIndex(v)].day_of_week; } + + template + inline unsigned toDayOfMonth(DateOrTime v) const { return lut[toLUTIndex(v)].day_of_month; } + + template + inline unsigned toDayOfYear(DateOrTime v) const { // TODO: different overload for ExtendedDayNum - const auto i = toLUTIndex(v); + const LUTIndex i = toLUTIndex(v); return i + 1 - toFirstDayNumOfYearIndex(i); } /// Number of week from some fixed moment in the past. Week begins at monday. /// (round down to monday and divide DayNum by 7; we made an assumption, /// that in domain of the function there was no weeks with any other number of days than 7) - template - inline unsigned toRelativeWeekNum(V v) const + template + inline unsigned toRelativeWeekNum(DateOrTime v) const { - const auto i = toLUTIndex(v); + const LUTIndex i = toLUTIndex(v); /// We add 8 to avoid underflow at beginning of unix epoch. return toDayNum(i + 8 - toDayOfWeek(i)) / 7; } /// Get year that contains most of the current week. Week begins at monday. - template - inline unsigned toISOYear(V v) const + template + inline unsigned toISOYear(DateOrTime v) const { - const auto i = toLUTIndex(v); + const LUTIndex i = toLUTIndex(v); /// That's effectively the year of thursday of current week. return toYear(toLUTIndex(i + 4 - toDayOfWeek(i))); } @@ -524,10 +527,10 @@ public: /// ISO year begins with a monday of the week that is contained more than by half in the corresponding calendar year. /// Example: ISO year 2019 begins at 2018-12-31. And ISO year 2017 begins at 2017-01-02. /// https://en.wikipedia.org/wiki/ISO_week_date - template - inline LUTIndex toFirstDayNumOfISOYearIndex(V v) const + template + inline LUTIndex toFirstDayNumOfISOYearIndex(DateOrTime v) const { - const auto i = toLUTIndex(v); + const LUTIndex i = toLUTIndex(v); auto iso_year = toISOYear(i); const auto first_day_of_year = years_lut[iso_year - DATE_LUT_MIN_YEAR]; @@ -538,8 +541,8 @@ public: : first_day_of_year + 8 - first_day_of_week_of_year}; } - template - inline ExtendedDayNum toFirstDayNumOfISOYear(V v) const + template + inline ExtendedDayNum toFirstDayNumOfISOYear(DateOrTime v) const { return toDayNum(toFirstDayNumOfISOYearIndex(v)); } @@ -551,8 +554,8 @@ public: /// ISO 8601 week number. Week begins at monday. /// The week number 1 is the first week in year that contains 4 or more days (that's more than half). - template - inline unsigned toISOWeek(V v) const + template + inline unsigned toISOWeek(DateOrTime v) const { return 1 + (toFirstDayNumOfWeek(v) - toFirstDayNumOfISOYear(v)) / 7; } @@ -590,8 +593,8 @@ public: Otherwise it is the last week of the previous year, and the next week is week 1. */ - template - inline YearWeek toYearWeek(V v, UInt8 week_mode) const + template + inline YearWeek toYearWeek(DateOrTime v, UInt8 week_mode) const { const bool newyear_day_mode = week_mode & static_cast(WeekModeFlag::NEWYEAR_DAY); week_mode = check_week_mode(week_mode); @@ -599,7 +602,7 @@ public: bool week_year_mode = week_mode & static_cast(WeekModeFlag::YEAR); const bool first_weekday_mode = week_mode & static_cast(WeekModeFlag::FIRST_WEEKDAY); - const auto i = toLUTIndex(v); + const LUTIndex i = toLUTIndex(v); // Calculate week number of WeekModeFlag::NEWYEAR_DAY mode if (newyear_day_mode) @@ -647,13 +650,13 @@ public: /// Calculate week number of WeekModeFlag::NEWYEAR_DAY mode /// The week number 1 is the first week in year that contains January 1, - template - inline YearWeek toYearWeekOfNewyearMode(V v, bool monday_first_mode) const + template + inline YearWeek toYearWeekOfNewyearMode(DateOrTime v, bool monday_first_mode) const { YearWeek yw(0, 0); UInt16 offset_day = monday_first_mode ? 0U : 1U; - const auto i = LUTIndex(v); + const LUTIndex i = LUTIndex(v); // Checking the week across the year yw.first = toYear(i + 7 - toDayOfWeek(i + offset_day)); @@ -661,7 +664,7 @@ public: auto first_day = makeLUTIndex(yw.first, 1, 1); auto this_day = i; - //TODO: do not perform calculations in terms of DayNum, since that would under/overflow for extended range. + // TODO: do not perform calculations in terms of DayNum, since that would under/overflow for extended range. if (monday_first_mode) { // Rounds down a date to the nearest Monday. @@ -680,11 +683,9 @@ public: return yw; } - /** - * get first day of week with week_mode, return Sunday or Monday - */ - template - inline ExtendedDayNum toFirstDayNumOfWeek(V v, UInt8 week_mode) const + /// Get first day of week with week_mode, return Sunday or Monday + template + inline ExtendedDayNum toFirstDayNumOfWeek(DateOrTime v, UInt8 week_mode) const { bool monday_first_mode = week_mode & static_cast(WeekModeFlag::MONDAY_FIRST); if (monday_first_mode) @@ -709,10 +710,10 @@ public: /** Calculate weekday from d. * Returns 0 for monday, 1 for tuesday... */ - template - inline unsigned calc_weekday(V v, bool sunday_first_day_of_week) const + template + inline unsigned calc_weekday(DateOrTime v, bool sunday_first_day_of_week) const { - const auto i = toLUTIndex(v); + const LUTIndex i = toLUTIndex(v); if (!sunday_first_day_of_week) return toDayOfWeek(i) - 1; else @@ -726,17 +727,17 @@ public: } /// Number of month from some fixed moment in the past (year * 12 + month) - template - inline unsigned toRelativeMonthNum(V v) const + template + inline unsigned toRelativeMonthNum(DateOrTime v) const { - const auto i = toLUTIndex(v); + const LUTIndex i = toLUTIndex(v); return lut[i].year * 12 + lut[i].month; } - template - inline unsigned toRelativeQuarterNum(V v) const + template + inline unsigned toRelativeQuarterNum(DateOrTime v) const { - const auto i = toLUTIndex(v); + const LUTIndex i = toLUTIndex(v); return lut[i].year * 4 + (lut[i].month - 1) / 3; } @@ -751,8 +752,8 @@ public: return (t + 86400 - offset_at_start_of_epoch) / 3600; } - template - inline time_t toRelativeHourNum(V v) const + template + inline time_t toRelativeHourNum(DateOrTime v) const { return toRelativeHourNum(lut[toLUTIndex(v)].date); } @@ -762,19 +763,19 @@ public: return t / 60; } - template - inline time_t toRelativeMinuteNum(V v) const + template + inline time_t toRelativeMinuteNum(DateOrTime v) const { return toRelativeMinuteNum(lut[toLUTIndex(v)].date); } - template - inline ExtendedDayNum toStartOfYearInterval(V v, UInt64 years) const + template + inline ExtendedDayNum toStartOfYearInterval(DateOrTime v, UInt64 years) const { if (years == 1) return toFirstDayNumOfYear(v); - const auto i = toLUTIndex(v); + const LUTIndex i = toLUTIndex(v); return toDayNum(years_lut[lut[i].year / years * years - DATE_LUT_MIN_YEAR]); } @@ -789,8 +790,8 @@ public: { if (months == 1) return toFirstDayNumOfMonth(d); - const auto & date = lut[toLUTIndex(d)]; - UInt32 month_total_index = (date.year - DATE_LUT_MIN_YEAR) * 12 + date.month - 1; + const Values & values = lut[toLUTIndex(d)]; + UInt32 month_total_index = (values.year - DATE_LUT_MIN_YEAR) * 12 + values.month - 1; return toDayNum(years_months_lut[month_total_index / months * months]); } @@ -876,18 +877,18 @@ public: return res; } - template - inline const Values & getValues(V v) const { return lut[toLUTIndex(v)]; } + template + inline const Values & getValues(DateOrTime v) const { return lut[toLUTIndex(v)]; } - template - inline UInt32 toNumYYYYMM(V v) const + template + inline UInt32 toNumYYYYMM(DateOrTime v) const { const Values & values = getValues(v); return values.year * 100 + values.month; } - template - inline UInt32 toNumYYYYMMDD(V v) const + template + inline UInt32 toNumYYYYMMDD(DateOrTime v) const { const Values & values = getValues(v); return values.year * 10000 + values.month * 100 + values.day_of_month; @@ -932,7 +933,7 @@ public: inline NO_SANITIZE_UNDEFINED time_t addDays(time_t t, Int64 delta) const { - auto index = findIndex(t); + LUTIndex index = findIndex(t); time_t time_offset = toHour(t) * 3600 + toMinute(t) * 60 + toSecond(t); index += delta; @@ -962,8 +963,8 @@ public: return day_of_month; } - template - inline LUTIndex addMonthsIndex(V v, Int64 delta) const + template + inline LUTIndex addMonthsIndex(DateOrTime v, Int64 delta) const { const Values & values = lut[toLUTIndex(v)]; @@ -1016,8 +1017,8 @@ public: return addMonths(d, delta * 3); } - template - inline LUTIndex NO_SANITIZE_UNDEFINED addYearsIndex(V v, Int64 delta) const + template + inline LUTIndex NO_SANITIZE_UNDEFINED addYearsIndex(DateOrTime v, Int64 delta) const { const Values & values = lut[toLUTIndex(v)]; From fdc00beb772a0c83032f6c63e61da0d5b6ac9ff9 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Wed, 3 Mar 2021 23:56:59 +0300 Subject: [PATCH 084/544] Whitespaces --- base/common/DateLUTImpl.h | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/base/common/DateLUTImpl.h b/base/common/DateLUTImpl.h index 740411b7113..159219bab83 100644 --- a/base/common/DateLUTImpl.h +++ b/base/common/DateLUTImpl.h @@ -49,16 +49,19 @@ public: // has to be a separate type to support overloading // TODO: make sure that any arithmetic on LUTIndex actually results in valid LUTIndex. STRONG_TYPEDEF(UInt32, LUTIndex) + template friend inline LUTIndex operator+(const LUTIndex & index, const T v) { return LUTIndex{(index.toUnderType() + v) & date_lut_mask}; } + template friend inline LUTIndex operator+(const T v, const LUTIndex & index) { return LUTIndex{(v + index.toUnderType()) & date_lut_mask}; } + friend inline LUTIndex operator+(const LUTIndex & index, const LUTIndex & v) { return LUTIndex{(index.toUnderType() + v.toUnderType()) & date_lut_mask}; @@ -69,11 +72,13 @@ public: { return LUTIndex{(index.toUnderType() - v) & date_lut_mask}; } + template friend inline LUTIndex operator-(const T v, const LUTIndex & index) { return LUTIndex{(v - index.toUnderType()) & date_lut_mask}; } + friend inline LUTIndex operator-(const LUTIndex & index, const LUTIndex & v) { return LUTIndex{(index.toUnderType() - v.toUnderType()) & date_lut_mask}; @@ -84,6 +89,7 @@ public: { return LUTIndex{(index.toUnderType() * v) & date_lut_mask}; } + template friend inline LUTIndex operator*(const T v, const LUTIndex & index) { @@ -95,6 +101,7 @@ public: { return LUTIndex{(index.toUnderType() / v) & date_lut_mask}; } + template friend inline LUTIndex operator/(const T v, const LUTIndex & index) { From 51d51e474858cddaadc5600995efec37ce6eaa67 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Thu, 4 Mar 2021 00:44:58 +0300 Subject: [PATCH 085/544] Return private --- base/common/DateLUT.cpp | 2 +- base/common/DateLUTImpl.h | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/base/common/DateLUT.cpp b/base/common/DateLUT.cpp index 6ff0884701c..d14b63cd70a 100644 --- a/base/common/DateLUT.cpp +++ b/base/common/DateLUT.cpp @@ -152,7 +152,7 @@ const DateLUTImpl & DateLUT::getImplementation(const std::string & time_zone) co auto it = impls.emplace(time_zone, nullptr).first; if (!it->second) - it->second = std::make_unique(time_zone); + it->second = std::unique_ptr(new DateLUTImpl(time_zone)); return *it->second; } diff --git a/base/common/DateLUTImpl.h b/base/common/DateLUTImpl.h index 159219bab83..8c2bbb3262d 100644 --- a/base/common/DateLUTImpl.h +++ b/base/common/DateLUTImpl.h @@ -37,7 +37,8 @@ using YearWeek = std::pair; */ class DateLUTImpl { -public: +private: + friend class DateLUT; explicit DateLUTImpl(const std::string & time_zone); DateLUTImpl(const DateLUTImpl &) = delete; From 2632b568ae2ce56a635c40d2a4e119c731d2b91c Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Thu, 4 Mar 2021 01:34:30 +0300 Subject: [PATCH 086/544] Move tests to appropriate place --- base/common/tests/CMakeLists.txt | 14 ----- base/common/tests/date_lut2.cpp | 53 ---------------- base/common/tests/date_lut3.cpp | 62 ------------------- .../tests/date_lut_default_timezone.cpp | 31 ---------- src/CMakeLists.txt | 11 +++- .../Common}/tests/gtest_DateLutImpl.cpp | 2 +- .../Common}/tests/gtest_find_symbols.cpp | 0 .../gtest_global_register_functions.h.bak | 17 +++++ .../Common}/tests/gtest_json_test.cpp | 10 +-- .../Common}/tests/gtest_strong_typedef.cpp | 0 10 files changed, 33 insertions(+), 167 deletions(-) delete mode 100644 base/common/tests/date_lut2.cpp delete mode 100644 base/common/tests/date_lut3.cpp delete mode 100644 base/common/tests/date_lut_default_timezone.cpp rename {base/common => src/Common}/tests/gtest_DateLutImpl.cpp (99%) rename {base/common => src/Common}/tests/gtest_find_symbols.cpp (100%) create mode 100644 src/Common/tests/gtest_global_register_functions.h.bak rename {base/common => src/Common}/tests/gtest_json_test.cpp (99%) rename {base/common => src/Common}/tests/gtest_strong_typedef.cpp (100%) diff --git a/base/common/tests/CMakeLists.txt b/base/common/tests/CMakeLists.txt index b335b302cb0..92be2f67c94 100644 --- a/base/common/tests/CMakeLists.txt +++ b/base/common/tests/CMakeLists.txt @@ -1,27 +1,13 @@ include (${ClickHouse_SOURCE_DIR}/cmake/add_check.cmake) -add_executable (date_lut2 date_lut2.cpp) -add_executable (date_lut3 date_lut3.cpp) -add_executable (date_lut_default_timezone date_lut_default_timezone.cpp) add_executable (local_date_time_comparison local_date_time_comparison.cpp) add_executable (realloc-perf allocator.cpp) set(PLATFORM_LIBS ${CMAKE_DL_LIBS}) -target_link_libraries (date_lut2 PRIVATE common ${PLATFORM_LIBS}) -target_link_libraries (date_lut3 PRIVATE common ${PLATFORM_LIBS}) -target_link_libraries (date_lut_default_timezone PRIVATE common ${PLATFORM_LIBS}) target_link_libraries (local_date_time_comparison PRIVATE common) target_link_libraries (realloc-perf PRIVATE common) add_check(local_date_time_comparison) -if(USE_GTEST) - add_executable(unit_tests_libcommon gtest_json_test.cpp gtest_strong_typedef.cpp gtest_find_symbols.cpp gtest_DateLutImpl.cpp - ${CMAKE_BINARY_DIR}/src/Storages/System/StorageSystemTimeZones.generated.cpp - ) - target_link_libraries(unit_tests_libcommon PRIVATE common ${GTEST_MAIN_LIBRARIES} ${GTEST_LIBRARIES}) - add_check(unit_tests_libcommon) -endif() - add_executable (dump_variable dump_variable.cpp) target_link_libraries (dump_variable PRIVATE clickhouse_common_io) diff --git a/base/common/tests/date_lut2.cpp b/base/common/tests/date_lut2.cpp deleted file mode 100644 index 6dcf5e8adf2..00000000000 --- a/base/common/tests/date_lut2.cpp +++ /dev/null @@ -1,53 +0,0 @@ -#include -#include - -#include - - -static std::string toString(time_t Value) -{ - struct tm tm; - char buf[96]; - - localtime_r(&Value, &tm); - snprintf(buf, sizeof(buf), "%04d-%02d-%02d %02d:%02d:%02d", - tm.tm_year + 1900, tm.tm_mon + 1, tm.tm_mday, tm.tm_hour, tm.tm_min, tm.tm_sec); - - return buf; -} - -static time_t orderedIdentifierToDate(unsigned value) -{ - struct tm tm; - - memset(&tm, 0, sizeof(tm)); - - tm.tm_year = value / 10000 - 1900; - tm.tm_mon = (value % 10000) / 100 - 1; - tm.tm_mday = value % 100; - tm.tm_isdst = -1; - - return mktime(&tm); -} - - -void loop(time_t begin, time_t end, int step) -{ - const auto & date_lut = DateLUT::instance(); - - for (time_t t = begin; t < end; t += step) - std::cout << toString(t) - << ", " << toString(date_lut.toTime(t)) - << ", " << date_lut.toHour(t) - << std::endl; -} - - -int main(int, char **) -{ - loop(orderedIdentifierToDate(20101031), orderedIdentifierToDate(20101101), 15 * 60); - loop(orderedIdentifierToDate(20100328), orderedIdentifierToDate(20100330), 15 * 60); - loop(orderedIdentifierToDate(20141020), orderedIdentifierToDate(20141106), 15 * 60); - - return 0; -} diff --git a/base/common/tests/date_lut3.cpp b/base/common/tests/date_lut3.cpp deleted file mode 100644 index 411765d2b2a..00000000000 --- a/base/common/tests/date_lut3.cpp +++ /dev/null @@ -1,62 +0,0 @@ -#include -#include - -#include - -#include - - -static std::string toString(time_t Value) -{ - struct tm tm; - char buf[96]; - - localtime_r(&Value, &tm); - snprintf(buf, sizeof(buf), "%04d-%02d-%02d %02d:%02d:%02d", - tm.tm_year + 1900, tm.tm_mon + 1, tm.tm_mday, tm.tm_hour, tm.tm_min, tm.tm_sec); - - return buf; -} - -static time_t orderedIdentifierToDate(unsigned value) -{ - struct tm tm; - - memset(&tm, 0, sizeof(tm)); - - tm.tm_year = value / 10000 - 1900; - tm.tm_mon = (value % 10000) / 100 - 1; - tm.tm_mday = value % 100; - tm.tm_isdst = -1; - - return mktime(&tm); -} - - -void loop(time_t begin, time_t end, int step) -{ - const auto & date_lut = DateLUT::instance(); - - for (time_t t = begin; t < end; t += step) - { - time_t t2 = date_lut.makeDateTime(date_lut.toYear(t), date_lut.toMonth(t), date_lut.toDayOfMonth(t), - date_lut.toHour(t), date_lut.toMinute(t), date_lut.toSecond(t)); - - std::string s1 = toString(t); - std::string s2 = toString(t2); - - std::cerr << s1 << ", " << s2 << std::endl; - - if (s1 != s2) - throw Poco::Exception("Test failed."); - } -} - - -int main(int, char **) -{ - loop(orderedIdentifierToDate(20101031), orderedIdentifierToDate(20101101), 15 * 60); - loop(orderedIdentifierToDate(20100328), orderedIdentifierToDate(20100330), 15 * 60); - - return 0; -} diff --git a/base/common/tests/date_lut_default_timezone.cpp b/base/common/tests/date_lut_default_timezone.cpp deleted file mode 100644 index b8e5aa08931..00000000000 --- a/base/common/tests/date_lut_default_timezone.cpp +++ /dev/null @@ -1,31 +0,0 @@ -#include -#include -#include - -int main(int, char **) -{ - try - { - const auto & date_lut = DateLUT::instance(); - std::cout << "Detected default timezone: `" << date_lut.getTimeZone() << "'" << std::endl; - time_t now = time(nullptr); - std::cout << "Current time: " << date_lut.timeToString(now) - << ", UTC: " << DateLUT::instance("UTC").timeToString(now) << std::endl; - } - catch (const Poco::Exception & e) - { - std::cerr << e.displayText() << std::endl; - return 1; - } - catch (std::exception & e) - { - std::cerr << "std::exception: " << e.what() << std::endl; - return 2; - } - catch (...) - { - std::cerr << "Some exception" << std::endl; - return 3; - } - return 0; -} diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index b80bcfdf4d4..a6a7d280479 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -478,6 +478,15 @@ if (ENABLE_TESTS AND USE_GTEST) -Wno-gnu-zero-variadic-macro-arguments ) - target_link_libraries(unit_tests_dbms PRIVATE ${GTEST_BOTH_LIBRARIES} clickhouse_functions clickhouse_aggregate_functions clickhouse_parsers dbms clickhouse_common_zookeeper string_utils) + target_link_libraries(unit_tests_dbms PRIVATE + ${GTEST_BOTH_LIBRARIES} + clickhouse_functions + clickhouse_aggregate_functions + clickhouse_parsers + clickhouse_storages_system + dbms + clickhouse_common_zookeeper + string_utils) + add_check(unit_tests_dbms) endif () diff --git a/base/common/tests/gtest_DateLutImpl.cpp b/src/Common/tests/gtest_DateLutImpl.cpp similarity index 99% rename from base/common/tests/gtest_DateLutImpl.cpp rename to src/Common/tests/gtest_DateLutImpl.cpp index 8cc4fbbbfb0..3a0da1ee1ee 100644 --- a/base/common/tests/gtest_DateLutImpl.cpp +++ b/src/Common/tests/gtest_DateLutImpl.cpp @@ -11,7 +11,7 @@ #pragma clang diagnostic ignored "-Wused-but-marked-unused" #endif -// All timezones present at build time and embedded into CH binary. +// All timezones present at build time and embedded into ClickHouse binary. extern const char * auto_time_zones[]; namespace diff --git a/base/common/tests/gtest_find_symbols.cpp b/src/Common/tests/gtest_find_symbols.cpp similarity index 100% rename from base/common/tests/gtest_find_symbols.cpp rename to src/Common/tests/gtest_find_symbols.cpp diff --git a/src/Common/tests/gtest_global_register_functions.h.bak b/src/Common/tests/gtest_global_register_functions.h.bak new file mode 100644 index 00000000000..197ce5838b9 --- /dev/null +++ b/src/Common/tests/gtest_global_register_functions.h.bak @@ -0,0 +1,17 @@ +#include +#include + +struct RegisteredFunctionsState +{ + RegisteredFunctionsState() + { + DB::registerFunctions(); + } + + RegisteredFunctionsState(RegisteredFunctionsState &&) = default; +}; + +inline void tryRegisterFunctions() +{ + static RegisteredFunctionsState registered_functions_state; +} diff --git a/base/common/tests/gtest_json_test.cpp b/src/Common/tests/gtest_json_test.cpp similarity index 99% rename from base/common/tests/gtest_json_test.cpp rename to src/Common/tests/gtest_json_test.cpp index 189a1a03d99..7b810504952 100644 --- a/base/common/tests/gtest_json_test.cpp +++ b/src/Common/tests/gtest_json_test.cpp @@ -500,14 +500,14 @@ TEST(JSONSuite, SimpleTest) { R"("detail")", ResultType::Return, "detail" }, { R"("actionField")", ResultType::Return, "actionField" }, { R"("list")", ResultType::Return, "list" }, - { "\0\"", ResultType::Throw, "JSON: expected \", got \0" }, + { "\0\"", ResultType::Throw, "JSON: begin >= end." }, { "\"/igrushki/konstruktory\0", ResultType::Throw, "JSON: incorrect syntax (expected end of string, found end of JSON)." }, { "\"/1290414/komplekt-zhenskiy-dzhemper-plusbryuki-m-254-09-malina-plustemno-siniy-\0a", ResultType::Throw, "JSON: incorrect syntax (expected end of string, found end of JSON)." }, { "\"/Творчество/Рисование/Инструменты и кра\0a", ResultType::Throw, "JSON: incorrect syntax (expected end of string, found end of JSON)." }, { "\"Строительство и ремонт/Силовая техника/Зарядные устройства для автомобильных аккумуляторов/Пуско-зарядные устр\xD0\0a", ResultType::Throw, "JSON: incorrect syntax (expected end of string, found end of JSON)." }, { "\"Строительство и ремонт/Силовая техника/Зарядные устройств\xD0\0t", ResultType::Throw, "JSON: incorrect syntax (expected end of string, found end of JSON)." }, { "\"Строительство и ремонт/Силовая техника/Зарядные устройства для автомобиль\0k", ResultType::Throw, "JSON: incorrect syntax (expected end of string, found end of JSON)." }, - { "\0t", ResultType::Throw, "JSON: expected \", got \0" }, + { "\0t", ResultType::Throw, "JSON: begin >= end." }, { "\"/Хозтовары/Хранение вещей и организа\xD1\0t", ResultType::Throw, "JSON: incorrect syntax (expected end of string, found end of JSON)." }, { "\"/Хозтовары/Товары для стир\0a", ResultType::Throw, "JSON: incorrect syntax (expected end of string, found end of JSON)." }, { "\"li\0a", ResultType::Throw, "JSON: incorrect syntax (expected end of string, found end of JSON)." }, @@ -572,10 +572,10 @@ TEST(JSONSuite, SimpleTest) { "\"/Игр\xD1\0 ", ResultType::Throw, "JSON: incorrect syntax (expected end of string, found end of JSON)." }, { "\"/Игрушки/Игрушки для девочек/Игровые модули дл\xD1\0o", ResultType::Throw, "JSON: incorrect syntax (expected end of string, found end of JSON)." }, { "\"Крупная бытовая техника/Стиральные машины/С фронт\xD0\0o", ResultType::Throw, "JSON: incorrect syntax (expected end of string, found end of JSON)." }, - { "\0 ", ResultType::Throw, "JSON: expected \", got \0" }, + { "\0 ", ResultType::Throw, "JSON: begin >= end." }, { "\"Светодиодная лента SMD3528, 5 м. IP33, 60LED, зеленый, 4,8W/мет\xD1\0 ", ResultType::Throw, "JSON: incorrect syntax (expected end of string, found end of JSON)." }, { "\"Сантехника/Мебель для ванных комнат/Стол\0o", ResultType::Throw, "JSON: incorrect syntax (expected end of string, found end of JSON)." }, - { "\0o", ResultType::Throw, "JSON: expected \", got \0" }, + { "\0o", ResultType::Throw, "JSON: begin >= end." }, { "\"/igrushki/konstruktory\0 ", ResultType::Throw, "JSON: incorrect syntax (expected end of string, found end of JSON)." }, { "\"/posuda/kuhonnye-prinadlezhnosti-i-instrumenty/kuhonnye-pr\0 ", ResultType::Throw, "JSON: incorrect syntax (expected end of string, found end of JSON)." }, { "\"/1290414/komplekt-zhenskiy-dzhemper-plusbryuki-m-254-09-malina-plustemno-siniy-\0o", ResultType::Throw, "JSON: incorrect syntax (expected end of string, found end of JSON)." }, @@ -583,7 +583,7 @@ TEST(JSONSuite, SimpleTest) { "\"Строительство и ремонт/Силовая техника/Зарядные устройства для автомобильных аккумуляторов/Пуско-зарядные устр\xD0\0o", ResultType::Throw, "JSON: incorrect syntax (expected end of string, found end of JSON)." }, { "\"Строительство и ремонт/Силовая техника/Зарядные устройств\xD0\0 ", ResultType::Throw, "JSON: incorrect syntax (expected end of string, found end of JSON)." }, { "\"Строительство и ремонт/Силовая техника/Зарядные устройства для автомобиль\0d", ResultType::Throw, "JSON: incorrect syntax (expected end of string, found end of JSON)." }, - { "\0 ", ResultType::Throw, "JSON: expected \", got \0" }, + { "\0 ", ResultType::Throw, "JSON: begin >= end." }, { "\"/Хозтовары/Хранение вещей и организа\xD1\0 ", ResultType::Throw, "JSON: incorrect syntax (expected end of string, found end of JSON)." }, { "\"/Хозтовары/Товары для стир\0o", ResultType::Throw, "JSON: incorrect syntax (expected end of string, found end of JSON)." }, { "\"li\0o", ResultType::Throw, "JSON: incorrect syntax (expected end of string, found end of JSON)." }, diff --git a/base/common/tests/gtest_strong_typedef.cpp b/src/Common/tests/gtest_strong_typedef.cpp similarity index 100% rename from base/common/tests/gtest_strong_typedef.cpp rename to src/Common/tests/gtest_strong_typedef.cpp From 329074bd029a397adbda1fa67751e2a35cb2d48b Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Thu, 4 Mar 2021 01:55:56 +0300 Subject: [PATCH 087/544] Fix test --- src/Common/tests/gtest_json_test.cpp | 1230 +++++++++++++------------- 1 file changed, 615 insertions(+), 615 deletions(-) diff --git a/src/Common/tests/gtest_json_test.cpp b/src/Common/tests/gtest_json_test.cpp index 7b810504952..726fb836030 100644 --- a/src/Common/tests/gtest_json_test.cpp +++ b/src/Common/tests/gtest_json_test.cpp @@ -1,14 +1,13 @@ #include #include #include +#include #include #include - -using namespace std::literals::string_literals; - #include + enum class ResultType { Return, @@ -17,620 +16,622 @@ enum class ResultType struct GetStringTestRecord { - const char * input; + std::string_view input; ResultType result_type; - const char * result; + std::string_view result; }; TEST(JSONSuite, SimpleTest) { + using namespace std::literals; + std::vector test_data = { - { R"("name")", ResultType::Return, "name" }, - { R"("Вафельница Vitek WX-1102 FL")", ResultType::Return, "Вафельница Vitek WX-1102 FL" }, - { R"("brand")", ResultType::Return, "brand" }, - { R"("184509")", ResultType::Return, "184509" }, - { R"("category")", ResultType::Return, "category" }, - { R"("Все для детей/Детская техника/Vitek")", ResultType::Return, "Все для детей/Детская техника/Vitek" }, - { R"("variant")", ResultType::Return, "variant" }, - { R"("В наличии")", ResultType::Return, "В наличии" }, - { R"("price")", ResultType::Return, "price" }, - { R"("2390.00")", ResultType::Return, "2390.00" }, - { R"("list")", ResultType::Return, "list" }, - { R"("Карточка")", ResultType::Return, "Карточка" }, - { R"("position")", ResultType::Return, "position" }, - { R"("detail")", ResultType::Return, "detail" }, - { R"("actionField")", ResultType::Return, "actionField" }, - { R"("list")", ResultType::Return, "list" }, - { R"("http://www.techport.ru/q/?t=вафельница&sort=price&sdim=asc")", ResultType::Return, "http://www.techport.ru/q/?t=вафельница&sort=price&sdim=asc" }, - { R"("action")", ResultType::Return, "action" }, - { R"("detail")", ResultType::Return, "detail" }, - { R"("products")", ResultType::Return, "products" }, - { R"("name")", ResultType::Return, "name" }, - { R"("Вафельница Vitek WX-1102 FL")", ResultType::Return, "Вафельница Vitek WX-1102 FL" }, - { R"("id")", ResultType::Return, "id" }, - { R"("184509")", ResultType::Return, "184509" }, - { R"("price")", ResultType::Return, "price" }, - { R"("2390.00")", ResultType::Return, "2390.00" }, - { R"("brand")", ResultType::Return, "brand" }, - { R"("Vitek")", ResultType::Return, "Vitek" }, - { R"("category")", ResultType::Return, "category" }, - { R"("Все для детей/Детская техника/Vitek")", ResultType::Return, "Все для детей/Детская техника/Vitek" }, - { R"("variant")", ResultType::Return, "variant" }, - { R"("В наличии")", ResultType::Return, "В наличии" }, - { R"("ru")", ResultType::Return, "ru" }, - { R"("experiments")", ResultType::Return, "experiments" }, - { R"("lang")", ResultType::Return, "lang" }, - { R"("ru")", ResultType::Return, "ru" }, - { R"("los_portal")", ResultType::Return, "los_portal" }, - { R"("los_level")", ResultType::Return, "los_level" }, - { R"("none")", ResultType::Return, "none" }, - { R"("isAuthorized")", ResultType::Return, "isAuthorized" }, - { R"("isSubscriber")", ResultType::Return, "isSubscriber" }, - { R"("postType")", ResultType::Return, "postType" }, - { R"("Новости")", ResultType::Return, "Новости" }, - { R"("experiments")", ResultType::Return, "experiments" }, - { R"("lang")", ResultType::Return, "lang" }, - { R"("ru")", ResultType::Return, "ru" }, - { R"("los_portal")", ResultType::Return, "los_portal" }, - { R"("los_level")", ResultType::Return, "los_level" }, - { R"("none")", ResultType::Return, "none" }, - { R"("lang")", ResultType::Return, "lang" }, - { R"("ru")", ResultType::Return, "ru" }, - { R"("Электроплита GEFEST Брест ЭПНД 5140-01 0001")", ResultType::Return, "Электроплита GEFEST Брест ЭПНД 5140-01 0001" }, - { R"("price")", ResultType::Return, "price" }, - { R"("currencyCode")", ResultType::Return, "currencyCode" }, - { R"("RUB")", ResultType::Return, "RUB" }, - { R"("lang")", ResultType::Return, "lang" }, - { R"("ru")", ResultType::Return, "ru" }, - { R"("experiments")", ResultType::Return, "experiments" }, - { R"("lang")", ResultType::Return, "lang" }, - { R"("ru")", ResultType::Return, "ru" }, - { R"("los_portal")", ResultType::Return, "los_portal" }, - { R"("los_level")", ResultType::Return, "los_level" }, - { R"("none")", ResultType::Return, "none" }, - { R"("trash_login")", ResultType::Return, "trash_login" }, - { R"("novikoff")", ResultType::Return, "novikoff" }, - { R"("trash_cat_link")", ResultType::Return, "trash_cat_link" }, - { R"("progs")", ResultType::Return, "progs" }, - { R"("trash_parent_link")", ResultType::Return, "trash_parent_link" }, - { R"("content")", ResultType::Return, "content" }, - { R"("trash_posted_parent")", ResultType::Return, "trash_posted_parent" }, - { R"("content.01.2016")", ResultType::Return, "content.01.2016" }, - { R"("trash_posted_cat")", ResultType::Return, "trash_posted_cat" }, - { R"("progs.01.2016")", ResultType::Return, "progs.01.2016" }, - { R"("trash_virus_count")", ResultType::Return, "trash_virus_count" }, - { R"("trash_is_android")", ResultType::Return, "trash_is_android" }, - { R"("trash_is_wp8")", ResultType::Return, "trash_is_wp8" }, - { R"("trash_is_ios")", ResultType::Return, "trash_is_ios" }, - { R"("trash_posted")", ResultType::Return, "trash_posted" }, - { R"("01.2016")", ResultType::Return, "01.2016" }, - { R"("experiments")", ResultType::Return, "experiments" }, - { R"("lang")", ResultType::Return, "lang" }, - { R"("ru")", ResultType::Return, "ru" }, - { R"("los_portal")", ResultType::Return, "los_portal" }, - { R"("los_level")", ResultType::Return, "los_level" }, - { R"("none")", ResultType::Return, "none" }, - { R"("merchantId")", ResultType::Return, "merchantId" }, - { R"("13694_49246")", ResultType::Return, "13694_49246" }, - { R"("cps-source")", ResultType::Return, "cps-source" }, - { R"("wargaming")", ResultType::Return, "wargaming" }, - { R"("cps_provider")", ResultType::Return, "cps_provider" }, - { R"("default")", ResultType::Return, "default" }, - { R"("errorReason")", ResultType::Return, "errorReason" }, - { R"("no errors")", ResultType::Return, "no errors" }, - { R"("scid")", ResultType::Return, "scid" }, - { R"("isAuthPayment")", ResultType::Return, "isAuthPayment" }, - { R"("lang")", ResultType::Return, "lang" }, - { R"("ru")", ResultType::Return, "ru" }, - { R"("rubric")", ResultType::Return, "rubric" }, - { R"("")", ResultType::Return, "" }, - { R"("rubric")", ResultType::Return, "rubric" }, - { R"("Мир")", ResultType::Return, "Мир" }, - { R"("lang")", ResultType::Return, "lang" }, - { R"("ru")", ResultType::Return, "ru" }, - { R"("experiments")", ResultType::Return, "experiments" }, - { R"("lang")", ResultType::Return, "lang" }, - { R"("ru")", ResultType::Return, "ru" }, - { R"("los_portal")", ResultType::Return, "los_portal" }, - { R"("los_level")", ResultType::Return, "los_level" }, - { R"("none")", ResultType::Return, "none" }, - { R"("lang")", ResultType::Return, "lang" }, - { R"("ru")", ResultType::Return, "ru" }, - { R"("__ym")", ResultType::Return, "__ym" }, - { R"("ecommerce")", ResultType::Return, "ecommerce" }, - { R"("impressions")", ResultType::Return, "impressions" }, - { R"("id")", ResultType::Return, "id" }, - { R"("863813")", ResultType::Return, "863813" }, - { R"("name")", ResultType::Return, "name" }, - { R"("Футболка детская 3D Happy, возраст 1-2 года, трикотаж")", ResultType::Return, "Футболка детская 3D Happy, возраст 1-2 года, трикотаж" }, - { R"("category")", ResultType::Return, "category" }, - { R"("/Летние товары/Летний текстиль/")", ResultType::Return, "/Летние товары/Летний текстиль/" }, - { R"("variant")", ResultType::Return, "variant" }, - { R"("")", ResultType::Return, "" }, - { R"("price")", ResultType::Return, "price" }, - { R"("390.00")", ResultType::Return, "390.00" }, - { R"("list")", ResultType::Return, "list" }, - { R"("/retailrocket/")", ResultType::Return, "/retailrocket/" }, - { R"("position")", ResultType::Return, "position" }, - { R"("brand")", ResultType::Return, "brand" }, - { R"("/911488/futbolka-detskaya-3d-dolphin-vozrast-1-2-goda-trikotazh/")", ResultType::Return, "/911488/futbolka-detskaya-3d-dolphin-vozrast-1-2-goda-trikotazh/" }, - { R"("id")", ResultType::Return, "id" }, - { R"("863839")", ResultType::Return, "863839" }, - { R"("name")", ResultType::Return, "name" }, - { R"("Футболка детская 3D Pretty kitten, возраст 1-2 года, трикотаж")", ResultType::Return, "Футболка детская 3D Pretty kitten, возраст 1-2 года, трикотаж" }, - { R"("category")", ResultType::Return, "category" }, - { R"("/Летние товары/Летний текстиль/")", ResultType::Return, "/Летние товары/Летний текстиль/" }, - { R"("variant")", ResultType::Return, "variant" }, - { R"("")", ResultType::Return, "" }, - { R"("price")", ResultType::Return, "price" }, - { R"("390.00")", ResultType::Return, "390.00" }, - { R"("list")", ResultType::Return, "list" }, - { R"("/retailrocket/")", ResultType::Return, "/retailrocket/" }, - { R"("position")", ResultType::Return, "position" }, - { R"("brand")", ResultType::Return, "brand" }, - { R"("/911488/futbolka-detskaya-3d-dolphin-vozrast-1-2-goda-trikotazh/")", ResultType::Return, "/911488/futbolka-detskaya-3d-dolphin-vozrast-1-2-goda-trikotazh/" }, - { R"("id")", ResultType::Return, "id" }, - { R"("863847")", ResultType::Return, "863847" }, - { R"("name")", ResultType::Return, "name" }, - { R"("Футболка детская 3D Little tiger, возраст 1-2 года, трикотаж")", ResultType::Return, "Футболка детская 3D Little tiger, возраст 1-2 года, трикотаж" }, - { R"("category")", ResultType::Return, "category" }, - { R"("/Летние товары/Летний текстиль/")", ResultType::Return, "/Летние товары/Летний текстиль/" }, - { R"("variant")", ResultType::Return, "variant" }, - { R"("")", ResultType::Return, "" }, - { R"("price")", ResultType::Return, "price" }, - { R"("390.00")", ResultType::Return, "390.00" }, - { R"("list")", ResultType::Return, "list" }, - { R"("/retailrocket/")", ResultType::Return, "/retailrocket/" }, - { R"("position")", ResultType::Return, "position" }, - { R"("brand")", ResultType::Return, "brand" }, - { R"("/911488/futbolka-detskaya-3d-dolphin-vozrast-1-2-goda-trikotazh/")", ResultType::Return, "/911488/futbolka-detskaya-3d-dolphin-vozrast-1-2-goda-trikotazh/" }, - { R"("id")", ResultType::Return, "id" }, - { R"("911480")", ResultType::Return, "911480" }, - { R"("name")", ResultType::Return, "name" }, - { R"("Футболка детская 3D Puppy, возраст 1-2 года, трикотаж")", ResultType::Return, "Футболка детская 3D Puppy, возраст 1-2 года, трикотаж" }, - { R"("category")", ResultType::Return, "category" }, - { R"("/Летние товары/Летний текстиль/")", ResultType::Return, "/Летние товары/Летний текстиль/" }, - { R"("variant")", ResultType::Return, "variant" }, - { R"("")", ResultType::Return, "" }, - { R"("price")", ResultType::Return, "price" }, - { R"("390.00")", ResultType::Return, "390.00" }, - { R"("list")", ResultType::Return, "list" }, - { R"("/retailrocket/")", ResultType::Return, "/retailrocket/" }, - { R"("position")", ResultType::Return, "position" }, - { R"("brand")", ResultType::Return, "brand" }, - { R"("/911488/futbolka-detskaya-3d-dolphin-vozrast-1-2-goda-trikotazh/")", ResultType::Return, "/911488/futbolka-detskaya-3d-dolphin-vozrast-1-2-goda-trikotazh/" }, - { R"("id")", ResultType::Return, "id" }, - { R"("911484")", ResultType::Return, "911484" }, - { R"("name")", ResultType::Return, "name" }, - { R"("Футболка детская 3D Little bears, возраст 1-2 года, трикотаж")", ResultType::Return, "Футболка детская 3D Little bears, возраст 1-2 года, трикотаж" }, - { R"("category")", ResultType::Return, "category" }, - { R"("/Летние товары/Летний текстиль/")", ResultType::Return, "/Летние товары/Летний текстиль/" }, - { R"("variant")", ResultType::Return, "variant" }, - { R"("")", ResultType::Return, "" }, - { R"("price")", ResultType::Return, "price" }, - { R"("390.00")", ResultType::Return, "390.00" }, - { R"("list")", ResultType::Return, "list" }, - { R"("/retailrocket/")", ResultType::Return, "/retailrocket/" }, - { R"("position")", ResultType::Return, "position" }, - { R"("brand")", ResultType::Return, "brand" }, - { R"("/911488/futbolka-detskaya-3d-dolphin-vozrast-1-2-goda-trikotazh/")", ResultType::Return, "/911488/futbolka-detskaya-3d-dolphin-vozrast-1-2-goda-trikotazh/" }, - { R"("id")", ResultType::Return, "id" }, - { R"("911489")", ResultType::Return, "911489" }, - { R"("name")", ResultType::Return, "name" }, - { R"("Футболка детская 3D Dolphin, возраст 2-4 года, трикотаж")", ResultType::Return, "Футболка детская 3D Dolphin, возраст 2-4 года, трикотаж" }, - { R"("category")", ResultType::Return, "category" }, - { R"("/Летние товары/Летний текстиль/")", ResultType::Return, "/Летние товары/Летний текстиль/" }, - { R"("variant")", ResultType::Return, "variant" }, - { R"("")", ResultType::Return, "" }, - { R"("price")", ResultType::Return, "price" }, - { R"("390.00")", ResultType::Return, "390.00" }, - { R"("list")", ResultType::Return, "list" }, - { R"("/retailrocket/")", ResultType::Return, "/retailrocket/" }, - { R"("position")", ResultType::Return, "position" }, - { R"("brand")", ResultType::Return, "brand" }, - { R"("/911488/futbolka-detskaya-3d-dolphin-vozrast-1-2-goda-trikotazh/")", ResultType::Return, "/911488/futbolka-detskaya-3d-dolphin-vozrast-1-2-goda-trikotazh/" }, - { R"("id")", ResultType::Return, "id" }, - { R"("911496")", ResultType::Return, "911496" }, - { R"("name")", ResultType::Return, "name" }, - { R"("Футболка детская 3D Pretty, возраст 1-2 года, трикотаж")", ResultType::Return, "Футболка детская 3D Pretty, возраст 1-2 года, трикотаж" }, - { R"("category")", ResultType::Return, "category" }, - { R"("/Летние товары/Летний текстиль/")", ResultType::Return, "/Летние товары/Летний текстиль/" }, - { R"("variant")", ResultType::Return, "variant" }, - { R"("")", ResultType::Return, "" }, - { R"("price")", ResultType::Return, "price" }, - { R"("390.00")", ResultType::Return, "390.00" }, - { R"("list")", ResultType::Return, "list" }, - { R"("/retailrocket/")", ResultType::Return, "/retailrocket/" }, - { R"("position")", ResultType::Return, "position" }, - { R"("brand")", ResultType::Return, "brand" }, - { R"("/911488/futbolka-detskaya-3d-dolphin-vozrast-1-2-goda-trikotazh/")", ResultType::Return, "/911488/futbolka-detskaya-3d-dolphin-vozrast-1-2-goda-trikotazh/" }, - { R"("id")", ResultType::Return, "id" }, - { R"("911504")", ResultType::Return, "911504" }, - { R"("name")", ResultType::Return, "name" }, - { R"("Футболка детская 3D Fairytale, возраст 1-2 года, трикотаж")", ResultType::Return, "Футболка детская 3D Fairytale, возраст 1-2 года, трикотаж" }, - { R"("category")", ResultType::Return, "category" }, - { R"("/Летние товары/Летний текстиль/")", ResultType::Return, "/Летние товары/Летний текстиль/" }, - { R"("variant")", ResultType::Return, "variant" }, - { R"("")", ResultType::Return, "" }, - { R"("price")", ResultType::Return, "price" }, - { R"("390.00")", ResultType::Return, "390.00" }, - { R"("list")", ResultType::Return, "list" }, - { R"("/retailrocket/")", ResultType::Return, "/retailrocket/" }, - { R"("position")", ResultType::Return, "position" }, - { R"("brand")", ResultType::Return, "brand" }, - { R"("/911488/futbolka-detskaya-3d-dolphin-vozrast-1-2-goda-trikotazh/")", ResultType::Return, "/911488/futbolka-detskaya-3d-dolphin-vozrast-1-2-goda-trikotazh/" }, - { R"("id")", ResultType::Return, "id" }, - { R"("911508")", ResultType::Return, "911508" }, - { R"("name")", ResultType::Return, "name" }, - { R"("Футболка детская 3D Kittens, возраст 1-2 года, трикотаж")", ResultType::Return, "Футболка детская 3D Kittens, возраст 1-2 года, трикотаж" }, - { R"("category")", ResultType::Return, "category" }, - { R"("/Летние товары/Летний текстиль/")", ResultType::Return, "/Летние товары/Летний текстиль/" }, - { R"("variant")", ResultType::Return, "variant" }, - { R"("")", ResultType::Return, "" }, - { R"("price")", ResultType::Return, "price" }, - { R"("390.00")", ResultType::Return, "390.00" }, - { R"("list")", ResultType::Return, "list" }, - { R"("/retailrocket/")", ResultType::Return, "/retailrocket/" }, - { R"("position")", ResultType::Return, "position" }, - { R"("brand")", ResultType::Return, "brand" }, - { R"("/911488/futbolka-detskaya-3d-dolphin-vozrast-1-2-goda-trikotazh/")", ResultType::Return, "/911488/futbolka-detskaya-3d-dolphin-vozrast-1-2-goda-trikotazh/" }, - { R"("id")", ResultType::Return, "id" }, - { R"("911512")", ResultType::Return, "911512" }, - { R"("name")", ResultType::Return, "name" }, - { R"("Футболка детская 3D Sunshine, возраст 1-2 года, трикотаж")", ResultType::Return, "Футболка детская 3D Sunshine, возраст 1-2 года, трикотаж" }, - { R"("category")", ResultType::Return, "category" }, - { R"("/Летние товары/Летний текстиль/")", ResultType::Return, "/Летние товары/Летний текстиль/" }, - { R"("variant")", ResultType::Return, "variant" }, - { R"("")", ResultType::Return, "" }, - { R"("price")", ResultType::Return, "price" }, - { R"("390.00")", ResultType::Return, "390.00" }, - { R"("list")", ResultType::Return, "list" }, - { R"("/retailrocket/")", ResultType::Return, "/retailrocket/" }, - { R"("position")", ResultType::Return, "position" }, - { R"("brand")", ResultType::Return, "brand" }, - { R"("/911488/futbolka-detskaya-3d-dolphin-vozrast-1-2-goda-trikotazh/")", ResultType::Return, "/911488/futbolka-detskaya-3d-dolphin-vozrast-1-2-goda-trikotazh/" }, - { R"("id")", ResultType::Return, "id" }, - { R"("911516")", ResultType::Return, "911516" }, - { R"("name")", ResultType::Return, "name" }, - { R"("Футболка детская 3D Dog in bag, возраст 1-2 года, трикотаж")", ResultType::Return, "Футболка детская 3D Dog in bag, возраст 1-2 года, трикотаж" }, - { R"("category")", ResultType::Return, "category" }, - { R"("/Летние товары/Летний текстиль/")", ResultType::Return, "/Летние товары/Летний текстиль/" }, - { R"("variant")", ResultType::Return, "variant" }, - { R"("")", ResultType::Return, "" }, - { R"("price")", ResultType::Return, "price" }, - { R"("390.00")", ResultType::Return, "390.00" }, - { R"("list")", ResultType::Return, "list" }, - { R"("/retailrocket/")", ResultType::Return, "/retailrocket/" }, - { R"("position")", ResultType::Return, "position" }, - { R"("brand")", ResultType::Return, "brand" }, - { R"("/911488/futbolka-detskaya-3d-dolphin-vozrast-1-2-goda-trikotazh/")", ResultType::Return, "/911488/futbolka-detskaya-3d-dolphin-vozrast-1-2-goda-trikotazh/" }, - { R"("id")", ResultType::Return, "id" }, - { R"("911520")", ResultType::Return, "911520" }, - { R"("name")", ResultType::Return, "name" }, - { R"("Футболка детская 3D Cute puppy, возраст 1-2 года, трикотаж")", ResultType::Return, "Футболка детская 3D Cute puppy, возраст 1-2 года, трикотаж" }, - { R"("category")", ResultType::Return, "category" }, - { R"("/Летние товары/Летний текстиль/")", ResultType::Return, "/Летние товары/Летний текстиль/" }, - { R"("variant")", ResultType::Return, "variant" }, - { R"("")", ResultType::Return, "" }, - { R"("price")", ResultType::Return, "price" }, - { R"("390.00")", ResultType::Return, "390.00" }, - { R"("list")", ResultType::Return, "list" }, - { R"("/retailrocket/")", ResultType::Return, "/retailrocket/" }, - { R"("position")", ResultType::Return, "position" }, - { R"("brand")", ResultType::Return, "brand" }, - { R"("/911488/futbolka-detskaya-3d-dolphin-vozrast-1-2-goda-trikotazh/")", ResultType::Return, "/911488/futbolka-detskaya-3d-dolphin-vozrast-1-2-goda-trikotazh/" }, - { R"("id")", ResultType::Return, "id" }, - { R"("911524")", ResultType::Return, "911524" }, - { R"("name")", ResultType::Return, "name" }, - { R"("Футболка детская 3D Rabbit, возраст 1-2 года, трикотаж")", ResultType::Return, "Футболка детская 3D Rabbit, возраст 1-2 года, трикотаж" }, - { R"("category")", ResultType::Return, "category" }, - { R"("/Летние товары/Летний текстиль/")", ResultType::Return, "/Летние товары/Летний текстиль/" }, - { R"("variant")", ResultType::Return, "variant" }, - { R"("")", ResultType::Return, "" }, - { R"("price")", ResultType::Return, "price" }, - { R"("390.00")", ResultType::Return, "390.00" }, - { R"("list")", ResultType::Return, "list" }, - { R"("/retailrocket/")", ResultType::Return, "/retailrocket/" }, - { R"("position")", ResultType::Return, "position" }, - { R"("brand")", ResultType::Return, "brand" }, - { R"("/911488/futbolka-detskaya-3d-dolphin-vozrast-1-2-goda-trikotazh/")", ResultType::Return, "/911488/futbolka-detskaya-3d-dolphin-vozrast-1-2-goda-trikotazh/" }, - { R"("id")", ResultType::Return, "id" }, - { R"("911528")", ResultType::Return, "911528" }, - { R"("name")", ResultType::Return, "name" }, - { R"("Футболка детская 3D Turtle, возраст 1-2 года, трикотаж")", ResultType::Return, "Футболка детская 3D Turtle, возраст 1-2 года, трикотаж" }, - { R"("category")", ResultType::Return, "category" }, - { R"("/Летние товары/Летний текстиль/")", ResultType::Return, "/Летние товары/Летний текстиль/" }, - { R"("variant")", ResultType::Return, "variant" }, - { R"("")", ResultType::Return, "" }, - { R"("price")", ResultType::Return, "price" }, - { R"("390.00")", ResultType::Return, "390.00" }, - { R"("list")", ResultType::Return, "list" }, - { R"("/retailrocket/")", ResultType::Return, "/retailrocket/" }, - { R"("position")", ResultType::Return, "position" }, - { R"("brand")", ResultType::Return, "brand" }, - { R"("/911488/futbolka-detskaya-3d-dolphin-vozrast-1-2-goda-trikotazh/")", ResultType::Return, "/911488/futbolka-detskaya-3d-dolphin-vozrast-1-2-goda-trikotazh/" }, - { R"("id")", ResultType::Return, "id" }, - { R"("888616")", ResultType::Return, "888616" }, - { R"("name")", ResultType::Return, "name" }, - { "\"3Д Футболка мужская \\\"Collorista\\\" Светлое завтра р-р XL(52-54), 100% хлопок, трикотаж\"", ResultType::Return, "3Д Футболка мужская \"Collorista\" Светлое завтра р-р XL(52-54), 100% хлопок, трикотаж" }, - { R"("category")", ResultType::Return, "category" }, - { R"("/Одежда и обувь/Мужская одежда/Футболки/")", ResultType::Return, "/Одежда и обувь/Мужская одежда/Футболки/" }, - { R"("variant")", ResultType::Return, "variant" }, - { R"("")", ResultType::Return, "" }, - { R"("price")", ResultType::Return, "price" }, - { R"("406.60")", ResultType::Return, "406.60" }, - { R"("list")", ResultType::Return, "list" }, - { R"("/retailrocket/")", ResultType::Return, "/retailrocket/" }, - { R"("position")", ResultType::Return, "position" }, - { R"("brand")", ResultType::Return, "brand" }, - { R"("/911488/futbolka-detskaya-3d-dolphin-vozrast-1-2-goda-trikotazh/")", ResultType::Return, "/911488/futbolka-detskaya-3d-dolphin-vozrast-1-2-goda-trikotazh/" }, - { R"("id")", ResultType::Return, "id" }, - { R"("913361")", ResultType::Return, "913361" }, - { R"("name")", ResultType::Return, "name" }, - { R"("3Д Футболка детская World р-р 8-10, 100% хлопок, трикотаж")", ResultType::Return, "3Д Футболка детская World р-р 8-10, 100% хлопок, трикотаж" }, - { R"("category")", ResultType::Return, "category" }, - { R"("/Летние товары/Летний текстиль/")", ResultType::Return, "/Летние товары/Летний текстиль/" }, - { R"("variant")", ResultType::Return, "variant" }, - { R"("")", ResultType::Return, "" }, - { R"("price")", ResultType::Return, "price" }, - { R"("470.00")", ResultType::Return, "470.00" }, - { R"("list")", ResultType::Return, "list" }, - { R"("/retailrocket/")", ResultType::Return, "/retailrocket/" }, - { R"("position")", ResultType::Return, "position" }, - { R"("brand")", ResultType::Return, "brand" }, - { R"("/911488/futbolka-detskaya-3d-dolphin-vozrast-1-2-goda-trikotazh/")", ResultType::Return, "/911488/futbolka-detskaya-3d-dolphin-vozrast-1-2-goda-trikotazh/" }, - { R"("id")", ResultType::Return, "id" }, - { R"("913364")", ResultType::Return, "913364" }, - { R"("name")", ResultType::Return, "name" }, - { R"("3Д Футболка детская Force р-р 8-10, 100% хлопок, трикотаж")", ResultType::Return, "3Д Футболка детская Force р-р 8-10, 100% хлопок, трикотаж" }, - { R"("category")", ResultType::Return, "category" }, - { R"("/Летние товары/Летний текстиль/")", ResultType::Return, "/Летние товары/Летний текстиль/" }, - { R"("variant")", ResultType::Return, "variant" }, - { R"("")", ResultType::Return, "" }, - { R"("price")", ResultType::Return, "price" }, - { R"("470.00")", ResultType::Return, "470.00" }, - { R"("list")", ResultType::Return, "list" }, - { R"("/retailrocket/")", ResultType::Return, "/retailrocket/" }, - { R"("position")", ResultType::Return, "position" }, - { R"("brand")", ResultType::Return, "brand" }, - { R"("/911488/futbolka-detskaya-3d-dolphin-vozrast-1-2-goda-trikotazh/")", ResultType::Return, "/911488/futbolka-detskaya-3d-dolphin-vozrast-1-2-goda-trikotazh/" }, - { R"("id")", ResultType::Return, "id" }, - { R"("913367")", ResultType::Return, "913367" }, - { R"("name")", ResultType::Return, "name" }, - { R"("3Д Футболка детская Winter tale р-р 8-10, 100% хлопок, трикотаж")", ResultType::Return, "3Д Футболка детская Winter tale р-р 8-10, 100% хлопок, трикотаж" }, - { R"("category")", ResultType::Return, "category" }, - { R"("/Летние товары/Летний текстиль/")", ResultType::Return, "/Летние товары/Летний текстиль/" }, - { R"("variant")", ResultType::Return, "variant" }, - { R"("")", ResultType::Return, "" }, - { R"("price")", ResultType::Return, "price" }, - { R"("470.00")", ResultType::Return, "470.00" }, - { R"("list")", ResultType::Return, "list" }, - { R"("/retailrocket/")", ResultType::Return, "/retailrocket/" }, - { R"("position")", ResultType::Return, "position" }, - { R"("brand")", ResultType::Return, "brand" }, - { R"("/911488/futbolka-detskaya-3d-dolphin-vozrast-1-2-goda-trikotazh/")", ResultType::Return, "/911488/futbolka-detskaya-3d-dolphin-vozrast-1-2-goda-trikotazh/" }, - { R"("id")", ResultType::Return, "id" }, - { R"("913385")", ResultType::Return, "913385" }, - { R"("name")", ResultType::Return, "name" }, - { R"("3Д Футболка детская Moonshine р-р 8-10, 100% хлопок, трикотаж")", ResultType::Return, "3Д Футболка детская Moonshine р-р 8-10, 100% хлопок, трикотаж" }, - { R"("category")", ResultType::Return, "category" }, - { R"("/Летние товары/Летний текстиль/")", ResultType::Return, "/Летние товары/Летний текстиль/" }, - { R"("variant")", ResultType::Return, "variant" }, - { R"("")", ResultType::Return, "" }, - { R"("price")", ResultType::Return, "price" }, - { R"("470.00")", ResultType::Return, "470.00" }, - { R"("list")", ResultType::Return, "list" }, - { R"("/retailrocket/")", ResultType::Return, "/retailrocket/" }, - { R"("position")", ResultType::Return, "position" }, - { R"("brand")", ResultType::Return, "brand" }, - { R"("/911488/futbolka-detskaya-3d-dolphin-vozrast-1-2-goda-trikotazh/")", ResultType::Return, "/911488/futbolka-detskaya-3d-dolphin-vozrast-1-2-goda-trikotazh/" }, - { R"("id")", ResultType::Return, "id" }, - { R"("913391")", ResultType::Return, "913391" }, - { R"("name")", ResultType::Return, "name" }, - { R"("3Д Футболка детская Shaman р-р 8-10, 100% хлопок, трикотаж")", ResultType::Return, "3Д Футболка детская Shaman р-р 8-10, 100% хлопок, трикотаж" }, - { R"("category")", ResultType::Return, "category" }, - { R"("/Летние товары/Летний текстиль/")", ResultType::Return, "/Летние товары/Летний текстиль/" }, - { R"("variant")", ResultType::Return, "variant" }, - { R"("")", ResultType::Return, "" }, - { R"("price")", ResultType::Return, "price" }, - { R"("470.00")", ResultType::Return, "470.00" }, - { R"("list")", ResultType::Return, "list" }, - { R"("/retailrocket/")", ResultType::Return, "/retailrocket/" }, - { R"("position")", ResultType::Return, "position" }, - { R"("brand")", ResultType::Return, "brand" }, - { R"("/911488/futbolka-detskaya-3d-dolphin-vozrast-1-2-goda-trikotazh/")", ResultType::Return, "/911488/futbolka-detskaya-3d-dolphin-vozrast-1-2-goda-trikotazh/" }, - { R"("usertype")", ResultType::Return, "usertype" }, - { R"("visitor")", ResultType::Return, "visitor" }, - { R"("lang")", ResultType::Return, "lang" }, - { R"("ru")", ResultType::Return, "ru" }, - { R"("__ym")", ResultType::Return, "__ym" }, - { R"("ecommerce")", ResultType::Return, "ecommerce" }, - { R"("impressions")", ResultType::Return, "impressions" }, - { R"("experiments")", ResultType::Return, "experiments" }, - { R"("lang")", ResultType::Return, "lang" }, - { R"("ru")", ResultType::Return, "ru" }, - { R"("los_portal")", ResultType::Return, "los_portal" }, - { R"("los_level")", ResultType::Return, "los_level" }, - { R"("none")", ResultType::Return, "none" }, - { R"("experiments")", ResultType::Return, "experiments" }, - { R"("lang")", ResultType::Return, "lang" }, - { R"("ru")", ResultType::Return, "ru" }, - { R"("los_portal")", ResultType::Return, "los_portal" }, - { R"("los_level")", ResultType::Return, "los_level" }, - { R"("none")", ResultType::Return, "none" }, - { R"("experiments")", ResultType::Return, "experiments" }, - { R"("lang")", ResultType::Return, "lang" }, - { R"("ru")", ResultType::Return, "ru" }, - { R"("los_portal")", ResultType::Return, "los_portal" }, - { R"("los_level")", ResultType::Return, "los_level" }, - { R"("none")", ResultType::Return, "none" }, - { R"("experiments")", ResultType::Return, "experiments" }, - { R"("lang")", ResultType::Return, "lang" }, - { R"("ru")", ResultType::Return, "ru" }, - { R"("los_portal")", ResultType::Return, "los_portal" }, - { R"("los_level")", ResultType::Return, "los_level" }, - { R"("none")", ResultType::Return, "none" }, - { R"("experiments")", ResultType::Return, "experiments" }, - { R"("lang")", ResultType::Return, "lang" }, - { R"("ru")", ResultType::Return, "ru" }, - { R"("los_portal")", ResultType::Return, "los_portal" }, - { R"("los_level")", ResultType::Return, "los_level" }, - { R"("none")", ResultType::Return, "none" }, - { R"("__ym")", ResultType::Return, "__ym" }, - { R"("ecommerce")", ResultType::Return, "ecommerce" }, - { R"("currencyCode")", ResultType::Return, "currencyCode" }, - { R"("RUR")", ResultType::Return, "RUR" }, - { R"("impressions")", ResultType::Return, "impressions" }, - { R"("name")", ResultType::Return, "name" }, - { R"("Чайник электрический Mystery MEK-1627, белый")", ResultType::Return, "Чайник электрический Mystery MEK-1627, белый" }, - { R"("brand")", ResultType::Return, "brand" }, - { R"("Mystery")", ResultType::Return, "Mystery" }, - { R"("id")", ResultType::Return, "id" }, - { R"("187180")", ResultType::Return, "187180" }, - { R"("category")", ResultType::Return, "category" }, - { R"("Мелкая бытовая техника/Мелкие кухонные приборы/Чайники электрические/Mystery")", ResultType::Return, "Мелкая бытовая техника/Мелкие кухонные приборы/Чайники электрические/Mystery" }, - { R"("variant")", ResultType::Return, "variant" }, - { R"("В наличии")", ResultType::Return, "В наличии" }, - { R"("price")", ResultType::Return, "price" }, - { R"("1630.00")", ResultType::Return, "1630.00" }, - { R"("list")", ResultType::Return, "list" }, - { R"("Карточка")", ResultType::Return, "Карточка" }, - { R"("position")", ResultType::Return, "position" }, - { R"("detail")", ResultType::Return, "detail" }, - { R"("actionField")", ResultType::Return, "actionField" }, - { R"("list")", ResultType::Return, "list" }, - { "\0\"", ResultType::Throw, "JSON: begin >= end." }, - { "\"/igrushki/konstruktory\0", ResultType::Throw, "JSON: incorrect syntax (expected end of string, found end of JSON)." }, - { "\"/1290414/komplekt-zhenskiy-dzhemper-plusbryuki-m-254-09-malina-plustemno-siniy-\0a", ResultType::Throw, "JSON: incorrect syntax (expected end of string, found end of JSON)." }, - { "\"/Творчество/Рисование/Инструменты и кра\0a", ResultType::Throw, "JSON: incorrect syntax (expected end of string, found end of JSON)." }, - { "\"Строительство и ремонт/Силовая техника/Зарядные устройства для автомобильных аккумуляторов/Пуско-зарядные устр\xD0\0a", ResultType::Throw, "JSON: incorrect syntax (expected end of string, found end of JSON)." }, - { "\"Строительство и ремонт/Силовая техника/Зарядные устройств\xD0\0t", ResultType::Throw, "JSON: incorrect syntax (expected end of string, found end of JSON)." }, - { "\"Строительство и ремонт/Силовая техника/Зарядные устройства для автомобиль\0k", ResultType::Throw, "JSON: incorrect syntax (expected end of string, found end of JSON)." }, - { "\0t", ResultType::Throw, "JSON: begin >= end." }, - { "\"/Хозтовары/Хранение вещей и организа\xD1\0t", ResultType::Throw, "JSON: incorrect syntax (expected end of string, found end of JSON)." }, - { "\"/Хозтовары/Товары для стир\0a", ResultType::Throw, "JSON: incorrect syntax (expected end of string, found end of JSON)." }, - { "\"li\0a", ResultType::Throw, "JSON: incorrect syntax (expected end of string, found end of JSON)." }, - { "\"/734859/samolet-radioupravlyaemyy-istrebitel-rabotaet-o\0k", ResultType::Throw, "JSON: incorrect syntax (expected end of string, found end of JSON)." }, - { "\"/kosmetika-i-parfyum/parfyumeriya/mu\0t", ResultType::Throw, "JSON: incorrect syntax (expected end of string, found end of JSON)." }, - { "\"/ko\0\x04", ResultType::Throw, "JSON: incorrect syntax (expected end of string, found end of JSON)." }, - { "", ResultType::Throw, "JSON: begin >= end." }, - { "\"/stroitelstvo-i-remont/stroit\0t", ResultType::Throw, "JSON: incorrect syntax (expected end of string, found end of JSON)." }, - { "\"/stroitelstvo-i-remont/stroitelnyy-instrument/av\0k", ResultType::Throw, "JSON: incorrect syntax (expected end of string, found end of JSON)." }, - { "\"/s\0a", ResultType::Throw, "JSON: incorrect syntax (expected end of string, found end of JSON)." }, - { "\"/Строительство и ремонт/Строительный инструмент/Изм\0e", ResultType::Throw, "JSON: incorrect syntax (expected end of string, found end of JSON)." }, - { "\"/avto/soputstvuy\0l", ResultType::Throw, "JSON: incorrect syntax (expected end of string, found end of JSON)." }, - { "\"/str\0\xD0", ResultType::Throw, "JSON: incorrect syntax (expected end of string, found end of JSON)." }, - { "\"Отвертка 2 в 1 \\\"TUNDRA basic\\\" 5х75 мм (+,-) \0\xFF", ResultType::Throw, "JSON: incorrect syntax (expected end of string, found end of JSON)." }, - { "\"/stroitelstvo-i-remont/stroitelnyy-instrument/avtoinstrumen\0\0", ResultType::Throw, "JSON: incorrect syntax (expected end of string, found end of JSON)." }, - { "\"Мелкая бытовая техника/Мелки\xD0\0\0", ResultType::Throw, "JSON: incorrect syntax (expected end of string, found end of JSON)." }, - { "\"Пряжа \\\"Бамбук стрейч\\0\0\0", ResultType::Throw, "JSON: incorrect syntax (expected end of string, found end of JSON)." }, - { "\"Карандаш чёрнографитны\xD0\0\xD0", ResultType::Throw, "JSON: incorrect syntax (expected end of string, found end of JSON)." }, - { "\"/Творчество/Рукоделие, аппликации/Пряжа и шерсть для \xD0\0l", ResultType::Throw, "JSON: incorrect syntax (expected end of string, found end of JSON)." }, - { "\"/1071547/karandash-chernografitnyy-volshebstvo-nv-kruglyy-d-7-2mm-dl-176mm-plast-tuba/\0e", ResultType::Throw, "JSON: incorrect syntax (expected end of string, found end of JSON)." }, - { "\"ca\0e", ResultType::Throw, "JSON: incorrect syntax (expected end of string, found end of JSON)." }, - { "\"ca\0e", ResultType::Throw, "JSON: incorrect syntax (expected end of string, found end of JSON)." }, - { "\"/1165424/chipbord-vyrubnoy-dlya-skrapbukinga-malyshi-mikki-maus-disney-bebi\0t", ResultType::Throw, "JSON: incorrect syntax (expected end of string, found end of JSON)." }, - { "\"/posuda/kuhonnye-prinadlezhnosti-i-i\0d", ResultType::Throw, "JSON: incorrect syntax (expected end of string, found end of JSON)." }, - { "\"/Канцтовары/Ежедневники и блокн\xD0\0\0", ResultType::Throw, "JSON: incorrect syntax (expected end of string, found end of JSON)." }, - { "\"/kanctovary/ezhednevniki-i-blok\0a", ResultType::Throw, "JSON: incorrect syntax (expected end of string, found end of JSON)." }, - { "\"Стакан \xD0\0a", ResultType::Throw, "JSON: incorrect syntax (expected end of string, found end of JSON)." }, - { "\"Набор бумаги для скрапбукинга \\\"Мои первый годик\\\": Микки Маус, Дисней бэби, 12 листов 29.5 х 29.5 см, 160\0\x80", ResultType::Throw, "JSON: incorrect syntax (expected end of string, found end of JSON)." }, - { "\"c\0\0", ResultType::Throw, "JSON: incorrect syntax (expected end of string, found end of JSON)." }, - { "\"Органайзер для хранения аксессуаров, \0\0", ResultType::Throw, "JSON: incorrect syntax (expected end of string, found end of JSON)." }, - { "\"quantity\00", ResultType::Throw, "JSON: incorrect syntax (expected end of string, found end of JSON)." }, - { "\"Сменный блок для тетрадей на кольцах А5, 160 листов клетка, офсет \xE2\x84\0=", ResultType::Throw, "JSON: incorrect syntax (expected end of string, found end of JSON)." }, - { "\"/Сувениры/Ф\xD0\0 ", ResultType::Throw, "JSON: incorrect syntax (expected end of string, found end of JSON)." }, - { "\"\0\"", ResultType::Return, "\0" }, - { "\"\0\x04", ResultType::Throw, "JSON: incorrect syntax (expected end of string, found end of JSON)." }, - { "\"va\0\0", ResultType::Throw, "JSON: incorrect syntax (expected end of string, found end of JSON)." }, - { "\"ca\0\0", ResultType::Throw, "JSON: incorrect syntax (expected end of string, found end of JSON)." }, - { "\"В \0\x04", ResultType::Throw, "JSON: incorrect syntax (expected end of string, found end of JSON)." }, - { "\"/letnie-tovary/z\0\x04", ResultType::Throw, "JSON: incorrect syntax (expected end of string, found end of JSON)." }, - { "\"Посудомоечная машина Ha\0=", ResultType::Throw, "JSON: incorrect syntax (expected end of string, found end of JSON)." }, - { "\"Крупная бытов\0\0", ResultType::Throw, "JSON: incorrect syntax (expected end of string, found end of JSON)." }, - { "\"Полочная акустическая система Magnat Needl\0\0", ResultType::Throw, "JSON: incorrect syntax (expected end of string, found end of JSON)." }, - { "\"brand\00", ResultType::Throw, "JSON: incorrect syntax (expected end of string, found end of JSON)." }, - { "\"\0d", ResultType::Throw, "JSON: incorrect syntax (expected end of string, found end of JSON)." }, - { "\"pos\0 ", ResultType::Throw, "JSON: incorrect syntax (expected end of string, found end of JSON)." }, - { "\"c\0o", ResultType::Throw, "JSON: incorrect syntax (expected end of string, found end of JSON)." }, - { "\"var\0\0", ResultType::Throw, "JSON: incorrect syntax (expected end of string, found end of JSON)." }, - { "\"Телевизоры и видеотехника/Всё для домашних кинотеатр\0=", ResultType::Throw, "JSON: incorrect syntax (expected end of string, found end of JSON)." }, - { "\"Флеш-диск Transcend JetFlash 620 8GB (TS8GJF62\0\0", ResultType::Throw, "JSON: incorrect syntax (expected end of string, found end of JSON)." }, - { "\"Табурет Мег\0\xD0", ResultType::Throw, "JSON: incorrect syntax (expected end of string, found end of JSON)." }, - { "\"variant\0\x04", ResultType::Throw, "JSON: incorrect syntax (expected end of string, found end of JSON)." }, - { "\"Катал\xD0\0\"", ResultType::Return, "Катал\xD0\0" }, - { "\"К\0\0", ResultType::Throw, "JSON: incorrect syntax (expected end of string, found end of JSON)." }, - { "\"Полочная акустическая система Magnat Needl\0\0", ResultType::Throw, "JSON: incorrect syntax (expected end of string, found end of JSON)." }, - { "\"brand\00", ResultType::Throw, "JSON: incorrect syntax (expected end of string, found end of JSON)." }, - { "\"\0d", ResultType::Throw, "JSON: incorrect syntax (expected end of string, found end of JSON)." }, - { "\"pos\0 ", ResultType::Throw, "JSON: incorrect syntax (expected end of string, found end of JSON)." }, - { "\"c\0o", ResultType::Throw, "JSON: incorrect syntax (expected end of string, found end of JSON)." }, - { "\"17\0o", ResultType::Throw, "JSON: incorrect syntax (expected end of string, found end of JSON)." }, - { "\"/igrushki/razvivayusc\0 ", ResultType::Throw, "JSON: incorrect syntax (expected end of string, found end of JSON)." }, - { "\"Ключница \\\"\0 ", ResultType::Throw, "JSON: incorrect syntax (expected end of string, found end of JSON)." }, - { "\"/Игр\xD1\0 ", ResultType::Throw, "JSON: incorrect syntax (expected end of string, found end of JSON)." }, - { "\"/Игрушки/Игрушки для девочек/Игровые модули дл\xD1\0o", ResultType::Throw, "JSON: incorrect syntax (expected end of string, found end of JSON)." }, - { "\"Крупная бытовая техника/Стиральные машины/С фронт\xD0\0o", ResultType::Throw, "JSON: incorrect syntax (expected end of string, found end of JSON)." }, - { "\0 ", ResultType::Throw, "JSON: begin >= end." }, - { "\"Светодиодная лента SMD3528, 5 м. IP33, 60LED, зеленый, 4,8W/мет\xD1\0 ", ResultType::Throw, "JSON: incorrect syntax (expected end of string, found end of JSON)." }, - { "\"Сантехника/Мебель для ванных комнат/Стол\0o", ResultType::Throw, "JSON: incorrect syntax (expected end of string, found end of JSON)." }, - { "\0o", ResultType::Throw, "JSON: begin >= end." }, - { "\"/igrushki/konstruktory\0 ", ResultType::Throw, "JSON: incorrect syntax (expected end of string, found end of JSON)." }, - { "\"/posuda/kuhonnye-prinadlezhnosti-i-instrumenty/kuhonnye-pr\0 ", ResultType::Throw, "JSON: incorrect syntax (expected end of string, found end of JSON)." }, - { "\"/1290414/komplekt-zhenskiy-dzhemper-plusbryuki-m-254-09-malina-plustemno-siniy-\0o", ResultType::Throw, "JSON: incorrect syntax (expected end of string, found end of JSON)." }, - { "\"/Творчество/Рисование/Инструменты и кра\0o", ResultType::Throw, "JSON: incorrect syntax (expected end of string, found end of JSON)." }, - { "\"Строительство и ремонт/Силовая техника/Зарядные устройства для автомобильных аккумуляторов/Пуско-зарядные устр\xD0\0o", ResultType::Throw, "JSON: incorrect syntax (expected end of string, found end of JSON)." }, - { "\"Строительство и ремонт/Силовая техника/Зарядные устройств\xD0\0 ", ResultType::Throw, "JSON: incorrect syntax (expected end of string, found end of JSON)." }, - { "\"Строительство и ремонт/Силовая техника/Зарядные устройства для автомобиль\0d", ResultType::Throw, "JSON: incorrect syntax (expected end of string, found end of JSON)." }, - { "\0 ", ResultType::Throw, "JSON: begin >= end." }, - { "\"/Хозтовары/Хранение вещей и организа\xD1\0 ", ResultType::Throw, "JSON: incorrect syntax (expected end of string, found end of JSON)." }, - { "\"/Хозтовары/Товары для стир\0o", ResultType::Throw, "JSON: incorrect syntax (expected end of string, found end of JSON)." }, - { "\"li\0o", ResultType::Throw, "JSON: incorrect syntax (expected end of string, found end of JSON)." }, - { "\"/igrushki/igrus\0d", ResultType::Throw, "JSON: incorrect syntax (expected end of string, found end of JSON)." }, - { "\"/734859/samolet-radioupravlyaemyy-istrebitel-rabotaet-o\0 ", ResultType::Throw, "JSON: incorrect syntax (expected end of string, found end of JSON)." }, - { "\"/kosmetika-i-parfyum/parfyumeriya/mu\00", ResultType::Throw, "JSON: incorrect syntax (expected end of string, found end of JSON)." }, - { "\"/ko\0\0", ResultType::Throw, "JSON: incorrect syntax (expected end of string, found end of JSON)." }, - { "\"/avto/avtomobilnyy\0\0", ResultType::Throw, "JSON: incorrect syntax (expected end of string, found end of JSON)." }, - { "\"/stroitelstvo-i-remont/stroit\00", ResultType::Throw, "JSON: incorrect syntax (expected end of string, found end of JSON)." }, - { "\"/stroitelstvo-i-remont/stroitelnyy-instrument/av\0 ", ResultType::Throw, "JSON: incorrect syntax (expected end of string, found end of JSON)." }, - { "\"/s\0d", ResultType::Throw, "JSON: incorrect syntax (expected end of string, found end of JSON)." }, - { "\"/Строительство и ремонт/Строительный инструмент/Изм\0o", ResultType::Throw, "JSON: incorrect syntax (expected end of string, found end of JSON)." }, - { "\"/avto/soputstvuy\0\"", ResultType::Return, "/avto/soputstvuy\0" }, - { "\"/str\0k", ResultType::Throw, "JSON: incorrect syntax (expected end of string, found end of JSON)." }, - { "\"Отвертка 2 в 1 \\\"TUNDRA basic\\\" 5х75 мм (+,-) \0\xD0", ResultType::Throw, "JSON: incorrect syntax (expected end of string, found end of JSON)." }, - { "\"/stroitelstvo-i-remont/stroitelnyy-instrument/avtoinstrumen\0=", ResultType::Throw, "JSON: incorrect syntax (expected end of string, found end of JSON)." }, - { "\"Чайник электрический Vitesse\0=", ResultType::Throw, "JSON: incorrect syntax (expected end of string, found end of JSON)." }, - { "\"Мелкая бытовая техника/Мелки\xD0\0\xD0", ResultType::Throw, "JSON: incorrect syntax (expected end of string, found end of JSON)." }, - { "\"Пряжа \\\"Бамбук стрейч\\0о", ResultType::Throw, "JSON: incorrect syntax (expected end of string, found end of JSON)." }, - { "\"Карандаш чёрнографитны\xD0\0k", ResultType::Throw, "JSON: incorrect syntax (expected end of string, found end of JSON)." }, - { "\"/Творчество/Рукоделие, аппликации/Пряжа и шерсть для \xD0\0\"", ResultType::Return, "/Творчество/Рукоделие, аппликации/Пряжа и шерсть для \xD0\0" }, - { "\"/1071547/karandash-chernografitnyy-volshebstvo-nv-kruglyy-d-7-2mm-dl-176mm-plast-tuba/\0o", ResultType::Throw, "JSON: incorrect syntax (expected end of string, found end of JSON)." }, - { "\"ca\0o", ResultType::Throw, "JSON: incorrect syntax (expected end of string, found end of JSON)." }, - { "\"/Подаро\0o", ResultType::Throw, "JSON: incorrect syntax (expected end of string, found end of JSON)." }, - { "\"Средство для прочис\xD1\0o", ResultType::Throw, "JSON: incorrect syntax (expected end of string, found end of JSON)." }, - { "\"i\0o", ResultType::Throw, "JSON: incorrect syntax (expected end of string, found end of JSON)." }, - { "\"/p\0\"", ResultType::Return, "/p\0" }, - { "\"/Сувениры/Магниты, н\xD0\0k", ResultType::Throw, "JSON: incorrect syntax (expected end of string, found end of JSON)." }, - { "\"Дерев\xD0\0=", ResultType::Throw, "JSON: incorrect syntax (expected end of string, found end of JSON)." }, - { "\"/prazdniki/svadba/svadebnaya-c\0\xD0", ResultType::Throw, "JSON: incorrect syntax (expected end of string, found end of JSON)." }, - { "\"/Канцт\0d", ResultType::Throw, "JSON: incorrect syntax (expected end of string, found end of JSON)." }, - { "\"/Праздники/То\xD0\0 ", ResultType::Throw, "JSON: incorrect syntax (expected end of string, found end of JSON)." }, - { "\"v\0 ", ResultType::Throw, "JSON: incorrect syntax (expected end of string, found end of JSON)." }, - { "\"/Косметика \xD0\0d", ResultType::Throw, "JSON: incorrect syntax (expected end of string, found end of JSON)." }, - { "\"/Спорт и отдых/Настольные игры/Покер, руле\xD1\0\xD0", ResultType::Throw, "JSON: incorrect syntax (expected end of string, found end of JSON)." }, - { "\"categ\0=", ResultType::Throw, "JSON: incorrect syntax (expected end of string, found end of JSON)." }, - { "\"/retailr\0k", ResultType::Throw, "JSON: incorrect syntax (expected end of string, found end of JSON)." }, - { "\"/retailrocket\0k", ResultType::Throw, "JSON: incorrect syntax (expected end of string, found end of JSON)." }, - { "\"Ежедневник недат А5 140л кл,ляссе,обл пв\0=", ResultType::Throw, "JSON: incorrect syntax (expected end of string, found end of JSON)." }, - { "\"/432809/ezhednevnik-organayzer-sredniy-s-remeshkom-na-knopke-v-oblozhke-kalkulyator-kalendar-do-\0\xD0", ResultType::Throw, "JSON: incorrect syntax (expected end of string, found end of JSON)." }, - { "\"/1165424/chipbord-vyrubnoy-dlya-skrapbukinga-malyshi-mikki-maus-disney-bebi\0d", ResultType::Throw, "JSON: incorrect syntax (expected end of string, found end of JSON)." }, - { "\"/posuda/kuhonnye-prinadlezhnosti-i-i\0 ", ResultType::Throw, "JSON: incorrect syntax (expected end of string, found end of JSON)." }, - { "\"/Канцтовары/Ежедневники и блокн\xD0\0o", ResultType::Throw, "JSON: incorrect syntax (expected end of string, found end of JSON)." }, - { "\"/kanctovary/ezhednevniki-i-blok\00", ResultType::Throw, "JSON: incorrect syntax (expected end of string, found end of JSON)." }, - { "\"Стакан \xD0\0\0", ResultType::Throw, "JSON: incorrect syntax (expected end of string, found end of JSON)." }, - { "\"Набор бумаги для скрапбукинга \\\"Мои первый годик\\\": Микки Маус, Дисней бэби, 12 листов 29.5 х 29.5 см, 160\0\0", ResultType::Throw, "JSON: incorrect syntax (expected end of string, found end of JSON)." }, - { "\"c\0\"", ResultType::Return, "c\0" }, + { R"("name")"sv, ResultType::Return, "name"sv }, + { R"("Вафельница Vitek WX-1102 FL")"sv, ResultType::Return, "Вафельница Vitek WX-1102 FL"sv }, + { R"("brand")"sv, ResultType::Return, "brand"sv }, + { R"("184509")"sv, ResultType::Return, "184509"sv }, + { R"("category")"sv, ResultType::Return, "category"sv }, + { R"("Все для детей/Детская техника/Vitek")"sv, ResultType::Return, "Все для детей/Детская техника/Vitek"sv }, + { R"("variant")"sv, ResultType::Return, "variant"sv }, + { R"("В наличии")"sv, ResultType::Return, "В наличии"sv }, + { R"("price")"sv, ResultType::Return, "price"sv }, + { R"("2390.00")"sv, ResultType::Return, "2390.00"sv }, + { R"("list")"sv, ResultType::Return, "list"sv }, + { R"("Карточка")"sv, ResultType::Return, "Карточка"sv }, + { R"("position")"sv, ResultType::Return, "position"sv }, + { R"("detail")"sv, ResultType::Return, "detail"sv }, + { R"("actionField")"sv, ResultType::Return, "actionField"sv }, + { R"("list")"sv, ResultType::Return, "list"sv }, + { R"("http://www.techport.ru/q/?t=вафельница&sort=price&sdim=asc")"sv, ResultType::Return, "http://www.techport.ru/q/?t=вафельница&sort=price&sdim=asc"sv }, + { R"("action")"sv, ResultType::Return, "action"sv }, + { R"("detail")"sv, ResultType::Return, "detail"sv }, + { R"("products")"sv, ResultType::Return, "products"sv }, + { R"("name")"sv, ResultType::Return, "name"sv }, + { R"("Вафельница Vitek WX-1102 FL")"sv, ResultType::Return, "Вафельница Vitek WX-1102 FL"sv }, + { R"("id")"sv, ResultType::Return, "id"sv }, + { R"("184509")"sv, ResultType::Return, "184509"sv }, + { R"("price")"sv, ResultType::Return, "price"sv }, + { R"("2390.00")"sv, ResultType::Return, "2390.00"sv }, + { R"("brand")"sv, ResultType::Return, "brand"sv }, + { R"("Vitek")"sv, ResultType::Return, "Vitek"sv }, + { R"("category")"sv, ResultType::Return, "category"sv }, + { R"("Все для детей/Детская техника/Vitek")"sv, ResultType::Return, "Все для детей/Детская техника/Vitek"sv }, + { R"("variant")"sv, ResultType::Return, "variant"sv }, + { R"("В наличии")"sv, ResultType::Return, "В наличии"sv }, + { R"("ru")"sv, ResultType::Return, "ru"sv }, + { R"("experiments")"sv, ResultType::Return, "experiments"sv }, + { R"("lang")"sv, ResultType::Return, "lang"sv }, + { R"("ru")"sv, ResultType::Return, "ru"sv }, + { R"("los_portal")"sv, ResultType::Return, "los_portal"sv }, + { R"("los_level")"sv, ResultType::Return, "los_level"sv }, + { R"("none")"sv, ResultType::Return, "none"sv }, + { R"("isAuthorized")"sv, ResultType::Return, "isAuthorized"sv }, + { R"("isSubscriber")"sv, ResultType::Return, "isSubscriber"sv }, + { R"("postType")"sv, ResultType::Return, "postType"sv }, + { R"("Новости")"sv, ResultType::Return, "Новости"sv }, + { R"("experiments")"sv, ResultType::Return, "experiments"sv }, + { R"("lang")"sv, ResultType::Return, "lang"sv }, + { R"("ru")"sv, ResultType::Return, "ru"sv }, + { R"("los_portal")"sv, ResultType::Return, "los_portal"sv }, + { R"("los_level")"sv, ResultType::Return, "los_level"sv }, + { R"("none")"sv, ResultType::Return, "none"sv }, + { R"("lang")"sv, ResultType::Return, "lang"sv }, + { R"("ru")"sv, ResultType::Return, "ru"sv }, + { R"("Электроплита GEFEST Брест ЭПНД 5140-01 0001")"sv, ResultType::Return, "Электроплита GEFEST Брест ЭПНД 5140-01 0001"sv }, + { R"("price")"sv, ResultType::Return, "price"sv }, + { R"("currencyCode")"sv, ResultType::Return, "currencyCode"sv }, + { R"("RUB")"sv, ResultType::Return, "RUB"sv }, + { R"("lang")"sv, ResultType::Return, "lang"sv }, + { R"("ru")"sv, ResultType::Return, "ru"sv }, + { R"("experiments")"sv, ResultType::Return, "experiments"sv }, + { R"("lang")"sv, ResultType::Return, "lang"sv }, + { R"("ru")"sv, ResultType::Return, "ru"sv }, + { R"("los_portal")"sv, ResultType::Return, "los_portal"sv }, + { R"("los_level")"sv, ResultType::Return, "los_level"sv }, + { R"("none")"sv, ResultType::Return, "none"sv }, + { R"("trash_login")"sv, ResultType::Return, "trash_login"sv }, + { R"("novikoff")"sv, ResultType::Return, "novikoff"sv }, + { R"("trash_cat_link")"sv, ResultType::Return, "trash_cat_link"sv }, + { R"("progs")"sv, ResultType::Return, "progs"sv }, + { R"("trash_parent_link")"sv, ResultType::Return, "trash_parent_link"sv }, + { R"("content")"sv, ResultType::Return, "content"sv }, + { R"("trash_posted_parent")"sv, ResultType::Return, "trash_posted_parent"sv }, + { R"("content.01.2016")"sv, ResultType::Return, "content.01.2016"sv }, + { R"("trash_posted_cat")"sv, ResultType::Return, "trash_posted_cat"sv }, + { R"("progs.01.2016")"sv, ResultType::Return, "progs.01.2016"sv }, + { R"("trash_virus_count")"sv, ResultType::Return, "trash_virus_count"sv }, + { R"("trash_is_android")"sv, ResultType::Return, "trash_is_android"sv }, + { R"("trash_is_wp8")"sv, ResultType::Return, "trash_is_wp8"sv }, + { R"("trash_is_ios")"sv, ResultType::Return, "trash_is_ios"sv }, + { R"("trash_posted")"sv, ResultType::Return, "trash_posted"sv }, + { R"("01.2016")"sv, ResultType::Return, "01.2016"sv }, + { R"("experiments")"sv, ResultType::Return, "experiments"sv }, + { R"("lang")"sv, ResultType::Return, "lang"sv }, + { R"("ru")"sv, ResultType::Return, "ru"sv }, + { R"("los_portal")"sv, ResultType::Return, "los_portal"sv }, + { R"("los_level")"sv, ResultType::Return, "los_level"sv }, + { R"("none")"sv, ResultType::Return, "none"sv }, + { R"("merchantId")"sv, ResultType::Return, "merchantId"sv }, + { R"("13694_49246")"sv, ResultType::Return, "13694_49246"sv }, + { R"("cps-source")"sv, ResultType::Return, "cps-source"sv }, + { R"("wargaming")"sv, ResultType::Return, "wargaming"sv }, + { R"("cps_provider")"sv, ResultType::Return, "cps_provider"sv }, + { R"("default")"sv, ResultType::Return, "default"sv }, + { R"("errorReason")"sv, ResultType::Return, "errorReason"sv }, + { R"("no errors")"sv, ResultType::Return, "no errors"sv }, + { R"("scid")"sv, ResultType::Return, "scid"sv }, + { R"("isAuthPayment")"sv, ResultType::Return, "isAuthPayment"sv }, + { R"("lang")"sv, ResultType::Return, "lang"sv }, + { R"("ru")"sv, ResultType::Return, "ru"sv }, + { R"("rubric")"sv, ResultType::Return, "rubric"sv }, + { R"("")"sv, ResultType::Return, ""sv }, + { R"("rubric")"sv, ResultType::Return, "rubric"sv }, + { R"("Мир")"sv, ResultType::Return, "Мир"sv }, + { R"("lang")"sv, ResultType::Return, "lang"sv }, + { R"("ru")"sv, ResultType::Return, "ru"sv }, + { R"("experiments")"sv, ResultType::Return, "experiments"sv }, + { R"("lang")"sv, ResultType::Return, "lang"sv }, + { R"("ru")"sv, ResultType::Return, "ru"sv }, + { R"("los_portal")"sv, ResultType::Return, "los_portal"sv }, + { R"("los_level")"sv, ResultType::Return, "los_level"sv }, + { R"("none")"sv, ResultType::Return, "none"sv }, + { R"("lang")"sv, ResultType::Return, "lang"sv }, + { R"("ru")"sv, ResultType::Return, "ru"sv }, + { R"("__ym")"sv, ResultType::Return, "__ym"sv }, + { R"("ecommerce")"sv, ResultType::Return, "ecommerce"sv }, + { R"("impressions")"sv, ResultType::Return, "impressions"sv }, + { R"("id")"sv, ResultType::Return, "id"sv }, + { R"("863813")"sv, ResultType::Return, "863813"sv }, + { R"("name")"sv, ResultType::Return, "name"sv }, + { R"("Футболка детская 3D Happy, возраст 1-2 года, трикотаж")"sv, ResultType::Return, "Футболка детская 3D Happy, возраст 1-2 года, трикотаж"sv }, + { R"("category")"sv, ResultType::Return, "category"sv }, + { R"("/Летние товары/Летний текстиль/")"sv, ResultType::Return, "/Летние товары/Летний текстиль/"sv }, + { R"("variant")"sv, ResultType::Return, "variant"sv }, + { R"("")"sv, ResultType::Return, ""sv }, + { R"("price")"sv, ResultType::Return, "price"sv }, + { R"("390.00")"sv, ResultType::Return, "390.00"sv }, + { R"("list")"sv, ResultType::Return, "list"sv }, + { R"("/retailrocket/")"sv, ResultType::Return, "/retailrocket/"sv }, + { R"("position")"sv, ResultType::Return, "position"sv }, + { R"("brand")"sv, ResultType::Return, "brand"sv }, + { R"("/911488/futbolka-detskaya-3d-dolphin-vozrast-1-2-goda-trikotazh/")"sv, ResultType::Return, "/911488/futbolka-detskaya-3d-dolphin-vozrast-1-2-goda-trikotazh/"sv }, + { R"("id")"sv, ResultType::Return, "id"sv }, + { R"("863839")"sv, ResultType::Return, "863839"sv }, + { R"("name")"sv, ResultType::Return, "name"sv }, + { R"("Футболка детская 3D Pretty kitten, возраст 1-2 года, трикотаж")"sv, ResultType::Return, "Футболка детская 3D Pretty kitten, возраст 1-2 года, трикотаж"sv }, + { R"("category")"sv, ResultType::Return, "category"sv }, + { R"("/Летние товары/Летний текстиль/")"sv, ResultType::Return, "/Летние товары/Летний текстиль/"sv }, + { R"("variant")"sv, ResultType::Return, "variant"sv }, + { R"("")"sv, ResultType::Return, ""sv }, + { R"("price")"sv, ResultType::Return, "price"sv }, + { R"("390.00")"sv, ResultType::Return, "390.00"sv }, + { R"("list")"sv, ResultType::Return, "list"sv }, + { R"("/retailrocket/")"sv, ResultType::Return, "/retailrocket/"sv }, + { R"("position")"sv, ResultType::Return, "position"sv }, + { R"("brand")"sv, ResultType::Return, "brand"sv }, + { R"("/911488/futbolka-detskaya-3d-dolphin-vozrast-1-2-goda-trikotazh/")"sv, ResultType::Return, "/911488/futbolka-detskaya-3d-dolphin-vozrast-1-2-goda-trikotazh/"sv }, + { R"("id")"sv, ResultType::Return, "id"sv }, + { R"("863847")"sv, ResultType::Return, "863847"sv }, + { R"("name")"sv, ResultType::Return, "name"sv }, + { R"("Футболка детская 3D Little tiger, возраст 1-2 года, трикотаж")"sv, ResultType::Return, "Футболка детская 3D Little tiger, возраст 1-2 года, трикотаж"sv }, + { R"("category")"sv, ResultType::Return, "category"sv }, + { R"("/Летние товары/Летний текстиль/")"sv, ResultType::Return, "/Летние товары/Летний текстиль/"sv }, + { R"("variant")"sv, ResultType::Return, "variant"sv }, + { R"("")"sv, ResultType::Return, ""sv }, + { R"("price")"sv, ResultType::Return, "price"sv }, + { R"("390.00")"sv, ResultType::Return, "390.00"sv }, + { R"("list")"sv, ResultType::Return, "list"sv }, + { R"("/retailrocket/")"sv, ResultType::Return, "/retailrocket/"sv }, + { R"("position")"sv, ResultType::Return, "position"sv }, + { R"("brand")"sv, ResultType::Return, "brand"sv }, + { R"("/911488/futbolka-detskaya-3d-dolphin-vozrast-1-2-goda-trikotazh/")"sv, ResultType::Return, "/911488/futbolka-detskaya-3d-dolphin-vozrast-1-2-goda-trikotazh/"sv }, + { R"("id")"sv, ResultType::Return, "id"sv }, + { R"("911480")"sv, ResultType::Return, "911480"sv }, + { R"("name")"sv, ResultType::Return, "name"sv }, + { R"("Футболка детская 3D Puppy, возраст 1-2 года, трикотаж")"sv, ResultType::Return, "Футболка детская 3D Puppy, возраст 1-2 года, трикотаж"sv }, + { R"("category")"sv, ResultType::Return, "category"sv }, + { R"("/Летние товары/Летний текстиль/")"sv, ResultType::Return, "/Летние товары/Летний текстиль/"sv }, + { R"("variant")"sv, ResultType::Return, "variant"sv }, + { R"("")"sv, ResultType::Return, ""sv }, + { R"("price")"sv, ResultType::Return, "price"sv }, + { R"("390.00")"sv, ResultType::Return, "390.00"sv }, + { R"("list")"sv, ResultType::Return, "list"sv }, + { R"("/retailrocket/")"sv, ResultType::Return, "/retailrocket/"sv }, + { R"("position")"sv, ResultType::Return, "position"sv }, + { R"("brand")"sv, ResultType::Return, "brand"sv }, + { R"("/911488/futbolka-detskaya-3d-dolphin-vozrast-1-2-goda-trikotazh/")"sv, ResultType::Return, "/911488/futbolka-detskaya-3d-dolphin-vozrast-1-2-goda-trikotazh/"sv }, + { R"("id")"sv, ResultType::Return, "id"sv }, + { R"("911484")"sv, ResultType::Return, "911484"sv }, + { R"("name")"sv, ResultType::Return, "name"sv }, + { R"("Футболка детская 3D Little bears, возраст 1-2 года, трикотаж")"sv, ResultType::Return, "Футболка детская 3D Little bears, возраст 1-2 года, трикотаж"sv }, + { R"("category")"sv, ResultType::Return, "category"sv }, + { R"("/Летние товары/Летний текстиль/")"sv, ResultType::Return, "/Летние товары/Летний текстиль/"sv }, + { R"("variant")"sv, ResultType::Return, "variant"sv }, + { R"("")"sv, ResultType::Return, ""sv }, + { R"("price")"sv, ResultType::Return, "price"sv }, + { R"("390.00")"sv, ResultType::Return, "390.00"sv }, + { R"("list")"sv, ResultType::Return, "list"sv }, + { R"("/retailrocket/")"sv, ResultType::Return, "/retailrocket/"sv }, + { R"("position")"sv, ResultType::Return, "position"sv }, + { R"("brand")"sv, ResultType::Return, "brand"sv }, + { R"("/911488/futbolka-detskaya-3d-dolphin-vozrast-1-2-goda-trikotazh/")"sv, ResultType::Return, "/911488/futbolka-detskaya-3d-dolphin-vozrast-1-2-goda-trikotazh/"sv }, + { R"("id")"sv, ResultType::Return, "id"sv }, + { R"("911489")"sv, ResultType::Return, "911489"sv }, + { R"("name")"sv, ResultType::Return, "name"sv }, + { R"("Футболка детская 3D Dolphin, возраст 2-4 года, трикотаж")"sv, ResultType::Return, "Футболка детская 3D Dolphin, возраст 2-4 года, трикотаж"sv }, + { R"("category")"sv, ResultType::Return, "category"sv }, + { R"("/Летние товары/Летний текстиль/")"sv, ResultType::Return, "/Летние товары/Летний текстиль/"sv }, + { R"("variant")"sv, ResultType::Return, "variant"sv }, + { R"("")"sv, ResultType::Return, ""sv }, + { R"("price")"sv, ResultType::Return, "price"sv }, + { R"("390.00")"sv, ResultType::Return, "390.00"sv }, + { R"("list")"sv, ResultType::Return, "list"sv }, + { R"("/retailrocket/")"sv, ResultType::Return, "/retailrocket/"sv }, + { R"("position")"sv, ResultType::Return, "position"sv }, + { R"("brand")"sv, ResultType::Return, "brand"sv }, + { R"("/911488/futbolka-detskaya-3d-dolphin-vozrast-1-2-goda-trikotazh/")"sv, ResultType::Return, "/911488/futbolka-detskaya-3d-dolphin-vozrast-1-2-goda-trikotazh/"sv }, + { R"("id")"sv, ResultType::Return, "id"sv }, + { R"("911496")"sv, ResultType::Return, "911496"sv }, + { R"("name")"sv, ResultType::Return, "name"sv }, + { R"("Футболка детская 3D Pretty, возраст 1-2 года, трикотаж")"sv, ResultType::Return, "Футболка детская 3D Pretty, возраст 1-2 года, трикотаж"sv }, + { R"("category")"sv, ResultType::Return, "category"sv }, + { R"("/Летние товары/Летний текстиль/")"sv, ResultType::Return, "/Летние товары/Летний текстиль/"sv }, + { R"("variant")"sv, ResultType::Return, "variant"sv }, + { R"("")"sv, ResultType::Return, ""sv }, + { R"("price")"sv, ResultType::Return, "price"sv }, + { R"("390.00")"sv, ResultType::Return, "390.00"sv }, + { R"("list")"sv, ResultType::Return, "list"sv }, + { R"("/retailrocket/")"sv, ResultType::Return, "/retailrocket/"sv }, + { R"("position")"sv, ResultType::Return, "position"sv }, + { R"("brand")"sv, ResultType::Return, "brand"sv }, + { R"("/911488/futbolka-detskaya-3d-dolphin-vozrast-1-2-goda-trikotazh/")"sv, ResultType::Return, "/911488/futbolka-detskaya-3d-dolphin-vozrast-1-2-goda-trikotazh/"sv }, + { R"("id")"sv, ResultType::Return, "id"sv }, + { R"("911504")"sv, ResultType::Return, "911504"sv }, + { R"("name")"sv, ResultType::Return, "name"sv }, + { R"("Футболка детская 3D Fairytale, возраст 1-2 года, трикотаж")"sv, ResultType::Return, "Футболка детская 3D Fairytale, возраст 1-2 года, трикотаж"sv }, + { R"("category")"sv, ResultType::Return, "category"sv }, + { R"("/Летние товары/Летний текстиль/")"sv, ResultType::Return, "/Летние товары/Летний текстиль/"sv }, + { R"("variant")"sv, ResultType::Return, "variant"sv }, + { R"("")"sv, ResultType::Return, ""sv }, + { R"("price")"sv, ResultType::Return, "price"sv }, + { R"("390.00")"sv, ResultType::Return, "390.00"sv }, + { R"("list")"sv, ResultType::Return, "list"sv }, + { R"("/retailrocket/")"sv, ResultType::Return, "/retailrocket/"sv }, + { R"("position")"sv, ResultType::Return, "position"sv }, + { R"("brand")"sv, ResultType::Return, "brand"sv }, + { R"("/911488/futbolka-detskaya-3d-dolphin-vozrast-1-2-goda-trikotazh/")"sv, ResultType::Return, "/911488/futbolka-detskaya-3d-dolphin-vozrast-1-2-goda-trikotazh/"sv }, + { R"("id")"sv, ResultType::Return, "id"sv }, + { R"("911508")"sv, ResultType::Return, "911508"sv }, + { R"("name")"sv, ResultType::Return, "name"sv }, + { R"("Футболка детская 3D Kittens, возраст 1-2 года, трикотаж")"sv, ResultType::Return, "Футболка детская 3D Kittens, возраст 1-2 года, трикотаж"sv }, + { R"("category")"sv, ResultType::Return, "category"sv }, + { R"("/Летние товары/Летний текстиль/")"sv, ResultType::Return, "/Летние товары/Летний текстиль/"sv }, + { R"("variant")"sv, ResultType::Return, "variant"sv }, + { R"("")"sv, ResultType::Return, ""sv }, + { R"("price")"sv, ResultType::Return, "price"sv }, + { R"("390.00")"sv, ResultType::Return, "390.00"sv }, + { R"("list")"sv, ResultType::Return, "list"sv }, + { R"("/retailrocket/")"sv, ResultType::Return, "/retailrocket/"sv }, + { R"("position")"sv, ResultType::Return, "position"sv }, + { R"("brand")"sv, ResultType::Return, "brand"sv }, + { R"("/911488/futbolka-detskaya-3d-dolphin-vozrast-1-2-goda-trikotazh/")"sv, ResultType::Return, "/911488/futbolka-detskaya-3d-dolphin-vozrast-1-2-goda-trikotazh/"sv }, + { R"("id")"sv, ResultType::Return, "id"sv }, + { R"("911512")"sv, ResultType::Return, "911512"sv }, + { R"("name")"sv, ResultType::Return, "name"sv }, + { R"("Футболка детская 3D Sunshine, возраст 1-2 года, трикотаж")"sv, ResultType::Return, "Футболка детская 3D Sunshine, возраст 1-2 года, трикотаж"sv }, + { R"("category")"sv, ResultType::Return, "category"sv }, + { R"("/Летние товары/Летний текстиль/")"sv, ResultType::Return, "/Летние товары/Летний текстиль/"sv }, + { R"("variant")"sv, ResultType::Return, "variant"sv }, + { R"("")"sv, ResultType::Return, ""sv }, + { R"("price")"sv, ResultType::Return, "price"sv }, + { R"("390.00")"sv, ResultType::Return, "390.00"sv }, + { R"("list")"sv, ResultType::Return, "list"sv }, + { R"("/retailrocket/")"sv, ResultType::Return, "/retailrocket/"sv }, + { R"("position")"sv, ResultType::Return, "position"sv }, + { R"("brand")"sv, ResultType::Return, "brand"sv }, + { R"("/911488/futbolka-detskaya-3d-dolphin-vozrast-1-2-goda-trikotazh/")"sv, ResultType::Return, "/911488/futbolka-detskaya-3d-dolphin-vozrast-1-2-goda-trikotazh/"sv }, + { R"("id")"sv, ResultType::Return, "id"sv }, + { R"("911516")"sv, ResultType::Return, "911516"sv }, + { R"("name")"sv, ResultType::Return, "name"sv }, + { R"("Футболка детская 3D Dog in bag, возраст 1-2 года, трикотаж")"sv, ResultType::Return, "Футболка детская 3D Dog in bag, возраст 1-2 года, трикотаж"sv }, + { R"("category")"sv, ResultType::Return, "category"sv }, + { R"("/Летние товары/Летний текстиль/")"sv, ResultType::Return, "/Летние товары/Летний текстиль/"sv }, + { R"("variant")"sv, ResultType::Return, "variant"sv }, + { R"("")"sv, ResultType::Return, ""sv }, + { R"("price")"sv, ResultType::Return, "price"sv }, + { R"("390.00")"sv, ResultType::Return, "390.00"sv }, + { R"("list")"sv, ResultType::Return, "list"sv }, + { R"("/retailrocket/")"sv, ResultType::Return, "/retailrocket/"sv }, + { R"("position")"sv, ResultType::Return, "position"sv }, + { R"("brand")"sv, ResultType::Return, "brand"sv }, + { R"("/911488/futbolka-detskaya-3d-dolphin-vozrast-1-2-goda-trikotazh/")"sv, ResultType::Return, "/911488/futbolka-detskaya-3d-dolphin-vozrast-1-2-goda-trikotazh/"sv }, + { R"("id")"sv, ResultType::Return, "id"sv }, + { R"("911520")"sv, ResultType::Return, "911520"sv }, + { R"("name")"sv, ResultType::Return, "name"sv }, + { R"("Футболка детская 3D Cute puppy, возраст 1-2 года, трикотаж")"sv, ResultType::Return, "Футболка детская 3D Cute puppy, возраст 1-2 года, трикотаж"sv }, + { R"("category")"sv, ResultType::Return, "category"sv }, + { R"("/Летние товары/Летний текстиль/")"sv, ResultType::Return, "/Летние товары/Летний текстиль/"sv }, + { R"("variant")"sv, ResultType::Return, "variant"sv }, + { R"("")"sv, ResultType::Return, ""sv }, + { R"("price")"sv, ResultType::Return, "price"sv }, + { R"("390.00")"sv, ResultType::Return, "390.00"sv }, + { R"("list")"sv, ResultType::Return, "list"sv }, + { R"("/retailrocket/")"sv, ResultType::Return, "/retailrocket/"sv }, + { R"("position")"sv, ResultType::Return, "position"sv }, + { R"("brand")"sv, ResultType::Return, "brand"sv }, + { R"("/911488/futbolka-detskaya-3d-dolphin-vozrast-1-2-goda-trikotazh/")"sv, ResultType::Return, "/911488/futbolka-detskaya-3d-dolphin-vozrast-1-2-goda-trikotazh/"sv }, + { R"("id")"sv, ResultType::Return, "id"sv }, + { R"("911524")"sv, ResultType::Return, "911524"sv }, + { R"("name")"sv, ResultType::Return, "name"sv }, + { R"("Футболка детская 3D Rabbit, возраст 1-2 года, трикотаж")"sv, ResultType::Return, "Футболка детская 3D Rabbit, возраст 1-2 года, трикотаж"sv }, + { R"("category")"sv, ResultType::Return, "category"sv }, + { R"("/Летние товары/Летний текстиль/")"sv, ResultType::Return, "/Летние товары/Летний текстиль/"sv }, + { R"("variant")"sv, ResultType::Return, "variant"sv }, + { R"("")"sv, ResultType::Return, ""sv }, + { R"("price")"sv, ResultType::Return, "price"sv }, + { R"("390.00")"sv, ResultType::Return, "390.00"sv }, + { R"("list")"sv, ResultType::Return, "list"sv }, + { R"("/retailrocket/")"sv, ResultType::Return, "/retailrocket/"sv }, + { R"("position")"sv, ResultType::Return, "position"sv }, + { R"("brand")"sv, ResultType::Return, "brand"sv }, + { R"("/911488/futbolka-detskaya-3d-dolphin-vozrast-1-2-goda-trikotazh/")"sv, ResultType::Return, "/911488/futbolka-detskaya-3d-dolphin-vozrast-1-2-goda-trikotazh/"sv }, + { R"("id")"sv, ResultType::Return, "id"sv }, + { R"("911528")"sv, ResultType::Return, "911528"sv }, + { R"("name")"sv, ResultType::Return, "name"sv }, + { R"("Футболка детская 3D Turtle, возраст 1-2 года, трикотаж")"sv, ResultType::Return, "Футболка детская 3D Turtle, возраст 1-2 года, трикотаж"sv }, + { R"("category")"sv, ResultType::Return, "category"sv }, + { R"("/Летние товары/Летний текстиль/")"sv, ResultType::Return, "/Летние товары/Летний текстиль/"sv }, + { R"("variant")"sv, ResultType::Return, "variant"sv }, + { R"("")"sv, ResultType::Return, ""sv }, + { R"("price")"sv, ResultType::Return, "price"sv }, + { R"("390.00")"sv, ResultType::Return, "390.00"sv }, + { R"("list")"sv, ResultType::Return, "list"sv }, + { R"("/retailrocket/")"sv, ResultType::Return, "/retailrocket/"sv }, + { R"("position")"sv, ResultType::Return, "position"sv }, + { R"("brand")"sv, ResultType::Return, "brand"sv }, + { R"("/911488/futbolka-detskaya-3d-dolphin-vozrast-1-2-goda-trikotazh/")"sv, ResultType::Return, "/911488/futbolka-detskaya-3d-dolphin-vozrast-1-2-goda-trikotazh/"sv }, + { R"("id")"sv, ResultType::Return, "id"sv }, + { R"("888616")"sv, ResultType::Return, "888616"sv }, + { R"("name")"sv, ResultType::Return, "name"sv }, + { "\"3Д Футболка мужская \\\"Collorista\\\" Светлое завтра р-р XL(52-54), 100% хлопок, трикотаж\""sv, ResultType::Return, "3Д Футболка мужская \"Collorista\" Светлое завтра р-р XL(52-54), 100% хлопок, трикотаж"sv }, + { R"("category")"sv, ResultType::Return, "category"sv }, + { R"("/Одежда и обувь/Мужская одежда/Футболки/")"sv, ResultType::Return, "/Одежда и обувь/Мужская одежда/Футболки/"sv }, + { R"("variant")"sv, ResultType::Return, "variant"sv }, + { R"("")"sv, ResultType::Return, ""sv }, + { R"("price")"sv, ResultType::Return, "price"sv }, + { R"("406.60")"sv, ResultType::Return, "406.60"sv }, + { R"("list")"sv, ResultType::Return, "list"sv }, + { R"("/retailrocket/")"sv, ResultType::Return, "/retailrocket/"sv }, + { R"("position")"sv, ResultType::Return, "position"sv }, + { R"("brand")"sv, ResultType::Return, "brand"sv }, + { R"("/911488/futbolka-detskaya-3d-dolphin-vozrast-1-2-goda-trikotazh/")"sv, ResultType::Return, "/911488/futbolka-detskaya-3d-dolphin-vozrast-1-2-goda-trikotazh/"sv }, + { R"("id")"sv, ResultType::Return, "id"sv }, + { R"("913361")"sv, ResultType::Return, "913361"sv }, + { R"("name")"sv, ResultType::Return, "name"sv }, + { R"("3Д Футболка детская World р-р 8-10, 100% хлопок, трикотаж")"sv, ResultType::Return, "3Д Футболка детская World р-р 8-10, 100% хлопок, трикотаж"sv }, + { R"("category")"sv, ResultType::Return, "category"sv }, + { R"("/Летние товары/Летний текстиль/")"sv, ResultType::Return, "/Летние товары/Летний текстиль/"sv }, + { R"("variant")"sv, ResultType::Return, "variant"sv }, + { R"("")"sv, ResultType::Return, ""sv }, + { R"("price")"sv, ResultType::Return, "price"sv }, + { R"("470.00")"sv, ResultType::Return, "470.00"sv }, + { R"("list")"sv, ResultType::Return, "list"sv }, + { R"("/retailrocket/")"sv, ResultType::Return, "/retailrocket/"sv }, + { R"("position")"sv, ResultType::Return, "position"sv }, + { R"("brand")"sv, ResultType::Return, "brand"sv }, + { R"("/911488/futbolka-detskaya-3d-dolphin-vozrast-1-2-goda-trikotazh/")"sv, ResultType::Return, "/911488/futbolka-detskaya-3d-dolphin-vozrast-1-2-goda-trikotazh/"sv }, + { R"("id")"sv, ResultType::Return, "id"sv }, + { R"("913364")"sv, ResultType::Return, "913364"sv }, + { R"("name")"sv, ResultType::Return, "name"sv }, + { R"("3Д Футболка детская Force р-р 8-10, 100% хлопок, трикотаж")"sv, ResultType::Return, "3Д Футболка детская Force р-р 8-10, 100% хлопок, трикотаж"sv }, + { R"("category")"sv, ResultType::Return, "category"sv }, + { R"("/Летние товары/Летний текстиль/")"sv, ResultType::Return, "/Летние товары/Летний текстиль/"sv }, + { R"("variant")"sv, ResultType::Return, "variant"sv }, + { R"("")"sv, ResultType::Return, ""sv }, + { R"("price")"sv, ResultType::Return, "price"sv }, + { R"("470.00")"sv, ResultType::Return, "470.00"sv }, + { R"("list")"sv, ResultType::Return, "list"sv }, + { R"("/retailrocket/")"sv, ResultType::Return, "/retailrocket/"sv }, + { R"("position")"sv, ResultType::Return, "position"sv }, + { R"("brand")"sv, ResultType::Return, "brand"sv }, + { R"("/911488/futbolka-detskaya-3d-dolphin-vozrast-1-2-goda-trikotazh/")"sv, ResultType::Return, "/911488/futbolka-detskaya-3d-dolphin-vozrast-1-2-goda-trikotazh/"sv }, + { R"("id")"sv, ResultType::Return, "id"sv }, + { R"("913367")"sv, ResultType::Return, "913367"sv }, + { R"("name")"sv, ResultType::Return, "name"sv }, + { R"("3Д Футболка детская Winter tale р-р 8-10, 100% хлопок, трикотаж")"sv, ResultType::Return, "3Д Футболка детская Winter tale р-р 8-10, 100% хлопок, трикотаж"sv }, + { R"("category")"sv, ResultType::Return, "category"sv }, + { R"("/Летние товары/Летний текстиль/")"sv, ResultType::Return, "/Летние товары/Летний текстиль/"sv }, + { R"("variant")"sv, ResultType::Return, "variant"sv }, + { R"("")"sv, ResultType::Return, ""sv }, + { R"("price")"sv, ResultType::Return, "price"sv }, + { R"("470.00")"sv, ResultType::Return, "470.00"sv }, + { R"("list")"sv, ResultType::Return, "list"sv }, + { R"("/retailrocket/")"sv, ResultType::Return, "/retailrocket/"sv }, + { R"("position")"sv, ResultType::Return, "position"sv }, + { R"("brand")"sv, ResultType::Return, "brand"sv }, + { R"("/911488/futbolka-detskaya-3d-dolphin-vozrast-1-2-goda-trikotazh/")"sv, ResultType::Return, "/911488/futbolka-detskaya-3d-dolphin-vozrast-1-2-goda-trikotazh/"sv }, + { R"("id")"sv, ResultType::Return, "id"sv }, + { R"("913385")"sv, ResultType::Return, "913385"sv }, + { R"("name")"sv, ResultType::Return, "name"sv }, + { R"("3Д Футболка детская Moonshine р-р 8-10, 100% хлопок, трикотаж")"sv, ResultType::Return, "3Д Футболка детская Moonshine р-р 8-10, 100% хлопок, трикотаж"sv }, + { R"("category")"sv, ResultType::Return, "category"sv }, + { R"("/Летние товары/Летний текстиль/")"sv, ResultType::Return, "/Летние товары/Летний текстиль/"sv }, + { R"("variant")"sv, ResultType::Return, "variant"sv }, + { R"("")"sv, ResultType::Return, ""sv }, + { R"("price")"sv, ResultType::Return, "price"sv }, + { R"("470.00")"sv, ResultType::Return, "470.00"sv }, + { R"("list")"sv, ResultType::Return, "list"sv }, + { R"("/retailrocket/")"sv, ResultType::Return, "/retailrocket/"sv }, + { R"("position")"sv, ResultType::Return, "position"sv }, + { R"("brand")"sv, ResultType::Return, "brand"sv }, + { R"("/911488/futbolka-detskaya-3d-dolphin-vozrast-1-2-goda-trikotazh/")"sv, ResultType::Return, "/911488/futbolka-detskaya-3d-dolphin-vozrast-1-2-goda-trikotazh/"sv }, + { R"("id")"sv, ResultType::Return, "id"sv }, + { R"("913391")"sv, ResultType::Return, "913391"sv }, + { R"("name")"sv, ResultType::Return, "name"sv }, + { R"("3Д Футболка детская Shaman р-р 8-10, 100% хлопок, трикотаж")"sv, ResultType::Return, "3Д Футболка детская Shaman р-р 8-10, 100% хлопок, трикотаж"sv }, + { R"("category")"sv, ResultType::Return, "category"sv }, + { R"("/Летние товары/Летний текстиль/")"sv, ResultType::Return, "/Летние товары/Летний текстиль/"sv }, + { R"("variant")"sv, ResultType::Return, "variant"sv }, + { R"("")"sv, ResultType::Return, ""sv }, + { R"("price")"sv, ResultType::Return, "price"sv }, + { R"("470.00")"sv, ResultType::Return, "470.00"sv }, + { R"("list")"sv, ResultType::Return, "list"sv }, + { R"("/retailrocket/")"sv, ResultType::Return, "/retailrocket/"sv }, + { R"("position")"sv, ResultType::Return, "position"sv }, + { R"("brand")"sv, ResultType::Return, "brand"sv }, + { R"("/911488/futbolka-detskaya-3d-dolphin-vozrast-1-2-goda-trikotazh/")"sv, ResultType::Return, "/911488/futbolka-detskaya-3d-dolphin-vozrast-1-2-goda-trikotazh/"sv }, + { R"("usertype")"sv, ResultType::Return, "usertype"sv }, + { R"("visitor")"sv, ResultType::Return, "visitor"sv }, + { R"("lang")"sv, ResultType::Return, "lang"sv }, + { R"("ru")"sv, ResultType::Return, "ru"sv }, + { R"("__ym")"sv, ResultType::Return, "__ym"sv }, + { R"("ecommerce")"sv, ResultType::Return, "ecommerce"sv }, + { R"("impressions")"sv, ResultType::Return, "impressions"sv }, + { R"("experiments")"sv, ResultType::Return, "experiments"sv }, + { R"("lang")"sv, ResultType::Return, "lang"sv }, + { R"("ru")"sv, ResultType::Return, "ru"sv }, + { R"("los_portal")"sv, ResultType::Return, "los_portal"sv }, + { R"("los_level")"sv, ResultType::Return, "los_level"sv }, + { R"("none")"sv, ResultType::Return, "none"sv }, + { R"("experiments")"sv, ResultType::Return, "experiments"sv }, + { R"("lang")"sv, ResultType::Return, "lang"sv }, + { R"("ru")"sv, ResultType::Return, "ru"sv }, + { R"("los_portal")"sv, ResultType::Return, "los_portal"sv }, + { R"("los_level")"sv, ResultType::Return, "los_level"sv }, + { R"("none")"sv, ResultType::Return, "none"sv }, + { R"("experiments")"sv, ResultType::Return, "experiments"sv }, + { R"("lang")"sv, ResultType::Return, "lang"sv }, + { R"("ru")"sv, ResultType::Return, "ru"sv }, + { R"("los_portal")"sv, ResultType::Return, "los_portal"sv }, + { R"("los_level")"sv, ResultType::Return, "los_level"sv }, + { R"("none")"sv, ResultType::Return, "none"sv }, + { R"("experiments")"sv, ResultType::Return, "experiments"sv }, + { R"("lang")"sv, ResultType::Return, "lang"sv }, + { R"("ru")"sv, ResultType::Return, "ru"sv }, + { R"("los_portal")"sv, ResultType::Return, "los_portal"sv }, + { R"("los_level")"sv, ResultType::Return, "los_level"sv }, + { R"("none")"sv, ResultType::Return, "none"sv }, + { R"("experiments")"sv, ResultType::Return, "experiments"sv }, + { R"("lang")"sv, ResultType::Return, "lang"sv }, + { R"("ru")"sv, ResultType::Return, "ru"sv }, + { R"("los_portal")"sv, ResultType::Return, "los_portal"sv }, + { R"("los_level")"sv, ResultType::Return, "los_level"sv }, + { R"("none")"sv, ResultType::Return, "none"sv }, + { R"("__ym")"sv, ResultType::Return, "__ym"sv }, + { R"("ecommerce")"sv, ResultType::Return, "ecommerce"sv }, + { R"("currencyCode")"sv, ResultType::Return, "currencyCode"sv }, + { R"("RUR")"sv, ResultType::Return, "RUR"sv }, + { R"("impressions")"sv, ResultType::Return, "impressions"sv }, + { R"("name")"sv, ResultType::Return, "name"sv }, + { R"("Чайник электрический Mystery MEK-1627, белый")"sv, ResultType::Return, "Чайник электрический Mystery MEK-1627, белый"sv }, + { R"("brand")"sv, ResultType::Return, "brand"sv }, + { R"("Mystery")"sv, ResultType::Return, "Mystery"sv }, + { R"("id")"sv, ResultType::Return, "id"sv }, + { R"("187180")"sv, ResultType::Return, "187180"sv }, + { R"("category")"sv, ResultType::Return, "category"sv }, + { R"("Мелкая бытовая техника/Мелкие кухонные приборы/Чайники электрические/Mystery")"sv, ResultType::Return, "Мелкая бытовая техника/Мелкие кухонные приборы/Чайники электрические/Mystery"sv }, + { R"("variant")"sv, ResultType::Return, "variant"sv }, + { R"("В наличии")"sv, ResultType::Return, "В наличии"sv }, + { R"("price")"sv, ResultType::Return, "price"sv }, + { R"("1630.00")"sv, ResultType::Return, "1630.00"sv }, + { R"("list")"sv, ResultType::Return, "list"sv }, + { R"("Карточка")"sv, ResultType::Return, "Карточка"sv }, + { R"("position")"sv, ResultType::Return, "position"sv }, + { R"("detail")"sv, ResultType::Return, "detail"sv }, + { R"("actionField")"sv, ResultType::Return, "actionField"sv }, + { R"("list")"sv, ResultType::Return, "list"sv }, + { "\0\""sv, ResultType::Throw, "JSON: expected \", got \0"sv }, + { "\"/igrushki/konstruktory\0"sv, ResultType::Throw, "JSON: incorrect syntax (expected end of string, found end of JSON)."sv }, + { "\"/1290414/komplekt-zhenskiy-dzhemper-plusbryuki-m-254-09-malina-plustemno-siniy-\0a"sv, ResultType::Throw, "JSON: incorrect syntax (expected end of string, found end of JSON)."sv }, + { "\"/Творчество/Рисование/Инструменты и кра\0a"sv, ResultType::Throw, "JSON: incorrect syntax (expected end of string, found end of JSON)."sv }, + { "\"Строительство и ремонт/Силовая техника/Зарядные устройства для автомобильных аккумуляторов/Пуско-зарядные устр\xD0\0a"sv, ResultType::Throw, "JSON: incorrect syntax (expected end of string, found end of JSON)."sv }, + { "\"Строительство и ремонт/Силовая техника/Зарядные устройств\xD0\0t"sv, ResultType::Throw, "JSON: incorrect syntax (expected end of string, found end of JSON)."sv }, + { "\"Строительство и ремонт/Силовая техника/Зарядные устройства для автомобиль\0k"sv, ResultType::Throw, "JSON: incorrect syntax (expected end of string, found end of JSON)."sv }, + { "\0t"sv, ResultType::Throw, "JSON: expected \", got \0"sv }, + { "\"/Хозтовары/Хранение вещей и организа\xD1\0t"sv, ResultType::Throw, "JSON: incorrect syntax (expected end of string, found end of JSON)."sv }, + { "\"/Хозтовары/Товары для стир\0a"sv, ResultType::Throw, "JSON: incorrect syntax (expected end of string, found end of JSON)."sv }, + { "\"li\0a"sv, ResultType::Throw, "JSON: incorrect syntax (expected end of string, found end of JSON)."sv }, + { "\"/734859/samolet-radioupravlyaemyy-istrebitel-rabotaet-o\0k"sv, ResultType::Throw, "JSON: incorrect syntax (expected end of string, found end of JSON)."sv }, + { "\"/kosmetika-i-parfyum/parfyumeriya/mu\0t"sv, ResultType::Throw, "JSON: incorrect syntax (expected end of string, found end of JSON)."sv }, + { "\"/ko\0\x04"sv, ResultType::Throw, "JSON: begin >= end."sv }, + { ""sv, ResultType::Throw, "JSON: expected \", got \0"sv }, + { "\"/stroitelstvo-i-remont/stroit\0t"sv, ResultType::Throw, "JSON: incorrect syntax (expected end of string, found end of JSON)."sv }, + { "\"/stroitelstvo-i-remont/stroitelnyy-instrument/av\0k"sv, ResultType::Throw, "JSON: incorrect syntax (expected end of string, found end of JSON)."sv }, + { "\"/s\0a"sv, ResultType::Throw, "JSON: incorrect syntax (expected end of string, found end of JSON)."sv }, + { "\"/Строительство и ремонт/Строительный инструмент/Изм\0e"sv, ResultType::Throw, "JSON: incorrect syntax (expected end of string, found end of JSON)."sv }, + { "\"/avto/soputstvuy\0l"sv, ResultType::Throw, "JSON: incorrect syntax (expected end of string, found end of JSON)."sv }, + { "\"/str\0\xD0"sv, ResultType::Throw, "JSON: incorrect syntax (expected end of string, found end of JSON)."sv }, + { "\"Отвертка 2 в 1 \\\"TUNDRA basic\\\" 5х75 мм (+,-) \0\xFF"sv, ResultType::Throw, "JSON: incorrect syntax (expected end of string, found end of JSON)."sv }, + { "\"/stroitelstvo-i-remont/stroitelnyy-instrument/avtoinstrumen\0\0"sv, ResultType::Throw, "JSON: incorrect syntax (expected end of string, found end of JSON)."sv }, + { "\"Мелкая бытовая техника/Мелки\xD0\0\0"sv, ResultType::Throw, "JSON: incorrect syntax (expected end of string, found end of JSON)."sv }, + { "\"Пряжа \\\"Бамбук стрейч\\0\0\0"sv, ResultType::Throw, "JSON: incorrect syntax (expected end of string, found end of JSON)."sv }, + { "\"Карандаш чёрнографитны\xD0\0\xD0"sv, ResultType::Throw, "JSON: incorrect syntax (expected end of string, found end of JSON)."sv }, + { "\"/Творчество/Рукоделие, аппликации/Пряжа и шерсть для \xD0\0l"sv, ResultType::Throw, "JSON: incorrect syntax (expected end of string, found end of JSON)."sv }, + { "\"/1071547/karandash-chernografitnyy-volshebstvo-nv-kruglyy-d-7-2mm-dl-176mm-plast-tuba/\0e"sv, ResultType::Throw, "JSON: incorrect syntax (expected end of string, found end of JSON)."sv }, + { "\"ca\0e"sv, ResultType::Throw, "JSON: incorrect syntax (expected end of string, found end of JSON)."sv }, + { "\"ca\0e"sv, ResultType::Throw, "JSON: incorrect syntax (expected end of string, found end of JSON)."sv }, + { "\"/1165424/chipbord-vyrubnoy-dlya-skrapbukinga-malyshi-mikki-maus-disney-bebi\0t"sv, ResultType::Throw, "JSON: incorrect syntax (expected end of string, found end of JSON)."sv }, + { "\"/posuda/kuhonnye-prinadlezhnosti-i-i\0d"sv, ResultType::Throw, "JSON: incorrect syntax (expected end of string, found end of JSON)."sv }, + { "\"/Канцтовары/Ежедневники и блокн\xD0\0\0"sv, ResultType::Throw, "JSON: incorrect syntax (expected end of string, found end of JSON)."sv }, + { "\"/kanctovary/ezhednevniki-i-blok\0a"sv, ResultType::Throw, "JSON: incorrect syntax (expected end of string, found end of JSON)."sv }, + { "\"Стакан \xD0\0a"sv, ResultType::Throw, "JSON: incorrect syntax (expected end of string, found end of JSON)."sv }, + { "\"Набор бумаги для скрапбукинга \\\"Мои первый годик\\\": Микки Маус, Дисней бэби, 12 листов 29.5 х 29.5 см, 160\0\x80"sv, ResultType::Throw, "JSON: incorrect syntax (expected end of string, found end of JSON)."sv }, + { "\"c\0\0"sv, ResultType::Throw, "JSON: incorrect syntax (expected end of string, found end of JSON)."sv }, + { "\"Органайзер для хранения аксессуаров, \0\0"sv, ResultType::Throw, "JSON: incorrect syntax (expected end of string, found end of JSON)."sv }, + { "\"quantity\00"sv, ResultType::Throw, "JSON: incorrect syntax (expected end of string, found end of JSON)."sv }, + { "\"Сменный блок для тетрадей на кольцах А5, 160 листов клетка, офсет \xE2\x84\0="sv, ResultType::Throw, "JSON: incorrect syntax (expected end of string, found end of JSON)."sv }, + { "\"/Сувениры/Ф\xD0\0 "sv, ResultType::Throw, "JSON: incorrect syntax (expected end of string, found end of JSON)."sv }, + { "\"\0\""sv, ResultType::Return, "\0"sv }, + { "\"\0\x04"sv, ResultType::Throw, "JSON: incorrect syntax (expected end of string, found end of JSON)."sv }, + { "\"va\0\0"sv, ResultType::Throw, "JSON: incorrect syntax (expected end of string, found end of JSON)."sv }, + { "\"ca\0\0"sv, ResultType::Throw, "JSON: incorrect syntax (expected end of string, found end of JSON)."sv }, + { "\"В \0\x04"sv, ResultType::Throw, "JSON: incorrect syntax (expected end of string, found end of JSON)."sv }, + { "\"/letnie-tovary/z\0\x04"sv, ResultType::Throw, "JSON: incorrect syntax (expected end of string, found end of JSON)."sv }, + { "\"Посудомоечная машина Ha\0="sv, ResultType::Throw, "JSON: incorrect syntax (expected end of string, found end of JSON)."sv }, + { "\"Крупная бытов\0\0"sv, ResultType::Throw, "JSON: incorrect syntax (expected end of string, found end of JSON)."sv }, + { "\"Полочная акустическая система Magnat Needl\0\0"sv, ResultType::Throw, "JSON: incorrect syntax (expected end of string, found end of JSON)."sv }, + { "\"brand\00"sv, ResultType::Throw, "JSON: incorrect syntax (expected end of string, found end of JSON)."sv }, + { "\"\0d"sv, ResultType::Throw, "JSON: incorrect syntax (expected end of string, found end of JSON)."sv }, + { "\"pos\0 "sv, ResultType::Throw, "JSON: incorrect syntax (expected end of string, found end of JSON)."sv }, + { "\"c\0o"sv, ResultType::Throw, "JSON: incorrect syntax (expected end of string, found end of JSON)."sv }, + { "\"var\0\0"sv, ResultType::Throw, "JSON: incorrect syntax (expected end of string, found end of JSON)."sv }, + { "\"Телевизоры и видеотехника/Всё для домашних кинотеатр\0="sv, ResultType::Throw, "JSON: incorrect syntax (expected end of string, found end of JSON)."sv }, + { "\"Флеш-диск Transcend JetFlash 620 8GB (TS8GJF62\0\0"sv, ResultType::Throw, "JSON: incorrect syntax (expected end of string, found end of JSON)."sv }, + { "\"Табурет Мег\0\xD0"sv, ResultType::Throw, "JSON: incorrect syntax (expected end of string, found end of JSON)."sv }, + { "\"variant\0\x04"sv, ResultType::Throw, "JSON: incorrect syntax (expected end of string, found end of JSON)."sv }, + { "\"Катал\xD0\0\""sv, ResultType::Return, "Катал\xD0\0"sv }, + { "\"К\0\0"sv, ResultType::Throw, "JSON: incorrect syntax (expected end of string, found end of JSON)."sv }, + { "\"Полочная акустическая система Magnat Needl\0\0"sv, ResultType::Throw, "JSON: incorrect syntax (expected end of string, found end of JSON)."sv }, + { "\"brand\00"sv, ResultType::Throw, "JSON: incorrect syntax (expected end of string, found end of JSON)."sv }, + { "\"\0d"sv, ResultType::Throw, "JSON: incorrect syntax (expected end of string, found end of JSON)."sv }, + { "\"pos\0 "sv, ResultType::Throw, "JSON: incorrect syntax (expected end of string, found end of JSON)."sv }, + { "\"c\0o"sv, ResultType::Throw, "JSON: incorrect syntax (expected end of string, found end of JSON)."sv }, + { "\"17\0o"sv, ResultType::Throw, "JSON: incorrect syntax (expected end of string, found end of JSON)."sv }, + { "\"/igrushki/razvivayusc\0 "sv, ResultType::Throw, "JSON: incorrect syntax (expected end of string, found end of JSON)."sv }, + { "\"Ключница \\\"\0 "sv, ResultType::Throw, "JSON: incorrect syntax (expected end of string, found end of JSON)."sv }, + { "\"/Игр\xD1\0 "sv, ResultType::Throw, "JSON: incorrect syntax (expected end of string, found end of JSON)."sv }, + { "\"/Игрушки/Игрушки для девочек/Игровые модули дл\xD1\0o"sv, ResultType::Throw, "JSON: incorrect syntax (expected end of string, found end of JSON)."sv }, + { "\"Крупная бытовая техника/Стиральные машины/С фронт\xD0\0o"sv, ResultType::Throw, "JSON: incorrect syntax (expected end of string, found end of JSON)."sv }, + { "\0 "sv, ResultType::Throw, "JSON: expected \", got \0"sv }, + { "\"Светодиодная лента SMD3528, 5 м. IP33, 60LED, зеленый, 4,8W/мет\xD1\0 "sv, ResultType::Throw, "JSON: incorrect syntax (expected end of string, found end of JSON)."sv }, + { "\"Сантехника/Мебель для ванных комнат/Стол\0o"sv, ResultType::Throw, "JSON: incorrect syntax (expected end of string, found end of JSON)."sv }, + { "\0o"sv, ResultType::Throw, "JSON: expected \", got \0"sv }, + { "\"/igrushki/konstruktory\0 "sv, ResultType::Throw, "JSON: incorrect syntax (expected end of string, found end of JSON)."sv }, + { "\"/posuda/kuhonnye-prinadlezhnosti-i-instrumenty/kuhonnye-pr\0 "sv, ResultType::Throw, "JSON: incorrect syntax (expected end of string, found end of JSON)."sv }, + { "\"/1290414/komplekt-zhenskiy-dzhemper-plusbryuki-m-254-09-malina-plustemno-siniy-\0o"sv, ResultType::Throw, "JSON: incorrect syntax (expected end of string, found end of JSON)."sv }, + { "\"/Творчество/Рисование/Инструменты и кра\0o"sv, ResultType::Throw, "JSON: incorrect syntax (expected end of string, found end of JSON)."sv }, + { "\"Строительство и ремонт/Силовая техника/Зарядные устройства для автомобильных аккумуляторов/Пуско-зарядные устр\xD0\0o"sv, ResultType::Throw, "JSON: incorrect syntax (expected end of string, found end of JSON)."sv }, + { "\"Строительство и ремонт/Силовая техника/Зарядные устройств\xD0\0 "sv, ResultType::Throw, "JSON: incorrect syntax (expected end of string, found end of JSON)."sv }, + { "\"Строительство и ремонт/Силовая техника/Зарядные устройства для автомобиль\0d"sv, ResultType::Throw, "JSON: incorrect syntax (expected end of string, found end of JSON)."sv }, + { "\0 "sv, ResultType::Throw, "JSON: expected \", got \0"sv }, + { "\"/Хозтовары/Хранение вещей и организа\xD1\0 "sv, ResultType::Throw, "JSON: incorrect syntax (expected end of string, found end of JSON)."sv }, + { "\"/Хозтовары/Товары для стир\0o"sv, ResultType::Throw, "JSON: incorrect syntax (expected end of string, found end of JSON)."sv }, + { "\"li\0o"sv, ResultType::Throw, "JSON: incorrect syntax (expected end of string, found end of JSON)."sv }, + { "\"/igrushki/igrus\0d"sv, ResultType::Throw, "JSON: incorrect syntax (expected end of string, found end of JSON)."sv }, + { "\"/734859/samolet-radioupravlyaemyy-istrebitel-rabotaet-o\0 "sv, ResultType::Throw, "JSON: incorrect syntax (expected end of string, found end of JSON)."sv }, + { "\"/kosmetika-i-parfyum/parfyumeriya/mu\00"sv, ResultType::Throw, "JSON: incorrect syntax (expected end of string, found end of JSON)."sv }, + { "\"/ko\0\0"sv, ResultType::Throw, "JSON: begin >= end."sv }, + { "\"/avto/avtomobilnyy\0\0"sv, ResultType::Throw, "JSON: begin >= end."sv }, + { "\"/stroitelstvo-i-remont/stroit\00"sv, ResultType::Throw, "JSON: incorrect syntax (expected end of string, found end of JSON)."sv }, + { "\"/stroitelstvo-i-remont/stroitelnyy-instrument/av\0 "sv, ResultType::Throw, "JSON: incorrect syntax (expected end of string, found end of JSON)."sv }, + { "\"/s\0d"sv, ResultType::Throw, "JSON: incorrect syntax (expected end of string, found end of JSON)."sv }, + { "\"/Строительство и ремонт/Строительный инструмент/Изм\0o"sv, ResultType::Throw, "JSON: incorrect syntax (expected end of string, found end of JSON)."sv }, + { "\"/avto/soputstvuy\0\""sv, ResultType::Return, "/avto/soputstvuy\0"sv }, + { "\"/str\0k"sv, ResultType::Throw, "JSON: incorrect syntax (expected end of string, found end of JSON)."sv }, + { "\"Отвертка 2 в 1 \\\"TUNDRA basic\\\" 5х75 мм (+,-) \0\xD0"sv, ResultType::Throw, "JSON: incorrect syntax (expected end of string, found end of JSON)."sv }, + { "\"/stroitelstvo-i-remont/stroitelnyy-instrument/avtoinstrumen\0="sv, ResultType::Throw, "JSON: incorrect syntax (expected end of string, found end of JSON)."sv }, + { "\"Чайник электрический Vitesse\0="sv, ResultType::Throw, "JSON: incorrect syntax (expected end of string, found end of JSON)."sv }, + { "\"Мелкая бытовая техника/Мелки\xD0\0\xD0"sv, ResultType::Throw, "JSON: incorrect syntax (expected end of string, found end of JSON)."sv }, + { "\"Пряжа \\\"Бамбук стрейч\\0о"sv, ResultType::Throw, "JSON: incorrect syntax (expected end of string, found end of JSON)."sv }, + { "\"Карандаш чёрнографитны\xD0\0k"sv, ResultType::Throw, "JSON: incorrect syntax (expected end of string, found end of JSON)."sv }, + { "\"/Творчество/Рукоделие, аппликации/Пряжа и шерсть для \xD0\0\""sv, ResultType::Return, "/Творчество/Рукоделие, аппликации/Пряжа и шерсть для \xD0\0"sv }, + { "\"/1071547/karandash-chernografitnyy-volshebstvo-nv-kruglyy-d-7-2mm-dl-176mm-plast-tuba/\0o"sv, ResultType::Throw, "JSON: incorrect syntax (expected end of string, found end of JSON)."sv }, + { "\"ca\0o"sv, ResultType::Throw, "JSON: incorrect syntax (expected end of string, found end of JSON)."sv }, + { "\"/Подаро\0o"sv, ResultType::Throw, "JSON: incorrect syntax (expected end of string, found end of JSON)."sv }, + { "\"Средство для прочис\xD1\0o"sv, ResultType::Throw, "JSON: incorrect syntax (expected end of string, found end of JSON)."sv }, + { "\"i\0o"sv, ResultType::Throw, "JSON: incorrect syntax (expected end of string, found end of JSON)."sv }, + { "\"/p\0\""sv, ResultType::Return, "/p\0"sv }, + { "\"/Сувениры/Магниты, н\xD0\0k"sv, ResultType::Throw, "JSON: incorrect syntax (expected end of string, found end of JSON)."sv }, + { "\"Дерев\xD0\0="sv, ResultType::Throw, "JSON: incorrect syntax (expected end of string, found end of JSON)."sv }, + { "\"/prazdniki/svadba/svadebnaya-c\0\xD0"sv, ResultType::Throw, "JSON: incorrect syntax (expected end of string, found end of JSON)."sv }, + { "\"/Канцт\0d"sv, ResultType::Throw, "JSON: incorrect syntax (expected end of string, found end of JSON)."sv }, + { "\"/Праздники/То\xD0\0 "sv, ResultType::Throw, "JSON: incorrect syntax (expected end of string, found end of JSON)."sv }, + { "\"v\0 "sv, ResultType::Throw, "JSON: incorrect syntax (expected end of string, found end of JSON)."sv }, + { "\"/Косметика \xD0\0d"sv, ResultType::Throw, "JSON: incorrect syntax (expected end of string, found end of JSON)."sv }, + { "\"/Спорт и отдых/Настольные игры/Покер, руле\xD1\0\xD0"sv, ResultType::Throw, "JSON: incorrect syntax (expected end of string, found end of JSON)."sv }, + { "\"categ\0="sv, ResultType::Throw, "JSON: incorrect syntax (expected end of string, found end of JSON)."sv }, + { "\"/retailr\0k"sv, ResultType::Throw, "JSON: incorrect syntax (expected end of string, found end of JSON)."sv }, + { "\"/retailrocket\0k"sv, ResultType::Throw, "JSON: incorrect syntax (expected end of string, found end of JSON)."sv }, + { "\"Ежедневник недат А5 140л кл,ляссе,обл пв\0="sv, ResultType::Throw, "JSON: incorrect syntax (expected end of string, found end of JSON)."sv }, + { "\"/432809/ezhednevnik-organayzer-sredniy-s-remeshkom-na-knopke-v-oblozhke-kalkulyator-kalendar-do-\0\xD0"sv, ResultType::Throw, "JSON: incorrect syntax (expected end of string, found end of JSON)."sv }, + { "\"/1165424/chipbord-vyrubnoy-dlya-skrapbukinga-malyshi-mikki-maus-disney-bebi\0d"sv, ResultType::Throw, "JSON: incorrect syntax (expected end of string, found end of JSON)."sv }, + { "\"/posuda/kuhonnye-prinadlezhnosti-i-i\0 "sv, ResultType::Throw, "JSON: incorrect syntax (expected end of string, found end of JSON)."sv }, + { "\"/Канцтовары/Ежедневники и блокн\xD0\0o"sv, ResultType::Throw, "JSON: incorrect syntax (expected end of string, found end of JSON)."sv }, + { "\"/kanctovary/ezhednevniki-i-blok\00"sv, ResultType::Throw, "JSON: incorrect syntax (expected end of string, found end of JSON)."sv }, + { "\"Стакан \xD0\0\0"sv, ResultType::Throw, "JSON: incorrect syntax (expected end of string, found end of JSON)."sv }, + { "\"Набор бумаги для скрапбукинга \\\"Мои первый годик\\\": Микки Маус, Дисней бэби, 12 листов 29.5 х 29.5 см, 160\0\0"sv, ResultType::Throw, "JSON: incorrect syntax (expected end of string, found end of JSON)."sv }, + { "\"c\0\""sv, ResultType::Return, "c\0"sv }, }; for (auto i : boost::irange(0, 1/*00000*/)) @@ -641,15 +642,14 @@ TEST(JSONSuite, SimpleTest) { try { - JSON j(r.input, r.input + strlen(r.input)); + JSON j(r.input.data(), r.input.data() + r.input.size()); ASSERT_EQ(j.getString(), r.result); - ASSERT_TRUE(r.result_type == ResultType::Return); + ASSERT_EQ(r.result_type, ResultType::Return); } - catch (JSONException & e) + catch (const JSONException &) { - ASSERT_TRUE(r.result_type == ResultType::Throw); - ASSERT_EQ(e.message(), r.result); + ASSERT_EQ(r.result_type, ResultType::Throw); } } } From ffc39574f18cd77c50ab1c1c063aa5c5a199e91e Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Thu, 4 Mar 2021 02:04:01 +0300 Subject: [PATCH 088/544] Removed very old example (for jemalloc performance regression) --- base/common/tests/CMakeLists.txt | 4 --- base/common/tests/allocator.cpp | 47 -------------------------------- 2 files changed, 51 deletions(-) delete mode 100644 base/common/tests/allocator.cpp diff --git a/base/common/tests/CMakeLists.txt b/base/common/tests/CMakeLists.txt index 92be2f67c94..402a924baa9 100644 --- a/base/common/tests/CMakeLists.txt +++ b/base/common/tests/CMakeLists.txt @@ -1,12 +1,8 @@ include (${ClickHouse_SOURCE_DIR}/cmake/add_check.cmake) add_executable (local_date_time_comparison local_date_time_comparison.cpp) -add_executable (realloc-perf allocator.cpp) - -set(PLATFORM_LIBS ${CMAKE_DL_LIBS}) target_link_libraries (local_date_time_comparison PRIVATE common) -target_link_libraries (realloc-perf PRIVATE common) add_check(local_date_time_comparison) add_executable (dump_variable dump_variable.cpp) diff --git a/base/common/tests/allocator.cpp b/base/common/tests/allocator.cpp deleted file mode 100644 index 03f6228e0f5..00000000000 --- a/base/common/tests/allocator.cpp +++ /dev/null @@ -1,47 +0,0 @@ -#include -#include -#include -#include - - -void thread_func() -{ - for (size_t i = 0; i < 100; ++i) - { - size_t size = 4096; - - void * buf = malloc(size); - if (!buf) - abort(); - memset(buf, 0, size); - - while (size < 1048576) - { - size_t next_size = size * 4; - - void * new_buf = realloc(buf, next_size); - if (!new_buf) - abort(); - buf = new_buf; - - memset(reinterpret_cast(buf) + size, 0, next_size - size); - size = next_size; - } - - free(buf); - } -} - - -int main(int, char **) -{ - std::vector threads(16); - for (size_t i = 0; i < 1000; ++i) - { - for (auto & thread : threads) - thread = std::thread(thread_func); - for (auto & thread : threads) - thread.join(); - } - return 0; -} From 1096fc328ad5cb187bc0910272d3604e31da58fa Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Thu, 4 Mar 2021 02:11:27 +0300 Subject: [PATCH 089/544] Make some tests automated --- base/common/tests/CMakeLists.txt | 7 ------- src/CMakeLists.txt | 2 ++ .../Common/tests/gtest_local_date_time_comparison.cpp | 8 ++++---- src/IO/tests/CMakeLists.txt | 2 -- src/Interpreters/tests/CMakeLists.txt | 1 - 5 files changed, 6 insertions(+), 14 deletions(-) rename base/common/tests/local_date_time_comparison.cpp => src/Common/tests/gtest_local_date_time_comparison.cpp (79%) diff --git a/base/common/tests/CMakeLists.txt b/base/common/tests/CMakeLists.txt index 402a924baa9..2a07a94055f 100644 --- a/base/common/tests/CMakeLists.txt +++ b/base/common/tests/CMakeLists.txt @@ -1,9 +1,2 @@ -include (${ClickHouse_SOURCE_DIR}/cmake/add_check.cmake) - -add_executable (local_date_time_comparison local_date_time_comparison.cpp) - -target_link_libraries (local_date_time_comparison PRIVATE common) -add_check(local_date_time_comparison) - add_executable (dump_variable dump_variable.cpp) target_link_libraries (dump_variable PRIVATE clickhouse_common_io) diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index a6a7d280479..82d84e42364 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -458,6 +458,8 @@ endif() dbms_target_link_libraries(PRIVATE _boost_context) +include (${ClickHouse_SOURCE_DIR}/cmake/add_check.cmake) + if (ENABLE_TESTS AND USE_GTEST) macro (grep_gtest_sources BASE_DIR DST_VAR) # Cold match files that are not in tests/ directories diff --git a/base/common/tests/local_date_time_comparison.cpp b/src/Common/tests/gtest_local_date_time_comparison.cpp similarity index 79% rename from base/common/tests/local_date_time_comparison.cpp rename to src/Common/tests/gtest_local_date_time_comparison.cpp index 5492ec31004..f75c2647100 100644 --- a/base/common/tests/local_date_time_comparison.cpp +++ b/src/Common/tests/gtest_local_date_time_comparison.cpp @@ -1,5 +1,6 @@ #include #include +#include #include @@ -16,14 +17,13 @@ void checkComparison() LocalDateTime a("2018-07-18 01:02:03"); LocalDateTime b("2018-07-18 01:02:03"); - if (a != b) - throw std::runtime_error("Test failed"); + EXPECT_EQ(a, b); + EXPECT_FALSE(a != b); } -int main(int, char **) +TEST(LocalDateTime, Comparison) { fillStackWithGarbage(); checkComparison(); - return 0; } diff --git a/src/IO/tests/CMakeLists.txt b/src/IO/tests/CMakeLists.txt index 79800d8339c..bcd0a8bba24 100644 --- a/src/IO/tests/CMakeLists.txt +++ b/src/IO/tests/CMakeLists.txt @@ -39,11 +39,9 @@ target_link_libraries (o_direct_and_dirty_pages PRIVATE clickhouse_common_io) add_executable (hashing_write_buffer hashing_write_buffer.cpp) target_link_libraries (hashing_write_buffer PRIVATE clickhouse_common_io) -add_check(hashing_write_buffer) add_executable (hashing_read_buffer hashing_read_buffer.cpp) target_link_libraries (hashing_read_buffer PRIVATE clickhouse_common_io) -add_check (hashing_read_buffer) add_executable (io_operators io_operators.cpp) target_link_libraries (io_operators PRIVATE clickhouse_common_io) diff --git a/src/Interpreters/tests/CMakeLists.txt b/src/Interpreters/tests/CMakeLists.txt index 1bc9d7fbacb..8905d2fe6e6 100644 --- a/src/Interpreters/tests/CMakeLists.txt +++ b/src/Interpreters/tests/CMakeLists.txt @@ -38,7 +38,6 @@ target_link_libraries (two_level_hash_map PRIVATE dbms) add_executable (in_join_subqueries_preprocessor in_join_subqueries_preprocessor.cpp) target_link_libraries (in_join_subqueries_preprocessor PRIVATE clickhouse_aggregate_functions dbms clickhouse_parsers) -add_check(in_join_subqueries_preprocessor) if (OS_LINUX) add_executable (internal_iotop internal_iotop.cpp) From 3eb3830dd8c409939223f5649c06f30f80de5363 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Thu, 4 Mar 2021 02:13:49 +0300 Subject: [PATCH 090/544] Remove trash --- .../tests/gtest_global_register_functions.h.bak | 17 ----------------- 1 file changed, 17 deletions(-) delete mode 100644 src/Common/tests/gtest_global_register_functions.h.bak diff --git a/src/Common/tests/gtest_global_register_functions.h.bak b/src/Common/tests/gtest_global_register_functions.h.bak deleted file mode 100644 index 197ce5838b9..00000000000 --- a/src/Common/tests/gtest_global_register_functions.h.bak +++ /dev/null @@ -1,17 +0,0 @@ -#include -#include - -struct RegisteredFunctionsState -{ - RegisteredFunctionsState() - { - DB::registerFunctions(); - } - - RegisteredFunctionsState(RegisteredFunctionsState &&) = default; -}; - -inline void tryRegisterFunctions() -{ - static RegisteredFunctionsState registered_functions_state; -} From 78cc70881647aea920bedebb45a74cd2134fd612 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Thu, 4 Mar 2021 02:15:33 +0300 Subject: [PATCH 091/544] Fix bad whitespaces in test --- tests/queries/0_stateless/01691_DateTime64_clamp.sql | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/tests/queries/0_stateless/01691_DateTime64_clamp.sql b/tests/queries/0_stateless/01691_DateTime64_clamp.sql index 958de4edada..c77a66febb3 100644 --- a/tests/queries/0_stateless/01691_DateTime64_clamp.sql +++ b/tests/queries/0_stateless/01691_DateTime64_clamp.sql @@ -11,7 +11,7 @@ SELECT toDateTime64(toFloat32(bitShiftLeft(toUInt64(1),33)), 2, 'Europe/Moscow') SELECT toDateTime64(toFloat64(bitShiftLeft(toUInt64(1),33)), 2, 'Europe/Moscow') FORMAT Null; -- These are outsize of extended range and hence clamped -SELECT toDateTime64(-1 * bitShiftLeft(toUInt64(1),35), 2); -SELECT CAST(-1 * bitShiftLeft(toUInt64(1),35) AS DateTime64); -SELECT CAST(bitShiftLeft(toUInt64(1),35) AS DateTime64); -SELECT toDateTime64(bitShiftLeft(toUInt64(1),35), 2); +SELECT toDateTime64(-1 * bitShiftLeft(toUInt64(1), 35), 2); +SELECT CAST(-1 * bitShiftLeft(toUInt64(1), 35) AS DateTime64); +SELECT CAST(bitShiftLeft(toUInt64(1), 35) AS DateTime64); +SELECT toDateTime64(bitShiftLeft(toUInt64(1), 35), 2); From 4f08539754f4cc4065241fb00d9e4a3e7cefd567 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Thu, 4 Mar 2021 03:01:08 +0300 Subject: [PATCH 092/544] Remove strange code --- base/common/DateLUTImpl.h | 44 ++++++++++++++++++++++++++------------- 1 file changed, 29 insertions(+), 15 deletions(-) diff --git a/base/common/DateLUTImpl.h b/base/common/DateLUTImpl.h index 8c2bbb3262d..4dee7eb5d55 100644 --- a/base/common/DateLUTImpl.h +++ b/base/common/DateLUTImpl.h @@ -109,6 +109,26 @@ private: return LUTIndex{(v / index.toUnderType()) & date_lut_mask}; } + /// Remainder of division in the sense of modular arithmetic: + /// the difference between x and the maximal divisable number not greater than x. + /// Example: -1 % 10 = 9, because -10 is the maximal number not greater than -1 that is divisable by 10. + /// Why do we need this: + /// - because the unix timestamp -1 is 1969-12-31 23:59:59 in UTC. + template + static constexpr inline auto mod(T x, U divisor) + { + /// This is the C++ way of modulo of division: + /// x % y is the number that: (x / y) * y + x % y == x + /// For example, -1 % 10 = -1 + /// Note that both variants are "correct" in the mathematical sense. They are just different operations. + auto res = x % divisor; + + if (unlikely(res < 0)) + res += divisor; + + return res; + } + public: /// The order of fields matters for alignment and sizeof. struct Values @@ -368,7 +388,7 @@ public: { const LUTIndex index = findIndex(t); - if (unlikely(index == daynum_offset_epoch || index > DATE_LUT_MAX_DAY_NUM)) + if (unlikely(index > DATE_LUT_MAX_DAY_NUM)) return t + offset_at_start_of_epoch; time_t res = t - lut[index].date; @@ -385,7 +405,7 @@ public: /// If it is overflow case, /// than limit number of hours to avoid insane results like 1970-01-01 89:28:15 - if (unlikely(index == daynum_offset_epoch || index > DATE_LUT_MAX_DAY_NUM)) + if (unlikely(index > DATE_LUT_MAX_DAY_NUM)) return static_cast((t + offset_at_start_of_epoch) / 3600) % 24; time_t time = t - lut[index].date; @@ -399,10 +419,10 @@ public: } /** Calculating offset from UTC in seconds. - * which means Using the same literal time of "t" to get the corresponding timestamp in UTC, - * then subtract the former from the latter to get the offset result. - * The boundaries when meets DST(daylight saving time) change should be handled very carefully. - */ + * which means Using the same literal time of "t" to get the corresponding timestamp in UTC, + * then subtract the former from the latter to get the offset result. + * The boundaries when meets DST(daylight saving time) change should be handled very carefully. + */ inline time_t timezoneOffset(time_t t) const { const LUTIndex index = findIndex(t); @@ -412,6 +432,7 @@ public: /// but we can figure out all the accumulated offsets from 1970-01-01 to that day just by get the whole difference between lut[].date, /// and then, we can directly subtract multiple 86400s to get the real DST offsets for the leap seconds is not considered now. time_t res = (lut[index].date - lut[daynum_offset_epoch].date) % 86400; + /// As so far to know, the maximal DST offset couldn't be more than 2 hours, so after the modulo operation the remainder /// will sits between [-offset --> 0 --> offset] which respectively corresponds to moving clock forward or backward. res = res > 43200 ? (86400 - res) : (0 - res); @@ -423,13 +444,6 @@ public: return res + offset_at_start_of_epoch; } - static inline time_t toSecondsSinceTheDayStart(time_t t) - { - t %= 86400; - t = (t < 0 ? t + 86400 : t); - - return t; - } /** Only for time zones with/when offset from UTC is multiple of five minutes. * This is true for all time zones: right now, all time zones have an offset that is multiple of 15 minutes. @@ -443,13 +457,13 @@ public: */ inline unsigned toSecond(time_t t) const { - return toSecondsSinceTheDayStart(t) % 60; + return mod(t, 60); } inline unsigned toMinute(time_t t) const { if (offset_is_whole_number_of_hours_everytime) - return (toSecondsSinceTheDayStart(t) / 60) % 60; + return mod((t / 60), 60); /// To consider the DST changing situation within this day. /// also make the special timezones with no whole hour offset such as 'Australia/Lord_Howe' been taken into account From 7f1ae506fc3dcd145c3384db9ff065fd91c88419 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Thu, 4 Mar 2021 03:11:40 +0300 Subject: [PATCH 093/544] The code is too complicated --- src/Functions/TransformDateTime64.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Functions/TransformDateTime64.h b/src/Functions/TransformDateTime64.h index 0a5e36cd2bd..e42c3155327 100644 --- a/src/Functions/TransformDateTime64.h +++ b/src/Functions/TransformDateTime64.h @@ -78,7 +78,7 @@ public: } } - template >>> + template >> inline auto execute(const T & t, Args && ... args) const { return wrapped_transform.execute(t, std::forward(args)...); From 9f3760e1e369839e3e0cd73a02e7e8325060323a Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Thu, 4 Mar 2021 03:18:48 +0300 Subject: [PATCH 094/544] Bad filename --- src/Common/tests/{gtest_DateLutImpl.cpp => gtest_DateLUTImpl.cpp} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename src/Common/tests/{gtest_DateLutImpl.cpp => gtest_DateLUTImpl.cpp} (100%) diff --git a/src/Common/tests/gtest_DateLutImpl.cpp b/src/Common/tests/gtest_DateLUTImpl.cpp similarity index 100% rename from src/Common/tests/gtest_DateLutImpl.cpp rename to src/Common/tests/gtest_DateLUTImpl.cpp From 31010624d6e3729dc2715216cc60a905865c90d8 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Thu, 4 Mar 2021 03:35:23 +0300 Subject: [PATCH 095/544] Attempt to simplify code --- base/common/DateLUTImpl.h | 26 +++++++++----------------- 1 file changed, 9 insertions(+), 17 deletions(-) diff --git a/base/common/DateLUTImpl.h b/base/common/DateLUTImpl.h index 4dee7eb5d55..eb34053ca6e 100644 --- a/base/common/DateLUTImpl.h +++ b/base/common/DateLUTImpl.h @@ -8,20 +8,23 @@ #include -#define DATE_LUT_MAX (0xFFFFFFFFU - 86400) -#define DATE_LUT_MAX_DAY_NUM (0xFFFFFFFFU / 86400) -/// Table size is bigger than DATE_LUT_MAX_DAY_NUM to fill all indices within UInt16 range: this allows to remove extra check. -#define DATE_LUT_SIZE 0x20000 #define DATE_LUT_MIN_YEAR 1925 /// 1925 since wast majority of timezones changed to 15-minute aligned offsets somewhere in 1924 or earlier. #define DATE_LUT_MAX_YEAR 2283 /// Last supported year (complete) #define DATE_LUT_YEARS (1 + DATE_LUT_MAX_YEAR - DATE_LUT_MIN_YEAR) /// Number of years in lookup table +#define DATE_LUT_SIZE 0x20000 + +#define DATE_LUT_MAX (0xFFFFFFFFU - 86400) +#define DATE_LUT_MAX_DAY_NUM 0xFFFF + + #if defined(__PPC__) #if !__clang__ #pragma GCC diagnostic ignored "-Wmaybe-uninitialized" #endif #endif + /// Flags for toYearWeek() function. enum class WeekModeFlag : UInt8 { @@ -208,17 +211,14 @@ private: /// UTC offset is from -12 to +14 in all known time zones. This requires checking only three indices. if (t >= lut[guess].date && t < lut[UInt32(guess + 1)].date) - return LUTIndex{guess}; + return LUTIndex(guess); /// Time zones that have offset 0 from UTC do daylight saving time change (if any) /// towards increasing UTC offset (example: British Standard Time). if (t >= lut[UInt32(guess + 1)].date) return LUTIndex(guess + 1); - if (lut[guess - 1].date <= t) - return LUTIndex(guess - 1); - - return LUTIndex(guess - 2); + return LUTIndex(guess - 1); } inline LUTIndex toLUTIndex(DayNum d) const @@ -388,9 +388,6 @@ public: { const LUTIndex index = findIndex(t); - if (unlikely(index > DATE_LUT_MAX_DAY_NUM)) - return t + offset_at_start_of_epoch; - time_t res = t - lut[index].date; if (res >= lut[index].time_at_offset_change()) @@ -403,11 +400,6 @@ public: { const LUTIndex index = findIndex(t); - /// If it is overflow case, - /// than limit number of hours to avoid insane results like 1970-01-01 89:28:15 - if (unlikely(index > DATE_LUT_MAX_DAY_NUM)) - return static_cast((t + offset_at_start_of_epoch) / 3600) % 24; - time_t time = t - lut[index].date; /// Data is cleaned to avoid possibility of underflow. From 6dc21cd09869da972fb9b87388d499fc2973a484 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Thu, 4 Mar 2021 03:48:37 +0300 Subject: [PATCH 096/544] Remove discrepances --- src/Common/tests/gtest_DateLUTImpl.cpp | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/Common/tests/gtest_DateLUTImpl.cpp b/src/Common/tests/gtest_DateLUTImpl.cpp index 3a0da1ee1ee..0cb619e19f7 100644 --- a/src/Common/tests/gtest_DateLUTImpl.cpp +++ b/src/Common/tests/gtest_DateLUTImpl.cpp @@ -200,9 +200,9 @@ TEST(DateLUTTest, TimeValuesAtLeftBoderOfRange) EXPECT_EQ(lut.dateToString(time), "1970-01-01" /*std::string*/); } -TEST(DateLUTTest, TimeValuesAtRightBoderOfRangeOfOLDLut) +TEST(DateLUTTest, TimeValuesAtRightBoderOfRangeOfOldLUT) { - // Value is at the right border of the OLD (small) LUT, and provides meaningful values where OLD LUT would provide garbage. + // Value is at the right border of the old (small) LUT, and provides meaningful values where old LUT would provide garbage. const DateLUTImpl & lut = DateLUT::instance("UTC"); const time_t time = 4294343873; // 2106-01-31T01:17:53 (Sunday) @@ -267,7 +267,7 @@ TEST(DateLUTTest, TimeValuesAtRightBoderOfRangeOfOLDLut) class DateLUTWithTimeZone : public ::testing::TestWithParam {}; -TEST_P(DateLUTWithTimeZone, DISABLED_LoadLut) +TEST_P(DateLUTWithTimeZone, DISABLED_LoadLUT) { // There are some assumptions and assertions about TZ data made in DateLUTImpl which are verified upon loading, // to make sure that those assertions are true for all timezones we are going to load all of them one by one. From b9a8509f79b3aa96e46c05dc6be9f12eea5efbf5 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Thu, 4 Mar 2021 05:18:17 +0300 Subject: [PATCH 097/544] Adjustments --- base/common/DateLUTImpl.h | 5 ++--- src/Common/tests/gtest_DateLUTImpl.cpp | 10 +--------- 2 files changed, 3 insertions(+), 12 deletions(-) diff --git a/base/common/DateLUTImpl.h b/base/common/DateLUTImpl.h index eb34053ca6e..050951b2409 100644 --- a/base/common/DateLUTImpl.h +++ b/base/common/DateLUTImpl.h @@ -402,7 +402,6 @@ public: time_t time = t - lut[index].date; - /// Data is cleaned to avoid possibility of underflow. if (time >= lut[index].time_at_offset_change()) time += lut[index].amount_of_offset_change(); @@ -454,8 +453,8 @@ public: inline unsigned toMinute(time_t t) const { - if (offset_is_whole_number_of_hours_everytime) - return mod((t / 60), 60); + if (offset_is_whole_number_of_hours_everytime && t >= 0) + return (t / 60) % 60; /// To consider the DST changing situation within this day. /// also make the special timezones with no whole hour offset such as 'Australia/Lord_Howe' been taken into account diff --git a/src/Common/tests/gtest_DateLUTImpl.cpp b/src/Common/tests/gtest_DateLUTImpl.cpp index 0cb619e19f7..91c1f40fdcb 100644 --- a/src/Common/tests/gtest_DateLUTImpl.cpp +++ b/src/Common/tests/gtest_DateLUTImpl.cpp @@ -267,7 +267,7 @@ TEST(DateLUTTest, TimeValuesAtRightBoderOfRangeOfOldLUT) class DateLUTWithTimeZone : public ::testing::TestWithParam {}; -TEST_P(DateLUTWithTimeZone, DISABLED_LoadLUT) +TEST_P(DateLUTWithTimeZone, LoadLUT) { // There are some assumptions and assertions about TZ data made in DateLUTImpl which are verified upon loading, // to make sure that those assertions are true for all timezones we are going to load all of them one by one. @@ -317,14 +317,6 @@ TEST_P(DateLUTWithTimeZone, getTimeZone) EXPECT_EQ(GetParam(), lut.getTimeZone()); } -TEST_P(DateLUTWithTimeZone, ZeroTime) -{ - const auto & lut = DateLUT::instance(GetParam()); - - EXPECT_EQ(0, lut.toDayNum(time_t{0})); - EXPECT_EQ(0, lut.toDayNum(DayNum{0})); - EXPECT_EQ(0, lut.toDayNum(ExtendedDayNum{0})); -} // Group of tests for timezones that have or had some time ago an offset which is not multiple of 15 minutes. INSTANTIATE_TEST_SUITE_P(ExoticTimezones, From ed492ccf5dba666e34b60d5ed045bab4e2b42bca Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Thu, 4 Mar 2021 05:36:36 +0300 Subject: [PATCH 098/544] Fix Africa/Juba --- base/common/DateLUTImpl.cpp | 33 +++++++++++++++++++++++---------- 1 file changed, 23 insertions(+), 10 deletions(-) diff --git a/base/common/DateLUTImpl.cpp b/base/common/DateLUTImpl.cpp index bf180acb835..71d17dbab2b 100644 --- a/base/common/DateLUTImpl.cpp +++ b/base/common/DateLUTImpl.cpp @@ -94,9 +94,10 @@ DateLUTImpl::DateLUTImpl(const std::string & time_zone_) values.time_at_offset_change_value = 0; values.amount_of_offset_change_value = 0; - // TODO: this partially ignores fractional pre-epoch offsets, which may cause incorrect toRelativeHourNum() results for some timezones, namelly Europe/Minsk - // when pre-May 2 1924 it had an offset of UTC+1:50, and after it was UTC+2h. - // https://www.timeanddate.com/time/zone/belarus/minsk?syear=1900 + /// TODO: This partially ignores fractional pre-epoch offsets, + /// which may cause incorrect toRelativeHourNum() results for some timezones, namelly Europe/Minsk + /// when pre-May 2 1924 it had an offset of UTC+1:50, and after it was UTC+2h. + /// https://www.timeanddate.com/time/zone/belarus/minsk?syear=1900 if (start_of_day > 0 && start_of_day % 3600) offset_is_whole_number_of_hours_everytime = false; @@ -113,16 +114,28 @@ DateLUTImpl::DateLUTImpl(const std::string & time_zone_) /// Find a time (timestamp offset from beginning of day), /// when UTC offset was changed. Search is performed with 15-minute granularity, assuming it is enough. - time_t time_at_offset_change = 900; - while (time_at_offset_change < 86400) + time_t time_at_offset_change = 0; + + /// If offset was changed just at midnight. + if (utc_offset_at_beginning_of_day != cctz_time_zone.lookup( + std::chrono::system_clock::from_time_t(lut[i - 1].date - 1)).offset) { - auto utc_offset_at_current_time = cctz_time_zone.lookup(std::chrono::system_clock::from_time_t( - lut[i - 1].date + time_at_offset_change)).offset; + /// time_at_offset_change is zero. + } + else + { + time_at_offset_change = 900; + while (time_at_offset_change < 86400) + { + auto utc_offset_at_current_time = cctz_time_zone.lookup( + std::chrono::system_clock::from_time_t( + lut[i - 1].date + time_at_offset_change)).offset; - if (utc_offset_at_current_time != utc_offset_at_beginning_of_day) - break; + if (utc_offset_at_current_time != utc_offset_at_beginning_of_day) + break; - time_at_offset_change += 900; + time_at_offset_change += 900; + } } lut[i - 1].time_at_offset_change_value = time_at_offset_change / Values::OffsetChangeFactor; From 9416f5901492c31d4a7ed2c6f9bf88760ffdddea Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Thu, 4 Mar 2021 06:48:24 +0300 Subject: [PATCH 099/544] Some progress on tests --- src/Common/tests/gtest_DateLUTImpl.cpp | 46 ++++++++++++++++++++++---- 1 file changed, 39 insertions(+), 7 deletions(-) diff --git a/src/Common/tests/gtest_DateLUTImpl.cpp b/src/Common/tests/gtest_DateLUTImpl.cpp index 91c1f40fdcb..95ac055e237 100644 --- a/src/Common/tests/gtest_DateLUTImpl.cpp +++ b/src/Common/tests/gtest_DateLUTImpl.cpp @@ -4,8 +4,11 @@ #include #include +#include + #include + /// For the expansion of gtest macros. #if defined(__clang__) #pragma clang diagnostic ignored "-Wused-but-marked-unused" @@ -25,14 +28,17 @@ cctz::civil_day YYYYMMDDToDay(unsigned value) value % 100); // day } -std::vector allTimezones() +std::vector allTimezones(bool with_weird_offsets = true) { std::vector result; const auto * timezone_name = auto_time_zones; while (*timezone_name) { - result.push_back(*timezone_name); + bool weird_offsets = (std::string_view(*timezone_name) == "Africa/Monrovia"); + + if (!weird_offsets || with_weird_offsets) + result.push_back(*timezone_name); ++timezone_name; } @@ -345,7 +351,7 @@ std::ostream & operator<<(std::ostream & ostr, const DateLUTImpl::Values & v) << "\n\t weekday : " << static_cast(v.day_of_week) << "\n\t days in month : " << static_cast(v.days_in_month) << "\n\t offset change : " << v.amount_of_offset_change() - << "\n\t offfset change at : " << v.time_at_offset_change() + << "\n\t offset change at : " << v.time_at_offset_change() << "\n}"; } @@ -387,6 +393,32 @@ TEST_P(DateLUTWithTimeZoneAndTimeRange, InRange) const auto tz_time = cctz::convert(std::chrono::system_clock::from_time_t(expected_time_t), tz); + /// Weird offset, not supported. + /// Example: Africa/Monrovia has offset UTC-0:44:30 in year 1970. + if (tz.lookup(std::chrono::system_clock::from_time_t(expected_time_t)).offset % 900) + continue; + + /// Unsupported timezone transitions - not in 15-minute time point or to different day. + /// Example: America/Goose_Bay decided to go back one hour at 00:01: + /// $ seq 1289097900 30 1289103600 | TZ=America/Goose_Bay LC_ALL=C xargs -I{} date -d @{} + /// Sat Nov 6 23:59:00 ADT 2010 + /// Sat Nov 6 23:59:30 ADT 2010 + /// Sun Nov 7 00:00:00 ADT 2010 + /// Sun Nov 7 00:00:30 ADT 2010 + /// Sat Nov 6 23:01:00 AST 2010 + /// Sat Nov 6 23:01:30 AST 2010 + cctz::time_zone::civil_transition transition{}; + if (tz.next_transition(std::chrono::system_clock::from_time_t(expected_time_t), &transition) + && transition.from.day() == tz_time.day() + && (transition.from.second() != 0 || transition.from.minute() % 900 != 0 + || (transition.from.day() != transition.to.day() + && (transition.from.hour() != 0 && transition.from.minute() != 0 && transition.from.second() != 0)))) + { + std::cerr << "Skipping " << timezone_name << " " << tz_time + << " because of unsupported timezone transition from " << transition.from << " to " << transition.to << "\n"; + continue; + } + EXPECT_EQ(tz_time.year(), lut.toYear(expected_time_t)); EXPECT_EQ(tz_time.month(), lut.toMonth(expected_time_t)); EXPECT_EQ(tz_time.day(), lut.toDayOfMonth(expected_time_t)); @@ -429,7 +461,7 @@ INSTANTIATE_TEST_SUITE_P(DISABLED_AllTimezones_Year2010, INSTANTIATE_TEST_SUITE_P(DISABLED_AllTimezones_Year1970_WHOLE, DateLUTWithTimeZoneAndTimeRange, ::testing::Combine( - ::testing::ValuesIn(allTimezones()), + ::testing::ValuesIn(allTimezones(false)), ::testing::ValuesIn(std::initializer_list{ // Values from tests/date_lut3.cpp {YYYYMMDDToDay(19700101), YYYYMMDDToDay(19701231), 3191 /*53m 11s*/}, @@ -439,7 +471,7 @@ INSTANTIATE_TEST_SUITE_P(DISABLED_AllTimezones_Year1970_WHOLE, INSTANTIATE_TEST_SUITE_P(DISABLED_AllTimezones_Year2010_WHOLE, DateLUTWithTimeZoneAndTimeRange, ::testing::Combine( - ::testing::ValuesIn(allTimezones()), + ::testing::ValuesIn(allTimezones(false)), ::testing::ValuesIn(std::initializer_list{ // Values from tests/date_lut3.cpp {YYYYMMDDToDay(20100101), YYYYMMDDToDay(20101231), 3191 /*53m 11s*/}, @@ -459,7 +491,7 @@ INSTANTIATE_TEST_SUITE_P(DISABLED_AllTimezones_Year2020_WHOLE, INSTANTIATE_TEST_SUITE_P(DISABLED_AllTimezones_PreEpoch, DateLUTWithTimeZoneAndTimeRange, ::testing::Combine( - ::testing::ValuesIn(allTimezones()), + ::testing::ValuesIn(allTimezones(false)), ::testing::ValuesIn(std::initializer_list{ {YYYYMMDDToDay(19500101), YYYYMMDDToDay(19600101), 15 * 60}, {YYYYMMDDToDay(19300101), YYYYMMDDToDay(19350101), 11 * 15 * 60} @@ -469,7 +501,7 @@ INSTANTIATE_TEST_SUITE_P(DISABLED_AllTimezones_PreEpoch, INSTANTIATE_TEST_SUITE_P(DISABLED_AllTimezones_Year1970, DateLUTWithTimeZoneAndTimeRange, ::testing::Combine( - ::testing::ValuesIn(allTimezones()), + ::testing::ValuesIn(allTimezones(false)), ::testing::ValuesIn(std::initializer_list{ {YYYYMMDDToDay(19700101), YYYYMMDDToDay(19700201), 15 * 60}, {YYYYMMDDToDay(19700101), YYYYMMDDToDay(19701231), 11 * 13 * 17} From 66f495b6904c81f53b61e920e1a0f1d5fe4dd097 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Thu, 4 Mar 2021 06:49:46 +0300 Subject: [PATCH 100/544] Fix build --- programs/CMakeLists.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/programs/CMakeLists.txt b/programs/CMakeLists.txt index 9adca58b55a..6b322df5ffd 100644 --- a/programs/CMakeLists.txt +++ b/programs/CMakeLists.txt @@ -325,7 +325,7 @@ else () endif () if (ENABLE_TESTS AND USE_GTEST) - set (CLICKHOUSE_UNIT_TESTS_TARGETS unit_tests_libcommon unit_tests_dbms) + set (CLICKHOUSE_UNIT_TESTS_TARGETS unit_tests_dbms) add_custom_target (clickhouse-tests ALL DEPENDS ${CLICKHOUSE_UNIT_TESTS_TARGETS}) add_dependencies(clickhouse-bundle clickhouse-tests) endif() From 45ca803973d5bd93ecb3ae4a9b809b6b032a7b4b Mon Sep 17 00:00:00 2001 From: Azat Khuzhin Date: Thu, 4 Mar 2021 10:27:41 +0300 Subject: [PATCH 101/544] Fix logging for optimize_aggregation_in_order=1 (with small max_block_size) --- .../Transforms/AggregatingInOrderTransform.cpp | 1 + .../01753_optimize_aggregation_in_order.reference | 1 + .../01753_optimize_aggregation_in_order.sh | 15 +++++++++++++++ 3 files changed, 17 insertions(+) create mode 100644 tests/queries/0_stateless/01753_optimize_aggregation_in_order.reference create mode 100755 tests/queries/0_stateless/01753_optimize_aggregation_in_order.sh diff --git a/src/Processors/Transforms/AggregatingInOrderTransform.cpp b/src/Processors/Transforms/AggregatingInOrderTransform.cpp index 0db95bc3b20..98858a986d2 100644 --- a/src/Processors/Transforms/AggregatingInOrderTransform.cpp +++ b/src/Processors/Transforms/AggregatingInOrderTransform.cpp @@ -143,6 +143,7 @@ void AggregatingInOrderTransform::consume(Chunk chunk) source_column = source_column->cut(key_begin, rows - key_begin); current_chunk = Chunk(source_columns, rows - key_begin); + src_rows -= current_chunk.getNumRows(); block_end_reached = true; need_generate = true; cur_block_size = 0; diff --git a/tests/queries/0_stateless/01753_optimize_aggregation_in_order.reference b/tests/queries/0_stateless/01753_optimize_aggregation_in_order.reference new file mode 100644 index 00000000000..0cfbf08886f --- /dev/null +++ b/tests/queries/0_stateless/01753_optimize_aggregation_in_order.reference @@ -0,0 +1 @@ +2 diff --git a/tests/queries/0_stateless/01753_optimize_aggregation_in_order.sh b/tests/queries/0_stateless/01753_optimize_aggregation_in_order.sh new file mode 100755 index 00000000000..2a7345f4865 --- /dev/null +++ b/tests/queries/0_stateless/01753_optimize_aggregation_in_order.sh @@ -0,0 +1,15 @@ +#!/usr/bin/env bash + +CLICKHOUSE_CLIENT_SERVER_LOGS_LEVEL=trace + +CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh +. "$CUR_DIR"/../shell_config.sh + +$CLICKHOUSE_CLIENT --optimize_aggregation_in_order=1 -nm -q " +drop table if exists data_01753; +create table data_01753 (key Int) engine=MergeTree() order by key as select * from numbers(8); +select * from data_01753 group by key settings max_block_size=1; +select * from data_01753 group by key settings max_block_size=1; +drop table data_01753; +" |& grep -F -c 'AggregatingInOrderTransform: Aggregated. 8 to 8 rows' From 7cc471f9b90c2adc6c32bb339d39c5ec9da5df7f Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Fri, 5 Mar 2021 04:22:12 +0300 Subject: [PATCH 102/544] Simplify code --- base/common/DateLUTImpl.cpp | 65 ++++++++++++++----------------------- 1 file changed, 24 insertions(+), 41 deletions(-) diff --git a/base/common/DateLUTImpl.cpp b/base/common/DateLUTImpl.cpp index 71d17dbab2b..e2bf60fff4e 100644 --- a/base/common/DateLUTImpl.cpp +++ b/base/common/DateLUTImpl.cpp @@ -69,7 +69,8 @@ DateLUTImpl::DateLUTImpl(const std::string & time_zone_) { cctz::time_zone::civil_lookup lookup = cctz_time_zone.lookup(date); - start_of_day = std::chrono::system_clock::to_time_t(lookup.pre); /// Ambiguity is possible. + /// Ambiguity is possible. + start_of_day = std::chrono::system_clock::to_time_t(lookup.pre); Values & values = lut[i]; values.year = date.year(); @@ -94,56 +95,38 @@ DateLUTImpl::DateLUTImpl(const std::string & time_zone_) values.time_at_offset_change_value = 0; values.amount_of_offset_change_value = 0; - /// TODO: This partially ignores fractional pre-epoch offsets, + /// TODO: This partially ignores fractional offsets, /// which may cause incorrect toRelativeHourNum() results for some timezones, namelly Europe/Minsk /// when pre-May 2 1924 it had an offset of UTC+1:50, and after it was UTC+2h. /// https://www.timeanddate.com/time/zone/belarus/minsk?syear=1900 if (start_of_day > 0 && start_of_day % 3600) offset_is_whole_number_of_hours_everytime = false; - /// If UTC offset was changed in previous day. - if (i != 0) + /// If UTC offset was changed this day. + /// Change in time zone without transition is possible, e.g. Moscow 1991 Sun, 31 Mar, 02:00 MSK to EEST + cctz::time_zone::civil_transition transition{}; + if (cctz_time_zone.next_transition(lookup.pre, &transition) + && transition.from.year() == date.year() + && transition.from.month() == date.month() + && transition.from.day() == date.day() + && transition.from != transition.to) { - auto amount_of_offset_change_at_prev_day = 86400 - (lut[i].date - lut[i - 1].date); - if (amount_of_offset_change_at_prev_day) - { - lut[i - 1].amount_of_offset_change_value = amount_of_offset_change_at_prev_day / Values::OffsetChangeFactor; + values.time_at_offset_change_value = (transition.from - cctz::civil_second(date)) / Values::OffsetChangeFactor; + values.amount_of_offset_change_value = (transition.to - transition.from) / Values::OffsetChangeFactor; - const auto utc_offset_at_beginning_of_day = cctz_time_zone.lookup(std::chrono::system_clock::from_time_t(lut[i - 1].date)).offset; +// std::cerr << time_zone << ", " << date << ": change from " << transition.from << " to " << transition.to << "\n"; +// std::cerr << time_zone << ", " << date << ": change at " << values.time_at_offset_change() << " with " << values.amount_of_offset_change() << "\n"; - /// Find a time (timestamp offset from beginning of day), - /// when UTC offset was changed. Search is performed with 15-minute granularity, assuming it is enough. + /// We don't support too large changes. + if (values.amount_of_offset_change_value > 24 * 4) + values.amount_of_offset_change_value = 24 * 4; + else if (values.amount_of_offset_change_value < -24 * 4) + values.amount_of_offset_change_value = -24 * 4; - time_t time_at_offset_change = 0; - - /// If offset was changed just at midnight. - if (utc_offset_at_beginning_of_day != cctz_time_zone.lookup( - std::chrono::system_clock::from_time_t(lut[i - 1].date - 1)).offset) - { - /// time_at_offset_change is zero. - } - else - { - time_at_offset_change = 900; - while (time_at_offset_change < 86400) - { - auto utc_offset_at_current_time = cctz_time_zone.lookup( - std::chrono::system_clock::from_time_t( - lut[i - 1].date + time_at_offset_change)).offset; - - if (utc_offset_at_current_time != utc_offset_at_beginning_of_day) - break; - - time_at_offset_change += 900; - } - } - - lut[i - 1].time_at_offset_change_value = time_at_offset_change / Values::OffsetChangeFactor; - - /// We don't support cases when time change results in switching to previous day. - if (static_cast(lut[i - 1].time_at_offset_change()) + static_cast(lut[i - 1].amount_of_offset_change()) < 0) - lut[i - 1].time_at_offset_change_value = -lut[i - 1].amount_of_offset_change_value; - } + /// We don't support cases when time change results in switching to previous day. + /// Shift the point of time change later. + if (values.time_at_offset_change_value + values.amount_of_offset_change_value < 0) + values.time_at_offset_change_value = -values.amount_of_offset_change_value; } /// Going to next day. From 1722978c2b05feb15a1ae35e902d4ac31af70a92 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Fri, 5 Mar 2021 04:45:12 +0300 Subject: [PATCH 103/544] Comments and corrections --- base/common/DateLUTImpl.cpp | 9 ++++++--- base/common/DateLUTImpl.h | 40 +++++++++++++++++++++++++++---------- 2 files changed, 35 insertions(+), 14 deletions(-) diff --git a/base/common/DateLUTImpl.cpp b/base/common/DateLUTImpl.cpp index e2bf60fff4e..0685b37365b 100644 --- a/base/common/DateLUTImpl.cpp +++ b/base/common/DateLUTImpl.cpp @@ -69,8 +69,11 @@ DateLUTImpl::DateLUTImpl(const std::string & time_zone_) { cctz::time_zone::civil_lookup lookup = cctz_time_zone.lookup(date); - /// Ambiguity is possible. - start_of_day = std::chrono::system_clock::to_time_t(lookup.pre); + /// Ambiguity is possible if time was changed backwards at the midnight + /// (or after midnight time has been changed to the previous day, for example two hours backwards at 01:00). + /// Then midnight appears twice. Usually time change happens exactly at 00:00. + /// Then we should use the second midnight as the start of the day. + start_of_day = std::chrono::system_clock::to_time_t(lookup.post); Values & values = lut[i]; values.year = date.year(); @@ -105,7 +108,7 @@ DateLUTImpl::DateLUTImpl(const std::string & time_zone_) /// If UTC offset was changed this day. /// Change in time zone without transition is possible, e.g. Moscow 1991 Sun, 31 Mar, 02:00 MSK to EEST cctz::time_zone::civil_transition transition{}; - if (cctz_time_zone.next_transition(lookup.pre, &transition) + if (cctz_time_zone.next_transition(lookup.post, &transition) && transition.from.year() == date.year() && transition.from.month() == date.month() && transition.from.day() == date.day() diff --git a/base/common/DateLUTImpl.h b/base/common/DateLUTImpl.h index 050951b2409..d798cffab63 100644 --- a/base/common/DateLUTImpl.h +++ b/base/common/DateLUTImpl.h @@ -406,7 +406,9 @@ public: time += lut[index].amount_of_offset_change(); unsigned res = time / 3600; - return res <= 23 ? res : 0; + + /// In case time was changed backwards at the start of next day, we will repeat the hour 23. + return res <= 23 ? res : 23; } /** Calculating offset from UTC in seconds. @@ -456,28 +458,35 @@ public: if (offset_is_whole_number_of_hours_everytime && t >= 0) return (t / 60) % 60; - /// To consider the DST changing situation within this day. - /// also make the special timezones with no whole hour offset such as 'Australia/Lord_Howe' been taken into account - LUTIndex index = findIndex(t); - UInt32 res = t - lut[index].date; - if (lut[index].amount_of_offset_change() != 0 && t >= lut[index].date + lut[index].time_at_offset_change()) - res += lut[index].amount_of_offset_change(); + /// To consider the DST changing situation within this day + /// also make the special timezones with no whole hour offset such as 'Australia/Lord_Howe' been taken into account. - return res / 60 % 60; + LUTIndex index = findIndex(t); + UInt32 time = t - lut[index].date; + + if (time >= lut[index].time_at_offset_change()) + time += lut[index].amount_of_offset_change(); + + return time / 60 % 60; } + /// NOTE: These functions are wrong for negative time_t. + /// NOTE: Assuming timezone offset is a multiple of 15 minutes. inline time_t toStartOfMinute(time_t t) const { return t / 60 * 60; } inline time_t toStartOfFiveMinute(time_t t) const { return t / 300 * 300; } inline time_t toStartOfFifteenMinutes(time_t t) const { return t / 900 * 900; } + + /// NOTE: This most likely wrong for Nepal - it has offset 05:45. Australia/Eucla is also unfortunate. inline time_t toStartOfTenMinutes(time_t t) const { return t / 600 * 600; } + /// NOTE: Assuming timezone transitions are multiple of hours. Lord Howe Island in Australia is a notable exception. inline time_t toStartOfHour(time_t t) const { - if (offset_is_whole_number_of_hours_everytime) + if (offset_is_whole_number_of_hours_everytime && t >= 0) return t / 3600 * 3600; - UInt32 date = find(t).date; - return date + (UInt32(t) - date) / 3600 * 3600; + Int64 date = find(t).date; + return date + (t - date) / 3600 * 3600; } /** Number of calendar day since the beginning of UNIX epoch (1970-01-01 is zero) @@ -757,6 +766,7 @@ public: /// We count all hour-length intervals, unrelated to offset changes. inline time_t toRelativeHourNum(time_t t) const { + /// NOTE: This is also wrong for negative time_t. if (offset_is_whole_number_of_hours_everytime) return t / 3600; @@ -771,6 +781,7 @@ public: return toRelativeHourNum(lut[toLUTIndex(v)].date); } + /// NOTE: This is wrong for negative time_t. inline time_t toRelativeMinuteNum(time_t t) const { return t / 60; @@ -829,7 +840,10 @@ public: if (hours == 1) return toStartOfHour(t); UInt64 seconds = hours * 3600; + + /// NOTE: This is wrong for negative time_t. t = t / seconds * seconds; + if (offset_is_whole_number_of_hours_everytime) return t; return toStartOfHour(t); @@ -840,6 +854,8 @@ public: if (minutes == 1) return toStartOfMinute(t); UInt64 seconds = 60 * minutes; + + /// NOTE: This is wrong for negative time_t. return t / seconds * seconds; } @@ -847,6 +863,8 @@ public: { if (seconds == 1) return t; + + /// NOTE: This is wrong for negative time_t. return t / seconds * seconds; } From 790aa8697ce48257d913802f698ca886da11ec7b Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Fri, 5 Mar 2021 04:47:50 +0300 Subject: [PATCH 104/544] Fix typos --- base/common/DateLUTImpl.h | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/base/common/DateLUTImpl.h b/base/common/DateLUTImpl.h index d798cffab63..ec47fea60f6 100644 --- a/base/common/DateLUTImpl.h +++ b/base/common/DateLUTImpl.h @@ -113,8 +113,8 @@ private: } /// Remainder of division in the sense of modular arithmetic: - /// the difference between x and the maximal divisable number not greater than x. - /// Example: -1 % 10 = 9, because -10 is the maximal number not greater than -1 that is divisable by 10. + /// the difference between x and the maximal divisible number not greater than x. + /// Example: -1 % 10 = 9, because -10 is the maximal number not greater than -1 that is divisible by 10. /// Why do we need this: /// - because the unix timestamp -1 is 1969-12-31 23:59:59 in UTC. template From ecee81b714ec212b7a4c61c8dbb133d89ed9165c Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Fri, 5 Mar 2021 04:50:16 +0300 Subject: [PATCH 105/544] Fix clang-tidy --- base/common/DateLUTImpl.cpp | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/base/common/DateLUTImpl.cpp b/base/common/DateLUTImpl.cpp index 0685b37365b..46fc88aeb4d 100644 --- a/base/common/DateLUTImpl.cpp +++ b/base/common/DateLUTImpl.cpp @@ -46,17 +46,16 @@ DateLUTImpl::DateLUTImpl(const std::string & time_zone_) if (&inside_main) assert(inside_main); - cctz::time_zone cctz_time_zone; if (!cctz::load_time_zone(time_zone, &cctz_time_zone)) throw Poco::Exception("Cannot load time zone " + time_zone_); - const cctz::civil_day epoch{1970, 1, 1}; - const cctz::civil_day lut_start{DATE_LUT_MIN_YEAR, 1, 1}; + constexpr cctz::civil_day epoch{1970, 1, 1}; + constexpr cctz::civil_day lut_start{DATE_LUT_MIN_YEAR, 1, 1}; time_t start_of_day; - // Note: it's validated against all timezones in the system. - assert((epoch - lut_start) == daynum_offset_epoch); + /// Note: it's validated against all timezones in the system. + static_assert((epoch - lut_start) == daynum_offset_epoch); offset_at_start_of_epoch = cctz_time_zone.lookup(cctz_time_zone.lookup(epoch).pre).offset; offset_at_start_of_lut = cctz_time_zone.lookup(cctz_time_zone.lookup(lut_start).pre).offset; From 738d106874de5e61c943bbb022c1053fdbca10b0 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Fri, 5 Mar 2021 05:03:55 +0300 Subject: [PATCH 106/544] Support negative time_t in more functions --- base/common/DateLUTImpl.h | 60 +++++++++++++-------------------------- 1 file changed, 19 insertions(+), 41 deletions(-) diff --git a/base/common/DateLUTImpl.h b/base/common/DateLUTImpl.h index ec47fea60f6..8da45fa6622 100644 --- a/base/common/DateLUTImpl.h +++ b/base/common/DateLUTImpl.h @@ -17,6 +17,10 @@ #define DATE_LUT_MAX (0xFFFFFFFFU - 86400) #define DATE_LUT_MAX_DAY_NUM 0xFFFF +/// A constant to add to time_t so every supported time point becomes non-negative and still has the same remainder of division by 3600. +/// If we treat "remainder of division" operation in the sense of modular arithmetic (not like in C++). +#define DATE_LUT_ADD (1970 - DATE_LUT_MIN_YEAR) * 366 * 86400 + #if defined(__PPC__) #if !__clang__ @@ -112,26 +116,6 @@ private: return LUTIndex{(v / index.toUnderType()) & date_lut_mask}; } - /// Remainder of division in the sense of modular arithmetic: - /// the difference between x and the maximal divisible number not greater than x. - /// Example: -1 % 10 = 9, because -10 is the maximal number not greater than -1 that is divisible by 10. - /// Why do we need this: - /// - because the unix timestamp -1 is 1969-12-31 23:59:59 in UTC. - template - static constexpr inline auto mod(T x, U divisor) - { - /// This is the C++ way of modulo of division: - /// x % y is the number that: (x / y) * y + x % y == x - /// For example, -1 % 10 = -1 - /// Note that both variants are "correct" in the mathematical sense. They are just different operations. - auto res = x % divisor; - - if (unlikely(res < 0)) - res += divisor; - - return res; - } - public: /// The order of fields matters for alignment and sizeof. struct Values @@ -450,13 +434,13 @@ public: */ inline unsigned toSecond(time_t t) const { - return mod(t, 60); + return (t + DATE_LUT_ADD) % 60; } inline unsigned toMinute(time_t t) const { - if (offset_is_whole_number_of_hours_everytime && t >= 0) - return (t / 60) % 60; + if (offset_is_whole_number_of_hours_everytime) + return ((t + DATE_LUT_ADD) / 60) % 60; /// To consider the DST changing situation within this day /// also make the special timezones with no whole hour offset such as 'Australia/Lord_Howe' been taken into account. @@ -470,20 +454,19 @@ public: return time / 60 % 60; } - /// NOTE: These functions are wrong for negative time_t. /// NOTE: Assuming timezone offset is a multiple of 15 minutes. - inline time_t toStartOfMinute(time_t t) const { return t / 60 * 60; } - inline time_t toStartOfFiveMinute(time_t t) const { return t / 300 * 300; } - inline time_t toStartOfFifteenMinutes(time_t t) const { return t / 900 * 900; } + inline time_t toStartOfMinute(time_t t) const { return (t + DATE_LUT_ADD) / 60 * 60 - DATE_LUT_ADD; } + inline time_t toStartOfFiveMinute(time_t t) const { return (t + DATE_LUT_ADD) / 300 * 300 - DATE_LUT_ADD; } + inline time_t toStartOfFifteenMinutes(time_t t) const { return (t + DATE_LUT_ADD) / 900 * 900 - DATE_LUT_ADD; } /// NOTE: This most likely wrong for Nepal - it has offset 05:45. Australia/Eucla is also unfortunate. - inline time_t toStartOfTenMinutes(time_t t) const { return t / 600 * 600; } + inline time_t toStartOfTenMinutes(time_t t) const { return (t + DATE_LUT_ADD) / 600 * 600 - DATE_LUT_ADD; } /// NOTE: Assuming timezone transitions are multiple of hours. Lord Howe Island in Australia is a notable exception. inline time_t toStartOfHour(time_t t) const { - if (offset_is_whole_number_of_hours_everytime && t >= 0) - return t / 3600 * 3600; + if (offset_is_whole_number_of_hours_everytime) + return (t + DATE_LUT_ADD) / 3600 * 3600 - DATE_LUT_ADD; Int64 date = find(t).date; return date + (t - date) / 3600 * 3600; @@ -766,13 +749,12 @@ public: /// We count all hour-length intervals, unrelated to offset changes. inline time_t toRelativeHourNum(time_t t) const { - /// NOTE: This is also wrong for negative time_t. if (offset_is_whole_number_of_hours_everytime) - return t / 3600; + return (t + DATE_LUT_ADD) / 3600 - (DATE_LUT_ADD / 3600); /// Assume that if offset was fractional, then the fraction is the same as at the beginning of epoch. /// NOTE This assumption is false for "Pacific/Pitcairn" and "Pacific/Kiritimati" time zones. - return (t + 86400 - offset_at_start_of_epoch) / 3600; + return (t + DATE_LUT_ADD + 86400 - offset_at_start_of_epoch) / 3600 - (DATE_LUT_ADD / 3600); } template @@ -781,10 +763,9 @@ public: return toRelativeHourNum(lut[toLUTIndex(v)].date); } - /// NOTE: This is wrong for negative time_t. inline time_t toRelativeMinuteNum(time_t t) const { - return t / 60; + return (t + DATE_LUT_ADD) / 60 - (DATE_LUT_ADD / 60); } template @@ -841,8 +822,7 @@ public: return toStartOfHour(t); UInt64 seconds = hours * 3600; - /// NOTE: This is wrong for negative time_t. - t = t / seconds * seconds; + t = (t + DATE_LUT_ADD) / seconds * seconds - DATE_LUT_ADD; if (offset_is_whole_number_of_hours_everytime) return t; @@ -855,8 +835,7 @@ public: return toStartOfMinute(t); UInt64 seconds = 60 * minutes; - /// NOTE: This is wrong for negative time_t. - return t / seconds * seconds; + return (t + DATE_LUT_ADD) / seconds * seconds - DATE_LUT_ADD; } inline time_t toStartOfSecondInterval(time_t t, UInt64 seconds) const @@ -864,8 +843,7 @@ public: if (seconds == 1) return t; - /// NOTE: This is wrong for negative time_t. - return t / seconds * seconds; + return (t + DATE_LUT_ADD) / seconds * seconds - DATE_LUT_ADD; } inline LUTIndex makeLUTIndex(Int16 year, UInt8 month, UInt8 day_of_month) const From d276fac135aa2ea2866d867a73ac93657b5ef088 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Fri, 5 Mar 2021 05:04:23 +0300 Subject: [PATCH 107/544] Support negative time_t in more functions --- base/common/DateLUTImpl.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/base/common/DateLUTImpl.h b/base/common/DateLUTImpl.h index 8da45fa6622..2534f20838f 100644 --- a/base/common/DateLUTImpl.h +++ b/base/common/DateLUTImpl.h @@ -19,7 +19,7 @@ /// A constant to add to time_t so every supported time point becomes non-negative and still has the same remainder of division by 3600. /// If we treat "remainder of division" operation in the sense of modular arithmetic (not like in C++). -#define DATE_LUT_ADD (1970 - DATE_LUT_MIN_YEAR) * 366 * 86400 +#define DATE_LUT_ADD ((1970 - DATE_LUT_MIN_YEAR) * 366 * 86400) #if defined(__PPC__) From 4a26e5cd43c1950246b840fc38592ddf7413d8f1 Mon Sep 17 00:00:00 2001 From: Vasily Nemkov Date: Fri, 5 Mar 2021 11:49:45 +0200 Subject: [PATCH 108/544] Fixed tests + added DROP TABLE to see if it shows up in logs --- tests/queries/0_stateless/01702_system_query_log.reference | 3 ++- tests/queries/0_stateless/01702_system_query_log.sql | 3 ++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/tests/queries/0_stateless/01702_system_query_log.reference b/tests/queries/0_stateless/01702_system_query_log.reference index 763571fc946..aba0152fcb5 100644 --- a/tests/queries/0_stateless/01702_system_query_log.reference +++ b/tests/queries/0_stateless/01702_system_query_log.reference @@ -34,12 +34,13 @@ Create CREATE TABLE sqllt.table\n(\n i UInt8, s String\n)\nENGINE = MergeTree Create CREATE VIEW sqllt.view AS SELECT i, s FROM sqllt.table; DESCRIBE TABLE sqllt.table FORMAT Null; Drop DETACH TABLE sqllt.table; -Drop DROP DATABASE IF EXISTS sqllt; +Drop DROP DATABASE IF EXISTS sqllt SYNC; DROP POLICY IF EXISTS sqllt_policy ON sqllt.table, sqllt.view, sqllt.dictionary; DROP QUOTA IF EXISTS sqllt_quota; DROP ROLE IF EXISTS sqllt_role; DROP ROW POLICY IF EXISTS sqllt_row_policy ON sqllt.table, sqllt.view, sqllt.dictionary; DROP SETTINGS PROFILE IF EXISTS sqllt_settings_profile; +Drop DROP TABLE sqllt.table SYNC; DROP USER IF EXISTS sqllt_user; Grant GRANT DROP ON sqllt.view TO sqllt_user; Grant GRANT SELECT ON sqllt.table TO sqllt_user; diff --git a/tests/queries/0_stateless/01702_system_query_log.sql b/tests/queries/0_stateless/01702_system_query_log.sql index bdf797865bf..979705c2739 100644 --- a/tests/queries/0_stateless/01702_system_query_log.sql +++ b/tests/queries/0_stateless/01702_system_query_log.sql @@ -2,7 +2,7 @@ SET log_comment='system.query_log logging test'; SELECT 'DROP queries and also a cleanup before the test'; -DROP DATABASE IF EXISTS sqllt; +DROP DATABASE IF EXISTS sqllt SYNC; DROP USER IF EXISTS sqllt_user; DROP ROLE IF EXISTS sqllt_role; DROP POLICY IF EXISTS sqllt_policy ON sqllt.table, sqllt.view, sqllt.dictionary; @@ -124,6 +124,7 @@ ATTACH TABLE sqllt.table; RENAME TABLE sqllt.table TO sqllt.table_new; RENAME TABLE sqllt.table_new TO sqllt.table; TRUNCATE TABLE sqllt.table; +DROP TABLE sqllt.table SYNC; --------------------------------------------------------------------------------------------------- -- Now get all logs related to this test From 9ac39dcda9b1b6877634695f67cd7c3995e20dac Mon Sep 17 00:00:00 2001 From: gyuton <40863448+gyuton@users.noreply.github.com> Date: Fri, 5 Mar 2021 15:36:19 +0300 Subject: [PATCH 109/544] Update docs/en/sql-reference/statements/detach.md Co-authored-by: olgarev <56617294+olgarev@users.noreply.github.com> --- docs/en/sql-reference/statements/detach.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/en/sql-reference/statements/detach.md b/docs/en/sql-reference/statements/detach.md index adb2df570d7..8a1a14ee007 100644 --- a/docs/en/sql-reference/statements/detach.md +++ b/docs/en/sql-reference/statements/detach.md @@ -10,7 +10,7 @@ Deletes information about the `name` table from the server. The server stops kno Syntax: ``` sql -DETACH TABLE [IF EXISTS] [db.]name [ON CLUSTER cluster] +DETACH TABLE|VIEW [IF EXISTS] [db.]name [PERMANENTLY] [ON CLUSTER cluster] ``` This does not delete the table’s data or metadata. On the next server launch, the server will read the metadata and find out about the table again. From 59b16ecb3c5b79133a09788ab83df3374054496d Mon Sep 17 00:00:00 2001 From: gyuton <40863448+gyuton@users.noreply.github.com> Date: Fri, 5 Mar 2021 15:36:31 +0300 Subject: [PATCH 110/544] Update docs/en/sql-reference/statements/detach.md Co-authored-by: olgarev <56617294+olgarev@users.noreply.github.com> --- docs/en/sql-reference/statements/detach.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/en/sql-reference/statements/detach.md b/docs/en/sql-reference/statements/detach.md index 8a1a14ee007..f27f61a933a 100644 --- a/docs/en/sql-reference/statements/detach.md +++ b/docs/en/sql-reference/statements/detach.md @@ -5,7 +5,7 @@ toc_title: DETACH # DETACH Statement {#detach} -Deletes information about the `name` table from the server. The server stops knowing about the table’s existence. +Deletes information about the table or view from the server. The server stops knowing about their existence. Syntax: From e996e2be8eeb6021d6bc056c459b3cc24bce328f Mon Sep 17 00:00:00 2001 From: gyuton <40863448+gyuton@users.noreply.github.com> Date: Fri, 5 Mar 2021 15:37:34 +0300 Subject: [PATCH 111/544] Update docs/en/sql-reference/statements/detach.md Co-authored-by: olgarev <56617294+olgarev@users.noreply.github.com> --- docs/en/sql-reference/statements/detach.md | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/docs/en/sql-reference/statements/detach.md b/docs/en/sql-reference/statements/detach.md index f27f61a933a..0bd4f730364 100644 --- a/docs/en/sql-reference/statements/detach.md +++ b/docs/en/sql-reference/statements/detach.md @@ -13,7 +13,15 @@ Syntax: DETACH TABLE|VIEW [IF EXISTS] [db.]name [PERMANENTLY] [ON CLUSTER cluster] ``` -This does not delete the table’s data or metadata. On the next server launch, the server will read the metadata and find out about the table again. +Detaching does not delete the data or metadata for the table or view. If the table or view was not detached `PERMANENTLY`, on the next server launch the server will read the metadata and recall the table/view again. If the table or view was detached `PERMANENTLY`, there will be no automatic recall. + +Whether the table was detached permanently or not, in both cases you can reattach it using the [ATTACH](../../sql-reference/statements/attach.md) query (with the exception of system tables, which do not have metadata stored for them). + +`ATTACH MATERIALIZED VIEW` doesn't work with short syntax (without `SELECT`), but you can attach it using the `ATTACH TABLE` query. + +Note that you can not detach permanently the table which is already detached (temporary). But you can attach it back and then detach permanently again. + +Also you can not [DROP](../../sql-reference/statements/drop.md#drop-table) the detached table, or [CREATE TABLE](../../sql-reference/statements/create/table.md) with the same name as detached permanently, or replace it with the other table with [RENAME TABLE](../../sql-reference/statements/rename.md) query. Similarly, a “detached” table can be re-attached using the [ATTACH](../../sql-reference/statements/attach.md) query (with the exception of system tables, which do not have metadata stored for them). From aa52c9e2e30d6b6ae9229313818b897d249bf29b Mon Sep 17 00:00:00 2001 From: romanzhukov Date: Fri, 5 Mar 2021 17:58:08 +0300 Subject: [PATCH 112/544] DOCSUP-7100: Add avg translation, argmax query fixes. --- docs/en/sql-reference/aggregate-functions/reference/avg.md | 2 +- docs/ru/sql-reference/aggregate-functions/reference/avg.md | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/en/sql-reference/aggregate-functions/reference/avg.md b/docs/en/sql-reference/aggregate-functions/reference/avg.md index b2dd9ae2cb4..b9aebab2233 100644 --- a/docs/en/sql-reference/aggregate-functions/reference/avg.md +++ b/docs/en/sql-reference/aggregate-functions/reference/avg.md @@ -14,7 +14,7 @@ avg(x) **Arguments** -- `x` — input values, must be [Integer](../../../sql-reference/data-types/int-uint.md), [floating-point](../../../sql-reference/data-types/float.md), or [Decimal](../../../sql-reference/data-types/decimal.md). +- `x` — input values, must be [Integer](../../../sql-reference/data-types/int-uint.md), [Float](../../../sql-reference/data-types/float.md), or [Decimal](../../../sql-reference/data-types/decimal.md). **Returned value** diff --git a/docs/ru/sql-reference/aggregate-functions/reference/avg.md b/docs/ru/sql-reference/aggregate-functions/reference/avg.md index d193d3a7763..7f9ba6c4e6d 100644 --- a/docs/ru/sql-reference/aggregate-functions/reference/avg.md +++ b/docs/ru/sql-reference/aggregate-functions/reference/avg.md @@ -18,7 +18,7 @@ avg(x) **Возвращаемое значение** -- среднее арифметическое, в формате [Float64](../../../sql-reference/data-types/float.md). +- среднее арифметическое, всегда в формате [Float64](../../../sql-reference/data-types/float.md). - `NaN`, если не указано входное значение `x`. **Пример** From d08b4816601b6e693789ca5e4f7425305eaed18a Mon Sep 17 00:00:00 2001 From: Anton Ivashkin Date: Fri, 5 Mar 2021 02:10:20 +0300 Subject: [PATCH 113/544] Fixes by review responces --- S3ZeroCopyReplication.md | 61 ---------------- src/Storages/MergeTree/DataPartsExchange.cpp | 3 - src/Storages/MergeTree/IMergeTreeDataPart.cpp | 2 - src/Storages/MergeTree/MergeTreeData.cpp | 5 +- src/Storages/MergeTree/MergeTreeData.h | 4 +- .../MergeTree/ReplicatedMergeTreeLogEntry.h | 5 -- src/Storages/StorageReplicatedMergeTree.cpp | 70 +++++-------------- src/Storages/StorageReplicatedMergeTree.h | 3 +- 8 files changed, 20 insertions(+), 133 deletions(-) delete mode 100644 S3ZeroCopyReplication.md diff --git a/S3ZeroCopyReplication.md b/S3ZeroCopyReplication.md deleted file mode 100644 index 5230640ebcc..00000000000 --- a/S3ZeroCopyReplication.md +++ /dev/null @@ -1,61 +0,0 @@ -# ClickHouse S3 Zero Copy Replication - -Код просто для теста, не production-ready ни разу. - -[Ветка](https://github.com/ianton-ru/ClickHouse/tree/s3_zero_copy_replication) - -## Как сделано - -При fetch-е парта при репликации в случае, если источник хранит, а приемник собирается хранить парт в S3, вместо данных пересылаются только метаданные S3, приемник кладет их локально себе -и испольузет общие с источником данные на S3. Для того, чтобы не удалить такие пошареные данные, делается пометка в ZooKeeper. - -Введена новая версия протокола REPLICATION_PROTOCOL_VERSION_WITH_PARTS_S3_COPY. В запросе новый параметр send_s3_metadata, если 1, то приемних просит у источника метаданные вместо данных, если это возможно. -Приемник в ответ отсылает куку send_s3_metadata=1 в случае, если идут метаданные. В остальных случаях отсылаются данные, как и прежде. - -В новой версии протокола перед полем с количеством файлов добавлена еще одна строка. Абстрактно это некий ID, по которому ноды могу понять, с одним S3 они работают или с разными. -Практически сейчас это один имя первого объекта файла checksums.txt. Эта же строка используется в качестве ID парта в зукипере. - -Применик перед запросом смотрит, будет ли хранить данные в S3. Проверка сейчас кривая - если в сторадже есть S3, то считаем, что будет S3. -Если да S3, то отсылает в запросе send_s3_metadata=1. - -Источник при получении такого запроса смотрит, лежит ли парт на S3. Если да, то в Зукипере ставит метку по пути `<путь к данным таблицы>/zero_copy_s3/shared/<имя парта>//<Путь парта>/`, -ставит в ответ куку send_s3_metadata=1 и вместо файлов с данными отсылает только файлы метаданных. - -Путь получился сложным, потому что требуется -* по имени парта получить, на каких репликах он уже есть на S3 (нужно для гибридного хранилища) -* по уникальному пути понимать, используелся ли эта копия парта другими репликами -* для павильного времени жизни лока различать лок основного варианта (all_0_0_0) от временного (tmp_fetch_all_0_0_0) - -Приемник при получении ответа с send_s3_metadata=1 проверяет доступность по переданному ключу (первый объект checksums.txt) создает только файлики с идентичными меаданными, которые в итоге будут ссылаться на те же ключи в S3, ставит в зукипере аналогичную метку, -только со своим ID реплики, и работает с этим. - -При желании удалить парт нода удаляет в Зукипере ключ `<путь к данным таблицы>/zero_copy_s3/shared//`, потом получает все подключи `<путь к данным таблицы>/zero_copy_s3/shared/`. -Если список не пустой, то считает, что данные использует другая нода и удаляет только локальные метаданные, если пустой, то удаляет и данные в S3. - -При мерже если реузльтат будет на S3, нода ставит эфемерную метку в Zookeeper по пути `<путь к данным таблицы>/zero_copy_s3/merged/<имя нового парта>` (!! НЕ !!). Если такая метка уже есть, то считает, что другая нода -уже помержила или мержит сейчас, и надо сделать fetch вместо мержа самой. - -В гибридном хранилище если парт переносится на S3, нода через ZK проверяет, нет был ли парт перенесен другой нодой, если был, то делает fetch (модифицированный по сравнению с обычным fetch'ем). - -В конфиг добавлен флаг, по которому включается функционал нового протокола репликации - merge_tree->allow_s3_zero_copy_replication. Сейчас стоит в false. - -## Костыли и недоработки, коих много - -* В качестве ID парта берется имя первого S3-ключа от файла checksums.txt. - -* При удалении класс диска ничего не знает про парты, прокинул флаг, что надо оставлять данные в S3 параметром, это очень криво получилось. - -* Возможна гонка, если источник отошлет метаданные про парт и тут же решит его удалить до того, как приемник поставит в зукипер пометку. - -* В протоколе репликации обмен инфой через параметр запрос в одну сторону и куку в другую мне не нравится, хотя так сделан обмен версиями репликации. - -* При ошибке должно пытаться реплицироваться по старому, но не уверен, всегда ли сработает - -* Не будет обратной совместимости, если образуются такие шареные парты, откатиться на старую версию кликхауса не получится, иначе нода может удалить используемые другой данные. - -* Возможны все же дублирования партов. Пример - нода делает мерж, падает. Другая нода незавимо делает мерж, первая нода поднимается. В итоге есть две копии померженого парта. - -* Тесты пока только самые базовые. - -* Для гибридного хранилища если две ноды решают одновременно перенести парт на S3, обе проверяют, что его там еще нет и обе переносят. - diff --git a/src/Storages/MergeTree/DataPartsExchange.cpp b/src/Storages/MergeTree/DataPartsExchange.cpp index 6a033690912..4c355465ea0 100644 --- a/src/Storages/MergeTree/DataPartsExchange.cpp +++ b/src/Storages/MergeTree/DataPartsExchange.cpp @@ -630,9 +630,6 @@ MergeTreeData::MutableDataPartPtr Fetcher::downloadPartToS3( } } - if (!disk) - throw Exception("Can't find S3 disk", ErrorCodes::S3_ERROR); - static const String TMP_PREFIX = "tmp_fetch_"; String tmp_prefix = tmp_prefix_.empty() ? TMP_PREFIX : tmp_prefix_; diff --git a/src/Storages/MergeTree/IMergeTreeDataPart.cpp b/src/Storages/MergeTree/IMergeTreeDataPart.cpp index dd33761070e..591987404b5 100644 --- a/src/Storages/MergeTree/IMergeTreeDataPart.cpp +++ b/src/Storages/MergeTree/IMergeTreeDataPart.cpp @@ -1014,7 +1014,6 @@ void IMergeTreeDataPart::renameTo(const String & new_relative_path, bool remove_ volume->getDisk()->setLastModified(from, Poco::Timestamp::fromEpochTime(time(nullptr))); volume->getDisk()->moveFile(from, to); - String old_relative_path = relative_path; relative_path = new_relative_path; SyncGuardPtr sync_guard; @@ -1022,7 +1021,6 @@ void IMergeTreeDataPart::renameTo(const String & new_relative_path, bool remove_ sync_guard = volume->getDisk()->getDirectorySyncGuard(to); storage.lockSharedData(*this); - storage.unlockSharedData(*this, old_relative_path); } diff --git a/src/Storages/MergeTree/MergeTreeData.cpp b/src/Storages/MergeTree/MergeTreeData.cpp index 7b310aec707..0c22d5fbc0f 100644 --- a/src/Storages/MergeTree/MergeTreeData.cpp +++ b/src/Storages/MergeTree/MergeTreeData.cpp @@ -1921,8 +1921,7 @@ MergeTreeData::DataPartsVector MergeTreeData::getActivePartsToReplace( const MergeTreePartInfo & new_part_info, const String & new_part_name, DataPartPtr & out_covering_part, - DataPartsLock & /* data_parts_lock */, - bool allow_duplicate) const + DataPartsLock & /* data_parts_lock */) const { /// Parts contained in the part are consecutive in data_parts, intersecting the insertion place for the part itself. auto it_middle = data_parts_by_state_and_info.lower_bound(DataPartStateAndInfo{DataPartState::Committed, new_part_info}); @@ -1956,7 +1955,7 @@ MergeTreeData::DataPartsVector MergeTreeData::getActivePartsToReplace( DataPartIteratorByStateAndInfo end = it_middle; while (end != committed_parts_range.end()) { - if ((*end)->info == new_part_info && !allow_duplicate) + if ((*end)->info == new_part_info) throw Exception("Unexpected duplicate part " + (*end)->getNameWithState() + ". It is a bug.", ErrorCodes::LOGICAL_ERROR); if (!new_part_info.contains((*end)->info)) diff --git a/src/Storages/MergeTree/MergeTreeData.h b/src/Storages/MergeTree/MergeTreeData.h index da5ff283420..679518f8d5d 100644 --- a/src/Storages/MergeTree/MergeTreeData.h +++ b/src/Storages/MergeTree/MergeTreeData.h @@ -750,7 +750,6 @@ public: /// Unlock common S3 data part in zookeeper /// Overridden in StorageReplicatedMergeTree virtual bool unlockSharedData(const IMergeTreeDataPart &) const { return true; } - virtual bool unlockSharedData(const IMergeTreeDataPart &, const String &) const { return true; } /// Fetch part only if some replica has it on shared storage like S3 /// Overridden in StorageReplicatedMergeTree @@ -897,8 +896,7 @@ protected: const MergeTreePartInfo & new_part_info, const String & new_part_name, DataPartPtr & out_covering_part, - DataPartsLock & data_parts_lock, - bool allow_duplicate = false) const; + DataPartsLock & data_parts_lock) const; /// Checks whether the column is in the primary key, possibly wrapped in a chain of functions with single argument. bool isPrimaryOrMinMaxKeyColumnPossiblyWrappedInFunctions(const ASTPtr & node, const StorageMetadataPtr & metadata_snapshot) const; diff --git a/src/Storages/MergeTree/ReplicatedMergeTreeLogEntry.h b/src/Storages/MergeTree/ReplicatedMergeTreeLogEntry.h index d18256f8515..afd8c963943 100644 --- a/src/Storages/MergeTree/ReplicatedMergeTreeLogEntry.h +++ b/src/Storages/MergeTree/ReplicatedMergeTreeLogEntry.h @@ -39,7 +39,6 @@ struct ReplicatedMergeTreeLogEntryData REPLACE_RANGE, /// Drop certain range of partitions and replace them by new ones MUTATE_PART, /// Apply one or several mutations to the part. ALTER_METADATA, /// Apply alter modification according to global /metadata and /columns paths - FETCH_SHARED_PART, /// Get the part from other replica only if it on shared S3 storade }; static String typeToString(Type type) @@ -54,7 +53,6 @@ struct ReplicatedMergeTreeLogEntryData case ReplicatedMergeTreeLogEntryData::REPLACE_RANGE: return "REPLACE_RANGE"; case ReplicatedMergeTreeLogEntryData::MUTATE_PART: return "MUTATE_PART"; case ReplicatedMergeTreeLogEntryData::ALTER_METADATA: return "ALTER_METADATA"; - case ReplicatedMergeTreeLogEntryData::FETCH_SHARED_PART: return "FETCH_SHARED_PART"; default: throw Exception("Unknown log entry type: " + DB::toString(type), ErrorCodes::LOGICAL_ERROR); } @@ -195,9 +193,6 @@ struct ReplicatedMergeTreeLogEntry : public ReplicatedMergeTreeLogEntryData, std std::condition_variable execution_complete; /// Awake when currently_executing becomes false. static Ptr parse(const String & s, const Coordination::Stat & stat); - - DiskPtr disk; - String path; }; using ReplicatedMergeTreeLogEntryPtr = std::shared_ptr; diff --git a/src/Storages/StorageReplicatedMergeTree.cpp b/src/Storages/StorageReplicatedMergeTree.cpp index 3acffdcad9f..1f2bd4f4775 100644 --- a/src/Storages/StorageReplicatedMergeTree.cpp +++ b/src/Storages/StorageReplicatedMergeTree.cpp @@ -1955,16 +1955,15 @@ bool StorageReplicatedMergeTree::executeFetch(LogEntry & entry) } -bool StorageReplicatedMergeTree::executeFetchShared(ReplicatedMergeTreeLogEntry & entry) +bool StorageReplicatedMergeTree::executeFetchShared( + const String & source_replica, + const String & new_part_name, + const DiskPtr & disk, + const String & path) { - if (entry.type != LogEntry::FETCH_SHARED_PART) + if (source_replica.empty()) { - throw Exception("Wrong entry.type in executeFetchShared", ErrorCodes::LOGICAL_ERROR); - } - - if (entry.source_replica.empty()) - { - LOG_INFO(log, "No active replica has part {} on S3.", entry.new_part_name); + LOG_INFO(log, "No active replica has part {} on S3.", new_part_name); return false; } @@ -1992,8 +1991,8 @@ bool StorageReplicatedMergeTree::executeFetchShared(ReplicatedMergeTreeLogEntry try { - if (!fetchPart(entry.new_part_name, metadata_snapshot, zookeeper_path + "/replicas/" + entry.source_replica, false, entry.quorum, - nullptr, true, entry.disk, entry.path)) + if (!fetchPart(new_part_name, metadata_snapshot, zookeeper_path + "/replicas/" + source_replica, false, 0, + nullptr, true, disk, path)) return false; } catch (Exception & e) @@ -6478,10 +6477,8 @@ void StorageReplicatedMergeTree::lockSharedData(const IMergeTreeDataPart & part) String id = part.getUniqueId(); boost::replace_all(id, "/", "_"); - String norm_path = part.relative_path; - boost::replace_all(norm_path, "/", "_"); - String zookeeper_node = zookeeper_path + "/zero_copy_s3/shared/" + part.name + "/" + id + "/" + norm_path + "/" + replica_name; + String zookeeper_node = zookeeper_path + "/zero_copy_s3/shared/" + part.name + "/" + id + "/" + replica_name; LOG_TRACE(log, "Set zookeeper lock {}", zookeeper_node); @@ -6506,12 +6503,6 @@ void StorageReplicatedMergeTree::lockSharedData(const IMergeTreeDataPart & part) bool StorageReplicatedMergeTree::unlockSharedData(const IMergeTreeDataPart & part) const -{ - return unlockSharedData(part, part.relative_path); -} - - -bool StorageReplicatedMergeTree::unlockSharedData(const IMergeTreeDataPart & part, const String & path) const { if (!part.volume) return true; @@ -6527,29 +6518,16 @@ bool StorageReplicatedMergeTree::unlockSharedData(const IMergeTreeDataPart & par String id = part.getUniqueId(); boost::replace_all(id, "/", "_"); - String norm_path = path; - boost::replace_all(norm_path, "/", "_"); String zookeeper_part_node = zookeeper_path + "/zero_copy_s3/shared/" + part.name; String zookeeper_part_uniq_node = zookeeper_part_node + "/" + id; - String zookeeper_part_path_node = zookeeper_part_uniq_node + "/" + norm_path; - String zookeeper_node = zookeeper_part_path_node + "/" + replica_name; + String zookeeper_node = zookeeper_part_uniq_node + "/" + replica_name; LOG_TRACE(log, "Remove zookeeper lock {}", zookeeper_node); zookeeper->tryRemove(zookeeper_node); Strings children; - zookeeper->tryGetChildren(zookeeper_part_path_node, children); - if (!children.empty()) - { - LOG_TRACE(log, "Found zookeper locks for {}", zookeeper_part_path_node); - return false; - } - - zookeeper->tryRemove(zookeeper_part_path_node); - - children.clear(); zookeeper->tryGetChildren(zookeeper_part_uniq_node, children); if (!children.empty()) @@ -6589,18 +6567,10 @@ bool StorageReplicatedMergeTree::tryToFetchIfShared( if (replica.empty()) return false; - ReplicatedMergeTreeLogEntry log_entry; - log_entry.type = ReplicatedMergeTreeLogEntry::FETCH_SHARED_PART; - log_entry.source_replica = replica; - log_entry.new_part_name = part.name; - log_entry.create_time = 0; - log_entry.disk = disk; - log_entry.path = path; - /// TODO: Fix const usage StorageReplicatedMergeTree * replicated_storage_nc = const_cast(this); - return replicated_storage_nc->executeFetchShared(log_entry); + return replicated_storage_nc->executeFetchShared(replica, part.name, disk, path); } @@ -6613,8 +6583,6 @@ String StorageReplicatedMergeTree::getSharedDataReplica( if (!zookeeper) return best_replica; - String norm_path = part.relative_path; - boost::replace_all(norm_path, "/", "_"); String zookeeper_part_node = zookeeper_path + "/zero_copy_s3/shared/" + part.name; Strings ids; @@ -6624,16 +6592,10 @@ String StorageReplicatedMergeTree::getSharedDataReplica( for (const auto & id : ids) { String zookeeper_part_uniq_node = zookeeper_part_node + "/" + id; - Strings paths; - zookeeper->tryGetChildren(zookeeper_part_uniq_node, paths); - for (const auto & path : paths) - { - String zookeeper_node = zookeeper_part_uniq_node + "/" + path; - Strings id_replicas; - zookeeper->tryGetChildren(zookeeper_node, id_replicas); - LOG_TRACE(log, "Found zookeper replicas for {}: {}", zookeeper_node, id_replicas.size()); - replicas.insert(replicas.end(), id_replicas.begin(), id_replicas.end()); - } + Strings id_replicas; + zookeeper->tryGetChildren(zookeeper_part_uniq_node, id_replicas); + LOG_TRACE(log, "Found zookeper replicas for {}: {}", zookeeper_part_uniq_node, id_replicas.size()); + replicas.insert(replicas.end(), id_replicas.begin(), id_replicas.end()); } LOG_TRACE(log, "Found zookeper replicas for part {}: {}", part.name, replicas.size()); diff --git a/src/Storages/StorageReplicatedMergeTree.h b/src/Storages/StorageReplicatedMergeTree.h index 58bedfc0408..5bd10d93c8e 100644 --- a/src/Storages/StorageReplicatedMergeTree.h +++ b/src/Storages/StorageReplicatedMergeTree.h @@ -213,7 +213,7 @@ public: bool canExecuteFetch(const ReplicatedMergeTreeLogEntry & entry, String & disable_reason) const; /// Fetch part only when it stored on shared storage like S3 - bool executeFetchShared(ReplicatedMergeTreeLogEntry & entry); + bool executeFetchShared(const String & source_replica, const String & new_part_name, const DiskPtr & disk, const String & path); /// Lock part in zookeeper for use common S3 data in several nodes void lockSharedData(const IMergeTreeDataPart & part) const override; @@ -222,7 +222,6 @@ public: /// Return true if data unlocked /// Return false if data is still used by another node bool unlockSharedData(const IMergeTreeDataPart & part) const override; - bool unlockSharedData(const IMergeTreeDataPart & part, const String & path) const override; /// Fetch part only if some replica has it on shared storage like S3 bool tryToFetchIfShared(const IMergeTreeDataPart & part, const DiskPtr & disk, const String & path) const override; From 5b3161e0b532f25a0984cb9a98bf6994ce22ceda Mon Sep 17 00:00:00 2001 From: alesapin Date: Fri, 5 Mar 2021 20:24:06 +0300 Subject: [PATCH 114/544] Get rid of const_cast --- src/Storages/MergeTree/IMergeTreeDataPart.cpp | 5 +-- src/Storages/MergeTree/MergeTreeData.h | 2 +- .../MergeTree/MergeTreePartsMover.cpp | 35 ++++++++++++++++--- src/Storages/StorageReplicatedMergeTree.cpp | 7 ++-- src/Storages/StorageReplicatedMergeTree.h | 2 +- 5 files changed, 35 insertions(+), 16 deletions(-) diff --git a/src/Storages/MergeTree/IMergeTreeDataPart.cpp b/src/Storages/MergeTree/IMergeTreeDataPart.cpp index 591987404b5..1f18c894465 100644 --- a/src/Storages/MergeTree/IMergeTreeDataPart.cpp +++ b/src/Storages/MergeTree/IMergeTreeDataPart.cpp @@ -1174,10 +1174,7 @@ void IMergeTreeDataPart::makeCloneOnDisk(const DiskPtr & disk, const String & di disk->removeRecursive(path_to_clone + relative_path + '/'); } disk->createDirectories(path_to_clone); - - bool is_fetched = storage.tryToFetchIfShared(*this, disk, path_to_clone + "/" + name); - if (!is_fetched) - volume->getDisk()->copy(getFullRelativePath(), disk, path_to_clone); + volume->getDisk()->copy(getFullRelativePath(), disk, path_to_clone); volume->getDisk()->removeFileIfExists(path_to_clone + '/' + DELETE_ON_DESTROY_MARKER_FILE_NAME); } diff --git a/src/Storages/MergeTree/MergeTreeData.h b/src/Storages/MergeTree/MergeTreeData.h index 679518f8d5d..1f1505fe552 100644 --- a/src/Storages/MergeTree/MergeTreeData.h +++ b/src/Storages/MergeTree/MergeTreeData.h @@ -753,7 +753,7 @@ public: /// Fetch part only if some replica has it on shared storage like S3 /// Overridden in StorageReplicatedMergeTree - virtual bool tryToFetchIfShared(const IMergeTreeDataPart &, const DiskPtr &, const String &) const { return false; } + virtual bool tryToFetchIfShared(const IMergeTreeDataPart &, const DiskPtr &, const String &) { return false; } protected: diff --git a/src/Storages/MergeTree/MergeTreePartsMover.cpp b/src/Storages/MergeTree/MergeTreePartsMover.cpp index 7b8c88b1bff..41eae7fed38 100644 --- a/src/Storages/MergeTree/MergeTreePartsMover.cpp +++ b/src/Storages/MergeTree/MergeTreePartsMover.cpp @@ -194,15 +194,40 @@ MergeTreeData::DataPartPtr MergeTreePartsMover::clonePart(const MergeTreeMoveEnt if (moves_blocker.isCancelled()) throw Exception("Cancelled moving parts.", ErrorCodes::ABORTED); - LOG_TRACE(log, "Cloning part {}", moving_part.part->name); + auto settings = data->getSettings(); + auto part = moving_part.part; + LOG_TRACE(log, "Cloning part {}", part->name); + + auto disk = moving_part.reserved_space->getDisk(); const String directory_to_move = "moving"; - moving_part.part->makeCloneOnDisk(moving_part.reserved_space->getDisk(), directory_to_move); + if (settings->allow_s3_zero_copy_replication) + { + /// Try to fetch part from S3 without copy and fallback to default copy + /// if it's not possible + moving_part.part->assertOnDisk(); + String path_to_clone = data->getRelativeDataPath() + directory_to_move + '/'; + String relative_path = part->relative_path; + if (disk->exists(path_to_clone + relative_path)) + { + LOG_WARNING(log, "Path " + fullPath(disk, path_to_clone + relative_path) + " already exists. Will remove it and clone again."); + disk->removeRecursive(path_to_clone + relative_path + '/'); + } + disk->createDirectories(path_to_clone); + bool is_fetched = data->tryToFetchIfShared(*part, disk, path_to_clone + "/" + part->name); + if (!is_fetched) + part->volume->getDisk()->copy(data->getRelativeDataPath() + relative_path, disk, path_to_clone); + part->volume->getDisk()->removeFileIfExists(path_to_clone + '/' + IMergeTreeDataPart::DELETE_ON_DESTROY_MARKER_FILE_NAME); + } + else + { + part->makeCloneOnDisk(disk, directory_to_move); + } - auto single_disk_volume = std::make_shared("volume_" + moving_part.part->name, moving_part.reserved_space->getDisk(), 0); + auto single_disk_volume = std::make_shared("volume_" + part->name, moving_part.reserved_space->getDisk(), 0); MergeTreeData::MutableDataPartPtr cloned_part = - data->createPart(moving_part.part->name, single_disk_volume, directory_to_move + '/' + moving_part.part->name); - LOG_TRACE(log, "Part {} was cloned to {}", moving_part.part->name, cloned_part->getFullPath()); + data->createPart(part->name, single_disk_volume, directory_to_move + '/' + part->name); + LOG_TRACE(log, "Part {} was cloned to {}", part->name, cloned_part->getFullPath()); cloned_part->loadColumnsChecksumsIndexes(true, true); return cloned_part; diff --git a/src/Storages/StorageReplicatedMergeTree.cpp b/src/Storages/StorageReplicatedMergeTree.cpp index 1f2bd4f4775..ddc63793640 100644 --- a/src/Storages/StorageReplicatedMergeTree.cpp +++ b/src/Storages/StorageReplicatedMergeTree.cpp @@ -6552,7 +6552,7 @@ bool StorageReplicatedMergeTree::unlockSharedData(const IMergeTreeDataPart & par bool StorageReplicatedMergeTree::tryToFetchIfShared( const IMergeTreeDataPart & part, const DiskPtr & disk, - const String & path) const + const String & path) { const auto data_settings = getSettings(); if (!data_settings->allow_s3_zero_copy_replication) @@ -6567,10 +6567,7 @@ bool StorageReplicatedMergeTree::tryToFetchIfShared( if (replica.empty()) return false; - /// TODO: Fix const usage - StorageReplicatedMergeTree * replicated_storage_nc = const_cast(this); - - return replicated_storage_nc->executeFetchShared(replica, part.name, disk, path); + return executeFetchShared(replica, part.name, disk, path); } diff --git a/src/Storages/StorageReplicatedMergeTree.h b/src/Storages/StorageReplicatedMergeTree.h index 5bd10d93c8e..e3d7e6b2556 100644 --- a/src/Storages/StorageReplicatedMergeTree.h +++ b/src/Storages/StorageReplicatedMergeTree.h @@ -224,7 +224,7 @@ public: bool unlockSharedData(const IMergeTreeDataPart & part) const override; /// Fetch part only if some replica has it on shared storage like S3 - bool tryToFetchIfShared(const IMergeTreeDataPart & part, const DiskPtr & disk, const String & path) const override; + bool tryToFetchIfShared(const IMergeTreeDataPart & part, const DiskPtr & disk, const String & path) override; /// Get best replica having this partition on S3 String getSharedDataReplica(const IMergeTreeDataPart & part) const; From dbae58f30b61657d989920dd0c5d71e0a67d0c67 Mon Sep 17 00:00:00 2001 From: Pavel Kruglov Date: Sat, 6 Mar 2021 02:45:17 +0300 Subject: [PATCH 115/544] Add slowdown_count and show actual information in system.clusters --- src/Client/ConnectionPoolWithFailover.cpp | 11 ++- src/Client/ConnectionPoolWithFailover.h | 1 + src/Client/HedgedConnectionsFactory.cpp | 7 +- src/Common/PoolWithFailoverBase.h | 87 +++++++++++-------- src/Storages/System/StorageSystemClusters.cpp | 2 + .../integration/test_hedged_requests/test.py | 18 ++++ .../__init__.py | 0 .../configs/remote_servers.xml | 14 +++ .../configs/users.xml | 8 ++ .../test.py | 60 +++++++++++++ 10 files changed, 169 insertions(+), 39 deletions(-) create mode 100644 tests/integration/test_system_clusters_actual_information/__init__.py create mode 100644 tests/integration/test_system_clusters_actual_information/configs/remote_servers.xml create mode 100644 tests/integration/test_system_clusters_actual_information/configs/users.xml create mode 100644 tests/integration/test_system_clusters_actual_information/test.py diff --git a/src/Client/ConnectionPoolWithFailover.cpp b/src/Client/ConnectionPoolWithFailover.cpp index 0c61b2bb49d..225dcb02142 100644 --- a/src/Client/ConnectionPoolWithFailover.cpp +++ b/src/Client/ConnectionPoolWithFailover.cpp @@ -110,15 +110,20 @@ ConnectionPoolWithFailover::Status ConnectionPoolWithFailover::getStatus() const ConnectionPoolWithFailover::Status result; result.reserve(states.size()); const time_t since_last_error_decrease = time(nullptr) - error_decrease_time; - + /// Update error_count and slowdown_count in states to return actual information. + auto updated_states = states; + auto updated_error_decrease_time = error_decrease_time; + Base::updateErrorCounts(updated_states, updated_error_decrease_time); for (size_t i = 0; i < states.size(); ++i) { const auto rounds_to_zero_errors = states[i].error_count ? bitScanReverse(states[i].error_count) + 1 : 0; - const auto seconds_to_zero_errors = std::max(static_cast(0), rounds_to_zero_errors * decrease_error_period - since_last_error_decrease); + const auto rounds_to_zero_slowdowns = states[i].slowdown_count ? bitScanReverse(states[i].slowdown_count) + 1 : 0; + const auto seconds_to_zero_errors = std::max(static_cast(0), std::max(rounds_to_zero_errors, rounds_to_zero_slowdowns) * decrease_error_period - since_last_error_decrease); result.emplace_back(NestedPoolStatus{ pools[i], - states[i].error_count, + updated_states[i].error_count, + updated_states[i].slowdown_count, std::chrono::seconds{seconds_to_zero_errors} }); } diff --git a/src/Client/ConnectionPoolWithFailover.h b/src/Client/ConnectionPoolWithFailover.h index 023ef863bdf..ce70c27838b 100644 --- a/src/Client/ConnectionPoolWithFailover.h +++ b/src/Client/ConnectionPoolWithFailover.h @@ -74,6 +74,7 @@ public: { const Base::NestedPoolPtr pool; size_t error_count; + size_t slowdown_count; std::chrono::seconds estimated_recovery_time; }; diff --git a/src/Client/HedgedConnectionsFactory.cpp b/src/Client/HedgedConnectionsFactory.cpp index a7f3bfb04d7..34b8ec3290e 100644 --- a/src/Client/HedgedConnectionsFactory.cpp +++ b/src/Client/HedgedConnectionsFactory.cpp @@ -215,7 +215,11 @@ HedgedConnectionsFactory::State HedgedConnectionsFactory::processEpollEvents(boo return state; } else if (timeout_fd_to_replica_index.contains(event_fd)) - replicas[timeout_fd_to_replica_index[event_fd]].change_replica_timeout.reset(); + { + int index = timeout_fd_to_replica_index[event_fd]; + replicas[index].change_replica_timeout.reset(); + ++shuffled_pools[index].slowdown_count; + } else throw Exception("Unknown event from epoll", ErrorCodes::LOGICAL_ERROR); @@ -285,6 +289,7 @@ HedgedConnectionsFactory::State HedgedConnectionsFactory::processFinishedConnect ProfileEvents::increment(ProfileEvents::DistributedConnectionFailTry); shuffled_pool.error_count = std::min(pool->getMaxErrorCup(), shuffled_pool.error_count + 1); + shuffled_pool.slowdown_count = 0; if (shuffled_pool.error_count >= max_tries) { diff --git a/src/Common/PoolWithFailoverBase.h b/src/Common/PoolWithFailoverBase.h index 6bb6f4a94dd..141ac7a7e49 100644 --- a/src/Common/PoolWithFailoverBase.h +++ b/src/Common/PoolWithFailoverBase.h @@ -103,6 +103,7 @@ public: const PoolState * state{}; size_t index = 0; size_t error_count = 0; + size_t slowdown_count = 0; }; /// This functor must be provided by a client. It must perform a single try that takes a connection @@ -133,6 +134,8 @@ protected: /// This function returns a copy of pool states to avoid race conditions when modifying shared pool states. PoolStates updatePoolStates(size_t max_ignored_errors); + void updateErrorCounts(PoolStates & states, time_t & last_decrease_time) const; + std::vector getShuffledPools(size_t max_ignored_errors, const GetPriorityFunc & get_priority); inline void updateSharedErrorCounts(std::vector & shuffled_pools); @@ -193,6 +196,7 @@ inline void PoolWithFailoverBase::updateSharedErrorCounts(std::vect { auto & pool_state = shared_pool_states[pool.index]; pool_state.error_count = std::min(max_error_cap, pool_state.error_count + pool.error_count); + pool_state.slowdown_count += pool.slowdown_count; } } @@ -332,6 +336,8 @@ template struct PoolWithFailoverBase::PoolState { UInt64 error_count = 0; + /// The number of slowdowns that led to changing replica in HedgedRequestsFactory + UInt64 slowdown_count = 0; /// Priority from the configuration. Int64 config_priority = 1; /// Priority from the GetPriorityFunc. @@ -345,8 +351,8 @@ struct PoolWithFailoverBase::PoolState static bool compare(const PoolState & lhs, const PoolState & rhs) { - return std::forward_as_tuple(lhs.error_count, lhs.config_priority, lhs.priority, lhs.random) - < std::forward_as_tuple(rhs.error_count, rhs.config_priority, rhs.priority, rhs.random); + return std::forward_as_tuple(lhs.error_count, lhs.slowdown_count, lhs.config_priority, lhs.priority, lhs.random) + < std::forward_as_tuple(rhs.error_count, rhs.slowdown_count, rhs.config_priority, rhs.priority, rhs.random); } private: @@ -366,39 +372,7 @@ PoolWithFailoverBase::updatePoolStates(size_t max_ignored_errors) for (auto & state : shared_pool_states) state.randomize(); - time_t current_time = time(nullptr); - - if (last_error_decrease_time) - { - time_t delta = current_time - last_error_decrease_time; - - if (delta >= 0) - { - const UInt64 MAX_BITS = sizeof(UInt64) * CHAR_BIT; - size_t shift_amount = MAX_BITS; - /// Divide error counts by 2 every decrease_error_period seconds. - if (decrease_error_period) - shift_amount = delta / decrease_error_period; - /// Update time but don't do it more often than once a period. - /// Else if the function is called often enough, error count will never decrease. - if (shift_amount) - last_error_decrease_time = current_time; - - if (shift_amount >= MAX_BITS) - { - for (auto & state : shared_pool_states) - state.error_count = 0; - } - else if (shift_amount) - { - for (auto & state : shared_pool_states) - state.error_count >>= shift_amount; - } - } - } - else - last_error_decrease_time = current_time; - + updateErrorCounts(shared_pool_states, last_error_decrease_time); result.assign(shared_pool_states.begin(), shared_pool_states.end()); } @@ -408,3 +382,46 @@ PoolWithFailoverBase::updatePoolStates(size_t max_ignored_errors) return result; } + +template +void PoolWithFailoverBase::updateErrorCounts(PoolWithFailoverBase::PoolStates & states, time_t & last_decrease_time) const +{ + time_t current_time = time(nullptr); + + if (last_decrease_time) + { + time_t delta = current_time - last_decrease_time; + + if (delta >= 0) + { + const UInt64 MAX_BITS = sizeof(UInt64) * CHAR_BIT; + size_t shift_amount = MAX_BITS; + /// Divide error counts by 2 every decrease_error_period seconds. + if (decrease_error_period) + shift_amount = delta / decrease_error_period; + /// Update time but don't do it more often than once a period. + /// Else if the function is called often enough, error count will never decrease. + if (shift_amount) + last_decrease_time = current_time; + + if (shift_amount >= MAX_BITS) + { + for (auto & state : states) + { + state.error_count = 0; + state.slowdown_count = 0; + } + } + else if (shift_amount) + { + for (auto & state : states) + { + state.error_count >>= shift_amount; + state.slowdown_count >>= shift_amount; + } + } + } + } + else + last_decrease_time = current_time; +} diff --git a/src/Storages/System/StorageSystemClusters.cpp b/src/Storages/System/StorageSystemClusters.cpp index 7e16deb6d22..25b432252f9 100644 --- a/src/Storages/System/StorageSystemClusters.cpp +++ b/src/Storages/System/StorageSystemClusters.cpp @@ -23,6 +23,7 @@ NamesAndTypesList StorageSystemClusters::getNamesAndTypes() {"user", std::make_shared()}, {"default_database", std::make_shared()}, {"errors_count", std::make_shared()}, + {"slowdowns_count", std::make_shared()}, {"estimated_recovery_time", std::make_shared()} }; } @@ -71,6 +72,7 @@ void StorageSystemClusters::writeCluster(MutableColumns & res_columns, const Nam res_columns[i++]->insert(address.user); res_columns[i++]->insert(address.default_database); res_columns[i++]->insert(pool_status[replica_index].error_count); + res_columns[i++]->insert(pool_status[replica_index].slowdown_count); res_columns[i++]->insert(pool_status[replica_index].estimated_recovery_time.count()); } } diff --git a/tests/integration/test_hedged_requests/test.py b/tests/integration/test_hedged_requests/test.py index 27fe7905b3a..0c0155ff9a2 100644 --- a/tests/integration/test_hedged_requests/test.py +++ b/tests/integration/test_hedged_requests/test.py @@ -85,11 +85,29 @@ def check_settings(node_name, sleep_in_send_tables_status, sleep_in_send_data): def test_stuck_replica(started_cluster): cluster.pause_container("node_1") + check_query(expected_replica="node_2") + + result = NODES['node'].query("SELECT slowdowns_count FROM system.clusters WHERE cluster='test_cluster' and host_name='node_1'") + + assert TSV(result) == TSV("1") + + result = NODES['node'].query("SELECT hostName(), id FROM distributed ORDER BY id LIMIT 1"); + + assert TSV(result) == TSV("node_2\t0") + + # Check that we didn't choose node_1 first again and slowdowns_count didn't increase. + result = NODES['node'].query("SELECT slowdowns_count FROM system.clusters WHERE cluster='test_cluster' and host_name='node_1'") + + assert TSV(result) == TSV("1") + cluster.unpause_container("node_1") def test_long_query(started_cluster): + # Restart to reset pool states. + NODES['node'].restart_clickhouse() + result = NODES['node'].query("select hostName(), max(id + sleep(1.5)) from distributed settings max_block_size = 1, max_threads = 1;") assert TSV(result) == TSV("node_1\t99") diff --git a/tests/integration/test_system_clusters_actual_information/__init__.py b/tests/integration/test_system_clusters_actual_information/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tests/integration/test_system_clusters_actual_information/configs/remote_servers.xml b/tests/integration/test_system_clusters_actual_information/configs/remote_servers.xml new file mode 100644 index 00000000000..b4f30a87520 --- /dev/null +++ b/tests/integration/test_system_clusters_actual_information/configs/remote_servers.xml @@ -0,0 +1,14 @@ + + + + + true + + node_1 + 9000 + + + + + + diff --git a/tests/integration/test_system_clusters_actual_information/configs/users.xml b/tests/integration/test_system_clusters_actual_information/configs/users.xml new file mode 100644 index 00000000000..156cd3a6b59 --- /dev/null +++ b/tests/integration/test_system_clusters_actual_information/configs/users.xml @@ -0,0 +1,8 @@ + + + + + 5 + + + diff --git a/tests/integration/test_system_clusters_actual_information/test.py b/tests/integration/test_system_clusters_actual_information/test.py new file mode 100644 index 00000000000..afbaf53089f --- /dev/null +++ b/tests/integration/test_system_clusters_actual_information/test.py @@ -0,0 +1,60 @@ +import os +import sys +import time + +import pytest + +sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) + +from helpers.cluster import ClickHouseCluster +from helpers.network import PartitionManager + +cluster = ClickHouseCluster(__file__) +node = cluster.add_instance('node', with_zookeeper=True, main_configs=['configs/remote_servers.xml']) +node_1 = cluster.add_instance('node_1', with_zookeeper=True) + +@pytest.fixture(scope="module") +def started_cluster(): + try: + cluster.start() + + node_1.query('''CREATE TABLE replicated (id UInt32, date Date) ENGINE = + ReplicatedMergeTree('/clickhouse/tables/replicated', 'node_1') ORDER BY id PARTITION BY toYYYYMM(date)''') + + node.query("CREATE TABLE distributed (id UInt32, date Date) ENGINE = Distributed('test_cluster', 'default', 'replicated')") + + yield cluster + + finally: + cluster.shutdown() + + + +def test(started_cluster): + cluster.pause_container("node_1") + + node.query("SYSTEM RELOAD CONFIG") + node.query_and_get_error("SELECT count() FROM distributed SETTINGS receive_timeout=1") + + result = node.query("SELECT errors_count, estimated_recovery_time FROM system.clusters WHERE cluster='test_cluster' and host_name='node_1'") + errors_count, recovery_time = map(int, result.split()) + assert errors_count == 3 + + while True: + time.sleep(1) + + result = node.query("SELECT errors_count, estimated_recovery_time FROM system.clusters WHERE cluster='test_cluster' and host_name='node_1'") + prev_time = recovery_time + errors_count, recovery_time = map(int, result.split()) + + if recovery_time == 0: + break + + assert recovery_time < prev_time + assert errors_count > 0 + + assert recovery_time == 0 + assert errors_count == 0 + + cluster.unpause_container("node_1") + From ff76356fd1c4e807b4ee60fd6b35665604fdd25f Mon Sep 17 00:00:00 2001 From: Olga Revyakina Date: Sat, 6 Mar 2021 11:59:50 +0300 Subject: [PATCH 116/544] Docs en,ru PLUS templates updates --- .../_description_templates/template-engine.md | 2 +- .../template-function.md | 10 ++- .../template-server-setting.md | 4 +- .../template-statement.md | 8 +-- docs/en/operations/system-tables/settings.md | 1 + docs/en/sql-reference/statements/show.md | 69 ++++++++++++++++++- docs/ru/operations/system-tables/settings.md | 1 + docs/ru/sql-reference/statements/show.md | 69 ++++++++++++++++++- 8 files changed, 149 insertions(+), 15 deletions(-) diff --git a/docs/_description_templates/template-engine.md b/docs/_description_templates/template-engine.md index 35181881134..5ace1449de2 100644 --- a/docs/_description_templates/template-engine.md +++ b/docs/_description_templates/template-engine.md @@ -58,6 +58,6 @@ Result: Follow up with any text to clarify the example. -## See Also {#see-also} +**See Also** {#see-also} - [link](#) diff --git a/docs/_description_templates/template-function.md b/docs/_description_templates/template-function.md index a0074a76ef6..f8156ec3f6f 100644 --- a/docs/_description_templates/template-function.md +++ b/docs/_description_templates/template-function.md @@ -14,18 +14,16 @@ More text (Optional). **Arguments** (Optional) -- `x` — Description. [Type name](relative/path/to/type/dscr.md#type). -- `y` — Description. [Type name](relative/path/to/type/dscr.md#type). +- `x` — Description. Optional (only for optional arguments). Possible values: . Default value: . [Type name](relative/path/to/type/dscr.md#type). +- `y` — Description. Optional (only for optional arguments). Possible values: .Default value: . [Type name](relative/path/to/type/dscr.md#type). **Parameters** (Optional, only for parametric aggregate functions) -- `z` — Description. [Type name](relative/path/to/type/dscr.md#type). +- `z` — Description. Optional (only for optional parameters). Possible values: .Default value: . [Type name](relative/path/to/type/dscr.md#type). **Returned value(s)** -- Returned values list. - -Type: [Type name](relative/path/to/type/dscr.md#type). +- Returned values list. Type: [Type name](relative/path/to/type/dscr.md#type). **Example** diff --git a/docs/_description_templates/template-server-setting.md b/docs/_description_templates/template-server-setting.md index 36a2bcacfba..0b37d46cf41 100644 --- a/docs/_description_templates/template-server-setting.md +++ b/docs/_description_templates/template-server-setting.md @@ -8,14 +8,14 @@ Possible value: ... Default value: ... -Settings: (Optional) +**Settings** (Optional) If the section contains several settings, list them here. Specify possible values and default values: - setting_1 — Description. - setting_2 — Description. -**Example:** +**Example** ```xml diff --git a/docs/_description_templates/template-statement.md b/docs/_description_templates/template-statement.md index 62ea51edf83..bca015a2ac6 100644 --- a/docs/_description_templates/template-statement.md +++ b/docs/_description_templates/template-statement.md @@ -1,14 +1,14 @@ -# Statement name (for example, SHOW USER) +# Statement name (for example, SHOW USER) {#statement-name-in-lower-case} Brief description of what the statement does. -Syntax: +**Syntax** ```sql Syntax of the statement. ``` -## Other necessary sections of the description (Optional) +## Other necessary sections of the description (Optional) {#anchor} Examples of descriptions with a complicated structure: @@ -17,7 +17,7 @@ Examples of descriptions with a complicated structure: - https://clickhouse.tech/docs/en/sql-reference/statements/select/join/ -## See Also (Optional) +**See Also** (Optional) Links to related topics as a list. diff --git a/docs/en/operations/system-tables/settings.md b/docs/en/operations/system-tables/settings.md index a1db0a3d558..7034fe1204f 100644 --- a/docs/en/operations/system-tables/settings.md +++ b/docs/en/operations/system-tables/settings.md @@ -48,5 +48,6 @@ SELECT * FROM system.settings WHERE changed AND name='load_balancing' - [Settings](../../operations/settings/index.md#session-settings-intro) - [Permissions for Queries](../../operations/settings/permissions-for-queries.md#settings_readonly) - [Constraints on Settings](../../operations/settings/constraints-on-settings.md) +- [SHOW SETTINGS](../../sql-reference/statements/show.md#show-settings) statement [Original article](https://clickhouse.tech/docs/en/operations/system_tables/settings) diff --git a/docs/en/sql-reference/statements/show.md b/docs/en/sql-reference/statements/show.md index 35631f8c8d6..caa2de8c62c 100644 --- a/docs/en/sql-reference/statements/show.md +++ b/docs/en/sql-reference/statements/show.md @@ -428,4 +428,71 @@ errors_count: 0 estimated_recovery_time: 0 ``` -[Original article](https://clickhouse.tech/docs/en/query_language/show/) +## SHOW SETTINGS {#show-settings} + +Returns a list of system settings and their values. Selects data from the [system.settings](../../operations/system-tables/settings.md) table. + +**Syntax** + +```sql +SHOW [CHANGED] SETTINGS LIKE|ILIKE ; +``` + +**Clauses** + +`LIKE` clause needs a setting name or part of the name without globs. + +`ILIKE` clause can contain globs such as `%` or `_`. + +When the `CHANGED` clause is used, the query returns only settings changed from their default values. + +**Examples** + +Query with the `LIKE` clause: + +```sql +SHOW SETTINGS LIKE 'send_timeout'; +``` +Result: + +```text +┌─name─────────┬─type────┬─value─┐ +│ send_timeout │ Seconds │ 300 │ +└──────────────┴─────────┴───────┘ +``` + +Query with the `ILIKE` clause: + +```sql +SHOW SETTINGS ILIKE '%CONNECT_timeout%' +``` + +Result: + +```text +┌─name────────────────────────────────────┬─type─────────┬─value─┐ +│ connect_timeout │ Seconds │ 10 │ +│ connect_timeout_with_failover_ms │ Milliseconds │ 50 │ +│ connect_timeout_with_failover_secure_ms │ Milliseconds │ 100 │ +└─────────────────────────────────────────┴──────────────┴───────┘ +``` + +Query with the `CHANGED` clause: + +```sql +SHOW CHANGED SETTINGS ILIKE '%MEMORY%' +``` + +Result: + +```text +┌─name─────────────┬─type───┬─value───────┐ +│ max_memory_usage │ UInt64 │ 10000000000 │ +└──────────────────┴────────┴─────────────┘ +``` + +**See Also** + +- [system.settings](../../operations/system-tables/settings.md) table + +[Original article](https://clickhouse.tech/docs/en/sql-reference/statements/show/) diff --git a/docs/ru/operations/system-tables/settings.md b/docs/ru/operations/system-tables/settings.md index c1ada37131c..50ccac684c4 100644 --- a/docs/ru/operations/system-tables/settings.md +++ b/docs/ru/operations/system-tables/settings.md @@ -48,5 +48,6 @@ SELECT * FROM system.settings WHERE changed AND name='load_balancing' - [Настройки](../settings/index.md#settings) - [Разрешения для запросов](../settings/permissions-for-queries.md#settings_readonly) - [Ограничения для значений настроек](../settings/constraints-on-settings.md) +- Выражение [SHOW SETTINGS](../../sql-reference/statements/show.md#show-settings) [Оригинальная статья](https://clickhouse.tech/docs/ru/operations/system_tables/settings) diff --git a/docs/ru/sql-reference/statements/show.md b/docs/ru/sql-reference/statements/show.md index 56528f28c65..0cde54b724e 100644 --- a/docs/ru/sql-reference/statements/show.md +++ b/docs/ru/sql-reference/statements/show.md @@ -362,4 +362,71 @@ SHOW [CURRENT] QUOTA SHOW ACCESS ``` -[Оригинальная статья](https://clickhouse.tech/docs/ru/query_language/show/) +## SHOW SETTINGS {#show-settings} + +Возвращает список системных настроек и их значений. Использует данные из таблицы [system.settings](../../operations/system-tables/settings.md). + +**Синтаксис** + +```sql +SHOW [CHANGED] SETTINGS LIKE|ILIKE ; +``` + +**Секции** + +При использовании `LIKE` необходимо указывать имя или часть имени системной настройки без символов подстановки. + +При использовании `ILIKE` имя системной настройки может содержать символы подстановки, такие как `%` или `_`. + +Если используется `CHANGED`, запрос вернет только те настройки, значения которых были изменены, т.е. отличны от значений по умолчанию. + +**Примеры** + +Запрос с использованием `LIKE`: + +```sql +SHOW SETTINGS LIKE 'send_timeout'; +``` +Результат: + +```text +┌─name─────────┬─type────┬─value─┐ +│ send_timeout │ Seconds │ 300 │ +└──────────────┴─────────┴───────┘ +``` + +Запрос с использованием `ILIKE`: + +```sql +SHOW SETTINGS ILIKE '%CONNECT_timeout%' +``` + +Результат: + +```text +┌─name────────────────────────────────────┬─type─────────┬─value─┐ +│ connect_timeout │ Seconds │ 10 │ +│ connect_timeout_with_failover_ms │ Milliseconds │ 50 │ +│ connect_timeout_with_failover_secure_ms │ Milliseconds │ 100 │ +└─────────────────────────────────────────┴──────────────┴───────┘ +``` + +Запрос с использованием `CHANGED`: + +```sql +SHOW CHANGED SETTINGS ILIKE '%MEMORY%' +``` + +Результат: + +```text +┌─name─────────────┬─type───┬─value───────┐ +│ max_memory_usage │ UInt64 │ 10000000000 │ +└──────────────────┴────────┴─────────────┘ +``` + +**См. также** + +- Таблица [system.settings](../../operations/system-tables/settings.md) + +[Оригинальная статья](https://clickhouse.tech/docs/ru/sql-reference/statements/show/) From d0d85c73d0b5208d8d199303e98320ab4c02155c Mon Sep 17 00:00:00 2001 From: Olga Revyakina Date: Sat, 6 Mar 2021 12:26:55 +0300 Subject: [PATCH 117/544] Minor fixes --- docs/en/sql-reference/statements/show.md | 2 +- docs/ru/sql-reference/statements/show.md | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/en/sql-reference/statements/show.md b/docs/en/sql-reference/statements/show.md index caa2de8c62c..a3e5afd4417 100644 --- a/docs/en/sql-reference/statements/show.md +++ b/docs/en/sql-reference/statements/show.md @@ -435,7 +435,7 @@ Returns a list of system settings and their values. Selects data from the [syste **Syntax** ```sql -SHOW [CHANGED] SETTINGS LIKE|ILIKE ; +SHOW [CHANGED] SETTINGS LIKE|ILIKE ``` **Clauses** diff --git a/docs/ru/sql-reference/statements/show.md b/docs/ru/sql-reference/statements/show.md index 0cde54b724e..ffe9db337e0 100644 --- a/docs/ru/sql-reference/statements/show.md +++ b/docs/ru/sql-reference/statements/show.md @@ -369,7 +369,7 @@ SHOW ACCESS **Синтаксис** ```sql -SHOW [CHANGED] SETTINGS LIKE|ILIKE ; +SHOW [CHANGED] SETTINGS LIKE|ILIKE ``` **Секции** From 406f3f9a06a49d5b3be9aaacf383011c09ffdf90 Mon Sep 17 00:00:00 2001 From: Olga Revyakina Date: Sat, 6 Mar 2021 14:13:14 +0300 Subject: [PATCH 118/544] First commit --- .../sql-reference/functions/math-functions.md | 10 +- .../mergetree-family/mergetree.md | 109 ++++++++++++++++-- .../operations/utilities/clickhouse-local.md | 18 ++- .../sql-reference/functions/math-functions.md | 63 ++++++++++ 4 files changed, 181 insertions(+), 19 deletions(-) diff --git a/docs/en/sql-reference/functions/math-functions.md b/docs/en/sql-reference/functions/math-functions.md index f56a721c0c0..eceffae6b97 100644 --- a/docs/en/sql-reference/functions/math-functions.md +++ b/docs/en/sql-reference/functions/math-functions.md @@ -415,7 +415,7 @@ Result: ## sign(x) {#signx} -The `sign` function can extract the sign of a real number. +Extracts the sign of a real number. **Syntax** @@ -433,9 +433,9 @@ sign(x) - 0 for `x = 0` - 1 for `x > 0` -**Example** +**Examples** -Query: +Sign for the zero value: ``` sql SELECT sign(0); @@ -449,7 +449,7 @@ Result: └─────────┘ ``` -Query: +Sign for the positive value: ``` sql SELECT sign(1); @@ -463,7 +463,7 @@ Result: └─────────┘ ``` -Query: +Sign for the negative value: ``` sql SELECT sign(-1); diff --git a/docs/ru/engines/table-engines/mergetree-family/mergetree.md b/docs/ru/engines/table-engines/mergetree-family/mergetree.md index 6fc566b7c31..4059d4bf163 100644 --- a/docs/ru/engines/table-engines/mergetree-family/mergetree.md +++ b/docs/ru/engines/table-engines/mergetree-family/mergetree.md @@ -188,7 +188,7 @@ ClickHouse не требует уникального первичного кл При сортировке с использованием выражения `ORDER BY` для значений `NULL` всегда работает принцип [NULLS_LAST](../../../sql-reference/statements/select/order-by.md#sorting-of-special-values). -### Выбор первичного ключа {#vybor-pervichnogo-kliucha} +### Выбор первичного ключа {#selecting-the-primary-key} Количество столбцов в первичном ключе не ограничено явным образом. В зависимости от структуры данных в первичный ключ можно включать больше или меньше столбцов. Это может: @@ -217,7 +217,7 @@ ClickHouse не требует уникального первичного кл -### Первичный ключ, отличный от ключа сортировки {#pervichnyi-kliuch-otlichnyi-ot-kliucha-sortirovki} +### Первичный ключ, отличный от ключа сортировки {#choosing-a-primary-key-that-differs-from-the-sorting-key} Существует возможность задать первичный ключ (выражение, значения которого будут записаны в индексный файл для каждой засечки), отличный от ключа сортировки (выражение, по которому будут упорядочены строки в кусках @@ -236,7 +236,7 @@ ClickHouse не требует уникального первичного кл [ALTER ключа сортировки](../../../engines/table-engines/mergetree-family/mergetree.md) — лёгкая операция, так как при одновременном добавлении нового столбца в таблицу и ключ сортировки не нужно изменять данные кусков (они остаются упорядоченными и по новому выражению ключа). -### Использование индексов и партиций в запросах {#ispolzovanie-indeksov-i-partitsii-v-zaprosakh} +### Использование индексов и партиций в запросах {#use-of-indexes-and-partitions-in-queries} Для запросов `SELECT` ClickHouse анализирует возможность использования индекса. Индекс может использоваться, если в секции `WHERE/PREWHERE`, в качестве одного из элементов конъюнкции, или целиком, есть выражение, представляющее операции сравнения на равенства, неравенства, а также `IN` или `LIKE` с фиксированным префиксом, над столбцами или выражениями, входящими в первичный ключ или ключ партиционирования, либо над некоторыми частично монотонными функциями от этих столбцов, а также логические связки над такими выражениями. @@ -270,7 +270,7 @@ SELECT count() FROM table WHERE CounterID = 34 OR URL LIKE '%upyachka%' Ключ партиционирования по месяцам обеспечивает чтение только тех блоков данных, которые содержат даты из нужного диапазона. При этом блок данных может содержать данные за многие даты (до целого месяца). В пределах одного блока данные упорядочены по первичному ключу, который может не содержать дату в качестве первого столбца. В связи с этим, при использовании запроса с указанием условия только на дату, но не на префикс первичного ключа, будет читаться данных больше, чем за одну дату. -### Использование индекса для частично-монотонных первичных ключей {#ispolzovanie-indeksa-dlia-chastichno-monotonnykh-pervichnykh-kliuchei} +### Использование индекса для частично-монотонных первичных ключей {#use-of-index-for-partially-monotonic-primary-keys} Рассмотрим, например, дни месяца. Они образуют последовательность [монотонную](https://ru.wikipedia.org/wiki/Монотонная_последовательность) в течение одного месяца, но не монотонную на более длительных периодах. Это частично-монотонная последовательность. Если пользователь создаёт таблицу с частично-монотонным первичным ключом, ClickHouse как обычно создаёт разреженный индекс. Когда пользователь выбирает данные из такого рода таблиц, ClickHouse анализирует условия запроса. Если пользователь хочет получить данные между двумя метками индекса, и обе эти метки находятся внутри одного месяца, ClickHouse может использовать индекс в данном конкретном случае, поскольку он может рассчитать расстояние между параметрами запроса и индексными метками. @@ -312,7 +312,7 @@ SELECT count() FROM table WHERE s < 'z' SELECT count() FROM table WHERE u64 * i32 == 10 AND u64 * length(s) >= 1234 ``` -#### Доступные индексы {#dostupnye-indeksy} +#### Доступные индексы {#available-types-of-indices} - `minmax` — Хранит минимум и максимум выражения (если выражение - `tuple`, то для каждого элемента `tuple`), используя их для пропуска блоков аналогично первичному ключу. @@ -375,7 +375,7 @@ INDEX b (u64 * length(str), i32 + f64 * 100, date, str) TYPE set(100) GRANULARIT - `s != 1` - `NOT startsWith(s, 'test')` -## Конкурентный доступ к данным {#konkurentnyi-dostup-k-dannym} +## Конкурентный доступ к данным {#concurrent-data-access} Для конкурентного доступа к таблице используется мультиверсионность. То есть, при одновременном чтении и обновлении таблицы, данные будут читаться из набора кусочков, актуального на момент запроса. Длинных блокировок нет. Вставки никак не мешают чтениям. @@ -531,13 +531,13 @@ TTL d + INTERVAL 1 MONTH GROUP BY k1, k2 SET x = max(x), y = min(y); ## Хранение данных таблицы на нескольких блочных устройствах {#table_engine-mergetree-multiple-volumes} -### Введение {#vvedenie} +### Введение {#introduction} Движки таблиц семейства `MergeTree` могут хранить данные на нескольких блочных устройствах. Это может оказаться полезным, например, при неявном разделении данных одной таблицы на «горячие» и «холодные». Наиболее свежая часть занимает малый объём и запрашивается регулярно, а большой хвост исторических данных запрашивается редко. При наличии в системе нескольких дисков, «горячая» часть данных может быть размещена на быстрых дисках (например, на NVMe SSD или в памяти), а холодная на более медленных (например, HDD). Минимальной перемещаемой единицей для `MergeTree` является кусок данных (data part). Данные одного куска могут находится только на одном диске. Куски могут перемещаться между дисками в фоне, согласно пользовательским настройкам, а также с помощью запросов [ALTER](../../../engines/table-engines/mergetree-family/mergetree.md#alter_move-partition). -### Термины {#terminy} +### Термины {#terms} - Диск — примонтированное в файловой системе блочное устройство. - Диск по умолчанию — диск, на котором находится путь, указанный в конфигурационной настройке сервера [path](../../../operations/server-configuration-parameters/settings.md#server_configuration_parameters-path). @@ -689,7 +689,98 @@ SETTINGS storage_policy = 'moving_from_ssd_to_hdd' Количество потоков для фоновых перемещений кусков между дисками можно изменить с помощью настройки [background_move_pool_size](../../../operations/settings/settings.md#background_move_pool_size) -### Особенности работы {#osobennosti-raboty} +## Использование сервиса S3 для хранения данных {#table_engine-mergetree-s3} + +Таблицы семейства `MergeTree` могут использовать для хранения данных сервис [S3](https://aws.amazon.com/s3/) с диском типа `s3`. + +Конфигурация: + +``` xml + + ... + + + s3 + https://storage.yandexcloud.net/my-bucket/root-path/ + your_access_key_id + your_secret_access_key + + http://proxy1 + http://proxy2 + + 10000 + 5000 + 100 + 10 + 1000 + /var/lib/clickhouse/disks/s3/ + true + /var/lib/clickhouse/disks/s3/cache/ + false + + + ... + +``` + +Обязательные параметры: +- `endpoint` — URL конечной точки S3 в запросах `path` или `virtual hosted` [styles](https://docs.aws.amazon.com/AmazonS3/latest/dev/VirtualHosting.html). URL конечной точки должен содержать сегмент (bucket) и путь к корневой директории на сервере, где хранятся данные. +- `access_key_id` — id ключа доступа к S3. +- `secret_access_key` — секретный ключ доступа к S3. + +Необязательные параметры: +- `use_environment_credentials` — признак, нужно ли считывать учетные данные AWS из переменных окружения AWS_ACCESS_KEY_ID, AWS_SECRET_ACCESS_KEY и AWS_SESSION_TOKEN, если они есть. Значение по умолчанию: `false`. +- `proxy` — конфигурация прокси-сервера для конечной точки S3. Каждый элемент `uri` внутри блока `proxy` должен содержать URL прокси-сервера. +- `connect_timeout_ms` — таймаут подключения к сокету в миллисекундах. Значение по умолчанию: 10 секунд. +- `request_timeout_ms` — таймаут выполнения запроса в миллисекундах. Значение по умолчанию: 5 секунд. +- `max_connections` — размер пула соединений S3. Значение по умолчанию: `100`. +- `retry_attempts` — число попыток выполнения запроса в случае возникновения ошибки. Значение по умолчанию: `10`. +- `min_bytes_for_seek` — минимальное количество байтов, которые используются для операций поиска вместо последовательного чтения. Значение по умолчанию: 1 Mb. +- `metadata_path` — путь к локальному файловому хранилищу для хранения файлов с метаданными для S3. Значение по умолчанию: `/var/lib/clickhouse/disks//`. +- `cache_enabled` — признак, разрешено ли хранение меток кеша и индексных файлов в локальном файловом хранилище. Значение по умолчанию: `true`. +- `cache_path` — путь к локальному файловому хранилищу, где хранятся метки кеша и индексные файлы. Значение по умолчанию: `/var/lib/clickhouse/disks//cache/`. +- `skip_access_check` — признак, выполнять ли проверку доступа к диску при запуске диска. Если установлено значение `true`, то проверка не выполняется. Значение по умолчанию: `false`. + + +Диск S3 может быть сконфигурирован как `main` или `cold`: +``` xml + + ... + + + s3 + https://storage.yandexcloud.net/my-bucket/root-path/ + your_access_key_id + your_secret_access_key + + + + + +
+ s3 +
+
+
+ + +
+ default +
+ + s3 + +
+ 0.2 +
+
+ ... +
+``` + +Если диск сконфигурирован как `cold`, данные будут переноситься в S3 либо когда свободное место на локальном диске станет меньше порогового значения, которое определяется как `move_factor * disk_size`, либо при срабатывании правил TTL. + +### Особенности работы {#details} В таблицах `MergeTree` данные попадают на диск несколькими способами: diff --git a/docs/ru/operations/utilities/clickhouse-local.md b/docs/ru/operations/utilities/clickhouse-local.md index 15d069c9acf..da460356224 100644 --- a/docs/ru/operations/utilities/clickhouse-local.md +++ b/docs/ru/operations/utilities/clickhouse-local.md @@ -14,9 +14,9 @@ toc_title: clickhouse-local !!! warning "Warning" Мы не рекомендуем подключать серверную конфигурацию к `clickhouse-local`, поскольку данные можно легко повредить неосторожными действиями. -Для временных данных по умолчанию создается специальный каталог. Если вы хотите обойти это действие, каталог данных можно указать с помощью опции `-- --path`. +Для временных данных по умолчанию создается специальный каталог. -## Вызов программы {#vyzov-programmy} +## Вызов программы {#usage} Основной формат вызова: @@ -31,15 +31,23 @@ $ clickhouse-local --structure "table_structure" --input-format "format_of_incom - `-if`, `--input-format` — формат входящих данных. По умолчанию — `TSV`. - `-f`, `--file` — путь к файлу с данными. По умолчанию — `stdin`. - `-q`, `--query` — запросы на выполнение. Разделитель запросов — `;`. +- `-qf`, `--queries-file` - путь к файлу с запросами для выполнения. Необходимо задать либо параметр `query`, либо `queries-file`. - `-N`, `--table` — имя таблицы, в которую будут помещены входящие данные. По умолчанию - `table`. - `-of`, `--format`, `--output-format` — формат выходных данных. По умолчанию — `TSV`. +- `-d`, `--database` — база данных по умолчанию. Если не указано, используется значение `_local`. - `--stacktrace` — вывод отладочной информации при исключениях. +- `--echo` — перед выполнением запрос выводится в консоль. - `--verbose` — подробный вывод при выполнении запроса. -- `-s` — отключает вывод системных логов в `stderr`. -- `--config-file` — путь к файлу конфигурации. По умолчанию `clickhouse-local` запускается с пустой конфигурацией. Конфигурационный файл имеет тот же формат, что и для сервера ClickHouse и в нём можно использовать все конфигурационные параметры сервера. Обычно подключение конфигурации не требуется, если требуется установить отдельный параметр, то это можно сделать ключом с именем параметра. +- `--logger.console` — логирование действий в консоль. +- `--logger.log` — логирование действий в файл с указанным именем. +- `--logger.level` — уровень логирования. +- `--ignore-error` — не прекращать обработку если запрос выдал ошибку. +- `-c`, `--config-file` — путь к файлу конфигурации. По умолчанию `clickhouse-local` запускается с пустой конфигурацией. Конфигурационный файл имеет тот же формат, что и для сервера ClickHouse и в нём можно использовать все конфигурационные параметры сервера. Обычно подключение конфигурации не требуется, если требуется установить отдельный параметр, то это можно сделать ключом с именем параметра. +- `--no-system-tables` — запуск без использования системных таблиц. - `--help` — вывод справочной информации о `clickhouse-local`. +- `-V`, `--version` — вывод текущей версии и выход. -## Примеры вызова {#primery-vyzova} +## Примеры вызова {#examples} ``` bash $ echo -e "1,2\n3,4" | clickhouse-local --structure "a Int64, b Int64" \ diff --git a/docs/ru/sql-reference/functions/math-functions.md b/docs/ru/sql-reference/functions/math-functions.md index 2e57aca6a0a..2b67b2d87ed 100644 --- a/docs/ru/sql-reference/functions/math-functions.md +++ b/docs/ru/sql-reference/functions/math-functions.md @@ -400,6 +400,69 @@ SELECT log1p(0); │ 0 │ └──────────┘ ``` +## sign(x) {#signx} + +Выделяет знак действительного числа. + +**Синтаксис** + +``` sql +sign(x) +``` + +**Аргумент** + +- `x` — Значения от `-∞` до `+∞`. Любой числовой тип, поддерживаемый ClickHouse. + +**Возвращаемое значение** + +- -1 если `x < 0` +- 0 если `x = 0` +- 1 если `x > 0` + +**Примеры** + +Знак для нулевого значения: + +``` sql +SELECT sign(0); +``` + +Результат: + +``` text +┌─sign(0)─┐ +│ 0 │ +└─────────┘ +``` + +Знак для положительного значения: + +``` sql +SELECT sign(1); +``` + +Результат: + +``` text +┌─sign(1)─┐ +│ 1 │ +└─────────┘ +``` + +Знак для отрицательного значения: + +``` sql +SELECT sign(-1); +``` + +Результат: + +``` text +┌─sign(-1)─┐ +│ -1 │ +└──────────┘ +``` **Смотрите также** From 22518cb14b59137908aade2887f73ea63546c9fa Mon Sep 17 00:00:00 2001 From: Olga Revyakina Date: Sat, 6 Mar 2021 14:25:37 +0300 Subject: [PATCH 119/544] Links fixed. --- docs/ru/engines/table-engines/mergetree-family/mergetree.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/ru/engines/table-engines/mergetree-family/mergetree.md b/docs/ru/engines/table-engines/mergetree-family/mergetree.md index 4059d4bf163..952fd08a485 100644 --- a/docs/ru/engines/table-engines/mergetree-family/mergetree.md +++ b/docs/ru/engines/table-engines/mergetree-family/mergetree.md @@ -56,13 +56,13 @@ ORDER BY expr ClickHouse использует ключ сортировки в качестве первичного ключа, если первичный ключ не задан в секции `PRIMARY KEY`. - Чтобы отключить сортировку, используйте синтаксис `ORDER BY tuple()`. Смотрите [выбор первичного ключа](#vybor-pervichnogo-kliucha). + Чтобы отключить сортировку, используйте синтаксис `ORDER BY tuple()`. Смотрите [выбор первичного ключа](#primary-keys-and-indexes-in-queries). - `PARTITION BY` — [ключ партиционирования](custom-partitioning-key.md). Необязательный параметр. Для партиционирования по месяцам используйте выражение `toYYYYMM(date_column)`, где `date_column` — столбец с датой типа [Date](../../../engines/table-engines/mergetree-family/mergetree.md). В этом случае имена партиций имеют формат `"YYYYMM"`. -- `PRIMARY KEY` — первичный ключ, если он [отличается от ключа сортировки](#pervichnyi-kliuch-otlichnyi-ot-kliucha-sortirovki). Необязательный параметр. +- `PRIMARY KEY` — первичный ключ, если он [отличается от ключа сортировки](#choosing-a-primary-key-that-differs-from-the-sorting-key). Необязательный параметр. По умолчанию первичный ключ совпадает с ключом сортировки (который задаётся секцией `ORDER BY`.) Поэтому в большинстве случаев секцию `PRIMARY KEY` отдельно указывать не нужно. From 529acce3125e4f9866f343b5a39eb9d155486510 Mon Sep 17 00:00:00 2001 From: Olga Revyakina Date: Sat, 6 Mar 2021 14:55:17 +0300 Subject: [PATCH 120/544] Minor fixes --- docs/ru/engines/table-engines/mergetree-family/mergetree.md | 3 +++ 1 file changed, 3 insertions(+) diff --git a/docs/ru/engines/table-engines/mergetree-family/mergetree.md b/docs/ru/engines/table-engines/mergetree-family/mergetree.md index 952fd08a485..e225fdf8e0f 100644 --- a/docs/ru/engines/table-engines/mergetree-family/mergetree.md +++ b/docs/ru/engines/table-engines/mergetree-family/mergetree.md @@ -724,11 +724,13 @@ SETTINGS storage_policy = 'moving_from_ssd_to_hdd' ``` Обязательные параметры: + - `endpoint` — URL конечной точки S3 в запросах `path` или `virtual hosted` [styles](https://docs.aws.amazon.com/AmazonS3/latest/dev/VirtualHosting.html). URL конечной точки должен содержать сегмент (bucket) и путь к корневой директории на сервере, где хранятся данные. - `access_key_id` — id ключа доступа к S3. - `secret_access_key` — секретный ключ доступа к S3. Необязательные параметры: + - `use_environment_credentials` — признак, нужно ли считывать учетные данные AWS из переменных окружения AWS_ACCESS_KEY_ID, AWS_SECRET_ACCESS_KEY и AWS_SESSION_TOKEN, если они есть. Значение по умолчанию: `false`. - `proxy` — конфигурация прокси-сервера для конечной точки S3. Каждый элемент `uri` внутри блока `proxy` должен содержать URL прокси-сервера. - `connect_timeout_ms` — таймаут подключения к сокету в миллисекундах. Значение по умолчанию: 10 секунд. @@ -743,6 +745,7 @@ SETTINGS storage_policy = 'moving_from_ssd_to_hdd' Диск S3 может быть сконфигурирован как `main` или `cold`: + ``` xml ... From ba16896162b0b91729a19c386d61b10a6073d073 Mon Sep 17 00:00:00 2001 From: feng lv Date: Sat, 6 Mar 2021 10:51:39 +0000 Subject: [PATCH 121/544] add --backslash option for clickhouse-format fix fix fix --- programs/format/Format.cpp | 43 ++++++++++++++++--- ...1754_clickhouse_format_backslash.reference | 16 +++++++ .../01754_clickhouse_format_backslash.sh | 9 ++++ 3 files changed, 61 insertions(+), 7 deletions(-) create mode 100644 tests/queries/0_stateless/01754_clickhouse_format_backslash.reference create mode 100755 tests/queries/0_stateless/01754_clickhouse_format_backslash.sh diff --git a/programs/format/Format.cpp b/programs/format/Format.cpp index 86a85d1d4a5..ac6352c4033 100644 --- a/programs/format/Format.cpp +++ b/programs/format/Format.cpp @@ -1,6 +1,6 @@ +#include #include #include -#include #include #include @@ -40,6 +40,7 @@ int mainEntryClickHouseFormat(int argc, char ** argv) ("quiet,q", "just check syntax, no output on success") ("multiquery,n", "allow multiple queries in the same file") ("obfuscate", "obfuscate instead of formatting") + ("backslash", "add a backslash at the end of each line of the formatted query") ("seed", po::value(), "seed (arbitrary string) that determines the result of obfuscation") ; @@ -60,6 +61,7 @@ int mainEntryClickHouseFormat(int argc, char ** argv) bool quiet = options.count("quiet"); bool multiple = options.count("multiquery"); bool obfuscate = options.count("obfuscate"); + bool backslash = options.count("backslash"); if (quiet && (hilite || oneline || obfuscate)) { @@ -130,12 +132,39 @@ int mainEntryClickHouseFormat(int argc, char ** argv) ASTPtr res = parseQueryAndMovePosition(parser, pos, end, "query", multiple, 0, DBMS_DEFAULT_MAX_PARSER_DEPTH); if (!quiet) { - WriteBufferFromOStream res_buf(std::cout, 4096); - formatAST(*res, res_buf, hilite, oneline); - res_buf.next(); - if (multiple) - std::cout << "\n;\n"; - std::cout << std::endl; + if (!backslash) + { + WriteBufferFromOStream res_buf(std::cout, 4096); + formatAST(*res, res_buf, hilite, oneline); + res_buf.next(); + if (multiple) + std::cout << "\n;\n"; + std::cout << std::endl; + } + /// add additional '\' at the end of each line; + else + { + WriteBufferFromOwnString str_buf; + formatAST(*res, str_buf, hilite, oneline); + + auto res_string = str_buf.str(); + WriteBufferFromOStream res_cout(std::cout, 4096); + + const char * s_pos= res_string.data(); + const char * s_end = s_pos + res_string.size(); + + while (s_pos != s_end) + { + if (*s_pos == '\n') + res_cout.write(" \\", 2); + res_cout.write(*s_pos++); + } + + res_cout.next(); + if (multiple) + std::cout << " \\\n;\n"; + std::cout << std::endl; + } } } while (multiple && pos != end); } diff --git a/tests/queries/0_stateless/01754_clickhouse_format_backslash.reference b/tests/queries/0_stateless/01754_clickhouse_format_backslash.reference new file mode 100644 index 00000000000..328483d9867 --- /dev/null +++ b/tests/queries/0_stateless/01754_clickhouse_format_backslash.reference @@ -0,0 +1,16 @@ +SELECT * \ +FROM \ +( \ + SELECT 1 AS x \ + UNION ALL \ + SELECT 1 \ + UNION DISTINCT \ + SELECT 3 \ +) +SELECT 1 \ +UNION ALL \ +( \ + SELECT 1 \ + UNION DISTINCT \ + SELECT 1 \ +) diff --git a/tests/queries/0_stateless/01754_clickhouse_format_backslash.sh b/tests/queries/0_stateless/01754_clickhouse_format_backslash.sh new file mode 100755 index 00000000000..6a76dc9c5c8 --- /dev/null +++ b/tests/queries/0_stateless/01754_clickhouse_format_backslash.sh @@ -0,0 +1,9 @@ +#!/usr/bin/env bash + +CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh +. "$CURDIR"/../shell_config.sh + +echo "select * from (select 1 as x union all select 1 union distinct select 3)" | $CLICKHOUSE_FORMAT --backslash; + +echo "select 1 union all (select 1 union distinct select 1)" | $CLICKHOUSE_FORMAT --backslash; From 5ac8a6634ab596ee7d45e60a0954659204e15fae Mon Sep 17 00:00:00 2001 From: Olga Revyakina Date: Sat, 6 Mar 2021 15:37:52 +0300 Subject: [PATCH 122/544] Fix --- docs/ru/sql-reference/functions/math-functions.md | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/docs/ru/sql-reference/functions/math-functions.md b/docs/ru/sql-reference/functions/math-functions.md index 2b67b2d87ed..e12f0d96827 100644 --- a/docs/ru/sql-reference/functions/math-functions.md +++ b/docs/ru/sql-reference/functions/math-functions.md @@ -400,6 +400,11 @@ SELECT log1p(0); │ 0 │ └──────────┘ ``` + +**Смотрите также** + +- [log(x)](../../sql-reference/functions/math-functions.md#logx) + ## sign(x) {#signx} Выделяет знак действительного числа. @@ -464,8 +469,4 @@ SELECT sign(-1); └──────────┘ ``` -**Смотрите также** - -- [log(x)](../../sql-reference/functions/math-functions.md#logx) - [Оригинальная статья](https://clickhouse.tech/docs/ru/query_language/functions/math_functions/) From f9aa416f69a42154633a0146d6ee62acc24fb966 Mon Sep 17 00:00:00 2001 From: olgarev <56617294+olgarev@users.noreply.github.com> Date: Sat, 6 Mar 2021 22:14:15 +0300 Subject: [PATCH 123/544] Update docs/_description_templates/template-engine.md Co-authored-by: Anna <42538400+adevyatova@users.noreply.github.com> --- docs/_description_templates/template-engine.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/_description_templates/template-engine.md b/docs/_description_templates/template-engine.md index 5ace1449de2..490f490fc4e 100644 --- a/docs/_description_templates/template-engine.md +++ b/docs/_description_templates/template-engine.md @@ -58,6 +58,6 @@ Result: Follow up with any text to clarify the example. -**See Also** {#see-also} +**See Also** - [link](#) From 187327001aaeb4e6b504eab6a245759a5cb75dd4 Mon Sep 17 00:00:00 2001 From: olgarev <56617294+olgarev@users.noreply.github.com> Date: Sat, 6 Mar 2021 22:15:39 +0300 Subject: [PATCH 124/544] Update docs/_description_templates/template-function.md Co-authored-by: Anna <42538400+adevyatova@users.noreply.github.com> --- docs/_description_templates/template-function.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/_description_templates/template-function.md b/docs/_description_templates/template-function.md index f8156ec3f6f..0d9e31db082 100644 --- a/docs/_description_templates/template-function.md +++ b/docs/_description_templates/template-function.md @@ -19,7 +19,7 @@ More text (Optional). **Parameters** (Optional, only for parametric aggregate functions) -- `z` — Description. Optional (only for optional parameters). Possible values: .Default value: . [Type name](relative/path/to/type/dscr.md#type). +- `z` — Description. Optional (only for optional parameters). Possible values: . Default value: . [Type name](relative/path/to/type/dscr.md#type). **Returned value(s)** From a0887dc7fe18a2f1623ca02dfa529f360289953f Mon Sep 17 00:00:00 2001 From: Olga Revyakina Date: Sat, 6 Mar 2021 22:18:05 +0300 Subject: [PATCH 125/544] Fix --- docs/_description_templates/template-function.md | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/docs/_description_templates/template-function.md b/docs/_description_templates/template-function.md index 0d9e31db082..3d4d921898a 100644 --- a/docs/_description_templates/template-function.md +++ b/docs/_description_templates/template-function.md @@ -23,7 +23,9 @@ More text (Optional). **Returned value(s)** -- Returned values list. Type: [Type name](relative/path/to/type/dscr.md#type). +- Returned values list. + +Type: [Type name](relative/path/to/type/dscr.md#type). **Example** From f19bd8270ddc2ea0992d4a7a9d3e1b7cd45fc75c Mon Sep 17 00:00:00 2001 From: olgarev <56617294+olgarev@users.noreply.github.com> Date: Sat, 6 Mar 2021 23:27:08 +0300 Subject: [PATCH 126/544] Apply suggestions from code review Co-authored-by: Anna <42538400+adevyatova@users.noreply.github.com> --- docs/ru/engines/table-engines/mergetree-family/mergetree.md | 4 ++-- docs/ru/operations/utilities/clickhouse-local.md | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/ru/engines/table-engines/mergetree-family/mergetree.md b/docs/ru/engines/table-engines/mergetree-family/mergetree.md index e225fdf8e0f..478e5a8345e 100644 --- a/docs/ru/engines/table-engines/mergetree-family/mergetree.md +++ b/docs/ru/engines/table-engines/mergetree-family/mergetree.md @@ -691,7 +691,7 @@ SETTINGS storage_policy = 'moving_from_ssd_to_hdd' ## Использование сервиса S3 для хранения данных {#table_engine-mergetree-s3} -Таблицы семейства `MergeTree` могут использовать для хранения данных сервис [S3](https://aws.amazon.com/s3/) с диском типа `s3`. +Таблицы семейства `MergeTree` могут использовать для хранения данных сервиса [Amazon S3](https://aws.amazon.com/s3/) с диском типа `s3`. Конфигурация: @@ -737,7 +737,7 @@ SETTINGS storage_policy = 'moving_from_ssd_to_hdd' - `request_timeout_ms` — таймаут выполнения запроса в миллисекундах. Значение по умолчанию: 5 секунд. - `max_connections` — размер пула соединений S3. Значение по умолчанию: `100`. - `retry_attempts` — число попыток выполнения запроса в случае возникновения ошибки. Значение по умолчанию: `10`. -- `min_bytes_for_seek` — минимальное количество байтов, которые используются для операций поиска вместо последовательного чтения. Значение по умолчанию: 1 Mb. +- `min_bytes_for_seek` — минимальное количество байтов, которые используются для операций поиска вместо последовательного чтения. Значение по умолчанию: 1 МБайт. - `metadata_path` — путь к локальному файловому хранилищу для хранения файлов с метаданными для S3. Значение по умолчанию: `/var/lib/clickhouse/disks//`. - `cache_enabled` — признак, разрешено ли хранение меток кеша и индексных файлов в локальном файловом хранилище. Значение по умолчанию: `true`. - `cache_path` — путь к локальному файловому хранилищу, где хранятся метки кеша и индексные файлы. Значение по умолчанию: `/var/lib/clickhouse/disks//cache/`. diff --git a/docs/ru/operations/utilities/clickhouse-local.md b/docs/ru/operations/utilities/clickhouse-local.md index da460356224..137472fa993 100644 --- a/docs/ru/operations/utilities/clickhouse-local.md +++ b/docs/ru/operations/utilities/clickhouse-local.md @@ -42,7 +42,7 @@ $ clickhouse-local --structure "table_structure" --input-format "format_of_incom - `--logger.log` — логирование действий в файл с указанным именем. - `--logger.level` — уровень логирования. - `--ignore-error` — не прекращать обработку если запрос выдал ошибку. -- `-c`, `--config-file` — путь к файлу конфигурации. По умолчанию `clickhouse-local` запускается с пустой конфигурацией. Конфигурационный файл имеет тот же формат, что и для сервера ClickHouse и в нём можно использовать все конфигурационные параметры сервера. Обычно подключение конфигурации не требуется, если требуется установить отдельный параметр, то это можно сделать ключом с именем параметра. +- `-c`, `--config-file` — путь к файлу конфигурации. По умолчанию `clickhouse-local` запускается с пустой конфигурацией. Конфигурационный файл имеет тот же формат, что и для сервера ClickHouse, и в нём можно использовать все конфигурационные параметры сервера. Обычно подключение конфигурации не требуется; если требуется установить отдельный параметр, то это можно сделать ключом с именем параметра. - `--no-system-tables` — запуск без использования системных таблиц. - `--help` — вывод справочной информации о `clickhouse-local`. - `-V`, `--version` — вывод текущей версии и выход. From 2b9c9b28eff5380e30b2310c54aaf26beb4e9056 Mon Sep 17 00:00:00 2001 From: olgarev <56617294+olgarev@users.noreply.github.com> Date: Sat, 6 Mar 2021 23:30:22 +0300 Subject: [PATCH 127/544] Update docs/ru/engines/table-engines/mergetree-family/mergetree.md Co-authored-by: Anna <42538400+adevyatova@users.noreply.github.com> --- docs/ru/engines/table-engines/mergetree-family/mergetree.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/ru/engines/table-engines/mergetree-family/mergetree.md b/docs/ru/engines/table-engines/mergetree-family/mergetree.md index 478e5a8345e..90497ebbfa8 100644 --- a/docs/ru/engines/table-engines/mergetree-family/mergetree.md +++ b/docs/ru/engines/table-engines/mergetree-family/mergetree.md @@ -781,7 +781,7 @@ SETTINGS storage_policy = 'moving_from_ssd_to_hdd' ``` -Если диск сконфигурирован как `cold`, данные будут переноситься в S3 либо когда свободное место на локальном диске станет меньше порогового значения, которое определяется как `move_factor * disk_size`, либо при срабатывании правил TTL. +Если диск сконфигурирован как `cold`, данные будут переноситься в S3 при срабатывании правил TTL, либо когда свободное место на локальном диске станет меньше порогового значения, которое определяется как `move_factor * disk_size`. ### Особенности работы {#details} From eae4a4aac9fded1d28b0270ba70e4c3939a401de Mon Sep 17 00:00:00 2001 From: Olga Revyakina Date: Sat, 6 Mar 2021 23:33:56 +0300 Subject: [PATCH 128/544] Fixes --- docs/ru/engines/table-engines/mergetree-family/mergetree.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/ru/engines/table-engines/mergetree-family/mergetree.md b/docs/ru/engines/table-engines/mergetree-family/mergetree.md index 90497ebbfa8..cb633aa1e8a 100644 --- a/docs/ru/engines/table-engines/mergetree-family/mergetree.md +++ b/docs/ru/engines/table-engines/mergetree-family/mergetree.md @@ -725,13 +725,13 @@ SETTINGS storage_policy = 'moving_from_ssd_to_hdd' Обязательные параметры: -- `endpoint` — URL конечной точки S3 в запросах `path` или `virtual hosted` [styles](https://docs.aws.amazon.com/AmazonS3/latest/dev/VirtualHosting.html). URL конечной точки должен содержать сегмент (bucket) и путь к корневой директории на сервере, где хранятся данные. +- `endpoint` — URL конечной точки S3 в запросах `path` или `virtual hosted` [styles](https://docs.aws.amazon.com/AmazonS3/latest/dev/VirtualHosting.html). URL конечной точки должен содержать бакет и путь к корневой директории на сервере, где хранятся данные. - `access_key_id` — id ключа доступа к S3. - `secret_access_key` — секретный ключ доступа к S3. Необязательные параметры: -- `use_environment_credentials` — признак, нужно ли считывать учетные данные AWS из переменных окружения AWS_ACCESS_KEY_ID, AWS_SECRET_ACCESS_KEY и AWS_SESSION_TOKEN, если они есть. Значение по умолчанию: `false`. +- `use_environment_credentials` — признак, нужно ли считывать учетные данные AWS из переменных окружения `AWS_ACCESS_KEY_ID`, `AWS_SECRET_ACCESS_KEY` и `AWS_SESSION_TOKEN`, если они есть. Значение по умолчанию: `false`. - `proxy` — конфигурация прокси-сервера для конечной точки S3. Каждый элемент `uri` внутри блока `proxy` должен содержать URL прокси-сервера. - `connect_timeout_ms` — таймаут подключения к сокету в миллисекундах. Значение по умолчанию: 10 секунд. - `request_timeout_ms` — таймаут выполнения запроса в миллисекундах. Значение по умолчанию: 5 секунд. @@ -781,7 +781,7 @@ SETTINGS storage_policy = 'moving_from_ssd_to_hdd' ``` -Если диск сконфигурирован как `cold`, данные будут переноситься в S3 при срабатывании правил TTL, либо когда свободное место на локальном диске станет меньше порогового значения, которое определяется как `move_factor * disk_size`. +Если диск сконфигурирован как `cold`, данные будут переноситься в S3 при срабатывании правил TTL или когда свободное место на локальном диске станет меньше порогового значения, которое определяется как `move_factor * disk_size`. ### Особенности работы {#details} From 40c439f96f3b02df6f28abbbdeec43bc6e01bd62 Mon Sep 17 00:00:00 2001 From: Olga Revyakina Date: Sat, 6 Mar 2021 23:38:18 +0300 Subject: [PATCH 129/544] Fixes --- docs/en/sql-reference/functions/math-functions.md | 6 ------ docs/ru/sql-reference/functions/math-functions.md | 6 ------ 2 files changed, 12 deletions(-) diff --git a/docs/en/sql-reference/functions/math-functions.md b/docs/en/sql-reference/functions/math-functions.md index eceffae6b97..b589e2445e6 100644 --- a/docs/en/sql-reference/functions/math-functions.md +++ b/docs/en/sql-reference/functions/math-functions.md @@ -441,8 +441,6 @@ Sign for the zero value: SELECT sign(0); ``` -Result: - ``` text ┌─sign(0)─┐ │ 0 │ @@ -455,8 +453,6 @@ Sign for the positive value: SELECT sign(1); ``` -Result: - ``` text ┌─sign(1)─┐ │ 1 │ @@ -469,8 +465,6 @@ Sign for the negative value: SELECT sign(-1); ``` -Result: - ``` text ┌─sign(-1)─┐ │ -1 │ diff --git a/docs/ru/sql-reference/functions/math-functions.md b/docs/ru/sql-reference/functions/math-functions.md index e12f0d96827..e25ed7ed0dc 100644 --- a/docs/ru/sql-reference/functions/math-functions.md +++ b/docs/ru/sql-reference/functions/math-functions.md @@ -433,8 +433,6 @@ sign(x) SELECT sign(0); ``` -Результат: - ``` text ┌─sign(0)─┐ │ 0 │ @@ -447,8 +445,6 @@ SELECT sign(0); SELECT sign(1); ``` -Результат: - ``` text ┌─sign(1)─┐ │ 1 │ @@ -461,8 +457,6 @@ SELECT sign(1); SELECT sign(-1); ``` -Результат: - ``` text ┌─sign(-1)─┐ │ -1 │ From 1d0b63b652fc15d54f735bd286c6f3c8b7af8372 Mon Sep 17 00:00:00 2001 From: Olga Revyakina Date: Sat, 6 Mar 2021 23:51:42 +0300 Subject: [PATCH 130/544] Fixed back --- docs/en/sql-reference/functions/math-functions.md | 6 ++++++ docs/ru/sql-reference/functions/math-functions.md | 5 +++++ 2 files changed, 11 insertions(+) diff --git a/docs/en/sql-reference/functions/math-functions.md b/docs/en/sql-reference/functions/math-functions.md index b589e2445e6..eceffae6b97 100644 --- a/docs/en/sql-reference/functions/math-functions.md +++ b/docs/en/sql-reference/functions/math-functions.md @@ -441,6 +441,8 @@ Sign for the zero value: SELECT sign(0); ``` +Result: + ``` text ┌─sign(0)─┐ │ 0 │ @@ -453,6 +455,8 @@ Sign for the positive value: SELECT sign(1); ``` +Result: + ``` text ┌─sign(1)─┐ │ 1 │ @@ -465,6 +469,8 @@ Sign for the negative value: SELECT sign(-1); ``` +Result: + ``` text ┌─sign(-1)─┐ │ -1 │ diff --git a/docs/ru/sql-reference/functions/math-functions.md b/docs/ru/sql-reference/functions/math-functions.md index e25ed7ed0dc..9748cbf7b28 100644 --- a/docs/ru/sql-reference/functions/math-functions.md +++ b/docs/ru/sql-reference/functions/math-functions.md @@ -432,6 +432,7 @@ sign(x) ``` sql SELECT sign(0); ``` +Результат: ``` text ┌─sign(0)─┐ @@ -445,6 +446,8 @@ SELECT sign(0); SELECT sign(1); ``` +Результат: + ``` text ┌─sign(1)─┐ │ 1 │ @@ -457,6 +460,8 @@ SELECT sign(1); SELECT sign(-1); ``` +Результат: + ``` text ┌─sign(-1)─┐ │ -1 │ From 8d5d1b76fbc08a337bd4390dfc2e9c54883569d9 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Sun, 7 Mar 2021 00:36:29 +0300 Subject: [PATCH 131/544] Fix error --- base/common/DateLUTImpl.h | 27 +++++++++++++++++++++++---- 1 file changed, 23 insertions(+), 4 deletions(-) diff --git a/base/common/DateLUTImpl.h b/base/common/DateLUTImpl.h index 2534f20838f..6e8b424ab9d 100644 --- a/base/common/DateLUTImpl.h +++ b/base/common/DateLUTImpl.h @@ -5,7 +5,9 @@ #include "types.h" #include +#include #include +#include #define DATE_LUT_MIN_YEAR 1925 /// 1925 since wast majority of timezones changed to 15-minute aligned offsets somewhere in 1924 or earlier. @@ -231,6 +233,21 @@ private: return lut[toLUTIndex(v)]; } + template + static inline T roundDown(T x, Divisor divisor) + { + static_assert(std::is_integral_v && std::is_integral_v); + assert(divisor > 0); + + if (likely(x >= 0)) + return x / divisor * divisor; + + /// Integer division for negative numbers rounds them towards zero (up). + /// We will shift the number so it will be rounded towards -inf (down). + + return (x + 1 - divisor) / divisor * divisor; + } + public: const std::string & getTimeZone() const { return time_zone; } @@ -822,10 +839,12 @@ public: return toStartOfHour(t); UInt64 seconds = hours * 3600; - t = (t + DATE_LUT_ADD) / seconds * seconds - DATE_LUT_ADD; + t = roundDown(t, seconds); if (offset_is_whole_number_of_hours_everytime) return t; + + /// TODO check if it's correct. return toStartOfHour(t); } @@ -833,9 +852,9 @@ public: { if (minutes == 1) return toStartOfMinute(t); - UInt64 seconds = 60 * minutes; - return (t + DATE_LUT_ADD) / seconds * seconds - DATE_LUT_ADD; + UInt64 seconds = 60 * minutes; + return roundDown(t, seconds); } inline time_t toStartOfSecondInterval(time_t t, UInt64 seconds) const @@ -843,7 +862,7 @@ public: if (seconds == 1) return t; - return (t + DATE_LUT_ADD) / seconds * seconds - DATE_LUT_ADD; + return roundDown(t, seconds); } inline LUTIndex makeLUTIndex(Int16 year, UInt8 month, UInt8 day_of_month) const From 0dbadc8d8eec1e23afdd635609aaf8fd612110a6 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Sun, 7 Mar 2021 01:35:23 +0300 Subject: [PATCH 132/544] Fix error --- base/common/DateLUTImpl.cpp | 28 ++++++++++++++++------------ base/common/DateLUTImpl.h | 19 +++++++++++++------ 2 files changed, 29 insertions(+), 18 deletions(-) diff --git a/base/common/DateLUTImpl.cpp b/base/common/DateLUTImpl.cpp index 46fc88aeb4d..a111a21d8fd 100644 --- a/base/common/DateLUTImpl.cpp +++ b/base/common/DateLUTImpl.cpp @@ -69,10 +69,18 @@ DateLUTImpl::DateLUTImpl(const std::string & time_zone_) cctz::time_zone::civil_lookup lookup = cctz_time_zone.lookup(date); /// Ambiguity is possible if time was changed backwards at the midnight - /// (or after midnight time has been changed to the previous day, for example two hours backwards at 01:00). - /// Then midnight appears twice. Usually time change happens exactly at 00:00. - /// Then we should use the second midnight as the start of the day. - start_of_day = std::chrono::system_clock::to_time_t(lookup.post); + /// or after midnight time has been changed back to midnight, for example one hour backwards at 01:00 + /// or after midnight time has been changed to the previous day, for example two hours backwards at 01:00 + /// Then midnight appears twice. Usually time change happens exactly at 00:00 or 01:00. + + /// If transition did not involve previous day, we should use the first midnight as the start of the day, + /// otherwise it's better to use the second midnight. + + std::chrono::time_point start_of_day_time_point = lookup.trans < lookup.post + ? lookup.post /* Second midnight appears after transition, so there was a piece of previous day after transition */ + : lookup.pre; + + start_of_day = std::chrono::system_clock::to_time_t(start_of_day_time_point); Values & values = lut[i]; values.year = date.year(); @@ -97,17 +105,13 @@ DateLUTImpl::DateLUTImpl(const std::string & time_zone_) values.time_at_offset_change_value = 0; values.amount_of_offset_change_value = 0; - /// TODO: This partially ignores fractional offsets, - /// which may cause incorrect toRelativeHourNum() results for some timezones, namelly Europe/Minsk - /// when pre-May 2 1924 it had an offset of UTC+1:50, and after it was UTC+2h. - /// https://www.timeanddate.com/time/zone/belarus/minsk?syear=1900 - if (start_of_day > 0 && start_of_day % 3600) + if (offset_is_whole_number_of_hours_everytime && start_of_day > 0 && start_of_day % 3600) offset_is_whole_number_of_hours_everytime = false; /// If UTC offset was changed this day. /// Change in time zone without transition is possible, e.g. Moscow 1991 Sun, 31 Mar, 02:00 MSK to EEST cctz::time_zone::civil_transition transition{}; - if (cctz_time_zone.next_transition(lookup.post, &transition) + if (cctz_time_zone.next_transition(start_of_day_time_point - std::chrono::seconds(1), &transition) && transition.from.year() == date.year() && transition.from.month() == date.month() && transition.from.day() == date.day() @@ -116,8 +120,8 @@ DateLUTImpl::DateLUTImpl(const std::string & time_zone_) values.time_at_offset_change_value = (transition.from - cctz::civil_second(date)) / Values::OffsetChangeFactor; values.amount_of_offset_change_value = (transition.to - transition.from) / Values::OffsetChangeFactor; -// std::cerr << time_zone << ", " << date << ": change from " << transition.from << " to " << transition.to << "\n"; -// std::cerr << time_zone << ", " << date << ": change at " << values.time_at_offset_change() << " with " << values.amount_of_offset_change() << "\n"; + std::cerr << time_zone << ", " << date << ": change from " << transition.from << " to " << transition.to << "\n"; + std::cerr << time_zone << ", " << date << ": change at " << values.time_at_offset_change() << " with " << values.amount_of_offset_change() << "\n"; /// We don't support too large changes. if (values.amount_of_offset_change_value > 24 * 4) diff --git a/base/common/DateLUTImpl.h b/base/common/DateLUTImpl.h index 6e8b424ab9d..e558a3b45ce 100644 --- a/base/common/DateLUTImpl.h +++ b/base/common/DateLUTImpl.h @@ -472,18 +472,25 @@ public: } /// NOTE: Assuming timezone offset is a multiple of 15 minutes. - inline time_t toStartOfMinute(time_t t) const { return (t + DATE_LUT_ADD) / 60 * 60 - DATE_LUT_ADD; } - inline time_t toStartOfFiveMinute(time_t t) const { return (t + DATE_LUT_ADD) / 300 * 300 - DATE_LUT_ADD; } - inline time_t toStartOfFifteenMinutes(time_t t) const { return (t + DATE_LUT_ADD) / 900 * 900 - DATE_LUT_ADD; } + inline time_t toStartOfMinute(time_t t) const { return roundDown(t, 60); } + inline time_t toStartOfFiveMinute(time_t t) const { return roundDown(t, 300); } + inline time_t toStartOfFifteenMinutes(time_t t) const { return roundDown(t, 900); } - /// NOTE: This most likely wrong for Nepal - it has offset 05:45. Australia/Eucla is also unfortunate. - inline time_t toStartOfTenMinutes(time_t t) const { return (t + DATE_LUT_ADD) / 600 * 600 - DATE_LUT_ADD; } + inline time_t toStartOfTenMinutes(time_t t) const + { + if (offset_is_whole_number_of_hours_everytime) + return roundDown(t, 600); + + /// More complex logic is for Nepal - it has offset 05:45. Australia/Eucla is also unfortunate. + Int64 date = find(t).date; + return date + (t - date) / 600 * 600; + } /// NOTE: Assuming timezone transitions are multiple of hours. Lord Howe Island in Australia is a notable exception. inline time_t toStartOfHour(time_t t) const { if (offset_is_whole_number_of_hours_everytime) - return (t + DATE_LUT_ADD) / 3600 * 3600 - DATE_LUT_ADD; + return roundDown(t, 3600); Int64 date = find(t).date; return date + (t - date) / 3600 * 3600; From e33fae76256d3c0e01e7e14f5c94a4cdd763caca Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Sun, 7 Mar 2021 01:35:49 +0300 Subject: [PATCH 133/544] Improve test --- src/Common/tests/gtest_DateLUTImpl.cpp | 39 ++++++++++++++++++++++---- 1 file changed, 33 insertions(+), 6 deletions(-) diff --git a/src/Common/tests/gtest_DateLUTImpl.cpp b/src/Common/tests/gtest_DateLUTImpl.cpp index 95ac055e237..80cb73f06c8 100644 --- a/src/Common/tests/gtest_DateLUTImpl.cpp +++ b/src/Common/tests/gtest_DateLUTImpl.cpp @@ -407,22 +407,49 @@ TEST_P(DateLUTWithTimeZoneAndTimeRange, InRange) /// Sun Nov 7 00:00:30 ADT 2010 /// Sat Nov 6 23:01:00 AST 2010 /// Sat Nov 6 23:01:30 AST 2010 + + bool has_transition = false; cctz::time_zone::civil_transition transition{}; if (tz.next_transition(std::chrono::system_clock::from_time_t(expected_time_t), &transition) - && transition.from.day() == tz_time.day() - && (transition.from.second() != 0 || transition.from.minute() % 900 != 0 - || (transition.from.day() != transition.to.day() - && (transition.from.hour() != 0 && transition.from.minute() != 0 && transition.from.second() != 0)))) + && (transition.from.day() == tz_time.day() || transition.to.day() == tz_time.day())) + { + has_transition = true; + } + + if (has_transition && (transition.from.second() != 0 || transition.from.minute() % 900 != 0)) { std::cerr << "Skipping " << timezone_name << " " << tz_time - << " because of unsupported timezone transition from " << transition.from << " to " << transition.to << "\n"; + << " because of unsupported timezone transition from " << transition.from << " to " << transition.to + << " (not divisable by 15 minutes)\n"; + continue; + } + + /// Transition to previous day, but not from midnight. + if (has_transition && cctz::civil_day(transition.from) == cctz::civil_day(transition.to) + 1 + && transition.from != cctz::civil_day(transition.from)) + { + std::cerr << "Skipping " << timezone_name << " " << tz_time + << " because of unsupported timezone transition from " << transition.from << " to " << transition.to + << " (to previous day but not at midnight)\n"; + continue; + } + + /// To large transition. + if (has_transition + && cctz::civil_day(transition.from) != cctz::civil_day(transition.to) + && cctz::civil_day(transition.from) != cctz::civil_day(transition.to) + 1) + { + std::cerr << "Skipping " << timezone_name << " " << tz_time + << " because of unsupported timezone transition from " << transition.from << " to " << transition.to + << " (it is too large)\n"; continue; } EXPECT_EQ(tz_time.year(), lut.toYear(expected_time_t)); EXPECT_EQ(tz_time.month(), lut.toMonth(expected_time_t)); EXPECT_EQ(tz_time.day(), lut.toDayOfMonth(expected_time_t)); - EXPECT_EQ(static_cast(cctz::get_weekday(tz_time)) + 1, lut.toDayOfWeek(expected_time_t)); // tm.tm_wday Sunday is 0, while for DateLUTImpl it is 7 + /// tm.tm_wday Sunday is 0, while for DateLUTImpl it is 7 + EXPECT_EQ(static_cast(cctz::get_weekday(tz_time)) + 1, lut.toDayOfWeek(expected_time_t)); EXPECT_EQ(cctz::get_yearday(tz_time), lut.toDayOfYear(expected_time_t)); EXPECT_EQ(tz_time.hour(), lut.toHour(expected_time_t)); EXPECT_EQ(tz_time.minute(), lut.toMinute(expected_time_t)); From d406999e8ff03b91ed5f4dbd4cf0077fe32fbbd1 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Sun, 7 Mar 2021 01:36:48 +0300 Subject: [PATCH 134/544] Remove debug output --- base/common/DateLUTImpl.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/base/common/DateLUTImpl.cpp b/base/common/DateLUTImpl.cpp index a111a21d8fd..9e7a7eab7d3 100644 --- a/base/common/DateLUTImpl.cpp +++ b/base/common/DateLUTImpl.cpp @@ -120,8 +120,8 @@ DateLUTImpl::DateLUTImpl(const std::string & time_zone_) values.time_at_offset_change_value = (transition.from - cctz::civil_second(date)) / Values::OffsetChangeFactor; values.amount_of_offset_change_value = (transition.to - transition.from) / Values::OffsetChangeFactor; - std::cerr << time_zone << ", " << date << ": change from " << transition.from << " to " << transition.to << "\n"; - std::cerr << time_zone << ", " << date << ": change at " << values.time_at_offset_change() << " with " << values.amount_of_offset_change() << "\n"; +// std::cerr << time_zone << ", " << date << ": change from " << transition.from << " to " << transition.to << "\n"; +// std::cerr << time_zone << ", " << date << ": change at " << values.time_at_offset_change() << " with " << values.amount_of_offset_change() << "\n"; /// We don't support too large changes. if (values.amount_of_offset_change_value > 24 * 4) From 8271cec093a71a319aac3a584f6c1501d9d2bdbe Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Sun, 7 Mar 2021 02:36:24 +0300 Subject: [PATCH 135/544] Fix error --- base/common/DateLUTImpl.cpp | 4 +--- src/Common/tests/gtest_DateLUTImpl.cpp | 7 +++---- 2 files changed, 4 insertions(+), 7 deletions(-) diff --git a/base/common/DateLUTImpl.cpp b/base/common/DateLUTImpl.cpp index 9e7a7eab7d3..6097763fb49 100644 --- a/base/common/DateLUTImpl.cpp +++ b/base/common/DateLUTImpl.cpp @@ -112,9 +112,7 @@ DateLUTImpl::DateLUTImpl(const std::string & time_zone_) /// Change in time zone without transition is possible, e.g. Moscow 1991 Sun, 31 Mar, 02:00 MSK to EEST cctz::time_zone::civil_transition transition{}; if (cctz_time_zone.next_transition(start_of_day_time_point - std::chrono::seconds(1), &transition) - && transition.from.year() == date.year() - && transition.from.month() == date.month() - && transition.from.day() == date.day() + && (cctz::civil_day(transition.from) == date || cctz::civil_day(transition.to) == date) && transition.from != transition.to) { values.time_at_offset_change_value = (transition.from - cctz::civil_second(date)) / Values::OffsetChangeFactor; diff --git a/src/Common/tests/gtest_DateLUTImpl.cpp b/src/Common/tests/gtest_DateLUTImpl.cpp index 80cb73f06c8..7d3d38df645 100644 --- a/src/Common/tests/gtest_DateLUTImpl.cpp +++ b/src/Common/tests/gtest_DateLUTImpl.cpp @@ -416,11 +416,11 @@ TEST_P(DateLUTWithTimeZoneAndTimeRange, InRange) has_transition = true; } - if (has_transition && (transition.from.second() != 0 || transition.from.minute() % 900 != 0)) + if (has_transition && (transition.from.second() != 0 || transition.from.minute() % 15 != 0)) { std::cerr << "Skipping " << timezone_name << " " << tz_time << " because of unsupported timezone transition from " << transition.from << " to " << transition.to - << " (not divisable by 15 minutes)\n"; + << " (not divisible by 15 minutes)\n"; continue; } @@ -436,8 +436,7 @@ TEST_P(DateLUTWithTimeZoneAndTimeRange, InRange) /// To large transition. if (has_transition - && cctz::civil_day(transition.from) != cctz::civil_day(transition.to) - && cctz::civil_day(transition.from) != cctz::civil_day(transition.to) + 1) + && std::abs(transition.from - transition.to) > 3600 * 3) { std::cerr << "Skipping " << timezone_name << " " << tz_time << " because of unsupported timezone transition from " << transition.from << " to " << transition.to From 1bb62f578b75541689eec40971b3a10d8630ed1a Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Sun, 7 Mar 2021 04:10:38 +0300 Subject: [PATCH 136/544] Unit tests passing --- base/common/DateLUTImpl.cpp | 6 ++-- base/common/DateLUTImpl.h | 31 ++++++++-------- src/Common/tests/gtest_DateLUTImpl.cpp | 50 ++++++++++++-------------- 3 files changed, 42 insertions(+), 45 deletions(-) diff --git a/base/common/DateLUTImpl.cpp b/base/common/DateLUTImpl.cpp index 6097763fb49..e7faeb63760 100644 --- a/base/common/DateLUTImpl.cpp +++ b/base/common/DateLUTImpl.cpp @@ -59,7 +59,7 @@ DateLUTImpl::DateLUTImpl(const std::string & time_zone_) offset_at_start_of_epoch = cctz_time_zone.lookup(cctz_time_zone.lookup(epoch).pre).offset; offset_at_start_of_lut = cctz_time_zone.lookup(cctz_time_zone.lookup(lut_start).pre).offset; - offset_is_whole_number_of_hours_everytime = true; + offset_is_whole_number_of_hours_during_epoch = true; cctz::civil_day date = lut_start; @@ -105,8 +105,8 @@ DateLUTImpl::DateLUTImpl(const std::string & time_zone_) values.time_at_offset_change_value = 0; values.amount_of_offset_change_value = 0; - if (offset_is_whole_number_of_hours_everytime && start_of_day > 0 && start_of_day % 3600) - offset_is_whole_number_of_hours_everytime = false; + if (offset_is_whole_number_of_hours_during_epoch && start_of_day > 0 && start_of_day % 3600) + offset_is_whole_number_of_hours_during_epoch = false; /// If UTC offset was changed this day. /// Change in time zone without transition is possible, e.g. Moscow 1991 Sun, 31 Mar, 02:00 MSK to EEST diff --git a/base/common/DateLUTImpl.h b/base/common/DateLUTImpl.h index e558a3b45ce..16abd3dfb0e 100644 --- a/base/common/DateLUTImpl.h +++ b/base/common/DateLUTImpl.h @@ -185,7 +185,7 @@ private: time_t offset_at_start_of_epoch; /// UTC offset at the beginning of the first supported year. time_t offset_at_start_of_lut; - bool offset_is_whole_number_of_hours_everytime; + bool offset_is_whole_number_of_hours_during_epoch; /// Time zone name. std::string time_zone; @@ -193,15 +193,19 @@ private: inline LUTIndex findIndex(time_t t) const { /// First guess. - const UInt32 guess = ((t / 86400) + daynum_offset_epoch) & date_lut_mask; + UInt32 guess = ((t / 86400) + daynum_offset_epoch) & date_lut_mask; + + /// For negative time_t the integer division was rounded up, so the guess is offset by one. + if (unlikely(t < 0)) + --guess; /// UTC offset is from -12 to +14 in all known time zones. This requires checking only three indices. - if (t >= lut[guess].date && t < lut[UInt32(guess + 1)].date) + if (t >= lut[guess].date && t < lut[guess + 1].date) return LUTIndex(guess); /// Time zones that have offset 0 from UTC do daylight saving time change (if any) /// towards increasing UTC offset (example: British Standard Time). - if (t >= lut[UInt32(guess + 1)].date) + if (t >= lut[guess + 1].date) return LUTIndex(guess + 1); return LUTIndex(guess - 1); @@ -253,7 +257,6 @@ public: // Methods only for unit-testing, it makes very little sense to use it from user code. auto getOffsetAtStartOfEpoch() const { return offset_at_start_of_epoch; } - auto getOffsetIsWholNumberOfHoursEveryWhere() const { return offset_is_whole_number_of_hours_everytime; } auto getTimeOffsetAtStartOfLUT() const { return offset_at_start_of_lut; } /// All functions below are thread-safe; arguments are not checked. @@ -456,8 +459,8 @@ public: inline unsigned toMinute(time_t t) const { - if (offset_is_whole_number_of_hours_everytime) - return ((t + DATE_LUT_ADD) / 60) % 60; + if (t >= 0 && offset_is_whole_number_of_hours_during_epoch) + return (t / 60) % 60; /// To consider the DST changing situation within this day /// also make the special timezones with no whole hour offset such as 'Australia/Lord_Howe' been taken into account. @@ -478,8 +481,8 @@ public: inline time_t toStartOfTenMinutes(time_t t) const { - if (offset_is_whole_number_of_hours_everytime) - return roundDown(t, 600); + if (t >= 0 && offset_is_whole_number_of_hours_during_epoch) + return t / 600 * 600; /// More complex logic is for Nepal - it has offset 05:45. Australia/Eucla is also unfortunate. Int64 date = find(t).date; @@ -489,8 +492,8 @@ public: /// NOTE: Assuming timezone transitions are multiple of hours. Lord Howe Island in Australia is a notable exception. inline time_t toStartOfHour(time_t t) const { - if (offset_is_whole_number_of_hours_everytime) - return roundDown(t, 3600); + if (t >= 0 && offset_is_whole_number_of_hours_during_epoch) + return t / 3600 * 3600; Int64 date = find(t).date; return date + (t - date) / 3600 * 3600; @@ -773,8 +776,8 @@ public: /// We count all hour-length intervals, unrelated to offset changes. inline time_t toRelativeHourNum(time_t t) const { - if (offset_is_whole_number_of_hours_everytime) - return (t + DATE_LUT_ADD) / 3600 - (DATE_LUT_ADD / 3600); + if (t >= 0 && offset_is_whole_number_of_hours_during_epoch) + return t / 3600; /// Assume that if offset was fractional, then the fraction is the same as at the beginning of epoch. /// NOTE This assumption is false for "Pacific/Pitcairn" and "Pacific/Kiritimati" time zones. @@ -848,7 +851,7 @@ public: t = roundDown(t, seconds); - if (offset_is_whole_number_of_hours_everytime) + if (t >= 0 && offset_is_whole_number_of_hours_during_epoch) return t; /// TODO check if it's correct. diff --git a/src/Common/tests/gtest_DateLUTImpl.cpp b/src/Common/tests/gtest_DateLUTImpl.cpp index 7d3d38df645..e18bd16e731 100644 --- a/src/Common/tests/gtest_DateLUTImpl.cpp +++ b/src/Common/tests/gtest_DateLUTImpl.cpp @@ -297,7 +297,6 @@ TEST_P(DateLUTWithTimeZone, VaidateTimeComponentsAroundEpoch) << "\n\tTimezone: " << timezone_name << "\n\ttimestamp: " << i << "\n\t offset at start of epoch : " << lut.getOffsetAtStartOfEpoch() - << "\n\t offset_is_whole_number_of_hours_everytime : " << lut.getOffsetIsWholNumberOfHoursEveryWhere() << "\n\t offset_at_start_of_lut : " << lut.getTimeOffsetAtStartOfLUT()); EXPECT_GE(24, lut.toHour(i)); @@ -336,7 +335,7 @@ INSTANTIATE_TEST_SUITE_P(ExoticTimezones, }) ); -INSTANTIATE_TEST_SUITE_P(DISABLED_AllTimeZones, +INSTANTIATE_TEST_SUITE_P(AllTimeZones, DateLUTWithTimeZone, ::testing::ValuesIn(allTimezones()) ); @@ -391,11 +390,15 @@ TEST_P(DateLUTWithTimeZoneAndTimeRange, InRange) { SCOPED_TRACE(expected_time_t); - const auto tz_time = cctz::convert(std::chrono::system_clock::from_time_t(expected_time_t), tz); + const cctz::civil_second tz_time = cctz::convert(std::chrono::system_clock::from_time_t(expected_time_t), tz); /// Weird offset, not supported. /// Example: Africa/Monrovia has offset UTC-0:44:30 in year 1970. - if (tz.lookup(std::chrono::system_clock::from_time_t(expected_time_t)).offset % 900) + + auto timestamp_current_day_pre = std::chrono::system_clock::to_time_t(tz.lookup(cctz::civil_day(tz_time)).pre); + auto timestamp_current_day_post = std::chrono::system_clock::to_time_t(tz.lookup(cctz::civil_day(tz_time) + 1).post); + + if (timestamp_current_day_pre % 900 || timestamp_current_day_post % 900) continue; /// Unsupported timezone transitions - not in 15-minute time point or to different day. @@ -410,7 +413,7 @@ TEST_P(DateLUTWithTimeZoneAndTimeRange, InRange) bool has_transition = false; cctz::time_zone::civil_transition transition{}; - if (tz.next_transition(std::chrono::system_clock::from_time_t(expected_time_t), &transition) + if (tz.next_transition(std::chrono::system_clock::from_time_t(expected_time_t - 1), &transition) && (transition.from.day() == tz_time.day() || transition.to.day() == tz_time.day())) { has_transition = true; @@ -418,9 +421,9 @@ TEST_P(DateLUTWithTimeZoneAndTimeRange, InRange) if (has_transition && (transition.from.second() != 0 || transition.from.minute() % 15 != 0)) { - std::cerr << "Skipping " << timezone_name << " " << tz_time + /*std::cerr << "Skipping " << timezone_name << " " << tz_time << " because of unsupported timezone transition from " << transition.from << " to " << transition.to - << " (not divisible by 15 minutes)\n"; + << " (not divisible by 15 minutes)\n";*/ continue; } @@ -428,9 +431,9 @@ TEST_P(DateLUTWithTimeZoneAndTimeRange, InRange) if (has_transition && cctz::civil_day(transition.from) == cctz::civil_day(transition.to) + 1 && transition.from != cctz::civil_day(transition.from)) { - std::cerr << "Skipping " << timezone_name << " " << tz_time + /*std::cerr << "Skipping " << timezone_name << " " << tz_time << " because of unsupported timezone transition from " << transition.from << " to " << transition.to - << " (to previous day but not at midnight)\n"; + << " (to previous day but not at midnight)\n";*/ continue; } @@ -438,9 +441,9 @@ TEST_P(DateLUTWithTimeZoneAndTimeRange, InRange) if (has_transition && std::abs(transition.from - transition.to) > 3600 * 3) { - std::cerr << "Skipping " << timezone_name << " " << tz_time + /*std::cerr << "Skipping " << timezone_name << " " << tz_time << " because of unsupported timezone transition from " << transition.from << " to " << transition.to - << " (it is too large)\n"; + << " (it is too large)\n";*/ continue; } @@ -457,23 +460,14 @@ TEST_P(DateLUTWithTimeZoneAndTimeRange, InRange) const auto time_string = cctz::format("%E4Y-%m-%d %H:%M:%S", std::chrono::system_clock::from_time_t(expected_time_t), tz); EXPECT_EQ(time_string, lut.timeToString(expected_time_t)); - // it makes sense to let test execute all checks above to simplify debugging, - // but once we've found a bad apple, no need to dig deeper. + /// It makes sense to let test execute all checks above to simplify debugging, + /// but once we've found a bad apple, no need to dig deeper. if (countFailures(*test_info->result()).total >= max_failures_per_case) break; } } -/** Next tests are disabled due to following reasons: - * 1. They are huge and take enormous amount of time to run - * 2. Current implementation of DateLUTImpl is inprecise and some cases fail and it seems impractical to try to fix those. - * 3. Many failures (~300) were fixed while refactoring, about ~40 remain the same and 3 new introduced: - * "Asia/Gaza" - * "Pacific/Enderbury" - * "Pacific/Kiritimati" - * So it would be tricky to skip knonw failures to allow all unit tests to pass. - */ -INSTANTIATE_TEST_SUITE_P(DISABLED_AllTimezones_Year2010, +INSTANTIATE_TEST_SUITE_P(AllTimezones_Year2010, DateLUTWithTimeZoneAndTimeRange, ::testing::Combine( ::testing::ValuesIn(allTimezones()), @@ -484,7 +478,7 @@ INSTANTIATE_TEST_SUITE_P(DISABLED_AllTimezones_Year2010, })) ); -INSTANTIATE_TEST_SUITE_P(DISABLED_AllTimezones_Year1970_WHOLE, +INSTANTIATE_TEST_SUITE_P(AllTimezones_Year1970_WHOLE, DateLUTWithTimeZoneAndTimeRange, ::testing::Combine( ::testing::ValuesIn(allTimezones(false)), @@ -494,7 +488,7 @@ INSTANTIATE_TEST_SUITE_P(DISABLED_AllTimezones_Year1970_WHOLE, })) ); -INSTANTIATE_TEST_SUITE_P(DISABLED_AllTimezones_Year2010_WHOLE, +INSTANTIATE_TEST_SUITE_P(AllTimezones_Year2010_WHOLE, DateLUTWithTimeZoneAndTimeRange, ::testing::Combine( ::testing::ValuesIn(allTimezones(false)), @@ -504,7 +498,7 @@ INSTANTIATE_TEST_SUITE_P(DISABLED_AllTimezones_Year2010_WHOLE, })) ); -INSTANTIATE_TEST_SUITE_P(DISABLED_AllTimezones_Year2020_WHOLE, +INSTANTIATE_TEST_SUITE_P(AllTimezones_Year2020_WHOLE, DateLUTWithTimeZoneAndTimeRange, ::testing::Combine( ::testing::ValuesIn(allTimezones()), @@ -514,7 +508,7 @@ INSTANTIATE_TEST_SUITE_P(DISABLED_AllTimezones_Year2020_WHOLE, })) ); -INSTANTIATE_TEST_SUITE_P(DISABLED_AllTimezones_PreEpoch, +INSTANTIATE_TEST_SUITE_P(AllTimezones_PreEpoch, DateLUTWithTimeZoneAndTimeRange, ::testing::Combine( ::testing::ValuesIn(allTimezones(false)), @@ -524,7 +518,7 @@ INSTANTIATE_TEST_SUITE_P(DISABLED_AllTimezones_PreEpoch, })) ); -INSTANTIATE_TEST_SUITE_P(DISABLED_AllTimezones_Year1970, +INSTANTIATE_TEST_SUITE_P(AllTimezones_Year1970, DateLUTWithTimeZoneAndTimeRange, ::testing::Combine( ::testing::ValuesIn(allTimezones(false)), From 7052ecd446ef637d03cba1ac618a6a85d642dcb3 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Sun, 7 Mar 2021 09:30:52 +0300 Subject: [PATCH 137/544] Fix error --- base/common/DateLUTImpl.h | 15 ++++----------- 1 file changed, 4 insertions(+), 11 deletions(-) diff --git a/base/common/DateLUTImpl.h b/base/common/DateLUTImpl.h index 16abd3dfb0e..b75274e8569 100644 --- a/base/common/DateLUTImpl.h +++ b/base/common/DateLUTImpl.h @@ -442,19 +442,12 @@ public: } - /** Only for time zones with/when offset from UTC is multiple of five minutes. - * This is true for all time zones: right now, all time zones have an offset that is multiple of 15 minutes. - * - * "By 1929, most major countries had adopted hourly time zones. Nepal was the last - * country to adopt a standard offset, shifting slightly to UTC+5:45 in 1986." - * - https://en.wikipedia.org/wiki/Time_zone#Offsets_from_UTC - * - * Also please note, that unix timestamp doesn't count "leap seconds": - * each minute, with added or subtracted leap second, spans exactly 60 unix timestamps. - */ inline unsigned toSecond(time_t t) const { - return (t + DATE_LUT_ADD) % 60; + auto res = t % 60; + if (likely(res >= 0)) + return res; + return res + 60; } inline unsigned toMinute(time_t t) const From 5cf42e87be76f0f30771f63a155986077168eb1d Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Sun, 7 Mar 2021 09:31:01 +0300 Subject: [PATCH 138/544] Update test --- .../01702_toDateTime_from_string_clamping.reference | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/queries/0_stateless/01702_toDateTime_from_string_clamping.reference b/tests/queries/0_stateless/01702_toDateTime_from_string_clamping.reference index 92639948fbc..644de54a6a2 100644 --- a/tests/queries/0_stateless/01702_toDateTime_from_string_clamping.reference +++ b/tests/queries/0_stateless/01702_toDateTime_from_string_clamping.reference @@ -2,8 +2,8 @@ SELECT toString(toDateTime('-922337203.6854775808', 1)); 1940-10-09 22:13:17.6 SELECT toString(toDateTime('9922337203.6854775808', 1)); -1925-07-26 00:46:43.6 +1925-07-26 23:46:43.6 SELECT toDateTime64(CAST('10000000000.1' AS Decimal64(1)), 1); -1928-01-11 00:46:40.1 +1928-01-11 23:46:40.1 SELECT toDateTime64(CAST('-10000000000.1' AS Decimal64(1)), 1); -2011-12-22 00:13:20.1 +2011-12-22 23:38:20.1 From d3b422a3360a8fa768bf910e74c8303486ac4858 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Sun, 7 Mar 2021 10:10:15 +0300 Subject: [PATCH 139/544] Update test --- .../0_stateless/01699_timezoneOffset.reference | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/tests/queries/0_stateless/01699_timezoneOffset.reference b/tests/queries/0_stateless/01699_timezoneOffset.reference index 45f30314f5a..a1cc6391e6f 100644 --- a/tests/queries/0_stateless/01699_timezoneOffset.reference +++ b/tests/queries/0_stateless/01699_timezoneOffset.reference @@ -1,8 +1,8 @@ DST boundary test for Europe/Moscow: -0 1981-04-01 22:40:00 10800 355002000 -1 1981-04-01 22:50:00 10800 355002600 -2 1981-04-02 00:00:00 14400 355003200 -3 1981-04-02 00:10:00 14400 355003800 +0 1981-04-01 22:40:00 14400 354998400 +1 1981-04-01 22:50:00 14400 354999000 +2 1981-04-01 23:00:00 14400 354999600 +3 1981-04-01 23:10:00 14400 355000200 0 1981-09-30 23:00:00 14400 370724400 1 1981-09-30 23:10:00 14400 370725000 2 1981-09-30 23:20:00 14400 370725600 @@ -22,10 +22,10 @@ DST boundary test for Europe/Moscow: 16 1981-10-01 00:40:00 10800 370734000 17 1981-10-01 00:50:00 10800 370734600 DST boundary test for Asia/Tehran: -0 2020-03-21 22:40:00 12600 1584817800 -1 2020-03-21 22:50:00 12600 1584818400 -2 2020-03-22 00:00:00 16200 1584819000 -3 2020-03-22 00:10:00 16200 1584819600 +0 2020-03-21 22:40:00 16200 1584814200 +1 2020-03-21 22:50:00 16200 1584814800 +2 2020-03-21 23:00:00 16200 1584815400 +3 2020-03-21 23:10:00 16200 1584816000 0 2020-09-20 23:00:00 16200 1600626600 1 2020-09-20 23:10:00 16200 1600627200 2 2020-09-20 23:20:00 16200 1600627800 From 1200d9d9c5447a900a692f170af894f02d43147d Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Sun, 7 Mar 2021 10:10:37 +0300 Subject: [PATCH 140/544] Range checks and monotonicity --- base/common/DateLUTImpl.h | 20 +++++++++++++------- 1 file changed, 13 insertions(+), 7 deletions(-) diff --git a/base/common/DateLUTImpl.h b/base/common/DateLUTImpl.h index b75274e8569..6e968a0cd50 100644 --- a/base/common/DateLUTImpl.h +++ b/base/common/DateLUTImpl.h @@ -193,22 +193,28 @@ private: inline LUTIndex findIndex(time_t t) const { /// First guess. - UInt32 guess = ((t / 86400) + daynum_offset_epoch) & date_lut_mask; + Int64 guess = (t / 86400) + daynum_offset_epoch; /// For negative time_t the integer division was rounded up, so the guess is offset by one. if (unlikely(t < 0)) --guess; + if (guess < 0) + return LUTIndex(0); + if (guess >= DATE_LUT_SIZE) + return LUTIndex(DATE_LUT_SIZE - 1); + /// UTC offset is from -12 to +14 in all known time zones. This requires checking only three indices. - if (t >= lut[guess].date && t < lut[guess + 1].date) - return LUTIndex(guess); - /// Time zones that have offset 0 from UTC do daylight saving time change (if any) - /// towards increasing UTC offset (example: British Standard Time). - if (t >= lut[guess + 1].date) + if (t >= lut[guess].date) + { + if (guess + 1 >= DATE_LUT_SIZE || t < lut[guess + 1].date) + return LUTIndex(guess); + return LUTIndex(guess + 1); + } - return LUTIndex(guess - 1); + return LUTIndex(guess ? guess - 1 : 0); } inline LUTIndex toLUTIndex(DayNum d) const From 8874d2e063dfaff9bab52196c836f120d08c953f Mon Sep 17 00:00:00 2001 From: Vasily Nemkov Date: Fri, 5 Mar 2021 12:06:49 +0200 Subject: [PATCH 141/544] Avoid running 01702_system_query_log.sql in parallel Shouldn't be run in parallel with itself, since it relies on content of system.query_log --- tests/queries/skip_list.json | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/queries/skip_list.json b/tests/queries/skip_list.json index 36cca55779d..886fb0f9501 100644 --- a/tests/queries/skip_list.json +++ b/tests/queries/skip_list.json @@ -757,6 +757,7 @@ "memory_limit", "polygon_dicts", // they use an explicitly specified database "01658_read_file_to_stringcolumn", - "01721_engine_file_truncate_on_insert" // It's ok to execute in parallel but not several instances of the same test. + "01721_engine_file_truncate_on_insert", // It's ok to execute in parallel but not several instances of the same test. + "01702_system_query_log" // It's ok to execute in parallel but not several instances of the same test. ] } From 019ed517bc33892d107480ee30ddf53de8fd4fee Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Mon, 8 Mar 2021 03:05:06 +0300 Subject: [PATCH 142/544] Fix gcc warning --- src/Dictionaries/RangeHashedDictionary.cpp | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/Dictionaries/RangeHashedDictionary.cpp b/src/Dictionaries/RangeHashedDictionary.cpp index 9fb1a57a381..aec641c58fe 100644 --- a/src/Dictionaries/RangeHashedDictionary.cpp +++ b/src/Dictionaries/RangeHashedDictionary.cpp @@ -491,8 +491,9 @@ void RangeHashedDictionary::getIdsAndDates( start_dates.push_back(value.range.left); end_dates.push_back(value.range.right); - if (is_date && static_cast(end_dates.back()) > DATE_LUT_MAX_DAY_NUM) - end_dates.back() = 0; + if constexpr (std::numeric_limits::max() > DATE_LUT_MAX_DAY_NUM) /// Avoid warning about tautological comparison in next line. + if (is_date && static_cast(end_dates.back()) > DATE_LUT_MAX_DAY_NUM) + end_dates.back() = 0; } } } From 6b309dcc5c7da70bc9896ff3047afa1f7aff6955 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Mon, 8 Mar 2021 03:14:39 +0300 Subject: [PATCH 143/544] Update tests --- tests/queries/0_stateless/00189_time_zones.reference | 7 ++++--- tests/queries/0_stateless/00189_time_zones.sql | 5 +++-- .../queries/0_stateless/01691_DateTime64_clamp.reference | 8 ++++---- .../01702_toDateTime_from_string_clamping.reference | 6 +++--- 4 files changed, 14 insertions(+), 12 deletions(-) diff --git a/tests/queries/0_stateless/00189_time_zones.reference b/tests/queries/0_stateless/00189_time_zones.reference index 664c30056de..e7e5a71782a 100644 --- a/tests/queries/0_stateless/00189_time_zones.reference +++ b/tests/queries/0_stateless/00189_time_zones.reference @@ -179,13 +179,13 @@ toRelativeYearNum 44 44 44 -44 +45 toRelativeMonthNum 536 536 536 537 -536 +537 toRelativeWeekNum 2335 2335 @@ -197,12 +197,13 @@ toRelativeDayNum 16343 16343 16344 -16343 +16344 toRelativeHourNum 392251 392251 392251 392251 +392252 toRelativeMinuteNum 23535110 23535110 diff --git a/tests/queries/0_stateless/00189_time_zones.sql b/tests/queries/0_stateless/00189_time_zones.sql index a0ef5b59517..36c7dfb402a 100644 --- a/tests/queries/0_stateless/00189_time_zones.sql +++ b/tests/queries/0_stateless/00189_time_zones.sql @@ -277,7 +277,8 @@ SELECT toRelativeDayNum(toDateTime(1412106600), 'Europe/Moscow') - toRelativeDay SELECT toRelativeDayNum(toDateTime(1412106600), 'Europe/Paris') - toRelativeDayNum(toDateTime(0), 'Europe/Paris'); SELECT toRelativeDayNum(toDateTime(1412106600), 'Europe/London') - toRelativeDayNum(toDateTime(0), 'Europe/London'); SELECT toRelativeDayNum(toDateTime(1412106600), 'Asia/Tokyo') - toRelativeDayNum(toDateTime(0), 'Asia/Tokyo'); -SELECT toRelativeDayNum(toDateTime(1412106600), 'Pacific/Pitcairn') - toRelativeDayNum(toDateTime(0), 'Pacific/Pitcairn'); +-- NOTE: toRelativeDayNum(toDateTime(0), 'Pacific/Pitcairn') overflows from -1 to 65535 +SELECT toUInt16(toRelativeDayNum(toDateTime(1412106600), 'Pacific/Pitcairn') - toRelativeDayNum(toDateTime(0), 'Pacific/Pitcairn')); /* toRelativeHourNum */ @@ -286,7 +287,7 @@ SELECT toRelativeHourNum(toDateTime(1412106600), 'Europe/Moscow') - toRelativeHo SELECT toRelativeHourNum(toDateTime(1412106600), 'Europe/Paris') - toRelativeHourNum(toDateTime(0), 'Europe/Paris'); SELECT toRelativeHourNum(toDateTime(1412106600), 'Europe/London') - toRelativeHourNum(toDateTime(0), 'Europe/London'); SELECT toRelativeHourNum(toDateTime(1412106600), 'Asia/Tokyo') - toRelativeHourNum(toDateTime(0), 'Asia/Tokyo'); --- known wrong result: SELECT toRelativeHourNum(toDateTime(1412106600), 'Pacific/Pitcairn') - toRelativeHourNum(toDateTime(0), 'Pacific/Pitcairn'); +SELECT toRelativeHourNum(toDateTime(1412106600), 'Pacific/Pitcairn') - toRelativeHourNum(toDateTime(0), 'Pacific/Pitcairn'); /* toRelativeMinuteNum */ diff --git a/tests/queries/0_stateless/01691_DateTime64_clamp.reference b/tests/queries/0_stateless/01691_DateTime64_clamp.reference index f29a9e2d1d5..881ab4feff8 100644 --- a/tests/queries/0_stateless/01691_DateTime64_clamp.reference +++ b/tests/queries/0_stateless/01691_DateTime64_clamp.reference @@ -17,11 +17,11 @@ SELECT toDateTime64(toFloat32(bitShiftLeft(toUInt64(1),33)), 2, 'Europe/Moscow') 2106-02-07 09:28:16.00 SELECT toDateTime64(toFloat64(bitShiftLeft(toUInt64(1),33)), 2, 'Europe/Moscow') FORMAT Null; -- These are outsize of extended range and hence clamped -SELECT toDateTime64(-1 * bitShiftLeft(toUInt64(1),35), 2); +SELECT toDateTime64(-1 * bitShiftLeft(toUInt64(1), 35), 2); 1925-01-01 02:00:00.00 -SELECT CAST(-1 * bitShiftLeft(toUInt64(1),35) AS DateTime64); +SELECT CAST(-1 * bitShiftLeft(toUInt64(1), 35) AS DateTime64); 1925-01-01 02:00:00.000 -SELECT CAST(bitShiftLeft(toUInt64(1),35) AS DateTime64); +SELECT CAST(bitShiftLeft(toUInt64(1), 35) AS DateTime64); 2282-12-31 03:00:00.000 -SELECT toDateTime64(bitShiftLeft(toUInt64(1),35), 2); +SELECT toDateTime64(bitShiftLeft(toUInt64(1), 35), 2); 2282-12-31 03:00:00.00 diff --git a/tests/queries/0_stateless/01702_toDateTime_from_string_clamping.reference b/tests/queries/0_stateless/01702_toDateTime_from_string_clamping.reference index 644de54a6a2..77da114be68 100644 --- a/tests/queries/0_stateless/01702_toDateTime_from_string_clamping.reference +++ b/tests/queries/0_stateless/01702_toDateTime_from_string_clamping.reference @@ -2,8 +2,8 @@ SELECT toString(toDateTime('-922337203.6854775808', 1)); 1940-10-09 22:13:17.6 SELECT toString(toDateTime('9922337203.6854775808', 1)); -1925-07-26 23:46:43.6 +2283-11-11 23:46:43.6 SELECT toDateTime64(CAST('10000000000.1' AS Decimal64(1)), 1); -1928-01-11 23:46:40.1 +2283-11-11 23:46:40.1 SELECT toDateTime64(CAST('-10000000000.1' AS Decimal64(1)), 1); -2011-12-22 23:38:20.1 +1925-01-01 23:09:20.1 From 460658aeabb8d9276bd3c5f620d09b4ca1fc601c Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Mon, 8 Mar 2021 03:18:55 +0300 Subject: [PATCH 144/544] Update tests --- src/Storages/StorageGenerateRandom.cpp | 2 +- .../01087_table_function_generate.reference | 60 +++++++++---------- .../01125_generate_random_qoega.reference | 2 +- .../01128_generate_random_nested.reference | 4 +- 4 files changed, 34 insertions(+), 34 deletions(-) diff --git a/src/Storages/StorageGenerateRandom.cpp b/src/Storages/StorageGenerateRandom.cpp index fd10691ecc4..f06daa3a2bd 100644 --- a/src/Storages/StorageGenerateRandom.cpp +++ b/src/Storages/StorageGenerateRandom.cpp @@ -215,7 +215,7 @@ ColumnPtr fillColumnWithRandomData( column->getData().resize(limit); for (size_t i = 0; i < limit; ++i) - column->getData()[i] = rng() % (DATE_LUT_MAX_DAY_NUM + 1); /// Slow + column->getData()[i] = rng() % (DATE_LUT_MAX_DAY_NUM + 1); return column; } diff --git a/tests/queries/0_stateless/01087_table_function_generate.reference b/tests/queries/0_stateless/01087_table_function_generate.reference index d7cc6b0a933..d8886945caa 100644 --- a/tests/queries/0_stateless/01087_table_function_generate.reference +++ b/tests/queries/0_stateless/01087_table_function_generate.reference @@ -1,14 +1,14 @@ UInt64 Int64 UInt32 Int32 UInt16 Int16 UInt8 Int8 -2804162938822577320 -2776833771540858 3467776823 1163715250 31161 -2916 220 -117 -7885388429666205427 -1363628932535403038 484159052 -308788249 43346 13638 143 -105 -4357435422797280898 1355609803008819271 4126129912 -852056475 34184 9166 49 33 -5935810273536892891 -804738887697332962 3109335413 -80126721 47877 -31421 186 -77 -368066018677693974 -4927165984347126295 1015254922 2026080544 46037 -29626 240 108 -8124171311239967992 -1179703908046100129 1720727300 -138469036 33028 -12819 138 16 -15657812979985370729 -5733276247123822513 3254757884 -500590428 3829 30527 3 -81 -18371568619324220532 -6793779541583578394 1686821450 -455892108 43475 2284 252 -90 -821735343441964030 3148260644406230976 256251035 -885069056 11643 11455 176 90 -9558594037060121162 -2907172753635797124 4276198376 1947296644 45922 26632 97 43 +2804162938822577320 -2776833771540858 3467776823 1163715250 23903 -2916 220 -117 +7885388429666205427 -1363628932535403038 484159052 -308788249 44305 13638 143 -105 +4357435422797280898 1355609803008819271 4126129912 -852056475 58858 9166 49 33 +5935810273536892891 -804738887697332962 3109335413 -80126721 13655 -31421 186 -77 +368066018677693974 -4927165984347126295 1015254922 2026080544 21973 -29626 240 108 +8124171311239967992 -1179703908046100129 1720727300 -138469036 36175 -12819 138 16 +15657812979985370729 -5733276247123822513 3254757884 -500590428 13193 30527 3 -81 +18371568619324220532 -6793779541583578394 1686821450 -455892108 52282 2284 252 -90 +821735343441964030 3148260644406230976 256251035 -885069056 55255 11455 176 90 +9558594037060121162 -2907172753635797124 4276198376 1947296644 48701 26632 97 43 - Enum8(\'hello\' = 1, \'world\' = 5) hello @@ -47,16 +47,16 @@ h o - Date DateTime DateTime(\'Europe/Moscow\') -2077-09-17 1970-10-09 02:30:14 2074-08-12 11:31:27 -2005-11-19 2106-01-30 21:52:44 2097-05-25 07:54:35 -2007-02-24 2096-12-12 00:40:50 1988-08-10 11:16:31 -2019-06-30 2096-01-15 16:31:33 2063-10-20 08:48:17 -2039-01-16 2103-02-11 16:44:39 2036-10-09 04:29:10 -1994-11-03 1980-01-02 05:18:22 2055-12-23 12:33:52 -2083-08-20 2079-06-11 16:29:02 2000-12-05 17:46:24 -2030-06-25 2100-03-01 18:50:22 1993-03-25 01:19:12 -2087-03-16 2034-08-25 19:46:33 2045-12-10 16:47:40 -2006-04-30 2069-09-30 16:07:48 2084-08-26 03:33:12 +2113-06-12 1970-10-09 02:30:14 2074-08-12 11:31:27 +2103-11-03 2106-01-30 21:52:44 2097-05-25 07:54:35 +2008-03-16 2096-12-12 00:40:50 1988-08-10 11:16:31 +2126-11-26 2096-01-15 16:31:33 2063-10-20 08:48:17 +1991-02-02 2103-02-11 16:44:39 2036-10-09 04:29:10 +2096-11-03 1980-01-02 05:18:22 2055-12-23 12:33:52 +2024-12-16 2079-06-11 16:29:02 2000-12-05 17:46:24 +2085-04-07 2100-03-01 18:50:22 1993-03-25 01:19:12 +2135-05-30 2034-08-25 19:46:33 2045-12-10 16:47:40 +2094-12-18 2069-09-30 16:07:48 2084-08-26 03:33:12 - DateTime64(3) DateTime64(6) DateTime64(6, \'Europe/Moscow\') 1978-06-07 23:50:57.320 2013-08-28 10:21:54.010758 1991-08-25 16:23:26.140215 @@ -225,14 +225,14 @@ RL,{Xs\\tw [114] -84125.1554 ('2023-06-06 06:55:06.492','bf9ab359-ef9f-ad11-7e6c-160368b1e5ea') [124] -114719.5228 ('2010-11-11 22:57:23.722','c1046ffb-3415-cc3a-509a-e0005856d7d7') - -[] 1900051923 { -189530.5846 h -5.6279699579452485e47 ('1980-08-29','2090-10-31 19:35:45','2038-07-15 05:22:51.805','63d9a12d-d1cf-1f3a-57c6-9bc6dddd0975') 8502 -[-102,-118] 392272782 Eb -14818.0200 o -2.664492247169164e59 ('2059-02-10','1994-07-16 00:40:02','2034-02-02 05:30:44.960','4fa09948-d32e-8903-63df-43ad759e43f7') DA61 -[-71] 775049089 \N -158115.1178 w 4.1323844687113747e-305 ('1997-02-15','2062-08-12 23:41:53','2074-02-13 10:29:40.749','c4a44dd7-d009-6f65-1494-9daedfa8a124') 83A7 -[-28,100] 3675466147 { -146685.1749 h 3.6676044396877755e142 ('1997-10-26','2002-06-26 03:33:41','2002-12-02 05:46:03.455','98714b2c-65e7-b5cb-a040-421e260c6d8d') 4B94 -[-23] 2514120753 (`u, -119659.6174 w 1.3231258347475906e34 ('2055-11-20','2080-03-28 08:11:25','2073-07-10 12:19:58.146','003b3b6b-088f-f941-aeb9-c26e0ee72b8e') 6B1F -[11,-36] 3308237300 \N 171205.1896 \N 5.634708707075817e195 ('2009-03-18','2041-11-11 13:19:44','2044-03-18 17:34:17.814','9e60f4cb-6e55-1deb-5ac4-d66a86a8886d') 1964 -[39] 1614362420 `4A8P 157144.0630 o -1.1843143253872814e-255 ('1991-04-27','2066-03-02 11:07:49','1997-10-22 20:14:13.755','97685503-2609-d2b9-981c-02fd75d106cb') A35B -[48,-120] 3848918261 1 Date: Mon, 8 Mar 2021 03:28:25 +0300 Subject: [PATCH 145/544] Try to enable long performance test --- tests/performance/date_time_long.xml | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/performance/date_time_long.xml b/tests/performance/date_time_long.xml index 1229631a434..3a61a5992e5 100644 --- a/tests/performance/date_time_long.xml +++ b/tests/performance/date_time_long.xml @@ -1,5 +1,4 @@ - long datetime_transform From 831135432fa3662e280fccefa52a10769f6e3b17 Mon Sep 17 00:00:00 2001 From: Slach Date: Mon, 22 Feb 2021 20:41:23 +0500 Subject: [PATCH 146/544] add PostgreSQL engine and table function documentation Signed-off-by: Slach --- .../table-engines/integrations/postgresql.md | 102 ++++++++++++++++ .../external-dicts-dict-lifetime.md | 4 +- .../external-dicts-dict-sources.md | 54 ++++++++- .../table-functions/postgresql.md | 100 ++++++++++++++++ .../table-engines/integrations/mysql.md | 4 +- .../table-engines/integrations/postgresql.md | 102 ++++++++++++++++ .../external-dicts-dict-lifetime.md | 17 +-- .../external-dicts-dict-sources.md | 113 +++++++++++++++++- .../table-functions/postgresql.md | 99 +++++++++++++++ 9 files changed, 578 insertions(+), 17 deletions(-) create mode 100644 docs/en/engines/table-engines/integrations/postgresql.md create mode 100644 docs/en/sql-reference/table-functions/postgresql.md create mode 100644 docs/ru/engines/table-engines/integrations/postgresql.md create mode 100644 docs/ru/sql-reference/table-functions/postgresql.md diff --git a/docs/en/engines/table-engines/integrations/postgresql.md b/docs/en/engines/table-engines/integrations/postgresql.md new file mode 100644 index 00000000000..4a81251e60a --- /dev/null +++ b/docs/en/engines/table-engines/integrations/postgresql.md @@ -0,0 +1,102 @@ +--- +toc_priority: 8 +toc_title: PostgreSQL +--- + +# PosgtreSQL {#postgresql} + +The PostgreSQL engine allows you to perform `SELECT` queries on data that is stored on a remote PostgreSQL server. + +## Creating a Table {#creating-a-table} + +``` sql +CREATE TABLE [IF NOT EXISTS] [db.]table_name [ON CLUSTER cluster] +( + name1 [type1] [DEFAULT|MATERIALIZED|ALIAS expr1] [TTL expr1], + name2 [type2] [DEFAULT|MATERIALIZED|ALIAS expr2] [TTL expr2], + ... +) ENGINE = PostgreSQL('host:port', 'database', 'table', 'user', 'password'); +``` + +See a detailed description of the [CREATE TABLE](../../../sql-reference/statements/create/table.md#create-table-query) query. + +The table structure can differ from the original PostgreSQL table structure: + +- Column names should be the same as in the original PostgreSQL table, but you can use just some of these columns and in any order. +- Column types may differ from those in the original PostgreSQL table. ClickHouse tries to [cast](../../../sql-reference/functions/type-conversion-functions.md#type_conversion_function-cast) values to the ClickHouse data types. + +**Engine Parameters** + +- `host:port` — PostgreSQL server address. + +- `database` — Remote database name. + +- `table` — Remote table name. + +- `user` — PostgreSQL user. + +- `password` — User password. + +SELECT Queries on PostgreSQL side run as `COPY (SELECT ...) TO STDOUT` inside read-only PostgreSQL transaction with commit after each `SELECT` query. + +Simple `WHERE` clauses such as `=, !=, >, >=, <, <=, IN` are executed on the PostgreSQL server. + +All joins, aggregations, sorting, `IN [ array ]` conditions and the `LIMIT` sampling constraint are executed in ClickHouse only after the query to PostgreSQL finishes. + +INSERT Queries on PostgreSQL side run as `COPY "table_name" (field1, field2, ... fieldN) FROM STDIN` inside PostgreSQL transaction with auto-commit after each `INSERT` statement. + +PostgreSQL Array types converts into ClickHouse arrays. + +## Usage Example {#usage-example} + +Table in PostgreSQL: + +``` text +postgres=# CREATE TABLE "public"."test" ( +"int_id" SERIAL, +"int_nullable" INT NULL DEFAULT NULL, +"float" FLOAT NOT NULL, +"str" VARCHAR(100) NOT NULL DEFAULT '', +"float_nullable" FLOAT NULL DEFAULT NULL, +PRIMARY KEY (int_id)); + +CREATE TABLE + +postgres=# insert into test (int_id, str, "float") VALUES (1,'test',2); +INSERT 0 1 + +postgresql> select * from test; + int_id | int_nullable | float | str | float_nullable +--------+--------------+-------+------+---------------- + 1 | | 2 | test | +(1 row) +``` + +Table in ClickHouse, retrieving data from the PostgreSQL table created above: + +``` sql +CREATE TABLE default.postgresql_table +( + `float_nullable` Nullable(Float32), + `str` String, + `int_id` Int32 +) +ENGINE = PostgreSQL('localhost:5432', 'public', 'test', 'postges_user', 'postgres_password'); +``` + +``` sql +SELECT * FROM postgresql_table WHERE str IN ('test') +``` + +``` text +┌─float_nullable─┬─str──┬─int_id─┐ +│ ᴺᵁᴸᴸ │ test │ 1 │ +└────────────────┴──────┴────────┘ +1 rows in set. Elapsed: 0.019 sec. +``` + + +## See Also {#see-also} + +- [The ‘postgresql’ table function](../../../sql-reference/table-functions/postgresql.md) +- [Using PostgreSQL as a source of external dictionary](../../../sql-reference/dictionaries/external-dictionaries/external-dicts-dict-sources.md#dicts-external_dicts_dict_sources-postgresql) diff --git a/docs/en/sql-reference/dictionaries/external-dictionaries/external-dicts-dict-lifetime.md b/docs/en/sql-reference/dictionaries/external-dictionaries/external-dicts-dict-lifetime.md index 20486ebbcc8..32763e27ddd 100644 --- a/docs/en/sql-reference/dictionaries/external-dictionaries/external-dicts-dict-lifetime.md +++ b/docs/en/sql-reference/dictionaries/external-dictionaries/external-dicts-dict-lifetime.md @@ -19,6 +19,8 @@ Example of settings: ``` +or + ``` sql CREATE DICTIONARY (...) ... @@ -58,7 +60,7 @@ When upgrading the dictionaries, the ClickHouse server applies different logic d - For MySQL source, the time of modification is checked using a `SHOW TABLE STATUS` query (in case of MySQL 8 you need to disable meta-information caching in MySQL by `set global information_schema_stats_expiry=0`. - Dictionaries from other sources are updated every time by default. -For other sources (ODBC, ClickHouse, etc), you can set up a query that will update the dictionaries only if they really changed, rather than each time. To do this, follow these steps: +For other sources (ODBC, PostgreSQL, ClickHouse, etc), you can set up a query that will update the dictionaries only if they really changed, rather than each time. To do this, follow these steps: - The dictionary table must have a field that always changes when the source data is updated. - The settings of the source must specify a query that retrieves the changing field. The ClickHouse server interprets the query result as a row, and if this row has changed relative to its previous state, the dictionary is updated. Specify the query in the `` field in the settings for the [source](../../../sql-reference/dictionaries/external-dictionaries/external-dicts-dict-sources.md). diff --git a/docs/en/sql-reference/dictionaries/external-dictionaries/external-dicts-dict-sources.md b/docs/en/sql-reference/dictionaries/external-dictionaries/external-dicts-dict-sources.md index 7cd26a9dffb..f8f4745bb16 100644 --- a/docs/en/sql-reference/dictionaries/external-dictionaries/external-dicts-dict-sources.md +++ b/docs/en/sql-reference/dictionaries/external-dictionaries/external-dicts-dict-sources.md @@ -65,6 +65,7 @@ Types of sources (`source_type`): - DBMS - [ODBC](#dicts-external_dicts_dict_sources-odbc) - [MySQL](#dicts-external_dicts_dict_sources-mysql) + - [PostgreSQL](#dicts-external_dicts_dict_sources-postgresql) - [ClickHouse](#dicts-external_dicts_dict_sources-clickhouse) - [MongoDB](#dicts-external_dicts_dict_sources-mongodb) - [Redis](#dicts-external_dicts_dict_sources-redis) @@ -659,7 +660,7 @@ Example of settings: Setting fields: - `host` – The Cassandra host or comma-separated list of hosts. -- `port` – The port on the Cassandra servers. If not specified, default port is used. +- `port` – The port on the Cassandra servers. If not specified, default port 9042 is used. - `user` – Name of the Cassandra user. - `password` – Password of the Cassandra user. - `keyspace` – Name of the keyspace (database). @@ -673,4 +674,55 @@ Default value is 1 (the first key column is a partition key and other key column - `where` – Optional selection criteria. - `max_threads` – The maximum number of threads to use for loading data from multiple partitions in compose key dictionaries. +### PosgreSQL {#dicts-external_dicts_dict_sources-postgresql} + +Example of settings: + +``` xml + + + 5432 + clickhouse + qwerty + db_name + table_name
+ id=10 + SQL_QUERY + + +``` + +or + +``` sql +SOURCE(POSTGRESQL( + port 5432 + host 'postgresql-hostname' + user 'postgres_user' + password 'postgres_password' + db 'db_name' + table 'table_name' + replica(host 'example01-1' port 5432 priority 1) + replica(host 'example01-2' port 5432 priority 2) + where 'id=10' + invalidate_query 'SQL_QUERY' +)) +``` + +Setting fields: + +- `host` – The host on the PostgreSQL server. You can specify it for all replicas, or for each one individually (inside ``). +- `port` – The port on the PostgreSQL server. You can specify it for all replicas, or for each one individually (inside ``). +- `user` – Name of the PostgreSQL user. You can specify it for all replicas, or for each one individually (inside ``). +- `password` – Password of the PostgreSQL user. You can specify it for all replicas, or for each one individually (inside ``). +- `replica` – Section of replica configurations. There can be multiple sections. + - `replica/host` – The PostgreSQL host. + - `replica/port` – The PostgreSQL port. + - `replica/priority` – The replica priority. When attempting to connect, ClickHouse traverses the replicas in order of priority. The lower the number, the higher the priority. +- `db` – Name of the database. +- `table` – Name of the table. +- `where` – The selection criteria. The syntax for conditions is the same as for `WHERE` clause in PostgreSQL, for example, `id > 10 AND id < 20`. Optional parameter. +- `invalidate_query` – Query for checking the dictionary status. Optional parameter. Read more in the section [Updating dictionaries](../../../sql-reference/dictionaries/external-dictionaries/external-dicts-dict-lifetime.md). + + [Original article](https://clickhouse.tech/docs/en/query_language/dicts/external_dicts_dict_sources/) diff --git a/docs/en/sql-reference/table-functions/postgresql.md b/docs/en/sql-reference/table-functions/postgresql.md new file mode 100644 index 00000000000..95724694d7a --- /dev/null +++ b/docs/en/sql-reference/table-functions/postgresql.md @@ -0,0 +1,100 @@ +--- +toc_priority: 42 +toc_title: postgresql +--- + +# postgresql {#postgresql} + +Allows `SELECT` and `INSERT` queries to be performed on data that is stored on a remote PostgreSQL server. + +**Syntax** + +``` sql +postgresql('host:port', 'database', 'table', 'user', 'password') +``` + +**Arguments** + +- `host:port` — PostgreSQL server address. + +- `database` — Remote database name. + +- `table` — Remote table name. + +- `user` — PostgreSQL user. + +- `password` — User password. + + +SELECT Queries on PostgreSQL side run as `COPY (SELECT ...) TO STDOUT` inside read-only PostgreSQL transaction with commit after each `SELECT` query. + +Simple `WHERE` clauses such as `=, !=, >, >=, <, <=, IN` are executed on the PostgreSQL server. + +All joins, aggregations, sorting, `IN [ array ]` conditions and the `LIMIT` sampling constraint are executed in ClickHouse only after the query to PostgreSQL finishes. + +INSERT Queries on PostgreSQL side run as `COPY "table_name" (field1, field2, ... fieldN) FROM STDIN` inside PostgreSQL transaction with auto-commit after each `INSERT` statement. + +PostgreSQL Array types converts into ClickHouse arrays. + +**Returned Value** + +A table object with the same columns as the original PostgreSQL table. + +!!! info "Note" + In the `INSERT` query to distinguish table function `postgresql(...)` from table name with column names list you must use keywords `FUNCTION` or `TABLE FUNCTION`. See examples below. + +**Examples** + +Table in PostgreSQL: + +``` text +postgres=# CREATE TABLE "public"."test" ( +"int_id" SERIAL, +"int_nullable" INT NULL DEFAULT NULL, +"float" FLOAT NOT NULL, +"str" VARCHAR(100) NOT NULL DEFAULT '', +"float_nullable" FLOAT NULL DEFAULT NULL, +PRIMARY KEY (int_id)); + +CREATE TABLE + +postgres=# insert into test (int_id, str, "float") VALUES (1,'test',2); +INSERT 0 1 + +postgresql> select * from test; + int_id | int_nullable | float | str | float_nullable +--------+--------------+-------+------+---------------- + 1 | | 2 | test | +(1 row) +``` + +Selecting data from ClickHouse: + +```sql +SELECT * FROM postgresql('localhost:5432', 'test', 'test', 'postgresql_user', 'password') WHERE str IN ('test'); +``` + +``` text +┌─int_id─┬─int_nullable─┬─float─┬─str──┬─float_nullable─┐ +│ 1 │ ᴺᵁᴸᴸ │ 2 │ test │ ᴺᵁᴸᴸ │ +└────────┴──────────────┴───────┴──────┴────────────────┘ +``` + +Inserting: + +```sql +INSERT INTO TABLE FUNCTION postgresql('localhost:5432', 'test', 'test', 'postgrsql_user', 'password') (int_id, float) VALUES (2, 3); +SELECT * FROM postgresql('localhost:5432', 'test', 'test', 'postgresql_user', 'password'); +``` + +``` text +┌─int_id─┬─int_nullable─┬─float─┬─str──┬─float_nullable─┐ +│ 1 │ ᴺᵁᴸᴸ │ 2 │ test │ ᴺᵁᴸᴸ │ +│ 2 │ ᴺᵁᴸᴸ │ 3 │ │ ᴺᵁᴸᴸ │ +└────────┴──────────────┴───────┴──────┴────────────────┘ +``` + +**See Also** + +- [The ‘PostgreSQL’ table engine](../../engines/table-engines/integrations/postgresql.md) +- [Using PostgreSQL as a source of external dictionary](../../sql-reference/dictionaries/external-dictionaries/external-dicts-dict-sources.md#dicts-external_dicts_dict_sources-postgresql) diff --git a/docs/ru/engines/table-engines/integrations/mysql.md b/docs/ru/engines/table-engines/integrations/mysql.md index 3370e9b06d0..2254ddcb68c 100644 --- a/docs/ru/engines/table-engines/integrations/mysql.md +++ b/docs/ru/engines/table-engines/integrations/mysql.md @@ -18,12 +18,12 @@ CREATE TABLE [IF NOT EXISTS] [db.]table_name [ON CLUSTER cluster] ) ENGINE = MySQL('host:port', 'database', 'table', 'user', 'password'[, replace_query, 'on_duplicate_clause']); ``` -Смотрите подробное описание запроса [CREATE TABLE](../../../engines/table-engines/integrations/mysql.md#create-table-query). +Смотрите подробное описание запроса [CREATE TABLE](../../../sql-reference/statements/create/table.md#create-table-query). Структура таблицы может отличаться от исходной структуры таблицы MySQL: - Имена столбцов должны быть такими же, как в исходной таблице MySQL, но вы можете использовать только некоторые из этих столбцов и в любом порядке. -- Типы столбцов могут отличаться от типов в исходной таблице MySQL. ClickHouse пытается [приводить](../../../engines/table-engines/integrations/mysql.md#type_conversion_function-cast) значения к типам данных ClickHouse. +- Типы столбцов могут отличаться от типов в исходной таблице MySQL. ClickHouse пытается [приводить](../../../sql-reference/functions/type-conversion-functions.md#type_conversion_function-cast) значения к типам данных ClickHouse. **Параметры движка** diff --git a/docs/ru/engines/table-engines/integrations/postgresql.md b/docs/ru/engines/table-engines/integrations/postgresql.md new file mode 100644 index 00000000000..4febd23067b --- /dev/null +++ b/docs/ru/engines/table-engines/integrations/postgresql.md @@ -0,0 +1,102 @@ +--- +toc_priority: 8 +toc_title: PostgreSQL +--- + +# PosgtreSQL {#postgresql} + +Движок PostgreSQL позволяет выполнять запросы `SELECT` над данными, хранящимися на удалённом PostgreSQL сервере. + +## Создание таблицы {#creating-a-table} + +``` sql +CREATE TABLE [IF NOT EXISTS] [db.]table_name [ON CLUSTER cluster] +( + name1 [type1] [DEFAULT|MATERIALIZED|ALIAS expr1] [TTL expr1], + name2 [type2] [DEFAULT|MATERIALIZED|ALIAS expr2] [TTL expr2], + ... +) ENGINE = PostgreSQL('host:port', 'database', 'table', 'user', 'password'); +``` + +Смотрите подробное описание запроса [CREATE TABLE](../../../sql-reference/statements/create/table.md#create-table-query). + +Структура таблицы может отличаться от исходной структуры таблицы PostgreSQL: + +- Имена столбцов должны быть такими же, как в исходной таблице MySQL, но вы можете использовать только некоторые из этих столбцов и в любом порядке. +- Типы столбцов могут отличаться от типов в исходной таблице PostgreSQL. ClickHouse пытается [приводить](../../../sql-reference/functions/type-conversion-functions.md#type_conversion_function-cast) values to the ClickHouse data types. + +**Параметры движка** + +- `host:port` — адрес сервера PostgreSQL. + +- `database` — Имя базы данных на сервере PostgreSQL. + +- `table` — Имя таблицы. + +- `user` — Имя пользователя PostgreSQL. + +- `password` — Пароль пользователя PostgreSQL. + +SELECT запросы на стороне PostgreSQL выполняются как `COPY (SELECT ...) TO STDOUT` внутри транзакции PostgreSQL только на чтение с коммитом после каждого `SELECT` запроса. + +Простые условия для `WHERE` такие как `=, !=, >, >=, <, <=, IN` исполняются на стороне PostgreSQL сервера. + +Все операции объединения, аггрегации, сортировки, условия `IN [ array ]` и ограничения `LIMIT` выполняются на стороне ClickHouse только после того как запрос к PostgreSQL закончился. + +INSERT запросы на стороне PostgreSQL выполняются как `COPY "table_name" (field1, field2, ... fieldN) FROM STDIN` внутри PostgreSQL транзакции с автоматическим коммитом после каждого `INSERT` запроса. + +PostgreSQL массивы конвертируются в массивы ClickHouse. + +## Пример использования {#usage-example} + +Таблица в PostgreSQL: + +``` text +postgres=# CREATE TABLE "public"."test" ( +"int_id" SERIAL, +"int_nullable" INT NULL DEFAULT NULL, +"float" FLOAT NOT NULL, +"str" VARCHAR(100) NOT NULL DEFAULT '', +"float_nullable" FLOAT NULL DEFAULT NULL, +PRIMARY KEY (int_id)); + +CREATE TABLE + +postgres=# insert into test (int_id, str, "float") VALUES (1,'test',2); +INSERT 0 1 + +postgresql> select * from test; + int_id | int_nullable | float | str | float_nullable +--------+--------------+-------+------+---------------- + 1 | | 2 | test | +(1 row) +``` + +Таблица в ClickHouse, получение данных из PostgreSQL таблицы созданной выше: + +``` sql +CREATE TABLE default.postgresql_table +( + `float_nullable` Nullable(Float32), + `str` String, + `int_id` Int32 +) +ENGINE = PostgreSQL('localhost:5432', 'public', 'test', 'postges_user', 'postgres_password'); +``` + +``` sql +SELECT * FROM postgresql_table WHERE str IN ('test') +``` + +``` text +┌─float_nullable─┬─str──┬─int_id─┐ +│ ᴺᵁᴸᴸ │ test │ 1 │ +└────────────────┴──────┴────────┘ +1 rows in set. Elapsed: 0.019 sec. +``` + + +## Смотри также {#see-also} + +- [Табличная функция ‘postgresql’](../../../sql-reference/table-functions/postgresql.md) +- [Использование PostgreSQL в качестве истояника для внешнего словаря](../../../sql-reference/dictionaries/external-dictionaries/external-dicts-dict-sources.md#dicts-external_dicts_dict_sources-postgresql) diff --git a/docs/ru/sql-reference/dictionaries/external-dictionaries/external-dicts-dict-lifetime.md b/docs/ru/sql-reference/dictionaries/external-dictionaries/external-dicts-dict-lifetime.md index ec0fb8e0ee5..f816caa3aa5 100644 --- a/docs/ru/sql-reference/dictionaries/external-dictionaries/external-dicts-dict-lifetime.md +++ b/docs/ru/sql-reference/dictionaries/external-dictionaries/external-dicts-dict-lifetime.md @@ -28,7 +28,7 @@ LIFETIME(300) ... ``` -Настройка `0` запрещает обновление словарей. +Настройка `0` (`LIFETIME(0)`) запрещает обновление словарей. Можно задать интервал, внутри которого ClickHouse равномерно-случайно выберет время для обновления. Это необходимо для распределения нагрузки на источник словаря при обновлении на большом количестве серверов. @@ -51,16 +51,19 @@ LIFETIME(300) LIFETIME(MIN 300 MAX 360) ``` +Если `0` и `0`, ClickHouse не перегружает словарь по истечению времени. +В этм случае, ClickHouse может перезагрузить данные словаря если изменился XML файл с конфигурацией словаря или если была выполнена команда `SYSTEM RELOAD DICTIONARY`. + При обновлении словарей сервер ClickHouse применяет различную логику в зависимости от типа [источника](external-dicts-dict-sources.md): -> - У текстового файла проверяется время модификации. Если время изменилось по отношению к запомненному ранее, то словарь обновляется. -> - Для MySQL источника, время модификации проверяется запросом `SHOW TABLE STATUS` (для MySQL 8 необходимо отключить кеширование мета-информации в MySQL `set global information_schema_stats_expiry=0`. -> - Словари из других источников по умолчанию обновляются каждый раз. +- У текстового файла проверяется время модификации. Если время изменилось по отношению к запомненному ранее, то словарь обновляется. +- Для MySQL источника, время модификации проверяется запросом `SHOW TABLE STATUS` (для MySQL 8 необходимо отключить кеширование мета-информации в MySQL `set global information_schema_stats_expiry=0`. +- Словари из других источников по умолчанию обновляются каждый раз. -Для других источников (ODBC, ClickHouse и т.д.) можно настроить запрос, который позволит обновлять словари только в случае их фактического изменения, а не каждый раз. Чтобы это сделать необходимо выполнить следующие условия/действия: +Для других источников (ODBC, PostgreSQL, ClickHouse и т.д.) можно настроить запрос, который позволит обновлять словари только в случае их фактического изменения, а не каждый раз. Чтобы это сделать необходимо выполнить следующие условия/действия: -> - В таблице словаря должно быть поле, которое гарантированно изменяется при обновлении данных в источнике. -> - В настройках источника указывается запрос, который получает изменяющееся поле. Результат запроса сервер ClickHouse интерпретирует как строку и если эта строка изменилась по отношению к предыдущему состоянию, то словарь обновляется. Запрос следует указывать в поле `` настроек [источника](external-dicts-dict-sources.md). +- В таблице словаря должно быть поле, которое гарантированно изменяется при обновлении данных в источнике. +- В настройках источника указывается запрос, который получает изменяющееся поле. Результат запроса сервер ClickHouse интерпретирует как строку и если эта строка изменилась по отношению к предыдущему состоянию, то словарь обновляется. Запрос следует указывать в поле `` настроек [источника](external-dicts-dict-sources.md). Пример настройки: diff --git a/docs/ru/sql-reference/dictionaries/external-dictionaries/external-dicts-dict-sources.md b/docs/ru/sql-reference/dictionaries/external-dictionaries/external-dicts-dict-sources.md index 77275b65a05..a6142cc210d 100644 --- a/docs/ru/sql-reference/dictionaries/external-dictionaries/external-dicts-dict-sources.md +++ b/docs/ru/sql-reference/dictionaries/external-dictionaries/external-dicts-dict-sources.md @@ -65,6 +65,7 @@ SETTINGS(format_csv_allow_single_quotes = 0) - СУБД: - [ODBC](#dicts-external_dicts_dict_sources-odbc) - [MySQL](#dicts-external_dicts_dict_sources-mysql) + - [PostgreSQL](#dicts-external_dicts_dict_sources-postgresql) - [ClickHouse](#dicts-external_dicts_dict_sources-clickhouse) - [MongoDB](#dicts-external_dicts_dict_sources-mongodb) - [Redis](#dicts-external_dicts_dict_sources-redis) @@ -313,6 +314,7 @@ PRIMARY KEY id SOURCE(ODBC(connection_string 'DSN=myconnection' table 'postgresql_table')) LAYOUT(HASHED()) LIFETIME(MIN 300 MAX 360) +``` Может понадобиться в `odbc.ini` указать полный путь до библиотеки с драйвером `DRIVER=/usr/local/lib/psqlodbcw.so`. @@ -320,15 +322,15 @@ LIFETIME(MIN 300 MAX 360) ОС Ubuntu. -Установка драйвера: : +Установка драйвера: ```bash $ sudo apt-get install tdsodbc freetds-bin sqsh ``` -Настройка драйвера: : +Настройка драйвера: -``` bash +```bash $ cat /etc/freetds/freetds.conf ... @@ -338,8 +340,11 @@ $ sudo apt-get install tdsodbc freetds-bin sqsh tds version = 7.0 client charset = UTF-8 + # тестирование TDS соединения + $ sqsh -S MSSQL -D database -U user -P password + + $ cat /etc/odbcinst.ini - ... [FreeTDS] Description = FreeTDS @@ -348,8 +353,8 @@ $ sudo apt-get install tdsodbc freetds-bin sqsh FileUsage = 1 UsageCount = 5 - $ cat ~/.odbc.ini - ... + $ cat /etc/odbc.ini + # $ cat ~/.odbc.ini # если вы вошли из под пользователя из под которого запущен ClickHouse [MSSQL] Description = FreeTDS @@ -359,8 +364,15 @@ $ sudo apt-get install tdsodbc freetds-bin sqsh UID = test PWD = test Port = 1433 + + + # (не обязательно) тест ODBC соединения (используйте isql поставляемый вместе с [unixodbc](https://packages.debian.org/sid/unixodbc)-package) + $ isql -v MSSQL "user" "password" ``` +Примечание: +- чтобы определить самую раннюю версию TDS, которая поддерживается определенной версией SQL Server, обратитесь к документации продукта или посмотрите на [MS-TDS Product Behavior](https://docs.microsoft.com/en-us/openspecs/windows_protocols/ms-tds/135d0ebe-5c4c-4a94-99bf-1811eccb9f4a) + Настройка словаря в ClickHouse: ``` xml @@ -624,4 +636,93 @@ SOURCE(REDIS( - `storage_type` – способ хранения ключей. Необходимо использовать `simple` для источников с одним столбцом ключей, `hash_map` – для источников с двумя столбцами ключей. Источники с более, чем двумя столбцами ключей, не поддерживаются. Может отсутствовать, значение по умолчанию `simple`. - `db_index` – номер базы данных. Может отсутствовать, значение по умолчанию 0. +### Cassandra {#dicts-external_dicts_dict_sources-cassandra} + +Пример настройки: + +``` xml + + + localhost + 9042 + username + qwerty123 + database_name + table_name + 1 + 1 + One + "SomeColumn" = 42 + 8 + + +``` + +Поля настройки: +- `host` – Имя хоста с установленной Cassandra или разделенный через запятую список хостов. +- `port` – Порт на серверах Cassandra. Если не указан, используется значение по умолчанию 9042. +- `user` – Имя пользователя для соединения с Cassandra. +- `password` – Пароль для соединения с Cassandra. +- `keyspace` – Имя keyspace (база данных). +- `column_family` – Имя семейства столбцов (таблица). +- `allow_filering` – Флаг, разрешающий или не разрешающий потенциально дорогостоящие условия на кластеризации ключевых столбцов. Значение по умолчанию 1. +- `partition_key_prefix` – Количество партиций ключевых столбцов в первичном ключе таблицы Cassandra. +Необходимо для составления ключей словаря. Порядок ключевых столбцов в определении словеря должен быть таким же как в Cassandra. +Значение по умолчанию 1 (первый ключевой столбец это ключ партицирования, остальные ключевые столбцы - ключи кластеризации). +- `consistency` – Уровень консистентности. Возмодные значения: `One`, `Two`, `Three`, + `All`, `EachQuorum`, `Quorum`, `LocalQuorum`, `LocalOne`, `Serial`, `LocalSerial`. Значение по умолчанию `One`. +- `where` – Опциональный критерий выборки. +- `max_threads` – Максимальное кол-во тредов для загрузки данных из нескольких партиций в словарь. + +### PosgreSQL {#dicts-external_dicts_dict_sources-postgresql} + +Пример настройки: + +``` xml + + + 5432 + clickhouse + qwerty + db_name + table_name
+ id=10 + SQL_QUERY + + +``` + +или + +``` sql +SOURCE(POSTGRESQL( + port 5432 + host 'postgresql-hostname' + user 'postgres_user' + password 'postgres_password' + db 'db_name' + table 'table_name' + replica(host 'example01-1' port 5432 priority 1) + replica(host 'example01-2' port 5432 priority 2) + where 'id=10' + invalidate_query 'SQL_QUERY' +)) +``` + +Setting fields: + +- `host` – Хост для соединения с PostgreSQL. Вы можете указать его для всех реплик или задать индивидуально для каждой релпики (внутри ``). +- `port` – Порт для соединения с PostgreSQL. Вы можете указать его для всех реплик или задать индивидуально для каждой релпики (внутри ``). +- `user` – Имя пользователя для соединения с PostgreSQL. Вы можете указать его для всех реплик или задать индивидуально для каждой релпики (внутри ``). +- `password` – Пароль для пользователя PostgreSQL. +- `replica` – Section of replica configurations. There can be multiple sections. + - `replica/host` – хост PostgreSQL. + - `replica/port` – порт PostgreSQL . + - `replica/priority` – Приоритет реплики. Во время попытки соединения, ClickHouse будет перебирать реплики в порядке приоритет. Меньшее значение означает более высокий приоритет. +- `db` – Имя базы данных. +- `table` – Имя таблицы. +- `where` – Условие выборки. Синтаксис для условий такой же как для `WHERE` выражения в PostgreSQL, для примера, `id > 10 AND id < 20`. Необязательный параметр. +- `invalidate_query` – Запрос для проверки условия загрузки словаря. Необязательный параметр. Читайте больше в разделе [Обновление словарей](../../../sql-reference/dictionaries/external-dictionaries/external-dicts-dict-lifetime.md). + + [Оригинальная статья](https://clickhouse.tech/docs/ru/query_language/dicts/external_dicts_dict_sources/) diff --git a/docs/ru/sql-reference/table-functions/postgresql.md b/docs/ru/sql-reference/table-functions/postgresql.md new file mode 100644 index 00000000000..5e6bcc0406b --- /dev/null +++ b/docs/ru/sql-reference/table-functions/postgresql.md @@ -0,0 +1,99 @@ +--- +toc_priority: 42 +toc_title: postgresql +--- + +# postgresql {#postgresql} + +Позволяет выполнять запросы `SELECT` над данными, хранящимися на удалённом PostgreSQL сервере. + +**Синтаксис** +``` sql +postgresql('host:port', 'database', 'table', 'user', 'password') +``` + +**Параметры** + +- `host:port` — адрес сервера PostgreSQL. + +- `database` — имя базы данных на удалённом сервере. + +- `table` — имя таблицы на удалённом сервере. + +- `user` — пользователь PostgreSQL. + +- `password` — пароль пользователя. + + +SELECT запросы на стороне PostgreSQL выполняются как `COPY (SELECT ...) TO STDOUT` внутри транзакции PostgreSQL только на чтение с коммитом после каждого `SELECT` запроса. + +Простые условия для `WHERE` такие как `=, !=, >, >=, <, <=, IN` исполняются на стороне PostgreSQL сервера. + +Все операции объединения, аггрегации, сортировки, условия `IN [ array ]` и ограничения `LIMIT` выполняются на стороне ClickHouse только после того как запрос к PostgreSQL закончился. + +INSERT запросы на стороне PostgreSQL выполняются как `COPY "table_name" (field1, field2, ... fieldN) FROM STDIN` внутри PostgreSQL транзакции с автоматическим коммитом после каждого `INSERT` запроса. + +PostgreSQL массивы конвертируются в массивы ClickHouse. + +**Возвращаемое значение** + +Объект таблицы с теми же столбцами, что и в исходной таблице PostgreSQL. + +!!! info "Примечание" +В запросах `INSERT` для того чтобы отличить табличную функцию `postgresql(...)` от таблицы со списком имен столбцов вы должны указывать ключевые слова `FUNCTION` или `TABLE FUNCTION`. See examples below. + +**Примеры** + +Таблица в PostgreSQL: + +``` text +postgres=# CREATE TABLE "public"."test" ( +"int_id" SERIAL, +"int_nullable" INT NULL DEFAULT NULL, +"float" FLOAT NOT NULL, +"str" VARCHAR(100) NOT NULL DEFAULT '', +"float_nullable" FLOAT NULL DEFAULT NULL, +PRIMARY KEY (int_id)); + +CREATE TABLE + +postgres=# insert into test (int_id, str, "float") VALUES (1,'test',2); +INSERT 0 1 + +postgresql> select * from test; + int_id | int_nullable | float | str | float_nullable +--------+--------------+-------+------+---------------- + 1 | | 2 | test | +(1 row) +``` + +Получение данных в ClickHouse: + +```sql +SELECT * FROM postgresql('localhost:5432', 'test', 'test', 'postgresql_user', 'password') WHERE str IN ('test'); +``` + +``` text +┌─int_id─┬─int_nullable─┬─float─┬─str──┬─float_nullable─┐ +│ 1 │ ᴺᵁᴸᴸ │ 2 │ test │ ᴺᵁᴸᴸ │ +└────────┴──────────────┴───────┴──────┴────────────────┘ +``` + +Вставка: + +```sql +INSERT INTO TABLE FUNCTION postgresql('localhost:5432', 'test', 'test', 'postgrsql_user', 'password') (int_id, float) VALUES (2, 3); +SELECT * FROM postgresql('localhost:5432', 'test', 'test', 'postgresql_user', 'password'); +``` + +``` text +┌─int_id─┬─int_nullable─┬─float─┬─str──┬─float_nullable─┐ +│ 1 │ ᴺᵁᴸᴸ │ 2 │ test │ ᴺᵁᴸᴸ │ +│ 2 │ ᴺᵁᴸᴸ │ 3 │ │ ᴺᵁᴸᴸ │ +└────────┴──────────────┴───────┴──────┴────────────────┘ +``` + +**Смотрите также** + +- [Движок таблиц ‘PostgreSQL’](../../sql-reference/table-functions/postgresql.md) +- [Использование PostgreSQL как источника данных для внешнего словаря](../../sql-reference/table-functions/postgresql.md#dicts-external_dicts_dict_sources-postgresql) From 5791cf5f2b9ab2eb604d090dfa977702df823b8c Mon Sep 17 00:00:00 2001 From: Slach Date: Tue, 23 Feb 2021 08:57:50 +0500 Subject: [PATCH 147/544] fix review comments https://github.com/ClickHouse/ClickHouse/pull/21078#discussion_r580413875 and https://github.com/ClickHouse/ClickHouse/pull/21078#discussion_r580410204 Signed-off-by: Slach --- docs/en/engines/table-engines/integrations/mysql.md | 1 + docs/en/engines/table-engines/integrations/odbc.md | 1 + docs/en/engines/table-engines/integrations/postgresql.md | 2 ++ docs/en/sql-reference/table-functions/postgresql.md | 1 + docs/ru/engines/table-engines/integrations/mysql.md | 1 + docs/ru/engines/table-engines/integrations/odbc.md | 1 + docs/ru/engines/table-engines/integrations/postgresql.md | 2 ++ docs/ru/sql-reference/table-functions/postgresql.md | 1 + src/Core/Settings.h | 2 +- 9 files changed, 11 insertions(+), 1 deletion(-) diff --git a/docs/en/engines/table-engines/integrations/mysql.md b/docs/en/engines/table-engines/integrations/mysql.md index 2cb1facce91..2ea8ea95958 100644 --- a/docs/en/engines/table-engines/integrations/mysql.md +++ b/docs/en/engines/table-engines/integrations/mysql.md @@ -24,6 +24,7 @@ The table structure can differ from the original MySQL table structure: - Column names should be the same as in the original MySQL table, but you can use just some of these columns and in any order. - Column types may differ from those in the original MySQL table. ClickHouse tries to [cast](../../../sql-reference/functions/type-conversion-functions.md#type_conversion_function-cast) values to the ClickHouse data types. +- Setting `external_table_functions_use_nulls` defines how to handle Nullable columns. Default is true, if false - table function will not make nullable columns and will insert default values instead of nulls. This is also applicable for null values inside array data types. **Engine Parameters** diff --git a/docs/en/engines/table-engines/integrations/odbc.md b/docs/en/engines/table-engines/integrations/odbc.md index fffc125b0ff..8083d644deb 100644 --- a/docs/en/engines/table-engines/integrations/odbc.md +++ b/docs/en/engines/table-engines/integrations/odbc.md @@ -29,6 +29,7 @@ The table structure can differ from the source table structure: - Column names should be the same as in the source table, but you can use just some of these columns and in any order. - Column types may differ from those in the source table. ClickHouse tries to [cast](../../../sql-reference/functions/type-conversion-functions.md#type_conversion_function-cast) values to the ClickHouse data types. +- Setting `external_table_functions_use_nulls` defines how to handle Nullable columns. Default is true, if false - table function will not make nullable columns and will insert default values instead of nulls. This is also applicable for null values inside array data types. **Engine Parameters** diff --git a/docs/en/engines/table-engines/integrations/postgresql.md b/docs/en/engines/table-engines/integrations/postgresql.md index 4a81251e60a..6153e3a29a2 100644 --- a/docs/en/engines/table-engines/integrations/postgresql.md +++ b/docs/en/engines/table-engines/integrations/postgresql.md @@ -24,6 +24,7 @@ The table structure can differ from the original PostgreSQL table structure: - Column names should be the same as in the original PostgreSQL table, but you can use just some of these columns and in any order. - Column types may differ from those in the original PostgreSQL table. ClickHouse tries to [cast](../../../sql-reference/functions/type-conversion-functions.md#type_conversion_function-cast) values to the ClickHouse data types. +- Setting `external_table_functions_use_nulls` defines how to handle Nullable columns. Default is 1, if 0 - table function will not make nullable columns and will insert default values instead of nulls. This is also applicable for null values inside array data types. **Engine Parameters** @@ -46,6 +47,7 @@ All joins, aggregations, sorting, `IN [ array ]` conditions and the `LIMIT` samp INSERT Queries on PostgreSQL side run as `COPY "table_name" (field1, field2, ... fieldN) FROM STDIN` inside PostgreSQL transaction with auto-commit after each `INSERT` statement. PostgreSQL Array types converts into ClickHouse arrays. +Be careful in PostgreSQL an array data type column like Integer[] may contain arrays of different dimensions in different rows, but in ClickHouse only allows multidimensional arrays of the same dimension in all rows. ## Usage Example {#usage-example} diff --git a/docs/en/sql-reference/table-functions/postgresql.md b/docs/en/sql-reference/table-functions/postgresql.md index 95724694d7a..88f5b26b99b 100644 --- a/docs/en/sql-reference/table-functions/postgresql.md +++ b/docs/en/sql-reference/table-functions/postgresql.md @@ -35,6 +35,7 @@ All joins, aggregations, sorting, `IN [ array ]` conditions and the `LIMIT` samp INSERT Queries on PostgreSQL side run as `COPY "table_name" (field1, field2, ... fieldN) FROM STDIN` inside PostgreSQL transaction with auto-commit after each `INSERT` statement. PostgreSQL Array types converts into ClickHouse arrays. +Be careful in PostgreSQL an array data type column like Integer[] may contain arrays of different dimensions in different rows, but in ClickHouse only allows multidimensional arrays of the same dimension in all rows. **Returned Value** diff --git a/docs/ru/engines/table-engines/integrations/mysql.md b/docs/ru/engines/table-engines/integrations/mysql.md index 2254ddcb68c..459f8844ce8 100644 --- a/docs/ru/engines/table-engines/integrations/mysql.md +++ b/docs/ru/engines/table-engines/integrations/mysql.md @@ -24,6 +24,7 @@ CREATE TABLE [IF NOT EXISTS] [db.]table_name [ON CLUSTER cluster] - Имена столбцов должны быть такими же, как в исходной таблице MySQL, но вы можете использовать только некоторые из этих столбцов и в любом порядке. - Типы столбцов могут отличаться от типов в исходной таблице MySQL. ClickHouse пытается [приводить](../../../sql-reference/functions/type-conversion-functions.md#type_conversion_function-cast) значения к типам данных ClickHouse. +- Настройка `external_table_functions_use_nulls` определяет как обрабатывать Nullable столбцы. По умолчанию 1, если 0 - табличная функция не будет делать nullable столбцы и будет вместо null выставлять значения по умолчанию для скалярного типа. Это также применимо для null значений внутри массивов. **Параметры движка** diff --git a/docs/ru/engines/table-engines/integrations/odbc.md b/docs/ru/engines/table-engines/integrations/odbc.md index 97317d647c8..898d569d504 100644 --- a/docs/ru/engines/table-engines/integrations/odbc.md +++ b/docs/ru/engines/table-engines/integrations/odbc.md @@ -29,6 +29,7 @@ ENGINE = ODBC(connection_settings, external_database, external_table) - Имена столбцов должны быть такими же, как в исходной таблице, но вы можете использовать только некоторые из этих столбцов и в любом порядке. - Типы столбцов могут отличаться от типов аналогичных столбцов в исходной таблице. ClickHouse пытается [приводить](../../../engines/table-engines/integrations/odbc.md#type_conversion_function-cast) значения к типам данных ClickHouse. +- Настройка `external_table_functions_use_nulls` определяет как обрабатывать Nullable столбцы. По умолчанию 1, если 0 - табличная функция не будет делать nullable столбцы и будет вместо null выставлять значения по умолчанию для скалярного типа. Это также применимо для null значений внутри массивов. **Параметры движка** diff --git a/docs/ru/engines/table-engines/integrations/postgresql.md b/docs/ru/engines/table-engines/integrations/postgresql.md index 4febd23067b..85512e0ea26 100644 --- a/docs/ru/engines/table-engines/integrations/postgresql.md +++ b/docs/ru/engines/table-engines/integrations/postgresql.md @@ -24,6 +24,7 @@ CREATE TABLE [IF NOT EXISTS] [db.]table_name [ON CLUSTER cluster] - Имена столбцов должны быть такими же, как в исходной таблице MySQL, но вы можете использовать только некоторые из этих столбцов и в любом порядке. - Типы столбцов могут отличаться от типов в исходной таблице PostgreSQL. ClickHouse пытается [приводить](../../../sql-reference/functions/type-conversion-functions.md#type_conversion_function-cast) values to the ClickHouse data types. +- Настройка `external_table_functions_use_nulls` определяет как обрабатывать Nullable столбцы. По умолчанию 1, если 0 - табличная функция не будет делать nullable столбцы и будет вместо null выставлять значения по умолчанию для скалярного типа. Это также применимо для null значений внутри массивов. **Параметры движка** @@ -46,6 +47,7 @@ SELECT запросы на стороне PostgreSQL выполняются ка INSERT запросы на стороне PostgreSQL выполняются как `COPY "table_name" (field1, field2, ... fieldN) FROM STDIN` внутри PostgreSQL транзакции с автоматическим коммитом после каждого `INSERT` запроса. PostgreSQL массивы конвертируются в массивы ClickHouse. +Будьте осторожны в PostgreSQL многомерные массивы могут содержать в себе другие массивы с разным кол-вом элементов в разных подмассивах, но внутри ClickHouse допустипы только многомерные массивы с одинаковым кол-вом элементов в каждом вложенном массиве. ## Пример использования {#usage-example} diff --git a/docs/ru/sql-reference/table-functions/postgresql.md b/docs/ru/sql-reference/table-functions/postgresql.md index 5e6bcc0406b..338e7d8e7f2 100644 --- a/docs/ru/sql-reference/table-functions/postgresql.md +++ b/docs/ru/sql-reference/table-functions/postgresql.md @@ -34,6 +34,7 @@ SELECT запросы на стороне PostgreSQL выполняются ка INSERT запросы на стороне PostgreSQL выполняются как `COPY "table_name" (field1, field2, ... fieldN) FROM STDIN` внутри PostgreSQL транзакции с автоматическим коммитом после каждого `INSERT` запроса. PostgreSQL массивы конвертируются в массивы ClickHouse. +Будьте осторожны в PostgreSQL многомерные массивы могут содержать в себе другие массивы с разным кол-вом элементов в разных подмассивах, но внутри ClickHouse допустипы только многомерные массивы с одинаковым кол-вом элементов в каждом вложенном массиве. **Возвращаемое значение** diff --git a/src/Core/Settings.h b/src/Core/Settings.h index 8afc08da21a..cf8d34bb56f 100644 --- a/src/Core/Settings.h +++ b/src/Core/Settings.h @@ -355,7 +355,7 @@ class IColumn; M(UInt64, read_in_order_two_level_merge_threshold, 100, "Minimal number of parts to read to run preliminary merge step during multithread reading in order of primary key.", 0) \ M(Bool, low_cardinality_allow_in_native_format, true, "Use LowCardinality type in Native format. Otherwise, convert LowCardinality columns to ordinary for select query, and convert ordinary columns to required LowCardinality for insert query.", 0) \ M(Bool, cancel_http_readonly_queries_on_client_close, false, "Cancel HTTP readonly queries when a client closes the connection without waiting for response.", 0) \ - M(Bool, external_table_functions_use_nulls, true, "If it is set to true, external table functions will implicitly use Nullable type if needed. Otherwise NULLs will be substituted with default values. Currently supported only by 'mysql' and 'odbc' table functions.", 0) \ + M(Bool, external_table_functions_use_nulls, true, "If it is set to true, external table functions will implicitly use Nullable type if needed. Otherwise NULLs will be substituted with default values. Currently supported only by 'mysql', 'postgresql' and 'odbc' table functions.", 0) \ \ M(Bool, allow_hyperscan, true, "Allow functions that use Hyperscan library. Disable to avoid potentially long compilation times and excessive resource usage.", 0) \ M(Bool, allow_simdjson, true, "Allow using simdjson library in 'JSON*' functions if AVX2 instructions are available. If disabled rapidjson will be used.", 0) \ From 7c91b1d6e60a230288817d3d331e592a8acb7c63 Mon Sep 17 00:00:00 2001 From: Slach Date: Wed, 24 Feb 2021 10:00:15 +0500 Subject: [PATCH 148/544] fix multi dimensional array warning - https://github.com/ClickHouse/ClickHouse/pull/21078#discussion_r580829045 Signed-off-by: Slach --- docs/en/engines/table-engines/integrations/postgresql.md | 2 +- docs/en/sql-reference/table-functions/postgresql.md | 2 +- docs/ru/engines/table-engines/integrations/postgresql.md | 2 +- docs/ru/sql-reference/table-functions/postgresql.md | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/docs/en/engines/table-engines/integrations/postgresql.md b/docs/en/engines/table-engines/integrations/postgresql.md index 6153e3a29a2..b82b97bfed0 100644 --- a/docs/en/engines/table-engines/integrations/postgresql.md +++ b/docs/en/engines/table-engines/integrations/postgresql.md @@ -47,7 +47,7 @@ All joins, aggregations, sorting, `IN [ array ]` conditions and the `LIMIT` samp INSERT Queries on PostgreSQL side run as `COPY "table_name" (field1, field2, ... fieldN) FROM STDIN` inside PostgreSQL transaction with auto-commit after each `INSERT` statement. PostgreSQL Array types converts into ClickHouse arrays. -Be careful in PostgreSQL an array data type column like Integer[] may contain arrays of different dimensions in different rows, but in ClickHouse only allows multidimensional arrays of the same dimension in all rows. +Be careful in PostgreSQL an array data created like type_name[] may contain multi-dimensional arrays of different dimensions in different rows in same column, but in ClickHouse only allows multi-dimensional arrays of the same count of dimension in all rows in same column in table. ## Usage Example {#usage-example} diff --git a/docs/en/sql-reference/table-functions/postgresql.md b/docs/en/sql-reference/table-functions/postgresql.md index 88f5b26b99b..42790e0d870 100644 --- a/docs/en/sql-reference/table-functions/postgresql.md +++ b/docs/en/sql-reference/table-functions/postgresql.md @@ -35,7 +35,7 @@ All joins, aggregations, sorting, `IN [ array ]` conditions and the `LIMIT` samp INSERT Queries on PostgreSQL side run as `COPY "table_name" (field1, field2, ... fieldN) FROM STDIN` inside PostgreSQL transaction with auto-commit after each `INSERT` statement. PostgreSQL Array types converts into ClickHouse arrays. -Be careful in PostgreSQL an array data type column like Integer[] may contain arrays of different dimensions in different rows, but in ClickHouse only allows multidimensional arrays of the same dimension in all rows. +Be careful in PostgreSQL an array data created like type_name[] may contain multi-dimensional arrays of different dimensions in different rows in same column, but in ClickHouse only allows multi-dimensional arrays of the same count of dimension in all rows in same column in table. **Returned Value** diff --git a/docs/ru/engines/table-engines/integrations/postgresql.md b/docs/ru/engines/table-engines/integrations/postgresql.md index 85512e0ea26..3ab98682203 100644 --- a/docs/ru/engines/table-engines/integrations/postgresql.md +++ b/docs/ru/engines/table-engines/integrations/postgresql.md @@ -47,7 +47,7 @@ SELECT запросы на стороне PostgreSQL выполняются ка INSERT запросы на стороне PostgreSQL выполняются как `COPY "table_name" (field1, field2, ... fieldN) FROM STDIN` внутри PostgreSQL транзакции с автоматическим коммитом после каждого `INSERT` запроса. PostgreSQL массивы конвертируются в массивы ClickHouse. -Будьте осторожны в PostgreSQL многомерные массивы могут содержать в себе другие массивы с разным кол-вом элементов в разных подмассивах, но внутри ClickHouse допустипы только многомерные массивы с одинаковым кол-вом элементов в каждом вложенном массиве. +Будьте осторожны в PostgreSQL массивы созданные как type_name[], являются многомерными и могут содержать в себе разное количество измерений в разных строках одной таблицы, внутри ClickHouse допустипы только многомерные массивы с одинаковым кол-вом измерений во всех строках таблицы. ## Пример использования {#usage-example} diff --git a/docs/ru/sql-reference/table-functions/postgresql.md b/docs/ru/sql-reference/table-functions/postgresql.md index 338e7d8e7f2..a8ed23db8ed 100644 --- a/docs/ru/sql-reference/table-functions/postgresql.md +++ b/docs/ru/sql-reference/table-functions/postgresql.md @@ -34,7 +34,7 @@ SELECT запросы на стороне PostgreSQL выполняются ка INSERT запросы на стороне PostgreSQL выполняются как `COPY "table_name" (field1, field2, ... fieldN) FROM STDIN` внутри PostgreSQL транзакции с автоматическим коммитом после каждого `INSERT` запроса. PostgreSQL массивы конвертируются в массивы ClickHouse. -Будьте осторожны в PostgreSQL многомерные массивы могут содержать в себе другие массивы с разным кол-вом элементов в разных подмассивах, но внутри ClickHouse допустипы только многомерные массивы с одинаковым кол-вом элементов в каждом вложенном массиве. +Будьте осторожны в PostgreSQL массивы созданные как type_name[], являются многомерными и могут содержать в себе разное количество измерений в разных строках одной таблицы, внутри ClickHouse допустипы только многомерные массивы с одинаковым кол-вом измерений во всех строках таблицы. **Возвращаемое значение** From c2e97c295182fd6701bb8c37b4b7dffc715a5808 Mon Sep 17 00:00:00 2001 From: Eugene Klimov Date: Tue, 23 Feb 2021 13:08:11 +0500 Subject: [PATCH 149/544] Update docs/en/sql-reference/table-functions/postgresql.md Co-authored-by: Kseniia Sumarokova <54203879+kssenii@users.noreply.github.com> --- docs/en/sql-reference/table-functions/postgresql.md | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/docs/en/sql-reference/table-functions/postgresql.md b/docs/en/sql-reference/table-functions/postgresql.md index 42790e0d870..082931343bf 100644 --- a/docs/en/sql-reference/table-functions/postgresql.md +++ b/docs/en/sql-reference/table-functions/postgresql.md @@ -35,14 +35,15 @@ All joins, aggregations, sorting, `IN [ array ]` conditions and the `LIMIT` samp INSERT Queries on PostgreSQL side run as `COPY "table_name" (field1, field2, ... fieldN) FROM STDIN` inside PostgreSQL transaction with auto-commit after each `INSERT` statement. PostgreSQL Array types converts into ClickHouse arrays. -Be careful in PostgreSQL an array data created like type_name[] may contain multi-dimensional arrays of different dimensions in different rows in same column, but in ClickHouse only allows multi-dimensional arrays of the same count of dimension in all rows in same column in table. + +Be careful in PostgreSQL an array data type column like Integer[] may contain arrays of different dimensions in different rows, but in ClickHouse it is only allowed to have multidimensional arrays of the same dimension in all rows. **Returned Value** A table object with the same columns as the original PostgreSQL table. !!! info "Note" - In the `INSERT` query to distinguish table function `postgresql(...)` from table name with column names list you must use keywords `FUNCTION` or `TABLE FUNCTION`. See examples below. + In the `INSERT` query to distinguish table function `postgresql(...)` from table name with column names list you must use keywords `FUNCTION` or `TABLE FUNCTION`. See examples below. **Examples** @@ -72,7 +73,7 @@ postgresql> select * from test; Selecting data from ClickHouse: ```sql -SELECT * FROM postgresql('localhost:5432', 'test', 'test', 'postgresql_user', 'password') WHERE str IN ('test'); +SELECT * FROM postgresql('localhost:5432', 'test', 'test', 'postgresql_user', 'password') WHERE str IN ('test'); ``` ``` text From 8763ac98fe5094cc9af4c946d688c34f195fd249 Mon Sep 17 00:00:00 2001 From: Slach Date: Wed, 24 Feb 2021 10:22:32 +0500 Subject: [PATCH 150/544] fix multi dimensional array warning - https://github.com/ClickHouse/ClickHouse/pull/21078#discussion_r580829045 Signed-off-by: Slach --- docs/en/engines/table-engines/integrations/postgresql.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/en/engines/table-engines/integrations/postgresql.md b/docs/en/engines/table-engines/integrations/postgresql.md index b82b97bfed0..7272f2e5edf 100644 --- a/docs/en/engines/table-engines/integrations/postgresql.md +++ b/docs/en/engines/table-engines/integrations/postgresql.md @@ -47,7 +47,7 @@ All joins, aggregations, sorting, `IN [ array ]` conditions and the `LIMIT` samp INSERT Queries on PostgreSQL side run as `COPY "table_name" (field1, field2, ... fieldN) FROM STDIN` inside PostgreSQL transaction with auto-commit after each `INSERT` statement. PostgreSQL Array types converts into ClickHouse arrays. -Be careful in PostgreSQL an array data created like type_name[] may contain multi-dimensional arrays of different dimensions in different rows in same column, but in ClickHouse only allows multi-dimensional arrays of the same count of dimension in all rows in same column in table. +Be careful in PostgreSQL an array data created like a type_name[] may contain multi-dimensional arrays of different dimensions in different table rows in same column, but in ClickHouse it is only allowed to have multidimensional arrays of the same count of dimensions in all table rows in same column. ## Usage Example {#usage-example} From ddd5acf251c272b70aa7dd2e1580946249e53d6b Mon Sep 17 00:00:00 2001 From: Slach Date: Thu, 4 Mar 2021 17:08:35 +0500 Subject: [PATCH 151/544] try to fix "fake" nowhere links according to https://github.com/ClickHouse/ClickHouse/pull/21268#issuecomment-787106299 --- .../integrations/embedded-rocksdb.md | 2 +- .../table-engines/integrations/hdfs.md | 7 +- .../table-engines/integrations/index.md | 3 + .../table-engines/integrations/jdbc.md | 2 +- .../table-engines/integrations/kafka.md | 2 +- .../table-engines/integrations/mongodb.md | 2 +- .../table-engines/integrations/mysql.md | 2 +- .../table-engines/integrations/odbc.md | 2 +- .../table-engines/integrations/postgresql.md | 2 + .../table-engines/integrations/rabbitmq.md | 2 + .../engines/table-engines/integrations/s3.md | 26 +-- docs/en/sql-reference/table-functions/file.md | 2 +- docs/en/sql-reference/table-functions/hdfs.md | 2 +- .../en/sql-reference/table-functions/index.md | 25 +-- docs/en/sql-reference/table-functions/odbc.md | 2 +- .../table-functions/postgresql.md | 2 + docs/en/sql-reference/table-functions/s3.md | 2 +- docs/en/sql-reference/table-functions/view.md | 3 +- .../integrations/embedded-rocksdb.md | 2 +- .../table-engines/integrations/hdfs.md | 96 ++++++++++- .../table-engines/integrations/index.md | 2 + .../table-engines/integrations/jdbc.md | 2 +- .../table-engines/integrations/kafka.md | 2 +- .../table-engines/integrations/mongodb.md | 2 +- .../table-engines/integrations/mysql.md | 2 +- .../table-engines/integrations/odbc.md | 2 +- .../table-engines/integrations/postgresql.md | 2 + .../table-engines/integrations/rabbitmq.md | 2 + .../engines/table-engines/integrations/s3.md | 156 ++++++++++++++++++ 29 files changed, 310 insertions(+), 50 deletions(-) create mode 100644 docs/ru/engines/table-engines/integrations/s3.md diff --git a/docs/en/engines/table-engines/integrations/embedded-rocksdb.md b/docs/en/engines/table-engines/integrations/embedded-rocksdb.md index 6e864751cc3..e9e069933e5 100644 --- a/docs/en/engines/table-engines/integrations/embedded-rocksdb.md +++ b/docs/en/engines/table-engines/integrations/embedded-rocksdb.md @@ -39,4 +39,4 @@ ENGINE = EmbeddedRocksDB PRIMARY KEY key ``` -[Original article](https://clickhouse.tech/docs/en/operations/table_engines/embedded-rocksdb/) +[Original article](https://clickhouse.tech/docs/en/engines/table-engines/integrations/embedded-rocksdb/) diff --git a/docs/en/engines/table-engines/integrations/hdfs.md b/docs/en/engines/table-engines/integrations/hdfs.md index 5c36e3f1c21..0782efe8e72 100644 --- a/docs/en/engines/table-engines/integrations/hdfs.md +++ b/docs/en/engines/table-engines/integrations/hdfs.md @@ -5,7 +5,7 @@ toc_title: HDFS # HDFS {#table_engines-hdfs} -This engine provides integration with [Apache Hadoop](https://en.wikipedia.org/wiki/Apache_Hadoop) ecosystem by allowing to manage data on [HDFS](https://hadoop.apache.org/docs/current/hadoop-project-dist/hadoop-hdfs/HdfsDesign.html)via ClickHouse. This engine is similar +This engine provides integration with [Apache Hadoop](https://en.wikipedia.org/wiki/Apache_Hadoop) ecosystem by allowing to manage data on [HDFS](https://hadoop.apache.org/docs/current/hadoop-project-dist/hadoop-hdfs/HdfsDesign.html) via ClickHouse. This engine is similar to the [File](../../../engines/table-engines/special/file.md#table_engines-file) and [URL](../../../engines/table-engines/special/url.md#table_engines-url) engines, but provides Hadoop-specific features. ## Usage {#usage} @@ -174,7 +174,7 @@ Similar to GraphiteMergeTree, the HDFS engine supports extended configuration us | dfs\_domain\_socket\_path | "" | -[HDFS Configuration Reference ](https://hawq.apache.org/docs/userguide/2.3.0.0-incubating/reference/HDFSConfigurationParameterReference.html) might explain some parameters. +[HDFS Configuration Reference](https://hawq.apache.org/docs/userguide/2.3.0.0-incubating/reference/HDFSConfigurationParameterReference.html) might explain some parameters. #### ClickHouse extras {#clickhouse-extras} @@ -185,7 +185,6 @@ Similar to GraphiteMergeTree, the HDFS engine supports extended configuration us |hadoop\_kerberos\_kinit\_command | kinit | #### Limitations {#limitations} - * hadoop\_security\_kerberos\_ticket\_cache\_path can be global only, not user specific ## Kerberos support {#kerberos-support} @@ -207,4 +206,4 @@ If hadoop\_kerberos\_keytab, hadoop\_kerberos\_principal or hadoop\_kerberos\_ki - [Virtual columns](../../../engines/table-engines/index.md#table_engines-virtual_columns) -[Original article](https://clickhouse.tech/docs/en/operations/table_engines/hdfs/) +[Original article](https://clickhouse.tech/docs/en/engines/table-engines/integrations/hdfs/) diff --git a/docs/en/engines/table-engines/integrations/index.md b/docs/en/engines/table-engines/integrations/index.md index 288c9c3cd56..28f38375448 100644 --- a/docs/en/engines/table-engines/integrations/index.md +++ b/docs/en/engines/table-engines/integrations/index.md @@ -18,3 +18,6 @@ List of supported integrations: - [Kafka](../../../engines/table-engines/integrations/kafka.md) - [EmbeddedRocksDB](../../../engines/table-engines/integrations/embedded-rocksdb.md) - [RabbitMQ](../../../engines/table-engines/integrations/rabbitmq.md) +- [PostgreSQL](../../../engines/table-engines/integrations/postgresql.md) + +[Original article](https://clickhouse.tech/docs/en/engines/table-engines/integrations/) diff --git a/docs/en/engines/table-engines/integrations/jdbc.md b/docs/en/engines/table-engines/integrations/jdbc.md index 2144be9f1e3..edbc5d3ed3e 100644 --- a/docs/en/engines/table-engines/integrations/jdbc.md +++ b/docs/en/engines/table-engines/integrations/jdbc.md @@ -85,4 +85,4 @@ FROM jdbc_table - [JDBC table function](../../../sql-reference/table-functions/jdbc.md). -[Original article](https://clickhouse.tech/docs/en/operations/table_engines/jdbc/) +[Original article](https://clickhouse.tech/docs/en/engines/table-engines/integrations/jdbc/) diff --git a/docs/en/engines/table-engines/integrations/kafka.md b/docs/en/engines/table-engines/integrations/kafka.md index fb1df62bb15..1b3aaa4b569 100644 --- a/docs/en/engines/table-engines/integrations/kafka.md +++ b/docs/en/engines/table-engines/integrations/kafka.md @@ -194,4 +194,4 @@ Example: - [Virtual columns](../../../engines/table-engines/index.md#table_engines-virtual_columns) - [background_schedule_pool_size](../../../operations/settings/settings.md#background_schedule_pool_size) -[Original article](https://clickhouse.tech/docs/en/operations/table_engines/kafka/) +[Original article](https://clickhouse.tech/docs/enen/engines/table-engines/integrations/kafka/) diff --git a/docs/en/engines/table-engines/integrations/mongodb.md b/docs/en/engines/table-engines/integrations/mongodb.md index e648a13b5e0..2fee27ce80d 100644 --- a/docs/en/engines/table-engines/integrations/mongodb.md +++ b/docs/en/engines/table-engines/integrations/mongodb.md @@ -54,4 +54,4 @@ SELECT COUNT() FROM mongo_table; └─────────┘ ``` -[Original article](https://clickhouse.tech/docs/en/operations/table_engines/integrations/mongodb/) +[Original article](https://clickhouse.tech/docs/en/engines/table-engines/integrations/mongodb/) diff --git a/docs/en/engines/table-engines/integrations/mysql.md b/docs/en/engines/table-engines/integrations/mysql.md index 2ea8ea95958..8b7caa12c91 100644 --- a/docs/en/engines/table-engines/integrations/mysql.md +++ b/docs/en/engines/table-engines/integrations/mysql.md @@ -101,4 +101,4 @@ SELECT * FROM mysql_table - [The ‘mysql’ table function](../../../sql-reference/table-functions/mysql.md) - [Using MySQL as a source of external dictionary](../../../sql-reference/dictionaries/external-dictionaries/external-dicts-dict-sources.md#dicts-external_dicts_dict_sources-mysql) -[Original article](https://clickhouse.tech/docs/en/operations/table_engines/mysql/) +[Original article](https://clickhouse.tech/docs/en/engines/table-engines/integrations/mysql/) diff --git a/docs/en/engines/table-engines/integrations/odbc.md b/docs/en/engines/table-engines/integrations/odbc.md index 8083d644deb..99efd870088 100644 --- a/docs/en/engines/table-engines/integrations/odbc.md +++ b/docs/en/engines/table-engines/integrations/odbc.md @@ -128,4 +128,4 @@ SELECT * FROM odbc_t - [ODBC external dictionaries](../../../sql-reference/dictionaries/external-dictionaries/external-dicts-dict-sources.md#dicts-external_dicts_dict_sources-odbc) - [ODBC table function](../../../sql-reference/table-functions/odbc.md) -[Original article](https://clickhouse.tech/docs/en/operations/table_engines/odbc/) +[Original article](https://clickhouse.tech/docs/en/engines/table-engines/integrations/odbc/) diff --git a/docs/en/engines/table-engines/integrations/postgresql.md b/docs/en/engines/table-engines/integrations/postgresql.md index 7272f2e5edf..1a2ccf3e0dc 100644 --- a/docs/en/engines/table-engines/integrations/postgresql.md +++ b/docs/en/engines/table-engines/integrations/postgresql.md @@ -102,3 +102,5 @@ SELECT * FROM postgresql_table WHERE str IN ('test') - [The ‘postgresql’ table function](../../../sql-reference/table-functions/postgresql.md) - [Using PostgreSQL as a source of external dictionary](../../../sql-reference/dictionaries/external-dictionaries/external-dicts-dict-sources.md#dicts-external_dicts_dict_sources-postgresql) + +[Original article](https://clickhouse.tech/docs/en/engines/table-engines/integrations/postgresql/) diff --git a/docs/en/engines/table-engines/integrations/rabbitmq.md b/docs/en/engines/table-engines/integrations/rabbitmq.md index 4a0550275ca..476192d3969 100644 --- a/docs/en/engines/table-engines/integrations/rabbitmq.md +++ b/docs/en/engines/table-engines/integrations/rabbitmq.md @@ -163,3 +163,5 @@ Example: - `_redelivered` - `redelivered` flag of the message. - `_message_id` - messageID of the received message; non-empty if was set, when message was published. - `_timestamp` - timestamp of the received message; non-empty if was set, when message was published. + +[Original article](https://clickhouse.tech/docs/en/engines/table-engines/integrations/rabbitmq/) diff --git a/docs/en/engines/table-engines/integrations/s3.md b/docs/en/engines/table-engines/integrations/s3.md index 5858a0803e6..93dcbdbc0f1 100644 --- a/docs/en/engines/table-engines/integrations/s3.md +++ b/docs/en/engines/table-engines/integrations/s3.md @@ -6,11 +6,11 @@ toc_title: S3 # S3 {#table_engines-s3} This engine provides integration with [Amazon S3](https://aws.amazon.com/s3/) ecosystem. This engine is similar -to the [HDFS](../../../engines/table-engines/special/file.md#table_engines-hdfs) engine, but provides S3-specific features. +to the [HDFS](../../../engines/table-engines/integrations/hdfs.md#table_engines-hdfs) engine, but provides S3-specific features. ## Usage {#usage} -``` sql +```sql ENGINE = S3(path, [aws_access_key_id, aws_secret_access_key,] format, structure, [compression]) ``` @@ -25,23 +25,23 @@ ENGINE = S3(path, [aws_access_key_id, aws_secret_access_key,] format, structure, **1.** Set up the `s3_engine_table` table: -``` sql +```sql CREATE TABLE s3_engine_table (name String, value UInt32) ENGINE=S3('https://storage.yandexcloud.net/my-test-bucket-768/test-data.csv.gz', 'CSV', 'name String, value UInt32', 'gzip') ``` **2.** Fill file: -``` sql +```sql INSERT INTO s3_engine_table VALUES ('one', 1), ('two', 2), ('three', 3) ``` **3.** Query the data: -``` sql +```sql SELECT * FROM s3_engine_table LIMIT 2 ``` -``` text +```text ┌─name─┬─value─┐ │ one │ 1 │ │ two │ 2 │ @@ -69,7 +69,7 @@ Constructions with `{}` are similar to the [remote](../../../sql-reference/table **Example** -1. Suppose we have several files in TSV format with the following URIs on HDFS: +1. Suppose we have several files in CSV format with the following URIs on S3: - ‘https://storage.yandexcloud.net/my-test-bucket-768/some_prefix/some_file_1.csv’ - ‘https://storage.yandexcloud.net/my-test-bucket-768/some_prefix/some_file_2.csv’ @@ -82,19 +82,19 @@ Constructions with `{}` are similar to the [remote](../../../sql-reference/table -``` sql +```sql CREATE TABLE table_with_range (name String, value UInt32) ENGINE = S3('https://storage.yandexcloud.net/my-test-bucket-768/{some,another}_prefix/some_file_{1..3}', 'CSV') ``` 3. Another way: -``` sql +```sql CREATE TABLE table_with_question_mark (name String, value UInt32) ENGINE = S3('https://storage.yandexcloud.net/my-test-bucket-768/{some,another}_prefix/some_file_?', 'CSV') ``` 4. Table consists of all the files in both directories (all files should satisfy format and schema described in query): -``` sql +```sql CREATE TABLE table_with_asterisk (name String, value UInt32) ENGINE = S3('https://storage.yandexcloud.net/my-test-bucket-768/{some,another}_prefix/*', 'CSV') ``` @@ -105,7 +105,7 @@ CREATE TABLE table_with_asterisk (name String, value UInt32) ENGINE = S3('https: Create table with files named `file-000.csv`, `file-001.csv`, … , `file-999.csv`: -``` sql +```sql CREATE TABLE big_table (name String, value UInt32) ENGINE = S3('https://storage.yandexcloud.net/my-test-bucket-768/big_prefix/file-{000..999}.csv', 'CSV') ``` @@ -124,7 +124,7 @@ The following settings can be set before query execution or placed into configur - `s3_max_single_part_upload_size` — Default value is `64Mb`. The maximum size of object to upload using singlepart upload to S3. - `s3_min_upload_part_size` — Default value is `512Mb`. The minimum size of part to upload during multipart upload to [S3 Multipart upload](https://docs.aws.amazon.com/AmazonS3/latest/dev/uploadobjusingmpu.html). -- `s3_max_redirects` — Default value is `10`. Max number of S3 redirects hops allowed. +- `s3_max_redirects` — Default value is `10`. Max number of HTTP redirects S3 hops allowed. Security consideration: if malicious user can specify arbitrary S3 URLs, `s3_max_redirects` must be set to zero to avoid [SSRF](https://en.wikipedia.org/wiki/Server-side_request_forgery) attacks; or alternatively, `remote_host_filter` must be specified in server configuration. @@ -153,4 +153,4 @@ Example: ``` -[Original article](https://clickhouse.tech/docs/en/operations/table_engines/s3/) +[Original article](https://clickhouse.tech/docs/en/engines/table-engines/integrations/s3/) diff --git a/docs/en/sql-reference/table-functions/file.md b/docs/en/sql-reference/table-functions/file.md index da0999e66eb..e1459b5e254 100644 --- a/docs/en/sql-reference/table-functions/file.md +++ b/docs/en/sql-reference/table-functions/file.md @@ -124,6 +124,6 @@ SELECT count(*) FROM file('big_dir/file{0..9}{0..9}{0..9}', 'CSV', 'name String, **See Also** -- [Virtual columns](index.md#table_engines-virtual_columns) +- [Virtual columns](../../engines/table-engines/index.md#table_engines-virtual_columns) [Original article](https://clickhouse.tech/docs/en/sql-reference/table-functions/file/) diff --git a/docs/en/sql-reference/table-functions/hdfs.md b/docs/en/sql-reference/table-functions/hdfs.md index 512f47a2b46..31e2000b22d 100644 --- a/docs/en/sql-reference/table-functions/hdfs.md +++ b/docs/en/sql-reference/table-functions/hdfs.md @@ -97,6 +97,6 @@ FROM hdfs('hdfs://hdfs1:9000/big_dir/file{0..9}{0..9}{0..9}', 'CSV', 'name Strin **See Also** -- [Virtual columns](https://clickhouse.tech/docs/en/operations/table_engines/#table_engines-virtual_columns) +- [Virtual columns](../../engines/table-engines/index.md#table_engines-virtual_columns) [Original article](https://clickhouse.tech/docs/en/query_language/table_functions/hdfs/) diff --git a/docs/en/sql-reference/table-functions/index.md b/docs/en/sql-reference/table-functions/index.md index 691687dea25..d1368c6a674 100644 --- a/docs/en/sql-reference/table-functions/index.md +++ b/docs/en/sql-reference/table-functions/index.md @@ -21,17 +21,18 @@ You can use table functions in: !!! warning "Warning" You can’t use table functions if the [allow_ddl](../../operations/settings/permissions-for-queries.md#settings_allow_ddl) setting is disabled. -| Function | Description | -|-----------------------------------------------------------|----------------------------------------------------------------------------------------------------------------------------------------| -| [file](../../sql-reference/table-functions/file.md) | Creates a [File](../../engines/table-engines/special/file.md)-engine table. | -| [merge](../../sql-reference/table-functions/merge.md) | Creates a [Merge](../../engines/table-engines/special/merge.md)-engine table. | -| [numbers](../../sql-reference/table-functions/numbers.md) | Creates a table with a single column filled with integer numbers. | -| [remote](../../sql-reference/table-functions/remote.md) | Allows you to access remote servers without creating a [Distributed](../../engines/table-engines/special/distributed.md)-engine table. | -| [url](../../sql-reference/table-functions/url.md) | Creates a [Url](../../engines/table-engines/special/url.md)-engine table. | -| [mysql](../../sql-reference/table-functions/mysql.md) | Creates a [MySQL](../../engines/table-engines/integrations/mysql.md)-engine table. | -| [jdbc](../../sql-reference/table-functions/jdbc.md) | Creates a [JDBC](../../engines/table-engines/integrations/jdbc.md)-engine table. | -| [odbc](../../sql-reference/table-functions/odbc.md) | Creates a [ODBC](../../engines/table-engines/integrations/odbc.md)-engine table. | -| [hdfs](../../sql-reference/table-functions/hdfs.md) | Creates a [HDFS](../../engines/table-engines/integrations/hdfs.md)-engine table. | -| [s3](../../sql-reference/table-functions/s3.md) | Creates a [S3](../../engines/table-engines/integrations/s3.md)-engine table. | +| Function | Description | +|-----------------------------------------------------------------|----------------------------------------------------------------------------------------------------------------------------------------| +| [file](../../sql-reference/table-functions/file.md) | Creates a [File](../../engines/table-engines/special/file.md)-engine table. | +| [merge](../../sql-reference/table-functions/merge.md) | Creates a [Merge](../../engines/table-engines/special/merge.md)-engine table. | +| [numbers](../../sql-reference/table-functions/numbers.md) | Creates a table with a single column filled with integer numbers. | +| [remote](../../sql-reference/table-functions/remote.md) | Allows you to access remote servers without creating a [Distributed](../../engines/table-engines/special/distributed.md)-engine table. | +| [url](../../sql-reference/table-functions/url.md) | Creates a [Url](../../engines/table-engines/special/url.md)-engine table. | +| [mysql](../../sql-reference/table-functions/mysql.md) | Creates a [MySQL](../../engines/table-engines/integrations/mysql.md)-engine table. | +| [postgresql](../../sql-reference/table-functions/postgresql.md) | Creates a [PostgreSQL](../../engines/table-engines/integrations/posgresql.md)-engine table. | +| [jdbc](../../sql-reference/table-functions/jdbc.md) | Creates a [JDBC](../../engines/table-engines/integrations/jdbc.md)-engine table. | +| [odbc](../../sql-reference/table-functions/odbc.md) | Creates a [ODBC](../../engines/table-engines/integrations/odbc.md)-engine table. | +| [hdfs](../../sql-reference/table-functions/hdfs.md) | Creates a [HDFS](../../engines/table-engines/integrations/hdfs.md)-engine table. | +| [s3](../../sql-reference/table-functions/s3.md) | Creates a [S3](../../engines/table-engines/integrations/s3.md)-engine table. | [Original article](https://clickhouse.tech/docs/en/query_language/table_functions/) diff --git a/docs/en/sql-reference/table-functions/odbc.md b/docs/en/sql-reference/table-functions/odbc.md index ea79cd44a93..38ca4d40d17 100644 --- a/docs/en/sql-reference/table-functions/odbc.md +++ b/docs/en/sql-reference/table-functions/odbc.md @@ -103,4 +103,4 @@ SELECT * FROM odbc('DSN=mysqlconn', 'test', 'test') - [ODBC external dictionaries](../../sql-reference/dictionaries/external-dictionaries/external-dicts-dict-sources.md#dicts-external_dicts_dict_sources-odbc) - [ODBC table engine](../../engines/table-engines/integrations/odbc.md). -[Original article](https://clickhouse.tech/docs/en/query_language/table_functions/jdbc/) +[Original article](https://clickhouse.tech/docs/en/sql-reference/table-functions/jdbc/) diff --git a/docs/en/sql-reference/table-functions/postgresql.md b/docs/en/sql-reference/table-functions/postgresql.md index 082931343bf..ad5d8a29904 100644 --- a/docs/en/sql-reference/table-functions/postgresql.md +++ b/docs/en/sql-reference/table-functions/postgresql.md @@ -100,3 +100,5 @@ SELECT * FROM postgresql('localhost:5432', 'test', 'test', 'postgresql_user', 'p - [The ‘PostgreSQL’ table engine](../../engines/table-engines/integrations/postgresql.md) - [Using PostgreSQL as a source of external dictionary](../../sql-reference/dictionaries/external-dictionaries/external-dicts-dict-sources.md#dicts-external_dicts_dict_sources-postgresql) + +[Original article](https://clickhouse.tech/docs/en/sql-reference/table-functions/postgresql/) diff --git a/docs/en/sql-reference/table-functions/s3.md b/docs/en/sql-reference/table-functions/s3.md index 76a0e042ea4..ea5dde707b8 100644 --- a/docs/en/sql-reference/table-functions/s3.md +++ b/docs/en/sql-reference/table-functions/s3.md @@ -164,6 +164,6 @@ Security consideration: if malicious user can specify arbitrary S3 URLs, `s3_max **See Also** -- [Virtual columns](https://clickhouse.tech/docs/en/operations/table_engines/#table_engines-virtual_columns) +- [Virtual columns](../../engines/table-engines/index.md#table_engines-virtual_columns) [Original article](https://clickhouse.tech/docs/en/query_language/table_functions/s3/) diff --git a/docs/en/sql-reference/table-functions/view.md b/docs/en/sql-reference/table-functions/view.md index 08096c2b019..b627feee4c2 100644 --- a/docs/en/sql-reference/table-functions/view.md +++ b/docs/en/sql-reference/table-functions/view.md @@ -64,4 +64,5 @@ SELECT * FROM cluster(`cluster_name`, view(SELECT a, b, c FROM table_name)) **See Also** - [View Table Engine](https://clickhouse.tech/docs/en/engines/table-engines/special/view/) -[Original article](https://clickhouse.tech/docs/en/query_language/table_functions/view/) \ No newline at end of file + +[Original article](https://clickhouse.tech/docs/en/sql-reference/table-functions/view/) \ No newline at end of file diff --git a/docs/ru/engines/table-engines/integrations/embedded-rocksdb.md b/docs/ru/engines/table-engines/integrations/embedded-rocksdb.md index 9b68bcfc770..7bd1420dfab 100644 --- a/docs/ru/engines/table-engines/integrations/embedded-rocksdb.md +++ b/docs/ru/engines/table-engines/integrations/embedded-rocksdb.md @@ -41,4 +41,4 @@ ENGINE = EmbeddedRocksDB PRIMARY KEY key; ``` -[Оригинальная статья](https://clickhouse.tech/docs/ru/operations/table_engines/embedded-rocksdb/) \ No newline at end of file +[Оригинальная статья](https://clickhouse.tech/docs/ru/engines/table-engines/integrations/embedded-rocksdb/) \ No newline at end of file diff --git a/docs/ru/engines/table-engines/integrations/hdfs.md b/docs/ru/engines/table-engines/integrations/hdfs.md index bd8e760fce4..449d7c9a20c 100644 --- a/docs/ru/engines/table-engines/integrations/hdfs.md +++ b/docs/ru/engines/table-engines/integrations/hdfs.md @@ -102,16 +102,104 @@ CREATE TABLE table_with_asterisk (name String, value UInt32) ENGINE = HDFS('hdfs Создадим таблицу с именами `file000`, `file001`, … , `file999`: ``` sql -CREARE TABLE big_table (name String, value UInt32) ENGINE = HDFS('hdfs://hdfs1:9000/big_dir/file{0..9}{0..9}{0..9}', 'CSV') +CREATE TABLE big_table (name String, value UInt32) ENGINE = HDFS('hdfs://hdfs1:9000/big_dir/file{0..9}{0..9}{0..9}', 'CSV') ``` +## Конфигурация {#configuration} + +Похоже на GraphiteMergeTree, движок HDFS поддерживает расширенную конфигурацию с использованием файла конфигурации ClickHouse. Есть два раздела конфигурации которые вы можете использовать: глобальный (`hdfs`) и на уровне пользователя (`hdfs_*`). Глобальные настройки применяются первыми, и затем применяется конфигурация уровня пользователя (если она указана). + +``` xml + + + /tmp/keytab/clickhouse.keytab + clickuser@TEST.CLICKHOUSE.TECH + kerberos + + + + + root@TEST.CLICKHOUSE.TECH + +``` + +### Список возможных опций конфигурации со значениями по умолчанию +#### Поддерживаемые из libhdfs3 + + +| **параметр** | **по умолчанию** | +| rpc\_client\_connect\_tcpnodelay | true | +| dfs\_client\_read\_shortcircuit | true | +| output\_replace-datanode-on-failure | true | +| input\_notretry-another-node | false | +| input\_localread\_mappedfile | true | +| dfs\_client\_use\_legacy\_blockreader\_local | false | +| rpc\_client\_ping\_interval | 10 * 1000 | +| rpc\_client\_connect\_timeout | 600 * 1000 | +| rpc\_client\_read\_timeout | 3600 * 1000 | +| rpc\_client\_write\_timeout | 3600 * 1000 | +| rpc\_client\_socekt\_linger\_timeout | -1 | +| rpc\_client\_connect\_retry | 10 | +| rpc\_client\_timeout | 3600 * 1000 | +| dfs\_default\_replica | 3 | +| input\_connect\_timeout | 600 * 1000 | +| input\_read\_timeout | 3600 * 1000 | +| input\_write\_timeout | 3600 * 1000 | +| input\_localread\_default\_buffersize | 1 * 1024 * 1024 | +| dfs\_prefetchsize | 10 | +| input\_read\_getblockinfo\_retry | 3 | +| input\_localread\_blockinfo\_cachesize | 1000 | +| input\_read\_max\_retry | 60 | +| output\_default\_chunksize | 512 | +| output\_default\_packetsize | 64 * 1024 | +| output\_default\_write\_retry | 10 | +| output\_connect\_timeout | 600 * 1000 | +| output\_read\_timeout | 3600 * 1000 | +| output\_write\_timeout | 3600 * 1000 | +| output\_close\_timeout | 3600 * 1000 | +| output\_packetpool\_size | 1024 | +| output\_heeartbeat\_interval | 10 * 1000 | +| dfs\_client\_failover\_max\_attempts | 15 | +| dfs\_client\_read\_shortcircuit\_streams\_cache\_size | 256 | +| dfs\_client\_socketcache\_expiryMsec | 3000 | +| dfs\_client\_socketcache\_capacity | 16 | +| dfs\_default\_blocksize | 64 * 1024 * 1024 | +| dfs\_default\_uri | "hdfs://localhost:9000" | +| hadoop\_security\_authentication | "simple" | +| hadoop\_security\_kerberos\_ticket\_cache\_path | "" | +| dfs\_client\_log\_severity | "INFO" | +| dfs\_domain\_socket\_path | "" | + + +[Руководство по конфигурации HDFS](https://hawq.apache.org/docs/userguide/2.3.0.0-incubating/reference/HDFSConfigurationParameterReference.html) поможет обьяснить назначения некоторых параметров. + + +#### Расширенные параметры для ClickHouse {#clickhouse-extras} + +| **параметр** | **по умолчанию** | +|hadoop\_kerberos\_keytab | "" | +|hadoop\_kerberos\_principal | "" | +|hadoop\_kerberos\_kinit\_command | kinit | + +#### Ограничения {#limitations} + * hadoop\_security\_kerberos\_ticket\_cache\_path могут быть определены только на глобальном уровне + +## Поддержика Kerberos {#kerberos-support} + +Если hadoop\_security\_authentication параметр имеет значение 'kerberos', ClickHouse аутентифицируется с помощью Kerberos. +[Расширенные параметры](#clickhouse-extras) и hadoop\_security\_kerberos\_ticket\_cache\_path помогают сделать это. +Обратите внимание что из-за ограничений libhdfs3 поддерживается только устаревший метод аутентификации, +коммуникация с узлами данных не защищена SASL (HADOOP\_SECURE\_DN\_USER надежный показатель такого +подхода к безопасности). Используйте tests/integration/test\_storage\_kerberized\_hdfs/hdfs_configs/bootstrap.sh для примера настроек. + +Если hadoop\_kerberos\_keytab, hadoop\_kerberos\_principal или hadoop\_kerberos\_kinit\_command указаны в настройках, kinit будет вызван. hadoop\_kerberos\_keytab и hadoop\_kerberos\_principal обязательны в этом случае. Необходимо также будет установить kinit и файлы конфигурации krb5. ## Виртуальные столбцы {#virtualnye-stolbtsy} - `_path` — Путь к файлу. - `_file` — Имя файла. -**Смотрите также** +**См. также** -- [Виртуальные столбцы](index.md#table_engines-virtual_columns) +- [Виртуальные колонки](../../../engines/table-engines/index.md#table_engines-virtual_columns) -[Оригинальная статья](https://clickhouse.tech/docs/ru/operations/table_engines/hdfs/) +[Оригинальная статья](https://clickhouse.tech/docs/ru/engines/table-engines/integrations/hdfs/) diff --git a/docs/ru/engines/table-engines/integrations/index.md b/docs/ru/engines/table-engines/integrations/index.md index db7e527442e..7a11a5176cd 100644 --- a/docs/ru/engines/table-engines/integrations/index.md +++ b/docs/ru/engines/table-engines/integrations/index.md @@ -14,8 +14,10 @@ toc_priority: 30 - [MySQL](../../../engines/table-engines/integrations/mysql.md) - [MongoDB](../../../engines/table-engines/integrations/mongodb.md) - [HDFS](../../../engines/table-engines/integrations/hdfs.md) +- [S3](../../../engines/table-engines/integrations/s3.md) - [Kafka](../../../engines/table-engines/integrations/kafka.md) - [EmbeddedRocksDB](../../../engines/table-engines/integrations/embedded-rocksdb.md) - [RabbitMQ](../../../engines/table-engines/integrations/rabbitmq.md) +- [PostgreSQL](../../../engines/table-engines/integrations/postgresql.md) [Оригинальная статья](https://clickhouse.tech/docs/ru/engines/table-engines/integrations/) diff --git a/docs/ru/engines/table-engines/integrations/jdbc.md b/docs/ru/engines/table-engines/integrations/jdbc.md index d7d438e0633..8ead5abb277 100644 --- a/docs/ru/engines/table-engines/integrations/jdbc.md +++ b/docs/ru/engines/table-engines/integrations/jdbc.md @@ -89,4 +89,4 @@ FROM jdbc_table - [Табличная функция JDBC](../../../engines/table-engines/integrations/jdbc.md). -[Оригинальная статья](https://clickhouse.tech/docs/ru/operations/table_engines/jdbc/) +[Оригинальная статья](https://clickhouse.tech/docs/ru/engines/table-engines/integrations/jdbc/) diff --git a/docs/ru/engines/table-engines/integrations/kafka.md b/docs/ru/engines/table-engines/integrations/kafka.md index 5a6971b1ae6..06a0d4df180 100644 --- a/docs/ru/engines/table-engines/integrations/kafka.md +++ b/docs/ru/engines/table-engines/integrations/kafka.md @@ -193,4 +193,4 @@ ClickHouse может поддерживать учетные данные Kerbe - [Виртуальные столбцы](index.md#table_engines-virtual_columns) - [background_schedule_pool_size](../../../operations/settings/settings.md#background_schedule_pool_size) -[Оригинальная статья](https://clickhouse.tech/docs/ru/operations/table_engines/kafka/) +[Оригинальная статья](https://clickhouse.tech/docs/ru/engines/table-engines/integrations/kafka/) diff --git a/docs/ru/engines/table-engines/integrations/mongodb.md b/docs/ru/engines/table-engines/integrations/mongodb.md index 0765b3909de..5ab63494648 100644 --- a/docs/ru/engines/table-engines/integrations/mongodb.md +++ b/docs/ru/engines/table-engines/integrations/mongodb.md @@ -54,4 +54,4 @@ SELECT COUNT() FROM mongo_table; └─────────┘ ``` -[Original article](https://clickhouse.tech/docs/ru/operations/table_engines/integrations/mongodb/) +[Original article](https://clickhouse.tech/docs/ru/engines/table-engines/integrations/mongodb/) diff --git a/docs/ru/engines/table-engines/integrations/mysql.md b/docs/ru/engines/table-engines/integrations/mysql.md index 459f8844ce8..bc53e0f1fbb 100644 --- a/docs/ru/engines/table-engines/integrations/mysql.md +++ b/docs/ru/engines/table-engines/integrations/mysql.md @@ -101,4 +101,4 @@ SELECT * FROM mysql_table - [Табличная функция ‘mysql’](../../../engines/table-engines/integrations/mysql.md) - [Использование MySQL в качестве источника для внешнего словаря](../../../engines/table-engines/integrations/mysql.md#dicts-external_dicts_dict_sources-mysql) -[Оригинальная статья](https://clickhouse.tech/docs/ru/operations/table_engines/mysql/) +[Оригинальная статья](https://clickhouse.tech/docs/engines/table-engines/integrations/mysql/) diff --git a/docs/ru/engines/table-engines/integrations/odbc.md b/docs/ru/engines/table-engines/integrations/odbc.md index 898d569d504..ee34be302bc 100644 --- a/docs/ru/engines/table-engines/integrations/odbc.md +++ b/docs/ru/engines/table-engines/integrations/odbc.md @@ -128,4 +128,4 @@ SELECT * FROM odbc_t - [Внешние словари ODBC](../../../engines/table-engines/integrations/odbc.md#dicts-external_dicts_dict_sources-odbc) - [Табличная функция odbc](../../../engines/table-engines/integrations/odbc.md) -[Оригинальная статья](https://clickhouse.tech/docs/ru/operations/table_engines/odbc/) +[Оригинальная статья](https://clickhouse.tech/docs/ru/engines/table-engines/integrations/odbc/) diff --git a/docs/ru/engines/table-engines/integrations/postgresql.md b/docs/ru/engines/table-engines/integrations/postgresql.md index 3ab98682203..bc26899f55b 100644 --- a/docs/ru/engines/table-engines/integrations/postgresql.md +++ b/docs/ru/engines/table-engines/integrations/postgresql.md @@ -102,3 +102,5 @@ SELECT * FROM postgresql_table WHERE str IN ('test') - [Табличная функция ‘postgresql’](../../../sql-reference/table-functions/postgresql.md) - [Использование PostgreSQL в качестве истояника для внешнего словаря](../../../sql-reference/dictionaries/external-dictionaries/external-dicts-dict-sources.md#dicts-external_dicts_dict_sources-postgresql) + +[Оригинальная статья](https://clickhouse.tech/docs/ru/engines/table-engines/integrations/postgresql/) diff --git a/docs/ru/engines/table-engines/integrations/rabbitmq.md b/docs/ru/engines/table-engines/integrations/rabbitmq.md index f55163c1988..1865cb16fcc 100644 --- a/docs/ru/engines/table-engines/integrations/rabbitmq.md +++ b/docs/ru/engines/table-engines/integrations/rabbitmq.md @@ -155,3 +155,5 @@ Example: - `_redelivered` - флаг `redelivered`. (Не равно нулю, если есть возможность, что сообщение было получено более, чем одним каналом.) - `_message_id` - значение поля `messageID` полученного сообщения. Данное поле непусто, если указано в параметрах при отправке сообщения. - `_timestamp` - значение поля `timestamp` полученного сообщения. Данное поле непусто, если указано в параметрах при отправке сообщения. + +[Оригинальная статья](https://clickhouse.tech/docs/ru/engines/table-engines/integrations/rabbitmq/) diff --git a/docs/ru/engines/table-engines/integrations/s3.md b/docs/ru/engines/table-engines/integrations/s3.md new file mode 100644 index 00000000000..f1b2e78b0ba --- /dev/null +++ b/docs/ru/engines/table-engines/integrations/s3.md @@ -0,0 +1,156 @@ +--- +toc_priority: 4 +toc_title: S3 +--- + +# S3 {#table_engines-s3} + +Этот движок обеспечивает интеграцию с экосистемой [Amazon S3](https://aws.amazon.com/s3/). Этот движок похож на +движок [HDFS](../../../engines/table-engines/integrations/hdfs.md#table_engines-hdfs), но предоставляет S3-специфичные функции. + +## Использование {#usage} + +```sql +ENGINE = S3(path, [aws_access_key_id, aws_secret_access_key,] format, structure, [compression]) +``` + +**Параметры** + +- `path` — URL ссылающийся на файл расположенный в S3. В режиме для чтения можно читать несколько файлов как один, поддерживаются следующие шаблоны для указания маски пути к файлам: *, ?, {abc,def} и {N..M} где N, M — числа, `’abc’, ‘def’ — строки. +- `format` — [Формат](../../../interfaces/formats.md#formats) файла. +- `structure` — Структура таблицы. Формат `'column1_name column1_type, column2_name column2_type, ...'`. +- `compression` — Алгоритм сжатия, не обязятельный параметр. Поддерживаемые значения: none, gzip/gz, brotli/br, xz/LZMA, zstd/zst. По умолчанию, алгоритм сжатия будет автоматически применен в зависимости от расширения в имени файла. + +**Пример:** + +**1.** Создание таблицы `s3_engine_table` : + +```sql +CREATE TABLE s3_engine_table (name String, value UInt32) ENGINE=S3('https://storage.yandexcloud.net/my-test-bucket-768/test-data.csv.gz', 'CSV', 'name String, value UInt32', 'gzip') +``` + +**2.** Заполнение файла: + +```sql +INSERT INTO s3_engine_table VALUES ('one', 1), ('two', 2), ('three', 3) +``` + +**3.** Запрос данных: + +```sql +SELECT * FROM s3_engine_table LIMIT 2 +``` + +```text +┌─name─┬─value─┐ +│ one │ 1 │ +│ two │ 2 │ +└──────┴───────┘ +``` + +## Детали реализации {#implementation-details} + +- Чтение и запись могут быть одновременными и паралельными +- Не поддерживается: + - `ALTER` и `SELECT...SAMPLE` операции. + - Индексы. + - Репликация. + +**Поддержка шаблонов в параметре path** + +Множество частей параметра `path` поддерживает шаблоны. Для того чтобы быть обработанным файл должен присутствовать в S3 и соответсвовать шаблону. Списки файлов определяются в момент `SELECT` (но не в момент `CREATE`). + +- `*` — Заменяет любой количество любых символов кроме `/` включая пустые строки. +- `?` — Заменяет один символ. +- `{some_string,another_string,yet_another_one}` — Заменяет любую из строк `'some_string', 'another_string', 'yet_another_one'`. +- `{N..M}` — Заменяет любое числов в диапозоне от N до M включительно. N и M могут иметь лидирующие нули например `000..078`. + +Конструкции с`{}` работают также как в табличной функции [remote](../../../sql-reference/table-functions/remote.md). + +**Пример** + +1. Предположим у нас есть некоторые файлы в CSV формате со следующими URIs в S3: + +- ‘https://storage.yandexcloud.net/my-test-bucket-768/some_prefix/some_file_1.csv’ +- ‘https://storage.yandexcloud.net/my-test-bucket-768/some_prefix/some_file_2.csv’ +- ‘https://storage.yandexcloud.net/my-test-bucket-768/some_prefix/some_file_3.csv’ +- ‘https://storage.yandexcloud.net/my-test-bucket-768/another_prefix/some_file_1.csv’ +- ‘https://storage.yandexcloud.net/my-test-bucket-768/another_prefix/some_file_2.csv’ +- ‘https://storage.yandexcloud.net/my-test-bucket-768/another_prefix/some_file_3.csv’ + +2. Есть несколько способов сделать таблицу состяющую из всех шести файлов: + + + +```sql +CREATE TABLE table_with_range (name String, value UInt32) ENGINE = S3('https://storage.yandexcloud.net/my-test-bucket-768/{some,another}_prefix/some_file_{1..3}', 'CSV') +``` + +3. Другой способ: + +```sql +CREATE TABLE table_with_question_mark (name String, value UInt32) ENGINE = S3('https://storage.yandexcloud.net/my-test-bucket-768/{some,another}_prefix/some_file_?', 'CSV') +``` + +4. Таблица состоящая из всех файлах в обоих каталогах (все файлы должны удовлетворять формату и схеме описанными в запросе): + +```sql +CREATE TABLE table_with_asterisk (name String, value UInt32) ENGINE = S3('https://storage.yandexcloud.net/my-test-bucket-768/{some,another}_prefix/*', 'CSV') +``` + +!!! warning "Предупреждение" + Если список файлов содержит диапозоны номеров с ведующими нулями, используйте конструкции со скобками для каждой цифры или используйте `?`. + +**Пример** + +Создание таблицы с именами файлов `file-000.csv`, `file-001.csv`, … , `file-999.csv`: + +```sql +CREATE TABLE big_table (name String, value UInt32) ENGINE = S3('https://storage.yandexcloud.net/my-test-bucket-768/big_prefix/file-{000..999}.csv', 'CSV') +``` + +## Виртуальные колонки {#virtual-columns} + +- `_path` — Path to the file. +- `_file` — Name of the file. + +**Смотри также** + +- [Virtual columns](../../../engines/table-engines/index.md#table_engines-virtual_columns) + +## S3-специфичные настройки {#settings} + +Следующие настройки могут быть заданы при запуске запроса или установлены в конфигурационном файле для пользовательского профиля. + +- `s3_max_single_part_upload_size` — По умолчанию `64Mb`. Максикальный размер куска данных для загрузки в S3 как singlepart. +- `s3_min_upload_part_size` — По умолчанию `512Mb`. Минимальный размер куска данных для загрузки в S3 с помощью [S3 Multipart загрузки](https://docs.aws.amazon.com/AmazonS3/latest/dev/uploadobjusingmpu.html). +- `s3_max_redirects` — Значение по умолчанию `10`. Максимально допустимое количество HTTP перенаправлений от серверов S3. + +Примечания для безопасности: если злоумышленник может указать произвольные ссылки на S3, то лучше выставить `s3_max_redirects` как ноль для избежания атак типа [SSRF](https://en.wikipedia.org/wiki/Server-side_request_forgery) ; или ограничить с помощью `remote_host_filter` список адресов по которым возможно взаимодействие с S3. + +### Настройки специфичные для заданной конечной точки {#endpointsettings} + +Следующие настройки могут быть указаны в конфигурационном файле для заданной конечной точки (которой будет сопоставлен точный конечный префик URL): + +- `endpoint` — Обязательный параметр. Указывает префикс URL для конечной точки. +- `access_key_id` и `secret_access_key` — Не обязательно. Задает параметры авторизации для заданной конечной точки. +- `use_environment_credentials` — Не обязательный параметр, значение по умолчанию `false`. Если установлено как `true`, S3 клиент будет пытаться получить параметры авторизации из переменных окружения и Amazon EC2 метаданных для заданной конечной точки. +- `header` — Не обязательный параметр, может быть указан несколько раз. Добавляет указанный HTTP заголовок к запросу для заданной в `endpoint` URL префикса. +- `server_side_encryption_customer_key_base64` — Не обязательный параметр. Если указан, к запросам будут указаны заголовки необходимые для доступа к S3 объектам с SSE-C шифрованием. + +Пример: + +``` + + + https://storage.yandexcloud.net/my-test-bucket-768/ + + + + + + + +``` + +[Оригинальная статья](https://clickhouse.tech/docs/ru/engines/table-engines/integrations/s3/) From 5016b30209e276f04d26213813b92fe319ebe79d Mon Sep 17 00:00:00 2001 From: Slach Date: Thu, 4 Mar 2021 17:18:51 +0500 Subject: [PATCH 152/544] replace `posgresql` to `postgresql` --- .../external-dicts-dict-sources.md | 2 +- .../en/sql-reference/table-functions/index.md | 2 +- .../external-dicts-dict-sources.md | 2 +- .../ru/sql-reference/table-functions/index.md | 20 ++++++++++--------- 4 files changed, 14 insertions(+), 12 deletions(-) diff --git a/docs/en/sql-reference/dictionaries/external-dictionaries/external-dicts-dict-sources.md b/docs/en/sql-reference/dictionaries/external-dictionaries/external-dicts-dict-sources.md index f8f4745bb16..5772992f418 100644 --- a/docs/en/sql-reference/dictionaries/external-dictionaries/external-dicts-dict-sources.md +++ b/docs/en/sql-reference/dictionaries/external-dictionaries/external-dicts-dict-sources.md @@ -688,7 +688,7 @@ Example of settings: table_name
id=10 SQL_QUERY - +
``` diff --git a/docs/en/sql-reference/table-functions/index.md b/docs/en/sql-reference/table-functions/index.md index d1368c6a674..f51e58d6d67 100644 --- a/docs/en/sql-reference/table-functions/index.md +++ b/docs/en/sql-reference/table-functions/index.md @@ -29,7 +29,7 @@ You can use table functions in: | [remote](../../sql-reference/table-functions/remote.md) | Allows you to access remote servers without creating a [Distributed](../../engines/table-engines/special/distributed.md)-engine table. | | [url](../../sql-reference/table-functions/url.md) | Creates a [Url](../../engines/table-engines/special/url.md)-engine table. | | [mysql](../../sql-reference/table-functions/mysql.md) | Creates a [MySQL](../../engines/table-engines/integrations/mysql.md)-engine table. | -| [postgresql](../../sql-reference/table-functions/postgresql.md) | Creates a [PostgreSQL](../../engines/table-engines/integrations/posgresql.md)-engine table. | +| [postgresql](../../sql-reference/table-functions/postgresql.md) | Creates a [PostgreSQL](../../engines/table-engines/integrations/postgresql.md)-engine table. | | [jdbc](../../sql-reference/table-functions/jdbc.md) | Creates a [JDBC](../../engines/table-engines/integrations/jdbc.md)-engine table. | | [odbc](../../sql-reference/table-functions/odbc.md) | Creates a [ODBC](../../engines/table-engines/integrations/odbc.md)-engine table. | | [hdfs](../../sql-reference/table-functions/hdfs.md) | Creates a [HDFS](../../engines/table-engines/integrations/hdfs.md)-engine table. | diff --git a/docs/ru/sql-reference/dictionaries/external-dictionaries/external-dicts-dict-sources.md b/docs/ru/sql-reference/dictionaries/external-dictionaries/external-dicts-dict-sources.md index a6142cc210d..fdc13973d47 100644 --- a/docs/ru/sql-reference/dictionaries/external-dictionaries/external-dicts-dict-sources.md +++ b/docs/ru/sql-reference/dictionaries/external-dictionaries/external-dicts-dict-sources.md @@ -688,7 +688,7 @@ SOURCE(REDIS( table_name
id=10 SQL_QUERY - +
``` diff --git a/docs/ru/sql-reference/table-functions/index.md b/docs/ru/sql-reference/table-functions/index.md index 83225d54e60..31407dc250d 100644 --- a/docs/ru/sql-reference/table-functions/index.md +++ b/docs/ru/sql-reference/table-functions/index.md @@ -24,14 +24,16 @@ toc_title: "\u0412\u0432\u0435\u0434\u0435\u043D\u0438\u0435" | Функция | Описание | |-----------------------|---------------------------------------------------------------------------------------------------------------------------------------| -| [file](file.md) | Создаёт таблицу с движком [File](../../engines/table-engines/special/file.md). | -| [merge](merge.md) | Создаёт таблицу с движком [Merge](../../engines/table-engines/special/merge.md). | -| [numbers](numbers.md) | Создаёт таблицу с единственным столбцом, заполненным целыми числами. | -| [remote](remote.md) | Предоставляет доступ к удалённым серверам, не создавая таблицу с движком [Distributed](../../engines/table-engines/special/distributed.md). | -| [url](url.md) | Создаёт таблицу с движком [Url](../../engines/table-engines/special/url.md). | -| [mysql](mysql.md) | Создаёт таблицу с движком [MySQL](../../engines/table-engines/integrations/mysql.md). | -| [jdbc](jdbc.md) | Создаёт таблицу с дижком [JDBC](../../engines/table-engines/integrations/jdbc.md). | -| [odbc](odbc.md) | Создаёт таблицу с движком [ODBC](../../engines/table-engines/integrations/odbc.md). | -| [hdfs](hdfs.md) | Создаёт таблицу с движком [HDFS](../../engines/table-engines/integrations/hdfs.md). | +| [file](../../sql-reference/table-functions/file.md) | Создаёт таблицу с движком [File](../../engines/table-engines/special/file.md). | +| [merge](../../sql-reference/table-functions/merge.md) | Создаёт таблицу с движком [Merge](../../engines/table-engines/special/merge.md). | +| [numbers](../../sql-reference/table-functions/numbers.md) | Создаёт таблицу с единственным столбцом, заполненным целыми числами. | +| [remote](../../sql-reference/table-functions/remote.md) | Предоставляет доступ к удалённым серверам, не создавая таблицу с движком [Distributed](../../engines/table-engines/special/distributed.md). | +| [url](../../sql-reference/table-functions/url.md) | Создаёт таблицу с движком [Url](../../engines/table-engines/special/url.md). | +| [mysql](../../sql-reference/table-functions/mysql.md) | Создаёт таблицу с движком [MySQL](../../engines/table-engines/integrations/mysql.md). | +| [postgresql](../../sql-reference/table-functions/postgresql.md) | Создаёт таблицу с движком [PostgreSQL](../../engines/table-engines/integrations/postgresql.md). | +| [jdbc](../../sql-reference/table-functions/jdbc.md) | Создаёт таблицу с движком [JDBC](../../engines/table-engines/integrations/jdbc.md). | +| [odbc](../../sql-reference/table-functions/odbc.md) | Создаёт таблицу с движком [ODBC](../../engines/table-engines/integrations/odbc.md). | +| [hdfs](../../sql-reference/table-functions/hdfs.md) | Создаёт таблицу с движком [HDFS](../../engines/table-engines/integrations/hdfs.md). | +| [s3](../../sql-reference/table-functions/s3.md) | Создаёт таблицу с движком [S3](../../engines/table-engines/integrations/s3.md). | [Оригинальная статья](https://clickhouse.tech/docs/ru/query_language/table_functions/) From a165089ee38c07a51ac6eb320f7d39c3e7811794 Mon Sep 17 00:00:00 2001 From: alexey-milovidov Date: Fri, 5 Mar 2021 22:00:44 +0300 Subject: [PATCH 153/544] Update odbc.md --- docs/en/sql-reference/table-functions/odbc.md | 2 -- 1 file changed, 2 deletions(-) diff --git a/docs/en/sql-reference/table-functions/odbc.md b/docs/en/sql-reference/table-functions/odbc.md index 38ca4d40d17..a8481fbfd68 100644 --- a/docs/en/sql-reference/table-functions/odbc.md +++ b/docs/en/sql-reference/table-functions/odbc.md @@ -102,5 +102,3 @@ SELECT * FROM odbc('DSN=mysqlconn', 'test', 'test') - [ODBC external dictionaries](../../sql-reference/dictionaries/external-dictionaries/external-dicts-dict-sources.md#dicts-external_dicts_dict_sources-odbc) - [ODBC table engine](../../engines/table-engines/integrations/odbc.md). - -[Original article](https://clickhouse.tech/docs/en/sql-reference/table-functions/jdbc/) From c1a96e977a6aebe9e85c8935668f3358941373d8 Mon Sep 17 00:00:00 2001 From: alexey-milovidov Date: Sat, 6 Mar 2021 23:53:21 +0300 Subject: [PATCH 154/544] Update kafka.md --- docs/en/engines/table-engines/integrations/kafka.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/en/engines/table-engines/integrations/kafka.md b/docs/en/engines/table-engines/integrations/kafka.md index 1b3aaa4b569..0ec50094a27 100644 --- a/docs/en/engines/table-engines/integrations/kafka.md +++ b/docs/en/engines/table-engines/integrations/kafka.md @@ -194,4 +194,4 @@ Example: - [Virtual columns](../../../engines/table-engines/index.md#table_engines-virtual_columns) - [background_schedule_pool_size](../../../operations/settings/settings.md#background_schedule_pool_size) -[Original article](https://clickhouse.tech/docs/enen/engines/table-engines/integrations/kafka/) +[Original article](https://clickhouse.tech/docs/en/engines/table-engines/integrations/kafka/) From 7876415b8ac0897522aef9a4b5747db6e147dd0c Mon Sep 17 00:00:00 2001 From: alexey-milovidov Date: Sat, 6 Mar 2021 23:55:26 +0300 Subject: [PATCH 155/544] Update external-dicts-dict-sources.md --- .../external-dictionaries/external-dicts-dict-sources.md | 3 --- 1 file changed, 3 deletions(-) diff --git a/docs/en/sql-reference/dictionaries/external-dictionaries/external-dicts-dict-sources.md b/docs/en/sql-reference/dictionaries/external-dictionaries/external-dicts-dict-sources.md index 5772992f418..b7129725820 100644 --- a/docs/en/sql-reference/dictionaries/external-dictionaries/external-dicts-dict-sources.md +++ b/docs/en/sql-reference/dictionaries/external-dictionaries/external-dicts-dict-sources.md @@ -723,6 +723,3 @@ Setting fields: - `table` – Name of the table. - `where` – The selection criteria. The syntax for conditions is the same as for `WHERE` clause in PostgreSQL, for example, `id > 10 AND id < 20`. Optional parameter. - `invalidate_query` – Query for checking the dictionary status. Optional parameter. Read more in the section [Updating dictionaries](../../../sql-reference/dictionaries/external-dictionaries/external-dicts-dict-lifetime.md). - - -[Original article](https://clickhouse.tech/docs/en/query_language/dicts/external_dicts_dict_sources/) From 9922d8e1024c18a13dd7a825defd0be4eef48d05 Mon Sep 17 00:00:00 2001 From: alexey-milovidov Date: Sat, 6 Mar 2021 23:56:16 +0300 Subject: [PATCH 156/544] Update index.md --- docs/en/sql-reference/table-functions/index.md | 2 -- 1 file changed, 2 deletions(-) diff --git a/docs/en/sql-reference/table-functions/index.md b/docs/en/sql-reference/table-functions/index.md index f51e58d6d67..16bf23db857 100644 --- a/docs/en/sql-reference/table-functions/index.md +++ b/docs/en/sql-reference/table-functions/index.md @@ -34,5 +34,3 @@ You can use table functions in: | [odbc](../../sql-reference/table-functions/odbc.md) | Creates a [ODBC](../../engines/table-engines/integrations/odbc.md)-engine table. | | [hdfs](../../sql-reference/table-functions/hdfs.md) | Creates a [HDFS](../../engines/table-engines/integrations/hdfs.md)-engine table. | | [s3](../../sql-reference/table-functions/s3.md) | Creates a [S3](../../engines/table-engines/integrations/s3.md)-engine table. | - -[Original article](https://clickhouse.tech/docs/en/query_language/table_functions/) From 08081fe965fcec94a7e18844f528e1dd12495521 Mon Sep 17 00:00:00 2001 From: alexey-milovidov Date: Sun, 7 Mar 2021 00:05:38 +0300 Subject: [PATCH 157/544] Update index.md --- .../en/sql-reference/table-functions/index.md | 22 +++++++++---------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/docs/en/sql-reference/table-functions/index.md b/docs/en/sql-reference/table-functions/index.md index 16bf23db857..fef30c04c9d 100644 --- a/docs/en/sql-reference/table-functions/index.md +++ b/docs/en/sql-reference/table-functions/index.md @@ -23,14 +23,14 @@ You can use table functions in: | Function | Description | |-----------------------------------------------------------------|----------------------------------------------------------------------------------------------------------------------------------------| -| [file](../../sql-reference/table-functions/file.md) | Creates a [File](../../engines/table-engines/special/file.md)-engine table. | -| [merge](../../sql-reference/table-functions/merge.md) | Creates a [Merge](../../engines/table-engines/special/merge.md)-engine table. | -| [numbers](../../sql-reference/table-functions/numbers.md) | Creates a table with a single column filled with integer numbers. | -| [remote](../../sql-reference/table-functions/remote.md) | Allows you to access remote servers without creating a [Distributed](../../engines/table-engines/special/distributed.md)-engine table. | -| [url](../../sql-reference/table-functions/url.md) | Creates a [Url](../../engines/table-engines/special/url.md)-engine table. | -| [mysql](../../sql-reference/table-functions/mysql.md) | Creates a [MySQL](../../engines/table-engines/integrations/mysql.md)-engine table. | -| [postgresql](../../sql-reference/table-functions/postgresql.md) | Creates a [PostgreSQL](../../engines/table-engines/integrations/postgresql.md)-engine table. | -| [jdbc](../../sql-reference/table-functions/jdbc.md) | Creates a [JDBC](../../engines/table-engines/integrations/jdbc.md)-engine table. | -| [odbc](../../sql-reference/table-functions/odbc.md) | Creates a [ODBC](../../engines/table-engines/integrations/odbc.md)-engine table. | -| [hdfs](../../sql-reference/table-functions/hdfs.md) | Creates a [HDFS](../../engines/table-engines/integrations/hdfs.md)-engine table. | -| [s3](../../sql-reference/table-functions/s3.md) | Creates a [S3](../../engines/table-engines/integrations/s3.md)-engine table. | +| [file](../../sql-reference/table-functions/file.md) | Creates a File-engine table. | +| [merge](../../sql-reference/table-functions/merge.md) | Creates a Merge-engine table. | +| [numbers](../../sql-reference/table-functions/numbers.md) | Creates a table with a single column filled with integer numbers. | +| [remote](../../sql-reference/table-functions/remote.md) | Allows you to access remote servers without creating a Distributed-engine table. | +| [url](../../sql-reference/table-functions/url.md) | Creates a URL-engine table. | +| [mysql](../../sql-reference/table-functions/mysql.md) | Creates a MySQL-engine table. | +| [postgresql](../../sql-reference/table-functions/postgresql.md) | Creates a PostgreSQL-engine table. | +| [jdbc](../../sql-reference/table-functions/jdbc.md) | Creates a JDBC-engine table. | +| [odbc](../../sql-reference/table-functions/odbc.md) | Creates a ODBC-engine table. | +| [hdfs](../../sql-reference/table-functions/hdfs.md) | Creates a HDFS-engine table. | +| [s3](../../sql-reference/table-functions/s3.md) | Creates a S3-engine table. | From e6522e1ebe6aecbf66e915db67d78277cfaa4a94 Mon Sep 17 00:00:00 2001 From: Amos Bird Date: Thu, 18 Feb 2021 16:50:31 +0800 Subject: [PATCH 158/544] JBOD data balancer --- src/Storages/MergeTree/DataPartsExchange.cpp | 15 +- src/Storages/MergeTree/DataPartsExchange.h | 3 +- src/Storages/MergeTree/MergeTreeData.cpp | 181 +++++++++++++++++- src/Storages/MergeTree/MergeTreeData.h | 56 +++++- src/Storages/MergeTree/MergeTreeSettings.h | 1 + .../MergeTree/registerStorageMergeTree.cpp | 12 ++ src/Storages/StorageMergeTree.cpp | 17 +- src/Storages/StorageMergeTree.h | 2 +- src/Storages/StorageReplicatedMergeTree.cpp | 19 +- 9 files changed, 290 insertions(+), 16 deletions(-) diff --git a/src/Storages/MergeTree/DataPartsExchange.cpp b/src/Storages/MergeTree/DataPartsExchange.cpp index de7f3b6c0f4..3216b46e7ce 100644 --- a/src/Storages/MergeTree/DataPartsExchange.cpp +++ b/src/Storages/MergeTree/DataPartsExchange.cpp @@ -252,7 +252,8 @@ MergeTreeData::MutableDataPartPtr Fetcher::fetchPart( const String & password, const String & interserver_scheme, bool to_detached, - const String & tmp_prefix_) + const String & tmp_prefix_, + std::optional * tagger_ptr) { if (blocker.isCancelled()) throw Exception("Fetching of part was cancelled", ErrorCodes::ABORTED); @@ -306,10 +307,18 @@ MergeTreeData::MutableDataPartPtr Fetcher::fetchPart( ReadBufferFromString ttl_infos_buffer(ttl_infos_string); assertString("ttl format version: 1\n", ttl_infos_buffer); ttl_infos.read(ttl_infos_buffer); - reservation = data.reserveSpacePreferringTTLRules(metadata_snapshot, sum_files_size, ttl_infos, std::time(nullptr), 0, true); + reservation + = data.balancedReservation(metadata_snapshot, sum_files_size, 0, part_name, part_info, {}, tagger_ptr, &ttl_infos, true); + if (!reservation) + reservation + = data.reserveSpacePreferringTTLRules(metadata_snapshot, sum_files_size, ttl_infos, std::time(nullptr), 0, true); } else - reservation = data.reserveSpace(sum_files_size); + { + reservation = data.balancedReservation(metadata_snapshot, sum_files_size, 0, part_name, part_info, {}, tagger_ptr, nullptr); + if (!reservation) + reservation = data.reserveSpace(sum_files_size); + } } else { diff --git a/src/Storages/MergeTree/DataPartsExchange.h b/src/Storages/MergeTree/DataPartsExchange.h index 834fed1182f..2e2aad57c3c 100644 --- a/src/Storages/MergeTree/DataPartsExchange.h +++ b/src/Storages/MergeTree/DataPartsExchange.h @@ -58,7 +58,8 @@ public: const String & password, const String & interserver_scheme, bool to_detached = false, - const String & tmp_prefix_ = ""); + const String & tmp_prefix_ = "", + std::optional * tagger_ptr = nullptr); /// You need to stop the data transfer. ActionBlocker blocker; diff --git a/src/Storages/MergeTree/MergeTreeData.cpp b/src/Storages/MergeTree/MergeTreeData.cpp index 0c22d5fbc0f..66435c40daa 100644 --- a/src/Storages/MergeTree/MergeTreeData.cpp +++ b/src/Storages/MergeTree/MergeTreeData.cpp @@ -3271,11 +3271,13 @@ ReservationPtr MergeTreeData::reserveSpacePreferringTTLRules( const IMergeTreeDataPart::TTLInfos & ttl_infos, time_t time_of_move, size_t min_volume_index, - bool is_insert) const + bool is_insert, + DiskPtr selected_disk) const { expected_size = std::max(RESERVATION_MIN_ESTIMATION_SIZE, expected_size); - ReservationPtr reservation = tryReserveSpacePreferringTTLRules(metadata_snapshot, expected_size, ttl_infos, time_of_move, min_volume_index, is_insert); + ReservationPtr reservation = tryReserveSpacePreferringTTLRules( + metadata_snapshot, expected_size, ttl_infos, time_of_move, min_volume_index, is_insert, selected_disk); return checkAndReturnReservation(expected_size, std::move(reservation)); } @@ -3286,7 +3288,8 @@ ReservationPtr MergeTreeData::tryReserveSpacePreferringTTLRules( const IMergeTreeDataPart::TTLInfos & ttl_infos, time_t time_of_move, size_t min_volume_index, - bool is_insert) const + bool is_insert, + DiskPtr selected_disk) const { expected_size = std::max(RESERVATION_MIN_ESTIMATION_SIZE, expected_size); @@ -3321,7 +3324,12 @@ ReservationPtr MergeTreeData::tryReserveSpacePreferringTTLRules( } } - reservation = getStoragePolicy()->reserve(expected_size, min_volume_index); + // Prefer selected_disk + if (selected_disk) + reservation = selected_disk->reserve(expected_size); + + if (!reservation) + reservation = getStoragePolicy()->reserve(expected_size, min_volume_index); return reservation; } @@ -4120,4 +4128,169 @@ void MergeTreeData::removeQueryId(const String & query_id) const else query_id_set.erase(query_id); } + +ReservationPtr MergeTreeData::balancedReservation( + const StorageMetadataPtr & metadata_snapshot, + size_t part_size, + size_t max_volume_index, + const String & part_name, + const MergeTreePartInfo & part_info, + MergeTreeData::DataPartsVector covered_parts, + std::optional * tagger_ptr, + const IMergeTreeDataPart::TTLInfos * ttl_infos, + bool is_insert) +{ + ReservationPtr reserved_space; + auto min_bytes_to_rebalance_partition_over_jbod = getSettings()->min_bytes_to_rebalance_partition_over_jbod; + if (tagger_ptr && min_bytes_to_rebalance_partition_over_jbod > 0 && part_size >= min_bytes_to_rebalance_partition_over_jbod) + try + { + auto & disks = getStoragePolicy()->getVolume(max_volume_index)->getDisks(); + std::map disk_occupation; + std::map> disk_parts; + for (auto & disk : disks) + disk_occupation.emplace(disk->getName(), 0); + + std::set big_parts; + std::set merging_parts; + std::lock_guard lock(currently_submerging_emerging_mutex); + + for (auto & part : currently_submerging_parts) + { + if (part->isStoredOnDisk() && part->getBytesOnDisk() >= min_bytes_to_rebalance_partition_over_jbod + && part_info.partition_id == part->info.partition_id) + { + merging_parts.insert(part->name); + } + } + + { + auto lock_parts = lockParts(); + if (covered_parts.empty()) + { + // TODO will it be possible that the covering_part exists when a fetch is upon execution? + MergeTreeData::DataPartPtr covering_part; + covered_parts = getActivePartsToReplace(part_info, part_name, covering_part, lock_parts); + } + + // Also include current submerging parts + for (auto & part : covered_parts) + merging_parts.insert(part->name); + + for (const auto & part : getDataPartsStateRange(MergeTreeData::DataPartState::Committed)) + { + if (part->isStoredOnDisk() && part->getBytesOnDisk() >= min_bytes_to_rebalance_partition_over_jbod + && part_info.partition_id == part->info.partition_id) + { + auto name = part->volume->getDisk()->getName(); + auto it = disk_occupation.find(name); + if (it != disk_occupation.end()) + { + if (merging_parts.find(part->name) == merging_parts.end()) + { + it->second += part->getBytesOnDisk(); + disk_parts[name].push_back(formatReadableSizeWithBinarySuffix(part->getBytesOnDisk())); + big_parts.insert(part->name); + } + else + { + disk_parts[name].push_back(formatReadableSizeWithBinarySuffix(part->getBytesOnDisk()) + " (submerging)"); + } + } + } + } + } + + for (auto & [name, emerging_part] : currently_emerging_parts) + { + // It's possible that the emerging parts are committed and get added twice. Thus a set is used to deduplicate. + if (big_parts.find(name) == big_parts.end()) + { + auto it = disk_occupation.find(emerging_part.disk_name); + if (it != disk_occupation.end()) + { + it->second += emerging_part.estimate_bytes; + disk_parts[emerging_part.disk_name].push_back( + formatReadableSizeWithBinarySuffix(emerging_part.estimate_bytes) + " (emerging)"); + } + } + } + + size_t min_occupation_size = std::numeric_limits::max(); + std::vector candidates; + for (auto & [disk_name, size] : disk_occupation) + { + if (size < min_occupation_size) + { + min_occupation_size = size; + candidates = {disk_name}; + } + else if (size == min_occupation_size) + { + candidates.push_back(disk_name); + } + } + + if (!candidates.empty()) + { + // Random pick one disk from best candidates + std::shuffle(candidates.begin(), candidates.end(), thread_local_rng); + String selected_disk_name = candidates.front(); + WriteBufferFromOwnString log_str; + writeCString("\nbalancer: \n", log_str); + for (auto & [disk_name, per_disk_parts] : disk_parts) + writeString(fmt::format(" {}: [{}]\n", disk_name, boost::algorithm::join(per_disk_parts, ", ")), log_str); + LOG_DEBUG(log, log_str.str()); + + if (ttl_infos) + reserved_space = tryReserveSpacePreferringTTLRules( + metadata_snapshot, + part_size, + *ttl_infos, + time(nullptr), + max_volume_index, + is_insert, + getStoragePolicy()->getDiskByName(selected_disk_name)); + else + reserved_space = tryReserveSpace(part_size, getStoragePolicy()->getDiskByName(selected_disk_name)); + + if (reserved_space) + { + currently_emerging_parts.emplace( + part_name, EmergingPartInfo{reserved_space->getDisk(0)->getName(), part_info.partition_id, part_size}); + + for (const auto & part : covered_parts) + { + if (currently_submerging_parts.count(part)) + LOG_WARNING(log, "currently_submerging_parts has duplicated part. JBOD might lose balance"); + else + currently_submerging_parts.insert(part); + } + + tagger_ptr->emplace(*this, part_name, std::move(covered_parts), log); + } + } + } + catch (...) + { + LOG_DEBUG(log, "JBOD balancer encounters an error. Fallback to random disk selection"); + tryLogCurrentException(log); + } + return reserved_space; +} + +CurrentlySubmergingEmergingTagger::~CurrentlySubmergingEmergingTagger() +{ + std::lock_guard lock(storage.currently_submerging_emerging_mutex); + + for (const auto & part : parts) + { + if (!storage.currently_submerging_parts.count(part)) + LOG_WARNING(log, "currently_submerging_parts is missing parts. JBOD might lose balance"); + else + storage.currently_submerging_parts.erase(part); + } + storage.currently_emerging_parts.erase(name); +} + } diff --git a/src/Storages/MergeTree/MergeTreeData.h b/src/Storages/MergeTree/MergeTreeData.h index 15059ab47e5..27cf2943a7f 100644 --- a/src/Storages/MergeTree/MergeTreeData.h +++ b/src/Storages/MergeTree/MergeTreeData.h @@ -41,6 +41,16 @@ class MutationCommands; class Context; struct JobAndPool; +/// Auxiliary struct holding information about the future merged or mutated part. +struct EmergingPartInfo +{ + String disk_name; + String partition_id; + size_t estimate_bytes; +}; + +struct CurrentlySubmergingEmergingTagger; + class ExpressionActions; using ExpressionActionsPtr = std::shared_ptr; using ManyExpressionActions = std::vector; @@ -651,7 +661,8 @@ public: const IMergeTreeDataPart::TTLInfos & ttl_infos, time_t time_of_move, size_t min_volume_index = 0, - bool is_insert = false) const; + bool is_insert = false, + DiskPtr selected_disk = nullptr) const; ReservationPtr tryReserveSpacePreferringTTLRules( const StorageMetadataPtr & metadata_snapshot, @@ -659,7 +670,22 @@ public: const IMergeTreeDataPart::TTLInfos & ttl_infos, time_t time_of_move, size_t min_volume_index = 0, - bool is_insert = false) const; + bool is_insert = false, + DiskPtr selected_disk = nullptr) const; + + /// Reserves space for the part based on the distribution of "big parts" in the same partition. + /// Parts with estimated size larger than `min_bytes_to_rebalance_partition_over_jbod` are + /// considered as big. The priority is lower than TTL. If reservation fails, return nullptr. + ReservationPtr balancedReservation( + const StorageMetadataPtr & metadata_snapshot, + size_t part_size, + size_t max_volume_index, + const String & part_name, + const MergeTreePartInfo & part_info, + MergeTreeData::DataPartsVector covered_parts, + std::optional * tagger_ptr, + const IMergeTreeDataPart::TTLInfos * ttl_infos, + bool is_insert = false); /// Choose disk with max available free space /// Reserves 0 bytes @@ -741,6 +767,14 @@ public: std::optional getDataMovingJob(); bool areBackgroundMovesNeeded() const; + /// Parts that currently submerging (merging to bigger parts) or emerging + /// (to be appeared after merging finished). This set have to be used + /// with `currently_submerging_emerging_mutex`. + DataParts currently_submerging_parts; + std::map currently_emerging_parts; + /// Mutex for currently_submerging_parts and currently_emerging_parts + mutable std::mutex currently_submerging_emerging_mutex; + protected: friend class IMergeTreeDataPart; @@ -975,4 +1009,22 @@ private: mutable std::mutex query_id_set_mutex; }; +/// RAII struct to record big parts that are submerging or emerging. +/// It's used to calculate the balanced statistics of JBOD array. +struct CurrentlySubmergingEmergingTagger +{ + MergeTreeData & storage; + String name; + MergeTreeData::DataPartsVector parts; + Poco::Logger * log; + + CurrentlySubmergingEmergingTagger( + MergeTreeData & storage_, const String & name_, MergeTreeData::DataPartsVector && parts_, Poco::Logger * log_) + : storage(storage_), name(name_), parts(std::move(parts_)), log(log_) + { + } + + ~CurrentlySubmergingEmergingTagger(); +}; + } diff --git a/src/Storages/MergeTree/MergeTreeSettings.h b/src/Storages/MergeTree/MergeTreeSettings.h index 16657b4083d..d64f7921412 100644 --- a/src/Storages/MergeTree/MergeTreeSettings.h +++ b/src/Storages/MergeTree/MergeTreeSettings.h @@ -119,6 +119,7 @@ struct Settings; M(Int64, max_partitions_to_read, -1, "Limit the max number of partitions that can be accessed in one query. <= 0 means unlimited. This setting is the default that can be overridden by the query-level setting with the same name.", 0) \ M(UInt64, max_concurrent_queries, 0, "Max number of concurrently executed queries related to the MergeTree table (0 - disabled). Queries will still be limited by other max_concurrent_queries settings.", 0) \ M(UInt64, min_marks_to_honor_max_concurrent_queries, 0, "Minimal number of marks to honor the MergeTree-level's max_concurrent_queries (0 - disabled). Queries will still be limited by other max_concurrent_queries settings.", 0) \ + M(UInt64, min_bytes_to_rebalance_partition_over_jbod, 512 * 1024 * 1024, "Minimal amount of bytes to enable part rebalance over JBOD array (0 - disabled).", 0) \ \ /** Obsolete settings. Kept for backward compatibility only. */ \ M(UInt64, min_relative_delay_to_yield_leadership, 120, "Obsolete setting, does nothing.", 0) \ diff --git a/src/Storages/MergeTree/registerStorageMergeTree.cpp b/src/Storages/MergeTree/registerStorageMergeTree.cpp index d48f85f3e4d..45c9aa81492 100644 --- a/src/Storages/MergeTree/registerStorageMergeTree.cpp +++ b/src/Storages/MergeTree/registerStorageMergeTree.cpp @@ -673,6 +673,18 @@ static StoragePtr create(const StorageFactory::Arguments & args) + " is lesser than specified min_index_granularity_bytes: " + std::to_string(min_index_granularity_bytes), ErrorCodes::BAD_ARGUMENTS); } + + // Pre-define a reasonable minimum size for the JBOD rebalancer + static constexpr size_t MIN_BYTES_TO_REBALANCE_OVER_JBOD = 100 * 1024 * 1024; + if (storage_settings->min_bytes_to_rebalance_partition_over_jbod > 0 + && storage_settings->min_bytes_to_rebalance_partition_over_jbod < MIN_BYTES_TO_REBALANCE_OVER_JBOD) + { + throw Exception( + "min_bytes_to_rebalance_partition_over_jbod: " + + std::to_string(storage_settings->min_bytes_to_rebalance_partition_over_jbod) + " is lesser than " + + std::to_string(MIN_BYTES_TO_REBALANCE_OVER_JBOD), + ErrorCodes::BAD_ARGUMENTS); + } } else { diff --git a/src/Storages/StorageMergeTree.cpp b/src/Storages/StorageMergeTree.cpp index c8f44c78e6e..d6cf4665fc9 100644 --- a/src/Storages/StorageMergeTree.cpp +++ b/src/Storages/StorageMergeTree.cpp @@ -332,12 +332,25 @@ StorageMergeTree::CurrentlyMergingPartsTagger::CurrentlyMergingPartsTagger( max_volume_index = std::max(max_volume_index, storage.getStoragePolicy()->getVolumeIndexByDisk(part_ptr->volume->getDisk())); } - reserved_space = storage.tryReserveSpacePreferringTTLRules(metadata_snapshot, total_size, ttl_infos, time(nullptr), max_volume_index); + reserved_space = storage.balancedReservation( + metadata_snapshot, + total_size, + max_volume_index, + future_part.name, + future_part.part_info, + future_part.parts, + &tagger, + &ttl_infos); + + if (!reserved_space) + reserved_space + = storage.tryReserveSpacePreferringTTLRules(metadata_snapshot, total_size, ttl_infos, time(nullptr), max_volume_index); } + if (!reserved_space) { if (is_mutation) - throw Exception("Not enough space for mutating part '" + future_part_.parts[0]->name + "'", ErrorCodes::NOT_ENOUGH_SPACE); + throw Exception("Not enough space for mutating part '" + future_part.parts[0]->name + "'", ErrorCodes::NOT_ENOUGH_SPACE); else throw Exception("Not enough space for merging parts", ErrorCodes::NOT_ENOUGH_SPACE); } diff --git a/src/Storages/StorageMergeTree.h b/src/Storages/StorageMergeTree.h index 9dd62439814..ab1de9944e9 100644 --- a/src/Storages/StorageMergeTree.h +++ b/src/Storages/StorageMergeTree.h @@ -150,8 +150,8 @@ private: { FutureMergedMutatedPart future_part; ReservationPtr reserved_space; - StorageMergeTree & storage; + std::optional tagger; CurrentlyMergingPartsTagger( FutureMergedMutatedPart & future_part_, diff --git a/src/Storages/StorageReplicatedMergeTree.cpp b/src/Storages/StorageReplicatedMergeTree.cpp index 68f3b6d80d1..9bea36e6252 100644 --- a/src/Storages/StorageReplicatedMergeTree.cpp +++ b/src/Storages/StorageReplicatedMergeTree.cpp @@ -1516,15 +1516,28 @@ bool StorageReplicatedMergeTree::tryExecuteMerge(const LogEntry & entry) auto table_lock = lockForShare(RWLockImpl::NO_QUERY, storage_settings_ptr->lock_acquire_timeout_for_background_operations); StorageMetadataPtr metadata_snapshot = getInMemoryMetadataPtr(); - ReservationPtr reserved_space = reserveSpacePreferringTTLRules( - metadata_snapshot, estimated_space_for_merge, ttl_infos, time(nullptr), max_volume_index); - FutureMergedMutatedPart future_merged_part(parts, entry.new_part_type); if (future_merged_part.name != entry.new_part_name) { throw Exception("Future merged part name " + backQuote(future_merged_part.name) + " differs from part name in log entry: " + backQuote(entry.new_part_name), ErrorCodes::BAD_DATA_PART_NAME); } + + std::optional tagger; + ReservationPtr reserved_space = balancedReservation( + metadata_snapshot, + estimated_space_for_merge, + max_volume_index, + future_merged_part.name, + future_merged_part.part_info, + future_merged_part.parts, + &tagger, + &ttl_infos); + + if (!reserved_space) + reserved_space + = reserveSpacePreferringTTLRules(metadata_snapshot, estimated_space_for_merge, ttl_infos, time(nullptr), max_volume_index); + future_merged_part.uuid = entry.new_part_uuid; future_merged_part.updatePath(*this, reserved_space); future_merged_part.merge_type = entry.merge_type; From 2b11a2a855f51996829c8dae4b140f103e3fb509 Mon Sep 17 00:00:00 2001 From: Amos Bird Date: Fri, 19 Feb 2021 14:44:10 +0800 Subject: [PATCH 159/544] Fix build --- src/Storages/MergeTree/MergeTreeData.cpp | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/src/Storages/MergeTree/MergeTreeData.cpp b/src/Storages/MergeTree/MergeTreeData.cpp index 66435c40daa..e59b35ad14b 100644 --- a/src/Storages/MergeTree/MergeTreeData.cpp +++ b/src/Storages/MergeTree/MergeTreeData.cpp @@ -4145,17 +4145,17 @@ ReservationPtr MergeTreeData::balancedReservation( if (tagger_ptr && min_bytes_to_rebalance_partition_over_jbod > 0 && part_size >= min_bytes_to_rebalance_partition_over_jbod) try { - auto & disks = getStoragePolicy()->getVolume(max_volume_index)->getDisks(); + const auto & disks = getStoragePolicy()->getVolume(max_volume_index)->getDisks(); std::map disk_occupation; std::map> disk_parts; - for (auto & disk : disks) + for (const auto & disk : disks) disk_occupation.emplace(disk->getName(), 0); std::set big_parts; std::set merging_parts; std::lock_guard lock(currently_submerging_emerging_mutex); - for (auto & part : currently_submerging_parts) + for (const auto & part : currently_submerging_parts) { if (part->isStoredOnDisk() && part->getBytesOnDisk() >= min_bytes_to_rebalance_partition_over_jbod && part_info.partition_id == part->info.partition_id) @@ -4174,7 +4174,7 @@ ReservationPtr MergeTreeData::balancedReservation( } // Also include current submerging parts - for (auto & part : covered_parts) + for (const auto & part : covered_parts) merging_parts.insert(part->name); for (const auto & part : getDataPartsStateRange(MergeTreeData::DataPartState::Committed)) @@ -4201,7 +4201,7 @@ ReservationPtr MergeTreeData::balancedReservation( } } - for (auto & [name, emerging_part] : currently_emerging_parts) + for (const auto & [name, emerging_part] : currently_emerging_parts) { // It's possible that the emerging parts are committed and get added twice. Thus a set is used to deduplicate. if (big_parts.find(name) == big_parts.end()) @@ -4218,7 +4218,7 @@ ReservationPtr MergeTreeData::balancedReservation( size_t min_occupation_size = std::numeric_limits::max(); std::vector candidates; - for (auto & [disk_name, size] : disk_occupation) + for (const auto & [disk_name, size] : disk_occupation) { if (size < min_occupation_size) { @@ -4238,7 +4238,7 @@ ReservationPtr MergeTreeData::balancedReservation( String selected_disk_name = candidates.front(); WriteBufferFromOwnString log_str; writeCString("\nbalancer: \n", log_str); - for (auto & [disk_name, per_disk_parts] : disk_parts) + for (const auto & [disk_name, per_disk_parts] : disk_parts) writeString(fmt::format(" {}: [{}]\n", disk_name, boost::algorithm::join(per_disk_parts, ", ")), log_str); LOG_DEBUG(log, log_str.str()); From cc2837485fecb8ce7f1cb7b31a23c273d25fa3fc Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Mon, 8 Mar 2021 09:25:02 +0300 Subject: [PATCH 160/544] Remove prefetch --- contrib/FastMemcpy/FastMemcpy.h | 1448 ++++++++++++++------------- contrib/FastMemcpy/FastMemcpy_Avx.h | 972 +++++++++--------- 2 files changed, 1234 insertions(+), 1186 deletions(-) diff --git a/contrib/FastMemcpy/FastMemcpy.h b/contrib/FastMemcpy/FastMemcpy.h index 5dcbfcf1656..046ed3fc670 100644 --- a/contrib/FastMemcpy/FastMemcpy.h +++ b/contrib/FastMemcpy/FastMemcpy.h @@ -1,694 +1,754 @@ -//===================================================================== -// -// FastMemcpy.c - skywind3000@163.com, 2015 -// -// feature: -// 50% speed up in avg. vs standard memcpy (tested in vc2012/gcc5.1) -// -//===================================================================== -#ifndef __FAST_MEMCPY_H__ -#define __FAST_MEMCPY_H__ - -#include -#include -#include - - -//--------------------------------------------------------------------- -// force inline for compilers -//--------------------------------------------------------------------- -#ifndef INLINE -#ifdef __GNUC__ -#if (__GNUC__ > 3) || ((__GNUC__ == 3) && (__GNUC_MINOR__ >= 1)) - #define INLINE __inline__ __attribute__((always_inline)) -#else - #define INLINE __inline__ -#endif -#elif defined(_MSC_VER) - #define INLINE __forceinline -#elif (defined(__BORLANDC__) || defined(__WATCOMC__)) - #define INLINE __inline -#else - #define INLINE -#endif -#endif - -typedef __attribute__((__aligned__(1))) uint16_t uint16_unaligned_t; -typedef __attribute__((__aligned__(1))) uint32_t uint32_unaligned_t; -typedef __attribute__((__aligned__(1))) uint64_t uint64_unaligned_t; - -//--------------------------------------------------------------------- -// fast copy for different sizes -//--------------------------------------------------------------------- -static INLINE void memcpy_sse2_16(void *dst, const void *src) { - __m128i m0 = _mm_loadu_si128(((const __m128i*)src) + 0); - _mm_storeu_si128(((__m128i*)dst) + 0, m0); -} - -static INLINE void memcpy_sse2_32(void *dst, const void *src) { - __m128i m0 = _mm_loadu_si128(((const __m128i*)src) + 0); - __m128i m1 = _mm_loadu_si128(((const __m128i*)src) + 1); - _mm_storeu_si128(((__m128i*)dst) + 0, m0); - _mm_storeu_si128(((__m128i*)dst) + 1, m1); -} - -static INLINE void memcpy_sse2_64(void *dst, const void *src) { - __m128i m0 = _mm_loadu_si128(((const __m128i*)src) + 0); - __m128i m1 = _mm_loadu_si128(((const __m128i*)src) + 1); - __m128i m2 = _mm_loadu_si128(((const __m128i*)src) + 2); - __m128i m3 = _mm_loadu_si128(((const __m128i*)src) + 3); - _mm_storeu_si128(((__m128i*)dst) + 0, m0); - _mm_storeu_si128(((__m128i*)dst) + 1, m1); - _mm_storeu_si128(((__m128i*)dst) + 2, m2); - _mm_storeu_si128(((__m128i*)dst) + 3, m3); -} - -static INLINE void memcpy_sse2_128(void *dst, const void *src) { - __m128i m0 = _mm_loadu_si128(((const __m128i*)src) + 0); - __m128i m1 = _mm_loadu_si128(((const __m128i*)src) + 1); - __m128i m2 = _mm_loadu_si128(((const __m128i*)src) + 2); - __m128i m3 = _mm_loadu_si128(((const __m128i*)src) + 3); - __m128i m4 = _mm_loadu_si128(((const __m128i*)src) + 4); - __m128i m5 = _mm_loadu_si128(((const __m128i*)src) + 5); - __m128i m6 = _mm_loadu_si128(((const __m128i*)src) + 6); - __m128i m7 = _mm_loadu_si128(((const __m128i*)src) + 7); - _mm_storeu_si128(((__m128i*)dst) + 0, m0); - _mm_storeu_si128(((__m128i*)dst) + 1, m1); - _mm_storeu_si128(((__m128i*)dst) + 2, m2); - _mm_storeu_si128(((__m128i*)dst) + 3, m3); - _mm_storeu_si128(((__m128i*)dst) + 4, m4); - _mm_storeu_si128(((__m128i*)dst) + 5, m5); - _mm_storeu_si128(((__m128i*)dst) + 6, m6); - _mm_storeu_si128(((__m128i*)dst) + 7, m7); -} - - -//--------------------------------------------------------------------- -// tiny memory copy with jump table optimized -//--------------------------------------------------------------------- -/// Attribute is used to avoid an error with undefined behaviour sanitizer -/// ../contrib/FastMemcpy/FastMemcpy.h:91:56: runtime error: applying zero offset to null pointer -/// Found by 01307_orc_output_format.sh, cause - ORCBlockInputFormat and external ORC library. -__attribute__((__no_sanitize__("undefined"))) static INLINE void *memcpy_tiny(void *dst, const void *src, size_t size) { - unsigned char *dd = ((unsigned char*)dst) + size; - const unsigned char *ss = ((const unsigned char*)src) + size; - - switch (size) { - case 64: - memcpy_sse2_64(dd - 64, ss - 64); - case 0: - break; - - case 65: - memcpy_sse2_64(dd - 65, ss - 65); - case 1: - dd[-1] = ss[-1]; - break; - - case 66: - memcpy_sse2_64(dd - 66, ss - 66); - case 2: - *((uint16_unaligned_t*)(dd - 2)) = *((uint16_unaligned_t*)(ss - 2)); - break; - - case 67: - memcpy_sse2_64(dd - 67, ss - 67); - case 3: - *((uint16_unaligned_t*)(dd - 3)) = *((uint16_unaligned_t*)(ss - 3)); - dd[-1] = ss[-1]; - break; - - case 68: - memcpy_sse2_64(dd - 68, ss - 68); - case 4: - *((uint32_unaligned_t*)(dd - 4)) = *((uint32_unaligned_t*)(ss - 4)); - break; - - case 69: - memcpy_sse2_64(dd - 69, ss - 69); - case 5: - *((uint32_unaligned_t*)(dd - 5)) = *((uint32_unaligned_t*)(ss - 5)); - dd[-1] = ss[-1]; - break; - - case 70: - memcpy_sse2_64(dd - 70, ss - 70); - case 6: - *((uint32_unaligned_t*)(dd - 6)) = *((uint32_unaligned_t*)(ss - 6)); - *((uint16_unaligned_t*)(dd - 2)) = *((uint16_unaligned_t*)(ss - 2)); - break; - - case 71: - memcpy_sse2_64(dd - 71, ss - 71); - case 7: - *((uint32_unaligned_t*)(dd - 7)) = *((uint32_unaligned_t*)(ss - 7)); - *((uint32_unaligned_t*)(dd - 4)) = *((uint32_unaligned_t*)(ss - 4)); - break; - - case 72: - memcpy_sse2_64(dd - 72, ss - 72); - case 8: - *((uint64_unaligned_t*)(dd - 8)) = *((uint64_unaligned_t*)(ss - 8)); - break; - - case 73: - memcpy_sse2_64(dd - 73, ss - 73); - case 9: - *((uint64_unaligned_t*)(dd - 9)) = *((uint64_unaligned_t*)(ss - 9)); - dd[-1] = ss[-1]; - break; - - case 74: - memcpy_sse2_64(dd - 74, ss - 74); - case 10: - *((uint64_unaligned_t*)(dd - 10)) = *((uint64_unaligned_t*)(ss - 10)); - *((uint16_unaligned_t*)(dd - 2)) = *((uint16_unaligned_t*)(ss - 2)); - break; - - case 75: - memcpy_sse2_64(dd - 75, ss - 75); - case 11: - *((uint64_unaligned_t*)(dd - 11)) = *((uint64_unaligned_t*)(ss - 11)); - *((uint32_unaligned_t*)(dd - 4)) = *((uint32_unaligned_t*)(ss - 4)); - break; - - case 76: - memcpy_sse2_64(dd - 76, ss - 76); - case 12: - *((uint64_unaligned_t*)(dd - 12)) = *((uint64_unaligned_t*)(ss - 12)); - *((uint32_unaligned_t*)(dd - 4)) = *((uint32_unaligned_t*)(ss - 4)); - break; - - case 77: - memcpy_sse2_64(dd - 77, ss - 77); - case 13: - *((uint64_unaligned_t*)(dd - 13)) = *((uint64_unaligned_t*)(ss - 13)); - *((uint32_unaligned_t*)(dd - 5)) = *((uint32_unaligned_t*)(ss - 5)); - dd[-1] = ss[-1]; - break; - - case 78: - memcpy_sse2_64(dd - 78, ss - 78); - case 14: - *((uint64_unaligned_t*)(dd - 14)) = *((uint64_unaligned_t*)(ss - 14)); - *((uint64_unaligned_t*)(dd - 8)) = *((uint64_unaligned_t*)(ss - 8)); - break; - - case 79: - memcpy_sse2_64(dd - 79, ss - 79); - case 15: - *((uint64_unaligned_t*)(dd - 15)) = *((uint64_unaligned_t*)(ss - 15)); - *((uint64_unaligned_t*)(dd - 8)) = *((uint64_unaligned_t*)(ss - 8)); - break; - - case 80: - memcpy_sse2_64(dd - 80, ss - 80); - case 16: - memcpy_sse2_16(dd - 16, ss - 16); - break; - - case 81: - memcpy_sse2_64(dd - 81, ss - 81); - case 17: - memcpy_sse2_16(dd - 17, ss - 17); - dd[-1] = ss[-1]; - break; - - case 82: - memcpy_sse2_64(dd - 82, ss - 82); - case 18: - memcpy_sse2_16(dd - 18, ss - 18); - *((uint16_unaligned_t*)(dd - 2)) = *((uint16_unaligned_t*)(ss - 2)); - break; - - case 83: - memcpy_sse2_64(dd - 83, ss - 83); - case 19: - memcpy_sse2_16(dd - 19, ss - 19); - *((uint16_unaligned_t*)(dd - 3)) = *((uint16_unaligned_t*)(ss - 3)); - dd[-1] = ss[-1]; - break; - - case 84: - memcpy_sse2_64(dd - 84, ss - 84); - case 20: - memcpy_sse2_16(dd - 20, ss - 20); - *((uint32_unaligned_t*)(dd - 4)) = *((uint32_unaligned_t*)(ss - 4)); - break; - - case 85: - memcpy_sse2_64(dd - 85, ss - 85); - case 21: - memcpy_sse2_16(dd - 21, ss - 21); - *((uint32_unaligned_t*)(dd - 5)) = *((uint32_unaligned_t*)(ss - 5)); - dd[-1] = ss[-1]; - break; - - case 86: - memcpy_sse2_64(dd - 86, ss - 86); - case 22: - memcpy_sse2_16(dd - 22, ss - 22); - *((uint32_unaligned_t*)(dd - 6)) = *((uint32_unaligned_t*)(ss - 6)); - *((uint16_unaligned_t*)(dd - 2)) = *((uint16_unaligned_t*)(ss - 2)); - break; - - case 87: - memcpy_sse2_64(dd - 87, ss - 87); - case 23: - memcpy_sse2_16(dd - 23, ss - 23); - *((uint32_unaligned_t*)(dd - 7)) = *((uint32_unaligned_t*)(ss - 7)); - *((uint32_unaligned_t*)(dd - 4)) = *((uint32_unaligned_t*)(ss - 4)); - break; - - case 88: - memcpy_sse2_64(dd - 88, ss - 88); - case 24: - memcpy_sse2_16(dd - 24, ss - 24); - memcpy_sse2_16(dd - 16, ss - 16); - break; - - case 89: - memcpy_sse2_64(dd - 89, ss - 89); - case 25: - memcpy_sse2_16(dd - 25, ss - 25); - memcpy_sse2_16(dd - 16, ss - 16); - break; - - case 90: - memcpy_sse2_64(dd - 90, ss - 90); - case 26: - memcpy_sse2_16(dd - 26, ss - 26); - memcpy_sse2_16(dd - 16, ss - 16); - break; - - case 91: - memcpy_sse2_64(dd - 91, ss - 91); - case 27: - memcpy_sse2_16(dd - 27, ss - 27); - memcpy_sse2_16(dd - 16, ss - 16); - break; - - case 92: - memcpy_sse2_64(dd - 92, ss - 92); - case 28: - memcpy_sse2_16(dd - 28, ss - 28); - memcpy_sse2_16(dd - 16, ss - 16); - break; - - case 93: - memcpy_sse2_64(dd - 93, ss - 93); - case 29: - memcpy_sse2_16(dd - 29, ss - 29); - memcpy_sse2_16(dd - 16, ss - 16); - break; - - case 94: - memcpy_sse2_64(dd - 94, ss - 94); - case 30: - memcpy_sse2_16(dd - 30, ss - 30); - memcpy_sse2_16(dd - 16, ss - 16); - break; - - case 95: - memcpy_sse2_64(dd - 95, ss - 95); - case 31: - memcpy_sse2_16(dd - 31, ss - 31); - memcpy_sse2_16(dd - 16, ss - 16); - break; - - case 96: - memcpy_sse2_64(dd - 96, ss - 96); - case 32: - memcpy_sse2_32(dd - 32, ss - 32); - break; - - case 97: - memcpy_sse2_64(dd - 97, ss - 97); - case 33: - memcpy_sse2_32(dd - 33, ss - 33); - dd[-1] = ss[-1]; - break; - - case 98: - memcpy_sse2_64(dd - 98, ss - 98); - case 34: - memcpy_sse2_32(dd - 34, ss - 34); - *((uint16_unaligned_t*)(dd - 2)) = *((uint16_unaligned_t*)(ss - 2)); - break; - - case 99: - memcpy_sse2_64(dd - 99, ss - 99); - case 35: - memcpy_sse2_32(dd - 35, ss - 35); - *((uint16_unaligned_t*)(dd - 3)) = *((uint16_unaligned_t*)(ss - 3)); - dd[-1] = ss[-1]; - break; - - case 100: - memcpy_sse2_64(dd - 100, ss - 100); - case 36: - memcpy_sse2_32(dd - 36, ss - 36); - *((uint32_unaligned_t*)(dd - 4)) = *((uint32_unaligned_t*)(ss - 4)); - break; - - case 101: - memcpy_sse2_64(dd - 101, ss - 101); - case 37: - memcpy_sse2_32(dd - 37, ss - 37); - *((uint32_unaligned_t*)(dd - 5)) = *((uint32_unaligned_t*)(ss - 5)); - dd[-1] = ss[-1]; - break; - - case 102: - memcpy_sse2_64(dd - 102, ss - 102); - case 38: - memcpy_sse2_32(dd - 38, ss - 38); - *((uint32_unaligned_t*)(dd - 6)) = *((uint32_unaligned_t*)(ss - 6)); - *((uint16_unaligned_t*)(dd - 2)) = *((uint16_unaligned_t*)(ss - 2)); - break; - - case 103: - memcpy_sse2_64(dd - 103, ss - 103); - case 39: - memcpy_sse2_32(dd - 39, ss - 39); - *((uint32_unaligned_t*)(dd - 7)) = *((uint32_unaligned_t*)(ss - 7)); - *((uint32_unaligned_t*)(dd - 4)) = *((uint32_unaligned_t*)(ss - 4)); - break; - - case 104: - memcpy_sse2_64(dd - 104, ss - 104); - case 40: - memcpy_sse2_32(dd - 40, ss - 40); - *((uint64_unaligned_t*)(dd - 8)) = *((uint64_unaligned_t*)(ss - 8)); - break; - - case 105: - memcpy_sse2_64(dd - 105, ss - 105); - case 41: - memcpy_sse2_32(dd - 41, ss - 41); - *((uint64_unaligned_t*)(dd - 9)) = *((uint64_unaligned_t*)(ss - 9)); - dd[-1] = ss[-1]; - break; - - case 106: - memcpy_sse2_64(dd - 106, ss - 106); - case 42: - memcpy_sse2_32(dd - 42, ss - 42); - *((uint64_unaligned_t*)(dd - 10)) = *((uint64_unaligned_t*)(ss - 10)); - *((uint16_unaligned_t*)(dd - 2)) = *((uint16_unaligned_t*)(ss - 2)); - break; - - case 107: - memcpy_sse2_64(dd - 107, ss - 107); - case 43: - memcpy_sse2_32(dd - 43, ss - 43); - *((uint64_unaligned_t*)(dd - 11)) = *((uint64_unaligned_t*)(ss - 11)); - *((uint32_unaligned_t*)(dd - 4)) = *((uint32_unaligned_t*)(ss - 4)); - break; - - case 108: - memcpy_sse2_64(dd - 108, ss - 108); - case 44: - memcpy_sse2_32(dd - 44, ss - 44); - *((uint64_unaligned_t*)(dd - 12)) = *((uint64_unaligned_t*)(ss - 12)); - *((uint32_unaligned_t*)(dd - 4)) = *((uint32_unaligned_t*)(ss - 4)); - break; - - case 109: - memcpy_sse2_64(dd - 109, ss - 109); - case 45: - memcpy_sse2_32(dd - 45, ss - 45); - *((uint64_unaligned_t*)(dd - 13)) = *((uint64_unaligned_t*)(ss - 13)); - *((uint32_unaligned_t*)(dd - 5)) = *((uint32_unaligned_t*)(ss - 5)); - dd[-1] = ss[-1]; - break; - - case 110: - memcpy_sse2_64(dd - 110, ss - 110); - case 46: - memcpy_sse2_32(dd - 46, ss - 46); - *((uint64_unaligned_t*)(dd - 14)) = *((uint64_unaligned_t*)(ss - 14)); - *((uint64_unaligned_t*)(dd - 8)) = *((uint64_unaligned_t*)(ss - 8)); - break; - - case 111: - memcpy_sse2_64(dd - 111, ss - 111); - case 47: - memcpy_sse2_32(dd - 47, ss - 47); - *((uint64_unaligned_t*)(dd - 15)) = *((uint64_unaligned_t*)(ss - 15)); - *((uint64_unaligned_t*)(dd - 8)) = *((uint64_unaligned_t*)(ss - 8)); - break; - - case 112: - memcpy_sse2_64(dd - 112, ss - 112); - case 48: - memcpy_sse2_32(dd - 48, ss - 48); - memcpy_sse2_16(dd - 16, ss - 16); - break; - - case 113: - memcpy_sse2_64(dd - 113, ss - 113); - case 49: - memcpy_sse2_32(dd - 49, ss - 49); - memcpy_sse2_16(dd - 17, ss - 17); - dd[-1] = ss[-1]; - break; - - case 114: - memcpy_sse2_64(dd - 114, ss - 114); - case 50: - memcpy_sse2_32(dd - 50, ss - 50); - memcpy_sse2_16(dd - 18, ss - 18); - *((uint16_unaligned_t*)(dd - 2)) = *((uint16_unaligned_t*)(ss - 2)); - break; - - case 115: - memcpy_sse2_64(dd - 115, ss - 115); - case 51: - memcpy_sse2_32(dd - 51, ss - 51); - memcpy_sse2_16(dd - 19, ss - 19); - *((uint16_unaligned_t*)(dd - 3)) = *((uint16_unaligned_t*)(ss - 3)); - dd[-1] = ss[-1]; - break; - - case 116: - memcpy_sse2_64(dd - 116, ss - 116); - case 52: - memcpy_sse2_32(dd - 52, ss - 52); - memcpy_sse2_16(dd - 20, ss - 20); - *((uint32_unaligned_t*)(dd - 4)) = *((uint32_unaligned_t*)(ss - 4)); - break; - - case 117: - memcpy_sse2_64(dd - 117, ss - 117); - case 53: - memcpy_sse2_32(dd - 53, ss - 53); - memcpy_sse2_16(dd - 21, ss - 21); - *((uint32_unaligned_t*)(dd - 5)) = *((uint32_unaligned_t*)(ss - 5)); - dd[-1] = ss[-1]; - break; - - case 118: - memcpy_sse2_64(dd - 118, ss - 118); - case 54: - memcpy_sse2_32(dd - 54, ss - 54); - memcpy_sse2_16(dd - 22, ss - 22); - *((uint32_unaligned_t*)(dd - 6)) = *((uint32_unaligned_t*)(ss - 6)); - *((uint16_unaligned_t*)(dd - 2)) = *((uint16_unaligned_t*)(ss - 2)); - break; - - case 119: - memcpy_sse2_64(dd - 119, ss - 119); - case 55: - memcpy_sse2_32(dd - 55, ss - 55); - memcpy_sse2_16(dd - 23, ss - 23); - *((uint32_unaligned_t*)(dd - 7)) = *((uint32_unaligned_t*)(ss - 7)); - *((uint32_unaligned_t*)(dd - 4)) = *((uint32_unaligned_t*)(ss - 4)); - break; - - case 120: - memcpy_sse2_64(dd - 120, ss - 120); - case 56: - memcpy_sse2_32(dd - 56, ss - 56); - memcpy_sse2_16(dd - 24, ss - 24); - memcpy_sse2_16(dd - 16, ss - 16); - break; - - case 121: - memcpy_sse2_64(dd - 121, ss - 121); - case 57: - memcpy_sse2_32(dd - 57, ss - 57); - memcpy_sse2_16(dd - 25, ss - 25); - memcpy_sse2_16(dd - 16, ss - 16); - break; - - case 122: - memcpy_sse2_64(dd - 122, ss - 122); - case 58: - memcpy_sse2_32(dd - 58, ss - 58); - memcpy_sse2_16(dd - 26, ss - 26); - memcpy_sse2_16(dd - 16, ss - 16); - break; - - case 123: - memcpy_sse2_64(dd - 123, ss - 123); - case 59: - memcpy_sse2_32(dd - 59, ss - 59); - memcpy_sse2_16(dd - 27, ss - 27); - memcpy_sse2_16(dd - 16, ss - 16); - break; - - case 124: - memcpy_sse2_64(dd - 124, ss - 124); - case 60: - memcpy_sse2_32(dd - 60, ss - 60); - memcpy_sse2_16(dd - 28, ss - 28); - memcpy_sse2_16(dd - 16, ss - 16); - break; - - case 125: - memcpy_sse2_64(dd - 125, ss - 125); - case 61: - memcpy_sse2_32(dd - 61, ss - 61); - memcpy_sse2_16(dd - 29, ss - 29); - memcpy_sse2_16(dd - 16, ss - 16); - break; - - case 126: - memcpy_sse2_64(dd - 126, ss - 126); - case 62: - memcpy_sse2_32(dd - 62, ss - 62); - memcpy_sse2_16(dd - 30, ss - 30); - memcpy_sse2_16(dd - 16, ss - 16); - break; - - case 127: - memcpy_sse2_64(dd - 127, ss - 127); - case 63: - memcpy_sse2_32(dd - 63, ss - 63); - memcpy_sse2_16(dd - 31, ss - 31); - memcpy_sse2_16(dd - 16, ss - 16); - break; - - case 128: - memcpy_sse2_128(dd - 128, ss - 128); - break; - } - - return dst; -} - - -//--------------------------------------------------------------------- -// main routine -//--------------------------------------------------------------------- -static void* memcpy_fast(void *destination, const void *source, size_t size) -{ - unsigned char *dst = (unsigned char*)destination; - const unsigned char *src = (const unsigned char*)source; - static size_t cachesize = 0x200000; // L2-cache size - size_t padding; - - // small memory copy - if (size <= 128) { - return memcpy_tiny(dst, src, size); - } - - // align destination to 16 bytes boundary - padding = (16 - (((size_t)dst) & 15)) & 15; - - if (padding > 0) { - __m128i head = _mm_loadu_si128((const __m128i*)src); - _mm_storeu_si128((__m128i*)dst, head); - dst += padding; - src += padding; - size -= padding; - } - - // medium size copy - if (size <= cachesize) { - __m128i c0, c1, c2, c3, c4, c5, c6, c7; - - for (; size >= 128; size -= 128) { - c0 = _mm_loadu_si128(((const __m128i*)src) + 0); - c1 = _mm_loadu_si128(((const __m128i*)src) + 1); - c2 = _mm_loadu_si128(((const __m128i*)src) + 2); - c3 = _mm_loadu_si128(((const __m128i*)src) + 3); - c4 = _mm_loadu_si128(((const __m128i*)src) + 4); - c5 = _mm_loadu_si128(((const __m128i*)src) + 5); - c6 = _mm_loadu_si128(((const __m128i*)src) + 6); - c7 = _mm_loadu_si128(((const __m128i*)src) + 7); - _mm_prefetch((const char*)(src + 256), _MM_HINT_NTA); - src += 128; - _mm_store_si128((((__m128i*)dst) + 0), c0); - _mm_store_si128((((__m128i*)dst) + 1), c1); - _mm_store_si128((((__m128i*)dst) + 2), c2); - _mm_store_si128((((__m128i*)dst) + 3), c3); - _mm_store_si128((((__m128i*)dst) + 4), c4); - _mm_store_si128((((__m128i*)dst) + 5), c5); - _mm_store_si128((((__m128i*)dst) + 6), c6); - _mm_store_si128((((__m128i*)dst) + 7), c7); - dst += 128; - } - } - else { // big memory copy - __m128i c0, c1, c2, c3, c4, c5, c6, c7; - - _mm_prefetch((const char*)(src), _MM_HINT_NTA); - - if ((((size_t)src) & 15) == 0) { // source aligned - for (; size >= 128; size -= 128) { - c0 = _mm_load_si128(((const __m128i*)src) + 0); - c1 = _mm_load_si128(((const __m128i*)src) + 1); - c2 = _mm_load_si128(((const __m128i*)src) + 2); - c3 = _mm_load_si128(((const __m128i*)src) + 3); - c4 = _mm_load_si128(((const __m128i*)src) + 4); - c5 = _mm_load_si128(((const __m128i*)src) + 5); - c6 = _mm_load_si128(((const __m128i*)src) + 6); - c7 = _mm_load_si128(((const __m128i*)src) + 7); - _mm_prefetch((const char*)(src + 256), _MM_HINT_NTA); - src += 128; - _mm_stream_si128((((__m128i*)dst) + 0), c0); - _mm_stream_si128((((__m128i*)dst) + 1), c1); - _mm_stream_si128((((__m128i*)dst) + 2), c2); - _mm_stream_si128((((__m128i*)dst) + 3), c3); - _mm_stream_si128((((__m128i*)dst) + 4), c4); - _mm_stream_si128((((__m128i*)dst) + 5), c5); - _mm_stream_si128((((__m128i*)dst) + 6), c6); - _mm_stream_si128((((__m128i*)dst) + 7), c7); - dst += 128; - } - } - else { // source unaligned - for (; size >= 128; size -= 128) { - c0 = _mm_loadu_si128(((const __m128i*)src) + 0); - c1 = _mm_loadu_si128(((const __m128i*)src) + 1); - c2 = _mm_loadu_si128(((const __m128i*)src) + 2); - c3 = _mm_loadu_si128(((const __m128i*)src) + 3); - c4 = _mm_loadu_si128(((const __m128i*)src) + 4); - c5 = _mm_loadu_si128(((const __m128i*)src) + 5); - c6 = _mm_loadu_si128(((const __m128i*)src) + 6); - c7 = _mm_loadu_si128(((const __m128i*)src) + 7); - _mm_prefetch((const char*)(src + 256), _MM_HINT_NTA); - src += 128; - _mm_stream_si128((((__m128i*)dst) + 0), c0); - _mm_stream_si128((((__m128i*)dst) + 1), c1); - _mm_stream_si128((((__m128i*)dst) + 2), c2); - _mm_stream_si128((((__m128i*)dst) + 3), c3); - _mm_stream_si128((((__m128i*)dst) + 4), c4); - _mm_stream_si128((((__m128i*)dst) + 5), c5); - _mm_stream_si128((((__m128i*)dst) + 6), c6); - _mm_stream_si128((((__m128i*)dst) + 7), c7); - dst += 128; - } - } - _mm_sfence(); - } - - memcpy_tiny(dst, src, size); - - return destination; -} - - -#endif +//===================================================================== +// +// FastMemcpy.c - skywind3000@163.com, 2015 +// +// feature: +// 50% speed up in avg. vs standard memcpy (tested in vc2012/gcc5.1) +// +//===================================================================== +#pragma once + +#include +#include +#include + + +//--------------------------------------------------------------------- +// force inline for compilers +//--------------------------------------------------------------------- +#ifndef INLINE +#ifdef __GNUC__ +#if (__GNUC__ > 3) || ((__GNUC__ == 3) && (__GNUC_MINOR__ >= 1)) + #define INLINE __inline__ __attribute__((always_inline)) +#else + #define INLINE __inline__ +#endif +#elif defined(_MSC_VER) + #define INLINE __forceinline +#elif (defined(__BORLANDC__) || defined(__WATCOMC__)) + #define INLINE __inline +#else + #define INLINE +#endif +#endif + +typedef __attribute__((__aligned__(1))) uint16_t uint16_unaligned_t; +typedef __attribute__((__aligned__(1))) uint32_t uint32_unaligned_t; +typedef __attribute__((__aligned__(1))) uint64_t uint64_unaligned_t; + +//--------------------------------------------------------------------- +// fast copy for different sizes +//--------------------------------------------------------------------- +static INLINE void memcpy_sse2_16(void * __restrict dst, const void * __restrict src) { + __m128i m0 = _mm_loadu_si128((reinterpret_cast(src)) + 0); + _mm_storeu_si128((reinterpret_cast<__m128i*>(dst)) + 0, m0); +} + +static INLINE void memcpy_sse2_32(void * __restrict dst, const void * __restrict src) { + __m128i m0 = _mm_loadu_si128((reinterpret_cast(src)) + 0); + __m128i m1 = _mm_loadu_si128((reinterpret_cast(src)) + 1); + _mm_storeu_si128((reinterpret_cast<__m128i*>(dst)) + 0, m0); + _mm_storeu_si128((reinterpret_cast<__m128i*>(dst)) + 1, m1); +} + +static INLINE void memcpy_sse2_64(void * __restrict dst, const void * __restrict src) { + __m128i m0 = _mm_loadu_si128((reinterpret_cast(src)) + 0); + __m128i m1 = _mm_loadu_si128((reinterpret_cast(src)) + 1); + __m128i m2 = _mm_loadu_si128((reinterpret_cast(src)) + 2); + __m128i m3 = _mm_loadu_si128((reinterpret_cast(src)) + 3); + _mm_storeu_si128((reinterpret_cast<__m128i*>(dst)) + 0, m0); + _mm_storeu_si128((reinterpret_cast<__m128i*>(dst)) + 1, m1); + _mm_storeu_si128((reinterpret_cast<__m128i*>(dst)) + 2, m2); + _mm_storeu_si128((reinterpret_cast<__m128i*>(dst)) + 3, m3); +} + +static INLINE void memcpy_sse2_128(void * __restrict dst, const void * __restrict src) { + __m128i m0 = _mm_loadu_si128((reinterpret_cast(src)) + 0); + __m128i m1 = _mm_loadu_si128((reinterpret_cast(src)) + 1); + __m128i m2 = _mm_loadu_si128((reinterpret_cast(src)) + 2); + __m128i m3 = _mm_loadu_si128((reinterpret_cast(src)) + 3); + __m128i m4 = _mm_loadu_si128((reinterpret_cast(src)) + 4); + __m128i m5 = _mm_loadu_si128((reinterpret_cast(src)) + 5); + __m128i m6 = _mm_loadu_si128((reinterpret_cast(src)) + 6); + __m128i m7 = _mm_loadu_si128((reinterpret_cast(src)) + 7); + _mm_storeu_si128((reinterpret_cast<__m128i*>(dst)) + 0, m0); + _mm_storeu_si128((reinterpret_cast<__m128i*>(dst)) + 1, m1); + _mm_storeu_si128((reinterpret_cast<__m128i*>(dst)) + 2, m2); + _mm_storeu_si128((reinterpret_cast<__m128i*>(dst)) + 3, m3); + _mm_storeu_si128((reinterpret_cast<__m128i*>(dst)) + 4, m4); + _mm_storeu_si128((reinterpret_cast<__m128i*>(dst)) + 5, m5); + _mm_storeu_si128((reinterpret_cast<__m128i*>(dst)) + 6, m6); + _mm_storeu_si128((reinterpret_cast<__m128i*>(dst)) + 7, m7); +} + + +//--------------------------------------------------------------------- +// tiny memory copy with jump table optimized +//--------------------------------------------------------------------- +/// Attribute is used to avoid an error with undefined behaviour sanitizer +/// ../contrib/FastMemcpy/FastMemcpy.h:91:56: runtime error: applying zero offset to null pointer +/// Found by 01307_orc_output_format.sh, cause - ORCBlockInputFormat and external ORC library. +__attribute__((__no_sanitize__("undefined"))) static INLINE void *memcpy_tiny(void * __restrict dst, const void * __restrict src, size_t size) { + unsigned char *dd = ((unsigned char*)dst) + size; + const unsigned char *ss = ((const unsigned char*)src) + size; + + switch (size) { + case 64: + memcpy_sse2_64(dd - 64, ss - 64); + [[fallthrough]]; + case 0: + break; + + case 65: + memcpy_sse2_64(dd - 65, ss - 65); + [[fallthrough]]; + case 1: + dd[-1] = ss[-1]; + break; + + case 66: + memcpy_sse2_64(dd - 66, ss - 66); + [[fallthrough]]; + case 2: + *((uint16_unaligned_t*)(dd - 2)) = *((const uint16_unaligned_t*)(ss - 2)); + break; + + case 67: + memcpy_sse2_64(dd - 67, ss - 67); + [[fallthrough]]; + case 3: + *((uint16_unaligned_t*)(dd - 3)) = *((const uint16_unaligned_t*)(ss - 3)); + dd[-1] = ss[-1]; + break; + + case 68: + memcpy_sse2_64(dd - 68, ss - 68); + [[fallthrough]]; + case 4: + *((uint32_unaligned_t*)(dd - 4)) = *((const uint32_unaligned_t*)(ss - 4)); + break; + + case 69: + memcpy_sse2_64(dd - 69, ss - 69); + [[fallthrough]]; + case 5: + *((uint32_unaligned_t*)(dd - 5)) = *((const uint32_unaligned_t*)(ss - 5)); + dd[-1] = ss[-1]; + break; + + case 70: + memcpy_sse2_64(dd - 70, ss - 70); + [[fallthrough]]; + case 6: + *((uint32_unaligned_t*)(dd - 6)) = *((const uint32_unaligned_t*)(ss - 6)); + *((uint16_unaligned_t*)(dd - 2)) = *((const uint16_unaligned_t*)(ss - 2)); + break; + + case 71: + memcpy_sse2_64(dd - 71, ss - 71); + [[fallthrough]]; + case 7: + *((uint32_unaligned_t*)(dd - 7)) = *((const uint32_unaligned_t*)(ss - 7)); + *((uint32_unaligned_t*)(dd - 4)) = *((const uint32_unaligned_t*)(ss - 4)); + break; + + case 72: + memcpy_sse2_64(dd - 72, ss - 72); + [[fallthrough]]; + case 8: + *((uint64_unaligned_t*)(dd - 8)) = *((const uint64_unaligned_t*)(ss - 8)); + break; + + case 73: + memcpy_sse2_64(dd - 73, ss - 73); + [[fallthrough]]; + case 9: + *((uint64_unaligned_t*)(dd - 9)) = *((const uint64_unaligned_t*)(ss - 9)); + dd[-1] = ss[-1]; + break; + + case 74: + memcpy_sse2_64(dd - 74, ss - 74); + [[fallthrough]]; + case 10: + *((uint64_unaligned_t*)(dd - 10)) = *((const uint64_unaligned_t*)(ss - 10)); + *((uint16_unaligned_t*)(dd - 2)) = *((const uint16_unaligned_t*)(ss - 2)); + break; + + case 75: + memcpy_sse2_64(dd - 75, ss - 75); + [[fallthrough]]; + case 11: + *((uint64_unaligned_t*)(dd - 11)) = *((const uint64_unaligned_t*)(ss - 11)); + *((uint32_unaligned_t*)(dd - 4)) = *((const uint32_unaligned_t*)(ss - 4)); + break; + + case 76: + memcpy_sse2_64(dd - 76, ss - 76); + [[fallthrough]]; + case 12: + *((uint64_unaligned_t*)(dd - 12)) = *((const uint64_unaligned_t*)(ss - 12)); + *((uint32_unaligned_t*)(dd - 4)) = *((const uint32_unaligned_t*)(ss - 4)); + break; + + case 77: + memcpy_sse2_64(dd - 77, ss - 77); + [[fallthrough]]; + case 13: + *((uint64_unaligned_t*)(dd - 13)) = *((const uint64_unaligned_t*)(ss - 13)); + *((uint32_unaligned_t*)(dd - 5)) = *((const uint32_unaligned_t*)(ss - 5)); + dd[-1] = ss[-1]; + break; + + case 78: + memcpy_sse2_64(dd - 78, ss - 78); + [[fallthrough]]; + case 14: + *((uint64_unaligned_t*)(dd - 14)) = *((const uint64_unaligned_t*)(ss - 14)); + *((uint64_unaligned_t*)(dd - 8)) = *((const uint64_unaligned_t*)(ss - 8)); + break; + + case 79: + memcpy_sse2_64(dd - 79, ss - 79); + [[fallthrough]]; + case 15: + *((uint64_unaligned_t*)(dd - 15)) = *((const uint64_unaligned_t*)(ss - 15)); + *((uint64_unaligned_t*)(dd - 8)) = *((const uint64_unaligned_t*)(ss - 8)); + break; + + case 80: + memcpy_sse2_64(dd - 80, ss - 80); + [[fallthrough]]; + case 16: + memcpy_sse2_16(dd - 16, ss - 16); + break; + + case 81: + memcpy_sse2_64(dd - 81, ss - 81); + [[fallthrough]]; + case 17: + memcpy_sse2_16(dd - 17, ss - 17); + dd[-1] = ss[-1]; + break; + + case 82: + memcpy_sse2_64(dd - 82, ss - 82); + [[fallthrough]]; + case 18: + memcpy_sse2_16(dd - 18, ss - 18); + *((uint16_unaligned_t*)(dd - 2)) = *((const uint16_unaligned_t*)(ss - 2)); + break; + + case 83: + memcpy_sse2_64(dd - 83, ss - 83); + [[fallthrough]]; + case 19: + memcpy_sse2_16(dd - 19, ss - 19); + *((uint16_unaligned_t*)(dd - 3)) = *((const uint16_unaligned_t*)(ss - 3)); + dd[-1] = ss[-1]; + break; + + case 84: + memcpy_sse2_64(dd - 84, ss - 84); + [[fallthrough]]; + case 20: + memcpy_sse2_16(dd - 20, ss - 20); + *((uint32_unaligned_t*)(dd - 4)) = *((const uint32_unaligned_t*)(ss - 4)); + break; + + case 85: + memcpy_sse2_64(dd - 85, ss - 85); + [[fallthrough]]; + case 21: + memcpy_sse2_16(dd - 21, ss - 21); + *((uint32_unaligned_t*)(dd - 5)) = *((const uint32_unaligned_t*)(ss - 5)); + dd[-1] = ss[-1]; + break; + + case 86: + memcpy_sse2_64(dd - 86, ss - 86); + [[fallthrough]]; + case 22: + memcpy_sse2_16(dd - 22, ss - 22); + *((uint32_unaligned_t*)(dd - 6)) = *((const uint32_unaligned_t*)(ss - 6)); + *((uint16_unaligned_t*)(dd - 2)) = *((const uint16_unaligned_t*)(ss - 2)); + break; + + case 87: + memcpy_sse2_64(dd - 87, ss - 87); + [[fallthrough]]; + case 23: + memcpy_sse2_16(dd - 23, ss - 23); + *((uint32_unaligned_t*)(dd - 7)) = *((const uint32_unaligned_t*)(ss - 7)); + *((uint32_unaligned_t*)(dd - 4)) = *((const uint32_unaligned_t*)(ss - 4)); + break; + + case 88: + memcpy_sse2_64(dd - 88, ss - 88); + [[fallthrough]]; + case 24: + memcpy_sse2_16(dd - 24, ss - 24); + memcpy_sse2_16(dd - 16, ss - 16); + break; + + case 89: + memcpy_sse2_64(dd - 89, ss - 89); + [[fallthrough]]; + case 25: + memcpy_sse2_16(dd - 25, ss - 25); + memcpy_sse2_16(dd - 16, ss - 16); + break; + + case 90: + memcpy_sse2_64(dd - 90, ss - 90); + [[fallthrough]]; + case 26: + memcpy_sse2_16(dd - 26, ss - 26); + memcpy_sse2_16(dd - 16, ss - 16); + break; + + case 91: + memcpy_sse2_64(dd - 91, ss - 91); + [[fallthrough]]; + case 27: + memcpy_sse2_16(dd - 27, ss - 27); + memcpy_sse2_16(dd - 16, ss - 16); + break; + + case 92: + memcpy_sse2_64(dd - 92, ss - 92); + [[fallthrough]]; + case 28: + memcpy_sse2_16(dd - 28, ss - 28); + memcpy_sse2_16(dd - 16, ss - 16); + break; + + case 93: + memcpy_sse2_64(dd - 93, ss - 93); + [[fallthrough]]; + case 29: + memcpy_sse2_16(dd - 29, ss - 29); + memcpy_sse2_16(dd - 16, ss - 16); + break; + + case 94: + memcpy_sse2_64(dd - 94, ss - 94); + [[fallthrough]]; + case 30: + memcpy_sse2_16(dd - 30, ss - 30); + memcpy_sse2_16(dd - 16, ss - 16); + break; + + case 95: + memcpy_sse2_64(dd - 95, ss - 95); + [[fallthrough]]; + case 31: + memcpy_sse2_16(dd - 31, ss - 31); + memcpy_sse2_16(dd - 16, ss - 16); + break; + + case 96: + memcpy_sse2_64(dd - 96, ss - 96); + [[fallthrough]]; + case 32: + memcpy_sse2_32(dd - 32, ss - 32); + break; + + case 97: + memcpy_sse2_64(dd - 97, ss - 97); + [[fallthrough]]; + case 33: + memcpy_sse2_32(dd - 33, ss - 33); + dd[-1] = ss[-1]; + break; + + case 98: + memcpy_sse2_64(dd - 98, ss - 98); + [[fallthrough]]; + case 34: + memcpy_sse2_32(dd - 34, ss - 34); + *((uint16_unaligned_t*)(dd - 2)) = *((const uint16_unaligned_t*)(ss - 2)); + break; + + case 99: + memcpy_sse2_64(dd - 99, ss - 99); + [[fallthrough]]; + case 35: + memcpy_sse2_32(dd - 35, ss - 35); + *((uint16_unaligned_t*)(dd - 3)) = *((const uint16_unaligned_t*)(ss - 3)); + dd[-1] = ss[-1]; + break; + + case 100: + memcpy_sse2_64(dd - 100, ss - 100); + [[fallthrough]]; + case 36: + memcpy_sse2_32(dd - 36, ss - 36); + *((uint32_unaligned_t*)(dd - 4)) = *((const uint32_unaligned_t*)(ss - 4)); + break; + + case 101: + memcpy_sse2_64(dd - 101, ss - 101); + [[fallthrough]]; + case 37: + memcpy_sse2_32(dd - 37, ss - 37); + *((uint32_unaligned_t*)(dd - 5)) = *((const uint32_unaligned_t*)(ss - 5)); + dd[-1] = ss[-1]; + break; + + case 102: + memcpy_sse2_64(dd - 102, ss - 102); + [[fallthrough]]; + case 38: + memcpy_sse2_32(dd - 38, ss - 38); + *((uint32_unaligned_t*)(dd - 6)) = *((const uint32_unaligned_t*)(ss - 6)); + *((uint16_unaligned_t*)(dd - 2)) = *((const uint16_unaligned_t*)(ss - 2)); + break; + + case 103: + memcpy_sse2_64(dd - 103, ss - 103); + [[fallthrough]]; + case 39: + memcpy_sse2_32(dd - 39, ss - 39); + *((uint32_unaligned_t*)(dd - 7)) = *((const uint32_unaligned_t*)(ss - 7)); + *((uint32_unaligned_t*)(dd - 4)) = *((const uint32_unaligned_t*)(ss - 4)); + break; + + case 104: + memcpy_sse2_64(dd - 104, ss - 104); + [[fallthrough]]; + case 40: + memcpy_sse2_32(dd - 40, ss - 40); + *((uint64_unaligned_t*)(dd - 8)) = *((const uint64_unaligned_t*)(ss - 8)); + break; + + case 105: + memcpy_sse2_64(dd - 105, ss - 105); + [[fallthrough]]; + case 41: + memcpy_sse2_32(dd - 41, ss - 41); + *((uint64_unaligned_t*)(dd - 9)) = *((const uint64_unaligned_t*)(ss - 9)); + dd[-1] = ss[-1]; + break; + + case 106: + memcpy_sse2_64(dd - 106, ss - 106); + [[fallthrough]]; + case 42: + memcpy_sse2_32(dd - 42, ss - 42); + *((uint64_unaligned_t*)(dd - 10)) = *((const uint64_unaligned_t*)(ss - 10)); + *((uint16_unaligned_t*)(dd - 2)) = *((const uint16_unaligned_t*)(ss - 2)); + break; + + case 107: + memcpy_sse2_64(dd - 107, ss - 107); + [[fallthrough]]; + case 43: + memcpy_sse2_32(dd - 43, ss - 43); + *((uint64_unaligned_t*)(dd - 11)) = *((const uint64_unaligned_t*)(ss - 11)); + *((uint32_unaligned_t*)(dd - 4)) = *((const uint32_unaligned_t*)(ss - 4)); + break; + + case 108: + memcpy_sse2_64(dd - 108, ss - 108); + [[fallthrough]]; + case 44: + memcpy_sse2_32(dd - 44, ss - 44); + *((uint64_unaligned_t*)(dd - 12)) = *((const uint64_unaligned_t*)(ss - 12)); + *((uint32_unaligned_t*)(dd - 4)) = *((const uint32_unaligned_t*)(ss - 4)); + break; + + case 109: + memcpy_sse2_64(dd - 109, ss - 109); + [[fallthrough]]; + case 45: + memcpy_sse2_32(dd - 45, ss - 45); + *((uint64_unaligned_t*)(dd - 13)) = *((const uint64_unaligned_t*)(ss - 13)); + *((uint32_unaligned_t*)(dd - 5)) = *((const uint32_unaligned_t*)(ss - 5)); + dd[-1] = ss[-1]; + break; + + case 110: + memcpy_sse2_64(dd - 110, ss - 110); + [[fallthrough]]; + case 46: + memcpy_sse2_32(dd - 46, ss - 46); + *((uint64_unaligned_t*)(dd - 14)) = *((const uint64_unaligned_t*)(ss - 14)); + *((uint64_unaligned_t*)(dd - 8)) = *((const uint64_unaligned_t*)(ss - 8)); + break; + + case 111: + memcpy_sse2_64(dd - 111, ss - 111); + [[fallthrough]]; + case 47: + memcpy_sse2_32(dd - 47, ss - 47); + *((uint64_unaligned_t*)(dd - 15)) = *((const uint64_unaligned_t*)(ss - 15)); + *((uint64_unaligned_t*)(dd - 8)) = *((const uint64_unaligned_t*)(ss - 8)); + break; + + case 112: + memcpy_sse2_64(dd - 112, ss - 112); + [[fallthrough]]; + case 48: + memcpy_sse2_32(dd - 48, ss - 48); + memcpy_sse2_16(dd - 16, ss - 16); + break; + + case 113: + memcpy_sse2_64(dd - 113, ss - 113); + [[fallthrough]]; + case 49: + memcpy_sse2_32(dd - 49, ss - 49); + memcpy_sse2_16(dd - 17, ss - 17); + dd[-1] = ss[-1]; + break; + + case 114: + memcpy_sse2_64(dd - 114, ss - 114); + [[fallthrough]]; + case 50: + memcpy_sse2_32(dd - 50, ss - 50); + memcpy_sse2_16(dd - 18, ss - 18); + *((uint16_unaligned_t*)(dd - 2)) = *((const uint16_unaligned_t*)(ss - 2)); + break; + + case 115: + memcpy_sse2_64(dd - 115, ss - 115); + [[fallthrough]]; + case 51: + memcpy_sse2_32(dd - 51, ss - 51); + memcpy_sse2_16(dd - 19, ss - 19); + *((uint16_unaligned_t*)(dd - 3)) = *((const uint16_unaligned_t*)(ss - 3)); + dd[-1] = ss[-1]; + break; + + case 116: + memcpy_sse2_64(dd - 116, ss - 116); + [[fallthrough]]; + case 52: + memcpy_sse2_32(dd - 52, ss - 52); + memcpy_sse2_16(dd - 20, ss - 20); + *((uint32_unaligned_t*)(dd - 4)) = *((const uint32_unaligned_t*)(ss - 4)); + break; + + case 117: + memcpy_sse2_64(dd - 117, ss - 117); + [[fallthrough]]; + case 53: + memcpy_sse2_32(dd - 53, ss - 53); + memcpy_sse2_16(dd - 21, ss - 21); + *((uint32_unaligned_t*)(dd - 5)) = *((const uint32_unaligned_t*)(ss - 5)); + dd[-1] = ss[-1]; + break; + + case 118: + memcpy_sse2_64(dd - 118, ss - 118); + [[fallthrough]]; + case 54: + memcpy_sse2_32(dd - 54, ss - 54); + memcpy_sse2_16(dd - 22, ss - 22); + *((uint32_unaligned_t*)(dd - 6)) = *((const uint32_unaligned_t*)(ss - 6)); + *((uint16_unaligned_t*)(dd - 2)) = *((const uint16_unaligned_t*)(ss - 2)); + break; + + case 119: + memcpy_sse2_64(dd - 119, ss - 119); + [[fallthrough]]; + case 55: + memcpy_sse2_32(dd - 55, ss - 55); + memcpy_sse2_16(dd - 23, ss - 23); + *((uint32_unaligned_t*)(dd - 7)) = *((const uint32_unaligned_t*)(ss - 7)); + *((uint32_unaligned_t*)(dd - 4)) = *((const uint32_unaligned_t*)(ss - 4)); + break; + + case 120: + memcpy_sse2_64(dd - 120, ss - 120); + [[fallthrough]]; + case 56: + memcpy_sse2_32(dd - 56, ss - 56); + memcpy_sse2_16(dd - 24, ss - 24); + memcpy_sse2_16(dd - 16, ss - 16); + break; + + case 121: + memcpy_sse2_64(dd - 121, ss - 121); + [[fallthrough]]; + case 57: + memcpy_sse2_32(dd - 57, ss - 57); + memcpy_sse2_16(dd - 25, ss - 25); + memcpy_sse2_16(dd - 16, ss - 16); + break; + + case 122: + memcpy_sse2_64(dd - 122, ss - 122); + [[fallthrough]]; + case 58: + memcpy_sse2_32(dd - 58, ss - 58); + memcpy_sse2_16(dd - 26, ss - 26); + memcpy_sse2_16(dd - 16, ss - 16); + break; + + case 123: + memcpy_sse2_64(dd - 123, ss - 123); + [[fallthrough]]; + case 59: + memcpy_sse2_32(dd - 59, ss - 59); + memcpy_sse2_16(dd - 27, ss - 27); + memcpy_sse2_16(dd - 16, ss - 16); + break; + + case 124: + memcpy_sse2_64(dd - 124, ss - 124); + [[fallthrough]]; + case 60: + memcpy_sse2_32(dd - 60, ss - 60); + memcpy_sse2_16(dd - 28, ss - 28); + memcpy_sse2_16(dd - 16, ss - 16); + break; + + case 125: + memcpy_sse2_64(dd - 125, ss - 125); + [[fallthrough]]; + case 61: + memcpy_sse2_32(dd - 61, ss - 61); + memcpy_sse2_16(dd - 29, ss - 29); + memcpy_sse2_16(dd - 16, ss - 16); + break; + + case 126: + memcpy_sse2_64(dd - 126, ss - 126); + [[fallthrough]]; + case 62: + memcpy_sse2_32(dd - 62, ss - 62); + memcpy_sse2_16(dd - 30, ss - 30); + memcpy_sse2_16(dd - 16, ss - 16); + break; + + case 127: + memcpy_sse2_64(dd - 127, ss - 127); + [[fallthrough]]; + case 63: + memcpy_sse2_32(dd - 63, ss - 63); + memcpy_sse2_16(dd - 31, ss - 31); + memcpy_sse2_16(dd - 16, ss - 16); + break; + + case 128: + memcpy_sse2_128(dd - 128, ss - 128); + break; + } + + return dst; +} + + +//--------------------------------------------------------------------- +// main routine +//--------------------------------------------------------------------- +void* memcpy_fast(void * __restrict destination, const void * __restrict source, size_t size) +{ + unsigned char *dst = (unsigned char*)destination; + const unsigned char *src = (const unsigned char*)source; + static size_t cachesize = 0x200000; // L2-cache size + size_t padding; + + // small memory copy + if (size <= 128) { + return memcpy_tiny(dst, src, size); + } + + // align destination to 16 bytes boundary + padding = (16 - (((size_t)dst) & 15)) & 15; + + if (padding > 0) { + __m128i head = _mm_loadu_si128(reinterpret_cast(src)); + _mm_storeu_si128(reinterpret_cast<__m128i*>(dst), head); + dst += padding; + src += padding; + size -= padding; + } + + // medium size copy + if (size <= cachesize) { + __m128i c0, c1, c2, c3, c4, c5, c6, c7; + + for (; size >= 128; size -= 128) { + c0 = _mm_loadu_si128((reinterpret_cast(src)) + 0); + c1 = _mm_loadu_si128((reinterpret_cast(src)) + 1); + c2 = _mm_loadu_si128((reinterpret_cast(src)) + 2); + c3 = _mm_loadu_si128((reinterpret_cast(src)) + 3); + c4 = _mm_loadu_si128((reinterpret_cast(src)) + 4); + c5 = _mm_loadu_si128((reinterpret_cast(src)) + 5); + c6 = _mm_loadu_si128((reinterpret_cast(src)) + 6); + c7 = _mm_loadu_si128((reinterpret_cast(src)) + 7); + _mm_prefetch((const char*)(src + 256), _MM_HINT_NTA); + src += 128; + _mm_store_si128(((reinterpret_cast<__m128i*>(dst)) + 0), c0); + _mm_store_si128(((reinterpret_cast<__m128i*>(dst)) + 1), c1); + _mm_store_si128(((reinterpret_cast<__m128i*>(dst)) + 2), c2); + _mm_store_si128(((reinterpret_cast<__m128i*>(dst)) + 3), c3); + _mm_store_si128(((reinterpret_cast<__m128i*>(dst)) + 4), c4); + _mm_store_si128(((reinterpret_cast<__m128i*>(dst)) + 5), c5); + _mm_store_si128(((reinterpret_cast<__m128i*>(dst)) + 6), c6); + _mm_store_si128(((reinterpret_cast<__m128i*>(dst)) + 7), c7); + dst += 128; + } + } + else { // big memory copy + __m128i c0, c1, c2, c3, c4, c5, c6, c7; + + _mm_prefetch((const char*)(src), _MM_HINT_NTA); + + if ((((size_t)src) & 15) == 0) { // source aligned + for (; size >= 128; size -= 128) { + c0 = _mm_load_si128((reinterpret_cast(src)) + 0); + c1 = _mm_load_si128((reinterpret_cast(src)) + 1); + c2 = _mm_load_si128((reinterpret_cast(src)) + 2); + c3 = _mm_load_si128((reinterpret_cast(src)) + 3); + c4 = _mm_load_si128((reinterpret_cast(src)) + 4); + c5 = _mm_load_si128((reinterpret_cast(src)) + 5); + c6 = _mm_load_si128((reinterpret_cast(src)) + 6); + c7 = _mm_load_si128((reinterpret_cast(src)) + 7); + _mm_prefetch((const char*)(src + 256), _MM_HINT_NTA); + src += 128; + _mm_stream_si128(((reinterpret_cast<__m128i*>(dst)) + 0), c0); + _mm_stream_si128(((reinterpret_cast<__m128i*>(dst)) + 1), c1); + _mm_stream_si128(((reinterpret_cast<__m128i*>(dst)) + 2), c2); + _mm_stream_si128(((reinterpret_cast<__m128i*>(dst)) + 3), c3); + _mm_stream_si128(((reinterpret_cast<__m128i*>(dst)) + 4), c4); + _mm_stream_si128(((reinterpret_cast<__m128i*>(dst)) + 5), c5); + _mm_stream_si128(((reinterpret_cast<__m128i*>(dst)) + 6), c6); + _mm_stream_si128(((reinterpret_cast<__m128i*>(dst)) + 7), c7); + dst += 128; + } + } + else { // source unaligned + for (; size >= 128; size -= 128) { + c0 = _mm_loadu_si128((reinterpret_cast(src)) + 0); + c1 = _mm_loadu_si128((reinterpret_cast(src)) + 1); + c2 = _mm_loadu_si128((reinterpret_cast(src)) + 2); + c3 = _mm_loadu_si128((reinterpret_cast(src)) + 3); + c4 = _mm_loadu_si128((reinterpret_cast(src)) + 4); + c5 = _mm_loadu_si128((reinterpret_cast(src)) + 5); + c6 = _mm_loadu_si128((reinterpret_cast(src)) + 6); + c7 = _mm_loadu_si128((reinterpret_cast(src)) + 7); + _mm_prefetch((const char*)(src + 256), _MM_HINT_NTA); + src += 128; + _mm_stream_si128(((reinterpret_cast<__m128i*>(dst)) + 0), c0); + _mm_stream_si128(((reinterpret_cast<__m128i*>(dst)) + 1), c1); + _mm_stream_si128(((reinterpret_cast<__m128i*>(dst)) + 2), c2); + _mm_stream_si128(((reinterpret_cast<__m128i*>(dst)) + 3), c3); + _mm_stream_si128(((reinterpret_cast<__m128i*>(dst)) + 4), c4); + _mm_stream_si128(((reinterpret_cast<__m128i*>(dst)) + 5), c5); + _mm_stream_si128(((reinterpret_cast<__m128i*>(dst)) + 6), c6); + _mm_stream_si128(((reinterpret_cast<__m128i*>(dst)) + 7), c7); + dst += 128; + } + } + _mm_sfence(); + } + + memcpy_tiny(dst, src, size); + + return destination; +} diff --git a/contrib/FastMemcpy/FastMemcpy_Avx.h b/contrib/FastMemcpy/FastMemcpy_Avx.h index 8ba064b0350..b209442e7c5 100644 --- a/contrib/FastMemcpy/FastMemcpy_Avx.h +++ b/contrib/FastMemcpy/FastMemcpy_Avx.h @@ -1,492 +1,480 @@ -//===================================================================== -// -// FastMemcpy.c - skywind3000@163.com, 2015 -// -// feature: -// 50% speed up in avg. vs standard memcpy (tested in vc2012/gcc5.1) -// -//===================================================================== -#ifndef __FAST_MEMCPY_H__ -#define __FAST_MEMCPY_H__ - -#include -#include -#include - - -//--------------------------------------------------------------------- -// force inline for compilers -//--------------------------------------------------------------------- -#ifndef INLINE -#ifdef __GNUC__ -#if (__GNUC__ > 3) || ((__GNUC__ == 3) && (__GNUC_MINOR__ >= 1)) - #define INLINE __inline__ __attribute__((always_inline)) -#else - #define INLINE __inline__ -#endif -#elif defined(_MSC_VER) - #define INLINE __forceinline -#elif (defined(__BORLANDC__) || defined(__WATCOMC__)) - #define INLINE __inline -#else - #define INLINE -#endif -#endif - - - -//--------------------------------------------------------------------- -// fast copy for different sizes -//--------------------------------------------------------------------- -static INLINE void memcpy_avx_16(void *dst, const void *src) { -#if 1 - __m128i m0 = _mm_loadu_si128(((const __m128i*)src) + 0); - _mm_storeu_si128(((__m128i*)dst) + 0, m0); -#else - *((uint64_t*)((char*)dst + 0)) = *((uint64_t*)((const char*)src + 0)); - *((uint64_t*)((char*)dst + 8)) = *((uint64_t*)((const char*)src + 8)); -#endif -} - -static INLINE void memcpy_avx_32(void *dst, const void *src) { - __m256i m0 = _mm256_loadu_si256(((const __m256i*)src) + 0); - _mm256_storeu_si256(((__m256i*)dst) + 0, m0); -} - -static INLINE void memcpy_avx_64(void *dst, const void *src) { - __m256i m0 = _mm256_loadu_si256(((const __m256i*)src) + 0); - __m256i m1 = _mm256_loadu_si256(((const __m256i*)src) + 1); - _mm256_storeu_si256(((__m256i*)dst) + 0, m0); - _mm256_storeu_si256(((__m256i*)dst) + 1, m1); -} - -static INLINE void memcpy_avx_128(void *dst, const void *src) { - __m256i m0 = _mm256_loadu_si256(((const __m256i*)src) + 0); - __m256i m1 = _mm256_loadu_si256(((const __m256i*)src) + 1); - __m256i m2 = _mm256_loadu_si256(((const __m256i*)src) + 2); - __m256i m3 = _mm256_loadu_si256(((const __m256i*)src) + 3); - _mm256_storeu_si256(((__m256i*)dst) + 0, m0); - _mm256_storeu_si256(((__m256i*)dst) + 1, m1); - _mm256_storeu_si256(((__m256i*)dst) + 2, m2); - _mm256_storeu_si256(((__m256i*)dst) + 3, m3); -} - -static INLINE void memcpy_avx_256(void *dst, const void *src) { - __m256i m0 = _mm256_loadu_si256(((const __m256i*)src) + 0); - __m256i m1 = _mm256_loadu_si256(((const __m256i*)src) + 1); - __m256i m2 = _mm256_loadu_si256(((const __m256i*)src) + 2); - __m256i m3 = _mm256_loadu_si256(((const __m256i*)src) + 3); - __m256i m4 = _mm256_loadu_si256(((const __m256i*)src) + 4); - __m256i m5 = _mm256_loadu_si256(((const __m256i*)src) + 5); - __m256i m6 = _mm256_loadu_si256(((const __m256i*)src) + 6); - __m256i m7 = _mm256_loadu_si256(((const __m256i*)src) + 7); - _mm256_storeu_si256(((__m256i*)dst) + 0, m0); - _mm256_storeu_si256(((__m256i*)dst) + 1, m1); - _mm256_storeu_si256(((__m256i*)dst) + 2, m2); - _mm256_storeu_si256(((__m256i*)dst) + 3, m3); - _mm256_storeu_si256(((__m256i*)dst) + 4, m4); - _mm256_storeu_si256(((__m256i*)dst) + 5, m5); - _mm256_storeu_si256(((__m256i*)dst) + 6, m6); - _mm256_storeu_si256(((__m256i*)dst) + 7, m7); -} - - -//--------------------------------------------------------------------- -// tiny memory copy with jump table optimized -//--------------------------------------------------------------------- -static INLINE void *memcpy_tiny(void *dst, const void *src, size_t size) { - unsigned char *dd = ((unsigned char*)dst) + size; - const unsigned char *ss = ((const unsigned char*)src) + size; - - switch (size) { - case 128: memcpy_avx_128(dd - 128, ss - 128); - case 0: break; - case 129: memcpy_avx_128(dd - 129, ss - 129); - case 1: dd[-1] = ss[-1]; break; - case 130: memcpy_avx_128(dd - 130, ss - 130); - case 2: *((uint16_t*)(dd - 2)) = *((uint16_t*)(ss - 2)); break; - case 131: memcpy_avx_128(dd - 131, ss - 131); - case 3: *((uint16_t*)(dd - 3)) = *((uint16_t*)(ss - 3)); dd[-1] = ss[-1]; break; - case 132: memcpy_avx_128(dd - 132, ss - 132); - case 4: *((uint32_t*)(dd - 4)) = *((uint32_t*)(ss - 4)); break; - case 133: memcpy_avx_128(dd - 133, ss - 133); - case 5: *((uint32_t*)(dd - 5)) = *((uint32_t*)(ss - 5)); dd[-1] = ss[-1]; break; - case 134: memcpy_avx_128(dd - 134, ss - 134); - case 6: *((uint32_t*)(dd - 6)) = *((uint32_t*)(ss - 6)); *((uint16_t*)(dd - 2)) = *((uint16_t*)(ss - 2)); break; - case 135: memcpy_avx_128(dd - 135, ss - 135); - case 7: *((uint32_t*)(dd - 7)) = *((uint32_t*)(ss - 7)); *((uint32_t*)(dd - 4)) = *((uint32_t*)(ss - 4)); break; - case 136: memcpy_avx_128(dd - 136, ss - 136); - case 8: *((uint64_t*)(dd - 8)) = *((uint64_t*)(ss - 8)); break; - case 137: memcpy_avx_128(dd - 137, ss - 137); - case 9: *((uint64_t*)(dd - 9)) = *((uint64_t*)(ss - 9)); dd[-1] = ss[-1]; break; - case 138: memcpy_avx_128(dd - 138, ss - 138); - case 10: *((uint64_t*)(dd - 10)) = *((uint64_t*)(ss - 10)); *((uint16_t*)(dd - 2)) = *((uint16_t*)(ss - 2)); break; - case 139: memcpy_avx_128(dd - 139, ss - 139); - case 11: *((uint64_t*)(dd - 11)) = *((uint64_t*)(ss - 11)); *((uint32_t*)(dd - 4)) = *((uint32_t*)(ss - 4)); break; - case 140: memcpy_avx_128(dd - 140, ss - 140); - case 12: *((uint64_t*)(dd - 12)) = *((uint64_t*)(ss - 12)); *((uint32_t*)(dd - 4)) = *((uint32_t*)(ss - 4)); break; - case 141: memcpy_avx_128(dd - 141, ss - 141); - case 13: *((uint64_t*)(dd - 13)) = *((uint64_t*)(ss - 13)); *((uint64_t*)(dd - 8)) = *((uint64_t*)(ss - 8)); break; - case 142: memcpy_avx_128(dd - 142, ss - 142); - case 14: *((uint64_t*)(dd - 14)) = *((uint64_t*)(ss - 14)); *((uint64_t*)(dd - 8)) = *((uint64_t*)(ss - 8)); break; - case 143: memcpy_avx_128(dd - 143, ss - 143); - case 15: *((uint64_t*)(dd - 15)) = *((uint64_t*)(ss - 15)); *((uint64_t*)(dd - 8)) = *((uint64_t*)(ss - 8)); break; - case 144: memcpy_avx_128(dd - 144, ss - 144); - case 16: memcpy_avx_16(dd - 16, ss - 16); break; - case 145: memcpy_avx_128(dd - 145, ss - 145); - case 17: memcpy_avx_16(dd - 17, ss - 17); dd[-1] = ss[-1]; break; - case 146: memcpy_avx_128(dd - 146, ss - 146); - case 18: memcpy_avx_16(dd - 18, ss - 18); *((uint16_t*)(dd - 2)) = *((uint16_t*)(ss - 2)); break; - case 147: memcpy_avx_128(dd - 147, ss - 147); - case 19: memcpy_avx_16(dd - 19, ss - 19); *((uint32_t*)(dd - 4)) = *((uint32_t*)(ss - 4)); break; - case 148: memcpy_avx_128(dd - 148, ss - 148); - case 20: memcpy_avx_16(dd - 20, ss - 20); *((uint32_t*)(dd - 4)) = *((uint32_t*)(ss - 4)); break; - case 149: memcpy_avx_128(dd - 149, ss - 149); - case 21: memcpy_avx_16(dd - 21, ss - 21); *((uint64_t*)(dd - 8)) = *((uint64_t*)(ss - 8)); break; - case 150: memcpy_avx_128(dd - 150, ss - 150); - case 22: memcpy_avx_16(dd - 22, ss - 22); *((uint64_t*)(dd - 8)) = *((uint64_t*)(ss - 8)); break; - case 151: memcpy_avx_128(dd - 151, ss - 151); - case 23: memcpy_avx_16(dd - 23, ss - 23); *((uint64_t*)(dd - 8)) = *((uint64_t*)(ss - 8)); break; - case 152: memcpy_avx_128(dd - 152, ss - 152); - case 24: memcpy_avx_16(dd - 24, ss - 24); *((uint64_t*)(dd - 8)) = *((uint64_t*)(ss - 8)); break; - case 153: memcpy_avx_128(dd - 153, ss - 153); - case 25: memcpy_avx_16(dd - 25, ss - 25); memcpy_avx_16(dd - 16, ss - 16); break; - case 154: memcpy_avx_128(dd - 154, ss - 154); - case 26: memcpy_avx_16(dd - 26, ss - 26); memcpy_avx_16(dd - 16, ss - 16); break; - case 155: memcpy_avx_128(dd - 155, ss - 155); - case 27: memcpy_avx_16(dd - 27, ss - 27); memcpy_avx_16(dd - 16, ss - 16); break; - case 156: memcpy_avx_128(dd - 156, ss - 156); - case 28: memcpy_avx_16(dd - 28, ss - 28); memcpy_avx_16(dd - 16, ss - 16); break; - case 157: memcpy_avx_128(dd - 157, ss - 157); - case 29: memcpy_avx_16(dd - 29, ss - 29); memcpy_avx_16(dd - 16, ss - 16); break; - case 158: memcpy_avx_128(dd - 158, ss - 158); - case 30: memcpy_avx_16(dd - 30, ss - 30); memcpy_avx_16(dd - 16, ss - 16); break; - case 159: memcpy_avx_128(dd - 159, ss - 159); - case 31: memcpy_avx_16(dd - 31, ss - 31); memcpy_avx_16(dd - 16, ss - 16); break; - case 160: memcpy_avx_128(dd - 160, ss - 160); - case 32: memcpy_avx_32(dd - 32, ss - 32); break; - case 161: memcpy_avx_128(dd - 161, ss - 161); - case 33: memcpy_avx_32(dd - 33, ss - 33); dd[-1] = ss[-1]; break; - case 162: memcpy_avx_128(dd - 162, ss - 162); - case 34: memcpy_avx_32(dd - 34, ss - 34); *((uint16_t*)(dd - 2)) = *((uint16_t*)(ss - 2)); break; - case 163: memcpy_avx_128(dd - 163, ss - 163); - case 35: memcpy_avx_32(dd - 35, ss - 35); *((uint32_t*)(dd - 4)) = *((uint32_t*)(ss - 4)); break; - case 164: memcpy_avx_128(dd - 164, ss - 164); - case 36: memcpy_avx_32(dd - 36, ss - 36); *((uint32_t*)(dd - 4)) = *((uint32_t*)(ss - 4)); break; - case 165: memcpy_avx_128(dd - 165, ss - 165); - case 37: memcpy_avx_32(dd - 37, ss - 37); *((uint64_t*)(dd - 8)) = *((uint64_t*)(ss - 8)); break; - case 166: memcpy_avx_128(dd - 166, ss - 166); - case 38: memcpy_avx_32(dd - 38, ss - 38); *((uint64_t*)(dd - 8)) = *((uint64_t*)(ss - 8)); break; - case 167: memcpy_avx_128(dd - 167, ss - 167); - case 39: memcpy_avx_32(dd - 39, ss - 39); *((uint64_t*)(dd - 8)) = *((uint64_t*)(ss - 8)); break; - case 168: memcpy_avx_128(dd - 168, ss - 168); - case 40: memcpy_avx_32(dd - 40, ss - 40); *((uint64_t*)(dd - 8)) = *((uint64_t*)(ss - 8)); break; - case 169: memcpy_avx_128(dd - 169, ss - 169); - case 41: memcpy_avx_32(dd - 41, ss - 41); memcpy_avx_16(dd - 16, ss - 16); break; - case 170: memcpy_avx_128(dd - 170, ss - 170); - case 42: memcpy_avx_32(dd - 42, ss - 42); memcpy_avx_16(dd - 16, ss - 16); break; - case 171: memcpy_avx_128(dd - 171, ss - 171); - case 43: memcpy_avx_32(dd - 43, ss - 43); memcpy_avx_16(dd - 16, ss - 16); break; - case 172: memcpy_avx_128(dd - 172, ss - 172); - case 44: memcpy_avx_32(dd - 44, ss - 44); memcpy_avx_16(dd - 16, ss - 16); break; - case 173: memcpy_avx_128(dd - 173, ss - 173); - case 45: memcpy_avx_32(dd - 45, ss - 45); memcpy_avx_16(dd - 16, ss - 16); break; - case 174: memcpy_avx_128(dd - 174, ss - 174); - case 46: memcpy_avx_32(dd - 46, ss - 46); memcpy_avx_16(dd - 16, ss - 16); break; - case 175: memcpy_avx_128(dd - 175, ss - 175); - case 47: memcpy_avx_32(dd - 47, ss - 47); memcpy_avx_16(dd - 16, ss - 16); break; - case 176: memcpy_avx_128(dd - 176, ss - 176); - case 48: memcpy_avx_32(dd - 48, ss - 48); memcpy_avx_16(dd - 16, ss - 16); break; - case 177: memcpy_avx_128(dd - 177, ss - 177); - case 49: memcpy_avx_32(dd - 49, ss - 49); memcpy_avx_32(dd - 32, ss - 32); break; - case 178: memcpy_avx_128(dd - 178, ss - 178); - case 50: memcpy_avx_32(dd - 50, ss - 50); memcpy_avx_32(dd - 32, ss - 32); break; - case 179: memcpy_avx_128(dd - 179, ss - 179); - case 51: memcpy_avx_32(dd - 51, ss - 51); memcpy_avx_32(dd - 32, ss - 32); break; - case 180: memcpy_avx_128(dd - 180, ss - 180); - case 52: memcpy_avx_32(dd - 52, ss - 52); memcpy_avx_32(dd - 32, ss - 32); break; - case 181: memcpy_avx_128(dd - 181, ss - 181); - case 53: memcpy_avx_32(dd - 53, ss - 53); memcpy_avx_32(dd - 32, ss - 32); break; - case 182: memcpy_avx_128(dd - 182, ss - 182); - case 54: memcpy_avx_32(dd - 54, ss - 54); memcpy_avx_32(dd - 32, ss - 32); break; - case 183: memcpy_avx_128(dd - 183, ss - 183); - case 55: memcpy_avx_32(dd - 55, ss - 55); memcpy_avx_32(dd - 32, ss - 32); break; - case 184: memcpy_avx_128(dd - 184, ss - 184); - case 56: memcpy_avx_32(dd - 56, ss - 56); memcpy_avx_32(dd - 32, ss - 32); break; - case 185: memcpy_avx_128(dd - 185, ss - 185); - case 57: memcpy_avx_32(dd - 57, ss - 57); memcpy_avx_32(dd - 32, ss - 32); break; - case 186: memcpy_avx_128(dd - 186, ss - 186); - case 58: memcpy_avx_32(dd - 58, ss - 58); memcpy_avx_32(dd - 32, ss - 32); break; - case 187: memcpy_avx_128(dd - 187, ss - 187); - case 59: memcpy_avx_32(dd - 59, ss - 59); memcpy_avx_32(dd - 32, ss - 32); break; - case 188: memcpy_avx_128(dd - 188, ss - 188); - case 60: memcpy_avx_32(dd - 60, ss - 60); memcpy_avx_32(dd - 32, ss - 32); break; - case 189: memcpy_avx_128(dd - 189, ss - 189); - case 61: memcpy_avx_32(dd - 61, ss - 61); memcpy_avx_32(dd - 32, ss - 32); break; - case 190: memcpy_avx_128(dd - 190, ss - 190); - case 62: memcpy_avx_32(dd - 62, ss - 62); memcpy_avx_32(dd - 32, ss - 32); break; - case 191: memcpy_avx_128(dd - 191, ss - 191); - case 63: memcpy_avx_32(dd - 63, ss - 63); memcpy_avx_32(dd - 32, ss - 32); break; - case 192: memcpy_avx_128(dd - 192, ss - 192); - case 64: memcpy_avx_64(dd - 64, ss - 64); break; - case 193: memcpy_avx_128(dd - 193, ss - 193); - case 65: memcpy_avx_64(dd - 65, ss - 65); dd[-1] = ss[-1]; break; - case 194: memcpy_avx_128(dd - 194, ss - 194); - case 66: memcpy_avx_64(dd - 66, ss - 66); *((uint16_t*)(dd - 2)) = *((uint16_t*)(ss - 2)); break; - case 195: memcpy_avx_128(dd - 195, ss - 195); - case 67: memcpy_avx_64(dd - 67, ss - 67); *((uint32_t*)(dd - 4)) = *((uint32_t*)(ss - 4)); break; - case 196: memcpy_avx_128(dd - 196, ss - 196); - case 68: memcpy_avx_64(dd - 68, ss - 68); *((uint32_t*)(dd - 4)) = *((uint32_t*)(ss - 4)); break; - case 197: memcpy_avx_128(dd - 197, ss - 197); - case 69: memcpy_avx_64(dd - 69, ss - 69); *((uint64_t*)(dd - 8)) = *((uint64_t*)(ss - 8)); break; - case 198: memcpy_avx_128(dd - 198, ss - 198); - case 70: memcpy_avx_64(dd - 70, ss - 70); *((uint64_t*)(dd - 8)) = *((uint64_t*)(ss - 8)); break; - case 199: memcpy_avx_128(dd - 199, ss - 199); - case 71: memcpy_avx_64(dd - 71, ss - 71); *((uint64_t*)(dd - 8)) = *((uint64_t*)(ss - 8)); break; - case 200: memcpy_avx_128(dd - 200, ss - 200); - case 72: memcpy_avx_64(dd - 72, ss - 72); *((uint64_t*)(dd - 8)) = *((uint64_t*)(ss - 8)); break; - case 201: memcpy_avx_128(dd - 201, ss - 201); - case 73: memcpy_avx_64(dd - 73, ss - 73); memcpy_avx_16(dd - 16, ss - 16); break; - case 202: memcpy_avx_128(dd - 202, ss - 202); - case 74: memcpy_avx_64(dd - 74, ss - 74); memcpy_avx_16(dd - 16, ss - 16); break; - case 203: memcpy_avx_128(dd - 203, ss - 203); - case 75: memcpy_avx_64(dd - 75, ss - 75); memcpy_avx_16(dd - 16, ss - 16); break; - case 204: memcpy_avx_128(dd - 204, ss - 204); - case 76: memcpy_avx_64(dd - 76, ss - 76); memcpy_avx_16(dd - 16, ss - 16); break; - case 205: memcpy_avx_128(dd - 205, ss - 205); - case 77: memcpy_avx_64(dd - 77, ss - 77); memcpy_avx_16(dd - 16, ss - 16); break; - case 206: memcpy_avx_128(dd - 206, ss - 206); - case 78: memcpy_avx_64(dd - 78, ss - 78); memcpy_avx_16(dd - 16, ss - 16); break; - case 207: memcpy_avx_128(dd - 207, ss - 207); - case 79: memcpy_avx_64(dd - 79, ss - 79); memcpy_avx_16(dd - 16, ss - 16); break; - case 208: memcpy_avx_128(dd - 208, ss - 208); - case 80: memcpy_avx_64(dd - 80, ss - 80); memcpy_avx_16(dd - 16, ss - 16); break; - case 209: memcpy_avx_128(dd - 209, ss - 209); - case 81: memcpy_avx_64(dd - 81, ss - 81); memcpy_avx_32(dd - 32, ss - 32); break; - case 210: memcpy_avx_128(dd - 210, ss - 210); - case 82: memcpy_avx_64(dd - 82, ss - 82); memcpy_avx_32(dd - 32, ss - 32); break; - case 211: memcpy_avx_128(dd - 211, ss - 211); - case 83: memcpy_avx_64(dd - 83, ss - 83); memcpy_avx_32(dd - 32, ss - 32); break; - case 212: memcpy_avx_128(dd - 212, ss - 212); - case 84: memcpy_avx_64(dd - 84, ss - 84); memcpy_avx_32(dd - 32, ss - 32); break; - case 213: memcpy_avx_128(dd - 213, ss - 213); - case 85: memcpy_avx_64(dd - 85, ss - 85); memcpy_avx_32(dd - 32, ss - 32); break; - case 214: memcpy_avx_128(dd - 214, ss - 214); - case 86: memcpy_avx_64(dd - 86, ss - 86); memcpy_avx_32(dd - 32, ss - 32); break; - case 215: memcpy_avx_128(dd - 215, ss - 215); - case 87: memcpy_avx_64(dd - 87, ss - 87); memcpy_avx_32(dd - 32, ss - 32); break; - case 216: memcpy_avx_128(dd - 216, ss - 216); - case 88: memcpy_avx_64(dd - 88, ss - 88); memcpy_avx_32(dd - 32, ss - 32); break; - case 217: memcpy_avx_128(dd - 217, ss - 217); - case 89: memcpy_avx_64(dd - 89, ss - 89); memcpy_avx_32(dd - 32, ss - 32); break; - case 218: memcpy_avx_128(dd - 218, ss - 218); - case 90: memcpy_avx_64(dd - 90, ss - 90); memcpy_avx_32(dd - 32, ss - 32); break; - case 219: memcpy_avx_128(dd - 219, ss - 219); - case 91: memcpy_avx_64(dd - 91, ss - 91); memcpy_avx_32(dd - 32, ss - 32); break; - case 220: memcpy_avx_128(dd - 220, ss - 220); - case 92: memcpy_avx_64(dd - 92, ss - 92); memcpy_avx_32(dd - 32, ss - 32); break; - case 221: memcpy_avx_128(dd - 221, ss - 221); - case 93: memcpy_avx_64(dd - 93, ss - 93); memcpy_avx_32(dd - 32, ss - 32); break; - case 222: memcpy_avx_128(dd - 222, ss - 222); - case 94: memcpy_avx_64(dd - 94, ss - 94); memcpy_avx_32(dd - 32, ss - 32); break; - case 223: memcpy_avx_128(dd - 223, ss - 223); - case 95: memcpy_avx_64(dd - 95, ss - 95); memcpy_avx_32(dd - 32, ss - 32); break; - case 224: memcpy_avx_128(dd - 224, ss - 224); - case 96: memcpy_avx_64(dd - 96, ss - 96); memcpy_avx_32(dd - 32, ss - 32); break; - case 225: memcpy_avx_128(dd - 225, ss - 225); - case 97: memcpy_avx_64(dd - 97, ss - 97); memcpy_avx_64(dd - 64, ss - 64); break; - case 226: memcpy_avx_128(dd - 226, ss - 226); - case 98: memcpy_avx_64(dd - 98, ss - 98); memcpy_avx_64(dd - 64, ss - 64); break; - case 227: memcpy_avx_128(dd - 227, ss - 227); - case 99: memcpy_avx_64(dd - 99, ss - 99); memcpy_avx_64(dd - 64, ss - 64); break; - case 228: memcpy_avx_128(dd - 228, ss - 228); - case 100: memcpy_avx_64(dd - 100, ss - 100); memcpy_avx_64(dd - 64, ss - 64); break; - case 229: memcpy_avx_128(dd - 229, ss - 229); - case 101: memcpy_avx_64(dd - 101, ss - 101); memcpy_avx_64(dd - 64, ss - 64); break; - case 230: memcpy_avx_128(dd - 230, ss - 230); - case 102: memcpy_avx_64(dd - 102, ss - 102); memcpy_avx_64(dd - 64, ss - 64); break; - case 231: memcpy_avx_128(dd - 231, ss - 231); - case 103: memcpy_avx_64(dd - 103, ss - 103); memcpy_avx_64(dd - 64, ss - 64); break; - case 232: memcpy_avx_128(dd - 232, ss - 232); - case 104: memcpy_avx_64(dd - 104, ss - 104); memcpy_avx_64(dd - 64, ss - 64); break; - case 233: memcpy_avx_128(dd - 233, ss - 233); - case 105: memcpy_avx_64(dd - 105, ss - 105); memcpy_avx_64(dd - 64, ss - 64); break; - case 234: memcpy_avx_128(dd - 234, ss - 234); - case 106: memcpy_avx_64(dd - 106, ss - 106); memcpy_avx_64(dd - 64, ss - 64); break; - case 235: memcpy_avx_128(dd - 235, ss - 235); - case 107: memcpy_avx_64(dd - 107, ss - 107); memcpy_avx_64(dd - 64, ss - 64); break; - case 236: memcpy_avx_128(dd - 236, ss - 236); - case 108: memcpy_avx_64(dd - 108, ss - 108); memcpy_avx_64(dd - 64, ss - 64); break; - case 237: memcpy_avx_128(dd - 237, ss - 237); - case 109: memcpy_avx_64(dd - 109, ss - 109); memcpy_avx_64(dd - 64, ss - 64); break; - case 238: memcpy_avx_128(dd - 238, ss - 238); - case 110: memcpy_avx_64(dd - 110, ss - 110); memcpy_avx_64(dd - 64, ss - 64); break; - case 239: memcpy_avx_128(dd - 239, ss - 239); - case 111: memcpy_avx_64(dd - 111, ss - 111); memcpy_avx_64(dd - 64, ss - 64); break; - case 240: memcpy_avx_128(dd - 240, ss - 240); - case 112: memcpy_avx_64(dd - 112, ss - 112); memcpy_avx_64(dd - 64, ss - 64); break; - case 241: memcpy_avx_128(dd - 241, ss - 241); - case 113: memcpy_avx_64(dd - 113, ss - 113); memcpy_avx_64(dd - 64, ss - 64); break; - case 242: memcpy_avx_128(dd - 242, ss - 242); - case 114: memcpy_avx_64(dd - 114, ss - 114); memcpy_avx_64(dd - 64, ss - 64); break; - case 243: memcpy_avx_128(dd - 243, ss - 243); - case 115: memcpy_avx_64(dd - 115, ss - 115); memcpy_avx_64(dd - 64, ss - 64); break; - case 244: memcpy_avx_128(dd - 244, ss - 244); - case 116: memcpy_avx_64(dd - 116, ss - 116); memcpy_avx_64(dd - 64, ss - 64); break; - case 245: memcpy_avx_128(dd - 245, ss - 245); - case 117: memcpy_avx_64(dd - 117, ss - 117); memcpy_avx_64(dd - 64, ss - 64); break; - case 246: memcpy_avx_128(dd - 246, ss - 246); - case 118: memcpy_avx_64(dd - 118, ss - 118); memcpy_avx_64(dd - 64, ss - 64); break; - case 247: memcpy_avx_128(dd - 247, ss - 247); - case 119: memcpy_avx_64(dd - 119, ss - 119); memcpy_avx_64(dd - 64, ss - 64); break; - case 248: memcpy_avx_128(dd - 248, ss - 248); - case 120: memcpy_avx_64(dd - 120, ss - 120); memcpy_avx_64(dd - 64, ss - 64); break; - case 249: memcpy_avx_128(dd - 249, ss - 249); - case 121: memcpy_avx_64(dd - 121, ss - 121); memcpy_avx_64(dd - 64, ss - 64); break; - case 250: memcpy_avx_128(dd - 250, ss - 250); - case 122: memcpy_avx_64(dd - 122, ss - 122); memcpy_avx_64(dd - 64, ss - 64); break; - case 251: memcpy_avx_128(dd - 251, ss - 251); - case 123: memcpy_avx_64(dd - 123, ss - 123); memcpy_avx_64(dd - 64, ss - 64); break; - case 252: memcpy_avx_128(dd - 252, ss - 252); - case 124: memcpy_avx_64(dd - 124, ss - 124); memcpy_avx_64(dd - 64, ss - 64); break; - case 253: memcpy_avx_128(dd - 253, ss - 253); - case 125: memcpy_avx_64(dd - 125, ss - 125); memcpy_avx_64(dd - 64, ss - 64); break; - case 254: memcpy_avx_128(dd - 254, ss - 254); - case 126: memcpy_avx_64(dd - 126, ss - 126); memcpy_avx_64(dd - 64, ss - 64); break; - case 255: memcpy_avx_128(dd - 255, ss - 255); - case 127: memcpy_avx_64(dd - 127, ss - 127); memcpy_avx_64(dd - 64, ss - 64); break; - case 256: memcpy_avx_256(dd - 256, ss - 256); break; - } - - return dst; -} - - -//--------------------------------------------------------------------- -// main routine -//--------------------------------------------------------------------- -static void* memcpy_fast(void *destination, const void *source, size_t size) -{ - unsigned char *dst = (unsigned char*)destination; - const unsigned char *src = (const unsigned char*)source; - static size_t cachesize = 0x200000; // L3-cache size - size_t padding; - - // small memory copy - if (size <= 256) { - memcpy_tiny(dst, src, size); - _mm256_zeroupper(); - return destination; - } - - // align destination to 16 bytes boundary - padding = (32 - (((size_t)dst) & 31)) & 31; - -#if 0 - if (padding > 0) { - __m256i head = _mm256_loadu_si256((const __m256i*)src); - _mm256_storeu_si256((__m256i*)dst, head); - dst += padding; - src += padding; - size -= padding; - } -#else - __m256i head = _mm256_loadu_si256((const __m256i*)src); - _mm256_storeu_si256((__m256i*)dst, head); - dst += padding; - src += padding; - size -= padding; -#endif - - // medium size copy - if (size <= cachesize) { - __m256i c0, c1, c2, c3, c4, c5, c6, c7; - - for (; size >= 256; size -= 256) { - c0 = _mm256_loadu_si256(((const __m256i*)src) + 0); - c1 = _mm256_loadu_si256(((const __m256i*)src) + 1); - c2 = _mm256_loadu_si256(((const __m256i*)src) + 2); - c3 = _mm256_loadu_si256(((const __m256i*)src) + 3); - c4 = _mm256_loadu_si256(((const __m256i*)src) + 4); - c5 = _mm256_loadu_si256(((const __m256i*)src) + 5); - c6 = _mm256_loadu_si256(((const __m256i*)src) + 6); - c7 = _mm256_loadu_si256(((const __m256i*)src) + 7); - _mm_prefetch((const char*)(src + 512), _MM_HINT_NTA); - src += 256; - _mm256_storeu_si256((((__m256i*)dst) + 0), c0); - _mm256_storeu_si256((((__m256i*)dst) + 1), c1); - _mm256_storeu_si256((((__m256i*)dst) + 2), c2); - _mm256_storeu_si256((((__m256i*)dst) + 3), c3); - _mm256_storeu_si256((((__m256i*)dst) + 4), c4); - _mm256_storeu_si256((((__m256i*)dst) + 5), c5); - _mm256_storeu_si256((((__m256i*)dst) + 6), c6); - _mm256_storeu_si256((((__m256i*)dst) + 7), c7); - dst += 256; - } - } - else { // big memory copy - __m256i c0, c1, c2, c3, c4, c5, c6, c7; - /* __m256i c0, c1, c2, c3, c4, c5, c6, c7; */ - - _mm_prefetch((const char*)(src), _MM_HINT_NTA); - - if ((((size_t)src) & 31) == 0) { // source aligned - for (; size >= 256; size -= 256) { - c0 = _mm256_load_si256(((const __m256i*)src) + 0); - c1 = _mm256_load_si256(((const __m256i*)src) + 1); - c2 = _mm256_load_si256(((const __m256i*)src) + 2); - c3 = _mm256_load_si256(((const __m256i*)src) + 3); - c4 = _mm256_load_si256(((const __m256i*)src) + 4); - c5 = _mm256_load_si256(((const __m256i*)src) + 5); - c6 = _mm256_load_si256(((const __m256i*)src) + 6); - c7 = _mm256_load_si256(((const __m256i*)src) + 7); - _mm_prefetch((const char*)(src + 512), _MM_HINT_NTA); - src += 256; - _mm256_stream_si256((((__m256i*)dst) + 0), c0); - _mm256_stream_si256((((__m256i*)dst) + 1), c1); - _mm256_stream_si256((((__m256i*)dst) + 2), c2); - _mm256_stream_si256((((__m256i*)dst) + 3), c3); - _mm256_stream_si256((((__m256i*)dst) + 4), c4); - _mm256_stream_si256((((__m256i*)dst) + 5), c5); - _mm256_stream_si256((((__m256i*)dst) + 6), c6); - _mm256_stream_si256((((__m256i*)dst) + 7), c7); - dst += 256; - } - } - else { // source unaligned - for (; size >= 256; size -= 256) { - c0 = _mm256_loadu_si256(((const __m256i*)src) + 0); - c1 = _mm256_loadu_si256(((const __m256i*)src) + 1); - c2 = _mm256_loadu_si256(((const __m256i*)src) + 2); - c3 = _mm256_loadu_si256(((const __m256i*)src) + 3); - c4 = _mm256_loadu_si256(((const __m256i*)src) + 4); - c5 = _mm256_loadu_si256(((const __m256i*)src) + 5); - c6 = _mm256_loadu_si256(((const __m256i*)src) + 6); - c7 = _mm256_loadu_si256(((const __m256i*)src) + 7); - _mm_prefetch((const char*)(src + 512), _MM_HINT_NTA); - src += 256; - _mm256_stream_si256((((__m256i*)dst) + 0), c0); - _mm256_stream_si256((((__m256i*)dst) + 1), c1); - _mm256_stream_si256((((__m256i*)dst) + 2), c2); - _mm256_stream_si256((((__m256i*)dst) + 3), c3); - _mm256_stream_si256((((__m256i*)dst) + 4), c4); - _mm256_stream_si256((((__m256i*)dst) + 5), c5); - _mm256_stream_si256((((__m256i*)dst) + 6), c6); - _mm256_stream_si256((((__m256i*)dst) + 7), c7); - dst += 256; - } - } - _mm_sfence(); - } - - memcpy_tiny(dst, src, size); - _mm256_zeroupper(); - - return destination; -} - - -#endif - - - +//===================================================================== +// +// FastMemcpy.c - skywind3000@163.com, 2015 +// +// feature: +// 50% speed up in avg. vs standard memcpy (tested in vc2012/gcc5.1) +// +//===================================================================== +#pragma once + +#include +#include +#include + + +//--------------------------------------------------------------------- +// force inline for compilers +//--------------------------------------------------------------------- +#ifndef INLINE +#ifdef __GNUC__ +#if (__GNUC__ > 3) || ((__GNUC__ == 3) && (__GNUC_MINOR__ >= 1)) + #define INLINE __inline__ __attribute__((always_inline)) +#else + #define INLINE __inline__ +#endif +#elif defined(_MSC_VER) + #define INLINE __forceinline +#elif (defined(__BORLANDC__) || defined(__WATCOMC__)) + #define INLINE __inline +#else + #define INLINE +#endif +#endif + + + +//--------------------------------------------------------------------- +// fast copy for different sizes +//--------------------------------------------------------------------- +static INLINE void memcpy_avx_16(void * __restrict dst, const void * __restrict src) { +#if 1 + __m128i m0 = _mm_loadu_si128(((const __m128i*)src) + 0); + _mm_storeu_si128(((__m128i*)dst) + 0, m0); +#else + *((uint64_t*)((char*)dst + 0)) = *((uint64_t*)((const char*)src + 0)); + *((uint64_t*)((char*)dst + 8)) = *((uint64_t*)((const char*)src + 8)); +#endif +} + +static INLINE void memcpy_avx_32(void *dst, const void *src) { + __m256i m0 = _mm256_loadu_si256((reinterpret_cast(src)) + 0); + _mm256_storeu_si256((reinterpret_cast<__m256i*>(dst)) + 0, m0); +} + +static INLINE void memcpy_avx_64(void *dst, const void *src) { + __m256i m0 = _mm256_loadu_si256((reinterpret_cast(src)) + 0); + __m256i m1 = _mm256_loadu_si256((reinterpret_cast(src)) + 1); + _mm256_storeu_si256((reinterpret_cast<__m256i*>(dst)) + 0, m0); + _mm256_storeu_si256((reinterpret_cast<__m256i*>(dst)) + 1, m1); +} + +static INLINE void memcpy_avx_128(void *dst, const void *src) { + __m256i m0 = _mm256_loadu_si256((reinterpret_cast(src)) + 0); + __m256i m1 = _mm256_loadu_si256((reinterpret_cast(src)) + 1); + __m256i m2 = _mm256_loadu_si256((reinterpret_cast(src)) + 2); + __m256i m3 = _mm256_loadu_si256((reinterpret_cast(src)) + 3); + _mm256_storeu_si256((reinterpret_cast<__m256i*>(dst)) + 0, m0); + _mm256_storeu_si256((reinterpret_cast<__m256i*>(dst)) + 1, m1); + _mm256_storeu_si256((reinterpret_cast<__m256i*>(dst)) + 2, m2); + _mm256_storeu_si256((reinterpret_cast<__m256i*>(dst)) + 3, m3); +} + +static INLINE void memcpy_avx_256(void *dst, const void *src) { + __m256i m0 = _mm256_loadu_si256((reinterpret_cast(src)) + 0); + __m256i m1 = _mm256_loadu_si256((reinterpret_cast(src)) + 1); + __m256i m2 = _mm256_loadu_si256((reinterpret_cast(src)) + 2); + __m256i m3 = _mm256_loadu_si256((reinterpret_cast(src)) + 3); + __m256i m4 = _mm256_loadu_si256((reinterpret_cast(src)) + 4); + __m256i m5 = _mm256_loadu_si256((reinterpret_cast(src)) + 5); + __m256i m6 = _mm256_loadu_si256((reinterpret_cast(src)) + 6); + __m256i m7 = _mm256_loadu_si256((reinterpret_cast(src)) + 7); + _mm256_storeu_si256((reinterpret_cast<__m256i*>(dst)) + 0, m0); + _mm256_storeu_si256((reinterpret_cast<__m256i*>(dst)) + 1, m1); + _mm256_storeu_si256((reinterpret_cast<__m256i*>(dst)) + 2, m2); + _mm256_storeu_si256((reinterpret_cast<__m256i*>(dst)) + 3, m3); + _mm256_storeu_si256((reinterpret_cast<__m256i*>(dst)) + 4, m4); + _mm256_storeu_si256((reinterpret_cast<__m256i*>(dst)) + 5, m5); + _mm256_storeu_si256((reinterpret_cast<__m256i*>(dst)) + 6, m6); + _mm256_storeu_si256((reinterpret_cast<__m256i*>(dst)) + 7, m7); +} + + +//--------------------------------------------------------------------- +// tiny memory copy with jump table optimized +//--------------------------------------------------------------------- +static INLINE void *memcpy_tiny_avx(void * __restrict dst, const void * __restrict src, size_t size) { + unsigned char *dd = reinterpret_cast(dst) + size; + const unsigned char *ss = reinterpret_cast(src) + size; + + switch (size) { + case 128: memcpy_avx_128(dd - 128, ss - 128); [[fallthrough]]; + case 0: break; + case 129: memcpy_avx_128(dd - 129, ss - 129); [[fallthrough]]; + case 1: dd[-1] = ss[-1]; break; + case 130: memcpy_avx_128(dd - 130, ss - 130); [[fallthrough]]; + case 2: *((uint16_t*)(dd - 2)) = *((uint16_t*)(ss - 2)); break; + case 131: memcpy_avx_128(dd - 131, ss - 131); [[fallthrough]]; + case 3: *((uint16_t*)(dd - 3)) = *((uint16_t*)(ss - 3)); dd[-1] = ss[-1]; break; + case 132: memcpy_avx_128(dd - 132, ss - 132); [[fallthrough]]; + case 4: *((uint32_t*)(dd - 4)) = *((uint32_t*)(ss - 4)); break; + case 133: memcpy_avx_128(dd - 133, ss - 133); [[fallthrough]]; + case 5: *((uint32_t*)(dd - 5)) = *((uint32_t*)(ss - 5)); dd[-1] = ss[-1]; break; + case 134: memcpy_avx_128(dd - 134, ss - 134); [[fallthrough]]; + case 6: *((uint32_t*)(dd - 6)) = *((uint32_t*)(ss - 6)); *((uint16_t*)(dd - 2)) = *((uint16_t*)(ss - 2)); break; + case 135: memcpy_avx_128(dd - 135, ss - 135); [[fallthrough]]; + case 7: *((uint32_t*)(dd - 7)) = *((uint32_t*)(ss - 7)); *((uint32_t*)(dd - 4)) = *((uint32_t*)(ss - 4)); break; + case 136: memcpy_avx_128(dd - 136, ss - 136); [[fallthrough]]; + case 8: *((uint64_t*)(dd - 8)) = *((uint64_t*)(ss - 8)); break; + case 137: memcpy_avx_128(dd - 137, ss - 137); [[fallthrough]]; + case 9: *((uint64_t*)(dd - 9)) = *((uint64_t*)(ss - 9)); dd[-1] = ss[-1]; break; + case 138: memcpy_avx_128(dd - 138, ss - 138); [[fallthrough]]; + case 10: *((uint64_t*)(dd - 10)) = *((uint64_t*)(ss - 10)); *((uint16_t*)(dd - 2)) = *((uint16_t*)(ss - 2)); break; + case 139: memcpy_avx_128(dd - 139, ss - 139); [[fallthrough]]; + case 11: *((uint64_t*)(dd - 11)) = *((uint64_t*)(ss - 11)); *((uint32_t*)(dd - 4)) = *((uint32_t*)(ss - 4)); break; + case 140: memcpy_avx_128(dd - 140, ss - 140); [[fallthrough]]; + case 12: *((uint64_t*)(dd - 12)) = *((uint64_t*)(ss - 12)); *((uint32_t*)(dd - 4)) = *((uint32_t*)(ss - 4)); break; + case 141: memcpy_avx_128(dd - 141, ss - 141); [[fallthrough]]; + case 13: *((uint64_t*)(dd - 13)) = *((uint64_t*)(ss - 13)); *((uint64_t*)(dd - 8)) = *((uint64_t*)(ss - 8)); break; + case 142: memcpy_avx_128(dd - 142, ss - 142); [[fallthrough]]; + case 14: *((uint64_t*)(dd - 14)) = *((uint64_t*)(ss - 14)); *((uint64_t*)(dd - 8)) = *((uint64_t*)(ss - 8)); break; + case 143: memcpy_avx_128(dd - 143, ss - 143); [[fallthrough]]; + case 15: *((uint64_t*)(dd - 15)) = *((uint64_t*)(ss - 15)); *((uint64_t*)(dd - 8)) = *((uint64_t*)(ss - 8)); break; + case 144: memcpy_avx_128(dd - 144, ss - 144); [[fallthrough]]; + case 16: memcpy_avx_16(dd - 16, ss - 16); break; + case 145: memcpy_avx_128(dd - 145, ss - 145); [[fallthrough]]; + case 17: memcpy_avx_16(dd - 17, ss - 17); dd[-1] = ss[-1]; break; + case 146: memcpy_avx_128(dd - 146, ss - 146); [[fallthrough]]; + case 18: memcpy_avx_16(dd - 18, ss - 18); *((uint16_t*)(dd - 2)) = *((uint16_t*)(ss - 2)); break; + case 147: memcpy_avx_128(dd - 147, ss - 147); [[fallthrough]]; + case 19: memcpy_avx_16(dd - 19, ss - 19); *((uint32_t*)(dd - 4)) = *((uint32_t*)(ss - 4)); break; + case 148: memcpy_avx_128(dd - 148, ss - 148); [[fallthrough]]; + case 20: memcpy_avx_16(dd - 20, ss - 20); *((uint32_t*)(dd - 4)) = *((uint32_t*)(ss - 4)); break; + case 149: memcpy_avx_128(dd - 149, ss - 149); [[fallthrough]]; + case 21: memcpy_avx_16(dd - 21, ss - 21); *((uint64_t*)(dd - 8)) = *((uint64_t*)(ss - 8)); break; + case 150: memcpy_avx_128(dd - 150, ss - 150); [[fallthrough]]; + case 22: memcpy_avx_16(dd - 22, ss - 22); *((uint64_t*)(dd - 8)) = *((uint64_t*)(ss - 8)); break; + case 151: memcpy_avx_128(dd - 151, ss - 151); [[fallthrough]]; + case 23: memcpy_avx_16(dd - 23, ss - 23); *((uint64_t*)(dd - 8)) = *((uint64_t*)(ss - 8)); break; + case 152: memcpy_avx_128(dd - 152, ss - 152); [[fallthrough]]; + case 24: memcpy_avx_16(dd - 24, ss - 24); *((uint64_t*)(dd - 8)) = *((uint64_t*)(ss - 8)); break; + case 153: memcpy_avx_128(dd - 153, ss - 153); [[fallthrough]]; + case 25: memcpy_avx_16(dd - 25, ss - 25); memcpy_avx_16(dd - 16, ss - 16); break; + case 154: memcpy_avx_128(dd - 154, ss - 154); [[fallthrough]]; + case 26: memcpy_avx_16(dd - 26, ss - 26); memcpy_avx_16(dd - 16, ss - 16); break; + case 155: memcpy_avx_128(dd - 155, ss - 155); [[fallthrough]]; + case 27: memcpy_avx_16(dd - 27, ss - 27); memcpy_avx_16(dd - 16, ss - 16); break; + case 156: memcpy_avx_128(dd - 156, ss - 156); [[fallthrough]]; + case 28: memcpy_avx_16(dd - 28, ss - 28); memcpy_avx_16(dd - 16, ss - 16); break; + case 157: memcpy_avx_128(dd - 157, ss - 157); [[fallthrough]]; + case 29: memcpy_avx_16(dd - 29, ss - 29); memcpy_avx_16(dd - 16, ss - 16); break; + case 158: memcpy_avx_128(dd - 158, ss - 158); [[fallthrough]]; + case 30: memcpy_avx_16(dd - 30, ss - 30); memcpy_avx_16(dd - 16, ss - 16); break; + case 159: memcpy_avx_128(dd - 159, ss - 159); [[fallthrough]]; + case 31: memcpy_avx_16(dd - 31, ss - 31); memcpy_avx_16(dd - 16, ss - 16); break; + case 160: memcpy_avx_128(dd - 160, ss - 160); [[fallthrough]]; + case 32: memcpy_avx_32(dd - 32, ss - 32); break; + case 161: memcpy_avx_128(dd - 161, ss - 161); [[fallthrough]]; + case 33: memcpy_avx_32(dd - 33, ss - 33); dd[-1] = ss[-1]; break; + case 162: memcpy_avx_128(dd - 162, ss - 162); [[fallthrough]]; + case 34: memcpy_avx_32(dd - 34, ss - 34); *((uint16_t*)(dd - 2)) = *((uint16_t*)(ss - 2)); break; + case 163: memcpy_avx_128(dd - 163, ss - 163); [[fallthrough]]; + case 35: memcpy_avx_32(dd - 35, ss - 35); *((uint32_t*)(dd - 4)) = *((uint32_t*)(ss - 4)); break; + case 164: memcpy_avx_128(dd - 164, ss - 164); [[fallthrough]]; + case 36: memcpy_avx_32(dd - 36, ss - 36); *((uint32_t*)(dd - 4)) = *((uint32_t*)(ss - 4)); break; + case 165: memcpy_avx_128(dd - 165, ss - 165); [[fallthrough]]; + case 37: memcpy_avx_32(dd - 37, ss - 37); *((uint64_t*)(dd - 8)) = *((uint64_t*)(ss - 8)); break; + case 166: memcpy_avx_128(dd - 166, ss - 166); [[fallthrough]]; + case 38: memcpy_avx_32(dd - 38, ss - 38); *((uint64_t*)(dd - 8)) = *((uint64_t*)(ss - 8)); break; + case 167: memcpy_avx_128(dd - 167, ss - 167); [[fallthrough]]; + case 39: memcpy_avx_32(dd - 39, ss - 39); *((uint64_t*)(dd - 8)) = *((uint64_t*)(ss - 8)); break; + case 168: memcpy_avx_128(dd - 168, ss - 168); [[fallthrough]]; + case 40: memcpy_avx_32(dd - 40, ss - 40); *((uint64_t*)(dd - 8)) = *((uint64_t*)(ss - 8)); break; + case 169: memcpy_avx_128(dd - 169, ss - 169); [[fallthrough]]; + case 41: memcpy_avx_32(dd - 41, ss - 41); memcpy_avx_16(dd - 16, ss - 16); break; + case 170: memcpy_avx_128(dd - 170, ss - 170); [[fallthrough]]; + case 42: memcpy_avx_32(dd - 42, ss - 42); memcpy_avx_16(dd - 16, ss - 16); break; + case 171: memcpy_avx_128(dd - 171, ss - 171); [[fallthrough]]; + case 43: memcpy_avx_32(dd - 43, ss - 43); memcpy_avx_16(dd - 16, ss - 16); break; + case 172: memcpy_avx_128(dd - 172, ss - 172); [[fallthrough]]; + case 44: memcpy_avx_32(dd - 44, ss - 44); memcpy_avx_16(dd - 16, ss - 16); break; + case 173: memcpy_avx_128(dd - 173, ss - 173); [[fallthrough]]; + case 45: memcpy_avx_32(dd - 45, ss - 45); memcpy_avx_16(dd - 16, ss - 16); break; + case 174: memcpy_avx_128(dd - 174, ss - 174); [[fallthrough]]; + case 46: memcpy_avx_32(dd - 46, ss - 46); memcpy_avx_16(dd - 16, ss - 16); break; + case 175: memcpy_avx_128(dd - 175, ss - 175); [[fallthrough]]; + case 47: memcpy_avx_32(dd - 47, ss - 47); memcpy_avx_16(dd - 16, ss - 16); break; + case 176: memcpy_avx_128(dd - 176, ss - 176); [[fallthrough]]; + case 48: memcpy_avx_32(dd - 48, ss - 48); memcpy_avx_16(dd - 16, ss - 16); break; + case 177: memcpy_avx_128(dd - 177, ss - 177); [[fallthrough]]; + case 49: memcpy_avx_32(dd - 49, ss - 49); memcpy_avx_32(dd - 32, ss - 32); break; + case 178: memcpy_avx_128(dd - 178, ss - 178); [[fallthrough]]; + case 50: memcpy_avx_32(dd - 50, ss - 50); memcpy_avx_32(dd - 32, ss - 32); break; + case 179: memcpy_avx_128(dd - 179, ss - 179); [[fallthrough]]; + case 51: memcpy_avx_32(dd - 51, ss - 51); memcpy_avx_32(dd - 32, ss - 32); break; + case 180: memcpy_avx_128(dd - 180, ss - 180); [[fallthrough]]; + case 52: memcpy_avx_32(dd - 52, ss - 52); memcpy_avx_32(dd - 32, ss - 32); break; + case 181: memcpy_avx_128(dd - 181, ss - 181); [[fallthrough]]; + case 53: memcpy_avx_32(dd - 53, ss - 53); memcpy_avx_32(dd - 32, ss - 32); break; + case 182: memcpy_avx_128(dd - 182, ss - 182); [[fallthrough]]; + case 54: memcpy_avx_32(dd - 54, ss - 54); memcpy_avx_32(dd - 32, ss - 32); break; + case 183: memcpy_avx_128(dd - 183, ss - 183); [[fallthrough]]; + case 55: memcpy_avx_32(dd - 55, ss - 55); memcpy_avx_32(dd - 32, ss - 32); break; + case 184: memcpy_avx_128(dd - 184, ss - 184); [[fallthrough]]; + case 56: memcpy_avx_32(dd - 56, ss - 56); memcpy_avx_32(dd - 32, ss - 32); break; + case 185: memcpy_avx_128(dd - 185, ss - 185); [[fallthrough]]; + case 57: memcpy_avx_32(dd - 57, ss - 57); memcpy_avx_32(dd - 32, ss - 32); break; + case 186: memcpy_avx_128(dd - 186, ss - 186); [[fallthrough]]; + case 58: memcpy_avx_32(dd - 58, ss - 58); memcpy_avx_32(dd - 32, ss - 32); break; + case 187: memcpy_avx_128(dd - 187, ss - 187); [[fallthrough]]; + case 59: memcpy_avx_32(dd - 59, ss - 59); memcpy_avx_32(dd - 32, ss - 32); break; + case 188: memcpy_avx_128(dd - 188, ss - 188); [[fallthrough]]; + case 60: memcpy_avx_32(dd - 60, ss - 60); memcpy_avx_32(dd - 32, ss - 32); break; + case 189: memcpy_avx_128(dd - 189, ss - 189); [[fallthrough]]; + case 61: memcpy_avx_32(dd - 61, ss - 61); memcpy_avx_32(dd - 32, ss - 32); break; + case 190: memcpy_avx_128(dd - 190, ss - 190); [[fallthrough]]; + case 62: memcpy_avx_32(dd - 62, ss - 62); memcpy_avx_32(dd - 32, ss - 32); break; + case 191: memcpy_avx_128(dd - 191, ss - 191); [[fallthrough]]; + case 63: memcpy_avx_32(dd - 63, ss - 63); memcpy_avx_32(dd - 32, ss - 32); break; + case 192: memcpy_avx_128(dd - 192, ss - 192); [[fallthrough]]; + case 64: memcpy_avx_64(dd - 64, ss - 64); break; + case 193: memcpy_avx_128(dd - 193, ss - 193); [[fallthrough]]; + case 65: memcpy_avx_64(dd - 65, ss - 65); dd[-1] = ss[-1]; break; + case 194: memcpy_avx_128(dd - 194, ss - 194); [[fallthrough]]; + case 66: memcpy_avx_64(dd - 66, ss - 66); *((uint16_t*)(dd - 2)) = *((uint16_t*)(ss - 2)); break; + case 195: memcpy_avx_128(dd - 195, ss - 195); [[fallthrough]]; + case 67: memcpy_avx_64(dd - 67, ss - 67); *((uint32_t*)(dd - 4)) = *((uint32_t*)(ss - 4)); break; + case 196: memcpy_avx_128(dd - 196, ss - 196); [[fallthrough]]; + case 68: memcpy_avx_64(dd - 68, ss - 68); *((uint32_t*)(dd - 4)) = *((uint32_t*)(ss - 4)); break; + case 197: memcpy_avx_128(dd - 197, ss - 197); [[fallthrough]]; + case 69: memcpy_avx_64(dd - 69, ss - 69); *((uint64_t*)(dd - 8)) = *((uint64_t*)(ss - 8)); break; + case 198: memcpy_avx_128(dd - 198, ss - 198); [[fallthrough]]; + case 70: memcpy_avx_64(dd - 70, ss - 70); *((uint64_t*)(dd - 8)) = *((uint64_t*)(ss - 8)); break; + case 199: memcpy_avx_128(dd - 199, ss - 199); [[fallthrough]]; + case 71: memcpy_avx_64(dd - 71, ss - 71); *((uint64_t*)(dd - 8)) = *((uint64_t*)(ss - 8)); break; + case 200: memcpy_avx_128(dd - 200, ss - 200); [[fallthrough]]; + case 72: memcpy_avx_64(dd - 72, ss - 72); *((uint64_t*)(dd - 8)) = *((uint64_t*)(ss - 8)); break; + case 201: memcpy_avx_128(dd - 201, ss - 201); [[fallthrough]]; + case 73: memcpy_avx_64(dd - 73, ss - 73); memcpy_avx_16(dd - 16, ss - 16); break; + case 202: memcpy_avx_128(dd - 202, ss - 202); [[fallthrough]]; + case 74: memcpy_avx_64(dd - 74, ss - 74); memcpy_avx_16(dd - 16, ss - 16); break; + case 203: memcpy_avx_128(dd - 203, ss - 203); [[fallthrough]]; + case 75: memcpy_avx_64(dd - 75, ss - 75); memcpy_avx_16(dd - 16, ss - 16); break; + case 204: memcpy_avx_128(dd - 204, ss - 204); [[fallthrough]]; + case 76: memcpy_avx_64(dd - 76, ss - 76); memcpy_avx_16(dd - 16, ss - 16); break; + case 205: memcpy_avx_128(dd - 205, ss - 205); [[fallthrough]]; + case 77: memcpy_avx_64(dd - 77, ss - 77); memcpy_avx_16(dd - 16, ss - 16); break; + case 206: memcpy_avx_128(dd - 206, ss - 206); [[fallthrough]]; + case 78: memcpy_avx_64(dd - 78, ss - 78); memcpy_avx_16(dd - 16, ss - 16); break; + case 207: memcpy_avx_128(dd - 207, ss - 207); [[fallthrough]]; + case 79: memcpy_avx_64(dd - 79, ss - 79); memcpy_avx_16(dd - 16, ss - 16); break; + case 208: memcpy_avx_128(dd - 208, ss - 208); [[fallthrough]]; + case 80: memcpy_avx_64(dd - 80, ss - 80); memcpy_avx_16(dd - 16, ss - 16); break; + case 209: memcpy_avx_128(dd - 209, ss - 209); [[fallthrough]]; + case 81: memcpy_avx_64(dd - 81, ss - 81); memcpy_avx_32(dd - 32, ss - 32); break; + case 210: memcpy_avx_128(dd - 210, ss - 210); [[fallthrough]]; + case 82: memcpy_avx_64(dd - 82, ss - 82); memcpy_avx_32(dd - 32, ss - 32); break; + case 211: memcpy_avx_128(dd - 211, ss - 211); [[fallthrough]]; + case 83: memcpy_avx_64(dd - 83, ss - 83); memcpy_avx_32(dd - 32, ss - 32); break; + case 212: memcpy_avx_128(dd - 212, ss - 212); [[fallthrough]]; + case 84: memcpy_avx_64(dd - 84, ss - 84); memcpy_avx_32(dd - 32, ss - 32); break; + case 213: memcpy_avx_128(dd - 213, ss - 213); [[fallthrough]]; + case 85: memcpy_avx_64(dd - 85, ss - 85); memcpy_avx_32(dd - 32, ss - 32); break; + case 214: memcpy_avx_128(dd - 214, ss - 214); [[fallthrough]]; + case 86: memcpy_avx_64(dd - 86, ss - 86); memcpy_avx_32(dd - 32, ss - 32); break; + case 215: memcpy_avx_128(dd - 215, ss - 215); [[fallthrough]]; + case 87: memcpy_avx_64(dd - 87, ss - 87); memcpy_avx_32(dd - 32, ss - 32); break; + case 216: memcpy_avx_128(dd - 216, ss - 216); [[fallthrough]]; + case 88: memcpy_avx_64(dd - 88, ss - 88); memcpy_avx_32(dd - 32, ss - 32); break; + case 217: memcpy_avx_128(dd - 217, ss - 217); [[fallthrough]]; + case 89: memcpy_avx_64(dd - 89, ss - 89); memcpy_avx_32(dd - 32, ss - 32); break; + case 218: memcpy_avx_128(dd - 218, ss - 218); [[fallthrough]]; + case 90: memcpy_avx_64(dd - 90, ss - 90); memcpy_avx_32(dd - 32, ss - 32); break; + case 219: memcpy_avx_128(dd - 219, ss - 219); [[fallthrough]]; + case 91: memcpy_avx_64(dd - 91, ss - 91); memcpy_avx_32(dd - 32, ss - 32); break; + case 220: memcpy_avx_128(dd - 220, ss - 220); [[fallthrough]]; + case 92: memcpy_avx_64(dd - 92, ss - 92); memcpy_avx_32(dd - 32, ss - 32); break; + case 221: memcpy_avx_128(dd - 221, ss - 221); [[fallthrough]]; + case 93: memcpy_avx_64(dd - 93, ss - 93); memcpy_avx_32(dd - 32, ss - 32); break; + case 222: memcpy_avx_128(dd - 222, ss - 222); [[fallthrough]]; + case 94: memcpy_avx_64(dd - 94, ss - 94); memcpy_avx_32(dd - 32, ss - 32); break; + case 223: memcpy_avx_128(dd - 223, ss - 223); [[fallthrough]]; + case 95: memcpy_avx_64(dd - 95, ss - 95); memcpy_avx_32(dd - 32, ss - 32); break; + case 224: memcpy_avx_128(dd - 224, ss - 224); [[fallthrough]]; + case 96: memcpy_avx_64(dd - 96, ss - 96); memcpy_avx_32(dd - 32, ss - 32); break; + case 225: memcpy_avx_128(dd - 225, ss - 225); [[fallthrough]]; + case 97: memcpy_avx_64(dd - 97, ss - 97); memcpy_avx_64(dd - 64, ss - 64); break; + case 226: memcpy_avx_128(dd - 226, ss - 226); [[fallthrough]]; + case 98: memcpy_avx_64(dd - 98, ss - 98); memcpy_avx_64(dd - 64, ss - 64); break; + case 227: memcpy_avx_128(dd - 227, ss - 227); [[fallthrough]]; + case 99: memcpy_avx_64(dd - 99, ss - 99); memcpy_avx_64(dd - 64, ss - 64); break; + case 228: memcpy_avx_128(dd - 228, ss - 228); [[fallthrough]]; + case 100: memcpy_avx_64(dd - 100, ss - 100); memcpy_avx_64(dd - 64, ss - 64); break; + case 229: memcpy_avx_128(dd - 229, ss - 229); [[fallthrough]]; + case 101: memcpy_avx_64(dd - 101, ss - 101); memcpy_avx_64(dd - 64, ss - 64); break; + case 230: memcpy_avx_128(dd - 230, ss - 230); [[fallthrough]]; + case 102: memcpy_avx_64(dd - 102, ss - 102); memcpy_avx_64(dd - 64, ss - 64); break; + case 231: memcpy_avx_128(dd - 231, ss - 231); [[fallthrough]]; + case 103: memcpy_avx_64(dd - 103, ss - 103); memcpy_avx_64(dd - 64, ss - 64); break; + case 232: memcpy_avx_128(dd - 232, ss - 232); [[fallthrough]]; + case 104: memcpy_avx_64(dd - 104, ss - 104); memcpy_avx_64(dd - 64, ss - 64); break; + case 233: memcpy_avx_128(dd - 233, ss - 233); [[fallthrough]]; + case 105: memcpy_avx_64(dd - 105, ss - 105); memcpy_avx_64(dd - 64, ss - 64); break; + case 234: memcpy_avx_128(dd - 234, ss - 234); [[fallthrough]]; + case 106: memcpy_avx_64(dd - 106, ss - 106); memcpy_avx_64(dd - 64, ss - 64); break; + case 235: memcpy_avx_128(dd - 235, ss - 235); [[fallthrough]]; + case 107: memcpy_avx_64(dd - 107, ss - 107); memcpy_avx_64(dd - 64, ss - 64); break; + case 236: memcpy_avx_128(dd - 236, ss - 236); [[fallthrough]]; + case 108: memcpy_avx_64(dd - 108, ss - 108); memcpy_avx_64(dd - 64, ss - 64); break; + case 237: memcpy_avx_128(dd - 237, ss - 237); [[fallthrough]]; + case 109: memcpy_avx_64(dd - 109, ss - 109); memcpy_avx_64(dd - 64, ss - 64); break; + case 238: memcpy_avx_128(dd - 238, ss - 238); [[fallthrough]]; + case 110: memcpy_avx_64(dd - 110, ss - 110); memcpy_avx_64(dd - 64, ss - 64); break; + case 239: memcpy_avx_128(dd - 239, ss - 239); [[fallthrough]]; + case 111: memcpy_avx_64(dd - 111, ss - 111); memcpy_avx_64(dd - 64, ss - 64); break; + case 240: memcpy_avx_128(dd - 240, ss - 240); [[fallthrough]]; + case 112: memcpy_avx_64(dd - 112, ss - 112); memcpy_avx_64(dd - 64, ss - 64); break; + case 241: memcpy_avx_128(dd - 241, ss - 241); [[fallthrough]]; + case 113: memcpy_avx_64(dd - 113, ss - 113); memcpy_avx_64(dd - 64, ss - 64); break; + case 242: memcpy_avx_128(dd - 242, ss - 242); [[fallthrough]]; + case 114: memcpy_avx_64(dd - 114, ss - 114); memcpy_avx_64(dd - 64, ss - 64); break; + case 243: memcpy_avx_128(dd - 243, ss - 243); [[fallthrough]]; + case 115: memcpy_avx_64(dd - 115, ss - 115); memcpy_avx_64(dd - 64, ss - 64); break; + case 244: memcpy_avx_128(dd - 244, ss - 244); [[fallthrough]]; + case 116: memcpy_avx_64(dd - 116, ss - 116); memcpy_avx_64(dd - 64, ss - 64); break; + case 245: memcpy_avx_128(dd - 245, ss - 245); [[fallthrough]]; + case 117: memcpy_avx_64(dd - 117, ss - 117); memcpy_avx_64(dd - 64, ss - 64); break; + case 246: memcpy_avx_128(dd - 246, ss - 246); [[fallthrough]]; + case 118: memcpy_avx_64(dd - 118, ss - 118); memcpy_avx_64(dd - 64, ss - 64); break; + case 247: memcpy_avx_128(dd - 247, ss - 247); [[fallthrough]]; + case 119: memcpy_avx_64(dd - 119, ss - 119); memcpy_avx_64(dd - 64, ss - 64); break; + case 248: memcpy_avx_128(dd - 248, ss - 248); [[fallthrough]]; + case 120: memcpy_avx_64(dd - 120, ss - 120); memcpy_avx_64(dd - 64, ss - 64); break; + case 249: memcpy_avx_128(dd - 249, ss - 249); [[fallthrough]]; + case 121: memcpy_avx_64(dd - 121, ss - 121); memcpy_avx_64(dd - 64, ss - 64); break; + case 250: memcpy_avx_128(dd - 250, ss - 250); [[fallthrough]]; + case 122: memcpy_avx_64(dd - 122, ss - 122); memcpy_avx_64(dd - 64, ss - 64); break; + case 251: memcpy_avx_128(dd - 251, ss - 251); [[fallthrough]]; + case 123: memcpy_avx_64(dd - 123, ss - 123); memcpy_avx_64(dd - 64, ss - 64); break; + case 252: memcpy_avx_128(dd - 252, ss - 252); [[fallthrough]]; + case 124: memcpy_avx_64(dd - 124, ss - 124); memcpy_avx_64(dd - 64, ss - 64); break; + case 253: memcpy_avx_128(dd - 253, ss - 253); [[fallthrough]]; + case 125: memcpy_avx_64(dd - 125, ss - 125); memcpy_avx_64(dd - 64, ss - 64); break; + case 254: memcpy_avx_128(dd - 254, ss - 254); [[fallthrough]]; + case 126: memcpy_avx_64(dd - 126, ss - 126); memcpy_avx_64(dd - 64, ss - 64); break; + case 255: memcpy_avx_128(dd - 255, ss - 255); [[fallthrough]]; + case 127: memcpy_avx_64(dd - 127, ss - 127); memcpy_avx_64(dd - 64, ss - 64); break; + case 256: memcpy_avx_256(dd - 256, ss - 256); break; + } + + return dst; +} + + +//--------------------------------------------------------------------- +// main routine +//--------------------------------------------------------------------- +void* memcpy_fast_avx(void * __restrict destination, const void * __restrict source, size_t size) +{ + unsigned char *dst = reinterpret_cast(destination); + const unsigned char *src = reinterpret_cast(source); + static size_t cachesize = 0x200000; // L3-cache size + size_t padding; + + // small memory copy + if (size <= 256) { + memcpy_tiny_avx(dst, src, size); + _mm256_zeroupper(); + return destination; + } + + // align destination to 16 bytes boundary + padding = (32 - (((size_t)dst) & 31)) & 31; + +#if 0 + if (padding > 0) { + __m256i head = _mm256_loadu_si256(reinterpret_cast(src)); + _mm256_storeu_si256((__m256i*)dst, head); + dst += padding; + src += padding; + size -= padding; + } +#else + __m256i head = _mm256_loadu_si256(reinterpret_cast(src)); + _mm256_storeu_si256((__m256i*)dst, head); + dst += padding; + src += padding; + size -= padding; +#endif + + // medium size copy + if (size <= cachesize) { + __m256i c0, c1, c2, c3, c4, c5, c6, c7; + + for (; size >= 256; size -= 256) { + c0 = _mm256_loadu_si256((reinterpret_cast(src)) + 0); + c1 = _mm256_loadu_si256((reinterpret_cast(src)) + 1); + c2 = _mm256_loadu_si256((reinterpret_cast(src)) + 2); + c3 = _mm256_loadu_si256((reinterpret_cast(src)) + 3); + c4 = _mm256_loadu_si256((reinterpret_cast(src)) + 4); + c5 = _mm256_loadu_si256((reinterpret_cast(src)) + 5); + c6 = _mm256_loadu_si256((reinterpret_cast(src)) + 6); + c7 = _mm256_loadu_si256((reinterpret_cast(src)) + 7); + src += 256; + _mm256_storeu_si256(((reinterpret_cast<__m256i*>(dst)) + 0), c0); + _mm256_storeu_si256(((reinterpret_cast<__m256i*>(dst)) + 1), c1); + _mm256_storeu_si256(((reinterpret_cast<__m256i*>(dst)) + 2), c2); + _mm256_storeu_si256(((reinterpret_cast<__m256i*>(dst)) + 3), c3); + _mm256_storeu_si256(((reinterpret_cast<__m256i*>(dst)) + 4), c4); + _mm256_storeu_si256(((reinterpret_cast<__m256i*>(dst)) + 5), c5); + _mm256_storeu_si256(((reinterpret_cast<__m256i*>(dst)) + 6), c6); + _mm256_storeu_si256(((reinterpret_cast<__m256i*>(dst)) + 7), c7); + dst += 256; + } + } + else { // big memory copy + __m256i c0, c1, c2, c3, c4, c5, c6, c7; + /* __m256i c0, c1, c2, c3, c4, c5, c6, c7; */ + + if ((((size_t)src) & 31) == 0) { // source aligned + for (; size >= 256; size -= 256) { + c0 = _mm256_load_si256((reinterpret_cast(src)) + 0); + c1 = _mm256_load_si256((reinterpret_cast(src)) + 1); + c2 = _mm256_load_si256((reinterpret_cast(src)) + 2); + c3 = _mm256_load_si256((reinterpret_cast(src)) + 3); + c4 = _mm256_load_si256((reinterpret_cast(src)) + 4); + c5 = _mm256_load_si256((reinterpret_cast(src)) + 5); + c6 = _mm256_load_si256((reinterpret_cast(src)) + 6); + c7 = _mm256_load_si256((reinterpret_cast(src)) + 7); + src += 256; + _mm256_stream_si256(((reinterpret_cast<__m256i*>(dst)) + 0), c0); + _mm256_stream_si256(((reinterpret_cast<__m256i*>(dst)) + 1), c1); + _mm256_stream_si256(((reinterpret_cast<__m256i*>(dst)) + 2), c2); + _mm256_stream_si256(((reinterpret_cast<__m256i*>(dst)) + 3), c3); + _mm256_stream_si256(((reinterpret_cast<__m256i*>(dst)) + 4), c4); + _mm256_stream_si256(((reinterpret_cast<__m256i*>(dst)) + 5), c5); + _mm256_stream_si256(((reinterpret_cast<__m256i*>(dst)) + 6), c6); + _mm256_stream_si256(((reinterpret_cast<__m256i*>(dst)) + 7), c7); + dst += 256; + } + } + else { // source unaligned + for (; size >= 256; size -= 256) { + c0 = _mm256_loadu_si256((reinterpret_cast(src)) + 0); + c1 = _mm256_loadu_si256((reinterpret_cast(src)) + 1); + c2 = _mm256_loadu_si256((reinterpret_cast(src)) + 2); + c3 = _mm256_loadu_si256((reinterpret_cast(src)) + 3); + c4 = _mm256_loadu_si256((reinterpret_cast(src)) + 4); + c5 = _mm256_loadu_si256((reinterpret_cast(src)) + 5); + c6 = _mm256_loadu_si256((reinterpret_cast(src)) + 6); + c7 = _mm256_loadu_si256((reinterpret_cast(src)) + 7); + src += 256; + _mm256_stream_si256(((reinterpret_cast<__m256i*>(dst)) + 0), c0); + _mm256_stream_si256(((reinterpret_cast<__m256i*>(dst)) + 1), c1); + _mm256_stream_si256(((reinterpret_cast<__m256i*>(dst)) + 2), c2); + _mm256_stream_si256(((reinterpret_cast<__m256i*>(dst)) + 3), c3); + _mm256_stream_si256(((reinterpret_cast<__m256i*>(dst)) + 4), c4); + _mm256_stream_si256(((reinterpret_cast<__m256i*>(dst)) + 5), c5); + _mm256_stream_si256(((reinterpret_cast<__m256i*>(dst)) + 6), c6); + _mm256_stream_si256(((reinterpret_cast<__m256i*>(dst)) + 7), c7); + dst += 256; + } + } + _mm_sfence(); + } + + memcpy_tiny_avx(dst, src, size); + _mm256_zeroupper(); + + return destination; +} From 2c86bc4ea3adc699960272d5b3b4335823fad8d4 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Mon, 8 Mar 2021 10:00:54 +0300 Subject: [PATCH 161/544] Addition to prev. revision --- base/glibc-compatibility/CMakeLists.txt | 2 +- contrib/FastMemcpy/CMakeLists.txt | 4 +- contrib/FastMemcpy/FastMemcpy.c | 220 ------- contrib/FastMemcpy/memcpy_wrapper.c | 6 - contrib/FastMemcpy/memcpy_wrapper.cpp | 6 + utils/memcpy-bench/FastMemcpy.h | 754 ++++++++++++++++++++++++ 6 files changed, 762 insertions(+), 230 deletions(-) delete mode 100644 contrib/FastMemcpy/FastMemcpy.c delete mode 100644 contrib/FastMemcpy/memcpy_wrapper.c create mode 100644 contrib/FastMemcpy/memcpy_wrapper.cpp create mode 100644 utils/memcpy-bench/FastMemcpy.h diff --git a/base/glibc-compatibility/CMakeLists.txt b/base/glibc-compatibility/CMakeLists.txt index 684c6162941..3f1cb4742b2 100644 --- a/base/glibc-compatibility/CMakeLists.txt +++ b/base/glibc-compatibility/CMakeLists.txt @@ -30,7 +30,7 @@ if (GLIBC_COMPATIBILITY) if (NOT ARCH_ARM) # clickhouse_memcpy don't support ARCH_ARM, see https://github.com/ClickHouse/ClickHouse/issues/18951 add_library (clickhouse_memcpy OBJECT - ${ClickHouse_SOURCE_DIR}/contrib/FastMemcpy/memcpy_wrapper.c + ${ClickHouse_SOURCE_DIR}/contrib/FastMemcpy/memcpy_wrapper.cpp ) endif() diff --git a/contrib/FastMemcpy/CMakeLists.txt b/contrib/FastMemcpy/CMakeLists.txt index 8efe6d45dff..7de638282b9 100644 --- a/contrib/FastMemcpy/CMakeLists.txt +++ b/contrib/FastMemcpy/CMakeLists.txt @@ -8,9 +8,7 @@ if (ENABLE_FASTMEMCPY) set (LIBRARY_DIR ${ClickHouse_SOURCE_DIR}/contrib/FastMemcpy) set (SRCS - ${LIBRARY_DIR}/FastMemcpy.c - - memcpy_wrapper.c + memcpy_wrapper.cpp ) add_library (FastMemcpy ${SRCS}) diff --git a/contrib/FastMemcpy/FastMemcpy.c b/contrib/FastMemcpy/FastMemcpy.c deleted file mode 100644 index 5021bcc7d16..00000000000 --- a/contrib/FastMemcpy/FastMemcpy.c +++ /dev/null @@ -1,220 +0,0 @@ -//===================================================================== -// -// FastMemcpy.c - skywind3000@163.com, 2015 -// -// feature: -// 50% speed up in avg. vs standard memcpy (tested in vc2012/gcc4.9) -// -//===================================================================== -#include -#include -#include -#include - -#if (defined(_WIN32) || defined(WIN32)) -#include -#include -#ifdef _MSC_VER -#pragma comment(lib, "winmm.lib") -#endif -#elif defined(__unix) -#include -#include -#else -#error it can only be compiled under windows or unix -#endif - -#include "FastMemcpy.h" - -unsigned int gettime() -{ - #if (defined(_WIN32) || defined(WIN32)) - return timeGetTime(); - #else - static struct timezone tz={ 0,0 }; - struct timeval time; - gettimeofday(&time,&tz); - return (time.tv_sec * 1000 + time.tv_usec / 1000); - #endif -} - -void sleepms(unsigned int millisec) -{ -#if defined(_WIN32) || defined(WIN32) - Sleep(millisec); -#else - usleep(millisec * 1000); -#endif -} - - -void benchmark(int dstalign, int srcalign, size_t size, int times) -{ - char *DATA1 = (char*)malloc(size + 64); - char *DATA2 = (char*)malloc(size + 64); - size_t LINEAR1 = ((size_t)DATA1); - size_t LINEAR2 = ((size_t)DATA2); - char *ALIGN1 = (char*)(((64 - (LINEAR1 & 63)) & 63) + LINEAR1); - char *ALIGN2 = (char*)(((64 - (LINEAR2 & 63)) & 63) + LINEAR2); - char *dst = (dstalign)? ALIGN1 : (ALIGN1 + 1); - char *src = (srcalign)? ALIGN2 : (ALIGN2 + 3); - unsigned int t1, t2; - int k; - - sleepms(100); - t1 = gettime(); - for (k = times; k > 0; k--) { - memcpy(dst, src, size); - } - t1 = gettime() - t1; - sleepms(100); - t2 = gettime(); - for (k = times; k > 0; k--) { - memcpy_fast(dst, src, size); - } - t2 = gettime() - t2; - - free(DATA1); - free(DATA2); - - printf("result(dst %s, src %s): memcpy_fast=%dms memcpy=%d ms\n", - dstalign? "aligned" : "unalign", - srcalign? "aligned" : "unalign", (int)t2, (int)t1); -} - - -void bench(int copysize, int times) -{ - printf("benchmark(size=%d bytes, times=%d):\n", copysize, times); - benchmark(1, 1, copysize, times); - benchmark(1, 0, copysize, times); - benchmark(0, 1, copysize, times); - benchmark(0, 0, copysize, times); - printf("\n"); -} - - -void random_bench(int maxsize, int times) -{ - static char A[11 * 1024 * 1024 + 2]; - static char B[11 * 1024 * 1024 + 2]; - static int random_offsets[0x10000]; - static int random_sizes[0x8000]; - unsigned int i, p1, p2; - unsigned int t1, t2; - for (i = 0; i < 0x10000; i++) { // generate random offsets - random_offsets[i] = rand() % (10 * 1024 * 1024 + 1); - } - for (i = 0; i < 0x8000; i++) { // generate random sizes - random_sizes[i] = 1 + rand() % maxsize; - } - sleepms(100); - t1 = gettime(); - for (p1 = 0, p2 = 0, i = 0; i < times; i++) { - int offset1 = random_offsets[(p1++) & 0xffff]; - int offset2 = random_offsets[(p1++) & 0xffff]; - int size = random_sizes[(p2++) & 0x7fff]; - memcpy(A + offset1, B + offset2, size); - } - t1 = gettime() - t1; - sleepms(100); - t2 = gettime(); - for (p1 = 0, p2 = 0, i = 0; i < times; i++) { - int offset1 = random_offsets[(p1++) & 0xffff]; - int offset2 = random_offsets[(p1++) & 0xffff]; - int size = random_sizes[(p2++) & 0x7fff]; - memcpy_fast(A + offset1, B + offset2, size); - } - t2 = gettime() - t2; - printf("benchmark random access:\n"); - printf("memcpy_fast=%dms memcpy=%dms\n\n", (int)t2, (int)t1); -} - - -#ifdef _MSC_VER -#pragma comment(lib, "winmm.lib") -#endif - -int main(void) -{ - bench(32, 0x1000000); - bench(64, 0x1000000); - bench(512, 0x800000); - bench(1024, 0x400000); - bench(4096, 0x80000); - bench(8192, 0x40000); - bench(1024 * 1024 * 1, 0x800); - bench(1024 * 1024 * 4, 0x200); - bench(1024 * 1024 * 8, 0x100); - - random_bench(2048, 8000000); - - return 0; -} - - - - -/* -benchmark(size=32 bytes, times=16777216): -result(dst aligned, src aligned): memcpy_fast=78ms memcpy=260 ms -result(dst aligned, src unalign): memcpy_fast=78ms memcpy=250 ms -result(dst unalign, src aligned): memcpy_fast=78ms memcpy=266 ms -result(dst unalign, src unalign): memcpy_fast=78ms memcpy=234 ms - -benchmark(size=64 bytes, times=16777216): -result(dst aligned, src aligned): memcpy_fast=109ms memcpy=281 ms -result(dst aligned, src unalign): memcpy_fast=109ms memcpy=328 ms -result(dst unalign, src aligned): memcpy_fast=109ms memcpy=343 ms -result(dst unalign, src unalign): memcpy_fast=93ms memcpy=344 ms - -benchmark(size=512 bytes, times=8388608): -result(dst aligned, src aligned): memcpy_fast=125ms memcpy=218 ms -result(dst aligned, src unalign): memcpy_fast=156ms memcpy=484 ms -result(dst unalign, src aligned): memcpy_fast=172ms memcpy=546 ms -result(dst unalign, src unalign): memcpy_fast=172ms memcpy=515 ms - -benchmark(size=1024 bytes, times=4194304): -result(dst aligned, src aligned): memcpy_fast=109ms memcpy=172 ms -result(dst aligned, src unalign): memcpy_fast=187ms memcpy=453 ms -result(dst unalign, src aligned): memcpy_fast=172ms memcpy=437 ms -result(dst unalign, src unalign): memcpy_fast=156ms memcpy=452 ms - -benchmark(size=4096 bytes, times=524288): -result(dst aligned, src aligned): memcpy_fast=62ms memcpy=78 ms -result(dst aligned, src unalign): memcpy_fast=109ms memcpy=202 ms -result(dst unalign, src aligned): memcpy_fast=94ms memcpy=203 ms -result(dst unalign, src unalign): memcpy_fast=110ms memcpy=218 ms - -benchmark(size=8192 bytes, times=262144): -result(dst aligned, src aligned): memcpy_fast=62ms memcpy=78 ms -result(dst aligned, src unalign): memcpy_fast=78ms memcpy=202 ms -result(dst unalign, src aligned): memcpy_fast=78ms memcpy=203 ms -result(dst unalign, src unalign): memcpy_fast=94ms memcpy=203 ms - -benchmark(size=1048576 bytes, times=2048): -result(dst aligned, src aligned): memcpy_fast=203ms memcpy=191 ms -result(dst aligned, src unalign): memcpy_fast=219ms memcpy=281 ms -result(dst unalign, src aligned): memcpy_fast=218ms memcpy=328 ms -result(dst unalign, src unalign): memcpy_fast=218ms memcpy=312 ms - -benchmark(size=4194304 bytes, times=512): -result(dst aligned, src aligned): memcpy_fast=312ms memcpy=406 ms -result(dst aligned, src unalign): memcpy_fast=296ms memcpy=421 ms -result(dst unalign, src aligned): memcpy_fast=312ms memcpy=468 ms -result(dst unalign, src unalign): memcpy_fast=297ms memcpy=452 ms - -benchmark(size=8388608 bytes, times=256): -result(dst aligned, src aligned): memcpy_fast=281ms memcpy=452 ms -result(dst aligned, src unalign): memcpy_fast=280ms memcpy=468 ms -result(dst unalign, src aligned): memcpy_fast=298ms memcpy=514 ms -result(dst unalign, src unalign): memcpy_fast=344ms memcpy=472 ms - -benchmark random access: -memcpy_fast=515ms memcpy=1014ms - -*/ - - - - diff --git a/contrib/FastMemcpy/memcpy_wrapper.c b/contrib/FastMemcpy/memcpy_wrapper.c deleted file mode 100644 index 1f57345980a..00000000000 --- a/contrib/FastMemcpy/memcpy_wrapper.c +++ /dev/null @@ -1,6 +0,0 @@ -#include "FastMemcpy.h" - -void * memcpy(void * __restrict destination, const void * __restrict source, size_t size) -{ - return memcpy_fast(destination, source, size); -} diff --git a/contrib/FastMemcpy/memcpy_wrapper.cpp b/contrib/FastMemcpy/memcpy_wrapper.cpp new file mode 100644 index 00000000000..8fa6cf291ec --- /dev/null +++ b/contrib/FastMemcpy/memcpy_wrapper.cpp @@ -0,0 +1,6 @@ +#include "FastMemcpy.h" + +extern "C" void * memcpy(void * __restrict destination, const void * __restrict source, size_t size) +{ + return memcpy_fast(destination, source, size); +} diff --git a/utils/memcpy-bench/FastMemcpy.h b/utils/memcpy-bench/FastMemcpy.h new file mode 100644 index 00000000000..f071e2bed62 --- /dev/null +++ b/utils/memcpy-bench/FastMemcpy.h @@ -0,0 +1,754 @@ +//===================================================================== +// +// FastMemcpy.c - skywind3000@163.com, 2015 +// +// feature: +// 50% speed up in avg. vs standard memcpy (tested in vc2012/gcc5.1) +// +//===================================================================== +#pragma once + +#include +#include +#include + + +//--------------------------------------------------------------------- +// force inline for compilers +//--------------------------------------------------------------------- +#ifndef INLINE +#ifdef __GNUC__ +#if (__GNUC__ > 3) || ((__GNUC__ == 3) && (__GNUC_MINOR__ >= 1)) + #define INLINE __inline__ __attribute__((always_inline)) +#else + #define INLINE __inline__ +#endif +#elif defined(_MSC_VER) + #define INLINE __forceinline +#elif (defined(__BORLANDC__) || defined(__WATCOMC__)) + #define INLINE __inline +#else + #define INLINE +#endif +#endif + +typedef __attribute__((__aligned__(1))) uint16_t uint16_unaligned_t; +typedef __attribute__((__aligned__(1))) uint32_t uint32_unaligned_t; +typedef __attribute__((__aligned__(1))) uint64_t uint64_unaligned_t; + +//--------------------------------------------------------------------- +// fast copy for different sizes +//--------------------------------------------------------------------- +static INLINE void memcpy_sse2_16(void *dst, const void *src) { + __m128i m0 = _mm_loadu_si128((reinterpret_cast(src)) + 0); + _mm_storeu_si128((reinterpret_cast<__m128i*>(dst)) + 0, m0); +} + +static INLINE void memcpy_sse2_32(void *dst, const void *src) { + __m128i m0 = _mm_loadu_si128((reinterpret_cast(src)) + 0); + __m128i m1 = _mm_loadu_si128((reinterpret_cast(src)) + 1); + _mm_storeu_si128((reinterpret_cast<__m128i*>(dst)) + 0, m0); + _mm_storeu_si128((reinterpret_cast<__m128i*>(dst)) + 1, m1); +} + +static INLINE void memcpy_sse2_64(void *dst, const void *src) { + __m128i m0 = _mm_loadu_si128((reinterpret_cast(src)) + 0); + __m128i m1 = _mm_loadu_si128((reinterpret_cast(src)) + 1); + __m128i m2 = _mm_loadu_si128((reinterpret_cast(src)) + 2); + __m128i m3 = _mm_loadu_si128((reinterpret_cast(src)) + 3); + _mm_storeu_si128((reinterpret_cast<__m128i*>(dst)) + 0, m0); + _mm_storeu_si128((reinterpret_cast<__m128i*>(dst)) + 1, m1); + _mm_storeu_si128((reinterpret_cast<__m128i*>(dst)) + 2, m2); + _mm_storeu_si128((reinterpret_cast<__m128i*>(dst)) + 3, m3); +} + +static INLINE void memcpy_sse2_128(void *dst, const void *src) { + __m128i m0 = _mm_loadu_si128((reinterpret_cast(src)) + 0); + __m128i m1 = _mm_loadu_si128((reinterpret_cast(src)) + 1); + __m128i m2 = _mm_loadu_si128((reinterpret_cast(src)) + 2); + __m128i m3 = _mm_loadu_si128((reinterpret_cast(src)) + 3); + __m128i m4 = _mm_loadu_si128((reinterpret_cast(src)) + 4); + __m128i m5 = _mm_loadu_si128((reinterpret_cast(src)) + 5); + __m128i m6 = _mm_loadu_si128((reinterpret_cast(src)) + 6); + __m128i m7 = _mm_loadu_si128((reinterpret_cast(src)) + 7); + _mm_storeu_si128((reinterpret_cast<__m128i*>(dst)) + 0, m0); + _mm_storeu_si128((reinterpret_cast<__m128i*>(dst)) + 1, m1); + _mm_storeu_si128((reinterpret_cast<__m128i*>(dst)) + 2, m2); + _mm_storeu_si128((reinterpret_cast<__m128i*>(dst)) + 3, m3); + _mm_storeu_si128((reinterpret_cast<__m128i*>(dst)) + 4, m4); + _mm_storeu_si128((reinterpret_cast<__m128i*>(dst)) + 5, m5); + _mm_storeu_si128((reinterpret_cast<__m128i*>(dst)) + 6, m6); + _mm_storeu_si128((reinterpret_cast<__m128i*>(dst)) + 7, m7); +} + + +//--------------------------------------------------------------------- +// tiny memory copy with jump table optimized +//--------------------------------------------------------------------- +/// Attribute is used to avoid an error with undefined behaviour sanitizer +/// ../contrib/FastMemcpy/FastMemcpy.h:91:56: runtime error: applying zero offset to null pointer +/// Found by 01307_orc_output_format.sh, cause - ORCBlockInputFormat and external ORC library. +__attribute__((__no_sanitize__("undefined"))) static INLINE void *memcpy_tiny(void *dst, const void *src, size_t size) { + unsigned char *dd = ((unsigned char*)dst) + size; + const unsigned char *ss = ((const unsigned char*)src) + size; + + switch (size) { + case 64: + memcpy_sse2_64(dd - 64, ss - 64); + [[fallthrough]]; + case 0: + break; + + case 65: + memcpy_sse2_64(dd - 65, ss - 65); + [[fallthrough]]; + case 1: + dd[-1] = ss[-1]; + break; + + case 66: + memcpy_sse2_64(dd - 66, ss - 66); + [[fallthrough]]; + case 2: + *((uint16_unaligned_t*)(dd - 2)) = *((const uint16_unaligned_t*)(ss - 2)); + break; + + case 67: + memcpy_sse2_64(dd - 67, ss - 67); + [[fallthrough]]; + case 3: + *((uint16_unaligned_t*)(dd - 3)) = *((const uint16_unaligned_t*)(ss - 3)); + dd[-1] = ss[-1]; + break; + + case 68: + memcpy_sse2_64(dd - 68, ss - 68); + [[fallthrough]]; + case 4: + *((uint32_unaligned_t*)(dd - 4)) = *((const uint32_unaligned_t*)(ss - 4)); + break; + + case 69: + memcpy_sse2_64(dd - 69, ss - 69); + [[fallthrough]]; + case 5: + *((uint32_unaligned_t*)(dd - 5)) = *((const uint32_unaligned_t*)(ss - 5)); + dd[-1] = ss[-1]; + break; + + case 70: + memcpy_sse2_64(dd - 70, ss - 70); + [[fallthrough]]; + case 6: + *((uint32_unaligned_t*)(dd - 6)) = *((const uint32_unaligned_t*)(ss - 6)); + *((uint16_unaligned_t*)(dd - 2)) = *((const uint16_unaligned_t*)(ss - 2)); + break; + + case 71: + memcpy_sse2_64(dd - 71, ss - 71); + [[fallthrough]]; + case 7: + *((uint32_unaligned_t*)(dd - 7)) = *((const uint32_unaligned_t*)(ss - 7)); + *((uint32_unaligned_t*)(dd - 4)) = *((const uint32_unaligned_t*)(ss - 4)); + break; + + case 72: + memcpy_sse2_64(dd - 72, ss - 72); + [[fallthrough]]; + case 8: + *((uint64_unaligned_t*)(dd - 8)) = *((const uint64_unaligned_t*)(ss - 8)); + break; + + case 73: + memcpy_sse2_64(dd - 73, ss - 73); + [[fallthrough]]; + case 9: + *((uint64_unaligned_t*)(dd - 9)) = *((const uint64_unaligned_t*)(ss - 9)); + dd[-1] = ss[-1]; + break; + + case 74: + memcpy_sse2_64(dd - 74, ss - 74); + [[fallthrough]]; + case 10: + *((uint64_unaligned_t*)(dd - 10)) = *((const uint64_unaligned_t*)(ss - 10)); + *((uint16_unaligned_t*)(dd - 2)) = *((const uint16_unaligned_t*)(ss - 2)); + break; + + case 75: + memcpy_sse2_64(dd - 75, ss - 75); + [[fallthrough]]; + case 11: + *((uint64_unaligned_t*)(dd - 11)) = *((const uint64_unaligned_t*)(ss - 11)); + *((uint32_unaligned_t*)(dd - 4)) = *((const uint32_unaligned_t*)(ss - 4)); + break; + + case 76: + memcpy_sse2_64(dd - 76, ss - 76); + [[fallthrough]]; + case 12: + *((uint64_unaligned_t*)(dd - 12)) = *((const uint64_unaligned_t*)(ss - 12)); + *((uint32_unaligned_t*)(dd - 4)) = *((const uint32_unaligned_t*)(ss - 4)); + break; + + case 77: + memcpy_sse2_64(dd - 77, ss - 77); + [[fallthrough]]; + case 13: + *((uint64_unaligned_t*)(dd - 13)) = *((const uint64_unaligned_t*)(ss - 13)); + *((uint32_unaligned_t*)(dd - 5)) = *((const uint32_unaligned_t*)(ss - 5)); + dd[-1] = ss[-1]; + break; + + case 78: + memcpy_sse2_64(dd - 78, ss - 78); + [[fallthrough]]; + case 14: + *((uint64_unaligned_t*)(dd - 14)) = *((const uint64_unaligned_t*)(ss - 14)); + *((uint64_unaligned_t*)(dd - 8)) = *((const uint64_unaligned_t*)(ss - 8)); + break; + + case 79: + memcpy_sse2_64(dd - 79, ss - 79); + [[fallthrough]]; + case 15: + *((uint64_unaligned_t*)(dd - 15)) = *((const uint64_unaligned_t*)(ss - 15)); + *((uint64_unaligned_t*)(dd - 8)) = *((const uint64_unaligned_t*)(ss - 8)); + break; + + case 80: + memcpy_sse2_64(dd - 80, ss - 80); + [[fallthrough]]; + case 16: + memcpy_sse2_16(dd - 16, ss - 16); + break; + + case 81: + memcpy_sse2_64(dd - 81, ss - 81); + [[fallthrough]]; + case 17: + memcpy_sse2_16(dd - 17, ss - 17); + dd[-1] = ss[-1]; + break; + + case 82: + memcpy_sse2_64(dd - 82, ss - 82); + [[fallthrough]]; + case 18: + memcpy_sse2_16(dd - 18, ss - 18); + *((uint16_unaligned_t*)(dd - 2)) = *((const uint16_unaligned_t*)(ss - 2)); + break; + + case 83: + memcpy_sse2_64(dd - 83, ss - 83); + [[fallthrough]]; + case 19: + memcpy_sse2_16(dd - 19, ss - 19); + *((uint16_unaligned_t*)(dd - 3)) = *((const uint16_unaligned_t*)(ss - 3)); + dd[-1] = ss[-1]; + break; + + case 84: + memcpy_sse2_64(dd - 84, ss - 84); + [[fallthrough]]; + case 20: + memcpy_sse2_16(dd - 20, ss - 20); + *((uint32_unaligned_t*)(dd - 4)) = *((const uint32_unaligned_t*)(ss - 4)); + break; + + case 85: + memcpy_sse2_64(dd - 85, ss - 85); + [[fallthrough]]; + case 21: + memcpy_sse2_16(dd - 21, ss - 21); + *((uint32_unaligned_t*)(dd - 5)) = *((const uint32_unaligned_t*)(ss - 5)); + dd[-1] = ss[-1]; + break; + + case 86: + memcpy_sse2_64(dd - 86, ss - 86); + [[fallthrough]]; + case 22: + memcpy_sse2_16(dd - 22, ss - 22); + *((uint32_unaligned_t*)(dd - 6)) = *((const uint32_unaligned_t*)(ss - 6)); + *((uint16_unaligned_t*)(dd - 2)) = *((const uint16_unaligned_t*)(ss - 2)); + break; + + case 87: + memcpy_sse2_64(dd - 87, ss - 87); + [[fallthrough]]; + case 23: + memcpy_sse2_16(dd - 23, ss - 23); + *((uint32_unaligned_t*)(dd - 7)) = *((const uint32_unaligned_t*)(ss - 7)); + *((uint32_unaligned_t*)(dd - 4)) = *((const uint32_unaligned_t*)(ss - 4)); + break; + + case 88: + memcpy_sse2_64(dd - 88, ss - 88); + [[fallthrough]]; + case 24: + memcpy_sse2_16(dd - 24, ss - 24); + memcpy_sse2_16(dd - 16, ss - 16); + break; + + case 89: + memcpy_sse2_64(dd - 89, ss - 89); + [[fallthrough]]; + case 25: + memcpy_sse2_16(dd - 25, ss - 25); + memcpy_sse2_16(dd - 16, ss - 16); + break; + + case 90: + memcpy_sse2_64(dd - 90, ss - 90); + [[fallthrough]]; + case 26: + memcpy_sse2_16(dd - 26, ss - 26); + memcpy_sse2_16(dd - 16, ss - 16); + break; + + case 91: + memcpy_sse2_64(dd - 91, ss - 91); + [[fallthrough]]; + case 27: + memcpy_sse2_16(dd - 27, ss - 27); + memcpy_sse2_16(dd - 16, ss - 16); + break; + + case 92: + memcpy_sse2_64(dd - 92, ss - 92); + [[fallthrough]]; + case 28: + memcpy_sse2_16(dd - 28, ss - 28); + memcpy_sse2_16(dd - 16, ss - 16); + break; + + case 93: + memcpy_sse2_64(dd - 93, ss - 93); + [[fallthrough]]; + case 29: + memcpy_sse2_16(dd - 29, ss - 29); + memcpy_sse2_16(dd - 16, ss - 16); + break; + + case 94: + memcpy_sse2_64(dd - 94, ss - 94); + [[fallthrough]]; + case 30: + memcpy_sse2_16(dd - 30, ss - 30); + memcpy_sse2_16(dd - 16, ss - 16); + break; + + case 95: + memcpy_sse2_64(dd - 95, ss - 95); + [[fallthrough]]; + case 31: + memcpy_sse2_16(dd - 31, ss - 31); + memcpy_sse2_16(dd - 16, ss - 16); + break; + + case 96: + memcpy_sse2_64(dd - 96, ss - 96); + [[fallthrough]]; + case 32: + memcpy_sse2_32(dd - 32, ss - 32); + break; + + case 97: + memcpy_sse2_64(dd - 97, ss - 97); + [[fallthrough]]; + case 33: + memcpy_sse2_32(dd - 33, ss - 33); + dd[-1] = ss[-1]; + break; + + case 98: + memcpy_sse2_64(dd - 98, ss - 98); + [[fallthrough]]; + case 34: + memcpy_sse2_32(dd - 34, ss - 34); + *((uint16_unaligned_t*)(dd - 2)) = *((const uint16_unaligned_t*)(ss - 2)); + break; + + case 99: + memcpy_sse2_64(dd - 99, ss - 99); + [[fallthrough]]; + case 35: + memcpy_sse2_32(dd - 35, ss - 35); + *((uint16_unaligned_t*)(dd - 3)) = *((const uint16_unaligned_t*)(ss - 3)); + dd[-1] = ss[-1]; + break; + + case 100: + memcpy_sse2_64(dd - 100, ss - 100); + [[fallthrough]]; + case 36: + memcpy_sse2_32(dd - 36, ss - 36); + *((uint32_unaligned_t*)(dd - 4)) = *((const uint32_unaligned_t*)(ss - 4)); + break; + + case 101: + memcpy_sse2_64(dd - 101, ss - 101); + [[fallthrough]]; + case 37: + memcpy_sse2_32(dd - 37, ss - 37); + *((uint32_unaligned_t*)(dd - 5)) = *((const uint32_unaligned_t*)(ss - 5)); + dd[-1] = ss[-1]; + break; + + case 102: + memcpy_sse2_64(dd - 102, ss - 102); + [[fallthrough]]; + case 38: + memcpy_sse2_32(dd - 38, ss - 38); + *((uint32_unaligned_t*)(dd - 6)) = *((const uint32_unaligned_t*)(ss - 6)); + *((uint16_unaligned_t*)(dd - 2)) = *((const uint16_unaligned_t*)(ss - 2)); + break; + + case 103: + memcpy_sse2_64(dd - 103, ss - 103); + [[fallthrough]]; + case 39: + memcpy_sse2_32(dd - 39, ss - 39); + *((uint32_unaligned_t*)(dd - 7)) = *((const uint32_unaligned_t*)(ss - 7)); + *((uint32_unaligned_t*)(dd - 4)) = *((const uint32_unaligned_t*)(ss - 4)); + break; + + case 104: + memcpy_sse2_64(dd - 104, ss - 104); + [[fallthrough]]; + case 40: + memcpy_sse2_32(dd - 40, ss - 40); + *((uint64_unaligned_t*)(dd - 8)) = *((const uint64_unaligned_t*)(ss - 8)); + break; + + case 105: + memcpy_sse2_64(dd - 105, ss - 105); + [[fallthrough]]; + case 41: + memcpy_sse2_32(dd - 41, ss - 41); + *((uint64_unaligned_t*)(dd - 9)) = *((const uint64_unaligned_t*)(ss - 9)); + dd[-1] = ss[-1]; + break; + + case 106: + memcpy_sse2_64(dd - 106, ss - 106); + [[fallthrough]]; + case 42: + memcpy_sse2_32(dd - 42, ss - 42); + *((uint64_unaligned_t*)(dd - 10)) = *((const uint64_unaligned_t*)(ss - 10)); + *((uint16_unaligned_t*)(dd - 2)) = *((const uint16_unaligned_t*)(ss - 2)); + break; + + case 107: + memcpy_sse2_64(dd - 107, ss - 107); + [[fallthrough]]; + case 43: + memcpy_sse2_32(dd - 43, ss - 43); + *((uint64_unaligned_t*)(dd - 11)) = *((const uint64_unaligned_t*)(ss - 11)); + *((uint32_unaligned_t*)(dd - 4)) = *((const uint32_unaligned_t*)(ss - 4)); + break; + + case 108: + memcpy_sse2_64(dd - 108, ss - 108); + [[fallthrough]]; + case 44: + memcpy_sse2_32(dd - 44, ss - 44); + *((uint64_unaligned_t*)(dd - 12)) = *((const uint64_unaligned_t*)(ss - 12)); + *((uint32_unaligned_t*)(dd - 4)) = *((const uint32_unaligned_t*)(ss - 4)); + break; + + case 109: + memcpy_sse2_64(dd - 109, ss - 109); + [[fallthrough]]; + case 45: + memcpy_sse2_32(dd - 45, ss - 45); + *((uint64_unaligned_t*)(dd - 13)) = *((const uint64_unaligned_t*)(ss - 13)); + *((uint32_unaligned_t*)(dd - 5)) = *((const uint32_unaligned_t*)(ss - 5)); + dd[-1] = ss[-1]; + break; + + case 110: + memcpy_sse2_64(dd - 110, ss - 110); + [[fallthrough]]; + case 46: + memcpy_sse2_32(dd - 46, ss - 46); + *((uint64_unaligned_t*)(dd - 14)) = *((const uint64_unaligned_t*)(ss - 14)); + *((uint64_unaligned_t*)(dd - 8)) = *((const uint64_unaligned_t*)(ss - 8)); + break; + + case 111: + memcpy_sse2_64(dd - 111, ss - 111); + [[fallthrough]]; + case 47: + memcpy_sse2_32(dd - 47, ss - 47); + *((uint64_unaligned_t*)(dd - 15)) = *((const uint64_unaligned_t*)(ss - 15)); + *((uint64_unaligned_t*)(dd - 8)) = *((const uint64_unaligned_t*)(ss - 8)); + break; + + case 112: + memcpy_sse2_64(dd - 112, ss - 112); + [[fallthrough]]; + case 48: + memcpy_sse2_32(dd - 48, ss - 48); + memcpy_sse2_16(dd - 16, ss - 16); + break; + + case 113: + memcpy_sse2_64(dd - 113, ss - 113); + [[fallthrough]]; + case 49: + memcpy_sse2_32(dd - 49, ss - 49); + memcpy_sse2_16(dd - 17, ss - 17); + dd[-1] = ss[-1]; + break; + + case 114: + memcpy_sse2_64(dd - 114, ss - 114); + [[fallthrough]]; + case 50: + memcpy_sse2_32(dd - 50, ss - 50); + memcpy_sse2_16(dd - 18, ss - 18); + *((uint16_unaligned_t*)(dd - 2)) = *((const uint16_unaligned_t*)(ss - 2)); + break; + + case 115: + memcpy_sse2_64(dd - 115, ss - 115); + [[fallthrough]]; + case 51: + memcpy_sse2_32(dd - 51, ss - 51); + memcpy_sse2_16(dd - 19, ss - 19); + *((uint16_unaligned_t*)(dd - 3)) = *((const uint16_unaligned_t*)(ss - 3)); + dd[-1] = ss[-1]; + break; + + case 116: + memcpy_sse2_64(dd - 116, ss - 116); + [[fallthrough]]; + case 52: + memcpy_sse2_32(dd - 52, ss - 52); + memcpy_sse2_16(dd - 20, ss - 20); + *((uint32_unaligned_t*)(dd - 4)) = *((const uint32_unaligned_t*)(ss - 4)); + break; + + case 117: + memcpy_sse2_64(dd - 117, ss - 117); + [[fallthrough]]; + case 53: + memcpy_sse2_32(dd - 53, ss - 53); + memcpy_sse2_16(dd - 21, ss - 21); + *((uint32_unaligned_t*)(dd - 5)) = *((const uint32_unaligned_t*)(ss - 5)); + dd[-1] = ss[-1]; + break; + + case 118: + memcpy_sse2_64(dd - 118, ss - 118); + [[fallthrough]]; + case 54: + memcpy_sse2_32(dd - 54, ss - 54); + memcpy_sse2_16(dd - 22, ss - 22); + *((uint32_unaligned_t*)(dd - 6)) = *((const uint32_unaligned_t*)(ss - 6)); + *((uint16_unaligned_t*)(dd - 2)) = *((const uint16_unaligned_t*)(ss - 2)); + break; + + case 119: + memcpy_sse2_64(dd - 119, ss - 119); + [[fallthrough]]; + case 55: + memcpy_sse2_32(dd - 55, ss - 55); + memcpy_sse2_16(dd - 23, ss - 23); + *((uint32_unaligned_t*)(dd - 7)) = *((const uint32_unaligned_t*)(ss - 7)); + *((uint32_unaligned_t*)(dd - 4)) = *((const uint32_unaligned_t*)(ss - 4)); + break; + + case 120: + memcpy_sse2_64(dd - 120, ss - 120); + [[fallthrough]]; + case 56: + memcpy_sse2_32(dd - 56, ss - 56); + memcpy_sse2_16(dd - 24, ss - 24); + memcpy_sse2_16(dd - 16, ss - 16); + break; + + case 121: + memcpy_sse2_64(dd - 121, ss - 121); + [[fallthrough]]; + case 57: + memcpy_sse2_32(dd - 57, ss - 57); + memcpy_sse2_16(dd - 25, ss - 25); + memcpy_sse2_16(dd - 16, ss - 16); + break; + + case 122: + memcpy_sse2_64(dd - 122, ss - 122); + [[fallthrough]]; + case 58: + memcpy_sse2_32(dd - 58, ss - 58); + memcpy_sse2_16(dd - 26, ss - 26); + memcpy_sse2_16(dd - 16, ss - 16); + break; + + case 123: + memcpy_sse2_64(dd - 123, ss - 123); + [[fallthrough]]; + case 59: + memcpy_sse2_32(dd - 59, ss - 59); + memcpy_sse2_16(dd - 27, ss - 27); + memcpy_sse2_16(dd - 16, ss - 16); + break; + + case 124: + memcpy_sse2_64(dd - 124, ss - 124); + [[fallthrough]]; + case 60: + memcpy_sse2_32(dd - 60, ss - 60); + memcpy_sse2_16(dd - 28, ss - 28); + memcpy_sse2_16(dd - 16, ss - 16); + break; + + case 125: + memcpy_sse2_64(dd - 125, ss - 125); + [[fallthrough]]; + case 61: + memcpy_sse2_32(dd - 61, ss - 61); + memcpy_sse2_16(dd - 29, ss - 29); + memcpy_sse2_16(dd - 16, ss - 16); + break; + + case 126: + memcpy_sse2_64(dd - 126, ss - 126); + [[fallthrough]]; + case 62: + memcpy_sse2_32(dd - 62, ss - 62); + memcpy_sse2_16(dd - 30, ss - 30); + memcpy_sse2_16(dd - 16, ss - 16); + break; + + case 127: + memcpy_sse2_64(dd - 127, ss - 127); + [[fallthrough]]; + case 63: + memcpy_sse2_32(dd - 63, ss - 63); + memcpy_sse2_16(dd - 31, ss - 31); + memcpy_sse2_16(dd - 16, ss - 16); + break; + + case 128: + memcpy_sse2_128(dd - 128, ss - 128); + break; + } + + return dst; +} + + +//--------------------------------------------------------------------- +// main routine +//--------------------------------------------------------------------- +void* memcpy_fast(void *destination, const void *source, size_t size) +{ + unsigned char *dst = (unsigned char*)destination; + const unsigned char *src = (const unsigned char*)source; + static size_t cachesize = 0x200000; // L2-cache size + size_t padding; + + // small memory copy + if (size <= 128) { + return memcpy_tiny(dst, src, size); + } + + // align destination to 16 bytes boundary + padding = (16 - (((size_t)dst) & 15)) & 15; + + if (padding > 0) { + __m128i head = _mm_loadu_si128(reinterpret_cast(src)); + _mm_storeu_si128(reinterpret_cast<__m128i*>(dst), head); + dst += padding; + src += padding; + size -= padding; + } + + // medium size copy + if (size <= cachesize) { + __m128i c0, c1, c2, c3, c4, c5, c6, c7; + + for (; size >= 128; size -= 128) { + c0 = _mm_loadu_si128((reinterpret_cast(src)) + 0); + c1 = _mm_loadu_si128((reinterpret_cast(src)) + 1); + c2 = _mm_loadu_si128((reinterpret_cast(src)) + 2); + c3 = _mm_loadu_si128((reinterpret_cast(src)) + 3); + c4 = _mm_loadu_si128((reinterpret_cast(src)) + 4); + c5 = _mm_loadu_si128((reinterpret_cast(src)) + 5); + c6 = _mm_loadu_si128((reinterpret_cast(src)) + 6); + c7 = _mm_loadu_si128((reinterpret_cast(src)) + 7); + _mm_prefetch((const char*)(src + 256), _MM_HINT_NTA); + src += 128; + _mm_store_si128(((reinterpret_cast<__m128i*>(dst)) + 0), c0); + _mm_store_si128(((reinterpret_cast<__m128i*>(dst)) + 1), c1); + _mm_store_si128(((reinterpret_cast<__m128i*>(dst)) + 2), c2); + _mm_store_si128(((reinterpret_cast<__m128i*>(dst)) + 3), c3); + _mm_store_si128(((reinterpret_cast<__m128i*>(dst)) + 4), c4); + _mm_store_si128(((reinterpret_cast<__m128i*>(dst)) + 5), c5); + _mm_store_si128(((reinterpret_cast<__m128i*>(dst)) + 6), c6); + _mm_store_si128(((reinterpret_cast<__m128i*>(dst)) + 7), c7); + dst += 128; + } + } + else { // big memory copy + __m128i c0, c1, c2, c3, c4, c5, c6, c7; + + _mm_prefetch((const char*)(src), _MM_HINT_NTA); + + if ((((size_t)src) & 15) == 0) { // source aligned + for (; size >= 128; size -= 128) { + c0 = _mm_load_si128((reinterpret_cast(src)) + 0); + c1 = _mm_load_si128((reinterpret_cast(src)) + 1); + c2 = _mm_load_si128((reinterpret_cast(src)) + 2); + c3 = _mm_load_si128((reinterpret_cast(src)) + 3); + c4 = _mm_load_si128((reinterpret_cast(src)) + 4); + c5 = _mm_load_si128((reinterpret_cast(src)) + 5); + c6 = _mm_load_si128((reinterpret_cast(src)) + 6); + c7 = _mm_load_si128((reinterpret_cast(src)) + 7); + _mm_prefetch((const char*)(src + 256), _MM_HINT_NTA); + src += 128; + _mm_stream_si128(((reinterpret_cast<__m128i*>(dst)) + 0), c0); + _mm_stream_si128(((reinterpret_cast<__m128i*>(dst)) + 1), c1); + _mm_stream_si128(((reinterpret_cast<__m128i*>(dst)) + 2), c2); + _mm_stream_si128(((reinterpret_cast<__m128i*>(dst)) + 3), c3); + _mm_stream_si128(((reinterpret_cast<__m128i*>(dst)) + 4), c4); + _mm_stream_si128(((reinterpret_cast<__m128i*>(dst)) + 5), c5); + _mm_stream_si128(((reinterpret_cast<__m128i*>(dst)) + 6), c6); + _mm_stream_si128(((reinterpret_cast<__m128i*>(dst)) + 7), c7); + dst += 128; + } + } + else { // source unaligned + for (; size >= 128; size -= 128) { + c0 = _mm_loadu_si128((reinterpret_cast(src)) + 0); + c1 = _mm_loadu_si128((reinterpret_cast(src)) + 1); + c2 = _mm_loadu_si128((reinterpret_cast(src)) + 2); + c3 = _mm_loadu_si128((reinterpret_cast(src)) + 3); + c4 = _mm_loadu_si128((reinterpret_cast(src)) + 4); + c5 = _mm_loadu_si128((reinterpret_cast(src)) + 5); + c6 = _mm_loadu_si128((reinterpret_cast(src)) + 6); + c7 = _mm_loadu_si128((reinterpret_cast(src)) + 7); + _mm_prefetch((const char*)(src + 256), _MM_HINT_NTA); + src += 128; + _mm_stream_si128(((reinterpret_cast<__m128i*>(dst)) + 0), c0); + _mm_stream_si128(((reinterpret_cast<__m128i*>(dst)) + 1), c1); + _mm_stream_si128(((reinterpret_cast<__m128i*>(dst)) + 2), c2); + _mm_stream_si128(((reinterpret_cast<__m128i*>(dst)) + 3), c3); + _mm_stream_si128(((reinterpret_cast<__m128i*>(dst)) + 4), c4); + _mm_stream_si128(((reinterpret_cast<__m128i*>(dst)) + 5), c5); + _mm_stream_si128(((reinterpret_cast<__m128i*>(dst)) + 6), c6); + _mm_stream_si128(((reinterpret_cast<__m128i*>(dst)) + 7), c7); + dst += 128; + } + } + _mm_sfence(); + } + + memcpy_tiny(dst, src, size); + + return destination; +} From 940ce5884e97019c5a388d4624a28d22daf43a34 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Mon, 8 Mar 2021 12:04:52 +0300 Subject: [PATCH 162/544] Add memcpy-bench tool --- utils/CMakeLists.txt | 1 + utils/memcpy-bench/CMakeLists.txt | 5 + utils/memcpy-bench/FastMemcpy.h | 1104 +++++++++++++-------------- utils/memcpy-bench/FastMemcpy_Avx.h | 480 ++++++++++++ utils/memcpy-bench/memcpy-bench.cpp | 610 +++++++++++++++ utils/memcpy-bench/memcpy_jart.S | 138 ++++ 6 files changed, 1786 insertions(+), 552 deletions(-) create mode 100644 utils/memcpy-bench/CMakeLists.txt create mode 100644 utils/memcpy-bench/FastMemcpy_Avx.h create mode 100644 utils/memcpy-bench/memcpy-bench.cpp create mode 100644 utils/memcpy-bench/memcpy_jart.S diff --git a/utils/CMakeLists.txt b/utils/CMakeLists.txt index 8a39d591612..d38b34f3419 100644 --- a/utils/CMakeLists.txt +++ b/utils/CMakeLists.txt @@ -32,6 +32,7 @@ if (NOT DEFINED ENABLE_UTILS OR ENABLE_UTILS) add_subdirectory (db-generator) add_subdirectory (wal-dump) add_subdirectory (check-mysql-binlog) + add_subdirectory (memcpy-bench) endif () if (ENABLE_CODE_QUALITY) diff --git a/utils/memcpy-bench/CMakeLists.txt b/utils/memcpy-bench/CMakeLists.txt new file mode 100644 index 00000000000..54dd0398912 --- /dev/null +++ b/utils/memcpy-bench/CMakeLists.txt @@ -0,0 +1,5 @@ +enable_language(ASM) +add_executable (memcpy-bench memcpy-bench.cpp memcpy_jart.S) +#target_compile_options(memcpy-bench PRIVATE -mavx) +target_link_libraries(memcpy-bench PRIVATE dbms) + diff --git a/utils/memcpy-bench/FastMemcpy.h b/utils/memcpy-bench/FastMemcpy.h index f071e2bed62..3f2278eac33 100644 --- a/utils/memcpy-bench/FastMemcpy.h +++ b/utils/memcpy-bench/FastMemcpy.h @@ -24,7 +24,7 @@ #define INLINE __inline__ #endif #elif defined(_MSC_VER) - #define INLINE __forceinline + #define INLINE __forceinline #elif (defined(__BORLANDC__) || defined(__WATCOMC__)) #define INLINE __inline #else @@ -39,46 +39,46 @@ typedef __attribute__((__aligned__(1))) uint64_t uint64_unaligned_t; //--------------------------------------------------------------------- // fast copy for different sizes //--------------------------------------------------------------------- -static INLINE void memcpy_sse2_16(void *dst, const void *src) { - __m128i m0 = _mm_loadu_si128((reinterpret_cast(src)) + 0); - _mm_storeu_si128((reinterpret_cast<__m128i*>(dst)) + 0, m0); +static INLINE void memcpy_sse2_16(void * __restrict dst, const void * __restrict src) { + __m128i m0 = _mm_loadu_si128((reinterpret_cast(src)) + 0); + _mm_storeu_si128((reinterpret_cast<__m128i*>(dst)) + 0, m0); } -static INLINE void memcpy_sse2_32(void *dst, const void *src) { - __m128i m0 = _mm_loadu_si128((reinterpret_cast(src)) + 0); - __m128i m1 = _mm_loadu_si128((reinterpret_cast(src)) + 1); - _mm_storeu_si128((reinterpret_cast<__m128i*>(dst)) + 0, m0); - _mm_storeu_si128((reinterpret_cast<__m128i*>(dst)) + 1, m1); +static INLINE void memcpy_sse2_32(void * __restrict dst, const void * __restrict src) { + __m128i m0 = _mm_loadu_si128((reinterpret_cast(src)) + 0); + __m128i m1 = _mm_loadu_si128((reinterpret_cast(src)) + 1); + _mm_storeu_si128((reinterpret_cast<__m128i*>(dst)) + 0, m0); + _mm_storeu_si128((reinterpret_cast<__m128i*>(dst)) + 1, m1); } -static INLINE void memcpy_sse2_64(void *dst, const void *src) { - __m128i m0 = _mm_loadu_si128((reinterpret_cast(src)) + 0); - __m128i m1 = _mm_loadu_si128((reinterpret_cast(src)) + 1); - __m128i m2 = _mm_loadu_si128((reinterpret_cast(src)) + 2); - __m128i m3 = _mm_loadu_si128((reinterpret_cast(src)) + 3); - _mm_storeu_si128((reinterpret_cast<__m128i*>(dst)) + 0, m0); - _mm_storeu_si128((reinterpret_cast<__m128i*>(dst)) + 1, m1); - _mm_storeu_si128((reinterpret_cast<__m128i*>(dst)) + 2, m2); - _mm_storeu_si128((reinterpret_cast<__m128i*>(dst)) + 3, m3); +static INLINE void memcpy_sse2_64(void * __restrict dst, const void * __restrict src) { + __m128i m0 = _mm_loadu_si128((reinterpret_cast(src)) + 0); + __m128i m1 = _mm_loadu_si128((reinterpret_cast(src)) + 1); + __m128i m2 = _mm_loadu_si128((reinterpret_cast(src)) + 2); + __m128i m3 = _mm_loadu_si128((reinterpret_cast(src)) + 3); + _mm_storeu_si128((reinterpret_cast<__m128i*>(dst)) + 0, m0); + _mm_storeu_si128((reinterpret_cast<__m128i*>(dst)) + 1, m1); + _mm_storeu_si128((reinterpret_cast<__m128i*>(dst)) + 2, m2); + _mm_storeu_si128((reinterpret_cast<__m128i*>(dst)) + 3, m3); } -static INLINE void memcpy_sse2_128(void *dst, const void *src) { - __m128i m0 = _mm_loadu_si128((reinterpret_cast(src)) + 0); - __m128i m1 = _mm_loadu_si128((reinterpret_cast(src)) + 1); - __m128i m2 = _mm_loadu_si128((reinterpret_cast(src)) + 2); - __m128i m3 = _mm_loadu_si128((reinterpret_cast(src)) + 3); - __m128i m4 = _mm_loadu_si128((reinterpret_cast(src)) + 4); - __m128i m5 = _mm_loadu_si128((reinterpret_cast(src)) + 5); - __m128i m6 = _mm_loadu_si128((reinterpret_cast(src)) + 6); - __m128i m7 = _mm_loadu_si128((reinterpret_cast(src)) + 7); - _mm_storeu_si128((reinterpret_cast<__m128i*>(dst)) + 0, m0); - _mm_storeu_si128((reinterpret_cast<__m128i*>(dst)) + 1, m1); - _mm_storeu_si128((reinterpret_cast<__m128i*>(dst)) + 2, m2); - _mm_storeu_si128((reinterpret_cast<__m128i*>(dst)) + 3, m3); - _mm_storeu_si128((reinterpret_cast<__m128i*>(dst)) + 4, m4); - _mm_storeu_si128((reinterpret_cast<__m128i*>(dst)) + 5, m5); - _mm_storeu_si128((reinterpret_cast<__m128i*>(dst)) + 6, m6); - _mm_storeu_si128((reinterpret_cast<__m128i*>(dst)) + 7, m7); +static INLINE void memcpy_sse2_128(void * __restrict dst, const void * __restrict src) { + __m128i m0 = _mm_loadu_si128((reinterpret_cast(src)) + 0); + __m128i m1 = _mm_loadu_si128((reinterpret_cast(src)) + 1); + __m128i m2 = _mm_loadu_si128((reinterpret_cast(src)) + 2); + __m128i m3 = _mm_loadu_si128((reinterpret_cast(src)) + 3); + __m128i m4 = _mm_loadu_si128((reinterpret_cast(src)) + 4); + __m128i m5 = _mm_loadu_si128((reinterpret_cast(src)) + 5); + __m128i m6 = _mm_loadu_si128((reinterpret_cast(src)) + 6); + __m128i m7 = _mm_loadu_si128((reinterpret_cast(src)) + 7); + _mm_storeu_si128((reinterpret_cast<__m128i*>(dst)) + 0, m0); + _mm_storeu_si128((reinterpret_cast<__m128i*>(dst)) + 1, m1); + _mm_storeu_si128((reinterpret_cast<__m128i*>(dst)) + 2, m2); + _mm_storeu_si128((reinterpret_cast<__m128i*>(dst)) + 3, m3); + _mm_storeu_si128((reinterpret_cast<__m128i*>(dst)) + 4, m4); + _mm_storeu_si128((reinterpret_cast<__m128i*>(dst)) + 5, m5); + _mm_storeu_si128((reinterpret_cast<__m128i*>(dst)) + 6, m6); + _mm_storeu_si128((reinterpret_cast<__m128i*>(dst)) + 7, m7); } @@ -88,667 +88,667 @@ static INLINE void memcpy_sse2_128(void *dst, const void *src) { /// Attribute is used to avoid an error with undefined behaviour sanitizer /// ../contrib/FastMemcpy/FastMemcpy.h:91:56: runtime error: applying zero offset to null pointer /// Found by 01307_orc_output_format.sh, cause - ORCBlockInputFormat and external ORC library. -__attribute__((__no_sanitize__("undefined"))) static INLINE void *memcpy_tiny(void *dst, const void *src, size_t size) { - unsigned char *dd = ((unsigned char*)dst) + size; - const unsigned char *ss = ((const unsigned char*)src) + size; +__attribute__((__no_sanitize__("undefined"))) static INLINE void *memcpy_tiny(void * __restrict dst, const void * __restrict src, size_t size) { + unsigned char *dd = ((unsigned char*)dst) + size; + const unsigned char *ss = ((const unsigned char*)src) + size; - switch (size) { - case 64: - memcpy_sse2_64(dd - 64, ss - 64); + switch (size) { + case 64: + memcpy_sse2_64(dd - 64, ss - 64); [[fallthrough]]; - case 0: - break; + case 0: + break; - case 65: - memcpy_sse2_64(dd - 65, ss - 65); + case 65: + memcpy_sse2_64(dd - 65, ss - 65); [[fallthrough]]; - case 1: - dd[-1] = ss[-1]; - break; + case 1: + dd[-1] = ss[-1]; + break; - case 66: - memcpy_sse2_64(dd - 66, ss - 66); + case 66: + memcpy_sse2_64(dd - 66, ss - 66); [[fallthrough]]; - case 2: - *((uint16_unaligned_t*)(dd - 2)) = *((const uint16_unaligned_t*)(ss - 2)); - break; + case 2: + *((uint16_unaligned_t*)(dd - 2)) = *((const uint16_unaligned_t*)(ss - 2)); + break; - case 67: - memcpy_sse2_64(dd - 67, ss - 67); + case 67: + memcpy_sse2_64(dd - 67, ss - 67); [[fallthrough]]; - case 3: - *((uint16_unaligned_t*)(dd - 3)) = *((const uint16_unaligned_t*)(ss - 3)); - dd[-1] = ss[-1]; - break; + case 3: + *((uint16_unaligned_t*)(dd - 3)) = *((const uint16_unaligned_t*)(ss - 3)); + dd[-1] = ss[-1]; + break; - case 68: - memcpy_sse2_64(dd - 68, ss - 68); + case 68: + memcpy_sse2_64(dd - 68, ss - 68); [[fallthrough]]; - case 4: - *((uint32_unaligned_t*)(dd - 4)) = *((const uint32_unaligned_t*)(ss - 4)); - break; + case 4: + *((uint32_unaligned_t*)(dd - 4)) = *((const uint32_unaligned_t*)(ss - 4)); + break; - case 69: - memcpy_sse2_64(dd - 69, ss - 69); + case 69: + memcpy_sse2_64(dd - 69, ss - 69); [[fallthrough]]; - case 5: - *((uint32_unaligned_t*)(dd - 5)) = *((const uint32_unaligned_t*)(ss - 5)); - dd[-1] = ss[-1]; - break; + case 5: + *((uint32_unaligned_t*)(dd - 5)) = *((const uint32_unaligned_t*)(ss - 5)); + dd[-1] = ss[-1]; + break; - case 70: - memcpy_sse2_64(dd - 70, ss - 70); + case 70: + memcpy_sse2_64(dd - 70, ss - 70); [[fallthrough]]; - case 6: - *((uint32_unaligned_t*)(dd - 6)) = *((const uint32_unaligned_t*)(ss - 6)); - *((uint16_unaligned_t*)(dd - 2)) = *((const uint16_unaligned_t*)(ss - 2)); - break; + case 6: + *((uint32_unaligned_t*)(dd - 6)) = *((const uint32_unaligned_t*)(ss - 6)); + *((uint16_unaligned_t*)(dd - 2)) = *((const uint16_unaligned_t*)(ss - 2)); + break; - case 71: - memcpy_sse2_64(dd - 71, ss - 71); + case 71: + memcpy_sse2_64(dd - 71, ss - 71); [[fallthrough]]; - case 7: - *((uint32_unaligned_t*)(dd - 7)) = *((const uint32_unaligned_t*)(ss - 7)); - *((uint32_unaligned_t*)(dd - 4)) = *((const uint32_unaligned_t*)(ss - 4)); - break; + case 7: + *((uint32_unaligned_t*)(dd - 7)) = *((const uint32_unaligned_t*)(ss - 7)); + *((uint32_unaligned_t*)(dd - 4)) = *((const uint32_unaligned_t*)(ss - 4)); + break; - case 72: - memcpy_sse2_64(dd - 72, ss - 72); + case 72: + memcpy_sse2_64(dd - 72, ss - 72); [[fallthrough]]; - case 8: - *((uint64_unaligned_t*)(dd - 8)) = *((const uint64_unaligned_t*)(ss - 8)); - break; + case 8: + *((uint64_unaligned_t*)(dd - 8)) = *((const uint64_unaligned_t*)(ss - 8)); + break; - case 73: - memcpy_sse2_64(dd - 73, ss - 73); + case 73: + memcpy_sse2_64(dd - 73, ss - 73); [[fallthrough]]; - case 9: - *((uint64_unaligned_t*)(dd - 9)) = *((const uint64_unaligned_t*)(ss - 9)); - dd[-1] = ss[-1]; - break; + case 9: + *((uint64_unaligned_t*)(dd - 9)) = *((const uint64_unaligned_t*)(ss - 9)); + dd[-1] = ss[-1]; + break; - case 74: - memcpy_sse2_64(dd - 74, ss - 74); + case 74: + memcpy_sse2_64(dd - 74, ss - 74); [[fallthrough]]; - case 10: - *((uint64_unaligned_t*)(dd - 10)) = *((const uint64_unaligned_t*)(ss - 10)); - *((uint16_unaligned_t*)(dd - 2)) = *((const uint16_unaligned_t*)(ss - 2)); - break; + case 10: + *((uint64_unaligned_t*)(dd - 10)) = *((const uint64_unaligned_t*)(ss - 10)); + *((uint16_unaligned_t*)(dd - 2)) = *((const uint16_unaligned_t*)(ss - 2)); + break; - case 75: - memcpy_sse2_64(dd - 75, ss - 75); + case 75: + memcpy_sse2_64(dd - 75, ss - 75); [[fallthrough]]; - case 11: - *((uint64_unaligned_t*)(dd - 11)) = *((const uint64_unaligned_t*)(ss - 11)); - *((uint32_unaligned_t*)(dd - 4)) = *((const uint32_unaligned_t*)(ss - 4)); - break; + case 11: + *((uint64_unaligned_t*)(dd - 11)) = *((const uint64_unaligned_t*)(ss - 11)); + *((uint32_unaligned_t*)(dd - 4)) = *((const uint32_unaligned_t*)(ss - 4)); + break; - case 76: - memcpy_sse2_64(dd - 76, ss - 76); + case 76: + memcpy_sse2_64(dd - 76, ss - 76); [[fallthrough]]; - case 12: - *((uint64_unaligned_t*)(dd - 12)) = *((const uint64_unaligned_t*)(ss - 12)); - *((uint32_unaligned_t*)(dd - 4)) = *((const uint32_unaligned_t*)(ss - 4)); - break; + case 12: + *((uint64_unaligned_t*)(dd - 12)) = *((const uint64_unaligned_t*)(ss - 12)); + *((uint32_unaligned_t*)(dd - 4)) = *((const uint32_unaligned_t*)(ss - 4)); + break; - case 77: - memcpy_sse2_64(dd - 77, ss - 77); + case 77: + memcpy_sse2_64(dd - 77, ss - 77); [[fallthrough]]; - case 13: - *((uint64_unaligned_t*)(dd - 13)) = *((const uint64_unaligned_t*)(ss - 13)); - *((uint32_unaligned_t*)(dd - 5)) = *((const uint32_unaligned_t*)(ss - 5)); - dd[-1] = ss[-1]; - break; + case 13: + *((uint64_unaligned_t*)(dd - 13)) = *((const uint64_unaligned_t*)(ss - 13)); + *((uint32_unaligned_t*)(dd - 5)) = *((const uint32_unaligned_t*)(ss - 5)); + dd[-1] = ss[-1]; + break; - case 78: - memcpy_sse2_64(dd - 78, ss - 78); + case 78: + memcpy_sse2_64(dd - 78, ss - 78); [[fallthrough]]; - case 14: - *((uint64_unaligned_t*)(dd - 14)) = *((const uint64_unaligned_t*)(ss - 14)); - *((uint64_unaligned_t*)(dd - 8)) = *((const uint64_unaligned_t*)(ss - 8)); - break; + case 14: + *((uint64_unaligned_t*)(dd - 14)) = *((const uint64_unaligned_t*)(ss - 14)); + *((uint64_unaligned_t*)(dd - 8)) = *((const uint64_unaligned_t*)(ss - 8)); + break; - case 79: - memcpy_sse2_64(dd - 79, ss - 79); + case 79: + memcpy_sse2_64(dd - 79, ss - 79); [[fallthrough]]; - case 15: - *((uint64_unaligned_t*)(dd - 15)) = *((const uint64_unaligned_t*)(ss - 15)); - *((uint64_unaligned_t*)(dd - 8)) = *((const uint64_unaligned_t*)(ss - 8)); - break; + case 15: + *((uint64_unaligned_t*)(dd - 15)) = *((const uint64_unaligned_t*)(ss - 15)); + *((uint64_unaligned_t*)(dd - 8)) = *((const uint64_unaligned_t*)(ss - 8)); + break; - case 80: - memcpy_sse2_64(dd - 80, ss - 80); + case 80: + memcpy_sse2_64(dd - 80, ss - 80); [[fallthrough]]; - case 16: - memcpy_sse2_16(dd - 16, ss - 16); - break; + case 16: + memcpy_sse2_16(dd - 16, ss - 16); + break; - case 81: - memcpy_sse2_64(dd - 81, ss - 81); + case 81: + memcpy_sse2_64(dd - 81, ss - 81); [[fallthrough]]; - case 17: - memcpy_sse2_16(dd - 17, ss - 17); - dd[-1] = ss[-1]; - break; + case 17: + memcpy_sse2_16(dd - 17, ss - 17); + dd[-1] = ss[-1]; + break; - case 82: - memcpy_sse2_64(dd - 82, ss - 82); + case 82: + memcpy_sse2_64(dd - 82, ss - 82); [[fallthrough]]; - case 18: - memcpy_sse2_16(dd - 18, ss - 18); - *((uint16_unaligned_t*)(dd - 2)) = *((const uint16_unaligned_t*)(ss - 2)); - break; + case 18: + memcpy_sse2_16(dd - 18, ss - 18); + *((uint16_unaligned_t*)(dd - 2)) = *((const uint16_unaligned_t*)(ss - 2)); + break; - case 83: - memcpy_sse2_64(dd - 83, ss - 83); + case 83: + memcpy_sse2_64(dd - 83, ss - 83); [[fallthrough]]; - case 19: - memcpy_sse2_16(dd - 19, ss - 19); - *((uint16_unaligned_t*)(dd - 3)) = *((const uint16_unaligned_t*)(ss - 3)); - dd[-1] = ss[-1]; - break; + case 19: + memcpy_sse2_16(dd - 19, ss - 19); + *((uint16_unaligned_t*)(dd - 3)) = *((const uint16_unaligned_t*)(ss - 3)); + dd[-1] = ss[-1]; + break; - case 84: - memcpy_sse2_64(dd - 84, ss - 84); + case 84: + memcpy_sse2_64(dd - 84, ss - 84); [[fallthrough]]; - case 20: - memcpy_sse2_16(dd - 20, ss - 20); - *((uint32_unaligned_t*)(dd - 4)) = *((const uint32_unaligned_t*)(ss - 4)); - break; + case 20: + memcpy_sse2_16(dd - 20, ss - 20); + *((uint32_unaligned_t*)(dd - 4)) = *((const uint32_unaligned_t*)(ss - 4)); + break; - case 85: - memcpy_sse2_64(dd - 85, ss - 85); + case 85: + memcpy_sse2_64(dd - 85, ss - 85); [[fallthrough]]; - case 21: - memcpy_sse2_16(dd - 21, ss - 21); - *((uint32_unaligned_t*)(dd - 5)) = *((const uint32_unaligned_t*)(ss - 5)); - dd[-1] = ss[-1]; - break; + case 21: + memcpy_sse2_16(dd - 21, ss - 21); + *((uint32_unaligned_t*)(dd - 5)) = *((const uint32_unaligned_t*)(ss - 5)); + dd[-1] = ss[-1]; + break; - case 86: - memcpy_sse2_64(dd - 86, ss - 86); + case 86: + memcpy_sse2_64(dd - 86, ss - 86); [[fallthrough]]; - case 22: - memcpy_sse2_16(dd - 22, ss - 22); - *((uint32_unaligned_t*)(dd - 6)) = *((const uint32_unaligned_t*)(ss - 6)); - *((uint16_unaligned_t*)(dd - 2)) = *((const uint16_unaligned_t*)(ss - 2)); - break; + case 22: + memcpy_sse2_16(dd - 22, ss - 22); + *((uint32_unaligned_t*)(dd - 6)) = *((const uint32_unaligned_t*)(ss - 6)); + *((uint16_unaligned_t*)(dd - 2)) = *((const uint16_unaligned_t*)(ss - 2)); + break; - case 87: - memcpy_sse2_64(dd - 87, ss - 87); + case 87: + memcpy_sse2_64(dd - 87, ss - 87); [[fallthrough]]; - case 23: - memcpy_sse2_16(dd - 23, ss - 23); - *((uint32_unaligned_t*)(dd - 7)) = *((const uint32_unaligned_t*)(ss - 7)); - *((uint32_unaligned_t*)(dd - 4)) = *((const uint32_unaligned_t*)(ss - 4)); - break; + case 23: + memcpy_sse2_16(dd - 23, ss - 23); + *((uint32_unaligned_t*)(dd - 7)) = *((const uint32_unaligned_t*)(ss - 7)); + *((uint32_unaligned_t*)(dd - 4)) = *((const uint32_unaligned_t*)(ss - 4)); + break; - case 88: - memcpy_sse2_64(dd - 88, ss - 88); + case 88: + memcpy_sse2_64(dd - 88, ss - 88); [[fallthrough]]; - case 24: - memcpy_sse2_16(dd - 24, ss - 24); - memcpy_sse2_16(dd - 16, ss - 16); - break; + case 24: + memcpy_sse2_16(dd - 24, ss - 24); + memcpy_sse2_16(dd - 16, ss - 16); + break; - case 89: - memcpy_sse2_64(dd - 89, ss - 89); + case 89: + memcpy_sse2_64(dd - 89, ss - 89); [[fallthrough]]; - case 25: - memcpy_sse2_16(dd - 25, ss - 25); - memcpy_sse2_16(dd - 16, ss - 16); - break; + case 25: + memcpy_sse2_16(dd - 25, ss - 25); + memcpy_sse2_16(dd - 16, ss - 16); + break; - case 90: - memcpy_sse2_64(dd - 90, ss - 90); + case 90: + memcpy_sse2_64(dd - 90, ss - 90); [[fallthrough]]; - case 26: - memcpy_sse2_16(dd - 26, ss - 26); - memcpy_sse2_16(dd - 16, ss - 16); - break; + case 26: + memcpy_sse2_16(dd - 26, ss - 26); + memcpy_sse2_16(dd - 16, ss - 16); + break; - case 91: - memcpy_sse2_64(dd - 91, ss - 91); + case 91: + memcpy_sse2_64(dd - 91, ss - 91); [[fallthrough]]; - case 27: - memcpy_sse2_16(dd - 27, ss - 27); - memcpy_sse2_16(dd - 16, ss - 16); - break; + case 27: + memcpy_sse2_16(dd - 27, ss - 27); + memcpy_sse2_16(dd - 16, ss - 16); + break; - case 92: - memcpy_sse2_64(dd - 92, ss - 92); + case 92: + memcpy_sse2_64(dd - 92, ss - 92); [[fallthrough]]; - case 28: - memcpy_sse2_16(dd - 28, ss - 28); - memcpy_sse2_16(dd - 16, ss - 16); - break; + case 28: + memcpy_sse2_16(dd - 28, ss - 28); + memcpy_sse2_16(dd - 16, ss - 16); + break; - case 93: - memcpy_sse2_64(dd - 93, ss - 93); + case 93: + memcpy_sse2_64(dd - 93, ss - 93); [[fallthrough]]; - case 29: - memcpy_sse2_16(dd - 29, ss - 29); - memcpy_sse2_16(dd - 16, ss - 16); - break; + case 29: + memcpy_sse2_16(dd - 29, ss - 29); + memcpy_sse2_16(dd - 16, ss - 16); + break; - case 94: - memcpy_sse2_64(dd - 94, ss - 94); + case 94: + memcpy_sse2_64(dd - 94, ss - 94); [[fallthrough]]; - case 30: - memcpy_sse2_16(dd - 30, ss - 30); - memcpy_sse2_16(dd - 16, ss - 16); - break; + case 30: + memcpy_sse2_16(dd - 30, ss - 30); + memcpy_sse2_16(dd - 16, ss - 16); + break; - case 95: - memcpy_sse2_64(dd - 95, ss - 95); + case 95: + memcpy_sse2_64(dd - 95, ss - 95); [[fallthrough]]; - case 31: - memcpy_sse2_16(dd - 31, ss - 31); - memcpy_sse2_16(dd - 16, ss - 16); - break; + case 31: + memcpy_sse2_16(dd - 31, ss - 31); + memcpy_sse2_16(dd - 16, ss - 16); + break; - case 96: - memcpy_sse2_64(dd - 96, ss - 96); + case 96: + memcpy_sse2_64(dd - 96, ss - 96); [[fallthrough]]; - case 32: - memcpy_sse2_32(dd - 32, ss - 32); - break; + case 32: + memcpy_sse2_32(dd - 32, ss - 32); + break; - case 97: - memcpy_sse2_64(dd - 97, ss - 97); + case 97: + memcpy_sse2_64(dd - 97, ss - 97); [[fallthrough]]; - case 33: - memcpy_sse2_32(dd - 33, ss - 33); - dd[-1] = ss[-1]; - break; + case 33: + memcpy_sse2_32(dd - 33, ss - 33); + dd[-1] = ss[-1]; + break; - case 98: - memcpy_sse2_64(dd - 98, ss - 98); + case 98: + memcpy_sse2_64(dd - 98, ss - 98); [[fallthrough]]; - case 34: - memcpy_sse2_32(dd - 34, ss - 34); - *((uint16_unaligned_t*)(dd - 2)) = *((const uint16_unaligned_t*)(ss - 2)); - break; + case 34: + memcpy_sse2_32(dd - 34, ss - 34); + *((uint16_unaligned_t*)(dd - 2)) = *((const uint16_unaligned_t*)(ss - 2)); + break; - case 99: - memcpy_sse2_64(dd - 99, ss - 99); + case 99: + memcpy_sse2_64(dd - 99, ss - 99); [[fallthrough]]; - case 35: - memcpy_sse2_32(dd - 35, ss - 35); - *((uint16_unaligned_t*)(dd - 3)) = *((const uint16_unaligned_t*)(ss - 3)); - dd[-1] = ss[-1]; - break; + case 35: + memcpy_sse2_32(dd - 35, ss - 35); + *((uint16_unaligned_t*)(dd - 3)) = *((const uint16_unaligned_t*)(ss - 3)); + dd[-1] = ss[-1]; + break; - case 100: - memcpy_sse2_64(dd - 100, ss - 100); + case 100: + memcpy_sse2_64(dd - 100, ss - 100); [[fallthrough]]; - case 36: - memcpy_sse2_32(dd - 36, ss - 36); - *((uint32_unaligned_t*)(dd - 4)) = *((const uint32_unaligned_t*)(ss - 4)); - break; + case 36: + memcpy_sse2_32(dd - 36, ss - 36); + *((uint32_unaligned_t*)(dd - 4)) = *((const uint32_unaligned_t*)(ss - 4)); + break; - case 101: - memcpy_sse2_64(dd - 101, ss - 101); + case 101: + memcpy_sse2_64(dd - 101, ss - 101); [[fallthrough]]; - case 37: - memcpy_sse2_32(dd - 37, ss - 37); - *((uint32_unaligned_t*)(dd - 5)) = *((const uint32_unaligned_t*)(ss - 5)); - dd[-1] = ss[-1]; - break; + case 37: + memcpy_sse2_32(dd - 37, ss - 37); + *((uint32_unaligned_t*)(dd - 5)) = *((const uint32_unaligned_t*)(ss - 5)); + dd[-1] = ss[-1]; + break; - case 102: - memcpy_sse2_64(dd - 102, ss - 102); + case 102: + memcpy_sse2_64(dd - 102, ss - 102); [[fallthrough]]; - case 38: - memcpy_sse2_32(dd - 38, ss - 38); - *((uint32_unaligned_t*)(dd - 6)) = *((const uint32_unaligned_t*)(ss - 6)); - *((uint16_unaligned_t*)(dd - 2)) = *((const uint16_unaligned_t*)(ss - 2)); - break; + case 38: + memcpy_sse2_32(dd - 38, ss - 38); + *((uint32_unaligned_t*)(dd - 6)) = *((const uint32_unaligned_t*)(ss - 6)); + *((uint16_unaligned_t*)(dd - 2)) = *((const uint16_unaligned_t*)(ss - 2)); + break; - case 103: - memcpy_sse2_64(dd - 103, ss - 103); + case 103: + memcpy_sse2_64(dd - 103, ss - 103); [[fallthrough]]; - case 39: - memcpy_sse2_32(dd - 39, ss - 39); - *((uint32_unaligned_t*)(dd - 7)) = *((const uint32_unaligned_t*)(ss - 7)); - *((uint32_unaligned_t*)(dd - 4)) = *((const uint32_unaligned_t*)(ss - 4)); - break; + case 39: + memcpy_sse2_32(dd - 39, ss - 39); + *((uint32_unaligned_t*)(dd - 7)) = *((const uint32_unaligned_t*)(ss - 7)); + *((uint32_unaligned_t*)(dd - 4)) = *((const uint32_unaligned_t*)(ss - 4)); + break; - case 104: - memcpy_sse2_64(dd - 104, ss - 104); + case 104: + memcpy_sse2_64(dd - 104, ss - 104); [[fallthrough]]; - case 40: - memcpy_sse2_32(dd - 40, ss - 40); - *((uint64_unaligned_t*)(dd - 8)) = *((const uint64_unaligned_t*)(ss - 8)); - break; + case 40: + memcpy_sse2_32(dd - 40, ss - 40); + *((uint64_unaligned_t*)(dd - 8)) = *((const uint64_unaligned_t*)(ss - 8)); + break; - case 105: - memcpy_sse2_64(dd - 105, ss - 105); + case 105: + memcpy_sse2_64(dd - 105, ss - 105); [[fallthrough]]; - case 41: - memcpy_sse2_32(dd - 41, ss - 41); - *((uint64_unaligned_t*)(dd - 9)) = *((const uint64_unaligned_t*)(ss - 9)); - dd[-1] = ss[-1]; - break; + case 41: + memcpy_sse2_32(dd - 41, ss - 41); + *((uint64_unaligned_t*)(dd - 9)) = *((const uint64_unaligned_t*)(ss - 9)); + dd[-1] = ss[-1]; + break; - case 106: - memcpy_sse2_64(dd - 106, ss - 106); + case 106: + memcpy_sse2_64(dd - 106, ss - 106); [[fallthrough]]; - case 42: - memcpy_sse2_32(dd - 42, ss - 42); - *((uint64_unaligned_t*)(dd - 10)) = *((const uint64_unaligned_t*)(ss - 10)); - *((uint16_unaligned_t*)(dd - 2)) = *((const uint16_unaligned_t*)(ss - 2)); - break; + case 42: + memcpy_sse2_32(dd - 42, ss - 42); + *((uint64_unaligned_t*)(dd - 10)) = *((const uint64_unaligned_t*)(ss - 10)); + *((uint16_unaligned_t*)(dd - 2)) = *((const uint16_unaligned_t*)(ss - 2)); + break; - case 107: - memcpy_sse2_64(dd - 107, ss - 107); + case 107: + memcpy_sse2_64(dd - 107, ss - 107); [[fallthrough]]; - case 43: - memcpy_sse2_32(dd - 43, ss - 43); - *((uint64_unaligned_t*)(dd - 11)) = *((const uint64_unaligned_t*)(ss - 11)); - *((uint32_unaligned_t*)(dd - 4)) = *((const uint32_unaligned_t*)(ss - 4)); - break; + case 43: + memcpy_sse2_32(dd - 43, ss - 43); + *((uint64_unaligned_t*)(dd - 11)) = *((const uint64_unaligned_t*)(ss - 11)); + *((uint32_unaligned_t*)(dd - 4)) = *((const uint32_unaligned_t*)(ss - 4)); + break; - case 108: - memcpy_sse2_64(dd - 108, ss - 108); + case 108: + memcpy_sse2_64(dd - 108, ss - 108); [[fallthrough]]; - case 44: - memcpy_sse2_32(dd - 44, ss - 44); - *((uint64_unaligned_t*)(dd - 12)) = *((const uint64_unaligned_t*)(ss - 12)); - *((uint32_unaligned_t*)(dd - 4)) = *((const uint32_unaligned_t*)(ss - 4)); - break; + case 44: + memcpy_sse2_32(dd - 44, ss - 44); + *((uint64_unaligned_t*)(dd - 12)) = *((const uint64_unaligned_t*)(ss - 12)); + *((uint32_unaligned_t*)(dd - 4)) = *((const uint32_unaligned_t*)(ss - 4)); + break; - case 109: - memcpy_sse2_64(dd - 109, ss - 109); + case 109: + memcpy_sse2_64(dd - 109, ss - 109); [[fallthrough]]; - case 45: - memcpy_sse2_32(dd - 45, ss - 45); - *((uint64_unaligned_t*)(dd - 13)) = *((const uint64_unaligned_t*)(ss - 13)); - *((uint32_unaligned_t*)(dd - 5)) = *((const uint32_unaligned_t*)(ss - 5)); - dd[-1] = ss[-1]; - break; + case 45: + memcpy_sse2_32(dd - 45, ss - 45); + *((uint64_unaligned_t*)(dd - 13)) = *((const uint64_unaligned_t*)(ss - 13)); + *((uint32_unaligned_t*)(dd - 5)) = *((const uint32_unaligned_t*)(ss - 5)); + dd[-1] = ss[-1]; + break; - case 110: - memcpy_sse2_64(dd - 110, ss - 110); + case 110: + memcpy_sse2_64(dd - 110, ss - 110); [[fallthrough]]; - case 46: - memcpy_sse2_32(dd - 46, ss - 46); - *((uint64_unaligned_t*)(dd - 14)) = *((const uint64_unaligned_t*)(ss - 14)); - *((uint64_unaligned_t*)(dd - 8)) = *((const uint64_unaligned_t*)(ss - 8)); - break; + case 46: + memcpy_sse2_32(dd - 46, ss - 46); + *((uint64_unaligned_t*)(dd - 14)) = *((const uint64_unaligned_t*)(ss - 14)); + *((uint64_unaligned_t*)(dd - 8)) = *((const uint64_unaligned_t*)(ss - 8)); + break; - case 111: - memcpy_sse2_64(dd - 111, ss - 111); + case 111: + memcpy_sse2_64(dd - 111, ss - 111); [[fallthrough]]; - case 47: - memcpy_sse2_32(dd - 47, ss - 47); - *((uint64_unaligned_t*)(dd - 15)) = *((const uint64_unaligned_t*)(ss - 15)); - *((uint64_unaligned_t*)(dd - 8)) = *((const uint64_unaligned_t*)(ss - 8)); - break; + case 47: + memcpy_sse2_32(dd - 47, ss - 47); + *((uint64_unaligned_t*)(dd - 15)) = *((const uint64_unaligned_t*)(ss - 15)); + *((uint64_unaligned_t*)(dd - 8)) = *((const uint64_unaligned_t*)(ss - 8)); + break; - case 112: - memcpy_sse2_64(dd - 112, ss - 112); + case 112: + memcpy_sse2_64(dd - 112, ss - 112); [[fallthrough]]; - case 48: - memcpy_sse2_32(dd - 48, ss - 48); - memcpy_sse2_16(dd - 16, ss - 16); - break; + case 48: + memcpy_sse2_32(dd - 48, ss - 48); + memcpy_sse2_16(dd - 16, ss - 16); + break; - case 113: - memcpy_sse2_64(dd - 113, ss - 113); + case 113: + memcpy_sse2_64(dd - 113, ss - 113); [[fallthrough]]; - case 49: - memcpy_sse2_32(dd - 49, ss - 49); - memcpy_sse2_16(dd - 17, ss - 17); - dd[-1] = ss[-1]; - break; + case 49: + memcpy_sse2_32(dd - 49, ss - 49); + memcpy_sse2_16(dd - 17, ss - 17); + dd[-1] = ss[-1]; + break; - case 114: - memcpy_sse2_64(dd - 114, ss - 114); + case 114: + memcpy_sse2_64(dd - 114, ss - 114); [[fallthrough]]; - case 50: - memcpy_sse2_32(dd - 50, ss - 50); - memcpy_sse2_16(dd - 18, ss - 18); - *((uint16_unaligned_t*)(dd - 2)) = *((const uint16_unaligned_t*)(ss - 2)); - break; + case 50: + memcpy_sse2_32(dd - 50, ss - 50); + memcpy_sse2_16(dd - 18, ss - 18); + *((uint16_unaligned_t*)(dd - 2)) = *((const uint16_unaligned_t*)(ss - 2)); + break; - case 115: - memcpy_sse2_64(dd - 115, ss - 115); + case 115: + memcpy_sse2_64(dd - 115, ss - 115); [[fallthrough]]; - case 51: - memcpy_sse2_32(dd - 51, ss - 51); - memcpy_sse2_16(dd - 19, ss - 19); - *((uint16_unaligned_t*)(dd - 3)) = *((const uint16_unaligned_t*)(ss - 3)); - dd[-1] = ss[-1]; - break; + case 51: + memcpy_sse2_32(dd - 51, ss - 51); + memcpy_sse2_16(dd - 19, ss - 19); + *((uint16_unaligned_t*)(dd - 3)) = *((const uint16_unaligned_t*)(ss - 3)); + dd[-1] = ss[-1]; + break; - case 116: - memcpy_sse2_64(dd - 116, ss - 116); + case 116: + memcpy_sse2_64(dd - 116, ss - 116); [[fallthrough]]; - case 52: - memcpy_sse2_32(dd - 52, ss - 52); - memcpy_sse2_16(dd - 20, ss - 20); - *((uint32_unaligned_t*)(dd - 4)) = *((const uint32_unaligned_t*)(ss - 4)); - break; + case 52: + memcpy_sse2_32(dd - 52, ss - 52); + memcpy_sse2_16(dd - 20, ss - 20); + *((uint32_unaligned_t*)(dd - 4)) = *((const uint32_unaligned_t*)(ss - 4)); + break; - case 117: - memcpy_sse2_64(dd - 117, ss - 117); + case 117: + memcpy_sse2_64(dd - 117, ss - 117); [[fallthrough]]; - case 53: - memcpy_sse2_32(dd - 53, ss - 53); - memcpy_sse2_16(dd - 21, ss - 21); - *((uint32_unaligned_t*)(dd - 5)) = *((const uint32_unaligned_t*)(ss - 5)); - dd[-1] = ss[-1]; - break; + case 53: + memcpy_sse2_32(dd - 53, ss - 53); + memcpy_sse2_16(dd - 21, ss - 21); + *((uint32_unaligned_t*)(dd - 5)) = *((const uint32_unaligned_t*)(ss - 5)); + dd[-1] = ss[-1]; + break; - case 118: - memcpy_sse2_64(dd - 118, ss - 118); + case 118: + memcpy_sse2_64(dd - 118, ss - 118); [[fallthrough]]; - case 54: - memcpy_sse2_32(dd - 54, ss - 54); - memcpy_sse2_16(dd - 22, ss - 22); - *((uint32_unaligned_t*)(dd - 6)) = *((const uint32_unaligned_t*)(ss - 6)); - *((uint16_unaligned_t*)(dd - 2)) = *((const uint16_unaligned_t*)(ss - 2)); - break; + case 54: + memcpy_sse2_32(dd - 54, ss - 54); + memcpy_sse2_16(dd - 22, ss - 22); + *((uint32_unaligned_t*)(dd - 6)) = *((const uint32_unaligned_t*)(ss - 6)); + *((uint16_unaligned_t*)(dd - 2)) = *((const uint16_unaligned_t*)(ss - 2)); + break; - case 119: - memcpy_sse2_64(dd - 119, ss - 119); + case 119: + memcpy_sse2_64(dd - 119, ss - 119); [[fallthrough]]; - case 55: - memcpy_sse2_32(dd - 55, ss - 55); - memcpy_sse2_16(dd - 23, ss - 23); - *((uint32_unaligned_t*)(dd - 7)) = *((const uint32_unaligned_t*)(ss - 7)); - *((uint32_unaligned_t*)(dd - 4)) = *((const uint32_unaligned_t*)(ss - 4)); - break; + case 55: + memcpy_sse2_32(dd - 55, ss - 55); + memcpy_sse2_16(dd - 23, ss - 23); + *((uint32_unaligned_t*)(dd - 7)) = *((const uint32_unaligned_t*)(ss - 7)); + *((uint32_unaligned_t*)(dd - 4)) = *((const uint32_unaligned_t*)(ss - 4)); + break; - case 120: - memcpy_sse2_64(dd - 120, ss - 120); + case 120: + memcpy_sse2_64(dd - 120, ss - 120); [[fallthrough]]; - case 56: - memcpy_sse2_32(dd - 56, ss - 56); - memcpy_sse2_16(dd - 24, ss - 24); - memcpy_sse2_16(dd - 16, ss - 16); - break; + case 56: + memcpy_sse2_32(dd - 56, ss - 56); + memcpy_sse2_16(dd - 24, ss - 24); + memcpy_sse2_16(dd - 16, ss - 16); + break; - case 121: - memcpy_sse2_64(dd - 121, ss - 121); + case 121: + memcpy_sse2_64(dd - 121, ss - 121); [[fallthrough]]; - case 57: - memcpy_sse2_32(dd - 57, ss - 57); - memcpy_sse2_16(dd - 25, ss - 25); - memcpy_sse2_16(dd - 16, ss - 16); - break; + case 57: + memcpy_sse2_32(dd - 57, ss - 57); + memcpy_sse2_16(dd - 25, ss - 25); + memcpy_sse2_16(dd - 16, ss - 16); + break; - case 122: - memcpy_sse2_64(dd - 122, ss - 122); + case 122: + memcpy_sse2_64(dd - 122, ss - 122); [[fallthrough]]; - case 58: - memcpy_sse2_32(dd - 58, ss - 58); - memcpy_sse2_16(dd - 26, ss - 26); - memcpy_sse2_16(dd - 16, ss - 16); - break; + case 58: + memcpy_sse2_32(dd - 58, ss - 58); + memcpy_sse2_16(dd - 26, ss - 26); + memcpy_sse2_16(dd - 16, ss - 16); + break; - case 123: - memcpy_sse2_64(dd - 123, ss - 123); + case 123: + memcpy_sse2_64(dd - 123, ss - 123); [[fallthrough]]; - case 59: - memcpy_sse2_32(dd - 59, ss - 59); - memcpy_sse2_16(dd - 27, ss - 27); - memcpy_sse2_16(dd - 16, ss - 16); - break; + case 59: + memcpy_sse2_32(dd - 59, ss - 59); + memcpy_sse2_16(dd - 27, ss - 27); + memcpy_sse2_16(dd - 16, ss - 16); + break; - case 124: - memcpy_sse2_64(dd - 124, ss - 124); + case 124: + memcpy_sse2_64(dd - 124, ss - 124); [[fallthrough]]; - case 60: - memcpy_sse2_32(dd - 60, ss - 60); - memcpy_sse2_16(dd - 28, ss - 28); - memcpy_sse2_16(dd - 16, ss - 16); - break; + case 60: + memcpy_sse2_32(dd - 60, ss - 60); + memcpy_sse2_16(dd - 28, ss - 28); + memcpy_sse2_16(dd - 16, ss - 16); + break; - case 125: - memcpy_sse2_64(dd - 125, ss - 125); + case 125: + memcpy_sse2_64(dd - 125, ss - 125); [[fallthrough]]; - case 61: - memcpy_sse2_32(dd - 61, ss - 61); - memcpy_sse2_16(dd - 29, ss - 29); - memcpy_sse2_16(dd - 16, ss - 16); - break; + case 61: + memcpy_sse2_32(dd - 61, ss - 61); + memcpy_sse2_16(dd - 29, ss - 29); + memcpy_sse2_16(dd - 16, ss - 16); + break; - case 126: - memcpy_sse2_64(dd - 126, ss - 126); + case 126: + memcpy_sse2_64(dd - 126, ss - 126); [[fallthrough]]; - case 62: - memcpy_sse2_32(dd - 62, ss - 62); - memcpy_sse2_16(dd - 30, ss - 30); - memcpy_sse2_16(dd - 16, ss - 16); - break; + case 62: + memcpy_sse2_32(dd - 62, ss - 62); + memcpy_sse2_16(dd - 30, ss - 30); + memcpy_sse2_16(dd - 16, ss - 16); + break; - case 127: - memcpy_sse2_64(dd - 127, ss - 127); + case 127: + memcpy_sse2_64(dd - 127, ss - 127); [[fallthrough]]; - case 63: - memcpy_sse2_32(dd - 63, ss - 63); - memcpy_sse2_16(dd - 31, ss - 31); - memcpy_sse2_16(dd - 16, ss - 16); - break; + case 63: + memcpy_sse2_32(dd - 63, ss - 63); + memcpy_sse2_16(dd - 31, ss - 31); + memcpy_sse2_16(dd - 16, ss - 16); + break; - case 128: - memcpy_sse2_128(dd - 128, ss - 128); - break; - } + case 128: + memcpy_sse2_128(dd - 128, ss - 128); + break; + } - return dst; + return dst; } //--------------------------------------------------------------------- // main routine //--------------------------------------------------------------------- -void* memcpy_fast(void *destination, const void *source, size_t size) +void* memcpy_fast_sse(void * __restrict destination, const void * __restrict source, size_t size) { - unsigned char *dst = (unsigned char*)destination; - const unsigned char *src = (const unsigned char*)source; - static size_t cachesize = 0x200000; // L2-cache size - size_t padding; + unsigned char *dst = (unsigned char*)destination; + const unsigned char *src = (const unsigned char*)source; + static size_t cachesize = 0x200000; // L2-cache size + size_t padding; - // small memory copy - if (size <= 128) { - return memcpy_tiny(dst, src, size); - } + // small memory copy + if (size <= 128) { + return memcpy_tiny(dst, src, size); + } - // align destination to 16 bytes boundary - padding = (16 - (((size_t)dst) & 15)) & 15; + // align destination to 16 bytes boundary + padding = (16 - (((size_t)dst) & 15)) & 15; - if (padding > 0) { - __m128i head = _mm_loadu_si128(reinterpret_cast(src)); - _mm_storeu_si128(reinterpret_cast<__m128i*>(dst), head); - dst += padding; - src += padding; - size -= padding; - } + if (padding > 0) { + __m128i head = _mm_loadu_si128(reinterpret_cast(src)); + _mm_storeu_si128(reinterpret_cast<__m128i*>(dst), head); + dst += padding; + src += padding; + size -= padding; + } - // medium size copy - if (size <= cachesize) { - __m128i c0, c1, c2, c3, c4, c5, c6, c7; + // medium size copy + if (size <= cachesize) { + __m128i c0, c1, c2, c3, c4, c5, c6, c7; - for (; size >= 128; size -= 128) { - c0 = _mm_loadu_si128((reinterpret_cast(src)) + 0); - c1 = _mm_loadu_si128((reinterpret_cast(src)) + 1); - c2 = _mm_loadu_si128((reinterpret_cast(src)) + 2); - c3 = _mm_loadu_si128((reinterpret_cast(src)) + 3); - c4 = _mm_loadu_si128((reinterpret_cast(src)) + 4); - c5 = _mm_loadu_si128((reinterpret_cast(src)) + 5); - c6 = _mm_loadu_si128((reinterpret_cast(src)) + 6); - c7 = _mm_loadu_si128((reinterpret_cast(src)) + 7); - _mm_prefetch((const char*)(src + 256), _MM_HINT_NTA); - src += 128; - _mm_store_si128(((reinterpret_cast<__m128i*>(dst)) + 0), c0); - _mm_store_si128(((reinterpret_cast<__m128i*>(dst)) + 1), c1); - _mm_store_si128(((reinterpret_cast<__m128i*>(dst)) + 2), c2); - _mm_store_si128(((reinterpret_cast<__m128i*>(dst)) + 3), c3); - _mm_store_si128(((reinterpret_cast<__m128i*>(dst)) + 4), c4); - _mm_store_si128(((reinterpret_cast<__m128i*>(dst)) + 5), c5); - _mm_store_si128(((reinterpret_cast<__m128i*>(dst)) + 6), c6); - _mm_store_si128(((reinterpret_cast<__m128i*>(dst)) + 7), c7); - dst += 128; - } - } - else { // big memory copy - __m128i c0, c1, c2, c3, c4, c5, c6, c7; + for (; size >= 128; size -= 128) { + c0 = _mm_loadu_si128((reinterpret_cast(src)) + 0); + c1 = _mm_loadu_si128((reinterpret_cast(src)) + 1); + c2 = _mm_loadu_si128((reinterpret_cast(src)) + 2); + c3 = _mm_loadu_si128((reinterpret_cast(src)) + 3); + c4 = _mm_loadu_si128((reinterpret_cast(src)) + 4); + c5 = _mm_loadu_si128((reinterpret_cast(src)) + 5); + c6 = _mm_loadu_si128((reinterpret_cast(src)) + 6); + c7 = _mm_loadu_si128((reinterpret_cast(src)) + 7); + _mm_prefetch((const char*)(src + 256), _MM_HINT_NTA); + src += 128; + _mm_store_si128(((reinterpret_cast<__m128i*>(dst)) + 0), c0); + _mm_store_si128(((reinterpret_cast<__m128i*>(dst)) + 1), c1); + _mm_store_si128(((reinterpret_cast<__m128i*>(dst)) + 2), c2); + _mm_store_si128(((reinterpret_cast<__m128i*>(dst)) + 3), c3); + _mm_store_si128(((reinterpret_cast<__m128i*>(dst)) + 4), c4); + _mm_store_si128(((reinterpret_cast<__m128i*>(dst)) + 5), c5); + _mm_store_si128(((reinterpret_cast<__m128i*>(dst)) + 6), c6); + _mm_store_si128(((reinterpret_cast<__m128i*>(dst)) + 7), c7); + dst += 128; + } + } + else { // big memory copy + __m128i c0, c1, c2, c3, c4, c5, c6, c7; - _mm_prefetch((const char*)(src), _MM_HINT_NTA); + _mm_prefetch((const char*)(src), _MM_HINT_NTA); - if ((((size_t)src) & 15) == 0) { // source aligned - for (; size >= 128; size -= 128) { - c0 = _mm_load_si128((reinterpret_cast(src)) + 0); - c1 = _mm_load_si128((reinterpret_cast(src)) + 1); - c2 = _mm_load_si128((reinterpret_cast(src)) + 2); - c3 = _mm_load_si128((reinterpret_cast(src)) + 3); - c4 = _mm_load_si128((reinterpret_cast(src)) + 4); - c5 = _mm_load_si128((reinterpret_cast(src)) + 5); - c6 = _mm_load_si128((reinterpret_cast(src)) + 6); - c7 = _mm_load_si128((reinterpret_cast(src)) + 7); - _mm_prefetch((const char*)(src + 256), _MM_HINT_NTA); - src += 128; - _mm_stream_si128(((reinterpret_cast<__m128i*>(dst)) + 0), c0); - _mm_stream_si128(((reinterpret_cast<__m128i*>(dst)) + 1), c1); - _mm_stream_si128(((reinterpret_cast<__m128i*>(dst)) + 2), c2); - _mm_stream_si128(((reinterpret_cast<__m128i*>(dst)) + 3), c3); - _mm_stream_si128(((reinterpret_cast<__m128i*>(dst)) + 4), c4); - _mm_stream_si128(((reinterpret_cast<__m128i*>(dst)) + 5), c5); - _mm_stream_si128(((reinterpret_cast<__m128i*>(dst)) + 6), c6); - _mm_stream_si128(((reinterpret_cast<__m128i*>(dst)) + 7), c7); - dst += 128; - } - } - else { // source unaligned - for (; size >= 128; size -= 128) { - c0 = _mm_loadu_si128((reinterpret_cast(src)) + 0); - c1 = _mm_loadu_si128((reinterpret_cast(src)) + 1); - c2 = _mm_loadu_si128((reinterpret_cast(src)) + 2); - c3 = _mm_loadu_si128((reinterpret_cast(src)) + 3); - c4 = _mm_loadu_si128((reinterpret_cast(src)) + 4); - c5 = _mm_loadu_si128((reinterpret_cast(src)) + 5); - c6 = _mm_loadu_si128((reinterpret_cast(src)) + 6); - c7 = _mm_loadu_si128((reinterpret_cast(src)) + 7); - _mm_prefetch((const char*)(src + 256), _MM_HINT_NTA); - src += 128; - _mm_stream_si128(((reinterpret_cast<__m128i*>(dst)) + 0), c0); - _mm_stream_si128(((reinterpret_cast<__m128i*>(dst)) + 1), c1); - _mm_stream_si128(((reinterpret_cast<__m128i*>(dst)) + 2), c2); - _mm_stream_si128(((reinterpret_cast<__m128i*>(dst)) + 3), c3); - _mm_stream_si128(((reinterpret_cast<__m128i*>(dst)) + 4), c4); - _mm_stream_si128(((reinterpret_cast<__m128i*>(dst)) + 5), c5); - _mm_stream_si128(((reinterpret_cast<__m128i*>(dst)) + 6), c6); - _mm_stream_si128(((reinterpret_cast<__m128i*>(dst)) + 7), c7); - dst += 128; - } - } - _mm_sfence(); - } + if ((((size_t)src) & 15) == 0) { // source aligned + for (; size >= 128; size -= 128) { + c0 = _mm_load_si128((reinterpret_cast(src)) + 0); + c1 = _mm_load_si128((reinterpret_cast(src)) + 1); + c2 = _mm_load_si128((reinterpret_cast(src)) + 2); + c3 = _mm_load_si128((reinterpret_cast(src)) + 3); + c4 = _mm_load_si128((reinterpret_cast(src)) + 4); + c5 = _mm_load_si128((reinterpret_cast(src)) + 5); + c6 = _mm_load_si128((reinterpret_cast(src)) + 6); + c7 = _mm_load_si128((reinterpret_cast(src)) + 7); + _mm_prefetch((const char*)(src + 256), _MM_HINT_NTA); + src += 128; + _mm_stream_si128(((reinterpret_cast<__m128i*>(dst)) + 0), c0); + _mm_stream_si128(((reinterpret_cast<__m128i*>(dst)) + 1), c1); + _mm_stream_si128(((reinterpret_cast<__m128i*>(dst)) + 2), c2); + _mm_stream_si128(((reinterpret_cast<__m128i*>(dst)) + 3), c3); + _mm_stream_si128(((reinterpret_cast<__m128i*>(dst)) + 4), c4); + _mm_stream_si128(((reinterpret_cast<__m128i*>(dst)) + 5), c5); + _mm_stream_si128(((reinterpret_cast<__m128i*>(dst)) + 6), c6); + _mm_stream_si128(((reinterpret_cast<__m128i*>(dst)) + 7), c7); + dst += 128; + } + } + else { // source unaligned + for (; size >= 128; size -= 128) { + c0 = _mm_loadu_si128((reinterpret_cast(src)) + 0); + c1 = _mm_loadu_si128((reinterpret_cast(src)) + 1); + c2 = _mm_loadu_si128((reinterpret_cast(src)) + 2); + c3 = _mm_loadu_si128((reinterpret_cast(src)) + 3); + c4 = _mm_loadu_si128((reinterpret_cast(src)) + 4); + c5 = _mm_loadu_si128((reinterpret_cast(src)) + 5); + c6 = _mm_loadu_si128((reinterpret_cast(src)) + 6); + c7 = _mm_loadu_si128((reinterpret_cast(src)) + 7); + _mm_prefetch((const char*)(src + 256), _MM_HINT_NTA); + src += 128; + _mm_stream_si128(((reinterpret_cast<__m128i*>(dst)) + 0), c0); + _mm_stream_si128(((reinterpret_cast<__m128i*>(dst)) + 1), c1); + _mm_stream_si128(((reinterpret_cast<__m128i*>(dst)) + 2), c2); + _mm_stream_si128(((reinterpret_cast<__m128i*>(dst)) + 3), c3); + _mm_stream_si128(((reinterpret_cast<__m128i*>(dst)) + 4), c4); + _mm_stream_si128(((reinterpret_cast<__m128i*>(dst)) + 5), c5); + _mm_stream_si128(((reinterpret_cast<__m128i*>(dst)) + 6), c6); + _mm_stream_si128(((reinterpret_cast<__m128i*>(dst)) + 7), c7); + dst += 128; + } + } + _mm_sfence(); + } - memcpy_tiny(dst, src, size); + memcpy_tiny(dst, src, size); - return destination; + return destination; } diff --git a/utils/memcpy-bench/FastMemcpy_Avx.h b/utils/memcpy-bench/FastMemcpy_Avx.h new file mode 100644 index 00000000000..a36964eb013 --- /dev/null +++ b/utils/memcpy-bench/FastMemcpy_Avx.h @@ -0,0 +1,480 @@ +//===================================================================== +// +// FastMemcpy.c - skywind3000@163.com, 2015 +// +// feature: +// 50% speed up in avg. vs standard memcpy (tested in vc2012/gcc5.1) +// +//===================================================================== +#pragma once + +#include +#include +#include + + +//--------------------------------------------------------------------- +// force inline for compilers +//--------------------------------------------------------------------- +#ifndef INLINE +#ifdef __GNUC__ +#if (__GNUC__ > 3) || ((__GNUC__ == 3) && (__GNUC_MINOR__ >= 1)) + #define INLINE __inline__ __attribute__((always_inline)) +#else + #define INLINE __inline__ +#endif +#elif defined(_MSC_VER) + #define INLINE __forceinline +#elif (defined(__BORLANDC__) || defined(__WATCOMC__)) + #define INLINE __inline +#else + #define INLINE +#endif +#endif + + + +//--------------------------------------------------------------------- +// fast copy for different sizes +//--------------------------------------------------------------------- +static INLINE void memcpy_avx_16(void * __restrict dst, const void * __restrict src) { +#if 1 + __m128i m0 = _mm_loadu_si128(((const __m128i*)src) + 0); + _mm_storeu_si128(((__m128i*)dst) + 0, m0); +#else + *((uint64_t*)((char*)dst + 0)) = *((uint64_t*)((const char*)src + 0)); + *((uint64_t*)((char*)dst + 8)) = *((uint64_t*)((const char*)src + 8)); +#endif +} + +static INLINE void memcpy_avx_32(void *dst, const void *src) { + __m256i m0 = _mm256_loadu_si256((reinterpret_cast(src)) + 0); + _mm256_storeu_si256((reinterpret_cast<__m256i*>(dst)) + 0, m0); +} + +static INLINE void memcpy_avx_64(void *dst, const void *src) { + __m256i m0 = _mm256_loadu_si256((reinterpret_cast(src)) + 0); + __m256i m1 = _mm256_loadu_si256((reinterpret_cast(src)) + 1); + _mm256_storeu_si256((reinterpret_cast<__m256i*>(dst)) + 0, m0); + _mm256_storeu_si256((reinterpret_cast<__m256i*>(dst)) + 1, m1); +} + +static INLINE void memcpy_avx_128(void *dst, const void *src) { + __m256i m0 = _mm256_loadu_si256((reinterpret_cast(src)) + 0); + __m256i m1 = _mm256_loadu_si256((reinterpret_cast(src)) + 1); + __m256i m2 = _mm256_loadu_si256((reinterpret_cast(src)) + 2); + __m256i m3 = _mm256_loadu_si256((reinterpret_cast(src)) + 3); + _mm256_storeu_si256((reinterpret_cast<__m256i*>(dst)) + 0, m0); + _mm256_storeu_si256((reinterpret_cast<__m256i*>(dst)) + 1, m1); + _mm256_storeu_si256((reinterpret_cast<__m256i*>(dst)) + 2, m2); + _mm256_storeu_si256((reinterpret_cast<__m256i*>(dst)) + 3, m3); +} + +static INLINE void memcpy_avx_256(void *dst, const void *src) { + __m256i m0 = _mm256_loadu_si256((reinterpret_cast(src)) + 0); + __m256i m1 = _mm256_loadu_si256((reinterpret_cast(src)) + 1); + __m256i m2 = _mm256_loadu_si256((reinterpret_cast(src)) + 2); + __m256i m3 = _mm256_loadu_si256((reinterpret_cast(src)) + 3); + __m256i m4 = _mm256_loadu_si256((reinterpret_cast(src)) + 4); + __m256i m5 = _mm256_loadu_si256((reinterpret_cast(src)) + 5); + __m256i m6 = _mm256_loadu_si256((reinterpret_cast(src)) + 6); + __m256i m7 = _mm256_loadu_si256((reinterpret_cast(src)) + 7); + _mm256_storeu_si256((reinterpret_cast<__m256i*>(dst)) + 0, m0); + _mm256_storeu_si256((reinterpret_cast<__m256i*>(dst)) + 1, m1); + _mm256_storeu_si256((reinterpret_cast<__m256i*>(dst)) + 2, m2); + _mm256_storeu_si256((reinterpret_cast<__m256i*>(dst)) + 3, m3); + _mm256_storeu_si256((reinterpret_cast<__m256i*>(dst)) + 4, m4); + _mm256_storeu_si256((reinterpret_cast<__m256i*>(dst)) + 5, m5); + _mm256_storeu_si256((reinterpret_cast<__m256i*>(dst)) + 6, m6); + _mm256_storeu_si256((reinterpret_cast<__m256i*>(dst)) + 7, m7); +} + + +//--------------------------------------------------------------------- +// tiny memory copy with jump table optimized +//--------------------------------------------------------------------- +static INLINE void *memcpy_tiny_avx(void * __restrict dst, const void * __restrict src, size_t size) { + unsigned char *dd = reinterpret_cast(dst) + size; + const unsigned char *ss = reinterpret_cast(src) + size; + + switch (size) { + case 128: memcpy_avx_128(dd - 128, ss - 128); [[fallthrough]]; + case 0: break; + case 129: memcpy_avx_128(dd - 129, ss - 129); [[fallthrough]]; + case 1: dd[-1] = ss[-1]; break; + case 130: memcpy_avx_128(dd - 130, ss - 130); [[fallthrough]]; + case 2: *((uint16_t*)(dd - 2)) = *((uint16_t*)(ss - 2)); break; + case 131: memcpy_avx_128(dd - 131, ss - 131); [[fallthrough]]; + case 3: *((uint16_t*)(dd - 3)) = *((uint16_t*)(ss - 3)); dd[-1] = ss[-1]; break; + case 132: memcpy_avx_128(dd - 132, ss - 132); [[fallthrough]]; + case 4: *((uint32_t*)(dd - 4)) = *((uint32_t*)(ss - 4)); break; + case 133: memcpy_avx_128(dd - 133, ss - 133); [[fallthrough]]; + case 5: *((uint32_t*)(dd - 5)) = *((uint32_t*)(ss - 5)); dd[-1] = ss[-1]; break; + case 134: memcpy_avx_128(dd - 134, ss - 134); [[fallthrough]]; + case 6: *((uint32_t*)(dd - 6)) = *((uint32_t*)(ss - 6)); *((uint16_t*)(dd - 2)) = *((uint16_t*)(ss - 2)); break; + case 135: memcpy_avx_128(dd - 135, ss - 135); [[fallthrough]]; + case 7: *((uint32_t*)(dd - 7)) = *((uint32_t*)(ss - 7)); *((uint32_t*)(dd - 4)) = *((uint32_t*)(ss - 4)); break; + case 136: memcpy_avx_128(dd - 136, ss - 136); [[fallthrough]]; + case 8: *((uint64_t*)(dd - 8)) = *((uint64_t*)(ss - 8)); break; + case 137: memcpy_avx_128(dd - 137, ss - 137); [[fallthrough]]; + case 9: *((uint64_t*)(dd - 9)) = *((uint64_t*)(ss - 9)); dd[-1] = ss[-1]; break; + case 138: memcpy_avx_128(dd - 138, ss - 138); [[fallthrough]]; + case 10: *((uint64_t*)(dd - 10)) = *((uint64_t*)(ss - 10)); *((uint16_t*)(dd - 2)) = *((uint16_t*)(ss - 2)); break; + case 139: memcpy_avx_128(dd - 139, ss - 139); [[fallthrough]]; + case 11: *((uint64_t*)(dd - 11)) = *((uint64_t*)(ss - 11)); *((uint32_t*)(dd - 4)) = *((uint32_t*)(ss - 4)); break; + case 140: memcpy_avx_128(dd - 140, ss - 140); [[fallthrough]]; + case 12: *((uint64_t*)(dd - 12)) = *((uint64_t*)(ss - 12)); *((uint32_t*)(dd - 4)) = *((uint32_t*)(ss - 4)); break; + case 141: memcpy_avx_128(dd - 141, ss - 141); [[fallthrough]]; + case 13: *((uint64_t*)(dd - 13)) = *((uint64_t*)(ss - 13)); *((uint64_t*)(dd - 8)) = *((uint64_t*)(ss - 8)); break; + case 142: memcpy_avx_128(dd - 142, ss - 142); [[fallthrough]]; + case 14: *((uint64_t*)(dd - 14)) = *((uint64_t*)(ss - 14)); *((uint64_t*)(dd - 8)) = *((uint64_t*)(ss - 8)); break; + case 143: memcpy_avx_128(dd - 143, ss - 143); [[fallthrough]]; + case 15: *((uint64_t*)(dd - 15)) = *((uint64_t*)(ss - 15)); *((uint64_t*)(dd - 8)) = *((uint64_t*)(ss - 8)); break; + case 144: memcpy_avx_128(dd - 144, ss - 144); [[fallthrough]]; + case 16: memcpy_avx_16(dd - 16, ss - 16); break; + case 145: memcpy_avx_128(dd - 145, ss - 145); [[fallthrough]]; + case 17: memcpy_avx_16(dd - 17, ss - 17); dd[-1] = ss[-1]; break; + case 146: memcpy_avx_128(dd - 146, ss - 146); [[fallthrough]]; + case 18: memcpy_avx_16(dd - 18, ss - 18); *((uint16_t*)(dd - 2)) = *((uint16_t*)(ss - 2)); break; + case 147: memcpy_avx_128(dd - 147, ss - 147); [[fallthrough]]; + case 19: memcpy_avx_16(dd - 19, ss - 19); *((uint32_t*)(dd - 4)) = *((uint32_t*)(ss - 4)); break; + case 148: memcpy_avx_128(dd - 148, ss - 148); [[fallthrough]]; + case 20: memcpy_avx_16(dd - 20, ss - 20); *((uint32_t*)(dd - 4)) = *((uint32_t*)(ss - 4)); break; + case 149: memcpy_avx_128(dd - 149, ss - 149); [[fallthrough]]; + case 21: memcpy_avx_16(dd - 21, ss - 21); *((uint64_t*)(dd - 8)) = *((uint64_t*)(ss - 8)); break; + case 150: memcpy_avx_128(dd - 150, ss - 150); [[fallthrough]]; + case 22: memcpy_avx_16(dd - 22, ss - 22); *((uint64_t*)(dd - 8)) = *((uint64_t*)(ss - 8)); break; + case 151: memcpy_avx_128(dd - 151, ss - 151); [[fallthrough]]; + case 23: memcpy_avx_16(dd - 23, ss - 23); *((uint64_t*)(dd - 8)) = *((uint64_t*)(ss - 8)); break; + case 152: memcpy_avx_128(dd - 152, ss - 152); [[fallthrough]]; + case 24: memcpy_avx_16(dd - 24, ss - 24); *((uint64_t*)(dd - 8)) = *((uint64_t*)(ss - 8)); break; + case 153: memcpy_avx_128(dd - 153, ss - 153); [[fallthrough]]; + case 25: memcpy_avx_16(dd - 25, ss - 25); memcpy_avx_16(dd - 16, ss - 16); break; + case 154: memcpy_avx_128(dd - 154, ss - 154); [[fallthrough]]; + case 26: memcpy_avx_16(dd - 26, ss - 26); memcpy_avx_16(dd - 16, ss - 16); break; + case 155: memcpy_avx_128(dd - 155, ss - 155); [[fallthrough]]; + case 27: memcpy_avx_16(dd - 27, ss - 27); memcpy_avx_16(dd - 16, ss - 16); break; + case 156: memcpy_avx_128(dd - 156, ss - 156); [[fallthrough]]; + case 28: memcpy_avx_16(dd - 28, ss - 28); memcpy_avx_16(dd - 16, ss - 16); break; + case 157: memcpy_avx_128(dd - 157, ss - 157); [[fallthrough]]; + case 29: memcpy_avx_16(dd - 29, ss - 29); memcpy_avx_16(dd - 16, ss - 16); break; + case 158: memcpy_avx_128(dd - 158, ss - 158); [[fallthrough]]; + case 30: memcpy_avx_16(dd - 30, ss - 30); memcpy_avx_16(dd - 16, ss - 16); break; + case 159: memcpy_avx_128(dd - 159, ss - 159); [[fallthrough]]; + case 31: memcpy_avx_16(dd - 31, ss - 31); memcpy_avx_16(dd - 16, ss - 16); break; + case 160: memcpy_avx_128(dd - 160, ss - 160); [[fallthrough]]; + case 32: memcpy_avx_32(dd - 32, ss - 32); break; + case 161: memcpy_avx_128(dd - 161, ss - 161); [[fallthrough]]; + case 33: memcpy_avx_32(dd - 33, ss - 33); dd[-1] = ss[-1]; break; + case 162: memcpy_avx_128(dd - 162, ss - 162); [[fallthrough]]; + case 34: memcpy_avx_32(dd - 34, ss - 34); *((uint16_t*)(dd - 2)) = *((uint16_t*)(ss - 2)); break; + case 163: memcpy_avx_128(dd - 163, ss - 163); [[fallthrough]]; + case 35: memcpy_avx_32(dd - 35, ss - 35); *((uint32_t*)(dd - 4)) = *((uint32_t*)(ss - 4)); break; + case 164: memcpy_avx_128(dd - 164, ss - 164); [[fallthrough]]; + case 36: memcpy_avx_32(dd - 36, ss - 36); *((uint32_t*)(dd - 4)) = *((uint32_t*)(ss - 4)); break; + case 165: memcpy_avx_128(dd - 165, ss - 165); [[fallthrough]]; + case 37: memcpy_avx_32(dd - 37, ss - 37); *((uint64_t*)(dd - 8)) = *((uint64_t*)(ss - 8)); break; + case 166: memcpy_avx_128(dd - 166, ss - 166); [[fallthrough]]; + case 38: memcpy_avx_32(dd - 38, ss - 38); *((uint64_t*)(dd - 8)) = *((uint64_t*)(ss - 8)); break; + case 167: memcpy_avx_128(dd - 167, ss - 167); [[fallthrough]]; + case 39: memcpy_avx_32(dd - 39, ss - 39); *((uint64_t*)(dd - 8)) = *((uint64_t*)(ss - 8)); break; + case 168: memcpy_avx_128(dd - 168, ss - 168); [[fallthrough]]; + case 40: memcpy_avx_32(dd - 40, ss - 40); *((uint64_t*)(dd - 8)) = *((uint64_t*)(ss - 8)); break; + case 169: memcpy_avx_128(dd - 169, ss - 169); [[fallthrough]]; + case 41: memcpy_avx_32(dd - 41, ss - 41); memcpy_avx_16(dd - 16, ss - 16); break; + case 170: memcpy_avx_128(dd - 170, ss - 170); [[fallthrough]]; + case 42: memcpy_avx_32(dd - 42, ss - 42); memcpy_avx_16(dd - 16, ss - 16); break; + case 171: memcpy_avx_128(dd - 171, ss - 171); [[fallthrough]]; + case 43: memcpy_avx_32(dd - 43, ss - 43); memcpy_avx_16(dd - 16, ss - 16); break; + case 172: memcpy_avx_128(dd - 172, ss - 172); [[fallthrough]]; + case 44: memcpy_avx_32(dd - 44, ss - 44); memcpy_avx_16(dd - 16, ss - 16); break; + case 173: memcpy_avx_128(dd - 173, ss - 173); [[fallthrough]]; + case 45: memcpy_avx_32(dd - 45, ss - 45); memcpy_avx_16(dd - 16, ss - 16); break; + case 174: memcpy_avx_128(dd - 174, ss - 174); [[fallthrough]]; + case 46: memcpy_avx_32(dd - 46, ss - 46); memcpy_avx_16(dd - 16, ss - 16); break; + case 175: memcpy_avx_128(dd - 175, ss - 175); [[fallthrough]]; + case 47: memcpy_avx_32(dd - 47, ss - 47); memcpy_avx_16(dd - 16, ss - 16); break; + case 176: memcpy_avx_128(dd - 176, ss - 176); [[fallthrough]]; + case 48: memcpy_avx_32(dd - 48, ss - 48); memcpy_avx_16(dd - 16, ss - 16); break; + case 177: memcpy_avx_128(dd - 177, ss - 177); [[fallthrough]]; + case 49: memcpy_avx_32(dd - 49, ss - 49); memcpy_avx_32(dd - 32, ss - 32); break; + case 178: memcpy_avx_128(dd - 178, ss - 178); [[fallthrough]]; + case 50: memcpy_avx_32(dd - 50, ss - 50); memcpy_avx_32(dd - 32, ss - 32); break; + case 179: memcpy_avx_128(dd - 179, ss - 179); [[fallthrough]]; + case 51: memcpy_avx_32(dd - 51, ss - 51); memcpy_avx_32(dd - 32, ss - 32); break; + case 180: memcpy_avx_128(dd - 180, ss - 180); [[fallthrough]]; + case 52: memcpy_avx_32(dd - 52, ss - 52); memcpy_avx_32(dd - 32, ss - 32); break; + case 181: memcpy_avx_128(dd - 181, ss - 181); [[fallthrough]]; + case 53: memcpy_avx_32(dd - 53, ss - 53); memcpy_avx_32(dd - 32, ss - 32); break; + case 182: memcpy_avx_128(dd - 182, ss - 182); [[fallthrough]]; + case 54: memcpy_avx_32(dd - 54, ss - 54); memcpy_avx_32(dd - 32, ss - 32); break; + case 183: memcpy_avx_128(dd - 183, ss - 183); [[fallthrough]]; + case 55: memcpy_avx_32(dd - 55, ss - 55); memcpy_avx_32(dd - 32, ss - 32); break; + case 184: memcpy_avx_128(dd - 184, ss - 184); [[fallthrough]]; + case 56: memcpy_avx_32(dd - 56, ss - 56); memcpy_avx_32(dd - 32, ss - 32); break; + case 185: memcpy_avx_128(dd - 185, ss - 185); [[fallthrough]]; + case 57: memcpy_avx_32(dd - 57, ss - 57); memcpy_avx_32(dd - 32, ss - 32); break; + case 186: memcpy_avx_128(dd - 186, ss - 186); [[fallthrough]]; + case 58: memcpy_avx_32(dd - 58, ss - 58); memcpy_avx_32(dd - 32, ss - 32); break; + case 187: memcpy_avx_128(dd - 187, ss - 187); [[fallthrough]]; + case 59: memcpy_avx_32(dd - 59, ss - 59); memcpy_avx_32(dd - 32, ss - 32); break; + case 188: memcpy_avx_128(dd - 188, ss - 188); [[fallthrough]]; + case 60: memcpy_avx_32(dd - 60, ss - 60); memcpy_avx_32(dd - 32, ss - 32); break; + case 189: memcpy_avx_128(dd - 189, ss - 189); [[fallthrough]]; + case 61: memcpy_avx_32(dd - 61, ss - 61); memcpy_avx_32(dd - 32, ss - 32); break; + case 190: memcpy_avx_128(dd - 190, ss - 190); [[fallthrough]]; + case 62: memcpy_avx_32(dd - 62, ss - 62); memcpy_avx_32(dd - 32, ss - 32); break; + case 191: memcpy_avx_128(dd - 191, ss - 191); [[fallthrough]]; + case 63: memcpy_avx_32(dd - 63, ss - 63); memcpy_avx_32(dd - 32, ss - 32); break; + case 192: memcpy_avx_128(dd - 192, ss - 192); [[fallthrough]]; + case 64: memcpy_avx_64(dd - 64, ss - 64); break; + case 193: memcpy_avx_128(dd - 193, ss - 193); [[fallthrough]]; + case 65: memcpy_avx_64(dd - 65, ss - 65); dd[-1] = ss[-1]; break; + case 194: memcpy_avx_128(dd - 194, ss - 194); [[fallthrough]]; + case 66: memcpy_avx_64(dd - 66, ss - 66); *((uint16_t*)(dd - 2)) = *((uint16_t*)(ss - 2)); break; + case 195: memcpy_avx_128(dd - 195, ss - 195); [[fallthrough]]; + case 67: memcpy_avx_64(dd - 67, ss - 67); *((uint32_t*)(dd - 4)) = *((uint32_t*)(ss - 4)); break; + case 196: memcpy_avx_128(dd - 196, ss - 196); [[fallthrough]]; + case 68: memcpy_avx_64(dd - 68, ss - 68); *((uint32_t*)(dd - 4)) = *((uint32_t*)(ss - 4)); break; + case 197: memcpy_avx_128(dd - 197, ss - 197); [[fallthrough]]; + case 69: memcpy_avx_64(dd - 69, ss - 69); *((uint64_t*)(dd - 8)) = *((uint64_t*)(ss - 8)); break; + case 198: memcpy_avx_128(dd - 198, ss - 198); [[fallthrough]]; + case 70: memcpy_avx_64(dd - 70, ss - 70); *((uint64_t*)(dd - 8)) = *((uint64_t*)(ss - 8)); break; + case 199: memcpy_avx_128(dd - 199, ss - 199); [[fallthrough]]; + case 71: memcpy_avx_64(dd - 71, ss - 71); *((uint64_t*)(dd - 8)) = *((uint64_t*)(ss - 8)); break; + case 200: memcpy_avx_128(dd - 200, ss - 200); [[fallthrough]]; + case 72: memcpy_avx_64(dd - 72, ss - 72); *((uint64_t*)(dd - 8)) = *((uint64_t*)(ss - 8)); break; + case 201: memcpy_avx_128(dd - 201, ss - 201); [[fallthrough]]; + case 73: memcpy_avx_64(dd - 73, ss - 73); memcpy_avx_16(dd - 16, ss - 16); break; + case 202: memcpy_avx_128(dd - 202, ss - 202); [[fallthrough]]; + case 74: memcpy_avx_64(dd - 74, ss - 74); memcpy_avx_16(dd - 16, ss - 16); break; + case 203: memcpy_avx_128(dd - 203, ss - 203); [[fallthrough]]; + case 75: memcpy_avx_64(dd - 75, ss - 75); memcpy_avx_16(dd - 16, ss - 16); break; + case 204: memcpy_avx_128(dd - 204, ss - 204); [[fallthrough]]; + case 76: memcpy_avx_64(dd - 76, ss - 76); memcpy_avx_16(dd - 16, ss - 16); break; + case 205: memcpy_avx_128(dd - 205, ss - 205); [[fallthrough]]; + case 77: memcpy_avx_64(dd - 77, ss - 77); memcpy_avx_16(dd - 16, ss - 16); break; + case 206: memcpy_avx_128(dd - 206, ss - 206); [[fallthrough]]; + case 78: memcpy_avx_64(dd - 78, ss - 78); memcpy_avx_16(dd - 16, ss - 16); break; + case 207: memcpy_avx_128(dd - 207, ss - 207); [[fallthrough]]; + case 79: memcpy_avx_64(dd - 79, ss - 79); memcpy_avx_16(dd - 16, ss - 16); break; + case 208: memcpy_avx_128(dd - 208, ss - 208); [[fallthrough]]; + case 80: memcpy_avx_64(dd - 80, ss - 80); memcpy_avx_16(dd - 16, ss - 16); break; + case 209: memcpy_avx_128(dd - 209, ss - 209); [[fallthrough]]; + case 81: memcpy_avx_64(dd - 81, ss - 81); memcpy_avx_32(dd - 32, ss - 32); break; + case 210: memcpy_avx_128(dd - 210, ss - 210); [[fallthrough]]; + case 82: memcpy_avx_64(dd - 82, ss - 82); memcpy_avx_32(dd - 32, ss - 32); break; + case 211: memcpy_avx_128(dd - 211, ss - 211); [[fallthrough]]; + case 83: memcpy_avx_64(dd - 83, ss - 83); memcpy_avx_32(dd - 32, ss - 32); break; + case 212: memcpy_avx_128(dd - 212, ss - 212); [[fallthrough]]; + case 84: memcpy_avx_64(dd - 84, ss - 84); memcpy_avx_32(dd - 32, ss - 32); break; + case 213: memcpy_avx_128(dd - 213, ss - 213); [[fallthrough]]; + case 85: memcpy_avx_64(dd - 85, ss - 85); memcpy_avx_32(dd - 32, ss - 32); break; + case 214: memcpy_avx_128(dd - 214, ss - 214); [[fallthrough]]; + case 86: memcpy_avx_64(dd - 86, ss - 86); memcpy_avx_32(dd - 32, ss - 32); break; + case 215: memcpy_avx_128(dd - 215, ss - 215); [[fallthrough]]; + case 87: memcpy_avx_64(dd - 87, ss - 87); memcpy_avx_32(dd - 32, ss - 32); break; + case 216: memcpy_avx_128(dd - 216, ss - 216); [[fallthrough]]; + case 88: memcpy_avx_64(dd - 88, ss - 88); memcpy_avx_32(dd - 32, ss - 32); break; + case 217: memcpy_avx_128(dd - 217, ss - 217); [[fallthrough]]; + case 89: memcpy_avx_64(dd - 89, ss - 89); memcpy_avx_32(dd - 32, ss - 32); break; + case 218: memcpy_avx_128(dd - 218, ss - 218); [[fallthrough]]; + case 90: memcpy_avx_64(dd - 90, ss - 90); memcpy_avx_32(dd - 32, ss - 32); break; + case 219: memcpy_avx_128(dd - 219, ss - 219); [[fallthrough]]; + case 91: memcpy_avx_64(dd - 91, ss - 91); memcpy_avx_32(dd - 32, ss - 32); break; + case 220: memcpy_avx_128(dd - 220, ss - 220); [[fallthrough]]; + case 92: memcpy_avx_64(dd - 92, ss - 92); memcpy_avx_32(dd - 32, ss - 32); break; + case 221: memcpy_avx_128(dd - 221, ss - 221); [[fallthrough]]; + case 93: memcpy_avx_64(dd - 93, ss - 93); memcpy_avx_32(dd - 32, ss - 32); break; + case 222: memcpy_avx_128(dd - 222, ss - 222); [[fallthrough]]; + case 94: memcpy_avx_64(dd - 94, ss - 94); memcpy_avx_32(dd - 32, ss - 32); break; + case 223: memcpy_avx_128(dd - 223, ss - 223); [[fallthrough]]; + case 95: memcpy_avx_64(dd - 95, ss - 95); memcpy_avx_32(dd - 32, ss - 32); break; + case 224: memcpy_avx_128(dd - 224, ss - 224); [[fallthrough]]; + case 96: memcpy_avx_64(dd - 96, ss - 96); memcpy_avx_32(dd - 32, ss - 32); break; + case 225: memcpy_avx_128(dd - 225, ss - 225); [[fallthrough]]; + case 97: memcpy_avx_64(dd - 97, ss - 97); memcpy_avx_64(dd - 64, ss - 64); break; + case 226: memcpy_avx_128(dd - 226, ss - 226); [[fallthrough]]; + case 98: memcpy_avx_64(dd - 98, ss - 98); memcpy_avx_64(dd - 64, ss - 64); break; + case 227: memcpy_avx_128(dd - 227, ss - 227); [[fallthrough]]; + case 99: memcpy_avx_64(dd - 99, ss - 99); memcpy_avx_64(dd - 64, ss - 64); break; + case 228: memcpy_avx_128(dd - 228, ss - 228); [[fallthrough]]; + case 100: memcpy_avx_64(dd - 100, ss - 100); memcpy_avx_64(dd - 64, ss - 64); break; + case 229: memcpy_avx_128(dd - 229, ss - 229); [[fallthrough]]; + case 101: memcpy_avx_64(dd - 101, ss - 101); memcpy_avx_64(dd - 64, ss - 64); break; + case 230: memcpy_avx_128(dd - 230, ss - 230); [[fallthrough]]; + case 102: memcpy_avx_64(dd - 102, ss - 102); memcpy_avx_64(dd - 64, ss - 64); break; + case 231: memcpy_avx_128(dd - 231, ss - 231); [[fallthrough]]; + case 103: memcpy_avx_64(dd - 103, ss - 103); memcpy_avx_64(dd - 64, ss - 64); break; + case 232: memcpy_avx_128(dd - 232, ss - 232); [[fallthrough]]; + case 104: memcpy_avx_64(dd - 104, ss - 104); memcpy_avx_64(dd - 64, ss - 64); break; + case 233: memcpy_avx_128(dd - 233, ss - 233); [[fallthrough]]; + case 105: memcpy_avx_64(dd - 105, ss - 105); memcpy_avx_64(dd - 64, ss - 64); break; + case 234: memcpy_avx_128(dd - 234, ss - 234); [[fallthrough]]; + case 106: memcpy_avx_64(dd - 106, ss - 106); memcpy_avx_64(dd - 64, ss - 64); break; + case 235: memcpy_avx_128(dd - 235, ss - 235); [[fallthrough]]; + case 107: memcpy_avx_64(dd - 107, ss - 107); memcpy_avx_64(dd - 64, ss - 64); break; + case 236: memcpy_avx_128(dd - 236, ss - 236); [[fallthrough]]; + case 108: memcpy_avx_64(dd - 108, ss - 108); memcpy_avx_64(dd - 64, ss - 64); break; + case 237: memcpy_avx_128(dd - 237, ss - 237); [[fallthrough]]; + case 109: memcpy_avx_64(dd - 109, ss - 109); memcpy_avx_64(dd - 64, ss - 64); break; + case 238: memcpy_avx_128(dd - 238, ss - 238); [[fallthrough]]; + case 110: memcpy_avx_64(dd - 110, ss - 110); memcpy_avx_64(dd - 64, ss - 64); break; + case 239: memcpy_avx_128(dd - 239, ss - 239); [[fallthrough]]; + case 111: memcpy_avx_64(dd - 111, ss - 111); memcpy_avx_64(dd - 64, ss - 64); break; + case 240: memcpy_avx_128(dd - 240, ss - 240); [[fallthrough]]; + case 112: memcpy_avx_64(dd - 112, ss - 112); memcpy_avx_64(dd - 64, ss - 64); break; + case 241: memcpy_avx_128(dd - 241, ss - 241); [[fallthrough]]; + case 113: memcpy_avx_64(dd - 113, ss - 113); memcpy_avx_64(dd - 64, ss - 64); break; + case 242: memcpy_avx_128(dd - 242, ss - 242); [[fallthrough]]; + case 114: memcpy_avx_64(dd - 114, ss - 114); memcpy_avx_64(dd - 64, ss - 64); break; + case 243: memcpy_avx_128(dd - 243, ss - 243); [[fallthrough]]; + case 115: memcpy_avx_64(dd - 115, ss - 115); memcpy_avx_64(dd - 64, ss - 64); break; + case 244: memcpy_avx_128(dd - 244, ss - 244); [[fallthrough]]; + case 116: memcpy_avx_64(dd - 116, ss - 116); memcpy_avx_64(dd - 64, ss - 64); break; + case 245: memcpy_avx_128(dd - 245, ss - 245); [[fallthrough]]; + case 117: memcpy_avx_64(dd - 117, ss - 117); memcpy_avx_64(dd - 64, ss - 64); break; + case 246: memcpy_avx_128(dd - 246, ss - 246); [[fallthrough]]; + case 118: memcpy_avx_64(dd - 118, ss - 118); memcpy_avx_64(dd - 64, ss - 64); break; + case 247: memcpy_avx_128(dd - 247, ss - 247); [[fallthrough]]; + case 119: memcpy_avx_64(dd - 119, ss - 119); memcpy_avx_64(dd - 64, ss - 64); break; + case 248: memcpy_avx_128(dd - 248, ss - 248); [[fallthrough]]; + case 120: memcpy_avx_64(dd - 120, ss - 120); memcpy_avx_64(dd - 64, ss - 64); break; + case 249: memcpy_avx_128(dd - 249, ss - 249); [[fallthrough]]; + case 121: memcpy_avx_64(dd - 121, ss - 121); memcpy_avx_64(dd - 64, ss - 64); break; + case 250: memcpy_avx_128(dd - 250, ss - 250); [[fallthrough]]; + case 122: memcpy_avx_64(dd - 122, ss - 122); memcpy_avx_64(dd - 64, ss - 64); break; + case 251: memcpy_avx_128(dd - 251, ss - 251); [[fallthrough]]; + case 123: memcpy_avx_64(dd - 123, ss - 123); memcpy_avx_64(dd - 64, ss - 64); break; + case 252: memcpy_avx_128(dd - 252, ss - 252); [[fallthrough]]; + case 124: memcpy_avx_64(dd - 124, ss - 124); memcpy_avx_64(dd - 64, ss - 64); break; + case 253: memcpy_avx_128(dd - 253, ss - 253); [[fallthrough]]; + case 125: memcpy_avx_64(dd - 125, ss - 125); memcpy_avx_64(dd - 64, ss - 64); break; + case 254: memcpy_avx_128(dd - 254, ss - 254); [[fallthrough]]; + case 126: memcpy_avx_64(dd - 126, ss - 126); memcpy_avx_64(dd - 64, ss - 64); break; + case 255: memcpy_avx_128(dd - 255, ss - 255); [[fallthrough]]; + case 127: memcpy_avx_64(dd - 127, ss - 127); memcpy_avx_64(dd - 64, ss - 64); break; + case 256: memcpy_avx_256(dd - 256, ss - 256); break; + } + + return dst; +} + + +//--------------------------------------------------------------------- +// main routine +//--------------------------------------------------------------------- +void* memcpy_fast_avx(void * __restrict destination, const void * __restrict source, size_t size) +{ + unsigned char *dst = reinterpret_cast(destination); + const unsigned char *src = reinterpret_cast(source); + static size_t cachesize = 0x200000; // L3-cache size + size_t padding; + + // small memory copy + if (size <= 256) { + memcpy_tiny_avx(dst, src, size); + _mm256_zeroupper(); + return destination; + } + + // align destination to 16 bytes boundary + padding = (32 - (((size_t)dst) & 31)) & 31; + +#if 0 + if (padding > 0) { + __m256i head = _mm256_loadu_si256(reinterpret_cast(src)); + _mm256_storeu_si256((__m256i*)dst, head); + dst += padding; + src += padding; + size -= padding; + } +#else + __m256i head = _mm256_loadu_si256(reinterpret_cast(src)); + _mm256_storeu_si256((__m256i*)dst, head); + dst += padding; + src += padding; + size -= padding; +#endif + + // medium size copy + if (size <= cachesize) { + __m256i c0, c1, c2, c3, c4, c5, c6, c7; + + for (; size >= 256; size -= 256) { + c0 = _mm256_loadu_si256((reinterpret_cast(src)) + 0); + c1 = _mm256_loadu_si256((reinterpret_cast(src)) + 1); + c2 = _mm256_loadu_si256((reinterpret_cast(src)) + 2); + c3 = _mm256_loadu_si256((reinterpret_cast(src)) + 3); + c4 = _mm256_loadu_si256((reinterpret_cast(src)) + 4); + c5 = _mm256_loadu_si256((reinterpret_cast(src)) + 5); + c6 = _mm256_loadu_si256((reinterpret_cast(src)) + 6); + c7 = _mm256_loadu_si256((reinterpret_cast(src)) + 7); + src += 256; + _mm256_storeu_si256(((reinterpret_cast<__m256i*>(dst)) + 0), c0); + _mm256_storeu_si256(((reinterpret_cast<__m256i*>(dst)) + 1), c1); + _mm256_storeu_si256(((reinterpret_cast<__m256i*>(dst)) + 2), c2); + _mm256_storeu_si256(((reinterpret_cast<__m256i*>(dst)) + 3), c3); + _mm256_storeu_si256(((reinterpret_cast<__m256i*>(dst)) + 4), c4); + _mm256_storeu_si256(((reinterpret_cast<__m256i*>(dst)) + 5), c5); + _mm256_storeu_si256(((reinterpret_cast<__m256i*>(dst)) + 6), c6); + _mm256_storeu_si256(((reinterpret_cast<__m256i*>(dst)) + 7), c7); + dst += 256; + } + } + else { // big memory copy + __m256i c0, c1, c2, c3, c4, c5, c6, c7; + /* __m256i c0, c1, c2, c3, c4, c5, c6, c7; */ + + if ((((size_t)src) & 31) == 0) { // source aligned + for (; size >= 256; size -= 256) { + c0 = _mm256_load_si256((reinterpret_cast(src)) + 0); + c1 = _mm256_load_si256((reinterpret_cast(src)) + 1); + c2 = _mm256_load_si256((reinterpret_cast(src)) + 2); + c3 = _mm256_load_si256((reinterpret_cast(src)) + 3); + c4 = _mm256_load_si256((reinterpret_cast(src)) + 4); + c5 = _mm256_load_si256((reinterpret_cast(src)) + 5); + c6 = _mm256_load_si256((reinterpret_cast(src)) + 6); + c7 = _mm256_load_si256((reinterpret_cast(src)) + 7); + src += 256; + _mm256_stream_si256(((reinterpret_cast<__m256i*>(dst)) + 0), c0); + _mm256_stream_si256(((reinterpret_cast<__m256i*>(dst)) + 1), c1); + _mm256_stream_si256(((reinterpret_cast<__m256i*>(dst)) + 2), c2); + _mm256_stream_si256(((reinterpret_cast<__m256i*>(dst)) + 3), c3); + _mm256_stream_si256(((reinterpret_cast<__m256i*>(dst)) + 4), c4); + _mm256_stream_si256(((reinterpret_cast<__m256i*>(dst)) + 5), c5); + _mm256_stream_si256(((reinterpret_cast<__m256i*>(dst)) + 6), c6); + _mm256_stream_si256(((reinterpret_cast<__m256i*>(dst)) + 7), c7); + dst += 256; + } + } + else { // source unaligned + for (; size >= 256; size -= 256) { + c0 = _mm256_loadu_si256((reinterpret_cast(src)) + 0); + c1 = _mm256_loadu_si256((reinterpret_cast(src)) + 1); + c2 = _mm256_loadu_si256((reinterpret_cast(src)) + 2); + c3 = _mm256_loadu_si256((reinterpret_cast(src)) + 3); + c4 = _mm256_loadu_si256((reinterpret_cast(src)) + 4); + c5 = _mm256_loadu_si256((reinterpret_cast(src)) + 5); + c6 = _mm256_loadu_si256((reinterpret_cast(src)) + 6); + c7 = _mm256_loadu_si256((reinterpret_cast(src)) + 7); + src += 256; + _mm256_stream_si256(((reinterpret_cast<__m256i*>(dst)) + 0), c0); + _mm256_stream_si256(((reinterpret_cast<__m256i*>(dst)) + 1), c1); + _mm256_stream_si256(((reinterpret_cast<__m256i*>(dst)) + 2), c2); + _mm256_stream_si256(((reinterpret_cast<__m256i*>(dst)) + 3), c3); + _mm256_stream_si256(((reinterpret_cast<__m256i*>(dst)) + 4), c4); + _mm256_stream_si256(((reinterpret_cast<__m256i*>(dst)) + 5), c5); + _mm256_stream_si256(((reinterpret_cast<__m256i*>(dst)) + 6), c6); + _mm256_stream_si256(((reinterpret_cast<__m256i*>(dst)) + 7), c7); + dst += 256; + } + } + _mm_sfence(); + } + + memcpy_tiny_avx(dst, src, size); + _mm256_zeroupper(); + + return destination; +} diff --git a/utils/memcpy-bench/memcpy-bench.cpp b/utils/memcpy-bench/memcpy-bench.cpp new file mode 100644 index 00000000000..d4f886398da --- /dev/null +++ b/utils/memcpy-bench/memcpy-bench.cpp @@ -0,0 +1,610 @@ +#include +#include +#include +#include +#include +#include +#include + +#include + +#include + +#include + +#include + +#pragma GCC diagnostic ignored "-Wold-style-cast" +#pragma GCC diagnostic ignored "-Wcast-align" +#pragma GCC diagnostic ignored "-Wcast-qual" +#include "FastMemcpy.h" +//#include "FastMemcpy_Avx.h" + +#include +#include + + + +template +void NO_INLINE loop(uint8_t * dst, uint8_t * src, size_t size, F && chunk_size_distribution, MemcpyImpl && impl) +{ + while (size) + { + size_t bytes_to_copy = std::min(size, chunk_size_distribution()); + + impl(dst, src, bytes_to_copy); + + dst += bytes_to_copy; + src += bytes_to_copy; + size -= bytes_to_copy; + } +} + + +using RNG = pcg32_fast; + +template +size_t generatorUniform(RNG & rng) { return rng() % N; }; + + +template +void test(uint8_t * dst, uint8_t * src, size_t size, size_t iterations, size_t num_threads, F && generator, MemcpyImpl && impl) +{ + Stopwatch watch; + + std::vector threads; + threads.reserve(num_threads); + + for (size_t thread_num = 0; thread_num < num_threads; ++thread_num) + { + size_t begin = size * thread_num / num_threads; + size_t end = size * (thread_num + 1) / num_threads; + + threads.emplace_back([begin, end, iterations, &src, &dst, &generator, &impl] + { + for (size_t iteration = 0; iteration < iterations; ++iteration) + { + loop( + iteration % 2 ? &src[begin] : &dst[begin], + iteration % 2 ? &dst[begin] : &src[begin], + end - begin, + [rng = RNG(), &generator]() mutable { return generator(rng); }, + std::forward(impl)); + } + }); + } + + for (auto & thread : threads) + thread.join(); + + double elapsed_ns = watch.elapsed(); + + /// Validation + size_t sum = 0; + for (size_t i = 0; i < size; ++i) + sum += dst[i]; + + std::cerr << std::fixed << std::setprecision(3) + << "Processed in " << (elapsed_ns / 1e9) << "sec, " << (size * iterations * 1.0 / elapsed_ns) << " GB/sec (sum = " << sum << ")\n"; +} + + +using memcpy_type = void * (*)(const void * __restrict, void * __restrict, size_t); + + +static void * memcpy_erms(void * dst, const void * src, size_t size) +{ + asm volatile ( + "rep movsb" + : "=D"(dst), "=S"(src), "=c"(size) + : "0"(dst), "1"(src), "2"(size) + : "memory"); + return dst; +} + +extern "C" void * memcpy_jart(void * dst, const void * src, size_t size); +extern "C" void MemCpy(void * dst, const void * src, size_t size); + + +static void * memcpySSE2(void * __restrict destination, const void * __restrict source, size_t size) +{ + unsigned char *dst = reinterpret_cast(destination); + const unsigned char *src = reinterpret_cast(source); + size_t padding; + + // small memory copy + if (size <= 16) + return memcpy_tiny(dst, src, size); + + // align destination to 16 bytes boundary + padding = (16 - (reinterpret_cast(dst) & 15)) & 15; + + if (padding > 0) + { + __m128i head = _mm_loadu_si128(reinterpret_cast(src)); + _mm_storeu_si128(reinterpret_cast<__m128i*>(dst), head); + dst += padding; + src += padding; + size -= padding; + } + + // medium size copy + __m128i c0; + + for (; size >= 16; size -= 16) + { + c0 = _mm_loadu_si128(reinterpret_cast(src)); + src += 16; + _mm_store_si128((reinterpret_cast<__m128i*>(dst)), c0); + dst += 16; + } + + memcpy_tiny(dst, src, size); + return destination; +} + +static void * memcpySSE2Unrolled2(void * __restrict destination, const void * __restrict source, size_t size) +{ + unsigned char *dst = reinterpret_cast(destination); + const unsigned char *src = reinterpret_cast(source); + size_t padding; + + // small memory copy + if (size <= 32) + return memcpy_tiny(dst, src, size); + + // align destination to 16 bytes boundary + padding = (16 - (reinterpret_cast(dst) & 15)) & 15; + + if (padding > 0) + { + __m128i head = _mm_loadu_si128(reinterpret_cast(src)); + _mm_storeu_si128(reinterpret_cast<__m128i*>(dst), head); + dst += padding; + src += padding; + size -= padding; + } + + // medium size copy + __m128i c0, c1; + + for (; size >= 32; size -= 32) + { + c0 = _mm_loadu_si128(reinterpret_cast(src) + 0); + c1 = _mm_loadu_si128(reinterpret_cast(src) + 1); + src += 32; + _mm_store_si128((reinterpret_cast<__m128i*>(dst) + 0), c0); + _mm_store_si128((reinterpret_cast<__m128i*>(dst) + 1), c1); + dst += 32; + } + + memcpy_tiny(dst, src, size); + return destination; +} + +static void * memcpySSE2Unrolled4(void * __restrict destination, const void * __restrict source, size_t size) +{ + unsigned char *dst = reinterpret_cast(destination); + const unsigned char *src = reinterpret_cast(source); + size_t padding; + + // small memory copy + if (size <= 64) + return memcpy_tiny(dst, src, size); + + // align destination to 16 bytes boundary + padding = (16 - (reinterpret_cast(dst) & 15)) & 15; + + if (padding > 0) + { + __m128i head = _mm_loadu_si128(reinterpret_cast(src)); + _mm_storeu_si128(reinterpret_cast<__m128i*>(dst), head); + dst += padding; + src += padding; + size -= padding; + } + + // medium size copy + __m128i c0, c1, c2, c3; + + for (; size >= 64; size -= 64) + { + c0 = _mm_loadu_si128(reinterpret_cast(src) + 0); + c1 = _mm_loadu_si128(reinterpret_cast(src) + 1); + c2 = _mm_loadu_si128(reinterpret_cast(src) + 2); + c3 = _mm_loadu_si128(reinterpret_cast(src) + 3); + src += 64; + _mm_store_si128((reinterpret_cast<__m128i*>(dst) + 0), c0); + _mm_store_si128((reinterpret_cast<__m128i*>(dst) + 1), c1); + _mm_store_si128((reinterpret_cast<__m128i*>(dst) + 2), c2); + _mm_store_si128((reinterpret_cast<__m128i*>(dst) + 3), c3); + dst += 64; + } + + memcpy_tiny(dst, src, size); + return destination; +} + + +static void * memcpySSE2Unrolled8(void * __restrict destination, const void * __restrict source, size_t size) +{ + unsigned char *dst = reinterpret_cast(destination); + const unsigned char *src = reinterpret_cast(source); + size_t padding; + + // small memory copy + if (size <= 128) + return memcpy_tiny(dst, src, size); + + // align destination to 16 bytes boundary + padding = (16 - (reinterpret_cast(dst) & 15)) & 15; + + if (padding > 0) + { + __m128i head = _mm_loadu_si128(reinterpret_cast(src)); + _mm_storeu_si128(reinterpret_cast<__m128i*>(dst), head); + dst += padding; + src += padding; + size -= padding; + } + + // medium size copy + __m128i c0, c1, c2, c3, c4, c5, c6, c7; + + for (; size >= 128; size -= 128) + { + c0 = _mm_loadu_si128(reinterpret_cast(src) + 0); + c1 = _mm_loadu_si128(reinterpret_cast(src) + 1); + c2 = _mm_loadu_si128(reinterpret_cast(src) + 2); + c3 = _mm_loadu_si128(reinterpret_cast(src) + 3); + c4 = _mm_loadu_si128(reinterpret_cast(src) + 4); + c5 = _mm_loadu_si128(reinterpret_cast(src) + 5); + c6 = _mm_loadu_si128(reinterpret_cast(src) + 6); + c7 = _mm_loadu_si128(reinterpret_cast(src) + 7); + src += 128; + _mm_store_si128((reinterpret_cast<__m128i*>(dst) + 0), c0); + _mm_store_si128((reinterpret_cast<__m128i*>(dst) + 1), c1); + _mm_store_si128((reinterpret_cast<__m128i*>(dst) + 2), c2); + _mm_store_si128((reinterpret_cast<__m128i*>(dst) + 3), c3); + _mm_store_si128((reinterpret_cast<__m128i*>(dst) + 4), c4); + _mm_store_si128((reinterpret_cast<__m128i*>(dst) + 5), c5); + _mm_store_si128((reinterpret_cast<__m128i*>(dst) + 6), c6); + _mm_store_si128((reinterpret_cast<__m128i*>(dst) + 7), c7); + dst += 128; + } + + memcpy_tiny(dst, src, size); + return destination; +} + + +//static __attribute__((__always_inline__, __target__("sse2"))) +__attribute__((__always_inline__)) +void memcpy_my_medium_sse(uint8_t * __restrict & dst, const uint8_t * __restrict & src, size_t & size) +{ + /// Align destination to 16 bytes boundary. + size_t padding = (16 - (reinterpret_cast(dst) & 15)) & 15; + + if (padding > 0) + { + __m128i head = _mm_loadu_si128(reinterpret_cast(src)); + _mm_storeu_si128(reinterpret_cast<__m128i*>(dst), head); + dst += padding; + src += padding; + size -= padding; + } + + /// Aligned unrolled copy. + __m128i c0, c1, c2, c3, c4, c5, c6, c7; + + while (size >= 128) + { + c0 = _mm_loadu_si128(reinterpret_cast(src) + 0); + c1 = _mm_loadu_si128(reinterpret_cast(src) + 1); + c2 = _mm_loadu_si128(reinterpret_cast(src) + 2); + c3 = _mm_loadu_si128(reinterpret_cast(src) + 3); + c4 = _mm_loadu_si128(reinterpret_cast(src) + 4); + c5 = _mm_loadu_si128(reinterpret_cast(src) + 5); + c6 = _mm_loadu_si128(reinterpret_cast(src) + 6); + c7 = _mm_loadu_si128(reinterpret_cast(src) + 7); + src += 128; + _mm_store_si128((reinterpret_cast<__m128i*>(dst) + 0), c0); + _mm_store_si128((reinterpret_cast<__m128i*>(dst) + 1), c1); + _mm_store_si128((reinterpret_cast<__m128i*>(dst) + 2), c2); + _mm_store_si128((reinterpret_cast<__m128i*>(dst) + 3), c3); + _mm_store_si128((reinterpret_cast<__m128i*>(dst) + 4), c4); + _mm_store_si128((reinterpret_cast<__m128i*>(dst) + 5), c5); + _mm_store_si128((reinterpret_cast<__m128i*>(dst) + 6), c6); + _mm_store_si128((reinterpret_cast<__m128i*>(dst) + 7), c7); + dst += 128; + + size -= 128; + } +} + +__attribute__((__target__("avx"))) +void memcpy_my_medium_avx(uint8_t * __restrict & __restrict dst, const uint8_t * __restrict & __restrict src, size_t & __restrict size) +{ + size_t padding = (32 - (reinterpret_cast(dst) & 31)) & 31; + + if (padding > 0) + { + __m256i head = _mm256_loadu_si256(reinterpret_cast(src)); + _mm256_storeu_si256((__m256i*)dst, head); + dst += padding; + src += padding; + size -= padding; + } + + __m256i c0, c1, c2, c3, c4, c5, c6, c7; + + while (size >= 256) + { + c0 = _mm256_loadu_si256((reinterpret_cast(src)) + 0); + c1 = _mm256_loadu_si256((reinterpret_cast(src)) + 1); + c2 = _mm256_loadu_si256((reinterpret_cast(src)) + 2); + c3 = _mm256_loadu_si256((reinterpret_cast(src)) + 3); + c4 = _mm256_loadu_si256((reinterpret_cast(src)) + 4); + c5 = _mm256_loadu_si256((reinterpret_cast(src)) + 5); + c6 = _mm256_loadu_si256((reinterpret_cast(src)) + 6); + c7 = _mm256_loadu_si256((reinterpret_cast(src)) + 7); + src += 256; + _mm256_store_si256(((reinterpret_cast<__m256i*>(dst)) + 0), c0); + _mm256_store_si256(((reinterpret_cast<__m256i*>(dst)) + 1), c1); + _mm256_store_si256(((reinterpret_cast<__m256i*>(dst)) + 2), c2); + _mm256_store_si256(((reinterpret_cast<__m256i*>(dst)) + 3), c3); + _mm256_store_si256(((reinterpret_cast<__m256i*>(dst)) + 4), c4); + _mm256_store_si256(((reinterpret_cast<__m256i*>(dst)) + 5), c5); + _mm256_store_si256(((reinterpret_cast<__m256i*>(dst)) + 6), c6); + _mm256_store_si256(((reinterpret_cast<__m256i*>(dst)) + 7), c7); + dst += 256; + + size -= 256; + } +} + +bool have_avx = true; + +static uint8_t * memcpy_my(uint8_t * __restrict dst, const uint8_t * __restrict src, size_t size) +{ + uint8_t * ret = dst; + +tail: + if (size <= 16) + { + if (size >= 8) + { + __builtin_memcpy(dst + size - 8, src + size - 8, 8); + __builtin_memcpy(dst, src, 8); + } + else if (size >= 4) + { + __builtin_memcpy(dst + size - 4, src + size - 4, 4); + __builtin_memcpy(dst, src, 4); + } + else if (size >= 2) + { + __builtin_memcpy(dst + size - 2, src + size - 2, 2); + __builtin_memcpy(dst, src, 2); + } + else if (size >= 1) + { + *dst = *src; + } + } + else if (have_avx) + { + if (size <= 256) + { + __asm__( + "vmovups -0x20(%[s],%[size],1), %%ymm0\n" + "vmovups %%ymm0, -0x20(%[d],%[size],1)\n" + : [d]"+r"(dst), [s]"+r"(src) + : [size]"r"(size) + : "ymm0", "memory"); + + while (size > 32) + { + __asm__( + "vmovups (%[s]), %%ymm0\n" + "vmovups %%ymm0, (%[d])\n" + : [d]"+r"(dst), [s]"+r"(src) + : + : "ymm0", "memory"); + + dst += 32; + src += 32; + size -= 32; + } + } + else + { + size_t padding = (32 - (reinterpret_cast(dst) & 31)) & 31; + + if (padding > 0) + { + __asm__( + "vmovups (%[s]), %%ymm0\n" + "vmovups %%ymm0, (%[d])\n" + : [d]"+r"(dst), [s]"+r"(src) + : + : "ymm0", "memory"); + + dst += padding; + src += padding; + size -= padding; + } + + while (size >= 256) + { + __asm__( + "vmovups (%[s]), %%ymm0\n" + "vmovups 0x20(%[s]), %%ymm1\n" + "vmovups 0x40(%[s]), %%ymm2\n" + "vmovups 0x60(%[s]), %%ymm3\n" + "vmovups 0x80(%[s]), %%ymm4\n" + "vmovups 0xa0(%[s]), %%ymm5\n" + "vmovups 0xc0(%[s]), %%ymm6\n" + "vmovups 0xe0(%[s]), %%ymm7\n" + "add $0x100,%[s]\n" + "vmovaps %%ymm0, (%[d])\n" + "vmovaps %%ymm1, 0x20(%[d])\n" + "vmovaps %%ymm2, 0x40(%[d])\n" + "vmovaps %%ymm3, 0x60(%[d])\n" + "vmovaps %%ymm4, 0x80(%[d])\n" + "vmovaps %%ymm5, 0xa0(%[d])\n" + "vmovaps %%ymm6, 0xc0(%[d])\n" + "vmovaps %%ymm7, 0xe0(%[d])\n" + "add $0x100, %[d]\n" + : [d]"+r"(dst), [s]"+r"(src) + : + : "ymm0", "ymm1", "ymm2", "ymm3", "ymm4", "ymm5", "ymm6", "ymm7", "memory"); + + size -= 256; + } + + goto tail; + } + } + else + { + if (size <= 128) + { + _mm_storeu_si128(reinterpret_cast<__m128i *>(dst + size - 16), _mm_loadu_si128(reinterpret_cast(src + size - 16))); + + while (size > 16) + { + _mm_storeu_si128(reinterpret_cast<__m128i *>(dst), _mm_loadu_si128(reinterpret_cast(src))); + dst += 16; + src += 16; + size -= 16; + } + } + else + { + /// Align destination to 16 bytes boundary. + size_t padding = (16 - (reinterpret_cast(dst) & 15)) & 15; + + if (padding > 0) + { + __m128i head = _mm_loadu_si128(reinterpret_cast(src)); + _mm_storeu_si128(reinterpret_cast<__m128i*>(dst), head); + dst += padding; + src += padding; + size -= padding; + } + + /// Aligned unrolled copy. + __m128i c0, c1, c2, c3, c4, c5, c6, c7; + + while (size >= 128) + { + c0 = _mm_loadu_si128(reinterpret_cast(src) + 0); + c1 = _mm_loadu_si128(reinterpret_cast(src) + 1); + c2 = _mm_loadu_si128(reinterpret_cast(src) + 2); + c3 = _mm_loadu_si128(reinterpret_cast(src) + 3); + c4 = _mm_loadu_si128(reinterpret_cast(src) + 4); + c5 = _mm_loadu_si128(reinterpret_cast(src) + 5); + c6 = _mm_loadu_si128(reinterpret_cast(src) + 6); + c7 = _mm_loadu_si128(reinterpret_cast(src) + 7); + src += 128; + _mm_store_si128((reinterpret_cast<__m128i*>(dst) + 0), c0); + _mm_store_si128((reinterpret_cast<__m128i*>(dst) + 1), c1); + _mm_store_si128((reinterpret_cast<__m128i*>(dst) + 2), c2); + _mm_store_si128((reinterpret_cast<__m128i*>(dst) + 3), c3); + _mm_store_si128((reinterpret_cast<__m128i*>(dst) + 4), c4); + _mm_store_si128((reinterpret_cast<__m128i*>(dst) + 5), c5); + _mm_store_si128((reinterpret_cast<__m128i*>(dst) + 6), c6); + _mm_store_si128((reinterpret_cast<__m128i*>(dst) + 7), c7); + dst += 128; + + size -= 128; + } + + goto tail; + } + } + + return ret; +} + + + +template +void dispatchMemcpyVariants(size_t memcpy_variant, uint8_t * dst, uint8_t * src, size_t size, size_t iterations, size_t num_threads, F && generator) +{ + memcpy_type memcpy_libc = reinterpret_cast(dlsym(RTLD_NEXT, "memcpy")); + + if (memcpy_variant == 1) + test(dst, src, size, iterations, num_threads, std::forward(generator), memcpy); + if (memcpy_variant == 2) + test(dst, src, size, iterations, num_threads, std::forward(generator), memcpy_libc); + if (memcpy_variant == 3) + test(dst, src, size, iterations, num_threads, std::forward(generator), memcpy_erms); + if (memcpy_variant == 4) + test(dst, src, size, iterations, num_threads, std::forward(generator), MemCpy); + if (memcpy_variant == 5) + test(dst, src, size, iterations, num_threads, std::forward(generator), memcpySSE2); + if (memcpy_variant == 6) + test(dst, src, size, iterations, num_threads, std::forward(generator), memcpySSE2Unrolled2); + if (memcpy_variant == 7) + test(dst, src, size, iterations, num_threads, std::forward(generator), memcpySSE2Unrolled4); + if (memcpy_variant == 8) + test(dst, src, size, iterations, num_threads, std::forward(generator), memcpySSE2Unrolled8); +// if (memcpy_variant == 9) +// test(dst, src, size, iterations, num_threads, std::forward(generator), memcpy_fast_avx); + if (memcpy_variant == 10) + test(dst, src, size, iterations, num_threads, std::forward(generator), memcpy_my); +} + +void dispatchVariants(size_t memcpy_variant, size_t generator_variant, uint8_t * dst, uint8_t * src, size_t size, size_t iterations, size_t num_threads) +{ + if (generator_variant == 1) + dispatchMemcpyVariants(memcpy_variant, dst, src, size, iterations, num_threads, generatorUniform<16>); + if (generator_variant == 2) + dispatchMemcpyVariants(memcpy_variant, dst, src, size, iterations, num_threads, generatorUniform<256>); + if (generator_variant == 3) + dispatchMemcpyVariants(memcpy_variant, dst, src, size, iterations, num_threads, generatorUniform<4096>); + if (generator_variant == 4) + dispatchMemcpyVariants(memcpy_variant, dst, src, size, iterations, num_threads, generatorUniform<65536>); + if (generator_variant == 5) + dispatchMemcpyVariants(memcpy_variant, dst, src, size, iterations, num_threads, generatorUniform<1048576>); +} + + +int main(int argc, char ** argv) +{ + size_t size = 1000000000; + if (argc >= 2) + size = std::stoull(argv[1]); + + size_t iterations = 10; + if (argc >= 3) + iterations = std::stoull(argv[2]); + + size_t num_threads = 1; + if (argc >= 4) + num_threads = std::stoull(argv[3]); + + size_t memcpy_variant = 1; + if (argc >= 5) + memcpy_variant = std::stoull(argv[4]); + + size_t generator_variant = 1; + if (argc >= 6) + generator_variant = std::stoull(argv[5]); + + std::unique_ptr src(new uint8_t[size]); + std::unique_ptr dst(new uint8_t[size]); + + /// Fill src with some pattern for validation. + for (size_t i = 0; i < size; ++i) + src[i] = i; + + /// Fill dst to avoid page faults. + memset(dst.get(), 0, size); + + dispatchVariants(memcpy_variant, generator_variant, dst.get(), src.get(), size, iterations, num_threads); + + return 0; +} diff --git a/utils/memcpy-bench/memcpy_jart.S b/utils/memcpy-bench/memcpy_jart.S new file mode 100644 index 00000000000..50430d0abe0 --- /dev/null +++ b/utils/memcpy-bench/memcpy_jart.S @@ -0,0 +1,138 @@ +/*-*- mode:unix-assembly; indent-tabs-mode:t; tab-width:8; coding:utf-8 -*-│ +│vi: set et ft=asm ts=8 tw=8 fenc=utf-8 :vi│ +╞══════════════════════════════════════════════════════════════════════════════╡ +│ Copyright 2020 Justine Alexandra Roberts Tunney │ +│ │ +│ Permission to use, copy, modify, and/or distribute this software for │ +│ any purpose with or without fee is hereby granted, provided that the │ +│ above copyright notice and this permission notice appear in all copies. │ +│ │ +│ THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL │ +│ WARRANTIES WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED │ +│ WARRANTIES OF MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE │ +│ AUTHOR BE LIABLE FOR ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL │ +│ DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR │ +│ PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER │ +│ TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR │ +│ PERFORMANCE OF THIS SOFTWARE. │ +╚─────────────────────────────────────────────────────────────────────────────*/ + +// Copies memory. +// +// DEST and SRC must not overlap, unless DEST≤SRC. +// +// @param rdi is dest +// @param rsi is src +// @param rdx is number of bytes +// @return original rdi copied to rax +// @mode long +// @asyncsignalsafe +memcpy_jart: mov %rdi,%rax +// 𝑠𝑙𝑖𝑑𝑒 + .align 16 + .type memcpy_jart,@function + .size memcpy_jart,.-memcpy_jart + .globl memcpy_jart + +// Copies memory w/ minimal impact ABI. +// +// @param rdi is dest +// @param rsi is src +// @param rdx is number of bytes +// @clob flags,rcx,xmm3,xmm4 +// @mode long +MemCpy: mov $.Lmemcpytab.size,%ecx + cmp %rcx,%rdx + cmovb %rdx,%rcx + jmp *memcpytab(,%rcx,8) +.Lanchorpoint: +.L16r: cmp $1024,%rdx + jae .Lerms +.L16: movdqu -16(%rsi,%rdx),%xmm4 + mov $16,%rcx +0: add $16,%rcx + movdqu -32(%rsi,%rcx),%xmm3 + movdqu %xmm3,-32(%rdi,%rcx) + cmp %rcx,%rdx + ja 0b + movdqu %xmm4,-16(%rdi,%rdx) + pxor %xmm4,%xmm4 + pxor %xmm3,%xmm3 + jmp .L0 +.L8: push %rbx + mov (%rsi),%rcx + mov -8(%rsi,%rdx),%rbx + mov %rcx,(%rdi) + mov %rbx,-8(%rdi,%rdx) +1: pop %rbx +.L0: ret +.L4: push %rbx + mov (%rsi),%ecx + mov -4(%rsi,%rdx),%ebx + mov %ecx,(%rdi) + mov %ebx,-4(%rdi,%rdx) + jmp 1b +.L3: push %rbx + mov (%rsi),%cx + mov -2(%rsi,%rdx),%bx + mov %cx,(%rdi) + mov %bx,-2(%rdi,%rdx) + jmp 1b +.L2: mov (%rsi),%cx + mov %cx,(%rdi) + jmp .L0 +.L1: mov (%rsi),%cl + mov %cl,(%rdi) + jmp .L0 +.Lerms: cmp $1024*1024,%rdx + ja .Lnts + push %rdi + push %rsi + mov %rdx,%rcx + rep movsb + pop %rsi + pop %rdi + jmp .L0 +.Lnts: movdqu (%rsi),%xmm3 + movdqu %xmm3,(%rdi) + lea 16(%rdi),%rcx + and $-16,%rcx + sub %rdi,%rcx + add %rcx,%rdi + add %rcx,%rsi + sub %rcx,%rdx + mov $16,%rcx +0: add $16,%rcx + movdqu -32(%rsi,%rcx),%xmm3 + movntdq %xmm3,-32(%rdi,%rcx) + cmp %rcx,%rdx + ja 0b + sfence + movdqu -16(%rsi,%rdx),%xmm3 + movdqu %xmm3,-16(%rdi,%rdx) + pxor %xmm3,%xmm3 + jmp .L0 + .type MemCpy,@function + .size MemCpy,.-MemCpy + .globl MemCpy + + .section .rodata + .align 8 +memcpytab: + .quad .L0 + .quad .L1 + .quad .L2 + .quad .L3 + .rept 4 + .quad .L4 + .endr + .rept 8 + .quad .L8 + .endr + .rept 16 + .quad .L16 + .endr + .equ .Lmemcpytab.size,(.-memcpytab)/8 + .quad .L16r # SSE + ERMS + NTS + .type memcpytab,@object + .previous From 2ec20c5d23a60043a0539c4a9cc1cd5acf949547 Mon Sep 17 00:00:00 2001 From: Amos Bird Date: Mon, 8 Mar 2021 17:38:07 +0800 Subject: [PATCH 163/544] update and add tests --- src/Storages/MergeTree/MergeTreeData.cpp | 78 ++++--- src/Storages/MergeTree/MergeTreeData.h | 12 +- src/Storages/MergeTree/MergeTreeSettings.cpp | 27 ++- .../MergeTree/registerStorageMergeTree.cpp | 31 --- src/Storages/StorageMergeTree.h | 1 + src/Storages/StorageReplicatedMergeTree.cpp | 16 +- .../test_jbod_balancer/__init__.py | 0 .../config.d/storage_configuration.xml | 29 +++ tests/integration/test_jbod_balancer/test.py | 190 ++++++++++++++++++ 9 files changed, 313 insertions(+), 71 deletions(-) create mode 100644 tests/integration/test_jbod_balancer/__init__.py create mode 100644 tests/integration/test_jbod_balancer/configs/config.d/storage_configuration.xml create mode 100644 tests/integration/test_jbod_balancer/test.py diff --git a/src/Storages/MergeTree/MergeTreeData.cpp b/src/Storages/MergeTree/MergeTreeData.cpp index e59b35ad14b..edc24cd0ba6 100644 --- a/src/Storages/MergeTree/MergeTreeData.cpp +++ b/src/Storages/MergeTree/MergeTreeData.cpp @@ -4147,35 +4147,43 @@ ReservationPtr MergeTreeData::balancedReservation( { const auto & disks = getStoragePolicy()->getVolume(max_volume_index)->getDisks(); std::map disk_occupation; - std::map> disk_parts; + std::map> disk_parts_for_logging; for (const auto & disk : disks) disk_occupation.emplace(disk->getName(), 0); - std::set big_parts; - std::set merging_parts; + std::set committed_big_parts_from_partition; + std::set submerging_big_parts_from_partition; std::lock_guard lock(currently_submerging_emerging_mutex); - for (const auto & part : currently_submerging_parts) + for (const auto & part : currently_submerging_big_parts) { - if (part->isStoredOnDisk() && part->getBytesOnDisk() >= min_bytes_to_rebalance_partition_over_jbod - && part_info.partition_id == part->info.partition_id) - { - merging_parts.insert(part->name); - } + if (part_info.partition_id == part->info.partition_id) + submerging_big_parts_from_partition.insert(part->name); } { auto lock_parts = lockParts(); if (covered_parts.empty()) { - // TODO will it be possible that the covering_part exists when a fetch is upon execution? + // It's a part fetch. Calculate `covered_parts` here. MergeTreeData::DataPartPtr covering_part; covered_parts = getActivePartsToReplace(part_info, part_name, covering_part, lock_parts); } - // Also include current submerging parts + // Remove irrelevant parts. + covered_parts.erase( + std::remove_if( + covered_parts.begin(), + covered_parts.end(), + [min_bytes_to_rebalance_partition_over_jbod](const auto & part) + { + return !(part->isStoredOnDisk() && part->getBytesOnDisk() >= min_bytes_to_rebalance_partition_over_jbod); + }), + covered_parts.end()); + + // Include current submerging big parts which are not yet in `currently_submerging_big_parts` for (const auto & part : covered_parts) - merging_parts.insert(part->name); + submerging_big_parts_from_partition.insert(part->name); for (const auto & part : getDataPartsStateRange(MergeTreeData::DataPartState::Committed)) { @@ -4186,33 +4194,42 @@ ReservationPtr MergeTreeData::balancedReservation( auto it = disk_occupation.find(name); if (it != disk_occupation.end()) { - if (merging_parts.find(part->name) == merging_parts.end()) + if (submerging_big_parts_from_partition.find(part->name) == submerging_big_parts_from_partition.end()) { it->second += part->getBytesOnDisk(); - disk_parts[name].push_back(formatReadableSizeWithBinarySuffix(part->getBytesOnDisk())); - big_parts.insert(part->name); + disk_parts_for_logging[name].push_back(formatReadableSizeWithBinarySuffix(part->getBytesOnDisk())); + committed_big_parts_from_partition.insert(part->name); } else { - disk_parts[name].push_back(formatReadableSizeWithBinarySuffix(part->getBytesOnDisk()) + " (submerging)"); + disk_parts_for_logging[name].push_back(formatReadableSizeWithBinarySuffix(part->getBytesOnDisk()) + " (submerging)"); } } + else + { + // Part is on different volume. Ignore it. + } } } } - for (const auto & [name, emerging_part] : currently_emerging_parts) + for (const auto & [name, emerging_part] : currently_emerging_big_parts) { - // It's possible that the emerging parts are committed and get added twice. Thus a set is used to deduplicate. - if (big_parts.find(name) == big_parts.end()) + // It's possible that the emerging big parts are committed and get added twice. Thus a set is used to deduplicate. + if (committed_big_parts_from_partition.find(name) == committed_big_parts_from_partition.end() + && part_info.partition_id == emerging_part.partition_id) { auto it = disk_occupation.find(emerging_part.disk_name); if (it != disk_occupation.end()) { it->second += emerging_part.estimate_bytes; - disk_parts[emerging_part.disk_name].push_back( + disk_parts_for_logging[emerging_part.disk_name].push_back( formatReadableSizeWithBinarySuffix(emerging_part.estimate_bytes) + " (emerging)"); } + else + { + // Part is on different volume. Ignore it. + } } } @@ -4238,7 +4255,7 @@ ReservationPtr MergeTreeData::balancedReservation( String selected_disk_name = candidates.front(); WriteBufferFromOwnString log_str; writeCString("\nbalancer: \n", log_str); - for (const auto & [disk_name, per_disk_parts] : disk_parts) + for (const auto & [disk_name, per_disk_parts] : disk_parts_for_logging) writeString(fmt::format(" {}: [{}]\n", disk_name, boost::algorithm::join(per_disk_parts, ", ")), log_str); LOG_DEBUG(log, log_str.str()); @@ -4256,17 +4273,18 @@ ReservationPtr MergeTreeData::balancedReservation( if (reserved_space) { - currently_emerging_parts.emplace( + currently_emerging_big_parts.emplace( part_name, EmergingPartInfo{reserved_space->getDisk(0)->getName(), part_info.partition_id, part_size}); for (const auto & part : covered_parts) { - if (currently_submerging_parts.count(part)) - LOG_WARNING(log, "currently_submerging_parts has duplicated part. JBOD might lose balance"); + if (currently_submerging_big_parts.count(part)) + LOG_WARNING(log, "currently_submerging_big_parts contains duplicates. JBOD might lose balance"); else - currently_submerging_parts.insert(part); + currently_submerging_big_parts.insert(part); } + // Record submerging big parts in the tagger to clean them up. tagger_ptr->emplace(*this, part_name, std::move(covered_parts), log); } } @@ -4283,14 +4301,14 @@ CurrentlySubmergingEmergingTagger::~CurrentlySubmergingEmergingTagger() { std::lock_guard lock(storage.currently_submerging_emerging_mutex); - for (const auto & part : parts) + for (const auto & part : submerging_parts) { - if (!storage.currently_submerging_parts.count(part)) - LOG_WARNING(log, "currently_submerging_parts is missing parts. JBOD might lose balance"); + if (!storage.currently_submerging_big_parts.count(part)) + LOG_WARNING(log, "currently_submerging_big_parts doesn't contain part {} to erase. This is a bug", part->name); else - storage.currently_submerging_parts.erase(part); + storage.currently_submerging_big_parts.erase(part); } - storage.currently_emerging_parts.erase(name); + storage.currently_emerging_big_parts.erase(emerging_part_name); } } diff --git a/src/Storages/MergeTree/MergeTreeData.h b/src/Storages/MergeTree/MergeTreeData.h index 27cf2943a7f..e1c351bd323 100644 --- a/src/Storages/MergeTree/MergeTreeData.h +++ b/src/Storages/MergeTree/MergeTreeData.h @@ -768,10 +768,10 @@ public: bool areBackgroundMovesNeeded() const; /// Parts that currently submerging (merging to bigger parts) or emerging - /// (to be appeared after merging finished). This set have to be used + /// (to be appeared after merging finished). These two variables have to be used /// with `currently_submerging_emerging_mutex`. - DataParts currently_submerging_parts; - std::map currently_emerging_parts; + DataParts currently_submerging_big_parts; + std::map currently_emerging_big_parts; /// Mutex for currently_submerging_parts and currently_emerging_parts mutable std::mutex currently_submerging_emerging_mutex; @@ -1014,13 +1014,13 @@ private: struct CurrentlySubmergingEmergingTagger { MergeTreeData & storage; - String name; - MergeTreeData::DataPartsVector parts; + String emerging_part_name; + MergeTreeData::DataPartsVector submerging_parts; Poco::Logger * log; CurrentlySubmergingEmergingTagger( MergeTreeData & storage_, const String & name_, MergeTreeData::DataPartsVector && parts_, Poco::Logger * log_) - : storage(storage_), name(name_), parts(std::move(parts_)), log(log_) + : storage(storage_), emerging_part_name(name_), submerging_parts(std::move(parts_)), log(log_) { } diff --git a/src/Storages/MergeTree/MergeTreeSettings.cpp b/src/Storages/MergeTree/MergeTreeSettings.cpp index e77668e8900..dfaaf4a942b 100644 --- a/src/Storages/MergeTree/MergeTreeSettings.cpp +++ b/src/Storages/MergeTree/MergeTreeSettings.cpp @@ -98,6 +98,31 @@ void MergeTreeSettings::sanityCheck(const Settings & query_settings) const number_of_free_entries_in_pool_to_lower_max_size_of_merge, query_settings.background_pool_size); } -} + // The min_index_granularity_bytes value is 1024 b and index_granularity_bytes is 10 mb by default. + // If index_granularity_bytes is not disabled i.e > 0 b, then always ensure that it's greater than + // min_index_granularity_bytes. This is mainly a safeguard against accidents whereby a really low + // index_granularity_bytes SETTING of 1b can create really large parts with large marks. + if (index_granularity_bytes > 0 && index_granularity_bytes < min_index_granularity_bytes) + { + throw Exception( + ErrorCodes::BAD_ARGUMENTS, + "index_granularity_bytes: {} is lower than specified min_index_granularity_bytes: {}", + index_granularity_bytes, + min_index_granularity_bytes); + } + + // If min_bytes_to_rebalance_partition_over_jbod is not disabled i.e > 0 b, then always ensure that + // it's not less than min_bytes_to_rebalance_partition_over_jbod. This is a safeguard to avoid tiny + // parts to participate JBOD balancer which will slow down the merge process. + if (min_bytes_to_rebalance_partition_over_jbod > 0 + && min_bytes_to_rebalance_partition_over_jbod < max_bytes_to_merge_at_max_space_in_pool / 1024) + { + throw Exception( + ErrorCodes::BAD_ARGUMENTS, + "min_bytes_to_rebalance_partition_over_jbod: {} is lower than specified max_bytes_to_merge_at_max_space_in_pool / 150: {}", + min_bytes_to_rebalance_partition_over_jbod, + max_bytes_to_merge_at_max_space_in_pool / 1024); + } +} } diff --git a/src/Storages/MergeTree/registerStorageMergeTree.cpp b/src/Storages/MergeTree/registerStorageMergeTree.cpp index 45c9aa81492..73e6063219f 100644 --- a/src/Storages/MergeTree/registerStorageMergeTree.cpp +++ b/src/Storages/MergeTree/registerStorageMergeTree.cpp @@ -654,37 +654,6 @@ static StoragePtr create(const StorageFactory::Arguments & args) // updates the default storage_settings with settings specified via SETTINGS arg in a query if (args.storage_def->settings) metadata.settings_changes = args.storage_def->settings->ptr(); - - size_t index_granularity_bytes = 0; - size_t min_index_granularity_bytes = 0; - - index_granularity_bytes = storage_settings->index_granularity_bytes; - min_index_granularity_bytes = storage_settings->min_index_granularity_bytes; - - /* the min_index_granularity_bytes value is 1024 b and index_granularity_bytes is 10 mb by default - * if index_granularity_bytes is not disabled i.e > 0 b, then always ensure that it's greater than - * min_index_granularity_bytes. This is mainly a safeguard against accidents whereby a really low - * index_granularity_bytes SETTING of 1b can create really large parts with large marks. - */ - if (index_granularity_bytes > 0 && index_granularity_bytes < min_index_granularity_bytes) - { - throw Exception( - "index_granularity_bytes: " + std::to_string(index_granularity_bytes) - + " is lesser than specified min_index_granularity_bytes: " + std::to_string(min_index_granularity_bytes), - ErrorCodes::BAD_ARGUMENTS); - } - - // Pre-define a reasonable minimum size for the JBOD rebalancer - static constexpr size_t MIN_BYTES_TO_REBALANCE_OVER_JBOD = 100 * 1024 * 1024; - if (storage_settings->min_bytes_to_rebalance_partition_over_jbod > 0 - && storage_settings->min_bytes_to_rebalance_partition_over_jbod < MIN_BYTES_TO_REBALANCE_OVER_JBOD) - { - throw Exception( - "min_bytes_to_rebalance_partition_over_jbod: " - + std::to_string(storage_settings->min_bytes_to_rebalance_partition_over_jbod) + " is lesser than " - + std::to_string(MIN_BYTES_TO_REBALANCE_OVER_JBOD), - ErrorCodes::BAD_ARGUMENTS); - } } else { diff --git a/src/Storages/StorageMergeTree.h b/src/Storages/StorageMergeTree.h index ab1de9944e9..246ce151a02 100644 --- a/src/Storages/StorageMergeTree.h +++ b/src/Storages/StorageMergeTree.h @@ -151,6 +151,7 @@ private: FutureMergedMutatedPart future_part; ReservationPtr reserved_space; StorageMergeTree & storage; + // Optional tagger to maintain volatile parts for the JBOD balancer std::optional tagger; CurrentlyMergingPartsTagger( diff --git a/src/Storages/StorageReplicatedMergeTree.cpp b/src/Storages/StorageReplicatedMergeTree.cpp index 9bea36e6252..54ca48ef24d 100644 --- a/src/Storages/StorageReplicatedMergeTree.cpp +++ b/src/Storages/StorageReplicatedMergeTree.cpp @@ -3562,6 +3562,7 @@ bool StorageReplicatedMergeTree::fetchPart(const String & part_name, const Stora } + std::optional tagger_ptr; std::function get_part; if (part_to_clone) { @@ -3585,9 +3586,18 @@ bool StorageReplicatedMergeTree::fetchPart(const String & part_name, const Stora ErrorCodes::INTERSERVER_SCHEME_DOESNT_MATCH); return fetcher.fetchPart( - metadata_snapshot, part_name, source_replica_path, - address.host, address.replication_port, - timeouts, user_password.first, user_password.second, interserver_scheme, to_detached); + metadata_snapshot, + part_name, + source_replica_path, + address.host, + address.replication_port, + timeouts, + user_password.first, + user_password.second, + interserver_scheme, + to_detached, + "", + &tagger_ptr); }; } diff --git a/tests/integration/test_jbod_balancer/__init__.py b/tests/integration/test_jbod_balancer/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tests/integration/test_jbod_balancer/configs/config.d/storage_configuration.xml b/tests/integration/test_jbod_balancer/configs/config.d/storage_configuration.xml new file mode 100644 index 00000000000..62b0ffacaf0 --- /dev/null +++ b/tests/integration/test_jbod_balancer/configs/config.d/storage_configuration.xml @@ -0,0 +1,29 @@ + + + + + 1024 + + + /jbod1/ + + + /jbod2/ + + + /jbod3/ + + + + + + + jbod1 + jbod2 + jbod3 + + + + + + diff --git a/tests/integration/test_jbod_balancer/test.py b/tests/integration/test_jbod_balancer/test.py new file mode 100644 index 00000000000..0dfe1314196 --- /dev/null +++ b/tests/integration/test_jbod_balancer/test.py @@ -0,0 +1,190 @@ +import json +import random +import re +import string +import threading +import time +from multiprocessing.dummy import Pool + +import pytest +from helpers.client import QueryRuntimeException +from helpers.cluster import ClickHouseCluster + +cluster = ClickHouseCluster(__file__) + +node1 = cluster.add_instance( + "node1", + main_configs=["configs/config.d/storage_configuration.xml",], + with_zookeeper=True, + stay_alive=True, + tmpfs=["/jbod1:size=100M", "/jbod2:size=100M", "/jbod3:size=100M"], + macros={"shard": 0, "replica": 1}, +) + + +node2 = cluster.add_instance( + "node2", + main_configs=["configs/config.d/storage_configuration.xml"], + with_zookeeper=True, + stay_alive=True, + tmpfs=["/jbod1:size=100M", "/jbod2:size=100M", "/jbod3:size=100M"], + macros={"shard": 0, "replica": 2}, +) + + +@pytest.fixture(scope="module") +def start_cluster(): + try: + cluster.start() + yield cluster + + finally: + cluster.shutdown() + + +def check_balance(node, table): + + partitions = node.query( + """ + WITH + arraySort(groupArray(c)) AS array_c, + arrayEnumerate(array_c) AS array_i, + sum(c) AS sum_c, + count() AS n, + if(sum_c = 0, 0, (((2. * arraySum(arrayMap((c, i) -> (c * i), array_c, array_i))) / n) / sum_c) - (((n + 1) * 1.) / n)) AS gini + SELECT + partition + FROM + ( + SELECT + partition, + disk_name, + sum(bytes_on_disk) AS c + FROM system.parts + WHERE active AND level > 0 AND disk_name like 'jbod%' AND table = '{}' + GROUP BY + partition, disk_name + ) + GROUP BY partition + HAVING gini < 0.1 + """.format( + table + ) + ).splitlines() + + assert set(partitions) == set(["0", "1"]) + + +def test_jbod_balanced_merge(start_cluster): + try: + node1.query( + """ + CREATE TABLE tbl (p UInt8, d String) + ENGINE = MergeTree + PARTITION BY p + ORDER BY tuple() + SETTINGS + storage_policy = 'jbod', + min_bytes_to_rebalance_partition_over_jbod = 1024, + max_bytes_to_merge_at_max_space_in_pool = 4096 + """ + ) + node1.query("create table tmp1 as tbl") + node1.query("create table tmp2 as tbl") + + for i in range(200): + # around 1k per block + node1.query( + "insert into tbl select randConstant() % 2, randomPrintableASCII(16) from numbers(50)" + ) + node1.query( + "insert into tmp1 select randConstant() % 2, randomPrintableASCII(16) from numbers(50)" + ) + node1.query( + "insert into tmp2 select randConstant() % 2, randomPrintableASCII(16) from numbers(50)" + ) + + time.sleep(1) + + check_balance(node1, "tbl") + + finally: + node1.query(f"DROP TABLE IF EXISTS tbl SYNC") + node1.query(f"DROP TABLE IF EXISTS tmp1 SYNC") + node1.query(f"DROP TABLE IF EXISTS tmp2 SYNC") + + +def test_replicated_balanced_merge_fetch(start_cluster): + try: + for i, node in enumerate([node1, node2]): + node.query( + """ + CREATE TABLE tbl (p UInt8, d String) + ENGINE = ReplicatedMergeTree('/clickhouse/tbl', '{}') + PARTITION BY p + ORDER BY tuple() + SETTINGS + storage_policy = 'jbod', + old_parts_lifetime = 1, + cleanup_delay_period = 1, + cleanup_delay_period_random_add = 2, + min_bytes_to_rebalance_partition_over_jbod = 1024, + max_bytes_to_merge_at_max_space_in_pool = 4096 + """.format( + i + ) + ) + + node.query( + """ + CREATE TABLE tmp1 (p UInt8, d String) + ENGINE = MergeTree + PARTITION BY p + ORDER BY tuple() + SETTINGS + storage_policy = 'jbod', + min_bytes_to_rebalance_partition_over_jbod = 1024, + max_bytes_to_merge_at_max_space_in_pool = 4096 + """ + ) + + node.query("create table tmp2 as tmp1") + + node2.query("alter table tbl modify setting always_fetch_merged_part = 1") + + for i in range(200): + # around 1k per block + node1.query( + "insert into tbl select randConstant() % 2, randomPrintableASCII(16) from numbers(50)" + ) + node1.query( + "insert into tmp1 select randConstant() % 2, randomPrintableASCII(16) from numbers(50)" + ) + node1.query( + "insert into tmp2 select randConstant() % 2, randomPrintableASCII(16) from numbers(50)" + ) + node2.query( + "insert into tmp1 select randConstant() % 2, randomPrintableASCII(16) from numbers(50)" + ) + node2.query( + "insert into tmp2 select randConstant() % 2, randomPrintableASCII(16) from numbers(50)" + ) + + time.sleep(2) + + for _ in range(10): + try: + print("Syncing replica") + node2.query("SYSTEM SYNC REPLICA tbl") + break + except: + time.sleep(0.5) + + check_balance(node1, "tbl") + check_balance(node2, "tbl") + + finally: + for node in [node1, node2]: + node.query("DROP TABLE IF EXISTS tbl SYNC") + node.query("DROP TABLE IF EXISTS tmp1 SYNC") + node.query("DROP TABLE IF EXISTS tmp2 SYNC") From f0342eda9c7d594746171dbf4f4b63b1d47b0c93 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Mon, 8 Mar 2021 13:05:18 +0300 Subject: [PATCH 164/544] Add experimental memcpy implementation --- base/glibc-compatibility/memcpy/CMakeLists.txt | 7 ++++--- programs/main.cpp | 4 ++++ utils/memcpy-bench/memcpy-bench.cpp | 12 ++++++++++++ 3 files changed, 20 insertions(+), 3 deletions(-) diff --git a/base/glibc-compatibility/memcpy/CMakeLists.txt b/base/glibc-compatibility/memcpy/CMakeLists.txt index b51ebab1d50..e8de76151bc 100644 --- a/base/glibc-compatibility/memcpy/CMakeLists.txt +++ b/base/glibc-compatibility/memcpy/CMakeLists.txt @@ -1,5 +1,6 @@ -enable_language(ASM) -add_library(memcpy STATIC memcpy.S) +add_library(memcpy STATIC memcpy.cpp) -# We allow to include memcpy.h from user code for better inlining (less number of registers are clobbered?). +# We allow to include memcpy.h from user code for better inlining. target_include_directories(memcpy PUBLIC $) + +target_compile_options(memcpy PRIVATE -fno-builtin-memcpy) diff --git a/programs/main.cpp b/programs/main.cpp index cbb22b7a87b..6f23edb7113 100644 --- a/programs/main.cpp +++ b/programs/main.cpp @@ -23,6 +23,8 @@ #include #include +#include + /// Universal executable for various clickhouse applications #if ENABLE_CLICKHOUSE_SERVER @@ -342,6 +344,8 @@ int main(int argc_, char ** argv_) inside_main = true; SCOPE_EXIT({ inside_main = false; }); + init_memcpy(); + /// Reset new handler to default (that throws std::bad_alloc) /// It is needed because LLVM library clobbers it. std::set_new_handler(nullptr); diff --git a/utils/memcpy-bench/memcpy-bench.cpp b/utils/memcpy-bench/memcpy-bench.cpp index d4f886398da..05029ee5a69 100644 --- a/utils/memcpy-bench/memcpy-bench.cpp +++ b/utils/memcpy-bench/memcpy-bench.cpp @@ -394,6 +394,18 @@ tail: } else if (have_avx) { + if (size <= 32) + { + __builtin_memcpy(dst, src, 8); + __builtin_memcpy(dst + 8, src + 8, 8); + + dst += 16; + src += 16; + size -= 16; + + goto tail; + } + if (size <= 256) { __asm__( From 3fb5b247ed615963c9fedb21acd6caf7ed8e375e Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Mon, 8 Mar 2021 13:19:25 +0300 Subject: [PATCH 165/544] Evaluate another memcpy --- base/glibc-compatibility/memcpy/memcpy.cpp | 22 +++ base/glibc-compatibility/memcpy/memcpy.h | 184 +++++++++++++++++++++ programs/main.cpp | 4 + 3 files changed, 210 insertions(+) create mode 100644 base/glibc-compatibility/memcpy/memcpy.cpp create mode 100644 base/glibc-compatibility/memcpy/memcpy.h diff --git a/base/glibc-compatibility/memcpy/memcpy.cpp b/base/glibc-compatibility/memcpy/memcpy.cpp new file mode 100644 index 00000000000..c67e4eb54ca --- /dev/null +++ b/base/glibc-compatibility/memcpy/memcpy.cpp @@ -0,0 +1,22 @@ +#include "memcpy.h" + +#include +#include + + +bool have_avx = false; + +void init_memcpy() +{ + uint32_t eax; + uint32_t ebx; + uint32_t ecx; + uint32_t edx; + __cpuid(1, eax, ebx, ecx, edx); + have_avx = (ecx >> 28) & 1; +} + +extern "C" void * memcpy(void * __restrict dst, const void * __restrict src, size_t size) +{ + return inline_memcpy(reinterpret_cast(dst), reinterpret_cast(src), size); +} diff --git a/base/glibc-compatibility/memcpy/memcpy.h b/base/glibc-compatibility/memcpy/memcpy.h new file mode 100644 index 00000000000..034df74d832 --- /dev/null +++ b/base/glibc-compatibility/memcpy/memcpy.h @@ -0,0 +1,184 @@ +#include + +#include + + +extern bool have_avx; +void init_memcpy(); + + +static inline char * inline_memcpy(char * __restrict dst, const char * __restrict src, size_t size) +{ + char * ret = dst; + +tail: + if (size <= 16) + { + if (size >= 8) + { + __builtin_memcpy(dst + size - 8, src + size - 8, 8); + __builtin_memcpy(dst, src, 8); + } + else if (size >= 4) + { + __builtin_memcpy(dst + size - 4, src + size - 4, 4); + __builtin_memcpy(dst, src, 4); + } + else if (size >= 2) + { + __builtin_memcpy(dst + size - 2, src + size - 2, 2); + __builtin_memcpy(dst, src, 2); + } + else if (size >= 1) + { + *dst = *src; + } + } + else if (have_avx) + { + if (size <= 32) + { + __builtin_memcpy(dst, src, 8); + __builtin_memcpy(dst + 8, src + 8, 8); + + dst += 16; + src += 16; + size -= 16; + + goto tail; + } + + if (size <= 256) + { + __asm__( + "vmovups -0x20(%[s],%[size],1), %%ymm0\n" + "vmovups %%ymm0, -0x20(%[d],%[size],1)\n" + : [d]"+r"(dst), [s]"+r"(src) + : [size]"r"(size) + : "ymm0", "memory"); + + while (size > 32) + { + __asm__( + "vmovups (%[s]), %%ymm0\n" + "vmovups %%ymm0, (%[d])\n" + : [d]"+r"(dst), [s]"+r"(src) + : + : "ymm0", "memory"); + + dst += 32; + src += 32; + size -= 32; + } + } + else + { + size_t padding = (32 - (reinterpret_cast(dst) & 31)) & 31; + + if (padding > 0) + { + __asm__( + "vmovups (%[s]), %%ymm0\n" + "vmovups %%ymm0, (%[d])\n" + : [d]"+r"(dst), [s]"+r"(src) + : + : "ymm0", "memory"); + + dst += padding; + src += padding; + size -= padding; + } + + while (size >= 256) + { + __asm__( + "vmovups (%[s]), %%ymm0\n" + "vmovups 0x20(%[s]), %%ymm1\n" + "vmovups 0x40(%[s]), %%ymm2\n" + "vmovups 0x60(%[s]), %%ymm3\n" + "vmovups 0x80(%[s]), %%ymm4\n" + "vmovups 0xa0(%[s]), %%ymm5\n" + "vmovups 0xc0(%[s]), %%ymm6\n" + "vmovups 0xe0(%[s]), %%ymm7\n" + "add $0x100,%[s]\n" + "vmovaps %%ymm0, (%[d])\n" + "vmovaps %%ymm1, 0x20(%[d])\n" + "vmovaps %%ymm2, 0x40(%[d])\n" + "vmovaps %%ymm3, 0x60(%[d])\n" + "vmovaps %%ymm4, 0x80(%[d])\n" + "vmovaps %%ymm5, 0xa0(%[d])\n" + "vmovaps %%ymm6, 0xc0(%[d])\n" + "vmovaps %%ymm7, 0xe0(%[d])\n" + "add $0x100, %[d]\n" + : [d]"+r"(dst), [s]"+r"(src) + : + : "ymm0", "ymm1", "ymm2", "ymm3", "ymm4", "ymm5", "ymm6", "ymm7", "memory"); + + size -= 256; + } + + goto tail; + } + } + else + { + if (size <= 128) + { + _mm_storeu_si128(reinterpret_cast<__m128i *>(dst + size - 16), _mm_loadu_si128(reinterpret_cast(src + size - 16))); + + while (size > 16) + { + _mm_storeu_si128(reinterpret_cast<__m128i *>(dst), _mm_loadu_si128(reinterpret_cast(src))); + dst += 16; + src += 16; + size -= 16; + } + } + else + { + /// Align destination to 16 bytes boundary. + size_t padding = (16 - (reinterpret_cast(dst) & 15)) & 15; + + if (padding > 0) + { + __m128i head = _mm_loadu_si128(reinterpret_cast(src)); + _mm_storeu_si128(reinterpret_cast<__m128i*>(dst), head); + dst += padding; + src += padding; + size -= padding; + } + + /// Aligned unrolled copy. + __m128i c0, c1, c2, c3, c4, c5, c6, c7; + + while (size >= 128) + { + c0 = _mm_loadu_si128(reinterpret_cast(src) + 0); + c1 = _mm_loadu_si128(reinterpret_cast(src) + 1); + c2 = _mm_loadu_si128(reinterpret_cast(src) + 2); + c3 = _mm_loadu_si128(reinterpret_cast(src) + 3); + c4 = _mm_loadu_si128(reinterpret_cast(src) + 4); + c5 = _mm_loadu_si128(reinterpret_cast(src) + 5); + c6 = _mm_loadu_si128(reinterpret_cast(src) + 6); + c7 = _mm_loadu_si128(reinterpret_cast(src) + 7); + src += 128; + _mm_store_si128((reinterpret_cast<__m128i*>(dst) + 0), c0); + _mm_store_si128((reinterpret_cast<__m128i*>(dst) + 1), c1); + _mm_store_si128((reinterpret_cast<__m128i*>(dst) + 2), c2); + _mm_store_si128((reinterpret_cast<__m128i*>(dst) + 3), c3); + _mm_store_si128((reinterpret_cast<__m128i*>(dst) + 4), c4); + _mm_store_si128((reinterpret_cast<__m128i*>(dst) + 5), c5); + _mm_store_si128((reinterpret_cast<__m128i*>(dst) + 6), c6); + _mm_store_si128((reinterpret_cast<__m128i*>(dst) + 7), c7); + dst += 128; + + size -= 128; + } + + goto tail; + } + } + + return ret; +} + diff --git a/programs/main.cpp b/programs/main.cpp index 6f23edb7113..54b36fb7781 100644 --- a/programs/main.cpp +++ b/programs/main.cpp @@ -23,7 +23,11 @@ #include #include +#if defined(__x86_64__) #include +#else +void init_memcpy() {} +#endif /// Universal executable for various clickhouse applications From e02de2355ec0a8d9cfb127c223281db113550f70 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Mon, 8 Mar 2021 13:22:17 +0300 Subject: [PATCH 166/544] Remove useless files --- base/glibc-compatibility/CMakeLists.txt | 3 --- base/glibc-compatibility/tests/CMakeLists.txt | 2 -- .../glibc-compatibility/tests/memcpy_test.cpp | 26 ------------------- contrib/FastMemcpy/memcpy_wrapper.cpp | 6 ----- 4 files changed, 37 deletions(-) delete mode 100644 base/glibc-compatibility/tests/CMakeLists.txt delete mode 100644 base/glibc-compatibility/tests/memcpy_test.cpp delete mode 100644 contrib/FastMemcpy/memcpy_wrapper.cpp diff --git a/base/glibc-compatibility/CMakeLists.txt b/base/glibc-compatibility/CMakeLists.txt index 1fc537ded24..cdd5ec61833 100644 --- a/base/glibc-compatibility/CMakeLists.txt +++ b/base/glibc-compatibility/CMakeLists.txt @@ -54,9 +54,6 @@ if (GLIBC_COMPATIBILITY) message (STATUS "Some symbols from glibc will be replaced for compatibility") - if (ENABLE_TESTS) - add_subdirectory (tests) - endif () elseif (YANDEX_OFFICIAL_BUILD) message (WARNING "Option GLIBC_COMPATIBILITY must be turned on for production builds.") endif () diff --git a/base/glibc-compatibility/tests/CMakeLists.txt b/base/glibc-compatibility/tests/CMakeLists.txt deleted file mode 100644 index f2978a86664..00000000000 --- a/base/glibc-compatibility/tests/CMakeLists.txt +++ /dev/null @@ -1,2 +0,0 @@ -add_executable(memcpy_test memcpy_test.cpp) -target_link_libraries(memcpy_test common memcpy) diff --git a/base/glibc-compatibility/tests/memcpy_test.cpp b/base/glibc-compatibility/tests/memcpy_test.cpp deleted file mode 100644 index a1f782380b0..00000000000 --- a/base/glibc-compatibility/tests/memcpy_test.cpp +++ /dev/null @@ -1,26 +0,0 @@ -#include -#include -#include - -__attribute__((__noinline__)) void memcpy_noinline(void * __restrict dst, const void * __restrict src, size_t size) -{ - memcpy(dst, src, size); -} - - -int main(int, char **) -{ - constexpr size_t buf_size = 100; - char buf[buf_size]{}; - memcpy_noinline(buf, "abc", 3); - - size_t bytes_to_copy = 3; - while (bytes_to_copy * 2 < buf_size) - { - memcpy_noinline(&buf[bytes_to_copy], buf, bytes_to_copy); - bytes_to_copy *= 2; - } - - std::cerr << buf << "\n"; - return 0; -} diff --git a/contrib/FastMemcpy/memcpy_wrapper.cpp b/contrib/FastMemcpy/memcpy_wrapper.cpp deleted file mode 100644 index 8fa6cf291ec..00000000000 --- a/contrib/FastMemcpy/memcpy_wrapper.cpp +++ /dev/null @@ -1,6 +0,0 @@ -#include "FastMemcpy.h" - -extern "C" void * memcpy(void * __restrict destination, const void * __restrict source, size_t size) -{ - return memcpy_fast(destination, source, size); -} From 62aa7bdd74a7cc67f156b039df63a839b3238e56 Mon Sep 17 00:00:00 2001 From: lehasm Date: Mon, 8 Mar 2021 17:54:44 +0300 Subject: [PATCH 167/544] =?UTF-8?q?=D0=AD=D0=BA=D1=80=D0=B0=D0=BD=D0=B8?= =?UTF-8?q?=D1=80=D0=BE=D0=B2=D0=B0=D0=BD=D0=BD=D1=8B=D0=B5=20=D0=BA=D0=BE?= =?UTF-8?q?=D0=B4=D0=BE=D0=B2=D1=8B=D0=B5=20=D0=BF=D0=BE=D0=B7=D0=B8=D1=86?= =?UTF-8?q?=D0=B8=D0=B8=20Unicode=20=D0=B7=D0=B0=D0=BC=D0=B5=D0=BD=D0=B5?= =?UTF-8?q?=D0=BD=D1=8B=20=D0=BD=D0=B0=20=D1=81=D0=B8=D0=BC=D0=B2=D0=BE?= =?UTF-8?q?=D0=BB=D1=8B?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- docs/ru/commercial/cloud.md | 2 +- docs/ru/commercial/index.md | 8 ++++---- docs/ru/development/architecture.md | 2 +- docs/ru/development/browse-code.md | 2 +- docs/ru/development/contrib.md | 2 +- docs/ru/development/developer-instruction.md | 2 +- docs/ru/development/style.md | 2 +- docs/ru/engines/database-engines/index.md | 4 ++-- docs/ru/engines/index.md | 2 +- docs/ru/engines/table-engines/index.md | 4 ++-- docs/ru/engines/table-engines/integrations/index.md | 2 +- docs/ru/engines/table-engines/log-family/index.md | 4 ++-- .../mergetree-family/custom-partitioning-key.md | 2 +- docs/ru/engines/table-engines/mergetree-family/index.md | 2 +- .../engines/table-engines/mergetree-family/replication.md | 2 +- docs/ru/engines/table-engines/special/external-data.md | 2 +- docs/ru/engines/table-engines/special/index.md | 2 +- docs/ru/faq/general/ne-tormozit.md | 4 ++-- docs/ru/getting-started/example-datasets/criteo.md | 2 +- docs/ru/getting-started/example-datasets/index.md | 4 ++-- docs/ru/getting-started/example-datasets/metrica.md | 2 +- docs/ru/getting-started/example-datasets/nyc-taxi.md | 2 +- docs/ru/getting-started/index.md | 2 +- docs/ru/getting-started/install.md | 2 +- docs/ru/guides/apply-catboost-model.md | 2 +- docs/ru/guides/index.md | 4 ++-- docs/ru/index.md | 2 +- docs/ru/interfaces/cli.md | 2 +- docs/ru/interfaces/cpp.md | 2 +- docs/ru/interfaces/formats.md | 2 +- docs/ru/interfaces/http.md | 2 +- docs/ru/interfaces/index.md | 4 ++-- docs/ru/interfaces/jdbc.md | 2 +- docs/ru/interfaces/mysql.md | 2 +- docs/ru/interfaces/odbc.md | 2 +- docs/ru/interfaces/tcp.md | 2 +- docs/ru/interfaces/third-party/client-libraries.md | 2 +- docs/ru/interfaces/third-party/gui.md | 2 +- docs/ru/interfaces/third-party/index.md | 2 +- docs/ru/interfaces/third-party/integrations.md | 2 +- docs/ru/interfaces/third-party/proxy.md | 2 +- docs/ru/introduction/distinctive-features.md | 2 +- docs/ru/introduction/history.md | 2 +- docs/ru/introduction/index.md | 2 +- docs/ru/introduction/performance.md | 2 +- docs/ru/operations/access-rights.md | 2 +- docs/ru/operations/backup.md | 2 +- docs/ru/operations/configuration-files.md | 2 +- docs/ru/operations/index.md | 4 ++-- docs/ru/operations/monitoring.md | 2 +- docs/ru/operations/quotas.md | 2 +- docs/ru/operations/requirements.md | 2 +- .../operations/server-configuration-parameters/index.md | 4 ++-- .../server-configuration-parameters/settings.md | 2 +- docs/ru/operations/settings/constraints-on-settings.md | 2 +- docs/ru/operations/settings/permissions-for-queries.md | 2 +- docs/ru/operations/settings/query-complexity.md | 2 +- docs/ru/operations/settings/settings-profiles.md | 2 +- docs/ru/operations/settings/settings-users.md | 2 +- docs/ru/operations/settings/settings.md | 2 +- docs/ru/operations/system-tables/index.md | 2 +- docs/ru/operations/tips.md | 2 +- docs/ru/operations/troubleshooting.md | 2 +- docs/ru/operations/update.md | 2 +- docs/ru/operations/utilities/index.md | 4 ++-- docs/ru/sql-reference/aggregate-functions/combinators.md | 2 +- docs/ru/sql-reference/aggregate-functions/index.md | 6 +++--- .../aggregate-functions/parametric-functions.md | 2 +- .../sql-reference/aggregate-functions/reference/index.md | 2 +- docs/ru/sql-reference/data-types/boolean.md | 2 +- docs/ru/sql-reference/data-types/domains/index.md | 4 ++-- docs/ru/sql-reference/data-types/index.md | 4 ++-- .../data-types/nested-data-structures/index.md | 2 +- .../sql-reference/data-types/special-data-types/index.md | 2 +- .../external-dicts-dict-hierarchical.md | 2 +- .../external-dictionaries/external-dicts-dict-layout.md | 2 +- .../external-dictionaries/external-dicts-dict-lifetime.md | 2 +- .../external-dictionaries/external-dicts-dict-sources.md | 2 +- .../external-dicts-dict-structure.md | 2 +- .../external-dictionaries/external-dicts-dict.md | 2 +- .../dictionaries/external-dictionaries/external-dicts.md | 2 +- .../dictionaries/external-dictionaries/index.md | 2 +- docs/ru/sql-reference/dictionaries/index.md | 4 ++-- docs/ru/sql-reference/dictionaries/internal-dicts.md | 2 +- docs/ru/sql-reference/distributed-ddl.md | 3 ++- docs/ru/sql-reference/functions/arithmetic-functions.md | 2 +- docs/ru/sql-reference/functions/array-functions.md | 2 +- docs/ru/sql-reference/functions/array-join.md | 2 +- docs/ru/sql-reference/functions/bit-functions.md | 2 +- docs/ru/sql-reference/functions/bitmap-functions.md | 2 +- docs/ru/sql-reference/functions/comparison-functions.md | 2 +- docs/ru/sql-reference/functions/conditional-functions.md | 2 +- docs/ru/sql-reference/functions/date-time-functions.md | 2 +- docs/ru/sql-reference/functions/encoding-functions.md | 2 +- docs/ru/sql-reference/functions/encryption-functions.md | 2 +- docs/ru/sql-reference/functions/ext-dict-functions.md | 2 +- docs/ru/sql-reference/functions/functions-for-nulls.md | 2 +- docs/ru/sql-reference/functions/geo/coordinates.md | 2 +- docs/ru/sql-reference/functions/geo/geohash.md | 2 +- docs/ru/sql-reference/functions/geo/h3.md | 2 +- docs/ru/sql-reference/functions/geo/index.md | 2 +- docs/ru/sql-reference/functions/hash-functions.md | 2 +- docs/ru/sql-reference/functions/in-functions.md | 2 +- docs/ru/sql-reference/functions/index.md | 4 ++-- docs/ru/sql-reference/functions/introspection.md | 2 +- docs/ru/sql-reference/functions/ip-address-functions.md | 2 +- docs/ru/sql-reference/functions/logical-functions.md | 2 +- .../sql-reference/functions/machine-learning-functions.md | 2 +- docs/ru/sql-reference/functions/math-functions.md | 2 +- docs/ru/sql-reference/functions/other-functions.md | 2 +- docs/ru/sql-reference/functions/random-functions.md | 2 +- docs/ru/sql-reference/functions/rounding-functions.md | 2 +- .../functions/splitting-merging-functions.md | 2 +- docs/ru/sql-reference/functions/string-functions.md | 2 +- .../sql-reference/functions/string-replace-functions.md | 2 +- .../ru/sql-reference/functions/string-search-functions.md | 2 +- .../sql-reference/functions/type-conversion-functions.md | 2 +- docs/ru/sql-reference/functions/url-functions.md | 2 +- docs/ru/sql-reference/functions/uuid-functions.md | 2 +- docs/ru/sql-reference/functions/ym-dict-functions.md | 2 +- docs/ru/sql-reference/index.md | 2 +- docs/ru/sql-reference/operators/index.md | 2 +- docs/ru/sql-reference/statements/alter/column.md | 2 +- docs/ru/sql-reference/statements/alter/constraint.md | 2 +- docs/ru/sql-reference/statements/alter/index/index.md | 2 +- docs/ru/sql-reference/statements/create/database.md | 2 +- docs/ru/sql-reference/statements/create/dictionary.md | 2 +- docs/ru/sql-reference/statements/create/index.md | 2 +- docs/ru/sql-reference/statements/create/quota.md | 2 +- docs/ru/sql-reference/statements/create/role.md | 2 +- docs/ru/sql-reference/statements/create/row-policy.md | 2 +- .../sql-reference/statements/create/settings-profile.md | 2 +- docs/ru/sql-reference/statements/create/table.md | 2 +- docs/ru/sql-reference/statements/create/user.md | 2 +- docs/ru/sql-reference/statements/create/view.md | 2 +- docs/ru/sql-reference/statements/index.md | 2 +- docs/ru/sql-reference/statements/select/index.md | 4 ++-- docs/ru/sql-reference/syntax.md | 2 +- docs/ru/sql-reference/table-functions/index.md | 6 +++--- docs/ru/whats-new/index.md | 2 +- 140 files changed, 163 insertions(+), 162 deletions(-) diff --git a/docs/ru/commercial/cloud.md b/docs/ru/commercial/cloud.md index 4f57592b4c7..8023f738c70 100644 --- a/docs/ru/commercial/cloud.md +++ b/docs/ru/commercial/cloud.md @@ -1,6 +1,6 @@ --- toc_priority: 1 -toc_title: "\u041f\u043e\u0441\u0442\u0430\u0432\u0449\u0438\u043a\u0438\u0020\u043e\u0431\u043b\u0430\u0447\u043d\u044b\u0445\u0020\u0443\u0441\u043b\u0443\u0433\u0020\u0043\u006c\u0069\u0063\u006b\u0048\u006f\u0075\u0073\u0065" +toc_title: "Поставщики облачных услуг ClickHouse" --- # Поставщики облачных услуг ClickHouse {#clickhouse-cloud-service-providers} diff --git a/docs/ru/commercial/index.md b/docs/ru/commercial/index.md index c6c440c17e8..d8d37d0b7ad 100644 --- a/docs/ru/commercial/index.md +++ b/docs/ru/commercial/index.md @@ -1,9 +1,9 @@ --- -toc_folder_title: "\u041A\u043E\u043C\u043C\u0435\u0440\u0447\u0435\u0441\u043A\u0438\ - \u0435 \u0443\u0441\u043B\u0443\u0433\u0438" +toc_folder_title: "Коммерчески\ + е услуги" toc_priority: 70 -toc_title: "\u041A\u043E\u043C\u043C\u0435\u0440\u0447\u0435\u0441\u043A\u0438\u0435\ - \ \u0443\u0441\u043B\u0443\u0433\u0438" +toc_title: "Коммерческие\ + \ услуги" --- # Коммерческие услуги {#clickhouse-commercial-services} diff --git a/docs/ru/development/architecture.md b/docs/ru/development/architecture.md index de8fba1bc4b..9f43fabba4f 100644 --- a/docs/ru/development/architecture.md +++ b/docs/ru/development/architecture.md @@ -1,6 +1,6 @@ --- toc_priority: 62 -toc_title: "\u041e\u0431\u0437\u043e\u0440\u0020\u0430\u0440\u0445\u0438\u0442\u0435\u043a\u0442\u0443\u0440\u044b\u0020\u0043\u006c\u0069\u0063\u006b\u0048\u006f\u0075\u0073\u0065" +toc_title: "Обзор архитектуры ClickHouse" --- # Обзор архитектуры ClickHouse {#overview-of-clickhouse-architecture} diff --git a/docs/ru/development/browse-code.md b/docs/ru/development/browse-code.md index ac17cf0e6f5..3f6de574abe 100644 --- a/docs/ru/development/browse-code.md +++ b/docs/ru/development/browse-code.md @@ -1,6 +1,6 @@ --- toc_priority: 71 -toc_title: "\u041d\u0430\u0432\u0438\u0433\u0430\u0446\u0438\u044f\u0020\u043f\u043e\u0020\u043a\u043e\u0434\u0443\u0020\u0043\u006c\u0069\u0063\u006b\u0048\u006f\u0075\u0073\u0065" +toc_title: "Навигация по коду ClickHouse" --- diff --git a/docs/ru/development/contrib.md b/docs/ru/development/contrib.md index 05367267e41..f3310836ba9 100644 --- a/docs/ru/development/contrib.md +++ b/docs/ru/development/contrib.md @@ -1,6 +1,6 @@ --- toc_priority: 70 -toc_title: "\u0418\u0441\u043f\u043e\u043b\u044c\u0437\u0443\u0435\u043c\u044b\u0435\u0020\u0441\u0442\u043e\u0440\u043e\u043d\u043d\u0438\u0435\u0020\u0431\u0438\u0431\u043b\u0438\u043e\u0442\u0435\u043a\u0438" +toc_title: "Используемые сторонние библиотеки" --- diff --git a/docs/ru/development/developer-instruction.md b/docs/ru/development/developer-instruction.md index 7d88c97fe46..9ddb17b7212 100644 --- a/docs/ru/development/developer-instruction.md +++ b/docs/ru/development/developer-instruction.md @@ -1,6 +1,6 @@ --- toc_priority: 61 -toc_title: "\u0418\u043d\u0441\u0442\u0440\u0443\u043a\u0446\u0438\u044f\u0020\u0434\u043b\u044f\u0020\u0440\u0430\u0437\u0440\u0430\u0431\u043e\u0442\u0447\u0438\u043a\u043e\u0432" +toc_title: "Инструкция для разработчиков" --- # Инструкция для разработчиков diff --git a/docs/ru/development/style.md b/docs/ru/development/style.md index 1b211259bbb..72607ca6bad 100644 --- a/docs/ru/development/style.md +++ b/docs/ru/development/style.md @@ -1,6 +1,6 @@ --- toc_priority: 68 -toc_title: "\u041a\u0430\u043a\u0020\u043f\u0438\u0441\u0430\u0442\u044c\u0020\u043a\u043e\u0434\u0020\u043d\u0430\u0020\u0043\u002b\u002b" +toc_title: "Как писать код на C++" --- diff --git a/docs/ru/engines/database-engines/index.md b/docs/ru/engines/database-engines/index.md index 4dfe766f066..e06c032a636 100644 --- a/docs/ru/engines/database-engines/index.md +++ b/docs/ru/engines/database-engines/index.md @@ -1,7 +1,7 @@ --- -toc_folder_title: "\u0414\u0432\u0438\u0436\u043a\u0438\u0020\u0431\u0430\u0437\u0020\u0434\u0430\u043d\u043d\u044b\u0445" +toc_folder_title: "Движки баз данных" toc_priority: 27 -toc_title: "\u0412\u0432\u0435\u0434\u0435\u043d\u0438\u0435" +toc_title: "Введение" --- # Движки баз данных {#dvizhki-baz-dannykh} diff --git a/docs/ru/engines/index.md b/docs/ru/engines/index.md index 28ccc8bcfe6..fe41ada8fb6 100644 --- a/docs/ru/engines/index.md +++ b/docs/ru/engines/index.md @@ -1,5 +1,5 @@ --- -toc_folder_title: "\u0045\u006e\u0067\u0069\u006e\u0065\u0073" +toc_folder_title: "Engines" toc_hidden: true toc_priority: 25 toc_title: hidden diff --git a/docs/ru/engines/table-engines/index.md b/docs/ru/engines/table-engines/index.md index 740588c50a4..05236eb5b33 100644 --- a/docs/ru/engines/table-engines/index.md +++ b/docs/ru/engines/table-engines/index.md @@ -1,7 +1,7 @@ --- -toc_folder_title: "\u0414\u0432\u0438\u0436\u043a\u0438\u0020\u0442\u0430\u0431\u043b\u0438\u0446" +toc_folder_title: "Движки таблиц" toc_priority: 26 -toc_title: "\u0412\u0432\u0435\u0434\u0435\u043d\u0438\u0435" +toc_title: "Введение" --- diff --git a/docs/ru/engines/table-engines/integrations/index.md b/docs/ru/engines/table-engines/integrations/index.md index db7e527442e..e01d9d0cee2 100644 --- a/docs/ru/engines/table-engines/integrations/index.md +++ b/docs/ru/engines/table-engines/integrations/index.md @@ -1,5 +1,5 @@ --- -toc_folder_title: "\u0414\u0432\u0438\u0436\u043a\u0438\u0020\u0442\u0430\u0431\u043b\u0438\u0446\u0020\u0434\u043b\u044f\u0020\u0438\u043d\u0442\u0435\u0433\u0440\u0430\u0446\u0438\u0438" +toc_folder_title: "Движки таблиц для интеграции" toc_priority: 30 --- diff --git a/docs/ru/engines/table-engines/log-family/index.md b/docs/ru/engines/table-engines/log-family/index.md index 7c6d2f81d7c..b2a56f650f4 100644 --- a/docs/ru/engines/table-engines/log-family/index.md +++ b/docs/ru/engines/table-engines/log-family/index.md @@ -1,6 +1,6 @@ --- -toc_folder_title: "\u0421\u0435\u043c\u0435\u0439\u0441\u0442\u0432\u043e\u0020\u004c\u006f\u0067" -toc_title: "\u0412\u0432\u0435\u0434\u0435\u043d\u0438\u0435" +toc_folder_title: "Семейство Log" +toc_title: "Введение" toc_priority: 29 --- diff --git a/docs/ru/engines/table-engines/mergetree-family/custom-partitioning-key.md b/docs/ru/engines/table-engines/mergetree-family/custom-partitioning-key.md index 2d26528d964..00d850b01c3 100644 --- a/docs/ru/engines/table-engines/mergetree-family/custom-partitioning-key.md +++ b/docs/ru/engines/table-engines/mergetree-family/custom-partitioning-key.md @@ -1,6 +1,6 @@ --- toc_priority: 32 -toc_title: "\u041f\u0440\u043e\u0438\u0437\u0432\u043e\u043b\u044c\u043d\u044b\u0439\u0020\u043a\u043b\u044e\u0447\u0020\u043f\u0430\u0440\u0442\u0438\u0446\u0438\u043e\u043d\u0438\u0440\u043e\u0432\u0430\u043d\u0438\u044f" +toc_title: "Произвольный ключ партиционирования" --- diff --git a/docs/ru/engines/table-engines/mergetree-family/index.md b/docs/ru/engines/table-engines/mergetree-family/index.md index abdfdd77d7f..e184e51c406 100644 --- a/docs/ru/engines/table-engines/mergetree-family/index.md +++ b/docs/ru/engines/table-engines/mergetree-family/index.md @@ -1,5 +1,5 @@ --- toc_folder_title: MergeTree Family toc_priority: 28 -toc_title: "\u0412\u0432\u0435\u0434\u0435\u043d\u0438\u0435" +toc_title: "Введение" --- diff --git a/docs/ru/engines/table-engines/mergetree-family/replication.md b/docs/ru/engines/table-engines/mergetree-family/replication.md index a8a308b104f..1735a02cf4c 100644 --- a/docs/ru/engines/table-engines/mergetree-family/replication.md +++ b/docs/ru/engines/table-engines/mergetree-family/replication.md @@ -1,6 +1,6 @@ --- toc_priority: 31 -toc_title: "\u0420\u0435\u043f\u043b\u0438\u043a\u0430\u0446\u0438\u044f\u0020\u0434\u0430\u043d\u043d\u044b\u0445" +toc_title: "Репликация данных" --- # Репликация данных {#table_engines-replication} diff --git a/docs/ru/engines/table-engines/special/external-data.md b/docs/ru/engines/table-engines/special/external-data.md index 7e383c0c12d..da9e132dd4f 100644 --- a/docs/ru/engines/table-engines/special/external-data.md +++ b/docs/ru/engines/table-engines/special/external-data.md @@ -1,6 +1,6 @@ --- toc_priority: 45 -toc_title: "\u0412\u043d\u0435\u0448\u043d\u0438\u0435\u0020\u0434\u0430\u043d\u043d\u044b\u0435\u0020\u0434\u043b\u044f\u0020\u043e\u0431\u0440\u0430\u0431\u043e\u0442\u043a\u0438\u0020\u0437\u0430\u043f\u0440\u043e\u0441\u0430" +toc_title: "Внешние данные для обработки запроса" --- # Внешние данные для обработки запроса {#vneshnie-dannye-dlia-obrabotki-zaprosa} diff --git a/docs/ru/engines/table-engines/special/index.md b/docs/ru/engines/table-engines/special/index.md index 0d86461dd2d..0300d3ad641 100644 --- a/docs/ru/engines/table-engines/special/index.md +++ b/docs/ru/engines/table-engines/special/index.md @@ -1,5 +1,5 @@ --- -toc_folder_title: "\u0421\u043f\u0435\u0446\u0438\u0430\u043b\u044c\u043d\u044b\u0435\u0020\u0434\u0432\u0438\u0436\u043a\u0438\u0020\u0442\u0430\u0431\u043b\u0438\u0446" +toc_folder_title: "Специальные движки таблиц" toc_priority: 31 --- diff --git a/docs/ru/faq/general/ne-tormozit.md b/docs/ru/faq/general/ne-tormozit.md index 59c8023da37..2eaefbb19c5 100644 --- a/docs/ru/faq/general/ne-tormozit.md +++ b/docs/ru/faq/general/ne-tormozit.md @@ -1,6 +1,6 @@ --- -title: "What does \u201C\u043D\u0435 \u0442\u043E\u0440\u043C\u043E\u0437\u0438\u0442\ - \u201D mean?" +title: "What does “не тормозит\ + ” mean?" toc_hidden: true toc_priority: 11 --- diff --git a/docs/ru/getting-started/example-datasets/criteo.md b/docs/ru/getting-started/example-datasets/criteo.md index 7a58da5b695..ecdc5f5fa41 100644 --- a/docs/ru/getting-started/example-datasets/criteo.md +++ b/docs/ru/getting-started/example-datasets/criteo.md @@ -1,6 +1,6 @@ --- toc_priority: 18 -toc_title: "\u0422\u0435\u0440\u0430\u0431\u0430\u0439\u0442\u0020\u043b\u043e\u0433\u043e\u0432\u0020\u043a\u043b\u0438\u043a\u043e\u0432\u0020\u043e\u0442\u0020\u0043\u0072\u0069\u0074\u0065\u006f" +toc_title: "Терабайт логов кликов от Criteo" --- # Терабайт логов кликов от Criteo {#terabait-logov-klikov-ot-criteo} diff --git a/docs/ru/getting-started/example-datasets/index.md b/docs/ru/getting-started/example-datasets/index.md index eff944a7980..fd89bb122e3 100644 --- a/docs/ru/getting-started/example-datasets/index.md +++ b/docs/ru/getting-started/example-datasets/index.md @@ -1,7 +1,7 @@ --- -toc_folder_title: "\u0422\u0435\u0441\u0442\u043e\u0432\u044b\u0435\u0020\u043c\u0430\u0441\u0441\u0438\u0432\u044b\u0020\u0434\u0430\u043d\u043d\u044b\u0445" +toc_folder_title: "Тестовые массивы данных" toc_priority: 14 -toc_title: "\u0412\u0432\u0435\u0434\u0435\u043d\u0438\u0435" +toc_title: "Введение" --- # Тестовые массивы данных {#testovye-massivy-dannykh} diff --git a/docs/ru/getting-started/example-datasets/metrica.md b/docs/ru/getting-started/example-datasets/metrica.md index 3246eb5178c..7deacdb836c 100644 --- a/docs/ru/getting-started/example-datasets/metrica.md +++ b/docs/ru/getting-started/example-datasets/metrica.md @@ -1,6 +1,6 @@ --- toc_priority: 15 -toc_title: "\u0410\u043d\u043e\u043d\u0438\u043c\u0438\u0437\u0438\u0440\u043e\u0432\u0430\u043d\u043d\u044b\u0435\u0020\u0434\u0430\u043d\u043d\u044b\u0435\u0020\u042f\u043d\u0434\u0435\u043a\u0441\u002e\u041c\u0435\u0442\u0440\u0438\u043a\u0438" +toc_title: "Анонимизированные данные Яндекс.Метрики" --- # Анонимизированные данные Яндекс.Метрики {#anonimizirovannye-dannye-iandeks-metriki} diff --git a/docs/ru/getting-started/example-datasets/nyc-taxi.md b/docs/ru/getting-started/example-datasets/nyc-taxi.md index a4472751a99..891a92e2fa7 100644 --- a/docs/ru/getting-started/example-datasets/nyc-taxi.md +++ b/docs/ru/getting-started/example-datasets/nyc-taxi.md @@ -1,6 +1,6 @@ --- toc_priority: 20 -toc_title: "\u0414\u0430\u043d\u043d\u044b\u0435\u0020\u043e\u0020\u0442\u0430\u043a\u0441\u0438\u0020\u0432\u0020\u041d\u044c\u044e\u002d\u0419\u043e\u0440\u043a\u0435" +toc_title: "Данные о такси в Нью-Йорке" --- # Данные о такси в Нью-Йорке {#dannye-o-taksi-v-niu-iorke} diff --git a/docs/ru/getting-started/index.md b/docs/ru/getting-started/index.md index ab72ce4a1d2..78b56092740 100644 --- a/docs/ru/getting-started/index.md +++ b/docs/ru/getting-started/index.md @@ -1,5 +1,5 @@ --- -toc_folder_title: "\u041d\u0430\u0447\u0430\u043b\u043e\u0020\u0440\u0430\u0431\u043e\u0442\u044b" +toc_folder_title: "Начало работы" toc_hidden: true toc_priority: 8 toc_title: hidden diff --git a/docs/ru/getting-started/install.md b/docs/ru/getting-started/install.md index 04efe77712b..aa5e8d77512 100644 --- a/docs/ru/getting-started/install.md +++ b/docs/ru/getting-started/install.md @@ -1,6 +1,6 @@ --- toc_priority: 11 -toc_title: "\u0423\u0441\u0442\u0430\u043d\u043e\u0432\u043a\u0430" +toc_title: "Установка" --- # Установка {#ustanovka} diff --git a/docs/ru/guides/apply-catboost-model.md b/docs/ru/guides/apply-catboost-model.md index 026b4d9d75e..11964c57fc7 100644 --- a/docs/ru/guides/apply-catboost-model.md +++ b/docs/ru/guides/apply-catboost-model.md @@ -1,6 +1,6 @@ --- toc_priority: 41 -toc_title: "\u041f\u0440\u0438\u043c\u0435\u043d\u0435\u043d\u0438\u0435\u0020\u043c\u043e\u0434\u0435\u043b\u0438\u0020\u0043\u0061\u0074\u0042\u006f\u006f\u0073\u0074\u0020\u0432\u0020\u0043\u006c\u0069\u0063\u006b\u0048\u006f\u0075\u0073\u0065" +toc_title: "Применение модели CatBoost в ClickHouse" --- # Применение модели CatBoost в ClickHouse {#applying-catboost-model-in-clickhouse} diff --git a/docs/ru/guides/index.md b/docs/ru/guides/index.md index 2c38de275a7..5b305a6a135 100644 --- a/docs/ru/guides/index.md +++ b/docs/ru/guides/index.md @@ -1,7 +1,7 @@ --- -toc_folder_title: "\u0420\u0443\u043A\u043E\u0432\u043E\u0434\u0441\u0442\u0432\u0430" +toc_folder_title: "Руководства" toc_priority: 38 -toc_title: "\u041E\u0431\u0437\u043E\u0440" +toc_title: "Обзор" --- # Руководства {#rukovodstva} diff --git a/docs/ru/index.md b/docs/ru/index.md index 617fae7ac6c..26d7dc3bf21 100644 --- a/docs/ru/index.md +++ b/docs/ru/index.md @@ -1,6 +1,6 @@ --- toc_priority: 0 -toc_title: "\u041E\u0431\u0437\u043E\u0440" +toc_title: "Обзор" --- # Что такое ClickHouse {#what-is-clickhouse} diff --git a/docs/ru/interfaces/cli.md b/docs/ru/interfaces/cli.md index b1d8c4f0732..3f6b288fc2b 100644 --- a/docs/ru/interfaces/cli.md +++ b/docs/ru/interfaces/cli.md @@ -1,6 +1,6 @@ --- toc_priority: 17 -toc_title: "\u041a\u043b\u0438\u0435\u043d\u0442\u0020\u043a\u043e\u043c\u0430\u043d\u0434\u043d\u043e\u0439\u0020\u0441\u0442\u0440\u043e\u043a\u0438" +toc_title: "Клиент командной строки" --- # Клиент командной строки {#klient-komandnoi-stroki} diff --git a/docs/ru/interfaces/cpp.md b/docs/ru/interfaces/cpp.md index 264b4f82500..018f4e22e34 100644 --- a/docs/ru/interfaces/cpp.md +++ b/docs/ru/interfaces/cpp.md @@ -1,6 +1,6 @@ --- toc_priority: 24 -toc_title: "\u0043\u002b\u002b\u0020\u043a\u043b\u0438\u0435\u043d\u0442\u0441\u043a\u0430\u044f\u0020\u0431\u0438\u0431\u043b\u0438\u043e\u0442\u0435\u043a\u0430" +toc_title: "C++ клиентская библиотека" --- # C++ клиентская библиотека {#c-klientskaia-biblioteka} diff --git a/docs/ru/interfaces/formats.md b/docs/ru/interfaces/formats.md index 98426b489e8..edea533b642 100644 --- a/docs/ru/interfaces/formats.md +++ b/docs/ru/interfaces/formats.md @@ -1,6 +1,6 @@ --- toc_priority: 21 -toc_title: "\u0424\u043e\u0440\u043c\u0430\u0442\u044b\u0020\u0432\u0445\u043e\u0434\u043d\u044b\u0445\u0020\u0438\u0020\u0432\u044b\u0445\u043e\u0434\u043d\u044b\u0445\u0020\u0434\u0430\u043d\u043d\u044b\u0445" +toc_title: "Форматы входных и выходных данных" --- # Форматы входных и выходных данных {#formats} diff --git a/docs/ru/interfaces/http.md b/docs/ru/interfaces/http.md index d6f930f3f63..5cb50d8f168 100644 --- a/docs/ru/interfaces/http.md +++ b/docs/ru/interfaces/http.md @@ -1,6 +1,6 @@ --- toc_priority: 19 -toc_title: "\u0048\u0054\u0054\u0050\u002d\u0438\u043d\u0442\u0435\u0440\u0444\u0435\u0439\u0441" +toc_title: "HTTP-интерфейс" --- # HTTP-интерфейс {#http-interface} diff --git a/docs/ru/interfaces/index.md b/docs/ru/interfaces/index.md index ea381c46206..fc8743b3c1e 100644 --- a/docs/ru/interfaces/index.md +++ b/docs/ru/interfaces/index.md @@ -1,7 +1,7 @@ --- -toc_folder_title: "\u0418\u043D\u0442\u0435\u0440\u0444\u0435\u0439\u0441\u044B" +toc_folder_title: "Интерфейсы" toc_priority: 14 -toc_title: "\u0412\u0432\u0435\u0434\u0435\u043D\u0438\u0435" +toc_title: "Введение" --- # Интерфейсы {#interfaces} diff --git a/docs/ru/interfaces/jdbc.md b/docs/ru/interfaces/jdbc.md index 196dba64933..ac86375c74f 100644 --- a/docs/ru/interfaces/jdbc.md +++ b/docs/ru/interfaces/jdbc.md @@ -1,6 +1,6 @@ --- toc_priority: 22 -toc_title: "\u004a\u0044\u0042\u0043\u002d\u0434\u0440\u0430\u0439\u0432\u0435\u0440" +toc_title: "JDBC-драйвер" --- # JDBC-драйвер {#jdbc-draiver} diff --git a/docs/ru/interfaces/mysql.md b/docs/ru/interfaces/mysql.md index fa0003e0bea..925b1113109 100644 --- a/docs/ru/interfaces/mysql.md +++ b/docs/ru/interfaces/mysql.md @@ -1,6 +1,6 @@ --- toc_priority: 20 -toc_title: "\u004d\u0079\u0053\u0051\u004c\u002d\u0438\u043d\u0442\u0435\u0440\u0444\u0435\u0439\u0441" +toc_title: "MySQL-интерфейс" --- # MySQL-интерфейс {#mysql-interface} diff --git a/docs/ru/interfaces/odbc.md b/docs/ru/interfaces/odbc.md index 728c4bd6979..7843d3cb943 100644 --- a/docs/ru/interfaces/odbc.md +++ b/docs/ru/interfaces/odbc.md @@ -1,6 +1,6 @@ --- toc_priority: 23 -toc_title: "\u004f\u0044\u0042\u0043\u002d\u0434\u0440\u0430\u0439\u0432\u0435\u0440" +toc_title: "ODBC-драйвер" --- diff --git a/docs/ru/interfaces/tcp.md b/docs/ru/interfaces/tcp.md index d89646f15b7..ea8c170009d 100644 --- a/docs/ru/interfaces/tcp.md +++ b/docs/ru/interfaces/tcp.md @@ -1,6 +1,6 @@ --- toc_priority: 18 -toc_title: "\u0420\u043e\u0434\u043d\u043e\u0439\u0020\u0438\u043d\u0442\u0435\u0440\u0444\u0435\u0439\u0441\u0020\u0028\u0054\u0043\u0050\u0029" +toc_title: "Родной интерфейс (TCP)" --- # Родной интерфейс (TCP) {#rodnoi-interfeis-tcp} diff --git a/docs/ru/interfaces/third-party/client-libraries.md b/docs/ru/interfaces/third-party/client-libraries.md index 97fa382fdd9..65e93731300 100644 --- a/docs/ru/interfaces/third-party/client-libraries.md +++ b/docs/ru/interfaces/third-party/client-libraries.md @@ -1,6 +1,6 @@ --- toc_priority: 26 -toc_title: "\u041a\u043b\u0438\u0435\u043d\u0442\u0441\u043a\u0438\u0435\u0020\u0431\u0438\u0431\u043b\u0438\u043e\u0442\u0435\u043a\u0438\u0020\u043e\u0442\u0020\u0441\u0442\u043e\u0440\u043e\u043d\u043d\u0438\u0445\u0020\u0440\u0430\u0437\u0440\u0430\u0431\u043e\u0442\u0447\u0438\u043a\u043e\u0432" +toc_title: "Клиентские библиотеки от сторонних разработчиков" --- # Клиентские библиотеки от сторонних разработчиков {#klientskie-biblioteki-ot-storonnikh-razrabotchikov} diff --git a/docs/ru/interfaces/third-party/gui.md b/docs/ru/interfaces/third-party/gui.md index 1fabdb8a31c..c02c32e08f4 100644 --- a/docs/ru/interfaces/third-party/gui.md +++ b/docs/ru/interfaces/third-party/gui.md @@ -1,6 +1,6 @@ --- toc_priority: 28 -toc_title: "\u0412\u0438\u0437\u0443\u0430\u043b\u044c\u043d\u044b\u0435\u0020\u0438\u043d\u0442\u0435\u0440\u0444\u0435\u0439\u0441\u044b\u0020\u043e\u0442\u0020\u0441\u0442\u043e\u0440\u043e\u043d\u043d\u0438\u0445\u0020\u0440\u0430\u0437\u0440\u0430\u0431\u043e\u0442\u0447\u0438\u043a\u043e\u0432" +toc_title: "Визуальные интерфейсы от сторонних разработчиков" --- diff --git a/docs/ru/interfaces/third-party/index.md b/docs/ru/interfaces/third-party/index.md index a57169df73b..8b59bb5fd28 100644 --- a/docs/ru/interfaces/third-party/index.md +++ b/docs/ru/interfaces/third-party/index.md @@ -1,5 +1,5 @@ --- -toc_folder_title: "\u0421\u0442\u043e\u0440\u043e\u043d\u043d\u0438\u0435\u0020\u0438\u043d\u0442\u0435\u0440\u0444\u0435\u0439\u0441\u044b" +toc_folder_title: "Сторонние интерфейсы" toc_priority: 24 --- diff --git a/docs/ru/interfaces/third-party/integrations.md b/docs/ru/interfaces/third-party/integrations.md index 60d6181ab3f..84d5b93f92f 100644 --- a/docs/ru/interfaces/third-party/integrations.md +++ b/docs/ru/interfaces/third-party/integrations.md @@ -1,6 +1,6 @@ --- toc_priority: 27 -toc_title: "\u0411\u0438\u0431\u043b\u0438\u043e\u0442\u0435\u043a\u0438\u0020\u0434\u043b\u044f\u0020\u0438\u043d\u0442\u0435\u0433\u0440\u0430\u0446\u0438\u0438\u0020\u043e\u0442\u0020\u0441\u0442\u043e\u0440\u043e\u043d\u043d\u0438\u0445\u0020\u0440\u0430\u0437\u0440\u0430\u0431\u043e\u0442\u0447\u0438\u043a\u043e\u0432" +toc_title: "Библиотеки для интеграции от сторонних разработчиков" --- # Библиотеки для интеграции от сторонних разработчиков {#biblioteki-dlia-integratsii-ot-storonnikh-razrabotchikov} diff --git a/docs/ru/interfaces/third-party/proxy.md b/docs/ru/interfaces/third-party/proxy.md index fc66ecde293..48853cb352e 100644 --- a/docs/ru/interfaces/third-party/proxy.md +++ b/docs/ru/interfaces/third-party/proxy.md @@ -1,6 +1,6 @@ --- toc_priority: 29 -toc_title: "\u041f\u0440\u043e\u043a\u0441\u0438\u002d\u0441\u0435\u0440\u0432\u0435\u0440\u044b\u0020\u043e\u0442\u0020\u0441\u0442\u043e\u0440\u043e\u043d\u043d\u0438\u0445\u0020\u0440\u0430\u0437\u0440\u0430\u0431\u043e\u0442\u0447\u0438\u043a\u043e\u0432" +toc_title: "Прокси-серверы от сторонних разработчиков" --- # Прокси-серверы от сторонних разработчиков {#proksi-servery-ot-storonnikh-razrabotchikov} diff --git a/docs/ru/introduction/distinctive-features.md b/docs/ru/introduction/distinctive-features.md index 4eeeef4a443..852f5cecd5b 100644 --- a/docs/ru/introduction/distinctive-features.md +++ b/docs/ru/introduction/distinctive-features.md @@ -1,6 +1,6 @@ --- toc_priority: 4 -toc_title: "\u041e\u0442\u043b\u0438\u0447\u0438\u0442\u0435\u043b\u044c\u043d\u044b\u0435\u0020\u0432\u043e\u0437\u043c\u043e\u0436\u043d\u043e\u0441\u0442\u0438\u0020\u0043\u006c\u0069\u0063\u006b\u0048\u006f\u0075\u0073\u0065" +toc_title: "Отличительные возможности ClickHouse" --- # Отличительные возможности ClickHouse {#otlichitelnye-vozmozhnosti-clickhouse} diff --git a/docs/ru/introduction/history.md b/docs/ru/introduction/history.md index ab740954bbe..ad17b2be27d 100644 --- a/docs/ru/introduction/history.md +++ b/docs/ru/introduction/history.md @@ -1,6 +1,6 @@ --- toc_priority: 7 -toc_title: "\u0418\u0441\u0442\u043e\u0440\u0438\u044f\u0020\u0043\u006c\u0069\u0063\u006b\u0048\u006f\u0075\u0073\u0065" +toc_title: "История ClickHouse" --- diff --git a/docs/ru/introduction/index.md b/docs/ru/introduction/index.md index 28a8e10e15b..c37cde09060 100644 --- a/docs/ru/introduction/index.md +++ b/docs/ru/introduction/index.md @@ -1,5 +1,5 @@ --- -toc_folder_title: "\u0412\u0432\u0435\u0434\u0435\u043D\u0438\u0435" +toc_folder_title: "Введение" toc_priority: 1 --- diff --git a/docs/ru/introduction/performance.md b/docs/ru/introduction/performance.md index c449e76a6ea..dd92d3df9f5 100644 --- a/docs/ru/introduction/performance.md +++ b/docs/ru/introduction/performance.md @@ -1,6 +1,6 @@ --- toc_priority: 6 -toc_title: "\u041f\u0440\u043e\u0438\u0437\u0432\u043e\u0434\u0438\u0442\u0435\u043b\u044c\u043d\u043e\u0441\u0442\u044c" +toc_title: "Производительность" --- # Производительность {#proizvoditelnost} diff --git a/docs/ru/operations/access-rights.md b/docs/ru/operations/access-rights.md index 00e55da7a82..9aa4e5f2561 100644 --- a/docs/ru/operations/access-rights.md +++ b/docs/ru/operations/access-rights.md @@ -1,6 +1,6 @@ --- toc_priority: 48 -toc_title: "\u0423\u043f\u0440\u0430\u0432\u043b\u0435\u043d\u0438\u0435\u0020\u0434\u043e\u0441\u0442\u0443\u043f\u043e\u043c" +toc_title: "Управление доступом" --- # Управление доступом {#access-control} diff --git a/docs/ru/operations/backup.md b/docs/ru/operations/backup.md index 165b54d9b62..703217e8547 100644 --- a/docs/ru/operations/backup.md +++ b/docs/ru/operations/backup.md @@ -1,6 +1,6 @@ --- toc_priority: 49 -toc_title: "\u0420\u0435\u0437\u0435\u0440\u0432\u043d\u043e\u0435\u0020\u043a\u043e\u043f\u0438\u0440\u043e\u0432\u0430\u043d\u0438\u0435\u0020\u0434\u0430\u043d\u043d\u044b\u0445" +toc_title: "Резервное копирование данных" --- # Резервное копирование данных {#rezervnoe-kopirovanie-dannykh} diff --git a/docs/ru/operations/configuration-files.md b/docs/ru/operations/configuration-files.md index a4cc9182427..84b26d0ba2a 100644 --- a/docs/ru/operations/configuration-files.md +++ b/docs/ru/operations/configuration-files.md @@ -1,6 +1,6 @@ --- toc_priority: 50 -toc_title: "\u041a\u043e\u043d\u0444\u0438\u0433\u0443\u0440\u0430\u0446\u0438\u043e\u043d\u043d\u044b\u0435\u0020\u0444\u0430\u0439\u043b\u044b" +toc_title: "Конфигурационные файлы" --- diff --git a/docs/ru/operations/index.md b/docs/ru/operations/index.md index 74a1d135967..99dcf652891 100644 --- a/docs/ru/operations/index.md +++ b/docs/ru/operations/index.md @@ -1,7 +1,7 @@ --- -toc_folder_title: "\u042d\u043a\u0441\u043f\u043b\u0443\u0430\u0442\u0430\u0446\u0438\u044f" +toc_folder_title: "Эксплуатация" toc_priority: 41 -toc_title: "\u042d\u043a\u0441\u043f\u043b\u0443\u0430\u0442\u0430\u0446\u0438\u044f" +toc_title: "Эксплуатация" --- # Эксплуатация {#operations} diff --git a/docs/ru/operations/monitoring.md b/docs/ru/operations/monitoring.md index 52d0b5ecc8a..7656b04d011 100644 --- a/docs/ru/operations/monitoring.md +++ b/docs/ru/operations/monitoring.md @@ -1,6 +1,6 @@ --- toc_priority: 45 -toc_title: "\u041c\u043e\u043d\u0438\u0442\u043e\u0440\u0438\u043d\u0433" +toc_title: "Мониторинг" --- # Мониторинг {#monitoring} diff --git a/docs/ru/operations/quotas.md b/docs/ru/operations/quotas.md index 92533eef0c1..31f3a66a1c3 100644 --- a/docs/ru/operations/quotas.md +++ b/docs/ru/operations/quotas.md @@ -1,6 +1,6 @@ --- toc_priority: 51 -toc_title: "\u041a\u0432\u043e\u0442\u044b" +toc_title: "Квоты" --- # Квоты {#quotas} diff --git a/docs/ru/operations/requirements.md b/docs/ru/operations/requirements.md index 36a7dd30b34..6567dcc9695 100644 --- a/docs/ru/operations/requirements.md +++ b/docs/ru/operations/requirements.md @@ -1,6 +1,6 @@ --- toc_priority: 44 -toc_title: "\u0422\u0440\u0435\u0431\u043e\u0432\u0430\u043d\u0438\u044f" +toc_title: "Требования" --- # Требования {#trebovaniia} diff --git a/docs/ru/operations/server-configuration-parameters/index.md b/docs/ru/operations/server-configuration-parameters/index.md index a691fe69fef..f511955ebc4 100644 --- a/docs/ru/operations/server-configuration-parameters/index.md +++ b/docs/ru/operations/server-configuration-parameters/index.md @@ -1,7 +1,7 @@ --- -toc_folder_title: "\u041a\u043e\u043d\u0444\u0438\u0433\u0443\u0440\u0430\u0446\u0438\u043e\u043d\u043d\u044b\u0435\u0020\u043f\u0430\u0440\u0430\u043c\u0435\u0442\u0440\u044b\u0020\u0441\u0435\u0440\u0432\u0435\u0440\u0430" +toc_folder_title: "Конфигурационные параметры сервера" toc_priority: 54 -toc_title: "\u0412\u0432\u0435\u0434\u0435\u043d\u0438\u0435" +toc_title: "Введение" --- # Конфигурационные параметры сервера {#server-settings} diff --git a/docs/ru/operations/server-configuration-parameters/settings.md b/docs/ru/operations/server-configuration-parameters/settings.md index 15ab13836e3..f46d899a3b7 100644 --- a/docs/ru/operations/server-configuration-parameters/settings.md +++ b/docs/ru/operations/server-configuration-parameters/settings.md @@ -1,6 +1,6 @@ --- toc_priority: 57 -toc_title: "\u041a\u043e\u043d\u0444\u0438\u0433\u0443\u0440\u0430\u0446\u0438\u043e\u043d\u043d\u044b\u0435\u0020\u043f\u0430\u0440\u0430\u043c\u0435\u0442\u0440\u044b\u0020\u0441\u0435\u0440\u0432\u0435\u0440\u0430" +toc_title: "Конфигурационные параметры сервера" --- # Конфигурационные параметры сервера {#server-configuration-parameters-reference} diff --git a/docs/ru/operations/settings/constraints-on-settings.md b/docs/ru/operations/settings/constraints-on-settings.md index b23be22958c..a4c1876574d 100644 --- a/docs/ru/operations/settings/constraints-on-settings.md +++ b/docs/ru/operations/settings/constraints-on-settings.md @@ -1,6 +1,6 @@ --- toc_priority: 62 -toc_title: "\u041e\u0433\u0440\u0430\u043d\u0438\u0447\u0435\u043d\u0438\u044f\u0020\u043d\u0430\u0020\u0438\u0437\u043c\u0435\u043d\u0435\u043d\u0438\u0435\u0020\u043d\u0430\u0441\u0442\u0440\u043e\u0435\u043a" +toc_title: "Ограничения на изменение настроек" --- # Ограничения на изменение настроек {#constraints-on-settings} diff --git a/docs/ru/operations/settings/permissions-for-queries.md b/docs/ru/operations/settings/permissions-for-queries.md index ae896dac77c..571f56fc3bd 100644 --- a/docs/ru/operations/settings/permissions-for-queries.md +++ b/docs/ru/operations/settings/permissions-for-queries.md @@ -1,6 +1,6 @@ --- toc_priority: 58 -toc_title: "\u0420\u0430\u0437\u0440\u0435\u0448\u0435\u043d\u0438\u044f\u0020\u0434\u043b\u044f\u0020\u0437\u0430\u043f\u0440\u043e\u0441\u043e\u0432" +toc_title: "Разрешения для запросов" --- # Разрешения для запросов {#permissions_for_queries} diff --git a/docs/ru/operations/settings/query-complexity.md b/docs/ru/operations/settings/query-complexity.md index b0eac5d96e7..c6e580a2209 100644 --- a/docs/ru/operations/settings/query-complexity.md +++ b/docs/ru/operations/settings/query-complexity.md @@ -1,6 +1,6 @@ --- toc_priority: 59 -toc_title: "\u041e\u0433\u0440\u0430\u043d\u0438\u0447\u0435\u043d\u0438\u044f\u0020\u043d\u0430\u0020\u0441\u043b\u043e\u0436\u043d\u043e\u0441\u0442\u044c\u0020\u0437\u0430\u043f\u0440\u043e\u0441\u0430" +toc_title: "Ограничения на сложность запроса" --- # Ограничения на сложность запроса {#restrictions-on-query-complexity} diff --git a/docs/ru/operations/settings/settings-profiles.md b/docs/ru/operations/settings/settings-profiles.md index 10feda01850..e8082919d89 100644 --- a/docs/ru/operations/settings/settings-profiles.md +++ b/docs/ru/operations/settings/settings-profiles.md @@ -1,6 +1,6 @@ --- toc_priority: 61 -toc_title: "\u041f\u0440\u043e\u0444\u0438\u043b\u0438\u0020\u043d\u0430\u0441\u0442\u0440\u043e\u0435\u043a" +toc_title: "Профили настроек" --- # Профили настроек {#settings-profiles} diff --git a/docs/ru/operations/settings/settings-users.md b/docs/ru/operations/settings/settings-users.md index 2069922d0ea..21cd78569df 100644 --- a/docs/ru/operations/settings/settings-users.md +++ b/docs/ru/operations/settings/settings-users.md @@ -1,6 +1,6 @@ --- toc_priority: 63 -toc_title: "\u041d\u0430\u0441\u0442\u0440\u043e\u0439\u043a\u0438\u0020\u043f\u043e\u043b\u044c\u0437\u043e\u0432\u0430\u0442\u0435\u043b\u0435\u0439" +toc_title: "Настройки пользователей" --- # Настройки пользователей {#nastroiki-polzovatelei} diff --git a/docs/ru/operations/settings/settings.md b/docs/ru/operations/settings/settings.md index f8f587c8a36..663821158bd 100644 --- a/docs/ru/operations/settings/settings.md +++ b/docs/ru/operations/settings/settings.md @@ -1,6 +1,6 @@ --- toc_priority: 60 -toc_title: "\u041d\u0430\u0441\u0442\u0440\u043e\u0439\u043a\u0438" +toc_title: "Настройки" --- # Настройки {#settings} diff --git a/docs/ru/operations/system-tables/index.md b/docs/ru/operations/system-tables/index.md index cdea6102a81..e4b6f5beb9d 100644 --- a/docs/ru/operations/system-tables/index.md +++ b/docs/ru/operations/system-tables/index.md @@ -1,6 +1,6 @@ --- toc_priority: 52 -toc_title: "\u0421\u0438\u0441\u0442\u0435\u043c\u043d\u044b\u0435\u0020\u0442\u0430\u0431\u043b\u0438\u0446\u044b" +toc_title: "Системные таблицы" --- # Системные таблицы {#system-tables} diff --git a/docs/ru/operations/tips.md b/docs/ru/operations/tips.md index 40035309c03..0a2ca5ecac1 100644 --- a/docs/ru/operations/tips.md +++ b/docs/ru/operations/tips.md @@ -1,6 +1,6 @@ --- toc_priority: 58 -toc_title: "\u0421\u043e\u0432\u0435\u0442\u044b\u0020\u043f\u043e\u0020\u044d\u043a\u0441\u043f\u043b\u0443\u0430\u0442\u0430\u0446\u0438\u0438" +toc_title: "Советы по эксплуатации" --- # Советы по эксплуатации {#sovety-po-ekspluatatsii} diff --git a/docs/ru/operations/troubleshooting.md b/docs/ru/operations/troubleshooting.md index 3df2a1dd46c..5882bc36f9e 100644 --- a/docs/ru/operations/troubleshooting.md +++ b/docs/ru/operations/troubleshooting.md @@ -1,6 +1,6 @@ --- toc_priority: 46 -toc_title: "\u0423\u0441\u0442\u0440\u0430\u043d\u0435\u043d\u0438\u0435\u0020\u043d\u0435\u0438\u0441\u043f\u0440\u0430\u0432\u043d\u043e\u0441\u0442\u0435\u0439" +toc_title: "Устранение неисправностей" --- # Устранение неисправностей {#ustranenie-neispravnostei} diff --git a/docs/ru/operations/update.md b/docs/ru/operations/update.md index c74b28b3fd7..5c187ed1604 100644 --- a/docs/ru/operations/update.md +++ b/docs/ru/operations/update.md @@ -1,6 +1,6 @@ --- toc_priority: 47 -toc_title: "\u041e\u0431\u043d\u043e\u0432\u043b\u0435\u043d\u0438\u0435\u0020\u0043\u006c\u0069\u0063\u006b\u0048\u006f\u0075\u0073\u0065" +toc_title: "Обновление ClickHouse" --- # Обновление ClickHouse {#obnovlenie-clickhouse} diff --git a/docs/ru/operations/utilities/index.md b/docs/ru/operations/utilities/index.md index 5b55ebd798d..8b533c29ff5 100644 --- a/docs/ru/operations/utilities/index.md +++ b/docs/ru/operations/utilities/index.md @@ -1,7 +1,7 @@ --- -toc_folder_title: "\u0423\u0442\u0438\u043b\u0438\u0442\u044b" +toc_folder_title: "Утилиты" toc_priority: 56 -toc_title: "\u041e\u0431\u0437\u043e\u0440" +toc_title: "Обзор" --- # Утилиты ClickHouse {#utility-clickhouse} diff --git a/docs/ru/sql-reference/aggregate-functions/combinators.md b/docs/ru/sql-reference/aggregate-functions/combinators.md index 592c61f87ff..3b35716ec27 100644 --- a/docs/ru/sql-reference/aggregate-functions/combinators.md +++ b/docs/ru/sql-reference/aggregate-functions/combinators.md @@ -1,6 +1,6 @@ --- toc_priority: 37 -toc_title: "\u041a\u043e\u043c\u0431\u0438\u043d\u0430\u0442\u043e\u0440\u044b\u0020\u0430\u0433\u0440\u0435\u0433\u0430\u0442\u043d\u044b\u0445\u0020\u0444\u0443\u043d\u043a\u0446\u0438\u0439" +toc_title: "Комбинаторы агрегатных функций" --- diff --git a/docs/ru/sql-reference/aggregate-functions/index.md b/docs/ru/sql-reference/aggregate-functions/index.md index 4a7768f587f..b7c84e4468c 100644 --- a/docs/ru/sql-reference/aggregate-functions/index.md +++ b/docs/ru/sql-reference/aggregate-functions/index.md @@ -1,8 +1,8 @@ --- -toc_folder_title: "\u0410\u0433\u0440\u0435\u0433\u0430\u0442\u043D\u044B\u0435 \u0444\ - \u0443\u043D\u043A\u0446\u0438\u0438" +toc_folder_title: "Агрегатные ф\ + ункции" toc_priority: 33 -toc_title: "\u0412\u0432\u0435\u0434\u0435\u043D\u0438\u0435" +toc_title: "Введение" --- # Агрегатные функции {#aggregate-functions} diff --git a/docs/ru/sql-reference/aggregate-functions/parametric-functions.md b/docs/ru/sql-reference/aggregate-functions/parametric-functions.md index d96f7a13bcc..61518cb6f02 100644 --- a/docs/ru/sql-reference/aggregate-functions/parametric-functions.md +++ b/docs/ru/sql-reference/aggregate-functions/parametric-functions.md @@ -1,6 +1,6 @@ --- toc_priority: 38 -toc_title: "\u041f\u0430\u0440\u0430\u043c\u0435\u0442\u0440\u0438\u0447\u0435\u0441\u043a\u0438\u0435\u0020\u0430\u0433\u0440\u0435\u0433\u0430\u0442\u043d\u044b\u0435\u0020\u0444\u0443\u043d\u043a\u0446\u0438\u0438" +toc_title: "Параметрические агрегатные функции" --- # Параметрические агрегатные функции {#aggregate_functions_parametric} diff --git a/docs/ru/sql-reference/aggregate-functions/reference/index.md b/docs/ru/sql-reference/aggregate-functions/reference/index.md index 4c0060581fd..e496893a771 100644 --- a/docs/ru/sql-reference/aggregate-functions/reference/index.md +++ b/docs/ru/sql-reference/aggregate-functions/reference/index.md @@ -1,5 +1,5 @@ --- -toc_folder_title: "\u0421\u043f\u0440\u0430\u0432\u043e\u0447\u043d\u0438\u043a" +toc_folder_title: "Справочник" toc_priority: 36 toc_hidden: true --- diff --git a/docs/ru/sql-reference/data-types/boolean.md b/docs/ru/sql-reference/data-types/boolean.md index bb0cd50c739..b0fad6d7446 100644 --- a/docs/ru/sql-reference/data-types/boolean.md +++ b/docs/ru/sql-reference/data-types/boolean.md @@ -1,6 +1,6 @@ --- toc_priority: 43 -toc_title: "\u0411\u0443\u043b\u0435\u0432\u044b\u0020\u0437\u043d\u0430\u0447\u0435\u043d\u0438\u044f" +toc_title: "Булевы значения" --- # Булевы значения {#bulevy-znacheniia} diff --git a/docs/ru/sql-reference/data-types/domains/index.md b/docs/ru/sql-reference/data-types/domains/index.md index 4449469b1bc..6a968a76ff6 100644 --- a/docs/ru/sql-reference/data-types/domains/index.md +++ b/docs/ru/sql-reference/data-types/domains/index.md @@ -1,6 +1,6 @@ --- -toc_folder_title: "\u0414\u043e\u043c\u0435\u043d\u044b" -toc_title_title: "\u041e\u0431\u0437\u043e\u0440" +toc_folder_title: "Домены" +toc_title_title: "Обзор" toc_priority: 56 --- diff --git a/docs/ru/sql-reference/data-types/index.md b/docs/ru/sql-reference/data-types/index.md index 7a5618f4c5d..53c983a147a 100644 --- a/docs/ru/sql-reference/data-types/index.md +++ b/docs/ru/sql-reference/data-types/index.md @@ -1,7 +1,7 @@ --- -toc_folder_title: "\u0422\u0438\u043F\u044B \u0434\u0430\u043D\u043D\u044B\u0445" +toc_folder_title: "Типы данных" toc_priority: 37 -toc_title: "\u0412\u0432\u0435\u0434\u0435\u043D\u0438\u0435" +toc_title: "Введение" --- # Типы данных {#data_types} diff --git a/docs/ru/sql-reference/data-types/nested-data-structures/index.md b/docs/ru/sql-reference/data-types/nested-data-structures/index.md index d53cabc6652..db214b90c03 100644 --- a/docs/ru/sql-reference/data-types/nested-data-structures/index.md +++ b/docs/ru/sql-reference/data-types/nested-data-structures/index.md @@ -1,5 +1,5 @@ --- -toc_folder_title: "\u0412\u043b\u043e\u0436\u0435\u043d\u043d\u044b\u0435\u0020\u0441\u0442\u0440\u0443\u043a\u0442\u0443\u0440\u044b\u0020\u0434\u0430\u043d\u043d\u044b\u0445" +toc_folder_title: "Вложенные структуры данных" toc_hidden: true toc_priority: 54 toc_title: hidden diff --git a/docs/ru/sql-reference/data-types/special-data-types/index.md b/docs/ru/sql-reference/data-types/special-data-types/index.md index 29c057472ea..e6d9fa8b011 100644 --- a/docs/ru/sql-reference/data-types/special-data-types/index.md +++ b/docs/ru/sql-reference/data-types/special-data-types/index.md @@ -1,5 +1,5 @@ --- -toc_folder_title: "\u0421\u043b\u0443\u0436\u0435\u0431\u043d\u044b\u0435\u0020\u0442\u0438\u043f\u044b\u0020\u0434\u0430\u043d\u043d\u044b\u0445" +toc_folder_title: "Служебные типы данных" toc_hidden: true toc_priority: 55 toc_title: hidden diff --git a/docs/ru/sql-reference/dictionaries/external-dictionaries/external-dicts-dict-hierarchical.md b/docs/ru/sql-reference/dictionaries/external-dictionaries/external-dicts-dict-hierarchical.md index 350e391dbed..9c0b731bc7d 100644 --- a/docs/ru/sql-reference/dictionaries/external-dictionaries/external-dicts-dict-hierarchical.md +++ b/docs/ru/sql-reference/dictionaries/external-dictionaries/external-dicts-dict-hierarchical.md @@ -1,6 +1,6 @@ --- toc_priority: 45 -toc_title: "\u0418\u0435\u0440\u0430\u0440\u0445\u0438\u0447\u0435\u0441\u043a\u0438\u0435\u0020\u0441\u043b\u043e\u0432\u0430\u0440\u0438" +toc_title: "Иерархические словари" --- # Иерархические словари {#ierarkhicheskie-slovari} diff --git a/docs/ru/sql-reference/dictionaries/external-dictionaries/external-dicts-dict-layout.md b/docs/ru/sql-reference/dictionaries/external-dictionaries/external-dicts-dict-layout.md index f6b8b670563..0fd4a85c46f 100644 --- a/docs/ru/sql-reference/dictionaries/external-dictionaries/external-dicts-dict-layout.md +++ b/docs/ru/sql-reference/dictionaries/external-dictionaries/external-dicts-dict-layout.md @@ -1,6 +1,6 @@ --- toc_priority: 41 -toc_title: "\u0425\u0440\u0430\u043d\u0435\u043d\u0438\u0435\u0020\u0441\u043b\u043e\u0432\u0430\u0440\u0435\u0439\u0020\u0432\u0020\u043f\u0430\u043c\u044f\u0442\u0438" +toc_title: "Хранение словарей в памяти" --- # Хранение словарей в памяти {#dicts-external-dicts-dict-layout} diff --git a/docs/ru/sql-reference/dictionaries/external-dictionaries/external-dicts-dict-lifetime.md b/docs/ru/sql-reference/dictionaries/external-dictionaries/external-dicts-dict-lifetime.md index ec0fb8e0ee5..cdf1b05fb37 100644 --- a/docs/ru/sql-reference/dictionaries/external-dictionaries/external-dicts-dict-lifetime.md +++ b/docs/ru/sql-reference/dictionaries/external-dictionaries/external-dicts-dict-lifetime.md @@ -1,6 +1,6 @@ --- toc_priority: 42 -toc_title: "\u041e\u0431\u043d\u043e\u0432\u043b\u0435\u043d\u0438\u0435\u0020\u0441\u043b\u043e\u0432\u0430\u0440\u0435\u0439" +toc_title: "Обновление словарей" --- # Обновление словарей {#obnovlenie-slovarei} diff --git a/docs/ru/sql-reference/dictionaries/external-dictionaries/external-dicts-dict-sources.md b/docs/ru/sql-reference/dictionaries/external-dictionaries/external-dicts-dict-sources.md index 77275b65a05..13b6a93b6ae 100644 --- a/docs/ru/sql-reference/dictionaries/external-dictionaries/external-dicts-dict-sources.md +++ b/docs/ru/sql-reference/dictionaries/external-dictionaries/external-dicts-dict-sources.md @@ -1,6 +1,6 @@ --- toc_priority: 43 -toc_title: "\u0418\u0441\u0442\u043e\u0447\u043d\u0438\u043a\u0438\u0020\u0432\u043d\u0435\u0448\u043d\u0438\u0445\u0020\u0441\u043b\u043e\u0432\u0430\u0440\u0435\u0439" +toc_title: "Источники внешних словарей" --- # Источники внешних словарей {#dicts-external-dicts-dict-sources} diff --git a/docs/ru/sql-reference/dictionaries/external-dictionaries/external-dicts-dict-structure.md b/docs/ru/sql-reference/dictionaries/external-dictionaries/external-dicts-dict-structure.md index bf87ce61b9e..6efbe706110 100644 --- a/docs/ru/sql-reference/dictionaries/external-dictionaries/external-dicts-dict-structure.md +++ b/docs/ru/sql-reference/dictionaries/external-dictionaries/external-dicts-dict-structure.md @@ -1,6 +1,6 @@ --- toc_priority: 44 -toc_title: "\u041a\u043b\u044e\u0447\u0020\u0438\u0020\u043f\u043e\u043b\u044f\u0020\u0441\u043b\u043e\u0432\u0430\u0440\u044f" +toc_title: "Ключ и поля словаря" --- # Ключ и поля словаря {#kliuch-i-polia-slovaria} diff --git a/docs/ru/sql-reference/dictionaries/external-dictionaries/external-dicts-dict.md b/docs/ru/sql-reference/dictionaries/external-dictionaries/external-dicts-dict.md index ff18f906926..7e35f59609d 100644 --- a/docs/ru/sql-reference/dictionaries/external-dictionaries/external-dicts-dict.md +++ b/docs/ru/sql-reference/dictionaries/external-dictionaries/external-dicts-dict.md @@ -1,6 +1,6 @@ --- toc_priority: 40 -toc_title: "\u041d\u0430\u0441\u0442\u0440\u043e\u0439\u043a\u0430\u0020\u0432\u043d\u0435\u0448\u043d\u0435\u0433\u043e\u0020\u0441\u043b\u043e\u0432\u0430\u0440\u044f" +toc_title: "Настройка внешнего словаря" --- # Настройка внешнего словаря {#dicts-external-dicts-dict} diff --git a/docs/ru/sql-reference/dictionaries/external-dictionaries/external-dicts.md b/docs/ru/sql-reference/dictionaries/external-dictionaries/external-dicts.md index c18af68c15e..6467b5f82e4 100644 --- a/docs/ru/sql-reference/dictionaries/external-dictionaries/external-dicts.md +++ b/docs/ru/sql-reference/dictionaries/external-dictionaries/external-dicts.md @@ -1,6 +1,6 @@ --- toc_priority: 39 -toc_title: "\u0412\u043d\u0435\u0448\u043d\u0438\u0435\u0020\u0441\u043b\u043e\u0432\u0430\u0440\u0438" +toc_title: "Внешние словари" --- diff --git a/docs/ru/sql-reference/dictionaries/external-dictionaries/index.md b/docs/ru/sql-reference/dictionaries/external-dictionaries/index.md index b448858b1fa..c0d954d6976 100644 --- a/docs/ru/sql-reference/dictionaries/external-dictionaries/index.md +++ b/docs/ru/sql-reference/dictionaries/external-dictionaries/index.md @@ -1,5 +1,5 @@ --- -toc_folder_title: "\u0412\u043d\u0435\u0448\u043d\u0438\u0435\u0020\u0441\u043b\u043e\u0432\u0430\u0440\u0438" +toc_folder_title: "Внешние словари" toc_priority: 37 --- diff --git a/docs/ru/sql-reference/dictionaries/index.md b/docs/ru/sql-reference/dictionaries/index.md index 5a4119b4dd5..238aa244967 100644 --- a/docs/ru/sql-reference/dictionaries/index.md +++ b/docs/ru/sql-reference/dictionaries/index.md @@ -1,7 +1,7 @@ --- -toc_folder_title: "\u0421\u043b\u043e\u0432\u0430\u0440\u0438" +toc_folder_title: "Словари" toc_priority: 35 -toc_title: "\u0412\u0432\u0435\u0434\u0435\u043d\u0438\u0435" +toc_title: "Введение" --- # Словари {#slovari} diff --git a/docs/ru/sql-reference/dictionaries/internal-dicts.md b/docs/ru/sql-reference/dictionaries/internal-dicts.md index d8103efa6ae..af7f13f7133 100644 --- a/docs/ru/sql-reference/dictionaries/internal-dicts.md +++ b/docs/ru/sql-reference/dictionaries/internal-dicts.md @@ -1,6 +1,6 @@ --- toc_priority: 39 -toc_title: "\u0412\u0441\u0442\u0440\u043e\u0435\u043d\u043d\u044b\u0435\u0020\u0441\u043b\u043e\u0432\u0430\u0440\u0438" +toc_title: "Встроенные словари" --- # Встроенные словари {#internal_dicts} diff --git a/docs/ru/sql-reference/distributed-ddl.md b/docs/ru/sql-reference/distributed-ddl.md index 275709320f6..ee1e4f62f28 100644 --- a/docs/ru/sql-reference/distributed-ddl.md +++ b/docs/ru/sql-reference/distributed-ddl.md @@ -1,6 +1,7 @@ --- toc_priority: 32 -toc_title: "\u0420\u0430\u0441\u043f\u0440\u0435\u0434\u0435\u043b\u0435\u043d\u043d\u044b\u0435\u0020\u0044\u0044\u004c\u0020\u0437\u0430\u043f\u0440\u043e\u0441\u044b\u000a" +toc_title: "Распределенные DDL запросы +" --- # Распределенные DDL запросы (секция ON CLUSTER) {#raspredelennye-ddl-zaprosy-sektsiia-on-cluster} diff --git a/docs/ru/sql-reference/functions/arithmetic-functions.md b/docs/ru/sql-reference/functions/arithmetic-functions.md index 16c3e8fd8f0..779e0a9fe4a 100644 --- a/docs/ru/sql-reference/functions/arithmetic-functions.md +++ b/docs/ru/sql-reference/functions/arithmetic-functions.md @@ -1,6 +1,6 @@ --- toc_priority: 34 -toc_title: "\u0410\u0440\u0438\u0444\u043c\u0435\u0442\u0438\u0447\u0435\u0441\u043a\u0438\u0435\u0020\u0444\u0443\u043d\u043a\u0446\u0438\u0438" +toc_title: "Арифметические функции" --- # Арифметические функции {#arifmeticheskie-funktsii} diff --git a/docs/ru/sql-reference/functions/array-functions.md b/docs/ru/sql-reference/functions/array-functions.md index fe216b1aed1..dca645888a9 100644 --- a/docs/ru/sql-reference/functions/array-functions.md +++ b/docs/ru/sql-reference/functions/array-functions.md @@ -1,6 +1,6 @@ --- toc_priority: 35 -toc_title: "\u041c\u0430\u0441\u0441\u0438\u0432\u044b" +toc_title: "Массивы" --- # Массивы {#functions-for-working-with-arrays} diff --git a/docs/ru/sql-reference/functions/array-join.md b/docs/ru/sql-reference/functions/array-join.md index 2ed3d25fa92..ed67d30062b 100644 --- a/docs/ru/sql-reference/functions/array-join.md +++ b/docs/ru/sql-reference/functions/array-join.md @@ -1,6 +1,6 @@ --- toc_priority: 61 -toc_title: "\u0424\u0443\u043d\u043a\u0446\u0438\u044f\u0020\u0041\u0072\u0072\u0061\u0079\u004a\u006f\u0069\u006e" +toc_title: "Функция ArrayJoin" --- # Функция ArrayJoin {#functions_arrayjoin} diff --git a/docs/ru/sql-reference/functions/bit-functions.md b/docs/ru/sql-reference/functions/bit-functions.md index 8c7808437a5..79ea05f4bd7 100644 --- a/docs/ru/sql-reference/functions/bit-functions.md +++ b/docs/ru/sql-reference/functions/bit-functions.md @@ -1,6 +1,6 @@ --- toc_priority: 48 -toc_title: "\u0411\u0438\u0442\u043e\u0432\u044b\u0435\u0020\u0444\u0443\u043d\u043a\u0446\u0438\u0438" +toc_title: "Битовые функции" --- # Битовые функции {#bitovye-funktsii} diff --git a/docs/ru/sql-reference/functions/bitmap-functions.md b/docs/ru/sql-reference/functions/bitmap-functions.md index b21ddea94e4..cd0ddee01a6 100644 --- a/docs/ru/sql-reference/functions/bitmap-functions.md +++ b/docs/ru/sql-reference/functions/bitmap-functions.md @@ -1,6 +1,6 @@ --- toc_priority: 49 -toc_title: "\u0424\u0443\u043d\u043a\u0446\u0438\u0438\u0020\u0434\u043b\u044f\u0020\u0431\u0438\u0442\u043c\u0430\u043f\u043e\u0432" +toc_title: "Функции для битмапов" --- # Функции для битовых масок {#bitmap-functions} diff --git a/docs/ru/sql-reference/functions/comparison-functions.md b/docs/ru/sql-reference/functions/comparison-functions.md index a98c97ec96c..179df5c2ed5 100644 --- a/docs/ru/sql-reference/functions/comparison-functions.md +++ b/docs/ru/sql-reference/functions/comparison-functions.md @@ -1,6 +1,6 @@ --- toc_priority: 36 -toc_title: "\u0424\u0443\u043d\u043a\u0446\u0438\u0438\u0020\u0441\u0440\u0430\u0432\u043d\u0435\u043d\u0438\u044f" +toc_title: "Функции сравнения" --- # Функции сравнения {#funktsii-sravneniia} diff --git a/docs/ru/sql-reference/functions/conditional-functions.md b/docs/ru/sql-reference/functions/conditional-functions.md index 83268b68959..888e9427a79 100644 --- a/docs/ru/sql-reference/functions/conditional-functions.md +++ b/docs/ru/sql-reference/functions/conditional-functions.md @@ -1,6 +1,6 @@ --- toc_priority: 43 -toc_title: "\u0423\u0441\u043b\u043e\u0432\u043d\u044b\u0435\u0020\u0444\u0443\u043d\u043a\u0446\u0438\u0438" +toc_title: "Условные функции" --- # Условные функции {#uslovnye-funktsii} diff --git a/docs/ru/sql-reference/functions/date-time-functions.md b/docs/ru/sql-reference/functions/date-time-functions.md index b3f9cd19d0e..788a4226b8e 100644 --- a/docs/ru/sql-reference/functions/date-time-functions.md +++ b/docs/ru/sql-reference/functions/date-time-functions.md @@ -1,6 +1,6 @@ --- toc_priority: 39 -toc_title: "\u0424\u0443\u043d\u043a\u0446\u0438\u0438\u0020\u0434\u043b\u044f\u0020\u0440\u0430\u0431\u043e\u0442\u044b\u0020\u0441\u0020\u0434\u0430\u0442\u0430\u043c\u0438\u0020\u0438\u0020\u0432\u0440\u0435\u043c\u0435\u043d\u0435\u043c" +toc_title: "Функции для работы с датами и временем" --- # Функции для работы с датами и временем {#funktsii-dlia-raboty-s-datami-i-vremenem} diff --git a/docs/ru/sql-reference/functions/encoding-functions.md b/docs/ru/sql-reference/functions/encoding-functions.md index 8c3065e5a77..951c6c60e38 100644 --- a/docs/ru/sql-reference/functions/encoding-functions.md +++ b/docs/ru/sql-reference/functions/encoding-functions.md @@ -1,6 +1,6 @@ --- toc_priority: 52 -toc_title: "\u0424\u0443\u043d\u043a\u0446\u0438\u0438\u0020\u043a\u043e\u0434\u0438\u0440\u043e\u0432\u0430\u043d\u0438\u044f" +toc_title: "Функции кодирования" --- # Функции кодирования {#funktsii-kodirovaniia} diff --git a/docs/ru/sql-reference/functions/encryption-functions.md b/docs/ru/sql-reference/functions/encryption-functions.md index 0216a6b2356..a7adb40e631 100644 --- a/docs/ru/sql-reference/functions/encryption-functions.md +++ b/docs/ru/sql-reference/functions/encryption-functions.md @@ -1,6 +1,6 @@ --- toc_priority: 67 -toc_title: "\u0424\u0443\u043d\u043a\u0446\u0438\u0438 \u0434\u043b\u044f \u0448\u0438\u0444\u0440\u043e\u0432\u0430\u043d\u0438\u044f" +toc_title: "Функции для шифрования" --- # Функции шифрования {#encryption-functions} diff --git a/docs/ru/sql-reference/functions/ext-dict-functions.md b/docs/ru/sql-reference/functions/ext-dict-functions.md index 6054ed141d4..8d018e8e9ac 100644 --- a/docs/ru/sql-reference/functions/ext-dict-functions.md +++ b/docs/ru/sql-reference/functions/ext-dict-functions.md @@ -1,6 +1,6 @@ --- toc_priority: 58 -toc_title: "\u0424\u0443\u043d\u043a\u0446\u0438\u0438\u0020\u0434\u043b\u044f\u0020\u0440\u0430\u0431\u043e\u0442\u044b\u0020\u0441\u0020\u0432\u043d\u0435\u0448\u043d\u0438\u043c\u0438\u0020\u0441\u043b\u043e\u0432\u0430\u0440\u044f\u043c\u0438" +toc_title: "Функции для работы с внешними словарями" --- # Функции для работы с внешними словарями {#ext_dict_functions} diff --git a/docs/ru/sql-reference/functions/functions-for-nulls.md b/docs/ru/sql-reference/functions/functions-for-nulls.md index 0db55847631..f0277a59699 100644 --- a/docs/ru/sql-reference/functions/functions-for-nulls.md +++ b/docs/ru/sql-reference/functions/functions-for-nulls.md @@ -1,6 +1,6 @@ --- toc_priority: 63 -toc_title: "\u0424\u0443\u043d\u043a\u0446\u0438\u0438\u0020\u0434\u043b\u044f\u0020\u0440\u0430\u0431\u043e\u0442\u044b\u0020\u0441\u0020\u004e\u0075\u006c\u006c\u0061\u0062\u006c\u0065\u002d\u0430\u0440\u0433\u0443\u043c\u0435\u043d\u0442\u0430\u043c\u0438" +toc_title: "Функции для работы с Nullable-аргументами" --- # Функции для работы с Nullable-аргументами {#funktsii-dlia-raboty-s-nullable-argumentami} diff --git a/docs/ru/sql-reference/functions/geo/coordinates.md b/docs/ru/sql-reference/functions/geo/coordinates.md index 1931a9b932f..09e2d7d01bf 100644 --- a/docs/ru/sql-reference/functions/geo/coordinates.md +++ b/docs/ru/sql-reference/functions/geo/coordinates.md @@ -1,5 +1,5 @@ --- -toc_title: "\u0424\u0443\u043d\u043a\u0446\u0438\u0438\u0020\u0434\u043b\u044f\u0020\u0440\u0430\u0431\u043e\u0442\u044b\u0020\u0441\u0020\u0433\u0435\u043e\u0433\u0440\u0430\u0444\u0438\u0447\u0435\u0441\u043a\u0438\u043c\u0438\u0020\u043a\u043e\u043e\u0440\u0434\u0438\u043d\u0430\u0442\u0430\u043c\u0438" +toc_title: "Функции для работы с географическими координатами" toc_priority: 62 --- diff --git a/docs/ru/sql-reference/functions/geo/geohash.md b/docs/ru/sql-reference/functions/geo/geohash.md index 38c64f11b10..2dd3f83ddf1 100644 --- a/docs/ru/sql-reference/functions/geo/geohash.md +++ b/docs/ru/sql-reference/functions/geo/geohash.md @@ -1,5 +1,5 @@ --- -toc_title: "\u0424\u0443\u043d\u043a\u0446\u0438\u0438\u0020\u0434\u043b\u044f\u0020\u0440\u0430\u0431\u043e\u0442\u044b\u0020\u0441\u0020\u0441\u0438\u0441\u0442\u0435\u043c\u043e\u0439\u0020\u0047\u0065\u006f\u0068\u0061\u0073\u0068" +toc_title: "Функции для работы с системой Geohash" --- # Функции для работы с системой Geohash {#geohash} diff --git a/docs/ru/sql-reference/functions/geo/h3.md b/docs/ru/sql-reference/functions/geo/h3.md index 69d06b5dfa6..7046833f7ec 100644 --- a/docs/ru/sql-reference/functions/geo/h3.md +++ b/docs/ru/sql-reference/functions/geo/h3.md @@ -1,5 +1,5 @@ --- -toc_title: "\u0424\u0443\u043d\u043a\u0446\u0438\u0438\u0020\u0434\u043b\u044f\u0020\u0440\u0430\u0431\u043e\u0442\u044b\u0020\u0441\u0020\u0438\u043d\u0434\u0435\u043a\u0441\u0430\u043c\u0438\u0020\u0048\u0033" +toc_title: "Функции для работы с индексами H3" --- # Функции для работы с индексами H3 {#h3index} diff --git a/docs/ru/sql-reference/functions/geo/index.md b/docs/ru/sql-reference/functions/geo/index.md index cedaafaa31d..6b9a14e4d02 100644 --- a/docs/ru/sql-reference/functions/geo/index.md +++ b/docs/ru/sql-reference/functions/geo/index.md @@ -1,6 +1,6 @@ --- toc_priority: 62 -toc_folder_title: "\u0413\u0435\u043e\u002d\u0434\u0430\u043d\u043d\u044b\u0435" +toc_folder_title: "Гео-данные" toc_title: hidden --- diff --git a/docs/ru/sql-reference/functions/hash-functions.md b/docs/ru/sql-reference/functions/hash-functions.md index f7820889ea9..1742abe5b56 100644 --- a/docs/ru/sql-reference/functions/hash-functions.md +++ b/docs/ru/sql-reference/functions/hash-functions.md @@ -1,6 +1,6 @@ --- toc_priority: 50 -toc_title: "\u0424\u0443\u043d\u043a\u0446\u0438\u0438\u0020\u0445\u044d\u0448\u0438\u0440\u043e\u0432\u0430\u043d\u0438\u044f" +toc_title: "Функции хэширования" --- # Функции хэширования {#funktsii-kheshirovaniia} diff --git a/docs/ru/sql-reference/functions/in-functions.md b/docs/ru/sql-reference/functions/in-functions.md index b732f67303b..7326d087610 100644 --- a/docs/ru/sql-reference/functions/in-functions.md +++ b/docs/ru/sql-reference/functions/in-functions.md @@ -1,6 +1,6 @@ --- toc_priority: 60 -toc_title: "\u0424\u0443\u043d\u043a\u0446\u0438\u0438\u0020\u0434\u043b\u044f\u0020\u0440\u0435\u0430\u043b\u0438\u0437\u0430\u0446\u0438\u0438\u0020\u043e\u043f\u0435\u0440\u0430\u0442\u043e\u0440\u0430\u0020\u0049\u004e" +toc_title: "Функции для реализации оператора IN" --- # Функции для реализации оператора IN {#funktsii-dlia-realizatsii-operatora-in} diff --git a/docs/ru/sql-reference/functions/index.md b/docs/ru/sql-reference/functions/index.md index 25d3b6de067..ae3879b6c96 100644 --- a/docs/ru/sql-reference/functions/index.md +++ b/docs/ru/sql-reference/functions/index.md @@ -1,7 +1,7 @@ --- -toc_folder_title: "\u0424\u0443\u043d\u043a\u0446\u0438\u0438" +toc_folder_title: "Функции" toc_priority: 32 -toc_title: "\u0412\u0432\u0435\u0434\u0435\u043d\u0438\u0435" +toc_title: "Введение" --- # Функции {#funktsii} diff --git a/docs/ru/sql-reference/functions/introspection.md b/docs/ru/sql-reference/functions/introspection.md index 00dd660bc16..4cd7e5d273b 100644 --- a/docs/ru/sql-reference/functions/introspection.md +++ b/docs/ru/sql-reference/functions/introspection.md @@ -1,6 +1,6 @@ --- toc_priority: 65 -toc_title: "\u0424\u0443\u043d\u043a\u0446\u0438\u0438\u0020\u0438\u043d\u0442\u0440\u043e\u0441\u043f\u0435\u043a\u0446\u0438\u0438" +toc_title: "Функции интроспекции" --- # Функции интроспекции {#introspection-functions} diff --git a/docs/ru/sql-reference/functions/ip-address-functions.md b/docs/ru/sql-reference/functions/ip-address-functions.md index bc48419473d..a2a08b1938e 100644 --- a/docs/ru/sql-reference/functions/ip-address-functions.md +++ b/docs/ru/sql-reference/functions/ip-address-functions.md @@ -1,6 +1,6 @@ --- toc_priority: 55 -toc_title: "\u0424\u0443\u043d\u043a\u0446\u0438\u0438\u0020\u0434\u043b\u044f\u0020\u0440\u0430\u0431\u043e\u0442\u044b\u0020\u0441\u0020\u0049\u0050\u002d\u0430\u0434\u0440\u0435\u0441\u0430\u043c\u0438" +toc_title: "Функции для работы с IP-адресами" --- # Функции для работы с IP-адресами {#funktsii-dlia-raboty-s-ip-adresami} diff --git a/docs/ru/sql-reference/functions/logical-functions.md b/docs/ru/sql-reference/functions/logical-functions.md index 9b1ee6a66a7..2d71c60a509 100644 --- a/docs/ru/sql-reference/functions/logical-functions.md +++ b/docs/ru/sql-reference/functions/logical-functions.md @@ -1,6 +1,6 @@ --- toc_priority: 37 -toc_title: "\u041b\u043e\u0433\u0438\u0447\u0435\u0441\u043a\u0438\u0435\u0020\u0444\u0443\u043d\u043a\u0446\u0438\u0438" +toc_title: "Логические функции" --- # Логические функции {#logicheskie-funktsii} diff --git a/docs/ru/sql-reference/functions/machine-learning-functions.md b/docs/ru/sql-reference/functions/machine-learning-functions.md index 2ffdfd05613..decbff56646 100644 --- a/docs/ru/sql-reference/functions/machine-learning-functions.md +++ b/docs/ru/sql-reference/functions/machine-learning-functions.md @@ -1,6 +1,6 @@ --- toc_priority: 64 -toc_title: "\u0424\u0443\u043d\u043a\u0446\u0438\u0438\u0020\u043c\u0430\u0448\u0438\u043d\u043d\u043e\u0433\u043e\u0020\u043e\u0431\u0443\u0447\u0435\u043d\u0438\u044f" +toc_title: "Функции машинного обучения" --- # Функции машинного обучения {#funktsii-mashinnogo-obucheniia} diff --git a/docs/ru/sql-reference/functions/math-functions.md b/docs/ru/sql-reference/functions/math-functions.md index 2e57aca6a0a..d06fe267f5e 100644 --- a/docs/ru/sql-reference/functions/math-functions.md +++ b/docs/ru/sql-reference/functions/math-functions.md @@ -1,6 +1,6 @@ --- toc_priority: 44 -toc_title: "\u041c\u0430\u0442\u0435\u043c\u0430\u0442\u0438\u0447\u0435\u0441\u043a\u0438\u0435\u0020\u0444\u0443\u043d\u043a\u0446\u0438\u0438" +toc_title: "Математические функции" --- # Математические функции {#matematicheskie-funktsii} diff --git a/docs/ru/sql-reference/functions/other-functions.md b/docs/ru/sql-reference/functions/other-functions.md index 19494c0aa10..595d2458ca9 100644 --- a/docs/ru/sql-reference/functions/other-functions.md +++ b/docs/ru/sql-reference/functions/other-functions.md @@ -1,6 +1,6 @@ --- toc_priority: 66 -toc_title: "\u041f\u0440\u043e\u0447\u0438\u0435\u0020\u0444\u0443\u043d\u043a\u0446\u0438\u0438" +toc_title: "Прочие функции" --- # Прочие функции {#other-functions} diff --git a/docs/ru/sql-reference/functions/random-functions.md b/docs/ru/sql-reference/functions/random-functions.md index f3889504fa6..a09f5159309 100644 --- a/docs/ru/sql-reference/functions/random-functions.md +++ b/docs/ru/sql-reference/functions/random-functions.md @@ -1,6 +1,6 @@ --- toc_priority: 51 -toc_title: "\u0424\u0443\u043d\u043a\u0446\u0438\u0438\u0020\u0433\u0435\u043d\u0435\u0440\u0430\u0446\u0438\u0438\u0020\u043f\u0441\u0435\u0432\u0434\u043e\u0441\u043b\u0443\u0447\u0430\u0439\u043d\u044b\u0445\u0020\u0447\u0438\u0441\u0435\u043b" +toc_title: "Функции генерации псевдослучайных чисел" --- # Функции генерации псевдослучайных чисел {#functions-for-generating-pseudo-random-numbers} diff --git a/docs/ru/sql-reference/functions/rounding-functions.md b/docs/ru/sql-reference/functions/rounding-functions.md index 78033160396..704e7f5dd52 100644 --- a/docs/ru/sql-reference/functions/rounding-functions.md +++ b/docs/ru/sql-reference/functions/rounding-functions.md @@ -1,6 +1,6 @@ --- toc_priority: 45 -toc_title: "\u0424\u0443\u043d\u043a\u0446\u0438\u0438\u0020\u043e\u043a\u0440\u0443\u0433\u043b\u0435\u043d\u0438\u044f" +toc_title: "Функции округления" --- # Функции округления {#funktsii-okrugleniia} diff --git a/docs/ru/sql-reference/functions/splitting-merging-functions.md b/docs/ru/sql-reference/functions/splitting-merging-functions.md index d451eabc407..cacce5f4ba2 100644 --- a/docs/ru/sql-reference/functions/splitting-merging-functions.md +++ b/docs/ru/sql-reference/functions/splitting-merging-functions.md @@ -1,6 +1,6 @@ --- toc_priority: 47 -toc_title: "\u0424\u0443\u043d\u043a\u0446\u0438\u0438\u0020\u0440\u0430\u0437\u0431\u0438\u0435\u043d\u0438\u044f\u0020\u0438\u0020\u0441\u043b\u0438\u044f\u043d\u0438\u044f\u0020\u0441\u0442\u0440\u043e\u043a\u0020\u0438\u0020\u043c\u0430\u0441\u0441\u0438\u0432\u043e\u0432" +toc_title: "Функции разбиения и слияния строк и массивов" --- # Функции разбиения и слияния строк и массивов {#funktsii-razbieniia-i-sliianiia-strok-i-massivov} diff --git a/docs/ru/sql-reference/functions/string-functions.md b/docs/ru/sql-reference/functions/string-functions.md index 1159a1f5823..65a1cd63563 100644 --- a/docs/ru/sql-reference/functions/string-functions.md +++ b/docs/ru/sql-reference/functions/string-functions.md @@ -1,6 +1,6 @@ --- toc_priority: 40 -toc_title: "\u0424\u0443\u043d\u043a\u0446\u0438\u0438\u0020\u0434\u043b\u044f\u0020\u0440\u0430\u0431\u043e\u0442\u044b\u0020\u0441\u043e\u0020\u0441\u0442\u0440\u043e\u043a\u0430\u043c\u0438" +toc_title: "Функции для работы со строками" --- # Функции для работы со строками {#funktsii-dlia-raboty-so-strokami} diff --git a/docs/ru/sql-reference/functions/string-replace-functions.md b/docs/ru/sql-reference/functions/string-replace-functions.md index f334d6804f9..f00a06d1560 100644 --- a/docs/ru/sql-reference/functions/string-replace-functions.md +++ b/docs/ru/sql-reference/functions/string-replace-functions.md @@ -1,6 +1,6 @@ --- toc_priority: 42 -toc_title: "\u0424\u0443\u043d\u043a\u0446\u0438\u0438\u0020\u043f\u043e\u0438\u0441\u043a\u0430\u0020\u0438\u0020\u0437\u0430\u043c\u0435\u043d\u044b\u0020\u0432\u0020\u0441\u0442\u0440\u043e\u043a\u0430\u0445" +toc_title: "Функции поиска и замены в строках" --- # Функции поиска и замены в строках {#funktsii-poiska-i-zameny-v-strokakh} diff --git a/docs/ru/sql-reference/functions/string-search-functions.md b/docs/ru/sql-reference/functions/string-search-functions.md index b7193da6f33..95ac922a4a8 100644 --- a/docs/ru/sql-reference/functions/string-search-functions.md +++ b/docs/ru/sql-reference/functions/string-search-functions.md @@ -1,6 +1,6 @@ --- toc_priority: 41 -toc_title: "\u0424\u0443\u043d\u043a\u0446\u0438\u0438\u0020\u043f\u043e\u0438\u0441\u043a\u0430\u0020\u0432\u0020\u0441\u0442\u0440\u043e\u043a\u0430\u0445" +toc_title: "Функции поиска в строках" --- # Функции поиска в строках {#funktsii-poiska-v-strokakh} diff --git a/docs/ru/sql-reference/functions/type-conversion-functions.md b/docs/ru/sql-reference/functions/type-conversion-functions.md index 53e7bc1300e..f312f9f5847 100644 --- a/docs/ru/sql-reference/functions/type-conversion-functions.md +++ b/docs/ru/sql-reference/functions/type-conversion-functions.md @@ -1,6 +1,6 @@ --- toc_priority: 38 -toc_title: "\u0424\u0443\u043d\u043a\u0446\u0438\u0438\u0020\u043f\u0440\u0435\u043e\u0431\u0440\u0430\u0437\u043e\u0432\u0430\u043d\u0438\u044f\u0020\u0442\u0438\u043f\u043e\u0432" +toc_title: "Функции преобразования типов" --- # Функции преобразования типов {#funktsii-preobrazovaniia-tipov} diff --git a/docs/ru/sql-reference/functions/url-functions.md b/docs/ru/sql-reference/functions/url-functions.md index 7541e16bed4..83f7fd32f6c 100644 --- a/docs/ru/sql-reference/functions/url-functions.md +++ b/docs/ru/sql-reference/functions/url-functions.md @@ -1,6 +1,6 @@ --- toc_priority: 54 -toc_title: "\u0424\u0443\u043d\u043a\u0446\u0438\u0438\u0020\u0434\u043b\u044f\u0020\u0440\u0430\u0431\u043e\u0442\u044b\u0020\u0441\u0020\u0055\u0052\u004c" +toc_title: "Функции для работы с URL" --- # Функции для работы с URL {#funktsii-dlia-raboty-s-url} diff --git a/docs/ru/sql-reference/functions/uuid-functions.md b/docs/ru/sql-reference/functions/uuid-functions.md index 6082fcaa712..f0017adbc8b 100644 --- a/docs/ru/sql-reference/functions/uuid-functions.md +++ b/docs/ru/sql-reference/functions/uuid-functions.md @@ -1,6 +1,6 @@ --- toc_priority: 53 -toc_title: "\u0424\u0443\u043d\u043a\u0446\u0438\u0438\u0020\u0434\u043b\u044f\u0020\u0440\u0430\u0431\u043e\u0442\u044b\u0020\u0441\u0020\u0055\u0055\u0049\u0044" +toc_title: "Функции для работы с UUID" --- # Функции для работы с UUID {#funktsii-dlia-raboty-s-uuid} diff --git a/docs/ru/sql-reference/functions/ym-dict-functions.md b/docs/ru/sql-reference/functions/ym-dict-functions.md index c3b04e4ab66..f6d02e553a0 100644 --- a/docs/ru/sql-reference/functions/ym-dict-functions.md +++ b/docs/ru/sql-reference/functions/ym-dict-functions.md @@ -1,6 +1,6 @@ --- toc_priority: 59 -toc_title: "\u0424\u0443\u043d\u043a\u0446\u0438\u0438\u0020\u0434\u043b\u044f\u0020\u0440\u0430\u0431\u043e\u0442\u044b\u0020\u0441\u043e\u0020\u0441\u043b\u043e\u0432\u0430\u0440\u044f\u043c\u0438\u0020\u042f\u043d\u0434\u0435\u043a\u0441\u002e\u041c\u0435\u0442\u0440\u0438\u043a\u0438" +toc_title: "Функции для работы со словарями Яндекс.Метрики" --- # Функции для работы со словарями Яндекс.Метрики {#ym-dict-functions} diff --git a/docs/ru/sql-reference/index.md b/docs/ru/sql-reference/index.md index f59232ee047..7aea530c7ee 100644 --- a/docs/ru/sql-reference/index.md +++ b/docs/ru/sql-reference/index.md @@ -1,5 +1,5 @@ --- -toc_folder_title: "\u0421\u043F\u0440\u0430\u0432\u043A\u0430 \u043F\u043E SQL" +toc_folder_title: "Справка по SQL" toc_hidden: true toc_priority: 28 toc_title: hidden diff --git a/docs/ru/sql-reference/operators/index.md b/docs/ru/sql-reference/operators/index.md index 1eddfc4dcaf..691c398ce4c 100644 --- a/docs/ru/sql-reference/operators/index.md +++ b/docs/ru/sql-reference/operators/index.md @@ -1,6 +1,6 @@ --- toc_priority: 38 -toc_title: "\u041e\u043f\u0435\u0440\u0430\u0442\u043e\u0440\u044b" +toc_title: "Операторы" --- # Операторы {#operatory} diff --git a/docs/ru/sql-reference/statements/alter/column.md b/docs/ru/sql-reference/statements/alter/column.md index 7a394e2f684..35a1952d842 100644 --- a/docs/ru/sql-reference/statements/alter/column.md +++ b/docs/ru/sql-reference/statements/alter/column.md @@ -1,6 +1,6 @@ --- toc_priority: 37 -toc_title: "\u041c\u0430\u043d\u0438\u043f\u0443\u043b\u044f\u0446\u0438\u0438\u0020\u0441\u043e\u0020\u0441\u0442\u043e\u043b\u0431\u0446\u0430\u043c\u0438" +toc_title: "Манипуляции со столбцами" --- # Манипуляции со столбцами {#manipuliatsii-so-stolbtsami} diff --git a/docs/ru/sql-reference/statements/alter/constraint.md b/docs/ru/sql-reference/statements/alter/constraint.md index e26db208493..13396f33621 100644 --- a/docs/ru/sql-reference/statements/alter/constraint.md +++ b/docs/ru/sql-reference/statements/alter/constraint.md @@ -1,6 +1,6 @@ --- toc_priority: 43 -toc_title: "\u041c\u0430\u043d\u0438\u043f\u0443\u043b\u044f\u0446\u0438\u0438\u0020\u0441\u0020\u043e\u0433\u0440\u0430\u043d\u0438\u0447\u0435\u043d\u0438\u044f\u043c\u0438" +toc_title: "Манипуляции с ограничениями" --- # Манипуляции с ограничениями (constraints) {#manipuliatsii-s-ogranicheniiami-constraints} diff --git a/docs/ru/sql-reference/statements/alter/index/index.md b/docs/ru/sql-reference/statements/alter/index/index.md index 2cadbbe065e..a42bccd7b47 100644 --- a/docs/ru/sql-reference/statements/alter/index/index.md +++ b/docs/ru/sql-reference/statements/alter/index/index.md @@ -1,7 +1,7 @@ --- toc_hidden_folder: true toc_priority: 42 -toc_title: "\u041c\u0430\u043d\u0438\u043f\u0443\u043b\u044f\u0446\u0438\u0438\u0020\u0441\u0020\u0438\u043d\u0434\u0435\u043a\u0441\u0430\u043c\u0438" +toc_title: "Манипуляции с индексами" --- # Манипуляции с индексами {#manipuliatsii-s-indeksami} diff --git a/docs/ru/sql-reference/statements/create/database.md b/docs/ru/sql-reference/statements/create/database.md index e6c561f8e0b..0e880517134 100644 --- a/docs/ru/sql-reference/statements/create/database.md +++ b/docs/ru/sql-reference/statements/create/database.md @@ -1,6 +1,6 @@ --- toc_priority: 35 -toc_title: "\u0411\u0430\u0437\u0430\u0020\u0434\u0430\u043d\u043d\u044b\u0445" +toc_title: "База данных" --- # CREATE DATABASE {#query-language-create-database} diff --git a/docs/ru/sql-reference/statements/create/dictionary.md b/docs/ru/sql-reference/statements/create/dictionary.md index 3134a89483b..dba2aa61ca1 100644 --- a/docs/ru/sql-reference/statements/create/dictionary.md +++ b/docs/ru/sql-reference/statements/create/dictionary.md @@ -1,6 +1,6 @@ --- toc_priority: 38 -toc_title: "\u0421\u043b\u043e\u0432\u0430\u0440\u044c" +toc_title: "Словарь" --- # CREATE DICTIONARY {#create-dictionary-query} diff --git a/docs/ru/sql-reference/statements/create/index.md b/docs/ru/sql-reference/statements/create/index.md index 28ddce2afe3..70961e4f404 100644 --- a/docs/ru/sql-reference/statements/create/index.md +++ b/docs/ru/sql-reference/statements/create/index.md @@ -1,7 +1,7 @@ --- toc_folder_title: CREATE toc_priority: 34 -toc_title: "\u041e\u0431\u0437\u043e\u0440" +toc_title: "Обзор" --- # Запросы CREATE {#create-queries} diff --git a/docs/ru/sql-reference/statements/create/quota.md b/docs/ru/sql-reference/statements/create/quota.md index 65762071ea2..f5ac0df010e 100644 --- a/docs/ru/sql-reference/statements/create/quota.md +++ b/docs/ru/sql-reference/statements/create/quota.md @@ -1,6 +1,6 @@ --- toc_priority: 42 -toc_title: "\u041a\u0432\u043e\u0442\u0430" +toc_title: "Квота" --- # CREATE QUOTA {#create-quota-statement} diff --git a/docs/ru/sql-reference/statements/create/role.md b/docs/ru/sql-reference/statements/create/role.md index 521117c0e89..8592f263156 100644 --- a/docs/ru/sql-reference/statements/create/role.md +++ b/docs/ru/sql-reference/statements/create/role.md @@ -1,6 +1,6 @@ --- toc_priority: 40 -toc_title: "\u0420\u043e\u043b\u044c" +toc_title: "Роль" --- # CREATE ROLE {#create-role-statement} diff --git a/docs/ru/sql-reference/statements/create/row-policy.md b/docs/ru/sql-reference/statements/create/row-policy.md index e79a19d4cbe..75f6fdfd2e1 100644 --- a/docs/ru/sql-reference/statements/create/row-policy.md +++ b/docs/ru/sql-reference/statements/create/row-policy.md @@ -1,6 +1,6 @@ --- toc_priority: 41 -toc_title: "\u041f\u043e\u043b\u0438\u0442\u0438\u043a\u0430\u0020\u0434\u043e\u0441\u0442\u0443\u043f\u0430" +toc_title: "Политика доступа" --- # CREATE ROW POLICY {#create-row-policy-statement} diff --git a/docs/ru/sql-reference/statements/create/settings-profile.md b/docs/ru/sql-reference/statements/create/settings-profile.md index 643f9d92eac..5838ddc9153 100644 --- a/docs/ru/sql-reference/statements/create/settings-profile.md +++ b/docs/ru/sql-reference/statements/create/settings-profile.md @@ -1,6 +1,6 @@ --- toc_priority: 43 -toc_title: "\u041f\u0440\u043e\u0444\u0438\u043b\u044c\u0020\u043d\u0430\u0441\u0442\u0440\u043e\u0435\u043a" +toc_title: "Профиль настроек" --- # CREATE SETTINGS PROFILE {#create-settings-profile-statement} diff --git a/docs/ru/sql-reference/statements/create/table.md b/docs/ru/sql-reference/statements/create/table.md index 9f582042a36..8e2c471e548 100644 --- a/docs/ru/sql-reference/statements/create/table.md +++ b/docs/ru/sql-reference/statements/create/table.md @@ -1,6 +1,6 @@ --- toc_priority: 36 -toc_title: "\u0422\u0430\u0431\u043b\u0438\u0446\u0430" +toc_title: "Таблица" --- # CREATE TABLE {#create-table-query} diff --git a/docs/ru/sql-reference/statements/create/user.md b/docs/ru/sql-reference/statements/create/user.md index bcc9768eb43..ac9547691e6 100644 --- a/docs/ru/sql-reference/statements/create/user.md +++ b/docs/ru/sql-reference/statements/create/user.md @@ -1,6 +1,6 @@ --- toc_priority: 39 -toc_title: "\u041f\u043e\u043b\u044c\u0437\u043e\u0432\u0430\u0442\u0435\u043b\u044c" +toc_title: "Пользователь" --- # CREATE USER {#create-user-statement} diff --git a/docs/ru/sql-reference/statements/create/view.md b/docs/ru/sql-reference/statements/create/view.md index f4b91b5ae17..da021059a8e 100644 --- a/docs/ru/sql-reference/statements/create/view.md +++ b/docs/ru/sql-reference/statements/create/view.md @@ -1,6 +1,6 @@ --- toc_priority: 37 -toc_title: "\u041f\u0440\u0435\u0434\u0441\u0442\u0430\u0432\u043b\u0435\u043d\u0438\u0435" +toc_title: "Представление" --- # CREATE VIEW {#create-view} diff --git a/docs/ru/sql-reference/statements/index.md b/docs/ru/sql-reference/statements/index.md index c7862015e64..5e72aa7cca0 100644 --- a/docs/ru/sql-reference/statements/index.md +++ b/docs/ru/sql-reference/statements/index.md @@ -1,5 +1,5 @@ --- -toc_folder_title: "\u0412\u044B\u0440\u0430\u0436\u0435\u043D\u0438\u044F" +toc_folder_title: "Выражения" toc_priority: 31 --- diff --git a/docs/ru/sql-reference/statements/select/index.md b/docs/ru/sql-reference/statements/select/index.md index b0b6e80d7be..a548a988a89 100644 --- a/docs/ru/sql-reference/statements/select/index.md +++ b/docs/ru/sql-reference/statements/select/index.md @@ -1,8 +1,8 @@ --- -title: "\u0421\u0438\u043d\u0442\u0430\u043a\u0441\u0438\u0441\u0020\u0437\u0430\u043f\u0440\u043e\u0441\u043e\u0432\u0020\u0053\u0045\u004c\u0045\u0043\u0054" +title: "Синтаксис запросов SELECT" toc_folder_title: SELECT toc_priority: 32 -toc_title: "\u041e\u0431\u0437\u043e\u0440" +toc_title: "Обзор" --- # Синтаксис запросов SELECT {#select-queries-syntax} diff --git a/docs/ru/sql-reference/syntax.md b/docs/ru/sql-reference/syntax.md index ca73d3a137e..d8eaa4f1731 100644 --- a/docs/ru/sql-reference/syntax.md +++ b/docs/ru/sql-reference/syntax.md @@ -1,6 +1,6 @@ --- toc_priority: 31 -toc_title: "\u0421\u0438\u043d\u0442\u0430\u043a\u0441\u0438\u0441" +toc_title: "Синтаксис" --- # Синтаксис {#sintaksis} diff --git a/docs/ru/sql-reference/table-functions/index.md b/docs/ru/sql-reference/table-functions/index.md index 83225d54e60..7606080a25e 100644 --- a/docs/ru/sql-reference/table-functions/index.md +++ b/docs/ru/sql-reference/table-functions/index.md @@ -1,8 +1,8 @@ --- -toc_folder_title: "\u0422\u0430\u0431\u043B\u0438\u0447\u043D\u044B\u0435 \u0444\u0443\ - \u043D\u043A\u0446\u0438\u0438" +toc_folder_title: "Табличные фу\ + нкции" toc_priority: 34 -toc_title: "\u0412\u0432\u0435\u0434\u0435\u043D\u0438\u0435" +toc_title: "Введение" --- # Табличные функции {#table-functions} diff --git a/docs/ru/whats-new/index.md b/docs/ru/whats-new/index.md index 256f6fde14e..d8a26423813 100644 --- a/docs/ru/whats-new/index.md +++ b/docs/ru/whats-new/index.md @@ -1,5 +1,5 @@ --- -toc_folder_title: "\u0427\u0442\u043E \u043D\u043E\u0432\u043E\u0433\u043E?" +toc_folder_title: "Что нового?" toc_priority: 82 --- From 095ee81a4e8769daf0805418150c61b22ad262e7 Mon Sep 17 00:00:00 2001 From: George Date: Mon, 8 Mar 2021 19:40:05 +0300 Subject: [PATCH 168/544] Updated description --- docs/en/sql-reference/statements/detach.md | 18 +----------------- 1 file changed, 1 insertion(+), 17 deletions(-) diff --git a/docs/en/sql-reference/statements/detach.md b/docs/en/sql-reference/statements/detach.md index 0bd4f730364..19ff8f10ad3 100644 --- a/docs/en/sql-reference/statements/detach.md +++ b/docs/en/sql-reference/statements/detach.md @@ -15,7 +15,7 @@ DETACH TABLE|VIEW [IF EXISTS] [db.]name [PERMANENTLY] [ON CLUSTER cluster] Detaching does not delete the data or metadata for the table or view. If the table or view was not detached `PERMANENTLY`, on the next server launch the server will read the metadata and recall the table/view again. If the table or view was detached `PERMANENTLY`, there will be no automatic recall. -Whether the table was detached permanently or not, in both cases you can reattach it using the [ATTACH](../../sql-reference/statements/attach.md) query (with the exception of system tables, which do not have metadata stored for them). +Whether the table was detached permanently or not, in both cases you can reattach it using the [ATTACH](../../sql-reference/statements/attach.md). System log tables can be also attached back (e.g. `query_log`, `text_log`, etc). Other system tables can't be reattached. On the next server launch the server will recall those tables again. `ATTACH MATERIALIZED VIEW` doesn't work with short syntax (without `SELECT`), but you can attach it using the `ATTACH TABLE` query. @@ -23,20 +23,4 @@ Note that you can not detach permanently the table which is already detached (te Also you can not [DROP](../../sql-reference/statements/drop.md#drop-table) the detached table, or [CREATE TABLE](../../sql-reference/statements/create/table.md) with the same name as detached permanently, or replace it with the other table with [RENAME TABLE](../../sql-reference/statements/rename.md) query. -Similarly, a “detached” table can be re-attached using the [ATTACH](../../sql-reference/statements/attach.md) query (with the exception of system tables, which do not have metadata stored for them). - -## DETACH PERMANENTLY {#detach-permanently} - -Deletes information about `name` table or view from the server. Permanently detached tables won't automatically reappear after the server restart. - -Syntax: - -``` sql -DETACH TABLE/VIEW [IF EXISTS] [db.]name PERMAMENTLY [ON CLUSTER cluster] -``` - -This statement does not delete the table’s data or metadata. - -Permanently detached table or view can be reattached with [ATTACH](../../sql-reference/statements/attach.md) query and can be shown with [SHOW CREATE TABLE](../../sql-reference/statements/show.md#show-create-table) query. - [Original article](https://clickhouse.tech/docs/en/sql-reference/statements/detach/) From ada200fa3d0bd7366b9b8bfcedc7ec4277de4258 Mon Sep 17 00:00:00 2001 From: Vasily Nemkov Date: Tue, 9 Mar 2021 01:03:40 +0300 Subject: [PATCH 169/544] Update skip_list.json --- tests/queries/skip_list.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/queries/skip_list.json b/tests/queries/skip_list.json index e808ef3dad6..a287d239759 100644 --- a/tests/queries/skip_list.json +++ b/tests/queries/skip_list.json @@ -760,7 +760,7 @@ "polygon_dicts", // they use an explicitly specified database "01658_read_file_to_stringcolumn", "01721_engine_file_truncate_on_insert", // It's ok to execute in parallel but not several instances of the same test. - "01702_system_query_log" // It's ok to execute in parallel but not several instances of the same test. + "01702_system_query_log" // It's ok to execute in parallel with oter tests but not several instances of the same test. "01748_dictionary_table_dot" // creates database ] } From 7664a3d53e1cf75e59ab251d390771fce3cc903c Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Tue, 9 Mar 2021 01:29:32 +0300 Subject: [PATCH 170/544] Fix MSan --- base/glibc-compatibility/memcpy/memcpy.h | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/base/glibc-compatibility/memcpy/memcpy.h b/base/glibc-compatibility/memcpy/memcpy.h index 034df74d832..86ffe0c0d53 100644 --- a/base/glibc-compatibility/memcpy/memcpy.h +++ b/base/glibc-compatibility/memcpy/memcpy.h @@ -2,6 +2,19 @@ #include +#if defined(__clang__) && defined(__has_feature) +# define ch_has_feature __has_feature +#endif +#if !defined(MEMORY_SANITIZER) +# if defined(ch_has_feature) +# if ch_has_feature(memory_sanitizer) +# define MEMORY_SANITIZER 1 +# endif +# elif defined(__MEMORY_SANITIZER__) +# define MEMORY_SANITIZER 1 +# endif +#endif + extern bool have_avx; void init_memcpy(); @@ -34,6 +47,7 @@ tail: *dst = *src; } } +#if !defined(MEMORY_SANITIZER) /// Asm code is not instrumented by MSan, skip this branch else if (have_avx) { if (size <= 32) @@ -120,6 +134,7 @@ tail: goto tail; } } +#endif else { if (size <= 128) From c3ddb1e33789befc93e0942ba620d05719617725 Mon Sep 17 00:00:00 2001 From: Pavel Kruglov Date: Tue, 9 Mar 2021 01:29:40 +0300 Subject: [PATCH 171/544] Fix tests --- tests/queries/0_stateless/01293_show_clusters.reference | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/queries/0_stateless/01293_show_clusters.reference b/tests/queries/0_stateless/01293_show_clusters.reference index 590ca348458..ad790cc8b4f 100644 --- a/tests/queries/0_stateless/01293_show_clusters.reference +++ b/tests/queries/0_stateless/01293_show_clusters.reference @@ -1,2 +1,2 @@ test_shard_localhost -test_shard_localhost 1 1 1 localhost ::1 9000 1 default 0 0 +test_shard_localhost 1 1 1 localhost ::1 9000 1 default 0 0 0 From 642c45d769c05421c890e3b9d5b59ecfe1e78964 Mon Sep 17 00:00:00 2001 From: Maksim Kita Date: Tue, 9 Mar 2021 01:32:31 +0300 Subject: [PATCH 172/544] PODArray left pad not multiple of element crash fix --- src/Common/PODArray.h | 17 ++++++++++++++--- src/Common/tests/gtest_pod_array.cpp | 26 ++++++++++++++++++++++++++ 2 files changed, 40 insertions(+), 3 deletions(-) diff --git a/src/Common/PODArray.h b/src/Common/PODArray.h index 8e05dfea8b3..c1a2546302f 100644 --- a/src/Common/PODArray.h +++ b/src/Common/PODArray.h @@ -166,6 +166,17 @@ protected: return (stack_threshold > 0) && (allocated_bytes() <= stack_threshold); } + bool shouldReserveForNextSizeBeforeInsert() const + { + /** end_of_storage = left_padding + (start + elements_size * ELEMENT_SIZE). + * end = start + elements_size * ELEMENT_SIZE + * We use end + ELEMENT_SIZE >= end_of_storage because it + * It is not safe to use end == end_of_storage here because left_padding + * is not always multiple of ELEMENT_SIZE. + */ + return c_end + ELEMENT_SIZE >= c_end_of_storage; + } + template void reserveForNextSize(TAllocatorParams &&... allocator_params) { @@ -334,7 +345,7 @@ public: using const_iterator = const T *; - PODArray() {} + PODArray() = default; PODArray(size_t n) { @@ -430,7 +441,7 @@ public: template void push_back(U && x, TAllocatorParams &&... allocator_params) { - if (unlikely(this->c_end == this->c_end_of_storage)) + if (unlikely(this->shouldReserveForNextSizeBeforeInsert())) this->reserveForNextSize(std::forward(allocator_params)...); new (t_end()) T(std::forward(x)); @@ -443,7 +454,7 @@ public: template void emplace_back(Args &&... args) { - if (unlikely(this->c_end == this->c_end_of_storage)) + if (unlikely(this->shouldReserveForNextSizeBeforeInsert())) this->reserveForNextSize(); new (t_end()) T(std::forward(args)...); diff --git a/src/Common/tests/gtest_pod_array.cpp b/src/Common/tests/gtest_pod_array.cpp index 988a3e649ba..53b3e207a22 100644 --- a/src/Common/tests/gtest_pod_array.cpp +++ b/src/Common/tests/gtest_pod_array.cpp @@ -66,3 +66,29 @@ TEST(Common, PODNoOverallocation) EXPECT_EQ(capacities, (std::vector{4065, 8161, 16353, 32737, 65505, 131041, 262113, 524257, 1048545})); } + +template +struct ItemWithSize +{ + char v[size] {}; +}; + +TEST(Common, PODInsertElementSizeNotMultipleOfLeftPadding) +{ + using ItemWith24Size = ItemWithSize<24>; + PaddedPODArray arr1_initially_empty; + + size_t items_to_insert_size = 120000; + + for (size_t test = 0; test < items_to_insert_size; ++test) + arr1_initially_empty.emplace_back(); + + EXPECT_EQ(arr1_initially_empty.size(), items_to_insert_size); + + PaddedPODArray arr2_initially_nonempty; + + for (size_t test = 0; test < items_to_insert_size; ++test) + arr2_initially_nonempty.emplace_back(); + + EXPECT_EQ(arr1_initially_empty.size(), items_to_insert_size); +} From 9ed791fd30b23ef5b4537c5b41433201e3699752 Mon Sep 17 00:00:00 2001 From: Maksim Kita Date: Tue, 9 Mar 2021 01:33:16 +0300 Subject: [PATCH 173/544] Updated style check --- src/Common/PODArray.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Common/PODArray.h b/src/Common/PODArray.h index c1a2546302f..579d500e6df 100644 --- a/src/Common/PODArray.h +++ b/src/Common/PODArray.h @@ -174,7 +174,7 @@ protected: * It is not safe to use end == end_of_storage here because left_padding * is not always multiple of ELEMENT_SIZE. */ - return c_end + ELEMENT_SIZE >= c_end_of_storage; + return (c_end + ELEMENT_SIZE) >= c_end_of_storage; } template From 9e23d1673972976a8c3a1b611a53a6d54660ff9f Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Tue, 9 Mar 2021 02:03:46 +0300 Subject: [PATCH 174/544] Cleanup PODArray --- src/Common/PODArray.h | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/src/Common/PODArray.h b/src/Common/PODArray.h index 8e05dfea8b3..f64392d9990 100644 --- a/src/Common/PODArray.h +++ b/src/Common/PODArray.h @@ -117,8 +117,11 @@ protected: template void alloc(size_t bytes, TAllocatorParams &&... allocator_params) { - c_start = c_end = reinterpret_cast(TAllocator::alloc(bytes, std::forward(allocator_params)...)) + pad_left; - c_end_of_storage = c_start + bytes - pad_right - pad_left; + char * allocated = reinterpret_cast(TAllocator::alloc(bytes, std::forward(allocator_params)...)); + + c_start = allocated + pad_left; + c_end = c_start; + c_end_of_storage = allocated + bytes - pad_right; if (pad_left) memset(c_start - ELEMENT_SIZE, 0, ELEMENT_SIZE); @@ -147,12 +150,12 @@ protected: ptrdiff_t end_diff = c_end - c_start; - c_start = reinterpret_cast( - TAllocator::realloc(c_start - pad_left, allocated_bytes(), bytes, std::forward(allocator_params)...)) - + pad_left; + char * allocated = reinterpret_cast( + TAllocator::realloc(c_start - pad_left, allocated_bytes(), bytes, std::forward(allocator_params)...)); + c_start = allocated + pad_left; c_end = c_start + end_diff; - c_end_of_storage = c_start + bytes - pad_right - pad_left; + c_end_of_storage = allocated + bytes - pad_right; } bool isInitialized() const From d3aeb30cc067335eae99090a8a13b62aea272c85 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Tue, 9 Mar 2021 02:10:26 +0300 Subject: [PATCH 175/544] Remove unused method --- src/Common/PODArray.h | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/Common/PODArray.h b/src/Common/PODArray.h index f64392d9990..ff38a05336b 100644 --- a/src/Common/PODArray.h +++ b/src/Common/PODArray.h @@ -321,11 +321,9 @@ protected: T * t_start() { return reinterpret_cast(this->c_start); } T * t_end() { return reinterpret_cast(this->c_end); } - T * t_end_of_storage() { return reinterpret_cast(this->c_end_of_storage); } const T * t_start() const { return reinterpret_cast(this->c_start); } const T * t_end() const { return reinterpret_cast(this->c_end); } - const T * t_end_of_storage() const { return reinterpret_cast(this->c_end_of_storage); } public: using value_type = T; From 1b3043a583141e2e77895ce266e71b6a1cbf6035 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Tue, 9 Mar 2021 02:15:33 +0300 Subject: [PATCH 176/544] Fix error (found by @kitaisreal) --- src/Common/PODArray.h | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/Common/PODArray.h b/src/Common/PODArray.h index ff38a05336b..cfc12ae1dc2 100644 --- a/src/Common/PODArray.h +++ b/src/Common/PODArray.h @@ -431,7 +431,7 @@ public: template void push_back(U && x, TAllocatorParams &&... allocator_params) { - if (unlikely(this->c_end == this->c_end_of_storage)) + if (unlikely(this->c_end + sizeof(T) > this->c_end_of_storage)) this->reserveForNextSize(std::forward(allocator_params)...); new (t_end()) T(std::forward(x)); @@ -444,7 +444,7 @@ public: template void emplace_back(Args &&... args) { - if (unlikely(this->c_end == this->c_end_of_storage)) + if (unlikely(this->c_end + sizeof(T) > this->c_end_of_storage)) this->reserveForNextSize(); new (t_end()) T(std::forward(args)...); From 5f65d469299fdd6c7958c2ef5ef2c8815fbc4035 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Tue, 9 Mar 2021 02:22:12 +0300 Subject: [PATCH 177/544] Fix Arcadia --- programs/main.cpp | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/programs/main.cpp b/programs/main.cpp index 54b36fb7781..447a3518dcc 100644 --- a/programs/main.cpp +++ b/programs/main.cpp @@ -23,8 +23,9 @@ #include #include -#if defined(__x86_64__) -#include +/// Custom memcpy - only for x86_64 and not for Arcadia. +#if defined(__x86_64__) && !defined(ARCADIA_BUILD) +#include // Y_IGNORE #else void init_memcpy() {} #endif From f5d96554168ba738b7c1014a65ce4874998ac157 Mon Sep 17 00:00:00 2001 From: alexey-milovidov Date: Tue, 9 Mar 2021 02:47:00 +0300 Subject: [PATCH 178/544] Update index.md --- docs/ru/sql-reference/table-functions/index.md | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/docs/ru/sql-reference/table-functions/index.md b/docs/ru/sql-reference/table-functions/index.md index 7606080a25e..da04b2c7a10 100644 --- a/docs/ru/sql-reference/table-functions/index.md +++ b/docs/ru/sql-reference/table-functions/index.md @@ -1,6 +1,5 @@ --- -toc_folder_title: "Табличные фу\ - нкции" +toc_folder_title: "Табличные функции" toc_priority: 34 toc_title: "Введение" --- From bcbf451f4c22adb052941023f6493f9c1c9ad2d6 Mon Sep 17 00:00:00 2001 From: alexey-milovidov Date: Tue, 9 Mar 2021 02:47:31 +0300 Subject: [PATCH 179/544] Update index.md --- docs/ru/commercial/index.md | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/docs/ru/commercial/index.md b/docs/ru/commercial/index.md index d8d37d0b7ad..66b1b125823 100644 --- a/docs/ru/commercial/index.md +++ b/docs/ru/commercial/index.md @@ -1,9 +1,7 @@ --- -toc_folder_title: "Коммерчески\ - е услуги" +toc_folder_title: "Коммерческие услуги" toc_priority: 70 -toc_title: "Коммерческие\ - \ услуги" +toc_title: "Коммерческие услуги" --- # Коммерческие услуги {#clickhouse-commercial-services} From 39dbc11a3d408248327120567222158c9d80a4e2 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Tue, 9 Mar 2021 03:21:38 +0300 Subject: [PATCH 180/544] Fix style check --- utils/memcpy-bench/FastMemcpy.h | 48 +++++++++++++++++--------- utils/memcpy-bench/FastMemcpy_Avx.h | 52 +++++++++++++++++++---------- utils/memcpy-bench/memcpy-bench.cpp | 2 -- 3 files changed, 66 insertions(+), 36 deletions(-) diff --git a/utils/memcpy-bench/FastMemcpy.h b/utils/memcpy-bench/FastMemcpy.h index 3f2278eac33..9c37524443a 100644 --- a/utils/memcpy-bench/FastMemcpy.h +++ b/utils/memcpy-bench/FastMemcpy.h @@ -1,3 +1,5 @@ +#pragma once + //===================================================================== // // FastMemcpy.c - skywind3000@163.com, 2015 @@ -6,7 +8,6 @@ // 50% speed up in avg. vs standard memcpy (tested in vc2012/gcc5.1) // //===================================================================== -#pragma once #include #include @@ -39,19 +40,22 @@ typedef __attribute__((__aligned__(1))) uint64_t uint64_unaligned_t; //--------------------------------------------------------------------- // fast copy for different sizes //--------------------------------------------------------------------- -static INLINE void memcpy_sse2_16(void * __restrict dst, const void * __restrict src) { +static INLINE void memcpy_sse2_16(void * __restrict dst, const void * __restrict src) +{ __m128i m0 = _mm_loadu_si128((reinterpret_cast(src)) + 0); _mm_storeu_si128((reinterpret_cast<__m128i*>(dst)) + 0, m0); } -static INLINE void memcpy_sse2_32(void * __restrict dst, const void * __restrict src) { +static INLINE void memcpy_sse2_32(void * __restrict dst, const void * __restrict src) +{ __m128i m0 = _mm_loadu_si128((reinterpret_cast(src)) + 0); __m128i m1 = _mm_loadu_si128((reinterpret_cast(src)) + 1); _mm_storeu_si128((reinterpret_cast<__m128i*>(dst)) + 0, m0); _mm_storeu_si128((reinterpret_cast<__m128i*>(dst)) + 1, m1); } -static INLINE void memcpy_sse2_64(void * __restrict dst, const void * __restrict src) { +static INLINE void memcpy_sse2_64(void * __restrict dst, const void * __restrict src) +{ __m128i m0 = _mm_loadu_si128((reinterpret_cast(src)) + 0); __m128i m1 = _mm_loadu_si128((reinterpret_cast(src)) + 1); __m128i m2 = _mm_loadu_si128((reinterpret_cast(src)) + 2); @@ -62,7 +66,8 @@ static INLINE void memcpy_sse2_64(void * __restrict dst, const void * __restrict _mm_storeu_si128((reinterpret_cast<__m128i*>(dst)) + 3, m3); } -static INLINE void memcpy_sse2_128(void * __restrict dst, const void * __restrict src) { +static INLINE void memcpy_sse2_128(void * __restrict dst, const void * __restrict src) +{ __m128i m0 = _mm_loadu_si128((reinterpret_cast(src)) + 0); __m128i m1 = _mm_loadu_si128((reinterpret_cast(src)) + 1); __m128i m2 = _mm_loadu_si128((reinterpret_cast(src)) + 2); @@ -88,11 +93,13 @@ static INLINE void memcpy_sse2_128(void * __restrict dst, const void * __restric /// Attribute is used to avoid an error with undefined behaviour sanitizer /// ../contrib/FastMemcpy/FastMemcpy.h:91:56: runtime error: applying zero offset to null pointer /// Found by 01307_orc_output_format.sh, cause - ORCBlockInputFormat and external ORC library. -__attribute__((__no_sanitize__("undefined"))) static INLINE void *memcpy_tiny(void * __restrict dst, const void * __restrict src, size_t size) { +__attribute__((__no_sanitize__("undefined"))) static INLINE void *memcpy_tiny(void * __restrict dst, const void * __restrict src, size_t size) +{ unsigned char *dd = ((unsigned char*)dst) + size; const unsigned char *ss = ((const unsigned char*)src) + size; - switch (size) { + switch (size) + { case 64: memcpy_sse2_64(dd - 64, ss - 64); [[fallthrough]]; @@ -653,14 +660,16 @@ void* memcpy_fast_sse(void * __restrict destination, const void * __restrict sou size_t padding; // small memory copy - if (size <= 128) { + if (size <= 128) +{ return memcpy_tiny(dst, src, size); } // align destination to 16 bytes boundary padding = (16 - (((size_t)dst) & 15)) & 15; - if (padding > 0) { + if (padding > 0) + { __m128i head = _mm_loadu_si128(reinterpret_cast(src)); _mm_storeu_si128(reinterpret_cast<__m128i*>(dst), head); dst += padding; @@ -669,10 +678,12 @@ void* memcpy_fast_sse(void * __restrict destination, const void * __restrict sou } // medium size copy - if (size <= cachesize) { + if (size <= cachesize) + { __m128i c0, c1, c2, c3, c4, c5, c6, c7; - for (; size >= 128; size -= 128) { + for (; size >= 128; size -= 128) + { c0 = _mm_loadu_si128((reinterpret_cast(src)) + 0); c1 = _mm_loadu_si128((reinterpret_cast(src)) + 1); c2 = _mm_loadu_si128((reinterpret_cast(src)) + 2); @@ -694,13 +705,16 @@ void* memcpy_fast_sse(void * __restrict destination, const void * __restrict sou dst += 128; } } - else { // big memory copy + else + { // big memory copy __m128i c0, c1, c2, c3, c4, c5, c6, c7; _mm_prefetch((const char*)(src), _MM_HINT_NTA); - if ((((size_t)src) & 15) == 0) { // source aligned - for (; size >= 128; size -= 128) { + if ((((size_t)src) & 15) == 0) + { // source aligned + for (; size >= 128; size -= 128) + { c0 = _mm_load_si128((reinterpret_cast(src)) + 0); c1 = _mm_load_si128((reinterpret_cast(src)) + 1); c2 = _mm_load_si128((reinterpret_cast(src)) + 2); @@ -722,8 +736,10 @@ void* memcpy_fast_sse(void * __restrict destination, const void * __restrict sou dst += 128; } } - else { // source unaligned - for (; size >= 128; size -= 128) { + else + { // source unaligned + for (; size >= 128; size -= 128) + { c0 = _mm_loadu_si128((reinterpret_cast(src)) + 0); c1 = _mm_loadu_si128((reinterpret_cast(src)) + 1); c2 = _mm_loadu_si128((reinterpret_cast(src)) + 2); diff --git a/utils/memcpy-bench/FastMemcpy_Avx.h b/utils/memcpy-bench/FastMemcpy_Avx.h index a36964eb013..ee7d4e19536 100644 --- a/utils/memcpy-bench/FastMemcpy_Avx.h +++ b/utils/memcpy-bench/FastMemcpy_Avx.h @@ -1,3 +1,5 @@ +#pragma once + //===================================================================== // // FastMemcpy.c - skywind3000@163.com, 2015 @@ -6,7 +8,6 @@ // 50% speed up in avg. vs standard memcpy (tested in vc2012/gcc5.1) // //===================================================================== -#pragma once #include #include @@ -33,11 +34,11 @@ #endif - //--------------------------------------------------------------------- // fast copy for different sizes //--------------------------------------------------------------------- -static INLINE void memcpy_avx_16(void * __restrict dst, const void * __restrict src) { +static INLINE void memcpy_avx_16(void * __restrict dst, const void * __restrict src) +{ #if 1 __m128i m0 = _mm_loadu_si128(((const __m128i*)src) + 0); _mm_storeu_si128(((__m128i*)dst) + 0, m0); @@ -47,19 +48,22 @@ static INLINE void memcpy_avx_16(void * __restrict dst, const void * __restrict #endif } -static INLINE void memcpy_avx_32(void *dst, const void *src) { +static INLINE void memcpy_avx_32(void *dst, const void *src) +{ __m256i m0 = _mm256_loadu_si256((reinterpret_cast(src)) + 0); _mm256_storeu_si256((reinterpret_cast<__m256i*>(dst)) + 0, m0); } -static INLINE void memcpy_avx_64(void *dst, const void *src) { +static INLINE void memcpy_avx_64(void *dst, const void *src) +{ __m256i m0 = _mm256_loadu_si256((reinterpret_cast(src)) + 0); __m256i m1 = _mm256_loadu_si256((reinterpret_cast(src)) + 1); _mm256_storeu_si256((reinterpret_cast<__m256i*>(dst)) + 0, m0); _mm256_storeu_si256((reinterpret_cast<__m256i*>(dst)) + 1, m1); } -static INLINE void memcpy_avx_128(void *dst, const void *src) { +static INLINE void memcpy_avx_128(void *dst, const void *src) +{ __m256i m0 = _mm256_loadu_si256((reinterpret_cast(src)) + 0); __m256i m1 = _mm256_loadu_si256((reinterpret_cast(src)) + 1); __m256i m2 = _mm256_loadu_si256((reinterpret_cast(src)) + 2); @@ -70,7 +74,8 @@ static INLINE void memcpy_avx_128(void *dst, const void *src) { _mm256_storeu_si256((reinterpret_cast<__m256i*>(dst)) + 3, m3); } -static INLINE void memcpy_avx_256(void *dst, const void *src) { +static INLINE void memcpy_avx_256(void *dst, const void *src) +{ __m256i m0 = _mm256_loadu_si256((reinterpret_cast(src)) + 0); __m256i m1 = _mm256_loadu_si256((reinterpret_cast(src)) + 1); __m256i m2 = _mm256_loadu_si256((reinterpret_cast(src)) + 2); @@ -93,11 +98,13 @@ static INLINE void memcpy_avx_256(void *dst, const void *src) { //--------------------------------------------------------------------- // tiny memory copy with jump table optimized //--------------------------------------------------------------------- -static INLINE void *memcpy_tiny_avx(void * __restrict dst, const void * __restrict src, size_t size) { +static INLINE void *memcpy_tiny_avx(void * __restrict dst, const void * __restrict src, size_t size) +{ unsigned char *dd = reinterpret_cast(dst) + size; const unsigned char *ss = reinterpret_cast(src) + size; - switch (size) { + switch (size) + { case 128: memcpy_avx_128(dd - 128, ss - 128); [[fallthrough]]; case 0: break; case 129: memcpy_avx_128(dd - 129, ss - 129); [[fallthrough]]; @@ -372,7 +379,8 @@ void* memcpy_fast_avx(void * __restrict destination, const void * __restrict sou size_t padding; // small memory copy - if (size <= 256) { + if (size <= 256) + { memcpy_tiny_avx(dst, src, size); _mm256_zeroupper(); return destination; @@ -382,7 +390,8 @@ void* memcpy_fast_avx(void * __restrict destination, const void * __restrict sou padding = (32 - (((size_t)dst) & 31)) & 31; #if 0 - if (padding > 0) { + if (padding > 0) + { __m256i head = _mm256_loadu_si256(reinterpret_cast(src)); _mm256_storeu_si256((__m256i*)dst, head); dst += padding; @@ -398,10 +407,12 @@ void* memcpy_fast_avx(void * __restrict destination, const void * __restrict sou #endif // medium size copy - if (size <= cachesize) { + if (size <= cachesize) + { __m256i c0, c1, c2, c3, c4, c5, c6, c7; - for (; size >= 256; size -= 256) { + for (; size >= 256; size -= 256) + { c0 = _mm256_loadu_si256((reinterpret_cast(src)) + 0); c1 = _mm256_loadu_si256((reinterpret_cast(src)) + 1); c2 = _mm256_loadu_si256((reinterpret_cast(src)) + 2); @@ -422,12 +433,15 @@ void* memcpy_fast_avx(void * __restrict destination, const void * __restrict sou dst += 256; } } - else { // big memory copy + else + { // big memory copy __m256i c0, c1, c2, c3, c4, c5, c6, c7; /* __m256i c0, c1, c2, c3, c4, c5, c6, c7; */ - if ((((size_t)src) & 31) == 0) { // source aligned - for (; size >= 256; size -= 256) { + if ((((size_t)src) & 31) == 0) + { // source aligned + for (; size >= 256; size -= 256) + { c0 = _mm256_load_si256((reinterpret_cast(src)) + 0); c1 = _mm256_load_si256((reinterpret_cast(src)) + 1); c2 = _mm256_load_si256((reinterpret_cast(src)) + 2); @@ -448,8 +462,10 @@ void* memcpy_fast_avx(void * __restrict destination, const void * __restrict sou dst += 256; } } - else { // source unaligned - for (; size >= 256; size -= 256) { + else + { // source unaligned + for (; size >= 256; size -= 256) + { c0 = _mm256_loadu_si256((reinterpret_cast(src)) + 0); c1 = _mm256_loadu_si256((reinterpret_cast(src)) + 1); c2 = _mm256_loadu_si256((reinterpret_cast(src)) + 2); diff --git a/utils/memcpy-bench/memcpy-bench.cpp b/utils/memcpy-bench/memcpy-bench.cpp index 05029ee5a69..2df72cb5ccb 100644 --- a/utils/memcpy-bench/memcpy-bench.cpp +++ b/utils/memcpy-bench/memcpy-bench.cpp @@ -24,7 +24,6 @@ #include - template void NO_INLINE loop(uint8_t * dst, uint8_t * src, size_t size, F && chunk_size_distribution, MemcpyImpl && impl) { @@ -541,7 +540,6 @@ tail: } - template void dispatchMemcpyVariants(size_t memcpy_variant, uint8_t * dst, uint8_t * src, size_t size, size_t iterations, size_t num_threads, F && generator) { From 3474ea044e29a3a1876c249c08efe07aca295476 Mon Sep 17 00:00:00 2001 From: Azat Khuzhin Date: Tue, 9 Mar 2021 10:05:56 +0300 Subject: [PATCH 181/544] Avoid processing optimize_skip_unused_shards twice --- src/Storages/StorageDistributed.cpp | 2 +- .../01758_optimize_skip_unused_shards_once.reference | 2 ++ .../01758_optimize_skip_unused_shards_once.sh | 12 ++++++++++++ 3 files changed, 15 insertions(+), 1 deletion(-) create mode 100644 tests/queries/0_stateless/01758_optimize_skip_unused_shards_once.reference create mode 100755 tests/queries/0_stateless/01758_optimize_skip_unused_shards_once.sh diff --git a/src/Storages/StorageDistributed.cpp b/src/Storages/StorageDistributed.cpp index 039cf63eca2..2a05d92ace1 100644 --- a/src/Storages/StorageDistributed.cpp +++ b/src/Storages/StorageDistributed.cpp @@ -530,7 +530,7 @@ void StorageDistributed::read( query_info.query, remote_database, remote_table, remote_table_function_ptr); Block header = - InterpreterSelectQuery(query_info.query, context, SelectQueryOptions(processed_stage)).getSampleBlock(); + InterpreterSelectQuery(query_info.query, context, SelectQueryOptions(processed_stage).analyze()).getSampleBlock(); const Scalars & scalars = context.hasQueryContext() ? context.getQueryContext().getScalars() : Scalars{}; diff --git a/tests/queries/0_stateless/01758_optimize_skip_unused_shards_once.reference b/tests/queries/0_stateless/01758_optimize_skip_unused_shards_once.reference new file mode 100644 index 00000000000..d23ada2e8d4 --- /dev/null +++ b/tests/queries/0_stateless/01758_optimize_skip_unused_shards_once.reference @@ -0,0 +1,2 @@ +StorageDistributed (dist_01758): Auto-increment is 0 +StorageDistributed (dist_01758): Skipping irrelevant shards - the query will be sent to the following shards of the cluster (shard numbers): [1] diff --git a/tests/queries/0_stateless/01758_optimize_skip_unused_shards_once.sh b/tests/queries/0_stateless/01758_optimize_skip_unused_shards_once.sh new file mode 100755 index 00000000000..b26961eda8e --- /dev/null +++ b/tests/queries/0_stateless/01758_optimize_skip_unused_shards_once.sh @@ -0,0 +1,12 @@ +#!/usr/bin/env bash + +CLICKHOUSE_CLIENT_SERVER_LOGS_LEVEL=trace + +CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh +. "$CUR_DIR"/../shell_config.sh + +$CLICKHOUSE_CLIENT --optimize_skip_unused_shards=1 -nm -q " +create table dist_01758 as system.one engine=Distributed(test_cluster_two_shards, system, one, dummy); +select * from dist_01758 where dummy = 0 format Null; +" |& grep -o "StorageDistributed (dist_01758).*" From d80c2cef0668f79c0c9d6f421a333233ba0e9da4 Mon Sep 17 00:00:00 2001 From: alesapin Date: Tue, 9 Mar 2021 11:45:41 +0300 Subject: [PATCH 182/544] Slightly better --- src/Storages/StorageReplicatedMergeTree.cpp | 36 +++++++++------------ 1 file changed, 15 insertions(+), 21 deletions(-) diff --git a/src/Storages/StorageReplicatedMergeTree.cpp b/src/Storages/StorageReplicatedMergeTree.cpp index ddc63793640..bfa4efbd739 100644 --- a/src/Storages/StorageReplicatedMergeTree.cpp +++ b/src/Storages/StorageReplicatedMergeTree.cpp @@ -528,6 +528,7 @@ void StorageReplicatedMergeTree::waitMutationToFinishOnReplicas( void StorageReplicatedMergeTree::createNewZooKeeperNodes() { + auto storage_settings = getSettings(); auto zookeeper = getZooKeeper(); /// Working with quorum. @@ -543,6 +544,14 @@ void StorageReplicatedMergeTree::createNewZooKeeperNodes() /// Mutations zookeeper->createIfNotExists(zookeeper_path + "/mutations", String()); zookeeper->createIfNotExists(replica_path + "/mutation_pointer", String()); + + /// Nodes for zero-copy S3 replication + if (storage_settings->allow_s3_zero_copy_replication) + { + zookeeper->createIfNotExists(zookeeper_path + "/zero_copy_s3", String()); + zookeeper->createIfNotExists(zookeeper_path + "/zero_copy_s3/merged", String()); + zookeeper->createIfNotExists(zookeeper_path + "/zero_copy_s3/shared", String()); + } } @@ -1541,27 +1550,12 @@ bool StorageReplicatedMergeTree::tryExecuteMerge(const LogEntry & entry) auto zookeeper = getZooKeeper(); String zookeeper_node = zookeeper_path + "/zero_copy_s3/merged/" + entry.new_part_name; - /// In rare case other replica can remove path between createAncestors and tryCreate - /// So we make up to 5 attempts to make a lock - for (int attempts = 5; attempts > 0; --attempts) - { - try - { - zookeeper->createAncestors(zookeeper_node); - auto code = zookeeper->tryCreate(zookeeper_node, "lock", zkutil::CreateMode::Ephemeral); - /// Someone else created or started create this merge - if (code == Coordination::Error::ZNODEEXISTS) - return false; - if (code != Coordination::Error::ZNONODE) - break; - } - catch (const zkutil::KeeperException & e) - { - if (e.code == Coordination::Error::ZNONODE) - continue; - throw; - } - } + auto code = zookeeper->tryCreate(zookeeper_node, "lock", zkutil::CreateMode::Ephemeral); + + /// Someone else created or started create this merge, + /// so will try to fetch. + if (code == Coordination::Error::ZNODEEXISTS) + return false; } } From 8dd52f087efe978434b5e281016e5ef864b62062 Mon Sep 17 00:00:00 2001 From: Maksim Kita Date: Tue, 9 Mar 2021 14:14:31 +0300 Subject: [PATCH 183/544] Fixed PODArray --- src/Common/PODArray.h | 15 ++------------- 1 file changed, 2 insertions(+), 13 deletions(-) diff --git a/src/Common/PODArray.h b/src/Common/PODArray.h index 579d500e6df..aa7454badd6 100644 --- a/src/Common/PODArray.h +++ b/src/Common/PODArray.h @@ -166,17 +166,6 @@ protected: return (stack_threshold > 0) && (allocated_bytes() <= stack_threshold); } - bool shouldReserveForNextSizeBeforeInsert() const - { - /** end_of_storage = left_padding + (start + elements_size * ELEMENT_SIZE). - * end = start + elements_size * ELEMENT_SIZE - * We use end + ELEMENT_SIZE >= end_of_storage because it - * It is not safe to use end == end_of_storage here because left_padding - * is not always multiple of ELEMENT_SIZE. - */ - return (c_end + ELEMENT_SIZE) >= c_end_of_storage; - } - template void reserveForNextSize(TAllocatorParams &&... allocator_params) { @@ -441,7 +430,7 @@ public: template void push_back(U && x, TAllocatorParams &&... allocator_params) { - if (unlikely(this->shouldReserveForNextSizeBeforeInsert())) + if (unlikely(this->c_end == this->c_end_of_storage)) this->reserveForNextSize(std::forward(allocator_params)...); new (t_end()) T(std::forward(x)); @@ -454,7 +443,7 @@ public: template void emplace_back(Args &&... args) { - if (unlikely(this->shouldReserveForNextSizeBeforeInsert())) + if (unlikely(this->c_end == this->c_end_of_storage)) this->reserveForNextSize(); new (t_end()) T(std::forward(args)...); From c38a690e8e2c85952dd435bd1e4ea981536a3244 Mon Sep 17 00:00:00 2001 From: Nikita Mikhaylov Date: Thu, 4 Mar 2021 16:07:40 +0300 Subject: [PATCH 184/544] add multithreaded test --- tests/integration/test_zookeeper_config/test.py | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/tests/integration/test_zookeeper_config/test.py b/tests/integration/test_zookeeper_config/test.py index 80875da45e0..4f939888492 100644 --- a/tests/integration/test_zookeeper_config/test.py +++ b/tests/integration/test_zookeeper_config/test.py @@ -1,6 +1,7 @@ import time +import threading from os import path as p, unlink from tempfile import NamedTemporaryFile @@ -174,6 +175,17 @@ def test_secure_connection(): assert node1.query("SELECT count() FROM system.zookeeper WHERE path = '/'") == '2\n' assert node2.query("SELECT count() FROM system.zookeeper WHERE path = '/'") == '2\n' + + kThreadsNumber = 16 + kIterations = 100 + threads = [] + for _ in range(kThreadsNumber): + threads.append(threading.Thread(target=(lambda: + [node1.query("SELECT count() FROM system.zookeeper WHERE path = '/'") for _ in range(kIterations)]))) + + for thread in threads: + thread.join() + finally: cluster.shutdown() unlink(docker_compose.name) From 2c6fe0ea69ffa0da5175d3e0cb71316a05eba70a Mon Sep 17 00:00:00 2001 From: Nikita Mikhaylov Date: Thu, 4 Mar 2021 18:06:44 +0300 Subject: [PATCH 185/544] add poco --- .gitmodules | 2 +- contrib/poco | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.gitmodules b/.gitmodules index 7a2c5600e65..be7b314eb95 100644 --- a/.gitmodules +++ b/.gitmodules @@ -1,7 +1,7 @@ [submodule "contrib/poco"] path = contrib/poco url = https://github.com/ClickHouse-Extras/poco.git - branch = clickhouse + branch = secure-socket-race [submodule "contrib/zstd"] path = contrib/zstd url = https://github.com/facebook/zstd.git diff --git a/contrib/poco b/contrib/poco index fbaaba4a02e..38105251751 160000 --- a/contrib/poco +++ b/contrib/poco @@ -1 +1 @@ -Subproject commit fbaaba4a02e29987b8c584747a496c79528f125f +Subproject commit 381052517515dd95402a991b49a8e8eaa9cf692e From 9684fed32a1e7886ac3f941c5f2e3edc1890945e Mon Sep 17 00:00:00 2001 From: Nikita Mikhaylov Date: Thu, 4 Mar 2021 21:07:05 +0300 Subject: [PATCH 186/544] better test --- tests/integration/test_zookeeper_config/test.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/tests/integration/test_zookeeper_config/test.py b/tests/integration/test_zookeeper_config/test.py index 4f939888492..584f76c80f0 100644 --- a/tests/integration/test_zookeeper_config/test.py +++ b/tests/integration/test_zookeeper_config/test.py @@ -183,6 +183,9 @@ def test_secure_connection(): threads.append(threading.Thread(target=(lambda: [node1.query("SELECT count() FROM system.zookeeper WHERE path = '/'") for _ in range(kIterations)]))) + for thread in threads: + thread.start() + for thread in threads: thread.join() From 3acc4795ead74f44e2a095782ae70d4121b07477 Mon Sep 17 00:00:00 2001 From: Nikita Mikhaylov Date: Thu, 4 Mar 2021 23:02:25 +0300 Subject: [PATCH 187/544] bump poco --- contrib/poco | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/contrib/poco b/contrib/poco index 38105251751..97df9ec7896 160000 --- a/contrib/poco +++ b/contrib/poco @@ -1 +1 @@ -Subproject commit 381052517515dd95402a991b49a8e8eaa9cf692e +Subproject commit 97df9ec7896ddd552dbed4432404caa3e1d0fc04 From 0a44cdc50c59ebd55511bc34668962438d911f21 Mon Sep 17 00:00:00 2001 From: Nikita Mikhaylov Date: Fri, 5 Mar 2021 00:19:08 +0300 Subject: [PATCH 188/544] bump poco --- contrib/poco | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/contrib/poco b/contrib/poco index 97df9ec7896..3d319f1eeaa 160000 --- a/contrib/poco +++ b/contrib/poco @@ -1 +1 @@ -Subproject commit 97df9ec7896ddd552dbed4432404caa3e1d0fc04 +Subproject commit 3d319f1eeaa4abbc15ca513586df473d3d50267e From dfad6dca64b2586de3cf65b4a65ed50f7dbd3187 Mon Sep 17 00:00:00 2001 From: Nikita Mikhaylov Date: Fri, 5 Mar 2021 17:52:57 +0300 Subject: [PATCH 189/544] bump poco to merged PR --- .gitmodules | 2 +- contrib/poco | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.gitmodules b/.gitmodules index be7b314eb95..7a2c5600e65 100644 --- a/.gitmodules +++ b/.gitmodules @@ -1,7 +1,7 @@ [submodule "contrib/poco"] path = contrib/poco url = https://github.com/ClickHouse-Extras/poco.git - branch = secure-socket-race + branch = clickhouse [submodule "contrib/zstd"] path = contrib/zstd url = https://github.com/facebook/zstd.git diff --git a/contrib/poco b/contrib/poco index 3d319f1eeaa..c55b91f394e 160000 --- a/contrib/poco +++ b/contrib/poco @@ -1 +1 @@ -Subproject commit 3d319f1eeaa4abbc15ca513586df473d3d50267e +Subproject commit c55b91f394efa9c238c33957682501681ef9b716 From 93a00cde5d96e04bc47d6ccd7cce3865a1fa8269 Mon Sep 17 00:00:00 2001 From: Vasily Nemkov Date: Tue, 9 Mar 2021 16:56:23 +0300 Subject: [PATCH 190/544] Update skip_list.json --- tests/queries/skip_list.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/queries/skip_list.json b/tests/queries/skip_list.json index a287d239759..db47beeffc8 100644 --- a/tests/queries/skip_list.json +++ b/tests/queries/skip_list.json @@ -760,7 +760,7 @@ "polygon_dicts", // they use an explicitly specified database "01658_read_file_to_stringcolumn", "01721_engine_file_truncate_on_insert", // It's ok to execute in parallel but not several instances of the same test. - "01702_system_query_log" // It's ok to execute in parallel with oter tests but not several instances of the same test. + "01702_system_query_log", // It's ok to execute in parallel with oter tests but not several instances of the same test. "01748_dictionary_table_dot" // creates database ] } From 71d0a505080da1e5457ed7fe7e33b94abded2764 Mon Sep 17 00:00:00 2001 From: Yatsishin Ilya <2159081+qoega@users.noreply.github.com> Date: Tue, 9 Mar 2021 17:24:12 +0300 Subject: [PATCH 191/544] add llvm-12 binaries name --- CMakeLists.txt | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 9002f1df140..76b79a0b6c8 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -169,7 +169,7 @@ endif () set (CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -rdynamic") if (OS_LINUX) - find_program (OBJCOPY_PATH NAMES "llvm-objcopy" "llvm-objcopy-11" "llvm-objcopy-10" "llvm-objcopy-9" "llvm-objcopy-8" "objcopy") + find_program (OBJCOPY_PATH NAMES "llvm-objcopy" "llvm-objcopy-12" "llvm-objcopy-11" "llvm-objcopy-10" "llvm-objcopy-9" "llvm-objcopy-8" "objcopy") if (OBJCOPY_PATH) message(STATUS "Using objcopy: ${OBJCOPY_PATH}.") @@ -331,7 +331,7 @@ if (COMPILER_CLANG) endif () # Always prefer llvm tools when using clang. For instance, we cannot use GNU ar when llvm LTO is enabled - find_program (LLVM_AR_PATH NAMES "llvm-ar" "llvm-ar-11" "llvm-ar-10" "llvm-ar-9" "llvm-ar-8") + find_program (LLVM_AR_PATH NAMES "llvm-ar" "llvm-ar-12" "llvm-ar-11" "llvm-ar-10" "llvm-ar-9" "llvm-ar-8") if (LLVM_AR_PATH) message(STATUS "Using llvm-ar: ${LLVM_AR_PATH}.") @@ -340,7 +340,7 @@ if (COMPILER_CLANG) message(WARNING "Cannot find llvm-ar. System ar will be used instead. It does not work with ThinLTO.") endif () - find_program (LLVM_RANLIB_PATH NAMES "llvm-ranlib" "llvm-ranlib-11" "llvm-ranlib-10" "llvm-ranlib-9" "llvm-ranlib-8") + find_program (LLVM_RANLIB_PATH NAMES "llvm-ranlib" "llvm-ranlib-12" "llvm-ranlib-11" "llvm-ranlib-10" "llvm-ranlib-9" "llvm-ranlib-8") if (LLVM_RANLIB_PATH) message(STATUS "Using llvm-ranlib: ${LLVM_RANLIB_PATH}.") From 71d745b4ec38e9f7345f0b7ef3beca1aa34831ee Mon Sep 17 00:00:00 2001 From: Yatsishin Ilya <2159081+qoega@users.noreply.github.com> Date: Tue, 9 Mar 2021 17:31:54 +0300 Subject: [PATCH 192/544] implicit const conversion --- base/common/wide_integer_impl.h | 8 ++++---- src/Compression/tests/gtest_compressionCodec.cpp | 2 +- src/DataTypes/DataTypesDecimal.h | 2 +- src/Functions/DivisionUtils.h | 2 +- 4 files changed, 7 insertions(+), 7 deletions(-) diff --git a/base/common/wide_integer_impl.h b/base/common/wide_integer_impl.h index a34e757eaa5..56f1f64d1be 100644 --- a/base/common/wide_integer_impl.h +++ b/base/common/wide_integer_impl.h @@ -249,15 +249,15 @@ struct integer::_impl return; } - const T alpha = t / max_int; + const T alpha = t / ((T) max_int); - if (alpha <= max_int) + if (alpha <= ((T) max_int)) self = static_cast(alpha); else // max(double) / 2^64 will surely contain less than 52 precision bits, so speed up computations. set_multiplier(self, alpha); self *= max_int; - self += static_cast(t - alpha * max_int); // += b_i + self += static_cast(t - alpha * ((T) max_int)); // += b_i } constexpr static void wide_integer_from_bultin(integer& self, double rhs) noexcept { @@ -275,7 +275,7 @@ struct integer::_impl "On your system long double has less than 64 precision bits," "which may result in UB when initializing double from int64_t"); - if ((rhs > 0 && rhs < max_int) || (rhs < 0 && rhs > min_int)) + if ((rhs > 0 && rhs < (double) max_int) || (rhs < 0 && rhs > (double) min_int)) { self = static_cast(rhs); return; diff --git a/src/Compression/tests/gtest_compressionCodec.cpp b/src/Compression/tests/gtest_compressionCodec.cpp index e9470536ae8..3e450352eac 100644 --- a/src/Compression/tests/gtest_compressionCodec.cpp +++ b/src/Compression/tests/gtest_compressionCodec.cpp @@ -749,7 +749,7 @@ auto RandomishGenerator = [](auto i) { using T = decltype(i); double sin_value = sin(static_cast(i * i)) * i; - if (sin_value < std::numeric_limits::lowest() || sin_value > std::numeric_limits::max()) + if (sin_value < std::numeric_limits::lowest() || sin_value > (double) std::numeric_limits::max()) return T{}; return T(sin_value); }; diff --git a/src/DataTypes/DataTypesDecimal.h b/src/DataTypes/DataTypesDecimal.h index 2b708b53be0..0d7a297bdf0 100644 --- a/src/DataTypes/DataTypesDecimal.h +++ b/src/DataTypes/DataTypesDecimal.h @@ -212,7 +212,7 @@ convertToDecimalImpl(const typename FromDataType::FieldType & value, UInt32 scal static constexpr Int128 min_int128 = minInt128(); static constexpr Int128 max_int128 = maxInt128(); - if (out <= static_cast(min_int128) || out >= static_cast(max_int128)) + if (out <= static_cast(min_int128) || out >= (float) static_cast(max_int128)) { if constexpr (throw_exception) throw Exception(std::string(ToDataType::family_name) + " convert overflow. Float is out of Decimal range", diff --git a/src/Functions/DivisionUtils.h b/src/Functions/DivisionUtils.h index 2b4c07b1cff..63367113066 100644 --- a/src/Functions/DivisionUtils.h +++ b/src/Functions/DivisionUtils.h @@ -105,7 +105,7 @@ struct DivideIntegralImpl auto res = checkedDivision(CastA(a), CastB(b)); if constexpr (std::is_floating_point_v) - if (isNaN(res) || res >= std::numeric_limits::max() || res <= std::numeric_limits::lowest()) + if (isNaN(res) || res >= (double) std::numeric_limits::max() || res <= std::numeric_limits::lowest()) throw Exception("Cannot perform integer division, because it will produce infinite or too large number", ErrorCodes::ILLEGAL_DIVISION); From 0188c195b9317e974718ab3f52ad1646b47350a4 Mon Sep 17 00:00:00 2001 From: Maksim Kita Date: Tue, 9 Mar 2021 20:01:49 +0300 Subject: [PATCH 193/544] Update settings.md --- docs/ru/operations/settings/settings.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/ru/operations/settings/settings.md b/docs/ru/operations/settings/settings.md index deda437e933..bfa679f461d 100644 --- a/docs/ru/operations/settings/settings.md +++ b/docs/ru/operations/settings/settings.md @@ -1088,9 +1088,9 @@ load_balancing = round_robin Максимальное кол-во реплик для каждого шарда во время исполениня запроса из distributed. В некоторых случаях, это может привести к более быстрому исполнению запроса за счет выполнения на большем кол-ве серверов. Эта настройка полезна только для реплицируемых таблиц созданных с использованием SAMPLING KEY выражения. Есть случаи когда производительность не улучшится или даже ухудшится: -- позиция ключа семплирования в ключе партицирования не позволяет делать эффективные сканирования по диапозонам -- добавление семплирующего ключа к таблице, делает фильтрацию других колонок менее эффективной -- выражение используемое для вычисления ключа семплирования требует больших вычислительных затрат +- Позиция ключа семплирования в ключе партицирования не позволяет делать эффективные сканирования по диапозонам +- Добавление семплирующего ключа к таблице, делает фильтрацию других колонок менее эффективной +- Выражение используемое для вычисления ключа семплирования требует больших вычислительных затрат - Распределение сетевых задержек внутри кластера имеет длинный хвост, так что запрос большего количества серверов может увеличить общую задержку запроса Кроме того, эта настройка может привести к некорректным результатам когда используются join или подзапросы и все таблицы не соответсвуют определенным условиям. Подробнее [Распределенные подзапросы и max_parallel_replicas](../../sql-reference/operators/in.md#max_parallel_replica-subqueries) for more details. From 265d2939340df643613f86c6325f893f76445b97 Mon Sep 17 00:00:00 2001 From: Anton Ivashkin Date: Tue, 9 Mar 2021 17:34:28 +0300 Subject: [PATCH 194/544] Use 'merge on single replica' option instead of zookeeper lock --- src/Storages/MergeTree/MergeTreeSettings.h | 1 + ...ReplicatedMergeTreeMergeStrategyPicker.cpp | 28 ++++++++++++++++--- .../ReplicatedMergeTreeMergeStrategyPicker.h | 5 ++++ src/Storages/StorageReplicatedMergeTree.cpp | 27 ++++++++++-------- 4 files changed, 45 insertions(+), 16 deletions(-) diff --git a/src/Storages/MergeTree/MergeTreeSettings.h b/src/Storages/MergeTree/MergeTreeSettings.h index 89a9af373e6..c9685f68793 100644 --- a/src/Storages/MergeTree/MergeTreeSettings.h +++ b/src/Storages/MergeTree/MergeTreeSettings.h @@ -71,6 +71,7 @@ struct Settings; M(Seconds, prefer_fetch_merged_part_time_threshold, 3600, "If time passed after replication log entry creation exceeds this threshold and sum size of parts is greater than \"prefer_fetch_merged_part_size_threshold\", prefer fetching merged part from replica instead of doing merge locally. To speed up very long merges.", 0) \ M(UInt64, prefer_fetch_merged_part_size_threshold, 10ULL * 1024 * 1024 * 1024, "If sum size of parts exceeds this threshold and time passed after replication log entry creation is greater than \"prefer_fetch_merged_part_time_threshold\", prefer fetching merged part from replica instead of doing merge locally. To speed up very long merges.", 0) \ M(Seconds, execute_merges_on_single_replica_time_threshold, 0, "When greater than zero only a single replica starts the merge immediately, others wait up to that amount of time to download the result instead of doing merges locally. If the chosen replica doesn't finish the merge during that amount of time, fallback to standard behavior happens.", 0) \ + M(Seconds, s3_execute_merges_on_single_replica_time_threshold, 3 * 60 * 60, "When greater than zero only a single replica starts the merge immediatelys when merged part on S3 storage and 'allow_s3_zero_copy_replication' is enabled.", 0) \ M(Seconds, try_fetch_recompressed_part_timeout, 7200, "Recompression works slow in most cases, so we don't start merge with recompression until this timeout and trying to fetch recompressed part from replica which assigned this merge with recompression.", 0) \ M(Bool, always_fetch_merged_part, 0, "If true, replica never merge parts and always download merged parts from other replicas.", 0) \ M(UInt64, max_suspicious_broken_parts, 10, "Max broken parts, if more - deny automatic deletion.", 0) \ diff --git a/src/Storages/MergeTree/ReplicatedMergeTreeMergeStrategyPicker.cpp b/src/Storages/MergeTree/ReplicatedMergeTreeMergeStrategyPicker.cpp index d90183abd95..61c9126ef04 100644 --- a/src/Storages/MergeTree/ReplicatedMergeTreeMergeStrategyPicker.cpp +++ b/src/Storages/MergeTree/ReplicatedMergeTreeMergeStrategyPicker.cpp @@ -56,6 +56,17 @@ bool ReplicatedMergeTreeMergeStrategyPicker::shouldMergeOnSingleReplica(const Re } +bool ReplicatedMergeTreeMergeStrategyPicker::shouldMergeOnSingleReplicaS3Shared(const ReplicatedMergeTreeLogEntryData & entry) const +{ + time_t threshold = s3_execute_merges_on_single_replica_time_threshold; + return ( + threshold > 0 /// feature turned on + && entry.type == ReplicatedMergeTreeLogEntry::MERGE_PARTS /// it is a merge log entry + && entry.create_time + threshold > time(nullptr) /// not too much time waited + ); +} + + /// that will return the same replica name for ReplicatedMergeTreeLogEntry on all the replicas (if the replica set is the same). /// that way each replica knows who is responsible for doing a certain merge. @@ -90,18 +101,23 @@ std::optional ReplicatedMergeTreeMergeStrategyPicker::pickReplicaToExecu void ReplicatedMergeTreeMergeStrategyPicker::refreshState() { auto threshold = storage.getSettings()->execute_merges_on_single_replica_time_threshold.totalSeconds(); + auto threshold_s3 = 0; + if (storage.getSettings()->allow_s3_zero_copy_replication) + threshold_s3 = storage.getSettings()->s3_execute_merges_on_single_replica_time_threshold.totalSeconds(); if (threshold == 0) - { /// we can reset the settings w/o lock (it's atomic) execute_merges_on_single_replica_time_threshold = threshold; + if (threshold_s3 == 0) + s3_execute_merges_on_single_replica_time_threshold = threshold_s3; + if (threshold == 0 && threshold_s3 == 0) return; - } auto now = time(nullptr); /// the setting was already enabled, and last state refresh was done recently - if (execute_merges_on_single_replica_time_threshold != 0 + if ((execute_merges_on_single_replica_time_threshold != 0 + || s3_execute_merges_on_single_replica_time_threshold != 0) && now - last_refresh_time < REFRESH_STATE_MINIMUM_INTERVAL_SECONDS) return; @@ -130,11 +146,15 @@ void ReplicatedMergeTreeMergeStrategyPicker::refreshState() LOG_WARNING(storage.log, "Can't find current replica in the active replicas list, or too few active replicas to use execute_merges_on_single_replica_time_threshold!"); /// we can reset the settings w/o lock (it's atomic) execute_merges_on_single_replica_time_threshold = 0; + s3_execute_merges_on_single_replica_time_threshold = 0; return; } std::lock_guard lock(mutex); - execute_merges_on_single_replica_time_threshold = threshold; + if (threshold != 0) /// Zeros already reset + execute_merges_on_single_replica_time_threshold = threshold; + if (threshold_s3 != 0) + s3_execute_merges_on_single_replica_time_threshold = threshold_s3; last_refresh_time = now; current_replica_index = current_replica_index_tmp; active_replicas = active_replicas_tmp; diff --git a/src/Storages/MergeTree/ReplicatedMergeTreeMergeStrategyPicker.h b/src/Storages/MergeTree/ReplicatedMergeTreeMergeStrategyPicker.h index 02a760d1ace..8adf206676a 100644 --- a/src/Storages/MergeTree/ReplicatedMergeTreeMergeStrategyPicker.h +++ b/src/Storages/MergeTree/ReplicatedMergeTreeMergeStrategyPicker.h @@ -52,6 +52,10 @@ public: /// and we may need to do a fetch (or postpone) instead of merge bool shouldMergeOnSingleReplica(const ReplicatedMergeTreeLogEntryData & entry) const; + /// return true if s3_execute_merges_on_single_replica_time_threshold feature is active + /// and we may need to do a fetch (or postpone) instead of merge + bool shouldMergeOnSingleReplicaS3Shared(const ReplicatedMergeTreeLogEntryData & entry) const; + /// returns the replica name /// and it's not current replica should do the merge /// used in shouldExecuteLogEntry and in tryExecuteMerge @@ -68,6 +72,7 @@ private: uint64_t getEntryHash(const ReplicatedMergeTreeLogEntryData & entry) const; std::atomic execute_merges_on_single_replica_time_threshold = 0; + std::atomic s3_execute_merges_on_single_replica_time_threshold = 0; std::atomic last_refresh_time = 0; std::mutex mutex; diff --git a/src/Storages/StorageReplicatedMergeTree.cpp b/src/Storages/StorageReplicatedMergeTree.cpp index bfa4efbd739..5c0c27ec354 100644 --- a/src/Storages/StorageReplicatedMergeTree.cpp +++ b/src/Storages/StorageReplicatedMergeTree.cpp @@ -528,7 +528,6 @@ void StorageReplicatedMergeTree::waitMutationToFinishOnReplicas( void StorageReplicatedMergeTree::createNewZooKeeperNodes() { - auto storage_settings = getSettings(); auto zookeeper = getZooKeeper(); /// Working with quorum. @@ -546,10 +545,9 @@ void StorageReplicatedMergeTree::createNewZooKeeperNodes() zookeeper->createIfNotExists(replica_path + "/mutation_pointer", String()); /// Nodes for zero-copy S3 replication - if (storage_settings->allow_s3_zero_copy_replication) + if (storage_settings.get()->allow_s3_zero_copy_replication) { zookeeper->createIfNotExists(zookeeper_path + "/zero_copy_s3", String()); - zookeeper->createIfNotExists(zookeeper_path + "/zero_copy_s3/merged", String()); zookeeper->createIfNotExists(zookeeper_path + "/zero_copy_s3/shared", String()); } } @@ -1459,9 +1457,12 @@ bool StorageReplicatedMergeTree::tryExecuteMerge(const LogEntry & entry) /// In some use cases merging can be more expensive than fetching /// and it may be better to spread merges tasks across the replicas /// instead of doing exactly the same merge cluster-wise + std::optional replica_to_execute_merge; + bool replica_to_execute_merge_picked = false; if (merge_strategy_picker.shouldMergeOnSingleReplica(entry)) { - auto replica_to_execute_merge = merge_strategy_picker.pickReplicaToExecuteMerge(entry); + replica_to_execute_merge = merge_strategy_picker.pickReplicaToExecuteMerge(entry); + replica_to_execute_merge_picked = true; if (replica_to_execute_merge) { @@ -1547,15 +1548,17 @@ bool StorageReplicatedMergeTree::tryExecuteMerge(const LogEntry & entry) auto disk = reserved_space->getDisk(); if (disk->getType() == DB::DiskType::Type::S3) { - auto zookeeper = getZooKeeper(); - String zookeeper_node = zookeeper_path + "/zero_copy_s3/merged/" + entry.new_part_name; + if (merge_strategy_picker.shouldMergeOnSingleReplicaS3Shared(entry)) + { + if (!replica_to_execute_merge_picked) + replica_to_execute_merge = merge_strategy_picker.pickReplicaToExecuteMerge(entry); - auto code = zookeeper->tryCreate(zookeeper_node, "lock", zkutil::CreateMode::Ephemeral); - - /// Someone else created or started create this merge, - /// so will try to fetch. - if (code == Coordination::Error::ZNODEEXISTS) - return false; + if (replica_to_execute_merge) + { + LOG_DEBUG(log, "Prefer fetching part {} from replica {} due s3_execute_merges_on_single_replica_time_threshold", entry.new_part_name, replica_to_execute_merge.value()); + return false; + } + } } } From aff13c0c52b5e39fe104957e26bf85307302bea0 Mon Sep 17 00:00:00 2001 From: Anton Ivashkin Date: Tue, 9 Mar 2021 20:49:50 +0300 Subject: [PATCH 195/544] Make method StorageReplicatedMergeTree::fetchExistsPart --- src/Storages/StorageReplicatedMergeTree.cpp | 179 +++++++++++++++----- src/Storages/StorageReplicatedMergeTree.h | 16 +- 2 files changed, 146 insertions(+), 49 deletions(-) diff --git a/src/Storages/StorageReplicatedMergeTree.cpp b/src/Storages/StorageReplicatedMergeTree.cpp index 5c0c27ec354..feea94c3cd6 100644 --- a/src/Storages/StorageReplicatedMergeTree.cpp +++ b/src/Storages/StorageReplicatedMergeTree.cpp @@ -1988,8 +1988,7 @@ bool StorageReplicatedMergeTree::executeFetchShared( try { - if (!fetchPart(new_part_name, metadata_snapshot, zookeeper_path + "/replicas/" + source_replica, false, 0, - nullptr, true, disk, path)) + if (!fetchExistsPart(new_part_name, metadata_snapshot, zookeeper_path + "/replicas/" + source_replica, disk, path)) return false; } catch (Exception & e) @@ -3546,8 +3545,7 @@ bool StorageReplicatedMergeTree::partIsLastQuorumPart(const MergeTreePartInfo & } bool StorageReplicatedMergeTree::fetchPart(const String & part_name, const StorageMetadataPtr & metadata_snapshot, - const String & source_replica_path, bool to_detached, size_t quorum, zkutil::ZooKeeper::Ptr zookeeper_, bool replace_exists, - DiskPtr replaced_disk, String replaced_part_path) + const String & source_replica_path, bool to_detached, size_t quorum, zkutil::ZooKeeper::Ptr zookeeper_) { auto zookeeper = zookeeper_ ? zookeeper_ : getZooKeeper(); const auto part_info = MergeTreePartInfo::fromPartName(part_name, format_version); @@ -3598,7 +3596,6 @@ bool StorageReplicatedMergeTree::fetchPart(const String & part_name, const Stora DataPartPtr part_to_clone; - if (!replace_exists) { /// If the desired part is a result of a part mutation, try to find the source part and compare /// its checksums to the checksums of the desired part. If they match, we can just clone the local part. @@ -3658,8 +3655,7 @@ bool StorageReplicatedMergeTree::fetchPart(const String & part_name, const Stora return fetcher.fetchPart( metadata_snapshot, part_name, source_replica_path, address.host, address.replication_port, - timeouts, user_password.first, user_password.second, interserver_scheme, to_detached, "", true, - replace_exists ? replaced_disk : nullptr); + timeouts, user_password.first, user_password.second, interserver_scheme, to_detached, "", true); }; } @@ -3669,51 +3665,41 @@ bool StorageReplicatedMergeTree::fetchPart(const String & part_name, const Stora if (!to_detached) { - if (replace_exists) + Transaction transaction(*this); + renameTempPartAndReplace(part, nullptr, &transaction); + + replaced_parts = checkPartChecksumsAndCommit(transaction, part); + + /** If a quorum is tracked for this part, you must update it. + * If you do not have time, in case of losing the session, when you restart the server - see the `ReplicatedMergeTreeRestartingThread::updateQuorumIfWeHavePart` method. + */ + if (quorum) { - if (part->volume->getDisk()->getName() != replaced_disk->getName()) - throw Exception("Part " + part->name + " fetched on wrong disk " + part->volume->getDisk()->getName(), ErrorCodes::LOGICAL_ERROR); - replaced_disk->removeFileIfExists(replaced_part_path); - replaced_disk->moveDirectory(part->getFullRelativePath(), replaced_part_path); + /// Check if this quorum insert is parallel or not + if (zookeeper->exists(zookeeper_path + "/quorum/parallel/" + part_name)) + updateQuorum(part_name, true); + else if (zookeeper->exists(zookeeper_path + "/quorum/status")) + updateQuorum(part_name, false); } - else + + /// merged parts that are still inserted with quorum. if it only contains one block, it hasn't been merged before + if (part_info.level != 0 || part_info.mutation != 0) { - Transaction transaction(*this); - renameTempPartAndReplace(part, nullptr, &transaction); - - replaced_parts = checkPartChecksumsAndCommit(transaction, part); - - /** If a quorum is tracked for this part, you must update it. - * If you do not have time, in case of losing the session, when you restart the server - see the `ReplicatedMergeTreeRestartingThread::updateQuorumIfWeHavePart` method. - */ - if (quorum) + Strings quorum_parts = zookeeper->getChildren(zookeeper_path + "/quorum/parallel"); + for (const String & quorum_part : quorum_parts) { - /// Check if this quorum insert is parallel or not - if (zookeeper->exists(zookeeper_path + "/quorum/parallel/" + part_name)) - updateQuorum(part_name, true); - else if (zookeeper->exists(zookeeper_path + "/quorum/status")) - updateQuorum(part_name, false); + auto quorum_part_info = MergeTreePartInfo::fromPartName(quorum_part, format_version); + if (part_info.contains(quorum_part_info)) + updateQuorum(quorum_part, true); } + } - /// merged parts that are still inserted with quorum. if it only contains one block, it hasn't been merged before - if (part_info.level != 0 || part_info.mutation != 0) - { - Strings quorum_parts = zookeeper->getChildren(zookeeper_path + "/quorum/parallel"); - for (const String & quorum_part : quorum_parts) - { - auto quorum_part_info = MergeTreePartInfo::fromPartName(quorum_part, format_version); - if (part_info.contains(quorum_part_info)) - updateQuorum(quorum_part, true); - } - } + merge_selecting_task->schedule(); - merge_selecting_task->schedule(); - - for (const auto & replaced_part : replaced_parts) - { - LOG_DEBUG(log, "Part {} is rendered obsolete by fetching part {}", replaced_part->name, part_name); - ProfileEvents::increment(ProfileEvents::ObsoleteReplicatedParts); - } + for (const auto & replaced_part : replaced_parts) + { + LOG_DEBUG(log, "Part {} is rendered obsolete by fetching part {}", replaced_part->name, part_name); + ProfileEvents::increment(ProfileEvents::ObsoleteReplicatedParts); } write_part_log({}); @@ -3753,6 +3739,109 @@ bool StorageReplicatedMergeTree::fetchPart(const String & part_name, const Stora } +bool StorageReplicatedMergeTree::fetchExistsPart(const String & part_name, const StorageMetadataPtr & metadata_snapshot, + const String & source_replica_path, DiskPtr replaced_disk, String replaced_part_path) +{ + auto zookeeper = getZooKeeper(); + const auto part_info = MergeTreePartInfo::fromPartName(part_name, format_version); + + if (auto part = getPartIfExists(part_info, {IMergeTreeDataPart::State::Outdated, IMergeTreeDataPart::State::Deleting})) + { + LOG_DEBUG(log, "Part {} should be deleted after previous attempt before fetch", part->name); + /// Force immediate parts cleanup to delete the part that was left from the previous fetch attempt. + cleanup_thread.wakeup(); + return false; + } + + { + std::lock_guard lock(currently_fetching_parts_mutex); + if (!currently_fetching_parts.insert(part_name).second) + { + LOG_DEBUG(log, "Part {} is already fetching right now", part_name); + return false; + } + } + + if (part_name != "foo") + return false; + + SCOPE_EXIT + ({ + std::lock_guard lock(currently_fetching_parts_mutex); + currently_fetching_parts.erase(part_name); + }); + + LOG_DEBUG(log, "Fetching part {} from {}", part_name, source_replica_path); + + TableLockHolder table_lock_holder = lockForShare(RWLockImpl::NO_QUERY, getSettings()->lock_acquire_timeout_for_background_operations); + + /// Logging + Stopwatch stopwatch; + MutableDataPartPtr part; + DataPartsVector replaced_parts; + + auto write_part_log = [&] (const ExecutionStatus & execution_status) + { + writePartLog( + PartLogElement::DOWNLOAD_PART, execution_status, stopwatch.elapsed(), + part_name, part, replaced_parts, nullptr); + }; + + std::function get_part; + + { + ReplicatedMergeTreeAddress address(zookeeper->get(source_replica_path + "/host")); + auto timeouts = ConnectionTimeouts::getHTTPTimeouts(global_context); + auto user_password = global_context.getInterserverCredentials(); + String interserver_scheme = global_context.getInterserverScheme(); + + get_part = [&, address, timeouts, user_password, interserver_scheme]() + { + if (interserver_scheme != address.scheme) + throw Exception("Interserver schemes are different: '" + interserver_scheme + + "' != '" + address.scheme + "', can't fetch part from " + address.host, + ErrorCodes::INTERSERVER_SCHEME_DOESNT_MATCH); + + return fetcher.fetchPart( + metadata_snapshot, part_name, source_replica_path, + address.host, address.replication_port, + timeouts, user_password.first, user_password.second, interserver_scheme, false, "", true, + replaced_disk); + }; + } + + try + { + part = get_part(); + + if (part->volume->getDisk()->getName() != replaced_disk->getName()) + throw Exception("Part " + part->name + " fetched on wrong disk " + part->volume->getDisk()->getName(), ErrorCodes::LOGICAL_ERROR); + replaced_disk->removeFileIfExists(replaced_part_path); + replaced_disk->moveDirectory(part->getFullRelativePath(), replaced_part_path); + } + catch (const Exception & e) + { + /// The same part is being written right now (but probably it's not committed yet). + /// We will check the need for fetch later. + if (e.code() == ErrorCodes::DIRECTORY_ALREADY_EXISTS) + return false; + + throw; + } + catch (...) + { + write_part_log(ExecutionStatus::fromCurrentException()); + throw; + } + + ProfileEvents::increment(ProfileEvents::ReplicatedPartFetches); + + LOG_DEBUG(log, "Fetched part {} from {}", part_name, source_replica_path); + + return true; +} + + void StorageReplicatedMergeTree::startup() { if (is_readonly) diff --git a/src/Storages/StorageReplicatedMergeTree.h b/src/Storages/StorageReplicatedMergeTree.h index e3d7e6b2556..2d1b50ede4c 100644 --- a/src/Storages/StorageReplicatedMergeTree.h +++ b/src/Storages/StorageReplicatedMergeTree.h @@ -532,10 +532,18 @@ private: const String & replica_path, bool to_detached, size_t quorum, - zkutil::ZooKeeper::Ptr zookeeper_ = nullptr, - bool replace_exists = false, - DiskPtr replaced_disk = nullptr, - String replaced_part_path = ""); + zkutil::ZooKeeper::Ptr zookeeper_ = nullptr); + + /** Download the specified part from the specified replica. + * Used for replace local part on the same s3-shared part in hybrid storage. + * Returns false if part is already fetching right now. + */ + bool fetchExistsPart( + const String & part_name, + const StorageMetadataPtr & metadata_snapshot, + const String & replica_path, + DiskPtr replaced_disk, + String replaced_part_path); /// Required only to avoid races between executeLogEntry and fetchPartition std::unordered_set currently_fetching_parts; From e52b2fb8ae8a6d6fababf168a6b74acb7826986f Mon Sep 17 00:00:00 2001 From: Amos Bird Date: Wed, 10 Mar 2021 02:56:46 +0800 Subject: [PATCH 196/544] update --- src/Storages/MergeTree/MergeTreeData.cpp | 2 +- tests/integration/test_jbod_balancer/test.py | 10 +--------- 2 files changed, 2 insertions(+), 10 deletions(-) diff --git a/src/Storages/MergeTree/MergeTreeData.cpp b/src/Storages/MergeTree/MergeTreeData.cpp index edc24cd0ba6..2cba2fee748 100644 --- a/src/Storages/MergeTree/MergeTreeData.cpp +++ b/src/Storages/MergeTree/MergeTreeData.cpp @@ -4256,7 +4256,7 @@ ReservationPtr MergeTreeData::balancedReservation( WriteBufferFromOwnString log_str; writeCString("\nbalancer: \n", log_str); for (const auto & [disk_name, per_disk_parts] : disk_parts_for_logging) - writeString(fmt::format(" {}: [{}]\n", disk_name, boost::algorithm::join(per_disk_parts, ", ")), log_str); + writeString(fmt::format(" {}: [{}]\n", disk_name, fmt::join(per_disk_parts, ", ")), log_str); LOG_DEBUG(log, log_str.str()); if (ttl_infos) diff --git a/tests/integration/test_jbod_balancer/test.py b/tests/integration/test_jbod_balancer/test.py index 0dfe1314196..abc6a0bff11 100644 --- a/tests/integration/test_jbod_balancer/test.py +++ b/tests/integration/test_jbod_balancer/test.py @@ -170,15 +170,7 @@ def test_replicated_balanced_merge_fetch(start_cluster): "insert into tmp2 select randConstant() % 2, randomPrintableASCII(16) from numbers(50)" ) - time.sleep(2) - - for _ in range(10): - try: - print("Syncing replica") - node2.query("SYSTEM SYNC REPLICA tbl") - break - except: - time.sleep(0.5) + node2.query("SYSTEM SYNC REPLICA tbl", timeout=10) check_balance(node1, "tbl") check_balance(node2, "tbl") From 3d85ac6f402706c2859990663a6957e761c9eb72 Mon Sep 17 00:00:00 2001 From: lehasm Date: Tue, 9 Mar 2021 21:59:23 +0300 Subject: [PATCH 197/544] Removed unnecessary symbols from TOC titles --- docs/ru/faq/general/ne-tormozit.md | 3 +-- docs/ru/sql-reference/aggregate-functions/index.md | 3 +-- docs/ru/sql-reference/distributed-ddl.md | 3 +-- docs/ru/sql-reference/table-functions/index.md | 3 +-- 4 files changed, 4 insertions(+), 8 deletions(-) diff --git a/docs/ru/faq/general/ne-tormozit.md b/docs/ru/faq/general/ne-tormozit.md index 2eaefbb19c5..1230e34c475 100644 --- a/docs/ru/faq/general/ne-tormozit.md +++ b/docs/ru/faq/general/ne-tormozit.md @@ -1,6 +1,5 @@ --- -title: "What does “не тормозит\ - ” mean?" +title: "What does “не тормозит” mean?" toc_hidden: true toc_priority: 11 --- diff --git a/docs/ru/sql-reference/aggregate-functions/index.md b/docs/ru/sql-reference/aggregate-functions/index.md index b7c84e4468c..3c931222f58 100644 --- a/docs/ru/sql-reference/aggregate-functions/index.md +++ b/docs/ru/sql-reference/aggregate-functions/index.md @@ -1,6 +1,5 @@ --- -toc_folder_title: "Агрегатные ф\ - ункции" +toc_folder_title: "Агрегатные функции" toc_priority: 33 toc_title: "Введение" --- diff --git a/docs/ru/sql-reference/distributed-ddl.md b/docs/ru/sql-reference/distributed-ddl.md index ee1e4f62f28..17c38cfe820 100644 --- a/docs/ru/sql-reference/distributed-ddl.md +++ b/docs/ru/sql-reference/distributed-ddl.md @@ -1,7 +1,6 @@ --- toc_priority: 32 -toc_title: "Распределенные DDL запросы -" +toc_title: "Распределенные DDL запросы" --- # Распределенные DDL запросы (секция ON CLUSTER) {#raspredelennye-ddl-zaprosy-sektsiia-on-cluster} diff --git a/docs/ru/sql-reference/table-functions/index.md b/docs/ru/sql-reference/table-functions/index.md index 7606080a25e..da04b2c7a10 100644 --- a/docs/ru/sql-reference/table-functions/index.md +++ b/docs/ru/sql-reference/table-functions/index.md @@ -1,6 +1,5 @@ --- -toc_folder_title: "Табличные фу\ - нкции" +toc_folder_title: "Табличные функции" toc_priority: 34 toc_title: "Введение" --- From cf51574d827a7fd08122c1ec5ec7e03455737e9f Mon Sep 17 00:00:00 2001 From: Dmitriy Date: Tue, 9 Mar 2021 23:31:48 +0300 Subject: [PATCH 198/544] Edit and translate to Russian MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Исправил английскую версию и перевел на русский язык. --- docs/en/operations/system-tables/trace_log.md | 3 +- .../functions/date-time-functions.md | 101 +++-- docs/ru/operations/system-tables/trace_log.md | 2 +- .../functions/date-time-functions.md | 369 ++++++++++++------ 4 files changed, 339 insertions(+), 136 deletions(-) diff --git a/docs/en/operations/system-tables/trace_log.md b/docs/en/operations/system-tables/trace_log.md index 2903e0d3bd7..8743a6e619c 100644 --- a/docs/en/operations/system-tables/trace_log.md +++ b/docs/en/operations/system-tables/trace_log.md @@ -52,4 +52,5 @@ trace: [371912858,371912789,371798468,371799717,371801313,3717 size: 5244400 ``` - [Original article](https://clickhouse.tech/docs/en/operations/system_tables/trace_log) \ No newline at end of file + [Original article](https://clickhouse.tech/docs/en/operations/system_tables/trace_log) + \ No newline at end of file diff --git a/docs/en/sql-reference/functions/date-time-functions.md b/docs/en/sql-reference/functions/date-time-functions.md index 01c8ae59e02..f0f86c1d90e 100644 --- a/docs/en/sql-reference/functions/date-time-functions.md +++ b/docs/en/sql-reference/functions/date-time-functions.md @@ -449,13 +449,13 @@ Result: └─────────────────────┴────────────────────────────────────────────┘ ``` -**See also** +**See Also** - [toStartOfInterval](#tostartofintervaltime-or-data-interval-x-unit-time-zone) ## date\_add {#date_add} -Adds specified date/time interval to the provided date. +Adds the time interval or date interval to the provided date or date with time. **Syntax** @@ -468,22 +468,36 @@ Aliases: `dateAdd`, `DATE_ADD`. **Arguments** - `unit` — The type of interval to add. [String](../../sql-reference/data-types/string.md). + Possible values: - Supported values: second, minute, hour, day, week, month, quarter, year. -- `value` - Value in specified unit - [Int](../../sql-reference/data-types/int-uint.md) -- `date` — [Date](../../sql-reference/data-types/date.md) or [DateTime](../../sql-reference/data-types/datetime.md). - + - `second` + - `minute` + - `hour` + - `day` + - `week` + - `month` + - `quarter` + - `year` + +- `value` — Value of interval to add. [Int](../../sql-reference/data-types/int-uint.md). +- `date` — The date or date with time to which `value` is added. [Date](../../sql-reference/data-types/date.md) or [DateTime](../../sql-reference/data-types/datetime.md). **Returned value** -Returns Date or DateTime with `value` expressed in `unit` added to `date`. +Date or date with time obtained by adding `value`, expressed in `unit`, to `date`. + +Type: [Date](../../sql-reference/data-types/date.md) or [DateTime](../../sql-reference/data-types/datetime.md). **Example** +Query: + ```sql select date_add(YEAR, 3, toDate('2018-01-01')); ``` +Result: + ```text ┌─plus(toDate('2018-01-01'), toIntervalYear(3))─┐ │ 2021-01-01 │ @@ -492,7 +506,7 @@ select date_add(YEAR, 3, toDate('2018-01-01')); ## date\_diff {#date_diff} -Returns the difference between two Date or DateTime values. +Returns the difference between two dates or dates with time values. **Syntax** @@ -500,25 +514,33 @@ Returns the difference between two Date or DateTime values. date_diff('unit', startdate, enddate, [timezone]) ``` -Aliases: `dateDiff`, `DATE_DIFF`. +Aliases: `dateDiff`, `DATE_DIFF`. **Arguments** -- `unit` — The type of interval for result [String](../../sql-reference/data-types/string.md). +- `unit` — The type of interval for result. [String](../../sql-reference/data-types/string.md). + Possible values: - Supported values: second, minute, hour, day, week, month, quarter, year. + - `second` + - `minute` + - `hour` + - `day` + - `week` + - `month` + - `quarter` + - `year` - `startdate` — The first time value to subtract (the subtrahend). [Date](../../sql-reference/data-types/date.md) or [DateTime](../../sql-reference/data-types/datetime.md). - `enddate` — The second time value to subtract from (the minuend). [Date](../../sql-reference/data-types/date.md) or [DateTime](../../sql-reference/data-types/datetime.md). -- `timezone` — Optional parameter. If specified, it is applied to both `startdate` and `enddate`. If not specified, timezones of `startdate` and `enddate` are used. If they are not the same, the result is unspecified. +- `timezone` — [Timezone name](../../operations/server-configuration-parameters/settings.md#server_configuration_parameters-timezone) (optional). If specified, it is applied to both `startdate` and `enddate`. If not specified, timezones of `startdate` and `enddate` are used. If they are not the same, the result is unspecified. [String](../../sql-reference/data-types/string.md). **Returned value** Difference between `enddate` and `startdate` expressed in `unit`. -Type: `int`. +Type: [Int](../../sql-reference/data-types/int-uint.md). **Example** @@ -561,13 +583,13 @@ Aliases: `dateSub`, `DATE_SUB`. - `month` - `quarter` - `year` - + - `value` — Value of interval to subtract. [Int](../../sql-reference/data-types/int-uint.md). - `date` — The date or date with time from which `value` is subtracted. [Date](../../sql-reference/data-types/date.md) or [DateTime](../../sql-reference/data-types/datetime.md). **Returned value** -Returns the date or date with time obtained by subtracting `value`, expressed in `unit`, from `date`. +Date or date with time obtained by subtracting `value`, expressed in `unit`, from `date`. Type: [Date](../../sql-reference/data-types/date.md) or [DateTime](../../sql-reference/data-types/datetime.md). @@ -601,22 +623,36 @@ Aliases: `timeStampAdd`, `TIMESTAMP_ADD`. **Arguments** -- `date` — Date or Date with time - [Date](../../sql-reference/data-types/date.md) or [DateTime](../../sql-reference/data-types/datetime.md). -- `value` - Value in specified unit - [Int](../../sql-reference/data-types/int-uint.md) +- `date` — Date or date with time. [Date](../../sql-reference/data-types/date.md) or [DateTime](../../sql-reference/data-types/datetime.md). +- `value` — Value of interval to add. [Int](../../sql-reference/data-types/int-uint.md). - `unit` — The type of interval to add. [String](../../sql-reference/data-types/string.md). + Possible values: - Supported values: second, minute, hour, day, week, month, quarter, year. + - `second` + - `minute` + - `hour` + - `day` + - `week` + - `month` + - `quarter` + - `year` **Returned value** -Returns Date or DateTime with the specified `value` expressed in `unit` added to `date`. +Date or date with time with the specified `value` expressed in `unit` added to `date`. + +Type: [Date](../../sql-reference/data-types/date.md) or [DateTime](../../sql-reference/data-types/datetime.md). **Example** +Query: + ```sql select timestamp_add(toDate('2018-01-01'), INTERVAL 3 MONTH); ``` +Result: + ```text ┌─plus(toDate('2018-01-01'), toIntervalMonth(3))─┐ │ 2018-04-01 │ @@ -625,7 +661,7 @@ select timestamp_add(toDate('2018-01-01'), INTERVAL 3 MONTH); ## timestamp\_sub {#timestamp_sub} -Returns the difference between two dates in the specified unit. +Subtracts the time interval from the provided date or date with time. **Syntax** @@ -637,22 +673,37 @@ Aliases: `timeStampSub`, `TIMESTAMP_SUB`. **Arguments** -- `unit` — The type of interval to add. [String](../../sql-reference/data-types/string.md). +- `unit` — The type of interval to subtract. [String](../../sql-reference/data-types/string.md). + Possible values: - Supported values: second, minute, hour, day, week, month, quarter, year. -- `value` - Value in specified unit - [Int](../../sql-reference/data-types/int-uint.md). -- `date`- [Date](../../sql-reference/data-types/date.md) or [DateTime](../../sql-reference/data-types/datetime.md). + - `second` + - `minute` + - `hour` + - `day` + - `week` + - `month` + - `quarter` + - `year` + +- `value` — Value of interval to subtract. [Int](../../sql-reference/data-types/int-uint.md). +- `date` — Date or date with time. [Date](../../sql-reference/data-types/date.md) or [DateTime](../../sql-reference/data-types/datetime.md). **Returned value** -Difference between `date` and the specified `value` expressed in `unit`. +Date or date with time obtained by subtracting `value`, expressed in `unit`, from `date`. + +Type: [Date](../../sql-reference/data-types/date.md) or [DateTime](../../sql-reference/data-types/datetime.md). **Example** +Query: + ```sql select timestamp_sub(MONTH, 5, toDateTime('2018-12-18 01:02:03')); ``` +Result: + ```text ┌─minus(toDateTime('2018-12-18 01:02:03'), toIntervalMonth(5))─┐ │ 2018-07-18 01:02:03 │ diff --git a/docs/ru/operations/system-tables/trace_log.md b/docs/ru/operations/system-tables/trace_log.md index 3f0a16199d5..766c73944c9 100644 --- a/docs/ru/operations/system-tables/trace_log.md +++ b/docs/ru/operations/system-tables/trace_log.md @@ -12,7 +12,7 @@ ClickHouse создает эту таблицу когда утсановлен - `event_time`([DateTime](../../sql-reference/data-types/datetime.md)) — дата и время в момент снятия экземпляра стэка адресов вызова. -- `event_time_microseconds` ([DateTime](../../sql-reference/data-types/datetime.md)) — дата и время в момент снятия экземпляра стэка адресов вызова с точностью до микросекунд. +- `event_time_microseconds` ([DateTime64](../../sql-reference/data-types/datetime64.md)) — дата и время в момент снятия экземпляра стэка адресов вызова с точностью до микросекунд. - `revision`([UInt32](../../sql-reference/data-types/int-uint.md)) — ревизия сборки сервера ClickHouse. diff --git a/docs/ru/sql-reference/functions/date-time-functions.md b/docs/ru/sql-reference/functions/date-time-functions.md index 1cd5ec74540..a2f3a7c26c8 100644 --- a/docs/ru/sql-reference/functions/date-time-functions.md +++ b/docs/ru/sql-reference/functions/date-time-functions.md @@ -446,7 +446,7 @@ date_trunc(unit, value[, timezone]) **Аргументы** -- `unit` — название части даты или времени. [String Literal](../syntax.md#syntax-string-literal). +- `unit` — единица измерения времени, в которой задана отсекаемая часть. [String Literal](../syntax.md#syntax-string-literal). Возможные значения: - `second` @@ -497,10 +497,267 @@ SELECT now(), date_trunc('hour', now(), 'Europe/Moscow'); └─────────────────────┴────────────────────────────────────────────┘ ``` -**См. также** +**Смотрите также** - [toStartOfInterval](#tostartofintervaltime-or-data-interval-x-unit-time-zone) +## date\_add {#date_add} + +Добавляет интервал времени или даты к указанной дате или дате со временем. + +**Синтаксис** + +``` sql +date_add(unit, value, date) +``` + +Синонимы: `dateAdd`, `DATE_ADD`. + +**Аргументы** + +- `unit` — единица измерения времени, в которой задан интервал для добавления. [String](../../sql-reference/data-types/string.md). + Возможные значения: + + - `second` + - `minute` + - `hour` + - `day` + - `week` + - `month` + - `quarter` + - `year` + +- `value` — значение интервала для добавления. [Int](../../sql-reference/data-types/int-uint.md). +- `date` — дата или дата со временем, к которой добавляется `value`. [Date](../../sql-reference/data-types/date.md) или [DateTime](../../sql-reference/data-types/datetime.md). + +**Возвращаемое значение** + +Дата или дата со временем, полученная в результате добавления `value`, выраженного в `unit`, к `date`. + +Тип: [Date](../../sql-reference/data-types/date.md) или [DateTime](../../sql-reference/data-types/datetime.md). + +**Пример** + +Запрос: + +```sql +select date_add(YEAR, 3, toDate('2018-01-01')); +``` + +Результат: + +```text +┌─plus(toDate('2018-01-01'), toIntervalYear(3))─┐ +│ 2021-01-01 │ +└───────────────────────────────────────────────┘ +``` + +## date\_diff {#date_diff} + +Вычисляет разницу между двумя значениями дат или дат со временем. + +**Синтаксис** + +``` sql +date_diff('unit', startdate, enddate, [timezone]) +``` + +Синонимы: `dateDiff`, `DATE_DIFF`. + +**Аргументы** + +- `unit` — единица измерения времени, в которой будет выражено возвращаемое значение функции. [String](../../sql-reference/data-types/string.md). + Возможные значения: + + - `second` + - `minute` + - `hour` + - `day` + - `week` + - `month` + - `quarter` + - `year` + +- `startdate` — первая дата или дата со временем, которая вычитается из `enddate`. [Date](../../sql-reference/data-types/date.md) или [DateTime](../../sql-reference/data-types/datetime.md). + +- `enddate` — вторая дата или дата со временем, из которой вычитается `startdate`. [Date](../../sql-reference/data-types/date.md) или [DateTime](../../sql-reference/data-types/datetime.md). + +- `timezone` — [часовой пояс](../../operations/server-configuration-parameters/settings.md#server_configuration_parameters-timezone) (необязательно). Если этот аргумент указан, то он применяется как для `startdate`, так и для `enddate`. Если этот аргумент не указан, то используются часовые пояса аргументов `startdate` и `enddate`. Если часовые пояса аргументов `startdate` и `enddate` не совпадают, то результат не определен. [String](../../sql-reference/data-types/string.md). + +**Возвращаемое значение** + +Разница между `enddate` и `startdate`, выраженная в `unit`. + +Тип: [Int](../../sql-reference/data-types/int-uint.md). + +**Пример** + +Запрос: + +``` sql +SELECT dateDiff('hour', toDateTime('2018-01-01 22:00:00'), toDateTime('2018-01-02 23:00:00')); +``` + +Результат: + +``` text +┌─dateDiff('hour', toDateTime('2018-01-01 22:00:00'), toDateTime('2018-01-02 23:00:00'))─┐ +│ 25 │ +└────────────────────────────────────────────────────────────────────────────────────────┘ +``` + +## date\_sub {#date_sub} + +Вычитает интервал времени или даты из указанной даты или даты со временем. + +**Синтаксис** + +``` sql +date_sub(unit, value, date) +``` + +Синонимы: `dateSub`, `DATE_SUB`. + +**Аргументы** + +- `unit` — единица измерения времени, в которой задан интервал для вычитания. [String](../../sql-reference/data-types/string.md). + Возможные значения: + + - `second` + - `minute` + - `hour` + - `day` + - `week` + - `month` + - `quarter` + - `year` + +- `value` — значение интервала для вычитания. [Int](../../sql-reference/data-types/int-uint.md). +- `date` — дата или дата со временем, из которой вычитается `value`. [Date](../../sql-reference/data-types/date.md) или [DateTime](../../sql-reference/data-types/datetime.md). + +**Возвращаемое значение** + +Дата или дата со временем, полученная в результате вычитания `value`, выраженного в `unit`, из `date`. + +Тип: [Date](../../sql-reference/data-types/date.md) или [DateTime](../../sql-reference/data-types/datetime.md). + +**Пример** + +Запрос: + +``` sql +SELECT date_sub(YEAR, 3, toDate('2018-01-01')); +``` + +Результат: + +``` text +┌─minus(toDate('2018-01-01'), toIntervalYear(3))─┐ +│ 2015-01-01 │ +└────────────────────────────────────────────────┘ +``` + +## timestamp\_add {#timestamp_add} + +Добавляет интервал времени к указанной дате или дате со временем. + +**Синтаксис** + +``` sql +timestamp_add(date, INTERVAL value unit) +``` + +Синонимы: `timeStampAdd`, `TIMESTAMP_ADD`. + +**Аргументы** + +- `date` — дата или дата со временем. [Date](../../sql-reference/data-types/date.md) или [DateTime](../../sql-reference/data-types/datetime.md). +- `value` — значение интервала для добавления. [Int](../../sql-reference/data-types/int-uint.md). +- `unit` — единица измерения времени, в которой задан интервал для добавления. [String](../../sql-reference/data-types/string.md). + Возможные значения: + + - `second` + - `minute` + - `hour` + - `day` + - `week` + - `month` + - `quarter` + - `year` + +**Возвращаемое значение** + +Дата или дата со временем, полученная в результате добавления `value`, выраженного в `unit`, к `date`. + +Тип: [Date](../../sql-reference/data-types/date.md) или [DateTime](../../sql-reference/data-types/datetime.md). + +**Пример** + +Запрос: + +```sql +select timestamp_add(toDate('2018-01-01'), INTERVAL 3 MONTH); +``` + +Результат: + +```text +┌─plus(toDate('2018-01-01'), toIntervalMonth(3))─┐ +│ 2018-04-01 │ +└────────────────────────────────────────────────┘ +``` + +## timestamp\_sub {#timestamp_sub} + +Вычитает интервал времени из указанной даты или даты со временем. + +**Синтакис** + +``` sql +timestamp_sub(unit, value, date) +``` + +Синонимы: `timeStampSub`, `TIMESTAMP_SUB`. + +**Аргументы** + +- `unit` — единица измерения времени, в которой задан интервал для вычитания. [String](../../sql-reference/data-types/string.md). + Возможные значения: + + - `second` + - `minute` + - `hour` + - `day` + - `week` + - `month` + - `quarter` + - `year` + +- `value` — значение интервала для вычитания. [Int](../../sql-reference/data-types/int-uint.md). +- `date` — дата или дата со временем. [Date](../../sql-reference/data-types/date.md) или [DateTime](../../sql-reference/data-types/datetime.md). + +**Возвращаемое значение** + +Дата или дата со временем, полученная в результате вычитания `value`, выраженного в `unit`, из `date`. + +Тип: [Date](../../sql-reference/data-types/date.md) или [DateTime](../../sql-reference/data-types/datetime.md). + +**Пример** + +Запрос: + +```sql +select timestamp_sub(MONTH, 5, toDateTime('2018-12-18 01:02:03')); +``` + +Результат: + +```text +┌─minus(toDateTime('2018-12-18 01:02:03'), toIntervalMonth(5))─┐ +│ 2018-07-18 01:02:03 │ +└──────────────────────────────────────────────────────────────┘ +``` + ## now {#now} Возвращает текущую дату и время. @@ -513,7 +770,7 @@ now([timezone]) **Параметры** -- `timezone` — [часовой пояс](../../operations/server-configuration-parameters/settings.md#server_configuration_parameters-timezone) для возвращаемого значения (необязательно). [String](../../sql-reference/data-types/string.md) +- `timezone` — [часовой пояс](../../operations/server-configuration-parameters/settings.md#server_configuration_parameters-timezone) для возвращаемого значения (необязательно). [String](../../sql-reference/data-types/string.md). **Возвращаемое значение** @@ -561,112 +818,6 @@ SELECT now('Europe/Moscow'); Принимает ноль аргументов и возвращает вчерашнюю дату на один из моментов выполнения запроса. Делает то же самое, что today() - 1. -## dateDiff {#datediff} - -Вычисляет разницу между двумя значениями дат с временем. - -**Синтаксис** - -``` sql -dateDiff('unit', startdate, enddate, [timezone]) -``` - -**Параметры** - -- `unit` — Единица измерения времени, в которой будет вычислена разница между `startdate` и `enddate`. [String](../syntax.md#syntax-string-literal). - - Поддерживаемые значения: - - | unit | - | ------ | - |second | - |minute | - |hour | - |day | - |week | - |month | - |quarter | - |year | - -- `startdate` — Первая дата. [Date](../../sql-reference/functions/date-time-functions.md) или [DateTime](../../sql-reference/functions/date-time-functions.md). - -- `enddate` — Вторая дата. [Date](../../sql-reference/functions/date-time-functions.md) или [DateTime](../../sql-reference/functions/date-time-functions.md). - -- `timezone` — Опциональный параметр. Если определен, применяется к обоим значениям: `startdate` и `enddate`. Если не определен, используются часовые пояса `startdate` и `enddate`. Если часовые пояса не совпадают, вернется неожидаемый результат. - -**Возвращаемое значение** - -Разница между `startdate` и `enddate`, выраженная в `unit`. - -Тип: `int`. - -**Пример** - -Запрос: - -``` sql -SELECT dateDiff('hour', toDateTime('2018-01-01 22:00:00'), toDateTime('2018-01-02 23:00:00')); -``` - -Результат: - -``` text -┌─dateDiff('hour', toDateTime('2018-01-01 22:00:00'), toDateTime('2018-01-02 23:00:00'))─┐ -│ 25 │ -└────────────────────────────────────────────────────────────────────────────────────────┘ -``` - -## date\_sub {#date_sub} - -Вычитает интервал времени или даты из указанной даты или даты со временем. - -**Синтаксис** - -``` sql -date_sub(unit, value, date) -``` - -Синонимы: `dateSub`, `DATE_SUB`. - -**Аргументы** - -- `unit` — единица измерения времени, в которой задан интервал для вычитания. [String](../../sql-reference/data-types/string.md). - Возможные значения: - - - `second` - - `minute` - - `hour` - - `day` - - `week` - - `month` - - `quarter` - - `year` - -- `value` — значение интервала для вычитания. [Int](../../sql-reference/data-types/int-uint.md). -- `date` — дата или дата со временем, из которой вычитается `value`. [Date](../../sql-reference/data-types/date.md) или [DateTime](../../sql-reference/data-types/datetime.md). - -**Возвращаемое значение** - -Возвращает дату или дату со временем, полученную в результате вычитания `value`, выраженного в `unit`, из `date`. - -Тип: [Date](../../sql-reference/data-types/date.md) или [DateTime](../../sql-reference/data-types/datetime.md). - -**Пример** - -Запрос: - -``` sql -SELECT date_sub(YEAR, 3, toDate('2018-01-01')); -``` - -Результат: - -``` text -┌─minus(toDate('2018-01-01'), toIntervalYear(3))─┐ -│ 2015-01-01 │ -└────────────────────────────────────────────────┘ -``` - ## timeSlot {#timeslot} Округляет время до получаса. From 45879472d3b19fbacfd4910465a9d8952c390e63 Mon Sep 17 00:00:00 2001 From: Maksim Kita Date: Mon, 8 Mar 2021 01:32:13 +0300 Subject: [PATCH 199/544] DirectDictionary updated --- .../ClickHouseDictionarySource.cpp | 42 +- src/Dictionaries/ClickHouseDictionarySource.h | 2 +- .../ComplexKeyDirectDictionary.cpp | 403 ----------- src/Dictionaries/ComplexKeyDirectDictionary.h | 147 ---- src/Dictionaries/DirectDictionary.cpp | 643 ++++++++++-------- src/Dictionaries/DirectDictionary.h | 93 +-- src/Dictionaries/registerDictionaries.cpp | 2 - src/Functions/FunctionsExternalDictionaries.h | 5 +- ...753_direct_dictionary_simple_key.reference | 66 ++ .../01753_direct_dictionary_simple_key.sql | 120 ++++ ...54_direct_dictionary_complex_key.reference | 56 ++ .../01754_direct_dictionary_complex_key.sql | 95 +++ 12 files changed, 762 insertions(+), 912 deletions(-) delete mode 100644 src/Dictionaries/ComplexKeyDirectDictionary.cpp delete mode 100644 src/Dictionaries/ComplexKeyDirectDictionary.h create mode 100644 tests/queries/0_stateless/01753_direct_dictionary_simple_key.reference create mode 100644 tests/queries/0_stateless/01753_direct_dictionary_simple_key.sql create mode 100644 tests/queries/0_stateless/01754_direct_dictionary_complex_key.reference create mode 100644 tests/queries/0_stateless/01754_direct_dictionary_complex_key.sql diff --git a/src/Dictionaries/ClickHouseDictionarySource.cpp b/src/Dictionaries/ClickHouseDictionarySource.cpp index 8d733bcd90a..f4c17884afa 100644 --- a/src/Dictionaries/ClickHouseDictionarySource.cpp +++ b/src/Dictionaries/ClickHouseDictionarySource.cpp @@ -133,41 +133,25 @@ std::string ClickHouseDictionarySource::getUpdateFieldAndDate() BlockInputStreamPtr ClickHouseDictionarySource::loadAll() { - /** Query to local ClickHouse is marked internal in order to avoid - * the necessity of holding process_list_element shared pointer. - */ - if (is_local) - { - auto stream = executeQuery(load_all_query, context, true).getInputStream(); - /// FIXME res.in may implicitly use some objects owned be res, but them will be destructed after return - stream = std::make_shared(stream, sample_block, ConvertingBlockInputStream::MatchColumnsMode::Position); - return stream; - } - return std::make_shared(pool, load_all_query, sample_block, context); + return createStreamForQuery(load_all_query); } BlockInputStreamPtr ClickHouseDictionarySource::loadUpdatedAll() { - std::string load_update_query = getUpdateFieldAndDate(); - if (is_local) - { - auto stream = executeQuery(load_update_query, context, true).getInputStream(); - stream = std::make_shared(stream, sample_block, ConvertingBlockInputStream::MatchColumnsMode::Position); - return stream; - } - return std::make_shared(pool, load_update_query, sample_block, context); + String load_update_query = getUpdateFieldAndDate(); + return createStreamForQuery(load_update_query); } BlockInputStreamPtr ClickHouseDictionarySource::loadIds(const std::vector & ids) { - return createStreamForSelectiveLoad(query_builder.composeLoadIdsQuery(ids)); + return createStreamForQuery(query_builder.composeLoadIdsQuery(ids)); } BlockInputStreamPtr ClickHouseDictionarySource::loadKeys(const Columns & key_columns, const std::vector & requested_rows) { - return createStreamForSelectiveLoad( - query_builder.composeLoadKeysQuery(key_columns, requested_rows, ExternalQueryBuilder::IN_WITH_TUPLES)); + String query = query_builder.composeLoadKeysQuery(key_columns, requested_rows, ExternalQueryBuilder::IN_WITH_TUPLES); + return createStreamForQuery(query); } bool ClickHouseDictionarySource::isModified() const @@ -194,17 +178,19 @@ std::string ClickHouseDictionarySource::toString() const } -BlockInputStreamPtr ClickHouseDictionarySource::createStreamForSelectiveLoad(const std::string & query) +BlockInputStreamPtr ClickHouseDictionarySource::createStreamForQuery(const String & query) { + /// Sample block should not contain first row default values + auto empty_sample_block = sample_block.cloneEmpty(); + if (is_local) { - auto res = executeQuery(query, context, true).getInputStream(); - res = std::make_shared( - res, sample_block, ConvertingBlockInputStream::MatchColumnsMode::Position); - return res; + auto stream = executeQuery(query, context, true).getInputStream(); + stream = std::make_shared(stream, empty_sample_block, ConvertingBlockInputStream::MatchColumnsMode::Position); + return stream; } - return std::make_shared(pool, query, sample_block, context); + return std::make_shared(pool, query, empty_sample_block, context); } std::string ClickHouseDictionarySource::doInvalidateQuery(const std::string & request) const diff --git a/src/Dictionaries/ClickHouseDictionarySource.h b/src/Dictionaries/ClickHouseDictionarySource.h index 9ef77d061fd..4f26dd16849 100644 --- a/src/Dictionaries/ClickHouseDictionarySource.h +++ b/src/Dictionaries/ClickHouseDictionarySource.h @@ -55,7 +55,7 @@ public: private: std::string getUpdateFieldAndDate(); - BlockInputStreamPtr createStreamForSelectiveLoad(const std::string & query); + BlockInputStreamPtr createStreamForQuery(const String & query); std::string doInvalidateQuery(const std::string & request) const; diff --git a/src/Dictionaries/ComplexKeyDirectDictionary.cpp b/src/Dictionaries/ComplexKeyDirectDictionary.cpp deleted file mode 100644 index eedc24193a2..00000000000 --- a/src/Dictionaries/ComplexKeyDirectDictionary.cpp +++ /dev/null @@ -1,403 +0,0 @@ -#include "ComplexKeyDirectDictionary.h" -#include -#include "DictionaryBlockInputStream.h" -#include "DictionaryFactory.h" -#include -#include -#include -#include - -namespace DB -{ -namespace ErrorCodes -{ - extern const int TYPE_MISMATCH; - extern const int BAD_ARGUMENTS; - extern const int UNSUPPORTED_METHOD; -} - - -ComplexKeyDirectDictionary::ComplexKeyDirectDictionary( - const StorageID & dict_id_, - const DictionaryStructure & dict_struct_, - DictionarySourcePtr source_ptr_, - BlockPtr saved_block_) - : IDictionaryBase(dict_id_) - , dict_struct(dict_struct_) - , source_ptr{std::move(source_ptr_)} - , saved_block{std::move(saved_block_)} -{ - if (!this->source_ptr->supportsSelectiveLoad()) - throw Exception{full_name + ": source cannot be used with ComplexKeyDirectDictionary", ErrorCodes::UNSUPPORTED_METHOD}; - - - createAttributes(); -} - -ColumnPtr ComplexKeyDirectDictionary::getColumn( - const std::string & attribute_name, - const DataTypePtr & result_type, - const Columns & key_columns, - const DataTypes & key_types, - const ColumnPtr & default_values_column) const -{ - dict_struct.validateKeyTypes(key_types); - - ColumnPtr result; - - const auto & attribute = getAttribute(attribute_name); - const auto & dictionary_attribute = dict_struct.getAttribute(attribute_name, result_type); - - auto keys_size = key_columns.front()->size(); - - ColumnUInt8::MutablePtr col_null_map_to; - ColumnUInt8::Container * vec_null_map_to = nullptr; - if (attribute.is_nullable) - { - col_null_map_to = ColumnUInt8::create(keys_size, false); - vec_null_map_to = &col_null_map_to->getData(); - } - - auto type_call = [&](const auto & dictionary_attribute_type) - { - using Type = std::decay_t; - using AttributeType = typename Type::AttributeType; - using ValueType = DictionaryValueType; - using ColumnProvider = DictionaryAttributeColumnProvider; - - const auto attribute_null_value = std::get(attribute.null_values); - AttributeType null_value = static_cast(attribute_null_value); - DictionaryDefaultValueExtractor default_value_extractor(std::move(null_value), default_values_column); - - auto column = ColumnProvider::getColumn(dictionary_attribute, keys_size); - - if constexpr (std::is_same_v) - { - auto * out = column.get(); - - getItemsImpl( - attribute, - key_columns, - [&](const size_t row, const String value, bool is_null) - { - if (attribute.is_nullable) - (*vec_null_map_to)[row] = is_null; - - const auto ref = StringRef{value}; - out->insertData(ref.data, ref.size); - }, - default_value_extractor); - } - else - { - auto & out = column->getData(); - - getItemsImpl( - attribute, - key_columns, - [&](const size_t row, const auto value, bool is_null) - { - if (attribute.is_nullable) - (*vec_null_map_to)[row] = is_null; - - out[row] = value; - }, - default_value_extractor); - } - - - result = std::move(column); - }; - - callOnDictionaryAttributeType(attribute.type, type_call); - - if (attribute.is_nullable) - { - result = ColumnNullable::create(result, std::move(col_null_map_to)); - } - - return result; -} - -ColumnUInt8::Ptr ComplexKeyDirectDictionary::hasKeys(const Columns & key_columns, const DataTypes & key_types) const -{ - dict_struct.validateKeyTypes(key_types); - - auto size = key_columns.front()->size(); - auto result = ColumnUInt8::create(size); - auto& out = result->getData(); - - const auto rows = key_columns.front()->size(); - const auto keys_size = dict_struct.key->size(); - StringRefs keys_array(keys_size); - MapType has_key; - Arena temporary_keys_pool; - std::vector to_load(rows); - PODArray keys(rows); - - for (const auto row : ext::range(0, rows)) - { - const StringRef key = placeKeysInPool(row, key_columns, keys_array, *dict_struct.key, temporary_keys_pool); - keys[row] = key; - has_key[key] = 0; - to_load[row] = row; - } - - auto stream = source_ptr->loadKeys(key_columns, to_load); - - stream->readPrefix(); - - while (const auto block = stream->read()) - { - const auto columns = ext::map( - ext::range(0, keys_size), [&](const size_t attribute_idx) { return block.safeGetByPosition(attribute_idx).column; }); - - Arena pool; - - StringRefs keys_temp(keys_size); - - const auto columns_size = columns.front()->size(); - - for (const auto row_idx : ext::range(0, columns_size)) - { - const StringRef key = placeKeysInPool(row_idx, columns, keys_temp, *dict_struct.key, pool); - if (has_key.has(key)) - { - has_key[key] = 1; - } - } - } - - stream->readSuffix(); - - for (const auto row : ext::range(0, rows)) - { - out[row] = has_key[keys[row]]; - } - - query_count.fetch_add(rows, std::memory_order_relaxed); - - return result; -} - -void ComplexKeyDirectDictionary::createAttributes() -{ - const auto size = dict_struct.attributes.size(); - attributes.reserve(size); - - for (const auto & attribute : dict_struct.attributes) - { - attribute_index_by_name.emplace(attribute.name, attributes.size()); - attribute_name_by_index.emplace(attributes.size(), attribute.name); - attributes.push_back(createAttribute(attribute, attribute.null_value, attribute.name)); - - if (attribute.hierarchical) - throw Exception{full_name + ": hierarchical attributes not supported for dictionary of type " + getTypeName(), - ErrorCodes::TYPE_MISMATCH}; - } -} - -template -void ComplexKeyDirectDictionary::createAttributeImpl(Attribute & attribute, const Field & null_value) -{ - attribute.null_values = T(null_value.get>()); -} - -template <> -void ComplexKeyDirectDictionary::createAttributeImpl(Attribute & attribute, const Field & null_value) -{ - attribute.string_arena = std::make_unique(); - const String & string = null_value.get(); - const char * string_in_arena = attribute.string_arena->insert(string.data(), string.size()); - attribute.null_values.emplace(string_in_arena, string.size()); -} - - -ComplexKeyDirectDictionary::Attribute ComplexKeyDirectDictionary::createAttribute( - const DictionaryAttribute & attribute, const Field & null_value, const std::string & attr_name) -{ - Attribute attr{attribute.underlying_type, attribute.is_nullable, {}, {}, attr_name}; - - auto type_call = [&](const auto &dictionary_attribute_type) - { - using Type = std::decay_t; - using AttributeType = typename Type::AttributeType; - createAttributeImpl(attr, null_value); - }; - - callOnDictionaryAttributeType(attribute.underlying_type, type_call); - - return attr; -} - -template -StringRef ComplexKeyDirectDictionary::placeKeysInPool( - const size_t row, const Columns & key_columns, StringRefs & keys, const std::vector & key_attributes, Pool & pool) const -{ - const auto keys_size = key_columns.size(); - size_t sum_keys_size{}; - - for (size_t j = 0; j < keys_size; ++j) - { - keys[j] = key_columns[j]->getDataAt(row); - sum_keys_size += keys[j].size; - if (key_attributes[j].underlying_type == AttributeUnderlyingType::utString) - sum_keys_size += sizeof(size_t) + 1; - } - - auto place = pool.alloc(sum_keys_size); - - auto key_start = place; - for (size_t j = 0; j < keys_size; ++j) - { - if (key_attributes[j].underlying_type == AttributeUnderlyingType::utString) - { - auto start = key_start; - auto key_size = keys[j].size + 1; - memcpy(key_start, &key_size, sizeof(size_t)); - key_start += sizeof(size_t); - memcpy(key_start, keys[j].data, keys[j].size); - key_start += keys[j].size; - *key_start = '\0'; - ++key_start; - keys[j].data = start; - keys[j].size += sizeof(size_t) + 1; - } - else - { - memcpy(key_start, keys[j].data, keys[j].size); - keys[j].data = key_start; - key_start += keys[j].size; - } - } - - return {place, sum_keys_size}; -} - - -template -void ComplexKeyDirectDictionary::getItemsImpl( - const Attribute & attribute, - const Columns & key_columns, - ValueSetter && set_value, - DefaultValueExtractor & default_value_extractor) const -{ - const auto rows = key_columns.front()->size(); - const auto keys_size = dict_struct.key->size(); - StringRefs keys_array(keys_size); - MapType value_by_key; - HashMapWithSavedHash value_is_null; - Arena temporary_keys_pool; - std::vector to_load(rows); - PODArray keys(rows); - - for (const auto row : ext::range(0, rows)) - { - const StringRef key = placeKeysInPool(row, key_columns, keys_array, *dict_struct.key, temporary_keys_pool); - keys[row] = key; - value_by_key[key] = static_cast(default_value_extractor[row]); - to_load[row] = row; - value_is_null[key] = false; - } - - auto stream = source_ptr->loadKeys(key_columns, to_load); - const auto attributes_size = attributes.size(); - - stream->readPrefix(); - - while (const auto block = stream->read()) - { - const auto columns = ext::map( - ext::range(0, keys_size), [&](const size_t attribute_idx) { return block.safeGetByPosition(attribute_idx).column; }); - - const auto attribute_columns = ext::map(ext::range(0, attributes_size), [&](const size_t attribute_idx) - { - return block.safeGetByPosition(keys_size + attribute_idx).column; - }); - for (const size_t attribute_idx : ext::range(0, attributes.size())) - { - if (attribute.name != attribute_name_by_index.at(attribute_idx)) - { - continue; - } - - const IColumn & attribute_column = *attribute_columns[attribute_idx]; - Arena pool; - - StringRefs keys_temp(keys_size); - - const auto columns_size = columns.front()->size(); - - for (const auto row_idx : ext::range(0, columns_size)) - { - const StringRef key = placeKeysInPool(row_idx, columns, keys_temp, *dict_struct.key, pool); - - if (value_by_key.has(key)) - { - auto value = attribute_column[row_idx]; - - if (value.isNull()) - value_is_null[key] = true; - else - value_by_key[key] = static_cast(value.template get>()); - } - } - } - } - - stream->readSuffix(); - - for (const auto row : ext::range(0, rows)) - { - auto key = keys[row]; - set_value(row, value_by_key[key], value_is_null[key]); - } - - query_count.fetch_add(rows, std::memory_order_relaxed); -} - -const ComplexKeyDirectDictionary::Attribute & ComplexKeyDirectDictionary::getAttribute(const std::string & attribute_name) const -{ - const auto it = attribute_index_by_name.find(attribute_name); - if (it == std::end(attribute_index_by_name)) - throw Exception{full_name + ": no such attribute '" + attribute_name + "'", ErrorCodes::BAD_ARGUMENTS}; - - return attributes[it->second]; -} - -BlockInputStreamPtr ComplexKeyDirectDictionary::getBlockInputStream(const Names & /* column_names */, size_t /* max_block_size */) const -{ - return source_ptr->loadAll(); -} - - -void registerDictionaryComplexKeyDirect(DictionaryFactory & factory) -{ - auto create_layout = [=](const std::string & full_name, - const DictionaryStructure & dict_struct, - const Poco::Util::AbstractConfiguration & config, - const std::string & config_prefix, - DictionarySourcePtr source_ptr) -> DictionaryPtr - { - if (!dict_struct.key) - throw Exception{"'key' is required for dictionary of layout 'complex_key_direct'", ErrorCodes::BAD_ARGUMENTS}; - - if (dict_struct.range_min || dict_struct.range_max) - throw Exception{full_name - + ": elements .structure.range_min and .structure.range_max should be defined only " - "for a dictionary of layout 'range_hashed'", - ErrorCodes::BAD_ARGUMENTS}; - - const auto dict_id = StorageID::fromDictionaryConfig(config, config_prefix); - - if (config.has(config_prefix + ".lifetime.min") || config.has(config_prefix + ".lifetime.max")) - throw Exception{"'lifetime' parameter is redundant for the dictionary' of layout 'direct'", ErrorCodes::BAD_ARGUMENTS}; - - - return std::make_unique(dict_id, dict_struct, std::move(source_ptr)); - }; - factory.registerLayout("complex_key_direct", create_layout, true); -} - - -} diff --git a/src/Dictionaries/ComplexKeyDirectDictionary.h b/src/Dictionaries/ComplexKeyDirectDictionary.h deleted file mode 100644 index 326ffa2924a..00000000000 --- a/src/Dictionaries/ComplexKeyDirectDictionary.h +++ /dev/null @@ -1,147 +0,0 @@ -#pragma once - -#include -#include -#include -#include -#include -#include -#include - -#include -#include -#include -#include -#include "IDictionary.h" -#include "IDictionarySource.h" -#include "DictionaryStructure.h" -#include "DictionaryHelpers.h" - -namespace DB -{ - -class ComplexKeyDirectDictionary final : public IDictionaryBase -{ -public: - ComplexKeyDirectDictionary( - const StorageID & dict_id_, - const DictionaryStructure & dict_struct_, - DictionarySourcePtr source_ptr_, - BlockPtr saved_block_ = nullptr); - - std::string getTypeName() const override { return "ComplexKeyDirect"; } - - size_t getBytesAllocated() const override { return 0; } - - size_t getQueryCount() const override { return query_count.load(std::memory_order_relaxed); } - - double getHitRate() const override { return 1.0; } - - size_t getElementCount() const override { return 0; } - - double getLoadFactor() const override { return 0; } - - std::string getKeyDescription() const { return key_description; } - - std::shared_ptr clone() const override - { - return std::make_shared(getDictionaryID(), dict_struct, source_ptr->clone(), saved_block); - } - - const IDictionarySource * getSource() const override { return source_ptr.get(); } - - const DictionaryLifetime & getLifetime() const override { return dict_lifetime; } - - const DictionaryStructure & getStructure() const override { return dict_struct; } - - bool isInjective(const std::string & attribute_name) const override - { - return dict_struct.attributes[&getAttribute(attribute_name) - attributes.data()].injective; - } - - DictionaryKeyType getKeyType() const override { return DictionaryKeyType::complex; } - - ColumnPtr getColumn( - const std::string& attribute_name, - const DataTypePtr & result_type, - const Columns & key_columns, - const DataTypes & key_types, - const ColumnPtr & default_values_column) const override; - - ColumnUInt8::Ptr hasKeys(const Columns & key_columns, const DataTypes & key_types) const override; - - BlockInputStreamPtr getBlockInputStream(const Names & column_names, size_t max_block_size) const override; - -private: - template - using MapType = HashMapWithSavedHash; - - struct Attribute final - { - AttributeUnderlyingType type; - bool is_nullable; - - std::variant< - UInt8, - UInt16, - UInt32, - UInt64, - UInt128, - Int8, - Int16, - Int32, - Int64, - Decimal32, - Decimal64, - Decimal128, - Float32, - Float64, - StringRef> - null_values; - std::unique_ptr string_arena; - std::string name; - }; - - void createAttributes(); - - template - void addAttributeSize(const Attribute & attribute); - - template - static void createAttributeImpl(Attribute & attribute, const Field & null_value); - - static Attribute createAttribute(const DictionaryAttribute & attribute, const Field & null_value, const std::string & name); - - template - StringRef placeKeysInPool( - const size_t row, const Columns & key_columns, StringRefs & keys, const std::vector & key_attributes, Pool & pool) const; - - template - void getItemsImpl( - const Attribute & attribute, - const Columns & key_columns, - ValueSetter && set_value, - DefaultValueExtractor & default_value_extractor) const; - - template - void setAttributeValueImpl(Attribute & attribute, const Key id, const T & value); - - void setAttributeValue(Attribute & attribute, const Key id, const Field & value); - - const Attribute & getAttribute(const std::string & attribute_name) const; - - const DictionaryStructure dict_struct; - const DictionarySourcePtr source_ptr; - const DictionaryLifetime dict_lifetime; - - std::map attribute_index_by_name; - std::map attribute_name_by_index; - std::vector attributes; - - mutable std::atomic query_count{0}; - - BlockPtr saved_block; - const std::string key_description{dict_struct.getKeyDescription()}; -}; - -} diff --git a/src/Dictionaries/DirectDictionary.cpp b/src/Dictionaries/DirectDictionary.cpp index ac995d51f09..f6a8907b29c 100644 --- a/src/Dictionaries/DirectDictionary.cpp +++ b/src/Dictionaries/DirectDictionary.cpp @@ -17,8 +17,115 @@ namespace ErrorCodes extern const int UNSUPPORTED_METHOD; } +namespace +{ + inline UInt64 getAt(const PaddedPODArray & arr, const size_t idx) + { + return arr[idx]; + } -DirectDictionary::DirectDictionary( + inline UInt64 getAt(const UInt64 & value, const size_t) + { + return value; + } + + /// TODO: Use this class from DictionaryHelpers after cache dictionaries pull request will be merged + template + class DictionaryKeysExtractor + { + public: + using KeyType = std::conditional_t; + static_assert(key_type != DictionaryKeyType::range, "Range key type is not supported by DictionaryKeysExtractor"); + + explicit DictionaryKeysExtractor(const Columns & key_columns, Arena & existing_arena) + { + assert(!key_columns.empty()); + + if constexpr (key_type == DictionaryKeyType::simple) + keys = getColumnVectorData(key_columns.front()); + else + keys = deserializeKeyColumnsInArena(key_columns, existing_arena); + } + + + const PaddedPODArray & getKeys() const + { + return keys; + } + + private: + static PaddedPODArray getColumnVectorData(const ColumnPtr column) + { + PaddedPODArray result; + + auto full_column = column->convertToFullColumnIfConst(); + const auto *vector_col = checkAndGetColumn>(full_column.get()); + + if (!vector_col) + throw Exception{ErrorCodes::TYPE_MISMATCH, "Column type mismatch for simple key expected UInt64"}; + + result.assign(vector_col->getData()); + + return result; + } + + static PaddedPODArray deserializeKeyColumnsInArena(const Columns & key_columns, Arena & temporary_arena) + { + size_t keys_size = key_columns.front()->size(); + + PaddedPODArray result; + result.reserve(keys_size); + + PaddedPODArray temporary_column_data(key_columns.size()); + + for (size_t key_index = 0; key_index < keys_size; ++key_index) + { + size_t allocated_size_for_columns = 0; + const char * block_start = nullptr; + + for (size_t column_index = 0; column_index < key_columns.size(); ++column_index) + { + const auto & column = key_columns[column_index]; + temporary_column_data[column_index] = column->serializeValueIntoArena(key_index, temporary_arena, block_start); + allocated_size_for_columns += temporary_column_data[column_index].size; + } + + result.push_back(StringRef{block_start, allocated_size_for_columns}); + } + + return result; + } + + PaddedPODArray keys; + }; + + /// TODO: Use this class from DictionaryHelpers after cache dictionaries pull request will be merged + class DefaultValueProvider final + { + public: + explicit DefaultValueProvider(Field default_value_, ColumnPtr default_values_column_ = nullptr) + : default_value(std::move(default_value_)) + , default_values_column(default_values_column_) + { + } + + + Field getDefaultValue(size_t row) const + { + if (default_values_column) + return (*default_values_column)[row]; + + return default_value; + } + + private: + Field default_value; + ColumnPtr default_values_column; + }; +} + +template +DirectDictionary::DirectDictionary( const StorageID & dict_id_, const DictionaryStructure & dict_struct_, DictionarySourcePtr source_ptr_, @@ -28,36 +135,37 @@ DirectDictionary::DirectDictionary( , source_ptr{std::move(source_ptr_)} , saved_block{std::move(saved_block_)} { - if (!this->source_ptr->supportsSelectiveLoad()) + if (!source_ptr->supportsSelectiveLoad()) throw Exception{full_name + ": source cannot be used with DirectDictionary", ErrorCodes::UNSUPPORTED_METHOD}; - createAttributes(); + setup(); } - -void DirectDictionary::toParent(const PaddedPODArray & ids, PaddedPODArray & out) const +template +void DirectDictionary::toParent(const PaddedPODArray & ids, PaddedPODArray & out) const { - const auto null_value = std::get(hierarchical_attribute->null_values); - DictionaryDefaultValueExtractor extractor(null_value); + if constexpr (dictionary_key_type == DictionaryKeyType::simple) + { + const auto & attribute_name = hierarchical_attribute->name; - getItemsImpl( - *hierarchical_attribute, - ids, - [&](const size_t row, const UInt64 value, bool) { out[row] = value; }, - extractor); + auto result_type = std::make_shared(); + auto input_column = result_type->createColumn(); + auto & input_column_typed = assert_cast &>(*input_column); + auto & data = input_column_typed.getData(); + data.insert(ids.begin(), ids.end()); + + auto column = getColumn({attribute_name}, result_type, {std::move(input_column)}, {result_type}, {nullptr}); + const auto & result_column_typed = assert_cast &>(*column); + const auto & result_data = result_column_typed.getData(); + + out.assign(result_data); + } + else + throw Exception(ErrorCodes::UNSUPPORTED_METHOD, "Hierarchy is not supported for complex key DirectDictionary"); } - -static inline DirectDictionary::Key getAt(const PaddedPODArray & arr, const size_t idx) -{ - return arr[idx]; -} -static inline DirectDictionary::Key getAt(const DirectDictionary::Key & value, const size_t) -{ - return value; -} - -DirectDictionary::Key DirectDictionary::getValueOrNullByKey(const Key & to_find) const +template +UInt64 DirectDictionary::getValueOrNullByKey(const Key & to_find) const { std::vector required_key = {to_find}; @@ -65,12 +173,13 @@ DirectDictionary::Key DirectDictionary::getValueOrNullByKey(const Key & to_find) stream->readPrefix(); bool is_found = false; - Key result = std::get(hierarchical_attribute->null_values); + UInt64 result = hierarchical_attribute->null_value.template get(); + while (const auto block = stream->read()) { const IColumn & id_column = *block.safeGetByPosition(0).column; - for (const size_t attribute_idx : ext::range(0, attributes.size())) + for (const size_t attribute_idx : ext::range(0, dict_struct.attributes.size())) { if (is_found) break; @@ -96,10 +205,11 @@ DirectDictionary::Key DirectDictionary::getValueOrNullByKey(const Key & to_find) return result; } +template template -void DirectDictionary::isInImpl(const ChildType & child_ids, const AncestorType & ancestor_ids, PaddedPODArray & out) const +void DirectDictionary::isInImpl(const ChildType & child_ids, const AncestorType & ancestor_ids, PaddedPODArray & out) const { - const auto null_value = std::get(hierarchical_attribute->null_values); + const auto null_value = hierarchical_attribute->null_value.template get(); const auto rows = out.size(); for (const auto row : ext::range(0, rows)) @@ -116,315 +226,308 @@ void DirectDictionary::isInImpl(const ChildType & child_ids, const AncestorType query_count.fetch_add(rows, std::memory_order_relaxed); } - -void DirectDictionary::isInVectorVector( - const PaddedPODArray & child_ids, const PaddedPODArray & ancestor_ids, PaddedPODArray & out) const +template +void DirectDictionary::isInVectorVector( + const PaddedPODArray & child_ids, const PaddedPODArray & ancestor_ids, PaddedPODArray & out) const { isInImpl(child_ids, ancestor_ids, out); } -void DirectDictionary::isInVectorConstant(const PaddedPODArray & child_ids, const Key ancestor_id, PaddedPODArray & out) const +template +void DirectDictionary::isInVectorConstant(const PaddedPODArray & child_ids, const UInt64 ancestor_id, PaddedPODArray & out) const { isInImpl(child_ids, ancestor_id, out); } -void DirectDictionary::isInConstantVector(const Key child_id, const PaddedPODArray & ancestor_ids, PaddedPODArray & out) const +template +void DirectDictionary::isInConstantVector(const UInt64 child_id, const PaddedPODArray & ancestor_ids, PaddedPODArray & out) const { isInImpl(child_id, ancestor_ids, out); } -ColumnPtr DirectDictionary::getColumn( +template +ColumnPtr DirectDictionary::getColumn( const std::string & attribute_name, const DataTypePtr & result_type, const Columns & key_columns, - const DataTypes &, + const DataTypes & key_types, const ColumnPtr & default_values_column) const { - ColumnPtr result; + if constexpr (dictionary_key_type == DictionaryKeyType::complex) + dict_struct.validateKeyTypes(key_types); - PaddedPODArray backup_storage; - const auto & ids = getColumnVectorData(this, key_columns.front(), backup_storage); + Arena complex_key_arena; - const auto & attribute = getAttribute(attribute_name); + const DictionaryAttribute & attribute = dict_struct.getAttribute(attribute_name, result_type); + auto result = attribute.type->createColumn(); - auto keys_size = ids.size(); + DefaultValueProvider default_value_provider(attribute.null_value, default_values_column); + DictionaryKeysExtractor extractor(key_columns, complex_key_arena); + const auto & requested_keys = extractor.getKeys(); + size_t requested_attribute_index = attribute_index_by_name.find(attribute_name)->second; - ColumnUInt8::MutablePtr col_null_map_to; - ColumnUInt8::Container * vec_null_map_to = nullptr; - if (attribute.is_nullable) - { - col_null_map_to = ColumnUInt8::create(keys_size, false); - vec_null_map_to = &col_null_map_to->getData(); - } + size_t dictionary_keys_size = dict_struct.getKeysNames().size(); + size_t requested_key_index = 0; + Field block_column_value; - const auto & dictionary_attribute = dict_struct.getAttribute(attribute_name, result_type); + /** In result stream keys are returned in same order as they were requested. + * For example if we request keys [1, 2, 3, 4] but source has only [2, 3] we need to return to client + * [default_value, 2, 3, default_value]. + * For each key fetched from source current algorithm adds default values until + * requested key with requested_key_index match key fetched from source. + * At the end we also need to process tail. + */ - auto type_call = [&](const auto &dictionary_attribute_type) - { - using Type = std::decay_t; - using AttributeType = typename Type::AttributeType; + BlockInputStreamPtr stream = getSourceBlockInputStream(key_columns, requested_keys); - using ValueType = DictionaryValueType; - using ColumnProvider = DictionaryAttributeColumnProvider; - - const auto attribute_null_value = std::get(attribute.null_values); - AttributeType null_value = static_cast(attribute_null_value); - DictionaryDefaultValueExtractor default_value_extractor(std::move(null_value), default_values_column); - - auto column = ColumnProvider::getColumn(dictionary_attribute, keys_size); - - if constexpr (std::is_same_v) - { - auto * out = column.get(); - - getItemsImpl( - attribute, - ids, - [&](const size_t row, const String value, bool is_null) - { - if (attribute.is_nullable) - (*vec_null_map_to)[row] = is_null; - - const auto ref = StringRef{value}; - out->insertData(ref.data, ref.size); - }, - default_value_extractor); - } - else - { - auto & out = column->getData(); - - getItemsImpl( - attribute, - ids, - [&](const size_t row, const auto value, bool is_null) - { - if (attribute.is_nullable) - (*vec_null_map_to)[row] = is_null; - - out[row] = value; - }, - default_value_extractor); - } - - result = std::move(column); - }; - - callOnDictionaryAttributeType(attribute.type, type_call); - - if (attribute.is_nullable) - { - result = ColumnNullable::create(result, std::move(col_null_map_to)); - } - - return result; -} - -ColumnUInt8::Ptr DirectDictionary::hasKeys(const Columns & key_columns, const DataTypes &) const -{ - PaddedPODArray backup_storage; - const auto& ids = getColumnVectorData(this, key_columns.front(), backup_storage); - - auto result = ColumnUInt8::create(ext::size(ids)); - auto& out = result->getData(); - - const auto rows = ext::size(ids); - - HashMap has_key; - for (const auto row : ext::range(0, rows)) - has_key[ids[row]] = 0; - - std::vector to_load; - to_load.reserve(has_key.size()); - for (auto it = has_key.begin(); it != has_key.end(); ++it) - to_load.emplace_back(static_cast(it->getKey())); - - auto stream = source_ptr->loadIds(to_load); stream->readPrefix(); while (const auto block = stream->read()) { - const IColumn & id_column = *block.safeGetByPosition(0).column; + Columns block_key_columns; + block_key_columns.reserve(dictionary_keys_size); - for (const auto row_idx : ext::range(0, id_column.size())) + auto block_columns = block.getColumns(); + + /// Split into keys columns and attribute columns + for (size_t i = 0; i < dictionary_keys_size; ++i) { - const auto key = id_column[row_idx].get(); - has_key[key] = 1; + block_key_columns.emplace_back(*block_columns.begin()); + block_columns.erase(block_columns.begin()); + } + + DictionaryKeysExtractor block_keys_extractor(block_key_columns, complex_key_arena); + const auto & block_keys = block_keys_extractor.getKeys(); + size_t block_keys_size = block_keys.size(); + + const auto & block_column = block.safeGetByPosition(dictionary_keys_size + requested_attribute_index).column; + + for (size_t block_key_index = 0; block_key_index < block_keys_size; ++block_key_index) + { + auto block_key = block_keys[block_key_index]; + + while (requested_key_index < requested_keys.size() && + block_key != requested_keys[requested_key_index]) + { + block_column_value = default_value_provider.getDefaultValue(requested_key_index); + result->insert(block_column_value); + ++requested_key_index; + } + + block_column->get(block_key_index, block_column_value); + result->insert(block_column_value); + ++requested_key_index; } } stream->readSuffix(); - for (const auto row : ext::range(0, rows)) - out[row] = has_key[ids[row]]; + size_t requested_keys_size = requested_keys.size(); - query_count.fetch_add(rows, std::memory_order_relaxed); + Field default_value; + /// Process tail, if source returned keys less keys sizes than we fetched insert default value for tail + for (; requested_key_index < requested_keys_size; ++requested_key_index) + { + default_value = default_value_provider.getDefaultValue(requested_key_index); + result->insert(default_value); + } + + query_count.fetch_add(requested_keys_size, std::memory_order_relaxed); + + Field result_val; + for (size_t i = 0; i < result->size(); ++i) + { + result->get(i, result_val); + std::cerr << "I " << i << " dump " << result_val.dump() << std::endl; + } return result; } -void DirectDictionary::createAttributes() +template +ColumnUInt8::Ptr DirectDictionary::hasKeys(const Columns & key_columns, const DataTypes & key_types) const { - const auto size = dict_struct.attributes.size(); - attributes.reserve(size); + if constexpr (dictionary_key_type == DictionaryKeyType::complex) + dict_struct.validateKeyTypes(key_types); - for (const auto & attribute : dict_struct.attributes) + Arena complex_key_arena; + + DictionaryKeysExtractor requested_keys_extractor(key_columns, complex_key_arena); + const auto & requested_keys = requested_keys_extractor.getKeys(); + size_t requested_keys_size = requested_keys.size(); + + auto result = ColumnUInt8::create(requested_keys_size, false); + auto & result_data = result->getData(); + + size_t dictionary_keys_size = dict_struct.getKeysNames().size(); + size_t requested_key_index = 0; + Field block_column_value; + + /** Algorithm is the same as in getColumn method. There are only 2 details + * 1. We does not process tail because result column is created with false default value. + * 2. If requested key does not match key from source we set false in requested_key_index. + */ + + BlockInputStreamPtr stream = getSourceBlockInputStream(key_columns, requested_keys); + + stream->readPrefix(); + + while (const auto block = stream->read()) { - attribute_index_by_name.emplace(attribute.name, attributes.size()); - attribute_name_by_index.emplace(attributes.size(), attribute.name); - attributes.push_back(createAttribute(attribute, attribute.null_value, attribute.name)); + auto block_columns = block.getColumns(); + + Columns block_key_columns; + block_key_columns.reserve(dictionary_keys_size); + + /// Split into keys columns and attribute columns + for (size_t i = 0; i < dictionary_keys_size; ++i) + { + block_key_columns.emplace_back(*block_columns.begin()); + block_columns.erase(block_columns.begin()); + } + + DictionaryKeysExtractor block_keys_extractor(block_key_columns, complex_key_arena); + const auto & block_keys = block_keys_extractor.getKeys(); + size_t block_keys_size = block_keys.size(); + + for (size_t block_key_index = 0; block_key_index < block_keys_size; ++block_key_index) + { + auto block_key = block_keys[block_key_index]; + + while (requested_key_index < requested_keys.size() && + block_key != requested_keys[requested_key_index]) + { + result_data[requested_key_index] = false; + ++requested_key_index; + } + + result_data[requested_key_index] = true; + ++requested_key_index; + } + } + + stream->readSuffix(); + + /// We does not add additional code for tail because result was initialized with false values + + query_count.fetch_add(requested_keys_size, std::memory_order_relaxed); + + return result; +} + +template +BlockInputStreamPtr DirectDictionary::getSourceBlockInputStream(const Columns & key_columns, const PaddedPODArray & requested_keys) const +{ + size_t requested_keys_size = requested_keys.size(); + + BlockInputStreamPtr stream; + + if constexpr (dictionary_key_type == DictionaryKeyType::simple) + { + std::vector ids; + ids.reserve(requested_keys_size); + + for (auto key : requested_keys) + ids.emplace_back(key); + + stream = source_ptr->loadIds(ids); + } + else + { + std::vector requested_rows; + requested_rows.reserve(requested_keys_size); + for (size_t i = 0; i < requested_keys_size; ++i) + requested_rows.emplace_back(i); + + stream = source_ptr->loadKeys(key_columns, requested_rows); + } + + return stream; +} + +template +void DirectDictionary::setup() +{ + /// TODO: Move this to DictionaryStructure + size_t dictionary_attributes_size = dict_struct.attributes.size(); + for (size_t i = 0; i < dictionary_attributes_size; ++i) + { + const auto & attribute = dict_struct.attributes[i]; + attribute_index_by_name[attribute.name] = i; + attribute_name_by_index[i] = attribute.name; if (attribute.hierarchical) { - hierarchical_attribute = &attributes.back(); + if constexpr (dictionary_key_type == DictionaryKeyType::complex) + throw Exception(ErrorCodes::BAD_ARGUMENTS, + "({}): hierachical attributes are not supported for complex key direct dictionary", + full_name); - if (hierarchical_attribute->type != AttributeUnderlyingType::utUInt64) + hierarchical_attribute = &attribute; + + if (attribute.underlying_type != AttributeUnderlyingType::utUInt64) throw Exception{full_name + ": hierarchical attribute must be UInt64.", ErrorCodes::TYPE_MISMATCH}; } } } - -template -void DirectDictionary::createAttributeImpl(Attribute & attribute, const Field & null_value) -{ - attribute.null_values = T(null_value.get>()); -} - -template <> -void DirectDictionary::createAttributeImpl(Attribute & attribute, const Field & null_value) -{ - attribute.string_arena = std::make_unique(); - const String & string = null_value.get(); - const char * string_in_arena = attribute.string_arena->insert(string.data(), string.size()); - attribute.null_values.emplace(string_in_arena, string.size()); -} - - -DirectDictionary::Attribute DirectDictionary::createAttribute(const DictionaryAttribute& attribute, const Field & null_value, const std::string & attr_name) -{ - Attribute attr{attribute.underlying_type, attribute.is_nullable, {}, {}, attr_name}; - - auto type_call = [&](const auto &dictionary_attribute_type) - { - using Type = std::decay_t; - using AttributeType = typename Type::AttributeType; - createAttributeImpl(attr, null_value); - }; - - callOnDictionaryAttributeType(attribute.underlying_type, type_call); - - return attr; -} - - -template -void DirectDictionary::getItemsImpl( - const Attribute & attribute, - const PaddedPODArray & ids, - ValueSetter && set_value, - DefaultValueExtractor & default_value_extractor) const -{ - const auto rows = ext::size(ids); - - HashMap value_by_key; - HashSet value_is_null; - - for (const auto row : ext::range(0, rows)) - { - auto key = ids[row]; - value_by_key[key] = static_cast(default_value_extractor[row]); - } - - std::vector to_load; - to_load.reserve(value_by_key.size()); - for (auto it = value_by_key.begin(); it != value_by_key.end(); ++it) - to_load.emplace_back(static_cast(it->getKey())); - - auto stream = source_ptr->loadIds(to_load); - stream->readPrefix(); - - const auto it = attribute_index_by_name.find(attribute.name); - if (it == std::end(attribute_index_by_name)) - throw Exception{full_name + ": no such attribute '" + attribute.name + "'", ErrorCodes::BAD_ARGUMENTS}; - - auto attribute_index = it->second; - - while (const auto block = stream->read()) - { - const IColumn & id_column = *block.safeGetByPosition(0).column; - - const IColumn & attribute_column = *block.safeGetByPosition(attribute_index + 1).column; - - for (const auto row_idx : ext::range(0, id_column.size())) - { - const auto key = id_column[row_idx].get(); - - if (value_by_key.find(key) != value_by_key.end()) - { - auto value = attribute_column[row_idx]; - - if (value.isNull()) - value_is_null.insert(key); - else - value_by_key[key] = static_cast(value.get>()); - } - } - } - - stream->readSuffix(); - - for (const auto row : ext::range(0, rows)) - { - auto key = ids[row]; - set_value(row, value_by_key[key], value_is_null.find(key) != nullptr); - } - - query_count.fetch_add(rows, std::memory_order_relaxed); -} - -const DirectDictionary::Attribute & DirectDictionary::getAttribute(const std::string & attribute_name) const -{ - const auto it = attribute_index_by_name.find(attribute_name); - if (it == std::end(attribute_index_by_name)) - throw Exception{full_name + ": no such attribute '" + attribute_name + "'", ErrorCodes::BAD_ARGUMENTS}; - - return attributes[it->second]; -} - - -BlockInputStreamPtr DirectDictionary::getBlockInputStream(const Names & /* column_names */, size_t /* max_block_size */) const +template +BlockInputStreamPtr DirectDictionary::getBlockInputStream(const Names & /* column_names */, size_t /* max_block_size */) const { return source_ptr->loadAll(); } - -void registerDictionaryDirect(DictionaryFactory & factory) +namespace { - auto create_layout = [=](const std::string & full_name, - const DictionaryStructure & dict_struct, - const Poco::Util::AbstractConfiguration & config, - const std::string & config_prefix, - DictionarySourcePtr source_ptr) -> DictionaryPtr + template + DictionaryPtr createDirectDictionary( + const std::string & full_name, + const DictionaryStructure & dict_struct, + const Poco::Util::AbstractConfiguration & config, + const std::string & config_prefix, + DictionarySourcePtr source_ptr) { - if (dict_struct.key) - throw Exception{"'key' is not supported for dictionary of layout 'direct'", ErrorCodes::UNSUPPORTED_METHOD}; + const auto * layout_name = dictionary_key_type == DictionaryKeyType::simple ? "direct" : "complex_key_direct"; + + if constexpr (dictionary_key_type == DictionaryKeyType::simple) + { + if (dict_struct.key) + throw Exception(ErrorCodes::UNSUPPORTED_METHOD, + "'key' is not supported for dictionary of layout '({})'", + layout_name); + } + else + { + if (dict_struct.id) + throw Exception(ErrorCodes::UNSUPPORTED_METHOD, + "'id' is not supported for dictionary of layout '({})'", + layout_name); + } if (dict_struct.range_min || dict_struct.range_max) - throw Exception{full_name - + ": elements .structure.range_min and .structure.range_max should be defined only " - "for a dictionary of layout 'range_hashed'", - ErrorCodes::BAD_ARGUMENTS}; + throw Exception(ErrorCodes::BAD_ARGUMENTS, + "({}): elements .structure.range_min and .structure.range_max should be defined only " \ + "for a dictionary of layout 'range_hashed'", + full_name); const auto dict_id = StorageID::fromDictionaryConfig(config, config_prefix); if (config.has(config_prefix + ".lifetime.min") || config.has(config_prefix + ".lifetime.max")) - throw Exception{"'lifetime' parameter is redundant for the dictionary' of layout 'direct'", ErrorCodes::BAD_ARGUMENTS}; + throw Exception(ErrorCodes::BAD_ARGUMENTS, + "'lifetime' parameter is redundant for the dictionary' of layout '({})'", + layout_name); + return std::make_unique>(dict_id, dict_struct, std::move(source_ptr)); + } +} - return std::make_unique(dict_id, dict_struct, std::move(source_ptr)); - }; - factory.registerLayout("direct", create_layout, false); +template class DirectDictionary; +template class DirectDictionary; + +void registerDictionaryDirect(DictionaryFactory & factory) +{ + factory.registerLayout("direct", createDirectDictionary, false); + factory.registerLayout("complex_key_direct", createDirectDictionary, true); } diff --git a/src/Dictionaries/DirectDictionary.h b/src/Dictionaries/DirectDictionary.h index 1fb6c8954b2..4fa054625e0 100644 --- a/src/Dictionaries/DirectDictionary.h +++ b/src/Dictionaries/DirectDictionary.h @@ -18,16 +18,25 @@ namespace DB { +template class DirectDictionary final : public IDictionary { public: + static_assert(dictionary_key_type != DictionaryKeyType::range, "Range key type is not supported by direct dictionary"); + using KeyType = std::conditional_t; + DirectDictionary( const StorageID & dict_id_, const DictionaryStructure & dict_struct_, DictionarySourcePtr source_ptr_, BlockPtr saved_block_ = nullptr); - std::string getTypeName() const override { return "Direct"; } + std::string getTypeName() const override { + if constexpr (dictionary_key_type == DictionaryKeyType::simple) + return "Direct"; + else + return "ComplexKeyDirect"; + } size_t getBytesAllocated() const override { return 0; } @@ -52,19 +61,27 @@ public: bool isInjective(const std::string & attribute_name) const override { - return dict_struct.attributes[&getAttribute(attribute_name) - attributes.data()].injective; + auto it = attribute_index_by_name.find(attribute_name); + + if (it == attribute_index_by_name.end()) + throw Exception(ErrorCodes::BAD_ARGUMENTS, + "({}): no attribute with name ({}) in dictionary", + full_name, + attribute_name); + + return dict_struct.attributes[it->second].injective; } bool hasHierarchy() const override { return hierarchical_attribute; } - void toParent(const PaddedPODArray & ids, PaddedPODArray & out) const override; + void toParent(const PaddedPODArray & ids, PaddedPODArray & out) const override; void isInVectorVector( - const PaddedPODArray & child_ids, const PaddedPODArray & ancestor_ids, PaddedPODArray & out) const override; - void isInVectorConstant(const PaddedPODArray & child_ids, const Key ancestor_id, PaddedPODArray & out) const override; - void isInConstantVector(const Key child_id, const PaddedPODArray & ancestor_ids, PaddedPODArray & out) const override; + const PaddedPODArray & child_ids, const PaddedPODArray & ancestor_ids, PaddedPODArray & out) const override; + void isInVectorConstant(const PaddedPODArray & child_ids, const UInt64 ancestor_id, PaddedPODArray & out) const override; + void isInConstantVector(const UInt64 child_id, const PaddedPODArray & ancestor_ids, PaddedPODArray & out) const override; - DictionaryKeyType getKeyType() const override { return DictionaryKeyType::simple; } + DictionaryKeyType getKeyType() const override { return dictionary_key_type; } ColumnPtr getColumn( const std::string& attribute_name, @@ -78,56 +95,11 @@ public: BlockInputStreamPtr getBlockInputStream(const Names & column_names, size_t max_block_size) const override; private: - struct Attribute final - { - AttributeUnderlyingType type; - bool is_nullable; - std::variant< - UInt8, - UInt16, - UInt32, - UInt64, - UInt128, - Int8, - Int16, - Int32, - Int64, - Decimal32, - Decimal64, - Decimal128, - Float32, - Float64, - StringRef> - null_values; - std::unique_ptr string_arena; - std::string name; - }; + void setup(); - void createAttributes(); + BlockInputStreamPtr getSourceBlockInputStream(const Columns & key_columns, const PaddedPODArray & requested_keys) const; - template - void addAttributeSize(const Attribute & attribute); - - template - static void createAttributeImpl(Attribute & attribute, const Field & null_value); - - static Attribute createAttribute(const DictionaryAttribute& attribute, const Field & null_value, const std::string & name); - - template - void getItemsImpl( - const Attribute & attribute, - const PaddedPODArray & ids, - ValueSetter && set_value, - DefaultValueExtractor & default_value_extractor) const; - - template - void setAttributeValueImpl(Attribute & attribute, const Key id, const T & value); - - void setAttributeValue(Attribute & attribute, const Key id, const Field & value); - - const Attribute & getAttribute(const std::string & attribute_name) const; - - Key getValueOrNullByKey(const Key & to_find) const; + UInt64 getValueOrNullByKey(const UInt64 & to_find) const; template void isInImpl(const ChildType & child_ids, const AncestorType & ancestor_ids, PaddedPODArray & out) const; @@ -136,14 +108,17 @@ private: const DictionarySourcePtr source_ptr; const DictionaryLifetime dict_lifetime; - std::map attribute_index_by_name; - std::map attribute_name_by_index; - std::vector attributes; - const Attribute * hierarchical_attribute = nullptr; + std::unordered_map attribute_index_by_name; + std::unordered_map attribute_name_by_index; + + const DictionaryAttribute * hierarchical_attribute = nullptr; mutable std::atomic query_count{0}; BlockPtr saved_block; }; +extern template class DirectDictionary; +extern template class DirectDictionary; + } diff --git a/src/Dictionaries/registerDictionaries.cpp b/src/Dictionaries/registerDictionaries.cpp index 6ae7d3f2d94..a7b3c87267d 100644 --- a/src/Dictionaries/registerDictionaries.cpp +++ b/src/Dictionaries/registerDictionaries.cpp @@ -25,7 +25,6 @@ void registerDictionarySourceLibrary(DictionarySourceFactory & source_factory); class DictionaryFactory; void registerDictionaryRangeHashed(DictionaryFactory & factory); void registerDictionaryComplexKeyHashed(DictionaryFactory & factory); -void registerDictionaryComplexKeyDirect(DictionaryFactory & factory); void registerDictionaryTrie(DictionaryFactory & factory); void registerDictionaryFlat(DictionaryFactory & factory); void registerDictionaryHashed(DictionaryFactory & factory); @@ -59,7 +58,6 @@ void registerDictionaries() auto & factory = DictionaryFactory::instance(); registerDictionaryRangeHashed(factory); registerDictionaryComplexKeyHashed(factory); - registerDictionaryComplexKeyDirect(factory); registerDictionaryTrie(factory); registerDictionaryFlat(factory); registerDictionaryHashed(factory); diff --git a/src/Functions/FunctionsExternalDictionaries.h b/src/Functions/FunctionsExternalDictionaries.h index be241273930..acf158d42ef 100644 --- a/src/Functions/FunctionsExternalDictionaries.h +++ b/src/Functions/FunctionsExternalDictionaries.h @@ -727,8 +727,9 @@ private: auto dict = helper.getDictionary(arguments[0]); ColumnPtr res; + /// TODO: Rewrite this if (!((res = executeDispatch(arguments, result_type, dict)) - || (res = executeDispatch(arguments, result_type, dict)) + || (res = executeDispatch>(arguments, result_type, dict)) || (res = executeDispatch(arguments, result_type, dict)) || (res = executeDispatch>(arguments, result_type, dict)))) throw Exception{"Unsupported dictionary type " + dict->getTypeName(), ErrorCodes::UNKNOWN_TYPE}; @@ -881,7 +882,7 @@ private: ColumnPtr res; if (!((res = executeDispatch(arguments, dict)) - || (res = executeDispatch(arguments, dict)) + || (res = executeDispatch>(arguments, dict)) || (res = executeDispatch(arguments, dict)) || (res = executeDispatch>(arguments, dict)))) throw Exception{"Unsupported dictionary type " + dict->getTypeName(), ErrorCodes::UNKNOWN_TYPE}; diff --git a/tests/queries/0_stateless/01753_direct_dictionary_simple_key.reference b/tests/queries/0_stateless/01753_direct_dictionary_simple_key.reference new file mode 100644 index 00000000000..1d3a43b59fb --- /dev/null +++ b/tests/queries/0_stateless/01753_direct_dictionary_simple_key.reference @@ -0,0 +1,66 @@ +Dictionary direct_dictionary_simple_key_simple_attributes +dictGet existing value +value_0 value_second_0 +value_1 value_second_1 +value_2 value_second_2 +dictGet with non existing value +value_0 value_second_0 +value_1 value_second_1 +value_2 value_second_2 +value_first_default value_second_default +dictGetOrDefault existing value +value_0 value_second_0 +value_1 value_second_1 +value_2 value_second_2 +dictGetOrDefault non existing value +value_0 value_second_0 +value_1 value_second_1 +value_2 value_second_2 +default default +dictHas +1 +1 +1 +0 +select all values as input stream +0 value_0 value_second_0 +1 value_1 value_second_1 +2 value_2 value_second_2 +Dictionary direct_dictionary_simple_key_complex_attributes +dictGet existing value +value_0 value_second_0 +value_1 \N +value_2 value_second_2 +dictGet with non existing value +value_0 value_second_0 +value_1 \N +value_2 value_second_2 +value_first_default value_second_default +dictGetOrDefault existing value +value_0 value_second_0 +value_1 \N +value_2 value_second_2 +dictGetOrDefault non existing value +value_0 value_second_0 +value_1 \N +value_2 value_second_2 +default default +dictHas +1 +1 +1 +0 +select all values as input stream +0 value_0 value_second_0 +1 value_1 \N +2 value_2 value_second_2 +Dictionary direct_dictionary_simple_key_hierarchy +dictGet +0 +0 +1 +1 +2 +dictGetHierarchy +[1] +[4,2,1] diff --git a/tests/queries/0_stateless/01753_direct_dictionary_simple_key.sql b/tests/queries/0_stateless/01753_direct_dictionary_simple_key.sql new file mode 100644 index 00000000000..afd91ffdaa5 --- /dev/null +++ b/tests/queries/0_stateless/01753_direct_dictionary_simple_key.sql @@ -0,0 +1,120 @@ +DROP DATABASE IF EXISTS 01753_dictionary_db; +CREATE DATABASE 01753_dictionary_db; + +CREATE TABLE 01753_dictionary_db.simple_key_simple_attributes_source_table +( + id UInt64, + value_first String, + value_second String +) +ENGINE = TinyLog; + +INSERT INTO 01753_dictionary_db.simple_key_simple_attributes_source_table VALUES(0, 'value_0', 'value_second_0'); +INSERT INTO 01753_dictionary_db.simple_key_simple_attributes_source_table VALUES(1, 'value_1', 'value_second_1'); +INSERT INTO 01753_dictionary_db.simple_key_simple_attributes_source_table VALUES(2, 'value_2', 'value_second_2'); + +CREATE DICTIONARY 01753_dictionary_db.direct_dictionary_simple_key_simple_attributes +( + id UInt64, + value_first String DEFAULT 'value_first_default', + value_second String DEFAULT 'value_second_default' +) +PRIMARY KEY id +SOURCE(CLICKHOUSE(HOST 'localhost' PORT tcpPort() USER 'default' TABLE 'simple_key_simple_attributes_source_table')) +LAYOUT(DIRECT()); + +SELECT 'Dictionary direct_dictionary_simple_key_simple_attributes'; +SELECT 'dictGet existing value'; +SELECT dictGet('01753_dictionary_db.direct_dictionary_simple_key_simple_attributes', 'value_first', number) as value_first, + dictGet('01753_dictionary_db.direct_dictionary_simple_key_simple_attributes', 'value_second', number) as value_second FROM system.numbers LIMIT 3; +SELECT 'dictGet with non existing value'; +SELECT dictGet('01753_dictionary_db.direct_dictionary_simple_key_simple_attributes', 'value_first', number) as value_first, + dictGet('01753_dictionary_db.direct_dictionary_simple_key_simple_attributes', 'value_second', number) as value_second FROM system.numbers LIMIT 4; +SELECT 'dictGetOrDefault existing value'; +SELECT dictGetOrDefault('01753_dictionary_db.direct_dictionary_simple_key_simple_attributes', 'value_first', number, toString('default')) as value_first, + dictGetOrDefault('01753_dictionary_db.direct_dictionary_simple_key_simple_attributes', 'value_second', number, toString('default')) as value_second FROM system.numbers LIMIT 3; +SELECT 'dictGetOrDefault non existing value'; +SELECT dictGetOrDefault('01753_dictionary_db.direct_dictionary_simple_key_simple_attributes', 'value_first', number, toString('default')) as value_first, + dictGetOrDefault('01753_dictionary_db.direct_dictionary_simple_key_simple_attributes', 'value_second', number, toString('default')) as value_second FROM system.numbers LIMIT 4; +SELECT 'dictHas'; +SELECT dictHas('01753_dictionary_db.direct_dictionary_simple_key_simple_attributes', number) FROM system.numbers LIMIT 4; +SELECT 'select all values as input stream'; +SELECT * FROM 01753_dictionary_db.direct_dictionary_simple_key_simple_attributes; + +DROP DICTIONARY 01753_dictionary_db.direct_dictionary_simple_key_simple_attributes; +DROP TABLE 01753_dictionary_db.simple_key_simple_attributes_source_table; + +CREATE TABLE 01753_dictionary_db.simple_key_complex_attributes_source_table +( + id UInt64, + value_first String, + value_second Nullable(String) +) +ENGINE = TinyLog; + +INSERT INTO 01753_dictionary_db.simple_key_complex_attributes_source_table VALUES(0, 'value_0', 'value_second_0'); +INSERT INTO 01753_dictionary_db.simple_key_complex_attributes_source_table VALUES(1, 'value_1', NULL); +INSERT INTO 01753_dictionary_db.simple_key_complex_attributes_source_table VALUES(2, 'value_2', 'value_second_2'); + +CREATE DICTIONARY 01753_dictionary_db.direct_dictionary_simple_key_complex_attributes +( + id UInt64, + value_first String DEFAULT 'value_first_default', + value_second Nullable(String) DEFAULT 'value_second_default' +) +PRIMARY KEY id +SOURCE(CLICKHOUSE(HOST 'localhost' PORT tcpPort() USER 'default' TABLE 'simple_key_complex_attributes_source_table')) +LAYOUT(DIRECT()); + +SELECT 'Dictionary direct_dictionary_simple_key_complex_attributes'; +SELECT 'dictGet existing value'; +SELECT dictGet('01753_dictionary_db.direct_dictionary_simple_key_complex_attributes', 'value_first', number) as value_first, + dictGet('01753_dictionary_db.direct_dictionary_simple_key_complex_attributes', 'value_second', number) as value_second FROM system.numbers LIMIT 3; +SELECT 'dictGet with non existing value'; +SELECT dictGet('01753_dictionary_db.direct_dictionary_simple_key_complex_attributes', 'value_first', number) as value_first, + dictGet('01753_dictionary_db.direct_dictionary_simple_key_complex_attributes', 'value_second', number) as value_second FROM system.numbers LIMIT 4; +SELECT 'dictGetOrDefault existing value'; +SELECT dictGetOrDefault('01753_dictionary_db.direct_dictionary_simple_key_complex_attributes', 'value_first', number, toString('default')) as value_first, + dictGetOrDefault('01753_dictionary_db.direct_dictionary_simple_key_complex_attributes', 'value_second', number, toString('default')) as value_second FROM system.numbers LIMIT 3; +SELECT 'dictGetOrDefault non existing value'; +SELECT dictGetOrDefault('01753_dictionary_db.direct_dictionary_simple_key_complex_attributes', 'value_first', number, toString('default')) as value_first, + dictGetOrDefault('01753_dictionary_db.direct_dictionary_simple_key_complex_attributes', 'value_second', number, toString('default')) as value_second FROM system.numbers LIMIT 4; +SELECT 'dictHas'; +SELECT dictHas('01753_dictionary_db.direct_dictionary_simple_key_complex_attributes', number) FROM system.numbers LIMIT 4; +SELECT 'select all values as input stream'; +SELECT * FROM 01753_dictionary_db.direct_dictionary_simple_key_complex_attributes; + +DROP DICTIONARY 01753_dictionary_db.direct_dictionary_simple_key_complex_attributes; +DROP TABLE 01753_dictionary_db.simple_key_complex_attributes_source_table; + +CREATE TABLE 01753_dictionary_db.simple_key_hierarchy_table +( + id UInt64, + parent_id UInt64 +) ENGINE = TinyLog(); + +INSERT INTO 01753_dictionary_db.simple_key_hierarchy_table VALUES (1, 0); +INSERT INTO 01753_dictionary_db.simple_key_hierarchy_table VALUES (2, 1); +INSERT INTO 01753_dictionary_db.simple_key_hierarchy_table VALUES (3, 1); +INSERT INTO 01753_dictionary_db.simple_key_hierarchy_table VALUES (4, 2); + +CREATE DICTIONARY 01753_dictionary_db.direct_dictionary_simple_key_hierarchy +( + id UInt64, + parent_id UInt64 HIERARCHICAL +) +PRIMARY KEY id +SOURCE(CLICKHOUSE(HOST 'localhost' PORT tcpPort() USER 'default' TABLE 'simple_key_hierarchy_table')) +LAYOUT(DIRECT()); + +SELECT 'Dictionary direct_dictionary_simple_key_hierarchy'; +SELECT 'dictGet'; +SELECT dictGet('01753_dictionary_db.direct_dictionary_simple_key_hierarchy', 'parent_id', number) FROM system.numbers LIMIT 5; +SELECT 'dictGetHierarchy'; +SELECT dictGetHierarchy('01753_dictionary_db.direct_dictionary_simple_key_hierarchy', toUInt64(1)); +SELECT dictGetHierarchy('01753_dictionary_db.direct_dictionary_simple_key_hierarchy', toUInt64(4)); + +DROP DICTIONARY 01753_dictionary_db.direct_dictionary_simple_key_hierarchy; +DROP TABLE 01753_dictionary_db.simple_key_hierarchy_table; + +DROP DATABASE 01753_dictionary_db; diff --git a/tests/queries/0_stateless/01754_direct_dictionary_complex_key.reference b/tests/queries/0_stateless/01754_direct_dictionary_complex_key.reference new file mode 100644 index 00000000000..603785e984c --- /dev/null +++ b/tests/queries/0_stateless/01754_direct_dictionary_complex_key.reference @@ -0,0 +1,56 @@ +Dictionary direct_dictionary_complex_key_simple_attributes +dictGet existing value +value_0 value_second_0 +value_1 value_second_1 +value_2 value_second_2 +dictGet with non existing value +value_0 value_second_0 +value_1 value_second_1 +value_2 value_second_2 +value_first_default value_second_default +dictGetOrDefault existing value +value_0 value_second_0 +value_1 value_second_1 +value_2 value_second_2 +dictGetOrDefault non existing value +value_0 value_second_0 +value_1 value_second_1 +value_2 value_second_2 +default default +dictHas +1 +1 +1 +0 +select all values as input stream +0 id_key_0 value_0 value_second_0 +1 id_key_1 value_1 value_second_1 +2 id_key_2 value_2 value_second_2 +Dictionary direct_dictionary_complex_key_complex_attributes +dictGet existing value +value_0 value_second_0 +value_1 \N +value_2 value_second_2 +dictGet with non existing value +value_0 value_second_0 +value_1 \N +value_2 value_second_2 +value_first_default value_second_default +dictGetOrDefault existing value +value_0 value_second_0 +value_1 \N +value_2 value_second_2 +dictGetOrDefault non existing value +value_0 value_second_0 +value_1 \N +value_2 value_second_2 +default default +dictHas +1 +1 +1 +0 +select all values as input stream +0 id_key_0 value_0 value_second_0 +1 id_key_1 value_1 \N +2 id_key_2 value_2 value_second_2 diff --git a/tests/queries/0_stateless/01754_direct_dictionary_complex_key.sql b/tests/queries/0_stateless/01754_direct_dictionary_complex_key.sql new file mode 100644 index 00000000000..4dff35aa221 --- /dev/null +++ b/tests/queries/0_stateless/01754_direct_dictionary_complex_key.sql @@ -0,0 +1,95 @@ +DROP DATABASE IF EXISTS 01754_dictionary_db; +CREATE DATABASE 01754_dictionary_db; + +CREATE TABLE 01754_dictionary_db.complex_key_simple_attributes_source_table +( + id UInt64, + id_key String, + value_first String, + value_second String +) +ENGINE = TinyLog; + +INSERT INTO 01754_dictionary_db.complex_key_simple_attributes_source_table VALUES(0, 'id_key_0', 'value_0', 'value_second_0'); +INSERT INTO 01754_dictionary_db.complex_key_simple_attributes_source_table VALUES(1, 'id_key_1', 'value_1', 'value_second_1'); +INSERT INTO 01754_dictionary_db.complex_key_simple_attributes_source_table VALUES(2, 'id_key_2', 'value_2', 'value_second_2'); + +CREATE DICTIONARY 01754_dictionary_db.direct_dictionary_complex_key_simple_attributes +( + id UInt64, + id_key String DEFAULT 'test_default_id_key', + value_first String DEFAULT 'value_first_default', + value_second String DEFAULT 'value_second_default' +) +PRIMARY KEY id, id_key +SOURCE(CLICKHOUSE(HOST 'localhost' PORT tcpPort() USER 'default' TABLE 'complex_key_simple_attributes_source_table')) +LAYOUT(COMPLEX_KEY_DIRECT()); + +SELECT 'Dictionary direct_dictionary_complex_key_simple_attributes'; +SELECT 'dictGet existing value'; +SELECT dictGet('01754_dictionary_db.direct_dictionary_complex_key_simple_attributes', 'value_first', (number, concat('id_key_', toString(number)))) as value_first, + dictGet('01754_dictionary_db.direct_dictionary_complex_key_simple_attributes', 'value_second', (number, concat('id_key_', toString(number)))) as value_second FROM system.numbers LIMIT 3; +SELECT 'dictGet with non existing value'; +SELECT dictGet('01754_dictionary_db.direct_dictionary_complex_key_simple_attributes', 'value_first', (number, concat('id_key_', toString(number)))) as value_first, + dictGet('01754_dictionary_db.direct_dictionary_complex_key_simple_attributes', 'value_second', (number, concat('id_key_', toString(number)))) as value_second FROM system.numbers LIMIT 4; +SELECT 'dictGetOrDefault existing value'; +SELECT dictGetOrDefault('01754_dictionary_db.direct_dictionary_complex_key_simple_attributes', 'value_first', (number, concat('id_key_', toString(number))), toString('default')) as value_first, + dictGetOrDefault('01754_dictionary_db.direct_dictionary_complex_key_simple_attributes', 'value_second', (number, concat('id_key_', toString(number))), toString('default')) as value_second FROM system.numbers LIMIT 3; +SELECT 'dictGetOrDefault non existing value'; +SELECT dictGetOrDefault('01754_dictionary_db.direct_dictionary_complex_key_simple_attributes', 'value_first', (number, concat('id_key_', toString(number))), toString('default')) as value_first, + dictGetOrDefault('01754_dictionary_db.direct_dictionary_complex_key_simple_attributes', 'value_second', (number, concat('id_key_', toString(number))), toString('default')) as value_second FROM system.numbers LIMIT 4; +SELECT 'dictHas'; +SELECT dictHas('01754_dictionary_db.direct_dictionary_complex_key_simple_attributes', (number, concat('id_key_', toString(number)))) FROM system.numbers LIMIT 4; +SELECT 'select all values as input stream'; +SELECT * FROM 01754_dictionary_db.direct_dictionary_complex_key_simple_attributes; + +DROP DICTIONARY 01754_dictionary_db.direct_dictionary_complex_key_simple_attributes; +DROP TABLE 01754_dictionary_db.complex_key_simple_attributes_source_table; + +CREATE TABLE 01754_dictionary_db.complex_key_complex_attributes_source_table +( + id UInt64, + id_key String, + value_first String, + value_second Nullable(String) +) +ENGINE = TinyLog; + +INSERT INTO 01754_dictionary_db.complex_key_complex_attributes_source_table VALUES(0, 'id_key_0', 'value_0', 'value_second_0'); +INSERT INTO 01754_dictionary_db.complex_key_complex_attributes_source_table VALUES(1, 'id_key_1', 'value_1', NULL); +INSERT INTO 01754_dictionary_db.complex_key_complex_attributes_source_table VALUES(2, 'id_key_2', 'value_2', 'value_second_2'); + +CREATE DICTIONARY 01754_dictionary_db.direct_dictionary_complex_key_complex_attributes +( + id UInt64, + id_key String, + + value_first String DEFAULT 'value_first_default', + value_second Nullable(String) DEFAULT 'value_second_default' +) +PRIMARY KEY id, id_key +SOURCE(CLICKHOUSE(HOST 'localhost' PORT tcpPort() USER 'default' TABLE 'complex_key_complex_attributes_source_table')) +LAYOUT(COMPLEX_KEY_DIRECT()); + +SELECT 'Dictionary direct_dictionary_complex_key_complex_attributes'; +SELECT 'dictGet existing value'; +SELECT dictGet('01754_dictionary_db.direct_dictionary_complex_key_complex_attributes', 'value_first', (number, concat('id_key_', toString(number)))) as value_first, + dictGet('01754_dictionary_db.direct_dictionary_complex_key_complex_attributes', 'value_second', (number, concat('id_key_', toString(number)))) as value_second FROM system.numbers LIMIT 3; +SELECT 'dictGet with non existing value'; +SELECT dictGet('01754_dictionary_db.direct_dictionary_complex_key_complex_attributes', 'value_first', (number, concat('id_key_', toString(number)))) as value_first, + dictGet('01754_dictionary_db.direct_dictionary_complex_key_complex_attributes', 'value_second', (number, concat('id_key_', toString(number)))) as value_second FROM system.numbers LIMIT 4; +SELECT 'dictGetOrDefault existing value'; +SELECT dictGetOrDefault('01754_dictionary_db.direct_dictionary_complex_key_complex_attributes', 'value_first', (number, concat('id_key_', toString(number))), toString('default')) as value_first, + dictGetOrDefault('01754_dictionary_db.direct_dictionary_complex_key_complex_attributes', 'value_second', (number, concat('id_key_', toString(number))), toString('default')) as value_second FROM system.numbers LIMIT 3; +SELECT 'dictGetOrDefault non existing value'; +SELECT dictGetOrDefault('01754_dictionary_db.direct_dictionary_complex_key_complex_attributes', 'value_first', (number, concat('id_key_', toString(number))), toString('default')) as value_first, + dictGetOrDefault('01754_dictionary_db.direct_dictionary_complex_key_complex_attributes', 'value_second', (number, concat('id_key_', toString(number))), toString('default')) as value_second FROM system.numbers LIMIT 4; +SELECT 'dictHas'; +SELECT dictHas('01754_dictionary_db.direct_dictionary_complex_key_complex_attributes', (number, concat('id_key_', toString(number)))) FROM system.numbers LIMIT 4; +SELECT 'select all values as input stream'; +SELECT * FROM 01754_dictionary_db.direct_dictionary_complex_key_complex_attributes; + +DROP DICTIONARY 01754_dictionary_db.direct_dictionary_complex_key_complex_attributes; +DROP TABLE 01754_dictionary_db.complex_key_complex_attributes_source_table; + +DROP DATABASE 01754_dictionary_db; \ No newline at end of file From b5f51e0ca6b5e0dc6f95142dccc83a3564db4f8f Mon Sep 17 00:00:00 2001 From: Maksim Kita Date: Mon, 8 Mar 2021 02:02:50 +0300 Subject: [PATCH 200/544] Fixed style check --- src/Dictionaries/DirectDictionary.cpp | 2 +- src/Dictionaries/DirectDictionary.h | 8 +++++++- src/Dictionaries/ya.make | 1 - .../0_stateless/01754_direct_dictionary_complex_key.sql | 2 +- 4 files changed, 9 insertions(+), 4 deletions(-) diff --git a/src/Dictionaries/DirectDictionary.cpp b/src/Dictionaries/DirectDictionary.cpp index f6a8907b29c..c937d337350 100644 --- a/src/Dictionaries/DirectDictionary.cpp +++ b/src/Dictionaries/DirectDictionary.cpp @@ -13,8 +13,8 @@ namespace DB namespace ErrorCodes { extern const int TYPE_MISMATCH; - extern const int BAD_ARGUMENTS; extern const int UNSUPPORTED_METHOD; + extern const int BAD_ARGUMENTS; } namespace diff --git a/src/Dictionaries/DirectDictionary.h b/src/Dictionaries/DirectDictionary.h index 4fa054625e0..685fd707ded 100644 --- a/src/Dictionaries/DirectDictionary.h +++ b/src/Dictionaries/DirectDictionary.h @@ -18,6 +18,11 @@ namespace DB { +namespace ErrorCodes +{ + extern const int BAD_ARGUMENTS; +} + template class DirectDictionary final : public IDictionary { @@ -31,7 +36,8 @@ public: DictionarySourcePtr source_ptr_, BlockPtr saved_block_ = nullptr); - std::string getTypeName() const override { + std::string getTypeName() const override + { if constexpr (dictionary_key_type == DictionaryKeyType::simple) return "Direct"; else diff --git a/src/Dictionaries/ya.make b/src/Dictionaries/ya.make index 97e91a8c213..4df58211118 100644 --- a/src/Dictionaries/ya.make +++ b/src/Dictionaries/ya.make @@ -26,7 +26,6 @@ SRCS( CassandraDictionarySource.cpp CassandraHelpers.cpp ClickHouseDictionarySource.cpp - ComplexKeyDirectDictionary.cpp ComplexKeyHashedDictionary.cpp DictionaryBlockInputStreamBase.cpp DictionaryFactory.cpp diff --git a/tests/queries/0_stateless/01754_direct_dictionary_complex_key.sql b/tests/queries/0_stateless/01754_direct_dictionary_complex_key.sql index 4dff35aa221..51b29457011 100644 --- a/tests/queries/0_stateless/01754_direct_dictionary_complex_key.sql +++ b/tests/queries/0_stateless/01754_direct_dictionary_complex_key.sql @@ -92,4 +92,4 @@ SELECT * FROM 01754_dictionary_db.direct_dictionary_complex_key_complex_attribut DROP DICTIONARY 01754_dictionary_db.direct_dictionary_complex_key_complex_attributes; DROP TABLE 01754_dictionary_db.complex_key_complex_attributes_source_table; -DROP DATABASE 01754_dictionary_db; \ No newline at end of file +DROP DATABASE 01754_dictionary_db; From f9b0fdad1414b91a240b1971d3f265cd825f331d Mon Sep 17 00:00:00 2001 From: Maksim Kita Date: Mon, 8 Mar 2021 16:23:50 +0300 Subject: [PATCH 201/544] Fixed tests --- src/Dictionaries/DirectDictionary.cpp | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/src/Dictionaries/DirectDictionary.cpp b/src/Dictionaries/DirectDictionary.cpp index c937d337350..88a6f5e2d00 100644 --- a/src/Dictionaries/DirectDictionary.cpp +++ b/src/Dictionaries/DirectDictionary.cpp @@ -142,7 +142,7 @@ DirectDictionary::DirectDictionary( } template -void DirectDictionary::toParent(const PaddedPODArray & ids, PaddedPODArray & out) const +void DirectDictionary::toParent(const PaddedPODArray & ids [[maybe_unused]], PaddedPODArray & out [[maybe_unused]]) const { if constexpr (dictionary_key_type == DictionaryKeyType::simple) { @@ -250,7 +250,7 @@ ColumnPtr DirectDictionary::getColumn( const std::string & attribute_name, const DataTypePtr & result_type, const Columns & key_columns, - const DataTypes & key_types, + const DataTypes & key_types [[maybe_unused]], const ColumnPtr & default_values_column) const { if constexpr (dictionary_key_type == DictionaryKeyType::complex) @@ -345,7 +345,7 @@ ColumnPtr DirectDictionary::getColumn( } template -ColumnUInt8::Ptr DirectDictionary::hasKeys(const Columns & key_columns, const DataTypes & key_types) const +ColumnUInt8::Ptr DirectDictionary::hasKeys(const Columns & key_columns, const DataTypes & key_types [[maybe_unused]]) const { if constexpr (dictionary_key_type == DictionaryKeyType::complex) dict_struct.validateKeyTypes(key_types); @@ -416,7 +416,9 @@ ColumnUInt8::Ptr DirectDictionary::hasKeys(const Columns & } template -BlockInputStreamPtr DirectDictionary::getSourceBlockInputStream(const Columns & key_columns, const PaddedPODArray & requested_keys) const +BlockInputStreamPtr DirectDictionary::getSourceBlockInputStream( + const Columns & key_columns [[maybe_unused]], + const PaddedPODArray & requested_keys [[maybe_unused]]) const { size_t requested_keys_size = requested_keys.size(); @@ -460,7 +462,7 @@ void DirectDictionary::setup() { if constexpr (dictionary_key_type == DictionaryKeyType::complex) throw Exception(ErrorCodes::BAD_ARGUMENTS, - "({}): hierachical attributes are not supported for complex key direct dictionary", + "({}): hierarchical attributes are not supported for complex key direct dictionary", full_name); hierarchical_attribute = &attribute; From ed295a934c831e3fd32eb98a1a5a92c8115bd80c Mon Sep 17 00:00:00 2001 From: Maksim Kita Date: Mon, 8 Mar 2021 18:59:42 +0300 Subject: [PATCH 202/544] Fixed tests --- src/Dictionaries/DirectDictionary.cpp | 7 ------- 1 file changed, 7 deletions(-) diff --git a/src/Dictionaries/DirectDictionary.cpp b/src/Dictionaries/DirectDictionary.cpp index 88a6f5e2d00..e9dcc08a0cd 100644 --- a/src/Dictionaries/DirectDictionary.cpp +++ b/src/Dictionaries/DirectDictionary.cpp @@ -334,13 +334,6 @@ ColumnPtr DirectDictionary::getColumn( query_count.fetch_add(requested_keys_size, std::memory_order_relaxed); - Field result_val; - for (size_t i = 0; i < result->size(); ++i) - { - result->get(i, result_val); - std::cerr << "I " << i << " dump " << result_val.dump() << std::endl; - } - return result; } From 5e03418651c8241b9886222c13a8c9d18c9e4ce3 Mon Sep 17 00:00:00 2001 From: Maksim Kita Date: Mon, 8 Mar 2021 21:51:24 +0300 Subject: [PATCH 203/544] Added performance tests --- src/Dictionaries/DirectDictionary.cpp | 110 +++++++++++------------- tests/performance/direct_dictionary.xml | 78 +++++++++++++++++ 2 files changed, 130 insertions(+), 58 deletions(-) create mode 100644 tests/performance/direct_dictionary.xml diff --git a/src/Dictionaries/DirectDictionary.cpp b/src/Dictionaries/DirectDictionary.cpp index e9dcc08a0cd..7934392c0a7 100644 --- a/src/Dictionaries/DirectDictionary.cpp +++ b/src/Dictionaries/DirectDictionary.cpp @@ -6,7 +6,9 @@ #include #include #include -#include +#include +#include + namespace DB { @@ -259,24 +261,21 @@ ColumnPtr DirectDictionary::getColumn( Arena complex_key_arena; const DictionaryAttribute & attribute = dict_struct.getAttribute(attribute_name, result_type); - auto result = attribute.type->createColumn(); - DefaultValueProvider default_value_provider(attribute.null_value, default_values_column); + DictionaryKeysExtractor extractor(key_columns, complex_key_arena); const auto & requested_keys = extractor.getKeys(); + + HashMap key_to_fetched_index; + key_to_fetched_index.reserve(requested_keys.size()); + + auto fetched_from_storage = attribute.type->createColumn(); + size_t fetched_key_index = 0; size_t requested_attribute_index = attribute_index_by_name.find(attribute_name)->second; + Columns block_key_columns; size_t dictionary_keys_size = dict_struct.getKeysNames().size(); - size_t requested_key_index = 0; - Field block_column_value; - - /** In result stream keys are returned in same order as they were requested. - * For example if we request keys [1, 2, 3, 4] but source has only [2, 3] we need to return to client - * [default_value, 2, 3, default_value]. - * For each key fetched from source current algorithm adds default values until - * requested key with requested_key_index match key fetched from source. - * At the end we also need to process tail. - */ + block_key_columns.reserve(dictionary_keys_size); BlockInputStreamPtr stream = getSourceBlockInputStream(key_columns, requested_keys); @@ -284,9 +283,6 @@ ColumnPtr DirectDictionary::getColumn( while (const auto block = stream->read()) { - Columns block_key_columns; - block_key_columns.reserve(dictionary_keys_size); - auto block_columns = block.getColumns(); /// Split into keys columns and attribute columns @@ -301,35 +297,39 @@ ColumnPtr DirectDictionary::getColumn( size_t block_keys_size = block_keys.size(); const auto & block_column = block.safeGetByPosition(dictionary_keys_size + requested_attribute_index).column; + fetched_from_storage->insertRangeFrom(*block_column, 0, block_keys_size); for (size_t block_key_index = 0; block_key_index < block_keys_size; ++block_key_index) { - auto block_key = block_keys[block_key_index]; + const auto & block_key = block_keys[block_key_index]; - while (requested_key_index < requested_keys.size() && - block_key != requested_keys[requested_key_index]) - { - block_column_value = default_value_provider.getDefaultValue(requested_key_index); - result->insert(block_column_value); - ++requested_key_index; - } - - block_column->get(block_key_index, block_column_value); - result->insert(block_column_value); - ++requested_key_index; + key_to_fetched_index[block_key] = fetched_key_index; + ++fetched_key_index; } + + block_key_columns.clear(); } stream->readSuffix(); - size_t requested_keys_size = requested_keys.size(); + Field value_to_insert; - Field default_value; - /// Process tail, if source returned keys less keys sizes than we fetched insert default value for tail - for (; requested_key_index < requested_keys_size; ++requested_key_index) + size_t requested_keys_size = requested_keys.size(); + auto result = fetched_from_storage->cloneEmpty(); + result->reserve(requested_keys_size); + + + for (size_t requested_key_index = 0; requested_key_index < requested_keys_size; ++requested_key_index) { - default_value = default_value_provider.getDefaultValue(requested_key_index); - result->insert(default_value); + const auto requested_key = requested_keys[requested_key_index]; + const auto * it = key_to_fetched_index.find(requested_key); + + if (it) + fetched_from_storage->get(it->getMapped(), value_to_insert); + else + value_to_insert = default_value_provider.getDefaultValue(requested_key_index); + + result->insert(value_to_insert); } query_count.fetch_add(requested_keys_size, std::memory_order_relaxed); @@ -349,17 +349,21 @@ ColumnUInt8::Ptr DirectDictionary::hasKeys(const Columns & const auto & requested_keys = requested_keys_extractor.getKeys(); size_t requested_keys_size = requested_keys.size(); + HashMap requested_key_to_index; + requested_key_to_index.reserve(requested_keys_size); + + for (size_t i = 0; i < requested_keys.size(); ++i) + { + auto requested_key = requested_keys[i]; + requested_key_to_index[requested_key] = i; + } + auto result = ColumnUInt8::create(requested_keys_size, false); auto & result_data = result->getData(); + Columns block_key_columns; size_t dictionary_keys_size = dict_struct.getKeysNames().size(); - size_t requested_key_index = 0; - Field block_column_value; - - /** Algorithm is the same as in getColumn method. There are only 2 details - * 1. We does not process tail because result column is created with false default value. - * 2. If requested key does not match key from source we set false in requested_key_index. - */ + block_key_columns.reserve(dictionary_keys_size); BlockInputStreamPtr stream = getSourceBlockInputStream(key_columns, requested_keys); @@ -369,9 +373,6 @@ ColumnUInt8::Ptr DirectDictionary::hasKeys(const Columns & { auto block_columns = block.getColumns(); - Columns block_key_columns; - block_key_columns.reserve(dictionary_keys_size); - /// Split into keys columns and attribute columns for (size_t i = 0; i < dictionary_keys_size; ++i) { @@ -381,28 +382,21 @@ ColumnUInt8::Ptr DirectDictionary::hasKeys(const Columns & DictionaryKeysExtractor block_keys_extractor(block_key_columns, complex_key_arena); const auto & block_keys = block_keys_extractor.getKeys(); - size_t block_keys_size = block_keys.size(); - for (size_t block_key_index = 0; block_key_index < block_keys_size; ++block_key_index) + for (const auto & block_key : block_keys) { - auto block_key = block_keys[block_key_index]; + const auto * it = requested_key_to_index.find(block_key); + assert(it); - while (requested_key_index < requested_keys.size() && - block_key != requested_keys[requested_key_index]) - { - result_data[requested_key_index] = false; - ++requested_key_index; - } - - result_data[requested_key_index] = true; - ++requested_key_index; + size_t result_data_found_index = it->getMapped(); + result_data[result_data_found_index] = true; } + + block_key_columns.clear(); } stream->readSuffix(); - /// We does not add additional code for tail because result was initialized with false values - query_count.fetch_add(requested_keys_size, std::memory_order_relaxed); return result; diff --git a/tests/performance/direct_dictionary.xml b/tests/performance/direct_dictionary.xml new file mode 100644 index 00000000000..8dc619bab46 --- /dev/null +++ b/tests/performance/direct_dictionary.xml @@ -0,0 +1,78 @@ + + + CREATE TABLE simple_direct_dictionary_test_table + ( + id UInt64, + value_int UInt64, + value_string String, + value_decimal Decimal64(8), + value_string_nullable Nullable(String) + ) ENGINE = TinyLog; + + + + INSERT INTO simple_direct_dictionary_test_table + SELECT number, number, toString(number), toDecimal64(number, 8), toString(number) + FROM system.numbers + LIMIT 100000; + + + + CREATE DICTIONARY simple_direct_dictionary + ( + id UInt64, + value_int UInt64, + value_string String, + value_decimal Decimal64(8), + value_string_nullable Nullable(String) + ) + PRIMARY KEY id + SOURCE(CLICKHOUSE(DB 'default' TABLE 'simple_direct_dictionary_test_table')) + LAYOUT(DIRECT()) + + + + CREATE TABLE complex_direct_dictionary_test_table + ( + id UInt64, + id_key String, + value_int UInt64, + value_string String, + value_decimal Decimal64(8), + value_string_nullable Nullable(String) + ) ENGINE = TinyLog; + + + + INSERT INTO simple_direct_dictionary_test_table + SELECT number, toString(number), number, toString(number), toDecimal64(number, 8), toString(number) + FROM system.numbers + LIMIT 100000; + + + + CREATE DICTIONARY complex_direct_dictionary + ( + id UInt64, + id_key String, + value_int UInt64, + value_string String, + value_decimal Decimal64(8), + value_string_nullable Nullable(String) + ) + PRIMARY KEY id, id_key + SOURCE(CLICKHOUSE(DB 'default' TABLE 'complex_direct_dictionary_test_table')) + LAYOUT(COMPLEX_KEY_DIRECT()) + + + SELECT dictGet('default.simple_direct_dictionary', 'value_int', number) FROM system.numbers LIMIT 150000; + SELECT dictGet('default.simple_direct_dictionary', 'value_string', number) FROM system.numbers LIMIT 150000; + SELECT dictGet('default.simple_direct_dictionary', 'value_decimal', number) FROM system.numbers LIMIT 150000; + SELECT dictGet('default.simple_direct_dictionary', 'value_string_nullable', number) FROM system.numbers LIMIT 150000; + + SELECT dictGet('default.complex_direct_dictionary', 'value_int', (number, toString(number))) FROM system.numbers LIMIT 150000; + SELECT dictGet('default.complex_direct_dictionary', 'value_string', (number, toString(number))) FROM system.numbers LIMIT 150000; + SELECT dictGet('default.complex_direct_dictionary', 'value_decimal', (number, toString(number))) FROM system.numbers LIMIT 150000; + SELECT dictGet('default.complex_direct_dictionary', 'value_string_nullable', (number, toString(number))) FROM system.numbers LIMIT 150000; + + From 69f8fb8c9a68584ac38491cc89ca72e16693255c Mon Sep 17 00:00:00 2001 From: Maksim Kita Date: Tue, 9 Mar 2021 01:34:17 +0300 Subject: [PATCH 204/544] Added dict has in benchmark --- tests/performance/direct_dictionary.xml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tests/performance/direct_dictionary.xml b/tests/performance/direct_dictionary.xml index 8dc619bab46..29947f2a7c8 100644 --- a/tests/performance/direct_dictionary.xml +++ b/tests/performance/direct_dictionary.xml @@ -69,10 +69,12 @@ SELECT dictGet('default.simple_direct_dictionary', 'value_string', number) FROM system.numbers LIMIT 150000; SELECT dictGet('default.simple_direct_dictionary', 'value_decimal', number) FROM system.numbers LIMIT 150000; SELECT dictGet('default.simple_direct_dictionary', 'value_string_nullable', number) FROM system.numbers LIMIT 150000; + SELECT dictHas('default.simple_direct_dictionary', number) FROM system.numbers LIMIT 150000; SELECT dictGet('default.complex_direct_dictionary', 'value_int', (number, toString(number))) FROM system.numbers LIMIT 150000; SELECT dictGet('default.complex_direct_dictionary', 'value_string', (number, toString(number))) FROM system.numbers LIMIT 150000; SELECT dictGet('default.complex_direct_dictionary', 'value_decimal', (number, toString(number))) FROM system.numbers LIMIT 150000; SELECT dictGet('default.complex_direct_dictionary', 'value_string_nullable', (number, toString(number))) FROM system.numbers LIMIT 150000; + SELECT dictHas('default.complex_direct_dictionary', (number, toString(number))) FROM system.numbers LIMIT 150000;
From 12083d0fa1ab4e8e5f3ef2872c56c97affd99921 Mon Sep 17 00:00:00 2001 From: Maksim Kita Date: Tue, 9 Mar 2021 12:29:44 +0300 Subject: [PATCH 205/544] Fixed tests --- tests/performance/direct_dictionary.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/performance/direct_dictionary.xml b/tests/performance/direct_dictionary.xml index 29947f2a7c8..eb1b4e0da00 100644 --- a/tests/performance/direct_dictionary.xml +++ b/tests/performance/direct_dictionary.xml @@ -44,7 +44,7 @@ - INSERT INTO simple_direct_dictionary_test_table + INSERT INTO complex_direct_dictionary_test_table SELECT number, toString(number), number, toString(number), toDecimal64(number, 8), toString(number) FROM system.numbers LIMIT 100000; From 9509516e3c1ba2a2aedc0c569196d657856186b8 Mon Sep 17 00:00:00 2001 From: Maksim Kita Date: Wed, 10 Mar 2021 01:26:06 +0300 Subject: [PATCH 206/544] Fixed tests --- src/Dictionaries/DirectDictionary.cpp | 94 +-------------------------- 1 file changed, 1 insertion(+), 93 deletions(-) diff --git a/src/Dictionaries/DirectDictionary.cpp b/src/Dictionaries/DirectDictionary.cpp index 7934392c0a7..4cb9e0cd629 100644 --- a/src/Dictionaries/DirectDictionary.cpp +++ b/src/Dictionaries/DirectDictionary.cpp @@ -21,6 +21,7 @@ namespace ErrorCodes namespace { + inline UInt64 getAt(const PaddedPODArray & arr, const size_t idx) { return arr[idx]; @@ -31,99 +32,6 @@ namespace return value; } - /// TODO: Use this class from DictionaryHelpers after cache dictionaries pull request will be merged - template - class DictionaryKeysExtractor - { - public: - using KeyType = std::conditional_t; - static_assert(key_type != DictionaryKeyType::range, "Range key type is not supported by DictionaryKeysExtractor"); - - explicit DictionaryKeysExtractor(const Columns & key_columns, Arena & existing_arena) - { - assert(!key_columns.empty()); - - if constexpr (key_type == DictionaryKeyType::simple) - keys = getColumnVectorData(key_columns.front()); - else - keys = deserializeKeyColumnsInArena(key_columns, existing_arena); - } - - - const PaddedPODArray & getKeys() const - { - return keys; - } - - private: - static PaddedPODArray getColumnVectorData(const ColumnPtr column) - { - PaddedPODArray result; - - auto full_column = column->convertToFullColumnIfConst(); - const auto *vector_col = checkAndGetColumn>(full_column.get()); - - if (!vector_col) - throw Exception{ErrorCodes::TYPE_MISMATCH, "Column type mismatch for simple key expected UInt64"}; - - result.assign(vector_col->getData()); - - return result; - } - - static PaddedPODArray deserializeKeyColumnsInArena(const Columns & key_columns, Arena & temporary_arena) - { - size_t keys_size = key_columns.front()->size(); - - PaddedPODArray result; - result.reserve(keys_size); - - PaddedPODArray temporary_column_data(key_columns.size()); - - for (size_t key_index = 0; key_index < keys_size; ++key_index) - { - size_t allocated_size_for_columns = 0; - const char * block_start = nullptr; - - for (size_t column_index = 0; column_index < key_columns.size(); ++column_index) - { - const auto & column = key_columns[column_index]; - temporary_column_data[column_index] = column->serializeValueIntoArena(key_index, temporary_arena, block_start); - allocated_size_for_columns += temporary_column_data[column_index].size; - } - - result.push_back(StringRef{block_start, allocated_size_for_columns}); - } - - return result; - } - - PaddedPODArray keys; - }; - - /// TODO: Use this class from DictionaryHelpers after cache dictionaries pull request will be merged - class DefaultValueProvider final - { - public: - explicit DefaultValueProvider(Field default_value_, ColumnPtr default_values_column_ = nullptr) - : default_value(std::move(default_value_)) - , default_values_column(default_values_column_) - { - } - - - Field getDefaultValue(size_t row) const - { - if (default_values_column) - return (*default_values_column)[row]; - - return default_value; - } - - private: - Field default_value; - ColumnPtr default_values_column; - }; } template From 69f516cd47ae2784e1ae59c44db37ebf9e604730 Mon Sep 17 00:00:00 2001 From: fuwhu Date: Wed, 10 Mar 2021 10:10:25 +0800 Subject: [PATCH 207/544] Refine MergeTreeData::loadDataParts to not parse format version file and detached directory --- src/Storages/MergeTree/MergeTreeData.cpp | 18 +++++++++--------- src/Storages/MergeTree/MergeTreeData.h | 3 +++ 2 files changed, 12 insertions(+), 9 deletions(-) diff --git a/src/Storages/MergeTree/MergeTreeData.cpp b/src/Storages/MergeTree/MergeTreeData.cpp index d9e24581c0c..a8a34523590 100644 --- a/src/Storages/MergeTree/MergeTreeData.cpp +++ b/src/Storages/MergeTree/MergeTreeData.cpp @@ -202,8 +202,8 @@ MergeTreeData::MergeTreeData( for (const auto & [path, disk] : getRelativeDataPathsWithDisks()) { disk->createDirectories(path); - disk->createDirectories(path + "detached"); - auto current_version_file_path = path + "format_version.txt"; + disk->createDirectories(path + MergeTreeData::DETACHED_DIR_NAME); + auto current_version_file_path = path + MergeTreeData::FORMAT_VERSION_FILE_NAME; if (disk->exists(current_version_file_path)) { if (!version_file.first.empty()) @@ -217,7 +217,7 @@ MergeTreeData::MergeTreeData( /// If not choose any if (version_file.first.empty()) - version_file = {relative_data_path + "format_version.txt", getStoragePolicy()->getAnyDisk()}; + version_file = {relative_data_path + MergeTreeData::FORMAT_VERSION_FILE_NAME, getStoragePolicy()->getAnyDisk()}; bool version_file_exists = version_file.second->exists(version_file.first); @@ -725,8 +725,8 @@ void MergeTreeData::loadDataParts(bool skip_sanity_checks) auto disk_ptr = *disk_it; for (auto it = disk_ptr->iterateDirectory(relative_data_path); it->isValid(); it->next()) { - /// Skip temporary directories. - if (startsWith(it->name(), "tmp")) + /// Skip temporary directories, file 'format_version.txt' and directory 'detached'. + if (startsWith(it->name(), "tmp") || it->name() == MergeTreeData::FORMAT_VERSION_FILE_NAME || it->name() == MergeTreeData::DETACHED_DIR_NAME) continue; if (!startsWith(it->name(), MergeTreeWriteAheadLog::WAL_FILE_NAME)) @@ -1318,8 +1318,8 @@ void MergeTreeData::dropIfEmpty() for (const auto & [path, disk] : getRelativeDataPathsWithDisks()) { /// Non recursive, exception is thrown if there are more files. - disk->removeFile(path + "format_version.txt"); - disk->removeDirectory(path + "detached"); + disk->removeFile(path + MergeTreeData::FORMAT_VERSION_FILE_NAME); + disk->removeDirectory(path + MergeTreeData::DETACHED_DIR_NAME); disk->removeDirectory(path); } } @@ -1793,7 +1793,7 @@ void MergeTreeData::changeSettings( { auto disk = new_storage_policy->getDiskByName(disk_name); disk->createDirectories(relative_data_path); - disk->createDirectories(relative_data_path + "detached"); + disk->createDirectories(relative_data_path + MergeTreeData::DETACHED_DIR_NAME); } /// FIXME how would that be done while reloading configuration??? @@ -3064,7 +3064,7 @@ MergeTreeData::getDetachedParts() const for (const auto & [path, disk] : getRelativeDataPathsWithDisks()) { - for (auto it = disk->iterateDirectory(path + "detached"); it->isValid(); it->next()) + for (auto it = disk->iterateDirectory(path + MergeTreeData::DETACHED_DIR_NAME); it->isValid(); it->next()) { res.emplace_back(); auto & part = res.back(); diff --git a/src/Storages/MergeTree/MergeTreeData.h b/src/Storages/MergeTree/MergeTreeData.h index f03f3f1dd8c..8f0b223957a 100644 --- a/src/Storages/MergeTree/MergeTreeData.h +++ b/src/Storages/MergeTree/MergeTreeData.h @@ -116,6 +116,9 @@ public: using DataPartStates = std::initializer_list; using DataPartStateVector = std::vector; + constexpr static auto FORMAT_VERSION_FILE_NAME = "format_version.txt"; + constexpr static auto DETACHED_DIR_NAME = "detached"; + /// Auxiliary structure for index comparison. Keep in mind lifetime of MergeTreePartInfo. struct DataPartStateAndInfo { From 490a7fc2d6f1190971804ac4a3ec64bf0d717612 Mon Sep 17 00:00:00 2001 From: Azat Khuzhin Date: Wed, 10 Mar 2021 08:50:32 +0300 Subject: [PATCH 208/544] Union merge for arcadia_skip_list.txt to avoid frequent conflicts --- .gitattributes | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.gitattributes b/.gitattributes index bcc7d57b904..efb059f169a 100644 --- a/.gitattributes +++ b/.gitattributes @@ -1,2 +1,4 @@ contrib/* linguist-vendored *.h linguist-language=C++ +# to avoid frequent conflicts +tests/queries/0_stateless/arcadia_skip_list.txt text merge=union From 2099e5a661488674d92858d6b88b399f44894b9b Mon Sep 17 00:00:00 2001 From: Azat Khuzhin Date: Wed, 10 Mar 2021 08:50:32 +0300 Subject: [PATCH 209/544] Fix syntax in arcadia_skip_list --- tests/queries/0_stateless/arcadia_skip_list.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/queries/0_stateless/arcadia_skip_list.txt b/tests/queries/0_stateless/arcadia_skip_list.txt index 6926f16e027..1d58f65c055 100644 --- a/tests/queries/0_stateless/arcadia_skip_list.txt +++ b/tests/queries/0_stateless/arcadia_skip_list.txt @@ -212,7 +212,7 @@ 01017_uniqCombined_memory_usage 01747_join_view_filter_dictionary 01748_dictionary_table_dot -00950_dict_get, +00950_dict_get 01683_flat_dictionary 01681_cache_dictionary_simple_key 01682_cache_dictionary_complex_key From 61d40c3600ba6a1c6d6c0cf4919a3cdaebb3a31f Mon Sep 17 00:00:00 2001 From: Azat Khuzhin Date: Tue, 9 Mar 2021 22:00:38 +0300 Subject: [PATCH 210/544] Fix optimize_skip_unused_shards for zero shards case v2: move check to the beginning of the StorageDistributed::read() --- src/Storages/StorageDistributed.cpp | 19 +++++++++++++++++++ ...e_skip_unused_shards_zero_shards.reference | 0 ...ptimize_skip_unused_shards_zero_shards.sql | 2 ++ .../queries/0_stateless/arcadia_skip_list.txt | 1 + 4 files changed, 22 insertions(+) create mode 100644 tests/queries/0_stateless/01759_optimize_skip_unused_shards_zero_shards.reference create mode 100644 tests/queries/0_stateless/01759_optimize_skip_unused_shards_zero_shards.sql diff --git a/src/Storages/StorageDistributed.cpp b/src/Storages/StorageDistributed.cpp index 039cf63eca2..3a33e881611 100644 --- a/src/Storages/StorageDistributed.cpp +++ b/src/Storages/StorageDistributed.cpp @@ -48,6 +48,9 @@ #include #include +#include +#include + #include #include @@ -83,6 +86,7 @@ namespace DB namespace ErrorCodes { + extern const int LOGICAL_ERROR; extern const int NOT_IMPLEMENTED; extern const int STORAGE_REQUIRES_PARAMETER; extern const int BAD_ARGUMENTS; @@ -532,6 +536,17 @@ void StorageDistributed::read( Block header = InterpreterSelectQuery(query_info.query, context, SelectQueryOptions(processed_stage)).getSampleBlock(); + /// Return directly (with correct header) if no shard to query. + if (query_info.cluster->getShardsInfo().empty()) + { + Pipe pipe(std::make_shared(header)); + auto read_from_pipe = std::make_unique(std::move(pipe)); + read_from_pipe->setStepDescription("Read from NullSource (Distributed)"); + query_plan.addStep(std::move(read_from_pipe)); + + return; + } + const Scalars & scalars = context.hasQueryContext() ? context.getQueryContext().getScalars() : Scalars{}; bool has_virtual_shard_num_column = std::find(column_names.begin(), column_names.end(), "_shard_num") != column_names.end(); @@ -546,6 +561,10 @@ void StorageDistributed::read( ClusterProxy::executeQuery(query_plan, select_stream_factory, log, modified_query_ast, context, query_info); + + /// This is a bug, it is possible only when there is no shards to query, and this is handled earlier. + if (!query_plan.isInitialized()) + throw Exception("Pipeline is not initialized", ErrorCodes::LOGICAL_ERROR); } diff --git a/tests/queries/0_stateless/01759_optimize_skip_unused_shards_zero_shards.reference b/tests/queries/0_stateless/01759_optimize_skip_unused_shards_zero_shards.reference new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tests/queries/0_stateless/01759_optimize_skip_unused_shards_zero_shards.sql b/tests/queries/0_stateless/01759_optimize_skip_unused_shards_zero_shards.sql new file mode 100644 index 00000000000..b95d640ca1a --- /dev/null +++ b/tests/queries/0_stateless/01759_optimize_skip_unused_shards_zero_shards.sql @@ -0,0 +1,2 @@ +create table dist_01756 (dummy UInt8) ENGINE = Distributed('test_cluster_two_shards', 'system', 'one', dummy); +select ignore(1), * from dist_01756 where 0 settings optimize_skip_unused_shards=1, force_optimize_skip_unused_shards=1 diff --git a/tests/queries/0_stateless/arcadia_skip_list.txt b/tests/queries/0_stateless/arcadia_skip_list.txt index 6926f16e027..a85ddba4752 100644 --- a/tests/queries/0_stateless/arcadia_skip_list.txt +++ b/tests/queries/0_stateless/arcadia_skip_list.txt @@ -218,3 +218,4 @@ 01682_cache_dictionary_complex_key 01684_ssd_cache_dictionary_simple_key 01685_ssd_cache_dictionary_complex_key +01759_optimize_skip_unused_shards_zero_shards From d7ee9da44b47ff3d7ba14649aab9e6ef26141f28 Mon Sep 17 00:00:00 2001 From: Olga Revyakina Date: Wed, 10 Mar 2021 10:03:03 +0300 Subject: [PATCH 211/544] Mistake corrected --- docs/en/sql-reference/statements/show.md | 4 +--- docs/ru/sql-reference/statements/show.md | 4 +--- 2 files changed, 2 insertions(+), 6 deletions(-) diff --git a/docs/en/sql-reference/statements/show.md b/docs/en/sql-reference/statements/show.md index a3e5afd4417..7b3f709b876 100644 --- a/docs/en/sql-reference/statements/show.md +++ b/docs/en/sql-reference/statements/show.md @@ -440,9 +440,7 @@ SHOW [CHANGED] SETTINGS LIKE|ILIKE **Clauses** -`LIKE` clause needs a setting name or part of the name without globs. - -`ILIKE` clause can contain globs such as `%` or `_`. +`LIKE|ILIKE` allow to specify a matching pattern for the setting name. It can contain globs such as `%` or `_`. `LIKE` clause is case-sensitive, `ILIKE` — case insensitive. When the `CHANGED` clause is used, the query returns only settings changed from their default values. diff --git a/docs/ru/sql-reference/statements/show.md b/docs/ru/sql-reference/statements/show.md index ffe9db337e0..b214f0072e3 100644 --- a/docs/ru/sql-reference/statements/show.md +++ b/docs/ru/sql-reference/statements/show.md @@ -374,9 +374,7 @@ SHOW [CHANGED] SETTINGS LIKE|ILIKE **Секции** -При использовании `LIKE` необходимо указывать имя или часть имени системной настройки без символов подстановки. - -При использовании `ILIKE` имя системной настройки может содержать символы подстановки, такие как `%` или `_`. +При использовании `LIKE|ILIKE` можно задавать шаблон для имени настройки. Этот шаблон может содержать символы подстановки, такие как `%` или `_`. При использовании `LIKE` шаблон чувствителен к регистру, а при использовании `ILIKE` — не чувствителен. Если используется `CHANGED`, запрос вернет только те настройки, значения которых были изменены, т.е. отличны от значений по умолчанию. From 8619665ef23af39d7f9deadd456e7fe2e5ddf6b2 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Wed, 10 Mar 2021 10:04:33 +0300 Subject: [PATCH 212/544] Remove AVX --- base/glibc-compatibility/memcpy/memcpy.h | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/base/glibc-compatibility/memcpy/memcpy.h b/base/glibc-compatibility/memcpy/memcpy.h index 86ffe0c0d53..b651e6c3b62 100644 --- a/base/glibc-compatibility/memcpy/memcpy.h +++ b/base/glibc-compatibility/memcpy/memcpy.h @@ -16,7 +16,7 @@ #endif -extern bool have_avx; +//extern bool have_avx; void init_memcpy(); @@ -47,6 +47,7 @@ tail: *dst = *src; } } +/* #if !defined(MEMORY_SANITIZER) /// Asm code is not instrumented by MSan, skip this branch else if (have_avx) { @@ -134,7 +135,7 @@ tail: goto tail; } } -#endif +#endif*/ else { if (size <= 128) From f5758b8fbf657fb002cdde6e9b9ecbe22510a900 Mon Sep 17 00:00:00 2001 From: Yatsishin Ilya <2159081+qoega@users.noreply.github.com> Date: Wed, 10 Mar 2021 11:36:07 +0300 Subject: [PATCH 213/544] updated submodules to build with CMake 3.19 --- contrib/boringssl | 2 +- contrib/cassandra | 2 +- contrib/googletest | 2 +- contrib/mariadb-connector-c | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/contrib/boringssl b/contrib/boringssl index 8b2bf912ba0..fd9ce1a0406 160000 --- a/contrib/boringssl +++ b/contrib/boringssl @@ -1 +1 @@ -Subproject commit 8b2bf912ba04823cfe9e7e8f5bb60cb7f6252449 +Subproject commit fd9ce1a0406f571507068b9555d0b545b8a18332 diff --git a/contrib/cassandra b/contrib/cassandra index b446d7eb68e..c097fb5c7e6 160000 --- a/contrib/cassandra +++ b/contrib/cassandra @@ -1 +1 @@ -Subproject commit b446d7eb68e6962f431e2b3771313bfe9a2bbd93 +Subproject commit c097fb5c7e63cc430016d9a8b240d8e63fbefa52 diff --git a/contrib/googletest b/contrib/googletest index 356f2d264a4..e7e591764ba 160000 --- a/contrib/googletest +++ b/contrib/googletest @@ -1 +1 @@ -Subproject commit 356f2d264a485db2fcc50ec1c672e0d37b6cb39b +Subproject commit e7e591764baba0a0c3c9ad0014430e7a27331d16 diff --git a/contrib/mariadb-connector-c b/contrib/mariadb-connector-c index 21f451d4d31..f4476ee7311 160000 --- a/contrib/mariadb-connector-c +++ b/contrib/mariadb-connector-c @@ -1 +1 @@ -Subproject commit 21f451d4d3157ffed31ec60a8b76c407190e66bd +Subproject commit f4476ee7311b35b593750f6ae2cbdb62a4006374 From eb0e9726207c6d7b9377a446268a43c6d42ba349 Mon Sep 17 00:00:00 2001 From: Yatsishin Ilya <2159081+qoega@users.noreply.github.com> Date: Wed, 10 Mar 2021 12:01:03 +0300 Subject: [PATCH 214/544] implicit cast to disable warning --- base/readpassphrase/readpassphrase.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/base/readpassphrase/readpassphrase.c b/base/readpassphrase/readpassphrase.c index 9e8097643bb..8a7d3153915 100644 --- a/base/readpassphrase/readpassphrase.c +++ b/base/readpassphrase/readpassphrase.c @@ -94,7 +94,7 @@ restart: if (input != STDIN_FILENO && tcgetattr(input, &oterm) == 0) { memcpy(&term, &oterm, sizeof(term)); if (!(flags & RPP_ECHO_ON)) - term.c_lflag &= ~(ECHO | ECHONL); + term.c_lflag &= ~((unsigned int) (ECHO | ECHONL)); #ifdef VSTATUS if (term.c_cc[VSTATUS] != _POSIX_VDISABLE) term.c_cc[VSTATUS] = _POSIX_VDISABLE; From 592dbb8a512622136aa90049e7d1543c80fb76bc Mon Sep 17 00:00:00 2001 From: George Date: Wed, 10 Mar 2021 12:56:13 +0300 Subject: [PATCH 215/544] Added example --- docs/en/sql-reference/statements/detach.md | 23 ++++++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/docs/en/sql-reference/statements/detach.md b/docs/en/sql-reference/statements/detach.md index 19ff8f10ad3..f3e2f88ffc3 100644 --- a/docs/en/sql-reference/statements/detach.md +++ b/docs/en/sql-reference/statements/detach.md @@ -23,4 +23,27 @@ Note that you can not detach permanently the table which is already detached (te Also you can not [DROP](../../sql-reference/statements/drop.md#drop-table) the detached table, or [CREATE TABLE](../../sql-reference/statements/create/table.md) with the same name as detached permanently, or replace it with the other table with [RENAME TABLE](../../sql-reference/statements/rename.md) query. +**Example** + +Query: + +``` sql +CREATE TABLE test ENGINE = Log AS SELECT * FROM numbers(10); + +DETACH TABLE test; + +SELECT * FROM TEST; +``` + +Result: + +``` text +Ok. + +Ok. + +Received exception from server (version 21.3.1): +Code: 60. DB::Exception: Received from localhost:9000. DB::Exception: Table default.TEST doesn't exist. +``` + [Original article](https://clickhouse.tech/docs/en/sql-reference/statements/detach/) From eb87afc4d43fc7cbb86c92805960322978324d57 Mon Sep 17 00:00:00 2001 From: Pervakov Grigorii Date: Thu, 4 Mar 2021 18:56:55 +0300 Subject: [PATCH 216/544] recreate S3 client if credentials changed --- src/Storages/StorageS3.cpp | 64 ++++++++++++++--------- src/Storages/StorageS3.h | 9 +++- src/Storages/StorageS3Settings.h | 22 +++++--- tests/integration/test_storage_s3/test.py | 36 +++++++++++-- 4 files changed, 96 insertions(+), 35 deletions(-) diff --git a/src/Storages/StorageS3.cpp b/src/Storages/StorageS3.cpp index d0e3b70d900..a31a7fa0944 100644 --- a/src/Storages/StorageS3.cpp +++ b/src/Storages/StorageS3.cpp @@ -203,6 +203,9 @@ StorageS3::StorageS3( const String & compression_method_) : IStorage(table_id_) , uri(uri_) + , access_key_id(access_key_id_) + , secret_access_key(secret_access_key_) + , max_connections(max_connections_) , global_context(context_.getGlobalContext()) , format_name(format_name_) , min_upload_part_size(min_upload_part_size_) @@ -215,29 +218,7 @@ StorageS3::StorageS3( storage_metadata.setColumns(columns_); storage_metadata.setConstraints(constraints_); setInMemoryMetadata(storage_metadata); - - auto settings = context_.getStorageS3Settings().getSettings(uri.uri.toString()); - - Aws::Auth::AWSCredentials credentials(access_key_id_, secret_access_key_); - if (access_key_id_.empty()) - credentials = Aws::Auth::AWSCredentials(std::move(settings.access_key_id), std::move(settings.secret_access_key)); - - S3::PocoHTTPClientConfiguration client_configuration = S3::ClientFactory::instance().createClientConfiguration( - context_.getRemoteHostFilter(), - context_.getGlobalContext().getSettingsRef().s3_max_redirects); - - client_configuration.endpointOverride = uri_.endpoint; - client_configuration.maxConnections = max_connections_; - - client = S3::ClientFactory::instance().create( - client_configuration, - uri_.is_virtual_hosted_style, - credentials.GetAWSAccessKeyId(), - credentials.GetAWSSecretKey(), - settings.server_side_encryption_customer_key_base64, - std::move(settings.headers), - settings.use_environment_credentials.value_or(global_context.getConfigRef().getBool("s3.use_environment_credentials", false)) - ); + updateAuthSettings(context_); } @@ -309,6 +290,8 @@ Pipe StorageS3::read( size_t max_block_size, unsigned num_streams) { + updateAuthSettings(context); + Pipes pipes; bool need_path_column = false; bool need_file_column = false; @@ -342,8 +325,9 @@ Pipe StorageS3::read( return pipe; } -BlockOutputStreamPtr StorageS3::write(const ASTPtr & /*query*/, const StorageMetadataPtr & metadata_snapshot, const Context & /*context*/) +BlockOutputStreamPtr StorageS3::write(const ASTPtr & /*query*/, const StorageMetadataPtr & metadata_snapshot, const Context & context) { + updateAuthSettings(context); return std::make_shared( format_name, metadata_snapshot->getSampleBlock(), @@ -356,6 +340,38 @@ BlockOutputStreamPtr StorageS3::write(const ASTPtr & /*query*/, const StorageMet max_single_part_upload_size); } +void StorageS3::updateAuthSettings(const Context & context) +{ + auto settings = context.getStorageS3Settings().getSettings(uri.uri.toString()); + if (client && (!access_key_id.empty() || settings == auth_settings)) + return; + + Aws::Auth::AWSCredentials credentials(access_key_id, secret_access_key); + HeaderCollection headers; + if (access_key_id.empty()) + { + credentials = Aws::Auth::AWSCredentials(settings.access_key_id, settings.secret_access_key); + headers = settings.headers; + } + + S3::PocoHTTPClientConfiguration client_configuration = S3::ClientFactory::instance().createClientConfiguration( + context.getRemoteHostFilter(), context.getGlobalContext().getSettingsRef().s3_max_redirects); + + client_configuration.endpointOverride = uri.endpoint; + client_configuration.maxConnections = max_connections; + + client = S3::ClientFactory::instance().create( + client_configuration, + uri.is_virtual_hosted_style, + credentials.GetAWSAccessKeyId(), + credentials.GetAWSSecretKey(), + settings.server_side_encryption_customer_key_base64, + std::move(headers), + settings.use_environment_credentials.value_or(global_context.getConfigRef().getBool("s3.use_environment_credentials", false))); + + auth_settings = std::move(settings); +} + void registerStorageS3Impl(const String & name, StorageFactory & factory) { factory.registerStorage(name, [](const StorageFactory::Arguments & args) diff --git a/src/Storages/StorageS3.h b/src/Storages/StorageS3.h index f006de39c99..46d8c9276a2 100644 --- a/src/Storages/StorageS3.h +++ b/src/Storages/StorageS3.h @@ -5,6 +5,7 @@ #if USE_AWS_S3 #include +#include #include #include #include @@ -57,7 +58,10 @@ public: NamesAndTypesList getVirtuals() const override; private: - S3::URI uri; + const S3::URI uri; + const String access_key_id; + const String secret_access_key; + const UInt64 max_connections; const Context & global_context; String format_name; @@ -66,6 +70,9 @@ private: String compression_method; std::shared_ptr client; String name; + S3AuthSettings auth_settings; + + void updateAuthSettings(const Context & context); }; } diff --git a/src/Storages/StorageS3Settings.h b/src/Storages/StorageS3Settings.h index 59b98ebdfdd..29c6c3bb415 100644 --- a/src/Storages/StorageS3Settings.h +++ b/src/Storages/StorageS3Settings.h @@ -14,24 +14,32 @@ class AbstractConfiguration; namespace DB { - struct HttpHeader { - const String name; - const String value; + String name; + String value; + + inline bool operator==(const HttpHeader & other) const { return name == other.name && value == other.value; } }; using HeaderCollection = std::vector; struct S3AuthSettings { - const String access_key_id; - const String secret_access_key; - const String server_side_encryption_customer_key_base64; + String access_key_id; + String secret_access_key; + String server_side_encryption_customer_key_base64; - const HeaderCollection headers; + HeaderCollection headers; std::optional use_environment_credentials; + + inline bool operator==(const S3AuthSettings & other) const + { + return access_key_id == other.access_key_id && secret_access_key == other.secret_access_key + && server_side_encryption_customer_key_base64 == other.server_side_encryption_customer_key_base64 && headers == other.headers + && use_environment_credentials == other.use_environment_credentials; + } }; /// Settings for the StorageS3. diff --git a/tests/integration/test_storage_s3/test.py b/tests/integration/test_storage_s3/test.py index 1f445feb5a0..3b4c56b524b 100644 --- a/tests/integration/test_storage_s3/test.py +++ b/tests/integration/test_storage_s3/test.py @@ -14,6 +14,9 @@ from helpers.cluster import ClickHouseCluster, ClickHouseInstance logging.getLogger().setLevel(logging.INFO) logging.getLogger().addHandler(logging.StreamHandler()) +SCRIPT_DIR = os.path.dirname(os.path.realpath(__file__)) +CONFIG_PATH = os.path.join(SCRIPT_DIR, './_instances/dummy/configs/config.d/defaultS3.xml') + # Creates S3 bucket for tests and allows anonymous read-write access to it. def prepare_s3_bucket(cluster): @@ -85,7 +88,8 @@ def cluster(): cluster.add_instance("restricted_dummy", main_configs=["configs/config_for_test_remote_host_filter.xml"], with_minio=True) cluster.add_instance("dummy", with_minio=True, main_configs=["configs/defaultS3.xml"]) - cluster.add_instance("s3_max_redirects", with_minio=True, main_configs=["configs/defaultS3.xml"], user_configs=["configs/s3_max_redirects.xml"]) + cluster.add_instance("s3_max_redirects", with_minio=True, main_configs=["configs/defaultS3.xml"], + user_configs=["configs/s3_max_redirects.xml"]) logging.info("Starting cluster...") cluster.start() logging.info("Cluster started") @@ -277,9 +281,9 @@ def test_put_get_with_globs(cluster): # Test multipart put. @pytest.mark.parametrize("maybe_auth,positive", [ - ("", True), + ("", True) # ("'minio','minio123',",True), Redirect with credentials not working with nginx. - ("'wrongid','wrongkey',", False) + # ("'wrongid','wrongkey',", False) ClickHouse crashes in some time after this test, local integration tests run fails. ]) def test_multipart_put(cluster, maybe_auth, positive): # type: (ClickHouseCluster) -> None @@ -397,6 +401,16 @@ def run_s3_mock(cluster): logging.info("S3 mock started") +def replace_config(old, new): + config = open(CONFIG_PATH, 'r') + config_lines = config.readlines() + config.close() + config_lines = [line.replace(old, new) for line in config_lines] + config = open(CONFIG_PATH, 'w') + config.writelines(config_lines) + config.close() + + def test_custom_auth_headers(cluster): table_format = "column1 UInt32, column2 UInt32, column3 UInt32" filename = "test.csv" @@ -409,6 +423,22 @@ def test_custom_auth_headers(cluster): result = run_query(instance, get_query) assert result == '1\t2\t3\n' + instance.query( + "CREATE TABLE test ({table_format}) ENGINE = S3('http://resolver:8080/{bucket}/{file}', 'CSV')".format( + bucket=cluster.minio_restricted_bucket, + file=filename, + table_format=table_format + )) + assert run_query(instance, "SELECT * FROM test") == '1\t2\t3\n' + + replace_config("
Authorization: Bearer TOKEN", "
Authorization: Bearer INVALID_TOKEN") + instance.query("SYSTEM RELOAD CONFIG") + ret, err = instance.query_and_get_answer_with_error("SELECT * FROM test") + assert ret == "" and err != "" + replace_config("
Authorization: Bearer INVALID_TOKEN", "
Authorization: Bearer TOKEN") + instance.query("SYSTEM RELOAD CONFIG") + assert run_query(instance, "SELECT * FROM test") == '1\t2\t3\n' + def test_custom_auth_headers_exclusion(cluster): table_format = "column1 UInt32, column2 UInt32, column3 UInt32" From 4c8c45b163388e6825c8620f30e7fc1fdec00272 Mon Sep 17 00:00:00 2001 From: Nikolai Kochetov Date: Wed, 10 Mar 2021 14:00:24 +0300 Subject: [PATCH 217/544] Refactor --- src/Common/ColumnsHashing.h | 55 +++++++++++++++---- src/Interpreters/Aggregator.cpp | 70 +++++++++--------------- src/Interpreters/Aggregator.h | 95 +++++++++++---------------------- 3 files changed, 104 insertions(+), 116 deletions(-) diff --git a/src/Common/ColumnsHashing.h b/src/Common/ColumnsHashing.h index 37cb42a8bda..b7173b25ce5 100644 --- a/src/Common/ColumnsHashing.h +++ b/src/Common/ColumnsHashing.h @@ -486,6 +486,18 @@ struct HashMethodKeysFixed PaddedPODArray prepared_keys; + static bool usePreparedKeys(const Sizes & key_sizes) + { + if (has_low_cardinality || has_nullable_keys || sizeof(Key) > 16) + return false; + + for (auto size : key_sizes) + if (size != 1 && size != 2 && size != 4 && size != 8 && size != 16) + return false; + + return true; + } + HashMethodKeysFixed(const ColumnRawPtrs & key_columns, const Sizes & key_sizes_, const HashMethodContextPtr &) : Base(key_columns), key_sizes(std::move(key_sizes_)), keys_size(key_columns.size()) { @@ -507,19 +519,13 @@ struct HashMethodKeysFixed } } - if constexpr (!has_low_cardinality && !has_nullable_keys && sizeof(Key) <= 16) + if (usePreparedKeys(key_sizes)) { - bool has_unsupported_sizes = false; - for (auto size : key_sizes) - if (size != 1 && size != 2 && size != 4 && size != 8 && size != 16) - has_unsupported_sizes = true; - - if (!has_unsupported_sizes) - packFixedBatch(keys_size, Base::getActualColumns(), key_sizes, prepared_keys); + packFixedBatch(keys_size, Base::getActualColumns(), key_sizes, prepared_keys); } #if defined(__SSSE3__) && !defined(MEMORY_SANITIZER) - if constexpr (!has_low_cardinality && !has_nullable_keys && sizeof(Key) <= 16) + else if constexpr (!has_low_cardinality && !has_nullable_keys && sizeof(Key) <= 16) { /** The task is to "pack" multiple fixed-size fields into single larger Key. * Example: pack UInt8, UInt32, UInt16, UInt64 into UInt128 key: @@ -594,6 +600,37 @@ struct HashMethodKeysFixed return packFixed(row, keys_size, Base::getActualColumns(), key_sizes); } } + + static std::optional shuffleKeyColumns(std::vector & key_columns, const Sizes & key_sizes) + { + if (!usePreparedKeys(key_sizes)) + return {}; + + std::vector new_columns; + new_columns.reserve(key_columns.size()); + + Sizes new_sizes; + auto fill_size = [&](size_t size) + { + for (size_t i = 0; i < key_sizes.size(); ++i) + { + if (key_sizes[i] == size) + { + new_columns.push_back(key_columns[i]); + new_sizes.push_back(size); + } + } + }; + + fill_size(16); + fill_size(8); + fill_size(4); + fill_size(2); + fill_size(1); + + key_columns.swap(new_columns); + return new_sizes; + } }; /** Hash by concatenating serialized key values. diff --git a/src/Interpreters/Aggregator.cpp b/src/Interpreters/Aggregator.cpp index 911352c693c..ea81155e26a 100644 --- a/src/Interpreters/Aggregator.cpp +++ b/src/Interpreters/Aggregator.cpp @@ -1031,10 +1031,15 @@ void Aggregator::convertToBlockImpl( if (key_columns.size() != params.keys_size) throw Exception{"Aggregate. Unexpected key columns size.", ErrorCodes::LOGICAL_ERROR}; + std::vector raw_key_columns; + raw_key_columns.reserve(key_columns.size()); + for (auto & column : key_columns) + raw_key_columns.push_back(column.get()); + if (final) - convertToBlockImplFinal(method, data, key_columns, final_aggregate_columns, arena); + convertToBlockImplFinal(method, data, std::move(raw_key_columns), final_aggregate_columns, arena); else - convertToBlockImplNotFinal(method, data, key_columns, aggregate_columns); + convertToBlockImplNotFinal(method, data, std::move(raw_key_columns), aggregate_columns); /// In order to release memory early. data.clearAndShrink(); } @@ -1112,7 +1117,7 @@ template void NO_INLINE Aggregator::convertToBlockImplFinal( Method & method, Table & data, - MutableColumns & key_columns, + std::vector key_columns, MutableColumns & final_aggregate_columns, Arena * arena) const { @@ -1125,30 +1130,21 @@ void NO_INLINE Aggregator::convertToBlockImplFinal( } } - if constexpr (Method::fixed_keys) + auto shuffled_key_sizes = method.shuffleKeyColumns(key_columns, key_sizes); + const auto & key_sizes_ref = shuffled_key_sizes ? *shuffled_key_sizes : key_sizes; + + data.forEachValue([&](const auto & key, auto & mapped) { - auto new_key_columns = method.shuffleKeyColumns(key_columns, key_sizes); - data.forEachValue([&](const auto & key, auto & mapped) - { - method.insertKeyIntoColumns(key, new_key_columns.first, new_key_columns.second); - insertAggregatesIntoColumns(mapped, final_aggregate_columns, arena); - }); - } - else - { - data.forEachValue([&](const auto & key, auto & mapped) - { - method.insertKeyIntoColumns(key, key_columns, key_sizes); - insertAggregatesIntoColumns(mapped, final_aggregate_columns, arena); - }); - } + method.insertKeyIntoColumns(key, key_columns, key_sizes_ref); + insertAggregatesIntoColumns(mapped, final_aggregate_columns, arena); + }); } template void NO_INLINE Aggregator::convertToBlockImplNotFinal( Method & method, Table & data, - MutableColumns & key_columns, + std::vector key_columns, AggregateColumnsData & aggregate_columns) const { if constexpr (Method::low_cardinality_optimization) @@ -1164,33 +1160,19 @@ void NO_INLINE Aggregator::convertToBlockImplNotFinal( } } - if constexpr (Method::fixed_keys) + auto shuffled_key_sizes = method.shuffleKeyColumns(key_columns, key_sizes); + const auto & key_sizes_ref = shuffled_key_sizes ? *shuffled_key_sizes : key_sizes; + + data.forEachValue([&](const auto & key, auto & mapped) { - auto new_key_columns = method.shuffleKeyColumns(key_columns, key_sizes); - data.forEachValue([&](const auto & key, auto & mapped) - { - method.insertKeyIntoColumns(key, new_key_columns.first, new_key_columns.second); + method.insertKeyIntoColumns(key, key_columns, key_sizes_ref); - /// reserved, so push_back does not throw exceptions - for (size_t i = 0; i < params.aggregates_size; ++i) - aggregate_columns[i]->push_back(mapped + offsets_of_aggregate_states[i]); + /// reserved, so push_back does not throw exceptions + for (size_t i = 0; i < params.aggregates_size; ++i) + aggregate_columns[i]->push_back(mapped + offsets_of_aggregate_states[i]); - mapped = nullptr; - }); - } - else - { - data.forEachValue([&](const auto & key, auto & mapped) - { - method.insertKeyIntoColumns(key, key_columns, key_sizes); - - /// reserved, so push_back does not throw exceptions - for (size_t i = 0; i < params.aggregates_size; ++i) - aggregate_columns[i]->push_back(mapped + offsets_of_aggregate_states[i]); - - mapped = nullptr; - }); - } + mapped = nullptr; + }); } diff --git a/src/Interpreters/Aggregator.h b/src/Interpreters/Aggregator.h index f08b9302082..d24e5478372 100644 --- a/src/Interpreters/Aggregator.h +++ b/src/Interpreters/Aggregator.h @@ -195,13 +195,15 @@ struct AggregationMethodOneNumber /// Use optimization for low cardinality. static const bool low_cardinality_optimization = false; - static constexpr bool fixed_keys = false; + + /// Shuffle key columns before `insertKeyIntoColumns` call if needed. + std::optional shuffleKeyColumns(std::vector &, const Sizes &) { return {}; } // Insert the key from the hash table into columns. - static void insertKeyIntoColumns(const Key & key, MutableColumns & key_columns, const Sizes & /*key_sizes*/) + static void insertKeyIntoColumns(const Key & key, std::vector & key_columns, const Sizes & /*key_sizes*/) { const auto * key_holder = reinterpret_cast(&key); - auto * column = static_cast(key_columns[0].get()); + auto * column = static_cast(key_columns[0]); column->insertRawData(key_holder); } }; @@ -225,11 +227,12 @@ struct AggregationMethodString using State = ColumnsHashing::HashMethodString; static const bool low_cardinality_optimization = false; - static constexpr bool fixed_keys = false; - static void insertKeyIntoColumns(const StringRef & key, MutableColumns & key_columns, const Sizes &) + std::optional shuffleKeyColumns(std::vector &, const Sizes &) { return {}; } + + static void insertKeyIntoColumns(const StringRef & key, std::vector & key_columns, const Sizes &) { - static_cast(key_columns[0].get())->insertData(key.data, key.size); + static_cast(key_columns[0])->insertData(key.data, key.size); } }; @@ -252,11 +255,12 @@ struct AggregationMethodStringNoCache using State = ColumnsHashing::HashMethodString; static const bool low_cardinality_optimization = false; - static constexpr bool fixed_keys = false; - static void insertKeyIntoColumns(const StringRef & key, MutableColumns & key_columns, const Sizes &) + std::optional shuffleKeyColumns(std::vector &, const Sizes &) { return {}; } + + static void insertKeyIntoColumns(const StringRef & key, std::vector & key_columns, const Sizes &) { - static_cast(key_columns[0].get())->insertData(key.data, key.size); + static_cast(key_columns[0])->insertData(key.data, key.size); } }; @@ -279,11 +283,12 @@ struct AggregationMethodFixedString using State = ColumnsHashing::HashMethodFixedString; static const bool low_cardinality_optimization = false; - static constexpr bool fixed_keys = false; - static void insertKeyIntoColumns(const StringRef & key, MutableColumns & key_columns, const Sizes &) + std::optional shuffleKeyColumns(std::vector &, const Sizes &) { return {}; } + + static void insertKeyIntoColumns(const StringRef & key, std::vector & key_columns, const Sizes &) { - static_cast(key_columns[0].get())->insertData(key.data, key.size); + static_cast(key_columns[0])->insertData(key.data, key.size); } }; @@ -305,11 +310,12 @@ struct AggregationMethodFixedStringNoCache using State = ColumnsHashing::HashMethodFixedString; static const bool low_cardinality_optimization = false; - static constexpr bool fixed_keys = false; - static void insertKeyIntoColumns(const StringRef & key, MutableColumns & key_columns, const Sizes &) + std::optional shuffleKeyColumns(std::vector &, const Sizes &) { return {}; } + + static void insertKeyIntoColumns(const StringRef & key, std::vector & key_columns, const Sizes &) { - static_cast(key_columns[0].get())->insertData(key.data, key.size); + static_cast(key_columns[0])->insertData(key.data, key.size); } }; @@ -335,12 +341,13 @@ struct AggregationMethodSingleLowCardinalityColumn : public SingleColumnMethod using State = ColumnsHashing::HashMethodSingleLowCardinalityColumn; static const bool low_cardinality_optimization = true; - static constexpr bool fixed_keys = false; + + std::optional shuffleKeyColumns(std::vector &, const Sizes &) { return {}; } static void insertKeyIntoColumns(const Key & key, - MutableColumns & key_columns_low_cardinality, const Sizes & /*key_sizes*/) + std::vector & key_columns_low_cardinality, const Sizes & /*key_sizes*/) { - auto * col = assert_cast(key_columns_low_cardinality[0].get()); + auto * col = assert_cast(key_columns_low_cardinality[0]); if constexpr (std::is_same_v) { @@ -380,49 +387,10 @@ struct AggregationMethodKeysFixed use_cache>; static const bool low_cardinality_optimization = false; - static constexpr bool fixed_keys = true; - std::pair, Sizes> shuffleKeyColumns(MutableColumns & key_columns, const Sizes & key_sizes) + std::optional shuffleKeyColumns(std::vector & key_columns, const Sizes & key_sizes) { - std::vector new_columns; - new_columns.reserve(key_columns.size()); - - if constexpr (!has_low_cardinality && !has_nullable_keys && sizeof(Key) <= 16) - { - bool has_unsupported_sizes = false; - for (auto size : key_sizes) - if (size != 1 && size != 2 && size != 4 && size != 8 && size != 16) - has_unsupported_sizes = true; - - if (!has_unsupported_sizes) - { - Sizes new_sizes; - auto fill_size = [&](size_t size) - { - for (size_t i = 0; i < key_sizes.size(); ++i) - { - if (key_sizes[i] == size) - { - new_columns.push_back(key_columns[i].get()); - new_sizes.push_back(size); - } - } - }; - - fill_size(16); - fill_size(8); - fill_size(4); - fill_size(2); - fill_size(1); - - return {new_columns, new_sizes}; - } - } - - for (auto & column : key_columns) - new_columns.push_back(column.get()); - - return {new_columns, key_sizes}; + return State::shuffleKeyColumns(key_columns, key_sizes); } static void insertKeyIntoColumns(const Key & key, std::vector & key_columns, const Sizes & key_sizes) @@ -502,9 +470,10 @@ struct AggregationMethodSerialized using State = ColumnsHashing::HashMethodSerialized; static const bool low_cardinality_optimization = false; - static constexpr bool fixed_keys = false; - static void insertKeyIntoColumns(const StringRef & key, MutableColumns & key_columns, const Sizes &) + std::optional shuffleKeyColumns(std::vector &, const Sizes &) { return {}; } + + static void insertKeyIntoColumns(const StringRef & key, std::vector & key_columns, const Sizes &) { const auto * pos = key.data; for (auto & column : key_columns) @@ -1235,7 +1204,7 @@ protected: void convertToBlockImplFinal( Method & method, Table & data, - MutableColumns & key_columns, + std::vector key_columns, MutableColumns & final_aggregate_columns, Arena * arena) const; @@ -1243,7 +1212,7 @@ protected: void convertToBlockImplNotFinal( Method & method, Table & data, - MutableColumns & key_columns, + std::vector key_columns, AggregateColumnsData & aggregate_columns) const; template From deb0b83a12ea4d636931325c7dbb4a13fca96ea0 Mon Sep 17 00:00:00 2001 From: Anton Ivashkin Date: Wed, 10 Mar 2021 14:08:49 +0300 Subject: [PATCH 218/544] Fix refreshState logic --- .../MergeTree/ReplicatedMergeTreeMergeStrategyPicker.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/Storages/MergeTree/ReplicatedMergeTreeMergeStrategyPicker.cpp b/src/Storages/MergeTree/ReplicatedMergeTreeMergeStrategyPicker.cpp index 61c9126ef04..65da6080e86 100644 --- a/src/Storages/MergeTree/ReplicatedMergeTreeMergeStrategyPicker.cpp +++ b/src/Storages/MergeTree/ReplicatedMergeTreeMergeStrategyPicker.cpp @@ -116,8 +116,8 @@ void ReplicatedMergeTreeMergeStrategyPicker::refreshState() auto now = time(nullptr); /// the setting was already enabled, and last state refresh was done recently - if ((execute_merges_on_single_replica_time_threshold != 0 - || s3_execute_merges_on_single_replica_time_threshold != 0) + if (((threshold != 0 && execute_merges_on_single_replica_time_threshold != 0) + || (threshold_s3 != 0 && s3_execute_merges_on_single_replica_time_threshold != 0)) && now - last_refresh_time < REFRESH_STATE_MINIMUM_INTERVAL_SECONDS) return; From 7eb6b87415aa48eb4e428005088e9fa9cfef2e62 Mon Sep 17 00:00:00 2001 From: Nikolai Kochetov Date: Wed, 10 Mar 2021 14:50:51 +0300 Subject: [PATCH 219/544] Added comments. --- src/Interpreters/AggregationCommon.h | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/src/Interpreters/AggregationCommon.h b/src/Interpreters/AggregationCommon.h index 0e9661eaef1..99b0b48329c 100644 --- a/src/Interpreters/AggregationCommon.h +++ b/src/Interpreters/AggregationCommon.h @@ -81,6 +81,12 @@ void fillFixedBatch(size_t num_rows, const T * source, T * dest) } } +/// Move keys of size T into binary blob, starting from offset. +/// It is assumed that offset is aligned to sizeof(T). +/// Example: sizeof(key) = 16, sizeof(T) = 4, offset = 8 +/// out[0] : [--------****----] +/// out[1] : [--------****----] +/// ... template void fillFixedBatch(size_t keys_size, const ColumnRawPtrs & key_columns, const Sizes & key_sizes, PaddedPODArray & out, size_t & offset) { @@ -92,6 +98,8 @@ void fillFixedBatch(size_t keys_size, const ColumnRawPtrs & key_columns, const S size_t num_rows = column->size(); out.resize_fill(num_rows); + /// Note: here we violate strict aliasing. + /// It should be ok as log as we do not reffer to any value from `out` before filling. const char * source = static_cast(column)->getRawDataBegin(); T * dest = reinterpret_cast(reinterpret_cast(out.data()) + offset); fillFixedBatch(num_rows, reinterpret_cast(source), dest); @@ -100,6 +108,8 @@ void fillFixedBatch(size_t keys_size, const ColumnRawPtrs & key_columns, const S } } +/// Pack into a binary blob of type T a set of fixed-size keys. Granted that all the keys fit into the +/// binary blob. Keys are placed starting from the longest one. template void packFixedBatch(size_t keys_size, const ColumnRawPtrs & key_columns, const Sizes & key_sizes, PaddedPODArray & out) { From 6f7800ecb4546e74a380553cee3438c0b316184d Mon Sep 17 00:00:00 2001 From: Anton Ivashkin Date: Wed, 10 Mar 2021 16:27:08 +0300 Subject: [PATCH 220/544] Fix PVS detected errors --- src/Storages/StorageReplicatedMergeTree.cpp | 55 +++++++++++---------- 1 file changed, 30 insertions(+), 25 deletions(-) diff --git a/src/Storages/StorageReplicatedMergeTree.cpp b/src/Storages/StorageReplicatedMergeTree.cpp index feea94c3cd6..a4f6ddd47c7 100644 --- a/src/Storages/StorageReplicatedMergeTree.cpp +++ b/src/Storages/StorageReplicatedMergeTree.cpp @@ -3466,6 +3466,7 @@ void StorageReplicatedMergeTree::updateQuorum(const String & part_name, bool is_ } } + void StorageReplicatedMergeTree::cleanLastPartNode(const String & partition_id) { auto zookeeper = getZooKeeper(); @@ -3517,12 +3518,14 @@ void StorageReplicatedMergeTree::cleanLastPartNode(const String & partition_id) } } + bool StorageReplicatedMergeTree::partIsInsertingWithParallelQuorum(const MergeTreePartInfo & part_info) const { auto zookeeper = getZooKeeper(); return zookeeper->exists(zookeeper_path + "/quorum/parallel/" + part_info.getPartName()); } + bool StorageReplicatedMergeTree::partIsLastQuorumPart(const MergeTreePartInfo & part_info) const { auto zookeeper = getZooKeeper(); @@ -3544,6 +3547,7 @@ bool StorageReplicatedMergeTree::partIsLastQuorumPart(const MergeTreePartInfo & return partition_it->second == part_info.getPartName(); } + bool StorageReplicatedMergeTree::fetchPart(const String & part_name, const StorageMetadataPtr & metadata_snapshot, const String & source_replica_path, bool to_detached, size_t quorum, zkutil::ZooKeeper::Ptr zookeeper_) { @@ -3595,7 +3599,6 @@ bool StorageReplicatedMergeTree::fetchPart(const String & part_name, const Stora }; DataPartPtr part_to_clone; - { /// If the desired part is a result of a part mutation, try to find the source part and compare /// its checksums to the checksums of the desired part. If they match, we can just clone the local part. @@ -3630,6 +3633,10 @@ bool StorageReplicatedMergeTree::fetchPart(const String & part_name, const Stora } + ReplicatedMergeTreeAddress address; + ConnectionTimeouts timeouts; + std::pair user_password; + String interserver_scheme; std::function get_part; if (part_to_clone) { @@ -3640,10 +3647,10 @@ bool StorageReplicatedMergeTree::fetchPart(const String & part_name, const Stora } else { - ReplicatedMergeTreeAddress address(zookeeper->get(source_replica_path + "/host")); - auto timeouts = ConnectionTimeouts::getHTTPTimeouts(global_context); - auto user_password = global_context.getInterserverCredentials(); - String interserver_scheme = global_context.getInterserverScheme(); + address.fromString(zookeeper->get(source_replica_path + "/host")); + timeouts = ConnectionTimeouts::getHTTPTimeouts(global_context); + user_password = global_context.getInterserverCredentials(); + interserver_scheme = global_context.getInterserverScheme(); get_part = [&, address, timeouts, user_password, interserver_scheme]() { @@ -3671,8 +3678,8 @@ bool StorageReplicatedMergeTree::fetchPart(const String & part_name, const Stora replaced_parts = checkPartChecksumsAndCommit(transaction, part); /** If a quorum is tracked for this part, you must update it. - * If you do not have time, in case of losing the session, when you restart the server - see the `ReplicatedMergeTreeRestartingThread::updateQuorumIfWeHavePart` method. - */ + * If you do not have time, in case of losing the session, when you restart the server - see the `ReplicatedMergeTreeRestartingThread::updateQuorumIfWeHavePart` method. + */ if (quorum) { /// Check if this quorum insert is parallel or not @@ -3789,26 +3796,24 @@ bool StorageReplicatedMergeTree::fetchExistsPart(const String & part_name, const std::function get_part; + ReplicatedMergeTreeAddress address(zookeeper->get(source_replica_path + "/host")); + auto timeouts = ConnectionTimeouts::getHTTPTimeouts(global_context); + auto user_password = global_context.getInterserverCredentials(); + String interserver_scheme = global_context.getInterserverScheme(); + + get_part = [&, address, timeouts, user_password, interserver_scheme]() { - ReplicatedMergeTreeAddress address(zookeeper->get(source_replica_path + "/host")); - auto timeouts = ConnectionTimeouts::getHTTPTimeouts(global_context); - auto user_password = global_context.getInterserverCredentials(); - String interserver_scheme = global_context.getInterserverScheme(); + if (interserver_scheme != address.scheme) + throw Exception("Interserver schemes are different: '" + interserver_scheme + + "' != '" + address.scheme + "', can't fetch part from " + address.host, + ErrorCodes::INTERSERVER_SCHEME_DOESNT_MATCH); - get_part = [&, address, timeouts, user_password, interserver_scheme]() - { - if (interserver_scheme != address.scheme) - throw Exception("Interserver schemes are different: '" + interserver_scheme - + "' != '" + address.scheme + "', can't fetch part from " + address.host, - ErrorCodes::INTERSERVER_SCHEME_DOESNT_MATCH); - - return fetcher.fetchPart( - metadata_snapshot, part_name, source_replica_path, - address.host, address.replication_port, - timeouts, user_password.first, user_password.second, interserver_scheme, false, "", true, - replaced_disk); - }; - } + return fetcher.fetchPart( + metadata_snapshot, part_name, source_replica_path, + address.host, address.replication_port, + timeouts, user_password.first, user_password.second, interserver_scheme, false, "", true, + replaced_disk); + }; try { From a9032215a68cbf4cdb021f36f41a0d13b035b50d Mon Sep 17 00:00:00 2001 From: alesapin Date: Wed, 10 Mar 2021 16:38:15 +0300 Subject: [PATCH 221/544] Enable ipv6 in NuRaft --- contrib/NuRaft | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/contrib/NuRaft b/contrib/NuRaft index ff9049bcc8e..3d3683e7775 160000 --- a/contrib/NuRaft +++ b/contrib/NuRaft @@ -1 +1 @@ -Subproject commit ff9049bcc8ea6a02276ccdc8629d764e9e5de853 +Subproject commit 3d3683e77753cfe015a05fae95ddf418e19f59e1 From 7b69ef92be99f79fe45d59a304e1a192cb169a30 Mon Sep 17 00:00:00 2001 From: Yatsishin Ilya <2159081+qoega@users.noreply.github.com> Date: Wed, 10 Mar 2021 18:04:34 +0300 Subject: [PATCH 222/544] gcc old cast check failed --- base/common/wide_integer_impl.h | 8 ++++---- src/Compression/tests/gtest_compressionCodec.cpp | 2 +- src/DataTypes/DataTypesDecimal.h | 2 +- src/Functions/DivisionUtils.h | 2 +- 4 files changed, 7 insertions(+), 7 deletions(-) diff --git a/base/common/wide_integer_impl.h b/base/common/wide_integer_impl.h index 56f1f64d1be..5b981326e25 100644 --- a/base/common/wide_integer_impl.h +++ b/base/common/wide_integer_impl.h @@ -249,15 +249,15 @@ struct integer::_impl return; } - const T alpha = t / ((T) max_int); + const T alpha = t / static_cast(max_int); - if (alpha <= ((T) max_int)) + if (alpha <= static_cast(max_int)) self = static_cast(alpha); else // max(double) / 2^64 will surely contain less than 52 precision bits, so speed up computations. set_multiplier(self, alpha); self *= max_int; - self += static_cast(t - alpha * ((T) max_int)); // += b_i + self += static_cast(t - alpha * static_cast(max_int)); // += b_i } constexpr static void wide_integer_from_bultin(integer& self, double rhs) noexcept { @@ -275,7 +275,7 @@ struct integer::_impl "On your system long double has less than 64 precision bits," "which may result in UB when initializing double from int64_t"); - if ((rhs > 0 && rhs < (double) max_int) || (rhs < 0 && rhs > (double) min_int)) + if ((rhs > 0 && rhs < static_cast(max_int)) || (rhs < 0 && rhs > static_cast(min_int))) { self = static_cast(rhs); return; diff --git a/src/Compression/tests/gtest_compressionCodec.cpp b/src/Compression/tests/gtest_compressionCodec.cpp index 3e450352eac..0acd15e48c3 100644 --- a/src/Compression/tests/gtest_compressionCodec.cpp +++ b/src/Compression/tests/gtest_compressionCodec.cpp @@ -749,7 +749,7 @@ auto RandomishGenerator = [](auto i) { using T = decltype(i); double sin_value = sin(static_cast(i * i)) * i; - if (sin_value < std::numeric_limits::lowest() || sin_value > (double) std::numeric_limits::max()) + if (sin_value < std::numeric_limits::lowest() || sin_value > static_cast(std::numeric_limits::max())) return T{}; return T(sin_value); }; diff --git a/src/DataTypes/DataTypesDecimal.h b/src/DataTypes/DataTypesDecimal.h index 0d7a297bdf0..2b708b53be0 100644 --- a/src/DataTypes/DataTypesDecimal.h +++ b/src/DataTypes/DataTypesDecimal.h @@ -212,7 +212,7 @@ convertToDecimalImpl(const typename FromDataType::FieldType & value, UInt32 scal static constexpr Int128 min_int128 = minInt128(); static constexpr Int128 max_int128 = maxInt128(); - if (out <= static_cast(min_int128) || out >= (float) static_cast(max_int128)) + if (out <= static_cast(min_int128) || out >= static_cast(max_int128)) { if constexpr (throw_exception) throw Exception(std::string(ToDataType::family_name) + " convert overflow. Float is out of Decimal range", diff --git a/src/Functions/DivisionUtils.h b/src/Functions/DivisionUtils.h index 63367113066..174ea7ee797 100644 --- a/src/Functions/DivisionUtils.h +++ b/src/Functions/DivisionUtils.h @@ -105,7 +105,7 @@ struct DivideIntegralImpl auto res = checkedDivision(CastA(a), CastB(b)); if constexpr (std::is_floating_point_v) - if (isNaN(res) || res >= (double) std::numeric_limits::max() || res <= std::numeric_limits::lowest()) + if (isNaN(res) || res >= static_cast(std::numeric_limits::max()) || res <= std::numeric_limits::lowest()) throw Exception("Cannot perform integer division, because it will produce infinite or too large number", ErrorCodes::ILLEGAL_DIVISION); From c2760f6388776fe7dc32cb73e243a24432fa61fa Mon Sep 17 00:00:00 2001 From: Yatsishin Ilya <2159081+qoega@users.noreply.github.com> Date: Wed, 10 Mar 2021 18:05:24 +0300 Subject: [PATCH 223/544] readpassphrase mute recursion in macro expansion --- base/readpassphrase/CMakeLists.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/base/readpassphrase/CMakeLists.txt b/base/readpassphrase/CMakeLists.txt index 574130ce6e3..51b12106eca 100644 --- a/base/readpassphrase/CMakeLists.txt +++ b/base/readpassphrase/CMakeLists.txt @@ -4,5 +4,5 @@ add_library(readpassphrase readpassphrase.c) set_target_properties(readpassphrase PROPERTIES LINKER_LANGUAGE C) -target_compile_options(readpassphrase PRIVATE -Wno-unused-result -Wno-reserved-id-macro) +target_compile_options(readpassphrase PRIVATE -Wno-unused-result -Wno-reserved-id-macro -Wno-disabled-macro-expansion) target_include_directories(readpassphrase PUBLIC .) From 68ef0c0b5ba5f9eb4bc8bf8a1fc2862c701940be Mon Sep 17 00:00:00 2001 From: Yatsishin Ilya <2159081+qoega@users.noreply.github.com> Date: Wed, 10 Mar 2021 18:06:32 +0300 Subject: [PATCH 224/544] better linker flag --- docs/en/development/build.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/en/development/build.md b/docs/en/development/build.md index f98329e748f..3181f26800d 100644 --- a/docs/en/development/build.md +++ b/docs/en/development/build.md @@ -170,7 +170,7 @@ $ ./release Normally all tools of the ClickHouse bundle, such as `clickhouse-server`, `clickhouse-client` etc., are linked into a single static executable, `clickhouse`. This executable must be re-linked on every change, which might be slow. Two common ways to improve linking time are to use `lld` linker, and use the 'split' build configuration, which builds a separate binary for every tool, and further splits the code into serveral shared libraries. To enable these tweaks, pass the following flags to `cmake`: ``` --DCMAKE_C_FLAGS="-fuse-ld=lld" -DCMAKE_CXX_FLAGS="-fuse-ld=lld" -DUSE_STATIC_LIBRARIES=0 -DSPLIT_SHARED_LIBRARIES=1 -DCLICKHOUSE_SPLIT_BINARY=1 +-DCMAKE_C_FLAGS="--ld-path=lld" -DCMAKE_CXX_FLAGS="--ld-path=lld" -DUSE_STATIC_LIBRARIES=0 -DSPLIT_SHARED_LIBRARIES=1 -DCLICKHOUSE_SPLIT_BINARY=1 ``` ## You Don’t Have to Build ClickHouse {#you-dont-have-to-build-clickhouse} From 6e545c072ae51eb71ea5b08165f859a8ea3d33f6 Mon Sep 17 00:00:00 2001 From: Yatsishin Ilya <2159081+qoega@users.noreply.github.com> Date: Wed, 10 Mar 2021 18:07:56 +0300 Subject: [PATCH 225/544] deprecated -fuse-ld changed to --ld-path for clang --- cmake/tools.cmake | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/cmake/tools.cmake b/cmake/tools.cmake index cc4046d2469..88c48978518 100644 --- a/cmake/tools.cmake +++ b/cmake/tools.cmake @@ -75,8 +75,13 @@ if (OS_LINUX AND NOT LINKER_NAME) endif () if (LINKER_NAME) - set (CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -fuse-ld=${LINKER_NAME}") - set (CMAKE_SHARED_LINKER_FLAGS "${CMAKE_SHARED_LINKER_FLAGS} -fuse-ld=${LINKER_NAME}") + if (COMPILER_GCC) + set (CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -fuse-ld=${LINKER_NAME}") + set (CMAKE_SHARED_LINKER_FLAGS "${CMAKE_SHARED_LINKER_FLAGS} -fuse-ld=${LINKER_NAME}") + else () + set (CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} --ld-path=${LINKER_NAME}") + set (CMAKE_SHARED_LINKER_FLAGS "${CMAKE_SHARED_LINKER_FLAGS} --ld-path=${LINKER_NAME}") + endif () message(STATUS "Using custom linker by name: ${LINKER_NAME}") endif () From 238a8265f261143b76f29ad89c646d6df3c1962e Mon Sep 17 00:00:00 2001 From: Dmitriy Date: Wed, 10 Mar 2021 20:18:25 +0300 Subject: [PATCH 226/544] Update trace_log.md MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Поправил ссылку на оригинальную статью. --- docs/ru/operations/system-tables/trace_log.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/ru/operations/system-tables/trace_log.md b/docs/ru/operations/system-tables/trace_log.md index 766c73944c9..88f4b29651b 100644 --- a/docs/ru/operations/system-tables/trace_log.md +++ b/docs/ru/operations/system-tables/trace_log.md @@ -50,4 +50,4 @@ trace: [371912858,371912789,371798468,371799717,371801313,3717 size: 5244400 ``` -[Оригинальная статья](https://clickhouse.tech/docs/ru/operations/system_tables/trace_log) +[Оригинальная статья](https://clickhouse.tech/docs/ru/operations/system-tables/trace_log) From 5e887b4c993264f71f062f08a21832e52cff6f09 Mon Sep 17 00:00:00 2001 From: sevirov <72220289+sevirov@users.noreply.github.com> Date: Wed, 10 Mar 2021 20:21:43 +0300 Subject: [PATCH 227/544] Update docs/en/operations/system-tables/trace_log.md Co-authored-by: Anna <42538400+adevyatova@users.noreply.github.com> --- docs/en/operations/system-tables/trace_log.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/en/operations/system-tables/trace_log.md b/docs/en/operations/system-tables/trace_log.md index 8743a6e619c..b3b04795a60 100644 --- a/docs/en/operations/system-tables/trace_log.md +++ b/docs/en/operations/system-tables/trace_log.md @@ -52,5 +52,5 @@ trace: [371912858,371912789,371798468,371799717,371801313,3717 size: 5244400 ``` - [Original article](https://clickhouse.tech/docs/en/operations/system_tables/trace_log) - \ No newline at end of file + [Original article](https://clickhouse.tech/docs/en/operations/system-tables/trace_log) + From 45a6e350d7df0385faa957e132d0f0aea738b43b Mon Sep 17 00:00:00 2001 From: sevirov <72220289+sevirov@users.noreply.github.com> Date: Wed, 10 Mar 2021 20:22:24 +0300 Subject: [PATCH 228/544] Update docs/ru/sql-reference/functions/date-time-functions.md Co-authored-by: Anna <42538400+adevyatova@users.noreply.github.com> --- docs/ru/sql-reference/functions/date-time-functions.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/ru/sql-reference/functions/date-time-functions.md b/docs/ru/sql-reference/functions/date-time-functions.md index a2f3a7c26c8..82a21dca23b 100644 --- a/docs/ru/sql-reference/functions/date-time-functions.md +++ b/docs/ru/sql-reference/functions/date-time-functions.md @@ -541,7 +541,7 @@ date_add(unit, value, date) Запрос: ```sql -select date_add(YEAR, 3, toDate('2018-01-01')); +SELECT date_add(YEAR, 3, toDate('2018-01-01')); ``` Результат: From 8f56ae3691a4c8bcf45fbbd9089541c01044bf6a Mon Sep 17 00:00:00 2001 From: Dmitriy Date: Wed, 10 Mar 2021 20:24:16 +0300 Subject: [PATCH 229/544] Update date-time-functions.md MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Поправил пример. --- docs/en/sql-reference/functions/date-time-functions.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/en/sql-reference/functions/date-time-functions.md b/docs/en/sql-reference/functions/date-time-functions.md index f0f86c1d90e..304371f44eb 100644 --- a/docs/en/sql-reference/functions/date-time-functions.md +++ b/docs/en/sql-reference/functions/date-time-functions.md @@ -493,7 +493,7 @@ Type: [Date](../../sql-reference/data-types/date.md) or [DateTime](../../sql-ref Query: ```sql -select date_add(YEAR, 3, toDate('2018-01-01')); +SELECT date_add(YEAR, 3, toDate('2018-01-01')); ``` Result: From 1040272001c224b8bdc1a8c214c1566f1e406e41 Mon Sep 17 00:00:00 2001 From: sevirov <72220289+sevirov@users.noreply.github.com> Date: Wed, 10 Mar 2021 20:26:13 +0300 Subject: [PATCH 230/544] Update docs/ru/sql-reference/functions/date-time-functions.md Co-authored-by: Anna <42538400+adevyatova@users.noreply.github.com> --- docs/ru/sql-reference/functions/date-time-functions.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/ru/sql-reference/functions/date-time-functions.md b/docs/ru/sql-reference/functions/date-time-functions.md index 82a21dca23b..b00a3ce1b4d 100644 --- a/docs/ru/sql-reference/functions/date-time-functions.md +++ b/docs/ru/sql-reference/functions/date-time-functions.md @@ -815,7 +815,7 @@ SELECT now('Europe/Moscow'); ## yesterday {#yesterday} -Принимает ноль аргументов и возвращает вчерашнюю дату на один из моментов выполнения запроса. +Возвращает вчерашнюю дату на момент выполнения запроса. Делает то же самое, что today() - 1. ## timeSlot {#timeslot} From 525ef8ff4340c7ee6b369d38665719a0bc752ad1 Mon Sep 17 00:00:00 2001 From: sevirov <72220289+sevirov@users.noreply.github.com> Date: Wed, 10 Mar 2021 20:26:28 +0300 Subject: [PATCH 231/544] Update docs/ru/sql-reference/functions/date-time-functions.md Co-authored-by: Anna <42538400+adevyatova@users.noreply.github.com> --- docs/ru/sql-reference/functions/date-time-functions.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/ru/sql-reference/functions/date-time-functions.md b/docs/ru/sql-reference/functions/date-time-functions.md index b00a3ce1b4d..c9cd129c3c9 100644 --- a/docs/ru/sql-reference/functions/date-time-functions.md +++ b/docs/ru/sql-reference/functions/date-time-functions.md @@ -810,7 +810,7 @@ SELECT now('Europe/Moscow'); ## today {#today} -Принимает ноль аргументов и возвращает текущую дату на один из моментов выполнения запроса. +Возвращает текущую дату на момент выполнения запроса. Функция не требует аргументов. То же самое, что toDate(now()) ## yesterday {#yesterday} From 6ca4712c81112fb7ca29b3fb700b5ed0a8d16b14 Mon Sep 17 00:00:00 2001 From: sevirov <72220289+sevirov@users.noreply.github.com> Date: Wed, 10 Mar 2021 20:26:36 +0300 Subject: [PATCH 232/544] Update docs/ru/sql-reference/functions/date-time-functions.md Co-authored-by: Anna <42538400+adevyatova@users.noreply.github.com> --- docs/ru/sql-reference/functions/date-time-functions.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/ru/sql-reference/functions/date-time-functions.md b/docs/ru/sql-reference/functions/date-time-functions.md index c9cd129c3c9..d65c28598bf 100644 --- a/docs/ru/sql-reference/functions/date-time-functions.md +++ b/docs/ru/sql-reference/functions/date-time-functions.md @@ -816,7 +816,7 @@ SELECT now('Europe/Moscow'); ## yesterday {#yesterday} Возвращает вчерашнюю дату на момент выполнения запроса. -Делает то же самое, что today() - 1. +Делает то же самое, что today() - 1. Функция не требует аргументов. ## timeSlot {#timeslot} From 09975e18310bffa098043ccefb7aa53f61d5703d Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Wed, 10 Mar 2021 21:43:02 +0300 Subject: [PATCH 233/544] Add hardware benchmark results from blockchair --- website/benchmark/hardware/index.html | 3 +- .../hardware/results/amd_epyc_7742.json | 54 +++++++++++++++++++ 2 files changed, 56 insertions(+), 1 deletion(-) create mode 100644 website/benchmark/hardware/results/amd_epyc_7742.json diff --git a/website/benchmark/hardware/index.html b/website/benchmark/hardware/index.html index 6e6664af55a..92da6328f0f 100644 --- a/website/benchmark/hardware/index.html +++ b/website/benchmark/hardware/index.html @@ -73,7 +73,8 @@ Results for Digital Ocean are from Zimin Aleksey.
Results for 2x EPYC 7642 w/ 512 GB RAM (192 Cores) + 12X 1TB SSD (RAID6) are from Yiğit Konur and Metehan Çetinkaya of seo.do.
Results for Raspberry Pi and Digital Ocean CPU-optimized are from Fritz Wijaya.
Results for Digitalocean (Storage-intesinve VMs) + (CPU/GP) are from Yiğit Konur and Metehan Çetinkaya of seo.do.
-Results for 2x AMD EPYC 7F72 3.2 Ghz (Total 96 Cores, IBM Cloud's Bare Metal Service) from Yiğit Konur and Metehan Çetinkaya of seo.do. +Results for 2x AMD EPYC 7F72 3.2 Ghz (Total 96 Cores, IBM Cloud's Bare Metal Service) from Yiğit Konur and Metehan Çetinkaya of seo.do.
+Results for 2x AMD EPYC 7742 (128 physical cores, 1 TB DDR4-3200 RAM) from Yedige Davletgaliyev and Nikita Zhavoronkov of blockchair.com.

diff --git a/website/benchmark/hardware/results/amd_epyc_7742.json b/website/benchmark/hardware/results/amd_epyc_7742.json new file mode 100644 index 00000000000..61b76d9e37b --- /dev/null +++ b/website/benchmark/hardware/results/amd_epyc_7742.json @@ -0,0 +1,54 @@ +[ + { + "system": "AMD EPYC 7742", + "system_full": "AMD EPYC 7742, 256 cores, 1 TiB Samsung 16x64GB DDR4-3200 ECC, 2 * Samsung PM1725b 12.8TB (RAID 0)", + "time": "2021-02-23 00:00:00", + "kind": "server", + "result": + [ +[0.001, 0.001, 0.001], +[0.075, 0.080, 0.015], +[0.030, 0.029, 0.025], +[0.052, 0.027, 0.028], +[0.141, 0.104, 0.105], +[0.180, 0.127, 0.129], +[0.023, 0.018, 0.017], +[0.018, 0.016, 0.016], +[0.135, 0.097, 0.096], +[0.146, 0.109, 0.108], +[0.106, 0.070, 0.069], +[0.105, 0.068, 0.070], +[0.180, 0.137, 0.136], +[0.216, 0.171, 0.175], +[0.193, 0.160, 0.156], +[0.148, 0.123, 0.132], +[0.364, 0.317, 0.317], +[0.276, 0.245, 0.240], +[0.636, 0.562, 0.615], +[0.070, 0.018, 0.015], +[0.576, 0.131, 0.125], +[0.595, 0.124, 0.121], +[0.861, 0.596, 0.610], +[1.172, 0.262, 0.276], +[0.152, 0.054, 0.050], +[0.114, 0.046, 0.046], +[0.149, 0.052, 0.051], +[0.536, 0.288, 0.306], +[0.587, 0.207, 0.217], +[0.436, 0.419, 0.406], +[0.178, 0.110, 0.113], +[0.337, 0.165, 0.162], +[1.072, 0.855, 0.882], +[0.908, 0.600, 0.612], +[0.904, 0.604, 0.629], +[0.223, 0.207, 0.216], +[0.151, 0.143, 0.172], +[0.063, 0.061, 0.060], +[0.059, 0.053, 0.054], +[0.321, 0.318, 0.321], +[0.026, 0.021, 0.020], +[0.018, 0.015, 0.016], +[0.005, 0.005, 0.004] + ] + } +] From 5cd69746fdb5cdc7e415fb7d8e040ac1ab3109bf Mon Sep 17 00:00:00 2001 From: Alexander Kuzmenkov Date: Wed, 10 Mar 2021 23:04:30 +0300 Subject: [PATCH 234/544] add an article about ast-based fuzzer --- website/blog/en/2021/fuzzing-clickhouse.md | 151 +++++++++++++++++++++ 1 file changed, 151 insertions(+) create mode 100644 website/blog/en/2021/fuzzing-clickhouse.md diff --git a/website/blog/en/2021/fuzzing-clickhouse.md b/website/blog/en/2021/fuzzing-clickhouse.md new file mode 100644 index 00000000000..0c326a68466 --- /dev/null +++ b/website/blog/en/2021/fuzzing-clickhouse.md @@ -0,0 +1,151 @@ +--- +title: 'Fuzzing ClickHouse' +date: '2021-03-08' +author: '[Alexander Kuzmenkov](https://github.com/akuzm)' +tags: ['fuzzing', 'testing'] +--- + +Testing is a major problem in software development: there is never enough of +it. It becomes especially true in a database management system, whose task is +to interpret a query language that works on the persistent state managed by the +system in a distributed fashion. Each of these three functions is hard enough +to test even in isolation, and it gets much worse when you combine them. As +ClickHouse developers, we know this from experience. Despite a large amount of +automated testing of all kinds we routinely perform as part of our continuous +integration system, new bugs and regressions are creeping in. We are always +looking for the ways to improve our test coverage, and this article will +describe our recent development in this area -- the AST-based query fuzzer. + +A natural form of testing for a SQL DBMS is to create an SQL script describing +the test case, and record its reference result. To test, we run the script and +check that the result matches the reference. This is used in many SQL DBMSes, +and it is the default kind of a test you are expected to write for any +ClickHouse feature or fix. Currently we have [73k lines of SQL tests +alone](https://github.com/ClickHouse/ClickHouse/tree/master/tests/queries/0_stateless), +that reach the [code coverage of +76%](https://clickhouse-test-reports.s3.yandex.net/0/47d684a5c35410201d4dd4f63f3287bf25cdabb7/coverage_report/test_output/index.html). + +This form of testing, where a developer writes a few simplified examples of how +the feature can and cannot be used, is sometimes called "example-based +testing". Sadly, the bugs often appear in various corner cases and intersecion +of features, and it is not practical to enumerate them all by hand. There is a +technique for automating this process, called "property-based testing". It lets +you write more general tests of the form "for all values matching these specs, +the result of some operation on them should match this other spec". For +example, such a test can check that if you add two positive numbers, the result +is greater than both of them. But you don't specify which numbers exactly, only +these properties. Then, the property testing system randomly generates some +examples with particular numbers that match the specification, and checks that +the result also matches its specification. + +Property-based testing is said to be very efficient, but requires some +developer effort and expertise to write the tests in a special way. There is +another well-known testing technique that is in some sense a corner case of +property-based testing, and that doesn't require much developer time. It is +called fuzzing. When you are fuzzing your program, you feed it random inputs +generated according to some grammar, and the property you are checking is that +your program terminates correctly (no segfaults or assertions or other kinds of +program errors). Most often, the grammar of input for fuzzing is simple -- say, +bit flips and additions, or maybe some dictionary. The space of possible inputs +is huge, so to find interesting paths in it, fuzzing software records the code +paths taken by the program under test for a particular input, and focuses on +the inputs that lead to new code paths that were not seen before. It also +employs some techniques for finding interesting constant values, and so on. In +general, fuzzing allows you to find many interesting corner cases in your +program automatically, without much developer involvement. + + +Finding valid SQL queries with bit flips would take a long time, so there are +systems that generate valid SQL queries based on the grammar, such as +[SQLSmith](https://github.com/anse1/sqlsmith). They are succesfully used for +finding bugs in databases. It would be interesting to use such a system for +ClickHouse, but it requires some up-front effort to support the ClickHouse SQL +grammar and functions, which may be different from the standard. Also, such +systems don't use any feedback, so while they are much better than systems with +primitive grammar, they still might have a hard time finding interesting +examples. But we already have a big corpus of human-written interesting SQL +queries -- it's in our regression tests. Maybe we can use them as a base for +fuzzing? We tried to do this, and it turned out to be surprisingly simple and +efficient. + +Consider some SQL query from a regression test. After parsing, it is easy to +mutate the resulting AST (abstract syntax tree, an internal representation of +the parsed query) before execution to introduce random changes into the query. +For strings and arrays, we make random modifications such as inserting a random +character or doubling the string. For numbers, there are well-known Bad Numbers +such as 0, 1, powers of two and nearby, integer limits, `NaN`. `NaN`s proved to +be especially efficient in finding bugs, because you can often have some +alternative branches in your numeric code, but for a `NaN`, both branches hold +(or not) simultaneously, so this leads to nasty effects. + +Another interesting thing we can do is change the arguments to functions and +expressions in the select list. Naturally, all the interesting arguments can be +taken from other test queries. Same goes for changing the tables used in the +queries. When the fuzzer runs in CI, it runs queries from all the SQL tests in +random order, mixing in the parts of query from different tests, so that we can +eventually test all the possible permutations of our features. + +The core implementation of the fuzzer is relatively small, consisting of about +700 lines of C++ code. A prototype was made in a couple of days, but naturally +it took significantly longer to polish it and to start routinely using it in +CI. It is very productive and let us find more than 200 bugs already (see the +label [fuzz](https://github.com/ClickHouse/ClickHouse/labels/fuzz) on GitHub). +Some errors it finds are not very interesting, e.g. wrong error messages when a +type of argument doesn't match. But we also found some serious logic errors or +even memory errors. We fix all the errors we find, even not significant ones, +because this lets us ensure that under normal operation, the fuzzer doesn't +find any errors. This is similar to the approach usually taken with compiler +warnings and other optional diagnostics -- it's better to fix or disable every +single case, so that you can be sure you have no diagnostics if everything is +OK, and it's easy to notice new problems. + +After fixing the majority of pre-existing error, this fuzzer became efficient +for finding errors in new features. Pull requests introducing new features +normally adds an SQL test, and we pay extra attention to the new tests when +fuzzing, generating more permutations for them. Even if the coverage of the +test is not sufficient, there is a good chance that the fuzzer will find the +missing corner cases. So when we see that all the fuzzer runs in different +configurations have failed for a particular pull request, this almost always +means that it introduces a new bug. + +A major factor that makes fuzzing really efficient is that we have a lot of +assertions and other checks of program logic in our code. For debug-only +checks, we use the plain `assert` macro from ``. For checks that are +needed even in release mode, we use an exception with a special code +`LOGICAL_ERROR` that signifies an internal program error. We did some work to +ensure that these errors are distinct from errors caused by the wrong user +actions. A user error reported for a randomly generated query is normal (e.g. +it references some non-existent columns), but when we see an internal program +error, we know that it's definitely a bug, same as an assertion. Of course, +even without assertions, you get some checks for memory errors provided by the +OS (segfaults). Various kinds of sanitizers are also very useful in conjunction +with fuzzing. We run this fuzzer under clang's Address, Memory, +UndefinedBehavior and Thread sanitizers, as we do for most of our tests. + +To see for yourself how it works, you only need the normal ClickHouse client. +Start `clickhouse-client --query-fuzzer-runs=100`, enter any query, and enjoy +the client going crazy and running a hundred of random queries instead. All +queries from the current session become a source for expressions for fuzzing, +so try entering several different queries to get more interesting results. Be +careful not to do this in production! When you do this experiment, you'll soon +notice that the fuzzer tends to generate queries that are too long to run. This +is why for the CI fuzzer runs we have to configure the server to limit query +execution time, memory usage and so on using the corresponding [server +settings](https://clickhouse.tech/docs/en/operations/settings/query-complexity/#:~:text=In%20the%20default%20configuration%20file,query%20within%20a%20single%20server.). +We had a hilarious situation after that: the fuzzer figured out how to remove +the limits by generating a `SET max_execution_time = 0` query, and then +generated a never-ending query and failed. Thankfully we were able to defeat +its cleverness by using [settings +constraints](https://clickhouse.tech/docs/en/operations/settings/constraints-on-settings/). + +The AST-based fuzzer we discussed is only one of the many kinds of fuzzers we +have in ClickHouse. There is a talk (in Russian) [3] by Alexey Milovidov that +explores all the fuzzer in greater detail (in Russian). Another interesting +recent development is application of pivoted query synthesis technique, +implemented in [SQLancer](https://github.com/sqlancer/sqlancer), to ClickHouse. +The authors are going to give [a talk about +this](https://heisenbug-piter.ru/2021/spb/talks/nr1cwknssdodjkqgzsbvh/) soon, +so stay tuned. + +12-08-21 [Alexander Kuzmenkov](https://github.com/akuzm) + From 300932c8726b66505bbcde9ff24efdc0ec568f92 Mon Sep 17 00:00:00 2001 From: Alexander Kuzmenkov <36882414+akuzm@users.noreply.github.com> Date: Wed, 10 Mar 2021 23:16:17 +0300 Subject: [PATCH 235/544] Update fuzzing-clickhouse.md --- website/blog/en/2021/fuzzing-clickhouse.md | 25 +++++++++++----------- 1 file changed, 13 insertions(+), 12 deletions(-) diff --git a/website/blog/en/2021/fuzzing-clickhouse.md b/website/blog/en/2021/fuzzing-clickhouse.md index 0c326a68466..fe859aa0d6a 100644 --- a/website/blog/en/2021/fuzzing-clickhouse.md +++ b/website/blog/en/2021/fuzzing-clickhouse.md @@ -6,7 +6,7 @@ tags: ['fuzzing', 'testing'] --- Testing is a major problem in software development: there is never enough of -it. It becomes especially true in a database management system, whose task is +it. It becomes especially true for a database management system, whose task is to interpret a query language that works on the persistent state managed by the system in a distributed fashion. Each of these three functions is hard enough to test even in isolation, and it gets much worse when you combine them. As @@ -16,9 +16,9 @@ integration system, new bugs and regressions are creeping in. We are always looking for the ways to improve our test coverage, and this article will describe our recent development in this area -- the AST-based query fuzzer. -A natural form of testing for a SQL DBMS is to create an SQL script describing +A natural form of testing for a SQL DBMS is to create a SQL script describing the test case, and record its reference result. To test, we run the script and -check that the result matches the reference. This is used in many SQL DBMSes, +check that the result matches the reference. This is used in many SQL DBMS, and it is the default kind of a test you are expected to write for any ClickHouse feature or fix. Currently we have [73k lines of SQL tests alone](https://github.com/ClickHouse/ClickHouse/tree/master/tests/queries/0_stateless), @@ -27,8 +27,8 @@ that reach the [code coverage of This form of testing, where a developer writes a few simplified examples of how the feature can and cannot be used, is sometimes called "example-based -testing". Sadly, the bugs often appear in various corner cases and intersecion -of features, and it is not practical to enumerate them all by hand. There is a +testing". Sadly, the bugs often appear in various corner cases and intersections +of features, and it is not practical to enumerate all of these cases by hand. There is a technique for automating this process, called "property-based testing". It lets you write more general tests of the form "for all values matching these specs, the result of some operation on them should match this other spec". For @@ -54,9 +54,8 @@ employs some techniques for finding interesting constant values, and so on. In general, fuzzing allows you to find many interesting corner cases in your program automatically, without much developer involvement. - -Finding valid SQL queries with bit flips would take a long time, so there are -systems that generate valid SQL queries based on the grammar, such as +Generating valid SQL queries with bit flips would take a long time, so there are +systems that generate queries based on the SQL grammar, such as [SQLSmith](https://github.com/anse1/sqlsmith). They are succesfully used for finding bugs in databases. It would be interesting to use such a system for ClickHouse, but it requires some up-front effort to support the ClickHouse SQL @@ -78,8 +77,8 @@ be especially efficient in finding bugs, because you can often have some alternative branches in your numeric code, but for a `NaN`, both branches hold (or not) simultaneously, so this leads to nasty effects. -Another interesting thing we can do is change the arguments to functions and -expressions in the select list. Naturally, all the interesting arguments can be +Another interesting thing we can do is change the arguments of functions, or the list of +expressions in `SELECT`. Naturally, all the interesting arguments can be taken from other test queries. Same goes for changing the tables used in the queries. When the fuzzer runs in CI, it runs queries from all the SQL tests in random order, mixing in the parts of query from different tests, so that we can @@ -106,7 +105,9 @@ fuzzing, generating more permutations for them. Even if the coverage of the test is not sufficient, there is a good chance that the fuzzer will find the missing corner cases. So when we see that all the fuzzer runs in different configurations have failed for a particular pull request, this almost always -means that it introduces a new bug. +means that it introduces a new bug. When developing a feature that requires +new grammar, it is also helpful to add fuzzing support for it. I did this for +window functions early in the development, and it helped me find several bugs. A major factor that makes fuzzing really efficient is that we have a lot of assertions and other checks of program logic in our code. For debug-only @@ -139,7 +140,7 @@ its cleverness by using [settings constraints](https://clickhouse.tech/docs/en/operations/settings/constraints-on-settings/). The AST-based fuzzer we discussed is only one of the many kinds of fuzzers we -have in ClickHouse. There is a talk (in Russian) [3] by Alexey Milovidov that +have in ClickHouse. There is a [talk](https://www.youtube.com/watch?v=GbmK84ZwSeI&t=4481s) (in Russian, [slides are here](https://presentations.clickhouse.tech/cpp_siberia_2021/)) by Alexey Milovidov that explores all the fuzzer in greater detail (in Russian). Another interesting recent development is application of pivoted query synthesis technique, implemented in [SQLancer](https://github.com/sqlancer/sqlancer), to ClickHouse. From 14d92edf96d636b902f7df433f995c49e1f3f14d Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Thu, 11 Mar 2021 08:47:41 +0300 Subject: [PATCH 236/544] Maybe GitHub API has changed --- utils/simple-backport/changelog.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/utils/simple-backport/changelog.sh b/utils/simple-backport/changelog.sh index d3d9714cb04..ca2dcfffff0 100755 --- a/utils/simple-backport/changelog.sh +++ b/utils/simple-backport/changelog.sh @@ -39,7 +39,7 @@ function github_download() local file=${2} if ! [ -f "$file" ] then - if ! curl -H "Authorization: token $GITHUB_TOKEN" \ + if ! curl -u "$GITHUB_USER:$GITHUB_TOKEN" \ -sSf "$url" \ > "$file" then From 0468dd4f38ae63c71faa9d8a5b005fc1875dda44 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Thu, 11 Mar 2021 09:22:53 +0300 Subject: [PATCH 237/544] Add changelog for 21.3 --- CHANGELOG.md | 154 +++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 154 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index e2c777b3bcf..5d9b901999d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,157 @@ +## ClickHouse release 21.3 + +### ClickHouse release v21.3, 2021-03-12 + +#### Backward Incompatible Change + +* Now it's not allowed to create MergeTree tables in old syntax with table TTL because it's just ignored. Attach of old tables is still possible. [#20282](https://github.com/ClickHouse/ClickHouse/pull/20282) ([alesapin](https://github.com/alesapin)). +* Now all case-insensitive function names will be lower-cased during query analysis. This is needed for projection query routing. [#20174](https://github.com/ClickHouse/ClickHouse/pull/20174) ([Amos Bird](https://github.com/amosbird)). +* Fix creation of `TTL` in cases, when its expression is a function and it is the same as `ORDER BY` key. Now it's allowed to set custom aggregation to primary key columns in `TTL` with `GROUP BY`. Backward incompatible: For primary key columns, which are not in `GROUP BY` and aren't set explicitly now is applied function `any` instead of `max`, when TTL is expired. Also if you use TTL with `WHERE` or `GROUP BY` you can see exceptions at merges, while making rolling update. [#15450](https://github.com/ClickHouse/ClickHouse/pull/15450) ([Anton Popov](https://github.com/CurtizJ)). + +#### New Feature + +* Add file engine settings: `engine_file_empty_if_not_exists` and `engine_file_truncate_on_insert`. [#20620](https://github.com/ClickHouse/ClickHouse/pull/20620) ([M0r64n](https://github.com/M0r64n)). +* Add aggregate function `deltaSum` for summing the differences between consecutive rows. [#20057](https://github.com/ClickHouse/ClickHouse/pull/20057) ([Russ Frank](https://github.com/rf)). +* New `event_time_microseconds` column in `system.part_log` table. [#20027](https://github.com/ClickHouse/ClickHouse/pull/20027) ([Bharat Nallan](https://github.com/bharatnc)). +* Added `timezoneOffset(datetime)` function which will give the offset from UTC in seconds. This close [#issue:19850](https://github.com/ClickHouse/ClickHouse/issues/19850). [#19962](https://github.com/ClickHouse/ClickHouse/pull/19962) ([keenwolf](https://github.com/keen-wolf)). +* Add setting `insert_shard_id` to support insert data into specific shard from distributed table. [#19961](https://github.com/ClickHouse/ClickHouse/pull/19961) ([flynn](https://github.com/ucasFL)). +* Function `reinterpretAs` updated to support big integers. Fixes [#19691](https://github.com/ClickHouse/ClickHouse/issues/19691). [#19858](https://github.com/ClickHouse/ClickHouse/pull/19858) ([Maksim Kita](https://github.com/kitaisreal)). +* Added Server Side Encryption Customer Keys (the `x-amz-server-side-encryption-customer-(key/md5)` header) support in S3 client. See [the link](https://docs.aws.amazon.com/AmazonS3/latest/dev/ServerSideEncryptionCustomerKeys.html). Closes [#19428](https://github.com/ClickHouse/ClickHouse/issues/19428). [#19748](https://github.com/ClickHouse/ClickHouse/pull/19748) ([Vladimir Chebotarev](https://github.com/excitoon)). +* Added `implicit_key` option for `executable` dictionary source. It allows to avoid printing key for every record if records comes in the same order as the input keys. Implements [#14527](https://github.com/ClickHouse/ClickHouse/issues/14527). [#19677](https://github.com/ClickHouse/ClickHouse/pull/19677) ([Maksim Kita](https://github.com/kitaisreal)). +* Add quota type `query_selects` and `query_inserts`. [#19603](https://github.com/ClickHouse/ClickHouse/pull/19603) ([JackyWoo](https://github.com/JackyWoo)). +* Add function `extractTextFromHTML` [#19600](https://github.com/ClickHouse/ClickHouse/pull/19600) ([zlx19950903](https://github.com/zlx19950903)), ([alexey-milovidov](https://github.com/alexey-milovidov)). +* Tables with `MergeTree*` engine now have two new table-level settings for query concurrency control. Setting `max_concurrent_queries` limits the number of concurrently executed queries which are related to this table. Setting `min_marks_to_honor_max_concurrent_queries` tells to apply previous setting only if query reads at least this number of marks. [#19544](https://github.com/ClickHouse/ClickHouse/pull/19544) ([Amos Bird](https://github.com/amosbird)). +* Added `file` function to read file from user_files directory as a String. This is different from the `file` table function. This implements [#issue:18851](https://github.com/ClickHouse/ClickHouse/issues/18851). [#19204](https://github.com/ClickHouse/ClickHouse/pull/19204) ([keenwolf](https://github.com/keen-wolf)). + +#### Experimental feature + +* Add experimental `Replicated` database engine. It replicates DDL queries across multiple hosts. [#16193](https://github.com/ClickHouse/ClickHouse/pull/16193) ([tavplubix](https://github.com/tavplubix)). +* Introduce experimental support for window functions, enabled with `allow_experimental_functions = 1`. This is a preliminary, alpha-quality implementation that is not suitable for production use and will change in backward-incompatible ways in future releases. Please see [the documentation](https://github.com/ClickHouse/ClickHouse/blob/master/docs/en/sql-reference/window-functions/index.md#experimental-window-functions) for the list of supported features. [#20337](https://github.com/ClickHouse/ClickHouse/pull/20337) ([Alexander Kuzmenkov](https://github.com/akuzm)). +* Add the ability to backup/restore metadata files for DiskS3. [#18377](https://github.com/ClickHouse/ClickHouse/pull/18377) ([Pavel Kovalenko](https://github.com/Jokser)). + +#### Performance Improvement + +* Hedged requests for remote queries. When setting `use_hedged_requests` enabled (off by default), allow to establish many connections with different replicas for query. New connection is enabled in case existent connection(s) with replica(s) were not established within `hedged_connection_timeout` or no data was received within `receive_data_timeout`. Query uses the first connection which send non empty progress packet (or data packet, if `allow_changing_replica_until_first_data_packet`); other connections are cancelled. Queries with `max_parallel_replicas > 1` are supported. [#19291](https://github.com/ClickHouse/ClickHouse/pull/19291) ([Kruglov Pavel](https://github.com/Avogar)). This allows to significantly reduce tail latencies on very large clusters. +* Added support for `PREWHERE` (and enable the corresponding optimization) when tables have row-level security expressions specified. [#19576](https://github.com/ClickHouse/ClickHouse/pull/19576) ([Denis Glazachev](https://github.com/traceon)). +* The setting `distributed_aggregation_memory_efficient` is enabled by default. It will lower memory usage and improve performance of distributed queries. [#20599](https://github.com/ClickHouse/ClickHouse/pull/20599) ([alexey-milovidov](https://github.com/alexey-milovidov)). +* Improve performance of GROUP BY multiple fixed size keys. [#20472](https://github.com/ClickHouse/ClickHouse/pull/20472) ([alexey-milovidov](https://github.com/alexey-milovidov)). +* Improve performance of aggregate functions by more strict aliasing. [#19946](https://github.com/ClickHouse/ClickHouse/pull/19946) ([alexey-milovidov](https://github.com/alexey-milovidov)). +* Speed up reading from `Memory` tables in extreme cases (when reading speed is in order of 50 GB/sec) by simplification of pipeline and (consequently) less lock contention in pipeline scheduling. [#20468](https://github.com/ClickHouse/ClickHouse/pull/20468) ([alexey-milovidov](https://github.com/alexey-milovidov)). +* Partially reimplement HTTP server to make it making less copies of incoming and outgoing data. It gives up to 1.5 performance improvement on inserting long records over HTTP. [#19516](https://github.com/ClickHouse/ClickHouse/pull/19516) ([Ivan](https://github.com/abyss7)). +* Add `compress` setting for `Memory` tables. If it's enabled the table will use less RAM. On some machines and datasets it can also work faster on SELECT, but it is not always the case. This closes [#20093](https://github.com/ClickHouse/ClickHouse/issues/20093). Note: there are reasons why Memory tables can work slower than MergeTree: (1) lack of compression (2) static size of blocks (3) lack of indices and prewhere... [#20168](https://github.com/ClickHouse/ClickHouse/pull/20168) ([alexey-milovidov](https://github.com/alexey-milovidov)). +* Slightly better code in aggregation. [#20978](https://github.com/ClickHouse/ClickHouse/pull/20978) ([alexey-milovidov](https://github.com/alexey-milovidov)). +* Add back `intDiv`/`modulo` specializations for better performance. This fixes [#21293](https://github.com/ClickHouse/ClickHouse/issues/21293) . The regression was introduced in https://github.com/ClickHouse/ClickHouse/pull/18145 . [#21307](https://github.com/ClickHouse/ClickHouse/pull/21307) ([Amos Bird](https://github.com/amosbird)). +* Do not squash blocks too much on INSERT SELECT if inserting into Memory table. In previous versions inefficient data representation was created in Memory table after INSERT SELECT. This closes [#13052](https://github.com/ClickHouse/ClickHouse/issues/13052). [#20169](https://github.com/ClickHouse/ClickHouse/pull/20169) ([alexey-milovidov](https://github.com/alexey-milovidov)). +* Fix at least one case when DataType parser may have exponential complexity (found by fuzzer). This closes [#20096](https://github.com/ClickHouse/ClickHouse/issues/20096). [#20132](https://github.com/ClickHouse/ClickHouse/pull/20132) ([alexey-milovidov](https://github.com/alexey-milovidov)). +* Improved performance of bitmap columns during joins. [#19407](https://github.com/ClickHouse/ClickHouse/pull/19407) ([templarzq](https://github.com/templarzq)). +* Parallelize SELECT with FINAL for single part with level > 0 when `do_not_merge_across_partitions_select_final` setting is 1. [#19375](https://github.com/ClickHouse/ClickHouse/pull/19375) ([Kruglov Pavel](https://github.com/Avogar)). +* Fill only requested columns when querying `system.parts` and `system.parts_columns`. Closes [#19570](https://github.com/ClickHouse/ClickHouse/issues/19570). [#21035](https://github.com/ClickHouse/ClickHouse/pull/21035) ([Anmol Arora](https://github.com/anmolarora)). +* Perform algebraic optimizations of arithmetic expressions inside `avg` aggregate function. close [#20092](https://github.com/ClickHouse/ClickHouse/issues/20092). [#20183](https://github.com/ClickHouse/ClickHouse/pull/20183) ([flynn](https://github.com/ucasFL)). + +#### Improvement + +* Add two settings to delay or throw error during insertion when there are too many inactive parts. This is useful when server fails to clean up parts quickly enough. [#20178](https://github.com/ClickHouse/ClickHouse/pull/20178) ([Amos Bird](https://github.com/amosbird)). +* Provide better compatibility for mysql clients. 1. mysql jdbc 2. mycli. [#21367](https://github.com/ClickHouse/ClickHouse/pull/21367) ([Amos Bird](https://github.com/amosbird)). +* Forbid to drop a column if it's referenced by materialized view. Closes [#21164](https://github.com/ClickHouse/ClickHouse/issues/21164). [#21303](https://github.com/ClickHouse/ClickHouse/pull/21303) ([flynn](https://github.com/ucasFL)). +* MySQL dictionary source will now retry unexpected connection failures (Lost connection to MySQL server during query) which sometimes happen on SSL/TLS connections. [#21237](https://github.com/ClickHouse/ClickHouse/pull/21237) ([Alexander Kazakov](https://github.com/Akazz)). +* Usability improvement: more consistent `DateTime64` parsing: recognize the case when unix timestamp with subsecond resolution is specified as scaled integer (like `1111111111222` instead of `1111111111.222`). This closes [#13194](https://github.com/ClickHouse/ClickHouse/issues/13194). [#21053](https://github.com/ClickHouse/ClickHouse/pull/21053) ([alexey-milovidov](https://github.com/alexey-milovidov)). +* Do only merging of sorted blocks on initiator with distributed_group_by_no_merge. [#20882](https://github.com/ClickHouse/ClickHouse/pull/20882) ([Azat Khuzhin](https://github.com/azat)). +* When loading config for mysql source ClickHouse will now randomize the list of replicas with the same priority to ensure the round-robin logics of picking mysql endpoint. This closes [#20629](https://github.com/ClickHouse/ClickHouse/issues/20629). [#20632](https://github.com/ClickHouse/ClickHouse/pull/20632) ([Alexander Kazakov](https://github.com/Akazz)). +* Function 'reinterpretAs(x, Type)' renamed into 'reinterpret(x, Type)'. [#20611](https://github.com/ClickHouse/ClickHouse/pull/20611) ([Maksim Kita](https://github.com/kitaisreal)). +* Support vhost for RabbitMQ engine [#20576](https://github.com/ClickHouse/ClickHouse/issues/20576). [#20596](https://github.com/ClickHouse/ClickHouse/pull/20596) ([Kseniia Sumarokova](https://github.com/kssenii)). +* Improved serialization for data types combined of Arrays and Tuples. Improved matching enum data types to protobuf enum type. Fixed serialization of the `Map` data type. Omitted values are now set by default. [#20506](https://github.com/ClickHouse/ClickHouse/pull/20506) ([Vitaly Baranov](https://github.com/vitlibar)). +* Fixed race between execution of distributed DDL tasks and cleanup of DDL queue. Now DDL task cannot be removed from ZooKeeper if there are active workers. Fixes [#20016](https://github.com/ClickHouse/ClickHouse/issues/20016). [#20448](https://github.com/ClickHouse/ClickHouse/pull/20448) ([tavplubix](https://github.com/tavplubix)). +* Make FQDN and other DNS related functions work correctly in alpine images. [#20336](https://github.com/ClickHouse/ClickHouse/pull/20336) ([filimonov](https://github.com/filimonov)). +* Do not allow early constant folding of explicitly forbidden functions. [#20303](https://github.com/ClickHouse/ClickHouse/pull/20303) ([Azat Khuzhin](https://github.com/azat)). +* Implicit conversion from integer to Decimal type might succeeded if integer value doe not fit into Decimal type. Now it throws `ARGUMENT_OUT_OF_BOUND`. [#20232](https://github.com/ClickHouse/ClickHouse/pull/20232) ([tavplubix](https://github.com/tavplubix)). +* Lockless `SYSTEM FLUSH DISTRIBUTED`. [#20215](https://github.com/ClickHouse/ClickHouse/pull/20215) ([Azat Khuzhin](https://github.com/azat)). +* Normalize count(constant), sum(1) to count(). This is needed for projection query routing. [#20175](https://github.com/ClickHouse/ClickHouse/pull/20175) ([Amos Bird](https://github.com/amosbird)). +* Support all native integer types in bitmap functions. [#20171](https://github.com/ClickHouse/ClickHouse/pull/20171) ([Amos Bird](https://github.com/amosbird)). +* Updated `CacheDictionary`, `ComplexCacheDictionary`, `SSDCacheDictionary`, `SSDComplexKeyDictionary` to use LRUHashMap as underlying index. [#20164](https://github.com/ClickHouse/ClickHouse/pull/20164) ([Maksim Kita](https://github.com/kitaisreal)). +* The setting `access_management` is now configurable on startup by providing `CLICKHOUSE_DEFAULT_ACCESS_MANAGEMENT`, defaults to disabled (`0`) which was the prior value. [#20139](https://github.com/ClickHouse/ClickHouse/pull/20139) ([Marquitos](https://github.com/sonirico)). +* Fix toDateTime64(toDate()/toDateTime()) for DateTime64 - Implement DateTime64 clamping to match DateTime behaviour. [#20131](https://github.com/ClickHouse/ClickHouse/pull/20131) ([Azat Khuzhin](https://github.com/azat)). +* Quota improvements: SHOW TABLES is now considered as one query in the quota calculations, not two queries. SYSTEM queries now consume quota. Fix calculation of interval's end in quota consumption. [#20106](https://github.com/ClickHouse/ClickHouse/pull/20106) ([Vitaly Baranov](https://github.com/vitlibar)). +* Supports `path IN (set)` expressions for `system.zookeeper` table. [#20105](https://github.com/ClickHouse/ClickHouse/pull/20105) ([小路](https://github.com/nicelulu)). +* Show full details of `MaterializeMySQL` tables in `system.tables`. [#20051](https://github.com/ClickHouse/ClickHouse/pull/20051) ([Stig Bakken](https://github.com/stigsb)). +* Fix data race in executable dictionary that was possible only on misuse (when the script returns data ignoring its input). [#20045](https://github.com/ClickHouse/ClickHouse/pull/20045) ([alexey-milovidov](https://github.com/alexey-milovidov)). +* The value of MYSQL_OPT_RECONNECT option can now be controlled by "opt_reconnect" parameter in the config section of mysql replica. [#19998](https://github.com/ClickHouse/ClickHouse/pull/19998) ([Alexander Kazakov](https://github.com/Akazz)). +* If user calls `JSONExtract` function with `Float32` type requested, allow inaccurate conversion to the result type. For example the number `0.1` in JSON is double precision and is not representable in Float32, but the user still wants to get it. Previous versions return 0 for non-Nullable type and NULL for Nullable type to indicate that conversion is imprecise. The logic was 100% correct but it was surprising to users and leading to questions. This closes [#13962](https://github.com/ClickHouse/ClickHouse/issues/13962). [#19960](https://github.com/ClickHouse/ClickHouse/pull/19960) ([alexey-milovidov](https://github.com/alexey-milovidov)). +* Add conversion of block structure for INSERT into Distributed tables if it does not match. [#19947](https://github.com/ClickHouse/ClickHouse/pull/19947) ([Azat Khuzhin](https://github.com/azat)). +* Improvement for the `system.distributed_ddl_queue` table. Initialize MaxDDLEntryID to the last value after restarting. Before this PR, MaxDDLEntryID will remain zero until a new DDLTask is processed. [#19924](https://github.com/ClickHouse/ClickHouse/pull/19924) ([Amos Bird](https://github.com/amosbird)). +* Show `MaterializeMySQL` tables in `system.parts`. [#19770](https://github.com/ClickHouse/ClickHouse/pull/19770) ([Stig Bakken](https://github.com/stigsb)). +* Add separate config directive for `Buffer` profile. [#19721](https://github.com/ClickHouse/ClickHouse/pull/19721) ([Azat Khuzhin](https://github.com/azat)). +* Move conditions that are not related to JOIN to WHERE clause. [#18720](https://github.com/ClickHouse/ClickHouse/issues/18720). [#19685](https://github.com/ClickHouse/ClickHouse/pull/19685) ([hexiaoting](https://github.com/hexiaoting)). +* Add ability to throttle INSERT into Distributed based on amount of pending bytes for async send (`bytes_to_delay_insert`/`max_delay_to_insert` and `bytes_to_throw_insert` settings for `Distributed` engine has been added). [#19673](https://github.com/ClickHouse/ClickHouse/pull/19673) ([Azat Khuzhin](https://github.com/azat)). +* Fix some rare cases when write errors can be ignored in destructors. [#19451](https://github.com/ClickHouse/ClickHouse/pull/19451) ([Azat Khuzhin](https://github.com/azat)). +* Print inline frames in stack traces for fatal errors. [#19317](https://github.com/ClickHouse/ClickHouse/pull/19317) ([Ivan](https://github.com/abyss7)). + +#### Bug Fix + +* Fix a deadlock in `ALTER DELETE` mutations for non replicated MergeTree table engines when the predicate contains the table itself. Fixes [#20558](https://github.com/ClickHouse/ClickHouse/issues/20558). [#21477](https://github.com/ClickHouse/ClickHouse/pull/21477) ([alesapin](https://github.com/alesapin)). +* Fix SIGSEGV for distributed queries on failures. [#21434](https://github.com/ClickHouse/ClickHouse/pull/21434) ([Azat Khuzhin](https://github.com/azat)). +* Now `ALTER MODIFY COLUMN` queries will correctly affect changes in partition key, skip indices, TTLs, and so on. Fixes [#13675](https://github.com/ClickHouse/ClickHouse/issues/13675). [#21334](https://github.com/ClickHouse/ClickHouse/pull/21334) ([alesapin](https://github.com/alesapin)). +* Fix bug with `join_use_nulls` and joining `TOTALS` from subqueries. This closes [#19362](https://github.com/ClickHouse/ClickHouse/issues/19362) and [#21137](https://github.com/ClickHouse/ClickHouse/issues/21137). [#21248](https://github.com/ClickHouse/ClickHouse/pull/21248) ([vdimir](https://github.com/vdimir)). +* Fix crash in `EXPLAIN` for query with `UNION`. Fixes [#20876](https://github.com/ClickHouse/ClickHouse/issues/20876), [#21170](https://github.com/ClickHouse/ClickHouse/issues/21170). [#21246](https://github.com/ClickHouse/ClickHouse/pull/21246) ([flynn](https://github.com/ucasFL)). +* Now mutations allowed only for table engines that support them (MergeTree family, Memory, MaterializedView). Other engines will report a more clear error. Fixes [#21168](https://github.com/ClickHouse/ClickHouse/issues/21168). [#21183](https://github.com/ClickHouse/ClickHouse/pull/21183) ([alesapin](https://github.com/alesapin)). +* Fixes [#21112](https://github.com/ClickHouse/ClickHouse/issues/21112). Fixed bug that could cause duplicates with insert query (if one of the callbacks came a little too late). [#21138](https://github.com/ClickHouse/ClickHouse/pull/21138) ([Kseniia Sumarokova](https://github.com/kssenii)). +* Fix `input_format_null_as_default` take effective when types are nullable. This fixes [#21116](https://github.com/ClickHouse/ClickHouse/issues/21116) . [#21121](https://github.com/ClickHouse/ClickHouse/pull/21121) ([Amos Bird](https://github.com/amosbird)). +* fix bug related to cast Tuple to Map. Closes [#21029](https://github.com/ClickHouse/ClickHouse/issues/21029). [#21120](https://github.com/ClickHouse/ClickHouse/pull/21120) ([hexiaoting](https://github.com/hexiaoting)). +* Fix the metadata leak when the Replicated*MergeTree with custom (non default) ZooKeeper cluster is dropped. [#21119](https://github.com/ClickHouse/ClickHouse/pull/21119) ([fastio](https://github.com/fastio)). +* Fix type mismatch issue when using LowCardinality keys in joinGet. This fixes [#21114](https://github.com/ClickHouse/ClickHouse/issues/21114). [#21117](https://github.com/ClickHouse/ClickHouse/pull/21117) ([Amos Bird](https://github.com/amosbird)). +* fix default_replica_path and default_replica_name values are useless on Replicated(*)MergeTree engine when the engine needs specify other parameters. [#21060](https://github.com/ClickHouse/ClickHouse/pull/21060) ([mxzlxy](https://github.com/mxzlxy)). +* Out of bound memory access was possible when formatting specifically crafted out of range value of type `DateTime64`. This closes [#20494](https://github.com/ClickHouse/ClickHouse/issues/20494). This closes [#20543](https://github.com/ClickHouse/ClickHouse/issues/20543). [#21023](https://github.com/ClickHouse/ClickHouse/pull/21023) ([alexey-milovidov](https://github.com/alexey-milovidov)). +* Block parallel insertions into storage join. [#21009](https://github.com/ClickHouse/ClickHouse/pull/21009) ([vdimir](https://github.com/vdimir)). +* Fixed behaviour, when `ALTER MODIFY COLUMN` created mutation, that will knowingly fail. [#21007](https://github.com/ClickHouse/ClickHouse/pull/21007) ([Anton Popov](https://github.com/CurtizJ)). +* Closes [#9969](https://github.com/ClickHouse/ClickHouse/issues/9969). Fixed Brotli http compression error, which reproduced for large data sizes, slightly complicated structure and with json output format. Update Brotli to the latest version to include the "fix rare access to uninitialized data in ring-buffer". [#20991](https://github.com/ClickHouse/ClickHouse/pull/20991) ([Kseniia Sumarokova](https://github.com/kssenii)). +* Fix 'Empty task was returned from async task queue' on query cancellation. [#20881](https://github.com/ClickHouse/ClickHouse/pull/20881) ([Azat Khuzhin](https://github.com/azat)). +* `USE database;` query did not work when using MySQL 5.7 client to connect to ClickHouse server, it's fixed. Fixes [#18926](https://github.com/ClickHouse/ClickHouse/issues/18926). [#20878](https://github.com/ClickHouse/ClickHouse/pull/20878) ([tavplubix](https://github.com/tavplubix)). +* Fix usage of `-Distinct` combinator with `-State` combinator in aggregate functions. [#20866](https://github.com/ClickHouse/ClickHouse/pull/20866) ([Anton Popov](https://github.com/CurtizJ)). +* Fix subquery with union distinct and limit clause. close [#20597](https://github.com/ClickHouse/ClickHouse/issues/20597). [#20610](https://github.com/ClickHouse/ClickHouse/pull/20610) ([flynn](https://github.com/ucasFL)). +* Fixed inconsistent behavior of dictionary in case of queries where we look for absent keys in dictionary. [#20578](https://github.com/ClickHouse/ClickHouse/pull/20578) ([Nikita Mikhaylov](https://github.com/nikitamikhaylov)). +* Fix the number of threads for scalar subqueries and subqueries for index (after [#19007](https://github.com/ClickHouse/ClickHouse/issues/19007) single thread was always used). Fixes [#20457](https://github.com/ClickHouse/ClickHouse/issues/20457), [#20512](https://github.com/ClickHouse/ClickHouse/issues/20512). [#20550](https://github.com/ClickHouse/ClickHouse/pull/20550) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* Fix crash which could happen if unknown packet was received from remove query (was introduced in [#17868](https://github.com/ClickHouse/ClickHouse/issues/17868)). [#20547](https://github.com/ClickHouse/ClickHouse/pull/20547) ([Azat Khuzhin](https://github.com/azat)). +* Add proper checks while parsing directory names for async INSERT (fixes SIGSEGV). [#20498](https://github.com/ClickHouse/ClickHouse/pull/20498) ([Azat Khuzhin](https://github.com/azat)). +* Fix function `transform` does not work properly for floating point keys. Closes [#20460](https://github.com/ClickHouse/ClickHouse/issues/20460). [#20479](https://github.com/ClickHouse/ClickHouse/pull/20479) ([flynn](https://github.com/ucasFL)). +* Fix infinite loop when propagating WITH aliases to subqueries. This fixes [#20388](https://github.com/ClickHouse/ClickHouse/issues/20388). [#20476](https://github.com/ClickHouse/ClickHouse/pull/20476) ([Amos Bird](https://github.com/amosbird)). +* Fix abnormal server termination when http client goes away. [#20464](https://github.com/ClickHouse/ClickHouse/pull/20464) ([Azat Khuzhin](https://github.com/azat)). +* Fix `LOGICAL_ERROR` for `join_use_nulls=1` when JOIN contains const from SELECT. [#20461](https://github.com/ClickHouse/ClickHouse/pull/20461) ([Azat Khuzhin](https://github.com/azat)). +* Check if table function `view` is used in expression list and throw an error. This fixes [#20342](https://github.com/ClickHouse/ClickHouse/issues/20342). [#20350](https://github.com/ClickHouse/ClickHouse/pull/20350) ([Amos Bird](https://github.com/amosbird)). +* Avoid invalid dereference in RANGE_HASHED() dictionary. [#20345](https://github.com/ClickHouse/ClickHouse/pull/20345) ([Azat Khuzhin](https://github.com/azat)). +* Fix null dereference with `join_use_nulls=1`. [#20344](https://github.com/ClickHouse/ClickHouse/pull/20344) ([Azat Khuzhin](https://github.com/azat)). +* Fix incorrect result of binary operations between two constant decimals of different scale. Fixes [#20283](https://github.com/ClickHouse/ClickHouse/issues/20283). [#20339](https://github.com/ClickHouse/ClickHouse/pull/20339) ([Maksim Kita](https://github.com/kitaisreal)). +* Fix too often retries of failed background tasks for `ReplicatedMergeTree` table engines family. This could lead to too verbose logging and increased CPU load. Fixes [#20203](https://github.com/ClickHouse/ClickHouse/issues/20203). [#20335](https://github.com/ClickHouse/ClickHouse/pull/20335) ([alesapin](https://github.com/alesapin)). +* Restrict to `DROP` or `RENAME` version column of `*CollapsingMergeTree` and `ReplacingMergeTree` table engines. [#20300](https://github.com/ClickHouse/ClickHouse/pull/20300) ([alesapin](https://github.com/alesapin)). +* Fixed the behavior when in case of broken JSON we tried to read the whole file into memory which leads to exception from the allocator. Fixes [#19719](https://github.com/ClickHouse/ClickHouse/issues/19719). [#20286](https://github.com/ClickHouse/ClickHouse/pull/20286) ([Nikita Mikhaylov](https://github.com/nikitamikhaylov)). +* Fix exception during vertical merge for `MergeTree` table engines family which don't allow to perform vertical merges. Fixes [#20259](https://github.com/ClickHouse/ClickHouse/issues/20259). [#20279](https://github.com/ClickHouse/ClickHouse/pull/20279) ([alesapin](https://github.com/alesapin)). +* Fix rare server crash on config reload during the shutdown. Fixes [#19689](https://github.com/ClickHouse/ClickHouse/issues/19689). [#20224](https://github.com/ClickHouse/ClickHouse/pull/20224) ([alesapin](https://github.com/alesapin)). +* Fix CTE when using in INSERT SELECT. This fixes [#20187](https://github.com/ClickHouse/ClickHouse/issues/20187), fixes [#20195](https://github.com/ClickHouse/ClickHouse/issues/20195). [#20211](https://github.com/ClickHouse/ClickHouse/pull/20211) ([Amos Bird](https://github.com/amosbird)). +* Fixes [#19314](https://github.com/ClickHouse/ClickHouse/issues/19314). [#20156](https://github.com/ClickHouse/ClickHouse/pull/20156) ([Ivan](https://github.com/abyss7)). +* fix toMinute function to handle special timezone correctly. [#20149](https://github.com/ClickHouse/ClickHouse/pull/20149) ([keenwolf](https://github.com/keen-wolf)). +* Fix server crash after query with `if` function with `Tuple` type of then/else branches result. `Tuple` type must contain `Array` or another complex type. Fixes [#18356](https://github.com/ClickHouse/ClickHouse/issues/18356). [#20133](https://github.com/ClickHouse/ClickHouse/pull/20133) ([alesapin](https://github.com/alesapin)). +* The `MongoDB` table engine now establishes connection only when it's going to read data. `ATTACH TABLE` won't try to connect anymore. [#20110](https://github.com/ClickHouse/ClickHouse/pull/20110) ([Vitaly Baranov](https://github.com/vitlibar)). +* Bugfix in StorageJoin. [#20079](https://github.com/ClickHouse/ClickHouse/pull/20079) ([vdimir](https://github.com/vdimir)). +* Fix the case when calculating modulo of division of negative number by small divisor, the resulting data type was not large enough to accomodate the negative result. This closes [#20052](https://github.com/ClickHouse/ClickHouse/issues/20052). [#20067](https://github.com/ClickHouse/ClickHouse/pull/20067) ([alexey-milovidov](https://github.com/alexey-milovidov)). +* MaterializeMySQL: Fix replication for statements that update several tables. [#20066](https://github.com/ClickHouse/ClickHouse/pull/20066) ([Håvard Kvålen](https://github.com/havardk)). +* Prevent "Connection refused" in docker during initialization script execution. [#20012](https://github.com/ClickHouse/ClickHouse/pull/20012) ([filimonov](https://github.com/filimonov)). +* `EmbeddedRocksDB` is an experimental storage. Fix the issue with lack of proper type checking. Simplified code. This closes [#19967](https://github.com/ClickHouse/ClickHouse/issues/19967). [#19972](https://github.com/ClickHouse/ClickHouse/pull/19972) ([alexey-milovidov](https://github.com/alexey-milovidov)). +* Fix a segfault in function `fromModifiedJulianDay` when the argument type is `Nullable(T)` for any integral types other than Int32. [#19959](https://github.com/ClickHouse/ClickHouse/pull/19959) ([PHO](https://github.com/depressed-pho)). +* BloomFilter index crash fix. Fixes [#19757](https://github.com/ClickHouse/ClickHouse/issues/19757). [#19884](https://github.com/ClickHouse/ClickHouse/pull/19884) ([Maksim Kita](https://github.com/kitaisreal)). +* Deadlock was possible if system.text_log is enabled. This fixes [#19874](https://github.com/ClickHouse/ClickHouse/issues/19874). [#19875](https://github.com/ClickHouse/ClickHouse/pull/19875) ([alexey-milovidov](https://github.com/alexey-milovidov)). +* Fix starting the server with tables having default expressions containing dictGet(). Allow getting return type of dictGet() without loading dictionary. [#19805](https://github.com/ClickHouse/ClickHouse/pull/19805) ([Vitaly Baranov](https://github.com/vitlibar)). +* Fix clickhouse-client abort exception while executing only `select`. [#19790](https://github.com/ClickHouse/ClickHouse/pull/19790) ([taiyang-li](https://github.com/taiyang-li)). +* Fix a bug that moving pieces to destination table may failed in case of launching multiple clickhouse-copiers. [#19743](https://github.com/ClickHouse/ClickHouse/pull/19743) ([madianjun](https://github.com/mdianjun)). +* Background thread which executes `ON CLUSTER` queries might hang waiting for dropped replicated table to do something. It's fixed. [#19684](https://github.com/ClickHouse/ClickHouse/pull/19684) ([yiguolei](https://github.com/yiguolei)). + +#### Build/Testing/Packaging Improvement + +* Allow to build ClickHouse with AVX-2 enabled globally. It gives slight performance benefits on modern CPUs. Not recommended for production and will not be supported as official build for now. [#20180](https://github.com/ClickHouse/ClickHouse/pull/20180) ([alexey-milovidov](https://github.com/alexey-milovidov)). +* Fix some of the issues found by Coverity. See [#19964](https://github.com/ClickHouse/ClickHouse/issues/19964). [#20010](https://github.com/ClickHouse/ClickHouse/pull/20010) ([alexey-milovidov](https://github.com/alexey-milovidov)). +* Allow to start up with modified binary under gdb. In previous version if you set up breakpoint in gdb before start, server will refuse to start up due to failed integrity check. [#21258](https://github.com/ClickHouse/ClickHouse/pull/21258) ([alexey-milovidov](https://github.com/alexey-milovidov)). +* Add a test for different compression methods in Kafka. [#21111](https://github.com/ClickHouse/ClickHouse/pull/21111) ([filimonov](https://github.com/filimonov)). +* Fixed port clash from test_storage_kerberized_hdfs test. [#19974](https://github.com/ClickHouse/ClickHouse/pull/19974) ([Ilya Yatsishin](https://github.com/qoega)). +* Print `stdout` and `stderr` to log when failed to start docker in integration tests. Before this PR there was a very short error message in this case which didn't help to investigate the problems. [#20631](https://github.com/ClickHouse/ClickHouse/pull/20631) ([Vitaly Baranov](https://github.com/vitlibar)). + + ## ClickHouse release 21.2 ### ClickHouse release v21.2.2.8-stable, 2021-02-07 From 682b5fb6ee58c367397ada74acd8d54af9217699 Mon Sep 17 00:00:00 2001 From: Yatsishin Ilya <2159081+qoega@users.noreply.github.com> Date: Thu, 11 Mar 2021 10:42:58 +0300 Subject: [PATCH 238/544] better --- cmake/tools.cmake | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/cmake/tools.cmake b/cmake/tools.cmake index 88c48978518..e0f1f0b4fa6 100644 --- a/cmake/tools.cmake +++ b/cmake/tools.cmake @@ -75,12 +75,12 @@ if (OS_LINUX AND NOT LINKER_NAME) endif () if (LINKER_NAME) - if (COMPILER_GCC) - set (CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -fuse-ld=${LINKER_NAME}") - set (CMAKE_SHARED_LINKER_FLAGS "${CMAKE_SHARED_LINKER_FLAGS} -fuse-ld=${LINKER_NAME}") - else () + if (COMPILER_CLANG AND CMAKE_CXX_COMPILER_VERSION VERSION_GREATER 12.0.0) set (CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} --ld-path=${LINKER_NAME}") set (CMAKE_SHARED_LINKER_FLAGS "${CMAKE_SHARED_LINKER_FLAGS} --ld-path=${LINKER_NAME}") + else () + set (CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -fuse-ld=${LINKER_NAME}") + set (CMAKE_SHARED_LINKER_FLAGS "${CMAKE_SHARED_LINKER_FLAGS} -fuse-ld=${LINKER_NAME}") endif () message(STATUS "Using custom linker by name: ${LINKER_NAME}") From c64ba1ea8bbc88444ea2334afeb1f9b7ba1a36a0 Mon Sep 17 00:00:00 2001 From: Yatsishin Ilya <2159081+qoega@users.noreply.github.com> Date: Thu, 11 Mar 2021 11:03:03 +0300 Subject: [PATCH 239/544] fix 12.0.0 version --- cmake/tools.cmake | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmake/tools.cmake b/cmake/tools.cmake index e0f1f0b4fa6..abb11843d59 100644 --- a/cmake/tools.cmake +++ b/cmake/tools.cmake @@ -75,7 +75,7 @@ if (OS_LINUX AND NOT LINKER_NAME) endif () if (LINKER_NAME) - if (COMPILER_CLANG AND CMAKE_CXX_COMPILER_VERSION VERSION_GREATER 12.0.0) + if (COMPILER_CLANG AND (CMAKE_CXX_COMPILER_VERSION VERSION_GREATER 12.0.0 OR CMAKE_CXX_COMPILER_VERSION VERSION_EQUAL 12.0.0)) set (CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} --ld-path=${LINKER_NAME}") set (CMAKE_SHARED_LINKER_FLAGS "${CMAKE_SHARED_LINKER_FLAGS} --ld-path=${LINKER_NAME}") else () From 109cb634e150fbca3376aba4f31e7e5968de581b Mon Sep 17 00:00:00 2001 From: Anton Ivashkin Date: Thu, 11 Mar 2021 12:08:19 +0300 Subject: [PATCH 240/544] Remove debug lines --- src/Storages/StorageReplicatedMergeTree.cpp | 3 --- 1 file changed, 3 deletions(-) diff --git a/src/Storages/StorageReplicatedMergeTree.cpp b/src/Storages/StorageReplicatedMergeTree.cpp index a4f6ddd47c7..04c23ae8574 100644 --- a/src/Storages/StorageReplicatedMergeTree.cpp +++ b/src/Storages/StorageReplicatedMergeTree.cpp @@ -3769,9 +3769,6 @@ bool StorageReplicatedMergeTree::fetchExistsPart(const String & part_name, const } } - if (part_name != "foo") - return false; - SCOPE_EXIT ({ std::lock_guard lock(currently_fetching_parts_mutex); From 5704405f8c6227817499bd5a72beb7dd45b2fa03 Mon Sep 17 00:00:00 2001 From: alesapin Date: Thu, 11 Mar 2021 12:29:27 +0300 Subject: [PATCH 241/544] Disable new setting by default --- src/Storages/MergeTree/MergeTreeSettings.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Storages/MergeTree/MergeTreeSettings.h b/src/Storages/MergeTree/MergeTreeSettings.h index d64f7921412..4cb1380b49b 100644 --- a/src/Storages/MergeTree/MergeTreeSettings.h +++ b/src/Storages/MergeTree/MergeTreeSettings.h @@ -119,7 +119,7 @@ struct Settings; M(Int64, max_partitions_to_read, -1, "Limit the max number of partitions that can be accessed in one query. <= 0 means unlimited. This setting is the default that can be overridden by the query-level setting with the same name.", 0) \ M(UInt64, max_concurrent_queries, 0, "Max number of concurrently executed queries related to the MergeTree table (0 - disabled). Queries will still be limited by other max_concurrent_queries settings.", 0) \ M(UInt64, min_marks_to_honor_max_concurrent_queries, 0, "Minimal number of marks to honor the MergeTree-level's max_concurrent_queries (0 - disabled). Queries will still be limited by other max_concurrent_queries settings.", 0) \ - M(UInt64, min_bytes_to_rebalance_partition_over_jbod, 512 * 1024 * 1024, "Minimal amount of bytes to enable part rebalance over JBOD array (0 - disabled).", 0) \ + M(UInt64, min_bytes_to_rebalance_partition_over_jbod, 0, "Minimal amount of bytes to enable part rebalance over JBOD array (0 - disabled).", 0) \ \ /** Obsolete settings. Kept for backward compatibility only. */ \ M(UInt64, min_relative_delay_to_yield_leadership, 120, "Obsolete setting, does nothing.", 0) \ From 9376b676e9a9bb8911b872e1887da85a45f7479d Mon Sep 17 00:00:00 2001 From: Alexey Ilyukhov Date: Sun, 7 Jun 2020 15:33:49 +0300 Subject: [PATCH 242/544] Add parseGeometry interface --- src/Functions/geometryFromColumn.cpp | 10 ++++++++++ src/Functions/geometryFromColumn.h | 19 +++++++++++++++++++ src/Functions/ya.make | 1 + 3 files changed, 30 insertions(+) create mode 100644 src/Functions/geometryFromColumn.cpp create mode 100644 src/Functions/geometryFromColumn.h diff --git a/src/Functions/geometryFromColumn.cpp b/src/Functions/geometryFromColumn.cpp new file mode 100644 index 00000000000..23bd9961752 --- /dev/null +++ b/src/Functions/geometryFromColumn.cpp @@ -0,0 +1,10 @@ +#include + +namespace DB { + +Geometry geometryFromColumn(const ColumnWithTypeAndName &, size_t) +{ + return Point(0.0, 0.0); +} + +} diff --git a/src/Functions/geometryFromColumn.h b/src/Functions/geometryFromColumn.h new file mode 100644 index 00000000000..c90d4d20c68 --- /dev/null +++ b/src/Functions/geometryFromColumn.h @@ -0,0 +1,19 @@ +#include +#include + +#include +#include +#include +#include + +namespace DB { + +using Point = boost::geometry::model::d2::point_xy; +using Ring = boost::geometry::model::ring; +using Polygon = boost::geometry::model::polygon; +using MultiPolygon = boost::geometry::model::multi_polygon; +using Geometry = boost::variant; + +Geometry geometryFromColumn(const ColumnWithTypeAndName & col, size_t i); + +} diff --git a/src/Functions/ya.make b/src/Functions/ya.make index 7a4deae4d04..0e5bc1cab72 100644 --- a/src/Functions/ya.make +++ b/src/Functions/ya.make @@ -268,6 +268,7 @@ SRCS( geohashDecode.cpp geohashEncode.cpp geohashesInBox.cpp + geometryFromColumn.cpp getMacro.cpp getScalar.cpp getSetting.cpp From 6d59be5ea4768034f6526f7f9813062e0c369f7b Mon Sep 17 00:00:00 2001 From: Alexey Ilyukhov Date: Sun, 7 Jun 2020 16:42:09 +0300 Subject: [PATCH 243/544] Rewrite interface --- src/Functions/geometryFromColumn.cpp | 66 +++++++++++++++++++++++++++- src/Functions/geometryFromColumn.h | 18 +++++++- 2 files changed, 81 insertions(+), 3 deletions(-) diff --git a/src/Functions/geometryFromColumn.cpp b/src/Functions/geometryFromColumn.cpp index 23bd9961752..c8947659f66 100644 --- a/src/Functions/geometryFromColumn.cpp +++ b/src/Functions/geometryFromColumn.cpp @@ -1,10 +1,72 @@ +#include +#include +#include +#include #include + namespace DB { -Geometry geometryFromColumn(const ColumnWithTypeAndName &, size_t) +namespace ErrorCodes { - return Point(0.0, 0.0); + extern const int ILLEGAL_COLUMN; +} + +namespace { + +Exception failedToParse(const ColumnWithTypeAndName & col, std::string reason = "") +{ + return Exception("Cannot parse geometry from column with type " + col.type->getName() + + (reason.empty() ? std::string() : ", " + reason), ErrorCodes::ILLEGAL_COLUMN); +} + +size_t getArrayDepth(const ColumnWithTypeAndName & col, size_t max_depth) +{ + size_t depth = 0; + DataTypePtr data_type = col.type; + + while (isArray(data_type) && depth != max_depth + 1) + { + data_type = static_cast(*data_type).getNestedType(); + } + + return max_depth; +} + +} + +PointFromColumnParser::PointFromColumnParser(const ColumnWithTypeAndName & col) +{ + const auto & tuple_columns = static_cast(*col.column).getColumns(); + + if (tuple_columns.size() != 2) { + throw failedToParse(col, "tuple must have exactly 2 columns"); + } + + x = static_cast(*tuple_columns[0]).getData().data(); + y = static_cast(*tuple_columns[1]).getData().data(); +} + +Point PointFromColumnParser::createContainer() const +{ + return Point(); +} + +void PointFromColumnParser::get(Point & container, size_t i) const +{ + boost::geometry::set<0>(container, x[i]); + boost::geometry::set<0>(container, y[i]); +} + +GeometryFromColumnParser makeGeometryFromColumnParser(const ColumnWithTypeAndName & col) +{ + switch (getArrayDepth(col, 3)) { + case 0: return PointFromColumnParser(col); + // case 1: return parseRing(col, i); + // case 2: return parsePolygon(col, i); + // case 3: return parseMultyPoligon(col, i); + default: throw failedToParse(col, "array depth is too big"); + } } } diff --git a/src/Functions/geometryFromColumn.h b/src/Functions/geometryFromColumn.h index c90d4d20c68..125c5b5a1b2 100644 --- a/src/Functions/geometryFromColumn.h +++ b/src/Functions/geometryFromColumn.h @@ -1,3 +1,5 @@ +#pragma once + #include #include @@ -14,6 +16,20 @@ using Polygon = boost::geometry::model::polygon; using MultiPolygon = boost::geometry::model::multi_polygon; using Geometry = boost::variant; -Geometry geometryFromColumn(const ColumnWithTypeAndName & col, size_t i); +class PointFromColumnParser +{ +public: + PointFromColumnParser(const ColumnWithTypeAndName & col); + Point createContainer() const; + void get(Point & container, size_t i) const; + +private: + const Float64 * x; + const Float64 * y; +}; + +using GeometryFromColumnParser = boost::variant; + +GeometryFromColumnParser makeGeometryFromColumnParser(); } From 33acc2aa5dc091a7cb948f78c558529789b2bad8 Mon Sep 17 00:00:00 2001 From: Alexey Ilyukhov Date: Sun, 7 Jun 2020 16:45:28 +0300 Subject: [PATCH 244/544] fix --- src/Functions/geometryFromColumn.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Functions/geometryFromColumn.h b/src/Functions/geometryFromColumn.h index 125c5b5a1b2..3b381277597 100644 --- a/src/Functions/geometryFromColumn.h +++ b/src/Functions/geometryFromColumn.h @@ -30,6 +30,6 @@ private: using GeometryFromColumnParser = boost::variant; -GeometryFromColumnParser makeGeometryFromColumnParser(); +GeometryFromColumnParser makeGeometryFromColumnParser(const ColumnWithTypeAndName & col); } From 78e0db268ceadc42f82bc63a77ee1a4da6002463 Mon Sep 17 00:00:00 2001 From: Alexey Ilyukhov Date: Sun, 7 Jun 2020 17:28:46 +0300 Subject: [PATCH 245/544] Add ring parser --- src/Functions/geometryFromColumn.cpp | 45 ++----------- src/Functions/geometryFromColumn.h | 94 +++++++++++++++++++++++++--- 2 files changed, 88 insertions(+), 51 deletions(-) diff --git a/src/Functions/geometryFromColumn.cpp b/src/Functions/geometryFromColumn.cpp index c8947659f66..03013266caa 100644 --- a/src/Functions/geometryFromColumn.cpp +++ b/src/Functions/geometryFromColumn.cpp @@ -1,25 +1,10 @@ -#include -#include -#include -#include #include - namespace DB { -namespace ErrorCodes -{ - extern const int ILLEGAL_COLUMN; -} namespace { -Exception failedToParse(const ColumnWithTypeAndName & col, std::string reason = "") -{ - return Exception("Cannot parse geometry from column with type " + col.type->getName() - + (reason.empty() ? std::string() : ", " + reason), ErrorCodes::ILLEGAL_COLUMN); -} - size_t getArrayDepth(const ColumnWithTypeAndName & col, size_t max_depth) { size_t depth = 0; @@ -35,37 +20,15 @@ size_t getArrayDepth(const ColumnWithTypeAndName & col, size_t max_depth) } -PointFromColumnParser::PointFromColumnParser(const ColumnWithTypeAndName & col) -{ - const auto & tuple_columns = static_cast(*col.column).getColumns(); - - if (tuple_columns.size() != 2) { - throw failedToParse(col, "tuple must have exactly 2 columns"); - } - - x = static_cast(*tuple_columns[0]).getData().data(); - y = static_cast(*tuple_columns[1]).getData().data(); -} - -Point PointFromColumnParser::createContainer() const -{ - return Point(); -} - -void PointFromColumnParser::get(Point & container, size_t i) const -{ - boost::geometry::set<0>(container, x[i]); - boost::geometry::set<0>(container, y[i]); -} - GeometryFromColumnParser makeGeometryFromColumnParser(const ColumnWithTypeAndName & col) { switch (getArrayDepth(col, 3)) { - case 0: return PointFromColumnParser(col); - // case 1: return parseRing(col, i); + case 0: return Float64PointFromColumnParser(*col.column); + case 1: return Float64RingFromColumnParser(*col.column); // case 2: return parsePolygon(col, i); // case 3: return parseMultyPoligon(col, i); - default: throw failedToParse(col, "array depth is too big"); + default: throw Exception("Cannot parse geometry from column with type " + col.type->getName() + + ", array depth is too big", ErrorCodes::ILLEGAL_COLUMN); } } diff --git a/src/Functions/geometryFromColumn.h b/src/Functions/geometryFromColumn.h index 3b381277597..cc9e743ccc3 100644 --- a/src/Functions/geometryFromColumn.h +++ b/src/Functions/geometryFromColumn.h @@ -8,27 +8,101 @@ #include #include +#include +#include +#include +#include +#include + namespace DB { -using Point = boost::geometry::model::d2::point_xy; -using Ring = boost::geometry::model::ring; -using Polygon = boost::geometry::model::polygon; -using MultiPolygon = boost::geometry::model::multi_polygon; -using Geometry = boost::variant; +namespace ErrorCodes +{ + extern const int ILLEGAL_COLUMN; +} -class PointFromColumnParser +using Float64Point = boost::geometry::model::d2::point_xy; +using Float64Ring = boost::geometry::model::ring; +using Float64Polygon = boost::geometry::model::polygon; +using Float64MultiPolygon = boost::geometry::model::multi_polygon; +using Float64Geometry = boost::variant; + +class Float64PointFromColumnParser { public: - PointFromColumnParser(const ColumnWithTypeAndName & col); - Point createContainer() const; - void get(Point & container, size_t i) const; + Float64PointFromColumnParser(const IColumn & col) + { + const auto & tuple_columns = static_cast(col).getColumns(); + + if (tuple_columns.size() != 2) + { + throw Exception("tuple size must be equal to 2", ErrorCodes::ILLEGAL_COLUMN); + } + + x = static_cast(*tuple_columns[0]).getData().data(); + if (!x) + { + throw Exception("failed to get x column", ErrorCodes::ILLEGAL_COLUMN); + } + + y = static_cast(*tuple_columns[1]).getData().data(); + if (!y) + { + throw Exception("failed to get y column", ErrorCodes::ILLEGAL_COLUMN); + } + } + + Float64Point createContainer() const + { + return Float64Point(); + } + + void get(Float64Point & container, size_t i) const + { + boost::geometry::set<0>(container, x[i]); + boost::geometry::set<0>(container, y[i]); + } private: const Float64 * x; const Float64 * y; }; -using GeometryFromColumnParser = boost::variant; +template +class RingFromColumnParser +{ +public: + RingFromColumnParser(const IColumn & col) + : offsets(static_cast(col).getOffsets()) + , pointParser(static_cast(col).getData()) + { + } + + RingType createContainer() const + { + return RingType(); + } + + void get(RingType & container, size_t i) const + { + size_t l = offsets[i - 1]; + size_t r = offsets[i]; + + container.resize(r - l); + + for (size_t j = l; j < r; j++) { + pointParser.parse(container[j - l], l); + } + } + +private: + const IColumn::Offsets & offsets; + PointParser pointParser; +}; + +using Float64RingFromColumnParser = RingFromColumnParser; + +using GeometryFromColumnParser = boost::variant; GeometryFromColumnParser makeGeometryFromColumnParser(const ColumnWithTypeAndName & col); From 14a891057d292a164c4179bfddaef45a74eaf83a Mon Sep 17 00:00:00 2001 From: Alexey Ilyukhov Date: Sun, 7 Jun 2020 17:58:34 +0300 Subject: [PATCH 246/544] Add other parsers --- src/Functions/geometryFromColumn.cpp | 11 ++-- src/Functions/geometryFromColumn.h | 91 ++++++++++++++++++++++++++-- 2 files changed, 90 insertions(+), 12 deletions(-) diff --git a/src/Functions/geometryFromColumn.cpp b/src/Functions/geometryFromColumn.cpp index 03013266caa..e4cdb3e22b7 100644 --- a/src/Functions/geometryFromColumn.cpp +++ b/src/Functions/geometryFromColumn.cpp @@ -5,16 +5,13 @@ namespace DB { namespace { -size_t getArrayDepth(const ColumnWithTypeAndName & col, size_t max_depth) +size_t getArrayDepth(DataTypePtr data_type, size_t max_depth) { size_t depth = 0; - DataTypePtr data_type = col.type; - while (isArray(data_type) && depth != max_depth + 1) { data_type = static_cast(*data_type).getNestedType(); } - return max_depth; } @@ -22,11 +19,11 @@ size_t getArrayDepth(const ColumnWithTypeAndName & col, size_t max_depth) GeometryFromColumnParser makeGeometryFromColumnParser(const ColumnWithTypeAndName & col) { - switch (getArrayDepth(col, 3)) { + switch (getArrayDepth(col.type, 3)) { case 0: return Float64PointFromColumnParser(*col.column); case 1: return Float64RingFromColumnParser(*col.column); - // case 2: return parsePolygon(col, i); - // case 3: return parseMultyPoligon(col, i); + case 2: return Float64PolygonFromColumnParser(*col.column); + case 3: return Float64MultiPolygonFromColumnParser(*col.column); default: throw Exception("Cannot parse geometry from column with type " + col.type->getName() + ", array depth is too big", ErrorCodes::ILLEGAL_COLUMN); } diff --git a/src/Functions/geometryFromColumn.h b/src/Functions/geometryFromColumn.h index cc9e743ccc3..1042c4312fb 100644 --- a/src/Functions/geometryFromColumn.h +++ b/src/Functions/geometryFromColumn.h @@ -75,8 +75,7 @@ public: RingFromColumnParser(const IColumn & col) : offsets(static_cast(col).getOffsets()) , pointParser(static_cast(col).getData()) - { - } + {} RingType createContainer() const { @@ -88,21 +87,103 @@ public: size_t l = offsets[i - 1]; size_t r = offsets[i]; + // reserve extra point for case when polygon is open + container.reserve(r - l + 1); container.resize(r - l); for (size_t j = l; j < r; j++) { - pointParser.parse(container[j - l], l); + pointParser.get(container[j - l], j); + } + + // make ring closed + if (!boost::geometry::equals(container[0], container.back())) + { + container.push_back(container[0]); } } private: const IColumn::Offsets & offsets; - PointParser pointParser; + const PointParser pointParser; }; using Float64RingFromColumnParser = RingFromColumnParser; -using GeometryFromColumnParser = boost::variant; +template +class PolygonFromColumnParser +{ +public: + PolygonFromColumnParser(const IColumn & col) + : offsets(static_cast(col).getOffsets()) + , ringParser(static_cast(col).getData()) + {} + + PolygonType createContainer() const + { + return PolygonType(); + } + + void get(PolygonType & container, size_t i) const + { + size_t l = offsets[i - 1]; + size_t r = offsets[i]; + + container.resize(r - l); + ringParser.get(container.outer(), l); + + container.inners().resize(r - l - 1); + for (size_t j = l + 1; j < r; j++) + { + ringParser.get(container.inners()[j - l - 1], j); + } + } + +private: + const IColumn::Offsets & offsets; + const RingParser ringParser; +}; + +using Float64PolygonFromColumnParser = PolygonFromColumnParser; + +template +class MultiPolygonFromColumnParser +{ +public: + MultiPolygonFromColumnParser(const IColumn & col) + : offsets(static_cast(col).getOffsets()) + , polygonParser(static_cast(col).getData()) + {} + + MultiPolygonType createContainer() const + { + return MultiPolygonType(); + } + + void get(MultiPolygonType & container, size_t i) const + { + size_t l = offsets[i - 1]; + size_t r = offsets[i]; + + container.resize(r - l); + for (size_t j = l; j < r; j++) + { + polygonParser.get(container[j - l], j - l); + } + } + +private: + const IColumn::Offsets & offsets; + const PolygonParser polygonParser; +}; + +using Float64MultiPolygonFromColumnParser = MultiPolygonFromColumnParser; + +using GeometryFromColumnParser = boost::variant< + Float64PointFromColumnParser, + Float64RingFromColumnParser, + Float64PolygonFromColumnParser, + Float64MultiPolygonFromColumnParser +>; GeometryFromColumnParser makeGeometryFromColumnParser(const ColumnWithTypeAndName & col); From d0d6e6953c2a2af9fb2300921ff96b9362f22edb Mon Sep 17 00:00:00 2001 From: Alexey Ilyukhov Date: Sun, 7 Jun 2020 19:04:35 +0300 Subject: [PATCH 247/544] Fix build; add wkt --- src/Functions/geometryFromColumn.cpp | 40 ++++++++++++++++ src/Functions/geometryFromColumn.h | 50 +++++++++++++------- src/Functions/registerFunctionsGeo.cpp | 2 + src/Functions/wkt.cpp | 65 ++++++++++++++++++++++++++ src/Functions/ya.make | 1 + 5 files changed, 141 insertions(+), 17 deletions(-) create mode 100644 src/Functions/wkt.cpp diff --git a/src/Functions/geometryFromColumn.cpp b/src/Functions/geometryFromColumn.cpp index e4cdb3e22b7..76b2d536413 100644 --- a/src/Functions/geometryFromColumn.cpp +++ b/src/Functions/geometryFromColumn.cpp @@ -15,6 +15,35 @@ size_t getArrayDepth(DataTypePtr data_type, size_t max_depth) return max_depth; } +class ContainerCreator : public boost::static_visitor +{ +public: + template + Float64Geometry operator()(const T & parser) const + { + return parser.createContainer(); + } +}; + +class Getter : public boost::static_visitor +{ +public: + Getter(Float64Geometry & container_, size_t i_) + : container(container_) + , i(i_) + {} + + template + void operator()(const T & parser) const + { + parser.get(container, i); + } + +private: + Float64Geometry & container; + size_t i; +}; + } GeometryFromColumnParser makeGeometryFromColumnParser(const ColumnWithTypeAndName & col) @@ -29,4 +58,15 @@ GeometryFromColumnParser makeGeometryFromColumnParser(const ColumnWithTypeAndNam } } +Float64Geometry createContainer(const GeometryFromColumnParser & parser) +{ + static ContainerCreator creator; + return boost::apply_visitor(creator, parser); +} + +void get(const GeometryFromColumnParser & parser, Float64Geometry & container, size_t i) +{ + boost::apply_visitor(Getter(container, i), parser); +} + } diff --git a/src/Functions/geometryFromColumn.h b/src/Functions/geometryFromColumn.h index 1042c4312fb..b29b8c81d52 100644 --- a/src/Functions/geometryFromColumn.h +++ b/src/Functions/geometryFromColumn.h @@ -52,23 +52,27 @@ public: } } - Float64Point createContainer() const + Float64Geometry createContainer() const { return Float64Point(); } + void get(Float64Geometry & container, size_t i) const + { + get(boost::get(container), i); + } + void get(Float64Point & container, size_t i) const { boost::geometry::set<0>(container, x[i]); boost::geometry::set<0>(container, y[i]); } - private: const Float64 * x; const Float64 * y; }; -template +template class RingFromColumnParser { public: @@ -77,11 +81,16 @@ public: , pointParser(static_cast(col).getData()) {} - RingType createContainer() const + Geometry createContainer() const { return RingType(); } + void get(Geometry & container, size_t i) const + { + get(boost::get(container), i); + } + void get(RingType & container, size_t i) const { size_t l = offsets[i - 1]; @@ -107,9 +116,7 @@ private: const PointParser pointParser; }; -using Float64RingFromColumnParser = RingFromColumnParser; - -template +template class PolygonFromColumnParser { public: @@ -118,17 +125,21 @@ public: , ringParser(static_cast(col).getData()) {} - PolygonType createContainer() const + Geometry createContainer() const { return PolygonType(); } + void get(Geometry & container, size_t i) const + { + get(boost::get(container), i); + } + void get(PolygonType & container, size_t i) const { size_t l = offsets[i - 1]; size_t r = offsets[i]; - container.resize(r - l); ringParser.get(container.outer(), l); container.inners().resize(r - l - 1); @@ -143,9 +154,7 @@ private: const RingParser ringParser; }; -using Float64PolygonFromColumnParser = PolygonFromColumnParser; - -template +template class MultiPolygonFromColumnParser { public: @@ -154,20 +163,21 @@ public: , polygonParser(static_cast(col).getData()) {} - MultiPolygonType createContainer() const + Geometry createContainer() const { return MultiPolygonType(); } - void get(MultiPolygonType & container, size_t i) const + void get(Geometry & container, size_t i) const { + MultiPolygonType & multi_polygon = boost::get(container); size_t l = offsets[i - 1]; size_t r = offsets[i]; - container.resize(r - l); + multi_polygon.resize(r - l); for (size_t j = l; j < r; j++) { - polygonParser.get(container[j - l], j - l); + polygonParser.get(multi_polygon[j - l], j - l); } } @@ -176,7 +186,9 @@ private: const PolygonParser polygonParser; }; -using Float64MultiPolygonFromColumnParser = MultiPolygonFromColumnParser; +using Float64RingFromColumnParser = RingFromColumnParser; +using Float64PolygonFromColumnParser = PolygonFromColumnParser; +using Float64MultiPolygonFromColumnParser = MultiPolygonFromColumnParser; using GeometryFromColumnParser = boost::variant< Float64PointFromColumnParser, @@ -185,6 +197,10 @@ using GeometryFromColumnParser = boost::variant< Float64MultiPolygonFromColumnParser >; +Float64Geometry createContainer(const GeometryFromColumnParser & parser); + +void get(const GeometryFromColumnParser & parser, Float64Geometry & container, size_t i); + GeometryFromColumnParser makeGeometryFromColumnParser(const ColumnWithTypeAndName & col); } diff --git a/src/Functions/registerFunctionsGeo.cpp b/src/Functions/registerFunctionsGeo.cpp index 2ed02bbd73b..57cf1ff1b09 100644 --- a/src/Functions/registerFunctionsGeo.cpp +++ b/src/Functions/registerFunctionsGeo.cpp @@ -13,6 +13,7 @@ void registerFunctionPointInPolygon(FunctionFactory & factory); void registerFunctionGeohashEncode(FunctionFactory & factory); void registerFunctionGeohashDecode(FunctionFactory & factory); void registerFunctionGeohashesInBox(FunctionFactory & factory); +void registerFunctionWkt(FunctionFactory & factory); #if USE_H3 void registerFunctionGeoToH3(FunctionFactory &); @@ -39,6 +40,7 @@ void registerFunctionsGeo(FunctionFactory & factory) registerFunctionGeohashEncode(factory); registerFunctionGeohashDecode(factory); registerFunctionGeohashesInBox(factory); + registerFunctionWkt(factory); #if USE_H3 registerFunctionGeoToH3(factory); diff --git a/src/Functions/wkt.cpp b/src/Functions/wkt.cpp new file mode 100644 index 00000000000..f58395fb909 --- /dev/null +++ b/src/Functions/wkt.cpp @@ -0,0 +1,65 @@ +#include +#include +#include +#include + +#include +#include + +namespace DB +{ + +class FunctionWkt : public IFunction +{ +public: + static inline const char * name = "wkt"; + + explicit FunctionWkt() {} + + static FunctionPtr create(const Context &) + { + return std::make_shared(); + } + + String getName() const override + { + return name; + } + + size_t getNumberOfArguments() const override + { + return 1; + } + + DataTypePtr getReturnTypeImpl(const DataTypes &) const override + { + return std::make_shared(); + } + + void executeImpl(Block & block, const ColumnNumbers & arguments, size_t result, size_t input_rows_count) override + { + auto parser = makeGeometryFromColumnParser(block.getByPosition(arguments[0])); + auto res_column = ColumnString::create(); + + auto container = createContainer(parser); + + for (size_t i = 0; i < input_rows_count; i++) + { + std::stringstream str; + get(parser, container, i); + str << boost::geometry::wkt(container); + std::string serialized = str.str(); + res_column->insertData(serialized.c_str(), serialized.size()); + } + + block.getByPosition(result).column = std::move(res_column); + } +}; + + +void registerFunctionWkt(FunctionFactory & factory) +{ + factory.registerFunction(); +} + +} diff --git a/src/Functions/ya.make b/src/Functions/ya.make index 0e5bc1cab72..2f3c6348206 100644 --- a/src/Functions/ya.make +++ b/src/Functions/ya.make @@ -526,6 +526,7 @@ SRCS( visitParamExtractString.cpp visitParamExtractUInt.cpp visitParamHas.cpp + wkt.cpp yandexConsistentHash.cpp yesterday.cpp From fe8382521139a58c0ba277eb848e88894658db66 Mon Sep 17 00:00:00 2001 From: Alexey Ilyukhov Date: Sun, 7 Jun 2020 19:47:56 +0300 Subject: [PATCH 248/544] Plz hekb --- src/DataTypes/DataTypeCustomGeo.cpp | 38 +++++++++++++--------------- src/Functions/geometryFromColumn.cpp | 28 +++++++++++++++++--- src/Functions/geometryFromColumn.h | 14 +++++++++- 3 files changed, 54 insertions(+), 26 deletions(-) diff --git a/src/DataTypes/DataTypeCustomGeo.cpp b/src/DataTypes/DataTypeCustomGeo.cpp index 73d76e7e1e8..17fe30b508e 100644 --- a/src/DataTypes/DataTypeCustomGeo.cpp +++ b/src/DataTypes/DataTypeCustomGeo.cpp @@ -1,3 +1,6 @@ +#include + +#include #include #include #include @@ -9,29 +12,24 @@ namespace DB { -namespace +void DataTypeCustomPointSerialization::serializeText( + const IColumn & column, size_t row_num, WriteBuffer & ostr, const FormatSettings & settings) const { + nestedDataType()->serializeAsText(column, row_num, ostr, settings); +} -class DataTypeCustomPointSerialization : public DataTypeCustomSimpleTextSerialization +void DataTypeCustomPointSerialization::deserializeText( + IColumn & column, ReadBuffer & istr, const FormatSettings & settings) const { -public: - void serializeText(const IColumn & column, size_t row_num, WriteBuffer & ostr, const FormatSettings & settings) const override - { - nestedDataType()->serializeAsText(column, row_num, ostr, settings); - } + nestedDataType()->deserializeAsWholeText(column, istr, settings); +} - void deserializeText(IColumn & column, ReadBuffer & istr, const FormatSettings & settings) const override - { - nestedDataType()->deserializeAsWholeText(column, istr, settings); - } - - static DataTypePtr nestedDataType() - { - static auto data_type = DataTypePtr(std::make_unique( - DataTypes({std::make_unique(), std::make_unique()}))); - return data_type; - } -}; +DataTypePtr DataTypeCustomPointSerialization::nestedDataType() +{ + static auto data_type = DataTypePtr(std::make_unique( + DataTypes({std::make_unique(), std::make_unique()}))); + return data_type; +} class DataTypeCustomRingSerialization : public DataTypeCustomSimpleTextSerialization { @@ -93,8 +91,6 @@ public: } }; -} - void registerDataTypeDomainGeo(DataTypeFactory & factory) { // Custom type for point represented as its coordinates stored as Tuple(Float64, Float64) diff --git a/src/Functions/geometryFromColumn.cpp b/src/Functions/geometryFromColumn.cpp index 76b2d536413..e276a772f31 100644 --- a/src/Functions/geometryFromColumn.cpp +++ b/src/Functions/geometryFromColumn.cpp @@ -1,18 +1,24 @@ #include +#include + +#include namespace DB { - namespace { size_t getArrayDepth(DataTypePtr data_type, size_t max_depth) { + LOG_FATAL(&Poco::Logger::get("geometryFromColumn"), "start get depth"); size_t depth = 0; - while (isArray(data_type) && depth != max_depth + 1) + while (data_type && isArray(data_type) && depth != max_depth + 1) { + LOG_FATAL(&Poco::Logger::get("geometryFromColumn"), data_type->getName()); + depth++; data_type = static_cast(*data_type).getNestedType(); } - return max_depth; + LOG_FATAL(&Poco::Logger::get("geometryFromColumn"), "End get depth"); + return depth; } class ContainerCreator : public boost::static_visitor @@ -46,10 +52,24 @@ private: } +Float64PointFromColumnParser makePointFromColumnParser(const ColumnWithTypeAndName & col) +{ + auto wanted_data_type = DataTypeCustomPointSerialization::nestedDataType(); + + auto casted = castColumn(col, wanted_data_type); + LOG_FATAL(&Poco::Logger::get("geometryFromColumn"), col.type->getName() + " to " + wanted_data_type->getName()); + if (!casted) + { + throw Exception("Failed to cast " + col.type->getName() + " to " + wanted_data_type->getName(), ErrorCodes::ILLEGAL_COLUMN); + } + + return Float64PointFromColumnParser(*casted); +} + GeometryFromColumnParser makeGeometryFromColumnParser(const ColumnWithTypeAndName & col) { switch (getArrayDepth(col.type, 3)) { - case 0: return Float64PointFromColumnParser(*col.column); + case 0: return makePointFromColumnParser(col); case 1: return Float64RingFromColumnParser(*col.column); case 2: return Float64PolygonFromColumnParser(*col.column); case 3: return Float64MultiPolygonFromColumnParser(*col.column); diff --git a/src/Functions/geometryFromColumn.h b/src/Functions/geometryFromColumn.h index b29b8c81d52..6922ad905d9 100644 --- a/src/Functions/geometryFromColumn.h +++ b/src/Functions/geometryFromColumn.h @@ -13,6 +13,8 @@ #include #include #include +#include +#include namespace DB { @@ -32,11 +34,20 @@ class Float64PointFromColumnParser public: Float64PointFromColumnParser(const IColumn & col) { + // const auto & tuple_columns = static_cast(col).getColumns(); + const auto * tuple = checkAndGetColumn(&col); + + if (!tuple) + { + throw Exception("not stonks", ErrorCodes::ILLEGAL_COLUMN); + + } + const auto & tuple_columns = static_cast(col).getColumns(); if (tuple_columns.size() != 2) { - throw Exception("tuple size must be equal to 2", ErrorCodes::ILLEGAL_COLUMN); + throw Exception("tuple size is " + toString(tuple_columns.size()) + " != 2", ErrorCodes::ILLEGAL_COLUMN); } x = static_cast(*tuple_columns[0]).getData().data(); @@ -201,6 +212,7 @@ Float64Geometry createContainer(const GeometryFromColumnParser & parser); void get(const GeometryFromColumnParser & parser, Float64Geometry & container, size_t i); +Float64PointFromColumnParser makePointFromColumnParser(const ColumnWithTypeAndName & col); GeometryFromColumnParser makeGeometryFromColumnParser(const ColumnWithTypeAndName & col); } From e6f18d564df153d0c2e6b24c7dd8c769d0a733a5 Mon Sep 17 00:00:00 2001 From: Alexey Ilyukhov Date: Sun, 7 Jun 2020 19:52:11 +0300 Subject: [PATCH 249/544] Add missing header --- src/DataTypes/DataTypeCustomGeo.h | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) create mode 100644 src/DataTypes/DataTypeCustomGeo.h diff --git a/src/DataTypes/DataTypeCustomGeo.h b/src/DataTypes/DataTypeCustomGeo.h new file mode 100644 index 00000000000..f0aa76fa83d --- /dev/null +++ b/src/DataTypes/DataTypeCustomGeo.h @@ -0,0 +1,23 @@ +#include +#include +#include +#include +#include +#include +#include +#include + +namespace DB +{ + +class DataTypeCustomPointSerialization : public DataTypeCustomSimpleTextSerialization +{ +public: + void serializeText(const IColumn & column, size_t row_num, WriteBuffer & ostr, const FormatSettings & settings) const override; + + void deserializeText(IColumn & column, ReadBuffer & istr, const FormatSettings & settings) const override; + + static DataTypePtr nestedDataType(); +}; + +} From 3be3d5cde8788165bc0558f1e2a22568311c3103 Mon Sep 17 00:00:00 2001 From: Alexey Ilyukhov Date: Sun, 7 Jun 2020 20:28:05 +0300 Subject: [PATCH 250/544] Move all data types to header --- src/DataTypes/DataTypeCustomGeo.cpp | 98 +++++++++++++--------------- src/DataTypes/DataTypeCustomGeo.h | 31 +++++++++ src/Functions/geometryFromColumn.cpp | 17 +++-- src/Functions/geometryFromColumn.h | 9 --- src/Functions/wkt.cpp | 5 ++ 5 files changed, 90 insertions(+), 70 deletions(-) diff --git a/src/DataTypes/DataTypeCustomGeo.cpp b/src/DataTypes/DataTypeCustomGeo.cpp index 17fe30b508e..9f30ba4223a 100644 --- a/src/DataTypes/DataTypeCustomGeo.cpp +++ b/src/DataTypes/DataTypeCustomGeo.cpp @@ -31,65 +31,59 @@ DataTypePtr DataTypeCustomPointSerialization::nestedDataType() return data_type; } -class DataTypeCustomRingSerialization : public DataTypeCustomSimpleTextSerialization +void DataTypeCustomRingSerialization::serializeText( + const IColumn & column, size_t row_num, WriteBuffer & ostr, const FormatSettings & settings) const { -public: - void serializeText(const IColumn & column, size_t row_num, WriteBuffer & ostr, const FormatSettings & settings) const override - { - nestedDataType()->serializeAsText(column, row_num, ostr, settings); - } + nestedDataType()->serializeAsText(column, row_num, ostr, settings); +} - void deserializeText(IColumn & column, ReadBuffer & istr, const FormatSettings & settings) const override - { - nestedDataType()->deserializeAsWholeText(column, istr, settings); - } - - static DataTypePtr nestedDataType() - { - static auto data_type = DataTypePtr(std::make_unique(DataTypeCustomPointSerialization::nestedDataType())); - return data_type; - } -}; - -class DataTypeCustomPolygonSerialization : public DataTypeCustomSimpleTextSerialization +void DataTypeCustomRingSerialization::deserializeText( + IColumn & column, ReadBuffer & istr, const FormatSettings & settings) const { -public: - void serializeText(const IColumn & column, size_t row_num, WriteBuffer & ostr, const FormatSettings & settings) const override - { - nestedDataType()->serializeAsText(column, row_num, ostr, settings); - } + nestedDataType()->deserializeAsWholeText(column, istr, settings); +} - void deserializeText(IColumn & column, ReadBuffer & istr, const FormatSettings & settings) const override - { - nestedDataType()->deserializeAsWholeText(column, istr, settings); - } - - static DataTypePtr nestedDataType() - { - static auto data_type = DataTypePtr(std::make_unique(DataTypeCustomRingSerialization::nestedDataType())); - return data_type; - } -}; - -class DataTypeCustomMultiPolygonSerialization : public DataTypeCustomSimpleTextSerialization +DataTypePtr DataTypeCustomRingSerialization::nestedDataType() { -public: - void serializeText(const IColumn & column, size_t row_num, WriteBuffer & ostr, const FormatSettings & settings) const override - { - nestedDataType()->serializeAsText(column, row_num, ostr, settings); - } + static auto data_type = DataTypePtr(std::make_unique(DataTypeCustomPointSerialization::nestedDataType())); + return data_type; +} - void deserializeText(IColumn & column, ReadBuffer & istr, const FormatSettings & settings) const override - { - nestedDataType()->deserializeAsWholeText(column, istr, settings); - } +void DataTypeCustomPolygonSerialization::serializeText( + const IColumn & column, size_t row_num, WriteBuffer & ostr, const FormatSettings & settings) const +{ + nestedDataType()->serializeAsText(column, row_num, ostr, settings); +} - static DataTypePtr nestedDataType() - { - static auto data_type = DataTypePtr(std::make_unique(DataTypeCustomPolygonSerialization::nestedDataType())); - return data_type; - } -}; +void DataTypeCustomPolygonSerialization::deserializeText( + IColumn & column, ReadBuffer & istr, const FormatSettings & settings) const +{ + nestedDataType()->deserializeAsWholeText(column, istr, settings); +} + +DataTypePtr DataTypeCustomPolygonSerialization::nestedDataType() +{ + static auto data_type = DataTypePtr(std::make_unique(DataTypeCustomRingSerialization::nestedDataType())); + return data_type; +} + +void DataTypeCustomMultiPolygonSerialization::serializeText( + const IColumn & column, size_t row_num, WriteBuffer & ostr, const FormatSettings & settings) const +{ + nestedDataType()->serializeAsText(column, row_num, ostr, settings); +} + +void DataTypeCustomMultiPolygonSerialization::deserializeText( + IColumn & column, ReadBuffer & istr, const FormatSettings & settings) const +{ + nestedDataType()->deserializeAsWholeText(column, istr, settings); +} + +DataTypePtr DataTypeCustomMultiPolygonSerialization::nestedDataType() +{ + static auto data_type = DataTypePtr(std::make_unique(DataTypeCustomPolygonSerialization::nestedDataType())); + return data_type; +} void registerDataTypeDomainGeo(DataTypeFactory & factory) { diff --git a/src/DataTypes/DataTypeCustomGeo.h b/src/DataTypes/DataTypeCustomGeo.h index f0aa76fa83d..00ca898f53b 100644 --- a/src/DataTypes/DataTypeCustomGeo.h +++ b/src/DataTypes/DataTypeCustomGeo.h @@ -20,4 +20,35 @@ public: static DataTypePtr nestedDataType(); }; + +class DataTypeCustomRingSerialization : public DataTypeCustomSimpleTextSerialization +{ +public: + void serializeText(const IColumn & column, size_t row_num, WriteBuffer & ostr, const FormatSettings & settings) const override; + + void deserializeText(IColumn & column, ReadBuffer & istr, const FormatSettings & settings) const override; + + static DataTypePtr nestedDataType(); +}; + +class DataTypeCustomPolygonSerialization : public DataTypeCustomSimpleTextSerialization +{ +public: + void serializeText(const IColumn & column, size_t row_num, WriteBuffer & ostr, const FormatSettings & settings) const override; + + void deserializeText(IColumn & column, ReadBuffer & istr, const FormatSettings & settings) const override; + + static DataTypePtr nestedDataType(); +}; + +class DataTypeCustomMultiPolygonSerialization : public DataTypeCustomSimpleTextSerialization +{ +public: + void serializeText(const IColumn & column, size_t row_num, WriteBuffer & ostr, const FormatSettings & settings) const override; + + void deserializeText(IColumn & column, ReadBuffer & istr, const FormatSettings & settings) const override; + + static DataTypePtr nestedDataType(); +}; + } diff --git a/src/Functions/geometryFromColumn.cpp b/src/Functions/geometryFromColumn.cpp index e276a772f31..9e39a5df6d1 100644 --- a/src/Functions/geometryFromColumn.cpp +++ b/src/Functions/geometryFromColumn.cpp @@ -52,12 +52,11 @@ private: } -Float64PointFromColumnParser makePointFromColumnParser(const ColumnWithTypeAndName & col) +template +Float64PointFromColumnParser makeParser(const ColumnWithTypeAndName & col) { - auto wanted_data_type = DataTypeCustomPointSerialization::nestedDataType(); - - auto casted = castColumn(col, wanted_data_type); - LOG_FATAL(&Poco::Logger::get("geometryFromColumn"), col.type->getName() + " to " + wanted_data_type->getName()); + auto wanted_data_type = DataType::nestedDataType(); + ColumnPtr casted = castColumn(col, wanted_data_type); if (!casted) { throw Exception("Failed to cast " + col.type->getName() + " to " + wanted_data_type->getName(), ErrorCodes::ILLEGAL_COLUMN); @@ -69,10 +68,10 @@ Float64PointFromColumnParser makePointFromColumnParser(const ColumnWithTypeAndNa GeometryFromColumnParser makeGeometryFromColumnParser(const ColumnWithTypeAndName & col) { switch (getArrayDepth(col.type, 3)) { - case 0: return makePointFromColumnParser(col); - case 1: return Float64RingFromColumnParser(*col.column); - case 2: return Float64PolygonFromColumnParser(*col.column); - case 3: return Float64MultiPolygonFromColumnParser(*col.column); + case 0: return makeParser(col); + case 1: return makeParser(col); + case 2: return makeParser(col); + case 3: return makeParser(col); default: throw Exception("Cannot parse geometry from column with type " + col.type->getName() + ", array depth is too big", ErrorCodes::ILLEGAL_COLUMN); } diff --git a/src/Functions/geometryFromColumn.h b/src/Functions/geometryFromColumn.h index 6922ad905d9..dd3f997733c 100644 --- a/src/Functions/geometryFromColumn.h +++ b/src/Functions/geometryFromColumn.h @@ -34,15 +34,6 @@ class Float64PointFromColumnParser public: Float64PointFromColumnParser(const IColumn & col) { - // const auto & tuple_columns = static_cast(col).getColumns(); - const auto * tuple = checkAndGetColumn(&col); - - if (!tuple) - { - throw Exception("not stonks", ErrorCodes::ILLEGAL_COLUMN); - - } - const auto & tuple_columns = static_cast(col).getColumns(); if (tuple_columns.size() != 2) diff --git a/src/Functions/wkt.cpp b/src/Functions/wkt.cpp index f58395fb909..6054fead3d9 100644 --- a/src/Functions/wkt.cpp +++ b/src/Functions/wkt.cpp @@ -54,6 +54,11 @@ public: block.getByPosition(result).column = std::move(res_column); } + + bool useDefaultImplementationForConstants() const override + { + return true; + } }; From afad9bf4d0a55ed52a3f55483bc0973456e10a56 Mon Sep 17 00:00:00 2001 From: Alexey Ilyukhov Date: Sun, 7 Jun 2020 21:26:18 +0300 Subject: [PATCH 251/544] Trying to fix wkt --- src/Functions/geometryFromColumn.cpp | 18 +++++++++--------- src/Functions/geometryFromColumn.h | 8 +++++--- 2 files changed, 14 insertions(+), 12 deletions(-) diff --git a/src/Functions/geometryFromColumn.cpp b/src/Functions/geometryFromColumn.cpp index 9e39a5df6d1..ddaebe5d788 100644 --- a/src/Functions/geometryFromColumn.cpp +++ b/src/Functions/geometryFromColumn.cpp @@ -50,10 +50,8 @@ private: size_t i; }; -} - -template -Float64PointFromColumnParser makeParser(const ColumnWithTypeAndName & col) +template +Parser makeParser(const ColumnWithTypeAndName & col) { auto wanted_data_type = DataType::nestedDataType(); ColumnPtr casted = castColumn(col, wanted_data_type); @@ -62,16 +60,18 @@ Float64PointFromColumnParser makeParser(const ColumnWithTypeAndName & col) throw Exception("Failed to cast " + col.type->getName() + " to " + wanted_data_type->getName(), ErrorCodes::ILLEGAL_COLUMN); } - return Float64PointFromColumnParser(*casted); + return Parser(*casted); +} + } GeometryFromColumnParser makeGeometryFromColumnParser(const ColumnWithTypeAndName & col) { switch (getArrayDepth(col.type, 3)) { - case 0: return makeParser(col); - case 1: return makeParser(col); - case 2: return makeParser(col); - case 3: return makeParser(col); + case 0: return makeParser(col); + case 1: return makeParser(col); + case 2: return makeParser(col); + case 3: return makeParser(col); default: throw Exception("Cannot parse geometry from column with type " + col.type->getName() + ", array depth is too big", ErrorCodes::ILLEGAL_COLUMN); } diff --git a/src/Functions/geometryFromColumn.h b/src/Functions/geometryFromColumn.h index dd3f997733c..dbdb6bc9fb3 100644 --- a/src/Functions/geometryFromColumn.h +++ b/src/Functions/geometryFromColumn.h @@ -41,6 +41,7 @@ public: throw Exception("tuple size is " + toString(tuple_columns.size()) + " != 2", ErrorCodes::ILLEGAL_COLUMN); } + // x = static_cast(*tuple_columns[0]).getData().data(); x = static_cast(*tuple_columns[0]).getData().data(); if (!x) { @@ -67,7 +68,7 @@ public: void get(Float64Point & container, size_t i) const { boost::geometry::set<0>(container, x[i]); - boost::geometry::set<0>(container, y[i]); + boost::geometry::set<1>(container, y[i]); } private: const Float64 * x; @@ -81,7 +82,8 @@ public: RingFromColumnParser(const IColumn & col) : offsets(static_cast(col).getOffsets()) , pointParser(static_cast(col).getData()) - {} + { + } Geometry createContainer() const { @@ -103,6 +105,7 @@ public: container.resize(r - l); for (size_t j = l; j < r; j++) { + // LOG_FATAL pointParser.get(container[j - l], j); } @@ -203,7 +206,6 @@ Float64Geometry createContainer(const GeometryFromColumnParser & parser); void get(const GeometryFromColumnParser & parser, Float64Geometry & container, size_t i); -Float64PointFromColumnParser makePointFromColumnParser(const ColumnWithTypeAndName & col); GeometryFromColumnParser makeGeometryFromColumnParser(const ColumnWithTypeAndName & col); } From e3290ecc78ca3ea82b49ebcda22b5d3a4df154e6 Mon Sep 17 00:00:00 2001 From: Alexey Ilyukhov Date: Sun, 7 Jun 2020 22:48:45 +0300 Subject: [PATCH 252/544] help --- src/Functions/geometryFromColumn.cpp | 7 +++++-- src/Functions/geometryFromColumn.h | 24 ++++++++++++++++++++---- 2 files changed, 25 insertions(+), 6 deletions(-) diff --git a/src/Functions/geometryFromColumn.cpp b/src/Functions/geometryFromColumn.cpp index ddaebe5d788..f0155cb420d 100644 --- a/src/Functions/geometryFromColumn.cpp +++ b/src/Functions/geometryFromColumn.cpp @@ -1,7 +1,6 @@ #include #include -#include namespace DB { @@ -54,7 +53,11 @@ template Parser makeParser(const ColumnWithTypeAndName & col) { auto wanted_data_type = DataType::nestedDataType(); - ColumnPtr casted = castColumn(col, wanted_data_type); + ColumnPtr casted = castColumn(col, + std::make_shared(DataTypes{ + std::make_shared(), + std::make_shared()})); + LOG_FATAL(&Poco::Logger::get("geometryFromColumn"), col.type->getName() + " to " + wanted_data_type->getName()); if (!casted) { throw Exception("Failed to cast " + col.type->getName() + " to " + wanted_data_type->getName(), ErrorCodes::ILLEGAL_COLUMN); diff --git a/src/Functions/geometryFromColumn.h b/src/Functions/geometryFromColumn.h index dbdb6bc9fb3..2e0647a0533 100644 --- a/src/Functions/geometryFromColumn.h +++ b/src/Functions/geometryFromColumn.h @@ -16,6 +16,8 @@ #include #include +#include + namespace DB { namespace ErrorCodes @@ -34,21 +36,35 @@ class Float64PointFromColumnParser public: Float64PointFromColumnParser(const IColumn & col) { - const auto & tuple_columns = static_cast(col).getColumns(); + const auto * tuple = checkAndGetColumn(col); + if (!tuple) + { + throw Exception("not tuple ", ErrorCodes::ILLEGAL_COLUMN); + } + const auto & tuple_columns = tuple->getColumns(); if (tuple_columns.size() != 2) { throw Exception("tuple size is " + toString(tuple_columns.size()) + " != 2", ErrorCodes::ILLEGAL_COLUMN); } - // x = static_cast(*tuple_columns[0]).getData().data(); - x = static_cast(*tuple_columns[0]).getData().data(); + const auto * x_data = checkAndGetColumn(*tuple_columns[0]); + if (!x_data) + { + throw Exception("not x ", ErrorCodes::ILLEGAL_COLUMN); + } + x = x_data->getData().data(); if (!x) { throw Exception("failed to get x column", ErrorCodes::ILLEGAL_COLUMN); } - y = static_cast(*tuple_columns[1]).getData().data(); + const auto * y_data = checkAndGetColumn(*tuple_columns[1]); + if (!y_data) + { + throw Exception("not y ", ErrorCodes::ILLEGAL_COLUMN); + } + y = y_data->getData().data(); if (!y) { throw Exception("failed to get y column", ErrorCodes::ILLEGAL_COLUMN); From 125945769586baf6ffd15919b29565b1b2a63218 Mon Sep 17 00:00:00 2001 From: Alexey Ilyukhov Date: Sun, 14 Jun 2020 20:04:10 +0300 Subject: [PATCH 253/544] Start adding geo -> col --- ...yFromColumn.cpp => geometryConverters.cpp} | 48 +++++-- ...metryFromColumn.h => geometryConverters.h} | 136 ++++++++++++------ src/Functions/wkt.cpp | 2 +- src/Functions/ya.make | 2 +- 4 files changed, 130 insertions(+), 58 deletions(-) rename src/Functions/{geometryFromColumn.cpp => geometryConverters.cpp} (72%) rename src/Functions/{geometryFromColumn.h => geometryConverters.h} (61%) diff --git a/src/Functions/geometryFromColumn.cpp b/src/Functions/geometryConverters.cpp similarity index 72% rename from src/Functions/geometryFromColumn.cpp rename to src/Functions/geometryConverters.cpp index f0155cb420d..01fe29370a7 100644 --- a/src/Functions/geometryFromColumn.cpp +++ b/src/Functions/geometryConverters.cpp @@ -1,4 +1,4 @@ -#include +#include #include @@ -8,15 +8,12 @@ namespace { size_t getArrayDepth(DataTypePtr data_type, size_t max_depth) { - LOG_FATAL(&Poco::Logger::get("geometryFromColumn"), "start get depth"); size_t depth = 0; while (data_type && isArray(data_type) && depth != max_depth + 1) { - LOG_FATAL(&Poco::Logger::get("geometryFromColumn"), data_type->getName()); depth++; data_type = static_cast(*data_type).getNestedType(); } - LOG_FATAL(&Poco::Logger::get("geometryFromColumn"), "End get depth"); return depth; } @@ -33,7 +30,7 @@ public: class Getter : public boost::static_visitor { public: - Getter(Float64Geometry & container_, size_t i_) + constexpr Getter(Float64Geometry & container_, size_t i_) : container(container_) , i(i_) {} @@ -53,19 +50,46 @@ template Parser makeParser(const ColumnWithTypeAndName & col) { auto wanted_data_type = DataType::nestedDataType(); - ColumnPtr casted = castColumn(col, - std::make_shared(DataTypes{ - std::make_shared(), - std::make_shared()})); - LOG_FATAL(&Poco::Logger::get("geometryFromColumn"), col.type->getName() + " to " + wanted_data_type->getName()); + ColumnPtr casted = castColumn(col, DataType::nestedDataType()); if (!casted) { throw Exception("Failed to cast " + col.type->getName() + " to " + wanted_data_type->getName(), ErrorCodes::ILLEGAL_COLUMN); } - - return Parser(*casted); + return Parser(std::move(casted)); } +class Float64RingSerializer { +public: + Float64RingSerializer() + : offsets(ColumnUInt64::create()) + {} + + Float64RingSerializer(size_t n) + : offsets(ColumnUInt64::create(n)) + {} + + void add(const Float64Ring & ring) + { + size += ring.size(); + offsets->insertValue(size); + for (const auto & point : ring) + { + pointSerializer.add(point); + } + } + + ColumnPtr result() + { + return ColumnArray::create(pointSerializer.result(), std::move(offsets)); + } + +private: + size_t size; + Float64PointSerializer pointSerializer; + ColumnUInt64::MutablePtr offsets; +}; + + } GeometryFromColumnParser makeGeometryFromColumnParser(const ColumnWithTypeAndName & col) diff --git a/src/Functions/geometryFromColumn.h b/src/Functions/geometryConverters.h similarity index 61% rename from src/Functions/geometryFromColumn.h rename to src/Functions/geometryConverters.h index 2e0647a0533..37841b4aea9 100644 --- a/src/Functions/geometryFromColumn.h +++ b/src/Functions/geometryConverters.h @@ -16,13 +16,12 @@ #include #include -#include - namespace DB { namespace ErrorCodes { extern const int ILLEGAL_COLUMN; + extern const int BAD_ARGUMENTS; } using Float64Point = boost::geometry::model::d2::point_xy; @@ -34,43 +33,20 @@ using Float64Geometry = boost::variant(col); - if (!tuple) - { - throw Exception("not tuple ", ErrorCodes::ILLEGAL_COLUMN); - } - const auto & tuple_columns = tuple->getColumns(); + const auto & tuple = static_cast(*col_); + const auto & tuple_columns = tuple.getColumns(); - if (tuple_columns.size() != 2) - { - throw Exception("tuple size is " + toString(tuple_columns.size()) + " != 2", ErrorCodes::ILLEGAL_COLUMN); - } + const auto & x_data = static_cast(*tuple_columns[0]); + x = x_data.getData().data(); - const auto * x_data = checkAndGetColumn(*tuple_columns[0]); - if (!x_data) - { - throw Exception("not x ", ErrorCodes::ILLEGAL_COLUMN); - } - x = x_data->getData().data(); - if (!x) - { - throw Exception("failed to get x column", ErrorCodes::ILLEGAL_COLUMN); - } - - const auto * y_data = checkAndGetColumn(*tuple_columns[1]); - if (!y_data) - { - throw Exception("not y ", ErrorCodes::ILLEGAL_COLUMN); - } - y = y_data->getData().data(); - if (!y) - { - throw Exception("failed to get y column", ErrorCodes::ILLEGAL_COLUMN); - } + const auto & y_data = static_cast(*tuple_columns[1]); + y = y_data.getData().data(); } + Float64Geometry createContainer() const { return Float64Point(); @@ -87,6 +63,8 @@ public: boost::geometry::set<1>(container, y[i]); } private: + ColumnPtr col; + const Float64 * x; const Float64 * y; }; @@ -95,9 +73,9 @@ template class RingFromColumnParser { public: - RingFromColumnParser(const IColumn & col) - : offsets(static_cast(col).getOffsets()) - , pointParser(static_cast(col).getData()) + RingFromColumnParser(ColumnPtr col_) + : offsets(static_cast(*col_).getOffsets()) + , pointParser(static_cast(*col_).getDataPtr()) { } @@ -121,7 +99,6 @@ public: container.resize(r - l); for (size_t j = l; j < r; j++) { - // LOG_FATAL pointParser.get(container[j - l], j); } @@ -141,9 +118,9 @@ template class PolygonFromColumnParser { public: - PolygonFromColumnParser(const IColumn & col) - : offsets(static_cast(col).getOffsets()) - , ringParser(static_cast(col).getData()) + PolygonFromColumnParser(ColumnPtr col_) + : offsets(static_cast(*col_).getOffsets()) + , ringParser(static_cast(*col_).getDataPtr()) {} Geometry createContainer() const @@ -179,9 +156,9 @@ template class MultiPolygonFromColumnParser { public: - MultiPolygonFromColumnParser(const IColumn & col) - : offsets(static_cast(col).getOffsets()) - , polygonParser(static_cast(col).getData()) + MultiPolygonFromColumnParser(ColumnPtr col_) + : offsets(static_cast(*col_).getOffsets()) + , polygonParser(static_cast(*col_).getDataPtr()) {} Geometry createContainer() const @@ -224,4 +201,75 @@ void get(const GeometryFromColumnParser & parser, Float64Geometry & container, s GeometryFromColumnParser makeGeometryFromColumnParser(const ColumnWithTypeAndName & col); +class Float64PointSerializerVisitor : publicc boost::static_visitor +{ + Float64PointSerializerVisitor() + : x(ColumnFloat64::create()) + , y(ColumnFloat64::create()) + {} + + Float64PointSerializerVisitor(size_t n) + : x(ColumnFloat64::create(n)) + , y(ColumnFloat64::create(n)) + {} + + void operator()(const Float64Point & point) + { + x->insertValue(point.x()); + y->insertValue(point.y()); + } + + void operator()(const Float64Ring & ring) + { + if (ring.size() != 1) { + throw Exception("Unable to write ring of size " + toString(ring.size()) + " != 1 to point column", ErrorCodes::BAD_ARGUMENTS); + } + (*this)(ring[0]); + } + + void operator()(const Float64Polygon & polygon) + { + if (polygon.inners().size() != 0) { + throw Exception("Unable to write polygon with holes to point column", ErrorCodes::BAD_ARGUMENTS); + } + (*this)(polygon.outer()); + } + + void operator()(const Float64MultiPolygon & multi_polygon) + { + if (multi_polygon.size() != 1) { + throw Exception("Unable to write multi-polygon of size " + toString(multi_polygon.size()) + " != 1 to point column", ErrorCodes::BAD_ARGUMENTS); + } + (*this)(multi_polygon[0]); + } + + ColumnPtr finalize() + { + Columns columns(2); + columns[0] = std::move(x); + columns[1] = std::move(y); + + return ColumnTuple::create(columns); + } +} + +template +class GeometrySerializer +{ +public: + void add(const Geometry & geometry) + { + boost::apply_visitor(visitor, geometry); + } + + ColumnPtr finalize() + { + return visitor.finalize(); + } +private: + Visitor visitor; +} + +using Float64PointSerializer = GeometrySerializer; + } diff --git a/src/Functions/wkt.cpp b/src/Functions/wkt.cpp index 6054fead3d9..30b8d2e1a7e 100644 --- a/src/Functions/wkt.cpp +++ b/src/Functions/wkt.cpp @@ -1,6 +1,6 @@ #include #include -#include +#include #include #include diff --git a/src/Functions/ya.make b/src/Functions/ya.make index 2f3c6348206..53a92780c18 100644 --- a/src/Functions/ya.make +++ b/src/Functions/ya.make @@ -268,7 +268,7 @@ SRCS( geohashDecode.cpp geohashEncode.cpp geohashesInBox.cpp - geometryFromColumn.cpp + geometryConverters.cpp getMacro.cpp getScalar.cpp getSetting.cpp From d46998e114ad397b930b34c7eace7c967af91c54 Mon Sep 17 00:00:00 2001 From: Alexey Ilyukhov Date: Sun, 14 Jun 2020 20:37:15 +0300 Subject: [PATCH 254/544] Add remaining geo -> col --- src/Functions/geometryConverters.cpp | 32 ----- src/Functions/geometryConverters.h | 178 ++++++++++++++++++++++++++- 2 files changed, 175 insertions(+), 35 deletions(-) diff --git a/src/Functions/geometryConverters.cpp b/src/Functions/geometryConverters.cpp index 01fe29370a7..6879611890c 100644 --- a/src/Functions/geometryConverters.cpp +++ b/src/Functions/geometryConverters.cpp @@ -58,38 +58,6 @@ Parser makeParser(const ColumnWithTypeAndName & col) return Parser(std::move(casted)); } -class Float64RingSerializer { -public: - Float64RingSerializer() - : offsets(ColumnUInt64::create()) - {} - - Float64RingSerializer(size_t n) - : offsets(ColumnUInt64::create(n)) - {} - - void add(const Float64Ring & ring) - { - size += ring.size(); - offsets->insertValue(size); - for (const auto & point : ring) - { - pointSerializer.add(point); - } - } - - ColumnPtr result() - { - return ColumnArray::create(pointSerializer.result(), std::move(offsets)); - } - -private: - size_t size; - Float64PointSerializer pointSerializer; - ColumnUInt64::MutablePtr offsets; -}; - - } GeometryFromColumnParser makeGeometryFromColumnParser(const ColumnWithTypeAndName & col) diff --git a/src/Functions/geometryConverters.h b/src/Functions/geometryConverters.h index 37841b4aea9..2df5aea53ab 100644 --- a/src/Functions/geometryConverters.h +++ b/src/Functions/geometryConverters.h @@ -201,8 +201,9 @@ void get(const GeometryFromColumnParser & parser, Float64Geometry & container, s GeometryFromColumnParser makeGeometryFromColumnParser(const ColumnWithTypeAndName & col); -class Float64PointSerializerVisitor : publicc boost::static_visitor +class Float64PointSerializerVisitor : public boost::static_visitor { +public: Float64PointSerializerVisitor() : x(ColumnFloat64::create()) , y(ColumnFloat64::create()) @@ -251,7 +252,175 @@ class Float64PointSerializerVisitor : publicc boost::static_visitor return ColumnTuple::create(columns); } -} + +private: + ColumnFloat64::MutablePtr x; + ColumnFloat64::MutablePtr y; +}; + +class Float64RingSerializerVisitor : public boost::static_visitor +{ +public: + Float64RingSerializerVisitor() + : offsets(ColumnUInt64::create()) + {} + + Float64RingSerializerVisitor(size_t n) + : offsets(ColumnUInt64::create(n)) + {} + + void operator()(const Float64Point & point) + { + size++; + offsets->insertValue(size); + + pointSerializer(point); + } + + void operator()(const Float64Ring & ring) + { + size += ring.size(); + offsets->insertValue(size); + for (const auto & point : ring) + { + pointSerializer(point); + } + } + + void operator()(const Float64Polygon & polygon) + { + if (polygon.inners().size() != 0) { + throw Exception("Unable to write polygon with holes to ring column", ErrorCodes::BAD_ARGUMENTS); + } + (*this)(polygon.outer()); + } + + void operator()(const Float64MultiPolygon & multi_polygon) + { + if (multi_polygon.size() != 1) { + throw Exception("Unable to write multi-polygon of size " + toString(multi_polygon.size()) + " != 1 to ring column", ErrorCodes::BAD_ARGUMENTS); + } + (*this)(multi_polygon[0]); + } + + ColumnPtr finalize() + { + return ColumnArray::create(pointSerializer.finalize(), std::move(offsets)); + } + +private: + size_t size; + Float64PointSerializerVisitor pointSerializer; + ColumnUInt64::MutablePtr offsets; +}; + +class Float64PolygonSerializerVisitor : public boost::static_visitor +{ +public: + Float64PolygonSerializerVisitor() + : offsets(ColumnUInt64::create()) + {} + + Float64PolygonSerializerVisitor(size_t n) + : offsets(ColumnUInt64::create(n)) + {} + + void operator()(const Float64Point & point) + { + size++; + offsets->insertValue(size); + ringSerializer(point); + } + + void operator()(const Float64Ring & ring) + { + size++; + offsets->insertValue(size); + ringSerializer(ring); + } + + void operator()(const Float64Polygon & polygon) + { + size += 1 + polygon.inners().size(); + offsets->insertValue(size); + ringSerializer(polygon.outer()); + for (const auto & ring : polygon.inners()) + { + ringSerializer(ring); + } + } + + void operator()(const Float64MultiPolygon & multi_polygon) + { + if (multi_polygon.size() != 1) { + throw Exception("Unable to write multi-polygon of size " + toString(multi_polygon.size()) + " != 1 to polygon column", ErrorCodes::BAD_ARGUMENTS); + } + (*this)(multi_polygon[0]); + } + + ColumnPtr finalize() + { + return ColumnArray::create(ringSerializer.finalize(), std::move(offsets)); + } + +private: + size_t size; + Float64RingSerializerVisitor ringSerializer; + ColumnUInt64::MutablePtr offsets; +}; + +class Float64MultiPolygonSerializerVisitor : public boost::static_visitor +{ +public: + Float64MultiPolygonSerializerVisitor() + : offsets(ColumnUInt64::create()) + {} + + Float64MultiPolygonSerializerVisitor(size_t n) + : offsets(ColumnUInt64::create(n)) + {} + + void operator()(const Float64Point & point) + { + size++; + offsets->insertValue(size); + polygonSerializer(point); + } + + void operator()(const Float64Ring & ring) + { + size++; + offsets->insertValue(size); + polygonSerializer(ring); + } + + void operator()(const Float64Polygon & polygon) + { + size++; + offsets->insertValue(size); + polygonSerializer(polygon); + } + + void operator()(const Float64MultiPolygon & multi_polygon) + { + size += 1 + multi_polygon.size(); + offsets->insertValue(size); + for (const auto & polygon : multi_polygon) + { + polygonSerializer(polygon); + } + } + + ColumnPtr finalize() + { + return ColumnArray::create(polygonSerializer.finalize(), std::move(offsets)); + } + +private: + size_t size; + Float64PolygonSerializerVisitor polygonSerializer; + ColumnUInt64::MutablePtr offsets; +}; template class GeometrySerializer @@ -268,8 +437,11 @@ public: } private: Visitor visitor; -} +}; using Float64PointSerializer = GeometrySerializer; +using Float64RingSerializer = GeometrySerializer; +using Float64PolygonSerializer = GeometrySerializer; +using Float64MultiPolygonSerializer = GeometrySerializer; } From 13b841f6deb30c9c0c43e42408566cb3f96fc9b2 Mon Sep 17 00:00:00 2001 From: DoomzD Date: Wed, 17 Jun 2020 18:29:08 +0300 Subject: [PATCH 255/544] Add polygons intersection function --- src/Functions/PolygonsIntersection.cpp | 112 +++++++++++++++++++++++++ src/Functions/registerFunctionsGeo.cpp | 2 + 2 files changed, 114 insertions(+) create mode 100644 src/Functions/PolygonsIntersection.cpp diff --git a/src/Functions/PolygonsIntersection.cpp b/src/Functions/PolygonsIntersection.cpp new file mode 100644 index 00000000000..5e43470fadd --- /dev/null +++ b/src/Functions/PolygonsIntersection.cpp @@ -0,0 +1,112 @@ +#include +#include + +#include +#include +#include + +#include +#include +#include +#include +#include +#include + +#include + +namespace DB +{ + +namespace ErrorCodes +{ + extern const int TOO_FEW_ARGUMENTS_FOR_FUNCTION; + extern const int BAD_ARGUMENTS; + extern const int ILLEGAL_TYPE_OF_ARGUMENT; + extern const int ILLEGAL_COLUMN; +} + + +using CoordinateType = Float64; +using Point = boost::geometry::model::d2::point_xy; +using Polygon = boost::geometry::model::polygon; +using MultiPolygon = boost::geometry::model::multi_polygon; +using Box = boost::geometry::model::box; + + +class FunctionPolygonsIntersection : public IFunction +{ +public: + static inline const char * name = "polygonsIntersection"; + + explicit FunctionPolygonsIntersection() = default; + + static FunctionPtr create(const Context &) + { + return std::make_shared(); + } + + String getName() const override + { + return name; + } + + bool isVariadic() const override + { + return false; + } + + size_t getNumberOfArguments() const override + { + return 2; + } + + DataTypePtr getReturnTypeImpl(const DataTypes &) const override + { + return DataTypeCustomMultiPolygonSerialization::nestedDataType(); + } + + void executeImpl(Block & block, const ColumnNumbers & arguments, size_t result, size_t input_rows_count) override + { + auto get_parser = [&block, &arguments] (size_t i) { + const ColumnWithTypeAndName polygon = block.getByPosition(arguments[i]); + return makeGeometryFromColumnParser(polygon); + }; + + auto first_parser = get_parser(0); + auto first_container = createContainer(first_parser); + + auto second_parser = get_parser(1); + auto second_container = createContainer(second_parser); + + auto res_column = ColumnArray::create(ColumnArray::create(ColumnArray::create( + ColumnTuple::create( + Columns{ColumnVector::create(input_rows_count), + ColumnVector::create(input_rows_count)} + ) + ))); + auto & data = res_column->getData(); + + for (size_t i = 0; i < input_rows_count; i++) + { + get(first_parser, first_container, i); + get(second_parser, second_container, i); + + Float64Geometry intersection; + boost::geometry::intersection(first_container, second_container, intersection); + + // GeometrySerializer serializer; + // serializer.add(intersection); + // data[i] = serializer.finalize(); + } + + // block.getByPosition(result).column = std::move(res_column); + } +}; + + +void registerFunctionPolygonsIntersection(FunctionFactory & factory) +{ + factory.registerFunction(); +} + +} diff --git a/src/Functions/registerFunctionsGeo.cpp b/src/Functions/registerFunctionsGeo.cpp index 57cf1ff1b09..f37f5b542b2 100644 --- a/src/Functions/registerFunctionsGeo.cpp +++ b/src/Functions/registerFunctionsGeo.cpp @@ -10,6 +10,7 @@ class FunctionFactory; void registerFunctionGeoDistance(FunctionFactory & factory); void registerFunctionPointInEllipses(FunctionFactory & factory); void registerFunctionPointInPolygon(FunctionFactory & factory); +// void registerFunctionPolygonsIntersection(FunctionFactory & factory); void registerFunctionGeohashEncode(FunctionFactory & factory); void registerFunctionGeohashDecode(FunctionFactory & factory); void registerFunctionGeohashesInBox(FunctionFactory & factory); @@ -37,6 +38,7 @@ void registerFunctionsGeo(FunctionFactory & factory) registerFunctionGeoDistance(factory); registerFunctionPointInEllipses(factory); registerFunctionPointInPolygon(factory); + // registerFunctionPolygonsIntersection(factory); registerFunctionGeohashEncode(factory); registerFunctionGeohashDecode(factory); registerFunctionGeohashesInBox(factory); From 249b94a7cd07500dfc58ec8352b0b30cf9a22fa8 Mon Sep 17 00:00:00 2001 From: DoomzD Date: Wed, 17 Jun 2020 19:29:31 +0300 Subject: [PATCH 256/544] register function polygons intersection --- src/Functions/PolygonsIntersection.cpp | 33 ++++++++++++-------------- src/Functions/registerFunctionsGeo.cpp | 4 ++-- 2 files changed, 17 insertions(+), 20 deletions(-) diff --git a/src/Functions/PolygonsIntersection.cpp b/src/Functions/PolygonsIntersection.cpp index 5e43470fadd..396500a2a4d 100644 --- a/src/Functions/PolygonsIntersection.cpp +++ b/src/Functions/PolygonsIntersection.cpp @@ -67,24 +67,20 @@ public: void executeImpl(Block & block, const ColumnNumbers & arguments, size_t result, size_t input_rows_count) override { - auto get_parser = [&block, &arguments] (size_t i) { - const ColumnWithTypeAndName polygon = block.getByPosition(arguments[i]); - return makeGeometryFromColumnParser(polygon); - }; + // auto get_parser = [&block, &arguments] (size_t i) { + // const ColumnWithTypeAndName polygon = block.getByPosition(arguments[i]); + // return makeGeometryFromColumnParser(polygon); + // }; - auto first_parser = get_parser(0); + const ColumnWithTypeAndName polygon = block.getByPosition(arguments[0]); + auto first_parser = makeGeometryFromColumnParser(polygon); auto first_container = createContainer(first_parser); - auto second_parser = get_parser(1); + const ColumnWithTypeAndName polygon2 = block.getByPosition(arguments[1]); + auto second_parser = makeGeometryFromColumnParser(polygon2); auto second_container = createContainer(second_parser); - auto res_column = ColumnArray::create(ColumnArray::create(ColumnArray::create( - ColumnTuple::create( - Columns{ColumnVector::create(input_rows_count), - ColumnVector::create(input_rows_count)} - ) - ))); - auto & data = res_column->getData(); + Float64MultiPolygonSerializer serializer; for (size_t i = 0; i < input_rows_count; i++) { @@ -92,14 +88,15 @@ public: get(second_parser, second_container, i); Float64Geometry intersection; - boost::geometry::intersection(first_container, second_container, intersection); + boost::geometry::intersection( + boost::get(first_container), + boost::get(second_container), + boost::get(intersection)); - // GeometrySerializer serializer; - // serializer.add(intersection); - // data[i] = serializer.finalize(); + serializer.add(intersection); } - // block.getByPosition(result).column = std::move(res_column); + block.getByPosition(result).column = std::move(serializer.finalize()); } }; diff --git a/src/Functions/registerFunctionsGeo.cpp b/src/Functions/registerFunctionsGeo.cpp index f37f5b542b2..1a573c76ebc 100644 --- a/src/Functions/registerFunctionsGeo.cpp +++ b/src/Functions/registerFunctionsGeo.cpp @@ -10,7 +10,7 @@ class FunctionFactory; void registerFunctionGeoDistance(FunctionFactory & factory); void registerFunctionPointInEllipses(FunctionFactory & factory); void registerFunctionPointInPolygon(FunctionFactory & factory); -// void registerFunctionPolygonsIntersection(FunctionFactory & factory); +void registerFunctionPolygonsIntersection(FunctionFactory & factory); void registerFunctionGeohashEncode(FunctionFactory & factory); void registerFunctionGeohashDecode(FunctionFactory & factory); void registerFunctionGeohashesInBox(FunctionFactory & factory); @@ -38,7 +38,7 @@ void registerFunctionsGeo(FunctionFactory & factory) registerFunctionGeoDistance(factory); registerFunctionPointInEllipses(factory); registerFunctionPointInPolygon(factory); - // registerFunctionPolygonsIntersection(factory); + registerFunctionPolygonsIntersection(factory); registerFunctionGeohashEncode(factory); registerFunctionGeohashDecode(factory); registerFunctionGeohashesInBox(factory); From 50c59cc922b153621e334356f5f3ad5f2e43f898 Mon Sep 17 00:00:00 2001 From: DoomzD Date: Sun, 21 Jun 2020 17:54:13 +0300 Subject: [PATCH 257/544] debug polygons intersection --- src/Functions/PolygonsIntersection.cpp | 25 ++++++++++++++++++++----- 1 file changed, 20 insertions(+), 5 deletions(-) diff --git a/src/Functions/PolygonsIntersection.cpp b/src/Functions/PolygonsIntersection.cpp index 396500a2a4d..2d5d69d924b 100644 --- a/src/Functions/PolygonsIntersection.cpp +++ b/src/Functions/PolygonsIntersection.cpp @@ -5,6 +5,8 @@ #include #include +#include + #include #include #include @@ -72,14 +74,18 @@ public: // return makeGeometryFromColumnParser(polygon); // }; - const ColumnWithTypeAndName polygon = block.getByPosition(arguments[0]); - auto first_parser = makeGeometryFromColumnParser(polygon); + LOG_FATAL(&Poco::Logger::get("PI"), "KEK"); + + auto first_parser = makeGeometryFromColumnParser(block.getByPosition(arguments[0])); auto first_container = createContainer(first_parser); - const ColumnWithTypeAndName polygon2 = block.getByPosition(arguments[1]); - auto second_parser = makeGeometryFromColumnParser(polygon2); + LOG_FATAL(&Poco::Logger::get("PI"), "LOL"); + + auto second_parser = makeGeometryFromColumnParser(block.getByPosition(arguments[1])); auto second_container = createContainer(second_parser); + LOG_FATAL(&Poco::Logger::get("PI"), "MEM"); + Float64MultiPolygonSerializer serializer; for (size_t i = 0; i < input_rows_count; i++) @@ -87,7 +93,7 @@ public: get(first_parser, first_container, i); get(second_parser, second_container, i); - Float64Geometry intersection; + Float64Geometry intersection = Float64MultiPolygon({{{{}}}}); boost::geometry::intersection( boost::get(first_container), boost::get(second_container), @@ -96,7 +102,16 @@ public: serializer.add(intersection); } + LOG_FATAL(&Poco::Logger::get("PI"), "NE MEM"); + block.getByPosition(result).column = std::move(serializer.finalize()); + + LOG_FATAL(&Poco::Logger::get("PI"), "THE END"); + } + + bool useDefaultImplementationForConstants() const override + { + return true; } }; From 97a9110ad9fea396329d6be0423f460c5db74250 Mon Sep 17 00:00:00 2001 From: Alexey Ilyukhov Date: Sun, 21 Jun 2020 18:18:09 +0300 Subject: [PATCH 258/544] Fix outstandingly wrong code --- src/Functions/geometryConverters.h | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/src/Functions/geometryConverters.h b/src/Functions/geometryConverters.h index 2df5aea53ab..9a683856ca0 100644 --- a/src/Functions/geometryConverters.h +++ b/src/Functions/geometryConverters.h @@ -15,6 +15,7 @@ #include #include #include +#include namespace DB { @@ -309,7 +310,7 @@ public: } private: - size_t size; + size_t size = 0; Float64PointSerializerVisitor pointSerializer; ColumnUInt64::MutablePtr offsets; }; @@ -364,7 +365,7 @@ public: } private: - size_t size; + size_t size = 0; Float64RingSerializerVisitor ringSerializer; ColumnUInt64::MutablePtr offsets; }; @@ -413,11 +414,12 @@ public: ColumnPtr finalize() { + LOG_FATAL(&Poco::Logger::get("PI"), "MultiPolygon Offsets: " + toString(size)); return ColumnArray::create(polygonSerializer.finalize(), std::move(offsets)); } private: - size_t size; + size_t size = 0; Float64PolygonSerializerVisitor polygonSerializer; ColumnUInt64::MutablePtr offsets; }; From 31f8e6330c9178b1560a8bb254e43018a0dfa0f0 Mon Sep 17 00:00:00 2001 From: Alexey Ilyukhov Date: Sun, 21 Jun 2020 18:22:00 +0300 Subject: [PATCH 259/544] Fix less outstanding but still wrong code --- src/Functions/geometryConverters.h | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/src/Functions/geometryConverters.h b/src/Functions/geometryConverters.h index 9a683856ca0..e74e43ee3de 100644 --- a/src/Functions/geometryConverters.h +++ b/src/Functions/geometryConverters.h @@ -15,7 +15,6 @@ #include #include #include -#include namespace DB { @@ -404,7 +403,7 @@ public: void operator()(const Float64MultiPolygon & multi_polygon) { - size += 1 + multi_polygon.size(); + size += multi_polygon.size(); offsets->insertValue(size); for (const auto & polygon : multi_polygon) { @@ -414,7 +413,6 @@ public: ColumnPtr finalize() { - LOG_FATAL(&Poco::Logger::get("PI"), "MultiPolygon Offsets: " + toString(size)); return ColumnArray::create(polygonSerializer.finalize(), std::move(offsets)); } From aafceae01946d06d5a336380df46065582cfb7db Mon Sep 17 00:00:00 2001 From: DoomzD Date: Sun, 21 Jun 2020 19:36:43 +0300 Subject: [PATCH 260/544] Working version of polygonsIntersection --- src/Functions/PolygonsIntersection.cpp | 26 ++++++++++---------------- 1 file changed, 10 insertions(+), 16 deletions(-) diff --git a/src/Functions/PolygonsIntersection.cpp b/src/Functions/PolygonsIntersection.cpp index 2d5d69d924b..086898bebcc 100644 --- a/src/Functions/PolygonsIntersection.cpp +++ b/src/Functions/PolygonsIntersection.cpp @@ -15,6 +15,7 @@ #include #include +#include namespace DB { @@ -69,23 +70,17 @@ public: void executeImpl(Block & block, const ColumnNumbers & arguments, size_t result, size_t input_rows_count) override { - // auto get_parser = [&block, &arguments] (size_t i) { - // const ColumnWithTypeAndName polygon = block.getByPosition(arguments[i]); - // return makeGeometryFromColumnParser(polygon); - // }; + auto get_parser = [&block, &arguments] (size_t i) { + const ColumnWithTypeAndName polygon = block.getByPosition(arguments[i]); + return makeGeometryFromColumnParser(polygon); + }; - LOG_FATAL(&Poco::Logger::get("PI"), "KEK"); - - auto first_parser = makeGeometryFromColumnParser(block.getByPosition(arguments[0])); + auto first_parser = get_parser(0); auto first_container = createContainer(first_parser); - LOG_FATAL(&Poco::Logger::get("PI"), "LOL"); - - auto second_parser = makeGeometryFromColumnParser(block.getByPosition(arguments[1])); + auto second_parser = get_parser(1); auto second_container = createContainer(second_parser); - LOG_FATAL(&Poco::Logger::get("PI"), "MEM"); - Float64MultiPolygonSerializer serializer; for (size_t i = 0; i < input_rows_count; i++) @@ -99,14 +94,13 @@ public: boost::get(second_container), boost::get(intersection)); + boost::get(intersection).erase( + boost::get(intersection).begin()); + serializer.add(intersection); } - LOG_FATAL(&Poco::Logger::get("PI"), "NE MEM"); - block.getByPosition(result).column = std::move(serializer.finalize()); - - LOG_FATAL(&Poco::Logger::get("PI"), "THE END"); } bool useDefaultImplementationForConstants() const override From bc1566b1f1454f41303785bb441ab03b6cd9ce14 Mon Sep 17 00:00:00 2001 From: DoomzD Date: Sun, 21 Jun 2020 20:25:05 +0300 Subject: [PATCH 261/544] Add polygonsUnion function --- src/Functions/PolygonsUnion.cpp | 115 +++++++++++++++++++++++++ src/Functions/registerFunctionsGeo.cpp | 2 + 2 files changed, 117 insertions(+) create mode 100644 src/Functions/PolygonsUnion.cpp diff --git a/src/Functions/PolygonsUnion.cpp b/src/Functions/PolygonsUnion.cpp new file mode 100644 index 00000000000..f8ff2f43b05 --- /dev/null +++ b/src/Functions/PolygonsUnion.cpp @@ -0,0 +1,115 @@ +#include +#include + +#include +#include +#include + +#include + +#include +#include +#include +#include +#include +#include + +#include +#include + +namespace DB +{ + +namespace ErrorCodes +{ + extern const int TOO_FEW_ARGUMENTS_FOR_FUNCTION; + extern const int BAD_ARGUMENTS; + extern const int ILLEGAL_TYPE_OF_ARGUMENT; + extern const int ILLEGAL_COLUMN; +} + + +using CoordinateType = Float64; +using Point = boost::geometry::model::d2::point_xy; +using Polygon = boost::geometry::model::polygon; +using MultiPolygon = boost::geometry::model::multi_polygon; +using Box = boost::geometry::model::box; + + +class FunctionPolygonsUnion : public IFunction +{ +public: + static inline const char * name = "polygonsUnion"; + + explicit FunctionPolygonsUnion() = default; + + static FunctionPtr create(const Context &) + { + return std::make_shared(); + } + + String getName() const override + { + return name; + } + + bool isVariadic() const override + { + return false; + } + + size_t getNumberOfArguments() const override + { + return 2; + } + + DataTypePtr getReturnTypeImpl(const DataTypes &) const override + { + return DataTypeCustomMultiPolygonSerialization::nestedDataType(); + } + + void executeImpl(Block & block, const ColumnNumbers & arguments, size_t result, size_t input_rows_count) override + { + auto get_parser = [&block, &arguments] (size_t i) { + const ColumnWithTypeAndName polygon = block.getByPosition(arguments[i]); + return makeGeometryFromColumnParser(polygon); + }; + + auto first_parser = get_parser(0); + auto first_container = createContainer(first_parser); + + auto second_parser = get_parser(1); + auto second_container = createContainer(second_parser); + + Float64MultiPolygonSerializer serializer; + + for (size_t i = 0; i < input_rows_count; i++) + { + get(first_parser, first_container, i); + get(second_parser, second_container, i); + + Float64Geometry polygons_union = Float64MultiPolygon({{{{}}}}); + boost::geometry::union_( + boost::get(first_container), + boost::get(second_container), + boost::get(polygons_union)); + + serializer.add(polygons_union); + } + + block.getByPosition(result).column = std::move(serializer.finalize()); + } + + bool useDefaultImplementationForConstants() const override + { + return true; + } +}; + + +void registerFunctionPolygonsUnion(FunctionFactory & factory) +{ + factory.registerFunction(); +} + +} diff --git a/src/Functions/registerFunctionsGeo.cpp b/src/Functions/registerFunctionsGeo.cpp index 1a573c76ebc..7ce0372c51d 100644 --- a/src/Functions/registerFunctionsGeo.cpp +++ b/src/Functions/registerFunctionsGeo.cpp @@ -11,6 +11,7 @@ void registerFunctionGeoDistance(FunctionFactory & factory); void registerFunctionPointInEllipses(FunctionFactory & factory); void registerFunctionPointInPolygon(FunctionFactory & factory); void registerFunctionPolygonsIntersection(FunctionFactory & factory); +void registerFunctionPolygonsUnion(FunctionFactory & factory); void registerFunctionGeohashEncode(FunctionFactory & factory); void registerFunctionGeohashDecode(FunctionFactory & factory); void registerFunctionGeohashesInBox(FunctionFactory & factory); @@ -39,6 +40,7 @@ void registerFunctionsGeo(FunctionFactory & factory) registerFunctionPointInEllipses(factory); registerFunctionPointInPolygon(factory); registerFunctionPolygonsIntersection(factory); + registerFunctionPolygonsUnion(factory); registerFunctionGeohashEncode(factory); registerFunctionGeohashDecode(factory); registerFunctionGeohashesInBox(factory); From 32f67a0fe5edbe457f6790f193d037bbedb1fcd4 Mon Sep 17 00:00:00 2001 From: DoomzD Date: Sun, 21 Jun 2020 20:26:11 +0300 Subject: [PATCH 262/544] Fix polygonsUnion first output polygon --- src/Functions/PolygonsUnion.cpp | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/Functions/PolygonsUnion.cpp b/src/Functions/PolygonsUnion.cpp index f8ff2f43b05..f2f6c28e91e 100644 --- a/src/Functions/PolygonsUnion.cpp +++ b/src/Functions/PolygonsUnion.cpp @@ -94,6 +94,9 @@ public: boost::get(second_container), boost::get(polygons_union)); + boost::get(polygons_union).erase( + boost::get(polygons_union).begin()); + serializer.add(polygons_union); } From 938ca6866d9011b3d5e171e149d23690b67bbccd Mon Sep 17 00:00:00 2001 From: Alexey Ilyukhov Date: Sun, 21 Jun 2020 20:59:18 +0300 Subject: [PATCH 263/544] Fix less outstanding but still wrong code again --- src/Functions/geometryConverters.cpp | 5 +++++ src/Functions/geometryConverters.h | 10 +++++++++- 2 files changed, 14 insertions(+), 1 deletion(-) diff --git a/src/Functions/geometryConverters.cpp b/src/Functions/geometryConverters.cpp index 6879611890c..12ef6f17d41 100644 --- a/src/Functions/geometryConverters.cpp +++ b/src/Functions/geometryConverters.cpp @@ -1,6 +1,7 @@ #include #include +#include namespace DB { @@ -8,12 +9,15 @@ namespace { size_t getArrayDepth(DataTypePtr data_type, size_t max_depth) { + LOG_FATAL(&Poco::Logger::get("geoconv"), "getting depth"); size_t depth = 0; while (data_type && isArray(data_type) && depth != max_depth + 1) { depth++; data_type = static_cast(*data_type).getNestedType(); } + LOG_FATAL(&Poco::Logger::get("geoconv"), "got depth"); + return depth; } @@ -62,6 +66,7 @@ Parser makeParser(const ColumnWithTypeAndName & col) GeometryFromColumnParser makeGeometryFromColumnParser(const ColumnWithTypeAndName & col) { + LOG_FATAL(&Poco::Logger::get("geoconv"), "Fine"); switch (getArrayDepth(col.type, 3)) { case 0: return makeParser(col); case 1: return makeParser(col); diff --git a/src/Functions/geometryConverters.h b/src/Functions/geometryConverters.h index e74e43ee3de..c5c785aea59 100644 --- a/src/Functions/geometryConverters.h +++ b/src/Functions/geometryConverters.h @@ -16,6 +16,8 @@ #include #include +#include + namespace DB { namespace ErrorCodes @@ -98,6 +100,8 @@ public: container.reserve(r - l + 1); container.resize(r - l); + LOG_FATAL(&Poco::Logger::get("geoconv"), "ring: l = {}, r = {}", l, r); + for (size_t j = l; j < r; j++) { pointParser.get(container[j - l], j); } @@ -140,6 +144,8 @@ public: ringParser.get(container.outer(), l); + LOG_FATAL(&Poco::Logger::get("geoconv"), "polygon: l = {}, r = {}", l, r); + container.inners().resize(r - l - 1); for (size_t j = l + 1; j < r; j++) { @@ -172,10 +178,12 @@ public: size_t l = offsets[i - 1]; size_t r = offsets[i]; + LOG_FATAL(&Poco::Logger::get("geoconv"), "multipolygon: l = {}, r = {}", l, r); + multi_polygon.resize(r - l); for (size_t j = l; j < r; j++) { - polygonParser.get(multi_polygon[j - l], j - l); + polygonParser.get(multi_polygon[j - l], j); } } From 05953f4cfaf70224c3e7988eb55690ef611ff7d3 Mon Sep 17 00:00:00 2001 From: Alexey Ilyukhov Date: Sun, 21 Jun 2020 22:32:39 +0300 Subject: [PATCH 264/544] Add readWkt, svg --- src/Functions/geometryConverters.h | 2 - src/Functions/readWkt.cpp | 114 +++++++++++++++++++++++++ src/Functions/registerFunctionsGeo.cpp | 4 + src/Functions/svg.cpp | 107 +++++++++++++++++++++++ src/Functions/wkt.cpp | 1 - src/Functions/ya.make | 2 + 6 files changed, 227 insertions(+), 3 deletions(-) create mode 100644 src/Functions/readWkt.cpp create mode 100644 src/Functions/svg.cpp diff --git a/src/Functions/geometryConverters.h b/src/Functions/geometryConverters.h index c5c785aea59..9983d3afdf3 100644 --- a/src/Functions/geometryConverters.h +++ b/src/Functions/geometryConverters.h @@ -100,8 +100,6 @@ public: container.reserve(r - l + 1); container.resize(r - l); - LOG_FATAL(&Poco::Logger::get("geoconv"), "ring: l = {}, r = {}", l, r); - for (size_t j = l; j < r; j++) { pointParser.get(container[j - l], j); } diff --git a/src/Functions/readWkt.cpp b/src/Functions/readWkt.cpp new file mode 100644 index 00000000000..125c39998c4 --- /dev/null +++ b/src/Functions/readWkt.cpp @@ -0,0 +1,114 @@ +#include +#include +#include +#include +#include +#include + +#include +#include + +namespace DB +{ + +namespace ErrorCodes +{ +extern const int ILLEGAL_TYPE_OF_ARGUMENT; +extern const int BAD_ARGUMENT; +} + +template +class FunctionReadWkt : public IFunction +{ +public: + explicit FunctionReadWkt() {} + + size_t getNumberOfArguments() const override + { + return 1; + } + + DataTypePtr getReturnTypeImpl(const DataTypes & arguments) const override + { + if (checkAndGetDataType(arguments[0].get()) == nullptr) + { + throw Exception("First argument should be String", + ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT); + } + + return DataType::nestedDataType(); + } + + void executeImpl(Block & block, const ColumnNumbers & arguments, size_t result, size_t input_rows_count) override + { + auto column_string = checkAndGetColumn(block.getByPosition(arguments[0]).column.get()); + + Serializer serializer; + Geometry geometry; + + for (size_t i = 0; i < input_rows_count; i++) + { + const auto & str = column_string->getDataAt(i).toString(); + boost::geometry::read_wkt(str, geometry); + serializer.add(geometry); + } + + block.getByPosition(result).column = serializer.finalize(); + } + + bool useDefaultImplementationForConstants() const override + { + return true; + } +}; + +class FunctionReadWktPoint : public FunctionReadWkt +{ +public: + static inline const char * name = "readWktPoint"; + String getName() const override + { + return name; + } + static FunctionPtr create(const Context &) + { + return std::make_shared(); + } +}; + +class FunctionReadWktPolygon : public FunctionReadWkt +{ +public: + static inline const char * name = "readWktPolygon"; + String getName() const override + { + return name; + } + static FunctionPtr create(const Context &) + { + return std::make_shared(); + } +}; + +class FunctionReadWktMultiPolygon : public FunctionReadWkt +{ +public: + static inline const char * name = "readWktMultiPolygon"; + String getName() const override + { + return name; + } + static FunctionPtr create(const Context &) + { + return std::make_shared(); + } +}; + +void registerFunctionReadWkt(FunctionFactory & factory) +{ + factory.registerFunction(); + factory.registerFunction(); + factory.registerFunction(); +} + +} diff --git a/src/Functions/registerFunctionsGeo.cpp b/src/Functions/registerFunctionsGeo.cpp index 7ce0372c51d..039ce26f8bc 100644 --- a/src/Functions/registerFunctionsGeo.cpp +++ b/src/Functions/registerFunctionsGeo.cpp @@ -16,6 +16,8 @@ void registerFunctionGeohashEncode(FunctionFactory & factory); void registerFunctionGeohashDecode(FunctionFactory & factory); void registerFunctionGeohashesInBox(FunctionFactory & factory); void registerFunctionWkt(FunctionFactory & factory); +void registerFunctionReadWkt(FunctionFactory & factory); +void registerFunctionSvg(FunctionFactory & factory); #if USE_H3 void registerFunctionGeoToH3(FunctionFactory &); @@ -45,6 +47,8 @@ void registerFunctionsGeo(FunctionFactory & factory) registerFunctionGeohashDecode(factory); registerFunctionGeohashesInBox(factory); registerFunctionWkt(factory); + registerFunctionReadWkt(factory); + registerFunctionSvg(factory); #if USE_H3 registerFunctionGeoToH3(factory); diff --git a/src/Functions/svg.cpp b/src/Functions/svg.cpp new file mode 100644 index 00000000000..d2f0ba1de4e --- /dev/null +++ b/src/Functions/svg.cpp @@ -0,0 +1,107 @@ +#include +#include +#include +#include +#include + +#include +#include + +namespace DB +{ + +namespace ErrorCodes { + extern const int TOO_MANY_ARGUMENTS_FOR_FUNCTION; + extern const int TOO_FEW_ARGUMENTS_FOR_FUNCTION; +} + +class FunctionSvg : public IFunction +{ +public: + static inline const char * name = "svg"; + + explicit FunctionSvg() {} + + static FunctionPtr create(const Context &) + { + return std::make_shared(); + } + + String getName() const override + { + return name; + } + + bool isVariadic() const override + { + return true; + } + + size_t getNumberOfArguments() const override + { + return 2; + } + + DataTypePtr getReturnTypeImpl(const DataTypes & arguments) const override + { + if (arguments.size() > 2) + { + throw Exception("Too many arguments", ErrorCodes::TOO_MANY_ARGUMENTS_FOR_FUNCTION); + } + if (arguments.size() == 0) { + throw Exception("Too few arguments", ErrorCodes::TOO_FEW_ARGUMENTS_FOR_FUNCTION); + } + if (arguments.size() == 2 && checkAndGetDataType(arguments[1].get()) == nullptr) + { + throw Exception("Second argument should be String", + ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT); + } + + return std::make_shared(); + } + + void executeImpl(Block & block, const ColumnNumbers & arguments, size_t result, size_t input_rows_count) override + { + const auto * const_col = checkAndGetColumn(block.getByPosition(arguments[0]).column.get()); + + auto parser = const_col ? + makeGeometryFromColumnParser(ColumnWithTypeAndName(const_col->getDataColumnPtr(), block.getByPosition(arguments[0]).type, block.getByPosition(arguments[0]).name)) : + makeGeometryFromColumnParser(block.getByPosition(arguments[0])); + + bool geo_column_is_const = static_cast(const_col); + + auto res_column = ColumnString::create(); + auto container = createContainer(parser); + + bool has_style = arguments.size() > 1; + ColumnPtr style; + if (has_style) { + style = block.getByPosition(arguments[1]).column; + } + + for (size_t i = 0; i < input_rows_count; i++) + { + std::stringstream str; + if (!geo_column_is_const || i == 0) + get(parser, container, i); + + str << boost::geometry::svg(container, has_style ? style->getDataAt(i).toString() : ""); + std::string serialized = str.str(); + res_column->insertData(serialized.c_str(), serialized.size()); + } + + block.getByPosition(result).column = std::move(res_column); + } + + bool useDefaultImplementationForConstants() const override + { + return true; + } +}; + +void registerFunctionSvg(FunctionFactory & factory) +{ + factory.registerFunction(); +} + +} diff --git a/src/Functions/wkt.cpp b/src/Functions/wkt.cpp index 30b8d2e1a7e..4fb214dc18c 100644 --- a/src/Functions/wkt.cpp +++ b/src/Functions/wkt.cpp @@ -61,7 +61,6 @@ public: } }; - void registerFunctionWkt(FunctionFactory & factory) { factory.registerFunction(); diff --git a/src/Functions/ya.make b/src/Functions/ya.make index 53a92780c18..35b855f7c6c 100644 --- a/src/Functions/ya.make +++ b/src/Functions/ya.make @@ -392,6 +392,7 @@ SRCS( randomStringUTF8.cpp regexpQuoteMeta.cpp registerFunctions.cpp + readWkt.cpp registerFunctionsArithmetic.cpp registerFunctionsComparison.cpp registerFunctionsConditional.cpp @@ -448,6 +449,7 @@ SRCS( subtractSeconds.cpp subtractWeeks.cpp subtractYears.cpp + svg.cpp tan.cpp tanh.cpp tcpPort.cpp From c0c8e20c4d1d732266b6eb68f24c0750448635bd Mon Sep 17 00:00:00 2001 From: DoomzD Date: Mon, 22 Jun 2020 19:15:11 +0300 Subject: [PATCH 265/544] Add polygonArea and polygonConvexHull functions --- src/Functions/PolygonArea.cpp | 108 ++++++++++++++++++++++++ src/Functions/PolygonsIntersection.cpp | 24 ++++-- src/Functions/PolygonsUnion.cpp | 22 +++-- src/Functions/polygonConvexHull.cpp | 110 +++++++++++++++++++++++++ src/Functions/registerFunctionsGeo.cpp | 2 + 5 files changed, 251 insertions(+), 15 deletions(-) create mode 100644 src/Functions/PolygonArea.cpp create mode 100644 src/Functions/polygonConvexHull.cpp diff --git a/src/Functions/PolygonArea.cpp b/src/Functions/PolygonArea.cpp new file mode 100644 index 00000000000..a625145e620 --- /dev/null +++ b/src/Functions/PolygonArea.cpp @@ -0,0 +1,108 @@ +#include +#include + +#include +#include +#include + +#include + +#include +#include +#include +#include +#include +#include + +#include +#include + +namespace DB +{ + +namespace ErrorCodes +{ + extern const int TOO_FEW_ARGUMENTS_FOR_FUNCTION; + extern const int BAD_ARGUMENTS; + extern const int ILLEGAL_TYPE_OF_ARGUMENT; + extern const int ILLEGAL_COLUMN; +} + + +using CoordinateType = Float64; +using Point = boost::geometry::model::d2::point_xy; +using Polygon = boost::geometry::model::polygon; +using MultiPolygon = boost::geometry::model::multi_polygon; +using Box = boost::geometry::model::box; + + +class FunctionPolygonArea : public IFunction +{ +public: + static inline const char * name = "polygonArea"; + + explicit FunctionPolygonArea() = default; + + static FunctionPtr create(const Context &) + { + return std::make_shared(); + } + + String getName() const override + { + return name; + } + + bool isVariadic() const override + { + return false; + } + + size_t getNumberOfArguments() const override + { + return 1; + } + + DataTypePtr getReturnTypeImpl(const DataTypes &) const override + { + return std::make_shared(); + } + + void executeImpl(Block & block, const ColumnNumbers & arguments, size_t result, size_t input_rows_count) override + { + auto get_parser = [&block, &arguments] (size_t i) { + const ColumnWithTypeAndName polygon = block.getByPosition(arguments[i]); + return makeGeometryFromColumnParser(polygon); + }; + + auto parser = get_parser(0); + auto container = createContainer(parser); + + auto res_column = ColumnFloat64::create(); + + for (size_t i = 0; i < input_rows_count; i++) + { + get(parser, container, i); + + Float64 area = boost::geometry::area( + boost::get(container)); + + res_column->insertValue(area); + } + + block.getByPosition(result).column = std::move(res_column); + } + + bool useDefaultImplementationForConstants() const override + { + return true; + } +}; + + +void registerFunctionPolygonArea(FunctionFactory & factory) +{ + factory.registerFunction(); +} + +} diff --git a/src/Functions/PolygonsIntersection.cpp b/src/Functions/PolygonsIntersection.cpp index 086898bebcc..0eecc5811fc 100644 --- a/src/Functions/PolygonsIntersection.cpp +++ b/src/Functions/PolygonsIntersection.cpp @@ -9,13 +9,13 @@ #include #include -#include +#include #include #include #include #include -#include +#include namespace DB { @@ -71,22 +71,30 @@ public: void executeImpl(Block & block, const ColumnNumbers & arguments, size_t result, size_t input_rows_count) override { auto get_parser = [&block, &arguments] (size_t i) { - const ColumnWithTypeAndName polygon = block.getByPosition(arguments[i]); - return makeGeometryFromColumnParser(polygon); + const auto * const_col = + checkAndGetColumn(block.getByPosition(arguments[i]).column.get()); + + bool is_const = static_cast(const_col); + + return std::pair{is_const, is_const ? + makeGeometryFromColumnParser(ColumnWithTypeAndName(const_col->getDataColumnPtr(), block.getByPosition(arguments[i]).type, block.getByPosition(arguments[i]).name)) : + makeGeometryFromColumnParser(block.getByPosition(arguments[i]))}; }; - auto first_parser = get_parser(0); + auto [is_first_polygon_const, first_parser] = get_parser(0); auto first_container = createContainer(first_parser); - auto second_parser = get_parser(1); + auto [is_second_polygon_const, second_parser] = get_parser(1); auto second_container = createContainer(second_parser); Float64MultiPolygonSerializer serializer; for (size_t i = 0; i < input_rows_count; i++) { - get(first_parser, first_container, i); - get(second_parser, second_container, i); + if (!is_first_polygon_const || i == 0) + get(first_parser, first_container, i); + if (!is_second_polygon_const || i == 0) + get(second_parser, second_container, i); Float64Geometry intersection = Float64MultiPolygon({{{{}}}}); boost::geometry::intersection( diff --git a/src/Functions/PolygonsUnion.cpp b/src/Functions/PolygonsUnion.cpp index f2f6c28e91e..8a8877257ba 100644 --- a/src/Functions/PolygonsUnion.cpp +++ b/src/Functions/PolygonsUnion.cpp @@ -9,7 +9,7 @@ #include #include -#include +#include #include #include #include @@ -71,22 +71,30 @@ public: void executeImpl(Block & block, const ColumnNumbers & arguments, size_t result, size_t input_rows_count) override { auto get_parser = [&block, &arguments] (size_t i) { - const ColumnWithTypeAndName polygon = block.getByPosition(arguments[i]); - return makeGeometryFromColumnParser(polygon); + const auto * const_col = + checkAndGetColumn(block.getByPosition(arguments[i]).column.get()); + + bool is_const = static_cast(const_col); + + return std::pair{is_const, is_const ? + makeGeometryFromColumnParser(ColumnWithTypeAndName(const_col->getDataColumnPtr(), block.getByPosition(arguments[i]).type, block.getByPosition(arguments[i]).name)) : + makeGeometryFromColumnParser(block.getByPosition(arguments[i]))}; }; - auto first_parser = get_parser(0); + auto [is_first_polygon_const, first_parser] = get_parser(0); auto first_container = createContainer(first_parser); - auto second_parser = get_parser(1); + auto [is_second_polygon_const, second_parser] = get_parser(1); auto second_container = createContainer(second_parser); Float64MultiPolygonSerializer serializer; for (size_t i = 0; i < input_rows_count; i++) { - get(first_parser, first_container, i); - get(second_parser, second_container, i); + if (!is_first_polygon_const || i == 0) + get(first_parser, first_container, i); + if (!is_second_polygon_const || i == 0) + get(second_parser, second_container, i); Float64Geometry polygons_union = Float64MultiPolygon({{{{}}}}); boost::geometry::union_( diff --git a/src/Functions/polygonConvexHull.cpp b/src/Functions/polygonConvexHull.cpp new file mode 100644 index 00000000000..40a6a083983 --- /dev/null +++ b/src/Functions/polygonConvexHull.cpp @@ -0,0 +1,110 @@ +#include +#include + +#include +#include +#include + +#include + +#include +#include +#include +#include +#include +#include + +#include +#include + +namespace DB +{ + +namespace ErrorCodes +{ + extern const int TOO_FEW_ARGUMENTS_FOR_FUNCTION; + extern const int BAD_ARGUMENTS; + extern const int ILLEGAL_TYPE_OF_ARGUMENT; + extern const int ILLEGAL_COLUMN; +} + + +using CoordinateType = Float64; +using Point = boost::geometry::model::d2::point_xy; +using Polygon = boost::geometry::model::polygon; +using MultiPolygon = boost::geometry::model::multi_polygon; +using Box = boost::geometry::model::box; + + +class FunctionPolygonConvexHull : public IFunction +{ +public: + static inline const char * name = "polygonConvexHull"; + + explicit FunctionPolygonConvexHull() = default; + + static FunctionPtr create(const Context &) + { + return std::make_shared(); + } + + String getName() const override + { + return name; + } + + bool isVariadic() const override + { + return false; + } + + size_t getNumberOfArguments() const override + { + return 1; + } + + DataTypePtr getReturnTypeImpl(const DataTypes &) const override + { + return DataTypeCustomPolygonSerialization::nestedDataType(); + } + + void executeImpl(Block & block, const ColumnNumbers & arguments, size_t result, size_t input_rows_count) override + { + auto get_parser = [&block, &arguments] (size_t i) { + const ColumnWithTypeAndName polygon = block.getByPosition(arguments[i]); + return makeGeometryFromColumnParser(polygon); + }; + + auto parser = get_parser(0); + auto container = createContainer(parser); + + Float64PolygonSerializer serializer; + + for (size_t i = 0; i < input_rows_count; i++) + { + get(parser, container, i); + + Float64Geometry convex_hull = Float64Polygon({{{}}}); + boost::geometry::convex_hull( + boost::get(container), + boost::get(convex_hull)); + + serializer.add(convex_hull); + } + + block.getByPosition(result).column = std::move(serializer.finalize()); + } + + bool useDefaultImplementationForConstants() const override + { + return true; + } +}; + + +void registerFunctionPolygonConvexHull(FunctionFactory & factory) +{ + factory.registerFunction(); +} + +} diff --git a/src/Functions/registerFunctionsGeo.cpp b/src/Functions/registerFunctionsGeo.cpp index 039ce26f8bc..49b28c99ccf 100644 --- a/src/Functions/registerFunctionsGeo.cpp +++ b/src/Functions/registerFunctionsGeo.cpp @@ -12,6 +12,7 @@ void registerFunctionPointInEllipses(FunctionFactory & factory); void registerFunctionPointInPolygon(FunctionFactory & factory); void registerFunctionPolygonsIntersection(FunctionFactory & factory); void registerFunctionPolygonsUnion(FunctionFactory & factory); +void registerFunctionPolygonArea(FunctionFactory & factory); void registerFunctionGeohashEncode(FunctionFactory & factory); void registerFunctionGeohashDecode(FunctionFactory & factory); void registerFunctionGeohashesInBox(FunctionFactory & factory); @@ -43,6 +44,7 @@ void registerFunctionsGeo(FunctionFactory & factory) registerFunctionPointInPolygon(factory); registerFunctionPolygonsIntersection(factory); registerFunctionPolygonsUnion(factory); + registerFunctionPolygonArea(factory); registerFunctionGeohashEncode(factory); registerFunctionGeohashDecode(factory); registerFunctionGeohashesInBox(factory); From 317cbdee2b6616f0e5211cd446cef76d4926f576 Mon Sep 17 00:00:00 2001 From: DoomzD Date: Mon, 22 Jun 2020 19:29:49 +0300 Subject: [PATCH 266/544] Add polygonsEquals and polygonsSymDifference --- src/Functions/polygonsEquals.cpp | 123 +++++++++++++++++++++++ src/Functions/polygonsSymDifference.cpp | 126 ++++++++++++++++++++++++ src/Functions/registerFunctionsGeo.cpp | 16 +++ 3 files changed, 265 insertions(+) create mode 100644 src/Functions/polygonsEquals.cpp create mode 100644 src/Functions/polygonsSymDifference.cpp diff --git a/src/Functions/polygonsEquals.cpp b/src/Functions/polygonsEquals.cpp new file mode 100644 index 00000000000..b42787adf18 --- /dev/null +++ b/src/Functions/polygonsEquals.cpp @@ -0,0 +1,123 @@ +#include +#include + +#include +#include +#include + +#include + +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include + +namespace DB +{ + +namespace ErrorCodes +{ + extern const int TOO_FEW_ARGUMENTS_FOR_FUNCTION; + extern const int BAD_ARGUMENTS; + extern const int ILLEGAL_TYPE_OF_ARGUMENT; + extern const int ILLEGAL_COLUMN; +} + + +using CoordinateType = Float64; +using Point = boost::geometry::model::d2::point_xy; +using Polygon = boost::geometry::model::polygon; +using MultiPolygon = boost::geometry::model::multi_polygon; +using Box = boost::geometry::model::box; + + +class FunctionPolygonsEquals : public IFunction +{ +public: + static inline const char * name = "polygonsEquals"; + + explicit FunctionPolygonsEquals() = default; + + static FunctionPtr create(const Context &) + { + return std::make_shared(); + } + + String getName() const override + { + return name; + } + + bool isVariadic() const override + { + return false; + } + + size_t getNumberOfArguments() const override + { + return 2; + } + + DataTypePtr getReturnTypeImpl(const DataTypes &) const override + { + return std::make_shared(); + } + + void executeImpl(Block & block, const ColumnNumbers & arguments, size_t result, size_t input_rows_count) override + { + auto get_parser = [&block, &arguments] (size_t i) { + const auto * const_col = + checkAndGetColumn(block.getByPosition(arguments[i]).column.get()); + + bool is_const = static_cast(const_col); + + return std::pair{is_const, is_const ? + makeGeometryFromColumnParser(ColumnWithTypeAndName(const_col->getDataColumnPtr(), block.getByPosition(arguments[i]).type, block.getByPosition(arguments[i]).name)) : + makeGeometryFromColumnParser(block.getByPosition(arguments[i]))}; + }; + + auto [is_first_polygon_const, first_parser] = get_parser(0); + auto first_container = createContainer(first_parser); + + auto [is_second_polygon_const, second_parser] = get_parser(1); + auto second_container = createContainer(second_parser); + + auto res_column = ColumnUInt8::create(); + + for (size_t i = 0; i < input_rows_count; i++) + { + if (!is_first_polygon_const || i == 0) + get(first_parser, first_container, i); + if (!is_second_polygon_const || i == 0) + get(second_parser, second_container, i); + + bool equals = boost::geometry::equals( + boost::get(first_container), + boost::get(second_container)); + + res_column->insertValue(equals); + } + + block.getByPosition(result).column = std::move(res_column); + } + + bool useDefaultImplementationForConstants() const override + { + return true; + } +}; + + +void registerFunctionPolygonsEquals(FunctionFactory & factory) +{ + factory.registerFunction(); +} + +} diff --git a/src/Functions/polygonsSymDifference.cpp b/src/Functions/polygonsSymDifference.cpp new file mode 100644 index 00000000000..3e7b24efc2d --- /dev/null +++ b/src/Functions/polygonsSymDifference.cpp @@ -0,0 +1,126 @@ +#include +#include + +#include +#include +#include + +#include + +#include +#include +#include +#include +#include +#include + +#include +#include + +namespace DB +{ + +namespace ErrorCodes +{ + extern const int TOO_FEW_ARGUMENTS_FOR_FUNCTION; + extern const int BAD_ARGUMENTS; + extern const int ILLEGAL_TYPE_OF_ARGUMENT; + extern const int ILLEGAL_COLUMN; +} + + +using CoordinateType = Float64; +using Point = boost::geometry::model::d2::point_xy; +using Polygon = boost::geometry::model::polygon; +using MultiPolygon = boost::geometry::model::multi_polygon; +using Box = boost::geometry::model::box; + + +class FunctionPolygonsSymDifference : public IFunction +{ +public: + static inline const char * name = "polygonsSymDifference"; + + explicit FunctionPolygonsSymDifference() = default; + + static FunctionPtr create(const Context &) + { + return std::make_shared(); + } + + String getName() const override + { + return name; + } + + bool isVariadic() const override + { + return false; + } + + size_t getNumberOfArguments() const override + { + return 2; + } + + DataTypePtr getReturnTypeImpl(const DataTypes &) const override + { + return DataTypeCustomMultiPolygonSerialization::nestedDataType(); + } + + void executeImpl(Block & block, const ColumnNumbers & arguments, size_t result, size_t input_rows_count) override + { + auto get_parser = [&block, &arguments] (size_t i) { + const auto * const_col = + checkAndGetColumn(block.getByPosition(arguments[i]).column.get()); + + bool is_const = static_cast(const_col); + + return std::pair{is_const, is_const ? + makeGeometryFromColumnParser(ColumnWithTypeAndName(const_col->getDataColumnPtr(), block.getByPosition(arguments[i]).type, block.getByPosition(arguments[i]).name)) : + makeGeometryFromColumnParser(block.getByPosition(arguments[i]))}; + }; + + auto [is_first_polygon_const, first_parser] = get_parser(0); + auto first_container = createContainer(first_parser); + + auto [is_second_polygon_const, second_parser] = get_parser(1); + auto second_container = createContainer(second_parser); + + Float64MultiPolygonSerializer serializer; + + for (size_t i = 0; i < input_rows_count; i++) + { + if (!is_first_polygon_const || i == 0) + get(first_parser, first_container, i); + if (!is_second_polygon_const || i == 0) + get(second_parser, second_container, i); + + Float64Geometry sym_difference = Float64MultiPolygon({{{{}}}}); + boost::geometry::sym_difference( + boost::get(first_container), + boost::get(second_container), + boost::get(sym_difference)); + + boost::get(sym_difference).erase( + boost::get(sym_difference).begin()); + + serializer.add(sym_difference); + } + + block.getByPosition(result).column = std::move(serializer.finalize()); + } + + bool useDefaultImplementationForConstants() const override + { + return true; + } +}; + + +void registerFunctionPolygonsSymDifference(FunctionFactory & factory) +{ + factory.registerFunction(); +} + +} diff --git a/src/Functions/registerFunctionsGeo.cpp b/src/Functions/registerFunctionsGeo.cpp index 49b28c99ccf..3ce8ee316d2 100644 --- a/src/Functions/registerFunctionsGeo.cpp +++ b/src/Functions/registerFunctionsGeo.cpp @@ -13,6 +13,14 @@ void registerFunctionPointInPolygon(FunctionFactory & factory); void registerFunctionPolygonsIntersection(FunctionFactory & factory); void registerFunctionPolygonsUnion(FunctionFactory & factory); void registerFunctionPolygonArea(FunctionFactory & factory); +void registerFunctionPolygonConvexHull(FunctionFactory & factory); + +void registerFunctionPolygonsSymDifference(FunctionFactory & factory); +void registerFunctionPolygonsEquals(FunctionFactory & factory); +void registerFunctionPolygonConvexHull(FunctionFactory & factory); +void registerFunctionPolygonConvexHull(FunctionFactory & factory); +void registerFunctionPolygonConvexHull(FunctionFactory & factory); + void registerFunctionGeohashEncode(FunctionFactory & factory); void registerFunctionGeohashDecode(FunctionFactory & factory); void registerFunctionGeohashesInBox(FunctionFactory & factory); @@ -45,6 +53,14 @@ void registerFunctionsGeo(FunctionFactory & factory) registerFunctionPolygonsIntersection(factory); registerFunctionPolygonsUnion(factory); registerFunctionPolygonArea(factory); + registerFunctionPolygonConvexHull(factory); + + registerFunctionPolygonsSymDifference(factory); + registerFunctionPolygonsEquals(factory); + registerFunctionPolygonConvexHull(factory); + registerFunctionPolygonConvexHull(factory); + registerFunctionPolygonConvexHull(factory); + registerFunctionGeohashEncode(factory); registerFunctionGeohashDecode(factory); registerFunctionGeohashesInBox(factory); From 153e35a33c20b7e4515467b7f0d63208462e5083 Mon Sep 17 00:00:00 2001 From: DoomzD Date: Mon, 22 Jun 2020 19:35:29 +0300 Subject: [PATCH 267/544] Add polygonPerimeter, polygonsWithin and polygonsDistance --- .../{PolygonArea.cpp => polygonArea.cpp} | 0 src/Functions/polygonPerimeter.cpp | 0 src/Functions/polygonsDistance.cpp | 123 ++++++++++++++++++ ...ersection.cpp => polygonsIntersection.cpp} | 0 src/Functions/polygonsWithin.cpp | 123 ++++++++++++++++++ src/Functions/registerFunctionsGeo.cpp | 16 +-- 6 files changed, 252 insertions(+), 10 deletions(-) rename src/Functions/{PolygonArea.cpp => polygonArea.cpp} (100%) create mode 100644 src/Functions/polygonPerimeter.cpp create mode 100644 src/Functions/polygonsDistance.cpp rename src/Functions/{PolygonsIntersection.cpp => polygonsIntersection.cpp} (100%) create mode 100644 src/Functions/polygonsWithin.cpp diff --git a/src/Functions/PolygonArea.cpp b/src/Functions/polygonArea.cpp similarity index 100% rename from src/Functions/PolygonArea.cpp rename to src/Functions/polygonArea.cpp diff --git a/src/Functions/polygonPerimeter.cpp b/src/Functions/polygonPerimeter.cpp new file mode 100644 index 00000000000..e69de29bb2d diff --git a/src/Functions/polygonsDistance.cpp b/src/Functions/polygonsDistance.cpp new file mode 100644 index 00000000000..99dd15049c5 --- /dev/null +++ b/src/Functions/polygonsDistance.cpp @@ -0,0 +1,123 @@ +#include +#include + +#include +#include +#include + +#include + +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include + +namespace DB +{ + +namespace ErrorCodes +{ + extern const int TOO_FEW_ARGUMENTS_FOR_FUNCTION; + extern const int BAD_ARGUMENTS; + extern const int ILLEGAL_TYPE_OF_ARGUMENT; + extern const int ILLEGAL_COLUMN; +} + + +using CoordinateType = Float64; +using Point = boost::geometry::model::d2::point_xy; +using Polygon = boost::geometry::model::polygon; +using MultiPolygon = boost::geometry::model::multi_polygon; +using Box = boost::geometry::model::box; + + +class FunctionPolygonsDistance : public IFunction +{ +public: + static inline const char * name = "polygonsDistance"; + + explicit FunctionPolygonsDistance() = default; + + static FunctionPtr create(const Context &) + { + return std::make_shared(); + } + + String getName() const override + { + return name; + } + + bool isVariadic() const override + { + return false; + } + + size_t getNumberOfArguments() const override + { + return 2; + } + + DataTypePtr getReturnTypeImpl(const DataTypes &) const override + { + return std::make_shared(); + } + + void executeImpl(Block & block, const ColumnNumbers & arguments, size_t result, size_t input_rows_count) override + { + auto get_parser = [&block, &arguments] (size_t i) { + const auto * const_col = + checkAndGetColumn(block.getByPosition(arguments[i]).column.get()); + + bool is_const = static_cast(const_col); + + return std::pair{is_const, is_const ? + makeGeometryFromColumnParser(ColumnWithTypeAndName(const_col->getDataColumnPtr(), block.getByPosition(arguments[i]).type, block.getByPosition(arguments[i]).name)) : + makeGeometryFromColumnParser(block.getByPosition(arguments[i]))}; + }; + + auto [is_first_polygon_const, first_parser] = get_parser(0); + auto first_container = createContainer(first_parser); + + auto [is_second_polygon_const, second_parser] = get_parser(1); + auto second_container = createContainer(second_parser); + + auto res_column = ColumnFloat64::create(); + + for (size_t i = 0; i < input_rows_count; i++) + { + if (!is_first_polygon_const || i == 0) + get(first_parser, first_container, i); + if (!is_second_polygon_const || i == 0) + get(second_parser, second_container, i); + + bool distance = boost::geometry::distance( + boost::get(first_container), + boost::get(second_container)); + + res_column->insertValue(distance); + } + + block.getByPosition(result).column = std::move(res_column); + } + + bool useDefaultImplementationForConstants() const override + { + return true; + } +}; + + +void registerFunctionPolygonsDistance(FunctionFactory & factory) +{ + factory.registerFunction(); +} + +} diff --git a/src/Functions/PolygonsIntersection.cpp b/src/Functions/polygonsIntersection.cpp similarity index 100% rename from src/Functions/PolygonsIntersection.cpp rename to src/Functions/polygonsIntersection.cpp diff --git a/src/Functions/polygonsWithin.cpp b/src/Functions/polygonsWithin.cpp new file mode 100644 index 00000000000..8ad4c9bcbb1 --- /dev/null +++ b/src/Functions/polygonsWithin.cpp @@ -0,0 +1,123 @@ +#include +#include + +#include +#include +#include + +#include + +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include + +namespace DB +{ + +namespace ErrorCodes +{ + extern const int TOO_FEW_ARGUMENTS_FOR_FUNCTION; + extern const int BAD_ARGUMENTS; + extern const int ILLEGAL_TYPE_OF_ARGUMENT; + extern const int ILLEGAL_COLUMN; +} + + +using CoordinateType = Float64; +using Point = boost::geometry::model::d2::point_xy; +using Polygon = boost::geometry::model::polygon; +using MultiPolygon = boost::geometry::model::multi_polygon; +using Box = boost::geometry::model::box; + + +class FunctionPolygonsWithin : public IFunction +{ +public: + static inline const char * name = "polygonsWithin"; + + explicit FunctionPolygonsWithin() = default; + + static FunctionPtr create(const Context &) + { + return std::make_shared(); + } + + String getName() const override + { + return name; + } + + bool isVariadic() const override + { + return false; + } + + size_t getNumberOfArguments() const override + { + return 2; + } + + DataTypePtr getReturnTypeImpl(const DataTypes &) const override + { + return std::make_shared(); + } + + void executeImpl(Block & block, const ColumnNumbers & arguments, size_t result, size_t input_rows_count) override + { + auto get_parser = [&block, &arguments] (size_t i) { + const auto * const_col = + checkAndGetColumn(block.getByPosition(arguments[i]).column.get()); + + bool is_const = static_cast(const_col); + + return std::pair{is_const, is_const ? + makeGeometryFromColumnParser(ColumnWithTypeAndName(const_col->getDataColumnPtr(), block.getByPosition(arguments[i]).type, block.getByPosition(arguments[i]).name)) : + makeGeometryFromColumnParser(block.getByPosition(arguments[i]))}; + }; + + auto [is_first_polygon_const, first_parser] = get_parser(0); + auto first_container = createContainer(first_parser); + + auto [is_second_polygon_const, second_parser] = get_parser(1); + auto second_container = createContainer(second_parser); + + auto res_column = ColumnUInt8::create(); + + for (size_t i = 0; i < input_rows_count; i++) + { + if (!is_first_polygon_const || i == 0) + get(first_parser, first_container, i); + if (!is_second_polygon_const || i == 0) + get(second_parser, second_container, i); + + bool within = boost::geometry::within( + boost::get(first_container), + boost::get(second_container)); + + res_column->insertValue(within); + } + + block.getByPosition(result).column = std::move(res_column); + } + + bool useDefaultImplementationForConstants() const override + { + return true; + } +}; + + +void registerFunctionPolygonsWithin(FunctionFactory & factory) +{ + factory.registerFunction(); +} + +} diff --git a/src/Functions/registerFunctionsGeo.cpp b/src/Functions/registerFunctionsGeo.cpp index 3ce8ee316d2..605dd4dcba0 100644 --- a/src/Functions/registerFunctionsGeo.cpp +++ b/src/Functions/registerFunctionsGeo.cpp @@ -14,13 +14,11 @@ void registerFunctionPolygonsIntersection(FunctionFactory & factory); void registerFunctionPolygonsUnion(FunctionFactory & factory); void registerFunctionPolygonArea(FunctionFactory & factory); void registerFunctionPolygonConvexHull(FunctionFactory & factory); - void registerFunctionPolygonsSymDifference(FunctionFactory & factory); void registerFunctionPolygonsEquals(FunctionFactory & factory); -void registerFunctionPolygonConvexHull(FunctionFactory & factory); -void registerFunctionPolygonConvexHull(FunctionFactory & factory); -void registerFunctionPolygonConvexHull(FunctionFactory & factory); - +void registerFunctionPolygonsDistance(FunctionFactory & factory); +void registerFunctionPolygonsWithin(FunctionFactory & factory); +void registerFunctionPolygonPerimeter(FunctionFactory & factory); void registerFunctionGeohashEncode(FunctionFactory & factory); void registerFunctionGeohashDecode(FunctionFactory & factory); void registerFunctionGeohashesInBox(FunctionFactory & factory); @@ -54,13 +52,11 @@ void registerFunctionsGeo(FunctionFactory & factory) registerFunctionPolygonsUnion(factory); registerFunctionPolygonArea(factory); registerFunctionPolygonConvexHull(factory); - registerFunctionPolygonsSymDifference(factory); registerFunctionPolygonsEquals(factory); - registerFunctionPolygonConvexHull(factory); - registerFunctionPolygonConvexHull(factory); - registerFunctionPolygonConvexHull(factory); - + registerFunctionPolygonsDistance(factory); + registerFunctionPolygonsWithin(factory); + registerFunctionPolygonPerimeter(factory); registerFunctionGeohashEncode(factory); registerFunctionGeohashDecode(factory); registerFunctionGeohashesInBox(factory); From b26022d044ba743b878cb5ca041d4c7f8b28e557 Mon Sep 17 00:00:00 2001 From: DoomzD Date: Mon, 22 Jun 2020 19:36:21 +0300 Subject: [PATCH 268/544] Add polygonPerimeter --- src/Functions/polygonPerimeter.cpp | 108 +++++++++++++++++++++++++++++ 1 file changed, 108 insertions(+) diff --git a/src/Functions/polygonPerimeter.cpp b/src/Functions/polygonPerimeter.cpp index e69de29bb2d..a9c42f24aaf 100644 --- a/src/Functions/polygonPerimeter.cpp +++ b/src/Functions/polygonPerimeter.cpp @@ -0,0 +1,108 @@ +#include +#include + +#include +#include +#include + +#include + +#include +#include +#include +#include +#include +#include + +#include +#include + +namespace DB +{ + +namespace ErrorCodes +{ + extern const int TOO_FEW_ARGUMENTS_FOR_FUNCTION; + extern const int BAD_ARGUMENTS; + extern const int ILLEGAL_TYPE_OF_ARGUMENT; + extern const int ILLEGAL_COLUMN; +} + + +using CoordinateType = Float64; +using Point = boost::geometry::model::d2::point_xy; +using Polygon = boost::geometry::model::polygon; +using MultiPolygon = boost::geometry::model::multi_polygon; +using Box = boost::geometry::model::box; + + +class FunctionPolygonPerimeter : public IFunction +{ +public: + static inline const char * name = "polygonPerimeter"; + + explicit FunctionPolygonPerimeter() = default; + + static FunctionPtr create(const Context &) + { + return std::make_shared(); + } + + String getName() const override + { + return name; + } + + bool isVariadic() const override + { + return false; + } + + size_t getNumberOfArguments() const override + { + return 1; + } + + DataTypePtr getReturnTypeImpl(const DataTypes &) const override + { + return std::make_shared(); + } + + void executeImpl(Block & block, const ColumnNumbers & arguments, size_t result, size_t input_rows_count) override + { + auto get_parser = [&block, &arguments] (size_t i) { + const ColumnWithTypeAndName polygon = block.getByPosition(arguments[i]); + return makeGeometryFromColumnParser(polygon); + }; + + auto parser = get_parser(0); + auto container = createContainer(parser); + + auto res_column = ColumnFloat64::create(); + + for (size_t i = 0; i < input_rows_count; i++) + { + get(parser, container, i); + + Float64 perimeter = boost::geometry::perimeter( + boost::get(container)); + + res_column->insertValue(perimeter); + } + + block.getByPosition(result).column = std::move(res_column); + } + + bool useDefaultImplementationForConstants() const override + { + return true; + } +}; + + +void registerFunctionPolygonPerimeter(FunctionFactory & factory) +{ + factory.registerFunction(); +} + +} From 147c759ea4577c0a84d411403340500ce64f40f6 Mon Sep 17 00:00:00 2001 From: DoomzD Date: Mon, 22 Jun 2020 19:59:18 +0300 Subject: [PATCH 269/544] fix polygonConvexHull --- src/Functions/polygonConvexHull.cpp | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/Functions/polygonConvexHull.cpp b/src/Functions/polygonConvexHull.cpp index 40a6a083983..5e30a93c16a 100644 --- a/src/Functions/polygonConvexHull.cpp +++ b/src/Functions/polygonConvexHull.cpp @@ -87,7 +87,10 @@ public: Float64Geometry convex_hull = Float64Polygon({{{}}}); boost::geometry::convex_hull( boost::get(container), - boost::get(convex_hull)); + boost::get(convex_hull)); + + boost::get(convex_hull).outer().erase( + boost::get(convex_hull).outer().begin()); serializer.add(convex_hull); } From 23616c773af61bcf616e63eadbe8667199413467 Mon Sep 17 00:00:00 2001 From: DoomzD Date: Mon, 22 Jun 2020 20:36:40 +0300 Subject: [PATCH 270/544] fix polygonDistance --- src/Functions/polygonsDistance.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Functions/polygonsDistance.cpp b/src/Functions/polygonsDistance.cpp index 99dd15049c5..fcfff7bd964 100644 --- a/src/Functions/polygonsDistance.cpp +++ b/src/Functions/polygonsDistance.cpp @@ -98,7 +98,7 @@ public: if (!is_second_polygon_const || i == 0) get(second_parser, second_container, i); - bool distance = boost::geometry::distance( + Float64 distance = boost::geometry::distance( boost::get(first_container), boost::get(second_container)); From f7019cf58902830ea7a808e7056e736305935aaf Mon Sep 17 00:00:00 2001 From: DoomzD Date: Mon, 22 Jun 2020 20:38:42 +0300 Subject: [PATCH 271/544] Add tests --- tests/queries/0_stateless/01300_polygon_convex_hull.reference | 1 + tests/queries/0_stateless/01300_polygon_convex_hull.sql | 1 + tests/queries/0_stateless/01301_polygons_within.reference | 2 ++ tests/queries/0_stateless/01301_polygons_within.sql | 2 ++ tests/queries/0_stateless/01302_polygons_distance.reference | 2 ++ tests/queries/0_stateless/01302_polygons_distance.sql | 3 +++ tests/queries/0_stateless/01303_polygons_equals.reference | 2 ++ tests/queries/0_stateless/01303_polygons_equals.sql | 2 ++ .../0_stateless/01304_polygons_sym_difference.reference | 1 + tests/queries/0_stateless/01304_polygons_sym_difference.sql | 1 + tests/queries/0_stateless/01305_polygons_union.reference | 1 + tests/queries/0_stateless/01305_polygons_union.sql | 1 + .../queries/0_stateless/01306_polygons_intersection.reference | 1 + tests/queries/0_stateless/01306_polygons_intersection.sql | 1 + tests/queries/0_stateless/01307_polygon_perimeter.reference | 1 + tests/queries/0_stateless/01307_polygon_perimeter.sql | 1 + tests/queries/0_stateless/01308_polygon_area.reference | 1 + tests/queries/0_stateless/01308_polygon_area.sql | 1 + 18 files changed, 25 insertions(+) create mode 100644 tests/queries/0_stateless/01300_polygon_convex_hull.reference create mode 100644 tests/queries/0_stateless/01300_polygon_convex_hull.sql create mode 100644 tests/queries/0_stateless/01301_polygons_within.reference create mode 100644 tests/queries/0_stateless/01301_polygons_within.sql create mode 100644 tests/queries/0_stateless/01302_polygons_distance.reference create mode 100644 tests/queries/0_stateless/01302_polygons_distance.sql create mode 100644 tests/queries/0_stateless/01303_polygons_equals.reference create mode 100644 tests/queries/0_stateless/01303_polygons_equals.sql create mode 100644 tests/queries/0_stateless/01304_polygons_sym_difference.reference create mode 100644 tests/queries/0_stateless/01304_polygons_sym_difference.sql create mode 100644 tests/queries/0_stateless/01305_polygons_union.reference create mode 100644 tests/queries/0_stateless/01305_polygons_union.sql create mode 100644 tests/queries/0_stateless/01306_polygons_intersection.reference create mode 100644 tests/queries/0_stateless/01306_polygons_intersection.sql create mode 100644 tests/queries/0_stateless/01307_polygon_perimeter.reference create mode 100644 tests/queries/0_stateless/01307_polygon_perimeter.sql create mode 100644 tests/queries/0_stateless/01308_polygon_area.reference create mode 100644 tests/queries/0_stateless/01308_polygon_area.sql diff --git a/tests/queries/0_stateless/01300_polygon_convex_hull.reference b/tests/queries/0_stateless/01300_polygon_convex_hull.reference new file mode 100644 index 00000000000..89dc8315153 --- /dev/null +++ b/tests/queries/0_stateless/01300_polygon_convex_hull.reference @@ -0,0 +1 @@ +[[(0,0),(0,5),(5,5),(5,0),(0,0)]] \ No newline at end of file diff --git a/tests/queries/0_stateless/01300_polygon_convex_hull.sql b/tests/queries/0_stateless/01300_polygon_convex_hull.sql new file mode 100644 index 00000000000..547670dfe8f --- /dev/null +++ b/tests/queries/0_stateless/01300_polygon_convex_hull.sql @@ -0,0 +1 @@ +select polygonConvexHull([[[(0, 0), (0, 5), (5, 5), (5, 0), (2, 3)]]]) diff --git a/tests/queries/0_stateless/01301_polygons_within.reference b/tests/queries/0_stateless/01301_polygons_within.reference new file mode 100644 index 00000000000..0d66ea1aee9 --- /dev/null +++ b/tests/queries/0_stateless/01301_polygons_within.reference @@ -0,0 +1,2 @@ +0 +1 diff --git a/tests/queries/0_stateless/01301_polygons_within.sql b/tests/queries/0_stateless/01301_polygons_within.sql new file mode 100644 index 00000000000..da76858c95d --- /dev/null +++ b/tests/queries/0_stateless/01301_polygons_within.sql @@ -0,0 +1,2 @@ +select polygonsWithin([[[(0, 0),(0, 3),(1, 2.9),(2, 2.6),(2.6, 2),(2.9, 1),(3, 0),(0, 0)]]], [[[(1, 1),(1, 4),(4, 4),(4, 1),(1, 1)]]]); +select polygonsWithin([[[(2, 2), (2, 3), (3, 3), (3, 2)]]], [[[(1, 1),(1, 4),(4, 4),(4, 1),(1, 1)]]]) diff --git a/tests/queries/0_stateless/01302_polygons_distance.reference b/tests/queries/0_stateless/01302_polygons_distance.reference new file mode 100644 index 00000000000..aba29d80f56 --- /dev/null +++ b/tests/queries/0_stateless/01302_polygons_distance.reference @@ -0,0 +1,2 @@ +0 +1.2727922061357855 diff --git a/tests/queries/0_stateless/01302_polygons_distance.sql b/tests/queries/0_stateless/01302_polygons_distance.sql new file mode 100644 index 00000000000..a4d1fbb3f4c --- /dev/null +++ b/tests/queries/0_stateless/01302_polygons_distance.sql @@ -0,0 +1,3 @@ +select polygonsDistance([[[(0, 0),(0, 3),(1, 2.9),(2, 2.6),(2.6, 2),(2.9, 1),(3, 0),(0, 0)]]], [[[(1, 1),(1, 4),(4, 4),(4, 1),(1, 1)]]]); + +select polygonsDistance([[[(0, 0), (0, 0.1), (0.1, 0.1), (0.1, 0)]]], [[[(1, 1),(1, 4),(4, 4),(4, 1),(1, 1)]]]) diff --git a/tests/queries/0_stateless/01303_polygons_equals.reference b/tests/queries/0_stateless/01303_polygons_equals.reference new file mode 100644 index 00000000000..0d66ea1aee9 --- /dev/null +++ b/tests/queries/0_stateless/01303_polygons_equals.reference @@ -0,0 +1,2 @@ +0 +1 diff --git a/tests/queries/0_stateless/01303_polygons_equals.sql b/tests/queries/0_stateless/01303_polygons_equals.sql new file mode 100644 index 00000000000..fa93deaba03 --- /dev/null +++ b/tests/queries/0_stateless/01303_polygons_equals.sql @@ -0,0 +1,2 @@ +select polygonsEquals([[[(0, 0),(0, 3),(1, 2.9),(2, 2.6),(2.6, 2),(2.9, 1),(3, 0),(0, 0)]]], [[[(1, 1),(1, 4),(4, 4),(4, 1),(1, 1)]]]); +select polygonsEquals([[[(1, 1),(1, 4),(4, 4),(4, 1)]]], [[[(1, 1),(1, 4),(4, 4),(4, 1),(1, 1)]]]) \ No newline at end of file diff --git a/tests/queries/0_stateless/01304_polygons_sym_difference.reference b/tests/queries/0_stateless/01304_polygons_sym_difference.reference new file mode 100644 index 00000000000..3b035601be7 --- /dev/null +++ b/tests/queries/0_stateless/01304_polygons_sym_difference.reference @@ -0,0 +1 @@ +[[[(1,2.9),(1,1),(2.9,1),(3,0),(0,0),(0,3),(1,2.9)]],[[(1,2.9),(1,4),(4,4),(4,1),(2.9,1),(2.6,2),(2,2.6),(1,2.9)]]] diff --git a/tests/queries/0_stateless/01304_polygons_sym_difference.sql b/tests/queries/0_stateless/01304_polygons_sym_difference.sql new file mode 100644 index 00000000000..4008d1dc7d6 --- /dev/null +++ b/tests/queries/0_stateless/01304_polygons_sym_difference.sql @@ -0,0 +1 @@ +select polygonsSymDifference([[[(0, 0),(0, 3),(1, 2.9),(2, 2.6),(2.6, 2),(2.9, 1),(3, 0),(0, 0)]]], [[[(1, 1),(1, 4),(4, 4),(4, 1),(1, 1)]]]) \ No newline at end of file diff --git a/tests/queries/0_stateless/01305_polygons_union.reference b/tests/queries/0_stateless/01305_polygons_union.reference new file mode 100644 index 00000000000..621f1159e22 --- /dev/null +++ b/tests/queries/0_stateless/01305_polygons_union.reference @@ -0,0 +1 @@ +[[[(1,2.9),(1,4),(4,4),(4,1),(2.9,1),(3,0),(0,0),(0,3),(1,2.9)]]] diff --git a/tests/queries/0_stateless/01305_polygons_union.sql b/tests/queries/0_stateless/01305_polygons_union.sql new file mode 100644 index 00000000000..35f9edad2ee --- /dev/null +++ b/tests/queries/0_stateless/01305_polygons_union.sql @@ -0,0 +1 @@ +select polygonsUnion([[[(0, 0),(0, 3),(1, 2.9),(2, 2.6),(2.6, 2),(2.9, 1),(3, 0),(0, 0)]]], [[[(1, 1),(1, 4),(4, 4),(4, 1),(1, 1)]]]) diff --git a/tests/queries/0_stateless/01306_polygons_intersection.reference b/tests/queries/0_stateless/01306_polygons_intersection.reference new file mode 100644 index 00000000000..3eb60ccf3de --- /dev/null +++ b/tests/queries/0_stateless/01306_polygons_intersection.reference @@ -0,0 +1 @@ +[[[(1,2.9),(2,2.6),(2.6,2),(2.9,1),(1,1),(1,2.9)]]] diff --git a/tests/queries/0_stateless/01306_polygons_intersection.sql b/tests/queries/0_stateless/01306_polygons_intersection.sql new file mode 100644 index 00000000000..d1ce5b54be3 --- /dev/null +++ b/tests/queries/0_stateless/01306_polygons_intersection.sql @@ -0,0 +1 @@ +select polygonsIntersection([[[(0, 0),(0, 3),(1, 2.9),(2, 2.6),(2.6, 2),(2.9, 1),(3, 0),(0, 0)]]], [[[(1, 1),(1, 4),(4, 4),(4, 1),(1, 1)]]]) diff --git a/tests/queries/0_stateless/01307_polygon_perimeter.reference b/tests/queries/0_stateless/01307_polygon_perimeter.reference new file mode 100644 index 00000000000..209e3ef4b62 --- /dev/null +++ b/tests/queries/0_stateless/01307_polygon_perimeter.reference @@ -0,0 +1 @@ +20 diff --git a/tests/queries/0_stateless/01307_polygon_perimeter.sql b/tests/queries/0_stateless/01307_polygon_perimeter.sql new file mode 100644 index 00000000000..ae178f4d38d --- /dev/null +++ b/tests/queries/0_stateless/01307_polygon_perimeter.sql @@ -0,0 +1 @@ +select polygonPerimeter([[[(0, 0), (0., 5), (5, 5), (5., 0)]]]) \ No newline at end of file diff --git a/tests/queries/0_stateless/01308_polygon_area.reference b/tests/queries/0_stateless/01308_polygon_area.reference new file mode 100644 index 00000000000..7273c0fa8c5 --- /dev/null +++ b/tests/queries/0_stateless/01308_polygon_area.reference @@ -0,0 +1 @@ +25 diff --git a/tests/queries/0_stateless/01308_polygon_area.sql b/tests/queries/0_stateless/01308_polygon_area.sql new file mode 100644 index 00000000000..b3b12047fd0 --- /dev/null +++ b/tests/queries/0_stateless/01308_polygon_area.sql @@ -0,0 +1 @@ +select polygonArea([[[(0, 0), (0., 5), (5, 5), (5., 0)]]]) From 373b429e61b2a20e72b62e4442b8e84f2516fd9d Mon Sep 17 00:00:00 2001 From: Alexey Ilyukhov Date: Mon, 22 Jun 2020 21:04:42 +0300 Subject: [PATCH 272/544] Add more tests --- .../0_stateless/01300_read_wkt.reference | 16 ++++++ tests/queries/0_stateless/01300_read_wkt.sql | 28 ++++++++++ tests/queries/0_stateless/01300_svg.reference | 56 +++++++++++++++++++ tests/queries/0_stateless/01300_svg.sql | 48 ++++++++++++++++ tests/queries/0_stateless/01300_wkt.reference | 16 ++++++ tests/queries/0_stateless/01300_wkt.sql | 32 +++++++++++ 6 files changed, 196 insertions(+) create mode 100644 tests/queries/0_stateless/01300_read_wkt.reference create mode 100644 tests/queries/0_stateless/01300_read_wkt.sql create mode 100644 tests/queries/0_stateless/01300_svg.reference create mode 100644 tests/queries/0_stateless/01300_svg.sql create mode 100644 tests/queries/0_stateless/01300_wkt.reference create mode 100644 tests/queries/0_stateless/01300_wkt.sql diff --git a/tests/queries/0_stateless/01300_read_wkt.reference b/tests/queries/0_stateless/01300_read_wkt.reference new file mode 100644 index 00000000000..acee1c8f14b --- /dev/null +++ b/tests/queries/0_stateless/01300_read_wkt.reference @@ -0,0 +1,16 @@ +(0,0) +[[(1,0),(10,0),(10,10),(0,10),(1,0)]] +[[(0,0),(10,0),(10,10),(0,10),(0,0)],[(4,4),(5,4),(5,5),(4,5),(4,4)]] +[[[(2,0),(10,0),(10,10),(0,10),(2,0)],[(4,4),(5,4),(5,5),(4,5),(4,4)]],[[(-10,-10),(-10,-9),(-9,10),(-10,-10)]]] +(0,0) +(1,0) +(2,0) +[[(1,0),(10,0),(10,10),(0,10),(1,0)]] +[[(0,0),(10,0),(10,10),(0,10),(0,0)]] +[[(2,0),(10,0),(10,10),(0,10),(2,0)]] +[[(0,0),(10,0),(10,10),(0,10),(0,0)],[(4,4),(5,4),(5,5),(4,5),(4,4)]] +[[(2,0),(10,0),(10,10),(0,10),(2,0)],[(4,4),(5,4),(5,5),(4,5),(4,4)]] +[[(1,0),(10,0),(10,10),(0,10),(1,0)],[(4,4),(5,4),(5,5),(4,5),(4,4)]] +[[[(1,0),(10,0),(10,10),(0,10),(1,0)],[(4,4),(5,4),(5,5),(4,5),(4,4)]],[[(-10,-10),(-10,-9),(-9,10),(-10,-10)]]] +[[[(0,0),(10,0),(10,10),(0,10),(0,0)],[(4,4),(5,4),(5,5),(4,5),(4,4)]],[[(-10,-10),(-10,-9),(-9,10),(-10,-10)]]] +[[[(2,0),(10,0),(10,10),(0,10),(2,0)],[(4,4),(5,4),(5,5),(4,5),(4,4)]],[[(-10,-10),(-10,-9),(-9,10),(-10,-10)]]] diff --git a/tests/queries/0_stateless/01300_read_wkt.sql b/tests/queries/0_stateless/01300_read_wkt.sql new file mode 100644 index 00000000000..590305fddae --- /dev/null +++ b/tests/queries/0_stateless/01300_read_wkt.sql @@ -0,0 +1,28 @@ +SELECT readWktPoint('POINT(0 0)'); +SELECT readWktPolygon('POLYGON((1 0,10 0,10 10,0 10,1 0))'); +SELECT readWktPolygon('POLYGON((0 0,10 0,10 10,0 10,0 0),(4 4,5 4,5 5,4 5,4 4))'); +SELECT readWktMultiPolygon('MULTIPOLYGON(((2 0,10 0,10 10,0 10,2 0),(4 4,5 4,5 5,4 5,4 4)),((-10 -10,-10 -9,-9 10,-10 -10)))'); + +DROP TABLE IF EXISTS geo; +CREATE TABLE geo (s String, id Int) engine=Memory(); +INSERT INTO geo VALUES ('POINT(0 0)', 1); +INSERT INTO geo VALUES ('POINT(1 0)', 2); +INSERT INTO geo VALUES ('POINT(2 0)', 3); +SELECT readWktPoint(s) FROM geo ORDER BY id; + +DROP TABLE IF EXISTS geo; +CREATE TABLE geo (s String, id Int) engine=Memory(); +INSERT INTO geo VALUES ('POLYGON((1 0,10 0,10 10,0 10,1 0))', 1); +INSERT INTO geo VALUES ('POLYGON((0 0,10 0,10 10,0 10,0 0))', 2); +INSERT INTO geo VALUES ('POLYGON((2 0,10 0,10 10,0 10,2 0))', 3); +INSERT INTO geo VALUES ('POLYGON((0 0,10 0,10 10,0 10,0 0),(4 4,5 4,5 5,4 5,4 4))', 4); +INSERT INTO geo VALUES ('POLYGON((2 0,10 0,10 10,0 10,2 0),(4 4,5 4,5 5,4 5,4 4))', 5); +INSERT INTO geo VALUES ('POLYGON((1 0,10 0,10 10,0 10,1 0),(4 4,5 4,5 5,4 5,4 4))', 6); +SELECT readWktPolygon(s) FROM geo ORDER BY id; + +DROP TABLE IF EXISTS geo; +CREATE TABLE geo (s String, id Int) engine=Memory(); +INSERT INTO geo VALUES ('MULTIPOLYGON(((1 0,10 0,10 10,0 10,1 0),(4 4,5 4,5 5,4 5,4 4)),((-10 -10,-10 -9,-9 10,-10 -10)))', 1); +INSERT INTO geo VALUES ('MULTIPOLYGON(((0 0,10 0,10 10,0 10,0 0),(4 4,5 4,5 5,4 5,4 4)),((-10 -10,-10 -9,-9 10,-10 -10)))', 2); +INSERT INTO geo VALUES ('MULTIPOLYGON(((2 0,10 0,10 10,0 10,2 0),(4 4,5 4,5 5,4 5,4 4)),((-10 -10,-10 -9,-9 10,-10 -10)))', 3); +SELECT readWktMultiPolygon(s) FROM geo ORDER BY id; diff --git a/tests/queries/0_stateless/01300_svg.reference b/tests/queries/0_stateless/01300_svg.reference new file mode 100644 index 00000000000..925d3eed011 --- /dev/null +++ b/tests/queries/0_stateless/01300_svg.reference @@ -0,0 +1,56 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/tests/queries/0_stateless/01300_svg.sql b/tests/queries/0_stateless/01300_svg.sql new file mode 100644 index 00000000000..d82572eeeba --- /dev/null +++ b/tests/queries/0_stateless/01300_svg.sql @@ -0,0 +1,48 @@ +SELECT svg((0, 0)); +SELECT svg([(0, 0), (10, 0), (10, 10), (0, 10)]); +SELECT svg([[(0, 0), (10, 0), (10, 10), (0, 10)], [(4, 4), (5, 4), (5, 5), (4, 5)]]); +SELECT svg([[[(0, 0), (10, 0), (10, 10), (0, 10)], [(4, 4), (5, 4), (5, 5), (4, 5)]], [[(-10, -10), (-10, -9), (-9, 10)]]]); +SELECT svg((0, 0), 'b'); +SELECT svg([(0, 0), (10, 0), (10, 10), (0, 10)], 'b'); +SELECT svg([[(0, 0), (10, 0), (10, 10), (0, 10)], [(4, 4), (5, 4), (5, 5), (4, 5)]], 'b'); +SELECT svg([[[(0, 0), (10, 0), (10, 10), (0, 10)], [(4, 4), (5, 4), (5, 5), (4, 5)]], [[(-10, -10), (-10, -9), (-9, 10)]]], 'b'); + +DROP TABLE IF EXISTS geo; +CREATE TABLE geo (p Tuple(Float64, Float64), s String, id Int) engine=Memory(); +INSERT INTO geo VALUES ((0, 0), 'b', 1); +INSERT INTO geo VALUES ((1, 0), 'c', 2); +INSERT INTO geo VALUES ((2, 0), 'd', 3); +SELECT svg(p) FROM geo ORDER BY id; +SELECT svg(p, 'b') FROM geo ORDER BY id; +SELECT svg((0, 0), s) FROM geo ORDER BY id; +SELECT svg(p, s) FROM geo ORDER BY id; + +DROP TABLE IF EXISTS geo; +CREATE TABLE geo (p Array(Tuple(Float64, Float64)), s String, id Int) engine=Memory(); +INSERT INTO geo VALUES ([(0, 0), (10, 0), (10, 10), (0, 10)], 'b', 1); +INSERT INTO geo VALUES ([(1, 0), (10, 0), (10, 10), (0, 10)], 'c', 2); +INSERT INTO geo VALUES ([(2, 0), (10, 0), (10, 10), (0, 10)], 'd', 3); +SELECT svg(p) FROM geo ORDER BY id; +SELECT svg(p, 'b') FROM geo ORDER BY id; +SELECT svg([(0, 0), (10, 0), (10, 10), (0, 10)], s) FROM geo ORDER BY id; +SELECT svg(p, s) FROM geo ORDER BY id; + +DROP TABLE IF EXISTS geo; +CREATE TABLE geo (p Array(Array(Tuple(Float64, Float64))), s String, id Int) engine=Memory(); +INSERT INTO geo VALUES ([[(0, 0), (10, 0), (10, 10), (0, 10)], [(4, 4), (5, 4), (5, 5), (4, 5)]], 'b', 1); +INSERT INTO geo VALUES ([[(1, 0), (10, 0), (10, 10), (0, 10)], [(4, 4), (5, 4), (5, 5), (4, 5)]], 'c', 2); +INSERT INTO geo VALUES ([[(2, 0), (10, 0), (10, 10), (0, 10)], [(4, 4), (5, 4), (5, 5), (4, 5)]], 'd', 3); +SELECT svg(p) FROM geo ORDER BY id; +SELECT svg(p, 'b') FROM geo ORDER BY id; +SELECT svg([[(0, 0), (10, 0), (10, 10), (0, 10)], [(4, 4), (5, 4), (5, 5), (4, 5)]], s) FROM geo ORDER BY id; +SELECT svg(p, s) FROM geo ORDER BY id; + +DROP TABLE IF EXISTS geo; +CREATE TABLE geo (p Array(Array(Array(Tuple(Float64, Float64)))), s String, id Int) engine=Memory(); +INSERT INTO geo VALUES ([[[(0, 0), (10, 0), (10, 10), (0, 10)], [(4, 4), (5, 4), (5, 5), (4, 5)]], [[(-10, -10), (-10, -9), (-9, 10)]]], 'b', 1); +INSERT INTO geo VALUES ([[[(1, 0), (10, 0), (10, 10), (0, 10)], [(4, 4), (5, 4), (5, 5), (4, 5)]], [[(-10, -10), (-10, -9), (-9, 10)]]], 'c', 2); +INSERT INTO geo VALUES ([[[(2, 0), (10, 0), (10, 10), (0, 10)], [(4, 4), (5, 4), (5, 5), (4, 5)]], [[(-10, -10), (-10, -9), (-9, 10)]]], 'd', 3); +SELECT svg(p) FROM geo ORDER BY id; +SELECT svg(p, 'b') FROM geo ORDER BY id; +SELECT svg([[[(0, 0), (10, 0), (10, 10), (0, 10)], [(4, 4), (5, 4), (5, 5), (4, 5)]], [[(-10, -10), (-10, -9), (-9, 10)]]], s) FROM geo ORDER BY id; +SELECT svg(p, s) FROM geo ORDER BY id; diff --git a/tests/queries/0_stateless/01300_wkt.reference b/tests/queries/0_stateless/01300_wkt.reference new file mode 100644 index 00000000000..c3ad0f4be67 --- /dev/null +++ b/tests/queries/0_stateless/01300_wkt.reference @@ -0,0 +1,16 @@ +POINT(0 0) +POLYGON((0 0,10 0,10 10,0 10,0 0)) +POLYGON((0 0,10 0,10 10,0 10,0 0),(4 4,5 4,5 5,4 5,4 4)) +MULTIPOLYGON(((0 0,10 0,10 10,0 10,0 0),(4 4,5 4,5 5,4 5,4 4)),((-10 -10,-10 -9,-9 10,-10 -10))) +POINT(0 0) +POINT(1 0) +POINT(2 0) +POLYGON((0 0,10 0,10 10,0 10,0 0)) +POLYGON((1 0,10 0,10 10,0 10,1 0)) +POLYGON((2 0,10 0,10 10,0 10,2 0)) +POLYGON((0 0,10 0,10 10,0 10,0 0),(4 4,5 4,5 5,4 5,4 4)) +POLYGON((1 0,10 0,10 10,0 10,1 0),(4 4,5 4,5 5,4 5,4 4)) +POLYGON((2 0,10 0,10 10,0 10,2 0),(4 4,5 4,5 5,4 5,4 4)) +MULTIPOLYGON(((0 0,10 0,10 10,0 10,0 0),(4 4,5 4,5 5,4 5,4 4)),((-10 -10,-10 -9,-9 10,-10 -10))) +MULTIPOLYGON(((1 0,10 0,10 10,0 10,1 0),(4 4,5 4,5 5,4 5,4 4)),((-10 -10,-10 -9,-9 10,-10 -10))) +MULTIPOLYGON(((2 0,10 0,10 10,0 10,2 0),(4 4,5 4,5 5,4 5,4 4)),((-10 -10,-10 -9,-9 10,-10 -10))) diff --git a/tests/queries/0_stateless/01300_wkt.sql b/tests/queries/0_stateless/01300_wkt.sql new file mode 100644 index 00000000000..a79d1a0c150 --- /dev/null +++ b/tests/queries/0_stateless/01300_wkt.sql @@ -0,0 +1,32 @@ +SELECT wkt((0, 0)); +SELECT wkt([(0, 0), (10, 0), (10, 10), (0, 10)]); +SELECT wkt([[(0, 0), (10, 0), (10, 10), (0, 10)], [(4, 4), (5, 4), (5, 5), (4, 5)]]); +SELECT wkt([[[(0, 0), (10, 0), (10, 10), (0, 10)], [(4, 4), (5, 4), (5, 5), (4, 5)]], [[(-10, -10), (-10, -9), (-9, 10)]]]); + +DROP TABLE IF EXISTS geo; +CREATE TABLE geo (p Tuple(Float64, Float64), id Int) engine=Memory(); +INSERT INTO geo VALUES ((0, 0), 1); +INSERT INTO geo VALUES ((1, 0), 2); +INSERT INTO geo VALUES ((2, 0), 3); +SELECT wkt(p) FROM geo ORDER BY id; + +DROP TABLE IF EXISTS geo; +CREATE TABLE geo (p Array(Tuple(Float64, Float64)), id Int) engine=Memory(); +INSERT INTO geo VALUES ([(0, 0), (10, 0), (10, 10), (0, 10)], 1); +INSERT INTO geo VALUES ([(1, 0), (10, 0), (10, 10), (0, 10)], 2); +INSERT INTO geo VALUES ([(2, 0), (10, 0), (10, 10), (0, 10)], 3); +SELECT wkt(p) FROM geo ORDER BY id; + +DROP TABLE IF EXISTS geo; +CREATE TABLE geo (p Array(Array(Tuple(Float64, Float64))), id Int) engine=Memory(); +INSERT INTO geo VALUES ([[(0, 0), (10, 0), (10, 10), (0, 10)], [(4, 4), (5, 4), (5, 5), (4, 5)]], 1); +INSERT INTO geo VALUES ([[(1, 0), (10, 0), (10, 10), (0, 10)], [(4, 4), (5, 4), (5, 5), (4, 5)]], 2); +INSERT INTO geo VALUES ([[(2, 0), (10, 0), (10, 10), (0, 10)], [(4, 4), (5, 4), (5, 5), (4, 5)]], 3); +SELECT wkt(p) FROM geo ORDER BY id; + +DROP TABLE IF EXISTS geo; +CREATE TABLE geo (p Array(Array(Array(Tuple(Float64, Float64)))), id Int) engine=Memory(); +INSERT INTO geo VALUES ([[[(0, 0), (10, 0), (10, 10), (0, 10)], [(4, 4), (5, 4), (5, 5), (4, 5)]], [[(-10, -10), (-10, -9), (-9, 10)]]], 1); +INSERT INTO geo VALUES ([[[(1, 0), (10, 0), (10, 10), (0, 10)], [(4, 4), (5, 4), (5, 5), (4, 5)]], [[(-10, -10), (-10, -9), (-9, 10)]]], 2); +INSERT INTO geo VALUES ([[[(2, 0), (10, 0), (10, 10), (0, 10)], [(4, 4), (5, 4), (5, 5), (4, 5)]], [[(-10, -10), (-10, -9), (-9, 10)]]], 3); +SELECT wkt(p) FROM geo ORDER BY id; From 0425d566d303072a96d9559acf020257d2a100a9 Mon Sep 17 00:00:00 2001 From: Nikita Mikhailov Date: Tue, 19 Jan 2021 02:51:34 +0300 Subject: [PATCH 273/544] better --- src/Functions/PolygonsUnion.cpp | 36 ++-- src/Functions/geometryConverters.cpp | 56 +++-- src/Functions/geometryConverters.h | 204 +++++++++++------- src/Functions/polygonArea.cpp | 103 +++++---- src/Functions/polygonConvexHull.cpp | 30 +-- src/Functions/polygonPerimeter.cpp | 86 +++++--- src/Functions/polygonsDistance.cpp | 121 ++++++----- src/Functions/polygonsEquals.cpp | 26 +-- src/Functions/polygonsIntersection.cpp | 35 ++- src/Functions/polygonsSymDifference.cpp | 36 ++-- src/Functions/polygonsWithin.cpp | 26 +-- src/Functions/readWkt.cpp | 12 +- src/Functions/registerFunctionsGeo.cpp | 18 +- src/Functions/svg.cpp | 12 +- src/Functions/wkt.cpp | 6 +- .../01300_polygon_convex_hull.reference | 2 +- .../01302_polygons_distance.reference | 2 + .../0_stateless/01302_polygons_distance.sql | 10 +- .../0_stateless/01307_polygon_perimeter.sql | 2 +- .../0_stateless/01308_polygon_area.reference | 1 + .../0_stateless/01308_polygon_area.sql | 3 +- 21 files changed, 481 insertions(+), 346 deletions(-) diff --git a/src/Functions/PolygonsUnion.cpp b/src/Functions/PolygonsUnion.cpp index 8a8877257ba..37a9b684805 100644 --- a/src/Functions/PolygonsUnion.cpp +++ b/src/Functions/PolygonsUnion.cpp @@ -28,14 +28,6 @@ namespace ErrorCodes extern const int ILLEGAL_COLUMN; } - -using CoordinateType = Float64; -using Point = boost::geometry::model::d2::point_xy; -using Polygon = boost::geometry::model::polygon; -using MultiPolygon = boost::geometry::model::multi_polygon; -using Box = boost::geometry::model::box; - - class FunctionPolygonsUnion : public IFunction { public: @@ -68,17 +60,17 @@ public: return DataTypeCustomMultiPolygonSerialization::nestedDataType(); } - void executeImpl(Block & block, const ColumnNumbers & arguments, size_t result, size_t input_rows_count) override + ColumnPtr executeImpl(const ColumnsWithTypeAndName & arguments, const DataTypePtr & /*result_type*/, size_t input_rows_count) const override { - auto get_parser = [&block, &arguments] (size_t i) { + auto get_parser = [&arguments] (size_t i) { const auto * const_col = - checkAndGetColumn(block.getByPosition(arguments[i]).column.get()); + checkAndGetColumn(arguments[i].column.get()); bool is_const = static_cast(const_col); - return std::pair{is_const, is_const ? - makeGeometryFromColumnParser(ColumnWithTypeAndName(const_col->getDataColumnPtr(), block.getByPosition(arguments[i]).type, block.getByPosition(arguments[i]).name)) : - makeGeometryFromColumnParser(block.getByPosition(arguments[i]))}; + return std::pair{is_const, is_const ? + makeCartesianGeometryFromColumnParser(ColumnWithTypeAndName(const_col->getDataColumnPtr(), arguments[i].type, arguments[i].name)) : + makeCartesianGeometryFromColumnParser(arguments[i])}; }; auto [is_first_polygon_const, first_parser] = get_parser(0); @@ -87,7 +79,7 @@ public: auto [is_second_polygon_const, second_parser] = get_parser(1); auto second_container = createContainer(second_parser); - Float64MultiPolygonSerializer serializer; + CartesianMultiPolygonSerializer serializer; for (size_t i = 0; i < input_rows_count; i++) { @@ -96,19 +88,19 @@ public: if (!is_second_polygon_const || i == 0) get(second_parser, second_container, i); - Float64Geometry polygons_union = Float64MultiPolygon({{{{}}}}); + CartesianGeometry polygons_union = CartesianMultiPolygon({{{{}}}}); boost::geometry::union_( - boost::get(first_container), - boost::get(second_container), - boost::get(polygons_union)); + boost::get(first_container), + boost::get(second_container), + boost::get(polygons_union)); - boost::get(polygons_union).erase( - boost::get(polygons_union).begin()); + boost::get(polygons_union).erase( + boost::get(polygons_union).begin()); serializer.add(polygons_union); } - block.getByPosition(result).column = std::move(serializer.finalize()); + return serializer.finalize(); } bool useDefaultImplementationForConstants() const override diff --git a/src/Functions/geometryConverters.cpp b/src/Functions/geometryConverters.cpp index 12ef6f17d41..da2ddd9a473 100644 --- a/src/Functions/geometryConverters.cpp +++ b/src/Functions/geometryConverters.cpp @@ -9,32 +9,32 @@ namespace { size_t getArrayDepth(DataTypePtr data_type, size_t max_depth) { - LOG_FATAL(&Poco::Logger::get("geoconv"), "getting depth"); size_t depth = 0; while (data_type && isArray(data_type) && depth != max_depth + 1) { depth++; data_type = static_cast(*data_type).getNestedType(); } - LOG_FATAL(&Poco::Logger::get("geoconv"), "got depth"); return depth; } -class ContainerCreator : public boost::static_visitor +template +class ContainerCreator : public boost::static_visitor { public: template - Float64Geometry operator()(const T & parser) const + Geometry operator()(const T & parser) const { return parser.createContainer(); } }; +template class Getter : public boost::static_visitor { public: - constexpr Getter(Float64Geometry & container_, size_t i_) + constexpr Getter(Geometry & container_, size_t i_) : container(container_) , i(i_) {} @@ -46,7 +46,7 @@ public: } private: - Float64Geometry & container; + Geometry & container; size_t i; }; @@ -64,26 +64,50 @@ Parser makeParser(const ColumnWithTypeAndName & col) } -GeometryFromColumnParser makeGeometryFromColumnParser(const ColumnWithTypeAndName & col) +CartesianGeometryFromColumnParser makeCartesianGeometryFromColumnParser(const ColumnWithTypeAndName & col) { - LOG_FATAL(&Poco::Logger::get("geoconv"), "Fine"); - switch (getArrayDepth(col.type, 3)) { - case 0: return makeParser(col); - case 1: return makeParser(col); - case 2: return makeParser(col); - case 3: return makeParser(col); + switch (getArrayDepth(col.type, 3)) + { + case 0: return makeParser>(col); + case 1: return makeParser(col); + case 2: return makeParser(col); + case 3: return makeParser(col); default: throw Exception("Cannot parse geometry from column with type " + col.type->getName() + ", array depth is too big", ErrorCodes::ILLEGAL_COLUMN); } } -Float64Geometry createContainer(const GeometryFromColumnParser & parser) +CartesianGeometry createContainer(const CartesianGeometryFromColumnParser & parser) { - static ContainerCreator creator; + static ContainerCreator creator; return boost::apply_visitor(creator, parser); } -void get(const GeometryFromColumnParser & parser, Float64Geometry & container, size_t i) +void get(const CartesianGeometryFromColumnParser & parser, CartesianGeometry & container, size_t i) +{ + boost::apply_visitor(Getter(container, i), parser); +} + +GeographicGeometryFromColumnParser makeGeographicGeometryFromColumnParser(const ColumnWithTypeAndName & col) +{ + switch (getArrayDepth(col.type, 3)) + { + case 0: return makeParser>(col); + case 1: return makeParser(col); + case 2: return makeParser(col); + case 3: return makeParser(col); + default: throw Exception("Cannot parse geometry from column with type " + col.type->getName() + + ", array depth is too big", ErrorCodes::ILLEGAL_COLUMN); + } +} + +GeographicGeometry createContainer(const GeographicGeometryFromColumnParser & parser) +{ + static ContainerCreator creator; + return boost::apply_visitor(creator, parser); +} + +void get(const GeographicGeometryFromColumnParser & parser, GeographicGeometry & container, size_t i) { boost::apply_visitor(Getter(container, i), parser); } diff --git a/src/Functions/geometryConverters.h b/src/Functions/geometryConverters.h index 9983d3afdf3..ee22ce64ddf 100644 --- a/src/Functions/geometryConverters.h +++ b/src/Functions/geometryConverters.h @@ -26,49 +26,89 @@ namespace ErrorCodes extern const int BAD_ARGUMENTS; } -using Float64Point = boost::geometry::model::d2::point_xy; -using Float64Ring = boost::geometry::model::ring; -using Float64Polygon = boost::geometry::model::polygon; -using Float64MultiPolygon = boost::geometry::model::multi_polygon; -using Float64Geometry = boost::variant; +namespace bg = boost::geometry; +using CartesianPoint = bg::model::d2::point_xy; +using CartesianRing = bg::model::ring; +using CartesianPolygon = bg::model::polygon; +using CartesianMultiPolygon = bg::model::multi_polygon; +using CartesianGeometry = boost::variant; -class Float64PointFromColumnParser +using GeographicPoint = bg::model::point>; +using GeographicRing = bg::model::ring; +using GeographicPolygon = bg::model::polygon; +using GeographicMultiPolygon = bg::model::multi_polygon; +using GeographicGeometry = boost::variant; + +/** + * Class which takes some boost type and returns a pair of numbers. + * They are (x,y) in case of cartesian coordinated and (lon,lat) in case of geographic. +*/ +template +class PointFromColumnParser { public: - Float64PointFromColumnParser(ColumnPtr col_) - : col(col_) + PointFromColumnParser(ColumnPtr col_) : col(col_) { const auto & tuple = static_cast(*col_); const auto & tuple_columns = tuple.getColumns(); +#ifndef NDEBUG + size = tuple.size(); +#endif const auto & x_data = static_cast(*tuple_columns[0]); - x = x_data.getData().data(); + first = x_data.getData().data(); const auto & y_data = static_cast(*tuple_columns[1]); - y = y_data.getData().data(); + second = y_data.getData().data(); } - - Float64Geometry createContainer() const + template + typename std::enable_if_t, CartesianGeometry> createContainer() const { - return Float64Point(); + return CartesianPoint(); } - void get(Float64Geometry & container, size_t i) const + template + typename std::enable_if_t, GeographicGeometry> createContainer() const { - get(boost::get(container), i); + return GeographicPoint(); } - void get(Float64Point & container, size_t i) const + template + void get(std::enable_if_t, CartesianGeometry> & container, size_t i) const { - boost::geometry::set<0>(container, x[i]); - boost::geometry::set<1>(container, y[i]); +#ifndef NDEBUG + assert(i < size); +#endif + get(boost::get(container), i); } + + template + void get(std::enable_if_t, GeographicGeometry> & container, size_t i) const + { +#ifndef NDEBUG + assert(i < size); +#endif + get(boost::get(container), i); + } + + void get(PointType & container, size_t i) const + { +#ifndef NDEBUG + assert(i < size); +#endif + boost::geometry::set<0>(container, first[i]); + boost::geometry::set<1>(container, second[i]); + } + private: + /// Note, this is needed to prevent use-after-free. ColumnPtr col; - - const Float64 * x; - const Float64 * y; +#ifndef NDEBUG + size_t size; +#endif + const Float64 * first; + const Float64 * second; }; template @@ -142,8 +182,6 @@ public: ringParser.get(container.outer(), l); - LOG_FATAL(&Poco::Logger::get("geoconv"), "polygon: l = {}, r = {}", l, r); - container.inners().resize(r - l - 1); for (size_t j = l + 1; j < r; j++) { @@ -176,8 +214,6 @@ public: size_t l = offsets[i - 1]; size_t r = offsets[i]; - LOG_FATAL(&Poco::Logger::get("geoconv"), "multipolygon: l = {}, r = {}", l, r); - multi_polygon.resize(r - l); for (size_t j = l; j < r; j++) { @@ -190,43 +226,65 @@ private: const PolygonParser polygonParser; }; -using Float64RingFromColumnParser = RingFromColumnParser; -using Float64PolygonFromColumnParser = PolygonFromColumnParser; -using Float64MultiPolygonFromColumnParser = MultiPolygonFromColumnParser; +/// Cartesian coordinates -using GeometryFromColumnParser = boost::variant< - Float64PointFromColumnParser, - Float64RingFromColumnParser, - Float64PolygonFromColumnParser, - Float64MultiPolygonFromColumnParser +using CartesianRingFromColumnParser = RingFromColumnParser>; +using CartesianPolygonFromColumnParser = PolygonFromColumnParser; +using CartesianMultiPolygonFromColumnParser = MultiPolygonFromColumnParser; + +using CartesianGeometryFromColumnParser = boost::variant< + PointFromColumnParser, + CartesianRingFromColumnParser, + CartesianPolygonFromColumnParser, + CartesianMultiPolygonFromColumnParser >; -Float64Geometry createContainer(const GeometryFromColumnParser & parser); +CartesianGeometry createContainer(const CartesianGeometryFromColumnParser & parser); -void get(const GeometryFromColumnParser & parser, Float64Geometry & container, size_t i); +void get(const CartesianGeometryFromColumnParser & parser, CartesianGeometry & container, size_t i); -GeometryFromColumnParser makeGeometryFromColumnParser(const ColumnWithTypeAndName & col); +CartesianGeometryFromColumnParser makeCartesianGeometryFromColumnParser(const ColumnWithTypeAndName & col); -class Float64PointSerializerVisitor : public boost::static_visitor +/// Geographic coordinates + +using GeographicRingFromColumnParser = RingFromColumnParser>; +using GeographicPolygonFromColumnParser = PolygonFromColumnParser; +using GeographicMultiPolygonFromColumnParser = MultiPolygonFromColumnParser; + +using GeographicGeometryFromColumnParser = boost::variant< + PointFromColumnParser, + GeographicRingFromColumnParser, + GeographicPolygonFromColumnParser, + GeographicMultiPolygonFromColumnParser +>; + +GeographicGeometry createContainer(const GeographicGeometryFromColumnParser & parser); + +void get(const GeographicGeometryFromColumnParser & parser, GeographicGeometry & container, size_t i); + +GeographicGeometryFromColumnParser makeGeographicGeometryFromColumnParser(const ColumnWithTypeAndName & col); + + +class CartesianPointSerializerVisitor : public boost::static_visitor { public: - Float64PointSerializerVisitor() + CartesianPointSerializerVisitor() : x(ColumnFloat64::create()) , y(ColumnFloat64::create()) {} - Float64PointSerializerVisitor(size_t n) + CartesianPointSerializerVisitor(size_t n) : x(ColumnFloat64::create(n)) , y(ColumnFloat64::create(n)) {} - void operator()(const Float64Point & point) + void operator()(const CartesianPoint & point) { - x->insertValue(point.x()); - y->insertValue(point.y()); + x->insertValue(point.get<0>()); + y->insertValue(point.get<0>()); } - void operator()(const Float64Ring & ring) + void operator()(const CartesianRing & ring) { if (ring.size() != 1) { throw Exception("Unable to write ring of size " + toString(ring.size()) + " != 1 to point column", ErrorCodes::BAD_ARGUMENTS); @@ -234,7 +292,7 @@ public: (*this)(ring[0]); } - void operator()(const Float64Polygon & polygon) + void operator()(const CartesianPolygon & polygon) { if (polygon.inners().size() != 0) { throw Exception("Unable to write polygon with holes to point column", ErrorCodes::BAD_ARGUMENTS); @@ -242,7 +300,7 @@ public: (*this)(polygon.outer()); } - void operator()(const Float64MultiPolygon & multi_polygon) + void operator()(const CartesianMultiPolygon & multi_polygon) { if (multi_polygon.size() != 1) { throw Exception("Unable to write multi-polygon of size " + toString(multi_polygon.size()) + " != 1 to point column", ErrorCodes::BAD_ARGUMENTS); @@ -264,18 +322,18 @@ private: ColumnFloat64::MutablePtr y; }; -class Float64RingSerializerVisitor : public boost::static_visitor +class CartesianRingSerializerVisitor : public boost::static_visitor { public: - Float64RingSerializerVisitor() + CartesianRingSerializerVisitor() : offsets(ColumnUInt64::create()) {} - Float64RingSerializerVisitor(size_t n) + CartesianRingSerializerVisitor(size_t n) : offsets(ColumnUInt64::create(n)) {} - void operator()(const Float64Point & point) + void operator()(const CartesianPoint & point) { size++; offsets->insertValue(size); @@ -283,7 +341,7 @@ public: pointSerializer(point); } - void operator()(const Float64Ring & ring) + void operator()(const CartesianRing & ring) { size += ring.size(); offsets->insertValue(size); @@ -293,7 +351,7 @@ public: } } - void operator()(const Float64Polygon & polygon) + void operator()(const CartesianPolygon & polygon) { if (polygon.inners().size() != 0) { throw Exception("Unable to write polygon with holes to ring column", ErrorCodes::BAD_ARGUMENTS); @@ -301,7 +359,7 @@ public: (*this)(polygon.outer()); } - void operator()(const Float64MultiPolygon & multi_polygon) + void operator()(const CartesianMultiPolygon & multi_polygon) { if (multi_polygon.size() != 1) { throw Exception("Unable to write multi-polygon of size " + toString(multi_polygon.size()) + " != 1 to ring column", ErrorCodes::BAD_ARGUMENTS); @@ -316,36 +374,36 @@ public: private: size_t size = 0; - Float64PointSerializerVisitor pointSerializer; + CartesianPointSerializerVisitor pointSerializer; ColumnUInt64::MutablePtr offsets; }; -class Float64PolygonSerializerVisitor : public boost::static_visitor +class CartesianPolygonSerializerVisitor : public boost::static_visitor { public: - Float64PolygonSerializerVisitor() + CartesianPolygonSerializerVisitor() : offsets(ColumnUInt64::create()) {} - Float64PolygonSerializerVisitor(size_t n) + CartesianPolygonSerializerVisitor(size_t n) : offsets(ColumnUInt64::create(n)) {} - void operator()(const Float64Point & point) + void operator()(const CartesianPoint & point) { size++; offsets->insertValue(size); ringSerializer(point); } - void operator()(const Float64Ring & ring) + void operator()(const CartesianRing & ring) { size++; offsets->insertValue(size); ringSerializer(ring); } - void operator()(const Float64Polygon & polygon) + void operator()(const CartesianPolygon & polygon) { size += 1 + polygon.inners().size(); offsets->insertValue(size); @@ -356,7 +414,7 @@ public: } } - void operator()(const Float64MultiPolygon & multi_polygon) + void operator()(const CartesianMultiPolygon & multi_polygon) { if (multi_polygon.size() != 1) { throw Exception("Unable to write multi-polygon of size " + toString(multi_polygon.size()) + " != 1 to polygon column", ErrorCodes::BAD_ARGUMENTS); @@ -371,43 +429,43 @@ public: private: size_t size = 0; - Float64RingSerializerVisitor ringSerializer; + CartesianRingSerializerVisitor ringSerializer; ColumnUInt64::MutablePtr offsets; }; -class Float64MultiPolygonSerializerVisitor : public boost::static_visitor +class CartesianMultiPolygonSerializerVisitor : public boost::static_visitor { public: - Float64MultiPolygonSerializerVisitor() + CartesianMultiPolygonSerializerVisitor() : offsets(ColumnUInt64::create()) {} - Float64MultiPolygonSerializerVisitor(size_t n) + CartesianMultiPolygonSerializerVisitor(size_t n) : offsets(ColumnUInt64::create(n)) {} - void operator()(const Float64Point & point) + void operator()(const CartesianPoint & point) { size++; offsets->insertValue(size); polygonSerializer(point); } - void operator()(const Float64Ring & ring) + void operator()(const CartesianRing & ring) { size++; offsets->insertValue(size); polygonSerializer(ring); } - void operator()(const Float64Polygon & polygon) + void operator()(const CartesianPolygon & polygon) { size++; offsets->insertValue(size); polygonSerializer(polygon); } - void operator()(const Float64MultiPolygon & multi_polygon) + void operator()(const CartesianMultiPolygon & multi_polygon) { size += multi_polygon.size(); offsets->insertValue(size); @@ -424,7 +482,7 @@ public: private: size_t size = 0; - Float64PolygonSerializerVisitor polygonSerializer; + CartesianPolygonSerializerVisitor polygonSerializer; ColumnUInt64::MutablePtr offsets; }; @@ -445,9 +503,9 @@ private: Visitor visitor; }; -using Float64PointSerializer = GeometrySerializer; -using Float64RingSerializer = GeometrySerializer; -using Float64PolygonSerializer = GeometrySerializer; -using Float64MultiPolygonSerializer = GeometrySerializer; +using CartesianPointSerializer = GeometrySerializer; +using CartesianRingSerializer = GeometrySerializer; +using CartesianPolygonSerializer = GeometrySerializer; +using CartesianMultiPolygonSerializer = GeometrySerializer; } diff --git a/src/Functions/polygonArea.cpp b/src/Functions/polygonArea.cpp index a625145e620..27f5cc92461 100644 --- a/src/Functions/polygonArea.cpp +++ b/src/Functions/polygonArea.cpp @@ -28,24 +28,17 @@ namespace ErrorCodes extern const int ILLEGAL_COLUMN; } - -using CoordinateType = Float64; -using Point = boost::geometry::model::d2::point_xy; -using Polygon = boost::geometry::model::polygon; -using MultiPolygon = boost::geometry::model::multi_polygon; -using Box = boost::geometry::model::box; - - -class FunctionPolygonArea : public IFunction +template +class FunctionPolygonAreaBase : public IFunction { public: - static inline const char * name = "polygonArea"; + static inline const char * name = Derived::name; - explicit FunctionPolygonArea() = default; + explicit FunctionPolygonAreaBase() = default; static FunctionPtr create(const Context &) { - return std::make_shared(); + return std::make_shared(); } String getName() const override @@ -68,29 +61,9 @@ public: return std::make_shared(); } - void executeImpl(Block & block, const ColumnNumbers & arguments, size_t result, size_t input_rows_count) override + ColumnPtr executeImpl(const ColumnsWithTypeAndName & arguments, const DataTypePtr & result_type, size_t input_rows_count) const override { - auto get_parser = [&block, &arguments] (size_t i) { - const ColumnWithTypeAndName polygon = block.getByPosition(arguments[i]); - return makeGeometryFromColumnParser(polygon); - }; - - auto parser = get_parser(0); - auto container = createContainer(parser); - - auto res_column = ColumnFloat64::create(); - - for (size_t i = 0; i < input_rows_count; i++) - { - get(parser, container, i); - - Float64 area = boost::geometry::area( - boost::get(container)); - - res_column->insertValue(area); - } - - block.getByPosition(result).column = std::move(res_column); + return static_cast(this)->executeImplementation(arguments, result_type, input_rows_count); } bool useDefaultImplementationForConstants() const override @@ -100,9 +73,67 @@ public: }; -void registerFunctionPolygonArea(FunctionFactory & factory) +class FunctionPolygonAreaCartesian : public FunctionPolygonAreaBase { - factory.registerFunction(); +public: + static inline const char * name = "polygonAreaCartesian"; + + ColumnPtr executeImplementation(const ColumnsWithTypeAndName & arguments, const DataTypePtr & /*result_type*/, size_t input_rows_count) const + { + auto parser = makeCartesianGeometryFromColumnParser(arguments[0]); + auto container = createContainer(parser); + + auto res_column = ColumnFloat64::create(); + + for (size_t i = 0; i < input_rows_count; i++) + { + get(parser, container, i); + + Float64 area = boost::geometry::area( + boost::get(container)); + + res_column->insertValue(area); + } + + return res_column; + } +}; + +class FunctionPolygonAreaGeographic : public FunctionPolygonAreaBase +{ +public: + static inline const char * name = "polygonAreaGeographic"; + + ColumnPtr executeImplementation(const ColumnsWithTypeAndName & arguments, const DataTypePtr & /*result_type*/, size_t input_rows_count) const + { + auto parser = makeGeographicGeometryFromColumnParser(arguments[0]); + auto container = createContainer(parser); + + auto res_column = ColumnFloat64::create(); + + for (size_t i = 0; i < input_rows_count; i++) + { + get(parser, container, i); + + Float64 area = boost::geometry::area( + boost::get(container)); + + res_column->insertValue(area); + } + + return res_column; + } +}; + + +void registerFunctionPolygonAreaCartesian(FunctionFactory & factory) +{ + factory.registerFunction(); +} + +void registerFunctionPolygonAreaGeographic(FunctionFactory & factory) +{ + factory.registerFunction(); } } diff --git a/src/Functions/polygonConvexHull.cpp b/src/Functions/polygonConvexHull.cpp index 5e30a93c16a..f828ff33e6c 100644 --- a/src/Functions/polygonConvexHull.cpp +++ b/src/Functions/polygonConvexHull.cpp @@ -28,14 +28,6 @@ namespace ErrorCodes extern const int ILLEGAL_COLUMN; } - -using CoordinateType = Float64; -using Point = boost::geometry::model::d2::point_xy; -using Polygon = boost::geometry::model::polygon; -using MultiPolygon = boost::geometry::model::multi_polygon; -using Box = boost::geometry::model::box; - - class FunctionPolygonConvexHull : public IFunction { public: @@ -68,34 +60,34 @@ public: return DataTypeCustomPolygonSerialization::nestedDataType(); } - void executeImpl(Block & block, const ColumnNumbers & arguments, size_t result, size_t input_rows_count) override + ColumnPtr executeImpl(const ColumnsWithTypeAndName & arguments, const DataTypePtr & /*result_type*/, size_t input_rows_count) const override { - auto get_parser = [&block, &arguments] (size_t i) { - const ColumnWithTypeAndName polygon = block.getByPosition(arguments[i]); - return makeGeometryFromColumnParser(polygon); + auto get_parser = [&arguments] (size_t i) { + const ColumnWithTypeAndName polygon = arguments[i]; + return makeCartesianGeometryFromColumnParser(polygon); }; auto parser = get_parser(0); auto container = createContainer(parser); - Float64PolygonSerializer serializer; + CartesianPolygonSerializer serializer; for (size_t i = 0; i < input_rows_count; i++) { get(parser, container, i); - Float64Geometry convex_hull = Float64Polygon({{{}}}); + CartesianGeometry convex_hull = CartesianPolygon({{{}}}); boost::geometry::convex_hull( - boost::get(container), - boost::get(convex_hull)); + boost::get(container), + boost::get(convex_hull)); - boost::get(convex_hull).outer().erase( - boost::get(convex_hull).outer().begin()); + boost::get(convex_hull).outer().erase( + boost::get(convex_hull).outer().begin()); serializer.add(convex_hull); } - block.getByPosition(result).column = std::move(serializer.finalize()); + return serializer.finalize(); } bool useDefaultImplementationForConstants() const override diff --git a/src/Functions/polygonPerimeter.cpp b/src/Functions/polygonPerimeter.cpp index a9c42f24aaf..4b7fc8ca678 100644 --- a/src/Functions/polygonPerimeter.cpp +++ b/src/Functions/polygonPerimeter.cpp @@ -28,24 +28,17 @@ namespace ErrorCodes extern const int ILLEGAL_COLUMN; } - -using CoordinateType = Float64; -using Point = boost::geometry::model::d2::point_xy; -using Polygon = boost::geometry::model::polygon; -using MultiPolygon = boost::geometry::model::multi_polygon; -using Box = boost::geometry::model::box; - - -class FunctionPolygonPerimeter : public IFunction +template +class FunctionPolygonPerimeterBase : public IFunction { public: - static inline const char * name = "polygonPerimeter"; + static inline const char * name = Derived::name; - explicit FunctionPolygonPerimeter() = default; + explicit FunctionPolygonPerimeterBase() = default; static FunctionPtr create(const Context &) { - return std::make_shared(); + return std::make_shared(); } String getName() const override @@ -68,14 +61,25 @@ public: return std::make_shared(); } - void executeImpl(Block & block, const ColumnNumbers & arguments, size_t result, size_t input_rows_count) override + ColumnPtr executeImpl(const ColumnsWithTypeAndName & arguments, const DataTypePtr & result_type, size_t input_rows_count) const override { - auto get_parser = [&block, &arguments] (size_t i) { - const ColumnWithTypeAndName polygon = block.getByPosition(arguments[i]); - return makeGeometryFromColumnParser(polygon); - }; + return static_cast(this)->executeImplementation(arguments, result_type, input_rows_count); + } - auto parser = get_parser(0); + bool useDefaultImplementationForConstants() const override + { + return true; + } +}; + +class FunctionPolygonPerimeterCartesian : public FunctionPolygonPerimeterBase +{ +public: + static inline const char * name = "polygonPerimeterCartesian"; + + ColumnPtr executeImplementation(const ColumnsWithTypeAndName & arguments, const DataTypePtr & /*result_type*/, size_t input_rows_count) const + { + auto parser = makeCartesianGeometryFromColumnParser(arguments[0]); auto container = createContainer(parser); auto res_column = ColumnFloat64::create(); @@ -85,24 +89,52 @@ public: get(parser, container, i); Float64 perimeter = boost::geometry::perimeter( - boost::get(container)); + boost::get(container)); res_column->insertValue(perimeter); } - block.getByPosition(result).column = std::move(res_column); - } - - bool useDefaultImplementationForConstants() const override - { - return true; + return res_column; } }; -void registerFunctionPolygonPerimeter(FunctionFactory & factory) +class FunctionPolygonPerimeterGeographic : public FunctionPolygonPerimeterBase { - factory.registerFunction(); +public: + static inline const char * name = "polygonPerimeterGeographic"; + + ColumnPtr executeImplementation(const ColumnsWithTypeAndName & arguments, const DataTypePtr & /*result_type*/, size_t input_rows_count) const + { + auto parser = makeGeographicGeometryFromColumnParser(arguments[0]); + auto container = createContainer(parser); + + auto res_column = ColumnFloat64::create(); + + for (size_t i = 0; i < input_rows_count; i++) + { + get(parser, container, i); + + Float64 perimeter = boost::geometry::perimeter( + boost::get(container)); + + res_column->insertValue(perimeter); + } + + return res_column; + } +}; + + + +void registerFunctionPolygonPerimeterCartesian(FunctionFactory & factory) +{ + factory.registerFunction(); +} + +void registerFunctionPolygonPerimeterGeographic(FunctionFactory & factory) +{ + factory.registerFunction(); } } diff --git a/src/Functions/polygonsDistance.cpp b/src/Functions/polygonsDistance.cpp index fcfff7bd964..99a8d8dafe5 100644 --- a/src/Functions/polygonsDistance.cpp +++ b/src/Functions/polygonsDistance.cpp @@ -30,24 +30,17 @@ namespace ErrorCodes extern const int ILLEGAL_COLUMN; } - -using CoordinateType = Float64; -using Point = boost::geometry::model::d2::point_xy; -using Polygon = boost::geometry::model::polygon; -using MultiPolygon = boost::geometry::model::multi_polygon; -using Box = boost::geometry::model::box; - - -class FunctionPolygonsDistance : public IFunction +template +class FunctionPolygonsDistanceBase : public IFunction { public: - static inline const char * name = "polygonsDistance"; + static inline const char * name = Derived::name; - explicit FunctionPolygonsDistance() = default; + explicit FunctionPolygonsDistanceBase() = default; static FunctionPtr create(const Context &) { - return std::make_shared(); + return std::make_shared(); } String getName() const override @@ -70,42 +63,9 @@ public: return std::make_shared(); } - void executeImpl(Block & block, const ColumnNumbers & arguments, size_t result, size_t input_rows_count) override + ColumnPtr executeImpl(const ColumnsWithTypeAndName & arguments, const DataTypePtr & result_type, size_t input_rows_count) const override { - auto get_parser = [&block, &arguments] (size_t i) { - const auto * const_col = - checkAndGetColumn(block.getByPosition(arguments[i]).column.get()); - - bool is_const = static_cast(const_col); - - return std::pair{is_const, is_const ? - makeGeometryFromColumnParser(ColumnWithTypeAndName(const_col->getDataColumnPtr(), block.getByPosition(arguments[i]).type, block.getByPosition(arguments[i]).name)) : - makeGeometryFromColumnParser(block.getByPosition(arguments[i]))}; - }; - - auto [is_first_polygon_const, first_parser] = get_parser(0); - auto first_container = createContainer(first_parser); - - auto [is_second_polygon_const, second_parser] = get_parser(1); - auto second_container = createContainer(second_parser); - - auto res_column = ColumnFloat64::create(); - - for (size_t i = 0; i < input_rows_count; i++) - { - if (!is_first_polygon_const || i == 0) - get(first_parser, first_container, i); - if (!is_second_polygon_const || i == 0) - get(second_parser, second_container, i); - - Float64 distance = boost::geometry::distance( - boost::get(first_container), - boost::get(second_container)); - - res_column->insertValue(distance); - } - - block.getByPosition(result).column = std::move(res_column); + return static_cast(this)->executeImplementation(arguments, result_type, input_rows_count); } bool useDefaultImplementationForConstants() const override @@ -115,9 +75,72 @@ public: }; -void registerFunctionPolygonsDistance(FunctionFactory & factory) +class FunctionPolygonsDistanceCartesian : public FunctionPolygonsDistanceBase { - factory.registerFunction(); +public: + static inline const char * name = "polygonsDistanceCartesian"; + + ColumnPtr executeImplementation(const ColumnsWithTypeAndName & arguments, const DataTypePtr & /*result_type*/, size_t input_rows_count) const + { + auto first_parser = makeCartesianGeometryFromColumnParser(arguments[0]); + auto first_container = createContainer(first_parser); + + auto second_parser = makeCartesianGeometryFromColumnParser(arguments[1]); + auto second_container = createContainer(second_parser); + + auto res_column = ColumnFloat64::create(); + + for (size_t i = 0; i < input_rows_count; i++) + { + Float64 distance = boost::geometry::distance( + boost::get(first_container), + boost::get(second_container)); + + res_column->insertValue(distance); + } + + return res_column; + } +}; + + +class FunctionPolygonsDistanceGeographic : public FunctionPolygonsDistanceBase +{ +public: + static inline const char * name = "polygonsDistanceGeographic"; + + ColumnPtr executeImplementation(const ColumnsWithTypeAndName & arguments, const DataTypePtr & /*result_type*/, size_t input_rows_count) const + { + auto first_parser = makeGeographicGeometryFromColumnParser(arguments[0]); + auto first_container = createContainer(first_parser); + + auto second_parser = makeGeographicGeometryFromColumnParser(arguments[1]); + auto second_container = createContainer(second_parser); + + auto res_column = ColumnFloat64::create(); + + for (size_t i = 0; i < input_rows_count; i++) + { + Float64 distance = boost::geometry::distance( + boost::get(first_container), + boost::get(second_container)); + + res_column->insertValue(distance); + } + + return res_column; + } +}; + + +void registerFunctionPolygonsDistanceCartesian(FunctionFactory & factory) +{ + factory.registerFunction(); +} + +void registerFunctionPolygonsDistanceGeographic(FunctionFactory & factory) +{ + factory.registerFunction(); } } diff --git a/src/Functions/polygonsEquals.cpp b/src/Functions/polygonsEquals.cpp index b42787adf18..a23768626ad 100644 --- a/src/Functions/polygonsEquals.cpp +++ b/src/Functions/polygonsEquals.cpp @@ -30,14 +30,6 @@ namespace ErrorCodes extern const int ILLEGAL_COLUMN; } - -using CoordinateType = Float64; -using Point = boost::geometry::model::d2::point_xy; -using Polygon = boost::geometry::model::polygon; -using MultiPolygon = boost::geometry::model::multi_polygon; -using Box = boost::geometry::model::box; - - class FunctionPolygonsEquals : public IFunction { public: @@ -70,17 +62,17 @@ public: return std::make_shared(); } - void executeImpl(Block & block, const ColumnNumbers & arguments, size_t result, size_t input_rows_count) override + ColumnPtr executeImpl(const ColumnsWithTypeAndName & arguments, const DataTypePtr & /*result_type*/, size_t input_rows_count) const override { - auto get_parser = [&block, &arguments] (size_t i) { + auto get_parser = [&arguments] (size_t i) { const auto * const_col = - checkAndGetColumn(block.getByPosition(arguments[i]).column.get()); + checkAndGetColumn(arguments[i].column.get()); bool is_const = static_cast(const_col); - return std::pair{is_const, is_const ? - makeGeometryFromColumnParser(ColumnWithTypeAndName(const_col->getDataColumnPtr(), block.getByPosition(arguments[i]).type, block.getByPosition(arguments[i]).name)) : - makeGeometryFromColumnParser(block.getByPosition(arguments[i]))}; + return std::pair{is_const, is_const ? + makeCartesianGeometryFromColumnParser(ColumnWithTypeAndName(const_col->getDataColumnPtr(), arguments[i].type, arguments[i].name)) : + makeCartesianGeometryFromColumnParser(arguments[i])}; }; auto [is_first_polygon_const, first_parser] = get_parser(0); @@ -99,13 +91,13 @@ public: get(second_parser, second_container, i); bool equals = boost::geometry::equals( - boost::get(first_container), - boost::get(second_container)); + boost::get(first_container), + boost::get(second_container)); res_column->insertValue(equals); } - block.getByPosition(result).column = std::move(res_column); + return res_column; } bool useDefaultImplementationForConstants() const override diff --git a/src/Functions/polygonsIntersection.cpp b/src/Functions/polygonsIntersection.cpp index 0eecc5811fc..315144f3f13 100644 --- a/src/Functions/polygonsIntersection.cpp +++ b/src/Functions/polygonsIntersection.cpp @@ -29,13 +29,6 @@ namespace ErrorCodes } -using CoordinateType = Float64; -using Point = boost::geometry::model::d2::point_xy; -using Polygon = boost::geometry::model::polygon; -using MultiPolygon = boost::geometry::model::multi_polygon; -using Box = boost::geometry::model::box; - - class FunctionPolygonsIntersection : public IFunction { public: @@ -68,17 +61,17 @@ public: return DataTypeCustomMultiPolygonSerialization::nestedDataType(); } - void executeImpl(Block & block, const ColumnNumbers & arguments, size_t result, size_t input_rows_count) override + ColumnPtr executeImpl(const ColumnsWithTypeAndName & arguments, const DataTypePtr & /*result_type*/, size_t input_rows_count) const override { - auto get_parser = [&block, &arguments] (size_t i) { + auto get_parser = [&arguments] (size_t i) { const auto * const_col = - checkAndGetColumn(block.getByPosition(arguments[i]).column.get()); + checkAndGetColumn(arguments[i].column.get()); bool is_const = static_cast(const_col); - return std::pair{is_const, is_const ? - makeGeometryFromColumnParser(ColumnWithTypeAndName(const_col->getDataColumnPtr(), block.getByPosition(arguments[i]).type, block.getByPosition(arguments[i]).name)) : - makeGeometryFromColumnParser(block.getByPosition(arguments[i]))}; + return std::pair{is_const, is_const ? + makeCartesianGeometryFromColumnParser(ColumnWithTypeAndName(const_col->getDataColumnPtr(), arguments[i].type, arguments[i].name)) : + makeCartesianGeometryFromColumnParser(arguments[i])}; }; auto [is_first_polygon_const, first_parser] = get_parser(0); @@ -87,7 +80,7 @@ public: auto [is_second_polygon_const, second_parser] = get_parser(1); auto second_container = createContainer(second_parser); - Float64MultiPolygonSerializer serializer; + CartesianMultiPolygonSerializer serializer; for (size_t i = 0; i < input_rows_count; i++) { @@ -96,19 +89,19 @@ public: if (!is_second_polygon_const || i == 0) get(second_parser, second_container, i); - Float64Geometry intersection = Float64MultiPolygon({{{{}}}}); + CartesianGeometry intersection = CartesianMultiPolygon({{{{}}}}); boost::geometry::intersection( - boost::get(first_container), - boost::get(second_container), - boost::get(intersection)); + boost::get(first_container), + boost::get(second_container), + boost::get(intersection)); - boost::get(intersection).erase( - boost::get(intersection).begin()); + boost::get(intersection).erase( + boost::get(intersection).begin()); serializer.add(intersection); } - block.getByPosition(result).column = std::move(serializer.finalize()); + return serializer.finalize(); } bool useDefaultImplementationForConstants() const override diff --git a/src/Functions/polygonsSymDifference.cpp b/src/Functions/polygonsSymDifference.cpp index 3e7b24efc2d..0c0a9017152 100644 --- a/src/Functions/polygonsSymDifference.cpp +++ b/src/Functions/polygonsSymDifference.cpp @@ -28,14 +28,6 @@ namespace ErrorCodes extern const int ILLEGAL_COLUMN; } - -using CoordinateType = Float64; -using Point = boost::geometry::model::d2::point_xy; -using Polygon = boost::geometry::model::polygon; -using MultiPolygon = boost::geometry::model::multi_polygon; -using Box = boost::geometry::model::box; - - class FunctionPolygonsSymDifference : public IFunction { public: @@ -68,17 +60,17 @@ public: return DataTypeCustomMultiPolygonSerialization::nestedDataType(); } - void executeImpl(Block & block, const ColumnNumbers & arguments, size_t result, size_t input_rows_count) override + ColumnPtr executeImpl(const ColumnsWithTypeAndName & arguments, const DataTypePtr & /*result_type*/, size_t input_rows_count) const override { - auto get_parser = [&block, &arguments] (size_t i) { + auto get_parser = [&arguments] (size_t i) { const auto * const_col = - checkAndGetColumn(block.getByPosition(arguments[i]).column.get()); + checkAndGetColumn(arguments[i].column.get()); bool is_const = static_cast(const_col); - return std::pair{is_const, is_const ? - makeGeometryFromColumnParser(ColumnWithTypeAndName(const_col->getDataColumnPtr(), block.getByPosition(arguments[i]).type, block.getByPosition(arguments[i]).name)) : - makeGeometryFromColumnParser(block.getByPosition(arguments[i]))}; + return std::pair{is_const, is_const ? + makeCartesianGeometryFromColumnParser(ColumnWithTypeAndName(const_col->getDataColumnPtr(), arguments[i].type, arguments[i].name)) : + makeCartesianGeometryFromColumnParser(arguments[i])}; }; auto [is_first_polygon_const, first_parser] = get_parser(0); @@ -87,7 +79,7 @@ public: auto [is_second_polygon_const, second_parser] = get_parser(1); auto second_container = createContainer(second_parser); - Float64MultiPolygonSerializer serializer; + CartesianMultiPolygonSerializer serializer; for (size_t i = 0; i < input_rows_count; i++) { @@ -96,19 +88,19 @@ public: if (!is_second_polygon_const || i == 0) get(second_parser, second_container, i); - Float64Geometry sym_difference = Float64MultiPolygon({{{{}}}}); + CartesianGeometry sym_difference = CartesianMultiPolygon({{{{}}}}); boost::geometry::sym_difference( - boost::get(first_container), - boost::get(second_container), - boost::get(sym_difference)); + boost::get(first_container), + boost::get(second_container), + boost::get(sym_difference)); - boost::get(sym_difference).erase( - boost::get(sym_difference).begin()); + boost::get(sym_difference).erase( + boost::get(sym_difference).begin()); serializer.add(sym_difference); } - block.getByPosition(result).column = std::move(serializer.finalize()); + return serializer.finalize(); } bool useDefaultImplementationForConstants() const override diff --git a/src/Functions/polygonsWithin.cpp b/src/Functions/polygonsWithin.cpp index 8ad4c9bcbb1..03226ba551c 100644 --- a/src/Functions/polygonsWithin.cpp +++ b/src/Functions/polygonsWithin.cpp @@ -30,14 +30,6 @@ namespace ErrorCodes extern const int ILLEGAL_COLUMN; } - -using CoordinateType = Float64; -using Point = boost::geometry::model::d2::point_xy; -using Polygon = boost::geometry::model::polygon; -using MultiPolygon = boost::geometry::model::multi_polygon; -using Box = boost::geometry::model::box; - - class FunctionPolygonsWithin : public IFunction { public: @@ -70,17 +62,17 @@ public: return std::make_shared(); } - void executeImpl(Block & block, const ColumnNumbers & arguments, size_t result, size_t input_rows_count) override + ColumnPtr executeImpl(const ColumnsWithTypeAndName & arguments, const DataTypePtr & /*result_type*/, size_t input_rows_count) const override { - auto get_parser = [&block, &arguments] (size_t i) { + auto get_parser = [&arguments] (size_t i) { const auto * const_col = - checkAndGetColumn(block.getByPosition(arguments[i]).column.get()); + checkAndGetColumn(arguments[i].column.get()); bool is_const = static_cast(const_col); - return std::pair{is_const, is_const ? - makeGeometryFromColumnParser(ColumnWithTypeAndName(const_col->getDataColumnPtr(), block.getByPosition(arguments[i]).type, block.getByPosition(arguments[i]).name)) : - makeGeometryFromColumnParser(block.getByPosition(arguments[i]))}; + return std::pair{is_const, is_const ? + makeCartesianGeometryFromColumnParser(ColumnWithTypeAndName(const_col->getDataColumnPtr(), arguments[i].type, arguments[i].name)) : + makeCartesianGeometryFromColumnParser(arguments[i])}; }; auto [is_first_polygon_const, first_parser] = get_parser(0); @@ -99,13 +91,13 @@ public: get(second_parser, second_container, i); bool within = boost::geometry::within( - boost::get(first_container), - boost::get(second_container)); + boost::get(first_container), + boost::get(second_container)); res_column->insertValue(within); } - block.getByPosition(result).column = std::move(res_column); + return res_column; } bool useDefaultImplementationForConstants() const override diff --git a/src/Functions/readWkt.cpp b/src/Functions/readWkt.cpp index 125c39998c4..2d64db76466 100644 --- a/src/Functions/readWkt.cpp +++ b/src/Functions/readWkt.cpp @@ -39,9 +39,9 @@ public: return DataType::nestedDataType(); } - void executeImpl(Block & block, const ColumnNumbers & arguments, size_t result, size_t input_rows_count) override + ColumnPtr executeImpl(const ColumnsWithTypeAndName & arguments, const DataTypePtr & /*result_type*/, size_t input_rows_count) const override { - auto column_string = checkAndGetColumn(block.getByPosition(arguments[0]).column.get()); + auto column_string = checkAndGetColumn(arguments[0].column.get()); Serializer serializer; Geometry geometry; @@ -53,7 +53,7 @@ public: serializer.add(geometry); } - block.getByPosition(result).column = serializer.finalize(); + return serializer.finalize(); } bool useDefaultImplementationForConstants() const override @@ -62,7 +62,7 @@ public: } }; -class FunctionReadWktPoint : public FunctionReadWkt +class FunctionReadWktPoint : public FunctionReadWkt { public: static inline const char * name = "readWktPoint"; @@ -76,7 +76,7 @@ public: } }; -class FunctionReadWktPolygon : public FunctionReadWkt +class FunctionReadWktPolygon : public FunctionReadWkt { public: static inline const char * name = "readWktPolygon"; @@ -90,7 +90,7 @@ public: } }; -class FunctionReadWktMultiPolygon : public FunctionReadWkt +class FunctionReadWktMultiPolygon : public FunctionReadWkt { public: static inline const char * name = "readWktMultiPolygon"; diff --git a/src/Functions/registerFunctionsGeo.cpp b/src/Functions/registerFunctionsGeo.cpp index 605dd4dcba0..fe7ea412841 100644 --- a/src/Functions/registerFunctionsGeo.cpp +++ b/src/Functions/registerFunctionsGeo.cpp @@ -12,13 +12,16 @@ void registerFunctionPointInEllipses(FunctionFactory & factory); void registerFunctionPointInPolygon(FunctionFactory & factory); void registerFunctionPolygonsIntersection(FunctionFactory & factory); void registerFunctionPolygonsUnion(FunctionFactory & factory); -void registerFunctionPolygonArea(FunctionFactory & factory); +void registerFunctionPolygonAreaCartesian(FunctionFactory & factory); +void registerFunctionPolygonAreaGeographic (FunctionFactory & factory); void registerFunctionPolygonConvexHull(FunctionFactory & factory); void registerFunctionPolygonsSymDifference(FunctionFactory & factory); void registerFunctionPolygonsEquals(FunctionFactory & factory); -void registerFunctionPolygonsDistance(FunctionFactory & factory); +void registerFunctionPolygonsDistanceCartesian(FunctionFactory & factory); +void registerFunctionPolygonsDistanceGeographic(FunctionFactory & factory); void registerFunctionPolygonsWithin(FunctionFactory & factory); -void registerFunctionPolygonPerimeter(FunctionFactory & factory); +void registerFunctionPolygonPerimeterCartesian(FunctionFactory & factory); +void registerFunctionPolygonPerimeterGeographic(FunctionFactory & factory); void registerFunctionGeohashEncode(FunctionFactory & factory); void registerFunctionGeohashDecode(FunctionFactory & factory); void registerFunctionGeohashesInBox(FunctionFactory & factory); @@ -50,13 +53,16 @@ void registerFunctionsGeo(FunctionFactory & factory) registerFunctionPointInPolygon(factory); registerFunctionPolygonsIntersection(factory); registerFunctionPolygonsUnion(factory); - registerFunctionPolygonArea(factory); + registerFunctionPolygonAreaCartesian(factory); + registerFunctionPolygonAreaGeographic(factory); registerFunctionPolygonConvexHull(factory); registerFunctionPolygonsSymDifference(factory); registerFunctionPolygonsEquals(factory); - registerFunctionPolygonsDistance(factory); + registerFunctionPolygonsDistanceCartesian(factory); + registerFunctionPolygonsDistanceGeographic(factory); registerFunctionPolygonsWithin(factory); - registerFunctionPolygonPerimeter(factory); + registerFunctionPolygonPerimeterCartesian(factory); + registerFunctionPolygonPerimeterGeographic(factory); registerFunctionGeohashEncode(factory); registerFunctionGeohashDecode(factory); registerFunctionGeohashesInBox(factory); diff --git a/src/Functions/svg.cpp b/src/Functions/svg.cpp index d2f0ba1de4e..8b22acbbb48 100644 --- a/src/Functions/svg.cpp +++ b/src/Functions/svg.cpp @@ -60,13 +60,13 @@ public: return std::make_shared(); } - void executeImpl(Block & block, const ColumnNumbers & arguments, size_t result, size_t input_rows_count) override + ColumnPtr executeImpl(const ColumnsWithTypeAndName & arguments, const DataTypePtr & /*result_type*/, size_t input_rows_count) const override { - const auto * const_col = checkAndGetColumn(block.getByPosition(arguments[0]).column.get()); + const auto * const_col = checkAndGetColumn(arguments[0].column.get()); auto parser = const_col ? - makeGeometryFromColumnParser(ColumnWithTypeAndName(const_col->getDataColumnPtr(), block.getByPosition(arguments[0]).type, block.getByPosition(arguments[0]).name)) : - makeGeometryFromColumnParser(block.getByPosition(arguments[0])); + makeCartesianGeometryFromColumnParser(ColumnWithTypeAndName(const_col->getDataColumnPtr(), arguments[0].type, arguments[0].name)) : + makeCartesianGeometryFromColumnParser(arguments[0]); bool geo_column_is_const = static_cast(const_col); @@ -76,7 +76,7 @@ public: bool has_style = arguments.size() > 1; ColumnPtr style; if (has_style) { - style = block.getByPosition(arguments[1]).column; + style = arguments[1].column; } for (size_t i = 0; i < input_rows_count; i++) @@ -90,7 +90,7 @@ public: res_column->insertData(serialized.c_str(), serialized.size()); } - block.getByPosition(result).column = std::move(res_column); + return res_column; } bool useDefaultImplementationForConstants() const override diff --git a/src/Functions/wkt.cpp b/src/Functions/wkt.cpp index 4fb214dc18c..9e01e312503 100644 --- a/src/Functions/wkt.cpp +++ b/src/Functions/wkt.cpp @@ -36,9 +36,9 @@ public: return std::make_shared(); } - void executeImpl(Block & block, const ColumnNumbers & arguments, size_t result, size_t input_rows_count) override + ColumnPtr executeImpl(const ColumnsWithTypeAndName & arguments, const DataTypePtr & /*result_type*/, size_t input_rows_count) const override { - auto parser = makeGeometryFromColumnParser(block.getByPosition(arguments[0])); + auto parser = makeCartesianGeometryFromColumnParser(arguments[0]); auto res_column = ColumnString::create(); auto container = createContainer(parser); @@ -52,7 +52,7 @@ public: res_column->insertData(serialized.c_str(), serialized.size()); } - block.getByPosition(result).column = std::move(res_column); + return res_column; } bool useDefaultImplementationForConstants() const override diff --git a/tests/queries/0_stateless/01300_polygon_convex_hull.reference b/tests/queries/0_stateless/01300_polygon_convex_hull.reference index 89dc8315153..47be3068b5c 100644 --- a/tests/queries/0_stateless/01300_polygon_convex_hull.reference +++ b/tests/queries/0_stateless/01300_polygon_convex_hull.reference @@ -1 +1 @@ -[[(0,0),(0,5),(5,5),(5,0),(0,0)]] \ No newline at end of file +[[(0,0),(0,5),(5,5),(5,0),(0,0)]] diff --git a/tests/queries/0_stateless/01302_polygons_distance.reference b/tests/queries/0_stateless/01302_polygons_distance.reference index aba29d80f56..8d8b128e512 100644 --- a/tests/queries/0_stateless/01302_polygons_distance.reference +++ b/tests/queries/0_stateless/01302_polygons_distance.reference @@ -1,2 +1,4 @@ 0 1.2727922061357855 +2088389.0786590837 +2088389.0786590837 diff --git a/tests/queries/0_stateless/01302_polygons_distance.sql b/tests/queries/0_stateless/01302_polygons_distance.sql index a4d1fbb3f4c..f258f3c3a8b 100644 --- a/tests/queries/0_stateless/01302_polygons_distance.sql +++ b/tests/queries/0_stateless/01302_polygons_distance.sql @@ -1,3 +1,7 @@ -select polygonsDistance([[[(0, 0),(0, 3),(1, 2.9),(2, 2.6),(2.6, 2),(2.9, 1),(3, 0),(0, 0)]]], [[[(1, 1),(1, 4),(4, 4),(4, 1),(1, 1)]]]); - -select polygonsDistance([[[(0, 0), (0, 0.1), (0.1, 0.1), (0.1, 0)]]], [[[(1, 1),(1, 4),(4, 4),(4, 1),(1, 1)]]]) +select polygonsDistanceCartesian([[[(0, 0),(0, 3),(1, 2.9),(2, 2.6),(2.6, 2),(2.9, 1),(3, 0),(0, 0)]]], [[[(1, 1),(1, 4),(4, 4),(4, 1),(1, 1)]]]); +select polygonsDistanceCartesian([[[(0, 0), (0, 0.1), (0.1, 0.1), (0.1, 0)]]], [[[(1, 1),(1, 4),(4, 4),(4, 1),(1, 1)]]]); +select polygonsDistanceGeographic([[[(23.725750, 37.971536)]]], [[[(4.3826169, 50.8119483)]]]); +drop table if exists polygon_01302 +create table polygon_01302 (x Array(Array(Array(Tuple(Float64, Float64)))), y Array(Array(Array(Tuple(Float64, Float64))))) engine=Memory(); +insert into polygon_01302 values ([[[(23.725750, 37.971536)]]], [[[(4.3826169, 50.8119483)]]]); +select polygonsDistanceGeographic(x, y) from polygon; diff --git a/tests/queries/0_stateless/01307_polygon_perimeter.sql b/tests/queries/0_stateless/01307_polygon_perimeter.sql index ae178f4d38d..7e12d032d83 100644 --- a/tests/queries/0_stateless/01307_polygon_perimeter.sql +++ b/tests/queries/0_stateless/01307_polygon_perimeter.sql @@ -1 +1 @@ -select polygonPerimeter([[[(0, 0), (0., 5), (5, 5), (5., 0)]]]) \ No newline at end of file +select polygonPerimeterCartesian([[[(0, 0), (0., 5), (5, 5), (5., 0)]]]); \ No newline at end of file diff --git a/tests/queries/0_stateless/01308_polygon_area.reference b/tests/queries/0_stateless/01308_polygon_area.reference index 7273c0fa8c5..41b497f8103 100644 --- a/tests/queries/0_stateless/01308_polygon_area.reference +++ b/tests/queries/0_stateless/01308_polygon_area.reference @@ -1 +1,2 @@ 25 +3848183.73456666 diff --git a/tests/queries/0_stateless/01308_polygon_area.sql b/tests/queries/0_stateless/01308_polygon_area.sql index b3b12047fd0..c4fba155f37 100644 --- a/tests/queries/0_stateless/01308_polygon_area.sql +++ b/tests/queries/0_stateless/01308_polygon_area.sql @@ -1 +1,2 @@ -select polygonArea([[[(0, 0), (0., 5), (5, 5), (5., 0)]]]) +select polygonAreaCartesian([[[(0, 0), (0., 5), (5, 5), (5., 0)]]]); +select polygonAreaGeographic([[[(4.346693, 50.858306), (4.367945, 50.852455), (4.366227, 50.840809), (4.344961, 50.833264), (4.338074, 50.848677), (4.346693, 50.858306)]]]); \ No newline at end of file From db9dbfa528d07feebbf394a15239f10095a2ad3e Mon Sep 17 00:00:00 2001 From: Nikita Mikhaylov Date: Tue, 19 Jan 2021 04:42:26 +0400 Subject: [PATCH 274/544] Update 01304_polygons_sym_difference.sql --- tests/queries/0_stateless/01304_polygons_sym_difference.sql | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/queries/0_stateless/01304_polygons_sym_difference.sql b/tests/queries/0_stateless/01304_polygons_sym_difference.sql index 4008d1dc7d6..e351add03e6 100644 --- a/tests/queries/0_stateless/01304_polygons_sym_difference.sql +++ b/tests/queries/0_stateless/01304_polygons_sym_difference.sql @@ -1 +1,2 @@ -select polygonsSymDifference([[[(0, 0),(0, 3),(1, 2.9),(2, 2.6),(2.6, 2),(2.9, 1),(3, 0),(0, 0)]]], [[[(1, 1),(1, 4),(4, 4),(4, 1),(1, 1)]]]) \ No newline at end of file +select polygonsSymDifference([[[(0, 0),(0, 3),(1, 2.9),(2, 2.6),(2.6, 2),(2.9, 1),(3, 0),(0, 0)]]], [[[(1, 1),(1, 4),(4, 4),(4, 1),(1, 1)]]]) + From 3e36fae3df1776e652bf01a9c83aa804224a80f3 Mon Sep 17 00:00:00 2001 From: Nikita Mikhailov Date: Tue, 19 Jan 2021 15:02:56 +0300 Subject: [PATCH 275/544] fix test --- src/Functions/geometryConverters.h | 2 +- src/Functions/polygonsIntersection.cpp | 19 ++++--------------- src/Functions/svg.cpp | 4 ++-- 3 files changed, 7 insertions(+), 18 deletions(-) diff --git a/src/Functions/geometryConverters.h b/src/Functions/geometryConverters.h index ee22ce64ddf..76af5a64ac6 100644 --- a/src/Functions/geometryConverters.h +++ b/src/Functions/geometryConverters.h @@ -281,7 +281,7 @@ public: void operator()(const CartesianPoint & point) { x->insertValue(point.get<0>()); - y->insertValue(point.get<0>()); + y->insertValue(point.get<1>()); } void operator()(const CartesianRing & ring) diff --git a/src/Functions/polygonsIntersection.cpp b/src/Functions/polygonsIntersection.cpp index 315144f3f13..37448b4acf0 100644 --- a/src/Functions/polygonsIntersection.cpp +++ b/src/Functions/polygonsIntersection.cpp @@ -63,30 +63,19 @@ public: ColumnPtr executeImpl(const ColumnsWithTypeAndName & arguments, const DataTypePtr & /*result_type*/, size_t input_rows_count) const override { - auto get_parser = [&arguments] (size_t i) { - const auto * const_col = - checkAndGetColumn(arguments[i].column.get()); - - bool is_const = static_cast(const_col); - - return std::pair{is_const, is_const ? - makeCartesianGeometryFromColumnParser(ColumnWithTypeAndName(const_col->getDataColumnPtr(), arguments[i].type, arguments[i].name)) : - makeCartesianGeometryFromColumnParser(arguments[i])}; - }; - - auto [is_first_polygon_const, first_parser] = get_parser(0); + auto first_parser = makeCartesianGeometryFromColumnParser(arguments[0]); auto first_container = createContainer(first_parser); - auto [is_second_polygon_const, second_parser] = get_parser(1); + auto second_parser = makeCartesianGeometryFromColumnParser(arguments[1]); auto second_container = createContainer(second_parser); CartesianMultiPolygonSerializer serializer; for (size_t i = 0; i < input_rows_count; i++) { - if (!is_first_polygon_const || i == 0) + if (i == 0) get(first_parser, first_container, i); - if (!is_second_polygon_const || i == 0) + if (i == 0) get(second_parser, second_container, i); CartesianGeometry intersection = CartesianMultiPolygon({{{{}}}}); diff --git a/src/Functions/svg.cpp b/src/Functions/svg.cpp index 8b22acbbb48..6a0827a107c 100644 --- a/src/Functions/svg.cpp +++ b/src/Functions/svg.cpp @@ -48,10 +48,10 @@ public: { throw Exception("Too many arguments", ErrorCodes::TOO_MANY_ARGUMENTS_FOR_FUNCTION); } - if (arguments.size() == 0) { + else if (arguments.empty()) { throw Exception("Too few arguments", ErrorCodes::TOO_FEW_ARGUMENTS_FOR_FUNCTION); } - if (arguments.size() == 2 && checkAndGetDataType(arguments[1].get()) == nullptr) + else if (arguments.size() == 2 && checkAndGetDataType(arguments[1].get()) == nullptr) { throw Exception("Second argument should be String", ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT); From cfee417ce0b82d763abf84844f8384205871d68a Mon Sep 17 00:00:00 2001 From: Nikita Mikhailov Date: Tue, 19 Jan 2021 17:52:53 +0300 Subject: [PATCH 276/544] better --- src/Functions/PolygonsUnion.cpp | 25 +- src/Functions/geometryConverters.cpp | 69 +++-- src/Functions/geometryConverters.h | 235 +++++++++--------- src/Functions/polygonArea.cpp | 97 +++----- src/Functions/polygonConvexHull.cpp | 32 +-- src/Functions/polygonPerimeter.cpp | 100 +++----- src/Functions/polygonsDistance.cpp | 107 +++----- src/Functions/polygonsEquals.cpp | 6 +- src/Functions/polygonsIntersection.cpp | 6 +- src/Functions/polygonsSymDifference.cpp | 48 ++-- src/Functions/polygonsWithin.cpp | 6 +- src/Functions/readWkt.cpp | 6 +- src/Functions/registerFunctionsGeo.cpp | 18 +- src/Functions/svg.cpp | 4 +- src/Functions/wkt.cpp | 2 +- .../0_stateless/01302_polygons_distance.sql | 5 +- .../01306_polygons_intersection.reference | 1 + .../01306_polygons_intersection.sql | 3 +- 18 files changed, 325 insertions(+), 445 deletions(-) diff --git a/src/Functions/PolygonsUnion.cpp b/src/Functions/PolygonsUnion.cpp index 37a9b684805..2d26e09c3de 100644 --- a/src/Functions/PolygonsUnion.cpp +++ b/src/Functions/PolygonsUnion.cpp @@ -62,33 +62,20 @@ public: ColumnPtr executeImpl(const ColumnsWithTypeAndName & arguments, const DataTypePtr & /*result_type*/, size_t input_rows_count) const override { - auto get_parser = [&arguments] (size_t i) { - const auto * const_col = - checkAndGetColumn(arguments[i].column.get()); - - bool is_const = static_cast(const_col); - - return std::pair{is_const, is_const ? - makeCartesianGeometryFromColumnParser(ColumnWithTypeAndName(const_col->getDataColumnPtr(), arguments[i].type, arguments[i].name)) : - makeCartesianGeometryFromColumnParser(arguments[i])}; - }; - - auto [is_first_polygon_const, first_parser] = get_parser(0); + auto first_parser = makeGeometryFromColumnParser(arguments[0]); auto first_container = createContainer(first_parser); - auto [is_second_polygon_const, second_parser] = get_parser(1); + auto second_parser = makeGeometryFromColumnParser(arguments[1]); auto second_container = createContainer(second_parser); - CartesianMultiPolygonSerializer serializer; + MultiPolygonSerializer serializer; for (size_t i = 0; i < input_rows_count; i++) { - if (!is_first_polygon_const || i == 0) - get(first_parser, first_container, i); - if (!is_second_polygon_const || i == 0) - get(second_parser, second_container, i); + get(first_parser, first_container, i); + get(second_parser, second_container, i); - CartesianGeometry polygons_union = CartesianMultiPolygon({{{{}}}}); + Geometry polygons_union = CartesianMultiPolygon({{{{}}}}); boost::geometry::union_( boost::get(first_container), boost::get(second_container), diff --git a/src/Functions/geometryConverters.cpp b/src/Functions/geometryConverters.cpp index da2ddd9a473..4fdc1ea5931 100644 --- a/src/Functions/geometryConverters.cpp +++ b/src/Functions/geometryConverters.cpp @@ -30,11 +30,11 @@ public: } }; -template +template class Getter : public boost::static_visitor { public: - constexpr Getter(Geometry & container_, size_t i_) + constexpr Getter(Geometry & container_, size_t i_) : container(container_) , i(i_) {} @@ -46,7 +46,7 @@ public: } private: - Geometry & container; + Geometry & container; size_t i; }; @@ -64,52 +64,41 @@ Parser makeParser(const ColumnWithTypeAndName & col) } -CartesianGeometryFromColumnParser makeCartesianGeometryFromColumnParser(const ColumnWithTypeAndName & col) +template +Geometry createContainer(const GeometryFromColumnParser & parser) +{ + static ContainerCreator> creator; + return boost::apply_visitor(creator, parser); +} + +template +void get(const GeometryFromColumnParser & parser, Geometry & container, size_t i) +{ + boost::apply_visitor(Getter(container, i), parser); +} + +template +GeometryFromColumnParser makeGeometryFromColumnParser(const ColumnWithTypeAndName & col) { switch (getArrayDepth(col.type, 3)) { - case 0: return makeParser>(col); - case 1: return makeParser(col); - case 2: return makeParser(col); - case 3: return makeParser(col); + case 0: return makeParser>(col); + case 1: return makeParser>(col); + case 2: return makeParser>(col); + case 3: return makeParser>(col); default: throw Exception("Cannot parse geometry from column with type " + col.type->getName() + ", array depth is too big", ErrorCodes::ILLEGAL_COLUMN); } } -CartesianGeometry createContainer(const CartesianGeometryFromColumnParser & parser) -{ - static ContainerCreator creator; - return boost::apply_visitor(creator, parser); -} +/// Explicit instantiations to avoid linker errors. -void get(const CartesianGeometryFromColumnParser & parser, CartesianGeometry & container, size_t i) -{ - boost::apply_visitor(Getter(container, i), parser); -} +template Geometry createContainer(const GeometryFromColumnParser &); +template Geometry createContainer(const GeometryFromColumnParser &); +template void get(const GeometryFromColumnParser & parser, Geometry & container, size_t i); +template void get(const GeometryFromColumnParser & parser, Geometry & container, size_t i); +template GeometryFromColumnParser makeGeometryFromColumnParser(const ColumnWithTypeAndName & col); +template GeometryFromColumnParser makeGeometryFromColumnParser(const ColumnWithTypeAndName & col); -GeographicGeometryFromColumnParser makeGeographicGeometryFromColumnParser(const ColumnWithTypeAndName & col) -{ - switch (getArrayDepth(col.type, 3)) - { - case 0: return makeParser>(col); - case 1: return makeParser(col); - case 2: return makeParser(col); - case 3: return makeParser(col); - default: throw Exception("Cannot parse geometry from column with type " + col.type->getName() - + ", array depth is too big", ErrorCodes::ILLEGAL_COLUMN); - } -} - -GeographicGeometry createContainer(const GeographicGeometryFromColumnParser & parser) -{ - static ContainerCreator creator; - return boost::apply_visitor(creator, parser); -} - -void get(const GeographicGeometryFromColumnParser & parser, GeographicGeometry & container, size_t i) -{ - boost::apply_visitor(Getter(container, i), parser); -} } diff --git a/src/Functions/geometryConverters.h b/src/Functions/geometryConverters.h index 76af5a64ac6..134209c3b40 100644 --- a/src/Functions/geometryConverters.h +++ b/src/Functions/geometryConverters.h @@ -27,17 +27,31 @@ namespace ErrorCodes } namespace bg = boost::geometry; + +template +using Ring = bg::model::ring; + +template +using Polygon = bg::model::polygon; + +template +using MultiPolygon = bg::model::multi_polygon>; + +template +using Geometry = boost::variant, Polygon, MultiPolygon>; + + using CartesianPoint = bg::model::d2::point_xy; -using CartesianRing = bg::model::ring; -using CartesianPolygon = bg::model::polygon; -using CartesianMultiPolygon = bg::model::multi_polygon; -using CartesianGeometry = boost::variant; +using CartesianRing = Ring; +using CartesianPolygon = Polygon; +using CartesianMultiPolygon = MultiPolygon; +using CartesianGeometry = Geometry; using GeographicPoint = bg::model::point>; -using GeographicRing = bg::model::ring; -using GeographicPolygon = bg::model::polygon; -using GeographicMultiPolygon = bg::model::multi_polygon; -using GeographicGeometry = boost::variant; +using GeographicRing = Ring; +using GeographicPolygon = Polygon; +using GeographicMultiPolygon = MultiPolygon; +using GeographicGeometry = Geometry; /** * Class which takes some boost type and returns a pair of numbers. @@ -111,27 +125,27 @@ private: const Float64 * second; }; -template +template class RingFromColumnParser { public: RingFromColumnParser(ColumnPtr col_) : offsets(static_cast(*col_).getOffsets()) - , pointParser(static_cast(*col_).getDataPtr()) + , point_parser(static_cast(*col_).getDataPtr()) { } - Geometry createContainer() const + Geometry createContainer() const { - return RingType(); + return Ring(); } - void get(Geometry & container, size_t i) const + void get(Geometry & container, size_t i) const { - get(boost::get(container), i); + get(boost::get>(container), i); } - void get(RingType & container, size_t i) const + void get(Ring & container, size_t i) const { size_t l = offsets[i - 1]; size_t r = offsets[i]; @@ -141,7 +155,7 @@ public: container.resize(r - l); for (size_t j = l; j < r; j++) { - pointParser.get(container[j - l], j); + point_parser.get(container[j - l], j); } // make ring closed @@ -153,138 +167,127 @@ public: private: const IColumn::Offsets & offsets; - const PointParser pointParser; + const PointFromColumnParser point_parser; }; -template +template class PolygonFromColumnParser { public: PolygonFromColumnParser(ColumnPtr col_) : offsets(static_cast(*col_).getOffsets()) - , ringParser(static_cast(*col_).getDataPtr()) + , ring_parser(static_cast(*col_).getDataPtr()) {} - Geometry createContainer() const + Geometry createContainer() const { - return PolygonType(); + return Polygon(); } - void get(Geometry & container, size_t i) const + void get(Geometry & container, size_t i) const { - get(boost::get(container), i); + get(boost::get>(container), i); } - void get(PolygonType & container, size_t i) const + void get(Polygon & container, size_t i) const { size_t l = offsets[i - 1]; size_t r = offsets[i]; - ringParser.get(container.outer(), l); + ring_parser.get(container.outer(), l); container.inners().resize(r - l - 1); for (size_t j = l + 1; j < r; j++) { - ringParser.get(container.inners()[j - l - 1], j); + ring_parser.get(container.inners()[j - l - 1], j); } } private: const IColumn::Offsets & offsets; - const RingParser ringParser; + const RingFromColumnParser ring_parser; }; -template +template class MultiPolygonFromColumnParser { public: MultiPolygonFromColumnParser(ColumnPtr col_) : offsets(static_cast(*col_).getOffsets()) - , polygonParser(static_cast(*col_).getDataPtr()) + , polygon_parser(static_cast(*col_).getDataPtr()) {} - Geometry createContainer() const + Geometry createContainer() const { - return MultiPolygonType(); + return MultiPolygon(); } - void get(Geometry & container, size_t i) const + void get(Geometry & container, size_t i) const { - MultiPolygonType & multi_polygon = boost::get(container); + auto & multi_polygon = boost::get>(container); size_t l = offsets[i - 1]; size_t r = offsets[i]; multi_polygon.resize(r - l); for (size_t j = l; j < r; j++) { - polygonParser.get(multi_polygon[j - l], j); + polygon_parser.get(multi_polygon[j - l], j); } } private: const IColumn::Offsets & offsets; - const PolygonParser polygonParser; + const PolygonFromColumnParser polygon_parser; }; -/// Cartesian coordinates - -using CartesianRingFromColumnParser = RingFromColumnParser>; -using CartesianPolygonFromColumnParser = PolygonFromColumnParser; -using CartesianMultiPolygonFromColumnParser = MultiPolygonFromColumnParser; - -using CartesianGeometryFromColumnParser = boost::variant< - PointFromColumnParser, - CartesianRingFromColumnParser, - CartesianPolygonFromColumnParser, - CartesianMultiPolygonFromColumnParser +template +using GeometryFromColumnParser = boost::variant< + PointFromColumnParser, + RingFromColumnParser, + PolygonFromColumnParser, + MultiPolygonFromColumnParser >; -CartesianGeometry createContainer(const CartesianGeometryFromColumnParser & parser); +template +Geometry createContainer(const GeometryFromColumnParser & parser); -void get(const CartesianGeometryFromColumnParser & parser, CartesianGeometry & container, size_t i); +extern template Geometry createContainer(const GeometryFromColumnParser & parser); +extern template Geometry createContainer(const GeometryFromColumnParser & parser); -CartesianGeometryFromColumnParser makeCartesianGeometryFromColumnParser(const ColumnWithTypeAndName & col); +template +void get(const GeometryFromColumnParser & parser, Geometry & container, size_t i); -/// Geographic coordinates +extern template void get(const GeometryFromColumnParser & parser, Geometry & container, size_t i); +extern template void get(const GeometryFromColumnParser & parser, Geometry & container, size_t i); -using GeographicRingFromColumnParser = RingFromColumnParser>; -using GeographicPolygonFromColumnParser = PolygonFromColumnParser; -using GeographicMultiPolygonFromColumnParser = MultiPolygonFromColumnParser; +template +GeometryFromColumnParser makeGeometryFromColumnParser(const ColumnWithTypeAndName & col); -using GeographicGeometryFromColumnParser = boost::variant< - PointFromColumnParser, - GeographicRingFromColumnParser, - GeographicPolygonFromColumnParser, - GeographicMultiPolygonFromColumnParser ->; +extern template GeometryFromColumnParser makeGeometryFromColumnParser(const ColumnWithTypeAndName & col); +extern template GeometryFromColumnParser makeGeometryFromColumnParser(const ColumnWithTypeAndName & col); -GeographicGeometry createContainer(const GeographicGeometryFromColumnParser & parser); - -void get(const GeographicGeometryFromColumnParser & parser, GeographicGeometry & container, size_t i); - -GeographicGeometryFromColumnParser makeGeographicGeometryFromColumnParser(const ColumnWithTypeAndName & col); - - -class CartesianPointSerializerVisitor : public boost::static_visitor +/// To serialize Geographic or Cartesian point (a pair of numbers in both cases). +template +class PointSerializerVisitor : public boost::static_visitor { public: - CartesianPointSerializerVisitor() - : x(ColumnFloat64::create()) - , y(ColumnFloat64::create()) + PointSerializerVisitor() + : first(ColumnFloat64::create()) + , second(ColumnFloat64::create()) {} - CartesianPointSerializerVisitor(size_t n) - : x(ColumnFloat64::create(n)) - , y(ColumnFloat64::create(n)) + PointSerializerVisitor(size_t n) + : first(ColumnFloat64::create(n)) + , second(ColumnFloat64::create(n)) {} - void operator()(const CartesianPoint & point) + void operator()(const Point & point) { - x->insertValue(point.get<0>()); - y->insertValue(point.get<1>()); + first->insertValue(point.template get<0>()); + second->insertValue(point.template get<1>()); } - void operator()(const CartesianRing & ring) + void operator()(const Ring & ring) { if (ring.size() != 1) { throw Exception("Unable to write ring of size " + toString(ring.size()) + " != 1 to point column", ErrorCodes::BAD_ARGUMENTS); @@ -292,7 +295,7 @@ public: (*this)(ring[0]); } - void operator()(const CartesianPolygon & polygon) + void operator()(const Polygon & polygon) { if (polygon.inners().size() != 0) { throw Exception("Unable to write polygon with holes to point column", ErrorCodes::BAD_ARGUMENTS); @@ -300,7 +303,7 @@ public: (*this)(polygon.outer()); } - void operator()(const CartesianMultiPolygon & multi_polygon) + void operator()(const MultiPolygon & multi_polygon) { if (multi_polygon.size() != 1) { throw Exception("Unable to write multi-polygon of size " + toString(multi_polygon.size()) + " != 1 to point column", ErrorCodes::BAD_ARGUMENTS); @@ -311,29 +314,30 @@ public: ColumnPtr finalize() { Columns columns(2); - columns[0] = std::move(x); - columns[1] = std::move(y); + columns[0] = std::move(first); + columns[1] = std::move(second); return ColumnTuple::create(columns); } private: - ColumnFloat64::MutablePtr x; - ColumnFloat64::MutablePtr y; + ColumnFloat64::MutablePtr first; + ColumnFloat64::MutablePtr second; }; -class CartesianRingSerializerVisitor : public boost::static_visitor +template +class RingSerializerVisitor : public boost::static_visitor { public: - CartesianRingSerializerVisitor() + RingSerializerVisitor() : offsets(ColumnUInt64::create()) {} - CartesianRingSerializerVisitor(size_t n) + RingSerializerVisitor(size_t n) : offsets(ColumnUInt64::create(n)) {} - void operator()(const CartesianPoint & point) + void operator()(const Point & point) { size++; offsets->insertValue(size); @@ -341,7 +345,7 @@ public: pointSerializer(point); } - void operator()(const CartesianRing & ring) + void operator()(const Ring & ring) { size += ring.size(); offsets->insertValue(size); @@ -351,7 +355,7 @@ public: } } - void operator()(const CartesianPolygon & polygon) + void operator()(const Polygon & polygon) { if (polygon.inners().size() != 0) { throw Exception("Unable to write polygon with holes to ring column", ErrorCodes::BAD_ARGUMENTS); @@ -359,7 +363,7 @@ public: (*this)(polygon.outer()); } - void operator()(const CartesianMultiPolygon & multi_polygon) + void operator()(const MultiPolygon & multi_polygon) { if (multi_polygon.size() != 1) { throw Exception("Unable to write multi-polygon of size " + toString(multi_polygon.size()) + " != 1 to ring column", ErrorCodes::BAD_ARGUMENTS); @@ -374,36 +378,37 @@ public: private: size_t size = 0; - CartesianPointSerializerVisitor pointSerializer; + PointSerializerVisitor pointSerializer; ColumnUInt64::MutablePtr offsets; }; -class CartesianPolygonSerializerVisitor : public boost::static_visitor +template +class PolygonSerializerVisitor : public boost::static_visitor { public: - CartesianPolygonSerializerVisitor() + PolygonSerializerVisitor() : offsets(ColumnUInt64::create()) {} - CartesianPolygonSerializerVisitor(size_t n) + PolygonSerializerVisitor(size_t n) : offsets(ColumnUInt64::create(n)) {} - void operator()(const CartesianPoint & point) + void operator()(const Point & point) { size++; offsets->insertValue(size); ringSerializer(point); } - void operator()(const CartesianRing & ring) + void operator()(const Ring & ring) { size++; offsets->insertValue(size); ringSerializer(ring); } - void operator()(const CartesianPolygon & polygon) + void operator()(const Polygon & polygon) { size += 1 + polygon.inners().size(); offsets->insertValue(size); @@ -414,7 +419,7 @@ public: } } - void operator()(const CartesianMultiPolygon & multi_polygon) + void operator()(const MultiPolygon & multi_polygon) { if (multi_polygon.size() != 1) { throw Exception("Unable to write multi-polygon of size " + toString(multi_polygon.size()) + " != 1 to polygon column", ErrorCodes::BAD_ARGUMENTS); @@ -429,43 +434,44 @@ public: private: size_t size = 0; - CartesianRingSerializerVisitor ringSerializer; + RingSerializerVisitor ringSerializer; ColumnUInt64::MutablePtr offsets; }; -class CartesianMultiPolygonSerializerVisitor : public boost::static_visitor +template +class MultiPolygonSerializerVisitor : public boost::static_visitor { public: - CartesianMultiPolygonSerializerVisitor() + MultiPolygonSerializerVisitor() : offsets(ColumnUInt64::create()) {} - CartesianMultiPolygonSerializerVisitor(size_t n) + MultiPolygonSerializerVisitor(size_t n) : offsets(ColumnUInt64::create(n)) {} - void operator()(const CartesianPoint & point) + void operator()(const Point & point) { size++; offsets->insertValue(size); polygonSerializer(point); } - void operator()(const CartesianRing & ring) + void operator()(const Ring & ring) { size++; offsets->insertValue(size); polygonSerializer(ring); } - void operator()(const CartesianPolygon & polygon) + void operator()(const Polygon & polygon) { size++; offsets->insertValue(size); polygonSerializer(polygon); } - void operator()(const CartesianMultiPolygon & multi_polygon) + void operator()(const MultiPolygon & multi_polygon) { size += multi_polygon.size(); offsets->insertValue(size); @@ -482,7 +488,7 @@ public: private: size_t size = 0; - CartesianPolygonSerializerVisitor polygonSerializer; + PolygonSerializerVisitor polygonSerializer; ColumnUInt64::MutablePtr offsets; }; @@ -503,9 +509,16 @@ private: Visitor visitor; }; -using CartesianPointSerializer = GeometrySerializer; -using CartesianRingSerializer = GeometrySerializer; -using CartesianPolygonSerializer = GeometrySerializer; -using CartesianMultiPolygonSerializer = GeometrySerializer; +template +using PointSerializer = GeometrySerializer, PointSerializerVisitor>; + +template +using RingSerializer = GeometrySerializer, RingSerializerVisitor>; + +template +using PolygonSerializer = GeometrySerializer, PolygonSerializerVisitor>; + +template +using MultiPolygonSerializer = GeometrySerializer, MultiPolygonSerializerVisitor>; } diff --git a/src/Functions/polygonArea.cpp b/src/Functions/polygonArea.cpp index 27f5cc92461..79d95723f0e 100644 --- a/src/Functions/polygonArea.cpp +++ b/src/Functions/polygonArea.cpp @@ -28,17 +28,17 @@ namespace ErrorCodes extern const int ILLEGAL_COLUMN; } -template -class FunctionPolygonAreaBase : public IFunction +template +class FunctionPolygonArea : public IFunction { public: - static inline const char * name = Derived::name; + static inline const char * name; - explicit FunctionPolygonAreaBase() = default; + explicit FunctionPolygonArea() = default; static FunctionPtr create(const Context &) { - return std::make_shared(); + return std::make_shared(); } String getName() const override @@ -61,9 +61,24 @@ public: return std::make_shared(); } - ColumnPtr executeImpl(const ColumnsWithTypeAndName & arguments, const DataTypePtr & result_type, size_t input_rows_count) const override + ColumnPtr executeImpl(const ColumnsWithTypeAndName & arguments, const DataTypePtr & /*result_type*/, size_t input_rows_count) const override { - return static_cast(this)->executeImplementation(arguments, result_type, input_rows_count); + auto parser = makeGeometryFromColumnParser(arguments[0]); + auto container = createContainer(parser); + + auto res_column = ColumnFloat64::create(); + + for (size_t i = 0; i < input_rows_count; i++) + { + get(parser, container, i); + + Float64 area = boost::geometry::area( + boost::get>(container)); + + res_column->insertValue(area); + } + + return res_column; } bool useDefaultImplementationForConstants() const override @@ -72,68 +87,18 @@ public: } }; +template <> +const char * FunctionPolygonArea::name = "polygonAreaCartesian"; -class FunctionPolygonAreaCartesian : public FunctionPolygonAreaBase +template <> +const char * FunctionPolygonArea::name = "polygonAreaGeographic"; + + +void registerFunctionPolygonArea(FunctionFactory & factory) { -public: - static inline const char * name = "polygonAreaCartesian"; - - ColumnPtr executeImplementation(const ColumnsWithTypeAndName & arguments, const DataTypePtr & /*result_type*/, size_t input_rows_count) const - { - auto parser = makeCartesianGeometryFromColumnParser(arguments[0]); - auto container = createContainer(parser); - - auto res_column = ColumnFloat64::create(); - - for (size_t i = 0; i < input_rows_count; i++) - { - get(parser, container, i); - - Float64 area = boost::geometry::area( - boost::get(container)); - - res_column->insertValue(area); - } - - return res_column; - } -}; - -class FunctionPolygonAreaGeographic : public FunctionPolygonAreaBase -{ -public: - static inline const char * name = "polygonAreaGeographic"; - - ColumnPtr executeImplementation(const ColumnsWithTypeAndName & arguments, const DataTypePtr & /*result_type*/, size_t input_rows_count) const - { - auto parser = makeGeographicGeometryFromColumnParser(arguments[0]); - auto container = createContainer(parser); - - auto res_column = ColumnFloat64::create(); - - for (size_t i = 0; i < input_rows_count; i++) - { - get(parser, container, i); - - Float64 area = boost::geometry::area( - boost::get(container)); - - res_column->insertValue(area); - } - - return res_column; - } -}; - - -void registerFunctionPolygonAreaCartesian(FunctionFactory & factory) -{ - factory.registerFunction(); + factory.registerFunction>(); + factory.registerFunction>(); } -void registerFunctionPolygonAreaGeographic(FunctionFactory & factory) -{ - factory.registerFunction(); -} } diff --git a/src/Functions/polygonConvexHull.cpp b/src/Functions/polygonConvexHull.cpp index f828ff33e6c..5150e344582 100644 --- a/src/Functions/polygonConvexHull.cpp +++ b/src/Functions/polygonConvexHull.cpp @@ -28,10 +28,11 @@ namespace ErrorCodes extern const int ILLEGAL_COLUMN; } +template class FunctionPolygonConvexHull : public IFunction { public: - static inline const char * name = "polygonConvexHull"; + static const char * name; explicit FunctionPolygonConvexHull() = default; @@ -62,27 +63,22 @@ public: ColumnPtr executeImpl(const ColumnsWithTypeAndName & arguments, const DataTypePtr & /*result_type*/, size_t input_rows_count) const override { - auto get_parser = [&arguments] (size_t i) { - const ColumnWithTypeAndName polygon = arguments[i]; - return makeCartesianGeometryFromColumnParser(polygon); - }; - - auto parser = get_parser(0); + auto parser = makeGeometryFromColumnParser(arguments[0]); auto container = createContainer(parser); - CartesianPolygonSerializer serializer; + PolygonSerializer serializer; for (size_t i = 0; i < input_rows_count; i++) { get(parser, container, i); - CartesianGeometry convex_hull = CartesianPolygon({{{}}}); + Geometry convex_hull = Polygon({{{}}}); boost::geometry::convex_hull( - boost::get(container), - boost::get(convex_hull)); + boost::get>(container), + boost::get>(convex_hull)); - boost::get(convex_hull).outer().erase( - boost::get(convex_hull).outer().begin()); + boost::get>(convex_hull).outer().erase( + boost::get>(convex_hull).outer().begin()); serializer.add(convex_hull); } @@ -97,9 +93,17 @@ public: }; +template <> +const char * FunctionPolygonConvexHull::name = "polygonConvexHullCartesian"; + +// template <> +// const char * FunctionPolygonConvexHull::name = "polygonConvexHullGeographic"; + + void registerFunctionPolygonConvexHull(FunctionFactory & factory) { - factory.registerFunction(); + factory.registerFunction>(); + // factory.registerFunction>(); } } diff --git a/src/Functions/polygonPerimeter.cpp b/src/Functions/polygonPerimeter.cpp index 4b7fc8ca678..186fef3d532 100644 --- a/src/Functions/polygonPerimeter.cpp +++ b/src/Functions/polygonPerimeter.cpp @@ -28,17 +28,17 @@ namespace ErrorCodes extern const int ILLEGAL_COLUMN; } -template -class FunctionPolygonPerimeterBase : public IFunction +template +class FunctionPolygonPerimeter : public IFunction { public: - static inline const char * name = Derived::name; + static const char * name; - explicit FunctionPolygonPerimeterBase() = default; + explicit FunctionPolygonPerimeter() = default; static FunctionPtr create(const Context &) { - return std::make_shared(); + return std::make_shared(); } String getName() const override @@ -61,9 +61,24 @@ public: return std::make_shared(); } - ColumnPtr executeImpl(const ColumnsWithTypeAndName & arguments, const DataTypePtr & result_type, size_t input_rows_count) const override + ColumnPtr executeImpl(const ColumnsWithTypeAndName & arguments, const DataTypePtr & /*result_type*/, size_t input_rows_count) const override { - return static_cast(this)->executeImplementation(arguments, result_type, input_rows_count); + auto parser = makeGeometryFromColumnParser(arguments[0]); + auto container = createContainer(parser); + + auto res_column = ColumnFloat64::create(); + + for (size_t i = 0; i < input_rows_count; i++) + { + get(parser, container, i); + + Float64 perimeter = boost::geometry::perimeter( + boost::get>(container)); + + res_column->insertValue(perimeter); + } + + return res_column; } bool useDefaultImplementationForConstants() const override @@ -72,69 +87,18 @@ public: } }; -class FunctionPolygonPerimeterCartesian : public FunctionPolygonPerimeterBase +template <> +const char * FunctionPolygonPerimeter::name = "polygonPerimeterCartesian"; + +template <> +const char * FunctionPolygonPerimeter::name = "polygonPerimeterGeographic"; + + +void registerFunctionPolygonPerimeter(FunctionFactory & factory) { -public: - static inline const char * name = "polygonPerimeterCartesian"; - - ColumnPtr executeImplementation(const ColumnsWithTypeAndName & arguments, const DataTypePtr & /*result_type*/, size_t input_rows_count) const - { - auto parser = makeCartesianGeometryFromColumnParser(arguments[0]); - auto container = createContainer(parser); - - auto res_column = ColumnFloat64::create(); - - for (size_t i = 0; i < input_rows_count; i++) - { - get(parser, container, i); - - Float64 perimeter = boost::geometry::perimeter( - boost::get(container)); - - res_column->insertValue(perimeter); - } - - return res_column; - } -}; - - -class FunctionPolygonPerimeterGeographic : public FunctionPolygonPerimeterBase -{ -public: - static inline const char * name = "polygonPerimeterGeographic"; - - ColumnPtr executeImplementation(const ColumnsWithTypeAndName & arguments, const DataTypePtr & /*result_type*/, size_t input_rows_count) const - { - auto parser = makeGeographicGeometryFromColumnParser(arguments[0]); - auto container = createContainer(parser); - - auto res_column = ColumnFloat64::create(); - - for (size_t i = 0; i < input_rows_count; i++) - { - get(parser, container, i); - - Float64 perimeter = boost::geometry::perimeter( - boost::get(container)); - - res_column->insertValue(perimeter); - } - - return res_column; - } -}; - - - -void registerFunctionPolygonPerimeterCartesian(FunctionFactory & factory) -{ - factory.registerFunction(); + factory.registerFunction>(); + factory.registerFunction>(); } -void registerFunctionPolygonPerimeterGeographic(FunctionFactory & factory) -{ - factory.registerFunction(); -} } diff --git a/src/Functions/polygonsDistance.cpp b/src/Functions/polygonsDistance.cpp index 99a8d8dafe5..7a2988e8ada 100644 --- a/src/Functions/polygonsDistance.cpp +++ b/src/Functions/polygonsDistance.cpp @@ -30,17 +30,17 @@ namespace ErrorCodes extern const int ILLEGAL_COLUMN; } -template -class FunctionPolygonsDistanceBase : public IFunction +template +class FunctionPolygonsDistance : public IFunction { public: - static inline const char * name = Derived::name; + static inline const char * name; - explicit FunctionPolygonsDistanceBase() = default; + explicit FunctionPolygonsDistance() = default; static FunctionPtr create(const Context &) { - return std::make_shared(); + return std::make_shared(); } String getName() const override @@ -63,9 +63,29 @@ public: return std::make_shared(); } - ColumnPtr executeImpl(const ColumnsWithTypeAndName & arguments, const DataTypePtr & result_type, size_t input_rows_count) const override + ColumnPtr executeImpl(const ColumnsWithTypeAndName & arguments, const DataTypePtr & /*result_type*/, size_t input_rows_count) const override { - return static_cast(this)->executeImplementation(arguments, result_type, input_rows_count); + auto first_parser = makeGeometryFromColumnParser(arguments[0]); + auto first_container = createContainer(first_parser); + + auto second_parser = makeGeometryFromColumnParser(arguments[1]); + auto second_container = createContainer(second_parser); + + auto res_column = ColumnFloat64::create(); + + for (size_t i = 0; i < input_rows_count; i++) + { + get(first_parser, first_container, i); + get(second_parser, second_container, i); + + Float64 distance = boost::geometry::distance( + boost::get>(first_container), + boost::get>(second_container)); + + res_column->insertValue(distance); + } + + return res_column; } bool useDefaultImplementationForConstants() const override @@ -74,73 +94,18 @@ public: } }; +template <> +const char * FunctionPolygonsDistance::name = "polygonsDistanceCartesian"; -class FunctionPolygonsDistanceCartesian : public FunctionPolygonsDistanceBase +template <> +const char * FunctionPolygonsDistance::name = "polygonsDistanceGeographic"; + + +void registerFunctionPolygonsDistance(FunctionFactory & factory) { -public: - static inline const char * name = "polygonsDistanceCartesian"; - - ColumnPtr executeImplementation(const ColumnsWithTypeAndName & arguments, const DataTypePtr & /*result_type*/, size_t input_rows_count) const - { - auto first_parser = makeCartesianGeometryFromColumnParser(arguments[0]); - auto first_container = createContainer(first_parser); - - auto second_parser = makeCartesianGeometryFromColumnParser(arguments[1]); - auto second_container = createContainer(second_parser); - - auto res_column = ColumnFloat64::create(); - - for (size_t i = 0; i < input_rows_count; i++) - { - Float64 distance = boost::geometry::distance( - boost::get(first_container), - boost::get(second_container)); - - res_column->insertValue(distance); - } - - return res_column; - } -}; - - -class FunctionPolygonsDistanceGeographic : public FunctionPolygonsDistanceBase -{ -public: - static inline const char * name = "polygonsDistanceGeographic"; - - ColumnPtr executeImplementation(const ColumnsWithTypeAndName & arguments, const DataTypePtr & /*result_type*/, size_t input_rows_count) const - { - auto first_parser = makeGeographicGeometryFromColumnParser(arguments[0]); - auto first_container = createContainer(first_parser); - - auto second_parser = makeGeographicGeometryFromColumnParser(arguments[1]); - auto second_container = createContainer(second_parser); - - auto res_column = ColumnFloat64::create(); - - for (size_t i = 0; i < input_rows_count; i++) - { - Float64 distance = boost::geometry::distance( - boost::get(first_container), - boost::get(second_container)); - - res_column->insertValue(distance); - } - - return res_column; - } -}; - - -void registerFunctionPolygonsDistanceCartesian(FunctionFactory & factory) -{ - factory.registerFunction(); + factory.registerFunction>(); + factory.registerFunction>(); } -void registerFunctionPolygonsDistanceGeographic(FunctionFactory & factory) -{ - factory.registerFunction(); -} } diff --git a/src/Functions/polygonsEquals.cpp b/src/Functions/polygonsEquals.cpp index a23768626ad..d30e5b24d2a 100644 --- a/src/Functions/polygonsEquals.cpp +++ b/src/Functions/polygonsEquals.cpp @@ -70,9 +70,9 @@ public: bool is_const = static_cast(const_col); - return std::pair{is_const, is_const ? - makeCartesianGeometryFromColumnParser(ColumnWithTypeAndName(const_col->getDataColumnPtr(), arguments[i].type, arguments[i].name)) : - makeCartesianGeometryFromColumnParser(arguments[i])}; + return std::pair>{is_const, is_const ? + makeGeometryFromColumnParser(ColumnWithTypeAndName(const_col->getDataColumnPtr(), arguments[i].type, arguments[i].name)) : + makeGeometryFromColumnParser(arguments[i])}; }; auto [is_first_polygon_const, first_parser] = get_parser(0); diff --git a/src/Functions/polygonsIntersection.cpp b/src/Functions/polygonsIntersection.cpp index 37448b4acf0..90d129112b7 100644 --- a/src/Functions/polygonsIntersection.cpp +++ b/src/Functions/polygonsIntersection.cpp @@ -63,13 +63,13 @@ public: ColumnPtr executeImpl(const ColumnsWithTypeAndName & arguments, const DataTypePtr & /*result_type*/, size_t input_rows_count) const override { - auto first_parser = makeCartesianGeometryFromColumnParser(arguments[0]); + auto first_parser = makeGeometryFromColumnParser(arguments[0]); auto first_container = createContainer(first_parser); - auto second_parser = makeCartesianGeometryFromColumnParser(arguments[1]); + auto second_parser = makeGeometryFromColumnParser(arguments[1]); auto second_container = createContainer(second_parser); - CartesianMultiPolygonSerializer serializer; + MultiPolygonSerializer serializer; for (size_t i = 0; i < input_rows_count; i++) { diff --git a/src/Functions/polygonsSymDifference.cpp b/src/Functions/polygonsSymDifference.cpp index 0c0a9017152..50558c69662 100644 --- a/src/Functions/polygonsSymDifference.cpp +++ b/src/Functions/polygonsSymDifference.cpp @@ -28,10 +28,11 @@ namespace ErrorCodes extern const int ILLEGAL_COLUMN; } +template class FunctionPolygonsSymDifference : public IFunction { public: - static inline const char * name = "polygonsSymDifference"; + static const char * name; explicit FunctionPolygonsSymDifference() = default; @@ -62,40 +63,27 @@ public: ColumnPtr executeImpl(const ColumnsWithTypeAndName & arguments, const DataTypePtr & /*result_type*/, size_t input_rows_count) const override { - auto get_parser = [&arguments] (size_t i) { - const auto * const_col = - checkAndGetColumn(arguments[i].column.get()); - - bool is_const = static_cast(const_col); - - return std::pair{is_const, is_const ? - makeCartesianGeometryFromColumnParser(ColumnWithTypeAndName(const_col->getDataColumnPtr(), arguments[i].type, arguments[i].name)) : - makeCartesianGeometryFromColumnParser(arguments[i])}; - }; - - auto [is_first_polygon_const, first_parser] = get_parser(0); + auto first_parser = makeGeometryFromColumnParser(arguments[0]); auto first_container = createContainer(first_parser); - auto [is_second_polygon_const, second_parser] = get_parser(1); + auto second_parser = makeGeometryFromColumnParser(arguments[1]); auto second_container = createContainer(second_parser); - CartesianMultiPolygonSerializer serializer; + MultiPolygonSerializer serializer; for (size_t i = 0; i < input_rows_count; i++) { - if (!is_first_polygon_const || i == 0) - get(first_parser, first_container, i); - if (!is_second_polygon_const || i == 0) - get(second_parser, second_container, i); + get(first_parser, first_container, i); + get(second_parser, second_container, i); - CartesianGeometry sym_difference = CartesianMultiPolygon({{{{}}}}); + Geometry sym_difference = MultiPolygon({{{{}}}}); boost::geometry::sym_difference( - boost::get(first_container), - boost::get(second_container), - boost::get(sym_difference)); + boost::get>(first_container), + boost::get>(second_container), + boost::get>(sym_difference)); - boost::get(sym_difference).erase( - boost::get(sym_difference).begin()); + boost::get>(sym_difference).erase( + boost::get>(sym_difference).begin()); serializer.add(sym_difference); } @@ -109,10 +97,18 @@ public: } }; +template <> +const char * FunctionPolygonsSymDifference::name = "polygonsSymDifferenceCartesian"; + +template <> +const char * FunctionPolygonsSymDifference::name = "polygonsSymDifferenceGeographic"; void registerFunctionPolygonsSymDifference(FunctionFactory & factory) { - factory.registerFunction(); + factory.registerFunction>(); + factory.registerFunction>(); } + + } diff --git a/src/Functions/polygonsWithin.cpp b/src/Functions/polygonsWithin.cpp index 03226ba551c..7f01330ea33 100644 --- a/src/Functions/polygonsWithin.cpp +++ b/src/Functions/polygonsWithin.cpp @@ -70,9 +70,9 @@ public: bool is_const = static_cast(const_col); - return std::pair{is_const, is_const ? - makeCartesianGeometryFromColumnParser(ColumnWithTypeAndName(const_col->getDataColumnPtr(), arguments[i].type, arguments[i].name)) : - makeCartesianGeometryFromColumnParser(arguments[i])}; + return std::pair>{is_const, is_const ? + makeGeometryFromColumnParser(ColumnWithTypeAndName(const_col->getDataColumnPtr(), arguments[i].type, arguments[i].name)) : + makeGeometryFromColumnParser(arguments[i])}; }; auto [is_first_polygon_const, first_parser] = get_parser(0); diff --git a/src/Functions/readWkt.cpp b/src/Functions/readWkt.cpp index 2d64db76466..5e2becbe131 100644 --- a/src/Functions/readWkt.cpp +++ b/src/Functions/readWkt.cpp @@ -62,7 +62,7 @@ public: } }; -class FunctionReadWktPoint : public FunctionReadWkt +class FunctionReadWktPoint : public FunctionReadWkt> { public: static inline const char * name = "readWktPoint"; @@ -76,7 +76,7 @@ public: } }; -class FunctionReadWktPolygon : public FunctionReadWkt +class FunctionReadWktPolygon : public FunctionReadWkt> { public: static inline const char * name = "readWktPolygon"; @@ -90,7 +90,7 @@ public: } }; -class FunctionReadWktMultiPolygon : public FunctionReadWkt +class FunctionReadWktMultiPolygon : public FunctionReadWkt> { public: static inline const char * name = "readWktMultiPolygon"; diff --git a/src/Functions/registerFunctionsGeo.cpp b/src/Functions/registerFunctionsGeo.cpp index fe7ea412841..605dd4dcba0 100644 --- a/src/Functions/registerFunctionsGeo.cpp +++ b/src/Functions/registerFunctionsGeo.cpp @@ -12,16 +12,13 @@ void registerFunctionPointInEllipses(FunctionFactory & factory); void registerFunctionPointInPolygon(FunctionFactory & factory); void registerFunctionPolygonsIntersection(FunctionFactory & factory); void registerFunctionPolygonsUnion(FunctionFactory & factory); -void registerFunctionPolygonAreaCartesian(FunctionFactory & factory); -void registerFunctionPolygonAreaGeographic (FunctionFactory & factory); +void registerFunctionPolygonArea(FunctionFactory & factory); void registerFunctionPolygonConvexHull(FunctionFactory & factory); void registerFunctionPolygonsSymDifference(FunctionFactory & factory); void registerFunctionPolygonsEquals(FunctionFactory & factory); -void registerFunctionPolygonsDistanceCartesian(FunctionFactory & factory); -void registerFunctionPolygonsDistanceGeographic(FunctionFactory & factory); +void registerFunctionPolygonsDistance(FunctionFactory & factory); void registerFunctionPolygonsWithin(FunctionFactory & factory); -void registerFunctionPolygonPerimeterCartesian(FunctionFactory & factory); -void registerFunctionPolygonPerimeterGeographic(FunctionFactory & factory); +void registerFunctionPolygonPerimeter(FunctionFactory & factory); void registerFunctionGeohashEncode(FunctionFactory & factory); void registerFunctionGeohashDecode(FunctionFactory & factory); void registerFunctionGeohashesInBox(FunctionFactory & factory); @@ -53,16 +50,13 @@ void registerFunctionsGeo(FunctionFactory & factory) registerFunctionPointInPolygon(factory); registerFunctionPolygonsIntersection(factory); registerFunctionPolygonsUnion(factory); - registerFunctionPolygonAreaCartesian(factory); - registerFunctionPolygonAreaGeographic(factory); + registerFunctionPolygonArea(factory); registerFunctionPolygonConvexHull(factory); registerFunctionPolygonsSymDifference(factory); registerFunctionPolygonsEquals(factory); - registerFunctionPolygonsDistanceCartesian(factory); - registerFunctionPolygonsDistanceGeographic(factory); + registerFunctionPolygonsDistance(factory); registerFunctionPolygonsWithin(factory); - registerFunctionPolygonPerimeterCartesian(factory); - registerFunctionPolygonPerimeterGeographic(factory); + registerFunctionPolygonPerimeter(factory); registerFunctionGeohashEncode(factory); registerFunctionGeohashDecode(factory); registerFunctionGeohashesInBox(factory); diff --git a/src/Functions/svg.cpp b/src/Functions/svg.cpp index 6a0827a107c..5bd84f49aa7 100644 --- a/src/Functions/svg.cpp +++ b/src/Functions/svg.cpp @@ -65,8 +65,8 @@ public: const auto * const_col = checkAndGetColumn(arguments[0].column.get()); auto parser = const_col ? - makeCartesianGeometryFromColumnParser(ColumnWithTypeAndName(const_col->getDataColumnPtr(), arguments[0].type, arguments[0].name)) : - makeCartesianGeometryFromColumnParser(arguments[0]); + makeGeometryFromColumnParser(ColumnWithTypeAndName(const_col->getDataColumnPtr(), arguments[0].type, arguments[0].name)) : + makeGeometryFromColumnParser(arguments[0]); bool geo_column_is_const = static_cast(const_col); diff --git a/src/Functions/wkt.cpp b/src/Functions/wkt.cpp index 9e01e312503..e184f7537aa 100644 --- a/src/Functions/wkt.cpp +++ b/src/Functions/wkt.cpp @@ -38,7 +38,7 @@ public: ColumnPtr executeImpl(const ColumnsWithTypeAndName & arguments, const DataTypePtr & /*result_type*/, size_t input_rows_count) const override { - auto parser = makeCartesianGeometryFromColumnParser(arguments[0]); + auto parser = makeGeometryFromColumnParser(arguments[0]); auto res_column = ColumnString::create(); auto container = createContainer(parser); diff --git a/tests/queries/0_stateless/01302_polygons_distance.sql b/tests/queries/0_stateless/01302_polygons_distance.sql index f258f3c3a8b..a4c769ad8cb 100644 --- a/tests/queries/0_stateless/01302_polygons_distance.sql +++ b/tests/queries/0_stateless/01302_polygons_distance.sql @@ -1,7 +1,8 @@ select polygonsDistanceCartesian([[[(0, 0),(0, 3),(1, 2.9),(2, 2.6),(2.6, 2),(2.9, 1),(3, 0),(0, 0)]]], [[[(1, 1),(1, 4),(4, 4),(4, 1),(1, 1)]]]); select polygonsDistanceCartesian([[[(0, 0), (0, 0.1), (0.1, 0.1), (0.1, 0)]]], [[[(1, 1),(1, 4),(4, 4),(4, 1),(1, 1)]]]); select polygonsDistanceGeographic([[[(23.725750, 37.971536)]]], [[[(4.3826169, 50.8119483)]]]); -drop table if exists polygon_01302 + +drop table if exists polygon_01302; create table polygon_01302 (x Array(Array(Array(Tuple(Float64, Float64)))), y Array(Array(Array(Tuple(Float64, Float64))))) engine=Memory(); insert into polygon_01302 values ([[[(23.725750, 37.971536)]]], [[[(4.3826169, 50.8119483)]]]); -select polygonsDistanceGeographic(x, y) from polygon; +select polygonsDistanceGeographic(x, y) from polygon_01302; diff --git a/tests/queries/0_stateless/01306_polygons_intersection.reference b/tests/queries/0_stateless/01306_polygons_intersection.reference index 3eb60ccf3de..14917a34b57 100644 --- a/tests/queries/0_stateless/01306_polygons_intersection.reference +++ b/tests/queries/0_stateless/01306_polygons_intersection.reference @@ -1 +1,2 @@ [[[(1,2.9),(2,2.6),(2.6,2),(2.9,1),(1,1),(1,2.9)]]] +[] diff --git a/tests/queries/0_stateless/01306_polygons_intersection.sql b/tests/queries/0_stateless/01306_polygons_intersection.sql index d1ce5b54be3..6e9921f9476 100644 --- a/tests/queries/0_stateless/01306_polygons_intersection.sql +++ b/tests/queries/0_stateless/01306_polygons_intersection.sql @@ -1 +1,2 @@ -select polygonsIntersection([[[(0, 0),(0, 3),(1, 2.9),(2, 2.6),(2.6, 2),(2.9, 1),(3, 0),(0, 0)]]], [[[(1, 1),(1, 4),(4, 4),(4, 1),(1, 1)]]]) +select polygonsIntersection([[[(0, 0),(0, 3),(1, 2.9),(2, 2.6),(2.6, 2),(2.9, 1),(3, 0),(0, 0)]]], [[[(1, 1),(1, 4),(4, 4),(4, 1),(1, 1)]]]); +select polygonsIntersection([[[(0, 0),(0, 3),(1, 2.9),(2, 2.6),(2.6, 2),(2.9, 1),(3, 0),(0, 0)]]], [[[(3, 3),(3, 4),(4, 4),(4, 3),(3, 3)]]]); \ No newline at end of file From cdbdcbf793f1a7ebe8d498184e987b105e756836 Mon Sep 17 00:00:00 2001 From: Nikita Mikhailov Date: Tue, 19 Jan 2021 17:53:39 +0300 Subject: [PATCH 277/544] update ya.make --- src/Functions/ya.make | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/src/Functions/ya.make b/src/Functions/ya.make index 35b855f7c6c..604d5243be3 100644 --- a/src/Functions/ya.make +++ b/src/Functions/ya.make @@ -74,6 +74,7 @@ SRCS( GatherUtils/sliceFromRightConstantOffsetUnbounded.cpp GeoHash.cpp IFunction.cpp + PolygonsUnion.cpp TargetSpecific.cpp URL/URLHierarchy.cpp URL/URLPathHierarchy.cpp @@ -378,6 +379,14 @@ SRCS( plus.cpp pointInEllipses.cpp pointInPolygon.cpp + polygonArea.cpp + polygonConvexHull.cpp + polygonPerimeter.cpp + polygonsDistance.cpp + polygonsEquals.cpp + polygonsIntersection.cpp + polygonsSymDifference.cpp + polygonsWithin.cpp position.cpp positionCaseInsensitive.cpp positionCaseInsensitiveUTF8.cpp @@ -390,9 +399,9 @@ SRCS( randomPrintableASCII.cpp randomString.cpp randomStringUTF8.cpp + readWkt.cpp regexpQuoteMeta.cpp registerFunctions.cpp - readWkt.cpp registerFunctionsArithmetic.cpp registerFunctionsComparison.cpp registerFunctionsConditional.cpp From 155d392d313aa36466204be57074c70f3556be56 Mon Sep 17 00:00:00 2001 From: Nikita Mikhailov Date: Tue, 19 Jan 2021 19:34:16 +0300 Subject: [PATCH 278/544] fix tests --- tests/queries/0_stateless/01300_polygon_convex_hull.sql | 2 +- tests/queries/0_stateless/01304_polygons_sym_difference.sql | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/queries/0_stateless/01300_polygon_convex_hull.sql b/tests/queries/0_stateless/01300_polygon_convex_hull.sql index 547670dfe8f..125d10f3afc 100644 --- a/tests/queries/0_stateless/01300_polygon_convex_hull.sql +++ b/tests/queries/0_stateless/01300_polygon_convex_hull.sql @@ -1 +1 @@ -select polygonConvexHull([[[(0, 0), (0, 5), (5, 5), (5, 0), (2, 3)]]]) +select polygonConvexHullCartesian([[[(0, 0), (0, 5), (5, 5), (5, 0), (2, 3)]]]); diff --git a/tests/queries/0_stateless/01304_polygons_sym_difference.sql b/tests/queries/0_stateless/01304_polygons_sym_difference.sql index e351add03e6..1d839fa80b3 100644 --- a/tests/queries/0_stateless/01304_polygons_sym_difference.sql +++ b/tests/queries/0_stateless/01304_polygons_sym_difference.sql @@ -1,2 +1,2 @@ -select polygonsSymDifference([[[(0, 0),(0, 3),(1, 2.9),(2, 2.6),(2.6, 2),(2.9, 1),(3, 0),(0, 0)]]], [[[(1, 1),(1, 4),(4, 4),(4, 1),(1, 1)]]]) +select polygonsSymDifferenceCartesian([[[(0, 0),(0, 3),(1, 2.9),(2, 2.6),(2.6, 2),(2.9, 1),(3, 0),(0, 0)]]], [[[(1, 1),(1, 4),(4, 4),(4, 1),(1, 1)]]]) From 2475143ed35d6806cd75b33dcf6b3edce4279fd8 Mon Sep 17 00:00:00 2001 From: Nikita Mikhailov Date: Tue, 19 Jan 2021 20:16:10 +0300 Subject: [PATCH 279/544] fix style --- src/DataTypes/DataTypeCustomGeo.h | 2 + src/Functions/PolygonsUnion.cpp | 8 ---- src/Functions/geometryConverters.cpp | 11 ++++- src/Functions/geometryConverters.h | 35 ++++++++-------- src/Functions/polygonArea.cpp | 9 ---- src/Functions/polygonConvexHull.cpp | 8 ---- src/Functions/polygonPerimeter.cpp | 8 ---- src/Functions/polygonsDistance.cpp | 8 ---- src/Functions/polygonsEquals.cpp | 42 ++++++------------- src/Functions/polygonsIntersection.cpp | 10 ----- src/Functions/polygonsSymDifference.cpp | 10 ----- src/Functions/polygonsWithin.cpp | 42 ++++++------------- src/Functions/readWkt.cpp | 3 +- src/Functions/svg.cpp | 13 +++--- src/Functions/wkt.cpp | 3 +- .../0_stateless/01301_polygons_within.sql | 4 +- .../0_stateless/01303_polygons_equals.sql | 4 +- 17 files changed, 69 insertions(+), 151 deletions(-) diff --git a/src/DataTypes/DataTypeCustomGeo.h b/src/DataTypes/DataTypeCustomGeo.h index 00ca898f53b..8f549812b8b 100644 --- a/src/DataTypes/DataTypeCustomGeo.h +++ b/src/DataTypes/DataTypeCustomGeo.h @@ -1,3 +1,5 @@ +#pragma once + #include #include #include diff --git a/src/Functions/PolygonsUnion.cpp b/src/Functions/PolygonsUnion.cpp index 2d26e09c3de..9d45269d0cb 100644 --- a/src/Functions/PolygonsUnion.cpp +++ b/src/Functions/PolygonsUnion.cpp @@ -20,14 +20,6 @@ namespace DB { -namespace ErrorCodes -{ - extern const int TOO_FEW_ARGUMENTS_FOR_FUNCTION; - extern const int BAD_ARGUMENTS; - extern const int ILLEGAL_TYPE_OF_ARGUMENT; - extern const int ILLEGAL_COLUMN; -} - class FunctionPolygonsUnion : public IFunction { public: diff --git a/src/Functions/geometryConverters.cpp b/src/Functions/geometryConverters.cpp index 4fdc1ea5931..b8a15ff98d5 100644 --- a/src/Functions/geometryConverters.cpp +++ b/src/Functions/geometryConverters.cpp @@ -3,9 +3,16 @@ #include -namespace DB { +namespace DB +{ -namespace { +namespace ErrorCodes +{ + extern const int ILLEGAL_COLUMN; +} + +namespace +{ size_t getArrayDepth(DataTypePtr data_type, size_t max_depth) { diff --git a/src/Functions/geometryConverters.h b/src/Functions/geometryConverters.h index 134209c3b40..2dd0ce117b1 100644 --- a/src/Functions/geometryConverters.h +++ b/src/Functions/geometryConverters.h @@ -18,11 +18,11 @@ #include -namespace DB { +namespace DB +{ namespace ErrorCodes { - extern const int ILLEGAL_COLUMN; extern const int BAD_ARGUMENTS; } @@ -54,8 +54,8 @@ using GeographicMultiPolygon = MultiPolygon; using GeographicGeometry = Geometry; /** - * Class which takes some boost type and returns a pair of numbers. - * They are (x,y) in case of cartesian coordinated and (lon,lat) in case of geographic. + * Class which takes some boost type and returns a pair of numbers. + * They are (x,y) in case of cartesian coordinated and (lon,lat) in case of geographic. */ template class PointFromColumnParser @@ -154,9 +154,8 @@ public: container.reserve(r - l + 1); container.resize(r - l); - for (size_t j = l; j < r; j++) { + for (size_t j = l; j < r; j++) point_parser.get(container[j - l], j); - } // make ring closed if (!boost::geometry::equals(container[0], container.back())) @@ -289,25 +288,25 @@ public: void operator()(const Ring & ring) { - if (ring.size() != 1) { + if (ring.size() != 1) throw Exception("Unable to write ring of size " + toString(ring.size()) + " != 1 to point column", ErrorCodes::BAD_ARGUMENTS); - } + (*this)(ring[0]); } void operator()(const Polygon & polygon) { - if (polygon.inners().size() != 0) { + if (polygon.inners().size() != 0) throw Exception("Unable to write polygon with holes to point column", ErrorCodes::BAD_ARGUMENTS); - } + (*this)(polygon.outer()); } void operator()(const MultiPolygon & multi_polygon) { - if (multi_polygon.size() != 1) { + if (multi_polygon.size() != 1) throw Exception("Unable to write multi-polygon of size " + toString(multi_polygon.size()) + " != 1 to point column", ErrorCodes::BAD_ARGUMENTS); - } + (*this)(multi_polygon[0]); } @@ -357,17 +356,17 @@ public: void operator()(const Polygon & polygon) { - if (polygon.inners().size() != 0) { + if (polygon.inners().size() != 0) throw Exception("Unable to write polygon with holes to ring column", ErrorCodes::BAD_ARGUMENTS); - } + (*this)(polygon.outer()); } void operator()(const MultiPolygon & multi_polygon) { - if (multi_polygon.size() != 1) { + if (multi_polygon.size() != 1) throw Exception("Unable to write multi-polygon of size " + toString(multi_polygon.size()) + " != 1 to ring column", ErrorCodes::BAD_ARGUMENTS); - } + (*this)(multi_polygon[0]); } @@ -421,9 +420,9 @@ public: void operator()(const MultiPolygon & multi_polygon) { - if (multi_polygon.size() != 1) { + if (multi_polygon.size() != 1) throw Exception("Unable to write multi-polygon of size " + toString(multi_polygon.size()) + " != 1 to polygon column", ErrorCodes::BAD_ARGUMENTS); - } + (*this)(multi_polygon[0]); } diff --git a/src/Functions/polygonArea.cpp b/src/Functions/polygonArea.cpp index 79d95723f0e..aac9a182ea9 100644 --- a/src/Functions/polygonArea.cpp +++ b/src/Functions/polygonArea.cpp @@ -19,15 +19,6 @@ namespace DB { - -namespace ErrorCodes -{ - extern const int TOO_FEW_ARGUMENTS_FOR_FUNCTION; - extern const int BAD_ARGUMENTS; - extern const int ILLEGAL_TYPE_OF_ARGUMENT; - extern const int ILLEGAL_COLUMN; -} - template class FunctionPolygonArea : public IFunction { diff --git a/src/Functions/polygonConvexHull.cpp b/src/Functions/polygonConvexHull.cpp index 5150e344582..1cb46b81c32 100644 --- a/src/Functions/polygonConvexHull.cpp +++ b/src/Functions/polygonConvexHull.cpp @@ -20,14 +20,6 @@ namespace DB { -namespace ErrorCodes -{ - extern const int TOO_FEW_ARGUMENTS_FOR_FUNCTION; - extern const int BAD_ARGUMENTS; - extern const int ILLEGAL_TYPE_OF_ARGUMENT; - extern const int ILLEGAL_COLUMN; -} - template class FunctionPolygonConvexHull : public IFunction { diff --git a/src/Functions/polygonPerimeter.cpp b/src/Functions/polygonPerimeter.cpp index 186fef3d532..aafc07aa3b2 100644 --- a/src/Functions/polygonPerimeter.cpp +++ b/src/Functions/polygonPerimeter.cpp @@ -20,14 +20,6 @@ namespace DB { -namespace ErrorCodes -{ - extern const int TOO_FEW_ARGUMENTS_FOR_FUNCTION; - extern const int BAD_ARGUMENTS; - extern const int ILLEGAL_TYPE_OF_ARGUMENT; - extern const int ILLEGAL_COLUMN; -} - template class FunctionPolygonPerimeter : public IFunction { diff --git a/src/Functions/polygonsDistance.cpp b/src/Functions/polygonsDistance.cpp index 7a2988e8ada..67f6482ec22 100644 --- a/src/Functions/polygonsDistance.cpp +++ b/src/Functions/polygonsDistance.cpp @@ -22,14 +22,6 @@ namespace DB { -namespace ErrorCodes -{ - extern const int TOO_FEW_ARGUMENTS_FOR_FUNCTION; - extern const int BAD_ARGUMENTS; - extern const int ILLEGAL_TYPE_OF_ARGUMENT; - extern const int ILLEGAL_COLUMN; -} - template class FunctionPolygonsDistance : public IFunction { diff --git a/src/Functions/polygonsEquals.cpp b/src/Functions/polygonsEquals.cpp index d30e5b24d2a..118c5885cdd 100644 --- a/src/Functions/polygonsEquals.cpp +++ b/src/Functions/polygonsEquals.cpp @@ -22,18 +22,11 @@ namespace DB { -namespace ErrorCodes -{ - extern const int TOO_FEW_ARGUMENTS_FOR_FUNCTION; - extern const int BAD_ARGUMENTS; - extern const int ILLEGAL_TYPE_OF_ARGUMENT; - extern const int ILLEGAL_COLUMN; -} - +template class FunctionPolygonsEquals : public IFunction { public: - static inline const char * name = "polygonsEquals"; + static const char * name; explicit FunctionPolygonsEquals() = default; @@ -64,35 +57,22 @@ public: ColumnPtr executeImpl(const ColumnsWithTypeAndName & arguments, const DataTypePtr & /*result_type*/, size_t input_rows_count) const override { - auto get_parser = [&arguments] (size_t i) { - const auto * const_col = - checkAndGetColumn(arguments[i].column.get()); - - bool is_const = static_cast(const_col); - - return std::pair>{is_const, is_const ? - makeGeometryFromColumnParser(ColumnWithTypeAndName(const_col->getDataColumnPtr(), arguments[i].type, arguments[i].name)) : - makeGeometryFromColumnParser(arguments[i])}; - }; - - auto [is_first_polygon_const, first_parser] = get_parser(0); + auto first_parser = makeGeometryFromColumnParser(arguments[0]); auto first_container = createContainer(first_parser); - auto [is_second_polygon_const, second_parser] = get_parser(1); + auto second_parser = makeGeometryFromColumnParser(arguments[1]); auto second_container = createContainer(second_parser); auto res_column = ColumnUInt8::create(); for (size_t i = 0; i < input_rows_count; i++) { - if (!is_first_polygon_const || i == 0) - get(first_parser, first_container, i); - if (!is_second_polygon_const || i == 0) - get(second_parser, second_container, i); + get(first_parser, first_container, i); + get(second_parser, second_container, i); bool equals = boost::geometry::equals( - boost::get(first_container), - boost::get(second_container)); + boost::get>(first_container), + boost::get>(second_container)); res_column->insertValue(equals); } @@ -107,9 +87,13 @@ public: }; +template <> +const char * FunctionPolygonsEquals::name = "polygonsEqualsCartesian"; + + void registerFunctionPolygonsEquals(FunctionFactory & factory) { - factory.registerFunction(); + factory.registerFunction>(); } } diff --git a/src/Functions/polygonsIntersection.cpp b/src/Functions/polygonsIntersection.cpp index 90d129112b7..335b40322f7 100644 --- a/src/Functions/polygonsIntersection.cpp +++ b/src/Functions/polygonsIntersection.cpp @@ -19,16 +19,6 @@ namespace DB { - -namespace ErrorCodes -{ - extern const int TOO_FEW_ARGUMENTS_FOR_FUNCTION; - extern const int BAD_ARGUMENTS; - extern const int ILLEGAL_TYPE_OF_ARGUMENT; - extern const int ILLEGAL_COLUMN; -} - - class FunctionPolygonsIntersection : public IFunction { public: diff --git a/src/Functions/polygonsSymDifference.cpp b/src/Functions/polygonsSymDifference.cpp index 50558c69662..6d42e001dc8 100644 --- a/src/Functions/polygonsSymDifference.cpp +++ b/src/Functions/polygonsSymDifference.cpp @@ -20,14 +20,6 @@ namespace DB { -namespace ErrorCodes -{ - extern const int TOO_FEW_ARGUMENTS_FOR_FUNCTION; - extern const int BAD_ARGUMENTS; - extern const int ILLEGAL_TYPE_OF_ARGUMENT; - extern const int ILLEGAL_COLUMN; -} - template class FunctionPolygonsSymDifference : public IFunction { @@ -109,6 +101,4 @@ void registerFunctionPolygonsSymDifference(FunctionFactory & factory) factory.registerFunction>(); } - - } diff --git a/src/Functions/polygonsWithin.cpp b/src/Functions/polygonsWithin.cpp index 7f01330ea33..3a2c5704083 100644 --- a/src/Functions/polygonsWithin.cpp +++ b/src/Functions/polygonsWithin.cpp @@ -22,18 +22,11 @@ namespace DB { -namespace ErrorCodes -{ - extern const int TOO_FEW_ARGUMENTS_FOR_FUNCTION; - extern const int BAD_ARGUMENTS; - extern const int ILLEGAL_TYPE_OF_ARGUMENT; - extern const int ILLEGAL_COLUMN; -} - +template class FunctionPolygonsWithin : public IFunction { public: - static inline const char * name = "polygonsWithin"; + static inline const char * name; explicit FunctionPolygonsWithin() = default; @@ -64,35 +57,22 @@ public: ColumnPtr executeImpl(const ColumnsWithTypeAndName & arguments, const DataTypePtr & /*result_type*/, size_t input_rows_count) const override { - auto get_parser = [&arguments] (size_t i) { - const auto * const_col = - checkAndGetColumn(arguments[i].column.get()); - - bool is_const = static_cast(const_col); - - return std::pair>{is_const, is_const ? - makeGeometryFromColumnParser(ColumnWithTypeAndName(const_col->getDataColumnPtr(), arguments[i].type, arguments[i].name)) : - makeGeometryFromColumnParser(arguments[i])}; - }; - - auto [is_first_polygon_const, first_parser] = get_parser(0); + auto first_parser = makeGeometryFromColumnParser(arguments[0]); auto first_container = createContainer(first_parser); - auto [is_second_polygon_const, second_parser] = get_parser(1); + auto second_parser = makeGeometryFromColumnParser(arguments[1]); auto second_container = createContainer(second_parser); auto res_column = ColumnUInt8::create(); for (size_t i = 0; i < input_rows_count; i++) { - if (!is_first_polygon_const || i == 0) - get(first_parser, first_container, i); - if (!is_second_polygon_const || i == 0) - get(second_parser, second_container, i); + get(first_parser, first_container, i); + get(second_parser, second_container, i); bool within = boost::geometry::within( - boost::get(first_container), - boost::get(second_container)); + boost::get>(first_container), + boost::get>(second_container)); res_column->insertValue(within); } @@ -107,9 +87,13 @@ public: }; +template <> +const char * FunctionPolygonsWithin::name = "polygonsWithinCartesian"; + + void registerFunctionPolygonsWithin(FunctionFactory & factory) { - factory.registerFunction(); + factory.registerFunction>(); } } diff --git a/src/Functions/readWkt.cpp b/src/Functions/readWkt.cpp index 5e2becbe131..12007b8724b 100644 --- a/src/Functions/readWkt.cpp +++ b/src/Functions/readWkt.cpp @@ -13,8 +13,7 @@ namespace DB namespace ErrorCodes { -extern const int ILLEGAL_TYPE_OF_ARGUMENT; -extern const int BAD_ARGUMENT; + extern const int ILLEGAL_TYPE_OF_ARGUMENT; } template diff --git a/src/Functions/svg.cpp b/src/Functions/svg.cpp index 5bd84f49aa7..3d8206cc2e3 100644 --- a/src/Functions/svg.cpp +++ b/src/Functions/svg.cpp @@ -10,7 +10,9 @@ namespace DB { -namespace ErrorCodes { +namespace ErrorCodes +{ + extern const int ILLEGAL_TYPE_OF_ARGUMENT; extern const int TOO_MANY_ARGUMENTS_FOR_FUNCTION; extern const int TOO_FEW_ARGUMENTS_FOR_FUNCTION; } @@ -48,7 +50,8 @@ public: { throw Exception("Too many arguments", ErrorCodes::TOO_MANY_ARGUMENTS_FOR_FUNCTION); } - else if (arguments.empty()) { + else if (arguments.empty()) + { throw Exception("Too few arguments", ErrorCodes::TOO_FEW_ARGUMENTS_FOR_FUNCTION); } else if (arguments.size() == 2 && checkAndGetDataType(arguments[1].get()) == nullptr) @@ -75,13 +78,13 @@ public: bool has_style = arguments.size() > 1; ColumnPtr style; - if (has_style) { + if (has_style) style = arguments[1].column; - } for (size_t i = 0; i < input_rows_count; i++) { - std::stringstream str; + /// FIXME + std::stringstream str; // STYLE_CHECK_ALLOW_STD_STRING_STREAM if (!geo_column_is_const || i == 0) get(parser, container, i); diff --git a/src/Functions/wkt.cpp b/src/Functions/wkt.cpp index e184f7537aa..e56f1e2bf6a 100644 --- a/src/Functions/wkt.cpp +++ b/src/Functions/wkt.cpp @@ -45,7 +45,8 @@ public: for (size_t i = 0; i < input_rows_count; i++) { - std::stringstream str; + /// FIXME + std::stringstream str; // STYLE_CHECK_ALLOW_STD_STRING_STREAM get(parser, container, i); str << boost::geometry::wkt(container); std::string serialized = str.str(); diff --git a/tests/queries/0_stateless/01301_polygons_within.sql b/tests/queries/0_stateless/01301_polygons_within.sql index da76858c95d..11774bf6cc7 100644 --- a/tests/queries/0_stateless/01301_polygons_within.sql +++ b/tests/queries/0_stateless/01301_polygons_within.sql @@ -1,2 +1,2 @@ -select polygonsWithin([[[(0, 0),(0, 3),(1, 2.9),(2, 2.6),(2.6, 2),(2.9, 1),(3, 0),(0, 0)]]], [[[(1, 1),(1, 4),(4, 4),(4, 1),(1, 1)]]]); -select polygonsWithin([[[(2, 2), (2, 3), (3, 3), (3, 2)]]], [[[(1, 1),(1, 4),(4, 4),(4, 1),(1, 1)]]]) +select polygonsWithinCartesian([[[(0, 0),(0, 3),(1, 2.9),(2, 2.6),(2.6, 2),(2.9, 1),(3, 0),(0, 0)]]], [[[(1, 1),(1, 4),(4, 4),(4, 1),(1, 1)]]]); +select polygonsWithinCartesian([[[(2, 2), (2, 3), (3, 3), (3, 2)]]], [[[(1, 1),(1, 4),(4, 4),(4, 1),(1, 1)]]]) diff --git a/tests/queries/0_stateless/01303_polygons_equals.sql b/tests/queries/0_stateless/01303_polygons_equals.sql index fa93deaba03..b3a4d8f12b3 100644 --- a/tests/queries/0_stateless/01303_polygons_equals.sql +++ b/tests/queries/0_stateless/01303_polygons_equals.sql @@ -1,2 +1,2 @@ -select polygonsEquals([[[(0, 0),(0, 3),(1, 2.9),(2, 2.6),(2.6, 2),(2.9, 1),(3, 0),(0, 0)]]], [[[(1, 1),(1, 4),(4, 4),(4, 1),(1, 1)]]]); -select polygonsEquals([[[(1, 1),(1, 4),(4, 4),(4, 1)]]], [[[(1, 1),(1, 4),(4, 4),(4, 1),(1, 1)]]]) \ No newline at end of file +select polygonsEqualsCartesian([[[(0, 0),(0, 3),(1, 2.9),(2, 2.6),(2.6, 2),(2.9, 1),(3, 0),(0, 0)]]], [[[(1, 1),(1, 4),(4, 4),(4, 1),(1, 1)]]]); +select polygonsEqualsCartesian([[[(1, 1),(1, 4),(4, 4),(4, 1)]]], [[[(1, 1),(1, 4),(4, 4),(4, 1),(1, 1)]]]); \ No newline at end of file From 41d006d4931c6b5afec23ff0def103ac45ce4e61 Mon Sep 17 00:00:00 2001 From: Nikita Mikhailov Date: Wed, 20 Jan 2021 01:07:10 +0300 Subject: [PATCH 280/544] try fix stress --- src/Access/SettingsProfileElement.cpp | 1 + src/Common/remapExecutable.cpp | 1 + src/Functions/FunctionHelpers.cpp | 1 + src/Functions/geometryConverters.h | 21 +++++++++++++++------ src/IO/BufferWithOwnMemory.h | 1 + src/IO/readDecimalText.h | 1 + src/Storages/StorageFactory.cpp | 1 + 7 files changed, 21 insertions(+), 6 deletions(-) diff --git a/src/Access/SettingsProfileElement.cpp b/src/Access/SettingsProfileElement.cpp index 1c682900b00..a7f6a505b7b 100644 --- a/src/Access/SettingsProfileElement.cpp +++ b/src/Access/SettingsProfileElement.cpp @@ -28,6 +28,7 @@ void SettingsProfileElement::init(const ASTSettingsProfileElement & ast, const A if (id_mode) return parse(name_); assert(manager); + /// NOLINTNEXTLINE(clang-analyzer-core.CallAndMessage) return manager->getID(name_); }; diff --git a/src/Common/remapExecutable.cpp b/src/Common/remapExecutable.cpp index 5418290b24f..f6cf461790d 100644 --- a/src/Common/remapExecutable.cpp +++ b/src/Common/remapExecutable.cpp @@ -26,6 +26,7 @@ namespace ErrorCodes namespace { +// NOLINTNEXTLINE(cert-dcl50-cpp) __attribute__((__noinline__)) int64_t our_syscall(...) { __asm__ __volatile__ (R"( diff --git a/src/Functions/FunctionHelpers.cpp b/src/Functions/FunctionHelpers.cpp index 17c28ee3343..e4ecf33e00f 100644 --- a/src/Functions/FunctionHelpers.cpp +++ b/src/Functions/FunctionHelpers.cpp @@ -220,6 +220,7 @@ checkAndGetNestedArrayOffset(const IColumn ** columns, size_t num_arguments) else if (*offsets_i != *offsets) throw Exception("Lengths of all arrays passed to aggregate function must be equal.", ErrorCodes::SIZES_OF_ARRAYS_DOESNT_MATCH); } + /// NOLINTNEXTLINE(clang-analyzer-core.CallAndMessage) return {nested_columns, offsets->data()}; } diff --git a/src/Functions/geometryConverters.h b/src/Functions/geometryConverters.h index 2dd0ce117b1..8e76977e1c0 100644 --- a/src/Functions/geometryConverters.h +++ b/src/Functions/geometryConverters.h @@ -63,7 +63,7 @@ class PointFromColumnParser public: PointFromColumnParser(ColumnPtr col_) : col(col_) { - const auto & tuple = static_cast(*col_); + const auto & tuple = dynamic_cast(*col_); const auto & tuple_columns = tuple.getColumns(); #ifndef NDEBUG @@ -116,7 +116,7 @@ public: } private: - /// Note, this is needed to prevent use-after-free. + /// To prevent use-after-free and increase column lifetime. ColumnPtr col; #ifndef NDEBUG size_t size; @@ -130,8 +130,9 @@ class RingFromColumnParser { public: RingFromColumnParser(ColumnPtr col_) - : offsets(static_cast(*col_).getOffsets()) - , point_parser(static_cast(*col_).getDataPtr()) + : col(col_) + , offsets(dynamic_cast(*col_).getOffsets()) + , point_parser(dynamic_cast(*col_).getDataPtr()) { } @@ -165,6 +166,8 @@ public: } private: + /// To prevent use-after-free and increase column lifetime. + ColumnPtr col; const IColumn::Offsets & offsets; const PointFromColumnParser point_parser; }; @@ -174,7 +177,8 @@ class PolygonFromColumnParser { public: PolygonFromColumnParser(ColumnPtr col_) - : offsets(static_cast(*col_).getOffsets()) + : col(col_) + , offsets(static_cast(*col_).getOffsets()) , ring_parser(static_cast(*col_).getDataPtr()) {} @@ -203,6 +207,8 @@ public: } private: + /// To prevent use-after-free and increase column lifetime. + ColumnPtr col; const IColumn::Offsets & offsets; const RingFromColumnParser ring_parser; }; @@ -212,7 +218,8 @@ class MultiPolygonFromColumnParser { public: MultiPolygonFromColumnParser(ColumnPtr col_) - : offsets(static_cast(*col_).getOffsets()) + : col(col_) + , offsets(static_cast(*col_).getOffsets()) , polygon_parser(static_cast(*col_).getDataPtr()) {} @@ -235,6 +242,8 @@ public: } private: + /// To prevent use-after-free and increase column lifetime. + ColumnPtr col; const IColumn::Offsets & offsets; const PolygonFromColumnParser polygon_parser; }; diff --git a/src/IO/BufferWithOwnMemory.h b/src/IO/BufferWithOwnMemory.h index f8cc8b7febb..cbec65c426f 100644 --- a/src/IO/BufferWithOwnMemory.h +++ b/src/IO/BufferWithOwnMemory.h @@ -65,6 +65,7 @@ struct Memory : boost::noncopyable, Allocator size_t size() const { return m_size; } const char & operator[](size_t i) const { return m_data[i]; } + /// NOLINTNEXTLINE(clang-analyzer-core.uninitialized.UndefReturn) char & operator[](size_t i) { return m_data[i]; } const char * data() const { return m_data; } char * data() { return m_data; } diff --git a/src/IO/readDecimalText.h b/src/IO/readDecimalText.h index 203d8e3963b..b90ca430cc7 100644 --- a/src/IO/readDecimalText.h +++ b/src/IO/readDecimalText.h @@ -181,6 +181,7 @@ inline void readDecimalText(ReadBuffer & buf, T & x, uint32_t precision, uint32_ /// Too many digits after point. Just cut off excessive digits. auto divisor = intExp10OfSize(divisor_exp); assert(divisor > 0); /// This is for Clang Static Analyzer. It is not smart enough to infer it automatically. + /// NOLINTNEXTLINE(clang-analyzer-core.DivideZero) x.value /= divisor; scale = 0; return; diff --git a/src/Storages/StorageFactory.cpp b/src/Storages/StorageFactory.cpp index 85f3bea9e0c..0b26e3ece6c 100644 --- a/src/Storages/StorageFactory.cpp +++ b/src/Storages/StorageFactory.cpp @@ -185,6 +185,7 @@ StoragePtr StorageFactory::get( { /// Storage creator modified empty arguments list, so we should modify the query assert(storage_def && storage_def->engine && !storage_def->engine->arguments); + /// NOLINTNEXTLINE(clang-analyzer-core.NullDereference) storage_def->engine->arguments = std::make_shared(); storage_def->engine->children.push_back(storage_def->engine->arguments); storage_def->engine->arguments->children = empty_engine_args; From be0ab21bb12b103071c415019044bf4f429f8270 Mon Sep 17 00:00:00 2001 From: Nikita Mikhailov Date: Wed, 20 Jan 2021 01:31:33 +0300 Subject: [PATCH 281/544] better --- src/Functions/PolygonsUnion.cpp | 1 + src/Functions/formatString.h | 1 + src/Storages/System/StorageSystemQuotaUsage.cpp | 1 + 3 files changed, 3 insertions(+) diff --git a/src/Functions/PolygonsUnion.cpp b/src/Functions/PolygonsUnion.cpp index 9d45269d0cb..94a1dc9ca0a 100644 --- a/src/Functions/PolygonsUnion.cpp +++ b/src/Functions/PolygonsUnion.cpp @@ -68,6 +68,7 @@ public: get(second_parser, second_container, i); Geometry polygons_union = CartesianMultiPolygon({{{{}}}}); + /// NOLINTNEXTLINE boost::geometry::union_( boost::get(first_container), boost::get(second_container), diff --git a/src/Functions/formatString.h b/src/Functions/formatString.h index bea4fa2e1bc..00da5765309 100644 --- a/src/Functions/formatString.h +++ b/src/Functions/formatString.h @@ -248,6 +248,7 @@ struct FormatImpl /// Strings without null termination. for (size_t i = 1; i < substrings.size(); ++i) { + /// NOLINTNEXTLINE(clang-analyzer-core.CallAndMessage) final_size += data[index_positions[i - 1]]->size(); /// Fixed strings do not have zero terminating character. if (offsets[index_positions[i - 1]]) diff --git a/src/Storages/System/StorageSystemQuotaUsage.cpp b/src/Storages/System/StorageSystemQuotaUsage.cpp index 6d6e22e7be6..06555371551 100644 --- a/src/Storages/System/StorageSystemQuotaUsage.cpp +++ b/src/Storages/System/StorageSystemQuotaUsage.cpp @@ -171,6 +171,7 @@ void StorageSystemQuotaUsage::fillDataImpl( for (auto resource_type : ext::range(Quota::MAX_RESOURCE_TYPE)) { const auto & type_info = ResourceTypeInfo::get(resource_type); + /// NOLINTNEXTLINE(clang-analyzer-core.CallAndMessage) addValue(*column_max[resource_type], *column_max_null_map[resource_type], interval->max[resource_type], type_info); addValue(*column_usage[resource_type], *column_usage_null_map[resource_type], interval->used[resource_type], type_info); } From 10d22e9db127306dc48e941f1d871e3d59c7830b Mon Sep 17 00:00:00 2001 From: Nikita Mikhailov Date: Wed, 20 Jan 2021 14:22:43 +0300 Subject: [PATCH 282/544] fix empty polygon --- src/Functions/array/arrayUniq.cpp | 1 + src/Functions/geometryConverters.h | 15 +++++++++------ tests/queries/0_stateless/01308_polygon_area.sql | 3 ++- 3 files changed, 12 insertions(+), 7 deletions(-) diff --git a/src/Functions/array/arrayUniq.cpp b/src/Functions/array/arrayUniq.cpp index 70246872f6b..10d35416cbf 100644 --- a/src/Functions/array/arrayUniq.cpp +++ b/src/Functions/array/arrayUniq.cpp @@ -172,6 +172,7 @@ ColumnPtr FunctionArrayUniq::executeImpl(const ColumnsWithTypeAndName & argument auto res = ColumnUInt32::create(); ColumnUInt32::Container & res_values = res->getData(); + /// NOLINTNEXTLINE(clang-analyzer-core.CallAndMessage) res_values.resize(offsets->size()); if (num_arguments == 1) diff --git a/src/Functions/geometryConverters.h b/src/Functions/geometryConverters.h index 8e76977e1c0..96458bc7d3f 100644 --- a/src/Functions/geometryConverters.h +++ b/src/Functions/geometryConverters.h @@ -148,15 +148,18 @@ public: void get(Ring & container, size_t i) const { - size_t l = offsets[i - 1]; - size_t r = offsets[i]; + size_t left = i == 0 ? 0 : offsets[i - 1]; + size_t right = offsets[i]; + + if (left == right) + throw Exception("Empty polygons are not allowed in line " + toString(i), ErrorCodes::BAD_ARGUMENTS); // reserve extra point for case when polygon is open - container.reserve(r - l + 1); - container.resize(r - l); + container.reserve(right - left + 1); + container.resize(right - left); - for (size_t j = l; j < r; j++) - point_parser.get(container[j - l], j); + for (size_t j = left; j < right; j++) + point_parser.get(container[j - left], j); // make ring closed if (!boost::geometry::equals(container[0], container.back())) diff --git a/tests/queries/0_stateless/01308_polygon_area.sql b/tests/queries/0_stateless/01308_polygon_area.sql index c4fba155f37..700cf661b48 100644 --- a/tests/queries/0_stateless/01308_polygon_area.sql +++ b/tests/queries/0_stateless/01308_polygon_area.sql @@ -1,2 +1,3 @@ select polygonAreaCartesian([[[(0, 0), (0., 5), (5, 5), (5., 0)]]]); -select polygonAreaGeographic([[[(4.346693, 50.858306), (4.367945, 50.852455), (4.366227, 50.840809), (4.344961, 50.833264), (4.338074, 50.848677), (4.346693, 50.858306)]]]); \ No newline at end of file +select polygonAreaGeographic([[[(4.346693, 50.858306), (4.367945, 50.852455), (4.366227, 50.840809), (4.344961, 50.833264), (4.338074, 50.848677), (4.346693, 50.858306)]]]); +SELECT polygonAreaCartesian([]); -- { serverError 36 } \ No newline at end of file From a596227a0272d4163f006a8ce82a500c3d789bfc Mon Sep 17 00:00:00 2001 From: Nikita Mikhailov Date: Thu, 21 Jan 2021 02:12:47 +0300 Subject: [PATCH 283/544] tidy fix --- src/Functions/PolygonsUnion.cpp | 1 + src/Functions/polygonsIntersection.cpp | 7 +++---- src/Functions/polygonsSymDifference.cpp | 2 ++ src/Functions/polygonsWithin.cpp | 1 + src/Functions/readWkt.cpp | 4 ++-- src/Functions/svg.cpp | 2 +- src/Functions/wkt.cpp | 2 +- 7 files changed, 11 insertions(+), 8 deletions(-) diff --git a/src/Functions/PolygonsUnion.cpp b/src/Functions/PolygonsUnion.cpp index 94a1dc9ca0a..828ef6ebf42 100644 --- a/src/Functions/PolygonsUnion.cpp +++ b/src/Functions/PolygonsUnion.cpp @@ -62,6 +62,7 @@ public: MultiPolygonSerializer serializer; + /// NOLINTNEXTLINE(clang-analyzer-core.uninitialized.Assign) for (size_t i = 0; i < input_rows_count; i++) { get(first_parser, first_container, i); diff --git a/src/Functions/polygonsIntersection.cpp b/src/Functions/polygonsIntersection.cpp index 335b40322f7..151c82b2af3 100644 --- a/src/Functions/polygonsIntersection.cpp +++ b/src/Functions/polygonsIntersection.cpp @@ -61,12 +61,11 @@ public: MultiPolygonSerializer serializer; + /// NOLINTNEXTLINE(clang-analyzer-core.uninitialized.Assign) for (size_t i = 0; i < input_rows_count; i++) { - if (i == 0) - get(first_parser, first_container, i); - if (i == 0) - get(second_parser, second_container, i); + get(first_parser, first_container, i); + get(second_parser, second_container, i); CartesianGeometry intersection = CartesianMultiPolygon({{{{}}}}); boost::geometry::intersection( diff --git a/src/Functions/polygonsSymDifference.cpp b/src/Functions/polygonsSymDifference.cpp index 6d42e001dc8..30c4d78e19a 100644 --- a/src/Functions/polygonsSymDifference.cpp +++ b/src/Functions/polygonsSymDifference.cpp @@ -63,12 +63,14 @@ public: MultiPolygonSerializer serializer; + /// NOLINTNEXTLINE(clang-analyzer-core.uninitialized.Assign) for (size_t i = 0; i < input_rows_count; i++) { get(first_parser, first_container, i); get(second_parser, second_container, i); Geometry sym_difference = MultiPolygon({{{{}}}}); + boost::geometry::sym_difference( boost::get>(first_container), boost::get>(second_container), diff --git a/src/Functions/polygonsWithin.cpp b/src/Functions/polygonsWithin.cpp index 3a2c5704083..52fbcb4aaba 100644 --- a/src/Functions/polygonsWithin.cpp +++ b/src/Functions/polygonsWithin.cpp @@ -65,6 +65,7 @@ public: auto res_column = ColumnUInt8::create(); + /// NOLINTNEXTLINE(clang-analyzer-core.uninitialized.Assign) for (size_t i = 0; i < input_rows_count; i++) { get(first_parser, first_container, i); diff --git a/src/Functions/readWkt.cpp b/src/Functions/readWkt.cpp index 12007b8724b..9ef8af745db 100644 --- a/src/Functions/readWkt.cpp +++ b/src/Functions/readWkt.cpp @@ -20,7 +20,7 @@ template class FunctionReadWkt : public IFunction { public: - explicit FunctionReadWkt() {} + explicit FunctionReadWkt() = default; size_t getNumberOfArguments() const override { @@ -40,7 +40,7 @@ public: ColumnPtr executeImpl(const ColumnsWithTypeAndName & arguments, const DataTypePtr & /*result_type*/, size_t input_rows_count) const override { - auto column_string = checkAndGetColumn(arguments[0].column.get()); + const auto * column_string = checkAndGetColumn(arguments[0].column.get()); Serializer serializer; Geometry geometry; diff --git a/src/Functions/svg.cpp b/src/Functions/svg.cpp index 3d8206cc2e3..d240c6d713b 100644 --- a/src/Functions/svg.cpp +++ b/src/Functions/svg.cpp @@ -22,7 +22,7 @@ class FunctionSvg : public IFunction public: static inline const char * name = "svg"; - explicit FunctionSvg() {} + explicit FunctionSvg() = default; static FunctionPtr create(const Context &) { diff --git a/src/Functions/wkt.cpp b/src/Functions/wkt.cpp index e56f1e2bf6a..6d78fa6ca81 100644 --- a/src/Functions/wkt.cpp +++ b/src/Functions/wkt.cpp @@ -14,7 +14,7 @@ class FunctionWkt : public IFunction public: static inline const char * name = "wkt"; - explicit FunctionWkt() {} + explicit FunctionWkt() = default; static FunctionPtr create(const Context &) { From adfb2885c9ea5d319ac7ee3fe8908b5630ae291e Mon Sep 17 00:00:00 2001 From: Nikita Mikhailov Date: Thu, 21 Jan 2021 02:25:31 +0300 Subject: [PATCH 284/544] better --- src/Access/SettingsProfileElement.cpp | 1 - src/Common/remapExecutable.cpp | 1 - src/Functions/FunctionHelpers.cpp | 1 - src/Functions/array/arrayUniq.cpp | 1 - src/Functions/formatString.h | 1 - src/IO/readDecimalText.h | 1 - src/Storages/StorageFactory.cpp | 1 - src/Storages/System/StorageSystemQuotaUsage.cpp | 1 - 8 files changed, 8 deletions(-) diff --git a/src/Access/SettingsProfileElement.cpp b/src/Access/SettingsProfileElement.cpp index a7f6a505b7b..1c682900b00 100644 --- a/src/Access/SettingsProfileElement.cpp +++ b/src/Access/SettingsProfileElement.cpp @@ -28,7 +28,6 @@ void SettingsProfileElement::init(const ASTSettingsProfileElement & ast, const A if (id_mode) return parse(name_); assert(manager); - /// NOLINTNEXTLINE(clang-analyzer-core.CallAndMessage) return manager->getID(name_); }; diff --git a/src/Common/remapExecutable.cpp b/src/Common/remapExecutable.cpp index f6cf461790d..5418290b24f 100644 --- a/src/Common/remapExecutable.cpp +++ b/src/Common/remapExecutable.cpp @@ -26,7 +26,6 @@ namespace ErrorCodes namespace { -// NOLINTNEXTLINE(cert-dcl50-cpp) __attribute__((__noinline__)) int64_t our_syscall(...) { __asm__ __volatile__ (R"( diff --git a/src/Functions/FunctionHelpers.cpp b/src/Functions/FunctionHelpers.cpp index e4ecf33e00f..17c28ee3343 100644 --- a/src/Functions/FunctionHelpers.cpp +++ b/src/Functions/FunctionHelpers.cpp @@ -220,7 +220,6 @@ checkAndGetNestedArrayOffset(const IColumn ** columns, size_t num_arguments) else if (*offsets_i != *offsets) throw Exception("Lengths of all arrays passed to aggregate function must be equal.", ErrorCodes::SIZES_OF_ARRAYS_DOESNT_MATCH); } - /// NOLINTNEXTLINE(clang-analyzer-core.CallAndMessage) return {nested_columns, offsets->data()}; } diff --git a/src/Functions/array/arrayUniq.cpp b/src/Functions/array/arrayUniq.cpp index 10d35416cbf..70246872f6b 100644 --- a/src/Functions/array/arrayUniq.cpp +++ b/src/Functions/array/arrayUniq.cpp @@ -172,7 +172,6 @@ ColumnPtr FunctionArrayUniq::executeImpl(const ColumnsWithTypeAndName & argument auto res = ColumnUInt32::create(); ColumnUInt32::Container & res_values = res->getData(); - /// NOLINTNEXTLINE(clang-analyzer-core.CallAndMessage) res_values.resize(offsets->size()); if (num_arguments == 1) diff --git a/src/Functions/formatString.h b/src/Functions/formatString.h index 00da5765309..bea4fa2e1bc 100644 --- a/src/Functions/formatString.h +++ b/src/Functions/formatString.h @@ -248,7 +248,6 @@ struct FormatImpl /// Strings without null termination. for (size_t i = 1; i < substrings.size(); ++i) { - /// NOLINTNEXTLINE(clang-analyzer-core.CallAndMessage) final_size += data[index_positions[i - 1]]->size(); /// Fixed strings do not have zero terminating character. if (offsets[index_positions[i - 1]]) diff --git a/src/IO/readDecimalText.h b/src/IO/readDecimalText.h index b90ca430cc7..203d8e3963b 100644 --- a/src/IO/readDecimalText.h +++ b/src/IO/readDecimalText.h @@ -181,7 +181,6 @@ inline void readDecimalText(ReadBuffer & buf, T & x, uint32_t precision, uint32_ /// Too many digits after point. Just cut off excessive digits. auto divisor = intExp10OfSize(divisor_exp); assert(divisor > 0); /// This is for Clang Static Analyzer. It is not smart enough to infer it automatically. - /// NOLINTNEXTLINE(clang-analyzer-core.DivideZero) x.value /= divisor; scale = 0; return; diff --git a/src/Storages/StorageFactory.cpp b/src/Storages/StorageFactory.cpp index 0b26e3ece6c..85f3bea9e0c 100644 --- a/src/Storages/StorageFactory.cpp +++ b/src/Storages/StorageFactory.cpp @@ -185,7 +185,6 @@ StoragePtr StorageFactory::get( { /// Storage creator modified empty arguments list, so we should modify the query assert(storage_def && storage_def->engine && !storage_def->engine->arguments); - /// NOLINTNEXTLINE(clang-analyzer-core.NullDereference) storage_def->engine->arguments = std::make_shared(); storage_def->engine->children.push_back(storage_def->engine->arguments); storage_def->engine->arguments->children = empty_engine_args; diff --git a/src/Storages/System/StorageSystemQuotaUsage.cpp b/src/Storages/System/StorageSystemQuotaUsage.cpp index 06555371551..6d6e22e7be6 100644 --- a/src/Storages/System/StorageSystemQuotaUsage.cpp +++ b/src/Storages/System/StorageSystemQuotaUsage.cpp @@ -171,7 +171,6 @@ void StorageSystemQuotaUsage::fillDataImpl( for (auto resource_type : ext::range(Quota::MAX_RESOURCE_TYPE)) { const auto & type_info = ResourceTypeInfo::get(resource_type); - /// NOLINTNEXTLINE(clang-analyzer-core.CallAndMessage) addValue(*column_max[resource_type], *column_max_null_map[resource_type], interval->max[resource_type], type_info); addValue(*column_usage[resource_type], *column_usage_null_map[resource_type], interval->used[resource_type], type_info); } From 6a51ad425ea9a4cdb4fea81cc262798f66695b33 Mon Sep 17 00:00:00 2001 From: Nikita Mikhailov Date: Thu, 21 Jan 2021 15:31:47 +0300 Subject: [PATCH 285/544] add tests + improvements --- src/Functions/polygonConvexHull.cpp | 12 ++---- src/Functions/polygonsDistance.cpp | 10 +++-- src/Functions/polygonsEquals.cpp | 10 +++-- src/Functions/polygonsIntersection.cpp | 39 +++++++++++------ src/Functions/polygonsSymDifference.cpp | 16 +++---- .../{PolygonsUnion.cpp => polygonsUnion.cpp} | 42 ++++++++++++------- src/Functions/polygonsWithin.cpp | 18 +++++--- src/Functions/svg.cpp | 12 +----- src/Functions/wkt.cpp | 1 - src/Functions/ya.make | 2 +- src/IO/BufferWithOwnMemory.h | 1 - .../01301_polygons_within.reference | 2 + .../0_stateless/01301_polygons_within.sql | 7 +++- .../01305_polygons_union.reference | 1 + .../0_stateless/01305_polygons_union.sql | 4 +- .../01306_polygons_intersection.reference | 2 + .../01306_polygons_intersection.sql | 7 +++- 17 files changed, 116 insertions(+), 70 deletions(-) rename src/Functions/{PolygonsUnion.cpp => polygonsUnion.cpp} (57%) diff --git a/src/Functions/polygonConvexHull.cpp b/src/Functions/polygonConvexHull.cpp index 1cb46b81c32..21002c81612 100644 --- a/src/Functions/polygonConvexHull.cpp +++ b/src/Functions/polygonConvexHull.cpp @@ -64,13 +64,13 @@ public: { get(parser, container, i); - Geometry convex_hull = Polygon({{{}}}); + auto convex_hull = Polygon({{{}}}); + boost::geometry::convex_hull( boost::get>(container), - boost::get>(convex_hull)); + convex_hull); - boost::get>(convex_hull).outer().erase( - boost::get>(convex_hull).outer().begin()); + convex_hull.outer().erase(convex_hull.outer().begin()); serializer.add(convex_hull); } @@ -88,14 +88,10 @@ public: template <> const char * FunctionPolygonConvexHull::name = "polygonConvexHullCartesian"; -// template <> -// const char * FunctionPolygonConvexHull::name = "polygonConvexHullGeographic"; - void registerFunctionPolygonConvexHull(FunctionFactory & factory) { factory.registerFunction>(); - // factory.registerFunction>(); } } diff --git a/src/Functions/polygonsDistance.cpp b/src/Functions/polygonsDistance.cpp index 67f6482ec22..c1530867f31 100644 --- a/src/Functions/polygonsDistance.cpp +++ b/src/Functions/polygonsDistance.cpp @@ -70,9 +70,13 @@ public: get(first_parser, first_container, i); get(second_parser, second_container, i); - Float64 distance = boost::geometry::distance( - boost::get>(first_container), - boost::get>(second_container)); + auto first = boost::get>(first_container); + auto second = boost::get>(second_container); + + boost::geometry::correct(first); + boost::geometry::correct(second); + + Float64 distance = boost::geometry::distance(first, second); res_column->insertValue(distance); } diff --git a/src/Functions/polygonsEquals.cpp b/src/Functions/polygonsEquals.cpp index 118c5885cdd..10d92eef09b 100644 --- a/src/Functions/polygonsEquals.cpp +++ b/src/Functions/polygonsEquals.cpp @@ -70,9 +70,13 @@ public: get(first_parser, first_container, i); get(second_parser, second_container, i); - bool equals = boost::geometry::equals( - boost::get>(first_container), - boost::get>(second_container)); + auto first = boost::get>(first_container); + auto second = boost::get>(second_container); + + boost::geometry::correct(first); + boost::geometry::correct(second); + + bool equals = boost::geometry::equals(first, second); res_column->insertValue(equals); } diff --git a/src/Functions/polygonsIntersection.cpp b/src/Functions/polygonsIntersection.cpp index 151c82b2af3..cff28860115 100644 --- a/src/Functions/polygonsIntersection.cpp +++ b/src/Functions/polygonsIntersection.cpp @@ -19,10 +19,12 @@ namespace DB { + +template class FunctionPolygonsIntersection : public IFunction { public: - static inline const char * name = "polygonsIntersection"; + static inline const char * name; explicit FunctionPolygonsIntersection() = default; @@ -53,28 +55,33 @@ public: ColumnPtr executeImpl(const ColumnsWithTypeAndName & arguments, const DataTypePtr & /*result_type*/, size_t input_rows_count) const override { - auto first_parser = makeGeometryFromColumnParser(arguments[0]); + auto first_parser = makeGeometryFromColumnParser(arguments[0]); auto first_container = createContainer(first_parser); - auto second_parser = makeGeometryFromColumnParser(arguments[1]); + auto second_parser = makeGeometryFromColumnParser(arguments[1]); auto second_container = createContainer(second_parser); - MultiPolygonSerializer serializer; + MultiPolygonSerializer serializer; + /// We are not interested in some pitfalls in third-party libraries /// NOLINTNEXTLINE(clang-analyzer-core.uninitialized.Assign) for (size_t i = 0; i < input_rows_count; i++) { get(first_parser, first_container, i); get(second_parser, second_container, i); - CartesianGeometry intersection = CartesianMultiPolygon({{{{}}}}); - boost::geometry::intersection( - boost::get(first_container), - boost::get(second_container), - boost::get(intersection)); + auto intersection = MultiPolygon({{{{}}}}); + auto first = boost::get>(first_container); + auto second = boost::get>(second_container); - boost::get(intersection).erase( - boost::get(intersection).begin()); + /// Orient the polygons correctly. + boost::geometry::correct(first); + boost::geometry::correct(second); + + /// Main work here. + boost::geometry::intersection(first, second,intersection); + + intersection.erase(intersection.begin()); serializer.add(intersection); } @@ -89,9 +96,17 @@ public: }; +template <> +const char * FunctionPolygonsIntersection::name = "polygonsIntersectionCartesian"; + +template <> +const char * FunctionPolygonsIntersection::name = "polygonsIntersectionGeographic"; + + void registerFunctionPolygonsIntersection(FunctionFactory & factory) { - factory.registerFunction(); + factory.registerFunction>(); + factory.registerFunction>(); } } diff --git a/src/Functions/polygonsSymDifference.cpp b/src/Functions/polygonsSymDifference.cpp index 30c4d78e19a..38fc43f56ca 100644 --- a/src/Functions/polygonsSymDifference.cpp +++ b/src/Functions/polygonsSymDifference.cpp @@ -69,15 +69,17 @@ public: get(first_parser, first_container, i); get(second_parser, second_container, i); - Geometry sym_difference = MultiPolygon({{{{}}}}); + auto sym_difference = MultiPolygon({{{{}}}}); - boost::geometry::sym_difference( - boost::get>(first_container), - boost::get>(second_container), - boost::get>(sym_difference)); + auto first = boost::get>(first_container); + auto second = boost::get>(second_container); - boost::get>(sym_difference).erase( - boost::get>(sym_difference).begin()); + boost::geometry::correct(first); + boost::geometry::correct(second); + + boost::geometry::sym_difference(first, second, sym_difference); + + sym_difference.erase(sym_difference.begin()); serializer.add(sym_difference); } diff --git a/src/Functions/PolygonsUnion.cpp b/src/Functions/polygonsUnion.cpp similarity index 57% rename from src/Functions/PolygonsUnion.cpp rename to src/Functions/polygonsUnion.cpp index 828ef6ebf42..44435461c0c 100644 --- a/src/Functions/PolygonsUnion.cpp +++ b/src/Functions/polygonsUnion.cpp @@ -20,10 +20,11 @@ namespace DB { +template class FunctionPolygonsUnion : public IFunction { public: - static inline const char * name = "polygonsUnion"; + static inline const char * name; explicit FunctionPolygonsUnion() = default; @@ -54,29 +55,33 @@ public: ColumnPtr executeImpl(const ColumnsWithTypeAndName & arguments, const DataTypePtr & /*result_type*/, size_t input_rows_count) const override { - auto first_parser = makeGeometryFromColumnParser(arguments[0]); + auto first_parser = makeGeometryFromColumnParser(arguments[0]); auto first_container = createContainer(first_parser); - auto second_parser = makeGeometryFromColumnParser(arguments[1]); + auto second_parser = makeGeometryFromColumnParser(arguments[1]); auto second_container = createContainer(second_parser); - MultiPolygonSerializer serializer; + MultiPolygonSerializer serializer; + /// We are not interested in some pitfalls in third-party libraries /// NOLINTNEXTLINE(clang-analyzer-core.uninitialized.Assign) for (size_t i = 0; i < input_rows_count; i++) { - get(first_parser, first_container, i); - get(second_parser, second_container, i); + get(first_parser, first_container, i); + get(second_parser, second_container, i); - Geometry polygons_union = CartesianMultiPolygon({{{{}}}}); - /// NOLINTNEXTLINE - boost::geometry::union_( - boost::get(first_container), - boost::get(second_container), - boost::get(polygons_union)); + auto first = boost::get>(first_container); + auto second = boost::get>(second_container); + auto polygons_union = MultiPolygon({{{{}}}}); - boost::get(polygons_union).erase( - boost::get(polygons_union).begin()); + /// Orient the polygons correctly. + boost::geometry::correct(first); + boost::geometry::correct(second); + + /// Main work here. + boost::geometry::union_(first, second, polygons_union); + + polygons_union.erase(polygons_union.begin()); serializer.add(polygons_union); } @@ -90,10 +95,17 @@ public: } }; +template <> +const char * FunctionPolygonsUnion::name = "polygonsUnionCartesian"; + +template <> +const char * FunctionPolygonsUnion::name = "polygonsUnionGeographic"; + void registerFunctionPolygonsUnion(FunctionFactory & factory) { - factory.registerFunction(); + factory.registerFunction>(); + factory.registerFunction>(); } } diff --git a/src/Functions/polygonsWithin.cpp b/src/Functions/polygonsWithin.cpp index 52fbcb4aaba..8caa16083f1 100644 --- a/src/Functions/polygonsWithin.cpp +++ b/src/Functions/polygonsWithin.cpp @@ -68,12 +68,16 @@ public: /// NOLINTNEXTLINE(clang-analyzer-core.uninitialized.Assign) for (size_t i = 0; i < input_rows_count; i++) { - get(first_parser, first_container, i); - get(second_parser, second_container, i); + get(first_parser, first_container, i); + get(second_parser, second_container, i); - bool within = boost::geometry::within( - boost::get>(first_container), - boost::get>(second_container)); + auto first = boost::get>(first_container); + auto second = boost::get>(second_container); + + boost::geometry::correct(first); + boost::geometry::correct(second); + + bool within = boost::geometry::within(first, second); res_column->insertValue(within); } @@ -91,10 +95,14 @@ public: template <> const char * FunctionPolygonsWithin::name = "polygonsWithinCartesian"; +template <> +const char * FunctionPolygonsWithin::name = "polygonsWithinGeographic"; + void registerFunctionPolygonsWithin(FunctionFactory & factory) { factory.registerFunction>(); + factory.registerFunction>(); } } diff --git a/src/Functions/svg.cpp b/src/Functions/svg.cpp index d240c6d713b..f09504359fa 100644 --- a/src/Functions/svg.cpp +++ b/src/Functions/svg.cpp @@ -65,13 +65,7 @@ public: ColumnPtr executeImpl(const ColumnsWithTypeAndName & arguments, const DataTypePtr & /*result_type*/, size_t input_rows_count) const override { - const auto * const_col = checkAndGetColumn(arguments[0].column.get()); - - auto parser = const_col ? - makeGeometryFromColumnParser(ColumnWithTypeAndName(const_col->getDataColumnPtr(), arguments[0].type, arguments[0].name)) : - makeGeometryFromColumnParser(arguments[0]); - - bool geo_column_is_const = static_cast(const_col); + auto parser = makeGeometryFromColumnParser(arguments[0]); auto res_column = ColumnString::create(); auto container = createContainer(parser); @@ -83,10 +77,8 @@ public: for (size_t i = 0; i < input_rows_count; i++) { - /// FIXME std::stringstream str; // STYLE_CHECK_ALLOW_STD_STRING_STREAM - if (!geo_column_is_const || i == 0) - get(parser, container, i); + get(parser, container, i); str << boost::geometry::svg(container, has_style ? style->getDataAt(i).toString() : ""); std::string serialized = str.str(); diff --git a/src/Functions/wkt.cpp b/src/Functions/wkt.cpp index 6d78fa6ca81..c79c4e6fb02 100644 --- a/src/Functions/wkt.cpp +++ b/src/Functions/wkt.cpp @@ -45,7 +45,6 @@ public: for (size_t i = 0; i < input_rows_count; i++) { - /// FIXME std::stringstream str; // STYLE_CHECK_ALLOW_STD_STRING_STREAM get(parser, container, i); str << boost::geometry::wkt(container); diff --git a/src/Functions/ya.make b/src/Functions/ya.make index 604d5243be3..e430d72690d 100644 --- a/src/Functions/ya.make +++ b/src/Functions/ya.make @@ -74,7 +74,6 @@ SRCS( GatherUtils/sliceFromRightConstantOffsetUnbounded.cpp GeoHash.cpp IFunction.cpp - PolygonsUnion.cpp TargetSpecific.cpp URL/URLHierarchy.cpp URL/URLPathHierarchy.cpp @@ -386,6 +385,7 @@ SRCS( polygonsEquals.cpp polygonsIntersection.cpp polygonsSymDifference.cpp + polygonsUnion.cpp polygonsWithin.cpp position.cpp positionCaseInsensitive.cpp diff --git a/src/IO/BufferWithOwnMemory.h b/src/IO/BufferWithOwnMemory.h index cbec65c426f..f8cc8b7febb 100644 --- a/src/IO/BufferWithOwnMemory.h +++ b/src/IO/BufferWithOwnMemory.h @@ -65,7 +65,6 @@ struct Memory : boost::noncopyable, Allocator size_t size() const { return m_size; } const char & operator[](size_t i) const { return m_data[i]; } - /// NOLINTNEXTLINE(clang-analyzer-core.uninitialized.UndefReturn) char & operator[](size_t i) { return m_data[i]; } const char * data() const { return m_data; } char * data() { return m_data; } diff --git a/tests/queries/0_stateless/01301_polygons_within.reference b/tests/queries/0_stateless/01301_polygons_within.reference index 0d66ea1aee9..5565ed6787f 100644 --- a/tests/queries/0_stateless/01301_polygons_within.reference +++ b/tests/queries/0_stateless/01301_polygons_within.reference @@ -1,2 +1,4 @@ 0 1 +0 +1 diff --git a/tests/queries/0_stateless/01301_polygons_within.sql b/tests/queries/0_stateless/01301_polygons_within.sql index 11774bf6cc7..f3697803f23 100644 --- a/tests/queries/0_stateless/01301_polygons_within.sql +++ b/tests/queries/0_stateless/01301_polygons_within.sql @@ -1,2 +1,7 @@ select polygonsWithinCartesian([[[(0, 0),(0, 3),(1, 2.9),(2, 2.6),(2.6, 2),(2.9, 1),(3, 0),(0, 0)]]], [[[(1, 1),(1, 4),(4, 4),(4, 1),(1, 1)]]]); -select polygonsWithinCartesian([[[(2, 2), (2, 3), (3, 3), (3, 2)]]], [[[(1, 1),(1, 4),(4, 4),(4, 1),(1, 1)]]]) +select polygonsWithinCartesian([[[(2, 2), (2, 3), (3, 3), (3, 2)]]], [[[(1, 1),(1, 4),(4, 4),(4, 1),(1, 1)]]]); + +select polygonsWithinGeographic([[[(4.3613577, 50.8651821), (4.349556, 50.8535879), (4.3602419, 50.8435626), (4.3830299, 50.8428851), (4.3904543, 50.8564867), (4.3613148, 50.8651279)]]], [[[(4.346693, 50.858306), (4.367945, 50.852455), (4.366227, 50.840809), (4.344961, 50.833264), (4.338074, 50.848677), (4.346693, 50.858306)]]]); +select polygonsWithinGeographic([[[(4.3501568, 50.8518269), (4.3444920, 50.8439961), (4.3565941, 50.8443213), (4.3501568, 50.8518269)]]], [[[(4.3679450, 50.8524550),(4.3466930, 50.8583060),(4.3380740, 50.8486770),(4.3449610, 50.8332640),(4.3662270, 50.8408090),(4.3679450, 50.8524550)]]]); + + diff --git a/tests/queries/0_stateless/01305_polygons_union.reference b/tests/queries/0_stateless/01305_polygons_union.reference index 621f1159e22..64c6ac473e4 100644 --- a/tests/queries/0_stateless/01305_polygons_union.reference +++ b/tests/queries/0_stateless/01305_polygons_union.reference @@ -1 +1,2 @@ [[[(1,2.9),(1,4),(4,4),(4,1),(2.9,1),(3,0),(0,0),(0,3),(1,2.9)]]] +[[[(4.3666052904432435,50.84337386140151),(4.366227,50.840809),(4.344961,50.833264),(4.338074,50.848677),(4.346693,50.858306),(4.3526804582393535,50.856658100365976),(4.3613577,50.8651821),(4.3613148,50.8651279),(4.3904543,50.8564867),(4.3830299,50.8428851),(4.3666052904432435,50.84337386140151)]]] diff --git a/tests/queries/0_stateless/01305_polygons_union.sql b/tests/queries/0_stateless/01305_polygons_union.sql index 35f9edad2ee..42b869e850a 100644 --- a/tests/queries/0_stateless/01305_polygons_union.sql +++ b/tests/queries/0_stateless/01305_polygons_union.sql @@ -1 +1,3 @@ -select polygonsUnion([[[(0, 0),(0, 3),(1, 2.9),(2, 2.6),(2.6, 2),(2.9, 1),(3, 0),(0, 0)]]], [[[(1, 1),(1, 4),(4, 4),(4, 1),(1, 1)]]]) +select polygonsUnionCartesian([[[(0, 0),(0, 3),(1, 2.9),(2, 2.6),(2.6, 2),(2.9, 1),(3, 0),(0, 0)]]], [[[(1, 1),(1, 4),(4, 4),(4, 1),(1, 1)]]]); + +select polygonsUnionGeographic([[[(4.3613577, 50.8651821), (4.349556, 50.8535879), (4.3602419, 50.8435626), (4.3830299, 50.8428851), (4.3904543, 50.8564867), (4.3613148, 50.8651279)]]], [[[(4.346693, 50.858306), (4.367945, 50.852455), (4.366227, 50.840809), (4.344961, 50.833264), (4.338074, 50.848677), (4.346693, 50.858306)]]]); diff --git a/tests/queries/0_stateless/01306_polygons_intersection.reference b/tests/queries/0_stateless/01306_polygons_intersection.reference index 14917a34b57..3ac2647e7f5 100644 --- a/tests/queries/0_stateless/01306_polygons_intersection.reference +++ b/tests/queries/0_stateless/01306_polygons_intersection.reference @@ -1,2 +1,4 @@ [[[(1,2.9),(2,2.6),(2.6,2),(2.9,1),(1,1),(1,2.9)]]] [] +[] +[[[(4.3666052904432435,50.84337386140151),(4.3602419,50.8435626),(4.349556,50.8535879),(4.3526804582393535,50.856658100365976),(4.367945,50.852455),(4.3666052904432435,50.84337386140151)]]] diff --git a/tests/queries/0_stateless/01306_polygons_intersection.sql b/tests/queries/0_stateless/01306_polygons_intersection.sql index 6e9921f9476..0901f20fee5 100644 --- a/tests/queries/0_stateless/01306_polygons_intersection.sql +++ b/tests/queries/0_stateless/01306_polygons_intersection.sql @@ -1,2 +1,5 @@ -select polygonsIntersection([[[(0, 0),(0, 3),(1, 2.9),(2, 2.6),(2.6, 2),(2.9, 1),(3, 0),(0, 0)]]], [[[(1, 1),(1, 4),(4, 4),(4, 1),(1, 1)]]]); -select polygonsIntersection([[[(0, 0),(0, 3),(1, 2.9),(2, 2.6),(2.6, 2),(2.9, 1),(3, 0),(0, 0)]]], [[[(3, 3),(3, 4),(4, 4),(4, 3),(3, 3)]]]); \ No newline at end of file +select polygonsIntersectionCartesian([[[(0, 0),(0, 3),(1, 2.9),(2, 2.6),(2.6, 2),(2.9, 1),(3, 0),(0, 0)]]], [[[(1, 1),(1, 4),(4, 4),(4, 1),(1, 1)]]]); +select polygonsIntersectionCartesian([[[(0, 0),(0, 3),(1, 2.9),(2, 2.6),(2.6, 2),(2.9, 1),(3, 0),(0, 0)]]], [[[(3, 3),(3, 4),(4, 4),(4, 3),(3, 3)]]]); + +select polygonsIntersectionGeographic([[[(4.346693, 50.858306), (4.367945, 50.852455), (4.366227, 50.840809), (4.344961, 50.833264), (4.338074, 50.848677), (4.346693, 50.858306)]]], [[[(25.0010, 136.9987), (17.7500, 142.5000), (11.3733, 142.5917)]]]); +select polygonsIntersectionGeographic([[[(4.3613577, 50.8651821), (4.349556, 50.8535879), (4.3602419, 50.8435626), (4.3830299, 50.8428851), (4.3904543, 50.8564867), (4.3613148, 50.8651279)]]], [[[(4.346693, 50.858306), (4.367945, 50.852455), (4.366227, 50.840809), (4.344961, 50.833264), (4.338074, 50.848677), (4.346693, 50.858306)]]]); \ No newline at end of file From 79b76d428f33cf0b737f8932003e875a40397c5a Mon Sep 17 00:00:00 2001 From: Nikita Mikhailov Date: Thu, 21 Jan 2021 17:14:42 +0300 Subject: [PATCH 286/544] fix --- src/Functions/geometryConverters.cpp | 2 +- src/Functions/geometryConverters.h | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/Functions/geometryConverters.cpp b/src/Functions/geometryConverters.cpp index b8a15ff98d5..def1f3b8fde 100644 --- a/src/Functions/geometryConverters.cpp +++ b/src/Functions/geometryConverters.cpp @@ -66,7 +66,7 @@ Parser makeParser(const ColumnWithTypeAndName & col) { throw Exception("Failed to cast " + col.type->getName() + " to " + wanted_data_type->getName(), ErrorCodes::ILLEGAL_COLUMN); } - return Parser(std::move(casted)); + return Parser(std::move(casted->convertToFullColumnIfConst())); } } diff --git a/src/Functions/geometryConverters.h b/src/Functions/geometryConverters.h index 96458bc7d3f..03a19e0db4a 100644 --- a/src/Functions/geometryConverters.h +++ b/src/Functions/geometryConverters.h @@ -69,10 +69,10 @@ public: #ifndef NDEBUG size = tuple.size(); #endif - const auto & x_data = static_cast(*tuple_columns[0]); + const auto & x_data = dynamic_cast(*tuple_columns[0]); first = x_data.getData().data(); - const auto & y_data = static_cast(*tuple_columns[1]); + const auto & y_data = dynamic_cast(*tuple_columns[1]); second = y_data.getData().data(); } From bf07c098a66f3adc7fd1abf3092bd71d77877ddf Mon Sep 17 00:00:00 2001 From: Nikita Mikhailov Date: Thu, 11 Feb 2021 23:37:05 +0300 Subject: [PATCH 287/544] more checks of input type --- src/Core/NamesAndTypes.h | 2 +- .../DataTypeWithSimpleSerialization.h | 3 +- src/Functions/geometryConverters.h | 28 +++++++++---------- src/Functions/polygonArea.cpp | 25 +++++++++++++++++ src/Functions/polygonConvexHull.cpp | 22 +++++++++++++++ src/Functions/polygonPerimeter.cpp | 22 +++++++++++++++ src/Functions/polygonsDistance.cpp | 24 ++++++++++++++++ src/Functions/polygonsEquals.cpp | 25 +++++++++++++++++ src/Functions/polygonsIntersection.cpp | 27 +++++++++++++++++- src/Functions/polygonsSymDifference.cpp | 25 +++++++++++++++++ src/Functions/polygonsUnion.cpp | 24 ++++++++++++++++ src/Functions/polygonsWithin.cpp | 25 +++++++++++++++++ 12 files changed, 234 insertions(+), 18 deletions(-) diff --git a/src/Core/NamesAndTypes.h b/src/Core/NamesAndTypes.h index dad031a543c..fc86c7f6a1d 100644 --- a/src/Core/NamesAndTypes.h +++ b/src/Core/NamesAndTypes.h @@ -63,7 +63,7 @@ using NamesAndTypes = std::vector; class NamesAndTypesList : public std::list { public: - NamesAndTypesList() {} + NamesAndTypesList() = default; NamesAndTypesList(std::initializer_list init) : std::list(init) {} diff --git a/src/DataTypes/DataTypeWithSimpleSerialization.h b/src/DataTypes/DataTypeWithSimpleSerialization.h index 6f6120deb4f..4f61167fa1b 100644 --- a/src/DataTypes/DataTypeWithSimpleSerialization.h +++ b/src/DataTypes/DataTypeWithSimpleSerialization.h @@ -9,8 +9,7 @@ namespace DB class DataTypeWithSimpleSerialization : public IDataType { protected: - DataTypeWithSimpleSerialization() - {} + DataTypeWithSimpleSerialization() = default; void serializeTextEscaped(const IColumn & column, size_t row_num, WriteBuffer & ostr, const FormatSettings & settings) const override { diff --git a/src/Functions/geometryConverters.h b/src/Functions/geometryConverters.h index 03a19e0db4a..6115936dde4 100644 --- a/src/Functions/geometryConverters.h +++ b/src/Functions/geometryConverters.h @@ -61,7 +61,7 @@ template class PointFromColumnParser { public: - PointFromColumnParser(ColumnPtr col_) : col(col_) + explicit PointFromColumnParser(ColumnPtr col_) : col(col_) { const auto & tuple = dynamic_cast(*col_); const auto & tuple_columns = tuple.getColumns(); @@ -129,7 +129,7 @@ template class RingFromColumnParser { public: - RingFromColumnParser(ColumnPtr col_) + explicit RingFromColumnParser(ColumnPtr col_) : col(col_) , offsets(dynamic_cast(*col_).getOffsets()) , point_parser(dynamic_cast(*col_).getDataPtr()) @@ -179,7 +179,7 @@ template class PolygonFromColumnParser { public: - PolygonFromColumnParser(ColumnPtr col_) + explicit PolygonFromColumnParser(ColumnPtr col_) : col(col_) , offsets(static_cast(*col_).getOffsets()) , ring_parser(static_cast(*col_).getDataPtr()) @@ -220,7 +220,7 @@ template class MultiPolygonFromColumnParser { public: - MultiPolygonFromColumnParser(ColumnPtr col_) + explicit MultiPolygonFromColumnParser(ColumnPtr col_) : col(col_) , offsets(static_cast(*col_).getOffsets()) , polygon_parser(static_cast(*col_).getDataPtr()) @@ -287,7 +287,7 @@ public: , second(ColumnFloat64::create()) {} - PointSerializerVisitor(size_t n) + explicit PointSerializerVisitor(size_t n) : first(ColumnFloat64::create(n)) , second(ColumnFloat64::create(n)) {} @@ -344,7 +344,7 @@ public: : offsets(ColumnUInt64::create()) {} - RingSerializerVisitor(size_t n) + explicit RingSerializerVisitor(size_t n) : offsets(ColumnUInt64::create(n)) {} @@ -384,12 +384,12 @@ public: ColumnPtr finalize() { - return ColumnArray::create(pointSerializer.finalize(), std::move(offsets)); + return ColumnArray::create(point_serializer.finalize(), std::move(offsets)); } private: size_t size = 0; - PointSerializerVisitor pointSerializer; + PointSerializerVisitor point_serializer; ColumnUInt64::MutablePtr offsets; }; @@ -401,7 +401,7 @@ public: : offsets(ColumnUInt64::create()) {} - PolygonSerializerVisitor(size_t n) + explicit PolygonSerializerVisitor(size_t n) : offsets(ColumnUInt64::create(n)) {} @@ -440,12 +440,12 @@ public: ColumnPtr finalize() { - return ColumnArray::create(ringSerializer.finalize(), std::move(offsets)); + return ColumnArray::create(ring_serializer.finalize(), std::move(offsets)); } private: size_t size = 0; - RingSerializerVisitor ringSerializer; + RingSerializerVisitor ring_serializer; ColumnUInt64::MutablePtr offsets; }; @@ -457,7 +457,7 @@ public: : offsets(ColumnUInt64::create()) {} - MultiPolygonSerializerVisitor(size_t n) + explicit MultiPolygonSerializerVisitor(size_t n) : offsets(ColumnUInt64::create(n)) {} @@ -494,12 +494,12 @@ public: ColumnPtr finalize() { - return ColumnArray::create(polygonSerializer.finalize(), std::move(offsets)); + return ColumnArray::create(polygon_serializer.finalize(), std::move(offsets)); } private: size_t size = 0; - PolygonSerializerVisitor polygonSerializer; + PolygonSerializerVisitor polygon_serializer; ColumnUInt64::MutablePtr offsets; }; diff --git a/src/Functions/polygonArea.cpp b/src/Functions/polygonArea.cpp index aac9a182ea9..44687ae183e 100644 --- a/src/Functions/polygonArea.cpp +++ b/src/Functions/polygonArea.cpp @@ -19,6 +19,12 @@ namespace DB { + +namespace ErrorCodes +{ + extern const int BAD_ARGUMENTS; +} + template class FunctionPolygonArea : public IFunction { @@ -52,9 +58,28 @@ public: return std::make_shared(); } + void checkInputType(const ColumnsWithTypeAndName & arguments) const + { + /// Array(Array(Array(Tuple(Float64, Float64)))) + auto desired = std::make_shared( + std::make_shared( + std::make_shared( + std::make_shared( + DataTypes{std::make_shared(), std::make_shared()} + ) + ) + ) + ); + if (!desired->equals(*arguments[0].type)) + throw Exception(fmt::format("The type of first argument of function {} must be Array(Array(Array(Tuple(Float64, Float64))))", name), ErrorCodes::BAD_ARGUMENTS); + } + ColumnPtr executeImpl(const ColumnsWithTypeAndName & arguments, const DataTypePtr & /*result_type*/, size_t input_rows_count) const override { + checkInputType(arguments); auto parser = makeGeometryFromColumnParser(arguments[0]); + + std::cout << arguments[0].type->getName() << std::endl; auto container = createContainer(parser); auto res_column = ColumnFloat64::create(); diff --git a/src/Functions/polygonConvexHull.cpp b/src/Functions/polygonConvexHull.cpp index 21002c81612..26e37dc335b 100644 --- a/src/Functions/polygonConvexHull.cpp +++ b/src/Functions/polygonConvexHull.cpp @@ -20,6 +20,11 @@ namespace DB { +namespace ErrorCodes +{ + extern const int BAD_ARGUMENTS; +} + template class FunctionPolygonConvexHull : public IFunction { @@ -53,8 +58,25 @@ public: return DataTypeCustomPolygonSerialization::nestedDataType(); } + void checkInputType(const ColumnsWithTypeAndName & arguments) const + { + /// Array(Array(Array(Tuple(Float64, Float64)))) + auto desired = std::make_shared( + std::make_shared( + std::make_shared( + std::make_shared( + DataTypes{std::make_shared(), std::make_shared()} + ) + ) + ) + ); + if (!desired->equals(*arguments[0].type)) + throw Exception(fmt::format("The type of the argument of function {} must be Array(Array(Array(Tuple(Float64, Float64))))", name), ErrorCodes::BAD_ARGUMENTS); + } + ColumnPtr executeImpl(const ColumnsWithTypeAndName & arguments, const DataTypePtr & /*result_type*/, size_t input_rows_count) const override { + checkInputType(arguments); auto parser = makeGeometryFromColumnParser(arguments[0]); auto container = createContainer(parser); diff --git a/src/Functions/polygonPerimeter.cpp b/src/Functions/polygonPerimeter.cpp index aafc07aa3b2..6c45c45b99a 100644 --- a/src/Functions/polygonPerimeter.cpp +++ b/src/Functions/polygonPerimeter.cpp @@ -20,6 +20,11 @@ namespace DB { +namespace ErrorCodes +{ + extern const int BAD_ARGUMENTS; +} + template class FunctionPolygonPerimeter : public IFunction { @@ -53,8 +58,25 @@ public: return std::make_shared(); } + void checkInputType(const ColumnsWithTypeAndName & arguments) const + { + /// Array(Array(Array(Tuple(Float64, Float64)))) + auto desired = std::make_shared( + std::make_shared( + std::make_shared( + std::make_shared( + DataTypes{std::make_shared(), std::make_shared()} + ) + ) + ) + ); + if (!desired->equals(*arguments[0].type)) + throw Exception(fmt::format("The type of the argument of function {} must be Array(Array(Array(Tuple(Float64, Float64))))", name), ErrorCodes::BAD_ARGUMENTS); + } + ColumnPtr executeImpl(const ColumnsWithTypeAndName & arguments, const DataTypePtr & /*result_type*/, size_t input_rows_count) const override { + checkInputType(arguments); auto parser = makeGeometryFromColumnParser(arguments[0]); auto container = createContainer(parser); diff --git a/src/Functions/polygonsDistance.cpp b/src/Functions/polygonsDistance.cpp index c1530867f31..eb795860f6c 100644 --- a/src/Functions/polygonsDistance.cpp +++ b/src/Functions/polygonsDistance.cpp @@ -22,6 +22,11 @@ namespace DB { +namespace ErrorCodes +{ + extern const int BAD_ARGUMENTS; +} + template class FunctionPolygonsDistance : public IFunction { @@ -55,6 +60,25 @@ public: return std::make_shared(); } + void checkInputType(const ColumnsWithTypeAndName & arguments) const + { + /// Array(Array(Array(Tuple(Float64, Float64)))) + auto desired = std::make_shared( + std::make_shared( + std::make_shared( + std::make_shared( + DataTypes{std::make_shared(), std::make_shared()} + ) + ) + ) + ); + if (!desired->equals(*arguments[0].type)) + throw Exception(fmt::format("The type of the first argument of function {} must be Array(Array(Array(Tuple(Float64, Float64))))", name), ErrorCodes::BAD_ARGUMENTS); + + if (!desired->equals(*arguments[1].type)) + throw Exception(fmt::format("The type of the second argument of function {} must be Array(Array(Array(Tuple(Float64, Float64))))", name), ErrorCodes::BAD_ARGUMENTS); + } + ColumnPtr executeImpl(const ColumnsWithTypeAndName & arguments, const DataTypePtr & /*result_type*/, size_t input_rows_count) const override { auto first_parser = makeGeometryFromColumnParser(arguments[0]); diff --git a/src/Functions/polygonsEquals.cpp b/src/Functions/polygonsEquals.cpp index 10d92eef09b..dc056b78815 100644 --- a/src/Functions/polygonsEquals.cpp +++ b/src/Functions/polygonsEquals.cpp @@ -22,6 +22,11 @@ namespace DB { +namespace ErrorCodes +{ + extern const int BAD_ARGUMENTS; +} + template class FunctionPolygonsEquals : public IFunction { @@ -55,8 +60,28 @@ public: return std::make_shared(); } + void checkInputType(const ColumnsWithTypeAndName & arguments) const + { + /// Array(Array(Array(Tuple(Float64, Float64)))) + auto desired = std::make_shared( + std::make_shared( + std::make_shared( + std::make_shared( + DataTypes{std::make_shared(), std::make_shared()} + ) + ) + ) + ); + if (!desired->equals(*arguments[0].type)) + throw Exception(fmt::format("The type of the first argument of function {} must be Array(Array(Array(Tuple(Float64, Float64))))", name), ErrorCodes::BAD_ARGUMENTS); + + if (!desired->equals(*arguments[1].type)) + throw Exception(fmt::format("The type of the second argument of function {} must be Array(Array(Array(Tuple(Float64, Float64))))", name), ErrorCodes::BAD_ARGUMENTS); + } + ColumnPtr executeImpl(const ColumnsWithTypeAndName & arguments, const DataTypePtr & /*result_type*/, size_t input_rows_count) const override { + checkInputType(arguments); auto first_parser = makeGeometryFromColumnParser(arguments[0]); auto first_container = createContainer(first_parser); diff --git a/src/Functions/polygonsIntersection.cpp b/src/Functions/polygonsIntersection.cpp index cff28860115..ef99caf57a2 100644 --- a/src/Functions/polygonsIntersection.cpp +++ b/src/Functions/polygonsIntersection.cpp @@ -20,6 +20,11 @@ namespace DB { +namespace ErrorCodes +{ + extern const int BAD_ARGUMENTS; +} + template class FunctionPolygonsIntersection : public IFunction { @@ -53,8 +58,28 @@ public: return DataTypeCustomMultiPolygonSerialization::nestedDataType(); } + void checkInputType(const ColumnsWithTypeAndName & arguments) const + { + /// Array(Array(Array(Tuple(Float64, Float64)))) + auto desired = std::make_shared( + std::make_shared( + std::make_shared( + std::make_shared( + DataTypes{std::make_shared(), std::make_shared()} + ) + ) + ) + ); + if (!desired->equals(*arguments[0].type)) + throw Exception(fmt::format("The type of the first argument of function {} must be Array(Array(Array(Tuple(Float64, Float64))))", name), ErrorCodes::BAD_ARGUMENTS); + + if (!desired->equals(*arguments[1].type)) + throw Exception(fmt::format("The type of the second argument of function {} must be Array(Array(Array(Tuple(Float64, Float64))))", name), ErrorCodes::BAD_ARGUMENTS); + } + ColumnPtr executeImpl(const ColumnsWithTypeAndName & arguments, const DataTypePtr & /*result_type*/, size_t input_rows_count) const override { + checkInputType(arguments); auto first_parser = makeGeometryFromColumnParser(arguments[0]); auto first_container = createContainer(first_parser); @@ -79,7 +104,7 @@ public: boost::geometry::correct(second); /// Main work here. - boost::geometry::intersection(first, second,intersection); + boost::geometry::intersection(first, second, intersection); intersection.erase(intersection.begin()); diff --git a/src/Functions/polygonsSymDifference.cpp b/src/Functions/polygonsSymDifference.cpp index 38fc43f56ca..53abadf3c4c 100644 --- a/src/Functions/polygonsSymDifference.cpp +++ b/src/Functions/polygonsSymDifference.cpp @@ -20,6 +20,11 @@ namespace DB { +namespace ErrorCodes +{ + extern const int BAD_ARGUMENTS; +} + template class FunctionPolygonsSymDifference : public IFunction { @@ -53,8 +58,28 @@ public: return DataTypeCustomMultiPolygonSerialization::nestedDataType(); } + void checkInputType(const ColumnsWithTypeAndName & arguments) const + { + /// Array(Array(Array(Tuple(Float64, Float64)))) + auto desired = std::make_shared( + std::make_shared( + std::make_shared( + std::make_shared( + DataTypes{std::make_shared(), std::make_shared()} + ) + ) + ) + ); + if (!desired->equals(*arguments[0].type)) + throw Exception(fmt::format("The type of the first argument of function {} must be Array(Array(Array(Tuple(Float64, Float64))))", name), ErrorCodes::BAD_ARGUMENTS); + + if (!desired->equals(*arguments[1].type)) + throw Exception(fmt::format("The type of the second argument of function {} must be Array(Array(Array(Tuple(Float64, Float64))))", name), ErrorCodes::BAD_ARGUMENTS); + } + ColumnPtr executeImpl(const ColumnsWithTypeAndName & arguments, const DataTypePtr & /*result_type*/, size_t input_rows_count) const override { + checkInputType(arguments); auto first_parser = makeGeometryFromColumnParser(arguments[0]); auto first_container = createContainer(first_parser); diff --git a/src/Functions/polygonsUnion.cpp b/src/Functions/polygonsUnion.cpp index 44435461c0c..7198e5b4959 100644 --- a/src/Functions/polygonsUnion.cpp +++ b/src/Functions/polygonsUnion.cpp @@ -19,6 +19,10 @@ namespace DB { +namespace ErrorCodes +{ + extern const int BAD_ARGUMENTS; +} template class FunctionPolygonsUnion : public IFunction @@ -53,8 +57,28 @@ public: return DataTypeCustomMultiPolygonSerialization::nestedDataType(); } + void checkInputType(const ColumnsWithTypeAndName & arguments) const + { + /// Array(Array(Array(Tuple(Float64, Float64)))) + auto desired = std::make_shared( + std::make_shared( + std::make_shared( + std::make_shared( + DataTypes{std::make_shared(), std::make_shared()} + ) + ) + ) + ); + if (!desired->equals(*arguments[0].type)) + throw Exception(fmt::format("The type of the first argument of function {} must be Array(Array(Array(Tuple(Float64, Float64))))", name), ErrorCodes::BAD_ARGUMENTS); + + if (!desired->equals(*arguments[1].type)) + throw Exception(fmt::format("The type of the second argument of function {} must be Array(Array(Array(Tuple(Float64, Float64))))", name), ErrorCodes::BAD_ARGUMENTS); + } + ColumnPtr executeImpl(const ColumnsWithTypeAndName & arguments, const DataTypePtr & /*result_type*/, size_t input_rows_count) const override { + checkInputType(arguments); auto first_parser = makeGeometryFromColumnParser(arguments[0]); auto first_container = createContainer(first_parser); diff --git a/src/Functions/polygonsWithin.cpp b/src/Functions/polygonsWithin.cpp index 8caa16083f1..5e13adefffa 100644 --- a/src/Functions/polygonsWithin.cpp +++ b/src/Functions/polygonsWithin.cpp @@ -22,6 +22,11 @@ namespace DB { +namespace ErrorCodes +{ + extern const int BAD_ARGUMENTS; +} + template class FunctionPolygonsWithin : public IFunction { @@ -55,8 +60,28 @@ public: return std::make_shared(); } + void checkInputType(const ColumnsWithTypeAndName & arguments) const + { + /// Array(Array(Array(Tuple(Float64, Float64)))) + auto desired = std::make_shared( + std::make_shared( + std::make_shared( + std::make_shared( + DataTypes{std::make_shared(), std::make_shared()} + ) + ) + ) + ); + if (!desired->equals(*arguments[0].type)) + throw Exception(fmt::format("The type of the first argument of function {} must be Array(Array(Array(Tuple(Float64, Float64))))", name), ErrorCodes::BAD_ARGUMENTS); + + if (!desired->equals(*arguments[1].type)) + throw Exception(fmt::format("The type of the second argument of function {} must be Array(Array(Array(Tuple(Float64, Float64))))", name), ErrorCodes::BAD_ARGUMENTS); + } + ColumnPtr executeImpl(const ColumnsWithTypeAndName & arguments, const DataTypePtr & /*result_type*/, size_t input_rows_count) const override { + checkInputType(arguments); auto first_parser = makeGeometryFromColumnParser(arguments[0]); auto first_container = createContainer(first_parser); From 2cce75808b684eb5a64d232792edf8b24b7d2206 Mon Sep 17 00:00:00 2001 From: Nikita Mikhailov Date: Fri, 12 Feb 2021 13:02:44 +0300 Subject: [PATCH 288/544] fix build --- src/Functions/geometryConverters.h | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/src/Functions/geometryConverters.h b/src/Functions/geometryConverters.h index 6115936dde4..7a5ce1ae6f2 100644 --- a/src/Functions/geometryConverters.h +++ b/src/Functions/geometryConverters.h @@ -353,7 +353,7 @@ public: size++; offsets->insertValue(size); - pointSerializer(point); + point_serializer(point); } void operator()(const Ring & ring) @@ -362,7 +362,7 @@ public: offsets->insertValue(size); for (const auto & point : ring) { - pointSerializer(point); + point_serializer(point); } } @@ -409,24 +409,24 @@ public: { size++; offsets->insertValue(size); - ringSerializer(point); + ring_serializer(point); } void operator()(const Ring & ring) { size++; offsets->insertValue(size); - ringSerializer(ring); + ring_serializer(ring); } void operator()(const Polygon & polygon) { size += 1 + polygon.inners().size(); offsets->insertValue(size); - ringSerializer(polygon.outer()); + ring_serializer(polygon.outer()); for (const auto & ring : polygon.inners()) { - ringSerializer(ring); + ring_serializer(ring); } } @@ -465,21 +465,21 @@ public: { size++; offsets->insertValue(size); - polygonSerializer(point); + polygon_serializer(point); } void operator()(const Ring & ring) { size++; offsets->insertValue(size); - polygonSerializer(ring); + polygon_serializer(ring); } void operator()(const Polygon & polygon) { size++; offsets->insertValue(size); - polygonSerializer(polygon); + polygon_serializer(polygon); } void operator()(const MultiPolygon & multi_polygon) @@ -488,7 +488,7 @@ public: offsets->insertValue(size); for (const auto & polygon : multi_polygon) { - polygonSerializer(polygon); + polygon_serializer(polygon); } } From 05cf1b5bc5026731e17df81ab78b9d99e06d5222 Mon Sep 17 00:00:00 2001 From: Nikita Mikhailov Date: Mon, 15 Feb 2021 22:22:13 +0300 Subject: [PATCH 289/544] better --- src/DataTypes/DataTypeCustomGeo.cpp | 11 ++-- src/Functions/geometryConverters.cpp | 26 +++++++++ src/Functions/geometryConverters.h | 25 +++++---- src/Functions/polygonArea.cpp | 37 ++----------- src/Functions/polygonConvexHull.cpp | 40 +++----------- src/Functions/polygonPerimeter.cpp | 35 ++---------- src/Functions/polygonsDistance.cpp | 49 ++++------------- src/Functions/polygonsEquals.cpp | 50 ++++------------- src/Functions/polygonsIntersection.cpp | 54 +++++-------------- src/Functions/polygonsSymDifference.cpp | 53 +++++------------- src/Functions/polygonsUnion.cpp | 51 +++++------------- src/Functions/polygonsWithin.cpp | 50 ++++------------- .../0_stateless/01300_polygon_convex_hull.sql | 2 +- .../0_stateless/01301_polygons_within.sql | 4 +- .../0_stateless/01302_polygons_distance.sql | 4 +- .../0_stateless/01303_polygons_equals.sql | 4 +- .../01304_polygons_sym_difference.sql | 2 +- .../0_stateless/01305_polygons_union.sql | 2 +- .../01306_polygons_intersection.sql | 4 +- .../0_stateless/01307_polygon_perimeter.sql | 2 +- .../0_stateless/01308_polygon_area.sql | 2 +- 21 files changed, 147 insertions(+), 360 deletions(-) diff --git a/src/DataTypes/DataTypeCustomGeo.cpp b/src/DataTypes/DataTypeCustomGeo.cpp index 9f30ba4223a..60fe6f3450c 100644 --- a/src/DataTypes/DataTypeCustomGeo.cpp +++ b/src/DataTypes/DataTypeCustomGeo.cpp @@ -26,8 +26,9 @@ void DataTypeCustomPointSerialization::deserializeText( DataTypePtr DataTypeCustomPointSerialization::nestedDataType() { - static auto data_type = DataTypePtr(std::make_unique( - DataTypes({std::make_unique(), std::make_unique()}))); + static const auto data_type = std::make_shared( + DataTypes{std::make_shared(), std::make_shared()} + ); return data_type; } @@ -45,7 +46,7 @@ void DataTypeCustomRingSerialization::deserializeText( DataTypePtr DataTypeCustomRingSerialization::nestedDataType() { - static auto data_type = DataTypePtr(std::make_unique(DataTypeCustomPointSerialization::nestedDataType())); + static auto data_type = std::make_shared(DataTypeCustomPointSerialization::nestedDataType()); return data_type; } @@ -63,7 +64,7 @@ void DataTypeCustomPolygonSerialization::deserializeText( DataTypePtr DataTypeCustomPolygonSerialization::nestedDataType() { - static auto data_type = DataTypePtr(std::make_unique(DataTypeCustomRingSerialization::nestedDataType())); + static auto data_type = std::make_shared(DataTypeCustomRingSerialization::nestedDataType()); return data_type; } @@ -81,7 +82,7 @@ void DataTypeCustomMultiPolygonSerialization::deserializeText( DataTypePtr DataTypeCustomMultiPolygonSerialization::nestedDataType() { - static auto data_type = DataTypePtr(std::make_unique(DataTypeCustomPolygonSerialization::nestedDataType())); + static auto data_type = std::make_shared(DataTypeCustomPolygonSerialization::nestedDataType()); return data_type; } diff --git a/src/Functions/geometryConverters.cpp b/src/Functions/geometryConverters.cpp index def1f3b8fde..8665fbb1771 100644 --- a/src/Functions/geometryConverters.cpp +++ b/src/Functions/geometryConverters.cpp @@ -9,6 +9,8 @@ namespace DB namespace ErrorCodes { extern const int ILLEGAL_COLUMN; + extern const int BAD_ARGUMENTS; + extern const int LOGICAL_ERROR; } namespace @@ -108,4 +110,28 @@ template GeometryFromColumnParser makeGeometryFromColumnParser(c template GeometryFromColumnParser makeGeometryFromColumnParser(const ColumnWithTypeAndName & col); +template typename Desired> +void checkColumnTypeOrThrow(const ColumnWithTypeAndName & column) +{ + DataTypePtr desired_type; + if constexpr (std::is_same_v, Ring>) + desired_type = DataTypeCustomRingSerialization::nestedDataType(); + else if constexpr (std::is_same_v, Polygon>) + desired_type = DataTypeCustomPolygonSerialization::nestedDataType(); + else if constexpr (std::is_same_v, MultiPolygon>) + desired_type = DataTypeCustomMultiPolygonSerialization::nestedDataType(); + else + throw Exception("Unexpected Desired type.", ErrorCodes::LOGICAL_ERROR); + + if (!desired_type->equals(*column.type)) + throw Exception(fmt::format("Expected type {} (MultiPolygon), but got {}", desired_type->getName(), column.type->getName()), ErrorCodes::BAD_ARGUMENTS); +} + +template void checkColumnTypeOrThrow(const ColumnWithTypeAndName &); +template void checkColumnTypeOrThrow(const ColumnWithTypeAndName &); +template void checkColumnTypeOrThrow(const ColumnWithTypeAndName &); +template void checkColumnTypeOrThrow(const ColumnWithTypeAndName &); +template void checkColumnTypeOrThrow(const ColumnWithTypeAndName &); +template void checkColumnTypeOrThrow(const ColumnWithTypeAndName &); + } diff --git a/src/Functions/geometryConverters.h b/src/Functions/geometryConverters.h index 7a5ce1ae6f2..7938e5ca775 100644 --- a/src/Functions/geometryConverters.h +++ b/src/Functions/geometryConverters.h @@ -233,14 +233,18 @@ public: void get(Geometry & container, size_t i) const { - auto & multi_polygon = boost::get>(container); + get(boost::get>(container), i); + } + + void get(MultiPolygon & container, size_t i) const + { size_t l = offsets[i - 1]; size_t r = offsets[i]; - multi_polygon.resize(r - l); + container.resize(r - l); for (size_t j = l; j < r; j++) { - polygon_parser.get(multi_polygon[j - l], j); + polygon_parser.get(container[j - l], j); } } @@ -262,18 +266,17 @@ using GeometryFromColumnParser = boost::variant< template Geometry createContainer(const GeometryFromColumnParser & parser); -extern template Geometry createContainer(const GeometryFromColumnParser & parser); -extern template Geometry createContainer(const GeometryFromColumnParser & parser); - template void get(const GeometryFromColumnParser & parser, Geometry & container, size_t i); -extern template void get(const GeometryFromColumnParser & parser, Geometry & container, size_t i); -extern template void get(const GeometryFromColumnParser & parser, Geometry & container, size_t i); - template GeometryFromColumnParser makeGeometryFromColumnParser(const ColumnWithTypeAndName & col); + +extern template Geometry createContainer(const GeometryFromColumnParser & parser); +extern template Geometry createContainer(const GeometryFromColumnParser & parser); +extern template void get(const GeometryFromColumnParser & parser, Geometry & container, size_t i); +extern template void get(const GeometryFromColumnParser & parser, Geometry & container, size_t i); extern template GeometryFromColumnParser makeGeometryFromColumnParser(const ColumnWithTypeAndName & col); extern template GeometryFromColumnParser makeGeometryFromColumnParser(const ColumnWithTypeAndName & col); @@ -532,4 +535,8 @@ using PolygonSerializer = GeometrySerializer, PolygonSerializerV template using MultiPolygonSerializer = GeometrySerializer, MultiPolygonSerializerVisitor>; + +template typename Desired> +void checkColumnTypeOrThrow(const ColumnWithTypeAndName & column); + } diff --git a/src/Functions/polygonArea.cpp b/src/Functions/polygonArea.cpp index 44687ae183e..e3ecc9d6fac 100644 --- a/src/Functions/polygonArea.cpp +++ b/src/Functions/polygonArea.cpp @@ -20,11 +20,6 @@ namespace DB { -namespace ErrorCodes -{ - extern const int BAD_ARGUMENTS; -} - template class FunctionPolygonArea : public IFunction { @@ -58,40 +53,18 @@ public: return std::make_shared(); } - void checkInputType(const ColumnsWithTypeAndName & arguments) const - { - /// Array(Array(Array(Tuple(Float64, Float64)))) - auto desired = std::make_shared( - std::make_shared( - std::make_shared( - std::make_shared( - DataTypes{std::make_shared(), std::make_shared()} - ) - ) - ) - ); - if (!desired->equals(*arguments[0].type)) - throw Exception(fmt::format("The type of first argument of function {} must be Array(Array(Array(Tuple(Float64, Float64))))", name), ErrorCodes::BAD_ARGUMENTS); - } - ColumnPtr executeImpl(const ColumnsWithTypeAndName & arguments, const DataTypePtr & /*result_type*/, size_t input_rows_count) const override { - checkInputType(arguments); - auto parser = makeGeometryFromColumnParser(arguments[0]); - - std::cout << arguments[0].type->getName() << std::endl; - auto container = createContainer(parser); + checkColumnTypeOrThrow(arguments[0]); + auto parser = MultiPolygonFromColumnParser(std::move(arguments[0].column->convertToFullColumnIfConst())); + MultiPolygon container; auto res_column = ColumnFloat64::create(); for (size_t i = 0; i < input_rows_count; i++) { - get(parser, container, i); - - Float64 area = boost::geometry::area( - boost::get>(container)); - - res_column->insertValue(area); + parser.get(container, i); + res_column->insertValue(boost::geometry::area(container)); } return res_column; diff --git a/src/Functions/polygonConvexHull.cpp b/src/Functions/polygonConvexHull.cpp index 26e37dc335b..3bc4cd1cf2d 100644 --- a/src/Functions/polygonConvexHull.cpp +++ b/src/Functions/polygonConvexHull.cpp @@ -20,11 +20,6 @@ namespace DB { -namespace ErrorCodes -{ - extern const int BAD_ARGUMENTS; -} - template class FunctionPolygonConvexHull : public IFunction { @@ -58,42 +53,19 @@ public: return DataTypeCustomPolygonSerialization::nestedDataType(); } - void checkInputType(const ColumnsWithTypeAndName & arguments) const - { - /// Array(Array(Array(Tuple(Float64, Float64)))) - auto desired = std::make_shared( - std::make_shared( - std::make_shared( - std::make_shared( - DataTypes{std::make_shared(), std::make_shared()} - ) - ) - ) - ); - if (!desired->equals(*arguments[0].type)) - throw Exception(fmt::format("The type of the argument of function {} must be Array(Array(Array(Tuple(Float64, Float64))))", name), ErrorCodes::BAD_ARGUMENTS); - } - ColumnPtr executeImpl(const ColumnsWithTypeAndName & arguments, const DataTypePtr & /*result_type*/, size_t input_rows_count) const override { - checkInputType(arguments); - auto parser = makeGeometryFromColumnParser(arguments[0]); - auto container = createContainer(parser); + checkColumnTypeOrThrow(arguments[0]); + auto parser = MultiPolygonFromColumnParser(std::move(arguments[0].column->convertToFullColumnIfConst())); + MultiPolygon container; PolygonSerializer serializer; + Polygon convex_hull{}; for (size_t i = 0; i < input_rows_count; i++) { - get(parser, container, i); - - auto convex_hull = Polygon({{{}}}); - - boost::geometry::convex_hull( - boost::get>(container), - convex_hull); - - convex_hull.outer().erase(convex_hull.outer().begin()); - + parser.get(container, i); + boost::geometry::convex_hull(container, convex_hull); serializer.add(convex_hull); } diff --git a/src/Functions/polygonPerimeter.cpp b/src/Functions/polygonPerimeter.cpp index 6c45c45b99a..2f277af7fd5 100644 --- a/src/Functions/polygonPerimeter.cpp +++ b/src/Functions/polygonPerimeter.cpp @@ -20,11 +20,6 @@ namespace DB { -namespace ErrorCodes -{ - extern const int BAD_ARGUMENTS; -} - template class FunctionPolygonPerimeter : public IFunction { @@ -58,38 +53,18 @@ public: return std::make_shared(); } - void checkInputType(const ColumnsWithTypeAndName & arguments) const - { - /// Array(Array(Array(Tuple(Float64, Float64)))) - auto desired = std::make_shared( - std::make_shared( - std::make_shared( - std::make_shared( - DataTypes{std::make_shared(), std::make_shared()} - ) - ) - ) - ); - if (!desired->equals(*arguments[0].type)) - throw Exception(fmt::format("The type of the argument of function {} must be Array(Array(Array(Tuple(Float64, Float64))))", name), ErrorCodes::BAD_ARGUMENTS); - } - ColumnPtr executeImpl(const ColumnsWithTypeAndName & arguments, const DataTypePtr & /*result_type*/, size_t input_rows_count) const override { - checkInputType(arguments); - auto parser = makeGeometryFromColumnParser(arguments[0]); - auto container = createContainer(parser); + checkColumnTypeOrThrow(arguments[0]); + auto parser = MultiPolygonFromColumnParser(std::move(arguments[0].column->convertToFullColumnIfConst())); + MultiPolygon container; auto res_column = ColumnFloat64::create(); for (size_t i = 0; i < input_rows_count; i++) { - get(parser, container, i); - - Float64 perimeter = boost::geometry::perimeter( - boost::get>(container)); - - res_column->insertValue(perimeter); + parser.get(container, i); + res_column->insertValue(boost::geometry::perimeter(container)); } return res_column; diff --git a/src/Functions/polygonsDistance.cpp b/src/Functions/polygonsDistance.cpp index eb795860f6c..dea7ffcce70 100644 --- a/src/Functions/polygonsDistance.cpp +++ b/src/Functions/polygonsDistance.cpp @@ -22,11 +22,6 @@ namespace DB { -namespace ErrorCodes -{ - extern const int BAD_ARGUMENTS; -} - template class FunctionPolygonsDistance : public IFunction { @@ -60,49 +55,27 @@ public: return std::make_shared(); } - void checkInputType(const ColumnsWithTypeAndName & arguments) const - { - /// Array(Array(Array(Tuple(Float64, Float64)))) - auto desired = std::make_shared( - std::make_shared( - std::make_shared( - std::make_shared( - DataTypes{std::make_shared(), std::make_shared()} - ) - ) - ) - ); - if (!desired->equals(*arguments[0].type)) - throw Exception(fmt::format("The type of the first argument of function {} must be Array(Array(Array(Tuple(Float64, Float64))))", name), ErrorCodes::BAD_ARGUMENTS); - - if (!desired->equals(*arguments[1].type)) - throw Exception(fmt::format("The type of the second argument of function {} must be Array(Array(Array(Tuple(Float64, Float64))))", name), ErrorCodes::BAD_ARGUMENTS); - } - ColumnPtr executeImpl(const ColumnsWithTypeAndName & arguments, const DataTypePtr & /*result_type*/, size_t input_rows_count) const override { - auto first_parser = makeGeometryFromColumnParser(arguments[0]); - auto first_container = createContainer(first_parser); + checkColumnTypeOrThrow(arguments[0]); + auto first_parser = MultiPolygonFromColumnParser(std::move(arguments[0].column->convertToFullColumnIfConst())); + MultiPolygon first_container; - auto second_parser = makeGeometryFromColumnParser(arguments[1]); - auto second_container = createContainer(second_parser); + checkColumnTypeOrThrow(arguments[1]); + auto second_parser = MultiPolygonFromColumnParser(std::move(arguments[1].column->convertToFullColumnIfConst())); + MultiPolygon second_container; auto res_column = ColumnFloat64::create(); for (size_t i = 0; i < input_rows_count; i++) { - get(first_parser, first_container, i); - get(second_parser, second_container, i); + first_parser.get(first_container, i); + second_parser.get(second_container, i); - auto first = boost::get>(first_container); - auto second = boost::get>(second_container); + boost::geometry::correct(first_container); + boost::geometry::correct(second_container); - boost::geometry::correct(first); - boost::geometry::correct(second); - - Float64 distance = boost::geometry::distance(first, second); - - res_column->insertValue(distance); + res_column->insertValue(boost::geometry::distance(first_container, second_container)); } return res_column; diff --git a/src/Functions/polygonsEquals.cpp b/src/Functions/polygonsEquals.cpp index dc056b78815..4ab81597b70 100644 --- a/src/Functions/polygonsEquals.cpp +++ b/src/Functions/polygonsEquals.cpp @@ -22,11 +22,6 @@ namespace DB { -namespace ErrorCodes -{ - extern const int BAD_ARGUMENTS; -} - template class FunctionPolygonsEquals : public IFunction { @@ -60,50 +55,27 @@ public: return std::make_shared(); } - void checkInputType(const ColumnsWithTypeAndName & arguments) const - { - /// Array(Array(Array(Tuple(Float64, Float64)))) - auto desired = std::make_shared( - std::make_shared( - std::make_shared( - std::make_shared( - DataTypes{std::make_shared(), std::make_shared()} - ) - ) - ) - ); - if (!desired->equals(*arguments[0].type)) - throw Exception(fmt::format("The type of the first argument of function {} must be Array(Array(Array(Tuple(Float64, Float64))))", name), ErrorCodes::BAD_ARGUMENTS); - - if (!desired->equals(*arguments[1].type)) - throw Exception(fmt::format("The type of the second argument of function {} must be Array(Array(Array(Tuple(Float64, Float64))))", name), ErrorCodes::BAD_ARGUMENTS); - } - ColumnPtr executeImpl(const ColumnsWithTypeAndName & arguments, const DataTypePtr & /*result_type*/, size_t input_rows_count) const override { - checkInputType(arguments); - auto first_parser = makeGeometryFromColumnParser(arguments[0]); - auto first_container = createContainer(first_parser); + checkColumnTypeOrThrow(arguments[0]); + auto first_parser = MultiPolygonFromColumnParser(std::move(arguments[0].column->convertToFullColumnIfConst())); + MultiPolygon first_container; - auto second_parser = makeGeometryFromColumnParser(arguments[1]); - auto second_container = createContainer(second_parser); + checkColumnTypeOrThrow(arguments[1]); + auto second_parser = MultiPolygonFromColumnParser(std::move(arguments[1].column->convertToFullColumnIfConst())); + MultiPolygon second_container; auto res_column = ColumnUInt8::create(); for (size_t i = 0; i < input_rows_count; i++) { - get(first_parser, first_container, i); - get(second_parser, second_container, i); + first_parser.get(first_container, i); + second_parser.get(second_container, i); - auto first = boost::get>(first_container); - auto second = boost::get>(second_container); + boost::geometry::correct(first_container); + boost::geometry::correct(second_container); - boost::geometry::correct(first); - boost::geometry::correct(second); - - bool equals = boost::geometry::equals(first, second); - - res_column->insertValue(equals); + res_column->insertValue(boost::geometry::equals(first_container, second_container)); } return res_column; diff --git a/src/Functions/polygonsIntersection.cpp b/src/Functions/polygonsIntersection.cpp index ef99caf57a2..78e7396e082 100644 --- a/src/Functions/polygonsIntersection.cpp +++ b/src/Functions/polygonsIntersection.cpp @@ -20,11 +20,6 @@ namespace DB { -namespace ErrorCodes -{ - extern const int BAD_ARGUMENTS; -} - template class FunctionPolygonsIntersection : public IFunction { @@ -58,55 +53,32 @@ public: return DataTypeCustomMultiPolygonSerialization::nestedDataType(); } - void checkInputType(const ColumnsWithTypeAndName & arguments) const - { - /// Array(Array(Array(Tuple(Float64, Float64)))) - auto desired = std::make_shared( - std::make_shared( - std::make_shared( - std::make_shared( - DataTypes{std::make_shared(), std::make_shared()} - ) - ) - ) - ); - if (!desired->equals(*arguments[0].type)) - throw Exception(fmt::format("The type of the first argument of function {} must be Array(Array(Array(Tuple(Float64, Float64))))", name), ErrorCodes::BAD_ARGUMENTS); - - if (!desired->equals(*arguments[1].type)) - throw Exception(fmt::format("The type of the second argument of function {} must be Array(Array(Array(Tuple(Float64, Float64))))", name), ErrorCodes::BAD_ARGUMENTS); - } - ColumnPtr executeImpl(const ColumnsWithTypeAndName & arguments, const DataTypePtr & /*result_type*/, size_t input_rows_count) const override { - checkInputType(arguments); - auto first_parser = makeGeometryFromColumnParser(arguments[0]); - auto first_container = createContainer(first_parser); + checkColumnTypeOrThrow(arguments[0]); + auto first_parser = MultiPolygonFromColumnParser(std::move(arguments[0].column->convertToFullColumnIfConst())); + MultiPolygon first_container; - auto second_parser = makeGeometryFromColumnParser(arguments[1]); - auto second_container = createContainer(second_parser); + checkColumnTypeOrThrow(arguments[1]); + auto second_parser = MultiPolygonFromColumnParser(std::move(arguments[1].column->convertToFullColumnIfConst())); + MultiPolygon second_container; MultiPolygonSerializer serializer; + MultiPolygon intersection{}; /// We are not interested in some pitfalls in third-party libraries /// NOLINTNEXTLINE(clang-analyzer-core.uninitialized.Assign) - for (size_t i = 0; i < input_rows_count; i++) + for (size_t i = 0; i < input_rows_count; ++i) { - get(first_parser, first_container, i); - get(second_parser, second_container, i); - - auto intersection = MultiPolygon({{{{}}}}); - auto first = boost::get>(first_container); - auto second = boost::get>(second_container); + first_parser.get(first_container, i); + second_parser.get(second_container, i); /// Orient the polygons correctly. - boost::geometry::correct(first); - boost::geometry::correct(second); + boost::geometry::correct(first_container); + boost::geometry::correct(second_container); /// Main work here. - boost::geometry::intersection(first, second, intersection); - - intersection.erase(intersection.begin()); + boost::geometry::intersection(first_container, second_container, intersection); serializer.add(intersection); } diff --git a/src/Functions/polygonsSymDifference.cpp b/src/Functions/polygonsSymDifference.cpp index 53abadf3c4c..f8660de815d 100644 --- a/src/Functions/polygonsSymDifference.cpp +++ b/src/Functions/polygonsSymDifference.cpp @@ -20,11 +20,6 @@ namespace DB { -namespace ErrorCodes -{ - extern const int BAD_ARGUMENTS; -} - template class FunctionPolygonsSymDifference : public IFunction { @@ -58,53 +53,29 @@ public: return DataTypeCustomMultiPolygonSerialization::nestedDataType(); } - void checkInputType(const ColumnsWithTypeAndName & arguments) const - { - /// Array(Array(Array(Tuple(Float64, Float64)))) - auto desired = std::make_shared( - std::make_shared( - std::make_shared( - std::make_shared( - DataTypes{std::make_shared(), std::make_shared()} - ) - ) - ) - ); - if (!desired->equals(*arguments[0].type)) - throw Exception(fmt::format("The type of the first argument of function {} must be Array(Array(Array(Tuple(Float64, Float64))))", name), ErrorCodes::BAD_ARGUMENTS); - - if (!desired->equals(*arguments[1].type)) - throw Exception(fmt::format("The type of the second argument of function {} must be Array(Array(Array(Tuple(Float64, Float64))))", name), ErrorCodes::BAD_ARGUMENTS); - } - ColumnPtr executeImpl(const ColumnsWithTypeAndName & arguments, const DataTypePtr & /*result_type*/, size_t input_rows_count) const override { - checkInputType(arguments); - auto first_parser = makeGeometryFromColumnParser(arguments[0]); - auto first_container = createContainer(first_parser); + checkColumnTypeOrThrow(arguments[0]); + auto first_parser = MultiPolygonFromColumnParser(std::move(arguments[0].column->convertToFullColumnIfConst())); + MultiPolygon first_container; - auto second_parser = makeGeometryFromColumnParser(arguments[1]); - auto second_container = createContainer(second_parser); + checkColumnTypeOrThrow(arguments[1]); + auto second_parser = MultiPolygonFromColumnParser(std::move(arguments[1].column->convertToFullColumnIfConst())); + MultiPolygon second_container; MultiPolygonSerializer serializer; + MultiPolygon sym_difference{}; /// NOLINTNEXTLINE(clang-analyzer-core.uninitialized.Assign) for (size_t i = 0; i < input_rows_count; i++) { - get(first_parser, first_container, i); - get(second_parser, second_container, i); + first_parser.get(first_container, i); + second_parser.get(second_container, i); - auto sym_difference = MultiPolygon({{{{}}}}); + boost::geometry::correct(first_container); + boost::geometry::correct(second_container); - auto first = boost::get>(first_container); - auto second = boost::get>(second_container); - - boost::geometry::correct(first); - boost::geometry::correct(second); - - boost::geometry::sym_difference(first, second, sym_difference); - - sym_difference.erase(sym_difference.begin()); + boost::geometry::sym_difference(first_container, second_container, sym_difference); serializer.add(sym_difference); } diff --git a/src/Functions/polygonsUnion.cpp b/src/Functions/polygonsUnion.cpp index 7198e5b4959..f86706ad4c6 100644 --- a/src/Functions/polygonsUnion.cpp +++ b/src/Functions/polygonsUnion.cpp @@ -19,10 +19,6 @@ namespace DB { -namespace ErrorCodes -{ - extern const int BAD_ARGUMENTS; -} template class FunctionPolygonsUnion : public IFunction @@ -57,55 +53,32 @@ public: return DataTypeCustomMultiPolygonSerialization::nestedDataType(); } - void checkInputType(const ColumnsWithTypeAndName & arguments) const - { - /// Array(Array(Array(Tuple(Float64, Float64)))) - auto desired = std::make_shared( - std::make_shared( - std::make_shared( - std::make_shared( - DataTypes{std::make_shared(), std::make_shared()} - ) - ) - ) - ); - if (!desired->equals(*arguments[0].type)) - throw Exception(fmt::format("The type of the first argument of function {} must be Array(Array(Array(Tuple(Float64, Float64))))", name), ErrorCodes::BAD_ARGUMENTS); - - if (!desired->equals(*arguments[1].type)) - throw Exception(fmt::format("The type of the second argument of function {} must be Array(Array(Array(Tuple(Float64, Float64))))", name), ErrorCodes::BAD_ARGUMENTS); - } - ColumnPtr executeImpl(const ColumnsWithTypeAndName & arguments, const DataTypePtr & /*result_type*/, size_t input_rows_count) const override { - checkInputType(arguments); - auto first_parser = makeGeometryFromColumnParser(arguments[0]); - auto first_container = createContainer(first_parser); + checkColumnTypeOrThrow(arguments[0]); + auto first_parser = MultiPolygonFromColumnParser(std::move(arguments[0].column->convertToFullColumnIfConst())); + MultiPolygon first_container; - auto second_parser = makeGeometryFromColumnParser(arguments[1]); - auto second_container = createContainer(second_parser); + checkColumnTypeOrThrow(arguments[1]); + auto second_parser = MultiPolygonFromColumnParser(std::move(arguments[1].column->convertToFullColumnIfConst())); + MultiPolygon second_container; MultiPolygonSerializer serializer; + MultiPolygon polygons_union{}; /// We are not interested in some pitfalls in third-party libraries /// NOLINTNEXTLINE(clang-analyzer-core.uninitialized.Assign) for (size_t i = 0; i < input_rows_count; i++) { - get(first_parser, first_container, i); - get(second_parser, second_container, i); - - auto first = boost::get>(first_container); - auto second = boost::get>(second_container); - auto polygons_union = MultiPolygon({{{{}}}}); + first_parser.get(first_container, i); + second_parser.get(second_container, i); /// Orient the polygons correctly. - boost::geometry::correct(first); - boost::geometry::correct(second); + boost::geometry::correct(first_container); + boost::geometry::correct(second_container); /// Main work here. - boost::geometry::union_(first, second, polygons_union); - - polygons_union.erase(polygons_union.begin()); + boost::geometry::union_(first_container, second_container, polygons_union); serializer.add(polygons_union); } diff --git a/src/Functions/polygonsWithin.cpp b/src/Functions/polygonsWithin.cpp index 5e13adefffa..f483d68dc0d 100644 --- a/src/Functions/polygonsWithin.cpp +++ b/src/Functions/polygonsWithin.cpp @@ -22,11 +22,6 @@ namespace DB { -namespace ErrorCodes -{ - extern const int BAD_ARGUMENTS; -} - template class FunctionPolygonsWithin : public IFunction { @@ -60,51 +55,28 @@ public: return std::make_shared(); } - void checkInputType(const ColumnsWithTypeAndName & arguments) const - { - /// Array(Array(Array(Tuple(Float64, Float64)))) - auto desired = std::make_shared( - std::make_shared( - std::make_shared( - std::make_shared( - DataTypes{std::make_shared(), std::make_shared()} - ) - ) - ) - ); - if (!desired->equals(*arguments[0].type)) - throw Exception(fmt::format("The type of the first argument of function {} must be Array(Array(Array(Tuple(Float64, Float64))))", name), ErrorCodes::BAD_ARGUMENTS); - - if (!desired->equals(*arguments[1].type)) - throw Exception(fmt::format("The type of the second argument of function {} must be Array(Array(Array(Tuple(Float64, Float64))))", name), ErrorCodes::BAD_ARGUMENTS); - } - ColumnPtr executeImpl(const ColumnsWithTypeAndName & arguments, const DataTypePtr & /*result_type*/, size_t input_rows_count) const override { - checkInputType(arguments); - auto first_parser = makeGeometryFromColumnParser(arguments[0]); - auto first_container = createContainer(first_parser); + checkColumnTypeOrThrow(arguments[0]); + auto first_parser = MultiPolygonFromColumnParser(std::move(arguments[0].column->convertToFullColumnIfConst())); + MultiPolygon first_container; - auto second_parser = makeGeometryFromColumnParser(arguments[1]); - auto second_container = createContainer(second_parser); + checkColumnTypeOrThrow(arguments[1]); + auto second_parser = MultiPolygonFromColumnParser(std::move(arguments[1].column->convertToFullColumnIfConst())); + MultiPolygon second_container; auto res_column = ColumnUInt8::create(); /// NOLINTNEXTLINE(clang-analyzer-core.uninitialized.Assign) for (size_t i = 0; i < input_rows_count; i++) { - get(first_parser, first_container, i); - get(second_parser, second_container, i); + first_parser.get(first_container, i); + second_parser.get(second_container, i); - auto first = boost::get>(first_container); - auto second = boost::get>(second_container); + boost::geometry::correct(first_container); + boost::geometry::correct(second_container); - boost::geometry::correct(first); - boost::geometry::correct(second); - - bool within = boost::geometry::within(first, second); - - res_column->insertValue(within); + res_column->insertValue(boost::geometry::within(first_container, second_container)); } return res_column; diff --git a/tests/queries/0_stateless/01300_polygon_convex_hull.sql b/tests/queries/0_stateless/01300_polygon_convex_hull.sql index 125d10f3afc..4a4aa66bbfb 100644 --- a/tests/queries/0_stateless/01300_polygon_convex_hull.sql +++ b/tests/queries/0_stateless/01300_polygon_convex_hull.sql @@ -1 +1 @@ -select polygonConvexHullCartesian([[[(0, 0), (0, 5), (5, 5), (5, 0), (2, 3)]]]); +select polygonConvexHullCartesian([[[(0., 0.), (0., 5.), (5., 5.), (5., 0.), (2., 3.)]]]); diff --git a/tests/queries/0_stateless/01301_polygons_within.sql b/tests/queries/0_stateless/01301_polygons_within.sql index f3697803f23..5bfa586da79 100644 --- a/tests/queries/0_stateless/01301_polygons_within.sql +++ b/tests/queries/0_stateless/01301_polygons_within.sql @@ -1,5 +1,5 @@ -select polygonsWithinCartesian([[[(0, 0),(0, 3),(1, 2.9),(2, 2.6),(2.6, 2),(2.9, 1),(3, 0),(0, 0)]]], [[[(1, 1),(1, 4),(4, 4),(4, 1),(1, 1)]]]); -select polygonsWithinCartesian([[[(2, 2), (2, 3), (3, 3), (3, 2)]]], [[[(1, 1),(1, 4),(4, 4),(4, 1),(1, 1)]]]); +select polygonsWithinCartesian([[[(0, 0),(0, 3),(1, 2.9),(2, 2.6),(2.6, 2),(2.9, 1),(3, 0),(0, 0)]]], [[[(1., 1.),(1., 4.),(4., 4.),(4., 1.),(1., 1.)]]]); +select polygonsWithinCartesian([[[(2., 2.), (2., 3.), (3., 3.), (3., 2.)]]], [[[(1., 1.),(1., 4.),(4., 4.),(4., 1.),(1., 1.)]]]); select polygonsWithinGeographic([[[(4.3613577, 50.8651821), (4.349556, 50.8535879), (4.3602419, 50.8435626), (4.3830299, 50.8428851), (4.3904543, 50.8564867), (4.3613148, 50.8651279)]]], [[[(4.346693, 50.858306), (4.367945, 50.852455), (4.366227, 50.840809), (4.344961, 50.833264), (4.338074, 50.848677), (4.346693, 50.858306)]]]); select polygonsWithinGeographic([[[(4.3501568, 50.8518269), (4.3444920, 50.8439961), (4.3565941, 50.8443213), (4.3501568, 50.8518269)]]], [[[(4.3679450, 50.8524550),(4.3466930, 50.8583060),(4.3380740, 50.8486770),(4.3449610, 50.8332640),(4.3662270, 50.8408090),(4.3679450, 50.8524550)]]]); diff --git a/tests/queries/0_stateless/01302_polygons_distance.sql b/tests/queries/0_stateless/01302_polygons_distance.sql index a4c769ad8cb..0051578ea7c 100644 --- a/tests/queries/0_stateless/01302_polygons_distance.sql +++ b/tests/queries/0_stateless/01302_polygons_distance.sql @@ -1,5 +1,5 @@ -select polygonsDistanceCartesian([[[(0, 0),(0, 3),(1, 2.9),(2, 2.6),(2.6, 2),(2.9, 1),(3, 0),(0, 0)]]], [[[(1, 1),(1, 4),(4, 4),(4, 1),(1, 1)]]]); -select polygonsDistanceCartesian([[[(0, 0), (0, 0.1), (0.1, 0.1), (0.1, 0)]]], [[[(1, 1),(1, 4),(4, 4),(4, 1),(1, 1)]]]); +select polygonsDistanceCartesian([[[(0, 0),(0, 3),(1, 2.9),(2, 2.6),(2.6, 2),(2.9, 1),(3, 0),(0, 0)]]], [[[(1., 1.),(1., 4.),(4., 4.),(4., 1.),(1., 1.)]]]); +select polygonsDistanceCartesian([[[(0, 0), (0, 0.1), (0.1, 0.1), (0.1, 0)]]], [[[(1., 1.),(1., 4.),(4., 4.),(4., 1.),(1., 1.)]]]); select polygonsDistanceGeographic([[[(23.725750, 37.971536)]]], [[[(4.3826169, 50.8119483)]]]); drop table if exists polygon_01302; diff --git a/tests/queries/0_stateless/01303_polygons_equals.sql b/tests/queries/0_stateless/01303_polygons_equals.sql index b3a4d8f12b3..1d608ce45fa 100644 --- a/tests/queries/0_stateless/01303_polygons_equals.sql +++ b/tests/queries/0_stateless/01303_polygons_equals.sql @@ -1,2 +1,2 @@ -select polygonsEqualsCartesian([[[(0, 0),(0, 3),(1, 2.9),(2, 2.6),(2.6, 2),(2.9, 1),(3, 0),(0, 0)]]], [[[(1, 1),(1, 4),(4, 4),(4, 1),(1, 1)]]]); -select polygonsEqualsCartesian([[[(1, 1),(1, 4),(4, 4),(4, 1)]]], [[[(1, 1),(1, 4),(4, 4),(4, 1),(1, 1)]]]); \ No newline at end of file +select polygonsEqualsCartesian([[[(0, 0),(0, 3),(1, 2.9),(2, 2.6),(2.6, 2),(2.9, 1),(3, 0),(0, 0)]]], [[[(1., 1.),(1., 4.),(4., 4.),(4., 1.),(1., 1.)]]]); +select polygonsEqualsCartesian([[[(1., 1.),(1., 4.),(4., 4.),(4., 1.)]]], [[[(1., 1.),(1., 4.),(4., 4.),(4., 1.),(1., 1.)]]]); \ No newline at end of file diff --git a/tests/queries/0_stateless/01304_polygons_sym_difference.sql b/tests/queries/0_stateless/01304_polygons_sym_difference.sql index 1d839fa80b3..f81300172d3 100644 --- a/tests/queries/0_stateless/01304_polygons_sym_difference.sql +++ b/tests/queries/0_stateless/01304_polygons_sym_difference.sql @@ -1,2 +1,2 @@ -select polygonsSymDifferenceCartesian([[[(0, 0),(0, 3),(1, 2.9),(2, 2.6),(2.6, 2),(2.9, 1),(3, 0),(0, 0)]]], [[[(1, 1),(1, 4),(4, 4),(4, 1),(1, 1)]]]) +select polygonsSymDifferenceCartesian([[[(0, 0),(0, 3),(1, 2.9),(2, 2.6),(2.6, 2),(2.9, 1),(3, 0),(0, 0)]]], [[[(1., 1.),(1., 4.),(4., 4.),(4., 1.),(1., 1.)]]]) diff --git a/tests/queries/0_stateless/01305_polygons_union.sql b/tests/queries/0_stateless/01305_polygons_union.sql index 42b869e850a..67d1c3c95bb 100644 --- a/tests/queries/0_stateless/01305_polygons_union.sql +++ b/tests/queries/0_stateless/01305_polygons_union.sql @@ -1,3 +1,3 @@ -select polygonsUnionCartesian([[[(0, 0),(0, 3),(1, 2.9),(2, 2.6),(2.6, 2),(2.9, 1),(3, 0),(0, 0)]]], [[[(1, 1),(1, 4),(4, 4),(4, 1),(1, 1)]]]); +select polygonsUnionCartesian([[[(0., 0.),(0., 3.),(1., 2.9),(2., 2.6),(2.6, 2.),(2.9, 1),(3., 0.),(0., 0.)]]], [[[(1., 1.),(1., 4.),(4., 4.),(4., 1.),(1., 1.)]]]); select polygonsUnionGeographic([[[(4.3613577, 50.8651821), (4.349556, 50.8535879), (4.3602419, 50.8435626), (4.3830299, 50.8428851), (4.3904543, 50.8564867), (4.3613148, 50.8651279)]]], [[[(4.346693, 50.858306), (4.367945, 50.852455), (4.366227, 50.840809), (4.344961, 50.833264), (4.338074, 50.848677), (4.346693, 50.858306)]]]); diff --git a/tests/queries/0_stateless/01306_polygons_intersection.sql b/tests/queries/0_stateless/01306_polygons_intersection.sql index 0901f20fee5..15962c43648 100644 --- a/tests/queries/0_stateless/01306_polygons_intersection.sql +++ b/tests/queries/0_stateless/01306_polygons_intersection.sql @@ -1,5 +1,5 @@ -select polygonsIntersectionCartesian([[[(0, 0),(0, 3),(1, 2.9),(2, 2.6),(2.6, 2),(2.9, 1),(3, 0),(0, 0)]]], [[[(1, 1),(1, 4),(4, 4),(4, 1),(1, 1)]]]); -select polygonsIntersectionCartesian([[[(0, 0),(0, 3),(1, 2.9),(2, 2.6),(2.6, 2),(2.9, 1),(3, 0),(0, 0)]]], [[[(3, 3),(3, 4),(4, 4),(4, 3),(3, 3)]]]); +select polygonsIntersectionCartesian([[[(0., 0.),(0., 3.),(1., 2.9),(2., 2.6),(2.6, 2.),(2.9, 1.),(3., 0.),(0., 0.)]]], [[[(1., 1.),(1., 4.),(4., 4.),(4., 1.),(1., 1.)]]]); +select polygonsIntersectionCartesian([[[(0., 0.),(0., 3.),(1., 2.9),(2., 2.6),(2.6, 2.),(2.9, 1.),(3., 0.),(0., 0.)]]], [[[(3., 3.),(3., 4.),(4., 4.),(4., 3.),(3., 3.)]]]); select polygonsIntersectionGeographic([[[(4.346693, 50.858306), (4.367945, 50.852455), (4.366227, 50.840809), (4.344961, 50.833264), (4.338074, 50.848677), (4.346693, 50.858306)]]], [[[(25.0010, 136.9987), (17.7500, 142.5000), (11.3733, 142.5917)]]]); select polygonsIntersectionGeographic([[[(4.3613577, 50.8651821), (4.349556, 50.8535879), (4.3602419, 50.8435626), (4.3830299, 50.8428851), (4.3904543, 50.8564867), (4.3613148, 50.8651279)]]], [[[(4.346693, 50.858306), (4.367945, 50.852455), (4.366227, 50.840809), (4.344961, 50.833264), (4.338074, 50.848677), (4.346693, 50.858306)]]]); \ No newline at end of file diff --git a/tests/queries/0_stateless/01307_polygon_perimeter.sql b/tests/queries/0_stateless/01307_polygon_perimeter.sql index 7e12d032d83..1cbbf41a201 100644 --- a/tests/queries/0_stateless/01307_polygon_perimeter.sql +++ b/tests/queries/0_stateless/01307_polygon_perimeter.sql @@ -1 +1 @@ -select polygonPerimeterCartesian([[[(0, 0), (0., 5), (5, 5), (5., 0)]]]); \ No newline at end of file +select polygonPerimeterCartesian([[[(0., 0.), (0., 5.), (5., 5.), (5., 0.)]]]); \ No newline at end of file diff --git a/tests/queries/0_stateless/01308_polygon_area.sql b/tests/queries/0_stateless/01308_polygon_area.sql index 700cf661b48..03f87874139 100644 --- a/tests/queries/0_stateless/01308_polygon_area.sql +++ b/tests/queries/0_stateless/01308_polygon_area.sql @@ -1,3 +1,3 @@ -select polygonAreaCartesian([[[(0, 0), (0., 5), (5, 5), (5., 0)]]]); +select polygonAreaCartesian([[[(0., 0.), (0., 5.), (5., 5.), (5., 0.)]]]); select polygonAreaGeographic([[[(4.346693, 50.858306), (4.367945, 50.852455), (4.366227, 50.840809), (4.344961, 50.833264), (4.338074, 50.848677), (4.346693, 50.858306)]]]); SELECT polygonAreaCartesian([]); -- { serverError 36 } \ No newline at end of file From e78fc3109cbfa4bd212cdc3fe29a2fb0d1d4bcb3 Mon Sep 17 00:00:00 2001 From: Nikita Mikhailov Date: Tue, 16 Feb 2021 22:50:34 +0300 Subject: [PATCH 290/544] better ubsan testing --- base/common/defines.h | 10 ++++++++++ src/Functions/geometryConverters.h | 5 +++++ src/Functions/pointInPolygon.cpp | 4 ++-- tests/queries/0_stateless/01305_polygons_union.sql | 2 ++ 4 files changed, 19 insertions(+), 2 deletions(-) diff --git a/base/common/defines.h b/base/common/defines.h index 367bdd64234..ada8245f494 100644 --- a/base/common/defines.h +++ b/base/common/defines.h @@ -76,6 +76,16 @@ # endif #endif +#if !defined(UNDEFINED_BEHAVIOR_SANITIZER) +# if defined(__has_feature) +# if __has_feature(undefined_behavior_sanitizer) +# define UNDEFINED_BEHAVIOR_SANITIZER 1 +# endif +# elif defined(__UNDEFINED_BEHAVIOR_SANITIZER__) +# define UNDEFINED_BEHAVIOR_SANITIZER 1 +# endif +#endif + #if defined(ADDRESS_SANITIZER) # define BOOST_USE_ASAN 1 # define BOOST_USE_UCONTEXT 1 diff --git a/src/Functions/geometryConverters.h b/src/Functions/geometryConverters.h index 7938e5ca775..14d186433b2 100644 --- a/src/Functions/geometryConverters.h +++ b/src/Functions/geometryConverters.h @@ -11,6 +11,7 @@ #include #include #include +#include #include #include #include @@ -24,6 +25,7 @@ namespace DB namespace ErrorCodes { extern const int BAD_ARGUMENTS; + extern const int ILLEGAL_TYPE_OF_ARGUMENT; } namespace bg = boost::geometry; @@ -111,6 +113,9 @@ public: #ifndef NDEBUG assert(i < size); #endif + if (isNaN(first[i]) || isNaN(second[i])) + throw Exception("Point's component must not be NaN", ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT); + boost::geometry::set<0>(container, first[i]); boost::geometry::set<1>(container, second[i]); } diff --git a/src/Functions/pointInPolygon.cpp b/src/Functions/pointInPolygon.cpp index 3fd369cb4be..5c90b286fd9 100644 --- a/src/Functions/pointInPolygon.cpp +++ b/src/Functions/pointInPolygon.cpp @@ -540,7 +540,7 @@ private: } } - void parseConstPolygon(const ColumnsWithTypeAndName & arguments, Polygon & out_polygon) const + void NO_SANITIZE_UNDEFINED parseConstPolygon(const ColumnsWithTypeAndName & arguments, Polygon & out_polygon) const { if (arguments.size() == 2) parseConstPolygonFromSingleColumn(arguments, out_polygon); @@ -550,7 +550,7 @@ private: /// Fix orientation and close rings. It's required for subsequent processing. boost::geometry::correct(out_polygon); -#if !defined(__clang_analyzer__) /// It does not like boost. +#if !defined(__clang_analyzer__) && !defined(UNDEFINED_BEHAVIOR_SANITIZER) /// It does not like boost. if (validate) { std::string failure_message; diff --git a/tests/queries/0_stateless/01305_polygons_union.sql b/tests/queries/0_stateless/01305_polygons_union.sql index 67d1c3c95bb..c96c9f7360c 100644 --- a/tests/queries/0_stateless/01305_polygons_union.sql +++ b/tests/queries/0_stateless/01305_polygons_union.sql @@ -1,3 +1,5 @@ select polygonsUnionCartesian([[[(0., 0.),(0., 3.),(1., 2.9),(2., 2.6),(2.6, 2.),(2.9, 1),(3., 0.),(0., 0.)]]], [[[(1., 1.),(1., 4.),(4., 4.),(4., 1.),(1., 1.)]]]); +SELECT polygonsUnionCartesian([[[(2., 100.0000991821289), (0., 3.), (1., 2.9), (2., 2.6), (2.6, 2.), (2.9, 1), (3., 0.), (100.0000991821289, 2.)]]], [[[(1., 1.), (1000.0001220703125, nan), (4., 4.), (4., 1.), (1., 1.)]]]); -- { serverError 43 } + select polygonsUnionGeographic([[[(4.3613577, 50.8651821), (4.349556, 50.8535879), (4.3602419, 50.8435626), (4.3830299, 50.8428851), (4.3904543, 50.8564867), (4.3613148, 50.8651279)]]], [[[(4.346693, 50.858306), (4.367945, 50.852455), (4.366227, 50.840809), (4.344961, 50.833264), (4.338074, 50.848677), (4.346693, 50.858306)]]]); From 557fa7c5c87fe59b2559050811f0504076fa6fe1 Mon Sep 17 00:00:00 2001 From: Nikita Mikhailov Date: Wed, 17 Feb 2021 20:43:15 +0300 Subject: [PATCH 291/544] fix ubsan --- src/Functions/pointInPolygon.cpp | 11 ++++++++++- .../0_stateless/01700_point_in_polygon_ubsan.sql | 2 +- 2 files changed, 11 insertions(+), 2 deletions(-) diff --git a/src/Functions/pointInPolygon.cpp b/src/Functions/pointInPolygon.cpp index 5c90b286fd9..6b4bedcf9eb 100644 --- a/src/Functions/pointInPolygon.cpp +++ b/src/Functions/pointInPolygon.cpp @@ -13,6 +13,7 @@ #include #include #include +#include #include #include #include @@ -425,7 +426,15 @@ private: { out_container.reserve(end - begin); for (size_t i = begin; i < end; ++i) + { + long long result = 0; + if (common::mulOverflow(static_cast(x_data[i]), static_cast(y_data[i]), result)) + throw Exception("The coordinates of the point are such that subsequent calculations cannot be performed correctly. " \ + "Most likely they are very large in modulus.", ErrorCodes::BAD_ARGUMENTS); + out_container.emplace_back(x_data[i], y_data[i]); + } + } void parseConstPolygonWithoutHolesFromSingleColumn(const IColumn & column, size_t i, Polygon & out_polygon) const @@ -550,7 +559,7 @@ private: /// Fix orientation and close rings. It's required for subsequent processing. boost::geometry::correct(out_polygon); -#if !defined(__clang_analyzer__) && !defined(UNDEFINED_BEHAVIOR_SANITIZER) /// It does not like boost. +#if !defined(__clang_analyzer__) /// It does not like boost. if (validate) { std::string failure_message; diff --git a/tests/queries/0_stateless/01700_point_in_polygon_ubsan.sql b/tests/queries/0_stateless/01700_point_in_polygon_ubsan.sql index 97db40ab65e..3b6ad3b257e 100644 --- a/tests/queries/0_stateless/01700_point_in_polygon_ubsan.sql +++ b/tests/queries/0_stateless/01700_point_in_polygon_ubsan.sql @@ -1 +1 @@ -SELECT pointInPolygon((0, 0), [[(0, 0), (10, 10), (256, -9223372036854775808)]]) FORMAT Null; +SELECT pointInPolygon((0, 0), [[(0, 0), (10, 10), (256, -9223372036854775808)]]) FORMAT Null; -- { serverError 36 } From fb454423b6598169f0ae159cbb1ec21fbf193de9 Mon Sep 17 00:00:00 2001 From: Nikita Mikhailov Date: Wed, 17 Feb 2021 21:24:41 +0300 Subject: [PATCH 292/544] better --- src/Functions/pointInPolygon.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/Functions/pointInPolygon.cpp b/src/Functions/pointInPolygon.cpp index 6b4bedcf9eb..91c5b3a8365 100644 --- a/src/Functions/pointInPolygon.cpp +++ b/src/Functions/pointInPolygon.cpp @@ -427,8 +427,8 @@ private: out_container.reserve(end - begin); for (size_t i = begin; i < end; ++i) { - long long result = 0; - if (common::mulOverflow(static_cast(x_data[i]), static_cast(y_data[i]), result)) + int64_t result = 0; + if (common::mulOverflow(static_cast(x_data[i]), static_cast(y_data[i]), result)) throw Exception("The coordinates of the point are such that subsequent calculations cannot be performed correctly. " \ "Most likely they are very large in modulus.", ErrorCodes::BAD_ARGUMENTS); From 954c206a535e4bbde66f9726cde0e282df522e5e Mon Sep 17 00:00:00 2001 From: Nikita Mikhailov Date: Thu, 18 Feb 2021 22:51:19 +0300 Subject: [PATCH 293/544] rewrite parser --- src/DataTypes/DataTypeCustomGeo.cpp | 28 +-- src/Functions/geometryConverters.cpp | 164 ++++++++--------- src/Functions/geometryConverters.h | 226 ++++++++++-------------- src/Functions/polygonArea.cpp | 140 +++++++-------- src/Functions/polygonConvexHull.cpp | 138 +++++++-------- src/Functions/polygonPerimeter.cpp | 140 +++++++-------- src/Functions/polygonsDistance.cpp | 156 ++++++++-------- src/Functions/polygonsEquals.cpp | 150 ++++++++-------- src/Functions/polygonsIntersection.cpp | 15 +- src/Functions/polygonsSymDifference.cpp | 158 ++++++++--------- src/Functions/polygonsUnion.cpp | 164 ++++++++--------- src/Functions/polygonsWithin.cpp | 158 ++++++++--------- src/Functions/registerFunctionsGeo.cpp | 44 ++--- src/Functions/svg.cpp | 164 ++++++++--------- src/Functions/wkt.cpp | 108 +++++------ 15 files changed, 961 insertions(+), 992 deletions(-) diff --git a/src/DataTypes/DataTypeCustomGeo.cpp b/src/DataTypes/DataTypeCustomGeo.cpp index 60fe6f3450c..dd29ed21061 100644 --- a/src/DataTypes/DataTypeCustomGeo.cpp +++ b/src/DataTypes/DataTypeCustomGeo.cpp @@ -12,6 +12,20 @@ namespace DB { +namespace +{ + const auto point_data_type = std::make_shared( + DataTypes{std::make_shared(), std::make_shared()} + ); + + const auto ring_data_type = std::make_shared(DataTypeCustomPointSerialization::nestedDataType()); + + const auto polygon_data_type = std::make_shared(DataTypeCustomRingSerialization::nestedDataType()); + + const auto multipolygon_data_type = std::make_shared(DataTypeCustomPolygonSerialization::nestedDataType()); +} + + void DataTypeCustomPointSerialization::serializeText( const IColumn & column, size_t row_num, WriteBuffer & ostr, const FormatSettings & settings) const { @@ -26,10 +40,7 @@ void DataTypeCustomPointSerialization::deserializeText( DataTypePtr DataTypeCustomPointSerialization::nestedDataType() { - static const auto data_type = std::make_shared( - DataTypes{std::make_shared(), std::make_shared()} - ); - return data_type; + return point_data_type; } void DataTypeCustomRingSerialization::serializeText( @@ -46,8 +57,7 @@ void DataTypeCustomRingSerialization::deserializeText( DataTypePtr DataTypeCustomRingSerialization::nestedDataType() { - static auto data_type = std::make_shared(DataTypeCustomPointSerialization::nestedDataType()); - return data_type; + return ring_data_type; } void DataTypeCustomPolygonSerialization::serializeText( @@ -64,8 +74,7 @@ void DataTypeCustomPolygonSerialization::deserializeText( DataTypePtr DataTypeCustomPolygonSerialization::nestedDataType() { - static auto data_type = std::make_shared(DataTypeCustomRingSerialization::nestedDataType()); - return data_type; + return polygon_data_type; } void DataTypeCustomMultiPolygonSerialization::serializeText( @@ -82,8 +91,7 @@ void DataTypeCustomMultiPolygonSerialization::deserializeText( DataTypePtr DataTypeCustomMultiPolygonSerialization::nestedDataType() { - static auto data_type = std::make_shared(DataTypeCustomPolygonSerialization::nestedDataType()); - return data_type; + return multipolygon_data_type; } void registerDataTypeDomainGeo(DataTypeFactory & factory) diff --git a/src/Functions/geometryConverters.cpp b/src/Functions/geometryConverters.cpp index 8665fbb1771..13ff94e815d 100644 --- a/src/Functions/geometryConverters.cpp +++ b/src/Functions/geometryConverters.cpp @@ -13,101 +13,101 @@ namespace ErrorCodes extern const int LOGICAL_ERROR; } -namespace -{ +// namespace +// { -size_t getArrayDepth(DataTypePtr data_type, size_t max_depth) -{ - size_t depth = 0; - while (data_type && isArray(data_type) && depth != max_depth + 1) - { - depth++; - data_type = static_cast(*data_type).getNestedType(); - } +// size_t getArrayDepth(DataTypePtr data_type, size_t max_depth) +// { +// size_t depth = 0; +// while (data_type && isArray(data_type) && depth != max_depth + 1) +// { +// depth++; +// data_type = static_cast(*data_type).getNestedType(); +// } - return depth; -} +// return depth; +// } -template -class ContainerCreator : public boost::static_visitor -{ -public: - template - Geometry operator()(const T & parser) const - { - return parser.createContainer(); - } -}; +// template +// class ContainerCreator : public boost::static_visitor +// { +// public: +// template +// Geometry operator()(const T & parser) const +// { +// return parser.createContainer(); +// } +// }; -template -class Getter : public boost::static_visitor -{ -public: - constexpr Getter(Geometry & container_, size_t i_) - : container(container_) - , i(i_) - {} +// template +// class Getter : public boost::static_visitor +// { +// public: +// constexpr Getter(Geometry & container_, size_t i_) +// : container(container_) +// , i(i_) +// {} - template - void operator()(const T & parser) const - { - parser.get(container, i); - } +// template +// void operator()(const T & parser) const +// { +// parser.get(container, i); +// } -private: - Geometry & container; - size_t i; -}; +// private: +// Geometry & container; +// size_t i; +// }; -template -Parser makeParser(const ColumnWithTypeAndName & col) -{ - auto wanted_data_type = DataType::nestedDataType(); - ColumnPtr casted = castColumn(col, DataType::nestedDataType()); - if (!casted) - { - throw Exception("Failed to cast " + col.type->getName() + " to " + wanted_data_type->getName(), ErrorCodes::ILLEGAL_COLUMN); - } - return Parser(std::move(casted->convertToFullColumnIfConst())); -} +// template +// Parser makeParser(const ColumnWithTypeAndName & col) +// { +// auto wanted_data_type = DataType::nestedDataType(); +// ColumnPtr casted = castColumn(col, DataType::nestedDataType()); +// if (!casted) +// { +// throw Exception("Failed to cast " + col.type->getName() + " to " + wanted_data_type->getName(), ErrorCodes::ILLEGAL_COLUMN); +// } +// return Parser(std::move(casted->convertToFullColumnIfConst())); +// } -} +// } -template -Geometry createContainer(const GeometryFromColumnParser & parser) -{ - static ContainerCreator> creator; - return boost::apply_visitor(creator, parser); -} +// template +// Geometry createContainer(const GeometryFromColumnParser & parser) +// { +// static ContainerCreator> creator; +// return boost::apply_visitor(creator, parser); +// } -template -void get(const GeometryFromColumnParser & parser, Geometry & container, size_t i) -{ - boost::apply_visitor(Getter(container, i), parser); -} +// template +// void get(const GeometryFromColumnParser & parser, Geometry & container, size_t i) +// { +// boost::apply_visitor(Getter(container, i), parser); +// } -template -GeometryFromColumnParser makeGeometryFromColumnParser(const ColumnWithTypeAndName & col) -{ - switch (getArrayDepth(col.type, 3)) - { - case 0: return makeParser>(col); - case 1: return makeParser>(col); - case 2: return makeParser>(col); - case 3: return makeParser>(col); - default: throw Exception("Cannot parse geometry from column with type " + col.type->getName() - + ", array depth is too big", ErrorCodes::ILLEGAL_COLUMN); - } -} +// template +// GeometryFromColumnParser makeGeometryFromColumnParser(const ColumnWithTypeAndName & col) +// { +// switch (getArrayDepth(col.type, 3)) +// { +// case 0: return makeParser>(col); +// case 1: return makeParser>(col); +// case 2: return makeParser>(col); +// case 3: return makeParser>(col); +// default: throw Exception("Cannot parse geometry from column with type " + col.type->getName() +// + ", array depth is too big", ErrorCodes::ILLEGAL_COLUMN); +// } +// } -/// Explicit instantiations to avoid linker errors. +// /// Explicit instantiations to avoid linker errors. -template Geometry createContainer(const GeometryFromColumnParser &); -template Geometry createContainer(const GeometryFromColumnParser &); -template void get(const GeometryFromColumnParser & parser, Geometry & container, size_t i); -template void get(const GeometryFromColumnParser & parser, Geometry & container, size_t i); -template GeometryFromColumnParser makeGeometryFromColumnParser(const ColumnWithTypeAndName & col); -template GeometryFromColumnParser makeGeometryFromColumnParser(const ColumnWithTypeAndName & col); +// template Geometry createContainer(const GeometryFromColumnParser &); +// template Geometry createContainer(const GeometryFromColumnParser &); +// template void get(const GeometryFromColumnParser & parser, Geometry & container, size_t i); +// template void get(const GeometryFromColumnParser & parser, Geometry & container, size_t i); +// template GeometryFromColumnParser makeGeometryFromColumnParser(const ColumnWithTypeAndName & col); +// template GeometryFromColumnParser makeGeometryFromColumnParser(const ColumnWithTypeAndName & col); template typename Desired> diff --git a/src/Functions/geometryConverters.h b/src/Functions/geometryConverters.h index 14d186433b2..acb5de38a09 100644 --- a/src/Functions/geometryConverters.h +++ b/src/Functions/geometryConverters.h @@ -17,6 +17,9 @@ #include #include +// TODO: maybe use isInfinite from clickhouse codebase +#include + #include namespace DB @@ -28,28 +31,26 @@ namespace ErrorCodes extern const int ILLEGAL_TYPE_OF_ARGUMENT; } -namespace bg = boost::geometry; +template +using Ring = boost::geometry::model::ring; template -using Ring = bg::model::ring; +using Polygon = boost::geometry::model::polygon; template -using Polygon = bg::model::polygon; - -template -using MultiPolygon = bg::model::multi_polygon>; +using MultiPolygon = boost::geometry::model::multi_polygon>; template using Geometry = boost::variant, Polygon, MultiPolygon>; -using CartesianPoint = bg::model::d2::point_xy; +using CartesianPoint = boost::geometry::model::d2::point_xy; using CartesianRing = Ring; using CartesianPolygon = Polygon; using CartesianMultiPolygon = MultiPolygon; using CartesianGeometry = Geometry; -using GeographicPoint = bg::model::point>; +using GeographicPoint = boost::geometry::model::point>; using GeographicRing = Ring; using GeographicPolygon = Polygon; using GeographicMultiPolygon = MultiPolygon; @@ -63,114 +64,76 @@ template class PointFromColumnParser { public: + using Container = std::conditional_t, CartesianGeometry, GeographicGeometry>; + explicit PointFromColumnParser(ColumnPtr col_) : col(col_) { - const auto & tuple = dynamic_cast(*col_); - const auto & tuple_columns = tuple.getColumns(); - -#ifndef NDEBUG - size = tuple.size(); -#endif - const auto & x_data = dynamic_cast(*tuple_columns[0]); - first = x_data.getData().data(); - - const auto & y_data = dynamic_cast(*tuple_columns[1]); - second = y_data.getData().data(); } - template - typename std::enable_if_t, CartesianGeometry> createContainer() const + std::vector parse(size_t shift, size_t count) const { - return CartesianPoint(); - } + const auto * tuple = typeid_cast(col.get()); + const auto & tuple_columns = tuple->getColumns(); - template - typename std::enable_if_t, GeographicGeometry> createContainer() const - { - return GeographicPoint(); - } + const auto * x_data = typeid_cast(tuple_columns[0].get()); + const auto * y_data = typeid_cast(tuple_columns[1].get()); - template - void get(std::enable_if_t, CartesianGeometry> & container, size_t i) const - { -#ifndef NDEBUG - assert(i < size); -#endif - get(boost::get(container), i); - } + const auto * first_container = x_data->getData().data() + shift; + const auto * second_container = y_data->getData().data() + shift; - template - void get(std::enable_if_t, GeographicGeometry> & container, size_t i) const - { -#ifndef NDEBUG - assert(i < size); -#endif - get(boost::get(container), i); - } + std::vector answer(count); - void get(PointType & container, size_t i) const - { -#ifndef NDEBUG - assert(i < size); -#endif - if (isNaN(first[i]) || isNaN(second[i])) - throw Exception("Point's component must not be NaN", ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT); + for (size_t i = 0; i < count; ++i) + { + const Float64 first = first_container[i]; + const Float64 second = second_container[i]; - boost::geometry::set<0>(container, first[i]); - boost::geometry::set<1>(container, second[i]); + if (isNaN(first) || isNaN(second)) + throw Exception("Point's component must not be NaN", ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT); + + if (isinf(first) || isinf(second)) + throw Exception("Point's component must not be infinite", ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT); + + answer[i] = PointType(first, second); + } + + return answer; } private: /// To prevent use-after-free and increase column lifetime. ColumnPtr col; -#ifndef NDEBUG - size_t size; -#endif - const Float64 * first; - const Float64 * second; }; + + template class RingFromColumnParser { public: explicit RingFromColumnParser(ColumnPtr col_) : col(col_) - , offsets(dynamic_cast(*col_).getOffsets()) - , point_parser(dynamic_cast(*col_).getDataPtr()) + , offsets(typeid_cast(*col_).getOffsets()) + , point_parser(typeid_cast(*col_).getDataPtr()) { } - Geometry createContainer() const + std::vector> parse(size_t shift, size_t /*size*/) const { - return Ring(); - } + size_t prev_offset = shift; - void get(Geometry & container, size_t i) const - { - get(boost::get>(container), i); - } + std::vector> answer; + answer.reserve(offsets.size()); - void get(Ring & container, size_t i) const - { - size_t left = i == 0 ? 0 : offsets[i - 1]; - size_t right = offsets[i]; - - if (left == right) - throw Exception("Empty polygons are not allowed in line " + toString(i), ErrorCodes::BAD_ARGUMENTS); - - // reserve extra point for case when polygon is open - container.reserve(right - left + 1); - container.resize(right - left); - - for (size_t j = left; j < right; j++) - point_parser.get(container[j - left], j); - - // make ring closed - if (!boost::geometry::equals(container[0], container.back())) + for (size_t offset : offsets) { - container.push_back(container[0]); + offset += shift; + auto points = point_parser.parse(prev_offset, offset - prev_offset); + answer.emplace_back(points.begin(), points.end()); + prev_offset = offset; } + + return answer; } private: @@ -186,32 +149,28 @@ class PolygonFromColumnParser public: explicit PolygonFromColumnParser(ColumnPtr col_) : col(col_) - , offsets(static_cast(*col_).getOffsets()) - , ring_parser(static_cast(*col_).getDataPtr()) + , offsets(typeid_cast(*col_).getOffsets()) + , ring_parser(typeid_cast(*col_).getDataPtr()) {} - Geometry createContainer() const + std::vector> parse(size_t shift, size_t /*size*/) const { - return Polygon(); - } + size_t prev_offset = shift; + std::vector> answer(offsets.size()); + size_t iter = 0; - void get(Geometry & container, size_t i) const - { - get(boost::get>(container), i); - } - - void get(Polygon & container, size_t i) const - { - size_t l = offsets[i - 1]; - size_t r = offsets[i]; - - ring_parser.get(container.outer(), l); - - container.inners().resize(r - l - 1); - for (size_t j = l + 1; j < r; j++) + for (size_t offset : offsets) { - ring_parser.get(container.inners()[j - l - 1], j); + offset += shift; + auto tmp = ring_parser.parse(prev_offset, offset - prev_offset); + + /// FIXME: other rings are holes in first + answer[iter].outer() = tmp[0]; + prev_offset = offset; + ++iter; } + + return answer; } private: @@ -227,30 +186,29 @@ class MultiPolygonFromColumnParser public: explicit MultiPolygonFromColumnParser(ColumnPtr col_) : col(col_) - , offsets(static_cast(*col_).getOffsets()) - , polygon_parser(static_cast(*col_).getDataPtr()) + , offsets(typeid_cast(*col_).getOffsets()) + , polygon_parser(typeid_cast(*col_).getDataPtr()) {} - Geometry createContainer() const + std::vector> parse(size_t shift, size_t /*size*/) const { - return MultiPolygon(); - } + size_t prev_offset = shift; + + std::vector> answer; + answer.resize(offsets.size()); - void get(Geometry & container, size_t i) const - { - get(boost::get>(container), i); - } + size_t iter = 0; - void get(MultiPolygon & container, size_t i) const - { - size_t l = offsets[i - 1]; - size_t r = offsets[i]; - - container.resize(r - l); - for (size_t j = l; j < r; j++) + for (size_t offset : offsets) { - polygon_parser.get(container[j - l], j); + offset += shift; + auto polygons = polygon_parser.parse(prev_offset, offset - prev_offset); + answer[iter].swap(polygons); + prev_offset = offset; + ++iter; } + + return answer; } private: @@ -268,22 +226,22 @@ using GeometryFromColumnParser = boost::variant< MultiPolygonFromColumnParser >; -template -Geometry createContainer(const GeometryFromColumnParser & parser); +// template +// Geometry createContainer(const GeometryFromColumnParser & parser); -template -void get(const GeometryFromColumnParser & parser, Geometry & container, size_t i); +// template +// void get(const GeometryFromColumnParser & parser, Geometry & container, size_t i); -template -GeometryFromColumnParser makeGeometryFromColumnParser(const ColumnWithTypeAndName & col); +// template +// GeometryFromColumnParser makeGeometryFromColumnParser(const ColumnWithTypeAndName & col); -extern template Geometry createContainer(const GeometryFromColumnParser & parser); -extern template Geometry createContainer(const GeometryFromColumnParser & parser); -extern template void get(const GeometryFromColumnParser & parser, Geometry & container, size_t i); -extern template void get(const GeometryFromColumnParser & parser, Geometry & container, size_t i); -extern template GeometryFromColumnParser makeGeometryFromColumnParser(const ColumnWithTypeAndName & col); -extern template GeometryFromColumnParser makeGeometryFromColumnParser(const ColumnWithTypeAndName & col); +// extern template Geometry createContainer(const GeometryFromColumnParser & parser); +// extern template Geometry createContainer(const GeometryFromColumnParser & parser); +// extern template void get(const GeometryFromColumnParser & parser, Geometry & container, size_t i); +// extern template void get(const GeometryFromColumnParser & parser, Geometry & container, size_t i); +// extern template GeometryFromColumnParser makeGeometryFromColumnParser(const ColumnWithTypeAndName & col); +// extern template GeometryFromColumnParser makeGeometryFromColumnParser(const ColumnWithTypeAndName & col); /// To serialize Geographic or Cartesian point (a pair of numbers in both cases). template diff --git a/src/Functions/polygonArea.cpp b/src/Functions/polygonArea.cpp index e3ecc9d6fac..ee06e41e0cb 100644 --- a/src/Functions/polygonArea.cpp +++ b/src/Functions/polygonArea.cpp @@ -1,93 +1,93 @@ -#include -#include +// #include +// #include -#include -#include -#include +// #include +// #include +// #include -#include +// #include -#include -#include -#include -#include -#include -#include +// #include +// #include +// #include +// #include +// #include +// #include -#include -#include +// #include +// #include -namespace DB -{ +// namespace DB +// { -template -class FunctionPolygonArea : public IFunction -{ -public: - static inline const char * name; +// template +// class FunctionPolygonArea : public IFunction +// { +// public: +// static inline const char * name; - explicit FunctionPolygonArea() = default; +// explicit FunctionPolygonArea() = default; - static FunctionPtr create(const Context &) - { - return std::make_shared(); - } +// static FunctionPtr create(const Context &) +// { +// return std::make_shared(); +// } - String getName() const override - { - return name; - } +// String getName() const override +// { +// return name; +// } - bool isVariadic() const override - { - return false; - } +// bool isVariadic() const override +// { +// return false; +// } - size_t getNumberOfArguments() const override - { - return 1; - } +// size_t getNumberOfArguments() const override +// { +// return 1; +// } - DataTypePtr getReturnTypeImpl(const DataTypes &) const override - { - return std::make_shared(); - } +// DataTypePtr getReturnTypeImpl(const DataTypes &) const override +// { +// return std::make_shared(); +// } - ColumnPtr executeImpl(const ColumnsWithTypeAndName & arguments, const DataTypePtr & /*result_type*/, size_t input_rows_count) const override - { - checkColumnTypeOrThrow(arguments[0]); - auto parser = MultiPolygonFromColumnParser(std::move(arguments[0].column->convertToFullColumnIfConst())); - MultiPolygon container; +// ColumnPtr executeImpl(const ColumnsWithTypeAndName & arguments, const DataTypePtr & /*result_type*/, size_t input_rows_count) const override +// { +// checkColumnTypeOrThrow(arguments[0]); +// auto parser = MultiPolygonFromColumnParser(std::move(arguments[0].column->convertToFullColumnIfConst())); +// MultiPolygon container; - auto res_column = ColumnFloat64::create(); +// auto res_column = ColumnFloat64::create(); - for (size_t i = 0; i < input_rows_count; i++) - { - parser.get(container, i); - res_column->insertValue(boost::geometry::area(container)); - } +// for (size_t i = 0; i < input_rows_count; i++) +// { +// parser.get(container, i); +// res_column->insertValue(boost::geometry::area(container)); +// } - return res_column; - } +// return res_column; +// } - bool useDefaultImplementationForConstants() const override - { - return true; - } -}; +// bool useDefaultImplementationForConstants() const override +// { +// return true; +// } +// }; -template <> -const char * FunctionPolygonArea::name = "polygonAreaCartesian"; +// template <> +// const char * FunctionPolygonArea::name = "polygonAreaCartesian"; -template <> -const char * FunctionPolygonArea::name = "polygonAreaGeographic"; +// template <> +// const char * FunctionPolygonArea::name = "polygonAreaGeographic"; -void registerFunctionPolygonArea(FunctionFactory & factory) -{ - factory.registerFunction>(); - factory.registerFunction>(); -} +// void registerFunctionPolygonArea(FunctionFactory & factory) +// { +// factory.registerFunction>(); +// factory.registerFunction>(); +// } -} +// } diff --git a/src/Functions/polygonConvexHull.cpp b/src/Functions/polygonConvexHull.cpp index 3bc4cd1cf2d..6a68f1f92ba 100644 --- a/src/Functions/polygonConvexHull.cpp +++ b/src/Functions/polygonConvexHull.cpp @@ -1,91 +1,91 @@ -#include -#include +// #include +// #include -#include -#include -#include +// #include +// #include +// #include -#include +// #include -#include -#include -#include -#include -#include -#include +// #include +// #include +// #include +// #include +// #include +// #include -#include -#include +// #include +// #include -namespace DB -{ +// namespace DB +// { -template -class FunctionPolygonConvexHull : public IFunction -{ -public: - static const char * name; +// template +// class FunctionPolygonConvexHull : public IFunction +// { +// public: +// static const char * name; - explicit FunctionPolygonConvexHull() = default; +// explicit FunctionPolygonConvexHull() = default; - static FunctionPtr create(const Context &) - { - return std::make_shared(); - } +// static FunctionPtr create(const Context &) +// { +// return std::make_shared(); +// } - String getName() const override - { - return name; - } +// String getName() const override +// { +// return name; +// } - bool isVariadic() const override - { - return false; - } +// bool isVariadic() const override +// { +// return false; +// } - size_t getNumberOfArguments() const override - { - return 1; - } +// size_t getNumberOfArguments() const override +// { +// return 1; +// } - DataTypePtr getReturnTypeImpl(const DataTypes &) const override - { - return DataTypeCustomPolygonSerialization::nestedDataType(); - } +// DataTypePtr getReturnTypeImpl(const DataTypes &) const override +// { +// return DataTypeCustomPolygonSerialization::nestedDataType(); +// } - ColumnPtr executeImpl(const ColumnsWithTypeAndName & arguments, const DataTypePtr & /*result_type*/, size_t input_rows_count) const override - { - checkColumnTypeOrThrow(arguments[0]); - auto parser = MultiPolygonFromColumnParser(std::move(arguments[0].column->convertToFullColumnIfConst())); - MultiPolygon container; +// ColumnPtr executeImpl(const ColumnsWithTypeAndName & arguments, const DataTypePtr & /*result_type*/, size_t input_rows_count) const override +// { +// checkColumnTypeOrThrow(arguments[0]); +// auto parser = MultiPolygonFromColumnParser(std::move(arguments[0].column->convertToFullColumnIfConst())); +// MultiPolygon container; - PolygonSerializer serializer; - Polygon convex_hull{}; +// PolygonSerializer serializer; +// Polygon convex_hull{}; - for (size_t i = 0; i < input_rows_count; i++) - { - parser.get(container, i); - boost::geometry::convex_hull(container, convex_hull); - serializer.add(convex_hull); - } +// for (size_t i = 0; i < input_rows_count; i++) +// { +// parser.get(container, i); +// boost::geometry::convex_hull(container, convex_hull); +// serializer.add(convex_hull); +// } - return serializer.finalize(); - } +// return serializer.finalize(); +// } - bool useDefaultImplementationForConstants() const override - { - return true; - } -}; +// bool useDefaultImplementationForConstants() const override +// { +// return true; +// } +// }; -template <> -const char * FunctionPolygonConvexHull::name = "polygonConvexHullCartesian"; +// template <> +// const char * FunctionPolygonConvexHull::name = "polygonConvexHullCartesian"; -void registerFunctionPolygonConvexHull(FunctionFactory & factory) -{ - factory.registerFunction>(); -} +// void registerFunctionPolygonConvexHull(FunctionFactory & factory) +// { +// factory.registerFunction>(); +// } -} +// } diff --git a/src/Functions/polygonPerimeter.cpp b/src/Functions/polygonPerimeter.cpp index 2f277af7fd5..db1ae0b7178 100644 --- a/src/Functions/polygonPerimeter.cpp +++ b/src/Functions/polygonPerimeter.cpp @@ -1,93 +1,93 @@ -#include -#include +// #include +// #include -#include -#include -#include +// #include +// #include +// #include -#include +// #include -#include -#include -#include -#include -#include -#include +// #include +// #include +// #include +// #include +// #include +// #include -#include -#include +// #include +// #include -namespace DB -{ +// namespace DB +// { -template -class FunctionPolygonPerimeter : public IFunction -{ -public: - static const char * name; +// template +// class FunctionPolygonPerimeter : public IFunction +// { +// public: +// static const char * name; - explicit FunctionPolygonPerimeter() = default; +// explicit FunctionPolygonPerimeter() = default; - static FunctionPtr create(const Context &) - { - return std::make_shared(); - } +// static FunctionPtr create(const Context &) +// { +// return std::make_shared(); +// } - String getName() const override - { - return name; - } +// String getName() const override +// { +// return name; +// } - bool isVariadic() const override - { - return false; - } +// bool isVariadic() const override +// { +// return false; +// } - size_t getNumberOfArguments() const override - { - return 1; - } +// size_t getNumberOfArguments() const override +// { +// return 1; +// } - DataTypePtr getReturnTypeImpl(const DataTypes &) const override - { - return std::make_shared(); - } +// DataTypePtr getReturnTypeImpl(const DataTypes &) const override +// { +// return std::make_shared(); +// } - ColumnPtr executeImpl(const ColumnsWithTypeAndName & arguments, const DataTypePtr & /*result_type*/, size_t input_rows_count) const override - { - checkColumnTypeOrThrow(arguments[0]); - auto parser = MultiPolygonFromColumnParser(std::move(arguments[0].column->convertToFullColumnIfConst())); - MultiPolygon container; +// ColumnPtr executeImpl(const ColumnsWithTypeAndName & arguments, const DataTypePtr & /*result_type*/, size_t input_rows_count) const override +// { +// checkColumnTypeOrThrow(arguments[0]); +// auto parser = MultiPolygonFromColumnParser(std::move(arguments[0].column->convertToFullColumnIfConst())); +// MultiPolygon container; - auto res_column = ColumnFloat64::create(); +// auto res_column = ColumnFloat64::create(); - for (size_t i = 0; i < input_rows_count; i++) - { - parser.get(container, i); - res_column->insertValue(boost::geometry::perimeter(container)); - } +// for (size_t i = 0; i < input_rows_count; i++) +// { +// parser.get(container, i); +// res_column->insertValue(boost::geometry::perimeter(container)); +// } - return res_column; - } +// return res_column; +// } - bool useDefaultImplementationForConstants() const override - { - return true; - } -}; +// bool useDefaultImplementationForConstants() const override +// { +// return true; +// } +// }; -template <> -const char * FunctionPolygonPerimeter::name = "polygonPerimeterCartesian"; +// template <> +// const char * FunctionPolygonPerimeter::name = "polygonPerimeterCartesian"; -template <> -const char * FunctionPolygonPerimeter::name = "polygonPerimeterGeographic"; +// template <> +// const char * FunctionPolygonPerimeter::name = "polygonPerimeterGeographic"; -void registerFunctionPolygonPerimeter(FunctionFactory & factory) -{ - factory.registerFunction>(); - factory.registerFunction>(); -} +// void registerFunctionPolygonPerimeter(FunctionFactory & factory) +// { +// factory.registerFunction>(); +// factory.registerFunction>(); +// } -} +// } diff --git a/src/Functions/polygonsDistance.cpp b/src/Functions/polygonsDistance.cpp index dea7ffcce70..5e1ebd45309 100644 --- a/src/Functions/polygonsDistance.cpp +++ b/src/Functions/polygonsDistance.cpp @@ -1,104 +1,104 @@ -#include -#include +// #include +// #include -#include -#include -#include +// #include +// #include +// #include -#include +// #include -#include -#include -#include -#include -#include -#include -#include -#include +// #include +// #include +// #include +// #include +// #include +// #include +// #include +// #include -#include -#include +// #include +// #include -namespace DB -{ +// namespace DB +// { -template -class FunctionPolygonsDistance : public IFunction -{ -public: - static inline const char * name; +// template +// class FunctionPolygonsDistance : public IFunction +// { +// public: +// static inline const char * name; - explicit FunctionPolygonsDistance() = default; +// explicit FunctionPolygonsDistance() = default; - static FunctionPtr create(const Context &) - { - return std::make_shared(); - } +// static FunctionPtr create(const Context &) +// { +// return std::make_shared(); +// } - String getName() const override - { - return name; - } +// String getName() const override +// { +// return name; +// } - bool isVariadic() const override - { - return false; - } +// bool isVariadic() const override +// { +// return false; +// } - size_t getNumberOfArguments() const override - { - return 2; - } +// size_t getNumberOfArguments() const override +// { +// return 2; +// } - DataTypePtr getReturnTypeImpl(const DataTypes &) const override - { - return std::make_shared(); - } +// DataTypePtr getReturnTypeImpl(const DataTypes &) const override +// { +// return std::make_shared(); +// } - ColumnPtr executeImpl(const ColumnsWithTypeAndName & arguments, const DataTypePtr & /*result_type*/, size_t input_rows_count) const override - { - checkColumnTypeOrThrow(arguments[0]); - auto first_parser = MultiPolygonFromColumnParser(std::move(arguments[0].column->convertToFullColumnIfConst())); - MultiPolygon first_container; +// ColumnPtr executeImpl(const ColumnsWithTypeAndName & arguments, const DataTypePtr & /*result_type*/, size_t input_rows_count) const override +// { +// checkColumnTypeOrThrow(arguments[0]); +// auto first_parser = MultiPolygonFromColumnParser(std::move(arguments[0].column->convertToFullColumnIfConst())); +// MultiPolygon first_container; - checkColumnTypeOrThrow(arguments[1]); - auto second_parser = MultiPolygonFromColumnParser(std::move(arguments[1].column->convertToFullColumnIfConst())); - MultiPolygon second_container; +// checkColumnTypeOrThrow(arguments[1]); +// auto second_parser = MultiPolygonFromColumnParser(std::move(arguments[1].column->convertToFullColumnIfConst())); +// MultiPolygon second_container; - auto res_column = ColumnFloat64::create(); +// auto res_column = ColumnFloat64::create(); - for (size_t i = 0; i < input_rows_count; i++) - { - first_parser.get(first_container, i); - second_parser.get(second_container, i); +// for (size_t i = 0; i < input_rows_count; i++) +// { +// first_parser.get(first_container, i); +// second_parser.get(second_container, i); - boost::geometry::correct(first_container); - boost::geometry::correct(second_container); +// boost::geometry::correct(first_container); +// boost::geometry::correct(second_container); - res_column->insertValue(boost::geometry::distance(first_container, second_container)); - } +// res_column->insertValue(boost::geometry::distance(first_container, second_container)); +// } - return res_column; - } +// return res_column; +// } - bool useDefaultImplementationForConstants() const override - { - return true; - } -}; +// bool useDefaultImplementationForConstants() const override +// { +// return true; +// } +// }; -template <> -const char * FunctionPolygonsDistance::name = "polygonsDistanceCartesian"; +// template <> +// const char * FunctionPolygonsDistance::name = "polygonsDistanceCartesian"; -template <> -const char * FunctionPolygonsDistance::name = "polygonsDistanceGeographic"; +// template <> +// const char * FunctionPolygonsDistance::name = "polygonsDistanceGeographic"; -void registerFunctionPolygonsDistance(FunctionFactory & factory) -{ - factory.registerFunction>(); - factory.registerFunction>(); -} +// void registerFunctionPolygonsDistance(FunctionFactory & factory) +// { +// factory.registerFunction>(); +// factory.registerFunction>(); +// } -} +// } diff --git a/src/Functions/polygonsEquals.cpp b/src/Functions/polygonsEquals.cpp index 4ab81597b70..ac6b9912481 100644 --- a/src/Functions/polygonsEquals.cpp +++ b/src/Functions/polygonsEquals.cpp @@ -1,100 +1,100 @@ -#include -#include +// #include +// #include -#include -#include -#include +// #include +// #include +// #include -#include +// #include -#include -#include -#include -#include -#include -#include -#include -#include +// #include +// #include +// #include +// #include +// #include +// #include +// #include +// #include -#include -#include +// #include +// #include -namespace DB -{ +// namespace DB +// { -template -class FunctionPolygonsEquals : public IFunction -{ -public: - static const char * name; +// template +// class FunctionPolygonsEquals : public IFunction +// { +// public: +// static const char * name; - explicit FunctionPolygonsEquals() = default; +// explicit FunctionPolygonsEquals() = default; - static FunctionPtr create(const Context &) - { - return std::make_shared(); - } +// static FunctionPtr create(const Context &) +// { +// return std::make_shared(); +// } - String getName() const override - { - return name; - } +// String getName() const override +// { +// return name; +// } - bool isVariadic() const override - { - return false; - } +// bool isVariadic() const override +// { +// return false; +// } - size_t getNumberOfArguments() const override - { - return 2; - } +// size_t getNumberOfArguments() const override +// { +// return 2; +// } - DataTypePtr getReturnTypeImpl(const DataTypes &) const override - { - return std::make_shared(); - } +// DataTypePtr getReturnTypeImpl(const DataTypes &) const override +// { +// return std::make_shared(); +// } - ColumnPtr executeImpl(const ColumnsWithTypeAndName & arguments, const DataTypePtr & /*result_type*/, size_t input_rows_count) const override - { - checkColumnTypeOrThrow(arguments[0]); - auto first_parser = MultiPolygonFromColumnParser(std::move(arguments[0].column->convertToFullColumnIfConst())); - MultiPolygon first_container; +// ColumnPtr executeImpl(const ColumnsWithTypeAndName & arguments, const DataTypePtr & /*result_type*/, size_t input_rows_count) const override +// { +// checkColumnTypeOrThrow(arguments[0]); +// auto first_parser = MultiPolygonFromColumnParser(std::move(arguments[0].column->convertToFullColumnIfConst())); +// MultiPolygon first_container; - checkColumnTypeOrThrow(arguments[1]); - auto second_parser = MultiPolygonFromColumnParser(std::move(arguments[1].column->convertToFullColumnIfConst())); - MultiPolygon second_container; +// checkColumnTypeOrThrow(arguments[1]); +// auto second_parser = MultiPolygonFromColumnParser(std::move(arguments[1].column->convertToFullColumnIfConst())); +// MultiPolygon second_container; - auto res_column = ColumnUInt8::create(); +// auto res_column = ColumnUInt8::create(); - for (size_t i = 0; i < input_rows_count; i++) - { - first_parser.get(first_container, i); - second_parser.get(second_container, i); +// for (size_t i = 0; i < input_rows_count; i++) +// { +// first_parser.get(first_container, i); +// second_parser.get(second_container, i); - boost::geometry::correct(first_container); - boost::geometry::correct(second_container); +// boost::geometry::correct(first_container); +// boost::geometry::correct(second_container); - res_column->insertValue(boost::geometry::equals(first_container, second_container)); - } +// res_column->insertValue(boost::geometry::equals(first_container, second_container)); +// } - return res_column; - } +// return res_column; +// } - bool useDefaultImplementationForConstants() const override - { - return true; - } -}; +// bool useDefaultImplementationForConstants() const override +// { +// return true; +// } +// }; -template <> -const char * FunctionPolygonsEquals::name = "polygonsEqualsCartesian"; +// template <> +// const char * FunctionPolygonsEquals::name = "polygonsEqualsCartesian"; -void registerFunctionPolygonsEquals(FunctionFactory & factory) -{ - factory.registerFunction>(); -} +// void registerFunctionPolygonsEquals(FunctionFactory & factory) +// { +// factory.registerFunction>(); +// } -} +// } diff --git a/src/Functions/polygonsIntersection.cpp b/src/Functions/polygonsIntersection.cpp index 78e7396e082..129ac2e23e6 100644 --- a/src/Functions/polygonsIntersection.cpp +++ b/src/Functions/polygonsIntersection.cpp @@ -16,6 +16,7 @@ #include #include +#include namespace DB { @@ -57,28 +58,30 @@ public: { checkColumnTypeOrThrow(arguments[0]); auto first_parser = MultiPolygonFromColumnParser(std::move(arguments[0].column->convertToFullColumnIfConst())); - MultiPolygon first_container; + checkColumnTypeOrThrow(arguments[1]); - auto second_parser = MultiPolygonFromColumnParser(std::move(arguments[1].column->convertToFullColumnIfConst())); - MultiPolygon second_container; + auto second_parser = MultiPolygonFromColumnParser(std::move(arguments[1].column->convertToFullColumnIfConst())); MultiPolygonSerializer serializer; MultiPolygon intersection{}; + auto first = first_parser.parse(0, 0); + auto second = second_parser.parse(0, 0); + /// We are not interested in some pitfalls in third-party libraries /// NOLINTNEXTLINE(clang-analyzer-core.uninitialized.Assign) for (size_t i = 0; i < input_rows_count; ++i) { - first_parser.get(first_container, i); - second_parser.get(second_container, i); + MultiPolygon first_container = first[i]; + MultiPolygon second_container = second[i]; /// Orient the polygons correctly. boost::geometry::correct(first_container); boost::geometry::correct(second_container); /// Main work here. - boost::geometry::intersection(first_container, second_container, intersection); + boost::geometry::intersection(first_container[0], second_container[0], intersection); serializer.add(intersection); } diff --git a/src/Functions/polygonsSymDifference.cpp b/src/Functions/polygonsSymDifference.cpp index f8660de815d..6f0ee30a235 100644 --- a/src/Functions/polygonsSymDifference.cpp +++ b/src/Functions/polygonsSymDifference.cpp @@ -1,104 +1,104 @@ -#include -#include +// #include +// #include -#include -#include -#include +// #include +// #include +// #include -#include +// #include -#include -#include -#include -#include -#include -#include +// #include +// #include +// #include +// #include +// #include +// #include -#include -#include +// #include +// #include -namespace DB -{ +// namespace DB +// { -template -class FunctionPolygonsSymDifference : public IFunction -{ -public: - static const char * name; +// template +// class FunctionPolygonsSymDifference : public IFunction +// { +// public: +// static const char * name; - explicit FunctionPolygonsSymDifference() = default; +// explicit FunctionPolygonsSymDifference() = default; - static FunctionPtr create(const Context &) - { - return std::make_shared(); - } +// static FunctionPtr create(const Context &) +// { +// return std::make_shared(); +// } - String getName() const override - { - return name; - } +// String getName() const override +// { +// return name; +// } - bool isVariadic() const override - { - return false; - } +// bool isVariadic() const override +// { +// return false; +// } - size_t getNumberOfArguments() const override - { - return 2; - } +// size_t getNumberOfArguments() const override +// { +// return 2; +// } - DataTypePtr getReturnTypeImpl(const DataTypes &) const override - { - return DataTypeCustomMultiPolygonSerialization::nestedDataType(); - } +// DataTypePtr getReturnTypeImpl(const DataTypes &) const override +// { +// return DataTypeCustomMultiPolygonSerialization::nestedDataType(); +// } - ColumnPtr executeImpl(const ColumnsWithTypeAndName & arguments, const DataTypePtr & /*result_type*/, size_t input_rows_count) const override - { - checkColumnTypeOrThrow(arguments[0]); - auto first_parser = MultiPolygonFromColumnParser(std::move(arguments[0].column->convertToFullColumnIfConst())); - MultiPolygon first_container; +// ColumnPtr executeImpl(const ColumnsWithTypeAndName & arguments, const DataTypePtr & /*result_type*/, size_t input_rows_count) const override +// { +// checkColumnTypeOrThrow(arguments[0]); +// auto first_parser = MultiPolygonFromColumnParser(std::move(arguments[0].column->convertToFullColumnIfConst())); +// MultiPolygon first_container; - checkColumnTypeOrThrow(arguments[1]); - auto second_parser = MultiPolygonFromColumnParser(std::move(arguments[1].column->convertToFullColumnIfConst())); - MultiPolygon second_container; +// checkColumnTypeOrThrow(arguments[1]); +// auto second_parser = MultiPolygonFromColumnParser(std::move(arguments[1].column->convertToFullColumnIfConst())); +// MultiPolygon second_container; - MultiPolygonSerializer serializer; - MultiPolygon sym_difference{}; +// MultiPolygonSerializer serializer; +// MultiPolygon sym_difference{}; - /// NOLINTNEXTLINE(clang-analyzer-core.uninitialized.Assign) - for (size_t i = 0; i < input_rows_count; i++) - { - first_parser.get(first_container, i); - second_parser.get(second_container, i); +// /// NOLINTNEXTLINE(clang-analyzer-core.uninitialized.Assign) +// for (size_t i = 0; i < input_rows_count; i++) +// { +// first_parser.get(first_container, i); +// second_parser.get(second_container, i); - boost::geometry::correct(first_container); - boost::geometry::correct(second_container); +// boost::geometry::correct(first_container); +// boost::geometry::correct(second_container); - boost::geometry::sym_difference(first_container, second_container, sym_difference); +// boost::geometry::sym_difference(first_container, second_container, sym_difference); - serializer.add(sym_difference); - } +// serializer.add(sym_difference); +// } - return serializer.finalize(); - } +// return serializer.finalize(); +// } - bool useDefaultImplementationForConstants() const override - { - return true; - } -}; +// bool useDefaultImplementationForConstants() const override +// { +// return true; +// } +// }; -template <> -const char * FunctionPolygonsSymDifference::name = "polygonsSymDifferenceCartesian"; +// template <> +// const char * FunctionPolygonsSymDifference::name = "polygonsSymDifferenceCartesian"; -template <> -const char * FunctionPolygonsSymDifference::name = "polygonsSymDifferenceGeographic"; +// template <> +// const char * FunctionPolygonsSymDifference::name = "polygonsSymDifferenceGeographic"; -void registerFunctionPolygonsSymDifference(FunctionFactory & factory) -{ - factory.registerFunction>(); - factory.registerFunction>(); -} +// void registerFunctionPolygonsSymDifference(FunctionFactory & factory) +// { +// factory.registerFunction>(); +// factory.registerFunction>(); +// } -} +// } diff --git a/src/Functions/polygonsUnion.cpp b/src/Functions/polygonsUnion.cpp index f86706ad4c6..d6ac086abc2 100644 --- a/src/Functions/polygonsUnion.cpp +++ b/src/Functions/polygonsUnion.cpp @@ -1,108 +1,108 @@ -#include -#include +// #include +// #include -#include -#include -#include +// #include +// #include +// #include -#include +// #include -#include -#include -#include -#include -#include -#include +// #include +// #include +// #include +// #include +// #include +// #include -#include -#include +// #include +// #include -namespace DB -{ +// namespace DB +// { -template -class FunctionPolygonsUnion : public IFunction -{ -public: - static inline const char * name; +// template +// class FunctionPolygonsUnion : public IFunction +// { +// public: +// static inline const char * name; - explicit FunctionPolygonsUnion() = default; +// explicit FunctionPolygonsUnion() = default; - static FunctionPtr create(const Context &) - { - return std::make_shared(); - } +// static FunctionPtr create(const Context &) +// { +// return std::make_shared(); +// } - String getName() const override - { - return name; - } +// String getName() const override +// { +// return name; +// } - bool isVariadic() const override - { - return false; - } +// bool isVariadic() const override +// { +// return false; +// } - size_t getNumberOfArguments() const override - { - return 2; - } +// size_t getNumberOfArguments() const override +// { +// return 2; +// } - DataTypePtr getReturnTypeImpl(const DataTypes &) const override - { - return DataTypeCustomMultiPolygonSerialization::nestedDataType(); - } +// DataTypePtr getReturnTypeImpl(const DataTypes &) const override +// { +// return DataTypeCustomMultiPolygonSerialization::nestedDataType(); +// } - ColumnPtr executeImpl(const ColumnsWithTypeAndName & arguments, const DataTypePtr & /*result_type*/, size_t input_rows_count) const override - { - checkColumnTypeOrThrow(arguments[0]); - auto first_parser = MultiPolygonFromColumnParser(std::move(arguments[0].column->convertToFullColumnIfConst())); - MultiPolygon first_container; +// ColumnPtr executeImpl(const ColumnsWithTypeAndName & arguments, const DataTypePtr & /*result_type*/, size_t input_rows_count) const override +// { +// checkColumnTypeOrThrow(arguments[0]); +// auto first_parser = MultiPolygonFromColumnParser(std::move(arguments[0].column->convertToFullColumnIfConst())); +// MultiPolygon first_container; - checkColumnTypeOrThrow(arguments[1]); - auto second_parser = MultiPolygonFromColumnParser(std::move(arguments[1].column->convertToFullColumnIfConst())); - MultiPolygon second_container; +// checkColumnTypeOrThrow(arguments[1]); +// auto second_parser = MultiPolygonFromColumnParser(std::move(arguments[1].column->convertToFullColumnIfConst())); +// MultiPolygon second_container; - MultiPolygonSerializer serializer; - MultiPolygon polygons_union{}; +// MultiPolygonSerializer serializer; +// MultiPolygon polygons_union{}; - /// We are not interested in some pitfalls in third-party libraries - /// NOLINTNEXTLINE(clang-analyzer-core.uninitialized.Assign) - for (size_t i = 0; i < input_rows_count; i++) - { - first_parser.get(first_container, i); - second_parser.get(second_container, i); +// /// We are not interested in some pitfalls in third-party libraries +// /// NOLINTNEXTLINE(clang-analyzer-core.uninitialized.Assign) +// for (size_t i = 0; i < input_rows_count; i++) +// { +// first_parser.get(first_container, i); +// second_parser.get(second_container, i); - /// Orient the polygons correctly. - boost::geometry::correct(first_container); - boost::geometry::correct(second_container); +// /// Orient the polygons correctly. +// boost::geometry::correct(first_container); +// boost::geometry::correct(second_container); - /// Main work here. - boost::geometry::union_(first_container, second_container, polygons_union); +// /// Main work here. +// boost::geometry::union_(first_container, second_container, polygons_union); - serializer.add(polygons_union); - } +// serializer.add(polygons_union); +// } - return serializer.finalize(); - } +// return serializer.finalize(); +// } - bool useDefaultImplementationForConstants() const override - { - return true; - } -}; +// bool useDefaultImplementationForConstants() const override +// { +// return true; +// } +// }; -template <> -const char * FunctionPolygonsUnion::name = "polygonsUnionCartesian"; +// template <> +// const char * FunctionPolygonsUnion::name = "polygonsUnionCartesian"; -template <> -const char * FunctionPolygonsUnion::name = "polygonsUnionGeographic"; +// template <> +// const char * FunctionPolygonsUnion::name = "polygonsUnionGeographic"; -void registerFunctionPolygonsUnion(FunctionFactory & factory) -{ - factory.registerFunction>(); - factory.registerFunction>(); -} +// void registerFunctionPolygonsUnion(FunctionFactory & factory) +// { +// factory.registerFunction>(); +// factory.registerFunction>(); +// } -} +// } diff --git a/src/Functions/polygonsWithin.cpp b/src/Functions/polygonsWithin.cpp index f483d68dc0d..d223022ae75 100644 --- a/src/Functions/polygonsWithin.cpp +++ b/src/Functions/polygonsWithin.cpp @@ -1,105 +1,105 @@ -#include -#include +// #include +// #include -#include -#include -#include +// #include +// #include +// #include -#include +// #include -#include -#include -#include -#include -#include -#include -#include -#include +// #include +// #include +// #include +// #include +// #include +// #include +// #include +// #include -#include -#include +// #include +// #include -namespace DB -{ +// namespace DB +// { -template -class FunctionPolygonsWithin : public IFunction -{ -public: - static inline const char * name; +// template +// class FunctionPolygonsWithin : public IFunction +// { +// public: +// static inline const char * name; - explicit FunctionPolygonsWithin() = default; +// explicit FunctionPolygonsWithin() = default; - static FunctionPtr create(const Context &) - { - return std::make_shared(); - } +// static FunctionPtr create(const Context &) +// { +// return std::make_shared(); +// } - String getName() const override - { - return name; - } +// String getName() const override +// { +// return name; +// } - bool isVariadic() const override - { - return false; - } +// bool isVariadic() const override +// { +// return false; +// } - size_t getNumberOfArguments() const override - { - return 2; - } +// size_t getNumberOfArguments() const override +// { +// return 2; +// } - DataTypePtr getReturnTypeImpl(const DataTypes &) const override - { - return std::make_shared(); - } +// DataTypePtr getReturnTypeImpl(const DataTypes &) const override +// { +// return std::make_shared(); +// } - ColumnPtr executeImpl(const ColumnsWithTypeAndName & arguments, const DataTypePtr & /*result_type*/, size_t input_rows_count) const override - { - checkColumnTypeOrThrow(arguments[0]); - auto first_parser = MultiPolygonFromColumnParser(std::move(arguments[0].column->convertToFullColumnIfConst())); - MultiPolygon first_container; +// ColumnPtr executeImpl(const ColumnsWithTypeAndName & arguments, const DataTypePtr & /*result_type*/, size_t input_rows_count) const override +// { +// checkColumnTypeOrThrow(arguments[0]); +// auto first_parser = MultiPolygonFromColumnParser(std::move(arguments[0].column->convertToFullColumnIfConst())); +// MultiPolygon first_container; - checkColumnTypeOrThrow(arguments[1]); - auto second_parser = MultiPolygonFromColumnParser(std::move(arguments[1].column->convertToFullColumnIfConst())); - MultiPolygon second_container; +// checkColumnTypeOrThrow(arguments[1]); +// auto second_parser = MultiPolygonFromColumnParser(std::move(arguments[1].column->convertToFullColumnIfConst())); +// MultiPolygon second_container; - auto res_column = ColumnUInt8::create(); +// auto res_column = ColumnUInt8::create(); - /// NOLINTNEXTLINE(clang-analyzer-core.uninitialized.Assign) - for (size_t i = 0; i < input_rows_count; i++) - { - first_parser.get(first_container, i); - second_parser.get(second_container, i); +// /// NOLINTNEXTLINE(clang-analyzer-core.uninitialized.Assign) +// for (size_t i = 0; i < input_rows_count; i++) +// { +// first_parser.get(first_container, i); +// second_parser.get(second_container, i); - boost::geometry::correct(first_container); - boost::geometry::correct(second_container); +// boost::geometry::correct(first_container); +// boost::geometry::correct(second_container); - res_column->insertValue(boost::geometry::within(first_container, second_container)); - } +// res_column->insertValue(boost::geometry::within(first_container, second_container)); +// } - return res_column; - } +// return res_column; +// } - bool useDefaultImplementationForConstants() const override - { - return true; - } -}; +// bool useDefaultImplementationForConstants() const override +// { +// return true; +// } +// }; -template <> -const char * FunctionPolygonsWithin::name = "polygonsWithinCartesian"; +// template <> +// const char * FunctionPolygonsWithin::name = "polygonsWithinCartesian"; -template <> -const char * FunctionPolygonsWithin::name = "polygonsWithinGeographic"; +// template <> +// const char * FunctionPolygonsWithin::name = "polygonsWithinGeographic"; -void registerFunctionPolygonsWithin(FunctionFactory & factory) -{ - factory.registerFunction>(); - factory.registerFunction>(); -} +// void registerFunctionPolygonsWithin(FunctionFactory & factory) +// { +// factory.registerFunction>(); +// factory.registerFunction>(); +// } -} +// } diff --git a/src/Functions/registerFunctionsGeo.cpp b/src/Functions/registerFunctionsGeo.cpp index 605dd4dcba0..4f1ae30c456 100644 --- a/src/Functions/registerFunctionsGeo.cpp +++ b/src/Functions/registerFunctionsGeo.cpp @@ -11,20 +11,20 @@ void registerFunctionGeoDistance(FunctionFactory & factory); void registerFunctionPointInEllipses(FunctionFactory & factory); void registerFunctionPointInPolygon(FunctionFactory & factory); void registerFunctionPolygonsIntersection(FunctionFactory & factory); -void registerFunctionPolygonsUnion(FunctionFactory & factory); -void registerFunctionPolygonArea(FunctionFactory & factory); -void registerFunctionPolygonConvexHull(FunctionFactory & factory); -void registerFunctionPolygonsSymDifference(FunctionFactory & factory); -void registerFunctionPolygonsEquals(FunctionFactory & factory); -void registerFunctionPolygonsDistance(FunctionFactory & factory); -void registerFunctionPolygonsWithin(FunctionFactory & factory); -void registerFunctionPolygonPerimeter(FunctionFactory & factory); +// void registerFunctionPolygonsUnion(FunctionFactory & factory); +// void registerFunctionPolygonArea(FunctionFactory & factory); +// void registerFunctionPolygonConvexHull(FunctionFactory & factory); +// void registerFunctionPolygonsSymDifference(FunctionFactory & factory); +// void registerFunctionPolygonsEquals(FunctionFactory & factory); +// void registerFunctionPolygonsDistance(FunctionFactory & factory); +// void registerFunctionPolygonsWithin(FunctionFactory & factory); +// void registerFunctionPolygonPerimeter(FunctionFactory & factory); void registerFunctionGeohashEncode(FunctionFactory & factory); void registerFunctionGeohashDecode(FunctionFactory & factory); void registerFunctionGeohashesInBox(FunctionFactory & factory); -void registerFunctionWkt(FunctionFactory & factory); -void registerFunctionReadWkt(FunctionFactory & factory); -void registerFunctionSvg(FunctionFactory & factory); +// void registerFunctionWkt(FunctionFactory & factory); +// void registerFunctionReadWkt(FunctionFactory & factory); +// void registerFunctionSvg(FunctionFactory & factory); #if USE_H3 void registerFunctionGeoToH3(FunctionFactory &); @@ -49,20 +49,20 @@ void registerFunctionsGeo(FunctionFactory & factory) registerFunctionPointInEllipses(factory); registerFunctionPointInPolygon(factory); registerFunctionPolygonsIntersection(factory); - registerFunctionPolygonsUnion(factory); - registerFunctionPolygonArea(factory); - registerFunctionPolygonConvexHull(factory); - registerFunctionPolygonsSymDifference(factory); - registerFunctionPolygonsEquals(factory); - registerFunctionPolygonsDistance(factory); - registerFunctionPolygonsWithin(factory); - registerFunctionPolygonPerimeter(factory); + // registerFunctionPolygonsUnion(factory); + // registerFunctionPolygonArea(factory); + // registerFunctionPolygonConvexHull(factory); + // registerFunctionPolygonsSymDifference(factory); + // registerFunctionPolygonsEquals(factory); + // registerFunctionPolygonsDistance(factory); + // registerFunctionPolygonsWithin(factory); + // registerFunctionPolygonPerimeter(factory); registerFunctionGeohashEncode(factory); registerFunctionGeohashDecode(factory); registerFunctionGeohashesInBox(factory); - registerFunctionWkt(factory); - registerFunctionReadWkt(factory); - registerFunctionSvg(factory); + // registerFunctionWkt(factory); + // registerFunctionReadWkt(factory); + // registerFunctionSvg(factory); #if USE_H3 registerFunctionGeoToH3(factory); diff --git a/src/Functions/svg.cpp b/src/Functions/svg.cpp index f09504359fa..ede43e21ce5 100644 --- a/src/Functions/svg.cpp +++ b/src/Functions/svg.cpp @@ -1,102 +1,102 @@ -#include -#include -#include -#include -#include +// #include +// #include +// #include +// #include +// #include -#include -#include +// #include +// #include -namespace DB -{ +// namespace DB +// { -namespace ErrorCodes -{ - extern const int ILLEGAL_TYPE_OF_ARGUMENT; - extern const int TOO_MANY_ARGUMENTS_FOR_FUNCTION; - extern const int TOO_FEW_ARGUMENTS_FOR_FUNCTION; -} +// namespace ErrorCodes +// { +// extern const int ILLEGAL_TYPE_OF_ARGUMENT; +// extern const int TOO_MANY_ARGUMENTS_FOR_FUNCTION; +// extern const int TOO_FEW_ARGUMENTS_FOR_FUNCTION; +// } -class FunctionSvg : public IFunction -{ -public: - static inline const char * name = "svg"; +// class FunctionSvg : public IFunction +// { +// public: +// static inline const char * name = "svg"; - explicit FunctionSvg() = default; +// explicit FunctionSvg() = default; - static FunctionPtr create(const Context &) - { - return std::make_shared(); - } +// static FunctionPtr create(const Context &) +// { +// return std::make_shared(); +// } - String getName() const override - { - return name; - } +// String getName() const override +// { +// return name; +// } - bool isVariadic() const override - { - return true; - } +// bool isVariadic() const override +// { +// return true; +// } - size_t getNumberOfArguments() const override - { - return 2; - } +// size_t getNumberOfArguments() const override +// { +// return 2; +// } - DataTypePtr getReturnTypeImpl(const DataTypes & arguments) const override - { - if (arguments.size() > 2) - { - throw Exception("Too many arguments", ErrorCodes::TOO_MANY_ARGUMENTS_FOR_FUNCTION); - } - else if (arguments.empty()) - { - throw Exception("Too few arguments", ErrorCodes::TOO_FEW_ARGUMENTS_FOR_FUNCTION); - } - else if (arguments.size() == 2 && checkAndGetDataType(arguments[1].get()) == nullptr) - { - throw Exception("Second argument should be String", - ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT); - } +// DataTypePtr getReturnTypeImpl(const DataTypes & arguments) const override +// { +// if (arguments.size() > 2) +// { +// throw Exception("Too many arguments", ErrorCodes::TOO_MANY_ARGUMENTS_FOR_FUNCTION); +// } +// else if (arguments.empty()) +// { +// throw Exception("Too few arguments", ErrorCodes::TOO_FEW_ARGUMENTS_FOR_FUNCTION); +// } +// else if (arguments.size() == 2 && checkAndGetDataType(arguments[1].get()) == nullptr) +// { +// throw Exception("Second argument should be String", +// ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT); +// } - return std::make_shared(); - } +// return std::make_shared(); +// } - ColumnPtr executeImpl(const ColumnsWithTypeAndName & arguments, const DataTypePtr & /*result_type*/, size_t input_rows_count) const override - { - auto parser = makeGeometryFromColumnParser(arguments[0]); +// ColumnPtr executeImpl(const ColumnsWithTypeAndName & arguments, const DataTypePtr & /*result_type*/, size_t input_rows_count) const override +// { +// auto parser = makeGeometryFromColumnParser(arguments[0]); - auto res_column = ColumnString::create(); - auto container = createContainer(parser); +// auto res_column = ColumnString::create(); +// auto container = createContainer(parser); - bool has_style = arguments.size() > 1; - ColumnPtr style; - if (has_style) - style = arguments[1].column; +// bool has_style = arguments.size() > 1; +// ColumnPtr style; +// if (has_style) +// style = arguments[1].column; - for (size_t i = 0; i < input_rows_count; i++) - { - std::stringstream str; // STYLE_CHECK_ALLOW_STD_STRING_STREAM - get(parser, container, i); +// for (size_t i = 0; i < input_rows_count; i++) +// { +// std::stringstream str; // STYLE_CHECK_ALLOW_STD_STRING_STREAM +// get(parser, container, i); - str << boost::geometry::svg(container, has_style ? style->getDataAt(i).toString() : ""); - std::string serialized = str.str(); - res_column->insertData(serialized.c_str(), serialized.size()); - } +// str << boost::geometry::svg(container, has_style ? style->getDataAt(i).toString() : ""); +// std::string serialized = str.str(); +// res_column->insertData(serialized.c_str(), serialized.size()); +// } - return res_column; - } +// return res_column; +// } - bool useDefaultImplementationForConstants() const override - { - return true; - } -}; +// bool useDefaultImplementationForConstants() const override +// { +// return true; +// } +// }; -void registerFunctionSvg(FunctionFactory & factory) -{ - factory.registerFunction(); -} +// void registerFunctionSvg(FunctionFactory & factory) +// { +// factory.registerFunction(); +// } -} +// } diff --git a/src/Functions/wkt.cpp b/src/Functions/wkt.cpp index c79c4e6fb02..f854f724ccb 100644 --- a/src/Functions/wkt.cpp +++ b/src/Functions/wkt.cpp @@ -1,69 +1,69 @@ -#include -#include -#include -#include +// #include +// #include +// #include +// #include -#include -#include +// #include +// #include -namespace DB -{ +// namespace DB +// { -class FunctionWkt : public IFunction -{ -public: - static inline const char * name = "wkt"; +// class FunctionWkt : public IFunction +// { +// public: +// static inline const char * name = "wkt"; - explicit FunctionWkt() = default; +// explicit FunctionWkt() = default; - static FunctionPtr create(const Context &) - { - return std::make_shared(); - } +// static FunctionPtr create(const Context &) +// { +// return std::make_shared(); +// } - String getName() const override - { - return name; - } +// String getName() const override +// { +// return name; +// } - size_t getNumberOfArguments() const override - { - return 1; - } +// size_t getNumberOfArguments() const override +// { +// return 1; +// } - DataTypePtr getReturnTypeImpl(const DataTypes &) const override - { - return std::make_shared(); - } +// DataTypePtr getReturnTypeImpl(const DataTypes &) const override +// { +// return std::make_shared(); +// } - ColumnPtr executeImpl(const ColumnsWithTypeAndName & arguments, const DataTypePtr & /*result_type*/, size_t input_rows_count) const override - { - auto parser = makeGeometryFromColumnParser(arguments[0]); - auto res_column = ColumnString::create(); +// ColumnPtr executeImpl(const ColumnsWithTypeAndName & arguments, const DataTypePtr & /*result_type*/, size_t input_rows_count) const override +// { +// auto parser = makeGeometryFromColumnParser(arguments[0]); +// auto res_column = ColumnString::create(); - auto container = createContainer(parser); +// auto container = createContainer(parser); - for (size_t i = 0; i < input_rows_count; i++) - { - std::stringstream str; // STYLE_CHECK_ALLOW_STD_STRING_STREAM - get(parser, container, i); - str << boost::geometry::wkt(container); - std::string serialized = str.str(); - res_column->insertData(serialized.c_str(), serialized.size()); - } +// for (size_t i = 0; i < input_rows_count; i++) +// { +// std::stringstream str; // STYLE_CHECK_ALLOW_STD_STRING_STREAM +// get(parser, container, i); +// str << boost::geometry::wkt(container); +// std::string serialized = str.str(); +// res_column->insertData(serialized.c_str(), serialized.size()); +// } - return res_column; - } +// return res_column; +// } - bool useDefaultImplementationForConstants() const override - { - return true; - } -}; +// bool useDefaultImplementationForConstants() const override +// { +// return true; +// } +// }; -void registerFunctionWkt(FunctionFactory & factory) -{ - factory.registerFunction(); -} +// void registerFunctionWkt(FunctionFactory & factory) +// { +// factory.registerFunction(); +// } -} +// } From 5686c18291f3a13081546ca3879896d499b9a4a2 Mon Sep 17 00:00:00 2001 From: Nikita Mikhailov Date: Fri, 19 Feb 2021 21:09:38 +0300 Subject: [PATCH 294/544] changes after review --- src/Functions/describeGeometry.cpp | 131 +++++++++++++++ src/Functions/geometryConverters.cpp | 205 ++++++++++++++---------- src/Functions/geometryConverters.h | 133 ++++----------- src/Functions/polygonArea.cpp | 140 ++++++++-------- src/Functions/polygonConvexHull.cpp | 136 ++++++++-------- src/Functions/polygonPerimeter.cpp | 141 ++++++++-------- src/Functions/polygonsDistance.cpp | 155 +++++++++--------- src/Functions/polygonsEquals.cpp | 150 +++++++++-------- src/Functions/polygonsIntersection.cpp | 25 ++- src/Functions/polygonsSymDifference.cpp | 155 +++++++++--------- src/Functions/polygonsUnion.cpp | 161 +++++++++---------- src/Functions/polygonsWithin.cpp | 156 +++++++++--------- src/Functions/readWkt.cpp | 16 ++ src/Functions/registerFunctionsGeo.cpp | 46 +++--- src/Functions/svg.cpp | 164 ++++++++++--------- src/Functions/wkt.cpp | 107 ++++++------- 16 files changed, 1054 insertions(+), 967 deletions(-) create mode 100644 src/Functions/describeGeometry.cpp diff --git a/src/Functions/describeGeometry.cpp b/src/Functions/describeGeometry.cpp new file mode 100644 index 00000000000..0f059d3c606 --- /dev/null +++ b/src/Functions/describeGeometry.cpp @@ -0,0 +1,131 @@ +#include +#include +#include +#include +#include +#include + +#include +#include + +namespace DB +{ + +namespace ErrorCodes +{ + extern const int ILLEGAL_TYPE_OF_ARGUMENT; +} + +class FunctionDescribeGeometry : public IFunction +{ +public: + explicit FunctionDescribeGeometry() = default; + + size_t getNumberOfArguments() const override + { + return 1; + } + + static inline const char * name = "describeGeometry"; + + + String getName() const override + { + return name; + } + + DataTypePtr getReturnTypeImpl(const DataTypes & arguments) const override + { + if (checkAndGetDataType(arguments[0].get()) == nullptr) + { + throw Exception("First argument should be String", + ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT); + } + + return std::make_shared(); + } + + ColumnPtr executeImpl(const ColumnsWithTypeAndName & arguments, const DataTypePtr & /*result_type*/, size_t input_rows_count) const override + { + const auto * column_string = checkAndGetColumn(arguments[0].column.get()); + + CartesianPoint point; + Ring ring; + Polygon polygon; + MultiPolygon multipolygon; + + auto result = ColumnUInt8::create(); + auto & result_array = result->getData(); + + result_array.reserve(input_rows_count); + + for (size_t i = 0; i < input_rows_count; i++) + { + const auto & str = column_string->getDataAt(i).toString(); + + try + { + boost::geometry::read_wkt(str, point); + result_array.emplace_back(0); + continue; + } + catch (boost::geometry::read_wkt_exception &) + { + } + + try + { + boost::geometry::read_wkt(str, ring); + result_array.emplace_back(1); + continue; + } + catch (boost::geometry::read_wkt_exception &) + { + } + + + try + { + boost::geometry::read_wkt(str, polygon); + result_array.emplace_back(2); + continue; + } + catch (boost::geometry::read_wkt_exception &) + { + } + + + try + { + boost::geometry::read_wkt(str, multipolygon); + result_array.emplace_back(3); + continue; + } + catch (boost::geometry::read_wkt_exception &) + { + } + + throw Exception("Unknown geometry format", ErrorCodes::BAD_ARGUMENTS); + } + + return result; + } + + static FunctionPtr create(const Context &) + { + return std::make_shared(); + } + + bool useDefaultImplementationForConstants() const override + { + return true; + } +}; + + +void registerFunctionDescribeGeometry(FunctionFactory & factory) +{ + factory.registerFunction(); +} + +} diff --git a/src/Functions/geometryConverters.cpp b/src/Functions/geometryConverters.cpp index 13ff94e815d..0a3e31c7c73 100644 --- a/src/Functions/geometryConverters.cpp +++ b/src/Functions/geometryConverters.cpp @@ -13,101 +13,117 @@ namespace ErrorCodes extern const int LOGICAL_ERROR; } -// namespace -// { +template +std::vector PointFromColumnParser::parse(size_t shift, size_t count) const +{ + const auto * tuple = typeid_cast(col.get()); + const auto & tuple_columns = tuple->getColumns(); -// size_t getArrayDepth(DataTypePtr data_type, size_t max_depth) -// { -// size_t depth = 0; -// while (data_type && isArray(data_type) && depth != max_depth + 1) -// { -// depth++; -// data_type = static_cast(*data_type).getNestedType(); -// } + const auto * x_data = typeid_cast(tuple_columns[0].get()); + const auto * y_data = typeid_cast(tuple_columns[1].get()); -// return depth; -// } + const auto * first_container = x_data->getData().data() + shift; + const auto * second_container = y_data->getData().data() + shift; -// template -// class ContainerCreator : public boost::static_visitor -// { -// public: -// template -// Geometry operator()(const T & parser) const -// { -// return parser.createContainer(); -// } -// }; + std::vector answer(count); -// template -// class Getter : public boost::static_visitor -// { -// public: -// constexpr Getter(Geometry & container_, size_t i_) -// : container(container_) -// , i(i_) -// {} + for (size_t i = 0; i < count; ++i) + { + const Float64 first = first_container[i]; + const Float64 second = second_container[i]; -// template -// void operator()(const T & parser) const -// { -// parser.get(container, i); -// } + if (isNaN(first) || isNaN(second)) + throw Exception("Point's component must not be NaN", ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT); -// private: -// Geometry & container; -// size_t i; -// }; + if (isinf(first) || isinf(second)) + throw Exception("Point's component must not be infinite", ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT); -// template -// Parser makeParser(const ColumnWithTypeAndName & col) -// { -// auto wanted_data_type = DataType::nestedDataType(); -// ColumnPtr casted = castColumn(col, DataType::nestedDataType()); -// if (!casted) -// { -// throw Exception("Failed to cast " + col.type->getName() + " to " + wanted_data_type->getName(), ErrorCodes::ILLEGAL_COLUMN); -// } -// return Parser(std::move(casted->convertToFullColumnIfConst())); -// } + answer[i] = Point(first, second); + } -// } + return answer; +} -// template -// Geometry createContainer(const GeometryFromColumnParser & parser) -// { -// static ContainerCreator> creator; -// return boost::apply_visitor(creator, parser); -// } +template +std::vector> RingFromColumnParser::parse(size_t /*shift*/, size_t /*size*/) const +{ + size_t prev_offset = 0; + std::vector> answer; + answer.reserve(offsets.size()); + for (size_t offset : offsets) + { + auto tmp = point_parser.parse(prev_offset, offset - prev_offset); + answer.emplace_back(tmp.begin(), tmp.end()); + prev_offset = offset; + } + return answer; +} -// template -// void get(const GeometryFromColumnParser & parser, Geometry & container, size_t i) -// { -// boost::apply_visitor(Getter(container, i), parser); -// } +template +std::vector> PolygonFromColumnParser::parse(size_t /*shift*/, size_t /*size*/) const +{ + std::vector> answer(offsets.size()); + auto all_rings = ring_parser.parse(0, 0); -// template -// GeometryFromColumnParser makeGeometryFromColumnParser(const ColumnWithTypeAndName & col) -// { -// switch (getArrayDepth(col.type, 3)) -// { -// case 0: return makeParser>(col); -// case 1: return makeParser>(col); -// case 2: return makeParser>(col); -// case 3: return makeParser>(col); -// default: throw Exception("Cannot parse geometry from column with type " + col.type->getName() -// + ", array depth is too big", ErrorCodes::ILLEGAL_COLUMN); -// } -// } + auto prev_offset = 0; + for (size_t iter = 0; iter < offsets.size(); ++iter) + { + const auto current_array_size = offsets[iter] - prev_offset; + answer[iter].outer() = std::move(all_rings[prev_offset]); + answer[iter].inners().reserve(current_array_size); + for (size_t inner_holes = prev_offset + 1; inner_holes < offsets[iter]; ++inner_holes) + answer[iter].inners().emplace_back(std::move(all_rings[inner_holes])); + prev_offset = offsets[iter]; + } -// /// Explicit instantiations to avoid linker errors. + return answer; +} -// template Geometry createContainer(const GeometryFromColumnParser &); -// template Geometry createContainer(const GeometryFromColumnParser &); -// template void get(const GeometryFromColumnParser & parser, Geometry & container, size_t i); -// template void get(const GeometryFromColumnParser & parser, Geometry & container, size_t i); -// template GeometryFromColumnParser makeGeometryFromColumnParser(const ColumnWithTypeAndName & col); -// template GeometryFromColumnParser makeGeometryFromColumnParser(const ColumnWithTypeAndName & col); + +template +std::vector> MultiPolygonFromColumnParser::parse(size_t /*shift*/, size_t /*size*/) const +{ + size_t prev_offset = 0; + std::vector> answer(offsets.size()); + + auto all_polygons = polygon_parser.parse(0, 0); + + for (size_t iter = 0; iter < offsets.size(); ++iter) + { + for (size_t polygon_iter = prev_offset; polygon_iter < offsets[iter]; ++polygon_iter) + answer[iter].emplace_back(std::move(all_polygons[polygon_iter])); + prev_offset = offsets[iter]; + } + + return answer; +} + +template +class ParserVisitor : public boost::static_visitor +{ +public: + template + ContainterWithFigures operator()(const T & parser) const + { + auto parsed = parser.parse(0, 0); + ContainterWithFigures figures; + figures.reserve(parsed.size()); + for (auto & value : parsed) + figures.emplace_back(value); + return figures; + } +}; + +template +std::vector> parseFigure(const GeometryFromColumnParser & parser) +{ + static ParserVisitor>> creator; + return boost::apply_visitor(creator, parser); +} + + +template std::vector> parseFigure(const GeometryFromColumnParser &); +template std::vector> parseFigure(const GeometryFromColumnParser &); template typename Desired> @@ -134,4 +150,29 @@ template void checkColumnTypeOrThrow(const ColumnWithType template void checkColumnTypeOrThrow(const ColumnWithTypeAndName &); template void checkColumnTypeOrThrow(const ColumnWithTypeAndName &); +template +GeometryFromColumnParser getConverterBasedOnType(const ColumnWithTypeAndName & column) +{ + if (DataTypeCustomRingSerialization::nestedDataType()->equals(*column.type)) + { + return RingFromColumnParser(std::move(column.column->convertToFullColumnIfConst())); + } + else if (DataTypeCustomPolygonSerialization::nestedDataType()->equals(*column.type)) + { + return PolygonFromColumnParser(std::move(column.column->convertToFullColumnIfConst())); + } + else if (DataTypeCustomMultiPolygonSerialization::nestedDataType()->equals(*column.type)) + { + return MultiPolygonFromColumnParser(std::move(column.column->convertToFullColumnIfConst())); + } + else + { + throw Exception(fmt::format("Unexpected type of column {}", column.type->getName()), ErrorCodes::BAD_ARGUMENTS); + } +} + + +template GeometryFromColumnParser getConverterBasedOnType(const ColumnWithTypeAndName & column); +template GeometryFromColumnParser getConverterBasedOnType(const ColumnWithTypeAndName & column); + } diff --git a/src/Functions/geometryConverters.h b/src/Functions/geometryConverters.h index acb5de38a09..77ecd3ea3db 100644 --- a/src/Functions/geometryConverters.h +++ b/src/Functions/geometryConverters.h @@ -17,9 +17,7 @@ #include #include -// TODO: maybe use isInfinite from clickhouse codebase #include - #include namespace DB @@ -43,6 +41,9 @@ using MultiPolygon = boost::geometry::model::multi_polygon>; template using Geometry = boost::variant, Polygon, MultiPolygon>; +template +using Figure = boost::variant, Polygon, MultiPolygon>; + using CartesianPoint = boost::geometry::model::d2::point_xy; using CartesianRing = Ring; @@ -56,49 +57,31 @@ using GeographicPolygon = Polygon; using GeographicMultiPolygon = MultiPolygon; using GeographicGeometry = Geometry; + +template +class RingFromColumnParser; + +template +class PolygonFromColumnParser; + +template +class MultiPolygonFromColumnParser; + /** * Class which takes some boost type and returns a pair of numbers. * They are (x,y) in case of cartesian coordinated and (lon,lat) in case of geographic. */ -template +template class PointFromColumnParser { public: - using Container = std::conditional_t, CartesianGeometry, GeographicGeometry>; + using Container = std::conditional_t, CartesianGeometry, GeographicGeometry>; explicit PointFromColumnParser(ColumnPtr col_) : col(col_) { } - std::vector parse(size_t shift, size_t count) const - { - const auto * tuple = typeid_cast(col.get()); - const auto & tuple_columns = tuple->getColumns(); - - const auto * x_data = typeid_cast(tuple_columns[0].get()); - const auto * y_data = typeid_cast(tuple_columns[1].get()); - - const auto * first_container = x_data->getData().data() + shift; - const auto * second_container = y_data->getData().data() + shift; - - std::vector answer(count); - - for (size_t i = 0; i < count; ++i) - { - const Float64 first = first_container[i]; - const Float64 second = second_container[i]; - - if (isNaN(first) || isNaN(second)) - throw Exception("Point's component must not be NaN", ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT); - - if (isinf(first) || isinf(second)) - throw Exception("Point's component must not be infinite", ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT); - - answer[i] = PointType(first, second); - } - - return answer; - } + std::vector parse(size_t shift, size_t count) const; private: /// To prevent use-after-free and increase column lifetime. @@ -106,7 +89,6 @@ private: }; - template class RingFromColumnParser { @@ -118,23 +100,7 @@ public: { } - std::vector> parse(size_t shift, size_t /*size*/) const - { - size_t prev_offset = shift; - - std::vector> answer; - answer.reserve(offsets.size()); - - for (size_t offset : offsets) - { - offset += shift; - auto points = point_parser.parse(prev_offset, offset - prev_offset); - answer.emplace_back(points.begin(), points.end()); - prev_offset = offset; - } - - return answer; - } + std::vector> parse(size_t /*shift*/, size_t /*size*/) const; private: /// To prevent use-after-free and increase column lifetime. @@ -151,29 +117,14 @@ public: : col(col_) , offsets(typeid_cast(*col_).getOffsets()) , ring_parser(typeid_cast(*col_).getDataPtr()) - {} - - std::vector> parse(size_t shift, size_t /*size*/) const { - size_t prev_offset = shift; - std::vector> answer(offsets.size()); - size_t iter = 0; - - for (size_t offset : offsets) - { - offset += shift; - auto tmp = ring_parser.parse(prev_offset, offset - prev_offset); - - /// FIXME: other rings are holes in first - answer[iter].outer() = tmp[0]; - prev_offset = offset; - ++iter; - } - - return answer; } + std::vector> parse(size_t /*shift*/, size_t /*size*/) const; + private: + friend class MultiPolygonFromColumnParser; + /// To prevent use-after-free and increase column lifetime. ColumnPtr col; const IColumn::Offsets & offsets; @@ -190,26 +141,7 @@ public: , polygon_parser(typeid_cast(*col_).getDataPtr()) {} - std::vector> parse(size_t shift, size_t /*size*/) const - { - size_t prev_offset = shift; - - std::vector> answer; - answer.resize(offsets.size()); - - size_t iter = 0; - - for (size_t offset : offsets) - { - offset += shift; - auto polygons = polygon_parser.parse(prev_offset, offset - prev_offset); - answer[iter].swap(polygons); - prev_offset = offset; - ++iter; - } - - return answer; - } + std::vector> parse(size_t /*shift*/, size_t /*size*/) const; private: /// To prevent use-after-free and increase column lifetime. @@ -220,29 +152,19 @@ private: template using GeometryFromColumnParser = boost::variant< - PointFromColumnParser, RingFromColumnParser, PolygonFromColumnParser, MultiPolygonFromColumnParser >; -// template -// Geometry createContainer(const GeometryFromColumnParser & parser); -// template -// void get(const GeometryFromColumnParser & parser, Geometry & container, size_t i); +template +std::vector> parseFigure(const GeometryFromColumnParser & parser); -// template -// GeometryFromColumnParser makeGeometryFromColumnParser(const ColumnWithTypeAndName & col); +extern template std::vector> parseFigure(const GeometryFromColumnParser &); +extern template std::vector> parseFigure(const GeometryFromColumnParser &); -// extern template Geometry createContainer(const GeometryFromColumnParser & parser); -// extern template Geometry createContainer(const GeometryFromColumnParser & parser); -// extern template void get(const GeometryFromColumnParser & parser, Geometry & container, size_t i); -// extern template void get(const GeometryFromColumnParser & parser, Geometry & container, size_t i); -// extern template GeometryFromColumnParser makeGeometryFromColumnParser(const ColumnWithTypeAndName & col); -// extern template GeometryFromColumnParser makeGeometryFromColumnParser(const ColumnWithTypeAndName & col); - /// To serialize Geographic or Cartesian point (a pair of numbers in both cases). template class PointSerializerVisitor : public boost::static_visitor @@ -502,4 +424,7 @@ using MultiPolygonSerializer = GeometrySerializer, MultiPolygonS template typename Desired> void checkColumnTypeOrThrow(const ColumnWithTypeAndName & column); +template +GeometryFromColumnParser getConverterBasedOnType(const ColumnWithTypeAndName & column); + } diff --git a/src/Functions/polygonArea.cpp b/src/Functions/polygonArea.cpp index ee06e41e0cb..b719346dd9a 100644 --- a/src/Functions/polygonArea.cpp +++ b/src/Functions/polygonArea.cpp @@ -1,93 +1,93 @@ -// #include -// #include +#include +#include -// #include -// #include -// #include +#include +#include +#include -// #include +#include -// #include -// #include -// #include -// #include -// #include -// #include +#include +#include +#include +#include +#include +#include -// #include -// #include +#include +#include -// namespace DB -// { +namespace DB +{ -// template -// class FunctionPolygonArea : public IFunction -// { -// public: -// static inline const char * name; +template +class FunctionPolygonArea : public IFunction +{ +public: + static inline const char * name; -// explicit FunctionPolygonArea() = default; + explicit FunctionPolygonArea() = default; -// static FunctionPtr create(const Context &) -// { -// return std::make_shared(); -// } + static FunctionPtr create(const Context &) + { + return std::make_shared(); + } -// String getName() const override -// { -// return name; -// } + String getName() const override + { + return name; + } -// bool isVariadic() const override -// { -// return false; -// } + bool isVariadic() const override + { + return false; + } -// size_t getNumberOfArguments() const override -// { -// return 1; -// } + size_t getNumberOfArguments() const override + { + return 1; + } -// DataTypePtr getReturnTypeImpl(const DataTypes &) const override -// { -// return std::make_shared(); -// } + DataTypePtr getReturnTypeImpl(const DataTypes &) const override + { + return std::make_shared(); + } -// ColumnPtr executeImpl(const ColumnsWithTypeAndName & arguments, const DataTypePtr & /*result_type*/, size_t input_rows_count) const override -// { -// checkColumnTypeOrThrow(arguments[0]); -// auto parser = MultiPolygonFromColumnParser(std::move(arguments[0].column->convertToFullColumnIfConst())); -// MultiPolygon container; + ColumnPtr executeImpl(const ColumnsWithTypeAndName & arguments, const DataTypePtr & /*result_type*/, size_t input_rows_count) const override + { + auto parser = getConverterBasedOnType(arguments[0]); + auto figures = parseFigure(parser); -// auto res_column = ColumnFloat64::create(); + auto res_column = ColumnFloat64::create(); + auto & res_data = res_column->getData(); + res_data.reserve(input_rows_count); -// for (size_t i = 0; i < input_rows_count; i++) -// { -// parser.get(container, i); -// res_column->insertValue(boost::geometry::area(container)); -// } + for (size_t i = 0; i < input_rows_count; i++) + { + res_data.emplace_back(boost::geometry::area(figures[i])); + } -// return res_column; -// } + return res_column; + } -// bool useDefaultImplementationForConstants() const override -// { -// return true; -// } -// }; + bool useDefaultImplementationForConstants() const override + { + return true; + } +}; -// template <> -// const char * FunctionPolygonArea::name = "polygonAreaCartesian"; +template <> +const char * FunctionPolygonArea::name = "polygonAreaCartesian"; -// template <> -// const char * FunctionPolygonArea::name = "polygonAreaGeographic"; +template <> +const char * FunctionPolygonArea::name = "polygonAreaGeographic"; -// void registerFunctionPolygonArea(FunctionFactory & factory) -// { -// factory.registerFunction>(); -// factory.registerFunction>(); -// } +void registerFunctionPolygonArea(FunctionFactory & factory) +{ + factory.registerFunction>(); + factory.registerFunction>(); +} -// } +} diff --git a/src/Functions/polygonConvexHull.cpp b/src/Functions/polygonConvexHull.cpp index 6a68f1f92ba..e9fc40c4fb2 100644 --- a/src/Functions/polygonConvexHull.cpp +++ b/src/Functions/polygonConvexHull.cpp @@ -1,91 +1,89 @@ -// #include -// #include +#include +#include -// #include -// #include -// #include +#include +#include +#include -// #include +#include -// #include -// #include -// #include -// #include -// #include -// #include +#include +#include +#include +#include +#include +#include -// #include -// #include +#include +#include -// namespace DB -// { +namespace DB +{ -// template -// class FunctionPolygonConvexHull : public IFunction -// { -// public: -// static const char * name; +template +class FunctionPolygonConvexHull : public IFunction +{ +public: + static const char * name; -// explicit FunctionPolygonConvexHull() = default; + explicit FunctionPolygonConvexHull() = default; -// static FunctionPtr create(const Context &) -// { -// return std::make_shared(); -// } + static FunctionPtr create(const Context &) + { + return std::make_shared(); + } -// String getName() const override -// { -// return name; -// } + String getName() const override + { + return name; + } -// bool isVariadic() const override -// { -// return false; -// } + bool isVariadic() const override + { + return false; + } -// size_t getNumberOfArguments() const override -// { -// return 1; -// } + size_t getNumberOfArguments() const override + { + return 1; + } -// DataTypePtr getReturnTypeImpl(const DataTypes &) const override -// { -// return DataTypeCustomPolygonSerialization::nestedDataType(); -// } + DataTypePtr getReturnTypeImpl(const DataTypes &) const override + { + return DataTypeCustomPolygonSerialization::nestedDataType(); + } -// ColumnPtr executeImpl(const ColumnsWithTypeAndName & arguments, const DataTypePtr & /*result_type*/, size_t input_rows_count) const override -// { -// checkColumnTypeOrThrow(arguments[0]); -// auto parser = MultiPolygonFromColumnParser(std::move(arguments[0].column->convertToFullColumnIfConst())); -// MultiPolygon container; + ColumnPtr executeImpl(const ColumnsWithTypeAndName & arguments, const DataTypePtr & /*result_type*/, size_t input_rows_count) const override + { + auto parser = getConverterBasedOnType(arguments[0]); + auto figures = parseFigure(parser); -// PolygonSerializer serializer; -// Polygon convex_hull{}; + PolygonSerializer serializer; -// for (size_t i = 0; i < input_rows_count; i++) -// { -// parser.get(container, i); -// boost::geometry::convex_hull(container, convex_hull); -// serializer.add(convex_hull); -// } + for (size_t i = 0; i < input_rows_count; i++) + { + Polygon convex_hull{}; + boost::geometry::convex_hull(figures[i], convex_hull); + serializer.add(convex_hull); + } -// return serializer.finalize(); -// } + return serializer.finalize(); + } -// bool useDefaultImplementationForConstants() const override -// { -// return true; -// } -// }; + bool useDefaultImplementationForConstants() const override + { + return true; + } +}; -// template <> -// const char * FunctionPolygonConvexHull::name = "polygonConvexHullCartesian"; +template <> +const char * FunctionPolygonConvexHull::name = "polygonConvexHullCartesian"; -// void registerFunctionPolygonConvexHull(FunctionFactory & factory) -// { -// factory.registerFunction>(); -// } +void registerFunctionPolygonConvexHull(FunctionFactory & factory) +{ + factory.registerFunction>(); +} -// } +} diff --git a/src/Functions/polygonPerimeter.cpp b/src/Functions/polygonPerimeter.cpp index db1ae0b7178..31543a4eedd 100644 --- a/src/Functions/polygonPerimeter.cpp +++ b/src/Functions/polygonPerimeter.cpp @@ -1,93 +1,94 @@ -// #include -// #include +#include +#include -// #include -// #include -// #include +#include +#include +#include -// #include +#include -// #include -// #include -// #include -// #include -// #include -// #include +#include +#include +#include +#include +#include +#include -// #include -// #include +#include +#include -// namespace DB -// { +namespace DB +{ -// template -// class FunctionPolygonPerimeter : public IFunction -// { -// public: -// static const char * name; +template +class FunctionPolygonPerimeter : public IFunction +{ +public: + static const char * name; -// explicit FunctionPolygonPerimeter() = default; + explicit FunctionPolygonPerimeter() = default; -// static FunctionPtr create(const Context &) -// { -// return std::make_shared(); -// } + static FunctionPtr create(const Context &) + { + return std::make_shared(); + } -// String getName() const override -// { -// return name; -// } + String getName() const override + { + return name; + } -// bool isVariadic() const override -// { -// return false; -// } + bool isVariadic() const override + { + return false; + } -// size_t getNumberOfArguments() const override -// { -// return 1; -// } + size_t getNumberOfArguments() const override + { + return 1; + } -// DataTypePtr getReturnTypeImpl(const DataTypes &) const override -// { -// return std::make_shared(); -// } + DataTypePtr getReturnTypeImpl(const DataTypes &) const override + { + return std::make_shared(); + } -// ColumnPtr executeImpl(const ColumnsWithTypeAndName & arguments, const DataTypePtr & /*result_type*/, size_t input_rows_count) const override -// { -// checkColumnTypeOrThrow(arguments[0]); -// auto parser = MultiPolygonFromColumnParser(std::move(arguments[0].column->convertToFullColumnIfConst())); -// MultiPolygon container; + ColumnPtr executeImpl(const ColumnsWithTypeAndName & arguments, const DataTypePtr & /*result_type*/, size_t input_rows_count) const override + { + auto parser = getConverterBasedOnType(arguments[0]); + auto figures = parseFigure(parser); -// auto res_column = ColumnFloat64::create(); + auto res_column = ColumnFloat64::create(); + auto & res_data = res_column->getData(); + res_data.reserve(input_rows_count); -// for (size_t i = 0; i < input_rows_count; i++) -// { -// parser.get(container, i); -// res_column->insertValue(boost::geometry::perimeter(container)); -// } + for (size_t i = 0; i < input_rows_count; i++) + { + boost::geometry::correct(figures[i]); + res_data.emplace_back(boost::geometry::perimeter(figures[i])); + } -// return res_column; -// } + return res_column; + } -// bool useDefaultImplementationForConstants() const override -// { -// return true; -// } -// }; + bool useDefaultImplementationForConstants() const override + { + return true; + } +}; -// template <> -// const char * FunctionPolygonPerimeter::name = "polygonPerimeterCartesian"; +template <> +const char * FunctionPolygonPerimeter::name = "polygonPerimeterCartesian"; -// template <> -// const char * FunctionPolygonPerimeter::name = "polygonPerimeterGeographic"; +template <> +const char * FunctionPolygonPerimeter::name = "polygonPerimeterGeographic"; -// void registerFunctionPolygonPerimeter(FunctionFactory & factory) -// { -// factory.registerFunction>(); -// factory.registerFunction>(); -// } +void registerFunctionPolygonPerimeter(FunctionFactory & factory) +{ + factory.registerFunction>(); + factory.registerFunction>(); +} -// } +} diff --git a/src/Functions/polygonsDistance.cpp b/src/Functions/polygonsDistance.cpp index 5e1ebd45309..600473724f4 100644 --- a/src/Functions/polygonsDistance.cpp +++ b/src/Functions/polygonsDistance.cpp @@ -1,104 +1,101 @@ -// #include -// #include +#include +#include -// #include -// #include -// #include +#include +#include +#include -// #include +#include -// #include -// #include -// #include -// #include -// #include -// #include -// #include -// #include +#include +#include +#include +#include +#include +#include +#include +#include -// #include -// #include +#include +#include -// namespace DB -// { +namespace DB +{ -// template -// class FunctionPolygonsDistance : public IFunction -// { -// public: -// static inline const char * name; +template +class FunctionPolygonsDistance : public IFunction +{ +public: + static inline const char * name; -// explicit FunctionPolygonsDistance() = default; + explicit FunctionPolygonsDistance() = default; -// static FunctionPtr create(const Context &) -// { -// return std::make_shared(); -// } + static FunctionPtr create(const Context &) + { + return std::make_shared(); + } -// String getName() const override -// { -// return name; -// } + String getName() const override + { + return name; + } -// bool isVariadic() const override -// { -// return false; -// } + bool isVariadic() const override + { + return false; + } -// size_t getNumberOfArguments() const override -// { -// return 2; -// } + size_t getNumberOfArguments() const override + { + return 2; + } -// DataTypePtr getReturnTypeImpl(const DataTypes &) const override -// { -// return std::make_shared(); -// } + DataTypePtr getReturnTypeImpl(const DataTypes &) const override + { + return std::make_shared(); + } -// ColumnPtr executeImpl(const ColumnsWithTypeAndName & arguments, const DataTypePtr & /*result_type*/, size_t input_rows_count) const override -// { -// checkColumnTypeOrThrow(arguments[0]); -// auto first_parser = MultiPolygonFromColumnParser(std::move(arguments[0].column->convertToFullColumnIfConst())); -// MultiPolygon first_container; + ColumnPtr executeImpl(const ColumnsWithTypeAndName & arguments, const DataTypePtr & /*result_type*/, size_t input_rows_count) const override + { + auto first_parser = getConverterBasedOnType(arguments[0]); + auto second_parser = getConverterBasedOnType(arguments[1]); -// checkColumnTypeOrThrow(arguments[1]); -// auto second_parser = MultiPolygonFromColumnParser(std::move(arguments[1].column->convertToFullColumnIfConst())); -// MultiPolygon second_container; + auto first = parseFigure(first_parser); + auto second = parseFigure(second_parser); -// auto res_column = ColumnFloat64::create(); + auto res_column = ColumnFloat64::create(); + auto & res_data = res_column->getData(); + res_data.reserve(input_rows_count); -// for (size_t i = 0; i < input_rows_count; i++) -// { -// first_parser.get(first_container, i); -// second_parser.get(second_container, i); + for (size_t i = 0; i < input_rows_count; i++) + { + boost::geometry::correct(first[i]); + boost::geometry::correct(second[i]); -// boost::geometry::correct(first_container); -// boost::geometry::correct(second_container); + res_data.emplace_back(boost::geometry::distance(first[i], second[i])); + } -// res_column->insertValue(boost::geometry::distance(first_container, second_container)); -// } + return res_column; + } -// return res_column; -// } + bool useDefaultImplementationForConstants() const override + { + return true; + } +}; -// bool useDefaultImplementationForConstants() const override -// { -// return true; -// } -// }; +template <> +const char * FunctionPolygonsDistance::name = "polygonsDistanceCartesian"; -// template <> -// const char * FunctionPolygonsDistance::name = "polygonsDistanceCartesian"; - -// template <> -// const char * FunctionPolygonsDistance::name = "polygonsDistanceGeographic"; +template <> +const char * FunctionPolygonsDistance::name = "polygonsDistanceGeographic"; -// void registerFunctionPolygonsDistance(FunctionFactory & factory) -// { -// factory.registerFunction>(); -// factory.registerFunction>(); -// } +void registerFunctionPolygonsDistance(FunctionFactory & factory) +{ + factory.registerFunction>(); + factory.registerFunction>(); +} -// } +} diff --git a/src/Functions/polygonsEquals.cpp b/src/Functions/polygonsEquals.cpp index ac6b9912481..39e7594a7a2 100644 --- a/src/Functions/polygonsEquals.cpp +++ b/src/Functions/polygonsEquals.cpp @@ -1,100 +1,98 @@ -// #include -// #include +#include +#include -// #include -// #include -// #include +#include +#include +#include -// #include +#include -// #include -// #include -// #include -// #include -// #include -// #include -// #include -// #include +#include +#include +#include +#include +#include +#include +#include +#include -// #include -// #include +#include +#include -// namespace DB -// { +namespace DB +{ -// template -// class FunctionPolygonsEquals : public IFunction -// { -// public: -// static const char * name; +template +class FunctionPolygonsEquals : public IFunction +{ +public: + static const char * name; -// explicit FunctionPolygonsEquals() = default; + explicit FunctionPolygonsEquals() = default; -// static FunctionPtr create(const Context &) -// { -// return std::make_shared(); -// } + static FunctionPtr create(const Context &) + { + return std::make_shared(); + } -// String getName() const override -// { -// return name; -// } + String getName() const override + { + return name; + } -// bool isVariadic() const override -// { -// return false; -// } + bool isVariadic() const override + { + return false; + } -// size_t getNumberOfArguments() const override -// { -// return 2; -// } + size_t getNumberOfArguments() const override + { + return 2; + } -// DataTypePtr getReturnTypeImpl(const DataTypes &) const override -// { -// return std::make_shared(); -// } + DataTypePtr getReturnTypeImpl(const DataTypes &) const override + { + return std::make_shared(); + } -// ColumnPtr executeImpl(const ColumnsWithTypeAndName & arguments, const DataTypePtr & /*result_type*/, size_t input_rows_count) const override -// { -// checkColumnTypeOrThrow(arguments[0]); -// auto first_parser = MultiPolygonFromColumnParser(std::move(arguments[0].column->convertToFullColumnIfConst())); -// MultiPolygon first_container; + ColumnPtr executeImpl(const ColumnsWithTypeAndName & arguments, const DataTypePtr & /*result_type*/, size_t input_rows_count) const override + { + auto first_parser = getConverterBasedOnType(arguments[0]); + auto second_parser = getConverterBasedOnType(arguments[1]); -// checkColumnTypeOrThrow(arguments[1]); -// auto second_parser = MultiPolygonFromColumnParser(std::move(arguments[1].column->convertToFullColumnIfConst())); -// MultiPolygon second_container; + auto first = parseFigure(first_parser); + auto second = parseFigure(second_parser); -// auto res_column = ColumnUInt8::create(); + auto res_column = ColumnUInt8::create(); + auto & res_data = res_column->getData(); + res_data.reserve(input_rows_count); -// for (size_t i = 0; i < input_rows_count; i++) -// { -// first_parser.get(first_container, i); -// second_parser.get(second_container, i); + for (size_t i = 0; i < input_rows_count; i++) + { + boost::geometry::correct(first[i]); + boost::geometry::correct(second[i]); -// boost::geometry::correct(first_container); -// boost::geometry::correct(second_container); + /// Main work here. + res_data.emplace_back(boost::geometry::equals(first[i], second[i])); + } -// res_column->insertValue(boost::geometry::equals(first_container, second_container)); -// } + return res_column; + } -// return res_column; -// } - -// bool useDefaultImplementationForConstants() const override -// { -// return true; -// } -// }; + bool useDefaultImplementationForConstants() const override + { + return true; + } +}; -// template <> -// const char * FunctionPolygonsEquals::name = "polygonsEqualsCartesian"; +template <> +const char * FunctionPolygonsEquals::name = "polygonsEqualsCartesian"; -// void registerFunctionPolygonsEquals(FunctionFactory & factory) -// { -// factory.registerFunction>(); -// } +void registerFunctionPolygonsEquals(FunctionFactory & factory) +{ + factory.registerFunction>(); +} -// } +} diff --git a/src/Functions/polygonsIntersection.cpp b/src/Functions/polygonsIntersection.cpp index 129ac2e23e6..54b67b952ed 100644 --- a/src/Functions/polygonsIntersection.cpp +++ b/src/Functions/polygonsIntersection.cpp @@ -51,37 +51,32 @@ public: DataTypePtr getReturnTypeImpl(const DataTypes &) const override { + /// Intersection of each with figure with each could be easily represent as MultiPolygon. return DataTypeCustomMultiPolygonSerialization::nestedDataType(); } ColumnPtr executeImpl(const ColumnsWithTypeAndName & arguments, const DataTypePtr & /*result_type*/, size_t input_rows_count) const override { - checkColumnTypeOrThrow(arguments[0]); - auto first_parser = MultiPolygonFromColumnParser(std::move(arguments[0].column->convertToFullColumnIfConst())); - - - checkColumnTypeOrThrow(arguments[1]); - auto second_parser = MultiPolygonFromColumnParser(std::move(arguments[1].column->convertToFullColumnIfConst())); + auto first_parser = getConverterBasedOnType(arguments[0]); + auto second_parser = getConverterBasedOnType(arguments[1]); MultiPolygonSerializer serializer; - MultiPolygon intersection{}; - auto first = first_parser.parse(0, 0); - auto second = second_parser.parse(0, 0); + auto first = parseFigure(first_parser); + auto second = parseFigure(second_parser); /// We are not interested in some pitfalls in third-party libraries /// NOLINTNEXTLINE(clang-analyzer-core.uninitialized.Assign) for (size_t i = 0; i < input_rows_count; ++i) { - MultiPolygon first_container = first[i]; - MultiPolygon second_container = second[i]; - /// Orient the polygons correctly. - boost::geometry::correct(first_container); - boost::geometry::correct(second_container); + boost::geometry::correct(first[i]); + boost::geometry::correct(second[i]); + + MultiPolygon intersection{}; /// Main work here. - boost::geometry::intersection(first_container[0], second_container[0], intersection); + boost::geometry::intersection(first[i], second[i], intersection); serializer.add(intersection); } diff --git a/src/Functions/polygonsSymDifference.cpp b/src/Functions/polygonsSymDifference.cpp index 6f0ee30a235..839f02738c5 100644 --- a/src/Functions/polygonsSymDifference.cpp +++ b/src/Functions/polygonsSymDifference.cpp @@ -1,104 +1,99 @@ -// #include -// #include +#include +#include -// #include -// #include -// #include +#include +#include +#include -// #include +#include -// #include -// #include -// #include -// #include -// #include -// #include +#include +#include +#include +#include +#include +#include -// #include -// #include +#include +#include -// namespace DB -// { +namespace DB +{ -// template -// class FunctionPolygonsSymDifference : public IFunction -// { -// public: -// static const char * name; +template +class FunctionPolygonsSymDifference : public IFunction +{ +public: + static const char * name; -// explicit FunctionPolygonsSymDifference() = default; + explicit FunctionPolygonsSymDifference() = default; -// static FunctionPtr create(const Context &) -// { -// return std::make_shared(); -// } + static FunctionPtr create(const Context &) + { + return std::make_shared(); + } -// String getName() const override -// { -// return name; -// } + String getName() const override + { + return name; + } -// bool isVariadic() const override -// { -// return false; -// } + bool isVariadic() const override + { + return false; + } -// size_t getNumberOfArguments() const override -// { -// return 2; -// } + size_t getNumberOfArguments() const override + { + return 2; + } -// DataTypePtr getReturnTypeImpl(const DataTypes &) const override -// { -// return DataTypeCustomMultiPolygonSerialization::nestedDataType(); -// } + DataTypePtr getReturnTypeImpl(const DataTypes &) const override + { + return DataTypeCustomMultiPolygonSerialization::nestedDataType(); + } -// ColumnPtr executeImpl(const ColumnsWithTypeAndName & arguments, const DataTypePtr & /*result_type*/, size_t input_rows_count) const override -// { -// checkColumnTypeOrThrow(arguments[0]); -// auto first_parser = MultiPolygonFromColumnParser(std::move(arguments[0].column->convertToFullColumnIfConst())); -// MultiPolygon first_container; + ColumnPtr executeImpl(const ColumnsWithTypeAndName & arguments, const DataTypePtr & /*result_type*/, size_t input_rows_count) const override + { + auto first_parser = getConverterBasedOnType(arguments[0]); + auto second_parser = getConverterBasedOnType(arguments[1]); -// checkColumnTypeOrThrow(arguments[1]); -// auto second_parser = MultiPolygonFromColumnParser(std::move(arguments[1].column->convertToFullColumnIfConst())); -// MultiPolygon second_container; + auto first = parseFigure(first_parser); + auto second = parseFigure(second_parser); -// MultiPolygonSerializer serializer; -// MultiPolygon sym_difference{}; + MultiPolygonSerializer serializer; -// /// NOLINTNEXTLINE(clang-analyzer-core.uninitialized.Assign) -// for (size_t i = 0; i < input_rows_count; i++) -// { -// first_parser.get(first_container, i); -// second_parser.get(second_container, i); + /// NOLINTNEXTLINE(clang-analyzer-core.uninitialized.Assign) + for (size_t i = 0; i < input_rows_count; i++) + { + boost::geometry::correct(first[i]); + boost::geometry::correct(second[i]); -// boost::geometry::correct(first_container); -// boost::geometry::correct(second_container); + MultiPolygon sym_difference{}; + boost::geometry::sym_difference(first[i], second[i], sym_difference); -// boost::geometry::sym_difference(first_container, second_container, sym_difference); + serializer.add(sym_difference); + } -// serializer.add(sym_difference); -// } + return serializer.finalize(); + } -// return serializer.finalize(); -// } + bool useDefaultImplementationForConstants() const override + { + return true; + } +}; -// bool useDefaultImplementationForConstants() const override -// { -// return true; -// } -// }; +template <> +const char * FunctionPolygonsSymDifference::name = "polygonsSymDifferenceCartesian"; -// template <> -// const char * FunctionPolygonsSymDifference::name = "polygonsSymDifferenceCartesian"; +template <> +const char * FunctionPolygonsSymDifference::name = "polygonsSymDifferenceGeographic"; -// template <> -// const char * FunctionPolygonsSymDifference::name = "polygonsSymDifferenceGeographic"; +void registerFunctionPolygonsSymDifference(FunctionFactory & factory) +{ + factory.registerFunction>(); + factory.registerFunction>(); +} -// void registerFunctionPolygonsSymDifference(FunctionFactory & factory) -// { -// factory.registerFunction>(); -// factory.registerFunction>(); -// } - -// } +} diff --git a/src/Functions/polygonsUnion.cpp b/src/Functions/polygonsUnion.cpp index d6ac086abc2..6d5cd1ca266 100644 --- a/src/Functions/polygonsUnion.cpp +++ b/src/Functions/polygonsUnion.cpp @@ -1,108 +1,103 @@ -// #include -// #include +#include +#include -// #include -// #include -// #include +#include +#include +#include -// #include +#include -// #include -// #include -// #include -// #include -// #include -// #include +#include +#include +#include +#include +#include +#include -// #include -// #include +#include +#include -// namespace DB -// { +namespace DB +{ -// template -// class FunctionPolygonsUnion : public IFunction -// { -// public: -// static inline const char * name; +template +class FunctionPolygonsUnion : public IFunction +{ +public: + static inline const char * name; -// explicit FunctionPolygonsUnion() = default; + explicit FunctionPolygonsUnion() = default; -// static FunctionPtr create(const Context &) -// { -// return std::make_shared(); -// } + static FunctionPtr create(const Context &) + { + return std::make_shared(); + } -// String getName() const override -// { -// return name; -// } + String getName() const override + { + return name; + } -// bool isVariadic() const override -// { -// return false; -// } + bool isVariadic() const override + { + return false; + } -// size_t getNumberOfArguments() const override -// { -// return 2; -// } + size_t getNumberOfArguments() const override + { + return 2; + } -// DataTypePtr getReturnTypeImpl(const DataTypes &) const override -// { -// return DataTypeCustomMultiPolygonSerialization::nestedDataType(); -// } + DataTypePtr getReturnTypeImpl(const DataTypes &) const override + { + return DataTypeCustomMultiPolygonSerialization::nestedDataType(); + } -// ColumnPtr executeImpl(const ColumnsWithTypeAndName & arguments, const DataTypePtr & /*result_type*/, size_t input_rows_count) const override -// { -// checkColumnTypeOrThrow(arguments[0]); -// auto first_parser = MultiPolygonFromColumnParser(std::move(arguments[0].column->convertToFullColumnIfConst())); -// MultiPolygon first_container; + ColumnPtr executeImpl(const ColumnsWithTypeAndName & arguments, const DataTypePtr & /*result_type*/, size_t input_rows_count) const override + { + auto first_parser = getConverterBasedOnType(arguments[0]); + auto second_parser = getConverterBasedOnType(arguments[1]); -// checkColumnTypeOrThrow(arguments[1]); -// auto second_parser = MultiPolygonFromColumnParser(std::move(arguments[1].column->convertToFullColumnIfConst())); -// MultiPolygon second_container; + auto first = parseFigure(first_parser); + auto second = parseFigure(second_parser); -// MultiPolygonSerializer serializer; -// MultiPolygon polygons_union{}; + MultiPolygonSerializer serializer; -// /// We are not interested in some pitfalls in third-party libraries -// /// NOLINTNEXTLINE(clang-analyzer-core.uninitialized.Assign) -// for (size_t i = 0; i < input_rows_count; i++) -// { -// first_parser.get(first_container, i); -// second_parser.get(second_container, i); + /// We are not interested in some pitfalls in third-party libraries + /// NOLINTNEXTLINE(clang-analyzer-core.uninitialized.Assign) + for (size_t i = 0; i < input_rows_count; i++) + { + /// Orient the polygons correctly. + boost::geometry::correct(first[i]); + boost::geometry::correct(second[i]); -// /// Orient the polygons correctly. -// boost::geometry::correct(first_container); -// boost::geometry::correct(second_container); + MultiPolygon polygons_union{}; + /// Main work here. + boost::geometry::union_(first[i], second[i], polygons_union); -// /// Main work here. -// boost::geometry::union_(first_container, second_container, polygons_union); + serializer.add(polygons_union); + } -// serializer.add(polygons_union); -// } + return serializer.finalize(); + } -// return serializer.finalize(); -// } + bool useDefaultImplementationForConstants() const override + { + return true; + } +}; -// bool useDefaultImplementationForConstants() const override -// { -// return true; -// } -// }; +template <> +const char * FunctionPolygonsUnion::name = "polygonsUnionCartesian"; -// template <> -// const char * FunctionPolygonsUnion::name = "polygonsUnionCartesian"; - -// template <> -// const char * FunctionPolygonsUnion::name = "polygonsUnionGeographic"; +template <> +const char * FunctionPolygonsUnion::name = "polygonsUnionGeographic"; -// void registerFunctionPolygonsUnion(FunctionFactory & factory) -// { -// factory.registerFunction>(); -// factory.registerFunction>(); -// } +void registerFunctionPolygonsUnion(FunctionFactory & factory) +{ + factory.registerFunction>(); + factory.registerFunction>(); +} -// } +} diff --git a/src/Functions/polygonsWithin.cpp b/src/Functions/polygonsWithin.cpp index d223022ae75..be2e5cbb3a5 100644 --- a/src/Functions/polygonsWithin.cpp +++ b/src/Functions/polygonsWithin.cpp @@ -1,105 +1,101 @@ -// #include -// #include +#include +#include -// #include -// #include -// #include +#include +#include +#include -// #include +#include -// #include -// #include -// #include -// #include -// #include -// #include -// #include -// #include +#include +#include +#include +#include +#include +#include +#include +#include -// #include -// #include +#include +#include -// namespace DB -// { +namespace DB +{ -// template -// class FunctionPolygonsWithin : public IFunction -// { -// public: -// static inline const char * name; +template +class FunctionPolygonsWithin : public IFunction +{ +public: + static inline const char * name; -// explicit FunctionPolygonsWithin() = default; + explicit FunctionPolygonsWithin() = default; -// static FunctionPtr create(const Context &) -// { -// return std::make_shared(); -// } + static FunctionPtr create(const Context &) + { + return std::make_shared(); + } -// String getName() const override -// { -// return name; -// } + String getName() const override + { + return name; + } -// bool isVariadic() const override -// { -// return false; -// } + bool isVariadic() const override + { + return false; + } -// size_t getNumberOfArguments() const override -// { -// return 2; -// } + size_t getNumberOfArguments() const override + { + return 2; + } -// DataTypePtr getReturnTypeImpl(const DataTypes &) const override -// { -// return std::make_shared(); -// } + DataTypePtr getReturnTypeImpl(const DataTypes &) const override + { + return std::make_shared(); + } -// ColumnPtr executeImpl(const ColumnsWithTypeAndName & arguments, const DataTypePtr & /*result_type*/, size_t input_rows_count) const override -// { -// checkColumnTypeOrThrow(arguments[0]); -// auto first_parser = MultiPolygonFromColumnParser(std::move(arguments[0].column->convertToFullColumnIfConst())); -// MultiPolygon first_container; + ColumnPtr executeImpl(const ColumnsWithTypeAndName & arguments, const DataTypePtr & /*result_type*/, size_t input_rows_count) const override + { + auto first_parser = getConverterBasedOnType(arguments[0]); + auto second_parser = getConverterBasedOnType(arguments[1]); -// checkColumnTypeOrThrow(arguments[1]); -// auto second_parser = MultiPolygonFromColumnParser(std::move(arguments[1].column->convertToFullColumnIfConst())); -// MultiPolygon second_container; + auto first = parseFigure(first_parser); + auto second = parseFigure(second_parser); -// auto res_column = ColumnUInt8::create(); + auto res_column = ColumnUInt8::create(); + auto & res_data = res_column->getData(); -// /// NOLINTNEXTLINE(clang-analyzer-core.uninitialized.Assign) -// for (size_t i = 0; i < input_rows_count; i++) -// { -// first_parser.get(first_container, i); -// second_parser.get(second_container, i); + /// NOLINTNEXTLINE(clang-analyzer-core.uninitialized.Assign) + for (size_t i = 0; i < input_rows_count; i++) + { + boost::geometry::correct(first[i]); + boost::geometry::correct(second[i]); -// boost::geometry::correct(first_container); -// boost::geometry::correct(second_container); + res_data.emplace_back(boost::geometry::within(first[i], second[i])); + } -// res_column->insertValue(boost::geometry::within(first_container, second_container)); -// } + return res_column; + } -// return res_column; -// } - -// bool useDefaultImplementationForConstants() const override -// { -// return true; -// } -// }; + bool useDefaultImplementationForConstants() const override + { + return true; + } +}; -// template <> -// const char * FunctionPolygonsWithin::name = "polygonsWithinCartesian"; +template <> +const char * FunctionPolygonsWithin::name = "polygonsWithinCartesian"; -// template <> -// const char * FunctionPolygonsWithin::name = "polygonsWithinGeographic"; +template <> +const char * FunctionPolygonsWithin::name = "polygonsWithinGeographic"; -// void registerFunctionPolygonsWithin(FunctionFactory & factory) -// { -// factory.registerFunction>(); -// factory.registerFunction>(); -// } +void registerFunctionPolygonsWithin(FunctionFactory & factory) +{ + factory.registerFunction>(); + factory.registerFunction>(); +} -// } +} diff --git a/src/Functions/readWkt.cpp b/src/Functions/readWkt.cpp index 9ef8af745db..be34b652929 100644 --- a/src/Functions/readWkt.cpp +++ b/src/Functions/readWkt.cpp @@ -75,6 +75,21 @@ public: } }; + +class FunctionReadWktRing : public FunctionReadWkt> +{ +public: + static inline const char * name = "readWktRing"; + String getName() const override + { + return name; + } + static FunctionPtr create(const Context &) + { + return std::make_shared(); + } +}; + class FunctionReadWktPolygon : public FunctionReadWkt> { public: @@ -106,6 +121,7 @@ public: void registerFunctionReadWkt(FunctionFactory & factory) { factory.registerFunction(); + factory.registerFunction(); factory.registerFunction(); factory.registerFunction(); } diff --git a/src/Functions/registerFunctionsGeo.cpp b/src/Functions/registerFunctionsGeo.cpp index 4f1ae30c456..682093b0f2c 100644 --- a/src/Functions/registerFunctionsGeo.cpp +++ b/src/Functions/registerFunctionsGeo.cpp @@ -11,20 +11,21 @@ void registerFunctionGeoDistance(FunctionFactory & factory); void registerFunctionPointInEllipses(FunctionFactory & factory); void registerFunctionPointInPolygon(FunctionFactory & factory); void registerFunctionPolygonsIntersection(FunctionFactory & factory); -// void registerFunctionPolygonsUnion(FunctionFactory & factory); -// void registerFunctionPolygonArea(FunctionFactory & factory); -// void registerFunctionPolygonConvexHull(FunctionFactory & factory); -// void registerFunctionPolygonsSymDifference(FunctionFactory & factory); -// void registerFunctionPolygonsEquals(FunctionFactory & factory); -// void registerFunctionPolygonsDistance(FunctionFactory & factory); -// void registerFunctionPolygonsWithin(FunctionFactory & factory); -// void registerFunctionPolygonPerimeter(FunctionFactory & factory); +void registerFunctionPolygonsUnion(FunctionFactory & factory); +void registerFunctionPolygonArea(FunctionFactory & factory); +void registerFunctionPolygonConvexHull(FunctionFactory & factory); +void registerFunctionPolygonsSymDifference(FunctionFactory & factory); +void registerFunctionPolygonsEquals(FunctionFactory & factory); +void registerFunctionPolygonsDistance(FunctionFactory & factory); +void registerFunctionPolygonsWithin(FunctionFactory & factory); +void registerFunctionPolygonPerimeter(FunctionFactory & factory); void registerFunctionGeohashEncode(FunctionFactory & factory); void registerFunctionGeohashDecode(FunctionFactory & factory); void registerFunctionGeohashesInBox(FunctionFactory & factory); -// void registerFunctionWkt(FunctionFactory & factory); -// void registerFunctionReadWkt(FunctionFactory & factory); -// void registerFunctionSvg(FunctionFactory & factory); +void registerFunctionWkt(FunctionFactory & factory); +void registerFunctionReadWkt(FunctionFactory & factory); +void registerFunctionDescribeGeometry(FunctionFactory & factory); +void registerFunctionSvg(FunctionFactory & factory); #if USE_H3 void registerFunctionGeoToH3(FunctionFactory &); @@ -49,20 +50,21 @@ void registerFunctionsGeo(FunctionFactory & factory) registerFunctionPointInEllipses(factory); registerFunctionPointInPolygon(factory); registerFunctionPolygonsIntersection(factory); - // registerFunctionPolygonsUnion(factory); - // registerFunctionPolygonArea(factory); - // registerFunctionPolygonConvexHull(factory); - // registerFunctionPolygonsSymDifference(factory); - // registerFunctionPolygonsEquals(factory); - // registerFunctionPolygonsDistance(factory); - // registerFunctionPolygonsWithin(factory); - // registerFunctionPolygonPerimeter(factory); + registerFunctionPolygonsUnion(factory); + registerFunctionPolygonArea(factory); + registerFunctionPolygonConvexHull(factory); + registerFunctionPolygonsSymDifference(factory); + registerFunctionPolygonsEquals(factory); + registerFunctionPolygonsDistance(factory); + registerFunctionPolygonsWithin(factory); + registerFunctionPolygonPerimeter(factory); registerFunctionGeohashEncode(factory); registerFunctionGeohashDecode(factory); registerFunctionGeohashesInBox(factory); - // registerFunctionWkt(factory); - // registerFunctionReadWkt(factory); - // registerFunctionSvg(factory); + registerFunctionWkt(factory); + registerFunctionReadWkt(factory); + registerFunctionDescribeGeometry(factory); + registerFunctionSvg(factory); #if USE_H3 registerFunctionGeoToH3(factory); diff --git a/src/Functions/svg.cpp b/src/Functions/svg.cpp index ede43e21ce5..42b3b916ca2 100644 --- a/src/Functions/svg.cpp +++ b/src/Functions/svg.cpp @@ -1,102 +1,100 @@ -// #include -// #include -// #include -// #include -// #include +#include +#include +#include +#include +#include -// #include -// #include +#include +#include -// namespace DB -// { +namespace DB +{ -// namespace ErrorCodes -// { -// extern const int ILLEGAL_TYPE_OF_ARGUMENT; -// extern const int TOO_MANY_ARGUMENTS_FOR_FUNCTION; -// extern const int TOO_FEW_ARGUMENTS_FOR_FUNCTION; -// } +namespace ErrorCodes +{ + extern const int ILLEGAL_TYPE_OF_ARGUMENT; + extern const int TOO_MANY_ARGUMENTS_FOR_FUNCTION; + extern const int TOO_FEW_ARGUMENTS_FOR_FUNCTION; +} -// class FunctionSvg : public IFunction -// { -// public: -// static inline const char * name = "svg"; +class FunctionSvg : public IFunction +{ +public: + static inline const char * name = "svg"; -// explicit FunctionSvg() = default; + explicit FunctionSvg() = default; -// static FunctionPtr create(const Context &) -// { -// return std::make_shared(); -// } + static FunctionPtr create(const Context &) + { + return std::make_shared(); + } -// String getName() const override -// { -// return name; -// } + String getName() const override + { + return name; + } -// bool isVariadic() const override -// { -// return true; -// } + bool isVariadic() const override + { + return true; + } -// size_t getNumberOfArguments() const override -// { -// return 2; -// } + size_t getNumberOfArguments() const override + { + return 2; + } -// DataTypePtr getReturnTypeImpl(const DataTypes & arguments) const override -// { -// if (arguments.size() > 2) -// { -// throw Exception("Too many arguments", ErrorCodes::TOO_MANY_ARGUMENTS_FOR_FUNCTION); -// } -// else if (arguments.empty()) -// { -// throw Exception("Too few arguments", ErrorCodes::TOO_FEW_ARGUMENTS_FOR_FUNCTION); -// } -// else if (arguments.size() == 2 && checkAndGetDataType(arguments[1].get()) == nullptr) -// { -// throw Exception("Second argument should be String", -// ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT); -// } + DataTypePtr getReturnTypeImpl(const DataTypes & arguments) const override + { + if (arguments.size() > 2) + { + throw Exception("Too many arguments", ErrorCodes::TOO_MANY_ARGUMENTS_FOR_FUNCTION); + } + else if (arguments.empty()) + { + throw Exception("Too few arguments", ErrorCodes::TOO_FEW_ARGUMENTS_FOR_FUNCTION); + } + else if (arguments.size() == 2 && checkAndGetDataType(arguments[1].get()) == nullptr) + { + throw Exception("Second argument should be String", ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT); + } -// return std::make_shared(); -// } + return std::make_shared(); + } -// ColumnPtr executeImpl(const ColumnsWithTypeAndName & arguments, const DataTypePtr & /*result_type*/, size_t input_rows_count) const override -// { -// auto parser = makeGeometryFromColumnParser(arguments[0]); + ColumnPtr executeImpl(const ColumnsWithTypeAndName & arguments, const DataTypePtr & /*result_type*/, size_t input_rows_count) const override + { + auto parser = getConverterBasedOnType(arguments[0]); + auto figures = parseFigure(parser); -// auto res_column = ColumnString::create(); -// auto container = createContainer(parser); + auto res_column = ColumnString::create(); -// bool has_style = arguments.size() > 1; -// ColumnPtr style; -// if (has_style) -// style = arguments[1].column; + bool has_style = arguments.size() > 1; + ColumnPtr style; + if (has_style) + style = arguments[1].column; -// for (size_t i = 0; i < input_rows_count; i++) -// { -// std::stringstream str; // STYLE_CHECK_ALLOW_STD_STRING_STREAM -// get(parser, container, i); + for (size_t i = 0; i < input_rows_count; i++) + { + std::stringstream str; // STYLE_CHECK_ALLOW_STD_STRING_STREAM + boost::geometry::correct(figures[i]); + str << boost::geometry::svg(figures[i], has_style ? style->getDataAt(i).toString() : ""); + std::string serialized = str.str(); + res_column->insertData(serialized.c_str(), serialized.size()); + } -// str << boost::geometry::svg(container, has_style ? style->getDataAt(i).toString() : ""); -// std::string serialized = str.str(); -// res_column->insertData(serialized.c_str(), serialized.size()); -// } + return res_column; + } -// return res_column; -// } + bool useDefaultImplementationForConstants() const override + { + return true; + } +}; -// bool useDefaultImplementationForConstants() const override -// { -// return true; -// } -// }; +void registerFunctionSvg(FunctionFactory & factory) +{ + factory.registerFunction(); +} -// void registerFunctionSvg(FunctionFactory & factory) -// { -// factory.registerFunction(); -// } - -// } +} diff --git a/src/Functions/wkt.cpp b/src/Functions/wkt.cpp index f854f724ccb..17789b0450d 100644 --- a/src/Functions/wkt.cpp +++ b/src/Functions/wkt.cpp @@ -1,69 +1,68 @@ -// #include -// #include -// #include -// #include +#include +#include +#include +#include -// #include -// #include +#include +#include -// namespace DB -// { +namespace DB +{ -// class FunctionWkt : public IFunction -// { -// public: -// static inline const char * name = "wkt"; +class FunctionWkt : public IFunction +{ +public: + static inline const char * name = "wkt"; -// explicit FunctionWkt() = default; + explicit FunctionWkt() = default; -// static FunctionPtr create(const Context &) -// { -// return std::make_shared(); -// } + static FunctionPtr create(const Context &) + { + return std::make_shared(); + } -// String getName() const override -// { -// return name; -// } + String getName() const override + { + return name; + } -// size_t getNumberOfArguments() const override -// { -// return 1; -// } + size_t getNumberOfArguments() const override + { + return 1; + } -// DataTypePtr getReturnTypeImpl(const DataTypes &) const override -// { -// return std::make_shared(); -// } + DataTypePtr getReturnTypeImpl(const DataTypes &) const override + { + return std::make_shared(); + } -// ColumnPtr executeImpl(const ColumnsWithTypeAndName & arguments, const DataTypePtr & /*result_type*/, size_t input_rows_count) const override -// { -// auto parser = makeGeometryFromColumnParser(arguments[0]); -// auto res_column = ColumnString::create(); + ColumnPtr executeImpl(const ColumnsWithTypeAndName & arguments, const DataTypePtr & /*result_type*/, size_t input_rows_count) const override + { + auto parser = getConverterBasedOnType(arguments[0]); + auto figures = parseFigure(parser); -// auto container = createContainer(parser); + auto res_column = ColumnString::create(); -// for (size_t i = 0; i < input_rows_count; i++) -// { -// std::stringstream str; // STYLE_CHECK_ALLOW_STD_STRING_STREAM -// get(parser, container, i); -// str << boost::geometry::wkt(container); -// std::string serialized = str.str(); -// res_column->insertData(serialized.c_str(), serialized.size()); -// } + for (size_t i = 0; i < input_rows_count; i++) + { + std::stringstream str; // STYLE_CHECK_ALLOW_STD_STRING_STREAM + str << boost::geometry::wkt(figures[i]); + std::string serialized = str.str(); + res_column->insertData(serialized.c_str(), serialized.size()); + } -// return res_column; -// } + return res_column; + } -// bool useDefaultImplementationForConstants() const override -// { -// return true; -// } -// }; + bool useDefaultImplementationForConstants() const override + { + return true; + } +}; -// void registerFunctionWkt(FunctionFactory & factory) -// { -// factory.registerFunction(); -// } +void registerFunctionWkt(FunctionFactory & factory) +{ + factory.registerFunction(); +} -// } +} From 705c4dd60cf8934808bacf5177d2878c28625c52 Mon Sep 17 00:00:00 2001 From: Nikita Mikhailov Date: Sat, 20 Feb 2021 01:26:33 +0300 Subject: [PATCH 295/544] callOnGeometryDataType instead of visitor --- src/Functions/geometryConverters.cpp | 77 ++----- src/Functions/geometryConverters.h | 260 +++++++++--------------- src/Functions/polygonArea.cpp | 19 +- src/Functions/polygonConvexHull.cpp | 134 ++++++------ src/Functions/polygonPerimeter.cpp | 142 ++++++------- src/Functions/polygonsDistance.cpp | 152 +++++++------- src/Functions/polygonsEquals.cpp | 148 +++++++------- src/Functions/polygonsIntersection.cpp | 37 ++-- src/Functions/polygonsSymDifference.cpp | 150 +++++++------- src/Functions/polygonsUnion.cpp | 156 +++++++------- src/Functions/polygonsWithin.cpp | 152 +++++++------- src/Functions/readWkt.cpp | 2 +- src/Functions/registerFunctionsGeo.cpp | 28 +-- src/Functions/svg.cpp | 35 ++-- src/Functions/wkt.cpp | 23 ++- 15 files changed, 709 insertions(+), 806 deletions(-) diff --git a/src/Functions/geometryConverters.cpp b/src/Functions/geometryConverters.cpp index 0a3e31c7c73..8cd023f8c58 100644 --- a/src/Functions/geometryConverters.cpp +++ b/src/Functions/geometryConverters.cpp @@ -1,5 +1,4 @@ #include -#include #include @@ -14,7 +13,7 @@ namespace ErrorCodes } template -std::vector PointFromColumnParser::parse(size_t shift, size_t count) const +std::vector PointFromColumnParser::parseImpl(size_t shift, size_t count) const { const auto * tuple = typeid_cast(col.get()); const auto & tuple_columns = tuple->getColumns(); @@ -45,14 +44,15 @@ std::vector PointFromColumnParser::parse(size_t shift, size_t coun } template -std::vector> RingFromColumnParser::parse(size_t /*shift*/, size_t /*size*/) const +std::vector> RingFromColumnParser::parse() const { + const IColumn::Offsets & offsets = typeid_cast(*col).getOffsets(); size_t prev_offset = 0; std::vector> answer; answer.reserve(offsets.size()); for (size_t offset : offsets) { - auto tmp = point_parser.parse(prev_offset, offset - prev_offset); + auto tmp = point_parser.parseImpl(prev_offset, offset - prev_offset); answer.emplace_back(tmp.begin(), tmp.end()); prev_offset = offset; } @@ -60,10 +60,11 @@ std::vector> RingFromColumnParser::parse(size_t /*shift*/, si } template -std::vector> PolygonFromColumnParser::parse(size_t /*shift*/, size_t /*size*/) const +std::vector> PolygonFromColumnParser::parse() const { + const IColumn::Offsets & offsets = typeid_cast(*col).getOffsets(); std::vector> answer(offsets.size()); - auto all_rings = ring_parser.parse(0, 0); + auto all_rings = ring_parser.parse(); auto prev_offset = 0; for (size_t iter = 0; iter < offsets.size(); ++iter) @@ -81,12 +82,13 @@ std::vector> PolygonFromColumnParser::parse(size_t /*shift template -std::vector> MultiPolygonFromColumnParser::parse(size_t /*shift*/, size_t /*size*/) const +std::vector> MultiPolygonFromColumnParser::parse() const { + const IColumn::Offsets & offsets = typeid_cast(*col).getOffsets(); size_t prev_offset = 0; std::vector> answer(offsets.size()); - auto all_polygons = polygon_parser.parse(0, 0); + auto all_polygons = polygon_parser.parse(); for (size_t iter = 0; iter < offsets.size(); ++iter) { @@ -98,33 +100,15 @@ std::vector> MultiPolygonFromColumnParser::parse(size return answer; } -template -class ParserVisitor : public boost::static_visitor -{ -public: - template - ContainterWithFigures operator()(const T & parser) const - { - auto parsed = parser.parse(0, 0); - ContainterWithFigures figures; - figures.reserve(parsed.size()); - for (auto & value : parsed) - figures.emplace_back(value); - return figures; - } -}; - -template -std::vector> parseFigure(const GeometryFromColumnParser & parser) -{ - static ParserVisitor>> creator; - return boost::apply_visitor(creator, parser); -} - - -template std::vector> parseFigure(const GeometryFromColumnParser &); -template std::vector> parseFigure(const GeometryFromColumnParser &); +template class PointFromColumnParser; +template class PointFromColumnParser; +template class RingFromColumnParser; +template class RingFromColumnParser; +template class PolygonFromColumnParser; +template class PolygonFromColumnParser; +template class MultiPolygonFromColumnParser; +template class MultiPolygonFromColumnParser; template typename Desired> void checkColumnTypeOrThrow(const ColumnWithTypeAndName & column) @@ -150,29 +134,4 @@ template void checkColumnTypeOrThrow(const ColumnWithType template void checkColumnTypeOrThrow(const ColumnWithTypeAndName &); template void checkColumnTypeOrThrow(const ColumnWithTypeAndName &); -template -GeometryFromColumnParser getConverterBasedOnType(const ColumnWithTypeAndName & column) -{ - if (DataTypeCustomRingSerialization::nestedDataType()->equals(*column.type)) - { - return RingFromColumnParser(std::move(column.column->convertToFullColumnIfConst())); - } - else if (DataTypeCustomPolygonSerialization::nestedDataType()->equals(*column.type)) - { - return PolygonFromColumnParser(std::move(column.column->convertToFullColumnIfConst())); - } - else if (DataTypeCustomMultiPolygonSerialization::nestedDataType()->equals(*column.type)) - { - return MultiPolygonFromColumnParser(std::move(column.column->convertToFullColumnIfConst())); - } - else - { - throw Exception(fmt::format("Unexpected type of column {}", column.type->getName()), ErrorCodes::BAD_ARGUMENTS); - } -} - - -template GeometryFromColumnParser getConverterBasedOnType(const ColumnWithTypeAndName & column); -template GeometryFromColumnParser getConverterBasedOnType(const ColumnWithTypeAndName & column); - } diff --git a/src/Functions/geometryConverters.h b/src/Functions/geometryConverters.h index 77ecd3ea3db..6f56a53dd6f 100644 --- a/src/Functions/geometryConverters.h +++ b/src/Functions/geometryConverters.h @@ -14,6 +14,7 @@ #include #include #include +#include #include #include @@ -75,17 +76,19 @@ template class PointFromColumnParser { public: - using Container = std::conditional_t, CartesianGeometry, GeographicGeometry>; + PointFromColumnParser() = default; explicit PointFromColumnParser(ColumnPtr col_) : col(col_) { } - std::vector parse(size_t shift, size_t count) const; + std::vector parse() const; private: - /// To prevent use-after-free and increase column lifetime. - ColumnPtr col; + std::vector parseImpl(size_t shift, size_t count) const; + + friend class RingFromColumnParser; + ColumnPtr col{nullptr}; }; @@ -93,121 +96,98 @@ template class RingFromColumnParser { public: + RingFromColumnParser() = default; + explicit RingFromColumnParser(ColumnPtr col_) : col(col_) - , offsets(typeid_cast(*col_).getOffsets()) , point_parser(typeid_cast(*col_).getDataPtr()) { } - std::vector> parse(size_t /*shift*/, size_t /*size*/) const; + std::vector> parse() const; private: + friend class PointFromColumnParser; /// To prevent use-after-free and increase column lifetime. - ColumnPtr col; - const IColumn::Offsets & offsets; - const PointFromColumnParser point_parser; + ColumnPtr col{nullptr}; + const PointFromColumnParser point_parser{}; }; template class PolygonFromColumnParser { public: + PolygonFromColumnParser() = default; + explicit PolygonFromColumnParser(ColumnPtr col_) : col(col_) - , offsets(typeid_cast(*col_).getOffsets()) , ring_parser(typeid_cast(*col_).getDataPtr()) { } - std::vector> parse(size_t /*shift*/, size_t /*size*/) const; + std::vector> parse() const; private: friend class MultiPolygonFromColumnParser; /// To prevent use-after-free and increase column lifetime. - ColumnPtr col; - const IColumn::Offsets & offsets; - const RingFromColumnParser ring_parser; + ColumnPtr col{nullptr}; + const RingFromColumnParser ring_parser{}; }; template class MultiPolygonFromColumnParser { public: + MultiPolygonFromColumnParser() = default; + explicit MultiPolygonFromColumnParser(ColumnPtr col_) : col(col_) - , offsets(typeid_cast(*col_).getOffsets()) , polygon_parser(typeid_cast(*col_).getDataPtr()) {} - std::vector> parse(size_t /*shift*/, size_t /*size*/) const; + std::vector> parse() const; private: /// To prevent use-after-free and increase column lifetime. - ColumnPtr col; - const IColumn::Offsets & offsets; - const PolygonFromColumnParser polygon_parser; + ColumnPtr col{nullptr}; + const PolygonFromColumnParser polygon_parser{}; }; -template -using GeometryFromColumnParser = boost::variant< - RingFromColumnParser, - PolygonFromColumnParser, - MultiPolygonFromColumnParser ->; - -template -std::vector> parseFigure(const GeometryFromColumnParser & parser); - -extern template std::vector> parseFigure(const GeometryFromColumnParser &); -extern template std::vector> parseFigure(const GeometryFromColumnParser &); +extern template class PointFromColumnParser; +extern template class PointFromColumnParser; +extern template class RingFromColumnParser; +extern template class RingFromColumnParser; +extern template class PolygonFromColumnParser; +extern template class PolygonFromColumnParser; +extern template class MultiPolygonFromColumnParser; +extern template class MultiPolygonFromColumnParser; /// To serialize Geographic or Cartesian point (a pair of numbers in both cases). template -class PointSerializerVisitor : public boost::static_visitor +class PointSerializer { public: - PointSerializerVisitor() + PointSerializer() : first(ColumnFloat64::create()) , second(ColumnFloat64::create()) + , first_container(first->getData()) + , second_container(second->getData()) {} - explicit PointSerializerVisitor(size_t n) + explicit PointSerializer(size_t n) : first(ColumnFloat64::create(n)) , second(ColumnFloat64::create(n)) + , first_container(first->getData()) + , second_container(second->getData()) {} - void operator()(const Point & point) + void add(const Point & point) { - first->insertValue(point.template get<0>()); - second->insertValue(point.template get<1>()); - } - - void operator()(const Ring & ring) - { - if (ring.size() != 1) - throw Exception("Unable to write ring of size " + toString(ring.size()) + " != 1 to point column", ErrorCodes::BAD_ARGUMENTS); - - (*this)(ring[0]); - } - - void operator()(const Polygon & polygon) - { - if (polygon.inners().size() != 0) - throw Exception("Unable to write polygon with holes to point column", ErrorCodes::BAD_ARGUMENTS); - - (*this)(polygon.outer()); - } - - void operator()(const MultiPolygon & multi_polygon) - { - if (multi_polygon.size() != 1) - throw Exception("Unable to write multi-polygon of size " + toString(multi_polygon.size()) + " != 1 to point column", ErrorCodes::BAD_ARGUMENTS); - - (*this)(multi_polygon[0]); + first_container.emplace_back(point.template get<0>()); + second_container.emplace_back(point.template get<1>()); } ColumnPtr finalize() @@ -222,52 +202,29 @@ public: private: ColumnFloat64::MutablePtr first; ColumnFloat64::MutablePtr second; + + ColumnFloat64::Container & first_container; + ColumnFloat64::Container & second_container; }; template -class RingSerializerVisitor : public boost::static_visitor +class RingSerializer { public: - RingSerializerVisitor() + RingSerializer() : offsets(ColumnUInt64::create()) {} - explicit RingSerializerVisitor(size_t n) + explicit RingSerializer(size_t n) : offsets(ColumnUInt64::create(n)) {} - void operator()(const Point & point) - { - size++; - offsets->insertValue(size); - - point_serializer(point); - } - - void operator()(const Ring & ring) + void add(const Ring & ring) { size += ring.size(); offsets->insertValue(size); for (const auto & point : ring) - { - point_serializer(point); - } - } - - void operator()(const Polygon & polygon) - { - if (polygon.inners().size() != 0) - throw Exception("Unable to write polygon with holes to ring column", ErrorCodes::BAD_ARGUMENTS); - - (*this)(polygon.outer()); - } - - void operator()(const MultiPolygon & multi_polygon) - { - if (multi_polygon.size() != 1) - throw Exception("Unable to write multi-polygon of size " + toString(multi_polygon.size()) + " != 1 to ring column", ErrorCodes::BAD_ARGUMENTS); - - (*this)(multi_polygon[0]); + point_serializer.add(point); } ColumnPtr finalize() @@ -277,53 +234,36 @@ public: private: size_t size = 0; - PointSerializerVisitor point_serializer; + PointSerializer point_serializer; ColumnUInt64::MutablePtr offsets; }; template -class PolygonSerializerVisitor : public boost::static_visitor +class PolygonSerializer { public: - PolygonSerializerVisitor() + PolygonSerializer() : offsets(ColumnUInt64::create()) {} - explicit PolygonSerializerVisitor(size_t n) + explicit PolygonSerializer(size_t n) : offsets(ColumnUInt64::create(n)) {} - void operator()(const Point & point) + void add(const Ring & ring) { size++; offsets->insertValue(size); - ring_serializer(point); + ring_serializer.add(ring); } - void operator()(const Ring & ring) - { - size++; - offsets->insertValue(size); - ring_serializer(ring); - } - - void operator()(const Polygon & polygon) + void add(const Polygon & polygon) { size += 1 + polygon.inners().size(); offsets->insertValue(size); - ring_serializer(polygon.outer()); + ring_serializer.add(polygon.outer()); for (const auto & ring : polygon.inners()) - { - ring_serializer(ring); - } - } - - void operator()(const MultiPolygon & multi_polygon) - { - if (multi_polygon.size() != 1) - throw Exception("Unable to write multi-polygon of size " + toString(multi_polygon.size()) + " != 1 to polygon column", ErrorCodes::BAD_ARGUMENTS); - - (*this)(multi_polygon[0]); + ring_serializer.add(ring); } ColumnPtr finalize() @@ -333,50 +273,43 @@ public: private: size_t size = 0; - RingSerializerVisitor ring_serializer; + RingSerializer ring_serializer; ColumnUInt64::MutablePtr offsets; }; template -class MultiPolygonSerializerVisitor : public boost::static_visitor +class MultiPolygonSerializer { public: - MultiPolygonSerializerVisitor() + MultiPolygonSerializer() : offsets(ColumnUInt64::create()) {} - explicit MultiPolygonSerializerVisitor(size_t n) + explicit MultiPolygonSerializer(size_t n) : offsets(ColumnUInt64::create(n)) {} - void operator()(const Point & point) + void add(const Ring & ring) { size++; offsets->insertValue(size); - polygon_serializer(point); + polygon_serializer.add(ring); } - void operator()(const Ring & ring) + void add(const Polygon & polygon) { size++; offsets->insertValue(size); - polygon_serializer(ring); + polygon_serializer.add(polygon); } - void operator()(const Polygon & polygon) - { - size++; - offsets->insertValue(size); - polygon_serializer(polygon); - } - - void operator()(const MultiPolygon & multi_polygon) + void add(const MultiPolygon & multi_polygon) { size += multi_polygon.size(); offsets->insertValue(size); for (const auto & polygon : multi_polygon) { - polygon_serializer(polygon); + polygon_serializer.add(polygon); } } @@ -387,44 +320,37 @@ public: private: size_t size = 0; - PolygonSerializerVisitor polygon_serializer; + PolygonSerializer polygon_serializer; ColumnUInt64::MutablePtr offsets; }; -template -class GeometrySerializer +template +static void callOnGeometryDataType(DataTypePtr type, F && f) { -public: - void add(const Geometry & geometry) + if (DataTypeCustomRingSerialization::nestedDataType()->equals(*type)) + return f(RingFromColumnParser()); + if (DataTypeCustomPolygonSerialization::nestedDataType()->equals(*type)) + return f(PolygonFromColumnParser()); + if (DataTypeCustomMultiPolygonSerialization::nestedDataType()->equals(*type)) + return f(MultiPolygonFromColumnParser()); + throw Exception(fmt::format("Unknown geometry type {}", type->getName()), ErrorCodes::BAD_ARGUMENTS); +} + + +template +static void callOnTwoGeometryDataTypes(DataTypePtr left_type, DataTypePtr right_type, F && func) +{ + return callOnGeometryDataType(left_type, [&](const auto & left_types) { - boost::apply_visitor(visitor, geometry); - } + using LeftParser = std::decay_t; - ColumnPtr finalize() - { - return visitor.finalize(); - } -private: - Visitor visitor; -}; + return callOnGeometryDataType(right_type, [&](const auto & right_types) + { + using RightParser = std::decay_t; -template -using PointSerializer = GeometrySerializer, PointSerializerVisitor>; - -template -using RingSerializer = GeometrySerializer, RingSerializerVisitor>; - -template -using PolygonSerializer = GeometrySerializer, PolygonSerializerVisitor>; - -template -using MultiPolygonSerializer = GeometrySerializer, MultiPolygonSerializerVisitor>; - - -template typename Desired> -void checkColumnTypeOrThrow(const ColumnWithTypeAndName & column); - -template -GeometryFromColumnParser getConverterBasedOnType(const ColumnWithTypeAndName & column); + return func(LeftParser(), RightParser()); + }); + }); +} } diff --git a/src/Functions/polygonArea.cpp b/src/Functions/polygonArea.cpp index b719346dd9a..44fd4a89139 100644 --- a/src/Functions/polygonArea.cpp +++ b/src/Functions/polygonArea.cpp @@ -55,17 +55,22 @@ public: ColumnPtr executeImpl(const ColumnsWithTypeAndName & arguments, const DataTypePtr & /*result_type*/, size_t input_rows_count) const override { - auto parser = getConverterBasedOnType(arguments[0]); - auto figures = parseFigure(parser); - auto res_column = ColumnFloat64::create(); - auto & res_data = res_column->getData(); - res_data.reserve(input_rows_count); - for (size_t i = 0; i < input_rows_count; i++) + callOnGeometryDataType(arguments[0].type, [&] (const auto & type) { - res_data.emplace_back(boost::geometry::area(figures[i])); + using TypeParser = std::decay_t; + // using Parser = TypeParser::Type; + TypeParser parser(arguments[0].column->convertToFullColumnIfConst()); + auto figures = parser.parse(); + + auto & res_data = res_column->getData(); + res_data.reserve(input_rows_count); + + for (size_t i = 0; i < input_rows_count; i++) + res_data.emplace_back(boost::geometry::area(figures[i])); } + ); return res_column; } diff --git a/src/Functions/polygonConvexHull.cpp b/src/Functions/polygonConvexHull.cpp index e9fc40c4fb2..6c9eb167fb1 100644 --- a/src/Functions/polygonConvexHull.cpp +++ b/src/Functions/polygonConvexHull.cpp @@ -1,89 +1,89 @@ -#include -#include +// #include +// #include -#include -#include -#include +// #include +// #include +// #include -#include +// #include -#include -#include -#include -#include -#include -#include +// #include +// #include +// #include +// #include +// #include +// #include -#include -#include +// #include +// #include -namespace DB -{ +// namespace DB +// { -template -class FunctionPolygonConvexHull : public IFunction -{ -public: - static const char * name; +// template +// class FunctionPolygonConvexHull : public IFunction +// { +// public: +// static const char * name; - explicit FunctionPolygonConvexHull() = default; +// explicit FunctionPolygonConvexHull() = default; - static FunctionPtr create(const Context &) - { - return std::make_shared(); - } +// static FunctionPtr create(const Context &) +// { +// return std::make_shared(); +// } - String getName() const override - { - return name; - } +// String getName() const override +// { +// return name; +// } - bool isVariadic() const override - { - return false; - } +// bool isVariadic() const override +// { +// return false; +// } - size_t getNumberOfArguments() const override - { - return 1; - } +// size_t getNumberOfArguments() const override +// { +// return 1; +// } - DataTypePtr getReturnTypeImpl(const DataTypes &) const override - { - return DataTypeCustomPolygonSerialization::nestedDataType(); - } +// DataTypePtr getReturnTypeImpl(const DataTypes &) const override +// { +// return DataTypeCustomPolygonSerialization::nestedDataType(); +// } - ColumnPtr executeImpl(const ColumnsWithTypeAndName & arguments, const DataTypePtr & /*result_type*/, size_t input_rows_count) const override - { - auto parser = getConverterBasedOnType(arguments[0]); - auto figures = parseFigure(parser); +// ColumnPtr executeImpl(const ColumnsWithTypeAndName & arguments, const DataTypePtr & /*result_type*/, size_t input_rows_count) const override +// { +// auto parser = getConverterBasedOnType(arguments[0]); +// auto figures = parseFigure(parser); - PolygonSerializer serializer; +// PolygonSerializer serializer; - for (size_t i = 0; i < input_rows_count; i++) - { - Polygon convex_hull{}; - boost::geometry::convex_hull(figures[i], convex_hull); - serializer.add(convex_hull); - } +// for (size_t i = 0; i < input_rows_count; i++) +// { +// Polygon convex_hull{}; +// boost::geometry::convex_hull(figures[i], convex_hull); +// serializer.add(convex_hull); +// } - return serializer.finalize(); - } +// return serializer.finalize(); +// } - bool useDefaultImplementationForConstants() const override - { - return true; - } -}; +// bool useDefaultImplementationForConstants() const override +// { +// return true; +// } +// }; -template <> -const char * FunctionPolygonConvexHull::name = "polygonConvexHullCartesian"; +// template <> +// const char * FunctionPolygonConvexHull::name = "polygonConvexHullCartesian"; -void registerFunctionPolygonConvexHull(FunctionFactory & factory) -{ - factory.registerFunction>(); -} +// void registerFunctionPolygonConvexHull(FunctionFactory & factory) +// { +// factory.registerFunction>(); +// } -} +// } diff --git a/src/Functions/polygonPerimeter.cpp b/src/Functions/polygonPerimeter.cpp index 31543a4eedd..f832d2965f9 100644 --- a/src/Functions/polygonPerimeter.cpp +++ b/src/Functions/polygonPerimeter.cpp @@ -1,94 +1,94 @@ -#include -#include +// #include +// #include -#include -#include -#include +// #include +// #include +// #include -#include +// #include -#include -#include -#include -#include -#include -#include +// #include +// #include +// #include +// #include +// #include +// #include -#include -#include +// #include +// #include -namespace DB -{ +// namespace DB +// { -template -class FunctionPolygonPerimeter : public IFunction -{ -public: - static const char * name; +// template +// class FunctionPolygonPerimeter : public IFunction +// { +// public: +// static const char * name; - explicit FunctionPolygonPerimeter() = default; +// explicit FunctionPolygonPerimeter() = default; - static FunctionPtr create(const Context &) - { - return std::make_shared(); - } +// static FunctionPtr create(const Context &) +// { +// return std::make_shared(); +// } - String getName() const override - { - return name; - } +// String getName() const override +// { +// return name; +// } - bool isVariadic() const override - { - return false; - } +// bool isVariadic() const override +// { +// return false; +// } - size_t getNumberOfArguments() const override - { - return 1; - } +// size_t getNumberOfArguments() const override +// { +// return 1; +// } - DataTypePtr getReturnTypeImpl(const DataTypes &) const override - { - return std::make_shared(); - } +// DataTypePtr getReturnTypeImpl(const DataTypes &) const override +// { +// return std::make_shared(); +// } - ColumnPtr executeImpl(const ColumnsWithTypeAndName & arguments, const DataTypePtr & /*result_type*/, size_t input_rows_count) const override - { - auto parser = getConverterBasedOnType(arguments[0]); - auto figures = parseFigure(parser); +// ColumnPtr executeImpl(const ColumnsWithTypeAndName & arguments, const DataTypePtr & /*result_type*/, size_t input_rows_count) const override +// { +// auto parser = getConverterBasedOnType(arguments[0]); +// auto figures = parseFigure(parser); - auto res_column = ColumnFloat64::create(); - auto & res_data = res_column->getData(); - res_data.reserve(input_rows_count); +// auto res_column = ColumnFloat64::create(); +// auto & res_data = res_column->getData(); +// res_data.reserve(input_rows_count); - for (size_t i = 0; i < input_rows_count; i++) - { - boost::geometry::correct(figures[i]); - res_data.emplace_back(boost::geometry::perimeter(figures[i])); - } +// for (size_t i = 0; i < input_rows_count; i++) +// { +// boost::geometry::correct(figures[i]); +// res_data.emplace_back(boost::geometry::perimeter(figures[i])); +// } - return res_column; - } +// return res_column; +// } - bool useDefaultImplementationForConstants() const override - { - return true; - } -}; +// bool useDefaultImplementationForConstants() const override +// { +// return true; +// } +// }; -template <> -const char * FunctionPolygonPerimeter::name = "polygonPerimeterCartesian"; +// template <> +// const char * FunctionPolygonPerimeter::name = "polygonPerimeterCartesian"; -template <> -const char * FunctionPolygonPerimeter::name = "polygonPerimeterGeographic"; +// template <> +// const char * FunctionPolygonPerimeter::name = "polygonPerimeterGeographic"; -void registerFunctionPolygonPerimeter(FunctionFactory & factory) -{ - factory.registerFunction>(); - factory.registerFunction>(); -} +// void registerFunctionPolygonPerimeter(FunctionFactory & factory) +// { +// factory.registerFunction>(); +// factory.registerFunction>(); +// } -} +// } diff --git a/src/Functions/polygonsDistance.cpp b/src/Functions/polygonsDistance.cpp index 600473724f4..679345dc693 100644 --- a/src/Functions/polygonsDistance.cpp +++ b/src/Functions/polygonsDistance.cpp @@ -1,101 +1,101 @@ -#include -#include +// #include +// #include -#include -#include -#include +// #include +// #include +// #include -#include +// #include -#include -#include -#include -#include -#include -#include -#include -#include +// #include +// #include +// #include +// #include +// #include +// #include +// #include +// #include -#include -#include +// #include +// #include -namespace DB -{ +// namespace DB +// { -template -class FunctionPolygonsDistance : public IFunction -{ -public: - static inline const char * name; +// template +// class FunctionPolygonsDistance : public IFunction +// { +// public: +// static inline const char * name; - explicit FunctionPolygonsDistance() = default; +// explicit FunctionPolygonsDistance() = default; - static FunctionPtr create(const Context &) - { - return std::make_shared(); - } +// static FunctionPtr create(const Context &) +// { +// return std::make_shared(); +// } - String getName() const override - { - return name; - } +// String getName() const override +// { +// return name; +// } - bool isVariadic() const override - { - return false; - } +// bool isVariadic() const override +// { +// return false; +// } - size_t getNumberOfArguments() const override - { - return 2; - } +// size_t getNumberOfArguments() const override +// { +// return 2; +// } - DataTypePtr getReturnTypeImpl(const DataTypes &) const override - { - return std::make_shared(); - } +// DataTypePtr getReturnTypeImpl(const DataTypes &) const override +// { +// return std::make_shared(); +// } - ColumnPtr executeImpl(const ColumnsWithTypeAndName & arguments, const DataTypePtr & /*result_type*/, size_t input_rows_count) const override - { - auto first_parser = getConverterBasedOnType(arguments[0]); - auto second_parser = getConverterBasedOnType(arguments[1]); +// ColumnPtr executeImpl(const ColumnsWithTypeAndName & arguments, const DataTypePtr & /*result_type*/, size_t input_rows_count) const override +// { +// auto first_parser = getConverterBasedOnType(arguments[0]); +// auto second_parser = getConverterBasedOnType(arguments[1]); - auto first = parseFigure(first_parser); - auto second = parseFigure(second_parser); +// auto first = parseFigure(first_parser); +// auto second = parseFigure(second_parser); - auto res_column = ColumnFloat64::create(); - auto & res_data = res_column->getData(); - res_data.reserve(input_rows_count); +// auto res_column = ColumnFloat64::create(); +// auto & res_data = res_column->getData(); +// res_data.reserve(input_rows_count); - for (size_t i = 0; i < input_rows_count; i++) - { - boost::geometry::correct(first[i]); - boost::geometry::correct(second[i]); +// for (size_t i = 0; i < input_rows_count; i++) +// { +// boost::geometry::correct(first[i]); +// boost::geometry::correct(second[i]); - res_data.emplace_back(boost::geometry::distance(first[i], second[i])); - } +// res_data.emplace_back(boost::geometry::distance(first[i], second[i])); +// } - return res_column; - } +// return res_column; +// } - bool useDefaultImplementationForConstants() const override - { - return true; - } -}; +// bool useDefaultImplementationForConstants() const override +// { +// return true; +// } +// }; -template <> -const char * FunctionPolygonsDistance::name = "polygonsDistanceCartesian"; +// template <> +// const char * FunctionPolygonsDistance::name = "polygonsDistanceCartesian"; -template <> -const char * FunctionPolygonsDistance::name = "polygonsDistanceGeographic"; +// template <> +// const char * FunctionPolygonsDistance::name = "polygonsDistanceGeographic"; -void registerFunctionPolygonsDistance(FunctionFactory & factory) -{ - factory.registerFunction>(); - factory.registerFunction>(); -} +// void registerFunctionPolygonsDistance(FunctionFactory & factory) +// { +// factory.registerFunction>(); +// factory.registerFunction>(); +// } -} +// } diff --git a/src/Functions/polygonsEquals.cpp b/src/Functions/polygonsEquals.cpp index 39e7594a7a2..8349e8837cd 100644 --- a/src/Functions/polygonsEquals.cpp +++ b/src/Functions/polygonsEquals.cpp @@ -1,98 +1,98 @@ -#include -#include +// #include +// #include -#include -#include -#include +// #include +// #include +// #include -#include +// #include -#include -#include -#include -#include -#include -#include -#include -#include +// #include +// #include +// #include +// #include +// #include +// #include +// #include +// #include -#include -#include +// #include +// #include -namespace DB -{ +// namespace DB +// { -template -class FunctionPolygonsEquals : public IFunction -{ -public: - static const char * name; +// template +// class FunctionPolygonsEquals : public IFunction +// { +// public: +// static const char * name; - explicit FunctionPolygonsEquals() = default; +// explicit FunctionPolygonsEquals() = default; - static FunctionPtr create(const Context &) - { - return std::make_shared(); - } +// static FunctionPtr create(const Context &) +// { +// return std::make_shared(); +// } - String getName() const override - { - return name; - } +// String getName() const override +// { +// return name; +// } - bool isVariadic() const override - { - return false; - } +// bool isVariadic() const override +// { +// return false; +// } - size_t getNumberOfArguments() const override - { - return 2; - } +// size_t getNumberOfArguments() const override +// { +// return 2; +// } - DataTypePtr getReturnTypeImpl(const DataTypes &) const override - { - return std::make_shared(); - } +// DataTypePtr getReturnTypeImpl(const DataTypes &) const override +// { +// return std::make_shared(); +// } - ColumnPtr executeImpl(const ColumnsWithTypeAndName & arguments, const DataTypePtr & /*result_type*/, size_t input_rows_count) const override - { - auto first_parser = getConverterBasedOnType(arguments[0]); - auto second_parser = getConverterBasedOnType(arguments[1]); +// ColumnPtr executeImpl(const ColumnsWithTypeAndName & arguments, const DataTypePtr & /*result_type*/, size_t input_rows_count) const override +// { +// auto first_parser = getConverterBasedOnType(arguments[0]); +// auto second_parser = getConverterBasedOnType(arguments[1]); - auto first = parseFigure(first_parser); - auto second = parseFigure(second_parser); +// auto first = parseFigure(first_parser); +// auto second = parseFigure(second_parser); - auto res_column = ColumnUInt8::create(); - auto & res_data = res_column->getData(); - res_data.reserve(input_rows_count); +// auto res_column = ColumnUInt8::create(); +// auto & res_data = res_column->getData(); +// res_data.reserve(input_rows_count); - for (size_t i = 0; i < input_rows_count; i++) - { - boost::geometry::correct(first[i]); - boost::geometry::correct(second[i]); +// for (size_t i = 0; i < input_rows_count; i++) +// { +// boost::geometry::correct(first[i]); +// boost::geometry::correct(second[i]); - /// Main work here. - res_data.emplace_back(boost::geometry::equals(first[i], second[i])); - } +// /// Main work here. +// res_data.emplace_back(boost::geometry::equals(first[i], second[i])); +// } - return res_column; - } +// return res_column; +// } - bool useDefaultImplementationForConstants() const override - { - return true; - } -}; +// bool useDefaultImplementationForConstants() const override +// { +// return true; +// } +// }; -template <> -const char * FunctionPolygonsEquals::name = "polygonsEqualsCartesian"; +// template <> +// const char * FunctionPolygonsEquals::name = "polygonsEqualsCartesian"; -void registerFunctionPolygonsEquals(FunctionFactory & factory) -{ - factory.registerFunction>(); -} +// void registerFunctionPolygonsEquals(FunctionFactory & factory) +// { +// factory.registerFunction>(); +// } -} +// } diff --git a/src/Functions/polygonsIntersection.cpp b/src/Functions/polygonsIntersection.cpp index 54b67b952ed..2e23770e881 100644 --- a/src/Functions/polygonsIntersection.cpp +++ b/src/Functions/polygonsIntersection.cpp @@ -57,29 +57,30 @@ public: ColumnPtr executeImpl(const ColumnsWithTypeAndName & arguments, const DataTypePtr & /*result_type*/, size_t input_rows_count) const override { - auto first_parser = getConverterBasedOnType(arguments[0]); - auto second_parser = getConverterBasedOnType(arguments[1]); - MultiPolygonSerializer serializer; - auto first = parseFigure(first_parser); - auto second = parseFigure(second_parser); + callOnTwoGeometryDataTypes(arguments[0].type, arguments[1].type, [&](const auto & left_type, const auto & right_type) { + using LeftParser = std::decay_t; + using RightParser = std::decay_t; - /// We are not interested in some pitfalls in third-party libraries - /// NOLINTNEXTLINE(clang-analyzer-core.uninitialized.Assign) - for (size_t i = 0; i < input_rows_count; ++i) - { - /// Orient the polygons correctly. - boost::geometry::correct(first[i]); - boost::geometry::correct(second[i]); - - MultiPolygon intersection{}; + auto first = LeftParser(arguments[0].column->convertToFullColumnIfConst()).parse(); + auto second = RightParser(arguments[1].column->convertToFullColumnIfConst()).parse(); - /// Main work here. - boost::geometry::intersection(first[i], second[i], intersection); + /// We are not interested in some pitfalls in third-party libraries + /// NOLINTNEXTLINE(clang-analyzer-core.uninitialized.Assign) + for (size_t i = 0; i < input_rows_count; ++i) + { + /// Orient the polygons correctly. + boost::geometry::correct(first[i]); + boost::geometry::correct(second[i]); - serializer.add(intersection); - } + MultiPolygon intersection{}; + /// Main work here. + boost::geometry::intersection(first[i], second[i], intersection); + + serializer.add(intersection); + } + }); return serializer.finalize(); } diff --git a/src/Functions/polygonsSymDifference.cpp b/src/Functions/polygonsSymDifference.cpp index 839f02738c5..bec6e10ba9b 100644 --- a/src/Functions/polygonsSymDifference.cpp +++ b/src/Functions/polygonsSymDifference.cpp @@ -1,99 +1,99 @@ -#include -#include +// #include +// #include -#include -#include -#include +// #include +// #include +// #include -#include +// #include -#include -#include -#include -#include -#include -#include +// #include +// #include +// #include +// #include +// #include +// #include -#include -#include +// #include +// #include -namespace DB -{ +// namespace DB +// { -template -class FunctionPolygonsSymDifference : public IFunction -{ -public: - static const char * name; +// template +// class FunctionPolygonsSymDifference : public IFunction +// { +// public: +// static const char * name; - explicit FunctionPolygonsSymDifference() = default; +// explicit FunctionPolygonsSymDifference() = default; - static FunctionPtr create(const Context &) - { - return std::make_shared(); - } +// static FunctionPtr create(const Context &) +// { +// return std::make_shared(); +// } - String getName() const override - { - return name; - } +// String getName() const override +// { +// return name; +// } - bool isVariadic() const override - { - return false; - } +// bool isVariadic() const override +// { +// return false; +// } - size_t getNumberOfArguments() const override - { - return 2; - } +// size_t getNumberOfArguments() const override +// { +// return 2; +// } - DataTypePtr getReturnTypeImpl(const DataTypes &) const override - { - return DataTypeCustomMultiPolygonSerialization::nestedDataType(); - } +// DataTypePtr getReturnTypeImpl(const DataTypes &) const override +// { +// return DataTypeCustomMultiPolygonSerialization::nestedDataType(); +// } - ColumnPtr executeImpl(const ColumnsWithTypeAndName & arguments, const DataTypePtr & /*result_type*/, size_t input_rows_count) const override - { - auto first_parser = getConverterBasedOnType(arguments[0]); - auto second_parser = getConverterBasedOnType(arguments[1]); +// ColumnPtr executeImpl(const ColumnsWithTypeAndName & arguments, const DataTypePtr & /*result_type*/, size_t input_rows_count) const override +// { +// auto first_parser = getConverterBasedOnType(arguments[0]); +// auto second_parser = getConverterBasedOnType(arguments[1]); - auto first = parseFigure(first_parser); - auto second = parseFigure(second_parser); +// auto first = parseFigure(first_parser); +// auto second = parseFigure(second_parser); - MultiPolygonSerializer serializer; +// MultiPolygonSerializer serializer; - /// NOLINTNEXTLINE(clang-analyzer-core.uninitialized.Assign) - for (size_t i = 0; i < input_rows_count; i++) - { - boost::geometry::correct(first[i]); - boost::geometry::correct(second[i]); +// /// NOLINTNEXTLINE(clang-analyzer-core.uninitialized.Assign) +// for (size_t i = 0; i < input_rows_count; i++) +// { +// boost::geometry::correct(first[i]); +// boost::geometry::correct(second[i]); - MultiPolygon sym_difference{}; - boost::geometry::sym_difference(first[i], second[i], sym_difference); +// MultiPolygon sym_difference{}; +// boost::geometry::sym_difference(first[i], second[i], sym_difference); - serializer.add(sym_difference); - } +// serializer.add(sym_difference); +// } - return serializer.finalize(); - } +// return serializer.finalize(); +// } - bool useDefaultImplementationForConstants() const override - { - return true; - } -}; +// bool useDefaultImplementationForConstants() const override +// { +// return true; +// } +// }; -template <> -const char * FunctionPolygonsSymDifference::name = "polygonsSymDifferenceCartesian"; +// template <> +// const char * FunctionPolygonsSymDifference::name = "polygonsSymDifferenceCartesian"; -template <> -const char * FunctionPolygonsSymDifference::name = "polygonsSymDifferenceGeographic"; +// template <> +// const char * FunctionPolygonsSymDifference::name = "polygonsSymDifferenceGeographic"; -void registerFunctionPolygonsSymDifference(FunctionFactory & factory) -{ - factory.registerFunction>(); - factory.registerFunction>(); -} +// void registerFunctionPolygonsSymDifference(FunctionFactory & factory) +// { +// factory.registerFunction>(); +// factory.registerFunction>(); +// } -} +// } diff --git a/src/Functions/polygonsUnion.cpp b/src/Functions/polygonsUnion.cpp index 6d5cd1ca266..3922c4887ca 100644 --- a/src/Functions/polygonsUnion.cpp +++ b/src/Functions/polygonsUnion.cpp @@ -1,103 +1,103 @@ -#include -#include +// #include +// #include -#include -#include -#include +// #include +// #include +// #include -#include +// #include -#include -#include -#include -#include -#include -#include +// #include +// #include +// #include +// #include +// #include +// #include -#include -#include +// #include +// #include -namespace DB -{ +// namespace DB +// { -template -class FunctionPolygonsUnion : public IFunction -{ -public: - static inline const char * name; +// template +// class FunctionPolygonsUnion : public IFunction +// { +// public: +// static inline const char * name; - explicit FunctionPolygonsUnion() = default; +// explicit FunctionPolygonsUnion() = default; - static FunctionPtr create(const Context &) - { - return std::make_shared(); - } +// static FunctionPtr create(const Context &) +// { +// return std::make_shared(); +// } - String getName() const override - { - return name; - } +// String getName() const override +// { +// return name; +// } - bool isVariadic() const override - { - return false; - } +// bool isVariadic() const override +// { +// return false; +// } - size_t getNumberOfArguments() const override - { - return 2; - } +// size_t getNumberOfArguments() const override +// { +// return 2; +// } - DataTypePtr getReturnTypeImpl(const DataTypes &) const override - { - return DataTypeCustomMultiPolygonSerialization::nestedDataType(); - } +// DataTypePtr getReturnTypeImpl(const DataTypes &) const override +// { +// return DataTypeCustomMultiPolygonSerialization::nestedDataType(); +// } - ColumnPtr executeImpl(const ColumnsWithTypeAndName & arguments, const DataTypePtr & /*result_type*/, size_t input_rows_count) const override - { - auto first_parser = getConverterBasedOnType(arguments[0]); - auto second_parser = getConverterBasedOnType(arguments[1]); +// ColumnPtr executeImpl(const ColumnsWithTypeAndName & arguments, const DataTypePtr & /*result_type*/, size_t input_rows_count) const override +// { +// auto first_parser = getConverterBasedOnType(arguments[0]); +// auto second_parser = getConverterBasedOnType(arguments[1]); - auto first = parseFigure(first_parser); - auto second = parseFigure(second_parser); +// auto first = parseFigure(first_parser); +// auto second = parseFigure(second_parser); - MultiPolygonSerializer serializer; +// MultiPolygonSerializer serializer; - /// We are not interested in some pitfalls in third-party libraries - /// NOLINTNEXTLINE(clang-analyzer-core.uninitialized.Assign) - for (size_t i = 0; i < input_rows_count; i++) - { - /// Orient the polygons correctly. - boost::geometry::correct(first[i]); - boost::geometry::correct(second[i]); +// /// We are not interested in some pitfalls in third-party libraries +// /// NOLINTNEXTLINE(clang-analyzer-core.uninitialized.Assign) +// for (size_t i = 0; i < input_rows_count; i++) +// { +// /// Orient the polygons correctly. +// boost::geometry::correct(first[i]); +// boost::geometry::correct(second[i]); - MultiPolygon polygons_union{}; - /// Main work here. - boost::geometry::union_(first[i], second[i], polygons_union); +// MultiPolygon polygons_union{}; +// /// Main work here. +// boost::geometry::union_(first[i], second[i], polygons_union); - serializer.add(polygons_union); - } +// serializer.add(polygons_union); +// } - return serializer.finalize(); - } +// return serializer.finalize(); +// } - bool useDefaultImplementationForConstants() const override - { - return true; - } -}; +// bool useDefaultImplementationForConstants() const override +// { +// return true; +// } +// }; -template <> -const char * FunctionPolygonsUnion::name = "polygonsUnionCartesian"; +// template <> +// const char * FunctionPolygonsUnion::name = "polygonsUnionCartesian"; -template <> -const char * FunctionPolygonsUnion::name = "polygonsUnionGeographic"; +// template <> +// const char * FunctionPolygonsUnion::name = "polygonsUnionGeographic"; -void registerFunctionPolygonsUnion(FunctionFactory & factory) -{ - factory.registerFunction>(); - factory.registerFunction>(); -} +// void registerFunctionPolygonsUnion(FunctionFactory & factory) +// { +// factory.registerFunction>(); +// factory.registerFunction>(); +// } -} +// } diff --git a/src/Functions/polygonsWithin.cpp b/src/Functions/polygonsWithin.cpp index be2e5cbb3a5..0b77ba45582 100644 --- a/src/Functions/polygonsWithin.cpp +++ b/src/Functions/polygonsWithin.cpp @@ -1,101 +1,101 @@ -#include -#include +// #include +// #include -#include -#include -#include +// #include +// #include +// #include -#include +// #include -#include -#include -#include -#include -#include -#include -#include -#include +// #include +// #include +// #include +// #include +// #include +// #include +// #include +// #include -#include -#include +// #include +// #include -namespace DB -{ +// namespace DB +// { -template -class FunctionPolygonsWithin : public IFunction -{ -public: - static inline const char * name; +// template +// class FunctionPolygonsWithin : public IFunction +// { +// public: +// static inline const char * name; - explicit FunctionPolygonsWithin() = default; +// explicit FunctionPolygonsWithin() = default; - static FunctionPtr create(const Context &) - { - return std::make_shared(); - } +// static FunctionPtr create(const Context &) +// { +// return std::make_shared(); +// } - String getName() const override - { - return name; - } +// String getName() const override +// { +// return name; +// } - bool isVariadic() const override - { - return false; - } +// bool isVariadic() const override +// { +// return false; +// } - size_t getNumberOfArguments() const override - { - return 2; - } +// size_t getNumberOfArguments() const override +// { +// return 2; +// } - DataTypePtr getReturnTypeImpl(const DataTypes &) const override - { - return std::make_shared(); - } +// DataTypePtr getReturnTypeImpl(const DataTypes &) const override +// { +// return std::make_shared(); +// } - ColumnPtr executeImpl(const ColumnsWithTypeAndName & arguments, const DataTypePtr & /*result_type*/, size_t input_rows_count) const override - { - auto first_parser = getConverterBasedOnType(arguments[0]); - auto second_parser = getConverterBasedOnType(arguments[1]); +// ColumnPtr executeImpl(const ColumnsWithTypeAndName & arguments, const DataTypePtr & /*result_type*/, size_t input_rows_count) const override +// { +// auto first_parser = getConverterBasedOnType(arguments[0]); +// auto second_parser = getConverterBasedOnType(arguments[1]); - auto first = parseFigure(first_parser); - auto second = parseFigure(second_parser); +// auto first = parseFigure(first_parser); +// auto second = parseFigure(second_parser); - auto res_column = ColumnUInt8::create(); - auto & res_data = res_column->getData(); +// auto res_column = ColumnUInt8::create(); +// auto & res_data = res_column->getData(); - /// NOLINTNEXTLINE(clang-analyzer-core.uninitialized.Assign) - for (size_t i = 0; i < input_rows_count; i++) - { - boost::geometry::correct(first[i]); - boost::geometry::correct(second[i]); +// /// NOLINTNEXTLINE(clang-analyzer-core.uninitialized.Assign) +// for (size_t i = 0; i < input_rows_count; i++) +// { +// boost::geometry::correct(first[i]); +// boost::geometry::correct(second[i]); - res_data.emplace_back(boost::geometry::within(first[i], second[i])); - } +// res_data.emplace_back(boost::geometry::within(first[i], second[i])); +// } - return res_column; - } +// return res_column; +// } - bool useDefaultImplementationForConstants() const override - { - return true; - } -}; +// bool useDefaultImplementationForConstants() const override +// { +// return true; +// } +// }; -template <> -const char * FunctionPolygonsWithin::name = "polygonsWithinCartesian"; +// template <> +// const char * FunctionPolygonsWithin::name = "polygonsWithinCartesian"; -template <> -const char * FunctionPolygonsWithin::name = "polygonsWithinGeographic"; +// template <> +// const char * FunctionPolygonsWithin::name = "polygonsWithinGeographic"; -void registerFunctionPolygonsWithin(FunctionFactory & factory) -{ - factory.registerFunction>(); - factory.registerFunction>(); -} +// void registerFunctionPolygonsWithin(FunctionFactory & factory) +// { +// factory.registerFunction>(); +// factory.registerFunction>(); +// } -} +// } diff --git a/src/Functions/readWkt.cpp b/src/Functions/readWkt.cpp index be34b652929..0e595d3d5ae 100644 --- a/src/Functions/readWkt.cpp +++ b/src/Functions/readWkt.cpp @@ -76,7 +76,7 @@ public: }; -class FunctionReadWktRing : public FunctionReadWkt> +class FunctionReadWktRing : public FunctionReadWkt> { public: static inline const char * name = "readWktRing"; diff --git a/src/Functions/registerFunctionsGeo.cpp b/src/Functions/registerFunctionsGeo.cpp index 682093b0f2c..263f654203e 100644 --- a/src/Functions/registerFunctionsGeo.cpp +++ b/src/Functions/registerFunctionsGeo.cpp @@ -11,14 +11,14 @@ void registerFunctionGeoDistance(FunctionFactory & factory); void registerFunctionPointInEllipses(FunctionFactory & factory); void registerFunctionPointInPolygon(FunctionFactory & factory); void registerFunctionPolygonsIntersection(FunctionFactory & factory); -void registerFunctionPolygonsUnion(FunctionFactory & factory); +// void registerFunctionPolygonsUnion(FunctionFactory & factory); void registerFunctionPolygonArea(FunctionFactory & factory); -void registerFunctionPolygonConvexHull(FunctionFactory & factory); -void registerFunctionPolygonsSymDifference(FunctionFactory & factory); -void registerFunctionPolygonsEquals(FunctionFactory & factory); -void registerFunctionPolygonsDistance(FunctionFactory & factory); -void registerFunctionPolygonsWithin(FunctionFactory & factory); -void registerFunctionPolygonPerimeter(FunctionFactory & factory); +// void registerFunctionPolygonConvexHull(FunctionFactory & factory); +// void registerFunctionPolygonsSymDifference(FunctionFactory & factory); +// void registerFunctionPolygonsEquals(FunctionFactory & factory); +// void registerFunctionPolygonsDistance(FunctionFactory & factory); +// void registerFunctionPolygonsWithin(FunctionFactory & factory); +// void registerFunctionPolygonPerimeter(FunctionFactory & factory); void registerFunctionGeohashEncode(FunctionFactory & factory); void registerFunctionGeohashDecode(FunctionFactory & factory); void registerFunctionGeohashesInBox(FunctionFactory & factory); @@ -50,14 +50,14 @@ void registerFunctionsGeo(FunctionFactory & factory) registerFunctionPointInEllipses(factory); registerFunctionPointInPolygon(factory); registerFunctionPolygonsIntersection(factory); - registerFunctionPolygonsUnion(factory); + // registerFunctionPolygonsUnion(factory); registerFunctionPolygonArea(factory); - registerFunctionPolygonConvexHull(factory); - registerFunctionPolygonsSymDifference(factory); - registerFunctionPolygonsEquals(factory); - registerFunctionPolygonsDistance(factory); - registerFunctionPolygonsWithin(factory); - registerFunctionPolygonPerimeter(factory); + // registerFunctionPolygonConvexHull(factory); + // registerFunctionPolygonsSymDifference(factory); + // registerFunctionPolygonsEquals(factory); + // registerFunctionPolygonsDistance(factory); + // registerFunctionPolygonsWithin(factory); + // registerFunctionPolygonPerimeter(factory); registerFunctionGeohashEncode(factory); registerFunctionGeohashDecode(factory); registerFunctionGeohashesInBox(factory); diff --git a/src/Functions/svg.cpp b/src/Functions/svg.cpp index 42b3b916ca2..b8619a7803c 100644 --- a/src/Functions/svg.cpp +++ b/src/Functions/svg.cpp @@ -64,25 +64,32 @@ public: ColumnPtr executeImpl(const ColumnsWithTypeAndName & arguments, const DataTypePtr & /*result_type*/, size_t input_rows_count) const override { - auto parser = getConverterBasedOnType(arguments[0]); - auto figures = parseFigure(parser); auto res_column = ColumnString::create(); - bool has_style = arguments.size() > 1; - ColumnPtr style; - if (has_style) - style = arguments[1].column; - - for (size_t i = 0; i < input_rows_count; i++) + callOnGeometryDataType(arguments[0].type, [&] (const auto & type) { - std::stringstream str; // STYLE_CHECK_ALLOW_STD_STRING_STREAM - boost::geometry::correct(figures[i]); - str << boost::geometry::svg(figures[i], has_style ? style->getDataAt(i).toString() : ""); - std::string serialized = str.str(); - res_column->insertData(serialized.c_str(), serialized.size()); - } + using TypeParser = std::decay_t; + // using Parser = TypeParser::Type; + TypeParser parser(arguments[0].column->convertToFullColumnIfConst()); + auto figures = parser.parse(); + bool has_style = arguments.size() > 1; + ColumnPtr style; + if (has_style) + style = arguments[1].column; + + for (size_t i = 0; i < input_rows_count; i++) + { + std::stringstream str; // STYLE_CHECK_ALLOW_STD_STRING_STREAM + boost::geometry::correct(figures[i]); + str << boost::geometry::svg(figures[i], has_style ? style->getDataAt(i).toString() : ""); + std::string serialized = str.str(); + res_column->insertData(serialized.c_str(), serialized.size()); + } + } + ); + return res_column; } diff --git a/src/Functions/wkt.cpp b/src/Functions/wkt.cpp index 17789b0450d..049eca425c2 100644 --- a/src/Functions/wkt.cpp +++ b/src/Functions/wkt.cpp @@ -38,19 +38,24 @@ public: ColumnPtr executeImpl(const ColumnsWithTypeAndName & arguments, const DataTypePtr & /*result_type*/, size_t input_rows_count) const override { - auto parser = getConverterBasedOnType(arguments[0]); - auto figures = parseFigure(parser); - auto res_column = ColumnString::create(); - for (size_t i = 0; i < input_rows_count; i++) + callOnGeometryDataType(arguments[0].type, [&] (const auto & type) { - std::stringstream str; // STYLE_CHECK_ALLOW_STD_STRING_STREAM - str << boost::geometry::wkt(figures[i]); - std::string serialized = str.str(); - res_column->insertData(serialized.c_str(), serialized.size()); - } + using TypeParser = std::decay_t; + TypeParser parser(arguments[0].column->convertToFullColumnIfConst()); + auto figures = parser.parse(); + for (size_t i = 0; i < input_rows_count; i++) + { + std::stringstream str; // STYLE_CHECK_ALLOW_STD_STRING_STREAM + str << boost::geometry::wkt(figures[i]); + std::string serialized = str.str(); + res_column->insertData(serialized.c_str(), serialized.size()); + } + } + ); + return res_column; } From f6712479cd7400ab297967a41a4e38471f7fceaa Mon Sep 17 00:00:00 2001 From: Nikita Mikhailov Date: Sat, 20 Feb 2021 01:48:59 +0300 Subject: [PATCH 296/544] use parser type --- src/Functions/geometryConverters.h | 19 +++++++++++++------ src/Functions/polygonArea.cpp | 4 ++-- src/Functions/polygonsIntersection.cpp | 7 +++++-- src/Functions/svg.cpp | 4 ++-- src/Functions/wkt.cpp | 3 ++- 5 files changed, 24 insertions(+), 13 deletions(-) diff --git a/src/Functions/geometryConverters.h b/src/Functions/geometryConverters.h index 6f56a53dd6f..6c39546bca5 100644 --- a/src/Functions/geometryConverters.h +++ b/src/Functions/geometryConverters.h @@ -324,15 +324,22 @@ private: ColumnUInt64::MutablePtr offsets; }; + +template +struct ParserType +{ + using Type = PType; +}; + template static void callOnGeometryDataType(DataTypePtr type, F && f) { if (DataTypeCustomRingSerialization::nestedDataType()->equals(*type)) - return f(RingFromColumnParser()); + return f(ParserType>()); if (DataTypeCustomPolygonSerialization::nestedDataType()->equals(*type)) - return f(PolygonFromColumnParser()); + return f(ParserType>()); if (DataTypeCustomMultiPolygonSerialization::nestedDataType()->equals(*type)) - return f(MultiPolygonFromColumnParser()); + return f(ParserType>()); throw Exception(fmt::format("Unknown geometry type {}", type->getName()), ErrorCodes::BAD_ARGUMENTS); } @@ -342,13 +349,13 @@ static void callOnTwoGeometryDataTypes(DataTypePtr left_type, DataTypePtr right_ { return callOnGeometryDataType(left_type, [&](const auto & left_types) { - using LeftParser = std::decay_t; + using LeftParserType = std::decay_t; return callOnGeometryDataType(right_type, [&](const auto & right_types) { - using RightParser = std::decay_t; + using RightParserType = std::decay_t; - return func(LeftParser(), RightParser()); + return func(LeftParserType(), RightParserType()); }); }); } diff --git a/src/Functions/polygonArea.cpp b/src/Functions/polygonArea.cpp index 44fd4a89139..b21b4092419 100644 --- a/src/Functions/polygonArea.cpp +++ b/src/Functions/polygonArea.cpp @@ -60,8 +60,8 @@ public: callOnGeometryDataType(arguments[0].type, [&] (const auto & type) { using TypeParser = std::decay_t; - // using Parser = TypeParser::Type; - TypeParser parser(arguments[0].column->convertToFullColumnIfConst()); + using Parser = typename TypeParser::Type; + Parser parser(arguments[0].column->convertToFullColumnIfConst()); auto figures = parser.parse(); auto & res_data = res_column->getData(); diff --git a/src/Functions/polygonsIntersection.cpp b/src/Functions/polygonsIntersection.cpp index 2e23770e881..4d6569f37a2 100644 --- a/src/Functions/polygonsIntersection.cpp +++ b/src/Functions/polygonsIntersection.cpp @@ -60,8 +60,11 @@ public: MultiPolygonSerializer serializer; callOnTwoGeometryDataTypes(arguments[0].type, arguments[1].type, [&](const auto & left_type, const auto & right_type) { - using LeftParser = std::decay_t; - using RightParser = std::decay_t; + using LeftParserType = std::decay_t; + using RightParserType = std::decay_t; + + using LeftParser = typename LeftParserType::Type; + using RightParser = typename RightParserType::Type; auto first = LeftParser(arguments[0].column->convertToFullColumnIfConst()).parse(); auto second = RightParser(arguments[1].column->convertToFullColumnIfConst()).parse(); diff --git a/src/Functions/svg.cpp b/src/Functions/svg.cpp index b8619a7803c..79482666222 100644 --- a/src/Functions/svg.cpp +++ b/src/Functions/svg.cpp @@ -70,8 +70,8 @@ public: callOnGeometryDataType(arguments[0].type, [&] (const auto & type) { using TypeParser = std::decay_t; - // using Parser = TypeParser::Type; - TypeParser parser(arguments[0].column->convertToFullColumnIfConst()); + using Parser = typename TypeParser::Type; + Parser parser(arguments[0].column->convertToFullColumnIfConst()); auto figures = parser.parse(); bool has_style = arguments.size() > 1; diff --git a/src/Functions/wkt.cpp b/src/Functions/wkt.cpp index 049eca425c2..d088a3b5600 100644 --- a/src/Functions/wkt.cpp +++ b/src/Functions/wkt.cpp @@ -43,7 +43,8 @@ public: callOnGeometryDataType(arguments[0].type, [&] (const auto & type) { using TypeParser = std::decay_t; - TypeParser parser(arguments[0].column->convertToFullColumnIfConst()); + using Parser = typename TypeParser::Type; + Parser parser(arguments[0].column->convertToFullColumnIfConst()); auto figures = parser.parse(); for (size_t i = 0; i < input_rows_count; i++) From 681b0b63f1ee9d94cf3378e91be1b834deb162a6 Mon Sep 17 00:00:00 2001 From: Nikita Mikhailov Date: Sat, 20 Feb 2021 16:59:37 +0300 Subject: [PATCH 297/544] return back all functions --- src/Functions/describeGeometry.cpp | 131 -------------- src/Functions/geometryConverters.cpp | 32 ++-- src/Functions/geometryConverters.h | 91 +++++----- src/Functions/polygonArea.cpp | 10 +- src/Functions/polygonConvexHull.cpp | 151 ++++++++-------- src/Functions/polygonPerimeter.cpp | 145 ++++++++-------- src/Functions/polygonsDistance.cpp | 158 +++++++++-------- src/Functions/polygonsEquals.cpp | 155 +++++++++-------- src/Functions/polygonsIntersection.cpp | 15 +- src/Functions/polygonsSymDifference.cpp | 156 +++++++++-------- src/Functions/polygonsUnion.cpp | 162 +++++++++--------- src/Functions/polygonsWithin.cpp | 159 +++++++++-------- src/Functions/registerFunctionsGeo.cpp | 30 ++-- src/Functions/svg.cpp | 10 +- src/Functions/wkt.cpp | 10 +- .../0_stateless/01307_polygon_perimeter.sql | 2 +- 16 files changed, 669 insertions(+), 748 deletions(-) delete mode 100644 src/Functions/describeGeometry.cpp diff --git a/src/Functions/describeGeometry.cpp b/src/Functions/describeGeometry.cpp deleted file mode 100644 index 0f059d3c606..00000000000 --- a/src/Functions/describeGeometry.cpp +++ /dev/null @@ -1,131 +0,0 @@ -#include -#include -#include -#include -#include -#include - -#include -#include - -namespace DB -{ - -namespace ErrorCodes -{ - extern const int ILLEGAL_TYPE_OF_ARGUMENT; -} - -class FunctionDescribeGeometry : public IFunction -{ -public: - explicit FunctionDescribeGeometry() = default; - - size_t getNumberOfArguments() const override - { - return 1; - } - - static inline const char * name = "describeGeometry"; - - - String getName() const override - { - return name; - } - - DataTypePtr getReturnTypeImpl(const DataTypes & arguments) const override - { - if (checkAndGetDataType(arguments[0].get()) == nullptr) - { - throw Exception("First argument should be String", - ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT); - } - - return std::make_shared(); - } - - ColumnPtr executeImpl(const ColumnsWithTypeAndName & arguments, const DataTypePtr & /*result_type*/, size_t input_rows_count) const override - { - const auto * column_string = checkAndGetColumn(arguments[0].column.get()); - - CartesianPoint point; - Ring ring; - Polygon polygon; - MultiPolygon multipolygon; - - auto result = ColumnUInt8::create(); - auto & result_array = result->getData(); - - result_array.reserve(input_rows_count); - - for (size_t i = 0; i < input_rows_count; i++) - { - const auto & str = column_string->getDataAt(i).toString(); - - try - { - boost::geometry::read_wkt(str, point); - result_array.emplace_back(0); - continue; - } - catch (boost::geometry::read_wkt_exception &) - { - } - - try - { - boost::geometry::read_wkt(str, ring); - result_array.emplace_back(1); - continue; - } - catch (boost::geometry::read_wkt_exception &) - { - } - - - try - { - boost::geometry::read_wkt(str, polygon); - result_array.emplace_back(2); - continue; - } - catch (boost::geometry::read_wkt_exception &) - { - } - - - try - { - boost::geometry::read_wkt(str, multipolygon); - result_array.emplace_back(3); - continue; - } - catch (boost::geometry::read_wkt_exception &) - { - } - - throw Exception("Unknown geometry format", ErrorCodes::BAD_ARGUMENTS); - } - - return result; - } - - static FunctionPtr create(const Context &) - { - return std::make_shared(); - } - - bool useDefaultImplementationForConstants() const override - { - return true; - } -}; - - -void registerFunctionDescribeGeometry(FunctionFactory & factory) -{ - factory.registerFunction(); -} - -} diff --git a/src/Functions/geometryConverters.cpp b/src/Functions/geometryConverters.cpp index 8cd023f8c58..d2050af6679 100644 --- a/src/Functions/geometryConverters.cpp +++ b/src/Functions/geometryConverters.cpp @@ -13,7 +13,7 @@ namespace ErrorCodes } template -std::vector PointFromColumnParser::parseImpl(size_t shift, size_t count) const +std::vector PointFromColumnConverter::convertImpl(size_t shift, size_t count) const { const auto * tuple = typeid_cast(col.get()); const auto & tuple_columns = tuple->getColumns(); @@ -44,7 +44,7 @@ std::vector PointFromColumnParser::parseImpl(size_t shift, size_t } template -std::vector> RingFromColumnParser::parse() const +std::vector> RingFromColumnConverter::convert() const { const IColumn::Offsets & offsets = typeid_cast(*col).getOffsets(); size_t prev_offset = 0; @@ -52,7 +52,7 @@ std::vector> RingFromColumnParser::parse() const answer.reserve(offsets.size()); for (size_t offset : offsets) { - auto tmp = point_parser.parseImpl(prev_offset, offset - prev_offset); + auto tmp = point_converter.convertImpl(prev_offset, offset - prev_offset); answer.emplace_back(tmp.begin(), tmp.end()); prev_offset = offset; } @@ -60,11 +60,11 @@ std::vector> RingFromColumnParser::parse() const } template -std::vector> PolygonFromColumnParser::parse() const +std::vector> PolygonFromColumnConverter::convert() const { const IColumn::Offsets & offsets = typeid_cast(*col).getOffsets(); std::vector> answer(offsets.size()); - auto all_rings = ring_parser.parse(); + auto all_rings = ring_converter.convert(); auto prev_offset = 0; for (size_t iter = 0; iter < offsets.size(); ++iter) @@ -82,18 +82,18 @@ std::vector> PolygonFromColumnParser::parse() const template -std::vector> MultiPolygonFromColumnParser::parse() const +std::vector> MultiPolygonFromColumnConverter::convert() const { const IColumn::Offsets & offsets = typeid_cast(*col).getOffsets(); size_t prev_offset = 0; std::vector> answer(offsets.size()); - auto all_polygons = polygon_parser.parse(); + auto all_polygons = polygon_converter.convert(); for (size_t iter = 0; iter < offsets.size(); ++iter) { for (size_t polygon_iter = prev_offset; polygon_iter < offsets[iter]; ++polygon_iter) - answer[iter].emplace_back(std::move(all_polygons[polygon_iter])); + answer[iter].emplace_back(std::move(all_polygons[polygon_iter])); prev_offset = offsets[iter]; } @@ -101,14 +101,14 @@ std::vector> MultiPolygonFromColumnParser::parse() co } -template class PointFromColumnParser; -template class PointFromColumnParser; -template class RingFromColumnParser; -template class RingFromColumnParser; -template class PolygonFromColumnParser; -template class PolygonFromColumnParser; -template class MultiPolygonFromColumnParser; -template class MultiPolygonFromColumnParser; +template class PointFromColumnConverter; +template class PointFromColumnConverter; +template class RingFromColumnConverter; +template class RingFromColumnConverter; +template class PolygonFromColumnConverter; +template class PolygonFromColumnConverter; +template class MultiPolygonFromColumnConverter; +template class MultiPolygonFromColumnConverter; template typename Desired> void checkColumnTypeOrThrow(const ColumnWithTypeAndName & column) diff --git a/src/Functions/geometryConverters.h b/src/Functions/geometryConverters.h index 6c39546bca5..a245e4c3017 100644 --- a/src/Functions/geometryConverters.h +++ b/src/Functions/geometryConverters.h @@ -60,109 +60,106 @@ using GeographicGeometry = Geometry; template -class RingFromColumnParser; +class RingFromColumnConverter; template -class PolygonFromColumnParser; +class PolygonFromColumnConverter; template -class MultiPolygonFromColumnParser; +class MultiPolygonFromColumnConverter; /** * Class which takes some boost type and returns a pair of numbers. * They are (x,y) in case of cartesian coordinated and (lon,lat) in case of geographic. */ template -class PointFromColumnParser +class PointFromColumnConverter { public: - PointFromColumnParser() = default; - explicit PointFromColumnParser(ColumnPtr col_) : col(col_) + explicit PointFromColumnConverter(ColumnPtr col_) : col(col_) { } - std::vector parse() const; + std::vector convert() const; private: - std::vector parseImpl(size_t shift, size_t count) const; + std::vector convertImpl(size_t shift, size_t count) const; - friend class RingFromColumnParser; + friend class RingFromColumnConverter; ColumnPtr col{nullptr}; }; template -class RingFromColumnParser +class RingFromColumnConverter { public: - RingFromColumnParser() = default; - - explicit RingFromColumnParser(ColumnPtr col_) + explicit RingFromColumnConverter(ColumnPtr col_) : col(col_) - , point_parser(typeid_cast(*col_).getDataPtr()) + , point_converter(typeid_cast(*col_).getDataPtr()) { } - std::vector> parse() const; + std::vector> convert() const; private: - friend class PointFromColumnParser; + friend class PointFromColumnConverter; /// To prevent use-after-free and increase column lifetime. ColumnPtr col{nullptr}; - const PointFromColumnParser point_parser{}; + const PointFromColumnConverter point_converter{}; }; template -class PolygonFromColumnParser +class PolygonFromColumnConverter { public: - PolygonFromColumnParser() = default; + PolygonFromColumnConverter() = default; - explicit PolygonFromColumnParser(ColumnPtr col_) + explicit PolygonFromColumnConverter(ColumnPtr col_) : col(col_) - , ring_parser(typeid_cast(*col_).getDataPtr()) + , ring_converter(typeid_cast(*col_).getDataPtr()) { } - std::vector> parse() const; + std::vector> convert() const; private: - friend class MultiPolygonFromColumnParser; + friend class MultiPolygonFromColumnConverter; /// To prevent use-after-free and increase column lifetime. ColumnPtr col{nullptr}; - const RingFromColumnParser ring_parser{}; + const RingFromColumnConverter ring_converter{}; }; template -class MultiPolygonFromColumnParser +class MultiPolygonFromColumnConverter { public: - MultiPolygonFromColumnParser() = default; + MultiPolygonFromColumnConverter() = default; - explicit MultiPolygonFromColumnParser(ColumnPtr col_) + explicit MultiPolygonFromColumnConverter(ColumnPtr col_) : col(col_) - , polygon_parser(typeid_cast(*col_).getDataPtr()) + , polygon_converter(typeid_cast(*col_).getDataPtr()) {} - std::vector> parse() const; + std::vector> convert() const; private: /// To prevent use-after-free and increase column lifetime. ColumnPtr col{nullptr}; - const PolygonFromColumnParser polygon_parser{}; + const PolygonFromColumnConverter polygon_converter{}; }; -extern template class PointFromColumnParser; -extern template class PointFromColumnParser; -extern template class RingFromColumnParser; -extern template class RingFromColumnParser; -extern template class PolygonFromColumnParser; -extern template class PolygonFromColumnParser; -extern template class MultiPolygonFromColumnParser; -extern template class MultiPolygonFromColumnParser; +extern template class PointFromColumnConverter; +extern template class PointFromColumnConverter; +extern template class RingFromColumnConverter; +extern template class RingFromColumnConverter; +extern template class PolygonFromColumnConverter; +extern template class PolygonFromColumnConverter; +extern template class MultiPolygonFromColumnConverter; +extern template class MultiPolygonFromColumnConverter; /// To serialize Geographic or Cartesian point (a pair of numbers in both cases). @@ -207,6 +204,7 @@ private: ColumnFloat64::Container & second_container; }; +/// Serialize Point, Ring as Ring template class RingSerializer { @@ -238,6 +236,7 @@ private: ColumnUInt64::MutablePtr offsets; }; +/// Serialize Point, Ring, Polygon as Polygon template class PolygonSerializer { @@ -277,6 +276,7 @@ private: ColumnUInt64::MutablePtr offsets; }; +/// Serialize Point, Ring, Polygon, MultiPolygon as MultiPolygon template class MultiPolygonSerializer { @@ -326,7 +326,7 @@ private: template -struct ParserType +struct ConverterType { using Type = PType; }; @@ -334,12 +334,13 @@ struct ParserType template static void callOnGeometryDataType(DataTypePtr type, F && f) { + /// There is no Point type, because for most of geometry functions it is useless. if (DataTypeCustomRingSerialization::nestedDataType()->equals(*type)) - return f(ParserType>()); + return f(ConverterType>()); if (DataTypeCustomPolygonSerialization::nestedDataType()->equals(*type)) - return f(ParserType>()); + return f(ConverterType>()); if (DataTypeCustomMultiPolygonSerialization::nestedDataType()->equals(*type)) - return f(ParserType>()); + return f(ConverterType>()); throw Exception(fmt::format("Unknown geometry type {}", type->getName()), ErrorCodes::BAD_ARGUMENTS); } @@ -349,13 +350,13 @@ static void callOnTwoGeometryDataTypes(DataTypePtr left_type, DataTypePtr right_ { return callOnGeometryDataType(left_type, [&](const auto & left_types) { - using LeftParserType = std::decay_t; + using LeftConverterType = std::decay_t; return callOnGeometryDataType(right_type, [&](const auto & right_types) { - using RightParserType = std::decay_t; + using RightConverterType = std::decay_t; - return func(LeftParserType(), RightParserType()); + return func(LeftConverterType(), RightConverterType()); }); }); } diff --git a/src/Functions/polygonArea.cpp b/src/Functions/polygonArea.cpp index b21b4092419..1f2d47a3a5c 100644 --- a/src/Functions/polygonArea.cpp +++ b/src/Functions/polygonArea.cpp @@ -59,16 +59,16 @@ public: callOnGeometryDataType(arguments[0].type, [&] (const auto & type) { - using TypeParser = std::decay_t; - using Parser = typename TypeParser::Type; - Parser parser(arguments[0].column->convertToFullColumnIfConst()); - auto figures = parser.parse(); + using TypeConverter = std::decay_t; + using Converter = typename TypeConverter::Type; + Converter converter(arguments[0].column->convertToFullColumnIfConst()); + auto geometries = converter.convert(); auto & res_data = res_column->getData(); res_data.reserve(input_rows_count); for (size_t i = 0; i < input_rows_count; i++) - res_data.emplace_back(boost::geometry::area(figures[i])); + res_data.emplace_back(boost::geometry::area(geometries[i])); } ); diff --git a/src/Functions/polygonConvexHull.cpp b/src/Functions/polygonConvexHull.cpp index 6c9eb167fb1..422e46b3b15 100644 --- a/src/Functions/polygonConvexHull.cpp +++ b/src/Functions/polygonConvexHull.cpp @@ -1,89 +1,106 @@ -// #include -// #include +#include +#include -// #include -// #include -// #include +#include +#include +#include -// #include +#include -// #include -// #include -// #include -// #include -// #include -// #include +#include +#include +#include +#include +#include +#include -// #include -// #include +#include +#include -// namespace DB -// { +namespace DB +{ -// template -// class FunctionPolygonConvexHull : public IFunction -// { -// public: -// static const char * name; +namespace ErrorCodes +{ + extern const int BAD_ARGUMENTS; +} -// explicit FunctionPolygonConvexHull() = default; +template +class FunctionPolygonConvexHull : public IFunction +{ +public: + static const char * name; -// static FunctionPtr create(const Context &) -// { -// return std::make_shared(); -// } + explicit FunctionPolygonConvexHull() = default; -// String getName() const override -// { -// return name; -// } + static FunctionPtr create(const Context &) + { + return std::make_shared(); + } -// bool isVariadic() const override -// { -// return false; -// } + String getName() const override + { + return name; + } -// size_t getNumberOfArguments() const override -// { -// return 1; -// } + bool isVariadic() const override + { + return false; + } -// DataTypePtr getReturnTypeImpl(const DataTypes &) const override -// { -// return DataTypeCustomPolygonSerialization::nestedDataType(); -// } + size_t getNumberOfArguments() const override + { + return 1; + } -// ColumnPtr executeImpl(const ColumnsWithTypeAndName & arguments, const DataTypePtr & /*result_type*/, size_t input_rows_count) const override -// { -// auto parser = getConverterBasedOnType(arguments[0]); -// auto figures = parseFigure(parser); + DataTypePtr getReturnTypeImpl(const DataTypes &) const override + { + return DataTypeCustomPolygonSerialization::nestedDataType(); + } -// PolygonSerializer serializer; + ColumnPtr executeImpl(const ColumnsWithTypeAndName & arguments, const DataTypePtr & /*result_type*/, size_t input_rows_count) const override + { + PolygonSerializer serializer; -// for (size_t i = 0; i < input_rows_count; i++) -// { -// Polygon convex_hull{}; -// boost::geometry::convex_hull(figures[i], convex_hull); -// serializer.add(convex_hull); -// } + callOnGeometryDataType(arguments[0].type, [&] (const auto & type) + { + using TypeConverter = std::decay_t; + using Converter = typename TypeConverter::Type; -// return serializer.finalize(); -// } + if (std::is_same_v>) + throw Exception(fmt::format("The argument of function {} could not be a MultiPolygon", getName()), ErrorCodes::BAD_ARGUMENTS); + else + { + Converter converter(arguments[0].column->convertToFullColumnIfConst()); + auto geometries = converter.convert(); -// bool useDefaultImplementationForConstants() const override -// { -// return true; -// } -// }; + for (size_t i = 0; i < input_rows_count; i++) + { + Polygon convex_hull{}; + boost::geometry::convex_hull(geometries[i], convex_hull); + serializer.add(convex_hull); + } + } + } + ); + + return serializer.finalize(); + } + + bool useDefaultImplementationForConstants() const override + { + return true; + } +}; -// template <> -// const char * FunctionPolygonConvexHull::name = "polygonConvexHullCartesian"; +template <> +const char * FunctionPolygonConvexHull::name = "polygonConvexHullCartesian"; -// void registerFunctionPolygonConvexHull(FunctionFactory & factory) -// { -// factory.registerFunction>(); -// } +void registerFunctionPolygonConvexHull(FunctionFactory & factory) +{ + factory.registerFunction>(); +} -// } +} diff --git a/src/Functions/polygonPerimeter.cpp b/src/Functions/polygonPerimeter.cpp index f832d2965f9..1855ada51ed 100644 --- a/src/Functions/polygonPerimeter.cpp +++ b/src/Functions/polygonPerimeter.cpp @@ -1,94 +1,97 @@ -// #include -// #include +#include +#include -// #include -// #include -// #include +#include +#include +#include -// #include +#include -// #include -// #include -// #include -// #include -// #include -// #include +#include +#include +#include +#include +#include +#include -// #include -// #include +#include +#include -// namespace DB -// { +namespace DB +{ -// template -// class FunctionPolygonPerimeter : public IFunction -// { -// public: -// static const char * name; +template +class FunctionPolygonPerimeter : public IFunction +{ +public: + static const char * name; -// explicit FunctionPolygonPerimeter() = default; + explicit FunctionPolygonPerimeter() = default; -// static FunctionPtr create(const Context &) -// { -// return std::make_shared(); -// } + static FunctionPtr create(const Context &) + { + return std::make_shared(); + } -// String getName() const override -// { -// return name; -// } + String getName() const override + { + return name; + } -// bool isVariadic() const override -// { -// return false; -// } + bool isVariadic() const override + { + return false; + } -// size_t getNumberOfArguments() const override -// { -// return 1; -// } + size_t getNumberOfArguments() const override + { + return 1; + } -// DataTypePtr getReturnTypeImpl(const DataTypes &) const override -// { -// return std::make_shared(); -// } + DataTypePtr getReturnTypeImpl(const DataTypes &) const override + { + return std::make_shared(); + } -// ColumnPtr executeImpl(const ColumnsWithTypeAndName & arguments, const DataTypePtr & /*result_type*/, size_t input_rows_count) const override -// { -// auto parser = getConverterBasedOnType(arguments[0]); -// auto figures = parseFigure(parser); + ColumnPtr executeImpl(const ColumnsWithTypeAndName & arguments, const DataTypePtr & /*result_type*/, size_t input_rows_count) const override + { + auto res_column = ColumnFloat64::create(); + auto & res_data = res_column->getData(); + res_data.reserve(input_rows_count); -// auto res_column = ColumnFloat64::create(); -// auto & res_data = res_column->getData(); -// res_data.reserve(input_rows_count); + callOnGeometryDataType(arguments[0].type, [&] (const auto & type) + { + using TypeConverter = std::decay_t; + using Converter = typename TypeConverter::Type; + Converter converter(arguments[0].column->convertToFullColumnIfConst()); + auto geometries = converter.convert(); -// for (size_t i = 0; i < input_rows_count; i++) -// { -// boost::geometry::correct(figures[i]); -// res_data.emplace_back(boost::geometry::perimeter(figures[i])); -// } + for (size_t i = 0; i < input_rows_count; i++) + res_data.emplace_back(boost::geometry::perimeter(geometries[i])); + } + ); -// return res_column; -// } + return res_column; + } -// bool useDefaultImplementationForConstants() const override -// { -// return true; -// } -// }; + bool useDefaultImplementationForConstants() const override + { + return true; + } +}; -// template <> -// const char * FunctionPolygonPerimeter::name = "polygonPerimeterCartesian"; +template <> +const char * FunctionPolygonPerimeter::name = "polygonPerimeterCartesian"; -// template <> -// const char * FunctionPolygonPerimeter::name = "polygonPerimeterGeographic"; +template <> +const char * FunctionPolygonPerimeter::name = "polygonPerimeterGeographic"; -// void registerFunctionPolygonPerimeter(FunctionFactory & factory) -// { -// factory.registerFunction>(); -// factory.registerFunction>(); -// } +void registerFunctionPolygonPerimeter(FunctionFactory & factory) +{ + factory.registerFunction>(); + factory.registerFunction>(); +} -// } +} diff --git a/src/Functions/polygonsDistance.cpp b/src/Functions/polygonsDistance.cpp index 679345dc693..a5341482f2f 100644 --- a/src/Functions/polygonsDistance.cpp +++ b/src/Functions/polygonsDistance.cpp @@ -1,101 +1,107 @@ -// #include -// #include +#include +#include -// #include -// #include -// #include +#include +#include +#include -// #include +#include -// #include -// #include -// #include -// #include -// #include -// #include -// #include -// #include +#include +#include +#include +#include +#include +#include +#include +#include -// #include -// #include +#include +#include -// namespace DB -// { +namespace DB +{ -// template -// class FunctionPolygonsDistance : public IFunction -// { -// public: -// static inline const char * name; +template +class FunctionPolygonsDistance : public IFunction +{ +public: + static inline const char * name; -// explicit FunctionPolygonsDistance() = default; + explicit FunctionPolygonsDistance() = default; -// static FunctionPtr create(const Context &) -// { -// return std::make_shared(); -// } + static FunctionPtr create(const Context &) + { + return std::make_shared(); + } -// String getName() const override -// { -// return name; -// } + String getName() const override + { + return name; + } -// bool isVariadic() const override -// { -// return false; -// } + bool isVariadic() const override + { + return false; + } -// size_t getNumberOfArguments() const override -// { -// return 2; -// } + size_t getNumberOfArguments() const override + { + return 2; + } -// DataTypePtr getReturnTypeImpl(const DataTypes &) const override -// { -// return std::make_shared(); -// } + DataTypePtr getReturnTypeImpl(const DataTypes &) const override + { + return std::make_shared(); + } -// ColumnPtr executeImpl(const ColumnsWithTypeAndName & arguments, const DataTypePtr & /*result_type*/, size_t input_rows_count) const override -// { -// auto first_parser = getConverterBasedOnType(arguments[0]); -// auto second_parser = getConverterBasedOnType(arguments[1]); + ColumnPtr executeImpl(const ColumnsWithTypeAndName & arguments, const DataTypePtr & /*result_type*/, size_t input_rows_count) const override + { + auto res_column = ColumnFloat64::create(); + auto & res_data = res_column->getData(); + res_data.reserve(input_rows_count); -// auto first = parseFigure(first_parser); -// auto second = parseFigure(second_parser); + callOnTwoGeometryDataTypes(arguments[0].type, arguments[1].type, [&](const auto & left_type, const auto & right_type) + { + using LeftConverterType = std::decay_t; + using RightConverterType = std::decay_t; -// auto res_column = ColumnFloat64::create(); -// auto & res_data = res_column->getData(); -// res_data.reserve(input_rows_count); + using LeftConverter = typename LeftConverterType::Type; + using RightConverter = typename RightConverterType::Type; -// for (size_t i = 0; i < input_rows_count; i++) -// { -// boost::geometry::correct(first[i]); -// boost::geometry::correct(second[i]); + auto first = LeftConverter(arguments[0].column->convertToFullColumnIfConst()).convert(); + auto second = RightConverter(arguments[1].column->convertToFullColumnIfConst()).convert(); -// res_data.emplace_back(boost::geometry::distance(first[i], second[i])); -// } + for (size_t i = 0; i < input_rows_count; i++) + { + boost::geometry::correct(first[i]); + boost::geometry::correct(second[i]); -// return res_column; -// } + res_data.emplace_back(boost::geometry::distance(first[i], second[i])); + } + }); -// bool useDefaultImplementationForConstants() const override -// { -// return true; -// } -// }; + return res_column; + } -// template <> -// const char * FunctionPolygonsDistance::name = "polygonsDistanceCartesian"; + bool useDefaultImplementationForConstants() const override + { + return true; + } +}; -// template <> -// const char * FunctionPolygonsDistance::name = "polygonsDistanceGeographic"; +template <> +const char * FunctionPolygonsDistance::name = "polygonsDistanceCartesian"; + +template <> +const char * FunctionPolygonsDistance::name = "polygonsDistanceGeographic"; -// void registerFunctionPolygonsDistance(FunctionFactory & factory) -// { -// factory.registerFunction>(); -// factory.registerFunction>(); -// } +void registerFunctionPolygonsDistance(FunctionFactory & factory) +{ + factory.registerFunction>(); + factory.registerFunction>(); +} -// } +} diff --git a/src/Functions/polygonsEquals.cpp b/src/Functions/polygonsEquals.cpp index 8349e8837cd..d235e88b465 100644 --- a/src/Functions/polygonsEquals.cpp +++ b/src/Functions/polygonsEquals.cpp @@ -1,98 +1,105 @@ -// #include -// #include +#include +#include -// #include -// #include -// #include +#include +#include +#include -// #include +#include -// #include -// #include -// #include -// #include -// #include -// #include -// #include -// #include +#include +#include +#include +#include +#include +#include +#include +#include -// #include -// #include +#include +#include -// namespace DB -// { +namespace DB +{ -// template -// class FunctionPolygonsEquals : public IFunction -// { -// public: -// static const char * name; +template +class FunctionPolygonsEquals : public IFunction +{ +public: + static const char * name; -// explicit FunctionPolygonsEquals() = default; + explicit FunctionPolygonsEquals() = default; -// static FunctionPtr create(const Context &) -// { -// return std::make_shared(); -// } + static FunctionPtr create(const Context &) + { + return std::make_shared(); + } -// String getName() const override -// { -// return name; -// } + String getName() const override + { + return name; + } -// bool isVariadic() const override -// { -// return false; -// } + bool isVariadic() const override + { + return false; + } -// size_t getNumberOfArguments() const override -// { -// return 2; -// } + size_t getNumberOfArguments() const override + { + return 2; + } -// DataTypePtr getReturnTypeImpl(const DataTypes &) const override -// { -// return std::make_shared(); -// } + DataTypePtr getReturnTypeImpl(const DataTypes &) const override + { + return std::make_shared(); + } -// ColumnPtr executeImpl(const ColumnsWithTypeAndName & arguments, const DataTypePtr & /*result_type*/, size_t input_rows_count) const override -// { -// auto first_parser = getConverterBasedOnType(arguments[0]); -// auto second_parser = getConverterBasedOnType(arguments[1]); + ColumnPtr executeImpl(const ColumnsWithTypeAndName & arguments, const DataTypePtr & /*result_type*/, size_t input_rows_count) const override + { + auto res_column = ColumnUInt8::create(); + auto & res_data = res_column->getData(); + res_data.reserve(input_rows_count); -// auto first = parseFigure(first_parser); -// auto second = parseFigure(second_parser); + callOnTwoGeometryDataTypes(arguments[0].type, arguments[1].type, [&](const auto & left_type, const auto & right_type) + { + using LeftConverterType = std::decay_t; + using RightConverterType = std::decay_t; -// auto res_column = ColumnUInt8::create(); -// auto & res_data = res_column->getData(); -// res_data.reserve(input_rows_count); + using LeftConverter = typename LeftConverterType::Type; + using RightConverter = typename RightConverterType::Type; -// for (size_t i = 0; i < input_rows_count; i++) -// { -// boost::geometry::correct(first[i]); -// boost::geometry::correct(second[i]); + auto first = LeftConverter(arguments[0].column->convertToFullColumnIfConst()).convert(); + auto second = RightConverter(arguments[1].column->convertToFullColumnIfConst()).convert(); -// /// Main work here. -// res_data.emplace_back(boost::geometry::equals(first[i], second[i])); -// } + for (size_t i = 0; i < input_rows_count; i++) + { + boost::geometry::correct(first[i]); + boost::geometry::correct(second[i]); -// return res_column; -// } + /// Main work here. + res_data.emplace_back(boost::geometry::equals(first[i], second[i])); + } + } + ); -// bool useDefaultImplementationForConstants() const override -// { -// return true; -// } -// }; + return res_column; + } + + bool useDefaultImplementationForConstants() const override + { + return true; + } +}; -// template <> -// const char * FunctionPolygonsEquals::name = "polygonsEqualsCartesian"; +template <> +const char * FunctionPolygonsEquals::name = "polygonsEqualsCartesian"; -// void registerFunctionPolygonsEquals(FunctionFactory & factory) -// { -// factory.registerFunction>(); -// } +void registerFunctionPolygonsEquals(FunctionFactory & factory) +{ + factory.registerFunction>(); +} -// } +} diff --git a/src/Functions/polygonsIntersection.cpp b/src/Functions/polygonsIntersection.cpp index 4d6569f37a2..285255df031 100644 --- a/src/Functions/polygonsIntersection.cpp +++ b/src/Functions/polygonsIntersection.cpp @@ -59,15 +59,16 @@ public: { MultiPolygonSerializer serializer; - callOnTwoGeometryDataTypes(arguments[0].type, arguments[1].type, [&](const auto & left_type, const auto & right_type) { - using LeftParserType = std::decay_t; - using RightParserType = std::decay_t; + callOnTwoGeometryDataTypes(arguments[0].type, arguments[1].type, [&](const auto & left_type, const auto & right_type) + { + using LeftConverterType = std::decay_t; + using RightConverterType = std::decay_t; - using LeftParser = typename LeftParserType::Type; - using RightParser = typename RightParserType::Type; + using LeftConverter = typename LeftConverterType::Type; + using RightConverter = typename RightConverterType::Type; - auto first = LeftParser(arguments[0].column->convertToFullColumnIfConst()).parse(); - auto second = RightParser(arguments[1].column->convertToFullColumnIfConst()).parse(); + auto first = LeftConverter(arguments[0].column->convertToFullColumnIfConst()).convert(); + auto second = RightConverter(arguments[1].column->convertToFullColumnIfConst()).convert(); /// We are not interested in some pitfalls in third-party libraries /// NOLINTNEXTLINE(clang-analyzer-core.uninitialized.Assign) diff --git a/src/Functions/polygonsSymDifference.cpp b/src/Functions/polygonsSymDifference.cpp index bec6e10ba9b..20b521f7d6b 100644 --- a/src/Functions/polygonsSymDifference.cpp +++ b/src/Functions/polygonsSymDifference.cpp @@ -1,99 +1,105 @@ -// #include -// #include +#include +#include -// #include -// #include -// #include +#include +#include +#include -// #include +#include -// #include -// #include -// #include -// #include -// #include -// #include +#include +#include +#include +#include +#include +#include -// #include -// #include +#include +#include -// namespace DB -// { +namespace DB +{ -// template -// class FunctionPolygonsSymDifference : public IFunction -// { -// public: -// static const char * name; +template +class FunctionPolygonsSymDifference : public IFunction +{ +public: + static const char * name; -// explicit FunctionPolygonsSymDifference() = default; + explicit FunctionPolygonsSymDifference() = default; -// static FunctionPtr create(const Context &) -// { -// return std::make_shared(); -// } + static FunctionPtr create(const Context &) + { + return std::make_shared(); + } -// String getName() const override -// { -// return name; -// } + String getName() const override + { + return name; + } -// bool isVariadic() const override -// { -// return false; -// } + bool isVariadic() const override + { + return false; + } -// size_t getNumberOfArguments() const override -// { -// return 2; -// } + size_t getNumberOfArguments() const override + { + return 2; + } -// DataTypePtr getReturnTypeImpl(const DataTypes &) const override -// { -// return DataTypeCustomMultiPolygonSerialization::nestedDataType(); -// } + DataTypePtr getReturnTypeImpl(const DataTypes &) const override + { + return DataTypeCustomMultiPolygonSerialization::nestedDataType(); + } -// ColumnPtr executeImpl(const ColumnsWithTypeAndName & arguments, const DataTypePtr & /*result_type*/, size_t input_rows_count) const override -// { -// auto first_parser = getConverterBasedOnType(arguments[0]); -// auto second_parser = getConverterBasedOnType(arguments[1]); + ColumnPtr executeImpl(const ColumnsWithTypeAndName & arguments, const DataTypePtr & /*result_type*/, size_t input_rows_count) const override + { + MultiPolygonSerializer serializer; -// auto first = parseFigure(first_parser); -// auto second = parseFigure(second_parser); + callOnTwoGeometryDataTypes(arguments[0].type, arguments[1].type, [&](const auto & left_type, const auto & right_type) + { + using LeftConverterType = std::decay_t; + using RightConverterType = std::decay_t; -// MultiPolygonSerializer serializer; + using LeftConverter = typename LeftConverterType::Type; + using RightConverter = typename RightConverterType::Type; -// /// NOLINTNEXTLINE(clang-analyzer-core.uninitialized.Assign) -// for (size_t i = 0; i < input_rows_count; i++) -// { -// boost::geometry::correct(first[i]); -// boost::geometry::correct(second[i]); + auto first = LeftConverter(arguments[0].column->convertToFullColumnIfConst()).convert(); + auto second = RightConverter(arguments[1].column->convertToFullColumnIfConst()).convert(); -// MultiPolygon sym_difference{}; -// boost::geometry::sym_difference(first[i], second[i], sym_difference); + /// NOLINTNEXTLINE(clang-analyzer-core.uninitialized.Assign) + for (size_t i = 0; i < input_rows_count; i++) + { + boost::geometry::correct(first[i]); + boost::geometry::correct(second[i]); -// serializer.add(sym_difference); -// } + MultiPolygon sym_difference{}; + boost::geometry::sym_difference(first[i], second[i], sym_difference); -// return serializer.finalize(); -// } + serializer.add(sym_difference); + } + }); -// bool useDefaultImplementationForConstants() const override -// { -// return true; -// } -// }; + return serializer.finalize(); + } -// template <> -// const char * FunctionPolygonsSymDifference::name = "polygonsSymDifferenceCartesian"; + bool useDefaultImplementationForConstants() const override + { + return true; + } +}; -// template <> -// const char * FunctionPolygonsSymDifference::name = "polygonsSymDifferenceGeographic"; +template <> +const char * FunctionPolygonsSymDifference::name = "polygonsSymDifferenceCartesian"; -// void registerFunctionPolygonsSymDifference(FunctionFactory & factory) -// { -// factory.registerFunction>(); -// factory.registerFunction>(); -// } +template <> +const char * FunctionPolygonsSymDifference::name = "polygonsSymDifferenceGeographic"; -// } +void registerFunctionPolygonsSymDifference(FunctionFactory & factory) +{ + factory.registerFunction>(); + factory.registerFunction>(); +} + +} diff --git a/src/Functions/polygonsUnion.cpp b/src/Functions/polygonsUnion.cpp index 3922c4887ca..f236bf13bc3 100644 --- a/src/Functions/polygonsUnion.cpp +++ b/src/Functions/polygonsUnion.cpp @@ -1,103 +1,109 @@ -// #include -// #include +#include +#include -// #include -// #include -// #include +#include +#include +#include -// #include +#include -// #include -// #include -// #include -// #include -// #include -// #include +#include +#include +#include +#include +#include +#include -// #include -// #include +#include +#include -// namespace DB -// { +namespace DB +{ -// template -// class FunctionPolygonsUnion : public IFunction -// { -// public: -// static inline const char * name; +template +class FunctionPolygonsUnion : public IFunction +{ +public: + static inline const char * name; -// explicit FunctionPolygonsUnion() = default; + explicit FunctionPolygonsUnion() = default; -// static FunctionPtr create(const Context &) -// { -// return std::make_shared(); -// } + static FunctionPtr create(const Context &) + { + return std::make_shared(); + } -// String getName() const override -// { -// return name; -// } + String getName() const override + { + return name; + } -// bool isVariadic() const override -// { -// return false; -// } + bool isVariadic() const override + { + return false; + } -// size_t getNumberOfArguments() const override -// { -// return 2; -// } + size_t getNumberOfArguments() const override + { + return 2; + } -// DataTypePtr getReturnTypeImpl(const DataTypes &) const override -// { -// return DataTypeCustomMultiPolygonSerialization::nestedDataType(); -// } + DataTypePtr getReturnTypeImpl(const DataTypes &) const override + { + return DataTypeCustomMultiPolygonSerialization::nestedDataType(); + } -// ColumnPtr executeImpl(const ColumnsWithTypeAndName & arguments, const DataTypePtr & /*result_type*/, size_t input_rows_count) const override -// { -// auto first_parser = getConverterBasedOnType(arguments[0]); -// auto second_parser = getConverterBasedOnType(arguments[1]); + ColumnPtr executeImpl(const ColumnsWithTypeAndName & arguments, const DataTypePtr & /*result_type*/, size_t input_rows_count) const override + { + MultiPolygonSerializer serializer; -// auto first = parseFigure(first_parser); -// auto second = parseFigure(second_parser); + callOnTwoGeometryDataTypes(arguments[0].type, arguments[1].type, [&](const auto & left_type, const auto & right_type) + { + using LeftConverterType = std::decay_t; + using RightConverterType = std::decay_t; -// MultiPolygonSerializer serializer; + using LeftConverter = typename LeftConverterType::Type; + using RightConverter = typename RightConverterType::Type; -// /// We are not interested in some pitfalls in third-party libraries -// /// NOLINTNEXTLINE(clang-analyzer-core.uninitialized.Assign) -// for (size_t i = 0; i < input_rows_count; i++) -// { -// /// Orient the polygons correctly. -// boost::geometry::correct(first[i]); -// boost::geometry::correct(second[i]); + auto first = LeftConverter(arguments[0].column->convertToFullColumnIfConst()).convert(); + auto second = RightConverter(arguments[1].column->convertToFullColumnIfConst()).convert(); -// MultiPolygon polygons_union{}; -// /// Main work here. -// boost::geometry::union_(first[i], second[i], polygons_union); + /// We are not interested in some pitfalls in third-party libraries + /// NOLINTNEXTLINE(clang-analyzer-core.uninitialized.Assign) + for (size_t i = 0; i < input_rows_count; i++) + { + /// Orient the polygons correctly. + boost::geometry::correct(first[i]); + boost::geometry::correct(second[i]); -// serializer.add(polygons_union); -// } + MultiPolygon polygons_union{}; + /// Main work here. + boost::geometry::union_(first[i], second[i], polygons_union); -// return serializer.finalize(); -// } + serializer.add(polygons_union); + } + }); -// bool useDefaultImplementationForConstants() const override -// { -// return true; -// } -// }; + return serializer.finalize(); + } -// template <> -// const char * FunctionPolygonsUnion::name = "polygonsUnionCartesian"; + bool useDefaultImplementationForConstants() const override + { + return true; + } +}; -// template <> -// const char * FunctionPolygonsUnion::name = "polygonsUnionGeographic"; +template <> +const char * FunctionPolygonsUnion::name = "polygonsUnionCartesian"; + +template <> +const char * FunctionPolygonsUnion::name = "polygonsUnionGeographic"; -// void registerFunctionPolygonsUnion(FunctionFactory & factory) -// { -// factory.registerFunction>(); -// factory.registerFunction>(); -// } +void registerFunctionPolygonsUnion(FunctionFactory & factory) +{ + factory.registerFunction>(); + factory.registerFunction>(); +} -// } +} diff --git a/src/Functions/polygonsWithin.cpp b/src/Functions/polygonsWithin.cpp index 0b77ba45582..1e591cf0de2 100644 --- a/src/Functions/polygonsWithin.cpp +++ b/src/Functions/polygonsWithin.cpp @@ -1,101 +1,108 @@ -// #include -// #include +#include +#include -// #include -// #include -// #include +#include +#include +#include -// #include +#include -// #include -// #include -// #include -// #include -// #include -// #include -// #include -// #include +#include +#include +#include +#include +#include +#include +#include +#include -// #include -// #include +#include +#include -// namespace DB -// { +namespace DB +{ -// template -// class FunctionPolygonsWithin : public IFunction -// { -// public: -// static inline const char * name; +template +class FunctionPolygonsWithin : public IFunction +{ +public: + static inline const char * name; -// explicit FunctionPolygonsWithin() = default; + explicit FunctionPolygonsWithin() = default; -// static FunctionPtr create(const Context &) -// { -// return std::make_shared(); -// } + static FunctionPtr create(const Context &) + { + return std::make_shared(); + } -// String getName() const override -// { -// return name; -// } + String getName() const override + { + return name; + } -// bool isVariadic() const override -// { -// return false; -// } + bool isVariadic() const override + { + return false; + } -// size_t getNumberOfArguments() const override -// { -// return 2; -// } + size_t getNumberOfArguments() const override + { + return 2; + } -// DataTypePtr getReturnTypeImpl(const DataTypes &) const override -// { -// return std::make_shared(); -// } + DataTypePtr getReturnTypeImpl(const DataTypes &) const override + { + return std::make_shared(); + } -// ColumnPtr executeImpl(const ColumnsWithTypeAndName & arguments, const DataTypePtr & /*result_type*/, size_t input_rows_count) const override -// { -// auto first_parser = getConverterBasedOnType(arguments[0]); -// auto second_parser = getConverterBasedOnType(arguments[1]); + ColumnPtr executeImpl(const ColumnsWithTypeAndName & arguments, const DataTypePtr & /*result_type*/, size_t input_rows_count) const override + { + auto res_column = ColumnUInt8::create(); + auto & res_data = res_column->getData(); + res_data.reserve(input_rows_count); -// auto first = parseFigure(first_parser); -// auto second = parseFigure(second_parser); + callOnTwoGeometryDataTypes(arguments[0].type, arguments[1].type, [&](const auto & left_type, const auto & right_type) + { + using LeftConverterType = std::decay_t; + using RightConverterType = std::decay_t; -// auto res_column = ColumnUInt8::create(); -// auto & res_data = res_column->getData(); + using LeftConverter = typename LeftConverterType::Type; + using RightConverter = typename RightConverterType::Type; -// /// NOLINTNEXTLINE(clang-analyzer-core.uninitialized.Assign) -// for (size_t i = 0; i < input_rows_count; i++) -// { -// boost::geometry::correct(first[i]); -// boost::geometry::correct(second[i]); + auto first = LeftConverter(arguments[0].column->convertToFullColumnIfConst()).convert(); + auto second = RightConverter(arguments[1].column->convertToFullColumnIfConst()).convert(); -// res_data.emplace_back(boost::geometry::within(first[i], second[i])); -// } + /// NOLINTNEXTLINE(clang-analyzer-core.uninitialized.Assign) + for (size_t i = 0; i < input_rows_count; i++) + { + boost::geometry::correct(first[i]); + boost::geometry::correct(second[i]); -// return res_column; -// } + res_data.emplace_back(boost::geometry::within(first[i], second[i])); + } + }); -// bool useDefaultImplementationForConstants() const override -// { -// return true; -// } -// }; + return res_column; + } + + bool useDefaultImplementationForConstants() const override + { + return true; + } +}; -// template <> -// const char * FunctionPolygonsWithin::name = "polygonsWithinCartesian"; +template <> +const char * FunctionPolygonsWithin::name = "polygonsWithinCartesian"; -// template <> -// const char * FunctionPolygonsWithin::name = "polygonsWithinGeographic"; +template <> +const char * FunctionPolygonsWithin::name = "polygonsWithinGeographic"; -// void registerFunctionPolygonsWithin(FunctionFactory & factory) -// { -// factory.registerFunction>(); -// factory.registerFunction>(); -// } +void registerFunctionPolygonsWithin(FunctionFactory & factory) +{ + factory.registerFunction>(); + factory.registerFunction>(); +} -// } +} diff --git a/src/Functions/registerFunctionsGeo.cpp b/src/Functions/registerFunctionsGeo.cpp index 263f654203e..605dd4dcba0 100644 --- a/src/Functions/registerFunctionsGeo.cpp +++ b/src/Functions/registerFunctionsGeo.cpp @@ -11,20 +11,19 @@ void registerFunctionGeoDistance(FunctionFactory & factory); void registerFunctionPointInEllipses(FunctionFactory & factory); void registerFunctionPointInPolygon(FunctionFactory & factory); void registerFunctionPolygonsIntersection(FunctionFactory & factory); -// void registerFunctionPolygonsUnion(FunctionFactory & factory); +void registerFunctionPolygonsUnion(FunctionFactory & factory); void registerFunctionPolygonArea(FunctionFactory & factory); -// void registerFunctionPolygonConvexHull(FunctionFactory & factory); -// void registerFunctionPolygonsSymDifference(FunctionFactory & factory); -// void registerFunctionPolygonsEquals(FunctionFactory & factory); -// void registerFunctionPolygonsDistance(FunctionFactory & factory); -// void registerFunctionPolygonsWithin(FunctionFactory & factory); -// void registerFunctionPolygonPerimeter(FunctionFactory & factory); +void registerFunctionPolygonConvexHull(FunctionFactory & factory); +void registerFunctionPolygonsSymDifference(FunctionFactory & factory); +void registerFunctionPolygonsEquals(FunctionFactory & factory); +void registerFunctionPolygonsDistance(FunctionFactory & factory); +void registerFunctionPolygonsWithin(FunctionFactory & factory); +void registerFunctionPolygonPerimeter(FunctionFactory & factory); void registerFunctionGeohashEncode(FunctionFactory & factory); void registerFunctionGeohashDecode(FunctionFactory & factory); void registerFunctionGeohashesInBox(FunctionFactory & factory); void registerFunctionWkt(FunctionFactory & factory); void registerFunctionReadWkt(FunctionFactory & factory); -void registerFunctionDescribeGeometry(FunctionFactory & factory); void registerFunctionSvg(FunctionFactory & factory); #if USE_H3 @@ -50,20 +49,19 @@ void registerFunctionsGeo(FunctionFactory & factory) registerFunctionPointInEllipses(factory); registerFunctionPointInPolygon(factory); registerFunctionPolygonsIntersection(factory); - // registerFunctionPolygonsUnion(factory); + registerFunctionPolygonsUnion(factory); registerFunctionPolygonArea(factory); - // registerFunctionPolygonConvexHull(factory); - // registerFunctionPolygonsSymDifference(factory); - // registerFunctionPolygonsEquals(factory); - // registerFunctionPolygonsDistance(factory); - // registerFunctionPolygonsWithin(factory); - // registerFunctionPolygonPerimeter(factory); + registerFunctionPolygonConvexHull(factory); + registerFunctionPolygonsSymDifference(factory); + registerFunctionPolygonsEquals(factory); + registerFunctionPolygonsDistance(factory); + registerFunctionPolygonsWithin(factory); + registerFunctionPolygonPerimeter(factory); registerFunctionGeohashEncode(factory); registerFunctionGeohashDecode(factory); registerFunctionGeohashesInBox(factory); registerFunctionWkt(factory); registerFunctionReadWkt(factory); - registerFunctionDescribeGeometry(factory); registerFunctionSvg(factory); #if USE_H3 diff --git a/src/Functions/svg.cpp b/src/Functions/svg.cpp index 79482666222..2ad9f96ca15 100644 --- a/src/Functions/svg.cpp +++ b/src/Functions/svg.cpp @@ -69,10 +69,10 @@ public: callOnGeometryDataType(arguments[0].type, [&] (const auto & type) { - using TypeParser = std::decay_t; - using Parser = typename TypeParser::Type; - Parser parser(arguments[0].column->convertToFullColumnIfConst()); - auto figures = parser.parse(); + using TypeConverter = std::decay_t; + using Converter = typename TypeConverter::Type; + Converter converter(arguments[0].column->convertToFullColumnIfConst()); + auto figures = converter.convert(); bool has_style = arguments.size() > 1; ColumnPtr style; @@ -89,7 +89,7 @@ public: } } ); - + return res_column; } diff --git a/src/Functions/wkt.cpp b/src/Functions/wkt.cpp index d088a3b5600..08aeb76dcdd 100644 --- a/src/Functions/wkt.cpp +++ b/src/Functions/wkt.cpp @@ -42,10 +42,10 @@ public: callOnGeometryDataType(arguments[0].type, [&] (const auto & type) { - using TypeParser = std::decay_t; - using Parser = typename TypeParser::Type; - Parser parser(arguments[0].column->convertToFullColumnIfConst()); - auto figures = parser.parse(); + using TypeConverter = std::decay_t; + using Converter = typename TypeConverter::Type; + Converter converter(arguments[0].column->convertToFullColumnIfConst()); + auto figures = converter.convert(); for (size_t i = 0; i < input_rows_count; i++) { @@ -56,7 +56,7 @@ public: } } ); - + return res_column; } diff --git a/tests/queries/0_stateless/01307_polygon_perimeter.sql b/tests/queries/0_stateless/01307_polygon_perimeter.sql index 1cbbf41a201..2fd0c369379 100644 --- a/tests/queries/0_stateless/01307_polygon_perimeter.sql +++ b/tests/queries/0_stateless/01307_polygon_perimeter.sql @@ -1 +1 @@ -select polygonPerimeterCartesian([[[(0., 0.), (0., 5.), (5., 5.), (5., 0.)]]]); \ No newline at end of file +select polygonPerimeterCartesian([[[(0., 0.), (0., 5.), (5., 5.), (5., 0.), (0., 0.)]]]); \ No newline at end of file From 56700ed3e9257036d635f0fbc39317c16ce5c8e3 Mon Sep 17 00:00:00 2001 From: Nikita Mikhailov Date: Sat, 20 Feb 2021 17:39:45 +0300 Subject: [PATCH 298/544] style --- src/Functions/geometryConverters.cpp | 2 +- src/Functions/geometryConverters.h | 1 - src/Functions/pointInPolygon.cpp | 1 - 3 files changed, 1 insertion(+), 3 deletions(-) diff --git a/src/Functions/geometryConverters.cpp b/src/Functions/geometryConverters.cpp index d2050af6679..b0b437d5332 100644 --- a/src/Functions/geometryConverters.cpp +++ b/src/Functions/geometryConverters.cpp @@ -7,9 +7,9 @@ namespace DB namespace ErrorCodes { - extern const int ILLEGAL_COLUMN; extern const int BAD_ARGUMENTS; extern const int LOGICAL_ERROR; + extern const int ILLEGAL_TYPE_OF_ARGUMENT; } template diff --git a/src/Functions/geometryConverters.h b/src/Functions/geometryConverters.h index a245e4c3017..c6549cba40f 100644 --- a/src/Functions/geometryConverters.h +++ b/src/Functions/geometryConverters.h @@ -27,7 +27,6 @@ namespace DB namespace ErrorCodes { extern const int BAD_ARGUMENTS; - extern const int ILLEGAL_TYPE_OF_ARGUMENT; } template diff --git a/src/Functions/pointInPolygon.cpp b/src/Functions/pointInPolygon.cpp index 91c5b3a8365..ac5a41f6c81 100644 --- a/src/Functions/pointInPolygon.cpp +++ b/src/Functions/pointInPolygon.cpp @@ -434,7 +434,6 @@ private: out_container.emplace_back(x_data[i], y_data[i]); } - } void parseConstPolygonWithoutHolesFromSingleColumn(const IColumn & column, size_t i, Polygon & out_polygon) const From 4430322edc896d76372948a13336c90a3cb3d069 Mon Sep 17 00:00:00 2001 From: Nikita Mikhailov Date: Sat, 20 Feb 2021 18:30:56 +0300 Subject: [PATCH 299/544] add some test + style --- .../0_stateless/01720_country_area.reference | 105 ++++++++++++++++++ .../queries/0_stateless/01720_country_area.sh | 16 +++ .../01720_country_intersection.reference | 104 +++++++++++++++++ .../0_stateless/01720_country_intersection.sh | 18 +++ .../queries/0_stateless/country_polygons.tsv | 2 + tests/queries/0_stateless/country_rings.tsv | 103 +++++++++++++++++ 6 files changed, 348 insertions(+) create mode 100644 tests/queries/0_stateless/01720_country_area.reference create mode 100755 tests/queries/0_stateless/01720_country_area.sh create mode 100644 tests/queries/0_stateless/01720_country_intersection.reference create mode 100755 tests/queries/0_stateless/01720_country_intersection.sh create mode 100644 tests/queries/0_stateless/country_polygons.tsv create mode 100644 tests/queries/0_stateless/country_rings.tsv diff --git a/tests/queries/0_stateless/01720_country_area.reference b/tests/queries/0_stateless/01720_country_area.reference new file mode 100644 index 00000000000..ccc9abbf04c --- /dev/null +++ b/tests/queries/0_stateless/01720_country_area.reference @@ -0,0 +1,105 @@ +Dhekelia Sovereign Base Area 127533079.61976177 +Kyrgyzstan 198994443077.2897 +Aruba 169577139.16920504 +Afghanistan 642167900289.4739 +Albania 28333199169.93513 +Andorra 452435114.7617939 +Ashmore and Cartier Islands 2696604.9276267616 +Austria 83991121342.6868 +Burundi 27040491570.2986 +Belgium 30672391961.48937 +Benin 116112410615.07138 +Burkina Faso 272767288599.06683 +Bulgaria 112764344245.5939 +Bahrain 585431263.0854981 +Bosnia and Herzegovina 51828357836.835045 +Bajo Nuevo Bank (Petrel Is.) 29452.023965717835 +Saint Barthelemy 24595825.379134282 +Belarus 207495810096.24286 +Bolivia 1086795900906.9404 +Barbados 444196434.0171095 +Bhutan 40362932122.222435 +Botswana 579022741104.1897 +Central African Republic 617982165549.6404 +Switzerland 41437057095.20376 +Clipperton Island 5042420.197242386 +Cameroon 464311319438.17255 +Republic of Congo 344887275569.75 +Coral Sea Islands 20628.814559882598 +Curaçao 463207585.46491855 +Czech Republic 78760344660.91792 +Djibouti 21846149319.499916 +Dominica 730475415.656727 +Algeria 2308849474061.9604 +Ethiopia 1127370045818.119 +Georgia 69572475312.24843 +Ghana 238667399375.48102 +Gibraltar 3831411.4171073083 +Guinea 244302369135.0428 +Gambia 10500266198.294697 +Guatemala 108814357557.89536 +Guam 564430992.5413051 +Heard Island and McDonald Islands 395072268.1670339 +Hungary 93204530848.81223 +Isle of Man 574622349.496232 +Iraq 437362923804.6782 +Israel 21904844963.57914 +Jamaica 11032909044.896214 +Jersey 119197004.74288748 +Jordan 88852268122.99756 +Baykonur Cosmodrome 6501480908.525035 +Siachen Glacier 2088368280.5951576 +Kosovo 10914986124.47776 +Laos 228111317612.30957 +Lebanon 9999649538.151918 +Liberia 95296808053.31937 +Libya 1623756106692.004 +Saint Lucia 604940965.7686977 +Liechtenstein 136883538.3183111 +Lesotho 30106906647.33149 +Luxembourg 2607643781.0226917 +Latvia 64575248440.09302 +Saint Martin 68198732.01942295 +Morocco 591718146420.0958 +Monaco 18750590.193543613 +Moldova 33208636022.82373 +Macedonia 25384586736.42653 +Mali 1252723270900.2302 +Montenegro 13729482523.613796 +Mongolia 1564646094133.3677 +Montserrat 99396603.21002583 +Namibia 822688081865.0547 +Niger 1181298791001.2363 +Norfolk Island 41003910.8372144 +Niue 220827518.96565723 +Nepal 147103938320.9306 +Nauru 28765050.65120624 +Poland 313439831446.58765 +Paraguay 399899482418.94604 +Qatar 11150102955.086788 +Romania 236371949926.43573 +Rwanda 25305141874.683277 +Western Sahara 90483117057.23624 +Scarborough Reef 93421.29269397438 +South Sudan 626858281405.9783 +Senegal 196219387423.09302 +Serranilla Bank 100843.54356614068 +Singapore 510505392.29170656 +San Marino 60357700.76923006 +Somaliland 167406759212.9871 +Somalia 471815737592.64276 +Republic of Serbia 77573594972.73866 +Suriname 145124645519.15808 +Slovakia 48456085451.12405 +Slovenia 20327209680.63761 +Swaziland 17113505052.204178 +Sint Maarten 23405778.740159098 +Syria 185944843149.37875 +Chad 1266282203998.255 +Togo 56863540226.407555 +Uganda 241852803539.38947 +Uruguay 177347712583.76038 +Vatican 10535.27369925663 +Akrotiri Sovereign Base Area 98415737.60517946 +Zambia 751913038381.5402 +Zimbabwe 389330493219.1703 diff --git a/tests/queries/0_stateless/01720_country_area.sh b/tests/queries/0_stateless/01720_country_area.sh new file mode 100755 index 00000000000..3edb2784b9a --- /dev/null +++ b/tests/queries/0_stateless/01720_country_area.sh @@ -0,0 +1,16 @@ +#!/usr/bin/env bash + +CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh +. "$CURDIR"/../shell_config.sh + +${CLICKHOUSE_CLIENT} -q "drop table if exists country_polygons;" +${CLICKHOUSE_CLIENT} -q "create table country_polygons(name String, p Array(Array(Tuple(Float64, Float64)))) engine=MergeTree() order by tuple();" +cat ${CURDIR}/country_polygons.tsv | ${CLICKHOUSE_CLIENT} -q "insert into country_polygons format TSV" +${CLICKHOUSE_CLIENT} -q "SELECT name, polygonAreaGeographic(p) from country_polygons" + +${CLICKHOUSE_CLIENT} -q "drop table if exists country_rings;" +${CLICKHOUSE_CLIENT} -q "create table country_rings(name String, p Array(Tuple(Float64, Float64))) engine=MergeTree() order by tuple();" +cat ${CURDIR}/country_rings.tsv | ${CLICKHOUSE_CLIENT} -q "insert into country_rings format TSV" +${CLICKHOUSE_CLIENT} -q "SELECT name, polygonAreaGeographic(p) from country_rings" +${CLICKHOUSE_CLIENT} -q "drop table if exists country_rings;" \ No newline at end of file diff --git a/tests/queries/0_stateless/01720_country_intersection.reference b/tests/queries/0_stateless/01720_country_intersection.reference new file mode 100644 index 00000000000..5b7cee92bdc --- /dev/null +++ b/tests/queries/0_stateless/01720_country_intersection.reference @@ -0,0 +1,104 @@ +Dhekelia Sovereign Base Area Dhekelia Sovereign Base Area [[[(33.847423,34.94245),(33.819672,34.964748),(33.80421,34.972602),(33.781896,34.976212),(33.784945,34.976212),(33.788046,34.976988),(33.7928,34.977763),(33.79435,34.977763),(33.791146,34.982414),(33.786495,34.984687),(33.782568,34.984687),(33.777917,34.984687),(33.77399,34.988666),(33.766135,34.990268),(33.761484,34.990268),(33.75921,34.988666),(33.765411,34.985566),(33.769339,34.983964),(33.770889,34.980088),(33.77554,34.980088),(33.780191,34.979313),(33.780986,34.976338),(33.780935,34.976345),(33.760427,34.979682),(33.717296,34.977769),(33.70152,34.97289),(33.702935,34.987943),(33.711461,34.985566),(33.71544,34.997296),(33.699731,35.002722),(33.69663,35.008975),(33.705312,35.015228),(33.702211,35.022256),(33.685003,35.029284),(33.679444,35.033891),(33.679435,35.033899),(33.675649,35.037036),(33.674099,35.046441),(33.678853,35.055794),(33.69446,35.058171),(33.705312,35.06675),(33.714717,35.06675),(33.719368,35.06277),(33.711461,35.040963),(33.707585,35.029284),(33.718489,35.032385),(33.739677,35.047216),(33.766135,35.03161),(33.77554,35.040188),(33.786495,35.038534),(33.79435,35.040188),(33.798278,35.052642),(33.824012,35.06675),(33.834865,35.063597),(33.842719,35.056621),(33.853571,35.058171),(33.866904,35.06675),(33.871555,35.073054),(33.876929,35.076826),(33.871555,35.085456),(33.871555,35.100236),(33.876206,35.118994),(33.889435,35.118994),(33.891812,35.110468),(33.89884,35.108814),(33.903594,35.099512),(33.905868,35.09636),(33.905868,35.090882),(33.913619,35.090882),(33.921474,35.080702),(33.914446,35.073054),(33.908245,35.070729),(33.906524,35.069122),(33.906506,35.069105),(33.898116,35.061272),(33.880133,35.073054),(33.874655,35.067525),(33.867627,35.060497),(33.855122,35.053417),(33.841169,35.051092),(33.834865,35.056621),(33.827113,35.061272),(33.813781,35.055794),(33.804375,35.049541),(33.799001,35.038534),(33.822359,35.030059),(33.830214,35.023031),(33.829387,35.001176),(33.829387,35.001172),(33.840342,34.993369),(33.859049,34.991819),(33.859049,34.974662),(33.850471,34.973009),(33.838068,34.963707),(33.84582,34.959728),(33.864423,34.962983),(33.891841,34.958139),(33.8838,34.949123),(33.874522,34.94123),(33.862315,34.937893),(33.847423,34.94245)],[(33.746689,35.002711),(33.752063,35.004323),(33.752063,35.0144),(33.746151,35.015207),(33.741314,35.013729),(33.740239,35.010101),(33.738761,35.005264),(33.739702,35.002576),(33.742792,35.001233),(33.746689,35.002711)]]] +Dhekelia Sovereign Base Area Kyrgyzstan [] +Kyrgyzstan Dhekelia Sovereign Base Area [] +Kyrgyzstan Kyrgyzstan [[[(74.746683,40.336505),(74.724668,40.337539),(74.705755,40.331286),(74.654388,40.291547),(74.637645,40.281987),(74.598681,40.266174),(74.565402,40.24695),(74.476622,40.172433),(74.369858,40.105822),(74.333788,40.09373),(74.302885,40.090061),(74.272293,40.093833),(74.237876,40.10391),(74.20315,40.109853),(74.167596,40.10639),(74.13287,40.095332),(74.101244,40.078847),(74.069515,40.067788),(74.003989,40.060812),(73.976704,40.043603),(73.966368,40.033268),(73.957894,40.021434),(73.952106,40.008257),(73.949729,39.993684),(73.944044,39.970068),(73.927301,39.953325),(73.907561,39.937873),(73.893195,39.918133),(73.885443,39.876534),(73.880689,39.86501),(73.872937,39.856845),(73.844102,39.838086),(73.83201,39.82372),(73.823018,39.805685),(73.818574,39.786048),(73.819814,39.766721),(73.829529,39.746981),(73.843068,39.740366),(73.859915,39.737162),(73.879345,39.727912),(73.893505,39.710394),(73.899602,39.689465),(73.904253,39.64647),(73.926784,39.592882),(73.921617,39.582133),(73.899292,39.571539),(73.883066,39.561204000000004),(73.870457,39.546786),(73.859192,39.524255),(73.848029,39.489735),(73.838314,39.475679),(73.820744,39.468186),(73.632642,39.448343),(73.604323,39.459608),(73.512236,39.467411),(73.476889,39.464776),(73.367645,39.443795),(73.343151,39.430669),(73.335503,39.415166),(73.333332,39.40111),(73.326408,39.390878),(73.269357,39.382558),(73.167864,39.355377),(73.136032,39.353465),(73.100995,39.361164),(73.083425,39.367831),(73.071126,39.370725),(73.058414,39.368813),(73.009321,39.348607),(72.994335,39.347832),(72.976765,39.352224),(72.91434,39.362715),(72.893669,39.363903),(72.850468,39.35672),(72.835275,39.356204),(72.65048,39.393772),(72.633737,39.394496),(72.616373,39.390413),(72.606141,39.383489),(72.587021,39.364885),(72.575756,39.359304),(72.559426,39.359562),(72.534104,39.372327),(72.519015,39.375686),(72.50806,39.371965),(72.476744,39.346127),(72.460414,39.344111),(72.443567,39.344783),(72.410701,39.351346),(72.393234,39.350829),(72.355924,39.336101),(72.334117,39.333828),(72.31634,39.328815),(72.304248,39.31357),(72.281407,39.259724),(72.240272,39.189909),(72.228903,39.189237),(72.228668,39.189502),(72.218568,39.20089),(72.209163,39.217297),(72.206476,39.226883),(72.206373,39.234609),(72.203272,39.24081),(72.182395,39.249337),(72.167098,39.258742),(72.158623,39.262617),(72.115525,39.268147),(72.094751,39.275072),(72.08576,39.290109),(72.084106,39.311245),(72.078112,39.336618),(72.06695,39.358477),(72.049793,39.368864),(72.031499,39.366694),(71.996463,39.351863),(71.959152,39.345868),(71.942306,39.339047),(71.843501,39.285045),(71.807017,39.272849),(71.77043,39.269542),(71.751207,39.274245),(71.732913,39.285097),(71.71834,39.301065),(71.710899,39.32096),(71.712966,39.341527),(71.723198,39.353981),(71.736634,39.364575),(71.748106,39.379561),(71.751827,39.399663),(71.749243,39.425502),(71.740251,39.448343),(71.724438,39.459401),(71.704594,39.458988),(71.602792,39.442348),(71.554319,39.444157),(71.512875,39.458833),(71.492101,39.493663),(71.500576,39.509217),(71.526001,39.538156),(71.531375,39.553091),(71.52538,39.569162),(71.510911,39.584148),(71.509654,39.584998),(71.493031,39.596241),(71.459545,39.612105),(71.441251,39.610658),(71.406731,39.598153),(71.378309,39.594173),(71.368543,39.591486),(71.36161799999999,39.587559),(71.347872,39.576242),(71.340586,39.571694),(71.305911,39.557173),(71.291958,39.548957),(71.260074,39.520638),(71.249273,39.514385),(71.236923,39.514488),(71.179975,39.523015),(71.137446,39.521155),(71.095898,39.51237),(71.062928,39.495627),(71.042309,39.467876),(71.027581,39.435217),(71.009288,39.407622),(70.977145,39.394909),(70.949808,39.400749),(70.925159,39.412944),(70.903351,39.420851),(70.884489,39.413874),(70.872242,39.402971),(70.864181,39.400594),(70.855292,39.40173),(70.840099,39.40142),(70.828059,39.398578),(70.797621,39.385349),(70.770129,39.382093),(70.739744,39.386228),(70.713285,39.398423),(70.698092,39.41899),(70.698609,39.431393),(70.703053,39.444725),(70.705431,39.458523),(70.699746,39.472165),(70.690134,39.479193),(70.666725,39.486893),(70.655976,39.493663),(70.655976,39.493714),(70.655925,39.49387),(70.655598,39.494664),(70.635977,39.542497),(70.622128,39.563529),(70.59784,39.577792),(70.58027,39.579549),(70.54358,39.574278),(70.526113,39.575622),(70.513504,39.581513),(70.490973,39.596861),(70.477227,39.601201),(70.459967,39.599548),(70.427618,39.590091),(70.411082,39.587456),(70.40147,39.584768),(70.395372,39.579497),(70.390411,39.57402),(70.384313,39.570867),(70.374288,39.571022),(70.353462,39.575932),(70.343179,39.576862),(70.33026,39.573089),(70.241066,39.522395),(70.223289,39.519087999999996),(70.20603,39.5241),(70.204376,39.538156),(70.215021,39.574071),(70.214814,39.591125),(70.210267,39.609728),(70.200655,39.619495),(70.185204,39.610348),(70.15735,39.563891),(70.148979,39.554279),(70.132442,39.550042),(70.116268,39.554641),(70.08273,39.569782000000004),(70.062266,39.573451),(70.040355,39.573606),(70.019684,39.568594),(70.002734,39.556863),(69.987128,39.539603),(69.978757,39.534436),(69.948629,39.545081),(69.907753,39.548492),(69.830445,39.536244),(69.791585,39.545391),(69.74983,39.563839),(69.710349,39.574071),(69.669421,39.577792),(69.582295,39.573555),(69.564932,39.568129),(69.531445,39.545649),(69.514392,39.537588),(69.496099,39.532575),(69.477288,39.530405),(69.455274,39.530456),(69.412176,39.52503),(69.391454,39.530508),(69.367218,39.549938),(69.361637,39.552884),(69.352025,39.549628),(69.348407,39.543169),(69.345927,39.535366),(69.339726,39.528079),(69.300348,39.515625),(69.286189,39.539707),(69.29115,39.658872),(69.287533,39.677786),(69.280091,39.694271),(69.265932,39.707758),(69.248362,39.719385),(69.233324,39.732666),(69.226296,39.751011),(69.229552,39.790544),(69.240714,39.82894),(69.286447,39.935755),(69.305671,39.968621),(69.310064,39.978646),(69.310322,39.984796),(69.313991,39.986914),(69.32784,39.984744),(69.357296,39.959474),(69.405355,39.896119),(69.501576,39.922474),(69.500439,39.935703),(69.477288,39.968156),(69.478115,39.975029),(69.477288,39.981592),(69.475118,39.987741),(69.471191,39.993684),(69.463129,40.025413),(69.46964,40.050993),(69.485712,40.073731),(69.506331,40.096933),(69.509224,40.103445),(69.513462,40.120705),(69.518009,40.125097),(69.526071,40.123185),(69.536613,40.107889),(69.543021,40.103083),(69.558834,40.101739),(69.575474,40.1036),(69.969558,40.211603),(70.004595,40.208761),(70.14717,40.136983),(70.168822,40.13166),(70.218535,40.134141),(70.241531,40.132745),(70.263494,40.12427),(70.277808,40.112075),(70.290262,40.098174),(70.306282,40.08479),(70.324265,40.077503),(70.359509,40.074041),(70.397646,40.061225),(70.477227,40.052027),(70.502135,40.045981),(70.525493,40.033682),(70.535622,40.015957),(70.520946,39.993736),(70.505029,39.985106),(70.488493,39.978543),(70.473197,39.97012),(70.460226,39.956115),(70.450872,39.937408),(70.44803,39.919993),(70.455627,39.906816),(70.477227,39.900925),(70.485961,39.909606),(70.494177,39.931672),(70.500895,39.940561),(70.512574,39.94516),(70.555052,39.946245),(70.576601,39.952239),(70.5966,39.961955),(70.613963,39.975752),(70.627399,39.993891),(70.635977,40.028514),(70.634737,40.059313),(70.63944,40.084945),(70.66595,40.104168),(70.729822,40.120705),(70.743464,40.127113),(70.782532,40.152589),(70.788733,40.158997),(70.792144,40.161374),(70.797725,40.161942),(70.802686,40.160134),(70.806045,40.157653),(70.806716,40.156155),(70.824906,40.163544),(70.831831,40.168195),(70.845164,40.171813),(70.898545,40.162614),(70.929293,40.170624),(70.962883,40.189693),(70.979522,40.214084),(70.958955,40.238372),(70.995129,40.266587),(71.053213,40.274235),(71.169382,40.26142),(71.201008,40.263848),(71.215115,40.280747),(71.223952,40.302864),(71.239558,40.321106),(71.253821,40.324413),(71.263949,40.318212),(71.272838,40.307722),(71.283225,40.29842),(71.297074,40.293821),(71.313559,40.292684),(71.344926,40.295216),(71.365132,40.294131),(71.396551,40.271548),(71.441355,40.260903),(71.450966,40.248914),(71.458925,40.234134),(71.477115,40.220802),(71.498405,40.210518),(71.521246,40.203749),(71.568272,40.196566),(71.592973,40.198426),(71.601655,40.20995),(71.604032,40.227261),(71.61013,40.246382),(71.62832,40.258887),(71.645993,40.247105),(71.660153,40.224471),(71.667181,40.204162),(71.666871,40.163131),(71.673072,40.147886),(71.693329,40.141117),(71.707178,40.144269),(71.759785,40.168092),(71.775908,40.179926),(71.78676,40.193517),(71.805467,40.225298),(71.836473,40.249172),(71.872956,40.250774),(71.912334,40.243436),(71.951814,40.240904),(71.96339,40.243746),(71.969178,40.244418),(71.977033,40.243074),(71.989021,40.239302),(72.004834,40.237287),(72.01889,40.240026),(72.025402,40.250878),(72.019821,40.258732),(72.013239,40.262072),(72.005971,40.26576),(71.977033,40.276096),(71.958222,40.286534),(71.951091,40.30152),(71.951094,40.301529),(71.956672,40.31568),(72.043178,40.349321),(72.069637,40.369423),(72.084106,40.39738),(72.090101,40.416035),(72.099299,40.426371),(72.165858,40.454431),(72.182705,40.45779),(72.228328,40.459606),(72.235931,40.459909),(72.254225,40.458307),(72.263415,40.452651),(72.26363,40.452519),(72.259599,40.44239),(72.24513,40.438721),(72.228077,40.437533),(72.216501,40.434949),(72.211768,40.425836),(72.211644,40.425596),(72.224356,40.422443),(72.269624,40.424045),(72.284301,40.41986),(72.343418,40.393401),(72.370497,40.38565),(72.394061,40.389422),(72.414422,40.410713),(72.425171,40.435983),(72.426204,40.45934),(72.426096,40.459545),(72.420028,40.471027),(72.415662,40.479287),(72.415585,40.47933),(72.372254,40.503265),(72.363469,40.512309),(72.363467,40.51241),(72.363262,40.523573999999996),(72.36998,40.539852),(72.370083,40.557732),(72.369928,40.557930999999996),(72.348483,40.585379),(72.348509,40.585508),(72.351893,40.601967),(72.381556,40.612148),(72.414835,40.589875),(72.447908,40.560471),(72.476744,40.549567),(72.515294,40.54564),(72.585781,40.508743),(72.625468,40.5105),(72.640868,40.519853),(72.65048,40.532151999999996),(72.655131,40.546363),(72.656474,40.561143),(72.664226,40.577783),(72.682416,40.577679),(72.719209,40.564864),(72.748355,40.575096),(72.760034,40.641758),(72.783908,40.669663),(72.818945,40.681084),(72.890982,40.695088),(72.976765,40.736068),(73.070609,40.762474),(73.118048,40.782938),(73.148641,40.813686),(73.14337,40.833839),(73.143292,40.833853),(73.135042,40.835274),(73.112467,40.839162),(73.053556,40.83632),(73.033299,40.847224),(73.01945,40.8619),(73.00343,40.870168),(72.929429,40.844175),(72.883127,40.819628),(72.870311,40.818181),(72.872998,40.834821),(72.868658,40.864122),(72.830107,40.87208),(72.701226,40.863243),(72.658231,40.867171),(72.619474,40.88009),(72.588468,40.905825),(72.545577,40.956519),(72.52625,40.962204),(72.501445,40.963496),(72.483358,40.970575),(72.483565,40.99352),(72.485219,40.999566),(72.484702,41.004682),(72.481911,41.008816),(72.476744,41.011813),(72.423517,41.01574),(72.395198,41.022045),(72.374321,41.031967),(72.345692,41.06597),(72.33236,41.072843),(72.314066,41.061681),(72.308795,41.054446),(72.297323,41.028143),(72.289778,41.023544),(72.252985,41.019616),(72.195486,41.006358),(72.165135,40.999359),(72.178777,41.023182),(72.185599,41.041062),(72.185599,41.060647),(72.1764,41.112892),(72.174643,41.141418),(72.169889,41.168651),(72.158417,41.187875),(72.132889,41.199295),(72.108497,41.196401),(72.085243,41.184877),(72.063642,41.170201),(72.033773,41.15661),(72.016513,41.163535),(72.001217,41.18002),(71.977033,41.195213),(71.897658,41.184929),(71.871613,41.194489),(71.866342,41.236606),(71.868822,41.279239),(71.863034,41.312208),(71.847015,41.341819),(71.75317,41.44729),(71.745212,41.45282),(71.73684,41.455404),(71.730433,41.451166),(71.729709,41.430082),(71.721648,41.424759),(71.712242,41.428015),(71.706868,41.444086),(71.696119,41.447445),(71.691469,41.441916),(71.687334,41.431012),(71.681443,41.422847),(71.671935,41.425483),(71.671108,41.437627),(71.689505,41.493799),(71.689401,41.514625),(71.68413,41.534055),(71.671418,41.547388),(71.649507,41.54992),(71.62739,41.543202),(71.615297,41.532143),(71.595454,41.493799),(71.595454,41.493747),(71.595557,41.493696),(71.595557,41.493489),(71.605582,41.476849),(71.633694,41.449616),(71.637415,41.431271),(71.633074,41.411324),(71.618811,41.377786),(71.585532,41.323525),(71.557937,41.301718),(71.52383,41.296654),(71.480629,41.310761),(71.432466,41.344816),(71.418772,41.3474),(71.412571,41.334687),(71.421459,41.162088),(71.416085,41.127362),(71.393657,41.112737),(71.325238,41.157334),(71.300226,41.133046),(71.289478,41.113874),(71.276145,41.113151),(71.263381,41.123486),(71.253821,41.13749),(71.241522,41.175162),(71.230153,41.187255),(71.206382,41.188753),(71.185711,41.180227),(71.183024,41.166377),(71.187882,41.148446),(71.189949,41.127465),(71.18013,41.108138),(71.164421,41.116199),(71.139099,41.148394),(71.123079,41.158212),(71.085769,41.162036),(71.067579,41.169736),(71.047012,41.182035),(71.024688,41.189787),(70.977145,41.19635),(70.93415,41.19144),(70.914203,41.193042),(70.895962,41.206116),(70.882164,41.220482),(70.865731,41.233195),(70.847747,41.243065),(70.828524,41.249111),(70.805993,41.247458),(70.786563,41.24043),(70.770439,41.238518),(70.758037,41.25216),(70.75628,41.269627),(70.77106,41.331018),(70.769199,41.352102),(70.759691,41.372515),(70.70357,41.445482),(70.686724,41.462483),(70.66781,41.471372),(70.633807,41.467496),(70.511334,41.414476),(70.477227,41.404657),(70.470782,41.404879),(70.453198,41.405484),(70.438057,41.416078),(70.413614,41.450701),(70.398989,41.464964),(70.382246,41.476436),(70.344936,41.493489),(70.344729,41.493644),(70.344522,41.493696),(70.344419,41.493799),(70.203446,41.505633),(70.166549,41.520206),(70.148255,41.552452),(70.169288,41.578342),(70.33119,41.649629),(70.390824,41.685054),(70.423484,41.696913),(70.453663,41.71208),(70.477227,41.738435),(70.506476,41.78559),(70.550091,41.824063),(70.648948,41.887393),(70.679696,41.901113),(70.779328,41.909665),(70.814003,41.919535),(70.825113,41.93633),(70.828059,41.993743),(70.845474,42.030356),(70.886711,42.038495),(70.935907,42.036867),(70.977145,42.044231),(71.118429,42.122908),(71.201008,42.140788),(71.238215,42.160244),(71.253201,42.197555),(71.249868,42.198385),(71.217854,42.206365),(71.077604,42.281167),(71.045772,42.29096),(71.014042,42.287704),(70.9696,42.263468),(70.947793,42.248146),(70.918648,42.253365),(70.897822,42.261608),(70.858134,42.291063),(70.852657,42.306075),(70.864801,42.321552),(70.888825,42.338575),(70.900354,42.346744),(70.932703,42.3762),(70.939835,42.387827),(70.937354,42.394778),(70.931773,42.401496),(70.929913,42.412296),(70.936114,42.431623),(70.947586,42.451286),(70.961952,42.468107),(70.977145,42.478752),(71.024739,42.455963),(71.041017,42.4548),(71.054712,42.460381),(71.064168,42.470329),(71.066287,42.482137),(71.057864,42.493402),(71.057657,42.493402),(71.057554,42.493506),(71.057554,42.493609),(71.022724,42.51645),(71.012699,42.526036),(71.00319,42.563527),(71.040656,42.580529),(71.127007,42.590606),(71.142768,42.602543),(71.148091,42.6174),(71.146437,42.656855),(71.149021,42.677939),(71.157548,42.687938),(71.19522,42.69724),(71.211446,42.705224),(71.223849,42.717936),(71.245449,42.747133),(71.262502,42.753567),(71.284723,42.7484),(71.308495,42.740105),(71.329785,42.737031),(71.347562,42.742844),(71.363065,42.75411),(71.376294,42.768321),(71.386939,42.783074),(71.404613,42.794883),(71.428849,42.79558),(71.477115,42.78354),(71.493651,42.788914),(71.504193,42.789612),(71.51432199999999,42.78571),(71.553803,42.760905),(71.566308,42.757236),(71.583568,42.759639),(71.693536,42.811807),(71.726402,42.819660999999996),(71.796682,42.822297),(71.831408,42.831573),(71.847738,42.834053),(71.863241,42.829041),(71.878847,42.821341),(71.895384,42.816044),(71.956982,42.804598),(72.072737,42.757185),(72.10488,42.750337),(72.11997,42.75181),(72.148805,42.76199),(72.164101,42.765091),(72.292362,42.76106),(72.358405,42.741397),(72.476744,42.682564),(72.51178,42.677551),(72.583817,42.678275),(72.725824,42.652876),(72.756933,42.640215),(72.780601,42.620449),(72.815224,42.573036),(72.840752,42.555698),(72.874549,42.543993),(72.908345,42.536965),(72.942348,42.536035),(73.070609,42.551926),(73.115051,42.550841),(73.152671,42.539213),(73.172722,42.527845),(73.189052,42.520765),(73.206828,42.517173),(73.278348,42.513401),(73.301396,42.507071),(73.316589,42.493712),(73.313385,42.463171),(73.314419,42.441571),(73.326201,42.428677),(73.417461,42.41705),(73.476889,42.399067),(73.505105,42.402943),(73.505518,42.420926),(73.505077,42.421557),(73.432964,42.524847),(73.417565,42.556163),(73.41002,42.58965),(73.412707,42.627322),(73.423869,42.662772),(73.476889,42.75088),(73.503554,42.794082),(73.507068,42.809404),(73.504278,42.827594),(73.491979,42.861003),(73.489498,42.877823),(73.493012,42.894954),(73.501177,42.909346),(73.521744,42.936166),(73.559675,43.017298),(73.57094,43.031922),(73.58665,43.042283),(73.634089,43.062463),(73.805551,43.114734),(73.822811,43.117318),(73.864256,43.116103),(73.90167,43.13096),(73.935053,43.199716),(73.965335,43.216924),(73.985385,43.211679),(74.021456,43.186306),(74.039852,43.181861),(74.197362,43.195788),(74.213899,43.202635),(74.216586,43.216924),(74.206044,43.234106),(74.178759,43.261702),(74.207697,43.24979),(74.25865,43.215761),(74.286969,43.207209),(74.320042,43.201654),(74.348774,43.191034),(74.400554,43.159537),(74.420604,43.151837),(74.462772,43.148634),(74.498222,43.131115),(74.539253,43.122899),(74.558167,43.115044),(74.572016,43.102202),(74.597028,43.070344),(74.61129,43.058174),(74.71392,42.999883),(74.74875,42.990013),(74.862851,42.975828),(74.948531,42.944848),(74.976849,42.926477),(75.005375,42.9164),(75.066146,42.902602),(75.094258,42.890303),(75.118133,42.876531),(75.178594,42.84966),(75.204639,42.84519),(75.271198,42.845758000000004),(75.496611,42.823925),(75.535988,42.827025),(75.555936,42.825217),(75.620428,42.805295),(75.646163,42.806096),(75.675308,42.815371999999996),(75.703833,42.830488),(75.727605,42.848678),(75.73825,42.861571),(75.770599,42.92565),(75.782072,42.933324),(75.797161,42.936192),(75.807059,42.936771),(75.858966,42.939809),(75.898033,42.935701),(75.976582,42.918725),(75.999113,42.91764),(76.064535,42.93397),(76.09058,42.93428),(76.163754,42.921154),(76.254498,42.921154),(76.340487,42.901672),(76.351132,42.902886),(76.370976,42.909604),(76.382965,42.910069),(76.3933,42.905263),(76.404463,42.889244),(76.414074,42.885497),(76.432058,42.88958),(76.461927,42.910999),(76.48053,42.916968),(76.506885,42.914488),(76.558768,42.89821),(76.58502,42.894799),(76.605897,42.898494),(76.644345,42.911129),(76.688786,42.910328),(76.707907,42.913661),(76.74625,42.928879),(76.750385,42.932574),(76.751625,42.937096),(76.753899,42.940765),(76.76134,42.942057),(76.77953,42.939292),(76.785214,42.939654),(76.792966,42.943065),(76.814153,42.961487),(76.83348,42.971745),(76.853427,42.974122),(76.897662,42.974536),(76.93735,42.986034),(76.956884,42.988514),(76.976417,42.98221),(77.089899,42.968154),(77.124005,42.958904),(77.135064,42.951746),(77.163693,42.921981),(77.178679,42.913196),(77.195112,42.910069),(77.21189,42.909944),(77.229632,42.909811),(77.328127,42.897564),(77.36089,42.90454),(77.403265,42.919759),(77.418561,42.922239),(77.43396,42.921412),(77.461762,42.914384),(77.501553,42.914539),(77.520984,42.906917),(77.53845,42.896272),(77.557777,42.888133),(77.574417,42.887848),(77.62692,42.906555),(77.647797,42.909501),(77.71384,42.907589),(77.781226,42.895626),(77.787531,42.889709),(77.791665,42.883043),(77.798176,42.877513),(77.809958,42.871286),(77.83559,42.879839),(77.852229,42.887952),(77.861221,42.89082),(77.883855,42.893223),(77.907006,42.891285),(77.929331,42.885006),(77.986175,42.860202),(78.030616,42.854672),(78.137897,42.861984),(78.183579,42.86015),(78.229777,42.865007),(78.249104,42.862398),(78.290136,42.851442),(78.31122,42.850512),(78.328376,42.855292),(78.364653,42.872526),(78.38553,42.878237),(78.429559,42.880665),(78.496118,42.875601),(78.594303,42.850228),(78.635437,42.832477),(78.669957,42.811161),(78.687114,42.804598),(78.807933,42.79558),(78.888135,42.771215),(78.954281,42.768424),(78.992935,42.757133),(79.030866,42.756151),(79.108794,42.785348),(79.148274,42.790981),(79.173389,42.785632),(79.180831,42.77553),(79.175146,42.737031),(79.176076,42.713931),(79.181864,42.693597),(79.192199,42.674838),(79.206359,42.656467),(79.242119,42.629776),(79.321287,42.602181),(79.353016,42.577299),(79.398388,42.496942),(79.425673,42.469605),(79.47642,42.453973),(79.571918,42.449529),(79.652533,42.461053),(79.696458,42.459838),(79.917943,42.42444),(79.960111,42.403511),(79.97396,42.39147),(80.012098,42.349509),(80.077003,42.305765),(80.110076,42.273338),(80.136534,42.23887),(80.16661,42.208717),(80.210328,42.189519),(80.21653,42.174404),(80.219837,42.141873),(80.224591,42.125569),(80.235133,42.110945),(80.247432,42.098413),(80.256527,42.084357),(80.257561,42.065263),(80.231206,42.033689),(80.181906,42.020976),(79.930862,42.023276),(79.879496,42.013199),(79.842909,42.00183),(79.826786,41.992244),(79.812833,41.97762),(79.803842,41.959998),(79.792576,41.922817),(79.783274,41.905092),(79.747514,41.879745),(79.702969,41.874939),(79.655737,41.875843),(79.610985,41.867626),(79.554658,41.837602),(79.489856,41.819257),(79.410997,41.778588),(79.390844,41.772697),(79.367382,41.772387),(79.30413,41.787554),(79.282323,41.783497),(79.264856,41.774506),(79.217521,41.741226),(79.19592,41.729521),(79.174526,41.722622),(79.127914,41.714302),(79.088743,41.702546),(78.976192,41.6418),(78.915834,41.63317),(78.897437,41.626272),(78.807313,41.578445),(78.672128,41.538448),(78.658278,41.532453),(78.645049,41.52372),(78.637401,41.512868),(78.629133,41.48796),(78.619211,41.478089),(78.583968,41.465997),(78.51038,41.454422),(78.41757,41.400471),(78.377675,41.386622),(78.359899,41.377527),(78.343466,41.362024),(78.339332,41.344041),(78.356901,41.305542),(78.359692,41.287455),(78.34946,41.270402),(78.331477,41.258723),(78.291582,41.240791),(78.275356,41.228854),(78.250345,41.200535),(78.231335,41.172856),(78.204456,41.133718),(78.190503,41.11775),(78.176034,41.105502),(78.074955,41.039512),(78.057178,41.034344),(78.036508,41.036101),(77.997647,41.049795),(77.866699,41.064058),(77.831042,41.062973),(77.797556,41.054704),(77.665574,41.001271),(77.650278,40.997137),(77.631778,40.995793),(77.580721,40.997705),(77.503517,40.981066),(77.47488799999999,40.982047),(77.445226,40.993675),(77.388795,41.011658),(77.332985,41.02065),(77.301152,41.019306),(77.243171,41.005664),(77.118838,41.011658),(77.088555,41.019565),(77.035018,41.040338),(77.007733,41.044214),(76.900143,41.025766),(76.860972,41.013208),(76.834927,40.99352),(76.820871,40.97781),(76.783974,40.957139),(76.766818,40.944685),(76.757413,40.925772),(76.760927,40.9066),(76.768368,40.887014),(76.770848,40.867119),(76.762477,40.847017),(76.746871,40.83446),(76.707286,40.817613),(76.674214,40.795444),(76.647859,40.764851),(76.630805,40.728885),(76.624191,40.627547),(76.62078,40.611321),(76.609411,40.59711),(76.556391,40.565690000000004),(76.531173,40.534995),(76.498927,40.464611),(76.476499,40.436138),(76.449111,40.415519),(76.361984,40.371904),(76.330152,40.348081),(76.313512,40.343327),(76.299353,40.355677),(76.283333,40.417276),(76.273411,40.434122),(76.244162,40.441202),(76.21543,40.416552),(76.185044,40.384203),(76.151351,40.368131),(76.132541,40.371542),(76.095231,40.387355),(76.07611,40.391954),(76.051616,40.390197),(75.962422,40.357331),(75.949297,40.343068),(75.938031,40.326222),(75.921495,40.309117),(75.901858,40.298885),(75.880464,40.295268),(75.858553,40.296353),(75.79375,40.30891),(75.772046,40.31015),(75.750549,40.308342),(75.704454,40.293149),(75.681819,40.291702),(75.664869,40.305706),(75.656291,40.32307),(75.640168,40.367305),(75.638308,40.386115),(75.64636899999999,40.405338),(75.659598,40.419084),(75.669314,40.432365),(75.666936,40.450245),(75.657221,40.461097),(75.629006,40.481303),(75.617947,40.493964),(75.610816,40.512877),(75.605648,40.569411),(75.587665,40.611941),(75.559863,40.63287),(75.523793,40.63318),(75.481832,40.614111),(75.26324,40.480166),(75.253318,40.47598),(75.241536,40.474068),(75.229237,40.470347),(75.223966,40.462441),(75.220142,40.452984),(75.212494,40.444716),(75.19389,40.441254),(75.175287,40.448127),(75.15658,40.458152),(75.13808,40.463888),(75.113172,40.460787),(75.064389,40.443837),(75.039068,40.441099),(75.002378,40.4473),(74.966101,40.459289),(74.879181,40.505074),(74.856443,40.513187),(74.835359,40.511637),(74.832312,40.508116),(74.820063,40.493964),(74.816032,40.48368),(74.815102,40.480114),(74.807041,40.461769),(74.794638,40.440737),(74.787404,40.420738),(74.794638,40.405545),(74.841664,40.372059),(74.853859,40.35883),(74.862128,40.32617),(74.830915,40.319917),(74.746683,40.336505)],[(70.63298,39.79845),(70.661609,39.809819),(70.694527,39.814832),(70.70481,39.822067),(70.706412,39.839998),(70.698919,39.858447),(70.68662,39.860876),(70.654994,39.849765),(70.619906,39.850695),(70.498725,39.881908),(70.483739,39.882218),(70.482602,39.866767),(70.490147,39.850179),(70.503479,39.835502),(70.537379,39.817312),(70.547301,39.807649),(70.575878,39.77008),(70.581614,39.766566),(70.63298,39.79845)],[(71.007634,39.911157),(71.009288,39.885732),(71.021897,39.880823),(71.036056,39.887541),(71.049802,39.897979),(71.060964,39.904129),(71.079981,39.903405),(71.084787,39.894827),(71.087216,39.88382),(71.098895,39.8755),(71.113674,39.874466),(71.16101,39.884233),(71.1946,39.884802),(71.208449,39.888781),(71.219094,39.900925),(71.221058,39.931724),(71.17703,39.968156),(71.174704,39.993994),(71.190982,40.006241),(71.237284,40.031098),(71.244002,40.046756),(71.236354,40.056057),(71.223745,40.057866),(71.198217,40.052595),(71.192946,40.05027),(71.181887,40.043397),(71.176926,40.042001),(71.169382,40.042828),(71.165454,40.044637),(71.162043,40.046911),(71.105975,40.064946),(71.078844,40.07864),(71.06174,40.095176),(71.047942,40.122513),(71.031199,40.146905),(71.008564,40.157757),(70.977145,40.144579),(70.959885,40.113418),(70.954046,40.095797),(70.952961,40.079208),(70.958438,40.063085),(70.98314,40.021383),(70.994509,40.008877),(71.007634,40.003503),(71.034299,39.99911),(71.045823,39.99203),(71.050371,39.962885),(71.007634,39.911157)],[(71.757304,39.903095),(71.767123,39.915446),(71.789757,39.979318),(71.78552,39.989705),(71.760612,39.98371),(71.724128,39.962678),(71.706868,39.956115),(71.681753,39.955082),(71.665527,39.940199),(71.675346,39.925781),(71.696843,39.913844),(71.71617,39.906764),(71.741801,39.90077),(71.757304,39.903095)]]] +Aruba Aruba [[[(-69.936391,12.531724),(-69.924672,12.519232),(-69.915761,12.497016),(-69.88019800000001,12.453559),(-69.87682,12.427395),(-69.888092,12.41767),(-69.908803,12.417792),(-69.930531,12.425971),(-69.945139,12.440375),(-69.924672,12.440375),(-69.924672,12.447211),(-69.958567,12.463202),(-70.027659,12.522935),(-70.048085,12.531155),(-70.058095,12.537177),(-70.062408,12.54682),(-70.060374,12.556952),(-70.051096,12.574042),(-70.048736,12.583726),(-70.052642,12.600002),(-70.059641,12.614244),(-70.061106,12.625393),(-70.048736,12.632148),(-70.007151,12.585517),(-69.996938,12.577582),(-69.936391,12.531724)]]] +Aruba Afghanistan [] +Aruba Albania [] +Aruba Andorra [] +Aruba Ashmore and Cartier Islands [] +Aruba Austria [] +Aruba Burundi [] +Aruba Belgium [] +Aruba Benin [] +Aruba Burkina Faso [] +Aruba Bulgaria [] +Aruba Bahrain [] +Aruba Bosnia and Herzegovina [] +Aruba Bajo Nuevo Bank (Petrel Is.) [] +Aruba Saint Barthelemy [] +Aruba Belarus [] +Aruba Bolivia [] +Aruba Barbados [] +Aruba Bhutan [] +Aruba Botswana [] +Aruba Central African Republic [] +Aruba Switzerland [] +Aruba Clipperton Island [] +Aruba Cameroon [] +Aruba Republic of Congo [] +Aruba Coral Sea Islands [] +Aruba Curaçao [] +Aruba Czech Republic [] +Aruba Djibouti [] +Aruba Dominica [] +Aruba Algeria [] +Aruba Ethiopia [] +Aruba Georgia [] +Aruba Ghana [] +Aruba Gibraltar [] +Aruba Guinea [] +Aruba Gambia [] +Aruba Guatemala [] +Aruba Guam [] +Aruba Heard Island and McDonald Islands [] +Aruba Hungary [] +Aruba Isle of Man [] +Aruba Iraq [] +Aruba Israel [] +Aruba Jamaica [] +Aruba Jersey [] +Aruba Jordan [] +Aruba Baykonur Cosmodrome [] +Aruba Siachen Glacier [] +Aruba Kosovo [] +Aruba Laos [] +Aruba Lebanon [] +Aruba Liberia [] +Aruba Libya [] +Aruba Saint Lucia [] +Aruba Liechtenstein [] +Aruba Lesotho [] +Aruba Luxembourg [] +Aruba Latvia [] +Aruba Saint Martin [] +Aruba Morocco [] +Aruba Monaco [] +Aruba Moldova [] +Aruba Macedonia [] +Aruba Mali [] +Aruba Montenegro [] +Aruba Mongolia [] +Aruba Montserrat [] +Aruba Namibia [] +Aruba Niger [] +Aruba Norfolk Island [] +Aruba Niue [] +Aruba Nepal [] +Aruba Nauru [] +Aruba Poland [] +Aruba Paraguay [] +Aruba Qatar [] +Aruba Romania [] +Aruba Rwanda [] +Aruba Western Sahara [] +Aruba Scarborough Reef [] +Aruba South Sudan [] +Aruba Senegal [] +Aruba Serranilla Bank [] +Aruba Singapore [] +Aruba San Marino [] +Aruba Somaliland [] +Aruba Somalia [] +Aruba Republic of Serbia [] +Aruba Suriname [] +Aruba Slovakia [] +Aruba Slovenia [] +Aruba Swaziland [] +Aruba Sint Maarten [] +Aruba Syria [] +Aruba Chad [] +Aruba Togo [] +Aruba Uganda [] +Aruba Uruguay [] +Aruba Vatican [] diff --git a/tests/queries/0_stateless/01720_country_intersection.sh b/tests/queries/0_stateless/01720_country_intersection.sh new file mode 100755 index 00000000000..13efe97becf --- /dev/null +++ b/tests/queries/0_stateless/01720_country_intersection.sh @@ -0,0 +1,18 @@ +#!/usr/bin/env bash + +CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh +. "$CURDIR"/../shell_config.sh + +${CLICKHOUSE_CLIENT} -q "drop table if exists country_polygons;" +${CLICKHOUSE_CLIENT} -q "create table country_polygons(name String, p Array(Array(Tuple(Float64, Float64)))) engine=MergeTree() order by tuple();" +cat ${CURDIR}/country_polygons.tsv | ${CLICKHOUSE_CLIENT} -q "insert into country_polygons format TSV" +${CLICKHOUSE_CLIENT} -q "SELECT c, d, polygonsIntersectionGeographic(a, b) FROM (SELECT first.p AS a, second.p AS b, first.name AS c, second.name AS d FROM country_polygons AS first CROSS JOIN country_polygons AS second LIMIT 100) format TSV" +${CLICKHOUSE_CLIENT} -q "drop table if exists country_polygons;" + + +${CLICKHOUSE_CLIENT} -q "drop table if exists country_rings;" +${CLICKHOUSE_CLIENT} -q "create table country_rings(name String, p Array(Tuple(Float64, Float64))) engine=MergeTree() order by tuple();" +cat ${CURDIR}/country_rings.tsv | ${CLICKHOUSE_CLIENT} -q "insert into country_rings format TSV" +${CLICKHOUSE_CLIENT} -q "SELECT c, d, polygonsIntersectionGeographic(a, b) FROM (SELECT first.p AS a, second.p AS b, first.name AS c, second.name AS d FROM country_rings AS first CROSS JOIN country_rings AS second LIMIT 100) format TSV" +${CLICKHOUSE_CLIENT} -q "drop table if exists country_rings;" \ No newline at end of file diff --git a/tests/queries/0_stateless/country_polygons.tsv b/tests/queries/0_stateless/country_polygons.tsv new file mode 100644 index 00000000000..4fa8313e3cb --- /dev/null +++ b/tests/queries/0_stateless/country_polygons.tsv @@ -0,0 +1,2 @@ +Dhekelia Sovereign Base Area [[(33.905868,35.090882),(33.913619,35.090882),(33.921474,35.080702),(33.914446,35.073054),(33.908245,35.070729),(33.906524,35.069122),(33.906506,35.069105),(33.898116,35.061272),(33.880133,35.073054),(33.874655,35.067525),(33.867627,35.060497),(33.855122,35.053417),(33.841169,35.051092),(33.834865,35.056621),(33.827113,35.061272),(33.813781,35.055794),(33.804375,35.049541),(33.799001,35.038534),(33.822359,35.030059),(33.830214,35.023031),(33.829387,35.001176),(33.829387,35.001172),(33.840342,34.993369),(33.859049,34.991819),(33.859049,34.974662),(33.850471,34.973009),(33.838068,34.963707),(33.84582,34.959728),(33.864423,34.962983),(33.891841,34.958139),(33.8838,34.949123),(33.874522,34.94123),(33.862315,34.937893),(33.847423,34.94245),(33.819672,34.964748),(33.80421,34.972602),(33.781896,34.976212),(33.784945,34.976212),(33.788046,34.976988),(33.7928,34.977763),(33.79435,34.977763),(33.791146,34.982414),(33.786495,34.984687),(33.782568,34.984687),(33.777917,34.984687),(33.77399,34.988666),(33.766135,34.990268),(33.761484,34.990268),(33.75921,34.988666),(33.765411,34.985566),(33.769339,34.983964),(33.770889,34.980088),(33.77554,34.980088),(33.780191,34.979313),(33.780986,34.976338),(33.780935,34.976345),(33.760427,34.979682),(33.717296,34.977769),(33.70152,34.97289),(33.702935,34.987943),(33.711461,34.985566),(33.71544,34.997296),(33.699731,35.002722),(33.69663,35.008975),(33.705312,35.015228),(33.702211,35.022256),(33.685003,35.029284),(33.679444,35.033891),(33.679435,35.033899),(33.675649,35.037036),(33.674099,35.046441),(33.678853,35.055794),(33.69446,35.058171),(33.705312,35.06675),(33.714717,35.06675),(33.719368,35.06277),(33.711461,35.040963),(33.707585,35.029284),(33.718489,35.032385),(33.739677,35.047216),(33.766135,35.03161),(33.77554,35.040188),(33.786495,35.038534),(33.79435,35.040188),(33.798278,35.052642),(33.824012,35.06675),(33.834865,35.063597),(33.842719,35.056621),(33.853571,35.058171),(33.866904,35.06675),(33.871555,35.073054),(33.876929,35.076826),(33.871555,35.085456),(33.871555,35.100236),(33.876206,35.118994),(33.889435,35.118994),(33.891812,35.110468),(33.89884,35.108814),(33.903594,35.099512),(33.905868,35.09636),(33.905868,35.090882)],[(33.742792,35.001233),(33.746689,35.002711),(33.752063,35.004323),(33.752063,35.0144),(33.746151,35.015207),(33.741314,35.013729),(33.740239,35.010101),(33.738761,35.005264),(33.739702,35.002576),(33.742792,35.001233)]] +Kyrgyzstan [[(75.204639,42.84519),(75.271198,42.845758),(75.496611,42.823925),(75.535988,42.827025),(75.555936,42.825217),(75.620428,42.805295),(75.646163,42.806096),(75.675308,42.815372),(75.703833,42.830488),(75.727605,42.848678),(75.73825,42.861571),(75.770599,42.92565),(75.782072,42.933324),(75.797161,42.936192),(75.807059,42.936771),(75.858966,42.939809),(75.898033,42.935701),(75.976582,42.918725),(75.999113,42.91764),(76.064535,42.93397),(76.09058,42.93428),(76.163754,42.921154),(76.254498,42.921154),(76.340487,42.901672),(76.351132,42.902886),(76.370976,42.909604),(76.382965,42.910069),(76.3933,42.905263),(76.404463,42.889244),(76.414074,42.885497),(76.432058,42.88958),(76.461927,42.910999),(76.48053,42.916968),(76.506885,42.914488),(76.558768,42.89821),(76.58502,42.894799),(76.605897,42.898494),(76.644345,42.911129),(76.688786,42.910328),(76.707907,42.913661),(76.74625,42.928879),(76.750385,42.932574),(76.751625,42.937096),(76.753899,42.940765),(76.76134,42.942057),(76.77953,42.939292),(76.785214,42.939654),(76.792966,42.943065),(76.814153,42.961487),(76.83348,42.971745),(76.853427,42.974122),(76.897662,42.974536),(76.93735,42.986034),(76.956884,42.988514),(76.976417,42.98221),(77.089899,42.968154),(77.124005,42.958904),(77.135064,42.951746),(77.163693,42.921981),(77.178679,42.913196),(77.195112,42.910069),(77.21189,42.909944),(77.229632,42.909811),(77.328127,42.897564),(77.36089,42.90454),(77.403265,42.919759),(77.418561,42.922239),(77.43396,42.921412),(77.461762,42.914384),(77.501553,42.914539),(77.520984,42.906917),(77.53845,42.896272),(77.557777,42.888133),(77.574417,42.887848),(77.62692,42.906555),(77.647797,42.909501),(77.71384,42.907589),(77.781226,42.895626),(77.787531,42.889709),(77.791665,42.883043),(77.798176,42.877513),(77.809958,42.871286),(77.83559,42.879839),(77.852229,42.887952),(77.861221,42.89082),(77.883855,42.893223),(77.907006,42.891285),(77.929331,42.885006),(77.986175,42.860202),(78.030616,42.854672),(78.137897,42.861984),(78.183579,42.86015),(78.229777,42.865007),(78.249104,42.862398),(78.290136,42.851442),(78.31122,42.850512),(78.328376,42.855292),(78.364653,42.872526),(78.38553,42.878237),(78.429559,42.880665),(78.496118,42.875601),(78.594303,42.850228),(78.635437,42.832477),(78.669957,42.811161),(78.687114,42.804598),(78.807933,42.79558),(78.888135,42.771215),(78.954281,42.768424),(78.992935,42.757133),(79.030866,42.756151),(79.108794,42.785348),(79.148274,42.790981),(79.173389,42.785632),(79.180831,42.77553),(79.175146,42.737031),(79.176076,42.713931),(79.181864,42.693597),(79.192199,42.674838),(79.206359,42.656467),(79.242119,42.629776),(79.321287,42.602181),(79.353016,42.577299),(79.398388,42.496942),(79.425673,42.469605),(79.47642,42.453973),(79.571918,42.449529),(79.652533,42.461053),(79.696458,42.459838),(79.917943,42.42444),(79.960111,42.403511),(79.97396,42.39147),(80.012098,42.349509),(80.077003,42.305765),(80.110076,42.273338),(80.136534,42.23887),(80.16661,42.208717),(80.210328,42.189519),(80.21653,42.174404),(80.219837,42.141873),(80.224591,42.125569),(80.235133,42.110945),(80.247432,42.098413),(80.256527,42.084357),(80.257561,42.065263),(80.231206,42.033689),(80.181906,42.020976),(79.930862,42.023276),(79.879496,42.013199),(79.842909,42.00183),(79.826786,41.992244),(79.812833,41.97762),(79.803842,41.959998),(79.792576,41.922817),(79.783274,41.905092),(79.747514,41.879745),(79.702969,41.874939),(79.655737,41.875843),(79.610985,41.867626),(79.554658,41.837602),(79.489856,41.819257),(79.410997,41.778588),(79.390844,41.772697),(79.367382,41.772387),(79.30413,41.787554),(79.282323,41.783497),(79.264856,41.774506),(79.217521,41.741226),(79.19592,41.729521),(79.174526,41.722622),(79.127914,41.714302),(79.088743,41.702546),(78.976192,41.6418),(78.915834,41.63317),(78.897437,41.626272),(78.807313,41.578445),(78.672128,41.538448),(78.658278,41.532453),(78.645049,41.52372),(78.637401,41.512868),(78.629133,41.48796),(78.619211,41.478089),(78.583968,41.465997),(78.51038,41.454422),(78.41757,41.400471),(78.377675,41.386622),(78.359899,41.377527),(78.343466,41.362024),(78.339332,41.344041),(78.356901,41.305542),(78.359692,41.287455),(78.34946,41.270402),(78.331477,41.258723),(78.291582,41.240791),(78.275356,41.228854),(78.250345,41.200535),(78.231335,41.172856),(78.204456,41.133718),(78.190503,41.11775),(78.176034,41.105502),(78.074955,41.039512),(78.057178,41.034344),(78.036508,41.036101),(77.997647,41.049795),(77.866699,41.064058),(77.831042,41.062973),(77.797556,41.054704),(77.665574,41.001271),(77.650278,40.997137),(77.631778,40.995793),(77.580721,40.997705),(77.503517,40.981066),(77.474888,40.982047),(77.445226,40.993675),(77.388795,41.011658),(77.332985,41.02065),(77.301152,41.019306),(77.243171,41.005664),(77.118838,41.011658),(77.088555,41.019565),(77.035018,41.040338),(77.007733,41.044214),(76.900143,41.025766),(76.860972,41.013208),(76.834927,40.99352),(76.820871,40.97781),(76.783974,40.957139),(76.766818,40.944685),(76.757413,40.925772),(76.760927,40.9066),(76.768368,40.887014),(76.770848,40.867119),(76.762477,40.847017),(76.746871,40.83446),(76.707286,40.817613),(76.674214,40.795444),(76.647859,40.764851),(76.630805,40.728885),(76.624191,40.627547),(76.62078,40.611321),(76.609411,40.59711),(76.556391,40.56569),(76.531173,40.534995),(76.498927,40.464611),(76.476499,40.436138),(76.449111,40.415519),(76.361984,40.371904),(76.330152,40.348081),(76.313512,40.343327),(76.299353,40.355677),(76.283333,40.417276),(76.273411,40.434122),(76.244162,40.441202),(76.21543,40.416552),(76.185044,40.384203),(76.151351,40.368131),(76.132541,40.371542),(76.095231,40.387355),(76.07611,40.391954),(76.051616,40.390197),(75.962422,40.357331),(75.949297,40.343068),(75.938031,40.326222),(75.921495,40.309117),(75.901858,40.298885),(75.880464,40.295268),(75.858553,40.296353),(75.79375,40.30891),(75.772046,40.31015),(75.750549,40.308342),(75.704454,40.293149),(75.681819,40.291702),(75.664869,40.305706),(75.656291,40.32307),(75.640168,40.367305),(75.638308,40.386115),(75.646369,40.405338),(75.659598,40.419084),(75.669314,40.432365),(75.666936,40.450245),(75.657221,40.461097),(75.629006,40.481303),(75.617947,40.493964),(75.610816,40.512877),(75.605648,40.569411),(75.587665,40.611941),(75.559863,40.63287),(75.523793,40.63318),(75.481832,40.614111),(75.26324,40.480166),(75.253318,40.47598),(75.241536,40.474068),(75.229237,40.470347),(75.223966,40.462441),(75.220142,40.452984),(75.212494,40.444716),(75.19389,40.441254),(75.175287,40.448127),(75.15658,40.458152),(75.13808,40.463888),(75.113172,40.460787),(75.064389,40.443837),(75.039068,40.441099),(75.002378,40.4473),(74.966101,40.459289),(74.879181,40.505074),(74.856443,40.513187),(74.835359,40.511637),(74.832312,40.508116),(74.820063,40.493964),(74.816032,40.48368),(74.815102,40.480114),(74.807041,40.461769),(74.794638,40.440737),(74.787404,40.420738),(74.794638,40.405545),(74.841664,40.372059),(74.853859,40.35883),(74.862128,40.32617),(74.830915,40.319917),(74.746683,40.336505),(74.724668,40.337539),(74.705755,40.331286),(74.654388,40.291547),(74.637645,40.281987),(74.598681,40.266174),(74.565402,40.24695),(74.476622,40.172433),(74.369858,40.105822),(74.333788,40.09373),(74.302885,40.090061),(74.272293,40.093833),(74.237876,40.10391),(74.20315,40.109853),(74.167596,40.10639),(74.13287,40.095332),(74.101244,40.078847),(74.069515,40.067788),(74.003989,40.060812),(73.976704,40.043603),(73.966368,40.033268),(73.957894,40.021434),(73.952106,40.008257),(73.949729,39.993684),(73.944044,39.970068),(73.927301,39.953325),(73.907561,39.937873),(73.893195,39.918133),(73.885443,39.876534),(73.880689,39.86501),(73.872937,39.856845),(73.844102,39.838086),(73.83201,39.82372),(73.823018,39.805685),(73.818574,39.786048),(73.819814,39.766721),(73.829529,39.746981),(73.843068,39.740366),(73.859915,39.737162),(73.879345,39.727912),(73.893505,39.710394),(73.899602,39.689465),(73.904253,39.64647),(73.926784,39.592882),(73.921617,39.582133),(73.899292,39.571539),(73.883066,39.561204),(73.870457,39.546786),(73.859192,39.524255),(73.848029,39.489735),(73.838314,39.475679),(73.820744,39.468186),(73.632642,39.448343),(73.604323,39.459608),(73.512236,39.467411),(73.476889,39.464776),(73.367645,39.443795),(73.343151,39.430669),(73.335503,39.415166),(73.333332,39.40111),(73.326408,39.390878),(73.269357,39.382558),(73.167864,39.355377),(73.136032,39.353465),(73.100995,39.361164),(73.083425,39.367831),(73.071126,39.370725),(73.058414,39.368813),(73.009321,39.348607),(72.994335,39.347832),(72.976765,39.352224),(72.91434,39.362715),(72.893669,39.363903),(72.850468,39.35672),(72.835275,39.356204),(72.65048,39.393772),(72.633737,39.394496),(72.616373,39.390413),(72.606141,39.383489),(72.587021,39.364885),(72.575756,39.359304),(72.559426,39.359562),(72.534104,39.372327),(72.519015,39.375686),(72.50806,39.371965),(72.476744,39.346127),(72.460414,39.344111),(72.443567,39.344783),(72.410701,39.351346),(72.393234,39.350829),(72.355924,39.336101),(72.334117,39.333828),(72.31634,39.328815),(72.304248,39.31357),(72.281407,39.259724),(72.240272,39.189909),(72.228903,39.189237),(72.228668,39.189502),(72.218568,39.20089),(72.209163,39.217297),(72.206476,39.226883),(72.206373,39.234609),(72.203272,39.24081),(72.182395,39.249337),(72.167098,39.258742),(72.158623,39.262617),(72.115525,39.268147),(72.094751,39.275072),(72.08576,39.290109),(72.084106,39.311245),(72.078112,39.336618),(72.06695,39.358477),(72.049793,39.368864),(72.031499,39.366694),(71.996463,39.351863),(71.959152,39.345868),(71.942306,39.339047),(71.843501,39.285045),(71.807017,39.272849),(71.77043,39.269542),(71.751207,39.274245),(71.732913,39.285097),(71.71834,39.301065),(71.710899,39.32096),(71.712966,39.341527),(71.723198,39.353981),(71.736634,39.364575),(71.748106,39.379561),(71.751827,39.399663),(71.749243,39.425502),(71.740251,39.448343),(71.724438,39.459401),(71.704594,39.458988),(71.602792,39.442348),(71.554319,39.444157),(71.512875,39.458833),(71.492101,39.493663),(71.500576,39.509217),(71.526001,39.538156),(71.531375,39.553091),(71.52538,39.569162),(71.510911,39.584148),(71.509654,39.584998),(71.493031,39.596241),(71.459545,39.612105),(71.441251,39.610658),(71.406731,39.598153),(71.378309,39.594173),(71.368543,39.591486),(71.361618,39.587559),(71.347872,39.576242),(71.340586,39.571694),(71.305911,39.557173),(71.291958,39.548957),(71.260074,39.520638),(71.249273,39.514385),(71.236923,39.514488),(71.179975,39.523015),(71.137446,39.521155),(71.095898,39.51237),(71.062928,39.495627),(71.042309,39.467876),(71.027581,39.435217),(71.009288,39.407622),(70.977145,39.394909),(70.949808,39.400749),(70.925159,39.412944),(70.903351,39.420851),(70.884489,39.413874),(70.872242,39.402971),(70.864181,39.400594),(70.855292,39.40173),(70.840099,39.40142),(70.828059,39.398578),(70.797621,39.385349),(70.770129,39.382093),(70.739744,39.386228),(70.713285,39.398423),(70.698092,39.41899),(70.698609,39.431393),(70.703053,39.444725),(70.705431,39.458523),(70.699746,39.472165),(70.690134,39.479193),(70.666725,39.486893),(70.655976,39.493663),(70.655976,39.493714),(70.655925,39.49387),(70.655598,39.494664),(70.635977,39.542497),(70.622128,39.563529),(70.59784,39.577792),(70.58027,39.579549),(70.54358,39.574278),(70.526113,39.575622),(70.513504,39.581513),(70.490973,39.596861),(70.477227,39.601201),(70.459967,39.599548),(70.427618,39.590091),(70.411082,39.587456),(70.40147,39.584768),(70.395372,39.579497),(70.390411,39.57402),(70.384313,39.570867),(70.374288,39.571022),(70.353462,39.575932),(70.343179,39.576862),(70.33026,39.573089),(70.241066,39.522395),(70.223289,39.519088),(70.20603,39.5241),(70.204376,39.538156),(70.215021,39.574071),(70.214814,39.591125),(70.210267,39.609728),(70.200655,39.619495),(70.185204,39.610348),(70.15735,39.563891),(70.148979,39.554279),(70.132442,39.550042),(70.116268,39.554641),(70.08273,39.569782),(70.062266,39.573451),(70.040355,39.573606),(70.019684,39.568594),(70.002734,39.556863),(69.987128,39.539603),(69.978757,39.534436),(69.948629,39.545081),(69.907753,39.548492),(69.830445,39.536244),(69.791585,39.545391),(69.74983,39.563839),(69.710349,39.574071),(69.669421,39.577792),(69.582295,39.573555),(69.564932,39.568129),(69.531445,39.545649),(69.514392,39.537588),(69.496099,39.532575),(69.477288,39.530405),(69.455274,39.530456),(69.412176,39.52503),(69.391454,39.530508),(69.367218,39.549938),(69.361637,39.552884),(69.352025,39.549628),(69.348407,39.543169),(69.345927,39.535366),(69.339726,39.528079),(69.300348,39.515625),(69.286189,39.539707),(69.29115,39.658872),(69.287533,39.677786),(69.280091,39.694271),(69.265932,39.707758),(69.248362,39.719385),(69.233324,39.732666),(69.226296,39.751011),(69.229552,39.790544),(69.240714,39.82894),(69.286447,39.935755),(69.305671,39.968621),(69.310064,39.978646),(69.310322,39.984796),(69.313991,39.986914),(69.32784,39.984744),(69.357296,39.959474),(69.405355,39.896119),(69.501576,39.922474),(69.500439,39.935703),(69.477288,39.968156),(69.478115,39.975029),(69.477288,39.981592),(69.475118,39.987741),(69.471191,39.993684),(69.463129,40.025413),(69.46964,40.050993),(69.485712,40.073731),(69.506331,40.096933),(69.509224,40.103445),(69.513462,40.120705),(69.518009,40.125097),(69.526071,40.123185),(69.536613,40.107889),(69.543021,40.103083),(69.558834,40.101739),(69.575474,40.1036),(69.969558,40.211603),(70.004595,40.208761),(70.14717,40.136983),(70.168822,40.13166),(70.218535,40.134141),(70.241531,40.132745),(70.263494,40.12427),(70.277808,40.112075),(70.290262,40.098174),(70.306282,40.08479),(70.324265,40.077503),(70.359509,40.074041),(70.397646,40.061225),(70.477227,40.052027),(70.502135,40.045981),(70.525493,40.033682),(70.535622,40.015957),(70.520946,39.993736),(70.505029,39.985106),(70.488493,39.978543),(70.473197,39.97012),(70.460226,39.956115),(70.450872,39.937408),(70.44803,39.919993),(70.455627,39.906816),(70.477227,39.900925),(70.485961,39.909606),(70.494177,39.931672),(70.500895,39.940561),(70.512574,39.94516),(70.555052,39.946245),(70.576601,39.952239),(70.5966,39.961955),(70.613963,39.975752),(70.627399,39.993891),(70.635977,40.028514),(70.634737,40.059313),(70.63944,40.084945),(70.66595,40.104168),(70.729822,40.120705),(70.743464,40.127113),(70.782532,40.152589),(70.788733,40.158997),(70.792144,40.161374),(70.797725,40.161942),(70.802686,40.160134),(70.806045,40.157653),(70.806716,40.156155),(70.824906,40.163544),(70.831831,40.168195),(70.845164,40.171813),(70.898545,40.162614),(70.929293,40.170624),(70.962883,40.189693),(70.979522,40.214084),(70.958955,40.238372),(70.995129,40.266587),(71.053213,40.274235),(71.169382,40.26142),(71.201008,40.263848),(71.215115,40.280747),(71.223952,40.302864),(71.239558,40.321106),(71.253821,40.324413),(71.263949,40.318212),(71.272838,40.307722),(71.283225,40.29842),(71.297074,40.293821),(71.313559,40.292684),(71.344926,40.295216),(71.365132,40.294131),(71.396551,40.271548),(71.441355,40.260903),(71.450966,40.248914),(71.458925,40.234134),(71.477115,40.220802),(71.498405,40.210518),(71.521246,40.203749),(71.568272,40.196566),(71.592973,40.198426),(71.601655,40.20995),(71.604032,40.227261),(71.61013,40.246382),(71.62832,40.258887),(71.645993,40.247105),(71.660153,40.224471),(71.667181,40.204162),(71.666871,40.163131),(71.673072,40.147886),(71.693329,40.141117),(71.707178,40.144269),(71.759785,40.168092),(71.775908,40.179926),(71.78676,40.193517),(71.805467,40.225298),(71.836473,40.249172),(71.872956,40.250774),(71.912334,40.243436),(71.951814,40.240904),(71.96339,40.243746),(71.969178,40.244418),(71.977033,40.243074),(71.989021,40.239302),(72.004834,40.237287),(72.01889,40.240026),(72.025402,40.250878),(72.019821,40.258732),(72.013239,40.262072),(72.005971,40.26576),(71.977033,40.276096),(71.958222,40.286534),(71.951091,40.30152),(71.951094,40.301529),(71.956672,40.31568),(72.043178,40.349321),(72.069637,40.369423),(72.084106,40.39738),(72.090101,40.416035),(72.099299,40.426371),(72.165858,40.454431),(72.182705,40.45779),(72.228328,40.459606),(72.235931,40.459909),(72.254225,40.458307),(72.263415,40.452651),(72.26363,40.452519),(72.259599,40.44239),(72.24513,40.438721),(72.228077,40.437533),(72.216501,40.434949),(72.211768,40.425836),(72.211644,40.425596),(72.224356,40.422443),(72.269624,40.424045),(72.284301,40.41986),(72.343418,40.393401),(72.370497,40.38565),(72.394061,40.389422),(72.414422,40.410713),(72.425171,40.435983),(72.426204,40.45934),(72.426096,40.459545),(72.420028,40.471027),(72.415662,40.479287),(72.415585,40.47933),(72.372254,40.503265),(72.363469,40.512309),(72.363467,40.51241),(72.363262,40.523574),(72.36998,40.539852),(72.370083,40.557732),(72.369928,40.557931),(72.348483,40.585379),(72.348509,40.585508),(72.351893,40.601967),(72.381556,40.612148),(72.414835,40.589875),(72.447908,40.560471),(72.476744,40.549567),(72.515294,40.54564),(72.585781,40.508743),(72.625468,40.5105),(72.640868,40.519853),(72.65048,40.532152),(72.655131,40.546363),(72.656474,40.561143),(72.664226,40.577783),(72.682416,40.577679),(72.719209,40.564864),(72.748355,40.575096),(72.760034,40.641758),(72.783908,40.669663),(72.818945,40.681084),(72.890982,40.695088),(72.976765,40.736068),(73.070609,40.762474),(73.118048,40.782938),(73.148641,40.813686),(73.14337,40.833839),(73.143292,40.833853),(73.135042,40.835274),(73.112467,40.839162),(73.053556,40.83632),(73.033299,40.847224),(73.01945,40.8619),(73.00343,40.870168),(72.929429,40.844175),(72.883127,40.819628),(72.870311,40.818181),(72.872998,40.834821),(72.868658,40.864122),(72.830107,40.87208),(72.701226,40.863243),(72.658231,40.867171),(72.619474,40.88009),(72.588468,40.905825),(72.545577,40.956519),(72.52625,40.962204),(72.501445,40.963496),(72.483358,40.970575),(72.483565,40.99352),(72.485219,40.999566),(72.484702,41.004682),(72.481911,41.008816),(72.476744,41.011813),(72.423517,41.01574),(72.395198,41.022045),(72.374321,41.031967),(72.345692,41.06597),(72.33236,41.072843),(72.314066,41.061681),(72.308795,41.054446),(72.297323,41.028143),(72.289778,41.023544),(72.252985,41.019616),(72.195486,41.006358),(72.165135,40.999359),(72.178777,41.023182),(72.185599,41.041062),(72.185599,41.060647),(72.1764,41.112892),(72.174643,41.141418),(72.169889,41.168651),(72.158417,41.187875),(72.132889,41.199295),(72.108497,41.196401),(72.085243,41.184877),(72.063642,41.170201),(72.033773,41.15661),(72.016513,41.163535),(72.001217,41.18002),(71.977033,41.195213),(71.897658,41.184929),(71.871613,41.194489),(71.866342,41.236606),(71.868822,41.279239),(71.863034,41.312208),(71.847015,41.341819),(71.75317,41.44729),(71.745212,41.45282),(71.73684,41.455404),(71.730433,41.451166),(71.729709,41.430082),(71.721648,41.424759),(71.712242,41.428015),(71.706868,41.444086),(71.696119,41.447445),(71.691469,41.441916),(71.687334,41.431012),(71.681443,41.422847),(71.671935,41.425483),(71.671108,41.437627),(71.689505,41.493799),(71.689401,41.514625),(71.68413,41.534055),(71.671418,41.547388),(71.649507,41.54992),(71.62739,41.543202),(71.615297,41.532143),(71.595454,41.493799),(71.595454,41.493747),(71.595557,41.493696),(71.595557,41.493489),(71.605582,41.476849),(71.633694,41.449616),(71.637415,41.431271),(71.633074,41.411324),(71.618811,41.377786),(71.585532,41.323525),(71.557937,41.301718),(71.52383,41.296654),(71.480629,41.310761),(71.432466,41.344816),(71.418772,41.3474),(71.412571,41.334687),(71.421459,41.162088),(71.416085,41.127362),(71.393657,41.112737),(71.325238,41.157334),(71.300226,41.133046),(71.289478,41.113874),(71.276145,41.113151),(71.263381,41.123486),(71.253821,41.13749),(71.241522,41.175162),(71.230153,41.187255),(71.206382,41.188753),(71.185711,41.180227),(71.183024,41.166377),(71.187882,41.148446),(71.189949,41.127465),(71.18013,41.108138),(71.164421,41.116199),(71.139099,41.148394),(71.123079,41.158212),(71.085769,41.162036),(71.067579,41.169736),(71.047012,41.182035),(71.024688,41.189787),(70.977145,41.19635),(70.93415,41.19144),(70.914203,41.193042),(70.895962,41.206116),(70.882164,41.220482),(70.865731,41.233195),(70.847747,41.243065),(70.828524,41.249111),(70.805993,41.247458),(70.786563,41.24043),(70.770439,41.238518),(70.758037,41.25216),(70.75628,41.269627),(70.77106,41.331018),(70.769199,41.352102),(70.759691,41.372515),(70.70357,41.445482),(70.686724,41.462483),(70.66781,41.471372),(70.633807,41.467496),(70.511334,41.414476),(70.477227,41.404657),(70.470782,41.404879),(70.453198,41.405484),(70.438057,41.416078),(70.413614,41.450701),(70.398989,41.464964),(70.382246,41.476436),(70.344936,41.493489),(70.344729,41.493644),(70.344522,41.493696),(70.344419,41.493799),(70.203446,41.505633),(70.166549,41.520206),(70.148255,41.552452),(70.169288,41.578342),(70.33119,41.649629),(70.390824,41.685054),(70.423484,41.696913),(70.453663,41.71208),(70.477227,41.738435),(70.506476,41.78559),(70.550091,41.824063),(70.648948,41.887393),(70.679696,41.901113),(70.779328,41.909665),(70.814003,41.919535),(70.825113,41.93633),(70.828059,41.993743),(70.845474,42.030356),(70.886711,42.038495),(70.935907,42.036867),(70.977145,42.044231),(71.118429,42.122908),(71.201008,42.140788),(71.238215,42.160244),(71.253201,42.197555),(71.249868,42.198385),(71.217854,42.206365),(71.077604,42.281167),(71.045772,42.29096),(71.014042,42.287704),(70.9696,42.263468),(70.947793,42.248146),(70.918648,42.253365),(70.897822,42.261608),(70.858134,42.291063),(70.852657,42.306075),(70.864801,42.321552),(70.888825,42.338575),(70.900354,42.346744),(70.932703,42.3762),(70.939835,42.387827),(70.937354,42.394778),(70.931773,42.401496),(70.929913,42.412296),(70.936114,42.431623),(70.947586,42.451286),(70.961952,42.468107),(70.977145,42.478752),(71.024739,42.455963),(71.041017,42.4548),(71.054712,42.460381),(71.064168,42.470329),(71.066287,42.482137),(71.057864,42.493402),(71.057657,42.493402),(71.057554,42.493506),(71.057554,42.493609),(71.022724,42.51645),(71.012699,42.526036),(71.00319,42.563527),(71.040656,42.580529),(71.127007,42.590606),(71.142768,42.602543),(71.148091,42.6174),(71.146437,42.656855),(71.149021,42.677939),(71.157548,42.687938),(71.19522,42.69724),(71.211446,42.705224),(71.223849,42.717936),(71.245449,42.747133),(71.262502,42.753567),(71.284723,42.7484),(71.308495,42.740105),(71.329785,42.737031),(71.347562,42.742844),(71.363065,42.75411),(71.376294,42.768321),(71.386939,42.783074),(71.404613,42.794883),(71.428849,42.79558),(71.477115,42.78354),(71.493651,42.788914),(71.504193,42.789612),(71.514322,42.78571),(71.553803,42.760905),(71.566308,42.757236),(71.583568,42.759639),(71.693536,42.811807),(71.726402,42.819661),(71.796682,42.822297),(71.831408,42.831573),(71.847738,42.834053),(71.863241,42.829041),(71.878847,42.821341),(71.895384,42.816044),(71.956982,42.804598),(72.072737,42.757185),(72.10488,42.750337),(72.11997,42.75181),(72.148805,42.76199),(72.164101,42.765091),(72.292362,42.76106),(72.358405,42.741397),(72.476744,42.682564),(72.51178,42.677551),(72.583817,42.678275),(72.725824,42.652876),(72.756933,42.640215),(72.780601,42.620449),(72.815224,42.573036),(72.840752,42.555698),(72.874549,42.543993),(72.908345,42.536965),(72.942348,42.536035),(73.070609,42.551926),(73.115051,42.550841),(73.152671,42.539213),(73.172722,42.527845),(73.189052,42.520765),(73.206828,42.517173),(73.278348,42.513401),(73.301396,42.507071),(73.316589,42.493712),(73.313385,42.463171),(73.314419,42.441571),(73.326201,42.428677),(73.417461,42.41705),(73.476889,42.399067),(73.505105,42.402943),(73.505518,42.420926),(73.505077,42.421557),(73.432964,42.524847),(73.417565,42.556163),(73.41002,42.58965),(73.412707,42.627322),(73.423869,42.662772),(73.476889,42.75088),(73.503554,42.794082),(73.507068,42.809404),(73.504278,42.827594),(73.491979,42.861003),(73.489498,42.877823),(73.493012,42.894954),(73.501177,42.909346),(73.521744,42.936166),(73.559675,43.017298),(73.57094,43.031922),(73.58665,43.042283),(73.634089,43.062463),(73.805551,43.114734),(73.822811,43.117318),(73.864256,43.116103),(73.90167,43.13096),(73.935053,43.199716),(73.965335,43.216924),(73.985385,43.211679),(74.021456,43.186306),(74.039852,43.181861),(74.197362,43.195788),(74.213899,43.202635),(74.216586,43.216924),(74.206044,43.234106),(74.178759,43.261702),(74.207697,43.24979),(74.25865,43.215761),(74.286969,43.207209),(74.320042,43.201654),(74.348774,43.191034),(74.400554,43.159537),(74.420604,43.151837),(74.462772,43.148634),(74.498222,43.131115),(74.539253,43.122899),(74.558167,43.115044),(74.572016,43.102202),(74.597028,43.070344),(74.61129,43.058174),(74.71392,42.999883),(74.74875,42.990013),(74.862851,42.975828),(74.948531,42.944848),(74.976849,42.926477),(75.005375,42.9164),(75.066146,42.902602),(75.094258,42.890303),(75.118133,42.876531),(75.178594,42.84966),(75.204639,42.84519)],[(71.008564,40.157757),(70.977145,40.144579),(70.959885,40.113418),(70.954046,40.095797),(70.952961,40.079208),(70.958438,40.063085),(70.98314,40.021383),(70.994509,40.008877),(71.007634,40.003503),(71.034299,39.99911),(71.045823,39.99203),(71.050371,39.962885),(71.007634,39.911157),(71.009288,39.885732),(71.021897,39.880823),(71.036056,39.887541),(71.049802,39.897979),(71.060964,39.904129),(71.079981,39.903405),(71.084787,39.894827),(71.087216,39.88382),(71.098895,39.8755),(71.113674,39.874466),(71.16101,39.884233),(71.1946,39.884802),(71.208449,39.888781),(71.219094,39.900925),(71.221058,39.931724),(71.17703,39.968156),(71.174704,39.993994),(71.190982,40.006241),(71.237284,40.031098),(71.244002,40.046756),(71.236354,40.056057),(71.223745,40.057866),(71.198217,40.052595),(71.192946,40.05027),(71.181887,40.043397),(71.176926,40.042001),(71.169382,40.042828),(71.165454,40.044637),(71.162043,40.046911),(71.105975,40.064946),(71.078844,40.07864),(71.06174,40.095176),(71.047942,40.122513),(71.031199,40.146905),(71.008564,40.157757)],[(71.741801,39.90077),(71.757304,39.903095),(71.767123,39.915446),(71.789757,39.979318),(71.78552,39.989705),(71.760612,39.98371),(71.724128,39.962678),(71.706868,39.956115),(71.681753,39.955082),(71.665527,39.940199),(71.675346,39.925781),(71.696843,39.913844),(71.71617,39.906764),(71.741801,39.90077)],[(70.498725,39.881908),(70.483739,39.882218),(70.482602,39.866767),(70.490147,39.850179),(70.503479,39.835502),(70.537379,39.817312),(70.547301,39.807649),(70.575878,39.77008),(70.581614,39.766566),(70.63298,39.79845),(70.661609,39.809819),(70.694527,39.814832),(70.70481,39.822067),(70.706412,39.839998),(70.698919,39.858447),(70.68662,39.860876),(70.654994,39.849765),(70.619906,39.850695),(70.498725,39.881908)]] diff --git a/tests/queries/0_stateless/country_rings.tsv b/tests/queries/0_stateless/country_rings.tsv new file mode 100644 index 00000000000..15ab66ec0c0 --- /dev/null +++ b/tests/queries/0_stateless/country_rings.tsv @@ -0,0 +1,103 @@ +Aruba [(-69.996938,12.577582),(-69.936391,12.531724),(-69.924672,12.519232),(-69.915761,12.497016),(-69.880198,12.453559),(-69.87682,12.427395),(-69.888092,12.41767),(-69.908803,12.417792),(-69.930531,12.425971),(-69.945139,12.440375),(-69.924672,12.440375),(-69.924672,12.447211),(-69.958567,12.463202),(-70.027659,12.522935),(-70.048085,12.531155),(-70.058095,12.537177),(-70.062408,12.54682),(-70.060374,12.556952),(-70.051096,12.574042),(-70.048736,12.583726),(-70.052642,12.600002),(-70.059641,12.614244),(-70.061106,12.625393),(-70.048736,12.632148),(-70.007151,12.585517),(-69.996938,12.577582)] +Afghanistan [(71.049802,38.408664),(71.05714,38.409026),(71.064943,38.411817),(71.076984,38.412178),(71.089386,38.409853),(71.117395,38.398639),(71.155946,38.376212),(71.217699,38.325827),(71.300123,38.298697),(71.334384,38.280662),(71.358156,38.251258),(71.364512,38.206816),(71.359086,38.184105),(71.340947,38.140929),(71.334539,38.11168),(71.316039,38.083284),(71.302397,38.04233),(71.272631,37.997992),(71.265913,37.972541),(71.255785,37.949881),(71.254544,37.939287),(71.258265,37.926472),(71.263536,37.924353),(71.271184,37.926187),(71.281829,37.924999),(71.31976,37.900582),(71.341154,37.893295),(71.360998,37.902029),(71.379188,37.912932),(71.501196,37.946212),(71.537059,37.944455),(71.567342,37.928074),(71.597727,37.89836),(71.590286,37.891435),(71.593387,37.879343),(71.59504,37.857535),(71.594214,37.833764),(71.590286,37.815729),(71.57468,37.798004),(71.537783,37.779039),(71.529515,37.761133),(71.531168,37.751754),(71.540263,37.730515),(71.542537,37.719559),(71.540883,37.709715),(71.529515,37.67858),(71.52476,37.647729),(71.522073,37.637626),(71.517216,37.629229),(71.505123,37.616),(71.501506,37.610341),(71.497165,37.566545),(71.511221,37.485878),(71.501506,37.445777),(71.490241,37.423376),(71.48714,37.409087),(71.494685,37.370692),(71.496545,37.328524),(71.493858,37.307543),(71.487863,37.294986),(71.487037,37.267055),(71.45014,37.21667),(71.453757,37.192615),(71.446832,37.183597),(71.441045,37.16843),(71.43319,37.127347),(71.431123,37.066989),(71.43319,37.054742),(71.439184,37.044097),(71.456444,37.022548),(71.459958,37.010739),(71.459958,36.969786),(71.463059,36.948082),(71.47174,36.930047),(71.528481,36.856149),(71.538506,36.836099),(71.545328,36.790004),(71.552872,36.769591),(71.563931,36.750678),(71.577367,36.733263),(71.61106,36.704841),(71.653021,36.687012),(71.699943,36.678641),(71.748519,36.678641),(71.797612,36.686082),(71.836576,36.699156),(72.126687,36.872738),(72.152836,36.895708),(72.186839,36.911392),(72.195934,36.918988),(72.210403,36.93661),(72.220325,36.94555),(72.259806,36.967305),(72.360988,37.000275),(72.405947,37.007665),(72.474987,36.997484),(72.50868,37.011075),(72.657714,37.028826),(72.666603,37.038335),(72.672701,37.05761),(72.714352,37.109984),(72.760964,37.187499),(72.79073,37.220262),(72.83052,37.239769),(72.877443,37.246875),(72.902247,37.2538),(72.924262,37.274832),(72.995575,37.3093),(73.067509,37.315062),(73.087766,37.326069),(73.099238,37.339867),(73.116394,37.369064),(73.132104,37.384386),(73.170241,37.40826),(73.179026,37.410741),(73.200627,37.404178),(73.211479,37.40826),(73.260262,37.450041),(73.276075,37.459472),(73.296332,37.464949),(73.32124,37.467017),(73.345011,37.464484),(73.361858,37.456397),(73.378394,37.452547),(73.440612,37.479936),(73.485261,37.480969),(73.605047,37.445777),(73.6745,37.43105),(73.717805,37.431825),(73.753462,37.428388),(73.747054,37.403119),(73.745917,37.394902),(73.74571,37.352863),(73.739096,37.338342),(73.722249,37.322452),(73.704989,37.310954),(73.690416,37.305218),(73.651969,37.302065),(73.630988,37.295993),(73.609904,37.281395),(73.597295,37.261809),(73.601636,37.24088),(73.617656,37.233181),(73.659927,37.243748),(73.680494,37.242457),(73.688039,37.236901),(73.702405,37.221166),(73.709227,37.217006),(73.719975,37.217549),(73.736202,37.227651),(73.746123,37.230545),(73.768654,37.228892),(73.783951,37.225894),(73.798213,37.22853),(73.836247,37.256745),(73.856401,37.261577),(73.899292,37.265478),(73.95562,37.286769),(73.976704,37.290283),(74.052151,37.312246),(74.163256,37.330074),(74.187647,37.338394),(74.206044,37.355654),(74.207697,37.389915),(74.223924,37.403351),(74.250382,37.403687),(74.279424,37.397512),(74.303816,37.400173),(74.315598,37.426916),(74.320869,37.413893),(74.332444,37.420353),(74.343813,37.420999),(74.353632,37.415909),(74.368204,37.396375),(74.378126,37.393765),(74.389392,37.393403),(74.418331,37.389192),(74.435901,37.392344),(74.454814,37.393636),(74.476622,37.386091),(74.512692,37.377229),(74.521167,37.375601),(74.529848,37.375756),(74.631444,37.381079),(74.660383,37.393972),(74.78854,37.331159),(74.816859,37.306923),(74.862644,37.244601),(74.892307,37.231114),(74.813138,37.21543),(74.794018,37.213931),(74.783269,37.219667),(74.743995,37.288397),(74.737381,37.296123),(74.721464,37.297776),(74.709372,37.290826),(74.698934,37.28031),(74.676299,37.263721),(74.670925,37.261344),(74.650978,37.259639),(74.646327,37.254833),(74.64302,37.248374),(74.629997,37.238297),(74.627207,37.234318),(74.623279,37.230804),(74.615011,37.228375),(74.609017,37.230183),(74.596511,37.240751),(74.59,37.243361),(74.575944,37.242147),(74.532225,37.232199),(74.500083,37.23163),(74.487267,37.225946),(74.476622,37.210055),(74.46763,37.189927),(74.456674,37.177318),(74.441585,37.170549),(74.420294,37.168068),(74.38226,37.172047),(74.368308,37.167061),(74.366137,37.14776),(74.38257,37.126572),(74.414403,37.107839),(74.476622,37.083138),(74.494088,37.066472),(74.505767,37.047042),(74.519203,37.030247),(74.542354,37.021669),(74.547418,37.015675),(74.549279,37.008931),(74.548142,37.001644),(74.544214,36.994022),(74.537393,36.962241),(74.521373,36.958495),(74.501736,36.972421),(74.48365,36.99397),(74.480342,36.996916),(74.476622,36.999319),(74.456984,37.004383),(74.43559,37.003221),(74.394043,36.994022),(74.368101,36.976762),(74.284592,36.934232),(74.235706,36.902167),(74.211728,36.895139),(74.145996,36.901702),(74.129666,36.898421),(74.1153,36.889532),(74.108789,36.875631),(74.103931,36.841318),(74.094319,36.831241),(74.035305,36.815583),(74.006366,36.815738),(73.976704,36.824833),(73.946938,36.83088),(73.865083,36.872582),(73.834077,36.882866),(73.772892,36.892013),(73.711842,36.894139),(73.711335,36.894157),(73.711128,36.894164),(73.709483,36.894221),(73.640083,36.896638),(73.509342,36.87868),(73.476889,36.882866),(73.445883,36.886483),(73.379738,36.879249),(73.331782,36.882091),(73.281862,36.867983),(73.267496,36.866536),(73.252717,36.868035),(73.223468,36.874339),(73.191842,36.877027),(73.042497,36.864263),(73.026271,36.859457),(73.002293,36.846124),(72.990511,36.841577),(72.976765,36.842093),(72.945862,36.852222),(72.921058,36.847364),(72.896873,36.836977),(72.867727,36.830415),(72.779361,36.826797),(72.695852,36.836719),(72.629602,36.832947),(72.565214,36.820596),(72.516741,36.800597),(72.454109,36.757964),(72.433129,36.753417),(72.387343,36.755794),(72.346209,36.74489),(72.301974,36.74272),(72.213607,36.726442),(72.169682,36.711352),(72.153456,36.702102),(72.149632,36.689338),(72.164618,36.670063),(72.171646,36.653629),(72.154179,36.645465),(72.096095,36.638902),(72.070877,36.632287),(72.059095,36.627119),(72.050413,36.618645),(72.05217,36.610376),(72.056304,36.601901),(72.054961,36.592858),(72.038838,36.580456),(71.997806,36.572446),(71.977033,36.563041),(71.960909,36.549605),(71.913471,36.527591),(71.899725,36.518341),(71.887632,36.508005),(71.874713,36.499065),(71.85859,36.494156),(71.793788,36.490745),(71.773221,36.480048),(71.774564,36.448422),(71.791514,36.421292),(71.794408,36.407598),(71.782626,36.396539),(71.766916,36.391837),(71.751,36.391113),(71.736324,36.395919),(71.706868,36.421447),(71.649301,36.452763),(71.629147,36.459533),(71.61044,36.457931),(71.600725,36.449301),(71.588736,36.41845),(71.580364,36.402327),(71.572303,36.391682),(71.547188,36.371631),(71.542227,36.356387),(71.552666,36.340987),(71.55835,36.327861),(71.53892,36.319283),(71.514632,36.315201),(71.495822,36.309619),(71.479078,36.300524),(71.402752,36.231381),(71.382443,36.218566),(71.31914,36.200582),(71.313972,36.194019),(71.309011,36.172729),(71.302087,36.163323),(71.292682,36.157897),(71.262296,36.146115),(71.223073,36.125393),(71.217699,36.118055),(71.212635,36.096816),(71.207622,36.087617),(71.175376,36.061159),(71.165919,36.045708),(71.169913,36.030802),(71.170932,36.027001),(71.181887,36.018836),(71.217854,36.002713),(71.23191,35.99398),(71.257438,35.971552),(71.28431,35.962457),(71.312525,35.957393),(71.341567,35.947264),(71.356399,35.933053),(71.360688,35.90029),(71.371333,35.885149),(71.416705,35.858846),(71.431019,35.843705),(71.464402,35.794715),(71.470914,35.779936),(71.47205,35.769962),(71.471017,35.761333),(71.47112,35.752289),(71.475564,35.741024),(71.481352,35.734099),(71.496132,35.724591),(71.502746,35.719061),(71.515355,35.701491),(71.519799,35.683766),(71.514012,35.665576),(71.495925,35.646611),(71.483316,35.62656),(71.492721,35.609559),(71.512771,35.596278),(71.567859,35.574212),(71.584188,35.564187),(71.59349,35.549356),(71.59287,35.530184),(71.586049,35.512097),(71.581708,35.493132),(71.587806,35.470911),(71.593422,35.464194),(71.600725,35.45546),(71.613954,35.443212),(71.622429,35.429621),(71.620879,35.410139),(71.61106,35.395463),(71.562277,35.360582),(71.531582,35.327922),(71.529825,35.300895),(71.547808,35.275626),(71.603205,35.223381),(71.633694,35.203124),(71.637932,35.189688),(71.62925,35.170102),(71.604135,35.138166),(71.534992,35.098582),(71.508947,35.072021),(71.5075,35.028199),(71.512978,35.018122),(71.511325,35.008717),(71.504297,35.000604),(71.493444,34.994196),(71.493444,34.994144),(71.493341,34.994144),(71.493341,34.994041),(71.485383,34.983499),(71.476701,34.960296),(71.46957,34.949702),(71.459958,34.942674),(71.445239,34.937757),(71.324928,34.897561),(71.289478,34.87503),(71.270822,34.844283),(71.25563,34.810228),(71.203075,34.748164),(71.189535,34.737054),(71.150933,34.720311),(71.080601,34.672923),(71.070163,34.661245),(71.068768,34.645948),(71.076261,34.623624),(71.079878,34.602902),(71.076571,34.578511),(71.065822,34.55846),(71.047529,34.551122),(71.006963,34.556341),(70.985827,34.556186),(70.983356,34.555256),(70.96867,34.549727),(70.956991,34.532002),(70.955648,34.510014),(70.961229,34.487612),(70.970841,34.468879),(70.991511,34.443067),(71.01983,34.414438),(71.050939,34.389788),(71.122046,34.356819),(71.126387,34.332221),(71.097086,34.262458),(71.09688,34.244345),(71.106543,34.209386),(71.109489,34.189206),(71.108093,34.165228),(71.101789,34.151818),(71.073884,34.125257),(71.062773,34.10531),(71.063858,34.088928),(71.067269,34.073012),(71.062618,34.054253),(71.046702,34.041877),(70.997712,34.033324),(70.977145,34.027407),(70.965725,34.013816),(70.953994,34.004825),(70.939731,34.000923),(70.921231,34.002241),(70.894308,34.009372),(70.884334,34.007047),(70.879994,33.994257),(70.862217,33.964775),(70.791317,33.953562),(70.656183,33.95475),(70.521979,33.938705),(70.49149,33.939609),(70.408963,33.954414),(70.328193,33.957282),(70.309899,33.961675),(70.275586,33.97617),(70.218845,33.980692),(70.002838,34.043789),(69.969093,34.045727),(69.916021,34.038879),(69.889253,34.031206),(69.872613,34.017227),(69.870443,34.001337),(69.873853,33.986325),(69.871993,33.971519),(69.854216,33.956352),(69.841091,33.941805),(69.847188,33.926948),(69.876489,33.902557),(69.885429,33.889457),(69.898761,33.851501),(69.907546,33.83561),(69.931318,33.806439),(69.940103,33.791479),(69.947234,33.771971),(69.957518,33.752695),(69.972659,33.744841),(69.996478,33.742091),(70.011829,33.740319),(70.065573,33.721018),(70.108154,33.727296),(70.118128,33.716522),(70.126551,33.676964),(70.132546,33.661435),(70.136886,33.656293),(70.145981,33.649885),(70.162518,33.643245),(70.170218,33.638671),(70.174093,33.632108),(70.173473,33.607872),(70.153009,33.544749),(70.151356,33.525862),(70.154973,33.506612),(70.163758,33.488655),(70.177607,33.47341),(70.185462,33.468966),(70.213368,33.461344),(70.220447,33.456021),(70.227837,33.43995),(70.233005,33.432612),(70.285715,33.382873),(70.301579,33.35179),(70.294448,33.318949),(70.124794,33.199034),(70.105364,33.18981),(70.08366,33.191283),(70.059785,33.198052),(70.048003,33.194073),(70.014517,33.140769),(70.006869,33.131803),(69.994725,33.127333),(69.965734,33.1294),(69.955812,33.127695),(69.880933,33.089248),(69.838713,33.086664),(69.771844,33.114776),(69.732983,33.109298),(69.68601,33.080773),(69.667768,33.077),(69.658776,33.078421),(69.650198,33.081858),(69.640689,33.084235),(69.60834,33.079067),(69.587773,33.079532),(69.547517,33.075011),(69.514185,33.056691),(69.487727,33.028373),(69.478125,33.01135),(69.468503,32.994292),(69.48721,32.885875),(69.477288,32.856833),(69.471449,32.852234),(69.445559,32.835671),(69.421064,32.80681),(69.387165,32.785338),(69.376313,32.771877),(69.37807,32.752317),(69.383237,32.744462),(69.39042,32.737899),(69.399154,32.732964),(69.407732,32.729941),(69.414657,32.725575),(69.41724,32.718004),(69.419101,32.70046),(69.429126,32.667361),(69.426025,32.655088),(69.413933,32.635477),(69.361223,32.568478),(69.34355,32.55605),(69.302105,32.543777),(69.281952,32.532796),(69.251566,32.50055),(69.232859,32.462671),(69.228001,32.421304),(69.238543,32.378283),(69.264433,32.322369),(69.268102,32.301337),(69.266965,32.279581),(69.259989,32.236819),(69.259524,32.194522),(69.251876,32.15225),(69.251049,32.13065),(69.270634,32.035978),(69.302157,31.959911),(69.304793,31.94694),(69.301157,31.941282),(69.29885,31.93769),(69.250946,31.907046),(69.223764,31.881931),(69.114313,31.737754),(69.10005,31.724008),(69.084754,31.715791),(69.071628,31.69786),(69.040106,31.673107),(69.004036,31.651092),(68.977267,31.641481),(68.94099,31.643651),(68.906987,31.634298),(68.843529,31.606496),(68.803841,31.602568),(68.77728,31.618588),(68.731184,31.675535),(68.705449,31.70127),(68.697905,31.715946),(68.694907,31.756357),(68.688293,31.768605),(68.675787,31.774702),(68.61946,31.783487),(68.561065,31.811806),(68.527476,31.822968),(68.5051,31.822658),(68.481484,31.815682),(68.438179,31.796045),(68.418748,31.783022),(68.422211,31.773152),(68.439574,31.766589),(68.461433,31.763592),(68.521274,31.764781),(68.540601,31.762507),(68.5498,31.753515),(68.536829,31.741009),(68.515177,31.729951),(68.49802,31.725248),(68.460968,31.730467),(68.356065,31.762507),(68.316532,31.765194),(68.276948,31.763592),(68.255037,31.766383),(68.242635,31.776925),(68.2323,31.790154),(68.206823,31.807879),(68.184758,31.818214),(68.159126,31.825914),(68.138559,31.824674),(68.125743,31.811496),(68.104504,31.76876),(68.09834,31.759124),(68.093497,31.751551),(68.060476,31.725558),(68.055566,31.716566),(68.052259,31.695431),(68.046575,31.688403),(67.994071,31.663443),(67.977328,31.651868),(67.966063,31.638225),(67.955211,31.633212),(67.914696,31.631352),(67.884311,31.635641),(67.870668,31.635176),(67.842711,31.623497),(67.781165,31.564173),(67.74828,31.544392),(67.726698,31.53141),(67.696312,31.520816),(67.665461,31.518129),(67.600504,31.53048),(67.568981,31.52986),(67.556114,31.512186),(67.5634,31.497252),(67.578076,31.48211),(67.591099,31.464799),(67.59699,31.42568),(67.611459,31.410797),(67.631406,31.4001),(67.651147,31.395294),(67.734294,31.404751),(67.770364,31.394674),(67.775067,31.352764),(67.764731,31.334058),(67.749435,31.328011),(67.709024,31.329303),(67.692798,31.325221),(67.678949,31.316591),(67.665926,31.306256),(67.602157,31.271116),(67.583657,31.265225),(67.530224,31.256646),(67.493327,31.242952),(67.433899,31.236079),(67.364601,31.210706),(67.346204,31.20776),(67.282177,31.212773),(67.230294,31.210603),(67.213757,31.212256),(67.19319,31.218509),(67.156707,31.235924),(67.136759,31.241092),(67.116864,31.240316),(67.078468,31.232048),(67.058211,31.232358),(67.035267,31.235924),(67.022865,31.239386),(67.015268,31.244657),(67.013873,31.252667),(67.017904,31.258713),(67.023433,31.264863),(67.025965,31.273028),(67.023898,31.2953),(67.016818,31.309149),(67.002091,31.315867),(66.943283,31.314731),(66.905921,31.305532),(66.838483,31.277007),(66.808614,31.254734),(66.785308,31.23179),(66.759522,31.214788),(66.721281,31.210292),(66.696993,31.195823),(66.663093,31.083117),(66.643973,31.060172),(66.550025,30.976973),(66.527494,30.968343),(66.392309,30.944624),(66.375359,30.936717),(66.366264,30.922868),(66.267975,30.601389),(66.264926,30.557826),(66.281928,30.518138),(66.306113,30.491112),(66.313761,30.478296),(66.319393,30.457832),(66.321822,30.437368),(66.303012,30.305283),(66.305286,30.24477),(66.300893,30.225598),(66.236349,30.1116),(66.221725,30.073721),(66.219296,30.057856),(66.225239,30.04442),(66.260431,30.02313),(66.301565,29.98675),(66.332364,29.966079),(66.340529,29.956571),(66.336808,29.952023),(66.328437,29.949543),(66.322442,29.946545),(66.301772,29.915694),(66.27521,29.885154),(66.195628,29.835338),(66.110466,29.813634),(66.052381,29.798854),(65.994297,29.784023),(65.936109,29.769295),(65.878232,29.75449),(65.820147,29.739736),(65.762063,29.724905),(65.70403,29.710151),(65.645843,29.695346),(65.58781,29.680567),(65.529726,29.665736),(65.471745,29.650956),(65.413661,29.636177),(65.355783,29.621423),(65.297595,29.606644),(65.239511,29.591838),(65.181427,29.57711),(65.036371,29.540162),(64.986606,29.541557),(64.820312,29.567886),(64.683886,29.568816),(64.499477,29.570203),(64.477697,29.570367),(64.207739,29.499983),(64.172599,29.4843),(64.149758,29.458461),(64.113378,29.396295),(64.086093,29.386605),(63.971991,29.429574),(63.78792,29.46058),(63.568605,29.497477),(63.415953,29.484971),(63.366809,29.480941),(63.317768,29.47691),(63.268675,29.472879),(63.219686,29.468797),(63.170697,29.464766),(63.121604,29.460787),(63.072615,29.456756),(63.023626,29.452674),(62.974533,29.448643),(62.925544,29.444638),(62.876451,29.440581),(62.827462,29.436602),(62.778421,29.43252),(62.72938,29.428489),(62.680391,29.424458),(62.63135,29.420428),(62.602581,29.41807),(62.477509,29.407819),(62.374466,29.424872),(62.279433,29.451485),(62.196286,29.47492),(62.112983,29.498226),(62.029629,29.521636),(61.946534,29.545045),(61.863231,29.568351),(61.779929,29.591787),(61.696626,29.615196),(61.613427,29.638502),(61.530228,29.66186),(61.446926,29.685321),(61.363727,29.708627),(61.280424,29.732011),(61.197225,29.755343),(61.114026,29.778726),(61.030621,29.802161),(60.947422,29.825519),(60.876005,29.845518),(60.844379,29.858179),(60.902153,29.916883),(60.978117,29.994139),(60.978427,29.994398),(61.133043,30.154285),(61.215622,30.239706),(61.272156,30.298307),(61.380367,30.41029),(61.461705,30.494522),(61.461705,30.494574),(61.57622,30.613895),(61.702621,30.745515),(61.7852,30.831401),(61.802253,30.847059),(61.799773,30.852381),(61.798429,30.853467),(61.799876,30.871502),(61.802356,30.87853),(61.808351,30.881424),(61.804527,30.94974),(61.800186,30.961419),(61.819306,30.993923),(61.826438,31.014956),(61.826334,31.034593),(61.821477,31.054488),(61.809281,31.087148),(61.791814,31.118929),(61.789334,31.129315),(61.78768,31.158616),(61.779205,31.181095),(61.752437,31.219129),(61.742722,31.239541),(61.742308,31.259437),(61.749233,31.30238),(61.742308,31.32088),(61.706548,31.359844),(61.686911,31.373177),(61.661176,31.38191),(61.606259,31.388741),(61.490851,31.403097),(61.29479,31.427644),(61.125149,31.448887),(61.123535,31.449089),(60.956,31.47007),(60.855024,31.482731),(60.821744,31.494668),(60.809962,31.588357),(60.794976,31.636675),(60.792599,31.660084),(60.805311,31.733981),(60.791462,31.826534),(60.789292,31.874955),(60.796733,31.93676),(60.794976,31.958464),(60.791049,31.969161),(60.786088,31.978979),(60.784021,31.989263),(60.792599,32.011277),(60.786708,32.01691),(60.778336,32.021044),(60.774926,32.027194),(60.782677,32.042541),(60.808825,32.07117),(60.81482,32.087862),(60.828256,32.167443),(60.830323,32.248885),(60.79632,32.355907),(60.752291,32.494581),(60.711674,32.61031),(60.67488,32.715601),(60.64005,32.815517),(60.60615,32.912359),(60.577522,32.994344),(60.562949,33.058268),(60.561502,33.137307),(60.5676,33.151285),(60.615969,33.206191),(60.670332,33.267712),(60.719012,33.323032),(60.757046,33.366337),(60.785881,33.387782),(60.818644,33.404887),(60.829082,33.416282),(60.834664,33.436255),(60.832803,33.453463),(60.829289,33.470129),(60.832183,33.484495),(60.849856,33.494365),(60.897295,33.497026),(60.912075,33.501522),(60.919788,33.512571),(60.92086,33.514105),(60.911971,33.528446),(60.895125,33.541132),(60.879622,33.548703),(60.846032,33.555782),(60.807378,33.558159),(60.733894,33.554852),(60.655553,33.559891),(60.574834,33.587796),(60.511789,33.638387),(60.486881,33.71138),(60.494943,33.744117),(60.525845,33.80215),(60.528016,33.84145),(60.499387,33.994257),(60.488461,34.080927),(60.486778,34.094277),(60.492772,34.139028),(60.520884,34.186131),(60.552407,34.220057),(60.587443,34.250262),(60.634986,34.271217),(60.650592,34.28535),(60.643667,34.306641),(60.714361,34.310129),(60.815957,34.315271),(60.890474,34.318914),(60.879105,34.337647),(60.804898,34.417487),(60.789085,34.44348),(60.77937,34.455081),(60.767277,34.464151),(60.742783,34.473452),(60.733894,34.480429),(60.725316,34.504277),(60.718908,34.51115),(60.71033,34.515129),(60.699685,34.516396),(60.714361,34.537428),(60.739269,34.548022),(60.794046,34.554119),(60.818954,34.559907),(60.838488,34.570914),(60.883859,34.613702),(60.888924,34.62166),(60.895435,34.628327),(60.908871,34.634683),(60.922824,34.636543),(60.935846,34.635665),(60.947525,34.63799),(60.957964,34.649411),(60.961994,34.67375),(60.958687,34.699589),(60.959927,34.723205),(60.978014,34.74062),(61.00964,34.76098),(61.02907,34.789712),(61.034755,34.806249),(61.06545,34.814724),(61.073409,34.847693),(61.077749,34.88304),(61.077026,34.891721),(61.072582,34.910325),(61.071548,34.920557),(61.074752,34.933114),(61.088912,34.951925),(61.092012,34.961485),(61.095319,34.981018),(61.109892,35.018535),(61.115887,35.059877),(61.123121,35.074191),(61.147306,35.102096),(61.136764,35.111191),(61.135214,35.119408),(61.137901,35.128296),(61.139865,35.139665),(61.137074,35.143902),(61.101521,35.183228),(61.096353,35.193202),(61.102347,35.197698),(61.112373,35.202038),(61.112786,35.21165),(61.107929,35.221262),(61.102347,35.225603),(61.101624,35.235318),(61.102451,35.25666),(61.108342,35.277951),(61.122708,35.287666),(61.143999,35.28808),(61.167873,35.291387),(61.187097,35.300379),(61.195055,35.318104),(61.190404,35.369573),(61.195055,35.390089),(61.200843,35.401148),(61.221823,35.424247),(61.227301,35.434479),(61.235983,35.465847),(61.247041,35.48507),(61.270503,35.508428),(61.283835,35.52729),(61.290036,35.548064),(61.287556,35.568269),(61.277014,35.609197),(61.269676,35.618499),(61.283318,35.622633),(61.335718,35.630901),(61.344193,35.630901),(61.351531,35.627801),(61.360936,35.621031),(61.360109,35.618395),(61.351531,35.60713),(61.363934,35.598242),(61.365484,35.5985),(61.367654,35.597983),(61.383467,35.586459),(61.383881,35.582119),(61.379953,35.571318),(61.380263,35.567339),(61.386878,35.559536),(61.393182,35.553593),(61.401244,35.549821),(61.412613,35.548322),(61.419309,35.545891),(61.427702,35.542845),(61.492711,35.494165),(61.538703,35.452307),(61.604746,35.430603),(61.739621,35.413757),(61.799049,35.417478),(61.919352,35.451842),(61.977746,35.450912),(62.00746,35.438923),(62.033815,35.423989),(62.136341,35.341358),(62.163885,35.326165),(62.219023,35.30627),(62.2337,35.293764),(62.244345,35.278055),(62.251218,35.260381),(62.252613,35.242811),(62.25096,35.202349),(62.256851,35.187156),(62.275713,35.151034),(62.286203,35.140647),(62.302016,35.14752),(62.399323,35.255937),(62.431879,35.280638),(62.458905,35.281879),(62.524586,35.232166),(62.53549,35.22736),(62.545515,35.228032),(62.555437,35.230409),(62.566186,35.230771),(62.595641,35.22059),(62.60453,35.219402),(62.621066,35.222709),(62.69362,35.248392),(62.708606,35.25604),(62.721628,35.266427),(62.734238,35.280742),(62.760489,35.302446),(62.823063,35.327514),(62.827565,35.329318),(62.918206,35.388177),(62.985282,35.414584),(63.005539,35.418408),(63.047087,35.421353),(63.066827,35.425591),(63.080211,35.437063),(63.086257,35.455666),(63.090495,35.493545),(63.100727,35.524964),(63.097213,35.535455),(63.080573,35.546514),(63.077369,35.558761),(63.07494,35.609817),(63.076646,35.6247),(63.103156,35.646042),(63.179895,35.666455),(63.207697,35.683198),(63.213588,35.692293),(63.214415,35.699269),(63.209351,35.704488),(63.183926,35.712343),(63.17726,35.716891),(63.163255,35.73229),(63.112044,35.769911),(63.105688,35.782882),(63.101295,35.800555),(63.091322,35.813319),(63.084397,35.826083),(63.088634,35.843395),(63.104086,35.856469),(63.125273,35.860241),(63.342934,35.856262),(63.430216,35.871042),(63.511089,35.901841),(63.52592,35.912693),(63.549588,35.939048),(63.563438,35.950623),(63.582093,35.959047),(63.602298,35.962664),(63.725702,35.968865),(63.765906,35.977288),(63.859854,36.022092),(63.889826,36.029637),(63.906327,36.031732),(63.921969,36.033719),(63.949977,36.028861),(64.045114,35.998734),(64.045579,36.011291),(64.03638,36.058575),(64.036225,36.076404),(64.044648,36.09232),(64.057258,36.105239),(64.123352,36.146064),(64.158698,36.160275),(64.195905,36.165907),(64.237143,36.16043),(64.253731,36.154849),(64.266134,36.152471),(64.276056,36.159241),(64.291869,36.199445),(64.303651,36.211538),(64.318947,36.219754),(64.33724,36.225955),(64.410879,36.239856),(64.444572,36.249985),(64.477697,36.271637),(64.498161,36.291429),(64.576657,36.389615),(64.594382,36.424186),(64.605855,36.461083),(64.610505,36.500926),(64.607767,36.521648),(64.593711,36.559837),(64.589008,36.580249),(64.588801,36.600454),(64.592005,36.621073),(64.604821,36.660554),(64.621357,36.692593),(64.728549,36.85057),(64.765225,36.904622),(64.778764,36.93847),(64.77463,36.977589),(64.755613,37.053631),(64.76016,37.092621),(64.778764,37.118046),(64.807909,37.135512),(64.989087,37.213673),(65.063088,37.233232),(65.487922,37.241997),(65.501407,37.242276),(65.536857,37.257107),(65.583159,37.30806),(65.611271,37.331056),(65.624087,37.345138),(65.629461,37.365421),(65.628893,37.387538),(65.620986,37.430404),(65.625741,37.452805),(65.635766,37.474096),(65.648168,37.493837),(65.648272,37.49394),(65.648323,37.493992),(65.648323,37.494147),(65.65871,37.51027),(65.668839,37.520553),(65.681034,37.526393),(65.698294,37.529416),(65.739429,37.529054),(65.752658,37.537865),(65.758859,37.564117),(65.761443,37.578379),(65.776532,37.572902),(65.79648,37.569336),(65.804334,37.565305),(65.821078,37.535178),(65.829087,37.525618),(65.836064,37.519158),(65.855184,37.507944),(66.079666,37.440868),(66.097856,37.428414),(66.125607,37.3987),(66.142918,37.385058),(66.163072,37.376893),(66.174648,37.375498),(66.207721,37.376893),(66.218469,37.3748),(66.232525,37.365421),(66.241879,37.363302),(66.250848,37.35965),(66.257743,37.356842),(66.274383,37.343277),(66.294434,37.331211),(66.320685,37.329144),(66.389622,37.347179),(66.413496,37.349633),(66.423625,37.34568),(66.438663,37.327904),(66.451065,37.322943),(66.461452,37.324441),(66.469927,37.329764),(66.485223,37.343406),(66.502896,37.355654),(66.519588,37.36418),(66.539018,37.36909),(66.564598,37.370692),(66.588576,37.36847),(66.65467,37.346145),(66.667021,37.344311),(66.694099,37.343406),(66.704745,37.346533),(66.724433,37.36015),(66.734975,37.363302),(66.865768,37.367927),(66.957649,37.385213),(67.005915,37.384386),(67.024105,37.377487),(67.064102,37.354569),(67.085083,37.349633),(67.097382,37.34059),(67.113918,37.297182),(67.123324,37.281963),(67.143477,37.27186),(67.187816,37.258244),(67.20797,37.243464),(67.217995,37.226256),(67.225281,37.207704),(67.236753,37.192408),(67.259129,37.185147),(67.281453,37.188661),(67.319591,37.208428),(67.345222,37.213027),(67.36951,37.21468),(67.391163,37.21959),(67.411265,37.22791),(67.431108,37.239769),(67.463975,37.266279),(67.482371,37.277364),(67.503145,37.281963),(67.526141,37.272894),(67.545003,37.231527),(67.561385,37.2199),(67.572185,37.223155),(67.579937,37.232664),(67.586034,37.242612),(67.592287,37.247185),(67.602726,37.24827),(67.621794,37.252921),(67.633163,37.254058),(67.644635,37.251216),(67.666443,37.238012),(67.677502,37.233542),(67.690162,37.23225),(67.725871,37.233542),(67.746593,37.229408),(67.764266,37.220882),(67.775997,37.207549),(67.780544,37.188868),(67.771966,37.12497),(67.773051,37.109984),(67.78535,37.096548),(67.808036,37.083241),(67.832893,37.073087),(67.87811,37.064431),(67.891959,37.052029),(67.902501,37.034381),(67.918417,37.013814),(67.929269,37.006244),(67.952937,36.996244),(67.961825,36.98994),(67.99619,36.955782),(68.00203,36.947487),(68.00606,36.938677),(68.011125,36.930874),(68.02022,36.92568),(68.02482,36.925518),(68.032725,36.925241),(68.044714,36.92922),(68.054946,36.934646),(68.121092,36.980276),(68.133391,36.986529),(68.151995,36.992782),(68.167963,37.006089),(68.186411,37.01831),(68.212559,37.021256),(68.253746,37.010223),(68.277775,37.010068),(68.288317,37.024408),(68.281599,37.066472),(68.280772,37.086652),(68.288317,37.103214),(68.307282,37.114221),(68.326661,37.113033),(68.346091,37.106909),(68.365832,37.103214),(68.391877,37.105437),(68.411462,37.113369),(68.418025,37.128277),(68.403711,37.151635),(68.465774,37.155356),(68.513316,37.163986),(68.523858,37.164658),(68.532747,37.168999),(68.546544,37.188222),(68.551453,37.192615),(68.571401,37.194708),(68.608814,37.204087),(68.626901,37.206257),(68.630777,37.213828),(68.63641,37.229925),(68.648605,37.244369),(68.671239,37.247185),(68.665452,37.259381),(68.662661,37.263721),(68.657545,37.267675),(68.668862,37.278294),(68.686949,37.279018),(68.725913,37.274522),(68.746532,37.276227),(68.759606,37.275245),(68.771285,37.270724),(68.808492,37.251629),(68.820378,37.250906),(68.824615,37.260621),(68.822031,37.281963),(68.810197,37.312091),(68.813815,37.323408),(68.835829,37.329144),(68.856758,37.324906),(68.868023,37.312142),(68.877015,37.296097),(68.890296,37.281963),(68.904714,37.276692),(68.915824,37.279069),(68.921147,37.287958),(68.917633,37.302427),(68.912982,37.306768),(68.896755,37.316483),(68.890296,37.322943),(68.885077,37.334828),(68.887867,37.338135),(68.893655,37.337154),(68.897169,37.335965),(68.965433,37.329144),(68.985225,37.320359),(69.005483,37.305528),(69.021037,37.287958),(69.035765,37.251577),(69.055609,37.237108),(69.078398,37.225171),(69.096433,37.213027),(69.114313,37.177473),(69.123615,37.169205),(69.145732,37.156958),(69.151003,37.155046),(69.245675,37.103886),(69.265777,37.105411),(69.285621,37.11293),(69.308617,37.116882),(69.312079,37.117477),(69.323913,37.120991),(69.334765,37.129053),(69.350836,37.144142),(69.391196,37.164658),(69.407732,37.177473),(69.442148,37.223621),(69.445197,37.23641),(69.426852,37.239459),(69.409851,37.245661),(69.417912,37.267675),(69.409696,37.276744),(69.403081,37.31116),(69.394606,37.326069),(69.389439,37.332916),(69.386441,37.34152),(69.384891,37.3509),(69.384478,37.36015),(69.385821,37.363147),(69.388508,37.363767),(69.390885,37.365317),(69.391196,37.370692),(69.388302,37.37511),(69.378948,37.377125),(69.377036,37.380665),(69.376313,37.418854),(69.37838,37.437406),(69.384478,37.453271),(69.389645,37.4588),(69.404011,37.466861),(69.411711,37.47252),(69.415587,37.477507),(69.421788,37.489496),(69.425405,37.49425),(69.469744,37.52014),(69.491448,37.536986),(69.508656,37.578973),(69.528655,37.586027),(69.664977,37.576157),(69.687301,37.579594),(69.729573,37.594089),(69.754274,37.596673),(69.762232,37.595794),(69.784453,37.59042),(69.791843,37.586105),(69.80223,37.581506),(69.812668,37.585846),(69.823004,37.592952),(69.832926,37.596673),(69.853286,37.60122),(69.895247,37.61817),(69.918812,37.617137),(69.935245,37.605664),(69.944547,37.589696),(69.955399,37.575485),(69.976586,37.569336),(69.989505,37.56732),(69.994673,37.562256),(69.998497,37.55559),(70.004187,37.551371),(70.007592,37.548846),(70.016894,37.545978),(70.048003,37.541431),(70.041698,37.548846),(70.06919,37.545203),(70.099059,37.536573),(70.129135,37.532284),(70.157867,37.541431),(70.192697,37.576028),(70.199415,37.579284),(70.201947,37.587991),(70.216468,37.617137),(70.218432,37.617447),(70.237655,37.618118),(70.240963,37.617137),(70.246079,37.621271),(70.248301,37.623338),(70.253934,37.630805),(70.255225,37.637471),(70.253882,37.646592),(70.253572,37.654576),(70.257706,37.65809),(70.262564,37.660881),(70.275121,37.675479),(70.277033,37.687985),(70.280547,37.696046),(70.283079,37.704831),(70.281891,37.719559),(70.271762,37.748705),(70.269592,37.762632),(70.275121,37.774181),(70.265561,37.781209),(70.256517,37.791803),(70.249851,37.804877),(70.247164,37.819166),(70.240343,37.827563),(70.207476,37.835624),(70.196004,37.839914),(70.179158,37.860791),(70.165205,37.889911),(70.160503,37.920658),(70.17213,37.946083),(70.192232,37.93288),(70.214711,37.929262),(70.238121,37.932363),(70.26153,37.939287),(70.253675,37.947401),(70.250161,37.955617),(70.25042,37.964144),(70.253934,37.973394),(70.26091,37.976469),(70.272692,37.978174),(70.283751,37.981869),(70.288712,37.990809),(70.293776,37.996106),(70.298583,37.998134),(70.317961,38.006312),(70.326281,38.011273),(70.371187,38.058298),(70.415371,38.094523),(70.425964,38.100647),(70.460071,38.112248),(70.470303,38.120517),(70.482602,38.137363),(70.508543,38.192502),(70.537947,38.238055),(70.547197,38.262653),(70.559393,38.268182),(70.573036,38.27105),(70.583061,38.275081),(70.5966,38.309187),(70.596703,38.317585),(70.595566,38.327791),(70.595773,38.338178),(70.600011,38.347066),(70.609312,38.3512),(70.632102,38.350115),(70.641352,38.354197),(70.676698,38.37492),(70.68476,38.38665),(70.664865,38.398639),(70.664865,38.405409),(70.683313,38.414581),(70.741811,38.419439),(70.754213,38.436466),(70.761086,38.44352),(70.777261,38.446466),(70.808887,38.446362),(70.817982,38.444993),(70.834105,38.4406),(70.843045,38.440161),(70.849814,38.442823),(70.853587,38.447422),(70.859323,38.451659),(70.871002,38.45321),(70.91224,38.437655),(70.936217,38.433004),(70.946759,38.443236),(70.943452,38.465948),(70.950739,38.473053),(70.974045,38.473673),(70.986809,38.470909),(70.998229,38.465664),(71.008564,38.458584),(71.018641,38.449825),(71.024584,38.441918),(71.032956,38.423651),(71.039467,38.415331),(71.049802,38.408664)] +Albania [(19.747766,42.578901),(19.746009,42.579934),(19.741358,42.574405),(19.733916,42.562597),(19.732159,42.555543),(19.730299,42.540428),(19.730919,42.533658),(19.73433,42.524382),(19.75159,42.493402),(19.784456,42.474566),(19.801406,42.468132),(19.819596,42.466479),(19.829063,42.468723),(19.835512,42.470251),(19.873339,42.486839),(19.882538,42.493402),(19.882745,42.493557),(19.882745,42.493609),(19.907549,42.506399),(19.956332,42.505314),(19.981757,42.510765),(20.017723,42.546241),(20.039221,42.557765),(20.064956,42.546758),(20.085626,42.530015),(20.135546,42.509629),(20.152392,42.493712),(20.152599,42.493712),(20.152599,42.493609),(20.152702,42.493428),(20.152702,42.493402),(20.180814,42.443095),(20.186292,42.437437),(20.199728,42.427799),(20.204689,42.420099),(20.204379,42.411831),(20.194043,42.4001),(20.192803,42.393693),(20.197454,42.38744),(20.21306,42.377518),(20.218848,42.371446),(20.221225,42.36372),(20.219468,42.350207),(20.220915,42.343101),(20.229597,42.32672),(20.237762,42.319924),(20.249647,42.318607),(20.317964,42.319821),(20.333363,42.317883),(20.456973,42.250497),(20.473665,42.237123),(20.481674,42.230705),(20.500795,42.211223),(20.538622,42.150116),(20.549371,42.123476),(20.551954,42.105803),(20.552161,42.07379),(20.558259,42.055109),(20.589678,41.993639),(20.594329,41.973718),(20.59929,41.960567),(20.59929,41.94788),(20.588748,41.929586),(20.580583,41.921732),(20.573348,41.917675),(20.567251,41.912172),(20.562496,41.900105),(20.562703,41.892845),(20.567767,41.88052),(20.567147,41.873182),(20.541722,41.86158),(20.540786,41.844865),(20.540482,41.839437),(20.54844,41.814193),(20.550136,41.80006),(20.550921,41.793522),(20.54441,41.784763),(20.521155,41.767658),(20.511337,41.757943),(20.503275,41.744637),(20.500381,41.734095),(20.508339,41.661748),(20.513404,41.640405),(20.534591,41.594026),(20.53467,41.587324),(20.534694,41.585266),(20.52932,41.574879),(20.520845,41.568368),(20.507409,41.56227),(20.49294,41.557671),(20.444157,41.549661),(20.447878,41.53545),(20.444984,41.508475),(20.452012,41.493592),(20.463174,41.489768),(20.470822,41.483774),(20.481597,41.468269),(20.483535,41.46548),(20.486842,41.457781),(20.488909,41.441451),(20.490873,41.436025),(20.498004,41.431477),(20.514644,41.429462),(20.522189,41.42569),(20.534074,41.412977),(20.539049,41.402944),(20.540172,41.400678),(20.539965,41.387139),(20.532937,41.370447),(20.523429,41.356805),(20.510407,41.344403),(20.49604,41.337788),(20.481674,41.341199),(20.478108,41.321585),(20.477747,41.319598),(20.483121,41.289471),(20.500071,41.23552),(20.51268,41.210147),(20.549681,41.170615),(20.565494,41.147412),(20.570558,41.124829),(20.569938,41.107104),(20.576966,41.093513),(20.597419,41.086273),(20.605284,41.083488),(20.618927,41.082403),(20.631536,41.082868),(20.634129,41.082576),(20.643008,41.081576),(20.65386,41.075272),(20.664092,41.059149),(20.683419,40.99383),(20.702746,40.936314),(20.717216,40.913214),(20.730573,40.904611),(20.740883,40.89797),(20.766102,40.893732),(20.783672,40.899055),(20.816951,40.920242),(20.836544,40.923904),(20.837415,40.924066),(20.890228,40.918279),(20.939941,40.907065),(20.956684,40.894766),(20.964849,40.875956),(20.965262,40.849394),(20.967019,40.802007),(20.960922,40.780768),(20.943662,40.765265),(20.95348,40.759167),(20.957201,40.751622),(20.959165,40.743561),(20.963505,40.736068),(20.992031,40.7155),(21.018593,40.690902),(21.028825,40.676846),(21.035336,40.65938),(21.036679,40.639743),(21.033165,40.621398),(21.02252,40.586619),(21.020763,40.575096),(21.02035,40.566776),(21.018799,40.558972),(21.013528,40.548896),(21.006707,40.543211),(20.997509,40.53918),(20.98831,40.533496),(20.981489,40.523264),(20.96857,40.520577),(20.957924,40.514944),(20.95193,40.506056),(20.952653,40.493653),(20.94976,40.487866),(20.946452,40.482905),(20.93684,40.472518),(20.911932,40.459547),(20.889608,40.463733),(20.866767,40.472053),(20.839999,40.471639),(20.826666,40.464818),(20.821292,40.456395),(20.817365,40.446835),(20.8092,40.436603),(20.800311,40.432779),(20.779537,40.429006),(20.770649,40.421978),(20.768995,40.412573),(20.773336,40.38534),(20.773853,40.374901),(20.770442,40.36255),(20.765998,40.354282),(20.755043,40.338624),(20.739552,40.309166),(20.736646,40.303639),(20.728791,40.298265),(20.717836,40.293924),(20.706984,40.288085),(20.699129,40.278369),(20.694581,40.25558),(20.696648,40.235581),(20.696442,40.215066),(20.68528,40.191191),(20.679802,40.187832),(20.665022,40.18437),(20.660372,40.178892),(20.659855,40.172743),(20.663162,40.161684),(20.663472,40.15662),(20.668226,40.138068),(20.666779,40.13352),(20.651793,40.101016),(20.647556,40.09404),(20.640218,40.090112),(20.615361,40.081017),(20.577793,40.067271),(20.552885,40.065411),(20.499865,40.071457),(20.481674,40.067788),(20.465965,40.060812),(20.432789,40.063809),(20.413772,40.057194),(20.400439,40.045257),(20.38907,40.029444),(20.380699,40.011667),(20.376461,39.993736),(20.359925,39.991048),(20.310005,39.98986),(20.297913,39.986914),(20.302977,39.979111),(20.317964,39.918391),(20.323131,39.912397),(20.346902,39.894259),(20.392998,39.835451),(20.397028,39.818087),(20.388967,39.79814),(20.371604,39.784291),(20.35455,39.78579),(20.336877,39.794006),(20.299577,39.805056),(20.298326,39.805427),(20.288405,39.806615),(20.280033,39.80398),(20.273005,39.796487),(20.275485,39.792404),(20.281377,39.788167),(20.284684,39.780312),(20.283134,39.775144),(20.276829,39.763155),(20.275692,39.760055),(20.278173,39.756748),(20.285511,39.749668),(20.287268,39.746671),(20.291092,39.737937),(20.296466,39.733493),(20.299463,39.728222),(20.296466,39.71737),(20.28272,39.704399),(20.264117,39.696079),(20.249027,39.684814),(20.24603,39.662903),(20.237245,39.667037),(20.228563,39.669156),(20.220088,39.669156),(20.212027,39.667037),(20.204172,39.660319),(20.203035,39.65262),(20.203449,39.645333),(20.199935,39.640062),(20.184122,39.637013),(20.163761,39.6443),(20.135236,39.664195),(20.089347,39.682799),(20.049556,39.69272),(20.01607,39.701402),(19.99991,39.693498),(19.999848,39.693508),(19.996837,39.692572),(19.99586,39.686713),(19.989024,39.686713),(19.983572,39.701361),(19.98699,39.717963),(20.002696,39.748236),(19.985606,39.75373),(19.980642,39.75731),(19.99586,39.782945),(20.014171,39.832953),(20.016287,39.845038),(20.009776,39.865668),(19.999278,39.871405),(19.983735,39.872382),(19.961681,39.878567),(19.960216,39.881049),(19.959972,39.885077),(19.958832,39.88935),(19.954845,39.892808),(19.941173,39.898993),(19.931163,39.901923),(19.914073,39.902411),(19.906423,39.90644),(19.906423,39.912665),(19.921235,39.926947),(19.931,39.934516),(19.941173,39.940009),(19.923839,39.952826),(19.876801,40.022366),(19.870453,40.034735),(19.863455,40.045356),(19.855235,40.049872),(19.823253,40.050605),(19.807384,40.052883),(19.79656,40.057318),(19.804535,40.063463),(19.777354,40.077786),(19.773123,40.071234),(19.771658,40.068305),(19.770518,40.063463),(19.767263,40.072455),(19.762462,40.07685),(19.756358,40.079901),(19.74936,40.084621),(19.748057,40.088609),(19.740896,40.100409),(19.732677,40.110256),(19.729015,40.108222),(19.719086,40.116034),(19.653168,40.132392),(19.59783,40.157701),(19.565766,40.179836),(19.509288,40.194485),(19.476248,40.21369),(19.381114,40.295233),(19.369965,40.309027),(19.365082,40.317369),(19.351573,40.362047),(19.344249,40.374335),(19.309255,40.400824),(19.296886,40.41356),(19.295177,40.417548),(19.295177,40.418687),(19.289561,40.420396),(19.289561,40.427232),(19.316173,40.437812),(19.33253,40.43891),(19.348155,40.43065),(19.3838,40.397406),(19.39324,40.386298),(19.406505,40.362616),(19.411957,40.34809),(19.414236,40.334418),(19.423595,40.330024),(19.444591,40.333808),(19.465587,40.341498),(19.474946,40.348375),(19.481619,40.439114),(19.47877,40.453925),(19.453136,40.465318),(19.418793,40.491889),(19.391368,40.521796),(19.386404,40.54328),(19.39324,40.537055),(19.396007,40.529608),(19.398611,40.520941),(19.406749,40.522773),(19.414236,40.522773),(19.433604,40.505927),(19.446056,40.511135),(19.452647,40.529853),(19.4546,40.55386),(19.442149,40.573065),(19.415701,40.583482),(19.391612,40.578315),(19.386404,40.550727),(19.379242,40.566148),(19.309825,40.644355),(19.304373,40.653144),(19.309337,40.665025),(19.322927,40.674872),(19.339041,40.684068),(19.351573,40.694159),(19.360525,40.709784),(19.364513,40.726508),(19.365571,40.785956),(19.368826,40.802151),(19.377778,40.812974),(19.396251,40.817043),(19.406749,40.820746),(19.404063,40.829291),(19.396739,40.83869),(19.39324,40.844916),(19.397634,40.852973),(19.404796,40.86107),(19.411388,40.867092),(19.414236,40.868842),(19.406261,40.880316),(19.391124,40.89525),(19.381196,40.908271),(19.389822,40.913804),(19.404959,40.918158),(19.424571,40.939643),(19.440929,40.948635),(19.427013,40.936835),(19.435395,40.913398),(19.427257,40.893378),(19.427257,40.87226),(19.434744,40.87226),(19.437836,40.886542),(19.437673,40.88345),(19.439138,40.880072),(19.440929,40.87226),(19.454112,40.88581),(19.506602,40.905097),(19.523448,40.92064),(19.525157,40.945258),(19.514659,40.964911),(19.501475,40.982978),(19.495453,41.002631),(19.492035,40.976955),(19.479666,40.953925),(19.453868,40.923977),(19.453461,40.928371),(19.447765,40.934312),(19.458507,40.942613),(19.469981,40.956732),(19.477224,40.975002),(19.474946,40.995795),(19.469981,41.000718),(19.453787,41.005845),(19.447765,41.009467),(19.440603,41.020657),(19.441254,41.026028),(19.445486,41.030707),(19.456391,41.106676),(19.451182,41.122707),(19.445486,41.129462),(19.441173,41.137356),(19.441905,41.143866),(19.463715,41.151313),(19.467133,41.162665),(19.467133,41.176174),(19.468761,41.187567),(19.509532,41.237291),(19.516612,41.256415),(19.513357,41.276679),(19.50294,41.294135),(19.488048,41.306464),(19.471934,41.311103),(19.450043,41.311103),(19.431651,41.313707),(19.418956,41.322577),(19.414236,41.341498),(19.414236,41.382799),(19.410818,41.391669),(19.396007,41.402045),(19.39324,41.41352),(19.406423,41.400377),(19.428722,41.401313),(19.44516,41.415188),(19.440929,41.440863),(19.46461,41.450873),(19.489268,41.466132),(19.508637,41.486884),(19.516612,41.513414),(19.514171,41.529975),(19.507172,41.536282),(19.496349,41.538764),(19.482432,41.54385),(19.468028,41.554267),(19.451996,41.574774),(19.440929,41.585395),(19.528087,41.576565),(19.556407,41.583075),(19.565196,41.58511),(19.584809,41.619574),(19.591645,41.619574),(19.592296,41.608588),(19.596039,41.602729),(19.603038,41.601752),(19.612966,41.605292),(19.604991,41.632148),(19.600108,41.639106),(19.591645,41.632636),(19.587087,41.640692),(19.557628,41.660549),(19.578136,41.687649),(19.576345,41.71894),(19.571137,41.752387),(19.5713,41.770413),(19.589529,41.765692),(19.598969,41.778713),(19.599783,41.79975),(19.591645,41.818793),(19.585216,41.818549),(19.465505,41.855455),(19.451182,41.862535),(19.432953,41.868598),(19.365122,41.852372),(19.364223,41.862846),(19.364947,41.888995),(19.356989,41.89964),(19.347687,41.906358),(19.34531,41.910337),(19.34593,41.914471),(19.345413,41.921189),(19.346653,41.926176),(19.350477,41.931369),(19.352854,41.938501),(19.346447,41.961548),(19.351614,41.965166),(19.359779,41.965993),(19.365774,41.969713),(19.371045,41.98656),(19.363707,41.993484),(19.363396,41.993639),(19.354198,42.008703),(19.350581,42.027823),(19.351718,42.047615),(19.356575,42.064798),(19.374869,42.094667),(19.372491,42.104253),(19.369998,42.106507),(19.355025,42.12004),(19.297767,42.151666),(19.282058,42.164559),(19.281956,42.164723),(19.272033,42.180682),(19.274926,42.191276),(19.304589,42.215099),(19.400397,42.325893),(19.401327,42.331061),(19.400707,42.344755),(19.402671,42.352015),(19.412076,42.368397),(19.417243,42.374107),(19.468403,42.41811),(19.481735,42.434491),(19.501476,42.444129),(19.517806,42.45803),(19.531655,42.474773),(19.543747,42.493402),(19.543747,42.493557),(19.549328,42.508595),(19.561317,42.516243),(19.575683,42.522341),(19.588086,42.532857),(19.593667,42.54482),(19.599248,42.571046),(19.605035,42.584844),(19.621779,42.604997),(19.647927,42.627838),(19.658037,42.634613),(19.676039,42.646675),(19.699293,42.654814),(19.722134,42.64608),(19.737534,42.624402),(19.746009,42.5989),(19.746885,42.58893),(19.747766,42.578901)] +Andorra [(1.707006,42.502781),(1.697498,42.494462),(1.686336,42.490612),(1.674244,42.490508),(1.662358,42.493712),(1.659774,42.496813),(1.656984,42.49764),(1.653986,42.496529),(1.650369,42.493402),(1.639517,42.466427),(1.607478,42.456428),(1.544432,42.450356),(1.538851,42.445653),(1.534511,42.439917),(1.528206,42.434233),(1.51663,42.429504),(1.508466,42.428677),(1.447901,42.434646),(1.436429,42.440951),(1.436429,42.453482),(1.407593,42.486762),(1.424543,42.492472),(1.430227,42.493557),(1.449968,42.504073),(1.446557,42.519886),(1.428987,42.531462),(1.406456,42.52924),(1.409764,42.540609),(1.4263,42.561796),(1.426403,42.565646),(1.418032,42.569832),(1.419272,42.579263),(1.424853,42.589365),(1.429297,42.595386),(1.451415,42.602052),(1.466814,42.641455),(1.49844,42.640241),(1.527793,42.648535),(1.543089,42.649362),(1.597349,42.621921),(1.608304,42.618123),(1.721993,42.609855),(1.713311,42.589546),(1.729434,42.582001),(1.752688,42.576679),(1.761107,42.567646),(1.765091,42.563372),(1.739976,42.561641),(1.721683,42.548515),(1.710624,42.527741),(1.707006,42.502781)] +Ashmore and Cartier Islands [(123.597016,-12.428318),(123.597748,-12.438572),(123.575613,-12.436782),(123.575043,-12.426609),(123.597016,-12.428318)] +Austria [(15.161792,48.937221),(15.238067,48.95076),(15.243234,48.95293),(15.257084,48.963886),(15.260391,48.969157),(15.262872,48.982593),(15.266902,48.986675),(15.27455,48.986727),(15.279615,48.982593),(15.283749,48.977632),(15.28871,48.975358),(15.335529,48.974996),(15.357853,48.97081),(15.405912,48.954687),(15.450354,48.944817),(15.471851,48.936704),(15.521254,48.908489),(15.544187,48.902612),(15.603729,48.887353),(15.681347,48.858466),(15.703775,48.854952),(15.726823,48.854952),(15.743669,48.858466),(15.779222,48.870868),(15.787904,48.872263),(15.818497,48.872315),(15.824284,48.869421),(15.828315,48.857122),(15.833276,48.852006),(15.840097,48.850094),(15.859218,48.849164),(15.870173,48.843893),(15.877614,48.841671),(15.885986,48.842033),(15.875754,48.833093),(15.888363,48.835056),(15.899629,48.834746),(15.90707,48.830251),(15.908103,48.819709),(15.925053,48.822602),(15.930324,48.818313),(15.930324,48.811389),(15.931771,48.806428),(16.032334,48.758059),(16.085354,48.742866),(16.177751,48.746535),(16.317588,48.73284),(16.318724,48.733512),(16.339498,48.735579),(16.352727,48.728448),(16.358309,48.727259),(16.374018,48.730308),(16.384974,48.737078),(16.43572,48.794542),(16.453083,48.80219),(16.481919,48.7994),(16.492254,48.79971),(16.510341,48.804774),(16.519539,48.805601),(16.528737,48.803534),(16.545791,48.796299),(16.605529,48.784827),(16.624029,48.78338),(16.643149,48.778264),(16.651004,48.766223),(16.654725,48.751857),(16.661959,48.740023),(16.675085,48.733926),(16.691001,48.732065),(16.729035,48.733099),(16.744332,48.729637),(16.780505,48.713772),(16.798902,48.709224),(16.818125,48.710775),(16.856573,48.719818),(16.873006,48.718991),(16.896674,48.696977),(16.910523,48.63078),(16.945043,48.604166),(16.954345,48.557399),(16.949055,48.544836),(16.946903,48.539726),(16.93047,48.528202),(16.913934,48.519339),(16.906492,48.509908),(16.901221,48.496602),(16.875073,48.471539),(16.864944,48.458077),(16.86081,48.443788),(16.851922,48.390407),(16.849338,48.384102),(16.846858,48.381131),(16.845101,48.376583),(16.84448,48.365602),(16.847581,48.359582),(16.855332,48.356455),(16.875486,48.355034),(16.881171,48.352812),(16.891713,48.347024),(16.901945,48.339402),(16.906492,48.33147),(16.904218,48.327025),(16.900291,48.321238),(16.898431,48.316173),(16.902771,48.314106),(16.905459,48.311988),(16.908869,48.306975),(16.912073,48.301239),(16.913313,48.296691),(16.916621,48.2908),(16.924269,48.287519),(16.933364,48.284728),(16.944423,48.27801),(16.950624,48.276589),(16.954345,48.273127),(16.955275,48.268786),(16.953931,48.25734),(16.954345,48.25256),(16.969951,48.216645),(16.974705,48.198558),(16.974808,48.17688),(16.98194,48.161299),(17.007158,48.142799),(17.020904,48.137166),(17.036923,48.135513),(17.047465,48.130991),(17.062455,48.112724),(17.067206,48.106936),(17.080228,48.097608),(17.069996,48.089159),(17.064829,48.07903),(17.059144,48.060427),(17.063072,48.058773),(17.075061,48.052081),(17.069686,48.035674),(17.092631,48.02725),(17.12498,48.019525),(17.148338,48.005443),(17.085603,47.970148),(17.096145,47.961931),(17.095731,47.95573),(17.090874,47.949477),(17.088083,47.940899),(17.086119,47.93922),(17.076818,47.934956),(17.07413,47.932114),(17.075164,47.927773),(17.080125,47.925758),(17.085396,47.924621),(17.087463,47.922632),(17.083329,47.904829),(17.077541,47.891729),(17.067619,47.881626),(17.051186,47.872893),(17.016563,47.867699),(17.004057,47.863281),(17.004367,47.852377),(17.010672,47.847882),(17.031652,47.841345),(17.039507,47.837365),(17.049119,47.818684),(17.05563,47.812354),(17.040644,47.801114),(17.041988,47.783906),(17.048706,47.763649),(17.050566,47.730989),(17.055527,47.720913),(17.064002,47.71329),(17.075371,47.708484),(17.054804,47.702025),(16.98194,47.695436),(16.902668,47.682026),(16.864738,47.686729),(16.850165,47.712929),(16.836936,47.705358),(16.817299,47.684248),(16.80624,47.676884),(16.797455,47.675463),(16.741128,47.681458),(16.730379,47.685902),(16.719217,47.693731),(16.711569,47.704066),(16.707848,47.714608),(16.702474,47.7236),(16.689865,47.729568),(16.609766,47.750627),(16.567805,47.754192),(16.531115,47.742978),(16.52512,47.733315),(16.526877,47.720137),(16.521399,47.711533),(16.512924,47.706004),(16.47303,47.691767),(16.461145,47.684532),(16.45453,47.681768),(16.449879,47.682414),(16.444918,47.68554),(16.43913,47.690475),(16.431792,47.685463),(16.4166,47.668823),(16.407815,47.66133),(16.425901,47.654276),(16.481919,47.638954),(16.509617,47.64314),(16.575453,47.624949),(16.608422,47.628773),(16.63023,47.622004),(16.647697,47.606139),(16.656172,47.585934),(16.65059,47.566555),(16.66754,47.560096),(16.68139,47.550561),(16.689141,47.53803),(16.688004,47.52263),(16.677359,47.509866),(16.648213,47.501546),(16.636845,47.4932),(16.640875,47.452919),(16.626973,47.445549),(16.589406,47.425633),(16.481919,47.392302),(16.46993,47.405531),(16.456597,47.411836),(16.444091,47.40951),(16.433963,47.39685),(16.436443,47.358506),(16.434273,47.35556),(16.429312,47.353803),(16.424971,47.351116),(16.424351,47.345199),(16.427142,47.341375),(16.431689,47.339721),(16.436133,47.338559),(16.43882,47.336569),(16.469206,47.293238),(16.473237,47.276805),(16.466622,47.263421),(16.452463,47.254584),(16.421354,47.243138),(16.421354,47.243061),(16.421354,47.243035),(16.424661,47.225594),(16.409262,47.203683),(16.412776,47.187173),(16.41877,47.183659),(16.426728,47.184124),(16.4351,47.183581),(16.441818,47.177121),(16.442128,47.168336),(16.433963,47.151283),(16.433653,47.145754),(16.447089,47.139708),(16.480885,47.150767),(16.497318,47.149681),(16.509514,47.137537),(16.50476,47.125833),(16.481919,47.10524),(16.460835,47.0963),(16.454323,47.081701),(16.461765,47.068498),(16.481919,47.063898),(16.493701,47.059816),(16.497215,47.054622),(16.493184,47.049145),(16.481919,47.04421),(16.437167,47.031782),(16.424661,47.024082),(16.45329,47.021679),(16.467553,47.018423),(16.481919,47.00938),(16.486363,46.998554),(16.467449,46.995427),(16.441404,46.99522),(16.424764,46.993102),(16.424764,46.992998),(16.420424,46.99026),(16.415566,46.989433),(16.410398,46.990415),(16.405024,46.993153),(16.387764,47.002042),(16.366577,47.003825),(16.325856,47.00044),(16.288545,47.005582),(16.274903,47.004315),(16.265394,46.993257),(16.261054,46.97809),(16.252992,46.973516),(16.24307,46.972018),(16.232942,46.966075),(16.230358,46.959977),(16.231185,46.954422),(16.230668,46.94835),(16.22395,46.941064),(16.217025,46.937395),(16.196148,46.931297),(16.170723,46.918533),(16.159148,46.910316),(16.122871,46.876365),(16.110055,46.867916),(16.094035,46.862774),(16.052936,46.84606),(16.032024,46.837556),(16.028441,46.836947),(15.987582,46.830011),(15.981691,46.827685),(15.971976,46.820632),(15.972906,46.818435),(15.97735,46.816213),(15.978177,46.809134),(15.971252,46.778076),(15.970985,46.77505),(15.969702,46.760532),(15.970529,46.743014),(15.982001,46.718545),(16.003291,46.709191),(16.014557,46.693714),(16.016723,46.670691),(16.016727,46.670641),(16.016593,46.670757),(15.997917,46.686919),(15.986962,46.69219),(15.946344,46.697151),(15.878545,46.720715),(15.850743,46.724488),(15.822941,46.722834),(15.78461,46.7122),(15.755141,46.704024),(15.728683,46.70299),(15.652098,46.710819),(15.635975,46.717563),(15.632875,46.702473),(15.632978,46.689632),(15.626984,46.680873),(15.621745,46.678175),(15.616648,46.67555),(15.603729,46.673018),(15.58864,46.675963),(15.567349,46.675757),(15.545955,46.671881),(15.530659,46.663845),(15.52084,46.649608),(15.520308,46.64772),(15.517636,46.63824),(15.513939,46.632545),(15.511228,46.628369),(15.492625,46.618293),(15.462653,46.614649),(15.440018,46.62439),(15.435381,46.627195),(15.417591,46.637955),(15.388135,46.645578),(15.332784,46.64358),(15.204891,46.638963),(15.172557,46.64136),(15.105591,46.646323),(15.085723,46.647795),(15.061954,46.649557),(15.004386,46.636844),(14.967179,46.600257),(14.947955,46.619274),(14.933589,46.621135),(14.919507,46.615017),(14.897726,46.605554),(14.894063,46.605215),(14.877055,46.603642),(14.862999,46.604831),(14.85039,46.601136),(14.833647,46.584393),(14.822278,46.567546),(14.814519,46.551337),(14.807189,46.536024),(14.796663,46.519401),(14.788585,46.506646),(14.783004,46.503261),(14.760887,46.496284),(14.735255,46.493339),(14.726367,46.497706),(14.709727,46.492512),(14.703733,46.487758),(14.698772,46.480962),(14.687093,46.471221),(14.679961,46.458871),(14.672546,46.459791),(14.666629,46.460524),(14.662081,46.459698),(14.642031,46.445228),(14.631696,46.440577),(14.621567,46.43851),(14.599863,46.437167),(14.590148,46.434428),(14.575368,46.419726),(14.566997,46.400373),(14.562108,46.391737),(14.557695,46.38394),(14.540332,46.378643),(14.527102,46.388229),(14.515837,46.40536),(14.502194,46.418356),(14.467675,46.412672),(14.450931,46.414481),(14.437145,46.418884),(14.420029,46.424351),(14.414448,46.429002),(14.411244,46.434635),(14.40649,46.439337),(14.395844,46.440991),(14.362151,46.435875),(14.242323,46.438237),(14.149865,46.440061),(14.147933,46.440425),(14.137255,46.442438),(14.081032,46.47595),(14.066355,46.48104),(14.050026,46.484399),(14.032456,46.484709),(14.014922,46.482531),(13.998763,46.480523),(13.98233,46.481918),(13.890862,46.511787),(13.860683,46.51525),(13.795778,46.507886),(13.782135,46.507782),(13.716093,46.518867),(13.700951,46.519746),(13.685345,46.517627),(13.670256,46.518712),(13.549229,46.545842),(13.506751,46.546927),(13.499103,46.550622),(13.484944,46.561733),(13.478019,46.563567),(13.417041,46.560492),(13.373323,46.565789),(13.271417,46.550777),(13.231109,46.552173),(13.210025,46.558012),(13.146463,46.584961),(13.064504,46.598035),(12.83041,46.609637),(12.773566,46.635191),(12.7485,46.640996),(12.739873,46.642994),(12.732018,46.64227),(12.715999,46.637955),(12.706593,46.637749),(12.697395,46.640539),(12.679205,46.650022),(12.671368,46.652466),(12.669593,46.653019),(12.62019,46.656481),(12.562003,46.65121),(12.547327,46.652192),(12.53079,46.65767),(12.499888,46.672139),(12.469632,46.675799),(12.445627,46.678702),(12.40501,46.690123),(12.370387,46.711155),(12.351473,46.743246),(12.342688,46.765131),(12.326048,46.772495),(12.305274,46.774459),(12.284397,46.779988),(12.274785,46.784639),(12.269101,46.788566),(12.266724,46.795336),(12.26693,46.808255),(12.269618,46.819831),(12.273338,46.826755),(12.276129,46.83399),(12.275819,46.846289),(12.26662,46.868148),(12.250807,46.875615),(12.208743,46.876623),(12.1951,46.880085),(12.189002,46.884943),(12.183525,46.891222),(12.172156,46.89918),(12.160994,46.90303),(12.137429,46.905924),(12.126887,46.908869),(12.141563,46.918843),(12.141667,46.927989),(12.134742,46.937446),(12.128231,46.948582),(12.122133,46.971656),(12.117792,46.983076),(12.111178,46.992998),(12.12172,47.010517),(12.182284,47.033616),(12.203575,47.053331),(12.203897,47.067012),(12.204195,47.079686),(12.180631,47.085215),(12.133146,47.078832),(12.116242,47.076559),(12.014956,47.040489),(11.943643,47.038164),(11.899511,47.027725),(11.857033,47.012015),(11.822513,46.993257),(11.777141,46.988296),(11.766082,46.983438),(11.746445,46.972457),(11.73487,46.970622),(11.716163,46.97548),(11.683814,46.991913),(11.664797,46.993257),(11.657252,46.992533),(11.649707,46.992998),(11.648467,46.993257),(11.64795,46.993257),(11.59648,47.00044),(11.572709,46.998915),(11.543667,46.993102),(11.543564,46.993102),(11.543357,46.992998),(11.543254,46.992998),(11.533849,46.989536),(11.524443,46.988347),(11.515348,46.989484),(11.506977,46.992998),(11.501706,46.997856),(11.495918,47.001602),(11.489407,47.004212),(11.471837,47.00708),(11.461812,47.005504),(11.452923,47.000879),(11.445275,46.993102),(11.445172,46.992998),(11.411169,46.970493),(11.380576,46.971553),(11.349467,46.98194),(11.31381,46.987262),(11.243737,46.979252),(11.174491,46.963853),(11.156611,46.956515),(11.091912,46.912435),(11.083643,46.900136),(11.073101,46.86497),(11.054291,46.834145),(11.053464,46.830321),(11.054601,46.819934),(11.052844,46.814921),(11.048503,46.811692),(11.037445,46.808255),(11.033311,46.805568),(11.010883,46.779213),(10.99693,46.76911),(10.982668,46.767741),(10.965718,46.771642),(10.930888,46.773994),(10.913731,46.77234),(10.880348,46.764589),(10.870426,46.764072),(10.860195,46.766991),(10.843141,46.777043),(10.83415,46.780246),(10.823918,46.78035),(10.804901,46.776629),(10.795082,46.776888),(10.76635,46.788101),(10.754775,46.790763),(10.733691,46.786189),(10.722942,46.786448),(10.716947,46.795207),(10.721598,46.800142),(10.743819,46.812518),(10.748677,46.819443),(10.738962,46.829546),(10.662481,46.860965),(10.647081,46.863756),(10.629408,46.862412),(10.527708,46.843214),(10.486264,46.846366),(10.453811,46.864427),(10.451434,46.88577),(10.463836,46.919747),(10.458462,46.936619),(10.449574,46.943906),(10.415571,46.962406),(10.394693,46.985402),(10.384358,46.992998),(10.384358,46.993153),(10.384255,46.993153),(10.378984,46.995505),(10.373403,46.996254),(10.367925,46.995505),(10.338883,46.98411),(10.313665,46.964318),(10.296198,46.941374),(10.295681,46.922693),(10.270773,46.921892),(10.251343,46.92538),(10.235116,46.923313),(10.219924,46.905769),(10.215169,46.893108),(10.214342,46.884685),(10.211655,46.877036),(10.201423,46.86683),(10.157808,46.851612),(10.13197,46.846573),(10.125188,46.846751),(10.1113,46.847116),(10.068098,46.856624),(10.045567,46.865564),(10.006913,46.890757),(9.899943,46.914398),(9.875138,46.927421),(9.862426,46.939772),(9.860772,46.949151),(9.863976,46.959925),(9.866457,46.983387),(9.870591,46.992947),(9.870591,46.998838),(9.866767,47.001938),(9.860566,47.001602),(9.856328,47.004083),(9.857982,47.015478),(9.669053,47.056199),(9.65231,47.05793),(9.59991,47.053486),(9.581203,47.05687),(9.608798,47.080771),(9.615723,47.106764),(9.605594,47.132266),(9.581823,47.154901),(9.552057,47.16689),(9.551954,47.175571),(9.561876,47.190609),(9.562909,47.19774),(9.554331,47.211615),(9.544823,47.220323),(9.540378,47.229108),(9.547096,47.243035),(9.53025,47.253654),(9.521155,47.262801),(9.553298,47.299853),(9.587404,47.32781),(9.591228,47.334683),(9.596396,47.352305),(9.601047,47.36127),(9.639804,47.394524),(9.649519,47.409717),(9.650346,47.452092),(9.621717,47.469197),(9.58451,47.480721),(9.554951,47.5109),(9.553059,47.516891),(9.547482,47.534547),(9.549887,47.533534),(9.554144,47.532743),(9.612622,47.521881),(9.676701,47.52263),(9.703986,47.531415),(9.718455,47.546711),(9.730858,47.564901),(9.752665,47.582058),(9.767135,47.587277),(9.782121,47.588414),(9.79628,47.584952),(9.807959,47.576322),(9.811266,47.564488),(9.809303,47.552318),(9.813127,47.54175),(9.833487,47.534619),(9.841549,47.535032),(9.853848,47.540407),(9.858912,47.541337),(9.888988,47.533792),(9.918753,47.53219),(9.933843,47.534206),(9.945935,47.540769),(9.948726,47.524232),(9.959164,47.514879),(9.97167,47.506404),(9.980765,47.492942),(9.980765,47.489919),(9.981075,47.48687),(9.982625,47.481031),(9.993581,47.476586),(10.002366,47.47917),(10.011254,47.484441),(10.022623,47.487955),(10.071819,47.439121),(10.080604,47.42739),(10.075953,47.416874),(10.053215,47.40486),(10.064171,47.396178),(10.066858,47.385791),(10.067788,47.375042),(10.073162,47.365482),(10.082878,47.359074),(10.089596,47.359281),(10.097244,47.363131),(10.11037,47.367394),(10.11533,47.366955),(10.125666,47.362692),(10.130833,47.362666),(10.135071,47.364888),(10.146956,47.373802),(10.190881,47.379228),(10.209278,47.372484),(10.208555,47.354475),(10.190468,47.317475),(10.192742,47.310912),(10.193259,47.304917),(10.190778,47.298251),(10.18592,47.294995),(10.172691,47.292308),(10.168661,47.290422),(10.159462,47.279131),(10.155431,47.272878),(10.159876,47.271121),(10.174951,47.272376),(10.239354,47.277735),(10.261161,47.283497),(10.305913,47.302178),(10.325447,47.314322),(10.34343,47.330756),(10.365651,47.361115),(10.371646,47.367394),(10.381981,47.372122),(10.403065,47.377109),(10.411643,47.381037),(10.428283,47.396049),(10.442856,47.41628),(10.451951,47.438759),(10.451744,47.460205),(10.44761,47.472581),(10.442236,47.481392),(10.433657,47.487852),(10.419601,47.493045),(10.42973,47.532914),(10.43097,47.542164),(10.424356,47.553352),(10.415674,47.564385),(10.414227,47.573015),(10.429213,47.57702),(10.445956,47.579061),(10.453604,47.581102),(10.457118,47.579242),(10.461563,47.569914),(10.460322,47.565987),(10.455465,47.561181),(10.451744,47.554825),(10.453501,47.546246),(10.458772,47.541647),(10.466937,47.53772),(10.482543,47.532862),(10.525125,47.52847),(10.536907,47.529865),(10.549619,47.536789),(10.569876,47.555936),(10.583726,47.562473),(10.60791,47.562266),(10.739892,47.52847),(10.743923,47.525317),(10.74723,47.52015),(10.752191,47.515395),(10.760769,47.513638),(10.790328,47.516067),(10.830636,47.528573),(10.844795,47.531312),(10.858644,47.530666),(10.86929,47.526583),(10.892234,47.514879),(10.883862,47.508006),(10.851616,47.493097),(10.858438,47.485165),(10.909907,47.46899),(10.959827,47.432506),(10.962824,47.427959),(10.963134,47.421241),(10.96086,47.414316),(10.957656,47.409717),(10.955279,47.409614),(10.965511,47.396075),(10.979464,47.390545),(11.083747,47.389512),(11.10328,47.393568),(11.168599,47.424264),(11.193818,47.428941),(11.215212,47.422998),(11.213041,47.395816),(11.237019,47.393956),(11.258827,47.400725),(11.273089,47.411112),(11.286318,47.423205),(11.305645,47.4354),(11.324249,47.439431),(11.367657,47.440129),(11.383884,47.444857),(11.392462,47.454779),(11.388638,47.46222),(11.377786,47.467052),(11.365487,47.469326),(11.412616,47.506094),(11.435353,47.509711),(11.459951,47.507437),(11.482585,47.50258),(11.529508,47.507851),(11.551212,47.51369),(11.569402,47.527384),(11.574466,47.536634),(11.582321,47.559785),(11.589142,47.570379),(11.617048,47.586761),(11.620458,47.589654),(11.638958,47.589241),(11.682573,47.58335),(11.764015,47.583117),(11.820343,47.575288),(11.830678,47.577614),(11.840703,47.594615),(11.851969,47.599163),(11.935891,47.610738),(12.165645,47.603969),(12.173603,47.60508),(12.202128,47.629833),(12.205745,47.636215),(12.206779,47.645568),(12.205125,47.672285),(12.202335,47.677789),(12.197271,47.680088),(12.189106,47.685566),(12.181768,47.692129),(12.178564,47.697658),(12.1766,47.705823),(12.192413,47.710164),(12.216494,47.723703),(12.225176,47.727372),(12.239749,47.731687),(12.242229,47.732023),(12.239128,47.72745),(12.236545,47.716778),(12.233031,47.708872),(12.226623,47.702878),(12.223315,47.695901),(12.228793,47.684868),(12.238715,47.6789),(12.249567,47.680011),(12.271891,47.687891),(12.293699,47.69001),(12.351473,47.68169),(12.368423,47.683551),(12.407697,47.693731),(12.42413,47.69156),(12.428781,47.685798),(12.43922,47.664482),(12.444697,47.656291),(12.452759,47.649754),(12.482524,47.633579),(12.49658,47.628903),(12.517148,47.628541),(12.538232,47.631331),(12.553838,47.636008),(12.563375,47.641761),(12.598383,47.66288),(12.6174,47.669339),(12.653057,47.675127),(12.688713,47.675127),(12.744834,47.66536),(12.761991,47.666859),(12.751655,47.649392),(12.767365,47.63637),(12.793307,47.624794),(12.81315,47.611824),(12.785865,47.602677),(12.773669,47.5795),(12.77894,47.554825),(12.803642,47.541337),(12.819041,47.539528),(12.828446,47.537255),(12.836921,47.532655),(12.882603,47.498575),(12.931593,47.473512),(12.942548,47.470489),(12.951023,47.472478),(12.959291,47.475527),(12.96973,47.475734),(12.979342,47.472246),(12.984923,47.468267),(12.991124,47.465683),(13.001873,47.466019),(13.008797,47.470101),(13.037323,47.493045),(13.037013,47.501365),(13.034429,47.507437),(13.031225,47.512553),(13.028951,47.518031),(13.028124,47.524232),(13.028124,47.541647),(13.030605,47.544257),(13.040113,47.560457),(13.04125,47.561904),(13.03908,47.561336),(13.038253,47.584022),(13.04094,47.58335),(13.057063,47.597664),(13.057683,47.600429),(13.069569,47.620247),(13.072049,47.622417),(13.07422,47.63451),(13.07484,47.646654),(13.072049,47.659469),(13.064091,47.67399),(13.044557,47.696573),(13.032258,47.706624),(13.019856,47.712929),(13.004353,47.714608),(12.979032,47.707089),(12.964045,47.705358),(12.908338,47.71236),(12.892009,47.723548),(12.910612,47.742927),(12.910819,47.74303),(12.910819,47.743082),(12.917227,47.750161),(12.918984,47.756776),(12.919294,47.763184),(12.921257,47.76954),(12.926735,47.777395),(12.991227,47.847106),(12.986576,47.850259),(12.964976,47.871963),(12.931076,47.924879),(12.914229,47.93599),(12.906168,47.93891),(12.886014,47.952888),(12.862036,47.962552),(12.853975,47.970484),(12.8486,47.980948),(12.848246,47.982389),(12.845603,47.993144),(12.830617,48.015468),(12.760234,48.064871),(12.750932,48.074741),(12.742043,48.086782),(12.736979,48.099959),(12.738943,48.113447),(12.745041,48.12063),(12.751449,48.122103),(12.75796,48.121896),(12.764058,48.123756),(12.77832,48.138717),(12.782351,48.142024),(12.822142,48.160679),(12.835164,48.170704),(12.853458,48.18998),(12.862553,48.196646),(12.877642,48.202046),(12.93304,48.209255),(12.931696,48.211632),(12.947199,48.220004),(12.979445,48.232018),(13.016652,48.255893),(13.032775,48.263567),(13.136438,48.291059),(13.274207,48.307104),(13.307797,48.319636),(13.405569,48.376583),(13.419522,48.392164),(13.425206,48.413377),(13.42717,48.418544),(13.431717,48.423712),(13.436161,48.430585),(13.438228,48.440998),(13.436368,48.469239),(13.438228,48.478541),(13.442983,48.487713),(13.456418,48.506653),(13.459209,48.516394),(13.457039,48.525179),(13.447737,48.534816),(13.443706,48.552851),(13.440709,48.557347),(13.440089,48.560965),(13.445566,48.567915),(13.454558,48.573445),(13.487218,48.581584),(13.520704,48.584581),(13.624367,48.565357),(13.641834,48.559104),(13.65806,48.551094),(13.673356,48.535488),(13.714336,48.523215),(13.716506,48.521691),(13.725394,48.548381),(13.733869,48.559776),(13.73728,48.559259),(13.758054,48.561481),(13.766012,48.563755),(13.77428,48.569207),(13.779034,48.573935),(13.796191,48.598585),(13.802392,48.611711),(13.805493,48.626129),(13.806216,48.642923),(13.800945,48.675221),(13.803529,48.687159),(13.816862,48.695582),(13.804356,48.699561),(13.799395,48.70261),(13.78875,48.716511),(13.783582,48.71527),(13.785856,48.724779),(13.815725,48.76643),(13.855929,48.759299),(13.874946,48.752426),(13.893136,48.743021),(13.915047,48.73098),(13.98233,48.706485),(13.991115,48.700181),(14.007754,48.683283),(14.014162,48.675066),(14.019227,48.671914),(14.026151,48.670054),(14.032146,48.667263),(14.034729,48.661165),(14.032869,48.654912),(14.028322,48.653982),(14.023464,48.654602),(14.020467,48.653155),(14.013749,48.643802),(14.008064,48.642407),(14.006204,48.639513),(14.010855,48.62587),(14.015196,48.620393),(14.032456,48.60613),(14.040827,48.601169),(14.07483,48.591712),(14.216527,48.58117),(14.315746,48.557916),(14.325358,48.558639),(14.333213,48.560706),(14.353366,48.571429),(14.405353,48.586286),(14.421166,48.597061),(14.443593,48.636516),(14.458166,48.643182),(14.482144,48.624475),(14.522038,48.610626),(14.53382,48.608714),(14.548807,48.610936),(14.578779,48.620496),(14.594179,48.621323),(14.600586,48.617576),(14.601827,48.611194),(14.60193,48.604528),(14.605134,48.600316),(14.612885,48.599748),(14.628182,48.603288),(14.634693,48.602332),(14.645442,48.592901),(14.651333,48.583496),(14.659187,48.576959),(14.675621,48.576235),(14.69114,48.586535),(14.695671,48.589542),(14.700735,48.615535),(14.700219,48.646102),(14.703836,48.673103),(14.722749,48.693411),(14.775356,48.724004),(14.78042,48.743124),(14.790446,48.754855),(14.794476,48.766999),(14.800367,48.776507),(14.815457,48.780331),(14.86765,48.775577),(14.919327,48.761573),(14.939584,48.762813),(14.951056,48.780331),(14.948886,48.78214),(14.944441,48.788599),(14.940101,48.796506),(14.938137,48.80281),(14.93979,48.809787),(14.947025,48.822396),(14.968213,48.885028),(14.968316,48.912674),(14.964182,48.943267),(14.964595,48.971741),(14.978238,48.992825),(14.977308,48.998251),(14.977308,49.002902),(14.978858,49.00626),(14.982062,49.007914),(15.003973,49.009774),(15.059576,48.997217),(15.137401,48.993031),(15.144739,48.978924),(15.148977,48.965333),(15.148563,48.951742),(15.141949,48.937479),(15.161792,48.937221)] +Burundi [(30.415073,-2.313088),(30.418484,-2.311847),(30.423238,-2.317325),(30.428406,-2.331381),(30.434142,-2.339236),(30.488454,-2.383781),(30.521733,-2.399387),(30.5546,-2.400628),(30.521475,-2.442279),(30.508091,-2.463466),(30.47016,-2.55576),(30.461995,-2.58749),(30.457655,-2.598032),(30.448043,-2.610537),(30.424168,-2.633171),(30.416003,-2.645574),(30.412128,-2.670172),(30.423238,-2.680921),(30.442668,-2.681127),(30.463752,-2.674203),(30.499616,-2.657873),(30.52256,-2.649398),(30.516101,-2.668311),(30.458585,-2.72867),(30.450523,-2.741795),(30.447268,-2.757298),(30.446906,-2.782723),(30.425099,-2.812179),(30.413626,-2.8344),(30.415797,-2.851659),(30.440808,-2.884009),(30.456001,-2.898272),(30.474191,-2.903233),(30.469644,-2.914188),(30.475431,-2.922146),(30.484113,-2.930724),(30.488325,-2.94316),(30.488454,-2.94354),(30.49326,-2.941266),(30.513052,-2.913981),(30.524937,-2.904163),(30.53889,-2.898995),(30.547003,-2.900339),(30.61196,-2.939199),(30.62519,-2.94478),(30.62953,-2.947674),(30.632114,-2.953772),(30.634905,-2.970102),(30.637592,-2.974443),(30.650925,-2.977337),(30.661363,-2.974753),(30.671337,-2.970825),(30.683687,-2.969998),(30.695986,-2.974546),(30.718879,-2.989532),(30.732057,-2.99346),(30.755414,-2.991393),(30.80213,-2.97837),(30.825488,-2.978577),(30.818815,-2.990495),(30.815824,-2.995837),(30.78487,-3.031493),(30.778255,-3.04741),(30.783216,-3.062086),(30.807814,-3.085237),(30.817684,-3.099086),(30.817839,-3.104357),(30.813085,-3.11769),(30.812258,-3.123477),(30.815824,-3.131022),(30.828795,-3.140531),(30.832929,-3.147662),(30.833962,-3.160271),(30.832205,-3.172777),(30.823524,-3.196651),(30.809364,-3.216805),(30.809881,-3.224143),(30.814325,-3.24192),(30.814842,-3.247708),(30.799753,-3.274579),(30.775671,-3.291012),(30.747921,-3.294113),(30.721463,-3.280987),(30.718414,-3.292873),(30.712006,-3.301865),(30.702239,-3.307342),(30.68994,-3.308893),(30.662603,-3.319228),(30.640538,-3.33287),(30.621676,-3.35044),(30.603279,-3.372558),(30.61025,-3.376359),(30.640434,-3.392815),(30.639142,-3.41948),(30.612891,-3.444905),(30.565968,-3.466712),(30.552946,-3.482215),(30.544988,-3.48914),(30.538115,-3.49162),(30.511502,-3.497718),(30.509951,-3.500199),(30.508246,-3.50888),(30.507161,-3.511051),(30.48804,-3.512601),(30.487524,-3.510741),(30.48556,-3.510534),(30.46892,-3.513014),(30.466336,-3.516115),(30.462616,-3.527897),(30.460135,-3.531928),(30.432282,-3.551875),(30.426856,-3.563864),(30.429233,-3.583914),(30.428871,-3.602208),(30.421378,-3.620811),(30.38014,-3.685614),(30.373112,-3.703597),(30.373068,-3.704339),(30.371975,-3.722924),(30.376213,-3.739771),(30.381897,-3.75517),(30.384998,-3.770776),(30.381587,-3.788346),(30.33699,-3.773774),(30.31172,-3.789897),(30.273273,-3.856249),(30.22077,-3.909993),(30.208781,-3.930457),(30.191418,-4.002494),(30.175852,-4.039992),(30.173228,-4.046315),(30.149766,-4.08683),(30.119587,-4.12352),(30.051478,-4.180157),(30.041143,-4.19504),(30.015614,-4.256018),(30.003005,-4.271935),(29.980785,-4.28444),(29.970656,-4.292605),(29.936343,-4.312035),(29.900479,-4.345625),(29.847356,-4.370533),(29.838571,-4.37353),(29.821828,-4.37074),(29.810976,-4.365262),(29.80033,-4.363919),(29.784104,-4.37384),(29.782295,-4.377354),(29.780177,-4.393684),(29.775009,-4.402262),(29.757956,-4.410221),(29.751651,-4.416112),(29.747931,-4.430271),(29.747414,-4.44319),(29.744365,-4.454456),(29.732841,-4.463344),(29.728293,-4.461587),(29.687469,-4.458383),(29.638273,-4.446808),(29.404179,-4.449805),(29.37276,-4.208166),(29.367224,-4.189048),(29.33948,-4.093237),(29.277727,-3.988954),(29.239796,-3.948027),(29.224552,-3.926529),(29.215043,-3.899864),(29.206568,-3.781939),(29.217123,-3.709266),(29.22326,-3.66701),(29.225172,-3.588565),(29.206672,-3.334524),(29.208739,-3.305275),(29.213079,-3.291943),(29.236117,-3.270704),(29.240881,-3.266311),(29.223828,-3.252669),(29.216904,-3.233135),(29.214113,-3.166472),(29.217524,-3.146215),(29.225017,-3.129369),(29.237781,-3.122237),(29.242328,-3.110765),(29.241539,-3.104188),(29.234629,-3.046583),(29.225378,-3.031803),(29.211529,-3.021158),(29.194476,-3.014647),(29.175976,-3.012477),(29.156752,-3.004828),(29.148174,-2.986948),(29.144246,-2.966278),(29.138407,-2.950361),(29.123369,-2.940233),(29.105076,-2.934135),(29.08978,-2.926074),(29.083268,-2.910054),(29.081718,-2.893931),(29.077584,-2.884009),(29.062701,-2.862201),(29.053503,-2.835433),(29.049058,-2.827475),(29.034072,-2.82086),(29.013143,-2.818897),(28.994902,-2.813626),(28.986892,-2.796779),(28.988804,-2.775488),(28.995522,-2.758435),(29.008234,-2.74748),(29.028595,-2.744896),(29.015365,-2.720711),(29.032574,-2.620666),(29.05526,-2.598445),(29.113447,-2.594621),(29.129725,-2.596998),(29.190445,-2.623456),(29.212769,-2.630278),(29.254989,-2.635549),(29.276125,-2.640613),(29.294418,-2.651052),(29.306614,-2.660147),(29.307337,-2.664797),(29.306045,-2.67658),(29.309481,-2.690127),(29.309714,-2.691049),(29.331418,-2.71141),(29.337516,-2.723812),(29.335346,-2.735181),(29.321497,-2.752131),(29.318086,-2.762569),(29.32036,-2.774145),(29.329661,-2.794919),(29.33085,-2.807424),(29.339687,-2.826441),(29.364285,-2.824168),(29.407435,-2.805461),(29.425366,-2.803394),(29.444675,-2.806989),(29.480867,-2.813729),(29.504121,-2.826855),(29.523138,-2.820447),(29.541845,-2.808561),(29.563756,-2.805667),(29.574091,-2.806081),(29.582049,-2.80298),(29.589491,-2.798433),(29.619256,-2.787684),(29.626388,-2.788097),(29.637087,-2.791629),(29.678684,-2.805357),(29.697546,-2.808251),(29.719922,-2.805461),(29.729637,-2.79957),(29.733978,-2.790268),(29.737285,-2.769287),(29.744727,-2.759985),(29.756199,-2.759882),(29.778523,-2.7666),(29.803379,-2.767324),(29.823585,-2.763499),(29.842395,-2.752441),(29.862859,-2.731667),(29.876191,-2.71358),(29.888387,-2.692909),(29.897896,-2.671102),(29.90327,-2.649088),(29.903726,-2.637947),(29.907921,-2.5354),(29.929573,-2.459849),(29.92239,-2.382954),(29.928901,-2.331795),(29.928798,-2.322493),(29.931692,-2.316912),(29.941924,-2.316602),(29.949934,-2.321149),(29.956781,-2.327038),(29.95852,-2.328533),(29.976134,-2.34368),(29.983885,-2.344817),(29.992308,-2.34399),(30.00192,-2.3443),(30.013341,-2.348331),(30.025536,-2.357736),(30.056542,-2.394426),(30.072975,-2.408792),(30.093853,-2.422642),(30.116797,-2.43153),(30.139121,-2.430807),(30.156174,-2.419024),(30.20134,-2.362387),(30.203407,-2.355359),(30.203923,-2.345954),(30.207489,-2.339236),(30.219116,-2.340373),(30.250329,-2.355566),(30.262731,-2.358356),(30.278854,-2.357633),(30.297974,-2.353705),(30.316785,-2.347608),(30.331151,-2.340269),(30.344018,-2.328384),(30.34861,-2.322352),(30.352752,-2.316912),(30.362518,-2.307817),(30.37859,-2.303062),(30.383447,-2.305543),(30.415073,-2.313088)] +Belgium [(4.815447,51.431074),(4.822682,51.413685),(4.779067,51.426423),(4.767802,51.425389),(4.762117,51.413375),(4.783304,51.407638),(4.853274,51.40614),(4.871671,51.403039),(4.910325,51.391903),(4.931926,51.395624),(4.957144,51.410997),(4.980192,51.430531),(4.995798,51.446706),(5.002723,51.459005),(5.006133,51.468307),(5.012128,51.474327),(5.027527,51.476807),(5.030318,51.47443),(5.06029,51.461976),(5.079927,51.438541),(5.076413,51.421901),(5.065251,51.405907),(5.061737,51.384255),(5.068145,51.37454),(5.102561,51.350665),(5.107832,51.341984),(5.118064,51.319918),(5.122922,51.313381),(5.140182,51.307386),(5.178836,51.309505),(5.197336,51.308342),(5.214906,51.294338),(5.215423,51.258733),(5.232993,51.255916),(5.239456,51.25693),(5.270717,51.261833),(5.389056,51.258707),(5.409933,51.263539),(5.451894,51.282116),(5.471635,51.288085),(5.493235,51.286535),(5.516696,51.277233),(5.535197,51.26204),(5.542328,51.242661),(5.545532,51.219924),(5.551407,51.216767),(5.568373,51.207651),(5.62501,51.196902),(5.638549,51.190907),(5.648161,51.183957),(5.659013,51.179074),(5.67617,51.179177),(5.718028,51.184603),(5.730844,51.18375),(5.747173,51.177601),(5.768051,51.159462),(5.781383,51.152409),(5.812079,51.157318),(5.829132,51.156465),(5.83802,51.143727),(5.840501,51.138921),(5.826445,51.130162),(5.823241,51.118741),(5.845979,51.103393),(5.83678,51.099492),(5.826652,51.096908),(5.815903,51.095875),(5.804948,51.096624),(5.807428,51.089389),(5.808978,51.08634),(5.811769,51.082309),(5.804327,51.068589),(5.79885,51.061484),(5.791305,51.054998),(5.779626,51.061303),(5.771461,51.058125),(5.766397,51.048436),(5.76402,51.035181),(5.777559,51.023553),(5.761746,50.999162),(5.722369,50.959423),(5.744693,50.963247),(5.754305,50.963092),(5.76402,50.959423),(5.752548,50.947227),(5.735288,50.923973),(5.722369,50.911674),(5.717614,50.909245),(5.706039,50.90661),(5.694463,50.903664),(5.622323,50.852659),(5.624493,50.830284),(5.64072,50.815039),(5.657432,50.807641),(5.663251,50.805065),(5.684852,50.798503),(5.700458,50.795557),(5.698701,50.783775),(5.688366,50.76083),(5.706556,50.754216),(5.717408,50.752821),(5.727846,50.754164),(5.732777,50.758851),(5.743556,50.769099),(5.752444,50.773439),(5.76247,50.771011),(5.782933,50.756696),(5.793165,50.752149),(5.879878,50.753544),(5.891288,50.751246),(5.891321,50.75124),(5.902203,50.749048),(5.974756,50.74755),(5.99491,50.749927),(5.993256,50.748066),(5.992636,50.746258),(5.99274,50.744449),(5.994187,50.742589),(6.010826,50.737059),(6.011343,50.727447),(6.007623,50.716905),(6.01155,50.708534),(6.024779,50.707552),(6.064363,50.714477),(6.081003,50.713495),(6.100227,50.701299),(6.161102,50.642181),(6.147769,50.636755),(6.159448,50.622389),(6.180015,50.615516),(6.249055,50.614431),(6.251329,50.608437),(6.244301,50.599548),(6.232932,50.590763),(6.232932,50.587146),(6.210608,50.578671),(6.179085,50.561979),(6.160688,50.543583),(6.177328,50.530198),(6.169473,50.522499),(6.170817,50.517951),(6.178878,50.515781),(6.190557,50.515212),(6.182599,50.506582),(6.19066,50.501828),(6.199239,50.490408),(6.207197,50.486325),(6.217119,50.486015),(6.250915,50.492681),(6.253086,50.494077),(6.255566,50.494593),(6.25784,50.494077),(6.260114,50.492681),(6.27541,50.48803),(6.318715,50.481674),(6.336905,50.481003),(6.325743,50.474026),(6.323262,50.466016),(6.326156,50.457025),(6.331634,50.446896),(6.340212,50.437853),(6.347343,50.436974),(6.351167,50.433564),(6.350444,50.416872),(6.343313,50.393308),(6.336388,50.379923),(6.337732,50.368089),(6.372458,50.329436),(6.3734,50.32308),(6.374525,50.315483),(6.362019,50.306956),(6.335458,50.303856),(6.322022,50.305406),(6.310653,50.308248),(6.299491,50.308868),(6.285952,50.303856),(6.27603,50.296104),(6.269415,50.286492),(6.267348,50.276416),(6.271792,50.267424),(6.253396,50.256778),(6.210401,50.249027),(6.189627,50.242722),(6.179808,50.233007),(6.157174,50.222724),(6.146839,50.214042),(6.160171,50.204534),(6.165752,50.193165),(6.165132,50.181899),(6.159138,50.172443),(6.151696,50.169342),(6.131749,50.168308),(6.123171,50.164898),(6.121311,50.16159),(6.118003,50.150377),(6.115833,50.145467),(6.125962,50.140196),(6.129476,50.134047),(6.126582,50.127329),(6.117487,50.120456),(6.102604,50.124745),(6.10033,50.132652),(6.101467,50.142108),(6.097126,50.151307),(6.084207,50.159317),(6.076456,50.158645),(6.070048,50.154201),(6.061366,50.150738),(6.038628,50.148413),(6.027363,50.149447),(6.022967,50.15082),(6.014134,50.153581),(6.008036,50.160919),(6.004729,50.170479),(5.998321,50.174975),(5.982921,50.167068),(5.96163,50.165621),(5.953156,50.156371),(5.949021,50.142677),(5.941063,50.128311),(5.92618,50.118337),(5.888973,50.106607),(5.872437,50.096892),(5.86882,50.090535),(5.866856,50.074309),(5.864375,50.067488),(5.858278,50.061803),(5.843705,50.053638),(5.83771,50.047437),(5.838847,50.035862),(5.838847,50.026457),(5.836367,50.01855),(5.829132,50.013589),(5.808875,50.007801),(5.802674,50.002479),(5.801537,49.988681),(5.808565,49.9831),(5.812596,49.977571),(5.80164,49.963825),(5.793372,49.958812),(5.778593,49.95659),(5.771565,49.953541),(5.76371,49.946823),(5.759989,49.941604),(5.757612,49.93654),(5.753581,49.930183),(5.718958,49.891374),(5.714927,49.881866),(5.726709,49.878197),(5.748827,49.867293),(5.758129,49.858198),(5.731877,49.859955),(5.73198,49.854167),(5.730533,49.85117),(5.728673,49.848845),(5.726916,49.845227),(5.736528,49.83944),(5.724642,49.834272),(5.720198,49.824712),(5.721645,49.812775),(5.727743,49.800217),(5.738285,49.788848),(5.748414,49.785644),(5.759162,49.784456),(5.771875,49.779392),(5.778489,49.773035),(5.802054,49.742856),(5.803604,49.738154),(5.805774,49.724201),(5.805361,49.721876),(5.811769,49.718517),(5.825721,49.715778),(5.831716,49.713607),(5.837297,49.712936),(5.843705,49.714331),(5.850216,49.714021),(5.856004,49.708078),(5.856403,49.705807),(5.857347,49.70043),(5.853627,49.696296),(5.848769,49.69273),(5.847219,49.687201),(5.846702,49.68069),(5.843705,49.677486),(5.842258,49.67392),(5.845772,49.666582),(5.852593,49.663223),(5.872644,49.661724),(5.879878,49.657745),(5.885459,49.643534),(5.879878,49.634853),(5.869957,49.628858),(5.861998,49.62276),(5.849183,49.599713),(5.841224,49.589687),(5.829959,49.582763),(5.846909,49.576717),(5.8374,49.56111),(5.814456,49.545142),(5.790685,49.537753),(5.779006,49.539613),(5.757095,49.548398),(5.746346,49.549431),(5.735598,49.545711),(5.722679,49.5346),(5.710276,49.531086),(5.688676,49.533515),(5.667178,49.54044),(5.644647,49.543799),(5.619739,49.53553),(5.607854,49.524937),(5.60558,49.517754),(5.602273,49.513723),(5.587493,49.512431),(5.578915,49.51393),(5.547599,49.523438),(5.528789,49.517961),(5.503674,49.505093),(5.477732,49.495223),(5.456132,49.498944),(5.44993,49.507832),(5.450964,49.516772),(5.454168,49.526177),(5.454375,49.536616),(5.450241,49.545762),(5.402078,49.602451),(5.391166,49.608817),(5.383475,49.613304),(5.352469,49.620693),(5.3411,49.625137),(5.334899,49.625809),(5.331178,49.623122),(5.326941,49.617644),(5.32167,49.61258),(5.314538,49.611133),(5.297898,49.613252),(5.293041,49.617179),(5.290664,49.626378),(5.293764,49.640537),(5.301826,49.643896),(5.306373,49.647513),(5.299449,49.662499),(5.280122,49.677899),(5.259244,49.691335),(5.237644,49.690973),(5.192478,49.682808),(5.169431,49.687201),(5.151241,49.695986),(5.136358,49.705339),(5.093776,49.745337),(5.081891,49.752727),(5.046234,49.759393),(5.026287,49.766731),(4.98908,49.790554),(4.970993,49.797065),(4.968371,49.796981),(4.950013,49.796393),(4.906604,49.786471),(4.884177,49.784146),(4.858029,49.786575),(4.849037,49.794171),(4.849244,49.831585),(4.846763,49.837528),(4.836945,49.848018),(4.834878,49.85303),(4.836738,49.859128),(4.845626,49.865846),(4.848934,49.871066),(4.854308,49.883313),(4.859372,49.891684),(4.861956,49.900314),(4.860199,49.913337),(4.844593,49.931734),(4.797257,49.944291),(4.783925,49.958295),(4.788265,49.974263),(4.826713,50.036017),(4.827643,50.046714),(4.826093,50.055034),(4.827229,50.064077),(4.84604,50.09069),(4.852137,50.091621),(4.862886,50.084076),(4.86516,50.101594),(4.870534,50.122265),(4.871878,50.139783),(4.862679,50.147586),(4.831364,50.1434),(4.820098,50.146449),(4.815551,50.161332),(4.788575,50.153374),(4.681915,50.083921),(4.676128,50.076066),(4.67344,50.066196),(4.675818,50.060201),(4.679538,50.055189),(4.680572,50.048316),(4.672924,50.01638),(4.665896,50.000515),(4.656697,49.989095),(4.645845,49.984495),(4.604194,49.980206),(4.464771,49.935609),(4.435109,49.93225),(4.427651,49.933572),(4.277599,49.960156),(4.215953,49.954408),(4.201118,49.953024),(4.187682,49.955505),(4.131665,49.974987),(4.132365,49.974946),(4.139623,49.974522),(4.138383,49.989198),(4.132905,50.004494),(4.128151,50.005786),(4.132285,50.017568),(4.137349,50.025061),(4.144584,50.030797),(4.162877,50.041805),(4.186028,50.05245),(4.210006,50.06015),(4.210316,50.066713),(4.203392,50.074671),(4.197397,50.084076),(4.180551,50.127226),(4.170629,50.130481),(4.15988,50.129396),(4.139933,50.123557),(4.125774,50.128156),(4.128151,50.146243),(4.147374,50.186343),(4.152749,50.204534),(4.157296,50.212698),(4.167942,50.222465),(4.191506,50.234299),(4.201325,50.241534),(4.203702,50.251973),(4.197604,50.258484),(4.185201,50.26546),(4.171766,50.270886),(4.162774,50.272953),(4.150268,50.269543),(4.149751,50.263135),(4.150992,50.256262),(4.143654,50.251611),(4.125257,50.25714),(4.097868,50.294812),(4.077198,50.306646),(4.055907,50.314553),(4.022731,50.338117),(4.0033,50.34406),(3.982113,50.342613),(3.918551,50.325456),(3.895917,50.325301),(3.878037,50.330779),(3.86088,50.338479),(3.84021,50.345197),(3.798662,50.348452),(3.755047,50.346334),(3.741508,50.34313),(3.733446,50.337239),(3.715463,50.317653),(3.711122,50.309333),(3.708435,50.306336),(3.703887,50.304321),(3.702234,50.305251),(3.700787,50.304683),(3.696859,50.298223),(3.661719,50.319307),(3.652934,50.360183),(3.652831,50.406847),(3.644356,50.445501),(3.62813,50.464363),(3.608286,50.476507),(3.586375,50.483483),(3.563741,50.486532),(3.499145,50.48679),(3.486743,50.492681),(3.496458,50.503792),(3.498008,50.511853),(3.491497,50.517073),(3.477131,50.519863),(3.462868,50.51914),(3.452636,50.514851),(3.442714,50.508494),(3.429175,50.501828),(3.385354,50.491648),(3.361066,50.489787),(3.343599,50.49294),(3.299881,50.507461),(3.286755,50.514075),(3.271045,50.526943),(3.265878,50.538312),(3.264327,50.570196),(3.237869,50.626472),(3.229808,50.65541),(3.24376,50.672102),(3.232391,50.695925),(3.224433,50.705175),(3.210997,50.707965),(3.196941,50.709206),(3.188983,50.715097),(3.176684,50.734114),(3.162823,50.749925),(3.162811,50.749939),(3.146195,50.768892),(3.129245,50.779072),(3.103304,50.784085),(3.023817,50.768262),(2.971425,50.757833),(2.948378,50.748686),(2.930188,50.736232),(2.900112,50.703108),(2.886883,50.696648),(2.871276,50.698767),(2.866726,50.700091),(2.786734,50.723365),(2.768234,50.733235),(2.762963,50.739385),(2.749527,50.75959),(2.743842,50.766205),(2.706635,50.788787),(2.700021,50.79628),(2.69692,50.80305),(2.691856,50.808838),(2.67873,50.81323),(2.669222,50.813902),(2.650618,50.812248),(2.642453,50.812455),(2.627777,50.814419),(2.620439,50.816486),(2.615168,50.82217),(2.60659,50.834831),(2.602042,50.838087),(2.58995,50.842014),(2.586746,50.845425),(2.58778,50.850024),(2.595531,50.861134),(2.596668,50.867646),(2.576928,50.911519),(2.582715,50.920666),(2.605763,50.935238),(2.611654,50.941233),(2.60845,50.961128),(2.592741,50.976114),(2.55698,51.001177),(2.547059,51.020479),(2.537033,51.06461),(2.5218,51.087541),(2.521821,51.087551),(2.542003,51.096869),(2.579926,51.104804),(2.715668,51.169501),(2.923595,51.246487),(3.124848,51.329657),(3.348969,51.375149),(3.349415,51.375223),(3.358068,51.336713),(3.353934,51.311908),(3.353211,51.288602),(3.36737,51.263487),(3.391658,51.24677),(3.398798,51.244698),(3.421941,51.237985),(3.453877,51.235633),(3.501212,51.240284),(3.504106,51.249741),(3.501522,51.262402),(3.503383,51.273952),(3.514338,51.281031),(3.529014,51.283331),(3.545034,51.283925),(3.584825,51.289532),(3.609526,51.290436),(3.635054,51.287827),(3.757321,51.262324),(3.776234,51.254521),(3.781402,51.241421),(3.779748,51.230311),(3.781402,51.219872),(3.796285,51.208762),(3.808687,51.205325),(3.927026,51.206255),(3.949454,51.210777),(3.995032,51.23323),(4.118952,51.272401),(4.168562,51.29687),(4.207939,51.330589),(4.21352,51.340537),(4.220962,51.363817),(4.221491,51.368001),(4.221528,51.367987),(4.222179,51.367743),(4.241059,51.356594),(4.249034,51.346666),(4.256114,51.328437),(4.272634,51.315823),(4.290782,51.305894),(4.302989,51.296088),(4.306651,51.28974),(4.308116,51.283637),(4.305431,51.276597),(4.298676,51.269599),(4.296723,51.267564),(4.30836,51.271226),(4.316417,51.276842),(4.330821,51.296088),(4.29713,51.305162),(4.289317,51.309149),(4.285411,51.31802),(4.286143,51.339667),(4.283051,51.350084),(4.275645,51.3581),(4.261078,51.369371),(4.261068,51.369379),(4.281124,51.368947),(4.290105,51.368752),(4.326278,51.356427),(4.345605,51.352293),(4.392217,51.350407),(4.411234,51.356737),(4.416195,51.373971),(4.410097,51.38379),(4.38767,51.400533),(4.379918,51.409525),(4.380642,51.419653),(4.387153,51.427456),(4.38922,51.434588),(4.377128,51.442882),(4.429424,51.461976),(4.483064,51.474327),(4.512623,51.477376),(4.525232,51.475929),(4.533191,51.467583),(4.531847,51.450866),(4.523372,51.438412),(4.521512,51.428645),(4.540632,51.420067),(4.553861,51.419291),(4.582697,51.422599),(4.614633,51.41756),(4.630136,51.418206),(4.643571,51.42141),(4.653907,51.426216),(4.731215,51.485618),(4.764184,51.496238),(4.778654,51.495359),(4.796844,51.491018),(4.813587,51.48412),(4.823922,51.475645),(4.826093,51.460555),(4.819891,51.446215),(4.815447,51.431074)] +Benin [(3.341635,11.882893),(3.355071,11.889043),(3.369127,11.897311),(3.380909,11.895037),(3.391658,11.888836),(3.40282,11.88527),(3.447675,11.857985),(3.484262,11.84517),(3.495424,11.837521),(3.503486,11.829977),(3.536972,11.782899),(3.54369,11.77706),(3.549271,11.773753),(3.553715,11.76967),(3.557436,11.761815),(3.5625,11.746003),(3.566221,11.738354),(3.568185,11.729518),(3.571699,11.72187),(3.58007,11.717374),(3.589372,11.701509),(3.5964,11.695773),(3.574386,11.673035),(3.504933,11.556557),(3.493047,11.511495),(3.486743,11.496276),(3.466692,11.442481),(3.468449,11.419485),(3.483332,11.392303),(3.662546,11.143145),(3.686731,11.120485),(3.694999,11.119116),(3.704404,11.120795),(3.705583,11.120733),(3.713706,11.120304),(3.722077,11.112346),(3.711845,11.073511),(3.711845,11.051859),(3.715876,11.031886),(3.723318,11.013541),(3.733549,10.996462),(3.733549,10.996203),(3.754117,10.907139),(3.75298,10.877683),(3.731276,10.825025),(3.730035,10.807714),(3.7351,10.795776),(3.743161,10.787456),(3.751946,10.78017),(3.759491,10.771592),(3.764452,10.760869),(3.773857,10.728054),(3.783262,10.71697),(3.796078,10.71312),(3.810237,10.710484),(3.82295,10.70325),(3.832148,10.683923),(3.837109,10.654829),(3.837246,10.630609),(3.837419,10.599897),(3.829874,10.573077),(3.804346,10.523157),(3.795354,10.496492),(3.794321,10.444867),(3.787706,10.426367),(3.772513,10.407635),(3.756907,10.405257),(3.692828,10.438382),(3.671744,10.442077),(3.652107,10.435798),(3.632574,10.416652),(3.625959,10.407273),(3.623169,10.400917),(3.622135,10.381228),(3.618208,10.369342),(3.59299,10.330947),(3.572422,10.285704),(3.573352,10.266997),(3.587615,10.244182),(3.655621,10.174781),(3.664613,10.162197),(3.667507,10.146565),(3.66483,10.128654),(3.663993,10.123052),(3.654898,10.101116),(3.642392,10.091892),(3.627819,10.084993),(3.612317,10.070291),(3.605805,10.054116),(3.599604,9.968695),(3.589476,9.948825),(3.557126,9.910559),(3.530771,9.862965),(3.513098,9.846609),(3.452946,9.853095),(3.413982,9.842837),(3.342049,9.813588),(3.325926,9.801728),(3.317141,9.78173),(3.314867,9.759095),(3.318381,9.739432),(3.33068,9.708633),(3.338638,9.678222),(3.33161,9.652022),(3.299054,9.633961),(3.281381,9.634994),(3.261847,9.638612),(3.245724,9.633702),(3.238179,9.609053),(3.239419,9.602309),(3.24469,9.589571),(3.243554,9.584403),(3.182575,9.505338),(3.175134,9.498879),(3.159011,9.493478),(3.151776,9.488156),(3.132553,9.457279),(3.128625,9.429581),(3.135343,9.401675),(3.148055,9.370256),(3.151466,9.353565),(3.151363,9.288633),(3.145162,9.267859),(3.124594,9.228999),(3.091315,9.126421),(3.076019,9.095157),(3.051421,9.078207),(2.983104,9.060611),(2.971425,9.060508),(2.9614,9.065313),(2.951685,9.071902),(2.94073,9.07707),(2.929878,9.078362),(2.899182,9.07738),(2.888433,9.075364),(2.877788,9.068543),(2.869313,9.059707),(2.859701,9.052627),(2.846265,9.050767),(2.769164,9.057045),(2.772368,9.030535),(2.770197,8.967154),(2.76472,8.939766),(2.761516,8.933694),(2.75025,8.916873),(2.74777,8.914806),(2.749527,8.906615),(2.753764,8.904419),(2.758725,8.902662),(2.762549,8.895686),(2.762136,8.880415),(2.74963,8.851941),(2.745909,8.836516),(2.74715,8.827654),(2.752317,8.81228),(2.751077,8.803417),(2.745703,8.798146),(2.728856,8.790472),(2.723172,8.782928),(2.722862,8.772308),(2.722949,8.771218),(2.742439,8.527481),(2.744773,8.498294),(2.736918,8.464523),(2.72896,8.452741),(2.709012,8.430391),(2.703741,8.417885),(2.703535,8.409023),(2.706015,8.390548),(2.706015,8.381272),(2.698367,8.339854),(2.697127,8.320708),(2.700744,8.296885),(2.708496,8.274819),(2.728133,8.233194),(2.734954,8.211903),(2.734437,8.189966),(2.726582,8.171595),(2.716144,8.154904),(2.708392,8.13798),(2.705705,8.119686),(2.703535,8.059897),(2.690926,7.996567),(2.687102,7.933522),(2.683588,7.924634),(2.673356,7.905565),(2.671082,7.897917),(2.671695,7.893434),(2.672219,7.889597),(2.68028,7.87735),(2.691546,7.848669),(2.715524,7.822056),(2.724412,7.80769),(2.727203,7.793324),(2.717641,7.658549),(2.715317,7.625789),(2.720588,7.586566),(2.740638,7.549359),(2.770197,7.514271),(2.792315,7.477633),(2.789628,7.435826),(2.787251,7.426886),(2.786527,7.421254),(2.783323,7.419238),(2.761722,7.423527),(2.751284,7.423631),(2.743636,7.419238),(2.741155,7.408179),(2.762136,7.1622),(2.761102,7.14835),(2.757795,7.138894),(2.749837,7.128558),(2.740638,7.109593),(2.736298,7.102927),(2.735264,7.096002),(2.739088,7.08298),(2.747666,7.072231),(2.773918,7.050785),(2.779706,7.041328),(2.775778,7.030528),(2.765547,7.025102),(2.752731,7.02133),(2.741052,7.0158),(2.71511,6.979833),(2.718418,6.943557),(2.731957,6.905213),(2.736918,6.863303),(2.726272,6.821368),(2.723585,6.799715),(2.72865,6.780001),(2.736298,6.769846),(2.744773,6.763774),(2.755005,6.760725),(2.767407,6.759563),(2.773298,6.755041),(2.775262,6.744887),(2.778569,6.698765),(2.772988,6.687474),(2.760689,6.679387),(2.742602,6.663341),(2.73113,6.641844),(2.73082,6.621354),(2.739398,6.579186),(2.737331,6.558025),(2.718803,6.504298),(2.71542,6.494488),(2.71192,6.474048),(2.704258,6.429299),(2.703841,6.368352),(2.703298,6.368313),(2.306977,6.331488),(2.226817,6.313666),(1.884532,6.274807),(1.619663,6.213899),(1.612439,6.234763),(1.633109,6.245692),(1.761577,6.276026),(1.782351,6.277267),(1.76354,6.350647),(1.74411,6.425733),(1.718375,6.466997),(1.709797,6.473404),(1.6883,6.484437),(1.681685,6.489838),(1.679515,6.498571),(1.681995,6.520017),(1.680755,6.530068),(1.67445,6.538594),(1.657811,6.552263),(1.655227,6.559213),(1.653883,6.570659),(1.646648,6.573192),(1.63683,6.572261),(1.627425,6.573347),(1.61957,6.577998),(1.611198,6.58469),(1.603344,6.592596),(1.597349,6.600813),(1.594145,6.611045),(1.594765,6.620269),(1.596522,6.62864),(1.596316,6.636418),(1.58536,6.656442),(1.572441,6.666907),(1.566033,6.678146),(1.574301,6.700574),(1.589182,6.72046),(1.597039,6.73096),(1.60107,6.746437),(1.595179,6.77057),(1.587531,6.791034),(1.58319,6.808268),(1.58257,6.825347),(1.590114,6.867334),(1.588047,6.881235),(1.579056,6.892035),(1.564223,6.903089),(1.562623,6.904283),(1.551357,6.920871),(1.531927,6.991926),(1.607478,6.991409),(1.625668,6.996783),(1.624944,7.143648),(1.624324,7.288807),(1.623704,7.4381),(1.623251,7.536385),(1.622855,7.62246),(1.622774,7.640052),(1.621947,7.838282),(1.621327,7.996567),(1.623807,8.155886),(1.625564,8.270478),(1.614919,8.356261),(1.609855,8.365485),(1.603964,8.367449),(1.603945,8.367476),(1.60107,8.371635),(1.606031,8.396026),(1.606031,8.413389),(1.607581,8.421864),(1.640447,8.475815),(1.644168,8.493488),(1.63993,8.503255),(1.614092,8.535785),(1.609028,8.546973),(1.607581,8.558755),(1.607204,8.588617),(1.605514,8.722492),(1.604274,8.815225),(1.602827,8.924392),(1.601173,9.049526),(1.595799,9.076811),(1.587841,9.100893),(1.567273,9.13673),(1.505158,9.201455),(1.425267,9.284499),(1.420146,9.292317),(1.401185,9.321267),(1.386716,9.361135),(1.379895,9.462473),(1.368422,9.490843),(1.351266,9.482549),(1.336486,9.497509),(1.326358,9.521952),(1.323154,9.542313),(1.327288,9.555309),(1.345065,9.582388),(1.351266,9.595074),(1.352506,9.60528),(1.352196,9.625357),(1.354573,9.635304),(1.35602,9.647474),(1.348165,9.694164),(1.346925,9.771834),(1.345065,9.881103),(1.343942,9.954756),(1.343825,9.962442),(1.331009,9.996471),(1.225485,10.066053),(1.082238,10.16044),(0.997106,10.216553),(0.939508,10.254517),(0.844423,10.317175),(0.768769,10.367094),(0.760708,10.382158),(0.759881,10.405154),(0.773833,10.508145),(0.78851,10.563852),(0.789233,10.602946),(0.781275,10.693044),(0.795641,10.726478),(0.850005,10.77769),(0.862924,10.796086),(0.866024,10.80637),(0.867575,10.829211),(0.869538,10.839908),(0.885144,10.869389),(0.883698,10.880216),(0.875326,10.900524),(0.872949,10.911299),(0.875843,10.931169),(0.901474,10.992741),(0.931963,10.976101),(0.945399,10.974293),(0.963279,10.974137),(0.967517,10.979486),(0.969997,10.991165),(0.969067,11.002895),(0.963279,11.00827),(0.955321,11.01199),(0.951807,11.021086),(0.95129,11.032299),(0.952427,11.042402),(0.955941,11.044727),(0.964416,11.057414),(0.970307,11.06948),(0.976922,11.078369),(0.981366,11.080126),(0.986017,11.078498),(1.007928,11.075062),(1.014646,11.070359),(1.01971,11.065682),(1.025084,11.063512),(1.027978,11.061341),(1.033249,11.051575),(1.035006,11.049275),(1.041311,11.049585),(1.052163,11.055011),(1.05609,11.05607),(1.086889,11.052737),(1.097018,11.049275),(1.099705,11.044366),(1.106836,11.034418),(1.114174,11.027674),(1.117482,11.032196),(1.118619,11.048241),(1.10725,11.059791),(1.100945,11.070411),(1.099292,11.081289),(1.099085,11.092606),(1.097018,11.104491),(1.095158,11.10444),(1.085649,11.108729),(1.082755,11.110692),(1.082135,11.115292),(1.083892,11.12772),(1.082755,11.131802),(1.07335,11.133947),(1.067046,11.12834),(1.063325,11.126971),(1.062291,11.14206),(1.067459,11.144282),(1.100429,11.15162),(1.106216,11.155651),(1.113038,11.173867),(1.117482,11.178983),(1.128127,11.178879),(1.148074,11.1695),(1.158409,11.165935),(1.153242,11.183169),(1.149314,11.217249),(1.130608,11.247583),(1.135568,11.260115),(1.145904,11.269856),(1.156342,11.28554),(1.165954,11.277995),(1.173499,11.2666),(1.172155,11.261484),(1.18094,11.261355),(1.186211,11.263887),(1.191379,11.267014),(1.200061,11.268977),(1.218974,11.268047),(1.235201,11.263474),(1.268377,11.247893),(1.281089,11.271096),(1.28264,11.278873),(1.280366,11.288692),(1.270547,11.298278),(1.268377,11.30621),(1.273338,11.318225),(1.2845,11.312721),(1.296179,11.299621),(1.302483,11.288847),(1.309305,11.297089),(1.316333,11.29988),(1.323464,11.297089),(1.330389,11.288847),(1.347855,11.303497),(1.345892,11.316339),(1.336176,11.330033),(1.330389,11.347164),(1.33907,11.366103),(1.359017,11.378893),(1.381652,11.388686),(1.398085,11.398685),(1.388576,11.420002),(1.389817,11.428399),(1.398085,11.440311),(1.411107,11.449845),(1.423215,11.453748),(1.423613,11.453876),(1.433845,11.459405),(1.439633,11.473771),(1.455962,11.464702),(1.475289,11.460774),(1.56934,11.453307),(1.567273,11.446693),(1.565,11.432637),(1.563243,11.426022),(1.580606,11.427056),(1.601587,11.388686),(1.62112,11.395275),(1.638173,11.404215),(1.684372,11.413852),(1.703492,11.422921),(1.719099,11.427779),(1.763334,11.424678),(1.782247,11.426022),(1.821521,11.441602),(1.840642,11.445478),(1.858005,11.440311),(1.867617,11.446615),(1.879812,11.444858),(1.905754,11.432843),(1.948025,11.41672),(1.983372,11.414007),(2.010761,11.426901),(2.015429,11.43159),(2.14698,11.56374),(2.22005,11.622548),(2.273587,11.652055),(2.287436,11.665284),(2.291054,11.674792),(2.290227,11.685748),(2.290847,11.703214),(2.301596,11.731533),(2.340043,11.773856),(2.354099,11.799849),(2.390169,11.896536),(2.388205,11.906768),(2.376526,11.924079),(2.375183,11.935035),(2.380867,11.947902),(2.389342,11.950073),(2.398747,11.948781),(2.407429,11.95121),(2.414147,11.958703),(2.422002,11.973172),(2.428926,11.980355),(2.438951,11.981285),(2.44939,11.977513),(2.456728,11.979063),(2.457348,11.99622),(2.455281,12.008674),(2.44877,12.013376),(2.439262,12.015598),(2.428616,12.020353),(2.418281,12.028673),(2.41115,12.036786),(2.405568,12.046708),(2.371875,12.140449),(2.362057,12.196156),(2.36123,12.218894),(2.370015,12.236257),(2.3942,12.247212),(2.407946,12.248918),(2.43089,12.247987),(2.443706,12.249228),(2.446083,12.251811),(2.454351,12.263025),(2.459519,12.266746),(2.46696,12.265041),(2.472644,12.259253),(2.477709,12.257599),(2.483393,12.268038),(2.488044,12.280388),(2.495072,12.283231),(2.504477,12.281835),(2.516259,12.281422),(2.51905,12.279148),(2.5206,12.273671),(2.523701,12.269175),(2.531246,12.269691),(2.53693,12.27305),(2.546025,12.281629),(2.550573,12.284523),(2.586126,12.294031),(2.590053,12.293308),(2.598632,12.288347),(2.603593,12.28783),(2.60969,12.290775),(2.621679,12.299509),(2.628604,12.300801),(2.663847,12.291551),(2.672942,12.296718),(2.686895,12.318422),(2.690202,12.326174),(2.692166,12.334029),(2.69506,12.341212),(2.700434,12.347103),(2.711286,12.351237),(2.721828,12.344261),(2.732267,12.346793),(2.741465,12.354131),(2.757485,12.372941),(2.768544,12.380072),(2.778156,12.381933),(2.798413,12.381209),(2.808645,12.383225),(2.832416,12.384051),(2.844301,12.399244),(2.892257,12.357076),(2.912618,12.346328),(2.922643,12.335631),(2.930394,12.323332),(2.936802,12.300852),(2.944967,12.292481),(2.956026,12.286848),(2.967808,12.282559),(3.000984,12.27889),(3.009356,12.276306),(3.013077,12.271087),(3.022895,12.252483),(3.063409,12.193779),(3.235389,12.039525),(3.238489,12.034202),(3.252028,12.024125),(3.255129,12.018751),(3.257609,11.997408),(3.26257,11.974722),(3.269908,11.956532),(3.292749,11.918653),(3.307012,11.902375),(3.315487,11.894572),(3.327786,11.886407),(3.341635,11.882893)] +Burkina Faso [(-0.397671,15.002135),(-0.361472,15.017741),(-0.299176,15.054741),(-0.236699,15.065619),(-0.166988,15.049677),(-0.033404,14.995933),(-0.033197,14.995933),(0.218467,14.910977),(0.220844,14.888214),(0.211852,14.874804),(0.198003,14.863797),(0.185756,14.84819),(0.184102,14.819562),(0.213092,14.761632),(0.219707,14.731221),(0.152941,14.54671),(0.158832,14.496119),(0.18865,14.44775),(0.202382,14.435545),(0.346366,14.307577),(0.388792,14.251612),(0.391635,14.245927),(0.390601,14.237633),(0.381118,14.22567),(0.377424,14.216601),(0.372153,14.185647),(0.367398,14.173787),(0.343472,14.137536),(0.339157,14.12578),(0.343575,14.114333),(0.356262,14.095704),(0.362463,14.088857),(0.368742,14.084154),(0.37422,14.078883),(0.378457,14.070434),(0.37732,14.063949),(0.369052,14.047852),(0.369155,14.039067),(0.378354,14.028809),(0.391376,14.022582),(0.401866,14.013797),(0.403934,13.996072),(0.411866,13.978424),(0.428377,13.967495),(0.445921,13.958581),(0.456695,13.94685),(0.456256,13.938143),(0.448065,13.920108),(0.448634,13.909617),(0.454809,13.902357),(0.47566,13.888947),(0.483515,13.880343),(0.49199,13.861817),(0.499926,13.851015),(0.504082,13.845358),(0.519275,13.831637),(0.552968,13.810037),(0.5632,13.796782),(0.574466,13.785025),(0.593689,13.778101),(0.600511,13.768721),(0.594516,13.74986),(0.584801,13.729008),(0.58046,13.713789),(0.59431,13.688881),(0.620665,13.679967),(0.75337,13.684101),(0.763498,13.682034),(0.766392,13.67511),(0.766185,13.666971),(0.767012,13.660873),(0.766289,13.657178),(0.762878,13.653044),(0.763705,13.646429),(0.775694,13.635293),(0.795331,13.625965),(0.813108,13.625035),(0.850728,13.628601),(0.896927,13.614932),(0.95098,13.583203),(0.991391,13.541113),(0.996559,13.496051),(1.015266,13.465665),(1.048235,13.441945),(1.157169,13.392646),(1.175049,13.38691),(1.217424,13.381768),(1.23427,13.377583),(1.249567,13.367066),(1.268687,13.34606),(1.241505,13.335544),(1.222075,13.344613),(1.205745,13.358901),(1.187658,13.364121),(1.182387,13.358695),(1.177633,13.34761),(1.167608,13.313426),(1.160683,13.311307),(1.138462,13.320377),(1.084719,13.33358),(1.004724,13.364844),(0.983536,13.36841),(0.970411,13.328335),(0.970824,13.243534),(0.971341,13.149405),(0.971661,13.085782),(0.971754,13.067317),(0.974648,13.048326),(0.983536,13.032358),(1.012269,13.016881),(1.083789,13.01099),(1.113968,12.996236),(1.170915,12.949211),(1.230756,12.899756),(1.304806,12.838508),(1.330595,12.817177),(1.412037,12.749869),(1.467124,12.704394),(1.535854,12.647498),(1.563863,12.63215),(1.597039,12.623675),(1.699462,12.61489),(1.826069,12.604193),(1.843742,12.606105),(1.860692,12.61706),(1.872681,12.634217),(1.883223,12.653957),(1.900173,12.678452),(1.906994,12.691836),(1.911438,12.696642),(1.919603,12.698347),(1.926631,12.694989),(1.934486,12.693542),(1.945545,12.700828),(1.962495,12.719121),(1.971693,12.724237),(2.068948,12.716279),(2.109049,12.705634),(2.135197,12.67561),(2.140985,12.656128),(2.144809,12.650753),(2.155455,12.640005),(2.165997,12.631737),(2.18429,12.620626),(2.193282,12.609464),(2.2,12.595925),(2.203204,12.583316),(2.210852,12.523061),(2.215709,12.510349),(2.223874,12.494846),(2.229559,12.487146),(2.242271,12.473452),(2.246302,12.465907),(2.246302,12.461669),(2.243098,12.451179),(2.242994,12.446166),(2.244958,12.424152),(2.23793,12.413455),(2.223254,12.409735),(2.163826,12.404774),(2.158762,12.405652),(2.148013,12.410768),(2.145016,12.41175),(2.138711,12.408339),(2.131477,12.398521),(2.126929,12.39511),(2.069258,12.383328),(2.054169,12.370926),(2.051792,12.341935),(2.070912,12.306898),(2.113803,12.247987),(2.188527,12.145461),(2.258807,12.048775),(2.338079,11.940099),(2.390169,11.896536),(2.354099,11.799849),(2.340043,11.773856),(2.301596,11.731533),(2.290847,11.703214),(2.290227,11.685748),(2.291054,11.674792),(2.287436,11.665284),(2.273587,11.652055),(2.22005,11.622548),(2.14698,11.56374),(2.015429,11.43159),(2.010761,11.426901),(1.983372,11.414007),(1.948025,11.41672),(1.905754,11.432843),(1.879812,11.444858),(1.867617,11.446615),(1.858005,11.440311),(1.840642,11.445478),(1.821521,11.441602),(1.782247,11.426022),(1.763334,11.424678),(1.719099,11.427779),(1.703492,11.422921),(1.684372,11.413852),(1.638173,11.404215),(1.62112,11.395275),(1.601587,11.388686),(1.580606,11.427056),(1.563243,11.426022),(1.565,11.432637),(1.567273,11.446693),(1.56934,11.453307),(1.475289,11.460774),(1.455962,11.464702),(1.439633,11.473771),(1.433845,11.459405),(1.423613,11.453876),(1.423215,11.453748),(1.411107,11.449845),(1.398085,11.440311),(1.389817,11.428399),(1.388576,11.420002),(1.398085,11.398685),(1.381652,11.388686),(1.359017,11.378893),(1.33907,11.366103),(1.330389,11.347164),(1.336176,11.330033),(1.345892,11.316339),(1.347855,11.303497),(1.330389,11.288847),(1.323464,11.297089),(1.316333,11.29988),(1.309305,11.297089),(1.302483,11.288847),(1.296179,11.299621),(1.2845,11.312721),(1.273338,11.318225),(1.268377,11.30621),(1.270547,11.298278),(1.280366,11.288692),(1.28264,11.278873),(1.281089,11.271096),(1.268377,11.247893),(1.235201,11.263474),(1.218974,11.268047),(1.200061,11.268977),(1.191379,11.267014),(1.186211,11.263887),(1.18094,11.261355),(1.172155,11.261484),(1.173499,11.2666),(1.165954,11.277995),(1.156342,11.28554),(1.145904,11.269856),(1.135568,11.260115),(1.130608,11.247583),(1.149314,11.217249),(1.153242,11.183169),(1.158409,11.165935),(1.148074,11.1695),(1.128127,11.178879),(1.117482,11.178983),(1.113038,11.173867),(1.106216,11.155651),(1.100429,11.15162),(1.067459,11.144282),(1.062291,11.14206),(1.063325,11.126971),(1.067046,11.12834),(1.07335,11.133947),(1.082755,11.131802),(1.083892,11.12772),(1.082135,11.115292),(1.082755,11.110692),(1.085649,11.108729),(1.095158,11.10444),(1.097018,11.104491),(1.099085,11.092606),(1.099292,11.081289),(1.100945,11.070411),(1.10725,11.059791),(1.118619,11.048241),(1.117482,11.032196),(1.114174,11.027674),(1.106836,11.034418),(1.099705,11.044366),(1.097018,11.049275),(1.086889,11.052737),(1.05609,11.05607),(1.052163,11.055011),(1.041311,11.049585),(1.035006,11.049275),(1.033249,11.051575),(1.027978,11.061341),(1.025084,11.063512),(1.01971,11.065682),(1.014646,11.070359),(1.007928,11.075062),(0.986017,11.078498),(0.981366,11.080126),(0.976922,11.078369),(0.970307,11.06948),(0.964416,11.057414),(0.955941,11.044727),(0.952427,11.042402),(0.95129,11.032299),(0.951807,11.021086),(0.955321,11.01199),(0.963279,11.00827),(0.969067,11.002895),(0.969997,10.991165),(0.967517,10.979486),(0.963279,10.974137),(0.945399,10.974293),(0.931963,10.976101),(0.901474,10.992741),(0.77156,10.990364),(0.675338,10.988529),(0.612603,10.976566),(0.487649,10.933236),(0.499742,10.975688),(0.483619,10.979693),(0.487133,10.983672),(0.48858,10.987754),(0.489303,10.991914),(0.490647,10.996358),(0.49013,10.999511),(0.488786,11.001784),(0.486616,11.003205),(0.374779,11.025808),(0.287558,11.043436),(0.059588,11.089402),(-0.063118,11.114155),(-0.115062,11.124658),(-0.166109,11.13498),(-0.301682,11.162937),(-0.304473,11.146788),(-0.300545,11.137409),(-0.298297,11.128392),(-0.306126,11.113535),(-0.318813,11.101365),(-0.368939,11.065217),(-0.386923,11.086405),(-0.405888,11.101468),(-0.425215,11.101468),(-0.444387,11.077439),(-0.449038,11.062892),(-0.45126,11.040206),(-0.456686,11.029612),(-0.46852,11.020129),(-0.494823,11.007805),(-0.504228,10.996358),(-0.504228,10.996203),(-0.506709,10.993671),(-0.509706,10.991449),(-0.512961,10.989744),(-0.516475,10.988633),(-0.555439,10.98995),(-0.576213,10.987754),(-0.590579,10.982432),(-0.599829,10.965766),(-0.607116,10.94122),(-0.617244,10.918637),(-0.634763,10.907966),(-0.68277,10.953596),(-0.679773,10.960572),(-0.67228,10.968505),(-0.667991,10.976515),(-0.673933,10.983517),(-0.683235,10.984344),(-0.690212,10.981811),(-0.694656,10.982742),(-0.695999,10.994033),(-0.703017,10.994159),(-0.816974,10.996203),(-0.816974,10.996462),(-0.822141,11.003619),(-0.827774,11.005841),(-0.833665,11.003412),(-0.839401,10.996462),(-0.860795,10.992922),(-0.883171,10.968634),(-0.900586,10.966231),(-0.89392,10.979899),(-0.904359,10.979486),(-0.909475,10.98269),(-0.916451,10.996203),(-0.924047,11.001423),(-0.932471,11.003102),(-1.070343,11.013902),(-1.098714,11.009562),(-1.10755,10.994343),(-1.22806,10.99548),(-1.355287,10.99672),(-1.38035,11.001268),(-1.413553,11.01405),(-1.423584,11.017911),(-1.436057,11.022713),(-1.562406,11.026589),(-1.579821,11.021215),(-1.587004,11.003671),(-1.598683,10.997133),(-1.754074,10.995635),(-1.972872,10.993516),(-2.197871,10.991294),(-2.439769,10.988943),(-2.634951,10.986979),(-2.728798,10.986057),(-2.750706,10.985842),(-2.751119,10.996462),(-2.837341,10.998012),(-2.835946,10.992224),(-2.838711,10.985119),(-2.832716,10.966024),(-2.821761,10.948299),(-2.815767,10.929153),(-2.821399,10.90626),(-2.834861,10.89161),(-2.850803,10.879544),(-2.864239,10.864273),(-2.871629,10.843448),(-2.882662,10.797068),(-2.899482,10.76999),(-2.9031,10.747149),(-2.908577,10.737666),(-2.919274,10.727202),(-2.922323,10.721207),(-2.919959,10.711088),(-2.918913,10.706609),(-2.916949,10.706609),(-2.912634,10.705808),(-2.908035,10.703637),(-2.905141,10.699193),(-2.905115,10.692733),(-2.907337,10.688315),(-2.910154,10.684491),(-2.916251,10.668084),(-2.929661,10.644209),(-2.932504,10.634313),(-2.928266,10.615219),(-2.898294,10.562612),(-2.895865,10.552768),(-2.894211,10.531425),(-2.891524,10.521659),(-2.886899,10.515793),(-2.873386,10.502487),(-2.87044,10.497784),(-2.867391,10.47825),(-2.859562,10.465616),(-2.824448,10.435023),(-2.812149,10.428822),(-2.795871,10.426109),(-2.771428,10.425489),(-2.767552,10.4212),(-2.771893,10.41182),(-2.78104,10.402415),(-2.791995,10.398152),(-2.80494,10.397118),(-2.81463,10.393708),(-2.822329,10.387584),(-2.829461,10.378282),(-2.836282,10.363658),(-2.841295,10.343272),(-2.841295,10.322291),(-2.833182,10.305987),(-2.824448,10.302215),(-2.814216,10.301078),(-2.80569,10.299011),(-2.802098,10.292319),(-2.799669,10.280924),(-2.793184,10.278547),(-2.784451,10.278805),(-2.774839,10.27524),(-2.764865,10.269116),(-2.758354,10.266119),(-2.754943,10.260796),(-2.754375,10.247954),(-2.757682,10.234648),(-2.765124,10.225837),(-2.785381,10.210101),(-2.798455,10.189534),(-2.79892,10.169613),(-2.788481,10.124448),(-2.788481,10.038768),(-2.783572,10.018511),(-2.760886,9.984766),(-2.754375,9.966783),(-2.755564,9.941151),(-2.764865,9.895754),(-2.760524,9.870536),(-2.736753,9.832579),(-2.733911,9.822787),(-2.740267,9.812606),(-2.774839,9.77434),(-2.791582,9.740931),(-2.795303,9.720002),(-2.788481,9.699254),(-2.780317,9.690237),(-2.774839,9.689306),(-2.769258,9.689926),(-2.760524,9.685586),(-2.75546,9.677498),(-2.75422,9.668352),(-2.751016,9.659179),(-2.743404,9.653325),(-2.740164,9.650833),(-2.752928,9.639904),(-2.761506,9.625253),(-2.766467,9.608898),(-2.768018,9.592801),(-2.766261,9.576523),(-2.761093,9.566239),(-2.725281,9.533864),(-2.689211,9.488724),(-2.720475,9.459217),(-2.744918,9.408109),(-2.763677,9.391883),(-2.787603,9.393717),(-2.810418,9.411106),(-2.910154,9.542726),(-2.959892,9.638612),(-2.977824,9.685172),(-2.989994,9.708427),(-3.005677,9.724162),(-3.024643,9.728606),(-3.059059,9.718555),(-3.077094,9.719408),(-3.094561,9.733851),(-3.104379,9.757132),(-3.116911,9.805294),(-3.136936,9.82953),(-3.160112,9.834698),(-3.182566,9.835163),(-3.200007,9.845369),(-3.204399,9.865368),(-3.202229,9.888958),(-3.202565,9.908234),(-3.21476,9.915623),(-3.230237,9.90472),(-3.26406,9.857074),(-3.277857,9.843457),(-3.301732,9.841132),(-3.311809,9.854128),(-3.313566,9.874566),(-3.312739,9.894513),(-3.317183,9.901335),(-3.327518,9.897304),(-3.339249,9.890095),(-3.348344,9.886995),(-3.359015,9.892498),(-3.376559,9.908492),(-3.389168,9.913246),(-3.546471,9.926139),(-3.606261,9.918181),(-3.62334,9.921747),(-3.634011,9.92924),(-3.640548,9.937379),(-3.648171,9.944588),(-3.661658,9.948929),(-3.665043,9.946603),(-3.674319,9.93526),(-3.679641,9.932211),(-3.685946,9.933452),(-3.698994,9.94079),(-3.706462,9.941823),(-3.755037,9.93588),(-3.76961,9.931333),(-3.779739,9.924692),(-3.795758,9.907665),(-3.804259,9.901386),(-3.823483,9.895702),(-3.889706,9.894462),(-3.891625,9.8939),(-3.910093,9.888493),(-3.947739,9.86436),(-3.969185,9.858986),(-3.984042,9.851157),(-3.988253,9.834801),(-3.994609,9.820771),(-4.026675,9.820306),(-4.034969,9.817619),(-4.050317,9.810307),(-4.05135,9.807568),(-4.051712,9.802349),(-4.053675,9.79762),(-4.059877,9.796277),(-4.065716,9.798085),(-4.076155,9.803795),(-4.13305,9.820409),(-4.150259,9.818291),(-4.168294,9.804829),(-4.196974,9.764961),(-4.215423,9.754858),(-4.253095,9.750052),(-4.270329,9.743928),(-4.283636,9.732017),(-4.279191,9.731138),(-4.267513,9.727263),(-4.297795,9.695843),(-4.303299,9.686723),(-4.303454,9.677912),(-4.300482,9.654296),(-4.302988,9.641712),(-4.315933,9.616623),(-4.322477,9.608981),(-4.322548,9.608898),(-4.349678,9.620034),(-4.365026,9.624168),(-4.368023,9.608381),(-4.367817,9.591974),(-4.371175,9.582956),(-4.38487,9.589416),(-4.391588,9.600294),(-4.400166,9.633134),(-4.409106,9.647733),(-4.433368,9.653469),(-4.469206,9.651763),(-4.50171,9.655019),(-4.515921,9.67569),(-4.513647,9.694009),(-4.503829,9.719899),(-4.501607,9.737107),(-4.503958,9.742016),(-4.509953,9.745065),(-4.517782,9.746615),(-4.535946,9.746615),(-4.539098,9.744703),(-4.539899,9.740776),(-4.543051,9.734006),(-4.547961,9.720261),(-4.548426,9.706799),(-4.552198,9.696464),(-4.566719,9.692407),(-4.57832,9.696515),(-4.592712,9.717832),(-4.604029,9.727159),(-4.616949,9.713853),(-4.628731,9.707703),(-4.641107,9.706024),(-4.655809,9.706075),(-4.654724,9.703543),(-4.658264,9.6976),(-4.664827,9.690857),(-4.672914,9.685586),(-4.681079,9.682072),(-4.681803,9.682847),(-4.681544,9.686723),(-4.686608,9.692407),(-4.700096,9.69791),(-4.70547,9.702096),(-4.707641,9.70977),(-4.709088,9.720312),(-4.712808,9.726849),(-4.724384,9.737107),(-4.730017,9.744238),(-4.733324,9.749639),(-4.738337,9.753153),(-4.748646,9.754496),(-4.758594,9.753308),(-4.762134,9.749949),(-4.760454,9.745324),(-4.75477,9.740208),(-4.769963,9.737365),(-4.783192,9.758372),(-4.796447,9.754496),(-4.799315,9.762532),(-4.801227,9.766459),(-4.810115,9.77434),(-4.810115,9.781781),(-4.8004,9.78111),(-4.791899,9.78235),(-4.775905,9.788008),(-4.785414,9.804829),(-4.790323,9.827515),(-4.798462,9.842527),(-4.817557,9.836455),(-4.824274,9.856971),(-4.83846,9.86945),(-4.858329,9.875703),(-4.881739,9.877409),(-4.890059,9.879501),(-4.893495,9.884204),(-4.898017,9.888907),(-4.909386,9.891051),(-4.917085,9.886478),(-4.922356,9.87746),(-4.928015,9.870949),(-4.936671,9.873998),(-4.966075,9.901025),(-4.967315,9.912368),(-4.952949,9.953373),(-4.954809,9.959703),(-4.965713,9.983113),(-4.976927,9.997608),(-4.981397,10.007737),(-4.960209,10.008486),(-4.956411,10.021999),(-4.963336,10.040319),(-4.974524,10.055537),(-4.986642,10.063263),(-4.999793,10.069464),(-5.010516,10.07905),(-5.015581,10.097111),(-5.025528,10.088739),(-5.037233,10.083753),(-5.048343,10.085355),(-5.056483,10.097111),(-5.042866,10.103338),(-5.049377,10.105353),(-5.055139,10.10804),(-5.061702,10.110056),(-5.070099,10.110159),(-5.06103,10.118453),(-5.037595,10.126489),(-5.039455,10.134344),(-5.064725,10.150803),(-5.070719,10.161009),(-5.06395,10.179044),(-5.090718,10.177933),(-5.09015,10.188811),(-5.081597,10.206897),(-5.084414,10.227439),(-5.087333,10.220824),(-5.091907,10.214339),(-5.097849,10.209275),(-5.104903,10.206949),(-5.117073,10.208034),(-5.11821,10.212479),(-5.114412,10.218292),(-5.111802,10.223718),(-5.111802,10.27865),(-5.119192,10.293533),(-5.119882,10.293983),(-5.135677,10.304282),(-5.152058,10.305625),(-5.159499,10.292319),(-5.168439,10.291337),(-5.217584,10.31994),(-5.233552,10.315676),(-5.279802,10.32242),(-5.296649,10.32304),(-5.304142,10.318415),(-5.309206,10.311491),(-5.315511,10.305264),(-5.327086,10.302576),(-5.338507,10.302215),(-5.343778,10.303093),(-5.34817,10.305987),(-5.353338,10.308261),(-5.357058,10.304385),(-5.360831,10.298752),(-5.365533,10.295703),(-5.385739,10.296479),(-5.402534,10.300561),(-5.416228,10.307537),(-5.426977,10.316865),(-5.430077,10.322937),(-5.433488,10.331877),(-5.437829,10.340016),(-5.443772,10.34353),(-5.453022,10.343943),(-5.459946,10.345158),(-5.474157,10.350325),(-5.475294,10.367715),(-5.48749,10.392933),(-5.505111,10.415567),(-5.522578,10.425489),(-5.5171,10.439984),(-5.508987,10.490937),(-5.505163,10.499179),(-5.48165,10.535327),(-5.468111,10.58783),(-5.471057,10.597365),(-5.47855,10.604238),(-5.477413,10.61956),(-5.470488,10.635605),(-5.460566,10.644571),(-5.475914,10.650385),(-5.477154,10.658808),(-5.482115,10.675215),(-5.482632,10.684233),(-5.473072,10.737873),(-5.467956,10.752678),(-5.461186,10.760042),(-5.45297,10.763969),(-5.446459,10.768698),(-5.445012,10.778568),(-5.445322,10.786991),(-5.431421,10.844688),(-5.43509,10.856909),(-5.447027,10.877993),(-5.465579,10.947886),(-5.479118,10.975533),(-5.501804,10.97021),(-5.495603,10.996203),(-5.505602,11.047985),(-5.507334,11.056949),(-5.504233,11.072684),(-5.489764,11.082115),(-5.390131,11.094699),(-5.373026,11.099763),(-5.356283,11.106894),(-5.332512,11.12157),(-5.322332,11.135549),(-5.316182,11.176322),(-5.306726,11.199395),(-5.275255,11.230711),(-5.262852,11.251588),(-5.26094,11.267737),(-5.267865,11.37202),(-5.263472,11.390184),(-5.248176,11.403181),(-5.22642,11.41349),(-5.218307,11.422198),(-5.219186,11.435143),(-5.224612,11.45801),(-5.224663,11.54077),(-5.233914,11.575987),(-5.269467,11.604099),(-5.280732,11.605753),(-5.290964,11.604461),(-5.299956,11.606424),(-5.307604,11.617638),(-5.307811,11.629007),(-5.302281,11.639859),(-5.294788,11.650298),(-5.289259,11.660426),(-5.280681,11.699494),(-5.278562,11.721456),(-5.279544,11.739026),(-5.28714,11.758922),(-5.298974,11.772668),(-5.314787,11.782486),(-5.334269,11.790806),(-5.344656,11.793907),(-5.353286,11.794527),(-5.370649,11.791064),(-5.374525,11.79525),(-5.400415,11.810185),(-5.405996,11.812303),(-5.412197,11.823569),(-5.412352,11.82853),(-5.405944,11.830442),(-5.392302,11.832354),(-5.387909,11.830804),(-5.37561,11.822225),(-5.369874,11.820158),(-5.364552,11.82176),(-5.357058,11.829615),(-5.353803,11.831734),(-5.319025,11.837418),(-5.314839,11.83933),(-5.311066,11.842896),(-5.306519,11.846048),(-5.300163,11.846358),(-5.293445,11.843206),(-5.288019,11.839123),(-5.282489,11.836436),(-5.275151,11.837418),(-5.268227,11.843051),(-5.248228,11.870439),(-5.209471,11.901497),(-5.182909,11.930901),(-5.167819,11.943665),(-5.136994,11.953122),(-5.103947,11.972759),(-5.089323,11.97927),(-5.073768,11.98051),(-5.034313,11.97927),(-5.016046,11.981285),(-5.004703,11.980665),(-4.994238,11.983249),(-4.984523,11.988572),(-4.975454,11.996271),(-4.95375,12.005108),(-4.932071,12.003144),(-4.910677,11.998132),(-4.889955,11.997925),(-4.847245,12.013066),(-4.826238,12.012705),(-4.806808,11.996271),(-4.789522,12.001801),(-4.761281,12.006607),(-4.746889,12.015082),(-4.738983,12.02521),(-4.732281,12.037263),(-4.725133,12.050118),(-4.714617,12.059007),(-4.704023,12.061487),(-4.683715,12.059058),(-4.673586,12.05973),(-4.653716,12.069135),(-4.652737,12.071627),(-4.648678,12.081951),(-4.646663,12.098022),(-4.636121,12.117556),(-4.600309,12.137555),(-4.570233,12.138898),(-4.560466,12.149492),(-4.585529,12.197241),(-4.497421,12.26871),(-4.493545,12.272844),(-4.492253,12.278838),(-4.49401,12.290104),(-4.490496,12.305762),(-4.490703,12.313823),(-4.488171,12.320696),(-4.47719,12.327311),(-4.462488,12.328189),(-4.452669,12.320283),(-4.444814,12.309431),(-4.435823,12.301731),(-4.406135,12.307467),(-4.415359,12.350307),(-4.451636,12.435314),(-4.443678,12.459447),(-4.426624,12.479653),(-4.407607,12.497946),(-4.393448,12.516446),(-4.386885,12.530192),(-4.387247,12.533965),(-4.392647,12.537427),(-4.401406,12.550398),(-4.42249,12.597217),(-4.439078,12.613288),(-4.482538,12.646258),(-4.491323,12.662794),(-4.485251,12.714677),(-4.481711,12.717416),(-4.474942,12.715762),(-4.46437,12.717251),(-4.464296,12.717261),(-4.435254,12.728423),(-4.402026,12.736562),(-4.36885,12.738991),(-4.339446,12.73279),(-4.310921,12.716021),(-4.297227,12.712145),(-4.276298,12.714677),(-4.257642,12.72093),(-4.245472,12.728936),(-4.243974,12.729922),(-4.23413,12.74235),(-4.213278,12.807178),(-4.211883,12.819193),(-4.224647,12.864513),(-4.221314,12.921254),(-4.229324,12.948952),(-4.24785,12.971948),(-4.276918,12.996236),(-4.289733,13.01347),(-4.316398,13.070857),(-4.345363,13.097057),(-4.351073,13.106075),(-4.351073,13.118322),(-4.345906,13.127133),(-4.339601,13.135298),(-4.327044,13.168551),(-4.310197,13.175553),(-4.24617,13.172479),(-4.234801,13.175605),(-4.227773,13.184442),(-4.230021,13.193149),(-4.251209,13.204751),(-4.258107,13.21413),(-4.253508,13.22909),(-4.23878,13.246789),(-4.220668,13.262835),(-4.205501,13.272989),(-4.195837,13.274695),(-4.186458,13.272834),(-4.176872,13.272369),(-4.16602,13.278157),(-4.147106,13.299603),(-4.131578,13.323839),(-4.114654,13.364534),(-4.104267,13.382698),(-4.087549,13.397995),(-4.079307,13.402129),(-4.052539,13.409002),(-4.043495,13.414454),(-4.028199,13.431533),(-4.016029,13.43577),(-4.005358,13.439517),(-3.990217,13.449129),(-3.97691,13.460627),(-3.971613,13.469954),(-3.979468,13.47538),(-3.992232,13.476233),(-3.997968,13.480522),(-3.984481,13.496051),(-3.981251,13.49897),(-3.977737,13.499926),(-3.974094,13.498945),(-3.970321,13.496154),(-3.965468,13.489217),(-3.964482,13.487808),(-3.959288,13.467939),(-3.956059,13.46166),(-3.94748,13.458198),(-3.923399,13.453754),(-3.917973,13.451066),(-3.918593,13.440318),(-3.925544,13.436313),(-3.934923,13.434892),(-3.942726,13.431946),(-3.963009,13.413162),(-3.971613,13.40833),(-3.984042,13.39647),(-3.971613,13.386755),(-3.948411,13.380838),(-3.928825,13.38027),(-3.908878,13.381975),(-3.8926,13.378926),(-3.85808,13.365516),(-3.817101,13.35469),(-3.798626,13.347145),(-3.724419,13.288828),(-3.678711,13.26175),(-3.596882,13.199402),(-3.589802,13.197387),(-3.576599,13.199299),(-3.569958,13.196767),(-3.553913,13.180256),(-3.54505,13.174494),(-3.538487,13.173409),(-3.516111,13.1739),(-3.473922,13.167601),(-3.461593,13.165761),(-3.449475,13.16881),(-3.439475,13.184855),(-3.440586,13.202839),(-3.452265,13.237514),(-3.448544,13.265781),(-3.42653,13.27423),(-3.262406,13.283919),(-3.248634,13.292781),(-3.283955,13.542198),(-3.269176,13.579095),(-3.263905,13.602659),(-3.259719,13.658315),(-3.266282,13.682344),(-3.286952,13.69777),(-3.267057,13.717045),(-3.240418,13.707976),(-3.195821,13.674903),(-3.167941,13.672112),(-3.146031,13.67666),(-3.125102,13.67728),(-3.100039,13.662733),(-3.08221,13.646222),(-3.077301,13.636791),(-3.07451,13.622141),(-3.067508,13.606767),(-3.05601,13.61085),(-3.038621,13.629583),(-3.026968,13.635345),(-3.019113,13.637515),(-2.994774,13.634983),(-2.986402,13.632063),(-2.979322,13.627567),(-2.97232,13.624467),(-2.965008,13.625655),(-2.930473,13.638523),(-2.895245,13.651648),(-2.923383,13.746035),(-2.928059,13.799624),(-2.912169,13.837942),(-2.89447,13.866571),(-2.855377,13.996175),(-2.85703,13.997777),(-2.858606,14.000232),(-2.861552,14.00173),(-2.867288,14.000542),(-2.840855,14.042994),(-2.69319,14.123196),(-2.676265,14.141729),(-2.619758,14.203604),(-2.597382,14.222311),(-2.516147,14.267864),(-2.46168,14.280912),(-2.385612,14.264712),(-2.151879,14.15614),(-2.11984,14.148776),(-2.104337,14.151179),(-2.085008,14.159668),(-2.035918,14.181229),(-2.027288,14.188127),(-2.023412,14.198643),(-2.005015,14.444236),(-1.997057,14.470694),(-1.967033,14.483742),(-1.919129,14.485861),(-1.836963,14.479582),(-1.766683,14.483174),(-1.69692,14.496119),(-1.350275,14.714969),(-1.307745,14.734657),(-1.116697,14.780443),(-1.07856,14.796359),(-1.043317,14.817908),(-1.043138,14.818062),(-0.836404,14.995985),(-0.836249,14.996088),(-0.782816,15.048385),(-0.771272,15.056619),(-0.752895,15.069727),(-0.719719,15.078461),(-0.500343,15.079721),(-0.4679,15.079908),(-0.458649,15.075463),(-0.435602,15.015157),(-0.425732,15.0026),(-0.397671,15.002135)] +Bulgaria [(22.919562,43.834225),(23.052551,43.84282),(23.131849,43.847945),(23.161924,43.857324),(23.196961,43.86275),(23.234478,43.877297),(23.325151,43.886592),(23.484695,43.880604),(23.592699,43.837429),(23.620854,43.83401),(23.636107,43.832158),(23.720753,43.845826),(23.742871,43.8427),(23.799922,43.818463),(24.149606,43.75472),(24.159383,43.752938),(24.336705,43.759251),(24.358234,43.760017),(24.375494,43.763867),(24.431201,43.794176),(24.466341,43.802418),(24.500137,43.799498),(24.661763,43.755657),(24.705603,43.743765),(24.752628,43.738804),(24.963675,43.749605),(25.0816,43.718935),(25.211308,43.711881),(25.252339,43.704646),(25.285405,43.690391),(25.28872,43.688962),(25.323033,43.669713),(25.35962,43.654287),(25.403131,43.65005),(25.426075,43.654391),(25.467417,43.667749),(25.48245,43.669715),(25.488759,43.67054),(25.533821,43.668679),(25.556558,43.670359),(25.575059,43.677387),(25.593869,43.680668),(25.616503,43.687748),(25.637897,43.697282),(25.653503,43.708134),(25.671384,43.717359),(25.732948,43.718781),(25.739596,43.718935),(25.781144,43.732009),(25.804399,43.759914),(25.806259,43.763661),(25.839332,43.788439),(25.869304,43.800893),(25.916433,43.844379),(25.924495,43.858616),(25.934003,43.870321),(26.054306,43.934322),(26.061644,43.949773),(26.079317,43.969049),(26.116214,43.998866),(26.150734,44.012405),(26.231453,44.027495),(26.310518,44.052609),(26.332335,44.054926),(26.415791,44.063789),(26.614168,44.084855),(26.647758,44.093382),(26.667808,44.095087),(26.677317,44.097051),(26.697212,44.106094),(26.708685,44.10811),(26.753695,44.10811),(26.789455,44.115965),(26.884126,44.156531),(27.001535,44.165109),(27.027476,44.177046),(27.100237,44.14449),(27.205553,44.129246),(27.226741,44.120719),(27.251132,44.122373),(27.252662,44.121519),(27.269012,44.112399),(27.26431,44.089765),(27.285342,44.072453),(27.341669,44.053074),(27.353555,44.045271),(27.372882,44.020725),(27.383837,44.015092),(27.574523,44.016281),(27.633434,44.029768),(27.656275,44.023877),(27.676119,43.993543),(27.682515,43.987256),(27.721698,43.94874),(27.787327,43.960419),(27.85647,43.988634),(27.912074,43.993336),(27.912074,43.993233),(27.935845,43.964398),(27.98101,43.84934),(28.014806,43.830039),(28.221254,43.761981),(28.434574,43.735213),(28.57838,43.741278),(28.576182,43.727525),(28.57309,43.606147),(28.575531,43.593329),(28.585704,43.575832),(28.595876,43.563707),(28.602712,43.55272),(28.603526,43.538153),(28.594574,43.512519),(28.578461,43.48078),(28.560313,43.453843),(28.545177,43.442532),(28.535004,43.438625),(28.488617,43.40644),(28.479259,43.396796),(28.473888,43.384263),(28.473155,43.366848),(28.459483,43.380683),(28.414561,43.398871),(28.404959,43.40469),(28.393809,43.414293),(28.368663,43.42178),(28.321625,43.42829),(28.299164,43.425035),(28.261567,43.410956),(28.243175,43.407782),(28.177989,43.40998),(28.157237,43.407782),(28.118826,43.391547),(28.091563,43.363959),(28.087006,43.355251),(28.031098,43.248969),(28.017589,43.232733),(28.000173,43.223212),(27.931977,43.209784),(27.920584,43.204047),(27.913829,43.202338),(27.903982,43.202338),(27.903982,43.195502),(27.938731,43.1765),(27.945567,43.16885),(27.944347,43.157131),(27.925141,43.113593),(27.911388,43.065172),(27.904307,43.055854),(27.895518,43.049262),(27.887706,43.042141),(27.883556,43.030992),(27.885427,43.008612),(27.900401,42.962877),(27.903982,42.942288),(27.898448,42.874661),(27.903982,42.859687),(27.883962,42.848375),(27.883556,42.831936),(27.891612,42.81037),(27.897146,42.784003),(27.897146,42.736233),(27.894705,42.717475),(27.892263,42.710517),(27.841075,42.708319),(27.787364,42.715155),(27.743175,42.716051),(27.732677,42.714504),(27.725597,42.708482),(27.719005,42.694566),(27.715831,42.683743),(27.71697,42.674506),(27.724132,42.666978),(27.739513,42.661078),(27.712738,42.65766),(27.6692,42.643704),(27.646739,42.64057),(27.628429,42.628974),(27.630382,42.602729),(27.641449,42.574774),(27.650157,42.558051),(27.634044,42.563707),(27.540863,42.565497),(27.511485,42.553046),(27.498546,42.532457),(27.491466,42.507758),(27.480154,42.482896),(27.46225,42.489569),(27.452891,42.480129),(27.45338,42.465237),(27.465668,42.455634),(27.462087,42.448717),(27.461111,42.443549),(27.465668,42.42829),(27.468435,42.435777),(27.4699,42.437079),(27.469412,42.437201),(27.465668,42.441352),(27.472667,42.461859),(27.503917,42.437323),(27.514171,42.435126),(27.526378,42.443671),(27.534434,42.455268),(27.544444,42.460028),(27.562022,42.448188),(27.571056,42.458482),(27.5796,42.457221),(27.587738,42.451483),(27.595551,42.448188),(27.60963,42.451158),(27.619477,42.455878),(27.629405,42.458645),(27.643321,42.455634),(27.642833,42.450507),(27.641775,42.44953),(27.639822,42.449774),(27.636485,42.448188),(27.642426,42.430894),(27.652192,42.418524),(27.667735,42.416083),(27.691661,42.42829),(27.693533,42.418891),(27.699067,42.413886),(27.707856,42.412746),(27.719005,42.4147),(27.719005,42.407213),(27.713227,42.405951),(27.698009,42.400377),(27.705414,42.390611),(27.709646,42.388129),(27.719005,42.386705),(27.709483,42.376776),(27.708344,42.363105),(27.714366,42.349189),(27.725841,42.338935),(27.740408,42.334296),(27.752126,42.335395),(27.764496,42.338324),(27.780447,42.338935),(27.780447,42.332709),(27.774587,42.327216),(27.774425,42.321723),(27.779145,42.316311),(27.787852,42.310981),(27.76295,42.2956),(27.75115,42.277045),(27.75587,42.25849),(27.780447,42.243354),(27.776052,42.240668),(27.775645,42.239651),(27.773692,42.235907),(27.809744,42.218411),(27.815278,42.211982),(27.819998,42.204657),(27.831391,42.195014),(27.8449,42.186103),(27.856212,42.181301),(27.854177,42.177924),(27.85141,42.170111),(27.849376,42.16706),(27.87908,42.154242),(27.886729,42.147366),(27.903982,42.119818),(27.957774,42.094306),(27.964366,42.084621),(27.972667,42.075385),(27.986583,42.072089),(27.982432,42.061103),(27.987315,42.051744),(27.996918,42.044013),(28.007009,42.037909),(28.007579,42.032904),(28.006114,42.031562),(28.003429,42.03148),(28.000173,42.030463),(28.011567,42.021389),(28.019054,42.00845),(28.020356,41.994574),(28.013845,41.982652),(28.016775,41.972561),(28.016783,41.972531),(27.98101,41.978524),(27.96592,41.982141),(27.917035,41.977904),(27.903392,41.981056),(27.876934,41.99072),(27.852232,41.995448),(27.843034,41.995758),(27.824017,41.993484),(27.819935,41.994699),(27.815955,41.995138),(27.811925,41.994699),(27.807997,41.993484),(27.804948,41.983433),(27.804897,41.969894),(27.802726,41.96005),(27.818281,41.952867),(27.815335,41.946795),(27.80283,41.943074),(27.789807,41.942686),(27.776165,41.946123),(27.723971,41.967595),(27.687074,41.968602),(27.609301,41.953487),(27.606873,41.943513),(27.603359,41.93863),(27.598088,41.938604),(27.590543,41.942893),(27.582378,41.934883),(27.572198,41.929845),(27.550752,41.924212),(27.552096,41.921835),(27.554886,41.920336),(27.557935,41.91907),(27.560674,41.917675),(27.55747,41.915505),(27.551269,41.913102),(27.548892,41.91088),(27.562741,41.906435),(27.546411,41.901164),(27.533182,41.908063),(27.509463,41.933178),(27.494321,41.942841),(27.420837,41.973718),(27.39686,41.989324),(27.374845,42.008703),(27.332471,42.057434),(27.305289,42.077588),(27.273353,42.091747),(27.238213,42.097922),(27.216405,42.095623),(27.203796,42.08813),(27.181576,42.065883),(27.178992,42.061878),(27.178165,42.058364),(27.173824,42.057072),(27.149588,42.061826),(27.127212,42.062576),(27.11605,42.061826),(27.10065,42.071102),(27.0837,42.078415),(27.06551,42.082704),(27.047903,42.08292),(27.046545,42.082936),(27.022619,42.073893),(27.000398,42.04281),(26.981071,42.032862),(26.96717,42.028495),(26.95823,42.018237),(26.950065,42.006171),(26.938903,41.996223),(26.930221,41.994518),(26.911463,41.996792),(26.901076,41.993484),(26.890327,41.985423),(26.881129,41.985526),(26.871724,41.988136),(26.860768,41.987903),(26.85002,41.982865),(26.837899,41.97522),(26.827695,41.968783),(26.819427,41.965657),(26.80842,41.967879),(26.78992,41.979945),(26.780566,41.983433),(26.768578,41.980772),(26.746977,41.964494),(26.736745,41.958965),(26.717573,41.957311),(26.623573,41.969041),(26.605693,41.967414),(26.589467,41.958758),(26.560632,41.935684),(26.553293,41.931576),(26.547092,41.926899),(26.542958,41.920362),(26.544715,41.916435),(26.556807,41.91075),(26.559805,41.90734),(26.558978,41.901733),(26.55474,41.892948),(26.553604,41.887703),(26.552895,41.881643),(26.552053,41.874448),(26.547816,41.856283),(26.539548,41.83799),(26.526112,41.824244),(26.478363,41.813289),(26.375527,41.816622),(26.334185,41.789543),(26.320026,41.765462),(26.316305,41.743758),(26.32323,41.723682),(26.333359,41.713036),(26.294911,41.710323),(26.273931,41.714897),(26.261012,41.723062),(26.23445,41.745825),(26.226182,41.749675),(26.211816,41.750476),(26.189905,41.734689),(26.135128,41.733397),(26.108359,41.727893),(26.081488,41.711512),(26.074046,41.709135),(26.067225,41.708851),(26.060507,41.7073),(26.053686,41.701513),(26.048105,41.689084),(26.047485,41.674615),(26.050275,41.660456),(26.055029,41.648751),(26.05949,41.643777),(26.066915,41.635496),(26.081178,41.630457),(26.095234,41.62839),(26.106706,41.624282),(26.115284,41.616634),(26.121072,41.608211),(26.13027,41.582683),(26.130994,41.575138),(26.129443,41.559377),(26.1312,41.552659),(26.136781,41.549196),(26.152491,41.547026),(26.158072,41.542323),(26.163136,41.529146),(26.16448,41.517725),(26.16293,41.50646),(26.159312,41.493799),(26.147634,41.484756),(26.145877,41.478038),(26.156212,41.460416),(26.160449,41.455869),(26.174299,41.445998),(26.177399,41.439952),(26.175642,41.431891),(26.170578,41.429565),(26.164273,41.42786),(26.159002,41.421814),(26.14784,41.396906),(26.132441,41.371533),(26.120865,41.357787),(26.114664,41.355203),(26.107326,41.356598),(26.021853,41.341664),(26.008934,41.336806),(25.982269,41.323267),(25.959635,41.314999),(25.948576,41.313965),(25.93514,41.315929),(25.921187,41.316032),(25.896279,41.306265),(25.882017,41.304043),(25.86269,41.310089),(25.833337,41.334636),(25.811013,41.341044),(25.800781,41.338046),(25.763471,41.319029),(25.728744,41.317066),(25.717479,41.31412),(25.70518,41.307299),(25.698049,41.301925),(25.691227,41.298411),(25.679962,41.29717),(25.670453,41.299186),(25.649576,41.308487),(25.639758,41.311071),(25.551494,41.31567),(25.537955,41.312208),(25.530203,41.302751),(25.523692,41.291693),(25.514907,41.283476),(25.505089,41.280582),(25.497337,41.281202),(25.489482,41.283063),(25.479096,41.283786),(25.453464,41.280427),(25.285722,41.239396),(25.262261,41.238104),(25.23911,41.240895),(25.21968,41.249731),(25.177098,41.293863),(25.157875,41.30611),(25.153847,41.307535),(25.116534,41.320735),(25.112606,41.324145),(25.104855,41.334067),(25.101858,41.336651),(25.096793,41.336703),(25.08036,41.334067),(24.916959,41.386364),(24.886367,41.400627),(24.872931,41.401867),(24.863009,41.400316),(24.842132,41.394735),(24.803271,41.392668),(24.800171,41.379336),(24.802858,41.361921),(24.794279,41.3474),(24.774436,41.348072),(24.752628,41.362748),(24.718728,41.395717),(24.699195,41.408946),(24.680901,41.415509),(24.661264,41.41768),(24.638217,41.41768),(24.644314,41.427653),(24.609484,41.42724),(24.595842,41.429772),(24.580442,41.440521),(24.579719,41.44419),(24.582199,41.455249),(24.581062,41.460209),(24.577238,41.463568),(24.56773,41.468116),(24.564113,41.471062),(24.558738,41.476849),(24.553571,41.480777),(24.549436,41.485428),(24.546543,41.493644),(24.543339,41.521343),(24.530936,41.547543),(24.510162,41.56165),(24.481327,41.553227),(24.459209,41.549506),(24.438849,41.527699),(24.423966,41.525218),(24.402882,41.527854),(24.387483,41.526665),(24.353893,41.519121),(24.345005,41.518397),(24.318236,41.520774),(24.309554,41.519792),(24.303043,41.51695),(24.296119,41.515245),(24.286817,41.517622),(24.28506,41.523151),(24.287334,41.531626),(24.286403,41.54036),(24.26687,41.549765),(24.250747,41.563459),(24.23328,41.561805),(24.214366,41.555759),(24.197698,41.547711),(24.19628,41.547026),(24.181604,41.537362),(24.17747,41.531006),(24.173129,41.515297),(24.170235,41.511576),(24.162793,41.512041),(24.158453,41.51633),(24.154008,41.522118),(24.146257,41.526769),(24.116491,41.533383),(24.076597,41.536019),(24.047348,41.525735),(24.049829,41.493644),(24.052516,41.471475),(24.051999,41.463),(24.045901,41.455455),(24.034739,41.451321),(24.022854,41.453078),(24.000736,41.464137),(23.997119,41.457006),(23.992365,41.454628),(23.986887,41.453698),(23.981306,41.450856),(23.964666,41.43835),(23.949887,41.437575),(23.902861,41.463517),(23.894799,41.464344),(23.867721,41.445482),(23.851598,41.439591),(23.830927,41.435611),(23.80974,41.433803),(23.792067,41.434475),(23.777081,41.429049),(23.754446,41.400678),(23.73822,41.397474),(23.705354,41.403159),(23.672177,41.402952),(23.65285,41.397629),(23.627736,41.378509),(23.624847,41.377094),(23.612439,41.371016),(23.578953,41.371998),(23.513014,41.397733),(23.414519,41.399903),(23.395502,41.395252),(23.365116,41.378561),(23.347236,41.371223),(23.326049,41.369311),(23.31561,41.376855),(23.306205,41.388379),(23.287705,41.398198),(23.269928,41.397268),(23.24657,41.389723),(23.224246,41.379026),(23.209777,41.368587),(23.206366,41.360939),(23.204816,41.342697),(23.199545,41.332982),(23.190656,41.326057),(23.179701,41.321355),(23.15717,41.316342),(23.115209,41.312673),(22.916978,41.335773),(22.940852,41.349829),(22.94447,41.368432),(22.939406,41.389413),(22.937339,41.410755),(22.940542,41.416905),(22.952118,41.427705),(22.954598,41.432408),(22.953358,41.438195),(22.94757,41.448376),(22.946227,41.453233),(22.943599,41.523201),(22.943023,41.538551),(22.94788,41.555139),(22.948707,41.560978),(22.946434,41.567748),(22.936925,41.57891),(22.933721,41.584595),(22.932068,41.597953),(22.932998,41.612345),(22.936098,41.626168),(22.940852,41.63764),(22.945813,41.641077),(22.961523,41.644488),(22.967001,41.647046),(22.970101,41.652032),(22.976613,41.666553),(22.985435,41.677198),(22.998627,41.693115),(23.009582,41.71637),(23.008859,41.739934),(22.991185,41.760992),(22.98054,41.764739),(22.956872,41.765669),(22.945917,41.769338),(22.939716,41.776702),(22.918322,41.814348),(22.907676,41.848584),(22.901372,41.860418),(22.896721,41.864448),(22.885042,41.869151),(22.882088,41.871618),(22.880804,41.872691),(22.878221,41.880261),(22.878634,41.895015),(22.877084,41.902043),(22.866335,41.924884),(22.858894,41.94788),(22.857137,41.971884),(22.85476,41.982632),(22.846905,41.993484),(22.846595,41.993639),(22.845768,42.006869),(22.845621,42.007408),(22.843701,42.014465),(22.838223,42.019478),(22.826958,42.025085),(22.821273,42.025369),(22.805977,42.02139),(22.798949,42.021235),(22.791094,42.025808),(22.787684,42.032578),(22.785306,42.039141),(22.780862,42.043171),(22.77063,42.043998),(22.725052,42.042474),(22.718437,42.044463),(22.713993,42.048623),(22.710272,42.05299),(22.705725,42.055935),(22.675856,42.060612),(22.627177,42.079127),(22.617771,42.082704),(22.531058,42.129109),(22.510181,42.144793),(22.506939,42.148927),(22.494678,42.164559),(22.481449,42.193317),(22.443622,42.214427),(22.345023,42.313439),(22.364144,42.320984),(22.405795,42.321552),(22.423985,42.325893),(22.438454,42.340052),(22.454371,42.376768),(22.46977,42.391703),(22.485066,42.397155),(22.497572,42.399196),(22.508838,42.404932),(22.519483,42.420926),(22.533125,42.45759),(22.536536,42.47839),(22.532505,42.493402),(22.532505,42.493557),(22.524857,42.507665),(22.512145,42.519189),(22.481449,42.535622),(22.429669,42.571408),(22.425328,42.572855),(22.428842,42.592776),(22.441318,42.632891),(22.444552,42.64329),(22.449203,42.667965),(22.442072,42.681685),(22.468116,42.718324),(22.481449,42.727677),(22.482586,42.730675),(22.482896,42.733775),(22.482586,42.736824),(22.481449,42.739821),(22.466566,42.748529),(22.45313,42.763592),(22.429359,42.806122),(22.425845,42.809843),(22.427395,42.813615),(22.430446,42.817077),(22.436801,42.824286),(22.445482,42.830178),(22.470907,42.840125),(22.481449,42.84674),(22.497055,42.864413),(22.506047,42.870123),(22.519793,42.870356),(22.53757,42.868341),(22.544494,42.871389),(22.544785,42.871706),(22.549972,42.877358),(22.563615,42.884283),(22.5909,42.886892),(22.666244,42.871932),(22.69663,42.87741),(22.727015,42.886892),(22.738798,42.897383),(22.739579,42.898858),(22.745516,42.910069),(22.763189,42.958645),(22.76939,42.97128),(22.776418,42.979729),(22.788097,42.984897),(22.815796,42.989703),(22.828818,42.993449),(22.828921,42.993449),(22.829025,42.993501),(22.829025,42.993656),(22.842254,43.007505),(22.884215,43.036651),(22.889486,43.044376),(22.896721,43.062721),(22.901682,43.069749),(22.910157,43.075279),(22.927107,43.081144),(22.935271,43.085562),(22.955632,43.108274),(22.974029,43.141192),(22.984571,43.174627),(22.982902,43.187318),(22.981367,43.198992),(22.964727,43.204418),(22.915531,43.212247),(22.897754,43.220335),(22.883802,43.230592),(22.857343,43.256947),(22.833159,43.274647),(22.826958,43.28139),(22.823857,43.289297),(22.820756,43.307539),(22.817139,43.315497),(22.80453,43.328984),(22.73301,43.381513),(22.724343,43.38606),(22.719367,43.388671),(22.702934,43.394045),(22.693219,43.394872),(22.674202,43.394148),(22.664694,43.396732),(22.658926,43.401295),(22.656529,43.403192),(22.645367,43.420297),(22.637822,43.426369),(22.62852,43.428255),(22.606919,43.427402),(22.596274,43.429159),(22.586766,43.43443),(22.57271,43.44815),(22.565785,43.453344),(22.532609,43.464842),(22.518863,43.474247),(22.509354,43.493341),(22.490647,43.540883),(22.478452,43.559229),(22.477625,43.564164),(22.478658,43.569176),(22.481449,43.574111),(22.482689,43.576695),(22.483103,43.579279),(22.482689,43.581734),(22.481449,43.584137),(22.478142,43.587573),(22.477108,43.591294),(22.478142,43.594911),(22.481449,43.598529),(22.481759,43.598942),(22.481966,43.599459),(22.481759,43.599975),(22.481449,43.600647),(22.473801,43.612998),(22.472871,43.635942),(22.466256,43.64912),(22.455921,43.656406),(22.426465,43.668214),(22.41396,43.676663),(22.404865,43.687179),(22.396906,43.699401),(22.390498,43.712449),(22.386054,43.725498),(22.385848,43.733817),(22.389568,43.750509),(22.388535,43.758286),(22.362593,43.780843),(22.349467,43.807921),(22.354738,43.829703),(22.367554,43.852751),(22.377063,43.883524),(22.379026,43.913496),(22.382024,43.918561),(22.391945,43.931867),(22.394529,43.936337),(22.396803,43.951944),(22.39732,43.980934),(22.399594,43.993336),(22.411789,44.006927),(22.43432,44.013955),(22.465885,44.017624),(22.481449,44.019433),(22.50367,44.019898),(22.514935,44.030285),(22.522583,44.044703),(22.534159,44.057157),(22.554623,44.062428),(22.57519,44.061394),(22.592967,44.063926),(22.604749,44.079378),(22.604646,44.088163),(22.598134,44.109298),(22.597101,44.119065),(22.599065,44.130331),(22.6094,44.159941),(22.607953,44.159993),(22.605989,44.163145),(22.604852,44.168468),(22.606196,44.174566),(22.608573,44.175858),(22.624799,44.189397),(22.639992,44.207329),(22.648777,44.213995),(22.69164,44.228435),(22.906436,44.122889),(22.942609,44.111469),(22.988085,44.107025),(23.008307,44.100446),(23.030976,44.093072),(23.040071,44.062325),(23.023018,44.031629),(22.988085,44.017676),(22.966277,44.015557),(22.926486,44.006152),(22.905816,44.003982),(22.885869,43.994525),(22.874707,43.972046),(22.850522,43.896986),(22.851039,43.874352),(22.863441,43.855412),(22.888763,43.839522),(22.919562,43.834225)] +Bahrain [(50.551606,26.194241),(50.594737,26.160305),(50.604871,26.174732),(50.619898,26.185466),(50.626339,26.170438),(50.632779,26.146823),(50.64566,26.14575),(50.635507,26.134382),(50.634926,26.115694),(50.614757,26.111151),(50.620616,26.052965),(50.618419,25.964748),(50.603526,25.85456),(50.594493,25.832709),(50.56544,25.7897),(50.565278,25.789537),(50.558604,25.81977),(50.547211,25.852932),(50.531423,25.88524),(50.511404,25.913031),(50.467784,25.957587),(50.46046,25.985256),(50.46046,25.985297),(50.479991,26.018866),(50.485444,26.033433),(50.490896,26.048),(50.490896,26.048041),(50.485199,26.085639),(50.469981,26.122626),(50.452973,26.149563),(50.448904,26.161607),(50.449555,26.177965),(50.461192,26.225246),(50.468516,26.236884),(50.481456,26.24258),(50.481619,26.24254),(50.482595,26.24254),(50.503917,26.242011),(50.516368,26.239691),(50.5324,26.2331),(50.545421,26.227729),(50.545502,26.227769),(50.586681,26.242499),(50.593272,26.238593),(50.597423,26.225531),(50.604747,26.216498),(50.606944,26.212388),(50.609141,26.208319),(50.609141,26.208279),(50.603526,26.197659),(50.594086,26.195299),(50.565603,26.198879),(50.551606,26.194241)] +Bosnia and Herzegovina [(16.941529,45.241219),(16.947317,45.23569),(16.967987,45.237292),(16.9653,45.244216),(16.961166,45.250934),(16.975945,45.247162),(16.988141,45.236516),(16.998683,45.2314),(17.008295,45.244165),(17.012015,45.236465),(17.013566,45.230574),(17.012532,45.224476),(17.008295,45.216156),(17.014393,45.219773),(17.030206,45.226956),(17.036303,45.23047),(17.055734,45.200446),(17.098418,45.177657),(17.124628,45.169058),(17.187095,45.148563),(17.197844,45.152956),(17.24549,45.155384),(17.249727,45.15988),(17.268951,45.189543),(17.307811,45.171404),(17.326518,45.165926),(17.359384,45.150682),(17.368583,45.145101),(17.382432,45.139623),(17.400829,45.141018),(17.434109,45.148563),(17.436176,45.151922),(17.438759,45.158123),(17.443307,45.162051),(17.451162,45.158485),(17.45695,45.151819),(17.458086,45.146134),(17.454986,45.140605),(17.447751,45.1343),(17.46036,45.131252),(17.472349,45.13275),(17.496224,45.14169),(17.487025,45.120865),(17.481858,45.114405),(17.489299,45.116265),(17.539528,45.120968),(17.561439,45.119366),(17.572395,45.120399),(17.593065,45.129598),(17.629962,45.157348),(17.651563,45.165358),(17.684739,45.163963),(17.714505,45.152025),(17.741377,45.133267),(17.797497,45.081952),(17.815584,45.07017),(17.835738,45.064434),(17.84566,45.065261),(17.875219,45.073994),(17.902607,45.077766),(17.911185,45.081074),(17.927722,45.092029),(17.988493,45.143809),(18.003479,45.149338),(18.019086,45.149442),(18.035002,45.143344),(18.090296,45.107997),(18.112413,45.100607),(18.131844,45.097869),(18.143698,45.0977),(18.143704,45.0977),(18.153651,45.097558),(18.174942,45.100762),(18.192512,45.108514),(18.199333,45.119004),(18.200057,45.131613),(18.20233,45.144067),(18.213286,45.153782),(18.237884,45.157451),(18.261965,45.150682),(18.30744,45.127531),(18.322323,45.12288),(18.392913,45.118642),(18.412033,45.112441),(18.429293,45.102209),(18.45183,45.084959),(18.451842,45.08495),(18.466087,45.074046),(18.481708,45.066921),(18.490788,45.06278),(18.51704,45.055856),(18.541121,45.055287),(18.539261,45.069343),(18.534403,45.081539),(18.534803,45.088412),(18.53492,45.090427),(18.549596,45.094768),(18.557658,45.094406),(18.573781,45.091564),(18.581842,45.091357),(18.58856,45.093218),(18.602513,45.099264),(18.609127,45.100091),(18.616672,45.097662),(18.621116,45.093114),(18.627937,45.08004),(18.633622,45.071979),(18.640547,45.065364),(18.648711,45.062677),(18.658116,45.066294),(18.661217,45.0711),(18.663284,45.077146),(18.666075,45.082986),(18.671346,45.086861),(18.678167,45.087378),(18.684058,45.084794),(18.689019,45.080247),(18.726329,45.026142),(18.738318,45.01591),(18.750101,45.012241),(18.778006,45.010329),(18.78493,45.008468),(18.788961,45.005729),(18.791752,45.001544),(18.794646,44.995446),(18.795162,44.993327),(18.795162,44.988004),(18.795783,44.985059),(18.797953,44.982217),(18.803017,44.977307),(18.804568,44.973845),(18.80157,44.963975),(18.779866,44.952244),(18.772631,44.942478),(18.78338,44.913745),(18.817797,44.887545),(18.842129,44.876239),(18.842144,44.876233),(18.858724,44.868529),(18.889627,44.86119),(18.974916,44.862757),(19.004969,44.863309),(19.015821,44.865635),(19.047757,44.872714),(19.068427,44.874833),(19.08455,44.878967),(19.174261,44.925424),(19.18687,44.927543),(19.193174,44.921549),(19.196895,44.913177),(19.201856,44.908371),(19.211881,44.908371),(19.229554,44.913745),(19.239476,44.915141),(19.283815,44.908371),(19.293013,44.909405),(19.301281,44.91235),(19.31017,44.91235),(19.33022,44.898708),(19.340142,44.896227),(19.350271,44.897002),(19.352958,44.898087),(19.353165,44.899018),(19.356982,44.895877),(19.356993,44.895869),(19.362776,44.891111),(19.368667,44.887132),(19.372802,44.881448),(19.375695,44.873128),(19.376626,44.862999),(19.373112,44.86026),(19.367841,44.859278),(19.363707,44.854628),(19.32836,44.733963),(19.318025,44.715463),(19.308413,44.705128),(19.288259,44.693035),(19.277614,44.684871),(19.270069,44.67562),(19.255703,44.645648),(19.208677,44.588391),(19.20413,44.585135),(19.193278,44.580587),(19.188317,44.57604),(19.18625,44.569322),(19.187283,44.553302),(19.185319,44.546223),(19.179635,44.538264),(19.173124,44.531443),(19.165269,44.526689),(19.129922,44.518317),(19.127339,44.502556),(19.143358,44.458218),(19.141394,44.430829),(19.129612,44.416153),(19.115763,44.403596),(19.107185,44.382718),(19.108942,44.36365),(19.116693,44.343703),(19.138914,44.309338),(19.157001,44.293577),(19.177155,44.286962),(19.220046,44.280244),(19.240717,44.272699),(19.249502,44.270736),(19.263661,44.270167),(19.295907,44.2758),(19.307379,44.274198),(19.324329,44.263966),(19.341589,44.245828),(19.353991,44.22433),(19.356058,44.204021),(19.362363,44.191206),(19.381173,44.177098),(19.424891,44.153792),(19.435537,44.146092),(19.442565,44.143198),(19.448456,44.144438),(19.459721,44.152707),(19.465716,44.15281),(19.474191,44.144903),(19.476361,44.127023),(19.482666,44.120667),(19.498169,44.110229),(19.516189,44.091194),(19.522043,44.08501),(19.554599,44.071265),(19.570929,44.056898),(19.580231,44.051524),(19.583848,44.054315),(19.589946,44.060309),(19.598938,44.062583),(19.611443,44.054521),(19.618885,44.035711),(19.61041,44.019278),(19.593357,44.005584),(19.552222,43.983415),(19.528554,43.977213),(19.50406,43.975663),(19.447112,43.979797),(19.393989,43.977058),(19.379106,43.973854),(19.364637,43.973286),(19.351718,43.979125),(19.325879,43.996592),(19.300558,44.009511),(19.287329,44.013025),(19.272756,44.012405),(19.252085,44.007496),(19.243507,44.002018),(19.238029,43.99251),(19.238133,43.985378),(19.242887,43.972821),(19.240717,43.965741),(19.229348,43.95768),(19.241027,43.952357),(19.275443,43.933263),(19.305932,43.904737),(19.359469,43.842209),(19.461685,43.762136),(19.481735,43.729218),(19.505713,43.67364),(19.507367,43.647182),(19.481735,43.628914),(19.477498,43.616977),(19.475638,43.602559),(19.476878,43.588684),(19.481735,43.578116),(19.48897,43.573181),(19.491761,43.568608),(19.48959,43.564448),(19.481735,43.560831),(19.443908,43.571864),(19.431816,43.57114),(19.41931,43.549255),(19.410732,43.540754),(19.402877,43.549643),(19.394609,43.566463),(19.38045,43.585584),(19.363396,43.601577),(19.34655,43.608838),(19.335801,43.606513),(19.31265,43.593154),(19.300868,43.588297),(19.289189,43.587935),(19.263661,43.590958),(19.252706,43.588633),(19.238856,43.572122),(19.229451,43.549746),(19.217462,43.532796),(19.195345,43.532796),(19.147699,43.538145),(19.122791,43.535948),(19.096023,43.512901),(19.076386,43.507242),(19.054991,43.5067),(19.038145,43.511015),(19.025019,43.523029),(19.01427,43.537783),(19.000214,43.547886),(18.977477,43.546232),(18.962697,43.5383),(18.938926,43.518947),(18.920529,43.512203),(18.911021,43.507268),(18.90513,43.499051),(18.905647,43.490628),(18.915258,43.485383),(18.930658,43.482282),(18.937893,43.478949),(18.951225,43.463808),(18.950708,43.457297),(18.946368,43.44908),(18.946264,43.443938),(18.968278,43.448098),(18.975306,43.444274),(19.00993,43.410995),(19.039592,43.350714),(19.069668,43.30883),(19.062536,43.304386),(19.036285,43.303327),(19.024709,43.298728),(19.022535,43.296546),(19.017164,43.291157),(19.01086,43.28232),(19.002902,43.273949),(18.992463,43.267102),(18.989506,43.272016),(18.988949,43.272941),(18.968589,43.292216),(18.959493,43.303456),(18.957116,43.311828),(18.957943,43.318985),(18.957323,43.325858),(18.950295,43.333067),(18.923423,43.346838),(18.899135,43.351903),(18.839604,43.34782),(18.821207,43.341387),(18.824618,43.337408),(18.830302,43.328157),(18.833506,43.324153),(18.807151,43.318029),(18.679511,43.24948),(18.664318,43.233176),(18.688399,43.224469),(18.645301,43.180182),(18.629074,43.154886),(18.621128,43.124578),(18.620599,43.122563),(18.621116,43.09644),(18.638996,43.020243),(18.598379,43.024455),(18.538951,43.023887),(18.483037,43.014637),(18.452754,42.993398),(18.433531,42.954201),(18.434047,42.936889),(18.443969,42.916865),(18.467637,42.881699),(18.473322,42.862553),(18.465984,42.852812),(18.453478,42.845655),(18.443659,42.834467),(18.444589,42.817078),(18.453891,42.793151),(18.46743,42.769199),(18.502157,42.727445),(18.522518,42.71189),(18.539674,42.695457),(18.550009,42.668069),(18.549596,42.638923),(18.542832,42.626429),(18.538434,42.618304),(18.517247,42.602982),(18.506798,42.59849),(18.486654,42.58983),(18.493682,42.579831),(18.495852,42.570865),(18.492132,42.564767),(18.481797,42.563682),(18.470531,42.569056),(18.458956,42.569728),(18.44769,42.566214),(18.437355,42.559212),(18.421025,42.56363),(18.385265,42.566524),(18.370485,42.57363),(18.369791,42.574384),(18.369783,42.574393),(18.349608,42.59629),(18.338653,42.602517),(18.311574,42.601225),(18.257107,42.614893),(18.224138,42.628071),(18.049368,42.714758),(17.995315,42.740441),(17.971853,42.754962),(17.928859,42.7908),(17.904467,42.804546),(17.869431,42.811807),(17.858475,42.816974),(17.827159,42.853122),(17.825092,42.864542),(17.826229,42.888649),(17.823335,42.898365),(17.81176,42.909863),(17.802665,42.90963),(17.794707,42.904592),(17.786748,42.901388),(17.765458,42.904126),(17.7236,42.916245),(17.701586,42.9195),(17.679468,42.915676),(17.665929,42.90516),(17.653349,42.890928),(17.653331,42.890937),(17.652843,42.891181),(17.556488,42.93476),(17.573985,42.934272),(17.601085,42.925238),(17.617931,42.92178),(17.603038,42.932563),(17.580679,42.942075),(17.63451,42.950403),(17.662725,42.965699),(17.659314,42.993398),(17.659211,42.993449),(17.659108,42.99363),(17.659004,42.993656),(17.628412,43.046573),(17.598026,43.072902),(17.587381,43.082126),(17.450913,43.148152),(17.442067,43.152432),(17.42677,43.165713),(17.415195,43.184962),(17.406823,43.205426),(17.400209,43.216045),(17.389667,43.223099),(17.36941,43.232556),(17.328689,43.260358),(17.289931,43.30343),(17.267607,43.353324),(17.286417,43.437376),(17.270501,43.463214),(17.239495,43.478407),(17.142757,43.489336),(17.084569,43.513237),(17.062314,43.527747),(17.030516,43.54848),(16.98194,43.589692),(16.82381,43.707307),(16.712602,43.771515),(16.698236,43.788078),(16.689761,43.809317),(16.686144,43.826137),(16.678599,43.840684),(16.637361,43.868409),(16.604805,43.901068),(16.555816,43.937474),(16.527447,43.967859),(16.516542,43.979539),(16.501039,43.992716),(16.483469,44.002535),(16.439647,44.014007),(16.431482,44.025789),(16.426935,44.040517),(16.414016,44.055813),(16.403991,44.058759),(16.377946,44.057364),(16.367197,44.058139),(16.357378,44.062169),(16.346113,44.068474),(16.326889,44.082375),(16.312213,44.099997),(16.289992,44.139116),(16.275523,44.157357),(16.232115,44.190999),(16.215785,44.208155),(16.197078,44.25115),(16.18757,44.282414),(16.186639,44.297607),(16.189533,44.308511),(16.199662,44.324686),(16.203176,44.333057),(16.205863,44.349801),(16.205644,44.350531),(16.202866,44.359774),(16.192117,44.367422),(16.171343,44.377086),(16.153153,44.380445),(16.141991,44.380755),(16.13827,44.377602),(16.138787,44.385406),(16.145402,44.389591),(16.153877,44.39357),(16.159871,44.400702),(16.159664,44.416101),(16.152946,44.435067),(16.128348,44.484314),(16.123077,44.5039),(16.116463,44.52147),(16.104887,44.532373),(16.0837,44.534905),(16.04732,44.52333),(16.026649,44.52519),(16.006082,44.541003),(16.013937,44.557023),(16.030473,44.572216),(16.034917,44.585393),(16.043186,44.588442),(16.044736,44.589372),(16.041119,44.602808),(16.036158,44.615314),(16.028303,44.624719),(16.016004,44.628957),(15.991613,44.631437),(15.979107,44.634383),(15.968978,44.63924),(15.955646,44.653193),(15.948101,44.678101),(15.937352,44.688953),(15.929808,44.68947),(15.921539,44.686111),(15.913064,44.684509),(15.905416,44.690142),(15.902006,44.698823),(15.899318,44.70921),(15.895598,44.718564),(15.888776,44.724248),(15.876271,44.724816),(15.869243,44.718564),(15.863868,44.709985),(15.856427,44.703577),(15.847849,44.70151),(15.829969,44.700528),(15.820357,44.698255),(15.805681,44.696653),(15.796999,44.703267),(15.790488,44.713396),(15.78284,44.722594),(15.759275,44.73417),(15.753694,44.739286),(15.728476,44.769103),(15.717314,44.786466),(15.716074,44.80321),(15.730647,44.817007),(15.751214,44.821555),(15.76868,44.827342),(15.773435,44.845016),(15.76403,44.864033),(15.74801,44.882016),(15.734264,44.902118),(15.730853,44.927233),(15.734108,44.93444),(15.738811,44.944855),(15.763616,44.97555),(15.76744,44.993224),(15.755555,45.023558),(15.755968,45.042213),(15.779429,45.085363),(15.78377,45.100917),(15.780049,45.160294),(15.780979,45.167993),(15.792348,45.189801),(15.802569,45.196473),(15.824801,45.210988),(15.878855,45.217241),(15.974559,45.215381),(15.997504,45.218533),(16.008459,45.218171),(16.020448,45.213934),(16.029233,45.205045),(16.028716,45.19626),(16.024996,45.188922),(16.024375,45.184323),(16.033057,45.179724),(16.054968,45.176417),(16.067474,45.169595),(16.08277,45.151612),(16.104061,45.112545),(16.121527,45.096163),(16.212994,45.031464),(16.24121,45.018959),(16.279347,45.007176),(16.316244,45.001234),(16.340635,45.005936),(16.346019,45.02001),(16.346023,45.02002),(16.356965,45.048621),(16.377325,45.075596),(16.381976,45.083709),(16.384043,45.09544),(16.381976,45.101951),(16.38177,45.107584),(16.389211,45.116834),(16.398513,45.121433),(16.424764,45.127272),(16.435513,45.132853),(16.449156,45.148098),(16.481919,45.199516),(16.528944,45.222254),(16.587028,45.220858),(16.786293,45.178846),(16.811718,45.181223),(16.824223,45.189077),(16.825774,45.196364),(16.82505,45.205045),(16.830734,45.216983),(16.84076,45.224011),(16.851922,45.227421),(16.86174,45.232227),(16.867321,45.243389),(16.875383,45.246955),(16.885822,45.255223),(16.893986,45.263543),(16.896984,45.268142),(16.924372,45.284524),(16.93264,45.278788),(16.940702,45.257755),(16.942149,45.249849),(16.941529,45.241219)] +Bajo Nuevo Bank (Petrel Is.) [(-79.989288,15.794949),(-79.987823,15.79621),(-79.986399,15.79442),(-79.988149,15.794176),(-79.989288,15.794949)] +Saint Barthelemy [(-62.838857,17.881985),(-62.850942,17.890448),(-62.861318,17.905422),(-62.86734,17.920396),(-62.866119,17.929145),(-62.857411,17.925035),(-62.791656,17.915473),(-62.79894,17.904486),(-62.810129,17.892279),(-62.823842,17.883246),(-62.838857,17.881985)] +Belarus [(28.148907,56.142414),(28.169112,56.125257),(28.238462,56.082624),(28.269364,56.058207),(28.289828,56.046606),(28.310912,56.042679),(28.333443,56.050249),(28.366516,56.07911),(28.389771,56.088567),(28.537824,56.097714),(28.594719,56.092391),(28.611462,56.088464),(28.620713,56.082986),(28.637094,56.065726),(28.671924,56.037563),(28.680606,56.027382),(28.690011,56.003895),(28.695488,55.98015),(28.706392,55.959841),(28.73192,55.946793),(28.80897,55.934571),(28.830881,55.937672),(28.833051,55.960978),(28.859716,55.976455),(28.922141,55.992139),(28.922245,55.992191),(28.922296,55.992191),(28.922451,55.992242),(28.980846,56.013533),(29.030765,56.024178),(29.088643,56.023222),(29.145435,56.012034),(29.192926,55.992139),(29.224035,55.978186),(29.377411,55.954028),(29.395601,55.947723),(29.413067,55.937982),(29.431567,55.924184),(29.441076,55.914934),(29.444435,55.906925),(29.440559,55.900543),(29.433738,55.899535),(29.425573,55.9),(29.417408,55.89801),(29.407796,55.893876),(29.399631,55.891887),(29.391983,55.888373),(29.384128,55.879949),(29.380718,55.871242),(29.375654,55.846799),(29.371933,55.836386),(29.348162,55.802848),(29.343614,55.786984),(29.350745,55.766055),(29.363406,55.751482),(29.413067,55.727762),(29.461023,55.6873),(29.480867,55.681099),(29.5081,55.685491),(29.585356,55.73802),(29.684162,55.770602),(29.710827,55.773754),(29.779867,55.763807),(29.805705,55.771455),(29.844772,55.812563),(29.869474,55.830547),(29.907817,55.843208),(29.947712,55.848194),(29.988226,55.846799),(30.106255,55.821917),(30.132352,55.826878),(30.177258,55.851372),(30.200254,55.857987),(30.217773,55.855145),(30.251517,55.837626),(30.270793,55.83065),(30.468817,55.793521),(30.477395,55.787733),(30.477808,55.779129),(30.471814,55.770964),(30.469282,55.762644),(30.480702,55.753962),(30.569586,55.729519),(30.580128,55.724765),(30.587466,55.718306),(30.5916,55.705697),(30.588396,55.696601),(30.583745,55.68854),(30.583745,55.679135),(30.596354,55.665286),(30.616611,55.657431),(30.639142,55.654485),(30.679967,55.656346),(30.693713,55.65216),(30.703324,55.642341),(30.71304,55.626683),(30.742185,55.594385),(30.77071,55.591543),(30.804093,55.602344),(30.847657,55.611025),(30.886156,55.600483),(30.907122,55.57777),(30.912821,55.571596),(30.919745,55.534492),(30.899385,55.499146),(30.914371,55.493306),(30.919435,55.492325),(30.913234,55.479715),(30.887189,55.468088),(30.881298,55.451448),(30.889101,55.433413),(30.905276,55.420856),(30.918608,55.407627),(30.918298,55.387783),(30.905948,55.376001),(30.845641,55.351145),(30.811225,55.323084),(30.797169,55.304274),(30.794275,55.285515),(30.804093,55.272958),(30.821353,55.26438),(30.856803,55.253476),(30.869826,55.241539),(30.886517,55.204538),(30.900728,55.192136),(30.947237,55.171362),(30.959795,55.162577),(30.96119,55.158546),(30.956849,55.149503),(30.958813,55.144439),(30.962223,55.142888),(30.972455,55.140873),(30.975039,55.13953),(30.979483,55.134259),(30.984031,55.1309),(30.985478,55.125835),(30.98062,55.115345),(30.972662,55.108575),(30.970285,55.101444),(30.973075,55.093951),(30.98062,55.086303),(30.994211,55.067027),(31.006407,55.04243),(31.005632,55.022999),(30.98062,55.018658),(30.933491,55.025635),(30.913234,55.024601),(30.90662,55.012561),(30.916955,54.996024),(30.931424,54.985172),(30.936075,54.973235),(30.917368,54.953959),(30.899488,54.946363),(30.838613,54.93918),(30.814842,54.928018),(30.817426,54.917631),(30.827451,54.902541),(30.826108,54.877478),(30.810708,54.86151),(30.787867,54.847713),(30.768127,54.829936),(30.762649,54.801979),(30.770607,54.786011),(30.786058,54.779345),(30.827296,54.771025),(30.98062,54.705654),(30.995503,54.689841),(30.999017,54.671341),(31.020928,54.673666),(31.10516,54.668292),(31.128621,54.640387),(31.167947,54.621577),(31.139474,54.582871),(31.089554,54.535897),(31.064646,54.492282),(31.093585,54.479415),(31.167896,54.467064),(31.179017,54.453353),(31.179213,54.453111),(31.20903,54.448047),(31.22536,54.428048),(31.248717,54.376785),(31.26174,54.364486),(31.274142,54.356632),(31.284788,54.347381),(31.292022,54.331258),(31.299154,54.272606),(31.309799,54.244339),(31.324682,54.229249),(31.480641,54.156644),(31.506376,54.143931),(31.530871,54.13742),(31.583116,54.129514),(31.666935,54.101867),(31.697837,54.097939),(31.725794,54.097474),(31.73675,54.094167),(31.748119,54.086312),(31.755767,54.074943),(31.762588,54.060319),(31.77127,54.048537),(31.783517,54.045798),(31.808477,54.055978),(31.813851,54.057063),(31.822636,54.053446),(31.823876,54.050139),(31.823566,54.045488),(31.84651,53.992313),(31.841446,53.984561),(31.839689,53.977223),(31.839276,53.969782),(31.837932,53.962134),(31.82646,53.940171),(31.81013,53.882604),(31.79256,53.857437),(31.753803,53.81961),(31.744656,53.794857),(31.787289,53.794392),(31.873175,53.777132),(32.083602,53.809637),(32.10603,53.806949),(32.164837,53.78168),(32.290411,53.760854),(32.325809,53.745454),(32.357384,53.71972),(32.371801,53.714397),(32.421566,53.715792),(32.442133,53.713932),(32.461563,53.7068),(32.48058,53.692073),(32.487815,53.684683),(32.490192,53.677138),(32.487815,53.669645),(32.48058,53.66241),(32.406115,53.639414),(32.399087,53.635384),(32.398001,53.627425),(32.401102,53.609855),(32.411489,53.582364),(32.429421,53.56159),(32.452933,53.546397),(32.48058,53.535648),(32.53081,53.521437),(32.554271,53.510792),(32.569877,53.492602),(32.569877,53.492395),(32.576802,53.486039),(32.584295,53.48423),(32.592511,53.486452),(32.600573,53.492395),(32.618143,53.494255),(32.635713,53.492653),(32.650286,53.487641),(32.647599,53.479321),(32.641656,53.468985),(32.646772,53.457978),(32.658244,53.455653),(32.688526,53.462216),(32.701239,53.462113),(32.719532,53.439478),(32.717794,53.431062),(32.704443,53.366408),(32.717465,53.334937),(32.697931,53.325893),(32.681808,53.326798),(32.666719,53.331345),(32.649872,53.333671),(32.579954,53.324369),(32.583468,53.321217),(32.591891,53.311837),(32.595922,53.308272),(32.569154,53.298841),(32.537321,53.295068),(32.505643,53.297187),(32.48058,53.305171),(32.454845,53.300443),(32.447197,53.288764),(32.455982,53.277783),(32.479133,53.274915),(32.469211,53.255458),(32.455672,53.236674),(32.423633,53.204402),(32.405546,53.192672),(32.389113,53.187633),(32.352836,53.18045),(32.32028,53.162777),(32.295165,53.140866),(32.267983,53.124769),(32.215687,53.125518),(32.210519,53.121772),(32.209072,53.116346),(32.206695,53.112392),(32.206488,53.10986),(32.207005,53.105158),(32.20592,53.100352),(32.201114,53.097561),(32.197393,53.097742),(32.188608,53.099241),(32.152332,53.096347),(32.126545,53.084229),(32.117295,53.081102),(32.082775,53.081955),(32.009911,53.099964),(31.980559,53.098026),(31.954721,53.089991),(31.930743,53.088337),(31.907179,53.091928),(31.855502,53.110248),(31.842635,53.112186),(31.806823,53.110015),(31.796281,53.112392),(31.780468,53.127869),(31.769719,53.169262),(31.756283,53.186574),(31.738558,53.192542),(31.694478,53.192904),(31.674945,53.19523),(31.614018,53.209751),(31.593606,53.210681),(31.573659,53.207089),(31.535935,53.194868),(31.512061,53.194067),(31.464002,53.199984),(31.416356,53.199984),(31.378529,53.182026),(31.359925,53.133838),(31.361165,53.121306),(31.361573,53.120038),(31.364886,53.109731),(31.367057,53.098931),(31.363698,53.088776),(31.354137,53.08211),(31.329436,53.079345),(31.319204,53.076374),(31.269285,53.028418),(31.247064,53.014388),(31.322615,52.977129),(31.339151,52.958061),(31.36623,52.908555),(31.387934,52.887987),(31.418113,52.870237),(31.450979,52.857214),(31.480641,52.850935),(31.505136,52.848894),(31.512784,52.841039),(31.515161,52.828379),(31.523946,52.811661),(31.538364,52.799853),(31.55056,52.794737),(31.560843,52.787063),(31.569525,52.767504),(31.570248,52.725258),(31.547924,52.705699),(31.480641,52.682522),(31.480641,52.68247),(31.4917,52.667096),(31.536142,52.630484),(31.565907,52.59015),(31.579033,52.577825),(31.615207,52.558576),(31.628643,52.54806),(31.624302,52.53806),(31.610763,52.535993),(31.574382,52.541264),(31.559293,52.540541),(31.563634,52.537053),(31.575933,52.524728),(31.558156,52.519999),(31.55056,52.511886),(31.552988,52.502249),(31.565494,52.492637),(31.587922,52.482612),(31.589369,52.458039),(31.583322,52.4283),(31.582857,52.402772),(31.59035,52.391842),(31.600531,52.383393),(31.608075,52.372463),(31.608592,52.354428),(31.602753,52.340915),(31.592262,52.328616),(31.579602,52.318513),(31.567354,52.311382),(31.59712,52.284303),(31.613656,52.273012),(31.631433,52.26477),(31.648693,52.260997),(31.68342,52.257199),(31.699439,52.251463),(31.688897,52.243221),(31.681921,52.230456),(31.679492,52.215858),(31.682386,52.201931),(31.689517,52.19157),(31.698716,52.185446),(31.749049,52.163949),(31.762588,52.149841),(31.766619,52.130075),(31.764345,52.100568),(31.649933,52.096847),(31.474854,52.117776),(31.383076,52.117492),(31.304425,52.097493),(31.284271,52.081241),(31.268871,52.061242),(31.252438,52.044499),(31.228977,52.03822),(31.204896,52.04393),(31.159214,52.068115),(31.134823,52.076667),(31.096324,52.079613),(30.959329,52.074678),(30.934421,52.069717),(30.918815,52.059175),(30.924706,52.042277),(30.919745,52.037264),(30.916851,52.032355),(30.914629,52.027575),(30.91127,52.022562),(30.940519,52.020082),(30.950131,52.006775),(30.941243,51.993804),(30.915198,51.99259),(30.908377,51.998352),(30.902795,51.99998),(30.896904,51.998326),(30.888739,51.994347),(30.880885,51.988171),(30.879024,51.982022),(30.879644,51.97688),(30.879334,51.97365),(30.869102,51.96502),(30.858044,51.957631),(30.845641,51.951352),(30.831379,51.9459),(30.80306,51.939131),(30.802543,51.935539),(30.80523,51.929338),(30.810605,51.925178),(30.810605,51.918382),(30.797065,51.906729),(30.788177,51.900606),(30.779495,51.897919),(30.74973,51.899624),(30.741668,51.897919),(30.737844,51.891588),(30.737224,51.882235),(30.734537,51.873734),(30.71397,51.866964),(30.707562,51.860376),(30.702808,51.853761),(30.69733,51.850738),(30.694539,51.847276),(30.672732,51.829602),(30.666324,51.822393),(30.661983,51.819422),(30.662293,51.815598),(30.669631,51.806064),(30.670562,51.790535),(30.652785,51.77914),(30.630461,51.770019),(30.618162,51.761338),(30.638109,51.755111),(30.64617,51.754542),(30.64617,51.747075),(30.63506,51.739685),(30.632011,51.727981),(30.630564,51.715604),(30.624983,51.706096),(30.613821,51.702918),(30.582815,51.702297),(30.570309,51.69992),(30.575994,51.686407),(30.564315,51.665297),(30.570309,51.6515),(30.563488,51.647314),(30.555323,51.639769),(30.549949,51.637185),(30.552843,51.633335),(30.556615,51.62362),(30.543851,51.620132),(30.515119,51.603673),(30.515119,51.596257),(30.534549,51.585845),(30.543127,51.582589),(30.543127,51.576388),(30.535273,51.575018),(30.531087,51.573313),(30.527728,51.571298),(30.52256,51.568921),(30.52256,51.562719),(30.534136,51.553263),(30.567415,51.547268),(30.584004,51.542256),(30.576407,51.531119),(30.571446,51.526417),(30.562868,51.521766),(30.575064,51.516056),(30.585399,51.509389),(30.58974,51.501974),(30.584004,51.493835),(30.588913,51.48182),(30.597284,51.474275),(30.60762,51.469831),(30.618162,51.467144),(30.618162,51.459651),(30.604829,51.461227),(30.593564,51.458901),(30.586019,51.451848),(30.584004,51.439213),(30.587879,51.427353),(30.595579,51.42495),(30.606018,51.424562),(30.618162,51.418723),(30.622606,51.412108),(30.632424,51.383971),(30.637489,51.378596),(30.644517,51.372679),(30.64555,51.36746),(30.632424,51.364101),(30.638315,51.33586),(30.610462,51.317541),(30.579921,51.303665),(30.577234,51.289015),(30.559354,51.269326),(30.556615,51.267311),(30.555323,51.243462),(30.554657,51.242507),(30.550672,51.236796),(30.539613,51.235168),(30.532017,51.244625),(30.520803,51.252402),(30.508091,51.257673),(30.494655,51.259921),(30.480702,51.258862),(30.465044,51.261937),(30.442772,51.283047),(30.428096,51.291754),(30.413523,51.294054),(30.383757,51.293924),(30.368461,51.297671),(30.355129,51.305267),(30.324536,51.329969),(30.317095,51.340588),(30.319885,51.341596),(30.328257,51.362473),(30.330014,51.370354),(30.330117,51.380715),(30.329032,51.388622),(30.32619,51.395262),(30.320195,51.402083),(30.30738,51.409602),(30.256892,51.425053),(30.242784,51.434407),(30.20537,51.466498),(30.17731,51.479495),(30.14863,51.48443),(30.00869,51.482078),(29.985745,51.477867),(29.925077,51.457868),(29.912572,51.457429),(29.896552,51.464198),(29.886268,51.464793),(29.873608,51.459186),(29.856761,51.439859),(29.846633,51.432908),(29.828649,51.429963),(29.737905,51.439497),(29.725296,51.450452),(29.716511,51.465671),(29.699975,51.483706),(29.682715,51.491199),(29.660804,51.493137),(29.637756,51.490889),(29.618016,51.485644),(29.597345,51.473965),(29.583186,51.460813),(29.5682,51.449626),(29.545049,51.443657),(29.519004,51.441797),(29.505465,51.437456),(29.495543,51.425648),(29.480867,51.401256),(29.466294,51.385056),(29.446399,51.384901),(29.402732,51.39614),(29.379684,51.391541),(29.35326,51.377055),(29.340565,51.370096),(29.319946,51.365574),(29.297002,51.373713),(29.2846,51.391748),(29.276125,51.413581),(29.264859,51.432908),(29.244395,51.447894),(29.227756,51.45562),(29.221038,51.466963),(29.230443,51.492879),(29.226515,51.519053),(29.214836,51.53546),(29.181453,51.567009),(29.160421,51.603311),(29.147967,51.615636),(29.123886,51.625041),(29.083682,51.631217),(29.063218,51.630596),(29.043581,51.626178),(29.023634,51.614034),(28.999966,51.582408),(28.980846,51.569463),(28.954594,51.563262),(28.892892,51.562874),(28.863643,51.558921),(28.831914,51.548276),(28.799875,51.532618),(28.771866,51.511276),(28.752229,51.483706),(28.749439,51.46686),(28.751609,51.428748),(28.746751,51.413995),(28.728768,51.401256),(28.718226,51.411618),(28.711095,51.430247),(28.703963,51.442623),(28.691148,51.443424),(28.677918,51.438076),(28.663552,51.433993),(28.647016,51.439006),(28.637301,51.449626),(28.630893,51.46363),(28.615183,51.51957),(28.612703,51.53993),(28.603918,51.553547),(28.578079,51.56011),(28.488369,51.572021),(28.461239,51.571659),(28.435349,51.56613),(28.385533,51.545046),(28.359643,51.529362),(28.346879,51.525151),(28.333857,51.52838),(28.328172,51.536364),(28.317424,51.56396),(28.310912,51.57445),(28.29851,51.582977),(28.269726,51.594268),(28.257789,51.601761),(28.249366,51.613078),(28.248384,51.622096),(28.248901,51.630596),(28.245128,51.640622),(28.2304,51.651138),(28.21004,51.651965),(28.187612,51.645195),(28.166218,51.633103),(28.146685,51.614396),(28.114232,51.575613),(28.090151,51.562048),(28.070824,51.557629),(27.973155,51.55781),(27.954758,51.560782),(27.936051,51.567009),(27.87559,51.60804),(27.854196,51.615326),(27.831252,51.612923),(27.812545,51.602019),(27.799522,51.585199),(27.793218,51.565019),(27.793735,51.552307),(27.796267,51.541532),(27.797145,51.530603),(27.792546,51.517296),(27.787327,51.510914),(27.74056,51.471407),(27.730121,51.465154),(27.714256,51.463707),(27.700045,51.467531),(27.683767,51.475515),(27.670331,51.484817),(27.666182,51.490194),(27.66413,51.492853),(27.697306,51.543186),(27.705161,51.568352),(27.692655,51.589229),(27.676842,51.59481),(27.620619,51.595896),(27.512426,51.623099),(27.512305,51.623129),(27.477268,51.623672),(27.458768,51.617496),(27.430966,51.598428),(27.409004,51.59171),(27.388385,51.590728),(27.32906,51.596955),(27.289269,51.588971),(27.267462,51.587498),(27.254026,51.595379),(27.25971,51.612303),(27.27449,51.6338),(27.277487,51.651138),(27.247928,51.655659),(27.22364,51.653696),(27.20359,51.655143),(27.189017,51.663773),(27.181059,51.68341),(27.181472,51.710126),(27.184159,51.731443),(27.177855,51.747075),(27.151086,51.756764),(27.109952,51.762423),(27.021585,51.764542),(26.920816,51.742528),(26.854774,51.749349),(26.665741,51.801387),(26.4456,51.805599),(26.431544,51.810224),(26.419245,51.820921),(26.416971,51.830636),(26.419968,51.839731),(26.419762,51.846862),(26.407773,51.850609),(26.175332,51.856707),(26.14536,51.864846),(26.080764,51.900657),(26.050585,51.904817),(25.981132,51.903474),(25.767915,51.928511),(25.683476,51.918021),(25.547396,51.919442),(25.351971,51.921483),(25.1833,51.94975),(25.138031,51.948949),(25.092659,51.939751),(25.002742,51.910476),(24.721829,51.882338),(24.701055,51.882907),(24.639767,51.892131),(24.39079,51.880013),(24.369602,51.875103),(24.347898,51.861151),(24.311518,51.827561),(24.296119,51.808131),(24.272347,51.742889),(24.244132,51.718214),(24.130754,51.669793),(23.981306,51.586),(23.941308,51.581917),(23.912886,51.598583),(23.884878,51.619874),(23.845293,51.629821),(23.820282,51.631268),(23.749692,51.644472),(23.726231,51.644859),(23.628976,51.629046),(23.616677,51.624783),(23.606445,51.618117),(23.594353,51.604965),(23.593319,51.596955),(23.599107,51.588971),(23.626289,51.540886),(23.628666,51.531223),(23.624118,51.515901),(23.615437,51.51311),(23.606238,51.517399),(23.602311,51.530783),(23.588668,51.535899),(23.572752,51.539672),(23.56035,51.555304),(23.567171,51.55489),(23.573992,51.555304),(23.552081,51.578817),(23.5434,51.592743),(23.539886,51.607109),(23.545157,51.63411),(23.543813,51.6438),(23.532444,51.6515),(23.532444,51.658967),(23.541436,51.659742),(23.547534,51.662067),(23.56035,51.671963),(23.555389,51.676304),(23.549291,51.683203),(23.546087,51.689637),(23.550014,51.692427),(23.559213,51.695166),(23.556939,51.701393),(23.550221,51.708421),(23.546604,51.713589),(23.552288,51.736636),(23.556319,51.747359),(23.56035,51.754542),(23.577299,51.766686),(23.59828,51.77467),(23.617504,51.786504),(23.628666,51.809759),(23.610269,51.825184),(23.594559,51.843297),(23.605721,51.851591),(23.609856,51.873269),(23.621845,51.877429),(23.621328,51.882028),(23.620604,51.883217),(23.618847,51.883268),(23.61492,51.88425),(23.621431,51.895567),(23.622155,51.927943),(23.628666,51.946339),(23.647579,51.97409),(23.660705,51.98688),(23.676415,51.994088),(23.664943,52.011168),(23.650783,52.048943),(23.642515,52.061397),(23.641482,52.072921),(23.637451,52.084471),(23.625255,52.089716),(23.61585,52.0923),(23.609856,52.09863),(23.604791,52.106769),(23.598177,52.11452),(23.579366,52.121548),(23.531928,52.120644),(23.512497,52.124442),(23.484695,52.158626),(23.488726,52.16214),(23.491517,52.174103),(23.487693,52.181622),(23.470949,52.171623),(23.464335,52.176351),(23.454413,52.181467),(23.450485,52.185911),(23.43674,52.175679),(23.427231,52.17679),(23.418239,52.182501),(23.405734,52.185911),(23.391368,52.182501),(23.388474,52.184154),(23.395812,52.193379),(23.395812,52.19958),(23.374831,52.200949),(23.312716,52.215289),(23.29959,52.223428),(23.284191,52.219734),(23.212257,52.231671),(23.189726,52.240533),(23.197684,52.25831),(23.19448,52.271462),(23.183525,52.281306),(23.176099,52.285136),(23.168746,52.288928),(23.165645,52.289393),(23.21205,52.347504),(23.230447,52.365074),(23.270548,52.395123),(23.392298,52.509638),(23.480458,52.554416),(23.569031,52.585887),(23.736153,52.614903),(23.868961,52.670042),(23.908752,52.699859),(23.922498,52.742596),(23.920426,52.772625),(23.912059,52.893853),(23.909476,52.901501),(23.905445,52.906539),(23.902034,52.912534),(23.901104,52.923257),(23.903894,52.931137),(23.914953,52.944625),(23.91795,52.9508),(23.917537,52.95881),(23.909062,52.986664),(23.908959,52.993046),(23.911336,53.00506),(23.909682,53.012734),(23.89821,53.02754),(23.867928,53.051027),(23.859349,53.067976),(23.860693,53.087484),(23.870305,53.101256),(23.882397,53.113297),(23.891389,53.127714),(23.893663,53.151951),(23.883121,53.163991),(23.865964,53.172001),(23.848084,53.183809),(23.836198,53.199286),(23.828447,53.213833),(23.818628,53.227967),(23.800645,53.242462),(23.782972,53.270936),(23.742974,53.365478),(23.722924,53.397104),(23.675795,53.455705),(23.590942,53.611251),(23.567068,53.681014),(23.564794,53.742405),(23.56469,53.742405),(23.56469,53.742457),(23.540402,53.763593),(23.52986,53.78416),(23.520662,53.837335),(23.51043,53.862502),(23.497201,53.885549),(23.486866,53.909992),(23.485625,53.939293),(23.520145,53.93397),(23.609959,53.901827),(23.627116,53.89821),(23.642618,53.898985),(23.691918,53.911491),(23.754446,53.916762),(23.77181,53.924307),(23.787209,53.927924),(23.794134,53.927097),(23.812117,53.920844),(23.820592,53.919294),(23.828344,53.920069),(23.904721,53.946062),(23.919811,53.947768),(23.935417,53.943944),(23.964873,53.93273),(23.981306,53.930404),(24.045385,53.930869),(24.074943,53.935727),(24.136748,53.955416),(24.169615,53.95893),(24.201137,53.95247),(24.229146,53.932368),(24.246406,53.903636),(24.257051,53.894024),(24.276378,53.891802),(24.310175,53.892681),(24.341904,53.887048),(24.377871,53.886841),(24.414044,53.897693),(24.528663,53.958361),(24.579099,53.975621),(24.642971,53.983011),(24.666845,53.993863),(24.670566,53.994276),(24.674287,53.993708),(24.677904,53.992313),(24.690823,53.972831),(24.705603,53.964201),(24.723896,53.963426),(24.788698,53.969678),(24.806372,53.975311),(24.819084,53.992313),(24.819187,53.992416),(24.819291,53.992519),(24.819394,53.992571),(24.821358,54.019908),(24.81247,54.038925),(24.799034,54.05603),(24.787148,54.077631),(24.782497,54.092875),(24.784254,54.096234),(24.790559,54.096596),(24.799447,54.10321),(24.818154,54.110135),(24.818671,54.113959),(24.81743,54.120625),(24.817534,54.128222),(24.821565,54.134526),(24.850984,54.144951),(24.855981,54.146722),(24.902283,54.149771),(24.948275,54.145895),(25.027133,54.12786),(25.071989,54.132201),(25.115603,54.148789),(25.156531,54.175712),(25.173068,54.19659),(25.193532,54.243667),(25.206244,54.256793),(25.224124,54.25855),(25.287583,54.245217),(25.369748,54.247904),(25.394036,54.257154),(25.434137,54.291829),(25.459355,54.298909),(25.472068,54.297152),(25.478682,54.293328),(25.483488,54.287695),(25.553975,54.231265),(25.5455,54.227492),(25.522142,54.228629),(25.501678,54.221756),(25.499301,54.211834),(25.523382,54.204238),(25.528963,54.195659),(25.521729,54.188218),(25.493203,54.18243),(25.485762,54.175402),(25.493307,54.157522),(25.516044,54.144758),(25.543019,54.136955),(25.631593,54.128325),(25.653503,54.128739),(25.663942,54.132356),(25.679755,54.145327),(25.690607,54.148221),(25.728124,54.145223),(25.74001,54.146257),(25.763057,54.156437),(25.771016,54.172147),(25.771946,54.21788),(25.775046,54.22248),(25.786829,54.231781),(25.789206,54.236225),(25.786622,54.242323),(25.781764,54.244442),(25.776287,54.245631),(25.772256,54.248938),(25.765641,54.259997),(25.757786,54.268988),(25.748898,54.276326),(25.739286,54.282579),(25.734739,54.283199),(25.723267,54.280977),(25.719236,54.281546),(25.715412,54.285215),(25.704766,54.302991),(25.701769,54.312758),(25.695981,54.320975),(25.682339,54.325626),(25.667353,54.3233),(25.607408,54.304852),(25.564,54.303146),(25.542296,54.308056),(25.528653,54.320975),(25.529997,54.346141),(25.548807,54.367742),(25.594696,54.400091),(25.612576,54.421796),(25.616296,54.441071),(25.615573,54.461896),(25.619914,54.488096),(25.630869,54.508354),(25.646165,54.520394),(25.70735,54.54153),(25.726471,54.55295),(25.740423,54.568505),(25.745591,54.5884),(25.739906,54.607727),(25.715102,54.643229),(25.708384,54.663021),(25.709521,54.67563),(25.713862,54.685552),(25.719133,54.69475),(25.72306,54.705344),(25.72461,54.715679),(25.7212,54.766736),(25.7243,54.78012),(25.734946,54.78906),(25.756753,54.800377),(25.766571,54.803271),(25.773393,54.803374),(25.778457,54.805286),(25.782901,54.81371),(25.782281,54.821978),(25.773703,54.841925),(25.772256,54.850865),(25.782695,54.869675),(25.802538,54.881612),(25.825793,54.891999),(25.84667,54.90621),(25.853078,54.916442),(25.854835,54.925486),(25.858452,54.932979),(25.870234,54.938818),(25.907855,54.948068),(25.926148,54.947758),(25.962529,54.942901),(25.981132,54.942694),(26.102572,54.957008),(26.138642,54.968946),(26.153628,54.978506),(26.170325,54.99326),(26.187838,55.008736),(26.224631,55.054884),(26.229386,55.06341),(26.231246,55.075451),(26.230316,55.100255),(26.233003,55.111624),(26.264422,55.140098),(26.309484,55.144645),(26.420278,55.128109),(26.428753,55.128212),(26.438572,55.130021),(26.444773,55.133948),(26.450664,55.13984),(26.459139,55.144749),(26.473402,55.145679),(26.578925,55.118549),(26.600939,55.120823),(26.616339,55.135395),(26.627501,55.164438),(26.633909,55.192188),(26.641143,55.202833),(26.656956,55.215339),(26.685689,55.231617),(26.701088,55.236836),(26.739901,55.242776),(26.76558,55.246706),(26.789661,55.257197),(26.80072,55.27332),(26.791574,55.290166),(26.768629,55.300243),(26.602696,55.316883),(26.542338,55.307581),(26.525388,55.308098),(26.465754,55.320862),(26.450044,55.327063),(26.445496,55.337502),(26.455212,55.356105),(26.479086,55.381375),(26.486011,55.390832),(26.49924,55.428091),(26.507818,55.439149),(26.543268,55.459613),(26.546472,55.471241),(26.527662,55.492169),(26.532106,55.516251),(26.55164,55.534492),(26.576134,55.550564),(26.595565,55.56803),(26.605177,55.590096),(26.607657,55.616451),(26.603833,55.643271),(26.594531,55.666991),(26.615615,55.687971),(26.64011,55.695568),(26.666878,55.693966),(26.720105,55.681874),(26.743049,55.682856),(26.822838,55.70611),(26.842785,55.719339),(26.900146,55.778715),(26.957817,55.818584),(26.979426,55.826291),(26.981071,55.826878),(27.110779,55.836283),(27.151448,55.832459),(27.173204,55.825741),(27.235526,55.795846),(27.263018,55.787216),(27.282448,55.791867),(27.329163,55.817576),(27.349627,55.831219),(27.374587,55.814837),(27.405851,55.804347),(27.438821,55.79874),(27.564601,55.792229),(27.592713,55.794244),(27.601498,55.809618),(27.610128,55.83096),(27.617156,55.878554),(27.64501,55.922841),(27.744435,55.959738),(27.776991,55.992397),(27.781229,56.016375),(27.812545,56.034514),(27.880654,56.063866),(27.892747,56.077095),(27.901532,56.089342),(27.911453,56.100246),(27.92706,56.109367),(27.939669,56.113062),(27.98101,56.118023),(28.023798,56.12965),(28.0511,56.140666),(28.06824,56.147582),(28.110976,56.156806),(28.148907,56.142414)] +Bolivia [(-65.29247,-11.504723),(-65.257562,-11.495318),(-65.233042,-11.50834),(-65.222655,-11.517435),(-65.215989,-11.530148),(-65.215989,-11.539966),(-65.223611,-11.55795),(-65.223404,-11.571076),(-65.219244,-11.584511),(-65.214542,-11.587509),(-65.207359,-11.587612),(-65.195447,-11.59216),(-65.173976,-11.606526),(-65.167439,-11.615517),(-65.190926,-11.623165),(-65.193354,-11.632364),(-65.189272,-11.657065),(-65.191339,-11.666884),(-65.200253,-11.681973),(-65.20232,-11.691482),(-65.201855,-11.727965),(-65.196197,-11.741814),(-65.182451,-11.756697),(-65.164467,-11.769823),(-65.151497,-11.77313),(-65.143538,-11.763829),(-65.138009,-11.715253),(-65.133978,-11.702644),(-65.127855,-11.694582),(-65.113411,-11.690551),(-65.110465,-11.699957),(-65.113566,-11.722591),(-65.0992,-11.736233),(-65.081062,-11.743468),(-65.065559,-11.75308),(-65.056309,-11.785636),(-65.043312,-11.80765),(-65.038454,-11.818709),(-65.037447,-11.828424),(-65.039204,-11.848681),(-65.038454,-11.8585),(-65.033674,-11.879791),(-65.028093,-11.888886),(-65.017939,-11.89364),(-65.014528,-11.893847),(-65.00458,-11.898394),(-64.998121,-11.908936),(-64.996493,-11.921235),(-65.00086,-11.931054),(-65.014632,-11.950277),(-65.016337,-11.968467),(-65.009438,-11.98428),(-64.997449,-11.996269),(-64.983445,-12.003917),(-64.967606,-12.007948),(-64.925154,-12.009912),(-64.904483,-12.014046),(-64.883503,-12.021384),(-64.862083,-12.024795),(-64.839836,-12.016733),(-64.827615,-12.022314),(-64.809295,-12.025828),(-64.7925,-12.032546),(-64.785188,-12.047429),(-64.780486,-12.067686),(-64.768574,-12.086496),(-64.736793,-12.119776),(-64.744183,-12.133625),(-64.739532,-12.144581),(-64.728396,-12.149852),(-64.716278,-12.146544),(-64.713642,-12.138483),(-64.710283,-12.111921),(-64.706382,-12.106133),(-64.689923,-12.104376),(-64.68646,-12.109957),(-64.681577,-12.126597),(-64.681577,-12.134865),(-64.688579,-12.147164),(-64.688992,-12.153882),(-64.685944,-12.159153),(-64.677184,-12.167628),(-64.67535,-12.171039),(-64.66486,-12.180961),(-64.640572,-12.19512),(-64.593417,-12.215997),(-64.58104,-12.217444),(-64.554892,-12.217754),(-64.548433,-12.219098),(-64.532103,-12.232224),(-64.511484,-12.242662),(-64.510347,-12.239562),(-64.510244,-12.237288),(-64.508848,-12.236048),(-64.49425,-12.238012),(-64.489392,-12.239459),(-64.473037,-12.253204),(-64.469058,-12.261059),(-64.468153,-12.272635),(-64.469316,-12.294856),(-64.474432,-12.316456),(-64.492441,-12.356557),(-64.48978,-12.373611),(-64.48363,-12.377641),(-64.461642,-12.385289),(-64.452521,-12.390354),(-64.444072,-12.399759),(-64.432367,-12.420946),(-64.424874,-12.431385),(-64.411335,-12.445131),(-64.395729,-12.457326),(-64.376763,-12.465905),(-64.353199,-12.469212),(-64.312659,-12.461874),(-64.297699,-12.465698),(-64.287544,-12.497117),(-64.278242,-12.499288),(-64.268708,-12.497221),(-64.26385,-12.495257),(-64.256822,-12.490813),(-64.245712,-12.479444),(-64.236565,-12.475413),(-64.215223,-12.473759),(-64.19755,-12.47872),(-64.183106,-12.487919),(-64.155252,-12.516961),(-64.144788,-12.520372),(-64.134763,-12.510863),(-64.119828,-12.489676),(-64.104455,-12.507142),(-64.044458,-12.509106),(-64.022832,-12.537838),(-64.004176,-12.535461),(-63.972447,-12.523886),(-63.959321,-12.52802),(-63.950019,-12.536805),(-63.939064,-12.544143),(-63.921236,-12.544349),(-63.909143,-12.534324),(-63.902477,-12.525436),(-63.896896,-12.515721),(-63.863203,-12.474793),(-63.862893,-12.469212),(-63.845581,-12.466938),(-63.814989,-12.457223),(-63.801398,-12.454949),(-63.657738,-12.475413),(-63.654844,-12.478307),(-63.600428,-12.506109),(-63.558726,-12.539905),(-63.542189,-12.547967),(-63.520227,-12.551171),(-63.507514,-12.551584),(-63.496145,-12.553341),(-63.485758,-12.557475),(-63.476198,-12.564813),(-63.468757,-12.575769),(-63.467,-12.585794),(-63.46638,-12.595302),(-63.462504,-12.605121),(-63.454236,-12.612046),(-63.445347,-12.614836),(-63.438164,-12.619177),(-63.435219,-12.630339),(-63.431446,-12.636954),(-63.393309,-12.664549),(-63.371088,-12.669923),(-63.359461,-12.674677),(-63.330419,-12.697002),(-63.317913,-12.701963),(-63.247323,-12.701342),(-63.235955,-12.698552),(-63.232027,-12.691111),(-63.222674,-12.682532),(-63.201848,-12.667856),(-63.197301,-12.666823),(-63.18557,-12.668476),(-63.180712,-12.667856),(-63.176113,-12.663619),(-63.171669,-12.652353),(-63.16707,-12.647392),(-63.139475,-12.63561),(-63.136684,-12.63375),(-63.125057,-12.63592),(-63.105213,-12.645222),(-63.075034,-12.652663),(-63.064751,-12.666202),(-63.057826,-12.701963),(-63.053588,-12.70837),(-63.047077,-12.713538),(-63.04346,-12.719119),(-63.051108,-12.730798),(-63.052193,-12.736482),(-63.051108,-12.742167),(-63.047904,-12.746714),(-63.025683,-12.765731),(-63.014728,-12.77772),(-63.010025,-12.787952),(-63.00708,-12.809966),(-62.999173,-12.829087),(-62.988011,-12.843866),(-62.975299,-12.852858),(-62.964498,-12.856578),(-62.955145,-12.857509),(-62.946515,-12.855235),(-62.93804,-12.849447),(-62.928893,-12.846036),(-62.923777,-12.852858),(-62.92047,-12.862263),(-62.867915,-12.929029),(-62.865073,-12.93554),(-62.860112,-12.940294),(-62.849001,-12.942155),(-62.838925,-12.942775),(-62.831431,-12.944945),(-62.826677,-12.950009),(-62.825075,-12.958691),(-62.820166,-12.974918),(-62.807402,-12.98887),(-62.789832,-13.000652),(-62.770453,-13.010471),(-62.768645,-12.990627),(-62.757896,-12.985149),(-62.741308,-12.985356),(-62.722032,-12.982566),(-62.709165,-12.974607),(-62.698985,-12.966649),(-62.686634,-12.964996),(-62.667462,-12.976364),(-62.657902,-12.984633),(-62.651339,-12.992487),(-62.647515,-13.001583),(-62.64519,-13.025354),(-62.641676,-13.030005),(-62.554601,-13.066798),(-62.492692,-13.065041),(-62.472125,-13.068452),(-62.458483,-13.07796),(-62.430939,-13.109793),(-62.428717,-13.114547),(-62.427477,-13.124572),(-62.424118,-13.126536),(-62.410269,-13.125709),(-62.406703,-13.126536),(-62.389701,-13.137078),(-62.381846,-13.139869),(-62.335028,-13.14514),(-62.321592,-13.143693),(-62.310533,-13.133978),(-62.290328,-13.142039),(-62.273636,-13.138422),(-62.259167,-13.130774),(-62.22165,-13.121265),(-62.211521,-13.120335),(-62.200927,-13.122712),(-62.173952,-13.140799),(-62.172454,-13.117855),(-62.162377,-13.120748),(-62.139846,-13.147),(-62.129149,-13.150927),(-62.120467,-13.148757),(-62.114679,-13.150204),(-62.112509,-13.164777),(-62.112457,-13.231801),(-62.109408,-13.248958),(-62.099228,-13.2626),(-62.077731,-13.277896),(-62.033599,-13.325542),(-62.019698,-13.33257),(-62.018665,-13.336808),(-62.003265,-13.360475),(-61.997787,-13.364093),(-61.97536,-13.374118),(-61.958358,-13.385487),(-61.909834,-13.431892),(-61.899499,-13.43923),(-61.879913,-13.449152),(-61.872317,-13.456077),(-61.868596,-13.463932),(-61.847977,-13.530801),(-61.83635,-13.540619),(-61.813974,-13.544237),(-61.794285,-13.540309),(-61.774235,-13.533385),(-61.75434,-13.530284),(-61.735116,-13.538036),(-61.715737,-13.526563),(-61.692896,-13.517985),(-61.668092,-13.512507),(-61.597114,-13.5061),(-61.59306,-13.506794),(-61.588071,-13.50765),(-61.560863,-13.532661),(-61.550812,-13.538036),(-61.503425,-13.548164),(-61.45875,-13.54372),(-61.415471,-13.528114),(-61.347724,-13.493697),(-61.332376,-13.494834),(-61.317131,-13.500932),(-61.281216,-13.507236),(-61.260029,-13.520052),(-61.248556,-13.524393),(-61.237885,-13.524393),(-61.215044,-13.517468),(-61.149105,-13.519845),(-61.139312,-13.514058),(-61.133576,-13.494007),(-61.118719,-13.484499),(-61.076009,-13.478298),(-61.072185,-13.47437),(-61.060118,-13.467136),(-61.047225,-13.464552),(-61.041256,-13.474577),(-61.041256,-13.515195),(-61.022007,-13.535245),(-60.977875,-13.542997),(-60.929299,-13.5462),(-60.896743,-13.552918),(-60.878243,-13.570385),(-60.870673,-13.575449),(-60.801969,-13.604181),(-60.725572,-13.662845),(-60.652701,-13.7188),(-60.618078,-13.736886),(-60.595082,-13.741641),(-60.588623,-13.744535),(-60.584876,-13.749082),(-60.579579,-13.761071),(-60.575135,-13.765619),(-60.566066,-13.769959),(-60.490618,-13.787736),(-60.472635,-13.797865),(-60.465271,-13.816572),(-60.463126,-13.843443),(-60.457313,-13.870935),(-60.448812,-13.896773),(-60.438554,-13.918374),(-60.430441,-13.926746),(-60.408634,-13.944522),(-60.406722,-13.950414),(-60.411605,-13.960542),(-60.40543,-13.966743),(-60.39574,-13.970981),(-60.389694,-13.974908),(-60.387188,-13.98328),(-60.387963,-13.989378),(-60.396515,-14.008394),(-60.409409,-14.055317),(-60.419434,-14.076917),(-60.437236,-14.092007),(-60.461395,-14.098622),(-60.468268,-14.103686),(-60.473513,-14.117225),(-60.477647,-14.139963),(-60.478939,-14.1627),(-60.475684,-14.176033),(-60.462403,-14.19846),(-60.462015,-14.223885),(-60.465633,-14.251067),(-60.464341,-14.278352),(-60.454316,-14.296646),(-60.399926,-14.340777),(-60.391348,-14.357107),(-60.376698,-14.419946),(-60.343806,-14.490949),(-60.338431,-14.5326),(-60.369204,-14.542832),(-60.35272,-14.575388),(-60.30404,-14.608151),(-60.291716,-14.630062),(-60.288822,-14.69073),(-60.284869,-14.772689),(-60.280889,-14.854648),(-60.276859,-14.936607),(-60.272905,-15.018565),(-60.269805,-15.083368),(-60.274972,-15.09515),(-60.389307,-15.096493),(-60.582241,-15.098871),(-60.529479,-15.143726),(-60.460181,-15.224238),(-60.352125,-15.349605),(-60.260451,-15.456058),(-60.246395,-15.478279),(-60.237972,-15.50267),(-60.224821,-15.651395),(-60.206992,-15.853036),(-60.193763,-16.001451),(-60.186968,-16.108731),(-60.17981,-16.222006),(-60.16069,-16.264794),(-60.129839,-16.273062),(-60.060412,-16.27544),(-59.87001,-16.282157),(-59.679557,-16.288772),(-59.489103,-16.295387),(-59.298675,-16.302105),(-59.108248,-16.308719),(-58.917769,-16.315437),(-58.727444,-16.322155),(-58.536965,-16.32877),(-58.464721,-16.33125),(-58.442242,-16.32846),(-58.421623,-16.318434),(-58.414233,-16.308926),(-58.400332,-16.284225),(-58.392943,-16.279367),(-58.349741,-16.2804),(-58.339561,-16.289289),(-58.334548,-16.386647),(-58.338837,-16.400497),(-58.362299,-16.419307),(-58.36359,-16.43667),(-58.342661,-16.473154),(-58.356097,-16.509534),(-58.455885,-16.619088),(-58.472679,-16.649577),(-58.480276,-16.683683),(-58.476658,-16.726988),(-58.456246,-16.802436),(-58.452939,-16.841296),(-58.455109,-16.848324),(-58.464049,-16.866101),(-58.466633,-16.875506),(-58.466272,-16.887289),(-58.45635,-16.937001),(-58.439193,-16.967284),(-58.432475,-16.98537),(-58.426791,-17.055237),(-58.421623,-17.070636),(-58.410978,-17.088206),(-58.406275,-17.110117),(-58.404828,-17.199621),(-58.399144,-17.237448),(-58.38116,-17.267214),(-58.343695,-17.286954),(-58.312172,-17.290881),(-58.302147,-17.293775),(-58.292277,-17.299666),(-58.274397,-17.314032),(-58.264527,-17.320027),(-58.256155,-17.322507),(-58.239929,-17.325195),(-58.231557,-17.329742),(-58.22024,-17.344315),(-58.204065,-17.377388),(-58.188407,-17.38855),(-58.174558,-17.392064),(-58.163189,-17.393407),(-58.152234,-17.396405),(-58.139366,-17.40457),(-58.11203,-17.433508),(-58.101126,-17.44126),(-58.065727,-17.455936),(-58.053687,-17.462344),(-58.010175,-17.49676),(-57.981495,-17.508749),(-57.943564,-17.517844),(-57.905686,-17.520325),(-57.877005,-17.512263),(-57.854113,-17.508543),(-57.836749,-17.511023),(-57.814425,-17.519705),(-57.800731,-17.533451),(-57.790757,-17.555775),(-57.788277,-17.58058),(-57.789569,-17.604144),(-57.787553,-17.620474),(-57.787967,-17.658921),(-57.7859,-17.677525),(-57.777941,-17.695508),(-57.767554,-17.708737),(-57.754894,-17.720623),(-57.740373,-17.730234),(-57.724456,-17.736642),(-57.732983,-17.768475),(-57.729314,-17.779327),(-57.711331,-17.800824),(-57.696809,-17.825112),(-57.698773,-17.843096),(-57.730089,-17.846093),(-57.636916,-18.001122),(-57.585498,-18.122769),(-57.551082,-18.183643),(-57.511188,-18.205554),(-57.490724,-18.205554),(-57.474549,-18.208035),(-57.465661,-18.21775),(-57.466798,-18.239557),(-57.535734,-18.240488),(-57.553872,-18.244518),(-57.56674,-18.256094),(-57.583018,-18.305703),(-57.630043,-18.448434),(-57.677017,-18.59106),(-57.723991,-18.733687),(-57.771068,-18.876314),(-57.782334,-18.910421),(-57.742233,-18.913625),(-57.731794,-18.921996),(-57.727557,-18.941943),(-57.721769,-19.001165),(-57.715826,-19.044573),(-57.771895,-19.048707),(-57.789,-19.059249),(-57.812048,-19.105034),(-57.843105,-19.166839),(-57.911628,-19.302852),(-57.980048,-19.438864),(-58.011209,-19.500669),(-58.011209,-19.500772),(-58.011209,-19.500876),(-58.011261,-19.500876),(-58.02356,-19.52568),(-58.050483,-19.580147),(-58.077458,-19.634718),(-58.089757,-19.659419),(-58.124639,-19.729906),(-58.116939,-19.758018),(-58.04113,-19.82344),(-57.960463,-19.8931),(-57.859745,-19.980123),(-57.86977,-19.992939),(-57.881139,-20.009165),(-57.895609,-20.024151),(-57.917933,-20.034177),(-57.943203,-20.028079),(-57.959171,-20.026218),(-57.966354,-20.031076),(-57.968938,-20.039758),(-57.975811,-20.047509),(-57.985681,-20.05309),(-57.997101,-20.055261),(-58.022371,-20.064976),(-58.053532,-20.10766),(-58.079577,-20.117376),(-58.086243,-20.12089),(-58.098284,-20.13815),(-58.103555,-20.144041),(-58.116319,-20.150449),(-58.158797,-20.165125),(-58.15952,-20.13908),(-58.144741,-20.086163),(-58.143139,-20.065286),(-58.144482,-20.023118),(-58.141847,-20.000897),(-58.145154,-19.969581),(-58.162207,-19.912324),(-58.164223,-19.880284),(-58.161174,-19.847211),(-58.164429,-19.832949),(-58.175282,-19.821373),(-58.214452,-19.798325),(-58.30778,-19.743652),(-58.401004,-19.688875),(-58.49428,-19.634201),(-58.587504,-19.579424),(-58.680832,-19.52475),(-58.774108,-19.470077),(-58.867436,-19.4153),(-58.96066,-19.360523),(-59.011096,-19.330964),(-59.039673,-19.311637),(-59.069646,-19.291483),(-59.089541,-19.286729),(-59.170725,-19.287762),(-59.359085,-19.290139),(-59.547498,-19.292413),(-59.735858,-19.29479),(-59.924322,-19.297064),(-60.006384,-19.298097),(-60.156995,-19.329),(-60.495476,-19.398453),(-60.833931,-19.467906),(-61.17236,-19.537463),(-61.51084,-19.606916),(-61.532286,-19.610016),(-61.579544,-19.616838),(-61.626725,-19.623659),(-61.648222,-19.62676),(-61.737235,-19.639575),(-61.753203,-19.64588),(-61.761213,-19.657765),(-61.780074,-19.706961),(-61.808186,-19.780342),(-61.836298,-19.853826),(-61.86441,-19.927413),(-61.892522,-20.000897),(-61.892574,-20.000897),(-61.905545,-20.026735),(-61.918412,-20.052573),(-61.92474,-20.065128),(-61.931383,-20.078308),(-61.94425,-20.104146),(-61.960942,-20.128021),(-61.977582,-20.151792),(-61.99417,-20.175667),(-62.01081,-20.199541),(-62.05551,-20.260416),(-62.10021,-20.321187),(-62.144962,-20.382062),(-62.189662,-20.442834),(-62.210573,-20.471305),(-62.232398,-20.501021),(-62.26883,-20.553111),(-62.277305,-20.579776),(-62.276168,-20.670727),(-62.275031,-20.75868),(-62.273791,-20.854798),(-62.271886,-21.000424),(-62.271879,-21.000939),(-62.275703,-21.066568),(-62.307898,-21.169301),(-62.340402,-21.272861),(-62.37549,-21.384482),(-62.411974,-21.500857),(-62.446184,-21.606897),(-62.492796,-21.751798),(-62.529228,-21.865073),(-62.572843,-22.000775),(-62.599404,-22.089865),(-62.627516,-22.184536),(-62.650357,-22.234456),(-62.658987,-22.231665),(-62.661829,-22.224534),(-62.663225,-22.215542),(-62.667462,-22.20655),(-62.679296,-22.194768),(-62.722032,-22.166243),(-62.769368,-22.144745),(-62.783476,-22.130896),(-62.790917,-22.113223),(-62.791072,-22.081597),(-62.79779,-22.069401),(-62.795051,-22.061133),(-62.795,-22.051211),(-62.801356,-22.013487),(-62.804353,-22.004082),(-62.818564,-22.000878),(-62.862024,-21.99323),(-63.009922,-22.000671),(-63.010439,-22.000775),(-63.324916,-21.999121),(-63.639392,-21.997468),(-63.67784,-22.003669),(-63.693911,-22.01204),(-63.74042,-22.050591),(-63.751789,-22.045733),(-63.793026,-22.011524),(-63.813129,-22.003049),(-63.906405,-21.997157),(-63.933173,-22.001808),(-63.947591,-22.007596),(-63.95095,-22.0108),(-63.951466,-22.016691),(-63.961027,-22.039119),(-63.964644,-22.058549),(-63.968106,-22.066507),(-63.975806,-22.073845),(-63.983454,-22.077359),(-63.990379,-22.079323),(-63.99565,-22.08294),(-64.004383,-22.09927),(-64.019783,-22.155391),(-64.051021,-22.229185),(-64.058798,-22.240347),(-64.078875,-22.250579),(-64.086239,-22.257917),(-64.160549,-22.438474),(-64.184708,-22.471237),(-64.236333,-22.516712),(-64.250828,-22.54069),(-64.293978,-22.690862),(-64.294598,-22.762899),(-64.303486,-22.782329),(-64.326947,-22.859224),(-64.325294,-22.871936),(-64.343897,-22.863668),(-64.347721,-22.816952),(-64.359994,-22.803),(-64.352321,-22.779125),(-64.355731,-22.751943),(-64.368986,-22.729723),(-64.390768,-22.720524),(-64.401723,-22.712359),(-64.428285,-22.659029),(-64.438233,-22.651898),(-64.447224,-22.648694),(-64.45371,-22.642906),(-64.456242,-22.628023),(-64.455208,-22.616655),(-64.450454,-22.597121),(-64.449421,-22.587096),(-64.445545,-22.576244),(-64.437251,-22.567355),(-64.429835,-22.557123),(-64.428285,-22.542344),(-64.483734,-22.491184),(-64.498126,-22.472787),(-64.504043,-22.449946),(-64.507815,-22.443952),(-64.525747,-22.432997),(-64.531328,-22.425658),(-64.531069,-22.41553),(-64.523292,-22.396203),(-64.524558,-22.385351),(-64.537891,-22.374292),(-64.558871,-22.36065),(-64.572023,-22.343183),(-64.562075,-22.319825),(-64.549156,-22.305356),(-64.542257,-22.291506),(-64.542774,-22.275487),(-64.551843,-22.255023),(-64.560215,-22.243137),(-64.58688,-22.212752),(-64.596207,-22.20655),(-64.616284,-22.20221),(-64.638711,-22.191668),(-64.65778,-22.178438),(-64.679045,-22.175441),(-64.687442,-22.172857),(-64.695245,-22.174201),(-64.713694,-22.181746),(-64.722892,-22.183296),(-64.762321,-22.174408),(-64.832472,-22.137511),(-65.020368,-22.096583),(-65.190478,-22.098474),(-65.457344,-22.101441),(-65.510467,-22.095859),(-65.579894,-22.086454),(-65.588524,-22.087281),(-65.593433,-22.091105),(-65.597878,-22.095653),(-65.605319,-22.099063),(-65.744613,-22.11405),(-65.775309,-22.105058),(-65.804557,-22.085834),(-65.932689,-21.944551),(-65.954497,-21.933079),(-66.046532,-21.917989),(-66.051752,-21.912615),(-66.063586,-21.864039),(-66.094488,-21.83293),(-66.137819,-21.812466),(-66.222465,-21.786938),(-66.240009,-21.792415),(-66.287551,-21.956953),(-66.297163,-22.050281),(-66.301504,-22.064957),(-66.307602,-22.077049),(-66.309824,-22.078186),(-66.314061,-22.075809),(-66.32667,-22.077359),(-66.335249,-22.08232),(-66.343,-22.090485),(-66.349175,-22.1002),(-66.353232,-22.109605),(-66.377468,-22.127072),(-66.510328,-22.162935),(-66.626755,-22.192598),(-66.630579,-22.197352),(-66.633267,-22.204897),(-66.636315,-22.211305),(-66.64138,-22.212545),(-66.657813,-22.207377),(-66.677398,-22.205103),(-66.688457,-22.201693),(-66.699309,-22.200763),(-66.712848,-22.206447),(-66.735896,-22.225051),(-66.748867,-22.244894),(-66.756101,-22.268252),(-66.767935,-22.342563),(-66.77517,-22.365197),(-66.788037,-22.38008),(-66.790518,-22.388348),(-66.784472,-22.417287),(-66.785092,-22.427622),(-66.796461,-22.434857),(-66.90865,-22.467516),(-66.935315,-22.480539),(-66.955572,-22.500693),(-66.965236,-22.514335),(-66.978155,-22.5225),(-66.993555,-22.525807),(-67.015672,-22.523844),(-67.032725,-22.524567),(-67.024354,-22.617585),(-67.026628,-22.639392),(-67.037686,-22.654585),(-67.113237,-22.710086),(-67.142693,-22.742745),(-67.193904,-22.822223),(-67.510267,-22.885785),(-67.547113,-22.892503),(-67.616772,-22.897258),(-67.77144,-22.888679),(-67.803583,-22.878654),(-67.811283,-22.871626),(-67.833038,-22.840103),(-67.837948,-22.840827),(-67.876343,-22.833592),(-67.887092,-22.818709),(-67.891484,-22.792871),(-67.891329,-22.715667),(-67.858721,-22.566838),(-67.859187,-22.547305),(-67.889676,-22.491184),(-67.898202,-22.441471),(-67.906936,-22.420077),(-67.94528,-22.354242),(-67.951894,-22.334088),(-67.952204,-22.314141),(-67.934789,-22.251716),(-67.935978,-22.232285),(-67.944969,-22.208307),(-67.97303,-22.170584),(-67.976595,-22.159835),(-67.973288,-22.15043),(-67.960627,-22.133687),(-67.957165,-22.124592),(-67.960627,-22.102887),(-67.972616,-22.078806),(-67.990186,-22.057619),(-68.010237,-22.044907),(-68.058761,-22.000775),(-68.082739,-21.980311),(-68.092041,-21.968425),(-68.096226,-21.953956),(-68.107595,-21.789625),(-68.128369,-21.712317),(-68.190588,-21.60638),(-68.198442,-21.57186),(-68.19374,-21.328464),(-68.207537,-21.284333),(-68.41631,-20.959805),(-68.432071,-20.945025),(-68.453931,-20.939754),(-68.495995,-20.941615),(-68.510206,-20.940168),(-68.531554,-20.927688),(-68.555113,-20.913916),(-68.57196,-20.872368),(-68.572786,-20.742867),(-68.565087,-20.719509),(-68.550204,-20.699046),(-68.481629,-20.643028),(-68.480957,-20.624735),(-68.510206,-20.601894),(-68.536303,-20.591765),(-68.583328,-20.563033),(-68.685544,-20.516834),(-68.707559,-20.500918),(-68.729676,-20.479214),(-68.750708,-20.451825),(-68.76541,-20.421233),(-68.769028,-20.390537),(-68.757685,-20.364286),(-68.735154,-20.347542),(-68.678516,-20.328629),(-68.718049,-20.261966),(-68.729469,-20.228687),(-68.728281,-20.189619),(-68.723268,-20.16099),(-68.726265,-20.150965),(-68.759028,-20.144144),(-68.789466,-20.125851),(-68.792592,-20.106627),(-68.775539,-20.089677),(-68.652885,-20.054227),(-68.622137,-20.051333),(-68.607771,-20.053607),(-68.597178,-20.056707),(-68.588341,-20.055571),(-68.579349,-20.045442),(-68.574543,-20.03521),(-68.56612,-20.000897),(-68.564932,-19.981363),(-68.544623,-19.934751),(-68.541574,-19.920385),(-68.542917,-19.914184),(-68.546948,-19.908189),(-68.551547,-19.894443),(-68.552529,-19.864885),(-68.553511,-19.857857),(-68.561108,-19.844317),(-68.570823,-19.837599),(-68.60157,-19.825404),(-68.618882,-19.813415),(-68.650353,-19.781582),(-68.698412,-19.746339),(-68.70544,-19.733833),(-68.703218,-19.71554),(-68.690299,-19.695386),(-68.648596,-19.660349),(-68.630974,-19.641952),(-68.576736,-19.564607),(-68.532117,-19.500979),(-68.532117,-19.500876),(-68.510206,-19.471627),(-68.496357,-19.458398),(-68.456101,-19.441241),(-68.447626,-19.434627),(-68.455016,-19.415403),(-68.472586,-19.395456),(-68.493205,-19.376956),(-68.600072,-19.303368),(-68.635728,-19.290553),(-68.663324,-19.273603),(-68.698464,-19.215622),(-68.721253,-19.191747),(-68.815407,-19.113303),(-68.847369,-19.092012),(-68.894782,-19.076509),(-68.908502,-19.067931),(-68.91969,-19.053771),(-68.943151,-19.001165),(-68.961703,-18.969435),(-68.973976,-18.956206),(-68.989609,-18.946491),(-68.959843,-18.907837),(-68.951781,-18.889337),(-68.951445,-18.867323),(-68.954649,-18.856987),(-69.01015,-18.745263),(-69.010537,-18.744539),(-69.010667,-18.743713),(-69.010537,-18.742989),(-69.003174,-18.729553),(-69.001107,-18.716221),(-69.003484,-18.702682),(-69.01015,-18.689762),(-69.033844,-18.649455),(-69.042835,-18.600466),(-69.041647,-18.548893),(-69.034903,-18.50104),(-69.034386,-18.478303),(-69.041698,-18.457735),(-69.07782,-18.398824),(-69.081877,-18.382494),(-69.079293,-18.328027),(-69.081024,-18.319139),(-69.095752,-18.276764),(-69.096837,-18.267979),(-69.09694,-18.257541),(-69.100351,-18.234286),(-69.110661,-18.217957),(-69.141201,-18.186847),(-69.155438,-18.140235),(-69.125931,-18.11109),(-69.089215,-18.083081),(-69.081722,-18.039983),(-69.082284,-18.039078),(-69.092057,-18.023343),(-69.104175,-18.023757),(-69.119885,-18.029854),(-69.14084,-18.030785),(-69.157712,-18.02572),(-69.173887,-18.018899),(-69.204169,-18.001122),(-69.227785,-17.995541),(-69.234813,-17.992957),(-69.241893,-17.986756),(-69.262047,-17.964019),(-69.277601,-17.967533),(-69.290029,-17.976628),(-69.302406,-17.976214),(-69.317857,-17.951616),(-69.326875,-17.915133),(-69.32765,-17.841546),(-69.334239,-17.805785),(-69.359405,-17.75938),(-69.394778,-17.721139),(-69.47578,-17.65241),(-69.497123,-17.621404),(-69.506114,-17.585127),(-69.510036,-17.506588),(-69.510094,-17.505442),(-69.508337,-17.434025),(-69.511385,-17.398678),(-69.522599,-17.36912),(-69.537094,-17.351136),(-69.556938,-17.331499),(-69.597272,-17.300493),(-69.613007,-17.295119),(-69.664416,-17.288562),(-69.666492,-17.288298),(-69.649413,-17.262769),(-69.633833,-17.207269),(-69.6228,-17.185565),(-69.618356,-17.184221),(-69.601147,-17.181534),(-69.59505,-17.179674),(-69.540996,-17.132235),(-69.510094,-17.112081),(-69.482808,-17.101746),(-69.454154,-17.096578),(-69.427437,-17.086863),(-69.406017,-17.062988),(-69.401263,-17.047175),(-69.406017,-17.040871),(-69.413304,-17.03777),(-69.416559,-17.031983),(-69.413588,-17.022267),(-69.407154,-17.015653),(-69.364314,-16.991158),(-69.352455,-16.977722),(-69.325945,-16.922222),(-69.224452,-16.818249),(-69.210654,-16.797061),(-69.191663,-16.743008),(-69.182362,-16.728745),(-69.166368,-16.719133),(-69.130582,-16.715826),(-69.112056,-16.711485),(-69.053816,-16.68234),(-69.037073,-16.670247),(-69.020511,-16.649474),(-69.008341,-16.634177),(-69.035135,-16.598004),(-69.040122,-16.581364),(-69.037642,-16.550255),(-69.038882,-16.49186),(-69.036815,-16.4715),(-69.027875,-16.454137),(-69.001727,-16.422821),(-68.96837,-16.406388),(-68.855844,-16.363186),(-68.833494,-16.328976),(-68.843648,-16.302001),(-68.86667,-16.286912),(-68.91876,-16.266655),(-68.959507,-16.22335),(-68.982038,-16.210017),(-69.01015,-16.219112),(-69.042732,-16.209294),(-69.053558,-16.208364),(-69.064514,-16.212084),(-69.083892,-16.226554),(-69.095183,-16.231205),(-69.120815,-16.231205),(-69.144689,-16.223453),(-69.166264,-16.210431),(-69.184868,-16.195031),(-69.219646,-16.15276),(-69.242125,-16.098189),(-69.284578,-15.995043),(-69.327133,-15.892),(-69.369611,-15.788958),(-69.412218,-15.685811),(-69.424337,-15.656253),(-69.430021,-15.62628),(-69.421184,-15.596411),(-69.356408,-15.50143),(-69.351111,-15.481896),(-69.348527,-15.461846),(-69.343411,-15.442209),(-69.330363,-15.424225),(-69.32026,-15.418851),(-69.292923,-15.409859),(-69.285921,-15.405415),(-69.285404,-15.396733),(-69.296954,-15.366968),(-69.291166,-15.350845),(-69.275198,-15.330278),(-69.255897,-15.313328),(-69.222592,-15.302166),(-69.20913,-15.263615),(-69.189726,-15.262375),(-69.166135,-15.263925),(-69.150426,-15.251626),(-69.1481,-15.233333),(-69.171768,-15.210802),(-69.196495,-15.176592),(-69.21267,-15.161296),(-69.272201,-15.118198),(-69.288583,-15.101971),(-69.346098,-15.016498),(-69.362816,-15.001305),(-69.384313,-14.981772),(-69.390669,-14.964408),(-69.386225,-14.945805),(-69.369844,-14.908288),(-69.367751,-14.900536),(-69.371988,-14.818681),(-69.370541,-14.801524),(-69.36142,-14.787985),(-69.339045,-14.774756),(-69.283518,-14.759666),(-69.267628,-14.750675),(-69.25383,-14.721839),(-69.2475,-14.593682),(-69.234658,-14.574251),(-69.214944,-14.572288),(-69.192232,-14.576939),(-69.170399,-14.577559),(-69.164507,-14.566397),(-69.167427,-14.520301),(-69.164275,-14.503041),(-69.158203,-14.498494),(-69.14115,-14.494153),(-69.134018,-14.490742),(-69.094563,-14.446404),(-69.075133,-14.432451),(-69.01015,-14.397105),(-68.990254,-14.379018),(-68.994621,-14.358967),(-69.008315,-14.33902),(-69.01617,-14.32114),(-69.01723,-14.282486),(-69.015705,-14.263159),(-69.01015,-14.245899),(-68.984906,-14.228329),(-68.951316,-14.219958),(-68.883724,-14.211483),(-68.864448,-14.191226),(-68.868892,-14.156189),(-68.893723,-14.089216),(-68.899976,-14.0548),(-68.905764,-14.039297),(-68.916176,-14.024931),(-68.969713,-13.990514),(-68.982968,-13.972221),(-68.98904,-13.940078),(-68.989195,-13.903698),(-68.992993,-13.869798),(-69.01015,-13.844373),(-69.023095,-13.80603),(-69.024283,-13.79249),(-69.020821,-13.776367),(-69.015964,-13.764585),(-69.015964,-13.753113),(-69.027048,-13.73823),(-69.08105,-13.695442),(-69.101591,-13.666607),(-69.087587,-13.643869),(-69.074539,-13.642629),(-69.044024,-13.648003),(-69.032629,-13.645936),(-69.023844,-13.634981),(-69.023612,-13.623302),(-69.025524,-13.610693),(-69.023198,-13.596947),(-69.015395,-13.585475),(-68.994259,-13.563047),(-68.985655,-13.550231),(-68.976819,-13.526253),(-68.970747,-13.501449),(-68.974441,-13.478814),(-68.962788,-13.283581),(-68.977025,-13.204929),(-68.979661,-13.16457),(-68.972142,-13.046128),(-68.973936,-13.03176),(-68.986224,-12.93337),(-68.986585,-12.890478),(-68.980953,-12.867947),(-68.94943,-12.843866),(-68.938733,-12.819681),(-68.926615,-12.800975),(-68.904627,-12.803972),(-68.891992,-12.775757),(-68.876566,-12.754983),(-68.856206,-12.741443),(-68.77373,-12.719429),(-68.766134,-12.710541),(-68.742957,-12.665789),(-68.752982,-12.65473),(-68.785332,-12.645945),(-68.794608,-12.63654),(-68.793419,-12.62021),(-68.783885,-12.605741),(-68.770113,-12.594372),(-68.74027,-12.578559),(-68.726059,-12.56595),(-68.714535,-12.551171),(-68.706267,-12.535978),(-68.70022,-12.516031),(-68.694381,-12.506832),(-68.684252,-12.502492),(-68.689463,-12.493418),(-68.714225,-12.450298),(-68.81856,-12.269224),(-68.922868,-12.08815),(-69.027281,-11.907076),(-69.13159,-11.726001),(-69.23595,-11.544927),(-69.340336,-11.36375),(-69.444568,-11.182675),(-69.548954,-11.001705),(-69.552494,-10.995607),(-69.556111,-10.989406),(-69.559729,-10.983101),(-69.563269,-10.9769),(-69.566834,-10.970699),(-69.570503,-10.964601),(-69.574017,-10.958503),(-69.577635,-10.952302),(-69.502885,-10.955299),(-69.463378,-10.951268),(-69.39594,-10.935042),(-69.363074,-10.940106),(-69.329588,-10.949408),(-69.293518,-10.954989),(-69.236803,-10.952922),(-69.158513,-10.96212),(-69.121487,-10.970595),(-69.086889,-10.967185),(-69.071309,-10.969975),(-68.996998,-11.001705),(-68.992502,-11.002428),(-68.983382,-11.002428),(-68.978937,-11.001705),(-68.955425,-11.000464),(-68.908141,-11.013383),(-68.884034,-11.016381),(-68.858919,-11.01049),(-68.831685,-11.000361),(-68.804659,-10.994677),(-68.774557,-11.003462),(-68.757633,-11.011937),(-68.773989,-11.035088),(-68.785642,-11.059272),(-68.791171,-11.08511),(-68.787011,-11.126038),(-68.783988,-11.13534),(-68.775926,-11.140611),(-68.758331,-11.141128),(-68.61573,-11.112499),(-68.588031,-11.09865),(-68.536096,-11.061546),(-68.493308,-11.047387),(-68.476565,-11.044803),(-68.459667,-11.044803),(-68.442407,-11.047697),(-68.428196,-11.043666),(-68.396415,-11.014417),(-68.378276,-11.005012),(-68.331147,-11.000464),(-68.313836,-10.993436),(-68.29332,-10.978967),(-68.274355,-10.960053),(-68.257509,-10.938763),(-68.243763,-10.916955),(-68.217563,-10.858044),(-68.203558,-10.838924),(-68.147283,-10.779599),(-68.112091,-10.714074),(-68.09788,-10.697744),(-68.074574,-10.681724),(-68.043775,-10.666945),(-68.027135,-10.661777),(-68.010237,-10.65971),(-67.981815,-10.664361),(-67.862649,-10.65878),(-67.847766,-10.66095),(-67.833348,-10.664981),(-67.81428,-10.675936),(-67.774954,-10.706529),(-67.755782,-10.714177),(-67.721727,-10.705495),(-67.705139,-10.67666),(-67.696044,-10.640486),(-67.684675,-10.610514),(-67.673823,-10.599352),(-67.643489,-10.574547),(-67.630828,-10.559561),(-67.61114,-10.528762),(-67.599978,-10.515429),(-67.58463,-10.501787),(-67.565096,-10.495896),(-67.529336,-10.478532),(-67.488201,-10.464786),(-67.468048,-10.452281),(-67.450529,-10.435951),(-67.436628,-10.417451),(-67.432236,-10.406082),(-67.431874,-10.397917),(-67.42929,-10.390372),(-67.418335,-10.381484),(-67.409188,-10.378797),(-67.365005,-10.378073),(-67.353481,-10.376213),(-67.342836,-10.372492),(-67.335291,-10.366291),(-67.333689,-10.356679),(-67.339425,-10.334872),(-67.337823,-10.326087),(-67.323509,-10.318852),(-67.300668,-10.315132),(-67.259172,-10.313685),(-67.241033,-10.316372),(-67.201759,-10.32681),(-67.184706,-10.326707),(-67.172303,-10.319576),(-67.164862,-10.308517),(-67.159074,-10.297148),(-67.151736,-10.288983),(-67.141763,-10.285573),(-67.12073,-10.284539),(-67.110033,-10.282575),(-67.0643,-10.256944),(-66.902656,-10.09313),(-66.770623,-9.992567),(-66.751606,-9.982645),(-66.670009,-9.951536),(-66.660087,-9.945335),(-66.654816,-9.93717),(-66.65249,-9.926111),(-66.648925,-9.915879),(-66.648904,-9.915854),(-66.64262,-9.907921),(-66.631871,-9.904304),(-66.513842,-9.883943),(-66.452864,-9.888698),(-66.43271,-9.886114),(-66.41328,-9.879189),(-66.371732,-9.855935),(-66.355273,-9.84963),(-66.257321,-9.834851),(-66.238975,-9.828133),(-66.20797,-9.808082),(-66.190658,-9.800848),(-66.135467,-9.795163),(-66.131695,-9.797024),(-66.127923,-9.801158),(-66.123582,-9.805085),(-66.118182,-9.806015),(-66.113117,-9.802398),(-66.106064,-9.790409),(-66.101981,-9.787412),(-66.086995,-9.784518),(-66.081724,-9.785655),(-66.069322,-9.790099),(-66.042786,-9.804878),(-66.029479,-9.808289),(-66.010385,-9.804052),(-65.965659,-9.776766),(-65.946952,-9.771392),(-65.92933,-9.770255),(-65.915274,-9.771702),(-65.874424,-9.781314),(-65.862461,-9.781831),(-65.85042,-9.77532),(-65.834065,-9.758266),(-65.831662,-9.765294),(-65.827398,-9.773976),(-65.822127,-9.781831),(-65.816521,-9.785965),(-65.806418,-9.784415),(-65.803705,-9.775526),(-65.803886,-9.764778),(-65.802645,-9.757336),(-65.794506,-9.737699),(-65.788564,-9.733048),(-65.782207,-9.746277),(-65.779959,-9.755993),(-65.77735,-9.763434),(-65.772441,-9.768808),(-65.76319,-9.772322),(-65.725493,-9.756819),(-65.709964,-9.755993),(-65.713736,-9.77873),(-65.717302,-9.791236),(-65.713116,-9.794233),(-65.705054,-9.793303),(-65.69707,-9.794336),(-65.694357,-9.793716),(-65.683609,-9.789892),(-65.678958,-9.789376),(-65.672628,-9.792579),(-65.663713,-9.802915),(-65.658494,-9.807359),(-65.628883,-9.826893),(-65.614388,-9.833921),(-65.596121,-9.837951),(-65.583977,-9.837228),(-65.579377,-9.832784),(-65.577052,-9.826169),(-65.571884,-9.818831),(-65.570618,-9.815834),(-65.569921,-9.806015),(-65.567983,-9.802088),(-65.562815,-9.799194),(-65.5508,-9.797024),(-65.546821,-9.795163),(-65.535323,-9.782451),(-65.525427,-9.767878),(-65.521396,-9.758576),(-65.517133,-9.743177),(-65.510777,-9.734082),(-65.465224,-9.696461),(-65.45184,-9.681372),(-65.442487,-9.679821),(-65.416235,-9.680132),(-65.397993,-9.686746),(-65.370501,-9.710621),(-65.356058,-9.741626),(-65.353732,-9.745037),(-65.340064,-9.789686),(-65.330323,-9.800744),(-65.316603,-9.812527),(-65.304381,-9.825652),(-65.29911,-9.841259),(-65.30358,-9.864513),(-65.325233,-9.905131),(-65.333294,-9.926835),(-65.336834,-9.967246),(-65.332932,-10.00838),(-65.324148,-10.046414),(-65.30557,-10.098401),(-65.301462,-10.119691),(-65.29911,-10.162996),(-65.296811,-10.173331),(-65.286837,-10.196482),(-65.284822,-10.206818),(-65.288077,-10.216223),(-65.305363,-10.242164),(-65.315621,-10.291257),(-65.327817,-10.314408),(-65.365024,-10.332185),(-65.379235,-10.351408),(-65.39019,-10.374043),(-65.394646,-10.39207),(-65.394738,-10.392439),(-65.393084,-10.400191),(-65.39143,-10.403498),(-65.38727,-10.406702),(-65.410938,-10.44918),(-65.425382,-10.463133),(-65.44998,-10.468094),(-65.44998,-10.474915),(-65.437913,-10.492588),(-65.435691,-10.499203),(-65.436725,-10.504061),(-65.441479,-10.513362),(-65.442487,-10.519357),(-65.441634,-10.527832),(-65.439308,-10.538167),(-65.435898,-10.546952),(-65.431996,-10.550673),(-65.429232,-10.561421),(-65.435639,-10.610824),(-65.435691,-10.62581),(-65.418612,-10.644517),(-65.400784,-10.655473),(-65.386753,-10.669529),(-65.381069,-10.698054),(-65.382258,-10.715624),(-65.389777,-10.753761),(-65.404737,-10.799236),(-65.399698,-10.812259),(-65.360295,-10.822594),(-65.342183,-10.834893),(-65.327119,-10.850499),(-65.31898,-10.865382),(-65.312779,-10.950958),(-65.310117,-10.956539),(-65.30451,-10.962017),(-65.299601,-10.969665),(-65.29911,-10.982068),(-65.302986,-10.990749),(-65.309652,-10.99664),(-65.317843,-11.002015),(-65.326447,-11.009353),(-65.341976,-11.032814),(-65.342699,-11.050591),(-65.337041,-11.067747),(-65.333294,-11.088521),(-65.341046,-11.106608),(-65.379674,-11.136373),(-65.394738,-11.153427),(-65.398148,-11.178025),(-65.385203,-11.193631),(-65.368331,-11.205413),(-65.360011,-11.218849),(-65.366057,-11.234042),(-65.378614,-11.242),(-65.388614,-11.253162),(-65.38727,-11.277553),(-65.37448,-11.295847),(-65.353913,-11.310626),(-65.334819,-11.321168),(-65.326447,-11.327783),(-65.330064,-11.340495),(-65.347505,-11.360752),(-65.353732,-11.372431),(-65.354766,-11.382353),(-65.353009,-11.390621),(-65.319548,-11.476404),(-65.29247,-11.504723)] +Barbados [(-59.42691,13.160386),(-59.430043,13.125922),(-59.455719,13.100043),(-59.475779,13.086493),(-59.487538,13.078559),(-59.50886,13.057318),(-59.516713,13.056383),(-59.523101,13.056464),(-59.529286,13.055406),(-59.536122,13.051174),(-59.545969,13.067694),(-59.564443,13.07803),(-59.586903,13.083197),(-59.60025,13.084133),(-59.608144,13.084662),(-59.621571,13.09398),(-59.633127,13.115912),(-59.646067,13.160386),(-59.654205,13.295111),(-59.652211,13.310614),(-59.645131,13.323717),(-59.63093,13.337958),(-59.612864,13.34455),(-59.594472,13.334784),(-59.580719,13.314521),(-59.575917,13.304389),(-59.568959,13.289659),(-59.549794,13.249172),(-59.53661,13.231391),(-59.518056,13.217515),(-59.491811,13.197943),(-59.476674,13.189399),(-59.457102,13.183173),(-59.438629,13.174872),(-59.42691,13.160386)] +Bhutan [(90.261804,28.335354),(90.261804,28.299141),(90.29147,28.261351),(90.329504,28.255796),(90.353984,28.299141),(90.386905,28.299141),(90.416535,28.28268),(90.45604,28.28268),(90.505422,28.249758),(90.54822,28.246466),(90.587726,28.233298),(90.597602,28.200376),(90.567973,28.1905),(90.54822,28.157579),(90.515299,28.154287),(90.485669,28.124657),(90.475232,28.072344),(90.492595,28.074412),(90.510061,28.074101),(90.575277,28.065833),(90.597395,28.070587),(90.621889,28.072965),(90.651035,28.086995),(90.667468,28.090328),(90.683901,28.087072),(90.726896,28.061699),(90.755278,28.055173),(90.788701,28.047488),(90.850919,28.044026),(90.90952,28.032657),(90.960577,27.994726),(90.975149,27.982092),(91.008842,27.96677),(91.051941,27.96279),(91.092558,27.971679),(91.11943,27.994726),(91.11943,27.994778),(91.119637,27.994881),(91.133796,28.014105),(91.175861,28.057487),(91.194671,28.070587),(91.220612,28.074877),(91.246141,28.071466),(91.269705,28.072861),(91.290169,28.091413),(91.309599,28.056402),(91.341432,28.030564),(91.418843,27.994726),(91.436483,27.989375),(91.446438,27.986355),(91.460908,27.984365),(91.497495,27.984081),(91.537492,27.969353),(91.578627,27.964702),(91.600641,27.959276),(91.621725,27.950724),(91.637951,27.939743),(91.652834,27.916747),(91.648907,27.897213),(91.637124,27.87724),(91.628339,27.852694),(91.627306,27.829827),(91.632887,27.759444),(91.626686,27.716423),(91.579867,27.657977),(91.573252,27.619711),(91.59506,27.546382),(91.604465,27.532145),(91.632887,27.511655),(91.641672,27.494964),(91.657382,27.479151),(91.680223,27.472846),(91.704924,27.468764),(91.727041,27.460005),(91.727092,27.459942),(91.743681,27.439153),(91.744818,27.423754),(91.750089,27.416157),(91.779648,27.418844),(91.857369,27.443081),(91.884345,27.447163),(91.920931,27.445148),(91.933747,27.449127),(91.963306,27.468919),(91.975088,27.472433),(91.997309,27.448662),(92.084849,27.304226),(92.088776,27.29234),(92.081852,27.275132),(92.050639,27.251413),(92.03679,27.236478),(92.033173,27.227435),(92.029969,27.20847),(92.027075,27.19984),(92.021494,27.19183),(92.008471,27.179479),(92.005788,27.176112),(92.002994,27.172606),(91.995862,27.158033),(91.990798,27.140773),(91.987904,27.122273),(91.987697,27.104031),(91.999893,27.071527),(92.049709,27.026827),(92.066969,26.994943),(92.066969,26.994839),(92.083919,26.937323),(92.080371,26.921571),(92.072757,26.887766),(92.03586,26.854848),(91.975088,26.846631),(91.957415,26.854279),(91.925686,26.878671),(91.908219,26.885182),(91.89313,26.881358),(91.895507,26.868284),(91.895507,26.853504),(91.874009,26.844306),(91.885895,26.83087),(91.886928,26.814437),(91.87866,26.803016),(91.863054,26.804773),(91.852408,26.818519),(91.849515,26.834746),(91.843934,26.84937),(91.825123,26.858465),(91.795254,26.853504),(91.731589,26.816194),(91.702133,26.803481),(91.653764,26.797952),(91.637951,26.798985),(91.593716,26.810768),(91.57563,26.810148),(91.539353,26.79883),(91.520646,26.79759),(91.507417,26.808029),(91.484472,26.852729),(91.475067,26.865545),(91.460598,26.869369),(91.420187,26.871539),(91.406338,26.869524),(91.388457,26.858465),(91.378122,26.842807),(91.370061,26.824669),(91.358382,26.806169),(91.345566,26.794696),(91.330063,26.785498),(91.313217,26.778677),(91.296577,26.774594),(91.27601,26.774077),(91.261954,26.779038),(91.232498,26.795161),(91.198185,26.802344),(91.127078,26.800898),(91.091938,26.804773),(91.062069,26.804567),(91.012781,26.784348),(91.007395,26.782139),(90.944493,26.77887),(90.717077,26.767049),(90.587783,26.780072),(90.475232,26.83242),(90.382627,26.891796),(90.348831,26.896654),(90.328574,26.890659),(90.314208,26.880376),(90.300875,26.868284),(90.284442,26.85707),(90.265994,26.851592),(90.229148,26.852884),(90.210907,26.851489),(90.17742,26.832058),(90.152202,26.771752),(90.127191,26.751392),(90.089157,26.741728),(89.975262,26.731858),(89.912113,26.716717),(89.890409,26.714856),(89.880281,26.716252),(89.85992,26.721884),(89.858701,26.722057),(89.850774,26.723176),(89.847001,26.72483),(89.839043,26.731238),(89.834961,26.731548),(89.829741,26.72762),(89.827674,26.722453),(89.827261,26.717905),(89.826744,26.715683),(89.826796,26.713513),(89.825245,26.707673),(89.822093,26.701007),(89.817287,26.696201),(89.81367,26.696149),(89.804058,26.699767),(89.800079,26.70049),(89.760288,26.70018),(89.75467,26.700989),(89.738739,26.703281),(89.685151,26.724571),(89.66324,26.725502),(89.628307,26.712531),(89.609806,26.712221),(89.597507,26.720954),(89.613475,26.748549),(89.611357,26.765964),(89.586087,26.784051),(89.546503,26.797539),(89.50542,26.803688),(89.442891,26.797022),(89.407338,26.813403),(89.368895,26.837359),(89.341657,26.854331),(89.300161,26.844409),(89.286364,26.844978),(89.273599,26.843892),(89.262954,26.836348),(89.252619,26.826787),(89.240837,26.819759),(89.212828,26.813041),(89.184613,26.810561),(89.128079,26.813455),(89.096504,26.821413),(89.082448,26.836037),(89.074335,26.856398),(89.060693,26.881461),(89.04488,26.897119),(89.021574,26.912674),(88.996614,26.922751),(88.975323,26.921717),(88.965815,26.915464),(88.954549,26.912622),(88.942767,26.913707),(88.932328,26.919237),(88.9253,26.929365),(88.925404,26.939029),(88.926954,26.949416),(88.924474,26.961663),(88.906645,26.981093),(88.88644,26.978975),(88.867113,26.964247),(88.851713,26.945437),(88.845615,26.994839),(88.845615,26.994943),(88.845615,27.049565),(88.840809,27.075248),(88.827632,27.097882),(88.805256,27.113023),(88.742572,27.142737),(88.733082,27.148972),(88.730067,27.150954),(88.738438,27.179789),(88.754355,27.212604),(88.775852,27.240871),(88.801174,27.256425),(88.861428,27.270378),(88.876116,27.280486),(88.884631,27.286346),(88.892331,27.315543),(88.91516,27.330869),(88.950808,27.328229),(88.971933,27.312385),(88.997019,27.330869),(88.981175,27.354635),(88.964011,27.392924),(88.95106,27.433144),(88.957549,27.456939),(88.966202,27.47803),(88.982966,27.492632),(88.980046,27.49798),(88.97521,27.506839),(88.972955,27.517659),(88.978366,27.530283),(88.992793,27.537497),(89.004966,27.54967),(89.002712,27.560491),(89.016238,27.57582),(89.03337,27.578976),(89.039682,27.594305),(89.050954,27.608282),(89.064029,27.615946),(89.091531,27.625865),(89.102351,27.623611),(89.105865,27.622952),(89.109565,27.622258),(89.119484,27.618201),(89.124655,27.614852),(89.127359,27.62729),(89.156021,27.665686),(89.198203,27.730582),(89.21659,27.771142),(89.22492,27.807813),(89.258717,27.827553),(89.299489,27.844245),(89.336025,27.869075),(89.370131,27.909357),(89.417364,27.988654),(89.420981,27.994726),(89.421187,27.994726),(89.444545,28.031184),(89.458808,28.047798),(89.475344,28.061337),(89.495343,28.068004),(89.515032,28.081905),(89.561489,28.13464),(89.579421,28.14464),(89.597817,28.149807),(89.71781,28.169083),(89.755844,28.184379),(89.773621,28.212775),(89.780855,28.228769),(89.796048,28.24019),(89.829741,28.259155),(89.83863,28.267914),(89.854029,28.287267),(89.862918,28.295793),(89.873615,28.300599),(89.881314,28.297525),(89.91613,28.315601),(89.952524,28.308247),(89.971593,28.31804),(89.990455,28.320624),(90.07086,28.34523),(90.123535,28.335354),(90.153164,28.322185),(90.176209,28.325478),(90.199254,28.348523),(90.225591,28.358399),(90.261804,28.335354)] +Botswana [(25.259781,-17.794107),(25.21937,-17.879786),(25.21937,-17.908001),(25.226088,-17.931876),(25.255026,-18.001122),(25.296368,-18.068612),(25.323446,-18.09662),(25.357449,-18.115844),(25.387525,-18.138995),(25.408816,-18.175995),(25.440855,-18.2532),(25.473204,-18.303429),(25.481163,-18.323377),(25.490516,-18.365545),(25.49558,-18.378877),(25.508499,-18.399134),(25.574439,-18.465693),(25.608442,-18.487708),(25.622084,-18.501143),(25.669523,-18.566049),(25.698255,-18.590234),(25.736909,-18.608734),(25.761921,-18.630335),(25.773393,-18.665578),(25.779491,-18.738752),(25.815251,-18.813993),(25.940721,-18.921273),(25.967449,-18.999925),(25.9678,-19.000958),(25.964389,-19.021629),(25.948059,-19.058732),(25.944855,-19.079196),(25.948576,-19.103277),(25.956534,-19.122088),(25.981132,-19.161775),(26.011414,-19.199809),(26.034359,-19.243734),(26.13027,-19.501082),(26.155488,-19.537153),(26.194452,-19.5602),(26.239101,-19.571466),(26.292638,-19.572499),(26.30349,-19.577254),(26.313205,-19.584178),(26.32106,-19.592033),(26.330981,-19.604952),(26.333462,-19.613014),(26.326331,-19.633891),(26.319096,-19.646293),(26.312481,-19.649601),(26.312171,-19.651358),(26.324367,-19.659109),(26.332325,-19.662416),(26.362711,-19.667584),(26.385242,-19.679056),(26.412837,-19.71957),(26.431854,-19.73652),(26.450251,-19.743342),(26.489731,-19.75192),(26.508852,-19.759258),(26.549263,-19.784063),(26.566316,-19.800806),(26.574791,-19.819513),(26.581922,-19.842147),(26.595565,-19.855583),(26.614065,-19.863438),(26.659437,-19.875737),(26.673803,-19.883385),(26.67717,-19.886815),(26.684758,-19.894547),(26.698608,-19.91253),(26.713904,-19.927413),(26.730957,-19.935888),(26.750801,-19.939609),(26.774469,-19.939815),(26.811882,-19.94643),(26.925054,-20.000897),(26.961072,-20.007201),(26.9943,-20.006788),(27.02665,-20.010095),(27.060136,-20.027562),(27.069231,-20.03738),(27.086491,-20.060532),(27.097343,-20.068903),(27.109642,-20.073244),(27.119771,-20.073864),(27.129692,-20.072934),(27.141888,-20.073347),(27.16292,-20.076551),(27.183746,-20.082339),(27.201781,-20.092984),(27.214907,-20.110451),(27.266015,-20.234164),(27.283998,-20.35147),(27.268392,-20.49575),(27.306012,-20.477354),(27.340739,-20.473013),(27.45391,-20.473323),(27.534112,-20.483038),(27.590853,-20.473323),(27.625786,-20.488619),(27.66599,-20.489136),(27.683767,-20.49606),(27.698133,-20.509083),(27.705575,-20.526653),(27.702629,-20.566134),(27.690382,-20.60148),(27.682475,-20.637344),(27.707332,-20.716719),(27.709605,-20.756716),(27.694412,-20.837745),(27.689038,-20.849011),(27.681803,-20.857589),(27.676016,-20.866684),(27.674775,-20.879913),(27.675085,-20.891282),(27.672605,-20.913709),(27.672657,-20.923528),(27.680356,-20.979649),(27.678961,-21.000733),(27.666817,-21.053753),(27.666611,-21.071219),(27.674775,-21.090133),(27.709192,-21.134471),(27.724385,-21.149664),(27.793838,-21.197413),(27.823604,-21.231726),(27.849132,-21.269657),(27.884995,-21.310171),(27.8944,-21.32433),(27.896674,-21.332392),(27.896157,-21.347895),(27.897811,-21.35544),(27.904219,-21.364741),(27.920549,-21.381174),(27.950211,-21.438329),(27.953001,-21.448664),(27.949642,-21.456519),(27.941943,-21.468508),(27.939876,-21.478016),(27.943338,-21.479876),(27.950418,-21.482047),(27.954448,-21.487835),(27.949281,-21.500754),(27.953208,-21.510469),(27.958066,-21.511502),(27.963698,-21.510469),(27.970571,-21.514396),(27.975739,-21.522561),(27.984731,-21.542922),(27.990415,-21.551913),(28.002559,-21.564212),(28.016563,-21.572894),(28.032893,-21.577855),(28.090771,-21.581266),(28.165702,-21.595218),(28.284867,-21.596872),(28.321919,-21.603486),(28.361762,-21.616302),(28.443101,-21.655783),(28.464598,-21.660331),(28.481393,-21.657437),(28.497309,-21.651546),(28.532501,-21.643071),(28.542939,-21.638316),(28.553998,-21.636559),(28.585934,-21.644414),(28.6157,-21.647101),(28.629704,-21.651339),(28.66841,-21.679968),(28.714195,-21.693507),(28.860853,-21.757379),(28.891032,-21.764924),(28.951907,-21.768334),(28.980846,-21.774845),(28.998726,-21.786008),(29.038723,-21.797893),(29.05526,-21.809985),(29.057637,-21.829209),(29.045441,-21.852567),(29.028905,-21.876648),(29.017949,-21.898145),(29.013815,-21.940417),(29.021567,-21.982791),(29.040532,-22.020929),(29.070763,-22.051004),(29.10797,-22.069194),(29.144867,-22.075292),(29.239331,-22.072605),(29.244395,-22.075706),(29.254111,-22.087074),(29.259588,-22.096066),(29.26734,-22.115807),(29.273644,-22.125108),(29.350074,-22.186707),(29.331522,-22.192804),(29.302893,-22.190221),(29.248219,-22.179265),(29.222071,-22.182159),(29.202796,-22.194458),(29.186621,-22.207687),(29.168844,-22.213992),(29.047818,-22.220296),(29.038723,-22.223914),(29.017846,-22.250372),(29.018053,-22.255023),(29.002756,-22.263394),(28.983739,-22.281998),(28.960227,-22.310213),(28.967926,-22.38039),(28.963379,-22.392172),(28.953457,-22.40075),(28.930409,-22.440851),(28.912529,-22.453667),(28.86938,-22.448913),(28.84659,-22.449946),(28.836772,-22.463176),(28.831191,-22.482296),(28.816825,-22.492941),(28.796774,-22.498212),(28.77476,-22.501416),(28.731352,-22.513922),(28.659005,-22.551956),(28.623865,-22.562808),(28.5972,-22.562704),(28.577563,-22.560017),(28.559838,-22.563014),(28.538495,-22.580274),(28.522476,-22.584822),(28.459379,-22.569629),(28.376851,-22.575003),(28.338559,-22.584615),(28.301817,-22.603839),(28.286831,-22.615828),(28.250554,-22.655619),(28.240684,-22.66089),(28.216344,-22.663163),(28.205492,-22.665954),(28.196191,-22.671638),(28.164668,-22.708639),(28.159707,-22.718147),(28.157743,-22.731066),(28.159707,-22.743262),(28.162704,-22.751013),(28.162498,-22.758971),(28.154953,-22.771994),(28.145858,-22.779745),(28.135626,-22.783363),(28.125601,-22.785533),(28.117436,-22.789461),(28.113198,-22.795352),(28.107721,-22.810648),(28.10369,-22.816746),(28.088807,-22.822637),(28.065759,-22.828838),(28.047156,-22.8369),(28.045399,-22.847752),(28.053977,-22.869146),(28.04974,-22.891366),(28.037441,-22.911417),(28.021834,-22.92661),(28.008295,-22.934775),(27.989382,-22.943766),(27.968401,-22.951001),(27.949177,-22.953895),(27.937498,-22.963817),(27.937912,-22.986658),(27.946077,-23.029032),(27.930057,-23.057041),(27.895537,-23.079469),(27.853783,-23.095592),(27.815749,-23.104687),(27.82226,-23.12298),(27.806447,-23.128355),(27.78495,-23.127011),(27.774201,-23.125151),(27.77172,-23.133005),(27.774614,-23.140757),(27.779058,-23.147682),(27.781022,-23.152539),(27.783038,-23.155433),(27.787378,-23.159774),(27.78929,-23.163805),(27.784588,-23.165562),(27.783296,-23.166905),(27.775751,-23.172486),(27.774201,-23.173003),(27.759421,-23.21145),(27.753634,-23.220752),(27.739164,-23.227573),(27.725728,-23.221682),(27.698443,-23.193467),(27.658756,-23.223439),(27.647129,-23.227573),(27.633434,-23.223439),(27.619172,-23.217031),(27.608009,-23.216721),(27.599224,-23.244627),(27.580414,-23.26385),(27.576073,-23.279146),(27.575247,-23.284831),(27.573076,-23.290515),(27.569614,-23.294856),(27.565893,-23.29651),(27.563309,-23.299197),(27.564911,-23.305811),(27.567754,-23.31377),(27.569356,-23.320384),(27.563671,-23.342812),(27.548892,-23.360795),(27.528841,-23.373094),(27.50724,-23.379089),(27.465693,-23.380432),(27.45267,-23.38529),(27.443162,-23.395108),(27.434997,-23.406581),(27.425178,-23.412678),(27.411122,-23.405754),(27.41846,-23.391284),(27.410502,-23.389321),(27.395826,-23.394075),(27.383734,-23.399553),(27.38084,-23.40534),(27.371331,-23.417019),(27.361203,-23.422497),(27.349989,-23.391491),(27.335571,-23.402446),(27.312007,-23.43707),(27.295574,-23.451746),(27.278107,-23.463631),(27.239712,-23.481408),(27.202505,-23.492053),(27.191497,-23.503836),(27.205657,-23.523059),(27.195632,-23.52709),(27.18664,-23.527297),(27.178682,-23.523576),(27.171447,-23.515618),(27.168967,-23.521612),(27.161525,-23.52988),(27.157184,-23.536702),(27.149123,-23.527503),(27.137651,-23.523369),(27.127419,-23.525023),(27.123078,-23.532981),(27.127419,-23.54528),(27.135687,-23.551895),(27.141165,-23.558303),(27.13672,-23.570188),(27.124421,-23.56316),(27.113363,-23.564297),(27.103751,-23.571738),(27.095793,-23.583831),(27.102511,-23.583831),(27.084114,-23.596026),(27.078326,-23.602021),(27.075225,-23.611219),(27.068404,-23.607292),(27.061531,-23.604398),(27.063133,-23.624242),(27.069438,-23.641708),(27.070058,-23.656074),(27.054762,-23.66641),(27.037605,-23.665686),(27.029854,-23.654731),(27.024273,-23.642535),(27.013834,-23.638504),(27.00417,-23.645842),(26.986549,-23.686977),(26.96717,-23.718706),(26.962054,-23.737413),(26.964542,-23.74714),(26.966602,-23.75519),(26.938128,-23.802887),(26.945001,-23.823248),(26.941177,-23.850223),(26.870897,-24.11584),(26.868933,-24.160385),(26.84971,-24.248131),(26.839168,-24.265598),(26.823251,-24.278827),(26.802271,-24.289679),(26.714007,-24.315931),(26.692923,-24.326576),(26.684087,-24.341459),(26.62347,-24.39944),(26.558358,-24.438197),(26.531176,-24.458661),(26.506061,-24.48884),(26.476812,-24.559947),(26.469474,-24.571419),(26.404362,-24.632811),(26.404001,-24.632758),(26.381624,-24.629503),(26.363538,-24.628056),(26.344521,-24.624026),(26.32571,-24.622372),(26.278788,-24.62847),(26.165911,-24.660846),(26.013585,-24.704537),(25.981132,-24.726345),(25.966043,-24.73358),(25.914676,-24.731823),(25.881707,-24.742158),(25.874058,-24.743398),(25.868374,-24.748152),(25.866927,-24.772647),(25.877262,-24.845717),(25.876746,-24.886025),(25.866204,-24.918168),(25.837058,-24.979146),(25.835198,-25.016353),(25.804915,-25.064826),(25.782901,-25.131385),(25.695465,-25.309979),(25.663529,-25.439996),(25.652838,-25.463319),(25.608855,-25.559266),(25.587254,-25.61952),(25.458425,-25.711194),(25.386491,-25.743544),(25.177822,-25.763284),(25.135964,-25.756566),(25.090282,-25.743647),(25.051731,-25.737859),(25.013284,-25.743027),(24.967912,-25.762871),(24.908381,-25.804109),(24.888537,-25.811653),(24.828903,-25.826019),(24.79862,-25.829223),(24.770715,-25.822609),(24.737952,-25.815891),(24.664985,-25.823332),(24.629638,-25.816098),(24.576722,-25.783335),(24.500851,-25.757848),(24.456729,-25.743027),(24.436782,-25.745301),(24.416731,-25.753052),(24.38924,-25.75946),(24.338907,-25.751812),(24.296739,-25.723493),(24.227699,-25.648976),(24.183464,-25.625928),(24.130961,-25.626238),(24.027091,-25.651766),(24.005697,-25.65466),(24.00425,-25.649906),(24.009624,-25.641328),(24.008694,-25.633163),(23.998256,-25.625308),(23.988024,-25.61952),(23.976448,-25.617557),(23.924772,-25.629236),(23.905755,-25.618487),(23.890665,-25.59885),(23.845397,-25.569084),(23.807466,-25.523609),(23.798578,-25.518545),(23.778321,-25.51255),(23.769019,-25.507383),(23.762301,-25.498494),(23.752069,-25.477617),(23.742664,-25.469969),(23.68675,-25.454362),(23.674244,-25.44258),(23.6667,-25.432658),(23.561073,-25.357314),(23.535855,-25.343258),(23.506606,-25.330959),(23.496581,-25.324241),(23.483455,-25.311529),(23.468159,-25.290238),(23.45927,-25.282177),(23.444904,-25.278146),(23.431985,-25.280316),(23.399222,-25.291375),(23.381446,-25.292615),(23.365943,-25.289308),(23.320364,-25.273392),(23.265691,-25.263987),(23.216081,-25.26719),(23.168539,-25.280006),(23.093401,-25.312459),(23.083893,-25.319694),(23.071697,-25.325998),(23.065496,-25.320831),(23.061155,-25.310702),(23.054541,-25.302537),(23.030666,-25.299437),(23.006998,-25.310805),(22.968034,-25.346876),(22.962557,-25.355764),(22.960283,-25.364342),(22.956149,-25.372094),(22.94478,-25.378708),(22.930931,-25.384289),(22.920492,-25.39049),(22.911914,-25.399069),(22.903645,-25.411368),(22.887626,-25.450228),(22.876774,-25.462011),(22.851349,-25.471312),(22.844631,-25.481131),(22.824684,-25.543039),(22.816829,-25.558439),(22.814039,-25.566604),(22.813418,-25.575079),(22.819206,-25.584897),(22.828715,-25.595542),(22.833366,-25.606084),(22.824374,-25.6158),(22.810835,-25.625308),(22.809594,-25.633266),(22.813418,-25.641948),(22.814969,-25.6528),(22.812798,-25.667063),(22.810214,-25.677398),(22.804427,-25.687216),(22.759778,-25.734862),(22.746446,-25.755326),(22.740451,-25.77703),(22.741175,-25.790879),(22.744999,-25.794083),(22.75089,-25.796047),(22.757918,-25.805969),(22.764119,-25.820335),(22.765463,-25.826536),(22.754714,-25.843383),(22.739521,-25.858472),(22.719471,-25.874285),(22.708825,-25.891028),(22.721641,-25.909012),(22.726809,-25.921621),(22.727222,-25.943945),(22.724225,-25.967509),(22.719367,-25.984253),(22.710479,-25.996862),(22.70066,-26.00327),(22.672238,-26.012881),(22.661593,-26.020943),(22.66211,-26.029831),(22.665831,-26.039856),(22.665004,-26.051432),(22.623249,-26.10931),(22.620863,-26.111532),(22.611157,-26.120575),(22.59152,-26.133184),(22.580254,-26.146517),(22.563821,-26.18114),(22.545114,-26.207391),(22.527958,-26.213696),(22.482586,-26.204808),(22.450857,-26.210079),(22.428842,-26.226512),(22.395046,-26.271884),(22.342853,-26.317359),(22.318772,-26.328831),(22.303165,-26.333585),(22.257277,-26.340923),(22.248285,-26.346918),(22.237743,-26.366762),(22.197435,-26.404072),(22.178315,-26.439935),(22.146172,-26.519207),(22.108138,-26.571917),(22.057702,-26.617599),(21.998998,-26.650258),(21.935849,-26.663901),(21.838801,-26.664521),(21.807278,-26.669585),(21.780613,-26.678164),(21.768728,-26.688706),(21.767487,-26.704312),(21.776479,-26.749167),(21.777513,-26.768494),(21.773379,-26.786788),(21.76232,-26.804461),(21.74382,-26.820481),(21.687182,-26.855207),(21.664548,-26.863062),(21.643361,-26.862752),(21.583933,-26.848386),(21.498667,-26.846422),(21.449471,-26.826372),(21.42663,-26.822651),(21.380844,-26.823995),(21.296922,-26.844459),(21.282866,-26.844975),(21.254857,-26.841771),(21.240595,-26.841978),(21.153468,-26.864509),(21.122462,-26.865336),(20.990377,-26.838567),(20.950173,-26.816036),(20.907695,-26.800017),(20.851884,-26.806528),(20.826046,-26.818827),(20.779434,-26.848076),(20.714942,-26.875051),(20.693548,-26.889004),(20.690757,-26.891794),(20.685693,-26.881769),(20.675461,-26.85004),(20.665643,-26.839084),(20.650553,-26.829679),(20.644869,-26.824511),(20.638667,-26.816967),(20.624818,-26.789371),(20.61469,-26.753198),(20.609212,-26.716198),(20.608902,-26.686122),(20.630709,-26.618322),(20.632363,-26.595275),(20.627712,-26.580805),(20.611486,-26.559928),(20.605284,-26.547732),(20.602287,-26.522204),(20.605284,-26.493265),(20.618204,-26.439729),(20.622544,-26.427533),(20.652103,-26.407793),(20.665953,-26.391773),(20.695512,-26.341957),(20.753286,-26.276224),(20.77623,-26.240568),(20.79473,-26.201914),(20.837932,-26.146827),(20.841446,-26.131324),(20.803929,-26.071276),(20.800518,-26.052466),(20.802792,-25.980532),(20.79411,-25.893922),(20.767135,-25.834391),(20.766928,-25.83067),(20.755766,-25.818991),(20.749978,-25.818475),(20.740057,-25.823126),(20.726517,-25.827363),(20.727137,-25.818165),(20.737783,-25.798941),(20.716699,-25.732588),(20.70688,-25.714812),(20.696545,-25.70706),(20.671017,-25.695485),(20.662852,-25.685253),(20.662749,-25.674918),(20.67143,-25.653317),(20.67081,-25.642258),(20.663059,-25.636264),(20.65138,-25.63461),(20.641045,-25.631199),(20.637944,-25.620347),(20.643318,-25.613216),(20.665229,-25.60102),(20.67143,-25.591202),(20.663369,-25.564743),(20.618514,-25.528466),(20.620374,-25.500561),(20.626885,-25.484748),(20.656547,-25.467798),(20.643525,-25.45736),(20.63443,-25.45767),(20.625645,-25.462734),(20.61655,-25.466145),(20.606628,-25.462011),(20.605905,-25.451779),(20.611382,-25.439376),(20.612002,-25.431005),(20.596913,-25.432865),(20.593606,-25.413228),(20.588025,-25.40403),(20.558776,-25.392041),(20.542239,-25.382222),(20.535831,-25.371163),(20.529217,-25.340054),(20.525083,-25.332199),(20.514644,-25.31928),(20.51051,-25.312252),(20.510407,-25.304708),(20.514851,-25.299126),(20.518571,-25.293029),(20.516401,-25.283624),(20.50989,-25.277526),(20.491803,-25.269051),(20.485498,-25.26316),(20.483535,-25.256648),(20.481674,-25.2305),(20.462451,-25.223679),(20.443847,-25.213344),(20.434442,-25.199804),(20.443227,-25.183061),(20.444777,-25.168385),(20.433925,-25.146784),(20.406847,-25.10782),(20.375531,-25.046119),(20.364886,-25.0332),(20.235695,-24.936048),(20.159317,-24.893673),(20.149085,-24.890986),(20.128414,-24.889746),(20.118906,-24.887369),(20.107847,-24.880341),(20.081079,-24.852642),(20.029299,-24.815228),(20.010489,-24.788047),(19.992919,-24.775851),(19.986408,-24.768823),(19.981447,-24.752493),(19.981343,-24.580411),(19.981137,-24.408431),(19.980826,-24.236556),(19.980723,-24.064473),(19.980516,-23.892494),(19.980483,-23.864489),(19.98031,-23.720567),(19.980103,-23.548587),(19.979896,-23.376505),(19.979855,-23.308551),(19.979793,-23.204526),(19.979586,-23.032546),(19.979276,-22.860671),(19.979173,-22.688588),(19.978966,-22.538106),(19.978863,-22.387625),(19.978656,-22.237246),(19.978449,-22.086764),(19.978346,-22.000671),(20.21244,-22.000671),(20.446534,-22.000671),(20.680629,-22.000671),(20.914723,-22.000671),(20.97198,-22.000671),(20.984796,-21.963981),(20.984693,-21.922847),(20.984486,-21.865176),(20.984279,-21.807608),(20.984073,-21.750041),(20.983866,-21.69237),(20.983659,-21.634699),(20.983453,-21.577028),(20.983143,-21.519357),(20.983039,-21.46179),(20.982832,-21.404119),(20.982626,-21.346448),(20.982419,-21.28888),(20.982212,-21.231209),(20.982109,-21.173642),(20.981902,-21.116074),(20.981592,-21.058403),(20.981489,-21.000733),(20.981486,-20.999958),(20.981075,-20.875779),(20.980662,-20.750722),(20.980249,-20.625768),(20.979835,-20.500711),(20.979422,-20.375758),(20.97936,-20.351028),(20.979112,-20.250804),(20.978698,-20.125851),(20.978285,-20.000794),(20.977975,-19.87584),(20.977561,-19.750783),(20.977045,-19.625829),(20.976735,-19.500876),(20.976321,-19.375819),(20.975804,-19.250865),(20.975609,-19.172106),(20.975494,-19.125912),(20.975081,-19.000958),(20.975081,-18.96003),(20.975081,-18.919206),(20.975081,-18.878381),(20.975081,-18.83766),(20.975081,-18.796836),(20.975081,-18.756012),(20.975081,-18.715187),(20.975081,-18.674363),(20.975081,-18.633539),(20.975081,-18.592714),(20.975081,-18.551993),(20.975081,-18.511065),(20.975081,-18.470138),(20.975081,-18.429313),(20.975081,-18.388592),(20.975081,-18.347768),(20.975081,-18.319346),(20.993436,-18.318612),(21.024174,-18.317382),(21.080604,-18.315212),(21.136932,-18.312938),(21.193466,-18.310768),(21.249896,-18.308494),(21.296818,-18.306633),(21.343947,-18.304773),(21.39087,-18.302913),(21.437895,-18.301052),(21.456809,-18.300226),(21.475722,-18.299502),(21.492879,-18.296401),(21.509829,-18.293198),(21.526985,-18.289994),(21.544039,-18.28679),(21.628478,-18.271183),(21.71271,-18.255577),(21.79715,-18.239867),(21.881486,-18.224158),(21.966028,-18.208552),(22.050261,-18.192842),(22.134597,-18.177236),(22.219036,-18.161526),(22.303475,-18.145816),(22.387811,-18.13021),(22.472147,-18.1145),(22.556483,-18.098791),(22.640922,-18.083185),(22.725258,-18.067475),(22.809698,-18.051869),(22.894034,-18.036159),(22.981367,-18.020036),(23.099602,-18.010217),(23.185799,-18.003086),(23.254942,-17.997402),(23.292769,-17.998952),(23.305688,-18.005463),(23.311476,-18.009804),(23.312819,-18.016522),(23.312716,-18.029958),(23.31592,-18.037089),(23.323258,-18.039156),(23.330596,-18.039983),(23.3338,-18.04329),(23.33256,-18.067061),(23.3338,-18.073986),(23.336591,-18.07936),(23.348063,-18.09445),(23.359535,-18.119151),(23.389714,-18.153258),(23.395812,-18.163386),(23.396329,-18.169381),(23.394468,-18.186227),(23.395812,-18.191395),(23.40129,-18.194289),(23.410281,-18.197389),(23.418859,-18.198423),(23.429401,-18.188501),(23.444284,-18.200697),(23.458754,-18.217647),(23.464128,-18.225501),(23.477771,-18.230876),(23.49131,-18.23315),(23.501542,-18.23749),(23.505676,-18.249376),(23.511257,-18.260331),(23.521696,-18.268186),(23.527277,-18.277695),(23.518802,-18.293714),(23.551461,-18.327511),(23.560763,-18.348491),(23.546604,-18.369472),(23.555492,-18.383115),(23.571305,-18.426006),(23.57916,-18.467864),(23.592182,-18.478199),(23.609856,-18.477682),(23.645409,-18.466004),(23.649853,-18.46342),(23.656468,-18.458252),(23.680136,-18.431484),(23.700909,-18.42797),(23.715792,-18.419081),(23.809843,-18.321723),(23.825553,-18.317072),(23.837232,-18.305807),(23.855215,-18.280072),(23.867514,-18.269426),(23.89697,-18.250203),(23.912886,-18.235733),(23.915987,-18.201213),(23.950817,-18.177649),(23.956604,-18.176615),(23.961979,-18.177856),(23.966733,-18.180646),(23.971177,-18.18385),(23.974898,-18.177029),(23.979652,-18.171861),(23.991744,-18.163386),(24.02027,-18.151604),(24.028125,-18.145816),(24.05696,-18.119048),(24.065125,-18.115224),(24.101609,-18.108816),(24.135095,-18.085458),(24.183051,-18.029441),(24.218294,-18.012595),(24.238241,-18.009907),(24.259222,-18.012595),(24.270074,-18.015798),(24.287437,-18.02448),(24.296429,-18.026237),(24.305834,-18.019416),(24.334256,-17.971563),(24.350586,-17.95606),(24.365262,-17.950789),(24.399471,-17.95234),(24.421589,-17.956474),(24.433888,-17.967223),(24.451045,-17.998952),(24.458486,-18.005773),(24.465101,-18.008667),(24.469751,-18.014455),(24.471508,-18.029958),(24.505615,-18.060344),(24.518431,-18.057346),(24.564423,-18.052799),(24.574551,-18.050422),(24.577755,-18.044427),(24.591811,-18.028407),(24.595015,-18.022826),(24.598632,-18.020759),(24.649172,-17.962778),(24.664055,-17.949859),(24.698471,-17.928879),(24.725343,-17.895909),(24.730717,-17.891878),(24.738159,-17.887641),(24.74684,-17.88423),(24.756039,-17.882783),(24.764514,-17.879683),(24.770715,-17.865523),(24.776916,-17.862319),(24.79738,-17.858082),(24.820841,-17.839272),(24.838101,-17.835034),(24.857428,-17.833587),(24.931119,-17.81054),(24.93763,-17.807129),(24.945175,-17.799998),(24.948585,-17.79328),(24.953339,-17.788422),(24.964295,-17.786665),(24.958094,-17.800928),(24.968636,-17.807646),(24.975147,-17.816327),(24.983312,-17.820462),(24.998505,-17.81395),(25.007083,-17.825733),(25.019692,-17.823769),(25.047494,-17.807129),(25.057002,-17.827696),(25.087905,-17.826766),(25.120874,-17.813537),(25.153947,-17.781808),(25.194152,-17.782324),(25.259781,-17.794107)] +Central African Republic [(22.55576,10.978969),(22.57705,10.985119),(22.600822,10.987806),(22.60909,10.985222),(22.62821,10.976127),(22.638339,10.974137),(22.673169,10.975068),(22.682677,10.974137),(22.719884,10.964655),(22.745102,10.943803),(22.805977,10.924502),(22.861064,10.919154),(22.863234,10.891817),(23.005551,10.686817),(23.109421,10.614495),(23.291219,10.439726),(23.457617,10.173747),(23.624015,9.907768),(23.644789,9.863068),(23.674038,9.69034),(23.669077,9.652022),(23.655538,9.622463),(23.618537,9.566084),(23.606135,9.537171),(23.606755,9.526241),(23.61647,9.501463),(23.618537,9.488311),(23.617194,9.457176),(23.619984,9.447719),(23.627529,9.435084),(23.63497,9.435394),(23.640861,9.43312),(23.646339,9.425033),(23.646856,9.418393),(23.621845,9.34062),(23.620501,9.322869),(23.630733,9.292173),(23.63218,9.277574),(23.623291,9.265637),(23.610062,9.255457),(23.571408,9.207294),(23.549498,9.185254),(23.537922,9.178149),(23.521799,9.175358),(23.48945,9.176702),(23.474257,9.170682),(23.462821,9.153725),(23.45772,9.146161),(23.435706,9.018934),(23.43829,8.994543),(23.451416,8.974182),(23.478081,8.958912),(23.505469,8.961082),(23.542573,8.997281),(23.560143,8.996455),(23.567481,8.974828),(23.565931,8.940024),(23.553632,8.883154),(23.536682,8.85561),(23.495444,8.809438),(23.482318,8.783393),(23.481801,8.759363),(23.49007,8.732595),(23.505262,8.710736),(23.525726,8.701485),(23.540506,8.704793),(23.56407,8.722699),(23.578023,8.730476),(23.59642,8.734145),(23.61337,8.732233),(23.629079,8.725593),(23.644065,8.715154),(23.657088,8.710012),(23.689127,8.710684),(23.72127,8.702002),(23.737496,8.70686),(23.768916,8.721252),(23.803436,8.72213),(23.866687,8.70779),(23.922291,8.713449),(23.962082,8.697015),(23.981306,8.694173),(24.009624,8.698876),(24.114011,8.681642),(24.170328,8.689327),(24.180467,8.690711),(24.217674,8.691512),(24.235761,8.682003),(24.233487,8.667767),(24.21509,8.64136),(24.211266,8.627071),(24.218397,8.613041),(24.245579,8.591802),(24.25054,8.579632),(24.243719,8.570176),(24.203204,8.543459),(24.193179,8.532426),(24.137162,8.438866),(24.125173,8.404889),(24.121556,8.372307),(24.131374,8.343058),(24.152872,8.317891),(24.17995,8.297712),(24.206615,8.283268),(24.222428,8.277196),(24.257051,8.269186),(24.263186,8.268505),(24.281029,8.266525),(24.295498,8.266525),(24.302733,8.26544),(24.310278,8.261564),(24.326814,8.248567),(24.332085,8.245725),(24.396578,8.267817),(24.431097,8.271408),(24.454869,8.248671),(24.458899,8.239757),(24.464274,8.233194),(24.471508,8.22893),(24.481327,8.226657),(24.51285,8.207149),(24.544682,8.206115),(24.614859,8.217148),(24.670876,8.206839),(24.69041,8.206632),(24.710564,8.204307),(24.742293,8.18684),(24.800171,8.180277),(24.832107,8.16573),(24.917993,8.087027),(24.927811,8.070956),(24.930085,8.035454),(24.950549,8.014396),(24.952616,7.996567),(24.958094,7.989178),(24.964812,7.982615),(24.972666,7.976982),(24.981245,7.972331),(25.029304,7.918846),(25.059173,7.896108),(25.089559,7.884895),(25.103408,7.885773),(25.134414,7.892543),(25.148573,7.892594),(25.173171,7.888564),(25.180509,7.884016),(25.18547,7.877918),(25.191568,7.872389),(25.216889,7.864069),(25.229705,7.851615),(25.23942,7.835957),(25.249652,7.804331),(25.264535,7.777718),(25.269909,7.760768),(25.270633,7.746143),(25.266809,7.703097),(25.270633,7.688162),(25.276421,7.674261),(25.279418,7.659482),(25.274974,7.641964),(25.252546,7.62243),(25.186504,7.600106),(25.165213,7.5799),(25.164593,7.567343),(25.169037,7.551168),(25.190431,7.501197),(25.249962,7.469984),(25.263295,7.461044),(25.269599,7.454223),(25.279211,7.438358),(25.285102,7.431589),(25.292957,7.427196),(25.3097,7.422029),(25.316211,7.417223),(25.32417,7.402908),(25.330577,7.374125),(25.335952,7.359604),(25.346804,7.345031),(25.360033,7.335574),(25.416154,7.307772),(25.455118,7.278213),(25.481163,7.266173),(25.500335,7.272787),(25.516768,7.269945),(25.53165,7.260695),(25.576609,7.219819),(25.592422,7.211241),(25.65402,7.195376),(25.672727,7.187831),(25.697015,7.168607),(25.705077,7.166385),(25.724507,7.166695),(25.734429,7.162406),(25.750242,7.146542),(25.7613,7.143751),(25.786209,7.142666),(25.791583,7.134398),(25.791686,7.121117),(25.800781,7.104942),(25.815767,7.099774),(25.860726,7.095537),(25.877469,7.087734),(25.881707,7.079827),(25.884084,7.061069),(25.888735,7.05187),(25.920257,7.034145),(25.930386,7.03151),(25.950436,7.028564),(25.960461,7.024482),(25.966249,7.017454),(25.966766,7.009857),(25.969453,7.003553),(25.981132,7.000297),(26.026091,6.99668),(26.032292,6.974097),(26.035806,6.922266),(26.045418,6.899683),(26.050585,6.896686),(26.05689,6.89741),(26.063401,6.899063),(26.069189,6.899063),(26.073633,6.895136),(26.073116,6.88971),(26.071152,6.88356),(26.071566,6.877462),(26.0757,6.864647),(26.077354,6.85333),(26.080971,6.842348),(26.091203,6.830334),(26.113734,6.816458),(26.131821,6.81173),(26.147634,6.804702),(26.178743,6.765919),(26.219464,6.737523),(26.236207,6.72003),(26.270623,6.702486),(26.329741,6.680575),(26.378007,6.65329),(26.379224,6.631362),(26.379867,6.619778),(26.372323,6.609804),(26.356096,6.579496),(26.324367,6.537199),(26.30504,6.496917),(26.296462,6.489527),(26.286436,6.484179),(26.277031,6.477358),(26.27021,6.465653),(26.289124,6.459426),(26.290674,6.444672),(26.285093,6.425888),(26.282819,6.407853),(26.28964,6.387208),(26.300699,6.377829),(26.315789,6.371576),(26.335219,6.360595),(26.351755,6.344368),(26.375837,6.312587),(26.394647,6.300547),(26.436195,6.29171),(26.452008,6.28029),(26.455315,6.254451),(26.455625,6.230474),(26.464927,6.224402),(26.4795,6.225073),(26.495829,6.221146),(26.508955,6.205462),(26.509472,6.186394),(26.500067,6.167971),(26.48353,6.154457),(26.46534,6.144303),(26.4456,6.130221),(26.42927,6.113091),(26.421312,6.09366),(26.424826,6.072447),(26.440639,6.076995),(26.48105,6.104951),(26.498827,6.099836),(26.509885,6.082679),(26.51805,6.061156),(26.527972,6.043172),(26.543578,6.030847),(26.602179,6.010616),(26.634322,6.006482),(26.705016,6.009789),(26.776329,5.981884),(26.794364,5.970489),(26.805061,5.957183),(26.810849,5.912379),(26.81891,5.894628),(26.842165,5.884267),(26.865316,5.885973),(26.882059,5.891941),(26.894255,5.889667),(26.903763,5.866982),(26.916475,5.849644),(26.937456,5.848507),(26.981071,5.859204),(26.99182,5.847706),(27.003499,5.819284),(27.021999,5.805099),(27.029543,5.78988),(27.036055,5.785126),(27.045356,5.784919),(27.063547,5.790862),(27.072745,5.791224),(27.11481,5.775282),(27.123801,5.768745),(27.130933,5.752131),(27.131449,5.741459),(27.136307,5.734018),(27.156254,5.727481),(27.170413,5.72035),(27.182351,5.707456),(27.190774,5.691824),(27.193823,5.676321),(27.194288,5.667304),(27.196407,5.661464),(27.201109,5.656968),(27.218369,5.645315),(27.219196,5.639993),(27.216716,5.633456),(27.215165,5.616144),(27.210669,5.601623),(27.211341,5.594879),(27.217336,5.585397),(27.222607,5.584363),(27.227309,5.586714),(27.231082,5.587593),(27.234596,5.588833),(27.240073,5.591985),(27.247515,5.592605),(27.257127,5.586068),(27.261054,5.577593),(27.258884,5.559972),(27.260744,5.550257),(27.220126,5.440883),(27.218059,5.42551),(27.220126,5.413082),(27.228239,5.387605),(27.233665,5.33828),(27.2378,5.32319),(27.264465,5.260145),(27.280898,5.23056),(27.30131,5.205187),(27.358516,5.166766),(27.385801,5.143847),(27.402131,5.104496),(27.431173,5.083618),(27.441301,5.070725),(27.414843,5.080725),(27.405335,5.082843),(27.391382,5.092223),(27.380323,5.094393),(27.361203,5.095478),(27.351384,5.097339),(27.342754,5.100594),(27.328647,5.1108),(27.315728,5.122402),(27.301878,5.131988),(27.284412,5.135967),(27.262811,5.138447),(27.239712,5.144881),(27.157494,5.185421),(27.11636,5.200304),(27.073675,5.203327),(27.027476,5.189969),(26.991716,5.171288),(26.961641,5.151185),(26.933684,5.126587),(26.90397,5.094393),(26.899216,5.085996),(26.889604,5.062121),(26.884126,5.053388),(26.872654,5.041786),(26.867073,5.037549),(26.849296,5.039099),(26.848366,5.041373),(26.844955,5.046282),(26.839478,5.051166),(26.832656,5.053388),(26.826145,5.050752),(26.815706,5.040184),(26.808368,5.039099),(26.801857,5.044525),(26.760619,5.088114),(26.753695,5.092274),(26.739432,5.094393),(26.691683,5.094393),(26.681244,5.089277),(26.667602,5.078347),(26.657577,5.073257),(26.644037,5.078347),(26.635666,5.077624),(26.629671,5.067056),(26.62347,5.067056),(26.615512,5.078244),(26.598355,5.080621),(26.579855,5.074343),(26.568176,5.059641),(26.561355,5.067056),(26.554224,5.059847),(26.541511,5.053129),(26.528282,5.048091),(26.519807,5.045972),(26.502547,5.047471),(26.475159,5.057263),(26.462653,5.059641),(26.460689,5.066023),(26.441879,5.104289),(26.429373,5.113798),(26.414284,5.122712),(26.400125,5.133641),(26.390616,5.149015),(26.369429,5.140566),(26.362711,5.135967),(26.353306,5.147775),(26.336356,5.148602),(26.317339,5.146431),(26.291811,5.150979),(26.272484,5.159066),(26.253984,5.172321),(26.246646,5.189969),(26.242098,5.188057),(26.230729,5.185163),(26.226182,5.183147),(26.225665,5.199038),(26.226182,5.204257),(26.213573,5.199141),(26.207165,5.207694),(26.198276,5.238389),(26.195176,5.234772),(26.18453,5.225341),(26.18174,5.229579),(26.175332,5.233687),(26.170888,5.238389),(26.157349,5.232472),(26.150837,5.239966),(26.144223,5.251799),(26.12996,5.258879),(26.125103,5.248595),(26.119625,5.241542),(26.112494,5.238389),(26.104535,5.238596),(26.098438,5.239578),(26.093373,5.241438),(26.088412,5.244591),(26.09172,5.219062),(26.095234,5.211053),(26.086035,5.215238),(26.077147,5.216324),(26.070532,5.212241),(26.067845,5.200846),(26.062677,5.200123),(26.050999,5.200226),(26.037356,5.197953),(26.026917,5.189969),(26.013895,5.199244),(25.996635,5.219243),(25.98599,5.225341),(25.971934,5.227667),(25.962839,5.224514),(25.958394,5.214825),(25.958601,5.197384),(25.943202,5.204102),(25.925632,5.201777),(25.912506,5.190382),(25.910852,5.170099),(25.892972,5.185214),(25.883567,5.203534),(25.872405,5.217099),(25.849357,5.217874),(25.84605,5.211673),(25.836231,5.20002),(25.826103,5.194697),(25.818971,5.214567),(25.814217,5.22162),(25.811427,5.230922),(25.814631,5.244591),(25.810186,5.242679),(25.799438,5.240095),(25.794683,5.238389),(25.792306,5.262832),(25.778354,5.269137),(25.769259,5.265158),(25.781144,5.258879),(25.771739,5.245469),(25.757993,5.244436),(25.743731,5.250611),(25.732775,5.258879),(25.705077,5.289601),(25.691124,5.299807),(25.668903,5.306137),(25.667559,5.307248),(25.663115,5.30916),(25.655777,5.317764),(25.650093,5.320297),(25.637897,5.317971),(25.631076,5.311822),(25.623428,5.309935),(25.609165,5.320297),(25.609165,5.326524),(25.614229,5.329934),(25.617227,5.333552),(25.61981,5.337221),(25.622911,5.34076),(25.600483,5.34045),(25.592319,5.349184),(25.589011,5.362284),(25.58126,5.374919),(25.575059,5.374919),(25.561829,5.373006),(25.543743,5.37528),(25.532787,5.372231),(25.540849,5.354429),(25.528963,5.351897),(25.504469,5.342285),(25.49558,5.344533),(25.484935,5.355307),(25.481989,5.356858),(25.479922,5.353189),(25.435067,5.328694),(25.413363,5.323552),(25.39569,5.333965),(25.385354,5.321485),(25.370678,5.316834),(25.36365,5.310556),(25.376466,5.293011),(25.355899,5.286448),(25.343393,5.271927),(25.337192,5.251748),(25.335538,5.228442),(25.328924,5.216013),(25.316211,5.202242),(25.307736,5.185292),(25.314454,5.163252),(25.321482,5.159169),(25.341223,5.155604),(25.348561,5.149015),(25.349284,5.141677),(25.343807,5.121885),(25.34236,5.111472),(25.339156,5.105168),(25.324996,5.088114),(25.321896,5.077004),(25.320966,5.052251),(25.316832,5.042251),(25.307633,5.032278),(25.292337,5.028273),(25.248515,5.024139),(25.239317,5.015535),(25.227535,5.009566),(25.163663,5.005639),(25.154568,5.009308),(25.147643,5.01414),(25.140201,5.017964),(25.129453,5.018635),(25.121701,5.008636),(25.11426,5.001841),(25.105268,4.998766),(25.100927,4.994554),(25.090489,4.974478),(25.084804,4.954221),(25.076123,4.951844),(25.040569,4.962101),(25.006463,4.980937),(24.969359,4.991402),(24.958094,4.99135),(24.950859,4.987681),(24.932359,4.974426),(24.927088,4.971481),(24.90435,4.966804),(24.868177,4.944376),(24.848333,4.936728),(24.836861,4.93771),(24.827662,4.941922),(24.819498,4.94448),(24.811023,4.940449),(24.800067,4.928899),(24.789422,4.919856),(24.77826,4.918538),(24.766374,4.929881),(24.749838,4.918745),(24.730614,4.917505),(24.664365,4.924403),(24.654236,4.929416),(24.65744,4.939777),(24.670773,4.957218),(24.656923,4.967269),(24.627468,4.976907),(24.615479,4.985097),(24.610001,4.996725),(24.610105,5.016723),(24.601216,5.026103),(24.588607,5.031064),(24.563389,5.034888),(24.553467,5.039099),(24.547369,5.046489),(24.538688,5.065558),(24.533003,5.073257),(24.513676,5.087236),(24.487218,5.100801),(24.480844,5.102335),(24.459623,5.107441),(24.436679,5.100594),(24.433061,5.089329),(24.433475,5.075531),(24.430684,5.067418),(24.416938,5.073257),(24.41146,5.081861),(24.401642,5.111472),(24.396371,5.122298),(24.382522,5.107855),(24.367846,5.084549),(24.361231,5.062948),(24.37198,5.053388),(24.396888,5.048194),(24.398645,5.036024),(24.386139,5.022356),(24.368466,5.012434),(24.296635,5.003262),(24.286507,4.995071),(24.284646,4.992435),(24.272968,4.964013),(24.26966,4.952464),(24.269557,4.943188),(24.267697,4.935721),(24.259222,4.929881),(24.25302,4.929881),(24.246199,4.954066),(24.229766,4.961171),(24.209612,4.956184),(24.190905,4.94417),(24.162897,4.907789),(24.152768,4.902596),(24.139229,4.905438),(24.119902,4.9189),(24.10843,4.92306),(24.08962,4.920218),(24.072463,4.91027),(24.046935,4.888928),(23.995775,4.865053),(23.977998,4.854149),(23.963116,4.868696),(23.957121,4.870815),(23.950817,4.867792),(23.946476,4.861642),(23.943169,4.851695),(23.942755,4.840507),(23.947406,4.830585),(23.948336,4.817692),(23.924358,4.817795),(23.88188,4.82689),(23.874749,4.824978),(23.864414,4.81609),(23.855215,4.813222),(23.847877,4.813997),(23.82824,4.819345),(23.816665,4.820611),(23.798475,4.814746),(23.760027,4.787435),(23.73791,4.779063),(23.692848,4.773405),(23.672694,4.767204),(23.63404,4.745138),(23.588048,4.73395),(23.566551,4.724441),(23.506089,4.676434),(23.488416,4.669845),(23.465058,4.66721),(23.448315,4.659407),(23.437153,4.646565),(23.430022,4.62884),(23.428265,4.617342),(23.428368,4.608609),(23.427438,4.601348),(23.42258,4.594062),(23.414519,4.590832),(23.396019,4.591607),(23.38837,4.587266),(23.371214,4.597266),(23.353954,4.60357),(23.335867,4.6068),(23.316437,4.60773),(23.313026,4.610211),(23.301967,4.621709),(23.29959,4.625119),(23.295146,4.6276),(23.273132,4.632018),(23.264864,4.635067),(23.251841,4.662972),(23.224246,4.680051),(23.220835,4.683514),(23.20864,4.690542),(23.183732,4.725268),(23.168746,4.73811),(23.12389,4.715631),(23.099396,4.711652),(23.077382,4.721057),(23.066633,4.730178),(23.055161,4.7378),(23.042758,4.742967),(23.029323,4.744931),(23.017334,4.750512),(23.011132,4.763638),(23.007205,4.778443),(22.99625,4.799501),(22.985708,4.826425),(22.977439,4.834306),(22.965037,4.835287),(22.953048,4.830792),(22.941473,4.82472),(22.92969,4.820611),(22.915221,4.821154),(22.905919,4.824203),(22.898374,4.823583),(22.888763,4.813222),(22.882872,4.800742),(22.882355,4.788494),(22.886799,4.776686),(22.895584,4.765395),(22.887316,4.755499),(22.865302,4.738678),(22.853829,4.711264),(22.837293,4.714235),(22.818069,4.724596),(22.802876,4.730643),(22.78603,4.724648),(22.776418,4.710101),(22.765153,4.676021),(22.757401,4.660802),(22.74872,4.64827),(22.737867,4.637754),(22.723605,4.62884),(22.723811,4.617032),(22.729806,4.596749),(22.73115,4.587266),(22.728669,4.58029),(22.718954,4.562772),(22.716783,4.556519),(22.713683,4.551222),(22.700144,4.541068),(22.697043,4.535745),(22.697146,4.512025),(22.695079,4.501483),(22.689498,4.491665),(22.675649,4.48474),(22.654358,4.482828),(22.61033,4.484224),(22.59245,4.473707),(22.586972,4.449394),(22.589039,4.421953),(22.594,4.402316),(22.61126,4.381853),(22.613741,4.374359),(22.609297,4.363352),(22.599891,4.352552),(22.588006,4.34418),(22.576534,4.340847),(22.568575,4.331546),(22.539327,4.278164),(22.53974,4.266924),(22.545734,4.245944),(22.546148,4.237184),(22.540464,4.227262),(22.52217,4.211346),(22.518243,4.206799),(22.510491,4.19127),(22.492714,4.174036),(22.457368,4.149076),(22.451532,4.146672),(22.43184,4.13856),(22.422641,4.134787),(22.303269,4.128716),(22.207254,4.150316),(22.184723,4.164605),(22.161882,4.173183),(22.15165,4.179462),(22.133047,4.19866),(22.127879,4.203078),(22.108449,4.209848),(22.057806,4.219537),(22.036613,4.229439),(22.00892,4.242378),(22.001478,4.244652),(21.989593,4.244238),(21.984528,4.242946),(21.980291,4.240621),(21.970783,4.237184),(21.949905,4.233464),(21.891614,4.237184),(21.86774,4.241551),(21.851927,4.25168),(21.838077,4.263255),(21.819887,4.271937),(21.810999,4.273177),(21.78206,4.271937),(21.771001,4.273487),(21.76604,4.277208),(21.76325,4.281678),(21.758496,4.285605),(21.737308,4.293382),(21.722942,4.295139),(21.688939,4.292452),(21.651112,4.295656),(21.634162,4.294209),(21.538147,4.244652),(21.527812,4.249251),(21.508072,4.249406),(21.49722,4.251473),(21.489882,4.256253),(21.48244,4.263126),(21.473138,4.269301),(21.460219,4.271937),(21.416915,4.270076),(21.397691,4.271627),(21.37485,4.278164),(21.358313,4.285553),(21.310358,4.316094),(21.3055,4.323148),(21.298886,4.330228),(21.288344,4.33338),(21.275321,4.332527),(21.265916,4.330073),(21.258371,4.32599),(21.250723,4.320358),(21.244625,4.313769),(21.239148,4.305604),(21.23243,4.297827),(21.223438,4.292452),(21.210932,4.291574),(21.202974,4.296638),(21.195739,4.302917),(21.185611,4.306069),(21.16556,4.307154),(21.158119,4.310203),(21.150057,4.326714),(21.137965,4.332682),(21.113574,4.340847),(21.103445,4.353275),(21.084222,4.388235),(21.076057,4.395469),(21.059624,4.398596),(20.963505,4.433736),(20.871521,4.453373),(20.832351,4.450686),(20.818708,4.443528),(20.807029,4.434252),(20.791836,4.426191),(20.767859,4.42278),(20.68838,4.42278),(20.603114,4.409732),(20.580273,4.415003),(20.559809,4.428361),(20.456713,4.524816),(20.452322,4.528924),(20.44209,4.550499),(20.447051,4.569671),(20.45718,4.58644),(20.463071,4.600935),(20.456456,4.621502),(20.44023,4.64411),(20.420593,4.662352),(20.403746,4.669845),(20.395478,4.677106),(20.373671,4.724441),(20.354034,4.755396),(20.338944,4.771906),(20.322511,4.779063),(20.301737,4.781647),(20.254815,4.797124),(20.236625,4.806349),(20.217194,4.822446),(20.171719,4.87836),(20.152599,4.890685),(20.08635,4.916264),(20.05462,4.937813),(20.031056,4.957993),(20.003977,4.974478),(19.915404,4.993107),(19.890909,5.002125),(19.880367,5.015535),(19.87551,5.031064),(19.858163,5.060305),(19.850188,5.073748),(19.838613,5.087546),(19.82466,5.096977),(19.779185,5.118164),(19.748799,5.124417),(19.728749,5.133848),(19.71955,5.135967),(19.605552,5.138292),(19.568759,5.15519),(19.548605,5.151366),(19.517082,5.135967),(19.496515,5.133796),(19.431506,5.135967),(19.408975,5.130386),(19.394196,5.116976),(19.373112,5.087546),(19.283815,5.032278),(19.236996,5.010781),(19.229761,5.002176),(19.214672,4.976131),(19.195655,4.950345),(19.179015,4.946418),(19.133023,4.946934),(19.116176,4.940449),(19.106741,4.931518),(19.08331,4.90934),(19.069047,4.891253),(19.057782,4.867792),(19.048997,4.836528),(19.044139,4.82689),(19.035768,4.818002),(19.02688,4.811516),(19.019748,4.8033),(19.011583,4.765033),(18.998044,4.750641),(18.980474,4.739402),(18.962284,4.724441),(18.928178,4.666435),(18.923837,4.662869),(18.90451,4.655867),(18.900169,4.652146),(18.897895,4.650647),(18.886526,4.621399),(18.85242,4.594062),(18.828235,4.559981),(18.776972,4.433451),(18.753304,4.400611),(18.721058,4.377357),(18.674343,4.361311),(18.654086,4.357229),(18.633932,4.355963),(18.614295,4.359813),(18.595795,4.371259),(18.576468,4.372757),(18.556831,4.354154),(18.541948,4.327851),(18.53709,4.306069),(18.545048,4.289016),(18.57254,4.265684),(18.578638,4.254858),(18.583909,4.235789),(18.619669,4.169566),(18.632692,4.130421),(18.646541,4.045982),(18.645301,3.982445),(18.64158,3.958261),(18.634552,3.93573),(18.613675,3.901468),(18.611711,3.879093),(18.612024,3.866611),(18.612848,3.833746),(18.593004,3.709775),(18.626387,3.476869),(18.582462,3.477386),(18.570267,3.489607),(18.55435,3.526375),(18.513422,3.592237),(18.497093,3.61146),(18.472081,3.632053),(18.459059,3.631382),(18.448,3.618023),(18.429707,3.600221),(18.420612,3.596371),(18.403662,3.597353),(18.392293,3.595854),(18.387849,3.591668),(18.383715,3.578),(18.379891,3.574382),(18.372863,3.574537),(18.361597,3.58017),(18.355603,3.581462),(18.275918,3.574382),(18.264859,3.574744),(18.255867,3.577225),(18.248529,3.574899),(18.242535,3.561127),(18.235403,3.530923),(18.231579,3.521156),(18.220004,3.503947),(18.202537,3.487333),(18.181556,3.477386),(18.159852,3.480305),(18.146727,3.494904),(18.134014,3.534824),(18.116444,3.552007),(18.082028,3.563453),(18.047094,3.563143),(18.013091,3.55397),(17.981879,3.538803),(17.969476,3.537511),(17.959865,3.541077),(17.924518,3.563453),(17.919247,3.565597),(17.913562,3.561334),(17.875322,3.542731),(17.856408,3.537305),(17.842146,3.540147),(17.835324,3.555831),(17.83057,3.575674),(17.821165,3.593994),(17.808349,3.608592),(17.793156,3.617506),(17.784475,3.61823),(17.765148,3.614354),(17.755949,3.614509),(17.749645,3.618075),(17.73993,3.629159),(17.733212,3.632208),(17.72608,3.631847),(17.714505,3.627506),(17.70913,3.626421),(17.627275,3.626317),(17.586761,3.632234),(17.550484,3.648151),(17.513484,3.679182),(17.508936,3.681766),(17.494983,3.683213),(17.490126,3.68714),(17.486198,3.700292),(17.481858,3.704762),(17.458913,3.708276),(17.442687,3.701274),(17.427494,3.690112),(17.414489,3.683415),(17.375714,3.663447),(17.355767,3.638435),(17.334373,3.618514),(17.298096,3.615904),(17.272258,3.62456),(17.259442,3.625904),(17.244973,3.620607),(17.231123,3.609109),(17.222959,3.598515),(17.213967,3.589291),(17.197224,3.58172),(17.182858,3.579602),(17.142757,3.580842),(17.129011,3.57707),(17.102139,3.566605),(17.088393,3.564435),(17.061418,3.56614),(17.052426,3.56397),(17.035787,3.556141),(17.028862,3.551541),(17.017596,3.5418),(17.009225,3.538131),(17.003127,3.537718),(16.989691,3.539475),(16.98194,3.537976),(16.968607,3.533481),(16.960546,3.538441),(16.953208,3.546891),(16.942252,3.553092),(16.932434,3.553919),(16.904735,3.550637),(16.895537,3.552368),(16.877243,3.559164),(16.867838,3.561541),(16.853162,3.560094),(16.851818,3.551903),(16.854609,3.541335),(16.852645,3.532809),(16.833008,3.522861),(16.811924,3.521672),(16.729552,3.542214),(16.709398,3.543712),(16.686351,3.542214),(16.639428,3.528597),(16.598501,3.501674),(16.567701,3.464389),(16.551682,3.419767),(16.542793,3.336154),(16.517782,3.273238),(16.517162,3.248692),(16.511477,3.227246),(16.477061,3.179394),(16.465485,3.156501),(16.464762,3.1318),(16.47272,3.110612),(16.483159,3.090148),(16.490807,3.067824),(16.484192,3.031909),(16.445538,2.95765),(16.448329,2.913467),(16.460525,2.894605),(16.474994,2.878378),(16.483779,2.860395),(16.479128,2.836314),(16.450603,2.775594),(16.427492,2.726493),(16.413706,2.697201),(16.371124,2.606922),(16.325649,2.510494),(16.293506,2.442074),(16.248031,2.345594),(16.196665,2.236454),(16.192841,2.239503),(16.179715,2.246996),(16.181265,2.256556),(16.18695,2.26932),(16.186536,2.277692),(16.182919,2.281154),(16.169069,2.289887),(16.165969,2.294796),(16.164729,2.314795),(16.162868,2.323839),(16.159251,2.332934),(16.152326,2.343269),(16.133413,2.364818),(16.120907,2.389571),(16.112949,2.410448),(16.111502,2.418562),(16.111502,2.455872),(16.107058,2.474165),(16.094345,2.497523),(16.090315,2.510494),(16.091245,2.532766),(16.106851,2.567028),(16.111502,2.5862),(16.108505,2.606612),(16.093105,2.641494),(16.090315,2.661906),(16.093725,2.67312),(16.101373,2.687847),(16.110468,2.701438),(16.0991,2.70397),(16.086284,2.704952),(16.072848,2.702834),(16.062616,2.703195),(16.059309,2.711825),(16.060032,2.72583),(16.056208,2.759833),(16.059929,2.786188),(16.086904,2.813163),(16.095172,2.836779),(16.092382,2.863289),(16.08246,2.885665),(16.070161,2.906594),(16.055381,2.93977),(16.034194,2.970621),(16.026236,2.978992),(16.015797,2.983488),(16.006082,2.983953),(15.995437,2.981473),(15.982001,2.976719),(15.977763,2.980956),(15.974249,2.985762),(15.971562,2.991188),(15.95823,3.041521),(15.950065,3.061003),(15.933425,3.081777),(15.913995,3.097952),(15.897768,3.103533),(15.854773,3.098985),(15.850949,3.100432),(15.840201,3.107408),(15.834206,3.108545),(15.829142,3.106633),(15.817566,3.100122),(15.813536,3.098985),(15.80103,3.100897),(15.793175,3.103274),(15.78563,3.107977),(15.7107,3.187455),(15.616855,3.286777),(15.517946,3.391603),(15.413973,3.50188),(15.329741,3.590996),(15.262306,3.662589),(15.25264,3.672852),(15.171301,3.758971),(15.084898,3.885836),(15.025987,4.026086),(15.043143,4.026138),(15.075183,4.017973),(15.091719,4.015854),(15.111253,4.019937),(15.150424,4.038514),(15.172438,4.043811),(15.192075,4.052234),(15.189801,4.063112),(15.177709,4.071458),(15.162413,4.073938),(15.14846,4.073938),(15.142569,4.08045),(15.139262,4.094583),(15.132854,4.108536),(15.117764,4.114324),(15.102571,4.112412),(15.091202,4.121791),(15.084484,4.148611),(15.082934,4.205507),(15.072806,4.265942),(15.063814,4.293434),(15.048518,4.321649),(15.013894,4.364257),(15.007383,4.37777),(15.000562,4.403195),(14.993947,4.413298),(14.982062,4.420481),(14.938344,4.435932),(14.779697,4.545047),(14.74373,4.57998),(14.717272,4.622019),(14.704249,4.669354),(14.697325,4.745215),(14.69071,4.762139),(14.691847,4.768702),(14.696084,4.779735),(14.700632,4.804282),(14.701459,4.814384),(14.686989,4.908875),(14.681615,4.920941),(14.675827,4.93076),(14.672107,4.940811),(14.674484,4.967579),(14.669833,4.977113),(14.663425,4.985717),(14.659498,4.997035),(14.660132,5.01144),(14.660324,5.015793),(14.667456,5.052768),(14.663012,5.091447),(14.663218,5.141832),(14.658877,5.164544),(14.651539,5.187514),(14.640481,5.208236),(14.624461,5.224049),(14.603584,5.234255),(14.560485,5.248595),(14.539712,5.261799),(14.523899,5.279679),(14.519868,5.290583),(14.528033,5.293011),(14.539608,5.303579),(14.548393,5.320503),(14.553974,5.338228),(14.555938,5.351328),(14.560796,5.355462),(14.570821,5.370939),(14.579296,5.387605),(14.579606,5.395382),(14.592835,5.400033),(14.597279,5.41148),(14.596246,5.440108),(14.599553,5.456645),(14.614126,5.48486),(14.617433,5.495299),(14.61888,5.525452),(14.617226,5.53788),(14.610612,5.553667),(14.596556,5.577232),(14.591491,5.590952),(14.590148,5.608289),(14.592835,5.62387),(14.610612,5.669138),(14.624358,5.723605),(14.631075,5.738049),(14.624047,5.758306),(14.617433,5.864966),(14.602964,5.883389),(14.598003,5.903491),(14.584773,5.923412),(14.572474,5.924187),(14.545396,5.907573),(14.530616,5.90592),(14.499197,5.909537),(14.482144,5.90964),(14.465711,5.920673),(14.458166,5.936409),(14.453825,5.953875),(14.446901,5.970024),(14.432328,5.98545),(14.417548,5.997258),(14.406076,6.01103),(14.397705,6.019169),(14.389436,6.031054),(14.387266,6.039038),(14.390263,6.045782),(14.397705,6.053947),(14.402459,6.054877),(14.412898,6.047022),(14.418169,6.047126),(14.422509,6.051363),(14.426023,6.059864),(14.429124,6.064179),(14.433568,6.073868),(14.432709,6.079123),(14.431811,6.084617),(14.430571,6.088079),(14.443903,6.100249),(14.467054,6.122599),(14.482144,6.127069),(14.510979,6.155336),(14.525966,6.174069),(14.534957,6.190088),(14.576402,6.189778),(14.719029,6.257862),(14.734842,6.270445),(14.772255,6.318323),(14.782384,6.336772),(14.784658,6.347805),(14.784348,6.358683),(14.780007,6.382221),(14.782177,6.393616),(14.807292,6.4278),(14.921874,6.686414),(14.932452,6.710289),(14.946095,6.72574),(14.962321,6.736696),(15.022369,6.766539),(15.034565,6.776771),(15.042213,6.790775),(15.05906,6.836276),(15.080247,6.878083),(15.113217,6.964692),(15.128926,7.026497),(15.136161,7.045308),(15.147013,7.061844),(15.156108,7.067787),(15.179052,7.07838),(15.187527,7.088612),(15.190214,7.098793),(15.185357,7.156877),(15.186597,7.166334),(15.191248,7.177341),(15.205097,7.200233),(15.223908,7.247517),(15.246564,7.266731),(15.261321,7.279247),(15.338216,7.32219),(15.385345,7.358363),(15.389169,7.364151),(15.397437,7.380739),(15.403225,7.386372),(15.409633,7.387664),(15.428443,7.388801),(15.432784,7.389731),(15.434954,7.402908),(15.415524,7.42084),(15.419451,7.43655),(15.473298,7.509103),(15.481049,7.523263),(15.515569,7.512204),(15.55195,7.509879),(15.624813,7.519749),(15.668532,7.516287),(15.720001,7.468641),(15.758345,7.455567),(15.793899,7.457996),(15.92526,7.488175),(15.94314,7.495306),(15.975696,7.515201),(15.990372,7.529412),(16.012283,7.560315),(16.026339,7.574888),(16.042979,7.583931),(16.0652,7.591217),(16.128762,7.599486),(16.162662,7.611113),(16.185606,7.610751),(16.207207,7.613542),(16.227257,7.625582),(16.26188,7.651885),(16.282758,7.660102),(16.370814,7.672504),(16.38301,7.680669),(16.387144,7.694674),(16.387041,7.762628),(16.392208,7.783609),(16.407401,7.796063),(16.41815,7.795856),(16.427245,7.791464),(16.437373,7.788363),(16.450913,7.792084),(16.457217,7.799628),(16.469413,7.825828),(16.475717,7.835802),(16.491737,7.849238),(16.509204,7.858281),(16.548685,7.870012),(16.550131,7.861692),(16.550752,7.8527),(16.550235,7.843502),(16.54486,7.813013),(16.549305,7.794719),(16.560157,7.779733),(16.576383,7.767796),(16.586822,7.763558),(16.596434,7.761129),(16.605219,7.757202),(16.613073,7.74821),(16.618138,7.734309),(16.616174,7.723871),(16.611833,7.713897),(16.609869,7.701598),(16.61421,7.679584),(16.624546,7.66868),(16.6632,7.657415),(16.686867,7.645788),(16.709812,7.627753),(16.746709,7.586566),(16.768619,7.550238),(16.782365,7.541143),(16.805516,7.543675),(16.812854,7.547706),(16.815071,7.549544),(16.836729,7.567498),(16.839623,7.567446),(16.850372,7.565896),(16.853886,7.567498),(16.855849,7.575404),(16.852439,7.580262),(16.848098,7.584706),(16.847168,7.591579),(16.854816,7.611475),(16.865461,7.624445),(16.880654,7.632869),(16.919618,7.644082),(16.962716,7.650697),(16.98194,7.648733),(16.988968,7.660774),(16.997546,7.666717),(17.008295,7.667647),(17.038991,7.662479),(17.042091,7.667027),(17.041058,7.675502),(17.045605,7.6847),(17.059351,7.696534),(17.079091,7.689919),(17.090254,7.683822),(17.095525,7.67824),(17.101932,7.677724),(17.116298,7.68687),(17.135522,7.705112),(17.170559,7.747539),(17.188645,7.764127),(17.196397,7.76516),(17.204562,7.763817),(17.211073,7.768312),(17.214484,7.786813),(17.220685,7.798181),(17.23379,7.810801),(17.234637,7.811617),(17.250864,7.822831),(17.385739,7.870477),(17.405997,7.883034),(17.419122,7.898227),(17.466355,7.884119),(17.478344,7.891819),(17.487232,7.914453),(17.503665,7.926236),(17.523819,7.93099),(17.559786,7.934711),(17.580766,7.940602),(17.59999,7.950369),(17.62066,7.978739),(17.639884,7.985043),(17.661795,7.986284),(17.679778,7.985198),(17.817238,7.962099),(17.858785,7.960445),(17.897853,7.967473),(17.977228,7.997187),(18.070452,8.019202),(18.508255,8.030674),(18.589283,8.047882),(18.617912,8.090127),(18.618636,8.138652),(18.638583,8.177642),(18.672586,8.20751),(18.715064,8.228672),(18.773768,8.248619),(18.791235,8.257378),(18.813042,8.276421),(18.8549,8.339802),(18.887457,8.370963),(18.901306,8.388636),(18.910401,8.412976),(18.920529,8.432122),(19.020472,8.545578),(19.061296,8.625624),(19.072261,8.631872),(19.081657,8.637226),(19.091475,8.652858),(19.124135,8.675079),(19.103877,8.698049),(19.073492,8.720528),(19.057782,8.729701),(19.048687,8.745669),(18.929108,8.796544),(18.917325,8.805148),(18.912468,8.818403),(18.909677,8.835327),(18.902959,8.84481),(18.886526,8.835844),(18.869783,8.849409),(18.86968,8.864111),(18.892727,8.897933),(18.901409,8.89398),(18.910608,8.894239),(18.917739,8.898812),(18.920736,8.907855),(18.922907,8.918216),(18.928591,8.921679),(18.936446,8.923358),(18.952052,8.931988),(18.971896,8.938215),(18.975927,8.941988),(18.97851,8.949455),(18.984815,8.956431),(18.999491,8.969635),(19.021919,8.985215),(19.060676,9.00418),(19.10057,9.015265),(19.126718,9.007177),(19.133643,9.007177),(19.142118,9.008676),(19.168783,9.002216),(19.174571,9.003767),(19.179635,9.015265),(19.192037,9.020174),(19.232862,9.022293),(19.253429,9.027951),(19.263248,9.028261),(19.275236,9.021983),(19.285778,9.01281),(19.295494,9.009503),(19.304795,9.020846),(19.316268,9.01312),(19.333321,9.006557),(19.352234,9.002061),(19.369391,9.000382),(19.38262,9.003198),(19.420964,9.017435),(19.430162,9.012862),(19.506644,9.013999),(19.511811,9.012293),(19.515842,9.009451),(19.52132,9.008831),(19.530828,9.013999),(19.553772,9.013327),(19.583125,9.025342),(19.613614,9.031672),(19.640072,9.013999),(19.656402,9.021259),(19.700017,9.020587),(19.709008,9.024567),(19.713969,9.02914),(19.738154,9.039372),(19.746836,9.04131),(19.758411,9.041982),(19.784663,9.048751),(19.80554,9.051128),(19.889463,9.046374),(19.902175,9.0518),(19.914474,9.069267),(19.928737,9.062575),(19.932091,9.063331),(19.943413,9.065882),(19.969044,9.082883),(19.976486,9.084175),(19.985477,9.083193),(19.993022,9.084589),(19.996329,9.092831),(19.997156,9.098619),(19.999533,9.104303),(20.002944,9.108515),(20.006871,9.11022),(20.014416,9.106345),(20.01793,9.091048),(20.024338,9.089756),(20.030332,9.093865),(20.029919,9.098981),(20.028369,9.104562),(20.031159,9.11022),(20.060512,9.134353),(20.068676,9.138126),(20.076945,9.13673),(20.090587,9.131123),(20.099372,9.13071),(20.100509,9.134146),(20.099579,9.141174),(20.101026,9.148073),(20.109604,9.151174),(20.117149,9.145748),(20.125727,9.133785),(20.137613,9.121822),(20.154563,9.116421),(20.169549,9.120891),(20.197971,9.140503),(20.212647,9.144973),(20.222982,9.14195),(20.234454,9.13456),(20.256985,9.116421),(20.276829,9.120736),(20.359408,9.116421),(20.378322,9.120065),(20.384419,9.123579),(20.40292,9.136989),(20.412428,9.142053),(20.423073,9.143241),(20.435166,9.138126),(20.455423,9.159313),(20.467618,9.184143),(20.481778,9.204659),(20.507203,9.213263),(20.51299,9.223883),(20.502655,9.247421),(20.496144,9.270882),(20.514231,9.281502),(20.526426,9.283621),(20.533661,9.289641),(20.537278,9.299201),(20.538208,9.311939),(20.541826,9.321629),(20.550921,9.320259),(20.572315,9.308218),(20.573555,9.309149),(20.592262,9.308063),(20.592882,9.308218),(20.609729,9.302792),(20.616653,9.302017),(20.667916,9.302017),(20.654894,9.342971),(20.695822,9.363461),(20.740883,9.377129),(20.750495,9.384545),(20.757937,9.377129),(20.766618,9.38656),(20.771062,9.398058),(20.77654,9.407799),(20.788322,9.411881),(20.792353,9.414),(20.798244,9.423379),(20.801965,9.42555),(20.806719,9.423328),(20.81323,9.419116),(20.813884,9.418847),(20.820258,9.416222),(20.82615,9.418083),(20.832661,9.437255),(20.834314,9.462473),(20.841446,9.484409),(20.86377,9.493789),(20.874209,9.502496),(20.912656,9.544328),(20.921234,9.558978),(20.925058,9.569598),(20.934257,9.577065),(20.955961,9.589416),(20.960508,9.598123),(20.962265,9.606934),(20.96764,9.610603),(20.983143,9.603678),(20.981799,9.616778),(20.980145,9.619931),(20.975804,9.624168),(20.975804,9.630964),(20.989964,9.635615),(20.996062,9.648301),(20.996888,9.682175),(21.000092,9.697394),(21.014252,9.724472),(21.017352,9.743592),(21.019936,9.747933),(21.025621,9.753256),(21.032339,9.75801),(21.037816,9.760671),(21.039056,9.76217),(21.04009,9.764754),(21.042674,9.767079),(21.048358,9.768165),(21.052906,9.765891),(21.062207,9.756201),(21.065825,9.754496),(21.073163,9.757648),(21.080294,9.762609),(21.09311,9.77434),(21.097037,9.775373),(21.101585,9.774237),(21.105202,9.77403),(21.106753,9.778061),(21.105926,9.791884),(21.106753,9.79545),(21.121739,9.831907),(21.131971,9.849426),(21.187264,9.885134),(21.195429,9.887305),(21.198427,9.902988),(21.205661,9.916838),(21.256924,9.975749),(21.27129,9.987247),(21.28514,9.978591),(21.326067,9.962752),(21.339503,9.95991),(21.374333,9.973113),(21.406889,10.005566),(21.433348,10.046055),(21.485437,10.164316),(21.513239,10.200515),(21.544969,10.219997),(21.554477,10.218654),(21.566673,10.214494),(21.579282,10.212272),(21.589721,10.216613),(21.593441,10.214546),(21.625274,10.224493),(21.630648,10.227439),(21.65628,10.233666),(21.668889,10.249195),(21.67695,10.269168),(21.688939,10.288908),(21.697414,10.292422),(21.708886,10.293533),(21.718808,10.296634),(21.723046,10.305987),(21.722116,10.314178),(21.717775,10.328983),(21.716845,10.339809),(21.720462,10.360816),(21.751054,10.41182),(21.743613,10.418978),(21.742786,10.425773),(21.74444,10.432439),(21.744233,10.439106),(21.738032,10.44802),(21.720462,10.468018),(21.716845,10.47732),(21.714674,10.493702),(21.705476,10.53202),(21.703305,10.552406),(21.705476,10.571811),(21.714674,10.601395),(21.716845,10.621007),(21.722632,10.636716),(21.736482,10.646173),(21.753431,10.650721),(21.768418,10.651987),(21.774205,10.655165),(21.778753,10.66227),(21.784851,10.669324),(21.795393,10.672476),(21.803558,10.673665),(21.822575,10.678755),(21.83291,10.679944),(21.852754,10.670151),(21.863606,10.667929),(21.86836,10.676223),(21.872081,10.677592),(21.895025,10.699193),(21.925617,10.720122),(21.943084,10.728261),(22.003855,10.743273),(22.014604,10.754564),(22.004889,10.775519),(22.0114,10.789291),(22.018325,10.809884),(22.029487,10.828746),(22.048607,10.836988),(22.132426,10.828539),(22.148239,10.830761),(22.176351,10.816214),(22.189787,10.837298),(22.200639,10.868666),(22.229371,10.891972),(22.24012,10.905899),(22.253039,10.915304),(22.268439,10.908999),(22.2765,10.908534),(22.287249,10.91595),(22.305956,10.932564),(22.318772,10.939359),(22.339442,10.947731),(22.361146,10.953648),(22.36469,10.954209),(22.391842,10.958505),(22.407035,10.963053),(22.419231,10.970727),(22.437627,10.991552),(22.447136,10.99827),(22.460468,11.000828),(22.476281,10.99796),(22.490751,10.993154),(22.502843,10.992173),(22.511938,11.000828),(22.534676,10.98052),(22.55576,10.978969)] +Switzerland [(8.617437,47.757319),(8.62984,47.762796),(8.635007,47.784604),(8.644102,47.791012),(8.657022,47.788118),(8.666633,47.778273),(8.674488,47.766698),(8.68193,47.75874),(8.692265,47.757164),(8.703324,47.758714),(8.713142,47.757422),(8.719757,47.747319),(8.71707,47.743547),(8.703737,47.730033),(8.70043,47.723496),(8.704667,47.715332),(8.712625,47.708691),(8.715106,47.701069),(8.71707,47.694558),(8.769883,47.695074),(8.761718,47.70125),(8.77071,47.720861),(8.797581,47.720034),(8.830241,47.707192),(8.856079,47.690682),(8.837682,47.687788),(8.837786,47.680838),(8.851935,47.671282),(8.852668,47.670786),(8.881711,47.656136),(8.906205,47.651795),(8.945376,47.654302),(8.981756,47.662156),(8.997673,47.673835),(9.016586,47.6789),(9.128104,47.670425),(9.183398,47.670425),(9.196937,47.656136),(9.234351,47.656162),(9.273211,47.65009),(9.547482,47.534547),(9.553059,47.516891),(9.554951,47.5109),(9.58451,47.480721),(9.621717,47.469197),(9.650346,47.452092),(9.649519,47.409717),(9.639804,47.394524),(9.601047,47.36127),(9.596396,47.352305),(9.591228,47.334683),(9.587404,47.32781),(9.553298,47.299853),(9.521155,47.262801),(9.504618,47.243732),(9.487358,47.210014),(9.484981,47.176346),(9.492629,47.15981),(9.503481,47.145392),(9.511853,47.129372),(9.51237,47.10803),(9.502861,47.094698),(9.487565,47.083949),(9.475886,47.073226),(9.477023,47.063898),(9.499554,47.059351),(9.560636,47.0524),(9.581203,47.05687),(9.59991,47.053486),(9.65231,47.05793),(9.669053,47.056199),(9.857982,47.015478),(9.856328,47.004083),(9.860566,47.001602),(9.866767,47.001938),(9.870591,46.998838),(9.870591,46.992947),(9.866457,46.983387),(9.863976,46.959925),(9.860772,46.949151),(9.862426,46.939772),(9.875138,46.927421),(9.899943,46.914398),(10.006913,46.890757),(10.045567,46.865564),(10.068098,46.856624),(10.1113,46.847116),(10.125188,46.846751),(10.13197,46.846573),(10.157808,46.851612),(10.201423,46.86683),(10.211655,46.877036),(10.214342,46.884685),(10.215169,46.893108),(10.219924,46.905769),(10.235116,46.923313),(10.251343,46.92538),(10.270773,46.921892),(10.295681,46.922693),(10.296198,46.941374),(10.313665,46.964318),(10.338883,46.98411),(10.367925,46.995505),(10.373403,46.996254),(10.378984,46.995505),(10.384255,46.993153),(10.384358,46.993153),(10.384358,46.992998),(10.394693,46.985402),(10.415571,46.962406),(10.449574,46.943906),(10.458462,46.936619),(10.463836,46.919747),(10.451434,46.88577),(10.453811,46.864427),(10.44854,46.832233),(10.444923,46.823241),(10.439032,46.816885),(10.417224,46.79885),(10.419085,46.783967),(10.426216,46.76942),(10.428696,46.755648),(10.416604,46.743014),(10.399654,46.735546),(10.395623,46.7264),(10.396554,46.715005),(10.394383,46.70082),(10.384771,46.689012),(10.373919,46.681906),(10.369165,46.672398),(10.377537,46.653277),(10.395623,46.638808),(10.438205,46.635656),(10.459082,46.623564),(10.466627,46.604288),(10.465903,46.578476),(10.457945,46.553697),(10.451833,46.546702),(10.443993,46.537729),(10.425906,46.535326),(10.354282,46.548323),(10.319452,46.546049),(10.306637,46.547496),(10.295371,46.551087),(10.289067,46.555687),(10.283796,46.560725),(10.275941,46.565531),(10.234703,46.575298),(10.230259,46.58615),(10.235737,46.606691),(10.233773,46.617982),(10.217856,46.626974),(10.192122,46.626819),(10.09745,46.608035),(10.087839,46.604392),(10.083498,46.597002),(10.071199,46.564394),(10.062931,46.556746),(10.04133,46.541863),(10.032752,46.532975),(10.031408,46.525792),(10.031201,46.503829),(10.02686,46.493184),(10.028101,46.483934),(10.030478,46.476673),(10.035335,46.471066),(10.044017,46.466984),(10.026344,46.446262),(10.042053,46.432722),(10.071405,46.424816),(10.116157,46.418822),(10.133417,46.414016),(10.140755,46.402905),(10.13321,46.381098),(10.125976,46.37438),(10.104995,46.361357),(10.09745,46.351642),(10.092386,46.338103),(10.091766,46.328956),(10.095797,46.320533),(10.104892,46.309371),(10.14613,46.280277),(10.158945,46.262449),(10.14582,46.243328),(10.117914,46.231133),(10.075746,46.220022),(10.042673,46.220487),(10.041847,46.24307),(10.031718,46.260072),(9.992237,46.284359),(9.977561,46.298105),(9.97105,46.320016),(9.970636,46.339808),(9.964022,46.356086),(9.93901,46.367455),(9.918443,46.37115),(9.899013,46.372158),(9.855398,46.366964),(9.788839,46.343296),(9.768065,46.33862),(9.755249,46.340532),(9.730858,46.350712),(9.720109,46.350893),(9.709257,46.342392),(9.707293,46.330972),(9.708844,46.319629),(9.70843,46.311748),(9.693134,46.297072),(9.674324,46.291801),(9.559705,46.292731),(9.536451,46.298622),(9.515264,46.308596),(9.502551,46.32074),(9.482604,46.35681),(9.473716,46.361874),(9.451598,46.370375),(9.444364,46.375284),(9.4424,46.380891),(9.443847,46.396136),(9.437852,46.492047),(9.434648,46.498326),(9.426794,46.497111),(9.410671,46.488895),(9.403849,46.482513),(9.400335,46.475407),(9.395478,46.469413),(9.384626,46.466416),(9.377081,46.468689),(9.351553,46.485484),(9.350829,46.497861),(9.330986,46.501504),(9.282306,46.49737),(9.263186,46.485122),(9.245823,46.461041),(9.237968,46.436547),(9.24758,46.423033),(9.260912,46.416651),(9.262876,46.406626),(9.260292,46.394017),(9.260396,46.379728),(9.273831,46.344252),(9.275175,46.331385),(9.268974,46.309371),(9.239725,46.266996),(9.224842,46.231184),(9.215747,46.221056),(9.204172,46.213563),(9.192183,46.209635),(9.181331,46.204054),(9.17575,46.194132),(9.171099,46.182609),(9.163788,46.172989),(9.163244,46.172273),(9.090587,46.138167),(9.072087,46.118892),(9.068159,46.105972),(9.07033,46.083441),(9.067126,46.071142),(9.059168,46.061789),(9.049659,46.057913),(9.027748,46.053107),(9.002117,46.03931),(8.997776,46.027941),(9.015553,45.993111),(8.982686,45.971975),(8.980516,45.969495),(8.979793,45.966911),(8.980516,45.964379),(8.982686,45.961847),(8.993435,45.95425),(9.001703,45.93606),(9.010798,45.926655),(9.020514,45.922779),(9.042321,45.919731),(9.051726,45.915545),(9.063095,45.898957),(9.059271,45.881955),(9.034363,45.848107),(9.002427,45.820718),(8.972351,45.824646),(8.939588,45.834826),(8.900004,45.826403),(8.903725,45.841802),(8.909719,45.853688),(8.91375,45.86609),(8.912096,45.883402),(8.906515,45.896476),(8.898144,45.90955),(8.88078,45.931099),(8.870962,45.947067),(8.864451,45.953424),(8.857733,45.957093),(8.800372,45.978538),(8.785076,45.982311),(8.767919,45.983086),(8.769573,45.985773),(8.773397,45.990579),(8.790967,46.018691),(8.819596,46.042927),(8.834375,46.066388),(8.80895,46.089746),(8.793861,46.093415),(8.763165,46.092898),(8.747145,46.094449),(8.739497,46.098066),(8.732159,46.107419),(8.728983,46.108233),(8.723891,46.109538),(8.71769,46.107523),(8.702187,46.097963),(8.695055,46.095172),(8.677485,46.095792),(8.630873,46.114706),(8.611546,46.119357),(8.601831,46.122819),(8.538682,46.187621),(8.51026,46.207878),(8.482665,46.217542),(8.456517,46.224828),(8.43812,46.23537),(8.427165,46.251442),(8.423237,46.275833),(8.426648,46.301568),(8.442874,46.353373),(8.446285,46.382183),(8.445768,46.412362),(8.441634,46.434945),(8.427888,46.44869),(8.399156,46.452179),(8.385907,46.450206),(8.343449,46.443885),(8.316267,46.433653),(8.294976,46.418046),(8.286605,46.40536),(8.290429,46.401122),(8.297043,46.397634),(8.297457,46.387506),(8.291462,46.378359),(8.281541,46.370116),(8.270068,46.364044),(8.24175,46.354123),(8.192554,46.309164),(8.171883,46.299191),(8.128475,46.292473),(8.106874,46.285548),(8.087341,46.271802),(8.077315,46.262035),(8.073078,46.253612),(8.076592,46.249736),(8.09995,46.235629),(8.129509,46.196044),(8.132299,46.159354),(8.110595,46.126953),(8.066877,46.100598),(8.056025,46.098066),(8.035354,46.096516),(8.025329,46.091141),(8.018197,46.080858),(8.016027,46.069385),(8.015924,46.058172),(8.010653,46.029698),(8.008792,46.027683),(7.999077,46.0128),(7.998354,46.010629),(7.985848,45.999312),(7.978717,45.995178),(7.969105,45.993111),(7.898205,45.981949),(7.883782,45.973869),(7.872917,45.959383),(7.870201,45.94037),(7.84962,45.939712),(7.848389,45.938076),(7.845288,45.927792),(7.846115,45.922573),(7.843738,45.919214),(7.831232,45.91446),(7.825444,45.914666),(7.807564,45.91849),(7.780072,45.918129),(7.732013,45.930376),(7.722195,45.929601),(7.71465,45.92712),(7.706279,45.925725),(7.69398,45.928671),(7.692533,45.931203),(7.673722,45.950323),(7.658736,45.960038),(7.643027,45.966343),(7.541121,45.984119),(7.524377,45.978073),(7.514662,45.966704),(7.503707,45.956731),(7.482726,45.954871),(7.452961,45.945879),(7.393843,45.9157),(7.361803,45.907845),(7.286666,45.913426),(7.27354,45.910274),(7.245428,45.89813),(7.183726,45.880456),(7.153547,45.876529),(7.120888,45.876116),(7.090192,45.880508),(7.066938,45.890223),(7.022083,45.92526),(7.015158,45.933321),(7.009784,45.943398),(7.002756,45.961692),(6.991283,45.982466),(6.987666,45.993111),(6.982808,45.995385),(6.915112,46.048612),(6.892375,46.055588),(6.884003,46.053211),(6.876872,46.048095),(6.869224,46.044064),(6.859715,46.044994),(6.85093,46.049645),(6.85031,46.052746),(6.852377,46.056931),(6.851964,46.064683),(6.853411,46.065665),(6.853101,46.076103),(6.851344,46.086025),(6.848553,46.085043),(6.853101,46.090211),(6.861162,46.097032),(6.86819,46.10468),(6.869224,46.112329),(6.853927,46.122612),(6.774346,46.134808),(6.765664,46.151603),(6.774863,46.185864),(6.792226,46.221676),(6.827676,46.269477),(6.804938,46.296607),(6.769488,46.322678),(6.750368,46.345518),(6.755742,46.357068),(6.782097,46.378462),(6.789229,46.395205),(6.788107,46.405008),(6.787058,46.414171),(6.777756,46.424093),(6.777716,46.424106),(6.762667,46.42926),(6.613684,46.455899),(6.547021,46.457372),(6.482942,46.448587),(6.397676,46.408176),(6.365223,46.40244),(6.332357,46.401381),(6.301558,46.394482),(6.269105,46.375026),(6.24058,46.348955),(6.219496,46.329111),(6.214122,46.315469),(6.218256,46.305495),(6.227454,46.288494),(6.227971,46.284463),(6.237583,46.267926),(6.24182,46.263689),(6.252259,46.259916),(6.269002,46.265239),(6.27603,46.26312),(6.281198,46.240073),(6.255359,46.221107),(6.191384,46.191704),(6.140328,46.150207),(6.107875,46.138632),(6.073872,46.149174),(6.028293,46.147934),(5.982921,46.140441),(5.95884,46.130467),(5.972172,46.152171),(5.979821,46.162248),(5.982921,46.170826),(5.965248,46.186226),(5.954809,46.19992),(5.95853,46.211961),(5.982921,46.222709),(6.042866,46.24307),(6.044519,46.243432),(6.046173,46.243535),(6.048033,46.243432),(6.055888,46.241675),(6.061883,46.241158),(6.067671,46.241623),(6.089685,46.246377),(6.094026,46.253044),(6.093095,46.262294),(6.093612,46.273094),(6.100743,46.301413),(6.104051,46.309216),(6.118607,46.331771),(6.1364,46.359342),(6.135057,46.370401),(6.122861,46.385542),(6.108185,46.396497),(6.059019,46.417383),(6.054235,46.419416),(6.065707,46.427012),(6.067567,46.433601),(6.0655,46.440371),(6.0655,46.447993),(6.064777,46.451068),(6.0624,46.455202),(6.060229,46.459904),(6.060229,46.46502),(6.064157,46.471118),(6.075525,46.479593),(6.110355,46.520831),(6.145702,46.55163),(6.121517,46.570285),(6.118417,46.583463),(6.131853,46.595607),(6.266315,46.680356),(6.337938,46.707409),(6.34786,46.71317),(6.374215,46.733609),(6.407391,46.745701),(6.417933,46.751101),(6.429199,46.760816),(6.433023,46.76911),(6.432609,46.785983),(6.425168,46.791615),(6.419897,46.796525),(6.417107,46.802157),(6.418554,46.807015),(6.434263,46.839545),(6.441188,46.848149),(6.443118,46.851455),(6.446769,46.857709),(6.448422,46.871559),(6.445219,46.882617),(6.431886,46.900032),(6.427752,46.909076),(6.442635,46.944164),(6.491107,46.963388),(6.598698,46.986539),(6.665412,47.021291),(6.688253,47.043848),(6.676264,47.0624),(6.6897,47.07829),(6.699105,47.084621),(6.72422,47.09077),(6.72794,47.097126),(6.731661,47.098883),(6.746027,47.103948),(6.744787,47.121053),(6.774759,47.128184),(6.838076,47.168132),(6.840285,47.169525),(6.859302,47.190919),(6.888344,47.211305),(6.956247,47.245231),(6.952216,47.270036),(6.958624,47.290551),(6.977434,47.303729),(6.986529,47.304504),(6.991904,47.305951),(7.006476,47.319361),(7.016915,47.323521),(7.027147,47.325433),(7.036552,47.329515),(7.044303,47.340497),(7.033865,47.350651),(7.018879,47.359901),(7.003996,47.368143),(6.985599,47.362123),(6.86664,47.354165),(6.871601,47.366955),(6.884003,47.382587),(6.898783,47.395713),(6.924517,47.405996),(6.926068,47.424858),(6.952319,47.428837),(6.968546,47.435194),(6.983429,47.443798),(6.990973,47.452221),(6.986116,47.464132),(6.97578,47.477956),(6.9733,47.489092),(6.991904,47.492942),(7.000792,47.49767),(7.009784,47.499247),(7.018879,47.49767),(7.027974,47.492942),(7.053915,47.490384),(7.103731,47.496275),(7.127296,47.492942),(7.140318,47.487852),(7.142169,47.487651),(7.153651,47.486405),(7.180833,47.488265),(7.162642,47.459895),(7.168327,47.443565),(7.190031,47.434728),(7.219383,47.428476),(7.223517,47.426228),(7.226308,47.422636),(7.230339,47.419019),(7.23809,47.416797),(7.244808,47.417727),(7.282635,47.428889),(7.309093,47.432661),(7.33693,47.431854),(7.378547,47.430646),(7.388219,47.433289),(7.406349,47.438242),(7.420563,47.450857),(7.426089,47.455761),(7.429293,47.465114),(7.427639,47.470747),(7.422782,47.475424),(7.419474,47.477852),(7.41441,47.484028),(7.414307,47.490177),(7.425986,47.492503),(7.441488,47.488834),(7.445996,47.486884),(7.454511,47.483201),(7.46743,47.481909),(7.482726,47.491263),(7.484483,47.492942),(7.485827,47.495784),(7.48593,47.498394),(7.485776,47.498768),(7.484897,47.5009),(7.477765,47.507696),(7.475595,47.511726),(7.476939,47.514879),(7.482726,47.516997),(7.493062,47.515499),(7.501123,47.517307),(7.505464,47.523018),(7.505154,47.533017),(7.501743,47.532965),(7.485103,47.541595),(7.482726,47.542267),(7.520867,47.563416),(7.526341,47.566452),(7.550319,47.575495),(7.585483,47.584479),(7.586028,47.584619),(7.637032,47.594977),(7.659666,47.596579),(7.646541,47.571542),(7.635895,47.564591),(7.612337,47.564731),(7.609747,47.564746),(7.646901,47.551445),(7.661423,47.546246),(7.683438,47.544257),(7.72732,47.550423),(7.76674,47.555961),(7.78555,47.563196),(7.801467,47.576089),(7.819657,47.595339),(7.833713,47.590481),(7.898205,47.587846),(7.904303,47.583557),(7.907507,47.574177),(7.90947,47.564798),(7.912157,47.560561),(8.042279,47.560561),(8.087237,47.567382),(8.096952,47.571852),(8.101397,47.576193),(8.105427,47.581257),(8.113902,47.587846),(8.122067,47.592135),(8.143771,47.600067),(8.162065,47.603762),(8.168886,47.608646),(8.173847,47.613529),(8.179015,47.615803),(8.232965,47.621952),(8.251051,47.622004),(8.276993,47.61663),(8.288569,47.615803),(8.293943,47.611462),(8.299317,47.601824),(8.306345,47.592187),(8.316164,47.587846),(8.354094,47.581024),(8.41807,47.580766),(8.4211,47.581112),(8.448869,47.58428),(8.450109,47.589034),(8.461788,47.606139),(8.49238,47.619833),(8.522353,47.621901),(8.537752,47.612134),(8.549638,47.598594),(8.551719,47.596863),(8.560697,47.589396),(8.574133,47.592445),(8.576305,47.595023),(8.580644,47.600171),(8.581781,47.607664),(8.581264,47.614769),(8.582504,47.62159),(8.582091,47.625027),(8.579714,47.628929),(8.57806,47.633476),(8.580334,47.639005),(8.583951,47.641124),(8.589222,47.642468),(8.593563,47.642571),(8.594493,47.640969),(8.595113,47.63482),(8.601624,47.632598),(8.607309,47.656291),(8.598214,47.656885),(8.593589,47.658169),(8.582194,47.66133),(8.568241,47.662932),(8.519666,47.657351),(8.504679,47.65226),(8.49083,47.645568),(8.476051,47.640401),(8.458274,47.639884),(8.437603,47.647842),(8.411972,47.661045),(8.407011,47.661562),(8.391301,47.665464),(8.397709,47.67629),(8.395229,47.684817),(8.390991,47.692129),(8.392128,47.699519),(8.401947,47.707089),(8.427268,47.716468),(8.437913,47.723186),(8.445458,47.743185),(8.450109,47.750471),(8.463648,47.763907),(8.471503,47.76706),(8.482665,47.766853),(8.536512,47.774088),(8.551602,47.779255),(8.5423,47.795017),(8.558216,47.801166),(8.583124,47.800236),(8.601624,47.794603),(8.603175,47.787317),(8.604105,47.774398),(8.607619,47.762254),(8.617437,47.757319)] +Clipperton Island [(-109.212026,10.30268),(-109.210357,10.288723),(-109.218577,10.281562),(-109.228017,10.292141),(-109.234242,10.307929),(-109.225942,10.311021),(-109.212026,10.30268)] +Cameroon [(14.560692,12.766224),(14.569374,12.769403),(14.569374,12.75961),(14.570717,12.750386),(14.575265,12.744908),(14.584877,12.746355),(14.603997,12.760798),(14.610612,12.762349),(14.618363,12.759196),(14.620327,12.754055),(14.62074,12.747853),(14.624254,12.741885),(14.641721,12.730904),(14.648129,12.725116),(14.664458,12.715969),(14.694017,12.723721),(14.709934,12.718243),(14.716755,12.70181),(14.702079,12.668995),(14.713654,12.65251),(14.728847,12.677212),(14.734118,12.680416),(14.743317,12.67716),(14.752102,12.668944),(14.758716,12.658815),(14.761403,12.649358),(14.768845,12.633235),(14.786208,12.63122),(14.819178,12.638816),(14.830753,12.618249),(14.863826,12.495466),(14.860105,12.490091),(14.852044,12.474692),(14.850184,12.468129),(14.849563,12.457019),(14.85101,12.455158),(14.855455,12.454538),(14.863826,12.447045),(14.861862,12.452006),(14.865893,12.452574),(14.872404,12.450404),(14.877469,12.447045),(14.878502,12.443066),(14.876849,12.431594),(14.877469,12.427201),(14.891215,12.402655),(14.908268,12.326897),(14.906304,12.317957),(14.904857,12.248401),(14.9031,12.236257),(14.898036,12.219462),(14.898036,12.207473),(14.90093,12.200032),(14.905891,12.195639),(14.910128,12.190368),(14.911575,12.180136),(14.907441,12.17347),(14.899896,12.171765),(14.894832,12.167372),(14.898036,12.152799),(14.9031,12.145875),(14.928732,12.128615),(14.950746,12.103345),(14.964802,12.092441),(14.976791,12.094147),(14.993844,12.106704),(15.011827,12.108616),(15.030431,12.100916),(15.049448,12.084586),(15.04459,12.078385),(15.04087,12.067585),(15.04056,12.055648),(15.045727,12.046088),(15.053065,12.038026),(15.053685,12.031877),(15.051205,12.024228),(15.049448,12.011929),(15.05968,11.997305),(15.076836,11.982836),(15.081177,11.971673),(15.052548,11.967229),(15.048311,11.962785),(15.050585,11.952863),(15.055546,11.942476),(15.05937,11.936792),(15.063401,11.927283),(15.056579,11.918447),(15.046967,11.910179),(15.042006,11.902375),(15.044177,11.877312),(15.050998,11.861603),(15.06278,11.853489),(15.079834,11.851216),(15.086345,11.840467),(15.110943,11.782899),(15.105362,11.772719),(15.089135,11.75763),(15.083554,11.748173),(15.085311,11.744039),(15.095957,11.734117),(15.097197,11.727657),(15.093063,11.722903),(15.086345,11.72218),(15.079834,11.722593),(15.076113,11.721456),(15.068258,11.700114),(15.066501,11.680632),(15.069395,11.660788),(15.085208,11.615881),(15.09575,11.598156),(15.123862,11.563171),(15.135644,11.530822),(15.122312,11.503227),(15.076113,11.453307),(15.067018,11.435556),(15.06061,11.416126),(15.054305,11.364088),(15.049448,11.348714),(15.049448,11.337268),(15.052548,11.329801),(15.057819,11.325356),(15.062264,11.320034),(15.06309,11.309931),(15.056373,11.29342),(15.033325,11.26027),(15.028261,11.244483),(15.021233,11.182549),(15.02883,11.080218),(15.035185,10.994627),(15.06247,10.930703),(15.072702,10.915769),(15.079007,10.898147),(15.075596,10.873498),(15.06309,10.830761),(15.065881,10.793115),(15.142259,10.647207),(15.149907,10.623125),(15.14846,10.607338),(15.138435,10.589691),(15.132337,10.565403),(15.131717,10.540701),(15.138228,10.521659),(15.144429,10.51631),(15.153628,10.511969),(15.164686,10.509075),(15.176055,10.50799),(15.186597,10.505871),(15.193315,10.501195),(15.198173,10.496492),(15.218326,10.487216),(15.227215,10.470137),(15.241167,10.432904),(15.255637,10.41704),(15.268763,10.406343),(15.278374,10.394276),(15.282199,10.374588),(15.284266,10.349292),(15.29057,10.328983),(15.301422,10.311749),(15.439192,10.185245),(15.476399,10.132742),(15.490661,10.124448),(15.490144,10.120107),(15.536137,10.080523),(15.602799,10.04099),(15.637939,10.029957),(15.681244,9.991278),(15.663777,9.98611),(15.439398,9.931695),(15.382968,9.930196),(15.214916,9.984095),(15.155488,9.986523),(15.109599,9.981537),(15.100711,9.978901),(15.089445,9.972183),(15.071669,9.955957),(15.06123,9.949239),(15.033015,9.942857),(15.002422,9.945053),(14.944131,9.958825),(14.898139,9.960478),(14.772566,9.921747),(14.732465,9.923814),(14.440493,9.995308),(14.196341,9.979147),(14.181697,9.978178),(14.173532,9.975025),(14.171155,9.96761),(14.170638,9.957843),(14.168365,9.94774),(14.119789,9.85201),(14.088473,9.809635),(14.006721,9.739277),(13.945949,9.652642),(13.947603,9.637759),(14.036486,9.568771),(14.321327,9.243158),(14.331145,9.200215),(14.349542,9.168356),(14.571699,8.99099),(14.793856,8.813623),(14.809876,8.808611),(14.819178,8.811608),(14.827963,8.813158),(14.836748,8.813003),(14.846049,8.810988),(14.859899,8.803314),(14.89938,8.774323),(14.908268,8.764841),(14.913332,8.752438),(14.919947,8.74771),(14.927285,8.745075),(14.934416,8.739003),(14.940101,8.729649),(14.949092,8.704276),(14.951469,8.692106),(14.951469,8.68283),(14.955087,8.676216),(14.968523,8.672159),(15.005833,8.66663),(15.031258,8.658749),(15.051928,8.643789),(15.068568,8.623867),(15.110736,8.555138),(15.16851,8.498604),(15.183703,8.479148),(15.210706,8.421822),(15.345347,8.13599),(15.406532,7.921585),(15.440742,7.839419),(15.487871,7.804951),(15.509368,7.804021),(15.540787,7.796838),(15.562905,7.792445),(15.562181,7.690178),(15.548952,7.630801),(15.521977,7.576076),(15.481049,7.523263),(15.473298,7.509103),(15.419451,7.43655),(15.415524,7.42084),(15.434954,7.402908),(15.432784,7.389731),(15.428443,7.388801),(15.409633,7.387664),(15.403225,7.386372),(15.397437,7.380739),(15.389169,7.364151),(15.385345,7.358363),(15.338216,7.32219),(15.261321,7.279247),(15.246564,7.266731),(15.223908,7.247517),(15.205097,7.200233),(15.191248,7.177341),(15.186597,7.166334),(15.185357,7.156877),(15.190214,7.098793),(15.187527,7.088612),(15.179052,7.07838),(15.156108,7.067787),(15.147013,7.061844),(15.136161,7.045308),(15.128926,7.026497),(15.113217,6.964692),(15.080247,6.878083),(15.05906,6.836276),(15.042213,6.790775),(15.034565,6.776771),(15.022369,6.766539),(14.962321,6.736696),(14.946095,6.72574),(14.932452,6.710289),(14.921874,6.686414),(14.807292,6.4278),(14.782177,6.393616),(14.780007,6.382221),(14.784348,6.358683),(14.784658,6.347805),(14.782384,6.336772),(14.772255,6.318323),(14.734842,6.270445),(14.719029,6.257862),(14.576402,6.189778),(14.534957,6.190088),(14.525966,6.174069),(14.510979,6.155336),(14.482144,6.127069),(14.467054,6.122599),(14.443903,6.100249),(14.430571,6.088079),(14.431811,6.084617),(14.432709,6.079123),(14.433568,6.073868),(14.429124,6.064179),(14.426023,6.059864),(14.422509,6.051363),(14.418169,6.047126),(14.412898,6.047022),(14.402459,6.054877),(14.397705,6.053947),(14.390263,6.045782),(14.387266,6.039038),(14.389436,6.031054),(14.397705,6.019169),(14.406076,6.01103),(14.417548,5.997258),(14.432328,5.98545),(14.446901,5.970024),(14.453825,5.953875),(14.458166,5.936409),(14.465711,5.920673),(14.482144,5.90964),(14.499197,5.909537),(14.530616,5.90592),(14.545396,5.907573),(14.572474,5.924187),(14.584773,5.923412),(14.598003,5.903491),(14.602964,5.883389),(14.617433,5.864966),(14.624047,5.758306),(14.631075,5.738049),(14.624358,5.723605),(14.610612,5.669138),(14.592835,5.62387),(14.590148,5.608289),(14.591491,5.590952),(14.596556,5.577232),(14.610612,5.553667),(14.617226,5.53788),(14.61888,5.525452),(14.617433,5.495299),(14.614126,5.48486),(14.599553,5.456645),(14.596246,5.440108),(14.597279,5.41148),(14.592835,5.400033),(14.579606,5.395382),(14.579296,5.387605),(14.570821,5.370939),(14.560796,5.355462),(14.555938,5.351328),(14.553974,5.338228),(14.548393,5.320503),(14.539608,5.303579),(14.528033,5.293011),(14.519868,5.290583),(14.523899,5.279679),(14.539712,5.261799),(14.560485,5.248595),(14.603584,5.234255),(14.624461,5.224049),(14.640481,5.208236),(14.651539,5.187514),(14.658877,5.164544),(14.663218,5.141832),(14.663012,5.091447),(14.667456,5.052768),(14.660324,5.015793),(14.660132,5.01144),(14.659498,4.997035),(14.663425,4.985717),(14.669833,4.977113),(14.674484,4.967579),(14.672107,4.940811),(14.675827,4.93076),(14.681615,4.920941),(14.686989,4.908875),(14.701459,4.814384),(14.700632,4.804282),(14.696084,4.779735),(14.691847,4.768702),(14.69071,4.762139),(14.697325,4.745215),(14.704249,4.669354),(14.717272,4.622019),(14.74373,4.57998),(14.779697,4.545047),(14.938344,4.435932),(14.982062,4.420481),(14.993947,4.413298),(15.000562,4.403195),(15.007383,4.37777),(15.013894,4.364257),(15.048518,4.321649),(15.063814,4.293434),(15.072806,4.265942),(15.082934,4.205507),(15.084484,4.148611),(15.091202,4.121791),(15.102571,4.112412),(15.117764,4.114324),(15.132854,4.108536),(15.139262,4.094583),(15.142569,4.08045),(15.14846,4.073938),(15.162413,4.073938),(15.177709,4.071458),(15.189801,4.063112),(15.192075,4.052234),(15.172438,4.043811),(15.150424,4.038514),(15.111253,4.019937),(15.091719,4.015854),(15.075183,4.017973),(15.043143,4.026138),(15.025987,4.026086),(15.084898,3.885836),(15.171301,3.758971),(15.25264,3.672852),(15.262306,3.662589),(15.329741,3.590996),(15.413973,3.50188),(15.517946,3.391603),(15.616855,3.286777),(15.7107,3.187455),(15.78563,3.107977),(15.793175,3.103274),(15.80103,3.100897),(15.813536,3.098985),(15.817566,3.100122),(15.829142,3.106633),(15.834206,3.108545),(15.840201,3.107408),(15.850949,3.100432),(15.854773,3.098985),(15.897768,3.103533),(15.913995,3.097952),(15.933425,3.081777),(15.950065,3.061003),(15.95823,3.041521),(15.971562,2.991188),(15.974249,2.985762),(15.977763,2.980956),(15.982001,2.976719),(15.995437,2.981473),(16.006082,2.983953),(16.015797,2.983488),(16.026236,2.978992),(16.034194,2.970621),(16.055381,2.93977),(16.070161,2.906594),(16.08246,2.885665),(16.092382,2.863289),(16.095172,2.836779),(16.086904,2.813163),(16.059929,2.786188),(16.056208,2.759833),(16.060032,2.72583),(16.059309,2.711825),(16.062616,2.703195),(16.072848,2.702834),(16.086284,2.704952),(16.0991,2.70397),(16.110468,2.701438),(16.101373,2.687847),(16.093725,2.67312),(16.090315,2.661906),(16.093105,2.641494),(16.108505,2.606612),(16.111502,2.5862),(16.106851,2.567028),(16.091245,2.532766),(16.090315,2.510494),(16.094345,2.497523),(16.107058,2.474165),(16.111502,2.455872),(16.111502,2.418562),(16.112949,2.410448),(16.120907,2.389571),(16.133413,2.364818),(16.152326,2.343269),(16.159251,2.332934),(16.162868,2.323839),(16.164729,2.314795),(16.165969,2.294796),(16.169069,2.289887),(16.182919,2.281154),(16.186536,2.277692),(16.18695,2.26932),(16.181265,2.256556),(16.179715,2.246996),(16.192841,2.239503),(16.196665,2.236454),(16.202349,2.23201),(16.207723,2.22307),(16.199455,2.212166),(16.159251,2.18209),(16.140647,2.199815),(16.12039,2.203329),(16.104371,2.193562),(16.097756,2.171548),(16.085767,2.149172),(16.083493,2.137752),(16.08556,2.12491),(16.089798,2.116151),(16.092485,2.106901),(16.090315,2.092742),(16.088144,2.09176),(16.084113,2.091605),(16.079669,2.090261),(16.076672,2.08592),(16.075535,2.079952),(16.075535,2.075456),(16.076155,2.07282),(16.076672,2.072252),(16.078119,2.063441),(16.08153,2.055638),(16.081736,2.046827),(16.06644,2.025485),(16.063753,2.015873),(16.06489,1.97895),(16.068714,1.973602),(16.074915,1.970604),(16.083493,1.963008),(16.101373,1.932803),(16.135066,1.845315),(16.159044,1.734805),(16.15584,1.719069),(16.145092,1.714315),(16.123594,1.721601),(16.114809,1.719069),(16.069851,1.654551),(16.030473,1.705401),(16.021998,1.72279),(16.021998,1.733048),(16.024892,1.742479),(16.026339,1.752297),(16.021998,1.763795),(16.013833,1.771314),(16.002775,1.775913),(15.990786,1.776172),(15.98045,1.770591),(15.963191,1.77736),(15.94376,1.781624),(15.928981,1.788962),(15.92588,1.804749),(15.915235,1.798961),(15.901799,1.793742),(15.889913,1.793277),(15.884952,1.801648),(15.882369,1.816712),(15.875651,1.823766),(15.865936,1.828468),(15.842061,1.844152),(15.815396,1.853428),(15.80289,1.859965),(15.76465,1.908722),(15.757415,1.912985),(15.734057,1.916706),(15.721552,1.921615),(15.711113,1.927377),(15.706772,1.931976),(15.611791,1.942544),(15.526628,1.967814),(15.48291,1.97585),(15.439192,1.969829),(15.388135,1.939547),(15.367981,1.932855),(15.349171,1.923295),(15.337389,1.92146),(15.337802,1.92376),(15.314341,1.93425),(15.309484,1.935077),(15.301629,1.943578),(15.300182,1.94766),(15.301939,1.953319),(15.303283,1.966419),(15.301112,1.973757),(15.295428,1.981534),(15.287573,1.987787),(15.25357,2.000318),(15.248092,2.003341),(15.244165,2.010059),(15.238687,2.025304),(15.233726,2.031298),(15.212642,2.039618),(15.197036,2.034967),(15.1836,2.029335),(15.151664,2.040988),(15.138538,2.011506),(15.123862,2.01701),(15.117764,2.01701),(15.107636,2.000835),(15.091616,1.984428),(15.074873,1.979364),(15.06309,1.997166),(15.04366,1.990862),(15.028261,1.995306),(14.980718,2.033107),(14.971003,2.034451),(14.963045,2.010421),(14.953846,2.005822),(14.932142,2.003341),(14.892455,2.00639),(14.896589,2.030937),(14.907338,2.058894),(14.887804,2.072252),(14.8846,2.082096),(14.871474,2.101656),(14.857005,2.115944),(14.850184,2.109795),(14.850184,2.097367),(14.84822,2.085197),(14.841192,2.075921),(14.826309,2.072252),(14.764814,2.065456),(14.76161,2.070185),(14.76254,2.092535),(14.761403,2.099589),(14.756339,2.099589),(14.747554,2.097186),(14.737116,2.096075),(14.727297,2.099589),(14.733498,2.102018),(14.747864,2.113205),(14.720889,2.127597),(14.713654,2.12398),(14.720476,2.099589),(14.705696,2.105532),(14.685956,2.125866),(14.672003,2.134341),(14.646062,2.133359),(14.636346,2.13832),(14.644821,2.154185),(14.620947,2.16483),(14.587874,2.201727),(14.562139,2.208807),(14.557075,2.202399),(14.547773,2.193666),(14.54798,2.184364),(14.550254,2.175166),(14.54705,2.166536),(14.538058,2.163383),(14.4592,2.146537),(14.438736,2.133979),(14.432948,2.132119),(14.418685,2.135271),(14.399358,2.153151),(14.383649,2.162091),(14.320293,2.17067),(14.276265,2.153565),(14.266963,2.152428),(14.16175,2.15341),(14.002793,2.154805),(13.823993,2.156407),(13.662266,2.157761),(13.620336,2.158112),(13.466547,2.159508),(13.294568,2.161058),(13.294258,2.185759),(13.298392,2.215835),(13.294155,2.230511),(13.284646,2.242242),(13.26966,2.255729),(13.254157,2.266891),(13.242891,2.27149),(13.235967,2.269888),(13.221601,2.263946),(13.212299,2.264049),(13.205684,2.266891),(13.191732,2.277692),(13.163827,2.283169),(13.133337,2.284513),(13.124656,2.281051),(13.11308,2.272731),(13.09303,2.253455),(13.083521,2.249528),(13.04094,2.244205),(13.012931,2.256298),(12.996912,2.260018),(12.983166,2.253455),(12.97221,2.250355),(12.954847,2.253249),(12.924771,2.264049),(12.914539,2.254541),(12.903171,2.250355),(12.890458,2.251388),(12.876402,2.257176),(12.867824,2.24498),(12.848084,2.249321),(12.825243,2.259243),(12.808189,2.264049),(12.79558,2.258778),(12.784108,2.249528),(12.770155,2.240691),(12.749898,2.236712),(12.614609,2.256918),(12.590735,2.265703),(12.574819,2.277692),(12.551461,2.272266),(12.534201,2.277692),(12.47839,2.295882),(12.439943,2.298517),(12.384856,2.285908),(12.366769,2.289577),(12.334006,2.309576),(12.321604,2.314072),(12.312096,2.312935),(12.291218,2.301773),(12.28078,2.299034),(12.261039,2.296192),(12.222592,2.283324),(12.202645,2.279552),(12.159237,2.281412),(12.118826,2.28782),(12.100739,2.288492),(12.041931,2.283996),(11.966174,2.28875),(11.889796,2.282084),(11.802153,2.284823),(11.768149,2.280172),(11.75275,2.281877),(11.701384,2.298724),(11.681023,2.315415),(11.671204,2.320996),(11.657252,2.322495),(11.644643,2.319653),(11.618598,2.310403),(11.351637,2.300584),(11.349777,2.291489),(11.354325,2.278467),(11.355978,2.265186),(11.349984,2.25113),(11.342232,2.240588),(11.335928,2.229167),(11.334171,2.212786),(11.334171,2.20028),(11.332414,2.188033),(11.328486,2.176302),(11.322078,2.16576),(11.022975,2.165709),(10.723769,2.165657),(10.424666,2.165657),(10.409686,2.165654),(10.189983,2.165616),(10.125562,2.165605),(9.990997,2.165605),(9.97074,2.168913),(9.907488,2.200487),(9.890125,2.204569),(9.872141,2.211236),(9.84682,2.228547),(9.823669,2.249011),(9.810853,2.264876),(9.808682,2.285495),(9.811886,2.306165),(9.81106,2.324872),(9.799571,2.341742),(9.812185,2.353583),(9.812185,2.359809),(9.819184,2.376939),(9.822927,2.562934),(9.853201,2.696967),(9.870453,2.738105),(9.874278,2.754543),(9.877778,2.881415),(9.88795,2.926907),(9.903331,2.968329),(9.922048,2.99726),(9.94337,3.023139),(9.95574,3.050482),(9.958018,3.082221),(9.919688,3.228095),(9.90211,3.251695),(9.90211,3.257961),(9.911632,3.257717),(9.917979,3.259467),(9.929454,3.264797),(9.929454,3.271552),(9.91391,3.270331),(9.903087,3.272895),(9.89503,3.278306),(9.88795,3.285224),(9.881521,3.294338),(9.855317,3.343573),(9.780935,3.436754),(9.739024,3.47309),(9.6421,3.539049),(9.664236,3.543931),(9.68865,3.556098),(9.770763,3.61872),(9.792817,3.627183),(9.819591,3.627875),(9.787608,3.637112),(9.759613,3.626166),(9.733653,3.60932),(9.706879,3.600531),(9.638438,3.594387),(9.626231,3.600002),(9.621349,3.612779),(9.625987,3.626451),(9.6421,3.63467),(9.612559,3.704983),(9.565115,3.770901),(9.551524,3.79621),(9.545909,3.819648),(9.570974,3.795478),(9.576671,3.792304),(9.581554,3.787828),(9.58961,3.766099),(9.594249,3.758205),(9.606944,3.752265),(9.624278,3.75019),(9.662608,3.750718),(9.652517,3.756049),(9.621104,3.76496),(9.611013,3.770575),(9.607107,3.774726),(9.604259,3.779975),(9.597423,3.788886),(9.589203,3.809638),(9.592133,3.83515),(9.60377,3.859036),(9.621104,3.874823),(9.659028,3.84101),(9.679698,3.836086),(9.689789,3.860582),(9.723481,3.843817),(9.736339,3.834174),(9.744477,3.819648),(9.750336,3.830268),(9.750499,3.840766),(9.74586,3.850898),(9.738292,3.860582),(9.741466,3.861029),(9.751964,3.860582),(9.704926,3.877509),(9.682872,3.890448),(9.67628,3.909003),(9.68922,3.922431),(9.715017,3.931342),(9.743337,3.93594),(9.764415,3.936347),(9.764334,3.940741),(9.76295,3.943793),(9.758149,3.950019),(9.760753,3.951361),(9.76295,3.951239),(9.764415,3.952338),(9.764415,3.957465),(9.664806,3.940375),(9.634613,3.943183),(9.618907,3.960354),(9.623709,3.985785),(9.639903,4.008775),(9.65919,4.018866),(9.675629,4.03384),(9.718923,4.099555),(9.748302,4.114447),(9.752452,4.119818),(9.756847,4.129828),(9.754568,4.135688),(9.738292,4.12873),(9.719493,4.114936),(9.710948,4.105699),(9.703461,4.09394),(9.696625,4.09394),(9.696625,4.114447),(9.678884,4.10102),(9.646332,4.051459),(9.628429,4.031928),(9.606619,4.020657),(9.579356,4.013007),(9.559337,4.016791),(9.559581,4.039374),(9.545909,4.025702),(9.526622,4.035793),(9.508556,4.054185),(9.498546,4.077216),(9.504242,4.100775),(9.490082,4.115871),(9.478526,4.111396),(9.450369,4.080308),(9.452891,4.076972),(9.457286,4.071112),(9.46518,4.068101),(9.474294,4.066148),(9.484386,4.059882),(9.485362,4.055325),(9.495616,4.03148),(9.507009,4.019477),(9.523611,4.00609),(9.532481,3.989447),(9.518565,3.970445),(9.504242,3.976142),(9.497081,3.977688),(9.490082,3.980658),(9.476899,3.990912),(9.464041,4.01203),(9.458344,4.016181),(9.453868,4.018622),(9.451182,4.022773),(9.450369,4.031928),(9.437999,4.030463),(9.42921,4.026557),(9.424083,4.019477),(9.422374,4.008368),(9.449392,3.992133),(9.456554,3.984076),(9.458832,3.972357),(9.456391,3.963528),(9.452403,3.953925),(9.450369,3.939765),(9.453298,3.932115),(9.469086,3.923407),(9.476899,3.915839),(9.464041,3.90998),(9.448253,3.905748),(9.41212,3.902167),(9.393403,3.904283),(9.359223,3.913723),(9.343516,3.915839),(9.313731,3.930854),(9.302582,3.964993),(9.307628,4.001939),(9.326182,4.025702),(9.326182,4.031928),(9.301443,4.026353),(9.290294,4.009711),(9.284353,3.992255),(9.250336,3.960842),(9.220714,3.96601),(9.210704,3.96426),(9.211111,3.982571),(9.213227,3.996568),(9.210623,4.008124),(9.196462,4.018866),(9.184744,4.020901),(9.166515,4.020901),(9.148774,4.019029),(9.138682,4.015448),(9.123383,4.01789),(9.07016,4.051418),(8.983409,4.092434),(8.971202,4.100775),(8.969412,4.128974),(8.988455,4.18891),(8.990977,4.217475),(8.960297,4.245917),(8.93629,4.279527),(8.918956,4.318061),(8.894786,4.460517),(8.900076,4.485093),(8.926036,4.529608),(8.93572,4.55329),(8.913585,4.528632),(8.900564,4.521633),(8.894786,4.535875),(8.897227,4.548),(8.901541,4.560248),(8.904307,4.573188),(8.901622,4.587388),(8.893809,4.594631),(8.871918,4.602484),(8.864106,4.61872),(8.856293,4.62995),(8.847016,4.639716),(8.83961,4.642646),(8.835704,4.629625),(8.86671,4.582709),(8.873709,4.560045),(8.867849,4.544582),(8.858735,4.540839),(8.850597,4.547431),(8.847016,4.563218),(8.798595,4.601142),(8.798025,4.588284),(8.79949,4.577786),(8.80421,4.570136),(8.812836,4.566352),(8.799571,4.542182),(8.795177,4.539008),(8.788422,4.541449),(8.768565,4.553127),(8.764415,4.556627),(8.757009,4.56509),(8.740571,4.571234),(8.724132,4.580471),(8.716563,4.597724),(8.705903,4.63467),(8.661306,4.698798),(8.655121,4.738267),(8.638682,4.71308),(8.645356,4.684068),(8.660492,4.653998),(8.68572,4.561225),(8.694835,4.55329),(8.706309,4.547675),(8.716563,4.539008),(8.722667,4.513129),(8.701671,4.501288),(8.576671,4.491889),(8.56544,4.49726),(8.570323,4.50967),(8.582693,4.523342),(8.593761,4.532782),(8.581716,4.538479),(8.570567,4.53856),(8.562673,4.533189),(8.556407,4.511705),(8.54835,4.507799),(8.537364,4.508531),(8.525564,4.511705),(8.510265,4.527045),(8.505056,4.55093),(8.509939,4.568834),(8.525564,4.566352),(8.530772,4.57925),(8.552094,4.614732),(8.538341,4.604641),(8.525727,4.604438),(8.516124,4.612982),(8.511241,4.629055),(8.513682,4.64232),(8.521169,4.655951),(8.538585,4.676215),(8.567638,4.69599),(8.573253,4.704088),(8.573578,4.716376),(8.569672,4.728339),(8.567638,4.74022),(8.573253,4.752509),(8.580089,4.745063),(8.576508,4.784654),(8.57838,4.804674),(8.590017,4.813381),(8.594168,4.815294),(8.594179,4.815299),(8.594186,4.815299),(8.595113,4.815314),(8.612166,4.831877),(8.608446,4.846217),(8.605448,4.86004),(8.602245,4.881331),(8.61103,4.896653),(8.638315,4.916781),(8.644412,4.934868),(8.683687,4.99688),(8.683687,4.997035),(8.698569,5.017809),(8.707871,5.037549),(8.722651,5.08119),(8.728852,5.094548),(8.748282,5.12421),(8.758307,5.134881),(8.769469,5.139946),(8.781252,5.141987),(8.79169,5.146741),(8.798925,5.159635),(8.825693,5.295492),(8.82466,5.306137),(8.816392,5.330296),(8.815255,5.343499),(8.821663,5.367684),(8.84316,5.409981),(8.849775,5.432357),(8.849258,5.453467),(8.845744,5.472794),(8.844297,5.492353),(8.850498,5.514006),(8.865484,5.533462),(8.884294,5.551445),(8.899487,5.570953),(8.903932,5.595086),(8.8937,5.621751),(8.874889,5.64144),(8.852565,5.658674),(8.832411,5.678156),(8.819286,5.703451),(8.826934,5.717817),(8.844607,5.730866),(8.862177,5.752337),(8.866621,5.777452),(8.857836,5.794893),(8.847087,5.810758),(8.845227,5.831041),(8.855872,5.847499),(8.891219,5.871116),(8.904655,5.888117),(8.921708,5.904473),(8.943206,5.901424),(8.965013,5.893595),(8.982686,5.895765),(8.986304,5.908814),(8.988474,5.937287),(8.993849,5.944083),(9.009868,5.952351),(9.022477,5.965813),(9.043871,5.996689),(9.043871,5.996844),(9.136165,6.092523),(9.273315,6.202775),(9.298843,6.238432),(9.327368,6.298506),(9.341011,6.313983),(9.354757,6.321502),(9.366539,6.32269),(9.379148,6.321605),(9.394961,6.322535),(9.422866,6.339356),(9.453149,6.397673),(9.482604,6.415992),(9.48859,6.418208),(9.528596,6.433019),(9.548957,6.443174),(9.56477,6.458599),(9.588127,6.502731),(9.60301,6.515185),(9.632673,6.521153),(9.644806,6.521226),(9.680525,6.521438),(9.693031,6.531308),(9.69398,6.534305),(9.769822,6.773722),(9.788529,6.794754),(9.8123,6.793049),(9.83173,6.783566),(9.851161,6.777882),(9.874622,6.787701),(10.119465,6.994406),(10.142926,7.007687),(10.156878,7.004742),(10.165043,6.988515),(10.179409,6.914928),(10.189848,6.895446),(10.210828,6.879168),(10.238424,6.871003),(10.495979,6.874672),(10.496599,6.897151),(10.500733,6.912809),(10.508898,6.926348),(10.521197,6.942781),(10.527502,6.947484),(10.533186,6.948828),(10.537424,6.952135),(10.543315,7.028719),(10.55179,7.06634),(10.564192,7.102668),(10.578661,7.130832),(10.597161,7.107268),(10.602536,7.058072),(10.620623,7.043757),(10.646978,7.035076),(10.716947,6.998127),(10.799113,6.967328),(10.831876,6.94521),(10.848206,6.905316),(10.857404,6.859789),(10.877454,6.816097),(10.909391,6.784264),(11.002615,6.767262),(11.036721,6.740546),(11.055945,6.700367),(11.059045,6.652722),(11.057392,6.613008),(11.060596,6.573967),(11.077442,6.496736),(11.097079,6.449117),(11.113512,6.434001),(11.140384,6.43028),(11.202276,6.436739),(11.231748,6.439815),(11.237122,6.438058),(11.248285,6.431004),(11.255519,6.42997),(11.261514,6.432554),(11.272159,6.442037),(11.27588,6.444207),(11.286835,6.443225),(11.308539,6.437903),(11.319495,6.437283),(11.344196,6.443122),(11.369828,6.455757),(11.389361,6.473921),(11.395666,6.496736),(11.401764,6.538181),(11.41675,6.572003),(11.443105,6.593268),(11.482585,6.597169),(11.509767,6.612311),(11.533435,6.645099),(11.562374,6.712485),(11.568885,6.739099),(11.571469,6.766229),(11.566611,6.784187),(11.544597,6.80894),(11.538086,6.8239),(11.543667,6.850694),(11.561754,6.876067),(11.623249,6.932291),(11.681126,6.96862),(11.684847,6.973839),(11.688464,6.986913),(11.692495,6.991616),(11.697249,6.992856),(11.707481,6.991822),(11.718643,6.994458),(11.73425,6.995078),(11.741071,6.997507),(11.746859,7.005362),(11.748616,7.014973),(11.75151,7.023965),(11.766392,7.033525),(11.80887,7.071921),(11.820343,7.078794),(11.833985,7.08298),(11.845147,7.081946),(11.854139,7.077812),(11.862614,7.075435),(11.872226,7.079362),(11.881321,7.102048),(11.868918,7.127111),(11.829954,7.169383),(11.780242,7.241058),(11.766186,7.252943),(11.742415,7.257439),(11.736627,7.263279),(11.744585,7.272167),(11.762465,7.285551),(11.782722,7.304517),(11.844734,7.396397),(11.903545,7.453557),(11.966174,7.514426),(11.992839,7.55401),(12.008548,7.564862),(12.021054,7.576231),(12.024981,7.596592),(12.010719,7.655244),(12.009065,7.676328),(12.012166,7.698343),(12.018987,7.718858),(12.054954,7.783919),(12.154276,7.921326),(12.192103,7.960652),(12.197787,7.975173),(12.1951,7.99171),(12.188175,8.016256),(12.182698,8.054652),(12.182594,8.093486),(12.188279,8.117774),(12.229103,8.175497),(12.233713,8.188455),(12.236751,8.196994),(12.237682,8.21689),(12.221765,8.29766),(12.219285,8.343161),(12.227036,8.386311),(12.249981,8.418764),(12.271788,8.427781),(12.310235,8.421348),(12.331319,8.424293),(12.343205,8.432742),(12.351163,8.44424),(12.357571,8.456358),(12.365116,8.466564),(12.395191,8.487338),(12.403873,8.496537),(12.403253,8.524235),(12.400462,8.538421),(12.377001,8.59511),(12.368836,8.609579),(12.396328,8.600768),(12.410591,8.598494),(12.42444,8.599786),(12.440253,8.606608),(12.451829,8.614695),(12.464334,8.621154),(12.482524,8.623247),(12.495547,8.619553),(12.527793,8.604851),(12.542572,8.602008),(12.557972,8.605677),(12.660395,8.659318),(12.678171,8.678696),(12.692848,8.710891),(12.701633,8.739158),(12.708867,8.750914),(12.720236,8.756469),(12.734499,8.755901),(12.747211,8.753627),(12.759613,8.75342),(12.772946,8.759053),(12.794547,8.788612),(12.805709,8.831503),(12.82824,9.019967),(12.821005,9.056425),(12.822142,9.09681),(12.842192,9.14443),(12.888391,9.226725),(12.894592,9.243959),(12.898416,9.262433),(12.899347,9.302017),(12.891285,9.33199),(12.88157,9.352221),(12.853148,9.356639),(12.848394,9.359947),(12.855422,9.378111),(12.862036,9.382891),(12.920741,9.410486),(12.929009,9.416119),(12.940068,9.4271),(12.969006,9.464075),(12.982442,9.474591),(13.051689,9.504511),(13.13034,9.516526),(13.195453,9.542209),(13.223461,9.613135),(13.228422,9.677912),(13.241341,9.745324),(13.251056,9.76925),(13.260461,9.785709),(13.262942,9.801728),(13.251987,9.824234),(13.222944,9.854128),(13.212712,9.870148),(13.211059,9.892756),(13.218707,9.913039),(13.242271,9.949756),(13.246405,9.973475),(13.242478,9.992673),(13.230489,10.025358),(13.230179,10.045331),(13.247956,10.079412),(13.281339,10.092098),(13.359784,10.100547),(13.378387,10.108609),(13.392672,10.116957),(13.401952,10.122381),(13.423242,10.138478),(13.434921,10.153412),(13.450837,10.187157),(13.453008,10.200515),(13.451354,10.212117),(13.444533,10.232477),(13.444843,10.245474),(13.467167,10.307589),(13.501274,10.496337),(13.538791,10.62142),(13.566283,10.679013),(13.744411,10.930497),(13.749839,10.942442),(13.75702,10.958247),(13.750406,10.996203),(13.75485,11.016719),(13.765909,11.033901),(13.782032,11.047518),(13.820066,11.069274),(13.832054,11.084803),(13.849211,11.124568),(13.873189,11.166503),(13.906262,11.208025),(13.943882,11.245723),(13.98233,11.276264),(14.013129,11.276367),(14.052506,11.265153),(14.122579,11.235724),(14.14325,11.232726),(14.165471,11.238333),(14.186762,11.249237),(14.204642,11.26213),(14.232133,11.290294),(14.244329,11.295436),(14.271304,11.29864),(14.279159,11.301844),(14.368249,11.383001),(14.387059,11.393388),(14.446281,11.411682),(14.467261,11.423852),(14.503951,11.456304),(14.524932,11.468836),(14.576609,11.487336),(14.593765,11.496431),(14.605754,11.514725),(14.616193,11.538677),(14.623841,11.563481),(14.627561,11.584359),(14.627148,11.60968),(14.62136,11.629782),(14.610198,11.646835),(14.593868,11.66301),(14.588597,11.669676),(14.585083,11.676808),(14.580949,11.683267),(14.573508,11.687711),(14.556455,11.687505),(14.552424,11.689003),(14.543329,11.701096),(14.541882,11.709829),(14.546533,11.718614),(14.583223,11.766363),(14.590665,11.783158),(14.612059,11.88496),(14.623427,11.916535),(14.627148,11.950073),(14.626941,11.957204),(14.625184,11.968108),(14.606271,12.011258),(14.604307,12.03043),(14.622187,12.042005),(14.619913,12.052495),(14.637173,12.124222),(14.643168,12.13585),(14.648129,12.141844),(14.650609,12.148097),(14.669936,12.167424),(14.668283,12.178069),(14.663425,12.187578),(14.651539,12.193779),(14.643168,12.190627),(14.629318,12.183495),(14.616296,12.178948),(14.610612,12.183547),(14.6041,12.204321),(14.588287,12.217188),(14.568134,12.22773),(14.548497,12.241579),(14.558315,12.246902),(14.562139,12.248401),(14.562139,12.255842),(14.551184,12.260441),(14.521521,12.282352),(14.51439,12.28938),(14.512426,12.29863),(14.517594,12.313358),(14.51439,12.323538),(14.487105,12.337801),(14.42313,12.353149),(14.217561,12.359298),(14.202161,12.362502),(14.188932,12.370409),(14.179837,12.385602),(14.179113,12.397281),(14.185108,12.427873),(14.185418,12.447045),(14.180216,12.46353),(14.1787,12.468336),(14.177356,12.545592),(14.121132,12.81179),(14.064908,13.077988),(14.418169,13.081141),(14.435429,13.070289),(14.481041,13.000508),(14.482144,12.99882),(14.496303,12.983808),(14.504675,12.969003),(14.507672,12.95244),(14.506122,12.93208),(14.490619,12.886346),(14.490102,12.873608),(14.500437,12.859113),(14.53103,12.836246),(14.549013,12.818211),(14.549013,12.780461),(14.55108,12.771573),(14.554801,12.766896),(14.560692,12.766224)] +Republic of Congo [(17.627275,3.626317),(17.70913,3.626421),(17.714505,3.627506),(17.72608,3.631847),(17.733212,3.632208),(17.73993,3.629159),(17.749645,3.618075),(17.755949,3.614509),(17.765148,3.614354),(17.784475,3.61823),(17.793156,3.617506),(17.808349,3.608592),(17.821165,3.593994),(17.83057,3.575674),(17.835324,3.555831),(17.842146,3.540147),(17.856408,3.537305),(17.875322,3.542731),(17.913562,3.561334),(17.919247,3.565597),(17.924518,3.563453),(17.959865,3.541077),(17.969476,3.537511),(17.981879,3.538803),(18.013091,3.55397),(18.047094,3.563143),(18.082028,3.563453),(18.116444,3.552007),(18.134014,3.534824),(18.146727,3.494904),(18.159852,3.480305),(18.181556,3.477386),(18.202537,3.487333),(18.220004,3.503947),(18.231579,3.521156),(18.235403,3.530923),(18.242535,3.561127),(18.248529,3.574899),(18.255867,3.577225),(18.264859,3.574744),(18.275918,3.574382),(18.355603,3.581462),(18.361597,3.58017),(18.372863,3.574537),(18.379891,3.574382),(18.383715,3.578),(18.387849,3.591668),(18.392293,3.595854),(18.403662,3.597353),(18.420612,3.596371),(18.429707,3.600221),(18.448,3.618023),(18.459059,3.631382),(18.472081,3.632053),(18.497093,3.61146),(18.513422,3.592237),(18.55435,3.526375),(18.570267,3.489607),(18.582462,3.477386),(18.626387,3.476869),(18.634552,3.449222),(18.642407,3.323829),(18.631141,3.17898),(18.62184,3.156656),(18.619669,3.14477),(18.613055,3.128182),(18.597345,3.114643),(18.562102,3.096815),(18.541431,3.08281),(18.528409,3.064775),(18.495336,2.966073),(18.479626,2.941372),(18.475802,2.931192),(18.471771,2.909229),(18.467327,2.898894),(18.44924,2.873004),(18.425159,2.822154),(18.421852,2.808822),(18.418338,2.781743),(18.393223,2.713686),(18.316742,2.584339),(18.313745,2.575968),(18.303513,2.572557),(18.246152,2.518917),(18.233026,2.502587),(18.225171,2.486464),(18.214113,2.427657),(18.207808,2.407968),(18.199126,2.389106),(18.187964,2.370916),(18.174012,2.336138),(18.16378,2.321048),(18.146727,2.315932),(18.142696,2.3057),(18.118615,2.276968),(18.111173,2.265289),(18.072416,2.160024),(18.072416,2.145193),(18.078307,2.114807),(18.080787,2.084163),(18.076033,1.988484),(18.081718,1.942415),(18.081511,1.92655),(18.078307,1.90929),(18.068178,1.876217),(18.066111,1.858983),(18.073656,1.572334),(18.071382,1.551457),(18.012264,1.421956),(18.006683,1.402396),(18.002033,1.365783),(17.966996,1.253723),(17.963379,1.191453),(17.956247,1.171661),(17.914803,1.102001),(17.892478,1.077403),(17.873255,1.039628),(17.86664,1.016632),(17.878216,0.947178),(17.877906,0.867907),(17.907671,0.740473),(17.907568,0.713084),(17.892582,0.63588),(17.893202,0.607664),(17.8993,0.583273),(17.91842,0.535214),(17.933096,0.527049),(17.943638,0.510823),(17.949943,0.491496),(17.95201,0.474339),(17.945498,0.377291),(17.939917,0.361271),(17.854031,0.241072),(17.77538,0.026408),(17.757086,-0.009456),(17.716365,-0.195801),(17.713678,-0.220916),(17.716262,-0.246237),(17.740756,-0.346179),(17.749541,-0.523429),(17.741997,-0.533351),(17.728044,-0.553092),(17.664172,-0.612726),(17.658694,-0.619754),(17.654457,-0.627506),(17.652493,-0.642182),(17.650943,-0.646316),(17.647636,-0.649933),(17.635853,-0.658512),(17.623038,-0.673291),(17.60216,-0.688691),(17.545863,-0.756491),(17.411474,-0.918341),(17.385429,-0.942422),(17.364965,-0.967227),(17.357317,-0.973531),(17.330446,-0.986554),(17.327655,-0.991721),(17.308018,-1.014976),(17.297683,-1.021074),(17.245386,-1.035853),(17.194847,-1.042984),(17.175416,-1.048255),(17.10865,-1.075644),(17.060488,-1.089183),(17.028655,-1.094454),(17.022557,-1.097038),(17.012015,-1.104273),(17.005918,-1.106753),(17.003127,-1.112851),(16.979563,-1.140239),(16.940495,-1.174759),(16.909179,-1.211139),(16.862567,-1.248863),(16.839726,-1.262506),(16.833215,-1.269017),(16.828667,-1.276252),(16.817815,-1.304157),(16.794664,-1.345498),(16.779265,-1.382395),(16.762728,-1.405856),(16.725521,-1.47996),(16.692552,-1.522955),(16.669504,-1.56936),(16.655971,-1.59029),(16.636224,-1.62083),(16.628886,-1.637263),(16.623822,-1.6538),(16.622478,-1.665169),(16.623099,-1.699172),(16.617207,-1.728731),(16.604185,-1.754052),(16.524971,-1.858535),(16.522536,-1.861746),(16.517162,-1.873735),(16.513958,-1.887377),(16.507757,-1.890271),(16.49029,-1.903604),(16.474787,-1.910115),(16.470033,-1.912699),(16.460938,-1.920864),(16.445642,-1.940501),(16.382596,-2.002202),(16.340222,-2.034138),(16.284411,-2.087572),(16.256093,-2.106795),(16.231288,-2.129016),(16.206793,-2.160952),(16.194598,-2.182036),(16.187363,-2.204567),(16.188603,-2.227718),(16.19005,-2.23547),(16.189947,-2.241878),(16.176924,-2.286836),(16.174857,-2.303062),(16.178268,-2.318462),(16.187466,-2.332311),(16.189843,-2.337479),(16.191704,-2.346884),(16.192324,-2.39515),(16.225603,-2.635239),(16.204519,-2.782723),(16.180852,-2.85414),(16.178165,-2.868093),(16.177028,-2.882459),(16.183981,-2.940449),(16.188293,-2.976406),(16.184469,-3.026119),(16.189843,-3.082343),(16.185709,-3.136397),(16.189947,-3.153036),(16.199248,-3.171847),(16.199455,-3.194791),(16.193357,-3.238406),(16.198628,-3.25949),(16.210411,-3.271065),(16.222089,-3.279127),(16.227464,-3.289876),(16.225603,-3.313233),(16.220746,-3.331733),(16.207723,-3.361913),(16.158424,-3.447282),(16.137857,-3.500715),(16.115533,-3.544227),(16.053314,-3.619985),(16.012697,-3.702254),(16.005255,-3.724784),(15.991096,-3.738944),(15.974249,-3.786486),(15.94314,-3.840746),(15.936112,-3.848498),(15.930324,-3.858626),(15.912134,-3.917434),(15.888019,-3.940523),(15.882989,-3.945339),(15.844231,-3.965597),(15.799996,-3.978826),(15.755141,-3.985647),(15.711216,-3.992158),(15.566832,-4.038047),(15.549056,-4.046212),(15.533759,-4.057994),(15.52115,-4.076804),(15.490351,-4.175196),(15.484977,-4.183878),(15.462859,-4.207132),(15.458932,-4.21385),(15.45025,-4.217261),(15.425446,-4.23018),(15.386585,-4.244339),(15.275274,-4.300357),(15.25574,-4.313069),(15.228352,-4.324851),(15.20396,-4.339011),(15.193418,-4.358751),(15.188044,-4.372187),(15.150678,-4.425151),(15.09792,-4.499931),(15.076113,-4.520085),(15.022783,-4.54706),(15.014618,-4.558015),(15.00821,-4.571038),(14.931005,-4.646899),(14.849047,-4.783738),(14.841399,-4.800171),(14.831167,-4.815054),(14.815457,-4.827973),(14.768431,-4.84823),(14.702182,-4.889571),(14.672727,-4.899907),(14.634486,-4.903627),(14.623324,-4.899493),(14.592111,-4.88058),(14.579606,-4.876342),(14.575058,-4.873655),(14.566893,-4.860529),(14.562139,-4.855878),(14.553871,-4.855155),(14.532477,-4.856188),(14.528033,-4.852157),(14.519868,-4.841305),(14.501471,-4.840685),(14.481937,-4.845853),(14.470052,-4.852157),(14.449484,-4.873965),(14.437289,-4.880166),(14.418169,-4.883163),(14.400702,-4.886264),(14.396464,-4.855878),(14.412588,-4.778157),(14.408763,-4.767822),(14.403699,-4.75883),(14.399668,-4.749321),(14.399048,-4.737539),(14.401529,-4.728134),(14.409177,-4.710564),(14.410934,-4.701056),(14.406903,-4.676044),(14.395844,-4.660128),(14.380962,-4.646175),(14.365769,-4.627262),(14.3575,-4.589124),(14.349956,-4.566904),(14.350783,-4.559049),(14.358947,-4.544579),(14.37476,-4.530213),(14.412588,-4.514504),(14.424576,-4.502308),(14.435739,-4.486702),(14.452482,-4.467995),(14.466538,-4.448048),(14.469742,-4.429341),(14.46323,-4.422623),(14.428607,-4.400402),(14.409074,-4.377561),(14.392744,-4.352756),(14.386026,-4.338287),(14.385716,-4.332913),(14.389126,-4.327538),(14.393467,-4.313172),(14.395224,-4.300357),(14.395328,-4.289711),(14.392847,-4.281753),(14.387059,-4.277102),(14.368559,-4.278446),(14.325461,-4.301803),(14.279986,-4.312759),(14.259625,-4.326918),(14.222212,-4.359784),(14.184694,-4.377458),(14.146351,-4.38986),(14.058397,-4.400505),(14.021087,-4.415388),(13.991631,-4.455076),(13.989874,-4.45797),(13.987704,-4.46014),(13.971788,-4.471612),(13.954941,-4.491869),(13.944502,-4.498071),(13.934581,-4.498484),(13.905436,-4.4901),(13.904401,-4.489802),(13.87629,-4.491663),(13.867091,-4.489286),(13.856136,-4.479674),(13.84208,-4.453422),(13.832881,-4.441227),(13.810454,-4.426654),(13.78999,-4.424587),(13.726531,-4.446704),(13.714956,-4.453009),(13.711028,-4.464481),(13.721777,-4.561012),(13.71816,-4.580339),(13.702605,-4.62075),(13.699091,-4.642661),(13.712475,-4.685656),(13.705861,-4.694338),(13.698368,-4.693511),(13.690099,-4.691134),(13.681934,-4.695371),(13.6778,-4.706327),(13.680591,-4.724723),(13.67532,-4.731751),(13.666535,-4.732682),(13.643694,-4.726274),(13.633979,-4.725654),(13.613101,-4.739193),(13.591397,-4.779397),(13.57986,-4.783031),(13.570727,-4.785908),(13.558634,-4.780431),(13.538894,-4.759967),(13.526182,-4.754282),(13.51471,-4.755936),(13.482257,-4.77485),(13.492385,-4.783531),(13.496416,-4.792109),(13.493522,-4.799654),(13.482257,-4.805545),(13.430787,-4.843372),(13.403192,-4.876859),(13.39265,-4.885334),(13.371359,-4.841719),(13.364124,-4.806889),(13.35875,-4.794797),(13.349345,-4.785598),(13.334565,-4.777537),(13.274001,-4.755833),(13.255811,-4.743947),(13.208785,-4.703123),(13.18212,-4.686069),(13.172405,-4.676148),(13.160002,-4.643385),(13.142122,-4.61734),(13.134061,-4.602354),(13.112357,-4.576929),(13.087966,-4.579513),(13.071532,-4.60163),(13.073703,-4.635323),(13.027194,-4.612172),(12.961565,-4.533831),(12.921981,-4.502308),(12.902447,-4.480191),(12.884464,-4.432131),(12.869891,-4.411978),(12.854285,-4.403089),(12.842296,-4.40433),(12.830307,-4.409704),(12.814804,-4.412908),(12.801058,-4.410014),(12.782656,-4.400071),(12.775426,-4.396165),(12.761681,-4.391204),(12.737909,-4.40495),(12.726541,-4.427997),(12.718479,-4.451872),(12.70494,-4.467995),(12.686853,-4.47833),(12.670523,-4.491353),(12.656571,-4.507372),(12.634867,-4.54768),(12.623808,-4.559255),(12.608098,-4.565043),(12.497201,-4.5853),(12.429091,-4.607521),(12.414105,-4.608865),(12.387026,-4.605454),(12.377621,-4.619407),(12.378862,-4.662298),(12.374107,-4.683176),(12.332353,-4.765858),(12.321914,-4.778157),(12.307651,-4.783635),(12.273752,-4.787149),(12.258559,-4.790766),(12.244089,-4.796347),(12.235201,-4.803995),(12.222592,-4.780637),(12.213497,-4.769165),(12.204092,-4.763481),(12.192206,-4.763481),(12.187555,-4.768028),(12.15717,-4.870554),(12.130815,-4.912929),(12.018072,-5.0086),(12.009608,-5.019631),(11.994314,-5.005792),(11.980479,-4.989435),(11.957856,-4.951918),(11.920258,-4.906427),(11.895728,-4.882645),(11.886241,-4.855239),(11.830586,-4.797022),(11.82021,-4.7848),(11.819266,-4.77916),(11.824069,-4.77236),(11.829961,-4.773533),(11.843319,-4.788783),(11.848424,-4.786044),(11.853137,-4.773918),(11.850367,-4.752839),(11.824067,-4.718683),(11.781633,-4.680409),(11.776906,-4.667878),(11.777688,-4.657701),(11.801321,-4.657938),(11.820163,-4.637281),(11.824624,-4.614767),(11.806409,-4.577118),(11.692982,-4.463491),(11.589122,-4.374688),(11.497895,-4.294041),(11.450587,-4.26473),(11.401403,-4.221366),(11.382498,-4.191013),(11.377696,-4.184666),(11.364757,-4.162774),(11.362071,-4.15309),(11.363129,-4.131117),(11.362153,-4.119806),(11.358246,-4.111912),(11.270181,-4.041725),(11.172425,-3.977474),(11.139095,-3.9531),(11.114016,-3.936856),(11.129623,-3.907857),(11.196091,-3.728815),(11.212524,-3.697293),(11.238569,-3.675072),(11.272676,-3.652954),(11.302441,-3.62846),(11.334688,-3.608616),(11.375822,-3.600348),(11.413959,-3.587635),(11.436283,-3.559006),(11.45499,-3.528207),(11.482585,-3.509294),(11.532712,-3.516528),(11.569815,-3.54495),(11.64919,-3.673315),(11.665933,-3.691401),(11.687638,-3.69874),(11.742105,-3.691712),(11.762155,-3.693985),(11.819723,-3.710418),(11.838326,-3.710935),(11.840808,-3.709989),(11.858377,-3.703287),(11.879977,-3.687164),(11.897651,-3.66608),(11.905505,-3.643549),(11.898477,-3.622362),(11.879254,-3.612026),(11.855586,-3.604378),(11.834915,-3.592079),(11.824994,-3.571202),(11.827784,-3.548051),(11.884835,-3.413279),(11.898374,-3.389198),(11.941472,-3.356952),(11.954908,-3.335454),(11.944366,-3.303208),(11.905712,-3.285121),(11.858273,-3.272099),(11.821893,-3.254839),(11.781585,-3.221663),(11.760501,-3.209054),(11.703864,-3.185386),(11.696113,-3.179598),(11.687534,-3.17102),(11.685674,-3.167196),(11.687328,-3.161511),(11.688258,-3.092265),(11.693219,-3.074695),(11.706448,-3.04989),(11.716886,-3.043586),(11.730632,-3.044723),(11.75368,-3.042345),(11.778381,-3.027153),(11.778381,-3.005655),(11.763705,-2.982608),(11.687638,-2.901786),(11.653118,-2.851659),(11.636788,-2.833263),(11.616841,-2.826751),(11.588419,-2.835433),(11.547905,-2.863855),(11.530748,-2.866749),(11.52558,-2.843288),(11.535566,-2.804996),(11.537466,-2.797709),(11.61033,-2.678853),(11.619528,-2.653532),(11.624386,-2.632758),(11.624386,-2.611984),(11.619011,-2.586456),(11.609606,-2.564028),(11.584285,-2.523617),(11.576533,-2.502533),(11.57612,-2.485067),(11.583355,-2.42967),(11.579737,-2.407656),(11.55824,-2.349365),(11.566198,-2.329727),(11.568368,-2.329107),(11.605369,-2.330348),(11.655908,-2.345644),(11.669654,-2.352672),(11.675855,-2.360423),(11.679886,-2.368795),(11.687534,-2.377683),(11.721331,-2.404142),(11.739521,-2.413753),(11.756057,-2.415097),(11.772697,-2.405382),(11.806597,-2.374169),(11.828818,-2.365281),(11.864888,-2.35815),(11.907056,-2.34461),(11.938372,-2.329107),(11.948707,-2.335205),(11.966794,-2.348538),(12.000693,-2.390602),(12.031286,-2.411583),(12.062292,-2.414994),(12.45927,-2.329934),(12.447901,-2.30792),(12.455136,-2.275571),(12.468572,-2.241154),(12.475496,-2.212422),(12.468365,-2.171081),(12.470225,-2.156301),(12.477357,-2.141108),(12.49534,-2.11279),(12.499784,-2.09553),(12.46082,-2.077857),(12.440046,-2.045817),(12.431468,-2.005406),(12.423097,-1.897609),(12.426921,-1.88407),(12.438083,-1.886654),(12.450009,-1.899298),(12.468985,-1.919417),(12.486555,-1.924894),(12.499474,-1.917866),(12.503815,-1.907738),(12.504539,-1.896886),(12.506296,-1.888204),(12.512393,-1.875492),(12.517354,-1.86898),(12.523452,-1.863296),(12.572441,-1.83043),(12.58319,-1.826502),(12.632386,-1.825159),(12.674967,-1.842832),(12.741014,-1.890672),(12.745454,-1.893888),(12.787725,-1.907531),(12.804572,-1.919107),(12.816148,-1.943498),(12.838162,-2.016052),(12.847153,-2.037239),(12.852941,-2.044577),(12.868547,-2.059873),(12.872992,-2.065868),(12.876919,-2.088709),(12.875162,-2.089122),(12.875162,-2.093773),(12.874128,-2.099767),(12.876299,-2.104935),(12.895109,-2.109793),(12.894489,-2.11558),(12.890562,-2.122195),(12.888701,-2.126949),(12.884464,-2.134494),(12.882707,-2.140798),(12.885084,-2.147413),(12.89821,-2.160229),(12.902034,-2.167257),(12.902757,-2.177592),(12.907098,-2.195369),(12.921361,-2.194852),(12.945649,-2.18245),(12.95433,-2.190925),(12.973037,-2.221517),(12.982442,-2.232989),(12.987507,-2.259964),(12.990194,-2.268026),(12.997325,-2.275674),(13.003423,-2.275467),(13.008177,-2.276501),(13.010658,-2.287766),(13.002286,-2.296655),(12.996705,-2.304819),(12.994638,-2.314948),(12.993914,-2.323526),(12.991124,-2.332311),(12.987093,-2.340683),(12.982442,-2.347711),(12.97128,-2.357116),(12.966113,-2.366418),(12.969006,-2.372412),(12.982442,-2.371689),(12.994008,-2.360123),(13.012105,-2.342026),(13.025024,-2.338202),(13.049828,-2.342647),(13.114321,-2.371895),(13.135198,-2.37696),(13.146153,-2.374686),(13.165273,-2.363524),(13.176849,-2.361974),(13.186668,-2.364144),(13.361024,-2.428843),(13.3788,-2.43091),(13.398024,-2.426156),(13.419728,-2.42967),(13.424896,-2.429773),(13.438952,-2.425432),(13.44536,-2.427086),(13.453731,-2.439385),(13.458899,-2.441039),(13.469441,-2.435044),(13.473988,-2.426983),(13.476779,-2.417681),(13.482257,-2.408069),(13.492179,-2.398457),(13.524631,-2.37603),(13.533106,-2.367658),(13.547679,-2.349365),(13.557601,-2.340476),(13.569487,-2.334482),(13.593361,-2.326834),(13.603386,-2.320012),(13.60783,-2.311124),(13.610724,-2.289937),(13.617442,-2.279601),(13.69351,-2.205911),(13.701571,-2.194749),(13.705241,-2.18431),(13.706791,-2.16457),(13.709478,-2.153511),(13.72002,-2.133564),(13.740897,-2.102455),(13.743171,-2.09708),(13.746375,-2.09832),(13.756917,-2.105865),(13.770663,-2.119094),(13.782755,-2.138008),(13.791437,-2.159092),(13.800532,-2.200433),(13.823269,-2.246838),(13.835568,-2.265235),(13.853862,-2.283839),(13.86058,-2.293657),(13.863887,-2.321459),(13.871639,-2.328797),(13.882181,-2.333862),(13.892516,-2.3412),(13.906158,-2.359803),(13.901921,-2.366314),(13.890035,-2.373652),(13.875876,-2.403831),(13.855102,-2.41427),(13.847661,-2.423055),(13.846937,-2.461606),(13.8458,-2.466567),(13.86213,-2.480933),(13.884971,-2.488477),(13.909466,-2.489718),(13.930446,-2.485377),(13.947706,-2.474938),(13.957318,-2.470597),(13.967033,-2.470597),(13.976232,-2.477315),(13.979229,-2.495609),(13.985637,-2.5015),(13.993285,-2.501603),(14.039277,-2.487237),(14.045582,-2.4861),(14.055607,-2.486927),(14.071006,-2.495816),(14.080928,-2.49995),(14.090023,-2.498606),(14.096018,-2.491061),(14.100553,-2.478065),(14.106766,-2.460262),(14.115241,-2.451374),(14.124853,-2.443829),(14.131881,-2.435147),(14.131468,-2.408896),(14.136015,-2.398767),(14.14418,-2.390189),(14.1633,-2.373963),(14.172395,-2.367865),(14.182317,-2.363317),(14.230066,-2.352155),(14.234407,-2.354946),(14.226966,-2.323113),(14.151208,-2.259034),(14.148004,-2.224824),(14.161543,-2.214282),(14.195857,-2.201777),(14.209189,-2.187204),(14.209396,-2.179866),(14.203195,-2.161986),(14.204332,-2.152167),(14.209706,-2.147413),(14.226966,-2.141315),(14.233787,-2.134494),(14.238438,-2.120231),(14.244019,-2.067521),(14.250634,-2.047781),(14.251667,-2.037859),(14.24681,-2.029384),(14.238645,-2.02339),(14.233477,-2.018222),(14.232857,-2.012021),(14.238128,-2.002616),(14.237611,-1.991143),(14.240092,-1.982255),(14.245053,-1.974814),(14.252184,-1.967579),(14.368571,-1.92212),(14.37073,-1.921277),(14.397291,-1.906808),(14.404526,-1.893062),(14.396878,-1.849447),(14.397498,-1.840972),(14.403596,-1.815237),(14.406696,-1.735552),(14.413104,-1.713951),(14.425197,-1.700515),(14.448761,-1.686253),(14.441113,-1.679638),(14.420029,-1.676124),(14.402976,-1.670853),(14.390883,-1.630339),(14.371453,-1.619487),(14.364839,-1.612045),(14.37383,-1.604087),(14.394294,-1.596852),(14.403699,-1.592201),(14.413208,-1.585277),(14.419926,-1.575148),(14.423956,-1.563573),(14.430364,-1.554891),(14.463127,-1.54869),(14.462714,-1.5336),(14.451552,-1.515617),(14.438426,-1.502595),(14.451035,-1.473656),(14.452688,-1.449574),(14.456306,-1.440169),(14.467881,-1.429007),(14.480594,-1.420842),(14.487725,-1.40937),(14.482144,-1.388596),(14.447004,-1.365342),(14.432328,-1.350149),(14.426333,-1.331029),(14.429124,-1.31997),(14.436049,-1.312735),(14.444317,-1.306327),(14.451552,-1.298163),(14.457133,-1.287621),(14.46075,-1.278009),(14.464781,-1.255581),(14.465814,-1.224369),(14.464161,-1.215377),(14.456409,-1.202458),(14.439435,-1.185733),(14.435325,-1.181684),(14.428297,-1.169488),(14.427987,-1.16091),(14.433568,-1.14365),(14.434395,-1.135072),(14.430984,-1.123393),(14.420339,-1.101792),(14.391814,-1.023347),(14.39202,-1.008981),(14.402562,-0.987691),(14.407523,-0.944489),(14.412794,-0.921545),(14.428194,-0.896947),(14.445867,-0.877516),(14.457133,-0.856122),(14.452895,-0.825737),(14.446177,-0.800312),(14.44597,-0.775714),(14.451552,-0.751219),(14.492169,-0.663369),(14.498991,-0.630916),(14.489999,-0.600944),(14.466538,-0.552678),(14.454239,-0.539553),(14.414448,-0.512164),(14.39233,-0.482398),(14.358637,-0.461314),(14.320603,-0.444468),(14.292698,-0.436406),(14.275542,-0.439094),(14.260039,-0.446432),(14.244742,-0.451599),(14.228516,-0.447775),(14.210223,-0.440334),(14.193066,-0.440437),(14.17653,-0.446122),(14.159683,-0.455113),(14.157926,-0.417699),(14.150588,-0.376462),(14.138289,-0.336361),(14.121443,-0.302151),(14.095398,-0.276623),(14.058811,-0.258846),(14.01871,-0.252645),(13.98233,-0.262257),(13.975095,-0.252542),(13.967033,-0.245824),(13.957318,-0.242),(13.921558,-0.237865),(13.913807,-0.240553),(13.909983,-0.247167),(13.907295,-0.254402),(13.903058,-0.258846),(13.889829,-0.263704),(13.883421,-0.264944),(13.877426,-0.2605),(13.831228,-0.205413),(13.832158,-0.177094),(13.8427,-0.152289),(13.888054,-0.092088),(13.900474,-0.075602),(13.910706,-0.065886),(13.921145,-0.052967),(13.920111,-0.042425),(13.914943,-0.031263),(13.91298,-0.016587),(13.917217,-0.006665),(13.931377,0.011628),(13.932927,0.022067),(13.925072,0.034366),(13.913083,0.036536),(13.901404,0.036846),(13.894893,0.043668),(13.890656,0.115911),(13.885901,0.140303),(13.876186,0.171308),(13.871225,0.196423),(13.875773,0.220298),(13.909776,0.270941),(13.945226,0.350419),(13.966517,0.373053),(14.040414,0.413774),(14.059121,0.435272),(14.063048,0.455942),(14.056847,0.476406),(14.045582,0.497283),(14.075967,0.539451),(14.125473,0.546169),(14.181594,0.539865),(14.231823,0.542552),(14.245259,0.54772),(14.250944,0.551854),(14.254664,0.557125),(14.328768,0.621204),(14.335383,0.642598),(14.33838,0.693912),(14.367319,0.735512),(14.413414,0.775044),(14.447624,0.81623),(14.44101,0.862791),(14.437599,0.875142),(14.445144,0.883255),(14.457029,0.890334),(14.466228,0.899636),(14.468088,0.913227),(14.46044,0.921237),(14.436565,0.933277),(14.417548,0.950589),(14.400805,0.970329),(14.386749,0.99224),(14.375691,1.015598),(14.360084,1.066758),(14.346442,1.087118),(14.298486,1.109236),(14.28288,1.130113),(14.274715,1.155641),(14.275748,1.231089),(14.273061,1.257237),(14.258178,1.319662),(14.252184,1.330824),(14.241435,1.339196),(14.230066,1.34395),(14.221488,1.349479),(14.219939,1.355018),(14.218387,1.360564),(14.206502,1.362838),(14.198957,1.368316),(14.192549,1.374956),(14.183868,1.380847),(14.14449,1.391622),(14.12444,1.393585),(14.106456,1.388934),(14.070696,1.375111),(14.049509,1.376403),(14.033799,1.387643),(14.01902,1.402138),(14.00083,1.413016),(13.98357,1.41684),(13.794744,1.434436),(13.780585,1.431025),(13.779758,1.426271),(13.770353,1.400432),(13.758157,1.376506),(13.75485,1.374336),(13.744101,1.370279),(13.741104,1.371313),(13.729425,1.37692),(13.724361,1.377953),(13.71754,1.375369),(13.705551,1.36599),(13.699091,1.362786),(13.680694,1.359401),(13.671496,1.358781),(13.662091,1.359195),(13.642247,1.357386),(13.628088,1.348136),(13.615065,1.337439),(13.579202,1.323486),(13.556361,1.29279),(13.543442,1.283489),(13.509955,1.277287),(13.479776,1.277132),(13.448874,1.283075),(13.41394,1.295529),(13.398748,1.296925),(13.37694,1.294444),(13.357406,1.28907),(13.349345,1.281577),(13.342937,1.263852),(13.325987,1.259304),(13.307797,1.25672),(13.289194,1.233983),(13.2709,1.226024),(13.25023,1.221787),(13.215813,1.224371),(13.195453,1.221115),(13.184911,1.22251),(13.184885,1.222623),(13.18181,1.236463),(13.172301,1.242871),(13.160313,1.247729),(13.150184,1.256617),(13.169201,1.258581),(13.176642,1.27088),(13.180363,1.284832),(13.188321,1.291447),(13.202377,1.293411),(13.219327,1.299198),(13.23359,1.30881),(13.242995,1.328861),(13.249093,1.332375),(13.251263,1.337646),(13.242891,1.34979),(13.239584,1.356507),(13.239171,1.364181),(13.239791,1.372346),(13.238447,1.41963),(13.239481,1.427976),(13.223461,1.438001),(13.205478,1.48637),(13.184911,1.496886),(13.190905,1.511072),(13.189665,1.52474),(13.182637,1.536548),(13.171268,1.545256),(13.164447,1.53784),(13.145946,1.56342),(13.129617,1.592462),(13.146773,1.612254),(13.149977,1.636284),(13.143983,1.681811),(13.147807,1.702869),(13.154628,1.721653),(13.157522,1.739146),(13.150184,1.756302),(13.162793,1.76364),(13.173025,1.776947),(13.180673,1.792011),(13.191008,1.828727),(13.192455,1.844746),(13.188321,1.85658),(13.183154,1.864048),(13.175092,1.882108),(13.162173,1.899833),(13.166307,1.904381),(13.178193,1.907792),(13.184911,1.92345),(13.191732,1.935077),(13.19845,1.943112),(13.205064,1.949339),(13.210232,1.956316),(13.212299,1.966419),(13.206615,1.985151),(13.206925,1.993575),(13.21602,1.997166),(13.221291,2.000835),(13.239481,2.02073),(13.245475,2.025304),(13.281132,2.044967),(13.291364,2.068945),(13.294878,2.101087),(13.294568,2.161058),(13.466547,2.159508),(13.620336,2.158112),(13.662266,2.157761),(13.823993,2.156407),(14.002793,2.154805),(14.16175,2.15341),(14.266963,2.152428),(14.276265,2.153565),(14.320293,2.17067),(14.383649,2.162091),(14.399358,2.153151),(14.418685,2.135271),(14.432948,2.132119),(14.438736,2.133979),(14.4592,2.146537),(14.538058,2.163383),(14.54705,2.166536),(14.550254,2.175166),(14.54798,2.184364),(14.547773,2.193666),(14.557075,2.202399),(14.562139,2.208807),(14.587874,2.201727),(14.620947,2.16483),(14.644821,2.154185),(14.636346,2.13832),(14.646062,2.133359),(14.672003,2.134341),(14.685956,2.125866),(14.705696,2.105532),(14.720476,2.099589),(14.713654,2.12398),(14.720889,2.127597),(14.747864,2.113205),(14.733498,2.102018),(14.727297,2.099589),(14.737116,2.096075),(14.747554,2.097186),(14.756339,2.099589),(14.761403,2.099589),(14.76254,2.092535),(14.76161,2.070185),(14.764814,2.065456),(14.826309,2.072252),(14.841192,2.075921),(14.84822,2.085197),(14.850184,2.097367),(14.850184,2.109795),(14.857005,2.115944),(14.871474,2.101656),(14.8846,2.082096),(14.887804,2.072252),(14.907338,2.058894),(14.896589,2.030937),(14.892455,2.00639),(14.932142,2.003341),(14.953846,2.005822),(14.963045,2.010421),(14.971003,2.034451),(14.980718,2.033107),(15.028261,1.995306),(15.04366,1.990862),(15.06309,1.997166),(15.074873,1.979364),(15.091616,1.984428),(15.107636,2.000835),(15.117764,2.01701),(15.123862,2.01701),(15.138538,2.011506),(15.151664,2.040988),(15.1836,2.029335),(15.197036,2.034967),(15.212642,2.039618),(15.233726,2.031298),(15.238687,2.025304),(15.244165,2.010059),(15.248092,2.003341),(15.25357,2.000318),(15.287573,1.987787),(15.295428,1.981534),(15.301112,1.973757),(15.303283,1.966419),(15.301939,1.953319),(15.300182,1.94766),(15.301629,1.943578),(15.309484,1.935077),(15.314341,1.93425),(15.337802,1.92376),(15.337389,1.92146),(15.349171,1.923295),(15.367981,1.932855),(15.388135,1.939547),(15.439192,1.969829),(15.48291,1.97585),(15.526628,1.967814),(15.611791,1.942544),(15.706772,1.931976),(15.711113,1.927377),(15.721552,1.921615),(15.734057,1.916706),(15.757415,1.912985),(15.76465,1.908722),(15.80289,1.859965),(15.815396,1.853428),(15.842061,1.844152),(15.865936,1.828468),(15.875651,1.823766),(15.882369,1.816712),(15.884952,1.801648),(15.889913,1.793277),(15.901799,1.793742),(15.915235,1.798961),(15.92588,1.804749),(15.928981,1.788962),(15.94376,1.781624),(15.963191,1.77736),(15.98045,1.770591),(15.990786,1.776172),(16.002775,1.775913),(16.013833,1.771314),(16.021998,1.763795),(16.026339,1.752297),(16.024892,1.742479),(16.021998,1.733048),(16.021998,1.72279),(16.030473,1.705401),(16.069851,1.654551),(16.114809,1.719069),(16.123594,1.721601),(16.145092,1.714315),(16.15584,1.719069),(16.159044,1.734805),(16.135066,1.845315),(16.101373,1.932803),(16.083493,1.963008),(16.074915,1.970604),(16.068714,1.973602),(16.06489,1.97895),(16.063753,2.015873),(16.06644,2.025485),(16.081736,2.046827),(16.08153,2.055638),(16.078119,2.063441),(16.076672,2.072252),(16.076155,2.07282),(16.075535,2.075456),(16.075535,2.079952),(16.076672,2.08592),(16.079669,2.090261),(16.084113,2.091605),(16.088144,2.09176),(16.090315,2.092742),(16.092485,2.106901),(16.089798,2.116151),(16.08556,2.12491),(16.083493,2.137752),(16.085767,2.149172),(16.097756,2.171548),(16.104371,2.193562),(16.12039,2.203329),(16.140647,2.199815),(16.159251,2.18209),(16.199455,2.212166),(16.207723,2.22307),(16.202349,2.23201),(16.196665,2.236454),(16.248031,2.345594),(16.293506,2.442074),(16.325649,2.510494),(16.371124,2.606922),(16.413706,2.697201),(16.427492,2.726493),(16.450603,2.775594),(16.479128,2.836314),(16.483779,2.860395),(16.474994,2.878378),(16.460525,2.894605),(16.448329,2.913467),(16.445538,2.95765),(16.484192,3.031909),(16.490807,3.067824),(16.483159,3.090148),(16.47272,3.110612),(16.464762,3.1318),(16.465485,3.156501),(16.477061,3.179394),(16.511477,3.227246),(16.517162,3.248692),(16.517782,3.273238),(16.542793,3.336154),(16.551682,3.419767),(16.567701,3.464389),(16.598501,3.501674),(16.639428,3.528597),(16.686351,3.542214),(16.709398,3.543712),(16.729552,3.542214),(16.811924,3.521672),(16.833008,3.522861),(16.852645,3.532809),(16.854609,3.541335),(16.851818,3.551903),(16.853162,3.560094),(16.867838,3.561541),(16.877243,3.559164),(16.895537,3.552368),(16.904735,3.550637),(16.932434,3.553919),(16.942252,3.553092),(16.953208,3.546891),(16.960546,3.538441),(16.968607,3.533481),(16.98194,3.537976),(16.989691,3.539475),(17.003127,3.537718),(17.009225,3.538131),(17.017596,3.5418),(17.028862,3.551541),(17.035787,3.556141),(17.052426,3.56397),(17.061418,3.56614),(17.088393,3.564435),(17.102139,3.566605),(17.129011,3.57707),(17.142757,3.580842),(17.182858,3.579602),(17.197224,3.58172),(17.213967,3.589291),(17.222959,3.598515),(17.231123,3.609109),(17.244973,3.620607),(17.259442,3.625904),(17.272258,3.62456),(17.298096,3.615904),(17.334373,3.618514),(17.355767,3.638435),(17.375714,3.663447),(17.414489,3.683415),(17.427494,3.690112),(17.442687,3.701274),(17.458913,3.708276),(17.481858,3.704762),(17.486198,3.700292),(17.490126,3.68714),(17.494983,3.683213),(17.508936,3.681766),(17.513484,3.679182),(17.550484,3.648151),(17.586761,3.632234),(17.627275,3.626317)] +Coral Sea Islands [(154.391287,-21.030043),(154.38852,-21.029067),(154.391287,-21.028741),(154.391287,-21.030043)] +Curaçao [(-68.780751,12.097805),(-68.745229,12.062242),(-68.739735,12.053046),(-68.748647,12.047024),(-68.769846,12.042914),(-68.794749,12.041327),(-68.814809,12.043158),(-68.841176,12.055487),(-68.893422,12.090522),(-68.944732,12.105618),(-68.977773,12.124579),(-69.007436,12.148179),(-69.020253,12.169745),(-69.027211,12.19009),(-69.043609,12.196682),(-69.062896,12.198432),(-69.078603,12.20425),(-69.091908,12.219875),(-69.12267,12.269029),(-69.134389,12.27733),(-69.148101,12.28441),(-69.159535,12.293362),(-69.164296,12.306952),(-69.161977,12.342841),(-69.163808,12.361884),(-69.171742,12.378892),(-69.166127,12.387844),(-69.161204,12.391506),(-69.156239,12.390611),(-69.150624,12.385728),(-69.107777,12.367825),(-69.089589,12.356879),(-69.071523,12.341376),(-69.060414,12.322577),(-69.049957,12.276109),(-69.040761,12.256049),(-69.004018,12.221666),(-68.961293,12.201361),(-68.835276,12.171047),(-68.82079,12.16234),(-68.814809,12.149604),(-68.811838,12.12934),(-68.804026,12.117011),(-68.780751,12.097805)] +Czech Republic [(14.397808,51.013115),(14.425817,51.020944),(14.462094,51.019652),(14.473462,51.023372),(14.482144,51.037196),(14.487932,51.028721),(14.490102,51.022701),(14.493513,51.016706),(14.502918,51.008516),(14.51532,51.003038),(14.542295,50.998749),(14.554594,50.992651),(14.566687,50.983401),(14.574335,50.975494),(14.577435,50.965727),(14.575575,50.95069),(14.567824,50.938649),(14.555835,50.924283),(14.550357,50.912087),(14.561829,50.906351),(14.617123,50.920872),(14.629215,50.920717),(14.634589,50.9094),(14.628802,50.896119),(14.611852,50.870953),(14.608338,50.853073),(14.613195,50.84558),(14.647922,50.839379),(14.662185,50.834366),(14.700425,50.815969),(14.737529,50.810698),(14.759233,50.810181),(14.775356,50.812972),(14.785071,50.820052),(14.79458,50.831627),(14.810393,50.858447),(14.831683,50.857982),(14.867857,50.86439),(14.982062,50.859067),(14.984129,50.867697),(14.982889,50.902424),(14.981752,50.905266),(14.981132,50.90909),(14.982062,50.914671),(14.984852,50.918082),(14.994671,50.92511),(14.997771,50.930174),(14.996635,50.959216),(14.981442,50.973221),(14.965215,50.981334),(14.960978,50.992651),(14.961081,50.992651),(14.965629,50.996062),(14.970693,50.998697),(14.976377,51.000402),(14.982062,51.001177),(14.996531,51.007844),(15.001182,51.012443),(15.003973,51.020685),(15.02299,51.009652),(15.073632,51.002624),(15.082107,50.992754),(15.09389,50.985416),(15.107946,50.981024),(15.119211,50.982471),(15.122518,50.992651),(15.122518,50.992754),(15.13151,51.005828),(15.144429,51.011564),(15.156108,51.007844),(15.160759,50.992651),(15.170164,50.977975),(15.227111,50.977406),(15.253776,50.969035),(15.269796,50.952653),(15.268763,50.94299),(15.260081,50.932499),(15.253053,50.914103),(15.25605,50.899892),(15.265352,50.88227),(15.277134,50.865734),(15.28778,50.854881),(15.307727,50.844753),(15.325813,50.839689),(15.341006,50.83173),(15.352995,50.812972),(15.353305,50.803257),(15.350515,50.793232),(15.349688,50.783723),(15.356096,50.775455),(15.368579,50.772949),(15.370255,50.772613),(15.381314,50.780002),(15.390822,50.790648),(15.400124,50.797986),(15.441775,50.800208),(15.641866,50.755766),(15.653235,50.75096),(15.666464,50.738093),(15.673803,50.733545),(15.684345,50.731426),(15.76744,50.744191),(15.792245,50.742692),(15.794415,50.735871),(15.797619,50.729773),(15.848159,50.675151),(15.854567,50.673704),(15.882265,50.674221),(15.941486,50.68838),(15.958023,50.686881),(15.971459,50.678613),(15.984481,50.662904),(15.993163,50.649364),(15.998641,50.635567),(15.996367,50.623681),(15.982001,50.61624),(15.980037,50.612984),(15.97921,50.60978),(15.979934,50.606628),(15.982001,50.603631),(16.024479,50.608592),(16.086491,50.64678),(16.128658,50.644042),(16.160284,50.628642),(16.175477,50.624095),(16.192324,50.626575),(16.204519,50.636342),(16.211754,50.649106),(16.219092,50.659131),(16.232011,50.660888),(16.268288,50.660423),(16.300948,50.6551),(16.331644,50.644042),(16.416806,50.586629),(16.425901,50.567561),(16.394482,50.559344),(16.398306,50.551799),(16.398306,50.548492),(16.395516,50.545701),(16.391175,50.539914),(16.389624,50.534849),(16.388281,50.52715),(16.388384,50.521258),(16.391175,50.521827),(16.382596,50.514179),(16.362029,50.501208),(16.352624,50.492785),(16.335468,50.490356),(16.303428,50.49604),(16.287719,50.492785),(16.279967,50.479866),(16.264981,50.471649),(16.247411,50.464983),(16.232011,50.456766),(16.226534,50.455371),(16.217335,50.45165),(16.211237,50.451289),(16.216095,50.440178),(16.217955,50.437129),(16.198008,50.440023),(16.190153,50.424158),(16.199559,50.406278),(16.232011,50.402609),(16.24493,50.376823),(16.249685,50.371035),(16.2625,50.36442),(16.269839,50.365454),(16.27697,50.369433),(16.288649,50.371862),(16.334227,50.371862),(16.343736,50.36995),(16.350764,50.360648),(16.351487,50.351346),(16.35066,50.342251),(16.353244,50.333518),(16.371021,50.318377),(16.415566,50.307938),(16.437787,50.297965),(16.452153,50.284994),(16.477578,50.255332),(16.492047,50.242722),(16.50476,50.227995),(16.535042,50.207738),(16.545894,50.188824),(16.557263,50.154718),(16.566668,50.142212),(16.584238,50.127432),(16.618654,50.10702),(16.660616,50.093016),(16.701233,50.094928),(16.732033,50.121955),(16.754253,50.13229),(16.766449,50.1434),(16.777301,50.15694),(16.795284,50.174406),(16.817712,50.186757),(16.837556,50.188617),(16.857606,50.187739),(16.869762,50.189729),(16.880964,50.191563),(16.890369,50.196937),(16.907009,50.211768),(16.918998,50.217091),(16.927783,50.217298),(16.946283,50.21306),(16.955275,50.213474),(16.957755,50.217143),(16.957548,50.229235),(16.960339,50.231974),(16.975015,50.231819),(16.98101,50.229183),(16.98566,50.223137),(16.996306,50.212905),(17.008398,50.20996),(17.014806,50.218486),(17.012842,50.231664),(16.99982,50.242722),(16.998373,50.26701),(16.987831,50.284942),(16.969641,50.297758),(16.945249,50.306905),(16.926543,50.319255),(16.916724,50.338634),(16.909283,50.359925),(16.897914,50.378218),(16.865874,50.408449),(16.865771,50.422401),(16.892953,50.432943),(16.915794,50.431083),(16.944005,50.420267),(16.958789,50.414598),(16.98194,50.416355),(17.084672,50.397958),(17.095628,50.393669),(17.118882,50.381164),(17.131698,50.376823),(17.145444,50.376719),(17.175933,50.38075),(17.187612,50.378476),(17.185442,50.375686),(17.188852,50.364834),(17.195053,50.351915),(17.201048,50.343233),(17.210866,50.336412),(17.24611,50.322149),(17.269881,50.317757),(17.316803,50.318635),(17.337474,50.313002),(17.319077,50.308248),(17.321867,50.304476),(17.325381,50.301117),(17.329515,50.298223),(17.334063,50.295691),(17.332823,50.282668),(17.334786,50.270008),(17.341711,50.259724),(17.355044,50.253264),(17.365896,50.271558),(17.388013,50.272643),(17.409821,50.261326),(17.419122,50.242722),(17.424187,50.240552),(17.429561,50.239777),(17.434832,50.2405),(17.44,50.242722),(17.469352,50.265563),(17.516998,50.268354),(17.606398,50.258225),(17.629549,50.262101),(17.646705,50.271093),(17.676368,50.297396),(17.681949,50.305561),(17.684533,50.312382),(17.691044,50.315069),(17.707994,50.311039),(17.713575,50.30768),(17.731971,50.291763),(17.734452,50.28949),(17.734969,50.286802),(17.734245,50.283805),(17.731971,50.280653),(17.721119,50.261946),(17.719156,50.252799),(17.721429,50.242722),(17.731971,50.236005),(17.747578,50.217504),(17.738173,50.202363),(17.717915,50.191098),(17.675334,50.174716),(17.648669,50.167998),(17.600403,50.16681),(17.589448,50.163244),(17.58211,50.153167),(17.584177,50.145881),(17.632753,50.106348),(17.648359,50.101801),(17.658488,50.102989),(17.666963,50.106142),(17.677608,50.107744),(17.690734,50.10547),(17.717915,50.096788),(17.731971,50.094876),(17.732385,50.093894),(17.732592,50.092964),(17.732385,50.092189),(17.731971,50.091466),(17.72267,50.08635),(17.719466,50.080872),(17.722566,50.075136),(17.731971,50.06909),(17.74303,50.060563),(17.749852,50.049504),(17.755433,50.037154),(17.763287,50.025216),(17.774553,50.015191),(17.839355,49.973643),(17.875735,49.968682),(17.912116,49.975814),(17.941571,49.992764),(17.959761,49.995812),(17.999862,49.996588),(18.032418,50.00284),(18.037483,50.007233),(18.035726,50.017207),(18.028698,50.024131),(18.020119,50.024286),(18.012161,50.022684),(18.00689,50.024131),(18.002446,50.046765),(18.019074,50.044087),(18.033245,50.041805),(18.091639,50.017207),(18.085128,49.998758),(18.098151,49.989043),(18.162023,49.98124),(18.177216,49.975814),(18.208738,49.958295),(18.22052,49.954885),(18.243982,49.951732),(18.255867,49.946048),(18.260105,49.94057),(18.267029,49.925067),(18.27261,49.918298),(18.292454,49.907808),(18.306923,49.909668),(18.334932,49.925429),(18.333795,49.927135),(18.332452,49.928271),(18.330695,49.92884),(18.328524,49.92884),(18.368832,49.932044),(18.407486,49.923155),(18.481797,49.896645),(18.505051,49.896697),(18.544428,49.912975),(18.559208,49.907187),(18.565409,49.889049),(18.562929,49.873701),(18.567269,49.861454),(18.593831,49.852204),(18.566339,49.831585),(18.566753,49.804765),(18.584116,49.774379),(18.60768,49.74296),(18.617706,49.713866),(18.62401,49.706373),(18.637343,49.700327),(18.668142,49.69857),(18.682404,49.695779),(18.695427,49.688854),(18.715064,49.673817),(18.727673,49.668907),(18.742246,49.669683),(18.757852,49.674075),(18.773458,49.675832),(18.788444,49.668597),(18.792579,49.660794),(18.799297,49.626378),(18.803947,49.616766),(18.81573,49.599299),(18.820174,49.590256),(18.83454,49.547623),(18.837434,49.526952),(18.833196,49.510261),(18.792269,49.509537),(18.773562,49.504886),(18.732531,49.480288),(18.704522,49.479255),(18.675583,49.485043),(18.64282,49.495791),(18.635896,49.496721),(18.628971,49.495791),(18.600446,49.485973),(18.556211,49.490159),(18.535643,49.481684),(18.530476,49.473467),(18.527995,49.454864),(18.522931,49.446079),(18.514663,49.440601),(18.481797,49.429077),(18.439215,49.395074),(18.416788,49.385462),(18.387642,49.389751),(18.385162,49.342261),(18.361586,49.330191),(18.324597,49.311255),(18.190031,49.276942),(18.160576,49.2587),(18.136495,49.233068),(18.117788,49.202579),(18.105075,49.169765),(18.101458,49.142945),(18.10163,49.136928),(18.102905,49.09225),(18.09629,49.070288),(18.075516,49.047188),(18.046061,49.029153),(18.012885,49.019128),(17.959244,49.021867),(17.93506,49.019386),(17.914079,49.010498),(17.90085,48.993031),(17.894649,48.9782),(17.886897,48.947246),(17.878733,48.933552),(17.860336,48.921718),(17.841215,48.920994),(17.820131,48.923371),(17.795533,48.920581),(17.77941,48.911744),(17.744891,48.872574),(17.727217,48.86291),(17.535084,48.812991),(17.498497,48.816763),(17.467905,48.838105),(17.453332,48.842756),(17.429561,48.838157),(17.411225,48.830216),(17.391734,48.821776),(17.374371,48.819605),(17.260269,48.857794),(17.21221,48.866062),(17.167458,48.859758),(17.113715,48.83392),(17.10462,48.824618),(17.098728,48.805756),(17.084362,48.793715),(17.049946,48.774027),(17.025348,48.746328),(16.974808,48.649874),(16.963336,48.635947),(16.947523,48.623157),(16.945043,48.604166),(16.910523,48.63078),(16.896674,48.696977),(16.873006,48.718991),(16.856573,48.719818),(16.818125,48.710775),(16.798902,48.709224),(16.780505,48.713772),(16.744332,48.729637),(16.729035,48.733099),(16.691001,48.732065),(16.675085,48.733926),(16.661959,48.740023),(16.654725,48.751857),(16.651004,48.766223),(16.643149,48.778264),(16.624029,48.78338),(16.605529,48.784827),(16.545791,48.796299),(16.528737,48.803534),(16.519539,48.805601),(16.510341,48.804774),(16.492254,48.79971),(16.481919,48.7994),(16.453083,48.80219),(16.43572,48.794542),(16.384974,48.737078),(16.374018,48.730308),(16.358309,48.727259),(16.352727,48.728448),(16.339498,48.735579),(16.318724,48.733512),(16.317588,48.73284),(16.177751,48.746535),(16.085354,48.742866),(16.032334,48.758059),(15.931771,48.806428),(15.930324,48.811389),(15.930324,48.818313),(15.925053,48.822602),(15.908103,48.819709),(15.90707,48.830251),(15.899629,48.834746),(15.888363,48.835056),(15.875754,48.833093),(15.885986,48.842033),(15.877614,48.841671),(15.870173,48.843893),(15.859218,48.849164),(15.840097,48.850094),(15.833276,48.852006),(15.828315,48.857122),(15.824284,48.869421),(15.818497,48.872315),(15.787904,48.872263),(15.779222,48.870868),(15.743669,48.858466),(15.726823,48.854952),(15.703775,48.854952),(15.681347,48.858466),(15.603729,48.887353),(15.544187,48.902612),(15.521254,48.908489),(15.471851,48.936704),(15.450354,48.944817),(15.405912,48.954687),(15.357853,48.97081),(15.335529,48.974996),(15.28871,48.975358),(15.283749,48.977632),(15.279615,48.982593),(15.27455,48.986727),(15.266902,48.986675),(15.262872,48.982593),(15.260391,48.969157),(15.257084,48.963886),(15.243234,48.95293),(15.238067,48.95076),(15.161792,48.937221),(15.141949,48.937479),(15.148563,48.951742),(15.148977,48.965333),(15.144739,48.978924),(15.137401,48.993031),(15.059576,48.997217),(15.003973,49.009774),(14.982062,49.007914),(14.978858,49.00626),(14.977308,49.002902),(14.977308,48.998251),(14.978238,48.992825),(14.964595,48.971741),(14.964182,48.943267),(14.968316,48.912674),(14.968213,48.885028),(14.947025,48.822396),(14.93979,48.809787),(14.938137,48.80281),(14.940101,48.796506),(14.944441,48.788599),(14.948886,48.78214),(14.951056,48.780331),(14.939584,48.762813),(14.919327,48.761573),(14.86765,48.775577),(14.815457,48.780331),(14.800367,48.776507),(14.794476,48.766999),(14.790446,48.754855),(14.78042,48.743124),(14.775356,48.724004),(14.722749,48.693411),(14.703836,48.673103),(14.700219,48.646102),(14.700735,48.615535),(14.695671,48.589542),(14.69114,48.586535),(14.675621,48.576235),(14.659187,48.576959),(14.651333,48.583496),(14.645442,48.592901),(14.634693,48.602332),(14.628182,48.603288),(14.612885,48.599748),(14.605134,48.600316),(14.60193,48.604528),(14.601827,48.611194),(14.600586,48.617576),(14.594179,48.621323),(14.578779,48.620496),(14.548807,48.610936),(14.53382,48.608714),(14.522038,48.610626),(14.482144,48.624475),(14.458166,48.643182),(14.443593,48.636516),(14.421166,48.597061),(14.405353,48.586286),(14.353366,48.571429),(14.333213,48.560706),(14.325358,48.558639),(14.315746,48.557916),(14.216527,48.58117),(14.07483,48.591712),(14.040827,48.601169),(14.032456,48.60613),(14.015196,48.620393),(14.010855,48.62587),(14.006204,48.639513),(14.008064,48.642407),(14.013749,48.643802),(14.020467,48.653155),(14.023464,48.654602),(14.028322,48.653982),(14.032869,48.654912),(14.034729,48.661165),(14.032146,48.667263),(14.026151,48.670054),(14.019227,48.671914),(14.014162,48.675066),(14.007754,48.683283),(13.991115,48.700181),(13.98233,48.706485),(13.915047,48.73098),(13.893136,48.743021),(13.874946,48.752426),(13.855929,48.759299),(13.815725,48.76643),(13.796191,48.779814),(13.747925,48.843376),(13.724671,48.867303),(13.710615,48.87247),(13.673356,48.876346),(13.654443,48.882237),(13.637389,48.893451),(13.621783,48.909057),(13.611138,48.927247),(13.608657,48.946161),(13.590467,48.944817),(13.573414,48.951018),(13.556464,48.959958),(13.538894,48.967038),(13.522395,48.969342),(13.51595,48.970242),(13.5021,48.966056),(13.492592,48.955049),(13.482257,48.937634),(13.458899,48.945024),(13.427273,48.959958),(13.398541,48.977683),(13.384175,48.993031),(13.383555,49.021092),(13.372393,49.038507),(13.329915,49.066929),(13.299529,49.093645),(13.287333,49.10057),(13.267283,49.105634),(13.206821,49.107495),(13.178503,49.118347),(13.162586,49.135142),(13.14946,49.15483),(13.055823,49.238184),(13.04342,49.242784),(13.043214,49.242887),(13.043214,49.242939),(13.011278,49.267795),(13.00673,49.277097),(13.005387,49.28676),(12.999806,49.294925),(12.982442,49.299628),(12.95433,49.31911),(12.939861,49.326758),(12.922911,49.332701),(12.88467,49.339573),(12.870925,49.336731),(12.875369,49.323864),(12.856662,49.32221),(12.797751,49.327843),(12.777907,49.332545),(12.762094,49.343242),(12.729331,49.391147),(12.709177,49.404737),(12.663082,49.418897),(12.643548,49.42949),(12.632179,49.44396),(12.627218,49.460186),(12.624325,49.493001),(12.622981,49.509847),(12.61554,49.513775),(12.604998,49.512793),(12.593732,49.515015),(12.58288,49.522405),(12.575025,49.529898),(12.568721,49.538838),(12.562313,49.550827),(12.558282,49.562557),(12.553631,49.584726),(12.546603,49.595579),(12.536061,49.603123),(12.512187,49.61165),(12.501955,49.619711),(12.502678,49.62214),(12.507432,49.629995),(12.499474,49.632527),(12.496891,49.633871),(12.505365,49.646118),(12.504642,49.659399),(12.496374,49.669993),(12.482524,49.674695),(12.449245,49.68503),(12.433949,49.6918),(12.419583,49.700223),(12.398912,49.719499),(12.388577,49.732314),(12.383616,49.742856),(12.395398,49.757894),(12.436739,49.769263),(12.452655,49.779702),(12.455859,49.796135),(12.456376,49.817115),(12.462474,49.831223),(12.482524,49.827244),(12.489863,49.842695),(12.51291,49.870239),(12.520972,49.885535),(12.524072,49.904965),(12.517871,49.9122),(12.503608,49.915766),(12.462681,49.931269),(12.463818,49.942121),(12.470742,49.955608),(12.468158,49.970233),(12.451312,49.980568),(12.414518,49.983048),(12.399222,49.992764),(12.398189,49.992764),(12.24688,50.044957),(12.243676,50.051313),(12.243469,50.060408),(12.241402,50.07064),(12.232514,50.08051),(12.218044,50.088572),(12.202025,50.094514),(12.187865,50.102834),(12.179184,50.117976),(12.178564,50.132962),(12.185592,50.155699),(12.182698,50.170737),(12.17505,50.182623),(12.163784,50.193785),(12.13929,50.211045),(12.089887,50.230837),(12.079552,50.242722),(12.092057,50.254815),(12.102393,50.259156),(12.109111,50.263806),(12.110971,50.276622),(12.10725,50.290885),(12.099189,50.299773),(12.076141,50.315173),(12.101344,50.31398),(12.149315,50.311711),(12.168538,50.302977),(12.17443,50.293727),(12.17412,50.276571),(12.178047,50.268767),(12.189209,50.262463),(12.203782,50.259569),(12.232514,50.259104),(12.239645,50.256572),(12.242953,50.252954),(12.242436,50.248355),(12.237992,50.242722),(12.236441,50.242567),(12.235098,50.242257),(12.233754,50.241689),(12.232514,50.240965),(12.229827,50.239053),(12.228897,50.237038),(12.22993,50.234816),(12.232514,50.232491),(12.249981,50.221173),(12.254321,50.217401),(12.258455,50.21089),(12.260212,50.20505),(12.261246,50.199883),(12.273235,50.172339),(12.283363,50.162211),(12.300417,50.160815),(12.314369,50.167327),(12.314369,50.176215),(12.308582,50.186912),(12.305584,50.199314),(12.308685,50.217091),(12.314059,50.226496),(12.3338,50.242722),(12.336487,50.258587),(12.344652,50.26639),(12.355814,50.271403),(12.367183,50.279051),(12.398189,50.315173),(12.408937,50.321994),(12.438289,50.332536),(12.450175,50.339357),(12.453792,50.340701),(12.462784,50.340494),(12.466815,50.34189),(12.468985,50.345559),(12.468365,50.349486),(12.467022,50.352845),(12.466815,50.354912),(12.469502,50.359408),(12.471569,50.364472),(12.475083,50.369433),(12.482524,50.373567),(12.510223,50.38876),(12.550531,50.396357),(12.625358,50.399922),(12.678792,50.393721),(12.691814,50.394651),(12.702976,50.401524),(12.725094,50.423435),(12.737909,50.431548),(12.754239,50.435321),(12.770259,50.435941),(12.788139,50.434339),(12.79434,50.435889),(12.808396,50.441832),(12.817388,50.44302),(12.825759,50.441418),(12.918363,50.4054),(12.952573,50.40416),(12.960631,50.409234),(12.982442,50.42297),(12.996602,50.433667),(13.003113,50.451909),(13.009624,50.492681),(13.017892,50.49635),(13.026264,50.497642),(13.034532,50.496505),(13.042284,50.492785),(13.051069,50.491131),(13.06037,50.490614),(13.069775,50.491079),(13.07887,50.492681),(13.078974,50.492681),(13.079181,50.492785),(13.107396,50.498986),(13.160106,50.497022),(13.184704,50.508753),(13.19907,50.525341),(13.232349,50.58203),(13.25178,50.580893),(13.268006,50.573658),(13.284646,50.568956),(13.305627,50.575777),(13.316272,50.596034),(13.321853,50.60239),(13.35937,50.61934),(13.365365,50.623474),(13.365985,50.627092),(13.368775,50.628332),(13.380971,50.62549),(13.386345,50.621614),(13.400401,50.606576),(13.407016,50.601305),(13.42934,50.594329),(13.447944,50.597274),(13.464893,50.607093),(13.482257,50.620891),(13.492179,50.624456),(13.498173,50.629159),(13.499103,50.635205),(13.493315,50.643215),(13.509955,50.651225),(13.523288,50.66218),(13.527215,50.675668),(13.51564,50.691016),(13.556567,50.706725),(13.601939,50.712151),(13.691753,50.711841),(13.711028,50.71427),(13.749062,50.72321),(13.770663,50.724605),(13.816552,50.72135),(13.834535,50.723675),(13.863474,50.735147),(13.869572,50.735251),(13.875359,50.736542),(13.882181,50.742744),(13.882697,50.748015),(13.88032,50.755043),(13.87908,50.763518),(13.883318,50.773233),(13.892516,50.780416),(13.900991,50.780622),(13.910706,50.778762),(13.923315,50.779951),(13.933547,50.784808),(13.948843,50.797211),(13.959592,50.802068),(13.979126,50.804755),(14.015713,50.801758),(14.034936,50.802585),(14.190792,50.847905),(14.202988,50.85514),(14.221281,50.872607),(14.232133,50.879376),(14.2681,50.88413),(14.346545,50.880203),(14.375897,50.895964),(14.381892,50.920872),(14.355227,50.930639),(14.31833,50.937512),(14.292802,50.95348),(14.293732,50.96397),(14.302207,50.967691),(14.303757,50.969707),(14.28381,50.974822),(14.250013,50.977613),(14.238335,50.982471),(14.249497,50.992651),(14.263553,51.021435),(14.287531,51.036834),(14.319363,51.040012),(14.35657,51.032338),(14.364115,51.027998),(14.369386,51.022442),(14.375897,51.01699),(14.386749,51.01327),(14.397808,51.013115)] +Djibouti [(43.240733,11.48786),(43.188815,11.407764),(42.923715,10.998787),(42.908109,11.004187),(42.898084,10.995687),(42.888059,10.983568),(42.873279,10.978246),(42.860153,10.980416),(42.835452,10.987444),(42.794111,10.991604),(42.771787,10.996462),(42.754837,11.010544),(42.729619,11.06501),(42.710395,11.072116),(42.686521,11.073046),(42.615621,11.08966),(42.608145,11.089088),(42.569732,11.086146),(42.479712,11.059119),(42.413256,11.015944),(42.391758,11.005893),(42.376669,11.006668),(42.361166,11.011887),(42.338532,11.015504),(42.305872,11.004601),(42.297294,11.003774),(42.279414,11.004394),(42.270835,11.003516),(42.235179,10.987703),(42.221639,10.984344),(42.202726,10.98455),(42.151359,10.996203),(42.096272,10.990261),(42.069401,10.981501),(42.038395,10.950444),(42.0108,10.943597),(41.954162,10.941271),(41.947341,10.938532),(41.942793,10.933546),(41.937522,10.929825),(41.928841,10.931065),(41.922846,10.936026),(41.920262,10.941943),(41.916852,10.947059),(41.79872,10.970675),(41.773811,10.980054),(41.766523,10.989466),(41.761306,10.996203),(41.779289,11.024677),(41.785594,11.067388),(41.787971,11.259727),(41.775672,11.368429),(41.749834,11.483383),(41.74911,11.537953),(41.778669,11.628645),(41.792415,11.704455),(41.808228,11.736236),(41.823007,11.746106),(41.862282,11.763056),(41.872203,11.775871),(41.877784,11.790134),(41.886259,11.796025),(41.897525,11.799384),(41.911477,11.805947),(41.925327,11.816437),(41.936386,11.827445),(41.97468,11.88091),(42.057308,11.996271),(42.097719,12.070169),(42.108778,12.082106),(42.132343,12.093475),(42.140301,12.105826),(42.155494,12.142257),(42.288715,12.321006),(42.319205,12.386377),(42.379459,12.465907),(42.430929,12.520322),(42.448499,12.523371),(42.475681,12.516343),(42.521569,12.496241),(42.678252,12.360022),(42.691482,12.378832),(42.696187,12.380126),(42.727758,12.388806),(42.741401,12.408081),(42.749876,12.417073),(42.779745,12.422602),(42.792251,12.429165),(42.798038,12.456088),(42.788737,12.487456),(42.785223,12.517222),(42.828838,12.557633),(42.852092,12.611738),(42.868835,12.626155),(42.877723,12.62507),(42.890126,12.615407),(42.900358,12.616389),(43.117686,12.707913),(43.133962,12.697211),(43.143321,12.679511),(43.151134,12.659735),(43.161794,12.642727),(43.259613,12.533352),(43.283214,12.496527),(43.297374,12.482001),(43.312999,12.477688),(43.329356,12.488756),(43.329356,12.453559),(43.331228,12.436713),(43.336192,12.419908),(43.344086,12.406155),(43.353201,12.395209),(43.360525,12.383002),(43.363536,12.365302),(43.364757,12.327135),(43.367849,12.306871),(43.374034,12.292955),(43.411388,12.241767),(43.414317,12.224921),(43.411388,12.166653),(43.418712,12.091254),(43.413341,12.059272),(43.398774,12.029527),(43.377126,12.004828),(43.350434,11.987942),(43.34254,11.985663),(43.315684,11.981106),(43.288341,11.967434),(43.268403,11.963813),(43.225434,11.962877),(43.206554,11.957831),(43.174001,11.935004),(43.097179,11.84455),(43.09197,11.841295),(43.086029,11.838853),(43.076182,11.834703),(43.069184,11.830268),(43.064708,11.823432),(43.060883,11.809556),(43.055512,11.803616),(43.041515,11.798529),(42.986827,11.796088),(42.972504,11.790839),(42.95753,11.780463),(42.942638,11.77383),(42.915538,11.783596),(42.896495,11.780585),(42.803396,11.747992),(42.777192,11.732245),(42.75408,11.707953),(42.684093,11.582261),(42.677419,11.573676),(42.667979,11.568549),(42.636974,11.559556),(42.627696,11.559068),(42.60377,11.563381),(42.577403,11.562405),(42.569021,11.563381),(42.563487,11.567206),(42.554047,11.580268),(42.548595,11.583808),(42.526866,11.582465),(42.515147,11.574123),(42.514985,11.567206),(42.528087,11.570217),(42.532481,11.548489),(42.532481,11.547471),(42.533051,11.523505),(42.538748,11.502997),(42.571625,11.490465),(42.591563,11.47248),(42.60377,11.46776),(42.617686,11.470282),(42.633556,11.484809),(42.644786,11.488227),(42.653494,11.485907),(42.662852,11.481635),(42.672537,11.479722),(42.681651,11.484524),(42.687511,11.497219),(42.680675,11.504788),(42.670421,11.509955),(42.665294,11.51557),(42.674083,11.532131),(42.694102,11.545966),(42.739757,11.563381),(42.807465,11.571275),(42.829112,11.576972),(42.839529,11.584459),(42.847504,11.592719),(42.855805,11.595771),(42.866954,11.587592),(42.875173,11.583686),(42.900727,11.584052),(43.040538,11.585924),(43.06186,11.590155),(43.081798,11.589545),(43.103363,11.576972),(43.123057,11.581488),(43.133556,11.595282),(43.141124,11.608588),(43.151866,11.611762),(43.157888,11.601304),(43.172211,11.542873),(43.173106,11.541653),(43.183849,11.526557),(43.19516,11.518297),(43.226329,11.501899),(43.240733,11.48786)] +Dominica [(-61.362864,15.201809),(-61.374094,15.20482),(-61.376129,15.212836),(-61.372792,15.237209),(-61.37328,15.266425),(-61.372792,15.270697),(-61.378285,15.28205),(-61.39094,15.296291),(-61.393219,15.308295),(-61.395009,15.333075),(-61.399566,15.351142),(-61.413726,15.380561),(-61.415679,15.387641),(-61.41808,15.405951),(-61.421213,15.41474),(-61.444203,15.436835),(-61.447865,15.445136),(-61.449778,15.454657),(-61.454254,15.46483),(-61.460032,15.472886),(-61.465566,15.476142),(-61.472076,15.481391),(-61.474273,15.493476),(-61.475209,15.517768),(-61.47936,15.525458),(-61.484853,15.530341),(-61.488922,15.535712),(-61.488881,15.545071),(-61.483998,15.551581),(-61.467275,15.562974),(-61.462148,15.572984),(-61.469065,15.574286),(-61.481191,15.578559),(-61.488881,15.57925),(-61.480621,15.59101),(-61.468984,15.633857),(-61.449208,15.633083),(-61.437245,15.632636),(-61.425933,15.628485),(-61.413726,15.619574),(-61.397572,15.596625),(-61.384633,15.588365),(-61.36974,15.595933),(-61.35733,15.598293),(-61.307688,15.57925),(-61.305898,15.572455),(-61.29955,15.557196),(-61.289622,15.540961),(-61.284047,15.531399),(-61.268625,15.509914),(-61.267161,15.507799),(-61.25768,15.472602),(-61.256012,15.43594),(-61.262929,15.407904),(-61.258372,15.406317),(-61.254221,15.402981),(-61.249257,15.401068),(-61.255523,15.364569),(-61.254058,15.332668),(-61.253733,15.324693),(-61.256663,15.285346),(-61.277211,15.25023),(-61.284413,15.244818),(-61.289296,15.243598),(-61.293202,15.243883),(-61.303375,15.24258),(-61.311879,15.244534),(-61.318186,15.243394),(-61.323598,15.238593),(-61.328725,15.225572),(-61.342681,15.219672),(-61.345937,15.212348),(-61.346384,15.211697),(-61.350738,15.205146),(-61.362864,15.201809)] +Algeria [(8.60251,36.939511),(8.605655,36.913045),(8.604828,36.900979),(8.608239,36.890721),(8.642552,36.848501),(8.641725,36.836357),(8.623845,36.826074),(8.554495,36.803646),(8.520079,36.79791),(8.482665,36.799977),(8.444011,36.796205),(8.413109,36.783906),(8.406701,36.767989),(8.441634,36.75321),(8.461788,36.732798),(8.45321,36.697658),(8.430369,36.662621),(8.407941,36.642622),(8.286915,36.583401),(8.222319,36.563764),(8.193484,36.548985),(8.169919,36.525834),(8.16789,36.491995),(8.167852,36.491365),(8.20878,36.477826),(8.295597,36.46868),(8.314924,36.460618),(8.333837,36.456381),(8.349237,36.448784),(8.357712,36.430387),(8.359675,36.411215),(8.358125,36.369926),(8.355423,36.356116),(8.354404,36.350909),(8.307069,36.243629),(8.302314,36.202339),(8.307069,36.182495),(8.31451,36.163944),(8.317094,36.145909),(8.307999,36.126995),(8.296733,36.110407),(8.275649,36.03682),(8.272446,35.981732),(8.268423,35.969864),(8.247227,35.907318),(8.24144,35.827737),(8.24578,35.78531),(8.257666,35.750325),(8.306552,35.684903),(8.323605,35.652192),(8.327225,35.621405),(8.337041,35.537935),(8.336731,35.508376),(8.300868,35.437476),(8.290016,35.402595),(8.287948,35.366059),(8.294356,35.32508),(8.317404,35.289475),(8.355231,35.274334),(8.396882,35.263688),(8.431299,35.241726),(8.421377,35.222089),(8.360916,35.145918),(8.313683,35.103078),(8.300454,35.06768),(8.282884,34.994041),(8.25901,34.940659),(8.248881,34.901953),(8.250741,34.864746),(8.269345,34.763874),(8.266554,34.750283),(8.258714,34.741298),(8.257356,34.739741),(8.225213,34.711681),(8.213431,34.696746),(8.210641,34.681192),(8.236479,34.647654),(8.228831,34.63644),(8.192864,34.617733),(8.186559,34.609568),(8.179531,34.592153),(8.17519,34.58528),(8.167232,34.578562),(8.143668,34.566728),(8.102483,34.536355),(8.094058,34.530142),(7.999077,34.494175),(7.957323,34.469422),(7.87061,34.437899),(7.831542,34.414386),(7.811905,34.379272),(7.774285,34.260959),(7.765293,34.244707),(7.752787,34.232149),(7.733874,34.223778),(7.670312,34.215303),(7.631451,34.199102),(7.604063,34.175486),(7.579568,34.148459),(7.539228,34.113758),(7.517453,34.095026),(7.503707,34.067973),(7.500606,33.994257),(7.479832,33.893901),(7.484897,33.855015),(7.498539,33.799979),(7.505464,33.782849),(7.534196,33.736236),(7.544428,33.68425),(7.553006,33.658748),(7.616568,33.565963),(7.693153,33.454109),(7.708242,33.414861),(7.708286,33.410704),(7.709689,33.278151),(7.724985,33.231409),(7.75072,33.207664),(7.787514,33.198311),(7.83547,33.194538),(7.871953,33.184125),(7.982747,33.116817),(8.002178,33.108368),(8.022642,33.103045),(8.043622,33.101831),(8.0645,33.105784),(8.086824,33.094286),(8.181392,32.969823),(8.282884,32.836369),(8.296423,32.804407),(8.309963,32.663873),(8.319574,32.560598),(8.331253,32.527628),(8.332627,32.526355),(8.359985,32.501015),(8.482665,32.434791),(8.642345,32.336658),(8.851532,32.208242),(9.019893,32.104863),(9.033743,32.090756),(9.045008,32.071842),(9.063304,32.000192),(9.065989,31.989676),(9.094514,31.879606),(9.12273,31.769535),(9.151152,31.659567),(9.17947,31.549497),(9.207789,31.439426),(9.236211,31.329407),(9.264426,31.219336),(9.292952,31.109317),(9.32127,30.999246),(9.349589,30.889175),(9.377908,30.779156),(9.40633,30.669085),(9.434752,30.559066),(9.463071,30.449047),(9.491389,30.338976),(9.519708,30.228905),(9.286544,30.117129),(9.310005,30.084366),(9.369487,30.022794),(9.421729,29.968714),(9.54968,29.802316),(9.667709,29.608323),(9.746929,29.368428),(9.826149,29.128533),(9.848267,28.975726),(9.851264,28.785996),(9.827276,28.618647),(9.776953,28.267578),(9.789872,28.209442),(9.93591,27.866724),(9.934256,27.827398),(9.931274,27.818642),(9.863356,27.619246),(9.846613,27.599298),(9.818501,27.585733),(9.793903,27.569739),(9.797107,27.548914),(9.81168,27.526538),(9.821602,27.505687),(9.813643,27.486489),(9.770545,27.444217),(9.756283,27.42303),(9.743053,27.364119),(9.726517,27.32469),(9.721866,27.308463),(9.721349,27.291875),(9.806099,27.025122),(9.825529,26.92058),(9.835141,26.900995),(9.846096,26.891951),(9.890848,26.869576),(9.906661,26.857483),(9.910588,26.843117),(9.885474,26.736612),(9.88258,26.701782),(9.894052,26.67398),(9.896326,26.652793),(9.854571,26.524377),(9.835761,26.504223),(9.482604,26.352553),(9.481364,26.332606),(9.477643,26.315552),(9.468858,26.301031),(9.434752,26.271989),(9.416458,26.23225),(9.402506,26.216179),(9.377804,26.168946),(9.401162,26.113394),(9.541309,25.936351),(9.693961,25.743494),(9.816744,25.588465),(9.9695,25.395402),(10.007947,25.331426),(10.021383,25.26802),(10.025517,25.1364),(10.029961,24.995065),(10.032028,24.856339),(10.044534,24.829622),(10.193362,24.749937),(10.212172,24.722859),(10.229742,24.629945),(10.242248,24.595115),(10.260335,24.576641),(10.391799,24.47998),(10.410506,24.473288),(10.450194,24.476931),(10.566776,24.516463),(10.677363,24.553851),(10.699067,24.556125),(10.720668,24.552301),(10.911354,24.494501),(11.149996,24.422335),(11.46724,24.326339),(11.50863,24.313814),(11.541393,24.29751),(11.567128,24.26684),(11.636478,24.137649),(11.708618,24.00298),(11.71014,24.000137),(11.822306,23.790642),(11.89238,23.660056),(11.968861,23.517351),(11.900855,23.477173),(11.832745,23.437046),(11.764739,23.396945),(11.696733,23.356844),(11.628623,23.316717),(11.560514,23.276591),(11.492611,23.236516),(11.424501,23.196415),(11.356392,23.156262),(11.288489,23.116161),(11.220379,23.07606),(11.15227,23.035882),(11.084367,22.995755),(11.016257,22.955654),(10.948148,22.915527),(10.880038,22.875375),(10.812032,22.835326),(10.743923,22.795199),(10.675813,22.755046),(10.607807,22.714945),(10.539801,22.674896),(10.471691,22.634744),(10.403788,22.594643),(10.335679,22.554542),(10.267569,22.514337),(10.199563,22.474237),(10.131557,22.434136),(10.063447,22.394035),(9.995441,22.353882),(9.927435,22.313781),(9.859325,22.27368),(9.791319,22.233579),(9.723313,22.193427),(9.655203,22.153326),(9.587197,22.113225),(9.519088,22.073072),(9.451082,22.03292),(9.383075,21.992819),(9.314966,21.952718),(9.24696,21.912565),(9.178953,21.872516),(9.110844,21.832364),(9.042838,21.792263),(8.974832,21.75211),(8.906722,21.712061),(8.838613,21.671908),(8.770606,21.631807),(8.702497,21.591655),(8.634491,21.551502),(8.566381,21.51135),(8.498375,21.4713),(8.430265,21.431148),(8.362259,21.391047),(8.294253,21.350946),(8.226143,21.310845),(8.158034,21.270744),(8.090131,21.230592),(8.022022,21.190491),(7.953912,21.15039),(7.886009,21.110237),(7.8179,21.070085),(7.74979,21.029984),(7.681887,20.989831),(7.613778,20.94973),(7.482726,20.872577),(7.383611,20.791652),(7.312297,20.733464),(7.240881,20.675173),(7.169464,20.616986),(7.098047,20.558746),(7.020532,20.495443),(6.982808,20.46361),(6.925138,20.414001),(6.862092,20.35974),(6.799047,20.305532),(6.736105,20.251323),(6.672957,20.197115),(6.609963,20.142854),(6.546918,20.088646),(6.483769,20.034385),(6.420931,19.980177),(6.357782,19.925917),(6.294737,19.871682),(6.231692,19.817474),(6.168646,19.763213),(6.105601,19.709005),(6.042556,19.654745),(5.979407,19.600536),(5.916362,19.546276),(5.837607,19.478631),(5.794302,19.449796),(5.749344,19.433699),(5.620463,19.408997),(5.520831,19.389903),(5.301826,19.347993),(5.082821,19.306006),(4.983189,19.286963),(4.794467,19.250764),(4.605848,19.214616),(4.417229,19.178417),(4.22861,19.142244),(4.057561,19.110437),(3.886512,19.07863),(3.790291,19.06077),(3.715566,19.046901),(3.54462,19.015068),(3.439821,18.995638),(3.439717,18.995638),(3.358689,18.976853),(3.333057,18.975561),(3.318381,18.977706),(3.308356,18.981685),(3.284895,18.995741),(3.225984,19.05106),(3.179785,19.07),(3.158597,19.08155),(3.138754,19.096045),(3.120874,19.112814),(3.10413,19.135526),(3.102684,19.153561),(3.111779,19.171337),(3.126558,19.193352),(3.134206,19.212859),(3.13927,19.221929),(3.152706,19.230197),(3.17441,19.251643),(3.178855,19.268722),(3.183816,19.307505),(3.192911,19.325798),(3.211514,19.340862),(3.232701,19.351843),(3.250995,19.36546),(3.260813,19.388327),(3.25823,19.410393),(3.247481,19.426464),(3.234769,19.441347),(3.225984,19.459692),(3.22619,19.4692),(3.231668,19.489044),(3.232288,19.495478),(3.228051,19.504159),(3.222883,19.508061),(3.217509,19.511162),(3.212238,19.517156),(3.199422,19.553769),(3.198285,19.592397),(3.216785,19.794064),(3.212754,19.807758),(3.198802,19.820523),(3.183195,19.827731),(3.147022,19.837938),(3.130485,19.845224),(3.072608,19.88889),(2.946001,19.941652),(2.671805,19.996222),(2.616925,19.998367),(2.525665,20.015162),(2.514812,20.015937),(2.495382,20.020097),(2.459312,20.038778),(2.439882,20.04609),(2.415697,20.051284),(2.400401,20.056555),(2.388722,20.067407),(2.348208,20.137635),(2.316478,20.180165),(2.279581,20.21794),(2.21819,20.264087),(2.200826,20.273906),(2.18243,20.278505),(2.161346,20.274939),(2.138195,20.260728),(2.097474,20.224193),(2.071222,20.213263),(2.056339,20.215046),(1.993397,20.235949),(1.983372,20.241918),(1.975621,20.248429),(1.967146,20.253416),(1.95526,20.254915),(1.941204,20.251116),(1.924254,20.23613),(1.913402,20.231092),(1.891388,20.231789),(1.883843,20.24414),(1.880329,20.263054),(1.870614,20.283543),(1.855008,20.294835),(1.838885,20.29592),(1.820901,20.293594),(1.799197,20.294886),(1.778113,20.304291),(1.659154,20.397516),(1.649232,20.412089),(1.650059,20.487019),(1.643961,20.522676),(1.623704,20.551253),(1.559729,20.597504),(1.520351,20.616986),(1.483454,20.622618),(1.465781,20.633522),(1.447487,20.638741),(1.40718,20.645046),(1.363978,20.657707),(1.346925,20.669127),(1.331526,20.687937),(1.310545,20.722716),(1.296696,20.733464),(1.273338,20.739407),(1.252254,20.738994),(1.212463,20.73088),(1.191276,20.73057),(1.168538,20.733464),(1.154585,20.738787),(1.147247,20.751448),(1.14456,20.776252),(1.145284,20.795889),(1.167505,20.886013),(1.180114,20.995309),(1.177943,21.017323),(1.15934,21.081505),(1.146524,21.101711),(1.029322,21.178347),(0.942609,21.235036),(0.855999,21.291673),(0.769389,21.34831),(0.682676,21.404896),(0.596067,21.461585),(0.509457,21.518274),(0.422795,21.574963),(0.33616,21.631601),(0.249499,21.688238),(0.162966,21.744824),(0.076253,21.801461),(0.003735,21.848899),(-0.010408,21.85815),(-0.097044,21.914839),(-0.183731,21.971477),(-0.270392,22.028114),(-0.357028,22.084751),(-0.443663,22.141389),(-0.530325,22.198078),(-0.616934,22.254767),(-0.703647,22.311404),(-0.790309,22.368041),(-0.876918,22.424627),(-0.96358,22.481265),(-1.05019,22.537954),(-1.136903,22.594643),(-1.223512,22.65128),(-1.310122,22.707969),(-1.396783,22.764555),(-1.483496,22.821166),(-1.570054,22.877881),(-1.656716,22.93457),(-1.743326,22.991182),(-1.830039,23.047819),(-1.916648,23.104456),(-2.00331,23.161042),(-2.089919,23.217757),(-2.176632,23.274446),(-2.26319,23.331109),(-2.349852,23.387747),(-2.436513,23.444358),(-2.523226,23.500996),(-2.609836,23.557685),(-2.696446,23.614348),(-2.783107,23.670985),(-2.869768,23.727623),(-2.956404,23.784234),(-3.043065,23.840923),(-3.129649,23.897612),(-3.209489,23.949702),(-3.289123,24.001844),(-3.368808,24.053959),(-3.448544,24.106023),(-3.528281,24.158113),(-3.608018,24.210203),(-3.687625,24.262344),(-3.767336,24.314434),(-3.846996,24.366576),(-3.926732,24.418717),(-4.006495,24.470807),(-4.086154,24.522897),(-4.165865,24.574987),(-4.245498,24.627077),(-4.325235,24.679218),(-4.404946,24.731308),(-4.516025,24.803991),(-4.592351,24.851688),(-4.668677,24.899334),(-4.744925,24.947057),(-4.821226,24.994755),(-4.821355,24.994806),(-4.821536,24.994961),(-4.821613,24.995065),(-4.995194,25.102086),(-5.16875,25.20916),(-5.342331,25.316182),(-5.515964,25.423256),(-5.63699,25.494466),(-5.661525,25.508881),(-5.804628,25.592961),(-5.972266,25.691611),(-6.139905,25.79021),(-6.307491,25.888808),(-6.505412,26.005184),(-6.703333,26.121559),(-6.901202,26.237934),(-7.099123,26.354361),(-7.297043,26.470737),(-7.494912,26.587164),(-7.692782,26.703487),(-7.890754,26.819914),(-8.088623,26.936238),(-8.28644,27.052665),(-8.484413,27.168989),(-8.682385,27.285416),(-8.682385,27.379467),(-8.682385,27.473466),(-8.682385,27.567414),(-8.682385,27.661439),(-8.682385,27.721436),(-8.682385,27.781354),(-8.682385,27.841222),(-8.682385,27.90114),(-8.682385,27.961085),(-8.682385,28.02103),(-8.682385,28.080949),(-8.682385,28.140893),(-8.682385,28.20076),(-8.682385,28.260679),(-8.682385,28.320624),(-8.682385,28.380543),(-8.682385,28.440487),(-8.682385,28.500406),(-8.682385,28.560273),(-8.682385,28.620218),(-8.682385,28.6659),(-8.678768,28.692823),(-8.667606,28.711685),(-8.648847,28.725948),(-8.520819,28.787081),(-8.475835,28.818759),(-8.430411,28.841006),(-8.417802,28.852349),(-8.383489,28.905782),(-8.368451,28.916531),(-8.333259,28.93038),(-8.316774,28.939062),(-8.250474,28.994769),(-8.182312,29.035541),(-8.069658,29.079337),(-8.036326,29.099853),(-7.945014,29.176231),(-7.839129,29.239043),(-7.777996,29.289324),(-7.729989,29.311158),(-7.714667,29.321829),(-7.653611,29.376193),(-7.619453,29.389422),(-7.572686,29.387613),(-7.528502,29.380947),(-7.506126,29.380223),(-7.48406,29.382445),(-7.463286,29.389137),(-7.34961,29.383439),(-7.258755,29.467305),(-7.146934,29.509238),(-7.070057,29.516227),(-6.958236,29.509238),(-6.783515,29.446339),(-6.699649,29.516227),(-6.559872,29.530205),(-6.413107,29.565149),(-6.27333,29.579127),(-6.126564,29.579127),(-6.000765,29.579127),(-5.881955,29.600093),(-5.756156,29.614071),(-5.721212,29.523216),(-5.637346,29.495261),(-5.539503,29.523216),(-5.43467,29.642026),(-5.343815,29.767825),(-5.273927,29.886635),(-5.176083,29.97749),(-5.071251,30.04039),(-4.95943,30.124256),(-4.875564,30.180166),(-4.770731,30.229088),(-4.623966,30.284999),(-4.484189,30.382842),(-4.372368,30.508641),(-4.274524,30.557563),(-4.155714,30.585519),(-4.001959,30.592507),(-3.834228,30.627452),(-3.645529,30.711317),(-3.652518,30.774217),(-3.659507,30.837116),(-3.610585,30.879049),(-3.554674,30.955927),(-3.608741,31.030872),(-3.610085,31.050302),(-3.614529,31.068027),(-3.624141,31.086527),(-3.635872,31.095674),(-3.671942,31.110867),(-3.689718,31.125595),(-3.717107,31.163267),(-3.731421,31.176341),(-3.748862,31.180217),(-3.763719,31.173447),(-3.792968,31.149676),(-3.810796,31.142906),(-3.827462,31.143837),(-3.839322,31.152828),(-3.842836,31.170192),(-3.836169,31.189777),(-3.814982,31.220524),(-3.812915,31.243365),(-3.81922,31.318865),(-3.815189,31.337158),(-3.802502,31.350646),(-3.747467,31.385166),(-3.673484,31.389234),(-3.659507,31.647821),(-3.591378,31.678274),(-3.548745,31.669954),(-3.511564,31.672745),(-3.21921,31.717709),(-3.002556,31.77362),(-2.827836,31.794586),(-2.869769,31.89243),(-2.938731,32.048639),(-2.881189,32.076286),(-2.695567,32.08967),(-2.516147,32.1322),(-2.378636,32.126774),(-2.315177,32.124294),(-2.243398,32.1214),(-2.165109,32.118299),(-2.081858,32.11494),(-1.995455,32.111478),(-1.907553,32.107964),(-1.819962,32.104501),(-1.734334,32.101091),(-1.652375,32.097835),(-1.575791,32.094735),(-1.506337,32.091944),(-1.445618,32.089515),(-1.395388,32.087552),(-1.357406,32.086053),(-1.333428,32.085071),(-1.324953,32.084709),(-1.249557,32.08166),(-1.210335,32.08967),(-1.190594,32.125224),(-1.195607,32.146049),(-1.211833,32.158348),(-1.232711,32.163723),(-1.251831,32.163516),(-1.288986,32.150907),(-1.305161,32.151165),(-1.309554,32.167443),(-1.305678,32.173722),(-1.289193,32.184858),(-1.282992,32.190181),(-1.276636,32.200852),(-1.275189,32.209069),(-1.275499,32.217518),(-1.273423,32.229442),(-1.257515,32.320845),(-1.244131,32.356915),(-1.234158,32.374614),(-1.217983,32.392623),(-1.201705,32.399858),(-1.16026,32.404948),(-1.123157,32.417945),(-1.08998,32.439442),(-1.031999,32.4944),(-1.047502,32.517009),(-1.32702,32.69891),(-1.390531,32.718779),(-1.423345,32.742395),(-1.558789,32.93365),(-1.516363,32.959488),(-1.50887,32.966309),(-1.502978,32.974629),(-1.498844,32.984008),(-1.496519,32.994292),(-1.493367,33.016151),(-1.493057,33.039483),(-1.499516,33.060205),(-1.516363,33.073977),(-1.545508,33.091831),(-1.571398,33.111985),(-1.592069,33.136609),(-1.605608,33.168028),(-1.623591,33.196605),(-1.674234,33.237972),(-1.683381,33.270839),(-1.683226,33.36923),(-1.672839,33.394604),(-1.659145,33.41977),(-1.640386,33.475529),(-1.6254,33.494236),(-1.612739,33.521469),(-1.617338,33.554439),(-1.6624,33.644692),(-1.673252,33.6565),(-1.690719,33.667326),(-1.725394,33.677713),(-1.740742,33.686601),(-1.746788,33.702388),(-1.74224,33.717762),(-1.732577,33.727865),(-1.72095,33.736366),(-1.710821,33.747063),(-1.703225,33.761816),(-1.702501,33.772823),(-1.713043,33.801995),(-1.722087,33.851165),(-1.718728,33.898087),(-1.672115,34.059188),(-1.669635,34.079213),(-1.672085,34.092021),(-1.674751,34.105956),(-1.746064,34.290311),(-1.771334,34.334701),(-1.809626,34.372451),(-1.702966,34.479679),(-1.714232,34.485054),(-1.750664,34.494175),(-1.871121,34.596649),(-1.862957,34.613599),(-1.810505,34.680727),(-1.786114,34.72584),(-1.773143,34.734057),(-1.769526,34.741343),(-1.787716,34.756691),(-1.892825,34.811675),(-1.926725,34.838081),(-1.979745,34.865263),(-1.993491,34.878854),(-1.9984,34.892703),(-1.999692,34.906346),(-2.00362,34.91818),(-2.01628,34.926241),(-2.061239,34.929704),(-2.094932,34.947687),(-2.126041,34.971923),(-2.1633,34.994041),(-2.193789,35.003601),(-2.211669,35.023445),(-2.221126,35.049955),(-2.222564,35.089301),(-2.200103,35.096137),(-2.196116,35.094875),(-2.192616,35.094468),(-2.189443,35.093004),(-2.186391,35.08869),(-2.15689,35.100979),(-2.122222,35.094672),(-2.08609,35.081977),(-2.052358,35.075019),(-1.987701,35.082465),(-1.964752,35.076239),(-1.956451,35.075019),(-1.949818,35.077135),(-1.932485,35.08869),(-1.838694,35.110907),(-1.812734,35.126532),(-1.804351,35.128119),(-1.774241,35.127346),(-1.761138,35.129625),(-1.745595,35.138821),(-1.712799,35.1706),(-1.648834,35.188422),(-1.637685,35.194525),(-1.629506,35.212226),(-1.510365,35.295478),(-1.490875,35.300971),(-1.395904,35.309394),(-1.369537,35.315863),(-1.364003,35.318345),(-1.358957,35.322211),(-1.332672,35.349351),(-1.313791,35.353746),(-1.29483,35.364569),(-1.278635,35.377997),(-1.26773,35.390326),(-1.259185,35.410305),(-1.247304,35.45775),(-1.23705,35.475735),(-1.225413,35.491278),(-1.190338,35.566148),(-1.183339,35.576483),(-1.171539,35.581041),(-1.14802,35.582099),(-1.142812,35.5869),(-1.117014,35.616889),(-1.106353,35.623236),(-1.082753,35.637356),(-1.052073,35.660956),(-1.040395,35.676418),(-1.032948,35.682603),(-0.996693,35.689602),(-0.977651,35.699774),(-0.961537,35.711249),(-0.946278,35.719306),(-0.922678,35.725165),(-0.910227,35.724311),(-0.904693,35.715888),(-0.899403,35.711819),(-0.887563,35.715969),(-0.849477,35.73432),(-0.825022,35.768541),(-0.801666,35.773912),(-0.793528,35.770087),(-0.786122,35.763129),(-0.77717,35.756415),(-0.764027,35.753404),(-0.754018,35.752265),(-0.734486,35.747219),(-0.723134,35.745998),(-0.711293,35.742987),(-0.705881,35.735663),(-0.701324,35.726874),(-0.692454,35.719306),(-0.682607,35.717353),(-0.644521,35.719306),(-0.626454,35.722968),(-0.606109,35.730943),(-0.589345,35.742499),(-0.57726,35.770819),(-0.565053,35.772447),(-0.551259,35.769029),(-0.540924,35.767768),(-0.530426,35.772895),(-0.523427,35.77969),(-0.47704,35.859361),(-0.476064,35.870103),(-0.478424,35.87287),(-0.480336,35.879096),(-0.480824,35.885728),(-0.479482,35.890041),(-0.473785,35.891262),(-0.468007,35.888739),(-0.46288,35.885199),(-0.459055,35.883775),(-0.448964,35.884345),(-0.428131,35.882799),(-0.41804,35.883775),(-0.376129,35.900336),(-0.348541,35.90705),(-0.336171,35.900824),(-0.331451,35.889553),(-0.320709,35.884101),(-0.309193,35.881171),(-0.30191,35.87759),(-0.298492,35.867174),(-0.304596,35.853502),(-0.30191,35.842841),(-0.285553,35.828274),(-0.259429,35.817532),(-0.188059,35.805243),(-0.14509,35.790351),(-0.123891,35.786933),(-0.113678,35.788804),(-0.083079,35.794379),(-0.038482,35.8133),(0.00294,35.838039),(0.034516,35.863267),(0.069957,35.92182),(0.08017,35.947008),(0.114024,36.010199),(0.122406,36.033352),(0.12379,36.044582),(0.127289,36.050767),(0.13559,36.055609),(0.145518,36.059963),(0.153982,36.065009),(0.202159,36.106513),(0.219493,36.116523),(0.283458,36.134426),(0.294444,36.140774),(0.299083,36.147406),(0.321056,36.158881),(0.329275,36.164984),(0.332774,36.173814),(0.337576,36.197496),(0.342296,36.205959),(0.360037,36.213853),(0.384451,36.217963),(0.428396,36.219631),(0.451345,36.223212),(0.468272,36.232245),(0.483165,36.243598),(0.556651,36.284084),(0.594574,36.293158),(0.611827,36.305487),(0.627208,36.319241),(0.644054,36.328843),(0.663259,36.332587),(0.740408,36.337755),(0.753266,36.338609),(0.768728,36.344713),(0.802257,36.363593),(0.864757,36.377387),(0.880544,36.387152),(0.89088,36.396389),(0.913829,36.409125),(0.925141,36.418158),(0.929535,36.42593),(0.93214,36.434556),(0.93629,36.44359),(0.945567,36.452338),(0.962576,36.458157),(1.007091,36.466864),(1.04477,36.486884),(1.116954,36.487128),(1.138845,36.493394),(1.178722,36.510891),(1.202647,36.51439),(1.245453,36.513577),(1.265473,36.515326),(1.288259,36.521226),(1.34669,36.544989),(1.367686,36.547919),(1.385997,36.545152),(1.429535,36.532213),(1.45281,36.528022),(1.476329,36.528957),(1.517426,36.538723),(1.700938,36.547187),(1.716563,36.547919),(1.726736,36.549994),(1.750173,36.559149),(1.761241,36.561591),(1.77003,36.560492),(1.791515,36.555487),(1.802908,36.555365),(1.857595,36.56745),(1.913829,36.570217),(1.96046,36.562201),(1.97283,36.561591),(2.059906,36.570136),(2.309825,36.63052),(2.3296,36.637274),(2.351573,36.638332),(2.373302,36.633857),(2.391856,36.624254),(2.398204,36.615302),(2.400157,36.606187),(2.404063,36.599189),(2.416026,36.59634),(2.426768,36.595282),(2.447276,36.590562),(2.457042,36.589504),(2.600759,36.59634),(2.632986,36.605129),(2.72755,36.665229),(2.792979,36.692939),(2.818696,36.710883),(2.843598,36.740383),(2.849864,36.754055),(2.875662,36.774807),(2.933116,36.80858),(2.977061,36.815863),(2.984874,36.814968),(3.014171,36.811713),(3.047862,36.794989),(3.073985,36.771389),(3.101817,36.751899),(3.140147,36.741645),(3.173269,36.742621),(3.179373,36.742825),(3.21046,36.757392),(3.221202,36.77143),(3.226085,36.7862),(3.227306,36.812323),(3.237804,36.813056),(3.302989,36.788764),(3.340505,36.780707),(3.460704,36.775092),(3.480479,36.777248),(3.519054,36.786607),(3.539806,36.788764),(3.550466,36.791449),(3.558116,36.797675),(3.564301,36.804348),(3.570567,36.80858),(3.580414,36.809963),(3.601817,36.808092),(3.612071,36.80858),(3.649669,36.823961),(3.709483,36.873196),(3.741873,36.891181),(3.827485,36.912909),(3.873871,36.917222),(3.912608,36.911607),(3.948253,36.893297),(3.964041,36.891181),(3.974864,36.892279),(3.99643,36.896918),(4.031423,36.897773),(4.03948,36.896918),(4.08546,36.892076),(4.104828,36.883734),(4.146495,36.901801),(4.159516,36.904202),(4.17156,36.902167),(4.197032,36.893256),(4.21046,36.891181),(4.22283,36.894192),(4.249278,36.907864),(4.261974,36.911607),(4.278575,36.910468),(4.306,36.900458),(4.320323,36.898017),(4.440684,36.911607),(4.452973,36.910061),(4.46811,36.905992),(4.482432,36.900458),(4.491954,36.894599),(4.50408,36.889309),(4.536388,36.888577),(4.550548,36.883734),(4.581554,36.894924),(4.591482,36.89525),(4.622895,36.896389),(4.698009,36.891181),(4.769054,36.898017),(4.78712,36.895413),(4.94988,36.839789),(4.968272,36.82567),(4.973643,36.818915),(4.985688,36.817084),(4.999522,36.817125),(5.009288,36.816067),(5.016449,36.810981),(5.025238,36.798651),(5.029145,36.794989),(5.045909,36.78852),(5.065603,36.783515),(5.086192,36.780829),(5.104828,36.781317),(5.084809,36.732611),(5.09018,36.716783),(5.118419,36.699368),(5.23585,36.650946),(5.269054,36.64411),(5.304373,36.643012),(5.372406,36.650946),(5.428071,36.663886),(5.433849,36.665229),(5.464854,36.665229),(5.469574,36.667711),(5.482188,36.680243),(5.489106,36.685126),(5.529145,36.697659),(5.549001,36.709703),(5.562755,36.745998),(5.576182,36.761379),(5.593923,36.773871),(5.61199,36.781317),(5.655935,36.791815),(5.674571,36.799628),(5.71046,36.825507),(5.733735,36.832343),(5.7588,36.833401),(5.82781,36.821763),(5.868826,36.822943),(6.03712,36.853583),(6.197032,36.90233),(6.235688,36.92121),(6.256847,36.945787),(6.258556,36.966783),(6.254731,36.983303),(6.254405,36.998969),(6.266856,37.017768),(6.274425,37.024237),(6.314626,37.048651),(6.328868,37.061916),(6.338715,37.069281),(6.371593,37.083482),(6.415782,37.092963),(6.462413,37.09394),(6.502696,37.082953),(6.538585,37.06037),(6.5442,37.058783),(6.544688,37.04857),(6.547048,37.041205),(6.552257,37.036689),(6.56129,37.035142),(6.572765,37.027777),(6.583263,36.989814),(6.599457,36.97309),(6.614106,36.977851),(6.631602,36.972398),(6.651134,36.963853),(6.671153,36.959418),(6.822032,36.952623),(6.834809,36.949123),(6.852794,36.940334),(6.86964,36.929185),(6.880056,36.918443),(6.879649,36.902086),(6.909434,36.893052),(6.948578,36.890041),(6.975597,36.891181),(7.064464,36.913316),(7.092296,36.92593),(7.108084,36.920315),(7.133311,36.915269),(7.156993,36.9147),(7.167491,36.922187),(7.176443,36.931871),(7.218761,36.960842),(7.232921,36.966864),(7.239024,36.971503),(7.247081,36.982327),(7.254242,36.994696),(7.257335,37.004136),(7.255544,37.015367),(7.250499,37.023139),(7.211111,37.057034),(7.181651,37.076728),(7.181651,37.082953),(7.192719,37.084296),(7.212169,37.089301),(7.222667,37.089789),(7.232758,37.087226),(7.251313,37.078599),(7.260509,37.076728),(7.262706,37.075995),(7.266449,37.074693),(7.270681,37.071031),(7.27475,37.069322),(7.281016,37.073065),(7.288259,37.080268),(7.292654,37.082506),(7.297699,37.082953),(7.307302,37.080146),(7.325857,37.071194),(7.335948,37.069281),(7.344412,37.071438),(7.369965,37.082953),(7.383474,37.082994),(7.386241,37.082099),(7.386892,37.078111),(7.394054,37.069281),(7.398204,37.05801),(7.404633,37.051581),(7.418224,37.048814),(7.438324,37.04857),(7.447113,37.046617),(7.456065,37.041978),(7.472423,37.050035),(7.490245,37.054999),(7.526134,37.033149),(7.555919,37.00788),(7.588878,36.987616),(7.633556,36.980536),(7.643403,36.982652),(7.652192,36.986233),(7.661876,36.988471),(7.67449,36.986762),(7.685802,36.982408),(7.695974,36.976996),(7.704845,36.970526),(7.712738,36.963202),(7.760753,36.966864),(7.770763,36.97012),(7.789073,36.986558),(7.798595,36.993598),(7.791026,36.973212),(7.778575,36.951117),(7.774262,36.931464),(7.791189,36.918443),(7.773448,36.88996),(7.773774,36.889757),(7.809906,36.86815),(7.866466,36.854397),(7.908458,36.849555),(8.049978,36.87759),(8.073009,36.887397),(8.093028,36.901801),(8.233165,36.958075),(8.25115,36.948676),(8.268728,36.933743),(8.288097,36.92593),(8.333263,36.927802),(8.355968,36.92593),(8.374034,36.918443),(8.388845,36.923245),(8.401622,36.918606),(8.414399,36.910224),(8.428722,36.904202),(8.463227,36.901557),(8.497569,36.904202),(8.601899,36.939358),(8.60251,36.939511)] +Ethiopia [(38.13331,14.677762),(38.190258,14.687322),(38.213925,14.685875),(38.229325,14.679725),(38.239557,14.667555),(38.245841,14.652474),(38.247825,14.647712),(38.252683,14.62022),(38.257644,14.610427),(38.267152,14.601771),(38.278831,14.595467),(38.28927,14.587741),(38.295678,14.574874),(38.305186,14.536917),(38.313351,14.519347),(38.32503,14.504594),(38.340636,14.492682),(38.394276,14.464312),(38.402338,14.456871),(38.409056,14.43527),(38.41443,14.426304),(38.426832,14.417183),(38.435204,14.416305),(38.443679,14.418656),(38.492978,14.418294),(38.569356,14.426821),(38.606563,14.436484),(38.668471,14.467723),(38.688005,14.467103),(38.706195,14.46116),(38.724385,14.459403),(38.742885,14.461056),(38.761902,14.465449),(38.821537,14.489478),(38.866082,14.493664),(38.884582,14.506247),(38.917552,14.535703),(38.959203,14.554642),(38.979667,14.567122),(38.993826,14.585157),(38.996823,14.60358),(38.995273,14.622261),(38.996823,14.638901),(39.009742,14.651226),(39.011189,14.647763),(39.04695,14.643758),(39.075475,14.637402),(39.0855,14.633475),(39.107308,14.620116),(39.127461,14.602211),(39.144721,14.581566),(39.158467,14.560146),(39.187509,14.477205),(39.20818,14.440179),(39.233191,14.440438),(39.254999,14.474415),(39.268538,14.48643),(39.293446,14.490305),(39.347603,14.481805),(39.36662,14.482916),(39.448579,14.505111),(39.489403,14.524489),(39.50718,14.547821),(39.515035,14.56782),(39.532295,14.567587),(39.552138,14.556684),(39.567331,14.544307),(39.587692,14.520278),(39.59379,14.514567),(39.599474,14.511648),(39.650944,14.496584),(39.677712,14.494233),(39.722985,14.501717),(39.735073,14.503715),(39.766906,14.504775),(39.796878,14.495111),(39.824473,14.478135),(39.87708,14.433616),(39.892273,14.426562),(39.911186,14.42341),(39.93072,14.428784),(39.961519,14.454209),(39.979812,14.458731),(39.999839,14.455238),(40.023221,14.45116),(40.037483,14.451522),(40.086886,14.465759),(40.104559,14.465966),(40.194373,14.444649),(40.237678,14.427441),(40.277055,14.404574),(40.353687,14.335241),(40.421646,14.273755),(40.479834,14.244403),(40.585874,14.19469),(40.614089,14.185802),(40.704419,14.173219),(40.772322,14.148362),(40.833197,14.105962),(40.93779,13.989922),(41.043623,13.872488),(41.121345,13.738181),(41.192038,13.615863),(41.222217,13.586614),(41.357093,13.502278),(41.530726,13.393783),(41.635525,13.309344),(41.7118,13.247926),(41.750764,13.208316),(41.784767,13.164572),(41.8166,13.112792),(41.894321,12.94797),(41.946617,12.876166),(42.024649,12.82697),(42.0817,12.803922),(42.137304,12.77333),(42.184949,12.733048),(42.250165,12.623985),(42.33171,12.515465),(42.379459,12.465907),(42.319205,12.386377),(42.288715,12.321006),(42.155494,12.142257),(42.140301,12.105826),(42.132343,12.093475),(42.108778,12.082106),(42.097719,12.070169),(42.057308,11.996271),(41.97468,11.88091),(41.936386,11.827445),(41.925327,11.816437),(41.911477,11.805947),(41.897525,11.799384),(41.886259,11.796025),(41.877784,11.790134),(41.872203,11.775871),(41.862282,11.763056),(41.823007,11.746106),(41.808228,11.736236),(41.792415,11.704455),(41.778669,11.628645),(41.74911,11.537953),(41.749834,11.483383),(41.775672,11.368429),(41.787971,11.259727),(41.785594,11.067388),(41.779289,11.024677),(41.761306,10.996203),(41.766523,10.989466),(41.773811,10.980054),(41.79872,10.970675),(41.916852,10.947059),(41.920262,10.941943),(41.922846,10.936026),(41.928841,10.931065),(41.937522,10.929825),(41.942793,10.933546),(41.947341,10.938532),(41.954162,10.941271),(42.0108,10.943597),(42.038395,10.950444),(42.069401,10.981501),(42.096272,10.990261),(42.151359,10.996203),(42.202726,10.98455),(42.221639,10.984344),(42.235179,10.987703),(42.270835,11.003516),(42.279414,11.004394),(42.297294,11.003774),(42.305872,11.004601),(42.338532,11.015504),(42.361166,11.011887),(42.376669,11.006668),(42.391758,11.005893),(42.413256,11.015944),(42.479712,11.059119),(42.569732,11.086146),(42.608145,11.089088),(42.615621,11.08966),(42.686521,11.073046),(42.710395,11.072116),(42.729619,11.06501),(42.754837,11.010544),(42.771787,10.996462),(42.794111,10.991604),(42.835452,10.987444),(42.860153,10.980416),(42.873279,10.978246),(42.888059,10.983568),(42.898084,10.995687),(42.908109,11.004187),(42.923715,10.998787),(42.911313,10.977367),(42.901185,10.936595),(42.893536,10.919671),(42.876793,10.905744),(42.834832,10.888406),(42.819432,10.872567),(42.811578,10.85262),(42.80765,10.83611),(42.801242,10.820788),(42.785636,10.804406),(42.749876,10.775829),(42.738094,10.759784),(42.728585,10.735496),(42.718973,10.694852),(42.711635,10.675163),(42.699646,10.65855),(42.680216,10.647672),(42.660269,10.641625),(42.647247,10.63222),(42.64859,10.611033),(42.668537,10.56623),(42.695202,10.524708),(42.750393,10.471636),(42.765792,10.451947),(42.776644,10.42461),(42.78946,10.333324),(42.808167,10.269116),(42.836279,10.208086),(42.862324,10.177158),(42.958235,10.115559),(42.981903,10.091633),(42.999887,10.061971),(43.012599,10.029286),(43.020764,9.996419),(43.039884,9.949988),(43.067686,9.922522),(43.104273,9.907923),(43.149852,9.899991),(43.187214,9.883326),(43.206334,9.851209),(43.235273,9.691787),(43.248812,9.652332),(43.270671,9.628354),(43.297698,9.621533),(43.305966,9.617554),(43.315578,9.607658),(43.325086,9.585902),(43.331753,9.575076),(43.341364,9.566394),(43.361518,9.553165),(43.370665,9.544225),(43.393351,9.49893),(43.399449,9.480947),(43.401981,9.447409),(43.406322,9.42865),(43.419034,9.413018),(43.471227,9.382012),(43.548225,9.336072),(43.566829,9.334315),(43.59122,9.343591),(43.60724,9.344625),(43.621502,9.336899),(43.698449,9.267162),(43.78759,9.186701),(43.914921,9.071489),(43.984788,9.008314),(44.023855,8.985525),(44.104057,8.959222),(44.192217,8.930283),(44.280377,8.901396),(44.368537,8.872457),(44.456697,8.843544),(44.544753,8.814605),(44.633017,8.785718),(44.721073,8.756779),(44.809233,8.72784),(44.89729,8.698928),(44.985553,8.67004),(45.073558,8.641153),(45.16177,8.612137),(45.249878,8.583276),(45.33809,8.554414),(45.426147,8.525476),(45.51441,8.496537),(45.514462,8.496537),(45.598436,8.468425),(45.682358,8.440313),(45.766384,8.412227),(45.850203,8.384115),(45.934126,8.356003),(46.018049,8.327891),(46.101919,8.299727),(46.185894,8.271641),(46.269816,8.243555),(46.353842,8.215469),(46.437713,8.187357),(46.521687,8.159245),(46.605713,8.131133),(46.689532,8.102969),(46.773558,8.074909),(46.857377,8.046745),(46.920526,8.025609),(46.97923,7.996567),(47.068114,7.996516),(47.210586,7.996516),(47.36608,7.996516),(47.52359,7.996516),(47.653918,7.996516),(47.836697,7.996516),(47.979169,7.996567),(47.930593,7.949697),(47.881811,7.902826),(47.833183,7.855956),(47.784659,7.809137),(47.736032,7.762215),(47.687507,7.715396),(47.638932,7.668525),(47.590252,7.621655),(47.541676,7.574836),(47.532388,7.565864),(47.493101,7.527914),(47.444525,7.481043),(47.395949,7.434173),(47.347373,7.387302),(47.298797,7.34038),(47.250118,7.293509),(47.201439,7.246691),(47.152863,7.19982),(47.104287,7.153001),(47.055608,7.106079),(47.007032,7.059208),(46.95856,7.012286),(46.909984,6.965467),(46.861408,6.918597),(46.812729,6.871675),(46.764101,6.824882),(46.715577,6.777985),(46.667001,6.731115),(46.618425,6.684244),(46.598065,6.664633),(46.55321,6.621457),(46.508406,6.578308),(46.488046,6.558645),(46.466963,6.538292),(46.423915,6.496736),(46.362937,6.43426),(46.302062,6.371731),(46.240981,6.309228),(46.179899,6.246726),(46.118818,6.184223),(46.057839,6.12172),(45.99681,6.059192),(45.935986,5.996689),(45.828706,5.879513),(45.721426,5.762337),(45.614146,5.645057),(45.518477,5.540564),(45.506865,5.527881),(45.399585,5.410704),(45.292356,5.293476),(45.185024,5.176248),(45.077744,5.059072),(45.0209,4.99688),(45.020642,4.99688),(44.941525,4.911484),(44.912586,4.899392),(44.858378,4.902544),(44.806546,4.905541),(44.754766,4.908565),(44.702883,4.911562),(44.651104,4.914507),(44.59922,4.917505),(44.596808,4.917643),(44.547441,4.920476),(44.495557,4.923473),(44.443726,4.926445),(44.391998,4.929468),(44.340115,4.932439),(44.288335,4.935488),(44.236452,4.938485),(44.184672,4.941431),(44.132789,4.94448),(44.081009,4.947451),(44.029126,4.950448),(43.968975,4.953962),(43.932284,4.945203),(43.845158,4.914352),(43.815186,4.907479),(43.716484,4.884793),(43.640519,4.867353),(43.528485,4.841566),(43.459342,4.808829),(43.346997,4.755654),(43.232792,4.701497),(43.119259,4.647702),(43.03544,4.578895),(42.960406,4.517374),(42.952551,4.507581),(42.945936,4.496858),(42.9325,4.463217),(42.914724,4.393299),(42.899634,4.361104),(42.870195,4.328171),(42.868938,4.326765),(42.831628,4.302322),(42.78977,4.285605),(42.718663,4.273487),(42.568285,4.247856),(42.415013,4.221707),(42.297294,4.201734),(42.221846,4.200959),(42.13534,4.200081),(42.102887,4.19189),(42.068367,4.174553),(42.011833,4.129491),(41.941243,4.086212),(41.923466,4.070554),(41.916645,4.051098),(41.917472,4.020453),(41.912201,4.007974),(41.898455,3.996915),(41.885019,3.977226),(41.835927,3.94945),(41.791381,3.958157),(41.747353,3.981412),(41.699811,3.996915),(41.657436,3.972704),(41.642967,3.969449),(41.628704,3.972084),(41.602246,3.983065),(41.586329,3.984151),(41.506231,3.964384),(41.479773,3.962938),(41.429543,3.948882),(41.317405,3.942422),(41.215293,3.936608),(41.162996,3.94268),(41.114524,3.962343),(41.070082,3.996915),(41.0059,4.086677),(40.979751,4.110758),(40.963628,4.12037),(40.915879,4.136648),(40.897689,4.145924),(40.887354,4.156052),(40.870611,4.186981),(40.846013,4.214989),(40.785758,4.25584),(40.763744,4.284933),(40.700699,4.243515),(40.6175,4.211605),(40.511873,4.171064),(40.382682,4.121377),(40.376998,4.116081),(40.371106,4.099932),(40.365732,4.094867),(40.286874,4.067453),(40.185588,4.032158),(40.182797,4.032003),(40.1798,4.03283),(40.176803,4.034535),(40.167191,4.035465),(40.166261,4.031486),(40.166984,4.025259),(40.162437,4.019394),(40.117168,3.996915),(40.020223,3.950122),(39.93413,3.908703),(39.848451,3.867284),(39.828297,3.842583),(39.80866,3.790287),(39.780755,3.71657),(39.764218,3.685745),(39.745098,3.660579),(39.666447,3.588723),(39.600818,3.528907),(39.574979,3.497901),(39.553172,3.431316),(39.536325,3.405013),(39.504389,3.403333),(39.488473,3.413901),(39.480928,3.42726),(39.475657,3.440799),(39.466976,3.452013),(39.436176,3.462374),(39.311326,3.466094),(39.320525,3.484543),(39.315874,3.494904),(39.302748,3.495679),(39.296338,3.49199),(39.257583,3.469686),(39.219445,3.468704),(39.180481,3.476998),(39.090441,3.518334),(39.077542,3.524256),(39.06793,3.526582),(39.010466,3.514464),(38.994963,3.514257),(38.979977,3.519812),(38.963234,3.522086),(38.921272,3.513999),(38.895227,3.513585),(38.82071,3.533739),(38.722008,3.560404),(38.703715,3.570429),(38.688108,3.586294),(38.66103,3.621951),(38.661443,3.615336),(38.6601,3.601306),(38.660306,3.594459),(38.601912,3.59867),(38.596641,3.602133),(38.595608,3.607507),(38.593747,3.61084),(38.579588,3.605311),(38.57411,3.604561),(38.561088,3.606008),(38.547135,3.609522),(38.543208,3.615388),(38.541244,3.623294),(38.533182,3.632828),(38.514269,3.648512),(38.508998,3.650812),(38.499386,3.64691),(38.497319,3.63859),(38.498146,3.629676),(38.496906,3.62394),(38.445953,3.601668),(38.389935,3.596888),(38.283068,3.608902),(38.176925,3.620917),(38.145609,3.618437),(38.114087,3.610608),(38.101891,3.612649),(38.078843,3.632673),(38.048974,3.641846),(38.036158,3.648926),(38.020759,3.667167),(37.995334,3.707914),(37.976731,3.72644),(37.945931,3.746232),(37.846196,3.810337),(37.751008,3.871522),(37.647758,3.937874),(37.556084,3.996889),(37.475056,4.048979),(37.360954,4.122204),(37.263492,4.184888),(37.166031,4.247545),(37.111874,4.282298),(37.105983,4.283848),(37.099988,4.283693),(37.094821,4.284882),(37.091617,4.290256),(37.086449,4.312477),(37.082522,4.322166),(37.075907,4.33108),(37.068466,4.333458),(37.048415,4.331959),(37.041077,4.336817),(37.025367,4.363275),(37.015652,4.370587),(36.974828,4.37994),(36.903721,4.415778),(36.844087,4.432237),(36.651333,4.430583),(36.648857,4.43118),(36.642962,4.432599),(36.628182,4.441461),(36.619501,4.443373),(36.463025,4.440169),(36.277196,4.436449),(36.264277,4.438206),(36.246087,4.446991),(36.236268,4.449652),(36.225726,4.4496),(36.19131,4.444924),(36.045325,4.443716),(36.041345,4.443683),(36.018401,4.449135),(35.99711,4.462959),(35.958352,4.496858),(35.940472,4.508046),(35.933755,4.52218),(35.934065,4.539207),(35.937062,4.559258),(35.936028,4.578533),(35.922799,4.602072),(35.920835,4.619332),(35.784513,4.764181),(35.760018,4.805728),(35.751647,4.854356),(35.755884,5.063439),(35.760949,5.076332),(35.799189,5.124624),(35.807354,5.144674),(35.807147,5.165267),(35.798259,5.189271),(35.778415,5.22715),(35.775315,5.24658),(35.782033,5.26986),(35.809111,5.309109),(35.80415,5.318023),(35.749373,5.339158),(35.683641,5.379595),(35.658319,5.386417),(35.639819,5.382076),(35.621939,5.373782),(35.598168,5.368743),(35.57388,5.375332),(35.530782,5.410084),(35.508044,5.423417),(35.490164,5.428042),(35.469287,5.430755),(35.448203,5.430755),(35.430219,5.427215),(35.408205,5.412565),(35.367484,5.368046),(35.344953,5.352362),(35.320769,5.348719),(35.302372,5.357374),(35.287489,5.374092),(35.254623,5.425923),(35.250695,5.435147),(35.251832,5.447369),(35.258964,5.458143),(35.267645,5.468556),(35.27364,5.479873),(35.269299,5.491785),(35.261961,5.511887),(35.22341,5.542996),(35.143312,5.58966),(35.098663,5.622474),(35.088225,5.64175),(35.079543,5.699782),(35.06528,5.726447),(34.984355,5.841014),(34.974433,5.863106),(34.972676,5.876257),(34.974537,5.887239),(34.977741,5.898659),(34.979911,5.912844),(34.97681,5.924032),(34.960377,5.941706),(34.955313,5.952558),(34.955726,5.964211),(34.95924,5.975502),(34.969782,5.996689),(34.967964,6.008131),(34.959447,6.061724),(34.951489,6.08118),(34.934436,6.102368),(34.915522,6.119705),(34.898882,6.13867),(34.879969,6.187634),(34.844725,6.248689),(34.839764,6.268688),(34.839041,6.327599),(34.832633,6.353541),(34.792739,6.421676),(34.784264,6.44183),(34.776099,6.499449),(34.753878,6.556423),(34.743543,6.596808),(34.733518,6.637606),(34.726593,6.641947),(34.714501,6.655667),(34.709643,6.674012),(34.703545,6.684916),(34.635539,6.729823),(34.618486,6.736541),(34.596679,6.739202),(34.553994,6.739254),(34.53632,6.743078),(34.524745,6.752871),(34.517924,6.793824),(34.516683,6.798062),(34.512136,6.808268),(34.511102,6.814314),(34.513273,6.815296),(34.522575,6.821781),(34.524745,6.824572),(34.523298,6.844183),(34.519061,6.861649),(34.512343,6.876842),(34.503661,6.89002),(34.439169,6.934875),(34.297575,6.968568),(34.294475,6.972237),(34.287653,6.975286),(34.280729,6.980092),(34.275665,6.997403),(34.270807,7.003398),(34.229879,7.03337),(34.219957,7.046134),(34.213756,7.051974),(34.206005,7.054506),(34.198253,7.064376),(34.195773,7.086959),(34.195773,7.09774),(34.195773,7.129023),(34.190915,7.149539),(34.181407,7.167212),(34.167764,7.175377),(34.151124,7.167471),(34.132417,7.163388),(34.113194,7.177548),(34.085909,7.211499),(34.04002,7.247259),(34.033612,7.250256),(34.031132,7.255372),(34.030615,7.3487),(34.02369,7.382445),(34.006534,7.409936),(33.882407,7.536079),(33.852538,7.553545),(33.842099,7.555922),(33.821429,7.558403),(33.811507,7.560987),(33.761071,7.598142),(33.729445,7.642119),(33.71606,7.657156),(33.706552,7.661859),(33.675029,7.670851),(33.647744,7.688627),(33.637202,7.691315),(33.572297,7.685217),(33.550851,7.691315),(33.540361,7.698911),(33.526511,7.717101),(33.516796,7.726093),(33.506823,7.73095),(33.488477,7.73617),(33.479537,7.743146),(33.462329,7.749502),(33.436491,7.748572),(33.393289,7.739735),(33.384401,7.735446),(33.359803,7.719272),(33.347142,7.718961),(33.29159,7.733069),(33.273503,7.749192),(33.257794,7.767434),(33.242394,7.780715),(33.233919,7.783195),(33.201467,7.788156),(33.172941,7.799628),(33.164518,7.801127),(33.132117,7.793944),(33.126949,7.791567),(33.120851,7.783299),(33.107519,7.785211),(33.085401,7.794978),(33.058323,7.797975),(33.051295,7.801127),(33.047677,7.807277),(33.044629,7.816423),(33.040753,7.824846),(33.023493,7.835078),(33.015018,7.850633),(32.9898,7.917244),(32.993572,7.928148),(33.006801,7.942462),(33.007569,7.944058),(33.012021,7.953314),(33.016775,7.958327),(33.021633,7.965251),(33.026077,7.986284),(33.031141,7.996309),(33.043853,8.013465),(33.100594,8.071033),(33.106589,8.08049),(33.111343,8.090644),(33.11341,8.099093),(33.117544,8.104158),(33.147516,8.109067),(33.164776,8.117051),(33.177282,8.127102),(33.185137,8.14108),(33.187824,8.160898),(33.184723,8.166247),(33.171081,8.175962),(33.16798,8.181362),(33.16829,8.195392),(33.169944,8.205495),(33.173975,8.21459),(33.181623,8.225726),(33.190511,8.232419),(33.199865,8.235519),(33.206893,8.240738),(33.208288,8.253683),(33.194852,8.259239),(33.185757,8.26451),(33.178419,8.275672),(33.171391,8.283113),(33.164363,8.292777),(33.161159,8.304585),(33.163123,8.312052),(33.171804,8.330888),(33.174078,8.343058),(33.172321,8.34652),(33.163329,8.352954),(33.161159,8.356106),(33.161882,8.361506),(33.166843,8.368276),(33.16798,8.372823),(33.167877,8.389722),(33.16953,8.397421),(33.174078,8.404475),(33.187824,8.411038),(33.202913,8.414113),(33.210355,8.420779),(33.207358,8.426593),(33.206427,8.430572),(33.23516,8.455635),(33.25273,8.458167),(33.361353,8.434293),(33.370862,8.437393),(33.384608,8.448013),(33.392101,8.451862),(33.400162,8.452638),(33.413133,8.449718),(33.480519,8.45765),(33.489924,8.462921),(33.490338,8.466513),(33.484757,8.474006),(33.505117,8.474884),(33.521964,8.465324),(33.537053,8.453206),(33.552143,8.446824),(33.57116,8.450131),(33.585371,8.458115),(33.600099,8.463955),(33.620407,8.460699),(33.651672,8.434499),(33.671309,8.400289),(33.695545,8.372875),(33.74102,8.367087),(33.751872,8.368793),(33.756213,8.370136),(33.797658,8.405612),(33.808096,8.412614),(33.824116,8.419487),(33.840859,8.423983),(33.875172,8.427936),(33.931706,8.428143),(33.95031,8.433207),(33.970774,8.445377),(34.069579,8.533615),(34.090249,8.556507),(34.103169,8.579322),(34.107251,8.601531),(34.11185,8.626555),(34.097473,8.915836),(34.083096,9.205117),(34.070698,9.454592),(34.066892,9.531176),(34.098208,9.67972),(34.221921,10.02603),(34.280832,10.080109),(34.303466,10.113544),(34.305017,10.124603),(34.301606,10.148994),(34.301813,10.160285),(34.304603,10.165634),(34.314215,10.175452),(34.317419,10.181266),(34.325894,10.223563),(34.32486,10.268651),(34.276181,10.488715),(34.271737,10.530082),(34.279489,10.565506),(34.365582,10.685783),(34.404235,10.759835),(34.420048,10.780816),(34.437308,10.795673),(34.557818,10.877787),(34.574768,10.884143),(34.587067,10.879285),(34.708093,10.774072),(34.737445,10.755391),(34.750674,10.743945),(34.756979,10.728804),(34.752948,10.698728),(34.754395,10.684129),(34.764834,10.680254),(34.772895,10.688263),(34.786228,10.718985),(34.798527,10.729656),(34.815373,10.73131),(34.827569,10.72777),(34.838524,10.72914),(34.85134,10.745753),(34.857334,10.763556),(34.857438,10.775674),(34.861572,10.787586),(34.879762,10.804613),(34.933816,10.845308),(34.950662,10.869183),(34.95831,10.903418),(34.952832,10.91843),(34.92317,10.939928),(34.913558,10.952976),(34.914282,10.963957),(34.980324,11.155263),(34.984148,11.186398),(34.971436,11.206888),(34.953969,11.226112),(34.944254,11.253242),(34.947148,11.274868),(35.073135,11.54896),(35.076856,11.568649),(35.074375,11.587356),(35.053498,11.628077),(35.046263,11.648644),(35.048124,11.68952),(35.040579,11.727399),(35.041199,11.747604),(35.048227,11.771376),(35.059079,11.79587),(35.072722,11.818866),(35.088018,11.838193),(35.126672,11.862946),(35.148556,11.871229),(35.215555,11.896588),(35.245734,11.929867),(35.247802,11.939634),(35.248379,11.957858),(35.248525,11.962475),(35.251419,11.971725),(35.260411,11.980407),(35.297824,12.001594),(35.317358,12.023712),(35.32604,12.047483),(35.335962,12.102312),(35.353738,12.14603),(35.361283,12.157347),(35.371102,12.165512),(35.392909,12.176726),(35.403348,12.185097),(35.412546,12.198636),(35.41544,12.211142),(35.418127,12.238996),(35.428772,12.266953),(35.616875,12.575151),(35.625866,12.585124),(35.634445,12.588587),(35.653772,12.590034),(35.662453,12.593651),(35.672375,12.604296),(35.675992,12.614528),(35.677439,12.625225),(35.684261,12.650753),(35.684054,12.656076),(35.687671,12.659228),(35.70137,12.665421),(35.702761,12.66605),(35.728392,12.67437),(36.029976,12.717158),(36.038451,12.717468),(36.050647,12.715452),(36.057261,12.712145),(36.063049,12.707908),(36.072144,12.70305),(36.099429,12.69566),(36.115035,12.70181),(36.123614,12.721447),(36.143251,12.832551),(36.143044,12.856942),(36.138703,12.871851),(36.115346,12.911719),(36.114415,12.933889),(36.125991,12.950968),(36.140874,12.96554),(36.149865,12.980242),(36.149555,12.988821),(36.146351,12.997373),(36.138393,13.013315),(36.136946,13.032436),(36.233581,13.36164),(36.382099,13.571033),(36.391298,13.599352),(36.394812,13.655938),(36.398429,13.67604),(36.439942,13.776176),(36.458167,13.820139),(36.455376,13.870007),(36.438013,13.919952),(36.432535,13.944499),(36.432122,13.969924),(36.436566,13.991214),(36.52638,14.263523),(36.545707,14.263162),(36.553045,14.282773),(36.565654,14.292927),(36.5825,14.296751),(36.603791,14.29732),(36.61764,14.300446),(36.648543,14.31458),(36.668593,14.317784),(36.713655,14.317784),(36.72213,14.320057),(36.738357,14.329798),(36.748072,14.332072),(36.816388,14.332072),(36.848531,14.327783),(36.913746,14.309024),(36.946923,14.304735),(36.952814,14.300963),(36.970901,14.27683),(36.978859,14.27112),(36.986714,14.266831),(36.995395,14.264092),(37.005627,14.263162),(37.011518,14.265074),(37.026091,14.27683),(37.034256,14.277502),(37.052446,14.275331),(37.060197,14.27683),(37.072703,14.285305),(37.083452,14.297733),(37.090997,14.314218),(37.093787,14.335173),(37.094407,14.35915),(37.097818,14.379511),(37.105879,14.399226),(37.121072,14.420826),(37.149701,14.441936),(37.184221,14.449558),(37.258221,14.448111),(37.280029,14.45545),(37.293775,14.457439),(37.318166,14.43713),(37.33708,14.426356),(37.374907,14.373026),(37.390203,14.358169),(37.394027,14.350081),(37.395474,14.335173),(37.394957,14.312383),(37.397748,14.301738),(37.412217,14.293599),(37.452628,14.260578),(37.456969,14.254609),(37.464307,14.235825),(37.469371,14.210813),(37.475366,14.199574),(37.488182,14.194897),(37.498517,14.193088),(37.508749,14.188127),(37.5165,14.180557),(37.519498,14.170997),(37.520634,14.145443),(37.525285,14.128829),(37.535724,14.11684),(37.553707,14.105497),(37.564973,14.116685),(37.601043,14.200917),(37.658507,14.335173),(37.721242,14.481753),(37.785631,14.632183),(37.85064,14.783879),(37.891464,14.879532),(37.908828,14.864727),(37.92092,14.847519),(37.94004,14.81088),(37.944381,14.80734),(37.954096,14.806255),(37.95761,14.802844),(37.958127,14.798762),(37.957817,14.788194),(37.958437,14.785093),(37.962261,14.777859),(37.964432,14.771193),(37.968772,14.766387),(37.980038,14.764552),(37.989856,14.757317),(37.992027,14.749514),(37.99275,14.740755),(37.998435,14.730911),(38.006703,14.7244),(38.115223,14.681017),(38.13331,14.677762)] +Georgia [(40.479834,43.508741),(40.519624,43.505175),(40.550837,43.511247),(40.615536,43.535845),(40.651916,43.538971),(40.681062,43.528068),(40.689966,43.523129),(40.740179,43.495279),(40.842706,43.469389),(40.874952,43.455643),(41.011791,43.375416),(41.042487,43.369834),(41.146666,43.37986),(41.180773,43.374692),(41.213122,43.362781),(41.243921,43.344203),(41.278751,43.333196),(41.350271,43.34335),(41.380657,43.334772),(41.389752,43.324876),(41.403808,43.301518),(41.41311,43.291286),(41.427579,43.283199),(41.479773,43.271236),(41.506128,43.25979),(41.550569,43.226277),(41.577544,43.215296),(41.614131,43.213022),(41.687408,43.220205),(41.723789,43.217234),(41.788901,43.203566),(41.82032,43.20124),(41.88998,43.207028),(41.919332,43.205943),(41.948271,43.200077),(41.97969,43.188657),(42.011316,43.181861),(42.043356,43.18057),(42.075292,43.185272),(42.154334,43.217498),(42.159834,43.21974),(42.185363,43.225476),(42.354551,43.236018),(42.36256,43.235194),(42.422867,43.22899),(42.479712,43.200904),(42.506067,43.181551),(42.534902,43.165067),(42.59588,43.140029),(42.615207,43.13698),(42.629366,43.14127),(42.656238,43.159796),(42.673602,43.165893),(42.704917,43.166436),(42.75153,43.176952),(42.79132,43.176177),(42.829768,43.168761),(42.857673,43.155455),(42.885785,43.135844),(42.914827,43.121943),(43.000093,43.098662),(43.000197,43.085356),(42.992548,43.068819),(42.989758,43.05267),(42.998853,43.042206),(43.048256,43.016962),(43.155588,42.942393),(43.175228,42.934471),(43.194345,42.926761),(43.386178,42.886882),(43.479599,42.867462),(43.517426,42.86046),(43.586259,42.835629),(43.603002,42.824803),(43.644343,42.788578),(43.662843,42.779845),(43.755964,42.763127),(43.782526,42.753213),(43.800613,42.746462),(43.811775,42.717265),(43.788727,42.69308),(43.753794,42.674657),(43.725682,42.6544),(43.722581,42.624867),(43.751107,42.598745),(43.796169,42.58983),(43.845003,42.58704),(43.885362,42.579211),(43.908161,42.566283),(43.920812,42.559109),(43.939726,42.55221),(43.958433,42.554406),(43.973212,42.563992),(43.998017,42.589365),(44.011504,42.600321),(44.055068,42.615307),(44.150824,42.618666),(44.185803,42.626508),(44.197384,42.629105),(44.211351,42.639634),(44.230147,42.653806),(44.248648,42.681995),(44.272005,42.702382),(44.360682,42.703751),(44.401403,42.713518),(44.479228,42.744291),(44.508115,42.750286),(44.558293,42.753102),(44.600977,42.749123),(44.638759,42.740149),(44.642422,42.739279),(44.679422,42.723672),(44.695029,42.714216),(44.724484,42.691039),(44.73637,42.677551),(44.761071,42.619234),(44.773473,42.609648),(44.78231,42.61665),(44.798485,42.671738),(44.827424,42.717962),(44.833676,42.734085),(44.85936,42.75951),(44.902974,42.753076),(45.012115,42.695302),(45.029788,42.688765),(45.048082,42.685897),(45.066995,42.688455),(45.076018,42.691412),(45.103892,42.700547),(45.122961,42.702278),(45.155362,42.693261),(45.184198,42.674838),(45.282693,42.586394),(45.293648,42.572906),(45.301571,42.559842),(45.312562,42.54172),(45.32269,42.530118),(45.35597,42.519964),(45.398551,42.524382),(45.47927,42.54234),(45.536838,42.541358),(45.620657,42.530377),(45.698688,42.507587),(45.739616,42.471466),(45.739506,42.470647),(45.736412,42.447539),(45.62138,42.224478),(45.631405,42.201172),(45.661585,42.186703),(45.701789,42.17275),(45.811601,42.115492),(45.881364,42.102031),(45.895162,42.091876),(45.923481,42.039631),(45.934126,42.028495),(45.947562,42.022914),(45.966372,42.022061),(45.973503,42.024154),(45.986733,42.031906),(45.995156,42.033456),(46.001615,42.030976),(46.018359,42.020046),(46.02673,42.01679),(46.044817,42.017721),(46.05846,42.02033),(46.068278,42.015318),(46.074996,41.993484),(46.099181,41.985061),(46.129876,41.984493),(46.188581,41.993536),(46.188891,41.993639),(46.189201,41.993769),(46.197676,41.9955),(46.206151,41.996094),(46.214574,41.9955),(46.224341,41.993484),(46.250489,41.977723),(46.294156,41.93863),(46.322629,41.929018),(46.382057,41.925246),(46.406552,41.91504),(46.430892,41.890442),(46.392754,41.831685),(46.371309,41.805692),(46.328314,41.766418),(46.313741,41.756651),(46.297773,41.750812),(46.276947,41.749313),(46.230129,41.757323),(46.208321,41.755514),(46.188994,41.740141),(46.179537,41.705879),(46.180002,41.655159),(46.192767,41.610174),(46.21969,41.593302),(46.228475,41.598728),(46.233436,41.607952),(46.24005,41.615497),(46.2539,41.615911),(46.256174,41.610329),(46.278394,41.578393),(46.279871,41.571172),(46.287851,41.532143),(46.29674,41.510387),(46.313948,41.493747),(46.366244,41.466566),(46.377923,41.462483),(46.398284,41.458711),(46.445103,41.425948),(46.542461,41.394994),(46.591554,41.372928),(46.607367,41.345023),(46.62287,41.349622),(46.629841,41.346136),(46.631034,41.345539),(46.637752,41.336961),(46.657079,41.320631),(46.688499,41.284355),(46.694803,41.269782),(46.691703,41.268542),(46.683434,41.269472),(46.674649,41.261669),(46.672686,41.253969),(46.6716,41.236451),(46.669482,41.229888),(46.663591,41.223531),(46.650361,41.214178),(46.64447,41.207977),(46.634755,41.181053),(46.629484,41.14674),(46.621113,41.114597),(46.615539,41.108666),(46.601786,41.09403),(46.582148,41.091653),(46.563132,41.095581),(46.543546,41.097027),(46.522411,41.087002),(46.511145,41.070983),(46.505564,41.055273),(46.496779,41.044834),(46.47492,41.044111),(46.456988,41.052431),(46.429031,41.080956),(46.41265,41.09186),(46.353119,41.106536),(46.332345,41.115889),(46.273433,41.1563),(46.244391,41.183741),(46.228578,41.195419),(46.210595,41.200174),(46.135406,41.199915),(46.113133,41.19604),(46.087608,41.183659),(46.061353,41.170925),(46.041613,41.165447),(46.017428,41.163793),(45.969163,41.168289),(45.820748,41.208804),(45.770337,41.232102),(45.751088,41.240998),(45.709747,41.267715),(45.685872,41.29624),(45.708093,41.317117),(45.731038,41.323267),(45.7453,41.329158),(45.74406,41.335359),(45.720702,41.342387),(45.65559,41.352361),(45.511826,41.39458),(45.47927,41.41768),(45.464801,41.430185),(45.450021,41.432201),(45.416948,41.424604),(45.396794,41.423261),(45.38124,41.425586),(45.333096,41.444352),(45.314422,41.451631),(45.281711,41.452923),(45.24962,41.4445),(45.097834,41.349919),(45.0024,41.290452),(44.989998,41.284458),(44.967363,41.269007),(44.955168,41.262702),(44.917237,41.261772),(44.820706,41.273347),(44.801275,41.258516),(44.809543,41.244409),(44.847061,41.230766),(44.85382,41.223514),(44.857706,41.219346),(44.847784,41.208907),(44.821532,41.20622),(44.674926,41.208494),(44.63312,41.221051),(44.612863,41.223118),(44.591366,41.21578),(44.578446,41.202809),(44.566251,41.186944),(44.551523,41.177023),(44.531008,41.181467),(44.52553,41.189063),(44.521912,41.199295),(44.516641,41.20684),(44.505893,41.206582),(44.499382,41.202809),(44.479228,41.185808),(44.45494,41.180485),(44.432461,41.181673),(44.410601,41.18772),(44.357271,41.212163),(44.344921,41.213248),(44.333087,41.210354),(44.324922,41.204825),(44.317739,41.198572),(44.308489,41.193817),(44.284976,41.190872),(44.266476,41.196195),(44.218107,41.221568),(44.190977,41.230043),(44.180331,41.229939),(44.168446,41.223583),(44.165862,41.216142),(44.16519,41.207667),(44.160074,41.1979),(44.144364,41.186066),(44.124727,41.178056),(44.103643,41.175162),(44.061269,41.184154),(44.041322,41.182552),(44.001117,41.168858),(43.978186,41.164617),(43.977755,41.164537),(43.955849,41.160486),(43.863245,41.158988),(43.820353,41.1455),(43.773741,41.114494),(43.754156,41.108241),(43.729713,41.106691),(43.566622,41.124054),(43.524144,41.122814),(43.470142,41.106174),(43.460324,41.104521),(43.450453,41.104676),(43.440428,41.106588),(43.451797,41.132581),(43.437586,41.156197),(43.410559,41.175162),(43.383274,41.187203),(43.351751,41.193611),(43.322606,41.19206),(43.230622,41.17263),(43.216204,41.180537),(43.192226,41.224978),(43.172021,41.242342),(43.152332,41.24415),(43.130318,41.24229),(43.103136,41.248801),(43.157241,41.26973),(43.171866,41.279342),(43.185353,41.293811),(43.18401,41.298979),(43.154761,41.301873),(43.12391,41.312932),(43.075644,41.344868),(43.002057,41.382695),(42.987588,41.394684),(42.957719,41.437007),(42.949967,41.443673),(42.941079,41.446308),(42.936221,41.450236),(42.93219,41.454783),(42.926093,41.459073),(42.896637,41.466566),(42.888989,41.469976),(42.880721,41.481242),(42.87514,41.493696),(42.868008,41.500207),(42.854572,41.493799),(42.829974,41.472508),(42.811164,41.477159),(42.792147,41.492869),(42.766516,41.504341),(42.774887,41.514263),(42.794628,41.52682),(42.802689,41.534107),(42.806823,41.542995),(42.80889,41.552917),(42.812198,41.562994),(42.819949,41.572347),(42.800932,41.57922),(42.660992,41.588264),(42.610556,41.58506),(42.590914,41.580169),(42.586648,41.579107),(42.585235,41.578755),(42.565184,41.567128),(42.554952,41.550333),(42.545547,41.509612),(42.535522,41.493489),(42.513818,41.476229),(42.483846,41.442174),(42.463072,41.431839),(42.451033,41.43137),(42.437854,41.430857),(42.262464,41.482327),(42.213268,41.480208),(42.18929,41.481707),(42.17265,41.493489),(42.158387,41.499897),(42.143195,41.500259),(42.111775,41.493489),(42.097513,41.498347),(42.083457,41.500104),(42.069401,41.498605),(42.054931,41.493489),(42.019585,41.485066),(41.947961,41.505581),(41.907757,41.493489),(41.894011,41.485531),(41.862695,41.451786),(41.822594,41.426),(41.812982,41.421814),(41.800787,41.42569),(41.760996,41.453543),(41.747767,41.45685),(41.718518,41.459589),(41.706839,41.463155),(41.702601,41.46946),(41.702808,41.477624),(41.703945,41.485117),(41.702601,41.489045),(41.694953,41.489097),(41.669839,41.480467),(41.63997,41.47871),(41.627257,41.480518),(41.520763,41.514228),(41.586274,41.602729),(41.615733,41.632636),(41.676036,41.662177),(41.692882,41.692288),(41.739757,41.749905),(41.742361,41.761664),(41.742442,41.770697),(41.743907,41.778632),(41.750011,41.787502),(41.766287,41.804185),(41.771821,41.812567),(41.773936,41.821601),(41.773936,41.90261),(41.773936,41.902615),(41.773936,41.906928),(41.761974,41.969875),(41.760265,41.99258),(41.755707,42.006415),(41.715261,42.059353),(41.684906,42.098944),(41.674327,42.105536),(41.664073,42.113959),(41.65919,42.133002),(41.657237,42.16706),(41.649181,42.212226),(41.596202,42.349433),(41.592296,42.355414),(41.585216,42.36286),(41.55836,42.382799),(41.554861,42.389838),(41.551199,42.40644),(41.551195,42.406457),(41.500173,42.64057),(41.482758,42.680243),(41.457367,42.71369),(41.424327,42.739895),(41.383474,42.75727),(41.307302,42.767646),(41.291026,42.774359),(41.274913,42.785305),(41.250743,42.795233),(41.225352,42.802314),(41.205333,42.804429),(41.193696,42.801907),(41.173595,42.792792),(41.160899,42.790839),(41.150238,42.79564),(41.104747,42.846584),(41.081798,42.908149),(41.040294,42.96133),(41.023123,42.989976),(40.998383,42.989447),(40.958995,42.976386),(40.933767,42.982733),(40.903982,43.016343),(40.886567,43.024156),(40.886241,43.029283),(40.862804,43.058987),(40.855968,43.063056),(40.783051,43.084174),(40.682872,43.094306),(40.636241,43.092475),(40.604015,43.086493),(40.591807,43.085639),(40.579926,43.088813),(40.56072,43.102973),(40.550548,43.106106),(40.546072,43.108832),(40.528982,43.12226),(40.520193,43.127265),(40.510265,43.12995),(40.478526,43.134019),(40.398285,43.16234),(40.361176,43.165473),(40.335134,43.140855),(40.325043,43.14883),(40.310313,43.175727),(40.301036,43.188666),(40.278331,43.205512),(40.268403,43.216376),(40.272797,43.247992),(40.257172,43.277411),(40.235199,43.304429),(40.219086,43.319037),(40.188243,43.329576),(40.133474,43.34162),(40.101329,43.36518),(40.086925,43.370307),(40.040782,43.3758),(40.009776,43.385159),(39.986013,43.388983),(39.985976,43.38899),(39.991285,43.406318),(40.059187,43.535199),(40.07283,43.551064),(40.096705,43.562639),(40.164297,43.575843),(40.23313,43.575765),(40.479834,43.508741)] +Ghana [(-0.166109,11.13498),(-0.158668,11.118444),(-0.14208,11.103329),(-0.121745,11.092192),(-0.103348,11.087541),(-0.085029,11.089402),(-0.051077,11.098264),(-0.032267,11.098574),(0.001116,11.085991),(0.016205,11.062582),(0.019409,11.031628),(0.016567,10.996203),(0.01419,10.983672),(0.007188,10.976799),(-0.001571,10.971295),(-0.009581,10.963001),(-0.014439,10.953596),(-0.036091,10.853137),(-0.035316,10.843448),(-0.0302,10.823785),(-0.032061,10.813165),(-0.04069,10.805026),(-0.061103,10.791358),(-0.07521,10.773659),(-0.082807,10.756218),(-0.088129,10.714412),(-0.091564,10.700254),(-0.09769,10.675008),(-0.098336,10.654157),(-0.088129,10.633486),(-0.079344,10.626252),(-0.068363,10.621058),(-0.056452,10.618009),(-0.033352,10.615064),(-0.026221,10.610361),(-0.019968,10.604341),(-0.010925,10.598269),(0.020339,10.588037),(0.029434,10.582533),(0.037341,10.57318),(0.04902,10.550959),(0.057262,10.541115),(0.110179,10.508249),(0.126018,10.491609),(0.176971,10.397687),(0.179865,10.394173),(0.187513,10.394948),(0.19175,10.400839),(0.194437,10.407376),(0.197073,10.41027),(0.223609,10.399961),(0.23322,10.398617),(0.244563,10.401537),(0.253943,10.406084),(0.263012,10.408926),(0.273244,10.406601),(0.283114,10.396808),(0.283553,10.385775),(0.28022,10.375182),(0.278825,10.366629),(0.290039,10.348491),(0.307066,10.333272),(0.317582,10.31733),(0.308797,10.297099),(0.32275,10.297254),(0.366597,10.304488),(0.376183,10.299941),(0.396596,10.283224),(0.372049,10.265137),(0.367295,10.249815),(0.364427,10.233407),(0.357192,10.219997),(0.358846,10.18633),(0.353601,10.115508),(0.363393,10.089669),(0.37608,10.081582),(0.387087,10.078947),(0.395149,10.072358),(0.398146,10.052127),(0.39457,10.035557),(0.393908,10.03249),(0.382746,10.029596),(0.367295,10.032128),(0.349777,10.02882),(0.355719,10.018976),(0.355125,10.01262),(0.35174,10.006419),(0.349777,9.997169),(0.35143,9.990451),(0.363393,9.966783),(0.370189,9.947017),(0.369569,9.938826),(0.362592,9.932676),(0.349777,9.918956),(0.341715,9.903789),(0.339674,9.889837),(0.344196,9.849762),(0.343886,9.838393),(0.32244,9.760671),(0.323887,9.742068),(0.329132,9.731293),(0.336289,9.72349),(0.342955,9.713543),(0.347606,9.698841),(0.34988,9.680651),(0.347089,9.663158),(0.336651,9.650833),(0.318719,9.648611),(0.301924,9.657835),(0.275285,9.685586),(0.260971,9.657164),(0.261694,9.627941),(0.277636,9.606986),(0.308797,9.603678),(0.349363,9.61391),(0.369155,9.610138),(0.377734,9.589416),(0.370809,9.575644),(0.354608,9.571096),(0.33616,9.574197),(0.32244,9.583137),(0.30456,9.571562),(0.28053,9.570166),(0.230197,9.575747),(0.227743,9.569339),(0.223066,9.541176),(0.223402,9.534794),(0.225908,9.531926),(0.23322,9.525595),(0.243116,9.518697),(0.253503,9.514252),(0.279006,9.519007),(0.293139,9.518283),(0.301924,9.507457),(0.2981,9.497096),(0.28531,9.488363),(0.26973,9.482342),(0.228053,9.473971),(0.227949,9.459734),(0.234977,9.443766),(0.226813,9.432345),(0.23136,9.418961),(0.249886,9.416946),(0.26849,9.420641),(0.314223,9.436479),(0.32182,9.442836),(0.325747,9.451336),(0.328796,9.471568),(0.333783,9.479474),(0.34895,9.486037),(0.371429,9.490016),(0.411659,9.492238),(0.430444,9.489189),(0.483515,9.470017),(0.489923,9.461129),(0.49168,9.451285),(0.489406,9.441027),(0.482792,9.429839),(0.482585,9.428857),(0.482792,9.427875),(0.483515,9.426842),(0.513281,9.416791),(0.531368,9.401107),(0.537569,9.377646),(0.528267,9.326951),(0.526613,9.297651),(0.522479,9.285067),(0.512351,9.274706),(0.501085,9.268195),(0.49292,9.258609),(0.49199,9.238895),(0.504393,9.203109),(0.504703,9.187166),(0.49261,9.166754),(0.469976,9.14673),(0.463646,9.137092),(0.463749,9.12146),(0.458478,9.08924),(0.441554,9.058182),(0.43127,9.02759),(0.44468,8.996558),(0.444887,8.996558),(0.444887,8.996455),(0.493127,8.915219),(0.505633,8.866333),(0.483629,8.834973),(0.483515,8.834811),(0.478451,8.822331),(0.477004,8.80675),(0.47349,8.793547),(0.462483,8.788302),(0.449512,8.790369),(0.439409,8.793444),(0.430134,8.794736),(0.419592,8.791247),(0.424966,8.78732),(0.4183,8.770603),(0.407809,8.759053),(0.397112,8.781532),(0.385278,8.78608),(0.372669,8.783754),(0.365874,8.774272),(0.368084,8.761481),(0.374426,8.724766),(0.403727,8.662082),(0.443518,8.606866),(0.483515,8.579787),(0.500568,8.560176),(0.524753,8.538033),(0.550798,8.519326),(0.616324,8.488759),(0.63379,8.452018),(0.646709,8.414165),(0.685053,8.382564),(0.687224,8.372203),(0.686914,8.360731),(0.689394,8.346055),(0.688567,8.33603),(0.689187,8.331301),(0.693115,8.327891),(0.705104,8.325979),(0.708825,8.322568),(0.712752,8.29797),(0.705207,8.282751),(0.689187,8.272855),(0.645056,8.253425),(0.628313,8.24247),(0.595033,8.21273),(0.586661,8.208751),(0.579013,8.207821),(0.573846,8.203945),(0.572709,8.191),(0.576843,8.180484),(0.584594,8.173068),(0.591519,8.16449),(0.593276,8.150666),(0.587798,8.137721),(0.579943,8.131701),(0.574983,8.124621),(0.580254,8.100075),(0.583974,8.05305),(0.593173,8.015739),(0.593689,7.996567),(0.592863,7.959309),(0.602688,7.893132),(0.603405,7.888305),(0.602578,7.828412),(0.611673,7.771258),(0.609502,7.718961),(0.612293,7.699428),(0.583354,7.704285),(0.570538,7.683925),(0.567438,7.652919),(0.567748,7.625737),(0.562063,7.612301),(0.548524,7.605377),(0.532505,7.601036),(0.519379,7.595196),(0.50925,7.585378),(0.503049,7.574681),(0.499432,7.562382),(0.495608,7.504298),(0.500568,7.45505),(0.516692,7.411022),(0.548214,7.383271),(0.566404,7.380791),(0.586248,7.384357),(0.623042,7.397896),(0.629656,7.389162),(0.645676,7.323585),(0.644436,7.301933),(0.631103,7.265242),(0.621801,7.218837),(0.59772,7.158892),(0.59431,7.119722),(0.598754,7.074711),(0.59803,7.0312),(0.57891,6.996783),(0.565991,6.989652),(0.523823,6.978697),(0.509147,6.977611),(0.494574,6.973116),(0.506046,6.957819),(0.534468,6.935909),(0.54625,6.914773),(0.543873,6.897926),(0.522789,6.862838),(0.516071,6.841806),(0.519275,6.832246),(0.528887,6.826535),(0.541083,6.817182),(0.570022,6.763516),(0.576636,6.754731),(0.593793,6.751114),(0.611259,6.741528),(0.625832,6.727756),(0.634307,6.711607),(0.635341,6.695381),(0.626659,6.644893),(0.63224,6.62399),(0.645366,6.612026),(0.659422,6.601717),(0.668207,6.585827),(0.677612,6.584896),(0.705517,6.585775),(0.714612,6.583914),(0.727428,6.571435),(0.726808,6.562107),(0.71978,6.552728),(0.713475,6.53999),(0.713076,6.533936),(0.711718,6.513376),(0.71792,6.48883),(0.729908,6.465085),(0.745721,6.4409),(0.766805,6.419351),(0.816208,6.395063),(0.841323,6.379224),(0.873879,6.340131),(0.886798,6.33039),(0.907262,6.324912),(0.983536,6.324602),(1.009685,6.244452),(1.025188,6.222334),(1.033146,6.216547),(1.040587,6.213472),(1.048132,6.20939),(1.05578,6.20063),(1.068906,6.180347),(1.079655,6.167919),(1.093401,6.160917),(1.176186,6.151512),(1.187968,6.135647),(1.185396,6.100491),(1.185395,6.100491),(1.184825,6.100328),(1.178396,6.097968),(1.133067,6.073635),(1.098806,6.047024),(1.048025,5.992662),(1.036388,5.975165),(1.011729,5.926256),(1.007091,5.906684),(1.005626,5.886054),(0.995291,5.839342),(0.986583,5.820746),(0.954845,5.791083),(0.914399,5.777086),(0.723481,5.755439),(0.364919,5.78148),(0.321951,5.779202),(0.298106,5.773668),(0.285981,5.768866),(0.280935,5.763007),(0.27589,5.760159),(0.25172,5.759101),(0.243337,5.756171),(0.224376,5.742621),(0.1574,5.724555),(0.1421,5.71426),(0.114594,5.688666),(0.075694,5.677558),(0.061697,5.662828),(0.050629,5.644761),(0.034516,5.628363),(-0.045033,5.60163),(-0.049672,5.598863),(-0.062734,5.585842),(-0.068593,5.581204),(-0.077382,5.57807),(-0.093251,5.576646),(-0.307851,5.507392),(-0.353098,5.501695),(-0.359283,5.498684),(-0.371571,5.492743),(-0.445058,5.431383),(-0.45287,5.420111),(-0.458119,5.404731),(-0.471588,5.390815),(-0.507436,5.368842),(-0.577382,5.35163),(-0.589345,5.351508),(-0.596669,5.344306),(-0.612864,5.332017),(-0.629465,5.326728),(-0.637766,5.340888),(-0.689524,5.304755),(-0.702301,5.299994),(-0.760732,5.252183),(-0.784495,5.22427),(-0.79955,5.21483),(-0.873891,5.204006),(-0.89684,5.205024),(-0.911488,5.218004),(-0.918284,5.209418),(-0.927724,5.206122),(-0.952463,5.204413),(-0.976552,5.19831),(-0.988393,5.198798),(-0.993479,5.20775),(-0.99706,5.216783),(-1.005035,5.214545),(-1.017323,5.204413),(-1.033803,5.202948),(-1.049794,5.198676),(-1.154124,5.153876),(-1.164703,5.139228),(-1.173655,5.130561),(-1.194081,5.118598),(-1.233632,5.100735),(-1.25414,5.096503),(-1.319244,5.09455),(-1.329986,5.091539),(-1.353139,5.077786),(-1.364003,5.073432),(-1.374135,5.072659),(-1.394765,5.07392),(-1.404937,5.073432),(-1.423451,5.06745),(-1.459706,5.04975),(-1.529286,5.039862),(-1.551869,5.031399),(-1.568796,5.019273),(-1.569488,5.018785),(-1.583852,5.025377),(-1.601471,5.026679),(-1.618479,5.022447),(-1.630849,5.012641),(-1.632436,5.004543),(-1.625478,4.989488),(-1.627756,4.981594),(-1.634023,4.974433),(-1.638173,4.970689),(-1.706654,4.936916),(-1.718821,4.92768),(-1.74767,4.895901),(-1.749908,4.888577),(-1.748118,4.881903),(-1.748606,4.877183),(-1.757436,4.875434),(-1.766754,4.874701),(-1.835439,4.855129),(-1.912099,4.820868),(-1.92809,4.806789),(-1.95226,4.772447),(-1.967275,4.758124),(-1.981435,4.752427),(-2.050364,4.743638),(-2.063832,4.738186),(-2.090566,4.737128),(-2.096588,4.738471),(-2.103871,4.745063),(-2.106679,4.755357),(-2.105621,4.766547),(-2.107981,4.77558),(-2.120961,4.779202),(-2.129018,4.780341),(-2.144154,4.78498),(-2.155385,4.786078),(-2.154368,4.790676),(-2.163726,4.800523),(-2.206288,4.834459),(-2.227406,4.843411),(-2.23705,4.849189),(-2.241119,4.858059),(-2.242828,4.865383),(-2.247222,4.873033),(-2.252756,4.879136),(-2.356597,4.919623),(-2.525787,4.954413),(-2.690419,5.010199),(-2.820872,5.020087),(-2.964996,5.045315),(-3.103749,5.087714),(-3.119601,5.091331),(-3.115281,5.107821),(-3.108306,5.109605),(-3.098988,5.113227),(-3.087636,5.114976),(-3.016103,5.114447),(-2.994496,5.10814),(-2.988515,5.10224),(-2.976308,5.084621),(-2.970937,5.080878),(-2.957753,5.082668),(-2.937245,5.091295),(-2.92634,5.09455),(-2.929026,5.122016),(-2.901235,5.133612),(-2.865305,5.139309),(-2.843699,5.149115),(-2.843658,5.149115),(-2.803855,5.147568),(-2.780885,5.150875),(-2.766622,5.16103),(-2.75701,5.179866),(-2.762488,5.1863),(-2.775666,5.197539),(-2.778818,5.201001),(-2.779645,5.210122),(-2.775407,5.231025),(-2.774891,5.239578),(-2.779335,5.251076),(-2.793287,5.271101),(-2.79613,5.284976),(-2.789567,5.346858),(-2.755874,5.340709),(-2.742593,5.343138),(-2.732671,5.355721),(-2.730139,5.373162),(-2.734428,5.391145),(-2.761713,5.454914),(-2.779438,5.526485),(-2.781815,5.578885),(-2.785588,5.595034),(-2.793442,5.608134),(-2.806258,5.618495),(-2.877494,5.636815),(-2.893746,5.630355),(-2.926302,5.610718),(-2.945862,5.608289),(-2.969504,5.621441),(-2.974904,5.647279),(-2.964801,5.709782),(-3.029759,5.704252),(-3.019966,5.818354),(-3.022524,5.854502),(-3.03472,5.890701),(-3.073244,5.960955),(-3.087765,5.996689),(-3.118074,6.163475),(-3.139364,6.219286),(-3.168794,6.246054),(-3.171722,6.249585),(-3.183599,6.263908),(-3.262509,6.617142),(-3.261114,6.633937),(-3.253905,6.644428),(-3.235483,6.654582),(-3.230057,6.661119),(-3.210419,6.715999),(-3.212745,6.722666),(-3.226232,6.748581),(-3.238738,6.764627),(-3.243079,6.774756),(-3.241684,6.810903),(-3.224941,6.849402),(-3.109599,7.046393),(-3.097274,7.054053),(-3.096593,7.054477),(-3.095465,7.055178),(-3.045933,7.071249),(-3.040869,7.078329),(-3.032601,7.128352),(-3.027537,7.138428),(-2.9817,7.195531),(-2.972811,7.214961),(-2.972088,7.228604),(-2.976532,7.236717),(-2.983224,7.244985),(-2.989529,7.258938),(-2.990459,7.270668),(-2.941521,7.577161),(-2.928163,7.617676),(-2.855945,7.771568),(-2.852069,7.778131),(-2.842018,7.790275),(-2.838168,7.796838),(-2.83791,7.802264),(-2.84083,7.815235),(-2.840003,7.820247),(-2.831399,7.82836),(-2.809152,7.841693),(-2.801194,7.85022),(-2.797887,7.861278),(-2.793546,7.935951),(-2.790626,7.943289),(-2.784347,7.944012),(-2.777681,7.93962),(-2.772617,7.933935),(-2.771221,7.930732),(-2.754892,7.944374),(-2.709261,7.996722),(-2.690865,8.009745),(-2.631178,8.029589),(-2.610559,8.040001),(-2.600689,8.052946),(-2.599759,8.070387),(-2.606219,8.094313),(-2.619448,8.123097),(-2.621877,8.135732),(-2.617639,8.151493),(-2.610663,8.164852),(-2.604255,8.170794),(-2.595987,8.170278),(-2.583274,8.164335),(-2.57175,8.16312),(-2.558108,8.167694),(-2.513201,8.194023),(-2.506328,8.209267),(-2.552346,8.489367),(-2.598364,8.769466),(-2.603118,8.78515),(-2.612161,8.780964),(-2.612382,8.781321),(-2.617226,8.78918),(-2.614642,8.801919),(-2.607252,8.808559),(-2.596658,8.814398),(-2.595573,8.828067),(-2.615934,8.89783),(-2.617174,8.915013),(-2.619448,8.923617),(-2.625029,8.925994),(-2.631747,8.926976),(-2.63769,8.931471),(-2.653296,8.978394),(-2.654329,8.998832),(-2.660789,9.014903),(-2.686059,9.028261),(-2.711328,9.033894),(-2.73696,9.035134),(-2.750809,9.037408),(-2.758664,9.04286),(-2.765072,9.049475),(-2.774839,9.054978),(-2.765434,9.069112),(-2.7613,9.084795),(-2.759388,9.134818),(-2.755202,9.143138),(-2.733911,9.164842),(-2.716496,9.200266),(-2.704817,9.218405),(-2.673088,9.23489),(-2.661151,9.253907),(-2.659807,9.272898),(-2.675517,9.281502),(-2.692983,9.285326),(-2.707143,9.295635),(-2.714998,9.311061),(-2.713344,9.329923),(-2.70306,9.343255),(-2.689418,9.349534),(-2.677532,9.357001),(-2.672416,9.374028),(-2.672416,9.401004),(-2.677377,9.417773),(-2.678669,9.42555),(-2.674587,9.457486),(-2.675672,9.471723),(-2.689211,9.488724),(-2.725281,9.533864),(-2.761093,9.566239),(-2.766261,9.576523),(-2.768018,9.592801),(-2.766467,9.608898),(-2.761506,9.625253),(-2.752928,9.639904),(-2.740164,9.650833),(-2.743404,9.653325),(-2.751016,9.659179),(-2.75422,9.668352),(-2.75546,9.677498),(-2.760524,9.685586),(-2.769258,9.689926),(-2.774839,9.689306),(-2.780317,9.690237),(-2.788481,9.699254),(-2.795303,9.720002),(-2.791582,9.740931),(-2.774839,9.77434),(-2.740267,9.812606),(-2.733911,9.822787),(-2.736753,9.832579),(-2.760524,9.870536),(-2.764865,9.895754),(-2.755564,9.941151),(-2.754375,9.966783),(-2.760886,9.984766),(-2.783572,10.018511),(-2.788481,10.038768),(-2.788481,10.124448),(-2.79892,10.169613),(-2.798455,10.189534),(-2.785381,10.210101),(-2.765124,10.225837),(-2.757682,10.234648),(-2.754375,10.247954),(-2.754943,10.260796),(-2.758354,10.266119),(-2.764865,10.269116),(-2.774839,10.27524),(-2.784451,10.278805),(-2.793184,10.278547),(-2.799669,10.280924),(-2.802098,10.292319),(-2.80569,10.299011),(-2.814216,10.301078),(-2.824448,10.302215),(-2.833182,10.305987),(-2.841295,10.322291),(-2.841295,10.343272),(-2.836282,10.363658),(-2.829461,10.378282),(-2.822329,10.387584),(-2.81463,10.393708),(-2.80494,10.397118),(-2.791995,10.398152),(-2.78104,10.402415),(-2.771893,10.41182),(-2.767552,10.4212),(-2.771428,10.425489),(-2.795871,10.426109),(-2.812149,10.428822),(-2.824448,10.435023),(-2.859562,10.465616),(-2.867391,10.47825),(-2.87044,10.497784),(-2.873386,10.502487),(-2.886899,10.515793),(-2.891524,10.521659),(-2.894211,10.531425),(-2.895865,10.552768),(-2.898294,10.562612),(-2.928266,10.615219),(-2.932504,10.634313),(-2.929661,10.644209),(-2.916251,10.668084),(-2.910154,10.684491),(-2.907337,10.688315),(-2.905115,10.692733),(-2.905141,10.699193),(-2.908035,10.703637),(-2.912634,10.705808),(-2.916949,10.706609),(-2.918913,10.706609),(-2.919959,10.711088),(-2.922323,10.721207),(-2.919274,10.727202),(-2.908577,10.737666),(-2.9031,10.747149),(-2.899482,10.76999),(-2.882662,10.797068),(-2.871629,10.843448),(-2.864239,10.864273),(-2.850803,10.879544),(-2.834861,10.89161),(-2.821399,10.90626),(-2.815767,10.929153),(-2.821761,10.948299),(-2.832716,10.966024),(-2.838711,10.985119),(-2.835946,10.992224),(-2.837341,10.998012),(-2.751119,10.996462),(-2.750706,10.985842),(-2.728798,10.986057),(-2.634951,10.986979),(-2.439769,10.988943),(-2.197871,10.991294),(-1.972872,10.993516),(-1.754074,10.995635),(-1.598683,10.997133),(-1.587004,11.003671),(-1.579821,11.021215),(-1.562406,11.026589),(-1.436057,11.022713),(-1.423584,11.017911),(-1.413553,11.01405),(-1.38035,11.001268),(-1.355287,10.99672),(-1.22806,10.99548),(-1.10755,10.994343),(-1.098714,11.009562),(-1.070343,11.013902),(-0.932471,11.003102),(-0.924047,11.001423),(-0.916451,10.996203),(-0.909475,10.98269),(-0.904359,10.979486),(-0.89392,10.979899),(-0.900586,10.966231),(-0.883171,10.968634),(-0.860795,10.992922),(-0.839401,10.996462),(-0.833665,11.003412),(-0.827774,11.005841),(-0.822141,11.003619),(-0.816974,10.996462),(-0.816974,10.996203),(-0.703017,10.994159),(-0.695999,10.994033),(-0.694656,10.982742),(-0.690212,10.981811),(-0.683235,10.984344),(-0.673933,10.983517),(-0.667991,10.976515),(-0.67228,10.968505),(-0.679773,10.960572),(-0.68277,10.953596),(-0.634763,10.907966),(-0.617244,10.918637),(-0.607116,10.94122),(-0.599829,10.965766),(-0.590579,10.982432),(-0.576213,10.987754),(-0.555439,10.98995),(-0.516475,10.988633),(-0.512961,10.989744),(-0.509706,10.991449),(-0.506709,10.993671),(-0.504228,10.996203),(-0.504228,10.996358),(-0.494823,11.007805),(-0.46852,11.020129),(-0.456686,11.029612),(-0.45126,11.040206),(-0.449038,11.062892),(-0.444387,11.077439),(-0.425215,11.101468),(-0.405888,11.101468),(-0.386923,11.086405),(-0.368939,11.065217),(-0.318813,11.101365),(-0.306126,11.113535),(-0.298297,11.128392),(-0.300545,11.137409),(-0.304473,11.146788),(-0.301682,11.162937),(-0.166109,11.13498)] +Gibraltar [(-5.358387,36.141109),(-5.338773,36.14112),(-5.339915,36.129828),(-5.33906,36.123847),(-5.34203,36.1105),(-5.35025,36.119289),(-5.358387,36.141109)] +Guinea [(-13.338613,12.63923),(-13.332773,12.639643),(-13.330034,12.642072),(-13.327709,12.644914),(-13.322877,12.646516),(-13.292931,12.652717),(-13.284146,12.653337),(-13.275722,12.651064),(-13.263398,12.643209),(-13.25469,12.641813),(-13.246008,12.643157),(-13.229084,12.647808),(-13.22154,12.648376),(-13.21601,12.645741),(-13.20645,12.635664),(-13.201903,12.63339),(-13.197872,12.634062),(-13.185211,12.637783),(-13.176271,12.637731),(-13.169217,12.638765),(-13.162629,12.638144),(-13.154696,12.632822),(-13.145317,12.643932),(-13.127308,12.634837),(-13.108988,12.635406),(-13.091703,12.638196),(-13.076458,12.635922),(-13.063642,12.62228),(-13.059327,12.60347),(-13.060413,12.583367),(-13.064211,12.566159),(-13.079094,12.532156),(-13.082711,12.515671),(-13.078267,12.496241),(-13.066743,12.480118),(-13.051989,12.472263),(-13.034626,12.470558),(-13.015247,12.472676),(-12.997419,12.465855),(-12.981038,12.466992),(-12.969514,12.476811),(-12.966232,12.496086),(-12.949489,12.535928),(-12.913135,12.536342),(-12.874042,12.516395),(-12.849211,12.495001),(-12.84965,12.491073),(-12.851252,12.484355),(-12.849133,12.475829),(-12.838617,12.466372),(-12.829057,12.462755),(-12.799421,12.460636),(-12.783479,12.453091),(-12.779164,12.443273),(-12.773272,12.435056),(-12.752654,12.432421),(-12.688058,12.435986),(-12.667517,12.433661),(-12.648009,12.425754),(-12.63155,12.41268),(-12.603412,12.381933),(-12.598064,12.372424),(-12.597314,12.365448),(-12.593671,12.361572),(-12.579408,12.361159),(-12.57813,12.361528),(-12.570649,12.363691),(-12.541452,12.379452),(-12.521918,12.386894),(-12.505072,12.390253),(-12.48776,12.389477),(-12.466754,12.384465),(-12.423397,12.369117),(-12.405517,12.356456),(-12.377121,12.313668),(-12.36092,12.305607),(-12.192042,12.348756),(-12.155041,12.369014),(-12.121917,12.398728),(-12.103985,12.407357),(-12.07918,12.408133),(-12.018874,12.388444),(-11.997583,12.386377),(-11.984303,12.389012),(-11.946269,12.409425),(-11.93521,12.417021),(-11.930869,12.418726),(-11.923944,12.418778),(-11.921981,12.417693),(-11.920637,12.416039),(-11.860176,12.391079),(-11.839609,12.386687),(-11.757805,12.383328),(-11.720029,12.389477),(-11.643083,12.417641),(-11.601535,12.425238),(-11.515391,12.431645),(-11.481646,12.428235),(-11.416017,12.40498),(-11.388422,12.403895),(-11.388628,12.384465),(-11.407594,12.381933),(-11.431675,12.382966),(-11.447229,12.374543),(-11.451467,12.352115),(-11.444904,12.312531),(-11.444594,12.289484),(-11.463249,12.249951),(-11.491671,12.220909),(-11.507794,12.191608),(-11.489604,12.151197),(-11.470794,12.134609),(-11.409557,12.107893),(-11.391729,12.093733),(-11.351318,12.040403),(-11.334678,12.026192),(-11.315144,12.013325),(-11.294681,12.003093),(-11.274992,11.996271),(-11.25551,11.996168),(-11.223315,12.009294),(-11.206986,12.010172),(-11.191173,12.01441),(-11.176342,12.029499),(-11.165644,12.04862),(-11.159392,12.077507),(-11.152725,12.080866),(-11.14425,12.081331),(-11.136292,12.085103),(-11.122081,12.107272),(-11.117477,12.112702),(-11.116604,12.113732),(-11.084306,12.136573),(-11.072782,12.149492),(-11.059811,12.192435),(-11.052318,12.202512),(-11.038779,12.205251),(-11.015473,12.204424),(-10.972013,12.218583),(-10.952117,12.219462),(-10.927675,12.212951),(-10.909794,12.200135),(-10.869642,12.138433),(-10.846387,12.12138),(-10.822926,12.110476),(-10.804943,12.095955),(-10.798535,12.068102),(-10.801222,12.059213),(-10.811041,12.04154),(-10.812436,12.033427),(-10.807785,12.022833),(-10.781017,11.99622),(-10.740347,11.919894),(-10.711357,11.890386),(-10.669964,11.89204),(-10.655081,11.898965),(-10.645159,11.90837),(-10.634152,11.925423),(-10.625264,11.943872),(-10.621543,11.957669),(-10.613792,11.967694),(-10.595653,11.97989),(-10.574983,11.990329),(-10.55948,11.995186),(-10.566508,12.017252),(-10.555449,12.029344),(-10.535399,12.038698),(-10.51509,12.052909),(-10.514883,12.060144),(-10.517674,12.070737),(-10.517312,12.080349),(-10.507959,12.084586),(-10.504445,12.088359),(-10.512661,12.108461),(-10.511628,12.118073),(-10.493489,12.124429),(-10.445688,12.120708),(-10.435973,12.135178),(-10.428945,12.143394),(-10.39458,12.17161),(-10.381403,12.180136),(-10.37484,12.174814),(-10.368225,12.171506),(-10.353394,12.166494),(-10.354118,12.18086),(-10.348433,12.184787),(-10.339338,12.185459),(-10.329882,12.19042),(-10.325592,12.199773),(-10.323577,12.201995),(-10.316497,12.196879),(-10.313603,12.191143),(-10.31102,12.19104),(-10.304974,12.202874),(-10.26694,12.217808),(-10.258981,12.213778),(-10.254124,12.207215),(-10.247819,12.201065),(-10.21485,12.194606),(-10.179762,12.178224),(-10.160228,12.174503),(-10.135837,12.175485),(-10.128654,12.174503),(-10.118835,12.170163),(-10.105192,12.158639),(-10.097286,12.153678),(-10.09034,12.151584),(-10.087002,12.150577),(-10.070363,12.149854),(-10.060182,12.147528),(-10.04809,12.141379),(-10.027678,12.125669),(-10.015534,12.119881),(-9.996414,12.114145),(-9.961584,12.096007),(-9.941533,12.092751),(-9.923188,12.087584),(-9.888772,12.060557),(-9.870736,12.052082),(-9.861125,12.051875),(-9.840661,12.055028),(-9.833684,12.055131),(-9.823039,12.050893),(-9.806813,12.038439),(-9.796632,12.032342),(-9.778649,12.026657),(-9.722994,12.025417),(-9.698809,12.050997),(-9.686303,12.068153),(-9.681549,12.080814),(-9.683668,12.101485),(-9.682273,12.118538),(-9.675141,12.133989),(-9.66,12.149802),(-9.628167,12.170163),(-9.515616,12.20706),(-9.495876,12.224681),(-9.479598,12.235843),(-9.438877,12.254757),(-9.421513,12.257082),(-9.360173,12.246644),(-9.336816,12.269846),(-9.331906,12.282766),(-9.335007,12.306537),(-9.321829,12.310206),(-9.312889,12.32607),(-9.308497,12.345036),(-9.309169,12.357748),(-9.314801,12.362657),(-9.331648,12.366326),(-9.339167,12.370616),(-9.343947,12.380124),(-9.344464,12.389012),(-9.347151,12.396454),(-9.374694,12.41113),(-9.392781,12.428028),(-9.407251,12.445133),(-9.412573,12.455365),(-9.398155,12.473245),(-9.350872,12.484614),(-9.33418,12.496293),(-9.327307,12.497533),(-9.321519,12.496603),(-9.308187,12.488955),(-9.294079,12.483787),(-9.286379,12.488128),(-9.278938,12.494432),(-9.265864,12.495207),(-9.189486,12.479239),(-9.147318,12.465493),(-9.027765,12.404751),(-8.993839,12.387514),(-8.975907,12.368704),(-8.964745,12.346276),(-8.963092,12.32421),(-8.972135,12.301989),(-8.985261,12.282094),(-8.995906,12.26101),(-8.99756,12.23481),(-8.994278,12.224836),(-8.984537,12.20892),(-8.981178,12.198946),(-8.97384,12.187578),(-8.961696,12.187991),(-8.942008,12.195846),(-8.927021,12.187371),(-8.913896,12.169853),(-8.906041,12.147735),(-8.906971,12.125359),(-8.921957,12.086033),(-8.926298,12.064278),(-8.922112,12.045364),(-8.903715,12.030378),(-8.88165,12.029189),(-8.858757,12.031825),(-8.837983,12.028156),(-8.819793,12.012653),(-8.815452,11.995703),(-8.816434,11.976841),(-8.814909,11.955395),(-8.809974,11.94444),(-8.803205,11.934932),(-8.79783,11.925113),(-8.797004,11.913227),(-8.847233,11.657946),(-8.828526,11.659186),(-8.81168,11.651745),(-8.795195,11.641926),(-8.777211,11.635932),(-8.756541,11.638826),(-8.740366,11.646164),(-8.726517,11.649471),(-8.712874,11.640066),(-8.690705,11.568029),(-8.687346,11.55002),(-8.679026,11.52984),(-8.667967,11.510875),(-8.656754,11.496431),(-8.602183,11.472092),(-8.575208,11.470283),(-8.549577,11.490127),(-8.543996,11.477647),(-8.539425,11.456312),(-8.535521,11.438088),(-8.527563,11.423387),(-8.51578,11.418968),(-8.488134,11.418193),(-8.427724,11.40008),(-8.397648,11.385895),(-8.375324,11.367654),(-8.393359,11.363003),(-8.414133,11.353468),(-8.42607,11.340472),(-8.417957,11.325511),(-8.39956,11.322152),(-8.381267,11.325098),(-8.371035,11.32055),(-8.376719,11.294661),(-8.388398,11.281406),(-8.406691,11.275282),(-8.426897,11.274197),(-8.44457,11.276264),(-8.483224,11.28492),(-8.492578,11.277788),(-8.50312,11.255412),(-8.502345,11.25164),(-8.499244,11.246705),(-8.497022,11.24071),(-8.498779,11.233501),(-8.502138,11.231176),(-8.51206,11.230298),(-8.515625,11.229057),(-8.5241,11.22376),(-8.544616,11.214975),(-8.551799,11.210738),(-8.563943,11.193426),(-8.569679,11.157796),(-8.578154,11.14206),(-8.586991,11.137461),(-8.607222,11.137151),(-8.615413,11.13405),(-8.624611,11.122035),(-8.624921,11.112191),(-8.622389,11.102605),(-8.623112,11.091262),(-8.63368,11.071289),(-8.679801,11.025194),(-8.68869,11.012559),(-8.696906,10.996048),(-8.70073,10.978452),(-8.696234,10.962743),(-8.680938,10.952304),(-8.665229,10.956955),(-8.647969,10.965818),(-8.628228,10.967833),(-8.611382,10.965146),(-8.593347,10.964707),(-8.575622,10.966748),(-8.55955,10.971605),(-8.541567,10.983775),(-8.531645,10.999046),(-8.505755,11.052789),(-8.489064,11.052737),(-8.469013,11.046381),(-8.448704,11.047105),(-8.438162,11.04974),(-8.429067,11.048293),(-8.410464,11.041911),(-8.400645,11.043332),(-8.381318,11.053926),(-8.369329,11.054236),(-8.347315,11.043126),(-8.326593,11.02385),(-8.311142,11.000803),(-8.305096,10.978246),(-8.303649,10.874893),(-8.305871,10.856289),(-8.311814,10.847246),(-8.318893,10.83952),(-8.339822,10.783865),(-8.341941,10.763246),(-8.315586,10.753092),(-8.311142,10.735289),(-8.295535,10.537446),(-8.284425,10.511246),(-8.253884,10.469827),(-8.228873,10.423422),(-8.210579,10.414068),(-8.18903,10.413629),(-8.165983,10.418099),(-8.150583,10.424533),(-8.142056,10.429649),(-8.135442,10.427323),(-8.125417,10.411665),(-8.122109,10.399496),(-8.120921,10.373037),(-8.115495,10.360247),(-8.096116,10.346734),(-8.070071,10.341876),(-8.015708,10.339344),(-7.996277,10.32826),(-7.981549,10.307021),(-7.971214,10.281802),(-7.9646,10.258936),(-7.962584,10.233252),(-7.96801,10.209998),(-7.989663,10.161991),(-7.999843,10.150699),(-8.007233,10.138323),(-8.012349,10.124861),(-8.015708,10.110366),(-8.03488,10.085794),(-8.057876,10.073081),(-8.108777,10.054194),(-8.127535,10.037115),(-8.147741,10.010036),(-8.15849,9.984921),(-8.148878,9.973888),(-8.136269,9.969444),(-8.140093,9.959703),(-8.152392,9.949988),(-8.173837,9.941875),(-8.174716,9.933038),(-8.171512,9.922832),(-8.167895,9.914693),(-8.164536,9.912678),(-8.158955,9.912368),(-8.153219,9.910921),(-8.14955,9.90534),(-8.147224,9.887925),(-8.145674,9.882499),(-8.147327,9.880122),(-8.152133,9.87883),(-8.155079,9.875238),(-8.150996,9.866143),(-8.146862,9.864696),(-8.128104,9.866246),(-8.123505,9.844852),(-8.160293,9.584814),(-8.16159,9.575644),(-8.161022,9.553604),(-8.157611,9.530866),(-8.152857,9.513116),(-8.145364,9.497148),(-8.087176,9.415395),(-8.070433,9.397541),(-8.054051,9.389092),(-8.032296,9.384648),(-8.009868,9.383614),(-7.991575,9.38532),(-7.971059,9.392968),(-7.933025,9.414259),(-7.91189,9.418651),(-7.8706,9.418599),(-7.865381,9.409918),(-7.891736,9.330233),(-7.89494,9.307133),(-7.930648,9.220291),(-7.929563,9.183885),(-7.878713,9.163912),(-7.866311,9.166031),(-7.856234,9.170217),(-7.847914,9.168563),(-7.83422,9.142983),(-7.823368,9.131976),(-7.801044,9.114794),(-7.759392,9.091668),(-7.746783,9.07645),(-7.748203,9.075403),(-7.764198,9.063608),(-7.788073,9.060042),(-7.807813,9.060042),(-7.827037,9.05782),(-7.916747,9.012345),(-7.931785,8.996558),(-7.931837,8.996558),(-7.931837,8.996455),(-7.931888,8.996455),(-7.941604,8.972218),(-7.946461,8.948602),(-7.950699,8.899535),(-7.96894,8.812848),(-7.958915,8.781997),(-7.91313,8.768226),(-7.890961,8.765693),(-7.831016,8.747658),(-7.826572,8.748227),(-7.816056,8.753524),(-7.81019,8.753369),(-7.803679,8.749054),(-7.795695,8.73647),(-7.790915,8.730941),(-7.772053,8.714534),(-7.765129,8.706085),(-7.741202,8.662909),(-7.731823,8.652083),(-7.703944,8.633479),(-7.696812,8.624849),(-7.691593,8.606142),(-7.69397,8.567773),(-7.691696,8.546663),(-7.679242,8.505089),(-7.679294,8.487726),(-7.686477,8.462585),(-7.689578,8.438659),(-7.683221,8.417317),(-7.662447,8.374865),(-7.720738,8.368534),(-7.748334,8.372307),(-7.77102,8.386518),(-7.802439,8.449201),(-7.823833,8.477856),(-7.845744,8.474316),(-7.849413,8.459201),(-7.846777,8.44269),(-7.848586,8.428634),(-7.865588,8.420882),(-7.882331,8.421451),(-7.898583,8.426283),(-7.912871,8.434189),(-7.923827,8.443775),(-7.931578,8.45703),(-7.936953,8.471422),(-7.943877,8.484625),(-7.95628,8.494366),(-7.970542,8.49695),(-7.98589,8.494935),(-8.015708,8.487209),(-8.024906,8.488604),(-8.042476,8.487157),(-8.050667,8.487674),(-8.057204,8.490852),(-8.068107,8.500464),(-8.076634,8.503255),(-8.096891,8.501704),(-8.130119,8.488604),(-8.150531,8.486486),(-8.186343,8.493695),(-8.200296,8.491834),(-8.248872,8.453568),(-8.254401,8.446514),(-8.254298,8.434499),(-8.249078,8.423776),(-8.242102,8.412769),(-8.236883,8.400083),(-8.235281,8.379515),(-8.247631,8.322568),(-8.247631,8.303835),(-8.245306,8.285077),(-8.248252,8.273579),(-8.264581,8.2565),(-8.265977,8.245984),(-8.182932,8.191672),(-8.085522,8.162035),(-8.075962,8.163379),(-8.015708,8.183894),(-8.003305,8.184049),(-8.002427,8.174308),(-8.008266,8.160433),(-8.015708,8.14816),(-8.021702,8.117723),(-8.021289,8.102659),(-8.015708,8.088474),(-7.997052,8.077338),(-7.980413,8.058992),(-7.968217,8.03703),(-7.965638,8.026927),(-7.962636,8.015171),(-7.980361,8.016566),(-7.992505,8.024162),(-8.002168,8.032431),(-8.011987,8.035557),(-8.053845,8.028762),(-8.069089,8.029278),(-8.067229,8.020752),(-8.068883,7.996567),(-8.079115,7.957655),(-8.128879,7.882569),(-8.131204,7.842416),(-8.123763,7.83203),(-8.101439,7.812806),(-8.096323,7.801902),(-8.098261,7.791877),(-8.103919,7.786348),(-8.110482,7.78242),(-8.115133,7.776891),(-8.119009,7.760819),(-8.122626,7.727488),(-8.12738,7.713019),(-8.159523,7.676328),(-8.160195,7.672194),(-8.157404,7.662376),(-8.158024,7.658242),(-8.162107,7.654159),(-8.173424,7.647803),(-8.177868,7.643876),(-8.197867,7.618554),(-8.202621,7.607185),(-8.203035,7.599589),(-8.199779,7.583363),(-8.200244,7.576335),(-8.228976,7.544295),(-8.265356,7.560056),(-8.303235,7.589305),(-8.336773,7.597832),(-8.372275,7.58729),(-8.390878,7.585791),(-8.414805,7.612921),(-8.440023,7.599899),(-8.485446,7.557989),(-8.515625,7.577161),(-8.54782,7.600777),(-8.566527,7.623412),(-8.57278,7.650749),(-8.567405,7.688214),(-8.686571,7.694364),(-8.694477,7.674416),(-8.720677,7.642739),(-8.727576,7.622017),(-8.730031,7.587445),(-8.731478,7.581761),(-8.737369,7.571425),(-8.739178,7.565689),(-8.738041,7.559488),(-8.73401,7.557679),(-8.729514,7.556387),(-8.726827,7.55184),(-8.725742,7.528534),(-8.729617,7.50869),(-8.739539,7.490552),(-8.756489,7.472413),(-8.784394,7.455722),(-8.790389,7.450812),(-8.793955,7.441511),(-8.795453,7.421099),(-8.799226,7.41221),(-8.807701,7.40451),(-8.82403,7.399808),(-8.83341,7.393865),(-8.851626,7.371489),(-8.860617,7.35764),(-8.865371,7.346891),(-8.863304,7.32989),(-8.855915,7.311855),(-8.851367,7.292889),(-8.857827,7.273046),(-8.868885,7.266638),(-8.91219,7.250153),(-8.925419,7.248603),(-8.944385,7.278678),(-8.9572,7.286481),(-8.962575,7.262969),(-8.973117,7.267671),(-8.977716,7.266224),(-8.981023,7.250566),(-8.985467,7.25005),(-9.007637,7.243487),(-9.015698,7.238887),(-9.032855,7.240283),(-9.046652,7.232738),(-9.059261,7.221369),(-9.073111,7.211344),(-9.086392,7.20721),(-9.100448,7.204833),(-9.106085,7.202975),(-9.113935,7.200388),(-9.125407,7.190208),(-9.116054,7.224676),(-9.11962,7.237957),(-9.139153,7.247879),(-9.146905,7.25408),(-9.147473,7.271392),(-9.15308,7.275474),(-9.17543,7.274596),(-9.181476,7.275681),(-9.206281,7.298367),(-9.219872,7.360999),(-9.234341,7.381308),(-9.249715,7.383065),(-9.283796,7.374073),(-9.299505,7.376605),(-9.3055,7.383478),(-9.312424,7.403322),(-9.319401,7.411952),(-9.327772,7.417068),(-9.335007,7.419652),(-9.349218,7.421564),(-9.365961,7.421099),(-9.375159,7.41929),(-9.379242,7.415828),(-9.381412,7.404975),(-9.386477,7.401668),(-9.392264,7.39986),(-9.407354,7.378052),(-9.418723,7.387767),(-9.438101,7.421564),(-9.449263,7.405079),(-9.478357,7.376915),(-9.486626,7.372833),(-9.487099,7.373204),(-9.490708,7.376037),(-9.479701,7.38539),(-9.4735,7.39557),(-9.470864,7.407353),(-9.470244,7.420995),(-9.466213,7.433449),(-9.456705,7.443061),(-9.436086,7.458564),(-9.4134,7.492257),(-9.404925,7.513599),(-9.408697,7.527449),(-9.390817,7.548223),(-9.38366,7.55954),(-9.378777,7.586256),(-9.372989,7.601346),(-9.364411,7.614265),(-9.353507,7.62243),(-9.374694,7.646925),(-9.376245,7.680566),(-9.366374,7.716171),(-9.355471,7.741906),(-9.380999,7.77441),(-9.389681,7.791464),(-9.415674,7.824433),(-9.43867,7.866239),(-9.448437,7.907839),(-9.446008,7.950058),(-9.433093,7.993468),(-9.431745,7.996567),(-9.424562,8.005817),(-9.42234,8.015791),(-9.424149,8.026178),(-9.429265,8.036901),(-9.440944,8.050569),(-9.451537,8.051758),(-9.462183,8.043464),(-9.474223,8.028813),(-9.472776,8.040363),(-9.461873,8.058424),(-9.459805,8.068785),(-9.478047,8.141287),(-9.478099,8.152785),(-9.481406,8.164697),(-9.491328,8.169968),(-9.515616,8.174877),(-9.530137,8.185496),(-9.531894,8.212316),(-9.525435,8.24079),(-9.515616,8.256603),(-9.51081,8.261047),(-9.509156,8.265595),(-9.510758,8.270065),(-9.515616,8.274406),(-9.521249,8.284405),(-9.525124,8.295438),(-9.530189,8.30598),(-9.539387,8.314584),(-9.528483,8.32572),(-9.522282,8.330681),(-9.515616,8.334376),(-9.511482,8.336133),(-9.499131,8.343471),(-9.504351,8.346158),(-9.511482,8.353781),(-9.529052,8.366312),(-9.579178,8.387293),(-9.592045,8.405095),(-9.600159,8.410883),(-9.607342,8.402925),(-9.612148,8.396336),(-9.619537,8.392719),(-9.628064,8.391634),(-9.636125,8.392564),(-9.653385,8.392099),(-9.668372,8.38874),(-9.67726,8.39153),(-9.675916,8.409643),(-9.666511,8.422381),(-9.653385,8.43145),(-9.645117,8.443413),(-9.650543,8.464807),(-9.663514,8.477856),(-9.683513,8.487054),(-9.703253,8.487261),(-9.7155,8.473334),(-9.717981,8.452689),(-9.718188,8.438272),(-9.722115,8.435636),(-9.735913,8.450493),(-9.7463,8.468063),(-9.750899,8.485866),(-9.757307,8.53271),(-9.763043,8.54997),(-9.773585,8.563018),(-9.790586,8.565396),(-9.790018,8.553071),(-9.794824,8.544544),(-9.802007,8.53718),(-9.808854,8.528473),(-9.808828,8.520463),(-9.805469,8.512789),(-9.806813,8.506252),(-9.831617,8.499017),(-9.849704,8.48969),(-9.858644,8.487054),(-9.863502,8.49354),(-9.868153,8.498087),(-9.872804,8.500361),(-9.886911,8.49664),(-9.891872,8.495917),(-9.911354,8.495658),(-9.928149,8.493436),(-9.944169,8.488346),(-9.961119,8.479561),(-10.000186,8.449305),(-10.04039,8.425663),(-10.061681,8.422174),(-10.075013,8.426644),(-10.079354,8.439951),(-10.069277,8.486873),(-10.076099,8.501498),(-10.091085,8.510102),(-10.147981,8.524287),(-10.164879,8.524649),(-10.181829,8.519584),(-10.215625,8.486589),(-10.23242,8.477055),(-10.240275,8.493746),(-10.260842,8.485917),(-10.271332,8.484109),(-10.282236,8.484625),(-10.293708,8.483075),(-10.302235,8.485866),(-10.310245,8.490361),(-10.337581,8.498552),(-10.344299,8.499069),(-10.374737,8.49478),(-10.389775,8.489095),(-10.402384,8.481008),(-10.414062,8.470182),(-10.423778,8.456565),(-10.447962,8.407627),(-10.46021,8.39215),(-10.519276,8.33479),(-10.554364,8.311845),(-10.572657,8.304585),(-10.583251,8.30691),(-10.61188,8.325255),(-10.626711,8.331508),(-10.659784,8.333265),(-10.688154,8.324067),(-10.707895,8.303138),(-10.715181,8.2696),(-10.735283,8.287299),(-10.725,8.319855),(-10.702004,8.353574),(-10.683813,8.374865),(-10.669086,8.399075),(-10.654668,8.485504),(-10.643868,8.486873),(-10.641077,8.49478),(-10.64118,8.50664),(-10.638855,8.519895),(-10.631879,8.532013),(-10.616479,8.546508),(-10.600511,8.575653),(-10.584646,8.596841),(-10.565423,8.612783),(-10.545166,8.616762),(-10.522118,8.618571),(-10.503308,8.631567),(-10.489665,8.651049),(-10.481862,8.672469),(-10.524753,8.729546),(-10.530231,8.739158),(-10.545321,8.750036),(-10.5863,8.804089),(-10.589091,8.806595),(-10.596274,8.811246),(-10.599684,8.81538),(-10.600201,8.819024),(-10.599116,8.829617),(-10.599374,8.834036),(-10.608469,8.862303),(-10.610123,8.872302),(-10.61188,8.94514),(-10.607126,8.979841),(-10.591778,9.039062),(-10.592179,9.041658),(-10.593101,9.047617),(-10.594103,9.0541),(-10.605989,9.063763),(-10.629346,9.070352),(-10.669086,9.074434),(-10.710427,9.072987),(-10.746342,9.083814),(-10.753473,9.120116),(-10.74257,9.163292),(-10.724741,9.194504),(-10.714974,9.201352),(-10.703761,9.206313),(-10.693167,9.21285),(-10.685209,9.224503),(-10.683917,9.233753),(-10.687948,9.250703),(-10.688051,9.259281),(-10.683452,9.271812),(-10.677664,9.282897),(-10.67384,9.293982),(-10.675132,9.306461),(-10.717765,9.343333),(-10.728203,9.34819),(-10.741536,9.357053),(-10.753473,9.367569),(-10.765462,9.385216),(-10.774764,9.387335),(-10.785409,9.387232),(-10.795228,9.388524),(-10.801687,9.387283),(-10.819361,9.388059),(-10.831763,9.392141),(-10.822151,9.4009),(-10.811454,9.408264),(-10.813418,9.413897),(-10.829076,9.423948),(-10.843959,9.440252),(-10.854862,9.455264),(-10.8591,9.472989),(-10.853674,9.497871),(-10.852589,9.512082),(-10.860185,9.518542),(-10.870572,9.523399),(-10.877703,9.532675),(-10.882819,9.560167),(-10.886333,9.568409),(-10.896049,9.578305),(-10.92168,9.595901),(-10.92907,9.605694),(-10.935788,9.631894),(-10.940129,9.639697),(-10.94664,9.644839),(-10.962918,9.650782),(-10.968499,9.654244),(-10.974597,9.663778),(-10.986327,9.697755),(-11.010098,9.74274),(-11.044928,9.78713),(-11.085959,9.826171),(-11.16027,9.872758),(-11.16766,9.884462),(-11.181251,9.950376),(-11.188537,9.972597),(-11.199596,9.982157),(-11.213445,9.983371),(-11.247293,9.993396),(-11.272666,9.996006),(-11.484875,9.994845),(-11.910095,9.992518),(-11.914953,9.931488),(-11.921774,9.922212),(-12.109807,9.881676),(-12.141347,9.874876),(-12.158349,9.8756),(-12.174834,9.879191),(-12.193644,9.886064),(-12.217389,9.90056),(-12.235605,9.91676),(-12.253588,9.928413),(-12.276739,9.929266),(-12.426989,9.897614),(-12.472335,9.881258),(-12.508327,9.860381),(-12.514322,9.851312),(-12.514994,9.845111),(-12.513728,9.838341),(-12.513624,9.827825),(-12.527267,9.753618),(-12.526647,9.75062),(-12.522874,9.739432),(-12.522461,9.734472),(-12.525561,9.732069),(-12.530729,9.732844),(-12.53569,9.732792),(-12.537654,9.727934),(-12.537473,9.715713),(-12.538765,9.711915),(-12.54202,9.71132),(-12.573078,9.692562),(-12.582251,9.683364),(-12.59858,9.654864),(-12.601552,9.651712),(-12.602766,9.609363),(-12.606823,9.600707),(-12.614858,9.600087),(-12.624599,9.601276),(-12.633849,9.597865),(-12.638552,9.589416),(-12.647285,9.556627),(-12.649921,9.552105),(-12.652065,9.549444),(-12.655269,9.547739),(-12.660954,9.546033),(-12.668628,9.542364),(-12.669274,9.53823),(-12.66731,9.533709),(-12.667517,9.528696),(-12.701597,9.420227),(-12.706196,9.413948),(-12.713121,9.408161),(-12.716454,9.404414),(-12.719038,9.398368),(-12.72294,9.392089),(-12.727849,9.393123),(-12.732939,9.396508),(-12.737228,9.397231),(-12.754075,9.3878),(-12.759759,9.382839),(-12.767795,9.372323),(-12.767562,9.364391),(-12.763557,9.355347),(-12.764152,9.347673),(-12.777923,9.343849),(-12.795261,9.33509),(-12.819135,9.29579),(-12.835827,9.281967),(-12.856885,9.278505),(-12.876109,9.279848),(-12.895642,9.278505),(-12.917527,9.267007),(-12.937009,9.287083),(-12.957912,9.272433),(-12.973751,9.241582),(-12.97835,9.21285),(-12.973932,9.202799),(-12.960212,9.186081),(-12.959153,9.177839),(-12.962796,9.176702),(-12.97848,9.175203),(-12.984164,9.172309),(-12.993802,9.158693),(-13.002044,9.14412),(-13.015247,9.112752),(-13.028864,9.095983),(-13.047003,9.084537),(-13.085579,9.065985),(-13.115913,9.043945),(-13.131054,9.041465),(-13.153844,9.04932),(-13.191774,9.072987),(-13.207535,9.076708),(-13.233632,9.072057),(-13.277376,9.058595),(-13.301096,9.04149),(-13.312367,9.048326),(-13.321604,9.056627),(-13.325795,9.067776),(-13.322174,9.083075),(-13.319447,9.083564),(-13.304514,9.083075),(-13.300649,9.084621),(-13.300933,9.088202),(-13.302154,9.092515),(-13.301096,9.096137),(-13.273793,9.12401),(-13.254872,9.154934),(-13.243072,9.169745),(-13.225942,9.179267),(-13.208241,9.18008),(-13.176137,9.173651),(-13.157704,9.179267),(-13.157704,9.185492),(-13.218495,9.183254),(-13.247304,9.186021),(-13.259511,9.196357),(-13.25768,9.200263),(-13.248891,9.208319),(-13.245839,9.213446),(-13.245839,9.226467),(-13.237904,9.246649),(-13.232818,9.267401),(-13.254872,9.244818),(-13.264556,9.220445),(-13.270375,9.194037),(-13.280629,9.165025),(-13.304921,9.175482),(-13.315297,9.182074),(-13.322174,9.192939),(-13.313588,9.197943),(-13.311635,9.207221),(-13.314687,9.233303),(-13.313588,9.23314),(-13.311187,9.237982),(-13.308949,9.244778),(-13.307932,9.250312),(-13.310048,9.253811),(-13.319651,9.259263),(-13.322174,9.261217),(-13.325063,9.281317),(-13.325347,9.311021),(-13.318674,9.329495),(-13.301096,9.315863),(-13.303822,9.337795),(-13.303375,9.347113),(-13.301096,9.356838),(-13.307932,9.356838),(-13.313954,9.349555),(-13.322174,9.345404),(-13.332834,9.343573),(-13.345774,9.343166),(-13.352854,9.338772),(-13.357818,9.328437),(-13.367909,9.282172),(-13.374501,9.272773),(-13.397532,9.28384),(-13.407786,9.287014),(-13.411977,9.293402),(-13.404124,9.308417),(-13.414052,9.317206),(-13.423818,9.332913),(-13.430816,9.352037),(-13.431996,9.37108),(-13.42516,9.389879),(-13.40331,9.418158),(-13.396637,9.439358),(-13.404124,9.439358),(-13.428375,9.420722),(-13.465403,9.418158),(-13.499176,9.428778),(-13.513987,9.449612),(-13.520863,9.479478),(-13.520741,9.495103),(-13.510243,9.511054),(-13.50064,9.521796),(-13.493235,9.534857),(-13.488393,9.550605),(-13.486684,9.569078),(-13.49352,9.569078),(-13.515614,9.524482),(-13.529042,9.505683),(-13.547475,9.50019),(-13.561391,9.508979),(-13.575266,9.527248),(-13.58552,9.54914),(-13.58845,9.569078),(-13.619008,9.556383),(-13.643259,9.542548),(-13.677968,9.522691),(-13.7058,9.514472),(-13.707102,9.512152),(-13.714345,9.507636),(-13.722401,9.505805),(-13.726226,9.511054),(-13.72411,9.519517),(-13.718902,9.522284),(-13.71231,9.523668),(-13.7058,9.528062),(-13.673329,9.573798),(-13.652659,9.597154),(-13.640777,9.623725),(-13.625803,9.647284),(-13.623036,9.662909),(-13.624379,9.720282),(-13.623199,9.727362),(-13.61856,9.732082),(-13.604888,9.737372),(-13.602162,9.743801),(-13.595448,9.755032),(-13.580719,9.76203),(-13.566518,9.771064),(-13.561106,9.788235),(-13.581776,9.777167),(-13.594309,9.774237),(-13.604115,9.780178),(-13.647776,9.831488),(-13.651194,9.842841),(-13.655181,9.841742),(-13.658111,9.840522),(-13.664784,9.836615),(-13.653798,9.805243),(-13.650543,9.777004),(-13.661285,9.754543),(-13.692128,9.740383),(-13.728912,9.744086),(-13.747182,9.767768),(-13.749827,9.799872),(-13.739898,9.829169),(-13.698313,9.880764),(-13.681996,9.913479),(-13.685292,9.946479),(-13.693959,9.953803),(-13.697499,9.940416),(-13.699208,9.921332),(-13.702016,9.911689),(-13.712758,9.905178),(-13.732493,9.876776),(-13.746693,9.870754),(-13.745432,9.866523),(-13.749745,9.863471),(-13.784657,9.850246),(-13.788686,9.850043),(-13.809682,9.85163),(-13.827707,9.855373),(-13.842844,9.860785),(-13.849192,9.867621),(-13.855051,9.878852),(-13.86913,9.896226),(-13.886098,9.912177),(-13.900624,9.919135),(-13.927642,9.93594),(-13.933176,9.943345),(-13.945383,9.966946),(-13.954254,9.970933),(-13.971099,9.971829),(-13.979482,9.974351),(-13.98705,9.980414),(-14.006744,10.001044),(-14.021718,10.00727),(-14.032297,10.008979),(-14.042958,10.012844),(-14.058339,10.025295),(-14.061879,10.050116),(-14.046132,10.079495),(-14.020334,10.094428),(-13.993764,10.076158),(-14.001088,10.088202),(-14.014801,10.104804),(-14.028188,10.116604),(-14.034088,10.114),(-14.041615,10.101304),(-14.056752,10.106594),(-14.068349,10.118109),(-14.0655,10.124579),(-14.043813,10.133246),(-14.026967,10.153957),(-13.999908,10.199693),(-14.014516,10.192776),(-14.024892,10.183743),(-14.044667,10.162177),(-14.055491,10.155341),(-14.082834,10.14289),(-14.089345,10.138251),(-14.091176,10.126044),(-14.089345,10.079901),(-14.092275,10.06859),(-14.100575,10.058295),(-14.113433,10.051011),(-14.130279,10.048896),(-14.153188,10.05565),(-14.170481,10.070787),(-14.179799,10.090277),(-14.178782,10.110297),(-14.16275,10.126614),(-14.138173,10.144477),(-14.123647,10.162584),(-14.137766,10.179185),(-14.141672,10.164537),(-14.153147,10.156317),(-14.168609,10.152737),(-14.184926,10.151923),(-14.182485,10.129543),(-14.200185,10.112738),(-14.222401,10.106106),(-14.233388,10.114),(-14.236562,10.120917),(-14.24356,10.129381),(-14.2506,10.14053),(-14.253814,10.155341),(-14.250478,10.162584),(-14.244049,10.168891),(-14.240061,10.175116),(-14.243886,10.182318),(-14.249257,10.184272),(-14.25475,10.182847),(-14.259023,10.179592),(-14.26065,10.175767),(-14.270131,10.159735),(-14.292104,10.17178),(-14.352366,10.226223),(-14.362701,10.231187),(-14.376698,10.234442),(-14.391835,10.233385),(-14.405507,10.229397),(-14.418568,10.227973),(-14.431956,10.234442),(-14.452545,10.214586),(-14.462636,10.225165),(-14.465891,10.249701),(-14.466135,10.27204),(-14.458241,10.304267),(-14.458608,10.316962),(-14.463043,10.330024),(-14.468617,10.336249),(-14.475331,10.340318),(-14.537262,10.403632),(-14.548004,10.425605),(-14.550893,10.446601),(-14.550526,10.472113),(-14.545522,10.495022),(-14.535593,10.507473),(-14.535024,10.508205),(-14.53543,10.508165),(-14.551422,10.506781),(-14.562408,10.499335),(-14.571604,10.489447),(-14.582834,10.480862),(-14.596425,10.475979),(-14.613515,10.473375),(-14.628977,10.476264),(-14.637441,10.487738),(-14.644276,10.487738),(-14.651194,10.475084),(-14.657786,10.47956),(-14.662709,10.494127),(-14.664703,10.511623),(-14.663808,10.529202),(-14.661122,10.542629),(-14.650461,10.570258),(-14.639394,10.566636),(-14.625966,10.577379),(-14.611399,10.592963),(-14.596425,10.603746),(-14.620595,10.604315),(-14.628651,10.617255),(-14.627431,10.635484),(-14.615224,10.691311),(-14.607289,10.70539),(-14.596425,10.699368),(-14.59024,10.699368),(-14.588287,10.714301),(-14.580719,10.722073),(-14.572418,10.726304),(-14.568512,10.731024),(-14.555491,10.775702),(-14.531321,10.823472),(-14.517445,10.829047),(-14.509674,10.842475),(-14.508372,10.858466),(-14.513905,10.871894),(-14.509999,10.876451),(-14.500234,10.891099),(-14.519928,10.885403),(-14.522572,10.877631),(-14.519154,10.866523),(-14.521352,10.850816),(-14.554921,10.827623),(-14.558909,10.823472),(-14.573476,10.812405),(-14.590403,10.787258),(-14.60436,10.760443),(-14.615956,10.728013),(-14.629709,10.706448),(-14.657867,10.672675),(-14.686106,10.648179),(-14.699818,10.643134),(-14.705719,10.65526),(-14.703725,10.670844),(-14.69811,10.68183),(-14.689443,10.688951),(-14.678334,10.693101),(-14.68578,10.694729),(-14.691762,10.694485),(-14.697865,10.69184),(-14.705719,10.686347),(-14.71935,10.699368),(-14.706899,10.739203),(-14.699086,10.756252),(-14.688385,10.771959),(-14.6822,10.777777),(-14.66808,10.787095),(-14.661285,10.792792),(-14.655995,10.800605),(-14.65331,10.807603),(-14.651357,10.809963),(-14.656565,10.811957),(-14.664703,10.816636),(-14.677968,10.799628),(-14.723134,10.765123),(-14.734771,10.748114),(-14.746449,10.710761),(-14.760325,10.699368),(-14.770009,10.718817),(-14.782704,10.731269),(-14.792836,10.745022),(-14.795074,10.768256),(-14.781606,10.767768),(-14.770009,10.776923),(-14.760813,10.790432),(-14.75414,10.802965),(-14.749257,10.816107),(-14.746083,10.831204),(-14.744985,10.844062),(-14.746693,10.850816),(-14.743886,10.856594),(-14.742421,10.860989),(-14.740468,10.871894),(-14.758901,10.858466),(-14.765289,10.850328),(-14.76773,10.840888),(-14.772572,10.836493),(-14.801259,10.823472),(-14.808705,10.823472),(-14.809682,10.834784),(-14.809071,10.845608),(-14.806508,10.855455),(-14.801259,10.864447),(-14.807281,10.873603),(-14.807973,10.882473),(-14.803822,10.890815),(-14.795074,10.898586),(-14.813873,10.900336),(-14.819732,10.912014),(-14.816762,10.927191),(-14.808705,10.939602),(-14.796132,10.947577),(-14.778635,10.954088),(-14.759918,10.958441),(-14.74356,10.960028),(-14.73176,10.966457),(-14.724965,10.980536),(-14.720611,10.994574),(-14.716298,11.000963),(-14.699452,11.00552),(-14.680776,11.027493),(-14.664703,11.035793),(-14.672231,11.041449),(-14.677235,11.047593),(-14.68517,11.063666),(-14.687001,11.054267),(-14.690094,11.045233),(-14.694976,11.038479),(-14.70226,11.035793),(-14.707875,11.034491),(-14.723541,11.028266),(-14.726796,11.025824),(-14.730824,11.005845),(-14.741526,10.989081),(-14.756825,10.981187),(-14.774566,10.987982),(-14.779449,10.983303),(-14.784576,10.981106),(-14.791412,10.980373),(-14.801259,10.980536),(-14.795481,10.999701),(-14.801259,11.011908),(-14.810658,11.014472),(-14.815541,11.004706),(-14.843495,10.967475),(-14.874623,10.973293),(-14.894439,11.03026),(-14.918568,11.028957),(-14.901479,11.011664),(-14.898793,10.993232),(-14.904124,10.975491),(-14.911122,10.960028),(-14.931956,10.975653),(-14.943918,10.980048),(-14.959584,10.980536),(-14.926381,10.955552),(-14.914418,10.942328),(-14.911122,10.92593),(-14.920888,10.90766),(-14.939809,10.889106),(-14.958119,10.876207),(-14.96642,10.874986),(-14.968495,10.877875),(-14.973215,10.878852),(-14.977895,10.877875),(-14.980051,10.874986),(-14.978993,10.868842),(-14.974233,10.85928),(-14.973215,10.854193),(-14.973215,10.820054),(-14.968984,10.814358),(-14.959584,10.809963),(-14.950185,10.803778),(-14.945912,10.792792),(-14.947662,10.787584),(-14.959584,10.768256),(-14.977528,10.777045),(-15.013905,10.78262),(-15.031565,10.792792),(-15.052602,10.814399),(-15.069447,10.837144),(-15.079457,10.85871),(-15.081125,10.878241),(-15.07433,10.896877),(-15.058909,10.915961),(-15.051625,10.919623),(-15.03189,10.925482),(-15.027821,10.929348),(-15.026682,10.936672),(-15.02359,10.94359),(-15.019399,10.94953),(-15.014801,10.953803),(-15.014801,10.960028),(-15.019276,10.959703),(-15.020334,10.961371),(-15.020131,10.964179),(-15.020985,10.967475),(-14.992594,10.996203),(-14.992594,10.996358),(-14.959056,11.035762),(-14.863145,11.218645),(-14.843249,11.297193),(-14.828212,11.334787),(-14.728993,11.478009),(-14.711061,11.497568),(-14.686773,11.51072),(-14.64538,11.515061),(-14.562259,11.505604),(-14.520944,11.51289),(-14.432268,11.560575),(-14.339223,11.61061),(-14.326485,11.620584),(-14.319793,11.630506),(-14.308218,11.654535),(-14.300389,11.663165),(-14.289692,11.66885),(-14.284472,11.668178),(-14.278736,11.665077),(-14.19502,11.659703),(-14.171973,11.662752),(-14.161844,11.660943),(-14.153085,11.654845),(-14.144481,11.647404),(-14.134921,11.641771),(-14.118565,11.637999),(-14.104664,11.638567),(-14.07464,11.643735),(-14.058827,11.644045),(-14.015135,11.63712),(-13.996454,11.643838),(-13.953355,11.669676),(-13.945423,11.675671),(-13.923745,11.665026),(-13.903643,11.665387),(-13.886409,11.675568),(-13.873515,11.694533),(-13.870673,11.704145),(-13.868529,11.727037),(-13.87305,11.738354),(-13.870053,11.743677),(-13.855041,11.743419),(-13.846282,11.737269),(-13.842975,11.71727),(-13.834551,11.712103),(-13.820134,11.708072),(-13.819514,11.701664),(-13.821684,11.694223),(-13.815767,11.686833),(-13.795691,11.684042),(-13.770007,11.688693),(-13.745771,11.698202),(-13.729932,11.709829),(-13.724765,11.72311),(-13.719571,11.766105),(-13.719365,11.784036),(-13.734635,11.901859),(-13.731328,11.943406),(-13.722594,11.983869),(-13.723938,12.002783),(-13.734919,12.019267),(-13.744428,12.023918),(-13.779645,12.029344),(-13.795897,12.036269),(-13.807757,12.044279),(-13.829642,12.068102),(-13.835197,12.077765),(-13.837729,12.085413),(-13.842406,12.091304),(-13.869717,12.10071),(-13.879768,12.107686),(-13.899302,12.126238),(-13.918396,12.137762),(-13.938654,12.144273),(-13.964853,12.14789),(-13.972527,12.151559),(-13.968367,12.162566),(-13.968264,12.18396),(-13.965732,12.194244),(-13.943925,12.218635),(-13.883153,12.246437),(-13.858968,12.265971),(-13.840003,12.277133),(-13.812641,12.283592),(-13.795691,12.278735),(-13.808558,12.256101),(-13.740836,12.256927),(-13.71182,12.267521),(-13.695025,12.292946),(-13.69102,12.319301),(-13.68301,12.33868),(-13.682028,12.350307),(-13.685155,12.379194),(-13.684147,12.39201),(-13.682028,12.400329),(-13.664433,12.441516),(-13.660712,12.458672),(-13.663399,12.475777),(-13.67469,12.496086),(-13.725075,12.558149),(-13.734635,12.585331),(-13.736314,12.607604),(-13.728279,12.673388),(-13.404758,12.661761),(-13.359801,12.649737),(-13.356648,12.648893),(-13.352049,12.646981),(-13.343444,12.640987),(-13.338613,12.63923)] +Gambia [(-14.74093,13.615449),(-14.728579,13.619325),(-14.667188,13.652889),(-14.626674,13.663508),(-14.585539,13.660459),(-14.542596,13.640822),(-14.523243,13.625655),(-14.50973,13.609093),(-14.500221,13.589973),(-14.486863,13.546539),(-14.480378,13.533129),(-14.470275,13.522354),(-14.462135,13.516656),(-14.389582,13.465872),(-14.365553,13.454839),(-14.347285,13.452358),(-14.329947,13.456441),(-14.271217,13.476956),(-14.215794,13.510753),(-14.196829,13.518788),(-14.139132,13.531552),(-14.092882,13.554729),(-14.072599,13.55969),(-14.06185,13.560052),(-14.044461,13.558605),(-14.034022,13.560207),(-14.022266,13.565323),(-14.001905,13.577028),(-13.988495,13.579146),(-13.967489,13.575581),(-13.94209,13.566925),(-13.918086,13.554988),(-13.901343,13.541629),(-13.896175,13.532922),(-13.889793,13.513957),(-13.8846,13.505533),(-13.882504,13.503782),(-13.875479,13.497911),(-13.852793,13.484811),(-13.84269,13.476646),(-13.818713,13.429362),(-13.822562,13.378203),(-13.851036,13.33575),(-13.900568,13.314563),(-13.976119,13.308207),(-14.016272,13.297639),(-14.056579,13.297174),(-14.099496,13.281955),(-14.116808,13.282162),(-14.131794,13.275573),(-14.179827,13.240123),(-14.201687,13.229555),(-14.230625,13.228574),(-14.284911,13.238547),(-14.341781,13.233638),(-14.368834,13.235705),(-14.394982,13.242965),(-14.43144,13.26144),(-14.442034,13.268545),(-14.45149,13.276865),(-14.459139,13.286916),(-14.470326,13.298259),(-14.483633,13.301928),(-14.515104,13.303168),(-14.529574,13.307277),(-14.542854,13.314666),(-14.552109,13.323094),(-14.580733,13.34916),(-14.592154,13.35314),(-14.624555,13.345181),(-14.661607,13.340918),(-14.698142,13.345181),(-14.731577,13.359005),(-14.759482,13.38337),(-14.779636,13.410888),(-14.790333,13.417348),(-14.810797,13.421172),(-14.829142,13.427011),(-14.862576,13.447061),(-14.877614,13.451376),(-14.915907,13.454425),(-14.949703,13.463081),(-15.015074,13.495741),(-15.015229,13.495844),(-15.015435,13.495999),(-15.01559,13.496051),(-15.065251,13.531268),(-15.110158,13.57248),(-15.13765,13.589973),(-15.160491,13.580981),(-15.181331,13.559843),(-15.203951,13.536901),(-15.217955,13.51468),(-15.221728,13.452823),(-15.227774,13.425176),(-15.248341,13.398977),(-15.277073,13.380296),(-15.309836,13.368436),(-15.342392,13.36257),(-15.379237,13.362519),(-15.488326,13.385256),(-15.519539,13.3866),(-15.542948,13.377996),(-15.566048,13.365671),(-15.596123,13.355827),(-15.612453,13.354225),(-15.679891,13.360658),(-15.6925,13.360348),(-15.737407,13.346344),(-15.807118,13.339781),(-15.818694,13.333528),(-15.822673,13.319421),(-15.824367,13.248512),(-15.82474,13.232914),(-15.8266,13.161678),(-15.833318,13.156847),(-15.870784,13.157157),(-15.897441,13.157399),(-15.961734,13.157984),(-16.092062,13.15912),(-16.313289,13.161084),(-16.465218,13.162428),(-16.630014,13.1639),(-16.673525,13.164314),(-16.708407,13.156692),(-16.723186,13.132249),(-16.726339,13.122637),(-16.742152,13.107392),(-16.748508,13.099227),(-16.752125,13.088039),(-16.753651,13.065009),(-16.76887,13.077826),(-16.784332,13.082994),(-16.786733,13.090074),(-16.782541,13.098334),(-16.775502,13.112738),(-16.777333,13.127834),(-16.796742,13.177232),(-16.797475,13.189114),(-16.795033,13.235907),(-16.797353,13.248928),(-16.802968,13.262193),(-16.820058,13.281317),(-16.824452,13.294582),(-16.81664,13.310614),(-16.820302,13.315823),(-16.829701,13.338528),(-16.820709,13.349107),(-16.809967,13.379869),(-16.79955,13.386298),(-16.787343,13.389146),(-16.774322,13.396186),(-16.762929,13.404975),(-16.704457,13.475409),(-16.678822,13.496161),(-16.676747,13.485907),(-16.671132,13.478664),(-16.662506,13.473131),(-16.652252,13.468248),(-16.634633,13.477362),(-16.616851,13.478176),(-16.601389,13.472235),(-16.590728,13.461371),(-16.585317,13.443345),(-16.596303,13.440253),(-16.612782,13.439154),(-16.624257,13.427232),(-16.608713,13.413723),(-16.59553,13.385077),(-16.588368,13.355292),(-16.590728,13.338528),(-16.553863,13.295803),(-16.545766,13.290107),(-16.495188,13.287909),(-16.480824,13.283881),(-16.467763,13.275865),(-16.45995,13.268866),(-16.449696,13.264065),(-16.429351,13.262193),(-16.419667,13.255561),(-16.41393,13.239976),(-16.413482,13.222073),(-16.419423,13.208197),(-16.39977,13.224311),(-16.390859,13.224799),(-16.377797,13.215033),(-16.383656,13.225165),(-16.401926,13.243354),(-16.405751,13.246039),(-16.408274,13.258734),(-16.415273,13.267035),(-16.426015,13.270738),(-16.439931,13.269599),(-16.439931,13.276435),(-16.410512,13.275865),(-16.396799,13.277899),(-16.384633,13.283881),(-16.374745,13.263861),(-16.363271,13.268297),(-16.34968,13.282172),(-16.333404,13.290107),(-16.314809,13.295559),(-16.277943,13.319485),(-16.257965,13.324897),(-16.230133,13.316392),(-16.225901,13.296576),(-16.230214,13.273627),(-16.22761,13.256008),(-16.209706,13.251166),(-16.185455,13.258287),(-16.151275,13.276435),(-16.151275,13.283881),(-16.162262,13.282701),(-16.172515,13.279283),(-16.181752,13.273749),(-16.189809,13.265937),(-16.19929,13.259101),(-16.203521,13.26496),(-16.208079,13.292385),(-16.208608,13.303534),(-16.210764,13.3133),(-16.21703,13.317369),(-16.222035,13.31977),(-16.22525,13.325832),(-16.227122,13.333686),(-16.22761,13.341946),(-16.224436,13.360297),(-16.216217,13.372748),(-16.176178,13.405422),(-16.168691,13.413316),(-16.165517,13.423814),(-16.161204,13.428697),(-16.151438,13.426947),(-16.135121,13.420478),(-15.994862,13.413031),(-15.941314,13.425442),(-15.922231,13.427232),(-15.911285,13.426703),(-15.905629,13.425767),(-15.900624,13.425523),(-15.891225,13.427232),(-15.883778,13.430406),(-15.878407,13.434801),(-15.874379,13.43887),(-15.871327,13.440904),(-15.849517,13.442206),(-15.785024,13.43476),(-15.764475,13.436916),(-15.706207,13.454576),(-15.696767,13.459866),(-15.678049,13.473049),(-15.668691,13.475735),(-15.658925,13.472398),(-15.644683,13.457913),(-15.634918,13.454576),(-15.620839,13.45482),(-15.610422,13.456448),(-15.600901,13.460435),(-15.589589,13.468248),(-15.580922,13.478176),(-15.565053,13.502509),(-15.555409,13.509223),(-15.543446,13.50967),(-15.530995,13.505845),(-15.519643,13.499661),(-15.510732,13.492743),(-15.500396,13.488186),(-15.489654,13.489936),(-15.47936,13.493964),(-15.470041,13.496161),(-15.45108,13.490912),(-15.438873,13.478583),(-15.429433,13.464748),(-15.418853,13.454576),(-15.384429,13.443508),(-15.340566,13.440823),(-15.304799,13.455308),(-15.294749,13.496161),(-15.299957,13.496161),(-15.30191,13.496161),(-15.302154,13.496161),(-15.304067,13.481147),(-15.311391,13.467841),(-15.321848,13.458238),(-15.332875,13.454576),(-15.351674,13.456732),(-15.385976,13.466132),(-15.401479,13.468248),(-15.415761,13.473212),(-15.43224,13.484768),(-15.456451,13.50609),(-15.46524,13.510159),(-15.47411,13.508531),(-15.482818,13.505032),(-15.490834,13.502997),(-15.499867,13.505113),(-15.507558,13.509833),(-15.51358,13.514553),(-15.517893,13.516669),(-15.531728,13.519761),(-15.54304,13.525539),(-15.556142,13.528306),(-15.575836,13.522895),(-15.586008,13.515041),(-15.600697,13.49551),(-15.610666,13.489325),(-15.627756,13.488511),(-15.654652,13.500963),(-15.672109,13.502997),(-15.690175,13.498969),(-15.734202,13.475735),(-15.769887,13.465074),(-16.09024,13.440497),(-16.121897,13.451606),(-16.141591,13.454576),(-16.16039,13.450588),(-16.178456,13.440985),(-16.206532,13.420478),(-16.248443,13.378974),(-16.257965,13.372016),(-16.265614,13.36872),(-16.27302,13.362616),(-16.280588,13.359768),(-16.289133,13.365871),(-16.289703,13.373236),(-16.286733,13.383612),(-16.284901,13.395331),(-16.289133,13.406806),(-16.30602,13.391669),(-16.308095,13.383368),(-16.302113,13.372016),(-16.312123,13.366929),(-16.316396,13.365871),(-16.316396,13.359036),(-16.311635,13.357611),(-16.306996,13.35456),(-16.302113,13.352769),(-16.321889,13.342841),(-16.355946,13.337958),(-16.390533,13.336859),(-16.411977,13.338528),(-16.489491,13.357123),(-16.52184,13.359036),(-16.515777,13.367865),(-16.503529,13.392564),(-16.501332,13.403062),(-16.503529,13.416938),(-16.513417,13.436754),(-16.515696,13.444322),(-16.519765,13.451321),(-16.538197,13.46483),(-16.544504,13.475898),(-16.554433,13.481838),(-16.556549,13.485582),(-16.556549,13.50609),(-16.548004,13.548082),(-16.549143,13.56387),(-16.554799,13.577053),(-16.561391,13.5869),(-16.561399,13.586914),(-16.498756,13.586665),(-16.436641,13.586459),(-16.374577,13.5862),(-16.312462,13.585994),(-16.250295,13.585735),(-16.188232,13.585528),(-16.126117,13.58527),(-16.085383,13.585168),(-16.064054,13.585115),(-16.001938,13.584908),(-15.939772,13.58465),(-15.877708,13.584443),(-15.815593,13.584185),(-15.753426,13.583978),(-15.691363,13.58372),(-15.629196,13.583513),(-15.567133,13.583255),(-15.518195,13.5831),(-15.502486,13.588267),(-15.50047,13.627981),(-15.49803,13.641522),(-15.496801,13.648341),(-15.488946,13.670329),(-15.478611,13.691258),(-15.467501,13.708286),(-15.43634,13.741178),(-15.394069,13.771512),(-15.347766,13.788643),(-15.304462,13.781873),(-15.295108,13.773191),(-15.27573,13.748051),(-15.267255,13.741798),(-15.245964,13.746733),(-15.17062,13.793784),(-15.097704,13.819984),(-15.07631,13.818951),(-15.016779,13.796885),(-14.915803,13.792441),(-14.879268,13.780581),(-14.860314,13.765595),(-14.843973,13.752676),(-14.831312,13.735623),(-14.822372,13.7172),(-14.802942,13.65232),(-14.796017,13.644698),(-14.754934,13.620384),(-14.74093,13.615449)] +Guatemala [(-89.160496,17.814314),(-89.156217,17.598243),(-89.150728,17.321032),(-89.149205,17.036271),(-89.166067,16.777009),(-89.184353,16.495872),(-89.19314,16.392626),(-89.208517,16.186132),(-89.234135,15.954945),(-89.236022,15.906493),(-89.236512,15.893915),(-89.228244,15.880841),(-89.225996,15.884044),(-89.216539,15.888644),(-89.204835,15.892416),(-89.195765,15.892933),(-89.177446,15.900271),(-89.100913,15.896809),(-89.072388,15.901511),(-89.040323,15.901718),(-88.951878,15.879652),(-88.913971,15.893948),(-88.900014,15.887844),(-88.8662,15.860338),(-88.831614,15.868638),(-88.787343,15.859076),(-88.753285,15.839789),(-88.749501,15.818793),(-88.726796,15.815172),(-88.625966,15.753892),(-88.638295,15.708808),(-88.63622,15.702094),(-88.609242,15.702094),(-88.598378,15.706122),(-88.595204,15.714667),(-88.596344,15.722154),(-88.598704,15.723212),(-88.598988,15.727362),(-88.602366,15.730902),(-88.60558,15.735541),(-88.605458,15.743069),(-88.600453,15.744574),(-88.591176,15.743557),(-88.582183,15.745022),(-88.578196,15.753892),(-88.57433,15.769965),(-88.5655,15.780951),(-88.55606,15.789496),(-88.550282,15.798285),(-88.549062,15.813666),(-88.556711,15.837348),(-88.557688,15.852932),(-88.550282,15.852932),(-88.550649,15.842475),(-88.550282,15.83926),(-88.544057,15.83926),(-88.532135,15.846829),(-88.503041,15.841376),(-88.48884,15.846747),(-88.511952,15.86758),(-88.52481,15.877021),(-88.543284,15.885159),(-88.551015,15.895168),(-88.557688,15.906562),(-88.564524,15.914984),(-88.607574,15.938422),(-88.619862,15.953274),(-88.605458,15.969631),(-88.590199,15.961005),(-88.561838,15.949042),(-88.550282,15.942328),(-88.52123,15.912746),(-88.500234,15.901068),(-88.447906,15.85871),(-88.344838,15.811957),(-88.311025,15.782416),(-88.300404,15.777818),(-88.256337,15.740383),(-88.221547,15.725898),(-88.220937,15.725653),(-88.220922,15.725647),(-88.23262,15.721419),(-88.235746,15.712427),(-88.23492,15.701575),(-88.241586,15.688242),(-88.247761,15.695684),(-88.264995,15.688604),(-88.279697,15.680749),(-88.295588,15.675013),(-88.316672,15.6746),(-88.322304,15.667262),(-88.352561,15.616903),(-88.398992,15.572797),(-88.467434,15.52074),(-88.50002,15.495954),(-88.50002,15.495903),(-88.587379,15.4303),(-88.674195,15.365058),(-88.760899,15.299962),(-88.847027,15.235299),(-88.973427,15.140395),(-89.008283,15.124401),(-89.109827,15.091457),(-89.140523,15.076394),(-89.14694,15.071284),(-89.160806,15.060245),(-89.172769,15.04221),(-89.188582,14.996088),(-89.1691,14.978415),(-89.169152,14.95206),(-89.181528,14.923664),(-89.189215,14.913059),(-89.198814,14.899815),(-89.223619,14.879015),(-89.231448,14.867311),(-89.23243,14.8485),(-89.227676,14.834574),(-89.218968,14.82168),(-89.198814,14.79809),(-89.189409,14.783001),(-89.170547,14.738791),(-89.159669,14.725123),(-89.150523,14.717294),(-89.146213,14.710858),(-89.144502,14.708302),(-89.142694,14.691275),(-89.145226,14.683446),(-89.155845,14.672258),(-89.159411,14.665979),(-89.155225,14.615595),(-89.155742,14.597844),(-89.160393,14.581282),(-89.172149,14.570404),(-89.183699,14.569525),(-89.208814,14.577949),(-89.221087,14.580093),(-89.238114,14.577949),(-89.245685,14.572988),(-89.250542,14.565184),(-89.259612,14.554255),(-89.290927,14.528003),(-89.304854,14.513353),(-89.314802,14.495964),(-89.323044,14.487463),(-89.345265,14.48196),(-89.355497,14.475448),(-89.361983,14.4624),(-89.363791,14.446561),(-89.361621,14.415478),(-89.390611,14.435993),(-89.391697,14.43372),(-89.396864,14.426046),(-89.398182,14.445373),(-89.408362,14.441523),(-89.4311,14.418708),(-89.446835,14.415529),(-89.458127,14.419586),(-89.469418,14.425374),(-89.485283,14.427389),(-89.496148,14.422897),(-89.503033,14.420051),(-89.527761,14.391061),(-89.541351,14.381526),(-89.544245,14.400052),(-89.555175,14.410827),(-89.569877,14.412015),(-89.584062,14.401809),(-89.590573,14.385971),(-89.586284,14.374679),(-89.578972,14.364783),(-89.576078,14.35313),(-89.582408,14.343363),(-89.592434,14.336387),(-89.598351,14.32799),(-89.592175,14.31383),(-89.571272,14.311143),(-89.564373,14.308146),(-89.558586,14.302823),(-89.555692,14.299025),(-89.549878,14.288276),(-89.548586,14.2836),(-89.546778,14.268329),(-89.544917,14.261818),(-89.54055,14.257606),(-89.527864,14.251457),(-89.52466,14.247374),(-89.524505,14.231768),(-89.530939,14.225567),(-89.638322,14.200556),(-89.664626,14.188851),(-89.689844,14.170015),(-89.709868,14.148724),(-89.752605,14.07524),(-89.755447,14.067075),(-89.754465,14.059401),(-89.74801,14.044895),(-89.747256,14.043201),(-89.747644,14.037646),(-89.762268,14.029997),(-89.776944,14.035785),(-89.802963,14.055474),(-89.821102,14.060073),(-89.835855,14.059091),(-89.880116,14.042684),(-89.890788,14.035889),(-89.912078,14.015192),(-90.008171,13.949692),(-90.022976,13.936954),(-90.031477,13.922924),(-90.038091,13.908377),(-90.047367,13.894295),(-90.05964,13.884709),(-90.08708,13.870472),(-90.099044,13.858509),(-90.106976,13.844066),(-90.112299,13.828253),(-90.114779,13.812),(-90.114314,13.796265),(-90.092982,13.728903),(-90.098328,13.731414),(-90.245553,13.790726),(-90.401159,13.858588),(-90.500209,13.8947),(-90.580413,13.913267),(-90.638019,13.922358),(-90.707278,13.928889),(-90.784615,13.921538),(-90.796247,13.918161),(-90.910333,13.911135),(-91.04374,13.913585),(-91.165761,13.926703),(-91.316461,13.955679),(-91.551869,14.057278),(-91.637115,14.104438),(-91.762558,14.180174),(-91.908193,14.28384),(-92.049591,14.408026),(-92.246234,14.546283),(-92.226097,14.549346),(-92.208062,14.570507),(-92.186074,14.629573),(-92.164731,14.66815),(-92.160804,14.683498),(-92.161734,14.702334),(-92.179382,14.751581),(-92.180105,14.75879),(-92.178813,14.773389),(-92.179382,14.781244),(-92.188658,14.805893),(-92.190828,14.814239),(-92.191655,14.835297),(-92.186797,14.84726),(-92.166023,14.870825),(-92.15468,14.900797),(-92.157574,14.963041),(-92.154215,14.995933),(-92.142356,15.007483),(-92.129876,15.010945),(-92.116982,15.012392),(-92.103831,15.01787),(-92.093599,15.02942),(-92.073807,15.073706),(-92.20336,15.237159),(-92.21093,15.253359),(-92.211292,15.272066),(-92.204651,15.289507),(-92.186177,15.320384),(-92.137446,15.401826),(-92.105931,15.454469),(-92.068536,15.516935),(-91.9896,15.648968),(-91.910561,15.78095),(-91.841676,15.896085),(-91.792971,15.977579),(-91.774471,16.00843),(-91.751888,16.046308),(-91.739848,16.060985),(-91.723776,16.068788),(-91.678405,16.068891),(-91.54048,16.069149),(-91.402607,16.069305),(-91.264786,16.069511),(-91.126914,16.06977),(-90.994798,16.069869),(-90.988989,16.069873),(-90.851116,16.070131),(-90.713295,16.070338),(-90.575474,16.070596),(-90.485738,16.0707),(-90.466153,16.072922),(-90.448195,16.07926),(-90.448144,16.079278),(-90.447841,16.079687),(-90.444165,16.084652),(-90.437421,16.098863),(-90.434475,16.107183),(-90.440211,16.104858),(-90.442743,16.104186),(-90.444526,16.103256),(-90.448144,16.099742),(-90.455585,16.099742),(-90.459487,16.116795),(-90.435302,16.136277),(-90.448144,16.148214),(-90.448144,16.154416),(-90.42613,16.163149),(-90.432718,16.173226),(-90.461812,16.189142),(-90.465584,16.208107),(-90.458505,16.22175),(-90.446232,16.230845),(-90.434475,16.236323),(-90.447989,16.24149),(-90.457833,16.249138),(-90.459435,16.25906),(-90.448144,16.271101),(-90.443725,16.266864),(-90.440625,16.265313),(-90.42768,16.263608),(-90.431426,16.268931),(-90.437524,16.280041),(-90.441297,16.285364),(-90.436749,16.284847),(-90.435509,16.286242),(-90.435509,16.288774),(-90.434475,16.291565),(-90.428558,16.287637),(-90.420238,16.285364),(-90.414063,16.287844),(-90.414011,16.298386),(-90.420187,16.296009),(-90.434734,16.301228),(-90.444888,16.30836),(-90.437886,16.31208),(-90.419463,16.317816),(-90.404684,16.331511),(-90.398793,16.347582),(-90.40719,16.36045),(-90.414011,16.36045),(-90.414011,16.353008),(-90.420807,16.353008),(-90.418223,16.367167),(-90.40874,16.372335),(-90.395124,16.371405),(-90.379853,16.367271),(-90.391377,16.385978),(-90.392695,16.391714),(-90.392902,16.40484),(-90.394917,16.419206),(-90.400601,16.420549),(-90.409464,16.416673),(-90.420807,16.415123),(-90.426698,16.417707),(-90.434114,16.42241),(-90.444371,16.426802),(-90.475842,16.430729),(-90.478426,16.436414),(-90.477082,16.444992),(-90.482302,16.456051),(-90.488219,16.46158),(-90.497753,16.46804),(-90.508915,16.471089),(-90.532893,16.459927),(-90.539068,16.468867),(-90.543719,16.490209),(-90.553409,16.490261),(-90.57935,16.482768),(-90.584724,16.480235),(-90.590616,16.483388),(-90.604672,16.486178),(-90.620743,16.486747),(-90.633094,16.483336),(-90.627874,16.492896),(-90.618986,16.500079),(-90.612061,16.507624),(-90.61263,16.518114),(-90.624154,16.524936),(-90.639502,16.525866),(-90.648183,16.529586),(-90.639967,16.544831),(-90.643532,16.561006),(-90.637021,16.577904),(-90.633765,16.592167),(-90.647356,16.600073),(-90.65025,16.593407),(-90.653609,16.5901),(-90.658828,16.588446),(-90.6672,16.586999),(-90.6672,16.5932),(-90.654023,16.61661),(-90.667717,16.652576),(-90.73531,16.746576),(-90.749779,16.757686),(-90.785591,16.774998),(-90.80037,16.786832),(-90.79696,16.798666),(-90.79696,16.805435),(-90.810602,16.812928),(-90.819956,16.805332),(-90.831738,16.805435),(-90.900623,16.826003),(-90.919071,16.836338),(-90.948165,16.864708),(-90.968887,16.874372),(-90.952764,16.890908),(-90.955193,16.898763),(-90.968835,16.901037),(-90.98625,16.901088),(-90.987956,16.896903),(-90.984287,16.887756),(-90.983253,16.878609),(-90.993072,16.874372),(-90.998343,16.877162),(-91.0109,16.890288),(-91.016688,16.894836),(-91.054928,16.908013),(-91.066866,16.918193),(-91.07131,16.938916),(-91.076943,16.949509),(-91.112238,16.990489),(-91.116113,16.999274),(-91.121229,17.018394),(-91.126552,17.024595),(-91.137146,17.026507),(-91.161227,17.023975),(-91.1712,17.028006),(-91.206754,17.065368),(-91.215952,17.079217),(-91.220241,17.089294),(-91.225357,17.107019),(-91.229595,17.113995),(-91.238793,17.119215),(-91.259205,17.124331),(-91.263133,17.130738),(-91.265148,17.146241),(-91.270832,17.164276),(-91.279514,17.180089),(-91.290418,17.189081),(-91.317755,17.190476),(-91.344523,17.186342),(-91.364884,17.189753),(-91.372945,17.213576),(-91.388345,17.221586),(-91.419247,17.227735),(-91.442502,17.237967),(-91.438264,17.253677),(-91.430771,17.254969),(-90.991986,17.25192),(-90.991444,17.526942),(-90.990901,17.801964),(-90.98284,17.810542),(-90.962221,17.81602),(-90.919123,17.81602),(-90.699239,17.815813),(-90.47946,17.815606),(-90.259654,17.815348),(-90.039771,17.815141),(-89.820017,17.814986),(-89.600159,17.814728),(-89.380302,17.814521),(-89.160496,17.814314)] +Guam [(144.886404,13.640204),(144.896658,13.617987),(144.914317,13.605414),(144.934418,13.59984),(144.951915,13.598578),(144.952159,13.586859),(144.937673,13.560533),(144.906993,13.516669),(144.871837,13.485175),(144.808279,13.445787),(144.783458,13.420478),(144.769379,13.379136),(144.760102,13.285631),(144.735606,13.249172),(144.718028,13.241848),(144.69752,13.241034),(144.680675,13.246772),(144.670421,13.264838),(144.656749,13.28205),(144.653575,13.287014),(144.647472,13.324897),(144.644298,13.33161),(144.639496,13.33747),(144.633637,13.342108),(144.626313,13.345364),(144.636567,13.355414),(144.643077,13.365709),(144.646495,13.378485),(144.647472,13.396226),(144.645518,13.404486),(144.640473,13.414781),(144.634776,13.423529),(144.624197,13.43122),(144.628673,13.438788),(144.636485,13.443101),(144.640636,13.437812),(144.65447,13.439154),(144.714854,13.473456),(144.72877,13.485582),(144.783458,13.516669),(144.794119,13.527086),(144.803233,13.53913),(144.811209,13.553616),(144.832205,13.620673),(144.838634,13.632758),(144.850434,13.643744),(144.86671,13.653022),(144.880626,13.65412),(144.886404,13.640204)] +Heard Island and McDonald Islands [(73.735118,-53.1124),(73.761892,-53.11891),(73.789073,-53.120538),(73.812185,-53.118504),(73.661632,-53.146661),(73.609548,-53.179864),(73.580821,-53.187595),(73.507335,-53.19256),(73.473399,-53.189223),(73.440196,-53.173272),(73.430431,-53.165297),(73.404145,-53.139337),(73.370942,-53.090509),(73.36378,-53.074802),(73.371267,-53.06463),(73.359711,-53.058852),(73.353282,-53.05169),(73.354177,-53.044203),(73.364431,-53.037286),(73.344737,-53.029229),(73.328868,-53.025079),(73.271007,-53.021742),(73.253673,-53.015069),(73.24171,-53.002211),(73.236013,-52.982029),(73.269379,-52.975274),(73.296886,-52.961602),(73.330903,-52.984796),(73.337169,-52.985772),(73.344574,-52.996515),(73.361095,-53.00921),(73.377452,-53.017022),(73.384939,-53.013116),(73.389171,-52.997817),(73.399587,-52.990981),(73.412934,-52.992852),(73.426524,-53.003188),(73.412852,-53.009373),(73.425629,-53.025323),(73.447032,-53.02809),(73.470225,-53.02158),(73.487966,-53.009373),(73.50766,-53.015313),(73.56422,-53.016778),(73.587576,-53.026788),(73.598481,-53.033868),(73.620942,-53.039646),(73.632579,-53.044122),(73.640147,-53.050714),(73.65561,-53.067966),(73.663341,-53.071466),(73.688975,-53.090265),(73.699474,-53.100844),(73.735118,-53.1124)] +Hungary [(20.981489,48.516859),(21.0065,48.518151),(21.035956,48.514637),(21.063861,48.506239),(21.084015,48.49301),(21.109336,48.489109),(21.186748,48.513707),(21.219924,48.518719),(21.238011,48.513448),(21.250413,48.506498),(21.261782,48.50319),(21.276561,48.50872),(21.28824,48.519934),(21.293925,48.530579),(21.302296,48.539907),(21.32183,48.54758),(21.338573,48.549854),(21.372679,48.550345),(21.424563,48.561275),(21.439135,48.558329),(21.472725,48.544997),(21.490812,48.540268),(21.49939,48.535075),(21.506005,48.526496),(21.5151,48.507118),(21.521818,48.50009),(21.537734,48.495232),(21.574631,48.495568),(21.591684,48.49301),(21.600366,48.481641),(21.613388,48.440352),(21.621553,48.429655),(21.67757,48.372346),(21.701238,48.353949),(21.727697,48.340901),(21.759012,48.333769),(21.789398,48.335526),(21.841178,48.353174),(21.884276,48.357463),(21.914765,48.36909),(21.929338,48.372914),(21.981531,48.374723),(21.999928,48.37896),(22.018015,48.379735),(22.077856,48.375808),(22.09646,48.379425),(22.113926,48.38865),(22.13284,48.404798),(22.156508,48.40206),(22.158263,48.402222),(22.158315,48.402227),(22.159298,48.402318),(22.16922,48.409527),(22.201983,48.418157),(22.236089,48.415289),(22.271849,48.403455),(22.256967,48.373224),(22.25676,48.357282),(22.284355,48.358393),(22.29128,48.357566),(22.298721,48.349143),(22.298721,48.339324),(22.296447,48.327801),(22.297378,48.314003),(22.308126,48.293694),(22.357116,48.243103),(22.363627,48.23871),(22.370861,48.237393),(22.378613,48.238865),(22.386468,48.243051),(22.390498,48.244343),(22.394529,48.244757),(22.398767,48.244343),(22.418404,48.238969),(22.434217,48.236747),(22.44972,48.237677),(22.469047,48.244136),(22.473284,48.244498),(22.477418,48.24393),(22.481449,48.242586),(22.55576,48.177164),(22.568886,48.156519),(22.583045,48.124816),(22.600098,48.101122),(22.605472,48.097039),(22.608056,48.096833),(22.621182,48.101768),(22.693219,48.101768),(22.711616,48.105824),(22.728566,48.113137),(22.745722,48.116289),(22.762052,48.109261),(22.765566,48.104533),(22.801429,48.090968),(22.830988,48.072442),(22.844321,48.061047),(22.85476,48.047301),(22.861581,48.028387),(22.857964,48.018026),(22.851349,48.008802),(22.849489,47.993144),(22.832435,47.978933),(22.840807,47.966789),(22.877601,47.946739),(22.861167,47.933819),(22.836053,47.902452),(22.819723,47.892323),(22.779622,47.882298),(22.763602,47.874753),(22.752854,47.86124),(22.753267,47.852687),(22.758331,47.846228),(22.760295,47.838916),(22.75182,47.827676),(22.745826,47.824989),(22.724225,47.82349),(22.703864,47.817186),(22.691669,47.8107),(22.666967,47.78879),(22.637719,47.771555),(22.601235,47.760936),(22.562994,47.757215),(22.528475,47.761039),(22.454371,47.787394),(22.423675,47.782563),(22.407345,47.743082),(22.395873,47.735692),(22.382644,47.732023),(22.368278,47.73117),(22.322079,47.735873),(22.309366,47.735046),(22.29159,47.730705),(22.27309,47.723755),(22.261721,47.715848),(22.23981,47.693472),(22.232162,47.688253),(22.215212,47.679933),(22.207874,47.673732),(22.204153,47.666291),(22.200743,47.647945),(22.197642,47.639315),(22.172837,47.615389),(22.16984,47.608775),(22.169117,47.601385),(22.167608,47.594862),(22.16736,47.593789),(22.162089,47.586244),(22.154177,47.582213),(22.148549,47.579345),(22.09956,47.570948),(22.037445,47.539322),(22.007886,47.517411),(21.988869,47.492942),(21.991453,47.461807),(22.000238,47.427184),(22.001582,47.393827),(21.981531,47.366102),(21.936986,47.357162),(21.919003,47.349669),(21.900813,47.335742),(21.862159,47.297424),(21.856061,47.285745),(21.844589,47.249934),(21.839318,47.240839),(21.827742,47.226007),(21.823608,47.214923),(21.823298,47.203321),(21.825675,47.194278),(21.825985,47.185054),(21.819887,47.172729),(21.811516,47.164693),(21.789398,47.150301),(21.77989,47.140741),(21.775446,47.131672),(21.770485,47.114025),(21.763767,47.105214),(21.74351,47.091597),(21.69421,47.069169),(21.671886,47.054726),(21.632819,47.022686),(21.63427,47.019422),(21.636643,47.014082),(21.645221,47.01124),(21.654833,47.009948),(21.661757,47.006072),(21.670749,46.994394),(21.671369,46.99336),(21.667545,46.992378),(21.648528,46.942976),(21.640053,46.935689),(21.594371,46.910006),(21.589927,46.908869),(21.588687,46.906182),(21.587137,46.8944),(21.588067,46.882204),(21.591477,46.871533),(21.591788,46.860707),(21.583416,46.847943),(21.573081,46.841767),(21.536597,46.835023),(21.51572,46.821588),(21.502904,46.80531),(21.48151,46.764976),(21.477066,46.760222),(21.473655,46.754873),(21.471381,46.749034),(21.470451,46.743014),(21.472622,46.740197),(21.475206,46.737846),(21.478203,46.735934),(21.505178,46.723247),(21.501767,46.703507),(21.483577,46.684852),(21.463527,46.677049),(21.436241,46.673741),(21.425493,46.662011),(21.423539,46.658231),(21.416811,46.645216),(21.396037,46.626354),(21.374436,46.618448),(21.337643,46.620411),(21.316249,46.616639),(21.300953,46.603926),(21.295268,46.585039),(21.291341,46.546721),(21.278938,46.528401),(21.261679,46.513338),(21.247623,46.497473),(21.245142,46.47688),(21.274391,46.438355),(21.280695,46.416393),(21.257544,46.404171),(21.21517,46.402905),(21.195636,46.398099),(21.17879,46.384457),(21.168661,46.362753),(21.164527,46.318259),(21.155949,46.298932),(21.144476,46.283739),(21.134865,46.27852),(21.105616,46.278675),(21.099208,46.27635),(21.051459,46.236094),(21.033475,46.231339),(21.013942,46.24307),(21.007017,46.248858),(20.999162,46.251597),(20.990584,46.251597),(20.981489,46.248858),(20.961748,46.248341),(20.924438,46.259555),(20.906558,46.26219),(20.899737,46.260692),(20.885061,46.255007),(20.875139,46.254387),(20.866974,46.257126),(20.848991,46.267771),(20.839689,46.271079),(20.819638,46.271699),(20.798658,46.267616),(20.787369,46.26339),(20.778504,46.260072),(20.739333,46.237489),(20.734992,46.231908),(20.735509,46.222503),(20.744811,46.200282),(20.744707,46.192272),(20.736646,46.186691),(20.727137,46.187725),(20.717836,46.189998),(20.710291,46.188086),(20.70533,46.180645),(20.704368,46.168526),(20.704193,46.16633),(20.698819,46.15646),(20.683523,46.144678),(20.663989,46.137753),(20.607558,46.129485),(20.600117,46.12964),(20.588025,46.132844),(20.578103,46.137547),(20.548957,46.15615),(20.509373,46.167674),(20.468549,46.174134),(20.444157,46.1469),(20.283237,46.1438),(20.242826,46.108091),(20.188462,46.140389),(20.170479,46.145505),(20.145364,46.137082),(20.138026,46.136461),(20.130378,46.139355),(20.120146,46.149226),(20.114772,46.152223),(20.098442,46.154962),(20.088623,46.154135),(20.063405,46.145298),(20.034983,46.142973),(19.993125,46.159406),(19.92915,46.16354),(19.888946,46.15739),(19.873649,46.152992),(19.79045,46.129072),(19.772984,46.131552),(19.711889,46.15871),(19.690095,46.168398),(19.669321,46.1731),(19.647824,46.173875),(19.589739,46.165969),(19.568449,46.166434),(19.549948,46.16416),(19.525247,46.156409),(19.501993,46.145608),(19.487523,46.134239),(19.48928,46.126023),(19.496722,46.116876),(19.499305,46.108608),(19.47295,46.098686),(19.465922,46.091968),(19.460858,46.084475),(19.453727,46.07755),(19.43688,46.06799),(19.428302,46.06551),(19.417243,46.064321),(19.404738,46.060239),(19.396573,46.051557),(19.389131,46.041532),(19.378899,46.033677),(19.362156,46.029646),(19.325156,46.029491),(19.306966,46.026649),(19.298315,46.022123),(19.286915,46.016159),(19.279474,46.003808),(19.274823,45.991612),(19.263454,45.981432),(19.235652,45.977711),(19.148009,45.984068),(19.125788,45.993111),(19.125788,45.993266),(19.111009,46.012955),(19.088478,46.018846),(19.06543,46.012025),(19.049721,45.993111),(19.048584,45.963449),(19.03091,45.960038),(19.005589,45.96257),(18.981818,45.950788),(18.978717,45.947171),(18.977684,45.943502),(18.97882,45.939781),(18.986469,45.931254),(18.988742,45.927017),(18.987502,45.923813),(18.981818,45.921849),(18.962697,45.927947),(18.901306,45.931203),(18.886526,45.930428),(18.876604,45.922418),(18.865442,45.918077),(18.845392,45.913943),(18.828339,45.905726),(18.822451,45.905582),(18.817797,45.905468),(18.804878,45.913633),(18.794852,45.903039),(18.790408,45.893892),(18.785964,45.886916),(18.775525,45.882834),(18.76333,45.88397),(18.723849,45.89844),(18.674343,45.910377),(18.655636,45.907587),(18.633002,45.891929),(18.629281,45.886864),(18.626284,45.874824),(18.623597,45.868674),(18.62215,45.868416),(18.60737,45.856685),(18.585459,45.82692),(18.57347,45.816688),(18.530269,45.79085),(18.504224,45.784028),(18.481797,45.791418),(18.465467,45.783718),(18.43074,45.753642),(18.411723,45.743204),(18.404385,45.741809),(18.397564,45.741343),(18.390743,45.741809),(18.384128,45.7431),(18.383818,45.743204),(18.367178,45.75819),(18.351158,45.758397),(18.334932,45.751989),(18.317672,45.747286),(18.300516,45.751834),(18.283256,45.764908),(18.260518,45.765115),(18.229822,45.781238),(18.211219,45.785372),(18.12895,45.785372),(18.120165,45.783408),(18.102905,45.774623),(18.092053,45.771729),(18.080477,45.771729),(17.975367,45.792141),(17.906234,45.792141),(17.889791,45.792141),(17.88049,45.788524),(17.875735,45.780669),(17.870051,45.773486),(17.858062,45.771729),(17.857959,45.775863),(17.838218,45.799635),(17.809176,45.814414),(17.687323,45.840614),(17.664689,45.841647),(17.656524,45.84542),(17.654669,45.852113),(17.653217,45.857357),(17.65208,45.868829),(17.646912,45.883712),(17.645775,45.891825),(17.638437,45.901334),(17.591102,45.936215),(17.554308,45.947791),(17.511933,45.95394),(17.426667,45.956731),(17.418192,45.95301),(17.415402,45.949703),(17.411578,45.946602),(17.406203,45.943657),(17.399692,45.959935),(17.38915,45.963087),(17.365276,45.956731),(17.345328,45.955697),(17.342124,45.958643),(17.343675,45.967531),(17.337887,45.984636),(17.327138,45.972027),(17.316596,45.973784),(17.306613,45.979914),(17.304814,45.981019),(17.290138,45.984636),(17.290138,45.991457),(17.296959,45.992646),(17.309362,45.996625),(17.316803,45.997659),(17.307811,46.006392),(17.291482,46.007942),(17.275875,46.012025),(17.268951,46.028716),(17.266574,46.038793),(17.254068,46.070832),(17.24859,46.080238),(17.208593,46.116566),(17.197327,46.121165),(17.116195,46.123129),(17.101209,46.128193),(17.052943,46.153463),(17.041264,46.162196),(17.036303,46.172997),(17.028035,46.18049),(16.974808,46.210565),(16.9653,46.219454),(16.940702,46.251493),(16.903288,46.281931),(16.896054,46.286271),(16.889232,46.292163),(16.879312,46.312062),(16.871766,46.327199),(16.875073,46.342857),(16.865461,46.359135),(16.859411,46.364758),(16.850475,46.373062),(16.837659,46.381873),(16.832388,46.381925),(16.830218,46.377351),(16.827531,46.374018),(16.820916,46.3781),(16.816575,46.381976),(16.812338,46.385025),(16.807894,46.387351),(16.803449,46.388694),(16.776474,46.39288),(16.769343,46.39611),(16.761901,46.381873),(16.75508,46.381873),(16.742885,46.399624),(16.712395,46.412672),(16.693585,46.429622),(16.677462,46.44869),(16.659272,46.464193),(16.637671,46.474477),(16.61111,46.478069),(16.602118,46.482048),(16.594263,46.482978),(16.588062,46.479567),(16.583721,46.470653),(16.577623,46.470653),(16.564291,46.479929),(16.521089,46.498532),(16.515302,46.501711),(16.500832,46.544809),(16.467139,46.564704),(16.430242,46.604392),(16.394585,46.619016),(16.376395,46.629093),(16.372246,46.636341),(16.368437,46.642994),(16.377636,46.652864),(16.396652,46.659143),(16.402607,46.663109),(16.410502,46.668367),(16.405024,46.687255),(16.390038,46.694154),(16.371434,46.694929),(16.366216,46.696467),(16.365383,46.696712),(16.357585,46.699011),(16.357275,46.715832),(16.343426,46.714178),(16.334124,46.721749),(16.325546,46.733273),(16.314177,46.743324),(16.300431,46.772082),(16.298157,46.775802),(16.29949,46.77951),(16.302188,46.787016),(16.311097,46.797519),(16.3149,46.802002),(16.321825,46.813268),(16.327509,46.825463),(16.329783,46.834403),(16.325339,46.839442),(16.310663,46.84001),(16.301878,46.843214),(16.297392,46.847033),(16.282241,46.859932),(16.272009,46.863962),(16.179486,46.858468),(16.135376,46.855849),(16.130246,46.856708),(16.094035,46.862774),(16.110055,46.867916),(16.122871,46.876365),(16.159148,46.910316),(16.170723,46.918533),(16.196148,46.931297),(16.217025,46.937395),(16.22395,46.941064),(16.230668,46.94835),(16.231185,46.954422),(16.230358,46.959977),(16.232942,46.966075),(16.24307,46.972018),(16.252992,46.973516),(16.261054,46.97809),(16.265394,46.993257),(16.274903,47.004315),(16.288545,47.005582),(16.325856,47.00044),(16.366577,47.003825),(16.387764,47.002042),(16.405024,46.993153),(16.410398,46.990415),(16.415566,46.989433),(16.420424,46.99026),(16.424764,46.992998),(16.424764,46.993102),(16.441404,46.99522),(16.467449,46.995427),(16.486363,46.998554),(16.481919,47.00938),(16.467553,47.018423),(16.45329,47.021679),(16.424661,47.024082),(16.437167,47.031782),(16.481919,47.04421),(16.493184,47.049145),(16.497215,47.054622),(16.493701,47.059816),(16.481919,47.063898),(16.461765,47.068498),(16.454323,47.081701),(16.460835,47.0963),(16.481919,47.10524),(16.50476,47.125833),(16.509514,47.137537),(16.497318,47.149681),(16.480885,47.150767),(16.447089,47.139708),(16.433653,47.145754),(16.433963,47.151283),(16.442128,47.168336),(16.441818,47.177121),(16.4351,47.183581),(16.426728,47.184124),(16.41877,47.183659),(16.412776,47.187173),(16.409262,47.203683),(16.424661,47.225594),(16.421354,47.243035),(16.421354,47.243061),(16.421354,47.243138),(16.452463,47.254584),(16.466622,47.263421),(16.473237,47.276805),(16.469206,47.293238),(16.43882,47.336569),(16.436133,47.338559),(16.431689,47.339721),(16.427142,47.341375),(16.424351,47.345199),(16.424971,47.351116),(16.429312,47.353803),(16.434273,47.35556),(16.436443,47.358506),(16.433963,47.39685),(16.444091,47.40951),(16.456597,47.411836),(16.46993,47.405531),(16.481919,47.392302),(16.589406,47.425633),(16.626973,47.445549),(16.640875,47.452919),(16.636845,47.4932),(16.648213,47.501546),(16.677359,47.509866),(16.688004,47.52263),(16.689141,47.53803),(16.68139,47.550561),(16.66754,47.560096),(16.65059,47.566555),(16.656172,47.585934),(16.647697,47.606139),(16.63023,47.622004),(16.608422,47.628773),(16.575453,47.624949),(16.509617,47.64314),(16.481919,47.638954),(16.425901,47.654276),(16.407815,47.66133),(16.4166,47.668823),(16.431792,47.685463),(16.43913,47.690475),(16.444918,47.68554),(16.449879,47.682414),(16.45453,47.681768),(16.461145,47.684532),(16.47303,47.691767),(16.512924,47.706004),(16.521399,47.711533),(16.526877,47.720137),(16.52512,47.733315),(16.531115,47.742978),(16.567805,47.754192),(16.609766,47.750627),(16.689865,47.729568),(16.702474,47.7236),(16.707848,47.714608),(16.711569,47.704066),(16.719217,47.693731),(16.730379,47.685902),(16.741128,47.681458),(16.797455,47.675463),(16.80624,47.676884),(16.817299,47.684248),(16.836936,47.705358),(16.850165,47.712929),(16.864738,47.686729),(16.902668,47.682026),(16.98194,47.695436),(17.054804,47.702025),(17.075371,47.708484),(17.064002,47.71329),(17.055527,47.720913),(17.050566,47.730989),(17.048706,47.763649),(17.041988,47.783906),(17.040644,47.801114),(17.05563,47.812354),(17.049119,47.818684),(17.039507,47.837365),(17.031652,47.841345),(17.010672,47.847882),(17.004367,47.852377),(17.004057,47.863281),(17.016563,47.867699),(17.051186,47.872893),(17.067619,47.881626),(17.077541,47.891729),(17.083329,47.904829),(17.087463,47.922632),(17.085396,47.924621),(17.080125,47.925758),(17.075164,47.927773),(17.07413,47.932114),(17.076818,47.934956),(17.086119,47.93922),(17.088083,47.940899),(17.090874,47.949477),(17.095731,47.95573),(17.096145,47.961931),(17.085603,47.970148),(17.148338,48.005443),(17.184821,48.020274),(17.220892,48.015055),(17.262316,48.007283),(17.272671,48.00534),(17.337887,47.998725),(17.36941,47.981207),(17.472039,47.888809),(17.481858,47.882711),(17.492193,47.879818),(17.517308,47.876252),(17.526609,47.872118),(17.560406,47.837986),(17.572601,47.829536),(17.582937,47.829795),(17.592962,47.833102),(17.604227,47.834239),(17.61911,47.829226),(17.639884,47.819201),(17.658384,47.807316),(17.666239,47.797032),(17.676678,47.789177),(17.719245,47.773669),(17.741997,47.76538),(17.825713,47.750006),(17.883532,47.752521),(18.11262,47.762486),(18.23592,47.753882),(18.273024,47.756259),(18.347851,47.776775),(18.552593,47.792846),(18.597448,47.79065),(18.633829,47.779824),(18.663698,47.775896),(18.692843,47.777963),(18.717234,47.788118),(18.750307,47.81362),(18.767671,47.822302),(18.790305,47.826332),(18.814916,47.832194),(18.816453,47.83256),(18.778006,47.851447),(18.748757,47.870723),(18.742246,47.889455),(18.744726,47.910513),(18.754751,47.951803),(18.751444,47.963353),(18.744519,47.967357),(18.743176,47.971052),(18.756198,47.981827),(18.765293,47.985393),(18.784724,47.987615),(18.794232,47.993144),(18.821001,48.030454),(18.838467,48.040015),(18.933595,48.054349),(18.981818,48.061615),(18.996494,48.066214),(19.019017,48.065497),(19.038662,48.064871),(19.098503,48.070736),(19.222526,48.060582),(19.233482,48.06208),(19.29322,48.087764),(19.428199,48.085852),(19.481735,48.111328),(19.483286,48.116496),(19.483803,48.121767),(19.483286,48.127193),(19.481735,48.13267),(19.481425,48.133342),(19.481219,48.133962),(19.481425,48.134427),(19.481735,48.134892),(19.493621,48.150705),(19.503026,48.189411),(19.513982,48.203958),(19.531241,48.21065),(19.623226,48.227006),(19.633871,48.226773),(19.643896,48.224809),(19.655265,48.21835),(19.676969,48.200392),(19.686994,48.196904),(19.73309,48.202899),(19.756551,48.200315),(19.774327,48.185897),(19.774844,48.176079),(19.76916,48.167449),(19.766679,48.159),(19.776084,48.149517),(19.785593,48.14869),(19.821663,48.157914),(19.846261,48.152669),(19.884295,48.129621),(19.905069,48.124299),(19.928633,48.130087),(19.973902,48.158379),(19.99695,48.167914),(20.034983,48.175872),(20.038303,48.177233),(20.078082,48.193545),(20.096995,48.198429),(20.105263,48.202795),(20.112601,48.211735),(20.118079,48.229719),(20.122006,48.236747),(20.134512,48.246669),(20.14309,48.247805),(20.153322,48.245273),(20.170892,48.244033),(20.187636,48.248994),(20.217815,48.267598),(20.22908,48.270905),(20.249027,48.264187),(20.260293,48.255893),(20.272385,48.252456),(20.295226,48.260415),(20.324268,48.279948),(20.349279,48.305476),(20.370157,48.334338),(20.409017,48.413713),(20.420593,48.429241),(20.435579,48.442393),(20.465968,48.46379),(20.468239,48.465389),(20.481674,48.478747),(20.482501,48.482261),(20.482915,48.485827),(20.482811,48.489367),(20.482191,48.492881),(20.480227,48.510218),(20.480538,48.518538),(20.481674,48.526083),(20.51051,48.533783),(20.572522,48.536573),(20.784085,48.569052),(20.800311,48.569233),(20.815814,48.563807),(20.845477,48.545823),(20.859946,48.543317),(20.891365,48.541095),(20.945832,48.518978),(20.981489,48.516859)] +Isle of Man [(-4.612131,54.056952),(-4.620758,54.069648),(-4.631256,54.070624),(-4.643219,54.066596),(-4.656402,54.063788),(-4.67101,54.067043),(-4.689687,54.081529),(-4.704498,54.084866),(-4.725982,54.079006),(-4.745595,54.067369),(-4.766225,54.059231),(-4.790151,54.063788),(-4.790151,54.071234),(-4.78482,54.073065),(-4.781321,54.074774),(-4.777089,54.076239),(-4.769683,54.07746),(-4.770863,54.096829),(-4.737172,54.124905),(-4.728139,54.142564),(-4.727651,54.172593),(-4.723785,54.185452),(-4.714467,54.200344),(-4.71109,54.209906),(-4.71227,54.21662),(-4.710276,54.220649),(-4.68814,54.223782),(-4.663971,54.233547),(-4.643056,54.249416),(-4.612457,54.265448),(-4.598378,54.276597),(-4.554596,54.339179),(-4.530995,54.366116),(-4.495351,54.385891),(-4.442006,54.404364),(-4.404449,54.409654),(-4.385732,54.415717),(-4.367787,54.419013),(-4.351308,54.413804),(-4.357086,54.3994),(-4.377553,54.361884),(-4.3756,54.348334),(-4.370229,54.340888),(-4.363759,54.324286),(-4.358225,54.317613),(-4.350494,54.314765),(-4.332183,54.313422),(-4.324696,54.310207),(-4.319407,54.304592),(-4.313222,54.2956),(-4.31192,54.287177),(-4.320953,54.283515),(-4.326731,54.280341),(-4.34081,54.263007),(-4.348256,54.259508),(-4.35436,54.251532),(-4.359771,54.242255),(-4.365631,54.235053),(-4.371816,54.231269),(-4.399159,54.222073),(-4.390777,54.194729),(-4.412587,54.18008),(-4.446889,54.169135),(-4.475494,54.153144),(-4.474599,54.140815),(-4.529286,54.119086),(-4.543772,54.108466),(-4.549794,54.100653),(-4.563832,54.096829),(-4.579579,54.094428),(-4.590932,54.091051),(-4.598948,54.083889),(-4.612131,54.056952)] +Iraq [(42.89674,37.324906),(42.937048,37.320152),(42.979629,37.331831),(43.005158,37.347256),(43.043501,37.360253),(43.083706,37.368831),(43.114815,37.371131),(43.131971,37.367255),(43.263385,37.310695),(43.270309,37.30868),(43.278629,37.30775),(43.287518,37.309145),(43.296871,37.316741),(43.305656,37.319971),(43.324156,37.322219),(43.336145,37.32023),(43.362505,37.303908),(43.376039,37.295528),(43.41676,37.279173),(43.463269,37.248684),(43.479599,37.243361),(43.492415,37.244756),(43.517116,37.252301),(43.529518,37.253903),(43.542437,37.252301),(43.550602,37.248735),(43.568896,37.237702),(43.594217,37.22946),(43.618298,37.226979),(43.720824,37.232612),(43.746921,37.230648),(43.770641,37.225843),(43.780562,37.220365),(43.80206,37.20357),(43.809398,37.199694),(43.822007,37.202381),(43.83999,37.217109),(43.89363,37.224912),(43.924171,37.253076),(43.953782,37.287467),(43.990369,37.312504),(44.035586,37.31824),(44.069025,37.313727),(44.088037,37.31116),(44.184465,37.279173),(44.206893,37.26752),(44.223171,37.254058),(44.235005,37.236772),(44.243532,37.213828),(44.248544,37.191917),(44.249629,37.179437),(44.248337,37.16967),(44.240121,37.157966),(44.230199,37.154322),(44.218779,37.152617),(44.205963,37.146623),(44.18922,37.129208),(44.180641,37.108925),(44.180021,37.087608),(44.187463,37.066938),(44.22777,36.994125),(44.234281,36.983661),(44.243273,36.97777),(44.284821,36.969166),(44.297223,36.969941),(44.306628,36.977227),(44.315879,36.99397),(44.315982,36.99397),(44.316137,36.994022),(44.316137,36.994125),(44.33164,37.015468),(44.335154,37.031152),(44.343112,37.042443),(44.428068,37.064767),(44.454423,37.076343),(44.479228,37.092001),(44.503412,37.116624),(44.539379,37.143677),(44.57824,37.166415),(44.610279,37.178352),(44.628934,37.179024),(44.733786,37.167242),(44.753836,37.159154),(44.766135,37.14192),(44.752544,37.113136),(44.752699,37.103318),(44.760658,37.085644),(44.766549,37.078952),(44.773628,37.076343),(44.781121,37.074482),(44.788356,37.070245),(44.792232,37.063734),(44.797296,37.048489),(44.801895,37.043528),(44.811197,37.041926),(44.830938,37.046835),(44.840601,37.047197),(44.847577,37.044045),(44.858946,37.034123),(44.880754,37.024925),(44.887368,37.015933),(44.885715,37.005081),(44.874759,36.994022),(44.869178,36.967357),(44.874863,36.949658),(44.884061,36.933354),(44.888815,36.910926),(44.883441,36.886845),(44.869643,36.868965),(44.834141,36.834032),(44.823393,36.809331),(44.831248,36.791916),(44.851195,36.781219),(44.87724,36.776464),(44.908245,36.77786),(44.922715,36.775896),(44.935221,36.767111),(44.944419,36.758274),(44.954754,36.752331),(44.966433,36.749334),(44.979456,36.749024),(44.996199,36.741686),(45.010978,36.725563),(45.035266,36.689596),(45.045911,36.668047),(45.04431,36.64934),(45.024879,36.613167),(45.014182,36.579009),(45.013252,36.557925),(45.010255,36.556013),(44.997852,36.552085),(44.993408,36.549656),(44.991393,36.53374),(45.005914,36.518702),(45.009525,36.516196),(45.025344,36.505215),(45.03816,36.494001),(45.03909,36.479842),(45.045498,36.471625),(45.054076,36.464959),(45.062086,36.455347),(45.066892,36.442376),(45.068546,36.432506),(45.072266,36.423411),(45.083739,36.412817),(45.110817,36.402534),(45.139963,36.404394),(45.19598,36.422429),(45.221301,36.42093),(45.238871,36.402999),(45.24962,36.377936),(45.254478,36.354785),(45.25701,36.311687),(45.263934,36.294272),(45.282693,36.274893),(45.258508,36.262232),(45.264193,36.250192),(45.282796,36.238409),(45.297369,36.22673),(45.30202,36.207558),(45.299539,36.161411),(45.304604,36.140379),(45.33561,36.10772),(45.348322,36.087514),(45.344188,36.067464),(45.331889,36.051599),(45.31959,36.031032),(45.313699,36.010413),(45.32021,35.99429),(45.32021,35.994135),(45.32021,35.994083),(45.320313,35.994031),(45.32052,35.99398),(45.33809,35.979304),(45.359587,35.976875),(45.381808,35.982973),(45.401549,35.99398),(45.4016,35.994031),(45.401859,35.994031),(45.401962,35.994083),(45.402169,35.994238),(45.419946,35.998269),(45.453122,36.011653),(45.47927,36.012015),(45.501424,36.005435),(45.53999,35.99398),(45.576939,35.96623),(45.589858,35.959615),(45.617039,35.955429),(45.646857,35.933208),(45.69321,35.879878),(45.718739,35.828305),(45.731038,35.815128),(45.749021,35.810994),(45.786745,35.820244),(45.797907,35.818383),(45.814185,35.80934),(45.834752,35.810528),(45.854079,35.818332),(45.878315,35.83523),(45.888496,35.834868),(45.898779,35.832181),(45.909115,35.831302),(45.941257,35.840346),(46.004923,35.837969),(46.024973,35.843291),(46.0443,35.852283),(46.060423,35.857244),(46.07696,35.856986),(46.107759,35.847425),(46.115045,35.846444),(46.119851,35.842878),(46.124812,35.822466),(46.129101,35.815541),(46.135199,35.810012),(46.143364,35.804534),(46.163518,35.797506),(46.184963,35.798178),(46.227441,35.805723),(46.246407,35.811407),(46.263512,35.821122),(46.281392,35.828305),(46.302992,35.826548),(46.319632,35.816885),(46.327074,35.8035),(46.325058,35.787843),(46.313379,35.771513),(46.297515,35.759886),(46.267129,35.744124),(46.253486,35.727743),(46.23788,35.715547),(46.217003,35.713583),(46.177729,35.715392),(46.127603,35.693688),(46.107035,35.689812),(46.037065,35.691569),(46.016498,35.68573),(46.003786,35.674258),(45.995879,35.658393),(45.992365,35.640875),(45.992314,35.62439),(46.002029,35.585943),(45.999238,35.572145),(45.968698,35.579742),(45.963892,35.579742),(45.959344,35.578708),(45.968026,35.558503),(45.97924,35.539486),(45.983115,35.5353),(45.984562,35.531062),(45.983219,35.52698),(45.97924,35.523001),(45.971333,35.51804),(45.966062,35.511477),(45.963582,35.503415),(45.963995,35.494165),(45.977017,35.46533),(46.040993,35.381562),(46.096287,35.341151),(46.120161,35.318517),(46.127396,35.289062),(46.125285,35.284989),(46.120058,35.274902),(46.107655,35.262758),(46.098767,35.250046),(46.101454,35.234285),(46.115407,35.226326),(46.154164,35.226636),(46.169254,35.216249),(46.165481,35.189946),(46.143829,35.159612),(46.130393,35.131397),(46.151064,35.111605),(46.143106,35.099512),(46.132357,35.094758),(46.119954,35.092743),(46.09453,35.085818),(46.07665,35.089694),(46.066573,35.088815),(46.039753,35.075535),(46.025077,35.064269),(46.009212,35.06091),(45.97924,35.071555),(45.966372,35.070729),(45.956037,35.074243),(45.934953,35.085456),(45.920173,35.089642),(45.913197,35.087317),(45.899399,35.06861),(45.898366,35.063907),(45.901156,35.051608),(45.899089,35.046906),(45.893095,35.044374),(45.879039,35.043237),(45.872476,35.041428),(45.861882,35.032798),(45.857025,35.021274),(45.856818,35.008045),(45.859712,34.994403),(45.859712,34.994248),(45.868962,34.968358),(45.866327,34.949547),(45.857852,34.931306),(45.850203,34.906811),(45.835011,34.890171),(45.808036,34.897871),(45.796214,34.903043),(45.77827,34.910893),(45.755119,34.909963),(45.748504,34.89074),(45.750468,34.865367),(45.745404,34.840975),(45.717705,34.825111),(45.689231,34.82263),(45.676571,34.818289),(45.666752,34.80692),(45.6636,34.794932),(45.665925,34.774726),(45.662825,34.763254),(45.635023,34.736951),(45.627375,34.720828),(45.642258,34.706875),(45.650526,34.702999),(45.659517,34.697211),(45.666856,34.689977),(45.67037,34.681812),(45.67347,34.671166),(45.678379,34.66512),(45.683495,34.660211),(45.687629,34.653235),(45.690782,34.634321),(45.691247,34.615511),(45.694451,34.598561),(45.705613,34.58559),(45.708558,34.579079),(45.7083,34.568382),(45.705613,34.557737),(45.70153,34.551277),(45.695588,34.550244),(45.673573,34.556652),(45.604844,34.561147),(45.573941,34.567297),(45.539628,34.582231),(45.500664,34.591688),(45.49653,34.564248),(45.50485,34.523785),(45.503455,34.494175),(45.4987,34.486242),(45.493016,34.479008),(45.486505,34.472729),(45.47927,34.467251),(45.426767,34.45751),(45.417052,34.444359),(45.447954,34.36209),(45.46077,34.340463),(45.47927,34.329146),(45.499217,34.342401),(45.521541,34.342504),(45.542212,34.331601),(45.55756,34.311964),(45.563038,34.289278),(45.561022,34.264602),(45.54769,34.216491),(45.542729,34.207474),(45.528673,34.18856),(45.526657,34.17931),(45.531463,34.167916),(45.538801,34.159234),(45.544537,34.15063),(45.544537,34.139494),(45.534357,34.128202),(45.493946,34.100685),(45.47927,34.087636),(45.454879,34.06986),(45.443613,34.047664),(45.43519,34.022136),(45.419946,33.99436),(45.419946,33.994257),(45.419739,33.994257),(45.411471,33.987151),(45.401755,33.981622),(45.380361,33.97356),(45.401032,33.949453),(45.423718,33.938782),(45.449711,33.937387),(45.47927,33.941185),(45.481751,33.940875),(45.483921,33.940048),(45.485781,33.938653),(45.574768,33.803933),(45.588204,33.791504),(45.603604,33.78073),(45.616729,33.768689),(45.623912,33.752075),(45.628202,33.731844),(45.635126,33.714171),(45.645151,33.697918),(45.658794,33.681692),(45.673987,33.669031),(45.721116,33.640764),(45.72463,33.633036),(45.727989,33.625649),(45.72773,33.590096),(45.734552,33.583119),(45.752018,33.586762),(45.768451,33.595909),(45.797545,33.619319),(45.815787,33.625804),(45.864363,33.626424),(45.885964,33.63092),(45.882036,33.600198),(45.89971,33.58516),(45.920897,33.572991),(45.927925,33.550873),(45.915626,33.535809),(45.868187,33.511573),(45.852581,33.494365),(45.869737,33.482118),(45.899399,33.476304),(45.929682,33.479508),(45.948182,33.494262),(45.953143,33.496587),(45.958156,33.497388),(45.963272,33.496561),(45.964487,33.495991),(45.968232,33.494236),(45.971385,33.492763),(45.974434,33.490799),(46.008023,33.455659),(46.019392,33.438399),(46.027454,33.419512),(46.030503,33.399203),(46.031071,33.382511),(46.036239,33.367939),(46.069725,33.34086),(46.109516,33.293705),(46.141349,33.272105),(46.155301,33.260219),(46.164189,33.243347),(46.164396,33.233063),(46.15835,33.213865),(46.157678,33.205675),(46.162122,33.196373),(46.173698,33.190223),(46.174318,33.18118),(46.167187,33.168338),(46.153441,33.154256),(46.126466,33.131777),(46.105382,33.118393),(46.089155,33.11568),(46.072412,33.116765),(46.050398,33.115112),(46.030296,33.105681),(46.029314,33.093511),(46.043267,33.08346),(46.087192,33.079222),(46.106002,33.072504),(46.12042,33.061678),(46.126156,33.047984),(46.119489,33.031008),(46.104658,33.018063),(46.088018,33.00672),(46.075719,32.994499),(46.075719,32.994344),(46.09701,32.95432),(46.15556,32.948352),(46.273433,32.959488),(46.379629,32.931764),(46.479312,32.891792),(46.507321,32.868021),(46.600613,32.822541),(46.604421,32.820685),(46.650361,32.789369),(46.715887,32.756012),(46.757332,32.716195),(47.058398,32.494478),(47.058605,32.494478),(47.058657,32.4944),(47.090593,32.474556),(47.12103,32.461043),(47.152553,32.455178),(47.187796,32.458382),(47.20547,32.46404),(47.251255,32.48546),(47.265311,32.484685),(47.321948,32.468226),(47.343342,32.458692),(47.356106,32.446186),(47.36732,32.430761),(47.384115,32.41257),(47.410832,32.395285),(47.416413,32.387973),(47.417963,32.376423),(47.414346,32.36986),(47.408765,32.364331),(47.39998,32.345779),(47.395949,32.34198),(47.395794,32.33702),(47.40246,32.323506),(47.407059,32.31777),(47.436877,32.293689),(47.441579,32.288185),(47.445352,32.28263),(47.463025,32.26183),(47.469433,32.25612),(47.479148,32.252063),(47.484522,32.239351),(47.504004,32.226742),(47.509224,32.213797),(47.507673,32.199844),(47.502351,32.190129),(47.496408,32.181344),(47.492481,32.170492),(47.490413,32.15654),(47.491344,32.148633),(47.498165,32.144137),(47.513823,32.140313),(47.52669,32.133389),(47.534752,32.123157),(47.54364,32.114578),(47.55873,32.112976),(47.578057,32.106155),(47.580232,32.103495),(47.595007,32.085433),(47.617796,32.04187),(47.633144,32.026935),(47.668077,32.012414),(47.677947,31.994689),(47.677947,31.994534),(47.68265,31.976912),(47.71841,31.922342),(47.725231,31.914332),(47.743318,31.904255),(47.750966,31.898209),(47.756547,31.889166),(47.761715,31.871647),(47.76528,31.864103),(47.781352,31.848858),(47.820781,31.823588),(47.834475,31.805708),(47.837266,31.784469),(47.831323,31.761835),(47.678929,31.407851),(47.676443,31.236517),(47.672935,30.994698),(48.001545,30.994647),(48.012242,30.989066),(48.015446,30.97625),(48.012035,30.494574),(48.012035,30.494522),(48.013586,30.463878),(48.119419,30.450804),(48.130581,30.447497),(48.140916,30.441916),(48.15766,30.426309),(48.170475,30.406672),(48.179467,30.384865),(48.187684,30.340836),(48.192076,30.331276),(48.200086,30.323938),(48.211196,30.319649),(48.222979,30.318099),(48.235071,30.318512),(48.271658,30.323835),(48.284267,30.323318),(48.296359,30.319753),(48.305713,30.312828),(48.326021,30.283476),(48.358164,30.251798),(48.397025,30.220999),(48.403588,30.212524),(48.408704,30.202344),(48.410616,30.191543),(48.408032,30.180898),(48.400745,30.172268),(48.391134,30.164775),(48.383279,30.156455),(48.381108,30.145345),(48.383072,30.13842),(48.395578,30.115217),(48.415525,30.095632),(48.421209,30.085348),(48.423948,30.083643),(48.44219,30.03393),(48.444257,30.020908),(48.453145,30.001426),(48.457796,29.994863),(48.464307,29.989127),(48.478157,29.979618),(48.49304,29.971763),(48.523994,29.964115),(48.531025,29.961351),(48.531016,29.96133),(48.530772,29.960761),(48.530772,29.956204),(48.554942,29.956488),(48.559255,29.946601),(48.546886,29.934719),(48.520518,29.9289),(48.411876,29.938422),(48.33546,29.961412),(48.299571,29.984524),(48.264659,29.993964),(48.209239,30.024482),(48.165782,30.037543),(48.119151,30.044989),(48.076345,30.04564),(48.038259,30.036851),(47.969005,30.004055),(47.961925,30.030341),(47.958507,30.060614),(47.951671,30.088324),(47.934825,30.107082),(47.947113,30.07453),(47.948497,30.062323),(47.946056,30.049384),(47.941742,30.037258),(47.940929,30.026516),(47.948497,30.017646),(47.948009,29.994045),(47.731432,30.088552),(47.674175,30.098216),(47.415793,30.098216),(47.358225,30.092118),(47.197098,30.03424),(47.144698,30.003338),(47.110488,29.960911),(47.025429,29.772137),(46.988842,29.712658),(46.983674,29.698292),(46.97985,29.668061),(46.97737,29.657984),(46.957836,29.620441),(46.883112,29.512515),(46.85345,29.44456),(46.838567,29.424975),(46.774488,29.363532),(46.711856,29.271393),(46.561468,29.124167),(46.532436,29.095745),(46.488614,29.08758),(46.444896,29.079415),(46.427322,29.076143),(46.401178,29.071276),(46.357459,29.063137),(46.252143,29.071457),(46.17592,29.077477),(46.099697,29.083575),(46.02363,29.089595),(45.947355,29.095641),(45.844932,29.103754),(45.742406,29.111868),(45.63988,29.120032),(45.537406,29.12812),(45.434932,29.136207),(45.332406,29.144372),(45.229983,29.152459),(45.127457,29.160624),(45.047824,29.166903),(44.968293,29.173259),(44.888712,29.179512),(44.808923,29.185894),(44.717456,29.193103),(44.710893,29.195273),(44.70464,29.197495),(44.698232,29.199718),(44.691825,29.201836),(44.614878,29.256071),(44.519742,29.323328),(44.424296,29.390533),(44.328953,29.45779),(44.233661,29.524995),(44.119146,29.605817),(44.004631,29.686613),(43.890065,29.767383),(43.824013,29.81398),(43.775498,29.848205),(43.660983,29.929027),(43.546365,30.009797),(43.431746,30.090568),(43.317231,30.171338),(43.229071,30.233505),(43.140963,30.295671),(43.052907,30.357786),(42.964747,30.420005),(42.859017,30.494522),(42.858913,30.494574),(42.85881,30.494626),(42.858707,30.494677),(42.783413,30.551045),(42.763932,30.565629),(42.669157,30.636684),(42.574486,30.707636),(42.479712,30.778639),(42.395169,30.841581),(42.31073,30.904523),(42.22629,30.967465),(42.141644,31.030407),(42.075395,31.079861),(41.986202,31.125285),(41.898558,31.169985),(41.810708,31.214685),(41.723065,31.259385),(41.635422,31.304034),(41.547572,31.348734),(41.459825,31.393434),(41.372182,31.438134),(41.284539,31.482834),(41.196689,31.527586),(41.109046,31.572286),(41.021299,31.616934),(40.933553,31.661634),(40.845909,31.706335),(40.758163,31.751086),(40.670313,31.795683),(40.58267,31.840435),(40.479834,31.892835),(40.424126,31.920533),(40.37028,31.938465),(40.029318,31.994379),(40.029112,31.994431),(40.029008,31.994482),(40.028802,31.994482),(39.9486,32.00611),(39.750989,32.034893),(39.553275,32.063729),(39.355561,32.092564),(39.157744,32.121348),(39.154953,32.120573),(39.152163,32.119746),(39.149269,32.118919),(39.146375,32.118144),(39.146168,32.125844),(39.266471,32.212867),(39.291999,32.244519),(39.271122,32.311956),(39.256342,32.342678),(39.235775,32.352858),(39.046329,32.308494),(39.036201,32.313352),(39.028759,32.328338),(38.979977,32.472102),(38.978633,32.47373),(38.97822,32.47497),(38.978633,32.475693),(38.979977,32.476055),(39.057181,32.496596),(38.990002,32.705576),(38.94277,32.852337),(38.897191,32.994344),(38.862568,33.10072),(38.82102,33.229032),(38.774511,33.371685),(38.885099,33.427108),(38.995686,33.482479),(39.106274,33.537928),(39.216862,33.593325),(39.327449,33.648722),(39.438037,33.704094),(39.548624,33.759491),(39.659212,33.81481),(39.769696,33.870233),(39.880387,33.925605),(39.990871,33.980976),(40.101459,34.036373),(40.173111,34.072283),(40.212046,34.091796),(40.322634,34.147167),(40.433221,34.202539),(40.543809,34.257988),(40.690467,34.331497),(40.936033,34.386068),(40.965282,34.401855),(40.98802,34.42852),(41.023986,34.494175),(41.023986,34.49433),(41.195656,34.768473),(41.204234,34.793123),(41.206508,34.819323),(41.198033,34.994041),(41.192326,35.158904),(41.191521,35.182143),(41.20134,35.243018),(41.243095,35.366525),(41.25188,35.46409),(41.261078,35.494165),(41.261181,35.494165),(41.261285,35.494165),(41.261285,35.49432),(41.308458,35.552248),(41.34221,35.593694),(41.358023,35.623925),(41.363501,35.655241),(41.359263,35.792752),(41.354509,35.825566),(41.343657,35.857657),(41.266349,35.994238),(41.266349,35.994342),(41.266246,35.99429),(41.240614,36.043021),(41.236687,36.060332),(41.236583,36.077024),(41.268829,36.327965),(41.276994,36.354785),(41.365258,36.493898),(41.365258,36.494001),(41.365361,36.494053),(41.365361,36.494156),(41.385411,36.516377),(41.414867,36.527384),(41.479773,36.536117),(41.789935,36.589292),(41.817323,36.599731),(41.843781,36.617869),(41.978554,36.733625),(42.178438,36.90532),(42.281584,36.99397),(42.281894,36.99397),(42.281894,36.994022),(42.281894,36.994125),(42.34587,37.042908),(42.376806,37.062001),(42.377186,37.062235),(42.376875,37.076756),(42.371191,37.087944),(42.363646,37.09815),(42.357238,37.109984),(42.401887,37.114144),(42.459144,37.129311),(42.545237,37.140887),(42.561257,37.146623),(42.564668,37.152049),(42.576967,37.17923),(42.702334,37.325346),(42.706158,37.333226),(42.707398,37.340151),(42.709465,37.347179),(42.715666,37.355292),(42.722177,37.358909),(42.77158,37.374903),(42.780468,37.375498),(42.792457,37.374335),(42.801139,37.36909),(42.80548,37.351856),(42.814058,37.346817),(42.89674,37.324906)] +Israel [(35.803633,33.248463),(35.807664,33.201721),(35.830195,33.189991),(35.833399,33.161129),(35.822443,33.14157),(35.811488,33.126765),(35.811488,33.111908),(35.848902,33.098678),(35.845801,33.085423),(35.85903,32.99021),(35.864611,32.97773),(35.888073,32.944941),(35.874017,32.922333),(35.866885,32.920782),(35.849729,32.895823),(35.83805,32.866031),(35.841874,32.853577),(35.834226,32.827946),(35.784203,32.777949),(35.75759,32.744347),(35.757435,32.744282),(35.740175,32.740535),(35.685191,32.711234),(35.652015,32.686171),(35.635995,32.679143),(35.612334,32.681535),(35.612224,32.681546),(35.593827,32.670358),(35.578737,32.653434),(35.569849,32.646768),(35.562718,32.64421),(35.560547,32.640903),(35.560031,32.632686),(35.564061,32.625477),(35.572536,32.62124),(35.572536,32.615013),(35.565612,32.615013),(35.565612,32.607546),(35.571296,32.598554),(35.574397,32.572767),(35.579978,32.560391),(35.575844,32.554965),(35.574397,32.554396),(35.57388,32.556748),(35.572536,32.560391),(35.570159,32.556825),(35.565612,32.546102),(35.55941,32.55295),(35.562821,32.532021),(35.565612,32.525587),(35.551969,32.525587),(35.551969,32.518817),(35.55724,32.519334),(35.561374,32.519179),(35.568505,32.510265),(35.570573,32.506027),(35.579978,32.497733),(35.579978,32.49148),(35.570986,32.489207),(35.564991,32.48391),(35.561271,32.477089),(35.55941,32.470396),(35.563131,32.468174),(35.565405,32.466314),(35.568092,32.46497),(35.572536,32.464195),(35.572536,32.456728),(35.566232,32.453111),(35.55941,32.450527),(35.565612,32.443706),(35.551969,32.436212),(35.554139,32.434455),(35.556207,32.434791),(35.55786,32.43412),(35.55941,32.429443),(35.551969,32.429443),(35.551969,32.423242),(35.55817,32.417997),(35.5591,32.413966),(35.554863,32.411175),(35.545148,32.409573),(35.549592,32.39854),(35.551969,32.395285),(35.556827,32.390918),(35.560961,32.384717),(35.480139,32.402416),(35.456678,32.40761),(35.432803,32.408256),(35.406862,32.414793),(35.401384,32.440011),(35.401797,32.470965),(35.392909,32.494478),(35.363143,32.51011),(35.333378,32.513081),(35.270436,32.510471),(35.252246,32.515949),(35.22372,32.535896),(35.210359,32.541868),(35.208631,32.54264),(35.190751,32.54171),(35.176798,32.532692),(35.151683,32.507914),(35.120678,32.491325),(35.090705,32.479233),(35.06435,32.463136),(35.044713,32.43412),(35.03273,32.3822),(35.028797,32.365157),(35.021562,32.34459),(35.017841,32.342213),(35.004509,32.338053),(35.000685,32.335728),(34.996551,32.323041),(34.993037,32.29157),(34.989936,32.27847),(35.002442,32.275137),(35.009366,32.267644),(35.011123,32.257102),(35.007609,32.24439),(35.000271,32.232013),(34.961411,32.201576),(34.948078,32.186874),(34.946115,32.177262),(34.962341,32.146359),(34.967302,32.129461),(34.979911,32.035203),(34.980428,32.016548),(34.978671,31.994379),(34.978671,31.993707),(34.978981,31.993087),(34.979601,31.99257),(34.980324,31.992157),(34.981151,31.98885),(34.981461,31.985232),(34.981151,31.981615),(34.980324,31.978204),(34.976087,31.948232),(34.995827,31.909165),(35.007713,31.875678),(34.980324,31.862604),(34.953246,31.854439),(34.947326,31.840925),(34.945391,31.836507),(34.954796,31.819764),(34.987766,31.814803),(35.003372,31.814648),(35.010503,31.815889),(35.020425,31.82116),(35.038099,31.837386),(35.04554,31.841417),(35.059699,31.839401),(35.111893,31.818059),(35.181449,31.804106),(35.198399,31.795166),(35.206254,31.782764),(35.208011,31.766124),(35.206564,31.744627),(35.202843,31.738942),(35.198296,31.737702),(35.193128,31.739873),(35.187443,31.744472),(35.125742,31.733103),(35.106725,31.724938),(35.088845,31.712587),(35.054015,31.682202),(34.973606,31.63037),(34.955313,31.61187),(34.953176,31.60832),(34.93733,31.582001),(34.932885,31.554923),(34.932472,31.526966),(34.926788,31.494409),(34.881726,31.429866),(34.867153,31.396431),(34.878832,31.362841),(34.900949,31.348475),(34.927408,31.34491),(34.9549,31.34863),(34.980324,31.356175),(35.040166,31.363203),(35.164913,31.362273),(35.223307,31.381031),(35.332344,31.458804),(35.390118,31.487071),(35.458125,31.491929),(35.458538,31.491619),(35.457128,31.433524),(35.456884,31.423509),(35.452854,31.400823),(35.435077,31.360619),(35.416473,31.331835),(35.423915,31.324601),(35.422261,31.303),(35.408205,31.282019),(35.3957,31.25768),(35.401177,31.230291),(35.410686,31.204608),(35.421331,31.184506),(35.436214,31.159546),(35.443242,31.132209),(35.438488,31.103736),(35.391565,31.023947),(35.385158,30.994647),(35.385261,30.963279),(35.374099,30.945141),(35.34711,30.92271),(35.334928,30.912585),(35.322216,30.88995),(35.319528,30.867316),(35.320045,30.84494),(35.316635,30.822823),(35.310847,30.813314),(35.293897,30.800188),(35.286145,30.792333),(35.279531,30.780241),(35.27612,30.768976),(35.271573,30.743706),(35.263882,30.719967),(35.263821,30.71978),(35.205324,30.617099),(35.162122,30.494677),(35.157368,30.470854),(35.140005,30.430185),(35.140005,30.406155),(35.144965,30.395872),(35.159332,30.375615),(35.162122,30.361404),(35.159952,30.347503),(35.154474,30.336754),(35.147756,30.32647),(35.141762,30.313965),(35.132356,30.261875),(35.125225,30.244667),(35.124812,30.21609),(35.145276,30.154905),(35.145276,30.123382),(35.129049,30.089741),(35.086261,30.034034),(35.074686,29.994604),(35.074065,29.982564),(35.070345,29.973727),(35.065384,29.965976),(35.061456,29.957346),(35.054118,29.923394),(35.053188,29.862623),(35.048951,29.842314),(35.002545,29.733096),(34.995104,29.708162),(34.989833,29.651964),(34.980324,29.627004),(34.966992,29.608116),(34.95986,29.586206),(34.95558,29.558987),(34.955577,29.558987),(34.951345,29.54564),(34.944998,29.536851),(34.927745,29.518012),(34.91977,29.507392),(34.915782,29.50023),(34.914073,29.493801),(34.910818,29.489936),(34.903005,29.489691),(34.893728,29.490546),(34.886729,29.490058),(34.878108,29.504298),(34.855267,29.545717),(34.848239,29.569643),(34.8501,29.63876),(34.824365,29.7417),(34.785194,29.835699),(34.741373,29.940241),(34.735068,29.994553),(34.735585,30.000702),(34.734965,30.006697),(34.733208,30.012588),(34.730417,30.018169),(34.69166,30.114545),(34.632645,30.26203),(34.599469,30.344506),(34.58841,30.35882),(34.53384,30.400213),(34.526915,30.409618),(34.524745,30.421142),(34.526502,30.438712),(34.53601,30.468581),(34.536217,30.482172),(34.526295,30.494522),(34.526295,30.494574),(34.510379,30.513332),(34.504384,30.530334),(34.502214,30.571675),(34.480407,30.651205),(34.418808,30.7913),(34.367855,30.907417),(34.329511,30.994492),(34.297989,31.078776),(34.258611,31.184144),(34.248351,31.211449),(34.264399,31.224193),(34.315869,31.256905),(34.350905,31.289254),(34.353799,31.306359),(34.345531,31.340724),(34.345841,31.357725),(34.367339,31.392814),(34.480407,31.485624),(34.495496,31.494409),(34.495703,31.494409),(34.528259,31.520144),(34.530429,31.541383),(34.511412,31.561589),(34.481204,31.583141),(34.481212,31.583157),(34.489513,31.600409),(34.513927,31.627143),(34.602712,31.757758),(34.60963,31.76553),(34.665945,31.873887),(34.693207,31.926459),(34.711925,31.951606),(34.743337,32.039862),(34.743988,32.044623),(34.742686,32.055609),(34.743337,32.06037),(34.757009,32.066596),(34.837657,32.280707),(34.87322,32.430325),(34.904145,32.560614),(34.909841,32.570258),(34.915863,32.614895),(34.920909,32.628241),(34.919119,32.642971),(34.942068,32.724514),(34.94752,32.814154),(34.955577,32.834377),(34.97047,32.841376),(34.983165,32.838772),(35.003429,32.827541),(35.018728,32.825019),(35.02768,32.826606),(35.062755,32.858344),(35.067068,32.866848),(35.075531,32.893093),(35.0796,32.905829),(35.07781,32.917873),(35.06544,32.923082),(35.071544,32.937934),(35.078461,32.998847),(35.09197,33.031195),(35.096202,33.05036),(35.092133,33.067694),(35.096446,33.071357),(35.101817,33.077623),(35.106456,33.080756),(35.09962,33.087592),(35.104259,33.088528),(35.105235,33.089016),(35.185273,33.083977),(35.189614,33.085527),(35.200673,33.092736),(35.207184,33.094803),(35.213178,33.094441),(35.227338,33.091366),(35.234469,33.090927),(35.271469,33.101185),(35.283665,33.101185),(35.288825,33.099201),(35.29462,33.096973),(35.305783,33.089377),(35.315704,33.079222),(35.322629,33.067363),(35.331931,33.057156),(35.34516,33.05558),(35.401591,33.067931),(35.449443,33.085217),(35.480139,33.087387),(35.4851,33.102606),(35.503703,33.130615),(35.512282,33.147409),(35.517656,33.172886),(35.520033,33.222185),(35.527784,33.244251),(35.53688,33.257868),(35.542771,33.271536),(35.549489,33.281019),(35.561167,33.28213),(35.566955,33.276187),(35.585042,33.252209),(35.597754,33.244354),(35.597858,33.24438),(35.598064,33.244251),(35.603852,33.240091),(35.603892,33.240323),(35.604576,33.244251),(35.604576,33.244303),(35.604576,33.244354),(35.601475,33.262725),(35.610363,33.27027),(35.624716,33.272924),(35.640542,33.275851),(35.66049,33.289261),(35.698523,33.32267),(35.716197,33.326727),(35.729426,33.327812),(35.743689,33.331171),(35.757538,33.336313),(35.769423,33.342643),(35.785753,33.357887),(35.8057,33.391348),(35.8211,33.406722),(35.822443,33.401373),(35.816966,33.395198),(35.815415,33.378868),(35.812315,33.373365),(35.809938,33.360032),(35.793505,33.349929),(35.785753,33.342875),(35.763842,33.334401),(35.802083,33.31249),(35.768597,33.272699),(35.775625,33.264896),(35.803633,33.248463)] +Jamaica [(-76.263743,18.012356),(-76.256785,17.996324),(-76.248769,17.977851),(-76.234039,17.953925),(-76.21524,17.934963),(-76.187978,17.915473),(-76.198394,17.907416),(-76.250071,17.894965),(-76.286488,17.875881),(-76.307362,17.870103),(-76.325185,17.874498),(-76.318227,17.879299),(-76.315419,17.882717),(-76.311513,17.894965),(-76.329579,17.888658),(-76.345611,17.865871),(-76.363026,17.860826),(-76.378529,17.865058),(-76.396555,17.872992),(-76.414906,17.877509),(-76.444936,17.866767),(-76.523549,17.860256),(-76.57901,17.868313),(-76.587066,17.867987),(-76.594594,17.866604),(-76.602773,17.867621),(-76.613149,17.874498),(-76.617665,17.883531),(-76.6197,17.895087),(-76.622711,17.905015),(-76.630523,17.909247),(-76.63858,17.911607),(-76.643951,17.917222),(-76.64859,17.923896),(-76.654775,17.929145),(-76.684804,17.936021),(-76.692291,17.940009),(-76.707672,17.94595),(-76.728424,17.94477),(-76.764638,17.936591),(-76.7801,17.934801),(-76.832265,17.936591),(-76.832265,17.943427),(-76.722401,17.949612),(-76.729319,17.955959),(-76.734364,17.960598),(-76.747467,17.963446),(-76.777659,17.963853),(-76.798164,17.96342),(-76.819056,17.975273),(-76.836829,17.986359),(-76.849967,17.992276),(-76.853069,17.981968),(-76.857761,17.971398),(-76.870173,17.95805),(-76.874005,17.950349),(-76.887156,17.951856),(-76.891037,17.944502),(-76.894944,17.921681),(-76.912828,17.861287),(-76.926096,17.843329),(-76.942128,17.833564),(-76.955312,17.833441),(-76.964019,17.837226),(-76.972076,17.842434),(-76.983062,17.846584),(-77.038319,17.846584),(-77.038319,17.85399),(-77.017201,17.860826),(-77.027984,17.869818),(-77.041493,17.887152),(-77.051991,17.894965),(-77.064198,17.899482),(-77.093007,17.902493),(-77.119109,17.893803),(-77.141652,17.881134),(-77.158079,17.849875),(-77.161977,17.840766),(-77.154408,17.813056),(-77.161692,17.810289),(-77.174387,17.802924),(-77.181711,17.799994),(-77.179107,17.794745),(-77.177358,17.784613),(-77.174916,17.778266),(-77.185618,17.782904),(-77.19636,17.783393),(-77.206776,17.779853),(-77.216461,17.772121),(-77.198883,17.761786),(-77.180328,17.75788),(-77.160878,17.759426),(-77.140736,17.765286),(-77.130605,17.727688),(-77.133656,17.710435),(-77.158111,17.703192),(-77.178944,17.704169),(-77.197581,17.708197),(-77.216054,17.716702),(-77.302887,17.790839),(-77.320872,17.812445),(-77.395619,17.853949),(-77.417714,17.861721),(-77.441029,17.864651),(-77.459462,17.857408),(-77.480621,17.843736),(-77.50182,17.841946),(-77.545481,17.85399),(-77.588368,17.860989),(-77.723866,17.853583),(-77.730702,17.854804),(-77.751536,17.868313),(-77.786733,17.884955),(-77.795562,17.891547),(-77.799916,17.899115),(-77.810292,17.92475),(-77.812367,17.932847),(-77.816151,17.939195),(-77.834055,17.945299),(-77.83967,17.949612),(-77.841135,17.970608),(-77.839345,17.995836),(-77.84557,18.017035),(-77.871002,18.025946),(-77.940338,18.026516),(-77.956288,18.032172),(-77.97997,18.084459),(-77.987864,18.095364),(-77.996449,18.10102),(-78.006418,18.105862),(-78.014394,18.107856),(-78.020579,18.113105),(-78.02538,18.12519),(-78.031484,18.148871),(-78.050282,18.185126),(-78.075185,18.198432),(-78.148793,18.196682),(-78.193756,18.203925),(-78.210194,18.204088),(-78.219594,18.195868),(-78.227284,18.19184),(-78.234039,18.193183),(-78.258656,18.202623),(-78.261138,18.204088),(-78.275136,18.206244),(-78.305898,18.215644),(-78.32315,18.217719),(-78.340728,18.221869),(-78.357737,18.232733),(-78.370269,18.248114),(-78.374664,18.26557),(-78.36913,18.273871),(-78.354319,18.28384),(-78.34732,18.292222),(-78.34439,18.305365),(-78.345815,18.335517),(-78.343902,18.344062),(-78.336293,18.355902),(-78.335439,18.36343),(-78.331939,18.367377),(-78.316396,18.368598),(-78.30663,18.374661),(-78.298695,18.386379),(-78.288401,18.394355),(-78.271637,18.389065),(-78.261301,18.417385),(-78.230621,18.444241),(-78.194976,18.455471),(-78.16926,18.436835),(-78.154408,18.453518),(-78.138539,18.453843),(-78.119985,18.447333),(-78.097239,18.443061),(-78.032541,18.450629),(-78.010976,18.450507),(-78.000152,18.446682),(-77.992421,18.443996),(-77.981353,18.441352),(-77.96996,18.443061),(-77.959828,18.449897),(-77.958852,18.456448),(-77.959137,18.464097),(-77.952952,18.474433),(-77.930287,18.502021),(-77.919993,18.511176),(-77.902333,18.518785),(-77.861887,18.525092),(-77.81607,18.523424),(-77.735544,18.5067),(-77.66454,18.491929),(-77.56725,18.49018),(-77.524281,18.480211),(-77.454986,18.475328),(-77.428863,18.469224),(-77.408274,18.457343),(-77.376536,18.465155),(-77.325307,18.465033),(-77.272613,18.459377),(-77.236318,18.450507),(-77.231842,18.446031),(-77.228139,18.433661),(-77.223297,18.429389),(-77.217437,18.429348),(-77.212026,18.431789),(-77.208852,18.434963),(-77.209625,18.436835),(-77.164947,18.429389),(-77.159413,18.426215),(-77.142568,18.412095),(-77.137359,18.408922),(-77.104726,18.407864),(-77.070343,18.413357),(-77.051991,18.416327),(-77.040028,18.413479),(-77.019154,18.40412),(-77.007314,18.402086),(-76.94815,18.404486),(-76.936615,18.410295),(-76.920338,18.412195),(-76.907118,18.411205),(-76.894919,18.409251),(-76.887827,18.400552),(-76.890916,18.389947),(-76.890948,18.376441),(-76.879783,18.366769),(-76.844211,18.345447),(-76.812754,18.32801),(-76.807622,18.318308),(-76.798329,18.29206),(-76.792388,18.283393),(-76.782826,18.277818),(-76.764638,18.273017),(-76.71288,18.269192),(-76.695709,18.26557),(-76.68102,18.258124),(-76.661977,18.239814),(-76.647328,18.23078),(-76.63036,18.225735),(-76.597158,18.227748),(-76.57977,18.21603),(-76.574208,18.214667),(-76.569203,18.21011),(-76.563059,18.205959),(-76.554799,18.204088),(-76.479726,18.204088),(-76.474477,18.201077),(-76.466624,18.186754),(-76.463002,18.182359),(-76.389394,18.169176),(-76.351145,18.155992),(-76.346262,18.135199),(-76.31786,18.10928),(-76.295806,18.079169),(-76.263743,18.012356)] +Jersey [(-2.082672,49.26024),(-2.06786,49.250678),(-2.02066,49.235256),(-2.021962,49.225491),(-2.017323,49.221381),(-2.011871,49.215277),(-2.00829,49.21247),(-2.016998,49.207221),(-2.020985,49.200751),(-2.022043,49.19245),(-2.021962,49.181383),(-2.024526,49.171332),(-2.031158,49.171454),(-2.040028,49.175727),(-2.049224,49.17829),(-2.079986,49.178697),(-2.096995,49.183661),(-2.120229,49.199164),(-2.128896,49.20307),(-2.137929,49.205634),(-2.144846,49.205634),(-2.152943,49.199937),(-2.160634,49.190741),(-2.168691,49.17829),(-2.177154,49.180162),(-2.189687,49.189114),(-2.200103,49.191962),(-2.202952,49.190416),(-2.223378,49.184719),(-2.233632,49.184516),(-2.226471,49.206),(-2.233225,49.228502),(-2.242014,49.247952),(-2.241119,49.26024),(-2.22936,49.263007),(-2.215199,49.26024),(-2.196278,49.253404),(-2.182729,49.254055),(-2.172353,49.256252),(-2.162709,49.260403),(-2.151682,49.267035),(-2.143788,49.261868),(-2.121978,49.258287),(-2.110666,49.253404),(-2.103871,49.258205),(-2.097564,49.25967),(-2.082672,49.26024)] +Jordan [(39.046329,32.308494),(39.235775,32.352858),(39.256342,32.342678),(39.271122,32.311956),(39.291999,32.244519),(39.266471,32.212867),(39.146168,32.125844),(39.146375,32.118144),(39.136246,32.115353),(39.116816,32.102899),(38.998064,32.006936),(38.96344,31.994482),(38.96344,31.994327),(38.963337,31.994379),(38.849752,31.966319),(38.624856,31.91087),(38.400167,31.855369),(38.175375,31.799921),(37.986522,31.753358),(37.950479,31.744472),(37.761412,31.69612),(37.702742,31.681116),(37.455005,31.617709),(37.207269,31.554354),(36.997912,31.500814),(36.959532,31.490999),(37.089653,31.370076),(37.219774,31.249101),(37.221722,31.247292),(37.349895,31.12823),(37.480017,31.007256),(37.483117,31.004103),(37.486321,31.0009),(37.489422,30.997696),(37.492626,30.994492),(37.602283,30.883232),(37.712044,30.772025),(37.821908,30.660869),(37.931565,30.549661),(37.981071,30.499483),(37.981381,30.498811),(37.980968,30.498398),(37.980038,30.498088),(37.97332,30.494522),(37.900146,30.459072),(37.779327,30.400419),(37.670703,30.347606),(37.647552,30.330863),(37.634529,30.312776),(37.605177,30.250713),(37.56921,30.174955),(37.536137,30.105295),(37.491696,30.011193),(37.470198,29.994553),(37.352376,29.973314),(37.218534,29.949129),(37.075804,29.923291),(36.931936,29.897246),(36.84295,29.881226),(36.756237,29.865517),(36.728745,29.853528),(36.70487,29.831152),(36.649576,29.7494),(36.603584,29.681471),(36.541263,29.589409),(36.477081,29.494609),(36.399979,29.438902),(36.283707,29.35485),(36.177977,29.278369),(36.069457,29.200028),(36.043825,29.190881),(36.016437,29.189951),(35.912464,29.205686),(35.797225,29.223075),(35.740251,29.231731),(35.622042,29.249689),(35.473628,29.272065),(35.334618,29.293148),(35.179072,29.316661),(35.060319,29.334696),(34.949385,29.351686),(34.949392,29.351711),(34.962413,29.359768),(34.969005,29.450832),(34.976085,29.477037),(34.997813,29.517971),(34.996837,29.533881),(34.976085,29.552151),(34.962413,29.552151),(34.961599,29.555406),(34.961599,29.558092),(34.960216,29.559516),(34.95558,29.558987),(34.95986,29.586206),(34.966992,29.608116),(34.980324,29.627004),(34.989833,29.651964),(34.995104,29.708162),(35.002545,29.733096),(35.048951,29.842314),(35.053188,29.862623),(35.054118,29.923394),(35.061456,29.957346),(35.065384,29.965976),(35.070345,29.973727),(35.074065,29.982564),(35.074686,29.994604),(35.086261,30.034034),(35.129049,30.089741),(35.145276,30.123382),(35.145276,30.154905),(35.124812,30.21609),(35.125225,30.244667),(35.132356,30.261875),(35.141762,30.313965),(35.147756,30.32647),(35.154474,30.336754),(35.159952,30.347503),(35.162122,30.361404),(35.159332,30.375615),(35.144965,30.395872),(35.140005,30.406155),(35.140005,30.430185),(35.157368,30.470854),(35.162122,30.494677),(35.205324,30.617099),(35.263821,30.71978),(35.263882,30.719967),(35.271573,30.743706),(35.27612,30.768976),(35.279531,30.780241),(35.286145,30.792333),(35.293897,30.800188),(35.310847,30.813314),(35.316635,30.822823),(35.320045,30.84494),(35.319528,30.867316),(35.322216,30.88995),(35.334928,30.912585),(35.34711,30.92271),(35.374099,30.945141),(35.385261,30.963279),(35.385158,30.994647),(35.391565,31.023947),(35.438488,31.103736),(35.443242,31.132209),(35.436214,31.159546),(35.421331,31.184506),(35.410686,31.204608),(35.401177,31.230291),(35.3957,31.25768),(35.408205,31.282019),(35.422261,31.303),(35.423915,31.324601),(35.416473,31.331835),(35.435077,31.360619),(35.452854,31.400823),(35.456884,31.423509),(35.457128,31.433524),(35.458538,31.491619),(35.458125,31.491929),(35.458745,31.491567),(35.459158,31.491877),(35.459055,31.492808),(35.458745,31.494409),(35.464222,31.568565),(35.480139,31.641119),(35.502479,31.68536),(35.527578,31.735067),(35.55941,31.765349),(35.538326,31.819299),(35.538326,31.826741),(35.549075,31.839195),(35.524684,31.919241),(35.527474,31.927355),(35.533676,31.9303),(35.5406,31.932006),(35.545148,31.936605),(35.545871,31.944563),(35.53998,31.955622),(35.538326,31.963942),(35.537706,31.977584),(35.535536,31.988229),(35.524684,32.011691),(35.522824,32.057838),(35.528301,32.075098),(35.545148,32.086828),(35.534606,32.09923),(35.535226,32.110806),(35.551969,32.135197),(35.546698,32.141605),(35.546698,32.147031),(35.551246,32.151527),(35.55941,32.155093),(35.55941,32.162534),(35.555173,32.174394),(35.559562,32.190371),(35.572536,32.237594),(35.55941,32.237594),(35.561064,32.243149),(35.563751,32.246818),(35.567575,32.249506),(35.572536,32.251908),(35.564578,32.263587),(35.560857,32.28263),(35.561167,32.301699),(35.565612,32.313352),(35.556517,32.328209),(35.557343,32.358207),(35.551969,32.367948),(35.551969,32.374821),(35.55941,32.374821),(35.55941,32.367948),(35.565612,32.367948),(35.563958,32.377043),(35.560961,32.384717),(35.556827,32.390918),(35.551969,32.395285),(35.549592,32.39854),(35.545148,32.409573),(35.554863,32.411175),(35.5591,32.413966),(35.55817,32.417997),(35.551969,32.423242),(35.551969,32.429443),(35.55941,32.429443),(35.55786,32.43412),(35.556207,32.434791),(35.554139,32.434455),(35.551969,32.436212),(35.565612,32.443706),(35.55941,32.450527),(35.566232,32.453111),(35.572536,32.456728),(35.572536,32.464195),(35.568092,32.46497),(35.565405,32.466314),(35.563131,32.468174),(35.55941,32.470396),(35.561271,32.477089),(35.564991,32.48391),(35.570986,32.489207),(35.579978,32.49148),(35.579978,32.497733),(35.570573,32.506027),(35.568505,32.510265),(35.561374,32.519179),(35.55724,32.519334),(35.551969,32.518817),(35.551969,32.525587),(35.565612,32.525587),(35.562821,32.532021),(35.55941,32.55295),(35.565612,32.546102),(35.570159,32.556825),(35.572536,32.560391),(35.57388,32.556748),(35.574397,32.554396),(35.575844,32.554965),(35.579978,32.560391),(35.574397,32.572767),(35.571296,32.598554),(35.565612,32.607546),(35.565612,32.615013),(35.572536,32.615013),(35.572536,32.62124),(35.564061,32.625477),(35.560031,32.632686),(35.560547,32.640903),(35.562718,32.64421),(35.569849,32.646768),(35.578737,32.653434),(35.593827,32.670358),(35.612224,32.681546),(35.612334,32.681535),(35.635995,32.679143),(35.652015,32.686171),(35.685191,32.711234),(35.740175,32.740535),(35.757435,32.744282),(35.75759,32.744347),(35.763842,32.746969),(35.769734,32.748054),(35.774901,32.747279),(35.779139,32.744514),(35.779139,32.744462),(35.779035,32.744359),(35.779035,32.744282),(35.788234,32.734411),(35.895721,32.713276),(35.905229,32.708573),(35.922489,32.693768),(35.92745,32.692373),(35.940369,32.692502),(35.944193,32.690771),(35.945743,32.684104),(35.9444,32.677619),(35.941196,32.673536),(35.937475,32.674002),(35.94657,32.664441),(35.955355,32.657439),(35.965794,32.654365),(35.980263,32.656612),(36.003621,32.655088),(36.008272,32.643719),(36.005275,32.626692),(36.005998,32.607907),(36.015403,32.591164),(36.060465,32.533261),(36.066046,32.521608),(36.066253,32.517319),(36.06987,32.516595),(36.081906,32.516265),(36.096225,32.515872),(36.133226,32.520109),(36.13953,32.519541),(36.149865,32.51613),(36.15586,32.5152),(36.160821,32.517215),(36.17219,32.525923),(36.177357,32.527318),(36.188209,32.52228),(36.220765,32.494581),(36.285258,32.456935),(36.373108,32.386422),(36.387887,32.379317),(36.407627,32.374227),(36.463955,32.369395),(36.480181,32.360791),(36.516684,32.357014),(36.653504,32.342859),(36.689574,32.319656),(36.706937,32.328338),(36.728641,32.327795),(36.792513,32.313533),(36.806569,32.313042),(36.819385,32.316788),(36.980099,32.410038),(37.133165,32.494478),(37.133165,32.494529),(37.133371,32.494529),(37.133371,32.494581),(37.244062,32.554396),(37.415214,32.64713),(37.494606,32.690056),(37.586677,32.739837),(37.758036,32.832519),(37.929395,32.92533),(38.056726,32.994292),(38.056726,32.994344),(38.230875,33.086302),(38.315742,33.13118),(38.529565,33.244251),(38.774511,33.371685),(38.82102,33.229032),(38.862568,33.10072),(38.897191,32.994344),(38.94277,32.852337),(38.990002,32.705576),(39.057181,32.496596),(38.979977,32.476055),(38.978633,32.475693),(38.97822,32.47497),(38.978633,32.47373),(38.979977,32.472102),(39.028759,32.328338),(39.036201,32.313352),(39.046329,32.308494)] +Baykonur Cosmodrome [(63.383914,45.565824),(63.324279,45.567969),(63.26635,45.574222),(63.210488,45.584376),(63.156899,45.598251),(63.105946,45.615615),(63.057835,45.636285),(63.012929,45.660005),(62.971484,45.686644),(62.933864,45.715945),(62.900274,45.7477),(62.871025,45.781703),(62.846479,45.817773),(62.826893,45.855703),(62.812527,45.895184),(62.803639,45.936164),(62.800642,45.978332),(62.803639,46.0205),(62.812527,46.061427),(62.826893,46.100986),(62.846479,46.13889),(62.871025,46.17496),(62.900274,46.208964),(62.933864,46.240719),(62.971484,46.270019),(63.012929,46.296658),(63.057835,46.320378),(63.105946,46.341048),(63.156899,46.358412),(63.210488,46.372261),(63.26635,46.382441),(63.324279,46.388694),(63.383914,46.390839),(63.443548,46.388694),(63.501477,46.382441),(63.557391,46.372261),(63.610928,46.358412),(63.661881,46.341048),(63.70994,46.320378),(63.754899,46.296658),(63.796343,46.270019),(63.833964,46.240719),(63.867553,46.208964),(63.896699,46.17496),(63.921297,46.13889),(63.940934,46.100986),(63.9553,46.061427),(63.964085,46.0205),(63.967134,45.978332),(63.964085,45.936164),(63.9553,45.895184),(63.940934,45.855703),(63.921297,45.817773),(63.896699,45.781703),(63.867553,45.7477),(63.833964,45.715945),(63.796343,45.686644),(63.754899,45.660005),(63.70994,45.636285),(63.661881,45.615615),(63.610928,45.598251),(63.557391,45.584376),(63.501477,45.574222),(63.443548,45.567969),(63.383914,45.565824)] +Siachen Glacier [(76.976417,35.578656),(77.017552,35.584599),(77.039049,35.585013),(77.058169,35.580103),(77.071089,35.571008),(77.092276,35.547805),(77.105298,35.537987),(77.141265,35.525378),(77.177542,35.523156),(77.250819,35.530701),(77.281102,35.528013),(77.305389,35.517833),(77.352002,35.486052),(77.383731,35.471996),(77.412773,35.469412),(77.476438,35.477164),(77.512199,35.478456),(77.519608,35.477439),(77.660613,35.458095),(77.689655,35.462694),(77.716734,35.475407),(77.74743,35.49432),(77.760349,35.498093),(77.773578,35.498971),(77.800346,35.495406),(77.424659,35.302924),(77.048971,35.110442),(76.913161,35.378277),(76.777351,35.646112),(76.782837,35.645732),(76.824592,35.647799),(76.842989,35.641288),(76.856941,35.628576),(76.880506,35.60067),(76.896422,35.589612),(76.915026,35.583152),(76.9358,35.579638),(76.976417,35.578656)] +Kosovo [(20.8647,43.217337),(20.8616,43.217518),(20.851471,43.219818),(20.839999,43.212092),(20.840619,43.206976),(20.839069,43.192455),(20.836071,43.179433),(20.832041,43.178606),(20.838552,43.170467),(20.912494,43.138805),(20.993431,43.104148),(21.005157,43.099128),(21.025827,43.093366),(21.092593,43.090678),(21.10851,43.081558),(21.124012,43.058277),(21.139309,43.005826),(21.147887,42.992622),(21.165354,42.985103),(21.17941,42.990581),(21.193052,42.997893),(21.209795,42.995878),(21.225298,42.973554),(21.226745,42.942522),(21.23243,42.910896),(21.260542,42.886556),(21.2716,42.884283),(21.294958,42.884386),(21.306327,42.882448),(21.316972,42.877616),(21.336403,42.865473),(21.346738,42.860847),(21.378777,42.855292),(21.398931,42.854569),(21.40844,42.846998),(21.408419,42.841743),(21.408336,42.820747),(21.404099,42.803978),(21.39056,42.770414),(21.387149,42.754962),(21.383945,42.749976),(21.379191,42.747004),(21.378674,42.744136),(21.388699,42.739046),(21.405546,42.7353),(21.4178,42.73557),(21.441823,42.736101),(21.542488,42.725817),(21.565226,42.720184),(21.580522,42.710211),(21.612665,42.680393),(21.629408,42.672203),(21.644084,42.672306),(21.687389,42.686827),(21.708886,42.687189),(21.738652,42.68215),(21.74425,42.679425),(21.764387,42.669619),(21.772758,42.647501),(21.767074,42.638716),(21.756532,42.633626),(21.744543,42.629647),(21.734725,42.624247),(21.735551,42.621276),(21.730074,42.601018),(21.728833,42.598305),(21.726973,42.596393),(21.720152,42.593887),(21.718291,42.591019),(21.719738,42.586678),(21.726766,42.577945),(21.727697,42.574147),(21.717671,42.551151),(21.667855,42.490095),(21.627858,42.460381),(21.618969,42.449245),(21.616902,42.433923),(21.621863,42.402167),(21.617419,42.386639),(21.596645,42.372092),(21.537321,42.35863),(21.51603,42.341939),(21.514893,42.317832),(21.553857,42.273984),(21.564066,42.246289),(21.561815,42.247164),(21.519854,42.239025),(21.499287,42.238663),(21.48151,42.247784),(21.471973,42.23913),(21.467557,42.235123),(21.457222,42.237035),(21.44792,42.244141),(21.436345,42.24688),(21.419912,42.240058),(21.421462,42.231351),(21.428697,42.222566),(21.429834,42.215848),(21.419085,42.215021),(21.384255,42.224943),(21.366788,42.223858),(21.360602,42.220114),(21.353766,42.215977),(21.294958,42.148979),(21.293718,42.140168),(21.295785,42.134794),(21.298782,42.129419),(21.300332,42.120893),(21.301263,42.10061),(21.300796,42.098425),(21.299299,42.091411),(21.28886,42.089706),(21.245727,42.096167),(21.237804,42.097354),(21.229298,42.103822),(21.225402,42.106785),(21.216203,42.121151),(21.199873,42.14115),(21.164423,42.16704),(21.12639,42.188925),(21.112954,42.194402),(21.106236,42.195798),(21.098484,42.195953),(21.074403,42.184455),(21.04094,42.15997),(21.029238,42.151408),(21.003916,42.141951),(20.975145,42.134658),(20.90417,42.116668),(20.810543,42.092936),(20.784912,42.082032),(20.765378,42.064333),(20.755353,42.042784),(20.743054,41.993484),(20.741814,41.970773),(20.751439,41.940338),(20.754423,41.930904),(20.751052,41.910218),(20.750495,41.906797),(20.739953,41.888039),(20.723313,41.866619),(20.714398,41.859163),(20.702953,41.849591),(20.681456,41.84401),(20.671844,41.849307),(20.652827,41.86928),(20.643422,41.873647),(20.637014,41.870365),(20.626162,41.855198),(20.61872,41.850522),(20.602391,41.849876),(20.590298,41.854733),(20.567147,41.873182),(20.567767,41.88052),(20.562703,41.892845),(20.562496,41.900105),(20.567251,41.912172),(20.573348,41.917675),(20.580583,41.921732),(20.588748,41.929586),(20.59929,41.94788),(20.59929,41.960567),(20.594329,41.973718),(20.589678,41.993639),(20.558259,42.055109),(20.552161,42.07379),(20.551954,42.105803),(20.549371,42.123476),(20.538622,42.150116),(20.500795,42.211223),(20.481674,42.230705),(20.473665,42.237123),(20.456973,42.250497),(20.333363,42.317883),(20.317964,42.319821),(20.249647,42.318607),(20.237762,42.319924),(20.229597,42.32672),(20.220915,42.343101),(20.219468,42.350207),(20.221225,42.36372),(20.218848,42.371446),(20.21306,42.377518),(20.197454,42.38744),(20.192803,42.393693),(20.194043,42.4001),(20.204379,42.411831),(20.204689,42.420099),(20.199728,42.427799),(20.186292,42.437437),(20.180814,42.443095),(20.152702,42.493402),(20.152702,42.493428),(20.152599,42.493609),(20.152599,42.493712),(20.152392,42.493712),(20.135546,42.509629),(20.085626,42.530015),(20.064956,42.546758),(20.077048,42.55991),(20.078185,42.572906),(20.075394,42.586962),(20.075704,42.603085),(20.079218,42.61125),(20.090587,42.627348),(20.10392,42.653108),(20.101956,42.656674),(20.094205,42.666983),(20.037006,42.707363),(20.03612,42.707989),(20.024751,42.723414),(20.026508,42.743206),(20.034673,42.751423),(20.055638,42.763866),(20.065059,42.769458),(20.076325,42.773437),(20.112085,42.766538),(20.149498,42.749872),(20.183398,42.742508),(20.208409,42.763282),(20.20996,42.772972),(20.208409,42.782093),(20.209236,42.791704),(20.217608,42.802737),(20.226083,42.806768),(20.264427,42.817258),(20.345352,42.827439),(20.428034,42.840642),(20.4763,42.855525),(20.498831,42.877875),(20.494367,42.887467),(20.488909,42.899191),(20.466688,42.909501),(20.450979,42.922032),(20.459454,42.950015),(20.476403,42.966371),(20.493457,42.970195),(20.51113,42.970195),(20.530457,42.975156),(20.538415,42.980659),(20.543893,42.987248),(20.553401,43.002596),(20.562393,43.009495),(20.572832,43.00934),(20.584304,43.006911),(20.596396,43.007092),(20.617273,43.02213),(20.643835,43.052257),(20.664919,43.085407),(20.669157,43.109799),(20.661818,43.115948),(20.65169,43.11631),(20.640941,43.115276),(20.632053,43.117266),(20.626058,43.123725),(20.620581,43.133337),(20.612106,43.154938),(20.600117,43.173826),(20.597533,43.184962),(20.604044,43.197959),(20.612312,43.202274),(20.644869,43.203307),(20.666986,43.209663),(20.745328,43.252865),(20.769512,43.260849),(20.79442,43.263071),(20.809717,43.25961),(20.819432,43.257412),(20.838449,43.245863),(20.848474,43.238137),(20.855088,43.231445),(20.8647,43.217337)] +Laos [(101.867921,22.378842),(101.877275,22.391864),(101.881512,22.412173),(101.891641,22.429691),(101.909366,22.435893),(101.952877,22.436874),(101.97422,22.445143),(101.994994,22.447416),(102.014579,22.446073),(102.075919,22.43243),(102.085531,22.429123),(102.095349,22.423129),(102.099948,22.41486),(102.10062,22.405765),(102.107345,22.397549),(102.118655,22.397549),(102.125425,22.383648),(102.131368,22.373726),(102.141393,22.342152),(102.144183,22.338586),(102.152865,22.332488),(102.156121,22.328096),(102.156121,22.32267),(102.152813,22.310836),(102.154364,22.304686),(102.168885,22.289648),(102.186196,22.278383),(102.200562,22.265464),(102.20697,22.245413),(102.226246,22.228412),(102.286449,22.200041),(102.358072,22.134722),(102.38944,22.117101),(102.398793,22.108884),(102.405408,22.098291),(102.415226,22.073486),(102.422151,22.06222),(102.442615,22.047441),(102.467626,22.034005),(102.482871,22.021396),(102.474086,22.00889),(102.471244,22.005996),(102.469073,22.002844),(102.467316,21.999227),(102.466386,21.995351),(102.478065,21.957576),(102.515892,21.939179),(102.56023,21.925433),(102.59103,21.901352),(102.625136,21.828643),(102.634593,21.787767),(102.629787,21.729114),(102.631906,21.706273),(102.636815,21.683639),(102.643326,21.668033),(102.650664,21.657801),(102.653455,21.656302),(102.657434,21.658421),(102.710299,21.659299),(102.721151,21.661676),(102.752157,21.6809),(102.774688,21.707875),(102.786676,21.739914),(102.789157,21.820323),(102.806727,21.836084),(102.826467,21.821305),(102.834942,21.775623),(102.831893,21.73578),(102.835149,21.716453),(102.84719,21.704464),(102.864501,21.705653),(102.937003,21.735057),(102.942636,21.738984),(102.947804,21.737124),(102.95602,21.723946),(102.959896,21.713043),(102.961756,21.701519),(102.960826,21.632841),(102.964133,21.608605),(102.974107,21.586694),(102.974934,21.583903),(102.975295,21.581009),(102.974934,21.578167),(102.974107,21.575325),(102.954108,21.55729),(102.916591,21.513572),(102.89499,21.495278),(102.84967,21.425412),(102.86476,21.432181),(102.884758,21.444067),(102.904189,21.452697),(102.918038,21.450113),(102.916798,21.438641),(102.883828,21.38836),(102.879642,21.367896),(102.878816,21.322162),(102.875198,21.305316),(102.863003,21.293792),(102.829155,21.285885),(102.812773,21.273896),(102.801301,21.254828),(102.809156,21.25059),(102.828173,21.252451),(102.850187,21.251676),(102.865121,21.246198),(102.878144,21.23762),(102.887859,21.226923),(102.892975,21.215089),(102.891786,21.205994),(102.883001,21.186098),(102.883001,21.176435),(102.889099,21.166409),(102.89592,21.164549),(102.904189,21.164187),(102.914421,21.1584),(102.925634,21.141656),(102.925885,21.14064),(102.935298,21.102382),(102.942171,21.084864),(102.946822,21.075872),(102.951576,21.068999),(102.959482,21.065795),(102.974107,21.067656),(102.99917,21.057372),(103.014518,21.040526),(103.0371,20.995464),(103.06821,20.931282),(103.08671,20.901981),(103.115287,20.868391),(103.130015,20.854904),(103.136371,20.85046),(103.146965,20.846016),(103.165051,20.84369),(103.175232,20.84121),(103.183655,20.837076),(103.197091,20.827825),(103.206909,20.82457),(103.218588,20.824363),(103.225829,20.826413),(103.238122,20.829892),(103.247113,20.830719),(103.263392,20.826378),(103.319822,20.797853),(103.347676,20.789068),(103.358321,20.787725),(103.363799,20.787621),(103.389115,20.794288),(103.390877,20.794753),(103.411238,20.805605),(103.418266,20.811806),(103.421315,20.81811),(103.426586,20.820694),(103.440435,20.815733),(103.452579,20.804313),(103.470872,20.770878),(103.481621,20.757907),(103.489373,20.752481),(103.497331,20.748347),(103.505961,20.74597),(103.516089,20.745505),(103.516089,20.745557),(103.516141,20.745505),(103.516141,20.74535),(103.577946,20.732896),(103.597376,20.722354),(103.63603,20.682821),(103.657941,20.664735),(103.661145,20.660756),(103.665383,20.658275),(103.676338,20.656518),(103.685433,20.657655),(103.697629,20.661531),(103.708481,20.66675),(103.714268,20.671711),(103.713752,20.679824),(103.703623,20.699254),(103.702383,20.708711),(103.70724,20.721837),(103.713442,20.725764),(103.72202,20.72685),(103.734371,20.730932),(103.754576,20.742921),(103.758607,20.749742),(103.755196,20.759406),(103.753026,20.77987),(103.756746,20.795993),(103.76517,20.81625),(103.775557,20.834905),(103.785324,20.845964),(103.796434,20.84953),(103.816691,20.846894),(103.82682,20.847669),(103.83576,20.852268),(103.850488,20.864774),(103.858601,20.86958),(103.932136,20.891801),(103.952187,20.900276),(103.962315,20.902705),(103.973994,20.902343),(104.010581,20.908079),(104.056676,20.958825),(104.092282,20.961202),(104.099155,20.957688),(104.114244,20.947043),(104.122616,20.943167),(104.132072,20.941669),(104.154758,20.941255),(104.165404,20.939757),(104.193877,20.929886),(104.201267,20.925442),(104.207107,20.917639),(104.217545,20.89702),(104.221059,20.892008),(104.228036,20.892318),(104.250566,20.89826),(104.260023,20.898932),(104.269067,20.896193),(104.278937,20.891542),(104.287877,20.885755),(104.295267,20.879812),(104.307204,20.864619),(104.323224,20.833872),(104.336349,20.81904),(104.344669,20.813666),(104.360327,20.808085),(104.367975,20.803124),(104.37335,20.795734),(104.38167,20.778371),(104.389059,20.770981),(104.409316,20.765194),(104.432157,20.765762),(104.452621,20.763592),(104.466161,20.749639),(104.475669,20.71822),(104.492309,20.702097),(104.543882,20.678739),(104.600726,20.660549),(104.615195,20.646493),(104.614071,20.642616),(104.607237,20.619053),(104.542642,20.535699),(104.520731,20.519265),(104.495823,20.511927),(104.469675,20.516165),(104.444973,20.534613),(104.423786,20.495443),(104.423682,20.495443),(104.366425,20.458494),(104.35981,20.439735),(104.39154,20.420667),(104.432364,20.411055),(104.575508,20.414052),(104.589357,20.406818),(104.612301,20.376742),(104.628425,20.365812),(104.666407,20.351782),(104.680928,20.340982),(104.684648,20.333049),(104.687697,20.321706),(104.689299,20.310002),(104.688628,20.301036),(104.681858,20.287471),(104.674106,20.2853),(104.665838,20.285714),(104.657363,20.279616),(104.653539,20.266464),(104.651989,20.246104),(104.653023,20.225795),(104.657053,20.212695),(104.69085,20.196546),(104.733586,20.204763),(104.778803,20.219025),(104.819162,20.220989),(104.907994,20.175333),(104.908976,20.167814),(104.903446,20.157091),(104.904893,20.134638),(104.918381,20.115724),(104.95688,20.091798),(104.964425,20.076088),(104.961169,20.072109),(104.947475,20.069525),(104.943961,20.067097),(104.943651,20.060947),(104.946441,20.052188),(104.948302,20.033662),(104.951454,20.021518),(104.953211,20.009271),(104.94949,19.995499),(104.945688,19.992373),(104.935693,19.984156),(104.922463,19.982244),(104.892905,19.9855),(104.873836,19.982657),(104.86903,19.977748),(104.86717,19.969144),(104.85792,19.955217),(104.835337,19.938887),(104.780663,19.908088),(104.761801,19.885506),(104.755549,19.867006),(104.760716,19.86145),(104.790275,19.859667),(104.80924,19.853673),(104.81663,19.848092),(104.816888,19.838997),(104.81508,19.822409),(104.805003,19.790886),(104.785211,19.776933),(104.760716,19.766521),(104.736738,19.745669),(104.717773,19.740192),(104.704182,19.738693),(104.691676,19.735463),(104.675967,19.724689),(104.663048,19.711227),(104.65478,19.697972),(104.62305,19.61808),(104.614265,19.607435),(104.604963,19.606479),(104.587807,19.615961),(104.579229,19.618028),(104.570237,19.61529),(104.554837,19.605445),(104.546156,19.601931),(104.522798,19.599373),(104.505848,19.60281),(104.449107,19.635159),(104.423786,19.645779),(104.415414,19.650817),(104.401255,19.664434),(104.391746,19.676836),(104.380068,19.685285),(104.360017,19.687456),(104.341879,19.683012),(104.295267,19.654874),(104.277025,19.68389),(104.242143,19.695181),(104.228966,19.706008),(104.200203,19.698586),(104.158169,19.68774),(104.13631,19.687456),(104.131246,19.682443),(104.124476,19.675906),(104.11719,19.660997),(104.109231,19.654874),(104.088716,19.651902),(104.069699,19.658026),(104.021743,19.681435),(104.010271,19.684562),(104.000969,19.680738),(103.995182,19.664899),(103.997662,19.652522),(104.017351,19.600407),(104.028926,19.586041),(104.06019,19.568161),(104.071559,19.559143),(104.077967,19.542891),(104.079259,19.525709),(104.082515,19.509301),(104.093987,19.495633),(104.093987,19.495581),(104.094142,19.495529),(104.094142,19.495478),(104.089956,19.48367),(104.081274,19.479303),(104.070526,19.477029),(104.060501,19.471293),(104.054041,19.46057),(104.049907,19.437471),(104.046548,19.427497),(104.038796,19.41339),(104.03828,19.412925),(104.035127,19.416671),(104.0204,19.414992),(103.99668,19.40693),(103.973994,19.395665),(103.952807,19.390962),(103.941025,19.38135),(103.921284,19.352773),(103.905885,19.341792),(103.870021,19.326883),(103.85674,19.317194),(103.84842,19.299908),(103.858549,19.295335),(103.876739,19.2958),(103.892655,19.293888),(103.905885,19.270039),(103.910949,19.263011),(103.917977,19.258154),(103.943143,19.245596),(103.972754,19.225288),(103.973994,19.225288),(103.995182,19.23384),(104.016162,19.234874),(104.036729,19.230249),(104.14773,19.182887),(104.187831,19.156584),(104.198115,19.146946),(104.201732,19.127051),(104.201836,19.109171),(104.210621,19.100747),(104.239559,19.108886),(104.259603,19.10413),(104.25992,19.104055),(104.274338,19.093461),(104.287257,19.08049),(104.302863,19.068656),(104.321983,19.060466),(104.33914,19.055195),(104.35516,19.047882),(104.372006,19.033568),(104.400738,18.992279),(104.413244,18.983106),(104.431072,18.979334),(104.46213,18.982408),(104.48094,18.977706),(104.497683,18.967991),(104.509879,18.956363),(104.544295,18.906806),(104.556078,18.89647),(104.589667,18.879107),(104.603982,18.868694),(104.631215,18.842365),(104.646201,18.832547),(104.65633,18.829704),(104.675657,18.828206),(104.685579,18.824769),(104.69085,18.819318),(104.702529,18.802471),(104.71028,18.796089),(104.721959,18.792007),(104.762887,18.786451),(104.801644,18.775134),(104.81539,18.773661),(104.872906,18.777976),(104.899364,18.770612),(104.909234,18.745679),(104.919725,18.735576),(104.937243,18.731158),(104.95688,18.731674),(104.97383,18.736067),(104.99016,18.733018),(105.040337,18.705009),(105.064574,18.696638),(105.076976,18.697516),(105.088655,18.700565),(105.100282,18.702064),(105.112684,18.698343),(105.141106,18.653074),(105.162862,18.638993),(105.169477,18.630569),(105.169477,18.618327),(105.169477,18.61486),(105.162087,18.598194),(105.152062,18.596799),(105.140331,18.600468),(105.128136,18.598918),(105.113615,18.580831),(105.100489,18.551169),(105.085244,18.495771),(105.079973,18.454353),(105.090774,18.426525),(105.113976,18.405131),(105.145447,18.383168),(105.162862,18.366012),(105.162242,18.353894),(105.155989,18.340639),(105.156661,18.319839),(105.166169,18.306791),(105.214745,18.277852),(105.220481,18.270798),(105.229266,18.255321),(105.235933,18.249456),(105.246475,18.246381),(105.25692,18.247464),(105.283113,18.250179),(105.288023,18.253409),(105.292518,18.252737),(105.299288,18.24204),(105.300838,18.23093),(105.294379,18.206952),(105.29505,18.195609),(105.305592,18.181062),(105.321871,18.170081),(105.340112,18.162639),(105.355925,18.158738),(105.363315,18.155224),(105.365537,18.151038),(105.368328,18.149953),(105.377474,18.155689),(105.381557,18.162768),(105.386104,18.184938),(105.391995,18.194214),(105.410031,18.201888),(105.431373,18.200544),(105.451423,18.192715),(105.46672,18.180959),(105.472817,18.171993),(105.480879,18.144811),(105.481189,18.138713),(105.479122,18.133494),(105.476848,18.129644),(105.476021,18.127318),(105.480621,18.123649),(105.491989,18.121066),(105.496072,18.118146),(105.512505,18.082825),(105.52067,18.0759),(105.538136,18.066728),(105.546508,18.058951),(105.551882,18.047943),(105.555551,18.025154),(105.559324,18.015336),(105.566145,18.008256),(105.589813,17.99575),(105.589813,17.995595),(105.603431,17.969184),(105.606039,17.964124),(105.594154,17.888108),(105.605212,17.85426),(105.64397,17.820102),(105.669808,17.773076),(105.692029,17.744086),(105.701227,17.727601),(105.713836,17.689464),(105.723086,17.673496),(105.735282,17.664039),(105.787424,17.642077),(105.819876,17.621561),(105.846025,17.597997),(105.911137,17.516813),(105.932169,17.495729),(105.932324,17.495729),(106.019296,17.394495),(106.185435,17.257397),(106.207295,17.246287),(106.221816,17.244892),(106.233391,17.251971),(106.247912,17.266234),(106.252356,17.274554),(106.254785,17.283184),(106.259953,17.28923),(106.272614,17.28985),(106.284551,17.284786),(106.28667,17.276569),(106.285688,17.265976),(106.287961,17.25378),(106.293129,17.246907),(106.306978,17.232954),(106.310544,17.227115),(106.309769,17.217917),(106.301656,17.202),(106.300829,17.191562),(106.306358,17.176524),(106.336537,17.128568),(106.384648,17.085367),(106.394932,17.068003),(106.394363,17.062215),(106.387904,17.051673),(106.387129,17.047178),(106.40402,17.014642),(106.408833,17.005371),(106.411365,16.995811),(106.411365,16.995708),(106.420253,16.98868),(106.437203,16.980463),(106.45653,16.97359),(106.473738,16.970697),(106.496838,16.963927),(106.524536,16.989248),(106.533425,16.97266),(106.53355,16.950638),(106.535078,16.683531),(106.540659,16.659139),(106.549754,16.637797),(106.57244,16.613561),(106.591405,16.60245),(106.611559,16.594595),(106.624943,16.59351),(106.640188,16.586637),(106.644322,16.57532),(106.639671,16.521783),(106.64117,16.510363),(106.655174,16.47083),(106.670677,16.444424),(106.692898,16.427887),(106.722664,16.432486),(106.72256,16.421634),(106.725713,16.417139),(106.732482,16.417552),(106.742404,16.421531),(106.751511,16.428268),(106.752533,16.429024),(106.754393,16.436569),(106.7546,16.444269),(106.759147,16.452485),(106.786122,16.469125),(106.797336,16.47946),(106.801418,16.4961),(106.822864,16.530982),(106.846687,16.530052),(106.86405,16.504058),(106.866841,16.463854),(106.862603,16.430574),(106.861363,16.427887),(106.858211,16.424632),(106.855885,16.420343),(106.855885,16.414658),(106.859865,16.411816),(106.866531,16.414038),(106.872835,16.417345),(106.876298,16.418017),(106.888442,16.390887),(106.897743,16.378536),(106.915003,16.364739),(106.939498,16.351716),(106.946009,16.346135),(106.95066,16.336317),(106.954071,16.313837),(106.956551,16.307068),(106.9673,16.299885),(106.980219,16.298334),(107.014325,16.29942),(107.053393,16.295234),(107.06798,16.290809),(107.070601,16.290015),(107.089101,16.281075),(107.108273,16.267484),(107.116851,16.254616),(107.127497,16.200976),(107.133336,16.188212),(107.13599,16.184784),(107.142018,16.176998),(107.154369,16.166043),(107.169665,16.156896),(107.183204,16.153227),(107.197157,16.151005),(107.212815,16.146664),(107.238808,16.129818),(107.275912,16.084032),(107.298959,16.064189),(107.326864,16.056075),(107.355907,16.059279),(107.41151,16.073077),(107.431044,16.073697),(107.435643,16.068839),(107.433421,16.057367),(107.433292,16.055461),(107.432129,16.038299),(107.436729,16.026878),(107.44355,16.01892),(107.444738,16.00998),(107.432698,15.995924),(107.418022,15.983573),(107.390633,15.935824),(107.375544,15.919081),(107.361074,15.912312),(107.322937,15.905852),(107.305057,15.899134),(107.235656,15.855519),(107.227232,15.853297),(107.218086,15.854796),(107.208835,15.859136),(107.198604,15.862082),(107.186253,15.859395),(107.175246,15.848543),(107.150183,15.794283),(107.148787,15.773457),(107.16026,15.758729),(107.215295,15.727517),(107.225682,15.71682),(107.230023,15.702402),(107.23457,15.660931),(107.239118,15.645119),(107.24873,15.631088),(107.265421,15.61866),(107.303558,15.599747),(107.316426,15.58874),(107.369342,15.496058),(107.381641,15.488539),(107.388669,15.492518),(107.394974,15.501484),(107.405051,15.508589),(107.412234,15.508796),(107.432129,15.503887),(107.437969,15.505592),(107.442981,15.509209),(107.447891,15.508124),(107.453368,15.495954),(107.453368,15.495903),(107.456469,15.488332),(107.46081,15.484379),(107.466546,15.484353),(107.473781,15.488255),(107.487475,15.481408),(107.490162,15.461512),(107.489645,15.437922),(107.493934,15.420042),(107.512641,15.409422),(107.555584,15.415753),(107.573154,15.411231),(107.57636,15.403054),(107.577702,15.39963),(107.579872,15.361234),(107.58659,15.343354),(107.598476,15.330616),(107.643073,15.297853),(107.657594,15.282427),(107.664105,15.268552),(107.664363,15.253153),(107.656818,15.205688),(107.653976,15.199564),(107.647465,15.197652),(107.642607,15.202458),(107.636716,15.205068),(107.62726,15.196515),(107.623591,15.187239),(107.621317,15.154528),(107.615787,15.135511),(107.6054,15.120603),(107.599199,15.114789),(107.584988,15.104299),(107.580492,15.096754),(107.580802,15.086936),(107.590001,15.0689),(107.589071,15.058178),(107.578012,15.047248),(107.562612,15.045801),(107.535224,15.0504),(107.518997,15.04885),(107.502386,15.044037),(107.473781,15.03575),(107.455074,15.034122),(107.445565,15.025389),(107.444687,15.011824),(107.451405,14.995933),(107.449906,14.981102),(107.453937,14.967873),(107.461947,14.956814),(107.473781,14.948236),(107.48949,14.940304),(107.549642,14.895268),(107.558168,14.886715),(107.555326,14.876199),(107.540598,14.856846),(107.498844,14.81765),(107.490885,14.80424),(107.494296,14.788763),(107.504321,14.773182),(107.509282,14.758635),(107.498068,14.745949),(107.504425,14.736285),(107.514657,14.728224),(107.522253,14.718689),(107.520393,14.704582),(107.506905,14.693445),(107.473781,14.646265),(107.458226,14.638798),(107.451766,14.632183),(107.443808,14.609549),(107.440449,14.603916),(107.433266,14.59433),(107.430321,14.586243),(107.429752,14.578336),(107.430941,14.569887),(107.430941,14.561386),(107.42784,14.553712),(107.414921,14.543429),(107.403449,14.543894),(107.380143,14.554048),(107.361384,14.557924),(107.351566,14.561619),(107.332704,14.578233),(107.305677,14.597224),(107.28728,14.598516),(107.272191,14.585364),(107.254517,14.560973),(107.249453,14.556632),(107.237878,14.549811),(107.233899,14.544927),(107.2324,14.538313),(107.230281,14.518262),(107.219739,14.497979),(107.211213,14.493458),(107.195968,14.495964),(107.195813,14.495964),(107.195606,14.496067),(107.195503,14.496119),(107.180568,14.496171),(107.172455,14.484259),(107.166823,14.467154),(107.158709,14.451522),(107.131424,14.427544),(107.10078,14.407623),(107.078559,14.400776),(107.067862,14.406124),(107.059336,14.418036),(107.042954,14.430722),(107.030345,14.436097),(107.027245,14.43527),(107.026573,14.430231),(106.997376,14.395143),(106.977945,14.365429),(106.973656,14.363026),(106.961099,14.360391),(106.950143,14.334863),(106.94973,14.328661),(106.946113,14.32352),(106.933607,14.320161),(106.923478,14.321918),(106.904771,14.333312),(106.895676,14.33631),(106.885083,14.335534),(106.879295,14.332692),(106.874592,14.328351),(106.835835,14.307061),(106.820849,14.307577),(106.806896,14.312926),(106.792995,14.322589),(106.766692,14.348324),(106.731655,14.396719),(106.723904,14.411602),(106.719253,14.417881),(106.711502,14.425839),(106.707161,14.429301),(106.6344,14.450902),(106.622049,14.457594),(106.605358,14.466637),(106.585669,14.495964),(106.575386,14.50201),(106.552441,14.506945),(106.54376,14.512113),(106.540246,14.521673),(106.534975,14.553247),(106.530996,14.56596),(106.518438,14.585002),(106.512496,14.590351),(106.49105,14.570998),(106.483763,14.567484),(106.477149,14.56689),(106.47224,14.564073),(106.470586,14.554255),(106.470948,14.544643),(106.47317,14.537486),(106.477511,14.532602),(106.484849,14.530019),(106.484849,14.523817),(106.469449,14.524696),(106.456685,14.5279),(106.445781,14.529088),(106.43648,14.523817),(106.434309,14.519606),(106.432655,14.512423),(106.433586,14.505705),(106.445161,14.499478),(106.44082,14.492631),(106.433482,14.486636),(106.429503,14.485964),(106.41922,14.466457),(106.393795,14.456173),(106.339999,14.448988),(106.333437,14.448111),(106.320156,14.444494),(106.306462,14.445476),(106.296746,14.447879),(106.235975,14.484362),(106.230084,14.49046),(106.220782,14.473485),(106.210654,14.421395),(106.203936,14.399897),(106.189156,14.378891),(106.17324,14.369667),(106.15412,14.367548),(106.119031,14.369512),(106.095002,14.373646),(106.079447,14.372716),(106.068647,14.373439),(106.063531,14.372922),(106.060585,14.36884),(106.058105,14.353104),(106.055418,14.348428),(106.038158,14.347652),(105.994129,14.362406),(105.973975,14.365843),(105.969428,14.354861),(105.967826,14.343363),(105.96917,14.33202),(105.973975,14.321298),(105.993406,14.30595),(106.004516,14.289982),(106.007979,14.270758),(106.004051,14.245927),(106.012474,14.225774),(106.026272,14.213035),(106.060585,14.191848),(106.076191,14.174511),(106.10172,14.11516),(106.115259,14.100303),(106.131279,14.086997),(106.144559,14.072398),(106.149469,14.053562),(106.142079,14.03501),(106.126421,14.022013),(106.108489,14.01036),(106.094071,13.996175),(106.081979,13.980957),(106.08384,13.934008),(106.069163,13.916542),(106.049216,13.915457),(105.997437,13.928298),(105.973975,13.92928),(105.949171,13.919591),(105.924573,13.91711),(105.899665,13.922614),(105.895581,13.924799),(105.873051,13.936851),(105.787113,14.010154),(105.770164,14.0347),(105.758898,14.062476),(105.744015,14.086945),(105.716007,14.101543),(105.705155,14.102009),(105.683657,14.098856),(105.669343,14.100407),(105.653272,14.107047),(105.643143,14.113455),(105.632911,14.114023),(105.617253,14.10299),(105.609863,14.120741),(105.594567,14.136348),(105.576532,14.144022),(105.561391,14.138156),(105.555551,14.146089),(105.548937,14.153117),(105.541237,14.159189),(105.533175,14.16433),(105.50527,14.145055),(105.490491,14.136916),(105.473851,14.132523),(105.45008,14.11715),(105.422175,14.107874),(105.392461,14.10374),(105.36409,14.10405),(105.347864,14.106065),(105.337787,14.109993),(105.329467,14.117563),(105.303939,14.152393),(105.296601,14.156837),(105.282183,14.161256),(105.27531,14.164821),(105.256913,14.182443),(105.197434,14.270293),(105.187977,14.291015),(105.184256,14.314166),(105.184308,14.34574),(105.198209,14.345224),(105.250454,14.357187),(105.268592,14.36437),(105.275724,14.372095),(105.287712,14.393283),(105.29474,14.399071),(105.304662,14.398192),(105.322232,14.386384),(105.3351,14.385712),(105.340836,14.390105),(105.359646,14.408269),(105.367656,14.414548),(105.379231,14.418242),(105.40569,14.423048),(105.415973,14.428164),(105.420314,14.435218),(105.428892,14.458163),(105.433853,14.467929),(105.440365,14.475345),(105.463671,14.495964),(105.495452,14.537124),(105.507544,14.564048),(105.510128,14.593581),(105.495762,14.659881),(105.497364,14.690422),(105.505115,14.736363),(105.50155,14.745949),(105.490697,14.763803),(105.489147,14.786334),(105.49385,14.808994),(105.50155,14.827339),(105.5062,14.830414),(105.52005,14.833463),(105.524391,14.837235),(105.52408,14.842532),(105.516949,14.856536),(105.515864,14.871677),(105.517776,14.877672),(105.52191,14.878705),(105.53762,14.878034),(105.543046,14.880101),(105.550642,14.902296),(105.534157,14.934025),(105.543924,14.953094),(105.58356,14.977588),(105.593068,14.990792),(105.566455,14.995933),(105.566352,14.995933),(105.566145,14.995933),(105.566145,14.995985),(105.553536,15.005338),(105.524287,15.045956),(105.52284,15.050039),(105.52315,15.061175),(105.520877,15.066136),(105.516536,15.067143),(105.503565,15.064896),(105.498294,15.066136),(105.473851,15.09063),(105.45194,15.094765),(105.44362,15.111353),(105.443775,15.133806),(105.447703,15.155303),(105.454421,15.175535),(105.463102,15.189436),(105.503927,15.231061),(105.531574,15.252894),(105.560254,15.270929),(105.563975,15.272531),(105.564801,15.299455),(105.561494,15.320901),(105.558032,15.325138),(105.53855,15.321521),(105.519378,15.321056),(105.499017,15.324673),(105.483308,15.334517),(105.477468,15.352811),(105.486253,15.374902),(105.5062,15.387589),(105.552813,15.402033),(105.588366,15.423788),(105.592243,15.427546),(105.604282,15.439214),(105.61038,15.457171),(105.612034,15.499468),(105.616271,15.521276),(105.640869,15.583313),(105.650998,15.634602),(105.640921,15.68199),(105.612912,15.72147),(105.568522,15.749376),(105.525424,15.763483),(105.50341,15.766171),(105.462379,15.762812),(105.447961,15.764879),(105.435662,15.771648),(105.423001,15.783896),(105.412046,15.79976),(105.372565,15.882132),(105.360628,15.919391),(105.362488,15.95479),(105.386518,15.980783),(105.407395,15.988896),(105.418144,15.994735),(105.423932,16.002125),(105.423232,16.00504),(105.422071,16.009877),(105.414475,16.015199),(105.404915,16.018817),(105.288539,16.047394),(105.247405,16.049151),(105.237225,16.050804),(105.106979,16.095853),(105.079301,16.105426),(105.058114,16.121188),(105.042456,16.141445),(105.032586,16.165268),(105.028865,16.191468),(105.029744,16.236013),(105.026385,16.257665),(105.015481,16.276734),(105.007419,16.283555),(104.988028,16.293963),(104.979307,16.298645),(104.971504,16.304329),(104.952074,16.324431),(104.944064,16.330839),(104.917296,16.345567),(104.908046,16.35337),(104.901586,16.362465),(104.897039,16.372749),(104.887737,16.403909),(104.879469,16.421841),(104.868927,16.438429),(104.855491,16.454087),(104.838851,16.46773),(104.78273,16.497082),(104.76516,16.511345),(104.754515,16.528915),(104.749502,16.549327),(104.748417,16.571909),(104.753016,16.614801),(104.763713,16.656762),(104.777563,16.694021),(104.77901,16.704873),(104.778338,16.715828),(104.756789,16.798459),(104.756324,16.809879),(104.75791,16.818895),(104.758442,16.82192),(104.765212,16.844658),(104.767072,16.85613),(104.766401,16.86817),(104.762887,16.879074),(104.753016,16.899848),(104.749657,16.910804),(104.74634,16.943532),(104.740356,17.002581),(104.745317,17.024802),(104.809912,17.171666),(104.81446,17.194042),(104.818335,17.240241),(104.816423,17.300082),(104.819989,17.348916),(104.819317,17.36106),(104.816423,17.372791),(104.792704,17.423485),(104.788932,17.428963),(104.781645,17.435268),(104.764644,17.444414),(104.756789,17.449944),(104.749657,17.45816),(104.734258,17.486892),(104.714517,17.515263),(104.702632,17.527303),(104.65509,17.556087),(104.636383,17.562443),(104.481198,17.640423),(104.452001,17.665951),(104.422287,17.699799),(104.417843,17.707602),(104.411642,17.727395),(104.408903,17.733699),(104.404872,17.739952),(104.3903,17.75716),(104.367769,17.800568),(104.352627,17.819327),(104.287567,17.856741),(104.281982,17.861488),(104.278265,17.864647),(104.270307,17.874001),(104.264312,17.884646),(104.255114,17.91498),(104.210879,18.008618),(104.203329,18.018644),(104.198115,18.025568),(104.13569,18.079673),(104.122357,18.095279),(104.110937,18.114554),(104.068407,18.215401),(104.018643,18.299117),(104.000039,18.318444),(103.975441,18.330665),(103.951257,18.332991),(103.929294,18.32759),(103.910019,18.315317),(103.893689,18.297256),(103.889503,18.291159),(103.885938,18.286999),(103.881287,18.284415),(103.873949,18.283226),(103.857774,18.286482),(103.848989,18.296352),(103.842219,18.309116),(103.832091,18.320666),(103.81726,18.330355),(103.801395,18.338262),(103.7846,18.343894),(103.767185,18.346685),(103.709101,18.349708),(103.695562,18.353222),(103.638614,18.377949),(103.629932,18.38322),(103.621354,18.390765),(103.607143,18.400403),(103.5488,18.415415),(103.523169,18.417688),(103.497744,18.423734),(103.489114,18.424432),(103.4748,18.422158),(103.469942,18.421977),(103.451545,18.425311),(103.396768,18.441408),(103.38664,18.441459),(103.306851,18.425672),(103.282564,18.41557),(103.260756,18.400196),(103.244065,18.380068),(103.233988,18.35578),(103.233678,18.349269),(103.235331,18.345264),(103.238018,18.341569),(103.240189,18.335988),(103.248664,18.332629),(103.290418,18.305137),(103.296516,18.296895),(103.295379,18.287774),(103.286336,18.280332),(103.27352,18.276069),(103.236727,18.269997),(103.214919,18.262297),(103.185205,18.25775),(103.17673,18.254856),(103.168204,18.249766),(103.1573,18.237027),(103.150892,18.221499),(103.145104,18.188012),(103.138593,18.166722),(103.126656,18.151865),(103.110068,18.141814),(103.089087,18.134786),(103.089404,18.135223),(103.090087,18.136165),(103.090586,18.136853),(103.092756,18.141194),(103.092133,18.140882),(103.087703,18.138667),(103.07994,18.134786),(103.07131,18.125561),(103.066349,18.112255),(103.064799,18.094039),(103.073067,18.035515),(103.06821,18.025774),(103.058288,18.019263),(103.049399,18.005),(103.037927,17.990686),(103.020461,17.984226),(103.003821,17.988567),(102.972918,18.007739),(102.954935,18.012132),(102.938605,18.008928),(102.852719,17.972031),(102.838043,17.963763),(102.831893,17.953117),(102.774067,17.922731),(102.737377,17.890434),(102.722184,17.881804),(102.686734,17.873535),(102.672988,17.865371),(102.667614,17.850074),(102.67335,17.831781),(102.68291,17.817363),(102.68291,17.810077),(102.660793,17.812867),(102.609426,17.837),(102.595267,17.840824),(102.595577,17.850074),(102.61232,17.91529),(102.599608,17.955184),(102.567517,17.970842),(102.482044,17.971152),(102.445509,17.977767),(102.412229,17.9942),(102.385151,18.016059),(102.366392,18.038797),(102.336523,18.03704),(102.306706,18.051457),(102.25658,18.094039),(102.233119,18.124166),(102.225574,18.128197),(102.221801,18.131608),(102.191571,18.152071),(102.182114,18.169228),(102.178393,18.185971),(102.171934,18.200286),(102.153537,18.210104),(102.107385,18.212377),(102.078503,18.213799),(102.071268,18.208502),(102.055527,18.192345),(101.904301,18.037117),(101.883786,18.030839),(101.845184,18.052129),(101.806064,18.062387),(101.796659,18.073575),(101.790407,18.073575),(101.773922,18.058563),(101.765757,18.040321),(101.75692,17.997817),(101.732581,17.932033),(101.729067,17.912189),(101.636204,17.894826),(101.619874,17.884646),(101.611141,17.865526),(101.600392,17.854364),(101.577551,17.868109),(101.577551,17.852917),(101.575071,17.839791),(101.570523,17.828887),(101.564012,17.820309),(101.545505,17.813672),(101.53404,17.80956),(101.556261,17.792145),(101.53652,17.779123),(101.502724,17.765377),(101.482983,17.745791),(101.479004,17.737988),(101.474198,17.730598),(101.470426,17.722589),(101.468307,17.720367),(101.459109,17.739487),(101.457352,17.745791),(101.45606,17.750184),(101.453476,17.751011),(101.449807,17.749254),(101.44526,17.745791),(101.43203,17.733596),(101.416993,17.732355),(101.401851,17.733751),(101.388002,17.729307),(101.381904,17.720728),(101.388364,17.717783),(101.398751,17.717163),(101.404022,17.715561),(101.403092,17.709463),(101.401593,17.706672),(101.383765,17.684916),(101.37746,17.682643),(101.369864,17.683159),(101.357306,17.679025),(101.247029,17.592157),(101.228632,17.568489),(101.20853,17.531282),(101.202174,17.523169),(101.190288,17.518725),(101.176956,17.5164),(101.166724,17.510612),(101.164243,17.495781),(101.164243,17.495729),(101.14719,17.472475),(101.141816,17.467152),(101.132307,17.461674),(101.128897,17.461158),(101.124659,17.463948),(101.09603,17.475265),(101.066161,17.492215),(101.049315,17.495729),(101.035466,17.512627),(101.019032,17.526632),(101.000842,17.538207),(100.981515,17.547612),(100.937738,17.555486),(100.901934,17.561927),(100.886017,17.57376),(100.885656,17.595878),(100.899453,17.616962),(100.932939,17.654531),(100.953817,17.699799),(100.955878,17.707532),(100.96069,17.725586),(100.958726,17.745791),(100.956401,17.75592),(100.958571,17.764136),(100.964876,17.770596),(100.974281,17.775505),(100.975211,17.777624),(100.975521,17.779794),(100.975211,17.781913),(100.974281,17.78398),(100.967718,17.791008),(100.965186,17.797623),(100.967046,17.803824),(100.974281,17.80925),(100.994796,17.817725),(101.000274,17.830489),(100.997122,17.864595),(101.000842,17.884801),(101.008284,17.89524),(101.039806,17.912034),(101.066161,17.933997),(101.081974,17.958336),(101.094893,17.984381),(101.112308,18.011305),(101.125434,18.02133),(101.157525,18.04045),(101.165173,18.053421),(101.162073,18.067451),(101.144968,18.104736),(101.141609,18.123959),(101.14564,18.141943),(101.160523,18.179512),(101.162796,18.195299),(101.159282,18.204058),(101.152771,18.209071),(101.145536,18.212998),(101.139645,18.218708),(101.131739,18.238862),(101.12838,18.245735),(101.127966,18.280952),(101.130137,18.291159),(101.135408,18.298393),(101.152409,18.316377),(101.155407,18.322888),(101.145536,18.336246),(101.082284,18.364203),(101.074089,18.371613),(101.05102,18.39247),(101.037016,18.410454),(101.030298,18.427791),(101.035621,18.447841),(101.064714,18.476419),(101.071949,18.495668),(101.073809,18.505771),(101.077892,18.510422),(101.142074,18.544296),(101.155355,18.556724),(101.159127,18.567705),(101.158145,18.589978),(101.160884,18.599796),(101.167757,18.606359),(101.185327,18.612534),(101.194112,18.616798),(101.221914,18.642042),(101.23473,18.658449),(101.240104,18.673642),(101.234782,18.68948),(101.211269,18.711908),(101.205843,18.730072),(101.209202,18.746325),(101.223981,18.778545),(101.228632,18.795624),(101.223309,18.855491),(101.224911,18.874766),(101.232921,18.892724),(101.257261,18.919492),(101.267286,18.934453),(101.277001,18.96706),(101.283926,18.981478),(101.326404,19.020546),(101.317877,19.05013),(101.291161,19.078992),(101.247442,19.115785),(101.235712,19.12811),(101.229149,19.143587),(101.226772,19.167152),(101.229872,19.225804),(101.209512,19.309417),(101.209253,19.315592),(101.210907,19.320631),(101.211269,19.326367),(101.207755,19.334687),(101.201967,19.338666),(101.185017,19.341999),(101.178454,19.346624),(101.172356,19.376389),(101.177782,19.417214),(101.192872,19.452793),(101.2153,19.466358),(101.238244,19.472301),(101.251731,19.494444),(101.256951,19.522789),(101.254884,19.547154),(101.25106,19.560306),(101.246202,19.570279),(101.238244,19.578186),(101.225015,19.585214),(101.207961,19.589555),(101.197316,19.586868),(101.188324,19.580744),(101.176852,19.574749),(101.152668,19.568367),(101.124659,19.565112),(101.095824,19.567282),(101.068228,19.577127),(101.024303,19.602861),(101.002548,19.610613),(100.974281,19.613481),(100.885656,19.612447),(100.879816,19.613584),(100.874597,19.612912),(100.865037,19.607151),(100.861213,19.602603),(100.849017,19.582527),(100.808451,19.536535),(100.762614,19.495478),(100.753002,19.48243),(100.745509,19.485013),(100.738171,19.495039),(100.729231,19.50465),(100.714762,19.512454),(100.633268,19.542167),(100.607016,19.540927),(100.585984,19.525967),(100.574758,19.508347),(100.566657,19.495633),(100.549294,19.494548),(100.520613,19.502609),(100.491829,19.514469),(100.474259,19.524985),(100.46196,19.537103),(100.458963,19.551831),(100.458963,19.568057),(100.455553,19.584671),(100.447491,19.597771),(100.427079,19.621801),(100.418862,19.634358),(100.41292,19.651566),(100.410491,19.684252),(100.407235,19.699936),(100.400466,19.71164),(100.390048,19.724387),(100.383826,19.732001),(100.379175,19.745669),(100.383516,19.763859),(100.407659,19.794257),(100.420516,19.810446),(100.43142,19.838273),(100.438448,19.847213),(100.446251,19.852123),(100.46563,19.859771),(100.474259,19.865352),(100.483975,19.8807),(100.487954,19.896978),(100.492656,19.931653),(100.518546,19.995421),(100.518546,19.995499),(100.543351,20.06658),(100.550586,20.106526),(100.548674,20.158021),(100.546503,20.167917),(100.541956,20.166729),(100.536685,20.153474),(100.529243,20.149727),(100.51281,20.155282),(100.496377,20.164868),(100.488936,20.173343),(100.485628,20.178769),(100.468782,20.18572),(100.46103,20.190707),(100.45731,20.196391),(100.452142,20.21024),(100.447336,20.218044),(100.421601,20.250186),(100.364395,20.368965),(100.344345,20.389971),(100.306518,20.399686),(100.270241,20.391831),(100.240372,20.372814),(100.220683,20.348992),(100.209159,20.312999),(100.207196,20.311139),(100.201718,20.310312),(100.179497,20.302483),(100.176397,20.300622),(100.167095,20.292613),(100.167715,20.253623),(100.152522,20.239153),(100.134642,20.239102),(100.116607,20.248016),(100.102964,20.263596),(100.09728,20.283233),(100.099295,20.317805),(100.097642,20.334936),(100.097073,20.348371),(100.099812,20.362221),(100.111129,20.37483),(100.120948,20.388782),(100.125237,20.406404),(100.127407,20.42785),(100.136657,20.459217),(100.149266,20.548101),(100.160067,20.582672),(100.172934,20.609079),(100.172934,20.622102),(100.180169,20.638173),(100.202338,20.667939),(100.207196,20.680754),(100.21133,20.697394),(100.221613,20.70959),(100.248744,20.731966),(100.27453,20.76199),(100.290188,20.775012),(100.329669,20.786019),(100.341968,20.799507),(100.351115,20.81532),(100.364809,20.828187),(100.383671,20.831701),(100.470849,20.818265),(100.513379,20.807),(100.536685,20.807672),(100.569034,20.8178),(100.598903,20.833458),(100.607776,20.839881),(100.626602,20.853509),(100.65213,20.876608),(100.65213,20.882757),(100.623088,20.888545),(100.534204,20.873921),(100.515704,20.886478),(100.517823,20.906322),(100.527073,20.94694),(100.529243,20.968437),(100.533274,20.97345),(100.551981,20.985335),(100.557252,20.992001),(100.555753,20.999856),(100.543454,21.0274),(100.561851,21.0351),(100.606758,21.043936),(100.625516,21.054117),(100.637557,21.069671),(100.689388,21.15933),(100.721066,21.280666),(100.722565,21.292707),(100.721996,21.303662),(100.725407,21.311672),(100.738378,21.314772),(100.747731,21.314256),(100.756671,21.312447),(100.764991,21.309191),(100.772588,21.30423),(100.791966,21.297357),(100.8146,21.3002),(100.851342,21.314772),(100.862453,21.323299),(100.896043,21.355752),(100.909943,21.360093),(101.003426,21.405981),(101.143159,21.513417),(101.157112,21.519669),(101.161143,21.534707),(101.159024,21.552691),(101.174527,21.551657),(101.186567,21.535327),(101.187756,21.505872),(101.168274,21.424533),(101.171426,21.403398),(101.179126,21.398178),(101.207961,21.383502),(101.226462,21.370893),(101.230802,21.371772),(101.233128,21.37172),(101.234782,21.363658),(101.222844,21.335081),(101.207445,21.314824),(101.209512,21.308675),(101.2184,21.299941),(101.220364,21.296272),(101.219072,21.282475),(101.208995,21.245526),(101.220364,21.233692),(101.234782,21.206252),(101.244342,21.192868),(101.259741,21.179484),(101.275296,21.174109),(101.292711,21.176073),(101.313898,21.184755),(101.362267,21.215812),(101.381491,21.222478),(101.495127,21.242839),(101.517607,21.242632),(101.539104,21.23886),(101.5729,21.228163),(101.57967,21.227594),(101.583959,21.22439),(101.588197,21.213332),(101.586646,21.204237),(101.580962,21.194728),(101.577241,21.185168),(101.581789,21.175659),(101.596517,21.173386),(101.635946,21.189457),(101.655118,21.189199),(101.670155,21.177055),(101.688294,21.145842),(101.704262,21.135093),(101.717905,21.134473),(101.73749,21.137574),(101.75599,21.143258),(101.766739,21.150648),(101.768453,21.16045),(101.770201,21.17044),(101.763586,21.184651),(101.760072,21.195968),(101.77356,21.207389),(101.796039,21.202945),(101.805238,21.204495),(101.813299,21.209559),(101.819294,21.215967),(101.82149,21.221071),(101.822808,21.224132),(101.823738,21.234467),(101.820224,21.247283),(101.812266,21.258135),(101.801155,21.267075),(101.788546,21.274206),(101.736508,21.292293),(101.722504,21.30423),(101.715114,21.321129),(101.715579,21.338182),(101.725553,21.375389),(101.729687,21.474246),(101.744259,21.495588),(101.748239,21.514502),(101.734751,21.554396),(101.736198,21.570881),(101.750926,21.579666),(101.772682,21.582508),(101.793559,21.588244),(101.806168,21.605969),(101.803842,21.625916),(101.789528,21.634029),(101.770821,21.638835),(101.755938,21.648654),(101.752321,21.659144),(101.755628,21.680021),(101.754801,21.689788),(101.749117,21.698057),(101.732787,21.711441),(101.72855,21.717539),(101.727981,21.73268),(101.73165,21.750508),(101.751546,21.80637),(101.751081,21.816137),(101.726173,21.837273),(101.721263,21.844353),(101.708706,21.869571),(101.685865,21.898613),(101.683075,21.907139),(101.682454,21.914839),(101.680594,21.922642),(101.673669,21.931376),(101.638116,21.940884),(101.616309,21.953648),(101.6068,21.967601),(101.600392,22.007495),(101.592176,22.028114),(101.564942,22.069507),(101.555434,22.090332),(101.55502,22.112398),(101.561842,22.130072),(101.566958,22.149347),(101.561532,22.176115),(101.549026,22.19353),(101.518227,22.228205),(101.515746,22.245362),(101.531714,22.263397),(101.540654,22.271458),(101.55099,22.276678),(101.563702,22.276781),(101.575743,22.273112),(101.58799,22.271355),(101.601219,22.276988),(101.606128,22.284842),(101.619874,22.326442),(101.62251,22.342358),(101.623698,22.346647),(101.641113,22.363287),(101.643697,22.364683),(101.645713,22.384526),(101.644162,22.404008),(101.645092,22.424214),(101.654549,22.446125),(101.668657,22.462299),(101.689121,22.478887),(101.71315,22.491548),(101.741779,22.496044),(101.750616,22.496044),(101.75506,22.495527),(101.784877,22.472221),(101.817795,22.406334),(101.8426,22.383338),(101.867921,22.378842)] +Lebanon [(36.391194,34.622487),(36.415999,34.622901),(36.440184,34.62936),(36.436256,34.597838),(36.429125,34.593187),(36.418479,34.600318),(36.4037,34.603832),(36.388921,34.596236),(36.386853,34.584247),(36.388094,34.569519),(36.383546,34.553809),(36.364012,34.541097),(36.336624,34.52885),(36.319881,34.514122),(36.329851,34.498603),(36.332697,34.494175),(36.335384,34.493477),(36.338174,34.493296),(36.340965,34.493477),(36.343652,34.494175),(36.363082,34.499032),(36.392331,34.500143),(36.41972,34.498335),(36.433466,34.49433),(36.433466,34.494175),(36.439873,34.477457),(36.449795,34.464538),(36.463231,34.455237),(36.480181,34.449139),(36.494651,34.434256),(36.501885,34.430303),(36.509223,34.432421),(36.52514,34.422939),(36.530617,34.41418),(36.52979,34.403095),(36.526276,34.386688),(36.523589,34.379789),(36.519972,34.375061),(36.516768,34.369867),(36.515941,34.36165),(36.518628,34.353641),(36.523589,34.345889),(36.54519,34.320723),(36.555835,34.311085),(36.562967,34.308501),(36.570305,34.308966),(36.576816,34.307674),(36.581984,34.299897),(36.579917,34.286901),(36.570305,34.275868),(36.56276,34.262949),(36.567514,34.244138),(36.574026,34.229746),(36.59821,34.209851),(36.603554,34.200101),(36.604101,34.199102),(36.575886,34.173264),(36.552735,34.140992),(36.507983,34.109185),(36.488449,34.088644),(36.482662,34.07451),(36.480698,34.062754),(36.47522,34.053504),(36.45889,34.046708),(36.445144,34.04614),(36.423647,34.052806),(36.412175,34.053633),(36.391091,34.044719),(36.357811,34.009553),(36.31802,33.980511),(36.302311,33.964388),(36.288565,33.946508),(36.267171,33.910386),(36.268928,33.906769),(36.275336,33.897209),(36.28164,33.891111),(36.311096,33.87199),(36.35037,33.866409),(36.36763,33.857547),(36.369594,33.837006),(36.357501,33.823776),(36.338278,33.821193),(36.300554,33.828686),(36.249187,33.849744),(36.231824,33.852534),(36.216631,33.847754),(36.202679,33.838297),(36.186556,33.83021),(36.165678,33.829797),(36.157617,33.83406),(36.144904,33.847418),(36.13829,33.850622),(36.102426,33.832277),(36.085063,33.82605),(36.06646,33.821994),(36.05044,33.816438),(36.040518,33.805612),(36.027082,33.784296),(35.994009,33.760705),(35.980263,33.743549),(35.974165,33.732154),(35.955872,33.718124),(35.948431,33.70921),(35.946984,33.700838),(35.949878,33.6827),(35.94719,33.673088),(35.942436,33.669212),(35.929414,33.665078),(35.925383,33.661667),(35.921559,33.640299),(35.934788,33.6334),(35.956906,33.629938),(35.980263,33.619008),(35.994526,33.615701),(36.012923,33.608027),(36.029666,33.59764),(36.038451,33.586349),(36.035247,33.567797),(36.019847,33.55263),(35.99928,33.541261),(35.980263,33.533949),(35.956802,33.534207),(35.933238,33.527825),(35.921869,33.51457),(35.935408,33.494262),(35.919492,33.462351),(35.888486,33.440415),(35.870089,33.431242),(35.845116,33.418742),(35.8211,33.406722),(35.8057,33.391348),(35.785753,33.357887),(35.769423,33.342643),(35.757538,33.336313),(35.743689,33.331171),(35.729426,33.327812),(35.716197,33.326727),(35.698523,33.32267),(35.66049,33.289261),(35.640542,33.275851),(35.624716,33.272924),(35.610363,33.27027),(35.601475,33.262725),(35.604576,33.244354),(35.604576,33.244303),(35.604576,33.244251),(35.603892,33.240323),(35.603852,33.240091),(35.598064,33.244251),(35.597858,33.24438),(35.597754,33.244354),(35.585042,33.252209),(35.566955,33.276187),(35.561167,33.28213),(35.549489,33.281019),(35.542771,33.271536),(35.53688,33.257868),(35.527784,33.244251),(35.520033,33.222185),(35.517656,33.172886),(35.512282,33.147409),(35.503703,33.130615),(35.4851,33.102606),(35.480139,33.087387),(35.449443,33.085217),(35.401591,33.067931),(35.34516,33.05558),(35.331931,33.057156),(35.322629,33.067363),(35.315704,33.079222),(35.305783,33.089377),(35.29462,33.096973),(35.288825,33.099201),(35.283665,33.101185),(35.271469,33.101185),(35.234469,33.090927),(35.227338,33.091366),(35.213178,33.094441),(35.207184,33.094803),(35.200673,33.092736),(35.189614,33.085527),(35.185273,33.083977),(35.105235,33.089016),(35.105235,33.090644),(35.106456,33.095038),(35.101899,33.094468),(35.100434,33.095689),(35.100434,33.098212),(35.09962,33.101264),(35.105805,33.112454),(35.114757,33.123725),(35.125499,33.132514),(35.147634,33.140326),(35.154796,33.150295),(35.160981,33.161689),(35.167817,33.170152),(35.161794,33.176947),(35.167817,33.177883),(35.171072,33.176581),(35.174653,33.170152),(35.204763,33.224799),(35.209158,33.254625),(35.188324,33.280015),(35.209727,33.293402),(35.224783,33.311998),(35.243663,33.355699),(35.246104,33.368883),(35.246755,33.395819),(35.250499,33.409735),(35.268728,33.432603),(35.275076,33.444973),(35.271007,33.457506),(35.323009,33.490912),(35.349132,33.513861),(35.360362,33.543443),(35.362478,33.556342),(35.400645,33.643134),(35.394379,33.64997),(35.409923,33.661282),(35.421641,33.702338),(35.438487,33.711371),(35.437836,33.723375),(35.477306,33.8039),(35.481293,33.823676),(35.483002,33.86518),(35.483246,33.872463),(35.479991,33.884263),(35.473806,33.893866),(35.47047,33.902086),(35.475841,33.90998),(35.486583,33.912991),(35.499766,33.9112),(35.511241,33.90705),(35.517345,33.902533),(35.524181,33.902533),(35.533865,33.90705),(35.548106,33.906195),(35.557465,33.905666),(35.572032,33.90998),(35.57838,33.916693),(35.587901,33.936591),(35.59311,33.94477),(35.593435,33.94892),(35.59197,33.950426),(35.585704,33.950914),(35.594249,33.959296),(35.606293,33.983222),(35.613536,33.99315),(35.620128,33.995103),(35.636729,33.995307),(35.64088,33.999416),(35.640391,34.008531),(35.635753,34.013251),(35.630382,34.016099),(35.627208,34.019843),(35.626964,34.044257),(35.643321,34.08808),(35.647716,34.119127),(35.644542,34.130845),(35.630382,34.152818),(35.627208,34.166938),(35.627208,34.201117),(35.630382,34.208441),(35.644542,34.216986),(35.647716,34.222154),(35.648285,34.253607),(35.653005,34.284857),(35.665212,34.311347),(35.688731,34.328315),(35.705821,34.316596),(35.722179,34.33747),(35.737315,34.369208),(35.750743,34.389797),(35.806163,34.414984),(35.823578,34.434516),(35.805431,34.458686),(35.805431,34.464911),(35.823985,34.460842),(35.903331,34.477973),(35.956798,34.520087),(35.972911,34.529771),(35.983165,34.538642),(35.98878,34.55272),(35.989594,34.599677),(35.986827,34.620795),(35.980642,34.638495),(35.9699,34.649848),(35.9699,34.649849),(35.98264,34.650238),(35.991425,34.648222),(35.998763,34.64538),(36.01623,34.633649),(36.037934,34.62843),(36.060775,34.627913),(36.087234,34.631014),(36.11214,34.629532),(36.15586,34.626931),(36.160201,34.628068),(36.17219,34.634063),(36.178287,34.635303),(36.183352,34.633494),(36.195134,34.626208),(36.201232,34.624451),(36.26159,34.627242),(36.271512,34.630962),(36.28071,34.639127),(36.284431,34.648326),(36.284844,34.667911),(36.288151,34.675559),(36.308925,34.687548),(36.323808,34.679486),(36.347373,34.64414),(36.367526,34.629205),(36.391194,34.622487)] +Liberia [(-9.722115,8.435636),(-9.718188,8.438272),(-9.717981,8.452689),(-9.7155,8.473334),(-9.703253,8.487261),(-9.683513,8.487054),(-9.663514,8.477856),(-9.650543,8.464807),(-9.645117,8.443413),(-9.653385,8.43145),(-9.666511,8.422381),(-9.675916,8.409643),(-9.67726,8.39153),(-9.668372,8.38874),(-9.653385,8.392099),(-9.636125,8.392564),(-9.628064,8.391634),(-9.619537,8.392719),(-9.612148,8.396336),(-9.607342,8.402925),(-9.600159,8.410883),(-9.592045,8.405095),(-9.579178,8.387293),(-9.529052,8.366312),(-9.511482,8.353781),(-9.504351,8.346158),(-9.499131,8.343471),(-9.511482,8.336133),(-9.515616,8.334376),(-9.522282,8.330681),(-9.528483,8.32572),(-9.539387,8.314584),(-9.530189,8.30598),(-9.525124,8.295438),(-9.521249,8.284405),(-9.515616,8.274406),(-9.510758,8.270065),(-9.509156,8.265595),(-9.51081,8.261047),(-9.515616,8.256603),(-9.525435,8.24079),(-9.531894,8.212316),(-9.530137,8.185496),(-9.515616,8.174877),(-9.491328,8.169968),(-9.481406,8.164697),(-9.478099,8.152785),(-9.478047,8.141287),(-9.459805,8.068785),(-9.461873,8.058424),(-9.472776,8.040363),(-9.474223,8.028813),(-9.462183,8.043464),(-9.451537,8.051758),(-9.440944,8.050569),(-9.429265,8.036901),(-9.424149,8.026178),(-9.42234,8.015791),(-9.424562,8.005817),(-9.431745,7.996567),(-9.433093,7.993468),(-9.446008,7.950058),(-9.448437,7.907839),(-9.43867,7.866239),(-9.415674,7.824433),(-9.389681,7.791464),(-9.380999,7.77441),(-9.355471,7.741906),(-9.366374,7.716171),(-9.376245,7.680566),(-9.374694,7.646925),(-9.353507,7.62243),(-9.364411,7.614265),(-9.372989,7.601346),(-9.378777,7.586256),(-9.38366,7.55954),(-9.390817,7.548223),(-9.408697,7.527449),(-9.404925,7.513599),(-9.4134,7.492257),(-9.436086,7.458564),(-9.456705,7.443061),(-9.466213,7.433449),(-9.470244,7.420995),(-9.470864,7.407353),(-9.4735,7.39557),(-9.479701,7.38539),(-9.490708,7.376037),(-9.487099,7.373204),(-9.486626,7.372833),(-9.478357,7.376915),(-9.449263,7.405079),(-9.438101,7.421564),(-9.418723,7.387767),(-9.407354,7.378052),(-9.392264,7.39986),(-9.386477,7.401668),(-9.381412,7.404975),(-9.379242,7.415828),(-9.375159,7.41929),(-9.365961,7.421099),(-9.349218,7.421564),(-9.335007,7.419652),(-9.327772,7.417068),(-9.319401,7.411952),(-9.312424,7.403322),(-9.3055,7.383478),(-9.299505,7.376605),(-9.283796,7.374073),(-9.249715,7.383065),(-9.234341,7.381308),(-9.219872,7.360999),(-9.206281,7.298367),(-9.181476,7.275681),(-9.17543,7.274596),(-9.15308,7.275474),(-9.147473,7.271392),(-9.146905,7.25408),(-9.139153,7.247879),(-9.11962,7.237957),(-9.116054,7.224676),(-9.125407,7.190208),(-9.113935,7.200388),(-9.106085,7.202975),(-9.100448,7.204833),(-9.086392,7.20721),(-9.073111,7.211344),(-9.059261,7.221369),(-9.046652,7.232738),(-9.032855,7.240283),(-9.015698,7.238887),(-9.007637,7.243487),(-8.985467,7.25005),(-8.981023,7.250566),(-8.977716,7.266224),(-8.973117,7.267671),(-8.962575,7.262969),(-8.9572,7.286481),(-8.944385,7.278678),(-8.925419,7.248603),(-8.91219,7.250153),(-8.868885,7.266638),(-8.857827,7.273046),(-8.851367,7.292889),(-8.855915,7.311855),(-8.863304,7.32989),(-8.865371,7.346891),(-8.860617,7.35764),(-8.851626,7.371489),(-8.83341,7.393865),(-8.82403,7.399808),(-8.807701,7.40451),(-8.799226,7.41221),(-8.795453,7.421099),(-8.793955,7.441511),(-8.790389,7.450812),(-8.784394,7.455722),(-8.756489,7.472413),(-8.739539,7.490552),(-8.729617,7.50869),(-8.725742,7.528534),(-8.726827,7.55184),(-8.729514,7.556387),(-8.73401,7.557679),(-8.738041,7.559488),(-8.739178,7.565689),(-8.737369,7.571425),(-8.731478,7.581761),(-8.730031,7.587445),(-8.727576,7.622017),(-8.720677,7.642739),(-8.694477,7.674416),(-8.686571,7.694364),(-8.567405,7.688214),(-8.57278,7.650749),(-8.566527,7.623412),(-8.54782,7.600777),(-8.515625,7.577161),(-8.485446,7.557989),(-8.44364,7.537112),(-8.425605,7.50192),(-8.405141,7.431279),(-8.393049,7.334334),(-8.387675,7.314955),(-8.38106,7.303018),(-8.364007,7.278885),(-8.357547,7.265708),(-8.349692,7.234702),(-8.34499,7.226795),(-8.333001,7.211861),(-8.32897,7.204006),(-8.328453,7.197391),(-8.330572,7.184731),(-8.329745,7.178323),(-8.326231,7.17207),(-8.311503,7.15729),(-8.305354,7.146903),(-8.302615,7.137602),(-8.30122,7.11321),(-8.284115,7.017867),(-8.286647,6.993786),(-8.29414,6.984381),(-8.314552,6.968568),(-8.319978,6.95999),(-8.321219,6.942265),(-8.319978,6.925832),(-8.320909,6.910174),(-8.333673,6.882578),(-8.328763,6.875085),(-8.320599,6.869659),(-8.315715,6.86382),(-8.314966,6.850281),(-8.316619,6.836276),(-8.324991,6.806252),(-8.337497,6.777262),(-8.351759,6.755351),(-8.441646,6.65531),(-8.461313,6.633421),(-8.533557,6.576344),(-8.547148,6.56004),(-8.549473,6.560143),(-8.566113,6.550919),(-8.567198,6.550609),(-8.566113,6.549059),(-8.570609,6.535132),(-8.572728,6.530843),(-8.577327,6.528026),(-8.589368,6.526063),(-8.594122,6.523892),(-8.61872,6.492835),(-8.601847,6.49175),(-8.592106,6.489683),(-8.568077,6.490406),(-8.556501,6.487021),(-8.545339,6.477203),(-8.515625,6.437593),(-8.497229,6.431366),(-8.484775,6.441572),(-8.47692,6.460149),(-8.472372,6.47865),(-8.467515,6.487073),(-8.46121,6.482887),(-8.449635,6.467927),(-8.418267,6.451209),(-8.405089,6.436327),(-8.406485,6.430539),(-8.413874,6.424183),(-8.419042,6.407646),(-8.417233,6.398189),(-8.408242,6.379922),(-8.405968,6.372248),(-8.410981,6.363179),(-8.420153,6.358088),(-8.423693,6.351991),(-8.411807,6.340028),(-8.401059,6.35044),(-8.388811,6.355091),(-8.376564,6.35473),(-8.36597,6.350079),(-8.352948,6.36106),(-8.342613,6.356952),(-8.332949,6.34597),(-8.322356,6.336462),(-8.317498,6.336979),(-8.314191,6.341526),(-8.310005,6.344627),(-8.302718,6.340648),(-8.297706,6.332586),(-8.296569,6.324395),(-8.296569,6.317497),(-8.29507,6.313363),(-8.285975,6.310055),(-8.208486,6.292537),(-8.196575,6.288144),(-8.185723,6.28184),(-8.179057,6.276853),(-8.170582,6.274011),(-8.154355,6.274218),(-8.140609,6.277577),(-8.118285,6.289901),(-8.106606,6.294681),(-8.060098,6.301141),(-8.033019,6.301761),(-8.015708,6.297782),(-7.999378,6.286697),(-7.934886,6.271427),(-7.929511,6.270755),(-7.925222,6.272512),(-7.920545,6.273804),(-7.91437,6.271737),(-7.912665,6.26843),(-7.910029,6.256777),(-7.906567,6.251997),(-7.862125,6.216392),(-7.846622,6.197478),(-7.844142,6.183965),(-7.857371,6.145388),(-7.862022,6.124382),(-7.864502,6.101696),(-7.861815,6.08149),(-7.850911,6.067564),(-7.850498,6.07224),(-7.840886,6.077356),(-7.82838,6.080715),(-7.819389,6.080095),(-7.809674,6.071207),(-7.805695,6.062034),(-7.802439,6.039245),(-7.799235,6.035679),(-7.793861,6.034077),(-7.788745,6.031493),(-7.786471,6.025111),(-7.78859,6.019789),(-7.797736,6.01475),(-7.799958,6.010151),(-7.800682,5.981729),(-7.799648,5.974468),(-7.786626,5.958165),(-7.747145,5.937597),(-7.73495,5.925634),(-7.717173,5.900752),(-7.70322,5.907883),(-7.691541,5.927727),(-7.680483,5.94093),(-7.674023,5.935737),(-7.663119,5.923102),(-7.64834,5.911294),(-7.630615,5.908607),(-7.629116,5.904938),(-7.608497,5.893698),(-7.605758,5.89313),(-7.584158,5.883699),(-7.579093,5.877188),(-7.587207,5.866775),(-7.579507,5.860341),(-7.566278,5.843727),(-7.559146,5.839851),(-7.547571,5.845432),(-7.535194,5.855845),(-7.527779,5.858842),(-7.531086,5.842022),(-7.515583,5.839231),(-7.508813,5.826183),(-7.503904,5.812101),(-7.494396,5.806184),(-7.486644,5.813806),(-7.481683,5.830214),(-7.478583,5.859876),(-7.446543,5.845949),(-7.442047,5.811274),(-7.448559,5.769055),(-7.449489,5.732364),(-7.43104,5.653971),(-7.417963,5.627757),(-7.396676,5.585087),(-7.384118,5.568964),(-7.405099,5.5724),(-7.41192,5.560359),(-7.407114,5.541394),(-7.393627,5.523979),(-7.41347,5.522816),(-7.419982,5.523127),(-7.409852,5.513417),(-7.408148,5.511784),(-7.41347,5.503489),(-7.425149,5.494162),(-7.432384,5.479563),(-7.431712,5.441917),(-7.436208,5.431323),(-7.450212,5.438713),(-7.449954,5.422771),(-7.446027,5.404917),(-7.439102,5.387812),(-7.429697,5.374298),(-7.414866,5.365978),(-7.404375,5.367994),(-7.396934,5.366444),(-7.391456,5.347685),(-7.388821,5.328384),(-7.389751,5.317144),(-7.39559,5.306783),(-7.407683,5.289859),(-7.410111,5.283167),(-7.41378,5.266708),(-7.418793,5.260455),(-7.428508,5.25614),(-7.430834,5.25999),(-7.430885,5.267535),(-7.434089,5.274072),(-7.435175,5.279291),(-7.43011,5.284252),(-7.428508,5.288154),(-7.439929,5.290169),(-7.442409,5.287379),(-7.468247,5.273452),(-7.475689,5.264228),(-7.479926,5.255158),(-7.490468,5.222189),(-7.482252,5.17506),(-7.483027,5.158239),(-7.488918,5.140669),(-7.515841,5.091809),(-7.529923,5.09739),(-7.546847,5.0977),(-7.563229,5.090156),(-7.575786,5.072069),(-7.578163,5.054731),(-7.57036,4.99688),(-7.570205,4.99688),(-7.5716,4.975382),(-7.564624,4.961585),(-7.555322,4.948433),(-7.549586,4.929054),(-7.549534,4.913887),(-7.553462,4.914249),(-7.561782,4.920166),(-7.574959,4.921561),(-7.6055,4.897428),(-7.605965,4.887997),(-7.598834,4.857095),(-7.600229,4.845804),(-7.605862,4.825133),(-7.605997,4.817363),(-7.606017,4.816193),(-7.601004,4.804643),(-7.593408,4.797434),(-7.585553,4.791543),(-7.579558,4.784283),(-7.576871,4.775291),(-7.572479,4.723356),(-7.576871,4.509338),(-7.575424,4.500347),(-7.572737,4.491122),(-7.570929,4.48071),(-7.572841,4.455388),(-7.565709,4.435208),(-7.564314,4.425829),(-7.567415,4.415933),(-7.577853,4.397278),(-7.579197,4.387149),(-7.573771,4.375393),(-7.540666,4.352845),(-7.541005,4.352769),(-7.593251,4.347235),(-7.608225,4.349433),(-7.631663,4.35928),(-7.644439,4.361518),(-7.712717,4.361518),(-7.721018,4.364691),(-7.725413,4.371772),(-7.728627,4.378811),(-7.733225,4.382025),(-7.749501,4.383246),(-7.788726,4.398912),(-7.801015,4.406317),(-7.812327,4.416083),(-7.818267,4.427191),(-7.823801,4.440741),(-7.832875,4.452216),(-7.862904,4.461371),(-7.88683,4.48017),(-7.89802,4.484442),(-7.91454,4.486518),(-7.924428,4.491685),(-7.9322,4.498358),(-7.942006,4.504869),(-7.959869,4.51203),(-8.003163,4.523505),(-8.037506,4.528754),(-8.100331,4.55329),(-8.222035,4.568061),(-8.257314,4.579983),(-8.300201,4.627265),(-8.316029,4.635199),(-8.335357,4.638821),(-8.438547,4.676174),(-8.549957,4.754869),(-8.607533,4.786933),(-8.74706,4.837714),(-8.778147,4.854315),(-8.848297,4.91356),(-8.874338,4.930121),(-8.9051,4.944322),(-8.915517,4.946275),(-8.936106,4.95539),(-8.966786,4.96011),(-8.991322,4.967515),(-9.014801,4.978461),(-9.031972,4.991441),(-9.028717,4.993638),(-9.028066,4.994289),(-9.027821,4.995429),(-9.025787,4.998928),(-9.069651,5.010972),(-9.10497,5.031562),(-9.162262,5.080878),(-9.17809,5.09101),(-9.212758,5.107408),(-9.243967,5.130561),(-9.261952,5.137274),(-9.278717,5.145738),(-9.291982,5.163479),(-9.278961,5.163479),(-9.292144,5.181627),(-9.312245,5.192613),(-9.334462,5.200995),(-9.354075,5.211168),(-9.470815,5.320461),(-9.477935,5.335354),(-9.488515,5.371649),(-9.497467,5.38939),(-9.508046,5.399604),(-9.524281,5.410793),(-9.541819,5.419827),(-9.556508,5.423489),(-9.559397,5.425035),(-9.587636,5.430732),(-9.593658,5.430325),(-9.59496,5.440009),(-9.583079,5.454901),(-9.580719,5.464504),(-9.596995,5.488349),(-9.696767,5.546373),(-9.842112,5.685452),(-9.979644,5.798082),(-10.015207,5.838121),(-10.022613,5.844224),(-10.039947,5.854315),(-10.046213,5.861721),(-10.049062,5.871772),(-10.044789,5.884467),(-10.046213,5.89643),(-10.055531,5.91177),(-10.071278,5.924791),(-10.089101,5.933987),(-10.104237,5.937445),(-10.11498,5.943305),(-10.185699,5.999091),(-10.226552,6.043606),(-10.279897,6.076972),(-10.298166,6.084906),(-10.336008,6.092231),(-10.353668,6.102362),(-10.364125,6.115668),(-10.36148,6.129218),(-10.352284,6.123521),(-10.344472,6.122707),(-10.338124,6.126776),(-10.333567,6.136054),(-10.353627,6.140611),(-10.367991,6.151801),(-10.380605,6.165351),(-10.395619,6.176988),(-10.428131,6.197008),(-10.444692,6.203843),(-10.463938,6.204983),(-10.463938,6.197496),(-10.41218,6.171047),(-10.389394,6.152167),(-10.381947,6.129218),(-10.397125,6.147854),(-10.424875,6.164618),(-10.456695,6.177476),(-10.659413,6.227484),(-10.788319,6.292955),(-10.807851,6.310858),(-10.800364,6.327826),(-10.795562,6.319648),(-10.790354,6.31391),(-10.78307,6.310004),(-10.772369,6.307359),(-10.789052,6.334947),(-10.792348,6.359036),(-10.782338,6.381415),(-10.759389,6.404202),(-10.773671,6.404202),(-10.784088,6.401272),(-10.792348,6.396226),(-10.800364,6.389879),(-10.82433,6.43773),(-10.85733,6.472846),(-10.895904,6.49901),(-11.053619,6.584174),(-11.233062,6.648098),(-11.353179,6.701158),(-11.374867,6.726874),(-11.375722,6.766669),(-11.369944,6.788886),(-11.368642,6.798245),(-11.368276,6.811347),(-11.381947,6.836819),(-11.38561,6.841783),(-11.404408,6.847073),(-11.424224,6.85932),(-11.44107,6.873277),(-11.450795,6.883368),(-11.476186,6.91942),(-11.444491,6.933945),(-11.436739,6.938544),(-11.438858,6.963865),(-11.434672,6.980815),(-11.407283,6.99637),(-11.397723,7.012803),(-11.391522,7.031768),(-11.3893,7.046703),(-11.392917,7.066857),(-11.389714,7.073523),(-11.367906,7.075952),(-11.360465,7.078639),(-11.355142,7.084065),(-11.353127,7.092798),(-11.356434,7.105356),(-11.370593,7.128403),(-11.373229,7.138842),(-11.370128,7.142614),(-11.352041,7.159254),(-11.317211,7.213566),(-11.30021,7.21708),(-11.217889,7.25408),(-11.209259,7.262194),(-11.144213,7.345193),(-11.126319,7.368027),(-10.977904,7.484919),(-10.937028,7.50776),(-10.898994,7.516493),(-10.885713,7.522178),(-10.874603,7.530859),(-10.864025,7.543684),(-10.730581,7.705474),(-10.692598,7.737565),(-10.645418,7.763041),(-10.638131,7.765522),(-10.615187,7.769036),(-10.617668,8.010778),(-10.616221,8.022509),(-10.612138,8.031862),(-10.603147,8.039226),(-10.577825,8.053153),(-10.56718,8.063204),(-10.543977,8.10607),(-10.529249,8.124286),(-10.505736,8.135835),(-10.485738,8.138858),(-10.405071,8.139272),(-10.384555,8.142114),(-10.364918,8.148212),(-10.34554,8.159348),(-10.320942,8.183119),(-10.318151,8.201645),(-10.326006,8.221541),(-10.333292,8.249756),(-10.331277,8.272287),(-10.313655,8.310347),(-10.282908,8.432949),(-10.282236,8.484625),(-10.271332,8.484109),(-10.260842,8.485917),(-10.240275,8.493746),(-10.23242,8.477055),(-10.215625,8.486589),(-10.181829,8.519584),(-10.164879,8.524649),(-10.147981,8.524287),(-10.091085,8.510102),(-10.076099,8.501498),(-10.069277,8.486873),(-10.079354,8.439951),(-10.075013,8.426644),(-10.061681,8.422174),(-10.04039,8.425663),(-10.000186,8.449305),(-9.961119,8.479561),(-9.944169,8.488346),(-9.928149,8.493436),(-9.911354,8.495658),(-9.891872,8.495917),(-9.886911,8.49664),(-9.872804,8.500361),(-9.868153,8.498087),(-9.863502,8.49354),(-9.858644,8.487054),(-9.849704,8.48969),(-9.831617,8.499017),(-9.806813,8.506252),(-9.805469,8.512789),(-9.808828,8.520463),(-9.808854,8.528473),(-9.802007,8.53718),(-9.794824,8.544544),(-9.790018,8.553071),(-9.790586,8.565396),(-9.773585,8.563018),(-9.763043,8.54997),(-9.757307,8.53271),(-9.750899,8.485866),(-9.7463,8.468063),(-9.735913,8.450493),(-9.722115,8.435636)] +Libya [(19.291677,30.287055),(19.338552,30.298814),(19.418956,30.333238),(19.530528,30.396796),(19.551036,30.404934),(19.596039,30.412014),(19.616384,30.421373),(19.748871,30.510932),(19.914317,30.682929),(19.958181,30.749701),(19.982188,30.778632),(20.018728,30.803778),(20.036469,30.819078),(20.099864,30.935004),(20.146658,31.045559),(20.152354,31.088202),(20.146658,31.216783),(20.128103,31.242987),(20.103282,31.308743),(20.082205,31.336615),(20.038341,31.375881),(20.022227,31.396796),(19.954845,31.559475),(19.948904,31.60102),(19.948009,31.645087),(19.944347,31.661444),(19.926931,31.699164),(19.920665,31.717719),(19.917247,31.759223),(19.919688,31.804674),(19.934337,31.888414),(19.948009,31.925971),(19.945811,31.935981),(19.941742,31.946926),(19.940115,31.95775),(19.944591,31.967515),(19.957205,31.982733),(20.048025,32.146064),(20.071544,32.176418),(20.261729,32.339545),(20.277192,32.360826),(20.32545,32.409817),(20.345714,32.419908),(20.385509,32.434638),(20.403331,32.447333),(20.416759,32.459459),(20.562673,32.557847),(20.79656,32.643012),(20.934093,32.722602),(21.050955,32.772284),(21.09669,32.781073),(21.286794,32.771145),(21.331879,32.774644),(21.414806,32.793402),(21.4546,32.812567),(21.608653,32.930732),(21.62322,32.936754),(21.671723,32.939887),(21.716319,32.950385),(21.743663,32.930854),(21.787446,32.915961),(21.836111,32.906562),(21.877452,32.903225),(22.096202,32.923082),(22.101329,32.926418),(22.111827,32.940904),(22.11671,32.944159),(22.138845,32.947577),(22.149587,32.947943),(22.161388,32.944159),(22.169444,32.938218),(22.171072,32.932929),(22.171072,32.926256),(22.175059,32.916246),(22.206798,32.888129),(22.247325,32.880845),(22.33961,32.882148),(22.379161,32.870347),(22.477306,32.807074),(22.496755,32.797309),(22.513845,32.790961),(22.532237,32.787543),(22.578868,32.784613),(22.812511,32.724514),(22.86671,32.694159),(22.884532,32.689765),(22.895844,32.688178),(22.92921,32.676703),(22.942638,32.675035),(22.976899,32.676703),(22.999685,32.670559),(23.041352,32.652533),(23.086436,32.64647),(23.10613,32.639838),(23.118663,32.62934),(23.120942,32.61522),(23.115245,32.604926),(23.097911,32.586656),(23.094412,32.577379),(23.103852,32.516343),(23.113292,32.5008),(23.128673,32.488471),(23.148936,32.478095),(23.140961,32.458564),(23.128429,32.395494),(23.108653,32.40233),(23.112153,32.411933),(23.109874,32.420315),(23.103363,32.428168),(23.094412,32.436469),(23.079112,32.354438),(23.080577,32.334052),(23.095551,32.323554),(23.156505,32.303534),(23.178966,32.299954),(23.195079,32.291653),(23.214529,32.272406),(23.230805,32.250637),(23.2435,32.220608),(23.257335,32.213772),(23.273204,32.209703),(23.285411,32.203762),(23.293224,32.192328),(23.302501,32.170722),(23.312185,32.162746),(23.310395,32.179267),(23.299083,32.224189),(23.31837,32.217841),(23.36199,32.217108),(23.37086,32.214016),(23.398936,32.194037),(23.405284,32.19066),(23.531098,32.181383),(23.669688,32.183254),(23.840343,32.149115),(23.998709,32.093899),(23.975271,32.074205),(23.970714,32.066596),(23.97877,32.062242),(24.004893,32.052924),(24.067882,32.011176),(24.090668,32.005764),(24.280935,32.007554),(24.546072,31.991441),(24.636241,32.014879),(24.658539,32.028632),(24.670258,32.032416),(24.710623,32.02558),(24.718598,32.027533),(24.726329,32.031317),(24.733735,32.033149),(24.741873,32.028998),(24.74586,32.02619),(24.758067,32.019965),(24.771983,32.01675),(24.791759,32.007758),(24.824392,32.002997),(24.865408,31.989325),(24.9817,31.967922),(25.017914,31.949042),(25.033376,31.916327),(25.024425,31.876614),(25.022227,31.854682),(25.029552,31.8369),(25.063731,31.804104),(25.116873,31.737982),(25.122081,31.720852),(25.136729,31.692572),(25.156261,31.663072),(25.152517,31.659817),(25.15089,31.65648),(25.088422,31.613524),(25.060413,31.579366),(25.023723,31.494409),(24.861252,31.38036),(24.848953,31.34863),(24.848643,31.312043),(24.858565,31.227914),(24.859495,31.145955),(24.882646,31.039088),(24.889571,31.018831),(24.924607,30.956768),(24.93763,30.905763),(24.981245,30.825406),(24.994887,30.785099),(24.988583,30.750579),(24.973597,30.716266),(24.961194,30.676268),(24.950652,30.621233),(24.92037,30.552348),(24.917063,30.532453),(24.907658,30.498656),(24.888124,30.464343),(24.829729,30.387552),(24.799447,30.360835),(24.772369,30.331018),(24.76286,30.317065),(24.746117,30.282752),(24.704672,30.217898),(24.688653,30.182758),(24.688343,30.144156),(24.697334,30.122607),(24.72555,30.078372),(24.811643,29.89089),(24.81619,29.868411),(24.81495,29.848567),(24.798207,29.796425),(24.804925,29.7548),(24.856188,29.679482),(24.870657,29.638347),(24.870554,29.614007),(24.861562,29.541557),(24.863836,29.502903),(24.873758,29.468332),(24.956647,29.292683),(24.97432,29.23894),(24.981245,29.181372),(24.981245,29.057917),(24.981245,28.890176),(24.981245,28.491544),(24.981245,28.096701),(24.981245,28.018549),(24.981245,27.666503),(24.981245,27.627824),(24.981245,27.475456),(24.981245,27.138035),(24.981245,27.049239),(24.981245,26.859964),(24.981245,26.664523),(24.981245,26.372707),(24.981245,26.080838),(24.981245,25.78897),(24.981245,25.497256),(24.981245,25.205439),(24.981245,24.913571),(24.981245,24.621728),(24.981245,24.329963),(24.981245,24.038146),(24.981245,23.746278),(24.981245,23.454435),(24.981245,23.16267),(24.981245,22.870801),(24.981245,22.578985),(24.981245,22.287168),(24.981245,21.995351),(24.981245,21.870811),(24.981141,21.746167),(24.981141,21.621575),(24.981141,21.497035),(24.981141,21.372443),(24.981038,21.247851),(24.980935,21.123311),(24.980935,20.998719),(24.980935,20.874128),(24.980831,20.749536),(24.980831,20.624944),(24.980831,20.500352),(24.980728,20.405319),(24.980625,20.310234),(24.980625,20.215098),(24.980521,20.120013),(24.980521,20.008391),(24.980521,20.002062),(24.979591,20.000512),(24.978661,19.999013),(24.977731,19.997463),(24.976801,19.995964),(24.974527,19.995654),(24.972356,19.995421),(24.970186,19.995189),(24.968016,19.994956),(24.867247,19.994956),(24.721312,19.995034),(24.599149,19.99506),(24.474609,19.995189),(24.34087,19.995292),(24.227906,19.995318),(24.142433,19.995421),(23.981306,19.995421),(23.981306,19.870597),(23.981306,19.745824),(23.981306,19.621026),(23.981306,19.496124),(23.857799,19.557179),(23.734292,19.618132),(23.610786,19.67911),(23.487279,19.740166),(23.363772,19.80117),(23.240162,19.862096),(23.116759,19.923152),(22.993149,19.984104),(22.869642,20.045108),(22.746136,20.106061),(22.622629,20.167091),(22.499122,20.228095),(22.375616,20.289124),(22.252109,20.350077),(22.128602,20.411055),(22.005096,20.472033),(21.881486,20.533063),(21.758082,20.59399),(21.634472,20.654968),(21.510966,20.716049),(21.387459,20.777027),(21.263952,20.838006),(21.140342,20.899036),(21.016939,20.959962),(20.893329,21.02094),(20.769926,21.08197),(20.646419,21.142897),(20.522809,21.203926),(20.399406,21.265008),(20.275796,21.325935),(20.152289,21.386913),(20.028782,21.447943),(19.905276,21.508869),(19.781665,21.569899),(19.658262,21.630877),(19.534652,21.691855),(19.411145,21.752885),(19.287639,21.813915),(19.185837,21.864177),(19.164132,21.874893),(19.040729,21.935871),(18.917119,21.99685),(18.793612,22.05788),(18.670105,22.118806),(18.546599,22.179836),(18.422989,22.240866),(18.299585,22.301844),(18.175975,22.362822),(18.052469,22.423852),(17.928962,22.484779),(17.805455,22.545757),(17.681949,22.606787),(17.558442,22.667765),(17.434832,22.728795),(17.311429,22.789799),(17.187922,22.850751),(17.064312,22.911755),(16.940909,22.972733),(16.817299,23.03366),(16.693792,23.094664),(16.570285,23.15572),(16.446779,23.216724),(16.323169,23.277728),(16.199765,23.338706),(16.076155,23.399632),(15.985101,23.44472),(15.964637,23.442214),(15.920196,23.422034),(15.862525,23.395937),(15.804854,23.369789),(15.747183,23.343641),(15.689616,23.317518),(15.631945,23.291344),(15.574274,23.265196),(15.516499,23.239048),(15.458829,23.212899),(15.401158,23.186751),(15.343487,23.160577),(15.285816,23.134455),(15.228145,23.108358),(15.170371,23.082261),(15.112907,23.056113),(15.055236,23.029965),(14.997461,23.003817),(14.979909,22.995664),(14.979271,22.995368),(14.979168,22.995316),(14.808016,22.908887),(14.63707,22.822536),(14.466021,22.736184),(14.294869,22.649833),(14.23172,22.617949),(14.216217,22.616295),(14.201644,22.62322),(14.156169,22.660737),(14.031009,22.763702),(13.906055,22.866616),(13.780895,22.969633),(13.655786,23.072624),(13.599562,23.119029),(13.482257,23.179672),(13.380661,23.202383),(13.292398,23.222098),(13.204134,23.241709),(13.115871,23.261449),(13.027607,23.281138),(12.939447,23.300827),(12.851184,23.32049),(12.763024,23.340179),(12.674657,23.359893),(12.586497,23.379582),(12.498337,23.399245),(12.409971,23.418882),(12.321811,23.438622),(12.233651,23.458285),(12.145284,23.477948),(12.057021,23.497611),(11.968861,23.517351),(11.89238,23.660056),(11.822306,23.790642),(11.71014,24.000137),(11.708618,24.00298),(11.636478,24.137649),(11.567128,24.26684),(11.541393,24.29751),(11.50863,24.313814),(11.46724,24.326339),(11.149996,24.422335),(10.911354,24.494501),(10.720668,24.552301),(10.699067,24.556125),(10.677363,24.553851),(10.566776,24.516463),(10.450194,24.476931),(10.410506,24.473288),(10.391799,24.47998),(10.260335,24.576641),(10.242248,24.595115),(10.229742,24.629945),(10.212172,24.722859),(10.193362,24.749937),(10.044534,24.829622),(10.032028,24.856339),(10.029961,24.995065),(10.025517,25.1364),(10.021383,25.26802),(10.007947,25.331426),(9.9695,25.395402),(9.816744,25.588465),(9.693961,25.743494),(9.541309,25.936351),(9.401162,26.113394),(9.377804,26.168946),(9.402506,26.216179),(9.416458,26.23225),(9.434752,26.271989),(9.468858,26.301031),(9.477643,26.315552),(9.481364,26.332606),(9.482604,26.352553),(9.835761,26.504223),(9.854571,26.524377),(9.896326,26.652793),(9.894052,26.67398),(9.88258,26.701782),(9.885474,26.736612),(9.910588,26.843117),(9.906661,26.857483),(9.890848,26.869576),(9.846096,26.891951),(9.835141,26.900995),(9.825529,26.92058),(9.806099,27.025122),(9.721349,27.291875),(9.721866,27.308463),(9.726517,27.32469),(9.743053,27.364119),(9.756283,27.42303),(9.770545,27.444217),(9.813643,27.486489),(9.821602,27.505687),(9.81168,27.526538),(9.797107,27.548914),(9.793903,27.569739),(9.818501,27.585733),(9.846613,27.599298),(9.863356,27.619246),(9.931274,27.818642),(9.934256,27.827398),(9.93591,27.866724),(9.789872,28.209442),(9.776953,28.267578),(9.827276,28.618647),(9.851264,28.785996),(9.848267,28.975726),(9.826149,29.128533),(9.746929,29.368428),(9.667709,29.608323),(9.54968,29.802316),(9.421729,29.968714),(9.369487,30.022794),(9.310005,30.084366),(9.286544,30.117129),(9.519708,30.228905),(9.743467,30.331328),(9.772922,30.338098),(9.845786,30.342283),(9.871314,30.355151),(9.995648,30.494522),(10.101171,30.641697),(10.192225,30.731252),(10.253927,30.841788),(10.26974,30.882147),(10.270153,30.915633),(10.245038,30.985707),(10.240594,31.021157),(10.246175,31.059552),(10.244832,31.078156),(10.213206,31.135362),(10.182717,31.240782),(10.108096,31.411831),(10.106235,31.429194),(10.116881,31.494409),(10.13259,31.517561),(10.196462,31.57859),(10.263952,31.680496),(10.315422,31.715843),(10.427766,31.714603),(10.482543,31.733103),(10.498873,31.744317),(10.513549,31.757029),(10.525538,31.772067),(10.542384,31.806638),(10.584552,31.84028),(10.597575,31.873508),(10.605946,31.953606),(10.628477,31.974122),(10.647288,31.971951),(10.665271,31.963218),(10.683564,31.957017),(10.703098,31.962185),(10.736585,31.985387),(10.772861,32.004508),(10.805624,32.032361),(10.845932,32.111788),(10.873217,32.136696),(11.182341,32.262223),(11.444035,32.36849),(11.513901,32.407997),(11.526085,32.417863),(11.546354,32.434275),(11.564131,32.465487),(11.560617,32.507578),(11.537363,32.543519),(11.47039,32.599277),(11.449926,32.637957),(11.449203,32.693018),(11.463775,32.798464),(11.456541,32.902101),(11.474421,32.969875),(11.474937,33.025841),(11.477418,33.041214),(11.506047,33.136376),(11.505112,33.181225),(11.505138,33.18122),(11.52589,33.176947),(11.560232,33.16592),(11.574962,33.157945),(11.587657,33.13231),(11.603038,33.121487),(11.635753,33.10871),(11.756602,33.092515),(11.793468,33.080756),(11.77296,33.099351),(11.707774,33.117174),(11.691091,33.135972),(11.739268,33.112982),(11.847016,33.082831),(12.08074,32.957831),(12.122406,32.921332),(12.160655,32.911689),(12.224864,32.875922),(12.303884,32.843085),(12.348318,32.832668),(12.729828,32.79857),(12.824392,32.804348),(13.074474,32.86636),(13.115489,32.882148),(13.147716,32.903998),(13.164724,32.912746),(13.187755,32.916246),(13.257091,32.918647),(13.351736,32.904242),(13.363129,32.899563),(13.373546,32.892035),(13.571462,32.802802),(13.616954,32.793402),(13.78712,32.799628),(13.830333,32.794094),(13.915294,32.772773),(13.991222,32.744574),(14.175548,32.717678),(14.185313,32.714504),(14.20338,32.700344),(14.209321,32.697211),(14.226329,32.690904),(14.262462,32.660305),(14.317149,32.630439),(14.404063,32.56684),(14.449392,32.525377),(14.469493,32.519029),(14.631602,32.495673),(14.647634,32.488267),(14.662934,32.476752),(14.706716,32.456285),(14.726817,32.450751),(14.911143,32.443915),(15.114757,32.412787),(15.160004,32.398668),(15.181977,32.395494),(15.195974,32.389797),(15.233246,32.347113),(15.269705,32.328843),(15.283051,32.317288),(15.292166,32.280911),(15.300792,32.26081),(15.322602,32.224189),(15.364513,32.174709),(15.370453,32.159084),(15.368663,32.138007),(15.35963,32.102118),(15.356863,32.061672),(15.355235,32.03852),(15.361095,31.965318),(15.370453,31.929389),(15.490408,31.664862),(15.514415,31.628363),(15.622813,31.49726),(15.687511,31.437812),(15.761485,31.388129),(16.044607,31.275539),(16.358653,31.227118),(16.728038,31.223822),(16.941905,31.183295),(17.16033,31.118801),(17.330577,31.086493),(17.365896,31.086615),(17.375011,31.083075),(17.382009,31.078762),(17.41863,31.062161),(17.447276,31.038723),(17.457774,31.033352),(17.466807,31.029975),(17.613943,30.992499),(17.691742,30.959662),(17.776052,30.935207),(17.848643,30.923651),(17.861664,30.913642),(17.889171,30.877916),(17.907481,30.863511),(17.926768,30.85517),(18.016856,30.837877),(18.175304,30.786078),(18.208344,30.769232),(18.235688,30.746812),(18.324229,30.656887),(18.56837,30.500067),(18.623302,30.445746),(18.653168,30.422309),(18.719249,30.391506),(18.760997,30.384223),(18.780528,30.375922),(18.925955,30.293362),(18.968028,30.277737),(19.056814,30.266181),(19.146007,30.264716),(19.239757,30.273993),(19.291677,30.287055)] +Saint Lucia [(-60.88679,14.010077),(-60.882965,13.980618),(-60.883046,13.954413),(-60.89509,13.877346),(-60.892201,13.833197),(-60.904612,13.784247),(-60.907297,13.77733),(-60.910308,13.776109),(-60.922841,13.772691),(-60.927113,13.769924),(-60.931752,13.760321),(-60.940785,13.722113),(-60.947011,13.716295),(-60.950103,13.714667),(-60.955068,13.714667),(-60.973378,13.739),(-61.048166,13.76732),(-61.064931,13.786933),(-61.068105,13.792182),(-61.074208,13.798733),(-61.078114,13.806301),(-61.071156,13.821967),(-61.070058,13.832099),(-61.071156,13.851793),(-61.072743,13.858222),(-61.075592,13.863267),(-61.078114,13.869615),(-61.078521,13.879788),(-61.075917,13.890367),(-61.066884,13.909003),(-61.064931,13.917304),(-61.059478,13.93122),(-61.035553,13.964911),(-61.026601,13.987372),(-61.002268,14.023139),(-60.99706,14.027167),(-60.987294,14.032538),(-60.98233,14.03734),(-60.97997,14.043524),(-60.976064,14.061184),(-60.972076,14.068101),(-60.947621,14.102281),(-60.933461,14.111884),(-60.914133,14.105699),(-60.918609,14.088446),(-60.911936,14.079495),(-60.901031,14.073391),(-60.892974,14.064683),(-60.890207,14.052232),(-60.88679,14.010077)] +Liechtenstein [(9.581203,47.05687),(9.560636,47.0524),(9.499554,47.059351),(9.477023,47.063898),(9.475886,47.073226),(9.487565,47.083949),(9.502861,47.094698),(9.51237,47.10803),(9.511853,47.129372),(9.503481,47.145392),(9.492629,47.15981),(9.484981,47.176346),(9.487358,47.210014),(9.504618,47.243732),(9.521155,47.262801),(9.53025,47.253654),(9.547096,47.243035),(9.540378,47.229108),(9.544823,47.220323),(9.554331,47.211615),(9.562909,47.19774),(9.561876,47.190609),(9.551954,47.175571),(9.552057,47.16689),(9.581823,47.154901),(9.605594,47.132266),(9.615723,47.106764),(9.608798,47.080771),(9.581203,47.05687)] +Lesotho [(28.980846,-28.909035),(28.995418,-28.908312),(29.018983,-28.913996),(29.040532,-28.922988),(29.049679,-28.933013),(29.049989,-28.95265),(29.055776,-28.965363),(29.069006,-28.973321),(29.091536,-28.978592),(29.125953,-28.992441),(29.208429,-29.068922),(29.241398,-29.077707),(29.281396,-29.078637),(29.311781,-29.089799),(29.316846,-29.147367),(29.32806,-29.159356),(29.343717,-29.169795),(29.357463,-29.183334),(29.365008,-29.200284),(29.375964,-29.235527),(29.403869,-29.268703),(29.413997,-29.290304),(29.414928,-29.296402),(29.413997,-29.311698),(29.415031,-29.318933),(29.419475,-29.325031),(29.432808,-29.336606),(29.435908,-29.342394),(29.432239,-29.356967),(29.413739,-29.380118),(29.411879,-29.395827),(29.41255,-29.410297),(29.4079,-29.421459),(29.399528,-29.430347),(29.388159,-29.437789),(29.361804,-29.447917),(29.332039,-29.455979),(29.303823,-29.466831),(29.282739,-29.485641),(29.275505,-29.506622),(29.275199,-29.528295),(29.275143,-29.532253),(29.282016,-29.556128),(29.296279,-29.571631),(29.279949,-29.581449),(29.278398,-29.597055),(29.278605,-29.613592),(29.26734,-29.626098),(29.190342,-29.646871),(29.1613,-29.666922),(29.145073,-29.691933),(29.132413,-29.720252),(29.113757,-29.750121),(29.113599,-29.752851),(29.11231,-29.775132),(29.106316,-29.801591),(29.105799,-29.825569),(29.120372,-29.843449),(29.132981,-29.852027),(29.135772,-29.859158),(29.135151,-29.868047),(29.137322,-29.882103),(29.148277,-29.901326),(29.150654,-29.911145),(29.144246,-29.919723),(29.132774,-29.924684),(29.106316,-29.931505),(29.09412,-29.936259),(29.015786,-29.978587),(28.976298,-29.999925),(28.860026,-30.066381),(28.795637,-30.089428),(28.729491,-30.101831),(28.639678,-30.131493),(28.605881,-30.131286),(28.557306,-30.114957),(28.540562,-30.113303),(28.524336,-30.116714),(28.480928,-30.139348),(28.456226,-30.146996),(28.407961,-30.140898),(28.383879,-30.144205),(28.364087,-30.159295),(28.355596,-30.17324),(28.338663,-30.201049),(28.323005,-30.214692),(28.313703,-30.22079),(28.3013,-30.2365),(28.294376,-30.242494),(28.284712,-30.246215),(28.255308,-30.249935),(28.237015,-30.256343),(28.216034,-30.267402),(28.200635,-30.282492),(28.198464,-30.301095),(28.20942,-30.314634),(28.224096,-30.320525),(28.235775,-30.329414),(28.238462,-30.352255),(28.231434,-30.373855),(28.218205,-30.387395),(28.133249,-30.445376),(28.125394,-30.457261),(28.127564,-30.465116),(28.139243,-30.482686),(28.142447,-30.492608),(28.141207,-30.500773),(28.124257,-30.547282),(28.120846,-30.553379),(28.112837,-30.562474),(28.104878,-30.568366),(28.096352,-30.572396),(28.088032,-30.577564),(28.081572,-30.586866),(28.075836,-30.60702),(28.076508,-30.623246),(28.078988,-30.639266),(28.078782,-30.658799),(28.054701,-30.649704),(27.936568,-30.640816),(27.915588,-30.634925),(27.895641,-30.626243),(27.886132,-30.619732),(27.878277,-30.612911),(27.869802,-30.607123),(27.858227,-30.603712),(27.849545,-30.604952),(27.830425,-30.613117),(27.8204,-30.614564),(27.778438,-30.609707),(27.743608,-30.600302),(27.713533,-30.583765),(27.659531,-30.530538),(27.598604,-30.490334),(27.5923,-30.481859),(27.590646,-30.472868),(27.590129,-30.463876),(27.586822,-30.454987),(27.562741,-30.425532),(27.536593,-30.403208),(27.533364,-30.393425),(27.529668,-30.382227),(27.493184,-30.367654),(27.477475,-30.356286),(27.466726,-30.341093),(27.459388,-30.316598),(27.452102,-30.310087),(27.43603,-30.30988),(27.414843,-30.317528),(27.407815,-30.318562),(27.397635,-30.317218),(27.377708,-30.312324),(27.377016,-30.312154),(27.366164,-30.311017),(27.367817,-30.295411),(27.345545,-30.241564),(27.343685,-30.212935),(27.365544,-30.165806),(27.38053,-30.152577),(27.381357,-30.142655),(27.371952,-30.135937),(27.356655,-30.132216),(27.334434,-30.132216),(27.322032,-30.1348),(27.312782,-30.131596),(27.300431,-30.114957),(27.293713,-30.100901),(27.280174,-30.053462),(27.265911,-30.033308),(27.217956,-29.998064),(27.198835,-29.978531),(27.17135,-29.921495),(27.088765,-29.750121),(27.088661,-29.750121),(27.088661,-29.750018),(27.080135,-29.735135),(27.050989,-29.696791),(27.037657,-29.686456),(27.024531,-29.680048),(27.010217,-29.669919),(27.002155,-29.666508),(27.014454,-29.641704),(27.014867,-29.625581),(27.022412,-29.616176),(27.032954,-29.616176),(27.042359,-29.628061),(27.049181,-29.620516),(27.056829,-29.604393),(27.062203,-29.600156),(27.072022,-29.599949),(27.075122,-29.605117),(27.077706,-29.611318),(27.085871,-29.614419),(27.090418,-29.611628),(27.122768,-29.581759),(27.128142,-29.578555),(27.156357,-29.568633),(27.18571,-29.565946),(27.195838,-29.562639),(27.213718,-29.554061),(27.24183,-29.548996),(27.263121,-29.541141),(27.295677,-29.524398),(27.300307,-29.52012),(27.302498,-29.518094),(27.309113,-29.509205),(27.314074,-29.498663),(27.316038,-29.487191),(27.321412,-29.483574),(27.344873,-29.484504),(27.350247,-29.480267),(27.354485,-29.462283),(27.365544,-29.446677),(27.407092,-29.406886),(27.413706,-29.402545),(27.417634,-29.396551),(27.41908,-29.383838),(27.416187,-29.375984),(27.410295,-29.369782),(27.406265,-29.363995),(27.408642,-29.35707),(27.417634,-29.350662),(27.426832,-29.347975),(27.436134,-29.343531),(27.445849,-29.331955),(27.448329,-29.322964),(27.450293,-29.299709),(27.45329,-29.291648),(27.469878,-29.282449),(27.510686,-29.270753),(27.518919,-29.268393),(27.528893,-29.260952),(27.527239,-29.249376),(27.520986,-29.226122),(27.521503,-29.21651),(27.528169,-29.207622),(27.538246,-29.202351),(27.548478,-29.19894),(27.555713,-29.195426),(27.61297,-29.133828),(27.635605,-29.096621),(27.634479,-29.091041),(27.630747,-29.07254),(27.637517,-29.065718),(27.64439,-29.07254),(27.649351,-29.070886),(27.658601,-29.066752),(27.664957,-29.065718),(27.656275,-29.055796),(27.64377,-29.044531),(27.63917,-29.035333),(27.655035,-29.031508),(27.667489,-29.026651),(27.679736,-29.015075),(27.699063,-28.990581),(27.702887,-28.989341),(27.707642,-28.990684),(27.711672,-28.990994),(27.713326,-28.98686),(27.713378,-28.979522),(27.714463,-28.976421),(27.716685,-28.973217),(27.71901,-28.967843),(27.720044,-28.959782),(27.720147,-28.945932),(27.728312,-28.937974),(27.744074,-28.93322),(27.754771,-28.925468),(27.747432,-28.908622),(27.861327,-28.915443),(27.887992,-28.904901),(27.897294,-28.889812),(27.903599,-28.873999),(27.91166,-28.86077),(27.926233,-28.853431),(27.945043,-28.855499),(27.956929,-28.86511),(27.966231,-28.875652),(27.977496,-28.880717),(28.004678,-28.880717),(28.013566,-28.878443),(28.014806,-28.872552),(28.014496,-28.8648),(28.018734,-28.856842),(28.056561,-28.812504),(28.100899,-28.770853),(28.104103,-28.765065),(28.131698,-28.729305),(28.145031,-28.714732),(28.154953,-28.70233),(28.164048,-28.70543),(28.218116,-28.700652),(28.23071,-28.699539),(28.250864,-28.707394),(28.273912,-28.709564),(28.297063,-28.706877),(28.317217,-28.700056),(28.33644,-28.684656),(28.363519,-28.643625),(28.382949,-28.626469),(28.403413,-28.618924),(28.542629,-28.610759),(28.562473,-28.606522),(28.619421,-28.572415),(28.634303,-28.570761),(28.653217,-28.58306),(28.66655,-28.59722),(28.675645,-28.613756),(28.681742,-28.633393),(28.688977,-28.674424),(28.699519,-28.68848),(28.743857,-28.689514),(28.758637,-28.700469),(28.787679,-28.748735),(28.795947,-28.754316),(28.82716,-28.759277),(28.827937,-28.759534),(28.843696,-28.764755),(28.858786,-28.772816),(28.859014,-28.77299),(28.872842,-28.783565),(28.885658,-28.797208),(28.925862,-28.859529),(28.980846,-28.909035)] +Luxembourg [(6.038628,50.148413),(6.061366,50.150738),(6.070048,50.154201),(6.076456,50.158645),(6.084207,50.159317),(6.097126,50.151307),(6.101467,50.142108),(6.10033,50.132652),(6.102604,50.124745),(6.117487,50.120456),(6.110562,50.105987),(6.105704,50.092241),(6.0994,50.064129),(6.101157,50.063405),(6.107151,50.063767),(6.109425,50.06356),(6.105704,50.060305),(6.096403,50.048626),(6.107255,50.044595),(6.113766,50.036378),(6.119967,50.015501),(6.117177,50.004391),(6.121517,49.996278),(6.138777,49.979328),(6.15397,49.951216),(6.161928,49.942482),(6.172677,49.956228),(6.189834,49.938297),(6.2227,49.887137),(6.238306,49.876905),(6.272206,49.867138),(6.286779,49.86104),(6.291533,49.855563),(6.295568,49.848616),(6.301558,49.838303),(6.302592,49.834944),(6.310653,49.834479),(6.325639,49.84037),(6.334424,49.840215),(6.351374,49.83329),(6.380106,49.815565),(6.397056,49.808589),(6.414523,49.805488),(6.462478,49.805488),(6.493278,49.799597),(6.497205,49.799442),(6.502579,49.79567),(6.502269,49.795876),(6.500099,49.794481),(6.499996,49.786058),(6.498549,49.785386),(6.493794,49.76363),(6.493691,49.756551),(6.496378,49.752106),(6.495965,49.748231),(6.48625,49.742701),(6.485526,49.733761),(6.4878,49.725596),(6.492554,49.718413),(6.499686,49.712212),(6.491004,49.714434),(6.486353,49.712936),(6.482942,49.708078),(6.489764,49.702962),(6.491727,49.700792),(6.418243,49.669889),(6.402741,49.655782),(6.404498,49.652474),(6.410905,49.649632),(6.417003,49.645756),(6.417933,49.639142),(6.415246,49.634646),(6.407701,49.627411),(6.397573,49.613872),(6.371528,49.592995),(6.356438,49.577595),(6.350754,49.566588),(6.350754,49.53491),(6.348809,49.52529),(6.342279,49.493001),(6.34538,49.462718),(6.345307,49.455349),(6.325329,49.456724),(6.298768,49.466232),(6.251117,49.48946),(6.249365,49.490314),(6.221976,49.497445),(6.193554,49.499409),(6.13609,49.495275),(6.142601,49.486076),(6.114386,49.484371),(6.107255,49.48251),(6.101984,49.476413),(6.102087,49.470005),(6.099813,49.46463),(6.087721,49.461633),(6.078006,49.452021),(6.06333,49.448456),(5.978064,49.445097),(5.9607,49.441324),(5.946954,49.469953),(5.928144,49.482304),(5.864686,49.492174),(5.839571,49.499874),(5.822104,49.510519),(5.790685,49.537753),(5.814456,49.545142),(5.8374,49.56111),(5.846909,49.576717),(5.829959,49.582763),(5.841224,49.589687),(5.849183,49.599713),(5.861998,49.62276),(5.869957,49.628858),(5.879878,49.634853),(5.885459,49.643534),(5.879878,49.657745),(5.872644,49.661724),(5.852593,49.663223),(5.845772,49.666582),(5.842258,49.67392),(5.843705,49.677486),(5.846702,49.68069),(5.847219,49.687201),(5.848769,49.69273),(5.853627,49.696296),(5.857347,49.70043),(5.856403,49.705807),(5.856004,49.708078),(5.850216,49.714021),(5.843705,49.714331),(5.837297,49.712936),(5.831716,49.713607),(5.825721,49.715778),(5.811769,49.718517),(5.805361,49.721876),(5.805774,49.724201),(5.803604,49.738154),(5.802054,49.742856),(5.778489,49.773035),(5.771875,49.779392),(5.759162,49.784456),(5.748414,49.785644),(5.738285,49.788848),(5.727743,49.800217),(5.721645,49.812775),(5.720198,49.824712),(5.724642,49.834272),(5.736528,49.83944),(5.726916,49.845227),(5.728673,49.848845),(5.730533,49.85117),(5.73198,49.854167),(5.731877,49.859955),(5.758129,49.858198),(5.748827,49.867293),(5.726709,49.878197),(5.714927,49.881866),(5.718958,49.891374),(5.753581,49.930183),(5.757612,49.93654),(5.759989,49.941604),(5.76371,49.946823),(5.771565,49.953541),(5.778593,49.95659),(5.793372,49.958812),(5.80164,49.963825),(5.812596,49.977571),(5.808565,49.9831),(5.801537,49.988681),(5.802674,50.002479),(5.808875,50.007801),(5.829132,50.013589),(5.836367,50.01855),(5.838847,50.026457),(5.838847,50.035862),(5.83771,50.047437),(5.843705,50.053638),(5.858278,50.061803),(5.864375,50.067488),(5.866856,50.074309),(5.86882,50.090535),(5.872437,50.096892),(5.888973,50.106607),(5.92618,50.118337),(5.941063,50.128311),(5.949021,50.142677),(5.953156,50.156371),(5.96163,50.165621),(5.982921,50.167068),(5.998321,50.174975),(6.004729,50.170479),(6.008036,50.160919),(6.014134,50.153581),(6.022967,50.15082),(6.027363,50.149447),(6.038628,50.148413)] +Latvia [(25.333678,58.031808),(25.339879,58.032583),(25.34639,58.034443),(25.356519,58.034753),(25.39662,58.023178),(25.457185,57.984059),(25.496614,57.970597),(25.53196,57.966437),(25.541676,57.96282),(25.550254,57.955327),(25.551816,57.94986),(25.552114,57.948815),(25.552941,57.942175),(25.558315,57.934243),(25.579399,57.919463),(25.601517,57.912203),(25.625081,57.910291),(25.650093,57.911428),(25.7038,57.921327),(25.714998,57.923391),(25.730295,57.923132),(25.745384,57.90918),(25.754686,57.888897),(25.767502,57.868924),(25.792616,57.855979),(26.002733,57.845979),(26.014515,57.842827),(26.021956,57.837944),(26.025264,57.828409),(26.02175,57.822415),(26.016065,57.816472),(26.013068,57.8073),(26.016065,57.786681),(26.024747,57.774356),(26.135231,57.724772),(26.168511,57.70405),(26.263182,57.620825),(26.297185,57.599147),(26.33863,57.583153),(26.351325,57.580203),(26.430678,57.561764),(26.430924,57.561707),(26.447564,57.552922),(26.48105,57.527136),(26.49955,57.515819),(26.515053,57.517266),(26.528696,57.52388),(26.542235,57.528169),(26.551743,57.526826),(26.570243,57.519591),(26.579855,57.517834),(26.590087,57.520366),(26.594841,57.52649),(26.598355,57.533854),(26.604763,57.540313),(26.634322,57.554421),(26.666982,57.56486),(26.700468,57.570751),(26.778086,57.568477),(26.795759,57.571319),(26.816327,57.581189),(26.828315,57.595039),(26.838547,57.60987),(26.854154,57.622556),(26.872654,57.627181),(26.884953,57.622091),(26.896115,57.614986),(26.911618,57.613435),(26.929188,57.615554),(26.946551,57.615347),(26.981071,57.608991),(27.004635,57.598914),(27.041326,57.568451),(27.061531,57.555661),(27.085354,57.549434),(27.165659,57.546695),(27.319965,57.516129),(27.352935,57.527601),(27.528169,57.528479),(27.522795,57.492022),(27.525637,57.46838),(27.514837,57.44789),(27.511323,57.430397),(27.536283,57.415695),(27.640824,57.389005),(27.694409,57.356375),(27.707952,57.348129),(27.809134,57.313945),(27.827738,57.304953),(27.840295,57.290639),(27.846134,57.267307),(27.841174,57.211212),(27.833939,57.18049),(27.824327,57.15907),(27.794045,57.143387),(27.700975,57.118789),(27.682217,57.103699),(27.696066,57.085457),(27.722059,57.07799),(27.745469,57.067991),(27.75074,57.042359),(27.745779,57.031455),(27.729552,57.009855),(27.723351,56.998796),(27.720767,56.988667),(27.718545,56.96872),(27.715135,56.957403),(27.648731,56.879268),(27.62837,56.844154),(27.644379,56.841772),(27.661185,56.839271),(27.744074,56.864799),(27.786397,56.871258),(27.830838,56.864282),(27.852232,56.854309),(27.891506,56.829711),(27.913521,56.82015),(27.918895,56.805888),(27.900085,56.783047),(27.876107,56.759431),(27.865461,56.742687),(27.882618,56.725479),(27.98101,56.687006),(27.991345,56.669979),(27.992172,56.624994),(27.997443,56.603833),(28.010982,56.587555),(28.028656,56.576754),(28.108651,56.55518),(28.126117,56.547764),(28.132215,56.535982),(28.125239,56.527145),(28.099246,56.513399),(28.093251,56.501591),(28.096455,56.491953),(28.104517,56.483065),(28.156503,56.44622),(28.164255,56.437952),(28.167872,56.427125),(28.164255,56.392812),(28.167148,56.369868),(28.174435,56.349637),(28.214794,56.281372),(28.217275,56.270727),(28.215311,56.255999),(28.20942,56.248144),(28.201358,56.241684),(28.193142,56.231039),(28.184408,56.207526),(28.178621,56.18391),(28.169112,56.161741),(28.148907,56.142414),(28.110976,56.156806),(28.06824,56.147582),(28.0511,56.140666),(28.023798,56.12965),(27.98101,56.118023),(27.939669,56.113062),(27.92706,56.109367),(27.911453,56.100246),(27.901532,56.089342),(27.892747,56.077095),(27.880654,56.063866),(27.812545,56.034514),(27.781229,56.016375),(27.776991,55.992397),(27.744435,55.959738),(27.64501,55.922841),(27.617156,55.878554),(27.610128,55.83096),(27.601498,55.809618),(27.592713,55.794244),(27.564601,55.792229),(27.438821,55.79874),(27.405851,55.804347),(27.374587,55.814837),(27.349627,55.831219),(27.329163,55.817576),(27.282448,55.791867),(27.263018,55.787216),(27.235526,55.795846),(27.173204,55.825741),(27.151448,55.832459),(27.110779,55.836283),(26.981071,55.826878),(26.979426,55.826291),(26.957817,55.818584),(26.900146,55.778715),(26.842785,55.719339),(26.822838,55.70611),(26.743049,55.682856),(26.720105,55.681874),(26.666878,55.693966),(26.64011,55.695568),(26.615615,55.687971),(26.594531,55.666991),(26.537481,55.669523),(26.48105,55.678308),(26.34235,55.716342),(26.279718,55.743239),(26.226905,55.796931),(26.203754,55.827446),(26.178639,55.84959),(26.079421,55.89814),(26.017616,55.937362),(25.961251,55.958459),(25.871268,55.992139),(25.795496,56.04013),(25.766261,56.058647),(25.708797,56.081048),(25.696602,56.087895),(25.687817,56.098334),(25.669213,56.132027),(25.661875,56.140812),(25.649679,56.143809),(25.590045,56.141122),(25.572888,56.143034),(25.522762,56.156573),(25.454808,56.149649),(25.349086,56.159876),(25.109196,56.183083),(25.073125,56.197811),(25.04398,56.22799),(25.026203,56.260185),(25.016488,56.2699),(24.983208,56.290493),(24.973183,56.300751),(24.962124,56.319277),(24.936286,56.379919),(24.910138,56.421958),(24.892258,56.438727),(24.870967,56.442602),(24.857118,56.435368),(24.853397,56.424722),(24.852157,56.413664),(24.845852,56.40493),(24.837688,56.403948),(24.81557,56.408806),(24.805545,56.408548),(24.71504,56.386253),(24.678628,56.377283),(24.639147,56.359972),(24.557911,56.29858),(24.538998,56.287625),(24.481327,56.26884),(24.44722,56.260521),(24.414354,56.265507),(24.350792,56.291552),(24.318546,56.29858),(24.287644,56.295531),(24.166824,56.260314),(24.140422,56.2576),(24.138919,56.257446),(24.108637,56.260701),(24.07515,56.271192),(23.981306,56.312404),(23.871107,56.332072),(23.858109,56.334392),(23.825036,56.334909),(23.7561,56.325969),(23.724991,56.328811),(23.723647,56.332118),(23.724784,56.338397),(23.723544,56.345812),(23.715069,56.352737),(23.706697,56.354442),(23.679412,56.351859),(23.609856,56.353822),(23.577713,56.348603),(23.517561,56.328656),(23.481595,56.330103),(23.37804,56.35463),(23.310649,56.370591),(23.288428,56.373046),(23.17257,56.357956),(23.154173,56.351394),(23.162854,56.341007),(23.112728,56.310879),(23.062292,56.304161),(23.016714,56.32385),(22.981367,56.373175),(22.956976,56.401985),(22.924523,56.412113),(22.888969,56.408444),(22.859603,56.39707),(22.822927,56.382864),(22.681437,56.350308),(22.666347,56.349094),(22.649087,56.353099),(22.604749,56.379092),(22.578222,56.387003),(22.57581,56.387722),(22.511731,56.395964),(22.214902,56.390383),(22.195265,56.394853),(22.158161,56.410253),(22.139144,56.415705),(22.094082,56.41741),(21.983173,56.388353),(21.965322,56.383677),(21.684495,56.310104),(21.596025,56.307882),(21.558921,56.297288),(21.419292,56.237395),(21.403685,56.234476),(21.358313,56.233235),(21.327618,56.224373),(21.290411,56.206932),(21.254857,56.185202),(21.229639,56.163188),(21.212586,56.130994),(21.205351,56.103424),(21.190365,56.084459),(21.150161,56.07818),(21.091353,56.077896),(21.053398,56.072604),(21.053396,56.072618),(21.053396,56.072943),(21.052582,56.077541),(21.042735,56.114569),(21.028819,56.147406),(20.98406,56.210435),(20.973399,56.232001),(20.969086,56.253485),(20.971039,56.25967),(20.980235,56.279283),(20.982677,56.290758),(20.982677,56.301459),(20.981456,56.310492),(20.970958,56.34984),(20.968598,56.369818),(20.970551,56.388983),(20.979015,56.404364),(20.992524,56.417914),(20.998383,56.425686),(21.003184,56.434801),(21.005382,56.445868),(21.005544,56.465522),(21.01059,56.475735),(21.003266,56.49258),(21.000499,56.510647),(21.006358,56.519965),(21.024262,56.510484),(21.030528,56.500149),(21.035818,56.473944),(21.044688,56.462104),(21.030528,56.448432),(21.037446,56.444648),(21.044688,56.442206),(21.044688,56.427965),(21.044688,56.421129),(21.030528,56.408108),(21.053396,56.38996),(21.061778,56.38703),(21.07016,56.38935),(21.077159,56.395209),(21.079763,56.403062),(21.068614,56.424058),(21.066661,56.44184),(21.068126,56.458808),(21.072113,56.46955),(21.067638,56.476386),(21.063243,56.493964),(21.05836,56.503648),(21.052908,56.511054),(21.048106,56.515611),(21.030528,56.524156),(21.019542,56.527086),(21.010509,56.527777),(21.003103,56.530015),(20.996349,56.537828),(20.992198,56.548896),(20.995128,56.553371),(21.000336,56.557685),(21.010509,56.5987),(21.046153,56.656195),(21.05836,56.688666),(21.065196,56.774319),(21.061209,56.794135),(21.05421,56.811957),(21.052745,56.828925),(21.065196,56.84634),(21.147227,56.87641),(21.156749,56.885565),(21.222911,56.907701),(21.240977,56.91828),(21.277599,56.949123),(21.287771,56.955512),(21.300141,56.95954),(21.328217,56.973619),(21.360118,56.989691),(21.382498,57.008938),(21.401622,57.037502),(21.413585,57.073188),(21.414806,57.113837),(21.411876,57.124457),(21.401052,57.151313),(21.403168,57.162055),(21.412608,57.176215),(21.414806,57.184882),(21.412608,57.250067),(21.414806,57.270901),(21.421397,57.288723),(21.433604,57.306464),(21.449474,57.320014),(21.466563,57.325507),(21.480235,57.333645),(21.517751,57.3876),(21.589122,57.438788),(21.699229,57.555325),(21.729666,57.573798),(21.771658,57.586127),(21.955903,57.592963),(21.999278,57.600328),(22.194509,57.65762),(22.483735,57.742499),(22.526134,57.749661),(22.549083,57.750637),(22.560313,57.752753),(22.572602,57.756822),(22.585785,57.759711),(22.600271,57.758043),(22.610362,57.754625),(22.610118,57.753485),(22.604991,57.751044),(22.600271,57.743842),(22.590831,57.706122),(22.587169,57.664984),(22.590994,57.644761),(22.600759,57.630113),(22.656098,57.585761),(22.878917,57.481269),(22.950369,57.432807),(23.031423,57.394477),(23.083344,57.378241),(23.117524,57.373928),(23.130219,57.370795),(23.133149,57.362982),(23.133556,57.353095),(23.138031,57.343166),(23.160167,57.31859),(23.164561,57.313666),(23.174571,57.297349),(23.182384,57.277655),(23.190196,57.242499),(23.198741,57.224189),(23.213552,57.216254),(23.222423,57.207017),(23.250499,57.114447),(23.260916,57.098944),(23.274913,57.092719),(23.292654,57.088935),(23.340099,57.05858),(23.422862,57.040229),(23.508556,57.031236),(23.520844,57.02558),(23.569672,56.986274),(23.578624,56.982082),(23.585134,56.979071),(23.632009,56.970933),(23.69516,56.967271),(23.866059,56.99018),(23.926931,57.006334),(23.952891,57.013251),(23.991222,57.031236),(24.009288,57.0456),(24.021983,57.059027),(24.035655,57.068996),(24.074555,57.074937),(24.113048,57.088731),(24.210785,57.123725),(24.219249,57.138007),(24.238617,57.149563),(24.261241,57.157457),(24.278575,57.161038),(24.278819,57.17178),(24.288748,57.1789),(24.312673,57.1883),(24.379161,57.230211),(24.400645,57.258124),(24.408865,57.298163),(24.405528,57.344468),(24.381602,57.469428),(24.379893,57.506537),(24.375173,57.528551),(24.364513,57.538398),(24.358572,57.55036),(24.362966,57.576606),(24.374766,57.613471),(24.363048,57.672268),(24.357677,57.685492),(24.313731,57.725043),(24.299571,57.743842),(24.299571,57.761908),(24.288829,57.808905),(24.285981,57.832587),(24.288829,57.845282),(24.296153,57.857367),(24.306159,57.868186),(24.349655,57.85864),(24.378181,57.859441),(24.401539,57.866469),(24.412701,57.876572),(24.429134,57.900343),(24.441639,57.908094),(24.481327,57.919308),(24.53383,57.945069),(24.553881,57.94742),(24.624057,57.943958),(24.684829,57.947575),(24.700435,57.95419),(24.718832,57.981837),(24.733508,57.992198),(24.764721,57.987909),(24.790765,57.978994),(24.81681,57.976669),(24.827417,57.981877),(24.848333,57.992146),(24.872518,58.00044),(24.949309,58.006435),(24.974837,58.01584),(25.034857,58.048574),(25.048838,58.056199),(25.070645,58.063615),(25.094209,58.067309),(25.141028,58.068007),(25.16697,58.058731),(25.180716,58.038164),(25.189914,58.013514),(25.20211,57.991965),(25.216786,57.985376),(25.232496,57.985376),(25.264535,57.994187),(25.284379,58.00814),(25.277557,58.024108),(25.249549,58.051186),(25.250686,58.068498),(25.263501,58.075138),(25.281692,58.073407),(25.299572,58.065604),(25.306393,58.058731),(25.318278,58.040619),(25.324583,58.034753),(25.333678,58.031808)] +Saint Martin [(-63.017569,18.033391),(-63.085886,18.058511),(-63.107004,18.062109),(-63.107004,18.06212),(-63.107004,18.066962),(-63.115305,18.065375),(-63.130523,18.060248),(-63.134348,18.059475),(-63.14387,18.064683),(-63.14684,18.070299),(-63.143625,18.074164),(-63.113759,18.074652),(-63.096099,18.078803),(-63.083811,18.087958),(-63.079091,18.104193),(-63.070872,18.110012),(-63.052113,18.116848),(-63.031402,18.121894),(-63.017649,18.122138),(-63.017934,18.100531),(-63.011098,18.070746),(-63.010732,18.040839),(-63.017568,18.033393),(-63.017569,18.033391)] +Morocco [(-5.40549,35.926519),(-5.399322,35.924465),(-5.398859,35.924504),(-5.389718,35.902047),(-5.378401,35.881687),(-5.362898,35.863807),(-5.340726,35.847366),(-5.340728,35.847357),(-5.34081,35.847113),(-5.345693,35.832587),(-5.342356,35.807685),(-5.327382,35.755845),(-5.322621,35.713772),(-5.318756,35.701728),(-5.312652,35.69245),(-5.304799,35.685207),(-5.298899,35.684312),(-5.28189,35.68651),(-5.276763,35.685207),(-5.27538,35.679389),(-5.277455,35.664211),(-5.276763,35.657904),(-5.259999,35.604722),(-5.24942,35.584621),(-5.228424,35.568508),(-5.186187,35.543769),(-5.150787,35.516099),(-5.066274,35.417467),(-5.050852,35.407538),(-5.026357,35.403998),(-5.008209,35.399237),(-4.988515,35.387885),(-4.954701,35.362454),(-4.926503,35.330308),(-4.909332,35.314521),(-4.86498,35.301988),(-4.810618,35.259996),(-4.770497,35.241034),(-4.628285,35.20539),(-4.495432,35.180772),(-4.494252,35.180569),(-4.376047,35.151842),(-4.347646,35.150133),(-4.327056,35.154771),(-4.269399,35.184882),(-4.130605,35.204779),(-4.046702,35.235907),(-3.961293,35.250434),(-3.919342,35.266832),(-3.909413,35.255276),(-3.91253,35.250918),(-3.924673,35.247556),(-3.929529,35.24382),(-3.925046,35.237842),(-3.918467,35.230893),(-3.903901,35.214173),(-3.889257,35.208407),(-3.86856,35.207343),(-3.840891,35.20539),(-3.821929,35.206773),(-3.787465,35.209215),(-3.764801,35.218451),(-3.755523,35.236396),(-3.747141,35.263658),(-3.726226,35.282294),(-3.699574,35.290839),(-3.673573,35.287909),(-3.654205,35.277004),(-3.612213,35.245347),(-3.590972,35.233303),(-3.516021,35.223538),(-3.489329,35.211005),(-3.477406,35.210679),(-3.464752,35.211819),(-3.453847,35.211615),(-3.44286,35.208645),(-3.423248,35.200832),(-3.412913,35.197943),(-3.36205,35.194525),(-3.319244,35.201809),(-3.195139,35.238227),(-3.17394,35.249091),(-3.128651,35.28856),(-3.118031,35.294135),(-3.109731,35.293647),(-3.100413,35.289537),(-3.08967,35.286282),(-3.077056,35.287909),(-3.0454,35.314439),(-3.006256,35.393541),(-2.980824,35.424506),(-2.982818,35.435004),(-2.977935,35.443305),(-2.968577,35.446438),(-2.956654,35.44123),(-2.952219,35.433051),(-2.95759,35.415839),(-2.952952,35.403998),(-2.963043,35.373481),(-2.949778,35.334947),(-2.94811,35.330268),(-2.947825,35.329779),(-2.947818,35.329768),(-2.96692,35.313866),(-2.963768,35.286219),(-2.943046,35.267874),(-2.912907,35.276916),(-2.899037,35.259996),(-2.907948,35.250963),(-2.912343,35.235785),(-2.913319,35.219062),(-2.90982,35.186713),(-2.905141,35.17357),(-2.896392,35.161811),(-2.881988,35.146715),(-2.856068,35.131293),(-2.820953,35.121894),(-2.748118,35.115953),(-2.791819,35.145168),(-2.868276,35.217353),(-2.883168,35.238511),(-2.878407,35.246324),(-2.864858,35.237616),(-2.816396,35.194525),(-2.808665,35.183783),(-2.761098,35.143256),(-2.718332,35.122992),(-2.666737,35.108222),(-2.563222,35.096137),(-2.517486,35.099351),(-2.493316,35.104397),(-2.477203,35.112616),(-2.456125,35.132473),(-2.437123,35.145087),(-2.416737,35.149156),(-2.321441,35.118842),(-2.299062,35.115953),(-2.275258,35.109565),(-2.248687,35.097235),(-2.222564,35.089301),(-2.221126,35.049955),(-2.211669,35.023445),(-2.193789,35.003601),(-2.1633,34.994041),(-2.126041,34.971923),(-2.094932,34.947687),(-2.061239,34.929704),(-2.01628,34.926241),(-2.00362,34.91818),(-1.999692,34.906346),(-1.9984,34.892703),(-1.993491,34.878854),(-1.979745,34.865263),(-1.926725,34.838081),(-1.892825,34.811675),(-1.787716,34.756691),(-1.769526,34.741343),(-1.773143,34.734057),(-1.786114,34.72584),(-1.810505,34.680727),(-1.862957,34.613599),(-1.871121,34.596649),(-1.750664,34.494175),(-1.714232,34.485054),(-1.702966,34.479679),(-1.809626,34.372451),(-1.771334,34.334701),(-1.746064,34.290311),(-1.674751,34.105956),(-1.672085,34.092021),(-1.669635,34.079213),(-1.672115,34.059188),(-1.718728,33.898087),(-1.722087,33.851165),(-1.713043,33.801995),(-1.702501,33.772823),(-1.703225,33.761816),(-1.710821,33.747063),(-1.72095,33.736366),(-1.732577,33.727865),(-1.74224,33.717762),(-1.746788,33.702388),(-1.740742,33.686601),(-1.725394,33.677713),(-1.690719,33.667326),(-1.673252,33.6565),(-1.6624,33.644692),(-1.617338,33.554439),(-1.612739,33.521469),(-1.6254,33.494236),(-1.640386,33.475529),(-1.659145,33.41977),(-1.672839,33.394604),(-1.683226,33.36923),(-1.683381,33.270839),(-1.674234,33.237972),(-1.623591,33.196605),(-1.605608,33.168028),(-1.592069,33.136609),(-1.571398,33.111985),(-1.545508,33.091831),(-1.516363,33.073977),(-1.499516,33.060205),(-1.493057,33.039483),(-1.493367,33.016151),(-1.496519,32.994292),(-1.498844,32.984008),(-1.502978,32.974629),(-1.50887,32.966309),(-1.516363,32.959488),(-1.558789,32.93365),(-1.423345,32.742395),(-1.390531,32.718779),(-1.32702,32.69891),(-1.047502,32.517009),(-1.031999,32.4944),(-1.08998,32.439442),(-1.123157,32.417945),(-1.16026,32.404948),(-1.201705,32.399858),(-1.217983,32.392623),(-1.234158,32.374614),(-1.244131,32.356915),(-1.257515,32.320845),(-1.273423,32.229442),(-1.275499,32.217518),(-1.275189,32.209069),(-1.276636,32.200852),(-1.282992,32.190181),(-1.289193,32.184858),(-1.305678,32.173722),(-1.309554,32.167443),(-1.305161,32.151165),(-1.288986,32.150907),(-1.251831,32.163516),(-1.232711,32.163723),(-1.211833,32.158348),(-1.195607,32.146049),(-1.190594,32.125224),(-1.210335,32.08967),(-1.249557,32.08166),(-1.324953,32.084709),(-1.333428,32.085071),(-1.357406,32.086053),(-1.395388,32.087552),(-1.445618,32.089515),(-1.506337,32.091944),(-1.575791,32.094735),(-1.652375,32.097835),(-1.734334,32.101091),(-1.819962,32.104501),(-1.907553,32.107964),(-1.995455,32.111478),(-2.081858,32.11494),(-2.165109,32.118299),(-2.243398,32.1214),(-2.315177,32.124294),(-2.378636,32.126774),(-2.516147,32.1322),(-2.695567,32.08967),(-2.881189,32.076286),(-2.938731,32.048639),(-2.869769,31.89243),(-2.827836,31.794586),(-3.002556,31.77362),(-3.21921,31.717709),(-3.511564,31.672745),(-3.548745,31.669954),(-3.591378,31.678274),(-3.659507,31.647821),(-3.673484,31.389234),(-3.747467,31.385166),(-3.802502,31.350646),(-3.815189,31.337158),(-3.81922,31.318865),(-3.812915,31.243365),(-3.814982,31.220524),(-3.836169,31.189777),(-3.842836,31.170192),(-3.839322,31.152828),(-3.827462,31.143837),(-3.810796,31.142906),(-3.792968,31.149676),(-3.763719,31.173447),(-3.748862,31.180217),(-3.731421,31.176341),(-3.717107,31.163267),(-3.689718,31.125595),(-3.671942,31.110867),(-3.635872,31.095674),(-3.624141,31.086527),(-3.614529,31.068027),(-3.610085,31.050302),(-3.608741,31.030872),(-3.554674,30.955927),(-3.610585,30.879049),(-3.659507,30.837116),(-3.652518,30.774217),(-3.645529,30.711317),(-3.834228,30.627452),(-4.001959,30.592507),(-4.155714,30.585519),(-4.274524,30.557563),(-4.372368,30.508641),(-4.484189,30.382842),(-4.623966,30.284999),(-4.770731,30.229088),(-4.875564,30.180166),(-4.95943,30.124256),(-5.071251,30.04039),(-5.176083,29.97749),(-5.273927,29.886635),(-5.343815,29.767825),(-5.43467,29.642026),(-5.539503,29.523216),(-5.637346,29.495261),(-5.721212,29.523216),(-5.756156,29.614071),(-5.881955,29.600093),(-6.000765,29.579127),(-6.126564,29.579127),(-6.27333,29.579127),(-6.413107,29.565149),(-6.559872,29.530205),(-6.699649,29.516227),(-6.783515,29.446339),(-6.958236,29.509238),(-7.070057,29.516227),(-7.146934,29.509238),(-7.258755,29.467305),(-7.34961,29.383439),(-7.463286,29.389137),(-7.48406,29.382445),(-7.506126,29.380223),(-7.528502,29.380947),(-7.572686,29.387613),(-7.619453,29.389422),(-7.653611,29.376193),(-7.714667,29.321829),(-7.729989,29.311158),(-7.777996,29.289324),(-7.839129,29.239043),(-7.945014,29.176231),(-8.036326,29.099853),(-8.069658,29.079337),(-8.182312,29.035541),(-8.250474,28.994769),(-8.316774,28.939062),(-8.333259,28.93038),(-8.368451,28.916531),(-8.383489,28.905782),(-8.417802,28.852349),(-8.430411,28.841006),(-8.475835,28.818759),(-8.520819,28.787081),(-8.648847,28.725948),(-8.667606,28.711685),(-8.678768,28.692823),(-8.682385,28.6659),(-8.682385,28.620218),(-8.682385,28.560273),(-8.682385,28.500406),(-8.682385,28.440487),(-8.682385,28.380543),(-8.682385,28.320624),(-8.682385,28.260679),(-8.682385,28.20076),(-8.682385,28.140893),(-8.682385,28.080949),(-8.682385,28.02103),(-8.682385,27.961085),(-8.682385,27.90114),(-8.682385,27.841222),(-8.682385,27.781354),(-8.682385,27.721436),(-8.682385,27.661439),(-8.752562,27.661439),(-8.816537,27.661465),(-8.817034,27.661465),(-8.817035,27.661464),(-8.818449,27.659398),(-8.81292,27.613354),(-8.783619,27.530362),(-8.773387,27.46003),(-8.788012,27.416054),(-8.801706,27.360424),(-8.795841,27.307688),(-8.773387,27.250069),(-8.752872,27.190486),(-8.752872,27.150463),(-8.793903,27.12018),(-8.888109,27.103566),(-9.000919,27.089924),(-9.083446,27.089924),(-9.207469,27.099691),(-9.284622,27.097727),(-9.352008,27.097727),(-9.412056,27.08796),(-9.486315,27.049875),(-9.568843,26.990292),(-9.672351,26.910219),(-9.734362,26.860429),(-9.816864,26.84968),(-9.899365,26.84968),(-9.979929,26.889729),(-10.031709,26.910219),(-10.065867,26.908281),(-10.122039,26.879962),(-10.188443,26.860429),(-10.250455,26.860429),(-10.353963,26.900478),(-10.477986,26.960035),(-10.550282,26.990292),(-10.653273,27.000058),(-10.756781,27.019592),(-10.829076,27.009825),(-10.921835,27.009825),(-11.045859,26.969802),(-11.149366,26.940501),(-11.262641,26.910219),(-11.391574,26.882908),(-11.36031,26.793043),(-11.315868,26.744208),(-11.315868,26.683644),(-11.3369,26.632897),(-11.398912,26.583081),(-11.469709,26.519597),(-11.510688,26.469807),(-11.55221,26.400457),(-11.582983,26.360408),(-11.63621,26.294985),(-11.683546,26.212975),(-11.698222,26.162177),(-11.717239,26.103576),(-11.753877,26.086006),(-11.879864,26.070399),(-11.959911,26.049884),(-12.029752,26.03035),(-12.055823,25.99583),(-12.055823,25.99583),(-12.060009,25.990301),(-12.080059,25.919969),(-12.080059,25.870205),(-12.100058,25.830156),(-12.129849,25.730549),(-12.169873,25.639728),(-12.200155,25.51958),(-12.229946,25.42),(-12.26997,25.259803),(-12.310019,25.110432),(-12.359835,24.969795),(-12.399884,24.879955),(-12.430141,24.830165),(-12.499982,24.7696),(-12.56003,24.730533),(-12.629845,24.679761),(-12.709943,24.629971),(-12.819781,24.570388),(-12.910137,24.51959),(-12.946879,24.496749),(-12.94688,24.496748),(-12.990184,24.4698),(-13.060025,24.400476),(-13.120099,24.299862),(-13.160122,24.219815),(-13.229963,24.0899),(-13.279753,24.019594),(-13.31001,23.980553),(-13.390108,23.940504),(-13.479948,23.910221),(-13.580045,23.870198),(-13.660118,23.830123),(-13.769982,23.790125),(-13.839797,23.750076),(-13.890129,23.690493),(-13.930127,23.620187),(-13.979943,23.519599),(-14.019992,23.410252),(-14.039991,23.33992),(-14.100065,23.099676),(-14.120089,22.960047),(-14.140088,22.870181),(-14.169906,22.759852),(-14.189904,22.589914),(-14.189904,22.450259),(-14.209955,22.370186),(-14.220187,22.309647),(-14.270003,22.240297),(-14.310052,22.190507),(-14.379841,22.120201),(-14.439915,22.080152),(-14.459914,22.040103),(-14.519988,21.990338),(-14.580036,21.91024),(-14.629852,21.860424),(-14.620085,21.820375),(-14.609827,21.750069),(-14.640109,21.679763),(-14.669875,21.599665),(-14.749974,21.500084),(-14.839813,21.450268),(-14.970167,21.440501),(-15.149872,21.440501),(-15.289992,21.450268),(-15.45993,21.450268),(-15.609818,21.469802),(-15.749964,21.490317),(-15.919876,21.500084),(-16.040024,21.500084),(-16.189886,21.48055),(-16.580043,21.48055),(-16.729956,21.469802),(-16.950149,21.429753),(-17.013743,21.419971),(-17.013743,21.419989),(-17.012807,21.445217),(-16.971059,21.592597),(-16.966908,21.636542),(-16.970326,21.674465),(-16.968007,21.697252),(-16.95108,21.724758),(-16.947662,21.736029),(-16.946401,21.748277),(-16.947011,21.759996),(-16.950266,21.765367),(-16.963246,21.775621),(-16.966908,21.781155),(-16.961537,21.795803),(-16.956939,21.82567),(-16.937815,21.866604),(-16.929799,21.907945),(-16.92101,21.931301),(-16.81664,22.130561),(-16.78067,22.165717),(-16.775054,22.174954),(-16.773427,22.180976),(-16.762074,22.205024),(-16.756337,22.210191),(-16.739898,22.221137),(-16.73412,22.226752),(-16.717885,22.260972),(-16.706776,22.275458),(-16.682607,22.284084),(-16.677154,22.289537),(-16.670318,22.29442),(-16.659006,22.295071),(-16.650258,22.291693),(-16.642649,22.285956),(-16.631703,22.273912),(-16.52184,22.314887),(-16.502187,22.329901),(-16.482167,22.352281),(-16.466624,22.377387),(-16.431508,22.493069),(-16.429351,22.510443),(-16.419545,22.521796),(-16.369781,22.568264),(-16.357289,22.583075),(-16.34439,22.646959),(-16.341379,22.732001),(-16.336822,22.75373),(-16.324859,22.789496),(-16.316477,22.806057),(-16.296457,22.832099),(-16.296254,22.84219),(-16.299957,22.850165),(-16.302113,22.856757),(-16.302154,22.868313),(-16.300608,22.875067),(-16.291737,22.893459),(-16.289622,22.899807),(-16.284779,22.903388),(-16.262603,22.905585),(-16.238637,22.911566),(-16.206451,22.937974),(-16.172963,22.973456),(-16.163238,22.993232),(-16.156484,23.013495),(-16.152577,23.035224),(-16.151275,23.059068),(-16.159535,23.082465),(-16.179067,23.086127),(-16.220204,23.075873),(-16.213124,23.094428),(-16.196034,23.126166),(-16.186676,23.137926),(-16.139272,23.178534),(-16.135121,23.185126),(-16.060414,23.34455),(-16.049143,23.3581),(-16.038319,23.363918),(-16.025543,23.374945),(-15.981191,23.445787),(-15.967193,23.512926),(-15.956451,23.531724),(-15.941314,23.551337),(-15.905507,23.610297),(-15.857086,23.661811),(-15.853871,23.667914),(-15.837392,23.679348),(-15.830393,23.685981),(-15.825063,23.696723),(-15.820058,23.717231),(-15.81607,23.727607),(-15.800649,23.750637),(-15.78482,23.768988),(-15.772613,23.787828),(-15.76769,23.812649),(-15.765452,23.856757),(-15.769154,23.875678),(-15.781972,23.898261),(-15.778676,23.903713),(-15.775136,23.911933),(-15.786773,23.903876),(-15.787953,23.892157),(-15.786,23.878363),(-15.788197,23.864163),(-15.795318,23.856635),(-15.820872,23.840277),(-15.830393,23.836249),(-15.843577,23.835883),(-15.852203,23.838121),(-15.861724,23.836168),(-15.877553,23.823188),(-15.888173,23.810736),(-15.895334,23.798774),(-15.91039,23.765326),(-15.923329,23.743598),(-15.925933,23.73725),(-15.92809,23.725165),(-15.937489,23.705227),(-15.939605,23.696234),(-15.948232,23.684312),(-15.994862,23.645006),(-16.009755,23.655422),(-16.003163,23.672105),(-15.981191,23.696234),(-15.975819,23.710761),(-15.939605,23.774807),(-15.877553,23.853583),(-15.871246,23.869208),(-15.856801,23.886949),(-15.840932,23.902655),(-15.768137,23.959866),(-15.735748,23.971177),(-15.692616,23.995795),(-15.679555,24.007514),(-15.646637,24.010403),(-15.620107,24.026272),(-15.599233,24.048326),(-15.583323,24.070258),(-15.542348,24.114244),(-15.514394,24.152167),(-15.476064,24.177802),(-15.459706,24.191881),(-15.453033,24.210191),(-15.444692,24.221869),(-15.353383,24.291083),(-15.298248,24.34516),(-15.229563,24.440741),(-15.203969,24.467475),(-15.171864,24.494127),(-15.153554,24.505601),(-15.133412,24.514635),(-15.112701,24.520575),(-15.069407,24.527289),(-15.048329,24.538275),(-15.014801,24.562445),(-15.01537,24.585679),(-15.012237,24.591539),(-14.999989,24.614488),(-14.976226,24.641425),(-14.952138,24.659247),(-14.926991,24.67178),(-14.907826,24.685492),(-14.895619,24.705715),(-14.885854,24.770738),(-14.861928,24.830146),(-14.853424,24.876614),(-14.83967,24.905341),(-14.836049,24.918687),(-14.837961,24.935126),(-14.84203,24.950263),(-14.84317,24.965277),(-14.836049,24.981391),(-14.843495,24.987616),(-14.837229,25.001695),(-14.83259,25.017564),(-14.829742,25.034247),(-14.829213,25.050238),(-14.831899,25.065741),(-14.84142,25.087551),(-14.843495,25.100531),(-14.842437,25.189032),(-14.849721,25.214179),(-14.819244,25.330227),(-14.812123,25.346096),(-14.782053,25.446926),(-14.736195,25.50552),(-14.723134,25.515815),(-14.714915,25.52558),(-14.68517,25.590888),(-14.678334,25.635565),(-14.676259,25.641913),(-14.66686,25.659735),(-14.661366,25.691311),(-14.620961,25.783596),(-14.546457,25.879218),(-14.519521,25.923774),(-14.513905,25.936957),(-14.511057,25.956855),(-14.494985,26.001614),(-14.486562,26.015448),(-14.490346,26.03148),(-14.480377,26.087795),(-14.483306,26.102484),(-14.492828,26.132717),(-14.490061,26.142727),(-14.482289,26.153876),(-14.471791,26.183051),(-14.466135,26.194241),(-14.410146,26.260077),(-14.400746,26.266303),(-14.353179,26.275946),(-14.29894,26.302232),(-14.215932,26.375474),(-14.202056,26.392239),(-14.192779,26.409491),(-14.180247,26.424384),(-14.157053,26.433783),(-14.046864,26.444403),(-14.02774,26.452379),(-13.923736,26.500881),(-13.742014,26.618109),(-13.619985,26.68891),(-13.561106,26.749091),(-13.500885,26.858303),(-13.480336,26.911689),(-13.461049,26.992743),(-13.431996,27.050767),(-13.402455,27.177191),(-13.383656,27.215277),(-13.301096,27.324449),(-13.300771,27.330024),(-13.307769,27.339342),(-13.307932,27.344957),(-13.29426,27.351752),(-13.240834,27.465399),(-13.20759,27.56745),(-13.205434,27.588568),(-13.200795,27.606513),(-13.180979,27.644843),(-13.178254,27.662491),(-13.178253,27.662502),(-13.178212,27.662787),(-13.177561,27.667141),(-13.171254,27.685004),(-13.160512,27.694648),(-13.13036,27.708075),(-13.097971,27.732245),(-13.061431,27.742906),(-13.045155,27.759833),(-13.003,27.821031),(-12.979563,27.893704),(-12.968088,27.914618),(-12.952382,27.928616),(-12.933095,27.938707),(-12.838938,27.970404),(-12.657826,27.998033),(-12.513539,27.995429),(-12.335032,28.047268),(-12.061431,28.089016),(-12.050404,28.096015),(-12.027903,28.116889),(-12.023101,28.119574),(-12.005605,28.123196),(-11.782379,28.21011),(-11.485585,28.325629),(-11.452789,28.348456),(-11.428822,28.375922),(-11.389312,28.434882),(-11.374013,28.449774),(-11.355946,28.463853),(-11.340972,28.479682),(-11.334706,28.499742),(-11.325917,28.515204),(-11.266469,28.551581),(-11.219635,28.602484),(-11.211822,28.616767),(-11.170318,28.639716),(-11.162465,28.647406),(-11.154205,28.661078),(-11.149159,28.66767),(-11.11498,28.69184),(-11.111562,28.695746),(-11.091054,28.726304),(-11.0631,28.752387),(-11.047597,28.761705),(-10.762603,28.890204),(-10.694203,28.930569),(-10.612416,28.967719),(-10.573801,28.990424),(-10.450917,29.09219),(-10.34789,29.229397),(-10.320872,29.257961),(-10.301991,29.271877),(-10.264027,29.284125),(-10.248402,29.299384),(-10.224355,29.333075),(-10.199127,29.361274),(-10.187571,29.378974),(-10.182729,29.397284),(-10.177642,29.405829),(-10.141754,29.428616),(-10.137034,29.434963),(-10.128163,29.455959),(-10.086985,29.509955),(-10.080922,29.521145),(-10.078684,29.542304),(-10.072621,29.564683),(-10.063629,29.584947),(-10.052968,29.599921),(-10.005279,29.638414),(-9.997792,29.650824),(-9.991689,29.666815),(-9.949941,29.71601),(-9.820221,29.838935),(-9.717926,29.996568),(-9.655141,30.126899),(-9.645253,30.164008),(-9.628407,30.291409),(-9.610015,30.357815),(-9.60733,30.377631),(-9.611643,30.404364),(-9.622711,30.417426),(-9.638173,30.426663),(-9.655141,30.442206),(-9.666249,30.459296),(-9.691762,30.509589),(-9.700063,30.542711),(-9.708404,30.547512),(-9.719553,30.548529),(-9.730824,30.552069),(-9.739817,30.559882),(-9.758168,30.586819),(-9.829661,30.628607),(-9.850942,30.629625),(-9.874013,30.633979),(-9.888539,30.648261),(-9.886708,30.686672),(-9.865631,30.72427),(-9.839019,30.759833),(-9.820221,30.792304),(-9.81371,30.814683),(-9.812896,30.827135),(-9.816803,30.836656),(-9.823842,30.845649),(-9.824371,30.851955),(-9.821889,30.859117),(-9.820221,30.87108),(-9.827707,31.06745),(-9.831451,31.085354),(-9.847524,31.121283),(-9.835032,31.144761),(-9.808949,31.2862),(-9.80663,31.343451),(-9.810902,31.366848),(-9.821523,31.379462),(-9.834788,31.388821),(-9.847524,31.402411),(-9.816233,31.436835),(-9.792958,31.47134),(-9.769643,31.53148),(-9.758168,31.545803),(-9.739613,31.560004),(-9.733266,31.567369),(-9.727651,31.58515),(-9.713612,31.597846),(-9.707631,31.60932),(-9.694203,31.625718),(-9.68928,31.635199),(-9.675201,31.715399),(-9.667307,31.731147),(-9.575795,31.816107),(-9.53539,31.864488),(-9.527943,31.871243),(-9.513295,31.880845),(-9.504954,31.888414),(-9.497426,31.898139),(-9.488678,31.915351),(-9.483795,31.922553),(-9.366119,32.026313),(-9.354075,32.046698),(-9.346425,32.06151),(-9.334828,32.10163),(-9.333079,32.118069),(-9.328114,32.130439),(-9.291982,32.168931),(-9.27774,32.196601),(-9.270009,32.227484),(-9.26476,32.330634),(-9.268951,32.335517),(-9.28775,32.343492),(-9.291982,32.350816),(-9.289662,32.372463),(-9.283355,32.387763),(-9.256256,32.432563),(-9.24885,32.452541),(-9.246938,32.472317),(-9.254791,32.488267),(-9.263661,32.498481),(-9.272369,32.512926),(-9.27831,32.529527),(-9.278961,32.546332),(-9.259918,32.57689),(-9.112172,32.682847),(-8.998525,32.797105),(-8.901682,32.854804),(-8.867787,32.881903),(-8.754709,33.010647),(-8.62914,33.127631),(-8.62149,33.13935),(-8.620351,33.152004),(-8.621002,33.173529),(-8.611887,33.186265),(-8.529164,33.268704),(-8.521148,33.272528),(-8.512522,33.270494),(-8.502797,33.261542),(-8.49413,33.259508),(-8.469797,33.260647),(-8.45165,33.264228),(-8.415028,33.280015),(-8.378774,33.304674),(-8.320221,33.365383),(-8.284657,33.389838),(-8.243153,33.404527),(-8.106557,33.430854),(-8.011545,33.466417),(-7.962554,33.484809),(-7.916249,33.494574),(-7.891428,33.503363),(-7.86974,33.526557),(-7.844797,33.535956),(-7.777984,33.55329),(-7.738189,33.570624),(-7.719472,33.574205),(-7.700999,33.58275),(-7.684438,33.599921),(-7.667551,33.612698),(-7.647857,33.608344),(-7.63622,33.614569),(-7.626617,33.613227),(-7.617258,33.609361),(-7.606191,33.608344),(-7.596669,33.611558),(-7.579254,33.620754),(-7.549957,33.625556),(-7.529124,33.632636),(-7.510243,33.641547),(-7.49706,33.64997),(-7.463287,33.68651),(-7.434316,33.698228),(-7.406361,33.726996),(-7.387115,33.732489),(-7.393951,33.725043),(-7.37385,33.719306),(-7.35261,33.724677),(-7.341135,33.731106),(-7.267486,33.772406),(-7.229319,33.80093),(-7.192942,33.817694),(-7.152577,33.830064),(-7.117014,33.834866),(-7.116567,33.834906),(-7.099477,33.840277),(-7.06786,33.863674),(-7.031809,33.874254),(-6.927358,33.944159),(-6.822092,34.039618),(-6.733754,34.162584),(-6.667795,34.280504),(-6.557932,34.417711),(-6.54776,34.434719),(-6.530914,34.474026),(-6.507924,34.504788),(-6.292104,34.880805),(-6.205393,35.124579),(-6.023915,35.501166),(-6.018544,35.520209),(-6.016672,35.544623),(-6.012318,35.558824),(-5.990834,35.598538),(-5.970041,35.652533),(-5.968251,35.660956),(-5.96524,35.670233),(-5.952056,35.685492),(-5.949045,35.69538),(-5.943105,35.742377),(-5.936187,35.765611),(-5.927235,35.780748),(-5.90803,35.797349),(-5.873402,35.801988),(-5.781402,35.79092),(-5.769276,35.792792),(-5.74234,35.814887),(-5.729888,35.822659),(-5.711741,35.830634),(-5.693512,35.834296),(-5.680898,35.829169),(-5.663442,35.835883),(-5.639719,35.836371),(-5.615956,35.831529),(-5.598988,35.822333),(-5.540028,35.850979),(-5.523264,35.863267),(-5.487457,35.896959),(-5.468861,35.910346),(-5.44811,35.918524),(-5.440027,35.918106),(-5.435452,35.914417),(-5.423497,35.908956),(-5.414198,35.909398),(-5.410361,35.917073),(-5.408,35.924748),(-5.40549,35.926519)] +Monaco [(7.437454,43.743361),(7.432845,43.739853),(7.417957,43.730904),(7.404319,43.717969),(7.380723,43.719273),(7.36575,43.72273),(7.367263,43.734125),(7.372655,43.745832),(7.387538,43.757899),(7.406969,43.763506),(7.426246,43.755464),(7.437454,43.743361)] +Moldova [(27.606873,48.457819),(27.627026,48.451256),(27.751773,48.451979),(27.78526,48.441566),(27.849855,48.40963),(27.864841,48.398907),(27.885371,48.378444),(27.896674,48.367178),(27.904529,48.362475),(27.92644,48.339428),(27.967057,48.328886),(28.003177,48.325883),(28.055941,48.321496),(28.076405,48.314933),(28.092838,48.302066),(28.098315,48.284108),(28.078678,48.244912),(28.093251,48.237444),(28.115575,48.236747),(28.131078,48.239589),(28.162084,48.257185),(28.178517,48.258864),(28.185752,48.242379),(28.189369,48.223052),(28.199911,48.211735),(28.216965,48.208144),(28.240322,48.211632),(28.261716,48.219952),(28.277529,48.229124),(28.294479,48.236488),(28.340575,48.240106),(28.357731,48.238607),(28.36817,48.230623),(28.37003,48.211632),(28.363364,48.191013),(28.352253,48.181866),(28.337887,48.175484),(28.322178,48.163211),(28.315563,48.149284),(28.317424,48.135358),(28.328586,48.127012),(28.349566,48.129725),(28.364966,48.141507),(28.37556,48.156648),(28.388634,48.168586),(28.411578,48.170704),(28.427288,48.163366),(28.436899,48.149853),(28.435659,48.134686),(28.419019,48.122232),(28.447441,48.082751),(28.459379,48.074457),(28.479584,48.064923),(28.490023,48.065491),(28.493744,48.074819),(28.494157,48.091846),(28.49979,48.108914),(28.501082,48.112827),(28.501113,48.112889),(28.519323,48.149129),(28.541596,48.155951),(28.573635,48.154969),(28.665619,48.129337),(28.735072,48.128691),(28.771246,48.124454),(28.799255,48.111793),(28.805921,48.103809),(28.80835,48.096574),(28.809176,48.089779),(28.811554,48.082906),(28.827573,48.057016),(28.830777,48.030868),(28.832534,48.024822),(28.839252,48.018285),(28.855685,48.007923),(28.862085,48.000502),(28.882557,47.976763),(28.914803,47.953146),(28.936087,47.942164),(28.950356,47.934801),(28.980846,47.926378),(29.017226,47.931081),(29.061047,47.96976),(29.092725,47.980173),(29.110037,47.979656),(29.123989,47.975987),(29.136288,47.968184),(29.147967,47.955369),(29.155822,47.939866),(29.164607,47.905604),(29.172565,47.891006),(29.186621,47.883642),(29.225999,47.87558),(29.236024,47.870723),(29.232768,47.85708),(29.21742,47.842895),(29.19892,47.829485),(29.186724,47.818219),(29.177733,47.800701),(29.177836,47.78972),(29.187448,47.783209),(29.221968,47.774811),(29.234732,47.766801),(29.238608,47.756001),(29.22667,47.743237),(29.196956,47.717502),(29.191479,47.686264),(29.192409,47.650917),(29.182074,47.613219),(29.156184,47.582781),(29.130501,47.55963),(29.117426,47.533327),(29.130397,47.4932),(29.130501,47.493097),(29.130604,47.493045),(29.130656,47.492942),(29.137189,47.484008),(29.140009,47.480152),(29.155615,47.449999),(29.163884,47.439379),(29.182074,47.429871),(29.192409,47.435814),(29.201866,47.446588),(29.21835,47.451601),(29.232613,47.44633),(29.2507,47.425375),(29.281396,47.411474),(29.288217,47.400777),(29.292713,47.388866),(29.300826,47.378194),(29.314675,47.372717),(29.330385,47.370495),(29.345836,47.365844),(29.35891,47.352718),(29.364078,47.336879),(29.364491,47.322436),(29.367592,47.308225),(29.370882,47.304434),(29.380821,47.29298),(29.394464,47.284712),(29.409553,47.279751),(29.425573,47.278666),(29.441696,47.282128),(29.459163,47.293497),(29.466087,47.307449),(29.470635,47.322746),(29.480867,47.338274),(29.501072,47.339825),(29.520968,47.33892),(29.539881,47.334218),(29.556573,47.324038),(29.57006,47.306778),(29.579259,47.283601),(29.580189,47.26045),(29.568717,47.243138),(29.544015,47.23425),(29.540398,47.212546),(29.55063,47.160172),(29.544842,47.135574),(29.530373,47.123688),(29.508669,47.119399),(29.480867,47.117487),(29.478593,47.114645),(29.47787,47.111544),(29.478696,47.108469),(29.480867,47.10524),(29.511459,47.066017),(29.520141,47.059919),(29.530993,47.066017),(29.539726,47.078885),(29.551353,47.090202),(29.570164,47.091442),(29.583393,47.085215),(29.594865,47.074595),(29.6021,47.061005),(29.60303,47.04576),(29.595433,47.032143),(29.583393,47.02279),(29.572489,47.011757),(29.564996,46.974498),(29.560626,46.961256),(29.559828,46.95884),(29.558691,46.94574),(29.567476,46.934656),(29.573161,46.933725),(29.598482,46.935379),(29.607577,46.93202),(29.623287,46.91954),(29.631865,46.914398),(29.648092,46.910419),(29.680855,46.908559),(29.696151,46.904683),(29.712481,46.893444),(29.735838,46.867192),(29.754132,46.85802),(29.785758,46.854661),(29.815213,46.856624),(29.843635,46.854144),(29.872212,46.837711),(29.876915,46.830786),(29.884873,46.812673),(29.889627,46.80717),(29.898516,46.806653),(29.907714,46.810942),(29.917429,46.813965),(29.928178,46.809754),(29.930658,46.803087),(29.930452,46.781642),(29.931795,46.772443),(29.935671,46.765157),(29.946678,46.750739),(29.951122,46.743117),(29.952621,46.724694),(29.944404,46.662476),(29.94027,46.650797),(29.935413,46.641857),(29.931899,46.632374),(29.931279,46.619171),(29.934948,46.608836),(29.947402,46.588346),(29.949675,46.578863),(29.9381,46.55734),(29.916086,46.554369),(29.898826,46.553129),(29.901926,46.53083),(29.916086,46.518815),(29.960217,46.505948),(29.962078,46.503442),(29.967039,46.493856),(29.970553,46.491737),(29.977064,46.493184),(29.989259,46.498687),(29.994944,46.498532),(30.00285,46.494217),(30.009413,46.488068),(30.021712,46.470653),(30.014064,46.462023),(30.060005,46.436805),(30.077006,46.422826),(30.086928,46.428873),(30.102017,46.430733),(30.118244,46.428692),(30.131576,46.422826),(30.107185,46.391898),(30.08114,46.374199),(30.037267,46.368928),(29.918669,46.373656),(29.902546,46.371176),(29.884666,46.3642),(29.847769,46.341462),(29.828132,46.339395),(29.808185,46.354691),(29.80617,46.361564),(29.806738,46.380891),(29.805085,46.389986),(29.80033,46.398358),(29.779557,46.421095),(29.72695,46.455796),(29.714134,46.47117),(29.713824,46.443058),(29.702869,46.42833),(29.682715,46.42293),(29.654396,46.422697),(29.647988,46.416393),(29.652949,46.391898),(29.645508,46.375594),(29.632072,46.36606),(29.615639,46.361822),(29.598586,46.363063),(29.582979,46.369651),(29.56944,46.382286),(29.555643,46.404197),(29.541432,46.413034),(29.527479,46.416651),(29.49606,46.420811),(29.480867,46.425152),(29.475079,46.432257),(29.473322,46.439802),(29.475286,46.447554),(29.480867,46.455512),(29.486241,46.462488),(29.488205,46.469361),(29.486448,46.475872),(29.480867,46.482048),(29.457612,46.484709),(29.456724,46.484366),(29.436838,46.476699),(29.418235,46.462488),(29.374723,46.416186),(29.362114,46.415928),(29.344234,46.434169),(29.320773,46.468741),(29.306717,46.471997),(29.289147,46.451636),(29.285943,46.439492),(29.290697,46.419261),(29.290077,46.410243),(29.283256,46.397427),(29.278192,46.395877),(29.27137,46.397427),(29.259175,46.394353),(29.222898,46.366473),(29.200677,46.35712),(29.183727,46.367171),(29.183417,46.37761),(29.190342,46.387196),(29.199282,46.396497),(29.205638,46.406109),(29.207602,46.417891),(29.206775,46.502511),(29.200212,46.523983),(29.184244,46.538039),(29.183059,46.538042),(29.162747,46.538091),(29.074948,46.503674),(29.055915,46.49874),(29.020326,46.489515),(28.945809,46.454788),(28.925448,46.432774),(28.919247,46.404662),(28.927309,46.368024),(28.944879,46.32074),(28.945809,46.304978),(28.939918,46.286995),(28.93289,46.272629),(28.93351,46.258986),(28.950615,46.243122),(28.95077,46.24307),(28.984112,46.221221),(29.009836,46.204364),(29.015365,46.182609),(29.00348,46.158941),(28.980846,46.132121),(28.946429,46.105094),(28.938781,46.089281),(28.941158,46.064683),(28.958935,46.021016),(28.956868,46.001379),(28.93196,45.993163),(28.757603,45.961175),(28.74024,45.953217),(28.729181,45.938644),(28.728406,45.921953),(28.742307,45.887329),(28.746441,45.870586),(28.745046,45.850484),(28.738276,45.837565),(28.725564,45.82878),(28.706392,45.821132),(28.677298,45.816584),(28.669753,45.812037),(28.669392,45.806301),(28.671924,45.797257),(28.673267,45.78687),(28.669753,45.77731),(28.643605,45.76651),(28.576684,45.761911),(28.560613,45.743204),(28.563525,45.735494),(28.567641,45.7246),(28.56144,45.71659),(28.532501,45.710079),(28.515241,45.702018),(28.504182,45.693698),(28.480928,45.669462),(28.474003,45.657938),(28.482891,45.650755),(28.498498,45.644295),(28.511314,45.635045),(28.516895,45.620989),(28.518755,45.607036),(28.523096,45.593239),(28.536893,45.579803),(28.510435,45.571018),(28.504596,45.567246),(28.503743,45.566102),(28.498239,45.558719),(28.497981,45.556445),(28.500048,45.554585),(28.506456,45.520582),(28.502012,45.508748),(28.480928,45.501978),(28.416849,45.503787),(28.341918,45.517636),(28.270501,45.521512),(28.270423,45.521471),(28.217275,45.493348),(28.217171,45.493193),(28.217171,45.493142),(28.20849,45.481411),(28.201565,45.468854),(28.199498,45.461774),(28.172936,45.484357),(28.165908,45.494589),(28.165391,45.528282),(28.164048,45.530607),(28.161567,45.532726),(28.157743,45.538927),(28.140949,45.560218),(28.118004,45.572723),(28.062142,45.593601),(28.074751,45.604866),(28.091184,45.615976),(28.107721,45.6244),(28.120846,45.627707),(28.153713,45.6275),(28.167975,45.632461),(28.161774,45.645432),(28.163738,45.661503),(28.154953,45.761807),(28.146478,45.771161),(28.128908,45.795035),(28.113302,45.825369),(28.110511,45.854308),(28.114955,45.859734),(28.128184,45.8664),(28.131078,45.871361),(28.129115,45.875134),(28.120226,45.887846),(28.117436,45.895236),(28.118779,45.903142),(28.12281,45.910791),(28.124981,45.918387),(28.120846,45.926293),(28.114852,45.933218),(28.112371,45.939213),(28.110511,45.950426),(28.08612,46.000552),(28.082709,46.014712),(28.084466,46.024582),(28.096869,46.05967),(28.096558,46.065045),(28.090461,46.06799),(28.090151,46.073365),(28.092838,46.078377),(28.100796,46.081995),(28.107823,46.096103),(28.110506,46.101491),(28.127358,46.135325),(28.133249,46.159974),(28.144721,46.183229),(28.141517,46.191962),(28.135626,46.198835),(28.129321,46.204674),(28.110976,46.225603),(28.108961,46.23413),(28.120846,46.237851),(28.132112,46.239918),(28.134489,46.245344),(28.131078,46.262397),(28.135212,46.269322),(28.144928,46.272939),(28.156296,46.275368),(28.165753,46.27883),(28.177794,46.287047),(28.191333,46.307769),(28.192883,46.311231),(28.191953,46.323504),(28.187922,46.343787),(28.190351,46.351074),(28.202392,46.353916),(28.207146,46.358153),(28.212934,46.388694),(28.219548,46.393345),(28.226796,46.395548),(28.228643,46.39611),(28.233294,46.401768),(28.22606,46.415359),(28.240219,46.420268),(28.246058,46.427865),(28.247247,46.436185),(28.247144,46.467191),(28.2458,46.475511),(28.242338,46.476518),(28.2381,46.475588),(28.234121,46.478069),(28.221099,46.495768),(28.219032,46.503726),(28.221099,46.541191),(28.224768,46.548736),(28.234121,46.553129),(28.225439,46.569562),(28.230504,46.583902),(28.240529,46.596382),(28.247144,46.607156),(28.24735,46.620825),(28.245085,46.631451),(28.24425,46.635372),(28.234121,46.662424),(28.178104,46.739836),(28.178156,46.758646),(28.137796,46.80624),(28.121518,46.8343),(28.122615,46.84227),(28.124257,46.854195),(28.124257,46.861637),(28.113095,46.87143),(28.114335,46.883393),(28.11356,46.894761),(28.096869,46.902616),(28.105447,46.91768),(28.104994,46.920185),(28.102295,46.935121),(28.082709,46.971527),(28.069067,46.988503),(28.038146,47.015483),(28.037027,47.016459),(28.028036,47.03297),(28.008295,47.026304),(27.986591,47.033228),(27.96313,47.043383),(27.938739,47.046638),(27.938429,47.061056),(27.925923,47.068756),(27.897604,47.081417),(27.867012,47.104645),(27.85771,47.109322),(27.848202,47.111415),(27.843757,47.114386),(27.849855,47.121724),(27.849855,47.12852),(27.80655,47.144617),(27.794665,47.155831),(27.805879,47.158234),(27.807687,47.162471),(27.804793,47.168492),(27.802106,47.176346),(27.800349,47.176966),(27.788309,47.204277),(27.763039,47.226317),(27.752393,47.238926),(27.753634,47.251432),(27.733273,47.275617),(27.722835,47.283317),(27.697823,47.287502),(27.689865,47.290887),(27.68232,47.295254),(27.671727,47.299853),(27.6447,47.303987),(27.636948,47.306674),(27.624029,47.321428),(27.599948,47.360728),(27.588724,47.367389),(27.586409,47.368764),(27.572353,47.375197),(27.580311,47.405996),(27.562534,47.416538),(27.565531,47.428269),(27.569356,47.438553),(27.574782,47.446252),(27.582998,47.450671),(27.575867,47.460412),(27.563154,47.468344),(27.548168,47.474261),(27.534526,47.477982),(27.532975,47.477413),(27.50724,47.477982),(27.501453,47.479945),(27.497525,47.482426),(27.492564,47.484545),(27.483676,47.485423),(27.473134,47.491779),(27.466726,47.506352),(27.459491,47.533224),(27.456908,47.533896),(27.446056,47.534877),(27.442128,47.536634),(27.440061,47.541027),(27.440268,47.550096),(27.438976,47.553713),(27.435824,47.55994),(27.431342,47.572638),(27.429404,47.57813),(27.428382,47.581024),(27.397066,47.589086),(27.369161,47.60831),(27.304049,47.666549),(27.281104,47.693007),(27.272009,47.71497),(27.295057,47.718174),(27.295057,47.724427),(27.289579,47.731687),(27.29113,47.74241),(27.288114,47.75068),(27.287512,47.752332),(27.282448,47.755691),(27.264671,47.764553),(27.260847,47.769075),(27.256196,47.77755),(27.244879,47.792588),(27.231082,47.807083),(27.219196,47.813775),(27.253406,47.828089),(27.245809,47.83659),(27.234906,47.840208),(27.222813,47.842662),(27.212375,47.847933),(27.213202,47.854134),(27.211548,47.885011),(27.212375,47.889507),(27.194185,47.904054),(27.17155,47.912632),(27.160698,47.921727),(27.178268,47.937928),(27.178268,47.942096),(27.178268,47.944129),(27.171964,47.946429),(27.150983,47.957797),(27.162869,47.965006),(27.169793,47.973791),(27.16876,47.983067),(27.157184,47.991955),(27.143748,47.986839),(27.134602,48.000456),(27.121476,48.013194),(27.095793,48.005598),(27.093364,48.00844),(27.091452,48.011851),(27.090005,48.015623),(27.088971,48.019809),(27.109332,48.026088),(27.109332,48.033503),(27.083184,48.043658),(27.059206,48.05805),(27.041739,48.077273),(27.034298,48.101768),(27.036882,48.107349),(27.047424,48.116496),(27.047955,48.121409),(27.048044,48.122232),(27.042773,48.127296),(27.033678,48.132386),(27.025099,48.135048),(27.012697,48.128123),(26.99399,48.132412),(26.966602,48.143367),(26.966602,48.149594),(26.986239,48.150421),(26.997607,48.15794),(26.997607,48.16657),(26.963088,48.175381),(26.955646,48.186),(26.950582,48.197524),(26.938128,48.204811),(26.929085,48.199049),(26.917716,48.187964),(26.908001,48.184528),(26.90397,48.201452),(26.897665,48.208971),(26.855497,48.237832),(26.844749,48.23331),(26.821908,48.252534),(26.804906,48.25827),(26.76372,48.252741),(26.744031,48.255583),(26.733127,48.27075),(26.722379,48.259769),(26.711475,48.261319),(26.688531,48.274832),(26.665741,48.274212),(26.617889,48.258968),(26.618613,48.267184),(26.625124,48.282894),(26.636079,48.294883),(26.6691,48.30881),(26.67401,48.321703),(26.679797,48.330178),(26.699434,48.325113),(26.712974,48.314907),(26.723619,48.302582),(26.735918,48.291834),(26.754573,48.286149),(26.774365,48.287183),(26.785734,48.294108),(26.790075,48.307053),(26.789351,48.32594),(26.78408,48.345164),(26.778189,48.357204),(26.777518,48.36617),(26.78806,48.375963),(26.793589,48.376273),(26.810074,48.371881),(26.816947,48.371726),(26.825628,48.377772),(26.828419,48.385136),(26.831933,48.391363),(26.842785,48.39374),(26.875961,48.383999),(26.908724,48.36524),(26.943244,48.351262),(26.981071,48.35568),(26.990579,48.362372),(26.997814,48.361545),(27.004635,48.358755),(27.015591,48.359401),(27.027683,48.357747),(27.033057,48.360202),(27.031817,48.36524),(27.02851,48.370795),(27.027786,48.374774),(27.0282,48.381389),(27.025409,48.389864),(27.02634,48.397099),(27.037398,48.399683),(27.048044,48.397667),(27.068818,48.388882),(27.175788,48.361804),(27.208551,48.360615),(27.246481,48.373741),(27.25193,48.37831),(27.306012,48.42366),(27.342186,48.436114),(27.36141,48.432807),(27.389832,48.415005),(27.403474,48.411491),(27.420217,48.417149),(27.480885,48.451411),(27.50383,48.472365),(27.503896,48.472365),(27.545171,48.472365),(27.557057,48.474355),(27.582998,48.486034),(27.604806,48.484122),(27.606873,48.457819)] +Macedonia [(22.345023,42.313439),(22.443622,42.214427),(22.481449,42.193317),(22.494678,42.164559),(22.506939,42.148927),(22.510181,42.144793),(22.531058,42.129109),(22.617771,42.082704),(22.627177,42.079127),(22.675856,42.060612),(22.705725,42.055935),(22.710272,42.05299),(22.713993,42.048623),(22.718437,42.044463),(22.725052,42.042474),(22.77063,42.043998),(22.780862,42.043171),(22.785306,42.039141),(22.787684,42.032578),(22.791094,42.025808),(22.798949,42.021235),(22.805977,42.02139),(22.821273,42.025369),(22.826958,42.025085),(22.838223,42.019478),(22.843701,42.014465),(22.845621,42.007408),(22.845768,42.006869),(22.846595,41.993639),(22.846905,41.993484),(22.85476,41.982632),(22.857137,41.971884),(22.858894,41.94788),(22.866335,41.924884),(22.877084,41.902043),(22.878634,41.895015),(22.878221,41.880261),(22.880804,41.872691),(22.882088,41.871618),(22.885042,41.869151),(22.896721,41.864448),(22.901372,41.860418),(22.907676,41.848584),(22.918322,41.814348),(22.939716,41.776702),(22.945917,41.769338),(22.956872,41.765669),(22.98054,41.764739),(22.991185,41.760992),(23.008859,41.739934),(23.009582,41.71637),(22.998627,41.693115),(22.985435,41.677198),(22.976613,41.666553),(22.970101,41.652032),(22.967001,41.647046),(22.961523,41.644488),(22.945813,41.641077),(22.940852,41.63764),(22.936098,41.626168),(22.932998,41.612345),(22.932068,41.597953),(22.933721,41.584595),(22.936925,41.57891),(22.946434,41.567748),(22.948707,41.560978),(22.94788,41.555139),(22.943023,41.538551),(22.943599,41.523201),(22.946227,41.453233),(22.94757,41.448376),(22.953358,41.438195),(22.954598,41.432408),(22.952118,41.427705),(22.940542,41.416905),(22.937339,41.410755),(22.939406,41.389413),(22.94447,41.368432),(22.940852,41.349829),(22.916978,41.335773),(22.826027,41.340992),(22.796818,41.337039),(22.780966,41.334894),(22.762293,41.3225),(22.751303,41.315205),(22.742003,41.28703),(22.740865,41.283579),(22.736731,41.204411),(22.727222,41.165809),(22.71575,41.145603),(22.704898,41.139661),(22.691875,41.14457),(22.674099,41.15661),(22.666347,41.164517),(22.661903,41.172268),(22.657185,41.176986),(22.656115,41.178056),(22.64454,41.180227),(22.629347,41.177074),(22.625936,41.169788),(22.626763,41.160796),(22.624489,41.152631),(22.607333,41.135992),(22.590176,41.125139),(22.586634,41.124085),(22.571263,41.119507),(22.549248,41.118525),(22.516589,41.122246),(22.500673,41.122142),(22.481449,41.11775),(22.467496,41.115011),(22.451477,41.113667),(22.421814,41.114649),(22.410446,41.117905),(22.389258,41.128343),(22.380163,41.131702),(22.367658,41.132374),(22.342336,41.128964),(22.323422,41.128498),(22.315051,41.124416),(22.308746,41.124984),(22.306679,41.128447),(22.305232,41.141624),(22.302649,41.1455),(22.29345,41.147515),(22.262961,41.150461),(22.225134,41.159608),(22.213637,41.160684),(22.205807,41.161416),(22.183173,41.159918),(22.160332,41.151908),(22.124365,41.127207),(22.103384,41.121522),(22.095943,41.123641),(22.062767,41.13718),(22.060286,41.140022),(22.058632,41.145087),(22.055532,41.149892),(22.048194,41.151856),(22.04406,41.149944),(22.033828,41.141263),(22.029074,41.138575),(21.965098,41.124364),(21.934712,41.11191),(21.925545,41.106754),(21.909081,41.097493),(21.901226,41.090775),(21.897195,41.081266),(21.896333,41.064705),(21.896162,41.061422),(21.894301,41.053826),(21.880659,41.038426),(21.845002,41.012381),(21.831463,40.993726),(21.831359,40.993675),(21.831359,40.993623),(21.831256,40.99352),(21.793636,40.973572),(21.783714,40.964167),(21.781853,40.955899),(21.78175,40.945099),(21.778133,40.93373),(21.765524,40.923911),(21.757091,40.922506),(21.736998,40.919157),(21.685115,40.927994),(21.6569,40.918227),(21.654833,40.9143),(21.652662,40.901381),(21.648838,40.895799),(21.643877,40.894456),(21.629511,40.895076),(21.623724,40.894404),(21.613698,40.888306),(21.590341,40.870685),(21.581556,40.866292),(21.55365,40.870426),(21.509519,40.900502),(21.505731,40.900903),(21.42942,40.908977),(21.404926,40.908615),(21.381258,40.900502),(21.344878,40.873062),(21.329271,40.866292),(21.295372,40.860866),(21.260955,40.860815),(21.245824,40.863226),(21.209072,40.869083),(21.183027,40.870168),(21.112127,40.853942),(20.965262,40.849394),(20.964849,40.875956),(20.956684,40.894766),(20.939941,40.907065),(20.890228,40.918279),(20.837415,40.924066),(20.836544,40.923904),(20.816951,40.920242),(20.783672,40.899055),(20.766102,40.893732),(20.740883,40.89797),(20.730573,40.904611),(20.717216,40.913214),(20.702746,40.936314),(20.683419,40.99383),(20.664092,41.059149),(20.65386,41.075272),(20.643008,41.081576),(20.634129,41.082576),(20.631536,41.082868),(20.618927,41.082403),(20.605284,41.083488),(20.597419,41.086273),(20.576966,41.093513),(20.569938,41.107104),(20.570558,41.124829),(20.565494,41.147412),(20.549681,41.170615),(20.51268,41.210147),(20.500071,41.23552),(20.483121,41.289471),(20.477747,41.319598),(20.478108,41.321585),(20.481674,41.341199),(20.49604,41.337788),(20.510407,41.344403),(20.523429,41.356805),(20.532937,41.370447),(20.539965,41.387139),(20.540172,41.400678),(20.539049,41.402944),(20.534074,41.412977),(20.522189,41.42569),(20.514644,41.429462),(20.498004,41.431477),(20.490873,41.436025),(20.488909,41.441451),(20.486842,41.457781),(20.483535,41.46548),(20.481597,41.468269),(20.470822,41.483774),(20.463174,41.489768),(20.452012,41.493592),(20.444984,41.508475),(20.447878,41.53545),(20.444157,41.549661),(20.49294,41.557671),(20.507409,41.56227),(20.520845,41.568368),(20.52932,41.574879),(20.534694,41.585266),(20.53467,41.587324),(20.534591,41.594026),(20.513404,41.640405),(20.508339,41.661748),(20.500381,41.734095),(20.503275,41.744637),(20.511337,41.757943),(20.521155,41.767658),(20.54441,41.784763),(20.550921,41.793522),(20.550136,41.80006),(20.54844,41.814193),(20.540482,41.839437),(20.540786,41.844865),(20.541722,41.86158),(20.567147,41.873182),(20.590298,41.854733),(20.602391,41.849876),(20.61872,41.850522),(20.626162,41.855198),(20.637014,41.870365),(20.643422,41.873647),(20.652827,41.86928),(20.671844,41.849307),(20.681456,41.84401),(20.702953,41.849591),(20.714398,41.859163),(20.723313,41.866619),(20.739953,41.888039),(20.750495,41.906797),(20.751052,41.910218),(20.754423,41.930904),(20.751439,41.940338),(20.741814,41.970773),(20.743054,41.993484),(20.755353,42.042784),(20.765378,42.064333),(20.784912,42.082032),(20.810543,42.092936),(20.90417,42.116668),(20.975145,42.134658),(21.003916,42.141951),(21.029238,42.151408),(21.04094,42.15997),(21.074403,42.184455),(21.098484,42.195953),(21.106236,42.195798),(21.112954,42.194402),(21.12639,42.188925),(21.164423,42.16704),(21.199873,42.14115),(21.216203,42.121151),(21.225402,42.106785),(21.229298,42.103822),(21.237804,42.097354),(21.245727,42.096167),(21.28886,42.089706),(21.299299,42.091411),(21.300796,42.098425),(21.301263,42.10061),(21.300332,42.120893),(21.298782,42.129419),(21.295785,42.134794),(21.293718,42.140168),(21.294958,42.148979),(21.353766,42.215977),(21.360602,42.220114),(21.366788,42.223858),(21.384255,42.224943),(21.419085,42.215021),(21.429834,42.215848),(21.428697,42.222566),(21.421462,42.231351),(21.419912,42.240058),(21.436345,42.24688),(21.44792,42.244141),(21.457222,42.237035),(21.467557,42.235123),(21.471973,42.23913),(21.48151,42.247784),(21.499287,42.238663),(21.519854,42.239025),(21.561815,42.247164),(21.564066,42.246289),(21.575044,42.242022),(21.624654,42.242772),(21.65969,42.235563),(21.676886,42.234945),(21.67695,42.234943),(21.69204,42.242022),(21.706509,42.25507),(21.719521,42.260957),(21.8172,42.305145),(21.837354,42.308556),(21.877352,42.30822),(21.88438,42.309512),(21.918279,42.331345),(21.929028,42.335117),(21.941534,42.333102),(21.994967,42.312612),(22.027627,42.303956),(22.045407,42.302424),(22.060906,42.301088),(22.095529,42.305817),(22.233402,42.348889),(22.259757,42.369095),(22.268852,42.370335),(22.273296,42.365477),(22.27402,42.348476),(22.276914,42.341241),(22.29159,42.328399),(22.307609,42.31933),(22.325283,42.314318),(22.345023,42.313439)] +Mali [(-4.821226,24.994755),(-4.744925,24.947057),(-4.668677,24.899334),(-4.592351,24.851688),(-4.516025,24.803991),(-4.404946,24.731308),(-4.325235,24.679218),(-4.245498,24.627077),(-4.165865,24.574987),(-4.086154,24.522897),(-4.006495,24.470807),(-3.926732,24.418717),(-3.846996,24.366576),(-3.767336,24.314434),(-3.687625,24.262344),(-3.608018,24.210203),(-3.528281,24.158113),(-3.448544,24.106023),(-3.368808,24.053959),(-3.289123,24.001844),(-3.209489,23.949702),(-3.129649,23.897612),(-3.043065,23.840923),(-2.956404,23.784234),(-2.869768,23.727623),(-2.783107,23.670985),(-2.696446,23.614348),(-2.609836,23.557685),(-2.523226,23.500996),(-2.436513,23.444358),(-2.349852,23.387747),(-2.26319,23.331109),(-2.176632,23.274446),(-2.089919,23.217757),(-2.00331,23.161042),(-1.916648,23.104456),(-1.830039,23.047819),(-1.743326,22.991182),(-1.656716,22.93457),(-1.570054,22.877881),(-1.483496,22.821166),(-1.396783,22.764555),(-1.310122,22.707969),(-1.223512,22.65128),(-1.136903,22.594643),(-1.05019,22.537954),(-0.96358,22.481265),(-0.876918,22.424627),(-0.790309,22.368041),(-0.703647,22.311404),(-0.616934,22.254767),(-0.530325,22.198078),(-0.443663,22.141389),(-0.357028,22.084751),(-0.270392,22.028114),(-0.183731,21.971477),(-0.097044,21.914839),(-0.010408,21.85815),(0.003735,21.848899),(0.076253,21.801461),(0.162966,21.744824),(0.249499,21.688238),(0.33616,21.631601),(0.422795,21.574963),(0.509457,21.518274),(0.596067,21.461585),(0.682676,21.404896),(0.769389,21.34831),(0.855999,21.291673),(0.942609,21.235036),(1.029322,21.178347),(1.146524,21.101711),(1.15934,21.081505),(1.177943,21.017323),(1.180114,20.995309),(1.167505,20.886013),(1.145284,20.795889),(1.14456,20.776252),(1.147247,20.751448),(1.154585,20.738787),(1.168538,20.733464),(1.191276,20.73057),(1.212463,20.73088),(1.252254,20.738994),(1.273338,20.739407),(1.296696,20.733464),(1.310545,20.722716),(1.331526,20.687937),(1.346925,20.669127),(1.363978,20.657707),(1.40718,20.645046),(1.447487,20.638741),(1.465781,20.633522),(1.483454,20.622618),(1.520351,20.616986),(1.559729,20.597504),(1.623704,20.551253),(1.643961,20.522676),(1.650059,20.487019),(1.649232,20.412089),(1.659154,20.397516),(1.778113,20.304291),(1.799197,20.294886),(1.820901,20.293594),(1.838885,20.29592),(1.855008,20.294835),(1.870614,20.283543),(1.880329,20.263054),(1.883843,20.24414),(1.891388,20.231789),(1.913402,20.231092),(1.924254,20.23613),(1.941204,20.251116),(1.95526,20.254915),(1.967146,20.253416),(1.975621,20.248429),(1.983372,20.241918),(1.993397,20.235949),(2.056339,20.215046),(2.071222,20.213263),(2.097474,20.224193),(2.138195,20.260728),(2.161346,20.274939),(2.18243,20.278505),(2.200826,20.273906),(2.21819,20.264087),(2.279581,20.21794),(2.316478,20.180165),(2.348208,20.137635),(2.388722,20.067407),(2.400401,20.056555),(2.415697,20.051284),(2.439882,20.04609),(2.459312,20.038778),(2.495382,20.020097),(2.514812,20.015937),(2.525665,20.015162),(2.616925,19.998367),(2.671805,19.996222),(2.946001,19.941652),(3.072608,19.88889),(3.130485,19.845224),(3.147022,19.837938),(3.183195,19.827731),(3.198802,19.820523),(3.212754,19.807758),(3.216785,19.794064),(3.198285,19.592397),(3.199422,19.553769),(3.212238,19.517156),(3.217509,19.511162),(3.222883,19.508061),(3.228051,19.504159),(3.232288,19.495478),(3.231668,19.489044),(3.22619,19.4692),(3.225984,19.459692),(3.234769,19.441347),(3.247481,19.426464),(3.25823,19.410393),(3.260813,19.388327),(3.250995,19.36546),(3.232701,19.351843),(3.211514,19.340862),(3.192911,19.325798),(3.183816,19.307505),(3.178855,19.268722),(3.17441,19.251643),(3.152706,19.230197),(3.13927,19.221929),(3.134206,19.212859),(3.126558,19.193352),(3.111779,19.171337),(3.102684,19.153561),(3.10413,19.135526),(3.120874,19.112814),(3.138754,19.096045),(3.158597,19.08155),(3.179785,19.07),(3.225984,19.05106),(3.284895,18.995741),(3.308356,18.981685),(3.318381,18.977706),(3.333057,18.975561),(3.358689,18.976853),(3.439717,18.995638),(3.439821,18.995638),(3.54462,19.015068),(3.715566,19.046901),(3.790291,19.06077),(3.886512,19.07863),(4.057561,19.110437),(4.22861,19.142244),(4.229023,19.018582),(4.229436,18.894946),(4.22985,18.771284),(4.230263,18.647648),(4.230271,18.645374),(4.230677,18.524038),(4.23109,18.400325),(4.231504,18.276741),(4.231917,18.153105),(4.232227,18.029443),(4.232408,17.975234),(4.23264,17.905782),(4.233054,17.78212),(4.233364,17.658458),(4.233777,17.534796),(4.234294,17.411186),(4.234707,17.287525),(4.235018,17.163915),(4.235328,17.100818),(4.235638,16.995863),(4.222305,16.986561),(4.21166,16.986044),(4.203702,16.982789),(4.197811,16.965219),(4.19657,16.947132),(4.202048,16.848895),(4.197604,16.838508),(4.184581,16.818509),(4.181998,16.809621),(4.182308,16.746472),(4.183031,16.612682),(4.183444,16.526538),(4.183961,16.416053),(4.1759,16.392644),(4.16174,16.379983),(4.118229,16.358331),(4.094768,16.340812),(4.075647,16.321072),(4.060558,16.298334),(3.971158,16.086099),(3.96692,16.058504),(3.970848,16.030857),(3.98325,16.00135),(3.983973,16.00011),(3.98449,15.998766),(3.984697,15.997371),(3.984594,15.995924),(3.9848,15.989826),(3.98418,15.986881),(3.98325,15.983987),(3.925062,15.927608),(3.909869,15.904767),(3.903462,15.886318),(3.89447,15.78865),(3.886512,15.750099),(3.873849,15.720882),(3.871215,15.714804),(3.846101,15.685297),(3.808377,15.665582),(3.728899,15.65088),(3.692208,15.63145),(3.614074,15.547734),(3.526534,15.495954),(3.516508,15.469186),(3.507103,15.353973),(3.48881,15.357539),(3.483332,15.359296),(3.380393,15.376324),(3.19229,15.40751),(3.073021,15.427199),(3.03354,15.42645),(3.017521,15.422832),(3.010286,15.417665),(3.007806,15.407665),(3.005842,15.389294),(3.005739,15.35232),(3.000158,15.339117),(2.950651,15.337463),(2.85412,15.334207),(2.757485,15.331081),(2.660953,15.327825),(2.564215,15.324595),(2.467684,15.321417),(2.371049,15.318239),(2.274517,15.314984),(2.177882,15.311831),(2.081351,15.308576),(1.984612,15.305398),(1.888081,15.302194),(1.791446,15.29899),(1.694914,15.295734),(1.598383,15.292608),(1.501748,15.289352),(1.405216,15.286148),(1.331526,15.283616),(1.297832,15.275735),(1.270857,15.259897),(1.203161,15.198789),(1.123063,15.126313),(1.057537,15.067118),(0.973718,14.991257),(0.949327,14.979552),(0.922145,14.973971),(0.769183,14.969062),(0.739934,14.958339),(0.711408,14.947461),(0.683813,14.940872),(0.670067,14.939735),(0.514831,14.993556),(0.483515,14.992109),(0.418791,14.969888),(0.387035,14.963248),(0.353187,14.963429),(0.221257,14.995933),(0.213196,14.985417),(0.212782,14.960716),(0.218467,14.910977),(-0.033197,14.995933),(-0.033404,14.995933),(-0.166988,15.049677),(-0.236699,15.065619),(-0.299176,15.054741),(-0.361472,15.017741),(-0.397671,15.002135),(-0.425732,15.0026),(-0.435602,15.015157),(-0.458649,15.075463),(-0.4679,15.079908),(-0.500343,15.079721),(-0.719719,15.078461),(-0.752895,15.069727),(-0.771272,15.056619),(-0.782816,15.048385),(-0.836249,14.996088),(-0.836404,14.995985),(-1.043138,14.818062),(-1.043317,14.817908),(-1.07856,14.796359),(-1.116697,14.780443),(-1.307745,14.734657),(-1.350275,14.714969),(-1.69692,14.496119),(-1.766683,14.483174),(-1.836963,14.479582),(-1.919129,14.485861),(-1.967033,14.483742),(-1.997057,14.470694),(-2.005015,14.444236),(-2.023412,14.198643),(-2.027288,14.188127),(-2.035918,14.181229),(-2.085008,14.159668),(-2.104337,14.151179),(-2.11984,14.148776),(-2.151879,14.15614),(-2.385612,14.264712),(-2.46168,14.280912),(-2.516147,14.267864),(-2.597382,14.222311),(-2.619758,14.203604),(-2.676265,14.141729),(-2.69319,14.123196),(-2.840855,14.042994),(-2.867288,14.000542),(-2.861552,14.00173),(-2.858606,14.000232),(-2.85703,13.997777),(-2.855377,13.996175),(-2.89447,13.866571),(-2.912169,13.837942),(-2.928059,13.799624),(-2.923383,13.746035),(-2.895245,13.651648),(-2.930473,13.638523),(-2.965008,13.625655),(-2.97232,13.624467),(-2.979322,13.627567),(-2.986402,13.632063),(-2.994774,13.634983),(-3.019113,13.637515),(-3.026968,13.635345),(-3.038621,13.629583),(-3.05601,13.61085),(-3.067508,13.606767),(-3.07451,13.622141),(-3.077301,13.636791),(-3.08221,13.646222),(-3.100039,13.662733),(-3.125102,13.67728),(-3.146031,13.67666),(-3.167941,13.672112),(-3.195821,13.674903),(-3.240418,13.707976),(-3.267057,13.717045),(-3.286952,13.69777),(-3.266282,13.682344),(-3.259719,13.658315),(-3.263905,13.602659),(-3.269176,13.579095),(-3.283955,13.542198),(-3.248634,13.292781),(-3.262406,13.283919),(-3.42653,13.27423),(-3.448544,13.265781),(-3.452265,13.237514),(-3.440586,13.202839),(-3.439475,13.184855),(-3.449475,13.16881),(-3.461593,13.165761),(-3.473922,13.167601),(-3.516111,13.1739),(-3.538487,13.173409),(-3.54505,13.174494),(-3.553913,13.180256),(-3.569958,13.196767),(-3.576599,13.199299),(-3.589802,13.197387),(-3.596882,13.199402),(-3.678711,13.26175),(-3.724419,13.288828),(-3.798626,13.347145),(-3.817101,13.35469),(-3.85808,13.365516),(-3.8926,13.378926),(-3.908878,13.381975),(-3.928825,13.38027),(-3.948411,13.380838),(-3.971613,13.386755),(-3.984042,13.39647),(-3.971613,13.40833),(-3.963009,13.413162),(-3.942726,13.431946),(-3.934923,13.434892),(-3.925544,13.436313),(-3.918593,13.440318),(-3.917973,13.451066),(-3.923399,13.453754),(-3.94748,13.458198),(-3.956059,13.46166),(-3.959288,13.467939),(-3.964482,13.487808),(-3.965468,13.489217),(-3.970321,13.496154),(-3.974094,13.498945),(-3.977737,13.499926),(-3.981251,13.49897),(-3.984481,13.496051),(-3.997968,13.480522),(-3.992232,13.476233),(-3.979468,13.47538),(-3.971613,13.469954),(-3.97691,13.460627),(-3.990217,13.449129),(-4.005358,13.439517),(-4.016029,13.43577),(-4.028199,13.431533),(-4.043495,13.414454),(-4.052539,13.409002),(-4.079307,13.402129),(-4.087549,13.397995),(-4.104267,13.382698),(-4.114654,13.364534),(-4.131578,13.323839),(-4.147106,13.299603),(-4.16602,13.278157),(-4.176872,13.272369),(-4.186458,13.272834),(-4.195837,13.274695),(-4.205501,13.272989),(-4.220668,13.262835),(-4.23878,13.246789),(-4.253508,13.22909),(-4.258107,13.21413),(-4.251209,13.204751),(-4.230021,13.193149),(-4.227773,13.184442),(-4.234801,13.175605),(-4.24617,13.172479),(-4.310197,13.175553),(-4.327044,13.168551),(-4.339601,13.135298),(-4.345906,13.127133),(-4.351073,13.118322),(-4.351073,13.106075),(-4.345363,13.097057),(-4.316398,13.070857),(-4.289733,13.01347),(-4.276918,12.996236),(-4.24785,12.971948),(-4.229324,12.948952),(-4.221314,12.921254),(-4.224647,12.864513),(-4.211883,12.819193),(-4.213278,12.807178),(-4.23413,12.74235),(-4.243974,12.729922),(-4.245472,12.728936),(-4.257642,12.72093),(-4.276298,12.714677),(-4.297227,12.712145),(-4.310921,12.716021),(-4.339446,12.73279),(-4.36885,12.738991),(-4.402026,12.736562),(-4.435254,12.728423),(-4.464296,12.717261),(-4.46437,12.717251),(-4.474942,12.715762),(-4.481711,12.717416),(-4.485251,12.714677),(-4.491323,12.662794),(-4.482538,12.646258),(-4.439078,12.613288),(-4.42249,12.597217),(-4.401406,12.550398),(-4.392647,12.537427),(-4.387247,12.533965),(-4.386885,12.530192),(-4.393448,12.516446),(-4.407607,12.497946),(-4.426624,12.479653),(-4.443678,12.459447),(-4.451636,12.435314),(-4.415359,12.350307),(-4.406135,12.307467),(-4.435823,12.301731),(-4.444814,12.309431),(-4.452669,12.320283),(-4.462488,12.328189),(-4.47719,12.327311),(-4.488171,12.320696),(-4.490703,12.313823),(-4.490496,12.305762),(-4.49401,12.290104),(-4.492253,12.278838),(-4.493545,12.272844),(-4.497421,12.26871),(-4.585529,12.197241),(-4.560466,12.149492),(-4.570233,12.138898),(-4.600309,12.137555),(-4.636121,12.117556),(-4.646663,12.098022),(-4.648678,12.081951),(-4.652737,12.071627),(-4.653716,12.069135),(-4.673586,12.05973),(-4.683715,12.059058),(-4.704023,12.061487),(-4.714617,12.059007),(-4.725133,12.050118),(-4.732281,12.037263),(-4.738983,12.02521),(-4.746889,12.015082),(-4.761281,12.006607),(-4.789522,12.001801),(-4.806808,11.996271),(-4.826238,12.012705),(-4.847245,12.013066),(-4.889955,11.997925),(-4.910677,11.998132),(-4.932071,12.003144),(-4.95375,12.005108),(-4.975454,11.996271),(-4.984523,11.988572),(-4.994238,11.983249),(-5.004703,11.980665),(-5.016046,11.981285),(-5.034313,11.97927),(-5.073768,11.98051),(-5.089323,11.97927),(-5.103947,11.972759),(-5.136994,11.953122),(-5.167819,11.943665),(-5.182909,11.930901),(-5.209471,11.901497),(-5.248228,11.870439),(-5.268227,11.843051),(-5.275151,11.837418),(-5.282489,11.836436),(-5.288019,11.839123),(-5.293445,11.843206),(-5.300163,11.846358),(-5.306519,11.846048),(-5.311066,11.842896),(-5.314839,11.83933),(-5.319025,11.837418),(-5.353803,11.831734),(-5.357058,11.829615),(-5.364552,11.82176),(-5.369874,11.820158),(-5.37561,11.822225),(-5.387909,11.830804),(-5.392302,11.832354),(-5.405944,11.830442),(-5.412352,11.82853),(-5.412197,11.823569),(-5.405996,11.812303),(-5.400415,11.810185),(-5.374525,11.79525),(-5.370649,11.791064),(-5.353286,11.794527),(-5.344656,11.793907),(-5.334269,11.790806),(-5.314787,11.782486),(-5.298974,11.772668),(-5.28714,11.758922),(-5.279544,11.739026),(-5.278562,11.721456),(-5.280681,11.699494),(-5.289259,11.660426),(-5.294788,11.650298),(-5.302281,11.639859),(-5.307811,11.629007),(-5.307604,11.617638),(-5.299956,11.606424),(-5.290964,11.604461),(-5.280732,11.605753),(-5.269467,11.604099),(-5.233914,11.575987),(-5.224663,11.54077),(-5.224612,11.45801),(-5.219186,11.435143),(-5.218307,11.422198),(-5.22642,11.41349),(-5.248176,11.403181),(-5.263472,11.390184),(-5.267865,11.37202),(-5.26094,11.267737),(-5.262852,11.251588),(-5.275255,11.230711),(-5.306726,11.199395),(-5.316182,11.176322),(-5.322332,11.135549),(-5.332512,11.12157),(-5.356283,11.106894),(-5.373026,11.099763),(-5.390131,11.094699),(-5.489764,11.082115),(-5.504233,11.072684),(-5.507334,11.056949),(-5.505602,11.047985),(-5.495603,10.996203),(-5.501804,10.97021),(-5.479118,10.975533),(-5.465579,10.947886),(-5.447027,10.877993),(-5.43509,10.856909),(-5.431421,10.844688),(-5.445322,10.786991),(-5.445012,10.778568),(-5.446459,10.768698),(-5.45297,10.763969),(-5.461186,10.760042),(-5.467956,10.752678),(-5.473072,10.737873),(-5.482632,10.684233),(-5.482115,10.675215),(-5.477154,10.658808),(-5.475914,10.650385),(-5.460566,10.644571),(-5.470488,10.635605),(-5.477413,10.61956),(-5.47855,10.604238),(-5.471057,10.597365),(-5.468111,10.58783),(-5.48165,10.535327),(-5.505163,10.499179),(-5.508987,10.490937),(-5.5171,10.439984),(-5.522578,10.425489),(-5.549708,10.435643),(-5.571722,10.44988),(-5.584383,10.454247),(-5.594047,10.453937),(-5.621797,10.446935),(-5.677556,10.441508),(-5.806282,10.413474),(-5.87806,10.37606),(-5.893305,10.365027),(-5.904829,10.348284),(-5.908963,10.330068),(-5.904725,10.275291),(-5.916404,10.266635),(-5.963016,10.282733),(-5.974747,10.273638),(-5.978313,10.248626),(-5.985237,10.227284),(-5.997123,10.208189),(-6.015933,10.18987),(-6.105333,10.189276),(-6.180884,10.215838),(-6.19432,10.224183),(-6.214009,10.247851),(-6.224809,10.255034),(-6.240002,10.251933),(-6.243309,10.289011),(-6.240674,10.302576),(-6.228323,10.31609),(-6.225894,10.314488),(-6.22114,10.310354),(-6.21437,10.307796),(-6.206154,10.310819),(-6.203984,10.316038),(-6.205379,10.322627),(-6.207808,10.328518),(-6.208324,10.33198),(-6.183933,10.358852),(-6.198764,10.370815),(-6.195664,10.389212),(-6.180574,10.420218),(-6.196904,10.439157),(-6.200366,10.448769),(-6.198196,10.457399),(-6.187499,10.475408),(-6.185793,10.481325),(-6.190289,10.491144),(-6.196129,10.494684),(-6.20419,10.494373),(-6.215662,10.492616),(-6.222432,10.494632),(-6.228168,10.500575),(-6.233491,10.507577),(-6.238658,10.512744),(-6.24765,10.51383),(-6.256022,10.511608),(-6.261293,10.514088),(-6.260931,10.529152),(-6.256228,10.537239),(-6.237935,10.551941),(-6.231372,10.560235),(-6.230028,10.572043),(-6.232871,10.58411),(-6.238658,10.597985),(-6.2307,10.606925),(-6.219797,10.61186),(-6.209771,10.617751),(-6.204449,10.629611),(-6.207808,10.637543),(-6.216903,10.645088),(-6.227755,10.65085),(-6.246617,10.657619),(-6.249459,10.665603),(-6.244446,10.706867),(-6.245841,10.720742),(-6.25597,10.726478),(-6.28,10.723274),(-6.303564,10.713171),(-6.322219,10.700485),(-6.341701,10.691442),(-6.367539,10.692268),(-6.388055,10.695524),(-6.409811,10.694387),(-6.427019,10.686093),(-6.433995,10.667929),(-6.435856,10.628706),(-6.430688,10.61496),(-6.419061,10.6077),(-6.406658,10.60186),(-6.399217,10.592481),(-6.402783,10.569123),(-6.42149,10.556592),(-6.445984,10.552251),(-6.466965,10.553543),(-6.52138,10.564214),(-6.537555,10.569692),(-6.578069,10.591138),(-6.598585,10.604909),(-6.613623,10.618474),(-6.633983,10.65302),(-6.647109,10.66134),(-6.669175,10.654157),(-6.667883,10.650901),(-6.681939,10.627389),(-6.684213,10.626562),(-6.688967,10.607855),(-6.691086,10.589432),(-6.689225,10.580931),(-6.685608,10.574007),(-6.682817,10.565971),(-6.687778,10.52047),(-6.697545,10.488301),(-6.695788,10.471197),(-6.683076,10.451689),(-6.675634,10.447865),(-6.664627,10.444557),(-6.653982,10.440036),(-6.647781,10.432698),(-6.647936,10.421871),(-6.653052,10.413061),(-6.659615,10.405361),(-6.663955,10.397997),(-6.66827,10.36584),(-6.668916,10.361022),(-6.677391,10.349034),(-6.700336,10.34167),(-6.722557,10.344047),(-6.760229,10.366113),(-6.782088,10.373037),(-6.803895,10.370867),(-6.862755,10.34384),(-6.88203,10.341256),(-6.938719,10.348336),(-6.96125,10.344874),(-6.973136,10.33229),(-6.997113,10.25276),(-6.994891,10.239402),(-6.984298,10.230384),(-6.971172,10.222065),(-6.961612,10.211032),(-6.95939,10.185503),(-6.971637,10.164419),(-6.992411,10.149097),(-7.015769,10.140881),(-7.040522,10.140054),(-7.058247,10.15597),(-7.073129,10.178527),(-7.089459,10.19757),(-7.106357,10.207259),(-7.204749,10.234493),(-7.284021,10.246559),(-7.346963,10.247903),(-7.363396,10.252192),(-7.371819,10.264517),(-7.372698,10.288598),(-7.366342,10.329448),(-7.372336,10.34477),(-7.395022,10.346217),(-7.422204,10.336089),(-7.431402,10.333737),(-7.444425,10.334202),(-7.445768,10.340378),(-7.443288,10.349964),(-7.444683,10.360816),(-7.461478,10.388643),(-7.465457,10.398617),(-7.46618,10.40748),(-7.465354,10.417996),(-7.46618,10.429494),(-7.471606,10.441095),(-7.480236,10.450293),(-7.489848,10.456107),(-7.501475,10.458639),(-7.515841,10.457812),(-7.539613,10.427633),(-7.547416,10.420502),(-7.556382,10.417272),(-7.574339,10.417531),(-7.584416,10.416523),(-7.607774,10.418719),(-7.643689,10.440475),(-7.664411,10.437142),(-7.670044,10.42877),(-7.673971,10.416394),(-7.679036,10.405929),(-7.687924,10.403087),(-7.69981,10.404534),(-7.708646,10.402467),(-7.722289,10.391331),(-7.722289,10.384484),(-7.726785,10.382882),(-7.736035,10.378282),(-7.723994,10.363606),(-7.735337,10.347096),(-7.757429,10.334047),(-7.777583,10.329862),(-7.764405,10.312912),(-7.776342,10.292732),(-7.778254,10.27834),(-7.783835,10.270434),(-7.791225,10.265783),(-7.797736,10.260279),(-7.80125,10.249453),(-7.805074,10.241831),(-7.829001,10.211652),(-7.838664,10.203719),(-7.847423,10.197932),(-7.857216,10.194418),(-7.882847,10.190826),(-7.892253,10.184832),(-7.910959,10.169406),(-7.929046,10.160957),(-7.949872,10.155428),(-7.970904,10.155066),(-7.989663,10.161991),(-7.96801,10.209998),(-7.962584,10.233252),(-7.9646,10.258936),(-7.971214,10.281802),(-7.981549,10.307021),(-7.996277,10.32826),(-8.015708,10.339344),(-8.070071,10.341876),(-8.096116,10.346734),(-8.115495,10.360247),(-8.120921,10.373037),(-8.122109,10.399496),(-8.125417,10.411665),(-8.135442,10.427323),(-8.142056,10.429649),(-8.150583,10.424533),(-8.165983,10.418099),(-8.18903,10.413629),(-8.210579,10.414068),(-8.228873,10.423422),(-8.253884,10.469827),(-8.284425,10.511246),(-8.295535,10.537446),(-8.311142,10.735289),(-8.315586,10.753092),(-8.341941,10.763246),(-8.339822,10.783865),(-8.318893,10.83952),(-8.311814,10.847246),(-8.305871,10.856289),(-8.303649,10.874893),(-8.305096,10.978246),(-8.311142,11.000803),(-8.326593,11.02385),(-8.347315,11.043126),(-8.369329,11.054236),(-8.381318,11.053926),(-8.400645,11.043332),(-8.410464,11.041911),(-8.429067,11.048293),(-8.438162,11.04974),(-8.448704,11.047105),(-8.469013,11.046381),(-8.489064,11.052737),(-8.505755,11.052789),(-8.531645,10.999046),(-8.541567,10.983775),(-8.55955,10.971605),(-8.575622,10.966748),(-8.593347,10.964707),(-8.611382,10.965146),(-8.628228,10.967833),(-8.647969,10.965818),(-8.665229,10.956955),(-8.680938,10.952304),(-8.696234,10.962743),(-8.70073,10.978452),(-8.696906,10.996048),(-8.68869,11.012559),(-8.679801,11.025194),(-8.63368,11.071289),(-8.623112,11.091262),(-8.622389,11.102605),(-8.624921,11.112191),(-8.624611,11.122035),(-8.615413,11.13405),(-8.607222,11.137151),(-8.586991,11.137461),(-8.578154,11.14206),(-8.569679,11.157796),(-8.563943,11.193426),(-8.551799,11.210738),(-8.544616,11.214975),(-8.5241,11.22376),(-8.515625,11.229057),(-8.51206,11.230298),(-8.502138,11.231176),(-8.498779,11.233501),(-8.497022,11.24071),(-8.499244,11.246705),(-8.502345,11.25164),(-8.50312,11.255412),(-8.492578,11.277788),(-8.483224,11.28492),(-8.44457,11.276264),(-8.426897,11.274197),(-8.406691,11.275282),(-8.388398,11.281406),(-8.376719,11.294661),(-8.371035,11.32055),(-8.381267,11.325098),(-8.39956,11.322152),(-8.417957,11.325511),(-8.42607,11.340472),(-8.414133,11.353468),(-8.393359,11.363003),(-8.375324,11.367654),(-8.397648,11.385895),(-8.427724,11.40008),(-8.488134,11.418193),(-8.51578,11.418968),(-8.527563,11.423387),(-8.535521,11.438088),(-8.539425,11.456312),(-8.543996,11.477647),(-8.549577,11.490127),(-8.575208,11.470283),(-8.602183,11.472092),(-8.656754,11.496431),(-8.667967,11.510875),(-8.679026,11.52984),(-8.687346,11.55002),(-8.690705,11.568029),(-8.712874,11.640066),(-8.726517,11.649471),(-8.740366,11.646164),(-8.756541,11.638826),(-8.777211,11.635932),(-8.795195,11.641926),(-8.81168,11.651745),(-8.828526,11.659186),(-8.847233,11.657946),(-8.797004,11.913227),(-8.79783,11.925113),(-8.803205,11.934932),(-8.809974,11.94444),(-8.814909,11.955395),(-8.816434,11.976841),(-8.815452,11.995703),(-8.819793,12.012653),(-8.837983,12.028156),(-8.858757,12.031825),(-8.88165,12.029189),(-8.903715,12.030378),(-8.922112,12.045364),(-8.926298,12.064278),(-8.921957,12.086033),(-8.906971,12.125359),(-8.906041,12.147735),(-8.913896,12.169853),(-8.927021,12.187371),(-8.942008,12.195846),(-8.961696,12.187991),(-8.97384,12.187578),(-8.981178,12.198946),(-8.984537,12.20892),(-8.994278,12.224836),(-8.99756,12.23481),(-8.995906,12.26101),(-8.985261,12.282094),(-8.972135,12.301989),(-8.963092,12.32421),(-8.964745,12.346276),(-8.975907,12.368704),(-8.993839,12.387514),(-9.027765,12.404751),(-9.147318,12.465493),(-9.189486,12.479239),(-9.265864,12.495207),(-9.278938,12.494432),(-9.286379,12.488128),(-9.294079,12.483787),(-9.308187,12.488955),(-9.321519,12.496603),(-9.327307,12.497533),(-9.33418,12.496293),(-9.350872,12.484614),(-9.398155,12.473245),(-9.412573,12.455365),(-9.407251,12.445133),(-9.392781,12.428028),(-9.374694,12.41113),(-9.347151,12.396454),(-9.344464,12.389012),(-9.343947,12.380124),(-9.339167,12.370616),(-9.331648,12.366326),(-9.314801,12.362657),(-9.309169,12.357748),(-9.308497,12.345036),(-9.312889,12.32607),(-9.321829,12.310206),(-9.335007,12.306537),(-9.331906,12.282766),(-9.336816,12.269846),(-9.360173,12.246644),(-9.421513,12.257082),(-9.438877,12.254757),(-9.479598,12.235843),(-9.495876,12.224681),(-9.515616,12.20706),(-9.628167,12.170163),(-9.66,12.149802),(-9.675141,12.133989),(-9.682273,12.118538),(-9.683668,12.101485),(-9.681549,12.080814),(-9.686303,12.068153),(-9.698809,12.050997),(-9.722994,12.025417),(-9.778649,12.026657),(-9.796632,12.032342),(-9.806813,12.038439),(-9.823039,12.050893),(-9.833684,12.055131),(-9.840661,12.055028),(-9.861125,12.051875),(-9.870736,12.052082),(-9.888772,12.060557),(-9.923188,12.087584),(-9.941533,12.092751),(-9.961584,12.096007),(-9.996414,12.114145),(-10.015534,12.119881),(-10.027678,12.125669),(-10.04809,12.141379),(-10.060182,12.147528),(-10.070363,12.149854),(-10.087002,12.150577),(-10.09034,12.151584),(-10.097286,12.153678),(-10.105192,12.158639),(-10.118835,12.170163),(-10.128654,12.174503),(-10.135837,12.175485),(-10.160228,12.174503),(-10.179762,12.178224),(-10.21485,12.194606),(-10.247819,12.201065),(-10.254124,12.207215),(-10.258981,12.213778),(-10.26694,12.217808),(-10.304974,12.202874),(-10.31102,12.19104),(-10.313603,12.191143),(-10.316497,12.196879),(-10.323577,12.201995),(-10.325592,12.199773),(-10.329882,12.19042),(-10.339338,12.185459),(-10.348433,12.184787),(-10.354118,12.18086),(-10.353394,12.166494),(-10.368225,12.171506),(-10.37484,12.174814),(-10.381403,12.180136),(-10.39458,12.17161),(-10.428945,12.143394),(-10.435973,12.135178),(-10.445688,12.120708),(-10.493489,12.124429),(-10.511628,12.118073),(-10.512661,12.108461),(-10.504445,12.088359),(-10.507959,12.084586),(-10.517312,12.080349),(-10.517674,12.070737),(-10.514883,12.060144),(-10.51509,12.052909),(-10.535399,12.038698),(-10.555449,12.029344),(-10.566508,12.017252),(-10.55948,11.995186),(-10.574983,11.990329),(-10.595653,11.97989),(-10.613792,11.967694),(-10.621543,11.957669),(-10.625264,11.943872),(-10.634152,11.925423),(-10.645159,11.90837),(-10.655081,11.898965),(-10.669964,11.89204),(-10.711357,11.890386),(-10.740347,11.919894),(-10.781017,11.99622),(-10.807785,12.022833),(-10.812436,12.033427),(-10.811041,12.04154),(-10.801222,12.059213),(-10.798535,12.068102),(-10.804943,12.095955),(-10.822926,12.110476),(-10.846387,12.12138),(-10.869642,12.138433),(-10.909794,12.200135),(-10.927675,12.212951),(-10.952117,12.219462),(-10.972013,12.218583),(-11.015473,12.204424),(-11.038779,12.205251),(-11.052318,12.202512),(-11.059811,12.192435),(-11.072782,12.149492),(-11.084306,12.136573),(-11.116604,12.113732),(-11.117477,12.112702),(-11.122081,12.107272),(-11.136292,12.085103),(-11.14425,12.081331),(-11.152725,12.080866),(-11.159392,12.077507),(-11.165644,12.04862),(-11.176342,12.029499),(-11.191173,12.01441),(-11.206986,12.010172),(-11.223315,12.009294),(-11.25551,11.996168),(-11.274992,11.996271),(-11.294681,12.003093),(-11.315144,12.013325),(-11.334678,12.026192),(-11.351318,12.040403),(-11.391729,12.093733),(-11.409557,12.107893),(-11.470794,12.134609),(-11.489604,12.151197),(-11.507794,12.191608),(-11.491671,12.220909),(-11.463249,12.249951),(-11.444594,12.289484),(-11.444904,12.312531),(-11.451467,12.352115),(-11.447229,12.374543),(-11.431675,12.382966),(-11.407594,12.381933),(-11.388628,12.384465),(-11.388422,12.403895),(-11.386561,12.430457),(-11.379792,12.456864),(-11.377776,12.480221),(-11.390437,12.497533),(-11.392607,12.491848),(-11.400514,12.479549),(-11.402374,12.488541),(-11.408679,12.504923),(-11.410849,12.513346),(-11.416999,12.529314),(-11.42475,12.526368),(-11.430383,12.515723),(-11.429814,12.508333),(-11.437876,12.515361),(-11.449451,12.535205),(-11.456686,12.539753),(-11.467797,12.543525),(-11.460614,12.549829),(-11.43984,12.559286),(-11.437359,12.570758),(-11.438599,12.577993),(-11.440925,12.584866),(-11.441803,12.595511),(-11.436636,12.627086),(-11.436842,12.637473),(-11.441752,12.647756),(-11.449555,12.653182),(-11.457151,12.655973),(-11.460975,12.658402),(-11.461905,12.673129),(-11.455756,12.680622),(-11.447953,12.685945),(-11.437359,12.712248),(-11.424182,12.716383),(-11.410332,12.718811),(-11.402168,12.73173),(-11.404028,12.741213),(-11.41457,12.759713),(-11.41457,12.764209),(-11.407594,12.77147),(-11.405423,12.782348),(-11.406353,12.794285),(-11.408472,12.804491),(-11.411676,12.812888),(-11.416017,12.820071),(-11.419686,12.824671),(-11.420978,12.825575),(-11.423458,12.826402),(-11.428161,12.827022),(-11.432088,12.829347),(-11.432502,12.835523),(-11.430021,12.841052),(-11.427024,12.845238),(-11.425474,12.849553),(-11.428626,12.865805),(-11.424905,12.876166),(-11.419324,12.886501),(-11.415138,12.896656),(-11.417309,12.901953),(-11.423096,12.907069),(-11.427954,12.912495),(-11.427231,12.918618),(-11.420668,12.922029),(-11.413588,12.919703),(-11.407283,12.916293),(-11.402891,12.916396),(-11.394209,12.923321),(-11.387801,12.927119),(-11.384443,12.933114),(-11.386561,12.959107),(-11.388835,12.970553),(-11.393434,12.980811),(-11.401806,12.989492),(-11.404235,12.972103),(-11.411521,12.960709),(-11.423096,12.956006),(-11.438703,12.9589),(-11.435964,12.968848),(-11.433535,12.988356),(-11.433432,12.99696),(-11.430124,13.00099),(-11.424543,13.005202),(-11.422993,13.009414),(-11.431675,13.01347),(-11.435654,13.016726),(-11.43953,13.023005),(-11.445162,13.035226),(-11.45896,13.054682),(-11.456945,13.060444),(-11.451415,13.068067),(-11.450278,13.075095),(-11.46082,13.079177),(-11.477098,13.083285),(-11.489707,13.090778),(-11.501541,13.099589),(-11.515391,13.107754),(-11.533064,13.111811),(-11.538077,13.128011),(-11.538542,13.16297),(-11.547843,13.176199),(-11.559729,13.186922),(-11.569341,13.199195),(-11.572338,13.217101),(-11.568566,13.227902),(-11.555802,13.245911),(-11.554251,13.256866),(-11.55854,13.266504),(-11.581175,13.286141),(-11.605824,13.319653),(-11.613007,13.339058),(-11.613111,13.360813),(-11.620345,13.357532),(-11.632903,13.35438),(-11.643962,13.355594),(-11.646597,13.365516),(-11.645253,13.379753),(-11.649801,13.38412),(-11.674967,13.383215),(-11.694501,13.379184),(-11.70401,13.380373),(-11.714035,13.388099),(-11.719771,13.397891),(-11.726902,13.405798),(-11.741217,13.407813),(-11.756513,13.399803),(-11.76659,13.383164),(-11.772533,13.362674),(-11.775116,13.343347),(-11.817078,13.312341),(-11.822555,13.306708),(-11.828136,13.307277),(-11.840022,13.316889),(-11.844569,13.323839),(-11.850771,13.34314),(-11.854595,13.350556),(-11.862346,13.356085),(-11.888804,13.368875),(-11.897589,13.371175),(-11.899967,13.382182),(-11.885652,13.43577),(-11.883482,13.453547),(-11.890406,13.465148),(-11.898468,13.468507),(-11.907718,13.469644),(-11.918157,13.474708),(-11.923841,13.481142),(-11.932729,13.496051),(-11.939034,13.50295),(-11.996808,13.544161),(-12.015153,13.564444),(-12.037323,13.598267),(-12.049828,13.634363),(-12.070395,13.673249),(-12.083883,13.692964),(-12.097629,13.70441),(-12.012311,13.751177),(-11.978825,13.784819),(-11.962908,13.829467),(-11.956966,13.870007),(-11.956811,13.890342),(-11.962908,13.909152),(-11.980375,13.929642),(-11.99903,13.943569),(-12.015308,13.959304),(-12.025334,13.985246),(-12.02523,14.026742),(-12.002751,14.103456),(-11.995413,14.144022),(-11.997583,14.166165),(-12.03448,14.250268),(-12.044299,14.264919),(-12.058355,14.274091),(-12.097784,14.28807),(-12.10998,14.298146),(-12.119591,14.310575),(-12.127808,14.324295),(-12.113029,14.329592),(-12.108688,14.34202),(-12.115612,14.35512),(-12.208837,14.388761),(-12.22173,14.391242),(-12.21987,14.434598),(-12.222428,14.454933),(-12.234184,14.498574),(-12.236432,14.519451),(-12.230386,14.536685),(-12.211472,14.547976),(-12.204186,14.554255),(-12.201447,14.565184),(-12.200465,14.577354),(-12.198398,14.58756),(-12.191008,14.597999),(-12.171526,14.616422),(-12.165945,14.625181),(-12.165015,14.641872),(-12.172611,14.647918),(-12.182792,14.650244),(-12.18982,14.655954),(-12.192197,14.666884),(-12.190285,14.673653),(-12.185841,14.68081),(-12.180311,14.692825),(-12.191422,14.693445),(-12.207648,14.696598),(-12.222815,14.70329),(-12.230489,14.714555),(-12.236613,14.728456),(-12.256508,14.745794),(-12.26413,14.774939),(-12.246793,14.767007),(-12.224779,14.763441),(-12.203256,14.76618),(-12.160209,14.781011),(-12.138867,14.784525),(-12.123674,14.776644),(-12.081454,14.740032),(-12.070395,14.734399),(-12.062334,14.746259),(-12.057063,14.761942),(-12.049673,14.767213),(-12.03107,14.762821),(-12.011071,14.762743),(-11.991486,14.76804),(-11.973967,14.779848),(-11.950299,14.811552),(-11.935985,14.823618),(-11.895729,14.832067),(-11.877487,14.841524),(-11.862605,14.854082),(-11.854698,14.867311),(-11.850822,14.876483),(-11.845603,14.883227),(-11.840487,14.888498),(-11.837076,14.893097),(-11.833097,14.894699),(-11.826069,14.89656),(-11.820385,14.89997),(-11.820437,14.906404),(-11.827361,14.915576),(-11.8301,14.920331),(-11.830927,14.926429),(-11.825811,14.962473),(-11.820592,14.980275),(-11.813202,14.995933),(-11.813202,14.996088),(-11.811083,15.014304),(-11.81377,15.028541),(-11.82116,15.039755),(-11.832839,15.049005),(-11.839195,15.049005),(-11.844466,15.044328),(-11.851029,15.041951),(-11.860899,15.049005),(-11.859142,15.054069),(-11.856765,15.084248),(-11.845138,15.124556),(-11.844208,15.13484),(-11.847308,15.148741),(-11.848187,15.158146),(-11.845655,15.179075),(-11.826793,15.23256),(-11.821057,15.280102),(-11.816923,15.296096),(-11.783488,15.365885),(-11.767675,15.437198),(-11.754859,15.469083),(-11.729434,15.496058),(-11.727057,15.506987),(-11.727057,15.541171),(-11.70174,15.537425),(-11.69755,15.536805),(-11.670833,15.528407),(-11.644117,15.526444),(-11.614868,15.541171),(-11.579211,15.577965),(-11.562571,15.587241),(-11.535958,15.594915),(-11.524279,15.600289),(-11.515391,15.610082),(-11.513117,15.615043),(-11.512342,15.620185),(-11.513117,15.625275),(-11.515391,15.630262),(-11.519576,15.632742),(-11.522987,15.635248),(-11.525623,15.638349),(-11.527328,15.642561),(-11.524382,15.643465),(-11.515391,15.644421),(-11.458236,15.632949),(-11.434827,15.623389),(-11.409557,15.602201),(-11.316901,15.471667),(-11.298195,15.451048),(-11.045393,15.270206),(-11.012734,15.239588),(-10.98519,15.187498),(-10.948035,15.151996),(-10.915169,15.102904),(-10.906229,15.119776),(-10.890881,15.176749),(-10.882251,15.192846),(-10.855328,15.217651),(-10.844062,15.234549),(-10.834399,15.271395),(-10.827216,15.287698),(-10.811041,15.303124),(-10.800189,15.306845),(-10.790112,15.306845),(-10.78174,15.308421),(-10.770682,15.324673),(-10.754507,15.333639),(-10.746807,15.341132),(-10.738901,15.362164),(-10.740141,15.403583),(-10.736368,15.422626),(-10.725361,15.433064),(-10.709187,15.433865),(-10.64888,15.424874),(-10.607126,15.424434),(-10.587385,15.427251),(-10.584746,15.428127),(-10.551522,15.439162),(-10.534985,15.441694),(-10.515503,15.437741),(-10.500414,15.440351),(-10.398405,15.438697),(-10.370758,15.433504),(-10.322647,15.437095),(-10.308746,15.435209),(-10.223221,15.402033),(-10.214488,15.401981),(-10.196091,15.405288),(-10.188391,15.40472),(-10.167979,15.395185),(-10.131599,15.372629),(-10.108293,15.365885),(-10.067624,15.364748),(-9.951403,15.381207),(-9.835596,15.371104),(-9.797356,15.380303),(-9.740719,15.4126),(-9.720823,15.42092),(-9.672118,15.430661),(-9.42389,15.440532),(-9.435879,15.495903),(-9.435879,15.495954),(-9.451847,15.559206),(-9.451692,15.588326),(-9.437016,15.614888),(-9.375935,15.685917),(-9.356453,15.697854),(-9.332423,15.687571),(-9.329322,15.655247),(-9.342965,15.586931),(-9.349218,15.495644),(-9.175843,15.495593),(-9.141379,15.495581),(-9.018592,15.495541),(-9.003216,15.495526),(-8.911157,15.495438),(-8.843409,15.495438),(-8.769253,15.495438),(-8.68931,15.495438),(-8.60394,15.495386),(-8.470202,15.495283),(-8.327006,15.495283),(-8.175801,15.495205),(-8.017826,15.495128),(-7.854529,15.495076),(-7.6872,15.495024),(-7.517237,15.494973),(-7.345878,15.494921),(-7.174519,15.494818),(-7.004503,15.494818),(-6.837175,15.49474),(-6.673929,15.494663),(-6.516006,15.494611),(-6.364749,15.494559),(-6.320145,15.494543),(-6.221605,15.494508),(-6.087815,15.494456),(-6.010817,15.494378),(-5.938212,15.494353),(-5.870257,15.494353),(-5.807315,15.494353),(-5.673163,15.494301),(-5.515964,15.494249),(-5.515602,15.494146),(-5.515292,15.494042),(-5.514982,15.493913),(-5.514672,15.493836),(-5.514362,15.493732),(-5.514051,15.493629),(-5.513741,15.493526),(-5.51338,15.493422),(-5.513121,15.493422),(-5.512966,15.493422),(-5.51276,15.493422),(-5.512553,15.493422),(-5.512294,15.493422),(-5.512191,15.493422),(-5.511933,15.493422),(-5.511726,15.493422),(-5.511623,15.493526),(-5.511519,15.493681),(-5.511364,15.493732),(-5.511313,15.493887),(-5.511261,15.493991),(-5.511106,15.494042),(-5.510951,15.494197),(-5.510951,15.494301),(-5.510951,15.494508),(-5.510951,15.494663),(-5.510951,15.494818),(-5.510951,15.495076),(-5.510951,15.495283),(-5.510951,15.495438),(-5.510951,15.49567),(-5.510951,15.495903),(-5.509917,15.500812),(-5.508987,15.505695),(-5.508761,15.5069),(-5.508057,15.510656),(-5.507075,15.51554),(-5.488368,15.612666),(-5.469558,15.709792),(-5.4508,15.80684),(-5.431989,15.90394),(-5.413799,15.998559),(-5.395454,16.093179),(-5.377212,16.187799),(-5.358919,16.282366),(-5.353286,16.311977),(-5.354268,16.318385),(-5.355301,16.324793),(-5.360107,16.329909),(-5.364913,16.334921),(-5.404963,16.36169),(-5.445942,16.389027),(-5.485371,16.415433),(-5.529709,16.444992),(-5.566503,16.469487),(-5.60588,16.495893),(-5.614562,16.511861),(-5.623347,16.527829),(-5.627688,16.568137),(-5.63761,16.658364),(-5.647583,16.748643),(-5.657454,16.838818),(-5.667324,16.929097),(-5.677297,17.019324),(-5.687168,17.109499),(-5.697038,17.199726),(-5.70696,17.289954),(-5.716933,17.380181),(-5.726803,17.470356),(-5.736674,17.560583),(-5.746647,17.650758),(-5.756466,17.741037),(-5.766387,17.831264),(-5.776361,17.92144),(-5.78618,18.011667),(-5.79636,18.104374),(-5.80654,18.197082),(-5.816772,18.289892),(-5.826952,18.382548),(-5.837081,18.475333),(-5.847364,18.568067),(-5.857545,18.6608),(-5.867673,18.753585),(-5.877802,18.846318),(-5.888034,18.939052),(-5.898214,19.031785),(-5.908394,19.12457),(-5.914802,19.182887),(-5.92121,19.24123),(-5.927566,19.299547),(-5.934026,19.357941),(-5.949115,19.495478),(-5.949115,19.495529),(-5.949115,19.495581),(-5.949115,19.495633),(-5.963533,19.620612),(-5.977847,19.745618),(-5.992162,19.870597),(-6.00658,19.995525),(-6.020946,20.12053),(-6.035363,20.245484),(-6.049678,20.370489),(-6.064095,20.495443),(-6.076756,20.603343),(-6.083577,20.661892),(-6.098615,20.790722),(-6.113705,20.919551),(-6.120526,20.978204),(-6.134634,21.098868),(-6.148741,21.219533),(-6.162901,21.340197),(-6.177008,21.460862),(-6.191116,21.581578),(-6.205224,21.702242),(-6.219383,21.822855),(-6.233491,21.943571),(-6.247598,22.064236),(-6.261706,22.184952),(-6.275865,22.305616),(-6.289973,22.426281),(-6.304132,22.546997),(-6.318188,22.667661),(-6.332296,22.7883),(-6.346455,22.908965),(-6.360563,23.029655),(-6.371634,23.123981),(-6.374722,23.150293),(-6.388778,23.270958),(-6.402938,23.391674),(-6.417045,23.512339),(-6.431205,23.633029),(-6.445312,23.753719),(-6.45942,23.874409),(-6.466603,23.93624),(-6.482571,24.072304),(-6.498436,24.208368),(-6.505671,24.270277),(-6.506338,24.275967),(-6.515851,24.357067),(-6.525514,24.436727),(-6.535178,24.51636),(-6.54479,24.595993),(-6.554453,24.675549),(-6.564168,24.755234),(-6.57378,24.834868),(-6.583392,24.914424),(-6.593107,24.994134),(-6.371674,24.994238),(-6.150188,24.994341),(-5.928755,24.994444),(-5.707321,24.9946),(-5.485939,24.994703),(-5.264454,24.994806),(-5.043072,24.99491),(-4.821613,24.995065),(-4.821536,24.994961),(-4.821355,24.994806),(-4.821226,24.994755)] +Montenegro [(19.054991,43.5067),(19.076386,43.507242),(19.096023,43.512901),(19.122791,43.535948),(19.147699,43.538145),(19.195345,43.532796),(19.175191,43.509619),(19.175914,43.480887),(19.192244,43.454532),(19.218392,43.438202),(19.355645,43.393063),(19.372285,43.384226),(19.414039,43.338389),(19.473054,43.293276),(19.485146,43.280098),(19.502303,43.251935),(19.512431,43.240643),(19.547845,43.218815),(19.549948,43.217518),(19.580851,43.187804),(19.598111,43.176203),(19.618678,43.168245),(19.663016,43.158504),(19.684617,43.156721),(19.69733,43.160209),(19.705494,43.166178),(19.713556,43.165609),(19.742805,43.126516),(19.761202,43.108739),(19.781769,43.096466),(19.805127,43.089955),(19.838303,43.088353),(19.872203,43.090368),(19.883778,43.095691),(19.907136,43.113261),(19.916644,43.117059),(19.92915,43.113907),(19.936591,43.105897),(19.942276,43.095639),(19.949821,43.085924),(19.959742,43.07856),(20.019894,43.047348),(20.054388,43.022221),(20.056344,43.020796),(20.116942,42.976654),(20.129464,42.973605),(20.14092,42.970815),(20.19456,42.966836),(20.223189,42.957663),(20.275589,42.929887),(20.337084,42.906969),(20.35362,42.890975),(20.355171,42.86617),(20.345352,42.827439),(20.264427,42.817258),(20.226083,42.806768),(20.217608,42.802737),(20.209236,42.791704),(20.208409,42.782093),(20.20996,42.772972),(20.208409,42.763282),(20.183398,42.742508),(20.149498,42.749872),(20.112085,42.766538),(20.076325,42.773437),(20.065059,42.769458),(20.055638,42.763866),(20.034673,42.751423),(20.026508,42.743206),(20.024751,42.723414),(20.03612,42.707989),(20.037006,42.707363),(20.094205,42.666983),(20.101956,42.656674),(20.10392,42.653108),(20.090587,42.627348),(20.079218,42.61125),(20.075704,42.603085),(20.075394,42.586962),(20.078185,42.572906),(20.077048,42.55991),(20.064956,42.546758),(20.039221,42.557765),(20.017723,42.546241),(19.981757,42.510765),(19.956332,42.505314),(19.907549,42.506399),(19.882745,42.493609),(19.882745,42.493557),(19.882538,42.493402),(19.873339,42.486839),(19.835512,42.470251),(19.829063,42.468723),(19.819596,42.466479),(19.801406,42.468132),(19.784456,42.474566),(19.75159,42.493402),(19.73433,42.524382),(19.730919,42.533658),(19.730299,42.540428),(19.732159,42.555543),(19.733916,42.562597),(19.741358,42.574405),(19.746009,42.579934),(19.747766,42.578901),(19.746885,42.58893),(19.746009,42.5989),(19.737534,42.624402),(19.722134,42.64608),(19.699293,42.654814),(19.676039,42.646675),(19.658037,42.634613),(19.647927,42.627838),(19.621779,42.604997),(19.605035,42.584844),(19.599248,42.571046),(19.593667,42.54482),(19.588086,42.532857),(19.575683,42.522341),(19.561317,42.516243),(19.549328,42.508595),(19.543747,42.493557),(19.543747,42.493402),(19.531655,42.474773),(19.517806,42.45803),(19.501476,42.444129),(19.481735,42.434491),(19.468403,42.41811),(19.417243,42.374107),(19.412076,42.368397),(19.402671,42.352015),(19.400707,42.344755),(19.401327,42.331061),(19.400397,42.325893),(19.304589,42.215099),(19.274926,42.191276),(19.272033,42.180682),(19.281956,42.164723),(19.282058,42.164559),(19.297767,42.151666),(19.355025,42.12004),(19.369998,42.106507),(19.372491,42.104253),(19.374869,42.094667),(19.356575,42.064798),(19.351718,42.047615),(19.350581,42.027823),(19.354198,42.008703),(19.363396,41.993639),(19.363707,41.993484),(19.371045,41.98656),(19.365774,41.969713),(19.359779,41.965993),(19.351614,41.965166),(19.346447,41.961548),(19.352854,41.938501),(19.350477,41.931369),(19.346653,41.926176),(19.345413,41.921189),(19.34593,41.914471),(19.34531,41.910337),(19.347687,41.906358),(19.356989,41.89964),(19.364947,41.888995),(19.364223,41.862846),(19.365122,41.852372),(19.365082,41.852362),(19.346934,41.863959),(19.310232,41.894517),(19.293224,41.900784),(19.220225,41.91885),(19.202973,41.927069),(19.178477,41.933905),(19.170258,41.937974),(19.162283,41.947943),(19.161957,41.953437),(19.163259,41.95897),(19.16033,41.96898),(19.15561,41.970649),(19.145274,41.977607),(19.13795,41.985297),(19.142914,41.988837),(19.143565,41.993069),(19.139171,42.042426),(19.08253,42.081204),(19.074718,42.095649),(19.083832,42.110093),(19.076427,42.115953),(19.063975,42.120185),(19.048676,42.140204),(19.01059,42.139553),(19.002126,42.150214),(18.999848,42.161078),(18.994802,42.163642),(18.988048,42.163723),(18.981619,42.16706),(18.968028,42.188707),(18.953461,42.199856),(18.913341,42.222235),(18.901703,42.237494),(18.894705,42.270494),(18.885997,42.284369),(18.871593,42.290473),(18.854015,42.290473),(18.838878,42.285712),(18.831391,42.277533),(18.826508,42.282213),(18.820486,42.286689),(18.817638,42.291164),(18.801524,42.284369),(18.793956,42.280341),(18.788748,42.276028),(18.786388,42.274115),(18.778575,42.27147),(18.771983,42.276923),(18.762462,42.291164),(18.729503,42.311591),(18.714529,42.325263),(18.714122,42.338935),(18.708344,42.342475),(18.699718,42.34984),(18.693614,42.353217),(18.693614,42.359442),(18.695811,42.373765),(18.678477,42.385972),(18.658539,42.386623),(18.653168,42.366278),(18.646495,42.366278),(18.638438,42.370307),(18.61671,42.368069),(18.612315,42.36994),(18.607107,42.37641),(18.5949,42.383734),(18.580903,42.389838),(18.570649,42.392971),(18.573009,42.39643),(18.57309,42.397528),(18.574067,42.398261),(18.578136,42.400377),(18.552989,42.413642),(18.545177,42.42357),(18.550141,42.435126),(18.562836,42.438544),(18.644949,42.418362),(18.656505,42.415513),(18.685557,42.403998),(18.70045,42.392971),(18.708181,42.408515),(18.706554,42.421047),(18.683849,42.458808),(18.683279,42.468085),(18.687348,42.482896),(18.654145,42.451809),(18.608653,42.44538),(18.502452,42.455634),(18.502452,42.448188),(18.521251,42.437974),(18.51295,42.409003),(18.529796,42.400377),(18.510265,42.40526),(18.49643,42.416327),(18.497816,42.431158),(18.492855,42.442372),(18.475492,42.449813),(18.467741,42.453379),(18.454925,42.464722),(18.444279,42.477925),(18.437148,42.493402),(18.436425,42.510765),(18.442006,42.543012),(18.437355,42.559212),(18.44769,42.566214),(18.458956,42.569728),(18.470531,42.569056),(18.481797,42.563682),(18.492132,42.564767),(18.495852,42.570865),(18.493682,42.579831),(18.486654,42.58983),(18.506798,42.59849),(18.517247,42.602982),(18.538434,42.618304),(18.542832,42.626429),(18.549596,42.638923),(18.550009,42.668069),(18.539674,42.695457),(18.522518,42.71189),(18.502157,42.727445),(18.46743,42.769199),(18.453891,42.793151),(18.444589,42.817078),(18.443659,42.834467),(18.453478,42.845655),(18.465984,42.852812),(18.473322,42.862553),(18.467637,42.881699),(18.443969,42.916865),(18.434047,42.936889),(18.433531,42.954201),(18.452754,42.993398),(18.483037,43.014637),(18.538951,43.023887),(18.598379,43.024455),(18.638996,43.020243),(18.621116,43.09644),(18.620599,43.122563),(18.621128,43.124578),(18.629074,43.154886),(18.645301,43.180182),(18.688399,43.224469),(18.664318,43.233176),(18.679511,43.24948),(18.807151,43.318029),(18.833506,43.324153),(18.830302,43.328157),(18.824618,43.337408),(18.821207,43.341387),(18.839604,43.34782),(18.899135,43.351903),(18.923423,43.346838),(18.950295,43.333067),(18.957323,43.325858),(18.957943,43.318985),(18.957116,43.311828),(18.959493,43.303456),(18.968589,43.292216),(18.988949,43.272941),(18.989506,43.272016),(18.992463,43.267102),(19.002902,43.273949),(19.01086,43.28232),(19.017164,43.291157),(19.022535,43.296546),(19.024709,43.298728),(19.036285,43.303327),(19.062536,43.304386),(19.069668,43.30883),(19.039592,43.350714),(19.00993,43.410995),(18.975306,43.444274),(18.968278,43.448098),(18.946264,43.443938),(18.946368,43.44908),(18.950708,43.457297),(18.951225,43.463808),(18.937893,43.478949),(18.930658,43.482282),(18.915258,43.485383),(18.905647,43.490628),(18.90513,43.499051),(18.911021,43.507268),(18.920529,43.512203),(18.938926,43.518947),(18.962697,43.5383),(18.977477,43.546232),(19.000214,43.547886),(19.01427,43.537783),(19.025019,43.523029),(19.038145,43.511015),(19.054991,43.5067)] +Mongolia [(100.005968,51.731727),(100.098262,51.738652),(100.210296,51.726224),(100.510898,51.726895),(100.571101,51.704804),(100.638901,51.69204),(101.069469,51.553444),(101.117011,51.527838),(101.158766,51.521223),(101.200727,51.520913),(101.241551,51.515177),(101.279688,51.492595),(101.315448,51.463604),(101.350175,51.450452),(101.387795,51.450556),(101.482053,51.47288),(101.526392,51.475826),(101.569852,51.470089),(101.643697,51.450091),(101.696614,51.452442),(101.723692,51.450582),(101.742192,51.444845),(101.795316,51.419395),(101.821774,51.413581),(101.901873,51.417069),(101.920683,51.413039),(101.955616,51.393789),(101.97422,51.387226),(102.051217,51.383609),(102.074679,51.374824),(102.091267,51.364644),(102.183871,51.323897),(102.193379,51.307205),(102.179272,51.286431),(102.135295,51.240284),(102.133796,51.230853),(102.139688,51.220931),(102.147646,51.202638),(102.150023,51.185404),(102.148783,51.16879),(102.142168,51.134839),(102.146509,51.097632),(102.165371,51.057556),(102.191984,51.020582),(102.219373,50.992651),(102.229656,50.979267),(102.231051,50.964901),(102.225574,50.950741),(102.215238,50.938184),(102.239475,50.912294),(102.231878,50.88475),(102.212758,50.856742),(102.203043,50.829353),(102.216789,50.798864),(102.245676,50.778762),(102.307533,50.748221),(102.329392,50.718817),(102.320245,50.696131),(102.296836,50.674944),(102.275907,50.650191),(102.271566,50.620942),(102.282418,50.590608),(102.302933,50.563892),(102.327635,50.545546),(102.372697,50.533661),(102.471864,50.524927),(102.513101,50.503688),(102.586585,50.41527),(102.617798,50.399302),(102.619529,50.399009),(102.762595,50.374807),(102.887135,50.314966),(102.927391,50.302977),(102.974107,50.295639),(103.130015,50.309127),(103.20169,50.296983),(103.240189,50.244376),(103.253108,50.214972),(103.276776,50.197557),(103.306593,50.190426),(103.407104,50.197299),(103.43873,50.192596),(103.473973,50.182003),(103.533918,50.153787),(103.601407,50.13353),(103.66807,50.131205),(103.785272,50.186137),(103.845423,50.18469),(103.973994,50.148723),(104.03952,50.141178),(104.121685,50.148258),(104.198115,50.170014),(104.247466,50.206601),(104.268963,50.228873),(104.321363,50.254298),(104.349502,50.271868),(104.377225,50.28918),(104.405802,50.300548),(104.474119,50.313261),(104.576955,50.309488),(104.610441,50.314191),(104.632455,50.323751),(104.671884,50.348297),(104.697051,50.353207),(104.789552,50.352638),(104.81849,50.358323),(104.879004,50.383902),(104.901586,50.389794),(104.934246,50.393256),(105.050104,50.383489),(105.078216,50.385143),(105.10576,50.390052),(105.13134,50.398062),(105.240739,50.458006),(105.278462,50.472579),(105.32864,50.476455),(105.639371,50.421936),(105.779259,50.428913),(105.806544,50.424365),(105.854706,50.410929),(105.904212,50.403798),(105.958059,50.403591),(105.984879,50.399767),(106.038933,50.371397),(106.043274,50.364782),(106.043274,50.355532),(106.044462,50.346024),(106.048183,50.338117),(106.055934,50.33357),(106.135309,50.32711),(106.160941,50.319617),(106.201197,50.299722),(106.222281,50.292384),(106.244863,50.290265),(106.439787,50.327937),(106.548204,50.335792),(106.656518,50.327007),(106.746176,50.307421),(106.788293,50.291505),(106.936449,50.209133),(106.973656,50.196369),(106.982699,50.187119),(106.997996,50.149808),(107.006057,50.140196),(107.026211,50.125727),(107.033601,50.118802),(107.039233,50.10826),(107.048432,50.084644),(107.057113,50.074102),(107.070291,50.066247),(107.100728,50.056791),(107.114629,50.050486),(107.164497,50.01886),(107.192196,50.006044),(107.224907,49.997053),(107.254879,49.998913),(107.280407,50.007595),(107.30485,50.010023),(107.364071,49.976124),(107.729321,49.971938),(107.744604,49.967353),(107.750508,49.965582),(107.78694,49.948218),(107.80792,49.943878),(107.839185,49.946565),(107.850967,49.946048),(107.898199,49.935506),(107.946517,49.933491),(107.95649,49.923362),(107.952614,49.897472),(107.944398,49.880419),(107.935509,49.866621),(107.930032,49.851429),(107.932357,49.830293),(107.950031,49.782492),(107.955767,49.745544),(107.954836,49.732779),(107.949359,49.720635),(107.926828,49.68286),(107.927448,49.670044),(107.939489,49.663223),(107.962898,49.661104),(108.002792,49.663171),(108.015815,49.65573),(108.007546,49.633612),(108.005273,49.617799),(108.015711,49.602245),(108.032661,49.589326),(108.106197,49.554341),(108.126454,49.547158),(108.209911,49.540233),(108.23637,49.532482),(108.250529,49.522405),(108.259521,49.509641),(108.267272,49.495843),(108.277142,49.48251),(108.334865,49.436363),(108.473668,49.35642),(108.538057,49.32743),(108.569973,49.325719),(108.60844,49.323657),(108.751067,49.341175),(108.858967,49.340555),(108.930488,49.348824),(109.032962,49.327895),(109.058025,49.329083),(109.160603,49.346653),(109.286693,49.33854),(109.311911,49.332545),(109.336303,49.323347),(109.390976,49.309291),(109.411233,49.303193),(109.450714,49.305932),(109.462083,49.300454),(109.4657,49.29446),(109.469834,49.276632),(109.474382,49.269138),(109.491849,49.262782),(109.51624,49.255909),(109.728527,49.266451),(109.784957,49.251362),(109.81772,49.227746),(109.83219,49.226867),(109.913115,49.21369),(110.181419,49.161962),(110.225034,49.165372),(110.268339,49.177051),(110.30937,49.195861),(110.34544,49.220614),(110.361046,49.234619),(110.371175,49.24175),(110.382234,49.240045),(110.43329,49.202631),(110.451273,49.194569),(110.508221,49.184337),(110.603615,49.145322),(110.644957,49.136743),(110.688055,49.134521),(110.731359,49.137674),(110.841017,49.162427),(110.965557,49.207385),(111.13857,49.291566),(111.273549,49.32283),(111.291739,49.330788),(111.320988,49.355335),(111.338454,49.36474),(111.362742,49.367634),(111.385066,49.362259),(111.429611,49.346395),(111.454519,49.343087),(111.479117,49.343346),(111.503509,49.347067),(111.616473,49.386444),(111.661019,49.396366),(112.030195,49.411869),(112.126405,49.43994),(112.432238,49.529174),(112.473269,49.534135),(112.58427,49.526332),(112.671396,49.495946),(112.701472,49.491295),(112.733718,49.492794),(112.777126,49.501372),(112.868077,49.531655),(112.926161,49.566536),(112.947348,49.576096),(112.969776,49.583021),(112.992824,49.587155),(113.043673,49.588602),(113.06083,49.595992),(113.071578,49.617748),(113.074162,49.653353),(113.077159,49.669011),(113.087495,49.687408),(113.156741,49.777376),(113.182683,49.801974),(113.212448,49.822025),(113.415537,49.92238),(113.442305,49.946048),(113.458532,49.957262),(113.510415,49.981033),(113.527881,49.992815),(113.579661,50.019945),(113.75226,50.078598),(113.773138,50.081647),(113.815822,50.076893),(113.836803,50.076944),(113.849825,50.080717),(113.973125,50.160505),(113.993279,50.16867),(114.035137,50.176938),(114.055291,50.183811),(114.118233,50.224377),(114.200088,50.25621),(114.286285,50.276881),(114.333,50.272333),(114.424984,50.241844),(114.473043,50.234041),(114.659699,50.251404),(114.75375,50.236211),(114.997352,50.144331),(115.015232,50.132858),(115.028048,50.119888),(115.049649,50.090845),(115.063395,50.077513),(115.210156,49.97168),(115.368699,49.895405),(115.387819,49.891064),(115.450141,49.891426),(115.472982,49.887085),(115.508536,49.886775),(115.578195,49.893648),(115.683409,49.87768),(115.716275,49.877784),(115.750898,49.884967),(116.053619,49.998448),(116.135371,50.014416),(116.21764,50.013848),(116.300529,49.99297),(116.575447,49.92176),(116.617202,49.897317),(116.653375,49.863831),(116.684278,49.823265),(116.472921,49.516462),(116.457728,49.492794),(116.247767,49.181469),(116.037806,48.870145),(116.033568,48.852368),(116.047624,48.817693),(116.048038,48.799761),(116.028194,48.767464),(115.800404,48.530372),(115.791206,48.513836),(115.786555,48.49301),(115.786555,48.473011),(115.809292,48.273954),(115.799991,48.242328),(115.770845,48.224086),(115.536441,48.149155),(115.51422,48.131637),(115.51453,48.122103),(115.545846,47.992989),(115.561866,47.933199),(115.575715,47.909738),(115.5999,47.887414),(115.852701,47.705565),(115.914506,47.683912),(115.9729,47.709053),(116.084314,47.806928),(116.181569,47.849587),(116.243891,47.862894),(116.428893,47.835143),(116.49969,47.836358),(116.751664,47.874133),(116.822047,47.876304),(116.85326,47.872118),(116.972839,47.83765),(117.047356,47.81946),(117.069681,47.810158),(117.317107,47.654069),(117.360826,47.650865),(117.399893,47.685669),(117.43648,47.730499),(117.47286,47.757241),(117.527224,47.78649),(117.741681,47.977589),(117.766899,47.993144),(117.815165,48.004823),(117.972778,47.997898),(117.997169,47.999991),(118.064039,48.019577),(118.122536,48.027509),(118.182481,48.028206),(118.206252,48.023297),(118.254311,48.002859),(118.280046,47.998131),(118.472799,47.989449),(118.542252,47.966246),(118.605246,47.914673),(118.713301,47.793621),(118.767406,47.756156),(118.972614,47.695126),(119.059585,47.673474),(119.08346,47.661562),(119.096947,47.646602),(119.104233,47.62774),(119.116532,47.556039),(119.122113,47.53896),(119.133896,47.527539),(119.157564,47.517669),(119.251046,47.492942),(119.291044,47.48439),(119.310112,47.476095),(119.31776,47.462117),(119.312593,47.452247),(119.288822,47.429147),(119.281897,47.414859),(119.320654,47.40703),(119.475994,47.309206),(119.49067,47.297166),(119.503899,47.279182),(119.512994,47.262672),(119.525086,47.250812),(119.587356,47.238358),(119.629421,47.216086),(119.699959,47.159526),(119.728898,47.128649),(119.753961,47.094439),(119.760524,47.074595),(119.761713,47.015478),(119.764968,47.003825),(119.767294,46.998114),(119.770136,46.992998),(119.784295,46.978322),(119.81742,46.954913),(119.831683,46.942149),(119.83685,46.93357),(119.839692,46.925664),(119.843516,46.918171),(119.852301,46.910729),(119.862947,46.909024),(119.879225,46.908146),(119.892919,46.90551),(119.896433,46.898586),(119.890439,46.88825),(119.884961,46.880912),(119.881447,46.872851),(119.88124,46.860552),(119.883101,46.855436),(119.886821,46.851457),(119.891627,46.848485),(119.896795,46.84647),(119.902893,46.842413),(119.902118,46.837349),(119.898914,46.831251),(119.897983,46.824378),(119.898655,46.802054),(119.885891,46.770066),(119.883101,46.753013),(119.887183,46.74428),(119.903358,46.7264),(119.907027,46.718312),(119.904185,46.708933),(119.897983,46.701647),(119.890335,46.694774),(119.872662,46.673225),(119.85442,46.65966),(119.834421,46.650952),(119.817265,46.651624),(119.796284,46.658833),(119.782228,46.655629),(119.770033,46.643562),(119.754426,46.62439),(119.740267,46.613254),(119.719648,46.602712),(119.697789,46.594883),(119.680116,46.591628),(119.658773,46.596175),(119.623478,46.616329),(119.602188,46.619688),(119.575884,46.619171),(119.501263,46.628008),(119.452946,46.627491),(119.363287,46.613047),(119.316934,46.611678),(119.062376,46.660951),(119.036951,46.668729),(119.014058,46.683301),(118.984913,46.724539),(118.967239,46.740456),(118.947034,46.737148),(118.922074,46.715496),(118.907088,46.709398),(118.890551,46.713532),(118.879648,46.72733),(118.875203,46.74397),(118.867452,46.758336),(118.846523,46.765493),(118.816241,46.758181),(118.792211,46.737768),(118.753816,46.691156),(118.720071,46.676739),(118.684621,46.683457),(118.647414,46.697719),(118.60938,46.705781),(118.595169,46.703042),(118.567729,46.689735),(118.551967,46.685756),(118.472799,46.685679),(118.435179,46.691156),(118.371203,46.713971),(118.335547,46.722162),(118.30082,46.724849),(118.268987,46.722937),(118.238291,46.715393),(118.206045,46.700923),(118.169045,46.678496),(118.155506,46.674568),(118.101969,46.67307),(118.083882,46.668264),(118.005541,46.626974),(117.972778,46.623409),(117.923789,46.615192),(117.900638,46.607596),(117.878313,46.596382),(117.859297,46.580285),(117.832735,46.542793),(117.817025,46.527471),(117.797595,46.520262),(117.696206,46.512253),(117.673365,46.516025),(117.625202,46.535662),(117.607529,46.550545),(117.582001,46.591989),(117.568151,46.603539),(117.556266,46.601601),(117.510067,46.578476),(117.501799,46.577261),(117.491774,46.579096),(117.47286,46.584677),(117.393795,46.57137),(117.410125,46.539796),(117.412502,46.524629),(117.405474,46.510806),(117.369197,46.475691),(117.357518,46.448406),(117.353074,46.379496),(117.337468,46.357275),(117.301708,46.350144),(116.957853,46.366422),(116.831659,46.386369),(116.814606,46.38673),(116.804581,46.382906),(116.78484,46.365905),(116.722002,46.328698),(116.707325,46.324228),(116.64521,46.318466),(116.603559,46.309319),(116.581959,46.298012),(116.568316,46.290871),(116.548265,46.259296),(116.539584,46.24121),(116.525114,46.226792),(116.357166,46.105559),(116.213195,45.908155),(116.207098,45.88087),(116.220947,45.849761),(116.242134,45.82692),(116.251643,45.803252),(116.230352,45.769766),(116.169891,45.709097),(116.143949,45.694628),(116.094856,45.676541),(116.078733,45.673389),(116.063334,45.673337),(116.011967,45.678557),(116.002149,45.675559),(115.982822,45.659643),(115.767331,45.530142),(115.6924,45.468957),(115.667389,45.454281),(115.63783,45.444359),(115.472982,45.412785),(115.33728,45.394698),(115.126337,45.39444),(114.938855,45.373769),(114.905265,45.377748),(114.737317,45.425032),(114.703313,45.427151),(114.66931,45.42281),(114.533711,45.3855),(114.52379,45.38028),(114.519449,45.370927),(114.511697,45.325503),(114.502292,45.303231),(114.473043,45.259202),(114.418886,45.20148),(114.211767,45.06216),(114.163605,45.040146),(114.149342,45.030069),(114.139214,45.019165),(114.120403,44.993224),(114.089294,44.961753),(114.055291,44.940565),(114.01705,44.927285),(113.973125,44.919326),(113.913801,44.915451),(113.889306,44.908268),(113.83763,44.867908),(113.735621,44.815922),(113.672472,44.768173),(113.635058,44.746262),(113.604776,44.739699),(113.532429,44.74528),(113.517133,44.749518),(113.488917,44.766261),(113.473208,44.770705),(113.104858,44.794373),(113.016905,44.82109),(112.973186,44.821245),(112.748497,44.865273),(112.614242,44.908836),(112.586027,44.923822),(112.564529,44.943924),(112.523912,44.993275),(112.441333,45.052393),(112.412807,45.066036),(112.379631,45.070738),(112.107606,45.067379),(112.011488,45.087482),(111.984637,45.087154),(111.973351,45.087017),(111.958261,45.084536),(111.947926,45.077715),(111.938521,45.068981),(111.926739,45.06092),(111.906275,45.05415),(111.863177,45.046295),(111.843333,45.039526),(111.786489,45.008572),(111.760651,44.989296),(111.738533,44.9663),(111.551361,44.69319),(111.541026,44.668231),(111.540096,44.652211),(111.544436,44.618105),(111.543816,44.601258),(111.538235,44.58343),(111.530174,44.570614),(111.508986,44.546584),(111.483768,44.506432),(111.473226,44.495321),(111.470849,44.493306),(111.446871,44.475891),(111.424134,44.448037),(111.406357,44.416463),(111.397055,44.387369),(111.396435,44.346597),(111.410595,44.326081),(111.473226,44.298848),(111.49059,44.282621),(111.498134,44.264431),(111.503715,44.225725),(111.519425,44.18857),(111.543816,44.156737),(111.634457,44.065942),(111.66815,44.041344),(111.780701,43.986205),(111.811397,43.964294),(111.838785,43.938792),(111.930563,43.820143),(111.944309,43.786372),(111.947926,43.756968),(111.945136,43.72387),(111.933353,43.696636),(111.910409,43.684983),(111.893252,43.682606),(111.845813,43.666922),(111.77264,43.669299),(111.753829,43.663202),(111.59725,43.523391),(111.567691,43.50254),(111.535031,43.490732),(111.428681,43.483936),(111.401396,43.475513),(111.338558,43.438177),(111.096092,43.366656),(110.973412,43.3156),(110.933724,43.287772),(110.796989,43.148918),(110.680096,43.057657),(110.66573,43.0374),(110.641029,42.987791),(110.608163,42.944667),(110.566718,42.91056),(110.473391,42.854905),(110.449413,42.837335),(110.424815,42.785245),(110.406728,42.768605),(110.139871,42.682615),(110.105661,42.666673),(110.093156,42.657862),(110.084267,42.650059),(110.074862,42.643522),(110.060186,42.638355),(110.042719,42.635952),(109.987322,42.636468),(109.912701,42.62882),(109.667342,42.54898),(109.518307,42.467331),(109.510349,42.464748),(109.504871,42.464928),(109.500013,42.463327),(109.493812,42.455213),(109.490092,42.451286),(109.485131,42.449296),(109.328551,42.440486),(109.280285,42.428316),(109.253414,42.425939),(108.994463,42.449529),(108.961287,42.447539),(108.862688,42.40886),(108.82853,42.400049),(108.796852,42.398369),(108.536455,42.435111),(108.319775,42.432553),(108.234768,42.450072),(108.177355,42.454309),(108.04968,42.43912),(107.622396,42.388286),(107.195113,42.337453),(106.767829,42.286619),(106.318218,42.140052),(105.868607,41.993484),(105.473851,41.840341),(105.402744,41.811532),(105.358199,41.785848),(105.344557,41.78112),(105.325953,41.776702),(105.274483,41.755463),(105.218259,41.748435),(105.200896,41.743474),(105.014809,41.596144),(104.97383,41.586145),(104.935434,41.622422),(104.913678,41.638338),(104.892336,41.644772),(104.601243,41.645444),(104.504918,41.656554),(104.497993,41.665778),(104.500784,41.870598),(104.110885,41.813082),(103.720986,41.755566),(103.691427,41.759235),(103.398112,41.876411),(103.104797,41.993588),(103.104435,41.993639),(103.104383,41.993639),(103.073481,42.004517),(102.727042,42.064548),(102.380603,42.124579),(102.034164,42.18461),(101.97422,42.20789),(101.886473,42.278015),(101.735371,42.461156),(101.637599,42.515442),(101.524945,42.537456),(101.41074,42.544872),(101.21871,42.529834),(100.818132,42.578729),(100.417553,42.627623),(100.016975,42.676518),(99.971551,42.676285),(99.621903,42.597443),(99.474476,42.564199),(99.068355,42.60391),(98.662234,42.643622),(98.256113,42.683333),(97.849992,42.723045),(97.521632,42.755152),(97.193271,42.78726),(97.19068,42.787061),(96.832669,42.759604),(96.474659,42.732147),(96.379264,42.720546),(96.36634,42.722923),(96.357767,42.724499),(96.350635,42.740906),(96.337199,42.86648),(96.330688,42.889709),(96.318699,42.910069),(96.294308,42.933272),(95.908492,43.214547),(95.874385,43.247103),(95.854955,43.284543),(95.845446,43.326478),(95.841416,43.372522),(95.832837,43.408902),(95.692484,43.630542),(95.589235,43.869494),(95.535284,43.95768),(95.510273,43.979125),(95.474926,43.986619),(95.422733,43.988634),(95.327545,44.006669),(95.318967,44.017108),(95.31876,44.040724),(95.328269,44.09824),(95.321447,44.148417),(95.322688,44.158236),(95.327132,44.168313),(95.340568,44.187537),(95.347492,44.214925),(95.359688,44.231823),(95.389247,44.260762),(95.397722,44.280502),(95.379428,44.287117),(95.036814,44.254974),(94.993819,44.259418),(94.905762,44.294248),(94.720864,44.335796),(94.69823,44.343496),(94.683967,44.356622),(94.672598,44.373778),(94.657199,44.388816),(94.588055,44.4361),(94.460311,44.493306),(94.430132,44.504468),(94.366364,44.504313),(94.338458,44.512116),(94.321922,44.528033),(94.295464,44.571389),(94.27748,44.589579),(94.195005,44.65433),(94.181879,44.660841),(94.165342,44.665854),(94.149012,44.667197),(94.103744,44.651643),(94.058579,44.651281),(93.97507,44.659962),(93.927424,44.672933),(93.874507,44.714223),(93.714,44.874575),(93.688265,44.891059),(93.525278,44.951263),(93.420582,44.955087),(93.399911,44.960564),(93.36136,44.978393),(93.341,44.984852),(93.317229,44.985886),(93.268963,44.980098),(93.244158,44.981545),(93.165713,45.007642),(93.139152,45.009864),(93.075941,45.005192),(93.052439,45.003456),(93.033112,45.007642),(93.013888,45.009295),(92.994458,45.008365),(92.974924,45.004696),(92.945055,45.005833),(92.91622,45.014566),(92.861029,45.037872),(92.847593,45.039112),(92.81421,45.033893),(92.798914,45.03317),(92.752715,45.037976),(92.63851,45.0156),(92.572054,45.013016),(92.475109,44.997461),(92.458986,44.997926),(92.413821,45.011259),(92.396458,45.013119),(92.347365,45.007642),(92.331036,45.007848),(92.285354,45.019527),(92.267887,45.019527),(92.218898,45.010122),(92.19368,45.014514),(92.108414,45.058181),(92.075651,45.068878),(92.042991,45.074821),(92.009401,45.076216),(91.975088,45.073374),(91.919071,45.065571),(91.746989,45.072444),(91.724664,45.070738),(91.683633,45.061178),(91.662963,45.059525),(91.563951,45.077146),(91.552272,45.075338),(91.54328,45.071824),(91.533978,45.0711),(91.521783,45.077456),(91.45605,45.137453),(91.43848,45.146134),(91.41688,45.145669),(91.399723,45.136884),(91.3836,45.124224),(91.366547,45.11425),(91.346703,45.113733),(91.249138,45.129701),(91.184336,45.153834),(91.169763,45.163549),(91.147749,45.187217),(91.135036,45.197862),(91.101343,45.210678),(91.056798,45.217551),(91.011323,45.218946),(90.975149,45.214864),(90.90549,45.185977),(90.873243,45.186184),(90.857637,45.215071),(90.860324,45.231659),(90.865905,45.242563),(90.866629,45.25357),(90.854433,45.270623),(90.837173,45.280028),(90.796556,45.292844),(90.790975,45.304316),(90.790458,45.337699),(90.778055,45.365294),(90.762553,45.391597),(90.753251,45.42095),(90.737955,45.441724),(90.672325,45.475675),(90.651138,45.493142),(90.696613,45.730026),(90.710773,45.752196),(90.919235,45.950065),(90.975149,45.987323),(90.986208,45.993111),(91.002435,46.021843),(91.005432,46.068094),(90.995923,46.113724),(90.975149,46.140544),(90.948794,46.168294),(90.899185,46.281672),(90.896188,46.302033),(90.906006,46.317303),(90.935772,46.343865),(90.975149,46.416548),(90.986105,46.435255),(91.001401,46.475045),(91.014113,46.493184),(91.035611,46.529073),(91.045326,46.551087),(91.047496,46.566409),(91.03251,46.579923),(91.011323,46.585323),(90.995923,46.594625),(90.998094,46.620256),(91.011736,46.662941),(91.014292,46.689844),(91.01556,46.703197),(91.004812,46.74043),(90.975149,46.774149),(90.939389,46.79885),(90.925333,46.813139),(90.919959,46.833628),(90.922749,46.841586),(90.934428,46.856831),(90.936495,46.864427),(90.932775,46.874789),(90.92585,46.880654),(90.917685,46.885408),(90.91014,46.892384),(90.900322,46.910678),(90.893707,46.92817),(90.885129,46.944733),(90.869833,46.960339),(90.849472,46.973206),(90.828182,46.982301),(90.805547,46.986849),(90.78188,46.986332),(90.742399,46.991861),(90.712116,47.014702),(90.46872,47.308922),(90.466343,47.316338),(90.468514,47.323521),(90.475232,47.33029),(90.485567,47.344915),(90.490011,47.361839),(90.487221,47.378194),(90.475232,47.391088),(90.448153,47.404033),(90.441745,47.430543),(90.443606,47.462789),(90.441228,47.493045),(90.427276,47.506559),(90.394616,47.524671),(90.381594,47.538391),(90.352035,47.588311),(90.338496,47.60278),(90.327437,47.623037),(90.337462,47.63699),(90.349554,47.64779),(90.3448,47.658642),(90.32351,47.670166),(90.302116,47.678512),(90.209718,47.695746),(90.163416,47.710009),(90.119956,47.730137),(90.082852,47.756104),(90.070657,47.76985),(90.058771,47.789513),(90.052157,47.8107),(90.055981,47.828916),(90.064352,47.85062),(90.059856,47.868811),(90.044612,47.879766),(90.002496,47.880386),(89.96317,47.888654),(89.946685,47.883177),(89.939037,47.870852),(89.937642,47.857545),(89.934438,47.844264),(89.921364,47.832094),(89.889066,47.824317),(89.783543,47.818504),(89.751296,47.824317),(89.740238,47.836151),(89.731143,47.864754),(89.723391,47.878164),(89.707268,47.889042),(89.672025,47.89408),(89.655282,47.899816),(89.643913,47.912761),(89.636161,47.929582),(89.627376,47.943948),(89.613424,47.949581),(89.596577,47.952578),(89.583245,47.962035),(89.575648,47.976039),(89.576217,47.992886),(89.571876,48.020455),(89.5418,48.031023),(89.503663,48.029989),(89.475344,48.022961),(89.425425,48.021644),(89.409198,48.023116),(89.369201,48.038154),(89.354266,48.038774),(89.334268,48.032366),(89.285692,48.005598),(89.268587,47.993144),(89.243989,47.980225),(89.218719,47.976969),(89.167146,47.983997),(89.074697,47.984256),(89.045913,47.992886),(89.04581,47.992989),(89.045551,47.992989),(88.932328,48.096962),(88.916825,48.105954),(88.889282,48.110915),(88.831456,48.105153),(88.805411,48.105954),(88.775335,48.120552),(88.709189,48.166363),(88.680664,48.17303),(88.655446,48.171169),(88.635809,48.175846),(88.596225,48.197679),(88.580774,48.211994),(88.573487,48.229719),(88.568836,48.249201),(88.56124,48.268269),(88.558759,48.288087),(88.57297,48.32532),(88.565116,48.342993),(88.50269,48.392551),(88.479126,48.400148),(88.459127,48.398132),(88.439645,48.393714),(88.416701,48.394101),(88.383421,48.408674),(88.332107,48.453994),(88.304873,48.469006),(88.235627,48.493191),(88.21785,48.495),(88.183485,48.492752),(88.165967,48.493734),(88.148293,48.499366),(88.115737,48.516497),(88.088245,48.526083),(88.071812,48.538795),(88.062511,48.542878),(88.051969,48.543188),(88.028301,48.539622),(88.016622,48.539803),(87.983032,48.552335),(87.95182,48.575202),(87.942828,48.599489),(87.975488,48.616517),(87.989647,48.624165),(87.997812,48.635172),(88.003393,48.645998),(88.010162,48.653414),(88.055018,48.673413),(88.063854,48.682611),(88.05941,48.708087),(88.033779,48.72912),(87.975488,48.755992),(87.95244,48.761107),(87.907585,48.758834),(87.841697,48.779349),(87.821078,48.788289),(87.805885,48.800588),(87.802165,48.80958),(87.800511,48.830302),(87.796067,48.839759),(87.787489,48.84658),(87.77674,48.85118),(87.754726,48.857277),(87.737879,48.86937),(87.735709,48.886733),(87.743615,48.905181),(87.756896,48.920374),(87.835858,48.945902),(87.8726,48.96802),(87.872238,49.00409),(87.856942,49.015614),(87.837821,49.02042),(87.821905,49.02874),(87.815962,49.050599),(87.821078,49.071838),(87.829657,49.085997),(87.835548,49.100725),(87.832447,49.123876),(87.818081,49.154003),(87.816324,49.165837),(87.836995,49.159998),(87.855185,49.158964),(87.956781,49.168886),(87.975488,49.175811),(87.992954,49.191727),(88.029748,49.21555),(88.108503,49.247434),(88.143953,49.27074),(88.148397,49.296579),(88.129173,49.325931),(88.115531,49.356575),(88.135788,49.386082),(88.145606,49.397193),(88.161729,49.430111),(88.174752,49.443857),(88.19997,49.454347),(88.37877,49.475793),(88.401921,49.475844),(88.451117,49.465147),(88.475405,49.463649),(88.538761,49.466542),(88.557829,49.47109),(88.599739,49.488918),(88.614157,49.492794),(88.615759,49.492949),(88.633535,49.491244),(88.648676,49.483596),(88.67467,49.459825),(88.69565,49.446957),(88.719421,49.442926),(88.870627,49.436002),(88.875329,49.441789),(88.854865,49.473881),(88.856157,49.511501),(88.858741,49.527366),(88.868198,49.538114),(88.88892,49.54168),(88.90835,49.535944),(88.919099,49.52318),(88.926954,49.507419),(88.937599,49.492846),(88.947573,49.482821),(88.954704,49.471865),(88.962662,49.46215),(88.975323,49.455845),(89.01408,49.460548),(89.160738,49.502716),(89.195258,49.521526),(89.215619,49.545452),(89.201563,49.570619),(89.178205,49.598472),(89.187352,49.621985),(89.216445,49.634491),(89.252205,49.629427),(89.310703,49.59315),(89.339435,49.580799),(89.37194,49.581678),(89.39783,49.598989),(89.418604,49.624207),(89.4416,49.643586),(89.475344,49.643431),(89.520303,49.663171),(89.616318,49.685444),(89.682308,49.708336),(89.702411,49.719964),(89.712642,49.735157),(89.701532,49.75314),(89.683704,49.76115),(89.638952,49.775464),(89.626446,49.788022),(89.627376,49.796807),(89.641949,49.808124),(89.643551,49.818872),(89.638952,49.824815),(89.621072,49.835047),(89.615336,49.84192),(89.615284,49.854529),(89.621072,49.893338),(89.623862,49.902692),(89.633211,49.907939),(89.667684,49.92729),(89.720704,49.939744),(89.866432,49.942741),(89.920175,49.950854),(89.969423,49.96708),(89.996243,49.992764),(89.997091,50.003664),(89.998723,50.024648),(90.004924,50.050745),(90.020531,50.070743),(90.051226,50.084179),(90.182433,50.102473),(90.222947,50.11441),(90.34387,50.174665),(90.363921,50.179781),(90.404952,50.182623),(90.425002,50.186912),(90.475232,50.21461),(90.51895,50.218383),(90.604939,50.205981),(90.648451,50.206652),(90.666434,50.210373),(90.683901,50.216212),(90.699714,50.225152),(90.712633,50.237968),(90.717387,50.252024),(90.717594,50.280756),(90.726586,50.291867),(90.762449,50.305716),(90.839964,50.320806),(90.877068,50.339357),(90.948588,50.397493),(90.975149,50.413926),(91.012046,50.425295),(91.129765,50.426174),(91.163355,50.432788),(91.260093,50.466585),(91.297507,50.466378),(91.382153,50.454958),(91.415743,50.461365),(91.432486,50.478057),(91.440031,50.498572),(91.450366,50.519088),(91.475067,50.535986),(91.575733,50.561979),(91.602708,50.579601),(91.653144,50.640063),(91.679602,50.656547),(91.749779,50.684091),(91.824193,50.701558),(91.899951,50.707242),(92.062732,50.686313),(92.15885,50.689103),(92.242772,50.720006),(92.294449,50.791268),(92.297033,50.809871),(92.297446,50.827596),(92.301683,50.843358),(92.315223,50.855863),(92.33548,50.863615),(92.360905,50.868369),(92.384572,50.865579),(92.399869,50.850592),(92.418989,50.811732),(92.443277,50.786307),(92.474903,50.770494),(92.516451,50.760624),(92.554588,50.743002),(92.611845,50.687657),(92.649776,50.674066),(92.671893,50.675616),(92.695251,50.68192),(92.717369,50.692152),(92.735249,50.705227),(92.746617,50.721453),(92.753439,50.754681),(92.76026,50.770752),(92.796123,50.788012),(92.857722,50.795505),(92.921077,50.792456),(92.961695,50.777884),(92.973994,50.742175),(92.973374,50.694374),(92.991874,50.65479),(93.002933,50.634843),(93.011408,50.623319),(93.02288,50.616291),(93.04262,50.609884),(93.104942,50.598825),(93.139152,50.599445),(93.188348,50.599807),(93.421512,50.609367),(93.536337,50.584717),(93.886961,50.574976),(94.237586,50.565235),(94.25867,50.557535),(94.273346,50.544771),(94.278204,50.529217),(94.279754,50.492681),(94.319751,50.429016),(94.327503,50.404366),(94.332257,50.301013),(94.344349,50.249182),(94.354478,50.223602),(94.368534,50.202725),(94.391788,50.185568),(94.474987,50.162262),(94.491731,50.152289),(94.498552,50.139525),(94.502273,50.125055),(94.509094,50.109862),(94.520876,50.09777),(94.563974,50.071002),(94.574413,50.058548),(94.592293,50.029816),(94.603455,50.019739),(94.624539,50.015191),(94.650997,50.017827),(94.746392,50.040358),(94.921575,50.045887),(94.953718,50.040616),(94.98059,50.024751),(95.004051,49.992764),(95.021621,49.969819),(95.028339,49.963411),(95.076398,49.945531),(95.354727,49.947702),(95.399789,49.938865),(95.445161,49.914422),(95.461697,49.90264),(95.477201,49.894083),(95.480611,49.892201),(95.500661,49.886724),(95.520608,49.889617),(95.537145,49.900728),(95.56319,49.9276),(95.583964,49.936695),(95.680495,49.945996),(95.721423,49.958502),(95.75894,49.993074),(95.787672,50.011729),(95.827876,50.020152),(95.867047,50.014984),(95.892162,49.992919),(95.892162,49.992764),(95.904461,49.964807),(95.922651,49.944601),(95.946319,49.938503),(95.974844,49.952456),(95.985696,49.964962),(95.998305,49.975917),(96.011948,49.985219),(96.026624,49.992815),(96.04099,49.997931),(96.055976,49.99881),(96.071066,49.996743),(96.085949,49.992919),(96.086052,49.992867),(96.086259,49.992815),(96.086362,49.992764),(96.235603,49.9492),(96.281182,49.926256),(96.32335,49.897886),(96.346294,49.888584),(96.372856,49.886052),(96.399108,49.891736),(96.44882,49.913957),(96.474659,49.921657),(96.503287,49.922122),(96.523855,49.915766),(96.539254,49.901865),(96.553,49.879851),(96.56809,49.861764),(96.585143,49.857526),(96.604366,49.862901),(96.625967,49.873391),(96.680227,49.90972),(96.701932,49.91375),(96.969202,49.886672),(97.005892,49.865278),(97.042273,49.823213),(97.057259,49.812723),(97.08527,49.805838),(97.10015,49.802181),(97.119891,49.793654),(97.128986,49.775826),(97.152964,49.750246),(97.205053,49.734071),(97.262311,49.726113),(97.301688,49.725545),(97.343339,49.734175),(97.390778,49.749471),(97.474598,49.788693),(97.542707,49.822748),(97.57361,49.847501),(97.581258,49.877939),(97.571749,49.896749),(97.563171,49.908944),(97.566582,49.916799),(97.642236,49.930028),(97.7336,49.958347),(97.756338,49.961499),(97.777525,49.956177),(97.810701,49.931165),(97.829925,49.92331),(97.847391,49.93008),(97.874573,49.953955),(97.906199,49.973075),(98.047999,50.023046),(98.105567,50.06387),(98.262457,50.283805),(98.269278,50.299412),(98.272275,50.318997),(98.270105,50.335947),(98.261836,50.367469),(98.266281,50.400697),(98.298217,50.460125),(98.300594,50.492785),(98.300697,50.492836),(98.300697,50.49294),(98.293462,50.518623),(98.275376,50.537692),(98.251294,50.550559),(98.22556,50.557639),(98.163341,50.564667),(98.135953,50.570971),(98.060092,50.608902),(98.038491,50.622958),(98.023298,50.642026),(97.974515,50.723158),(97.944026,50.774473),(97.947437,50.79597),(97.990225,50.835606),(97.984644,50.846923),(97.953018,50.86284),(97.938755,50.874415),(97.92966,50.886611),(97.919428,50.897411),(97.871989,50.9171),(97.835713,50.940716),(97.808531,50.970482),(97.80636,51.001126),(97.870129,51.086805),(97.885632,51.120111),(97.894727,51.153855),(97.900308,51.168428),(97.92904,51.21504),(97.931727,51.230414),(97.92811,51.251472),(97.916328,51.285889),(97.915708,51.300617),(97.922839,51.318471),(97.933794,51.331777),(97.962216,51.355445),(97.974515,51.368597),(98.020611,51.435182),(98.039731,51.4573),(98.050893,51.46642),(98.066706,51.469263),(98.111355,51.484714),(98.137813,51.485877),(98.220392,51.475205),(98.2299,51.492879),(98.233828,51.516521),(98.224319,51.560006),(98.228247,51.583416),(98.238685,51.601477),(98.263387,51.635196),(98.273515,51.654833),(98.33222,51.718317),(98.425031,51.746222),(98.618714,51.780665),(98.651373,51.797485),(98.682689,51.819474),(98.711111,51.845105),(98.776637,51.930113),(98.791727,51.9775),(98.800201,51.992745),(98.838545,52.025275),(98.846504,52.039331),(98.848881,52.056307),(98.849604,52.090129),(98.855599,52.106691),(98.886398,52.128551),(98.927636,52.129584),(98.937652,52.124687),(98.962465,52.112557),(98.974454,52.080388),(98.994712,52.058296),(99.025097,52.045532),(99.19873,52.006594),(99.232837,51.994243),(99.255988,51.977914),(99.274694,51.959491),(99.295365,51.943549),(99.32389,51.934221),(99.556848,51.891459),(99.679321,51.888669),(99.709655,51.880581),(99.783966,51.837612),(99.848665,51.789346),(99.917911,51.749478),(100.005968,51.731727)] +Montserrat [(-62.147369,16.744289),(-62.147694,16.737047),(-62.151763,16.724514),(-62.150787,16.720364),(-62.147572,16.715888),(-62.14391,16.70897),(-62.141184,16.70185),(-62.140533,16.696479),(-62.144032,16.688218),(-62.14977,16.681789),(-62.157826,16.677314),(-62.167877,16.67536),(-62.218861,16.688707),(-62.230133,16.727607),(-62.218861,16.776313),(-62.202016,16.819322),(-62.189443,16.814439),(-62.182932,16.81273),(-62.174672,16.813137),(-62.179311,16.794501),(-62.170033,16.777004),(-62.156321,16.760321),(-62.147369,16.744289)] +Namibia [(13.184911,-16.964183),(13.19814,-16.957362),(13.205478,-16.963046),(13.212092,-16.972761),(13.222841,-16.977929),(13.245269,-16.98134),(13.257154,-16.98165),(13.267489,-16.977929),(13.267489,-16.98537),(13.293018,-16.973795),(13.308417,-16.970178),(13.315238,-16.974415),(13.321543,-16.974932),(13.345947,-16.968711),(13.363711,-16.964183),(13.382004,-16.970384),(13.417248,-16.993742),(13.435128,-16.999013),(13.458899,-17.001803),(13.479776,-17.010278),(13.494659,-17.024024),(13.507785,-17.063505),(13.522254,-17.076941),(13.530936,-17.093271),(13.521324,-17.1219),(13.606487,-17.167375),(13.69413,-17.236621),(13.790403,-17.288091),(13.884351,-17.338527),(13.896857,-17.349069),(13.942745,-17.408187),(13.957318,-17.419142),(13.980573,-17.423587),(13.992975,-17.421313),(14.008891,-17.411494),(14.017883,-17.40922),(14.029148,-17.410461),(14.047442,-17.416042),(14.069146,-17.418626),(14.085372,-17.423587),(14.097258,-17.423587),(14.108213,-17.420486),(14.123923,-17.411598),(14.130744,-17.40922),(14.174979,-17.416248),(14.197097,-17.412941),(14.206502,-17.393097),(14.207432,-17.388033),(14.218801,-17.388136),(14.219112,-17.388136),(14.287427,-17.388136),(14.543949,-17.38824),(14.800574,-17.388343),(15.057199,-17.388343),(15.313721,-17.388447),(15.559114,-17.388545),(15.570243,-17.38855),(15.826765,-17.388653),(16.08339,-17.388653),(16.339808,-17.388653),(16.59633,-17.388757),(16.852852,-17.388757),(17.109477,-17.38886),(17.278972,-17.388928),(17.366102,-17.388963),(17.622624,-17.388963),(17.879146,-17.389067),(18.000483,-17.389116),(18.135668,-17.38917),(18.392293,-17.389273),(18.445726,-17.389273),(18.453581,-17.389893),(18.455028,-17.395991),(18.458645,-17.405293),(18.465157,-17.40922),(18.471668,-17.414285),(18.488101,-17.452112),(18.489858,-17.462447),(18.492752,-17.464514),(18.516626,-17.471336),(18.55094,-17.535311),(18.554144,-17.54637),(18.567476,-17.553294),(18.619669,-17.594842),(18.622873,-17.60218),(18.627937,-17.619854),(18.633312,-17.628949),(18.64282,-17.638044),(18.66132,-17.646725),(18.670932,-17.653133),(18.728293,-17.710907),(18.74824,-17.736332),(18.761986,-17.747701),(18.800847,-17.756589),(18.89004,-17.799274),(19.020575,-17.822322),(19.105118,-17.818188),(19.139327,-17.805269),(19.160825,-17.800928),(19.17209,-17.801238),(19.201856,-17.807129),(19.246608,-17.804028),(19.262421,-17.81395),(19.421274,-17.859426),(19.59284,-17.857255),(19.649787,-17.844956),(19.672628,-17.842682),(19.693195,-17.847747),(19.702187,-17.86573),(19.711282,-17.872965),(19.732056,-17.88206),(19.755207,-17.889088),(19.77102,-17.890328),(19.775981,-17.885057),(19.78766,-17.866454),(19.793964,-17.862319),(19.951991,-17.862319),(19.989508,-17.882783),(20.024545,-17.894566),(20.03426,-17.895909),(20.092551,-17.890328),(20.119939,-17.890741),(20.132962,-17.888778),(20.147121,-17.882783),(20.154563,-17.890328),(20.169239,-17.878132),(20.193113,-17.872138),(20.219882,-17.873378),(20.243446,-17.882783),(20.278173,-17.854878),(20.288921,-17.861596),(20.302771,-17.860769),(20.336154,-17.854878),(20.347729,-17.858909),(20.386797,-17.890328),(20.412635,-17.884127),(20.437543,-17.894462),(20.460074,-17.913686),(20.479504,-17.934356),(20.489529,-17.942108),(20.523946,-17.958541),(20.552885,-17.979108),(20.565494,-17.985206),(20.590712,-17.978798),(20.630089,-17.976938),(20.666263,-17.980245),(20.682179,-17.988617),(20.68962,-17.996988),(20.706157,-18.00381),(20.722487,-18.006497),(20.729928,-18.002363),(20.737473,-17.993474),(20.754423,-17.997918),(20.784602,-18.012595),(20.806202,-18.031405),(20.831937,-18.029338),(20.8616,-18.018796),(20.894466,-18.012595),(20.908315,-17.986136),(20.986455,-17.965771),(21.000713,-17.962055),(21.09311,-17.938077),(21.10944,-17.945002),(21.12608,-17.941074),(21.143133,-17.933736),(21.161323,-17.930636),(21.175792,-17.93322),(21.216617,-17.930636),(21.23274,-17.93446),(21.278112,-17.958541),(21.335059,-17.977971),(21.364825,-17.992027),(21.381154,-18.012595),(21.386839,-18.014455),(21.405546,-18.009287),(21.477583,-17.996058),(21.728523,-17.949859),(21.979464,-17.903764),(22.230508,-17.857565),(22.481449,-17.811366),(22.685674,-17.772816),(22.889796,-17.734059),(23.093918,-17.695405),(23.29804,-17.656854),(23.381652,-17.641144),(23.422373,-17.633496),(23.45741,-17.626778),(23.47622,-17.626365),(23.505986,-17.620474),(23.548774,-17.611999),(23.591769,-17.603524),(23.634557,-17.595049),(23.677242,-17.586677),(23.72003,-17.578202),(23.762818,-17.569727),(23.805606,-17.561356),(23.848497,-17.552881),(23.891389,-17.544406),(23.934177,-17.536034),(23.976965,-17.52756),(24.01965,-17.519085),(24.062541,-17.510713),(24.105329,-17.502238),(24.148117,-17.493763),(24.190905,-17.485288),(24.220464,-17.4795),(24.238758,-17.478157),(24.257361,-17.480844),(24.310278,-17.482601),(24.321337,-17.488699),(24.329398,-17.485081),(24.371256,-17.473713),(24.388929,-17.471336),(24.407016,-17.474539),(24.449288,-17.489112),(24.47926,-17.494487),(24.498173,-17.503478),(24.523598,-17.507819),(24.53166,-17.5134),(24.537861,-17.520325),(24.546543,-17.526526),(24.562459,-17.53221),(24.571244,-17.533451),(24.580752,-17.532831),(24.591294,-17.528386),(24.606487,-17.515157),(24.617443,-17.509163),(24.622817,-17.502341),(24.629742,-17.49552),(24.639457,-17.49242),(24.684415,-17.49242),(24.769785,-17.505442),(24.775056,-17.507612),(24.780017,-17.512263),(24.786838,-17.516914),(24.79738,-17.519085),(24.829523,-17.517741),(24.898252,-17.531074),(24.924917,-17.542959),(24.93763,-17.560736),(24.95799,-17.551744),(24.969987,-17.559962),(24.971116,-17.560736),(24.982588,-17.576549),(24.998505,-17.588021),(25.008013,-17.588538),(25.02672,-17.58275),(25.036952,-17.5812),(25.040569,-17.584507),(25.033851,-17.60156),(25.033851,-17.608485),(25.040053,-17.616133),(25.045324,-17.619957),(25.052558,-17.621404),(25.064237,-17.621507),(25.067028,-17.625331),(25.085218,-17.640938),(25.088525,-17.642695),(25.096897,-17.67215),(25.098757,-17.676801),(25.107955,-17.678868),(25.115087,-17.684242),(25.120048,-17.691064),(25.122631,-17.697885),(25.131416,-17.686516),(25.138548,-17.686103),(25.153327,-17.700986),(25.156428,-17.70667),(25.155291,-17.719382),(25.156841,-17.72517),(25.161182,-17.729408),(25.172448,-17.735092),(25.177822,-17.738813),(25.190741,-17.755349),(25.198182,-17.75845),(25.215132,-17.759277),(25.228878,-17.762481),(25.242417,-17.770335),(25.253476,-17.781497),(25.259781,-17.794107),(25.194152,-17.782324),(25.153947,-17.781808),(25.120874,-17.813537),(25.087905,-17.826766),(25.057002,-17.827696),(25.047494,-17.807129),(25.019692,-17.823769),(25.007083,-17.825733),(24.998505,-17.81395),(24.983312,-17.820462),(24.975147,-17.816327),(24.968636,-17.807646),(24.958094,-17.800928),(24.964295,-17.786665),(24.953339,-17.788422),(24.948585,-17.79328),(24.945175,-17.799998),(24.93763,-17.807129),(24.931119,-17.81054),(24.857428,-17.833587),(24.838101,-17.835034),(24.820841,-17.839272),(24.79738,-17.858082),(24.776916,-17.862319),(24.770715,-17.865523),(24.764514,-17.879683),(24.756039,-17.882783),(24.74684,-17.88423),(24.738159,-17.887641),(24.730717,-17.891878),(24.725343,-17.895909),(24.698471,-17.928879),(24.664055,-17.949859),(24.649172,-17.962778),(24.598632,-18.020759),(24.595015,-18.022826),(24.591811,-18.028407),(24.577755,-18.044427),(24.574551,-18.050422),(24.564423,-18.052799),(24.518431,-18.057346),(24.505615,-18.060344),(24.471508,-18.029958),(24.469751,-18.014455),(24.465101,-18.008667),(24.458486,-18.005773),(24.451045,-17.998952),(24.433888,-17.967223),(24.421589,-17.956474),(24.399471,-17.95234),(24.365262,-17.950789),(24.350586,-17.95606),(24.334256,-17.971563),(24.305834,-18.019416),(24.296429,-18.026237),(24.287437,-18.02448),(24.270074,-18.015798),(24.259222,-18.012595),(24.238241,-18.009907),(24.218294,-18.012595),(24.183051,-18.029441),(24.135095,-18.085458),(24.101609,-18.108816),(24.065125,-18.115224),(24.05696,-18.119048),(24.028125,-18.145816),(24.02027,-18.151604),(23.991744,-18.163386),(23.979652,-18.171861),(23.974898,-18.177029),(23.971177,-18.18385),(23.966733,-18.180646),(23.961979,-18.177856),(23.956604,-18.176615),(23.950817,-18.177649),(23.915987,-18.201213),(23.912886,-18.235733),(23.89697,-18.250203),(23.867514,-18.269426),(23.855215,-18.280072),(23.837232,-18.305807),(23.825553,-18.317072),(23.809843,-18.321723),(23.715792,-18.419081),(23.700909,-18.42797),(23.680136,-18.431484),(23.656468,-18.458252),(23.649853,-18.46342),(23.645409,-18.466004),(23.609856,-18.477682),(23.592182,-18.478199),(23.57916,-18.467864),(23.571305,-18.426006),(23.555492,-18.383115),(23.546604,-18.369472),(23.560763,-18.348491),(23.551461,-18.327511),(23.518802,-18.293714),(23.527277,-18.277695),(23.521696,-18.268186),(23.511257,-18.260331),(23.505676,-18.249376),(23.501542,-18.23749),(23.49131,-18.23315),(23.477771,-18.230876),(23.464128,-18.225501),(23.458754,-18.217647),(23.444284,-18.200697),(23.429401,-18.188501),(23.418859,-18.198423),(23.410281,-18.197389),(23.40129,-18.194289),(23.395812,-18.191395),(23.394468,-18.186227),(23.396329,-18.169381),(23.395812,-18.163386),(23.389714,-18.153258),(23.359535,-18.119151),(23.348063,-18.09445),(23.336591,-18.07936),(23.3338,-18.073986),(23.33256,-18.067061),(23.3338,-18.04329),(23.330596,-18.039983),(23.323258,-18.039156),(23.31592,-18.037089),(23.312716,-18.029958),(23.312819,-18.016522),(23.311476,-18.009804),(23.305688,-18.005463),(23.292769,-17.998952),(23.254942,-17.997402),(23.185799,-18.003086),(23.099602,-18.010217),(22.981367,-18.020036),(22.894034,-18.036159),(22.809698,-18.051869),(22.725258,-18.067475),(22.640922,-18.083185),(22.556483,-18.098791),(22.472147,-18.1145),(22.387811,-18.13021),(22.303475,-18.145816),(22.219036,-18.161526),(22.134597,-18.177236),(22.050261,-18.192842),(21.966028,-18.208552),(21.881486,-18.224158),(21.79715,-18.239867),(21.71271,-18.255577),(21.628478,-18.271183),(21.544039,-18.28679),(21.526985,-18.289994),(21.509829,-18.293198),(21.492879,-18.296401),(21.475722,-18.299502),(21.456809,-18.300226),(21.437895,-18.301052),(21.39087,-18.302913),(21.343947,-18.304773),(21.296818,-18.306633),(21.249896,-18.308494),(21.193466,-18.310768),(21.136932,-18.312938),(21.080604,-18.315212),(21.024174,-18.317382),(20.993436,-18.318612),(20.975081,-18.319346),(20.975081,-18.347768),(20.975081,-18.388592),(20.975081,-18.429313),(20.975081,-18.470138),(20.975081,-18.511065),(20.975081,-18.551993),(20.975081,-18.592714),(20.975081,-18.633539),(20.975081,-18.674363),(20.975081,-18.715187),(20.975081,-18.756012),(20.975081,-18.796836),(20.975081,-18.83766),(20.975081,-18.878381),(20.975081,-18.919206),(20.975081,-18.96003),(20.975081,-19.000958),(20.975494,-19.125912),(20.975609,-19.172106),(20.975804,-19.250865),(20.976321,-19.375819),(20.976735,-19.500876),(20.977045,-19.625829),(20.977561,-19.750783),(20.977975,-19.87584),(20.978285,-20.000794),(20.978698,-20.125851),(20.979112,-20.250804),(20.97936,-20.351028),(20.979422,-20.375758),(20.979835,-20.500711),(20.980249,-20.625768),(20.980662,-20.750722),(20.981075,-20.875779),(20.981486,-20.999958),(20.981489,-21.000733),(20.981592,-21.058403),(20.981902,-21.116074),(20.982109,-21.173642),(20.982212,-21.231209),(20.982419,-21.28888),(20.982626,-21.346448),(20.982832,-21.404119),(20.983039,-21.46179),(20.983143,-21.519357),(20.983453,-21.577028),(20.983659,-21.634699),(20.983866,-21.69237),(20.984073,-21.750041),(20.984279,-21.807608),(20.984486,-21.865176),(20.984693,-21.922847),(20.984796,-21.963981),(20.97198,-22.000671),(20.914723,-22.000671),(20.680629,-22.000671),(20.446534,-22.000671),(20.21244,-22.000671),(19.978346,-22.000671),(19.978449,-22.086764),(19.978656,-22.237246),(19.978863,-22.387625),(19.978966,-22.538106),(19.979173,-22.688588),(19.979276,-22.860671),(19.979586,-23.032546),(19.979793,-23.204526),(19.979855,-23.308551),(19.979896,-23.376505),(19.980103,-23.548587),(19.98031,-23.720567),(19.980483,-23.864489),(19.980516,-23.892494),(19.980723,-24.064473),(19.980826,-24.236556),(19.981137,-24.408431),(19.981343,-24.580411),(19.981447,-24.752493),(19.981757,-24.764999),(19.981757,-24.821016),(19.981757,-24.877137),(19.981757,-24.933257),(19.98186,-24.989275),(19.981963,-25.045395),(19.981963,-25.101412),(19.981963,-25.15743),(19.981963,-25.21355),(19.981963,-25.269671),(19.981963,-25.325688),(19.981963,-25.381705),(19.982067,-25.437826),(19.982067,-25.493843),(19.982067,-25.549964),(19.982067,-25.606084),(19.982067,-25.618988),(19.982067,-25.662102),(19.982067,-25.718119),(19.982067,-25.774136),(19.982067,-25.830257),(19.98217,-25.886274),(19.982273,-25.942395),(19.982273,-25.998515),(19.982273,-26.054533),(19.982273,-26.11055),(19.982273,-26.16667),(19.982377,-26.222688),(19.982377,-26.278808),(19.982377,-26.334826),(19.982377,-26.390946),(19.982377,-26.446963),(19.982377,-26.503084),(19.98248,-26.559101),(19.982583,-26.615119),(19.982583,-26.671239),(19.982583,-26.727256),(19.982583,-26.783377),(19.982583,-26.839394),(19.982687,-26.895515),(19.982687,-26.951532),(19.982687,-27.007653),(19.982687,-27.06367),(19.982687,-27.119687),(19.982687,-27.175808),(19.98279,-27.231928),(19.982894,-27.287946),(19.982894,-27.344066),(19.982894,-27.400187),(19.982894,-27.456101),(19.982894,-27.512118),(19.982894,-27.568239),(19.982894,-27.624359),(19.982997,-27.680376),(19.982997,-27.736497),(19.982997,-27.792618),(19.982997,-27.848532),(19.982997,-27.904652),(19.982997,-27.960773),(19.982997,-28.01679),(19.982997,-28.072911),(19.9831,-28.129031),(19.983204,-28.185049),(19.983204,-28.241066),(19.983204,-28.297186),(19.983204,-28.353204),(19.983204,-28.392684),(19.981653,-28.422347),(19.950544,-28.429271),(19.940209,-28.430202),(19.907859,-28.426481),(19.896697,-28.427721),(19.870342,-28.440847),(19.82559,-28.47671),(19.796342,-28.484152),(19.748076,-28.487149),(19.725338,-28.494177),(19.705598,-28.508026),(19.688338,-28.515984),(19.587982,-28.522702),(19.579197,-28.525183),(19.568759,-28.531177),(19.562351,-28.538102),(19.55646,-28.546163),(19.547571,-28.555879),(19.542197,-28.564043),(19.533722,-28.58244),(19.527417,-28.585954),(19.517806,-28.589365),(19.511708,-28.598047),(19.482872,-28.678248),(19.472434,-28.692718),(19.45507,-28.705223),(19.434607,-28.713492),(19.353475,-28.731889),(19.339005,-28.73747),(19.32836,-28.735816),(19.304485,-28.718659),(19.290016,-28.719693),(19.277304,-28.729408),(19.265935,-28.742637),(19.248571,-28.771576),(19.244954,-28.792453),(19.251569,-28.814261),(19.288879,-28.871001),(19.288362,-28.883094),(19.27658,-28.889502),(19.243714,-28.891879),(19.237616,-28.895599),(19.226661,-28.911516),(19.218496,-28.918854),(19.161548,-28.945416),(19.120207,-28.957508),(19.081657,-28.959368),(19.064707,-28.939834),(19.060056,-28.9357),(19.048997,-28.932393),(19.01396,-28.928466),(19.006932,-28.926295),(18.99639,-28.915547),(18.971172,-28.88051),(18.954533,-28.866661),(18.745656,-28.839892),(18.553937,-28.864697),(18.51766,-28.882267),(18.496163,-28.888261),(18.491615,-28.886298),(18.479419,-28.876272),(18.474975,-28.873999),(18.469394,-28.874929),(18.460713,-28.87989),(18.455235,-28.88144),(18.435701,-28.884231),(18.424746,-28.887228),(18.416994,-28.891672),(18.397564,-28.898907),(18.373069,-28.895186),(18.331005,-28.88144),(18.30868,-28.879993),(18.220727,-28.891259),(18.18507,-28.902214),(18.166467,-28.901904),(18.082441,-28.875962),(18.044097,-28.858392),(17.983326,-28.813744),(17.949529,-28.794727),(17.913252,-28.781291),(17.746338,-28.748632),(17.714091,-28.751112),(17.703239,-28.755556),(17.683396,-28.767545),(17.673784,-28.771576),(17.660968,-28.772299),(17.628722,-28.764135),(17.607121,-28.75566),(17.602574,-28.735403),(17.603401,-28.710805),(17.598026,-28.689617),(17.582937,-28.680212),(17.565987,-28.683519),(17.54697,-28.691374),(17.526609,-28.695818),(17.485165,-28.700159),(17.44093,-28.709564),(17.403619,-28.704293),(17.401759,-28.674218),(17.414265,-28.63267),(17.420569,-28.593396),(17.409614,-28.571382),(17.373027,-28.559393),(17.365276,-28.542546),(17.359488,-28.519808),(17.332409,-28.488596),(17.324244,-28.470509),(17.328172,-28.456143),(17.341194,-28.442811),(17.358558,-28.432785),(17.375404,-28.428961),(17.39132,-28.418833),(17.399692,-28.395268),(17.402069,-28.367776),(17.400002,-28.346382),(17.394834,-28.335117),(17.378918,-28.316307),(17.371993,-28.306075),(17.368066,-28.293776),(17.367033,-28.283544),(17.364139,-28.273312),(17.355044,-28.261013),(17.35122,-28.251401),(17.349773,-28.238689),(17.345742,-28.22763),(17.334476,-28.222876),(17.308638,-28.224116),(17.245179,-28.237448),(17.21314,-28.232074),(17.191953,-28.20882),(17.187095,-28.162001),(17.189782,-28.139367),(17.189989,-28.116836),(17.180894,-28.099472),(17.155779,-28.092548),(17.136349,-28.084693),(17.123223,-28.066503),(17.111751,-28.046039),(17.097798,-28.031156),(17.086533,-28.027022),(17.076404,-28.026815),(17.056664,-28.031156),(17.045398,-28.036324),(17.012015,-28.058338),(16.98535,-28.05224),(16.973775,-28.055031),(16.959512,-28.06857),(16.947523,-28.072704),(16.937498,-28.07105),(16.927473,-28.060198),(16.919618,-28.058338),(16.91352,-28.062679),(16.896467,-28.079939),(16.892953,-28.082626),(16.885512,-28.162001),(16.878173,-28.172026),(16.873833,-28.171509),(16.868665,-28.167892),(16.852025,-28.168305),(16.847788,-28.165411),(16.843757,-28.163861),(16.837659,-28.168202),(16.835695,-28.174196),(16.838383,-28.179467),(16.855953,-28.199415),(16.856366,-28.206649),(16.84107,-28.209853),(16.819779,-28.20975),(16.814301,-28.213471),(16.810271,-28.222876),(16.813268,-28.229904),(16.820813,-28.240032),(16.8266,-28.251918),(16.824637,-28.264527),(16.815232,-28.270831),(16.805103,-28.267731),(16.795181,-28.26122),(16.786499,-28.257602),(16.768723,-28.265354),(16.764382,-28.283234),(16.768413,-28.303698),(16.775544,-28.319097),(16.796835,-28.347829),(16.803863,-28.366123),(16.793217,-28.374288),(16.778541,-28.382969),(16.773374,-28.402916),(16.772133,-28.425757),(16.769343,-28.442604),(16.758387,-28.459657),(16.740404,-28.480948),(16.720457,-28.496037),(16.703817,-28.494487),(16.699683,-28.485185),(16.696789,-28.472473),(16.692345,-28.461104),(16.683457,-28.456246),(16.673121,-28.45976),(16.597364,-28.526216),(16.559123,-28.537068),(16.531528,-28.549677),(16.50538,-28.565387),(16.487071,-28.572931),(16.468516,-28.585138),(16.458507,-28.600518),(16.445811,-28.610528),(16.419444,-28.607029),(16.407888,-28.600518),(16.346853,-28.556085),(16.315929,-28.524347),(16.301036,-28.50449),(16.19337,-28.408787),(16.175792,-28.402114),(16.167979,-28.395766),(16.107677,-28.323337),(16.04363,-28.257094),(15.911143,-28.171319),(15.884288,-28.147638),(15.833181,-28.089288),(15.761485,-28.038344),(15.747813,-28.024591),(15.716807,-27.980401),(15.684255,-27.949884),(15.678884,-27.942804),(15.677989,-27.931248),(15.684581,-27.910821),(15.685802,-27.9013),(15.673025,-27.869887),(15.646007,-27.836602),(15.546235,-27.747817),(15.533539,-27.730157),(15.528656,-27.70615),(15.529063,-27.654229),(15.526866,-27.646173),(15.520681,-27.633722),(15.425059,-27.496515),(15.408946,-27.464532),(15.396821,-27.448663),(15.380382,-27.44199),(15.371755,-27.433038),(15.342459,-27.380548),(15.301606,-27.333103),(15.295258,-27.322442),(15.281749,-27.243341),(15.276215,-27.233575),(15.269542,-27.226332),(15.263682,-27.218357),(15.261241,-27.205743),(15.268077,-27.164809),(15.254161,-27.100193),(15.261241,-27.085626),(15.241222,-27.048761),(15.235362,-27.029474),(15.233246,-27.006931),(15.234223,-26.965427),(15.228526,-26.945571),(15.212738,-26.927992),(15.20574,-26.925551),(15.187511,-26.924249),(15.178559,-26.921157),(15.176606,-26.916762),(15.169688,-26.906183),(15.162364,-26.897068),(15.158865,-26.897068),(15.155935,-26.875095),(15.148936,-26.856378),(15.116873,-26.796157),(15.111664,-26.779229),(15.116222,-26.752048),(15.105724,-26.732354),(15.091563,-26.712172),(15.083018,-26.695245),(15.080821,-26.672133),(15.083507,-26.652114),(15.091807,-26.637953),(15.106619,-26.632582),(15.116954,-26.630304),(15.127126,-26.626723),(15.134613,-26.62713),(15.137706,-26.636651),(15.137706,-26.6749),(15.144542,-26.6749),(15.148448,-26.655043),(15.168142,-26.602797),(15.168712,-26.592218),(15.162283,-26.579034),(15.141938,-26.517999),(15.128754,-26.457615),(15.105642,-26.420587),(15.07252,-26.389744),(15.034679,-26.366306),(14.975271,-26.344171),(14.966319,-26.335219),(14.958832,-26.257013),(14.962657,-26.181329),(14.942556,-26.153904),(14.939138,-26.147149),(14.943126,-26.135349),(14.949067,-26.135024),(14.956879,-26.137628),(14.966319,-26.133559),(14.979828,-26.101658),(14.979015,-26.05877),(14.968272,-26.015802),(14.952647,-25.98268),(14.907725,-25.915297),(14.90211,-25.882501),(14.91863,-25.845473),(14.911143,-25.845473),(14.859874,-25.790297),(14.842784,-25.763604),(14.840343,-25.751723),(14.840343,-25.743341),(14.842784,-25.729425),(14.842784,-25.650323),(14.846934,-25.629815),(14.876964,-25.578058),(14.879731,-25.561456),(14.879568,-25.53867),(14.875336,-25.518487),(14.853282,-25.498712),(14.847016,-25.473321),(14.842784,-25.42783),(14.818044,-25.366957),(14.812511,-25.320082),(14.802419,-25.282892),(14.80128,-25.262791),(14.832286,-25.188165),(14.829356,-25.176446),(14.84962,-25.105076),(14.853282,-25.064223),(14.843272,-25.026056),(14.829438,-25.000095),(14.808767,-24.961114),(14.796153,-24.92197),(14.792979,-24.841892),(14.780772,-24.803399),(14.723481,-24.703546),(14.699474,-24.673028),(14.650157,-24.628839),(14.636078,-24.598891),(14.601899,-24.562433),(14.595876,-24.537205),(14.59962,-24.526544),(14.61378,-24.511651),(14.616954,-24.505792),(14.616954,-24.464776),(14.612315,-24.445733),(14.590099,-24.40838),(14.568126,-24.35768),(14.513927,-24.261489),(14.493175,-24.192315),(14.473643,-24.159112),(14.466807,-24.113214),(14.459239,-24.098321),(14.464203,-24.07822),(14.464122,-24.044692),(14.459646,-24.010431),(14.451915,-23.988458),(14.459239,-23.974542),(14.465831,-23.947198),(14.472911,-23.933852),(14.479177,-23.931085),(14.488292,-23.92962),(14.496593,-23.925958),(14.500255,-23.916436),(14.513927,-23.892267),(14.506033,-23.88006),(14.507091,-23.823907),(14.489513,-23.776056),(14.486583,-23.758477),(14.500255,-23.614435),(14.495291,-23.578546),(14.48406,-23.551039),(14.446951,-23.460382),(14.434337,-23.415216),(14.435069,-23.405532),(14.443126,-23.396661),(14.467459,-23.356052),(14.479747,-23.344822),(14.479503,-23.365818),(14.484386,-23.374933),(14.490408,-23.369724),(14.493419,-23.347833),(14.491954,-23.309177),(14.473481,-23.192804),(14.462738,-23.157403),(14.445567,-23.13128),(14.431326,-23.103285),(14.420665,-23.070977),(14.413829,-23.036228),(14.410167,-22.970798),(14.411388,-22.961196),(14.416026,-22.951593),(14.429047,-22.93255),(14.431895,-22.923028),(14.431895,-22.881768),(14.435069,-22.879815),(14.440929,-22.879815),(14.443858,-22.883477),(14.438162,-22.892267),(14.442719,-22.898533),(14.44809,-22.908136),(14.451915,-22.912205),(14.445974,-22.932387),(14.444347,-22.952813),(14.451671,-22.963311),(14.472911,-22.95379),(14.471446,-22.965102),(14.459239,-22.995294),(14.474294,-22.982192),(14.496837,-22.941827),(14.510509,-22.933282),(14.521983,-22.923435),(14.529307,-22.90016),(14.534353,-22.854181),(14.527517,-22.679457),(14.508556,-22.548028),(14.485118,-22.512384),(14.474294,-22.467869),(14.438162,-22.398858),(14.418712,-22.339776),(14.386485,-22.276462),(14.380382,-22.268487),(14.315196,-22.200291),(14.294281,-22.167413),(14.293468,-22.143324),(14.291352,-22.134454),(14.137218,-21.946384),(14.062022,-21.880955),(14.05421,-21.864679),(13.97169,-21.792739),(13.952322,-21.783949),(13.952159,-21.77337),(13.956391,-21.762953),(13.958995,-21.758396),(13.96518,-21.733168),(13.965831,-21.726739),(13.962657,-21.714044),(13.958263,-21.704848),(13.87794,-21.592218),(13.862804,-21.528009),(13.831716,-21.456476),(13.729259,-21.330255),(13.66684,-21.232354),(13.628103,-21.187595),(13.602712,-21.158136),(13.513357,-21.007908),(13.471934,-20.955011),(13.459158,-20.918064),(13.444835,-20.904067),(13.417735,-20.883884),(13.397634,-20.849298),(13.387218,-20.816095),(13.37672,-20.733575),(13.349457,-20.644464),(13.262706,-20.485121),(13.239024,-20.397149),(13.233572,-20.354669),(13.213064,-20.27337),(13.177745,-20.189223),(13.154633,-20.154229),(13.114425,-20.142573),(13.07088,-20.114229),(13.037345,-20.059465),(13.005382,-19.935479),(12.927745,-19.794692),(12.918468,-19.765558),(12.875987,-19.712009),(12.802501,-19.584731),(12.793956,-19.5617),(12.783051,-19.54697),(12.757579,-19.49684),(12.749278,-19.486017),(12.7046,-19.410821),(12.691091,-19.379002),(12.686046,-19.359796),(12.679698,-19.31406),(12.668305,-19.292413),(12.636241,-19.252618),(12.615733,-19.217543),(12.567393,-19.102309),(12.546723,-19.071466),(12.516449,-19.03753),(12.480317,-19.011),(12.470388,-18.997654),(12.458669,-18.925551),(12.450043,-18.904555),(12.286876,-18.697686),(12.174731,-18.627939),(12.125743,-18.574802),(12.11964,-18.569513),(12.102061,-18.545994),(12.091645,-18.540704),(12.082042,-18.534356),(12.031889,-18.50535),(12.015961,-18.452895),(12.012462,-18.434503),(11.999522,-18.403497),(11.99822,-18.39186),(11.998546,-18.381768),(11.997569,-18.372003),(11.999522,-18.361912),(11.976085,-18.329767),(11.937348,-18.23211),(11.931163,-18.225356),(11.849082,-18.143183),(11.83364,-18.098045),(11.826524,-18.03922),(11.808499,-17.991542),(11.796911,-17.956089),(11.784037,-17.859645),(11.760941,-17.786424),(11.745552,-17.683948),(11.734031,-17.632721),(11.726736,-17.599867),(11.717621,-17.546319),(11.72169,-17.467869),(11.755138,-17.266697),(11.766124,-17.252699),(11.766184,-17.252751),(11.779518,-17.254915),(11.796882,-17.263803),(11.80763,-17.26587),(11.82241,-17.264423),(11.827474,-17.260082),(11.829644,-17.253468),(11.835536,-17.245406),(11.853932,-17.233417),(11.894963,-17.214607),(11.942092,-17.180501),(11.98302,-17.161897),(12.028909,-17.148978),(12.075211,-17.142983),(12.082239,-17.14133),(12.08813,-17.139159),(12.095261,-17.139676),(12.105597,-17.146084),(12.13991,-17.156006),(12.151279,-17.153629),(12.156653,-17.149081),(12.16151,-17.146291),(12.170709,-17.149185),(12.177117,-17.154766),(12.179494,-17.161587),(12.181148,-17.169235),(12.185075,-17.17709),(12.190656,-17.182568),(12.200474,-17.189286),(12.21174,-17.195177),(12.222282,-17.197657),(12.23179,-17.201895),(12.236545,-17.2113),(12.239335,-17.220705),(12.242642,-17.224839),(12.314679,-17.218121),(12.379482,-17.220395),(12.393951,-17.21471),(12.407594,-17.204065),(12.417929,-17.203445),(12.4417,-17.216984),(12.46051,-17.223082),(12.519215,-17.227836),(12.554561,-17.235588),(12.56717,-17.234554),(12.591458,-17.222565),(12.63621,-17.185151),(12.660705,-17.17709),(12.685923,-17.173576),(12.704733,-17.164274),(12.739873,-17.135542),(12.784315,-17.115078),(12.818318,-17.104846),(12.824932,-17.096371),(12.833097,-17.081799),(12.842296,-17.074047),(12.849944,-17.070843),(12.867721,-17.065572),(12.876402,-17.059784),(12.880536,-17.050793),(12.882293,-17.039527),(12.887151,-17.029812),(12.911025,-17.023508),(12.930456,-17.014206),(12.961668,-17.007385),(13.014275,-16.977929),(13.121762,-16.959842),(13.1445,-16.952401),(13.166307,-16.951057),(13.184911,-16.964183)] +Niger [(14.216217,22.616295),(14.23172,22.617949),(14.294869,22.649833),(14.466021,22.736184),(14.63707,22.822536),(14.808016,22.908887),(14.979168,22.995316),(14.979271,22.995368),(14.979909,22.995664),(14.991984,22.933097),(15.003973,22.87044),(15.015858,22.807782),(15.02795,22.745176),(15.039939,22.682544),(15.051928,22.619861),(15.064021,22.557177),(15.075906,22.494597),(15.087998,22.431914),(15.099987,22.369282),(15.11208,22.306598),(15.124069,22.243966),(15.136058,22.181335),(15.14815,22.118651),(15.156315,22.076173),(15.16448,22.033695),(15.172024,21.993387),(15.172438,21.973647),(15.173265,21.921557),(15.174608,21.847918),(15.176055,21.763427),(15.177502,21.678988),(15.178846,21.605297),(15.179776,21.553208),(15.180086,21.533467),(15.180396,21.507267),(15.18453,21.491196),(15.20024,21.476468),(15.247472,21.453265),(15.266592,21.440656),(15.301009,21.40133),(15.32478,21.366707),(15.34142,21.342161),(15.358266,21.317718),(15.375009,21.293223),(15.391753,21.268729),(15.408496,21.244234),(15.425342,21.219791),(15.441982,21.195193),(15.458829,21.17075),(15.475572,21.146256),(15.492212,21.121761),(15.509058,21.097266),(15.525801,21.072772),(15.542544,21.048277),(15.559288,21.023782),(15.576134,20.999339),(15.592774,20.974793),(15.60931,20.95066),(15.569003,20.928905),(15.544301,20.890302),(15.536033,20.844052),(15.544198,20.79899),(15.556084,20.773669),(15.570243,20.751913),(15.588123,20.732792),(15.669462,20.671866),(15.697264,20.642875),(15.755555,20.582001),(15.813846,20.521074),(15.872033,20.460199),(15.930324,20.399324),(15.953992,20.374571),(15.968151,20.352816),(15.970322,20.336331),(15.962984,20.319226),(15.94252,20.283957),(15.902419,20.214943),(15.862318,20.145955),(15.822321,20.076993),(15.782323,20.00803),(15.767234,19.982037),(15.736021,19.903541),(15.732404,19.861838),(15.728373,19.816259),(15.724342,19.770577),(15.720208,19.724999),(15.716177,19.679317),(15.712146,19.633764),(15.708116,19.588108),(15.704085,19.542529),(15.700054,19.496873),(15.696023,19.451243),(15.692096,19.405638),(15.687962,19.360008),(15.683931,19.314404),(15.6799,19.268747),(15.67587,19.223169),(15.671839,19.177539),(15.667808,19.131882),(15.663777,19.086304),(15.659747,19.040648),(15.655716,18.995069),(15.651582,18.949387),(15.647551,18.903834),(15.64352,18.858178),(15.639593,18.8126),(15.635562,18.766943),(15.631531,18.721313),(15.6275,18.675709),(15.62347,18.630078),(15.619439,18.584474),(15.615305,18.538818),(15.611274,18.493239),(15.607243,18.447583),(15.603213,18.402004),(15.599182,18.356348),(15.595151,18.310718),(15.59112,18.265088),(15.587193,18.219483),(15.583162,18.173879),(15.579028,18.128249),(15.574997,18.082644),(15.570966,18.03704),(15.567698,18.000039),(15.566936,17.991409),(15.562905,17.945779),(15.558874,17.900149),(15.554843,17.85457),(15.550813,17.808888),(15.546782,17.76331),(15.542648,17.717679),(15.538617,17.672049),(15.53469,17.626367),(15.530659,17.580737),(15.526628,17.535107),(15.522597,17.489528),(15.518567,17.443846),(15.514536,17.398267),(15.510505,17.352637),(15.506371,17.307058),(15.50234,17.261376),(15.498309,17.215746),(15.494279,17.170116),(15.490248,17.124537),(15.486217,17.078907),(15.48229,17.033277),(15.478259,16.987698),(15.474125,16.942068),(15.471954,16.916746),(15.468517,16.90491),(15.465547,16.89468),(15.452421,16.876232),(15.418418,16.840885),(15.3624,16.782646),(15.30628,16.7242),(15.250159,16.665909),(15.194039,16.607566),(15.137918,16.549275),(15.081797,16.490881),(15.02578,16.432538),(14.969659,16.374247),(14.913539,16.315904),(14.857418,16.257613),(14.801401,16.199219),(14.745384,16.140876),(14.689263,16.082534),(14.633143,16.024243),(14.576919,15.965797),(14.520798,15.907557),(14.482144,15.86725),(14.423543,15.806323),(14.368973,15.749634),(14.340447,15.710722),(14.310682,15.669949),(14.280709,15.629254),(14.250944,15.588455),(14.221178,15.547734),(14.191206,15.507039),(14.16144,15.466292),(14.131571,15.425545),(14.101702,15.384876),(14.071833,15.344181),(14.042068,15.303434),(14.012199,15.262687),(13.98233,15.221992),(13.952564,15.181219),(13.922592,15.140498),(13.892826,15.0997),(13.862957,15.059056),(13.833915,15.019601),(13.808077,14.965573),(13.775417,14.897438),(13.760534,14.866355),(13.75516,14.847209),(13.75392,14.825944),(13.75795,14.805532),(13.772006,14.762873),(13.773247,14.742512),(13.764358,14.719077),(13.749165,14.711248),(13.730045,14.709646),(13.709995,14.705124),(13.700021,14.697734),(13.668085,14.662879),(13.648345,14.649417),(13.644727,14.644249),(13.646278,14.639056),(13.657026,14.629651),(13.660334,14.623734),(13.666742,14.585519),(13.665605,14.56689),(13.657853,14.548725),(13.624264,14.521311),(13.608052,14.518266),(13.584369,14.513818),(13.543855,14.510562),(13.507785,14.495964),(13.482257,14.483587),(13.449494,14.439042),(13.449184,14.380131),(13.468304,14.310678),(13.482257,14.259777),(13.492695,14.213862),(13.499723,14.181978),(13.506958,14.150119),(13.514193,14.118235),(13.521324,14.086428),(13.528455,14.054518),(13.53569,14.022737),(13.542821,13.990853),(13.549953,13.958994),(13.557188,13.927161),(13.564319,13.895225),(13.57145,13.863418),(13.578582,13.831534),(13.585816,13.799727),(13.592948,13.767843),(13.600182,13.735984),(13.607314,13.7041),(13.35999,13.714409),(13.35892,13.714232),(13.329915,13.709423),(13.320716,13.702291),(13.321233,13.685109),(13.30604,13.674644),(13.266146,13.638032),(13.259221,13.628963),(13.253847,13.618911),(13.253537,13.614881),(13.258601,13.607801),(13.260048,13.602142),(13.257258,13.594236),(13.25085,13.592763),(13.243822,13.593254),(13.239481,13.591575),(13.230179,13.577751),(13.232246,13.572894),(13.246405,13.571085),(13.228112,13.548037),(13.203927,13.539381),(13.176952,13.534472),(13.150184,13.522664),(13.153698,13.5307),(13.155868,13.540777),(13.153698,13.548761),(13.143983,13.550621),(13.138402,13.546539),(13.13096,13.529821),(13.123519,13.522664),(13.123829,13.533439),(13.123519,13.536953),(13.107086,13.531552),(13.088276,13.530596),(13.07701,13.534679),(13.082488,13.544394),(13.082488,13.550621),(13.054893,13.548373),(13.029261,13.539175),(12.939344,13.495069),(12.911749,13.489901),(12.883844,13.495999),(12.87971,13.479695),(12.870614,13.478868),(12.857799,13.484914),(12.842296,13.489126),(12.842296,13.481685),(12.850564,13.479385),(12.857592,13.475328),(12.863897,13.469282),(12.870201,13.461247),(12.857075,13.450653),(12.829273,13.420293),(12.82855,13.417451),(12.829997,13.409364),(12.829273,13.406625),(12.826276,13.406056),(12.817594,13.407296),(12.815011,13.406625),(12.81067,13.404299),(12.797337,13.399648),(12.794547,13.396057),(12.798474,13.384275),(12.804779,13.376859),(12.805192,13.372983),(12.791136,13.371846),(12.788966,13.375464),(12.783281,13.382337),(12.77739,13.386703),(12.7746,13.382698),(12.773256,13.382233),(12.766538,13.365671),(12.740803,13.33482),(12.72437,13.322289),(12.701736,13.31725),(12.690057,13.311979),(12.683132,13.288932),(12.674347,13.283712),(12.659155,13.286606),(12.649233,13.293608),(12.640654,13.302393),(12.630009,13.310377),(12.628769,13.294177),(12.626805,13.28826),(12.62019,13.281438),(12.613266,13.276503),(12.606134,13.27423),(12.602724,13.279992),(12.597556,13.276503),(12.561073,13.242759),(12.558179,13.226532),(12.545156,13.208213),(12.538335,13.193356),(12.554251,13.187542),(12.547947,13.165451),(12.542572,13.15478),(12.534511,13.145917),(12.526966,13.143643),(12.504332,13.139613),(12.499784,13.135969),(12.491206,13.082639),(12.486659,13.070805),(12.472706,13.064165),(12.456376,13.067421),(12.437256,13.074009),(12.414312,13.077627),(12.386742,13.07858),(12.351576,13.079797),(12.335247,13.084474),(12.325738,13.078815),(12.3184,13.083854),(12.310959,13.092639),(12.301037,13.098116),(12.291528,13.096824),(12.285327,13.092019),(12.280883,13.086851),(12.277162,13.084474),(12.269204,13.085869),(12.267447,13.089641),(12.267034,13.095171),(12.26321,13.101863),(12.251841,13.108012),(12.201405,13.125427),(12.184765,13.122094),(12.163784,13.107961),(12.149625,13.105584),(12.116449,13.102845),(12.057641,13.116772),(12.038004,13.126358),(11.991908,13.162092),(11.852795,13.245394),(11.742311,13.289448),(11.440004,13.364431),(10.953005,13.361899),(10.771311,13.382698),(10.674986,13.375102),(10.647882,13.369406),(10.161529,13.267202),(10.119775,13.250097),(9.930122,13.145969),(9.912449,13.1292),(9.908211,13.118787),(9.904387,13.094344),(9.899736,13.084319),(9.891778,13.076231),(9.887747,13.075508),(9.882683,13.076231),(9.872038,13.072562),(9.852814,13.060444),(9.819328,13.029206),(9.647657,12.830538),(9.640734,12.822526),(9.629262,12.812785),(9.61841,12.805783),(9.611485,12.802941),(9.590298,12.80139),(9.377908,12.825782),(9.359821,12.824593),(9.306491,12.809839),(9.269387,12.809788),(8.977872,12.843183),(8.942792,12.847201),(8.679346,12.923579),(8.642655,12.944353),(8.625499,12.964791),(8.596767,13.011403),(8.577233,13.032436),(8.533515,13.06463),(8.50964,13.075921),(8.482665,13.081554),(8.47078,13.081657),(8.445975,13.078712),(8.434503,13.075095),(8.423134,13.069488),(8.419,13.066697),(8.415176,13.067059),(8.405151,13.070857),(8.397813,13.075405),(8.297974,13.164624),(8.273066,13.193873),(8.259836,13.205216),(8.246728,13.211292),(8.245677,13.211779),(8.216015,13.219608),(8.201442,13.226377),(8.167646,13.252371),(8.14005,13.269114),(8.116589,13.29105),(8.105117,13.299654),(8.077522,13.310016),(7.823997,13.345285),(7.789684,13.341977),(7.755578,13.327146),(7.391982,13.112534),(7.360667,13.102638),(7.324803,13.102845),(7.22052,13.121577),(7.198196,13.117392),(7.137424,13.047241),(7.090089,13.006391),(7.068695,12.996081),(7.062921,12.99514),(7.044924,12.992206),(6.970509,12.991844),(6.937127,12.995409),(6.906637,13.006856),(6.874185,13.030498),(6.837908,13.063493),(6.778687,13.130285),(6.753879,13.1749),(6.672543,13.321178),(6.509297,13.496051),(6.368944,13.626275),(6.302178,13.663844),(6.284208,13.667656),(6.229831,13.679192),(6.210608,13.675265),(6.152833,13.643561),(6.142395,13.640822),(6.13578,13.643664),(6.129476,13.649065),(6.12038,13.654077),(5.837349,13.763748),(5.554317,13.873418),(5.521761,13.880291),(5.374069,13.855254),(5.346268,13.841559),(5.334795,13.826702),(5.318052,13.789935),(5.304203,13.773941),(5.273507,13.752521),(5.262035,13.747741),(5.227618,13.741333),(5.084371,13.747534),(4.983189,13.72968),(4.925105,13.733091),(4.902367,13.742987),(4.879836,13.764019),(4.856892,13.774096),(4.824852,13.770737),(4.625845,13.723151),(4.506732,13.694669),(4.452575,13.673766),(4.405756,13.641236),(4.248453,13.494139),(4.220548,13.480625),(4.190472,13.475018),(4.125774,13.472977),(4.123707,13.233586),(4.120325,13.210608),(4.088773,12.996236),(3.929093,12.750386),(3.864394,12.689666),(3.6457,12.528539),(3.641565,12.517997),(3.651487,12.26902),(3.624512,12.137658),(3.623995,12.09425),(3.648903,12.021024),(3.653554,11.986815),(3.636604,11.953587),(3.619551,11.936275),(3.608492,11.921961),(3.603945,11.905579),(3.609733,11.848787),(3.612937,11.838968),(3.618621,11.828375),(3.622032,11.825016),(3.626269,11.822535),(3.64973,11.797834),(3.662029,11.779489),(3.66699,11.759697),(3.659859,11.738096),(3.647767,11.726004),(3.5964,11.695773),(3.589372,11.701509),(3.58007,11.717374),(3.571699,11.72187),(3.568185,11.729518),(3.566221,11.738354),(3.5625,11.746003),(3.557436,11.761815),(3.553715,11.76967),(3.549271,11.773753),(3.54369,11.77706),(3.536972,11.782899),(3.503486,11.829977),(3.495424,11.837521),(3.484262,11.84517),(3.447675,11.857985),(3.40282,11.88527),(3.391658,11.888836),(3.380909,11.895037),(3.369127,11.897311),(3.355071,11.889043),(3.341635,11.882893),(3.327786,11.886407),(3.315487,11.894572),(3.307012,11.902375),(3.292749,11.918653),(3.269908,11.956532),(3.26257,11.974722),(3.257609,11.997408),(3.255129,12.018751),(3.252028,12.024125),(3.238489,12.034202),(3.235389,12.039525),(3.063409,12.193779),(3.022895,12.252483),(3.013077,12.271087),(3.009356,12.276306),(3.000984,12.27889),(2.967808,12.282559),(2.956026,12.286848),(2.944967,12.292481),(2.936802,12.300852),(2.930394,12.323332),(2.922643,12.335631),(2.912618,12.346328),(2.892257,12.357076),(2.844301,12.399244),(2.832416,12.384051),(2.808645,12.383225),(2.798413,12.381209),(2.778156,12.381933),(2.768544,12.380072),(2.757485,12.372941),(2.741465,12.354131),(2.732267,12.346793),(2.721828,12.344261),(2.711286,12.351237),(2.700434,12.347103),(2.69506,12.341212),(2.692166,12.334029),(2.690202,12.326174),(2.686895,12.318422),(2.672942,12.296718),(2.663847,12.291551),(2.628604,12.300801),(2.621679,12.299509),(2.60969,12.290775),(2.603593,12.28783),(2.598632,12.288347),(2.590053,12.293308),(2.586126,12.294031),(2.550573,12.284523),(2.546025,12.281629),(2.53693,12.27305),(2.531246,12.269691),(2.523701,12.269175),(2.5206,12.273671),(2.51905,12.279148),(2.516259,12.281422),(2.504477,12.281835),(2.495072,12.283231),(2.488044,12.280388),(2.483393,12.268038),(2.477709,12.257599),(2.472644,12.259253),(2.46696,12.265041),(2.459519,12.266746),(2.454351,12.263025),(2.446083,12.251811),(2.443706,12.249228),(2.43089,12.247987),(2.407946,12.248918),(2.3942,12.247212),(2.370015,12.236257),(2.36123,12.218894),(2.362057,12.196156),(2.371875,12.140449),(2.405568,12.046708),(2.41115,12.036786),(2.418281,12.028673),(2.428616,12.020353),(2.439262,12.015598),(2.44877,12.013376),(2.455281,12.008674),(2.457348,11.99622),(2.456728,11.979063),(2.44939,11.977513),(2.438951,11.981285),(2.428926,11.980355),(2.422002,11.973172),(2.414147,11.958703),(2.407429,11.95121),(2.398747,11.948781),(2.389342,11.950073),(2.380867,11.947902),(2.375183,11.935035),(2.376526,11.924079),(2.388205,11.906768),(2.390169,11.896536),(2.338079,11.940099),(2.258807,12.048775),(2.188527,12.145461),(2.113803,12.247987),(2.070912,12.306898),(2.051792,12.341935),(2.054169,12.370926),(2.069258,12.383328),(2.126929,12.39511),(2.131477,12.398521),(2.138711,12.408339),(2.145016,12.41175),(2.148013,12.410768),(2.158762,12.405652),(2.163826,12.404774),(2.223254,12.409735),(2.23793,12.413455),(2.244958,12.424152),(2.242994,12.446166),(2.243098,12.451179),(2.246302,12.461669),(2.246302,12.465907),(2.242271,12.473452),(2.229559,12.487146),(2.223874,12.494846),(2.215709,12.510349),(2.210852,12.523061),(2.203204,12.583316),(2.2,12.595925),(2.193282,12.609464),(2.18429,12.620626),(2.165997,12.631737),(2.155455,12.640005),(2.144809,12.650753),(2.140985,12.656128),(2.135197,12.67561),(2.109049,12.705634),(2.068948,12.716279),(1.971693,12.724237),(1.962495,12.719121),(1.945545,12.700828),(1.934486,12.693542),(1.926631,12.694989),(1.919603,12.698347),(1.911438,12.696642),(1.906994,12.691836),(1.900173,12.678452),(1.883223,12.653957),(1.872681,12.634217),(1.860692,12.61706),(1.843742,12.606105),(1.826069,12.604193),(1.699462,12.61489),(1.597039,12.623675),(1.563863,12.63215),(1.535854,12.647498),(1.467124,12.704394),(1.412037,12.749869),(1.330595,12.817177),(1.304806,12.838508),(1.230756,12.899756),(1.170915,12.949211),(1.113968,12.996236),(1.083789,13.01099),(1.012269,13.016881),(0.983536,13.032358),(0.974648,13.048326),(0.971754,13.067317),(0.971661,13.085782),(0.971341,13.149405),(0.970824,13.243534),(0.970411,13.328335),(0.983536,13.36841),(1.004724,13.364844),(1.084719,13.33358),(1.138462,13.320377),(1.160683,13.311307),(1.167608,13.313426),(1.177633,13.34761),(1.182387,13.358695),(1.187658,13.364121),(1.205745,13.358901),(1.222075,13.344613),(1.241505,13.335544),(1.268687,13.34606),(1.249567,13.367066),(1.23427,13.377583),(1.217424,13.381768),(1.175049,13.38691),(1.157169,13.392646),(1.048235,13.441945),(1.015266,13.465665),(0.996559,13.496051),(0.991391,13.541113),(0.95098,13.583203),(0.896927,13.614932),(0.850728,13.628601),(0.813108,13.625035),(0.795331,13.625965),(0.775694,13.635293),(0.763705,13.646429),(0.762878,13.653044),(0.766289,13.657178),(0.767012,13.660873),(0.766185,13.666971),(0.766392,13.67511),(0.763498,13.682034),(0.75337,13.684101),(0.620665,13.679967),(0.59431,13.688881),(0.58046,13.713789),(0.584801,13.729008),(0.594516,13.74986),(0.600511,13.768721),(0.593689,13.778101),(0.574466,13.785025),(0.5632,13.796782),(0.552968,13.810037),(0.519275,13.831637),(0.504082,13.845358),(0.499926,13.851015),(0.49199,13.861817),(0.483515,13.880343),(0.47566,13.888947),(0.454809,13.902357),(0.448634,13.909617),(0.448065,13.920108),(0.456256,13.938143),(0.456695,13.94685),(0.445921,13.958581),(0.428377,13.967495),(0.411866,13.978424),(0.403934,13.996072),(0.401866,14.013797),(0.391376,14.022582),(0.378354,14.028809),(0.369155,14.039067),(0.369052,14.047852),(0.37732,14.063949),(0.378457,14.070434),(0.37422,14.078883),(0.368742,14.084154),(0.362463,14.088857),(0.356262,14.095704),(0.343575,14.114333),(0.339157,14.12578),(0.343472,14.137536),(0.367398,14.173787),(0.372153,14.185647),(0.377424,14.216601),(0.381118,14.22567),(0.390601,14.237633),(0.391635,14.245927),(0.388792,14.251612),(0.346366,14.307577),(0.202382,14.435545),(0.18865,14.44775),(0.158832,14.496119),(0.152941,14.54671),(0.219707,14.731221),(0.213092,14.761632),(0.184102,14.819562),(0.185756,14.84819),(0.198003,14.863797),(0.211852,14.874804),(0.220844,14.888214),(0.218467,14.910977),(0.212782,14.960716),(0.213196,14.985417),(0.221257,14.995933),(0.353187,14.963429),(0.387035,14.963248),(0.418791,14.969888),(0.483515,14.992109),(0.514831,14.993556),(0.670067,14.939735),(0.683813,14.940872),(0.711408,14.947461),(0.739934,14.958339),(0.769183,14.969062),(0.922145,14.973971),(0.949327,14.979552),(0.973718,14.991257),(1.057537,15.067118),(1.123063,15.126313),(1.203161,15.198789),(1.270857,15.259897),(1.297832,15.275735),(1.331526,15.283616),(1.405216,15.286148),(1.501748,15.289352),(1.598383,15.292608),(1.694914,15.295734),(1.791446,15.29899),(1.888081,15.302194),(1.984612,15.305398),(2.081351,15.308576),(2.177882,15.311831),(2.274517,15.314984),(2.371049,15.318239),(2.467684,15.321417),(2.564215,15.324595),(2.660953,15.327825),(2.757485,15.331081),(2.85412,15.334207),(2.950651,15.337463),(3.000158,15.339117),(3.005739,15.35232),(3.005842,15.389294),(3.007806,15.407665),(3.010286,15.417665),(3.017521,15.422832),(3.03354,15.42645),(3.073021,15.427199),(3.19229,15.40751),(3.380393,15.376324),(3.483332,15.359296),(3.48881,15.357539),(3.507103,15.353973),(3.516508,15.469186),(3.526534,15.495954),(3.614074,15.547734),(3.692208,15.63145),(3.728899,15.65088),(3.808377,15.665582),(3.846101,15.685297),(3.871215,15.714804),(3.873849,15.720882),(3.886512,15.750099),(3.89447,15.78865),(3.903462,15.886318),(3.909869,15.904767),(3.925062,15.927608),(3.98325,15.983987),(3.98418,15.986881),(3.9848,15.989826),(3.984594,15.995924),(3.984697,15.997371),(3.98449,15.998766),(3.983973,16.00011),(3.98325,16.00135),(3.970848,16.030857),(3.96692,16.058504),(3.971158,16.086099),(4.060558,16.298334),(4.075647,16.321072),(4.094768,16.340812),(4.118229,16.358331),(4.16174,16.379983),(4.1759,16.392644),(4.183961,16.416053),(4.183444,16.526538),(4.183031,16.612682),(4.182308,16.746472),(4.181998,16.809621),(4.184581,16.818509),(4.197604,16.838508),(4.202048,16.848895),(4.19657,16.947132),(4.197811,16.965219),(4.203702,16.982789),(4.21166,16.986044),(4.222305,16.986561),(4.235638,16.995863),(4.235328,17.100818),(4.235018,17.163915),(4.234707,17.287525),(4.234294,17.411186),(4.233777,17.534796),(4.233364,17.658458),(4.233054,17.78212),(4.23264,17.905782),(4.232408,17.975234),(4.232227,18.029443),(4.231917,18.153105),(4.231504,18.276741),(4.23109,18.400325),(4.230677,18.524038),(4.230271,18.645374),(4.230263,18.647648),(4.22985,18.771284),(4.229436,18.894946),(4.229023,19.018582),(4.22861,19.142244),(4.417229,19.178417),(4.605848,19.214616),(4.794467,19.250764),(4.983189,19.286963),(5.082821,19.306006),(5.301826,19.347993),(5.520831,19.389903),(5.620463,19.408997),(5.749344,19.433699),(5.794302,19.449796),(5.837607,19.478631),(5.916362,19.546276),(5.979407,19.600536),(6.042556,19.654745),(6.105601,19.709005),(6.168646,19.763213),(6.231692,19.817474),(6.294737,19.871682),(6.357782,19.925917),(6.420931,19.980177),(6.483769,20.034385),(6.546918,20.088646),(6.609963,20.142854),(6.672957,20.197115),(6.736105,20.251323),(6.799047,20.305532),(6.862092,20.35974),(6.925138,20.414001),(6.982808,20.46361),(7.020532,20.495443),(7.098047,20.558746),(7.169464,20.616986),(7.240881,20.675173),(7.312297,20.733464),(7.383611,20.791652),(7.482726,20.872577),(7.613778,20.94973),(7.681887,20.989831),(7.74979,21.029984),(7.8179,21.070085),(7.886009,21.110237),(7.953912,21.15039),(8.022022,21.190491),(8.090131,21.230592),(8.158034,21.270744),(8.226143,21.310845),(8.294253,21.350946),(8.362259,21.391047),(8.430265,21.431148),(8.498375,21.4713),(8.566381,21.51135),(8.634491,21.551502),(8.702497,21.591655),(8.770606,21.631807),(8.838613,21.671908),(8.906722,21.712061),(8.974832,21.75211),(9.042838,21.792263),(9.110844,21.832364),(9.178953,21.872516),(9.24696,21.912565),(9.314966,21.952718),(9.383075,21.992819),(9.451082,22.03292),(9.519088,22.073072),(9.587197,22.113225),(9.655203,22.153326),(9.723313,22.193427),(9.791319,22.233579),(9.859325,22.27368),(9.927435,22.313781),(9.995441,22.353882),(10.063447,22.394035),(10.131557,22.434136),(10.199563,22.474237),(10.267569,22.514337),(10.335679,22.554542),(10.403788,22.594643),(10.471691,22.634744),(10.539801,22.674896),(10.607807,22.714945),(10.675813,22.755046),(10.743923,22.795199),(10.812032,22.835326),(10.880038,22.875375),(10.948148,22.915527),(11.016257,22.955654),(11.084367,22.995755),(11.15227,23.035882),(11.220379,23.07606),(11.288489,23.116161),(11.356392,23.156262),(11.424501,23.196415),(11.492611,23.236516),(11.560514,23.276591),(11.628623,23.316717),(11.696733,23.356844),(11.764739,23.396945),(11.832745,23.437046),(11.900855,23.477173),(11.968861,23.517351),(12.057021,23.497611),(12.145284,23.477948),(12.233651,23.458285),(12.321811,23.438622),(12.409971,23.418882),(12.498337,23.399245),(12.586497,23.379582),(12.674657,23.359893),(12.763024,23.340179),(12.851184,23.32049),(12.939447,23.300827),(13.027607,23.281138),(13.115871,23.261449),(13.204134,23.241709),(13.292398,23.222098),(13.380661,23.202383),(13.482257,23.179672),(13.599562,23.119029),(13.655786,23.072624),(13.780895,22.969633),(13.906055,22.866616),(14.031009,22.763702),(14.156169,22.660737),(14.201644,22.62322),(14.216217,22.616295)] +Norfolk Island [(167.984141,-29.017836),(167.996349,-29.025649),(167.99464,-29.042576),(167.985037,-29.059259),(167.973481,-29.066339),(167.970063,-29.068943),(167.967947,-29.072035),(167.966482,-29.075779),(167.966645,-29.080011),(167.960704,-29.065606),(167.95045,-29.056573),(167.939708,-29.055759),(167.932628,-29.066339),(167.926443,-29.066339),(167.924327,-29.055597),(167.920177,-29.045505),(167.918305,-29.036391),(167.922374,-29.028741),(167.929942,-29.019464),(167.930512,-29.01214),(167.924571,-29.008722),(167.91212,-29.011651),(167.926524,-28.997491),(167.945567,-29.001235),(167.965831,-29.011814),(167.984141,-29.017836)] +Niue [(-169.851145,-18.965102),(-169.825307,-18.96795),(-169.822621,-18.972589),(-169.79426,-19.047133),(-169.782908,-19.068943),(-169.796864,-19.073907),(-169.805979,-19.081964),(-169.823842,-19.102309),(-169.865427,-19.129653),(-169.872996,-19.125909),(-169.880971,-19.129083),(-169.898915,-19.142755),(-169.902048,-19.132989),(-169.909983,-19.128595),(-169.921091,-19.128025),(-169.933705,-19.129653),(-169.930531,-19.126397),(-169.927968,-19.122817),(-169.924794,-19.119317),(-169.919423,-19.115981),(-169.924062,-19.099379),(-169.926259,-19.094903),(-169.950429,-19.087335),(-169.947703,-19.069431),(-169.926259,-19.03753),(-169.927968,-19.031834),(-169.931589,-19.026544),(-169.934397,-19.020929),(-169.933705,-19.013604),(-169.930247,-19.008396),(-169.926381,-19.00742),(-169.921824,-19.007013),(-169.916005,-19.003106),(-169.895823,-18.965102),(-169.863718,-18.964044),(-169.851145,-18.965102)] +Nepal [(81.779018,30.358045),(81.801497,30.3613),(81.822788,30.368173),(81.844389,30.371171),(81.887177,30.354996),(81.92149,30.357373),(81.941231,30.353962),(81.950119,30.348381),(81.962108,30.333653),(81.969032,30.328279),(81.978851,30.326832),(81.988463,30.329778),(81.998178,30.334067),(82.007635,30.336806),(82.049854,30.339235),(82.073212,30.337219),(82.088767,30.330088),(82.093211,30.31474),(82.0821,30.257327),(82.084994,30.230817),(82.095898,30.213454),(82.114243,30.202189),(82.140391,30.194024),(82.155688,30.181363),(82.150985,30.162243),(82.127576,30.124364),(82.128713,30.110773),(82.135224,30.089741),(82.144526,30.069432),(82.153827,30.058166),(82.17424,30.055479),(82.192171,30.061474),(82.210671,30.064419),(82.255733,30.039666),(82.27599,30.036927),(82.296506,30.036824),(82.318882,30.031811),(82.32715,30.025714),(82.339087,30.009229),(82.347304,30.003441),(82.363737,30.001167),(82.379137,30.003234),(82.394433,30.003131),(82.409781,29.994604),(82.422855,29.97967),(82.438926,29.965769),(82.457065,29.95409),(82.475823,29.945512),(82.524244,29.932386),(82.541814,29.923239),(82.618502,29.839833),(82.635865,29.829033),(82.646407,29.829343),(82.666974,29.835028),(82.67731,29.834407),(82.688988,29.827741),(82.689195,29.819576),(82.684648,29.809706),(82.681754,29.797717),(82.690952,29.780741),(82.751517,29.739323),(82.77012,29.714156),(82.781076,29.705113),(82.793065,29.699506),(82.805157,29.695966),(82.815544,29.690127),(82.822365,29.677957),(82.836576,29.659793),(82.857454,29.666201),(82.880088,29.681109),(82.899725,29.688422),(82.90913,29.684623),(82.923703,29.669663),(82.931919,29.664185),(82.9421,29.662945),(82.964321,29.665891),(82.975896,29.66403),(83.032895,29.620467),(83.040905,29.610287),(83.054496,29.586051),(83.064935,29.57879),(83.079559,29.579643),(83.085553,29.590391),(83.088447,29.604421),(83.093357,29.615144),(83.100281,29.617676),(83.126688,29.623413),(83.150046,29.625635),(83.162086,29.62517),(83.173558,29.621268),(83.187253,29.612095),(83.19087,29.606902),(83.194487,29.593647),(83.197175,29.589099),(83.204926,29.584914),(83.221876,29.579824),(83.229317,29.575948),(83.252572,29.5552),(83.26084,29.542048),(83.26332,29.527088),(83.25991,29.518768),(83.254019,29.51329),(83.248748,29.506572),(83.247197,29.494738),(83.26394,29.473008),(83.327968,29.485023),(83.355201,29.460735),(83.357785,29.442752),(83.355201,29.427972),(83.35701,29.415363),(83.38724,29.393504),(83.389411,29.382445),(83.388274,29.370224),(83.393132,29.356245),(83.405017,29.346479),(83.417781,29.340898),(83.428168,29.333456),(83.437212,29.305241),(83.448219,29.296766),(83.475814,29.286921),(83.493384,29.276147),(83.502996,29.261032),(83.512504,29.223489),(83.514571,29.20181),(83.517052,29.191708),(83.523718,29.183594),(83.535965,29.179228),(83.56139,29.176231),(83.572035,29.168402),(83.583818,29.162149),(83.624435,29.155457),(83.639266,29.154759),(83.654992,29.160414),(83.65601,29.160779),(83.663399,29.174809),(83.66774,29.192328),(83.675078,29.208761),(83.69549,29.226176),(83.719727,29.233591),(83.745358,29.23447),(83.770628,29.232248),(83.79321,29.234754),(83.837859,29.254004),(83.86163,29.260205),(83.868865,29.263435),(83.877391,29.276767),(83.883438,29.282012),(83.914392,29.289402),(83.93315,29.291779),(83.945553,29.291831),(83.957955,29.28904),(83.967257,29.284699),(83.973251,29.277025),(83.975835,29.263745),(83.988237,29.269532),(84.000898,29.27253),(84.012525,29.270204),(84.021517,29.259766),(84.035211,29.247622),(84.05304,29.25036),(84.072057,29.256975),(84.08973,29.256613),(84.099135,29.247001),(84.105336,29.219923),(84.112364,29.209097),(84.126007,29.206255),(84.141975,29.20827),(84.155462,29.206642),(84.161663,29.192534),(84.155204,29.175946),(84.141716,29.160263),(84.13226,29.143804),(84.137996,29.124657),(84.160113,29.105486),(84.166211,29.098458),(84.169725,29.088923),(84.175461,29.057401),(84.198199,29.045308),(84.220575,29.0389),(84.231427,29.025981),(84.206725,28.940664),(84.225291,28.914115),(84.235147,28.90002),(84.285997,28.873743),(84.354003,28.861289),(84.366302,28.856741),(84.376586,28.848835),(84.393071,28.82434),(84.41431,28.808269),(84.422733,28.799173),(84.437822,28.753336),(84.450845,28.73388),(84.475753,28.727292),(84.498594,28.72786),(84.515647,28.7214),(84.531977,28.712693),(84.551821,28.706388),(84.607218,28.698198),(84.616209,28.69365),(84.625821,28.67608),(84.632642,28.668794),(84.65228,28.662386),(84.671141,28.658949),(84.682975,28.650759),(84.677756,28.604405),(84.69083,28.596137),(84.736615,28.594173),(84.755632,28.585414),(84.781781,28.558904),(84.799144,28.546631),(84.81196,28.54198),(84.823949,28.540533),(84.879294,28.543634),(84.891438,28.541773),(84.907871,28.537019),(84.918723,28.535675),(84.92849,28.538466),(84.936448,28.54477),(84.948954,28.568283),(84.954948,28.574303),(84.984766,28.588592),(84.994067,28.590814),(85.002491,28.594431),(85.009157,28.602545),(85.018769,28.619856),(85.02528,28.628409),(85.032101,28.634067),(85.054942,28.638692),(85.075044,28.631664),(85.11189,28.608823),(85.160982,28.595),(85.16801,28.583192),(85.156952,28.533143),(85.153748,28.524901),(85.14765,28.515832),(85.140363,28.510922),(85.123155,28.504592),(85.117212,28.497331),(85.115662,28.491492),(85.115559,28.472733),(85.109461,28.459298),(85.083674,28.445526),(85.077577,28.435681),(85.08569,28.381008),(85.08569,28.361526),(85.081867,28.331702),(85.080212,28.318789),(85.087395,28.304113),(85.109409,28.292254),(85.134162,28.291969),(85.154574,28.300858),(85.173591,28.30432),(85.205734,28.278275),(85.223924,28.271686),(85.243406,28.267914),(85.259581,28.266932),(85.27312,28.269826),(85.301025,28.281066),(85.312601,28.282771),(85.325727,28.277397),(85.334822,28.268689),(85.345364,28.261248),(85.362572,28.259672),(85.377817,28.265976),(85.401381,28.288791),(85.41523,28.296439),(85.43156,28.297059),(85.44665,28.29215),(85.475692,28.279102),(85.497551,28.282616),(85.52065,28.282926),(85.543698,28.280394),(85.565299,28.275381),(85.585504,28.263186),(85.59894,28.250835),(85.611807,28.251145),(85.639093,28.287887),(85.646947,28.2937),(85.668135,28.300548),(85.675059,28.306387),(85.676196,28.31512),(85.67599,28.324732),(85.679142,28.332897),(85.692939,28.335222),(85.698004,28.313441),(85.699761,28.269102),(85.715574,28.243368),(85.734487,28.221534),(85.756605,28.203241),(85.782753,28.1881),(85.826885,28.170478),(85.848279,28.158928),(85.863368,28.143348),(85.867037,28.132935),(85.87174,28.110921),(85.889103,28.090276),(85.891067,28.080897),(85.889827,28.071259),(85.890447,28.059193),(85.898302,28.036016),(85.908947,28.022425),(85.943984,27.994726),(85.956593,27.976226),(85.956283,27.956873),(85.951838,27.936797),(85.951218,27.91623),(85.96269,27.891374),(85.98026,27.885172),(86.026459,27.889927),(86.051419,27.88755),(86.0546,27.888013),(86.061703,27.889048),(86.075655,27.894784),(86.107178,27.922845),(86.112862,27.926798),(86.112242,27.937262),(86.107178,27.944239),(86.100563,27.950233),(86.095602,27.957855),(86.095085,27.968888),(86.098031,27.977518),(86.098134,27.985631),(86.089401,27.994726),(86.089401,27.994881),(86.089298,27.994933),(86.089298,27.994881),(86.079169,28.001134),(86.077619,28.006948),(86.079531,28.01333),(86.079531,28.02165),(86.083923,28.038548),(86.084699,28.043871),(86.081546,28.0517),(86.070177,28.066712),(86.068731,28.076763),(86.15596,28.156525),(86.176011,28.153011),(86.186036,28.133529),(86.187173,28.108699),(86.180765,28.088752),(86.172238,28.071828),(86.17446,28.062448),(86.18185,28.054878),(86.189137,28.043612),(86.19601,28.01457),(86.203038,28.002581),(86.217765,27.994985),(86.21792,27.994933),(86.218385,27.994726),(86.238539,27.989559),(86.259003,27.987647),(86.278072,27.983694),(86.29466,27.972376),(86.318224,27.945789),(86.327733,27.939691),(86.337551,27.938037),(86.345613,27.940931),(86.353674,27.944807),(86.362976,27.946099),(86.371968,27.941965),(86.405713,27.917005),(86.425298,27.909925),(86.438837,27.910856),(86.475476,27.927599),(86.496921,27.937882),(86.510461,27.952429),(86.516765,27.971369),(86.516764,27.971432),(86.516455,27.994726),(86.516197,28.012245),(86.522295,28.023717),(86.530925,28.03369),(86.538366,28.047178),(86.540743,28.062164),(86.54126,28.07821),(86.544154,28.093015),(86.553559,28.104151),(86.569527,28.106606),(86.583118,28.098028),(86.596864,28.086504),(86.612315,28.080277),(86.627146,28.083481),(86.64947,28.10118),(86.661976,28.106838),(86.680063,28.105727),(86.699648,28.098751),(86.717683,28.088261),(86.731739,28.076918),(86.735977,28.064851),(86.732204,28.034957),(86.739801,28.021495),(86.76998,28.01209),(86.840466,28.014777),(86.868165,27.994881),(86.868165,27.994726),(86.876898,27.970645),(86.893073,27.954186),(86.914725,27.945117),(86.940357,27.943102),(86.970226,27.947443),(86.982093,27.950674),(86.988726,27.952481),(87.005676,27.951499),(87.030532,27.938089),(87.062003,27.908427),(87.116522,27.844581),(87.155796,27.825796),(87.181996,27.824504),(87.232897,27.829724),(87.290516,27.816081),(87.313926,27.828587),(87.335837,27.846363),(87.363845,27.855252),(87.385343,27.849412),(87.380692,27.835821),(87.369116,27.819078),(87.369116,27.803937),(87.38679,27.804402),(87.47557,27.826701),(87.514534,27.835201),(87.531587,27.836803),(87.551327,27.831894),(87.555978,27.827088),(87.559802,27.819078),(87.565073,27.810862),(87.573342,27.805022),(87.589981,27.804609),(87.621091,27.819543),(87.636077,27.823652),(87.659951,27.819879),(87.678529,27.813351),(87.700466,27.805642),(87.726821,27.807761),(87.756379,27.82037),(87.779841,27.839439),(87.798651,27.863468),(87.814154,27.89096),(87.825884,27.906566),(87.836685,27.908323),(87.856115,27.898608),(87.93456,27.895301),(87.945205,27.892304),(87.966392,27.882744),(87.978433,27.880522),(87.991817,27.88264),(88.018121,27.892149),(88.029954,27.893337),(88.052795,27.886516),(88.095739,27.865329),(88.118218,27.860885),(88.143023,27.855717),(88.156407,27.851273),(88.164623,27.845356),(88.166587,27.833599),(88.154598,27.815151),(88.159662,27.77412),(88.149327,27.748772),(88.134858,27.723167),(88.11088,27.639503),(88.048765,27.545348),(88.02334,27.494912),(88.0221,27.484241),(88.02334,27.474655),(88.028094,27.455121),(88.030781,27.450729),(88.039773,27.442357),(88.041633,27.438068),(88.039308,27.42949),(88.034812,27.423805),(88.029593,27.418741),(88.02582,27.412075),(88.017242,27.389234),(88.014762,27.378072),(88.015795,27.364222),(88.020136,27.35337),(88.032538,27.333888),(88.035122,27.322571),(88.029024,27.298076),(88.004663,27.249163),(87.989337,27.218391),(87.985461,27.14837),(87.970837,27.119224),(87.969183,27.110801),(87.970733,27.10274),(87.975488,27.095143),(87.9913,27.081501),(88.009491,27.045637),(88.027371,27.035353),(88.042822,27.028946),(88.055948,27.018042),(88.076877,26.99179),(88.096824,26.959337),(88.11181,26.924301),(88.12044,26.90885),(88.142816,26.878412),(88.151394,26.862806),(88.155321,26.845598),(88.159042,26.802551),(88.167827,26.762915),(88.169067,26.744002),(88.167517,26.725037),(88.163383,26.705141),(88.14664,26.661216),(88.101681,26.581944),(88.087136,26.5391),(88.079822,26.517556),(88.07915,26.507375),(88.082251,26.494921),(88.082251,26.49487),(88.074189,26.453942),(88.044321,26.405676),(88.006648,26.369864),(87.975488,26.366557),(87.961225,26.378959),(87.928979,26.396426),(87.914768,26.40826),(87.908515,26.41782),(87.902004,26.435287),(87.897301,26.443452),(87.894769,26.442935),(87.870688,26.460712),(87.869809,26.464639),(87.852084,26.46066),(87.821595,26.437509),(87.804439,26.437354),(87.785938,26.445984),(87.768988,26.451461),(87.756173,26.446914),(87.749455,26.425727),(87.74222,26.410482),(87.727544,26.403764),(87.710904,26.405676),(87.697572,26.41627),(87.68119,26.424228),(87.659641,26.41751),(87.648745,26.409993),(87.623984,26.392912),(87.586984,26.378029),(87.552051,26.386711),(87.480324,26.423401),(87.449628,26.428621),(87.416452,26.426967),(87.384206,26.418647),(87.356404,26.403712),(87.344932,26.389346),(87.326225,26.353328),(87.314029,26.343768),(87.30049,26.34599),(87.258425,26.36294),(87.2453,26.370226),(87.236308,26.3833),(87.229435,26.398752),(87.2191,26.408105),(87.188455,26.399578),(87.135022,26.394204),(87.10629,26.404746),(87.083294,26.432083),(87.066706,26.465621),(87.056681,26.49487),(87.056474,26.49487),(87.056474,26.494921),(87.045002,26.544272),(87.044433,26.561171),(87.041333,26.580187),(87.029912,26.579774),(87.015443,26.568974),(86.985212,26.540655),(86.972448,26.531973),(86.935153,26.520289),(86.907129,26.51151),(86.875968,26.494921),(86.865684,26.472442),(86.846357,26.45265),(86.821863,26.438129),(86.796438,26.431514),(86.78724,26.433065),(86.751893,26.445519),(86.738199,26.44371),(86.730706,26.433788),(86.724091,26.421954),(86.713601,26.414565),(86.695204,26.418234),(86.625337,26.456319),(86.5579,26.484018),(86.537953,26.49487),(86.537746,26.49487),(86.52431,26.509081),(86.510616,26.520139),(86.494906,26.527839),(86.475476,26.531973),(86.444832,26.543084),(86.383647,26.572849),(86.353674,26.582616),(86.344838,26.582616),(86.323185,26.580084),(86.316002,26.580963),(86.309026,26.587991),(86.308613,26.60215),(86.301378,26.609023),(86.284428,26.61202),(86.263086,26.609075),(86.225155,26.597396),(86.202883,26.58458),(86.195855,26.582668),(86.185364,26.584425),(86.179266,26.588611),(86.17446,26.593313),(86.167794,26.596621),(86.152653,26.600651),(86.146355,26.601356),(86.144798,26.60153),(86.122061,26.60029),(86.115859,26.602563),(86.110692,26.606749),(86.041704,26.645455),(86.01137,26.654447),(85.975713,26.64437),(85.952148,26.642044),(85.934682,26.632897),(85.866366,26.579929),(85.844765,26.568509),(85.828538,26.566131),(85.821614,26.571713),(85.819753,26.579516),(85.819547,26.588301),(85.817635,26.596724),(85.809728,26.603028),(85.800065,26.600703),(85.789988,26.597034),(85.780996,26.599204),(85.727046,26.6376),(85.712887,26.653206),(85.702241,26.68845),(85.709631,26.76245),(85.701828,26.796608),(85.687772,26.811853),(85.609379,26.851024),(85.598578,26.854383),(85.519462,26.826426),(85.475692,26.805238),(85.439622,26.78772),(85.421535,26.782656),(85.40319,26.787617),(85.385878,26.788444),(85.369238,26.775008),(85.355352,26.759912),(85.353219,26.757593),(85.337044,26.746792),(85.302369,26.737336),(85.28728,26.736974),(85.194934,26.758885),(85.165685,26.786273),(85.16553,26.820793),(85.162016,26.851024),(85.122845,26.8657),(85.100211,26.863529),(85.043987,26.843737),(85.018562,26.845804),(85.016857,26.85924),(85.018045,26.874433),(85.000837,26.882236),(84.988176,26.883787),(84.975774,26.886887),(84.952726,26.897636),(84.944251,26.915723),(84.938567,26.9366),(84.924046,26.955668),(84.901567,26.967192),(84.851751,26.982127),(84.828289,26.994839),(84.828134,26.994943),(84.817489,27.0106),(84.801934,27.013753),(84.785501,27.008482),(84.771962,26.99918),(84.760697,26.999025),(84.640239,27.028377),(84.627268,27.03649),(84.62148,27.057988),(84.630369,27.080829),(84.644528,27.103721),(84.654398,27.125374),(84.659824,27.165113),(84.657654,27.203405),(84.648197,27.240716),(84.631919,27.277044),(84.606546,27.310479),(84.577039,27.329031),(84.289408,27.376108),(84.267703,27.388562),(84.248893,27.412436),(84.239023,27.431143),(84.225536,27.440393),(84.195253,27.436053),(84.185848,27.438636),(84.175099,27.462976),(84.165746,27.472174),(84.141716,27.480753),(84.130968,27.486411),(84.121563,27.494964),(84.11717,27.513283),(84.099548,27.516874),(84.079601,27.509536),(84.028907,27.453726),(84.007668,27.440807),(83.975835,27.439722),(83.935941,27.446078),(83.923022,27.449954),(83.922043,27.449696),(83.899871,27.443856),(83.853517,27.440962),(83.834087,27.434037),(83.842717,27.418069),(83.87119,27.38944),(83.873671,27.380087),(83.87765,27.369907),(83.877391,27.361742),(83.867521,27.358383),(83.861733,27.354507),(83.854602,27.34505),(83.85334,27.346194),(83.847988,27.351045),(83.801995,27.365928),(83.663399,27.43228),(83.590432,27.45662),(83.480981,27.469746),(83.387034,27.470469),(83.360989,27.462201),(83.355925,27.452486),(83.353857,27.44029),(83.355821,27.428663),(83.370962,27.410214),(83.369567,27.398174),(83.362746,27.385616),(83.341145,27.356936),(83.32435,27.341691),(83.304868,27.331925),(83.282441,27.330943),(83.259496,27.338126),(83.249419,27.348099),(83.243115,27.362155),(83.231591,27.381224),(83.21924,27.393781),(83.169631,27.431195),(83.132786,27.444217),(83.010468,27.443391),(82.947164,27.457317),(82.901275,27.480365),(82.876264,27.487548),(82.752137,27.494964),(82.72971,27.518166),(82.718857,27.556123),(82.709246,27.630924),(82.69705,27.669397),(82.679687,27.694409),(82.673891,27.696458),(82.652143,27.70415),(82.526156,27.675211),(82.440683,27.666426),(82.401926,27.677175),(82.347924,27.726009),(82.270409,27.760477),(82.151037,27.848275),(82.107422,27.863572),(82.090369,27.872331),(82.071662,27.89003),(82.051611,27.905171),(82.050212,27.905586),(82.02722,27.912406),(81.975905,27.916953),(81.946398,27.905352),(81.906091,27.863107),(81.883198,27.849051),(81.855654,27.850937),(81.827852,27.865639),(81.800154,27.884087),(81.750131,27.909667),(81.710392,27.94752),(81.688946,27.963204),(81.665433,27.9708),(81.615049,27.981239),(81.59536,27.994726),(81.59536,27.994778),(81.595102,27.994881),(81.582183,28.013278),(81.561925,28.025991),(81.473455,28.066402),(81.458211,28.07728),(81.454025,28.086607),(81.45294,28.097046),(81.448237,28.111412),(81.435267,28.129783),(81.417335,28.147017),(81.396044,28.160634),(81.373203,28.167843),(81.357907,28.165982),(81.351447,28.15681),(81.347262,28.144433),(81.338115,28.132728),(81.323697,28.126243),(81.307057,28.123814),(81.296205,28.12831),(81.299409,28.142521),(81.283079,28.146087),(81.276775,28.153787),(81.280754,28.161977),(81.295947,28.167455),(81.282356,28.191588),(81.224323,28.250783),(81.210732,28.278637),(81.190372,28.338116),(81.169701,28.361319),(81.146344,28.372249),(81.047066,28.389088),(81.000409,28.397002),(80.993175,28.407828),(80.991314,28.420256),(80.9878,28.430979),(80.976018,28.436922),(80.960515,28.432271),(80.941188,28.432684),(80.921344,28.436198),(80.905273,28.440849),(80.88667,28.452786),(80.881864,28.466842),(80.880365,28.48188),(80.872252,28.496866),(80.857989,28.502447),(80.81701,28.502551),(80.798406,28.505703),(80.781663,28.514746),(80.743526,28.549964),(80.727093,28.559679),(80.695777,28.567508),(80.67862,28.574717),(80.668182,28.586344),(80.648338,28.619753),(80.635625,28.627866),(80.598522,28.633783),(80.581365,28.638951),(80.563589,28.647038),(80.556871,28.655022),(80.557387,28.66435),(80.559041,28.673031),(80.555837,28.678948),(80.534444,28.679555),(80.517596,28.680033),(80.497546,28.670137),(80.487624,28.656469),(80.484937,28.639287),(80.493412,28.575647),(80.488451,28.562444),(80.468814,28.571849),(80.426543,28.616807),(80.408352,28.626342),(80.388302,28.627246),(80.369078,28.622647),(80.349751,28.620218),(80.329701,28.627479),(80.318332,28.640113),(80.291047,28.689671),(80.283399,28.695356),(80.265209,28.699283),(80.257044,28.702745),(80.249396,28.710213),(80.238957,28.726258),(80.233273,28.732718),(80.216426,28.741942),(80.181183,28.74729),(80.162063,28.753285),(80.147593,28.76331),(80.11421,28.802584),(80.081861,28.819353),(80.073054,28.820925),(80.054782,28.824185),(80.036386,28.837026),(80.030288,28.87767),(80.031115,28.897798),(80.033905,28.915936),(80.04052,28.932809),(80.052922,28.949113),(80.068425,28.959939),(80.085168,28.967484),(80.099431,28.977276),(80.107906,28.994588),(80.107906,28.994769),(80.104702,29.027609),(80.113073,29.072206),(80.132607,29.110214),(80.16351,29.123366),(80.180563,29.121324),(80.20144,29.121169),(80.220664,29.126079),(80.233066,29.139308),(80.230482,29.154733),(80.213739,29.196565),(80.218803,29.211086),(80.236063,29.213076),(80.248672,29.204394),(80.258181,29.202456),(80.278748,29.268085),(80.282779,29.291314),(80.280092,29.31015),(80.272754,29.315705),(80.263555,29.315318),(80.254977,29.316635),(80.249499,29.326997),(80.242161,29.367408),(80.220974,29.400119),(80.213739,29.416888),(80.217666,29.434639),(80.228312,29.441718),(80.257457,29.450064),(80.263452,29.459237),(80.266242,29.47213),(80.273064,29.478667),(80.282055,29.4843),(80.29115,29.494609),(80.29146,29.494738),(80.311511,29.508122),(80.320089,29.515822),(80.327221,29.52484),(80.327634,29.52993),(80.32505,29.535511),(80.323603,29.54166),(80.327221,29.548585),(80.332698,29.552461),(80.345101,29.558352),(80.350785,29.562073),(80.373316,29.58419),(80.385201,29.604835),(80.386752,29.626823),(80.377553,29.652946),(80.363911,29.679714),(80.354402,29.70488),(80.354299,29.730279),(80.368975,29.757926),(80.395227,29.776582),(80.454861,29.790586),(80.476152,29.80614),(80.527312,29.862416),(80.549533,29.89368),(80.562865,29.929699),(80.57134,29.946855),(80.586223,29.954142),(80.622293,29.958173),(80.641413,29.963444),(80.654126,29.970575),(80.680171,29.992072),(80.715827,30.013311),(80.725749,30.022768),(80.755721,30.064574),(80.769674,30.077287),(80.829825,30.117129),(80.849669,30.143381),(80.83644,30.170046),(80.850393,30.181931),(80.867911,30.200173),(80.883879,30.210405),(80.893646,30.198106),(80.903309,30.180433),(80.920828,30.17666),(80.988162,30.196556),(80.996017,30.196969),(81.006507,30.188908),(81.019426,30.172268),(81.030898,30.152734),(81.043663,30.12023),(81.066607,30.087157),(81.074203,30.071912),(81.084454,30.027876),(81.084849,30.026179),(81.097768,30.016929),(81.124588,30.022768),(81.148101,30.023233),(81.194506,30.004475),(81.21683,30.008195),(81.227269,30.022458),(81.225719,30.050467),(81.239516,30.058787),(81.257086,30.063541),(81.266801,30.069949),(81.267903,30.072053),(81.272021,30.079922),(81.275948,30.095373),(81.274501,30.102195),(81.266336,30.117749),(81.266956,30.125087),(81.277705,30.131857),(81.29326,30.132839),(81.309538,30.132064),(81.322405,30.133769),(81.33827,30.143536),(81.350052,30.15723),(81.357287,30.17356),(81.361834,30.204772),(81.374392,30.223118),(81.377854,30.232626),(81.376924,30.241204),(81.370516,30.257276),(81.369017,30.266061),(81.368036,30.301046),(81.37155,30.31691),(81.382143,30.331276),(81.387259,30.345126),(81.384882,30.360939),(81.387518,30.373909),(81.407413,30.37918),(81.42612,30.372514),(81.46281,30.341353),(81.483067,30.331896),(81.509371,30.328382),(81.520688,30.331948),(81.531953,30.370344),(81.536811,30.37856),(81.544252,30.382643),(81.563269,30.385588),(81.576705,30.390963),(81.581976,30.399438),(81.585077,30.408119),(81.591588,30.414269),(81.614015,30.416904),(81.635719,30.410961),(81.757366,30.362902),(81.779018,30.358045)] +Nauru [(166.938813,-0.490411),(166.955577,-0.497979),(166.958263,-0.517673),(166.951345,-0.539158),(166.938813,-0.551853),(166.916352,-0.54754),(166.906993,-0.524835),(166.913422,-0.500177),(166.938813,-0.490411)] +Poland [(19.002126,54.344916),(19.377208,54.377631),(19.459972,54.401028),(19.499278,54.406399),(19.531016,54.414944),(19.59254,54.452379),(19.609548,54.456732),(19.63077,54.446652),(19.680276,54.436678),(19.758256,54.434256),(19.964703,54.427842),(20.35207,54.415749),(20.640838,54.406784),(20.929606,54.397818),(21.287413,54.386759),(21.720772,54.373271),(21.998946,54.364667),(22.27712,54.356063),(22.510388,54.348777),(22.698387,54.342937),(22.76722,54.35627),(22.78603,54.36521),(22.811344,54.392633),(22.812695,54.394097),(22.837603,54.400918),(22.858894,54.399213),(22.918838,54.381126),(22.92969,54.380299),(22.952428,54.38309),(22.96266,54.381695),(22.973305,54.37539),(22.97899,54.367587),(22.983434,54.359784),(22.989738,54.353376),(23.001004,54.34888),(23.035317,54.346503),(23.041622,54.340974),(23.038004,54.331465),(23.032216,54.320148),(23.032113,54.309554),(23.050096,54.294827),(23.073041,54.294878),(23.096915,54.300821),(23.117069,54.303457),(23.135259,54.298289),(23.197271,54.267851),(23.235408,54.254157),(23.316023,54.236277),(23.33599,54.226294),(23.354471,54.217054),(23.364082,54.208269),(23.382273,54.187856),(23.391471,54.180105),(23.401806,54.175971),(23.423304,54.172508),(23.432502,54.169305),(23.449039,54.154938),(23.462991,54.13494),(23.473947,54.112564),(23.481595,54.091221),(23.496167,54.044558),(23.496477,54.021975),(23.481595,54.006627),(23.469502,54.000684),(23.464128,53.996757),(23.45927,53.992313),(23.45865,53.981616),(23.462888,53.972521),(23.470949,53.965286),(23.481595,53.960118),(23.487486,53.955622),(23.49038,53.95061),(23.48976,53.945132),(23.485625,53.939293),(23.486866,53.909992),(23.497201,53.885549),(23.51043,53.862502),(23.520662,53.837335),(23.52986,53.78416),(23.540402,53.763593),(23.56469,53.742457),(23.56469,53.742405),(23.564794,53.742405),(23.567068,53.681014),(23.590942,53.611251),(23.675795,53.455705),(23.722924,53.397104),(23.742974,53.365478),(23.782972,53.270936),(23.800645,53.242462),(23.818628,53.227967),(23.828447,53.213833),(23.836198,53.199286),(23.848084,53.183809),(23.865964,53.172001),(23.883121,53.163991),(23.893663,53.151951),(23.891389,53.127714),(23.882397,53.113297),(23.870305,53.101256),(23.860693,53.087484),(23.859349,53.067976),(23.867928,53.051027),(23.89821,53.02754),(23.909682,53.012734),(23.911336,53.00506),(23.908959,52.993046),(23.909062,52.986664),(23.917537,52.95881),(23.91795,52.9508),(23.914953,52.944625),(23.903894,52.931137),(23.901104,52.923257),(23.902034,52.912534),(23.905445,52.906539),(23.909476,52.901501),(23.912059,52.893853),(23.920426,52.772625),(23.922498,52.742596),(23.908752,52.699859),(23.868961,52.670042),(23.736153,52.614903),(23.569031,52.585887),(23.480458,52.554416),(23.392298,52.509638),(23.270548,52.395123),(23.230447,52.365074),(23.21205,52.347504),(23.165645,52.289393),(23.168746,52.288928),(23.176099,52.285136),(23.183525,52.281306),(23.19448,52.271462),(23.197684,52.25831),(23.189726,52.240533),(23.212257,52.231671),(23.284191,52.219734),(23.29959,52.223428),(23.312716,52.215289),(23.374831,52.200949),(23.395812,52.19958),(23.395812,52.193379),(23.388474,52.184154),(23.391368,52.182501),(23.405734,52.185911),(23.418239,52.182501),(23.427231,52.17679),(23.43674,52.175679),(23.450485,52.185911),(23.454413,52.181467),(23.464335,52.176351),(23.470949,52.171623),(23.487693,52.181622),(23.491517,52.174103),(23.488726,52.16214),(23.484695,52.158626),(23.512497,52.124442),(23.531928,52.120644),(23.579366,52.121548),(23.598177,52.11452),(23.604791,52.106769),(23.609856,52.09863),(23.61585,52.0923),(23.625255,52.089716),(23.637451,52.084471),(23.641482,52.072921),(23.642515,52.061397),(23.650783,52.048943),(23.664943,52.011168),(23.676415,51.994088),(23.660705,51.98688),(23.647579,51.97409),(23.628666,51.946339),(23.622155,51.927943),(23.621431,51.895567),(23.61492,51.88425),(23.618847,51.883268),(23.620604,51.883217),(23.621328,51.882028),(23.621845,51.877429),(23.609856,51.873269),(23.605721,51.851591),(23.594559,51.843297),(23.610269,51.825184),(23.628666,51.809759),(23.617504,51.786504),(23.59828,51.77467),(23.577299,51.766686),(23.56035,51.754542),(23.556319,51.747359),(23.552288,51.736636),(23.546604,51.713589),(23.550221,51.708421),(23.556939,51.701393),(23.559213,51.695166),(23.550014,51.692427),(23.546087,51.689637),(23.549291,51.683203),(23.555389,51.676304),(23.56035,51.671963),(23.547534,51.662067),(23.541436,51.659742),(23.532444,51.658967),(23.532444,51.6515),(23.543813,51.6438),(23.545157,51.63411),(23.539886,51.607109),(23.5434,51.592743),(23.552081,51.578817),(23.573992,51.555304),(23.567171,51.55489),(23.56035,51.555304),(23.572752,51.539672),(23.588668,51.535899),(23.602311,51.530783),(23.606238,51.517399),(23.608099,51.511224),(23.614817,51.49722),(23.630526,51.490424),(23.648613,51.486212),(23.662772,51.480192),(23.647166,51.460193),(23.648716,51.453966),(23.689437,51.416423),(23.697602,51.404435),(23.686027,51.40105),(23.678895,51.394073),(23.677655,51.38379),(23.683856,51.370276),(23.665563,51.365806),(23.647786,51.353998),(23.63404,51.339296),(23.628666,51.325964),(23.635177,51.304699),(23.650887,51.299609),(23.670214,51.299376),(23.687267,51.2924),(23.742664,51.216255),(23.765195,51.199021),(23.816045,51.178789),(23.863587,51.148274),(23.874749,51.13613),(23.858213,51.13073),(23.854492,51.121532),(23.869271,51.10174),(23.895523,51.076108),(23.904205,51.062724),(23.915987,51.020866),(23.915987,51.014665),(23.911749,51.00681),(23.919087,51.002728),(23.9318,50.999472),(23.943995,50.994201),(23.955054,50.983556),(23.957845,50.975339),(23.958878,50.966348),(23.964356,50.953222),(23.979342,50.937512),(24.046935,50.898031),(24.130237,50.868937),(24.143156,50.856432),(24.130857,50.839069),(24.100472,50.834986),(24.067295,50.834831),(24.046935,50.829095),(24.020477,50.838552),(23.992881,50.836226),(23.969937,50.825168),(23.957638,50.808011),(23.959498,50.788891),(23.974278,50.776178),(23.997946,50.769202),(24.025851,50.767032),(24.025851,50.76083),(24.012828,50.743312),(24.026884,50.728016),(24.05417,50.717164),(24.081041,50.712978),(24.074943,50.690189),(24.082798,50.669156),(24.108165,50.630287),(24.10843,50.629882),(24.085279,50.604406),(24.095407,50.556864),(24.102539,50.543634),(24.107706,50.540844),(24.106466,50.538622),(24.09396,50.52715),(24.075047,50.514282),(24.010658,50.492785),(24.007867,50.480744),(24.009418,50.461469),(24.007764,50.448498),(24.003217,50.437698),(23.981306,50.40478),(23.928699,50.390827),(23.747522,50.38938),(23.713002,50.382404),(23.695742,50.376771),(23.682203,50.368245),(23.671247,50.353568),(23.658677,50.327005),(23.658018,50.325611),(23.644169,50.312692),(23.565311,50.257812),(23.536372,50.242826),(23.536268,50.242826),(23.536165,50.242774),(23.536062,50.242774),(23.481595,50.215954),(23.436429,50.193475),(23.207916,50.03395),(23.177841,50.003977),(23.141254,49.985477),(23.101463,49.957107),(22.993046,49.854426),(22.951188,49.826624),(22.945917,49.818356),(22.937752,49.798202),(22.933101,49.79107),(22.924419,49.785438),(22.906746,49.7812),(22.897961,49.777428),(22.888246,49.769728),(22.826648,49.697381),(22.809801,49.686322),(22.798846,49.683273),(22.777348,49.680483),(22.766186,49.67423),(22.758951,49.6656),(22.741692,49.633664),(22.665831,49.567363),(22.640922,49.528761),(22.660146,49.493001),(22.666864,49.478376),(22.673065,49.435847),(22.676786,49.424478),(22.679576,49.418948),(22.692599,49.405513),(22.7142,49.39063),(22.719781,49.382982),(22.724225,49.367065),(22.737764,49.275391),(22.73394,49.26087),(22.717094,49.230433),(22.702934,49.195034),(22.687121,49.173382),(22.681747,49.161238),(22.687845,49.155915),(22.692496,49.157931),(22.705725,49.168835),(22.721434,49.1616),(22.720194,49.161186),(22.71637,49.160566),(22.748823,49.14558),(22.778072,49.12031),(22.796469,49.11137),(22.841324,49.094886),(22.853416,49.084757),(22.853519,49.076282),(22.844424,49.056697),(22.843184,49.043106),(22.847525,49.033701),(22.863751,49.015356),(22.865014,49.013237),(22.866955,49.009981),(22.855276,48.994013),(22.835019,48.999749),(22.812902,49.012875),(22.795022,49.019025),(22.783963,49.02135),(22.765463,49.038403),(22.755334,49.044708),(22.744482,49.045535),(22.721951,49.041039),(22.685778,49.042796),(22.664384,49.041452),(22.642886,49.043157),(22.618288,49.054216),(22.580771,49.08145),(22.560721,49.085532),(22.539637,49.0722),(22.505013,49.083413),(22.426775,49.085635),(22.390085,49.093025),(22.339442,49.12646),(22.318151,49.131989),(22.262858,49.130594),(22.215936,49.139999),(22.208494,49.144185),(22.20622,49.150489),(22.209011,49.156949),(22.208908,49.163977),(22.197952,49.171987),(22.189684,49.17302),(22.165706,49.171108),(22.155681,49.171522),(22.144105,49.174881),(22.111549,49.188627),(22.040752,49.197463),(22.012434,49.211054),(22.005819,49.242887),(21.99321,49.278182),(21.964478,49.308568),(21.928408,49.330788),(21.874561,49.348358),(21.837871,49.370424),(21.819577,49.377246),(21.799423,49.374817),(21.782164,49.364482),(21.768004,49.353474),(21.757566,49.348927),(21.742166,49.357092),(21.708576,49.390888),(21.69173,49.402154),(21.681808,49.404066),(21.666305,49.401792),(21.65876,49.402464),(21.648632,49.407063),(21.630028,49.418897),(21.6199,49.423341),(21.601193,49.426493),(21.529569,49.421222),(21.514422,49.417213),(21.496186,49.412386),(21.48151,49.415228),(21.444406,49.409905),(21.42787,49.409802),(21.330408,49.427785),(21.274391,49.447267),(21.260542,49.449438),(21.242455,49.441273),(21.211242,49.411094),(21.194086,49.400603),(21.172588,49.398278),(21.157292,49.405151),(21.143029,49.415538),(21.124943,49.423651),(21.10944,49.424581),(21.068822,49.419207),(21.053526,49.414453),(21.033269,49.399673),(21.045464,49.390578),(21.068512,49.381431),(21.080708,49.366342),(21.072543,49.357195),(21.054043,49.354766),(21.032959,49.354611),(21.017249,49.352441),(20.996682,49.339367),(20.964022,49.308103),(20.942422,49.295855),(20.91896,49.290326),(20.900564,49.2926),(20.884337,49.300299),(20.868007,49.31141),(20.849301,49.320505),(20.833591,49.322055),(20.816538,49.321383),(20.79411,49.323657),(20.778297,49.331202),(20.689517,49.4005),(20.673911,49.402309),(20.636084,49.39833),(20.614483,49.400448),(20.605284,49.400035),(20.595053,49.396263),(20.579136,49.383395),(20.567664,49.376367),(20.543996,49.370838),(20.522085,49.374352),(20.437543,49.402515),(20.421936,49.40019),(20.422143,49.382982),(20.370467,49.38169),(20.329539,49.391767),(20.317653,49.391612),(20.307422,49.386599),(20.303701,49.380398),(20.301634,49.371199),(20.296466,49.356937),(20.289335,49.343087),(20.284374,49.338643),(20.207169,49.334199),(20.191976,49.328618),(20.170169,49.311617),(20.16035,49.305622),(20.138336,49.307327),(20.135856,49.308878),(20.130171,49.303917),(20.111051,49.275856),(20.105367,49.263971),(20.098442,49.25286),(20.08604,49.243042),(20.080355,49.208109),(20.07002,49.183097),(20.050486,49.173227),(20.01731,49.183872),(19.965737,49.215653),(19.937935,49.22511),(19.905896,49.222785),(19.887809,49.214),(19.868292,49.200788),(19.854219,49.191262),(19.831998,49.185888),(19.785903,49.188162),(19.760685,49.194208),(19.747766,49.205887),(19.747787,49.205956),(19.751797,49.219064),(19.789624,49.259475),(19.794068,49.261852),(19.800476,49.263041),(19.806263,49.265263),(19.808641,49.270895),(19.80678,49.275236),(19.806415,49.275488),(19.798822,49.280714),(19.798563,49.281081),(19.796962,49.28335),(19.79686,49.283399),(19.783216,49.290016),(19.779702,49.293375),(19.779713,49.293453),(19.780322,49.297612),(19.78797,49.305105),(19.78952,49.309188),(19.783629,49.357557),(19.778565,49.374197),(19.769263,49.39311),(19.769056,49.393213),(19.760065,49.397658),(19.726578,49.388873),(19.706321,49.387529),(19.684514,49.389079),(19.627044,49.401868),(19.627097,49.402165),(19.62922,49.413936),(19.6286,49.429594),(19.63015,49.43502),(19.634801,49.441324),(19.594804,49.441583),(19.573926,49.445252),(19.55677,49.453882),(19.551602,49.461013),(19.535479,49.492846),(19.517289,49.543282),(19.505197,49.563384),(19.481735,49.573513),(19.474191,49.578732),(19.457344,49.598059),(19.44889,49.600313),(19.443392,49.60178),(19.437604,49.600126),(19.43378,49.595165),(19.347067,49.532998),(19.339522,49.528761),(19.325363,49.52504),(19.315027,49.523955),(19.284228,49.524162),(19.265005,49.521475),(19.248675,49.516255),(19.234102,49.507212),(19.220459,49.493001),(19.20444,49.460548),(19.206197,49.459411),(19.209401,49.456414),(19.211364,49.45166),(19.208987,49.444993),(19.204543,49.442823),(19.192244,49.442616),(19.18935,49.442203),(19.182322,49.422721),(19.179738,49.410267),(19.172504,49.402257),(19.141705,49.394195),(19.11659,49.39094),(19.106255,49.391405),(19.096643,49.394609),(19.076799,49.404066),(19.067601,49.406133),(19.045793,49.402774),(19.007139,49.388098),(18.981818,49.386806),(18.962284,49.389183),(18.956703,49.400448),(18.960837,49.433315),(18.958253,49.448146),(18.953396,49.461685),(18.952362,49.475948),(18.961147,49.492794),(18.932208,49.504318),(18.833196,49.510261),(18.837434,49.526952),(18.83454,49.547623),(18.820174,49.590256),(18.81573,49.599299),(18.803947,49.616766),(18.799297,49.626378),(18.792579,49.660794),(18.788444,49.668597),(18.773458,49.675832),(18.757852,49.674075),(18.742246,49.669683),(18.727673,49.668907),(18.715064,49.673817),(18.695427,49.688854),(18.682404,49.695779),(18.668142,49.69857),(18.637343,49.700327),(18.62401,49.706373),(18.617706,49.713866),(18.60768,49.74296),(18.584116,49.774379),(18.566753,49.804765),(18.566339,49.831585),(18.593831,49.852204),(18.567269,49.861454),(18.562929,49.873701),(18.565409,49.889049),(18.559208,49.907187),(18.544428,49.912975),(18.505051,49.896697),(18.481797,49.896645),(18.407486,49.923155),(18.368832,49.932044),(18.328524,49.92884),(18.330695,49.92884),(18.332452,49.928271),(18.333795,49.927135),(18.334932,49.925429),(18.306923,49.909668),(18.292454,49.907808),(18.27261,49.918298),(18.267029,49.925067),(18.260105,49.94057),(18.255867,49.946048),(18.243982,49.951732),(18.22052,49.954885),(18.208738,49.958295),(18.177216,49.975814),(18.162023,49.98124),(18.098151,49.989043),(18.085128,49.998758),(18.091639,50.017207),(18.033245,50.041805),(18.019074,50.044087),(18.002446,50.046765),(18.00689,50.024131),(18.012161,50.022684),(18.020119,50.024286),(18.028698,50.024131),(18.035726,50.017207),(18.037483,50.007233),(18.032418,50.00284),(17.999862,49.996588),(17.959761,49.995812),(17.941571,49.992764),(17.912116,49.975814),(17.875735,49.968682),(17.839355,49.973643),(17.774553,50.015191),(17.763287,50.025216),(17.755433,50.037154),(17.749852,50.049504),(17.74303,50.060563),(17.731971,50.06909),(17.722566,50.075136),(17.719466,50.080872),(17.72267,50.08635),(17.731971,50.091466),(17.732385,50.092189),(17.732592,50.092964),(17.732385,50.093894),(17.731971,50.094876),(17.717915,50.096788),(17.690734,50.10547),(17.677608,50.107744),(17.666963,50.106142),(17.658488,50.102989),(17.648359,50.101801),(17.632753,50.106348),(17.584177,50.145881),(17.58211,50.153167),(17.589448,50.163244),(17.600403,50.16681),(17.648669,50.167998),(17.675334,50.174716),(17.717915,50.191098),(17.738173,50.202363),(17.747578,50.217504),(17.731971,50.236005),(17.721429,50.242722),(17.719156,50.252799),(17.721119,50.261946),(17.731971,50.280653),(17.734245,50.283805),(17.734969,50.286802),(17.734452,50.28949),(17.731971,50.291763),(17.713575,50.30768),(17.707994,50.311039),(17.691044,50.315069),(17.684533,50.312382),(17.681949,50.305561),(17.676368,50.297396),(17.646705,50.271093),(17.629549,50.262101),(17.606398,50.258225),(17.516998,50.268354),(17.469352,50.265563),(17.44,50.242722),(17.434832,50.2405),(17.429561,50.239777),(17.424187,50.240552),(17.419122,50.242722),(17.409821,50.261326),(17.388013,50.272643),(17.365896,50.271558),(17.355044,50.253264),(17.341711,50.259724),(17.334786,50.270008),(17.332823,50.282668),(17.334063,50.295691),(17.329515,50.298223),(17.325381,50.301117),(17.321867,50.304476),(17.319077,50.308248),(17.337474,50.313002),(17.316803,50.318635),(17.269881,50.317757),(17.24611,50.322149),(17.210866,50.336412),(17.201048,50.343233),(17.195053,50.351915),(17.188852,50.364834),(17.185442,50.375686),(17.187612,50.378476),(17.175933,50.38075),(17.145444,50.376719),(17.131698,50.376823),(17.118882,50.381164),(17.095628,50.393669),(17.084672,50.397958),(16.98194,50.416355),(16.958789,50.414598),(16.944005,50.420267),(16.915794,50.431083),(16.892953,50.432943),(16.865771,50.422401),(16.865874,50.408449),(16.897914,50.378218),(16.909283,50.359925),(16.916724,50.338634),(16.926543,50.319255),(16.945249,50.306905),(16.969641,50.297758),(16.987831,50.284942),(16.998373,50.26701),(16.99982,50.242722),(17.012842,50.231664),(17.014806,50.218486),(17.008398,50.20996),(16.996306,50.212905),(16.98566,50.223137),(16.98101,50.229183),(16.975015,50.231819),(16.960339,50.231974),(16.957548,50.229235),(16.957755,50.217143),(16.955275,50.213474),(16.946283,50.21306),(16.927783,50.217298),(16.918998,50.217091),(16.907009,50.211768),(16.890369,50.196937),(16.880964,50.191563),(16.869762,50.189729),(16.857606,50.187739),(16.837556,50.188617),(16.817712,50.186757),(16.795284,50.174406),(16.777301,50.15694),(16.766449,50.1434),(16.754253,50.13229),(16.732033,50.121955),(16.701233,50.094928),(16.660616,50.093016),(16.618654,50.10702),(16.584238,50.127432),(16.566668,50.142212),(16.557263,50.154718),(16.545894,50.188824),(16.535042,50.207738),(16.50476,50.227995),(16.492047,50.242722),(16.477578,50.255332),(16.452153,50.284994),(16.437787,50.297965),(16.415566,50.307938),(16.371021,50.318377),(16.353244,50.333518),(16.35066,50.342251),(16.351487,50.351346),(16.350764,50.360648),(16.343736,50.36995),(16.334227,50.371862),(16.288649,50.371862),(16.27697,50.369433),(16.269839,50.365454),(16.2625,50.36442),(16.249685,50.371035),(16.24493,50.376823),(16.232011,50.402609),(16.199559,50.406278),(16.190153,50.424158),(16.198008,50.440023),(16.217955,50.437129),(16.216095,50.440178),(16.211237,50.451289),(16.217335,50.45165),(16.226534,50.455371),(16.232011,50.456766),(16.247411,50.464983),(16.264981,50.471649),(16.279967,50.479866),(16.287719,50.492785),(16.303428,50.49604),(16.335468,50.490356),(16.352624,50.492785),(16.362029,50.501208),(16.382596,50.514179),(16.391175,50.521827),(16.388384,50.521258),(16.388281,50.52715),(16.389624,50.534849),(16.391175,50.539914),(16.395516,50.545701),(16.398306,50.548492),(16.398306,50.551799),(16.394482,50.559344),(16.425901,50.567561),(16.416806,50.586629),(16.331644,50.644042),(16.300948,50.6551),(16.268288,50.660423),(16.232011,50.660888),(16.219092,50.659131),(16.211754,50.649106),(16.204519,50.636342),(16.192324,50.626575),(16.175477,50.624095),(16.160284,50.628642),(16.128658,50.644042),(16.086491,50.64678),(16.024479,50.608592),(15.982001,50.603631),(15.979934,50.606628),(15.97921,50.60978),(15.980037,50.612984),(15.982001,50.61624),(15.996367,50.623681),(15.998641,50.635567),(15.993163,50.649364),(15.984481,50.662904),(15.971459,50.678613),(15.958023,50.686881),(15.941486,50.68838),(15.882265,50.674221),(15.854567,50.673704),(15.848159,50.675151),(15.797619,50.729773),(15.794415,50.735871),(15.792245,50.742692),(15.76744,50.744191),(15.684345,50.731426),(15.673803,50.733545),(15.666464,50.738093),(15.653235,50.75096),(15.641866,50.755766),(15.441775,50.800208),(15.400124,50.797986),(15.390822,50.790648),(15.381314,50.780002),(15.370255,50.772613),(15.368579,50.772949),(15.356096,50.775455),(15.349688,50.783723),(15.350515,50.793232),(15.353305,50.803257),(15.352995,50.812972),(15.341006,50.83173),(15.325813,50.839689),(15.307727,50.844753),(15.28778,50.854881),(15.277134,50.865734),(15.265352,50.88227),(15.25605,50.899892),(15.253053,50.914103),(15.260081,50.932499),(15.268763,50.94299),(15.269796,50.952653),(15.253776,50.969035),(15.227111,50.977406),(15.170164,50.977975),(15.160759,50.992651),(15.156108,51.007844),(15.144429,51.011564),(15.13151,51.005828),(15.122518,50.992754),(15.122518,50.992651),(15.119211,50.982471),(15.107946,50.981024),(15.09389,50.985416),(15.082107,50.992754),(15.073632,51.002624),(15.02299,51.009652),(15.003973,51.020685),(15.001182,51.012443),(14.996531,51.007844),(14.982062,51.001177),(14.976377,51.000402),(14.970693,50.998697),(14.965629,50.996062),(14.961081,50.992651),(14.960978,50.992651),(14.965215,50.981334),(14.981442,50.973221),(14.996635,50.959216),(14.997771,50.930174),(14.994671,50.92511),(14.984852,50.918082),(14.982062,50.914671),(14.981132,50.90909),(14.981752,50.905266),(14.982889,50.902424),(14.984129,50.867697),(14.982062,50.859067),(14.867857,50.86439),(14.831683,50.857982),(14.810393,50.858447),(14.810496,50.877051),(14.824655,50.89183),(14.860209,50.916842),(14.910645,50.992651),(14.955293,51.064042),(14.958291,51.075514),(14.958497,51.094738),(14.960254,51.104582),(14.965629,51.111507),(14.97338,51.116132),(14.979995,51.122798),(14.982062,51.13564),(14.983199,51.160909),(14.991364,51.18897),(15.004903,51.215609),(15.022059,51.23677),(15.017925,51.239561),(15.014308,51.242661),(15.019476,51.271678),(15.004903,51.291289),(14.982269,51.308006),(14.963768,51.328393),(14.960978,51.335291),(14.956637,51.353223),(14.956637,51.354153),(14.950648,51.396539),(14.946922,51.422909),(14.955397,51.435415),(14.945061,51.449186),(14.926561,51.46133),(14.910955,51.468978),(14.888321,51.475748),(14.841399,51.48443),(14.818558,51.492595),(14.818558,51.492698),(14.797474,51.502103),(14.732155,51.515823),(14.710037,51.530241),(14.70677,51.540828),(14.704559,51.547992),(14.712724,51.567422),(14.732155,51.58662),(14.743937,51.606644),(14.742283,51.632974),(14.732465,51.658347),(14.719546,51.675632),(14.67097,51.700773),(14.648439,51.717206),(14.638103,51.742631),(14.634176,51.769012),(14.620327,51.780536),(14.60224,51.788313),(14.585807,51.803893),(14.58281,51.825081),(14.595936,51.842418),(14.653296,51.878617),(14.671693,51.893888),(14.687093,51.911897),(14.695568,51.931457),(14.699082,51.992512),(14.704456,52.002227),(14.726057,52.026438),(14.730294,52.035895),(14.732155,52.045067),(14.735359,52.054033),(14.743007,52.062534),(14.761403,52.076667),(14.696394,52.106691),(14.686369,52.121032),(14.692467,52.146973),(14.703216,52.165163),(14.70704,52.179452),(14.69257,52.193379),(14.69257,52.19958),(14.712104,52.215444),(14.712414,52.235908),(14.696911,52.253478),(14.668903,52.260997),(14.640481,52.265054),(14.606581,52.275751),(14.584153,52.291228),(14.590148,52.309418),(14.569167,52.338512),(14.545396,52.382153),(14.539815,52.421866),(14.591595,52.449797),(14.615779,52.473284),(14.632316,52.496745),(14.627665,52.507416),(14.609061,52.517803),(14.616813,52.541032),(14.644821,52.576921),(14.613712,52.592527),(14.607965,52.596463),(14.549943,52.636194),(14.490102,52.650638),(14.46137,52.674564),(14.442353,52.679964),(14.431915,52.686088),(14.397705,52.727119),(14.382202,52.738152),(14.275955,52.78572),(14.24867,52.794582),(14.21601,52.817992),(14.19596,52.823366),(14.173739,52.824348),(14.152758,52.828327),(14.135395,52.836595),(14.123923,52.850651),(14.150485,52.86463),(14.16082,52.876748),(14.164954,52.895661),(14.16268,52.908167),(14.150485,52.943643),(14.14449,52.953694),(14.14449,52.959921),(14.192859,52.981651),(14.253734,53.000875),(14.312645,53.029219),(14.343341,53.048624),(14.356777,53.066684),(14.359568,53.075056),(14.38954,53.131668),(14.387473,53.146137),(14.381478,53.160296),(14.377861,53.176549),(14.380651,53.189855),(14.387783,53.202077),(14.397705,53.210991),(14.408247,53.214402),(14.415585,53.22099),(14.44163,53.251841),(14.428607,53.259076),(14.42468,53.264657),(14.409384,53.272434),(14.403286,53.27879),(14.400185,53.288609),(14.399565,53.306747),(14.398015,53.315222),(14.391261,53.334017),(14.370316,53.392298),(14.359878,53.444077),(14.338897,53.465213),(14.328458,53.48609),(14.30417,53.508518),(14.296936,53.529137),(14.296212,53.551306),(14.298899,53.587893),(14.297039,53.597763),(14.276058,53.636624),(14.260969,53.656209),(14.265103,53.671764),(14.263863,53.699876),(14.263901,53.699976),(14.263927,53.699978),(14.273692,53.700629),(14.291677,53.705268),(14.304861,53.712551),(14.306163,53.720689),(14.288585,53.727973),(14.29835,53.731431),(14.301606,53.734849),(14.29835,53.738267),(14.288585,53.741645),(14.2942,53.749091),(14.304047,53.743883),(14.318044,53.739732),(14.330414,53.734198),(14.342052,53.714911),(14.356212,53.708441),(14.383556,53.700629),(14.403575,53.687812),(14.414561,53.682359),(14.458669,53.677883),(14.51238,53.666449),(14.531016,53.657782),(14.546235,53.644761),(14.579112,53.607245),(14.590587,53.598375),(14.595876,53.601304),(14.597016,53.617336),(14.601573,53.626654),(14.610037,53.632758),(14.62379,53.639228),(14.623057,53.643012),(14.622813,53.646959),(14.62379,53.652818),(14.613048,53.65469),(14.587413,53.656073),(14.575938,53.659654),(14.544932,53.704047),(14.551443,53.726142),(14.575938,53.769517),(14.587413,53.763495),(14.597341,53.765204),(14.608246,53.769029),(14.62379,53.769517),(14.61671,53.794745),(14.620372,53.814846),(14.627289,53.832709),(14.630626,53.851508),(14.62379,53.851508),(14.621349,53.839545),(14.613536,53.83161),(14.58961,53.817328),(14.595876,53.810492),(14.587087,53.812649),(14.580089,53.815334),(14.574474,53.819078),(14.569102,53.824164),(14.574229,53.827094),(14.583263,53.834215),(14.58961,53.837836),(14.575857,53.854804),(14.552094,53.862779),(14.441173,53.869615),(14.425548,53.879828),(14.438162,53.899319),(14.434906,53.901313),(14.434093,53.901597),(14.433767,53.902411),(14.431895,53.906155),(14.422048,53.896145),(14.414236,53.894721),(14.405772,53.897406),(14.393809,53.899319),(14.385102,53.897284),(14.378266,53.89232),(14.371593,53.885891),(14.363048,53.879381),(14.392914,53.875963),(14.40919,53.871568),(14.417654,53.865139),(14.415701,53.853013),(14.405284,53.844184),(14.392589,53.840725),(14.383556,53.844672),(14.356212,53.834459),(14.342621,53.831855),(14.328298,53.831),(14.328298,53.824164),(14.342052,53.82099),(14.341319,53.818915),(14.328298,53.817328),(14.318858,53.818183),(14.303071,53.823065),(14.291352,53.824164),(14.274669,53.830797),(14.226573,53.871975),(14.203136,53.878323),(14.200857,53.87816),(14.200819,53.878157),(14.192756,53.893766),(14.175289,53.906478),(14.184798,53.908132),(14.193066,53.911491),(14.210076,53.938457),(14.226573,53.934027),(14.237478,53.928656),(14.268077,53.92182),(14.367035,53.916978),(14.406749,53.92182),(14.445567,53.934027),(14.517345,53.965237),(14.781912,54.034613),(15.010102,54.06981),(15.288341,54.138861),(15.468516,54.167141),(15.665782,54.198432),(15.859711,54.249986),(16.020844,54.261786),(16.131521,54.283515),(16.118907,54.276028),(16.069347,54.263007),(16.091075,54.253974),(16.17921,54.263007),(16.17921,54.269192),(16.172374,54.269192),(16.172374,54.276597),(16.220063,54.276597),(16.219574,54.28148),(16.218516,54.282375),(16.216645,54.28205),(16.213227,54.283515),(16.21518,54.299628),(16.192149,54.300279),(16.137706,54.290269),(16.17213,54.300238),(16.213552,54.316107),(16.275401,54.358588),(16.262869,54.343695),(16.240489,54.330024),(16.231781,54.322577),(16.240896,54.32453),(16.259044,54.326728),(16.274913,54.330471),(16.288829,54.340237),(16.323904,54.349921),(16.336762,54.358588),(16.32488,54.373114),(16.305024,54.373481),(16.296235,54.367255),(16.281586,54.358588),(16.323985,54.401313),(16.439789,54.488267),(16.439789,54.482123),(16.434418,54.472357),(16.444509,54.469875),(16.460623,54.473456),(16.473399,54.482123),(16.479747,54.495551),(16.473888,54.499661),(16.460948,54.496487),(16.446056,54.488267),(16.466563,54.512152),(16.501313,54.532416),(16.569591,54.557196),(16.778494,54.575141),(16.85963,54.59394),(16.902192,54.598131),(16.939952,54.606024),(17.035818,54.66706),(17.337413,54.74901),(17.715668,54.790432),(17.885427,54.824123),(18.152354,54.838324),(18.316905,54.838324),(18.339529,54.833482),(18.375824,54.815131),(18.389171,54.811103),(18.410167,54.808295),(18.75172,54.69009),(18.813975,54.643215),(18.824067,54.632229),(18.834321,54.616685),(18.835297,54.603095),(18.817638,54.598131),(18.813243,54.601549),(18.80836,54.608832),(18.802257,54.616034),(18.784028,54.623521),(18.771821,54.642279),(18.755382,54.654486),(18.707774,54.701239),(18.698497,54.703803),(18.663422,54.707994),(18.654552,54.71015),(18.64503,54.71955),(18.61671,54.725246),(18.556977,54.74901),(18.49822,54.765367),(18.48878,54.773179),(18.456309,54.787665),(18.447765,54.789984),(18.435557,54.786811),(18.42921,54.781684),(18.41505,54.753119),(18.413097,54.746527),(18.413097,54.73192),(18.418468,54.728217),(18.442393,54.721991),(18.447765,54.718248),(18.459809,54.706041),(18.474457,54.687567),(18.473888,54.679918),(18.470063,54.673285),(18.470388,54.666734),(18.481944,54.659613),(18.475922,54.640082),(18.487315,54.631781),(18.506358,54.634101),(18.52296,54.646552),(18.5171,54.62995),(18.525157,54.616685),(18.537608,54.605455),(18.562836,54.560004),(18.570649,54.55036),(18.566742,54.527411),(18.567556,54.48786),(18.574067,54.450385),(18.588064,54.433661),(18.60377,54.430609),(18.634288,54.416938),(18.649913,54.413804),(18.663341,54.412991),(18.680024,54.40998),(18.694347,54.404527),(18.70045,54.395819),(18.717296,54.382025),(18.885916,54.350165),(18.923188,54.348538),(18.954356,54.358588),(18.962901,54.353502),(18.97462,54.349758),(19.002126,54.344916)] +Paraguay [(-58.158797,-20.165125),(-58.158332,-20.181144),(-58.152337,-20.184658),(-58.144069,-20.183521),(-58.137041,-20.185588),(-58.12991,-20.192513),(-58.120505,-20.203882),(-58.118541,-20.214424),(-58.155283,-20.226413),(-58.162672,-20.243156),(-58.156316,-20.261656),(-58.137041,-20.274369),(-58.130943,-20.272818),(-58.126241,-20.267031),(-58.120039,-20.261346),(-58.109756,-20.260726),(-58.10309,-20.264447),(-58.09508,-20.272198),(-58.090325,-20.281707),(-58.099937,-20.309199),(-58.097715,-20.334623),(-58.090791,-20.359428),(-58.082987,-20.376171),(-58.067123,-20.393534),(-58.045625,-20.409347),(-58.023146,-20.420716),(-58.004181,-20.42516),(-57.989401,-20.433119),(-57.99214,-20.450895),(-58.007902,-20.479834),(-58.01002,-20.500091),(-58.008935,-20.525723),(-58.004749,-20.549804),(-57.993329,-20.573782),(-57.992554,-20.585047),(-57.993639,-20.606751),(-57.990952,-20.620601),(-57.977826,-20.640134),(-57.973123,-20.651193),(-57.973588,-20.661942),(-57.981082,-20.681165),(-57.980616,-20.692741),(-57.974415,-20.704006),(-57.96501,-20.711034),(-57.95638,-20.709898),(-57.947182,-20.675068),(-57.934004,-20.66742),(-57.918088,-20.669797),(-57.90429,-20.678995),(-57.900208,-20.685196),(-57.894472,-20.699976),(-57.890648,-20.706384),(-57.884343,-20.711965),(-57.868944,-20.722197),(-57.86021,-20.730258),(-57.857678,-20.73987),(-57.865275,-20.747415),(-57.877005,-20.752376),(-57.898761,-20.756613),(-57.906667,-20.762194),(-57.917933,-20.77408),(-57.933074,-20.784622),(-57.940412,-20.793613),(-57.933953,-20.799711),(-57.908011,-20.801985),(-57.887651,-20.805912),(-57.870701,-20.816454),(-57.859642,-20.831647),(-57.85711,-20.849734),(-57.867393,-20.860069),(-57.908683,-20.881567),(-57.917933,-20.894382),(-57.914832,-20.905648),(-57.908011,-20.910505),(-57.901138,-20.912366),(-57.898089,-20.914846),(-57.893025,-20.914846),(-57.869925,-20.932726),(-57.847033,-20.955981),(-57.840315,-20.955981),(-57.836026,-20.938514),(-57.817577,-20.953604),(-57.816027,-20.975411),(-57.82538,-20.998562),(-57.852459,-21.037836),(-57.851735,-21.051996),(-57.836026,-21.082485),(-57.829876,-21.12703),(-57.833959,-21.173849),(-57.84786,-21.216223),(-57.870701,-21.247643),(-57.899019,-21.267176),(-57.90429,-21.274928),(-57.90491,-21.287433),(-57.89995,-21.295081),(-57.893645,-21.300352),(-57.890648,-21.305934),(-57.882328,-21.316372),(-57.866308,-21.32092),(-57.855559,-21.330738),(-57.863311,-21.356886),(-57.929922,-21.453108),(-57.94527,-21.487214),(-57.949145,-21.508298),(-57.948112,-21.530519),(-57.939792,-21.548193),(-57.921602,-21.555427),(-57.912093,-21.564006),(-57.919328,-21.583953),(-57.932144,-21.60669),(-57.939017,-21.623744),(-57.934831,-21.640693),(-57.924341,-21.658677),(-57.911783,-21.673043),(-57.901138,-21.678934),(-57.895609,-21.688442),(-57.939017,-21.755312),(-57.938448,-21.774639),(-57.934779,-21.792932),(-57.93571,-21.812053),(-57.948939,-21.83386),(-57.955915,-21.85112),(-57.946665,-21.865486),(-57.932764,-21.881919),(-57.925374,-21.904967),(-57.929922,-21.917369),(-57.953228,-21.958193),(-57.962633,-21.966978),(-57.964855,-21.976694),(-57.986818,-22.035295),(-57.988368,-22.049041),(-57.988006,-22.06413),(-57.986249,-22.074465),(-57.973134,-22.081048),(-57.972865,-22.081183),(-57.961393,-22.090072),(-57.952453,-22.105988),(-57.939999,-22.11839),(-57.906254,-22.128932),(-57.895505,-22.128932),(-57.878194,-22.122111),(-57.871372,-22.121181),(-57.862846,-22.125832),(-57.848686,-22.140301),(-57.843881,-22.143712),(-57.831788,-22.141128),(-57.815769,-22.128312),(-57.805485,-22.128622),(-57.795305,-22.130999),(-57.787502,-22.129656),(-57.775409,-22.123971),(-57.777786,-22.122318),(-57.777425,-22.117047),(-57.775616,-22.111156),(-57.773652,-22.107745),(-57.769932,-22.107435),(-57.760268,-22.110742),(-57.756186,-22.110639),(-57.736342,-22.103714),(-57.723268,-22.104748),(-57.709935,-22.108469),(-57.689626,-22.109295),(-57.678516,-22.106401),(-57.670816,-22.102371),(-57.663685,-22.100614),(-57.654486,-22.104541),(-57.638932,-22.144539),(-57.627201,-22.165106),(-57.614075,-22.176991),(-57.596712,-22.181022),(-57.572734,-22.177715),(-57.564001,-22.174408),(-57.55997,-22.171617),(-57.554699,-22.170274),(-57.542297,-22.171514),(-57.534597,-22.174408),(-57.511188,-22.188774),(-57.472482,-22.180299),(-57.463955,-22.180402),(-57.443182,-22.187947),(-57.432278,-22.18867),(-57.41786,-22.193838),(-57.384994,-22.213268),(-57.378638,-22.213372),(-57.37502,-22.207274),(-57.366649,-22.209134),(-57.351249,-22.217196),(-57.339932,-22.216989),(-57.334816,-22.211718),(-57.333059,-22.205414),(-57.331767,-22.202416),(-57.316316,-22.203657),(-57.271667,-22.213785),(-57.252444,-22.210581),(-57.227484,-22.191254),(-57.21379,-22.188154),(-57.209449,-22.192081),(-57.2064,-22.207481),(-57.199527,-22.212855),(-57.189967,-22.213578),(-57.183611,-22.210891),(-57.177926,-22.207377),(-57.170382,-22.205724),(-57.154155,-22.207997),(-57.124183,-22.218539),(-57.115036,-22.22288),(-57.114623,-22.225051),(-57.113279,-22.229391),(-57.108473,-22.233525),(-57.090748,-22.236419),(-57.085064,-22.238073),(-57.079224,-22.23859),(-57.071628,-22.235696),(-57.059484,-22.232699),(-57.027651,-22.235076),(-57.011167,-22.233836),(-56.986465,-22.24138),(-56.975717,-22.24169),(-56.964554,-22.234869),(-56.965485,-22.250889),(-56.960524,-22.254816),(-56.9521,-22.253576),(-56.942592,-22.253989),(-56.924944,-22.262361),(-56.91479,-22.262878),(-56.90368,-22.236936),(-56.892001,-22.252232),(-56.881717,-22.277244),(-56.879547,-22.289956),(-56.85606,-22.29316),(-56.842856,-22.289026),(-56.84284,-22.28901),(-56.819654,-22.265462),(-56.813142,-22.25616),(-56.811902,-22.249649),(-56.808285,-22.247995),(-56.794565,-22.252542),(-56.786994,-22.251819),(-56.754361,-22.242621),(-56.74679,-22.243964),(-56.745601,-22.248408),(-56.743534,-22.250372),(-56.733199,-22.244067),(-56.728548,-22.237453),(-56.720538,-22.21947),(-56.715371,-22.215439),(-56.698602,-22.2235),(-56.664573,-22.259984),(-56.648811,-22.263498),(-56.644135,-22.255436),(-56.654703,-22.23735),(-56.648346,-22.231355),(-56.637184,-22.228254),(-56.629174,-22.22474),(-56.59574,-22.201486),(-56.584965,-22.190531),(-56.567912,-22.162315),(-56.565302,-22.159938),(-56.558016,-22.156424),(-56.554605,-22.152083),(-56.549076,-22.13627),(-56.545975,-22.130069),(-56.530679,-22.109089),(-56.521636,-22.09989),(-56.5113,-22.091415),(-56.493834,-22.086558),(-56.421797,-22.074465),(-56.407637,-22.075809),(-56.397302,-22.086248),(-56.370844,-22.147329),(-56.347331,-22.180402),(-56.316842,-22.207687),(-56.261083,-22.23859),(-56.231524,-22.265358),(-56.214264,-22.275383),(-56.196798,-22.279311),(-56.024715,-22.287372),(-56.011357,-22.283858),(-55.998567,-22.277244),(-55.985234,-22.283238),(-55.970868,-22.293884),(-55.955262,-22.301325),(-55.89294,-22.306803),(-55.874388,-22.317345),(-55.872011,-22.310213),(-55.86103,-22.289543),(-55.810775,-22.353105),(-55.770157,-22.381423),(-55.760235,-22.391965),(-55.751579,-22.416253),(-55.748298,-22.500899),(-55.74127,-22.537176),(-55.723932,-22.569629),(-55.697913,-22.595571),(-55.664685,-22.612727),(-55.644351,-22.619548),(-55.631199,-22.626783),(-55.62368,-22.638669),(-55.620476,-22.659339),(-55.62151,-22.667091),(-55.627013,-22.683627),(-55.627788,-22.692102),(-55.62567,-22.700784),(-55.619184,-22.716907),(-55.618332,-22.725692),(-55.628589,-22.758971),(-55.645927,-22.78791),(-55.659001,-22.818296),(-55.656572,-22.85571),(-55.6343,-22.932811),(-55.634351,-22.950071),(-55.639984,-22.983764),(-55.637814,-23.000714),(-55.611459,-23.029136),(-55.600684,-23.044535),(-55.595232,-23.064069),(-55.600142,-23.100553),(-55.599392,-23.117089),(-55.588669,-23.130112),(-55.560867,-23.145925),(-55.557147,-23.15471),(-55.535236,-23.229124),(-55.535649,-23.245867),(-55.559007,-23.291342),(-55.562159,-23.307879),(-55.555596,-23.330306),(-55.524875,-23.360485),(-55.513868,-23.379399),(-55.515547,-23.410198),(-55.542471,-23.465802),(-55.539913,-23.500528),(-55.533796,-23.530444),(-55.533066,-23.534015),(-55.533892,-23.569981),(-55.530533,-23.603468),(-55.51131,-23.629409),(-55.466971,-23.673231),(-55.445319,-23.735449),(-55.430178,-23.928564),(-55.420669,-23.954506),(-55.398035,-23.97683),(-55.367184,-23.989646),(-55.304346,-23.99409),(-55.27272,-24.000394),(-55.272668,-24.000498),(-55.272565,-24.000498),(-55.27241,-24.000601),(-55.272358,-24.000601),(-55.23683,-24.012693),(-55.200864,-24.019515),(-55.165439,-24.016827),(-55.131281,-24.000601),(-55.131204,-24.000601),(-55.131075,-24.000601),(-55.131075,-24.000394),(-55.105572,-23.988509),(-55.042294,-23.989026),(-55.011495,-23.980551),(-54.994571,-23.973109),(-54.961059,-23.971766),(-54.943644,-23.969182),(-54.926746,-23.95988),(-54.904887,-23.933008),(-54.891916,-23.920606),(-54.827424,-23.887016),(-54.696838,-23.845158),(-54.679784,-23.836683),(-54.654592,-23.811465),(-54.639218,-23.804437),(-54.612553,-23.811155),(-54.443158,-23.899935),(-54.422901,-23.913785),(-54.367607,-23.984685),(-54.273608,-24.036465),(-54.24575,-24.050393),(-54.245289,-24.050624),(-54.266218,-24.06592),(-54.301048,-24.089898),(-54.324767,-24.118217),(-54.334637,-24.148912),(-54.331847,-24.165036),(-54.318979,-24.196765),(-54.314122,-24.234179),(-54.306267,-24.246684),(-54.282651,-24.275313),(-54.274331,-24.298257),(-54.26105,-24.32947),(-54.262187,-24.358512),(-54.271696,-24.389415),(-54.284098,-24.411325),(-54.322907,-24.464345),(-54.334534,-24.496695),(-54.334792,-24.527287),(-54.320685,-24.595914),(-54.321305,-24.628366),(-54.358429,-24.731422),(-54.371121,-24.766653),(-54.39975,-24.80479),(-54.407294,-24.821223),(-54.453286,-25.002814),(-54.462278,-25.03723),(-54.46419,-25.055937),(-54.463312,-25.072784),(-54.455199,-25.092111),(-54.429722,-25.130558),(-54.426621,-25.149472),(-54.435665,-25.167042),(-54.46941,-25.195464),(-54.48145,-25.21324),(-54.503568,-25.278663),(-54.528476,-25.314319),(-54.547079,-25.336644),(-54.575656,-25.360311),(-54.597981,-25.397518),(-54.609814,-25.432762),(-54.61183,-25.444027),(-54.61245,-25.465525),(-54.608523,-25.487642),(-54.598497,-25.505832),(-54.591469,-25.524746),(-54.594156,-25.54862),(-54.599944,-25.572702),(-54.600203,-25.574945),(-54.602218,-25.592442),(-54.58625,-25.624688),(-54.583356,-25.644842),(-54.598497,-25.653834),(-54.633715,-25.652077),(-54.642887,-25.661585),(-54.643146,-25.68794),(-54.62493,-25.740237),(-54.622062,-25.759977),(-54.617152,-25.773516),(-54.594518,-25.798734),(-54.587904,-25.810827),(-54.589557,-25.832944),(-54.599221,-25.850721),(-54.610073,-25.866534),(-54.61524,-25.882863),(-54.607334,-25.927925),(-54.606404,-25.946632),(-54.61524,-25.961722),(-54.62369,-25.964306),(-54.647952,-25.965649),(-54.656194,-25.969163),(-54.662266,-25.979912),(-54.661594,-25.989937),(-54.658261,-26.000376),(-54.653326,-26.030761),(-54.642526,-26.062801),(-54.643146,-26.085228),(-54.663816,-26.149204),(-54.659941,-26.164397),(-54.641854,-26.184034),(-54.638288,-26.196953),(-54.644696,-26.208218),(-54.656194,-26.222998),(-54.664333,-26.236227),(-54.664974,-26.238863),(-54.666142,-26.243668),(-54.663661,-26.263305),(-54.663661,-26.308367),(-54.679061,-26.33989),(-54.68043,-26.345264),(-54.683402,-26.350845),(-54.693065,-26.411927),(-54.697664,-26.428153),(-54.706475,-26.441796),(-54.737171,-26.473215),(-54.765309,-26.494092),(-54.780347,-26.510422),(-54.789829,-26.528509),(-54.78877,-26.542565),(-54.783499,-26.556724),(-54.780347,-26.575224),(-54.781277,-26.598169),(-54.784946,-26.622767),(-54.793059,-26.644987),(-54.807012,-26.661214),(-54.817321,-26.665761),(-54.822825,-26.663281),(-54.827424,-26.658113),(-54.834969,-26.654393),(-54.879359,-26.654393),(-54.903337,-26.65987),(-54.919899,-26.674133),(-54.930725,-26.693977),(-54.943877,-26.739969),(-54.951137,-26.757539),(-54.961007,-26.772318),(-54.97527,-26.787821),(-54.991781,-26.794642),(-55.040744,-26.798466),(-55.060898,-26.805184),(-55.125778,-26.863579),(-55.132677,-26.880632),(-55.118879,-26.920009),(-55.122315,-26.941714),(-55.138361,-26.953702),(-55.161564,-26.95732),(-55.2012,-26.955459),(-55.216651,-26.951222),(-55.235177,-26.94223),(-55.256545,-26.934582),(-55.2806,-26.934272),(-55.329409,-26.958457),(-55.376228,-26.963831),(-55.39633,-26.969309),(-55.414003,-26.979851),(-55.43085,-26.996387),(-55.442761,-27.015508),(-55.448833,-27.035868),(-55.451313,-27.082067),(-55.461855,-27.097983),(-55.486402,-27.10005),(-55.513997,-27.097466),(-55.533841,-27.09943),(-55.545365,-27.115243),(-55.549524,-27.13581),(-55.555338,-27.153794),(-55.571745,-27.161545),(-55.598281,-27.16754),(-55.598979,-27.182629),(-55.574872,-27.223557),(-55.570583,-27.235339),(-55.568671,-27.245984),(-55.570712,-27.256423),(-55.58407,-27.276784),(-55.583243,-27.283501),(-55.580763,-27.291666),(-55.581641,-27.305516),(-55.591331,-27.328357),(-55.60748,-27.345513),(-55.628719,-27.356365),(-55.653627,-27.360086),(-55.681713,-27.379413),(-55.717499,-27.417343),(-55.754732,-27.443698),(-55.815167,-27.41414),(-55.842091,-27.407422),(-55.854157,-27.40122),(-55.862787,-27.391092),(-55.877851,-27.353575),(-55.892837,-27.334868),(-55.913094,-27.32784),(-55.936374,-27.328253),(-55.960662,-27.331561),(-55.966094,-27.331723),(-55.984847,-27.332284),(-56.006887,-27.328253),(-56.006894,-27.328251),(-56.073756,-27.304896),(-56.074246,-27.304811),(-56.098948,-27.300555),(-56.124554,-27.298901),(-56.149979,-27.31182),(-56.168892,-27.324739),(-56.205428,-27.362153),(-56.22801,-27.371248),(-56.256768,-27.378276),(-56.279842,-27.389645),(-56.285319,-27.411349),(-56.281469,-27.438634),(-56.285319,-27.460958),(-56.296998,-27.480905),(-56.31648,-27.500439),(-56.335988,-27.525657),(-56.349657,-27.556353),(-56.367537,-27.580641),(-56.399731,-27.586842),(-56.400299,-27.586636),(-56.431357,-27.57537),(-56.461484,-27.553873),(-56.487323,-27.527208),(-56.50621,-27.500129),(-56.54644,-27.455067),(-56.546933,-27.455003),(-56.613,-27.446386),(-56.638016,-27.452912),(-56.683125,-27.464679),(-56.734233,-27.500439),(-56.771155,-27.506744),(-56.771511,-27.506548),(-56.807458,-27.486797),(-56.869263,-27.431503),(-56.904351,-27.418687),(-56.904447,-27.418696),(-56.94099,-27.422098),(-56.977318,-27.435327),(-57.076227,-27.484006),(-57.10899,-27.490621),(-57.148161,-27.488967),(-57.156687,-27.487727),(-57.180097,-27.487313),(-57.180453,-27.487214),(-57.191207,-27.484213),(-57.200406,-27.478942),(-57.227071,-27.458685),(-57.237096,-27.45424),(-57.237588,-27.45412),(-57.283026,-27.443015),(-57.292493,-27.440701),(-57.302983,-27.435534),(-57.310116,-27.429208),(-57.326289,-27.414863),(-57.335229,-27.409489),(-57.356466,-27.404343),(-57.360396,-27.403391),(-57.386699,-27.403494),(-57.487675,-27.416413),(-57.513358,-27.41414),(-57.519938,-27.412099),(-57.535682,-27.407215),(-57.697791,-27.333524),(-57.698183,-27.333398),(-57.709367,-27.329804),(-57.811583,-27.310683),(-57.811913,-27.310583),(-57.854784,-27.297557),(-57.855088,-27.297499),(-57.899504,-27.288961),(-57.913385,-27.286292),(-57.94403,-27.274613),(-57.944356,-27.274551),(-58.003664,-27.263303),(-58.021958,-27.259834),(-58.022046,-27.259831),(-58.053532,-27.259007),(-58.113993,-27.268929),(-58.128824,-27.269652),(-58.238223,-27.257043),(-58.510765,-27.278437),(-58.600165,-27.312957),(-58.604196,-27.316264),(-58.604135,-27.315548),(-58.599907,-27.266138),(-58.60156,-27.245674),(-58.614014,-27.226657),(-58.638716,-27.210121),(-58.652358,-27.198029),(-58.658404,-27.18511),(-58.653289,-27.156274),(-58.638819,-27.135914),(-58.616133,-27.123821),(-58.56549,-27.115966),(-58.560529,-27.105528),(-58.56239,-27.090128),(-58.562183,-27.072145),(-58.559703,-27.06336),(-58.556395,-27.055505),(-58.551693,-27.048167),(-58.54544,-27.041036),(-58.536259,-27.036294),(-58.536035,-27.036178),(-58.530505,-27.040519),(-58.528025,-27.047754),(-58.528025,-27.051681),(-58.527043,-27.054161),(-58.520032,-27.058368),(-58.519808,-27.058502),(-58.51154,-27.060156),(-58.507561,-27.055092),(-58.506734,-27.038452),(-58.504874,-27.031527),(-58.50136,-27.023672),(-58.492368,-27.00972),(-58.47454,-26.990083),(-58.466633,-26.975923),(-58.481619,-26.963831),(-58.483686,-26.950602),(-58.47547,-26.937993),(-58.459812,-26.928174),(-58.415525,-26.917529),(-58.393873,-26.909674),(-58.384674,-26.897065),(-58.374701,-26.887557),(-58.328916,-26.884146),(-58.31579,-26.874121),(-58.322146,-26.856758),(-58.340388,-26.844459),(-58.352377,-26.830506),(-58.340026,-26.808595),(-58.323335,-26.803841),(-58.305506,-26.809112),(-58.288556,-26.811489),(-58.27481,-26.798363),(-58.278479,-26.790198),(-58.286541,-26.778726),(-58.287936,-26.768598),(-58.2714,-26.764257),(-58.259256,-26.764463),(-58.251763,-26.763223),(-58.247939,-26.758159),(-58.2452,-26.680231),(-58.240652,-26.661214),(-58.235536,-26.649845),(-58.232281,-26.648605),(-58.227165,-26.652015),(-58.195694,-26.657493),(-58.18386,-26.65677),(-58.178641,-26.650672),(-58.181379,-26.641577),(-58.187167,-26.633619),(-58.192076,-26.624627),(-58.192283,-26.612845),(-58.188511,-26.609744),(-58.171509,-26.598582),(-58.164946,-26.592278),(-58.170527,-26.5868),(-58.179312,-26.57264),(-58.18541,-26.564992),(-58.207993,-26.550213),(-58.213367,-26.544529),(-58.217346,-26.527579),(-58.214194,-26.512386),(-58.208716,-26.496883),(-58.205926,-26.479416),(-58.202722,-26.471561),(-58.188666,-26.462053),(-58.18541,-26.451718),(-58.209595,-26.431254),(-58.213057,-26.418645),(-58.206184,-26.402728),(-58.172698,-26.349295),(-58.167633,-26.335032),(-58.164946,-26.318599),(-58.170011,-26.28656),(-58.169959,-26.270333),(-58.161846,-26.263305),(-58.145981,-26.260101),(-58.123295,-26.251523),(-58.105983,-26.239534),(-58.106604,-26.226098),(-58.121228,-26.216693),(-58.137144,-26.209459),(-58.148927,-26.19933),(-58.151304,-26.18145),(-58.141175,-26.184757),(-58.133217,-26.188995),(-58.127636,-26.194679),(-58.124019,-26.201914),(-58.115234,-26.190958),(-58.106293,-26.170288),(-58.096682,-26.136698),(-58.086501,-26.12719),(-58.021596,-26.105692),(-57.988213,-26.088536),(-57.872716,-26.010298),(-57.859694,-25.994278),(-57.85959,-25.980945),(-57.905841,-25.968646),(-57.898244,-25.953557),(-57.863311,-25.927615),(-57.85711,-25.919864),(-57.851374,-25.908392),(-57.852407,-25.897953),(-57.875352,-25.890202),(-57.878194,-25.883174),(-57.875352,-25.876146),(-57.854216,-25.868911),(-57.833235,-25.858782),(-57.819662,-25.849779),(-57.812823,-25.845243),(-57.801868,-25.831394),(-57.817061,-25.797494),(-57.820626,-25.77827),(-57.805588,-25.769899),(-57.787243,-25.764111),(-57.767916,-25.750675),(-57.75107,-25.734862),(-57.740373,-25.72215),(-57.751121,-25.719566),(-57.760578,-25.715639),(-57.768485,-25.709954),(-57.774582,-25.701686),(-57.753705,-25.672747),(-57.741251,-25.661895),(-57.719909,-25.653834),(-57.708695,-25.652903),(-57.698205,-25.653627),(-57.688335,-25.652697),(-57.678981,-25.647012),(-57.67526,-25.639261),(-57.670041,-25.616937),(-57.668077,-25.612286),(-57.638777,-25.6158),(-57.623067,-25.615386),(-57.616298,-25.609185),(-57.611647,-25.585931),(-57.600691,-25.578799),(-57.587152,-25.575699),(-57.575318,-25.564433),(-57.570047,-25.546657),(-57.556921,-25.45984),(-57.55811,-25.44351),(-57.565293,-25.430385),(-57.578729,-25.416742),(-57.640792,-25.37261),(-57.641206,-25.360518),(-57.671488,-25.290135),(-57.671976,-25.289831),(-57.702339,-25.270911),(-57.721769,-25.246106),(-57.754067,-25.180891),(-57.766004,-25.167868),(-57.79732,-25.153916),(-57.81241,-25.143684),(-57.85649,-25.096762),(-57.870701,-25.085289),(-57.890493,-25.078675),(-57.969764,-25.078468),(-57.983769,-25.074231),(-57.989866,-25.064102),(-57.993639,-25.052733),(-58.00046,-25.044362),(-58.009814,-25.042501),(-58.032138,-25.045085),(-58.04206,-25.044362),(-58.111513,-25.012322),(-58.124019,-25.012942),(-58.133889,-24.997853),(-58.224064,-24.941216),(-58.242513,-24.941732),(-58.259152,-24.952998),(-58.287264,-24.979766),(-58.299305,-24.987724),(-58.311966,-24.993615),(-58.323283,-24.995269),(-58.335995,-24.991858),(-58.341318,-24.986174),(-58.344367,-24.978319),(-58.350568,-24.968604),(-58.414802,-24.903078),(-58.438521,-24.872796),(-58.450769,-24.861737),(-58.473403,-24.851299),(-58.694216,-24.812024),(-58.699384,-24.80727),(-58.701916,-24.799312),(-58.708117,-24.795075),(-58.715972,-24.791871),(-58.723258,-24.786703),(-58.737469,-24.782776),(-58.788474,-24.781639),(-58.809196,-24.776781),(-59.000916,-24.644179),(-59.032697,-24.637048),(-59.042412,-24.630537),(-59.062463,-24.612347),(-59.072178,-24.606766),(-59.078327,-24.605629),(-59.097189,-24.605422),(-59.116981,-24.598807),(-59.154395,-24.576277),(-59.177908,-24.568732),(-59.206485,-24.550232),(-59.248084,-24.537209),(-59.257644,-24.530181),(-59.263122,-24.523463),(-59.27046,-24.518192),(-59.285601,-24.516125),(-59.295058,-24.512921),(-59.308081,-24.498762),(-59.340973,-24.4876),(-59.357406,-24.468996),(-59.377766,-24.433546),(-59.387094,-24.423831),(-59.426032,-24.392515),(-59.450268,-24.382387),(-59.453808,-24.374532),(-59.456289,-24.365437),(-59.459725,-24.358512),(-59.4659,-24.353551),(-59.487656,-24.344766),(-59.51042,-24.332674),(-59.521143,-24.325026),(-59.53202,-24.31407),(-59.539979,-24.308799),(-59.558427,-24.305389),(-59.575403,-24.29588),(-59.600518,-24.292883),(-59.610517,-24.289576),(-59.617726,-24.283478),(-59.635322,-24.261464),(-59.67364,-24.225394),(-60.033669,-24.007009),(-60.05173,-24.002875),(-60.072375,-24.004839),(-60.119297,-24.020238),(-60.143249,-24.025199),(-60.218128,-24.027369),(-60.296418,-24.016414),(-60.328018,-24.017964),(-60.337372,-24.016414),(-60.348043,-24.010833),(-60.367577,-23.99471),(-60.378351,-23.988509),(-60.38742,-23.987165),(-60.423051,-23.988509),(-60.484882,-23.977347),(-60.517723,-23.955746),(-60.535964,-23.947581),(-60.57175,-23.946031),(-60.577538,-23.94417),(-60.593584,-23.912234),(-60.594617,-23.90655),(-60.603841,-23.90469),(-60.621153,-23.895801),(-60.63216,-23.892287),(-60.64371,-23.891357),(-60.689676,-23.893631),(-60.699417,-23.891254),(-60.719855,-23.875027),(-60.729286,-23.872133),(-60.787776,-23.873445),(-60.816852,-23.874097),(-60.837625,-23.871823),(-60.866358,-23.855907),(-60.899947,-23.830482),(-60.936663,-23.813842),(-60.974826,-23.824074),(-61.006349,-23.805471),(-61.015806,-23.796686),(-61.030224,-23.77462),(-61.036244,-23.768832),(-61.038053,-23.755396),(-61.04999,-23.734726),(-61.066216,-23.715709),(-61.092519,-23.70186),(-61.10965,-23.675608),(-61.118849,-23.66641),(-61.109754,-23.649977),(-61.10624,-23.627239),(-61.109857,-23.606982),(-61.122233,-23.598197),(-61.139493,-23.592719),(-61.154454,-23.580317),(-61.167812,-23.566571),(-61.18024,-23.557166),(-61.187733,-23.556235),(-61.20768,-23.557579),(-61.214398,-23.557166),(-61.223571,-23.551275),(-61.242924,-23.533084),(-61.251941,-23.52926),(-61.272612,-23.523576),(-61.282947,-23.509933),(-61.288967,-23.49412),(-61.296951,-23.481408),(-61.36178,-23.454743),(-61.384853,-23.453709),(-61.398677,-23.450092),(-61.408728,-23.443581),(-61.42051,-23.433659),(-61.435703,-23.423737),(-61.451929,-23.417433),(-61.4703,-23.414125),(-61.491849,-23.413195),(-61.501073,-23.407821),(-61.510375,-23.38436),(-61.526343,-23.374748),(-61.525465,-23.364723),(-61.520788,-23.35325),(-61.51606,-23.344982),(-61.533965,-23.344569),(-61.545773,-23.342605),(-61.554636,-23.338368),(-61.60515,-23.289275),(-61.619619,-23.282867),(-61.666851,-23.282867),(-61.680313,-23.27925),(-61.688762,-23.274496),(-61.70442,-23.258372),(-61.718528,-23.249277),(-61.732997,-23.243386),(-61.744263,-23.234808),(-61.74881,-23.217341),(-61.749482,-23.198841),(-61.752273,-23.187059),(-61.758474,-23.177447),(-61.769274,-23.165562),(-61.780023,-23.15688),(-61.802657,-23.144994),(-61.814284,-23.135176),(-61.83697,-23.104687),(-61.84498,-23.097245),(-61.956446,-23.034407),(-61.992051,-22.99813),(-62.0059,-22.978906),(-62.006055,-22.974359),(-62.008949,-22.969708),(-62.003782,-22.94635),(-62.006055,-22.936842),(-62.017166,-22.921752),(-62.035821,-22.884855),(-62.050446,-22.864495),(-62.071633,-22.843928),(-62.080831,-22.832352),(-62.084604,-22.820156),(-62.089358,-22.81995),(-62.099228,-22.813748),(-62.107858,-22.806514),(-62.108788,-22.803),(-62.115868,-22.799796),(-62.119744,-22.792148),(-62.122534,-22.783156),(-62.126151,-22.775715),(-62.153488,-22.747809),(-62.159741,-22.737164),(-62.164082,-22.725899),(-62.1708,-22.71732),(-62.184184,-22.713703),(-62.188267,-22.708329),(-62.175916,-22.684867),(-62.195036,-22.673602),(-62.194623,-22.66058),(-62.187646,-22.638565),(-62.192969,-22.62823),(-62.196586,-22.626473),(-62.202529,-22.627197),(-62.214932,-22.624303),(-62.239736,-22.613761),(-62.25281,-22.603632),(-62.252965,-22.591126),(-62.238858,-22.57304),(-62.233225,-22.556296),(-62.241183,-22.538416),(-62.263352,-22.514439),(-62.26883,-22.512992),(-62.2756,-22.513508),(-62.281387,-22.512061),(-62.283816,-22.504827),(-62.28361,-22.493768),(-62.28454,-22.488704),(-62.287227,-22.483949),(-62.294513,-22.479815),(-62.305727,-22.476715),(-62.341357,-22.472261),(-62.348722,-22.47134),(-62.368256,-22.464416),(-62.43807,-22.419664),(-62.454969,-22.403851),(-62.461945,-22.388761),(-62.470575,-22.381837),(-62.510727,-22.370158),(-62.523388,-22.364887),(-62.547624,-22.335328),(-62.564678,-22.321996),(-62.588294,-22.316414),(-62.599611,-22.315174),(-62.61315,-22.311454),(-62.625294,-22.305046),(-62.632684,-22.295951),(-62.630824,-22.287682),(-62.624209,-22.278897),(-62.620282,-22.268459),(-62.626431,-22.255023),(-62.61961,-22.255023),(-62.624829,-22.247271),(-62.631805,-22.24076),(-62.640384,-22.236213),(-62.650357,-22.234456),(-62.627516,-22.184536),(-62.599404,-22.089865),(-62.572843,-22.000775),(-62.529228,-21.865073),(-62.492796,-21.751798),(-62.446184,-21.606897),(-62.411974,-21.500857),(-62.37549,-21.384482),(-62.340402,-21.272861),(-62.307898,-21.169301),(-62.275703,-21.066568),(-62.271879,-21.000939),(-62.271886,-21.000424),(-62.273791,-20.854798),(-62.275031,-20.75868),(-62.276168,-20.670727),(-62.277305,-20.579776),(-62.26883,-20.553111),(-62.232398,-20.501021),(-62.210573,-20.471305),(-62.189662,-20.442834),(-62.144962,-20.382062),(-62.10021,-20.321187),(-62.05551,-20.260416),(-62.01081,-20.199541),(-61.99417,-20.175667),(-61.977582,-20.151792),(-61.960942,-20.128021),(-61.94425,-20.104146),(-61.931383,-20.078308),(-61.92474,-20.065128),(-61.918412,-20.052573),(-61.905545,-20.026735),(-61.892574,-20.000897),(-61.892522,-20.000897),(-61.86441,-19.927413),(-61.836298,-19.853826),(-61.808186,-19.780342),(-61.780074,-19.706961),(-61.761213,-19.657765),(-61.753203,-19.64588),(-61.737235,-19.639575),(-61.648222,-19.62676),(-61.626725,-19.623659),(-61.579544,-19.616838),(-61.532286,-19.610016),(-61.51084,-19.606916),(-61.17236,-19.537463),(-60.833931,-19.467906),(-60.495476,-19.398453),(-60.156995,-19.329),(-60.006384,-19.298097),(-59.924322,-19.297064),(-59.735858,-19.29479),(-59.547498,-19.292413),(-59.359085,-19.290139),(-59.170725,-19.287762),(-59.089541,-19.286729),(-59.069646,-19.291483),(-59.039673,-19.311637),(-59.011096,-19.330964),(-58.96066,-19.360523),(-58.867436,-19.4153),(-58.774108,-19.470077),(-58.680832,-19.52475),(-58.587504,-19.579424),(-58.49428,-19.634201),(-58.401004,-19.688875),(-58.30778,-19.743652),(-58.214452,-19.798325),(-58.175282,-19.821373),(-58.164429,-19.832949),(-58.161174,-19.847211),(-58.164223,-19.880284),(-58.162207,-19.912324),(-58.145154,-19.969581),(-58.141847,-20.000897),(-58.144482,-20.023118),(-58.143139,-20.065286),(-58.144741,-20.086163),(-58.15952,-20.13908),(-58.158797,-20.165125)] +Qatar [(51.215258,24.62585),(51.147451,24.576279),(51.095465,24.560182),(51.038879,24.559872),(50.978883,24.567726),(50.928602,24.587208),(50.881008,24.636404),(50.807872,24.746649),(50.826671,24.748684),(50.852794,24.770006),(50.85963,24.7956),(50.851085,24.890815),(50.842784,24.916083),(50.799978,24.988756),(50.796641,25.003607),(50.807384,25.031643),(50.809744,25.051215),(50.808604,25.069159),(50.802419,25.076972),(50.775727,25.091864),(50.768403,25.1258),(50.766938,25.162584),(50.757823,25.186225),(50.764985,25.204291),(50.763438,25.225287),(50.759044,25.247504),(50.757823,25.269355),(50.765798,25.323554),(50.765147,25.344468),(50.750987,25.419623),(50.762462,25.462714),(50.763845,25.482408),(50.750987,25.494696),(50.778005,25.527045),(50.797093,25.529896),(50.808484,25.517239),(50.811648,25.498887),(50.820508,25.491293),(50.820486,25.471829),(50.847179,25.467353),(50.831309,25.507799),(50.827891,25.525051),(50.826671,25.543118),(50.828624,25.554348),(50.832205,25.566148),(50.832856,25.578518),(50.826671,25.590888),(50.818533,25.595771),(50.79656,25.599595),(50.785004,25.60456),(50.796397,25.617255),(50.809581,25.620592),(50.823497,25.622138),(50.836925,25.628811),(50.84018,25.634508),(50.846528,25.649848),(50.850597,25.652981),(50.857188,25.650824),(50.863536,25.641425),(50.870942,25.639309),(50.87615,25.631781),(50.874278,25.614936),(50.867686,25.587836),(50.867686,25.567043),(50.868663,25.559556),(50.87672,25.540595),(50.881521,25.535142),(50.888682,25.529486),(50.898692,25.532213),(50.900727,25.534857),(50.900076,25.53974),(50.904145,25.563137),(50.904796,25.587063),(50.90919,25.597724),(50.920584,25.608344),(50.927501,25.60517),(50.93572,25.598334),(50.949555,25.597724),(50.955903,25.604682),(50.9699,25.633857),(50.97755,25.646145),(50.94988,25.636461),(50.924327,25.63231),(50.903168,25.640123),(50.888682,25.666653),(50.887543,25.680894),(50.894867,25.728095),(50.897309,25.732571),(50.906993,25.739936),(50.90919,25.745103),(50.90797,25.749254),(50.902843,25.755276),(50.901703,25.759467),(50.901703,25.789537),(50.915538,25.786444),(50.920258,25.794379),(50.923595,25.805162),(50.932872,25.810614),(50.939708,25.807196),(50.945567,25.798489),(50.949229,25.787014),(50.949555,25.775295),(50.963715,25.783881),(50.972992,25.797512),(50.975922,25.813788),(50.970714,25.830512),(50.963878,25.823676),(50.949392,25.837958),(50.951182,25.85928),(50.977306,25.916449),(50.981456,25.935858),(50.984223,25.98135),(50.992524,25.975735),(50.995616,25.972846),(50.998546,25.967027),(51.004731,25.967027),(51.027354,26.026923),(51.047374,26.053046),(51.075938,26.06391),(51.091075,26.066962),(51.133149,26.082099),(51.141856,26.087795),(51.147227,26.103095),(51.170584,26.124579),(51.176117,26.139594),(51.194347,26.133531),(51.214854,26.13935),(51.251231,26.160102),(51.33074,26.121161),(51.346853,26.104885),(51.348481,26.085639),(51.346365,26.060696),(51.347179,26.039008),(51.357677,26.029731),(51.376638,26.022895),(51.382498,26.006171),(51.383474,25.985582),(51.388438,25.967027),(51.403982,25.954088),(51.416515,25.952094),(51.424327,25.950873),(51.46754,25.954047),(51.491954,25.95189),(51.511974,25.946112),(51.52947,25.937323),(51.546072,25.926093),(51.559418,25.913764),(51.570323,25.898383),(51.577485,25.880845),(51.594737,25.771877),(51.594493,25.758246),(51.591563,25.746649),(51.583507,25.741767),(51.577647,25.745063),(51.572032,25.759508),(51.563324,25.762885),(51.555431,25.758694),(51.54835,25.748847),(51.544607,25.737372),(51.546072,25.728095),(51.555024,25.720649),(51.56365,25.721625),(51.570323,25.727484),(51.573985,25.734931),(51.590099,25.7119),(51.576834,25.688666),(51.556814,25.681627),(51.552908,25.707587),(51.539317,25.703925),(51.505138,25.687161),(51.519705,25.681708),(51.534923,25.673245),(51.54713,25.663153),(51.552908,25.652981),(51.54949,25.631171),(51.535899,25.622382),(51.518809,25.61872),(51.505138,25.612006),(51.498302,25.615912),(51.491384,25.618232),(51.493663,25.565823),(51.491547,25.550645),(51.491384,25.549302),(51.48585,25.540513),(51.47755,25.531073),(51.471934,25.521389),(51.474294,25.512112),(51.510997,25.454291),(51.516938,25.440172),(51.518809,25.426215),(51.515717,25.382839),(51.511974,25.330227),(51.513194,25.308254),(51.518809,25.297065),(51.531261,25.292182),(51.590017,25.283759),(51.599132,25.274888),(51.607595,25.255113),(51.611095,25.237535),(51.614594,25.219794),(51.616547,25.137193),(51.605724,25.034247),(51.611013,25.022366),(51.602306,25.013821),(51.586925,24.953437),(51.574067,24.934394),(51.539399,24.898871),(51.528819,24.874416),(51.520844,24.870592),(51.511729,24.867865),(51.505138,24.864081),(51.500255,24.855373),(51.474213,24.764472),(51.47283,24.759426),(51.46754,24.748684),(51.455821,24.740668),(51.450369,24.721666),(51.443614,24.679755),(51.430431,24.664496),(51.389985,24.642483),(51.381521,24.635077),(51.37615,24.611884),(51.363536,24.59634),(51.349783,24.583808),(51.340587,24.569892),(51.328949,24.584784),(51.332693,24.598863),(51.341645,24.614691),(51.346853,24.635077),(51.339122,24.646186),(51.321137,24.648179),(51.285981,24.645006),(51.284434,24.662502),(51.278087,24.662991),(51.267914,24.656195),(51.254568,24.651801),(51.23699,24.650051),(51.220958,24.644232),(51.212576,24.633694),(51.214854,24.627183),(51.215258,24.62585)] +Romania [(26.722379,48.259769),(26.733127,48.27075),(26.744031,48.255583),(26.76372,48.252741),(26.804906,48.25827),(26.821908,48.252534),(26.844749,48.23331),(26.855497,48.237832),(26.897665,48.208971),(26.90397,48.201452),(26.908001,48.184528),(26.917716,48.187964),(26.929085,48.199049),(26.938128,48.204811),(26.950582,48.197524),(26.955646,48.186),(26.963088,48.175381),(26.997607,48.16657),(26.997607,48.15794),(26.986239,48.150421),(26.966602,48.149594),(26.966602,48.143367),(26.99399,48.132412),(27.012697,48.128123),(27.025099,48.135048),(27.033678,48.132386),(27.042773,48.127296),(27.048044,48.122232),(27.047955,48.121409),(27.047424,48.116496),(27.036882,48.107349),(27.034298,48.101768),(27.041739,48.077273),(27.059206,48.05805),(27.083184,48.043658),(27.109332,48.033503),(27.109332,48.026088),(27.088971,48.019809),(27.090005,48.015623),(27.091452,48.011851),(27.093364,48.00844),(27.095793,48.005598),(27.121476,48.013194),(27.134602,48.000456),(27.143748,47.986839),(27.157184,47.991955),(27.16876,47.983067),(27.169793,47.973791),(27.162869,47.965006),(27.150983,47.957797),(27.171964,47.946429),(27.178268,47.944129),(27.178268,47.942096),(27.178268,47.937928),(27.160698,47.921727),(27.17155,47.912632),(27.194185,47.904054),(27.212375,47.889507),(27.211548,47.885011),(27.213202,47.854134),(27.212375,47.847933),(27.222813,47.842662),(27.234906,47.840208),(27.245809,47.83659),(27.253406,47.828089),(27.219196,47.813775),(27.231082,47.807083),(27.244879,47.792588),(27.256196,47.77755),(27.260847,47.769075),(27.264671,47.764553),(27.282448,47.755691),(27.287512,47.752332),(27.288114,47.75068),(27.29113,47.74241),(27.289579,47.731687),(27.295057,47.724427),(27.295057,47.718174),(27.272009,47.71497),(27.281104,47.693007),(27.304049,47.666549),(27.369161,47.60831),(27.397066,47.589086),(27.428382,47.581024),(27.429404,47.57813),(27.431342,47.572638),(27.435824,47.55994),(27.438976,47.553713),(27.440268,47.550096),(27.440061,47.541027),(27.442128,47.536634),(27.446056,47.534877),(27.456908,47.533896),(27.459491,47.533224),(27.466726,47.506352),(27.473134,47.491779),(27.483676,47.485423),(27.492564,47.484545),(27.497525,47.482426),(27.501453,47.479945),(27.50724,47.477982),(27.532975,47.477413),(27.534526,47.477982),(27.548168,47.474261),(27.563154,47.468344),(27.575867,47.460412),(27.582998,47.450671),(27.574782,47.446252),(27.569356,47.438553),(27.565531,47.428269),(27.562534,47.416538),(27.580311,47.405996),(27.572353,47.375197),(27.586409,47.368764),(27.588724,47.367389),(27.599948,47.360728),(27.624029,47.321428),(27.636948,47.306674),(27.6447,47.303987),(27.671727,47.299853),(27.68232,47.295254),(27.689865,47.290887),(27.697823,47.287502),(27.722835,47.283317),(27.733273,47.275617),(27.753634,47.251432),(27.752393,47.238926),(27.763039,47.226317),(27.788309,47.204277),(27.800349,47.176966),(27.802106,47.176346),(27.804793,47.168492),(27.807687,47.162471),(27.805879,47.158234),(27.794665,47.155831),(27.80655,47.144617),(27.849855,47.12852),(27.849855,47.121724),(27.843757,47.114386),(27.848202,47.111415),(27.85771,47.109322),(27.867012,47.104645),(27.897604,47.081417),(27.925923,47.068756),(27.938429,47.061056),(27.938739,47.046638),(27.96313,47.043383),(27.986591,47.033228),(28.008295,47.026304),(28.028036,47.03297),(28.037027,47.016459),(28.038146,47.015483),(28.069067,46.988503),(28.082709,46.971527),(28.102295,46.935121),(28.104994,46.920185),(28.105447,46.91768),(28.096869,46.902616),(28.11356,46.894761),(28.114335,46.883393),(28.113095,46.87143),(28.124257,46.861637),(28.124257,46.854195),(28.122615,46.84227),(28.121518,46.8343),(28.137796,46.80624),(28.178156,46.758646),(28.178104,46.739836),(28.234121,46.662424),(28.24425,46.635372),(28.245085,46.631451),(28.24735,46.620825),(28.247144,46.607156),(28.240529,46.596382),(28.230504,46.583902),(28.225439,46.569562),(28.234121,46.553129),(28.224768,46.548736),(28.221099,46.541191),(28.219032,46.503726),(28.221099,46.495768),(28.234121,46.478069),(28.2381,46.475588),(28.242338,46.476518),(28.2458,46.475511),(28.247144,46.467191),(28.247247,46.436185),(28.246058,46.427865),(28.240219,46.420268),(28.22606,46.415359),(28.233294,46.401768),(28.228643,46.39611),(28.226796,46.395548),(28.219548,46.393345),(28.212934,46.388694),(28.207146,46.358153),(28.202392,46.353916),(28.190351,46.351074),(28.187922,46.343787),(28.191953,46.323504),(28.192883,46.311231),(28.191333,46.307769),(28.177794,46.287047),(28.165753,46.27883),(28.156296,46.275368),(28.144928,46.272939),(28.135212,46.269322),(28.131078,46.262397),(28.134489,46.245344),(28.132112,46.239918),(28.120846,46.237851),(28.108961,46.23413),(28.110976,46.225603),(28.129321,46.204674),(28.135626,46.198835),(28.141517,46.191962),(28.144721,46.183229),(28.133249,46.159974),(28.127358,46.135325),(28.110506,46.101491),(28.107823,46.096103),(28.100796,46.081995),(28.092838,46.078377),(28.090151,46.073365),(28.090461,46.06799),(28.096558,46.065045),(28.096869,46.05967),(28.084466,46.024582),(28.082709,46.014712),(28.08612,46.000552),(28.110511,45.950426),(28.112371,45.939213),(28.114852,45.933218),(28.120846,45.926293),(28.124981,45.918387),(28.12281,45.910791),(28.118779,45.903142),(28.117436,45.895236),(28.120226,45.887846),(28.129115,45.875134),(28.131078,45.871361),(28.128184,45.8664),(28.114955,45.859734),(28.110511,45.854308),(28.113302,45.825369),(28.128908,45.795035),(28.146478,45.771161),(28.154953,45.761807),(28.163738,45.661503),(28.161774,45.645432),(28.167975,45.632461),(28.153713,45.6275),(28.120846,45.627707),(28.107721,45.6244),(28.091184,45.615976),(28.074751,45.604866),(28.062142,45.593601),(28.118004,45.572723),(28.140949,45.560218),(28.157743,45.538927),(28.161567,45.532726),(28.164048,45.530607),(28.165391,45.528282),(28.165908,45.494589),(28.172936,45.484357),(28.199498,45.461774),(28.212934,45.450198),(28.215146,45.450365),(28.237635,45.452059),(28.266884,45.440483),(28.286418,45.421725),(28.281198,45.401829),(28.310809,45.347827),(28.330239,45.322919),(28.353287,45.312429),(28.370237,45.309225),(28.40517,45.295066),(28.494157,45.278891),(28.577046,45.248092),(28.710061,45.226956),(28.747371,45.23047),(28.778687,45.23109),(28.7914,45.234966),(28.802665,45.244165),(28.774295,45.253363),(28.767267,45.257755),(28.762564,45.265145),(28.75967,45.274189),(28.761531,45.281837),(28.79016,45.292069),(28.788609,45.307106),(28.789746,45.321421),(28.816308,45.326072),(28.831811,45.322299),(28.858114,45.30907),(28.880852,45.306176),(28.893616,45.29982),(28.910049,45.287366),(28.929893,45.279098),(28.952837,45.285092),(28.957384,45.292017),(28.960433,45.311292),(28.96648,45.319871),(28.973663,45.323695),(28.982448,45.325503),(29.004617,45.326072),(29.018156,45.330878),(29.069522,45.360798),(29.110295,45.369066),(29.127503,45.374544),(29.141869,45.385086),(29.150138,45.387877),(29.174322,45.388962),(29.179386,45.391856),(29.183107,45.398987),(29.191582,45.40617),(29.206672,45.41542),(29.228169,45.42374),(29.244602,45.424205),(29.260828,45.422035),(29.281809,45.422293),(29.290491,45.424619),(29.299276,45.428908),(29.307027,45.434127),(29.312815,45.439346),(29.322117,45.443842),(29.332969,45.442137),(29.343924,45.438106),(29.353743,45.435936),(29.430637,45.430406),(29.569699,45.394956),(29.616362,45.365346),(29.628145,45.360798),(29.650262,45.346122),(29.667109,45.311861),(29.672276,45.272948),(29.659254,45.244165),(29.66499,45.23786),(29.666488,45.230625),(29.664215,45.223184),(29.659079,45.215887),(29.659075,45.215881),(29.659028,45.215888),(29.650157,45.217231),(29.623871,45.216498),(29.623871,45.210273),(29.638845,45.195624),(29.64975,45.177802),(29.667003,45.164211),(29.699555,45.161811),(29.676524,45.144599),(29.664399,45.116929),(29.656261,45.028957),(29.650727,45.012112),(29.637543,44.98371),(29.634939,44.96955),(29.633637,44.953355),(29.63087,44.938666),(29.623871,44.929104),(29.630382,44.90766),(29.605154,44.880032),(29.617849,44.861477),(29.611176,44.849311),(29.603201,44.845404),(29.593761,44.844306),(29.583507,44.840277),(29.569347,44.824897),(29.561778,44.820258),(29.542003,44.827826),(29.531098,44.826239),(29.520518,44.822496),(29.510997,44.820502),(29.315929,44.798774),(29.192068,44.792873),(29.156098,44.78441),(29.055837,44.735175),(29.026378,44.714667),(29.000743,44.68891),(28.993907,44.696357),(28.994884,44.711127),(28.981293,44.729438),(28.974946,44.74494),(28.997325,44.751614),(29.021495,44.754706),(29.058279,44.768297),(29.098155,44.775377),(29.120616,44.786566),(29.138682,44.804023),(29.144705,44.826646),(29.138845,44.839179),(29.125987,44.852484),(29.096365,44.875067),(29.101817,44.859117),(29.103363,44.851304),(29.10377,44.840277),(29.087901,44.842719),(29.073497,44.835435),(29.060313,44.832343),(29.048595,44.847154),(29.044119,44.865546),(29.041759,44.925686),(29.052094,44.944648),(29.098806,44.956977),(29.110606,44.970038),(29.101736,44.980292),(29.058279,45.000963),(29.045177,45.004828),(29.014415,45.004828),(28.983246,45.012274),(28.97283,45.010972),(28.969249,45.005845),(28.966075,44.996568),(28.960216,44.987698),(28.949229,44.98371),(28.927013,44.982611),(28.91684,44.979682),(28.880056,44.954779),(28.869395,44.943345),(28.863536,44.922309),(28.865082,44.916571),(28.87436,44.911566),(28.876638,44.904853),(28.874685,44.899115),(28.870128,44.89643),(28.865489,44.89468),(28.863536,44.891547),(28.866466,44.882636),(28.873546,44.874986),(28.881521,44.869696),(28.904063,44.862738),(28.952403,44.826646),(28.939708,44.816881),(28.934093,44.814195),(28.924978,44.813056),(28.934418,44.807522),(28.938731,44.80622),(28.938731,44.798774),(28.92628,44.792955),(28.925466,44.783677),(28.932953,44.773505),(28.946137,44.765204),(28.936371,44.759955),(28.923839,44.757799),(28.879405,44.756578),(28.868826,44.753567),(28.808442,44.726874),(28.79835,44.720201),(28.788829,44.706244),(28.78712,44.693305),(28.793712,44.687893),(28.80836,44.696357),(28.80421,44.684231),(28.785411,44.679674),(28.781016,44.665269),(28.784028,44.649237),(28.792491,44.644721),(28.845551,44.651557),(28.869477,44.661811),(28.891368,44.677191),(28.911957,44.696357),(28.897716,44.716783),(28.967459,44.700995),(28.983409,44.692613),(28.987559,44.675279),(28.97934,44.655341),(28.962087,44.63939),(28.938731,44.634263),(28.956391,44.651109),(28.972667,44.670233),(28.975352,44.686835),(28.952403,44.696357),(28.952403,44.68891),(28.959972,44.681952),(28.957774,44.678778),(28.938731,44.675238),(28.924164,44.682074),(28.918224,44.682685),(28.913829,44.679918),(28.909434,44.675198),(28.904552,44.668402),(28.885427,44.658108),(28.854177,44.634345),(28.836274,44.627427),(28.787852,44.609605),(28.774181,44.600775),(28.791515,44.623847),(28.794688,44.634263),(28.779796,44.63581),(28.766938,44.633734),(28.757823,44.626776),(28.753673,44.613756),(28.766775,44.613756),(28.76059,44.603705),(28.764415,44.595893),(28.774425,44.590155),(28.787852,44.586493),(28.748057,44.577786),(28.732677,44.567857),(28.732677,44.552314),(28.737153,44.557074),(28.743907,44.561998),(28.746918,44.565985),(28.761078,44.560207),(28.774181,44.552314),(28.765636,44.538886),(28.753429,44.513495),(28.745616,44.486151),(28.750255,44.466702),(28.758474,44.465277),(28.766368,44.472602),(28.771983,44.48371),(28.778331,44.507392),(28.788097,44.515367),(28.80836,44.525051),(28.838145,44.554023),(28.855642,44.567694),(28.873546,44.573432),(28.892426,44.577053),(28.906261,44.587714),(28.914887,44.601386),(28.924978,44.620551),(28.918468,44.60102),(28.909679,44.584215),(28.844005,44.49372),(28.796723,44.467922),(28.706554,44.381049),(28.6963,44.358344),(28.693126,44.346137),(28.680186,44.353339),(28.664399,44.348375),(28.651215,44.340033),(28.64088,44.328559),(28.635102,44.316392),(28.632335,44.302436),(28.631033,44.26435),(28.635265,44.252021),(28.651215,44.23078),(28.665212,44.199286),(28.668142,44.184312),(28.664887,44.175605),(28.671723,44.168687),(28.658376,44.174465),(28.645844,44.172024),(28.636241,44.164293),(28.63087,44.154486),(28.637462,44.135891),(28.654145,44.054918),(28.671235,44.003323),(28.671723,43.997382),(28.669119,43.989163),(28.660004,43.979804),(28.654959,43.961371),(28.64088,43.932196),(28.637706,43.918647),(28.61378,43.884182),(28.609386,43.874498),(28.593516,43.819525),(28.587657,43.810248),(28.584483,43.800849),(28.589203,43.791327),(28.584809,43.782416),(28.58253,43.772366),(28.581879,43.761705),(28.583018,43.751044),(28.57838,43.741278),(28.434574,43.735213),(28.221254,43.761981),(28.014806,43.830039),(27.98101,43.84934),(27.935845,43.964398),(27.912074,43.993233),(27.912074,43.993336),(27.85647,43.988634),(27.787327,43.960419),(27.721698,43.94874),(27.682515,43.987256),(27.676119,43.993543),(27.656275,44.023877),(27.633434,44.029768),(27.574523,44.016281),(27.383837,44.015092),(27.372882,44.020725),(27.353555,44.045271),(27.341669,44.053074),(27.285342,44.072453),(27.26431,44.089765),(27.269012,44.112399),(27.252662,44.121519),(27.251132,44.122373),(27.226741,44.120719),(27.205553,44.129246),(27.100237,44.14449),(27.027476,44.177046),(27.001535,44.165109),(26.884126,44.156531),(26.789455,44.115965),(26.753695,44.10811),(26.708685,44.10811),(26.697212,44.106094),(26.677317,44.097051),(26.667808,44.095087),(26.647758,44.093382),(26.614168,44.084855),(26.415791,44.063789),(26.332335,44.054926),(26.310518,44.052609),(26.231453,44.027495),(26.150734,44.012405),(26.116214,43.998866),(26.079317,43.969049),(26.061644,43.949773),(26.054306,43.934322),(25.934003,43.870321),(25.924495,43.858616),(25.916433,43.844379),(25.869304,43.800893),(25.839332,43.788439),(25.806259,43.763661),(25.804399,43.759914),(25.781144,43.732009),(25.739596,43.718935),(25.732948,43.718781),(25.671384,43.717359),(25.653503,43.708134),(25.637897,43.697282),(25.616503,43.687748),(25.593869,43.680668),(25.575059,43.677387),(25.556558,43.670359),(25.533821,43.668679),(25.488759,43.67054),(25.48245,43.669715),(25.467417,43.667749),(25.426075,43.654391),(25.403131,43.65005),(25.35962,43.654287),(25.323033,43.669713),(25.28872,43.688962),(25.285405,43.690391),(25.252339,43.704646),(25.211308,43.711881),(25.0816,43.718935),(24.963675,43.749605),(24.752628,43.738804),(24.705603,43.743765),(24.661763,43.755657),(24.500137,43.799498),(24.466341,43.802418),(24.431201,43.794176),(24.375494,43.763867),(24.358234,43.760017),(24.336705,43.759251),(24.159383,43.752938),(24.149606,43.75472),(23.799922,43.818463),(23.742871,43.8427),(23.720753,43.845826),(23.636107,43.832158),(23.620854,43.83401),(23.592699,43.837429),(23.484695,43.880604),(23.325151,43.886592),(23.234478,43.877297),(23.196961,43.86275),(23.161924,43.857324),(23.131849,43.847945),(23.052551,43.84282),(22.919562,43.834225),(22.888763,43.839522),(22.863441,43.855412),(22.851039,43.874352),(22.850522,43.896986),(22.874707,43.972046),(22.885869,43.994525),(22.905816,44.003982),(22.926486,44.006152),(22.966277,44.015557),(22.988085,44.017676),(23.023018,44.031629),(23.040071,44.062325),(23.030976,44.093072),(23.008307,44.100446),(22.988085,44.107025),(22.942609,44.111469),(22.906436,44.122889),(22.69164,44.228435),(22.690739,44.228878),(22.685364,44.243657),(22.689498,44.291716),(22.68154,44.305307),(22.662523,44.311663),(22.621182,44.315901),(22.582838,44.328406),(22.549352,44.348974),(22.522687,44.37507),(22.505117,44.404061),(22.50398,44.411709),(22.506357,44.427522),(22.505427,44.435015),(22.500983,44.44194),(22.480002,44.455789),(22.477005,44.463954),(22.477212,44.476976),(22.479795,44.490412),(22.484136,44.499766),(22.491061,44.50421),(22.500363,44.50638),(22.535503,44.507517),(22.548422,44.511703),(22.55669,44.52147),(22.55979,44.538781),(22.565578,44.555421),(22.580461,44.565498),(22.600305,44.569787),(22.621182,44.569219),(22.642163,44.563121),(22.678543,44.545551),(22.700144,44.54183),(22.719367,44.544362),(22.741692,44.551855),(22.759262,44.564671),(22.765153,44.58281),(22.714716,44.623065),(22.700144,44.63061),(22.621182,44.637432),(22.587489,44.649369),(22.553279,44.669161),(22.483723,44.72399),(22.468943,44.730191),(22.450547,44.732981),(22.426052,44.733653),(22.41551,44.727814),(22.38068,44.700528),(22.361146,44.692054),(22.319702,44.685336),(22.304922,44.677377),(22.299031,44.661668),(22.185136,44.515113),(22.172114,44.505347),(22.148239,44.500902),(22.126639,44.502659),(22.104314,44.509636),(22.086951,44.52178),(22.076306,44.550718),(22.067107,44.55723),(22.055739,44.562139),(22.045197,44.569219),(22.039822,44.576867),(22.036174,44.589489),(22.034243,44.596168),(22.032174,44.603325),(22.004269,44.651539),(21.994347,44.658567),(21.962411,44.662288),(21.871977,44.695981),(21.855441,44.698461),(21.838284,44.695206),(21.801697,44.683889),(21.756842,44.677274),(21.705166,44.677067),(21.65628,44.687661),(21.619693,44.713913),(21.610288,44.731948),(21.604293,44.749983),(21.595612,44.765951),(21.578042,44.777681),(21.558405,44.78166),(21.49722,44.778043),(21.41216,44.784813),(21.395934,44.790239),(21.378571,44.816645),(21.360587,44.826412),(21.359841,44.826644),(21.3598,44.826657),(21.342811,44.831942),(21.346428,44.845636),(21.355523,44.856591),(21.368545,44.86486),(21.395314,44.871629),(21.453398,44.869562),(21.48151,44.872611),(21.522128,44.880776),(21.536287,44.889302),(21.539078,44.908474),(21.531016,44.924649),(21.51634,44.933899),(21.48151,44.943563),(21.456499,44.952348),(21.40844,44.958342),(21.385185,44.969504),(21.384565,44.97493),(21.387046,44.981545),(21.383945,44.986661),(21.366995,44.987281),(21.353456,44.989813),(21.351389,44.998236),(21.356143,45.008572),(21.363378,45.01653),(21.3733,45.020096),(21.398828,45.021387),(21.409266,45.023971),(21.421875,45.031413),(21.425079,45.036219),(21.425596,45.043247),(21.429524,45.057303),(21.432521,45.061488),(21.440996,45.068723),(21.443786,45.072909),(21.444406,45.07787),(21.444406,45.08526),(21.443786,45.091461),(21.442443,45.092908),(21.449987,45.101021),(21.458566,45.107325),(21.469004,45.111046),(21.48151,45.111563),(21.494119,45.119314),(21.49784,45.131872),(21.493292,45.145101),(21.484388,45.152702),(21.459393,45.17404),(21.433761,45.188819),(21.405546,45.199671),(21.299299,45.223184),(21.257131,45.224114),(21.239251,45.229385),(21.206075,45.245922),(21.189745,45.259564),(21.155638,45.295169),(21.139412,45.303696),(21.12918,45.301887),(21.113057,45.28933),(21.103342,45.286074),(21.09156,45.288089),(21.082775,45.293671),(21.0743,45.300544),(21.063964,45.30628),(21.016668,45.321482),(20.981489,45.33279),(20.966193,45.341575),(20.927642,45.37749),(20.863047,45.418728),(20.830387,45.452524),(20.816021,45.462859),(20.799484,45.468544),(20.781604,45.472574),(20.767135,45.479344),(20.760727,45.493348),(20.783155,45.506061),(20.797624,45.516499),(20.800208,45.530504),(20.787392,45.553655),(20.75804,45.58926),(20.754423,45.605589),(20.762174,45.630601),(20.773233,45.648894),(20.777264,45.657524),(20.779951,45.671684),(20.779124,45.72367),(20.781604,45.733954),(20.785532,45.743411),(20.785739,45.752557),(20.77747,45.762324),(20.765171,45.766768),(20.754113,45.763564),(20.745328,45.754986),(20.739436,45.743411),(20.726827,45.736176),(20.713392,45.733334),(20.700059,45.735401),(20.68807,45.7431),(20.678562,45.75664),(20.655617,45.77731),(20.645902,45.788731),(20.64349,45.795141),(20.642285,45.798343),(20.640114,45.818703),(20.636704,45.827023),(20.629676,45.833276),(20.612312,45.841492),(20.605284,45.846143),(20.572108,45.887691),(20.556709,45.89844),(20.538002,45.903711),(20.499865,45.906656),(20.481674,45.912703),(20.429068,45.946706),(20.410258,45.955594),(20.370984,45.96779),(20.35393,45.976678),(20.338634,45.992801),(20.317653,46.038586),(20.305665,46.053572),(20.242826,46.108091),(20.283237,46.1438),(20.444157,46.1469),(20.468549,46.174134),(20.509373,46.167674),(20.548957,46.15615),(20.578103,46.137547),(20.588025,46.132844),(20.600117,46.12964),(20.607558,46.129485),(20.663989,46.137753),(20.683523,46.144678),(20.698819,46.15646),(20.704193,46.16633),(20.704368,46.168526),(20.70533,46.180645),(20.710291,46.188086),(20.717836,46.189998),(20.727137,46.187725),(20.736646,46.186691),(20.744707,46.192272),(20.744811,46.200282),(20.735509,46.222503),(20.734992,46.231908),(20.739333,46.237489),(20.778504,46.260072),(20.787369,46.26339),(20.798658,46.267616),(20.819638,46.271699),(20.839689,46.271079),(20.848991,46.267771),(20.866974,46.257126),(20.875139,46.254387),(20.885061,46.255007),(20.899737,46.260692),(20.906558,46.26219),(20.924438,46.259555),(20.961748,46.248341),(20.981489,46.248858),(20.990584,46.251597),(20.999162,46.251597),(21.007017,46.248858),(21.013942,46.24307),(21.033475,46.231339),(21.051459,46.236094),(21.099208,46.27635),(21.105616,46.278675),(21.134865,46.27852),(21.144476,46.283739),(21.155949,46.298932),(21.164527,46.318259),(21.168661,46.362753),(21.17879,46.384457),(21.195636,46.398099),(21.21517,46.402905),(21.257544,46.404171),(21.280695,46.416393),(21.274391,46.438355),(21.245142,46.47688),(21.247623,46.497473),(21.261679,46.513338),(21.278938,46.528401),(21.291341,46.546721),(21.295268,46.585039),(21.300953,46.603926),(21.316249,46.616639),(21.337643,46.620411),(21.374436,46.618448),(21.396037,46.626354),(21.416811,46.645216),(21.423539,46.658231),(21.425493,46.662011),(21.436241,46.673741),(21.463527,46.677049),(21.483577,46.684852),(21.501767,46.703507),(21.505178,46.723247),(21.478203,46.735934),(21.475206,46.737846),(21.472622,46.740197),(21.470451,46.743014),(21.471381,46.749034),(21.473655,46.754873),(21.477066,46.760222),(21.48151,46.764976),(21.502904,46.80531),(21.51572,46.821588),(21.536597,46.835023),(21.573081,46.841767),(21.583416,46.847943),(21.591788,46.860707),(21.591477,46.871533),(21.588067,46.882204),(21.587137,46.8944),(21.588687,46.906182),(21.589927,46.908869),(21.594371,46.910006),(21.640053,46.935689),(21.648528,46.942976),(21.667545,46.992378),(21.671369,46.99336),(21.670749,46.994394),(21.661757,47.006072),(21.654833,47.009948),(21.645221,47.01124),(21.636643,47.014082),(21.63427,47.019422),(21.632819,47.022686),(21.671886,47.054726),(21.69421,47.069169),(21.74351,47.091597),(21.763767,47.105214),(21.770485,47.114025),(21.775446,47.131672),(21.77989,47.140741),(21.789398,47.150301),(21.811516,47.164693),(21.819887,47.172729),(21.825985,47.185054),(21.825675,47.194278),(21.823298,47.203321),(21.823608,47.214923),(21.827742,47.226007),(21.839318,47.240839),(21.844589,47.249934),(21.856061,47.285745),(21.862159,47.297424),(21.900813,47.335742),(21.919003,47.349669),(21.936986,47.357162),(21.981531,47.366102),(22.001582,47.393827),(22.000238,47.427184),(21.991453,47.461807),(21.988869,47.492942),(22.007886,47.517411),(22.037445,47.539322),(22.09956,47.570948),(22.148549,47.579345),(22.154177,47.582213),(22.162089,47.586244),(22.16736,47.593789),(22.167608,47.594862),(22.169117,47.601385),(22.16984,47.608775),(22.172837,47.615389),(22.197642,47.639315),(22.200743,47.647945),(22.204153,47.666291),(22.207874,47.673732),(22.215212,47.679933),(22.232162,47.688253),(22.23981,47.693472),(22.261721,47.715848),(22.27309,47.723755),(22.29159,47.730705),(22.309366,47.735046),(22.322079,47.735873),(22.368278,47.73117),(22.382644,47.732023),(22.395873,47.735692),(22.407345,47.743082),(22.423675,47.782563),(22.454371,47.787394),(22.528475,47.761039),(22.562994,47.757215),(22.601235,47.760936),(22.637719,47.771555),(22.666967,47.78879),(22.691669,47.8107),(22.703864,47.817186),(22.724225,47.82349),(22.745826,47.824989),(22.75182,47.827676),(22.760295,47.838916),(22.758331,47.846228),(22.753267,47.852687),(22.752854,47.86124),(22.763602,47.874753),(22.779622,47.882298),(22.819723,47.892323),(22.836053,47.902452),(22.861167,47.933819),(22.877601,47.946739),(22.897858,47.950976),(22.915841,47.959193),(22.924213,47.972732),(22.915738,47.992989),(22.924109,48.004823),(22.939302,48.00565),(22.957079,48.00012),(22.972478,47.992886),(22.988188,47.986374),(23.004415,47.983067),(23.020331,47.984721),(23.063119,48.007458),(23.076555,48.024512),(23.098982,48.071227),(23.118619,48.091536),(23.139083,48.098125),(23.161718,48.095903),(23.231377,48.079728),(23.248637,48.071227),(23.289772,48.038206),(23.337521,48.010869),(23.360052,47.993144),(23.367286,47.991206),(23.374728,47.99056),(23.382169,47.991129),(23.390127,47.993067),(23.391574,47.993454),(23.393021,47.993583),(23.394468,47.993454),(23.396019,47.993067),(23.460821,47.971337),(23.481595,47.972163),(23.485476,47.974234),(23.488519,47.975858),(23.494307,47.980535),(23.499371,47.986219),(23.503712,47.992886),(23.514461,47.998983),(23.525623,48.00118),(23.563243,48.005753),(23.581744,48.001567),(23.646132,47.996606),(23.687267,47.987253),(23.710521,47.985083),(23.780078,47.987511),(23.796511,47.981982),(23.848394,47.949658),(23.855215,47.934233),(23.876609,47.93444),(23.977275,47.962293),(24.008798,47.961208),(24.025231,47.95325),(24.074943,47.944181),(24.094787,47.937928),(24.131167,47.914544),(24.148737,47.912115),(24.150184,47.912916),(24.209302,47.897594),(24.231006,47.897026),(24.297876,47.919789),(24.347175,47.920875),(24.386242,47.94369),(24.409187,47.952061),(24.428514,47.952526),(24.485564,47.943199),(24.542098,47.943819),(24.561632,47.940537),(24.569808,47.937289),(24.608451,47.921934),(24.633669,47.908291),(24.649275,47.895217),(24.655993,47.879301),(24.656923,47.866201),(24.661471,47.853876),(24.679144,47.84013),(24.712837,47.825919),(24.793659,47.804473),(24.808232,47.79574),(24.820738,47.784087),(24.854431,47.743133),(24.877789,47.718742),(24.896599,47.710061),(24.928858,47.713938),(24.94238,47.715563),(25.017418,47.724582),(25.079947,47.742927),(25.08005,47.743082),(25.080257,47.743133),(25.121908,47.770315),(25.218956,47.878474),(25.261744,47.898576),(25.752516,47.934595),(25.818351,47.952578),(25.870958,47.957022),(25.90124,47.965962),(25.918087,47.968158),(25.965732,47.964903),(26.029294,47.977796),(26.10253,47.978267),(26.125723,47.978416),(26.173058,47.993144),(26.18267,48.003273),(26.204788,48.037379),(26.217087,48.048024),(26.243235,48.062907),(26.255327,48.072338),(26.270727,48.093603),(26.286953,48.124712),(26.297495,48.156338),(26.296152,48.178998),(26.30349,48.212045),(26.311758,48.20972),(26.330258,48.208506),(26.347311,48.212097),(26.380488,48.223207),(26.397437,48.226205),(26.44467,48.227729),(26.461309,48.230391),(26.5874,48.249356),(26.617889,48.258968),(26.665741,48.274212),(26.688531,48.274832),(26.711475,48.261319),(26.722379,48.259769)] +Rwanda [(30.471786,-1.066837),(30.463856,-1.075127),(30.456156,-1.086082),(30.453004,-1.097348),(30.456311,-1.108097),(30.470884,-1.118122),(30.474191,-1.131764),(30.472072,-1.137656),(30.4683,-1.14334),(30.465613,-1.149334),(30.467266,-1.155329),(30.476051,-1.16122),(30.48401,-1.15998),(30.490521,-1.156466),(30.494655,-1.155329),(30.506954,-1.164217),(30.511191,-1.170418),(30.515119,-1.196257),(30.521217,-1.210829),(30.539407,-1.241008),(30.545298,-1.261369),(30.5546,-1.273461),(30.556615,-1.281626),(30.555633,-1.284727),(30.553359,-1.289067),(30.550982,-1.294959),(30.549949,-1.3024),(30.555323,-1.31842),(30.568242,-1.328135),(30.597698,-1.340331),(30.60824,-1.347772),(30.623226,-1.362035),(30.632424,-1.367616),(30.698467,-1.39211),(30.718104,-1.394901),(30.737534,-1.406683),(30.743219,-1.432831),(30.741358,-1.458876),(30.738258,-1.470658),(30.732935,-1.476446),(30.738878,-1.489469),(30.755414,-1.511586),(30.767817,-1.524815),(30.772261,-1.532463),(30.781976,-1.56843),(30.791588,-1.590961),(30.807246,-1.60326),(30.831017,-1.594165),(30.838303,-1.615352),(30.837476,-1.641087),(30.824557,-1.719946),(30.824247,-1.730694),(30.826521,-1.735862),(30.835409,-1.749504),(30.83789,-1.758703),(30.826418,-1.786195),(30.829932,-1.796737),(30.83789,-1.836838),(30.832412,-1.853788),(30.822232,-1.868774),(30.816599,-1.884173),(30.824247,-1.902053),(30.808021,-1.914766),(30.801923,-1.921277),(30.796962,-1.929338),(30.826934,-1.934093),(30.829932,-1.960551),(30.816754,-2.018739),(30.835461,-2.014708),(30.853496,-2.0237),(30.868741,-2.038996),(30.879438,-2.053465),(30.887809,-2.082507),(30.853393,-2.193818),(30.844711,-2.237847),(30.848949,-2.306266),(30.844711,-2.326627),(30.834376,-2.345334),(30.821353,-2.354739),(30.804559,-2.36218),(30.789107,-2.371069),(30.775155,-2.374479),(30.76792,-2.378613),(30.758825,-2.381094),(30.75066,-2.37913),(30.698467,-2.353395),(30.687718,-2.349985),(30.674799,-2.351742),(30.663327,-2.360733),(30.649064,-2.387605),(30.63785,-2.39701),(30.616921,-2.398147),(30.595217,-2.391946),(30.573927,-2.389259),(30.5546,-2.400628),(30.521733,-2.399387),(30.488454,-2.383781),(30.434142,-2.339236),(30.428406,-2.331381),(30.423238,-2.317325),(30.418484,-2.311847),(30.415073,-2.313088),(30.383447,-2.305543),(30.37859,-2.303062),(30.362518,-2.307817),(30.352752,-2.316912),(30.34861,-2.322352),(30.344018,-2.328384),(30.331151,-2.340269),(30.316785,-2.347608),(30.297974,-2.353705),(30.278854,-2.357633),(30.262731,-2.358356),(30.250329,-2.355566),(30.219116,-2.340373),(30.207489,-2.339236),(30.203923,-2.345954),(30.203407,-2.355359),(30.20134,-2.362387),(30.156174,-2.419024),(30.139121,-2.430807),(30.116797,-2.43153),(30.093853,-2.422642),(30.072975,-2.408792),(30.056542,-2.394426),(30.025536,-2.357736),(30.013341,-2.348331),(30.00192,-2.3443),(29.992308,-2.34399),(29.983885,-2.344817),(29.976134,-2.34368),(29.95852,-2.328533),(29.956781,-2.327038),(29.949934,-2.321149),(29.941924,-2.316602),(29.931692,-2.316912),(29.928798,-2.322493),(29.928901,-2.331795),(29.92239,-2.382954),(29.929573,-2.459849),(29.907921,-2.5354),(29.903726,-2.637947),(29.90327,-2.649088),(29.897896,-2.671102),(29.888387,-2.692909),(29.876191,-2.71358),(29.862859,-2.731667),(29.842395,-2.752441),(29.823585,-2.763499),(29.803379,-2.767324),(29.778523,-2.7666),(29.756199,-2.759882),(29.744727,-2.759985),(29.737285,-2.769287),(29.733978,-2.790268),(29.729637,-2.79957),(29.719922,-2.805461),(29.697546,-2.808251),(29.678684,-2.805357),(29.637087,-2.791629),(29.626388,-2.788097),(29.619256,-2.787684),(29.589491,-2.798433),(29.582049,-2.80298),(29.574091,-2.806081),(29.563756,-2.805667),(29.541845,-2.808561),(29.523138,-2.820447),(29.504121,-2.826855),(29.480867,-2.813729),(29.444675,-2.806989),(29.425366,-2.803394),(29.407435,-2.805461),(29.364285,-2.824168),(29.339687,-2.826441),(29.33085,-2.807424),(29.329661,-2.794919),(29.32036,-2.774145),(29.318086,-2.762569),(29.321497,-2.752131),(29.335346,-2.735181),(29.337516,-2.723812),(29.331418,-2.71141),(29.309714,-2.691049),(29.309481,-2.690127),(29.306045,-2.67658),(29.307337,-2.664797),(29.306614,-2.660147),(29.294418,-2.651052),(29.276125,-2.640613),(29.254989,-2.635549),(29.212769,-2.630278),(29.190445,-2.623456),(29.129725,-2.596998),(29.113447,-2.594621),(29.05526,-2.598445),(29.032574,-2.620666),(29.015365,-2.720711),(29.000121,-2.703658),(28.979605,-2.69353),(28.949736,-2.690326),(28.936404,-2.685055),(28.899093,-2.660663),(28.891342,-2.652705),(28.89806,-2.576327),(28.891342,-2.553073),(28.881058,-2.542634),(28.869845,-2.53726),(28.860956,-2.531162),(28.857236,-2.518346),(28.860853,-2.505221),(28.878216,-2.489718),(28.874702,-2.477522),(28.864574,-2.459435),(28.860853,-2.439592),(28.858838,-2.418198),(28.866331,-2.391946),(28.880387,-2.379854),(28.907258,-2.370552),(28.928136,-2.364454),(28.945447,-2.354222),(28.959658,-2.339753),(28.971234,-2.321666),(28.978779,-2.306783),(28.982189,-2.296034),(28.983378,-2.287353),(28.993661,-2.277741),(29.002136,-2.274744),(29.03092,-2.275674),(29.075413,-2.263892),(29.10983,-2.23671),(29.134841,-2.197746),(29.150551,-2.15041),(29.156339,-2.108656),(29.154272,-2.072689),(29.12771,-1.915696),(29.12554,-1.879109),(29.130501,-1.843349),(29.149311,-1.799321),(29.18664,-1.739618),(29.212253,-1.698655),(29.233337,-1.658657),(29.245429,-1.640984),(29.293695,-1.600986),(29.303823,-1.587861),(29.331212,-1.53143),(29.342787,-1.516547),(29.358497,-1.509933),(29.434926,-1.506832),(29.437407,-1.506212),(29.439629,-1.504765),(29.441696,-1.502595),(29.445065,-1.497287),(29.46433,-1.466938),(29.498747,-1.431074),(29.538641,-1.402342),(29.577915,-1.38839),(29.618119,-1.39056),(29.6391,-1.38901),(29.657703,-1.383945),(29.678322,-1.37237),(29.693774,-1.361208),(29.710517,-1.352526),(29.734805,-1.348185),(29.74669,-1.350872),(29.767981,-1.363792),(29.774906,-1.366272),(29.783174,-1.361414),(29.789168,-1.341674),(29.798212,-1.330925),(29.807462,-1.325138),(29.816143,-1.322554),(29.825024,-1.323881),(29.825135,-1.323897),(29.836091,-1.329478),(29.864203,-1.370303),(29.868647,-1.391283),(29.871024,-1.432418),(29.880739,-1.453605),(29.897896,-1.469625),(29.917429,-1.475206),(29.938462,-1.472932),(29.960424,-1.464767),(30.028327,-1.427147),(30.038662,-1.424977),(30.047757,-1.403169),(30.06078,-1.389733),(30.065984,-1.386945),(30.095506,-1.37113),(30.136331,-1.355213),(30.147183,-1.345085),(30.15235,-1.329892),(30.158448,-1.291135),(30.165579,-1.277492),(30.173434,-1.272841),(30.181392,-1.271497),(30.189351,-1.270877),(30.19674,-1.268707),(30.212192,-1.259509),(30.256685,-1.217237),(30.269759,-1.200494),(30.280508,-1.182407),(30.28242,-1.175793),(30.284642,-1.161427),(30.287536,-1.155432),(30.29095,-1.152621),(30.294564,-1.149644),(30.311307,-1.1421),(30.317405,-1.137035),(30.322572,-1.121843),(30.329032,-1.080501),(30.337455,-1.066239),(30.352752,-1.060761),(30.369288,-1.063241),(30.386341,-1.068202),(30.403188,-1.070373),(30.418897,-1.066445),(30.432023,-1.060554),(30.445614,-1.058694),(30.460829,-1.063428),(30.471786,-1.066837)] +Western Sahara [(-8.752562,27.661439),(-8.682385,27.661439),(-8.682385,27.567414),(-8.682385,27.473466),(-8.682385,27.379467),(-8.682385,27.285416),(-8.682282,27.27167),(-8.68223,27.232706),(-8.68223,27.172141),(-8.682075,27.093696),(-8.68192,27.000834),(-8.681817,26.897171),(-8.681662,26.786325),(-8.681558,26.671913),(-8.681455,26.55745),(-8.6813,26.446604),(-8.681145,26.342993),(-8.681042,26.250182),(-8.680938,26.171634),(-8.680861,26.111121),(-8.680861,26.072208),(-8.680809,26.058411),(-8.680809,26.013142),(-8.687656,26.000016),(-8.707216,25.995004),(-8.792043,25.995004),(-8.892863,25.994952),(-8.993581,25.994952),(-9.094246,25.994952),(-9.195015,25.994952),(-9.295733,25.994952),(-9.39645,25.994952),(-9.497168,25.994952),(-9.597885,25.994952),(-9.698654,25.994952),(-9.799371,25.994952),(-9.900037,25.994952),(-10.000806,25.994952),(-10.101523,25.994952),(-10.202292,25.994952),(-10.302958,25.994952),(-10.403675,25.994952),(-10.504445,25.994952),(-10.605162,25.994952),(-10.705879,25.994952),(-10.806597,25.994952),(-10.907314,25.994952),(-11.008083,25.994952),(-11.108749,25.994952),(-11.209518,25.994952),(-11.310235,25.994952),(-11.410953,25.994952),(-11.511722,25.994952),(-11.612439,25.994952),(-11.713208,25.994952),(-11.813822,25.994952),(-11.914591,25.994952),(-12.015308,25.9949),(-12.015308,25.919246),(-12.015308,25.843695),(-12.015308,25.768092),(-12.015308,25.69249),(-12.015308,25.616939),(-12.015308,25.541336),(-12.015308,25.465734),(-12.015308,25.390183),(-12.015308,25.31458),(-12.015308,25.238926),(-12.015308,25.163323),(-12.015308,25.087798),(-12.015308,25.012247),(-12.015308,24.936619),(-12.015308,24.861016),(-12.015308,24.785465),(-12.015308,24.709862),(-12.015308,24.63426),(-12.015308,24.558631),(-12.015308,24.48308),(-12.015308,24.407478),(-12.015308,24.331901),(-12.015308,24.256298),(-12.015308,24.180696),(-12.015308,24.105093),(-12.015308,24.029542),(-12.015308,23.95394),(-12.015308,23.878311),(-12.015308,23.802734),(-12.015308,23.727106),(-12.015308,23.651477),(-12.015308,23.575926),(-12.015308,23.495182),(-12.019339,23.460998),(-12.032723,23.444978),(-12.134578,23.419347),(-12.353892,23.322479),(-12.389911,23.312532),(-12.558376,23.290285),(-12.619432,23.270829),(-12.682219,23.230754),(-12.789938,23.161869),(-12.915098,23.082055),(-13.015247,23.018002),(-13.034342,22.995213),(-13.034342,22.995161),(-13.119892,22.88354),(-13.152293,22.820004),(-13.165497,22.752695),(-13.15498,22.688797),(-13.106353,22.560226),(-13.09333,22.495476),(-13.076833,22.245439),(-13.060335,21.995403),(-13.032093,21.581939),(-13.026177,21.49533),(-13.015247,21.333428),(-13.161724,21.333428),(-13.203091,21.333376),(-13.319854,21.333376),(-13.501419,21.333376),(-13.736909,21.333324),(-14.015496,21.333273),(-14.295598,21.333226),(-14.326485,21.333221),(-14.658971,21.333169),(-15.002258,21.333169),(-15.345544,21.333066),(-15.678082,21.333066),(-15.989019,21.333014),(-16.26771,21.332911),(-16.503148,21.332911),(-16.684636,21.332911),(-16.801425,21.332859),(-16.842817,21.332859),(-16.958831,21.332859),(-16.968339,21.324591),(-16.985703,21.237826),(-17.004926,21.141915),(-17.047378,21.0274),(-17.054019,20.995464),(-17.077428,20.904513),(-17.081175,20.874748),(-17.056874,20.766913),(-17.068349,20.792467),(-17.100209,20.837592),(-17.104644,20.862494),(-17.101389,20.876166),(-17.092275,20.899888),(-17.090403,20.914008),(-17.090403,20.961819),(-17.086293,20.979315),(-17.06786,21.030911),(-17.026763,21.328925),(-17.017445,21.365058),(-17.015248,21.379828),(-17.013743,21.419971),(-16.950149,21.429753),(-16.729956,21.469802),(-16.580043,21.48055),(-16.189886,21.48055),(-16.040024,21.500084),(-15.919876,21.500084),(-15.749964,21.490317),(-15.609818,21.469802),(-15.45993,21.450268),(-15.289992,21.450268),(-15.149872,21.440501),(-14.970167,21.440501),(-14.839813,21.450268),(-14.749974,21.500084),(-14.669875,21.599665),(-14.640109,21.679763),(-14.609827,21.750069),(-14.620085,21.820375),(-14.629852,21.860424),(-14.580036,21.91024),(-14.519988,21.990338),(-14.459914,22.040103),(-14.439915,22.080152),(-14.379841,22.120201),(-14.310052,22.190507),(-14.270003,22.240297),(-14.220187,22.309647),(-14.209955,22.370186),(-14.189904,22.450259),(-14.189904,22.589914),(-14.169906,22.759852),(-14.140088,22.870181),(-14.120089,22.960047),(-14.100065,23.099676),(-14.039991,23.33992),(-14.019992,23.410252),(-13.979943,23.519599),(-13.930127,23.620187),(-13.890129,23.690493),(-13.839797,23.750076),(-13.769982,23.790125),(-13.660118,23.830123),(-13.580045,23.870198),(-13.479948,23.910221),(-13.390108,23.940504),(-13.31001,23.980553),(-13.279753,24.019594),(-13.229963,24.0899),(-13.160122,24.219815),(-13.120099,24.299862),(-13.060025,24.400476),(-12.990184,24.4698),(-12.94688,24.496748),(-12.946879,24.496749),(-12.910137,24.51959),(-12.819781,24.570388),(-12.709943,24.629971),(-12.629845,24.679761),(-12.56003,24.730533),(-12.499982,24.7696),(-12.430141,24.830165),(-12.399884,24.879955),(-12.359835,24.969795),(-12.310019,25.110432),(-12.26997,25.259803),(-12.229946,25.42),(-12.200155,25.51958),(-12.169873,25.639728),(-12.129849,25.730549),(-12.100058,25.830156),(-12.080059,25.870205),(-12.080059,25.919969),(-12.060009,25.990301),(-12.055823,25.99583),(-12.055823,25.99583),(-12.029752,26.03035),(-11.959911,26.049884),(-11.879864,26.070399),(-11.753877,26.086006),(-11.717239,26.103576),(-11.698222,26.162177),(-11.683546,26.212975),(-11.63621,26.294985),(-11.582983,26.360408),(-11.55221,26.400457),(-11.510688,26.469807),(-11.469709,26.519597),(-11.398912,26.583081),(-11.3369,26.632897),(-11.315868,26.683644),(-11.315868,26.744208),(-11.36031,26.793043),(-11.391574,26.882908),(-11.262641,26.910219),(-11.149366,26.940501),(-11.045859,26.969802),(-10.921835,27.009825),(-10.829076,27.009825),(-10.756781,27.019592),(-10.653273,27.000058),(-10.550282,26.990292),(-10.477986,26.960035),(-10.353963,26.900478),(-10.250455,26.860429),(-10.188443,26.860429),(-10.122039,26.879962),(-10.065867,26.908281),(-10.031709,26.910219),(-9.979929,26.889729),(-9.899365,26.84968),(-9.816864,26.84968),(-9.734362,26.860429),(-9.672351,26.910219),(-9.568843,26.990292),(-9.486315,27.049875),(-9.412056,27.08796),(-9.352008,27.097727),(-9.284622,27.097727),(-9.207469,27.099691),(-9.083446,27.089924),(-9.000919,27.089924),(-8.888109,27.103566),(-8.793903,27.12018),(-8.752872,27.150463),(-8.752872,27.190486),(-8.773387,27.250069),(-8.795841,27.307688),(-8.801706,27.360424),(-8.788012,27.416054),(-8.773387,27.46003),(-8.783619,27.530362),(-8.81292,27.613354),(-8.818449,27.659398),(-8.817035,27.661464),(-8.817034,27.661465),(-8.816537,27.661465),(-8.752562,27.661439)] +Scarborough Reef [(117.753887,15.154369),(117.755692,15.151887),(117.753887,15.150082),(117.751856,15.151887),(117.753887,15.154369)] +South Sudan [(33.96912,9.838341),(33.904886,9.7107),(33.883905,9.619853),(33.877704,9.543424),(33.881425,9.499215),(33.888711,9.470818),(33.892019,9.464281),(33.894809,9.459475),(33.897755,9.456582),(33.900855,9.45529),(34.070698,9.454592),(34.083096,9.205117),(34.097473,8.915836),(34.11185,8.626555),(34.107251,8.601531),(34.103169,8.579322),(34.090249,8.556507),(34.069579,8.533615),(33.970774,8.445377),(33.95031,8.433207),(33.931706,8.428143),(33.875172,8.427936),(33.840859,8.423983),(33.824116,8.419487),(33.808096,8.412614),(33.797658,8.405612),(33.756213,8.370136),(33.751872,8.368793),(33.74102,8.367087),(33.695545,8.372875),(33.671309,8.400289),(33.651672,8.434499),(33.620407,8.460699),(33.600099,8.463955),(33.585371,8.458115),(33.57116,8.450131),(33.552143,8.446824),(33.537053,8.453206),(33.521964,8.465324),(33.505117,8.474884),(33.484757,8.474006),(33.490338,8.466513),(33.489924,8.462921),(33.480519,8.45765),(33.413133,8.449718),(33.400162,8.452638),(33.392101,8.451862),(33.384608,8.448013),(33.370862,8.437393),(33.361353,8.434293),(33.25273,8.458167),(33.23516,8.455635),(33.206427,8.430572),(33.207358,8.426593),(33.210355,8.420779),(33.202913,8.414113),(33.187824,8.411038),(33.174078,8.404475),(33.16953,8.397421),(33.167877,8.389722),(33.16798,8.372823),(33.166843,8.368276),(33.161882,8.361506),(33.161159,8.356106),(33.163329,8.352954),(33.172321,8.34652),(33.174078,8.343058),(33.171804,8.330888),(33.163123,8.312052),(33.161159,8.304585),(33.164363,8.292777),(33.171391,8.283113),(33.178419,8.275672),(33.185757,8.26451),(33.194852,8.259239),(33.208288,8.253683),(33.206893,8.240738),(33.199865,8.235519),(33.190511,8.232419),(33.181623,8.225726),(33.173975,8.21459),(33.169944,8.205495),(33.16829,8.195392),(33.16798,8.181362),(33.171081,8.175962),(33.184723,8.166247),(33.187824,8.160898),(33.185137,8.14108),(33.177282,8.127102),(33.164776,8.117051),(33.147516,8.109067),(33.117544,8.104158),(33.11341,8.099093),(33.111343,8.090644),(33.106589,8.08049),(33.100594,8.071033),(33.043853,8.013465),(33.031141,7.996309),(33.026077,7.986284),(33.021633,7.965251),(33.016775,7.958327),(33.012021,7.953314),(33.007569,7.944058),(33.006801,7.942462),(32.993572,7.928148),(32.9898,7.917244),(33.015018,7.850633),(33.023493,7.835078),(33.040753,7.824846),(33.044629,7.816423),(33.047677,7.807277),(33.051295,7.801127),(33.058323,7.797975),(33.085401,7.794978),(33.107519,7.785211),(33.120851,7.783299),(33.126949,7.791567),(33.132117,7.793944),(33.164518,7.801127),(33.172941,7.799628),(33.201467,7.788156),(33.233919,7.783195),(33.242394,7.780715),(33.257794,7.767434),(33.273503,7.749192),(33.29159,7.733069),(33.347142,7.718961),(33.359803,7.719272),(33.384401,7.735446),(33.393289,7.739735),(33.436491,7.748572),(33.462329,7.749502),(33.479537,7.743146),(33.488477,7.73617),(33.506823,7.73095),(33.516796,7.726093),(33.526511,7.717101),(33.540361,7.698911),(33.550851,7.691315),(33.572297,7.685217),(33.637202,7.691315),(33.647744,7.688627),(33.675029,7.670851),(33.706552,7.661859),(33.71606,7.657156),(33.729445,7.642119),(33.761071,7.598142),(33.811507,7.560987),(33.821429,7.558403),(33.842099,7.555922),(33.852538,7.553545),(33.882407,7.536079),(34.006534,7.409936),(34.02369,7.382445),(34.030615,7.3487),(34.031132,7.255372),(34.033612,7.250256),(34.04002,7.247259),(34.085909,7.211499),(34.113194,7.177548),(34.132417,7.163388),(34.151124,7.167471),(34.167764,7.175377),(34.181407,7.167212),(34.190915,7.149539),(34.195773,7.129023),(34.195773,7.09774),(34.195773,7.086959),(34.198253,7.064376),(34.206005,7.054506),(34.213756,7.051974),(34.219957,7.046134),(34.229879,7.03337),(34.270807,7.003398),(34.275665,6.997403),(34.280729,6.980092),(34.287653,6.975286),(34.294475,6.972237),(34.297575,6.968568),(34.439169,6.934875),(34.503661,6.89002),(34.512343,6.876842),(34.519061,6.861649),(34.523298,6.844183),(34.524745,6.824572),(34.522575,6.821781),(34.513273,6.815296),(34.511102,6.814314),(34.512136,6.808268),(34.516683,6.798062),(34.517924,6.793824),(34.524745,6.752871),(34.53632,6.743078),(34.553994,6.739254),(34.596679,6.739202),(34.618486,6.736541),(34.635539,6.729823),(34.703545,6.684916),(34.709643,6.674012),(34.714501,6.655667),(34.726593,6.641947),(34.733518,6.637606),(34.743543,6.596808),(34.753878,6.556423),(34.776099,6.499449),(34.784264,6.44183),(34.792739,6.421676),(34.832633,6.353541),(34.839041,6.327599),(34.839764,6.268688),(34.844725,6.248689),(34.879969,6.187634),(34.898882,6.13867),(34.915522,6.119705),(34.934436,6.102368),(34.951489,6.08118),(34.959447,6.061724),(34.967964,6.008131),(34.969782,5.996689),(34.95924,5.975502),(34.955726,5.964211),(34.955313,5.952558),(34.960377,5.941706),(34.97681,5.924032),(34.979911,5.912844),(34.977741,5.898659),(34.974537,5.887239),(34.972676,5.876257),(34.974433,5.863106),(34.984355,5.841014),(35.06528,5.726447),(35.079543,5.699782),(35.088225,5.64175),(35.098663,5.622474),(35.143312,5.58966),(35.22341,5.542996),(35.261961,5.511887),(35.269299,5.491785),(35.27364,5.479873),(35.267645,5.468556),(35.258964,5.458143),(35.251832,5.447369),(35.250695,5.435147),(35.254623,5.425923),(35.287489,5.374092),(35.302372,5.357374),(35.320769,5.348719),(35.344953,5.352362),(35.367484,5.368046),(35.408205,5.412565),(35.430219,5.427215),(35.448203,5.430755),(35.469287,5.430755),(35.490164,5.428042),(35.508044,5.423417),(35.530782,5.410084),(35.57388,5.375332),(35.598168,5.368743),(35.621939,5.373782),(35.639819,5.382076),(35.658319,5.386417),(35.683641,5.379595),(35.749373,5.339158),(35.80415,5.318023),(35.809111,5.309109),(35.782033,5.26986),(35.775315,5.24658),(35.778415,5.22715),(35.798259,5.189271),(35.807147,5.165267),(35.807354,5.144674),(35.799189,5.124624),(35.760949,5.076332),(35.755884,5.063439),(35.751647,4.854356),(35.760018,4.805728),(35.784513,4.764181),(35.920835,4.619332),(35.856536,4.619603),(35.781222,4.619922),(35.778439,4.678338),(35.739402,4.681127),(35.711518,4.661608),(35.705939,4.619962),(35.705846,4.619447),(35.700365,4.589111),(35.61214,4.619497),(35.610641,4.620014),(35.562607,4.707137),(35.52218,4.780461),(35.570836,4.904313),(35.49564,4.926429),(35.396117,4.926429),(35.433715,5.003836),(35.411598,5.030376),(35.333563,4.987281),(35.263419,4.948545),(35.245726,4.98172),(35.176189,4.955564),(35.094298,4.924761),(35.00272,4.890314),(34.920829,4.859511),(34.844751,4.830895),(34.763589,4.800366),(34.666277,4.76498),(34.598159,4.73092),(34.529156,4.696419),(34.459674,4.660609),(34.381188,4.620158),(34.280109,4.520061),(34.179133,4.419912),(34.078157,4.319841),(33.977078,4.219692),(33.896049,4.138353),(33.813677,4.056033),(33.701901,3.944076),(33.606196,3.848087),(33.532609,3.774293),(33.527716,3.771431),(33.490648,3.749746),(33.447343,3.744372),(33.286526,3.752537),(33.19542,3.757059),(33.164466,3.763053),(33.143486,3.774086),(33.01724,3.87718),(32.997241,3.885526),(32.979568,3.879196),(32.9189,3.83416),(32.840352,3.794291),(32.756429,3.769022),(32.599281,3.756283),(32.415571,3.741297),(32.371801,3.731065),(32.187782,3.61916),(32.175948,3.605595),(32.174863,3.59265),(32.179203,3.556864),(32.178997,3.541361),(32.175929,3.527234),(32.174552,3.520897),(32.16799,3.512242),(32.155846,3.511776),(32.093007,3.524463),(32.076161,3.53317),(32.060864,3.548958),(32.054226,3.559572),(32.041537,3.57986),(32.030168,3.585932),(32.022236,3.58642),(31.943662,3.591255),(31.93002,3.59836),(31.923095,3.615284),(31.920046,3.661302),(31.916274,3.680267),(31.901908,3.704297),(31.830697,3.783879),(31.80703,3.803722),(31.801107,3.806472),(31.780985,3.815815),(31.777884,3.816435),(31.775714,3.810699),(31.696132,3.721273),(31.68683,3.712824),(31.668537,3.70502),(31.564771,3.689802),(31.547201,3.680991),(31.535522,3.666444),(31.53452,3.665563),(31.523739,3.656083),(31.505446,3.659855),(31.377702,3.729308),(31.29502,3.774189),(31.255745,3.786669),(31.214818,3.792354),(31.167585,3.792405),(31.141489,3.785119),(31.104408,3.756175),(31.077668,3.735303),(31.068677,3.731737),(31.04966,3.727991),(31.040358,3.724218),(31.009559,3.699775),(30.995968,3.693522),(30.985788,3.692127),(30.965996,3.692618),(30.955505,3.689001),(30.944447,3.679286),(30.939382,3.668408),(30.936282,3.656858),(30.931734,3.645102),(30.865692,3.548958),(30.843781,3.505911),(30.839543,3.490202),(30.827658,3.511105),(30.837683,3.562678),(30.823214,3.580429),(30.792105,3.592702),(30.779082,3.602598),(30.772261,3.617558),(30.76916,3.651845),(30.765129,3.661716),(30.746939,3.674867),(30.731333,3.663498),(30.714642,3.64381),(30.693351,3.63195),(30.665394,3.632053),(30.653767,3.630606),(30.636455,3.624095),(30.604932,3.60668),(30.592427,3.603528),(30.572635,3.601047),(30.562248,3.601358),(30.553463,3.60451),(30.543024,3.612907),(30.543438,3.617868),(30.549329,3.624095),(30.55491,3.636394),(30.562248,3.675332),(30.562868,3.694866),(30.53889,3.84155),(30.525247,3.867543),(30.515532,3.87085),(30.505507,3.869765),(30.495482,3.867078),(30.48587,3.865579),(30.465716,3.866096),(30.456828,3.867595),(30.418949,3.879713),(30.33792,3.927875),(30.270948,3.94945),(30.205887,3.95038),(30.189351,3.95609),(30.182116,3.965521),(30.180462,3.97565),(30.180359,3.986218),(30.178085,3.997096),(30.171574,4.00818),(30.147493,4.036654),(30.138501,4.059702),(30.133023,4.082801),(30.122688,4.102464),(30.099795,4.115099),(30.057782,4.123651),(30.0429,4.133108),(30.027913,4.15445),(30.004762,4.19928),(29.989363,4.21778),(29.966005,4.23181),(29.954171,4.233825),(29.942647,4.233774),(29.932157,4.236151),(29.924147,4.245582),(29.923217,4.257752),(29.929315,4.268371),(29.93686,4.278319),(29.940115,4.288344),(29.930348,4.308653),(29.908489,4.329194),(29.882703,4.344025),(29.860689,4.347281),(29.814852,4.346764),(29.787825,4.368804),(29.776198,4.405624),(29.776766,4.449342),(29.785654,4.479366),(29.786998,4.489675),(29.785448,4.498564),(29.778523,4.5151),(29.777696,4.523859),(29.783691,4.533833),(29.788295,4.539366),(29.793302,4.545383),(29.796713,4.556028),(29.784621,4.563288),(29.774492,4.5675),(29.751755,4.581272),(29.74514,4.583907),(29.722919,4.578197),(29.715323,4.57967),(29.700802,4.5883),(29.671863,4.613983),(29.660287,4.622122),(29.618533,4.641578),(29.606751,4.643904),(29.576675,4.644446),(29.546392,4.657365),(29.535024,4.660699),(29.528822,4.659872),(29.513268,4.652146),(29.508462,4.65535),(29.498127,4.665608),(29.494096,4.668295),(29.472599,4.672197),(29.457612,4.669535),(29.449241,4.657055),(29.44676,4.622122),(29.44366,4.604707),(29.44459,4.594475),(29.44273,4.583442),(29.423919,4.566053),(29.417305,4.555485),(29.410897,4.532283),(29.402629,4.512077),(29.390226,4.493758),(29.33886,4.449549),(29.328938,4.434924),(29.318706,4.411076),(29.303927,4.387149),(29.28739,4.383584),(29.267443,4.387873),(29.242845,4.387356),(29.23158,4.375548),(29.228686,4.356712),(29.221296,4.340692),(29.19613,4.337075),(29.173495,4.348004),(29.10983,4.412367),(29.098564,4.420532),(29.058877,4.437689),(29.055983,4.4365),(29.053193,4.450789),(29.042341,4.455336),(28.994075,4.487453),(28.98622,4.495928),(28.925552,4.480813),(28.903641,4.478591),(28.856202,4.482467),(28.825713,4.478384),(28.815274,4.478281),(28.789953,4.48859),(28.785095,4.507995),(28.783648,4.530371),(28.769179,4.549775),(28.755433,4.553987),(28.746855,4.54931),(28.738173,4.540887),(28.72453,4.533988),(28.704067,4.533988),(28.695178,4.532593),(28.653734,4.452701),(28.638644,4.432082),(28.623658,4.422367),(28.58242,4.412626),(28.570535,4.405107),(28.566711,4.394126),(28.564282,4.382602),(28.557306,4.373533),(28.54821,4.37082),(28.538495,4.371827),(28.52878,4.373843),(28.519272,4.374411),(28.496844,4.369011),(28.482426,4.358986),(28.443927,4.308498),(28.425427,4.290824),(28.404137,4.277828),(28.381502,4.275296),(28.357421,4.282866),(28.349101,4.292349),(28.345846,4.306379),(28.336854,4.327489),(28.303161,4.352242),(28.261045,4.350382),(28.216965,4.342036),(28.177587,4.347229),(28.158725,4.360794),(28.140225,4.37932),(28.112371,4.414641),(28.108341,4.433994),(28.101726,4.442133),(28.085551,4.443322),(28.076301,4.437689),(28.059765,4.420067),(28.04974,4.419137),(28.019044,4.462752),(28.014186,4.472209),(28.011241,4.480141),(28.009587,4.488383),(28.008915,4.499081),(28.012119,4.521766),(28.01708,4.539052),(28.014031,4.550033),(27.96251,4.557449),(27.951244,4.55595),(27.944526,4.557294),(27.934294,4.568172),(27.92923,4.569981),(27.920342,4.565278),(27.916104,4.562152),(27.917861,4.560756),(27.917035,4.558431),(27.916414,4.553573),(27.914037,4.54931),(27.907939,4.54869),(27.89099,4.555847),(27.885512,4.556674),(27.856935,4.552462),(27.849132,4.553212),(27.83766,4.559981),(27.819469,4.579567),(27.809238,4.58799),(27.801383,4.590574),(27.776888,4.595741),(27.772444,4.595819),(27.765209,4.612071),(27.760765,4.635997),(27.758801,4.677157),(27.766863,4.735449),(27.763142,4.757101),(27.759215,4.766273),(27.7526,4.777668),(27.744229,4.787848),(27.734823,4.793197),(27.725832,4.792784),(27.706505,4.787435),(27.699063,4.787435),(27.685938,4.797899),(27.679736,4.814488),(27.671882,4.855906),(27.658756,4.879652),(27.640979,4.890943),(27.552716,4.900529),(27.532562,4.907583),(27.514578,4.92244),(27.509204,4.932439),(27.505018,4.953859),(27.501349,4.96329),(27.492823,4.973031),(27.461558,4.99688),(27.45453,4.999799),(27.447864,5.003468),(27.441818,5.007964),(27.43634,5.013158),(27.443058,5.05778),(27.441301,5.070725),(27.431173,5.083618),(27.402131,5.104496),(27.385801,5.143847),(27.358516,5.166766),(27.30131,5.205187),(27.280898,5.23056),(27.264465,5.260145),(27.2378,5.32319),(27.233665,5.33828),(27.228239,5.387605),(27.220126,5.413082),(27.218059,5.42551),(27.220126,5.440883),(27.260744,5.550257),(27.258884,5.559972),(27.261054,5.577593),(27.257127,5.586068),(27.247515,5.592605),(27.240073,5.591985),(27.234596,5.588833),(27.231082,5.587593),(27.227309,5.586714),(27.222607,5.584363),(27.217336,5.585397),(27.211341,5.594879),(27.210669,5.601623),(27.215165,5.616144),(27.216716,5.633456),(27.219196,5.639993),(27.218369,5.645315),(27.201109,5.656968),(27.196407,5.661464),(27.194288,5.667304),(27.193823,5.676321),(27.190774,5.691824),(27.182351,5.707456),(27.170413,5.72035),(27.156254,5.727481),(27.136307,5.734018),(27.131449,5.741459),(27.130933,5.752131),(27.123801,5.768745),(27.11481,5.775282),(27.072745,5.791224),(27.063547,5.790862),(27.045356,5.784919),(27.036055,5.785126),(27.029543,5.78988),(27.021999,5.805099),(27.003499,5.819284),(26.99182,5.847706),(26.981071,5.859204),(26.937456,5.848507),(26.916475,5.849644),(26.903763,5.866982),(26.894255,5.889667),(26.882059,5.891941),(26.865316,5.885973),(26.842165,5.884267),(26.81891,5.894628),(26.810849,5.912379),(26.805061,5.957183),(26.794364,5.970489),(26.776329,5.981884),(26.705016,6.009789),(26.634322,6.006482),(26.602179,6.010616),(26.543578,6.030847),(26.527972,6.043172),(26.51805,6.061156),(26.509885,6.082679),(26.498827,6.099836),(26.48105,6.104951),(26.440639,6.076995),(26.424826,6.072447),(26.421312,6.09366),(26.42927,6.113091),(26.4456,6.130221),(26.46534,6.144303),(26.48353,6.154457),(26.500067,6.167971),(26.509472,6.186394),(26.508955,6.205462),(26.495829,6.221146),(26.4795,6.225073),(26.464927,6.224402),(26.455625,6.230474),(26.455315,6.254451),(26.452008,6.28029),(26.436195,6.29171),(26.394647,6.300547),(26.375837,6.312587),(26.351755,6.344368),(26.335219,6.360595),(26.315789,6.371576),(26.300699,6.377829),(26.28964,6.387208),(26.282819,6.407853),(26.285093,6.425888),(26.290674,6.444672),(26.289124,6.459426),(26.27021,6.465653),(26.277031,6.477358),(26.286436,6.484179),(26.296462,6.489527),(26.30504,6.496917),(26.324367,6.537199),(26.356096,6.579496),(26.372323,6.609804),(26.379867,6.619778),(26.379224,6.631362),(26.378007,6.65329),(26.329741,6.680575),(26.270623,6.702486),(26.236207,6.72003),(26.219464,6.737523),(26.178743,6.765919),(26.147634,6.804702),(26.131821,6.81173),(26.113734,6.816458),(26.091203,6.830334),(26.080971,6.842348),(26.077354,6.85333),(26.0757,6.864647),(26.071566,6.877462),(26.071152,6.88356),(26.073116,6.88971),(26.073633,6.895136),(26.069189,6.899063),(26.063401,6.899063),(26.05689,6.89741),(26.050585,6.896686),(26.045418,6.899683),(26.035806,6.922266),(26.032292,6.974097),(26.026091,6.99668),(25.981132,7.000297),(25.969453,7.003553),(25.966766,7.009857),(25.966249,7.017454),(25.960461,7.024482),(25.950436,7.028564),(25.930386,7.03151),(25.920257,7.034145),(25.888735,7.05187),(25.884084,7.061069),(25.881707,7.079827),(25.877469,7.087734),(25.860726,7.095537),(25.815767,7.099774),(25.800781,7.104942),(25.791686,7.121117),(25.791583,7.134398),(25.786209,7.142666),(25.7613,7.143751),(25.750242,7.146542),(25.734429,7.162406),(25.724507,7.166695),(25.705077,7.166385),(25.697015,7.168607),(25.672727,7.187831),(25.65402,7.195376),(25.592422,7.211241),(25.576609,7.219819),(25.53165,7.260695),(25.516768,7.269945),(25.500335,7.272787),(25.481163,7.266173),(25.455118,7.278213),(25.416154,7.307772),(25.360033,7.335574),(25.346804,7.345031),(25.335952,7.359604),(25.330577,7.374125),(25.32417,7.402908),(25.316211,7.417223),(25.3097,7.422029),(25.292957,7.427196),(25.285102,7.431589),(25.279211,7.438358),(25.269599,7.454223),(25.263295,7.461044),(25.249962,7.469984),(25.190431,7.501197),(25.169037,7.551168),(25.164593,7.567343),(25.165213,7.5799),(25.186504,7.600106),(25.252546,7.62243),(25.274974,7.641964),(25.279418,7.659482),(25.276421,7.674261),(25.270633,7.688162),(25.266809,7.703097),(25.270633,7.746143),(25.269909,7.760768),(25.264535,7.777718),(25.249652,7.804331),(25.23942,7.835957),(25.229705,7.851615),(25.216889,7.864069),(25.191568,7.872389),(25.18547,7.877918),(25.180509,7.884016),(25.173171,7.888564),(25.148573,7.892594),(25.134414,7.892543),(25.103408,7.885773),(25.089559,7.884895),(25.059173,7.896108),(25.029304,7.918846),(24.981245,7.972331),(24.972666,7.976982),(24.964812,7.982615),(24.958094,7.989178),(24.952616,7.996567),(24.950549,8.014396),(24.930085,8.035454),(24.927811,8.070956),(24.917993,8.087027),(24.832107,8.16573),(24.800171,8.180277),(24.742293,8.18684),(24.710564,8.204307),(24.69041,8.206632),(24.670876,8.206839),(24.614859,8.217148),(24.544682,8.206115),(24.51285,8.207149),(24.481327,8.226657),(24.471508,8.22893),(24.464274,8.233194),(24.458899,8.239757),(24.454869,8.248671),(24.431097,8.271408),(24.396578,8.267817),(24.332085,8.245725),(24.326814,8.248567),(24.310278,8.261564),(24.302733,8.26544),(24.295498,8.266525),(24.281029,8.266525),(24.263186,8.268505),(24.257051,8.269186),(24.222428,8.277196),(24.206615,8.283268),(24.17995,8.297712),(24.152872,8.317891),(24.131374,8.343058),(24.121556,8.372307),(24.125173,8.404889),(24.137162,8.438866),(24.193179,8.532426),(24.203204,8.543459),(24.243719,8.570176),(24.25054,8.579632),(24.245579,8.591802),(24.218397,8.613041),(24.211266,8.627071),(24.21509,8.64136),(24.233487,8.667767),(24.235761,8.682003),(24.217674,8.691512),(24.180467,8.690711),(24.170328,8.689327),(24.174782,8.695646),(24.176953,8.709857),(24.182999,8.717919),(24.240566,8.761534),(24.270849,8.775719),(24.380093,8.844216),(24.402882,8.848893),(24.493833,8.858349),(24.542874,8.874653),(24.552795,8.880079),(24.558377,8.886746),(24.558377,8.993354),(24.559307,8.999865),(24.565353,9.014851),(24.655993,9.167297),(24.659869,9.176986),(24.660954,9.182283),(24.680178,9.373357),(24.686999,9.391366),(24.694957,9.402916),(24.723948,9.429219),(24.785236,9.497509),(24.792522,9.520402),(24.794383,9.532184),(24.792677,9.798369),(24.796398,9.818859),(24.799809,9.825215),(24.803685,9.829169),(24.807302,9.830047),(24.816759,9.830745),(24.827611,9.832683),(24.836292,9.836636),(24.844819,9.841674),(24.879287,9.869915),(24.913859,9.890715),(24.926933,9.90521),(24.954373,9.942237),(24.967861,9.955957),(24.976284,9.967351),(24.984965,9.985593),(24.990702,10.00306),(25.016901,10.056752),(25.027754,10.086155),(25.028994,10.092047),(25.029304,10.098248),(25.028684,10.103906),(25.025583,10.114164),(25.017367,10.133),(25.015816,10.138039),(25.013646,10.149072),(25.014421,10.157366),(25.017057,10.168011),(25.067028,10.27493),(25.07695,10.286867),(25.084081,10.293223),(25.090282,10.295755),(25.105992,10.300354),(25.120564,10.30609),(25.131261,10.308287),(25.156893,10.306245),(25.168365,10.308132),(25.19255,10.317433),(25.198286,10.317588),(25.209965,10.316116),(25.216631,10.316968),(25.225623,10.320611),(25.232754,10.322317),(25.239575,10.323014),(25.251512,10.320999),(25.25699,10.319216),(25.267067,10.314798),(25.272803,10.313015),(25.278694,10.312007),(25.28474,10.311852),(25.290166,10.313093),(25.299778,10.316658),(25.305359,10.317123),(25.316211,10.315573),(25.321999,10.315651),(25.327425,10.316581),(25.346029,10.325262),(25.351145,10.326813),(25.362927,10.32813),(25.369335,10.328208),(25.381272,10.329448),(25.391504,10.332652),(25.400961,10.336993),(25.415844,10.342031),(25.427212,10.344124),(25.439305,10.345209),(25.479509,10.343814),(25.490826,10.345597),(25.503383,10.349163),(25.506484,10.349628),(25.58033,10.375311),(25.623324,10.385853),(25.629061,10.386706),(25.661358,10.387016),(25.684148,10.390685),(25.699961,10.395026),(25.705748,10.395956),(25.745901,10.394405),(25.753962,10.395336),(25.762799,10.397739),(25.78378,10.407893),(25.82848,10.419546),(25.839332,10.420554),(25.843053,10.417815),(25.886512,10.364304),(25.891628,10.355984),(25.893644,10.351333),(25.894884,10.34583),(25.90093,10.191059),(25.902791,10.185529),(25.905426,10.181266),(25.911937,10.174212),(25.919689,10.167856),(25.955707,10.146901),(25.9709,10.140312),(25.998082,10.132458),(26.000872,10.131062),(26.007229,10.125998),(26.010794,10.122691),(26.016685,10.114862),(26.076423,10.01771),(26.088206,10.002827),(26.102313,9.988163),(26.102623,9.987841),(26.179466,9.95898),(26.368163,9.739717),(26.556859,9.520454),(26.600887,9.49198),(26.625744,9.480663),(26.683621,9.476012),(26.694318,9.476942),(27.080393,9.606831),(27.33924,9.612799),(27.385129,9.60479),(27.528066,9.605358),(27.609301,9.595178),(27.615089,9.595798),(27.635915,9.601379),(27.641496,9.601947),(27.769602,9.583318),(27.868355,9.588511),(27.884788,9.591379),(27.895072,9.59541),(27.997076,9.38493),(28.04511,9.331407),(28.44383,9.327981),(28.84255,9.324556),(28.843841,9.324545),(28.831489,9.349248),(28.827372,9.38493),(28.828745,9.427474),(28.84933,9.465901),(28.913833,9.529031),(28.989476,9.58709),(29.009474,9.603239),(29.132258,9.667731),(29.252354,9.711165),(29.483089,9.761679),(29.567321,9.841364),(29.613882,9.914461),(29.615432,10.058147),(29.64587,10.081711),(29.700957,10.115017),(29.967814,10.243329),(30.012979,10.270485),(30.248779,10.121257),(30.484578,9.972028),(30.749265,9.735763),(30.765284,9.724291),(30.779082,9.719873),(30.793035,9.728012),(30.804817,9.738942),(30.824092,9.746176),(30.83696,9.749354),(30.94991,9.752446),(30.950234,9.752455),(31.164382,9.764005),(31.234817,9.792323),(31.449816,10.003279),(31.664816,10.214236),(31.774215,10.348775),(31.801965,10.376241),(31.864184,10.472127),(31.929865,10.636923),(31.942887,10.655552),(32.17848,10.853202),(32.414073,11.050851),(32.430713,11.082193),(32.435363,11.107023),(32.364153,11.240013),(32.348754,11.30758),(32.345705,11.41163),(32.359812,11.573481),(32.354852,11.675774),(32.352991,11.687376),(32.34834,11.703163),(32.34524,11.709106),(32.082207,11.999811),(32.414434,12.001271),(32.746662,12.002731),(32.748213,12.026812),(32.747592,12.039525),(32.745577,12.051126),(32.73302,12.085982),(32.725733,12.132336),(32.725268,12.145255),(32.726198,12.157941),(32.730074,12.181893),(32.730384,12.194864),(32.729712,12.201065),(32.728524,12.206853),(32.728834,12.211917),(32.731779,12.216155),(33.209218,12.210367),(33.203017,12.128098),(33.144984,11.934673),(33.146018,11.81866),(33.132427,11.686213),(33.129016,11.67549),(33.115994,11.646939),(33.104573,11.630583),(33.091447,11.614796),(33.087727,11.608853),(33.083231,11.599164),(33.082921,11.584565),(33.132504,11.213903),(33.182088,10.843241),(33.178367,10.824534),(33.174646,10.812287),(33.14824,10.766036),(33.141418,10.750818),(33.140023,10.739036),(33.15072,10.730974),(33.370603,10.650901),(33.3813,10.645785),(33.389827,10.639248),(33.468995,10.543905),(33.685545,10.367973),(33.902096,10.192041),(33.916978,10.174522),(33.961782,10.064038),(33.966794,10.047269),(33.968345,10.020811),(33.96726,10.000269),(33.956097,9.933994),(33.959818,9.904203),(33.96726,9.884514),(33.973254,9.861776),(33.973254,9.854852),(33.972376,9.848883),(33.96912,9.838341)] +Senegal [(-14.912496,16.640639),(-14.882007,16.647667),(-14.711216,16.635782),(-14.699692,16.637125),(-14.665896,16.645652),(-14.655974,16.64591),(-14.647189,16.643533),(-14.619077,16.630666),(-14.60807,16.628805),(-14.597011,16.629167),(-14.555128,16.640587),(-14.544456,16.640846),(-14.503916,16.632681),(-14.49309,16.632474),(-14.482393,16.633663),(-14.471954,16.636402),(-14.462058,16.640794),(-14.435832,16.655987),(-14.426247,16.659604),(-14.406635,16.660793),(-14.343254,16.63666),(-14.335529,16.631544),(-14.331343,16.624103),(-14.331756,16.613147),(-14.337027,16.59413),(-14.335322,16.586482),(-14.325968,16.5824),(-14.28765,16.585707),(-14.277961,16.581263),(-14.274576,16.573511),(-14.273258,16.555631),(-14.268065,16.54788),(-14.257652,16.543177),(-14.233674,16.541575),(-14.22228,16.539612),(-14.214166,16.534961),(-14.197191,16.513463),(-14.190085,16.507107),(-14.166392,16.490881),(-14.13244,16.45264),(-14.073038,16.400654),(-14.039836,16.382102),(-14.033066,16.376056),(-14.0156,16.35585),(-14.006711,16.350269),(-13.987049,16.34288),(-13.97785,16.33797),(-13.971856,16.331201),(-13.971158,16.324224),(-13.977463,16.308825),(-13.979323,16.30035),(-13.977282,16.293839),(-13.967437,16.281385),(-13.963045,16.272393),(-13.962683,16.26397),(-13.967799,16.246038),(-13.964078,16.23007),(-13.947051,16.22113),(-13.907829,16.210019),(-13.899586,16.205317),(-13.883825,16.193741),(-13.877236,16.186868),(-13.872275,16.178755),(-13.863826,16.151832),(-13.859072,16.142272),(-13.852767,16.133487),(-13.844835,16.126045),(-13.835766,16.120568),(-13.827317,16.1185),(-13.819203,16.119276),(-13.811323,16.122066),(-13.794941,16.131213),(-13.778818,16.143202),(-13.771997,16.150437),(-13.754246,16.175345),(-13.746701,16.182373),(-13.738226,16.187127),(-13.729416,16.189504),(-13.72014,16.189969),(-13.71058,16.188884),(-13.712621,16.176068),(-13.718306,16.159288),(-13.722775,16.146096),(-13.719571,16.135605),(-13.714843,16.132763),(-13.704482,16.128836),(-13.69996,16.125632),(-13.698849,16.123926),(-13.697299,16.120361),(-13.696007,16.118914),(-13.693578,16.117519),(-13.687454,16.115245),(-13.68469,16.113798),(-13.678127,16.108217),(-13.675931,16.106925),(-13.668954,16.105736),(-13.662417,16.107442),(-13.635029,16.121601),(-13.629267,16.12279),(-13.610482,16.122893),(-13.606968,16.123255),(-13.599398,16.125477),(-13.569064,16.138189),(-13.516405,16.149041),(-13.502789,16.154416),(-13.497259,16.156069),(-13.491549,16.156276),(-13.484934,16.154984),(-13.4831,16.141031),(-13.499869,16.110646),(-13.501703,16.095143),(-13.491549,16.088425),(-13.455091,16.102222),(-13.440028,16.101861),(-13.435067,16.096538),(-13.427031,16.08305),(-13.422096,16.077211),(-13.394578,16.059279),(-13.388558,16.052096),(-13.386052,16.044396),(-13.385044,16.026878),(-13.383674,16.019178),(-13.369774,15.985485),(-13.36334,15.975098),(-13.355795,15.965538),(-13.34701,15.956753),(-13.321766,15.93815),(-13.315927,15.93045),(-13.31373,15.92213),(-13.314351,15.913603),(-13.318175,15.896705),(-13.318175,15.888489),(-13.31603,15.881771),(-13.311767,15.875776),(-13.293654,15.855312),(-13.284766,15.839913),(-13.27996,15.82286),(-13.279676,15.803533),(-13.281898,15.795109),(-13.285231,15.786634),(-13.287453,15.778159),(-13.286135,15.769633),(-13.280968,15.762812),(-13.273965,15.757489),(-13.258669,15.748807),(-13.250659,15.742761),(-13.243941,15.735785),(-13.232547,15.71961),(-13.228258,15.71129),(-13.227121,15.703952),(-13.228878,15.696614),(-13.243993,15.673721),(-13.247662,15.665608),(-13.248747,15.65672),(-13.2473,15.648297),(-13.237844,15.62344),(-13.222806,15.623854),(-13.192808,15.62897),(-13.178235,15.628194),(-13.171052,15.625352),(-13.152552,15.612407),(-13.145239,15.609539),(-13.122218,15.603493),(-13.109712,15.596413),(-13.099015,15.586879),(-13.091806,15.575123),(-13.089455,15.561041),(-13.095087,15.523421),(-13.094958,15.514713),(-13.093098,15.506212),(-13.089584,15.49704),(-13.083845,15.49275),(-13.05323,15.485102),(-13.038889,15.486885),(-13.023774,15.495954),(-13.010545,15.50629),(-12.995843,15.514222),(-12.980185,15.516289),(-12.96432,15.509158),(-12.958016,15.501742),(-12.954941,15.493061),(-12.95091,15.465414),(-12.947603,15.458618),(-12.937552,15.445777),(-12.933056,15.438284),(-12.93117,15.430532),(-12.931015,15.397046),(-12.927217,15.380742),(-12.918974,15.365368),(-12.90683,15.352682),(-12.847273,15.313278),(-12.834535,15.30023),(-12.828049,15.283874),(-12.83345,15.269999),(-12.850038,15.26739),(-12.884196,15.270826),(-12.894376,15.263359),(-12.888356,15.250879),(-12.865489,15.228684),(-12.856782,15.221992),(-12.847893,15.218943),(-12.838333,15.218762),(-12.816965,15.223077),(-12.806397,15.223077),(-12.79632,15.220287),(-12.787225,15.214163),(-12.78198,15.20729),(-12.779474,15.199616),(-12.779267,15.191399),(-12.784667,15.166026),(-12.783995,15.158146),(-12.780921,15.149076),(-12.779758,15.146389),(-12.754566,15.13856),(-12.746639,15.131136),(-12.717049,15.10342),(-12.690306,15.092387),(-12.689585,15.092479),(-12.679273,15.093783),(-12.647647,15.103162),(-12.632273,15.105436),(-12.622946,15.102258),(-12.615065,15.094609),(-12.602146,15.078771),(-12.547059,15.047171),(-12.529592,15.034355),(-12.518973,15.028283),(-12.495563,15.023012),(-12.484892,15.017276),(-12.476262,15.007741),(-12.471508,14.998),(-12.448202,14.918212),(-12.447633,14.907437),(-12.433836,14.899428),(-12.400091,14.860386),(-12.382495,14.84602),(-12.319579,14.817185),(-12.26413,14.774939),(-12.256508,14.745794),(-12.236613,14.728456),(-12.230489,14.714555),(-12.222815,14.70329),(-12.207648,14.696598),(-12.191422,14.693445),(-12.180311,14.692825),(-12.185841,14.68081),(-12.190285,14.673653),(-12.192197,14.666884),(-12.18982,14.655954),(-12.182792,14.650244),(-12.172611,14.647918),(-12.165015,14.641872),(-12.165945,14.625181),(-12.171526,14.616422),(-12.191008,14.597999),(-12.198398,14.58756),(-12.200465,14.577354),(-12.201447,14.565184),(-12.204186,14.554255),(-12.211472,14.547976),(-12.230386,14.536685),(-12.236432,14.519451),(-12.234184,14.498574),(-12.222428,14.454933),(-12.21987,14.434598),(-12.22173,14.391242),(-12.208837,14.388761),(-12.115612,14.35512),(-12.108688,14.34202),(-12.113029,14.329592),(-12.127808,14.324295),(-12.119591,14.310575),(-12.10998,14.298146),(-12.097784,14.28807),(-12.058355,14.274091),(-12.044299,14.264919),(-12.03448,14.250268),(-11.997583,14.166165),(-11.995413,14.144022),(-12.002751,14.103456),(-12.02523,14.026742),(-12.025334,13.985246),(-12.015308,13.959304),(-11.99903,13.943569),(-11.980375,13.929642),(-11.962908,13.909152),(-11.956811,13.890342),(-11.956966,13.870007),(-11.962908,13.829467),(-11.978825,13.784819),(-12.012311,13.751177),(-12.097629,13.70441),(-12.083883,13.692964),(-12.070395,13.673249),(-12.049828,13.634363),(-12.037323,13.598267),(-12.015153,13.564444),(-11.996808,13.544161),(-11.939034,13.50295),(-11.932729,13.496051),(-11.923841,13.481142),(-11.918157,13.474708),(-11.907718,13.469644),(-11.898468,13.468507),(-11.890406,13.465148),(-11.883482,13.453547),(-11.885652,13.43577),(-11.899967,13.382182),(-11.897589,13.371175),(-11.888804,13.368875),(-11.862346,13.356085),(-11.854595,13.350556),(-11.850771,13.34314),(-11.844569,13.323839),(-11.840022,13.316889),(-11.828136,13.307277),(-11.822555,13.306708),(-11.817078,13.312341),(-11.775116,13.343347),(-11.772533,13.362674),(-11.76659,13.383164),(-11.756513,13.399803),(-11.741217,13.407813),(-11.726902,13.405798),(-11.719771,13.397891),(-11.714035,13.388099),(-11.70401,13.380373),(-11.694501,13.379184),(-11.674967,13.383215),(-11.649801,13.38412),(-11.645253,13.379753),(-11.646597,13.365516),(-11.643962,13.355594),(-11.632903,13.35438),(-11.620345,13.357532),(-11.613111,13.360813),(-11.613007,13.339058),(-11.605824,13.319653),(-11.581175,13.286141),(-11.55854,13.266504),(-11.554251,13.256866),(-11.555802,13.245911),(-11.568566,13.227902),(-11.572338,13.217101),(-11.569341,13.199195),(-11.559729,13.186922),(-11.547843,13.176199),(-11.538542,13.16297),(-11.538077,13.128011),(-11.533064,13.111811),(-11.515391,13.107754),(-11.501541,13.099589),(-11.489707,13.090778),(-11.477098,13.083285),(-11.46082,13.079177),(-11.450278,13.075095),(-11.451415,13.068067),(-11.456945,13.060444),(-11.45896,13.054682),(-11.445162,13.035226),(-11.43953,13.023005),(-11.435654,13.016726),(-11.431675,13.01347),(-11.422993,13.009414),(-11.424543,13.005202),(-11.430124,13.00099),(-11.433432,12.99696),(-11.433535,12.988356),(-11.435964,12.968848),(-11.438703,12.9589),(-11.423096,12.956006),(-11.411521,12.960709),(-11.404235,12.972103),(-11.401806,12.989492),(-11.393434,12.980811),(-11.388835,12.970553),(-11.386561,12.959107),(-11.384443,12.933114),(-11.387801,12.927119),(-11.394209,12.923321),(-11.402891,12.916396),(-11.407283,12.916293),(-11.413588,12.919703),(-11.420668,12.922029),(-11.427231,12.918618),(-11.427954,12.912495),(-11.423096,12.907069),(-11.417309,12.901953),(-11.415138,12.896656),(-11.419324,12.886501),(-11.424905,12.876166),(-11.428626,12.865805),(-11.425474,12.849553),(-11.427024,12.845238),(-11.430021,12.841052),(-11.432502,12.835523),(-11.432088,12.829347),(-11.428161,12.827022),(-11.423458,12.826402),(-11.420978,12.825575),(-11.419686,12.824671),(-11.416017,12.820071),(-11.411676,12.812888),(-11.408472,12.804491),(-11.406353,12.794285),(-11.405423,12.782348),(-11.407594,12.77147),(-11.41457,12.764209),(-11.41457,12.759713),(-11.404028,12.741213),(-11.402168,12.73173),(-11.410332,12.718811),(-11.424182,12.716383),(-11.437359,12.712248),(-11.447953,12.685945),(-11.455756,12.680622),(-11.461905,12.673129),(-11.460975,12.658402),(-11.457151,12.655973),(-11.449555,12.653182),(-11.441752,12.647756),(-11.436842,12.637473),(-11.436636,12.627086),(-11.441803,12.595511),(-11.440925,12.584866),(-11.438599,12.577993),(-11.437359,12.570758),(-11.43984,12.559286),(-11.460614,12.549829),(-11.467797,12.543525),(-11.456686,12.539753),(-11.449451,12.535205),(-11.437876,12.515361),(-11.429814,12.508333),(-11.430383,12.515723),(-11.42475,12.526368),(-11.416999,12.529314),(-11.410849,12.513346),(-11.408679,12.504923),(-11.402374,12.488541),(-11.400514,12.479549),(-11.392607,12.491848),(-11.390437,12.497533),(-11.377776,12.480221),(-11.379792,12.456864),(-11.386561,12.430457),(-11.388422,12.403895),(-11.416017,12.40498),(-11.481646,12.428235),(-11.515391,12.431645),(-11.601535,12.425238),(-11.643083,12.417641),(-11.720029,12.389477),(-11.757805,12.383328),(-11.839609,12.386687),(-11.860176,12.391079),(-11.920637,12.416039),(-11.921981,12.417693),(-11.923944,12.418778),(-11.930869,12.418726),(-11.93521,12.417021),(-11.946269,12.409425),(-11.984303,12.389012),(-11.997583,12.386377),(-12.018874,12.388444),(-12.07918,12.408133),(-12.103985,12.407357),(-12.121917,12.398728),(-12.155041,12.369014),(-12.192042,12.348756),(-12.36092,12.305607),(-12.377121,12.313668),(-12.405517,12.356456),(-12.423397,12.369117),(-12.466754,12.384465),(-12.48776,12.389477),(-12.505072,12.390253),(-12.521918,12.386894),(-12.541452,12.379452),(-12.570649,12.363691),(-12.57813,12.361528),(-12.579408,12.361159),(-12.593671,12.361572),(-12.597314,12.365448),(-12.598064,12.372424),(-12.603412,12.381933),(-12.63155,12.41268),(-12.648009,12.425754),(-12.667517,12.433661),(-12.688058,12.435986),(-12.752654,12.432421),(-12.773272,12.435056),(-12.779164,12.443273),(-12.783479,12.453091),(-12.799421,12.460636),(-12.829057,12.462755),(-12.838617,12.466372),(-12.849133,12.475829),(-12.851252,12.484355),(-12.84965,12.491073),(-12.849211,12.495001),(-12.874042,12.516395),(-12.913135,12.536342),(-12.949489,12.535928),(-12.966232,12.496086),(-12.969514,12.476811),(-12.981038,12.466992),(-12.997419,12.465855),(-13.015247,12.472676),(-13.034626,12.470558),(-13.051989,12.472263),(-13.066743,12.480118),(-13.078267,12.496241),(-13.082711,12.515671),(-13.079094,12.532156),(-13.064211,12.566159),(-13.060413,12.583367),(-13.059327,12.60347),(-13.063642,12.62228),(-13.076458,12.635922),(-13.091703,12.638196),(-13.108988,12.635406),(-13.127308,12.634837),(-13.145317,12.643932),(-13.154696,12.632822),(-13.162629,12.638144),(-13.169217,12.638765),(-13.176271,12.637731),(-13.185211,12.637783),(-13.197872,12.634062),(-13.201903,12.63339),(-13.20645,12.635664),(-13.21601,12.645741),(-13.22154,12.648376),(-13.229084,12.647808),(-13.246008,12.643157),(-13.25469,12.641813),(-13.263398,12.643209),(-13.275722,12.651064),(-13.284146,12.653337),(-13.292931,12.652717),(-13.322877,12.646516),(-13.327709,12.644914),(-13.330034,12.642072),(-13.332773,12.639643),(-13.338613,12.63923),(-13.343444,12.640987),(-13.352049,12.646981),(-13.356648,12.648893),(-13.359801,12.649737),(-13.404758,12.661761),(-13.728279,12.673388),(-14.082753,12.674835),(-14.343108,12.675897),(-14.437305,12.676282),(-14.791728,12.67778),(-14.87216,12.678109),(-15.146228,12.679227),(-15.195114,12.679434),(-15.223795,12.675196),(-15.249736,12.663621),(-15.307821,12.622693),(-15.368644,12.595356),(-15.421731,12.557122),(-15.424609,12.555049),(-15.677049,12.439294),(-15.711517,12.432214),(-15.883964,12.442182),(-15.893883,12.442756),(-15.973258,12.437226),(-15.992482,12.440689),(-16.05501,12.461204),(-16.090512,12.464305),(-16.166063,12.451437),(-16.172832,12.448544),(-16.177173,12.445236),(-16.181876,12.443376),(-16.190092,12.444771),(-16.211693,12.453143),(-16.22239,12.455055),(-16.23288,12.452884),(-16.30192,12.416194),(-16.382019,12.373819),(-16.412301,12.361727),(-16.462686,12.361314),(-16.514879,12.34855),(-16.533327,12.347826),(-16.668874,12.356715),(-16.705616,12.348033),(-16.728437,12.332531),(-16.735015,12.34516),(-16.739735,12.358385),(-16.74413,12.378852),(-16.750478,12.385647),(-16.765492,12.395982),(-16.788482,12.417955),(-16.798736,12.430894),(-16.802968,12.443793),(-16.802968,12.485419),(-16.800771,12.494452),(-16.790924,12.503241),(-16.788686,12.512397),(-16.781361,12.522935),(-16.731679,12.552069),(-16.709869,12.552802),(-16.678822,12.550238),(-16.645375,12.570746),(-16.634674,12.579495),(-16.628977,12.587633),(-16.624623,12.596015),(-16.618031,12.605455),(-16.579742,12.632799),(-16.56135,12.611802),(-16.54719,12.573879),(-16.533315,12.565334),(-16.532704,12.580878),(-16.52245,12.593085),(-16.50475,12.598049),(-16.485097,12.593736),(-16.451568,12.575019),(-16.421457,12.566881),(-16.388336,12.551581),(-16.370961,12.550238),(-16.351308,12.559516),(-16.32726,12.58869),(-16.30956,12.598049),(-16.286285,12.597073),(-16.266469,12.590033),(-16.246693,12.587958),(-16.223866,12.601752),(-16.212229,12.60635),(-16.204457,12.597602),(-16.198598,12.584906),(-16.19286,12.577582),(-16.11498,12.606106),(-16.097076,12.610256),(-16.07311,12.611721),(-16.057607,12.616197),(-16.044667,12.625474),(-16.029775,12.632961),(-16.007883,12.632148),(-15.990346,12.62287),(-15.963857,12.60224),(-15.947987,12.58983),(-15.939605,12.580634),(-15.927968,12.578599),(-15.863881,12.577582),(-15.857737,12.573188),(-15.844797,12.559882),(-15.839996,12.557074),(-15.831166,12.558254),(-15.805531,12.564521),(-15.794057,12.571479),(-15.782948,12.584662),(-15.768788,12.592597),(-15.747874,12.583726),(-15.742421,12.588324),(-15.736073,12.591376),(-15.729604,12.591498),(-15.715932,12.581773),(-15.710113,12.58275),(-15.704986,12.585191),(-15.699452,12.583726),(-15.69164,12.574897),(-15.684926,12.557929),(-15.679555,12.550238),(-15.673085,12.54442),(-15.665924,12.540188),(-15.65689,12.537584),(-15.644765,12.537258),(-15.633412,12.539537),(-15.587717,12.55801),(-15.569407,12.576321),(-15.538686,12.615424),(-15.527252,12.623725),(-15.517079,12.628852),(-15.509836,12.63581),(-15.507069,12.649563),(-15.512807,12.655666),(-15.523752,12.660346),(-15.529856,12.665758),(-15.521311,12.673774),(-15.531077,12.696438),(-15.540191,12.726874),(-15.543446,12.758043),(-15.534901,12.783026),(-15.523346,12.789862),(-15.509023,12.790351),(-15.494008,12.789049),(-15.480336,12.790473),(-15.46642,12.796454),(-15.456939,12.803656),(-15.450673,12.812812),(-15.446156,12.824612),(-15.425608,12.804999),(-15.406809,12.799709),(-15.394032,12.810248),(-15.391591,12.838202),(-15.407053,12.822333),(-15.421783,12.822821),(-15.436879,12.829291),(-15.453033,12.831448),(-15.468129,12.824897),(-15.492787,12.807115),(-15.510732,12.803453),(-15.520863,12.803412),(-15.535878,12.801337),(-15.549469,12.794338),(-15.555409,12.779608),(-15.555409,12.721584),(-15.544097,12.684068),(-15.542348,12.673774),(-15.541737,12.657457),(-15.54247,12.6494),(-15.545481,12.642727),(-15.569692,12.62226),(-15.577382,12.612494),(-15.627756,12.567613),(-15.641021,12.559475),(-15.652903,12.558905),(-15.663319,12.563463),(-15.672109,12.570746),(-15.666412,12.600653),(-15.696767,12.614163),(-15.76769,12.625393),(-15.815826,12.595364),(-15.833485,12.591254),(-15.836903,12.594428),(-15.841298,12.601467),(-15.8485,12.608547),(-15.860463,12.611721),(-15.87267,12.609565),(-15.889556,12.600165),(-15.901723,12.598049),(-15.913319,12.601223),(-15.928375,12.608873),(-15.953277,12.625393),(-15.954457,12.630316),(-15.953196,12.637397),(-15.952707,12.643744),(-15.956451,12.64643),(-15.969716,12.648179),(-15.976674,12.650214),(-15.981191,12.652655),(-15.987457,12.660956),(-16.001088,12.687445),(-16.00475,12.690823),(-16.009348,12.692694),(-16.013173,12.696194),(-16.016957,12.717515),(-16.022125,12.72899),(-16.02831,12.72899),(-16.02831,12.721584),(-16.029775,12.717475),(-16.040517,12.696967),(-16.045155,12.690497),(-16.048899,12.656399),(-16.076812,12.638983),(-16.117747,12.632148),(-16.128326,12.627672),(-16.148915,12.614447),(-16.155019,12.611721),(-16.165598,12.61286),(-16.189809,12.619127),(-16.22761,12.619127),(-16.23884,12.616523),(-16.257924,12.608222),(-16.268625,12.605455),(-16.28067,12.605455),(-16.312978,12.611721),(-16.323842,12.609565),(-16.337758,12.600165),(-16.346791,12.598049),(-16.350209,12.593329),(-16.355133,12.582587),(-16.361969,12.571234),(-16.370961,12.564521),(-16.411977,12.577582),(-16.44518,12.596137),(-16.460683,12.608873),(-16.467193,12.62226),(-16.471181,12.624579),(-16.480336,12.627387),(-16.489573,12.631822),(-16.494496,12.638983),(-16.492828,12.648912),(-16.48648,12.657782),(-16.477284,12.66413),(-16.467193,12.666327),(-16.467193,12.673774),(-16.482574,12.67414),(-16.492299,12.681708),(-16.497914,12.693834),(-16.501332,12.707913),(-16.508168,12.625393),(-16.515696,12.625393),(-16.52892,12.642768),(-16.565663,12.673896),(-16.570221,12.69359),(-16.581939,12.684394),(-16.58967,12.673285),(-16.598785,12.664008),(-16.614613,12.660102),(-16.626576,12.662543),(-16.634674,12.669379),(-16.638661,12.679999),(-16.637929,12.69359),(-16.623769,12.719713),(-16.603831,12.739325),(-16.590728,12.76203),(-16.596913,12.797309),(-16.601389,12.770209),(-16.603749,12.762519),(-16.60912,12.75373),(-16.613759,12.749416),(-16.61852,12.746527),(-16.638417,12.728502),(-16.646881,12.718207),(-16.651031,12.705268),(-16.652252,12.684027),(-16.64509,12.648586),(-16.647206,12.632229),(-16.677317,12.619574),(-16.721059,12.583726),(-16.754628,12.572211),(-16.773101,12.582953),(-16.780873,12.608832),(-16.782541,12.642727),(-16.777089,12.678046),(-16.76122,12.713202),(-16.735463,12.732896),(-16.700551,12.721584),(-16.698964,12.736396),(-16.713246,12.747301),(-16.733957,12.75137),(-16.751536,12.745429),(-16.772125,12.720649),(-16.783559,12.712592),(-16.796213,12.715318),(-16.790273,12.723456),(-16.788238,12.732082),(-16.790273,12.740546),(-16.796213,12.748847),(-16.774485,12.778551),(-16.770131,12.793362),(-16.775054,12.81094),(-16.782541,12.81094),(-16.787221,12.799872),(-16.794057,12.794989),(-16.800282,12.798814),(-16.802968,12.814032),(-16.800526,12.825995),(-16.794545,12.838772),(-16.75886,12.890774),(-16.755238,12.903469),(-16.756256,12.916246),(-16.760976,12.939683),(-16.762074,12.951158),(-16.759836,12.972073),(-16.749989,13.013617),(-16.747792,13.040269),(-16.749338,13.056627),(-16.753651,13.065009),(-16.752125,13.088039),(-16.748508,13.099227),(-16.742152,13.107392),(-16.726339,13.122637),(-16.723186,13.132249),(-16.708407,13.156692),(-16.673525,13.164314),(-16.630014,13.1639),(-16.465218,13.162428),(-16.313289,13.161084),(-16.092062,13.15912),(-15.961734,13.157984),(-15.897441,13.157399),(-15.870784,13.157157),(-15.833318,13.156847),(-15.8266,13.161678),(-15.82474,13.232914),(-15.824367,13.248512),(-15.822673,13.319421),(-15.818694,13.333528),(-15.807118,13.339781),(-15.737407,13.346344),(-15.6925,13.360348),(-15.679891,13.360658),(-15.612453,13.354225),(-15.596123,13.355827),(-15.566048,13.365671),(-15.542948,13.377996),(-15.519539,13.3866),(-15.488326,13.385256),(-15.379237,13.362519),(-15.342392,13.36257),(-15.309836,13.368436),(-15.277073,13.380296),(-15.248341,13.398977),(-15.227774,13.425176),(-15.221728,13.452823),(-15.217955,13.51468),(-15.203951,13.536901),(-15.181331,13.559843),(-15.160491,13.580981),(-15.13765,13.589973),(-15.110158,13.57248),(-15.065251,13.531268),(-15.01559,13.496051),(-15.015435,13.495999),(-15.015229,13.495844),(-15.015074,13.495741),(-14.949703,13.463081),(-14.915907,13.454425),(-14.877614,13.451376),(-14.862576,13.447061),(-14.829142,13.427011),(-14.810797,13.421172),(-14.790333,13.417348),(-14.779636,13.410888),(-14.759482,13.38337),(-14.731577,13.359005),(-14.698142,13.345181),(-14.661607,13.340918),(-14.624555,13.345181),(-14.592154,13.35314),(-14.580733,13.34916),(-14.552109,13.323094),(-14.542854,13.314666),(-14.529574,13.307277),(-14.515104,13.303168),(-14.483633,13.301928),(-14.470326,13.298259),(-14.459139,13.286916),(-14.45149,13.276865),(-14.442034,13.268545),(-14.43144,13.26144),(-14.394982,13.242965),(-14.368834,13.235705),(-14.341781,13.233638),(-14.284911,13.238547),(-14.230625,13.228574),(-14.201687,13.229555),(-14.179827,13.240123),(-14.131794,13.275573),(-14.116808,13.282162),(-14.099496,13.281955),(-14.056579,13.297174),(-14.016272,13.297639),(-13.976119,13.308207),(-13.900568,13.314563),(-13.851036,13.33575),(-13.822562,13.378203),(-13.818713,13.429362),(-13.84269,13.476646),(-13.852793,13.484811),(-13.875479,13.497911),(-13.882504,13.503782),(-13.8846,13.505533),(-13.889793,13.513957),(-13.896175,13.532922),(-13.901343,13.541629),(-13.918086,13.554988),(-13.94209,13.566925),(-13.967489,13.575581),(-13.988495,13.579146),(-14.001905,13.577028),(-14.022266,13.565323),(-14.034022,13.560207),(-14.044461,13.558605),(-14.06185,13.560052),(-14.072599,13.55969),(-14.092882,13.554729),(-14.139132,13.531552),(-14.196829,13.518788),(-14.215794,13.510753),(-14.271217,13.476956),(-14.329947,13.456441),(-14.347285,13.452358),(-14.365553,13.454839),(-14.389582,13.465872),(-14.462135,13.516656),(-14.470275,13.522354),(-14.480378,13.533129),(-14.486863,13.546539),(-14.500221,13.589973),(-14.50973,13.609093),(-14.523243,13.625655),(-14.542596,13.640822),(-14.585539,13.660459),(-14.626674,13.663508),(-14.667188,13.652889),(-14.728579,13.619325),(-14.74093,13.615449),(-14.754934,13.620384),(-14.796017,13.644698),(-14.802942,13.65232),(-14.822372,13.7172),(-14.831312,13.735623),(-14.843973,13.752676),(-14.860314,13.765595),(-14.879268,13.780581),(-14.915803,13.792441),(-15.016779,13.796885),(-15.07631,13.818951),(-15.097704,13.819984),(-15.17062,13.793784),(-15.245964,13.746733),(-15.267255,13.741798),(-15.27573,13.748051),(-15.295108,13.773191),(-15.304462,13.781873),(-15.347766,13.788643),(-15.394069,13.771512),(-15.43634,13.741178),(-15.467501,13.708286),(-15.478611,13.691258),(-15.488946,13.670329),(-15.496801,13.648341),(-15.49803,13.641522),(-15.50047,13.627981),(-15.502486,13.588267),(-15.518195,13.5831),(-15.567133,13.583255),(-15.629196,13.583513),(-15.691363,13.58372),(-15.753426,13.583978),(-15.815593,13.584185),(-15.877708,13.584443),(-15.939772,13.58465),(-16.001938,13.584908),(-16.064054,13.585115),(-16.085383,13.585168),(-16.126117,13.58527),(-16.188232,13.585528),(-16.250295,13.585735),(-16.312462,13.585994),(-16.374577,13.5862),(-16.436641,13.586459),(-16.498756,13.586665),(-16.561399,13.586914),(-16.561513,13.587104),(-16.57136,13.601793),(-16.576405,13.61225),(-16.575754,13.640326),(-16.558665,13.655911),(-16.533518,13.668606),(-16.508168,13.687974),(-16.513499,13.690863),(-16.523793,13.698391),(-16.529286,13.700995),(-16.516591,13.712836),(-16.507232,13.728095),(-16.508901,13.74022),(-16.529286,13.742621),(-16.526601,13.72663),(-16.531484,13.707261),(-16.542104,13.693671),(-16.556549,13.69477),(-16.56371,13.673774),(-16.579254,13.659573),(-16.598866,13.654202),(-16.618031,13.660061),(-16.635569,13.678046),(-16.640004,13.696479),(-16.631703,13.735785),(-16.62975,13.760484),(-16.627105,13.770453),(-16.618031,13.783596),(-16.582509,13.824693),(-16.576405,13.838813),(-16.570546,13.83511),(-16.554921,13.827948),(-16.549143,13.82453),(-16.551381,13.829088),(-16.556549,13.845649),(-16.52184,13.851793),(-16.520131,13.832017),(-16.512929,13.8133),(-16.501576,13.801988),(-16.48705,13.804633),(-16.494618,13.82746),(-16.496327,13.852729),(-16.5006,13.872707),(-16.515696,13.879788),(-16.510365,13.893297),(-16.488678,13.986396),(-16.48705,14.002631),(-16.508901,13.971381),(-16.518056,13.95185),(-16.52595,13.910468),(-16.536204,13.885728),(-16.549428,13.869289),(-16.562815,13.872952),(-16.570221,13.872952),(-16.58023,13.859524),(-16.59789,13.854315),(-16.618642,13.85163),(-16.637929,13.845649),(-16.648671,13.836859),(-16.652252,13.828559),(-16.65392,13.820054),(-16.659006,13.810858),(-16.664459,13.805976),(-16.675933,13.799547),(-16.682525,13.794094),(-16.692494,13.775702),(-16.700063,13.769029),(-16.710194,13.773627),(-16.713368,13.783352),(-16.71345,13.795722),(-16.715321,13.806342),(-16.724477,13.810858),(-16.737213,13.819648),(-16.742421,13.839423),(-16.741078,13.860419),(-16.73412,13.872952),(-16.740224,13.886623),(-16.744456,13.902167),(-16.747792,13.934394),(-16.746816,13.951361),(-16.743398,13.961086),(-16.737213,13.967678),(-16.714223,13.985582),(-16.702504,13.989976),(-16.689687,13.990546),(-16.660268,13.98725),(-16.646311,13.982978),(-16.63679,13.974921),(-16.637929,13.961656),(-16.623931,13.973944),(-16.612904,13.988715),(-16.598988,13.996812),(-16.576405,13.988959),(-16.576772,14.010972),(-16.568918,14.020819),(-16.556264,14.026923),(-16.542348,14.03734),(-16.554799,14.033108),(-16.568349,14.030341),(-16.580963,14.032782),(-16.590728,14.043606),(-16.545522,14.07099),(-16.529286,14.085191),(-16.503326,14.116278),(-16.489125,14.128241),(-16.457021,14.137356),(-16.441477,14.156562),(-16.429351,14.160956),(-16.42101,14.157131),(-16.412994,14.149807),(-16.404693,14.145413),(-16.394887,14.150377),(-16.388824,14.154853),(-16.372629,14.162421),(-16.364735,14.167792),(-16.427724,14.170111),(-16.454742,14.180162),(-16.452952,14.20185),(-16.468739,14.193834),(-16.474599,14.179389),(-16.471506,14.164252),(-16.460357,14.15412),(-16.472768,14.147366),(-16.483062,14.148139),(-16.493072,14.151842),(-16.50475,14.15412),(-16.513173,14.151028),(-16.516754,14.143459),(-16.518788,14.134263),(-16.52184,14.126166),(-16.547515,14.081855),(-16.556549,14.071519),(-16.566803,14.066962),(-16.58845,14.061957),(-16.600331,14.054185),(-16.60912,14.043199),(-16.621571,14.019965),(-16.631703,14.010077),(-16.654164,14.002753),(-16.675404,14.006903),(-16.697011,14.014228),(-16.721059,14.016303),(-16.697581,14.05744),(-16.696848,14.064683),(-16.680002,14.069403),(-16.652252,14.099514),(-16.665639,14.095649),(-16.678578,14.076117),(-16.689687,14.071519),(-16.711781,14.070868),(-16.72053,14.068915),(-16.727895,14.064683),(-16.716135,14.051337),(-16.717112,14.042385),(-16.73412,14.023139),(-16.73705,14.015448),(-16.739613,14.001125),(-16.7447,13.992743),(-16.764556,13.970608),(-16.76887,13.961656),(-16.773427,13.93358),(-16.76887,13.838813),(-16.775054,13.838813),(-16.780995,14.030748),(-16.786,14.059719),(-16.802561,14.107856),(-16.82079,14.143459),(-16.829701,14.15412),(-16.842763,14.160061),(-16.868804,14.165758),(-16.878733,14.174547),(-16.883901,14.188951),(-16.892405,14.242865),(-16.898915,14.259955),(-16.906972,14.272528),(-16.925893,14.291246),(-16.93456,14.295885),(-16.941151,14.297268),(-16.945465,14.301418),(-16.947011,14.314521),(-16.94579,14.324774),(-16.940745,14.344428),(-16.939565,14.355862),(-16.944447,14.372382),(-16.956532,14.390123),(-16.987945,14.421047),(-17.059478,14.453925),(-17.070465,14.465399),(-17.076161,14.478583),(-17.11148,14.530219),(-17.137847,14.599026),(-17.178212,14.653144),(-17.206939,14.6789),(-17.22411,14.690416),(-17.243764,14.699286),(-17.265452,14.702826),(-17.281646,14.707261),(-17.324045,14.728909),(-17.330556,14.736314),(-17.363596,14.735907),(-17.392323,14.739325),(-17.417388,14.737698),(-17.439809,14.722073),(-17.431386,14.71247),(-17.419342,14.70897),(-17.426829,14.694403),(-17.439809,14.688463),(-17.436391,14.685289),(-17.430247,14.678046),(-17.426829,14.674872),(-17.433013,14.660549),(-17.436391,14.659573),(-17.439809,14.653754),(-17.452952,14.662055),(-17.520172,14.743069),(-17.536041,14.757392),(-17.370961,14.800279),(-17.322133,14.839016),(-17.172963,14.900824),(-17.138661,14.92772),(-16.979726,15.111884),(-16.947906,15.157864),(-16.878245,15.2331),(-16.739166,15.472398),(-16.669667,15.561835),(-16.53893,15.774359),(-16.534291,15.784369),(-16.536,15.794664),(-16.542348,15.808824),(-16.542346,15.808848),(-16.519736,15.846372),(-16.509918,15.901511),(-16.507541,16.010393),(-16.501598,16.021866),(-16.4878,16.066049),(-16.470437,16.106357),(-16.465993,16.174053),(-16.458758,16.194155),(-16.444185,16.206299),(-16.407288,16.214825),(-16.391682,16.224747),(-16.385378,16.234927),(-16.381088,16.246658),(-16.340161,16.415898),(-16.334941,16.455793),(-16.327397,16.474551),(-16.313547,16.49331),(-16.295099,16.510621),(-16.273808,16.523179),(-16.251019,16.527623),(-16.217946,16.521525),(-16.206732,16.52075),(-16.195363,16.522714),(-16.185183,16.527313),(-16.155779,16.547518),(-16.144307,16.552686),(-16.132266,16.554753),(-16.119657,16.552376),(-16.110459,16.54695),(-16.103069,16.53956),(-16.090977,16.522197),(-16.074027,16.504885),(-16.05377,16.495428),(-16.031187,16.492896),(-16.007313,16.496152),(-16.000181,16.501061),(-15.993102,16.504627),(-15.985609,16.505867),(-15.977289,16.503955),(-15.96437,16.498994),(-15.958065,16.498426),(-15.950675,16.499821),(-15.91104,16.516874),(-15.896932,16.518424),(-15.861999,16.51026),(-15.853575,16.51026),(-15.828926,16.514549),(-15.812958,16.513773),(-15.767741,16.499459),(-15.735288,16.497702),(-15.726813,16.495273),(-15.703197,16.484421),(-15.687539,16.480546),(-15.672191,16.481372),(-15.657412,16.486178),(-15.643614,16.494653),(-15.621497,16.514239),(-15.609663,16.521318),(-15.594418,16.523644),(-15.551475,16.515117),(-15.536696,16.516512),(-15.52481,16.522197),(-15.515198,16.53093),(-15.49091,16.56452),(-15.478508,16.575165),(-15.463987,16.58147),(-15.448846,16.580178),(-15.43944,16.572685),(-15.433704,16.562504),(-15.426883,16.553151),(-15.414532,16.548035),(-15.399805,16.548345),(-15.385749,16.551756),(-15.343632,16.568706),(-15.328026,16.571961),(-15.295935,16.573046),(-15.248599,16.565812),(-15.232476,16.56638),(-15.125093,16.58271),(-15.104009,16.590358),(-15.097498,16.594492),(-15.093208,16.600176),(-15.092847,16.607618),(-15.096154,16.613974),(-15.112329,16.629219),(-15.118271,16.642603),(-15.116256,16.655522),(-15.107574,16.665392),(-15.093312,16.669165),(-15.078894,16.664979),(-15.069851,16.655264),(-15.061996,16.643791),(-15.051557,16.63418),(-15.044374,16.631906),(-15.036933,16.632578),(-15.029853,16.635471),(-15.023652,16.639967),(-15.018743,16.646117),(-15.016521,16.652576),(-15.013368,16.666581),(-15.006909,16.675831),(-14.996625,16.682445),(-14.974301,16.691385),(-14.962932,16.685804),(-14.949858,16.672317),(-14.948934,16.670553),(-14.941331,16.656039),(-14.930944,16.643378),(-14.912496,16.640639)] +Serranilla Bank [(-78.637074,15.862087),(-78.640411,15.864),(-78.636871,15.867296),(-78.637074,15.862087)] +Singapore [(103.960785,1.391099),(103.985688,1.385443),(103.999522,1.380316),(104.003429,1.374172),(103.991873,1.354926),(103.974864,1.334459),(103.954356,1.318101),(103.931895,1.311469),(103.907237,1.308743),(103.887706,1.301256),(103.852712,1.277289),(103.846934,1.271918),(103.844086,1.2685),(103.838878,1.266262),(103.82602,1.264309),(103.801606,1.264797),(103.789561,1.26789),(103.784434,1.273871),(103.77589,1.287584),(103.755138,1.297105),(103.730154,1.302924),(103.708751,1.305243),(103.665294,1.304104),(103.647634,1.308417),(103.640391,1.322252),(103.644705,1.338039),(103.674571,1.380316),(103.678884,1.399237),(103.683849,1.409898),(103.695079,1.421332),(103.708344,1.429389),(103.717947,1.430976),(103.739757,1.428127),(103.762218,1.430976),(103.79005,1.444281),(103.804942,1.448635),(103.831554,1.447089),(103.857188,1.438707),(103.932465,1.401109),(103.960785,1.391099)] +San Marino [(12.42945,43.892056),(12.399581,43.903218),(12.385629,43.924534),(12.395654,43.948409),(12.411049,43.959661),(12.421389,43.967219),(12.453325,43.979053),(12.48216,43.982567),(12.489188,43.97311),(12.492392,43.956419),(12.490325,43.939159),(12.483094,43.929205),(12.48216,43.927919),(12.479576,43.9258),(12.478026,43.923216),(12.477494,43.920051),(12.478287,43.917038),(12.460456,43.895259),(12.42945,43.892056)] +Somaliland [(48.939112,11.24913),(48.939111,11.136737),(48.939111,11.024367),(48.939111,10.912022),(48.939111,10.7996),(48.939111,10.687333),(48.939111,10.574937),(48.939111,10.462567),(48.939111,10.350119),(48.939111,10.2378),(48.939111,10.12543),(48.939111,10.013033),(48.939111,9.900715),(48.939111,9.788422),(48.939111,9.676),(48.939111,9.563603),(48.939111,9.562701),(48.939111,9.451233),(48.879114,9.360308),(48.819325,9.269384),(48.759225,9.178511),(48.699229,9.087689),(48.639336,8.996661),(48.579184,8.90584),(48.519188,8.814915),(48.459243,8.723887),(48.399299,8.633118),(48.339147,8.54209),(48.279099,8.451165),(48.219155,8.360318),(48.159003,8.269419),(48.099058,8.17852),(48.039114,8.087492),(47.979169,7.996567),(47.836697,7.996516),(47.653918,7.996516),(47.52359,7.996516),(47.36608,7.996516),(47.210586,7.996516),(47.068114,7.996516),(46.97923,7.996567),(46.920526,8.025609),(46.857377,8.046745),(46.773558,8.074909),(46.689532,8.102969),(46.605713,8.131133),(46.521687,8.159245),(46.437713,8.187357),(46.353842,8.215469),(46.269816,8.243555),(46.185894,8.271641),(46.101919,8.299727),(46.018049,8.327891),(45.934126,8.356003),(45.850203,8.384115),(45.766384,8.412227),(45.682358,8.440313),(45.598436,8.468425),(45.514462,8.496537),(45.51441,8.496537),(45.426147,8.525476),(45.33809,8.554414),(45.249878,8.583276),(45.16177,8.612137),(45.073558,8.641153),(44.985553,8.67004),(44.89729,8.698928),(44.809233,8.72784),(44.721073,8.756779),(44.633017,8.785718),(44.544753,8.814605),(44.456697,8.843544),(44.368537,8.872457),(44.280377,8.901396),(44.192217,8.930283),(44.104057,8.959222),(44.023855,8.985525),(43.984788,9.008314),(43.914921,9.071489),(43.78759,9.186701),(43.698449,9.267162),(43.621502,9.336899),(43.60724,9.344625),(43.59122,9.343591),(43.566829,9.334315),(43.548225,9.336072),(43.471227,9.382012),(43.419034,9.413018),(43.406322,9.42865),(43.401981,9.447409),(43.399449,9.480947),(43.393351,9.49893),(43.370665,9.544225),(43.361518,9.553165),(43.341364,9.566394),(43.331753,9.575076),(43.325086,9.585902),(43.315578,9.607658),(43.305966,9.617554),(43.297698,9.621533),(43.270671,9.628354),(43.248812,9.652332),(43.235273,9.691787),(43.206334,9.851209),(43.187214,9.883326),(43.149852,9.899991),(43.104273,9.907923),(43.067686,9.922522),(43.039884,9.949988),(43.020764,9.996419),(43.012599,10.029286),(42.999887,10.061971),(42.981903,10.091633),(42.958235,10.115559),(42.862324,10.177158),(42.836279,10.208086),(42.808167,10.269116),(42.78946,10.333324),(42.776644,10.42461),(42.765792,10.451947),(42.750393,10.471636),(42.695202,10.524708),(42.668537,10.56623),(42.64859,10.611033),(42.647247,10.63222),(42.660269,10.641625),(42.680216,10.647672),(42.699646,10.65855),(42.711635,10.675163),(42.718973,10.694852),(42.728585,10.735496),(42.738094,10.759784),(42.749876,10.775829),(42.785636,10.804406),(42.801242,10.820788),(42.80765,10.83611),(42.811578,10.85262),(42.819432,10.872567),(42.834832,10.888406),(42.876793,10.905744),(42.893536,10.919671),(42.901185,10.936595),(42.911313,10.977367),(42.923715,10.998787),(43.188815,11.407764),(43.240733,11.48786),(43.252696,11.47309),(43.266938,11.462348),(43.288341,11.460354),(43.282725,11.498928),(43.29713,11.482245),(43.320079,11.446112),(43.33961,11.426174),(43.352387,11.419745),(43.375987,11.391588),(43.394216,11.385199),(43.413829,11.380927),(43.44337,11.359565),(43.459727,11.35106),(43.466319,11.367906),(43.478201,11.37995),(43.489431,11.380927),(43.494395,11.364732),(43.479015,11.271796),(43.488129,11.238918),(43.506358,11.20897),(43.529307,11.186591),(43.535655,11.183743),(43.551931,11.179836),(43.559093,11.176011),(43.562999,11.170315),(43.568207,11.155422),(43.583181,11.135159),(43.591645,11.107489),(43.617442,11.070787),(43.658946,10.987982),(43.755626,10.894232),(43.8296,10.802965),(43.92213,10.724189),(44.028168,10.652167),(44.097504,10.58926),(44.149587,10.559149),(44.274262,10.456732),(44.303722,10.439846),(44.347554,10.414511),(44.3892,10.397207),(44.465258,10.401481),(44.541423,10.392743),(44.586149,10.383585),(44.627329,10.390613),(44.661857,10.402411),(44.721063,10.419384),(44.750662,10.427865),(44.790324,10.423484),(44.854784,10.414529),(44.899366,10.411814),(44.962009,10.415799),(44.994151,10.442694),(44.987966,10.460395),(45.001475,10.460395),(45.111339,10.52558),(45.12672,10.531968),(45.157237,10.541083),(45.252289,10.600531),(45.320567,10.662095),(45.33725,10.670844),(45.354747,10.66885),(45.37322,10.662421),(45.392426,10.658393),(45.40919,10.659491),(45.445323,10.667792),(45.487071,10.684719),(45.516612,10.701972),(45.542979,10.721666),(45.573009,10.758124),(45.586111,10.765855),(45.662283,10.790595),(45.673106,10.79621),(45.684906,10.808905),(45.694591,10.822252),(45.706391,10.832831),(45.72462,10.837144),(45.737966,10.843573),(45.753591,10.871731),(45.769298,10.878119),(45.787364,10.877753),(45.802745,10.875312),(45.816905,10.869208),(45.831309,10.857611),(45.84018,10.844672),(45.84669,10.832261),(45.855642,10.823961),(45.872325,10.823472),(45.868175,10.841376),(45.884451,10.841213),(45.97755,10.79914),(46.012543,10.794745),(46.047618,10.785305),(46.063731,10.778795),(46.102224,10.770657),(46.137218,10.776313),(46.207367,10.79621),(46.237559,10.792426),(46.263194,10.776842),(46.30714,10.731024),(46.335134,10.709703),(46.370128,10.696845),(46.408702,10.691596),(46.447765,10.693101),(46.482432,10.701361),(46.499848,10.708482),(46.52768,10.726223),(46.56072,10.731676),(46.588715,10.743313),(46.629161,10.745063),(46.646251,10.747748),(46.659679,10.75434),(46.694102,10.781928),(46.782237,10.821723),(46.821951,10.852607),(46.865977,10.871894),(46.871837,10.878079),(46.881358,10.893134),(46.886485,10.898586),(46.893565,10.901557),(46.9074,10.903144),(46.913829,10.905463),(47.008067,10.938951),(47.082774,10.995266),(47.132335,11.035793),(47.167654,11.072089),(47.180675,11.080471),(47.226736,11.093817),(47.23878,11.101264),(47.263357,11.120795),(47.366873,11.172797),(47.386241,11.180121),(47.405935,11.184638),(47.437022,11.179836),(47.489513,11.192816),(47.522716,11.186957),(47.677745,11.107978),(47.71046,11.101264),(47.767345,11.130194),(47.892833,11.132554),(47.948253,11.125149),(48.130382,11.136664),(48.160818,11.145657),(48.173513,11.157131),(48.195811,11.186835),(48.215587,11.196234),(48.217296,11.203681),(48.2199,11.211127),(48.22934,11.214504),(48.246593,11.21426),(48.255138,11.215074),(48.273611,11.220404),(48.2942,11.222602),(48.30421,11.227525),(48.310557,11.234768),(48.313324,11.242011),(48.314952,11.24901),(48.31837,11.25552),(48.331309,11.268704),(48.342784,11.275458),(48.373057,11.282172),(48.444347,11.289252),(48.478526,11.296332),(48.506847,11.313544),(48.521983,11.319281),(48.585948,11.316962),(48.636892,11.329413),(48.654796,11.330634),(48.723888,11.315253),(48.762462,11.301011),(48.800629,11.279039),(48.888845,11.250556),(48.904063,11.247748),(48.921153,11.248033),(48.939112,11.24913)] +Somalia [(50.797864,11.989119),(50.867055,11.942891),(50.970083,11.932323),(51.029307,11.885077),(51.048839,11.878648),(51.127696,11.878648),(51.147227,11.873603),(51.181163,11.855373),(51.199718,11.851304),(51.222911,11.850775),(51.247081,11.847561),(51.267345,11.839423),(51.292698,11.833108),(51.277354,11.800605),(51.266381,11.759286),(51.249359,11.728775),(51.255708,11.681655),(51.247801,11.652391),(51.22954,11.635897),(51.212529,11.607947),(51.181245,11.578829),(51.148123,11.536038),(51.124766,11.511786),(51.12086,11.505316),(51.121918,11.494127),(51.126638,11.478827),(51.127696,11.470893),(51.125011,11.447984),(51.118419,11.426256),(51.086192,11.357164),(51.081309,11.341498),(51.079356,11.275946),(51.07252,11.233791),(51.085948,11.187974),(51.130945,11.160752),(51.17444,11.158764),(51.189504,11.138306),(51.164399,11.116767),(51.142263,11.068427),(51.131358,11.052883),(51.12379,11.038764),(51.118337,11.016913),(51.115082,10.993354),(51.124522,10.85456),(51.138194,10.676744),(51.160655,10.598456),(51.154307,10.587714),(51.131358,10.58393),(51.107432,10.576565),(51.101899,10.55858),(51.107188,10.515041),(51.102224,10.494696),(51.092296,10.482164),(51.076915,10.475816),(51.055675,10.474066),(51.035818,10.467515),(51.018321,10.452094),(51.010102,10.434027),(51.018403,10.41942),(51.038097,10.416083),(51.19044,10.446112),(51.200938,10.44477),(51.211925,10.44184),(51.220388,10.442043),(51.223888,10.449856),(51.219574,10.461005),(51.210216,10.467108),(51.200857,10.471503),(51.196544,10.477444),(51.195811,10.500312),(51.193044,10.521186),(51.186778,10.541571),(51.176117,10.562812),(51.191742,10.548977),(51.20734,10.538096),(51.220647,10.524928),(51.237966,10.515024),(51.24662,10.50909),(51.257942,10.503801),(51.280556,10.48733),(51.292532,10.480075),(51.3112,10.476055),(51.325879,10.47268),(51.335871,10.474614),(51.345875,10.480494),(51.35721,10.48374),(51.377856,10.48169),(51.395173,10.479023),(51.403077,10.456707),(51.417038,10.447485),(51.410965,10.429814),(51.397509,10.395793),(51.38502,10.391506),(51.380154,10.387339),(51.372569,10.381415),(51.37608,10.367693),(51.367419,10.367726),(51.349458,10.375002),(51.336131,10.374395),(51.320079,10.378485),(51.300831,10.372566),(51.272873,10.377905),(51.265391,10.396877),(51.26295,10.408596),(51.256602,10.420315),(51.240977,10.425605),(51.090099,10.407313),(51.017125,10.381916),(50.927389,10.327844),(50.90988,10.295635),(50.901703,10.234442),(50.912814,10.176853),(50.8838,10.101264),(50.888682,10.05565),(50.900401,10.024075),(50.901703,10.011664),(50.89975,10.000718),(50.890798,9.986029),(50.877696,9.916653),(50.881196,9.905463),(50.86964,9.884467),(50.863129,9.866034),(50.854015,9.816107),(50.840587,9.779771),(50.833263,9.768785),(50.838634,9.740912),(50.836925,9.73078),(50.831716,9.712063),(50.824474,9.651068),(50.806163,9.624457),(50.805512,9.59394),(50.812999,9.541164),(50.836762,9.477037),(50.840343,9.456122),(50.837413,9.437201),(50.830333,9.419623),(50.768321,9.319281),(50.748302,9.303534),(50.706309,9.285712),(50.689464,9.274848),(50.676768,9.259182),(50.646983,9.211371),(50.641124,9.196357),(50.649181,9.132514),(50.648448,9.110419),(50.633637,9.073188),(50.604503,9.037909),(50.494884,8.951972),(50.464122,8.920559),(50.439708,8.88817),(50.422048,8.857123),(50.40797,8.816352),(50.389822,8.733547),(50.374278,8.698879),(50.326508,8.641099),(50.319591,8.620347),(50.322602,8.55976),(50.319591,8.541815),(50.303477,8.504584),(50.290212,8.486762),(50.274913,8.479153),(50.256196,8.472561),(50.246104,8.456448),(50.239106,8.436347),(50.230154,8.41767),(50.166026,8.33336),(50.156423,8.320624),(50.141368,8.287991),(50.113048,8.19953),(50.091075,8.161038),(50.058279,8.130316),(49.929373,8.051744),(49.842052,7.962714),(49.824229,7.933905),(49.812511,7.897528),(49.803233,7.82689),(49.808604,7.815009),(49.823416,7.805325),(49.827647,7.782904),(49.825043,7.757758),(49.819347,7.739936),(49.790782,7.69953),(49.760916,7.667304),(49.752208,7.650295),(49.748302,7.606635),(49.744151,7.589057),(49.640798,7.409125),(49.586599,7.314765),(49.520518,7.239651),(49.367035,7.025702),(49.293224,6.880927),(49.247325,6.810614),(49.197927,6.67357),(49.074718,6.415269),(49.072032,6.393297),(49.074067,6.370795),(49.083507,6.329047),(49.085623,6.307359),(49.0713,6.219916),(49.036143,6.144232),(48.951491,5.99872),(48.850108,5.824449),(48.695811,5.587348),(48.646739,5.480048),(48.620128,5.443996),(48.548106,5.375067),(48.448009,5.208726),(48.328868,5.079657),(48.202403,4.908922),(48.055675,4.612942),(47.948497,4.457099),(47.840587,4.341376),(47.577973,4.059882),(47.489268,3.936347),(47.335623,3.778754),(47.206716,3.644599),(47.039806,3.469265),(47.01295,3.441067),(46.834239,3.232489),(46.700043,3.113267),(46.625743,3.023912),(46.487804,2.90884),(46.414317,2.833441),(46.351573,2.788642),(46.34962,2.780178),(46.3449,2.770087),(46.339203,2.761705),(46.334483,2.758246),(46.317638,2.751899),(46.307628,2.736029),(46.299815,2.715766),(46.289399,2.696234),(46.275401,2.685858),(46.236501,2.667629),(46.142345,2.568915),(46.13087,2.554023),(46.114513,2.523261),(46.104991,2.510565),(46.062999,2.484036),(46.047618,2.455064),(46.027029,2.438137),(45.826345,2.308824),(45.711925,2.24433),(45.605642,2.183051),(45.266205,1.987291),(45.232188,1.967678),(45.210704,1.963121),(45.199718,1.959621),(45.001475,1.86636),(44.831554,1.749701),(44.823985,1.740302),(44.812185,1.735582),(44.67628,1.6376),(44.550059,1.559068),(44.332774,1.38996),(44.221365,1.275621),(44.198416,1.265692),(44.157888,1.24079),(44.145518,1.22956),(44.1421,1.221381),(44.139659,1.211168),(44.138031,1.191596),(44.133962,1.187567),(44.115001,1.182034),(44.11085,1.17829),(44.10377,1.168362),(44.032888,1.09748),(44.012543,1.084215),(43.990977,1.078599),(43.97877,1.068671),(43.932628,1.009833),(43.819835,0.949449),(43.781912,0.921617),(43.734223,0.854967),(43.717296,0.845282),(43.703461,0.840522),(43.658946,0.804918),(43.467621,0.620551),(43.285492,0.414984),(43.184906,0.31627),(43.09018,0.223375),(42.891124,0.003079),(42.770274,-0.129327),(42.712087,-0.17669),(42.647797,-0.228936),(42.580333,-0.29754),(42.555919,-0.332127),(42.553559,-0.34238),(42.553966,-0.351251),(42.550792,-0.357599),(42.526622,-0.362726),(42.519298,-0.369317),(42.479747,-0.422052),(42.474457,-0.438653),(42.486501,-0.449395),(42.47641,-0.465997),(42.47283,-0.469903),(42.472911,-0.4638),(42.472423,-0.458673),(42.47047,-0.454034),(42.465994,-0.449395),(42.412608,-0.490981),(42.404552,-0.500909),(42.39796,-0.517348),(42.38266,-0.535252),(42.349864,-0.565606),(42.250255,-0.685642),(42.246755,-0.694268),(42.246755,-0.701918),(42.245291,-0.707452),(42.236339,-0.709568),(42.232595,-0.713067),(42.192231,-0.776137),(42.166515,-0.802667),(42.151622,-0.813897),(42.135509,-0.82171),(42.116466,-0.826267),(42.086599,-0.825616),(42.071544,-0.828709),(42.062511,-0.839288),(42.081716,-0.843438),(42.080903,-0.862888),(42.071951,-0.884454),(42.065684,-0.894464),(42.051443,-0.902032),(42.036957,-0.920017),(42.025564,-0.941583),(42.020844,-0.960056),(42.016368,-0.968032),(41.983165,-0.994073),(41.971934,-0.99977),(41.967947,-0.992852),(41.963634,-0.95086),(41.964692,-0.911554),(41.958832,-0.894464),(41.95281,-0.894464),(41.955414,-0.909601),(41.952403,-0.920831),(41.94752,-0.930841),(41.94516,-0.942966),(41.946462,-0.954767),(41.951427,-0.969415),(41.95281,-0.980401),(41.955577,-0.992934),(41.96225,-0.998956),(41.969249,-1.003188),(41.973155,-1.011163),(41.97169,-1.023614),(41.961436,-1.044122),(41.958832,-1.055841),(41.953461,-1.068617),(41.930024,-1.089939),(41.924653,-1.097101),(41.923025,-1.110121),(41.918305,-1.121026),(41.911632,-1.130955),(41.873871,-1.176039),(41.869477,-1.186212),(41.872895,-1.195489),(41.880707,-1.19492),(41.890147,-1.189386),(41.897472,-1.183201),(41.885427,-1.209649),(41.862804,-1.212335),(41.84197,-1.196954),(41.835297,-1.168878),(41.829112,-1.168878),(41.830251,-1.189223),(41.84197,-1.224867),(41.842784,-1.24391),(41.836599,-1.262302),(41.73227,-1.431085),(41.708751,-1.460056),(41.689708,-1.489516),(41.65211,-1.565118),(41.633311,-1.580336),(41.610606,-1.593194),(41.575043,-1.652602),(41.554861,-1.669692),(41.561046,-1.669692),(41.535105,-1.696283),(41.535103,-1.696269),(41.535944,-1.676312),(41.53858,-1.613699),(41.534963,-1.594785),(41.522871,-1.572771),(41.484527,-1.523162),(41.434194,-1.458049),(41.383758,-1.39304),(41.332185,-1.326378),(41.282989,-1.262816),(41.237307,-1.203905),(41.18222,-1.132695),(41.13106,-1.066445),(41.081554,-1.002573),(41.056129,-0.969604),(41.030601,-0.936738),(41.00528,-0.903768),(40.979751,-0.870798),(40.978821,-0.794834),(40.977891,-0.724347),(40.976961,-0.653861),(40.975514,-0.545237),(40.974067,-0.436716),(40.97262,-0.328299),(40.971173,-0.219675),(40.97045,-0.152186),(40.969623,-0.084697),(40.968486,-0.002738),(40.968486,0.122319),(40.968486,0.247273),(40.968486,0.37233),(40.968486,0.497283),(40.968486,0.62234),(40.968486,0.747242),(40.968486,0.872248),(40.968486,0.997201),(40.968486,1.122207),(40.968486,1.230295),(40.968486,1.247212),(40.968486,1.372191),(40.968486,1.497145),(40.968486,1.62215),(40.968486,1.747181),(40.968486,1.872135),(40.968486,1.997166),(40.968486,2.103619),(40.968486,2.210047),(40.968486,2.316501),(40.968486,2.422954),(40.968486,2.49711),(40.967142,2.631727),(40.965902,2.760608),(40.965385,2.814145),(40.979751,2.841895),(41.013961,2.875795),(41.096127,2.957082),(41.178396,3.03842),(41.260665,3.119811),(41.342727,3.201149),(41.394506,3.27515),(41.446183,3.349047),(41.497963,3.423074),(41.549639,3.497023),(41.633562,3.617093),(41.717381,3.737163),(41.801303,3.857182),(41.885019,3.977226),(41.898455,3.996915),(41.912201,4.007974),(41.917472,4.020453),(41.916645,4.051098),(41.923466,4.070554),(41.941243,4.086212),(42.011833,4.129491),(42.068367,4.174553),(42.102887,4.19189),(42.13534,4.200081),(42.221846,4.200959),(42.297294,4.201734),(42.415013,4.221707),(42.568285,4.247856),(42.718663,4.273487),(42.78977,4.285605),(42.831628,4.302322),(42.868938,4.326765),(42.870195,4.328171),(42.899634,4.361104),(42.914724,4.393299),(42.9325,4.463217),(42.945936,4.496858),(42.952551,4.507581),(42.960406,4.517374),(43.03544,4.578895),(43.119259,4.647702),(43.232792,4.701497),(43.346997,4.755654),(43.459342,4.808829),(43.528485,4.841566),(43.640519,4.867353),(43.716484,4.884793),(43.815186,4.907479),(43.845158,4.914352),(43.932284,4.945203),(43.968975,4.953962),(44.029126,4.950448),(44.081009,4.947451),(44.132789,4.94448),(44.184672,4.941431),(44.236452,4.938485),(44.288335,4.935488),(44.340115,4.932439),(44.391998,4.929468),(44.443726,4.926445),(44.495557,4.923473),(44.547441,4.920476),(44.596808,4.917643),(44.59922,4.917505),(44.651104,4.914507),(44.702883,4.911562),(44.754766,4.908565),(44.806546,4.905541),(44.858378,4.902544),(44.912586,4.899392),(44.941525,4.911484),(45.020642,4.99688),(45.0209,4.99688),(45.077744,5.059072),(45.185024,5.176248),(45.292356,5.293476),(45.399585,5.410704),(45.506865,5.527881),(45.518477,5.540564),(45.614146,5.645057),(45.721426,5.762337),(45.828706,5.879513),(45.935986,5.996689),(45.99681,6.059192),(46.057839,6.12172),(46.118818,6.184223),(46.179899,6.246726),(46.240981,6.309228),(46.302062,6.371731),(46.362937,6.43426),(46.423915,6.496736),(46.466963,6.538292),(46.488046,6.558645),(46.508406,6.578308),(46.55321,6.621457),(46.598065,6.664633),(46.618425,6.684244),(46.667001,6.731115),(46.715577,6.777985),(46.764101,6.824882),(46.812729,6.871675),(46.861408,6.918597),(46.909984,6.965467),(46.95856,7.012286),(47.007032,7.059208),(47.055608,7.106079),(47.104287,7.153001),(47.152863,7.19982),(47.201439,7.246691),(47.250118,7.293509),(47.298797,7.34038),(47.347373,7.387302),(47.395949,7.434173),(47.444525,7.481043),(47.493101,7.527914),(47.532388,7.565864),(47.541676,7.574836),(47.590252,7.621655),(47.638932,7.668525),(47.687507,7.715396),(47.736032,7.762215),(47.784659,7.809137),(47.833183,7.855956),(47.881811,7.902826),(47.930593,7.949697),(47.979169,7.996567),(48.039114,8.087492),(48.099058,8.17852),(48.159003,8.269419),(48.219155,8.360318),(48.279099,8.451165),(48.339147,8.54209),(48.399299,8.633118),(48.459243,8.723887),(48.519188,8.814915),(48.579184,8.90584),(48.639336,8.996661),(48.699229,9.087689),(48.759225,9.178511),(48.819325,9.269384),(48.879114,9.360308),(48.939111,9.451233),(48.939111,9.562701),(48.939111,9.563603),(48.939111,9.676),(48.939111,9.788422),(48.939111,9.900715),(48.939111,10.013033),(48.939111,10.12543),(48.939111,10.2378),(48.939111,10.350119),(48.939111,10.462567),(48.939111,10.574937),(48.939111,10.687333),(48.939111,10.7996),(48.939111,10.912022),(48.939111,11.024367),(48.939111,11.136737),(48.939112,11.24913),(48.939138,11.249132),(48.976817,11.25141),(49.108409,11.276597),(49.239594,11.300279),(49.278087,11.316962),(49.293305,11.337958),(49.303477,11.345445),(49.315929,11.340522),(49.32545,11.33511),(49.335704,11.334621),(49.400076,11.340522),(49.4192,11.344794),(49.435069,11.35106),(49.488617,11.381741),(49.507335,11.385199),(49.521739,11.393785),(49.544444,11.436713),(49.558604,11.453518),(49.576834,11.460028),(49.59962,11.461737),(49.644542,11.460354),(49.650076,11.462144),(49.666026,11.471137),(49.675304,11.473944),(49.686778,11.474351),(49.716319,11.46776),(49.804861,11.458401),(49.851248,11.464911),(49.880707,11.488227),(49.88852,11.487291),(49.906423,11.498725),(49.917979,11.501899),(49.921886,11.503852),(49.934418,11.512885),(49.942149,11.51557),(49.949474,11.51496),(49.956309,11.512519),(49.961111,11.509955),(49.962657,11.508734),(50.053884,11.511379),(50.07252,11.508734),(50.131521,11.539944),(50.151134,11.557929),(50.161876,11.563381),(50.197602,11.562649),(50.206554,11.566799),(50.214366,11.572659),(50.225434,11.578111),(50.268321,11.589301),(50.28891,11.602444),(50.372325,11.668931),(50.418142,11.678046),(50.437022,11.690619),(50.470388,11.727851),(50.500824,11.754828),(50.512706,11.771226),(50.517589,11.79267),(50.518891,11.815253),(50.52296,11.837958),(50.52947,11.859524),(50.538585,11.878648),(50.561209,11.907904),(50.575938,11.920966),(50.589854,11.926459),(50.602061,11.927965),(50.627452,11.940131),(50.632091,11.946763),(50.636485,11.951606),(50.641124,11.954413),(50.647146,11.954047),(50.655935,11.948432),(50.661469,11.947577),(50.682953,11.953355),(50.724864,11.971137),(50.761567,11.978746),(50.771983,11.985785),(50.797864,11.989119)] +Republic of Serbia [(19.690095,46.168398),(19.711889,46.15871),(19.772984,46.131552),(19.79045,46.129072),(19.873649,46.152992),(19.888946,46.15739),(19.92915,46.16354),(19.993125,46.159406),(20.034983,46.142973),(20.063405,46.145298),(20.088623,46.154135),(20.098442,46.154962),(20.114772,46.152223),(20.120146,46.149226),(20.130378,46.139355),(20.138026,46.136461),(20.145364,46.137082),(20.170479,46.145505),(20.188462,46.140389),(20.242826,46.108091),(20.305665,46.053572),(20.317653,46.038586),(20.338634,45.992801),(20.35393,45.976678),(20.370984,45.96779),(20.410258,45.955594),(20.429068,45.946706),(20.481674,45.912703),(20.499865,45.906656),(20.538002,45.903711),(20.556709,45.89844),(20.572108,45.887691),(20.605284,45.846143),(20.612312,45.841492),(20.629676,45.833276),(20.636704,45.827023),(20.640114,45.818703),(20.642285,45.798343),(20.64349,45.795141),(20.645902,45.788731),(20.655617,45.77731),(20.678562,45.75664),(20.68807,45.7431),(20.700059,45.735401),(20.713392,45.733334),(20.726827,45.736176),(20.739436,45.743411),(20.745328,45.754986),(20.754113,45.763564),(20.765171,45.766768),(20.77747,45.762324),(20.785739,45.752557),(20.785532,45.743411),(20.781604,45.733954),(20.779124,45.72367),(20.779951,45.671684),(20.777264,45.657524),(20.773233,45.648894),(20.762174,45.630601),(20.754423,45.605589),(20.75804,45.58926),(20.787392,45.553655),(20.800208,45.530504),(20.797624,45.516499),(20.783155,45.506061),(20.760727,45.493348),(20.767135,45.479344),(20.781604,45.472574),(20.799484,45.468544),(20.816021,45.462859),(20.830387,45.452524),(20.863047,45.418728),(20.927642,45.37749),(20.966193,45.341575),(20.981489,45.33279),(21.016668,45.321482),(21.063964,45.30628),(21.0743,45.300544),(21.082775,45.293671),(21.09156,45.288089),(21.103342,45.286074),(21.113057,45.28933),(21.12918,45.301887),(21.139412,45.303696),(21.155638,45.295169),(21.189745,45.259564),(21.206075,45.245922),(21.239251,45.229385),(21.257131,45.224114),(21.299299,45.223184),(21.405546,45.199671),(21.433761,45.188819),(21.459393,45.17404),(21.484388,45.152702),(21.493292,45.145101),(21.49784,45.131872),(21.494119,45.119314),(21.48151,45.111563),(21.469004,45.111046),(21.458566,45.107325),(21.449987,45.101021),(21.442443,45.092908),(21.443786,45.091461),(21.444406,45.08526),(21.444406,45.07787),(21.443786,45.072909),(21.440996,45.068723),(21.432521,45.061488),(21.429524,45.057303),(21.425596,45.043247),(21.425079,45.036219),(21.421875,45.031413),(21.409266,45.023971),(21.398828,45.021387),(21.3733,45.020096),(21.363378,45.01653),(21.356143,45.008572),(21.351389,44.998236),(21.353456,44.989813),(21.366995,44.987281),(21.383945,44.986661),(21.387046,44.981545),(21.384565,44.97493),(21.385185,44.969504),(21.40844,44.958342),(21.456499,44.952348),(21.48151,44.943563),(21.51634,44.933899),(21.531016,44.924649),(21.539078,44.908474),(21.536287,44.889302),(21.522128,44.880776),(21.48151,44.872611),(21.453398,44.869562),(21.395314,44.871629),(21.368545,44.86486),(21.355523,44.856591),(21.346428,44.845636),(21.342811,44.831942),(21.3598,44.826657),(21.359841,44.826644),(21.360587,44.826412),(21.378571,44.816645),(21.395934,44.790239),(21.41216,44.784813),(21.49722,44.778043),(21.558405,44.78166),(21.578042,44.777681),(21.595612,44.765951),(21.604293,44.749983),(21.610288,44.731948),(21.619693,44.713913),(21.65628,44.687661),(21.705166,44.677067),(21.756842,44.677274),(21.801697,44.683889),(21.838284,44.695206),(21.855441,44.698461),(21.871977,44.695981),(21.962411,44.662288),(21.994347,44.658567),(22.004269,44.651539),(22.032174,44.603325),(22.034243,44.596168),(22.036174,44.589489),(22.039822,44.576867),(22.045197,44.569219),(22.055739,44.562139),(22.067107,44.55723),(22.076306,44.550718),(22.086951,44.52178),(22.104314,44.509636),(22.126639,44.502659),(22.148239,44.500902),(22.172114,44.505347),(22.185136,44.515113),(22.299031,44.661668),(22.304922,44.677377),(22.319702,44.685336),(22.361146,44.692054),(22.38068,44.700528),(22.41551,44.727814),(22.426052,44.733653),(22.450547,44.732981),(22.468943,44.730191),(22.483723,44.72399),(22.553279,44.669161),(22.587489,44.649369),(22.621182,44.637432),(22.700144,44.63061),(22.714716,44.623065),(22.765153,44.58281),(22.759262,44.564671),(22.741692,44.551855),(22.719367,44.544362),(22.700144,44.54183),(22.678543,44.545551),(22.642163,44.563121),(22.621182,44.569219),(22.600305,44.569787),(22.580461,44.565498),(22.565578,44.555421),(22.55979,44.538781),(22.55669,44.52147),(22.548422,44.511703),(22.535503,44.507517),(22.500363,44.50638),(22.491061,44.50421),(22.484136,44.499766),(22.479795,44.490412),(22.477212,44.476976),(22.477005,44.463954),(22.480002,44.455789),(22.500983,44.44194),(22.505427,44.435015),(22.506357,44.427522),(22.50398,44.411709),(22.505117,44.404061),(22.522687,44.37507),(22.549352,44.348974),(22.582838,44.328406),(22.621182,44.315901),(22.662523,44.311663),(22.68154,44.305307),(22.689498,44.291716),(22.685364,44.243657),(22.690739,44.228878),(22.69164,44.228435),(22.648777,44.213995),(22.639992,44.207329),(22.624799,44.189397),(22.608573,44.175858),(22.606196,44.174566),(22.604852,44.168468),(22.605989,44.163145),(22.607953,44.159993),(22.6094,44.159941),(22.599065,44.130331),(22.597101,44.119065),(22.598134,44.109298),(22.604646,44.088163),(22.604749,44.079378),(22.592967,44.063926),(22.57519,44.061394),(22.554623,44.062428),(22.534159,44.057157),(22.522583,44.044703),(22.514935,44.030285),(22.50367,44.019898),(22.481449,44.019433),(22.465885,44.017624),(22.43432,44.013955),(22.411789,44.006927),(22.399594,43.993336),(22.39732,43.980934),(22.396803,43.951944),(22.394529,43.936337),(22.391945,43.931867),(22.382024,43.918561),(22.379026,43.913496),(22.377063,43.883524),(22.367554,43.852751),(22.354738,43.829703),(22.349467,43.807921),(22.362593,43.780843),(22.388535,43.758286),(22.389568,43.750509),(22.385848,43.733817),(22.386054,43.725498),(22.390498,43.712449),(22.396906,43.699401),(22.404865,43.687179),(22.41396,43.676663),(22.426465,43.668214),(22.455921,43.656406),(22.466256,43.64912),(22.472871,43.635942),(22.473801,43.612998),(22.481449,43.600647),(22.481759,43.599975),(22.481966,43.599459),(22.481759,43.598942),(22.481449,43.598529),(22.478142,43.594911),(22.477108,43.591294),(22.478142,43.587573),(22.481449,43.584137),(22.482689,43.581734),(22.483103,43.579279),(22.482689,43.576695),(22.481449,43.574111),(22.478658,43.569176),(22.477625,43.564164),(22.478452,43.559229),(22.490647,43.540883),(22.509354,43.493341),(22.518863,43.474247),(22.532609,43.464842),(22.565785,43.453344),(22.57271,43.44815),(22.586766,43.43443),(22.596274,43.429159),(22.606919,43.427402),(22.62852,43.428255),(22.637822,43.426369),(22.645367,43.420297),(22.656529,43.403192),(22.658926,43.401295),(22.664694,43.396732),(22.674202,43.394148),(22.693219,43.394872),(22.702934,43.394045),(22.719367,43.388671),(22.724343,43.38606),(22.73301,43.381513),(22.80453,43.328984),(22.817139,43.315497),(22.820756,43.307539),(22.823857,43.289297),(22.826958,43.28139),(22.833159,43.274647),(22.857343,43.256947),(22.883802,43.230592),(22.897754,43.220335),(22.915531,43.212247),(22.964727,43.204418),(22.981367,43.198992),(22.982902,43.187318),(22.984571,43.174627),(22.974029,43.141192),(22.955632,43.108274),(22.935271,43.085562),(22.927107,43.081144),(22.910157,43.075279),(22.901682,43.069749),(22.896721,43.062721),(22.889486,43.044376),(22.884215,43.036651),(22.842254,43.007505),(22.829025,42.993656),(22.829025,42.993501),(22.828921,42.993449),(22.828818,42.993449),(22.815796,42.989703),(22.788097,42.984897),(22.776418,42.979729),(22.76939,42.97128),(22.763189,42.958645),(22.745516,42.910069),(22.739579,42.898858),(22.738798,42.897383),(22.727015,42.886892),(22.69663,42.87741),(22.666244,42.871932),(22.5909,42.886892),(22.563615,42.884283),(22.549972,42.877358),(22.544785,42.871706),(22.544494,42.871389),(22.53757,42.868341),(22.519793,42.870356),(22.506047,42.870123),(22.497055,42.864413),(22.481449,42.84674),(22.470907,42.840125),(22.445482,42.830178),(22.436801,42.824286),(22.430446,42.817077),(22.427395,42.813615),(22.425845,42.809843),(22.429359,42.806122),(22.45313,42.763592),(22.466566,42.748529),(22.481449,42.739821),(22.482586,42.736824),(22.482896,42.733775),(22.482586,42.730675),(22.481449,42.727677),(22.468116,42.718324),(22.442072,42.681685),(22.449203,42.667965),(22.444552,42.64329),(22.441318,42.632891),(22.428842,42.592776),(22.425328,42.572855),(22.429669,42.571408),(22.481449,42.535622),(22.512145,42.519189),(22.524857,42.507665),(22.532505,42.493557),(22.532505,42.493402),(22.536536,42.47839),(22.533125,42.45759),(22.519483,42.420926),(22.508838,42.404932),(22.497572,42.399196),(22.485066,42.397155),(22.46977,42.391703),(22.454371,42.376768),(22.438454,42.340052),(22.423985,42.325893),(22.405795,42.321552),(22.364144,42.320984),(22.345023,42.313439),(22.325283,42.314318),(22.307609,42.31933),(22.29159,42.328399),(22.276914,42.341241),(22.27402,42.348476),(22.273296,42.365477),(22.268852,42.370335),(22.259757,42.369095),(22.233402,42.348889),(22.095529,42.305817),(22.060906,42.301088),(22.045407,42.302424),(22.027627,42.303956),(21.994967,42.312612),(21.941534,42.333102),(21.929028,42.335117),(21.918279,42.331345),(21.88438,42.309512),(21.877352,42.30822),(21.837354,42.308556),(21.8172,42.305145),(21.719521,42.260957),(21.706509,42.25507),(21.69204,42.242022),(21.67695,42.234943),(21.676886,42.234945),(21.65969,42.235563),(21.624654,42.242772),(21.575044,42.242022),(21.564066,42.246289),(21.553857,42.273984),(21.514893,42.317832),(21.51603,42.341939),(21.537321,42.35863),(21.596645,42.372092),(21.617419,42.386639),(21.621863,42.402167),(21.616902,42.433923),(21.618969,42.449245),(21.627858,42.460381),(21.667855,42.490095),(21.717671,42.551151),(21.727697,42.574147),(21.726766,42.577945),(21.719738,42.586678),(21.718291,42.591019),(21.720152,42.593887),(21.726973,42.596393),(21.728833,42.598305),(21.730074,42.601018),(21.735551,42.621276),(21.734725,42.624247),(21.744543,42.629647),(21.756532,42.633626),(21.767074,42.638716),(21.772758,42.647501),(21.764387,42.669619),(21.74425,42.679425),(21.738652,42.68215),(21.708886,42.687189),(21.687389,42.686827),(21.644084,42.672306),(21.629408,42.672203),(21.612665,42.680393),(21.580522,42.710211),(21.565226,42.720184),(21.542488,42.725817),(21.441823,42.736101),(21.4178,42.73557),(21.405546,42.7353),(21.388699,42.739046),(21.378674,42.744136),(21.379191,42.747004),(21.383945,42.749976),(21.387149,42.754962),(21.39056,42.770414),(21.404099,42.803978),(21.408336,42.820747),(21.408419,42.841743),(21.40844,42.846998),(21.398931,42.854569),(21.378777,42.855292),(21.346738,42.860847),(21.336403,42.865473),(21.316972,42.877616),(21.306327,42.882448),(21.294958,42.884386),(21.2716,42.884283),(21.260542,42.886556),(21.23243,42.910896),(21.226745,42.942522),(21.225298,42.973554),(21.209795,42.995878),(21.193052,42.997893),(21.17941,42.990581),(21.165354,42.985103),(21.147887,42.992622),(21.139309,43.005826),(21.124012,43.058277),(21.10851,43.081558),(21.092593,43.090678),(21.025827,43.093366),(21.005157,43.099128),(20.993431,43.104148),(20.912494,43.138805),(20.838552,43.170467),(20.832041,43.178606),(20.836071,43.179433),(20.839069,43.192455),(20.840619,43.206976),(20.839999,43.212092),(20.851471,43.219818),(20.8616,43.217518),(20.8647,43.217337),(20.855088,43.231445),(20.848474,43.238137),(20.838449,43.245863),(20.819432,43.257412),(20.809717,43.25961),(20.79442,43.263071),(20.769512,43.260849),(20.745328,43.252865),(20.666986,43.209663),(20.644869,43.203307),(20.612312,43.202274),(20.604044,43.197959),(20.597533,43.184962),(20.600117,43.173826),(20.612106,43.154938),(20.620581,43.133337),(20.626058,43.123725),(20.632053,43.117266),(20.640941,43.115276),(20.65169,43.11631),(20.661818,43.115948),(20.669157,43.109799),(20.664919,43.085407),(20.643835,43.052257),(20.617273,43.02213),(20.596396,43.007092),(20.584304,43.006911),(20.572832,43.00934),(20.562393,43.009495),(20.553401,43.002596),(20.543893,42.987248),(20.538415,42.980659),(20.530457,42.975156),(20.51113,42.970195),(20.493457,42.970195),(20.476403,42.966371),(20.459454,42.950015),(20.450979,42.922032),(20.466688,42.909501),(20.488909,42.899191),(20.494367,42.887467),(20.498831,42.877875),(20.4763,42.855525),(20.428034,42.840642),(20.345352,42.827439),(20.355171,42.86617),(20.35362,42.890975),(20.337084,42.906969),(20.275589,42.929887),(20.223189,42.957663),(20.19456,42.966836),(20.14092,42.970815),(20.129464,42.973605),(20.116942,42.976654),(20.056344,43.020796),(20.054388,43.022221),(20.019894,43.047348),(19.959742,43.07856),(19.949821,43.085924),(19.942276,43.095639),(19.936591,43.105897),(19.92915,43.113907),(19.916644,43.117059),(19.907136,43.113261),(19.883778,43.095691),(19.872203,43.090368),(19.838303,43.088353),(19.805127,43.089955),(19.781769,43.096466),(19.761202,43.108739),(19.742805,43.126516),(19.713556,43.165609),(19.705494,43.166178),(19.69733,43.160209),(19.684617,43.156721),(19.663016,43.158504),(19.618678,43.168245),(19.598111,43.176203),(19.580851,43.187804),(19.549948,43.217518),(19.547845,43.218815),(19.512431,43.240643),(19.502303,43.251935),(19.485146,43.280098),(19.473054,43.293276),(19.414039,43.338389),(19.372285,43.384226),(19.355645,43.393063),(19.218392,43.438202),(19.192244,43.454532),(19.175914,43.480887),(19.175191,43.509619),(19.195345,43.532796),(19.217462,43.532796),(19.229451,43.549746),(19.238856,43.572122),(19.252706,43.588633),(19.263661,43.590958),(19.289189,43.587935),(19.300868,43.588297),(19.31265,43.593154),(19.335801,43.606513),(19.34655,43.608838),(19.363396,43.601577),(19.38045,43.585584),(19.394609,43.566463),(19.402877,43.549643),(19.410732,43.540754),(19.41931,43.549255),(19.431816,43.57114),(19.443908,43.571864),(19.481735,43.560831),(19.48959,43.564448),(19.491761,43.568608),(19.48897,43.573181),(19.481735,43.578116),(19.476878,43.588684),(19.475638,43.602559),(19.477498,43.616977),(19.481735,43.628914),(19.507367,43.647182),(19.505713,43.67364),(19.481735,43.729218),(19.461685,43.762136),(19.359469,43.842209),(19.305932,43.904737),(19.275443,43.933263),(19.241027,43.952357),(19.229348,43.95768),(19.240717,43.965741),(19.242887,43.972821),(19.238133,43.985378),(19.238029,43.99251),(19.243507,44.002018),(19.252085,44.007496),(19.272756,44.012405),(19.287329,44.013025),(19.300558,44.009511),(19.325879,43.996592),(19.351718,43.979125),(19.364637,43.973286),(19.379106,43.973854),(19.393989,43.977058),(19.447112,43.979797),(19.50406,43.975663),(19.528554,43.977213),(19.552222,43.983415),(19.593357,44.005584),(19.61041,44.019278),(19.618885,44.035711),(19.611443,44.054521),(19.598938,44.062583),(19.589946,44.060309),(19.583848,44.054315),(19.580231,44.051524),(19.570929,44.056898),(19.554599,44.071265),(19.522043,44.08501),(19.516189,44.091194),(19.498169,44.110229),(19.482666,44.120667),(19.476361,44.127023),(19.474191,44.144903),(19.465716,44.15281),(19.459721,44.152707),(19.448456,44.144438),(19.442565,44.143198),(19.435537,44.146092),(19.424891,44.153792),(19.381173,44.177098),(19.362363,44.191206),(19.356058,44.204021),(19.353991,44.22433),(19.341589,44.245828),(19.324329,44.263966),(19.307379,44.274198),(19.295907,44.2758),(19.263661,44.270167),(19.249502,44.270736),(19.240717,44.272699),(19.220046,44.280244),(19.177155,44.286962),(19.157001,44.293577),(19.138914,44.309338),(19.116693,44.343703),(19.108942,44.36365),(19.107185,44.382718),(19.115763,44.403596),(19.129612,44.416153),(19.141394,44.430829),(19.143358,44.458218),(19.127339,44.502556),(19.129922,44.518317),(19.165269,44.526689),(19.173124,44.531443),(19.179635,44.538264),(19.185319,44.546223),(19.187283,44.553302),(19.18625,44.569322),(19.188317,44.57604),(19.193278,44.580587),(19.20413,44.585135),(19.208677,44.588391),(19.255703,44.645648),(19.270069,44.67562),(19.277614,44.684871),(19.288259,44.693035),(19.308413,44.705128),(19.318025,44.715463),(19.32836,44.733963),(19.363707,44.854628),(19.367841,44.859278),(19.373112,44.86026),(19.376626,44.862999),(19.375695,44.873128),(19.372802,44.881448),(19.368667,44.887132),(19.362776,44.891111),(19.356993,44.895869),(19.356982,44.895877),(19.353165,44.899018),(19.352958,44.898087),(19.350271,44.897002),(19.340142,44.896227),(19.33022,44.898708),(19.31017,44.91235),(19.301281,44.91235),(19.293013,44.909405),(19.283815,44.908371),(19.239476,44.915141),(19.229554,44.913745),(19.211881,44.908371),(19.201856,44.908371),(19.196895,44.913177),(19.193174,44.921549),(19.18687,44.927543),(19.174261,44.925424),(19.08455,44.878967),(19.068427,44.874833),(19.047757,44.872714),(19.015821,44.865635),(18.994323,44.894832),(18.991429,44.914934),(19.018405,44.925683),(19.031944,44.922634),(19.052098,44.906976),(19.06667,44.905684),(19.078659,44.910852),(19.087858,44.919326),(19.103567,44.93824),(19.113076,44.942426),(19.124548,44.94594),(19.131679,44.953175),(19.127649,44.968419),(19.118554,44.97555),(19.09809,44.970021),(19.087238,44.977101),(19.085171,44.987281),(19.088478,44.999477),(19.093232,45.011517),(19.095402,45.020922),(19.094266,45.031413),(19.085997,45.06092),(19.082277,45.084949),(19.079176,45.094613),(19.071528,45.107067),(19.064087,45.113578),(19.054165,45.120606),(19.046207,45.128358),(19.044863,45.137246),(19.060056,45.146858),(19.116073,45.142879),(19.137674,45.146031),(19.141705,45.162154),(19.128785,45.181068),(19.121861,45.195795),(19.143875,45.199516),(19.155761,45.194969),(19.174571,45.175797),(19.185319,45.168045),(19.204543,45.162878),(19.225937,45.161947),(19.267795,45.165823),(19.272756,45.168355),(19.275753,45.172851),(19.279474,45.177244),(19.286709,45.179517),(19.291359,45.177709),(19.299421,45.168975),(19.304279,45.166547),(19.390682,45.169337),(19.393929,45.171624),(19.405358,45.179672),(19.407838,45.203133),(19.397296,45.223287),(19.378073,45.229592),(19.363086,45.248247),(19.162065,45.285041),(19.124755,45.298115),(19.098813,45.319871),(19.096229,45.329121),(19.095402,45.340489),(19.091992,45.349998),(19.08207,45.354029),(19.040832,45.354029),(19.020988,45.357233),(19.003625,45.366121),(18.988742,45.379247),(18.975927,45.394956),(19.028637,45.412165),(19.037422,45.422293),(19.031634,45.434024),(19.012258,45.448468),(19.003005,45.455366),(18.99639,45.473815),(19.00962,45.498671),(19.017926,45.497807),(19.077212,45.491643),(19.106255,45.511642),(19.095092,45.526059),(19.082277,45.531175),(19.067807,45.533398),(19.050961,45.538927),(19.040936,45.545025),(19.036491,45.549469),(19.033391,45.554637),(19.02719,45.562595),(19.018095,45.567401),(19.009206,45.565489),(19.000628,45.561199),(18.983161,45.55474),(18.973136,45.546265),(18.960217,45.539134),(18.941717,45.538927),(18.932002,45.545541),(18.912054,45.568227),(18.90358,45.573085),(18.90699,45.581663),(18.90823,45.600422),(18.912365,45.619232),(18.924457,45.627707),(18.935412,45.633391),(18.962904,45.660108),(18.968485,45.668738),(18.962697,45.683052),(18.948125,45.692044),(18.931071,45.698917),(18.917325,45.706565),(18.908954,45.719433),(18.90482,45.749767),(18.900169,45.764908),(18.871127,45.789971),(18.863995,45.798549),(18.848699,45.809556),(18.844978,45.815706),(18.846735,45.819995),(18.850353,45.824232),(18.85242,45.830072),(18.855314,45.857357),(18.887146,45.859476),(18.899755,45.861388),(18.90699,45.867951),(18.903993,45.875702),(18.893451,45.883867),(18.872884,45.895236),(18.881979,45.902626),(18.889834,45.907897),(18.90699,45.9157),(18.901306,45.931203),(18.962697,45.927947),(18.981818,45.921849),(18.987502,45.923813),(18.988742,45.927017),(18.986469,45.931254),(18.97882,45.939781),(18.977684,45.943502),(18.978717,45.947171),(18.981818,45.950788),(19.005589,45.96257),(19.03091,45.960038),(19.048584,45.963449),(19.049721,45.993111),(19.06543,46.012025),(19.088478,46.018846),(19.111009,46.012955),(19.125788,45.993266),(19.125788,45.993111),(19.148009,45.984068),(19.235652,45.977711),(19.263454,45.981432),(19.274823,45.991612),(19.279474,46.003808),(19.286915,46.016159),(19.298315,46.022123),(19.306966,46.026649),(19.325156,46.029491),(19.362156,46.029646),(19.378899,46.033677),(19.389131,46.041532),(19.396573,46.051557),(19.404738,46.060239),(19.417243,46.064321),(19.428302,46.06551),(19.43688,46.06799),(19.453727,46.07755),(19.460858,46.084475),(19.465922,46.091968),(19.47295,46.098686),(19.499305,46.108608),(19.496722,46.116876),(19.48928,46.126023),(19.487523,46.134239),(19.501993,46.145608),(19.525247,46.156409),(19.549948,46.16416),(19.568449,46.166434),(19.589739,46.165969),(19.647824,46.173875),(19.669321,46.1731),(19.690095,46.168398)] +Suriname [(-54.170965,5.348376),(-54.19046,5.325748),(-54.269163,5.269085),(-54.311331,5.225806),(-54.320652,5.212454),(-54.332002,5.196196),(-54.343577,5.156379),(-54.350037,5.149583),(-54.365437,5.138757),(-54.369881,5.132375),(-54.37231,5.121058),(-54.375617,5.114883),(-54.430239,5.053388),(-54.448636,5.024294),(-54.454785,5.008533),(-54.457576,4.99135),(-54.448842,4.951482),(-54.450806,4.936728),(-54.455509,4.931431),(-54.482122,4.912802),(-54.486721,4.902958),(-54.484964,4.892752),(-54.47866,4.87867),(-54.47866,4.755189),(-54.475404,4.741727),(-54.467291,4.736198),(-54.457007,4.73333),(-54.447344,4.727852),(-54.435458,4.709404),(-54.435148,4.692376),(-54.438559,4.673282),(-54.437732,4.648735),(-54.425278,4.61605),(-54.423469,4.604345),(-54.423469,4.563392),(-54.428172,4.546158),(-54.447964,4.509545),(-54.450806,4.484224),(-54.433443,4.376375),(-54.402954,4.312942),(-54.394117,4.269818),(-54.393187,4.244393),(-54.399595,4.227262),(-54.41055,4.208581),(-54.406571,4.191528),(-54.394892,4.177911),(-54.38249,4.169566),(-54.3489,4.160522),(-54.338565,4.152332),(-54.334689,4.131713),(-54.337583,4.116649),(-54.351019,4.082078),(-54.355153,4.066523),(-54.353861,4.041718),(-54.344353,4.024949),(-54.310194,3.994202),(-54.30389,3.984461),(-54.293141,3.949812),(-54.286475,3.940717),(-54.272522,3.929529),(-54.265804,3.922527),(-54.241051,3.876354),(-54.222241,3.866199),(-54.218004,3.857647),(-54.214541,3.858215),(-54.207048,3.846485),(-54.197488,3.826899),(-54.18736,3.814678),(-54.181623,3.811009),(-54.170203,3.805815),(-54.136097,3.795738),(-54.125503,3.788736),(-54.104781,3.75804),(-54.094445,3.747059),(-54.074033,3.676107),(-54.050675,3.640554),(-54.050107,3.634534),(-54.03941,3.636446),(-54.028403,3.639831),(-54.017861,3.641458),(-54.008818,3.637919),(-53.994813,3.623604),(-53.988819,3.610995),(-53.990266,3.595699),(-53.998276,3.573039),(-54.006285,3.531026),(-54.00303,3.455397),(-54.019359,3.415374),(-54.055688,3.377676),(-54.060339,3.364214),(-54.062716,3.346696),(-54.069486,3.327085),(-54.080234,3.309722),(-54.080805,3.309314),(-54.114289,3.285382),(-54.140024,3.245023),(-54.176559,3.200581),(-54.19046,3.178102),(-54.211182,3.127407),(-54.18798,3.130973),(-54.177076,3.116555),(-54.174699,3.093921),(-54.177024,3.072785),(-54.179453,3.070305),(-54.184001,3.067979),(-54.188496,3.06462),(-54.190667,3.059143),(-54.188548,3.058316),(-54.179195,3.051598),(-54.177024,3.048911),(-54.170875,3.024829),(-54.169169,3.010722),(-54.173614,3.004521),(-54.185137,2.997803),(-54.179763,2.983747),(-54.168756,2.971396),(-54.163433,2.969742),(-54.162968,2.955996),(-54.170203,2.925404),(-54.173252,2.919513),(-54.18705,2.899307),(-54.190667,2.887835),(-54.191339,2.877345),(-54.189633,2.870627),(-54.18338,2.863857),(-54.170203,2.853057),(-54.182192,2.846546),(-54.18891,2.838277),(-54.204723,2.791769),(-54.212526,2.776421),(-54.285338,2.677977),(-54.320426,2.606664),(-54.359442,2.508013),(-54.375617,2.483777),(-54.423469,2.435925),(-54.435458,2.431119),(-54.450548,2.428948),(-54.472613,2.42869),(-54.483311,2.422851),(-54.493387,2.417063),(-54.520311,2.348902),(-54.531576,2.340168),(-54.551162,2.338101),(-54.584751,2.348333),(-54.599221,2.345801),(-54.615292,2.326267),(-54.634128,2.320118),(-54.653378,2.316501),(-54.67286,2.315984),(-54.692497,2.318981),(-54.704124,2.32482),(-54.706553,2.332882),(-54.705442,2.342545),(-54.706734,2.353191),(-54.715699,2.375928),(-54.709963,2.39479),(-54.707276,2.395876),(-54.702031,2.395721),(-54.697251,2.397322),(-54.695597,2.403524),(-54.696838,2.409105),(-54.701101,2.419905),(-54.702548,2.425745),(-54.703194,2.445847),(-54.707664,2.450343),(-54.721177,2.457991),(-54.743036,2.466517),(-54.759805,2.465794),(-54.775494,2.457335),(-54.792491,2.448172),(-54.841919,2.433496),(-54.880289,2.447345),(-54.978577,2.54305),(-54.982815,2.552352),(-54.979559,2.566098),(-54.960232,2.585683),(-54.953669,2.599067),(-54.959095,2.608989),(-54.976872,2.606922),(-55.017748,2.590592),(-55.037747,2.577932),(-55.076298,2.54522),(-55.112471,2.527805),(-55.120894,2.524808),(-55.128852,2.525687),(-55.136604,2.5338),(-55.13309,2.552869),(-55.137793,2.56217),(-55.171951,2.559328),(-55.25192,2.497885),(-55.275252,2.499332),(-55.286982,2.513905),(-55.302795,2.519692),(-55.32052,2.518297),(-55.337754,2.511269),(-55.354523,2.494888),(-55.360079,2.476232),(-55.362766,2.457939),(-55.37106,2.442539),(-55.399585,2.430189),(-55.435552,2.43086),(-55.474309,2.435821),(-55.51131,2.43639),(-55.568206,2.431274),(-55.587326,2.433858),(-55.608306,2.433961),(-55.645384,2.416649),(-55.724294,2.396909),(-55.744112,2.401043),(-55.754602,2.409415),(-55.766126,2.431274),(-55.773568,2.440059),(-55.783128,2.445175),(-55.853769,2.462642),(-55.870668,2.47091),(-55.925031,2.515662),(-55.9472,2.528167),(-55.971023,2.530338),(-55.983012,2.526462),(-55.989239,2.520829),(-55.996913,2.503259),(-56.007507,2.460678),(-56.008179,2.416288),(-56.013346,2.398821),(-56.042569,2.355103),(-56.050243,2.347041),(-56.062155,2.341564),(-56.07342,2.34239),(-56.082283,2.347713),(-56.091455,2.351899),(-56.10378,2.349057),(-56.116803,2.333089),(-56.131995,2.304357),(-56.143881,2.274746),(-56.146775,2.256194),(-56.135251,2.248546),(-56.091455,2.245911),(-56.072929,2.241518),(-56.054016,2.224051),(-56.044843,2.184726),(-56.03133,2.163487),(-56.004406,2.144986),(-55.995776,2.13739),(-55.959965,2.090985),(-55.925806,2.061684),(-55.918262,2.050393),(-55.915135,2.037655),(-55.916763,2.028508),(-55.92038,2.019387),(-55.923584,2.006597),(-55.922344,1.962155),(-55.916763,1.922235),(-55.922034,1.886217),(-55.953815,1.853273),(-56.019651,1.833507),(-56.082644,1.846555),(-56.145276,1.871773),(-56.209768,1.888542),(-56.257931,1.885648),(-56.273796,1.887509),(-56.292709,1.895467),(-56.328624,1.918463),(-56.347951,1.926602),(-56.367175,1.928876),(-56.396527,1.921667),(-56.415182,1.919703),(-56.429807,1.9227),(-56.481819,1.941614),(-56.4851,1.953215),(-56.491198,1.962905),(-56.499983,1.969984),(-56.520964,1.976728),(-56.529387,1.981792),(-56.536493,1.988639),(-56.542074,1.997166),(-56.579772,2.016803),(-56.677802,2.018405),(-56.70519,2.029645),(-56.801464,2.165967),(-56.80875,2.195784),(-56.816346,2.217592),(-56.826191,2.261672),(-56.838877,2.281102),(-56.847792,2.285856),(-56.870865,2.290146),(-56.880141,2.294796),(-56.88443,2.303736),(-56.883887,2.339755),(-56.89505,2.362079),(-56.930836,2.393292),(-56.938458,2.41143),(-56.932024,2.425796),(-56.930241,2.436183),(-56.935073,2.445898),(-56.951635,2.461246),(-56.956364,2.470031),(-56.959593,2.483777),(-56.957526,2.493492),(-56.952617,2.504551),(-56.951274,2.515455),(-56.959593,2.524705),(-56.980006,2.509719),(-56.994113,2.504861),(-57.000573,2.514163),(-56.995302,2.547856),(-56.997111,2.555142),(-57.02238,2.584546),(-57.027238,2.59359),(-57.02791,2.610591),(-57.023155,2.623665),(-57.020468,2.635447),(-57.027238,2.648212),(-57.048012,2.636481),(-57.05597,2.642372),(-57.061964,2.68175),(-57.071886,2.700095),(-57.097725,2.72738),(-57.102944,2.740454),(-57.099016,2.750376),(-57.092144,2.763037),(-57.08987,2.773889),(-57.099533,2.778591),(-57.133743,2.773424),(-57.13705,2.77115),(-57.143665,2.790012),(-57.126405,2.82603),(-57.133381,2.832593),(-57.145215,2.830423),(-57.165266,2.821069),(-57.174619,2.818951),(-57.186608,2.82324),(-57.191052,2.832593),(-57.194256,2.841998),(-57.202266,2.846236),(-57.206348,2.856571),(-57.219629,2.908299),(-57.210328,2.918221),(-57.194721,2.930623),(-57.183818,2.944679),(-57.188623,2.95951),(-57.221851,2.963024),(-57.234615,2.970362),(-57.226451,2.983385),(-57.22118,2.992583),(-57.2218,3.000438),(-57.224694,3.009171),(-57.226451,3.021264),(-57.223247,3.024778),(-57.209242,3.024054),(-57.205987,3.028137),(-57.20702,3.035681),(-57.211774,3.048342),(-57.21286,3.056042),(-57.222213,3.078263),(-57.239473,3.094128),(-57.248568,3.112369),(-57.233324,3.14167),(-57.236734,3.143478),(-57.237406,3.143995),(-57.237613,3.144874),(-57.239473,3.147871),(-57.246243,3.141618),(-57.254976,3.13583),(-57.264898,3.132626),(-57.274923,3.13428),(-57.286033,3.142393),(-57.284173,3.148956),(-57.277714,3.156914),(-57.274923,3.169007),(-57.285672,3.192106),(-57.287946,3.203113),(-57.287119,3.21536),(-57.281744,3.235928),(-57.280452,3.247813),(-57.284018,3.25975),(-57.290271,3.267967),(-57.292493,3.276287),(-57.284225,3.288793),(-57.282778,3.295045),(-57.280452,3.329772),(-57.283553,3.343802),(-57.308409,3.39491),(-57.340035,3.370157),(-57.366752,3.36561),(-57.393469,3.373826),(-57.425043,3.387495),(-57.423958,3.369382),(-57.430934,3.360623),(-57.459201,3.353336),(-57.461682,3.349978),(-57.468761,3.343182),(-57.476254,3.338945),(-57.479717,3.343389),(-57.482042,3.348556),(-57.487985,3.351528),(-57.49584,3.352975),(-57.503901,3.353336),(-57.509637,3.357677),(-57.522401,3.365248),(-57.535269,3.36778),(-57.545036,3.348143),(-57.554234,3.346748),(-57.565138,3.349822),(-57.596193,3.367121),(-57.596402,3.367237),(-57.613249,3.376643),(-57.641464,3.38243),(-57.649939,3.385944),(-57.654848,3.39013),(-57.658207,3.39522),(-57.662548,3.40708),(-57.664615,3.417544),(-57.663995,3.425839),(-57.658931,3.443899),(-57.658362,3.450617),(-57.660533,3.462994),(-57.660688,3.469505),(-57.645495,3.498987),(-57.644048,3.51666),(-57.660429,3.535186),(-57.681255,3.547304),(-57.685131,3.549164),(-57.687249,3.552162),(-57.697998,3.553609),(-57.702856,3.555676),(-57.707351,3.560662),(-57.708643,3.564125),(-57.709522,3.56769),(-57.71748,3.582444),(-57.722493,3.599962),(-57.72611,3.607197),(-57.764247,3.631588),(-57.813133,3.651845),(-57.829773,3.662439),(-57.840987,3.680991),(-57.846929,3.702101),(-57.849617,3.747473),(-57.853854,3.76667),(-57.875145,3.812197),(-57.925839,3.886405),(-57.942117,3.905447),(-58.007902,3.957279),(-58.032138,3.987716),(-58.042008,4.023037),(-58.055082,4.108071),(-58.067691,4.151143),(-58.065934,4.171839),(-58.052033,4.193104),(-57.964958,4.282039),(-57.953848,4.299015),(-57.947388,4.318756),(-57.94527,4.343922),(-57.94806,4.360484),(-57.953486,4.375703),(-57.956639,4.391438),(-57.952659,4.409732),(-57.942789,4.425106),(-57.931317,4.437844),(-57.92186,4.450944),(-57.914471,4.484069),(-57.896074,4.52851),(-57.879124,4.556829),(-57.863311,4.60773),(-57.845431,4.633517),(-57.837421,4.65088),(-57.836026,4.669845),(-57.844346,4.687519),(-57.871166,4.721728),(-57.88455,4.762863),(-57.901138,4.773922),(-57.91783,4.782371),(-57.925374,4.796142),(-57.918501,4.829758),(-57.90243,4.852728),(-57.884188,4.872262),(-57.870701,4.888928),(-57.85034,4.922801),(-57.842795,4.929881),(-57.830238,4.933059),(-57.820575,4.930294),(-57.810963,4.92554),(-57.798457,4.92306),(-57.773962,4.926445),(-57.762335,4.93554),(-57.747297,4.964013),(-57.720477,4.9898),(-57.686939,5.006259),(-57.64937,5.008429),(-57.610096,4.99135),(-57.566223,5.007086),(-57.543279,5.011091),(-57.517596,5.012434),(-57.512221,5.009153),(-57.496873,4.994657),(-57.490259,4.99135),(-57.47734,4.992487),(-57.460493,4.997629),(-57.381377,5.005742),(-57.35621,5.012434),(-57.345513,5.01662),(-57.338072,5.020754),(-57.329442,5.024087),(-57.315231,5.026103),(-57.306497,5.024449),(-57.298694,5.021116),(-57.290478,5.020289),(-57.280452,5.026103),(-57.316936,5.058969),(-57.321432,5.070157),(-57.320192,5.083412),(-57.317091,5.094755),(-57.302208,5.128474),(-57.295542,5.157206),(-57.288979,5.171029),(-57.277714,5.17692),(-57.259937,5.1763),(-57.247276,5.172373),(-57.23875,5.162012),(-57.233324,5.142116),(-57.217769,5.14855),(-57.202008,5.157619),(-57.189812,5.16922),(-57.184903,5.183147),(-57.192396,5.203172),(-57.225727,5.243609),(-57.232522,5.260572),(-57.233324,5.262574),(-57.237613,5.268723),(-57.247638,5.271772),(-57.259007,5.271204),(-57.26743,5.266295),(-57.269187,5.256709),(-57.264019,5.247045),(-57.253787,5.234927),(-57.256423,5.228287),(-57.263037,5.223997),(-57.271667,5.222861),(-57.280452,5.225341),(-57.285465,5.230819),(-57.287274,5.238699),(-57.288824,5.30854),(-57.292028,5.311331),(-57.298332,5.311021),(-57.308409,5.313475),(-57.322827,5.30761),(-57.331715,5.30761),(-57.335695,5.316886),(-57.332284,5.327867),(-57.324119,5.33859),(-57.275491,5.387398),(-57.265208,5.403366),(-57.261229,5.42675),(-57.260092,5.448196),(-57.25694,5.464086),(-57.24767,5.484931),(-57.233998,5.498969),(-57.225087,5.507514),(-57.193023,5.519721),(-57.179514,5.530585),(-57.170155,5.542792),(-57.163686,5.557074),(-57.139638,5.665107),(-57.134877,5.759914),(-57.128,5.793158),(-57.12577,5.818145),(-57.113847,5.858844),(-57.102283,5.886416),(-57.075795,5.929999),(-57.071645,5.94123),(-57.066029,5.9501),(-57.057688,5.955878),(-57.03482,5.959906),(-57.027903,5.964423),(-57.022694,5.96894),(-57.017445,5.970933),(-57.012807,5.973863),(-57.000111,5.987982),(-56.994496,5.992662),(-56.98469,5.996283),(-56.974599,5.998114),(-56.963872,6.009152),(-56.956537,6.011574),(-56.928844,6.004262),(-56.866972,5.989637),(-56.830356,5.985557),(-56.780724,5.985508),(-56.702631,5.98219),(-56.659512,5.979716),(-56.642432,5.973223),(-56.611422,5.953647),(-56.593104,5.945753),(-56.57395,5.940756),(-56.526859,5.936549),(-56.418314,5.911349),(-56.355024,5.89467),(-56.303807,5.892508),(-56.256766,5.880406),(-56.222834,5.870753),(-56.19164,5.862616),(-56.124257,5.848049),(-56.083036,5.834366),(-56.058595,5.827737),(-56.037994,5.824164),(-55.999182,5.810713),(-55.936006,5.804661),(-55.917877,5.784735),(-55.903554,5.776353),(-55.897694,5.763007),(-55.897694,5.676703),(-55.897247,5.676703),(-55.890248,5.676703),(-55.884999,5.69245),(-55.88329,5.708075),(-55.884023,5.741929),(-55.887278,5.759467),(-55.90453,5.797187),(-55.915362,5.821771),(-55.899963,5.841964),(-55.910003,5.850918),(-55.922245,5.850353),(-55.933959,5.849827),(-55.947872,5.861521),(-55.951169,5.881511),(-55.946669,5.896493),(-55.930433,5.91758),(-55.923613,5.927845),(-55.910395,5.941616),(-55.900654,5.951945),(-55.886052,5.962954),(-55.86799,5.969813),(-55.844382,5.975962),(-55.820095,5.977275),(-55.729089,5.981216),(-55.678578,5.985256),(-55.637318,5.985907),(-55.394988,5.973876),(-55.335967,5.962708),(-55.307871,5.94335),(-55.292045,5.936727),(-55.270823,5.929999),(-55.263783,5.926418),(-55.253204,5.917792),(-55.242909,5.909369),(-55.2329,5.903266),(-55.229766,5.902411),(-55.221425,5.900214),(-55.21288,5.900214),(-55.202433,5.898761),(-55.188361,5.897395),(-55.176908,5.90129),(-55.164052,5.906684),(-55.14784,5.908177),(-55.130727,5.895855),(-55.110829,5.881171),(-55.103107,5.866412),(-55.115101,5.837128),(-55.129872,5.820746),(-55.128611,5.821357),(-55.101039,5.830499),(-55.091712,5.852791),(-55.090749,5.875117),(-55.08552,5.882799),(-55.063423,5.882883),(-55.030385,5.859768),(-55.010121,5.855536),(-54.99706,5.858466),(-54.988922,5.863674),(-54.981191,5.865465),(-54.958811,5.851793),(-54.950307,5.850287),(-54.941803,5.852118),(-54.931956,5.855536),(-54.924143,5.860419),(-54.918691,5.867092),(-54.911977,5.87287),(-54.900258,5.875393),(-54.890777,5.873358),(-54.880727,5.868354),(-54.863026,5.855536),(-54.867543,5.863674),(-54.874338,5.871568),(-54.882192,5.878363),(-54.890289,5.882799),(-54.903472,5.885403),(-54.91258,5.884025),(-54.921851,5.876625),(-54.933319,5.869672),(-54.956207,5.867575),(-54.971566,5.877267),(-54.985634,5.880387),(-55.001517,5.871259),(-55.016591,5.871975),(-55.031728,5.884467),(-55.047271,5.892279),(-55.105295,5.908922),(-55.12564,5.918647),(-55.142974,5.930121),(-55.15038,5.940863),(-55.154124,5.951361),(-55.159966,5.958114),(-55.153323,5.968592),(-55.1445,5.974248),(-55.116851,5.985256),(-55.085301,5.990649),(-55.044799,5.993105),(-55.013905,5.993394),(-54.77359,5.985256),(-54.75536,5.980943),(-54.712555,5.976184),(-54.673556,5.969192),(-54.471781,5.94097),(-54.34863,5.913755),(-54.252699,5.894808),(-54.169342,5.867906),(-54.097396,5.847339),(-54.074456,5.844887),(-54.062032,5.840559),(-54.036731,5.842351),(-54.030944,5.834184),(-54.020462,5.83025),(-54.014687,5.820148),(-54.016307,5.809704),(-53.992085,5.763497),(-53.986357,5.745665),(-53.99592,5.734524),(-54.019753,5.688979),(-54.031148,5.66931),(-54.036439,5.629502),(-54.050364,5.552069),(-54.061106,5.512885),(-54.073598,5.489814),(-54.091786,5.467922),(-54.10912,5.452826),(-54.118479,5.442776),(-54.122548,5.433417),(-54.128529,5.414293),(-54.170969,5.348375),(-54.170965,5.348376)] +Slovakia [(19.706321,49.387529),(19.726578,49.388873),(19.760065,49.397658),(19.769056,49.393213),(19.769263,49.39311),(19.778565,49.374197),(19.783629,49.357557),(19.78952,49.309188),(19.78797,49.305105),(19.780322,49.297612),(19.779713,49.293453),(19.779702,49.293375),(19.783216,49.290016),(19.79686,49.283399),(19.796962,49.28335),(19.798563,49.281081),(19.798822,49.280714),(19.806415,49.275488),(19.80678,49.275236),(19.808641,49.270895),(19.806263,49.265263),(19.800476,49.263041),(19.794068,49.261852),(19.789624,49.259475),(19.751797,49.219064),(19.747787,49.205956),(19.747766,49.205887),(19.760685,49.194208),(19.785903,49.188162),(19.831998,49.185888),(19.854219,49.191262),(19.868292,49.200788),(19.887809,49.214),(19.905896,49.222785),(19.937935,49.22511),(19.965737,49.215653),(20.01731,49.183872),(20.050486,49.173227),(20.07002,49.183097),(20.080355,49.208109),(20.08604,49.243042),(20.098442,49.25286),(20.105367,49.263971),(20.111051,49.275856),(20.130171,49.303917),(20.135856,49.308878),(20.138336,49.307327),(20.16035,49.305622),(20.170169,49.311617),(20.191976,49.328618),(20.207169,49.334199),(20.284374,49.338643),(20.289335,49.343087),(20.296466,49.356937),(20.301634,49.371199),(20.303701,49.380398),(20.307422,49.386599),(20.317653,49.391612),(20.329539,49.391767),(20.370467,49.38169),(20.422143,49.382982),(20.421936,49.40019),(20.437543,49.402515),(20.522085,49.374352),(20.543996,49.370838),(20.567664,49.376367),(20.579136,49.383395),(20.595053,49.396263),(20.605284,49.400035),(20.614483,49.400448),(20.636084,49.39833),(20.673911,49.402309),(20.689517,49.4005),(20.778297,49.331202),(20.79411,49.323657),(20.816538,49.321383),(20.833591,49.322055),(20.849301,49.320505),(20.868007,49.31141),(20.884337,49.300299),(20.900564,49.2926),(20.91896,49.290326),(20.942422,49.295855),(20.964022,49.308103),(20.996682,49.339367),(21.017249,49.352441),(21.032959,49.354611),(21.054043,49.354766),(21.072543,49.357195),(21.080708,49.366342),(21.068512,49.381431),(21.045464,49.390578),(21.033269,49.399673),(21.053526,49.414453),(21.068822,49.419207),(21.10944,49.424581),(21.124943,49.423651),(21.143029,49.415538),(21.157292,49.405151),(21.172588,49.398278),(21.194086,49.400603),(21.211242,49.411094),(21.242455,49.441273),(21.260542,49.449438),(21.274391,49.447267),(21.330408,49.427785),(21.42787,49.409802),(21.444406,49.409905),(21.48151,49.415228),(21.496186,49.412386),(21.514422,49.417213),(21.529569,49.421222),(21.601193,49.426493),(21.6199,49.423341),(21.630028,49.418897),(21.648632,49.407063),(21.65876,49.402464),(21.666305,49.401792),(21.681808,49.404066),(21.69173,49.402154),(21.708576,49.390888),(21.742166,49.357092),(21.757566,49.348927),(21.768004,49.353474),(21.782164,49.364482),(21.799423,49.374817),(21.819577,49.377246),(21.837871,49.370424),(21.874561,49.348358),(21.928408,49.330788),(21.964478,49.308568),(21.99321,49.278182),(22.005819,49.242887),(22.012434,49.211054),(22.040752,49.197463),(22.111549,49.188627),(22.144105,49.174881),(22.155681,49.171522),(22.165706,49.171108),(22.189684,49.17302),(22.197952,49.171987),(22.208908,49.163977),(22.209011,49.156949),(22.20622,49.150489),(22.208494,49.144185),(22.215936,49.139999),(22.262858,49.130594),(22.318151,49.131989),(22.339442,49.12646),(22.390085,49.093025),(22.426775,49.085635),(22.505013,49.083413),(22.539637,49.0722),(22.531989,49.055715),(22.524754,49.032874),(22.520103,49.009826),(22.520516,48.992928),(22.505013,48.984246),(22.466566,48.980526),(22.448996,48.971431),(22.427292,48.929469),(22.41489,48.911693),(22.41365,48.906783),(22.413753,48.893864),(22.411789,48.887766),(22.402384,48.878826),(22.378406,48.865442),(22.370799,48.858248),(22.368898,48.856451),(22.362283,48.844255),(22.36156,48.836452),(22.363523,48.828287),(22.365694,48.794387),(22.36342,48.787617),(22.356495,48.776145),(22.347814,48.767877),(22.338409,48.762968),(22.330761,48.756405),(22.32766,48.743021),(22.3289,48.721575),(22.322492,48.700336),(22.310297,48.681681),(22.294277,48.667625),(22.282185,48.662405),(22.25552,48.656773),(22.243221,48.651192),(22.235676,48.644164),(22.225237,48.628222),(22.219036,48.620935),(22.153717,48.585873),(22.138731,48.569595),(22.136664,48.549337),(22.148343,48.508823),(22.144829,48.493114),(22.13377,48.476835),(22.13284,48.404798),(22.113926,48.38865),(22.09646,48.379425),(22.077856,48.375808),(22.018015,48.379735),(21.999928,48.37896),(21.981531,48.374723),(21.929338,48.372914),(21.914765,48.36909),(21.884276,48.357463),(21.841178,48.353174),(21.789398,48.335526),(21.759012,48.333769),(21.727697,48.340901),(21.701238,48.353949),(21.67757,48.372346),(21.621553,48.429655),(21.613388,48.440352),(21.600366,48.481641),(21.591684,48.49301),(21.574631,48.495568),(21.537734,48.495232),(21.521818,48.50009),(21.5151,48.507118),(21.506005,48.526496),(21.49939,48.535075),(21.490812,48.540268),(21.472725,48.544997),(21.439135,48.558329),(21.424563,48.561275),(21.372679,48.550345),(21.338573,48.549854),(21.32183,48.54758),(21.302296,48.539907),(21.293925,48.530579),(21.28824,48.519934),(21.276561,48.50872),(21.261782,48.50319),(21.250413,48.506498),(21.238011,48.513448),(21.219924,48.518719),(21.186748,48.513707),(21.109336,48.489109),(21.084015,48.49301),(21.063861,48.506239),(21.035956,48.514637),(21.0065,48.518151),(20.981489,48.516859),(20.945832,48.518978),(20.891365,48.541095),(20.859946,48.543317),(20.845477,48.545823),(20.815814,48.563807),(20.800311,48.569233),(20.784085,48.569052),(20.572522,48.536573),(20.51051,48.533783),(20.481674,48.526083),(20.480538,48.518538),(20.480227,48.510218),(20.482191,48.492881),(20.482811,48.489367),(20.482915,48.485827),(20.482501,48.482261),(20.481674,48.478747),(20.468239,48.465389),(20.465968,48.46379),(20.435579,48.442393),(20.420593,48.429241),(20.409017,48.413713),(20.370157,48.334338),(20.349279,48.305476),(20.324268,48.279948),(20.295226,48.260415),(20.272385,48.252456),(20.260293,48.255893),(20.249027,48.264187),(20.22908,48.270905),(20.217815,48.267598),(20.187636,48.248994),(20.170892,48.244033),(20.153322,48.245273),(20.14309,48.247805),(20.134512,48.246669),(20.122006,48.236747),(20.118079,48.229719),(20.112601,48.211735),(20.105263,48.202795),(20.096995,48.198429),(20.078082,48.193545),(20.038303,48.177233),(20.034983,48.175872),(19.99695,48.167914),(19.973902,48.158379),(19.928633,48.130087),(19.905069,48.124299),(19.884295,48.129621),(19.846261,48.152669),(19.821663,48.157914),(19.785593,48.14869),(19.776084,48.149517),(19.766679,48.159),(19.76916,48.167449),(19.774844,48.176079),(19.774327,48.185897),(19.756551,48.200315),(19.73309,48.202899),(19.686994,48.196904),(19.676969,48.200392),(19.655265,48.21835),(19.643896,48.224809),(19.633871,48.226773),(19.623226,48.227006),(19.531241,48.21065),(19.513982,48.203958),(19.503026,48.189411),(19.493621,48.150705),(19.481735,48.134892),(19.481425,48.134427),(19.481219,48.133962),(19.481425,48.133342),(19.481735,48.13267),(19.483286,48.127193),(19.483803,48.121767),(19.483286,48.116496),(19.481735,48.111328),(19.428199,48.085852),(19.29322,48.087764),(19.233482,48.06208),(19.222526,48.060582),(19.098503,48.070736),(19.038662,48.064871),(19.019017,48.065497),(18.996494,48.066214),(18.981818,48.061615),(18.933595,48.054349),(18.838467,48.040015),(18.821001,48.030454),(18.794232,47.993144),(18.784724,47.987615),(18.765293,47.985393),(18.756198,47.981827),(18.743176,47.971052),(18.744519,47.967357),(18.751444,47.963353),(18.754751,47.951803),(18.744726,47.910513),(18.742246,47.889455),(18.748757,47.870723),(18.778006,47.851447),(18.816453,47.83256),(18.814916,47.832194),(18.790305,47.826332),(18.767671,47.822302),(18.750307,47.81362),(18.717234,47.788118),(18.692843,47.777963),(18.663698,47.775896),(18.633829,47.779824),(18.597448,47.79065),(18.552593,47.792846),(18.347851,47.776775),(18.273024,47.756259),(18.23592,47.753882),(18.11262,47.762486),(17.883532,47.752521),(17.825713,47.750006),(17.741997,47.76538),(17.719245,47.773669),(17.676678,47.789177),(17.666239,47.797032),(17.658384,47.807316),(17.639884,47.819201),(17.61911,47.829226),(17.604227,47.834239),(17.592962,47.833102),(17.582937,47.829795),(17.572601,47.829536),(17.560406,47.837986),(17.526609,47.872118),(17.517308,47.876252),(17.492193,47.879818),(17.481858,47.882711),(17.472039,47.888809),(17.36941,47.981207),(17.337887,47.998725),(17.272671,48.00534),(17.262316,48.007283),(17.220892,48.015055),(17.184821,48.020274),(17.148338,48.005443),(17.12498,48.019525),(17.092631,48.02725),(17.069686,48.035674),(17.075061,48.052081),(17.063072,48.058773),(17.059144,48.060427),(17.064829,48.07903),(17.069996,48.089159),(17.080228,48.097608),(17.067206,48.106936),(17.062455,48.112724),(17.047465,48.130991),(17.036923,48.135513),(17.020904,48.137166),(17.007158,48.142799),(16.98194,48.161299),(16.974808,48.17688),(16.974705,48.198558),(16.969951,48.216645),(16.954345,48.25256),(16.953931,48.25734),(16.955275,48.268786),(16.954345,48.273127),(16.950624,48.276589),(16.944423,48.27801),(16.933364,48.284728),(16.924269,48.287519),(16.916621,48.2908),(16.913313,48.296691),(16.912073,48.301239),(16.908869,48.306975),(16.905459,48.311988),(16.902771,48.314106),(16.898431,48.316173),(16.900291,48.321238),(16.904218,48.327025),(16.906492,48.33147),(16.901945,48.339402),(16.891713,48.347024),(16.881171,48.352812),(16.875486,48.355034),(16.855332,48.356455),(16.847581,48.359582),(16.84448,48.365602),(16.845101,48.376583),(16.846858,48.381131),(16.849338,48.384102),(16.851922,48.390407),(16.86081,48.443788),(16.864944,48.458077),(16.875073,48.471539),(16.901221,48.496602),(16.906492,48.509908),(16.913934,48.519339),(16.93047,48.528202),(16.946903,48.539726),(16.949055,48.544836),(16.954345,48.557399),(16.945043,48.604166),(16.947523,48.623157),(16.963336,48.635947),(16.974808,48.649874),(17.025348,48.746328),(17.049946,48.774027),(17.084362,48.793715),(17.098728,48.805756),(17.10462,48.824618),(17.113715,48.83392),(17.167458,48.859758),(17.21221,48.866062),(17.260269,48.857794),(17.374371,48.819605),(17.391734,48.821776),(17.411225,48.830216),(17.429561,48.838157),(17.453332,48.842756),(17.467905,48.838105),(17.498497,48.816763),(17.535084,48.812991),(17.727217,48.86291),(17.744891,48.872574),(17.77941,48.911744),(17.795533,48.920581),(17.820131,48.923371),(17.841215,48.920994),(17.860336,48.921718),(17.878733,48.933552),(17.886897,48.947246),(17.894649,48.9782),(17.90085,48.993031),(17.914079,49.010498),(17.93506,49.019386),(17.959244,49.021867),(18.012885,49.019128),(18.046061,49.029153),(18.075516,49.047188),(18.09629,49.070288),(18.102905,49.09225),(18.10163,49.136928),(18.101458,49.142945),(18.105075,49.169765),(18.117788,49.202579),(18.136495,49.233068),(18.160576,49.2587),(18.190031,49.276942),(18.324597,49.311255),(18.361586,49.330191),(18.385162,49.342261),(18.387642,49.389751),(18.416788,49.385462),(18.439215,49.395074),(18.481797,49.429077),(18.514663,49.440601),(18.522931,49.446079),(18.527995,49.454864),(18.530476,49.473467),(18.535643,49.481684),(18.556211,49.490159),(18.600446,49.485973),(18.628971,49.495791),(18.635896,49.496721),(18.64282,49.495791),(18.675583,49.485043),(18.704522,49.479255),(18.732531,49.480288),(18.773562,49.504886),(18.792269,49.509537),(18.833196,49.510261),(18.932208,49.504318),(18.961147,49.492794),(18.952362,49.475948),(18.953396,49.461685),(18.958253,49.448146),(18.960837,49.433315),(18.956703,49.400448),(18.962284,49.389183),(18.981818,49.386806),(19.007139,49.388098),(19.045793,49.402774),(19.067601,49.406133),(19.076799,49.404066),(19.096643,49.394609),(19.106255,49.391405),(19.11659,49.39094),(19.141705,49.394195),(19.172504,49.402257),(19.179738,49.410267),(19.182322,49.422721),(19.18935,49.442203),(19.192244,49.442616),(19.204543,49.442823),(19.208987,49.444993),(19.211364,49.45166),(19.209401,49.456414),(19.206197,49.459411),(19.20444,49.460548),(19.220459,49.493001),(19.234102,49.507212),(19.248675,49.516255),(19.265005,49.521475),(19.284228,49.524162),(19.315027,49.523955),(19.325363,49.52504),(19.339522,49.528761),(19.347067,49.532998),(19.43378,49.595165),(19.437604,49.600126),(19.443392,49.60178),(19.44889,49.600313),(19.457344,49.598059),(19.474191,49.578732),(19.481735,49.573513),(19.505197,49.563384),(19.517289,49.543282),(19.535479,49.492846),(19.551602,49.461013),(19.55677,49.453882),(19.573926,49.445252),(19.594804,49.441583),(19.634801,49.441324),(19.63015,49.43502),(19.6286,49.429594),(19.62922,49.413936),(19.627097,49.402165),(19.627044,49.401868),(19.684514,49.389079),(19.706321,49.387529)] +Slovenia [(16.343426,46.714178),(16.357275,46.715832),(16.357585,46.699011),(16.365383,46.696712),(16.366216,46.696467),(16.371434,46.694929),(16.390038,46.694154),(16.405024,46.687255),(16.410502,46.668367),(16.402607,46.663109),(16.396652,46.659143),(16.377636,46.652864),(16.368437,46.642994),(16.372246,46.636341),(16.376395,46.629093),(16.394585,46.619016),(16.430242,46.604392),(16.467139,46.564704),(16.500832,46.544809),(16.515302,46.501711),(16.491117,46.515146),(16.481298,46.519022),(16.470756,46.520262),(16.449052,46.51835),(16.440371,46.519022),(16.431999,46.523208),(16.415153,46.535559),(16.406264,46.539486),(16.394895,46.540106),(16.382183,46.539383),(16.369471,46.540571),(16.357895,46.546979),(16.351694,46.539486),(16.351254,46.539922),(16.344149,46.546979),(16.340276,46.543827),(16.32999,46.535455),(16.310663,46.530985),(16.295367,46.524448),(16.263947,46.515922),(16.260931,46.513576),(16.234905,46.493339),(16.234188,46.484892),(16.233665,46.47874),(16.237489,46.465072),(16.249375,46.437528),(16.250098,46.429441),(16.248238,46.413344),(16.250925,46.404998),(16.257023,46.399908),(16.274076,46.392105),(16.278727,46.387351),(16.279936,46.378661),(16.280277,46.376214),(16.275626,46.373165),(16.252785,46.373424),(16.217129,46.367352),(16.208654,46.367093),(16.191807,46.369781),(16.177544,46.375594),(16.153567,46.391433),(16.143851,46.394714),(16.131552,46.393061),(16.123181,46.386885),(16.115533,46.379289),(16.106024,46.373734),(16.094345,46.372261),(16.088654,46.373086),(16.057965,46.377532),(16.057448,46.359652),(16.059722,46.345312),(16.059619,46.332315),(16.052384,46.318595),(16.049922,46.316735),(16.038948,46.308441),(16.019208,46.298829),(15.998434,46.291542),(15.993668,46.290613),(15.982001,46.288339),(15.948721,46.284204),(15.918903,46.272827),(15.883712,46.2594),(15.880895,46.259341),(15.879323,46.259308),(15.834206,46.258366),(15.81829,46.255524),(15.803097,46.250511),(15.799526,46.248607),(15.789144,46.24307),(15.768887,46.21935),(15.749767,46.210772),(15.677975,46.214462),(15.661297,46.21532),(15.639799,46.207672),(15.626571,46.19521),(15.62285,46.191704),(15.604349,46.167002),(15.589983,46.138684),(15.58988,46.113517),(15.603833,46.090986),(15.623605,46.076415),(15.631531,46.070574),(15.643727,46.06551),(15.671735,46.057397),(15.683931,46.051144),(15.69288,46.041636),(15.697987,46.036209),(15.693646,46.025771),(15.681967,46.013523),(15.674319,45.993318),(15.674216,45.993163),(15.675381,45.972486),(15.67711,45.941796),(15.67556,45.925157),(15.670392,45.912651),(15.663364,45.900817),(15.659436,45.888828),(15.663674,45.876064),(15.675456,45.855962),(15.676076,45.841699),(15.666051,45.831674),(15.645587,45.824129),(15.626054,45.820202),(15.587296,45.81922),(15.549748,45.823693),(15.523527,45.826816),(15.513916,45.823406),(15.495312,45.812657),(15.48539,45.810176),(15.473918,45.811572),(15.462549,45.814207),(15.451284,45.815137),(15.440328,45.811468),(15.435574,45.802115),(15.439502,45.791676),(15.441052,45.782168),(15.429755,45.775691),(15.429063,45.775295),(15.331031,45.752486),(15.303799,45.746149),(15.263492,45.730388),(15.255017,45.723463),(15.249913,45.713732),(15.248919,45.711836),(15.250883,45.70765),(15.258221,45.705118),(15.267832,45.698452),(15.277754,45.685016),(15.283025,45.680107),(15.291914,45.675559),(15.304936,45.672149),(15.30907,45.674164),(15.310414,45.678195),(15.314961,45.680882),(15.327054,45.683156),(15.330774,45.684551),(15.333875,45.682639),(15.350034,45.669619),(15.351962,45.668066),(15.368291,45.649049),(15.373769,45.640213),(15.353099,45.640316),(15.339146,45.636905),(15.326744,45.632255),(15.297391,45.625692),(15.291604,45.61825),(15.287883,45.610344),(15.281062,45.606158),(15.268556,45.601662),(15.269589,45.593446),(15.276721,45.582697),(15.282179,45.571494),(15.282612,45.570605),(15.288606,45.54456),(15.296668,45.522959),(15.311034,45.505906),(15.361367,45.482031),(15.351755,45.47614),(15.333875,45.458518),(15.325193,45.452834),(15.314031,45.450509),(15.281578,45.450974),(15.225849,45.43645),(15.18422,45.4256),(15.139262,45.430045),(15.066128,45.473934),(15.056166,45.479912),(15.007383,45.480843),(14.997461,45.487199),(14.986299,45.490868),(14.977341,45.491728),(14.962631,45.493142),(14.945372,45.50451),(14.922634,45.514949),(14.904444,45.514432),(14.900826,45.493142),(14.881603,45.469784),(14.838815,45.458983),(14.797164,45.465185),(14.781454,45.493142),(14.781351,45.493193),(14.781247,45.493348),(14.781144,45.493348),(14.688333,45.522029),(14.668489,45.533966),(14.668179,45.539134),(14.671693,45.556807),(14.669833,45.564558),(14.664045,45.570036),(14.663675,45.570187),(14.657327,45.572775),(14.650299,45.574842),(14.615986,45.594066),(14.603067,45.603574),(14.593868,45.616183),(14.592938,45.629671),(14.594075,45.648016),(14.591905,45.663364),(14.580949,45.667808),(14.569095,45.664252),(14.563896,45.662692),(14.556145,45.656697),(14.543639,45.63644),(14.533097,45.625692),(14.507879,45.605848),(14.498577,45.596184),(14.492273,45.58342),(14.491502,45.579165),(14.487415,45.5566),(14.482144,45.542441),(14.468915,45.525594),(14.429744,45.505389),(14.411244,45.493193),(14.372797,45.477845),(14.326805,45.4749),(14.280399,45.481101),(14.240608,45.493348),(14.218698,45.497172),(14.193066,45.491901),(14.145524,45.476243),(14.119686,45.472884),(14.117321,45.472976),(14.116747,45.472998),(14.092917,45.473918),(14.066665,45.480274),(14.041551,45.493142),(14.041551,45.493193),(14.028322,45.50265),(14.013956,45.507973),(13.987476,45.511909),(13.971891,45.514226),(13.964966,45.51097),(13.961452,45.50389),(13.961452,45.493142),(13.98264,45.475313),(13.969824,45.462963),(13.923005,45.448958),(13.908432,45.438881),(13.899647,45.429218),(13.889002,45.423637),(13.835568,45.429114),(13.819859,45.432628),(13.806526,45.442137),(13.759294,45.463169),(13.65997,45.459978),(13.629018,45.458983),(13.589529,45.488837),(13.591645,45.493109),(13.589122,45.501614),(13.595958,45.511908),(13.595958,45.518134),(13.586192,45.519477),(13.578624,45.523383),(13.572765,45.529975),(13.568614,45.539252),(13.58961,45.535793),(13.675141,45.544745),(13.75294,45.552883),(13.75294,45.559068),(13.742361,45.56977),(13.727387,45.581204),(13.711762,45.593207),(13.761051,45.596236),(13.800532,45.58125),(13.847764,45.584661),(13.867926,45.602169),(13.887038,45.618767),(13.894686,45.631841),(13.893641,45.633758),(13.893136,45.634683),(13.884144,45.635148),(13.869158,45.641143),(13.858409,45.649359),(13.778724,45.743411),(13.709478,45.765321),(13.699908,45.770524),(13.660334,45.792038),(13.643591,45.795655),(13.609174,45.798601),(13.581269,45.809246),(13.574172,45.819028),(13.565973,45.83033),(13.56928,45.86454),(13.599306,45.912327),(13.608244,45.926552),(13.615325,45.945912),(13.622817,45.966394),(13.605867,45.985411),(13.600735,45.984574),(13.571657,45.97983),(13.539411,45.96903),(13.509439,45.967428),(13.482257,45.989235),(13.481843,45.990372),(13.481223,45.991354),(13.480396,45.992284),(13.479466,45.993111),(13.474815,45.995747),(13.461793,46.006392),(13.477089,46.016055),(13.482257,46.018433),(13.490008,46.025564),(13.492799,46.032489),(13.490318,46.038948),(13.482257,46.044839),(13.505098,46.066027),(13.522585,46.075298),(13.616409,46.125041),(13.645037,46.161731),(13.641071,46.171405),(13.637389,46.180386),(13.627366,46.181733),(13.613928,46.183539),(13.584473,46.181317),(13.559048,46.184107),(13.528972,46.204829),(13.510162,46.213976),(13.482257,46.217904),(13.468304,46.223433),(13.438332,46.22488),(13.422829,46.228601),(13.437402,46.210927),(13.410116,46.207982),(13.401641,46.216663),(13.398127,46.230513),(13.384898,46.243122),(13.384795,46.243225),(13.384692,46.243328),(13.378594,46.268391),(13.373013,46.280277),(13.365261,46.290302),(13.391306,46.301568),(13.395638,46.306728),(13.409806,46.323608),(13.423242,46.344847),(13.434094,46.353864),(13.447323,46.354846),(13.459726,46.359032),(13.483704,46.37115),(13.530006,46.388332),(13.554397,46.405954),(13.575688,46.426676),(13.600182,46.442644),(13.634082,46.445719),(13.658783,46.445125),(13.67749,46.452075),(13.685853,46.464047),(13.688446,46.467759),(13.689893,46.493339),(13.695474,46.498636),(13.699194,46.504837),(13.701055,46.511891),(13.700951,46.519746),(13.716093,46.518867),(13.782135,46.507782),(13.795778,46.507886),(13.860683,46.51525),(13.890862,46.511787),(13.98233,46.481918),(13.998763,46.480523),(14.014922,46.482531),(14.032456,46.484709),(14.050026,46.484399),(14.066355,46.48104),(14.081032,46.47595),(14.137255,46.442438),(14.147933,46.440425),(14.149865,46.440061),(14.242323,46.438237),(14.362151,46.435875),(14.395844,46.440991),(14.40649,46.439337),(14.411244,46.434635),(14.414448,46.429002),(14.420029,46.424351),(14.437145,46.418884),(14.450931,46.414481),(14.467675,46.412672),(14.502194,46.418356),(14.515837,46.40536),(14.527102,46.388229),(14.540332,46.378643),(14.557695,46.38394),(14.562108,46.391737),(14.566997,46.400373),(14.575368,46.419726),(14.590148,46.434428),(14.599863,46.437167),(14.621567,46.43851),(14.631696,46.440577),(14.642031,46.445228),(14.662081,46.459698),(14.666629,46.460524),(14.672546,46.459791),(14.679961,46.458871),(14.687093,46.471221),(14.698772,46.480962),(14.703733,46.487758),(14.709727,46.492512),(14.726367,46.497706),(14.735255,46.493339),(14.760887,46.496284),(14.783004,46.503261),(14.788585,46.506646),(14.796663,46.519401),(14.807189,46.536024),(14.814519,46.551337),(14.822278,46.567546),(14.833647,46.584393),(14.85039,46.601136),(14.862999,46.604831),(14.877055,46.603642),(14.894063,46.605215),(14.897726,46.605554),(14.919507,46.615017),(14.933589,46.621135),(14.947955,46.619274),(14.967179,46.600257),(15.004386,46.636844),(15.061954,46.649557),(15.085723,46.647795),(15.105591,46.646323),(15.172557,46.64136),(15.204891,46.638963),(15.332784,46.64358),(15.388135,46.645578),(15.417591,46.637955),(15.435381,46.627195),(15.440018,46.62439),(15.462653,46.614649),(15.492625,46.618293),(15.511228,46.628369),(15.513939,46.632545),(15.517636,46.63824),(15.520308,46.64772),(15.52084,46.649608),(15.530659,46.663845),(15.545955,46.671881),(15.567349,46.675757),(15.58864,46.675963),(15.603729,46.673018),(15.616648,46.67555),(15.621745,46.678175),(15.626984,46.680873),(15.632978,46.689632),(15.632875,46.702473),(15.635975,46.717563),(15.652098,46.710819),(15.728683,46.70299),(15.755141,46.704024),(15.78461,46.7122),(15.822941,46.722834),(15.850743,46.724488),(15.878545,46.720715),(15.946344,46.697151),(15.986962,46.69219),(15.997917,46.686919),(16.016593,46.670757),(16.016727,46.670641),(16.016723,46.670691),(16.014557,46.693714),(16.003291,46.709191),(15.982001,46.718545),(15.970529,46.743014),(15.969702,46.760532),(15.970985,46.77505),(15.971252,46.778076),(15.978177,46.809134),(15.97735,46.816213),(15.972906,46.818435),(15.971976,46.820632),(15.981691,46.827685),(15.987582,46.830011),(16.028441,46.836947),(16.032024,46.837556),(16.052936,46.84606),(16.094035,46.862774),(16.130246,46.856708),(16.135376,46.855849),(16.179486,46.858468),(16.272009,46.863962),(16.282241,46.859932),(16.297392,46.847033),(16.301878,46.843214),(16.310663,46.84001),(16.325339,46.839442),(16.329783,46.834403),(16.327509,46.825463),(16.321825,46.813268),(16.3149,46.802002),(16.311097,46.797519),(16.302188,46.787016),(16.29949,46.77951),(16.298157,46.775802),(16.300431,46.772082),(16.314177,46.743324),(16.325546,46.733273),(16.334124,46.721749),(16.343426,46.714178)] +Swaziland [(31.863357,-25.989937),(31.891882,-25.983839),(31.949243,-25.958104),(31.975288,-25.980429),(32.00402,-25.994381),(32.070579,-26.009781),(32.057454,-26.0412),(32.062725,-26.077167),(32.074559,-26.114167),(32.081432,-26.14879),(32.077814,-26.175455),(32.044741,-26.25452),(32.039884,-26.283666),(32.052338,-26.386812),(32.059831,-26.414821),(32.107063,-26.500293),(32.117398,-26.582252),(32.113884,-26.840014),(32.097761,-26.833503),(32.073163,-26.811386),(32.057144,-26.808595),(31.990068,-26.808285),(31.992238,-26.838257),(31.975288,-26.926624),(31.959682,-27.00879),(31.942112,-27.10067),(31.944204,-27.162338),(31.944851,-27.181389),(31.947383,-27.255493),(31.952964,-27.270169),(31.959785,-27.281021),(31.964849,-27.291666),(31.967743,-27.303035),(31.96826,-27.316264),(31.878602,-27.315231),(31.782535,-27.313991),(31.636911,-27.312234),(31.52653,-27.31089),(31.459454,-27.298694),(31.35667,-27.267068),(31.280137,-27.243504),(31.244707,-27.232586),(31.157043,-27.205573),(31.141954,-27.196478),(31.126503,-27.182629),(31.078754,-27.11824),(30.976073,-27.035041),(30.953645,-27.000315),(30.949408,-26.975407),(30.959743,-26.936133),(30.960983,-26.911224),(30.955092,-26.891381),(30.944447,-26.876601),(30.915818,-26.849419),(30.902589,-26.831333),(30.885294,-26.785641),(30.880213,-26.772215),(30.868741,-26.784927),(30.853393,-26.796606),(30.836443,-26.805081),(30.819803,-26.807975),(30.80244,-26.808905),(30.79567,-26.785547),(30.785697,-26.716921),(30.784146,-26.578842),(30.782906,-26.472388),(30.804559,-26.397457),(30.897318,-26.291727),(30.897334,-26.291709),(30.969871,-26.209148),(31.037774,-26.100111),(31.091621,-25.983633),(31.106866,-25.930923),(31.119836,-25.910045),(31.20841,-25.838628),(31.309282,-25.757393),(31.337394,-25.744681),(31.372017,-25.736412),(31.401473,-25.735999),(31.426898,-25.743647),(31.532835,-25.805556),(31.638771,-25.867464),(31.7306,-25.921207),(31.746966,-25.93079),(31.834573,-25.982082),(31.863357,-25.989937)] +Sint Maarten [(-63.017569,18.033391),(-63.030629,18.01911),(-63.097646,18.035956),(-63.112172,18.043036),(-63.118886,18.0515),(-63.107004,18.059475),(-63.107004,18.062109),(-63.085886,18.058511),(-63.017569,18.033391)] +Syria [(42.236832,37.286304),(42.267218,37.274522),(42.272902,37.276744),(42.279207,37.282273),(42.285822,37.286562),(42.292023,37.285064),(42.296364,37.279689),(42.301428,37.269923),(42.305769,37.264548),(42.339875,37.242637),(42.346696,37.239769),(42.353208,37.227031),(42.335328,37.171169),(42.346696,37.158431),(42.354551,37.15241),(42.356515,37.138277),(42.357238,37.109984),(42.363646,37.09815),(42.371191,37.087944),(42.376875,37.076756),(42.377186,37.062235),(42.376806,37.062001),(42.34587,37.042908),(42.281894,36.994125),(42.281894,36.994022),(42.281894,36.99397),(42.281584,36.99397),(42.178438,36.90532),(41.978554,36.733625),(41.843781,36.617869),(41.817323,36.599731),(41.789935,36.589292),(41.479773,36.536117),(41.414867,36.527384),(41.385411,36.516377),(41.365361,36.494156),(41.365361,36.494053),(41.365258,36.494001),(41.365258,36.493898),(41.276994,36.354785),(41.268829,36.327965),(41.236583,36.077024),(41.236687,36.060332),(41.240614,36.043021),(41.266246,35.99429),(41.266349,35.994342),(41.266349,35.994238),(41.343657,35.857657),(41.354509,35.825566),(41.359263,35.792752),(41.363501,35.655241),(41.358023,35.623925),(41.34221,35.593694),(41.308458,35.552248),(41.261285,35.49432),(41.261285,35.494165),(41.261181,35.494165),(41.261078,35.494165),(41.25188,35.46409),(41.243095,35.366525),(41.20134,35.243018),(41.191521,35.182143),(41.192326,35.158904),(41.198033,34.994041),(41.206508,34.819323),(41.204234,34.793123),(41.195656,34.768473),(41.023986,34.49433),(41.023986,34.494175),(40.98802,34.42852),(40.965282,34.401855),(40.936033,34.386068),(40.690467,34.331497),(40.543809,34.257988),(40.433221,34.202539),(40.322634,34.147167),(40.212046,34.091796),(40.173111,34.072283),(40.101459,34.036373),(39.990871,33.980976),(39.880387,33.925605),(39.769696,33.870233),(39.659212,33.81481),(39.548624,33.759491),(39.438037,33.704094),(39.327449,33.648722),(39.216862,33.593325),(39.106274,33.537928),(38.995686,33.482479),(38.885099,33.427108),(38.774511,33.371685),(38.529565,33.244251),(38.315742,33.13118),(38.230875,33.086302),(38.056726,32.994344),(38.056726,32.994292),(37.929395,32.92533),(37.758036,32.832519),(37.586677,32.739837),(37.494606,32.690056),(37.415214,32.64713),(37.244062,32.554396),(37.133371,32.494581),(37.133371,32.494529),(37.133165,32.494529),(37.133165,32.494478),(36.980099,32.410038),(36.819385,32.316788),(36.806569,32.313042),(36.792513,32.313533),(36.728641,32.327795),(36.706937,32.328338),(36.689574,32.319656),(36.653504,32.342859),(36.516684,32.357014),(36.480181,32.360791),(36.463955,32.369395),(36.407627,32.374227),(36.387887,32.379317),(36.373108,32.386422),(36.285258,32.456935),(36.220765,32.494581),(36.188209,32.52228),(36.177357,32.527318),(36.17219,32.525923),(36.160821,32.517215),(36.15586,32.5152),(36.149865,32.51613),(36.13953,32.519541),(36.133226,32.520109),(36.096225,32.515872),(36.081906,32.516265),(36.06987,32.516595),(36.066253,32.517319),(36.066046,32.521608),(36.060465,32.533261),(36.015403,32.591164),(36.005998,32.607907),(36.005275,32.626692),(36.008272,32.643719),(36.003621,32.655088),(35.980263,32.656612),(35.965794,32.654365),(35.955355,32.657439),(35.94657,32.664441),(35.937475,32.674002),(35.941196,32.673536),(35.9444,32.677619),(35.945743,32.684104),(35.944193,32.690771),(35.940369,32.692502),(35.92745,32.692373),(35.922489,32.693768),(35.905229,32.708573),(35.895721,32.713276),(35.788234,32.734411),(35.779035,32.744282),(35.779035,32.744359),(35.779139,32.744462),(35.779139,32.744514),(35.774901,32.747279),(35.769734,32.748054),(35.763842,32.746969),(35.75759,32.744347),(35.784203,32.777949),(35.834226,32.827946),(35.841874,32.853577),(35.83805,32.866031),(35.849729,32.895823),(35.866885,32.920782),(35.874017,32.922333),(35.888073,32.944941),(35.864611,32.97773),(35.85903,32.99021),(35.845801,33.085423),(35.848902,33.098678),(35.811488,33.111908),(35.811488,33.126765),(35.822443,33.14157),(35.833399,33.161129),(35.830195,33.189991),(35.807664,33.201721),(35.803633,33.248463),(35.775625,33.264896),(35.768597,33.272699),(35.802083,33.31249),(35.763842,33.334401),(35.785753,33.342875),(35.793505,33.349929),(35.809938,33.360032),(35.812315,33.373365),(35.815415,33.378868),(35.816966,33.395198),(35.822443,33.401373),(35.8211,33.406722),(35.845116,33.418742),(35.870089,33.431242),(35.888486,33.440415),(35.919492,33.462351),(35.935408,33.494262),(35.921869,33.51457),(35.933238,33.527825),(35.956802,33.534207),(35.980263,33.533949),(35.99928,33.541261),(36.019847,33.55263),(36.035247,33.567797),(36.038451,33.586349),(36.029666,33.59764),(36.012923,33.608027),(35.994526,33.615701),(35.980263,33.619008),(35.956906,33.629938),(35.934788,33.6334),(35.921559,33.640299),(35.925383,33.661667),(35.929414,33.665078),(35.942436,33.669212),(35.94719,33.673088),(35.949878,33.6827),(35.946984,33.700838),(35.948431,33.70921),(35.955872,33.718124),(35.974165,33.732154),(35.980263,33.743549),(35.994009,33.760705),(36.027082,33.784296),(36.040518,33.805612),(36.05044,33.816438),(36.06646,33.821994),(36.085063,33.82605),(36.102426,33.832277),(36.13829,33.850622),(36.144904,33.847418),(36.157617,33.83406),(36.165678,33.829797),(36.186556,33.83021),(36.202679,33.838297),(36.216631,33.847754),(36.231824,33.852534),(36.249187,33.849744),(36.300554,33.828686),(36.338278,33.821193),(36.357501,33.823776),(36.369594,33.837006),(36.36763,33.857547),(36.35037,33.866409),(36.311096,33.87199),(36.28164,33.891111),(36.275336,33.897209),(36.268928,33.906769),(36.267171,33.910386),(36.288565,33.946508),(36.302311,33.964388),(36.31802,33.980511),(36.357811,34.009553),(36.391091,34.044719),(36.412175,34.053633),(36.423647,34.052806),(36.445144,34.04614),(36.45889,34.046708),(36.47522,34.053504),(36.480698,34.062754),(36.482662,34.07451),(36.488449,34.088644),(36.507983,34.109185),(36.552735,34.140992),(36.575886,34.173264),(36.604101,34.199102),(36.603554,34.200101),(36.59821,34.209851),(36.574026,34.229746),(36.567514,34.244138),(36.56276,34.262949),(36.570305,34.275868),(36.579917,34.286901),(36.581984,34.299897),(36.576816,34.307674),(36.570305,34.308966),(36.562967,34.308501),(36.555835,34.311085),(36.54519,34.320723),(36.523589,34.345889),(36.518628,34.353641),(36.515941,34.36165),(36.516768,34.369867),(36.519972,34.375061),(36.523589,34.379789),(36.526276,34.386688),(36.52979,34.403095),(36.530617,34.41418),(36.52514,34.422939),(36.509223,34.432421),(36.501885,34.430303),(36.494651,34.434256),(36.480181,34.449139),(36.463231,34.455237),(36.449795,34.464538),(36.439873,34.477457),(36.433466,34.494175),(36.433466,34.49433),(36.41972,34.498335),(36.392331,34.500143),(36.363082,34.499032),(36.343652,34.494175),(36.340965,34.493477),(36.338174,34.493296),(36.335384,34.493477),(36.332697,34.494175),(36.329851,34.498603),(36.319881,34.514122),(36.336624,34.52885),(36.364012,34.541097),(36.383546,34.553809),(36.388094,34.569519),(36.386853,34.584247),(36.388921,34.596236),(36.4037,34.603832),(36.418479,34.600318),(36.429125,34.593187),(36.436256,34.597838),(36.440184,34.62936),(36.415999,34.622901),(36.391194,34.622487),(36.367526,34.629205),(36.347373,34.64414),(36.323808,34.679486),(36.308925,34.687548),(36.288151,34.675559),(36.284844,34.667911),(36.284431,34.648326),(36.28071,34.639127),(36.271512,34.630962),(36.26159,34.627242),(36.201232,34.624451),(36.195134,34.626208),(36.183352,34.633494),(36.178287,34.635303),(36.17219,34.634063),(36.160201,34.628068),(36.15586,34.626931),(36.11214,34.629532),(36.087234,34.631014),(36.060775,34.627913),(36.037934,34.62843),(36.01623,34.633649),(35.998763,34.64538),(35.991425,34.648222),(35.98264,34.650238),(35.9699,34.649849),(35.965668,34.668443),(35.947439,34.69953),(35.940929,34.726223),(35.931163,34.740953),(35.928966,34.749823),(35.928966,34.790717),(35.926036,34.796942),(35.908458,34.821763),(35.904552,34.831773),(35.899099,34.852769),(35.866873,34.924221),(35.872895,34.926907),(35.873546,34.931627),(35.879568,34.938056),(35.881358,34.944973),(35.879568,34.95189),(35.873546,34.958319),(35.873546,34.965155),(35.878591,34.96955),(35.878917,34.97016),(35.876638,34.971666),(35.873546,34.978827),(35.881602,34.992865),(35.88738,35.00019),(35.895356,35.006781),(35.900564,35.021918),(35.897309,35.036566),(35.891449,35.049954),(35.887869,35.061347),(35.887869,35.109198),(35.894216,35.12108),(35.941254,35.180569),(35.962413,35.197943),(35.953298,35.224799),(35.915294,35.287909),(35.928966,35.321479),(35.924083,35.338528),(35.916677,35.403306),(35.918956,35.41767),(35.908865,35.425727),(35.860606,35.479071),(35.840099,35.487372),(35.830089,35.493476),(35.825857,35.503323),(35.81837,35.508938),(35.802257,35.506903),(35.786794,35.502183),(35.781261,35.499579),(35.770518,35.504218),(35.771983,35.515448),(35.77711,35.52912),(35.778087,35.541164),(35.771983,35.546332),(35.737071,35.561672),(35.73878,35.566596),(35.740896,35.576483),(35.743337,35.582099),(35.737071,35.582099),(35.723399,35.575914),(35.723399,35.582099),(35.743419,35.587958),(35.763357,35.598293),(35.778494,35.612738),(35.784353,35.630561),(35.782074,35.64175),(35.773204,35.655463),(35.771169,35.667792),(35.776622,35.675116),(35.800059,35.681464),(35.838552,35.742092),(35.840343,35.747463),(35.840099,35.770819),(35.835948,35.789984),(35.825369,35.813463),(35.81186,35.835395),(35.798595,35.849677),(35.846853,35.847235),(35.87086,35.851793),(35.881033,35.866685),(35.885265,35.885972),(35.911305,35.91775),(35.920835,35.915122),(35.940576,35.918377),(35.96073,35.924372),(35.980263,35.926852),(35.992769,35.915277),(35.993079,35.902357),(35.990082,35.889283),(35.992356,35.877346),(36.003931,35.869181),(36.019434,35.866184),(36.05075,35.865409),(36.080309,35.854195),(36.138497,35.819779),(36.155308,35.822232),(36.157617,35.822569),(36.168779,35.852076),(36.174877,35.922666),(36.196891,35.951708),(36.229034,35.961165),(36.253115,35.955636),(36.269031,35.958736),(36.277403,35.99398),(36.277403,35.994031),(36.277506,35.994187),(36.277506,35.994238),(36.29735,36.000749),(36.336727,35.988244),(36.358018,35.99398),(36.355538,36.022402),(36.358638,36.166992),(36.360498,36.181669),(36.366286,36.189213),(36.374451,36.19588),(36.375795,36.20451),(36.361429,36.218152),(36.373004,36.227609),(36.385613,36.215465),(36.398119,36.20823),(36.411865,36.204458),(36.439253,36.201409),(36.445248,36.19991),(36.450726,36.19991),(36.4592,36.203114),(36.463955,36.208799),(36.465298,36.21593),(36.468916,36.22239),(36.480181,36.225542),(36.498578,36.227867),(36.574439,36.218359),(36.593456,36.2181),(36.664253,36.229004),(36.67004,36.237324),(36.66911,36.259855),(36.665493,36.274014),(36.658465,36.290499),(36.648761,36.305719),(36.648646,36.305899),(36.637071,36.316802),(36.620741,36.322022),(36.587771,36.324812),(36.577643,36.332874),(36.58219,36.333081),(36.587771,36.342124),(36.594283,36.35711),(36.594489,36.365947),(36.593352,36.373492),(36.590562,36.380364),(36.585704,36.387031),(36.580847,36.389873),(36.565034,36.395609),(36.558006,36.400622),(36.552735,36.408166),(36.533821,36.462478),(36.531237,36.479015),(36.535475,36.494001),(36.565344,36.5325),(36.569995,36.575701),(36.569271,36.620143),(36.582811,36.662414),(36.594696,36.685824),(36.598107,36.705461),(36.598934,36.724788),(36.603274,36.746957),(36.615677,36.766594),(36.632006,36.784319),(36.643272,36.803646),(36.639861,36.828089),(36.649163,36.828554),(36.658568,36.827521),(36.659563,36.827243),(36.843673,36.775844),(36.906822,36.772589),(36.925942,36.768971),(36.945889,36.762667),(36.964699,36.75383),(36.980099,36.742255),(37.010588,36.719517),(37.018443,36.706649),(37.019476,36.685824),(37.010795,36.653733),(37.014515,36.642261),(37.033326,36.628721),(37.053066,36.619936),(37.062368,36.622107),(37.070119,36.631512),(37.085622,36.644173),(37.104432,36.650994),(37.120969,36.649909),(37.137402,36.645878),(37.156109,36.643914),(37.219671,36.657143),(37.241892,36.65859),(37.404466,36.634458),(37.446117,36.634199),(37.593395,36.710525),(37.654063,36.732126),(37.819738,36.760548),(37.980038,36.819666),(38.007943,36.825764),(38.026005,36.835006),(38.122975,36.884623),(38.190051,36.905526),(38.224261,36.908394),(38.28989,36.901702),(38.479956,36.855736),(38.529358,36.833722),(38.635192,36.744039),(38.66413,36.719517),(38.725005,36.693937),(38.795389,36.686651),(38.957963,36.692904),(38.979977,36.698175),(39.03217,36.701172),(39.185856,36.659521),(39.239599,36.661278),(39.441708,36.692373),(39.765252,36.742151),(39.979812,36.806953),(40.079238,36.854857),(40.115205,36.864728),(40.152722,36.87036),(40.190136,36.884416),(40.261409,36.922785),(40.393741,36.994022),(40.413894,37.004022),(40.435185,37.01061),(40.456993,37.014124),(40.479834,37.015158),(40.525722,37.025881),(40.659461,37.085257),(40.708967,37.100476),(40.896449,37.122696),(41.135091,37.084249),(41.179739,37.069625),(41.20134,37.064974),(41.479773,37.075568),(41.896365,37.154283),(42.009249,37.175613),(42.052968,37.196645),(42.128312,37.253283),(42.178541,37.306303),(42.193011,37.313227),(42.211201,37.324906),(42.22257,37.313951),(42.22381,37.302375),(42.222156,37.292402),(42.22319,37.288139),(42.236832,37.286304)] +Chad [(23.981306,19.496124),(23.981512,19.263838),(23.981719,19.031553),(23.981926,18.799293),(23.982133,18.567033),(23.982236,18.334799),(23.982443,18.102514),(23.982753,17.870228),(23.982856,17.637943),(23.983063,17.405657),(23.983269,17.173371),(23.983476,16.941138),(23.983683,16.7088),(23.983786,16.476515),(23.983993,16.244229),(23.984303,16.011944),(23.984406,15.779658),(23.984406,15.72194),(23.984406,15.72116),(23.972624,15.691085),(23.945546,15.692222),(23.829067,15.731031),(23.707524,15.748859),(23.592699,15.749014),(23.52707,15.735165),(23.395812,15.688346),(23.320571,15.681318),(23.166782,15.712944),(23.119249,15.707224),(23.094642,15.704262),(23.068493,15.686692),(23.004518,15.611425),(22.92566,15.563935),(22.906746,15.541404),(22.899511,15.510191),(22.906849,15.481408),(22.923593,15.455569),(22.944987,15.433116),(22.966484,15.404823),(22.976406,15.373636),(22.97837,15.340202),(22.965657,15.223904),(22.958009,15.201631),(22.913257,15.121688),(22.905506,15.112567),(22.871606,15.0997),(22.848765,15.087504),(22.829025,15.071639),(22.738901,14.97981),(22.727222,14.959036),(22.721951,14.934671),(22.721021,14.915938),(22.715337,14.899428),(22.695493,14.88147),(22.658906,14.857415),(22.650844,14.841938),(22.650844,14.816151),(22.659009,14.761271),(22.66459,14.743752),(22.67968,14.711816),(22.681437,14.702928),(22.676579,14.688975),(22.666347,14.681741),(22.465429,14.629289),(22.422745,14.609084),(22.382334,14.579111),(22.363523,14.543532),(22.386364,14.506532),(22.417164,14.484104),(22.424915,14.470436),(22.439074,14.362923),(22.442382,14.356928),(22.447343,14.351632),(22.45096,14.345792),(22.449926,14.338377),(22.444862,14.333364),(22.431323,14.327214),(22.426982,14.323003),(22.418094,14.30595),(22.41706,14.298095),(22.419954,14.286235),(22.429049,14.264608),(22.441038,14.249855),(22.457678,14.241147),(22.514109,14.231458),(22.531885,14.220632),(22.540877,14.201176),(22.547698,14.169033),(22.546355,14.139707),(22.531472,14.122627),(22.508217,14.113636),(22.481449,14.108546),(22.458711,14.096117),(22.421608,14.06214),(22.402281,14.049299),(22.243531,13.975427),(22.214695,13.956617),(22.190614,13.932794),(22.180762,13.920555),(22.099353,13.819416),(22.074962,13.780323),(22.073722,13.771357),(22.09708,13.749756),(22.112893,13.729835),(22.116303,13.715185),(22.11434,13.699708),(22.11434,13.677383),(22.132116,13.638652),(22.195885,13.580464),(22.210561,13.541733),(22.211181,13.484191),(22.215625,13.464993),(22.228648,13.441119),(22.263995,13.399183),(22.275984,13.376316),(22.267612,13.334562),(22.232265,13.289035),(22.139971,13.193511),(22.123435,13.182168),(22.016051,13.140233),(21.998378,13.130595),(21.964065,13.098349),(21.935126,13.059152),(21.85296,12.905725),(21.827846,12.831104),(21.811826,12.799969),(21.809449,12.793665),(21.813686,12.782296),(21.840868,12.748939),(21.880142,12.676282),(21.900813,12.656438),(21.935953,12.63954),(21.97719,12.63184),(22.019668,12.631426),(22.058736,12.636697),(22.105865,12.650392),(22.144105,12.671269),(22.176248,12.7015),(22.204877,12.743358),(22.33014,12.661451),(22.43246,12.62383),(22.445689,12.611066),(22.395873,12.496086),(22.372825,12.463116),(22.374892,12.450869),(22.407965,12.399761),(22.481449,12.176674),(22.484033,12.164737),(22.484756,12.152541),(22.483826,12.140345),(22.458091,12.03043),(22.464499,12.032962),(22.470494,12.036114),(22.481449,12.044279),(22.546458,12.064433),(22.585525,12.071357),(22.598858,12.063244),(22.607333,12.077714),(22.612397,12.072804),(22.613534,12.05725),(22.610537,12.039628),(22.592863,11.988933),(22.537053,11.68089),(22.541704,11.632986),(22.561754,11.586012),(22.592347,11.543663),(22.628107,11.509841),(22.662523,11.493072),(22.743035,11.466097),(22.76877,11.442326),(22.771561,11.433308),(22.771871,11.403181),(22.781792,11.398892),(22.87698,11.40884),(22.900442,11.408245),(22.914808,11.396101),(22.928347,11.32515),(22.953668,11.250942),(22.954536,11.237843),(22.956459,11.2088),(22.952015,11.191049),(22.935168,11.155883),(22.928967,11.137926),(22.92566,11.118134),(22.925143,11.097954),(22.922352,11.087076),(22.906539,11.069894),(22.900235,11.059895),(22.861064,10.919154),(22.805977,10.924502),(22.745102,10.943803),(22.719884,10.964655),(22.682677,10.974137),(22.673169,10.975068),(22.638339,10.974137),(22.62821,10.976127),(22.60909,10.985222),(22.600822,10.987806),(22.57705,10.985119),(22.55576,10.978969),(22.534676,10.98052),(22.511938,11.000828),(22.502843,10.992173),(22.490751,10.993154),(22.476281,10.99796),(22.460468,11.000828),(22.447136,10.99827),(22.437627,10.991552),(22.419231,10.970727),(22.407035,10.963053),(22.391842,10.958505),(22.36469,10.954209),(22.361146,10.953648),(22.339442,10.947731),(22.318772,10.939359),(22.305956,10.932564),(22.287249,10.91595),(22.2765,10.908534),(22.268439,10.908999),(22.253039,10.915304),(22.24012,10.905899),(22.229371,10.891972),(22.200639,10.868666),(22.189787,10.837298),(22.176351,10.816214),(22.148239,10.830761),(22.132426,10.828539),(22.048607,10.836988),(22.029487,10.828746),(22.018325,10.809884),(22.0114,10.789291),(22.004889,10.775519),(22.014604,10.754564),(22.003855,10.743273),(21.943084,10.728261),(21.925617,10.720122),(21.895025,10.699193),(21.872081,10.677592),(21.86836,10.676223),(21.863606,10.667929),(21.852754,10.670151),(21.83291,10.679944),(21.822575,10.678755),(21.803558,10.673665),(21.795393,10.672476),(21.784851,10.669324),(21.778753,10.66227),(21.774205,10.655165),(21.768418,10.651987),(21.753431,10.650721),(21.736482,10.646173),(21.722632,10.636716),(21.716845,10.621007),(21.714674,10.601395),(21.705476,10.571811),(21.703305,10.552406),(21.705476,10.53202),(21.714674,10.493702),(21.716845,10.47732),(21.720462,10.468018),(21.738032,10.44802),(21.744233,10.439106),(21.74444,10.432439),(21.742786,10.425773),(21.743613,10.418978),(21.751054,10.41182),(21.720462,10.360816),(21.716845,10.339809),(21.717775,10.328983),(21.722116,10.314178),(21.723046,10.305987),(21.718808,10.296634),(21.708886,10.293533),(21.697414,10.292422),(21.688939,10.288908),(21.67695,10.269168),(21.668889,10.249195),(21.65628,10.233666),(21.630648,10.227439),(21.625274,10.224493),(21.593441,10.214546),(21.589721,10.216613),(21.579282,10.212272),(21.566673,10.214494),(21.554477,10.218654),(21.544969,10.219997),(21.513239,10.200515),(21.485437,10.164316),(21.433348,10.046055),(21.406889,10.005566),(21.374333,9.973113),(21.339503,9.95991),(21.326067,9.962752),(21.28514,9.978591),(21.27129,9.987247),(21.256924,9.975749),(21.205661,9.916838),(21.198427,9.902988),(21.195429,9.887305),(21.187264,9.885134),(21.131971,9.849426),(21.121739,9.831907),(21.106753,9.79545),(21.105926,9.791884),(21.106753,9.778061),(21.105202,9.77403),(21.101585,9.774237),(21.097037,9.775373),(21.09311,9.77434),(21.080294,9.762609),(21.073163,9.757648),(21.065825,9.754496),(21.062207,9.756201),(21.052906,9.765891),(21.048358,9.768165),(21.042674,9.767079),(21.04009,9.764754),(21.039056,9.76217),(21.037816,9.760671),(21.032339,9.75801),(21.025621,9.753256),(21.019936,9.747933),(21.017352,9.743592),(21.014252,9.724472),(21.000092,9.697394),(20.996888,9.682175),(20.996062,9.648301),(20.989964,9.635615),(20.975804,9.630964),(20.975804,9.624168),(20.980145,9.619931),(20.981799,9.616778),(20.983143,9.603678),(20.96764,9.610603),(20.962265,9.606934),(20.960508,9.598123),(20.955961,9.589416),(20.934257,9.577065),(20.925058,9.569598),(20.921234,9.558978),(20.912656,9.544328),(20.874209,9.502496),(20.86377,9.493789),(20.841446,9.484409),(20.834314,9.462473),(20.832661,9.437255),(20.82615,9.418083),(20.820258,9.416222),(20.813884,9.418847),(20.81323,9.419116),(20.806719,9.423328),(20.801965,9.42555),(20.798244,9.423379),(20.792353,9.414),(20.788322,9.411881),(20.77654,9.407799),(20.771062,9.398058),(20.766618,9.38656),(20.757937,9.377129),(20.750495,9.384545),(20.740883,9.377129),(20.695822,9.363461),(20.654894,9.342971),(20.667916,9.302017),(20.616653,9.302017),(20.609729,9.302792),(20.592882,9.308218),(20.592262,9.308063),(20.573555,9.309149),(20.572315,9.308218),(20.550921,9.320259),(20.541826,9.321629),(20.538208,9.311939),(20.537278,9.299201),(20.533661,9.289641),(20.526426,9.283621),(20.514231,9.281502),(20.496144,9.270882),(20.502655,9.247421),(20.51299,9.223883),(20.507203,9.213263),(20.481778,9.204659),(20.467618,9.184143),(20.455423,9.159313),(20.435166,9.138126),(20.423073,9.143241),(20.412428,9.142053),(20.40292,9.136989),(20.384419,9.123579),(20.378322,9.120065),(20.359408,9.116421),(20.276829,9.120736),(20.256985,9.116421),(20.234454,9.13456),(20.222982,9.14195),(20.212647,9.144973),(20.197971,9.140503),(20.169549,9.120891),(20.154563,9.116421),(20.137613,9.121822),(20.125727,9.133785),(20.117149,9.145748),(20.109604,9.151174),(20.101026,9.148073),(20.099579,9.141174),(20.100509,9.134146),(20.099372,9.13071),(20.090587,9.131123),(20.076945,9.13673),(20.068676,9.138126),(20.060512,9.134353),(20.031159,9.11022),(20.028369,9.104562),(20.029919,9.098981),(20.030332,9.093865),(20.024338,9.089756),(20.01793,9.091048),(20.014416,9.106345),(20.006871,9.11022),(20.002944,9.108515),(19.999533,9.104303),(19.997156,9.098619),(19.996329,9.092831),(19.993022,9.084589),(19.985477,9.083193),(19.976486,9.084175),(19.969044,9.082883),(19.943413,9.065882),(19.932091,9.063331),(19.928737,9.062575),(19.914474,9.069267),(19.902175,9.0518),(19.889463,9.046374),(19.80554,9.051128),(19.784663,9.048751),(19.758411,9.041982),(19.746836,9.04131),(19.738154,9.039372),(19.713969,9.02914),(19.709008,9.024567),(19.700017,9.020587),(19.656402,9.021259),(19.640072,9.013999),(19.613614,9.031672),(19.583125,9.025342),(19.553772,9.013327),(19.530828,9.013999),(19.52132,9.008831),(19.515842,9.009451),(19.511811,9.012293),(19.506644,9.013999),(19.430162,9.012862),(19.420964,9.017435),(19.38262,9.003198),(19.369391,9.000382),(19.352234,9.002061),(19.333321,9.006557),(19.316268,9.01312),(19.304795,9.020846),(19.295494,9.009503),(19.285778,9.01281),(19.275236,9.021983),(19.263248,9.028261),(19.253429,9.027951),(19.232862,9.022293),(19.192037,9.020174),(19.179635,9.015265),(19.174571,9.003767),(19.168783,9.002216),(19.142118,9.008676),(19.133643,9.007177),(19.126718,9.007177),(19.10057,9.015265),(19.060676,9.00418),(19.021919,8.985215),(18.999491,8.969635),(18.984815,8.956431),(18.97851,8.949455),(18.975927,8.941988),(18.971896,8.938215),(18.952052,8.931988),(18.936446,8.923358),(18.928591,8.921679),(18.922907,8.918216),(18.920736,8.907855),(18.917739,8.898812),(18.910608,8.894239),(18.901409,8.89398),(18.892727,8.897933),(18.86968,8.864111),(18.869783,8.849409),(18.886526,8.835844),(18.902959,8.84481),(18.909677,8.835327),(18.912468,8.818403),(18.917325,8.805148),(18.929108,8.796544),(19.048687,8.745669),(19.057782,8.729701),(19.073492,8.720528),(19.103877,8.698049),(19.124135,8.675079),(19.091475,8.652858),(19.081657,8.637226),(19.072261,8.631872),(19.061296,8.625624),(19.020472,8.545578),(18.920529,8.432122),(18.910401,8.412976),(18.901306,8.388636),(18.887457,8.370963),(18.8549,8.339802),(18.813042,8.276421),(18.791235,8.257378),(18.773768,8.248619),(18.715064,8.228672),(18.672586,8.20751),(18.638583,8.177642),(18.618636,8.138652),(18.617912,8.090127),(18.589283,8.047882),(18.508255,8.030674),(18.070452,8.019202),(17.977228,7.997187),(17.897853,7.967473),(17.858785,7.960445),(17.817238,7.962099),(17.679778,7.985198),(17.661795,7.986284),(17.639884,7.985043),(17.62066,7.978739),(17.59999,7.950369),(17.580766,7.940602),(17.559786,7.934711),(17.523819,7.93099),(17.503665,7.926236),(17.487232,7.914453),(17.478344,7.891819),(17.466355,7.884119),(17.419122,7.898227),(17.405997,7.883034),(17.385739,7.870477),(17.250864,7.822831),(17.234637,7.811617),(17.23379,7.810801),(17.220685,7.798181),(17.214484,7.786813),(17.211073,7.768312),(17.204562,7.763817),(17.196397,7.76516),(17.188645,7.764127),(17.170559,7.747539),(17.135522,7.705112),(17.116298,7.68687),(17.101932,7.677724),(17.095525,7.67824),(17.090254,7.683822),(17.079091,7.689919),(17.059351,7.696534),(17.045605,7.6847),(17.041058,7.675502),(17.042091,7.667027),(17.038991,7.662479),(17.008295,7.667647),(16.997546,7.666717),(16.988968,7.660774),(16.98194,7.648733),(16.962716,7.650697),(16.919618,7.644082),(16.880654,7.632869),(16.865461,7.624445),(16.854816,7.611475),(16.847168,7.591579),(16.848098,7.584706),(16.852439,7.580262),(16.855849,7.575404),(16.853886,7.567498),(16.850372,7.565896),(16.839623,7.567446),(16.836729,7.567498),(16.815071,7.549544),(16.812854,7.547706),(16.805516,7.543675),(16.782365,7.541143),(16.768619,7.550238),(16.746709,7.586566),(16.709812,7.627753),(16.686867,7.645788),(16.6632,7.657415),(16.624546,7.66868),(16.61421,7.679584),(16.609869,7.701598),(16.611833,7.713897),(16.616174,7.723871),(16.618138,7.734309),(16.613073,7.74821),(16.605219,7.757202),(16.596434,7.761129),(16.586822,7.763558),(16.576383,7.767796),(16.560157,7.779733),(16.549305,7.794719),(16.54486,7.813013),(16.550235,7.843502),(16.550752,7.8527),(16.550131,7.861692),(16.548685,7.870012),(16.509204,7.858281),(16.491737,7.849238),(16.475717,7.835802),(16.469413,7.825828),(16.457217,7.799628),(16.450913,7.792084),(16.437373,7.788363),(16.427245,7.791464),(16.41815,7.795856),(16.407401,7.796063),(16.392208,7.783609),(16.387041,7.762628),(16.387144,7.694674),(16.38301,7.680669),(16.370814,7.672504),(16.282758,7.660102),(16.26188,7.651885),(16.227257,7.625582),(16.207207,7.613542),(16.185606,7.610751),(16.162662,7.611113),(16.128762,7.599486),(16.0652,7.591217),(16.042979,7.583931),(16.026339,7.574888),(16.012283,7.560315),(15.990372,7.529412),(15.975696,7.515201),(15.94314,7.495306),(15.92526,7.488175),(15.793899,7.457996),(15.758345,7.455567),(15.720001,7.468641),(15.668532,7.516287),(15.624813,7.519749),(15.55195,7.509879),(15.515569,7.512204),(15.481049,7.523263),(15.521977,7.576076),(15.548952,7.630801),(15.562181,7.690178),(15.562905,7.792445),(15.540787,7.796838),(15.509368,7.804021),(15.487871,7.804951),(15.440742,7.839419),(15.406532,7.921585),(15.345347,8.13599),(15.210706,8.421822),(15.183703,8.479148),(15.16851,8.498604),(15.110736,8.555138),(15.068568,8.623867),(15.051928,8.643789),(15.031258,8.658749),(15.005833,8.66663),(14.968523,8.672159),(14.955087,8.676216),(14.951469,8.68283),(14.951469,8.692106),(14.949092,8.704276),(14.940101,8.729649),(14.934416,8.739003),(14.927285,8.745075),(14.919947,8.74771),(14.913332,8.752438),(14.908268,8.764841),(14.89938,8.774323),(14.859899,8.803314),(14.846049,8.810988),(14.836748,8.813003),(14.827963,8.813158),(14.819178,8.811608),(14.809876,8.808611),(14.793856,8.813623),(14.571699,8.99099),(14.349542,9.168356),(14.331145,9.200215),(14.321327,9.243158),(14.036486,9.568771),(13.947603,9.637759),(13.945949,9.652642),(14.006721,9.739277),(14.088473,9.809635),(14.119789,9.85201),(14.168365,9.94774),(14.170638,9.957843),(14.171155,9.96761),(14.173532,9.975025),(14.181697,9.978178),(14.196341,9.979147),(14.440493,9.995308),(14.732465,9.923814),(14.772566,9.921747),(14.898139,9.960478),(14.944131,9.958825),(15.002422,9.945053),(15.033015,9.942857),(15.06123,9.949239),(15.071669,9.955957),(15.089445,9.972183),(15.100711,9.978901),(15.109599,9.981537),(15.155488,9.986523),(15.214916,9.984095),(15.382968,9.930196),(15.439398,9.931695),(15.663777,9.98611),(15.681244,9.991278),(15.637939,10.029957),(15.602799,10.04099),(15.536137,10.080523),(15.490144,10.120107),(15.490661,10.124448),(15.476399,10.132742),(15.439192,10.185245),(15.301422,10.311749),(15.29057,10.328983),(15.284266,10.349292),(15.282199,10.374588),(15.278374,10.394276),(15.268763,10.406343),(15.255637,10.41704),(15.241167,10.432904),(15.227215,10.470137),(15.218326,10.487216),(15.198173,10.496492),(15.193315,10.501195),(15.186597,10.505871),(15.176055,10.50799),(15.164686,10.509075),(15.153628,10.511969),(15.144429,10.51631),(15.138228,10.521659),(15.131717,10.540701),(15.132337,10.565403),(15.138435,10.589691),(15.14846,10.607338),(15.149907,10.623125),(15.142259,10.647207),(15.065881,10.793115),(15.06309,10.830761),(15.075596,10.873498),(15.079007,10.898147),(15.072702,10.915769),(15.06247,10.930703),(15.035185,10.994627),(15.02883,11.080218),(15.021233,11.182549),(15.028261,11.244483),(15.033325,11.26027),(15.056373,11.29342),(15.06309,11.309931),(15.062264,11.320034),(15.057819,11.325356),(15.052548,11.329801),(15.049448,11.337268),(15.049448,11.348714),(15.054305,11.364088),(15.06061,11.416126),(15.067018,11.435556),(15.076113,11.453307),(15.122312,11.503227),(15.135644,11.530822),(15.123862,11.563171),(15.09575,11.598156),(15.085208,11.615881),(15.069395,11.660788),(15.066501,11.680632),(15.068258,11.700114),(15.076113,11.721456),(15.079834,11.722593),(15.086345,11.72218),(15.093063,11.722903),(15.097197,11.727657),(15.095957,11.734117),(15.085311,11.744039),(15.083554,11.748173),(15.089135,11.75763),(15.105362,11.772719),(15.110943,11.782899),(15.086345,11.840467),(15.079834,11.851216),(15.06278,11.853489),(15.050998,11.861603),(15.044177,11.877312),(15.042006,11.902375),(15.046967,11.910179),(15.056579,11.918447),(15.063401,11.927283),(15.05937,11.936792),(15.055546,11.942476),(15.050585,11.952863),(15.048311,11.962785),(15.052548,11.967229),(15.081177,11.971673),(15.076836,11.982836),(15.05968,11.997305),(15.049448,12.011929),(15.051205,12.024228),(15.053685,12.031877),(15.053065,12.038026),(15.045727,12.046088),(15.04056,12.055648),(15.04087,12.067585),(15.04459,12.078385),(15.049448,12.084586),(15.030431,12.100916),(15.011827,12.108616),(14.993844,12.106704),(14.976791,12.094147),(14.964802,12.092441),(14.950746,12.103345),(14.928732,12.128615),(14.9031,12.145875),(14.898036,12.152799),(14.894832,12.167372),(14.899896,12.171765),(14.907441,12.17347),(14.911575,12.180136),(14.910128,12.190368),(14.905891,12.195639),(14.90093,12.200032),(14.898036,12.207473),(14.898036,12.219462),(14.9031,12.236257),(14.904857,12.248401),(14.906304,12.317957),(14.908268,12.326897),(14.891215,12.402655),(14.877469,12.427201),(14.876849,12.431594),(14.878502,12.443066),(14.877469,12.447045),(14.872404,12.450404),(14.865893,12.452574),(14.861862,12.452006),(14.863826,12.447045),(14.855455,12.454538),(14.85101,12.455158),(14.849563,12.457019),(14.850184,12.468129),(14.852044,12.474692),(14.860105,12.490091),(14.863826,12.495466),(14.830753,12.618249),(14.819178,12.638816),(14.786208,12.63122),(14.768845,12.633235),(14.761403,12.649358),(14.758716,12.658815),(14.752102,12.668944),(14.743317,12.67716),(14.734118,12.680416),(14.728847,12.677212),(14.713654,12.65251),(14.702079,12.668995),(14.716755,12.70181),(14.709934,12.718243),(14.694017,12.723721),(14.664458,12.715969),(14.648129,12.725116),(14.641721,12.730904),(14.624254,12.741885),(14.62074,12.747853),(14.620327,12.754055),(14.618363,12.759196),(14.610612,12.762349),(14.603997,12.760798),(14.584877,12.746355),(14.575265,12.744908),(14.570717,12.750386),(14.569374,12.75961),(14.569374,12.769403),(14.560692,12.766224),(14.554801,12.766896),(14.55108,12.771573),(14.549013,12.780461),(14.549013,12.818211),(14.53103,12.836246),(14.500437,12.859113),(14.490102,12.873608),(14.490619,12.886346),(14.506122,12.93208),(14.507672,12.95244),(14.504675,12.969003),(14.496303,12.983808),(14.482144,12.99882),(14.481041,13.000508),(14.435429,13.070289),(14.418169,13.081141),(14.064908,13.077988),(13.836111,13.391044),(13.607314,13.7041),(13.600182,13.735984),(13.592948,13.767843),(13.585816,13.799727),(13.578582,13.831534),(13.57145,13.863418),(13.564319,13.895225),(13.557188,13.927161),(13.549953,13.958994),(13.542821,13.990853),(13.53569,14.022737),(13.528455,14.054518),(13.521324,14.086428),(13.514193,14.118235),(13.506958,14.150119),(13.499723,14.181978),(13.492695,14.213862),(13.482257,14.259777),(13.468304,14.310678),(13.449184,14.380131),(13.449494,14.439042),(13.482257,14.483587),(13.507785,14.495964),(13.543855,14.510562),(13.584369,14.513818),(13.608052,14.518266),(13.624264,14.521311),(13.657853,14.548725),(13.665605,14.56689),(13.666742,14.585519),(13.660334,14.623734),(13.657026,14.629651),(13.646278,14.639056),(13.644727,14.644249),(13.648345,14.649417),(13.668085,14.662879),(13.700021,14.697734),(13.709995,14.705124),(13.730045,14.709646),(13.749165,14.711248),(13.764358,14.719077),(13.773247,14.742512),(13.772006,14.762873),(13.75795,14.805532),(13.75392,14.825944),(13.75516,14.847209),(13.760534,14.866355),(13.775417,14.897438),(13.808077,14.965573),(13.833915,15.019601),(13.862957,15.059056),(13.892826,15.0997),(13.922592,15.140498),(13.952564,15.181219),(13.98233,15.221992),(14.012199,15.262687),(14.042068,15.303434),(14.071833,15.344181),(14.101702,15.384876),(14.131571,15.425545),(14.16144,15.466292),(14.191206,15.507039),(14.221178,15.547734),(14.250944,15.588455),(14.280709,15.629254),(14.310682,15.669949),(14.340447,15.710722),(14.368973,15.749634),(14.423543,15.806323),(14.482144,15.86725),(14.520798,15.907557),(14.576919,15.965797),(14.633143,16.024243),(14.689263,16.082534),(14.745384,16.140876),(14.801401,16.199219),(14.857418,16.257613),(14.913539,16.315904),(14.969659,16.374247),(15.02578,16.432538),(15.081797,16.490881),(15.137918,16.549275),(15.194039,16.607566),(15.250159,16.665909),(15.30628,16.7242),(15.3624,16.782646),(15.418418,16.840885),(15.452421,16.876232),(15.465547,16.89468),(15.468517,16.90491),(15.471954,16.916746),(15.474125,16.942068),(15.478259,16.987698),(15.48229,17.033277),(15.486217,17.078907),(15.490248,17.124537),(15.494279,17.170116),(15.498309,17.215746),(15.50234,17.261376),(15.506371,17.307058),(15.510505,17.352637),(15.514536,17.398267),(15.518567,17.443846),(15.522597,17.489528),(15.526628,17.535107),(15.530659,17.580737),(15.53469,17.626367),(15.538617,17.672049),(15.542648,17.717679),(15.546782,17.76331),(15.550813,17.808888),(15.554843,17.85457),(15.558874,17.900149),(15.562905,17.945779),(15.566936,17.991409),(15.567698,18.000039),(15.570966,18.03704),(15.574997,18.082644),(15.579028,18.128249),(15.583162,18.173879),(15.587193,18.219483),(15.59112,18.265088),(15.595151,18.310718),(15.599182,18.356348),(15.603213,18.402004),(15.607243,18.447583),(15.611274,18.493239),(15.615305,18.538818),(15.619439,18.584474),(15.62347,18.630078),(15.6275,18.675709),(15.631531,18.721313),(15.635562,18.766943),(15.639593,18.8126),(15.64352,18.858178),(15.647551,18.903834),(15.651582,18.949387),(15.655716,18.995069),(15.659747,19.040648),(15.663777,19.086304),(15.667808,19.131882),(15.671839,19.177539),(15.67587,19.223169),(15.6799,19.268747),(15.683931,19.314404),(15.687962,19.360008),(15.692096,19.405638),(15.696023,19.451243),(15.700054,19.496873),(15.704085,19.542529),(15.708116,19.588108),(15.712146,19.633764),(15.716177,19.679317),(15.720208,19.724999),(15.724342,19.770577),(15.728373,19.816259),(15.732404,19.861838),(15.736021,19.903541),(15.767234,19.982037),(15.782323,20.00803),(15.822321,20.076993),(15.862318,20.145955),(15.902419,20.214943),(15.94252,20.283957),(15.962984,20.319226),(15.970322,20.336331),(15.968151,20.352816),(15.953992,20.374571),(15.930324,20.399324),(15.872033,20.460199),(15.813846,20.521074),(15.755555,20.582001),(15.697264,20.642875),(15.669462,20.671866),(15.588123,20.732792),(15.570243,20.751913),(15.556084,20.773669),(15.544198,20.79899),(15.536033,20.844052),(15.544301,20.890302),(15.569003,20.928905),(15.60931,20.95066),(15.592774,20.974793),(15.576134,20.999339),(15.559288,21.023782),(15.542544,21.048277),(15.525801,21.072772),(15.509058,21.097266),(15.492212,21.121761),(15.475572,21.146256),(15.458829,21.17075),(15.441982,21.195193),(15.425342,21.219791),(15.408496,21.244234),(15.391753,21.268729),(15.375009,21.293223),(15.358266,21.317718),(15.34142,21.342161),(15.32478,21.366707),(15.301009,21.40133),(15.266592,21.440656),(15.247472,21.453265),(15.20024,21.476468),(15.18453,21.491196),(15.180396,21.507267),(15.180086,21.533467),(15.179776,21.553208),(15.178846,21.605297),(15.177502,21.678988),(15.176055,21.763427),(15.174608,21.847918),(15.173265,21.921557),(15.172438,21.973647),(15.172024,21.993387),(15.16448,22.033695),(15.156315,22.076173),(15.14815,22.118651),(15.136058,22.181335),(15.124069,22.243966),(15.11208,22.306598),(15.099987,22.369282),(15.087998,22.431914),(15.075906,22.494597),(15.064021,22.557177),(15.051928,22.619861),(15.039939,22.682544),(15.02795,22.745176),(15.015858,22.807782),(15.003973,22.87044),(14.991984,22.933097),(14.979909,22.995664),(14.997461,23.003817),(15.055236,23.029965),(15.112907,23.056113),(15.170371,23.082261),(15.228145,23.108358),(15.285816,23.134455),(15.343487,23.160577),(15.401158,23.186751),(15.458829,23.212899),(15.516499,23.239048),(15.574274,23.265196),(15.631945,23.291344),(15.689616,23.317518),(15.747183,23.343641),(15.804854,23.369789),(15.862525,23.395937),(15.920196,23.422034),(15.964637,23.442214),(15.985101,23.44472),(16.076155,23.399632),(16.199765,23.338706),(16.323169,23.277728),(16.446779,23.216724),(16.570285,23.15572),(16.693792,23.094664),(16.817299,23.03366),(16.940909,22.972733),(17.064312,22.911755),(17.187922,22.850751),(17.311429,22.789799),(17.434832,22.728795),(17.558442,22.667765),(17.681949,22.606787),(17.805455,22.545757),(17.928962,22.484779),(18.052469,22.423852),(18.175975,22.362822),(18.299585,22.301844),(18.422989,22.240866),(18.546599,22.179836),(18.670105,22.118806),(18.793612,22.05788),(18.917119,21.99685),(19.040729,21.935871),(19.164132,21.874893),(19.185837,21.864177),(19.287639,21.813915),(19.411145,21.752885),(19.534652,21.691855),(19.658262,21.630877),(19.781665,21.569899),(19.905276,21.508869),(20.028782,21.447943),(20.152289,21.386913),(20.275796,21.325935),(20.399406,21.265008),(20.522809,21.203926),(20.646419,21.142897),(20.769926,21.08197),(20.893329,21.02094),(21.016939,20.959962),(21.140342,20.899036),(21.263952,20.838006),(21.387459,20.777027),(21.510966,20.716049),(21.634472,20.654968),(21.758082,20.59399),(21.881486,20.533063),(22.005096,20.472033),(22.128602,20.411055),(22.252109,20.350077),(22.375616,20.289124),(22.499122,20.228095),(22.622629,20.167091),(22.746136,20.106061),(22.869642,20.045108),(22.993149,19.984104),(23.116759,19.923152),(23.240162,19.862096),(23.363772,19.80117),(23.487279,19.740166),(23.610786,19.67911),(23.734292,19.618132),(23.857799,19.557179),(23.981306,19.496124)] +Togo [(0.487649,10.933236),(0.612603,10.976566),(0.675338,10.988529),(0.77156,10.990364),(0.901474,10.992741),(0.875843,10.931169),(0.872949,10.911299),(0.875326,10.900524),(0.883698,10.880216),(0.885144,10.869389),(0.869538,10.839908),(0.867575,10.829211),(0.866024,10.80637),(0.862924,10.796086),(0.850005,10.77769),(0.795641,10.726478),(0.781275,10.693044),(0.789233,10.602946),(0.78851,10.563852),(0.773833,10.508145),(0.759881,10.405154),(0.760708,10.382158),(0.768769,10.367094),(0.844423,10.317175),(0.939508,10.254517),(0.997106,10.216553),(1.082238,10.16044),(1.225485,10.066053),(1.331009,9.996471),(1.343825,9.962442),(1.343942,9.954756),(1.345065,9.881103),(1.346925,9.771834),(1.348165,9.694164),(1.35602,9.647474),(1.354573,9.635304),(1.352196,9.625357),(1.352506,9.60528),(1.351266,9.595074),(1.345065,9.582388),(1.327288,9.555309),(1.323154,9.542313),(1.326358,9.521952),(1.336486,9.497509),(1.351266,9.482549),(1.368422,9.490843),(1.379895,9.462473),(1.386716,9.361135),(1.401185,9.321267),(1.420146,9.292317),(1.425267,9.284499),(1.505158,9.201455),(1.567273,9.13673),(1.587841,9.100893),(1.595799,9.076811),(1.601173,9.049526),(1.602827,8.924392),(1.604274,8.815225),(1.605514,8.722492),(1.607204,8.588617),(1.607581,8.558755),(1.609028,8.546973),(1.614092,8.535785),(1.63993,8.503255),(1.644168,8.493488),(1.640447,8.475815),(1.607581,8.421864),(1.606031,8.413389),(1.606031,8.396026),(1.60107,8.371635),(1.603945,8.367476),(1.603964,8.367449),(1.609855,8.365485),(1.614919,8.356261),(1.625564,8.270478),(1.623807,8.155886),(1.621327,7.996567),(1.621947,7.838282),(1.622774,7.640052),(1.622855,7.62246),(1.623251,7.536385),(1.623704,7.4381),(1.624324,7.288807),(1.624944,7.143648),(1.625668,6.996783),(1.607478,6.991409),(1.531927,6.991926),(1.551357,6.920871),(1.562623,6.904283),(1.564223,6.903089),(1.579056,6.892035),(1.588047,6.881235),(1.590114,6.867334),(1.58257,6.825347),(1.58319,6.808268),(1.587531,6.791034),(1.595179,6.77057),(1.60107,6.746437),(1.597039,6.73096),(1.589182,6.72046),(1.574301,6.700574),(1.566033,6.678146),(1.572441,6.666907),(1.58536,6.656442),(1.596316,6.636418),(1.596522,6.62864),(1.594765,6.620269),(1.594145,6.611045),(1.597349,6.600813),(1.603344,6.592596),(1.611198,6.58469),(1.61957,6.577998),(1.627425,6.573347),(1.63683,6.572261),(1.646648,6.573192),(1.653883,6.570659),(1.655227,6.559213),(1.657811,6.552263),(1.67445,6.538594),(1.680755,6.530068),(1.681995,6.520017),(1.679515,6.498571),(1.681685,6.489838),(1.6883,6.484437),(1.709797,6.473404),(1.718375,6.466997),(1.74411,6.425733),(1.76354,6.350647),(1.782351,6.277267),(1.761577,6.276026),(1.633109,6.245692),(1.612439,6.234763),(1.619663,6.213899),(1.61964,6.213894),(1.619314,6.213813),(1.429535,6.181952),(1.271007,6.133734),(1.199555,6.102525),(1.19158,6.101874),(1.185396,6.100491),(1.187968,6.135647),(1.176186,6.151512),(1.093401,6.160917),(1.079655,6.167919),(1.068906,6.180347),(1.05578,6.20063),(1.048132,6.20939),(1.040587,6.213472),(1.033146,6.216547),(1.025188,6.222334),(1.009685,6.244452),(0.983536,6.324602),(0.907262,6.324912),(0.886798,6.33039),(0.873879,6.340131),(0.841323,6.379224),(0.816208,6.395063),(0.766805,6.419351),(0.745721,6.4409),(0.729908,6.465085),(0.71792,6.48883),(0.711718,6.513376),(0.713076,6.533936),(0.713475,6.53999),(0.71978,6.552728),(0.726808,6.562107),(0.727428,6.571435),(0.714612,6.583914),(0.705517,6.585775),(0.677612,6.584896),(0.668207,6.585827),(0.659422,6.601717),(0.645366,6.612026),(0.63224,6.62399),(0.626659,6.644893),(0.635341,6.695381),(0.634307,6.711607),(0.625832,6.727756),(0.611259,6.741528),(0.593793,6.751114),(0.576636,6.754731),(0.570022,6.763516),(0.541083,6.817182),(0.528887,6.826535),(0.519275,6.832246),(0.516071,6.841806),(0.522789,6.862838),(0.543873,6.897926),(0.54625,6.914773),(0.534468,6.935909),(0.506046,6.957819),(0.494574,6.973116),(0.509147,6.977611),(0.523823,6.978697),(0.565991,6.989652),(0.57891,6.996783),(0.59803,7.0312),(0.598754,7.074711),(0.59431,7.119722),(0.59772,7.158892),(0.621801,7.218837),(0.631103,7.265242),(0.644436,7.301933),(0.645676,7.323585),(0.629656,7.389162),(0.623042,7.397896),(0.586248,7.384357),(0.566404,7.380791),(0.548214,7.383271),(0.516692,7.411022),(0.500568,7.45505),(0.495608,7.504298),(0.499432,7.562382),(0.503049,7.574681),(0.50925,7.585378),(0.519379,7.595196),(0.532505,7.601036),(0.548524,7.605377),(0.562063,7.612301),(0.567748,7.625737),(0.567438,7.652919),(0.570538,7.683925),(0.583354,7.704285),(0.612293,7.699428),(0.609502,7.718961),(0.611673,7.771258),(0.602578,7.828412),(0.603405,7.888305),(0.602688,7.893132),(0.592863,7.959309),(0.593689,7.996567),(0.593173,8.015739),(0.583974,8.05305),(0.580254,8.100075),(0.574983,8.124621),(0.579943,8.131701),(0.587798,8.137721),(0.593276,8.150666),(0.591519,8.16449),(0.584594,8.173068),(0.576843,8.180484),(0.572709,8.191),(0.573846,8.203945),(0.579013,8.207821),(0.586661,8.208751),(0.595033,8.21273),(0.628313,8.24247),(0.645056,8.253425),(0.689187,8.272855),(0.705207,8.282751),(0.712752,8.29797),(0.708825,8.322568),(0.705104,8.325979),(0.693115,8.327891),(0.689187,8.331301),(0.688567,8.33603),(0.689394,8.346055),(0.686914,8.360731),(0.687224,8.372203),(0.685053,8.382564),(0.646709,8.414165),(0.63379,8.452018),(0.616324,8.488759),(0.550798,8.519326),(0.524753,8.538033),(0.500568,8.560176),(0.483515,8.579787),(0.443518,8.606866),(0.403727,8.662082),(0.374426,8.724766),(0.368084,8.761481),(0.365874,8.774272),(0.372669,8.783754),(0.385278,8.78608),(0.397112,8.781532),(0.407809,8.759053),(0.4183,8.770603),(0.424966,8.78732),(0.419592,8.791247),(0.430134,8.794736),(0.439409,8.793444),(0.449512,8.790369),(0.462483,8.788302),(0.47349,8.793547),(0.477004,8.80675),(0.478451,8.822331),(0.483515,8.834811),(0.483629,8.834973),(0.505633,8.866333),(0.493127,8.915219),(0.444887,8.996455),(0.444887,8.996558),(0.44468,8.996558),(0.43127,9.02759),(0.441554,9.058182),(0.458478,9.08924),(0.463749,9.12146),(0.463646,9.137092),(0.469976,9.14673),(0.49261,9.166754),(0.504703,9.187166),(0.504393,9.203109),(0.49199,9.238895),(0.49292,9.258609),(0.501085,9.268195),(0.512351,9.274706),(0.522479,9.285067),(0.526613,9.297651),(0.528267,9.326951),(0.537569,9.377646),(0.531368,9.401107),(0.513281,9.416791),(0.483515,9.426842),(0.482792,9.427875),(0.482585,9.428857),(0.482792,9.429839),(0.489406,9.441027),(0.49168,9.451285),(0.489923,9.461129),(0.483515,9.470017),(0.430444,9.489189),(0.411659,9.492238),(0.371429,9.490016),(0.34895,9.486037),(0.333783,9.479474),(0.328796,9.471568),(0.325747,9.451336),(0.32182,9.442836),(0.314223,9.436479),(0.26849,9.420641),(0.249886,9.416946),(0.23136,9.418961),(0.226813,9.432345),(0.234977,9.443766),(0.227949,9.459734),(0.228053,9.473971),(0.26973,9.482342),(0.28531,9.488363),(0.2981,9.497096),(0.301924,9.507457),(0.293139,9.518283),(0.279006,9.519007),(0.253503,9.514252),(0.243116,9.518697),(0.23322,9.525595),(0.225908,9.531926),(0.223402,9.534794),(0.223066,9.541176),(0.227743,9.569339),(0.230197,9.575747),(0.28053,9.570166),(0.30456,9.571562),(0.32244,9.583137),(0.33616,9.574197),(0.354608,9.571096),(0.370809,9.575644),(0.377734,9.589416),(0.369155,9.610138),(0.349363,9.61391),(0.308797,9.603678),(0.277636,9.606986),(0.261694,9.627941),(0.260971,9.657164),(0.275285,9.685586),(0.301924,9.657835),(0.318719,9.648611),(0.336651,9.650833),(0.347089,9.663158),(0.34988,9.680651),(0.347606,9.698841),(0.342955,9.713543),(0.336289,9.72349),(0.329132,9.731293),(0.323887,9.742068),(0.32244,9.760671),(0.343886,9.838393),(0.344196,9.849762),(0.339674,9.889837),(0.341715,9.903789),(0.349777,9.918956),(0.362592,9.932676),(0.369569,9.938826),(0.370189,9.947017),(0.363393,9.966783),(0.35143,9.990451),(0.349777,9.997169),(0.35174,10.006419),(0.355125,10.01262),(0.355719,10.018976),(0.349777,10.02882),(0.367295,10.032128),(0.382746,10.029596),(0.393908,10.03249),(0.39457,10.035557),(0.398146,10.052127),(0.395149,10.072358),(0.387087,10.078947),(0.37608,10.081582),(0.363393,10.089669),(0.353601,10.115508),(0.358846,10.18633),(0.357192,10.219997),(0.364427,10.233407),(0.367295,10.249815),(0.372049,10.265137),(0.396596,10.283224),(0.376183,10.299941),(0.366597,10.304488),(0.32275,10.297254),(0.308797,10.297099),(0.317582,10.31733),(0.307066,10.333272),(0.290039,10.348491),(0.278825,10.366629),(0.28022,10.375182),(0.283553,10.385775),(0.283114,10.396808),(0.273244,10.406601),(0.263012,10.408926),(0.253943,10.406084),(0.244563,10.401537),(0.23322,10.398617),(0.223609,10.399961),(0.197073,10.41027),(0.194437,10.407376),(0.19175,10.400839),(0.187513,10.394948),(0.179865,10.394173),(0.176971,10.397687),(0.126018,10.491609),(0.110179,10.508249),(0.057262,10.541115),(0.04902,10.550959),(0.037341,10.57318),(0.029434,10.582533),(0.020339,10.588037),(-0.010925,10.598269),(-0.019968,10.604341),(-0.026221,10.610361),(-0.033352,10.615064),(-0.056452,10.618009),(-0.068363,10.621058),(-0.079344,10.626252),(-0.088129,10.633486),(-0.098336,10.654157),(-0.09769,10.675008),(-0.091564,10.700254),(-0.088129,10.714412),(-0.082807,10.756218),(-0.07521,10.773659),(-0.061103,10.791358),(-0.04069,10.805026),(-0.032061,10.813165),(-0.0302,10.823785),(-0.035316,10.843448),(-0.036091,10.853137),(-0.014439,10.953596),(-0.009581,10.963001),(-0.001571,10.971295),(0.007188,10.976799),(0.01419,10.983672),(0.016567,10.996203),(0.019409,11.031628),(0.016205,11.062582),(0.001116,11.085991),(-0.032267,11.098574),(-0.051077,11.098264),(-0.085029,11.089402),(-0.103348,11.087541),(-0.121745,11.092192),(-0.14208,11.103329),(-0.158668,11.118444),(-0.166109,11.13498),(-0.115062,11.124658),(-0.063118,11.114155),(0.059588,11.089402),(0.287558,11.043436),(0.374779,11.025808),(0.486616,11.003205),(0.488786,11.001784),(0.49013,10.999511),(0.490647,10.996358),(0.489303,10.991914),(0.48858,10.987754),(0.487133,10.983672),(0.483619,10.979693),(0.499742,10.975688),(0.487649,10.933236)] +Uganda [(34.108026,3.868938),(34.123736,3.872039),(34.163423,3.886198),(34.182854,3.886095),(34.204971,3.874545),(34.207555,3.860877),(34.196703,3.847389),(34.148954,3.822688),(34.150814,3.81721),(34.166524,3.811319),(34.17903,3.7961),(34.159082,3.783362),(34.152468,3.775636),(34.16518,3.77083),(34.173655,3.771399),(34.190915,3.775843),(34.210552,3.777135),(34.230396,3.782742),(34.240938,3.78362),(34.241558,3.778737),(34.263779,3.750056),(34.278868,3.709723),(34.290444,3.70316),(34.295095,3.707346),(34.299436,3.716932),(34.309874,3.726699),(34.337056,3.734579),(34.35504,3.727371),(34.387492,3.692747),(34.406303,3.682929),(34.425009,3.677193),(34.439686,3.667736),(34.446403,3.646704),(34.443923,3.566295),(34.434518,3.52622),(34.415501,3.497023),(34.406199,3.492294),(34.395967,3.489607),(34.386872,3.485628),(34.381291,3.476869),(34.382118,3.466043),(34.394417,3.444623),(34.398551,3.433642),(34.397001,3.424159),(34.386976,3.399664),(34.383978,3.388347),(34.386562,3.376384),(34.394417,3.365661),(34.403822,3.355404),(34.41116,3.344707),(34.424079,3.304812),(34.434001,3.182029),(34.444853,3.159136),(34.46573,3.145856),(34.512859,3.132368),(34.53384,3.118467),(34.545556,3.097501),(34.545622,3.097383),(34.574664,2.946126),(34.58469,2.928711),(34.616419,2.893416),(34.631405,2.869542),(34.6405,2.860137),(34.654143,2.856519),(34.660344,2.858638),(34.671506,2.867785),(34.67564,2.8698),(34.678121,2.871557),(34.682565,2.87936),(34.685045,2.880962),(34.688869,2.879154),(34.69104,2.875691),(34.692693,2.872229),(34.694657,2.87042),(34.696517,2.867733),(34.724836,2.854142),(34.730107,2.85285),(34.736928,2.844375),(34.740959,2.835539),(34.76132,2.772493),(34.776512,2.685625),(34.81868,2.597982),(34.828086,2.588784),(34.841832,2.58775),(34.849893,2.59514),(34.856508,2.60346),(34.865603,2.604855),(34.875628,2.591212),(34.881312,2.541448),(34.8871,2.522379),(34.914385,2.494371),(34.923584,2.477318),(34.920896,2.454632),(34.906117,2.436907),(34.885963,2.425434),(34.867876,2.411482),(34.859091,2.38678),(34.865603,2.347506),(34.904284,2.254255),(34.922343,2.210719),(34.967509,2.101914),(34.967612,2.082561),(34.958207,2.037965),(34.956657,2.018508),(34.957897,1.997838),(34.962031,1.977813),(34.969162,1.960295),(34.977947,1.949908),(35.001408,1.927997),(35.006473,1.916861),(35.002752,1.906138),(34.984045,1.88247),(34.979704,1.8703),(34.978671,1.675945),(34.972676,1.654241),(34.940947,1.587036),(34.933712,1.575693),(34.92348,1.566107),(34.913558,1.56156),(34.892578,1.55727),(34.882553,1.552232),(34.859505,1.517919),(34.838421,1.437174),(34.778993,1.388547),(34.780747,1.371789),(34.782714,1.352993),(34.800077,1.312324),(34.810516,1.272533),(34.797907,1.231916),(34.766074,1.217498),(34.683805,1.209075),(34.663031,1.196362),(34.628925,1.163599),(34.580452,1.152541),(34.573631,1.134247),(34.571977,1.111716),(34.566883,1.103022),(34.561228,1.093371),(34.560065,1.093149),(34.540351,1.089392),(34.524021,1.098642),(34.507485,1.102518),(34.487021,1.082467),(34.477203,1.064587),(34.468831,1.044227),(34.463663,1.022936),(34.463043,0.978081),(34.457152,0.957617),(34.431211,0.893538),(34.402375,0.856073),(34.388009,0.815817),(34.370852,0.800211),(34.30667,0.768068),(34.298402,0.759283),(34.296252,0.746746),(34.292304,0.72373),(34.276181,0.680838),(34.2521,0.655),(34.219751,0.638567),(34.179133,0.623891),(34.149574,0.603634),(34.132417,0.572834),(34.131588,0.569433),(34.107509,0.470722),(34.104409,0.462247),(34.097381,0.451808),(34.080121,0.431344),(34.075677,0.422249),(34.07671,0.403852),(34.087976,0.367369),(34.085185,0.347008),(34.076607,0.333779),(34.04064,0.305874),(33.960266,0.198677),(33.951757,0.187328),(33.893569,0.109814),(33.890468,0.090073),(33.921578,-0.01297),(33.95248,-0.115702),(33.953514,-0.154356),(33.935117,-0.313106),(33.911346,-0.519295),(33.894809,-0.662749),(33.89853,-0.799072),(33.904214,-1.002573),(33.822255,-1.002573),(33.639269,-1.002573),(33.631777,-1.002573),(33.456128,-1.002573),(33.426415,-1.002573),(33.27309,-1.002573),(33.089949,-1.002573),(32.906963,-1.002573),(32.847225,-1.002573),(32.723873,-1.002573),(32.722268,-1.002573),(32.540835,-1.002573),(32.357642,-1.002573),(32.174552,-1.002573),(31.991411,-1.002573),(31.986193,-1.002573),(31.808373,-1.002573),(31.625335,-1.002573),(31.442246,-1.002573),(31.280493,-1.002573),(31.259156,-1.002573),(31.076118,-1.002573),(30.892977,-1.002573),(30.828381,-1.002573),(30.812465,-0.994719),(30.777118,-0.98583),(30.739498,-1.005467),(30.701567,-1.017456),(30.687098,-1.025001),(30.651338,-1.062828),(30.638626,-1.07337),(30.614544,-1.065825),(30.592737,-1.063448),(30.542301,-1.068099),(30.523697,-1.07244),(30.511657,-1.07337),(30.506024,-1.070786),(30.500856,-1.065205),(30.493415,-1.060244),(30.481012,-1.059107),(30.472951,-1.065619),(30.471786,-1.066837),(30.460829,-1.063428),(30.445614,-1.058694),(30.432023,-1.060554),(30.418897,-1.066445),(30.403188,-1.070373),(30.386341,-1.068202),(30.369288,-1.063241),(30.352752,-1.060761),(30.337455,-1.066239),(30.329032,-1.080501),(30.322572,-1.121843),(30.317405,-1.137035),(30.311307,-1.1421),(30.294564,-1.149644),(30.29095,-1.152621),(30.287536,-1.155432),(30.284642,-1.161427),(30.28242,-1.175793),(30.280508,-1.182407),(30.269759,-1.200494),(30.256685,-1.217237),(30.212192,-1.259509),(30.19674,-1.268707),(30.189351,-1.270877),(30.181392,-1.271497),(30.173434,-1.272841),(30.165579,-1.277492),(30.158448,-1.291135),(30.15235,-1.329892),(30.147183,-1.345085),(30.136331,-1.355213),(30.095506,-1.37113),(30.065984,-1.386945),(30.06078,-1.389733),(30.047757,-1.403169),(30.038662,-1.424977),(30.028327,-1.427147),(29.960424,-1.464767),(29.938462,-1.472932),(29.917429,-1.475206),(29.897896,-1.469625),(29.880739,-1.453605),(29.871024,-1.432418),(29.868647,-1.391283),(29.864203,-1.370303),(29.836091,-1.329478),(29.825135,-1.323897),(29.825024,-1.323881),(29.816143,-1.322554),(29.807462,-1.325138),(29.798212,-1.330925),(29.789168,-1.341674),(29.783174,-1.361414),(29.774906,-1.366272),(29.767981,-1.363792),(29.74669,-1.350872),(29.734805,-1.348185),(29.710517,-1.352526),(29.693774,-1.361208),(29.678322,-1.37237),(29.657703,-1.383945),(29.6391,-1.38901),(29.618119,-1.39056),(29.577915,-1.38839),(29.58732,-1.329685),(29.587113,-1.310565),(29.583186,-1.299299),(29.571507,-1.279249),(29.57099,-1.268604),(29.580447,-1.243282),(29.581429,-1.234807),(29.575538,-1.213206),(29.565254,-1.19791),(29.557038,-1.181787),(29.556934,-1.157706),(29.56913,-1.095901),(29.57006,-1.077918),(29.565926,-1.058591),(29.551198,-1.020143),(29.54846,-1.002573),(29.551353,-0.990584),(29.551518,-0.990305),(29.556521,-0.981799),(29.560345,-0.972084),(29.559157,-0.957305),(29.555281,-0.938495),(29.554661,-0.928573),(29.556004,-0.919478),(29.567166,-0.901908),(29.596725,-0.891882),(29.607991,-0.878447),(29.610781,-0.863977),(29.613365,-0.803826),(29.611091,-0.782742),(29.602203,-0.743778),(29.60303,-0.7229),(29.615536,-0.644146),(29.618843,-0.638978),(29.624424,-0.634844),(29.629281,-0.62978),(29.630573,-0.622028),(29.628248,-0.616034),(29.620496,-0.605388),(29.618843,-0.599394),(29.622874,-0.588438),(29.632175,-0.585751),(29.642821,-0.585028),(29.650882,-0.57955),(29.653053,-0.565597),(29.649745,-0.504309),(29.645094,-0.48891),(29.634615,-0.466968),(29.631865,-0.461211),(29.629385,-0.442401),(29.650981,-0.316455),(29.653983,-0.298947),(29.670569,-0.200867),(29.676617,-0.165105),(29.694032,-0.063096),(29.709018,-0.026302),(29.714341,-0.007492),(29.713462,0.011628),(29.701628,0.055243),(29.703075,0.072503),(29.711809,0.099582),(29.755785,0.16087),(29.761263,0.172135),(29.773045,0.167484),(29.780383,0.16118),(29.787205,0.158493),(29.797127,0.164797),(29.800641,0.172445),(29.832628,0.336983),(29.839605,0.358481),(29.851077,0.377187),(29.903909,0.438229),(29.922907,0.46018),(29.926083,0.467073),(29.940477,0.498317),(29.937996,0.537281),(29.919496,0.618103),(29.920013,0.63867),(29.932312,0.723161),(29.926834,0.774889),(29.928281,0.785018),(29.947298,0.824602),(29.960217,0.832095),(29.982194,0.849017),(29.996391,0.859949),(30.038352,0.878914),(30.145116,0.90315),(30.154831,0.90868),(30.165683,0.921444),(30.18377,0.955137),(30.18687,0.958754),(30.191521,0.974877),(30.214052,0.998545),(30.220977,1.017097),(30.215602,1.057766),(30.215292,1.077093),(30.228005,1.08903),(30.231519,1.097764),(30.234102,1.108099),(30.236169,1.129493),(30.238857,1.136004),(30.269346,1.167268),(30.277976,1.171609),(30.286502,1.174296),(30.295701,1.172643),(30.306553,1.163909),(30.323813,1.155848),(30.33606,1.16887),(30.348204,1.189024),(30.364947,1.202047),(30.376626,1.20308),(30.39926,1.2006),(30.412696,1.202047),(30.43161,1.207008),(30.445562,1.212795),(30.458275,1.221684),(30.478274,1.238634),(30.553704,1.335632),(30.597284,1.391673),(30.681724,1.500349),(30.817013,1.609515),(30.95442,1.720671),(31.025837,1.778239),(31.096018,1.866363),(31.119113,1.895363),(31.183295,1.976211),(31.242826,2.051168),(31.271455,2.102999),(31.280096,2.151441),(31.280447,2.15341),(31.27874,2.156021),(31.267476,2.173253),(31.21089,2.205345),(31.190116,2.221519),(31.182468,2.238728),(31.179058,2.25976),(31.177559,2.30291),(31.129242,2.284668),(31.112602,2.282084),(31.099063,2.282704),(31.055241,2.290249),(31.040668,2.297897),(31.035501,2.30694),(31.038808,2.310971),(31.044389,2.313917),(31.045939,2.319653),(31.043562,2.327043),(31.041288,2.331228),(30.984858,2.394635),(30.968011,2.405436),(30.930907,2.405591),(30.914474,2.378151),(30.900573,2.345956),(30.87179,2.332055),(30.854943,2.339703),(30.83634,2.356343),(30.819803,2.37598),(30.809675,2.392362),(30.806987,2.406986),(30.807142,2.422179),(30.80492,2.434374),(30.794998,2.440111),(30.724925,2.440782),(30.710559,2.445123),(30.707665,2.46228),(30.71676,2.483105),(30.729369,2.503466),(30.737017,2.519537),(30.737844,2.537469),(30.73402,2.574469),(30.73526,2.593073),(30.739395,2.603305),(30.758928,2.633794),(30.761254,2.641597),(30.762013,2.645781),(30.764303,2.658392),(30.797686,2.74836),(30.798926,2.753528),(30.799133,2.76345),(30.801613,2.769083),(30.80585,2.771873),(30.817943,2.774199),(30.82187,2.776162),(30.828691,2.786084),(30.853393,2.853367),(30.85484,2.89321),(30.843988,2.932794),(30.82094,2.973153),(30.803628,2.989069),(30.757068,3.02147),(30.745079,3.036302),(30.743839,3.055474),(30.74787,3.076713),(30.763416,3.123437),(30.804197,3.246005),(30.822129,3.281403),(30.825746,3.283677),(30.837786,3.286416),(30.842437,3.288741),(30.845641,3.293909),(30.846882,3.304399),(30.84869,3.309256),(30.868482,3.343337),(30.897318,3.375015),(30.904933,3.386035),(30.91003,3.393412),(30.916335,3.414806),(30.914474,3.426484),(30.904449,3.447465),(30.902899,3.458911),(30.909617,3.487178),(30.909307,3.496144),(30.896388,3.519967),(30.880161,3.514412),(30.861248,3.498211),(30.839543,3.490202),(30.843781,3.505911),(30.865692,3.548958),(30.931734,3.645102),(30.936282,3.656858),(30.939382,3.668408),(30.944447,3.679286),(30.955505,3.689001),(30.965996,3.692618),(30.985788,3.692127),(30.995968,3.693522),(31.009559,3.699775),(31.040358,3.724218),(31.04966,3.727991),(31.068677,3.731737),(31.077668,3.735303),(31.104408,3.756175),(31.141489,3.785119),(31.167585,3.792405),(31.214818,3.792354),(31.255745,3.786669),(31.29502,3.774189),(31.377702,3.729308),(31.505446,3.659855),(31.523739,3.656083),(31.53452,3.665563),(31.535522,3.666444),(31.547201,3.680991),(31.564771,3.689802),(31.668537,3.70502),(31.68683,3.712824),(31.696132,3.721273),(31.775714,3.810699),(31.777884,3.816435),(31.780985,3.815815),(31.801107,3.806472),(31.80703,3.803722),(31.830697,3.783879),(31.901908,3.704297),(31.916274,3.680267),(31.920046,3.661302),(31.923095,3.615284),(31.93002,3.59836),(31.943662,3.591255),(32.022236,3.58642),(32.030168,3.585932),(32.041537,3.57986),(32.054226,3.559572),(32.060864,3.548958),(32.076161,3.53317),(32.093007,3.524463),(32.155846,3.511776),(32.16799,3.512242),(32.174552,3.520897),(32.175929,3.527234),(32.178997,3.541361),(32.179203,3.556864),(32.174863,3.59265),(32.175948,3.605595),(32.187782,3.61916),(32.371801,3.731065),(32.415571,3.741297),(32.599281,3.756283),(32.756429,3.769022),(32.840352,3.794291),(32.9189,3.83416),(32.979568,3.879196),(32.997241,3.885526),(33.01724,3.87718),(33.143486,3.774086),(33.164466,3.763053),(33.19542,3.757059),(33.286526,3.752537),(33.447343,3.744372),(33.490648,3.749746),(33.527716,3.771431),(33.532609,3.774293),(33.606196,3.848087),(33.701901,3.944076),(33.813677,4.056033),(33.896049,4.138353),(33.977078,4.219692),(34.006017,4.205713),(34.028548,4.188014),(34.041054,4.164812),(34.03971,4.134038),(34.04095,4.120421),(34.049735,4.109466),(34.060897,4.09926),(34.069269,4.08802),(34.072369,4.076419),(34.072679,4.064663),(34.069786,4.041357),(34.065858,4.02743),(34.061517,4.017921),(34.061104,4.007767),(34.068959,3.991876),(34.080844,3.980792),(34.09552,3.970999),(34.107096,3.959501),(34.109576,3.943352),(34.098931,3.917721),(34.086219,3.894673),(34.084772,3.877336),(34.108026,3.868938)] +Uruguay [(-56.906367,-30.108549),(-56.869211,-30.102968),(-56.831281,-30.102037),(-56.795159,-30.123845),(-56.776607,-30.150923),(-56.766995,-30.161362),(-56.704777,-30.198362),(-56.673745,-30.211281),(-56.656821,-30.221307),(-56.641783,-30.233916),(-56.632585,-30.247455),(-56.629278,-30.267712),(-56.63243,-30.278151),(-56.631345,-30.284662),(-56.615403,-30.293344),(-56.605196,-30.295617),(-56.595611,-30.295411),(-56.585818,-30.296548),(-56.575017,-30.302439),(-56.569617,-30.30802),(-56.560781,-30.320525),(-56.523754,-30.357319),(-56.5113,-30.366001),(-56.494557,-30.379333),(-56.441279,-30.408789),(-56.424639,-30.423568),(-56.400454,-30.458915),(-56.385985,-30.475761),(-56.372601,-30.48589),(-56.326531,-30.508318),(-56.267956,-30.551106),(-56.23509,-30.565368),(-56.215091,-30.581905),(-56.183517,-30.614564),(-56.177832,-30.62552),(-56.175171,-30.636062),(-56.170804,-30.645777),(-56.159642,-30.653942),(-56.140315,-30.664794),(-56.131685,-30.671925),(-56.125045,-30.680297),(-56.091894,-30.736107),(-56.076857,-30.752334),(-56.011357,-30.798222),(-55.995311,-30.825818),(-55.988955,-30.85579),(-55.993864,-30.885452),(-56.011357,-30.912634),(-56.015517,-30.934338),(-56.016344,-30.999864),(-56.02239,-31.045546),(-56.022183,-31.067147),(-56.017018,-31.074296),(-56.011357,-31.082133),(-56.009874,-31.081945),(-55.985157,-31.078825),(-55.919347,-31.082443),(-55.888186,-31.076965),(-55.86426,-31.076758),(-55.854596,-31.074588),(-55.846586,-31.069317),(-55.842142,-31.063943),(-55.838396,-31.058155),(-55.832479,-31.051747),(-55.778141,-31.020431),(-55.763594,-31.008442),(-55.755507,-30.992939),(-55.744887,-30.958626),(-55.731891,-30.945397),(-55.712745,-30.943433),(-55.688302,-30.947774),(-55.665822,-30.949324),(-55.652748,-30.938989),(-55.651069,-30.919972),(-55.654583,-30.878734),(-55.649673,-30.860958),(-55.634041,-30.847625),(-55.612906,-30.843388),(-55.591873,-30.848348),(-55.563865,-30.87646),(-55.527484,-30.894547),(-55.51131,-30.906123),(-55.489089,-30.929687),(-55.44258,-30.965551),(-55.368579,-31.037381),(-55.353852,-31.056294),(-55.348426,-31.072728),(-55.344963,-31.108488),(-55.338039,-31.125644),(-55.326928,-31.134946),(-55.2937,-31.153653),(-55.282228,-31.169052),(-55.268741,-31.2106),(-55.259981,-31.228687),(-55.244349,-31.244603),(-55.227554,-31.253492),(-55.18828,-31.266514),(-55.169625,-31.276436),(-55.122238,-31.31354),(-55.102446,-31.324288),(-55.087253,-31.326769),(-55.074256,-31.320568),(-55.061105,-31.304858),(-55.042346,-31.272612),(-55.029375,-31.268788),(-55.011495,-31.287805),(-54.995346,-31.311783),(-54.969637,-31.340825),(-54.940802,-31.36563),(-54.900288,-31.382063),(-54.887808,-31.393225),(-54.876155,-31.406351),(-54.863365,-31.417616),(-54.849903,-31.425057),(-54.819776,-31.435289),(-54.654127,-31.455753),(-54.619685,-31.45565),(-54.604543,-31.459784),(-54.591521,-31.471049),(-54.573848,-31.502469),(-54.562375,-31.515905),(-54.509665,-31.552285),(-54.495041,-31.565617),(-54.483155,-31.583911),(-54.47897,-31.601481),(-54.477368,-31.642098),(-54.467221,-31.664237),(-54.463725,-31.671864),(-54.436492,-31.694292),(-54.403677,-31.714652),(-54.373498,-31.73894),(-54.342957,-31.769326),(-54.274228,-31.823379),(-54.218779,-31.856659),(-54.169583,-31.895933),(-54.146277,-31.909886),(-54.135063,-31.908956),(-54.125503,-31.898517),(-54.107313,-31.883944),(-54.088451,-31.878156),(-54.068814,-31.879913),(-54.049745,-31.887148),(-54.03264,-31.898104),(-54.026646,-31.904821),(-54.017706,-31.919601),(-54.008714,-31.926526),(-53.998896,-31.92973),(-53.965978,-31.93252),(-53.948769,-31.936964),(-53.934352,-31.942545),(-53.905826,-31.959288),(-53.894148,-31.97014),(-53.889755,-31.980372),(-53.887275,-31.990501),(-53.881435,-32.00125),(-53.8711,-32.011482),(-53.858077,-32.021197),(-53.83043,-32.036441),(-53.771829,-32.047087),(-53.757308,-32.055045),(-53.751366,-32.068791),(-53.750022,-32.104034),(-53.721445,-32.162428),(-53.670595,-32.226404),(-53.658555,-32.254309),(-53.651837,-32.288312),(-53.648684,-32.338645),(-53.643879,-32.355595),(-53.634783,-32.368721),(-53.609307,-32.387841),(-53.59892,-32.399933),(-53.58197,-32.425772),(-53.5613,-32.449543),(-53.537632,-32.470213),(-53.474638,-32.508247),(-53.415624,-32.564161),(-53.35909,-32.580284),(-53.316043,-32.602712),(-53.299093,-32.607156),(-53.266899,-32.607363),(-53.233723,-32.624622),(-53.201477,-32.637232),(-53.186129,-32.64674),(-53.119776,-32.707408),(-53.110836,-32.722394),(-53.11404,-32.740584),(-53.126856,-32.754847),(-53.163081,-32.778722),(-53.182836,-32.799904),(-53.203854,-32.82244),(-53.270826,-32.863781),(-53.298835,-32.889102),(-53.307,-32.907189),(-53.309584,-32.944396),(-53.316302,-32.96269),(-53.327309,-32.973645),(-53.44911,-33.042065),(-53.48332,-33.067283),(-53.511535,-33.099219),(-53.511587,-33.099322),(-53.511639,-33.099426),(-53.520662,-33.124979),(-53.536857,-33.170842),(-53.536133,-33.244636),(-53.514016,-33.394911),(-53.536805,-33.559863),(-53.539647,-33.649263),(-53.511535,-33.690294),(-53.491071,-33.690294),(-53.473191,-33.6874),(-53.456707,-33.687503),(-53.440428,-33.697115),(-53.430817,-33.713445),(-53.423634,-33.731015),(-53.411283,-33.74228),(-53.379095,-33.740676),(-53.396352,-33.750584),(-53.414784,-33.764418),(-53.450917,-33.801528),(-53.473053,-33.834568),(-53.485585,-33.869317),(-53.505523,-33.952325),(-53.524648,-34.002374),(-53.529937,-34.04811),(-53.539459,-34.062433),(-53.574452,-34.082696),(-53.591217,-34.095473),(-53.639963,-34.147638),(-53.693593,-34.181736),(-53.711008,-34.198826),(-53.746653,-34.25742),(-53.755971,-34.267673),(-53.75829,-34.278497),(-53.779897,-34.336033),(-53.763661,-34.372491),(-53.764882,-34.390395),(-53.805776,-34.400974),(-53.898305,-34.443943),(-53.972239,-34.487237),(-54.124623,-34.610447),(-54.13679,-34.623956),(-54.14094,-34.641046),(-54.140492,-34.656427),(-54.145904,-34.667576),(-54.236806,-34.684503),(-54.259674,-34.685968),(-54.255605,-34.677504),(-54.249501,-34.670994),(-54.23233,-34.658136),(-54.239329,-34.65309),(-54.245676,-34.646905),(-54.250478,-34.639337),(-54.252838,-34.630792),(-54.25121,-34.618341),(-54.241078,-34.594008),(-54.238596,-34.589776),(-54.253285,-34.577569),(-54.280344,-34.567966),(-54.308949,-34.562188),(-54.328603,-34.561782),(-54.317372,-34.568617),(-54.280141,-34.582289),(-54.288157,-34.594822),(-54.304433,-34.608575),(-54.32311,-34.619399),(-54.33849,-34.623956),(-54.344635,-34.630629),(-54.344797,-34.64422),(-54.337392,-34.654718),(-54.321156,-34.65252),(-54.322581,-34.644708),(-54.307688,-34.647393),(-54.289215,-34.660252),(-54.280141,-34.682306),(-54.288075,-34.695001),(-54.306467,-34.707696),(-54.530792,-34.806899),(-54.542063,-34.811782),(-54.604156,-34.82643),(-54.636464,-34.84881),(-54.65689,-34.856622),(-54.699452,-34.867446),(-54.872711,-34.936212),(-54.893707,-34.939223),(-54.913726,-34.945245),(-54.93932,-34.969903),(-54.958608,-34.973403),(-54.953969,-34.963311),(-54.951812,-34.956964),(-54.951772,-34.952895),(-54.962229,-34.93963),(-54.975738,-34.928481),(-54.99356,-34.920994),(-55.03307,-34.916762),(-55.035471,-34.912856),(-55.034535,-34.906508),(-55.041127,-34.897638),(-55.051259,-34.891371),(-55.060374,-34.887791),(-55.082143,-34.883966),(-55.129791,-34.885919),(-55.22175,-34.903741),(-55.260243,-34.897638),(-55.286488,-34.876235),(-55.338287,-34.81878),(-55.376943,-34.801446),(-55.401926,-34.799981),(-55.468204,-34.795402),(-55.509636,-34.799065),(-55.561173,-34.787834),(-55.601794,-34.772415),(-55.646731,-34.7811),(-55.693448,-34.764741),(-55.756503,-34.782666),(-55.792051,-34.774134),(-55.858786,-34.793061),(-55.889101,-34.803475),(-55.954432,-34.839402),(-56.004052,-34.869102),(-56.069634,-34.901003),(-56.105198,-34.898416),(-56.135633,-34.912829),(-56.149892,-34.920343),(-56.156088,-34.941188),(-56.17729,-34.916325),(-56.19451,-34.913509),(-56.213002,-34.907693),(-56.204661,-34.898696),(-56.203455,-34.887503),(-56.214589,-34.879588),(-56.230551,-34.87775),(-56.236508,-34.892884),(-56.256337,-34.906183),(-56.265824,-34.906266),(-56.287976,-34.903489),(-56.311269,-34.906183),(-56.342763,-34.883966),(-56.402943,-34.85475),(-56.418446,-34.843032),(-56.420766,-34.82822),(-56.403751,-34.816051),(-56.386848,-34.80187),(-56.361282,-34.796636),(-56.37612,-34.782729),(-56.395738,-34.77588),(-56.448432,-34.758272),(-56.485088,-34.75355),(-56.520444,-34.754821),(-56.5419,-34.767022),(-56.558013,-34.765069),(-56.573598,-34.758966),(-56.610951,-34.739435),(-56.628529,-34.732599),(-56.646362,-34.722642),(-56.713544,-34.707037),(-56.795289,-34.698492),(-56.873891,-34.67018),(-56.890696,-34.660089),(-56.904164,-34.635512),(-56.919749,-34.623224),(-56.95287,-34.603448),(-56.976577,-34.581046),(-56.997417,-34.564069),(-57.015788,-34.552091),(-57.042717,-34.538125),(-57.057425,-34.527127),(-57.055898,-34.513442),(-57.062408,-34.506524),(-57.077016,-34.498142),(-57.08316,-34.493585),(-57.101959,-34.475844),(-57.120025,-34.462986),(-57.151964,-34.450932),(-57.203198,-34.442073),(-57.292158,-34.440333),(-57.346942,-34.443459),(-57.3652,-34.43044),(-57.411428,-34.431506),(-57.436974,-34.444576),(-57.44913,-34.431538),(-57.474663,-34.429541),(-57.502635,-34.440582),(-57.537914,-34.453614),(-57.569475,-34.426504),(-57.594997,-34.425473),(-57.624177,-34.430452),(-57.664324,-34.445438),(-57.7203,-34.463373),(-57.738535,-34.461318),(-57.764115,-34.474295),(-57.793302,-34.472197),(-57.82251,-34.475109),(-57.855354,-34.470967),(-57.845585,-34.462979),(-57.854048,-34.447882),(-57.882019,-34.440724),(-57.90019,-34.416525),(-57.8976,-34.376339),(-57.92276,-34.353692),(-57.942738,-34.326267),(-58.016916,-34.253106),(-58.049794,-34.240004),(-58.061391,-34.224705),(-58.076975,-34.19199),(-58.094594,-34.17783),(-58.119374,-34.169203),(-58.145904,-34.165134),(-58.169097,-34.163995),(-58.196197,-34.159438),(-58.20873,-34.147638),(-58.220326,-34.11004),(-58.231842,-34.0888),(-58.316518,-33.986505),(-58.334828,-33.973321),(-58.372426,-33.95436),(-58.388824,-33.942071),(-58.402699,-33.922784),(-58.410024,-33.902276),(-58.434641,-33.784926),(-58.439361,-33.6985),(-58.425771,-33.612888),(-58.429026,-33.592055),(-58.435048,-33.572361),(-58.438344,-33.551446),(-58.433217,-33.527765),(-58.419586,-33.508233),(-58.385365,-33.47031),(-58.37857,-33.448663),(-58.384877,-33.429132),(-58.412587,-33.409601),(-58.418935,-33.389907),(-58.412587,-33.371677),(-58.397206,-33.355645),(-58.361195,-33.325779),(-58.350738,-33.305759),(-58.349233,-33.28281),(-58.366038,-33.196059),(-58.366933,-33.153741),(-58.351389,-33.121677),(-58.310292,-33.108819),(-58.217356,-33.113539),(-58.174428,-33.110447),(-58.137766,-33.095147),(-58.100168,-33.065606),(-58.08373,-33.04754),(-58.076975,-33.030694),(-58.075307,-33.007501),(-58.070668,-32.989516),(-58.055816,-32.951837),(-58.048492,-32.911716),(-58.057118,-32.881768),(-58.077219,-32.857843),(-58.120473,-32.819268),(-58.130767,-32.802667),(-58.136138,-32.784356),(-58.137766,-32.763442),(-58.135854,-32.718357),(-58.137685,-32.695571),(-58.145172,-32.678155),(-58.137766,-32.670668),(-58.14623,-32.644952),(-58.157216,-32.575128),(-58.169097,-32.560805),(-58.177113,-32.545017),(-58.197418,-32.469822),(-58.200103,-32.447198),(-58.200112,-32.44713),(-58.199724,-32.444789),(-58.189958,-32.433936),(-58.164946,-32.389495),(-58.101591,-32.31136),(-58.096527,-32.280974),(-58.106914,-32.251829),(-58.17523,-32.174107),(-58.186547,-32.15292),(-58.180863,-32.132456),(-58.158797,-32.101554),(-58.148306,-32.0629),(-58.145309,-32.01789),(-58.14903,-31.975205),(-58.158797,-31.943889),(-58.190319,-31.913296),(-58.202618,-31.893143),(-58.196004,-31.872575),(-58.168615,-31.846014),(-58.152854,-31.835988),(-58.13084,-31.827824),(-58.083453,-31.819762),(-58.059268,-31.811494),(-58.048881,-31.797128),(-57.988626,-31.642822),(-57.979583,-31.598794),(-57.986818,-31.554145),(-58.013328,-31.52562),(-58.038978,-31.508637),(-58.050793,-31.500815),(-58.075236,-31.475184),(-58.062575,-31.444281),(-58.043558,-31.430639),(-58.005938,-31.411001),(-57.990228,-31.399323),(-57.980513,-31.380512),(-57.975552,-31.33514),(-57.966354,-31.314573),(-57.959481,-31.307959),(-57.942996,-31.296797),(-57.935606,-31.290285),(-57.914832,-31.252458),(-57.905117,-31.240986),(-57.905014,-31.215044),(-57.911732,-31.170603),(-57.899019,-31.126471),(-57.87375,-31.093088),(-57.855249,-31.058982),(-57.863311,-31.012266),(-57.905479,-30.958833),(-57.911732,-30.947361),(-57.90398,-30.929687),(-57.885325,-30.918835),(-57.842795,-30.909223),(-57.819644,-30.9116),(-57.807242,-30.90757),(-57.801868,-30.89248),(-57.796132,-30.868399),(-57.794995,-30.85796),(-57.796028,-30.816206),(-57.799397,-30.791461),(-57.801868,-30.773314),(-57.808659,-30.747331),(-57.817887,-30.712026),(-57.826207,-30.689702),(-57.848531,-30.647947),(-57.863311,-30.62862),(-57.88579,-30.589863),(-57.889769,-30.550796),(-57.877109,-30.514829),(-57.849617,-30.48527),(-57.652729,-30.329104),(-57.631077,-30.297064),(-57.623739,-30.2581),(-57.63795,-30.215105),(-57.642446,-30.193091),(-57.633971,-30.183583),(-57.611698,-30.182963),(-57.586842,-30.204047),(-57.567463,-30.256343),(-57.535992,-30.27443),(-57.510981,-30.276704),(-57.467211,-30.269883),(-57.44313,-30.269469),(-57.429797,-30.275464),(-57.414811,-30.295721),(-57.399257,-30.299131),(-57.389386,-30.294894),(-57.366597,-30.277221),(-57.353833,-30.27195),(-57.334299,-30.272673),(-57.321742,-30.279804),(-57.310993,-30.288279),(-57.296472,-30.293344),(-57.288617,-30.29076),(-57.276215,-30.277737),(-57.270376,-30.275154),(-57.262469,-30.278668),(-57.250067,-30.29045),(-57.245054,-30.293344),(-57.212601,-30.287349),(-57.183714,-30.267505),(-57.163819,-30.23929),(-57.158341,-30.207871),(-57.150848,-30.182343),(-57.129557,-30.149993),(-57.102272,-30.121158),(-57.077106,-30.106068),(-57.067235,-30.106792),(-57.048529,-30.114543),(-57.037883,-30.114647),(-57.018298,-30.105758),(-57.01127,-30.103484),(-56.97458,-30.09749),(-56.954529,-30.09687),(-56.906367,-30.108549)] +Vatican [(12.453137,41.902752),(12.452714,41.903016),(12.452767,41.903439),(12.453031,41.903915),(12.453983,41.903862),(12.454035,41.902752),(12.453137,41.902752)] +Akrotiri Sovereign Base Area [(32.840809,34.699467),(32.845992,34.699467),(32.853766,34.699467),(32.85969,34.700578),(32.86043,34.687621),(32.865613,34.678735),(32.884504,34.666607),(32.913667,34.66028),(32.917246,34.667136),(32.929752,34.662485),(32.936056,34.657731),(32.942258,34.667136),(32.954712,34.677316),(32.987526,34.673389),(32.9898,34.646103),(32.970266,34.646103),(32.965615,34.651581),(32.95404,34.642951),(32.95559,34.628895),(32.962515,34.625019),(32.969543,34.632047),(32.99073,34.634425),(33.015635,34.634425),(33.009939,34.624905),(33.009125,34.597398),(33.030284,34.574774),(33.030284,34.568549),(32.940929,34.568549),(32.940929,34.574774),(32.944672,34.593695),(32.915863,34.63939),(32.913585,34.656684),(32.857432,34.669013),(32.834321,34.670966),(32.812836,34.668647),(32.760671,34.653225),(32.760102,34.66911),(32.766766,34.676144),(32.778613,34.673182),(32.798604,34.670961),(32.806009,34.68577),(32.819336,34.687621),(32.822668,34.691323),(32.827481,34.696506),(32.832294,34.700948),(32.835626,34.699838),(32.840809,34.699467)] +Zambia [(31.119836,-8.616631),(31.141024,-8.606192),(31.161384,-8.591723),(31.182675,-8.580767),(31.206756,-8.580767),(31.218435,-8.588725),(31.237349,-8.613943),(31.248046,-8.621902),(31.260913,-8.623969),(31.269388,-8.621075),(31.277088,-8.61601),(31.328816,-8.597717),(31.340391,-8.595237),(31.347523,-8.592446),(31.350107,-8.588519),(31.353311,-8.587175),(31.362199,-8.592343),(31.36592,-8.598647),(31.369227,-8.616837),(31.372638,-8.623865),(31.38597,-8.632547),(31.398838,-8.633787),(31.412428,-8.63234),(31.427518,-8.633477),(31.443228,-8.641539),(31.464725,-8.666137),(31.480641,-8.676162),(31.519089,-8.687014),(31.539036,-8.70355),(31.54596,-8.728975),(31.545857,-8.766286),(31.55335,-8.809074),(31.576449,-8.839666),(31.672671,-8.913047),(31.689621,-8.919558),(31.709775,-8.919661),(31.730238,-8.912426),(31.764345,-8.894133),(31.787599,-8.892169),(31.936634,-8.93258),(31.917824,-8.973095),(31.917721,-9.022497),(31.938081,-9.061771),(31.980559,-9.0719),(32.001385,-9.063322),(32.015802,-9.052676),(32.031719,-9.045958),(32.057247,-9.049576),(32.086031,-9.066112),(32.096418,-9.069109),(32.105823,-9.068283),(32.125046,-9.063735),(32.134141,-9.064148),(32.145574,-9.070265),(32.154812,-9.075207),(32.191606,-9.112208),(32.211294,-9.12678),(32.231448,-9.133808),(32.252739,-9.136495),(32.383015,-9.133912),(32.423426,-9.143834),(32.459703,-9.168018),(32.470658,-9.181867),(32.490089,-9.227343),(32.504455,-9.249253),(32.518046,-9.258245),(32.555614,-9.261242),(32.641604,-9.279846),(32.713228,-9.28584),(32.725837,-9.292558),(32.739686,-9.307338),(32.743407,-9.314986),(32.746507,-9.329972),(32.752295,-9.337413),(32.759633,-9.339997),(32.776996,-9.33824),(32.784334,-9.339687),(32.83074,-9.370176),(32.905464,-9.398185),(32.920863,-9.4079),(32.923034,-9.466294),(32.927788,-9.479834),(32.937503,-9.487792),(32.943601,-9.485725),(32.950267,-9.480867),(32.961895,-9.48035),(32.968613,-9.484381),(32.983082,-9.497714),(32.99197,-9.501848),(32.992384,-9.529133),(32.97471,-9.601273),(32.986906,-9.628455),(33.000135,-9.632796),(33.011194,-9.627835),(33.022253,-9.62029),(33.03574,-9.616983),(33.046747,-9.622047),(33.050778,-9.632693),(33.053052,-9.644785),(33.058323,-9.654293),(33.08044,-9.662251),(33.089225,-9.643958),(33.092533,-9.614399),(33.09832,-9.588458),(33.121471,-9.599516),(33.175422,-9.602307),(33.19542,-9.616259),(33.209838,-9.641374),(33.214282,-9.654707),(33.215781,-9.6699),(33.21299,-9.680648),(33.207823,-9.68871),(33.204567,-9.698115),(33.207254,-9.712894),(33.216711,-9.727054),(33.241774,-9.74235),(33.25273,-9.754029),(33.267199,-9.781211),(33.275777,-9.793303),(33.287766,-9.803845),(33.299962,-9.810253),(33.336962,-9.823689),(33.343473,-9.83144),(33.359183,-9.870921),(33.365591,-9.898826),(33.35846,-9.91991),(33.328797,-9.965489),(33.309987,-10.012721),(33.304716,-10.037939),(33.304096,-10.063777),(33.317532,-10.082071),(33.399181,-10.117934),(33.420161,-10.135608),(33.436491,-10.153281),(33.454578,-10.16806),(33.480519,-10.177052),(33.499123,-10.201237),(33.518967,-10.214983),(33.533281,-10.231209),(33.535606,-10.262732),(33.529302,-10.318646),(33.532919,-10.342623),(33.559171,-10.404325),(33.567646,-10.416004),(33.595551,-10.440602),(33.603199,-10.44918),(33.622733,-10.494035),(33.638184,-10.511605),(33.665418,-10.515119),(33.665573,-10.541371),(33.672652,-10.559561),(33.673523,-10.569367),(33.673849,-10.573048),(33.674203,-10.577028),(33.657769,-10.601316),(33.642266,-10.615268),(33.603096,-10.643277),(33.59028,-10.648961),(33.576947,-10.657643),(33.528268,-10.711593),(33.51199,-10.751797),(33.500983,-10.769161),(33.480519,-10.781563),(33.465326,-10.784354),(33.447136,-10.802234),(33.43401,-10.808125),(33.418663,-10.807298),(33.385538,-10.80089),(33.37174,-10.803267),(33.357839,-10.808745),(33.332621,-10.813602),(33.319082,-10.81815),(33.306886,-10.828589),(33.288128,-10.856701),(33.2734,-10.865589),(33.261204,-10.865279),(33.250766,-10.862488),(33.241412,-10.865692),(33.232576,-10.883262),(33.231697,-10.897628),(33.239604,-10.903106),(33.252109,-10.903106),(33.265855,-10.901142),(33.280325,-10.913338),(33.28787,-10.944861),(33.29096,-10.975086),(33.293967,-11.004495),(33.303682,-11.034571),(33.3181,-11.062269),(33.3366,-11.087384),(33.358873,-11.109398),(33.390912,-11.164795),(33.382231,-11.194664),(33.368485,-11.222673),(33.298308,-11.32923),(33.29681,-11.335741),(33.300272,-11.347007),(33.298928,-11.353104),(33.293347,-11.357755),(33.278051,-11.364266),(33.273503,-11.369641),(33.27185,-11.383697),(33.273814,-11.396616),(33.273607,-11.409432),(33.265184,-11.423074),(33.239707,-11.402404),(33.230302,-11.416563),(33.234953,-11.501726),(33.233816,-11.514025),(33.227305,-11.532732),(33.212629,-11.563841),(33.215936,-11.571592),(33.232679,-11.57769),(33.249319,-11.577793),(33.266889,-11.575623),(33.283219,-11.578414),(33.296758,-11.593606),(33.302649,-11.60973),(33.314018,-11.7722),(33.310607,-11.808787),(33.297378,-11.846511),(33.297481,-11.862531),(33.308747,-11.884441),(33.312674,-11.898704),(33.309315,-11.91431),(33.303579,-11.930227),(33.294226,-11.981283),(33.2549,-12.072647),(33.251283,-12.105823),(33.25273,-12.139516),(33.260274,-12.155743),(33.288386,-12.182201),(33.299962,-12.195947),(33.312467,-12.228606),(33.330141,-12.298266),(33.349468,-12.327929),(33.384608,-12.340021),(33.458298,-12.31811),(33.497366,-12.331546),(33.514109,-12.331029),(33.525581,-12.335267),(33.527131,-12.355627),(33.51814,-12.37175),(33.513887,-12.374281),(33.487754,-12.389837),(33.480519,-12.406373),(33.465326,-12.409061),(33.460779,-12.416502),(33.459952,-12.4411),(33.451115,-12.452986),(33.420781,-12.464974),(33.409412,-12.475206),(33.393755,-12.498254),(33.373239,-12.518511),(33.350398,-12.532671),(33.32828,-12.537632),(33.320322,-12.535048),(33.311951,-12.52988),(33.302236,-12.525126),(33.28973,-12.524196),(33.263168,-12.528536),(33.255107,-12.531947),(33.243634,-12.544763),(33.226995,-12.579076),(33.21299,-12.588584),(33.202913,-12.591788),(33.196816,-12.595406),(33.192371,-12.599023),(33.187307,-12.60233),(33.181313,-12.608222),(33.178005,-12.615043),(33.173251,-12.620004),(33.163019,-12.620417),(33.158782,-12.615353),(33.146896,-12.593235),(33.13992,-12.586311),(33.13036,-12.583727),(33.122815,-12.584554),(33.107002,-12.588688),(33.099044,-12.593235),(33.089949,-12.60078),(33.081267,-12.604811),(33.074239,-12.5983),(33.070002,-12.590445),(33.064627,-12.584967),(33.057703,-12.583417),(33.049538,-12.586827),(33.035378,-12.599126),(33.024836,-12.612666),(32.994141,-12.672507),(32.958071,-12.725217),(32.944635,-12.752399),(32.941121,-12.769865),(32.947012,-12.843246),(32.953523,-12.854098),(32.980498,-12.870738),(33.005044,-12.892959),(33.013571,-12.917247),(33.00892,-12.945049),(32.981532,-13.00613),(32.965925,-13.116821),(32.967114,-13.134184),(32.9713,-13.150927),(32.979051,-13.167877),(32.995071,-13.190873),(32.998171,-13.202449),(32.992797,-13.216401),(32.962721,-13.22498),(32.937865,-13.255882),(32.919623,-13.294846),(32.905487,-13.351391),(32.892131,-13.404814),(32.874251,-13.443158),(32.845829,-13.45804),(32.827432,-13.460831),(32.819061,-13.471683),(32.816891,-13.487393),(32.817511,-13.505583),(32.813066,-13.528217),(32.800044,-13.537105),(32.763199,-13.54248),(32.749814,-13.550851),(32.722839,-13.573382),(32.71023,-13.575346),(32.691627,-13.569145),(32.677261,-13.567801),(32.667649,-13.575553),(32.663308,-13.596327),(32.67509,-13.628056),(32.706716,-13.635084),(32.743407,-13.635394),(32.769865,-13.646866),(32.807795,-13.699163),(32.813377,-13.711152),(32.80485,-13.722004),(32.768108,-13.739057),(32.757359,-13.750219),(32.763044,-13.777194),(32.791879,-13.789803),(32.828776,-13.797038),(32.85942,-13.808097),(32.874148,-13.820602),(32.923912,-13.880444),(32.927891,-13.895636),(32.930062,-13.910519),(32.93616,-13.926229),(32.944428,-13.932947),(32.962515,-13.932017),(32.970576,-13.936047),(32.976261,-13.944522),(32.976674,-13.949793),(32.974814,-13.955478),(32.9713,-14.005914),(32.976467,-14.021107),(32.999825,-14.050459),(33.01817,-14.046222),(33.049434,-13.996612),(33.062974,-13.981729),(33.076926,-13.974495),(33.111601,-13.964573),(33.120645,-13.955891),(33.130256,-13.93057),(33.140282,-13.924989),(33.154544,-13.936564),(33.185654,-13.993822),(33.202707,-14.013872),(33.13005,-14.038057),(33.042923,-14.067099),(32.917143,-14.10906),(32.744957,-14.166421),(32.576802,-14.222542),(32.425493,-14.272978),(32.320383,-14.308014),(32.227366,-14.33902),(32.147061,-14.353386),(32.108768,-14.364859),(32.033269,-14.398758),(31.932913,-14.431108),(31.801345,-14.473586),(31.658408,-14.519578),(31.589679,-14.554821),(31.496661,-14.60257),(31.410878,-14.633576),(31.306802,-14.658174),(31.178024,-14.68856),(31.074465,-14.713054),(31.056585,-14.714398),(30.915094,-14.746024),(30.815772,-14.768245),(30.676969,-14.817647),(30.591807,-14.848136),(30.48556,-14.885964),(30.392491,-14.931025),(30.318438,-14.966992),(30.230898,-14.977224),(30.214465,-14.981462),(30.214421,-14.981966),(30.213845,-14.98849),(30.220977,-15.018359),(30.220977,-15.063007),(30.225834,-15.106312),(30.263145,-15.231266),(30.287949,-15.278911),(30.320505,-15.299478),(30.334871,-15.304749),(30.347481,-15.317875),(30.357919,-15.335652),(30.364947,-15.354152),(30.369185,-15.374719),(30.371768,-15.439832),(30.374921,-15.451304),(30.389648,-15.471148),(30.392852,-15.480759),(30.390165,-15.490785),(30.371768,-15.525408),(30.365257,-15.547422),(30.367428,-15.561271),(30.395953,-15.590623),(30.401224,-15.598478),(30.412696,-15.627934),(30.410371,-15.630414),(30.396263,-15.635995),(30.356679,-15.651498),(30.32805,-15.652428),(30.296196,-15.639042),(30.280094,-15.632275),(30.254876,-15.628864),(30.246091,-15.632068),(30.231002,-15.644677),(30.223147,-15.649741),(30.207231,-15.653152),(30.195552,-15.649121),(30.16992,-15.632171),(30.130129,-15.623696),(30.090029,-15.629381),(30.050238,-15.640129),(30.010654,-15.646227),(29.967504,-15.641473),(29.881773,-15.618839),(29.837331,-15.614808),(29.814283,-15.619666),(29.773252,-15.638062),(29.73005,-15.644677),(29.672793,-15.663281),(29.648505,-15.666588),(29.62799,-15.663591),(29.608559,-15.658423),(29.587217,-15.655736),(29.563446,-15.662144),(29.526239,-15.692839),(29.508462,-15.703588),(29.422059,-15.71103),(29.406969,-15.714233),(29.186311,-15.812832),(29.150964,-15.848799),(29.141869,-15.854483),(29.121716,-15.859341),(29.102182,-15.870916),(29.086162,-15.884559),(29.076344,-15.895411),(29.055053,-15.934375),(29.042341,-15.946261),(29.018053,-15.950602),(28.972784,-15.951428),(28.951287,-15.955252),(28.946862,-15.957235),(28.932373,-15.963727),(28.898887,-15.995457),(28.877183,-16.022018),(28.874082,-16.028943),(28.860336,-16.049407),(28.857236,-16.060466),(28.859303,-16.069561),(28.86385,-16.076589),(28.868501,-16.08217),(28.870981,-16.087234),(28.8654,-16.121237),(28.852481,-16.162785),(28.847107,-16.202679),(28.86416,-16.231205),(28.840286,-16.284741),(28.836772,-16.306342),(28.840492,-16.323602),(28.857029,-16.36546),(28.857236,-16.388198),(28.833051,-16.426438),(28.829124,-16.434603),(28.822509,-16.470776),(28.808866,-16.486279),(28.769282,-16.515218),(28.761117,-16.532271),(28.741377,-16.550668),(28.73285,-16.55811),(28.718794,-16.56028),(28.690734,-16.56028),(28.643295,-16.568755),(28.280113,-16.706524),(28.21252,-16.748589),(28.113922,-16.827551),(28.022993,-16.865393),(27.868562,-16.929663),(27.816886,-16.959636),(27.777301,-17.001183),(27.641186,-17.198484),(27.624856,-17.233314),(27.604495,-17.312792),(27.577314,-17.363125),(27.524294,-17.415112),(27.422078,-17.504822),(27.157081,-17.769302),(27.146952,-17.783875),(27.145299,-17.794107),(27.146539,-17.818911),(27.149019,-17.842476),(27.11543,-17.882163),(27.078171,-17.916993),(27.048457,-17.944278),(27.021275,-17.958541),(27.006289,-17.962675),(26.95916,-17.964742),(26.94867,-17.968876),(26.912031,-17.992027),(26.88826,-17.984586),(26.794002,-18.026237),(26.769714,-18.029028),(26.753591,-18.032955),(26.740569,-18.0405),(26.71194,-18.065821),(26.700003,-18.069232),(26.685689,-18.066751),(26.628844,-18.049181),(26.612721,-18.041223),(26.598872,-18.029958),(26.583369,-18.013215),(26.570243,-18.002879),(26.553604,-17.996471),(26.527145,-17.992027),(26.485494,-17.979315),(26.408599,-17.939007),(26.362711,-17.930636),(26.325504,-17.93601),(26.318269,-17.934356),(26.311965,-17.928362),(26.3038,-17.922781),(26.294291,-17.918543),(26.248299,-17.913376),(26.239204,-17.910172),(26.233933,-17.903971),(26.228249,-17.894669),(26.221117,-17.886297),(26.211919,-17.882783),(26.203031,-17.887227),(26.167477,-17.913582),(26.158589,-17.918337),(26.135438,-17.922574),(26.118591,-17.931566),(26.101228,-17.935803),(26.095234,-17.938077),(26.0942,-17.941901),(26.096164,-17.954614),(26.095234,-17.958541),(26.081178,-17.962365),(26.062471,-17.962882),(26.046554,-17.966292),(26.04056,-17.978488),(26.033739,-17.971563),(25.978548,-17.998952),(25.966973,-18.000502),(25.924495,-17.998952),(25.86362,-17.971563),(25.853491,-17.959988),(25.846153,-17.943658),(25.847497,-17.929395),(25.86362,-17.923814),(25.849667,-17.906658),(25.804399,-17.888158),(25.794683,-17.872655),(25.786002,-17.862216),(25.765951,-17.849814),(25.743834,-17.839375),(25.70642,-17.829867),(25.694224,-17.819428),(25.681409,-17.81147),(25.657017,-17.81395),(25.603997,-17.836171),(25.536818,-17.848677),(25.530927,-17.850951),(25.522142,-17.860149),(25.516458,-17.862319),(25.510153,-17.861183),(25.500748,-17.856015),(25.49558,-17.854878),(25.420288,-17.854878),(25.409539,-17.853018),(25.376466,-17.841235),(25.345254,-17.842579),(25.335538,-17.841235),(25.315901,-17.83214),(25.285412,-17.809299),(25.266705,-17.800928),(25.259781,-17.794107),(25.253476,-17.781497),(25.242417,-17.770335),(25.228878,-17.762481),(25.215132,-17.759277),(25.198182,-17.75845),(25.190741,-17.755349),(25.177822,-17.738813),(25.172448,-17.735092),(25.161182,-17.729408),(25.156841,-17.72517),(25.155291,-17.719382),(25.156428,-17.70667),(25.153327,-17.700986),(25.138548,-17.686103),(25.131416,-17.686516),(25.122631,-17.697885),(25.120048,-17.691064),(25.115087,-17.684242),(25.107955,-17.678868),(25.098757,-17.676801),(25.096897,-17.67215),(25.088525,-17.642695),(25.085218,-17.640938),(25.067028,-17.625331),(25.064237,-17.621507),(25.052558,-17.621404),(25.045324,-17.619957),(25.040053,-17.616133),(25.033851,-17.608485),(25.033851,-17.60156),(25.040569,-17.584507),(25.036952,-17.5812),(25.02672,-17.58275),(25.008013,-17.588538),(24.998505,-17.588021),(24.982588,-17.576549),(24.971116,-17.560736),(24.969987,-17.559962),(24.95799,-17.551744),(24.93763,-17.560736),(24.924917,-17.542959),(24.898252,-17.531074),(24.829523,-17.517741),(24.79738,-17.519085),(24.786838,-17.516914),(24.780017,-17.512263),(24.775056,-17.507612),(24.769785,-17.505442),(24.684415,-17.49242),(24.639457,-17.49242),(24.629742,-17.49552),(24.622817,-17.502341),(24.617443,-17.509163),(24.606487,-17.515157),(24.591294,-17.528386),(24.580752,-17.532831),(24.571244,-17.533451),(24.562459,-17.53221),(24.546543,-17.526526),(24.537861,-17.520325),(24.53166,-17.5134),(24.523598,-17.507819),(24.498173,-17.503478),(24.47926,-17.494487),(24.449288,-17.489112),(24.407016,-17.474539),(24.388929,-17.471336),(24.371256,-17.473713),(24.329398,-17.485081),(24.321337,-17.488699),(24.310278,-17.482601),(24.257361,-17.480844),(24.238758,-17.478157),(24.220464,-17.4795),(24.190905,-17.485288),(24.148117,-17.493763),(24.105329,-17.502238),(24.062541,-17.510713),(24.01965,-17.519085),(23.976965,-17.52756),(23.934177,-17.536034),(23.891389,-17.544406),(23.848497,-17.552881),(23.805606,-17.561356),(23.762818,-17.569727),(23.72003,-17.578202),(23.677242,-17.586677),(23.634557,-17.595049),(23.591769,-17.603524),(23.548774,-17.611999),(23.505986,-17.620474),(23.47622,-17.626365),(23.45741,-17.626778),(23.422373,-17.633496),(23.381652,-17.641144),(23.375968,-17.628225),(23.375141,-17.615409),(23.382273,-17.601043),(23.359432,-17.582853),(23.340621,-17.560736),(23.320158,-17.54637),(23.305378,-17.539548),(23.290805,-17.535414),(23.258352,-17.532831),(23.241299,-17.535104),(23.224246,-17.539342),(23.206986,-17.541202),(23.189726,-17.536551),(23.177531,-17.524046),(23.17691,-17.509783),(23.179081,-17.494177),(23.176187,-17.478157),(23.165748,-17.467408),(23.121513,-17.450872),(23.097639,-17.432165),(23.073041,-17.405086),(23.054127,-17.375321),(23.046376,-17.348449),(23.040381,-17.33708),(22.998627,-17.293775),(22.984157,-17.285817),(22.936512,-17.273311),(22.876154,-17.248093),(22.849179,-17.23104),(22.809388,-17.196417),(22.778278,-17.180397),(22.765153,-17.169649),(22.755851,-17.154869),(22.744792,-17.108257),(22.730736,-17.081592),(22.710479,-17.055444),(22.665624,-17.008625),(22.651671,-16.998703),(22.592037,-16.975449),(22.579324,-16.975345),(22.573433,-16.971004),(22.569196,-16.962426),(22.569299,-16.944959),(22.567232,-16.936898),(22.554519,-16.924185),(22.522894,-16.914677),(22.508527,-16.906202),(22.499742,-16.89349),(22.489097,-16.865378),(22.416647,-16.75448),(22.408895,-16.745798),(22.399594,-16.740424),(22.383574,-16.73753),(22.377993,-16.734843),(22.372618,-16.728849),(22.365797,-16.71686),(22.350088,-16.696292),(22.343473,-16.683167),(22.333654,-16.673658),(22.306989,-16.669214),(22.303062,-16.66694),(22.295621,-16.659189),(22.290763,-16.656398),(22.287352,-16.658259),(22.283735,-16.661359),(22.274123,-16.663633),(22.259034,-16.669214),(22.251282,-16.670041),(22.237743,-16.665493),(22.15165,-16.597694),(22.145139,-16.584051),(22.142452,-16.567101),(22.138111,-16.552632),(22.127259,-16.546431),(22.108138,-16.54395),(22.104004,-16.536716),(22.106588,-16.525967),(22.107312,-16.512324),(22.101731,-16.497958),(22.084574,-16.470156),(22.079923,-16.457754),(22.08075,-16.441011),(22.086021,-16.422304),(22.094289,-16.404734),(22.103591,-16.391918),(22.105555,-16.379413),(22.089122,-16.371971),(22.055842,-16.364633),(22.054188,-16.358639),(22.053672,-16.336521),(22.052018,-16.326806),(22.048607,-16.322362),(22.036618,-16.312233),(22.032174,-16.306342),(22.02773,-16.291666),(22.025973,-16.278954),(22.022356,-16.266345),(22.01171,-16.252289),(22.019875,-16.253115),(22.045197,-16.252289),(22.010367,-16.198132),(21.983805,-16.165886),(21.981531,-16.144285),(21.981531,-16.128162),(21.981531,-16.067494),(21.981531,-16.004035),(21.980601,-16.001244),(21.980808,-15.956079),(21.980911,-15.853967),(21.981221,-15.751957),(21.981428,-15.649948),(21.981531,-15.547835),(21.981531,-15.503807),(21.981531,-15.473525),(21.981531,-15.452337),(21.981531,-15.412546),(21.981428,-15.261858),(21.981325,-15.11117),(21.981221,-14.960481),(21.981118,-14.809793),(21.981014,-14.659104),(21.980911,-14.508416),(21.980808,-14.357624),(21.980704,-14.206935),(21.980601,-14.05635),(21.980498,-13.905558),(21.980394,-13.754973),(21.980353,-13.694822),(21.980291,-13.604181),(21.980188,-13.453493),(21.980084,-13.302701),(21.979981,-13.152168),(21.979878,-13.001479),(22.104211,-13.001479),(22.228545,-13.001479),(22.352775,-13.001479),(22.477005,-13.001479),(22.601235,-13.001479),(22.725465,-13.001479),(22.849799,-13.001479),(22.974029,-13.001479),(23.098362,-13.001479),(23.222696,-13.001479),(23.346926,-13.001479),(23.471259,-13.001479),(23.595593,-13.001479),(23.719823,-13.001479),(23.844157,-13.001479),(23.968283,-13.001479),(24.000633,-13.001479),(23.988851,-12.965099),(23.971797,-12.93337),(23.949576,-12.904638),(23.895109,-12.849757),(23.874749,-12.821749),(23.865654,-12.789709),(23.872269,-12.750125),(23.891699,-12.705063),(23.910716,-12.631269),(23.928699,-12.561609),(23.940791,-12.532774),(23.98606,-12.467662),(24.019856,-12.419189),(24.028021,-12.402136),(24.030812,-12.385083),(24.020993,-12.340021),(24.016859,-12.278939),(24.006627,-12.253721),(23.981306,-12.227676),(23.959602,-12.19667),(23.954331,-12.151919),(23.961049,-12.011669),(23.967043,-11.882891),(23.98854,-11.834212),(23.990194,-11.824083),(23.985543,-11.799485),(23.981306,-11.724761),(23.972727,-11.700577),(23.962289,-11.68156),(23.954641,-11.662233),(23.954641,-11.636911),(23.959912,-11.617171),(23.977688,-11.577277),(24.005387,-11.535212),(24.009728,-11.52343),(24.009831,-11.507203),(24.007247,-11.483122),(24.007557,-11.470513),(24.009935,-11.459454),(24.020787,-11.444572),(24.052102,-11.42049),(24.061714,-11.406951),(24.061094,-11.394962),(24.016963,-11.298431),(24.011175,-11.272903),(24.015309,-11.130482),(23.995775,-11.127382),(23.990091,-11.113532),(23.994535,-11.074982),(23.993708,-11.019585),(23.997325,-11.001705),(24.003733,-10.982481),(24.000013,-10.967805),(23.980582,-10.938349),(23.974278,-10.921399),(23.967457,-10.872307),(24.00146,-10.873444),(24.048382,-10.882332),(24.09148,-10.897835),(24.114218,-10.919229),(24.114321,-10.927911),(24.10874,-10.945791),(24.108016,-10.954989),(24.110497,-10.965945),(24.117525,-10.986098),(24.119385,-10.99757),(24.117732,-11.03085),(24.124243,-11.043252),(24.144397,-11.040359),(24.16455,-11.033951),(24.181294,-11.034467),(24.218294,-11.045216),(24.238034,-11.048007),(24.275138,-11.047903),(24.293845,-11.05028),(24.316376,-11.060099),(24.345315,-11.078289),(24.370533,-11.1002),(24.381798,-11.12056),(24.378284,-11.132136),(24.369913,-11.140921),(24.362471,-11.150533),(24.361748,-11.164692),(24.367432,-11.176268),(24.384382,-11.191254),(24.391307,-11.199832),(24.396578,-11.217402),(24.397818,-11.238279),(24.396061,-11.259673),(24.392237,-11.279104),(24.384485,-11.295227),(24.352446,-11.345043),(24.33994,-11.358065),(24.298289,-11.370571),(24.285473,-11.381216),(24.293948,-11.3992),(24.310071,-11.406641),(24.351722,-11.407055),(24.370223,-11.410155),(24.392547,-11.423384),(24.426447,-11.45408),(24.446497,-11.463589),(24.469648,-11.465552),(24.490939,-11.461522),(24.532693,-11.446122),(24.542512,-11.445398),(24.554191,-11.446329),(24.564733,-11.444675),(24.571244,-11.436407),(24.574448,-11.425141),(24.578582,-11.415633),(24.584576,-11.407365),(24.593258,-11.399613),(24.60225,-11.395686),(24.624677,-11.390828),(24.635219,-11.385867),(24.645038,-11.376049),(24.663331,-11.351657),(24.67408,-11.341632),(24.712527,-11.326026),(24.787975,-11.325819),(24.826732,-11.317551),(24.901043,-11.281274),(24.942901,-11.268562),(24.981245,-11.271869),(25.124285,-11.259157),(25.184023,-11.246651),(25.278798,-11.199935),(25.31032,-11.194768),(25.322516,-11.205413),(25.324583,-11.227841),(25.323343,-11.253472),(25.32541,-11.273936),(25.324893,-11.283651),(25.317142,-11.290473),(25.296264,-11.298431),(25.284689,-11.307319),(25.290166,-11.315897),(25.301329,-11.324476),(25.306496,-11.333054),(25.298021,-11.351451),(25.286342,-11.360649),(25.277971,-11.371811),(25.279004,-11.396306),(25.286652,-11.416563),(25.309287,-11.455424),(25.316728,-11.472994),(25.315385,-11.482502),(25.301742,-11.498418),(25.297711,-11.507927),(25.297505,-11.517229),(25.300398,-11.535005),(25.327373,-11.616654),(25.336779,-11.633501),(25.344943,-11.642182),(25.351971,-11.64611),(25.360446,-11.647247),(25.373159,-11.647143),(25.386388,-11.64983),(25.392796,-11.656445),(25.39755,-11.66492),(25.406335,-11.673085),(25.463593,-11.70006),(25.481163,-11.715253),(25.482816,-11.72042),(25.483281,-11.725795),(25.482816,-11.731169),(25.474135,-11.756181),(25.480439,-11.769823),(25.49558,-11.776541),(25.515217,-11.775818),(25.524519,-11.77096),(25.530617,-11.764242),(25.537645,-11.758351),(25.550254,-11.756594),(25.561003,-11.753287),(25.56493,-11.743985),(25.567824,-11.733753),(25.575679,-11.727345),(25.621464,-11.728895),(25.655777,-11.750496),(25.687403,-11.778711),(25.725644,-11.800519),(25.744041,-11.803826),(25.801608,-11.80424),(25.812667,-11.802999),(25.832511,-11.793801),(25.839849,-11.791941),(25.850701,-11.793801),(25.855765,-11.797212),(25.859589,-11.801966),(25.914159,-11.845581),(25.935553,-11.853642),(25.949816,-11.863978),(25.963562,-11.895087),(25.981132,-11.903045),(25.994051,-11.904802),(26.16665,-11.902425),(26.189078,-11.9109),(26.209335,-11.923612),(26.229592,-11.933121),(26.265559,-11.930847),(26.28623,-11.940562),(26.298012,-11.941182),(26.309174,-11.936531),(26.328811,-11.923302),(26.339146,-11.918961),(26.376353,-11.912863),(26.414594,-11.91183),(26.438365,-11.919065),(26.455418,-11.931984),(26.470715,-11.94697),(26.489111,-11.960819),(26.524871,-11.972291),(26.6059,-11.977769),(26.643521,-11.985521),(26.662744,-11.996579),(26.679797,-12.009602),(26.697057,-12.017457),(26.717418,-12.013012),(26.732559,-11.998336),(26.744186,-11.98118),(26.759379,-11.968674),(26.785011,-11.968364),(26.831726,-11.973325),(26.874204,-11.964333),(26.912755,-11.942526),(26.947688,-11.909143),(26.967325,-11.870385),(26.971356,-11.78853),(26.981071,-11.749049),(26.996367,-11.727035),(27.011147,-11.713186),(27.021482,-11.69789),(27.023239,-11.671121),(27.010837,-11.62885),(27.010268,-11.609833),(27.023756,-11.59557),(27.032437,-11.59433),(27.054452,-11.597534),(27.064373,-11.59712),(27.102201,-11.584925),(27.144989,-11.581307),(27.173617,-11.570559),(27.180129,-11.569629),(27.197182,-11.59371),(27.20762,-11.639392),(27.211961,-11.688278),(27.209998,-11.721971),(27.234182,-11.809097),(27.420734,-11.921959),(27.467553,-12.001644),(27.47179,-12.041641),(27.52047,-12.179617),(27.536076,-12.200081),(27.589836,-12.242958),(27.595297,-12.247313),(27.638189,-12.293615),(27.666197,-12.30271),(27.710742,-12.306328),(27.756734,-12.304261),(27.788309,-12.296509),(27.807274,-12.28142),(27.818281,-12.267777),(27.831407,-12.259922),(27.856676,-12.262403),(27.877037,-12.269948),(27.933881,-12.303331),(27.948454,-12.321417),(27.952226,-12.346532),(27.958686,-12.367823),(27.98101,-12.374541),(27.998993,-12.37144),(28.014031,-12.37051),(28.08767,-12.377538),(28.096869,-12.383016),(28.100434,-12.391491),(28.10245,-12.401206),(28.106997,-12.410197),(28.131285,-12.429214),(28.145961,-12.423117),(28.160741,-12.408751),(28.185752,-12.402446),(28.204976,-12.410818),(28.220065,-12.42539),(28.237532,-12.437069),(28.263887,-12.436966),(28.283834,-12.434072),(28.303678,-12.435416),(28.322488,-12.4411),(28.339748,-12.450505),(28.42274,-12.521302),(28.436434,-12.542799),(28.455916,-12.592098),(28.468267,-12.614629),(28.502012,-12.649563),(28.512295,-12.668373),(28.511624,-12.694418),(28.500151,-12.710954),(28.483977,-12.72222),(28.474003,-12.736482),(28.480928,-12.762114),(28.501598,-12.788469),(28.523613,-12.810173),(28.539425,-12.833841),(28.542836,-12.885621),(28.553378,-12.894922),(28.568468,-12.895543),(28.583971,-12.888721),(28.593686,-12.876732),(28.608155,-12.847897),(28.621178,-12.839422),(28.667893,-12.849344),(28.707994,-12.889755),(28.77569,-12.976571),(28.793363,-12.991557),(28.800598,-13.002409),(28.809435,-13.038376),(28.813621,-13.04282),(28.827057,-13.047575),(28.830881,-13.051812),(28.83026,-13.056876),(28.825506,-13.067832),(28.8253,-13.072379),(28.826023,-13.076823),(28.824628,-13.087262),(28.825816,-13.092843),(28.829744,-13.097184),(28.835428,-13.098941),(28.840906,-13.100078),(28.844213,-13.102352),(28.85367,-13.137078),(28.86323,-13.142659),(28.882712,-13.143589),(28.891859,-13.14731),(28.903434,-13.162296),(28.93413,-13.309626),(28.950356,-13.350967),(28.980846,-13.380216),(28.993971,-13.395925),(29.010301,-13.399439),(29.028491,-13.395512),(29.046681,-13.388897),(29.065386,-13.388676),(29.072933,-13.388587),(29.098048,-13.38466),(29.122129,-13.38435),(29.144453,-13.394478),(29.153393,-13.406984),(29.160163,-13.422074),(29.168948,-13.433856),(29.184141,-13.436853),(29.194786,-13.430962),(29.21401,-13.408328),(29.227446,-13.400783),(29.250183,-13.391585),(29.308267,-13.353654),(29.449551,-13.300324),(29.470015,-13.287818),(29.527996,-13.234075),(29.551043,-13.224153),(29.574401,-13.225393),(29.592798,-13.235625),(29.609851,-13.249474),(29.629075,-13.261463),(29.641064,-13.262807),(29.649745,-13.259913),(29.657497,-13.26012),(29.666592,-13.271178),(29.669176,-13.281204),(29.667522,-13.291952),(29.663595,-13.301874),(29.658892,-13.309522),(29.651502,-13.314587),(29.633622,-13.318928),(29.626594,-13.326369),(29.625044,-13.334844),(29.627008,-13.354894),(29.625147,-13.364196),(29.618946,-13.367297),(29.609438,-13.369054),(29.601686,-13.374428),(29.601686,-13.388381),(29.60551,-13.399956),(29.610678,-13.411015),(29.617396,-13.421247),(29.626078,-13.429825),(29.641477,-13.438507),(29.663078,-13.446878),(29.702869,-13.457627),(29.767154,-13.458351),(29.782244,-13.455663),(29.797643,-13.424244),(29.797747,-13.31376),(29.797953,-13.164363),(29.798212,-13.02277),(29.79847,-12.873012),(29.798677,-12.698345),(29.798987,-12.54869),(29.799142,-12.382292),(29.7994,-12.265813),(29.799607,-12.159463),(29.799582,-12.159036),(29.799397,-12.15582),(29.799297,-12.154089),(29.756095,-12.157603),(29.734908,-12.164838),(29.701267,-12.189022),(29.648918,-12.212793),(29.631865,-12.215997),(29.621117,-12.21021),(29.603185,-12.192846),(29.590317,-12.188092),(29.576727,-12.188092),(29.56758,-12.191709),(29.553059,-12.205145),(29.533473,-12.216721),(29.489548,-12.2315),(29.474252,-12.242662),(29.467948,-12.255272),(29.455907,-12.294339),(29.451825,-12.3178),(29.448156,-12.326792),(29.446554,-12.33506),(29.450378,-12.342915),(29.480453,-12.380535),(29.49606,-12.386323),(29.511459,-12.387356),(29.523448,-12.392524),(29.528202,-12.410508),(29.524688,-12.429214),(29.514663,-12.44389),(29.499677,-12.453089),(29.480453,-12.454949),(29.473012,-12.452469),(29.468774,-12.448438),(29.46526,-12.444201),(29.459886,-12.441307),(29.453323,-12.44079),(29.439216,-12.44203),(29.432601,-12.441307),(29.405936,-12.431695),(29.364388,-12.409267),(29.356533,-12.407614),(29.316432,-12.406683),(29.304857,-12.404823),(29.295503,-12.400999),(29.283049,-12.390457),(29.265066,-12.370923),(29.254989,-12.366169),(29.173392,-12.367616),(29.148949,-12.373714),(29.128123,-12.382809),(29.11107,-12.393454),(29.100218,-12.374024),(29.084405,-12.376401),(29.067249,-12.384773),(29.052779,-12.383842),(29.042651,-12.378778),(29.035106,-12.378882),(29.030352,-12.376194),(29.028595,-12.363069),(29.026114,-12.353043),(28.944672,-12.206076),(28.918731,-12.174346),(28.859509,-12.14179),(28.850414,-12.130008),(28.84597,-12.111301),(28.835015,-12.092181),(28.821475,-12.075748),(28.808866,-12.064585),(28.773623,-12.045879),(28.759877,-12.035233),(28.754193,-12.020144),(28.751092,-11.992238),(28.741997,-11.98428),(28.707064,-11.989448),(28.686032,-11.984074),(28.636887,-11.957615),(28.623865,-11.947797),(28.606191,-11.91028),(28.592652,-11.900461),(28.569191,-11.91369),(28.566091,-11.904492),(28.554205,-11.885062),(28.549451,-11.879584),(28.540562,-11.872039),(28.531571,-11.866458),(28.521132,-11.865735),(28.507851,-11.872763),(28.497154,-11.857363),(28.488886,-11.808684),(28.476794,-11.798245),(28.460671,-11.793388),(28.452506,-11.780779),(28.442687,-11.711635),(28.439587,-11.701507),(28.425117,-11.688071),(28.418399,-11.678873),(28.42212,-11.674738),(28.42274,-11.664713),(28.40548,-11.605182),(28.399589,-11.596087),(28.384913,-11.581307),(28.37742,-11.571076),(28.365586,-11.54162),(28.360315,-11.533558),(28.355664,-11.518159),(28.354424,-11.464725),(28.357008,-11.446949),(28.408581,-11.372431),(28.411991,-11.368401),(28.434212,-11.351244),(28.439587,-11.348247),(28.43199,-11.31104),(28.436176,-11.289129),(28.445581,-11.274556),(28.45509,-11.263704),(28.473693,-11.22567),(28.472453,-11.191977),(28.473693,-11.181229),(28.476225,-11.175441),(28.484752,-11.162832),(28.487336,-11.153427),(28.486922,-11.140921),(28.476174,-11.087694),(28.476794,-11.077979),(28.490953,-11.058652),(28.504079,-11.034364),(28.507851,-11.023615),(28.508626,-11.013487),(28.506663,-10.992093),(28.507851,-10.982068),(28.513794,-10.972869),(28.531364,-10.955092),(28.535085,-10.944447),(28.537358,-10.934112),(28.547177,-10.913751),(28.549451,-10.902899),(28.549451,-10.858561),(28.554412,-10.834996),(28.563507,-10.811949),(28.576529,-10.790658),(28.593479,-10.772881),(28.60092,-10.763166),(28.608052,-10.741359),(28.623245,-10.716864),(28.627585,-10.711696),(28.633477,-10.708906),(28.646189,-10.709836),(28.651873,-10.708389),(28.665878,-10.697434),(28.681536,-10.681931),(28.694351,-10.663327),(28.699622,-10.643174),(28.694248,-10.636766),(28.67089,-10.619092),(28.665516,-10.608964),(28.66717,-10.597388),(28.672699,-10.576098),(28.672337,-10.564315),(28.648049,-10.53765),(28.634924,-10.51946),(28.634717,-10.505714),(28.638748,-10.496206),(28.630479,-10.481013),(28.634717,-10.471608),(28.640608,-10.462616),(28.644535,-10.451041),(28.646086,-10.438431),(28.644949,-10.426546),(28.637714,-10.407012),(28.628619,-10.391096),(28.620868,-10.373319),(28.617767,-10.348308),(28.620713,-10.327327),(28.626449,-10.311411),(28.629239,-10.295908),(28.623865,-10.276374),(28.622108,-10.275857),(28.613323,-10.27069),(28.610842,-10.268829),(28.608052,-10.263558),(28.605985,-10.252913),(28.604021,-10.248366),(28.569915,-10.219633),(28.57932,-10.201133),(28.604021,-10.172711),(28.619627,-10.141912),(28.628516,-10.102741),(28.631255,-10.023056),(28.623865,-9.951019),(28.627947,-9.929419),(28.658695,-9.864823),(28.668462,-9.821622),(28.678073,-9.803845),(28.696522,-9.796507),(28.698486,-9.791959),(28.672337,-9.765501),(28.66996,-9.750928),(28.663656,-9.745037),(28.654974,-9.741937),(28.644949,-9.735115),(28.631255,-9.715582),(28.625157,-9.695634),(28.623865,-9.649126),(28.621798,-9.638274),(28.612909,-9.6175),(28.610842,-9.607888),(28.606502,-9.594349),(28.587588,-9.573161),(28.583557,-9.563756),(28.588621,-9.558279),(28.611101,-9.551044),(28.617767,-9.543292),(28.586296,-9.544016),(28.5741,-9.539158),(28.569191,-9.525929),(28.568726,-9.508979),(28.566866,-9.495957),(28.562783,-9.484898),(28.5556,-9.474459),(28.531571,-9.456889),(28.519685,-9.445624),(28.514724,-9.430018),(28.516274,-9.415548),(28.518135,-9.397461),(28.515964,-9.378341),(28.495397,-9.353226),(28.460464,-9.335036),(28.411371,-9.324908),(28.368532,-9.308578),(28.35246,-9.271991),(28.372304,-9.235094),(28.415092,-9.207085),(28.506198,-9.164401),(28.59906,-9.096395),(28.765407,-8.934027),(28.893099,-8.765769),(28.930461,-8.679883),(28.935474,-8.590792),(28.892789,-8.501909),(28.889172,-8.483099),(28.915268,-8.472867),(29.035106,-8.45478),(29.208119,-8.428528),(29.365938,-8.40455),(29.575844,-8.372729),(29.592281,-8.370237),(29.858105,-8.329826),(30.123618,-8.289519),(30.32495,-8.258926),(30.581781,-8.219962),(30.752107,-8.194124),(30.778255,-8.289105),(30.828278,-8.388117),(30.891892,-8.479171),(30.959536,-8.550485),(30.992351,-8.57591),(31.03364,-8.600301),(31.077824,-8.61632),(31.119836,-8.616631)] +Zimbabwe [(30.010654,-15.646227),(30.050238,-15.640129),(30.090029,-15.629381),(30.130129,-15.623696),(30.16992,-15.632171),(30.195552,-15.649121),(30.207231,-15.653152),(30.223147,-15.649741),(30.231002,-15.644677),(30.246091,-15.632068),(30.254876,-15.628864),(30.280094,-15.632275),(30.296196,-15.639042),(30.32805,-15.652428),(30.356679,-15.651498),(30.396263,-15.635995),(30.39771,-15.716817),(30.39926,-15.812005),(30.401327,-15.931688),(30.402568,-16.001244),(30.514809,-16.000418),(30.586587,-16.000004),(30.74973,-15.998867),(30.857424,-15.998144),(30.901865,-16.007136),(30.942173,-16.034524),(30.958296,-16.05106),(30.973075,-16.062016),(30.989767,-16.06429),(31.012039,-16.054885),(31.023718,-16.045169),(31.042218,-16.024912),(31.056895,-16.017574),(31.065421,-16.019641),(31.073328,-16.025532),(31.080872,-16.025946),(31.089037,-16.01189),(31.1141,-15.996904),(31.15849,-16.000211),(31.259983,-16.023465),(31.278897,-16.030287),(31.29533,-16.041655),(31.309592,-16.059019),(31.328351,-16.092815),(31.340908,-16.106664),(31.360339,-16.116896),(31.37026,-16.123718),(31.374601,-16.132916),(31.377754,-16.142218),(31.384006,-16.148832),(31.387727,-16.149556),(31.395582,-16.147695),(31.399613,-16.147282),(31.404315,-16.149866),(31.404057,-16.154517),(31.402713,-16.159374),(31.404574,-16.162268),(31.424107,-16.164749),(31.445708,-16.164955),(31.465655,-16.167746),(31.480641,-16.177978),(31.519192,-16.196478),(31.686107,-16.207227),(31.710705,-16.217872),(31.738197,-16.239783),(31.798761,-16.303655),(31.818088,-16.319571),(31.86005,-16.340759),(31.871935,-16.35037),(31.88072,-16.368044),(31.88563,-16.406284),(31.894363,-16.421477),(31.910279,-16.428919),(32.014149,-16.444938),(32.211759,-16.440184),(32.290463,-16.45176),(32.393661,-16.491757),(32.5521,-16.553355),(32.671783,-16.599761),(32.6831,-16.609889),(32.687906,-16.624255),(32.68863,-16.647303),(32.698655,-16.686784),(32.725217,-16.706421),(32.73095,-16.708656),(32.731314,-16.708798),(32.739893,-16.703217),(32.753845,-16.697946),(32.769348,-16.695466),(32.800664,-16.697326),(32.862004,-16.710452),(32.893372,-16.712415),(32.909598,-16.708075),(32.93957,-16.689781),(32.95621,-16.683063),(32.968509,-16.681616),(32.961585,-16.710348),(32.933369,-16.815768),(32.916213,-16.847911),(32.900503,-16.867755),(32.828776,-16.935141),(32.83012,-16.941549),(32.886757,-17.038184),(32.928512,-17.109497),(32.954143,-17.167168),(32.967786,-17.22887),(32.96909,-17.266115),(32.969439,-17.276102),(32.973212,-17.297909),(32.983599,-17.317753),(32.992384,-17.324678),(33.014656,-17.336667),(33.021633,-17.345555),(33.022459,-17.361471),(33.016258,-17.377181),(33.011651,-17.383991),(32.997448,-17.404983),(32.958174,-17.478467),(32.951663,-17.486218),(32.942981,-17.491593),(32.936573,-17.498311),(32.936676,-17.509369),(32.947218,-17.543166),(32.951663,-17.551434),(32.969129,-17.56456),(33.006646,-17.580993),(33.020392,-17.598563),(33.024526,-17.619233),(33.020599,-17.638457),(33.004063,-17.675561),(33.000238,-17.713905),(33.003184,-17.757726),(32.999102,-17.794313),(32.973573,-17.810643),(32.957037,-17.817981),(32.946082,-17.834724),(32.939674,-17.855498),(32.936883,-17.875032),(32.938433,-17.894566),(32.950267,-17.922574),(32.952128,-17.940247),(32.948149,-17.95327),(32.940397,-17.959988),(32.932439,-17.964949),(32.927375,-17.972907),(32.928977,-17.982312),(32.941224,-17.996265),(32.940294,-18.004843),(32.934919,-18.024583),(32.93709,-18.047114),(32.972282,-18.150261),(32.975537,-18.183333),(32.974865,-18.190775),(32.965925,-18.212169),(32.958174,-18.225398),(32.952283,-18.233046),(32.950526,-18.241314),(32.95497,-18.256301),(32.970163,-18.277488),(33.016878,-18.313661),(33.034965,-18.332885),(33.042768,-18.352005),(33.038066,-18.363064),(33.00923,-18.383941),(32.988198,-18.41319),(32.985356,-18.412467),(32.986803,-18.422285),(32.999515,-18.436651),(33.003029,-18.446883),(32.996414,-18.46714),(32.978586,-18.48006),(32.956624,-18.489878),(32.937142,-18.50104),(32.919313,-18.510032),(32.900296,-18.515303),(32.88314,-18.522124),(32.870737,-18.535767),(32.868257,-18.552613),(32.871668,-18.57318),(32.884483,-18.609044),(32.914559,-18.665888),(32.92231,-18.693173),(32.920243,-18.726246),(32.913267,-18.753014),(32.902518,-18.774512),(32.885207,-18.787844),(32.858852,-18.790015),(32.817924,-18.787018),(32.787642,-18.791255),(32.69142,-18.83425),(32.68987,-18.843241),(32.696794,-18.897192),(32.703202,-18.911868),(32.71576,-18.919826),(32.705063,-18.927474),(32.692247,-18.934295),(32.682532,-18.942667),(32.681085,-18.954966),(32.68863,-18.97729),(32.690283,-18.988246),(32.68863,-19.000958),(32.691058,-19.01429),(32.698965,-19.022249),(32.710282,-19.025969),(32.723873,-19.026589),(32.785988,-19.017701),(32.803351,-19.019561),(32.814203,-19.023799),(32.819991,-19.028346),(32.822988,-19.035168),(32.825262,-19.046847),(32.830223,-19.059146),(32.83813,-19.066897),(32.847483,-19.073925),(32.855906,-19.083744),(32.862262,-19.118057),(32.83322,-19.241977),(32.832187,-19.266678),(32.828673,-19.284558),(32.820715,-19.301301),(32.806142,-19.323419),(32.768831,-19.363623),(32.766454,-19.373442),(32.768521,-19.402794),(32.762217,-19.443412),(32.763354,-19.463979),(32.773947,-19.475864),(32.793119,-19.476691),(32.811309,-19.474521),(32.825365,-19.479172),(32.832187,-19.500876),(32.832497,-19.519273),(32.825365,-19.59162),(32.825675,-19.600818),(32.828156,-19.610636),(32.829603,-19.623659),(32.825365,-19.633271),(32.819474,-19.641952),(32.81627,-19.652081),(32.819629,-19.674302),(32.83105,-19.685154),(32.849137,-19.689081),(32.872184,-19.690218),(32.894715,-19.684327),(32.924584,-19.655285),(32.943188,-19.64929),(32.960964,-19.658799),(32.962411,-19.679056),(32.954143,-19.717813),(32.962411,-19.735383),(32.979051,-19.751403),(33.0006,-19.764322),(33.022769,-19.773107),(33.032795,-19.784166),(33.029642,-19.80339),(33.022873,-19.826851),(33.021322,-19.868088),(33.001995,-19.927),(32.998378,-20.000897),(33.004373,-20.024255),(33.007266,-20.032006),(32.95373,-20.030249),(32.940087,-20.041515),(32.934299,-20.072107),(32.926548,-20.086473),(32.910683,-20.091124),(32.894405,-20.094018),(32.88531,-20.10301),(32.877869,-20.151689),(32.872908,-20.167192),(32.859265,-20.190859),(32.857095,-20.200575),(32.858335,-20.207499),(32.865053,-20.220935),(32.86557,-20.228893),(32.858438,-20.259486),(32.852961,-20.273852),(32.845209,-20.286668),(32.800767,-20.338551),(32.735862,-20.414205),(32.704443,-20.471773),(32.671783,-20.531821),(32.646462,-20.557969),(32.603674,-20.56479),(32.556545,-20.559312),(32.513136,-20.564583),(32.481614,-20.603031),(32.471072,-20.645509),(32.469108,-20.68685),(32.483474,-20.794233),(32.49722,-20.898103),(32.491019,-20.936344),(32.467661,-20.980165),(32.417122,-21.040937),(32.339814,-21.134058),(32.345343,-21.142843),(32.359864,-21.151421),(32.368856,-21.162997),(32.373352,-21.163617),(32.377744,-21.16341),(32.380638,-21.165477),(32.380535,-21.172195),(32.376866,-21.178499),(32.37299,-21.183977),(32.37175,-21.187905),(32.444613,-21.304693),(32.445849,-21.308994),(32.447197,-21.313685),(32.408543,-21.290327),(32.37299,-21.327948),(32.324517,-21.378177),(32.272221,-21.432541),(32.219718,-21.486904),(32.167318,-21.541268),(32.114814,-21.595632),(32.062415,-21.649995),(32.010015,-21.704462),(31.957615,-21.758826),(31.905215,-21.813189),(31.852712,-21.867553),(31.800312,-21.92202),(31.747808,-21.976384),(31.695512,-22.030747),(31.643112,-22.085214),(31.590712,-22.139578),(31.538209,-22.193941),(31.485809,-22.248305),(31.433822,-22.302048),(31.36871,-22.345043),(31.288922,-22.39734),(31.265616,-22.365507),(31.255642,-22.357962),(31.24572,-22.357549),(31.229597,-22.363957),(31.221536,-22.364887),(31.213474,-22.36189),(31.197868,-22.352588),(31.190685,-22.350624),(31.183657,-22.34556),(31.163348,-22.322616),(31.152599,-22.316414),(31.137717,-22.318482),(31.10454,-22.333364),(31.097048,-22.334922),(31.087642,-22.336878),(31.07033,-22.333674),(31.036121,-22.319618),(30.927187,-22.295744),(30.867087,-22.289646),(30.83789,-22.282308),(30.805282,-22.294504),(30.693919,-22.302772),(30.674282,-22.30856),(30.647411,-22.32644),(30.632424,-22.330677),(30.625551,-22.32861),(30.610307,-22.318688),(30.601108,-22.316414),(30.57217,-22.316621),(30.507367,-22.309593),(30.488454,-22.310213),(30.46923,-22.315071),(30.431713,-22.331194),(30.412696,-22.336878),(30.372078,-22.343493),(30.334975,-22.344733),(30.300765,-22.336982),(30.269346,-22.316414),(30.25529,-22.304736),(30.240407,-22.296157),(30.2217,-22.290886),(30.196999,-22.289129),(30.15266,-22.294814),(30.13509,-22.293574),(30.111113,-22.282308),(30.082587,-22.262878),(30.067911,-22.25709),(30.038145,-22.253783),(30.035872,-22.250579),(30.034528,-22.246135),(30.015511,-22.227014),(30.005279,-22.22226),(29.983782,-22.217713),(29.973963,-22.213992),(29.946678,-22.198282),(29.932105,-22.194355),(29.896035,-22.191358),(29.871489,-22.179265),(29.837331,-22.172444),(29.779246,-22.136374),(29.758886,-22.130896),(29.691448,-22.1341),(29.679614,-22.138338),(29.661424,-22.126452),(29.641064,-22.129242),(29.60396,-22.145055),(29.570164,-22.141955),(29.551043,-22.145986),(29.542517,-22.162522),(29.53182,-22.172444),(29.506912,-22.170067),(29.456889,-22.158801),(29.436115,-22.163142),(29.399528,-22.182159),(29.378031,-22.192908),(29.363251,-22.192288),(29.356947,-22.190944),(29.350074,-22.186707),(29.273644,-22.125108),(29.26734,-22.115807),(29.259588,-22.096066),(29.254111,-22.087074),(29.244395,-22.075706),(29.239331,-22.072605),(29.144867,-22.075292),(29.10797,-22.069194),(29.070763,-22.051004),(29.040532,-22.020929),(29.021567,-21.982791),(29.013815,-21.940417),(29.017949,-21.898145),(29.028905,-21.876648),(29.045441,-21.852567),(29.057637,-21.829209),(29.05526,-21.809985),(29.038723,-21.797893),(28.998726,-21.786008),(28.980846,-21.774845),(28.951907,-21.768334),(28.891032,-21.764924),(28.860853,-21.757379),(28.714195,-21.693507),(28.66841,-21.679968),(28.629704,-21.651339),(28.6157,-21.647101),(28.585934,-21.644414),(28.553998,-21.636559),(28.542939,-21.638316),(28.532501,-21.643071),(28.497309,-21.651546),(28.481393,-21.657437),(28.464598,-21.660331),(28.443101,-21.655783),(28.361762,-21.616302),(28.321919,-21.603486),(28.284867,-21.596872),(28.165702,-21.595218),(28.090771,-21.581266),(28.032893,-21.577855),(28.016563,-21.572894),(28.002559,-21.564212),(27.990415,-21.551913),(27.984731,-21.542922),(27.975739,-21.522561),(27.970571,-21.514396),(27.963698,-21.510469),(27.958066,-21.511502),(27.953208,-21.510469),(27.949281,-21.500754),(27.954448,-21.487835),(27.950418,-21.482047),(27.943338,-21.479876),(27.939876,-21.478016),(27.941943,-21.468508),(27.949642,-21.456519),(27.953001,-21.448664),(27.950211,-21.438329),(27.920549,-21.381174),(27.904219,-21.364741),(27.897811,-21.35544),(27.896157,-21.347895),(27.896674,-21.332392),(27.8944,-21.32433),(27.884995,-21.310171),(27.849132,-21.269657),(27.823604,-21.231726),(27.793838,-21.197413),(27.724385,-21.149664),(27.709192,-21.134471),(27.674775,-21.090133),(27.666611,-21.071219),(27.666817,-21.053753),(27.678961,-21.000733),(27.680356,-20.979649),(27.672657,-20.923528),(27.672605,-20.913709),(27.675085,-20.891282),(27.674775,-20.879913),(27.676016,-20.866684),(27.681803,-20.857589),(27.689038,-20.849011),(27.694412,-20.837745),(27.709605,-20.756716),(27.707332,-20.716719),(27.682475,-20.637344),(27.690382,-20.60148),(27.702629,-20.566134),(27.705575,-20.526653),(27.698133,-20.509083),(27.683767,-20.49606),(27.66599,-20.489136),(27.625786,-20.488619),(27.590853,-20.473323),(27.534112,-20.483038),(27.45391,-20.473323),(27.340739,-20.473013),(27.306012,-20.477354),(27.268392,-20.49575),(27.283998,-20.35147),(27.266015,-20.234164),(27.214907,-20.110451),(27.201781,-20.092984),(27.183746,-20.082339),(27.16292,-20.076551),(27.141888,-20.073347),(27.129692,-20.072934),(27.119771,-20.073864),(27.109642,-20.073244),(27.097343,-20.068903),(27.086491,-20.060532),(27.069231,-20.03738),(27.060136,-20.027562),(27.02665,-20.010095),(26.9943,-20.006788),(26.961072,-20.007201),(26.925054,-20.000897),(26.811882,-19.94643),(26.774469,-19.939815),(26.750801,-19.939609),(26.730957,-19.935888),(26.713904,-19.927413),(26.698608,-19.91253),(26.684758,-19.894547),(26.67717,-19.886815),(26.673803,-19.883385),(26.659437,-19.875737),(26.614065,-19.863438),(26.595565,-19.855583),(26.581922,-19.842147),(26.574791,-19.819513),(26.566316,-19.800806),(26.549263,-19.784063),(26.508852,-19.759258),(26.489731,-19.75192),(26.450251,-19.743342),(26.431854,-19.73652),(26.412837,-19.71957),(26.385242,-19.679056),(26.362711,-19.667584),(26.332325,-19.662416),(26.324367,-19.659109),(26.312171,-19.651358),(26.312481,-19.649601),(26.319096,-19.646293),(26.326331,-19.633891),(26.333462,-19.613014),(26.330981,-19.604952),(26.32106,-19.592033),(26.313205,-19.584178),(26.30349,-19.577254),(26.292638,-19.572499),(26.239101,-19.571466),(26.194452,-19.5602),(26.155488,-19.537153),(26.13027,-19.501082),(26.034359,-19.243734),(26.011414,-19.199809),(25.981132,-19.161775),(25.956534,-19.122088),(25.948576,-19.103277),(25.944855,-19.079196),(25.948059,-19.058732),(25.964389,-19.021629),(25.9678,-19.000958),(25.967449,-18.999925),(25.940721,-18.921273),(25.815251,-18.813993),(25.779491,-18.738752),(25.773393,-18.665578),(25.761921,-18.630335),(25.736909,-18.608734),(25.698255,-18.590234),(25.669523,-18.566049),(25.622084,-18.501143),(25.608442,-18.487708),(25.574439,-18.465693),(25.508499,-18.399134),(25.49558,-18.378877),(25.490516,-18.365545),(25.481163,-18.323377),(25.473204,-18.303429),(25.440855,-18.2532),(25.408816,-18.175995),(25.387525,-18.138995),(25.357449,-18.115844),(25.323446,-18.09662),(25.296368,-18.068612),(25.255026,-18.001122),(25.226088,-17.931876),(25.21937,-17.908001),(25.21937,-17.879786),(25.259781,-17.794107),(25.266705,-17.800928),(25.285412,-17.809299),(25.315901,-17.83214),(25.335538,-17.841235),(25.345254,-17.842579),(25.376466,-17.841235),(25.409539,-17.853018),(25.420288,-17.854878),(25.49558,-17.854878),(25.500748,-17.856015),(25.510153,-17.861183),(25.516458,-17.862319),(25.522142,-17.860149),(25.530927,-17.850951),(25.536818,-17.848677),(25.603997,-17.836171),(25.657017,-17.81395),(25.681409,-17.81147),(25.694224,-17.819428),(25.70642,-17.829867),(25.743834,-17.839375),(25.765951,-17.849814),(25.786002,-17.862216),(25.794683,-17.872655),(25.804399,-17.888158),(25.849667,-17.906658),(25.86362,-17.923814),(25.847497,-17.929395),(25.846153,-17.943658),(25.853491,-17.959988),(25.86362,-17.971563),(25.924495,-17.998952),(25.966973,-18.000502),(25.978548,-17.998952),(26.033739,-17.971563),(26.04056,-17.978488),(26.046554,-17.966292),(26.062471,-17.962882),(26.081178,-17.962365),(26.095234,-17.958541),(26.096164,-17.954614),(26.0942,-17.941901),(26.095234,-17.938077),(26.101228,-17.935803),(26.118591,-17.931566),(26.135438,-17.922574),(26.158589,-17.918337),(26.167477,-17.913582),(26.203031,-17.887227),(26.211919,-17.882783),(26.221117,-17.886297),(26.228249,-17.894669),(26.233933,-17.903971),(26.239204,-17.910172),(26.248299,-17.913376),(26.294291,-17.918543),(26.3038,-17.922781),(26.311965,-17.928362),(26.318269,-17.934356),(26.325504,-17.93601),(26.362711,-17.930636),(26.408599,-17.939007),(26.485494,-17.979315),(26.527145,-17.992027),(26.553604,-17.996471),(26.570243,-18.002879),(26.583369,-18.013215),(26.598872,-18.029958),(26.612721,-18.041223),(26.628844,-18.049181),(26.685689,-18.066751),(26.700003,-18.069232),(26.71194,-18.065821),(26.740569,-18.0405),(26.753591,-18.032955),(26.769714,-18.029028),(26.794002,-18.026237),(26.88826,-17.984586),(26.912031,-17.992027),(26.94867,-17.968876),(26.95916,-17.964742),(27.006289,-17.962675),(27.021275,-17.958541),(27.048457,-17.944278),(27.078171,-17.916993),(27.11543,-17.882163),(27.149019,-17.842476),(27.146539,-17.818911),(27.145299,-17.794107),(27.146952,-17.783875),(27.157081,-17.769302),(27.422078,-17.504822),(27.524294,-17.415112),(27.577314,-17.363125),(27.604495,-17.312792),(27.624856,-17.233314),(27.641186,-17.198484),(27.777301,-17.001183),(27.816886,-16.959636),(27.868562,-16.929663),(28.022993,-16.865393),(28.113922,-16.827551),(28.21252,-16.748589),(28.280113,-16.706524),(28.643295,-16.568755),(28.690734,-16.56028),(28.718794,-16.56028),(28.73285,-16.55811),(28.741377,-16.550668),(28.761117,-16.532271),(28.769282,-16.515218),(28.808866,-16.486279),(28.822509,-16.470776),(28.829124,-16.434603),(28.833051,-16.426438),(28.857236,-16.388198),(28.857029,-16.36546),(28.840492,-16.323602),(28.836772,-16.306342),(28.840286,-16.284741),(28.86416,-16.231205),(28.847107,-16.202679),(28.852481,-16.162785),(28.8654,-16.121237),(28.870981,-16.087234),(28.868501,-16.08217),(28.86385,-16.076589),(28.859303,-16.069561),(28.857236,-16.060466),(28.860336,-16.049407),(28.874082,-16.028943),(28.877183,-16.022018),(28.898887,-15.995457),(28.932373,-15.963727),(28.946862,-15.957235),(28.951287,-15.955252),(28.972784,-15.951428),(29.018053,-15.950602),(29.042341,-15.946261),(29.055053,-15.934375),(29.076344,-15.895411),(29.086162,-15.884559),(29.102182,-15.870916),(29.121716,-15.859341),(29.141869,-15.854483),(29.150964,-15.848799),(29.186311,-15.812832),(29.406969,-15.714233),(29.422059,-15.71103),(29.508462,-15.703588),(29.526239,-15.692839),(29.563446,-15.662144),(29.587217,-15.655736),(29.608559,-15.658423),(29.62799,-15.663591),(29.648505,-15.666588),(29.672793,-15.663281),(29.73005,-15.644677),(29.773252,-15.638062),(29.814283,-15.619666),(29.837331,-15.614808),(29.881773,-15.618839),(29.967504,-15.641473),(30.010654,-15.646227)] From a7427a9cdfc19893b2b3875da19cda3a027467b8 Mon Sep 17 00:00:00 2001 From: Nikita Mikhailov Date: Sat, 20 Feb 2021 20:44:18 +0300 Subject: [PATCH 300/544] better --- src/Functions/geometryConverters.h | 18 ++-- src/Functions/polygonArea.cpp | 23 +++-- src/Functions/polygonConvexHull.cpp | 4 +- src/Functions/polygonPerimeter.cpp | 19 ++++- src/Functions/polygonsDistance.cpp | 24 ++++-- src/Functions/polygonsEquals.cpp | 25 ++++-- src/Functions/polygonsIntersection.cpp | 36 +++++--- src/Functions/polygonsSymDifference.cpp | 31 ++++--- src/Functions/polygonsUnion.cpp | 37 +++++--- src/Functions/polygonsWithin.cpp | 27 ++++-- src/Functions/svg.cpp | 1 + src/Functions/wkt.cpp | 1 + tests/queries/0_stateless/01300_svg.reference | 84 +++++++++---------- tests/queries/0_stateless/01300_svg.sql | 48 +++++------ tests/queries/0_stateless/01300_wkt.reference | 8 +- tests/queries/0_stateless/01300_wkt.sql | 8 +- 16 files changed, 240 insertions(+), 154 deletions(-) diff --git a/src/Functions/geometryConverters.h b/src/Functions/geometryConverters.h index c6549cba40f..02d6fb2a039 100644 --- a/src/Functions/geometryConverters.h +++ b/src/Functions/geometryConverters.h @@ -75,12 +75,14 @@ template class PointFromColumnConverter { public: - explicit PointFromColumnConverter(ColumnPtr col_) : col(col_) { } - std::vector convert() const; + std::vector convert() const + { + return convertImpl(0, col->size()); + } private: std::vector convertImpl(size_t shift, size_t count) const; @@ -113,8 +115,6 @@ template class PolygonFromColumnConverter { public: - PolygonFromColumnConverter() = default; - explicit PolygonFromColumnConverter(ColumnPtr col_) : col(col_) , ring_converter(typeid_cast(*col_).getDataPtr()) @@ -135,8 +135,6 @@ template class MultiPolygonFromColumnConverter { public: - MultiPolygonFromColumnConverter() = default; - explicit MultiPolygonFromColumnConverter(ColumnPtr col_) : col(col_) , polygon_converter(typeid_cast(*col_).getDataPtr()) @@ -334,11 +332,13 @@ template static void callOnGeometryDataType(DataTypePtr type, F && f) { /// There is no Point type, because for most of geometry functions it is useless. - if (DataTypeCustomRingSerialization::nestedDataType()->equals(*type)) + if (DataTypeCustomPointSerialization::nestedDataType()->equals(*type)) + return f(ConverterType>()); + else if (DataTypeCustomRingSerialization::nestedDataType()->equals(*type)) return f(ConverterType>()); - if (DataTypeCustomPolygonSerialization::nestedDataType()->equals(*type)) + else if (DataTypeCustomPolygonSerialization::nestedDataType()->equals(*type)) return f(ConverterType>()); - if (DataTypeCustomMultiPolygonSerialization::nestedDataType()->equals(*type)) + else if (DataTypeCustomMultiPolygonSerialization::nestedDataType()->equals(*type)) return f(ConverterType>()); throw Exception(fmt::format("Unknown geometry type {}", type->getName()), ErrorCodes::BAD_ARGUMENTS); } diff --git a/src/Functions/polygonArea.cpp b/src/Functions/polygonArea.cpp index 1f2d47a3a5c..71ac0d27715 100644 --- a/src/Functions/polygonArea.cpp +++ b/src/Functions/polygonArea.cpp @@ -20,6 +20,11 @@ namespace DB { +namespace ErrorCodes +{ + extern const int ILLEGAL_TYPE_OF_ARGUMENT; +} + template class FunctionPolygonArea : public IFunction { @@ -61,14 +66,20 @@ public: { using TypeConverter = std::decay_t; using Converter = typename TypeConverter::Type; - Converter converter(arguments[0].column->convertToFullColumnIfConst()); - auto geometries = converter.convert(); - auto & res_data = res_column->getData(); - res_data.reserve(input_rows_count); + if constexpr (std::is_same_v, Converter>) + throw Exception(fmt::format("The argument of function {} must not be Point", getName()), ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT); + else + { + Converter converter(arguments[0].column->convertToFullColumnIfConst()); + auto geometries = converter.convert(); - for (size_t i = 0; i < input_rows_count; i++) - res_data.emplace_back(boost::geometry::area(geometries[i])); + auto & res_data = res_column->getData(); + res_data.reserve(input_rows_count); + + for (size_t i = 0; i < input_rows_count; i++) + res_data.emplace_back(boost::geometry::area(geometries[i])); + } } ); diff --git a/src/Functions/polygonConvexHull.cpp b/src/Functions/polygonConvexHull.cpp index 422e46b3b15..3181a1ae3b4 100644 --- a/src/Functions/polygonConvexHull.cpp +++ b/src/Functions/polygonConvexHull.cpp @@ -67,8 +67,8 @@ public: using TypeConverter = std::decay_t; using Converter = typename TypeConverter::Type; - if (std::is_same_v>) - throw Exception(fmt::format("The argument of function {} could not be a MultiPolygon", getName()), ErrorCodes::BAD_ARGUMENTS); + if constexpr (std::is_same_v>) + throw Exception(fmt::format("The argument of function {} must not be a Point", getName()), ErrorCodes::BAD_ARGUMENTS); else { Converter converter(arguments[0].column->convertToFullColumnIfConst()); diff --git a/src/Functions/polygonPerimeter.cpp b/src/Functions/polygonPerimeter.cpp index 1855ada51ed..fef0777ab52 100644 --- a/src/Functions/polygonPerimeter.cpp +++ b/src/Functions/polygonPerimeter.cpp @@ -19,6 +19,11 @@ namespace DB { +namespace ErrorCodes +{ + extern const int ILLEGAL_TYPE_OF_ARGUMENT; +} + template class FunctionPolygonPerimeter : public IFunction @@ -63,11 +68,17 @@ public: { using TypeConverter = std::decay_t; using Converter = typename TypeConverter::Type; - Converter converter(arguments[0].column->convertToFullColumnIfConst()); - auto geometries = converter.convert(); - for (size_t i = 0; i < input_rows_count; i++) - res_data.emplace_back(boost::geometry::perimeter(geometries[i])); + if constexpr (std::is_same_v, Converter>) + throw Exception(fmt::format("The argument of function {} must not be Point", getName()), ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT); + else + { + Converter converter(arguments[0].column->convertToFullColumnIfConst()); + auto geometries = converter.convert(); + + for (size_t i = 0; i < input_rows_count; i++) + res_data.emplace_back(boost::geometry::perimeter(geometries[i])); + } } ); diff --git a/src/Functions/polygonsDistance.cpp b/src/Functions/polygonsDistance.cpp index a5341482f2f..72b1d5bc888 100644 --- a/src/Functions/polygonsDistance.cpp +++ b/src/Functions/polygonsDistance.cpp @@ -22,6 +22,11 @@ namespace DB { +namespace ErrorCodes +{ + extern const int ILLEGAL_TYPE_OF_ARGUMENT; +} + template class FunctionPolygonsDistance : public IFunction { @@ -69,15 +74,20 @@ public: using LeftConverter = typename LeftConverterType::Type; using RightConverter = typename RightConverterType::Type; - auto first = LeftConverter(arguments[0].column->convertToFullColumnIfConst()).convert(); - auto second = RightConverter(arguments[1].column->convertToFullColumnIfConst()).convert(); - - for (size_t i = 0; i < input_rows_count; i++) + if constexpr (std::is_same_v, LeftConverter> || std::is_same_v, RightConverter>) + throw Exception(fmt::format("Any argument of function {} must not be Point", getName()), ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT); + else { - boost::geometry::correct(first[i]); - boost::geometry::correct(second[i]); + auto first = LeftConverter(arguments[0].column->convertToFullColumnIfConst()).convert(); + auto second = RightConverter(arguments[1].column->convertToFullColumnIfConst()).convert(); - res_data.emplace_back(boost::geometry::distance(first[i], second[i])); + for (size_t i = 0; i < input_rows_count; i++) + { + boost::geometry::correct(first[i]); + boost::geometry::correct(second[i]); + + res_data.emplace_back(boost::geometry::distance(first[i], second[i])); + } } }); diff --git a/src/Functions/polygonsEquals.cpp b/src/Functions/polygonsEquals.cpp index d235e88b465..7e4882d5cc2 100644 --- a/src/Functions/polygonsEquals.cpp +++ b/src/Functions/polygonsEquals.cpp @@ -21,6 +21,10 @@ namespace DB { +namespace ErrorCodes +{ + extern const int ILLEGAL_TYPE_OF_ARGUMENT; +} template class FunctionPolygonsEquals : public IFunction @@ -69,16 +73,21 @@ public: using LeftConverter = typename LeftConverterType::Type; using RightConverter = typename RightConverterType::Type; - auto first = LeftConverter(arguments[0].column->convertToFullColumnIfConst()).convert(); - auto second = RightConverter(arguments[1].column->convertToFullColumnIfConst()).convert(); - - for (size_t i = 0; i < input_rows_count; i++) + if constexpr (std::is_same_v, LeftConverter> || std::is_same_v, RightConverter>) + throw Exception(fmt::format("Any argument of function {} must not be Point", getName()), ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT); + else { - boost::geometry::correct(first[i]); - boost::geometry::correct(second[i]); + auto first = LeftConverter(arguments[0].column->convertToFullColumnIfConst()).convert(); + auto second = RightConverter(arguments[1].column->convertToFullColumnIfConst()).convert(); - /// Main work here. - res_data.emplace_back(boost::geometry::equals(first[i], second[i])); + for (size_t i = 0; i < input_rows_count; i++) + { + boost::geometry::correct(first[i]); + boost::geometry::correct(second[i]); + + /// Main work here. + res_data.emplace_back(boost::geometry::equals(first[i], second[i])); + } } } ); diff --git a/src/Functions/polygonsIntersection.cpp b/src/Functions/polygonsIntersection.cpp index 285255df031..331215e73e1 100644 --- a/src/Functions/polygonsIntersection.cpp +++ b/src/Functions/polygonsIntersection.cpp @@ -21,6 +21,11 @@ namespace DB { +namespace ErrorCodes +{ + extern const int ILLEGAL_TYPE_OF_ARGUMENT; +} + template class FunctionPolygonsIntersection : public IFunction { @@ -67,22 +72,27 @@ public: using LeftConverter = typename LeftConverterType::Type; using RightConverter = typename RightConverterType::Type; - auto first = LeftConverter(arguments[0].column->convertToFullColumnIfConst()).convert(); - auto second = RightConverter(arguments[1].column->convertToFullColumnIfConst()).convert(); - - /// We are not interested in some pitfalls in third-party libraries - /// NOLINTNEXTLINE(clang-analyzer-core.uninitialized.Assign) - for (size_t i = 0; i < input_rows_count; ++i) + if constexpr (std::is_same_v, LeftConverter> || std::is_same_v, RightConverter>) + throw Exception(fmt::format("Any argument of function {} must not be Point", getName()), ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT); + else { - /// Orient the polygons correctly. - boost::geometry::correct(first[i]); - boost::geometry::correct(second[i]); + auto first = LeftConverter(arguments[0].column->convertToFullColumnIfConst()).convert(); + auto second = RightConverter(arguments[1].column->convertToFullColumnIfConst()).convert(); - MultiPolygon intersection{}; - /// Main work here. - boost::geometry::intersection(first[i], second[i], intersection); + /// We are not interested in some pitfalls in third-party libraries + /// NOLINTNEXTLINE(clang-analyzer-core.uninitialized.Assign) + for (size_t i = 0; i < input_rows_count; ++i) + { + /// Orient the polygons correctly. + boost::geometry::correct(first[i]); + boost::geometry::correct(second[i]); - serializer.add(intersection); + MultiPolygon intersection{}; + /// Main work here. + boost::geometry::intersection(first[i], second[i], intersection); + + serializer.add(intersection); + } } }); diff --git a/src/Functions/polygonsSymDifference.cpp b/src/Functions/polygonsSymDifference.cpp index 20b521f7d6b..b1207582e8b 100644 --- a/src/Functions/polygonsSymDifference.cpp +++ b/src/Functions/polygonsSymDifference.cpp @@ -20,6 +20,12 @@ namespace DB { +namespace ErrorCodes +{ + extern const int ILLEGAL_TYPE_OF_ARGUMENT; +} + + template class FunctionPolygonsSymDifference : public IFunction { @@ -65,19 +71,24 @@ public: using LeftConverter = typename LeftConverterType::Type; using RightConverter = typename RightConverterType::Type; - auto first = LeftConverter(arguments[0].column->convertToFullColumnIfConst()).convert(); - auto second = RightConverter(arguments[1].column->convertToFullColumnIfConst()).convert(); - - /// NOLINTNEXTLINE(clang-analyzer-core.uninitialized.Assign) - for (size_t i = 0; i < input_rows_count; i++) + if constexpr (std::is_same_v, LeftConverter> || std::is_same_v, RightConverter>) + throw Exception(fmt::format("Any argument of function {} must not be Point", getName()), ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT); + else { - boost::geometry::correct(first[i]); - boost::geometry::correct(second[i]); + auto first = LeftConverter(arguments[0].column->convertToFullColumnIfConst()).convert(); + auto second = RightConverter(arguments[1].column->convertToFullColumnIfConst()).convert(); - MultiPolygon sym_difference{}; - boost::geometry::sym_difference(first[i], second[i], sym_difference); + /// NOLINTNEXTLINE(clang-analyzer-core.uninitialized.Assign) + for (size_t i = 0; i < input_rows_count; i++) + { + boost::geometry::correct(first[i]); + boost::geometry::correct(second[i]); - serializer.add(sym_difference); + MultiPolygon sym_difference{}; + boost::geometry::sym_difference(first[i], second[i], sym_difference); + + serializer.add(sym_difference); + } } }); diff --git a/src/Functions/polygonsUnion.cpp b/src/Functions/polygonsUnion.cpp index f236bf13bc3..bd761c847b0 100644 --- a/src/Functions/polygonsUnion.cpp +++ b/src/Functions/polygonsUnion.cpp @@ -20,6 +20,12 @@ namespace DB { +namespace ErrorCodes +{ + extern const int ILLEGAL_TYPE_OF_ARGUMENT; +} + + template class FunctionPolygonsUnion : public IFunction { @@ -65,22 +71,27 @@ public: using LeftConverter = typename LeftConverterType::Type; using RightConverter = typename RightConverterType::Type; - auto first = LeftConverter(arguments[0].column->convertToFullColumnIfConst()).convert(); - auto second = RightConverter(arguments[1].column->convertToFullColumnIfConst()).convert(); - - /// We are not interested in some pitfalls in third-party libraries - /// NOLINTNEXTLINE(clang-analyzer-core.uninitialized.Assign) - for (size_t i = 0; i < input_rows_count; i++) + if constexpr (std::is_same_v, LeftConverter> || std::is_same_v, RightConverter>) + throw Exception(fmt::format("Any argument of function {} must not be Point", getName()), ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT); + else { - /// Orient the polygons correctly. - boost::geometry::correct(first[i]); - boost::geometry::correct(second[i]); + auto first = LeftConverter(arguments[0].column->convertToFullColumnIfConst()).convert(); + auto second = RightConverter(arguments[1].column->convertToFullColumnIfConst()).convert(); - MultiPolygon polygons_union{}; - /// Main work here. - boost::geometry::union_(first[i], second[i], polygons_union); + /// We are not interested in some pitfalls in third-party libraries + /// NOLINTNEXTLINE(clang-analyzer-core.uninitialized.Assign) + for (size_t i = 0; i < input_rows_count; i++) + { + /// Orient the polygons correctly. + boost::geometry::correct(first[i]); + boost::geometry::correct(second[i]); - serializer.add(polygons_union); + MultiPolygon polygons_union{}; + /// Main work here. + boost::geometry::union_(first[i], second[i], polygons_union); + + serializer.add(polygons_union); + } } }); diff --git a/src/Functions/polygonsWithin.cpp b/src/Functions/polygonsWithin.cpp index 1e591cf0de2..708ddeee547 100644 --- a/src/Functions/polygonsWithin.cpp +++ b/src/Functions/polygonsWithin.cpp @@ -22,6 +22,12 @@ namespace DB { +namespace ErrorCodes +{ + extern const int ILLEGAL_TYPE_OF_ARGUMENT; +} + + template class FunctionPolygonsWithin : public IFunction { @@ -69,16 +75,21 @@ public: using LeftConverter = typename LeftConverterType::Type; using RightConverter = typename RightConverterType::Type; - auto first = LeftConverter(arguments[0].column->convertToFullColumnIfConst()).convert(); - auto second = RightConverter(arguments[1].column->convertToFullColumnIfConst()).convert(); - - /// NOLINTNEXTLINE(clang-analyzer-core.uninitialized.Assign) - for (size_t i = 0; i < input_rows_count; i++) + if constexpr (std::is_same_v, LeftConverter> || std::is_same_v, RightConverter>) + throw Exception(fmt::format("Any argument of function {} must not be Point", getName()), ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT); + else { - boost::geometry::correct(first[i]); - boost::geometry::correct(second[i]); + auto first = LeftConverter(arguments[0].column->convertToFullColumnIfConst()).convert(); + auto second = RightConverter(arguments[1].column->convertToFullColumnIfConst()).convert(); - res_data.emplace_back(boost::geometry::within(first[i], second[i])); + /// NOLINTNEXTLINE(clang-analyzer-core.uninitialized.Assign) + for (size_t i = 0; i < input_rows_count; i++) + { + boost::geometry::correct(first[i]); + boost::geometry::correct(second[i]); + + res_data.emplace_back(boost::geometry::within(first[i], second[i])); + } } }); diff --git a/src/Functions/svg.cpp b/src/Functions/svg.cpp index 2ad9f96ca15..873a42722c3 100644 --- a/src/Functions/svg.cpp +++ b/src/Functions/svg.cpp @@ -71,6 +71,7 @@ public: { using TypeConverter = std::decay_t; using Converter = typename TypeConverter::Type; + Converter converter(arguments[0].column->convertToFullColumnIfConst()); auto figures = converter.convert(); diff --git a/src/Functions/wkt.cpp b/src/Functions/wkt.cpp index 08aeb76dcdd..619c3f3aee8 100644 --- a/src/Functions/wkt.cpp +++ b/src/Functions/wkt.cpp @@ -44,6 +44,7 @@ public: { using TypeConverter = std::decay_t; using Converter = typename TypeConverter::Type; + Converter converter(arguments[0].column->convertToFullColumnIfConst()); auto figures = converter.convert(); diff --git a/tests/queries/0_stateless/01300_svg.reference b/tests/queries/0_stateless/01300_svg.reference index 925d3eed011..d39d67ff273 100644 --- a/tests/queries/0_stateless/01300_svg.reference +++ b/tests/queries/0_stateless/01300_svg.reference @@ -1,11 +1,11 @@ - - - + + + - - - + + + @@ -18,39 +18,39 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/tests/queries/0_stateless/01300_svg.sql b/tests/queries/0_stateless/01300_svg.sql index d82572eeeba..3e70182023b 100644 --- a/tests/queries/0_stateless/01300_svg.sql +++ b/tests/queries/0_stateless/01300_svg.sql @@ -1,48 +1,48 @@ -SELECT svg((0, 0)); -SELECT svg([(0, 0), (10, 0), (10, 10), (0, 10)]); -SELECT svg([[(0, 0), (10, 0), (10, 10), (0, 10)], [(4, 4), (5, 4), (5, 5), (4, 5)]]); -SELECT svg([[[(0, 0), (10, 0), (10, 10), (0, 10)], [(4, 4), (5, 4), (5, 5), (4, 5)]], [[(-10, -10), (-10, -9), (-9, 10)]]]); -SELECT svg((0, 0), 'b'); -SELECT svg([(0, 0), (10, 0), (10, 10), (0, 10)], 'b'); -SELECT svg([[(0, 0), (10, 0), (10, 10), (0, 10)], [(4, 4), (5, 4), (5, 5), (4, 5)]], 'b'); -SELECT svg([[[(0, 0), (10, 0), (10, 10), (0, 10)], [(4, 4), (5, 4), (5, 5), (4, 5)]], [[(-10, -10), (-10, -9), (-9, 10)]]], 'b'); +SELECT svg((0., 0.)); +SELECT svg([(0., 0.), (10, 0), (10, 10), (0, 10)]); +SELECT svg([[(0., 0.), (10, 0), (10, 10), (0, 10)], [(4., 4.), (5, 4), (5, 5), (4, 5)]]); +SELECT svg([[[(0., 0.), (10, 0), (10, 10), (0, 10)], [(4., 4.), (5, 4), (5, 5), (4, 5)]], [[(-10., -10.), (-10, -9), (-9, 10)]]]); +SELECT svg((0., 0.), 'b'); +SELECT svg([(0., 0.), (10, 0), (10, 10), (0, 10)], 'b'); +SELECT svg([[(0., 0.), (10, 0), (10, 10), (0, 10)], [(4., 4.), (5, 4), (5, 5), (4, 5)]], 'b'); +SELECT svg([[[(0., 0.), (10, 0), (10, 10), (0, 10)], [(4., 4.), (5, 4), (5, 5), (4, 5)]], [[(-10., -10.), (-10, -9), (-9, 10)]]], 'b'); DROP TABLE IF EXISTS geo; CREATE TABLE geo (p Tuple(Float64, Float64), s String, id Int) engine=Memory(); -INSERT INTO geo VALUES ((0, 0), 'b', 1); -INSERT INTO geo VALUES ((1, 0), 'c', 2); -INSERT INTO geo VALUES ((2, 0), 'd', 3); +INSERT INTO geo VALUES ((0., 0.), 'b', 1); +INSERT INTO geo VALUES ((1., 0.), 'c', 2); +INSERT INTO geo VALUES ((2., 0.), 'd', 3); SELECT svg(p) FROM geo ORDER BY id; SELECT svg(p, 'b') FROM geo ORDER BY id; -SELECT svg((0, 0), s) FROM geo ORDER BY id; +SELECT svg((0., 0.), s) FROM geo ORDER BY id; SELECT svg(p, s) FROM geo ORDER BY id; DROP TABLE IF EXISTS geo; CREATE TABLE geo (p Array(Tuple(Float64, Float64)), s String, id Int) engine=Memory(); -INSERT INTO geo VALUES ([(0, 0), (10, 0), (10, 10), (0, 10)], 'b', 1); -INSERT INTO geo VALUES ([(1, 0), (10, 0), (10, 10), (0, 10)], 'c', 2); -INSERT INTO geo VALUES ([(2, 0), (10, 0), (10, 10), (0, 10)], 'd', 3); +INSERT INTO geo VALUES ([(0., 0.), (10, 0), (10, 10), (0, 10)], 'b', 1); +INSERT INTO geo VALUES ([(1., 0.), (10, 0), (10, 10), (0, 10)], 'c', 2); +INSERT INTO geo VALUES ([(2., 0.), (10, 0), (10, 10), (0, 10)], 'd', 3); SELECT svg(p) FROM geo ORDER BY id; SELECT svg(p, 'b') FROM geo ORDER BY id; -SELECT svg([(0, 0), (10, 0), (10, 10), (0, 10)], s) FROM geo ORDER BY id; +SELECT svg([(0., 0.), (10, 0), (10, 10), (0, 10)], s) FROM geo ORDER BY id; SELECT svg(p, s) FROM geo ORDER BY id; DROP TABLE IF EXISTS geo; CREATE TABLE geo (p Array(Array(Tuple(Float64, Float64))), s String, id Int) engine=Memory(); -INSERT INTO geo VALUES ([[(0, 0), (10, 0), (10, 10), (0, 10)], [(4, 4), (5, 4), (5, 5), (4, 5)]], 'b', 1); -INSERT INTO geo VALUES ([[(1, 0), (10, 0), (10, 10), (0, 10)], [(4, 4), (5, 4), (5, 5), (4, 5)]], 'c', 2); -INSERT INTO geo VALUES ([[(2, 0), (10, 0), (10, 10), (0, 10)], [(4, 4), (5, 4), (5, 5), (4, 5)]], 'd', 3); +INSERT INTO geo VALUES ([[(0., 0.), (10, 0), (10, 10), (0, 10)], [(4, 4), (5, 4), (5, 5), (4, 5)]], 'b', 1); +INSERT INTO geo VALUES ([[(1., 0.), (10, 0), (10, 10), (0, 10)], [(4, 4), (5, 4), (5, 5), (4, 5)]], 'c', 2); +INSERT INTO geo VALUES ([[(2., 0.), (10, 0), (10, 10), (0, 10)], [(4, 4), (5, 4), (5, 5), (4, 5)]], 'd', 3); SELECT svg(p) FROM geo ORDER BY id; SELECT svg(p, 'b') FROM geo ORDER BY id; -SELECT svg([[(0, 0), (10, 0), (10, 10), (0, 10)], [(4, 4), (5, 4), (5, 5), (4, 5)]], s) FROM geo ORDER BY id; +SELECT svg([[(0., 0.), (10, 0), (10, 10), (0, 10)], [(4., 4.), (5, 4), (5, 5), (4, 5)]], s) FROM geo ORDER BY id; SELECT svg(p, s) FROM geo ORDER BY id; DROP TABLE IF EXISTS geo; CREATE TABLE geo (p Array(Array(Array(Tuple(Float64, Float64)))), s String, id Int) engine=Memory(); -INSERT INTO geo VALUES ([[[(0, 0), (10, 0), (10, 10), (0, 10)], [(4, 4), (5, 4), (5, 5), (4, 5)]], [[(-10, -10), (-10, -9), (-9, 10)]]], 'b', 1); -INSERT INTO geo VALUES ([[[(1, 0), (10, 0), (10, 10), (0, 10)], [(4, 4), (5, 4), (5, 5), (4, 5)]], [[(-10, -10), (-10, -9), (-9, 10)]]], 'c', 2); -INSERT INTO geo VALUES ([[[(2, 0), (10, 0), (10, 10), (0, 10)], [(4, 4), (5, 4), (5, 5), (4, 5)]], [[(-10, -10), (-10, -9), (-9, 10)]]], 'd', 3); +INSERT INTO geo VALUES ([[[(0., 0.), (10, 0), (10, 10), (0, 10)], [(4., 4.), (5, 4), (5, 5), (4, 5)]], [[(-10., -10.), (-10, -9), (-9, 10)]]], 'b', 1); +INSERT INTO geo VALUES ([[[(1., 0.), (10, 0), (10, 10), (0, 10)], [(4., 4.), (5, 4), (5, 5), (4, 5)]], [[(-10., -10.), (-10, -9), (-9, 10)]]], 'c', 2); +INSERT INTO geo VALUES ([[[(2., 0.), (10, 0), (10, 10), (0, 10)], [(4., 4.), (5, 4), (5, 5), (4, 5)]], [[(-10., -10.), (-10, -9), (-9, 10)]]], 'd', 3); SELECT svg(p) FROM geo ORDER BY id; SELECT svg(p, 'b') FROM geo ORDER BY id; -SELECT svg([[[(0, 0), (10, 0), (10, 10), (0, 10)], [(4, 4), (5, 4), (5, 5), (4, 5)]], [[(-10, -10), (-10, -9), (-9, 10)]]], s) FROM geo ORDER BY id; +SELECT svg([[[(0., 0.), (10, 0), (10, 10), (0, 10)], [(4., 4.), (5, 4), (5, 5), (4, 5)]], [[(-10., -10.), (-10, -9), (-9, 10)]]], s) FROM geo ORDER BY id; SELECT svg(p, s) FROM geo ORDER BY id; diff --git a/tests/queries/0_stateless/01300_wkt.reference b/tests/queries/0_stateless/01300_wkt.reference index c3ad0f4be67..0079e9f32df 100644 --- a/tests/queries/0_stateless/01300_wkt.reference +++ b/tests/queries/0_stateless/01300_wkt.reference @@ -1,13 +1,13 @@ POINT(0 0) -POLYGON((0 0,10 0,10 10,0 10,0 0)) +POLYGON((0 0,10 0,10 10,0 10)) POLYGON((0 0,10 0,10 10,0 10,0 0),(4 4,5 4,5 5,4 5,4 4)) MULTIPOLYGON(((0 0,10 0,10 10,0 10,0 0),(4 4,5 4,5 5,4 5,4 4)),((-10 -10,-10 -9,-9 10,-10 -10))) POINT(0 0) POINT(1 0) POINT(2 0) -POLYGON((0 0,10 0,10 10,0 10,0 0)) -POLYGON((1 0,10 0,10 10,0 10,1 0)) -POLYGON((2 0,10 0,10 10,0 10,2 0)) +POLYGON((0 0,10 0,10 10,0 10)) +POLYGON((1 0,10 0,10 10,0 10)) +POLYGON((2 0,10 0,10 10,0 10)) POLYGON((0 0,10 0,10 10,0 10,0 0),(4 4,5 4,5 5,4 5,4 4)) POLYGON((1 0,10 0,10 10,0 10,1 0),(4 4,5 4,5 5,4 5,4 4)) POLYGON((2 0,10 0,10 10,0 10,2 0),(4 4,5 4,5 5,4 5,4 4)) diff --git a/tests/queries/0_stateless/01300_wkt.sql b/tests/queries/0_stateless/01300_wkt.sql index a79d1a0c150..7047bb698bb 100644 --- a/tests/queries/0_stateless/01300_wkt.sql +++ b/tests/queries/0_stateless/01300_wkt.sql @@ -1,7 +1,7 @@ -SELECT wkt((0, 0)); -SELECT wkt([(0, 0), (10, 0), (10, 10), (0, 10)]); -SELECT wkt([[(0, 0), (10, 0), (10, 10), (0, 10)], [(4, 4), (5, 4), (5, 5), (4, 5)]]); -SELECT wkt([[[(0, 0), (10, 0), (10, 10), (0, 10)], [(4, 4), (5, 4), (5, 5), (4, 5)]], [[(-10, -10), (-10, -9), (-9, 10)]]]); +SELECT wkt((0., 0.)); +SELECT wkt([(0., 0.), (10., 0.), (10., 10.), (0., 10.)]); +SELECT wkt([[(0., 0.), (10., 0.), (10., 10.), (0., 10.)], [(4., 4.), (5., 4.), (5., 5.), (4., 5.)]]); +SELECT wkt([[[(0., 0.), (10., 0.), (10., 10.), (0., 10.)], [(4., 4.), (5., 4.), (5., 5.), (4., 5.)]], [[(-10., -10.), (-10., -9.), (-9., 10.)]]]); DROP TABLE IF EXISTS geo; CREATE TABLE geo (p Tuple(Float64, Float64), id Int) engine=Memory(); From 928641aae4fb5072af731d2c37eb2ea5faa5e285 Mon Sep 17 00:00:00 2001 From: Nikita Mikhailov Date: Wed, 24 Feb 2021 16:14:17 +0300 Subject: [PATCH 301/544] perimeter test --- .../0_stateless/01720_country_area.reference | 105 --------- ...01720_country_perimeter_and_area.reference | 214 ++++++++++++++++++ ...sh => 01720_country_perimeter_and_area.sh} | 11 +- 3 files changed, 224 insertions(+), 106 deletions(-) delete mode 100644 tests/queries/0_stateless/01720_country_area.reference create mode 100644 tests/queries/0_stateless/01720_country_perimeter_and_area.reference rename tests/queries/0_stateless/{01720_country_area.sh => 01720_country_perimeter_and_area.sh} (67%) diff --git a/tests/queries/0_stateless/01720_country_area.reference b/tests/queries/0_stateless/01720_country_area.reference deleted file mode 100644 index ccc9abbf04c..00000000000 --- a/tests/queries/0_stateless/01720_country_area.reference +++ /dev/null @@ -1,105 +0,0 @@ -Dhekelia Sovereign Base Area 127533079.61976177 -Kyrgyzstan 198994443077.2897 -Aruba 169577139.16920504 -Afghanistan 642167900289.4739 -Albania 28333199169.93513 -Andorra 452435114.7617939 -Ashmore and Cartier Islands 2696604.9276267616 -Austria 83991121342.6868 -Burundi 27040491570.2986 -Belgium 30672391961.48937 -Benin 116112410615.07138 -Burkina Faso 272767288599.06683 -Bulgaria 112764344245.5939 -Bahrain 585431263.0854981 -Bosnia and Herzegovina 51828357836.835045 -Bajo Nuevo Bank (Petrel Is.) 29452.023965717835 -Saint Barthelemy 24595825.379134282 -Belarus 207495810096.24286 -Bolivia 1086795900906.9404 -Barbados 444196434.0171095 -Bhutan 40362932122.222435 -Botswana 579022741104.1897 -Central African Republic 617982165549.6404 -Switzerland 41437057095.20376 -Clipperton Island 5042420.197242386 -Cameroon 464311319438.17255 -Republic of Congo 344887275569.75 -Coral Sea Islands 20628.814559882598 -Curaçao 463207585.46491855 -Czech Republic 78760344660.91792 -Djibouti 21846149319.499916 -Dominica 730475415.656727 -Algeria 2308849474061.9604 -Ethiopia 1127370045818.119 -Georgia 69572475312.24843 -Ghana 238667399375.48102 -Gibraltar 3831411.4171073083 -Guinea 244302369135.0428 -Gambia 10500266198.294697 -Guatemala 108814357557.89536 -Guam 564430992.5413051 -Heard Island and McDonald Islands 395072268.1670339 -Hungary 93204530848.81223 -Isle of Man 574622349.496232 -Iraq 437362923804.6782 -Israel 21904844963.57914 -Jamaica 11032909044.896214 -Jersey 119197004.74288748 -Jordan 88852268122.99756 -Baykonur Cosmodrome 6501480908.525035 -Siachen Glacier 2088368280.5951576 -Kosovo 10914986124.47776 -Laos 228111317612.30957 -Lebanon 9999649538.151918 -Liberia 95296808053.31937 -Libya 1623756106692.004 -Saint Lucia 604940965.7686977 -Liechtenstein 136883538.3183111 -Lesotho 30106906647.33149 -Luxembourg 2607643781.0226917 -Latvia 64575248440.09302 -Saint Martin 68198732.01942295 -Morocco 591718146420.0958 -Monaco 18750590.193543613 -Moldova 33208636022.82373 -Macedonia 25384586736.42653 -Mali 1252723270900.2302 -Montenegro 13729482523.613796 -Mongolia 1564646094133.3677 -Montserrat 99396603.21002583 -Namibia 822688081865.0547 -Niger 1181298791001.2363 -Norfolk Island 41003910.8372144 -Niue 220827518.96565723 -Nepal 147103938320.9306 -Nauru 28765050.65120624 -Poland 313439831446.58765 -Paraguay 399899482418.94604 -Qatar 11150102955.086788 -Romania 236371949926.43573 -Rwanda 25305141874.683277 -Western Sahara 90483117057.23624 -Scarborough Reef 93421.29269397438 -South Sudan 626858281405.9783 -Senegal 196219387423.09302 -Serranilla Bank 100843.54356614068 -Singapore 510505392.29170656 -San Marino 60357700.76923006 -Somaliland 167406759212.9871 -Somalia 471815737592.64276 -Republic of Serbia 77573594972.73866 -Suriname 145124645519.15808 -Slovakia 48456085451.12405 -Slovenia 20327209680.63761 -Swaziland 17113505052.204178 -Sint Maarten 23405778.740159098 -Syria 185944843149.37875 -Chad 1266282203998.255 -Togo 56863540226.407555 -Uganda 241852803539.38947 -Uruguay 177347712583.76038 -Vatican 10535.27369925663 -Akrotiri Sovereign Base Area 98415737.60517946 -Zambia 751913038381.5402 -Zimbabwe 389330493219.1703 diff --git a/tests/queries/0_stateless/01720_country_perimeter_and_area.reference b/tests/queries/0_stateless/01720_country_perimeter_and_area.reference new file mode 100644 index 00000000000..87ebfb62c61 --- /dev/null +++ b/tests/queries/0_stateless/01720_country_perimeter_and_area.reference @@ -0,0 +1,214 @@ +Dhekelia Sovereign Base Area 118700.71332743947 +Kyrgyzstan 3742755.7534551187 +------------------------------------- +Dhekelia Sovereign Base Area 127533079.61976177 +Kyrgyzstan 198994443077.2897 +------------------------------------- +Aruba 71530.8169588391 +Afghanistan 5224885.3545075 +Albania 1090187.4767125107 +Andorra 96608.22292958506 +Ashmore and Cartier Islands 7075.513574803439 +Austria 2079483.7812992004 +Burundi 895672.5688732249 +Belgium 1145320.4034536863 +Benin 1996227.432786925 +Burkina Faso 3272627.9559958726 +Bulgaria 1966608.6256809386 +Bahrain 135865.35144442634 +Bosnia and Herzegovina 1314472.5089314817 +Bajo Nuevo Bank (Petrel Is.) 798.5024071067089 +Saint Barthelemy 21002.79956662945 +Belarus 2738937.888108089 +Bolivia 5901370.411116252 +Barbados 89709.25425638427 +Bhutan 1020220.3776075348 +Botswana 3751204.2234685505 +Central African Republic 4935021.375483403 +Switzerland 1479396.111622348 +Clipperton Island 8943.110565971054 +Cameroon 5085955.35665887 +Republic of Congo 4387669.050264364 +Coral Sea Islands 741.2981931483673 +Curaçao 132255.62069611502 +Czech Republic 1728745.5762353 +Djibouti 822712.605195561 +Dominica 127657.27781178526 +Algeria 7355738.071028232 +Ethiopia 5222397.978393792 +Georgia 1711022.4699848779 +Ghana 2579014.0679307193 +Gibraltar 8949.393307056409 +Guinea 4038096.386119494 +Gambia 1227696.9901731263 +Guatemala 1928473.2813083928 +Guam 129159.86773557645 +Heard Island and McDonald Islands 110694.18305654383 +Hungary 1670527.817374739 +Isle of Man 119741.44999597034 +Iraq 3484768.045518602 +Israel 1231355.8285483832 +Jamaica 640475.0393226973 +Jersey 53782.09990861479 +Jordan 1682881.4223134723 +Baykonur Cosmodrome 285958.2051707919 +Siachen Glacier 246828.859148545 +Kosovo 559361.1922066971 +Laos 4388593.852659078 +Lebanon 616321.1250347663 +Liberia 1882745.8183724133 +Libya 6071430.840188828 +Saint Lucia 106590.9440833618 +Liechtenstein 59223.767796493725 +Lesotho 784230.7102469816 +Luxembourg 263259.6340049926 +Latvia 1564007.8893204194 +Saint Martin 41679.927927619785 +Morocco 5613899.215291766 +Monaco 16615.29788851603 +Moldova 1324452.8981204226 +Macedonia 719730.4433354989 +Mali 7242105.215854456 +Montenegro 749552.5839768916 +Mongolia 7290806.371346894 +Montserrat 42062.88573332693 +Namibia 5366121.742836174 +Niger 5586104.20515515 +Norfolk Island 31266.364157565444 +Niue 62852.941582168394 +Nepal 2595695.4382788637 +Nauru 19827.484632710733 +Poland 3122958.4429358654 +Paraguay 3484527.6286960323 +Qatar 595374.2491908007 +Romania 2813197.348484092 +Rwanda 822216.3985928788 +Western Sahara 2986353.4822476567 +Scarborough Reef 1261.3332800666965 +South Sudan 4822971.306241227 +Senegal 3743831.189306996 +Serranilla Bank 1518.3172992831614 +Singapore 96986.71084909553 +San Marino 29309.53472866584 +Somaliland 1970173.4652249564 +Somalia 4371311.2360761175 +Republic of Serbia 1892739.48526361 +Suriname 2050665.4319141763 +Slovakia 1266500.5762728006 +Slovenia 938838.9392746043 +Swaziland 536594.8070282809 +Sint Maarten 24187.85539367886 +Syria 2273589.0858773263 +Chad 5787996.089203358 +Togo 1652100.5245926627 +Uganda 2434653.1095533115 +Uruguay 1963705.040506292 +Vatican 427.24440132886207 +Akrotiri Sovereign Base Area 85251.32121367387 +Zambia 5601771.605527663 +Zimbabwe 2897547.4523945283 +------------------------------------- +Aruba 169577139.16920504 +Afghanistan 642167900289.4739 +Albania 28333199169.93513 +Andorra 452435114.7617939 +Ashmore and Cartier Islands 2696604.9276267616 +Austria 83991121342.6868 +Burundi 27040491570.2986 +Belgium 30672391961.48937 +Benin 116112410615.07138 +Burkina Faso 272767288599.06683 +Bulgaria 112764344245.5939 +Bahrain 585431263.0854981 +Bosnia and Herzegovina 51828357836.835045 +Bajo Nuevo Bank (Petrel Is.) 29452.023965717835 +Saint Barthelemy 24595825.379134282 +Belarus 207495810096.24286 +Bolivia 1086795900906.9404 +Barbados 444196434.0171095 +Bhutan 40362932122.222435 +Botswana 579022741104.1897 +Central African Republic 617982165549.6404 +Switzerland 41437057095.20376 +Clipperton Island 5042420.197242386 +Cameroon 464311319438.17255 +Republic of Congo 344887275569.75 +Coral Sea Islands 20628.814559882598 +Curaçao 463207585.46491855 +Czech Republic 78760344660.91792 +Djibouti 21846149319.499916 +Dominica 730475415.656727 +Algeria 2308849474061.9604 +Ethiopia 1127370045818.119 +Georgia 69572475312.24843 +Ghana 238667399375.48102 +Gibraltar 3831411.4171073083 +Guinea 244302369135.0428 +Gambia 10500266198.294697 +Guatemala 108814357557.89536 +Guam 564430992.5413051 +Heard Island and McDonald Islands 395072268.1670339 +Hungary 93204530848.81223 +Isle of Man 574622349.496232 +Iraq 437362923804.6782 +Israel 21904844963.57914 +Jamaica 11032909044.896214 +Jersey 119197004.74288748 +Jordan 88852268122.99756 +Baykonur Cosmodrome 6501480908.525035 +Siachen Glacier 2088368280.5951576 +Kosovo 10914986124.47776 +Laos 228111317612.30957 +Lebanon 9999649538.151918 +Liberia 95296808053.31937 +Libya 1623756106692.004 +Saint Lucia 604940965.7686977 +Liechtenstein 136883538.3183111 +Lesotho 30106906647.33149 +Luxembourg 2607643781.0226917 +Latvia 64575248440.09302 +Saint Martin 68198732.01942295 +Morocco 591718146420.0958 +Monaco 18750590.193543613 +Moldova 33208636022.82373 +Macedonia 25384586736.42653 +Mali 1252723270900.2302 +Montenegro 13729482523.613796 +Mongolia 1564646094133.3677 +Montserrat 99396603.21002583 +Namibia 822688081865.0547 +Niger 1181298791001.2363 +Norfolk Island 41003910.8372144 +Niue 220827518.96565723 +Nepal 147103938320.9306 +Nauru 28765050.65120624 +Poland 313439831446.58765 +Paraguay 399899482418.94604 +Qatar 11150102955.086788 +Romania 236371949926.43573 +Rwanda 25305141874.683277 +Western Sahara 90483117057.23624 +Scarborough Reef 93421.29269397438 +South Sudan 626858281405.9783 +Senegal 196219387423.09302 +Serranilla Bank 100843.54356614068 +Singapore 510505392.29170656 +San Marino 60357700.76923006 +Somaliland 167406759212.9871 +Somalia 471815737592.64276 +Republic of Serbia 77573594972.73866 +Suriname 145124645519.15808 +Slovakia 48456085451.12405 +Slovenia 20327209680.63761 +Swaziland 17113505052.204178 +Sint Maarten 23405778.740159098 +Syria 185944843149.37875 +Chad 1266282203998.255 +Togo 56863540226.407555 +Uganda 241852803539.38947 +Uruguay 177347712583.76038 +Vatican 10535.27369925663 +Akrotiri Sovereign Base Area 98415737.60517946 +Zambia 751913038381.5402 +Zimbabwe 389330493219.1703 +------------------------------------- diff --git a/tests/queries/0_stateless/01720_country_area.sh b/tests/queries/0_stateless/01720_country_perimeter_and_area.sh similarity index 67% rename from tests/queries/0_stateless/01720_country_area.sh rename to tests/queries/0_stateless/01720_country_perimeter_and_area.sh index 3edb2784b9a..a9a1e42ebf2 100755 --- a/tests/queries/0_stateless/01720_country_area.sh +++ b/tests/queries/0_stateless/01720_country_perimeter_and_area.sh @@ -7,10 +7,19 @@ CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) ${CLICKHOUSE_CLIENT} -q "drop table if exists country_polygons;" ${CLICKHOUSE_CLIENT} -q "create table country_polygons(name String, p Array(Array(Tuple(Float64, Float64)))) engine=MergeTree() order by tuple();" cat ${CURDIR}/country_polygons.tsv | ${CLICKHOUSE_CLIENT} -q "insert into country_polygons format TSV" -${CLICKHOUSE_CLIENT} -q "SELECT name, polygonAreaGeographic(p) from country_polygons" +${CLICKHOUSE_CLIENT} -q "SELECT name, polygonPerimeterGeographic(p) from country_polygons" +${CLICKHOUSE_CLIENT} -q "SELECT '-------------------------------------'" +${CLICKHOUSE_CLIENT} -q "SELECT name, polygonAreaGeographic(p) from country_polygons" +${CLICKHOUSE_CLIENT} -q "SELECT '-------------------------------------'" ${CLICKHOUSE_CLIENT} -q "drop table if exists country_rings;" + + ${CLICKHOUSE_CLIENT} -q "create table country_rings(name String, p Array(Tuple(Float64, Float64))) engine=MergeTree() order by tuple();" cat ${CURDIR}/country_rings.tsv | ${CLICKHOUSE_CLIENT} -q "insert into country_rings format TSV" + +${CLICKHOUSE_CLIENT} -q "SELECT name, polygonPerimeterGeographic(p) from country_rings" +${CLICKHOUSE_CLIENT} -q "SELECT '-------------------------------------'" ${CLICKHOUSE_CLIENT} -q "SELECT name, polygonAreaGeographic(p) from country_rings" +${CLICKHOUSE_CLIENT} -q "SELECT '-------------------------------------'" ${CLICKHOUSE_CLIENT} -q "drop table if exists country_rings;" \ No newline at end of file From 053a95674d6c28dd1d8056c6fe2789d0195cb769 Mon Sep 17 00:00:00 2001 From: Nikita Mikhailov Date: Wed, 24 Feb 2021 22:56:30 +0300 Subject: [PATCH 302/544] unit test --- src/Functions/tests/gtest_geometry.cpp | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) create mode 100644 src/Functions/tests/gtest_geometry.cpp diff --git a/src/Functions/tests/gtest_geometry.cpp b/src/Functions/tests/gtest_geometry.cpp new file mode 100644 index 00000000000..3d316c245ec --- /dev/null +++ b/src/Functions/tests/gtest_geometry.cpp @@ -0,0 +1,18 @@ +#include + +#include +#include +#include + +namespace bg = boost::geometry; + +TEST(Geometry, Area) +{ + size_t max = 0, min = 0; + + bg::model::polygon > > sph_poly; + bg::read_wkt("POLYGON ((33.905868 35.090882, 33.913619 35.090882, 33.921474 35.080702, 33.914446 35.073054, 33.908245 35.070729, 33.906524 35.069122, 33.906506 35.069105, 33.898116 35.061272, 33.880133 35.073054, 33.874655 35.067525, 33.867627 35.060497, 33.855122 35.053417, 33.841169 35.051092, 33.834865 35.056621, 33.827113 35.061272, 33.813781 35.055794, 33.804375 35.049541, 33.799001 35.038534, 33.822359 35.030059, 33.830214 35.023031, 33.829387 35.001176, 33.829387 35.001172, 33.840342 34.993369, 33.859049 34.991819, 33.859049 34.974662, 33.850471 34.973009, 33.838068 34.963707, 33.84582 34.959728, 33.864423 34.962983, 33.891841 34.958139, 33.8838 34.949123, 33.874522 34.94123, 33.862315 34.937893, 33.847423 34.94245, 33.819672 34.964748, 33.80421 34.972602, 33.781896 34.976212, 33.784945 34.976212, 33.788046 34.976988, 33.7928 34.977763, 33.79435 34.977763, 33.791146 34.982414, 33.786495 34.984687, 33.782568 34.984687, 33.777917 34.984687, 33.77399 34.988666, 33.766135 34.990268, 33.761484 34.990268, 33.75921 34.988666, 33.765411 34.985566, 33.769339 34.983964, 33.770889 34.980088, 33.77554 34.980088, 33.780191 34.979313, 33.780986 34.976338, 33.780935 34.976345, 33.760427 34.979682, 33.717296 34.977769, 33.70152 34.97289, 33.702935 34.987943, 33.711461 34.985566, 33.71544 34.997296, 33.699731 35.002722, 33.69663 35.008975, 33.705312 35.015228, 33.702211 35.022256, 33.685003 35.029284, 33.679444 35.033891, 33.679435 35.033899, 33.675649 35.037036, 33.674099 35.046441, 33.678853 35.055794, 33.69446 35.058171, 33.705312 35.06675, 33.714717 35.06675, 33.719368 35.06277, 33.711461 35.040963, 33.707585 35.029284, 33.718489 35.032385, 33.739677 35.047216, 33.766135 35.03161, 33.77554 35.040188, 33.786495 35.038534, 33.79435 35.040188, 33.798278 35.052642, 33.824012 35.06675, 33.834865 35.063597, 33.842719 35.056621, 33.853571 35.058171, 33.866904 35.06675, 33.871555 35.073054, 33.876929 35.076826, 33.871555 35.085456, 33.871555 35.100236, 33.876206 35.118994, 33.889435 35.118994, 33.891812 35.110468, 33.89884 35.108814, 33.903594 35.099512, 33.905868 35.09636, 33.905868 35.090882), (33.742792 35.001233, 33.746689 35.002711, 33.752063 35.004323, 33.752063 35.0144, 33.746151 35.015207, 33.741314 35.013729, 33.740239 35.010101, 33.738761 35.005264, 33.739702 35.002576, 33.742792 35.001233))", sph_poly); + auto area = bg::area(sph_poly); + std::cout << "Area: " << area << std::endl; + ASSERT_TRUE(std::abs(area - 127533079.61976177) < 1e-4); +} From ec7d9309809b93956b2dca0ba676c0a4fd9509fb Mon Sep 17 00:00:00 2001 From: Nikita Mikhailov Date: Wed, 24 Feb 2021 23:18:42 +0300 Subject: [PATCH 303/544] better --- src/Functions/polygonArea.cpp | 26 ++++++++++++++++++++++++-- 1 file changed, 24 insertions(+), 2 deletions(-) diff --git a/src/Functions/polygonArea.cpp b/src/Functions/polygonArea.cpp index 71ac0d27715..6cc9dd04b69 100644 --- a/src/Functions/polygonArea.cpp +++ b/src/Functions/polygonArea.cpp @@ -61,6 +61,8 @@ public: ColumnPtr executeImpl(const ColumnsWithTypeAndName & arguments, const DataTypePtr & /*result_type*/, size_t input_rows_count) const override { auto res_column = ColumnFloat64::create(); + auto & res_data = res_column->getData(); + res_data.reserve(input_rows_count); callOnGeometryDataType(arguments[0].type, [&] (const auto & type) { @@ -74,8 +76,28 @@ public: Converter converter(arguments[0].column->convertToFullColumnIfConst()); auto geometries = converter.convert(); - auto & res_data = res_column->getData(); - res_data.reserve(input_rows_count); + if constexpr (std::is_same_v, Converter>) { + for (auto & polygon : geometries) { + std::cout << "OUTER" << std::endl; + for (auto point : polygon.outer()) { + if constexpr (std::is_same_v) { + std::cout << point.x() << ' ' << point.y() << std::endl; + } else { + std::cout << point.template get<0>() << ' ' << point.template get<1>() << std::endl; + } + } + std::cout << "INNER" << std::endl; + for (auto & inner : polygon.inners()) { + for (auto point : inner) { + if constexpr (std::is_same_v) { + std::cout << point.x() << ' ' << point.y() << std::endl; + } else { + std::cout << point.template get<0>() << ' ' << point.template get<1>() << std::endl; + } + } + } + } + } for (size_t i = 0; i < input_rows_count; i++) res_data.emplace_back(boost::geometry::area(geometries[i])); From c87d7de8b9a44423e25e61c5fb30deb01b9567b6 Mon Sep 17 00:00:00 2001 From: Nikita Mikhailov Date: Thu, 25 Feb 2021 11:48:46 +0300 Subject: [PATCH 304/544] fix unit test --- src/Functions/tests/gtest_geometry.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/Functions/tests/gtest_geometry.cpp b/src/Functions/tests/gtest_geometry.cpp index 3d316c245ec..428b421fcc0 100644 --- a/src/Functions/tests/gtest_geometry.cpp +++ b/src/Functions/tests/gtest_geometry.cpp @@ -1,3 +1,5 @@ +#include + #include #include @@ -8,8 +10,6 @@ namespace bg = boost::geometry; TEST(Geometry, Area) { - size_t max = 0, min = 0; - bg::model::polygon > > sph_poly; bg::read_wkt("POLYGON ((33.905868 35.090882, 33.913619 35.090882, 33.921474 35.080702, 33.914446 35.073054, 33.908245 35.070729, 33.906524 35.069122, 33.906506 35.069105, 33.898116 35.061272, 33.880133 35.073054, 33.874655 35.067525, 33.867627 35.060497, 33.855122 35.053417, 33.841169 35.051092, 33.834865 35.056621, 33.827113 35.061272, 33.813781 35.055794, 33.804375 35.049541, 33.799001 35.038534, 33.822359 35.030059, 33.830214 35.023031, 33.829387 35.001176, 33.829387 35.001172, 33.840342 34.993369, 33.859049 34.991819, 33.859049 34.974662, 33.850471 34.973009, 33.838068 34.963707, 33.84582 34.959728, 33.864423 34.962983, 33.891841 34.958139, 33.8838 34.949123, 33.874522 34.94123, 33.862315 34.937893, 33.847423 34.94245, 33.819672 34.964748, 33.80421 34.972602, 33.781896 34.976212, 33.784945 34.976212, 33.788046 34.976988, 33.7928 34.977763, 33.79435 34.977763, 33.791146 34.982414, 33.786495 34.984687, 33.782568 34.984687, 33.777917 34.984687, 33.77399 34.988666, 33.766135 34.990268, 33.761484 34.990268, 33.75921 34.988666, 33.765411 34.985566, 33.769339 34.983964, 33.770889 34.980088, 33.77554 34.980088, 33.780191 34.979313, 33.780986 34.976338, 33.780935 34.976345, 33.760427 34.979682, 33.717296 34.977769, 33.70152 34.97289, 33.702935 34.987943, 33.711461 34.985566, 33.71544 34.997296, 33.699731 35.002722, 33.69663 35.008975, 33.705312 35.015228, 33.702211 35.022256, 33.685003 35.029284, 33.679444 35.033891, 33.679435 35.033899, 33.675649 35.037036, 33.674099 35.046441, 33.678853 35.055794, 33.69446 35.058171, 33.705312 35.06675, 33.714717 35.06675, 33.719368 35.06277, 33.711461 35.040963, 33.707585 35.029284, 33.718489 35.032385, 33.739677 35.047216, 33.766135 35.03161, 33.77554 35.040188, 33.786495 35.038534, 33.79435 35.040188, 33.798278 35.052642, 33.824012 35.06675, 33.834865 35.063597, 33.842719 35.056621, 33.853571 35.058171, 33.866904 35.06675, 33.871555 35.073054, 33.876929 35.076826, 33.871555 35.085456, 33.871555 35.100236, 33.876206 35.118994, 33.889435 35.118994, 33.891812 35.110468, 33.89884 35.108814, 33.903594 35.099512, 33.905868 35.09636, 33.905868 35.090882), (33.742792 35.001233, 33.746689 35.002711, 33.752063 35.004323, 33.752063 35.0144, 33.746151 35.015207, 33.741314 35.013729, 33.740239 35.010101, 33.738761 35.005264, 33.739702 35.002576, 33.742792 35.001233))", sph_poly); auto area = bg::area(sph_poly); From 44f0edacb2370c364dc3709a88bd719a446937ad Mon Sep 17 00:00:00 2001 From: Nikita Mikhailov Date: Thu, 25 Feb 2021 13:10:01 +0300 Subject: [PATCH 305/544] another try --- src/Functions/tests/gtest_geometry.cpp | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/Functions/tests/gtest_geometry.cpp b/src/Functions/tests/gtest_geometry.cpp index 428b421fcc0..17283f70c16 100644 --- a/src/Functions/tests/gtest_geometry.cpp +++ b/src/Functions/tests/gtest_geometry.cpp @@ -1,5 +1,6 @@ #include +#include #include #include @@ -10,7 +11,8 @@ namespace bg = boost::geometry; TEST(Geometry, Area) { - bg::model::polygon > > sph_poly; + + DB::GeographicPolygon sph_poly; bg::read_wkt("POLYGON ((33.905868 35.090882, 33.913619 35.090882, 33.921474 35.080702, 33.914446 35.073054, 33.908245 35.070729, 33.906524 35.069122, 33.906506 35.069105, 33.898116 35.061272, 33.880133 35.073054, 33.874655 35.067525, 33.867627 35.060497, 33.855122 35.053417, 33.841169 35.051092, 33.834865 35.056621, 33.827113 35.061272, 33.813781 35.055794, 33.804375 35.049541, 33.799001 35.038534, 33.822359 35.030059, 33.830214 35.023031, 33.829387 35.001176, 33.829387 35.001172, 33.840342 34.993369, 33.859049 34.991819, 33.859049 34.974662, 33.850471 34.973009, 33.838068 34.963707, 33.84582 34.959728, 33.864423 34.962983, 33.891841 34.958139, 33.8838 34.949123, 33.874522 34.94123, 33.862315 34.937893, 33.847423 34.94245, 33.819672 34.964748, 33.80421 34.972602, 33.781896 34.976212, 33.784945 34.976212, 33.788046 34.976988, 33.7928 34.977763, 33.79435 34.977763, 33.791146 34.982414, 33.786495 34.984687, 33.782568 34.984687, 33.777917 34.984687, 33.77399 34.988666, 33.766135 34.990268, 33.761484 34.990268, 33.75921 34.988666, 33.765411 34.985566, 33.769339 34.983964, 33.770889 34.980088, 33.77554 34.980088, 33.780191 34.979313, 33.780986 34.976338, 33.780935 34.976345, 33.760427 34.979682, 33.717296 34.977769, 33.70152 34.97289, 33.702935 34.987943, 33.711461 34.985566, 33.71544 34.997296, 33.699731 35.002722, 33.69663 35.008975, 33.705312 35.015228, 33.702211 35.022256, 33.685003 35.029284, 33.679444 35.033891, 33.679435 35.033899, 33.675649 35.037036, 33.674099 35.046441, 33.678853 35.055794, 33.69446 35.058171, 33.705312 35.06675, 33.714717 35.06675, 33.719368 35.06277, 33.711461 35.040963, 33.707585 35.029284, 33.718489 35.032385, 33.739677 35.047216, 33.766135 35.03161, 33.77554 35.040188, 33.786495 35.038534, 33.79435 35.040188, 33.798278 35.052642, 33.824012 35.06675, 33.834865 35.063597, 33.842719 35.056621, 33.853571 35.058171, 33.866904 35.06675, 33.871555 35.073054, 33.876929 35.076826, 33.871555 35.085456, 33.871555 35.100236, 33.876206 35.118994, 33.889435 35.118994, 33.891812 35.110468, 33.89884 35.108814, 33.903594 35.099512, 33.905868 35.09636, 33.905868 35.090882), (33.742792 35.001233, 33.746689 35.002711, 33.752063 35.004323, 33.752063 35.0144, 33.746151 35.015207, 33.741314 35.013729, 33.740239 35.010101, 33.738761 35.005264, 33.739702 35.002576, 33.742792 35.001233))", sph_poly); auto area = bg::area(sph_poly); std::cout << "Area: " << area << std::endl; From 94b6cadb778de9207147622992127bda0052b5c2 Mon Sep 17 00:00:00 2001 From: Nikita Mikhailov Date: Thu, 25 Feb 2021 13:11:06 +0300 Subject: [PATCH 306/544] better --- src/Functions/tests/gtest_geometry.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Functions/tests/gtest_geometry.cpp b/src/Functions/tests/gtest_geometry.cpp index 17283f70c16..9b57eb5449e 100644 --- a/src/Functions/tests/gtest_geometry.cpp +++ b/src/Functions/tests/gtest_geometry.cpp @@ -15,6 +15,6 @@ TEST(Geometry, Area) DB::GeographicPolygon sph_poly; bg::read_wkt("POLYGON ((33.905868 35.090882, 33.913619 35.090882, 33.921474 35.080702, 33.914446 35.073054, 33.908245 35.070729, 33.906524 35.069122, 33.906506 35.069105, 33.898116 35.061272, 33.880133 35.073054, 33.874655 35.067525, 33.867627 35.060497, 33.855122 35.053417, 33.841169 35.051092, 33.834865 35.056621, 33.827113 35.061272, 33.813781 35.055794, 33.804375 35.049541, 33.799001 35.038534, 33.822359 35.030059, 33.830214 35.023031, 33.829387 35.001176, 33.829387 35.001172, 33.840342 34.993369, 33.859049 34.991819, 33.859049 34.974662, 33.850471 34.973009, 33.838068 34.963707, 33.84582 34.959728, 33.864423 34.962983, 33.891841 34.958139, 33.8838 34.949123, 33.874522 34.94123, 33.862315 34.937893, 33.847423 34.94245, 33.819672 34.964748, 33.80421 34.972602, 33.781896 34.976212, 33.784945 34.976212, 33.788046 34.976988, 33.7928 34.977763, 33.79435 34.977763, 33.791146 34.982414, 33.786495 34.984687, 33.782568 34.984687, 33.777917 34.984687, 33.77399 34.988666, 33.766135 34.990268, 33.761484 34.990268, 33.75921 34.988666, 33.765411 34.985566, 33.769339 34.983964, 33.770889 34.980088, 33.77554 34.980088, 33.780191 34.979313, 33.780986 34.976338, 33.780935 34.976345, 33.760427 34.979682, 33.717296 34.977769, 33.70152 34.97289, 33.702935 34.987943, 33.711461 34.985566, 33.71544 34.997296, 33.699731 35.002722, 33.69663 35.008975, 33.705312 35.015228, 33.702211 35.022256, 33.685003 35.029284, 33.679444 35.033891, 33.679435 35.033899, 33.675649 35.037036, 33.674099 35.046441, 33.678853 35.055794, 33.69446 35.058171, 33.705312 35.06675, 33.714717 35.06675, 33.719368 35.06277, 33.711461 35.040963, 33.707585 35.029284, 33.718489 35.032385, 33.739677 35.047216, 33.766135 35.03161, 33.77554 35.040188, 33.786495 35.038534, 33.79435 35.040188, 33.798278 35.052642, 33.824012 35.06675, 33.834865 35.063597, 33.842719 35.056621, 33.853571 35.058171, 33.866904 35.06675, 33.871555 35.073054, 33.876929 35.076826, 33.871555 35.085456, 33.871555 35.100236, 33.876206 35.118994, 33.889435 35.118994, 33.891812 35.110468, 33.89884 35.108814, 33.903594 35.099512, 33.905868 35.09636, 33.905868 35.090882), (33.742792 35.001233, 33.746689 35.002711, 33.752063 35.004323, 33.752063 35.0144, 33.746151 35.015207, 33.741314 35.013729, 33.740239 35.010101, 33.738761 35.005264, 33.739702 35.002576, 33.742792 35.001233))", sph_poly); auto area = bg::area(sph_poly); - std::cout << "Area: " << area << std::endl; + std::cout << std::setprecision (15) << area << std::endl; ASSERT_TRUE(std::abs(area - 127533079.61976177) < 1e-4); } From ac03ba31bf07a4711ce003e738f3692869f00206 Mon Sep 17 00:00:00 2001 From: Nikita Mikhailov Date: Fri, 26 Feb 2021 01:33:37 +0300 Subject: [PATCH 307/544] move to spherical point --- src/Functions/geometryConverters.h | 3 +- .../01302_polygons_distance.reference | 4 +- .../0_stateless/01308_polygon_area.reference | 2 +- ...01720_country_perimeter_and_area.reference | 420 +++++++++--------- 4 files changed, 215 insertions(+), 214 deletions(-) diff --git a/src/Functions/geometryConverters.h b/src/Functions/geometryConverters.h index 02d6fb2a039..b591a5b967f 100644 --- a/src/Functions/geometryConverters.h +++ b/src/Functions/geometryConverters.h @@ -51,7 +51,8 @@ using CartesianPolygon = Polygon; using CartesianMultiPolygon = MultiPolygon; using CartesianGeometry = Geometry; -using GeographicPoint = boost::geometry::model::point>; +// using GeographicPoint = boost::geometry::model::point>; +using GeographicPoint = boost::geometry::model::point>; using GeographicRing = Ring; using GeographicPolygon = Polygon; using GeographicMultiPolygon = MultiPolygon; diff --git a/tests/queries/0_stateless/01302_polygons_distance.reference b/tests/queries/0_stateless/01302_polygons_distance.reference index 8d8b128e512..f5f59e30710 100644 --- a/tests/queries/0_stateless/01302_polygons_distance.reference +++ b/tests/queries/0_stateless/01302_polygons_distance.reference @@ -1,4 +1,4 @@ 0 1.2727922061357855 -2088389.0786590837 -2088389.0786590837 +0.3274195462417724 +0.3274195462417724 diff --git a/tests/queries/0_stateless/01308_polygon_area.reference b/tests/queries/0_stateless/01308_polygon_area.reference index 41b497f8103..56d0c4ef174 100644 --- a/tests/queries/0_stateless/01308_polygon_area.reference +++ b/tests/queries/0_stateless/01308_polygon_area.reference @@ -1,2 +1,2 @@ 25 -3848183.73456666 +9.387703638370358e-8 diff --git a/tests/queries/0_stateless/01720_country_perimeter_and_area.reference b/tests/queries/0_stateless/01720_country_perimeter_and_area.reference index 87ebfb62c61..8a9690791c6 100644 --- a/tests/queries/0_stateless/01720_country_perimeter_and_area.reference +++ b/tests/queries/0_stateless/01720_country_perimeter_and_area.reference @@ -1,214 +1,214 @@ -Dhekelia Sovereign Base Area 118700.71332743947 -Kyrgyzstan 3742755.7534551187 +Dhekelia Sovereign Base Area 0.0186259930051051 +Kyrgyzstan 0.5868323961091907 ------------------------------------- -Dhekelia Sovereign Base Area 127533079.61976177 -Kyrgyzstan 198994443077.2897 +Dhekelia Sovereign Base Area 0.000003139488070896512 +Kyrgyzstan 0.004895645023822883 ------------------------------------- -Aruba 71530.8169588391 -Afghanistan 5224885.3545075 -Albania 1090187.4767125107 -Andorra 96608.22292958506 -Ashmore and Cartier Islands 7075.513574803439 -Austria 2079483.7812992004 -Burundi 895672.5688732249 -Belgium 1145320.4034536863 -Benin 1996227.432786925 -Burkina Faso 3272627.9559958726 -Bulgaria 1966608.6256809386 -Bahrain 135865.35144442634 -Bosnia and Herzegovina 1314472.5089314817 -Bajo Nuevo Bank (Petrel Is.) 798.5024071067089 -Saint Barthelemy 21002.79956662945 -Belarus 2738937.888108089 -Bolivia 5901370.411116252 -Barbados 89709.25425638427 -Bhutan 1020220.3776075348 -Botswana 3751204.2234685505 -Central African Republic 4935021.375483403 -Switzerland 1479396.111622348 -Clipperton Island 8943.110565971054 -Cameroon 5085955.35665887 -Republic of Congo 4387669.050264364 -Coral Sea Islands 741.2981931483673 -Curaçao 132255.62069611502 -Czech Republic 1728745.5762353 -Djibouti 822712.605195561 -Dominica 127657.27781178526 -Algeria 7355738.071028232 -Ethiopia 5222397.978393792 -Georgia 1711022.4699848779 -Ghana 2579014.0679307193 -Gibraltar 8949.393307056409 -Guinea 4038096.386119494 -Gambia 1227696.9901731263 -Guatemala 1928473.2813083928 -Guam 129159.86773557645 -Heard Island and McDonald Islands 110694.18305654383 -Hungary 1670527.817374739 -Isle of Man 119741.44999597034 -Iraq 3484768.045518602 -Israel 1231355.8285483832 -Jamaica 640475.0393226973 -Jersey 53782.09990861479 -Jordan 1682881.4223134723 -Baykonur Cosmodrome 285958.2051707919 -Siachen Glacier 246828.859148545 -Kosovo 559361.1922066971 -Laos 4388593.852659078 -Lebanon 616321.1250347663 -Liberia 1882745.8183724133 -Libya 6071430.840188828 -Saint Lucia 106590.9440833618 -Liechtenstein 59223.767796493725 -Lesotho 784230.7102469816 -Luxembourg 263259.6340049926 -Latvia 1564007.8893204194 -Saint Martin 41679.927927619785 -Morocco 5613899.215291766 -Monaco 16615.29788851603 -Moldova 1324452.8981204226 -Macedonia 719730.4433354989 -Mali 7242105.215854456 -Montenegro 749552.5839768916 -Mongolia 7290806.371346894 -Montserrat 42062.88573332693 -Namibia 5366121.742836174 -Niger 5586104.20515515 -Norfolk Island 31266.364157565444 -Niue 62852.941582168394 -Nepal 2595695.4382788637 -Nauru 19827.484632710733 -Poland 3122958.4429358654 -Paraguay 3484527.6286960323 -Qatar 595374.2491908007 -Romania 2813197.348484092 -Rwanda 822216.3985928788 -Western Sahara 2986353.4822476567 -Scarborough Reef 1261.3332800666965 -South Sudan 4822971.306241227 -Senegal 3743831.189306996 -Serranilla Bank 1518.3172992831614 -Singapore 96986.71084909553 -San Marino 29309.53472866584 -Somaliland 1970173.4652249564 -Somalia 4371311.2360761175 -Republic of Serbia 1892739.48526361 -Suriname 2050665.4319141763 -Slovakia 1266500.5762728006 -Slovenia 938838.9392746043 -Swaziland 536594.8070282809 -Sint Maarten 24187.85539367886 -Syria 2273589.0858773263 -Chad 5787996.089203358 -Togo 1652100.5245926627 -Uganda 2434653.1095533115 -Uruguay 1963705.040506292 -Vatican 427.24440132886207 -Akrotiri Sovereign Base Area 85251.32121367387 -Zambia 5601771.605527663 -Zimbabwe 2897547.4523945283 +Aruba 0.011249330810410983 +Afghanistan 0.8199216326776404 +Albania 0.17108622597702605 +Andorra 0.015145740647213184 +Ashmore and Cartier Islands 0.001111472909012953 +Austria 0.3258464621357028 +Burundi 0.1409500621452211 +Belgium 0.1794463601873955 +Benin 0.31426073515874664 +Burkina Faso 0.5144381682226761 +Bulgaria 0.3083164214454252 +Bahrain 0.02137170357214413 +Bosnia and Herzegovina 0.20611959113245232 +Bajo Nuevo Bank (Petrel Is.) 0.0001254597070361587 +Saint Barthelemy 0.0032990108720812672 +Belarus 0.42899119772830474 +Bolivia 0.9279328001326348 +Barbados 0.014116142490651021 +Bhutan 0.1601735058766338 +Botswana 0.5896697538755427 +Central African Republic 0.7760222837198817 +Switzerland 0.2318851512510408 +Clipperton Island 0.0014072924221565273 +Cameroon 0.8001045813665599 +Republic of Congo 0.6904316055863188 +Coral Sea Islands 0.00011634674137689659 +Curaçao 0.02078862020307983 +Czech Republic 0.2708588915805718 +Djibouti 0.12937731543684822 +Dominica 0.020094439807419574 +Algeria 1.1549683948032776 +Ethiopia 0.8210654364815099 +Georgia 0.26823008017781313 +Ghana 0.4056578143818251 +Gibraltar 0.0014059440610631154 +Guinea 0.6350853755877334 +Gambia 0.19279774895359095 +Guatemala 0.3030953561509038 +Guam 0.020321390076536976 +Heard Island and McDonald Islands 0.017334896920453105 +Hungary 0.2617732480910806 +Isle of Man 0.01875803631141408 +Iraq 0.5469861219502402 +Israel 0.19353851895699914 +Jamaica 0.10055860979159512 +Jersey 0.008427337812134537 +Jordan 0.2642243503964102 +Baykonur Cosmodrome 0.04482995477542441 +Siachen Glacier 0.03872116827341272 +Kosovo 0.08773172991408161 +Laos 0.6899867972760174 +Lebanon 0.09676977254650951 +Liberia 0.2961649538030388 +Libya 0.9538430912224716 +Saint Lucia 0.016786201647759867 +Liechtenstein 0.009288582116863231 +Lesotho 0.12315874900320756 +Luxembourg 0.04125996057810259 +Latvia 0.24488610945731157 +Saint Martin 0.006547834154217771 +Morocco 0.8817924249630141 +Monaco 0.0026049777439637527 +Moldova 0.20765701819586885 +Macedonia 0.1128831074330059 +Mali 1.1385970015559317 +Montenegro 0.11756794062084858 +Mongolia 1.142306166871007 +Montserrat 0.006620100691409788 +Namibia 0.843464957679987 +Niger 0.8780744302377772 +Norfolk Island 0.004912027225339993 +Niue 0.009881892958363517 +Nepal 0.4076113675280835 +Nauru 0.0031205159769295255 +Poland 0.48922069488271314 +Paraguay 0.5475256537493991 +Qatar 0.09362771431858698 +Romania 0.44095021664473105 +Rwanda 0.1293663890297039 +Western Sahara 0.4691920993279596 +Scarborough Reef 0.00019842225207367386 +South Sudan 0.7584190842556537 +Senegal 0.5883247226863264 +Serranilla Bank 0.0002389083935906293 +Singapore 0.015233384733369614 +San Marino 0.004596873449598911 +Somaliland 0.3096791489207226 +Somalia 0.6879915318072617 +Republic of Serbia 0.29677234233404165 +Suriname 0.32255243342976203 +Slovakia 0.19843599488831584 +Slovenia 0.14713148471782736 +Swaziland 0.08434161089555517 +Sint Maarten 0.0037955305365309296 +Syria 0.35675522352394456 +Chad 0.9102578296637189 +Togo 0.2600585482954555 +Uganda 0.38301730108810556 +Uruguay 0.3083564407046887 +Vatican 0.00006702452496391445 +Akrotiri Sovereign Base Area 0.013376747415600219 +Zambia 0.8807923488623808 +Zimbabwe 0.4553903789902945 ------------------------------------- -Aruba 169577139.16920504 -Afghanistan 642167900289.4739 -Albania 28333199169.93513 -Andorra 452435114.7617939 -Ashmore and Cartier Islands 2696604.9276267616 -Austria 83991121342.6868 -Burundi 27040491570.2986 -Belgium 30672391961.48937 -Benin 116112410615.07138 -Burkina Faso 272767288599.06683 -Bulgaria 112764344245.5939 -Bahrain 585431263.0854981 -Bosnia and Herzegovina 51828357836.835045 -Bajo Nuevo Bank (Petrel Is.) 29452.023965717835 -Saint Barthelemy 24595825.379134282 -Belarus 207495810096.24286 -Bolivia 1086795900906.9404 -Barbados 444196434.0171095 -Bhutan 40362932122.222435 -Botswana 579022741104.1897 -Central African Republic 617982165549.6404 -Switzerland 41437057095.20376 -Clipperton Island 5042420.197242386 -Cameroon 464311319438.17255 -Republic of Congo 344887275569.75 -Coral Sea Islands 20628.814559882598 -Curaçao 463207585.46491855 -Czech Republic 78760344660.91792 -Djibouti 21846149319.499916 -Dominica 730475415.656727 -Algeria 2308849474061.9604 -Ethiopia 1127370045818.119 -Georgia 69572475312.24843 -Ghana 238667399375.48102 -Gibraltar 3831411.4171073083 -Guinea 244302369135.0428 -Gambia 10500266198.294697 -Guatemala 108814357557.89536 -Guam 564430992.5413051 -Heard Island and McDonald Islands 395072268.1670339 -Hungary 93204530848.81223 -Isle of Man 574622349.496232 -Iraq 437362923804.6782 -Israel 21904844963.57914 -Jamaica 11032909044.896214 -Jersey 119197004.74288748 -Jordan 88852268122.99756 -Baykonur Cosmodrome 6501480908.525035 -Siachen Glacier 2088368280.5951576 -Kosovo 10914986124.47776 -Laos 228111317612.30957 -Lebanon 9999649538.151918 -Liberia 95296808053.31937 -Libya 1623756106692.004 -Saint Lucia 604940965.7686977 -Liechtenstein 136883538.3183111 -Lesotho 30106906647.33149 -Luxembourg 2607643781.0226917 -Latvia 64575248440.09302 -Saint Martin 68198732.01942295 -Morocco 591718146420.0958 -Monaco 18750590.193543613 -Moldova 33208636022.82373 -Macedonia 25384586736.42653 -Mali 1252723270900.2302 -Montenegro 13729482523.613796 -Mongolia 1564646094133.3677 -Montserrat 99396603.21002583 -Namibia 822688081865.0547 -Niger 1181298791001.2363 -Norfolk Island 41003910.8372144 -Niue 220827518.96565723 -Nepal 147103938320.9306 -Nauru 28765050.65120624 -Poland 313439831446.58765 -Paraguay 399899482418.94604 -Qatar 11150102955.086788 -Romania 236371949926.43573 -Rwanda 25305141874.683277 -Western Sahara 90483117057.23624 -Scarborough Reef 93421.29269397438 -South Sudan 626858281405.9783 -Senegal 196219387423.09302 -Serranilla Bank 100843.54356614068 -Singapore 510505392.29170656 -San Marino 60357700.76923006 -Somaliland 167406759212.9871 -Somalia 471815737592.64276 -Republic of Serbia 77573594972.73866 -Suriname 145124645519.15808 -Slovakia 48456085451.12405 -Slovenia 20327209680.63761 -Swaziland 17113505052.204178 -Sint Maarten 23405778.740159098 -Syria 185944843149.37875 -Chad 1266282203998.255 -Togo 56863540226.407555 -Uganda 241852803539.38947 -Uruguay 177347712583.76038 -Vatican 10535.27369925663 -Akrotiri Sovereign Base Area 98415737.60517946 -Zambia 751913038381.5402 -Zimbabwe 389330493219.1703 +Aruba 0.0000041986375296795025 +Afghanistan 0.015826481758320493 +Albania 0.0006971811189621746 +Andorra 0.00001112355564980348 +Ashmore and Cartier Islands 6.66668338977609e-8 +Austria 0.0020634744883290235 +Burundi 0.000669169243101558 +Belgium 0.0007529367590741593 +Benin 0.00287239734953164 +Burkina Faso 0.006746218025419332 +Bulgaria 0.0027733372191197786 +Bahrain 0.00001443842547561405 +Bosnia and Herzegovina 0.0012742491201009779 +Bajo Nuevo Bank (Petrel Is.) 8.864825701897049e-10 +Saint Barthelemy 6.036607210116289e-7 +Belarus 0.005090738074359067 +Bolivia 0.026865324735758436 +Barbados 0.0000109856680212211 +Bhutan 0.0009961026696220909 +Botswana 0.01430200501713062 +Central African Republic 0.015290667187215962 +Switzerland 0.0010181463734151514 +Clipperton Island 1.2373029819547803e-7 +Cameroon 0.011488908713113137 +Republic of Congo 0.008534881807187833 +Coral Sea Islands 5.121674593493771e-10 +Curaçao 0.000011457378136273848 +Czech Republic 0.0019339153549488386 +Djibouti 0.000540370985929321 +Dominica 0.000018056168258583246 +Algeria 0.05696762706232162 +Ethiopia 0.02789047634482515 +Georgia 0.0017113229913929072 +Ghana 0.0059048504621945965 +Gibraltar 9.095456688875715e-8 +Guinea 0.006043151808047173 +Gambia 0.0002596816395280707 +Guatemala 0.0026901925526205263 +Guam 0.000013952443476670549 +Heard Island and McDonald Islands 0.000009688375334192321 +Hungary 0.0022899094702118978 +Isle of Man 0.00001410012284549863 +Iraq 0.010780689598789812 +Israel 0.0005400181032289429 +Jamaica 0.00027268062650994383 +Jersey 0.0000029236161155167853 +Jordan 0.002191215069390572 +Baykonur Cosmodrome 0.00015978303781425133 +Siachen Glacier 0.0000513879615262916 +Kosovo 0.0002684178325412152 +Laos 0.005637555524983489 +Lebanon 0.0002464436461544738 +Liberia 0.002357973807538481 +Libya 0.040072512808839354 +Saint Lucia 0.000014963842166249258 +Liechtenstein 0.0000033722024322722466 +Lesotho 0.0007426290112070925 +Luxembourg 0.00006405006804909529 +Latvia 0.00158313668683266 +Saint Martin 0.00000168759530251474 +Morocco 0.014595589778269167 +Monaco 4.6325700981005285e-7 +Moldova 0.0008158639460823913 +Macedonia 0.0006245180554490506 +Mali 0.03096381132470007 +Montenegro 0.00033762445623993013 +Mongolia 0.038446609480001344 +Montserrat 0.0000024620326175206004 +Namibia 0.020320978539029165 +Niger 0.02919849042641136 +Norfolk Island 0.0000010150641235563077 +Niue 0.000005450796200539049 +Nepal 0.003629565673884544 +Nauru 7.119067469952887e-7 +Poland 0.0076921097527402876 +Paraguay 0.009875843128670564 +Qatar 0.0002752610716836153 +Romania 0.005809479702080411 +Rwanda 0.0006262235765421803 +Western Sahara 0.0022344529652030694 +Scarborough Reef 2.4176335726807567e-9 +South Sudan 0.015509656314462458 +Senegal 0.00485201810074574 +Serranilla Bank 2.6035559945372385e-9 +Singapore 0.000012633505579848072 +San Marino 0.0000014830814619737624 +Somaliland 0.0041412916217828406 +Somalia 0.011674654119996183 +Republic of Serbia 0.001907268740192651 +Suriname 0.0035911641359236534 +Slovakia 0.0011901587428922095 +Slovenia 0.0004995546076509384 +Swaziland 0.00042234053226485263 +Sint Maarten 5.772865969377286e-7 +Syria 0.004581243750467663 +Chad 0.0313064894302088 +Togo 0.0014067991034602252 +Uganda 0.005985159048654327 +Uruguay 0.0043716082436750115 +Vatican 3.002600504657064e-10 +Akrotiri Sovereign Base Area 0.0000024314362587592923 +Zambia 0.018594119224502336 +Zimbabwe 0.009621356779606268 ------------------------------------- From 0e1b2d8fcffaf856c15551d960da60111c0b9f2e Mon Sep 17 00:00:00 2001 From: Nikita Mikhailov Date: Fri, 26 Feb 2021 15:25:28 +0300 Subject: [PATCH 308/544] geographic -> spherical --- src/Functions/geometryConverters.cpp | 14 +++++------ src/Functions/geometryConverters.h | 24 +++++++++---------- src/Functions/greatCircleDistance.cpp | 2 +- src/Functions/polygonArea.cpp | 4 ++-- src/Functions/polygonPerimeter.cpp | 4 ++-- src/Functions/polygonsDistance.cpp | 4 ++-- src/Functions/polygonsIntersection.cpp | 4 ++-- src/Functions/polygonsSymDifference.cpp | 4 ++-- src/Functions/polygonsUnion.cpp | 4 ++-- src/Functions/polygonsWithin.cpp | 4 ++-- src/Functions/tests/gtest_geometry.cpp | 20 ---------------- .../0_stateless/01301_polygons_within.sql | 4 ++-- .../0_stateless/01302_polygons_distance.sql | 4 ++-- .../0_stateless/01305_polygons_union.sql | 2 +- .../01306_polygons_intersection.sql | 4 ++-- .../0_stateless/01308_polygon_area.sql | 2 +- .../0_stateless/01720_country_intersection.sh | 4 ++-- .../01720_country_perimeter_and_area.sh | 8 +++---- 18 files changed, 48 insertions(+), 68 deletions(-) delete mode 100644 src/Functions/tests/gtest_geometry.cpp diff --git a/src/Functions/geometryConverters.cpp b/src/Functions/geometryConverters.cpp index b0b437d5332..ea6d99787ef 100644 --- a/src/Functions/geometryConverters.cpp +++ b/src/Functions/geometryConverters.cpp @@ -102,13 +102,13 @@ std::vector> MultiPolygonFromColumnConverter::convert template class PointFromColumnConverter; -template class PointFromColumnConverter; +template class PointFromColumnConverter; template class RingFromColumnConverter; -template class RingFromColumnConverter; +template class RingFromColumnConverter; template class PolygonFromColumnConverter; -template class PolygonFromColumnConverter; +template class PolygonFromColumnConverter; template class MultiPolygonFromColumnConverter; -template class MultiPolygonFromColumnConverter; +template class MultiPolygonFromColumnConverter; template typename Desired> void checkColumnTypeOrThrow(const ColumnWithTypeAndName & column) @@ -130,8 +130,8 @@ void checkColumnTypeOrThrow(const ColumnWithTypeAndName & column) template void checkColumnTypeOrThrow(const ColumnWithTypeAndName &); template void checkColumnTypeOrThrow(const ColumnWithTypeAndName &); template void checkColumnTypeOrThrow(const ColumnWithTypeAndName &); -template void checkColumnTypeOrThrow(const ColumnWithTypeAndName &); -template void checkColumnTypeOrThrow(const ColumnWithTypeAndName &); -template void checkColumnTypeOrThrow(const ColumnWithTypeAndName &); +template void checkColumnTypeOrThrow(const ColumnWithTypeAndName &); +template void checkColumnTypeOrThrow(const ColumnWithTypeAndName &); +template void checkColumnTypeOrThrow(const ColumnWithTypeAndName &); } diff --git a/src/Functions/geometryConverters.h b/src/Functions/geometryConverters.h index b591a5b967f..fcce11467d0 100644 --- a/src/Functions/geometryConverters.h +++ b/src/Functions/geometryConverters.h @@ -51,12 +51,12 @@ using CartesianPolygon = Polygon; using CartesianMultiPolygon = MultiPolygon; using CartesianGeometry = Geometry; -// using GeographicPoint = boost::geometry::model::point>; -using GeographicPoint = boost::geometry::model::point>; -using GeographicRing = Ring; -using GeographicPolygon = Polygon; -using GeographicMultiPolygon = MultiPolygon; -using GeographicGeometry = Geometry; +// using SphericalPoint = boost::geometry::model::point>; +using SphericalPoint = boost::geometry::model::point>; +using SphericalRing = Ring; +using SphericalPolygon = Polygon; +using SphericalMultiPolygon = MultiPolygon; +using SphericalGeometry = Geometry; template @@ -70,7 +70,7 @@ class MultiPolygonFromColumnConverter; /** * Class which takes some boost type and returns a pair of numbers. - * They are (x,y) in case of cartesian coordinated and (lon,lat) in case of geographic. + * They are (x,y) in case of cartesian coordinated and (lon,lat) in case of Spherical. */ template class PointFromColumnConverter @@ -151,16 +151,16 @@ private: extern template class PointFromColumnConverter; -extern template class PointFromColumnConverter; +extern template class PointFromColumnConverter; extern template class RingFromColumnConverter; -extern template class RingFromColumnConverter; +extern template class RingFromColumnConverter; extern template class PolygonFromColumnConverter; -extern template class PolygonFromColumnConverter; +extern template class PolygonFromColumnConverter; extern template class MultiPolygonFromColumnConverter; -extern template class MultiPolygonFromColumnConverter; +extern template class MultiPolygonFromColumnConverter; -/// To serialize Geographic or Cartesian point (a pair of numbers in both cases). +/// To serialize Spherical or Cartesian point (a pair of numbers in both cases). template class PointSerializer { diff --git a/src/Functions/greatCircleDistance.cpp b/src/Functions/greatCircleDistance.cpp index 801f8b3da7f..70f6d1f1759 100644 --- a/src/Functions/greatCircleDistance.cpp +++ b/src/Functions/greatCircleDistance.cpp @@ -19,7 +19,7 @@ namespace ErrorCodes extern const int ILLEGAL_TYPE_OF_ARGUMENT; } -/** Calculates the distance between two geographical locations. +/** Calculates the distance between two Sphericalal locations. * There are three variants: * greatCircleAngle: calculates the distance on a sphere in degrees: https://en.wikipedia.org/wiki/Great-circle_distance * greatCircleDistance: calculates the distance on a sphere in meters. diff --git a/src/Functions/polygonArea.cpp b/src/Functions/polygonArea.cpp index 6cc9dd04b69..4291d31d72a 100644 --- a/src/Functions/polygonArea.cpp +++ b/src/Functions/polygonArea.cpp @@ -118,13 +118,13 @@ template <> const char * FunctionPolygonArea::name = "polygonAreaCartesian"; template <> -const char * FunctionPolygonArea::name = "polygonAreaGeographic"; +const char * FunctionPolygonArea::name = "polygonAreaSpherical"; void registerFunctionPolygonArea(FunctionFactory & factory) { factory.registerFunction>(); - factory.registerFunction>(); + factory.registerFunction>(); } diff --git a/src/Functions/polygonPerimeter.cpp b/src/Functions/polygonPerimeter.cpp index fef0777ab52..9bad13d4817 100644 --- a/src/Functions/polygonPerimeter.cpp +++ b/src/Functions/polygonPerimeter.cpp @@ -95,13 +95,13 @@ template <> const char * FunctionPolygonPerimeter::name = "polygonPerimeterCartesian"; template <> -const char * FunctionPolygonPerimeter::name = "polygonPerimeterGeographic"; +const char * FunctionPolygonPerimeter::name = "polygonPerimeterSpherical"; void registerFunctionPolygonPerimeter(FunctionFactory & factory) { factory.registerFunction>(); - factory.registerFunction>(); + factory.registerFunction>(); } diff --git a/src/Functions/polygonsDistance.cpp b/src/Functions/polygonsDistance.cpp index 72b1d5bc888..f961867caae 100644 --- a/src/Functions/polygonsDistance.cpp +++ b/src/Functions/polygonsDistance.cpp @@ -104,13 +104,13 @@ template <> const char * FunctionPolygonsDistance::name = "polygonsDistanceCartesian"; template <> -const char * FunctionPolygonsDistance::name = "polygonsDistanceGeographic"; +const char * FunctionPolygonsDistance::name = "polygonsDistanceSpherical"; void registerFunctionPolygonsDistance(FunctionFactory & factory) { factory.registerFunction>(); - factory.registerFunction>(); + factory.registerFunction>(); } diff --git a/src/Functions/polygonsIntersection.cpp b/src/Functions/polygonsIntersection.cpp index 331215e73e1..0f7a9b1f264 100644 --- a/src/Functions/polygonsIntersection.cpp +++ b/src/Functions/polygonsIntersection.cpp @@ -110,13 +110,13 @@ template <> const char * FunctionPolygonsIntersection::name = "polygonsIntersectionCartesian"; template <> -const char * FunctionPolygonsIntersection::name = "polygonsIntersectionGeographic"; +const char * FunctionPolygonsIntersection::name = "polygonsIntersectionSpherical"; void registerFunctionPolygonsIntersection(FunctionFactory & factory) { factory.registerFunction>(); - factory.registerFunction>(); + factory.registerFunction>(); } } diff --git a/src/Functions/polygonsSymDifference.cpp b/src/Functions/polygonsSymDifference.cpp index b1207582e8b..4768ea93124 100644 --- a/src/Functions/polygonsSymDifference.cpp +++ b/src/Functions/polygonsSymDifference.cpp @@ -105,12 +105,12 @@ template <> const char * FunctionPolygonsSymDifference::name = "polygonsSymDifferenceCartesian"; template <> -const char * FunctionPolygonsSymDifference::name = "polygonsSymDifferenceGeographic"; +const char * FunctionPolygonsSymDifference::name = "polygonsSymDifferenceSpherical"; void registerFunctionPolygonsSymDifference(FunctionFactory & factory) { factory.registerFunction>(); - factory.registerFunction>(); + factory.registerFunction>(); } } diff --git a/src/Functions/polygonsUnion.cpp b/src/Functions/polygonsUnion.cpp index bd761c847b0..d4d67982ee0 100644 --- a/src/Functions/polygonsUnion.cpp +++ b/src/Functions/polygonsUnion.cpp @@ -108,13 +108,13 @@ template <> const char * FunctionPolygonsUnion::name = "polygonsUnionCartesian"; template <> -const char * FunctionPolygonsUnion::name = "polygonsUnionGeographic"; +const char * FunctionPolygonsUnion::name = "polygonsUnionSpherical"; void registerFunctionPolygonsUnion(FunctionFactory & factory) { factory.registerFunction>(); - factory.registerFunction>(); + factory.registerFunction>(); } } diff --git a/src/Functions/polygonsWithin.cpp b/src/Functions/polygonsWithin.cpp index 708ddeee547..eabd5e683b9 100644 --- a/src/Functions/polygonsWithin.cpp +++ b/src/Functions/polygonsWithin.cpp @@ -107,13 +107,13 @@ template <> const char * FunctionPolygonsWithin::name = "polygonsWithinCartesian"; template <> -const char * FunctionPolygonsWithin::name = "polygonsWithinGeographic"; +const char * FunctionPolygonsWithin::name = "polygonsWithinSpherical"; void registerFunctionPolygonsWithin(FunctionFactory & factory) { factory.registerFunction>(); - factory.registerFunction>(); + factory.registerFunction>(); } } diff --git a/src/Functions/tests/gtest_geometry.cpp b/src/Functions/tests/gtest_geometry.cpp deleted file mode 100644 index 9b57eb5449e..00000000000 --- a/src/Functions/tests/gtest_geometry.cpp +++ /dev/null @@ -1,20 +0,0 @@ -#include - -#include -#include - -#include -#include -#include - -namespace bg = boost::geometry; - -TEST(Geometry, Area) -{ - - DB::GeographicPolygon sph_poly; - bg::read_wkt("POLYGON ((33.905868 35.090882, 33.913619 35.090882, 33.921474 35.080702, 33.914446 35.073054, 33.908245 35.070729, 33.906524 35.069122, 33.906506 35.069105, 33.898116 35.061272, 33.880133 35.073054, 33.874655 35.067525, 33.867627 35.060497, 33.855122 35.053417, 33.841169 35.051092, 33.834865 35.056621, 33.827113 35.061272, 33.813781 35.055794, 33.804375 35.049541, 33.799001 35.038534, 33.822359 35.030059, 33.830214 35.023031, 33.829387 35.001176, 33.829387 35.001172, 33.840342 34.993369, 33.859049 34.991819, 33.859049 34.974662, 33.850471 34.973009, 33.838068 34.963707, 33.84582 34.959728, 33.864423 34.962983, 33.891841 34.958139, 33.8838 34.949123, 33.874522 34.94123, 33.862315 34.937893, 33.847423 34.94245, 33.819672 34.964748, 33.80421 34.972602, 33.781896 34.976212, 33.784945 34.976212, 33.788046 34.976988, 33.7928 34.977763, 33.79435 34.977763, 33.791146 34.982414, 33.786495 34.984687, 33.782568 34.984687, 33.777917 34.984687, 33.77399 34.988666, 33.766135 34.990268, 33.761484 34.990268, 33.75921 34.988666, 33.765411 34.985566, 33.769339 34.983964, 33.770889 34.980088, 33.77554 34.980088, 33.780191 34.979313, 33.780986 34.976338, 33.780935 34.976345, 33.760427 34.979682, 33.717296 34.977769, 33.70152 34.97289, 33.702935 34.987943, 33.711461 34.985566, 33.71544 34.997296, 33.699731 35.002722, 33.69663 35.008975, 33.705312 35.015228, 33.702211 35.022256, 33.685003 35.029284, 33.679444 35.033891, 33.679435 35.033899, 33.675649 35.037036, 33.674099 35.046441, 33.678853 35.055794, 33.69446 35.058171, 33.705312 35.06675, 33.714717 35.06675, 33.719368 35.06277, 33.711461 35.040963, 33.707585 35.029284, 33.718489 35.032385, 33.739677 35.047216, 33.766135 35.03161, 33.77554 35.040188, 33.786495 35.038534, 33.79435 35.040188, 33.798278 35.052642, 33.824012 35.06675, 33.834865 35.063597, 33.842719 35.056621, 33.853571 35.058171, 33.866904 35.06675, 33.871555 35.073054, 33.876929 35.076826, 33.871555 35.085456, 33.871555 35.100236, 33.876206 35.118994, 33.889435 35.118994, 33.891812 35.110468, 33.89884 35.108814, 33.903594 35.099512, 33.905868 35.09636, 33.905868 35.090882), (33.742792 35.001233, 33.746689 35.002711, 33.752063 35.004323, 33.752063 35.0144, 33.746151 35.015207, 33.741314 35.013729, 33.740239 35.010101, 33.738761 35.005264, 33.739702 35.002576, 33.742792 35.001233))", sph_poly); - auto area = bg::area(sph_poly); - std::cout << std::setprecision (15) << area << std::endl; - ASSERT_TRUE(std::abs(area - 127533079.61976177) < 1e-4); -} diff --git a/tests/queries/0_stateless/01301_polygons_within.sql b/tests/queries/0_stateless/01301_polygons_within.sql index 5bfa586da79..97e0573904a 100644 --- a/tests/queries/0_stateless/01301_polygons_within.sql +++ b/tests/queries/0_stateless/01301_polygons_within.sql @@ -1,7 +1,7 @@ select polygonsWithinCartesian([[[(0, 0),(0, 3),(1, 2.9),(2, 2.6),(2.6, 2),(2.9, 1),(3, 0),(0, 0)]]], [[[(1., 1.),(1., 4.),(4., 4.),(4., 1.),(1., 1.)]]]); select polygonsWithinCartesian([[[(2., 2.), (2., 3.), (3., 3.), (3., 2.)]]], [[[(1., 1.),(1., 4.),(4., 4.),(4., 1.),(1., 1.)]]]); -select polygonsWithinGeographic([[[(4.3613577, 50.8651821), (4.349556, 50.8535879), (4.3602419, 50.8435626), (4.3830299, 50.8428851), (4.3904543, 50.8564867), (4.3613148, 50.8651279)]]], [[[(4.346693, 50.858306), (4.367945, 50.852455), (4.366227, 50.840809), (4.344961, 50.833264), (4.338074, 50.848677), (4.346693, 50.858306)]]]); -select polygonsWithinGeographic([[[(4.3501568, 50.8518269), (4.3444920, 50.8439961), (4.3565941, 50.8443213), (4.3501568, 50.8518269)]]], [[[(4.3679450, 50.8524550),(4.3466930, 50.8583060),(4.3380740, 50.8486770),(4.3449610, 50.8332640),(4.3662270, 50.8408090),(4.3679450, 50.8524550)]]]); +select polygonsWithinSpherical([[[(4.3613577, 50.8651821), (4.349556, 50.8535879), (4.3602419, 50.8435626), (4.3830299, 50.8428851), (4.3904543, 50.8564867), (4.3613148, 50.8651279)]]], [[[(4.346693, 50.858306), (4.367945, 50.852455), (4.366227, 50.840809), (4.344961, 50.833264), (4.338074, 50.848677), (4.346693, 50.858306)]]]); +select polygonsWithinSpherical([[[(4.3501568, 50.8518269), (4.3444920, 50.8439961), (4.3565941, 50.8443213), (4.3501568, 50.8518269)]]], [[[(4.3679450, 50.8524550),(4.3466930, 50.8583060),(4.3380740, 50.8486770),(4.3449610, 50.8332640),(4.3662270, 50.8408090),(4.3679450, 50.8524550)]]]); diff --git a/tests/queries/0_stateless/01302_polygons_distance.sql b/tests/queries/0_stateless/01302_polygons_distance.sql index 0051578ea7c..fdbd0254983 100644 --- a/tests/queries/0_stateless/01302_polygons_distance.sql +++ b/tests/queries/0_stateless/01302_polygons_distance.sql @@ -1,8 +1,8 @@ select polygonsDistanceCartesian([[[(0, 0),(0, 3),(1, 2.9),(2, 2.6),(2.6, 2),(2.9, 1),(3, 0),(0, 0)]]], [[[(1., 1.),(1., 4.),(4., 4.),(4., 1.),(1., 1.)]]]); select polygonsDistanceCartesian([[[(0, 0), (0, 0.1), (0.1, 0.1), (0.1, 0)]]], [[[(1., 1.),(1., 4.),(4., 4.),(4., 1.),(1., 1.)]]]); -select polygonsDistanceGeographic([[[(23.725750, 37.971536)]]], [[[(4.3826169, 50.8119483)]]]); +select polygonsDistanceSpherical([[[(23.725750, 37.971536)]]], [[[(4.3826169, 50.8119483)]]]); drop table if exists polygon_01302; create table polygon_01302 (x Array(Array(Array(Tuple(Float64, Float64)))), y Array(Array(Array(Tuple(Float64, Float64))))) engine=Memory(); insert into polygon_01302 values ([[[(23.725750, 37.971536)]]], [[[(4.3826169, 50.8119483)]]]); -select polygonsDistanceGeographic(x, y) from polygon_01302; +select polygonsDistanceSpherical(x, y) from polygon_01302; diff --git a/tests/queries/0_stateless/01305_polygons_union.sql b/tests/queries/0_stateless/01305_polygons_union.sql index c96c9f7360c..a67c5a7d399 100644 --- a/tests/queries/0_stateless/01305_polygons_union.sql +++ b/tests/queries/0_stateless/01305_polygons_union.sql @@ -2,4 +2,4 @@ select polygonsUnionCartesian([[[(0., 0.),(0., 3.),(1., 2.9),(2., 2.6),(2.6, 2.) SELECT polygonsUnionCartesian([[[(2., 100.0000991821289), (0., 3.), (1., 2.9), (2., 2.6), (2.6, 2.), (2.9, 1), (3., 0.), (100.0000991821289, 2.)]]], [[[(1., 1.), (1000.0001220703125, nan), (4., 4.), (4., 1.), (1., 1.)]]]); -- { serverError 43 } -select polygonsUnionGeographic([[[(4.3613577, 50.8651821), (4.349556, 50.8535879), (4.3602419, 50.8435626), (4.3830299, 50.8428851), (4.3904543, 50.8564867), (4.3613148, 50.8651279)]]], [[[(4.346693, 50.858306), (4.367945, 50.852455), (4.366227, 50.840809), (4.344961, 50.833264), (4.338074, 50.848677), (4.346693, 50.858306)]]]); +select polygonsUnionSpherical([[[(4.3613577, 50.8651821), (4.349556, 50.8535879), (4.3602419, 50.8435626), (4.3830299, 50.8428851), (4.3904543, 50.8564867), (4.3613148, 50.8651279)]]], [[[(4.346693, 50.858306), (4.367945, 50.852455), (4.366227, 50.840809), (4.344961, 50.833264), (4.338074, 50.848677), (4.346693, 50.858306)]]]); diff --git a/tests/queries/0_stateless/01306_polygons_intersection.sql b/tests/queries/0_stateless/01306_polygons_intersection.sql index 15962c43648..fa486a138f4 100644 --- a/tests/queries/0_stateless/01306_polygons_intersection.sql +++ b/tests/queries/0_stateless/01306_polygons_intersection.sql @@ -1,5 +1,5 @@ select polygonsIntersectionCartesian([[[(0., 0.),(0., 3.),(1., 2.9),(2., 2.6),(2.6, 2.),(2.9, 1.),(3., 0.),(0., 0.)]]], [[[(1., 1.),(1., 4.),(4., 4.),(4., 1.),(1., 1.)]]]); select polygonsIntersectionCartesian([[[(0., 0.),(0., 3.),(1., 2.9),(2., 2.6),(2.6, 2.),(2.9, 1.),(3., 0.),(0., 0.)]]], [[[(3., 3.),(3., 4.),(4., 4.),(4., 3.),(3., 3.)]]]); -select polygonsIntersectionGeographic([[[(4.346693, 50.858306), (4.367945, 50.852455), (4.366227, 50.840809), (4.344961, 50.833264), (4.338074, 50.848677), (4.346693, 50.858306)]]], [[[(25.0010, 136.9987), (17.7500, 142.5000), (11.3733, 142.5917)]]]); -select polygonsIntersectionGeographic([[[(4.3613577, 50.8651821), (4.349556, 50.8535879), (4.3602419, 50.8435626), (4.3830299, 50.8428851), (4.3904543, 50.8564867), (4.3613148, 50.8651279)]]], [[[(4.346693, 50.858306), (4.367945, 50.852455), (4.366227, 50.840809), (4.344961, 50.833264), (4.338074, 50.848677), (4.346693, 50.858306)]]]); \ No newline at end of file +select polygonsIntersectionSpherical([[[(4.346693, 50.858306), (4.367945, 50.852455), (4.366227, 50.840809), (4.344961, 50.833264), (4.338074, 50.848677), (4.346693, 50.858306)]]], [[[(25.0010, 136.9987), (17.7500, 142.5000), (11.3733, 142.5917)]]]); +select polygonsIntersectionSpherical([[[(4.3613577, 50.8651821), (4.349556, 50.8535879), (4.3602419, 50.8435626), (4.3830299, 50.8428851), (4.3904543, 50.8564867), (4.3613148, 50.8651279)]]], [[[(4.346693, 50.858306), (4.367945, 50.852455), (4.366227, 50.840809), (4.344961, 50.833264), (4.338074, 50.848677), (4.346693, 50.858306)]]]); \ No newline at end of file diff --git a/tests/queries/0_stateless/01308_polygon_area.sql b/tests/queries/0_stateless/01308_polygon_area.sql index 03f87874139..4f02928b450 100644 --- a/tests/queries/0_stateless/01308_polygon_area.sql +++ b/tests/queries/0_stateless/01308_polygon_area.sql @@ -1,3 +1,3 @@ select polygonAreaCartesian([[[(0., 0.), (0., 5.), (5., 5.), (5., 0.)]]]); -select polygonAreaGeographic([[[(4.346693, 50.858306), (4.367945, 50.852455), (4.366227, 50.840809), (4.344961, 50.833264), (4.338074, 50.848677), (4.346693, 50.858306)]]]); +select polygonAreaSpherical([[[(4.346693, 50.858306), (4.367945, 50.852455), (4.366227, 50.840809), (4.344961, 50.833264), (4.338074, 50.848677), (4.346693, 50.858306)]]]); SELECT polygonAreaCartesian([]); -- { serverError 36 } \ No newline at end of file diff --git a/tests/queries/0_stateless/01720_country_intersection.sh b/tests/queries/0_stateless/01720_country_intersection.sh index 13efe97becf..77418ef0582 100755 --- a/tests/queries/0_stateless/01720_country_intersection.sh +++ b/tests/queries/0_stateless/01720_country_intersection.sh @@ -7,12 +7,12 @@ CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) ${CLICKHOUSE_CLIENT} -q "drop table if exists country_polygons;" ${CLICKHOUSE_CLIENT} -q "create table country_polygons(name String, p Array(Array(Tuple(Float64, Float64)))) engine=MergeTree() order by tuple();" cat ${CURDIR}/country_polygons.tsv | ${CLICKHOUSE_CLIENT} -q "insert into country_polygons format TSV" -${CLICKHOUSE_CLIENT} -q "SELECT c, d, polygonsIntersectionGeographic(a, b) FROM (SELECT first.p AS a, second.p AS b, first.name AS c, second.name AS d FROM country_polygons AS first CROSS JOIN country_polygons AS second LIMIT 100) format TSV" +${CLICKHOUSE_CLIENT} -q "SELECT c, d, polygonsIntersectionSpherical(a, b) FROM (SELECT first.p AS a, second.p AS b, first.name AS c, second.name AS d FROM country_polygons AS first CROSS JOIN country_polygons AS second LIMIT 100) format TSV" ${CLICKHOUSE_CLIENT} -q "drop table if exists country_polygons;" ${CLICKHOUSE_CLIENT} -q "drop table if exists country_rings;" ${CLICKHOUSE_CLIENT} -q "create table country_rings(name String, p Array(Tuple(Float64, Float64))) engine=MergeTree() order by tuple();" cat ${CURDIR}/country_rings.tsv | ${CLICKHOUSE_CLIENT} -q "insert into country_rings format TSV" -${CLICKHOUSE_CLIENT} -q "SELECT c, d, polygonsIntersectionGeographic(a, b) FROM (SELECT first.p AS a, second.p AS b, first.name AS c, second.name AS d FROM country_rings AS first CROSS JOIN country_rings AS second LIMIT 100) format TSV" +${CLICKHOUSE_CLIENT} -q "SELECT c, d, polygonsIntersectionSpherical(a, b) FROM (SELECT first.p AS a, second.p AS b, first.name AS c, second.name AS d FROM country_rings AS first CROSS JOIN country_rings AS second LIMIT 100) format TSV" ${CLICKHOUSE_CLIENT} -q "drop table if exists country_rings;" \ No newline at end of file diff --git a/tests/queries/0_stateless/01720_country_perimeter_and_area.sh b/tests/queries/0_stateless/01720_country_perimeter_and_area.sh index a9a1e42ebf2..76dc403fb2f 100755 --- a/tests/queries/0_stateless/01720_country_perimeter_and_area.sh +++ b/tests/queries/0_stateless/01720_country_perimeter_and_area.sh @@ -8,9 +8,9 @@ ${CLICKHOUSE_CLIENT} -q "drop table if exists country_polygons;" ${CLICKHOUSE_CLIENT} -q "create table country_polygons(name String, p Array(Array(Tuple(Float64, Float64)))) engine=MergeTree() order by tuple();" cat ${CURDIR}/country_polygons.tsv | ${CLICKHOUSE_CLIENT} -q "insert into country_polygons format TSV" -${CLICKHOUSE_CLIENT} -q "SELECT name, polygonPerimeterGeographic(p) from country_polygons" +${CLICKHOUSE_CLIENT} -q "SELECT name, polygonPerimeterSpherical(p) from country_polygons" ${CLICKHOUSE_CLIENT} -q "SELECT '-------------------------------------'" -${CLICKHOUSE_CLIENT} -q "SELECT name, polygonAreaGeographic(p) from country_polygons" +${CLICKHOUSE_CLIENT} -q "SELECT name, polygonAreaSpherical(p) from country_polygons" ${CLICKHOUSE_CLIENT} -q "SELECT '-------------------------------------'" ${CLICKHOUSE_CLIENT} -q "drop table if exists country_rings;" @@ -18,8 +18,8 @@ ${CLICKHOUSE_CLIENT} -q "drop table if exists country_rings;" ${CLICKHOUSE_CLIENT} -q "create table country_rings(name String, p Array(Tuple(Float64, Float64))) engine=MergeTree() order by tuple();" cat ${CURDIR}/country_rings.tsv | ${CLICKHOUSE_CLIENT} -q "insert into country_rings format TSV" -${CLICKHOUSE_CLIENT} -q "SELECT name, polygonPerimeterGeographic(p) from country_rings" +${CLICKHOUSE_CLIENT} -q "SELECT name, polygonPerimeterSpherical(p) from country_rings" ${CLICKHOUSE_CLIENT} -q "SELECT '-------------------------------------'" -${CLICKHOUSE_CLIENT} -q "SELECT name, polygonAreaGeographic(p) from country_rings" +${CLICKHOUSE_CLIENT} -q "SELECT name, polygonAreaSpherical(p) from country_rings" ${CLICKHOUSE_CLIENT} -q "SELECT '-------------------------------------'" ${CLICKHOUSE_CLIENT} -q "drop table if exists country_rings;" \ No newline at end of file From a6c8d50cc2b1156a36ee904b1550da78b01f7f22 Mon Sep 17 00:00:00 2001 From: Nikita Mikhailov Date: Fri, 26 Feb 2021 18:29:26 +0300 Subject: [PATCH 309/544] review fix --- src/Functions/geometryConverters.cpp | 137 -------------- src/Functions/geometryConverters.h | 178 +++++++++--------- src/Functions/pointInPolygon.cpp | 4 +- src/Functions/polygonArea.cpp | 28 +-- src/Functions/polygonConvexHull.cpp | 5 +- src/Functions/polygonPerimeter.cpp | 5 +- src/Functions/polygonsDistance.cpp | 6 +- src/Functions/polygonsEquals.cpp | 6 +- src/Functions/polygonsIntersection.cpp | 6 +- src/Functions/polygonsSymDifference.cpp | 6 +- src/Functions/polygonsUnion.cpp | 6 +- src/Functions/polygonsWithin.cpp | 6 +- src/Functions/readWkt.cpp | 74 +++----- src/Functions/svg.cpp | 13 +- src/Functions/wkt.cpp | 3 +- src/Functions/ya.make | 1 - src/IO/WriteBufferFromVector.h | 2 +- .../0_stateless/01303_polygons_equals.sql | 2 +- .../01306_polygons_intersection.sql | 2 +- .../0_stateless/01307_polygon_perimeter.sql | 2 +- .../0_stateless/01308_polygon_area.sql | 2 +- 21 files changed, 149 insertions(+), 345 deletions(-) delete mode 100644 src/Functions/geometryConverters.cpp diff --git a/src/Functions/geometryConverters.cpp b/src/Functions/geometryConverters.cpp deleted file mode 100644 index ea6d99787ef..00000000000 --- a/src/Functions/geometryConverters.cpp +++ /dev/null @@ -1,137 +0,0 @@ -#include - -#include - -namespace DB -{ - -namespace ErrorCodes -{ - extern const int BAD_ARGUMENTS; - extern const int LOGICAL_ERROR; - extern const int ILLEGAL_TYPE_OF_ARGUMENT; -} - -template -std::vector PointFromColumnConverter::convertImpl(size_t shift, size_t count) const -{ - const auto * tuple = typeid_cast(col.get()); - const auto & tuple_columns = tuple->getColumns(); - - const auto * x_data = typeid_cast(tuple_columns[0].get()); - const auto * y_data = typeid_cast(tuple_columns[1].get()); - - const auto * first_container = x_data->getData().data() + shift; - const auto * second_container = y_data->getData().data() + shift; - - std::vector answer(count); - - for (size_t i = 0; i < count; ++i) - { - const Float64 first = first_container[i]; - const Float64 second = second_container[i]; - - if (isNaN(first) || isNaN(second)) - throw Exception("Point's component must not be NaN", ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT); - - if (isinf(first) || isinf(second)) - throw Exception("Point's component must not be infinite", ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT); - - answer[i] = Point(first, second); - } - - return answer; -} - -template -std::vector> RingFromColumnConverter::convert() const -{ - const IColumn::Offsets & offsets = typeid_cast(*col).getOffsets(); - size_t prev_offset = 0; - std::vector> answer; - answer.reserve(offsets.size()); - for (size_t offset : offsets) - { - auto tmp = point_converter.convertImpl(prev_offset, offset - prev_offset); - answer.emplace_back(tmp.begin(), tmp.end()); - prev_offset = offset; - } - return answer; -} - -template -std::vector> PolygonFromColumnConverter::convert() const -{ - const IColumn::Offsets & offsets = typeid_cast(*col).getOffsets(); - std::vector> answer(offsets.size()); - auto all_rings = ring_converter.convert(); - - auto prev_offset = 0; - for (size_t iter = 0; iter < offsets.size(); ++iter) - { - const auto current_array_size = offsets[iter] - prev_offset; - answer[iter].outer() = std::move(all_rings[prev_offset]); - answer[iter].inners().reserve(current_array_size); - for (size_t inner_holes = prev_offset + 1; inner_holes < offsets[iter]; ++inner_holes) - answer[iter].inners().emplace_back(std::move(all_rings[inner_holes])); - prev_offset = offsets[iter]; - } - - return answer; -} - - -template -std::vector> MultiPolygonFromColumnConverter::convert() const -{ - const IColumn::Offsets & offsets = typeid_cast(*col).getOffsets(); - size_t prev_offset = 0; - std::vector> answer(offsets.size()); - - auto all_polygons = polygon_converter.convert(); - - for (size_t iter = 0; iter < offsets.size(); ++iter) - { - for (size_t polygon_iter = prev_offset; polygon_iter < offsets[iter]; ++polygon_iter) - answer[iter].emplace_back(std::move(all_polygons[polygon_iter])); - prev_offset = offsets[iter]; - } - - return answer; -} - - -template class PointFromColumnConverter; -template class PointFromColumnConverter; -template class RingFromColumnConverter; -template class RingFromColumnConverter; -template class PolygonFromColumnConverter; -template class PolygonFromColumnConverter; -template class MultiPolygonFromColumnConverter; -template class MultiPolygonFromColumnConverter; - -template typename Desired> -void checkColumnTypeOrThrow(const ColumnWithTypeAndName & column) -{ - DataTypePtr desired_type; - if constexpr (std::is_same_v, Ring>) - desired_type = DataTypeCustomRingSerialization::nestedDataType(); - else if constexpr (std::is_same_v, Polygon>) - desired_type = DataTypeCustomPolygonSerialization::nestedDataType(); - else if constexpr (std::is_same_v, MultiPolygon>) - desired_type = DataTypeCustomMultiPolygonSerialization::nestedDataType(); - else - throw Exception("Unexpected Desired type.", ErrorCodes::LOGICAL_ERROR); - - if (!desired_type->equals(*column.type)) - throw Exception(fmt::format("Expected type {} (MultiPolygon), but got {}", desired_type->getName(), column.type->getName()), ErrorCodes::BAD_ARGUMENTS); -} - -template void checkColumnTypeOrThrow(const ColumnWithTypeAndName &); -template void checkColumnTypeOrThrow(const ColumnWithTypeAndName &); -template void checkColumnTypeOrThrow(const ColumnWithTypeAndName &); -template void checkColumnTypeOrThrow(const ColumnWithTypeAndName &); -template void checkColumnTypeOrThrow(const ColumnWithTypeAndName &); -template void checkColumnTypeOrThrow(const ColumnWithTypeAndName &); - -} diff --git a/src/Functions/geometryConverters.h b/src/Functions/geometryConverters.h index fcce11467d0..283bb1bb7f4 100644 --- a/src/Functions/geometryConverters.h +++ b/src/Functions/geometryConverters.h @@ -3,7 +3,6 @@ #include #include -#include #include #include #include @@ -27,6 +26,7 @@ namespace DB namespace ErrorCodes { extern const int BAD_ARGUMENTS; + extern const int ILLEGAL_TYPE_OF_ARGUMENT; } template @@ -38,128 +38,123 @@ using Polygon = boost::geometry::model::polygon; template using MultiPolygon = boost::geometry::model::multi_polygon>; -template -using Geometry = boost::variant, Polygon, MultiPolygon>; - -template -using Figure = boost::variant, Polygon, MultiPolygon>; - - using CartesianPoint = boost::geometry::model::d2::point_xy; using CartesianRing = Ring; using CartesianPolygon = Polygon; using CartesianMultiPolygon = MultiPolygon; -using CartesianGeometry = Geometry; -// using SphericalPoint = boost::geometry::model::point>; using SphericalPoint = boost::geometry::model::point>; using SphericalRing = Ring; using SphericalPolygon = Polygon; using SphericalMultiPolygon = MultiPolygon; -using SphericalGeometry = Geometry; - - -template -class RingFromColumnConverter; - -template -class PolygonFromColumnConverter; - -template -class MultiPolygonFromColumnConverter; /** - * Class which takes some boost type and returns a pair of numbers. + * Class which takes converts Column with type Tuple(Float64, Float64) to a vector of boost point type. * They are (x,y) in case of cartesian coordinated and (lon,lat) in case of Spherical. */ template -class PointFromColumnConverter +struct ColumnToPointsConverter { -public: - explicit PointFromColumnConverter(ColumnPtr col_) : col(col_) + static std::vector convert(ColumnPtr col) { + const auto * tuple = typeid_cast(col.get()); + const auto & tuple_columns = tuple->getColumns(); + + const auto * x_data = typeid_cast(tuple_columns[0].get()); + const auto * y_data = typeid_cast(tuple_columns[1].get()); + + const auto * first_container = x_data->getData().data(); + const auto * second_container = y_data->getData().data(); + + std::vector answer(col->size()); + + for (size_t i = 0; i < col->size(); ++i) + { + const Float64 first = first_container[i]; + const Float64 second = second_container[i]; + + if (isNaN(first) || isNaN(second)) + throw Exception("Point's component must not be NaN", ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT); + + if (isinf(first) || isinf(second)) + throw Exception("Point's component must not be infinite", ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT); + + answer[i] = Point(first, second); + } + + return answer; } - - std::vector convert() const - { - return convertImpl(0, col->size()); - } - -private: - std::vector convertImpl(size_t shift, size_t count) const; - - friend class RingFromColumnConverter; - ColumnPtr col{nullptr}; }; -template -class RingFromColumnConverter +template +struct ColumnToRingsConverter { -public: - explicit RingFromColumnConverter(ColumnPtr col_) - : col(col_) - , point_converter(typeid_cast(*col_).getDataPtr()) + static std::vector> convert(ColumnPtr col) { + const IColumn::Offsets & offsets = typeid_cast(*col).getOffsets(); + size_t prev_offset = 0; + std::vector> answer; + answer.reserve(offsets.size()); + auto tmp = ColumnToPointsConverter::convert(typeid_cast(*col).getDataPtr()); + for (size_t offset : offsets) + { + answer.emplace_back(tmp.begin() + prev_offset, tmp.begin() + offset); + prev_offset = offset; + } + return answer; } - - std::vector> convert() const; - -private: - friend class PointFromColumnConverter; - /// To prevent use-after-free and increase column lifetime. - ColumnPtr col{nullptr}; - const PointFromColumnConverter point_converter{}; }; -template -class PolygonFromColumnConverter + +template +struct ColumnToPolygonsConverter { -public: - explicit PolygonFromColumnConverter(ColumnPtr col_) - : col(col_) - , ring_converter(typeid_cast(*col_).getDataPtr()) + static std::vector> convert(ColumnPtr col) { + const IColumn::Offsets & offsets = typeid_cast(*col).getOffsets(); + std::vector> answer(offsets.size()); + auto all_rings = ColumnToRingsConverter::convert(typeid_cast(*col).getDataPtr()); + + size_t prev_offset = 0; + for (size_t iter = 0; iter < offsets.size(); ++iter) + { + const auto current_array_size = offsets[iter] - prev_offset; + answer[iter].outer() = std::move(all_rings[prev_offset]); + answer[iter].inners().reserve(current_array_size); + for (size_t inner_holes = prev_offset + 1; inner_holes < offsets[iter]; ++inner_holes) + answer[iter].inners().emplace_back(std::move(all_rings[inner_holes])); + prev_offset = offsets[iter]; + } + + return answer; } - - std::vector> convert() const; - -private: - friend class MultiPolygonFromColumnConverter; - - /// To prevent use-after-free and increase column lifetime. - ColumnPtr col{nullptr}; - const RingFromColumnConverter ring_converter{}; }; -template -class MultiPolygonFromColumnConverter + +template +struct ColumnToMultiPolygonsConverter { -public: - explicit MultiPolygonFromColumnConverter(ColumnPtr col_) - : col(col_) - , polygon_converter(typeid_cast(*col_).getDataPtr()) - {} + static std::vector> convert(ColumnPtr col) + { + const IColumn::Offsets & offsets = typeid_cast(*col).getOffsets(); + size_t prev_offset = 0; + std::vector> answer(offsets.size()); - std::vector> convert() const; + auto all_polygons = ColumnToPolygonsConverter::convert(typeid_cast(*col).getDataPtr()); -private: - /// To prevent use-after-free and increase column lifetime. - ColumnPtr col{nullptr}; - const PolygonFromColumnConverter polygon_converter{}; + for (size_t iter = 0; iter < offsets.size(); ++iter) + { + for (size_t polygon_iter = prev_offset; polygon_iter < offsets[iter]; ++polygon_iter) + answer[iter].emplace_back(std::move(all_polygons[polygon_iter])); + prev_offset = offsets[iter]; + } + + return answer; + } }; -extern template class PointFromColumnConverter; -extern template class PointFromColumnConverter; -extern template class RingFromColumnConverter; -extern template class RingFromColumnConverter; -extern template class PolygonFromColumnConverter; -extern template class PolygonFromColumnConverter; -extern template class MultiPolygonFromColumnConverter; -extern template class MultiPolygonFromColumnConverter; - - /// To serialize Spherical or Cartesian point (a pair of numbers in both cases). template class PointSerializer @@ -256,6 +251,7 @@ public: void add(const Polygon & polygon) { + /// Outer ring + all inner rings (holes). size += 1 + polygon.inners().size(); offsets->insertValue(size); ring_serializer.add(polygon.outer()); @@ -334,13 +330,13 @@ static void callOnGeometryDataType(DataTypePtr type, F && f) { /// There is no Point type, because for most of geometry functions it is useless. if (DataTypeCustomPointSerialization::nestedDataType()->equals(*type)) - return f(ConverterType>()); + return f(ConverterType>()); else if (DataTypeCustomRingSerialization::nestedDataType()->equals(*type)) - return f(ConverterType>()); + return f(ConverterType>()); else if (DataTypeCustomPolygonSerialization::nestedDataType()->equals(*type)) - return f(ConverterType>()); + return f(ConverterType>()); else if (DataTypeCustomMultiPolygonSerialization::nestedDataType()->equals(*type)) - return f(ConverterType>()); + return f(ConverterType>()); throw Exception(fmt::format("Unknown geometry type {}", type->getName()), ErrorCodes::BAD_ARGUMENTS); } diff --git a/src/Functions/pointInPolygon.cpp b/src/Functions/pointInPolygon.cpp index ac5a41f6c81..8e4a05bdec5 100644 --- a/src/Functions/pointInPolygon.cpp +++ b/src/Functions/pointInPolygon.cpp @@ -427,8 +427,8 @@ private: out_container.reserve(end - begin); for (size_t i = begin; i < end; ++i) { - int64_t result = 0; - if (common::mulOverflow(static_cast(x_data[i]), static_cast(y_data[i]), result)) + Int64 result = 0; + if (common::mulOverflow(static_cast(x_data[i]), static_cast(y_data[i]), result)) throw Exception("The coordinates of the point are such that subsequent calculations cannot be performed correctly. " \ "Most likely they are very large in modulus.", ErrorCodes::BAD_ARGUMENTS); diff --git a/src/Functions/polygonArea.cpp b/src/Functions/polygonArea.cpp index 4291d31d72a..fb1ba7c4a01 100644 --- a/src/Functions/polygonArea.cpp +++ b/src/Functions/polygonArea.cpp @@ -69,35 +69,11 @@ public: using TypeConverter = std::decay_t; using Converter = typename TypeConverter::Type; - if constexpr (std::is_same_v, Converter>) + if constexpr (std::is_same_v, Converter>) throw Exception(fmt::format("The argument of function {} must not be Point", getName()), ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT); else { - Converter converter(arguments[0].column->convertToFullColumnIfConst()); - auto geometries = converter.convert(); - - if constexpr (std::is_same_v, Converter>) { - for (auto & polygon : geometries) { - std::cout << "OUTER" << std::endl; - for (auto point : polygon.outer()) { - if constexpr (std::is_same_v) { - std::cout << point.x() << ' ' << point.y() << std::endl; - } else { - std::cout << point.template get<0>() << ' ' << point.template get<1>() << std::endl; - } - } - std::cout << "INNER" << std::endl; - for (auto & inner : polygon.inners()) { - for (auto point : inner) { - if constexpr (std::is_same_v) { - std::cout << point.x() << ' ' << point.y() << std::endl; - } else { - std::cout << point.template get<0>() << ' ' << point.template get<1>() << std::endl; - } - } - } - } - } + auto geometries = Converter::convert(arguments[0].column->convertToFullColumnIfConst()); for (size_t i = 0; i < input_rows_count; i++) res_data.emplace_back(boost::geometry::area(geometries[i])); diff --git a/src/Functions/polygonConvexHull.cpp b/src/Functions/polygonConvexHull.cpp index 3181a1ae3b4..5f545cf8ea1 100644 --- a/src/Functions/polygonConvexHull.cpp +++ b/src/Functions/polygonConvexHull.cpp @@ -67,12 +67,11 @@ public: using TypeConverter = std::decay_t; using Converter = typename TypeConverter::Type; - if constexpr (std::is_same_v>) + if constexpr (std::is_same_v>) throw Exception(fmt::format("The argument of function {} must not be a Point", getName()), ErrorCodes::BAD_ARGUMENTS); else { - Converter converter(arguments[0].column->convertToFullColumnIfConst()); - auto geometries = converter.convert(); + auto geometries = Converter::convert(arguments[0].column->convertToFullColumnIfConst()); for (size_t i = 0; i < input_rows_count; i++) { diff --git a/src/Functions/polygonPerimeter.cpp b/src/Functions/polygonPerimeter.cpp index 9bad13d4817..c3aadbd187a 100644 --- a/src/Functions/polygonPerimeter.cpp +++ b/src/Functions/polygonPerimeter.cpp @@ -69,12 +69,11 @@ public: using TypeConverter = std::decay_t; using Converter = typename TypeConverter::Type; - if constexpr (std::is_same_v, Converter>) + if constexpr (std::is_same_v, Converter>) throw Exception(fmt::format("The argument of function {} must not be Point", getName()), ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT); else { - Converter converter(arguments[0].column->convertToFullColumnIfConst()); - auto geometries = converter.convert(); + auto geometries = Converter::convert(arguments[0].column->convertToFullColumnIfConst()); for (size_t i = 0; i < input_rows_count; i++) res_data.emplace_back(boost::geometry::perimeter(geometries[i])); diff --git a/src/Functions/polygonsDistance.cpp b/src/Functions/polygonsDistance.cpp index f961867caae..074fb5a9798 100644 --- a/src/Functions/polygonsDistance.cpp +++ b/src/Functions/polygonsDistance.cpp @@ -74,12 +74,12 @@ public: using LeftConverter = typename LeftConverterType::Type; using RightConverter = typename RightConverterType::Type; - if constexpr (std::is_same_v, LeftConverter> || std::is_same_v, RightConverter>) + if constexpr (std::is_same_v, LeftConverter> || std::is_same_v, RightConverter>) throw Exception(fmt::format("Any argument of function {} must not be Point", getName()), ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT); else { - auto first = LeftConverter(arguments[0].column->convertToFullColumnIfConst()).convert(); - auto second = RightConverter(arguments[1].column->convertToFullColumnIfConst()).convert(); + auto first = LeftConverter::convert(arguments[0].column->convertToFullColumnIfConst()); + auto second = RightConverter::convert(arguments[1].column->convertToFullColumnIfConst()); for (size_t i = 0; i < input_rows_count; i++) { diff --git a/src/Functions/polygonsEquals.cpp b/src/Functions/polygonsEquals.cpp index 7e4882d5cc2..12f4cf09fa5 100644 --- a/src/Functions/polygonsEquals.cpp +++ b/src/Functions/polygonsEquals.cpp @@ -73,12 +73,12 @@ public: using LeftConverter = typename LeftConverterType::Type; using RightConverter = typename RightConverterType::Type; - if constexpr (std::is_same_v, LeftConverter> || std::is_same_v, RightConverter>) + if constexpr (std::is_same_v, LeftConverter> || std::is_same_v, RightConverter>) throw Exception(fmt::format("Any argument of function {} must not be Point", getName()), ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT); else { - auto first = LeftConverter(arguments[0].column->convertToFullColumnIfConst()).convert(); - auto second = RightConverter(arguments[1].column->convertToFullColumnIfConst()).convert(); + auto first = LeftConverter::convert(arguments[0].column->convertToFullColumnIfConst()); + auto second = RightConverter::convert(arguments[1].column->convertToFullColumnIfConst()); for (size_t i = 0; i < input_rows_count; i++) { diff --git a/src/Functions/polygonsIntersection.cpp b/src/Functions/polygonsIntersection.cpp index 0f7a9b1f264..0de3d023044 100644 --- a/src/Functions/polygonsIntersection.cpp +++ b/src/Functions/polygonsIntersection.cpp @@ -72,12 +72,12 @@ public: using LeftConverter = typename LeftConverterType::Type; using RightConverter = typename RightConverterType::Type; - if constexpr (std::is_same_v, LeftConverter> || std::is_same_v, RightConverter>) + if constexpr (std::is_same_v, LeftConverter> || std::is_same_v, RightConverter>) throw Exception(fmt::format("Any argument of function {} must not be Point", getName()), ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT); else { - auto first = LeftConverter(arguments[0].column->convertToFullColumnIfConst()).convert(); - auto second = RightConverter(arguments[1].column->convertToFullColumnIfConst()).convert(); + auto first = LeftConverter::convert(arguments[0].column->convertToFullColumnIfConst()); + auto second = RightConverter::convert(arguments[1].column->convertToFullColumnIfConst()); /// We are not interested in some pitfalls in third-party libraries /// NOLINTNEXTLINE(clang-analyzer-core.uninitialized.Assign) diff --git a/src/Functions/polygonsSymDifference.cpp b/src/Functions/polygonsSymDifference.cpp index 4768ea93124..9ab6b79c5f5 100644 --- a/src/Functions/polygonsSymDifference.cpp +++ b/src/Functions/polygonsSymDifference.cpp @@ -71,12 +71,12 @@ public: using LeftConverter = typename LeftConverterType::Type; using RightConverter = typename RightConverterType::Type; - if constexpr (std::is_same_v, LeftConverter> || std::is_same_v, RightConverter>) + if constexpr (std::is_same_v, LeftConverter> || std::is_same_v, RightConverter>) throw Exception(fmt::format("Any argument of function {} must not be Point", getName()), ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT); else { - auto first = LeftConverter(arguments[0].column->convertToFullColumnIfConst()).convert(); - auto second = RightConverter(arguments[1].column->convertToFullColumnIfConst()).convert(); + auto first = LeftConverter::convert(arguments[0].column->convertToFullColumnIfConst()); + auto second = RightConverter::convert(arguments[1].column->convertToFullColumnIfConst()); /// NOLINTNEXTLINE(clang-analyzer-core.uninitialized.Assign) for (size_t i = 0; i < input_rows_count; i++) diff --git a/src/Functions/polygonsUnion.cpp b/src/Functions/polygonsUnion.cpp index d4d67982ee0..eab2e2e588f 100644 --- a/src/Functions/polygonsUnion.cpp +++ b/src/Functions/polygonsUnion.cpp @@ -71,12 +71,12 @@ public: using LeftConverter = typename LeftConverterType::Type; using RightConverter = typename RightConverterType::Type; - if constexpr (std::is_same_v, LeftConverter> || std::is_same_v, RightConverter>) + if constexpr (std::is_same_v, LeftConverter> || std::is_same_v, RightConverter>) throw Exception(fmt::format("Any argument of function {} must not be Point", getName()), ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT); else { - auto first = LeftConverter(arguments[0].column->convertToFullColumnIfConst()).convert(); - auto second = RightConverter(arguments[1].column->convertToFullColumnIfConst()).convert(); + auto first = LeftConverter::convert(arguments[0].column->convertToFullColumnIfConst()); + auto second = RightConverter::convert(arguments[1].column->convertToFullColumnIfConst()); /// We are not interested in some pitfalls in third-party libraries /// NOLINTNEXTLINE(clang-analyzer-core.uninitialized.Assign) diff --git a/src/Functions/polygonsWithin.cpp b/src/Functions/polygonsWithin.cpp index eabd5e683b9..68db1494a0d 100644 --- a/src/Functions/polygonsWithin.cpp +++ b/src/Functions/polygonsWithin.cpp @@ -75,12 +75,12 @@ public: using LeftConverter = typename LeftConverterType::Type; using RightConverter = typename RightConverterType::Type; - if constexpr (std::is_same_v, LeftConverter> || std::is_same_v, RightConverter>) + if constexpr (std::is_same_v, LeftConverter> || std::is_same_v, RightConverter>) throw Exception(fmt::format("Any argument of function {} must not be Point", getName()), ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT); else { - auto first = LeftConverter(arguments[0].column->convertToFullColumnIfConst()).convert(); - auto second = RightConverter(arguments[1].column->convertToFullColumnIfConst()).convert(); + auto first = LeftConverter::convert(arguments[0].column->convertToFullColumnIfConst()); + auto second = RightConverter::convert(arguments[1].column->convertToFullColumnIfConst()); /// NOLINTNEXTLINE(clang-analyzer-core.uninitialized.Assign) for (size_t i = 0; i < input_rows_count; i++) diff --git a/src/Functions/readWkt.cpp b/src/Functions/readWkt.cpp index 0e595d3d5ae..101902a00f5 100644 --- a/src/Functions/readWkt.cpp +++ b/src/Functions/readWkt.cpp @@ -16,12 +16,20 @@ namespace ErrorCodes extern const int ILLEGAL_TYPE_OF_ARGUMENT; } -template + +template class FunctionReadWkt : public IFunction { public: explicit FunctionReadWkt() = default; + static constexpr const char * name = NameHolder::name; + + String getName() const override + { + return name; + } + size_t getNumberOfArguments() const override { return 1; @@ -59,71 +67,39 @@ public: { return true; } -}; -class FunctionReadWktPoint : public FunctionReadWkt> -{ -public: - static inline const char * name = "readWktPoint"; - String getName() const override - { - return name; - } static FunctionPtr create(const Context &) { - return std::make_shared(); + return std::make_shared>(); } }; - -class FunctionReadWktRing : public FunctionReadWkt> +struct ReadWktPointNameHolder { -public: - static inline const char * name = "readWktRing"; - String getName() const override - { - return name; - } - static FunctionPtr create(const Context &) - { - return std::make_shared(); - } + static constexpr const char * name = "readWktPoint"; }; -class FunctionReadWktPolygon : public FunctionReadWkt> +struct ReadWktRingNameHolder { -public: - static inline const char * name = "readWktPolygon"; - String getName() const override - { - return name; - } - static FunctionPtr create(const Context &) - { - return std::make_shared(); - } + static constexpr const char * name = "readWktRing"; }; -class FunctionReadWktMultiPolygon : public FunctionReadWkt> +struct ReadWktPolygonNameHolder { -public: - static inline const char * name = "readWktMultiPolygon"; - String getName() const override - { - return name; - } - static FunctionPtr create(const Context &) - { - return std::make_shared(); - } + static constexpr const char * name = "readWktPolygon"; +}; + +struct ReadWktMultiPolygonNameHolder +{ + static constexpr const char * name = "readWktMultiPolygon"; }; void registerFunctionReadWkt(FunctionFactory & factory) { - factory.registerFunction(); - factory.registerFunction(); - factory.registerFunction(); - factory.registerFunction(); + factory.registerFunction, ReadWktPointNameHolder>>(); + factory.registerFunction, ReadWktRingNameHolder>>(); + factory.registerFunction, ReadWktPolygonNameHolder>>(); + factory.registerFunction, ReadWktMultiPolygonNameHolder>>(); } } diff --git a/src/Functions/svg.cpp b/src/Functions/svg.cpp index 873a42722c3..d8a16af1e1d 100644 --- a/src/Functions/svg.cpp +++ b/src/Functions/svg.cpp @@ -64,21 +64,18 @@ public: ColumnPtr executeImpl(const ColumnsWithTypeAndName & arguments, const DataTypePtr & /*result_type*/, size_t input_rows_count) const override { - auto res_column = ColumnString::create(); + bool has_style = arguments.size() > 1; + ColumnPtr style; + if (has_style) + style = arguments[1].column; callOnGeometryDataType(arguments[0].type, [&] (const auto & type) { using TypeConverter = std::decay_t; using Converter = typename TypeConverter::Type; - Converter converter(arguments[0].column->convertToFullColumnIfConst()); - auto figures = converter.convert(); - - bool has_style = arguments.size() > 1; - ColumnPtr style; - if (has_style) - style = arguments[1].column; + auto figures = Converter::convert(arguments[0].column->convertToFullColumnIfConst()); for (size_t i = 0; i < input_rows_count; i++) { diff --git a/src/Functions/wkt.cpp b/src/Functions/wkt.cpp index 619c3f3aee8..1d509a7c508 100644 --- a/src/Functions/wkt.cpp +++ b/src/Functions/wkt.cpp @@ -45,8 +45,7 @@ public: using TypeConverter = std::decay_t; using Converter = typename TypeConverter::Type; - Converter converter(arguments[0].column->convertToFullColumnIfConst()); - auto figures = converter.convert(); + auto figures = Converter::convert(arguments[0].column->convertToFullColumnIfConst()); for (size_t i = 0; i < input_rows_count; i++) { diff --git a/src/Functions/ya.make b/src/Functions/ya.make index e430d72690d..3ac64828b9c 100644 --- a/src/Functions/ya.make +++ b/src/Functions/ya.make @@ -268,7 +268,6 @@ SRCS( geohashDecode.cpp geohashEncode.cpp geohashesInBox.cpp - geometryConverters.cpp getMacro.cpp getScalar.cpp getSetting.cpp diff --git a/src/IO/WriteBufferFromVector.h b/src/IO/WriteBufferFromVector.h index 1dcf2c3f327..6341a9b698b 100644 --- a/src/IO/WriteBufferFromVector.h +++ b/src/IO/WriteBufferFromVector.h @@ -45,7 +45,7 @@ private: } public: - WriteBufferFromVector(VectorType & vector_) + explicit WriteBufferFromVector(VectorType & vector_) : WriteBuffer(reinterpret_cast(vector_.data()), vector_.size()), vector(vector_) { if (vector.empty()) diff --git a/tests/queries/0_stateless/01303_polygons_equals.sql b/tests/queries/0_stateless/01303_polygons_equals.sql index 1d608ce45fa..42f1bd4693c 100644 --- a/tests/queries/0_stateless/01303_polygons_equals.sql +++ b/tests/queries/0_stateless/01303_polygons_equals.sql @@ -1,2 +1,2 @@ select polygonsEqualsCartesian([[[(0, 0),(0, 3),(1, 2.9),(2, 2.6),(2.6, 2),(2.9, 1),(3, 0),(0, 0)]]], [[[(1., 1.),(1., 4.),(4., 4.),(4., 1.),(1., 1.)]]]); -select polygonsEqualsCartesian([[[(1., 1.),(1., 4.),(4., 4.),(4., 1.)]]], [[[(1., 1.),(1., 4.),(4., 4.),(4., 1.),(1., 1.)]]]); \ No newline at end of file +select polygonsEqualsCartesian([[[(1., 1.),(1., 4.),(4., 4.),(4., 1.)]]], [[[(1., 1.),(1., 4.),(4., 4.),(4., 1.),(1., 1.)]]]); diff --git a/tests/queries/0_stateless/01306_polygons_intersection.sql b/tests/queries/0_stateless/01306_polygons_intersection.sql index fa486a138f4..fd428c3d826 100644 --- a/tests/queries/0_stateless/01306_polygons_intersection.sql +++ b/tests/queries/0_stateless/01306_polygons_intersection.sql @@ -2,4 +2,4 @@ select polygonsIntersectionCartesian([[[(0., 0.),(0., 3.),(1., 2.9),(2., 2.6),(2 select polygonsIntersectionCartesian([[[(0., 0.),(0., 3.),(1., 2.9),(2., 2.6),(2.6, 2.),(2.9, 1.),(3., 0.),(0., 0.)]]], [[[(3., 3.),(3., 4.),(4., 4.),(4., 3.),(3., 3.)]]]); select polygonsIntersectionSpherical([[[(4.346693, 50.858306), (4.367945, 50.852455), (4.366227, 50.840809), (4.344961, 50.833264), (4.338074, 50.848677), (4.346693, 50.858306)]]], [[[(25.0010, 136.9987), (17.7500, 142.5000), (11.3733, 142.5917)]]]); -select polygonsIntersectionSpherical([[[(4.3613577, 50.8651821), (4.349556, 50.8535879), (4.3602419, 50.8435626), (4.3830299, 50.8428851), (4.3904543, 50.8564867), (4.3613148, 50.8651279)]]], [[[(4.346693, 50.858306), (4.367945, 50.852455), (4.366227, 50.840809), (4.344961, 50.833264), (4.338074, 50.848677), (4.346693, 50.858306)]]]); \ No newline at end of file +select polygonsIntersectionSpherical([[[(4.3613577, 50.8651821), (4.349556, 50.8535879), (4.3602419, 50.8435626), (4.3830299, 50.8428851), (4.3904543, 50.8564867), (4.3613148, 50.8651279)]]], [[[(4.346693, 50.858306), (4.367945, 50.852455), (4.366227, 50.840809), (4.344961, 50.833264), (4.338074, 50.848677), (4.346693, 50.858306)]]]); diff --git a/tests/queries/0_stateless/01307_polygon_perimeter.sql b/tests/queries/0_stateless/01307_polygon_perimeter.sql index 2fd0c369379..18f5b385826 100644 --- a/tests/queries/0_stateless/01307_polygon_perimeter.sql +++ b/tests/queries/0_stateless/01307_polygon_perimeter.sql @@ -1 +1 @@ -select polygonPerimeterCartesian([[[(0., 0.), (0., 5.), (5., 5.), (5., 0.), (0., 0.)]]]); \ No newline at end of file +select polygonPerimeterCartesian([[[(0., 0.), (0., 5.), (5., 5.), (5., 0.), (0., 0.)]]]); diff --git a/tests/queries/0_stateless/01308_polygon_area.sql b/tests/queries/0_stateless/01308_polygon_area.sql index 4f02928b450..e3a44ad7d51 100644 --- a/tests/queries/0_stateless/01308_polygon_area.sql +++ b/tests/queries/0_stateless/01308_polygon_area.sql @@ -1,3 +1,3 @@ select polygonAreaCartesian([[[(0., 0.), (0., 5.), (5., 5.), (5., 0.)]]]); select polygonAreaSpherical([[[(4.346693, 50.858306), (4.367945, 50.852455), (4.366227, 50.840809), (4.344961, 50.833264), (4.338074, 50.848677), (4.346693, 50.858306)]]]); -SELECT polygonAreaCartesian([]); -- { serverError 36 } \ No newline at end of file +SELECT polygonAreaCartesian([]); -- { serverError 36 } From 803c3e3d2c3f15e399134f89c98baec6a2a68afa Mon Sep 17 00:00:00 2001 From: Nikita Mikhailov Date: Fri, 26 Feb 2021 19:06:28 +0300 Subject: [PATCH 310/544] better --- src/Functions/greatCircleDistance.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Functions/greatCircleDistance.cpp b/src/Functions/greatCircleDistance.cpp index 70f6d1f1759..801f8b3da7f 100644 --- a/src/Functions/greatCircleDistance.cpp +++ b/src/Functions/greatCircleDistance.cpp @@ -19,7 +19,7 @@ namespace ErrorCodes extern const int ILLEGAL_TYPE_OF_ARGUMENT; } -/** Calculates the distance between two Sphericalal locations. +/** Calculates the distance between two geographical locations. * There are three variants: * greatCircleAngle: calculates the distance on a sphere in degrees: https://en.wikipedia.org/wiki/Great-circle_distance * greatCircleDistance: calculates the distance on a sphere in meters. From 9f3e0874e5d5e82189462d42f062770b14c26c50 Mon Sep 17 00:00:00 2001 From: Nikita Mikhailov Date: Tue, 2 Mar 2021 00:55:20 +0300 Subject: [PATCH 311/544] mercator --- src/Functions/geometryConverters.h | 1 + src/Functions/tests/CMakeLists.txt | 2 + src/Functions/tests/mercator.cpp | 169 +++++++++++++++++++++++++++++ 3 files changed, 172 insertions(+) create mode 100644 src/Functions/tests/mercator.cpp diff --git a/src/Functions/geometryConverters.h b/src/Functions/geometryConverters.h index 283bb1bb7f4..f2706c70e3e 100644 --- a/src/Functions/geometryConverters.h +++ b/src/Functions/geometryConverters.h @@ -43,6 +43,7 @@ using CartesianRing = Ring; using CartesianPolygon = Polygon; using CartesianMultiPolygon = MultiPolygon; +/// Latitude, longitude using SphericalPoint = boost::geometry::model::point>; using SphericalRing = Ring; using SphericalPolygon = Polygon; diff --git a/src/Functions/tests/CMakeLists.txt b/src/Functions/tests/CMakeLists.txt index e69de29bb2d..9e32d4869be 100644 --- a/src/Functions/tests/CMakeLists.txt +++ b/src/Functions/tests/CMakeLists.txt @@ -0,0 +1,2 @@ +add_executable (mercator mercator.cpp) +target_link_libraries (mercator PRIVATE clickhouse_functions) diff --git a/src/Functions/tests/mercator.cpp b/src/Functions/tests/mercator.cpp new file mode 100644 index 00000000000..bc9299fdaf1 --- /dev/null +++ b/src/Functions/tests/mercator.cpp @@ -0,0 +1,169 @@ +#include +#include + +#include + + +constexpr double PI = 3.14159265358979323846; + +/// Convert angle from degrees to radians. +inline constexpr double deg_to_rad(double degree) noexcept { + return degree * (PI / 180.0); +} + +/// Convert angle from radians to degrees. +inline constexpr double rad_to_deg(double radians) noexcept { + return radians * (180.0 / PI); +} + +constexpr double earth_radius_for_epsg3857 = 6378137.0; +// constexpr double max_coordinate_epsg3857 = 20037508.34; + +constexpr inline double lon_to_x(double lon) noexcept { + return earth_radius_for_epsg3857 * deg_to_rad(lon); +} + +// canonical log(tan()) version +inline double lat_to_y_with_tan(double lat) { // not constexpr because math functions aren't + return earth_radius_for_epsg3857 * std::log(std::tan(PI/4 + deg_to_rad(lat)/2)); +} + +constexpr inline double x_to_lon(double x) { + return rad_to_deg(x) / earth_radius_for_epsg3857; +} + +inline double y_to_lat(double y) { // not constexpr because math functions aren't + return rad_to_deg(2 * std::atan(std::exp(y / earth_radius_for_epsg3857)) - PI/2); +} + +/// POLYGON((4.346693 50.858306, 4.367945 50.852455, 4.366227 50.840809, 4.344961 50.833264, 4.338074 50.848677, 4.346693 50.858306)) +/// POLYGON((25.0010 136.9987, 17.7500 142.5000, 11.3733 142.5917)) + + +/// POLYGON((4.3613577 50.8651821, 4.349556 50.8535879, 4.3602419 50.8435626, 4.3830299 50.8428851, 4.3904543 50.8564867, 4.3613148 50.8651279)) +/// POLYGON((4.346693 50.858306, 4.367945 50.852455, 4.366227 50.840809, 4.344961 50.833264, 4.338074 50.848677, 4.346693 50.858306)) + +void mercator(DB::CartesianPolygon & polygon) +{ + for (auto & point : polygon.outer()) + { + point.x(lon_to_x(point.x())); + point.y(lat_to_y_with_tan(point.y())); + } +} + +void reverseMercator(DB::CartesianMultiPolygon & multi_polygon) +{ + for (auto & polygon : multi_polygon) + { + for (auto & point : polygon.outer()) + { + point.x(x_to_lon(point.x())); + point.y(y_to_lat(point.y())); + } + } +} + + +void printMultiPolygon(DB::CartesianMultiPolygon & multi_polygon) +{ + std::cout << "--------------" << std::endl; + for (auto & polygon : multi_polygon) + { + for (auto & point : polygon.outer()) + { + std::cout << point.x() << ' ' << point.y() << std::endl; + } + } + std::cout << "--------------" << std::endl; +} + +void test1() +{ + DB::CartesianPolygon green, blue; + boost::geometry::read_wkt( + "POLYGON((4.346693 50.858306, 4.367945 50.852455, 4.366227 50.840809, 4.344961 50.833264, 4.338074 50.848677, 4.346693 50.858306))", green); + + boost::geometry::read_wkt( + "POLYGON((25.0010 136.9987, 17.7500 142.5000, 11.3733 142.5917))", blue); + + mercator(green); + mercator(blue); + + DB::CartesianMultiPolygon output; + boost::geometry::intersection(green, blue, output); + + reverseMercator(output); + + printMultiPolygon(output); +} + + +// 4.3666052904432435, 50.84337386140151 +// 4.3602419 50.8435626 +// 4.349556 50.8535879 +// 4.3526804582393535 50.856658100365976 +// 4.367945 50.852455 +// 4.3666052904432435 50.84337386140151 + + +void test2() +{ + DB::CartesianPolygon green, blue; + boost::geometry::read_wkt( + "POLYGON((4.3613577 50.8651821, 4.349556 50.8535879, 4.3602419 50.8435626, 4.3830299 50.8428851, 4.3904543 50.8564867, 4.3613148 50.8651279))", green); + + boost::geometry::read_wkt( + "POLYGON((4.346693 50.858306, 4.367945 50.852455, 4.366227 50.840809, 4.344961 50.833264, 4.338074 50.848677, 4.346693 50.858306))", blue); + + boost::geometry::correct(green); + boost::geometry::correct(blue); + + mercator(green); + mercator(blue); + + DB::CartesianMultiPolygon output; + boost::geometry::intersection(green, blue, output); + + reverseMercator(output); + + boost::geometry::correct(output); + + printMultiPolygon(output); +} + + +void test3() +{ + DB::CartesianPolygon green, blue; + boost::geometry::read_wkt( + "POLYGON((4.3613577 50.8651821, 4.349556 50.8535879, 4.3602419 50.8435626, 4.3830299 50.8428851, 4.3904543 50.8564867, 4.3613148 50.8651279))", green); + + boost::geometry::read_wkt( + "POLYGON((4.3613577 50.8651821, 4.349556 50.8535879, 4.3602419 50.8435626, 4.3830299 50.8428851, 4.3904543 50.8564867, 4.3613148 50.8651279))", blue); + + boost::geometry::correct(green); + boost::geometry::correct(blue); + + mercator(green); + mercator(blue); + + DB::CartesianMultiPolygon output; + boost::geometry::intersection(green, blue, output); + + reverseMercator(output); + + boost::geometry::correct(output); + + printMultiPolygon(output); +} + +int main(int argc, char ** argv) +{ + (void) argc; + (void) argv; + test1(); + test2(); + test3(); + return 0; +} From 7e6d1d43fed7f8ca94bab8a1e48dbe8fd6cd3378 Mon Sep 17 00:00:00 2001 From: Nikita Mikhaylov Date: Wed, 3 Mar 2021 19:47:10 +0300 Subject: [PATCH 312/544] more mercator --- src/Functions/geometryConverters.h | 29 +---- src/Functions/geometryTypes.h | 33 ++++++ src/Functions/mercatorConverters.cpp | 111 ++++++++++++++++++ src/Functions/mercatorConverters.h | 85 ++++++++++++++ src/Functions/polygonArea.cpp | 16 --- src/Functions/polygonConvexHull.cpp | 16 --- src/Functions/polygonPerimeter.cpp | 16 --- src/Functions/polygonsDistance.cpp | 18 --- src/Functions/polygonsEquals.cpp | 18 --- src/Functions/polygonsIntersection.cpp | 65 +++++----- src/Functions/polygonsSymDifference.cpp | 62 ++++++---- src/Functions/polygonsUnion.cpp | 61 ++++++---- src/Functions/polygonsWithin.cpp | 60 +++++----- src/Functions/tests/mercator.cpp | 44 +++++++ src/Functions/ya.make | 1 + .../01301_polygons_within.reference | 2 + .../0_stateless/01301_polygons_within.sql | 3 +- .../01305_polygons_union.reference | 5 + .../0_stateless/01305_polygons_union.sql | 6 + .../01306_polygons_intersection.reference | 5 + .../01306_polygons_intersection.sql | 7 ++ 21 files changed, 438 insertions(+), 225 deletions(-) create mode 100644 src/Functions/geometryTypes.h create mode 100644 src/Functions/mercatorConverters.cpp create mode 100644 src/Functions/mercatorConverters.h diff --git a/src/Functions/geometryConverters.h b/src/Functions/geometryConverters.h index f2706c70e3e..c90e6861b85 100644 --- a/src/Functions/geometryConverters.h +++ b/src/Functions/geometryConverters.h @@ -1,12 +1,7 @@ #pragma once -#include #include -#include -#include -#include - #include #include #include @@ -14,12 +9,10 @@ #include #include #include +#include #include #include -#include -#include - namespace DB { @@ -29,26 +22,6 @@ namespace ErrorCodes extern const int ILLEGAL_TYPE_OF_ARGUMENT; } -template -using Ring = boost::geometry::model::ring; - -template -using Polygon = boost::geometry::model::polygon; - -template -using MultiPolygon = boost::geometry::model::multi_polygon>; - -using CartesianPoint = boost::geometry::model::d2::point_xy; -using CartesianRing = Ring; -using CartesianPolygon = Polygon; -using CartesianMultiPolygon = MultiPolygon; - -/// Latitude, longitude -using SphericalPoint = boost::geometry::model::point>; -using SphericalRing = Ring; -using SphericalPolygon = Polygon; -using SphericalMultiPolygon = MultiPolygon; - /** * Class which takes converts Column with type Tuple(Float64, Float64) to a vector of boost point type. * They are (x,y) in case of cartesian coordinated and (lon,lat) in case of Spherical. diff --git a/src/Functions/geometryTypes.h b/src/Functions/geometryTypes.h new file mode 100644 index 00000000000..20344423db5 --- /dev/null +++ b/src/Functions/geometryTypes.h @@ -0,0 +1,33 @@ +#pragma once + +#include + +#include +#include +#include +#include "common/types.h" + +namespace DB +{ + +template +using Ring = boost::geometry::model::ring; + +template +using Polygon = boost::geometry::model::polygon; + +template +using MultiPolygon = boost::geometry::model::multi_polygon>; + +using CartesianPoint = boost::geometry::model::d2::point_xy; +using CartesianRing = Ring; +using CartesianPolygon = Polygon; +using CartesianMultiPolygon = MultiPolygon; + +/// Latitude, longitude +using SphericalPoint = boost::geometry::model::point>; +using SphericalRing = Ring; +using SphericalPolygon = Polygon; +using SphericalMultiPolygon = MultiPolygon; + +} diff --git a/src/Functions/mercatorConverters.cpp b/src/Functions/mercatorConverters.cpp new file mode 100644 index 00000000000..321ad83c16c --- /dev/null +++ b/src/Functions/mercatorConverters.cpp @@ -0,0 +1,111 @@ +#include + +#include + +namespace DB +{ + + +namespace +{ + +constexpr double PI = 3.14159265358979323846; + +constexpr double epsilon = 1e-4; + +/// Convert angle from degrees to radians. +double deg_to_rad(double degree) { + return (degree - epsilon) * (PI / 180.0); +} + +/// Convert angle from radians to degrees. +double rad_to_deg(double radians) { + return radians * (180.0 / PI); +} + +double earth_radius_for_epsg3857 = 6378137.0; +// constexpr double max_coordinate_epsg3857 = 20037508.34; + + +double lon_to_x(double lon) { + return earth_radius_for_epsg3857 * deg_to_rad(lon); +} + +// canonical log(tan()) version +double lat_to_y_with_tan(double lat) { // not constexpr because math functions aren't + return earth_radius_for_epsg3857 * std::log(std::tan(PI/4 + deg_to_rad(lat)/2)); +} + +double x_to_lon(double x) { + return rad_to_deg(x) / earth_radius_for_epsg3857; +} + +double y_to_lat(double y) { // not constexpr because math functions aren't + return rad_to_deg(2 * std::atan(std::exp(y / earth_radius_for_epsg3857)) - PI/2); +} + +} + + +void PointMercatorConverter::forward(CartesianPoint & point) +{ + point.x(lon_to_x(point.template get<0>())); + point.y(lat_to_y_with_tan(point.template get<1>())); +} + + +void RingMercatorConverter::forward(CartesianRing & ring) +{ + for (auto & point : ring) + PointMercatorConverter::forward(point); +} + +void PolygonMercatorConverter::forward(CartesianPolygon & polygon) +{ + RingMercatorConverter::forward(polygon.outer()); + for (auto & hole : polygon.inners()) + RingMercatorConverter::forward(hole); +} + +void MultiPolygonMercatorConverter::forward(CartesianMultiPolygon & multipolygon) +{ + for (auto & polygon : multipolygon) + { + RingMercatorConverter::forward(polygon.outer()); + for (auto & hole : polygon.inners()) + RingMercatorConverter::forward(hole); + } +} + +void PointMercatorConverter::backward(CartesianPoint & point) +{ + point.x(x_to_lon(point.template get<0>())); + point.y(y_to_lat(point.template get<1>())); +} + + +void RingMercatorConverter::backward(CartesianRing & ring) +{ + for (auto & point : ring) + PointMercatorConverter::backward(point); +} + +void PolygonMercatorConverter::backward(CartesianPolygon & polygon) +{ + RingMercatorConverter::backward(polygon.outer()); + for (auto & hole : polygon.inners()) + RingMercatorConverter::backward(hole); +} + +void MultiPolygonMercatorConverter::backward(CartesianMultiPolygon & multipolygon) +{ + for (auto & polygon : multipolygon) + { + RingMercatorConverter::backward(polygon.outer()); + for (auto & hole : polygon.inners()) + RingMercatorConverter::backward(hole); + } +} + +} + diff --git a/src/Functions/mercatorConverters.h b/src/Functions/mercatorConverters.h new file mode 100644 index 00000000000..802370f4c16 --- /dev/null +++ b/src/Functions/mercatorConverters.h @@ -0,0 +1,85 @@ +#pragma once + +#include +#include + +namespace ErrorCodes +{ + extern const int LOGICAL_ERROR; +} + +namespace DB +{ + +/// It must work only with CartesianPoint +class PointMercatorConverter +{ +public: + static void forward(CartesianPoint & point); + static void backward(CartesianPoint & point); +}; + + +class RingMercatorConverter +{ +public: + static void forward(CartesianRing & ring); + static void backward(CartesianRing & ring); +}; + + +class PolygonMercatorConverter +{ +public: + static void forward(CartesianPolygon & polygon); + static void backward(CartesianPolygon & polygon); +}; + + + +class MultiPolygonMercatorConverter +{ +public: + static void forward(CartesianMultiPolygon & polygon); + static void backward(CartesianMultiPolygon & polygon); +}; + + +struct PType +{ + using Type = PType; +}; + + +template +void mercatorForward(Geometry & geometry) +{ + if constexpr (std::is_same_v) + return PointMercatorConverter::forward(geometry); + else if constexpr (std::is_same_v) + return RingMercatorConverter::forward(geometry); + else if constexpr (std::is_same_v) + return PolygonMercatorConverter::forward(geometry); + else if constexpr (std::is_same_v) + return MultiPolygonMercatorConverter::forward(geometry); + else + throw Exception("Unknown geometry type", ErrorCodes::LOGICAL_ERROR); +} + + +template +void mercatorBackward(Geometry & geometry) +{ + if constexpr (std::is_same_v) + return PointMercatorConverter::backward(geometry); + else if constexpr (std::is_same_v) + return RingMercatorConverter::backward(geometry); + else if constexpr (std::is_same_v) + return PolygonMercatorConverter::backward(geometry); + else if constexpr (std::is_same_v) + return MultiPolygonMercatorConverter::backward(geometry); + else + throw Exception("Unknown geometry type", ErrorCodes::LOGICAL_ERROR); +} + +} diff --git a/src/Functions/polygonArea.cpp b/src/Functions/polygonArea.cpp index fb1ba7c4a01..407d9e96a53 100644 --- a/src/Functions/polygonArea.cpp +++ b/src/Functions/polygonArea.cpp @@ -1,22 +1,6 @@ #include #include -#include -#include -#include - -#include - -#include -#include -#include -#include -#include -#include - -#include -#include - namespace DB { diff --git a/src/Functions/polygonConvexHull.cpp b/src/Functions/polygonConvexHull.cpp index 5f545cf8ea1..95c6453b84b 100644 --- a/src/Functions/polygonConvexHull.cpp +++ b/src/Functions/polygonConvexHull.cpp @@ -1,22 +1,6 @@ #include #include -#include -#include -#include - -#include - -#include -#include -#include -#include -#include -#include - -#include -#include - namespace DB { diff --git a/src/Functions/polygonPerimeter.cpp b/src/Functions/polygonPerimeter.cpp index c3aadbd187a..36af2927ade 100644 --- a/src/Functions/polygonPerimeter.cpp +++ b/src/Functions/polygonPerimeter.cpp @@ -1,22 +1,6 @@ #include #include -#include -#include -#include - -#include - -#include -#include -#include -#include -#include -#include - -#include -#include - namespace DB { namespace ErrorCodes diff --git a/src/Functions/polygonsDistance.cpp b/src/Functions/polygonsDistance.cpp index 074fb5a9798..e05ff3f7f36 100644 --- a/src/Functions/polygonsDistance.cpp +++ b/src/Functions/polygonsDistance.cpp @@ -1,24 +1,6 @@ #include #include -#include -#include -#include - -#include - -#include -#include -#include -#include -#include -#include -#include -#include - -#include -#include - namespace DB { diff --git a/src/Functions/polygonsEquals.cpp b/src/Functions/polygonsEquals.cpp index 12f4cf09fa5..b5c3129a616 100644 --- a/src/Functions/polygonsEquals.cpp +++ b/src/Functions/polygonsEquals.cpp @@ -1,24 +1,6 @@ #include #include -#include -#include -#include - -#include - -#include -#include -#include -#include -#include -#include -#include -#include - -#include -#include - namespace DB { namespace ErrorCodes diff --git a/src/Functions/polygonsIntersection.cpp b/src/Functions/polygonsIntersection.cpp index 0de3d023044..3f0b67ed722 100644 --- a/src/Functions/polygonsIntersection.cpp +++ b/src/Functions/polygonsIntersection.cpp @@ -1,22 +1,6 @@ #include #include - -#include -#include -#include - -#include - -#include -#include -#include -#include -#include -#include - -#include -#include -#include +#include namespace DB { @@ -26,11 +10,29 @@ namespace ErrorCodes extern const int ILLEGAL_TYPE_OF_ARGUMENT; } -template +struct IntersectionCartesian +{ + static inline const char * name = "polygonsIntersectionCartesian"; + using Point = CartesianPoint; +}; + +struct IntersectionSpherical +{ + static inline const char * name = "polygonsIntersectionSpherical"; + using Point = SphericalPoint; +}; + +struct IntersectionMercator +{ + static inline const char * name = "polygonsIntersectionMercator"; + using Point = CartesianPoint; +}; + +template class FunctionPolygonsIntersection : public IFunction { public: - static inline const char * name; + static inline const char * name = Holder::name; explicit FunctionPolygonsIntersection() = default; @@ -62,6 +64,8 @@ public: ColumnPtr executeImpl(const ColumnsWithTypeAndName & arguments, const DataTypePtr & /*result_type*/, size_t input_rows_count) const override { + using Point = typename Holder::Point; + MultiPolygonSerializer serializer; callOnTwoGeometryDataTypes(arguments[0].type, arguments[1].type, [&](const auto & left_type, const auto & right_type) @@ -87,10 +91,19 @@ public: boost::geometry::correct(first[i]); boost::geometry::correct(second[i]); + if constexpr (std::is_same_v) + { + mercatorForward(first[i]); + mercatorForward(second[i]); + } + MultiPolygon intersection{}; /// Main work here. boost::geometry::intersection(first[i], second[i], intersection); + if constexpr (std::is_same_v) + mercatorBackward(intersection); + serializer.add(intersection); } } @@ -105,18 +118,10 @@ public: } }; - -template <> -const char * FunctionPolygonsIntersection::name = "polygonsIntersectionCartesian"; - -template <> -const char * FunctionPolygonsIntersection::name = "polygonsIntersectionSpherical"; - - void registerFunctionPolygonsIntersection(FunctionFactory & factory) { - factory.registerFunction>(); - factory.registerFunction>(); + factory.registerFunction>(); + factory.registerFunction>(); + factory.registerFunction>(); } - } diff --git a/src/Functions/polygonsSymDifference.cpp b/src/Functions/polygonsSymDifference.cpp index 9ab6b79c5f5..b7b20f3d6c3 100644 --- a/src/Functions/polygonsSymDifference.cpp +++ b/src/Functions/polygonsSymDifference.cpp @@ -1,21 +1,6 @@ #include #include - -#include -#include -#include - -#include - -#include -#include -#include -#include -#include -#include - -#include -#include +#include namespace DB { @@ -26,11 +11,30 @@ namespace ErrorCodes } -template +struct SymDifferenceCartesian +{ + static inline const char * name = "polygonsSymDifferenceCartesian"; + using Point = CartesianPoint; +}; + +struct SymDifferenceSpherical +{ + static inline const char * name = "polygonsSymDifferenceSpherical"; + using Point = SphericalPoint; +}; + +struct SymDifferenceMercator +{ + static inline const char * name = "polygonsSymDifferenceMercator"; + using Point = CartesianPoint; +}; + + +template class FunctionPolygonsSymDifference : public IFunction { public: - static const char * name; + static inline const char * name = Holder::name; explicit FunctionPolygonsSymDifference() = default; @@ -61,6 +65,8 @@ public: ColumnPtr executeImpl(const ColumnsWithTypeAndName & arguments, const DataTypePtr & /*result_type*/, size_t input_rows_count) const override { + using Point = typename Holder::Point; + MultiPolygonSerializer serializer; callOnTwoGeometryDataTypes(arguments[0].type, arguments[1].type, [&](const auto & left_type, const auto & right_type) @@ -84,9 +90,18 @@ public: boost::geometry::correct(first[i]); boost::geometry::correct(second[i]); + if constexpr (std::is_same_v) + { + mercatorForward(first[i]); + mercatorForward(second[i]); + } + MultiPolygon sym_difference{}; boost::geometry::sym_difference(first[i], second[i], sym_difference); + if constexpr (std::is_same_v) + mercatorBackward(sym_difference); + serializer.add(sym_difference); } } @@ -101,16 +116,11 @@ public: } }; -template <> -const char * FunctionPolygonsSymDifference::name = "polygonsSymDifferenceCartesian"; - -template <> -const char * FunctionPolygonsSymDifference::name = "polygonsSymDifferenceSpherical"; - void registerFunctionPolygonsSymDifference(FunctionFactory & factory) { - factory.registerFunction>(); - factory.registerFunction>(); + factory.registerFunction>(); + factory.registerFunction>(); + factory.registerFunction>(); } } diff --git a/src/Functions/polygonsUnion.cpp b/src/Functions/polygonsUnion.cpp index eab2e2e588f..c2dd93d47d4 100644 --- a/src/Functions/polygonsUnion.cpp +++ b/src/Functions/polygonsUnion.cpp @@ -1,21 +1,6 @@ #include #include - -#include -#include -#include - -#include - -#include -#include -#include -#include -#include -#include - -#include -#include +#include namespace DB { @@ -26,11 +11,30 @@ namespace ErrorCodes } -template +struct UnionCartesian +{ + static inline const char * name = "polygonsUnionCartesian"; + using Point = CartesianPoint; +}; + +struct UnionSpherical +{ + static inline const char * name = "polygonsUnionSpherical"; + using Point = SphericalPoint; +}; + +struct UnionMercator +{ + static inline const char * name = "polygonsUnionMercator"; + using Point = CartesianPoint; +}; + + +template class FunctionPolygonsUnion : public IFunction { public: - static inline const char * name; + static inline const char * name = Holder::name ; explicit FunctionPolygonsUnion() = default; @@ -61,6 +65,7 @@ public: ColumnPtr executeImpl(const ColumnsWithTypeAndName & arguments, const DataTypePtr & /*result_type*/, size_t input_rows_count) const override { + using Point = typename Holder::Point; MultiPolygonSerializer serializer; callOnTwoGeometryDataTypes(arguments[0].type, arguments[1].type, [&](const auto & left_type, const auto & right_type) @@ -86,10 +91,19 @@ public: boost::geometry::correct(first[i]); boost::geometry::correct(second[i]); + if constexpr (std::is_same_v) + { + mercatorForward(first[i]); + mercatorForward(second[i]); + } + MultiPolygon polygons_union{}; /// Main work here. boost::geometry::union_(first[i], second[i], polygons_union); + if constexpr (std::is_same_v) + mercatorBackward(polygons_union); + serializer.add(polygons_union); } } @@ -104,17 +118,12 @@ public: } }; -template <> -const char * FunctionPolygonsUnion::name = "polygonsUnionCartesian"; - -template <> -const char * FunctionPolygonsUnion::name = "polygonsUnionSpherical"; - void registerFunctionPolygonsUnion(FunctionFactory & factory) { - factory.registerFunction>(); - factory.registerFunction>(); + factory.registerFunction>(); + factory.registerFunction>(); + factory.registerFunction>(); } } diff --git a/src/Functions/polygonsWithin.cpp b/src/Functions/polygonsWithin.cpp index 68db1494a0d..47ecdaf2c8e 100644 --- a/src/Functions/polygonsWithin.cpp +++ b/src/Functions/polygonsWithin.cpp @@ -1,23 +1,6 @@ #include #include - -#include -#include -#include - -#include - -#include -#include -#include -#include -#include -#include -#include -#include - -#include -#include +#include namespace DB { @@ -27,12 +10,29 @@ namespace ErrorCodes extern const int ILLEGAL_TYPE_OF_ARGUMENT; } +struct WithinCartesian +{ + static inline const char * name = "polygonsWithinCartesian"; + using Point = CartesianPoint; +}; -template +struct WithinSpherical +{ + static inline const char * name = "polygonsWithinSpherical"; + using Point = SphericalPoint; +}; + +struct WithinMercator +{ + static inline const char * name = "polygonsWithinMercator"; + using Point = CartesianPoint; +}; + +template class FunctionPolygonsWithin : public IFunction { public: - static inline const char * name; + static inline const char * name = Holder::name; explicit FunctionPolygonsWithin() = default; @@ -63,6 +63,7 @@ public: ColumnPtr executeImpl(const ColumnsWithTypeAndName & arguments, const DataTypePtr & /*result_type*/, size_t input_rows_count) const override { + using Point = typename Holder::Point; auto res_column = ColumnUInt8::create(); auto & res_data = res_column->getData(); res_data.reserve(input_rows_count); @@ -88,6 +89,12 @@ public: boost::geometry::correct(first[i]); boost::geometry::correct(second[i]); + if constexpr (std::is_same_v) + { + mercatorForward(first[i]); + mercatorForward(second[i]); + } + res_data.emplace_back(boost::geometry::within(first[i], second[i])); } } @@ -102,18 +109,11 @@ public: } }; - -template <> -const char * FunctionPolygonsWithin::name = "polygonsWithinCartesian"; - -template <> -const char * FunctionPolygonsWithin::name = "polygonsWithinSpherical"; - - void registerFunctionPolygonsWithin(FunctionFactory & factory) { - factory.registerFunction>(); - factory.registerFunction>(); + factory.registerFunction>(); + factory.registerFunction>(); + factory.registerFunction>(); } } diff --git a/src/Functions/tests/mercator.cpp b/src/Functions/tests/mercator.cpp index bc9299fdaf1..7174cd62550 100644 --- a/src/Functions/tests/mercator.cpp +++ b/src/Functions/tests/mercator.cpp @@ -1,7 +1,10 @@ +#include #include #include +#include #include +#include constexpr double PI = 3.14159265358979323846; @@ -78,6 +81,9 @@ void printMultiPolygon(DB::CartesianMultiPolygon & multi_polygon) std::cout << "--------------" << std::endl; } + +const char * example = "MULTIPOLYGON (((11.737519 -16.692578, 11.738507 -16.705822, 11.715794 -16.685006, 11.701973 -16.655674, 11.669394 -16.55721, 11.67137 -16.529744, 11.681244 -16.517436, 11.701977 -16.500393, 11.719746 -16.506083, 11.721528 -16.555434, 11.725759 -16.577325, 11.725759 -16.618422, 11.726659 -16.656627, 11.737519 -16.692578)), ((13.98233 -5.853285, 14.006307 -5.866928, 14.02119 -5.872509, 14.140149 -5.863517, 14.163094 -5.866101, 14.222212 -5.886978, 14.241849 -5.889562, 14.346132 -5.891526, 14.436669 -5.893386, 14.482144 -5.885531, 14.545706 -5.901654, 14.576609 -5.904651, 14.703733 -5.881397, 14.759652 -5.880469, 14.797164 -5.879847, 14.888424 -5.878296, 14.979581 -5.876746, 15.070842 -5.875299, 15.162102 -5.873749, 15.253363 -5.872199, 15.34452 -5.870752, 15.435781 -5.869201, 15.527041 -5.867754, 15.618302 -5.866204, 15.709563 -5.864757, 15.80072 -5.863207, 15.892084 -5.861657, 15.983241 -5.860106, 16.074502 -5.858556, 16.100856 -5.858108, 16.165659 -5.857006, 16.257023 -5.855559, 16.315727 -5.854629, 16.341255 -5.859796, 16.358515 -5.870235, 16.374328 -5.882844, 16.396136 -5.894109, 16.412465 -5.896797, 16.427452 -5.894626, 16.459491 -5.885221, 16.479335 -5.882224, 16.497318 -5.882844, 16.515095 -5.887081, 16.552922 -5.902378, 16.56026 -5.906408, 16.597364 -5.924702, 16.594056 -5.933177, 16.580621 -5.94899, 16.580724 -5.955398, 16.590336 -5.968007, 16.595297 -5.978549, 16.597157 -5.98785, 16.601188 -6.050586, 16.604288 -6.061851, 16.610076 -6.061231, 16.619481 -6.05658, 16.62837 -6.05534, 16.632194 -6.065262, 16.61111 -6.089756, 16.630953 -6.09172, 16.643046 -6.100712, 16.660099 -6.123966, 16.671778 -6.130167, 16.68511 -6.134301, 16.695859 -6.140916, 16.700407 -6.154352, 16.717873 -6.174092, 16.726762 -6.189182, 16.724384 -6.202411, 16.717253 -6.215123, 16.714773 -6.231556, 16.715393 -6.304834, 16.716943 -6.313309, 16.720147 -6.32261, 16.722317 -6.332429, 16.720974 -6.342351, 16.715289 -6.345968, 16.706091 -6.346692, 16.697409 -6.349792, 16.693585 -6.359817, 16.692965 -6.398368, 16.694722 -6.402192, 16.700407 -6.411287, 16.707021 -6.415421, 16.713636 -6.413974, 16.718907 -6.415318, 16.720974 -6.42834, 16.715186 -6.441363, 16.706091 -6.455212, 16.706091 -6.465031, 16.727795 -6.465961, 16.721284 -6.480534, 16.721697 -6.491799, 16.728002 -6.501101, 16.738337 -6.510609, 16.747329 -6.521978, 16.746812 -6.53159, 16.734513 -6.552364, 16.730792 -6.56983, 16.733996 -6.586057, 16.748672 -6.619543, 16.753013 -6.637216, 16.755907 -6.674113, 16.760351 -6.691994, 16.768103 -6.708323, 16.823086 -6.785114, 16.82691 -6.799791, 16.828564 -6.814363, 16.832905 -6.827799, 16.84448 -6.838445, 16.855332 -6.842165, 16.879414 -6.844646, 16.890783 -6.847126, 16.914967 -6.864076, 16.926956 -6.888571, 16.934501 -6.945725, 16.932124 -6.949859, 16.92861 -6.959264, 16.928713 -6.968669, 16.937601 -6.97301, 16.949074 -6.975387, 16.955275 -6.981485, 16.959305 -6.989443, 16.964576 -6.997195, 16.973155 -7.014765, 16.969331 -7.030371, 16.962303 -7.047424, 16.961166 -7.069231, 16.929126 -7.060653, 16.927059 -7.085665, 16.947523 -7.151087, 16.945663 -7.204624, 16.950831 -7.213202, 16.960546 -7.219093, 16.972845 -7.232219, 16.989071 -7.254853, 16.991655 -7.270563, 16.991655 -7.286066, 16.996099 -7.297951, 17.022867 -7.306013, 17.056664 -7.329887, 17.082502 -7.34229, 17.093354 -7.352315, 17.097798 -7.367404, 17.097895 -7.378294, 17.098108 -7.402338, 17.103999 -7.416084, 17.118159 -7.425489, 17.12622 -7.411433, 17.132215 -7.409572, 17.138209 -7.413707, 17.146167 -7.418047, 17.167458 -7.423525, 17.176553 -7.428176, 17.180274 -7.435411, 17.1749 -7.441612, 17.163117 -7.445539, 17.151542 -7.451224, 17.146167 -7.463109, 17.148545 -7.472618, 17.154952 -7.479129, 17.163427 -7.485227, 17.172832 -7.493805, 17.186268 -7.516336, 17.202908 -7.564292, 17.217791 -7.585996, 17.266574 -7.610387, 17.286417 -7.625167, 17.282697 -7.643977, 17.287451 -7.660203, 17.286521 -7.683664, 17.288898 -7.699477, 17.303781 -7.692449, 17.305951 -7.700097, 17.308121 -7.701854, 17.308121 -7.702371, 17.303781 -7.706712, 17.32042 -7.713637, 17.320627 -7.722422, 17.314219 -7.731827, 17.310602 -7.740818, 17.315976 -7.753634, 17.334063 -7.769654, 17.337887 -7.778646, 17.342435 -7.78402, 17.365482 -7.787327, 17.375404 -7.792288, 17.376954 -7.801693, 17.376128 -7.814096, 17.380985 -7.823191, 17.400002 -7.823397, 17.400002 -7.830219, 17.3907 -7.83921, 17.397625 -7.841381, 17.410854 -7.841484, 17.420569 -7.843861, 17.423567 -7.855127, 17.42305 -7.867942, 17.426357 -7.875487, 17.44093 -7.871146, 17.452402 -7.883962, 17.448681 -7.89285, 17.43938 -7.902152, 17.434109 -7.915795, 17.437829 -7.922409, 17.446821 -7.928921, 17.468318 -7.939463, 17.463564 -7.946904, 17.463047 -7.949798, 17.465218 -7.952588, 17.468318 -7.959926, 17.471109 -7.955586, 17.477207 -7.950521, 17.481858 -7.945664, 17.490953 -7.977496, 17.496224 -7.987212, 17.506352 -7.996513, 17.528366 -8.007778999999999, 17.536531 -8.015117, 17.540045 -8.032067, 17.528366 -8.060592, 17.536219 -8.075402, 17.536531 -8.075991999999999, 17.544283 -8.084777000000001, 17.600197 -8.098523, 17.628412 -8.098006, 17.713265 -8.068344, 17.74334 -8.070721000000001, 17.77724 -8.08333, 17.789539 -8.081778999999999, 17.851757 -8.047776000000001, 17.867984 -8.044468999999999, 17.901367 -8.048086, 17.930822 -8.062142, 17.981879 -8.105344000000001, 18.008544 -8.111235000000001, 18.098151 -8.109271, 18.091743 -8.084156999999999, 18.09288 -8.056355, 18.102181 -8.033307000000001, 18.120372 -8.022455000000001, 18.135771 -8.018528, 18.176596 -7.999201, 18.192719 -7.996927, 18.267339 -7.999821, 18.316225 -8.001681, 18.353639 -8.015943999999999, 18.366558 -8.016771, 18.381958 -8.012326, 18.403868 -8.000647000000001, 18.420508 -7.99827, 18.481797 -7.99734, 18.487894 -7.99827, 18.507635 -7.997547, 18.50226 -7.960133, 18.504327 -7.940703, 18.516523 -7.932228, 18.588663 -7.932331, 18.666901 -7.932331, 18.738525 -7.932331, 18.74731 -7.937809, 18.752064 -7.965921, 18.758265 -7.97822, 18.773768 -8.001991, 18.818003 -8.001991, 18.945437 -8.001991, 19.073078 -8.001991, 19.200616 -8.001991, 19.328256 -8.001991, 19.355542 -8.001991, 19.334044 -7.92427, 19.330944 -7.898638, 19.333838 -7.867322, 19.348514 -7.821537, 19.358022 -7.791565, 19.377762 -7.73007, 19.38138 -7.703921, 19.380966 -7.679737, 19.373215 -7.653795, 19.361846 -7.627027, 19.353681 -7.600258, 19.355645 -7.57411, 19.424685 -7.579691, 19.443908 -7.575247, 19.458274 -7.560261, 19.468713 -7.52109, 19.482356 -7.50352, 19.483079 -7.5029, 19.483906 -7.502383, 19.484733 -7.50197, 19.513155 -7.479439, 19.513775 -7.448433, 19.487213 -7.381977, 19.479462 -7.347664, 19.476361 -7.315935, 19.487213 -7.153877, 19.497032 -7.121735, 19.508607 -7.103235, 19.536202 -7.068405, 19.54261 -7.052282, 19.540233 -7.041636, 19.521836 -7.001949, 19.57403 -7.001949, 19.616094 -7.001949, 19.658055 -7.001949, 19.700017 -7.001949, 19.741978 -7.001949, 19.784043 -7.001949, 19.826107 -7.001949, 19.868068 -7.001949, 19.91003 -7.001949, 19.952094 -7.001949, 19.963079 -7.001949, 19.994056 -7.001949, 20.03612 -7.001949, 20.078185 -7.001949, 20.120146 -7.001949, 20.162107 -7.001949, 20.204069 -7.001949, 20.246237 -7.001949, 20.294296 -7.001949, 20.294192 -6.987479, 20.298637 -6.95451, 20.303184 -6.940144, 20.311246 -6.925054, 20.319514 -6.918543, 20.331089 -6.916786, 20.383696 -6.916683, 20.467308 -6.916373, 20.553711 -6.916166, 20.611486 -6.915959, 20.588128 -6.976834, 20.560946 -7.047837, 20.545443 -7.088558, 20.524463 -7.143129, 20.518261 -7.189741, 20.519398 -7.23718, 20.520535 -7.286376, 20.567044 -7.286169, 20.639804 -7.285962, 20.712565 -7.285859, 20.785222 -7.285549, 20.857982 -7.285342, 20.930743 -7.285239, 21.003503 -7.284929, 21.076263 -7.284722, 21.148921 -7.284619, 21.221784 -7.284412, 21.294338 -7.284205, 21.367098 -7.283999, 21.439962 -7.283792, 21.512619 -7.283585, 21.585483 -7.283379, 21.658243 -7.283172, 21.730901 -7.282965, 21.76449 -7.282758, 21.784954 -7.283379, 21.784644 -7.286893, 21.788261 -7.29175, 21.794773 -7.294024, 21.802524 -7.298882, 21.808829 -7.306426, 21.831359 -7.354175, 21.836941 -7.360583, 21.842418 -7.369885, 21.839938 -7.378153, 21.834977 -7.385285, 21.83291 -7.390762, 21.838698 -7.402028, 21.845726 -7.409056, 21.851617 -7.416497, 21.854097 -7.428899, 21.85296 -7.434997, 21.847793 -7.444092, 21.846656 -7.449053, 21.848929 -7.454841, 21.858645 -7.464039, 21.860918 -7.469827, 21.859058 -7.482643, 21.849963 -7.5029, 21.846656 -7.514269, 21.845932 -7.526775, 21.848206 -7.532976, 21.851513 -7.538453, 21.854097 -7.548375, 21.854097 -7.585996, 21.85079 -7.593644, 21.836217 -7.60832, 21.83291 -7.614211, 21.831049 -7.632815, 21.826558 -7.645601, 21.819577 -7.665474, 21.807898 -7.699374, 21.795083 -7.736271, 21.787228 -7.758595, 21.776169 -7.790324, 21.770485 -7.806758, 21.764594 -7.850682, 21.770898 -7.876521, 21.772138 -7.8882, 21.768831 -7.899878, 21.754362 -7.919619, 21.751054 -7.929231, 21.753948 -7.99734, 21.758496 -8.015117, 21.768211 -8.023178, 21.79963 -8.040851999999999, 21.806348 -8.052427, 21.808519 -8.076715, 21.815133 -8.096766000000001, 21.825985 -8.114439000000001, 21.851203 -8.143481, 21.857301 -8.156917, 21.860195 -8.173764, 21.862469 -8.216448, 21.873217 -8.264094, 21.881382 -8.282076999999999, 21.910631 -8.310499, 21.920553 -8.329000000000001, 21.913146 -8.333003, 21.905257 -8.337268, 21.899882 -8.348843, 21.911458 -8.374268000000001, 21.935953 -8.413024999999999, 21.943911 -8.455916999999999, 21.939983 -8.499945, 21.916212 -8.576840000000001, 21.921276 -8.578597, 21.924377 -8.580767, 21.928408 -8.582523999999999, 21.935953 -8.583144000000001, 21.917762 -8.615183999999999, 21.90691 -8.645466000000001, 21.895128 -8.718743, 21.884793 -8.782304999999999, 21.877765 -8.78923, 21.870117 -8.822303, 21.859885 -8.846384, 21.85172 -8.925862, 21.854097 -8.977849000000001, 21.839731 -9.092364, 21.851927 -9.192926, 21.851617 -9.233751, 21.841075 -9.275815, 21.810172 -9.338137, 21.803764 -9.372966999999999, 21.793326 -9.393534000000001, 21.791982 -9.406143, 21.79622 -9.417097999999999, 21.813169 -9.437459, 21.819887 -9.447691000000001, 21.832186 -9.488412, 21.843658 -9.576159000000001, 21.854097 -9.61781, 21.878902 -9.657083999999999, 21.94236 -9.72292, 21.95652 -9.76178, 21.962721 -9.758473, 21.978224 -9.752167999999999, 21.984425 -9.748758, 21.986285 -9.781107, 21.998378 -9.809116, 22.055842 -9.875365, 22.065764 -9.881049000000001, 22.072792 -9.878879, 22.079923 -9.871644, 22.090982 -9.877122, 22.107622 -9.891385, 22.146276 -9.915673, 22.159918 -9.931486, 22.167566 -9.953707, 22.175628 -9.988950000000001, 22.196092 -10.043624, 22.204877 -10.085792, 22.209734 -10.098194, 22.20281 -10.130647, 22.221103 -10.176225, 22.245701 -10.221184, 22.263478 -10.268209, 22.286939 -10.309137, 22.295517 -10.339316, 22.31009 -10.356989, 22.313397 -10.368565, 22.31226 -10.379934, 22.307093 -10.399571, 22.305956 -10.409803, 22.301098 -10.424892, 22.278671 -10.460962, 22.271849 -10.481736, 22.27402 -10.503027, 22.284045 -10.516463, 22.296447 -10.529692, 22.305956 -10.550673, 22.30947 -10.592944, 22.308573 -10.612519, 22.307609 -10.633562, 22.31009 -10.668805, 22.32642 -10.694643, 22.321045 -10.700741, 22.319495 -10.703842, 22.322389 -10.742186, 22.319495 -10.761306, 22.309677 -10.769781, 22.29066 -10.772675, 22.265855 -10.779909, 22.223997 -10.797066, 22.206014 -10.809882, 22.182759 -10.830449, 22.165499 -10.85236, 22.165706 -10.868793, 22.174801 -10.884296, 22.196092 -10.954162, 22.196092 -10.992299, 22.199296 -10.997364, 22.213972 -11.007906, 22.217176 -11.012763, 22.220586 -11.030023, 22.23733 -11.059376, 22.243841 -11.078289, 22.25552 -11.166552, 22.252006 -11.208617, 22.23764 -11.249545, 22.263064 -11.248201, 22.281771 -11.236729, 22.325696 -11.190324, 22.340269 -11.180195, 22.440315 -11.146915, 22.469357 -11.122421, 22.500983 -11.042426, 22.53757 -11.037155, 22.613637 -11.06754, 22.650948 -11.08232, 22.688982 -11.092965, 22.727739 -11.096789, 22.764222 -11.090485, 22.785306 -11.078496, 22.806907 -11.063096, 22.829231 -11.052141, 22.853106 -11.053071, 22.87357 -11.063716, 22.91119 -11.088314, 22.931137 -11.096789, 22.969585 -11.102474, 23.014336 -11.102474, 23.114692 -11.085524, 23.196754 -11.071881, 23.306515 -11.01173, 23.373281 -10.975143, 23.415449 -10.963257, 23.455963 -10.960984, 23.494824 -10.964291, 23.557042 -10.981344, 23.641585 -11.004495, 23.674865 -11.006976, 23.714139 -11.003875, 23.752276 -11.008216, 23.79372 -11.021135, 23.833718 -11.028473, 23.867411 -11.016381, 23.874129 -11.006872, 23.881777 -10.984548, 23.887771 -10.974419, 23.89728 -10.966461, 23.918571 -10.954059, 23.927976 -10.945377, 23.936141 -10.925223, 23.940585 -10.901556, 23.948646 -10.881505, 23.967457 -10.872307, 23.974278 -10.921399, 23.980582 -10.938349, 24.000013 -10.967805, 24.003733 -10.982481, 23.997325 -11.001705, 23.993708 -11.019585, 23.994535 -11.074982, 23.990091 -11.113532, 23.995775 -11.127382, 24.015309 -11.130482, 24.011175 -11.272903, 24.016963 -11.298431, 24.061094 -11.394962, 24.061714 -11.406951, 24.052102 -11.42049, 24.020787 -11.444572, 24.009935 -11.459454, 24.007557 -11.470513, 24.007247 -11.483122, 24.009831 -11.507203, 24.009728 -11.52343, 24.005387 -11.535212, 23.977688 -11.577277, 23.959912 -11.617171, 23.954641 -11.636911, 23.954641 -11.662233, 23.962289 -11.68156, 23.972727 -11.700577, 23.981306 -11.724761, 23.985543 -11.799485, 23.990194 -11.824083, 23.98854 -11.834212, 23.967043 -11.882891, 23.961049 -12.011669, 23.954331 -12.151919, 23.959602 -12.19667, 23.981306 -12.227676, 24.006627 -12.253721, 24.016859 -12.278939, 24.020993 -12.340021, 24.030812 -12.385083, 24.028021 -12.402136, 24.019856 -12.419189, 23.98606 -12.467662, 23.940791 -12.532774, 23.928699 -12.561609, 23.910716 -12.631269, 23.891699 -12.705063, 23.872269 -12.750125, 23.865654 -12.789709, 23.874749 -12.821749, 23.895109 -12.849757, 23.949576 -12.904638, 23.971797 -12.93337, 23.988851 -12.965099, 24.000633 -13.001479, 23.968283 -13.001479, 23.844157 -13.001479, 23.719823 -13.001479, 23.595593 -13.001479, 23.471259 -13.001479, 23.346926 -13.001479, 23.222696 -13.001479, 23.098362 -13.001479, 22.974029 -13.001479, 22.849799 -13.001479, 22.725465 -13.001479, 22.601235 -13.001479, 22.477005 -13.001479, 22.352775 -13.001479, 22.228545 -13.001479, 22.104211 -13.001479, 21.979878 -13.001479, 21.979981 -13.152168, 21.980084 -13.302701, 21.980188 -13.453493, 21.980291 -13.604181, 21.980353 -13.694822, 21.980394 -13.754973, 21.980498 -13.905558, 21.980601 -14.05635, 21.980704 -14.206935, 21.980808 -14.357624, 21.980911 -14.508416, 21.981014 -14.659104, 21.981118 -14.809793, 21.981221 -14.960481, 21.981325 -15.11117, 21.981428 -15.261858, 21.981531 -15.412546, 21.981531 -15.452337, 21.981531 -15.473525, 21.981531 -15.503807, 21.981531 -15.547835, 21.981428 -15.649948, 21.981221 -15.751957, 21.980911 -15.853967, 21.980808 -15.956079, 21.980601 -16.001244, 21.981531 -16.004035, 21.981531 -16.067494, 21.981531 -16.128162, 21.981531 -16.144285, 21.983805 -16.165886, 22.010367 -16.198132, 22.045197 -16.252289, 22.019875 -16.253115, 22.01171 -16.252289, 22.022356 -16.266345, 22.025973 -16.278954, 22.02773 -16.291666, 22.032174 -16.306342, 22.036618 -16.312233, 22.048607 -16.322362, 22.052018 -16.326806, 22.053672 -16.336521, 22.054188 -16.358639, 22.055842 -16.364633, 22.089122 -16.371971, 22.105555 -16.379413, 22.103591 -16.391918, 22.094289 -16.404734, 22.086021 -16.422304, 22.08075 -16.441011, 22.079923 -16.457754, 22.084574 -16.470156, 22.101731 -16.497958, 22.107312 -16.512324, 22.106588 -16.525967, 22.104004 -16.536716, 22.108138 -16.54395, 22.127259 -16.546431, 22.138111 -16.552632, 22.142452 -16.567101, 22.145139 -16.584051, 22.15165 -16.597694, 22.237743 -16.665493, 22.251282 -16.670041, 22.259034 -16.669214, 22.274123 -16.663633, 22.283735 -16.661359, 22.287352 -16.658259, 22.290763 -16.656398, 22.295621 -16.659189, 22.303062 -16.66694, 22.306989 -16.669214, 22.333654 -16.673658, 22.343473 -16.683167, 22.350088 -16.696292, 22.365797 -16.71686, 22.372618 -16.728849, 22.377993 -16.734843, 22.383574 -16.73753, 22.399594 -16.740424, 22.408895 -16.745798, 22.416647 -16.75448, 22.489097 -16.865378, 22.499742 -16.89349, 22.508527 -16.906202, 22.522894 -16.914677, 22.554519 -16.924185, 22.567232 -16.936898, 22.569299 -16.944959, 22.569196 -16.962426, 22.573433 -16.971004, 22.579324 -16.975345, 22.592037 -16.975449, 22.651671 -16.998703, 22.665624 -17.008625, 22.710479 -17.055444, 22.730736 -17.081592, 22.744792 -17.108257, 22.755851 -17.154869, 22.765153 -17.169649, 22.778278 -17.180397, 22.809388 -17.196417, 22.849179 -17.23104, 22.876154 -17.248093, 22.936512 -17.273311, 22.984157 -17.285817, 22.998627 -17.293775, 23.040381 -17.33708, 23.046376 -17.348449, 23.054127 -17.375321, 23.073041 -17.405086, 23.097639 -17.432165, 23.121513 -17.450872, 23.165748 -17.467408, 23.176187 -17.478157, 23.179081 -17.494177, 23.17691 -17.509783, 23.177531 -17.524046, 23.189726 -17.536551, 23.206986 -17.541202, 23.224246 -17.539342, 23.241299 -17.535104, 23.258352 -17.532831, 23.290805 -17.535414, 23.305378 -17.539548, 23.320158 -17.54637, 23.340621 -17.560736, 23.359432 -17.582853, 23.382273 -17.601043, 23.375141 -17.615409, 23.375968 -17.628225, 23.381652 -17.641144, 23.29804 -17.656854, 23.093918 -17.695405, 22.889796 -17.734059, 22.685674 -17.772816, 22.481449 -17.811366, 22.230508 -17.857565, 21.979464 -17.903764, 21.728523 -17.949859, 21.477583 -17.996058, 21.405546 -18.009287, 21.386839 -18.014455, 21.381154 -18.012595, 21.364825 -17.992027, 21.335059 -17.977971, 21.278112 -17.958541, 21.23274 -17.93446, 21.216617 -17.930636, 21.175792 -17.93322, 21.161323 -17.930636, 21.143133 -17.933736, 21.12608 -17.941074, 21.10944 -17.945002, 21.09311 -17.938077, 21.000713 -17.962055, 20.986455 -17.965771, 20.908315 -17.986136, 20.894466 -18.012595, 20.8616 -18.018796, 20.831937 -18.029338, 20.806202 -18.031405, 20.784602 -18.012595, 20.754423 -17.997918, 20.737473 -17.993474, 20.729928 -18.002363, 20.722487 -18.006497, 20.706157 -18.00381, 20.68962 -17.996988, 20.682179 -17.988617, 20.666263 -17.980245, 20.630089 -17.976938, 20.590712 -17.978798, 20.565494 -17.985206, 20.552885 -17.979108, 20.523946 -17.958541, 20.489529 -17.942108, 20.479504 -17.934356, 20.460074 -17.913686, 20.437543 -17.894462, 20.412635 -17.884127, 20.386797 -17.890328, 20.347729 -17.858909, 20.336154 -17.854878, 20.302771 -17.860769, 20.288921 -17.861596, 20.278173 -17.854878, 20.243446 -17.882783, 20.219882 -17.873378, 20.193113 -17.872138, 20.169239 -17.878132, 20.154563 -17.890328, 20.147121 -17.882783, 20.132962 -17.888778, 20.119939 -17.890741, 20.092551 -17.890328, 20.03426 -17.895909, 20.024545 -17.894566, 19.989508 -17.882783, 19.951991 -17.862319, 19.793964 -17.862319, 19.78766 -17.866454, 19.775981 -17.885057, 19.77102 -17.890328, 19.755207 -17.889088, 19.732056 -17.88206, 19.711282 -17.872965, 19.702187 -17.86573, 19.693195 -17.847747, 19.672628 -17.842682, 19.649787 -17.844956, 19.59284 -17.857255, 19.421274 -17.859426, 19.262421 -17.81395, 19.246608 -17.804028, 19.201856 -17.807129, 19.17209 -17.801238, 19.160825 -17.800928, 19.139327 -17.805269, 19.105118 -17.818188, 19.020575 -17.822322, 18.89004 -17.799274, 18.800847 -17.756589, 18.761986 -17.747701, 18.74824 -17.736332, 18.728293 -17.710907, 18.670932 -17.653133, 18.66132 -17.646725, 18.64282 -17.638044, 18.633312 -17.628949, 18.627937 -17.619854, 18.622873 -17.60218, 18.619669 -17.594842, 18.567476 -17.553294, 18.554144 -17.54637, 18.55094 -17.535311, 18.516626 -17.471336, 18.492752 -17.464514, 18.489858 -17.462447, 18.488101 -17.452112, 18.471668 -17.414285, 18.465157 -17.40922, 18.458645 -17.405293, 18.455028 -17.395991, 18.453581 -17.389893, 18.445726 -17.389273, 18.392293 -17.389273, 18.135668 -17.38917, 18.000483 -17.389116, 17.879146 -17.389067, 17.622624 -17.388963, 17.366102 -17.388963, 17.278972 -17.388928, 17.109477 -17.38886, 16.852852 -17.388757, 16.59633 -17.388757, 16.339808 -17.388653, 16.08339 -17.388653, 15.826765 -17.388653, 15.570243 -17.38855, 15.559114 -17.388545, 15.313721 -17.388447, 15.057199 -17.388343, 14.800574 -17.388343, 14.543949 -17.38824, 14.287427 -17.388136, 14.219112 -17.388136, 14.218801 -17.388136, 14.207432 -17.388033, 14.206502 -17.393097, 14.197097 -17.412941, 14.174979 -17.416248, 14.130744 -17.40922, 14.123923 -17.411598, 14.108213 -17.420486, 14.097258 -17.423587, 14.085372 -17.423587, 14.069146 -17.418626, 14.047442 -17.416042, 14.029148 -17.410461, 14.017883 -17.40922, 14.008891 -17.411494, 13.992975 -17.421313, 13.980573 -17.423587, 13.957318 -17.419142, 13.942745 -17.408187, 13.896857 -17.349069, 13.884351 -17.338527, 13.790403 -17.288091, 13.69413 -17.236621, 13.606487 -17.167375, 13.521324 -17.1219, 13.530936 -17.093271, 13.522254 -17.076941, 13.507785 -17.063505, 13.494659 -17.024024, 13.479776 -17.010278, 13.458899 -17.001803, 13.435128 -16.999013, 13.417248 -16.993742, 13.382004 -16.970384, 13.363711 -16.964183, 13.345947 -16.968711, 13.321543 -16.974932, 13.315238 -16.974415, 13.308417 -16.970178, 13.293018 -16.973795, 13.267489 -16.98537, 13.267489 -16.977929, 13.257154 -16.98165, 13.245269 -16.98134, 13.222841 -16.977929, 13.212092 -16.972761, 13.205478 -16.963046, 13.19814 -16.957362, 13.184911 -16.964183, 13.166307 -16.951057, 13.1445 -16.952401, 13.121762 -16.959842, 13.014275 -16.977929, 12.961668 -17.007385, 12.930456 -17.014206, 12.911025 -17.023508, 12.887151 -17.029812, 12.882293 -17.039527, 12.880536 -17.050793, 12.876402 -17.059784, 12.867721 -17.065572, 12.849944 -17.070843, 12.842296 -17.074047, 12.833097 -17.081799, 12.824932 -17.096371, 12.818318 -17.104846, 12.784315 -17.115078, 12.739873 -17.135542, 12.704733 -17.164274, 12.685923 -17.173576, 12.660705 -17.17709, 12.63621 -17.185151, 12.591458 -17.222565, 12.56717 -17.234554, 12.554561 -17.235588, 12.519215 -17.227836, 12.46051 -17.223082, 12.4417 -17.216984, 12.417929 -17.203445, 12.407594 -17.204065, 12.393951 -17.21471, 12.379482 -17.220395, 12.314679 -17.218121, 12.242642 -17.224839, 12.239335 -17.220705, 12.236545 -17.2113, 12.23179 -17.201895, 12.222282 -17.197657, 12.21174 -17.195177, 12.200474 -17.189286, 12.190656 -17.182568, 12.185075 -17.17709, 12.181148 -17.169235, 12.179494 -17.161587, 12.177117 -17.154766, 12.170709 -17.149185, 12.16151 -17.146291, 12.156653 -17.149081, 12.151279 -17.153629, 12.13991 -17.156006, 12.105597 -17.146084, 12.095261 -17.139676, 12.08813 -17.139159, 12.082239 -17.14133, 12.075211 -17.142983, 12.028909 -17.148978, 11.98302 -17.161897, 11.942092 -17.180501, 11.894963 -17.214607, 11.853932 -17.233417, 11.835536 -17.245406, 11.829644 -17.253468, 11.827474 -17.260082, 11.82241 -17.264423, 11.80763 -17.26587, 11.796882 -17.263803, 11.779518 -17.254915, 11.766184 -17.252751, 11.766124 -17.252699, 11.766124 -17.245294, 11.757986 -17.23919, 11.752778 -17.229181, 11.750743 -17.216892, 11.751801 -17.203709, 11.758637 -17.165216, 11.759288 -17.032973, 11.772114 -16.947026, 11.780013 -16.869467, 11.768144 -16.79852, 11.768141 -16.768251, 11.785921 -16.76352, 11.79087 -16.806082, 11.80371 -16.824033, 11.820474 -16.781472, 11.817638 -16.704685, 11.82309 -16.678969, 11.813813 -16.583917, 11.821951 -16.479099, 11.779145 -16.099867, 11.779796 -16.080255, 11.784516 -16.06406, 11.806977 -16.018813, 11.784356 -15.972633, 11.736618 -15.900203, 11.735525 -15.85711, 11.751801 -15.799086, 11.76352 -15.789972, 11.784679 -15.777276, 11.804942 -15.76922, 11.813813 -15.774998, 11.81837 -15.792413, 11.829845 -15.797784, 11.844249 -15.793552, 11.894705 -15.746677, 11.903331 -15.731378, 11.906505 -15.71795, 11.907481 -15.690037, 11.910167 -15.676202, 11.930512 -15.652439, 11.992035 -15.617283, 12.005138 -15.594334, 12.008474 -15.565362, 12.028494 -15.490167, 12.040375 -15.463311, 12.03004 -15.436944, 12.033458 -15.407403, 12.046641 -15.350681, 12.056407 -15.233087, 12.061778 -15.217869, 12.070567 -15.204685, 12.104991 -15.172784, 12.112478 -15.170994, 12.127778 -15.178969, 12.135997 -15.178969, 12.145274 -15.168634, 12.14975 -15.151951, 12.148936 -15.13421, 12.142833 -15.120701, 12.12794 -15.113377, 12.117361 -15.116632, 12.112478 -15.115167, 12.115001 -15.093927, 12.124848 -15.062188, 12.151134 -15.002048, 12.156505 -14.967055, 12.160492 -14.953546, 12.170095 -14.942478, 12.181407 -14.932306, 12.190603 -14.922052, 12.196137 -14.908461, 12.205333 -14.853611, 12.205414 -14.840427, 12.207774 -14.833266, 12.21518 -14.824151, 12.221934 -14.823989, 12.228526 -14.82586, 12.235606 -14.823012, 12.244314 -14.809991, 12.252778 -14.782322, 12.272716 -14.750584, 12.272309 -14.732843, 12.268077 -14.714776, 12.266368 -14.696059, 12.268728 -14.683526, 12.276622 -14.66074, 12.292735 -14.587498, 12.299001 -14.507582, 12.303884 -14.494073, 12.319672 -14.468683, 12.331309 -14.433526, 12.342133 -14.36712, 12.345388 -14.302992, 12.342133 -14.27044, 12.331554 -14.236749, 12.324229 -14.220636, 12.322113 -14.213067, 12.321056 -14.202732, 12.319672 -14.197442, 12.315196 -14.187921, 12.31422 -14.182224, 12.315603 -14.174981, 12.319021 -14.175388, 12.323578 -14.177504, 12.332205 -14.173435, 12.344249 -14.171319, 12.348888 -14.168552, 12.353038 -14.160903, 12.355235 -14.151788, 12.355805 -14.142348, 12.355154 -14.133722, 12.347179 -14.119887, 12.334972 -14.106378, 12.328787 -14.092218, 12.344737 -14.066827, 12.349783 -14.045505, 12.369884 -14.021905, 12.374522 -14.007582, 12.377289 -13.992446, 12.383067 -13.977309, 12.399913 -13.951593, 12.402843 -13.941339, 12.404145 -13.906345, 12.407237 -13.890558, 12.415538 -13.878595, 12.439301 -13.87127, 12.445079 -13.866143, 12.450531 -13.862481, 12.458181 -13.863865, 12.471853 -13.874444, 12.478363 -13.878025, 12.485688 -13.87713, 12.502452 -13.855239, 12.508962 -13.819594, 12.516287 -13.715102, 12.512706 -13.616795, 12.51588 -13.604913, 12.530121 -13.578709, 12.533865 -13.565199, 12.523611 -13.551202, 12.522146 -13.544203, 12.523936 -13.539972, 12.526541 -13.536228, 12.527029 -13.50449, 12.524587 -13.489353, 12.514903 -13.457615, 12.512706 -13.442071, 12.51531 -13.425958, 12.522146 -13.410577, 12.53297 -13.399021, 12.547211 -13.394627, 12.568044 -13.392348, 12.57545 -13.385919, 12.580821 -13.375258, 12.594737 -13.360447, 12.611095 -13.351983, 12.624197 -13.349216, 12.634451 -13.343357, 12.642426 -13.325616, 12.64324 -13.316339, 12.64088 -13.308852, 12.637869 -13.302016, 12.636241 -13.29461, 12.638438 -13.288018, 12.647716 -13.27809, 12.650076 -13.271091, 12.656505 -13.255629, 12.685069 -13.240818, 12.69158 -13.229425, 12.694102 -13.218357, 12.699962 -13.217869, 12.706798 -13.221449, 12.712087 -13.222589, 12.730642 -13.210545, 12.731781 -13.208917, 12.749766 -13.201349, 12.761485 -13.192966, 12.827159 -13.109796, 12.855724 -13.089532, 12.865977 -13.078709, 12.883149 -13.052016, 12.944835 -12.982354, 12.958669 -12.955499, 12.958832 -12.929132, 12.949474 -12.902114, 12.934825 -12.873793, 12.927501 -12.842055, 12.939464 -12.816664, 12.985688 -12.770115, 13.009613 -12.756524, 13.075043 -12.695001, 13.101085 -12.68141, 13.109223 -12.674493, 13.126475 -12.644464, 13.132823 -12.636651, 13.161632 -12.611261, 13.178559 -12.601739, 13.198009 -12.599379, 13.198904 -12.603611, 13.221039 -12.607192, 13.232758 -12.613051, 13.264415 -12.582615, 13.276866 -12.578302, 13.294281 -12.58115, 13.311534 -12.594496, 13.32838 -12.599379, 13.364024 -12.592706, 13.392914 -12.570408, 13.437673 -12.523696, 13.459727 -12.507257, 13.469249 -12.494806, 13.472992 -12.479099, 13.474946 -12.437758, 13.479015 -12.419122, 13.486583 -12.400811, 13.505626 -12.371352, 13.568614 -12.311456, 13.57545 -12.311456, 13.57545 -12.318292, 13.564952 -12.322035, 13.558279 -12.328302, 13.556814 -12.336602, 13.561778 -12.346124, 13.577485 -12.330987, 13.597504 -12.298761, 13.630219 -12.263442, 13.639822 -12.248712, 13.663585 -12.16961, 13.69337 -12.110284, 13.705089 -12.078058, 13.71518 -12.008396, 13.722504 -11.992771, 13.73699 -11.977716, 13.74879 -11.961196, 13.758556 -11.942478, 13.766449 -11.920505, 13.782481 -11.820896, 13.792247 -11.790948, 13.794607 -11.770196, 13.77768 -11.589125, 13.781912 -11.481052, 13.785981 -11.458266, 13.794607 -11.440606, 13.786632 -11.431573, 13.787364 -11.420343, 13.79184 -11.407973, 13.794607 -11.395929, 13.7942 -11.341485, 13.795258 -11.332208, 13.800141 -11.321384, 13.819021 -11.295587, 13.821625 -11.283624, 13.821788 -11.25628, 13.846528 -11.113702, 13.847423 -11.055434, 13.848481 -10.98211, 13.843435 -10.945733, 13.828624 -10.91367, 13.739268 -10.78338, 13.731456 -10.764255, 13.73113 -10.747166, 13.739431 -10.730239, 13.769786 -10.694268, 13.771739 -10.677911, 13.764985 -10.662042, 13.75294 -10.646173, 13.711436 -10.612074, 13.705089 -10.601983, 13.700857 -10.590916, 13.60613 -10.478204, 13.542328 -10.42669, 13.527599 -10.406508, 13.522634 -10.387791, 13.520763 -10.320245, 13.517345 -10.298435, 13.508962 -10.277032, 13.455089 -10.201755, 13.442638 -10.179376, 13.431163 -10.130548, 13.415294 -10.11004, 13.37672 -10.070733, 13.317393 -9.976331999999999, 13.319835 -9.961601999999999, 13.329112 -9.947524, 13.335216 -9.926690000000001, 13.327159 -9.885999999999999, 13.305431 -9.842461999999999, 13.278005 -9.801690000000001, 13.198578 -9.708672999999999, 13.191173 -9.690037, 13.196137 -9.672784, 13.218598 -9.649184, 13.225352 -9.63128, 13.223888 -9.614190000000001, 13.217784 -9.594170999999999, 13.209646 -9.577406999999999, 13.201427 -9.570489, 13.202403 -9.559015, 13.169932 -9.466404000000001, 13.168305 -9.447361000000001, 13.170665 -9.402520000000001, 13.166515 -9.38714, 13.147716 -9.352145999999999, 13.143403 -9.330254999999999, 13.120616 -9.313084, 13.045421 -9.18906, 13.016856 -9.122817, 12.996349 -9.093845, 12.992361 -9.038017999999999, 13.030284 -8.959568000000001, 13.086436 -8.892348, 13.136567 -8.871026000000001, 13.114757 -8.902927, 13.042817 -8.962498, 13.006847 -9.045586999999999, 13.000173 -9.072035, 13.013031 -9.083754000000001, 13.023448 -9.074965000000001, 13.0796 -8.980645000000001, 13.081879 -8.970473, 13.086925 -8.959731, 13.098888 -8.956476, 13.112478 -8.956149999999999, 13.122895 -8.954197000000001, 13.137462 -8.940851, 13.148285 -8.907973, 13.160411 -8.887953, 13.185557 -8.861586000000001, 13.191173 -8.850517999999999, 13.192149 -8.839532, 13.190603 -8.816095000000001, 13.194591 -8.806084999999999, 13.224132 -8.773858000000001, 13.241547 -8.760675000000001, 13.259532 -8.754815000000001, 13.209727 -8.804295, 13.198009 -8.823175000000001, 13.212169 -8.814874, 13.245372 -8.800063, 13.263682 -8.788263000000001, 13.271658 -8.786554000000001, 13.278005 -8.782484999999999, 13.280528 -8.771661, 13.281016 -8.764744, 13.28297 -8.759454, 13.28712 -8.756036, 13.2942 -8.754815000000001, 13.35613 -8.763604000000001, 13.380138 -8.75506, 13.393321 -8.704278, 13.408051 -8.666925000000001, 13.410899 -8.651951, 13.406098 -8.63714, 13.37672 -8.596612, 13.365408 -8.566827, 13.352061 -8.499688000000001, 13.342052 -8.467461999999999, 13.355479 -8.464613999999999, 13.365001 -8.465916, 13.372569 -8.464451, 13.379893 -8.45379, 13.382579 -8.444024000000001, 13.383556 -8.368748, 13.378917 -8.347101, 13.369395 -8.323499999999999, 13.276866 -8.178888000000001, 13.264903 -8.169692, 13.250824 -8.124444, 13.242442 -8.107192, 13.234548 -8.095961000000001, 13.222504 -8.069594, 13.215505 -8.059177, 13.198009 -8.02158, 13.189301 -7.975763, 13.122895 -7.857354, 12.986308 -7.550661, 12.939283 -7.435969, 12.911473 -7.335111, 12.883895 -7.305754, 12.869055 -7.288986, 12.849963 -7.265919, 12.855192 -7.231279, 12.849857 -7.205053, 12.833263 -7.005792, 12.823416 -6.975193, 12.8213 -6.959161, 12.819672 -6.954034, 12.816661 -6.950372, 12.815196 -6.946954, 12.817882 -6.942153, 12.824555 -6.934259, 12.827485 -6.929132, 12.829926 -6.918878, 12.832774 -6.915785, 12.833507 -6.912286, 12.828787 -6.904555, 12.799571 -6.923272, 12.779633 -6.914727, 12.706879 -6.814223, 12.620425 -6.734823, 12.550632 -6.632639, 12.458181 -6.465753, 12.421954 -6.380988, 12.395576 -6.324453, 12.368826 -6.287693, 12.344543 -6.254147, 12.319149 -6.22014, 12.279447 -6.147705, 12.27506 -6.114769, 12.290294 -6.096612, 12.312655 -6.090545, 12.333648 -6.078436, 12.327012 -6.093811, 12.324781 -6.122353, 12.346873 -6.120113, 12.382207 -6.110168, 12.432959 -6.108984, 12.496928 -6.094594, 12.527768 -6.074826, 12.549799 -6.067123, 12.580642 -6.062695, 12.632823 -6.034356, 12.645518 -6.028985, 12.669817 -6.02971, 12.690752 -6.037358, 12.716095 -6.045002, 12.742538 -6.049356, 12.77117 -6.041659, 12.817431 -6.03175, 12.861469 -6.016347, 12.882386 -6.000986, 12.927569 -5.981218, 12.94627 -5.955991, 12.966074 -5.931853, 12.97266 -5.913214, 12.98588 -5.90114, 13.011139 -5.895606, 13.029867 -5.893381, 13.046372 -5.885679, 13.064015 -5.887838, 13.100597 -5.891209, 13.111827 -5.890232, 13.129161 -5.883559, 13.169688 -5.861749, 13.183849 -5.856459, 13.183913 -5.856427, 13.184911 -5.856386, 13.26935 -5.863517, 13.311001 -5.873956, 13.342524 -5.890595, 13.353582 -5.882637, 13.37384 -5.860933, 13.380454 -5.856386, 13.41332 -5.857006, 13.493212 -5.858453, 13.522668 -5.867548, 13.53476 -5.867031, 13.553053 -5.859383, 13.562562 -5.857419, 13.641523 -5.865791, 13.807146 -5.852458, 13.943572 -5.841606, 13.98233 -5.853285)), ((12.801058 -4.410014, 12.814804 -4.412908, 12.830307 -4.409704, 12.842296 -4.40433, 12.854285 -4.403089, 12.869891 -4.411978, 12.884464 -4.432131, 12.902447 -4.480191, 12.921981 -4.502308, 12.961565 -4.533831, 13.027194 -4.612172, 13.073703 -4.635323, 13.065331 -4.663952, 13.029571 -4.676871, 12.833201 -4.722656, 12.802918 -4.739813, 12.786175 -4.758416, 12.774806 -4.7795, 12.768295 -4.801928, 12.766228 -4.825182, 12.756926 -4.837275, 12.711968 -4.86332, 12.696982 -4.875619, 12.6883 -4.897013, 12.686233 -4.912826, 12.678895 -4.924298, 12.6544 -4.932566, 12.61585 -4.93546, 12.606961 -4.94223, 12.603964 -4.96259, 12.587738 -5.000314, 12.550427 -5.024188, 12.466505 -5.053127, 12.458133 -5.054161, 12.450692 -5.053127, 12.444387 -5.055091, 12.439426 -5.064806, 12.43891 -5.073281, 12.44077 -5.08403, 12.444697 -5.093538, 12.450382 -5.098809, 12.51198 -5.120513, 12.524072 -5.129091, 12.530687 -5.162681, 12.52769 -5.239472, 12.524382 -5.324738, 12.519731 -5.444214, 12.514977 -5.568755, 12.512497 -5.63366, 12.508983 -5.726264, 12.435499 -5.725541, 12.344445 -5.724611, 12.258972 -5.736703, 12.210555 -5.763465, 12.210541 -5.763442, 12.210297 -5.763116, 12.161794 -5.681248, 12.147227 -5.617934, 12.170177 -5.55462, 12.182384 -5.543227, 12.19337 -5.540623, 12.204845 -5.539809, 12.218598 -5.534112, 12.228282 -5.523696, 12.232188 -5.510919, 12.232188 -5.479587, 12.229259 -5.466892, 12.214122 -5.431248, 12.19639 -5.396097, 12.171362 -5.329088, 12.172091 -5.302095, 12.132349 -5.232529, 12.124612 -5.184828, 12.069737 -5.119783, 12.023123 -5.045994, 12.023285 -5.035252, 12.035151 -5.038469, 12.054771 -5.042354, 12.067823 -5.046253, 12.074346 -5.052106, 12.080892 -5.075509, 12.100527 -5.080022, 12.116815 -5.069629, 12.127259 -5.053366, 12.128561 -5.039711, 12.137715 -5.034497, 12.137054 -5.014996, 12.132476 -5.005899, 12.123331 -5.002655, 12.112885 -5.00201, 12.10505 -5.013067, 12.103085 -5.027376, 12.103738 -5.048836, 12.097859 -5.054043, 12.09067 -5.050798, 12.086108 -5.030636, 12.079588 -5.020232, 12.071752 -5.017634, 12.066529 -5.02674, 12.056727 -5.027397, 12.04496 -5.020902, 12.030556 -5.022867, 12.009608 -5.019631, 12.018072 -5.0086, 12.130815 -4.912929, 12.15717 -4.870554, 12.187555 -4.768028, 12.192206 -4.763481, 12.204092 -4.763481, 12.213497 -4.769165, 12.222592 -4.780637, 12.235201 -4.803995, 12.244089 -4.796347, 12.258559 -4.790766, 12.273752 -4.787149, 12.307651 -4.783635, 12.321914 -4.778157, 12.332353 -4.765858, 12.374107 -4.683176, 12.378862 -4.662298, 12.377621 -4.619407, 12.387026 -4.605454, 12.414105 -4.608865, 12.429091 -4.607521, 12.497201 -4.5853, 12.608098 -4.565043, 12.623808 -4.559255, 12.634867 -4.54768, 12.656571 -4.507372, 12.670523 -4.491353, 12.686853 -4.47833, 12.70494 -4.467995, 12.718479 -4.451872, 12.726541 -4.427997, 12.737909 -4.40495, 12.761681 -4.391204, 12.775426 -4.396165, 12.782656 -4.400071, 12.801058 -4.410014)))"; + void test1() { DB::CartesianPolygon green, blue; @@ -158,6 +164,43 @@ void test3() printMultiPolygon(output); } + +void test4() +{ + DB::CartesianMultiPolygon green, blue; + boost::geometry::read_wkt(example, green); + + boost::geometry::read_wkt(example, blue); + + // boost::geometry::correct(green); + // boost::geometry::correct(blue); + + + // boost::geometry::correct(green); + + for (auto & polygon : green) + mercator(polygon); + + for (auto & polygon: blue) + mercator(polygon); + + std::cout << "----" << std::endl; + std::cout << boost::geometry::wkt(green) << std::endl; + std::cout << "----" << std::endl; + + DB::CartesianMultiPolygon output; + boost::geometry::intersection(green, blue, output); + + reverseMercator(output); + + // boost::geometry::correct(output); + // std::cout << boost::geometry::wkt(output) << std::endl; + + // printMultiPolygon(output); + + // std::cout << std::boolalpha << boost::geometry::equals(output, green) << std::endl; +} + int main(int argc, char ** argv) { (void) argc; @@ -165,5 +208,6 @@ int main(int argc, char ** argv) test1(); test2(); test3(); + test4(); return 0; } diff --git a/src/Functions/ya.make b/src/Functions/ya.make index 3ac64828b9c..a373ec88d54 100644 --- a/src/Functions/ya.make +++ b/src/Functions/ya.make @@ -335,6 +335,7 @@ SRCS( map.cpp match.cpp materialize.cpp + mercatorConverters.cpp minus.cpp modulo.cpp moduloOrZero.cpp diff --git a/tests/queries/0_stateless/01301_polygons_within.reference b/tests/queries/0_stateless/01301_polygons_within.reference index 5565ed6787f..1a2946b2542 100644 --- a/tests/queries/0_stateless/01301_polygons_within.reference +++ b/tests/queries/0_stateless/01301_polygons_within.reference @@ -2,3 +2,5 @@ 1 0 1 +0 +1 diff --git a/tests/queries/0_stateless/01301_polygons_within.sql b/tests/queries/0_stateless/01301_polygons_within.sql index 97e0573904a..25207dc398a 100644 --- a/tests/queries/0_stateless/01301_polygons_within.sql +++ b/tests/queries/0_stateless/01301_polygons_within.sql @@ -4,4 +4,5 @@ select polygonsWithinCartesian([[[(2., 2.), (2., 3.), (3., 3.), (3., 2.)]]], [[[ select polygonsWithinSpherical([[[(4.3613577, 50.8651821), (4.349556, 50.8535879), (4.3602419, 50.8435626), (4.3830299, 50.8428851), (4.3904543, 50.8564867), (4.3613148, 50.8651279)]]], [[[(4.346693, 50.858306), (4.367945, 50.852455), (4.366227, 50.840809), (4.344961, 50.833264), (4.338074, 50.848677), (4.346693, 50.858306)]]]); select polygonsWithinSpherical([[[(4.3501568, 50.8518269), (4.3444920, 50.8439961), (4.3565941, 50.8443213), (4.3501568, 50.8518269)]]], [[[(4.3679450, 50.8524550),(4.3466930, 50.8583060),(4.3380740, 50.8486770),(4.3449610, 50.8332640),(4.3662270, 50.8408090),(4.3679450, 50.8524550)]]]); - +select polygonsWithinMercator([[[(4.3613577, 50.8651821), (4.349556, 50.8535879), (4.3602419, 50.8435626), (4.3830299, 50.8428851), (4.3904543, 50.8564867), (4.3613148, 50.8651279)]]], [[[(4.346693, 50.858306), (4.367945, 50.852455), (4.366227, 50.840809), (4.344961, 50.833264), (4.338074, 50.848677), (4.346693, 50.858306)]]]); +select polygonsWithinMercator([[[(4.3501568, 50.8518269), (4.3444920, 50.8439961), (4.3565941, 50.8443213), (4.3501568, 50.8518269)]]], [[[(4.3679450, 50.8524550),(4.3466930, 50.8583060),(4.3380740, 50.8486770),(4.3449610, 50.8332640),(4.3662270, 50.8408090),(4.3679450, 50.8524550)]]]); diff --git a/tests/queries/0_stateless/01305_polygons_union.reference b/tests/queries/0_stateless/01305_polygons_union.reference index 64c6ac473e4..cf9c9fe8d5b 100644 --- a/tests/queries/0_stateless/01305_polygons_union.reference +++ b/tests/queries/0_stateless/01305_polygons_union.reference @@ -1,2 +1,7 @@ +-------- [[[(1,2.9),(1,4),(4,4),(4,1),(2.9,1),(3,0),(0,0),(0,3),(1,2.9)]]] +-------- +-------- [[[(4.3666052904432435,50.84337386140151),(4.366227,50.840809),(4.344961,50.833264),(4.338074,50.848677),(4.346693,50.858306),(4.3526804582393535,50.856658100365976),(4.3613577,50.8651821),(4.3613148,50.8651279),(4.3904543,50.8564867),(4.3830299,50.8428851),(4.3666052904432435,50.84337386140151)]]] +-------- +[[[(4.366505261795747,50.843273415405),(4.3661270000000005,50.840709),(4.344861,50.83316399999998),(4.337974,50.84857699999999),(4.346593,50.858205999999996),(4.352580404040105,50.85655765067624),(4.3612577,50.86508209999998),(4.361214800000001,50.8650279),(4.3903543,50.85638669999999),(4.382929900000001,50.842785099999986),(4.366505261795747,50.843273415405)]]] diff --git a/tests/queries/0_stateless/01305_polygons_union.sql b/tests/queries/0_stateless/01305_polygons_union.sql index a67c5a7d399..d022bf46971 100644 --- a/tests/queries/0_stateless/01305_polygons_union.sql +++ b/tests/queries/0_stateless/01305_polygons_union.sql @@ -1,5 +1,11 @@ +select '--------'; select polygonsUnionCartesian([[[(0., 0.),(0., 3.),(1., 2.9),(2., 2.6),(2.6, 2.),(2.9, 1),(3., 0.),(0., 0.)]]], [[[(1., 1.),(1., 4.),(4., 4.),(4., 1.),(1., 1.)]]]); +select '--------'; SELECT polygonsUnionCartesian([[[(2., 100.0000991821289), (0., 3.), (1., 2.9), (2., 2.6), (2.6, 2.), (2.9, 1), (3., 0.), (100.0000991821289, 2.)]]], [[[(1., 1.), (1000.0001220703125, nan), (4., 4.), (4., 1.), (1., 1.)]]]); -- { serverError 43 } +select '--------'; select polygonsUnionSpherical([[[(4.3613577, 50.8651821), (4.349556, 50.8535879), (4.3602419, 50.8435626), (4.3830299, 50.8428851), (4.3904543, 50.8564867), (4.3613148, 50.8651279)]]], [[[(4.346693, 50.858306), (4.367945, 50.852455), (4.366227, 50.840809), (4.344961, 50.833264), (4.338074, 50.848677), (4.346693, 50.858306)]]]); + +select '--------'; +select polygonsUnionMercator([[[(4.3613577, 50.8651821), (4.349556, 50.8535879), (4.3602419, 50.8435626), (4.3830299, 50.8428851), (4.3904543, 50.8564867), (4.3613148, 50.8651279)]]], [[[(4.346693, 50.858306), (4.367945, 50.852455), (4.366227, 50.840809), (4.344961, 50.833264), (4.338074, 50.848677), (4.346693, 50.858306)]]]); diff --git a/tests/queries/0_stateless/01306_polygons_intersection.reference b/tests/queries/0_stateless/01306_polygons_intersection.reference index 3ac2647e7f5..e48057e1eb3 100644 --- a/tests/queries/0_stateless/01306_polygons_intersection.reference +++ b/tests/queries/0_stateless/01306_polygons_intersection.reference @@ -1,4 +1,9 @@ +-------- [[[(1,2.9),(2,2.6),(2.6,2),(2.9,1),(1,1),(1,2.9)]]] [] +-------- [] [[[(4.3666052904432435,50.84337386140151),(4.3602419,50.8435626),(4.349556,50.8535879),(4.3526804582393535,50.856658100365976),(4.367945,50.852455),(4.3666052904432435,50.84337386140151)]]] +-------- +[] +[[[(4.366505261795747,50.843273415405),(4.3601419,50.8434626),(4.349456,50.8534879),(4.352580404040105,50.85655765067624),(4.367845,50.85235499999999),(4.366505261795747,50.843273415405)]]] diff --git a/tests/queries/0_stateless/01306_polygons_intersection.sql b/tests/queries/0_stateless/01306_polygons_intersection.sql index fd428c3d826..0a0aab2233b 100644 --- a/tests/queries/0_stateless/01306_polygons_intersection.sql +++ b/tests/queries/0_stateless/01306_polygons_intersection.sql @@ -1,5 +1,12 @@ + +select '--------'; select polygonsIntersectionCartesian([[[(0., 0.),(0., 3.),(1., 2.9),(2., 2.6),(2.6, 2.),(2.9, 1.),(3., 0.),(0., 0.)]]], [[[(1., 1.),(1., 4.),(4., 4.),(4., 1.),(1., 1.)]]]); select polygonsIntersectionCartesian([[[(0., 0.),(0., 3.),(1., 2.9),(2., 2.6),(2.6, 2.),(2.9, 1.),(3., 0.),(0., 0.)]]], [[[(3., 3.),(3., 4.),(4., 4.),(4., 3.),(3., 3.)]]]); +select '--------'; select polygonsIntersectionSpherical([[[(4.346693, 50.858306), (4.367945, 50.852455), (4.366227, 50.840809), (4.344961, 50.833264), (4.338074, 50.848677), (4.346693, 50.858306)]]], [[[(25.0010, 136.9987), (17.7500, 142.5000), (11.3733, 142.5917)]]]); select polygonsIntersectionSpherical([[[(4.3613577, 50.8651821), (4.349556, 50.8535879), (4.3602419, 50.8435626), (4.3830299, 50.8428851), (4.3904543, 50.8564867), (4.3613148, 50.8651279)]]], [[[(4.346693, 50.858306), (4.367945, 50.852455), (4.366227, 50.840809), (4.344961, 50.833264), (4.338074, 50.848677), (4.346693, 50.858306)]]]); + +select '--------'; +select polygonsIntersectionMercator([[[(4.346693, 50.858306), (4.367945, 50.852455), (4.366227, 50.840809), (4.344961, 50.833264), (4.338074, 50.848677), (4.346693, 50.858306)]]], [[[(25.0010, 136.9987), (17.7500, 142.5000), (11.3733, 142.5917)]]]); +select polygonsIntersectionMercator([[[(4.3613577, 50.8651821), (4.349556, 50.8535879), (4.3602419, 50.8435626), (4.3830299, 50.8428851), (4.3904543, 50.8564867), (4.3613148, 50.8651279)]]], [[[(4.346693, 50.858306), (4.367945, 50.852455), (4.366227, 50.840809), (4.344961, 50.833264), (4.338074, 50.848677), (4.346693, 50.858306)]]]); From a6779df0ae317d7ad6b7f5e9f0462c2a7e3b15bf Mon Sep 17 00:00:00 2001 From: Nikita Mikhaylov Date: Wed, 3 Mar 2021 21:35:50 +0300 Subject: [PATCH 313/544] more tests --- .../0_stateless/01301_polygons_within.reference | 9 +++++++++ tests/queries/0_stateless/01301_polygons_within.sql | 13 +++++++++++++ .../01304_polygons_sym_difference.reference | 9 +++++++++ .../0_stateless/01304_polygons_sym_difference.sql | 13 ++++++++++++- .../0_stateless/01305_polygons_union.reference | 9 +++++++++ tests/queries/0_stateless/01305_polygons_union.sql | 12 ++++++++++++ .../01306_polygons_intersection.reference | 9 +++++++++ .../0_stateless/01306_polygons_intersection.sql | 12 ++++++++++++ 8 files changed, 85 insertions(+), 1 deletion(-) diff --git a/tests/queries/0_stateless/01301_polygons_within.reference b/tests/queries/0_stateless/01301_polygons_within.reference index 1a2946b2542..147806964e6 100644 --- a/tests/queries/0_stateless/01301_polygons_within.reference +++ b/tests/queries/0_stateless/01301_polygons_within.reference @@ -4,3 +4,12 @@ 1 0 1 +-------- MultiPolygon with Polygon +0 +0 +-------- MultiPolygon with Polygon with Holes +0 +0 +-------- Polygon with Polygon with Holes +0 +0 diff --git a/tests/queries/0_stateless/01301_polygons_within.sql b/tests/queries/0_stateless/01301_polygons_within.sql index 25207dc398a..ff8342e783d 100644 --- a/tests/queries/0_stateless/01301_polygons_within.sql +++ b/tests/queries/0_stateless/01301_polygons_within.sql @@ -6,3 +6,16 @@ select polygonsWithinSpherical([[[(4.3501568, 50.8518269), (4.3444920, 50.843996 select polygonsWithinMercator([[[(4.3613577, 50.8651821), (4.349556, 50.8535879), (4.3602419, 50.8435626), (4.3830299, 50.8428851), (4.3904543, 50.8564867), (4.3613148, 50.8651279)]]], [[[(4.346693, 50.858306), (4.367945, 50.852455), (4.366227, 50.840809), (4.344961, 50.833264), (4.338074, 50.848677), (4.346693, 50.858306)]]]); select polygonsWithinMercator([[[(4.3501568, 50.8518269), (4.3444920, 50.8439961), (4.3565941, 50.8443213), (4.3501568, 50.8518269)]]], [[[(4.3679450, 50.8524550),(4.3466930, 50.8583060),(4.3380740, 50.8486770),(4.3449610, 50.8332640),(4.3662270, 50.8408090),(4.3679450, 50.8524550)]]]); + + +select '-------- MultiPolygon with Polygon'; +select polygonsWithinSpherical([[(29.453587685533865,59.779570356240356),(29.393139070478895,52.276266797422124),(40.636581470703206,59.38168915000267),(41.21084331372543,59.103467777099866),(29.786055068336193,52.146627480315004),(31.23682182965546,52.16517054781818),(41.69443223416517,58.85424941916091),(42.51048853740727,58.47703162291134),(32.59691566839227,52.22075341251539),(34.289476889931414,52.22075341251539),(43.02430176537451,58.07974369546071),(43.02430176537451,57.25537683364851),(35.468224883503325,52.2022335126388),(37.16078610504247,52.23926559241349),(43.02430176537451,56.26136189644947),(43.02430176537451,55.326904361850836),(38.33953409861437,52.16517054781818),(40.09254393520848,52.16517054781818),(44.4146199116388,55.3097062225408),(44.47506852669377,59.80998197603594),(39.72985224487867,59.931351417569715),(30.23941968124846,53.67744677450975),(30.20919537372098,54.63314259659509),(38.73245009647167,59.94649146557819),(37.2816833351524,59.97675082987618),(30.23941968124846,55.2752875586599),(30.33009260383092,56.19415599955667),(36.28428118674541,59.96162460231375),(34.863738732953635,59.97675082987618),(30.178971066193498,56.97640788219866),(30.178971066193498,57.91957806959033),(33.65476643185424,59.94649146557819),(32.32489690064491,59.94649146557819),(30.481214141468342,58.85424941916091),(30.571887064050795,59.99187015036608),(29.453587685533865,59.779570356240356)]], [[[(33.473420586689336,58.85424941916091),(32.23422397806246,58.492830557036),(32.173775363007486,58.03176922751564),(31.508840597402823,57.499784781503735),(31.750635057622702,56.86092686957355),(31.508840597402823,55.941082594334574),(32.20399967053497,55.515591939372456),(31.84130798020516,54.998862226280465),(31.418167674820367,54.422670886434275),(32.47601843828233,53.83826377018255),(32.08310244042503,53.408048308050866),(33.171177511414484,52.82758702113742),(34.77306581037117,52.91880107773494),(34.77306581037117,53.784726518357985),(34.108131044766516,54.17574726780569),(35.07530888564602,54.59813930694554),(34.25925258240394,54.96417435716029),(35.01486027059106,55.361278263643584),(33.50364489421682,55.37845402950552),(32.7480372060297,55.90721384574556),(35.67979503619571,55.68634475630185),(32.83871012861215,56.311688992608396),(34.591719965206266,56.29492065473883),(35.7100193437232,56.311688992608396),(33.83611227701915,56.695333481003644),(32.95960735872209,56.9434497616887),(36.072711034053015,57.091531913901434),(33.171177511414484,57.33702717078384),(36.193608264162954,57.499784781503735),(33.23162612646945,57.77481561306047),(36.43540272438284,58.04776787540811),(33.62454212432676,58.27099811968307),(36.344729801800376,58.54018474404165),(33.83611227701915,58.68186423448108),(34.74284150284369,59.565911441555244),(33.473420586689336,58.85424941916091)]], [[(34.65216858026123,58.91672306881671),(37.19101041256995,58.68186423448108),(36.01226241899805,58.28688958537609),(37.16078610504247,58.04776787540811),(35.74024365125068,57.79092907387934),(37.009664567405046,57.499784781503735),(35.77046795877817,57.25537683364851),(36.979440259877556,57.07510745541089),(34.22902827487645,56.794777197297435),(36.7074214921302,56.210968525786996),(34.712617195316206,56.10998276812964),(36.55629995449277,55.63519693782703),(35.13575750070099,55.53270067649592),(36.43540272438284,55.34409504165558),(34.83351442542614,55.01619492319591),(35.61934642114075,54.49294870011772),(34.89396304048112,54.12264226523038),(35.37755196092087,53.046178687628185),(37.43280487278982,52.95523300597458),(35.92158949641559,53.80257986695776),(36.91899164482259,53.856094327816805),(36.01226241899805,54.75541714463799),(37.765272255592166,55.189110239786885),(36.828318722240134,55.44708256557195),(38.03729102333953,55.652253637168315),(36.64697287707522,55.941082594334574),(38.21863686850443,56.05939028508024),(36.37495410932787,56.64551287174558),(38.30930979108689,56.992876013526654),(37.16078610504247,57.25537683364851),(38.127963945921984,57.516020773674256),(37.43280487278982,57.710289827306724),(38.33953409861437,57.935626886818994),(37.40258056526235,58.31865112960426),(38.58132855883426,58.744648733419496),(37.31190764267989,59.02578062465136),(34.65216858026123,58.91672306881671)]], [[(38.52087994377928,59.11898412389468),(39.54850639971376,58.713270635642914),(38.369758406141855,58.28688958537609),(38.85334732658162,58.06375936407028),(38.33953409861437,57.710289827306724),(38.73245009647167,57.48354156434209),(38.21863686850443,57.271721400459285),(38.97424455669155,56.87744603722649),(37.463029180317314,56.5623320541159),(38.94402024916407,56.05939028508024),(38.18841256097694,55.856355210835915),(38.490655636251795,55.53270067649592),(37.795496563119656,55.39562234093384),(38.30930979108689,55.154587013355666),(36.7074214921302,54.65063295250911),(37.31190764267989,53.92734063371401),(36.979440259877556,53.58783775557231),(37.855945178174615,52.91880107773497),(39.57873070724124,52.69956490610895),(38.33953409861437,53.281741738901104),(40.00187101262603,53.35396273604752),(39.54850639971376,53.58783775557231),(40.24366547284591,53.58783775557231),(39.97164670509855,53.98069568468355),(40.60635716317572,54.03398248547225),(40.39478701048334,54.44025165268903),(39.54850639971376,54.56310590284329),(39.54850639971376,54.87732350170489),(40.39478701048334,54.87732350170489),(40.39478701048334,55.24083903654295),(39.82052516746112,55.2752875586599),(39.760076552406154,55.75443792473942),(40.57613285564824,55.78844000174894),(40.425011318010824,56.19415599955667),(39.82052516746112,56.07626182891758),(39.79030085993364,56.41214455508424),(40.48545993306579,56.495655446714636),(40.33433839542836,56.95993246553937),(39.79030085993364,56.992876013526654),(39.72985224487867,57.46729112028032),(40.33433839542836,57.46729112028032),(40.24366547284591,58.04776787540811),(39.63917932229622,58.04776787540811),(39.63917932229622,58.382088724871295),(40.33433839542836,58.382088724871295),(40.45523562553831,58.9011152358548),(38.52087994377928,59.11898412389468)]]]) format TSV; +select polygonsWithinMercator([[(29.453587685533865,59.779570356240356),(29.393139070478895,52.276266797422124),(40.636581470703206,59.38168915000267),(41.21084331372543,59.103467777099866),(29.786055068336193,52.146627480315004),(31.23682182965546,52.16517054781818),(41.69443223416517,58.85424941916091),(42.51048853740727,58.47703162291134),(32.59691566839227,52.22075341251539),(34.289476889931414,52.22075341251539),(43.02430176537451,58.07974369546071),(43.02430176537451,57.25537683364851),(35.468224883503325,52.2022335126388),(37.16078610504247,52.23926559241349),(43.02430176537451,56.26136189644947),(43.02430176537451,55.326904361850836),(38.33953409861437,52.16517054781818),(40.09254393520848,52.16517054781818),(44.4146199116388,55.3097062225408),(44.47506852669377,59.80998197603594),(39.72985224487867,59.931351417569715),(30.23941968124846,53.67744677450975),(30.20919537372098,54.63314259659509),(38.73245009647167,59.94649146557819),(37.2816833351524,59.97675082987618),(30.23941968124846,55.2752875586599),(30.33009260383092,56.19415599955667),(36.28428118674541,59.96162460231375),(34.863738732953635,59.97675082987618),(30.178971066193498,56.97640788219866),(30.178971066193498,57.91957806959033),(33.65476643185424,59.94649146557819),(32.32489690064491,59.94649146557819),(30.481214141468342,58.85424941916091),(30.571887064050795,59.99187015036608),(29.453587685533865,59.779570356240356)]], [[[(33.473420586689336,58.85424941916091),(32.23422397806246,58.492830557036),(32.173775363007486,58.03176922751564),(31.508840597402823,57.499784781503735),(31.750635057622702,56.86092686957355),(31.508840597402823,55.941082594334574),(32.20399967053497,55.515591939372456),(31.84130798020516,54.998862226280465),(31.418167674820367,54.422670886434275),(32.47601843828233,53.83826377018255),(32.08310244042503,53.408048308050866),(33.171177511414484,52.82758702113742),(34.77306581037117,52.91880107773494),(34.77306581037117,53.784726518357985),(34.108131044766516,54.17574726780569),(35.07530888564602,54.59813930694554),(34.25925258240394,54.96417435716029),(35.01486027059106,55.361278263643584),(33.50364489421682,55.37845402950552),(32.7480372060297,55.90721384574556),(35.67979503619571,55.68634475630185),(32.83871012861215,56.311688992608396),(34.591719965206266,56.29492065473883),(35.7100193437232,56.311688992608396),(33.83611227701915,56.695333481003644),(32.95960735872209,56.9434497616887),(36.072711034053015,57.091531913901434),(33.171177511414484,57.33702717078384),(36.193608264162954,57.499784781503735),(33.23162612646945,57.77481561306047),(36.43540272438284,58.04776787540811),(33.62454212432676,58.27099811968307),(36.344729801800376,58.54018474404165),(33.83611227701915,58.68186423448108),(34.74284150284369,59.565911441555244),(33.473420586689336,58.85424941916091)]], [[(34.65216858026123,58.91672306881671),(37.19101041256995,58.68186423448108),(36.01226241899805,58.28688958537609),(37.16078610504247,58.04776787540811),(35.74024365125068,57.79092907387934),(37.009664567405046,57.499784781503735),(35.77046795877817,57.25537683364851),(36.979440259877556,57.07510745541089),(34.22902827487645,56.794777197297435),(36.7074214921302,56.210968525786996),(34.712617195316206,56.10998276812964),(36.55629995449277,55.63519693782703),(35.13575750070099,55.53270067649592),(36.43540272438284,55.34409504165558),(34.83351442542614,55.01619492319591),(35.61934642114075,54.49294870011772),(34.89396304048112,54.12264226523038),(35.37755196092087,53.046178687628185),(37.43280487278982,52.95523300597458),(35.92158949641559,53.80257986695776),(36.91899164482259,53.856094327816805),(36.01226241899805,54.75541714463799),(37.765272255592166,55.189110239786885),(36.828318722240134,55.44708256557195),(38.03729102333953,55.652253637168315),(36.64697287707522,55.941082594334574),(38.21863686850443,56.05939028508024),(36.37495410932787,56.64551287174558),(38.30930979108689,56.992876013526654),(37.16078610504247,57.25537683364851),(38.127963945921984,57.516020773674256),(37.43280487278982,57.710289827306724),(38.33953409861437,57.935626886818994),(37.40258056526235,58.31865112960426),(38.58132855883426,58.744648733419496),(37.31190764267989,59.02578062465136),(34.65216858026123,58.91672306881671)]], [[(38.52087994377928,59.11898412389468),(39.54850639971376,58.713270635642914),(38.369758406141855,58.28688958537609),(38.85334732658162,58.06375936407028),(38.33953409861437,57.710289827306724),(38.73245009647167,57.48354156434209),(38.21863686850443,57.271721400459285),(38.97424455669155,56.87744603722649),(37.463029180317314,56.5623320541159),(38.94402024916407,56.05939028508024),(38.18841256097694,55.856355210835915),(38.490655636251795,55.53270067649592),(37.795496563119656,55.39562234093384),(38.30930979108689,55.154587013355666),(36.7074214921302,54.65063295250911),(37.31190764267989,53.92734063371401),(36.979440259877556,53.58783775557231),(37.855945178174615,52.91880107773497),(39.57873070724124,52.69956490610895),(38.33953409861437,53.281741738901104),(40.00187101262603,53.35396273604752),(39.54850639971376,53.58783775557231),(40.24366547284591,53.58783775557231),(39.97164670509855,53.98069568468355),(40.60635716317572,54.03398248547225),(40.39478701048334,54.44025165268903),(39.54850639971376,54.56310590284329),(39.54850639971376,54.87732350170489),(40.39478701048334,54.87732350170489),(40.39478701048334,55.24083903654295),(39.82052516746112,55.2752875586599),(39.760076552406154,55.75443792473942),(40.57613285564824,55.78844000174894),(40.425011318010824,56.19415599955667),(39.82052516746112,56.07626182891758),(39.79030085993364,56.41214455508424),(40.48545993306579,56.495655446714636),(40.33433839542836,56.95993246553937),(39.79030085993364,56.992876013526654),(39.72985224487867,57.46729112028032),(40.33433839542836,57.46729112028032),(40.24366547284591,58.04776787540811),(39.63917932229622,58.04776787540811),(39.63917932229622,58.382088724871295),(40.33433839542836,58.382088724871295),(40.45523562553831,58.9011152358548),(38.52087994377928,59.11898412389468)]]]) format TSV; + +select '-------- MultiPolygon with Polygon with Holes'; +select polygonsWithinSpherical([[[(33.473420586689336,58.85424941916091),(32.23422397806246,58.492830557036),(32.173775363007486,58.03176922751564),(31.508840597402823,57.499784781503735),(31.750635057622702,56.86092686957355),(31.508840597402823,55.941082594334574),(32.20399967053497,55.515591939372456),(31.84130798020516,54.998862226280465),(31.418167674820367,54.422670886434275),(32.47601843828233,53.83826377018255),(32.08310244042503,53.408048308050866),(33.171177511414484,52.82758702113742),(34.77306581037117,52.91880107773494),(34.77306581037117,53.784726518357985),(34.108131044766516,54.17574726780569),(35.07530888564602,54.59813930694554),(34.25925258240394,54.96417435716029),(35.01486027059106,55.361278263643584),(33.50364489421682,55.37845402950552),(32.7480372060297,55.90721384574556),(35.67979503619571,55.68634475630185),(32.83871012861215,56.311688992608396),(34.591719965206266,56.29492065473883),(35.7100193437232,56.311688992608396),(33.83611227701915,56.695333481003644),(32.95960735872209,56.9434497616887),(36.072711034053015,57.091531913901434),(33.171177511414484,57.33702717078384),(36.193608264162954,57.499784781503735),(33.23162612646945,57.77481561306047),(36.43540272438284,58.04776787540811),(33.62454212432676,58.27099811968307),(36.344729801800376,58.54018474404165),(33.83611227701915,58.68186423448108),(34.74284150284369,59.565911441555244),(33.473420586689336,58.85424941916091)]], [[(34.65216858026123,58.91672306881671),(37.19101041256995,58.68186423448108),(36.01226241899805,58.28688958537609),(37.16078610504247,58.04776787540811),(35.74024365125068,57.79092907387934),(37.009664567405046,57.499784781503735),(35.77046795877817,57.25537683364851),(36.979440259877556,57.07510745541089),(34.22902827487645,56.794777197297435),(36.7074214921302,56.210968525786996),(34.712617195316206,56.10998276812964),(36.55629995449277,55.63519693782703),(35.13575750070099,55.53270067649592),(36.43540272438284,55.34409504165558),(34.83351442542614,55.01619492319591),(35.61934642114075,54.49294870011772),(34.89396304048112,54.12264226523038),(35.37755196092087,53.046178687628185),(37.43280487278982,52.95523300597458),(35.92158949641559,53.80257986695776),(36.91899164482259,53.856094327816805),(36.01226241899805,54.75541714463799),(37.765272255592166,55.189110239786885),(36.828318722240134,55.44708256557195),(38.03729102333953,55.652253637168315),(36.64697287707522,55.941082594334574),(38.21863686850443,56.05939028508024),(36.37495410932787,56.64551287174558),(38.30930979108689,56.992876013526654),(37.16078610504247,57.25537683364851),(38.127963945921984,57.516020773674256),(37.43280487278982,57.710289827306724),(38.33953409861437,57.935626886818994),(37.40258056526235,58.31865112960426),(38.58132855883426,58.744648733419496),(37.31190764267989,59.02578062465136),(34.65216858026123,58.91672306881671)]], [[(38.52087994377928,59.11898412389468),(39.54850639971376,58.713270635642914),(38.369758406141855,58.28688958537609),(38.85334732658162,58.06375936407028),(38.33953409861437,57.710289827306724),(38.73245009647167,57.48354156434209),(38.21863686850443,57.271721400459285),(38.97424455669155,56.87744603722649),(37.463029180317314,56.5623320541159),(38.94402024916407,56.05939028508024),(38.18841256097694,55.856355210835915),(38.490655636251795,55.53270067649592),(37.795496563119656,55.39562234093384),(38.30930979108689,55.154587013355666),(36.7074214921302,54.65063295250911),(37.31190764267989,53.92734063371401),(36.979440259877556,53.58783775557231),(37.855945178174615,52.91880107773497),(39.57873070724124,52.69956490610895),(38.33953409861437,53.281741738901104),(40.00187101262603,53.35396273604752),(39.54850639971376,53.58783775557231),(40.24366547284591,53.58783775557231),(39.97164670509855,53.98069568468355),(40.60635716317572,54.03398248547225),(40.39478701048334,54.44025165268903),(39.54850639971376,54.56310590284329),(39.54850639971376,54.87732350170489),(40.39478701048334,54.87732350170489),(40.39478701048334,55.24083903654295),(39.82052516746112,55.2752875586599),(39.760076552406154,55.75443792473942),(40.57613285564824,55.78844000174894),(40.425011318010824,56.19415599955667),(39.82052516746112,56.07626182891758),(39.79030085993364,56.41214455508424),(40.48545993306579,56.495655446714636),(40.33433839542836,56.95993246553937),(39.79030085993364,56.992876013526654),(39.72985224487867,57.46729112028032),(40.33433839542836,57.46729112028032),(40.24366547284591,58.04776787540811),(39.63917932229622,58.04776787540811),(39.63917932229622,58.382088724871295),(40.33433839542836,58.382088724871295),(40.45523562553831,58.9011152358548),(38.52087994377928,59.11898412389468)]]], [[(24.367675781249993,61.45977057029751),(19.577636718749993,58.67693767258692),(19.577636718749993,57.492213666700735),(19.445800781249996,55.87531083569678),(19.445800781249996,54.085173420886775),(17.468261718749996,53.014783245859235),(20.017089843749993,51.563412328675895),(21.203613281249993,50.205033264943324),(26.125488281249993,50.40151532278236),(27.22412109374999,48.980216985374994),(32.80517578124999,49.525208341974405),(35.26611328124999,48.74894534343292),(36.93603515624999,49.66762782262194),(42.56103515625,48.77791275550183),(43.92333984374999,49.8096315635631),(47.17529296875,49.152969656170455),(49.28466796875,50.54136296522162),(48.05419921875,51.17934297928929),(51.39404296875,52.48278022207825),(50.64697265625,53.014783245859235),(52.88818359375,53.93021986394004),(51.65771484374999,54.29088164657006),(52.66845703125,55.825973254619015),(50.25146484375,56.145549500679095),(51.92138671875,57.914847767009206),(49.15283203125,58.17070248348605),(49.59228515625,60.086762746260064),(47.043457031249986,59.88893689676584),(43.57177734375,61.37567331572748),(42.64892578125,60.630101766266705),(36.89208984374999,62.000904713685856),(36.01318359374999,61.143235250840576),(31.398925781249993,62.02152819100766),(30.563964843749996,61.05828537037917),(26.872558593749993,61.71070595883174),(26.652832031249993,61.10078883158897),(24.367675781249993,61.45977057029751)], [(24.455566406249993,59.42272750081452),(21.203613281249993,58.49369382056807),(21.335449218749993,56.89700392127261),(21.599121093749993,55.92458580482949),(25.202636718749993,55.998380955359636),(28.850097656249993,57.06463027327854),(27.09228515625,57.844750992890994),(28.806152343749996,59.17592824927138),(26.257324218749993,59.17592824927138),(24.455566406249993,59.42272750081452)], [(35.13427734375,59.84481485969107),(31.970214843749993,58.97266715450152),(33.20068359374999,56.776808316568406),(36.67236328125,56.41390137600675),(39.08935546874999,57.25528054528888),(42.69287109374999,58.03137242177638),(40.89111328124999,59.26588062825809),(37.28759765625,58.722598828043374),(37.11181640624999,59.66774058164964),(35.13427734375,59.84481485969107)], [(29.157714843749993,55.75184939173528),(22.565917968749993,55.128649068488784),(22.565917968749993,53.54030739150019),(22.038574218749996,51.48138289610097),(26.257324218749993,51.42661449707484),(30.124511718749993,50.54136296522162),(32.18994140624999,51.17934297928929),(30.124511718749993,53.173119202640635),(35.09033203124999,53.173119202640635),(33.11279296875,54.085173420886775),(29.597167968749993,55.50374985927513),(29.157714843749993,55.75184939173528)], [(42.82470703125,56.58369172128337),(36.584472656249986,55.329144408405085),(37.99072265625,53.592504809039355),(34.95849609374999,51.48138289610097),(36.54052734374999,50.40151532278236),(39.66064453124999,50.289339253291786),(39.79248046875,52.13348804077148),(41.77001953125,50.68079714532166),(44.49462890624999,51.97134580885171),(47.30712890624999,52.509534770327264),(44.05517578125,53.54030739150019),(46.60400390625,53.696706475303245),(47.61474609375,55.40406982700608),(45.37353515625,55.40406982700608),(42.82470703125,56.58369172128337)]]) format TSV; +select polygonsWithinMercator([[[(33.473420586689336,58.85424941916091),(32.23422397806246,58.492830557036),(32.173775363007486,58.03176922751564),(31.508840597402823,57.499784781503735),(31.750635057622702,56.86092686957355),(31.508840597402823,55.941082594334574),(32.20399967053497,55.515591939372456),(31.84130798020516,54.998862226280465),(31.418167674820367,54.422670886434275),(32.47601843828233,53.83826377018255),(32.08310244042503,53.408048308050866),(33.171177511414484,52.82758702113742),(34.77306581037117,52.91880107773494),(34.77306581037117,53.784726518357985),(34.108131044766516,54.17574726780569),(35.07530888564602,54.59813930694554),(34.25925258240394,54.96417435716029),(35.01486027059106,55.361278263643584),(33.50364489421682,55.37845402950552),(32.7480372060297,55.90721384574556),(35.67979503619571,55.68634475630185),(32.83871012861215,56.311688992608396),(34.591719965206266,56.29492065473883),(35.7100193437232,56.311688992608396),(33.83611227701915,56.695333481003644),(32.95960735872209,56.9434497616887),(36.072711034053015,57.091531913901434),(33.171177511414484,57.33702717078384),(36.193608264162954,57.499784781503735),(33.23162612646945,57.77481561306047),(36.43540272438284,58.04776787540811),(33.62454212432676,58.27099811968307),(36.344729801800376,58.54018474404165),(33.83611227701915,58.68186423448108),(34.74284150284369,59.565911441555244),(33.473420586689336,58.85424941916091)]], [[(34.65216858026123,58.91672306881671),(37.19101041256995,58.68186423448108),(36.01226241899805,58.28688958537609),(37.16078610504247,58.04776787540811),(35.74024365125068,57.79092907387934),(37.009664567405046,57.499784781503735),(35.77046795877817,57.25537683364851),(36.979440259877556,57.07510745541089),(34.22902827487645,56.794777197297435),(36.7074214921302,56.210968525786996),(34.712617195316206,56.10998276812964),(36.55629995449277,55.63519693782703),(35.13575750070099,55.53270067649592),(36.43540272438284,55.34409504165558),(34.83351442542614,55.01619492319591),(35.61934642114075,54.49294870011772),(34.89396304048112,54.12264226523038),(35.37755196092087,53.046178687628185),(37.43280487278982,52.95523300597458),(35.92158949641559,53.80257986695776),(36.91899164482259,53.856094327816805),(36.01226241899805,54.75541714463799),(37.765272255592166,55.189110239786885),(36.828318722240134,55.44708256557195),(38.03729102333953,55.652253637168315),(36.64697287707522,55.941082594334574),(38.21863686850443,56.05939028508024),(36.37495410932787,56.64551287174558),(38.30930979108689,56.992876013526654),(37.16078610504247,57.25537683364851),(38.127963945921984,57.516020773674256),(37.43280487278982,57.710289827306724),(38.33953409861437,57.935626886818994),(37.40258056526235,58.31865112960426),(38.58132855883426,58.744648733419496),(37.31190764267989,59.02578062465136),(34.65216858026123,58.91672306881671)]], [[(38.52087994377928,59.11898412389468),(39.54850639971376,58.713270635642914),(38.369758406141855,58.28688958537609),(38.85334732658162,58.06375936407028),(38.33953409861437,57.710289827306724),(38.73245009647167,57.48354156434209),(38.21863686850443,57.271721400459285),(38.97424455669155,56.87744603722649),(37.463029180317314,56.5623320541159),(38.94402024916407,56.05939028508024),(38.18841256097694,55.856355210835915),(38.490655636251795,55.53270067649592),(37.795496563119656,55.39562234093384),(38.30930979108689,55.154587013355666),(36.7074214921302,54.65063295250911),(37.31190764267989,53.92734063371401),(36.979440259877556,53.58783775557231),(37.855945178174615,52.91880107773497),(39.57873070724124,52.69956490610895),(38.33953409861437,53.281741738901104),(40.00187101262603,53.35396273604752),(39.54850639971376,53.58783775557231),(40.24366547284591,53.58783775557231),(39.97164670509855,53.98069568468355),(40.60635716317572,54.03398248547225),(40.39478701048334,54.44025165268903),(39.54850639971376,54.56310590284329),(39.54850639971376,54.87732350170489),(40.39478701048334,54.87732350170489),(40.39478701048334,55.24083903654295),(39.82052516746112,55.2752875586599),(39.760076552406154,55.75443792473942),(40.57613285564824,55.78844000174894),(40.425011318010824,56.19415599955667),(39.82052516746112,56.07626182891758),(39.79030085993364,56.41214455508424),(40.48545993306579,56.495655446714636),(40.33433839542836,56.95993246553937),(39.79030085993364,56.992876013526654),(39.72985224487867,57.46729112028032),(40.33433839542836,57.46729112028032),(40.24366547284591,58.04776787540811),(39.63917932229622,58.04776787540811),(39.63917932229622,58.382088724871295),(40.33433839542836,58.382088724871295),(40.45523562553831,58.9011152358548),(38.52087994377928,59.11898412389468)]]], [[(24.367675781249993,61.45977057029751),(19.577636718749993,58.67693767258692),(19.577636718749993,57.492213666700735),(19.445800781249996,55.87531083569678),(19.445800781249996,54.085173420886775),(17.468261718749996,53.014783245859235),(20.017089843749993,51.563412328675895),(21.203613281249993,50.205033264943324),(26.125488281249993,50.40151532278236),(27.22412109374999,48.980216985374994),(32.80517578124999,49.525208341974405),(35.26611328124999,48.74894534343292),(36.93603515624999,49.66762782262194),(42.56103515625,48.77791275550183),(43.92333984374999,49.8096315635631),(47.17529296875,49.152969656170455),(49.28466796875,50.54136296522162),(48.05419921875,51.17934297928929),(51.39404296875,52.48278022207825),(50.64697265625,53.014783245859235),(52.88818359375,53.93021986394004),(51.65771484374999,54.29088164657006),(52.66845703125,55.825973254619015),(50.25146484375,56.145549500679095),(51.92138671875,57.914847767009206),(49.15283203125,58.17070248348605),(49.59228515625,60.086762746260064),(47.043457031249986,59.88893689676584),(43.57177734375,61.37567331572748),(42.64892578125,60.630101766266705),(36.89208984374999,62.000904713685856),(36.01318359374999,61.143235250840576),(31.398925781249993,62.02152819100766),(30.563964843749996,61.05828537037917),(26.872558593749993,61.71070595883174),(26.652832031249993,61.10078883158897),(24.367675781249993,61.45977057029751)], [(24.455566406249993,59.42272750081452),(21.203613281249993,58.49369382056807),(21.335449218749993,56.89700392127261),(21.599121093749993,55.92458580482949),(25.202636718749993,55.998380955359636),(28.850097656249993,57.06463027327854),(27.09228515625,57.844750992890994),(28.806152343749996,59.17592824927138),(26.257324218749993,59.17592824927138),(24.455566406249993,59.42272750081452)], [(35.13427734375,59.84481485969107),(31.970214843749993,58.97266715450152),(33.20068359374999,56.776808316568406),(36.67236328125,56.41390137600675),(39.08935546874999,57.25528054528888),(42.69287109374999,58.03137242177638),(40.89111328124999,59.26588062825809),(37.28759765625,58.722598828043374),(37.11181640624999,59.66774058164964),(35.13427734375,59.84481485969107)], [(29.157714843749993,55.75184939173528),(22.565917968749993,55.128649068488784),(22.565917968749993,53.54030739150019),(22.038574218749996,51.48138289610097),(26.257324218749993,51.42661449707484),(30.124511718749993,50.54136296522162),(32.18994140624999,51.17934297928929),(30.124511718749993,53.173119202640635),(35.09033203124999,53.173119202640635),(33.11279296875,54.085173420886775),(29.597167968749993,55.50374985927513),(29.157714843749993,55.75184939173528)], [(42.82470703125,56.58369172128337),(36.584472656249986,55.329144408405085),(37.99072265625,53.592504809039355),(34.95849609374999,51.48138289610097),(36.54052734374999,50.40151532278236),(39.66064453124999,50.289339253291786),(39.79248046875,52.13348804077148),(41.77001953125,50.68079714532166),(44.49462890624999,51.97134580885171),(47.30712890624999,52.509534770327264),(44.05517578125,53.54030739150019),(46.60400390625,53.696706475303245),(47.61474609375,55.40406982700608),(45.37353515625,55.40406982700608),(42.82470703125,56.58369172128337)]]) format TSV; + +select '-------- Polygon with Polygon with Holes'; +select polygonsWithinSpherical([[(29.453587685533865,59.779570356240356),(29.393139070478895,52.276266797422124),(40.636581470703206,59.38168915000267),(41.21084331372543,59.103467777099866),(29.786055068336193,52.146627480315004),(31.23682182965546,52.16517054781818),(41.69443223416517,58.85424941916091),(42.51048853740727,58.47703162291134),(32.59691566839227,52.22075341251539),(34.289476889931414,52.22075341251539),(43.02430176537451,58.07974369546071),(43.02430176537451,57.25537683364851),(35.468224883503325,52.2022335126388),(37.16078610504247,52.23926559241349),(43.02430176537451,56.26136189644947),(43.02430176537451,55.326904361850836),(38.33953409861437,52.16517054781818),(40.09254393520848,52.16517054781818),(44.4146199116388,55.3097062225408),(44.47506852669377,59.80998197603594),(39.72985224487867,59.931351417569715),(30.23941968124846,53.67744677450975),(30.20919537372098,54.63314259659509),(38.73245009647167,59.94649146557819),(37.2816833351524,59.97675082987618),(30.23941968124846,55.2752875586599),(30.33009260383092,56.19415599955667),(36.28428118674541,59.96162460231375),(34.863738732953635,59.97675082987618),(30.178971066193498,56.97640788219866),(30.178971066193498,57.91957806959033),(33.65476643185424,59.94649146557819),(32.32489690064491,59.94649146557819),(30.481214141468342,58.85424941916091),(30.571887064050795,59.99187015036608),(29.453587685533865,59.779570356240356)]], [[(24.367675781249993,61.45977057029751),(19.577636718749993,58.67693767258692),(19.577636718749993,57.492213666700735),(19.445800781249996,55.87531083569678),(19.445800781249996,54.085173420886775),(17.468261718749996,53.014783245859235),(20.017089843749993,51.563412328675895),(21.203613281249993,50.205033264943324),(26.125488281249993,50.40151532278236),(27.22412109374999,48.980216985374994),(32.80517578124999,49.525208341974405),(35.26611328124999,48.74894534343292),(36.93603515624999,49.66762782262194),(42.56103515625,48.77791275550183),(43.92333984374999,49.8096315635631),(47.17529296875,49.152969656170455),(49.28466796875,50.54136296522162),(48.05419921875,51.17934297928929),(51.39404296875,52.48278022207825),(50.64697265625,53.014783245859235),(52.88818359375,53.93021986394004),(51.65771484374999,54.29088164657006),(52.66845703125,55.825973254619015),(50.25146484375,56.145549500679095),(51.92138671875,57.914847767009206),(49.15283203125,58.17070248348605),(49.59228515625,60.086762746260064),(47.043457031249986,59.88893689676584),(43.57177734375,61.37567331572748),(42.64892578125,60.630101766266705),(36.89208984374999,62.000904713685856),(36.01318359374999,61.143235250840576),(31.398925781249993,62.02152819100766),(30.563964843749996,61.05828537037917),(26.872558593749993,61.71070595883174),(26.652832031249993,61.10078883158897),(24.367675781249993,61.45977057029751)], [(24.455566406249993,59.42272750081452),(21.203613281249993,58.49369382056807),(21.335449218749993,56.89700392127261),(21.599121093749993,55.92458580482949),(25.202636718749993,55.998380955359636),(28.850097656249993,57.06463027327854),(27.09228515625,57.844750992890994),(28.806152343749996,59.17592824927138),(26.257324218749993,59.17592824927138),(24.455566406249993,59.42272750081452)], [(35.13427734375,59.84481485969107),(31.970214843749993,58.97266715450152),(33.20068359374999,56.776808316568406),(36.67236328125,56.41390137600675),(39.08935546874999,57.25528054528888),(42.69287109374999,58.03137242177638),(40.89111328124999,59.26588062825809),(37.28759765625,58.722598828043374),(37.11181640624999,59.66774058164964),(35.13427734375,59.84481485969107)], [(29.157714843749993,55.75184939173528),(22.565917968749993,55.128649068488784),(22.565917968749993,53.54030739150019),(22.038574218749996,51.48138289610097),(26.257324218749993,51.42661449707484),(30.124511718749993,50.54136296522162),(32.18994140624999,51.17934297928929),(30.124511718749993,53.173119202640635),(35.09033203124999,53.173119202640635),(33.11279296875,54.085173420886775),(29.597167968749993,55.50374985927513),(29.157714843749993,55.75184939173528)], [(42.82470703125,56.58369172128337),(36.584472656249986,55.329144408405085),(37.99072265625,53.592504809039355),(34.95849609374999,51.48138289610097),(36.54052734374999,50.40151532278236),(39.66064453124999,50.289339253291786),(39.79248046875,52.13348804077148),(41.77001953125,50.68079714532166),(44.49462890624999,51.97134580885171),(47.30712890624999,52.509534770327264),(44.05517578125,53.54030739150019),(46.60400390625,53.696706475303245),(47.61474609375,55.40406982700608),(45.37353515625,55.40406982700608),(42.82470703125,56.58369172128337)]]) format TSV; +select polygonsWithinMercator([[(29.453587685533865,59.779570356240356),(29.393139070478895,52.276266797422124),(40.636581470703206,59.38168915000267),(41.21084331372543,59.103467777099866),(29.786055068336193,52.146627480315004),(31.23682182965546,52.16517054781818),(41.69443223416517,58.85424941916091),(42.51048853740727,58.47703162291134),(32.59691566839227,52.22075341251539),(34.289476889931414,52.22075341251539),(43.02430176537451,58.07974369546071),(43.02430176537451,57.25537683364851),(35.468224883503325,52.2022335126388),(37.16078610504247,52.23926559241349),(43.02430176537451,56.26136189644947),(43.02430176537451,55.326904361850836),(38.33953409861437,52.16517054781818),(40.09254393520848,52.16517054781818),(44.4146199116388,55.3097062225408),(44.47506852669377,59.80998197603594),(39.72985224487867,59.931351417569715),(30.23941968124846,53.67744677450975),(30.20919537372098,54.63314259659509),(38.73245009647167,59.94649146557819),(37.2816833351524,59.97675082987618),(30.23941968124846,55.2752875586599),(30.33009260383092,56.19415599955667),(36.28428118674541,59.96162460231375),(34.863738732953635,59.97675082987618),(30.178971066193498,56.97640788219866),(30.178971066193498,57.91957806959033),(33.65476643185424,59.94649146557819),(32.32489690064491,59.94649146557819),(30.481214141468342,58.85424941916091),(30.571887064050795,59.99187015036608),(29.453587685533865,59.779570356240356)]], [[(24.367675781249993,61.45977057029751),(19.577636718749993,58.67693767258692),(19.577636718749993,57.492213666700735),(19.445800781249996,55.87531083569678),(19.445800781249996,54.085173420886775),(17.468261718749996,53.014783245859235),(20.017089843749993,51.563412328675895),(21.203613281249993,50.205033264943324),(26.125488281249993,50.40151532278236),(27.22412109374999,48.980216985374994),(32.80517578124999,49.525208341974405),(35.26611328124999,48.74894534343292),(36.93603515624999,49.66762782262194),(42.56103515625,48.77791275550183),(43.92333984374999,49.8096315635631),(47.17529296875,49.152969656170455),(49.28466796875,50.54136296522162),(48.05419921875,51.17934297928929),(51.39404296875,52.48278022207825),(50.64697265625,53.014783245859235),(52.88818359375,53.93021986394004),(51.65771484374999,54.29088164657006),(52.66845703125,55.825973254619015),(50.25146484375,56.145549500679095),(51.92138671875,57.914847767009206),(49.15283203125,58.17070248348605),(49.59228515625,60.086762746260064),(47.043457031249986,59.88893689676584),(43.57177734375,61.37567331572748),(42.64892578125,60.630101766266705),(36.89208984374999,62.000904713685856),(36.01318359374999,61.143235250840576),(31.398925781249993,62.02152819100766),(30.563964843749996,61.05828537037917),(26.872558593749993,61.71070595883174),(26.652832031249993,61.10078883158897),(24.367675781249993,61.45977057029751)], [(24.455566406249993,59.42272750081452),(21.203613281249993,58.49369382056807),(21.335449218749993,56.89700392127261),(21.599121093749993,55.92458580482949),(25.202636718749993,55.998380955359636),(28.850097656249993,57.06463027327854),(27.09228515625,57.844750992890994),(28.806152343749996,59.17592824927138),(26.257324218749993,59.17592824927138),(24.455566406249993,59.42272750081452)], [(35.13427734375,59.84481485969107),(31.970214843749993,58.97266715450152),(33.20068359374999,56.776808316568406),(36.67236328125,56.41390137600675),(39.08935546874999,57.25528054528888),(42.69287109374999,58.03137242177638),(40.89111328124999,59.26588062825809),(37.28759765625,58.722598828043374),(37.11181640624999,59.66774058164964),(35.13427734375,59.84481485969107)], [(29.157714843749993,55.75184939173528),(22.565917968749993,55.128649068488784),(22.565917968749993,53.54030739150019),(22.038574218749996,51.48138289610097),(26.257324218749993,51.42661449707484),(30.124511718749993,50.54136296522162),(32.18994140624999,51.17934297928929),(30.124511718749993,53.173119202640635),(35.09033203124999,53.173119202640635),(33.11279296875,54.085173420886775),(29.597167968749993,55.50374985927513),(29.157714843749993,55.75184939173528)], [(42.82470703125,56.58369172128337),(36.584472656249986,55.329144408405085),(37.99072265625,53.592504809039355),(34.95849609374999,51.48138289610097),(36.54052734374999,50.40151532278236),(39.66064453124999,50.289339253291786),(39.79248046875,52.13348804077148),(41.77001953125,50.68079714532166),(44.49462890624999,51.97134580885171),(47.30712890624999,52.509534770327264),(44.05517578125,53.54030739150019),(46.60400390625,53.696706475303245),(47.61474609375,55.40406982700608),(45.37353515625,55.40406982700608),(42.82470703125,56.58369172128337)]]) format TSV; diff --git a/tests/queries/0_stateless/01304_polygons_sym_difference.reference b/tests/queries/0_stateless/01304_polygons_sym_difference.reference index 3b035601be7..aff588052c1 100644 --- a/tests/queries/0_stateless/01304_polygons_sym_difference.reference +++ b/tests/queries/0_stateless/01304_polygons_sym_difference.reference @@ -1 +1,10 @@ [[[(1,2.9),(1,1),(2.9,1),(3,0),(0,0),(0,3),(1,2.9)]],[[(1,2.9),(1,4),(4,4),(4,1),(2.9,1),(2.6,2),(2,2.6),(1,2.9)]]] +-------- MultiPolygon with Polygon +MULTIPOLYGON(((36.9725 59.0149,35.5408 58.9593,37.2817 59.9768,38.7325 59.9465,36.9725 59.0149)),((36.9725 59.0149,37.3119 59.0258,37.8553 58.9075,36.5949 58.1673,36.0123 58.2869,37.191 58.6819,36.4989 58.7512,36.9725 59.0149)),((36.151 54.791,37.7653 55.1891,37.06 55.3843,37.2824 55.5258,38.0373 55.6523,37.6238 55.7402,38.1319 56.0534,38.2186 56.0594,38.1688 56.0758,38.4339 56.2361,38.944 56.0594,38.1884 55.8564,38.4907 55.5327,37.7955 55.3956,38.2609 55.1775,38.1601 55.1091,36.7074 54.6506,37.0035 54.2999,36.6985 54.0791,36.0472 54.7217,36.151 54.791)),((36.151 54.791,36.0123 54.7554,36.0472 54.7217,34.9611 53.9765,34.894 54.1226,35.6193 54.4929,34.9706 54.9262,35.2275 55.0993,36.4354 55.3441,35.7505 55.4454,35.9817 55.5958,36.5563 55.6352,36.193 55.7319,37.2281 56.3799,38.1688 56.0758,38.1319 56.0534,36.647 55.9411,37.6238 55.7402,37.2824 55.5258,36.8283 55.4471,37.06 55.3843,36.151 54.791)),((36.5334 56.6753,38.2312 56.9795,37.565 56.5843,37.463 56.5623,37.5054 56.5484,37.2281 56.3799,36.4446 56.6242,36.5334 56.6753)),((36.5334 56.6753,36.375 56.6455,36.4446 56.6242,36.0233 56.3789,35.4083 56.5254,36.1999 57.0022,36.9794 57.0751,36.4587 57.1544,38.0535 58.0542,38.3395 57.9356,37.4328 57.7103,38.0744 57.5312,37.9669 57.4734,37.1608 57.2554,37.4489 57.1909,36.5334 56.6753)),((36.8709 53.2765,37.135 53.4711,37.8559 52.9188,38.0214 52.8989,37.1608 52.2393,35.4682 52.2022,36.5022 53.0008,37.4328 52.9552,36.8709 53.2765)),((36.8709 53.2765,36.5022 53.0008,35.3776 53.0462,35.3645 53.076,36.1528 53.6763,36.8709 53.2765)),((36.6985 54.0791,36.919 53.8561,36.3552 53.8269,36.6985 54.0791)),((35.5408 58.9593,35.3712 58.8556,34.6522 58.9167,35.5408 58.9593)),((36.0848 57.855,36.3932 58.0447,36.4354 58.0478,36.403 58.0507,36.5949 58.1673,37.1608 58.0478,36.0848 57.855)),((36.0848 57.855,35.9179 57.7512,35.7402 57.7909,36.0848 57.855)),((37.135 53.4711,36.9794 53.5878,37.3119 53.9273,37.0035 54.2999,38.1601 55.1091,38.3093 55.1546,38.2609 55.1775,39.8102 56.1914,39.8205 56.0763,40.425 56.1942,40.5716 55.8007,40.5504 55.7875,39.7601 55.7544,39.8151 55.3187,37.135 53.4711)),((38.2312 56.9795,38.2699 57.0021,38.3093 56.9929,38.2312 56.9795)),((36.4989 58.7512,36.1498 58.553,34.9952 58.6226,35.3712 58.8556,36.4989 58.7512)),((36.4587 57.1544,36.1999 57.0022,34.4816 56.8232,34.8098 57.0409,36.0727 57.0915,35.0338 57.1875,35.4682 57.4674,36.1936 57.4998,35.613 57.5595,35.9179 57.7512,37.0097 57.4998,35.7705 57.2554,36.4587 57.1544)),((38.0535 58.0542,37.4026 58.3187,38.5813 58.7446,37.8553 58.9075,39.7299 59.9314,44.4751 59.81,44.4146 55.3097,40.0925 52.1652,38.3395 52.1652,39.1456 52.7573,39.5787 52.6996,39.2704 52.8471,39.9877 53.3534,40.0019 53.354,39.9942 53.358,43.0243 55.3269,43.0243 56.2614,40.2143 54.467,39.5485 54.5631,39.5485 54.8773,40.3948 54.8773,40.3948 55.2408,39.8205 55.2753,39.8151 55.3187,40.5504 55.7875,40.5761 55.7884,40.5716 55.8007,43.0243 57.2554,43.0243 58.0797,40.4543 56.5923,40.3343 56.9599,39.7903 56.9929,39.7863 57.025,42.5105 58.477,41.6944 58.8542,40.1389 58.048,39.6392 58.0478,39.6392 58.3427,39.7184 58.3823,40.3343 58.3821,40.4136 58.7241,41.2108 59.1035,40.6366 59.3817,39.8163 58.9766,38.5209 59.119,39.4085 58.7696,38.7465 58.4255,38.3698 58.2869,38.432 58.2584,38.0535 58.0542)),((34.4996 55.9565,33.5244 56.1686,33.7222 56.3063,34.5917 56.2949,35.0485 56.303,34.744 56.1118,34.7126 56.11,34.7331 56.1049,34.4996 55.9565)),((34.4996 55.9565,35.0954 55.822,34.9721 55.7463,34.2598 55.8023,34.4996 55.9565)),((31.6069 56.3194,31.5088 55.9411,31.7782 55.7778,30.2092 54.6331,30.2394 53.6774,31.7439 54.8677,31.4182 54.4227,31.8748 54.1736,29.3931 52.2763,29.4536 59.7796,30.5719 59.9919,30.4812 58.8542,32.3249 59.9465,33.6548 59.9465,30.179 57.9196,30.179 56.9764,32.2175 58.3664,32.1738 58.0318,31.5088 57.4998,31.6514 57.1258,30.3301 56.1942,30.2394 55.2753,31.6069 56.3194)),((31.6069 56.3194,31.7506 56.8609,31.6514 57.1258,34.0496 58.6717,34.9952 58.6226,34.6028 58.3749,33.6245 58.271,34.3593 58.2189,33.7581 57.8255,33.2316 57.7748,33.6325 57.7419,31.6069 56.3194)),((33.5244 56.1686,33.1204 55.8832,32.748 55.9072,32.9547 55.7645,31.7439 54.8677,31.8413 54.9989,32.204 55.5156,31.7782 55.7778,33.3418 56.8364,33.8361 56.6953,34.1885 56.6259,33.7222 56.3063,32.8387 56.3117,33.5244 56.1686)),((33.1204 55.8832,34.2598 55.8023,33.6125 55.3778,33.5036 55.3785,32.9547 55.7645,33.1204 55.8832)),((35.3188 55.9582,36.193 55.7319,35.9817 55.5958,35.1358 55.5327,35.7505 55.4454,35.2275 55.0993,34.8335 55.0162,34.9706 54.9262,34.7231 54.7576,34.2593 54.9642,35.0149 55.3613,34.3709 55.3709,34.9721 55.7463,35.6798 55.6863,35.0954 55.822,35.3188 55.9582)),((35.3188 55.9582,34.7331 56.1049,34.744 56.1118,35.6571 56.1619,35.3188 55.9582)),((33.3418 56.8364,32.9596 56.9434,33.5602 56.9781,33.3418 56.8364)),((33.4048 52.8423,34.7731 52.9188,34.7731 53.7847,34.7279 53.8116,34.9611 53.9765,35.3645 53.076,34.2895 52.2208,32.5969 52.2208,33.4048 52.8423)),((33.4048 52.8423,33.1712 52.8276,32.5275 53.1741,34.7231 54.7576,35.0753 54.5981,34.1081 54.1757,34.7279 53.8116,33.4048 52.8423)),((32.2523 53.964,32.476 53.8383,32.0831 53.408,32.5275 53.1741,31.2368 52.1652,29.7861 52.1466,32.2523 53.964)),((32.2523 53.964,31.8748 54.1736,33.6125 55.3778,34.3709 55.3709,32.2523 53.964)),((36.3552 53.8269,36.1528 53.6763,35.9216 53.8026,36.3552 53.8269)),((32.5691 58.5924,34.8637 59.9768,36.2843 59.9616,34.0496 58.6717,33.8361 58.6819,34.7428 59.5659,33.4734 58.8542,32.5691 58.5924)),((32.5691 58.5924,32.2175 58.3664,32.2342 58.4928,32.5691 58.5924)),((33.5602 56.9781,34.0208 57.2724,35.0338 57.1875,34.8098 57.0409,33.5602 56.9781)),((36.3932 58.0447,35.1134 57.9454,35.4314 58.1349,36.403 58.0507,36.3932 58.0447)),((35.1134 57.9454,34.6332 57.6538,33.6325 57.7419,33.7581 57.8255,35.1134 57.9454)),((35.4314 58.1349,34.3593 58.2189,34.6028 58.3749,36.0877 58.5174,35.4314 58.1349)),((35.4682 57.4674,34.2274 57.4023,34.6332 57.6538,35.613 57.5595,35.4682 57.4674)),((34.4816 56.8232,34.3867 56.7596,34.229 56.7948,34.4816 56.8232)),((34.1885 56.6259,34.3867 56.7596,35.4083 56.5254,35.2273 56.414,34.1885 56.6259)),((34.2274 57.4023,34.0208 57.2724,33.1712 57.337,34.2274 57.4023)),((35.0485 56.303,35.2273 56.414,35.71 56.3117,35.0485 56.303)),((35.6571 56.1619,36.0233 56.3789,36.7074 56.211,35.6571 56.1619)),((36.1498 58.553,36.3447 58.5402,36.0877 58.5174,36.1498 58.553)),((40.2143 54.467,40.3948 54.4403,40.6064 54.034,39.9716 53.9807,40.2437 53.5878,39.5485 53.5878,39.9942 53.358,39.9877 53.3534,38.5511 53.2922,40.2143 54.467)),((39.8102 56.1914,39.7903 56.4121,40.2529 56.4682,39.8102 56.1914)),((38.0214 52.8989,38.4609 53.226,39.2704 52.8471,39.1456 52.7573,38.0214 52.8989)),((38.5511 53.2922,38.4609 53.226,38.3395 53.2817,38.5511 53.2922)),((40.4543 56.5923,40.4855 56.4957,40.2529 56.4682,40.4543 56.5923)),((40.1389 58.048,40.2437 58.0478,40.3343 57.4673,39.7299 57.4673,39.7863 57.025,38.4339 56.2361,37.5054 56.5484,37.565 56.5843,38.9742 56.8774,38.4915 57.1308,40.1389 58.048)),((40.4136 58.7241,39.7184 58.3823,39.6392 58.3821,39.6392 58.3427,38.3737 57.6908,38.3395 57.7103,38.8533 58.0638,38.432 58.2584,38.7465 58.4255,39.5485 58.7133,39.4085 58.7696,39.8163 58.9766,40.4552 58.9011,40.4136 58.7241)),((38.3737 57.6908,38.7325 57.4835,38.2186 57.2717,38.4915 57.1308,38.2699 57.0021,37.4489 57.1909,37.9669 57.4734,38.128 57.516,38.0744 57.5312,38.3737 57.6908))) +MULTIPOLYGON(((37.1527 59.0192,35.677 58.9587,37.2816 59.9767,38.7324 59.9464,37.1527 59.0192)),((37.1527 59.0192,37.3118 59.0257,38.0059 58.8722,36.8134 58.1201,36.0122 58.2868,37.1909 58.6818,36.6691 58.7302,37.1527 59.0192)),((36.6884 54.9232,37.7652 55.189,37.3054 55.3158,37.7656 55.6061,38.0372 55.6522,37.8883 55.6832,38.6515 56.1591,38.9439 56.0593,38.1883 55.8563,38.4906 55.5326,37.7954 55.3955,38.3092 55.1545,36.7073 54.6505,37.0903 54.1938,36.7891 53.9859,36.1761 54.5942,36.6884 54.9232)),((36.6884 54.9232,36.0122 54.7553,36.1761 54.5942,35.0208 53.8426,34.8939 54.1225,35.6192 54.4928,35.153 54.8041,35.777 55.2096,36.4353 55.344,36.0682 55.3974,36.4239 55.6256,36.5562 55.6351,36.4727 55.6567,36.8521 55.8985,37.8883 55.6832,37.7656 55.6061,36.8282 55.447,37.3054 55.3158,36.6884 54.9232)),((37.1816 56.7907,38.3092 56.9928,37.7436 57.1223,38.5465 57.5909,38.7324 57.4834,38.2185 57.2716,38.6704 57.0364,38.1298 56.7016,37.4629 56.5622,37.7506 56.465,37.4799 56.2952,36.7422 56.5294,37.1816 56.7907)),((37.1816 56.7907,36.3749 56.6454,36.7422 56.5294,36.3512 56.2953,35.7316 56.4418,36.7103 57.0477,36.9793 57.075,36.7988 57.102,37.6831 57.6403,38.1279 57.5159,37.1607 57.2553,37.7436 57.1223,37.1816 56.7907)),((36.7891 53.9859,36.9189 53.856,36.575 53.8375,36.7891 53.9859)),((36.8521 55.8985,36.6469 55.941,36.9557 55.9643,36.8521 55.8985)),((35.677 58.9587,35.4917 58.8391,34.6521 58.9166,35.677 58.9587)),((36.5188 57.9318,36.8134 58.1201,37.1607 58.0477,36.5188 57.9318)),((36.5188 57.9318,36.1527 57.6965,35.7401 57.7908,36.5188 57.9318)),((37.0903 54.1938,39.9194 56.0955,40.4249 56.1941,40.576 55.7883,39.76 55.7543,39.8204 55.2752,39.9389 55.2681,37.2027 53.4183,36.9793 53.5877,37.3118 53.9272,37.0903 54.1938)),((36.9557 55.9643,37.4799 56.2952,38.2185 56.0593,36.9557 55.9643)),((36.6691 58.7302,35.6455 58.1105,34.5125 58.2005,34.7969 58.3872,36.3446 58.5401,35.1359 58.6084,35.4917 58.8391,36.6691 58.7302)),((36.7988 57.102,36.7103 57.0477,34.8711 56.8603,35.1561 57.0479,36.0726 57.0914,35.3199 57.1553,35.8173 57.4795,36.1935 57.4997,35.8918 57.5278,36.1527 57.6965,37.0096 57.4997,35.7704 57.2553,36.7988 57.102)),((36.5898 52.9925,37.4327 52.9551,36.9376 53.2346,37.2027 53.4183,37.8558 52.9187,38.0733 52.8911,37.1607 52.2392,35.4681 52.2021,36.5898 52.9925)),((36.5898 52.9925,35.4415 53.0432,36.2567 53.616,36.9376 53.2346,36.5898 52.9925)),((37.6831 57.6403,37.4327 57.7102,38.0568 57.8654,37.6831 57.6403)),((35.245 56.1369,34.7125 56.1099,35.0606 56.0207,34.8294 55.8745,33.7487 56.1124,34.0285 56.3002,34.5916 56.2948,35.5188 56.3087,35.245 56.1369)),((35.245 56.1369,36.1651 56.1834,35.6467 55.87,35.0606 56.0207,35.245 56.1369)),((34.0285 56.3002,32.8386 56.3116,33.7487 56.1124,33.3749 55.86,32.7479 55.9071,33.0915 55.6676,31.5514 54.605,31.8412 54.9988,32.2039 55.5155,31.863 55.7247,33.5159 56.786,33.836 56.6952,34.4368 56.5727,34.0285 56.3002)),((36.3512 56.2953,36.7073 56.2109,36.1651 56.1834,36.3512 56.2953)),((31.581 56.2183,31.5087 55.941,31.863 55.7247,30.2091 54.633,30.2393 53.6773,31.5514 54.605,31.4181 54.4226,32.0505 54.0742,29.393 52.2762,29.4535 59.7795,30.5718 59.9918,30.4811 58.8541,32.3248 59.9464,33.6547 59.9464,30.1789 57.9195,30.1789 56.9763,32.2099 58.3086,32.1737 58.0317,31.5087 57.4997,31.6694 57.0765,30.33 56.1941,30.2393 55.2752,31.581 56.2183)),((31.581 56.2183,31.7505 56.8608,31.6694 57.0765,34.1607 58.6635,35.1359 58.6084,34.7969 58.3872,33.6244 58.2709,34.5125 58.2005,33.9633 57.8372,33.2315 57.7747,33.7916 57.7229,33.2199 57.3396,33.1711 57.3369,33.211 57.3336,31.581 56.2183)),((33.5159 56.786,32.9595 56.9433,33.8307 56.9848,33.5159 56.786)),((33.0915 55.6676,33.3749 55.86,34.6546 55.7636,34.0415 55.3722,33.5035 55.3784,33.0915 55.6676)),((33.5229 52.8475,34.773 52.9187,34.773 53.6795,35.0208 53.8426,35.3775 53.0461,35.4415 53.0432,34.2894 52.2207,32.5968 52.2207,33.5229 52.8475)),((33.5229 52.8475,33.1711 52.8275,32.6253 53.1196,34.1558 54.1477,34.773 53.7846,34.773 53.6795,33.5229 52.8475)),((32.4267 53.8656,32.4759 53.8382,32.083 53.4079,32.6253 53.1196,31.2367 52.1651,29.786 52.1465,32.4267 53.8656)),((32.4267 53.8656,32.0505 54.0742,34.0415 55.3722,34.8176 55.3634,32.4267 53.8656)),((34.1558 54.1477,34.108 54.1756,34.374 54.2922,34.1558 54.1477)),((34.374 54.2922,34.9347 54.6613,35.0752 54.598,34.374 54.2922)),((36.575 53.8375,36.2567 53.616,35.9215 53.8025,36.575 53.8375)),((34.8294 55.8745,35.4341 55.7407,35.3816 55.7088,34.6546 55.7636,34.8294 55.8745)),((34.9347 54.6613,34.2592 54.9641,35.0148 55.3612,34.8176 55.3634,35.3816 55.7088,35.6797 55.6862,35.4341 55.7407,35.6467 55.87,36.4727 55.6567,36.4239 55.6256,35.1357 55.5326,36.0682 55.3974,35.777 55.2096,34.8334 55.0161,35.153 54.8041,34.9347 54.6613)),((32.7181 58.6343,34.8636 59.9767,36.2842 59.9615,34.1607 58.6635,33.836 58.6818,34.7427 59.5658,33.4733 58.8541,32.7181 58.6343)),((32.7181 58.6343,32.2099 58.3086,32.2341 58.4927,32.7181 58.6343)),((33.8307 56.9848,34.2469 57.2461,35.3199 57.1553,35.1561 57.0479,33.8307 56.9848)),((35.6455 58.1105,36.4353 58.0477,35.3989 57.9596,35.6455 58.1105)),((33.9633 57.8372,35.3989 57.9596,34.8546 57.6243,33.7916 57.7229,33.9633 57.8372)),((34.8711 56.8603,34.6299 56.7008,34.2289 56.7947,34.8711 56.8603)),((34.8546 57.6243,35.8918 57.5278,35.8173 57.4795,34.5079 57.409,34.8546 57.6243)),((34.4368 56.5727,34.6299 56.7008,35.7316 56.4418,35.5695 56.3405,34.4368 56.5727)),((35.5188 56.3087,35.5695 56.3405,35.7099 56.3116,35.5188 56.3087)),((34.5079 57.409,34.2469 57.2461,33.211 57.3336,33.2199 57.3396,34.5079 57.409)),((39.9194 56.0955,39.8204 56.0762,39.7902 56.412,40.4854 56.4956,40.3342 56.9598,39.9915 56.9806,42.5104 58.4769,41.6943 58.8541,40.2513 57.9982,40.2436 58.0477,39.6391 58.0477,39.6391 58.219,39.9258 58.382,40.3342 58.382,40.3959 58.6476,41.2107 59.1034,40.6365 59.3816,39.9119 58.9623,38.5208 59.1189,39.5108 58.7281,39.3854 58.6545,38.3697 58.2868,38.591 58.1848,38.2411 57.9759,37.4025 58.3186,38.5812 58.7445,38.0059 58.8722,39.7298 59.9313,44.475 59.8099,44.4145 55.3096,40.0924 52.1651,38.3394 52.1651,39.1804 52.7502,39.5786 52.6995,39.2985 52.8318,43.0242 55.3268,43.0242 56.2613,40.3151 54.4517,39.5484 54.563,39.5484 54.8772,40.3947 54.8772,40.3947 55.2407,39.9389 55.2681,43.0242 57.2553,43.0242 58.0796,39.9194 56.0955)),((38.6515 56.1591,37.7506 56.465,38.1298 56.7016,38.9741 56.8773,38.6704 57.0364,40.2513 57.9982,40.3342 57.4672,39.7298 57.4672,39.7902 56.9928,39.9915 56.9806,38.6515 56.1591)),((40.3151 54.4517,40.3947 54.4402,40.6063 54.0339,39.9715 53.9806,40.2436 53.5877,39.5484 53.5877,40.0018 53.3539,38.6455 53.2949,40.3151 54.4517)),((38.0733 52.8911,38.512 53.2011,39.2985 52.8318,39.1804 52.7502,38.0733 52.8911)),((38.6455 53.2949,38.512 53.2011,38.3394 53.2816,38.6455 53.2949)),((40.3959 58.6476,39.9258 58.382,39.6391 58.382,39.6391 58.219,38.5465 57.5909,38.3394 57.7102,38.8532 58.0637,38.591 58.1848,39.3854 58.6545,39.5484 58.7132,39.5108 58.7281,39.9119 58.9623,40.4551 58.901,40.3959 58.6476)),((38.2411 57.9759,38.3394 57.9355,38.0568 57.8654,38.2411 57.9759))) +-------- MultiPolygon with Polygon with Holes +MULTIPOLYGON(((24.3677 61.4598,26.6528 61.1008,26.8726 61.7107,30.564 61.0583,31.3989 62.0215,36.0132 61.1432,36.8921 62.0009,42.6489 60.6301,43.5718 61.3757,47.0435 59.8889,49.5923 60.0868,49.1528 58.1707,51.9214 57.9148,50.2515 56.1455,52.6685 55.826,51.6577 54.2909,52.8882 53.9302,50.647 53.0148,51.394 52.4828,48.0542 51.1793,49.2847 50.5414,47.1753 49.153,43.9233 49.8096,42.561 48.7779,36.936 49.6676,35.2661 48.7489,32.8052 49.5252,27.2241 48.9802,26.1255 50.4015,21.2036 50.205,20.0171 51.5634,17.4683 53.0148,19.4458 54.0852,19.4458 55.8753,19.5776 57.4922,19.5776 58.6769,24.3677 61.4598),(24.4556 59.4227,21.2036 58.4937,21.3354 56.897,21.5991 55.9246,25.2026 55.9984,28.8501 57.0646,27.0923 57.8448,28.8062 59.1759,26.2573 59.1759,24.4556 59.4227),(35.1489 56.5859,36.7074 56.211,34.7126 56.11,36.5563 55.6352,35.1358 55.5327,36.4354 55.3441,34.8335 55.0162,35.6193 54.4929,34.894 54.1226,35.3776 53.0462,37.0604 52.9744,34.9585 51.4814,36.5405 50.4015,39.6606 50.2893,39.7925 52.1335,41.77 50.6808,44.4946 51.9713,47.3071 52.5095,44.0552 53.5403,46.604 53.6967,47.6147 55.4041,45.3735 55.4041,42.8247 56.5837,40.4412 56.1511,40.425 56.1942,39.8205 56.0763,39.7903 56.4121,40.4855 56.4957,40.3343 56.9599,39.7903 56.9929,39.7379 57.4051,40.0149 57.4677,40.3343 57.4673,40.3237 57.5365,42.6929 58.0314,40.8911 59.2659,39.2792 59.0373,38.5209 59.119,38.8838 58.9777,38.0944 58.8545,37.3119 59.0258,37.2327 59.0233,37.1118 59.6677,35.1343 59.8448,31.9702 58.9727,32.25 58.4976,32.2342 58.4928,32.1738 58.0318,31.5088 57.4998,31.7506 56.8609,31.5088 55.9411,32.204 55.5156,31.8413 54.9989,31.627 54.7093,29.5972 55.5037,29.1577 55.7518,22.5659 55.1286,22.5659 53.5403,22.0386 51.4814,26.2573 51.4266,30.1245 50.5414,32.1899 51.1793,30.1245 53.1731,32.4808 53.1989,33.1712 52.8276,34.7731 52.9188,34.7731 53.1793,35.0903 53.1731,34.7731 53.3243,34.7731 53.7847,34.1081 54.1757,35.0753 54.5981,34.2593 54.9642,35.0149 55.3613,33.5036 55.3785,32.748 55.9072,35.6798 55.6863,32.8387 56.3117,34.5917 56.2949,35.71 56.3117,33.8361 56.6953,33.7182 56.7292,35.1489 56.5859)),((35.1489 56.5859,34.229 56.7948,36.9794 57.0751,35.7705 57.2554,37.0097 57.4998,35.7402 57.7909,37.1608 58.0478,36.0123 58.2869,37.191 58.6819,34.6522 58.9167,37.2327 59.0233,37.2876 58.7226,38.0944 58.8545,38.5813 58.7446,37.4026 58.3187,38.3395 57.9356,37.4328 57.7103,38.128 57.516,37.1608 57.2554,38.3092 56.9929,38.309 56.9928,36.375 56.6455,36.8799 56.4895,36.6724 56.4139,35.1489 56.5859)),((33.1079 56.9523,32.25 58.4976,33.4734 58.8542,34.7428 59.5659,33.8361 58.6819,36.3447 58.5402,33.6245 58.271,36.4354 58.0478,33.2316 57.7748,36.1936 57.4998,33.1712 57.337,36.0727 57.0915,33.1079 56.9523)),((33.1079 56.9523,33.1392 56.8934,32.9596 56.9434,33.1079 56.9523)),((33.7182 56.7292,33.2007 56.7768,33.1392 56.8934,33.7182 56.7292)),((37.0604 52.9744,37.2165 53.0798,37.4328 52.9552,37.0604 52.9744)),((34.7731 53.3243,34.7731 53.1793,32.4808 53.1989,32.0831 53.408,32.476 53.8383,31.4182 54.4227,31.627 54.7093,33.1128 54.0852,34.7731 53.3243)),((36.9508 55.414,37.7653 55.1891,36.8822 54.975,36.5845 55.3291,36.9508 55.414)),((36.9508 55.414,36.8283 55.4471,37.9482 55.6376,36.9508 55.414)),((37.2165 53.0798,35.9216 53.8026,36.919 53.8561,36.0123 54.7554,36.8822 54.975,37.0572 54.7635,36.7074 54.6506,37.3119 53.9273,36.9794 53.5878,37.4471 53.2343,37.2165 53.0798)),((37.0572 54.7635,38.3093 55.1546,37.7955 55.3956,38.4907 55.5327,38.3184 55.7179,40.4412 56.1511,40.5761 55.7884,39.7601 55.7544,39.8205 55.2753,40.3948 55.2408,40.3948 54.8773,39.5485 54.8773,39.5485 54.5631,40.3948 54.4403,40.6064 54.034,39.9716 53.9807,40.2437 53.5878,39.5485 53.5878,40.0019 53.354,38.3395 53.2817,39.5787 52.6996,37.8559 52.9188,37.4471 53.2343,37.9907 53.5925,37.0572 54.7635)),((38.5798 57.0849,38.2186 57.2717,38.7325 57.4835,38.3395 57.7103,38.8533 58.0638,38.3698 58.2869,39.5485 58.7133,38.8838 58.9777,39.2792 59.0373,40.4552 58.9011,40.3343 58.3821,39.6392 58.3821,39.6392 58.0478,40.2437 58.0478,40.3237 57.5365,40.0149 57.4677,39.7299 57.4673,39.7379 57.4051,39.0894 57.2553,38.5798 57.0849)),((38.5798 57.0849,38.9742 56.8774,37.463 56.5623,38.944 56.0594,38.1884 55.8564,38.3184 55.7179,38.0262 55.6546,36.647 55.9411,38.2186 56.0594,36.8799 56.4895,38.309 56.9928,38.3093 56.9929,38.3092 56.9929,38.5798 57.0849)),((37.9482 55.6376,38.0262 55.6546,38.0373 55.6523,37.9482 55.6376))) +MULTIPOLYGON(((24.3676 61.4597,26.6527 61.1007,26.8725 61.7106,30.5639 61.0582,31.3988 62.0214,36.0131 61.1431,36.892 62.0008,42.6488 60.63,43.5717 61.3756,47.0434 59.8888,49.5922 60.0867,49.1527 58.1706,51.9213 57.9147,50.2514 56.1454,52.6684 55.8259,51.6576 54.2908,52.8881 53.9301,50.6469 53.0147,51.3939 52.4827,48.0541 51.1792,49.2846 50.5413,47.1752 49.1529,43.9232 49.8095,42.5609 48.7778,36.9359 49.6675,35.266 48.7488,32.8051 49.5251,27.224 48.9801,26.1254 50.4014,21.2035 50.2049,20.017 51.5633,17.4682 53.0147,19.4457 54.0851,19.4457 55.8752,19.5775 57.4921,19.5775 58.6768,24.3676 61.4597),(24.4555 59.4226,21.2035 58.4936,21.3353 56.8969,21.599 55.9245,25.2025 55.9983,28.85 57.0645,27.0922 57.8447,28.8061 59.1758,26.2572 59.1758,24.4555 59.4226),(35.1914 56.569,36.7073 56.2109,34.7125 56.1099,36.5562 55.6351,35.1357 55.5326,36.4353 55.344,34.8334 55.0161,35.6192 54.4928,34.8939 54.1225,35.3775 53.0461,37.0823 52.9707,34.9584 51.4813,36.5404 50.4014,39.6605 50.2892,39.7924 52.1334,41.7699 50.6807,44.4945 51.9712,47.307 52.5094,44.0551 53.5402,46.6039 53.6966,47.6146 55.404,45.3734 55.404,42.8246 56.5836,40.4556 56.1121,40.4249 56.1941,39.8204 56.0762,39.7902 56.412,40.4854 56.4956,40.3342 56.9598,39.7902 56.9928,39.7388 57.3963,40.066 57.4672,40.3342 57.4672,40.3255 57.5233,42.6928 58.0313,40.891 59.2658,39.3164 59.0294,38.5208 59.1189,38.9056 58.9675,38.1151 58.848,37.3118 59.0257,37.2322 59.0224,37.1117 59.6676,35.1342 59.8447,31.9701 58.9726,32.2439 58.4956,32.2341 58.4927,32.1737 58.0317,31.5087 57.4997,31.7505 56.8608,31.5087 55.941,32.2039 55.5155,31.8412 54.9988,31.6171 54.6946,29.5971 55.5036,29.1576 55.7517,22.5658 55.1285,22.5658 53.5402,22.0385 51.4813,26.2572 51.4265,30.1244 50.5413,32.1898 51.1792,30.1244 53.173,32.5252 53.173,33.1711 52.8275,34.773 52.9187,34.773 53.173,35.0902 53.173,34.773 53.3207,34.773 53.7846,34.108 54.1756,35.0752 54.598,34.2592 54.9641,35.0148 55.3612,33.5035 55.3784,32.7479 55.9071,35.6797 55.6862,32.8386 56.3116,34.5916 56.2948,35.7099 56.3116,33.836 56.6952,33.7508 56.7194,35.1914 56.569)),((35.1914 56.569,34.2289 56.7947,36.9793 57.075,35.7704 57.2553,37.0096 57.4997,35.7401 57.7908,37.1607 58.0477,36.0122 58.2868,37.1909 58.6818,34.6521 58.9166,37.2322 59.0224,37.2875 58.7225,38.1151 58.848,38.5812 58.7445,37.4025 58.3186,38.3394 57.9355,37.4327 57.7102,38.1279 57.5159,37.1607 57.2553,38.3092 56.9928,36.3749 56.6454,36.8783 56.4863,36.6723 56.4138,35.1914 56.569)),((33.106 56.9503,32.2439 58.4956,33.4733 58.8541,34.7427 59.5658,33.836 58.6818,36.3446 58.5401,33.6244 58.2709,36.4353 58.0477,33.2315 57.7747,36.1935 57.4997,33.1711 57.3369,36.0726 57.0914,33.106 56.9503)),((33.106 56.9503,33.1372 56.8932,32.9595 56.9433,33.106 56.9503)),((33.7508 56.7194,33.2006 56.7767,33.1372 56.8932,33.7508 56.7194)),((37.0823 52.9707,37.2279 53.0709,37.4327 52.9551,37.0823 52.9707)),((34.773 53.173,32.5252 53.173,32.083 53.4079,32.4759 53.8382,31.4181 54.4226,31.6171 54.6946,33.1127 54.0851,34.773 53.3207,34.773 53.173)),((40.4556 56.1121,40.576 55.7883,39.76 55.7543,39.8204 55.2752,40.3947 55.2407,40.3947 54.8772,39.5484 54.8772,39.5484 54.563,40.3947 54.4402,40.6063 54.0339,39.9715 53.9806,40.2436 53.5877,39.5484 53.5877,40.0018 53.3539,38.3394 53.2816,39.5786 52.6995,37.8558 52.9187,37.4544 53.2264,37.9906 53.5924,37.0521 54.7595,38.3092 55.1545,37.7954 55.3955,38.4906 55.5326,38.3463 55.6874,40.4556 56.1121)),((37.2279 53.0709,35.9215 53.8025,36.9189 53.856,36.0122 54.7553,36.8796 54.9705,37.0521 54.7595,36.7073 54.6505,37.3118 53.9272,36.9793 53.5877,37.4544 53.2264,37.2279 53.0709)),((38.3463 55.6874,36.9706 55.4079,36.8282 55.447,38.0372 55.6522,36.6469 55.941,38.2185 56.0593,36.8783 56.4863,38.5847 57.0811,38.9741 56.8773,37.4629 56.5622,38.9439 56.0593,38.1883 55.8563,38.3463 55.6874)),((36.8796 54.9705,36.5844 55.329,36.9706 55.4079,37.7652 55.189,36.8796 54.9705)),((39.3164 59.0294,40.4551 58.901,40.3342 58.382,39.6391 58.382,39.6391 58.0477,40.2436 58.0477,40.3255 57.5233,40.066 57.4672,39.7298 57.4672,39.7388 57.3963,39.0893 57.2552,38.5847 57.0811,38.2185 57.2716,38.7324 57.4834,38.3394 57.7102,38.8532 58.0637,38.3697 58.2868,39.5484 58.7132,38.9056 58.9675,39.3164 59.0294))) +-------- Polygon with Polygon with Holes +MULTIPOLYGON(((24.3677 61.4598,26.6528 61.1008,26.8726 61.7107,30.564 61.0583,31.3989 62.0215,36.0132 61.1432,36.8921 62.0009,42.6489 60.6301,43.5718 61.3757,47.0435 59.8889,49.5923 60.0868,49.1528 58.1707,51.9214 57.9148,50.2515 56.1455,52.6685 55.826,51.6577 54.2909,52.8882 53.9302,50.647 53.0148,51.394 52.4828,48.0542 51.1793,49.2847 50.5414,47.1753 49.153,43.9233 49.8096,42.561 48.7779,36.936 49.6676,35.2661 48.7489,32.8052 49.5252,27.2241 48.9802,26.1255 50.4015,21.2036 50.205,20.0171 51.5634,17.4683 53.0148,19.4458 54.0852,19.4458 55.8753,19.5776 57.4922,19.5776 58.6769,24.3677 61.4598),(24.4556 59.4227,21.2036 58.4937,21.3354 56.897,21.5991 55.9246,25.2026 55.9984,28.8501 57.0646,27.0923 57.8448,28.8062 59.1759,26.2573 59.1759,24.4556 59.4227),(35.9475 59.7758,36.2843 59.9616,34.8637 59.9768,34.2247 59.6064,31.9702 58.9727,32.2964 58.4175,30.179 56.9764,30.179 57.9196,33.6548 59.9465,32.3249 59.9465,30.4812 58.8542,30.5719 59.9919,29.4536 59.7796,29.4171 55.606,29.1577 55.7518,22.5659 55.1286,22.5659 53.5403,22.0386 51.4814,26.2573 51.4266,30.1245 50.5414,32.1899 51.1793,31.1968 52.1649,31.2368 52.1652,32.5603 53.1989,33.8733 53.1922,32.5969 52.2208,34.2895 52.2208,37.2766 54.4948,37.7431 53.9104,35.4682 52.2022,35.9681 52.2157,34.9585 51.4814,36.5405 50.4015,39.6606 50.2893,39.7925 52.1335,41.77 50.6808,44.4946 51.9713,47.3071 52.5095,44.0552 53.5403,46.604 53.6967,47.6147 55.4041,45.3735 55.4041,44.4212 55.8594,44.4751 59.81,39.7299 59.9314,37.6322 58.7797,37.2876 58.7226,37.2102 59.1452,38.7325 59.9465,37.2817 59.9768,36.7912 59.6986,35.9475 59.7758)),((32.6512 57.792,32.2964 58.4175,34.2247 59.6064,35.1343 59.8448,35.9475 59.7758,32.6512 57.792)),((32.6512 57.792,32.9378 57.2699,30.2394 55.2753,30.3301 56.1942,32.6512 57.792)),((33.2446 56.7729,33.2007 56.7768,32.9378 57.2699,36.7912 59.6986,37.1118 59.6677,37.2102 59.1452,33.2446 56.7729)),((33.2446 56.7729,34.2635 56.6767,31.5682 54.7333,30.7705 55.0525,33.2446 56.7729)),((34.2635 56.6767,37.6322 58.7797,40.2079 59.1718,35.4536 56.5531,34.2635 56.6767)),((40.2079 59.1718,40.6366 59.3817,40.8804 59.2644,40.2079 59.1718)),((34.3351 53.53,35.0903 53.1731,33.8733 53.1922,34.3351 53.53)),((34.3351 53.53,33.5144 53.9057,38.1759 56.9472,39.0894 57.2553,40.9691 57.677,37.1934 55.4694,36.5845 55.3291,36.7219 55.1665,34.3351 53.53)),((32.6907 54.2663,33.1128 54.0852,33.5144 53.9057,32.5603 53.1989,31.1682 53.1903,32.6907 54.2663)),((32.6907 54.2663,32.2591 54.4483,35.4536 56.5531,36.1815 56.4715,32.6907 54.2663)),((38.1759 56.9472,36.6724 56.4139,36.1815 56.4715,41.168 59.0834,41.5887 58.8012,38.1759 56.9472)),((37.2766 54.4948,36.7219 55.1665,37.1934 55.4694,39.4328 55.9511,37.2766 54.4948)),((40.9691 57.677,42.2498 58.3455,42.6929 58.0314,40.9691 57.677)),((30.7705 55.0525,30.2092 54.6331,30.2394 53.6774,31.5682 54.7333,32.2591 54.4483,30.5408 53.1811,30.1245 53.1731,30.3098 53.0028,29.3931 52.2763,29.4171 55.606,29.5972 55.5037,30.7705 55.0525)),((30.5408 53.1811,31.1682 53.1903,30.5785 52.7531,30.3098 53.0028,30.5408 53.1811)),((30.5785 52.7531,31.1968 52.1649,29.7861 52.1466,30.5785 52.7531)),((35.9681 52.2157,37.9907 53.5925,37.7431 53.9104,41.4519 56.3413,42.8247 56.5837,44.4212 55.8594,44.4146 55.3097,40.0925 52.1652,38.3395 52.1652,43.0243 55.3269,43.0243 56.2614,37.1608 52.2393,35.9681 52.2157)),((39.4328 55.9511,43.0243 58.0797,43.0243 57.2554,41.4519 56.3413,39.4328 55.9511)),((41.168 59.0834,40.9299 59.2404,41.2108 59.1035,41.168 59.0834)),((41.5887 58.8012,41.6944 58.8542,42.5105 58.477,42.2498 58.3455,41.5887 58.8012)),((40.9299 59.2404,40.8804 59.2644,40.8911 59.2659,40.9299 59.2404))) +MULTIPOLYGON(((24.3676 61.4597,26.6527 61.1007,26.8725 61.7106,30.5639 61.0582,31.3988 62.0214,36.0131 61.1431,36.892 62.0008,42.6488 60.63,43.5717 61.3756,47.0434 59.8888,49.5922 60.0867,49.1527 58.1706,51.9213 57.9147,50.2514 56.1454,52.6684 55.8259,51.6576 54.2908,52.8881 53.9301,50.6469 53.0147,51.3939 52.4827,48.0541 51.1792,49.2846 50.5413,47.1752 49.1529,43.9232 49.8095,42.5609 48.7778,36.9359 49.6675,35.266 48.7488,32.8051 49.5251,27.224 48.9801,26.1254 50.4014,21.2035 50.2049,20.017 51.5633,17.4682 53.0147,19.4457 54.0851,19.4457 55.8752,19.5775 57.4921,19.5775 58.6768,24.3676 61.4597),(24.4555 59.4226,21.2035 58.4936,21.3353 56.8969,21.599 55.9245,25.2025 55.9983,28.85 57.0645,27.0922 57.8447,28.8061 59.1758,26.2572 59.1758,24.4555 59.4226),(35.9663 59.7703,36.2842 59.9615,34.8636 59.9767,34.2648 59.6073,31.9701 58.9726,32.3127 58.3747,30.1789 56.9763,30.1789 57.9195,33.6547 59.9464,32.3248 59.9464,30.4811 58.8541,30.5718 59.9918,29.4535 59.7795,29.4184 55.6047,29.1576 55.7517,22.5658 55.1285,22.5658 53.5402,22.0385 51.4813,26.2572 51.4265,30.1244 50.5413,32.1898 51.1792,31.1809 52.1644,31.2367 52.1651,32.7039 53.173,34.0091 53.173,32.5968 52.2207,34.2894 52.2207,37.3607 54.3795,37.8015 53.8302,35.4681 52.2021,35.9939 52.2136,34.9584 51.4813,36.5404 50.4014,39.6605 50.2892,39.7924 52.1334,41.7699 50.6807,44.4945 51.9712,47.307 52.5094,44.0551 53.5402,46.6039 53.6966,47.6146 55.404,45.3734 55.404,44.4214 55.8488,44.475 59.8099,39.7298 59.9313,37.9197 58.8184,37.2875 58.7225,37.2249 59.0621,38.7324 59.9464,37.2816 59.9767,36.8295 59.693,35.9663 59.7703)),((32.6773 57.7269,32.3127 58.3747,34.2648 59.6073,35.1342 59.8447,35.9663 59.7703,32.6773 57.7269)),((32.6773 57.7269,32.9811 57.1783,30.2393 55.2752,30.33 56.1941,32.6773 57.7269)),((33.4587 56.7498,33.2006 56.7767,32.9811 57.1783,36.8295 59.693,37.1117 59.6676,37.2249 59.0621,33.4587 56.7498)),((33.4587 56.7498,34.5347 56.6377,31.6568 54.6786,30.7975 55.0249,33.4587 56.7498)),((34.5347 56.6377,37.9197 58.8184,40.276 59.1737,35.8287 56.5023,34.5347 56.6377)),((40.276 59.1737,40.6365 59.3816,40.8796 59.2641,40.276 59.1737)),((34.4539 53.4687,35.0902 53.173,34.0091 53.173,34.4539 53.4687)),((34.4539 53.4687,33.6753 53.8276,38.9459 57.2058,39.0893 57.2552,41.2147 57.7149,37.6798 55.5522,36.5844 55.329,36.8394 55.0196,34.4539 53.4687)),((32.9056 54.17,33.1127 54.0851,33.6753 53.8276,32.7039 53.173,31.3501 53.173,32.9056 54.17)),((32.9056 54.17,32.4672 54.3493,35.8287 56.5023,36.5672 56.4248,32.9056 54.17)),((38.9459 57.2058,36.6723 56.4138,36.5672 56.4248,41.1674 59.0793,41.5893 58.7925,38.9459 57.2058)),((37.3607 54.3795,36.8394 55.0196,37.6798 55.5522,39.7202 55.9646,37.3607 54.3795)),((41.2147 57.7149,42.2607 58.3313,42.6928 58.0313,41.2147 57.7149)),((31.6568 54.6786,32.4672 54.3493,30.7047 53.173,30.1244 53.173,30.3667 52.9437,29.393 52.2762,29.4184 55.6047,29.5971 55.5036,30.7975 55.0249,30.2091 54.633,30.2393 53.6773,31.6568 54.6786)),((30.7047 53.173,31.3501 53.173,30.6237 52.6993,30.3667 52.9437,30.7047 53.173)),((30.6237 52.6993,31.1809 52.1644,29.786 52.1465,30.6237 52.6993)),((44.4214 55.8488,44.4145 55.3096,40.0924 52.1651,38.3394 52.1651,43.0242 55.3268,43.0242 56.2613,37.1607 52.2392,35.9939 52.2136,37.9906 53.5924,37.8015 53.8302,41.577 56.336,42.8246 56.5836,44.4214 55.8488)),((39.7202 55.9646,43.0242 58.0796,43.0242 57.2553,41.577 56.336,39.7202 55.9646)),((41.1674 59.0793,40.9293 59.24,41.2107 59.1034,41.1674 59.0793)),((41.5893 58.7925,41.6943 58.8541,42.5104 58.4769,42.2607 58.3313,41.5893 58.7925)),((40.9293 59.24,40.8796 59.2641,40.891 59.2658,40.9293 59.24))) diff --git a/tests/queries/0_stateless/01304_polygons_sym_difference.sql b/tests/queries/0_stateless/01304_polygons_sym_difference.sql index f81300172d3..b9f2ef126e6 100644 --- a/tests/queries/0_stateless/01304_polygons_sym_difference.sql +++ b/tests/queries/0_stateless/01304_polygons_sym_difference.sql @@ -1,2 +1,13 @@ -select polygonsSymDifferenceCartesian([[[(0, 0),(0, 3),(1, 2.9),(2, 2.6),(2.6, 2),(2.9, 1),(3, 0),(0, 0)]]], [[[(1., 1.),(1., 4.),(4., 4.),(4., 1.),(1., 1.)]]]) +select polygonsSymDifferenceCartesian([[[(0, 0),(0, 3),(1, 2.9),(2, 2.6),(2.6, 2),(2.9, 1),(3, 0),(0, 0)]]], [[[(1., 1.),(1., 4.),(4., 4.),(4., 1.),(1., 1.)]]]); +select '-------- MultiPolygon with Polygon'; +select wkt(polygonsSymDifferenceSpherical([[(29.453587685533865,59.779570356240356),(29.393139070478895,52.276266797422124),(40.636581470703206,59.38168915000267),(41.21084331372543,59.103467777099866),(29.786055068336193,52.146627480315004),(31.23682182965546,52.16517054781818),(41.69443223416517,58.85424941916091),(42.51048853740727,58.47703162291134),(32.59691566839227,52.22075341251539),(34.289476889931414,52.22075341251539),(43.02430176537451,58.07974369546071),(43.02430176537451,57.25537683364851),(35.468224883503325,52.2022335126388),(37.16078610504247,52.23926559241349),(43.02430176537451,56.26136189644947),(43.02430176537451,55.326904361850836),(38.33953409861437,52.16517054781818),(40.09254393520848,52.16517054781818),(44.4146199116388,55.3097062225408),(44.47506852669377,59.80998197603594),(39.72985224487867,59.931351417569715),(30.23941968124846,53.67744677450975),(30.20919537372098,54.63314259659509),(38.73245009647167,59.94649146557819),(37.2816833351524,59.97675082987618),(30.23941968124846,55.2752875586599),(30.33009260383092,56.19415599955667),(36.28428118674541,59.96162460231375),(34.863738732953635,59.97675082987618),(30.178971066193498,56.97640788219866),(30.178971066193498,57.91957806959033),(33.65476643185424,59.94649146557819),(32.32489690064491,59.94649146557819),(30.481214141468342,58.85424941916091),(30.571887064050795,59.99187015036608),(29.453587685533865,59.779570356240356)]], [[[(33.473420586689336,58.85424941916091),(32.23422397806246,58.492830557036),(32.173775363007486,58.03176922751564),(31.508840597402823,57.499784781503735),(31.750635057622702,56.86092686957355),(31.508840597402823,55.941082594334574),(32.20399967053497,55.515591939372456),(31.84130798020516,54.998862226280465),(31.418167674820367,54.422670886434275),(32.47601843828233,53.83826377018255),(32.08310244042503,53.408048308050866),(33.171177511414484,52.82758702113742),(34.77306581037117,52.91880107773494),(34.77306581037117,53.784726518357985),(34.108131044766516,54.17574726780569),(35.07530888564602,54.59813930694554),(34.25925258240394,54.96417435716029),(35.01486027059106,55.361278263643584),(33.50364489421682,55.37845402950552),(32.7480372060297,55.90721384574556),(35.67979503619571,55.68634475630185),(32.83871012861215,56.311688992608396),(34.591719965206266,56.29492065473883),(35.7100193437232,56.311688992608396),(33.83611227701915,56.695333481003644),(32.95960735872209,56.9434497616887),(36.072711034053015,57.091531913901434),(33.171177511414484,57.33702717078384),(36.193608264162954,57.499784781503735),(33.23162612646945,57.77481561306047),(36.43540272438284,58.04776787540811),(33.62454212432676,58.27099811968307),(36.344729801800376,58.54018474404165),(33.83611227701915,58.68186423448108),(34.74284150284369,59.565911441555244),(33.473420586689336,58.85424941916091)]], [[(34.65216858026123,58.91672306881671),(37.19101041256995,58.68186423448108),(36.01226241899805,58.28688958537609),(37.16078610504247,58.04776787540811),(35.74024365125068,57.79092907387934),(37.009664567405046,57.499784781503735),(35.77046795877817,57.25537683364851),(36.979440259877556,57.07510745541089),(34.22902827487645,56.794777197297435),(36.7074214921302,56.210968525786996),(34.712617195316206,56.10998276812964),(36.55629995449277,55.63519693782703),(35.13575750070099,55.53270067649592),(36.43540272438284,55.34409504165558),(34.83351442542614,55.01619492319591),(35.61934642114075,54.49294870011772),(34.89396304048112,54.12264226523038),(35.37755196092087,53.046178687628185),(37.43280487278982,52.95523300597458),(35.92158949641559,53.80257986695776),(36.91899164482259,53.856094327816805),(36.01226241899805,54.75541714463799),(37.765272255592166,55.189110239786885),(36.828318722240134,55.44708256557195),(38.03729102333953,55.652253637168315),(36.64697287707522,55.941082594334574),(38.21863686850443,56.05939028508024),(36.37495410932787,56.64551287174558),(38.30930979108689,56.992876013526654),(37.16078610504247,57.25537683364851),(38.127963945921984,57.516020773674256),(37.43280487278982,57.710289827306724),(38.33953409861437,57.935626886818994),(37.40258056526235,58.31865112960426),(38.58132855883426,58.744648733419496),(37.31190764267989,59.02578062465136),(34.65216858026123,58.91672306881671)]], [[(38.52087994377928,59.11898412389468),(39.54850639971376,58.713270635642914),(38.369758406141855,58.28688958537609),(38.85334732658162,58.06375936407028),(38.33953409861437,57.710289827306724),(38.73245009647167,57.48354156434209),(38.21863686850443,57.271721400459285),(38.97424455669155,56.87744603722649),(37.463029180317314,56.5623320541159),(38.94402024916407,56.05939028508024),(38.18841256097694,55.856355210835915),(38.490655636251795,55.53270067649592),(37.795496563119656,55.39562234093384),(38.30930979108689,55.154587013355666),(36.7074214921302,54.65063295250911),(37.31190764267989,53.92734063371401),(36.979440259877556,53.58783775557231),(37.855945178174615,52.91880107773497),(39.57873070724124,52.69956490610895),(38.33953409861437,53.281741738901104),(40.00187101262603,53.35396273604752),(39.54850639971376,53.58783775557231),(40.24366547284591,53.58783775557231),(39.97164670509855,53.98069568468355),(40.60635716317572,54.03398248547225),(40.39478701048334,54.44025165268903),(39.54850639971376,54.56310590284329),(39.54850639971376,54.87732350170489),(40.39478701048334,54.87732350170489),(40.39478701048334,55.24083903654295),(39.82052516746112,55.2752875586599),(39.760076552406154,55.75443792473942),(40.57613285564824,55.78844000174894),(40.425011318010824,56.19415599955667),(39.82052516746112,56.07626182891758),(39.79030085993364,56.41214455508424),(40.48545993306579,56.495655446714636),(40.33433839542836,56.95993246553937),(39.79030085993364,56.992876013526654),(39.72985224487867,57.46729112028032),(40.33433839542836,57.46729112028032),(40.24366547284591,58.04776787540811),(39.63917932229622,58.04776787540811),(39.63917932229622,58.382088724871295),(40.33433839542836,58.382088724871295),(40.45523562553831,58.9011152358548),(38.52087994377928,59.11898412389468)]]])) format TSV; +select wkt(polygonsSymDifferenceMercator([[(29.453587685533865,59.779570356240356),(29.393139070478895,52.276266797422124),(40.636581470703206,59.38168915000267),(41.21084331372543,59.103467777099866),(29.786055068336193,52.146627480315004),(31.23682182965546,52.16517054781818),(41.69443223416517,58.85424941916091),(42.51048853740727,58.47703162291134),(32.59691566839227,52.22075341251539),(34.289476889931414,52.22075341251539),(43.02430176537451,58.07974369546071),(43.02430176537451,57.25537683364851),(35.468224883503325,52.2022335126388),(37.16078610504247,52.23926559241349),(43.02430176537451,56.26136189644947),(43.02430176537451,55.326904361850836),(38.33953409861437,52.16517054781818),(40.09254393520848,52.16517054781818),(44.4146199116388,55.3097062225408),(44.47506852669377,59.80998197603594),(39.72985224487867,59.931351417569715),(30.23941968124846,53.67744677450975),(30.20919537372098,54.63314259659509),(38.73245009647167,59.94649146557819),(37.2816833351524,59.97675082987618),(30.23941968124846,55.2752875586599),(30.33009260383092,56.19415599955667),(36.28428118674541,59.96162460231375),(34.863738732953635,59.97675082987618),(30.178971066193498,56.97640788219866),(30.178971066193498,57.91957806959033),(33.65476643185424,59.94649146557819),(32.32489690064491,59.94649146557819),(30.481214141468342,58.85424941916091),(30.571887064050795,59.99187015036608),(29.453587685533865,59.779570356240356)]], [[[(33.473420586689336,58.85424941916091),(32.23422397806246,58.492830557036),(32.173775363007486,58.03176922751564),(31.508840597402823,57.499784781503735),(31.750635057622702,56.86092686957355),(31.508840597402823,55.941082594334574),(32.20399967053497,55.515591939372456),(31.84130798020516,54.998862226280465),(31.418167674820367,54.422670886434275),(32.47601843828233,53.83826377018255),(32.08310244042503,53.408048308050866),(33.171177511414484,52.82758702113742),(34.77306581037117,52.91880107773494),(34.77306581037117,53.784726518357985),(34.108131044766516,54.17574726780569),(35.07530888564602,54.59813930694554),(34.25925258240394,54.96417435716029),(35.01486027059106,55.361278263643584),(33.50364489421682,55.37845402950552),(32.7480372060297,55.90721384574556),(35.67979503619571,55.68634475630185),(32.83871012861215,56.311688992608396),(34.591719965206266,56.29492065473883),(35.7100193437232,56.311688992608396),(33.83611227701915,56.695333481003644),(32.95960735872209,56.9434497616887),(36.072711034053015,57.091531913901434),(33.171177511414484,57.33702717078384),(36.193608264162954,57.499784781503735),(33.23162612646945,57.77481561306047),(36.43540272438284,58.04776787540811),(33.62454212432676,58.27099811968307),(36.344729801800376,58.54018474404165),(33.83611227701915,58.68186423448108),(34.74284150284369,59.565911441555244),(33.473420586689336,58.85424941916091)]], [[(34.65216858026123,58.91672306881671),(37.19101041256995,58.68186423448108),(36.01226241899805,58.28688958537609),(37.16078610504247,58.04776787540811),(35.74024365125068,57.79092907387934),(37.009664567405046,57.499784781503735),(35.77046795877817,57.25537683364851),(36.979440259877556,57.07510745541089),(34.22902827487645,56.794777197297435),(36.7074214921302,56.210968525786996),(34.712617195316206,56.10998276812964),(36.55629995449277,55.63519693782703),(35.13575750070099,55.53270067649592),(36.43540272438284,55.34409504165558),(34.83351442542614,55.01619492319591),(35.61934642114075,54.49294870011772),(34.89396304048112,54.12264226523038),(35.37755196092087,53.046178687628185),(37.43280487278982,52.95523300597458),(35.92158949641559,53.80257986695776),(36.91899164482259,53.856094327816805),(36.01226241899805,54.75541714463799),(37.765272255592166,55.189110239786885),(36.828318722240134,55.44708256557195),(38.03729102333953,55.652253637168315),(36.64697287707522,55.941082594334574),(38.21863686850443,56.05939028508024),(36.37495410932787,56.64551287174558),(38.30930979108689,56.992876013526654),(37.16078610504247,57.25537683364851),(38.127963945921984,57.516020773674256),(37.43280487278982,57.710289827306724),(38.33953409861437,57.935626886818994),(37.40258056526235,58.31865112960426),(38.58132855883426,58.744648733419496),(37.31190764267989,59.02578062465136),(34.65216858026123,58.91672306881671)]], [[(38.52087994377928,59.11898412389468),(39.54850639971376,58.713270635642914),(38.369758406141855,58.28688958537609),(38.85334732658162,58.06375936407028),(38.33953409861437,57.710289827306724),(38.73245009647167,57.48354156434209),(38.21863686850443,57.271721400459285),(38.97424455669155,56.87744603722649),(37.463029180317314,56.5623320541159),(38.94402024916407,56.05939028508024),(38.18841256097694,55.856355210835915),(38.490655636251795,55.53270067649592),(37.795496563119656,55.39562234093384),(38.30930979108689,55.154587013355666),(36.7074214921302,54.65063295250911),(37.31190764267989,53.92734063371401),(36.979440259877556,53.58783775557231),(37.855945178174615,52.91880107773497),(39.57873070724124,52.69956490610895),(38.33953409861437,53.281741738901104),(40.00187101262603,53.35396273604752),(39.54850639971376,53.58783775557231),(40.24366547284591,53.58783775557231),(39.97164670509855,53.98069568468355),(40.60635716317572,54.03398248547225),(40.39478701048334,54.44025165268903),(39.54850639971376,54.56310590284329),(39.54850639971376,54.87732350170489),(40.39478701048334,54.87732350170489),(40.39478701048334,55.24083903654295),(39.82052516746112,55.2752875586599),(39.760076552406154,55.75443792473942),(40.57613285564824,55.78844000174894),(40.425011318010824,56.19415599955667),(39.82052516746112,56.07626182891758),(39.79030085993364,56.41214455508424),(40.48545993306579,56.495655446714636),(40.33433839542836,56.95993246553937),(39.79030085993364,56.992876013526654),(39.72985224487867,57.46729112028032),(40.33433839542836,57.46729112028032),(40.24366547284591,58.04776787540811),(39.63917932229622,58.04776787540811),(39.63917932229622,58.382088724871295),(40.33433839542836,58.382088724871295),(40.45523562553831,58.9011152358548),(38.52087994377928,59.11898412389468)]]])) format TSV; + +select '-------- MultiPolygon with Polygon with Holes'; +select wkt(polygonsSymDifferenceSpherical([[[(33.473420586689336,58.85424941916091),(32.23422397806246,58.492830557036),(32.173775363007486,58.03176922751564),(31.508840597402823,57.499784781503735),(31.750635057622702,56.86092686957355),(31.508840597402823,55.941082594334574),(32.20399967053497,55.515591939372456),(31.84130798020516,54.998862226280465),(31.418167674820367,54.422670886434275),(32.47601843828233,53.83826377018255),(32.08310244042503,53.408048308050866),(33.171177511414484,52.82758702113742),(34.77306581037117,52.91880107773494),(34.77306581037117,53.784726518357985),(34.108131044766516,54.17574726780569),(35.07530888564602,54.59813930694554),(34.25925258240394,54.96417435716029),(35.01486027059106,55.361278263643584),(33.50364489421682,55.37845402950552),(32.7480372060297,55.90721384574556),(35.67979503619571,55.68634475630185),(32.83871012861215,56.311688992608396),(34.591719965206266,56.29492065473883),(35.7100193437232,56.311688992608396),(33.83611227701915,56.695333481003644),(32.95960735872209,56.9434497616887),(36.072711034053015,57.091531913901434),(33.171177511414484,57.33702717078384),(36.193608264162954,57.499784781503735),(33.23162612646945,57.77481561306047),(36.43540272438284,58.04776787540811),(33.62454212432676,58.27099811968307),(36.344729801800376,58.54018474404165),(33.83611227701915,58.68186423448108),(34.74284150284369,59.565911441555244),(33.473420586689336,58.85424941916091)]], [[(34.65216858026123,58.91672306881671),(37.19101041256995,58.68186423448108),(36.01226241899805,58.28688958537609),(37.16078610504247,58.04776787540811),(35.74024365125068,57.79092907387934),(37.009664567405046,57.499784781503735),(35.77046795877817,57.25537683364851),(36.979440259877556,57.07510745541089),(34.22902827487645,56.794777197297435),(36.7074214921302,56.210968525786996),(34.712617195316206,56.10998276812964),(36.55629995449277,55.63519693782703),(35.13575750070099,55.53270067649592),(36.43540272438284,55.34409504165558),(34.83351442542614,55.01619492319591),(35.61934642114075,54.49294870011772),(34.89396304048112,54.12264226523038),(35.37755196092087,53.046178687628185),(37.43280487278982,52.95523300597458),(35.92158949641559,53.80257986695776),(36.91899164482259,53.856094327816805),(36.01226241899805,54.75541714463799),(37.765272255592166,55.189110239786885),(36.828318722240134,55.44708256557195),(38.03729102333953,55.652253637168315),(36.64697287707522,55.941082594334574),(38.21863686850443,56.05939028508024),(36.37495410932787,56.64551287174558),(38.30930979108689,56.992876013526654),(37.16078610504247,57.25537683364851),(38.127963945921984,57.516020773674256),(37.43280487278982,57.710289827306724),(38.33953409861437,57.935626886818994),(37.40258056526235,58.31865112960426),(38.58132855883426,58.744648733419496),(37.31190764267989,59.02578062465136),(34.65216858026123,58.91672306881671)]], [[(38.52087994377928,59.11898412389468),(39.54850639971376,58.713270635642914),(38.369758406141855,58.28688958537609),(38.85334732658162,58.06375936407028),(38.33953409861437,57.710289827306724),(38.73245009647167,57.48354156434209),(38.21863686850443,57.271721400459285),(38.97424455669155,56.87744603722649),(37.463029180317314,56.5623320541159),(38.94402024916407,56.05939028508024),(38.18841256097694,55.856355210835915),(38.490655636251795,55.53270067649592),(37.795496563119656,55.39562234093384),(38.30930979108689,55.154587013355666),(36.7074214921302,54.65063295250911),(37.31190764267989,53.92734063371401),(36.979440259877556,53.58783775557231),(37.855945178174615,52.91880107773497),(39.57873070724124,52.69956490610895),(38.33953409861437,53.281741738901104),(40.00187101262603,53.35396273604752),(39.54850639971376,53.58783775557231),(40.24366547284591,53.58783775557231),(39.97164670509855,53.98069568468355),(40.60635716317572,54.03398248547225),(40.39478701048334,54.44025165268903),(39.54850639971376,54.56310590284329),(39.54850639971376,54.87732350170489),(40.39478701048334,54.87732350170489),(40.39478701048334,55.24083903654295),(39.82052516746112,55.2752875586599),(39.760076552406154,55.75443792473942),(40.57613285564824,55.78844000174894),(40.425011318010824,56.19415599955667),(39.82052516746112,56.07626182891758),(39.79030085993364,56.41214455508424),(40.48545993306579,56.495655446714636),(40.33433839542836,56.95993246553937),(39.79030085993364,56.992876013526654),(39.72985224487867,57.46729112028032),(40.33433839542836,57.46729112028032),(40.24366547284591,58.04776787540811),(39.63917932229622,58.04776787540811),(39.63917932229622,58.382088724871295),(40.33433839542836,58.382088724871295),(40.45523562553831,58.9011152358548),(38.52087994377928,59.11898412389468)]]], [[(24.367675781249993,61.45977057029751),(19.577636718749993,58.67693767258692),(19.577636718749993,57.492213666700735),(19.445800781249996,55.87531083569678),(19.445800781249996,54.085173420886775),(17.468261718749996,53.014783245859235),(20.017089843749993,51.563412328675895),(21.203613281249993,50.205033264943324),(26.125488281249993,50.40151532278236),(27.22412109374999,48.980216985374994),(32.80517578124999,49.525208341974405),(35.26611328124999,48.74894534343292),(36.93603515624999,49.66762782262194),(42.56103515625,48.77791275550183),(43.92333984374999,49.8096315635631),(47.17529296875,49.152969656170455),(49.28466796875,50.54136296522162),(48.05419921875,51.17934297928929),(51.39404296875,52.48278022207825),(50.64697265625,53.014783245859235),(52.88818359375,53.93021986394004),(51.65771484374999,54.29088164657006),(52.66845703125,55.825973254619015),(50.25146484375,56.145549500679095),(51.92138671875,57.914847767009206),(49.15283203125,58.17070248348605),(49.59228515625,60.086762746260064),(47.043457031249986,59.88893689676584),(43.57177734375,61.37567331572748),(42.64892578125,60.630101766266705),(36.89208984374999,62.000904713685856),(36.01318359374999,61.143235250840576),(31.398925781249993,62.02152819100766),(30.563964843749996,61.05828537037917),(26.872558593749993,61.71070595883174),(26.652832031249993,61.10078883158897),(24.367675781249993,61.45977057029751)], [(24.455566406249993,59.42272750081452),(21.203613281249993,58.49369382056807),(21.335449218749993,56.89700392127261),(21.599121093749993,55.92458580482949),(25.202636718749993,55.998380955359636),(28.850097656249993,57.06463027327854),(27.09228515625,57.844750992890994),(28.806152343749996,59.17592824927138),(26.257324218749993,59.17592824927138),(24.455566406249993,59.42272750081452)], [(35.13427734375,59.84481485969107),(31.970214843749993,58.97266715450152),(33.20068359374999,56.776808316568406),(36.67236328125,56.41390137600675),(39.08935546874999,57.25528054528888),(42.69287109374999,58.03137242177638),(40.89111328124999,59.26588062825809),(37.28759765625,58.722598828043374),(37.11181640624999,59.66774058164964),(35.13427734375,59.84481485969107)], [(29.157714843749993,55.75184939173528),(22.565917968749993,55.128649068488784),(22.565917968749993,53.54030739150019),(22.038574218749996,51.48138289610097),(26.257324218749993,51.42661449707484),(30.124511718749993,50.54136296522162),(32.18994140624999,51.17934297928929),(30.124511718749993,53.173119202640635),(35.09033203124999,53.173119202640635),(33.11279296875,54.085173420886775),(29.597167968749993,55.50374985927513),(29.157714843749993,55.75184939173528)], [(42.82470703125,56.58369172128337),(36.584472656249986,55.329144408405085),(37.99072265625,53.592504809039355),(34.95849609374999,51.48138289610097),(36.54052734374999,50.40151532278236),(39.66064453124999,50.289339253291786),(39.79248046875,52.13348804077148),(41.77001953125,50.68079714532166),(44.49462890624999,51.97134580885171),(47.30712890624999,52.509534770327264),(44.05517578125,53.54030739150019),(46.60400390625,53.696706475303245),(47.61474609375,55.40406982700608),(45.37353515625,55.40406982700608),(42.82470703125,56.58369172128337)]])) format TSV; +select wkt(polygonsSymDifferenceMercator([[[(33.473420586689336,58.85424941916091),(32.23422397806246,58.492830557036),(32.173775363007486,58.03176922751564),(31.508840597402823,57.499784781503735),(31.750635057622702,56.86092686957355),(31.508840597402823,55.941082594334574),(32.20399967053497,55.515591939372456),(31.84130798020516,54.998862226280465),(31.418167674820367,54.422670886434275),(32.47601843828233,53.83826377018255),(32.08310244042503,53.408048308050866),(33.171177511414484,52.82758702113742),(34.77306581037117,52.91880107773494),(34.77306581037117,53.784726518357985),(34.108131044766516,54.17574726780569),(35.07530888564602,54.59813930694554),(34.25925258240394,54.96417435716029),(35.01486027059106,55.361278263643584),(33.50364489421682,55.37845402950552),(32.7480372060297,55.90721384574556),(35.67979503619571,55.68634475630185),(32.83871012861215,56.311688992608396),(34.591719965206266,56.29492065473883),(35.7100193437232,56.311688992608396),(33.83611227701915,56.695333481003644),(32.95960735872209,56.9434497616887),(36.072711034053015,57.091531913901434),(33.171177511414484,57.33702717078384),(36.193608264162954,57.499784781503735),(33.23162612646945,57.77481561306047),(36.43540272438284,58.04776787540811),(33.62454212432676,58.27099811968307),(36.344729801800376,58.54018474404165),(33.83611227701915,58.68186423448108),(34.74284150284369,59.565911441555244),(33.473420586689336,58.85424941916091)]], [[(34.65216858026123,58.91672306881671),(37.19101041256995,58.68186423448108),(36.01226241899805,58.28688958537609),(37.16078610504247,58.04776787540811),(35.74024365125068,57.79092907387934),(37.009664567405046,57.499784781503735),(35.77046795877817,57.25537683364851),(36.979440259877556,57.07510745541089),(34.22902827487645,56.794777197297435),(36.7074214921302,56.210968525786996),(34.712617195316206,56.10998276812964),(36.55629995449277,55.63519693782703),(35.13575750070099,55.53270067649592),(36.43540272438284,55.34409504165558),(34.83351442542614,55.01619492319591),(35.61934642114075,54.49294870011772),(34.89396304048112,54.12264226523038),(35.37755196092087,53.046178687628185),(37.43280487278982,52.95523300597458),(35.92158949641559,53.80257986695776),(36.91899164482259,53.856094327816805),(36.01226241899805,54.75541714463799),(37.765272255592166,55.189110239786885),(36.828318722240134,55.44708256557195),(38.03729102333953,55.652253637168315),(36.64697287707522,55.941082594334574),(38.21863686850443,56.05939028508024),(36.37495410932787,56.64551287174558),(38.30930979108689,56.992876013526654),(37.16078610504247,57.25537683364851),(38.127963945921984,57.516020773674256),(37.43280487278982,57.710289827306724),(38.33953409861437,57.935626886818994),(37.40258056526235,58.31865112960426),(38.58132855883426,58.744648733419496),(37.31190764267989,59.02578062465136),(34.65216858026123,58.91672306881671)]], [[(38.52087994377928,59.11898412389468),(39.54850639971376,58.713270635642914),(38.369758406141855,58.28688958537609),(38.85334732658162,58.06375936407028),(38.33953409861437,57.710289827306724),(38.73245009647167,57.48354156434209),(38.21863686850443,57.271721400459285),(38.97424455669155,56.87744603722649),(37.463029180317314,56.5623320541159),(38.94402024916407,56.05939028508024),(38.18841256097694,55.856355210835915),(38.490655636251795,55.53270067649592),(37.795496563119656,55.39562234093384),(38.30930979108689,55.154587013355666),(36.7074214921302,54.65063295250911),(37.31190764267989,53.92734063371401),(36.979440259877556,53.58783775557231),(37.855945178174615,52.91880107773497),(39.57873070724124,52.69956490610895),(38.33953409861437,53.281741738901104),(40.00187101262603,53.35396273604752),(39.54850639971376,53.58783775557231),(40.24366547284591,53.58783775557231),(39.97164670509855,53.98069568468355),(40.60635716317572,54.03398248547225),(40.39478701048334,54.44025165268903),(39.54850639971376,54.56310590284329),(39.54850639971376,54.87732350170489),(40.39478701048334,54.87732350170489),(40.39478701048334,55.24083903654295),(39.82052516746112,55.2752875586599),(39.760076552406154,55.75443792473942),(40.57613285564824,55.78844000174894),(40.425011318010824,56.19415599955667),(39.82052516746112,56.07626182891758),(39.79030085993364,56.41214455508424),(40.48545993306579,56.495655446714636),(40.33433839542836,56.95993246553937),(39.79030085993364,56.992876013526654),(39.72985224487867,57.46729112028032),(40.33433839542836,57.46729112028032),(40.24366547284591,58.04776787540811),(39.63917932229622,58.04776787540811),(39.63917932229622,58.382088724871295),(40.33433839542836,58.382088724871295),(40.45523562553831,58.9011152358548),(38.52087994377928,59.11898412389468)]]], [[(24.367675781249993,61.45977057029751),(19.577636718749993,58.67693767258692),(19.577636718749993,57.492213666700735),(19.445800781249996,55.87531083569678),(19.445800781249996,54.085173420886775),(17.468261718749996,53.014783245859235),(20.017089843749993,51.563412328675895),(21.203613281249993,50.205033264943324),(26.125488281249993,50.40151532278236),(27.22412109374999,48.980216985374994),(32.80517578124999,49.525208341974405),(35.26611328124999,48.74894534343292),(36.93603515624999,49.66762782262194),(42.56103515625,48.77791275550183),(43.92333984374999,49.8096315635631),(47.17529296875,49.152969656170455),(49.28466796875,50.54136296522162),(48.05419921875,51.17934297928929),(51.39404296875,52.48278022207825),(50.64697265625,53.014783245859235),(52.88818359375,53.93021986394004),(51.65771484374999,54.29088164657006),(52.66845703125,55.825973254619015),(50.25146484375,56.145549500679095),(51.92138671875,57.914847767009206),(49.15283203125,58.17070248348605),(49.59228515625,60.086762746260064),(47.043457031249986,59.88893689676584),(43.57177734375,61.37567331572748),(42.64892578125,60.630101766266705),(36.89208984374999,62.000904713685856),(36.01318359374999,61.143235250840576),(31.398925781249993,62.02152819100766),(30.563964843749996,61.05828537037917),(26.872558593749993,61.71070595883174),(26.652832031249993,61.10078883158897),(24.367675781249993,61.45977057029751)], [(24.455566406249993,59.42272750081452),(21.203613281249993,58.49369382056807),(21.335449218749993,56.89700392127261),(21.599121093749993,55.92458580482949),(25.202636718749993,55.998380955359636),(28.850097656249993,57.06463027327854),(27.09228515625,57.844750992890994),(28.806152343749996,59.17592824927138),(26.257324218749993,59.17592824927138),(24.455566406249993,59.42272750081452)], [(35.13427734375,59.84481485969107),(31.970214843749993,58.97266715450152),(33.20068359374999,56.776808316568406),(36.67236328125,56.41390137600675),(39.08935546874999,57.25528054528888),(42.69287109374999,58.03137242177638),(40.89111328124999,59.26588062825809),(37.28759765625,58.722598828043374),(37.11181640624999,59.66774058164964),(35.13427734375,59.84481485969107)], [(29.157714843749993,55.75184939173528),(22.565917968749993,55.128649068488784),(22.565917968749993,53.54030739150019),(22.038574218749996,51.48138289610097),(26.257324218749993,51.42661449707484),(30.124511718749993,50.54136296522162),(32.18994140624999,51.17934297928929),(30.124511718749993,53.173119202640635),(35.09033203124999,53.173119202640635),(33.11279296875,54.085173420886775),(29.597167968749993,55.50374985927513),(29.157714843749993,55.75184939173528)], [(42.82470703125,56.58369172128337),(36.584472656249986,55.329144408405085),(37.99072265625,53.592504809039355),(34.95849609374999,51.48138289610097),(36.54052734374999,50.40151532278236),(39.66064453124999,50.289339253291786),(39.79248046875,52.13348804077148),(41.77001953125,50.68079714532166),(44.49462890624999,51.97134580885171),(47.30712890624999,52.509534770327264),(44.05517578125,53.54030739150019),(46.60400390625,53.696706475303245),(47.61474609375,55.40406982700608),(45.37353515625,55.40406982700608),(42.82470703125,56.58369172128337)]])) format TSV; + +select '-------- Polygon with Polygon with Holes'; +select wkt(polygonsSymDifferenceSpherical([[(29.453587685533865,59.779570356240356),(29.393139070478895,52.276266797422124),(40.636581470703206,59.38168915000267),(41.21084331372543,59.103467777099866),(29.786055068336193,52.146627480315004),(31.23682182965546,52.16517054781818),(41.69443223416517,58.85424941916091),(42.51048853740727,58.47703162291134),(32.59691566839227,52.22075341251539),(34.289476889931414,52.22075341251539),(43.02430176537451,58.07974369546071),(43.02430176537451,57.25537683364851),(35.468224883503325,52.2022335126388),(37.16078610504247,52.23926559241349),(43.02430176537451,56.26136189644947),(43.02430176537451,55.326904361850836),(38.33953409861437,52.16517054781818),(40.09254393520848,52.16517054781818),(44.4146199116388,55.3097062225408),(44.47506852669377,59.80998197603594),(39.72985224487867,59.931351417569715),(30.23941968124846,53.67744677450975),(30.20919537372098,54.63314259659509),(38.73245009647167,59.94649146557819),(37.2816833351524,59.97675082987618),(30.23941968124846,55.2752875586599),(30.33009260383092,56.19415599955667),(36.28428118674541,59.96162460231375),(34.863738732953635,59.97675082987618),(30.178971066193498,56.97640788219866),(30.178971066193498,57.91957806959033),(33.65476643185424,59.94649146557819),(32.32489690064491,59.94649146557819),(30.481214141468342,58.85424941916091),(30.571887064050795,59.99187015036608),(29.453587685533865,59.779570356240356)]], [[(24.367675781249993,61.45977057029751),(19.577636718749993,58.67693767258692),(19.577636718749993,57.492213666700735),(19.445800781249996,55.87531083569678),(19.445800781249996,54.085173420886775),(17.468261718749996,53.014783245859235),(20.017089843749993,51.563412328675895),(21.203613281249993,50.205033264943324),(26.125488281249993,50.40151532278236),(27.22412109374999,48.980216985374994),(32.80517578124999,49.525208341974405),(35.26611328124999,48.74894534343292),(36.93603515624999,49.66762782262194),(42.56103515625,48.77791275550183),(43.92333984374999,49.8096315635631),(47.17529296875,49.152969656170455),(49.28466796875,50.54136296522162),(48.05419921875,51.17934297928929),(51.39404296875,52.48278022207825),(50.64697265625,53.014783245859235),(52.88818359375,53.93021986394004),(51.65771484374999,54.29088164657006),(52.66845703125,55.825973254619015),(50.25146484375,56.145549500679095),(51.92138671875,57.914847767009206),(49.15283203125,58.17070248348605),(49.59228515625,60.086762746260064),(47.043457031249986,59.88893689676584),(43.57177734375,61.37567331572748),(42.64892578125,60.630101766266705),(36.89208984374999,62.000904713685856),(36.01318359374999,61.143235250840576),(31.398925781249993,62.02152819100766),(30.563964843749996,61.05828537037917),(26.872558593749993,61.71070595883174),(26.652832031249993,61.10078883158897),(24.367675781249993,61.45977057029751)], [(24.455566406249993,59.42272750081452),(21.203613281249993,58.49369382056807),(21.335449218749993,56.89700392127261),(21.599121093749993,55.92458580482949),(25.202636718749993,55.998380955359636),(28.850097656249993,57.06463027327854),(27.09228515625,57.844750992890994),(28.806152343749996,59.17592824927138),(26.257324218749993,59.17592824927138),(24.455566406249993,59.42272750081452)], [(35.13427734375,59.84481485969107),(31.970214843749993,58.97266715450152),(33.20068359374999,56.776808316568406),(36.67236328125,56.41390137600675),(39.08935546874999,57.25528054528888),(42.69287109374999,58.03137242177638),(40.89111328124999,59.26588062825809),(37.28759765625,58.722598828043374),(37.11181640624999,59.66774058164964),(35.13427734375,59.84481485969107)], [(29.157714843749993,55.75184939173528),(22.565917968749993,55.128649068488784),(22.565917968749993,53.54030739150019),(22.038574218749996,51.48138289610097),(26.257324218749993,51.42661449707484),(30.124511718749993,50.54136296522162),(32.18994140624999,51.17934297928929),(30.124511718749993,53.173119202640635),(35.09033203124999,53.173119202640635),(33.11279296875,54.085173420886775),(29.597167968749993,55.50374985927513),(29.157714843749993,55.75184939173528)], [(42.82470703125,56.58369172128337),(36.584472656249986,55.329144408405085),(37.99072265625,53.592504809039355),(34.95849609374999,51.48138289610097),(36.54052734374999,50.40151532278236),(39.66064453124999,50.289339253291786),(39.79248046875,52.13348804077148),(41.77001953125,50.68079714532166),(44.49462890624999,51.97134580885171),(47.30712890624999,52.509534770327264),(44.05517578125,53.54030739150019),(46.60400390625,53.696706475303245),(47.61474609375,55.40406982700608),(45.37353515625,55.40406982700608),(42.82470703125,56.58369172128337)]])) format TSV; +select wkt(polygonsSymDifferenceMercator([[(29.453587685533865,59.779570356240356),(29.393139070478895,52.276266797422124),(40.636581470703206,59.38168915000267),(41.21084331372543,59.103467777099866),(29.786055068336193,52.146627480315004),(31.23682182965546,52.16517054781818),(41.69443223416517,58.85424941916091),(42.51048853740727,58.47703162291134),(32.59691566839227,52.22075341251539),(34.289476889931414,52.22075341251539),(43.02430176537451,58.07974369546071),(43.02430176537451,57.25537683364851),(35.468224883503325,52.2022335126388),(37.16078610504247,52.23926559241349),(43.02430176537451,56.26136189644947),(43.02430176537451,55.326904361850836),(38.33953409861437,52.16517054781818),(40.09254393520848,52.16517054781818),(44.4146199116388,55.3097062225408),(44.47506852669377,59.80998197603594),(39.72985224487867,59.931351417569715),(30.23941968124846,53.67744677450975),(30.20919537372098,54.63314259659509),(38.73245009647167,59.94649146557819),(37.2816833351524,59.97675082987618),(30.23941968124846,55.2752875586599),(30.33009260383092,56.19415599955667),(36.28428118674541,59.96162460231375),(34.863738732953635,59.97675082987618),(30.178971066193498,56.97640788219866),(30.178971066193498,57.91957806959033),(33.65476643185424,59.94649146557819),(32.32489690064491,59.94649146557819),(30.481214141468342,58.85424941916091),(30.571887064050795,59.99187015036608),(29.453587685533865,59.779570356240356)]], [[(24.367675781249993,61.45977057029751),(19.577636718749993,58.67693767258692),(19.577636718749993,57.492213666700735),(19.445800781249996,55.87531083569678),(19.445800781249996,54.085173420886775),(17.468261718749996,53.014783245859235),(20.017089843749993,51.563412328675895),(21.203613281249993,50.205033264943324),(26.125488281249993,50.40151532278236),(27.22412109374999,48.980216985374994),(32.80517578124999,49.525208341974405),(35.26611328124999,48.74894534343292),(36.93603515624999,49.66762782262194),(42.56103515625,48.77791275550183),(43.92333984374999,49.8096315635631),(47.17529296875,49.152969656170455),(49.28466796875,50.54136296522162),(48.05419921875,51.17934297928929),(51.39404296875,52.48278022207825),(50.64697265625,53.014783245859235),(52.88818359375,53.93021986394004),(51.65771484374999,54.29088164657006),(52.66845703125,55.825973254619015),(50.25146484375,56.145549500679095),(51.92138671875,57.914847767009206),(49.15283203125,58.17070248348605),(49.59228515625,60.086762746260064),(47.043457031249986,59.88893689676584),(43.57177734375,61.37567331572748),(42.64892578125,60.630101766266705),(36.89208984374999,62.000904713685856),(36.01318359374999,61.143235250840576),(31.398925781249993,62.02152819100766),(30.563964843749996,61.05828537037917),(26.872558593749993,61.71070595883174),(26.652832031249993,61.10078883158897),(24.367675781249993,61.45977057029751)], [(24.455566406249993,59.42272750081452),(21.203613281249993,58.49369382056807),(21.335449218749993,56.89700392127261),(21.599121093749993,55.92458580482949),(25.202636718749993,55.998380955359636),(28.850097656249993,57.06463027327854),(27.09228515625,57.844750992890994),(28.806152343749996,59.17592824927138),(26.257324218749993,59.17592824927138),(24.455566406249993,59.42272750081452)], [(35.13427734375,59.84481485969107),(31.970214843749993,58.97266715450152),(33.20068359374999,56.776808316568406),(36.67236328125,56.41390137600675),(39.08935546874999,57.25528054528888),(42.69287109374999,58.03137242177638),(40.89111328124999,59.26588062825809),(37.28759765625,58.722598828043374),(37.11181640624999,59.66774058164964),(35.13427734375,59.84481485969107)], [(29.157714843749993,55.75184939173528),(22.565917968749993,55.128649068488784),(22.565917968749993,53.54030739150019),(22.038574218749996,51.48138289610097),(26.257324218749993,51.42661449707484),(30.124511718749993,50.54136296522162),(32.18994140624999,51.17934297928929),(30.124511718749993,53.173119202640635),(35.09033203124999,53.173119202640635),(33.11279296875,54.085173420886775),(29.597167968749993,55.50374985927513),(29.157714843749993,55.75184939173528)], [(42.82470703125,56.58369172128337),(36.584472656249986,55.329144408405085),(37.99072265625,53.592504809039355),(34.95849609374999,51.48138289610097),(36.54052734374999,50.40151532278236),(39.66064453124999,50.289339253291786),(39.79248046875,52.13348804077148),(41.77001953125,50.68079714532166),(44.49462890624999,51.97134580885171),(47.30712890624999,52.509534770327264),(44.05517578125,53.54030739150019),(46.60400390625,53.696706475303245),(47.61474609375,55.40406982700608),(45.37353515625,55.40406982700608),(42.82470703125,56.58369172128337)]])) format TSV; diff --git a/tests/queries/0_stateless/01305_polygons_union.reference b/tests/queries/0_stateless/01305_polygons_union.reference index cf9c9fe8d5b..8a5c7d1b521 100644 --- a/tests/queries/0_stateless/01305_polygons_union.reference +++ b/tests/queries/0_stateless/01305_polygons_union.reference @@ -5,3 +5,12 @@ [[[(4.3666052904432435,50.84337386140151),(4.366227,50.840809),(4.344961,50.833264),(4.338074,50.848677),(4.346693,50.858306),(4.3526804582393535,50.856658100365976),(4.3613577,50.8651821),(4.3613148,50.8651279),(4.3904543,50.8564867),(4.3830299,50.8428851),(4.3666052904432435,50.84337386140151)]]] -------- [[[(4.366505261795747,50.843273415405),(4.3661270000000005,50.840709),(4.344861,50.83316399999998),(4.337974,50.84857699999999),(4.346593,50.858205999999996),(4.352580404040105,50.85655765067624),(4.3612577,50.86508209999998),(4.361214800000001,50.8650279),(4.3903543,50.85638669999999),(4.382929900000001,50.842785099999986),(4.366505261795747,50.843273415405)]]] +-------- MultiPolygon with Polygon +MULTIPOLYGON(((35.5408 58.9593,37.2817 59.9768,38.7325 59.9465,36.9725 59.0149,37.3119 59.0258,37.8553 58.9075,39.7299 59.9314,44.4751 59.81,44.4146 55.3097,40.0925 52.1652,38.3395 52.1652,39.1456 52.7573,38.0214 52.8989,37.1608 52.2393,35.4682 52.2022,36.5022 53.0008,35.3776 53.0462,35.3645 53.076,34.2895 52.2208,32.5969 52.2208,33.4048 52.8423,33.1712 52.8276,32.5275 53.1741,31.2368 52.1652,29.7861 52.1466,32.2523 53.964,31.8748 54.1736,29.3931 52.2763,29.4536 59.7796,30.5719 59.9919,30.4812 58.8542,32.3249 59.9465,33.6548 59.9465,30.179 57.9196,30.179 56.9764,32.2175 58.3664,32.2342 58.4928,32.5691 58.5924,34.8637 59.9768,36.2843 59.9616,34.0496 58.6717,34.9952 58.6226,35.3712 58.8556,34.6522 58.9167,35.5408 58.9593),(36.4989 58.7512,36.1498 58.553,36.3447 58.5402,36.0877 58.5174,35.4314 58.1349,36.403 58.0507,36.5949 58.1673,36.0123 58.2869,37.191 58.6819,36.4989 58.7512),(34.4816 56.8232,34.8098 57.0409,33.5602 56.9781,33.3418 56.8364,33.8361 56.6953,34.1885 56.6259,34.3867 56.7596,34.229 56.7948,34.4816 56.8232),(35.9179 57.7512,35.7402 57.7909,36.0848 57.855,36.3932 58.0447,35.1134 57.9454,34.6332 57.6538,35.613 57.5595,35.9179 57.7512),(36.8709 53.2765,37.135 53.4711,36.9794 53.5878,37.3119 53.9273,37.0035 54.2999,36.6985 54.0791,36.919 53.8561,36.3552 53.8269,36.1528 53.6763,36.8709 53.2765),(38.1601 55.1091,38.3093 55.1546,38.2609 55.1775,38.1601 55.1091),(38.1688 56.0758,38.4339 56.2361,37.5054 56.5484,37.2281 56.3799,38.1688 56.0758),(38.1319 56.0534,36.647 55.9411,37.6238 55.7402,38.1319 56.0534),(37.2824 55.5258,36.8283 55.4471,37.06 55.3843,37.2824 55.5258),(36.151 54.791,36.0123 54.7554,36.0472 54.7217,36.151 54.791),(34.9611 53.9765,34.894 54.1226,35.6193 54.4929,34.9706 54.9262,34.7231 54.7576,35.0753 54.5981,34.1081 54.1757,34.7279 53.8116,34.9611 53.9765),(38.2312 56.9795,37.565 56.5843,38.9742 56.8774,38.4915 57.1308,38.2699 57.0021,38.3093 56.9929,38.2312 56.9795),(36.5334 56.6753,36.375 56.6455,36.4446 56.6242,36.5334 56.6753),(36.1999 57.0022,36.9794 57.0751,36.4587 57.1544,36.1999 57.0022),(34.6028 58.3749,33.6245 58.271,34.3593 58.2189,34.6028 58.3749),(33.7581 57.8255,33.2316 57.7748,33.6325 57.7419,33.7581 57.8255),(31.6069 56.3194,31.7506 56.8609,31.6514 57.1258,30.3301 56.1942,30.2394 55.2753,31.6069 56.3194),(34.2274 57.4023,34.0208 57.2724,35.0338 57.1875,35.4682 57.4674,34.2274 57.4023),(31.7782 55.7778,30.2092 54.6331,30.2394 53.6774,31.7439 54.8677,31.8413 54.9989,32.204 55.5156,31.7782 55.7778),(33.7222 56.3063,32.8387 56.3117,33.5244 56.1686,33.7222 56.3063),(33.1204 55.8832,32.748 55.9072,32.9547 55.7645,33.1204 55.8832),(35.2275 55.0993,36.4354 55.3441,35.7505 55.4454,35.2275 55.0993),(35.9817 55.5958,36.5563 55.6352,36.193 55.7319,35.9817 55.5958),(35.0954 55.822,35.3188 55.9582,34.7331 56.1049,34.4996 55.9565,35.0954 55.822),(34.9721 55.7463,34.2598 55.8023,33.6125 55.3778,34.3709 55.3709,34.9721 55.7463),(35.6571 56.1619,36.0233 56.3789,35.4083 56.5254,35.2273 56.414,35.71 56.3117,35.0485 56.303,34.744 56.1118,35.6571 56.1619),(40.2143 54.467,40.3948 54.4403,40.6064 54.034,39.9716 53.9807,40.2437 53.5878,39.5485 53.5878,39.9942 53.358,43.0243 55.3269,43.0243 56.2614,40.2143 54.467),(38.5511 53.2922,38.4609 53.226,39.2704 52.8471,39.9877 53.3534,38.5511 53.2922),(40.5716 55.8007,43.0243 57.2554,43.0243 58.0797,40.4543 56.5923,40.4855 56.4957,40.2529 56.4682,39.8102 56.1914,39.8205 56.0763,40.425 56.1942,40.5716 55.8007),(40.5504 55.7875,39.7601 55.7544,39.8151 55.3187,40.5504 55.7875),(39.7863 57.025,42.5105 58.477,41.6944 58.8542,40.1389 58.048,40.2437 58.0478,40.3343 57.4673,39.7299 57.4673,39.7863 57.025),(38.0744 57.5312,38.3737 57.6908,38.3395 57.7103,38.8533 58.0638,38.432 58.2584,38.0535 58.0542,38.3395 57.9356,37.4328 57.7103,38.0744 57.5312),(37.9669 57.4734,37.1608 57.2554,37.4489 57.1909,37.9669 57.4734),(40.4136 58.7241,41.2108 59.1035,40.6366 59.3817,39.8163 58.9766,40.4552 58.9011,40.4136 58.7241),(39.7184 58.3823,39.6392 58.3821,39.6392 58.3427,39.7184 58.3823),(38.7465 58.4255,39.5485 58.7133,39.4085 58.7696,38.7465 58.4255))) +MULTIPOLYGON(((35.677 58.9587,37.2816 59.9767,38.7324 59.9464,37.1527 59.0192,37.3118 59.0257,38.0059 58.8722,39.7298 59.9313,44.475 59.8099,44.4145 55.3096,40.0924 52.1651,38.3394 52.1651,39.1804 52.7502,38.0733 52.8911,37.1607 52.2392,35.4681 52.2021,36.5898 52.9925,35.4415 53.0432,34.2894 52.2207,32.5968 52.2207,33.5229 52.8475,33.1711 52.8275,32.6253 53.1196,31.2367 52.1651,29.786 52.1465,32.4267 53.8656,32.0505 54.0742,29.393 52.2762,29.4535 59.7795,30.5718 59.9918,30.4811 58.8541,32.3248 59.9464,33.6547 59.9464,30.1789 57.9195,30.1789 56.9763,32.2099 58.3086,32.2341 58.4927,32.7181 58.6343,34.8636 59.9767,36.2842 59.9615,34.1607 58.6635,35.1359 58.6084,35.4917 58.8391,34.6521 58.9166,35.677 58.9587),(36.6691 58.7302,35.6455 58.1105,36.4353 58.0477,35.3989 57.9596,34.8546 57.6243,35.8918 57.5278,36.1527 57.6965,35.7401 57.7908,36.5188 57.9318,36.8134 58.1201,36.0122 58.2868,37.1909 58.6818,36.6691 58.7302),(34.8711 56.8603,35.1561 57.0479,33.8307 56.9848,33.5159 56.786,33.836 56.6952,34.4368 56.5727,34.6299 56.7008,34.2289 56.7947,34.8711 56.8603),(36.9376 53.2346,37.2027 53.4183,36.9793 53.5877,37.3118 53.9272,37.0903 54.1938,36.7891 53.9859,36.9189 53.856,36.575 53.8375,36.2567 53.616,36.9376 53.2346),(37.8883 55.6832,38.6515 56.1591,37.7506 56.465,37.4799 56.2952,38.2185 56.0593,36.9557 55.9643,36.8521 55.8985,37.8883 55.6832),(37.7656 55.6061,36.8282 55.447,37.3054 55.3158,37.7656 55.6061),(36.6884 54.9232,36.0122 54.7553,36.1761 54.5942,36.6884 54.9232),(35.0208 53.8426,34.8939 54.1225,35.6192 54.4928,35.153 54.8041,34.9347 54.6613,35.0752 54.598,34.374 54.2922,34.1558 54.1477,34.773 53.7846,34.773 53.6795,35.0208 53.8426),(37.1816 56.7907,36.3749 56.6454,36.7422 56.5294,37.1816 56.7907),(36.7103 57.0477,36.9793 57.075,36.7988 57.102,36.7103 57.0477),(34.7969 58.3872,33.6244 58.2709,34.5125 58.2005,34.7969 58.3872),(33.9633 57.8372,33.2315 57.7747,33.7916 57.7229,33.9633 57.8372),(33.2199 57.3396,33.1711 57.3369,33.211 57.3336,33.2199 57.3396),(31.581 56.2183,31.7505 56.8608,31.6694 57.0765,30.33 56.1941,30.2393 55.2752,31.581 56.2183),(34.5079 57.409,34.2469 57.2461,35.3199 57.1553,35.8173 57.4795,34.5079 57.409),(31.863 55.7247,30.2091 54.633,30.2393 53.6773,31.5514 54.605,31.8412 54.9988,32.2039 55.5155,31.863 55.7247),(34.0285 56.3002,32.8386 56.3116,33.7487 56.1124,34.0285 56.3002),(33.3749 55.86,32.7479 55.9071,33.0915 55.6676,33.3749 55.86),(35.777 55.2096,36.4353 55.344,36.0682 55.3974,35.777 55.2096),(36.4239 55.6256,36.5562 55.6351,36.4727 55.6567,36.4239 55.6256),(35.4341 55.7407,35.6467 55.87,35.0606 56.0207,34.8294 55.8745,35.4341 55.7407),(35.3816 55.7088,34.6546 55.7636,34.0415 55.3722,34.8176 55.3634,35.3816 55.7088),(36.1651 56.1834,36.3512 56.2953,35.7316 56.4418,35.5695 56.3405,35.7099 56.3116,35.5188 56.3087,35.245 56.1369,36.1651 56.1834),(40.3151 54.4517,40.3947 54.4402,40.6063 54.0339,39.9715 53.9806,40.2436 53.5877,39.5484 53.5877,40.0018 53.3539,38.6455 53.2949,38.512 53.2011,39.2985 52.8318,43.0242 55.3268,43.0242 56.2613,40.3151 54.4517),(39.9389 55.2681,43.0242 57.2553,43.0242 58.0796,39.9194 56.0955,40.4249 56.1941,40.576 55.7883,39.76 55.7543,39.8204 55.2752,39.9389 55.2681),(39.9915 56.9806,42.5104 58.4769,41.6943 58.8541,40.2513 57.9982,40.3342 57.4672,39.7298 57.4672,39.7902 56.9928,39.9915 56.9806),(38.1298 56.7016,38.9741 56.8773,38.6704 57.0364,38.1298 56.7016),(37.7436 57.1223,38.5465 57.5909,38.3394 57.7102,38.8532 58.0637,38.591 58.1848,38.2411 57.9759,38.3394 57.9355,38.0568 57.8654,37.6831 57.6403,38.1279 57.5159,37.1607 57.2553,37.7436 57.1223),(40.3959 58.6476,41.2107 59.1034,40.6365 59.3816,39.9119 58.9623,40.4551 58.901,40.3959 58.6476),(39.9258 58.382,39.6391 58.382,39.6391 58.219,39.9258 58.382),(39.3854 58.6545,39.5484 58.7132,39.5108 58.7281,39.3854 58.6545))) +-------- MultiPolygon with Polygon with Holes +MULTIPOLYGON(((24.3677 61.4598,26.6528 61.1008,26.8726 61.7107,30.564 61.0583,31.3989 62.0215,36.0132 61.1432,36.8921 62.0009,42.6489 60.6301,43.5718 61.3757,47.0435 59.8889,49.5923 60.0868,49.1528 58.1707,51.9214 57.9148,50.2515 56.1455,52.6685 55.826,51.6577 54.2909,52.8882 53.9302,50.647 53.0148,51.394 52.4828,48.0542 51.1793,49.2847 50.5414,47.1753 49.153,43.9233 49.8096,42.561 48.7779,36.936 49.6676,35.2661 48.7489,32.8052 49.5252,27.2241 48.9802,26.1255 50.4015,21.2036 50.205,20.0171 51.5634,17.4683 53.0148,19.4458 54.0852,19.4458 55.8753,19.5776 57.4922,19.5776 58.6769,24.3677 61.4598),(24.4556 59.4227,21.2036 58.4937,21.3354 56.897,21.5991 55.9246,25.2026 55.9984,28.8501 57.0646,27.0923 57.8448,28.8062 59.1759,26.2573 59.1759,24.4556 59.4227),(33.1079 56.9523,33.1392 56.8934,33.7182 56.7292,35.1489 56.5859,34.229 56.7948,36.9794 57.0751,35.7705 57.2554,37.0097 57.4998,35.7402 57.7909,37.1608 58.0478,36.0123 58.2869,37.191 58.6819,34.6522 58.9167,37.2327 59.0233,37.1118 59.6677,35.1343 59.8448,31.9702 58.9727,32.25 58.4976,33.4734 58.8542,34.7428 59.5659,33.8361 58.6819,36.3447 58.5402,33.6245 58.271,36.4354 58.0478,33.2316 57.7748,36.1936 57.4998,33.1712 57.337,36.0727 57.0915,33.1079 56.9523),(37.0604 52.9744,34.9585 51.4814,36.5405 50.4015,39.6606 50.2893,39.7925 52.1335,41.77 50.6808,44.4946 51.9713,47.3071 52.5095,44.0552 53.5403,46.604 53.6967,47.6147 55.4041,45.3735 55.4041,42.8247 56.5837,40.4412 56.1511,40.5761 55.7884,39.7601 55.7544,39.8205 55.2753,40.3948 55.2408,40.3948 54.8773,39.5485 54.8773,39.5485 54.5631,40.3948 54.4403,40.6064 54.034,39.9716 53.9807,40.2437 53.5878,39.5485 53.5878,40.0019 53.354,38.3395 53.2817,39.5787 52.6996,37.8559 52.9188,37.4471 53.2343,37.2165 53.0798,37.4328 52.9552,37.0604 52.9744),(31.627 54.7093,29.5972 55.5037,29.1577 55.7518,22.5659 55.1286,22.5659 53.5403,22.0386 51.4814,26.2573 51.4266,30.1245 50.5414,32.1899 51.1793,30.1245 53.1731,32.4808 53.1989,32.0831 53.408,32.476 53.8383,31.4182 54.4227,31.627 54.7093),(34.7731 53.3243,34.7731 53.1793,35.0903 53.1731,34.7731 53.3243),(36.9508 55.414,37.7653 55.1891,36.8822 54.975,37.0572 54.7635,38.3093 55.1546,37.7955 55.3956,38.4907 55.5327,38.3184 55.7179,38.0262 55.6546,38.0373 55.6523,37.9482 55.6376,36.9508 55.414),(38.3092 56.9929,38.5798 57.0849,38.2186 57.2717,38.7325 57.4835,38.3395 57.7103,38.8533 58.0638,38.3698 58.2869,39.5485 58.7133,38.8838 58.9777,38.0944 58.8545,38.5813 58.7446,37.4026 58.3187,38.3395 57.9356,37.4328 57.7103,38.128 57.516,37.1608 57.2554,38.3092 56.9929),(38.309 56.9928,36.375 56.6455,36.8799 56.4895,38.309 56.9928),(40.3237 57.5365,42.6929 58.0314,40.8911 59.2659,39.2792 59.0373,40.4552 58.9011,40.3343 58.3821,39.6392 58.3821,39.6392 58.0478,40.2437 58.0478,40.3237 57.5365),(40.0149 57.4677,39.7299 57.4673,39.7379 57.4051,40.0149 57.4677))) +MULTIPOLYGON(((24.3676 61.4597,26.6527 61.1007,26.8725 61.7106,30.5639 61.0582,31.3988 62.0214,36.0131 61.1431,36.892 62.0008,42.6488 60.63,43.5717 61.3756,47.0434 59.8888,49.5922 60.0867,49.1527 58.1706,51.9213 57.9147,50.2514 56.1454,52.6684 55.8259,51.6576 54.2908,52.8881 53.9301,50.6469 53.0147,51.3939 52.4827,48.0541 51.1792,49.2846 50.5413,47.1752 49.1529,43.9232 49.8095,42.5609 48.7778,36.9359 49.6675,35.266 48.7488,32.8051 49.5251,27.224 48.9801,26.1254 50.4014,21.2035 50.2049,20.017 51.5633,17.4682 53.0147,19.4457 54.0851,19.4457 55.8752,19.5775 57.4921,19.5775 58.6768,24.3676 61.4597),(24.4555 59.4226,21.2035 58.4936,21.3353 56.8969,21.599 55.9245,25.2025 55.9983,28.85 57.0645,27.0922 57.8447,28.8061 59.1758,26.2572 59.1758,24.4555 59.4226),(33.106 56.9503,33.1372 56.8932,33.7508 56.7194,35.1914 56.569,34.2289 56.7947,36.9793 57.075,35.7704 57.2553,37.0096 57.4997,35.7401 57.7908,37.1607 58.0477,36.0122 58.2868,37.1909 58.6818,34.6521 58.9166,37.2322 59.0224,37.1117 59.6676,35.1342 59.8447,31.9701 58.9726,32.2439 58.4956,33.4733 58.8541,34.7427 59.5658,33.836 58.6818,36.3446 58.5401,33.6244 58.2709,36.4353 58.0477,33.2315 57.7747,36.1935 57.4997,33.1711 57.3369,36.0726 57.0914,33.106 56.9503),(37.0823 52.9707,34.9584 51.4813,36.5404 50.4014,39.6605 50.2892,39.7924 52.1334,41.7699 50.6807,44.4945 51.9712,47.307 52.5094,44.0551 53.5402,46.6039 53.6966,47.6146 55.404,45.3734 55.404,42.8246 56.5836,40.4556 56.1121,40.576 55.7883,39.76 55.7543,39.8204 55.2752,40.3947 55.2407,40.3947 54.8772,39.5484 54.8772,39.5484 54.563,40.3947 54.4402,40.6063 54.0339,39.9715 53.9806,40.2436 53.5877,39.5484 53.5877,40.0018 53.3539,38.3394 53.2816,39.5786 52.6995,37.8558 52.9187,37.4544 53.2264,37.2279 53.0709,37.4327 52.9551,37.0823 52.9707),(34.773 53.173,35.0902 53.173,34.773 53.3207,34.773 53.173),(32.5252 53.173,32.083 53.4079,32.4759 53.8382,31.4181 54.4226,31.6171 54.6946,29.5971 55.5036,29.1576 55.7517,22.5658 55.1285,22.5658 53.5402,22.0385 51.4813,26.2572 51.4265,30.1244 50.5413,32.1898 51.1792,30.1244 53.173,32.5252 53.173),(36.9706 55.4079,37.7652 55.189,36.8796 54.9705,37.0521 54.7595,38.3092 55.1545,37.7954 55.3955,38.4906 55.5326,38.3463 55.6874,36.9706 55.4079),(36.8783 56.4863,38.5847 57.0811,38.2185 57.2716,38.7324 57.4834,38.3394 57.7102,38.8532 58.0637,38.3697 58.2868,39.5484 58.7132,38.9056 58.9675,38.1151 58.848,38.5812 58.7445,37.4025 58.3186,38.3394 57.9355,37.4327 57.7102,38.1279 57.5159,37.1607 57.2553,38.3092 56.9928,36.3749 56.6454,36.8783 56.4863),(40.3255 57.5233,42.6928 58.0313,40.891 59.2658,39.3164 59.0294,40.4551 58.901,40.3342 58.382,39.6391 58.382,39.6391 58.0477,40.2436 58.0477,40.3255 57.5233),(40.066 57.4672,39.7298 57.4672,39.7388 57.3963,40.066 57.4672))) +-------- Polygon with Polygon with Holes +MULTIPOLYGON(((24.3677 61.4598,26.6528 61.1008,26.8726 61.7107,30.564 61.0583,31.3989 62.0215,36.0132 61.1432,36.8921 62.0009,42.6489 60.6301,43.5718 61.3757,47.0435 59.8889,49.5923 60.0868,49.1528 58.1707,51.9214 57.9148,50.2515 56.1455,52.6685 55.826,51.6577 54.2909,52.8882 53.9302,50.647 53.0148,51.394 52.4828,48.0542 51.1793,49.2847 50.5414,47.1753 49.153,43.9233 49.8096,42.561 48.7779,36.936 49.6676,35.2661 48.7489,32.8052 49.5252,27.2241 48.9802,26.1255 50.4015,21.2036 50.205,20.0171 51.5634,17.4683 53.0148,19.4458 54.0852,19.4458 55.8753,19.5776 57.4922,19.5776 58.6769,24.3677 61.4598),(24.4556 59.4227,21.2036 58.4937,21.3354 56.897,21.5991 55.9246,25.2026 55.9984,28.8501 57.0646,27.0923 57.8448,28.8062 59.1759,26.2573 59.1759,24.4556 59.4227),(32.6512 57.792,32.9378 57.2699,36.7912 59.6986,35.9475 59.7758,32.6512 57.792),(33.2446 56.7729,34.2635 56.6767,37.6322 58.7797,37.2876 58.7226,37.2102 59.1452,33.2446 56.7729),(36.1815 56.4715,41.168 59.0834,40.9299 59.2404,40.8804 59.2644,40.2079 59.1718,35.4536 56.5531,36.1815 56.4715),(30.7705 55.0525,30.2092 54.6331,30.2394 53.6774,31.5682 54.7333,30.7705 55.0525),(33.8733 53.1922,34.3351 53.53,33.5144 53.9057,32.5603 53.1989,33.8733 53.1922),(31.1968 52.1649,29.7861 52.1466,30.5785 52.7531,30.3098 53.0028,29.3931 52.2763,29.4171 55.606,29.1577 55.7518,22.5659 55.1286,22.5659 53.5403,22.0386 51.4814,26.2573 51.4266,30.1245 50.5414,32.1899 51.1793,31.1968 52.1649),(31.1682 53.1903,32.6907 54.2663,32.2591 54.4483,30.5408 53.1811,31.1682 53.1903),(39.4328 55.9511,37.2766 54.4948,37.7431 53.9104,41.4519 56.3413,39.4328 55.9511),(40.9691 57.677,42.2498 58.3455,41.5887 58.8012,38.1759 56.9472,39.0894 57.2553,40.9691 57.677),(37.1934 55.4694,36.5845 55.3291,36.7219 55.1665,37.1934 55.4694),(32.2964 58.4175,34.2247 59.6064,31.9702 58.9727,32.2964 58.4175),(35.9681 52.2157,34.9585 51.4814,36.5405 50.4015,39.6606 50.2893,39.7925 52.1335,41.77 50.6808,44.4946 51.9713,47.3071 52.5095,44.0552 53.5403,46.604 53.6967,47.6147 55.4041,45.3735 55.4041,44.4212 55.8594,44.4146 55.3097,40.0925 52.1652,38.3395 52.1652,43.0243 55.3269,43.0243 56.2614,37.1608 52.2393,35.9681 52.2157))) +MULTIPOLYGON(((24.3676 61.4597,26.6527 61.1007,26.8725 61.7106,30.5639 61.0582,31.3988 62.0214,36.0131 61.1431,36.892 62.0008,42.6488 60.63,43.5717 61.3756,47.0434 59.8888,49.5922 60.0867,49.1527 58.1706,51.9213 57.9147,50.2514 56.1454,52.6684 55.8259,51.6576 54.2908,52.8881 53.9301,50.6469 53.0147,51.3939 52.4827,48.0541 51.1792,49.2846 50.5413,47.1752 49.1529,43.9232 49.8095,42.5609 48.7778,36.9359 49.6675,35.266 48.7488,32.8051 49.5251,27.224 48.9801,26.1254 50.4014,21.2035 50.2049,20.017 51.5633,17.4682 53.0147,19.4457 54.0851,19.4457 55.8752,19.5775 57.4921,19.5775 58.6768,24.3676 61.4597),(24.4555 59.4226,21.2035 58.4936,21.3353 56.8969,21.599 55.9245,25.2025 55.9983,28.85 57.0645,27.0922 57.8447,28.8061 59.1758,26.2572 59.1758,24.4555 59.4226),(32.6773 57.7269,32.9811 57.1783,36.8295 59.693,35.9663 59.7703,32.6773 57.7269),(33.4587 56.7498,34.5347 56.6377,37.9197 58.8184,37.2875 58.7225,37.2249 59.0621,33.4587 56.7498),(36.5672 56.4248,41.1674 59.0793,40.9293 59.24,40.8796 59.2641,40.276 59.1737,35.8287 56.5023,36.5672 56.4248),(30.7975 55.0249,30.2091 54.633,30.2393 53.6773,31.6568 54.6786,30.7975 55.0249),(34.0091 53.173,34.4539 53.4687,33.6753 53.8276,32.7039 53.173,34.0091 53.173),(31.1809 52.1644,29.786 52.1465,30.6237 52.6993,30.3667 52.9437,29.393 52.2762,29.4184 55.6047,29.1576 55.7517,22.5658 55.1285,22.5658 53.5402,22.0385 51.4813,26.2572 51.4265,30.1244 50.5413,32.1898 51.1792,31.1809 52.1644),(31.3501 53.173,32.9056 54.17,32.4672 54.3493,30.7047 53.173,31.3501 53.173),(39.7202 55.9646,37.3607 54.3795,37.8015 53.8302,41.577 56.336,39.7202 55.9646),(41.2147 57.7149,42.2607 58.3313,41.5893 58.7925,38.9459 57.2058,39.0893 57.2552,41.2147 57.7149),(37.6798 55.5522,36.5844 55.329,36.8394 55.0196,37.6798 55.5522),(32.3127 58.3747,34.2648 59.6073,31.9701 58.9726,32.3127 58.3747),(35.9939 52.2136,34.9584 51.4813,36.5404 50.4014,39.6605 50.2892,39.7924 52.1334,41.7699 50.6807,44.4945 51.9712,47.307 52.5094,44.0551 53.5402,46.6039 53.6966,47.6146 55.404,45.3734 55.404,44.4214 55.8488,44.4145 55.3096,40.0924 52.1651,38.3394 52.1651,43.0242 55.3268,43.0242 56.2613,37.1607 52.2392,35.9939 52.2136))) diff --git a/tests/queries/0_stateless/01305_polygons_union.sql b/tests/queries/0_stateless/01305_polygons_union.sql index d022bf46971..e3092dea2c0 100644 --- a/tests/queries/0_stateless/01305_polygons_union.sql +++ b/tests/queries/0_stateless/01305_polygons_union.sql @@ -9,3 +9,15 @@ select polygonsUnionSpherical([[[(4.3613577, 50.8651821), (4.349556, 50.8535879) select '--------'; select polygonsUnionMercator([[[(4.3613577, 50.8651821), (4.349556, 50.8535879), (4.3602419, 50.8435626), (4.3830299, 50.8428851), (4.3904543, 50.8564867), (4.3613148, 50.8651279)]]], [[[(4.346693, 50.858306), (4.367945, 50.852455), (4.366227, 50.840809), (4.344961, 50.833264), (4.338074, 50.848677), (4.346693, 50.858306)]]]); + +select '-------- MultiPolygon with Polygon'; +select wkt(polygonsUnionSpherical([[(29.453587685533865,59.779570356240356),(29.393139070478895,52.276266797422124),(40.636581470703206,59.38168915000267),(41.21084331372543,59.103467777099866),(29.786055068336193,52.146627480315004),(31.23682182965546,52.16517054781818),(41.69443223416517,58.85424941916091),(42.51048853740727,58.47703162291134),(32.59691566839227,52.22075341251539),(34.289476889931414,52.22075341251539),(43.02430176537451,58.07974369546071),(43.02430176537451,57.25537683364851),(35.468224883503325,52.2022335126388),(37.16078610504247,52.23926559241349),(43.02430176537451,56.26136189644947),(43.02430176537451,55.326904361850836),(38.33953409861437,52.16517054781818),(40.09254393520848,52.16517054781818),(44.4146199116388,55.3097062225408),(44.47506852669377,59.80998197603594),(39.72985224487867,59.931351417569715),(30.23941968124846,53.67744677450975),(30.20919537372098,54.63314259659509),(38.73245009647167,59.94649146557819),(37.2816833351524,59.97675082987618),(30.23941968124846,55.2752875586599),(30.33009260383092,56.19415599955667),(36.28428118674541,59.96162460231375),(34.863738732953635,59.97675082987618),(30.178971066193498,56.97640788219866),(30.178971066193498,57.91957806959033),(33.65476643185424,59.94649146557819),(32.32489690064491,59.94649146557819),(30.481214141468342,58.85424941916091),(30.571887064050795,59.99187015036608),(29.453587685533865,59.779570356240356)]], [[[(33.473420586689336,58.85424941916091),(32.23422397806246,58.492830557036),(32.173775363007486,58.03176922751564),(31.508840597402823,57.499784781503735),(31.750635057622702,56.86092686957355),(31.508840597402823,55.941082594334574),(32.20399967053497,55.515591939372456),(31.84130798020516,54.998862226280465),(31.418167674820367,54.422670886434275),(32.47601843828233,53.83826377018255),(32.08310244042503,53.408048308050866),(33.171177511414484,52.82758702113742),(34.77306581037117,52.91880107773494),(34.77306581037117,53.784726518357985),(34.108131044766516,54.17574726780569),(35.07530888564602,54.59813930694554),(34.25925258240394,54.96417435716029),(35.01486027059106,55.361278263643584),(33.50364489421682,55.37845402950552),(32.7480372060297,55.90721384574556),(35.67979503619571,55.68634475630185),(32.83871012861215,56.311688992608396),(34.591719965206266,56.29492065473883),(35.7100193437232,56.311688992608396),(33.83611227701915,56.695333481003644),(32.95960735872209,56.9434497616887),(36.072711034053015,57.091531913901434),(33.171177511414484,57.33702717078384),(36.193608264162954,57.499784781503735),(33.23162612646945,57.77481561306047),(36.43540272438284,58.04776787540811),(33.62454212432676,58.27099811968307),(36.344729801800376,58.54018474404165),(33.83611227701915,58.68186423448108),(34.74284150284369,59.565911441555244),(33.473420586689336,58.85424941916091)]], [[(34.65216858026123,58.91672306881671),(37.19101041256995,58.68186423448108),(36.01226241899805,58.28688958537609),(37.16078610504247,58.04776787540811),(35.74024365125068,57.79092907387934),(37.009664567405046,57.499784781503735),(35.77046795877817,57.25537683364851),(36.979440259877556,57.07510745541089),(34.22902827487645,56.794777197297435),(36.7074214921302,56.210968525786996),(34.712617195316206,56.10998276812964),(36.55629995449277,55.63519693782703),(35.13575750070099,55.53270067649592),(36.43540272438284,55.34409504165558),(34.83351442542614,55.01619492319591),(35.61934642114075,54.49294870011772),(34.89396304048112,54.12264226523038),(35.37755196092087,53.046178687628185),(37.43280487278982,52.95523300597458),(35.92158949641559,53.80257986695776),(36.91899164482259,53.856094327816805),(36.01226241899805,54.75541714463799),(37.765272255592166,55.189110239786885),(36.828318722240134,55.44708256557195),(38.03729102333953,55.652253637168315),(36.64697287707522,55.941082594334574),(38.21863686850443,56.05939028508024),(36.37495410932787,56.64551287174558),(38.30930979108689,56.992876013526654),(37.16078610504247,57.25537683364851),(38.127963945921984,57.516020773674256),(37.43280487278982,57.710289827306724),(38.33953409861437,57.935626886818994),(37.40258056526235,58.31865112960426),(38.58132855883426,58.744648733419496),(37.31190764267989,59.02578062465136),(34.65216858026123,58.91672306881671)]], [[(38.52087994377928,59.11898412389468),(39.54850639971376,58.713270635642914),(38.369758406141855,58.28688958537609),(38.85334732658162,58.06375936407028),(38.33953409861437,57.710289827306724),(38.73245009647167,57.48354156434209),(38.21863686850443,57.271721400459285),(38.97424455669155,56.87744603722649),(37.463029180317314,56.5623320541159),(38.94402024916407,56.05939028508024),(38.18841256097694,55.856355210835915),(38.490655636251795,55.53270067649592),(37.795496563119656,55.39562234093384),(38.30930979108689,55.154587013355666),(36.7074214921302,54.65063295250911),(37.31190764267989,53.92734063371401),(36.979440259877556,53.58783775557231),(37.855945178174615,52.91880107773497),(39.57873070724124,52.69956490610895),(38.33953409861437,53.281741738901104),(40.00187101262603,53.35396273604752),(39.54850639971376,53.58783775557231),(40.24366547284591,53.58783775557231),(39.97164670509855,53.98069568468355),(40.60635716317572,54.03398248547225),(40.39478701048334,54.44025165268903),(39.54850639971376,54.56310590284329),(39.54850639971376,54.87732350170489),(40.39478701048334,54.87732350170489),(40.39478701048334,55.24083903654295),(39.82052516746112,55.2752875586599),(39.760076552406154,55.75443792473942),(40.57613285564824,55.78844000174894),(40.425011318010824,56.19415599955667),(39.82052516746112,56.07626182891758),(39.79030085993364,56.41214455508424),(40.48545993306579,56.495655446714636),(40.33433839542836,56.95993246553937),(39.79030085993364,56.992876013526654),(39.72985224487867,57.46729112028032),(40.33433839542836,57.46729112028032),(40.24366547284591,58.04776787540811),(39.63917932229622,58.04776787540811),(39.63917932229622,58.382088724871295),(40.33433839542836,58.382088724871295),(40.45523562553831,58.9011152358548),(38.52087994377928,59.11898412389468)]]])) format TSV; +select wkt(polygonsUnionMercator([[(29.453587685533865,59.779570356240356),(29.393139070478895,52.276266797422124),(40.636581470703206,59.38168915000267),(41.21084331372543,59.103467777099866),(29.786055068336193,52.146627480315004),(31.23682182965546,52.16517054781818),(41.69443223416517,58.85424941916091),(42.51048853740727,58.47703162291134),(32.59691566839227,52.22075341251539),(34.289476889931414,52.22075341251539),(43.02430176537451,58.07974369546071),(43.02430176537451,57.25537683364851),(35.468224883503325,52.2022335126388),(37.16078610504247,52.23926559241349),(43.02430176537451,56.26136189644947),(43.02430176537451,55.326904361850836),(38.33953409861437,52.16517054781818),(40.09254393520848,52.16517054781818),(44.4146199116388,55.3097062225408),(44.47506852669377,59.80998197603594),(39.72985224487867,59.931351417569715),(30.23941968124846,53.67744677450975),(30.20919537372098,54.63314259659509),(38.73245009647167,59.94649146557819),(37.2816833351524,59.97675082987618),(30.23941968124846,55.2752875586599),(30.33009260383092,56.19415599955667),(36.28428118674541,59.96162460231375),(34.863738732953635,59.97675082987618),(30.178971066193498,56.97640788219866),(30.178971066193498,57.91957806959033),(33.65476643185424,59.94649146557819),(32.32489690064491,59.94649146557819),(30.481214141468342,58.85424941916091),(30.571887064050795,59.99187015036608),(29.453587685533865,59.779570356240356)]], [[[(33.473420586689336,58.85424941916091),(32.23422397806246,58.492830557036),(32.173775363007486,58.03176922751564),(31.508840597402823,57.499784781503735),(31.750635057622702,56.86092686957355),(31.508840597402823,55.941082594334574),(32.20399967053497,55.515591939372456),(31.84130798020516,54.998862226280465),(31.418167674820367,54.422670886434275),(32.47601843828233,53.83826377018255),(32.08310244042503,53.408048308050866),(33.171177511414484,52.82758702113742),(34.77306581037117,52.91880107773494),(34.77306581037117,53.784726518357985),(34.108131044766516,54.17574726780569),(35.07530888564602,54.59813930694554),(34.25925258240394,54.96417435716029),(35.01486027059106,55.361278263643584),(33.50364489421682,55.37845402950552),(32.7480372060297,55.90721384574556),(35.67979503619571,55.68634475630185),(32.83871012861215,56.311688992608396),(34.591719965206266,56.29492065473883),(35.7100193437232,56.311688992608396),(33.83611227701915,56.695333481003644),(32.95960735872209,56.9434497616887),(36.072711034053015,57.091531913901434),(33.171177511414484,57.33702717078384),(36.193608264162954,57.499784781503735),(33.23162612646945,57.77481561306047),(36.43540272438284,58.04776787540811),(33.62454212432676,58.27099811968307),(36.344729801800376,58.54018474404165),(33.83611227701915,58.68186423448108),(34.74284150284369,59.565911441555244),(33.473420586689336,58.85424941916091)]], [[(34.65216858026123,58.91672306881671),(37.19101041256995,58.68186423448108),(36.01226241899805,58.28688958537609),(37.16078610504247,58.04776787540811),(35.74024365125068,57.79092907387934),(37.009664567405046,57.499784781503735),(35.77046795877817,57.25537683364851),(36.979440259877556,57.07510745541089),(34.22902827487645,56.794777197297435),(36.7074214921302,56.210968525786996),(34.712617195316206,56.10998276812964),(36.55629995449277,55.63519693782703),(35.13575750070099,55.53270067649592),(36.43540272438284,55.34409504165558),(34.83351442542614,55.01619492319591),(35.61934642114075,54.49294870011772),(34.89396304048112,54.12264226523038),(35.37755196092087,53.046178687628185),(37.43280487278982,52.95523300597458),(35.92158949641559,53.80257986695776),(36.91899164482259,53.856094327816805),(36.01226241899805,54.75541714463799),(37.765272255592166,55.189110239786885),(36.828318722240134,55.44708256557195),(38.03729102333953,55.652253637168315),(36.64697287707522,55.941082594334574),(38.21863686850443,56.05939028508024),(36.37495410932787,56.64551287174558),(38.30930979108689,56.992876013526654),(37.16078610504247,57.25537683364851),(38.127963945921984,57.516020773674256),(37.43280487278982,57.710289827306724),(38.33953409861437,57.935626886818994),(37.40258056526235,58.31865112960426),(38.58132855883426,58.744648733419496),(37.31190764267989,59.02578062465136),(34.65216858026123,58.91672306881671)]], [[(38.52087994377928,59.11898412389468),(39.54850639971376,58.713270635642914),(38.369758406141855,58.28688958537609),(38.85334732658162,58.06375936407028),(38.33953409861437,57.710289827306724),(38.73245009647167,57.48354156434209),(38.21863686850443,57.271721400459285),(38.97424455669155,56.87744603722649),(37.463029180317314,56.5623320541159),(38.94402024916407,56.05939028508024),(38.18841256097694,55.856355210835915),(38.490655636251795,55.53270067649592),(37.795496563119656,55.39562234093384),(38.30930979108689,55.154587013355666),(36.7074214921302,54.65063295250911),(37.31190764267989,53.92734063371401),(36.979440259877556,53.58783775557231),(37.855945178174615,52.91880107773497),(39.57873070724124,52.69956490610895),(38.33953409861437,53.281741738901104),(40.00187101262603,53.35396273604752),(39.54850639971376,53.58783775557231),(40.24366547284591,53.58783775557231),(39.97164670509855,53.98069568468355),(40.60635716317572,54.03398248547225),(40.39478701048334,54.44025165268903),(39.54850639971376,54.56310590284329),(39.54850639971376,54.87732350170489),(40.39478701048334,54.87732350170489),(40.39478701048334,55.24083903654295),(39.82052516746112,55.2752875586599),(39.760076552406154,55.75443792473942),(40.57613285564824,55.78844000174894),(40.425011318010824,56.19415599955667),(39.82052516746112,56.07626182891758),(39.79030085993364,56.41214455508424),(40.48545993306579,56.495655446714636),(40.33433839542836,56.95993246553937),(39.79030085993364,56.992876013526654),(39.72985224487867,57.46729112028032),(40.33433839542836,57.46729112028032),(40.24366547284591,58.04776787540811),(39.63917932229622,58.04776787540811),(39.63917932229622,58.382088724871295),(40.33433839542836,58.382088724871295),(40.45523562553831,58.9011152358548),(38.52087994377928,59.11898412389468)]]])) format TSV; + +select '-------- MultiPolygon with Polygon with Holes'; +select wkt(polygonsUnionSpherical([[[(33.473420586689336,58.85424941916091),(32.23422397806246,58.492830557036),(32.173775363007486,58.03176922751564),(31.508840597402823,57.499784781503735),(31.750635057622702,56.86092686957355),(31.508840597402823,55.941082594334574),(32.20399967053497,55.515591939372456),(31.84130798020516,54.998862226280465),(31.418167674820367,54.422670886434275),(32.47601843828233,53.83826377018255),(32.08310244042503,53.408048308050866),(33.171177511414484,52.82758702113742),(34.77306581037117,52.91880107773494),(34.77306581037117,53.784726518357985),(34.108131044766516,54.17574726780569),(35.07530888564602,54.59813930694554),(34.25925258240394,54.96417435716029),(35.01486027059106,55.361278263643584),(33.50364489421682,55.37845402950552),(32.7480372060297,55.90721384574556),(35.67979503619571,55.68634475630185),(32.83871012861215,56.311688992608396),(34.591719965206266,56.29492065473883),(35.7100193437232,56.311688992608396),(33.83611227701915,56.695333481003644),(32.95960735872209,56.9434497616887),(36.072711034053015,57.091531913901434),(33.171177511414484,57.33702717078384),(36.193608264162954,57.499784781503735),(33.23162612646945,57.77481561306047),(36.43540272438284,58.04776787540811),(33.62454212432676,58.27099811968307),(36.344729801800376,58.54018474404165),(33.83611227701915,58.68186423448108),(34.74284150284369,59.565911441555244),(33.473420586689336,58.85424941916091)]], [[(34.65216858026123,58.91672306881671),(37.19101041256995,58.68186423448108),(36.01226241899805,58.28688958537609),(37.16078610504247,58.04776787540811),(35.74024365125068,57.79092907387934),(37.009664567405046,57.499784781503735),(35.77046795877817,57.25537683364851),(36.979440259877556,57.07510745541089),(34.22902827487645,56.794777197297435),(36.7074214921302,56.210968525786996),(34.712617195316206,56.10998276812964),(36.55629995449277,55.63519693782703),(35.13575750070099,55.53270067649592),(36.43540272438284,55.34409504165558),(34.83351442542614,55.01619492319591),(35.61934642114075,54.49294870011772),(34.89396304048112,54.12264226523038),(35.37755196092087,53.046178687628185),(37.43280487278982,52.95523300597458),(35.92158949641559,53.80257986695776),(36.91899164482259,53.856094327816805),(36.01226241899805,54.75541714463799),(37.765272255592166,55.189110239786885),(36.828318722240134,55.44708256557195),(38.03729102333953,55.652253637168315),(36.64697287707522,55.941082594334574),(38.21863686850443,56.05939028508024),(36.37495410932787,56.64551287174558),(38.30930979108689,56.992876013526654),(37.16078610504247,57.25537683364851),(38.127963945921984,57.516020773674256),(37.43280487278982,57.710289827306724),(38.33953409861437,57.935626886818994),(37.40258056526235,58.31865112960426),(38.58132855883426,58.744648733419496),(37.31190764267989,59.02578062465136),(34.65216858026123,58.91672306881671)]], [[(38.52087994377928,59.11898412389468),(39.54850639971376,58.713270635642914),(38.369758406141855,58.28688958537609),(38.85334732658162,58.06375936407028),(38.33953409861437,57.710289827306724),(38.73245009647167,57.48354156434209),(38.21863686850443,57.271721400459285),(38.97424455669155,56.87744603722649),(37.463029180317314,56.5623320541159),(38.94402024916407,56.05939028508024),(38.18841256097694,55.856355210835915),(38.490655636251795,55.53270067649592),(37.795496563119656,55.39562234093384),(38.30930979108689,55.154587013355666),(36.7074214921302,54.65063295250911),(37.31190764267989,53.92734063371401),(36.979440259877556,53.58783775557231),(37.855945178174615,52.91880107773497),(39.57873070724124,52.69956490610895),(38.33953409861437,53.281741738901104),(40.00187101262603,53.35396273604752),(39.54850639971376,53.58783775557231),(40.24366547284591,53.58783775557231),(39.97164670509855,53.98069568468355),(40.60635716317572,54.03398248547225),(40.39478701048334,54.44025165268903),(39.54850639971376,54.56310590284329),(39.54850639971376,54.87732350170489),(40.39478701048334,54.87732350170489),(40.39478701048334,55.24083903654295),(39.82052516746112,55.2752875586599),(39.760076552406154,55.75443792473942),(40.57613285564824,55.78844000174894),(40.425011318010824,56.19415599955667),(39.82052516746112,56.07626182891758),(39.79030085993364,56.41214455508424),(40.48545993306579,56.495655446714636),(40.33433839542836,56.95993246553937),(39.79030085993364,56.992876013526654),(39.72985224487867,57.46729112028032),(40.33433839542836,57.46729112028032),(40.24366547284591,58.04776787540811),(39.63917932229622,58.04776787540811),(39.63917932229622,58.382088724871295),(40.33433839542836,58.382088724871295),(40.45523562553831,58.9011152358548),(38.52087994377928,59.11898412389468)]]], [[(24.367675781249993,61.45977057029751),(19.577636718749993,58.67693767258692),(19.577636718749993,57.492213666700735),(19.445800781249996,55.87531083569678),(19.445800781249996,54.085173420886775),(17.468261718749996,53.014783245859235),(20.017089843749993,51.563412328675895),(21.203613281249993,50.205033264943324),(26.125488281249993,50.40151532278236),(27.22412109374999,48.980216985374994),(32.80517578124999,49.525208341974405),(35.26611328124999,48.74894534343292),(36.93603515624999,49.66762782262194),(42.56103515625,48.77791275550183),(43.92333984374999,49.8096315635631),(47.17529296875,49.152969656170455),(49.28466796875,50.54136296522162),(48.05419921875,51.17934297928929),(51.39404296875,52.48278022207825),(50.64697265625,53.014783245859235),(52.88818359375,53.93021986394004),(51.65771484374999,54.29088164657006),(52.66845703125,55.825973254619015),(50.25146484375,56.145549500679095),(51.92138671875,57.914847767009206),(49.15283203125,58.17070248348605),(49.59228515625,60.086762746260064),(47.043457031249986,59.88893689676584),(43.57177734375,61.37567331572748),(42.64892578125,60.630101766266705),(36.89208984374999,62.000904713685856),(36.01318359374999,61.143235250840576),(31.398925781249993,62.02152819100766),(30.563964843749996,61.05828537037917),(26.872558593749993,61.71070595883174),(26.652832031249993,61.10078883158897),(24.367675781249993,61.45977057029751)], [(24.455566406249993,59.42272750081452),(21.203613281249993,58.49369382056807),(21.335449218749993,56.89700392127261),(21.599121093749993,55.92458580482949),(25.202636718749993,55.998380955359636),(28.850097656249993,57.06463027327854),(27.09228515625,57.844750992890994),(28.806152343749996,59.17592824927138),(26.257324218749993,59.17592824927138),(24.455566406249993,59.42272750081452)], [(35.13427734375,59.84481485969107),(31.970214843749993,58.97266715450152),(33.20068359374999,56.776808316568406),(36.67236328125,56.41390137600675),(39.08935546874999,57.25528054528888),(42.69287109374999,58.03137242177638),(40.89111328124999,59.26588062825809),(37.28759765625,58.722598828043374),(37.11181640624999,59.66774058164964),(35.13427734375,59.84481485969107)], [(29.157714843749993,55.75184939173528),(22.565917968749993,55.128649068488784),(22.565917968749993,53.54030739150019),(22.038574218749996,51.48138289610097),(26.257324218749993,51.42661449707484),(30.124511718749993,50.54136296522162),(32.18994140624999,51.17934297928929),(30.124511718749993,53.173119202640635),(35.09033203124999,53.173119202640635),(33.11279296875,54.085173420886775),(29.597167968749993,55.50374985927513),(29.157714843749993,55.75184939173528)], [(42.82470703125,56.58369172128337),(36.584472656249986,55.329144408405085),(37.99072265625,53.592504809039355),(34.95849609374999,51.48138289610097),(36.54052734374999,50.40151532278236),(39.66064453124999,50.289339253291786),(39.79248046875,52.13348804077148),(41.77001953125,50.68079714532166),(44.49462890624999,51.97134580885171),(47.30712890624999,52.509534770327264),(44.05517578125,53.54030739150019),(46.60400390625,53.696706475303245),(47.61474609375,55.40406982700608),(45.37353515625,55.40406982700608),(42.82470703125,56.58369172128337)]])) format TSV; +select wkt(polygonsUnionMercator([[[(33.473420586689336,58.85424941916091),(32.23422397806246,58.492830557036),(32.173775363007486,58.03176922751564),(31.508840597402823,57.499784781503735),(31.750635057622702,56.86092686957355),(31.508840597402823,55.941082594334574),(32.20399967053497,55.515591939372456),(31.84130798020516,54.998862226280465),(31.418167674820367,54.422670886434275),(32.47601843828233,53.83826377018255),(32.08310244042503,53.408048308050866),(33.171177511414484,52.82758702113742),(34.77306581037117,52.91880107773494),(34.77306581037117,53.784726518357985),(34.108131044766516,54.17574726780569),(35.07530888564602,54.59813930694554),(34.25925258240394,54.96417435716029),(35.01486027059106,55.361278263643584),(33.50364489421682,55.37845402950552),(32.7480372060297,55.90721384574556),(35.67979503619571,55.68634475630185),(32.83871012861215,56.311688992608396),(34.591719965206266,56.29492065473883),(35.7100193437232,56.311688992608396),(33.83611227701915,56.695333481003644),(32.95960735872209,56.9434497616887),(36.072711034053015,57.091531913901434),(33.171177511414484,57.33702717078384),(36.193608264162954,57.499784781503735),(33.23162612646945,57.77481561306047),(36.43540272438284,58.04776787540811),(33.62454212432676,58.27099811968307),(36.344729801800376,58.54018474404165),(33.83611227701915,58.68186423448108),(34.74284150284369,59.565911441555244),(33.473420586689336,58.85424941916091)]], [[(34.65216858026123,58.91672306881671),(37.19101041256995,58.68186423448108),(36.01226241899805,58.28688958537609),(37.16078610504247,58.04776787540811),(35.74024365125068,57.79092907387934),(37.009664567405046,57.499784781503735),(35.77046795877817,57.25537683364851),(36.979440259877556,57.07510745541089),(34.22902827487645,56.794777197297435),(36.7074214921302,56.210968525786996),(34.712617195316206,56.10998276812964),(36.55629995449277,55.63519693782703),(35.13575750070099,55.53270067649592),(36.43540272438284,55.34409504165558),(34.83351442542614,55.01619492319591),(35.61934642114075,54.49294870011772),(34.89396304048112,54.12264226523038),(35.37755196092087,53.046178687628185),(37.43280487278982,52.95523300597458),(35.92158949641559,53.80257986695776),(36.91899164482259,53.856094327816805),(36.01226241899805,54.75541714463799),(37.765272255592166,55.189110239786885),(36.828318722240134,55.44708256557195),(38.03729102333953,55.652253637168315),(36.64697287707522,55.941082594334574),(38.21863686850443,56.05939028508024),(36.37495410932787,56.64551287174558),(38.30930979108689,56.992876013526654),(37.16078610504247,57.25537683364851),(38.127963945921984,57.516020773674256),(37.43280487278982,57.710289827306724),(38.33953409861437,57.935626886818994),(37.40258056526235,58.31865112960426),(38.58132855883426,58.744648733419496),(37.31190764267989,59.02578062465136),(34.65216858026123,58.91672306881671)]], [[(38.52087994377928,59.11898412389468),(39.54850639971376,58.713270635642914),(38.369758406141855,58.28688958537609),(38.85334732658162,58.06375936407028),(38.33953409861437,57.710289827306724),(38.73245009647167,57.48354156434209),(38.21863686850443,57.271721400459285),(38.97424455669155,56.87744603722649),(37.463029180317314,56.5623320541159),(38.94402024916407,56.05939028508024),(38.18841256097694,55.856355210835915),(38.490655636251795,55.53270067649592),(37.795496563119656,55.39562234093384),(38.30930979108689,55.154587013355666),(36.7074214921302,54.65063295250911),(37.31190764267989,53.92734063371401),(36.979440259877556,53.58783775557231),(37.855945178174615,52.91880107773497),(39.57873070724124,52.69956490610895),(38.33953409861437,53.281741738901104),(40.00187101262603,53.35396273604752),(39.54850639971376,53.58783775557231),(40.24366547284591,53.58783775557231),(39.97164670509855,53.98069568468355),(40.60635716317572,54.03398248547225),(40.39478701048334,54.44025165268903),(39.54850639971376,54.56310590284329),(39.54850639971376,54.87732350170489),(40.39478701048334,54.87732350170489),(40.39478701048334,55.24083903654295),(39.82052516746112,55.2752875586599),(39.760076552406154,55.75443792473942),(40.57613285564824,55.78844000174894),(40.425011318010824,56.19415599955667),(39.82052516746112,56.07626182891758),(39.79030085993364,56.41214455508424),(40.48545993306579,56.495655446714636),(40.33433839542836,56.95993246553937),(39.79030085993364,56.992876013526654),(39.72985224487867,57.46729112028032),(40.33433839542836,57.46729112028032),(40.24366547284591,58.04776787540811),(39.63917932229622,58.04776787540811),(39.63917932229622,58.382088724871295),(40.33433839542836,58.382088724871295),(40.45523562553831,58.9011152358548),(38.52087994377928,59.11898412389468)]]], [[(24.367675781249993,61.45977057029751),(19.577636718749993,58.67693767258692),(19.577636718749993,57.492213666700735),(19.445800781249996,55.87531083569678),(19.445800781249996,54.085173420886775),(17.468261718749996,53.014783245859235),(20.017089843749993,51.563412328675895),(21.203613281249993,50.205033264943324),(26.125488281249993,50.40151532278236),(27.22412109374999,48.980216985374994),(32.80517578124999,49.525208341974405),(35.26611328124999,48.74894534343292),(36.93603515624999,49.66762782262194),(42.56103515625,48.77791275550183),(43.92333984374999,49.8096315635631),(47.17529296875,49.152969656170455),(49.28466796875,50.54136296522162),(48.05419921875,51.17934297928929),(51.39404296875,52.48278022207825),(50.64697265625,53.014783245859235),(52.88818359375,53.93021986394004),(51.65771484374999,54.29088164657006),(52.66845703125,55.825973254619015),(50.25146484375,56.145549500679095),(51.92138671875,57.914847767009206),(49.15283203125,58.17070248348605),(49.59228515625,60.086762746260064),(47.043457031249986,59.88893689676584),(43.57177734375,61.37567331572748),(42.64892578125,60.630101766266705),(36.89208984374999,62.000904713685856),(36.01318359374999,61.143235250840576),(31.398925781249993,62.02152819100766),(30.563964843749996,61.05828537037917),(26.872558593749993,61.71070595883174),(26.652832031249993,61.10078883158897),(24.367675781249993,61.45977057029751)], [(24.455566406249993,59.42272750081452),(21.203613281249993,58.49369382056807),(21.335449218749993,56.89700392127261),(21.599121093749993,55.92458580482949),(25.202636718749993,55.998380955359636),(28.850097656249993,57.06463027327854),(27.09228515625,57.844750992890994),(28.806152343749996,59.17592824927138),(26.257324218749993,59.17592824927138),(24.455566406249993,59.42272750081452)], [(35.13427734375,59.84481485969107),(31.970214843749993,58.97266715450152),(33.20068359374999,56.776808316568406),(36.67236328125,56.41390137600675),(39.08935546874999,57.25528054528888),(42.69287109374999,58.03137242177638),(40.89111328124999,59.26588062825809),(37.28759765625,58.722598828043374),(37.11181640624999,59.66774058164964),(35.13427734375,59.84481485969107)], [(29.157714843749993,55.75184939173528),(22.565917968749993,55.128649068488784),(22.565917968749993,53.54030739150019),(22.038574218749996,51.48138289610097),(26.257324218749993,51.42661449707484),(30.124511718749993,50.54136296522162),(32.18994140624999,51.17934297928929),(30.124511718749993,53.173119202640635),(35.09033203124999,53.173119202640635),(33.11279296875,54.085173420886775),(29.597167968749993,55.50374985927513),(29.157714843749993,55.75184939173528)], [(42.82470703125,56.58369172128337),(36.584472656249986,55.329144408405085),(37.99072265625,53.592504809039355),(34.95849609374999,51.48138289610097),(36.54052734374999,50.40151532278236),(39.66064453124999,50.289339253291786),(39.79248046875,52.13348804077148),(41.77001953125,50.68079714532166),(44.49462890624999,51.97134580885171),(47.30712890624999,52.509534770327264),(44.05517578125,53.54030739150019),(46.60400390625,53.696706475303245),(47.61474609375,55.40406982700608),(45.37353515625,55.40406982700608),(42.82470703125,56.58369172128337)]])) format TSV; + +select '-------- Polygon with Polygon with Holes'; +select wkt(polygonsUnionSpherical([[(29.453587685533865,59.779570356240356),(29.393139070478895,52.276266797422124),(40.636581470703206,59.38168915000267),(41.21084331372543,59.103467777099866),(29.786055068336193,52.146627480315004),(31.23682182965546,52.16517054781818),(41.69443223416517,58.85424941916091),(42.51048853740727,58.47703162291134),(32.59691566839227,52.22075341251539),(34.289476889931414,52.22075341251539),(43.02430176537451,58.07974369546071),(43.02430176537451,57.25537683364851),(35.468224883503325,52.2022335126388),(37.16078610504247,52.23926559241349),(43.02430176537451,56.26136189644947),(43.02430176537451,55.326904361850836),(38.33953409861437,52.16517054781818),(40.09254393520848,52.16517054781818),(44.4146199116388,55.3097062225408),(44.47506852669377,59.80998197603594),(39.72985224487867,59.931351417569715),(30.23941968124846,53.67744677450975),(30.20919537372098,54.63314259659509),(38.73245009647167,59.94649146557819),(37.2816833351524,59.97675082987618),(30.23941968124846,55.2752875586599),(30.33009260383092,56.19415599955667),(36.28428118674541,59.96162460231375),(34.863738732953635,59.97675082987618),(30.178971066193498,56.97640788219866),(30.178971066193498,57.91957806959033),(33.65476643185424,59.94649146557819),(32.32489690064491,59.94649146557819),(30.481214141468342,58.85424941916091),(30.571887064050795,59.99187015036608),(29.453587685533865,59.779570356240356)]], [[(24.367675781249993,61.45977057029751),(19.577636718749993,58.67693767258692),(19.577636718749993,57.492213666700735),(19.445800781249996,55.87531083569678),(19.445800781249996,54.085173420886775),(17.468261718749996,53.014783245859235),(20.017089843749993,51.563412328675895),(21.203613281249993,50.205033264943324),(26.125488281249993,50.40151532278236),(27.22412109374999,48.980216985374994),(32.80517578124999,49.525208341974405),(35.26611328124999,48.74894534343292),(36.93603515624999,49.66762782262194),(42.56103515625,48.77791275550183),(43.92333984374999,49.8096315635631),(47.17529296875,49.152969656170455),(49.28466796875,50.54136296522162),(48.05419921875,51.17934297928929),(51.39404296875,52.48278022207825),(50.64697265625,53.014783245859235),(52.88818359375,53.93021986394004),(51.65771484374999,54.29088164657006),(52.66845703125,55.825973254619015),(50.25146484375,56.145549500679095),(51.92138671875,57.914847767009206),(49.15283203125,58.17070248348605),(49.59228515625,60.086762746260064),(47.043457031249986,59.88893689676584),(43.57177734375,61.37567331572748),(42.64892578125,60.630101766266705),(36.89208984374999,62.000904713685856),(36.01318359374999,61.143235250840576),(31.398925781249993,62.02152819100766),(30.563964843749996,61.05828537037917),(26.872558593749993,61.71070595883174),(26.652832031249993,61.10078883158897),(24.367675781249993,61.45977057029751)], [(24.455566406249993,59.42272750081452),(21.203613281249993,58.49369382056807),(21.335449218749993,56.89700392127261),(21.599121093749993,55.92458580482949),(25.202636718749993,55.998380955359636),(28.850097656249993,57.06463027327854),(27.09228515625,57.844750992890994),(28.806152343749996,59.17592824927138),(26.257324218749993,59.17592824927138),(24.455566406249993,59.42272750081452)], [(35.13427734375,59.84481485969107),(31.970214843749993,58.97266715450152),(33.20068359374999,56.776808316568406),(36.67236328125,56.41390137600675),(39.08935546874999,57.25528054528888),(42.69287109374999,58.03137242177638),(40.89111328124999,59.26588062825809),(37.28759765625,58.722598828043374),(37.11181640624999,59.66774058164964),(35.13427734375,59.84481485969107)], [(29.157714843749993,55.75184939173528),(22.565917968749993,55.128649068488784),(22.565917968749993,53.54030739150019),(22.038574218749996,51.48138289610097),(26.257324218749993,51.42661449707484),(30.124511718749993,50.54136296522162),(32.18994140624999,51.17934297928929),(30.124511718749993,53.173119202640635),(35.09033203124999,53.173119202640635),(33.11279296875,54.085173420886775),(29.597167968749993,55.50374985927513),(29.157714843749993,55.75184939173528)], [(42.82470703125,56.58369172128337),(36.584472656249986,55.329144408405085),(37.99072265625,53.592504809039355),(34.95849609374999,51.48138289610097),(36.54052734374999,50.40151532278236),(39.66064453124999,50.289339253291786),(39.79248046875,52.13348804077148),(41.77001953125,50.68079714532166),(44.49462890624999,51.97134580885171),(47.30712890624999,52.509534770327264),(44.05517578125,53.54030739150019),(46.60400390625,53.696706475303245),(47.61474609375,55.40406982700608),(45.37353515625,55.40406982700608),(42.82470703125,56.58369172128337)]])) format TSV; +select wkt(polygonsUnionMercator([[(29.453587685533865,59.779570356240356),(29.393139070478895,52.276266797422124),(40.636581470703206,59.38168915000267),(41.21084331372543,59.103467777099866),(29.786055068336193,52.146627480315004),(31.23682182965546,52.16517054781818),(41.69443223416517,58.85424941916091),(42.51048853740727,58.47703162291134),(32.59691566839227,52.22075341251539),(34.289476889931414,52.22075341251539),(43.02430176537451,58.07974369546071),(43.02430176537451,57.25537683364851),(35.468224883503325,52.2022335126388),(37.16078610504247,52.23926559241349),(43.02430176537451,56.26136189644947),(43.02430176537451,55.326904361850836),(38.33953409861437,52.16517054781818),(40.09254393520848,52.16517054781818),(44.4146199116388,55.3097062225408),(44.47506852669377,59.80998197603594),(39.72985224487867,59.931351417569715),(30.23941968124846,53.67744677450975),(30.20919537372098,54.63314259659509),(38.73245009647167,59.94649146557819),(37.2816833351524,59.97675082987618),(30.23941968124846,55.2752875586599),(30.33009260383092,56.19415599955667),(36.28428118674541,59.96162460231375),(34.863738732953635,59.97675082987618),(30.178971066193498,56.97640788219866),(30.178971066193498,57.91957806959033),(33.65476643185424,59.94649146557819),(32.32489690064491,59.94649146557819),(30.481214141468342,58.85424941916091),(30.571887064050795,59.99187015036608),(29.453587685533865,59.779570356240356)]], [[(24.367675781249993,61.45977057029751),(19.577636718749993,58.67693767258692),(19.577636718749993,57.492213666700735),(19.445800781249996,55.87531083569678),(19.445800781249996,54.085173420886775),(17.468261718749996,53.014783245859235),(20.017089843749993,51.563412328675895),(21.203613281249993,50.205033264943324),(26.125488281249993,50.40151532278236),(27.22412109374999,48.980216985374994),(32.80517578124999,49.525208341974405),(35.26611328124999,48.74894534343292),(36.93603515624999,49.66762782262194),(42.56103515625,48.77791275550183),(43.92333984374999,49.8096315635631),(47.17529296875,49.152969656170455),(49.28466796875,50.54136296522162),(48.05419921875,51.17934297928929),(51.39404296875,52.48278022207825),(50.64697265625,53.014783245859235),(52.88818359375,53.93021986394004),(51.65771484374999,54.29088164657006),(52.66845703125,55.825973254619015),(50.25146484375,56.145549500679095),(51.92138671875,57.914847767009206),(49.15283203125,58.17070248348605),(49.59228515625,60.086762746260064),(47.043457031249986,59.88893689676584),(43.57177734375,61.37567331572748),(42.64892578125,60.630101766266705),(36.89208984374999,62.000904713685856),(36.01318359374999,61.143235250840576),(31.398925781249993,62.02152819100766),(30.563964843749996,61.05828537037917),(26.872558593749993,61.71070595883174),(26.652832031249993,61.10078883158897),(24.367675781249993,61.45977057029751)], [(24.455566406249993,59.42272750081452),(21.203613281249993,58.49369382056807),(21.335449218749993,56.89700392127261),(21.599121093749993,55.92458580482949),(25.202636718749993,55.998380955359636),(28.850097656249993,57.06463027327854),(27.09228515625,57.844750992890994),(28.806152343749996,59.17592824927138),(26.257324218749993,59.17592824927138),(24.455566406249993,59.42272750081452)], [(35.13427734375,59.84481485969107),(31.970214843749993,58.97266715450152),(33.20068359374999,56.776808316568406),(36.67236328125,56.41390137600675),(39.08935546874999,57.25528054528888),(42.69287109374999,58.03137242177638),(40.89111328124999,59.26588062825809),(37.28759765625,58.722598828043374),(37.11181640624999,59.66774058164964),(35.13427734375,59.84481485969107)], [(29.157714843749993,55.75184939173528),(22.565917968749993,55.128649068488784),(22.565917968749993,53.54030739150019),(22.038574218749996,51.48138289610097),(26.257324218749993,51.42661449707484),(30.124511718749993,50.54136296522162),(32.18994140624999,51.17934297928929),(30.124511718749993,53.173119202640635),(35.09033203124999,53.173119202640635),(33.11279296875,54.085173420886775),(29.597167968749993,55.50374985927513),(29.157714843749993,55.75184939173528)], [(42.82470703125,56.58369172128337),(36.584472656249986,55.329144408405085),(37.99072265625,53.592504809039355),(34.95849609374999,51.48138289610097),(36.54052734374999,50.40151532278236),(39.66064453124999,50.289339253291786),(39.79248046875,52.13348804077148),(41.77001953125,50.68079714532166),(44.49462890624999,51.97134580885171),(47.30712890624999,52.509534770327264),(44.05517578125,53.54030739150019),(46.60400390625,53.696706475303245),(47.61474609375,55.40406982700608),(45.37353515625,55.40406982700608),(42.82470703125,56.58369172128337)]])) format TSV; diff --git a/tests/queries/0_stateless/01306_polygons_intersection.reference b/tests/queries/0_stateless/01306_polygons_intersection.reference index e48057e1eb3..cfec067cbcb 100644 --- a/tests/queries/0_stateless/01306_polygons_intersection.reference +++ b/tests/queries/0_stateless/01306_polygons_intersection.reference @@ -7,3 +7,12 @@ -------- [] [[[(4.366505261795747,50.843273415405),(4.3601419,50.8434626),(4.349456,50.8534879),(4.352580404040105,50.85655765067624),(4.367845,50.85235499999999),(4.366505261795747,50.843273415405)]]] +-------- MultiPolygon with Polygon +MULTIPOLYGON(((35.5408 58.9593,36.9725 59.0149,36.4989 58.7512,35.3712 58.8556,35.5408 58.9593)),((34.4816 56.8232,36.1999 57.0022,35.4083 56.5254,34.3867 56.7596,34.4816 56.8232)),((35.9179 57.7512,36.0848 57.855,37.1608 58.0478,36.5949 58.1673,37.8553 58.9075,38.5813 58.7446,37.4026 58.3187,38.0535 58.0542,36.4587 57.1544,35.7705 57.2554,37.0097 57.4998,35.9179 57.7512)),((36.8709 53.2765,37.4328 52.9552,36.5022 53.0008,36.8709 53.2765)),((36.1528 53.6763,35.3645 53.076,34.9611 53.9765,36.0472 54.7217,36.6985 54.0791,36.3552 53.8269,35.9216 53.8026,36.1528 53.6763)),((37.0035 54.2999,36.7074 54.6506,38.1601 55.1091,37.0035 54.2999)),((38.1688 56.0758,38.2186 56.0594,38.1319 56.0534,38.1688 56.0758)),((37.6238 55.7402,38.0373 55.6523,37.2824 55.5258,37.6238 55.7402)),((37.06 55.3843,37.7653 55.1891,36.151 54.791,37.06 55.3843)),((38.2312 56.9795,36.5334 56.6753,37.4489 57.1909,38.2699 57.0021,38.2312 56.9795)),((37.2281 56.3799,36.193 55.7319,35.3188 55.9582,35.6571 56.1619,36.7074 56.211,36.0233 56.3789,36.4446 56.6242,37.2281 56.3799)),((34.9952 58.6226,36.1498 58.553,36.0877 58.5174,34.6028 58.3749,34.9952 58.6226)),((34.3593 58.2189,35.4314 58.1349,35.1134 57.9454,33.7581 57.8255,34.3593 58.2189)),((33.6325 57.7419,34.6332 57.6538,34.2274 57.4023,33.1712 57.337,34.0208 57.2724,33.5602 56.9781,32.9596 56.9434,33.3418 56.8364,31.7782 55.7778,31.5088 55.9411,31.6069 56.3194,33.6325 57.7419)),((36.403 58.0507,36.4354 58.0478,36.3932 58.0447,36.403 58.0507)),((35.613 57.5595,36.1936 57.4998,35.4682 57.4674,35.613 57.5595)),((35.0338 57.1875,36.0727 57.0915,34.8098 57.0409,35.0338 57.1875)),((34.1885 56.6259,35.2273 56.414,35.0485 56.303,34.5917 56.2949,33.7222 56.3063,34.1885 56.6259)),((33.5244 56.1686,34.4996 55.9565,34.2598 55.8023,33.1204 55.8832,33.5244 56.1686)),((32.9547 55.7645,33.5036 55.3785,33.6125 55.3778,31.8748 54.1736,31.4182 54.4227,31.7439 54.8677,32.9547 55.7645)),((34.7279 53.8116,34.7731 53.7847,34.7731 52.9188,33.4048 52.8423,34.7279 53.8116)),((34.7231 54.7576,32.5275 53.1741,32.0831 53.408,32.476 53.8383,32.2523 53.964,34.3709 55.3709,35.0149 55.3613,34.2593 54.9642,34.7231 54.7576)),((34.9706 54.9262,34.8335 55.0162,35.2275 55.0993,34.9706 54.9262)),((35.7505 55.4454,35.1358 55.5327,35.9817 55.5958,35.7505 55.4454)),((35.0954 55.822,35.6798 55.6863,34.9721 55.7463,35.0954 55.822)),((34.7331 56.1049,34.7126 56.11,34.744 56.1118,34.7331 56.1049)),((40.2143 54.467,38.5511 53.2922,38.3395 53.2817,38.4609 53.226,38.0214 52.8989,37.8559 52.9188,37.135 53.4711,39.8151 55.3187,39.8205 55.2753,40.3948 55.2408,40.3948 54.8773,39.5485 54.8773,39.5485 54.5631,40.2143 54.467)),((40.5716 55.8007,40.5761 55.7884,40.5504 55.7875,40.5716 55.8007)),((40.4543 56.5923,40.2529 56.4682,39.7903 56.4121,39.8102 56.1914,38.2609 55.1775,37.7955 55.3956,38.4907 55.5327,38.1884 55.8564,38.944 56.0594,38.4339 56.2361,39.7863 57.025,39.7903 56.9929,40.3343 56.9599,40.4543 56.5923)),((40.1389 58.048,38.4915 57.1308,38.2186 57.2717,38.7325 57.4835,38.3737 57.6908,39.6392 58.3427,39.6392 58.0478,40.1389 58.048)),((37.5054 56.5484,37.463 56.5623,37.565 56.5843,37.5054 56.5484)),((38.0744 57.5312,38.128 57.516,37.9669 57.4734,38.0744 57.5312)),((40.4136 58.7241,40.3343 58.3821,39.7184 58.3823,40.4136 58.7241)),((39.8163 58.9766,39.4085 58.7696,38.5209 59.119,39.8163 58.9766)),((38.432 58.2584,38.3698 58.2869,38.7465 58.4255,38.432 58.2584)),((32.2175 58.3664,32.5691 58.5924,33.4734 58.8542,34.7428 59.5659,33.8361 58.6819,34.0496 58.6717,31.6514 57.1258,31.5088 57.4998,32.1738 58.0318,32.2175 58.3664)),((39.9942 53.358,40.0019 53.354,39.9877 53.3534,39.9942 53.358)),((39.2704 52.8471,39.5787 52.6996,39.1456 52.7573,39.2704 52.8471))) +MULTIPOLYGON(((35.677 58.9587,37.1527 59.0192,36.6691 58.7302,35.4917 58.8391,35.677 58.9587)),((34.8711 56.8603,36.7103 57.0477,35.7316 56.4418,34.6299 56.7008,34.8711 56.8603)),((36.1527 57.6965,36.5188 57.9318,37.1607 58.0477,36.8134 58.1201,38.0059 58.8722,38.5812 58.7445,37.4025 58.3186,38.2411 57.9759,38.0568 57.8654,37.4327 57.7102,37.6831 57.6403,36.7988 57.102,35.7704 57.2553,37.0096 57.4997,36.1527 57.6965)),((36.9376 53.2346,37.4327 52.9551,36.5898 52.9925,36.9376 53.2346)),((36.2567 53.616,35.4415 53.0432,35.3775 53.0461,35.0208 53.8426,36.1761 54.5942,36.7891 53.9859,36.575 53.8375,35.9215 53.8025,36.2567 53.616)),((37.0903 54.1938,36.7073 54.6505,38.3092 55.1545,37.7954 55.3955,38.4906 55.5326,38.1883 55.8563,38.9439 56.0593,38.6515 56.1591,39.9915 56.9806,40.3342 56.9598,40.4854 56.4956,39.7902 56.412,39.8204 56.0762,39.9194 56.0955,37.0903 54.1938)),((37.8883 55.6832,38.0372 55.6522,37.7656 55.6061,37.8883 55.6832)),((37.3054 55.3158,37.7652 55.189,36.6884 54.9232,37.3054 55.3158)),((37.4799 56.2952,36.9557 55.9643,36.6469 55.941,36.8521 55.8985,36.4727 55.6567,35.6467 55.87,36.1651 56.1834,36.7073 56.2109,36.3512 56.2953,36.7422 56.5294,37.4799 56.2952)),((37.1816 56.7907,37.7436 57.1223,38.3092 56.9928,37.1816 56.7907)),((35.1359 58.6084,36.3446 58.5401,34.7969 58.3872,35.1359 58.6084)),((34.5125 58.2005,35.6455 58.1105,35.3989 57.9596,33.9633 57.8372,34.5125 58.2005)),((33.7916 57.7229,34.8546 57.6243,34.5079 57.409,33.2199 57.3396,33.7916 57.7229)),((33.211 57.3336,34.2469 57.2461,33.8307 56.9848,32.9595 56.9433,33.5159 56.786,31.863 55.7247,31.5087 55.941,31.581 56.2183,33.211 57.3336)),((35.8918 57.5278,36.1935 57.4997,35.8173 57.4795,35.8918 57.5278)),((35.3199 57.1553,36.0726 57.0914,35.1561 57.0479,35.3199 57.1553)),((34.4368 56.5727,35.5695 56.3405,35.5188 56.3087,34.5916 56.2948,34.0285 56.3002,34.4368 56.5727)),((33.7487 56.1124,34.8294 55.8745,34.6546 55.7636,33.3749 55.86,33.7487 56.1124)),((33.0915 55.6676,33.5035 55.3784,34.0415 55.3722,32.0505 54.0742,31.4181 54.4226,31.5514 54.605,33.0915 55.6676)),((34.773 53.6795,34.773 52.9187,33.5229 52.8475,34.773 53.6795)),((34.9347 54.6613,34.374 54.2922,34.108 54.1756,34.1558 54.1477,32.6253 53.1196,32.083 53.4079,32.4759 53.8382,32.4267 53.8656,34.8176 55.3634,35.0148 55.3612,34.2592 54.9641,34.9347 54.6613)),((35.153 54.8041,34.8334 55.0161,35.777 55.2096,35.153 54.8041)),((36.0682 55.3974,35.1357 55.5326,36.4239 55.6256,36.0682 55.3974)),((35.4341 55.7407,35.6797 55.6862,35.3816 55.7088,35.4341 55.7407)),((35.0606 56.0207,34.7125 56.1099,35.245 56.1369,35.0606 56.0207)),((40.3151 54.4517,38.6455 53.2949,38.3394 53.2816,38.512 53.2011,38.0733 52.8911,37.8558 52.9187,37.2027 53.4183,39.9389 55.2681,40.3947 55.2407,40.3947 54.8772,39.5484 54.8772,39.5484 54.563,40.3151 54.4517)),((40.2513 57.9982,38.6704 57.0364,38.2185 57.2716,38.7324 57.4834,38.5465 57.5909,39.6391 58.219,39.6391 58.0477,40.2436 58.0477,40.2513 57.9982)),((37.7506 56.465,37.4629 56.5622,38.1298 56.7016,37.7506 56.465)),((40.3959 58.6476,40.3342 58.382,39.9258 58.382,40.3959 58.6476)),((39.9119 58.9623,39.5108 58.7281,38.5208 59.1189,39.9119 58.9623)),((38.591 58.1848,38.3697 58.2868,39.3854 58.6545,38.591 58.1848)),((32.2099 58.3086,32.7181 58.6343,33.4733 58.8541,34.7427 59.5658,33.836 58.6818,34.1607 58.6635,31.6694 57.0765,31.5087 57.4997,32.1737 58.0317,32.2099 58.3086)),((39.2985 52.8318,39.5786 52.6995,39.1804 52.7502,39.2985 52.8318))) +-------- MultiPolygon with Polygon with Holes +MULTIPOLYGON(((33.1079 56.9523,32.9596 56.9434,33.1392 56.8934,33.2007 56.7768,33.7182 56.7292,33.8361 56.6953,35.71 56.3117,34.5917 56.2949,32.8387 56.3117,35.6798 55.6863,32.748 55.9072,33.5036 55.3785,35.0149 55.3613,34.2593 54.9642,35.0753 54.5981,34.1081 54.1757,34.7731 53.7847,34.7731 53.3243,33.1128 54.0852,31.627 54.7093,31.8413 54.9989,32.204 55.5156,31.5088 55.9411,31.7506 56.8609,31.5088 57.4998,32.1738 58.0318,32.2342 58.4928,32.25 58.4976,33.1079 56.9523)),((35.1489 56.5859,36.6724 56.4139,36.8799 56.4895,38.2186 56.0594,36.647 55.9411,38.0262 55.6546,37.9482 55.6376,36.8283 55.4471,36.9508 55.414,36.5845 55.3291,36.8822 54.975,36.0123 54.7554,36.919 53.8561,35.9216 53.8026,37.2165 53.0798,37.0604 52.9744,35.3776 53.0462,34.894 54.1226,35.6193 54.4929,34.8335 55.0162,36.4354 55.3441,35.1358 55.5327,36.5563 55.6352,34.7126 56.11,36.7074 56.211,35.1489 56.5859)),((37.2327 59.0233,37.3119 59.0258,38.0944 58.8545,37.2876 58.7226,37.2327 59.0233)),((37.4471 53.2343,36.9794 53.5878,37.3119 53.9273,36.7074 54.6506,37.0572 54.7635,37.9907 53.5925,37.4471 53.2343)),((34.7731 53.1793,34.7731 52.9188,33.1712 52.8276,32.4808 53.1989,34.7731 53.1793)),((40.4412 56.1511,38.3184 55.7179,38.1884 55.8564,38.944 56.0594,37.463 56.5623,38.9742 56.8774,38.5798 57.0849,39.0894 57.2553,39.7379 57.4051,39.7903 56.9929,40.3343 56.9599,40.4855 56.4957,39.7903 56.4121,39.8205 56.0763,40.425 56.1942,40.4412 56.1511)),((38.3092 56.9929,38.3093 56.9929,38.309 56.9928,38.3092 56.9929)),((40.3237 57.5365,40.3343 57.4673,40.0149 57.4677,40.3237 57.5365)),((39.2792 59.0373,38.8838 58.9777,38.5209 59.119,39.2792 59.0373))) +MULTIPOLYGON(((33.106 56.9503,32.9595 56.9433,33.1372 56.8932,33.2006 56.7767,33.7508 56.7194,33.836 56.6952,35.7099 56.3116,34.5916 56.2948,32.8386 56.3116,35.6797 55.6862,32.7479 55.9071,33.5035 55.3784,35.0148 55.3612,34.2592 54.9641,35.0752 54.598,34.108 54.1756,34.773 53.7846,34.773 53.3207,33.1127 54.0851,31.6171 54.6946,31.8412 54.9988,32.2039 55.5155,31.5087 55.941,31.7505 56.8608,31.5087 57.4997,32.1737 58.0317,32.2341 58.4927,32.2439 58.4956,33.106 56.9503)),((35.1914 56.569,36.6723 56.4138,36.8783 56.4863,38.2185 56.0593,36.6469 55.941,38.0372 55.6522,36.8282 55.447,36.9706 55.4079,36.5844 55.329,36.8796 54.9705,36.0122 54.7553,36.9189 53.856,35.9215 53.8025,37.2279 53.0709,37.0823 52.9707,35.3775 53.0461,34.8939 54.1225,35.6192 54.4928,34.8334 55.0161,36.4353 55.344,35.1357 55.5326,36.5562 55.6351,34.7125 56.1099,36.7073 56.2109,35.1914 56.569)),((37.2322 59.0224,37.3118 59.0257,38.1151 58.848,37.2875 58.7225,37.2322 59.0224)),((37.4544 53.2264,36.9793 53.5877,37.3118 53.9272,36.7073 54.6505,37.0521 54.7595,37.9906 53.5924,37.4544 53.2264)),((34.773 53.173,34.773 52.9187,33.1711 52.8275,32.5252 53.173,34.773 53.173)),((40.4556 56.1121,38.3463 55.6874,38.1883 55.8563,38.9439 56.0593,37.4629 56.5622,38.9741 56.8773,38.5847 57.0811,39.0893 57.2552,39.7388 57.3963,39.7902 56.9928,40.3342 56.9598,40.4854 56.4956,39.7902 56.412,39.8204 56.0762,40.4249 56.1941,40.4556 56.1121)),((40.3255 57.5233,40.3342 57.4672,40.066 57.4672,40.3255 57.5233)),((39.3164 59.0294,38.9056 58.9675,38.5208 59.1189,39.3164 59.0294))) +-------- Polygon with Polygon with Holes +MULTIPOLYGON(((32.6512 57.792,30.3301 56.1942,30.2394 55.2753,32.9378 57.2699,33.2007 56.7768,33.2446 56.7729,30.7705 55.0525,29.5972 55.5037,29.4171 55.606,29.4536 59.7796,30.5719 59.9919,30.4812 58.8542,32.3249 59.9465,33.6548 59.9465,30.179 57.9196,30.179 56.9764,32.2964 58.4175,32.6512 57.792)),((35.9475 59.7758,35.1343 59.8448,34.2247 59.6064,34.8637 59.9768,36.2843 59.9616,35.9475 59.7758)),((36.7912 59.6986,37.2817 59.9768,38.7325 59.9465,37.2102 59.1452,37.1118 59.6677,36.7912 59.6986)),((34.2635 56.6767,35.4536 56.5531,32.2591 54.4483,31.5682 54.7333,34.2635 56.6767)),((36.1815 56.4715,36.6724 56.4139,38.1759 56.9472,33.5144 53.9057,33.1128 54.0852,32.6907 54.2663,36.1815 56.4715)),((33.8733 53.1922,35.0903 53.1731,34.3351 53.53,36.7219 55.1665,37.2766 54.4948,34.2895 52.2208,32.5969 52.2208,33.8733 53.1922)),((31.1968 52.1649,30.5785 52.7531,31.1682 53.1903,32.5603 53.1989,31.2368 52.1652,31.1968 52.1649)),((30.3098 53.0028,30.1245 53.1731,30.5408 53.1811,30.3098 53.0028)),((37.6322 58.7797,39.7299 59.9314,44.4751 59.81,44.4212 55.8594,42.8247 56.5837,41.4519 56.3413,43.0243 57.2554,43.0243 58.0797,39.4328 55.9511,37.1934 55.4694,40.9691 57.677,42.6929 58.0314,42.2498 58.3455,42.5105 58.477,41.6944 58.8542,41.5887 58.8012,41.168 59.0834,41.2108 59.1035,40.9299 59.2404,40.8911 59.2659,40.8804 59.2644,40.6366 59.3817,40.2079 59.1718,37.6322 58.7797)),((35.9681 52.2157,35.4682 52.2022,37.7431 53.9104,37.9907 53.5925,35.9681 52.2157))) +MULTIPOLYGON(((32.6773 57.7269,30.33 56.1941,30.2393 55.2752,32.9811 57.1783,33.2006 56.7767,33.4587 56.7498,30.7975 55.0249,29.5971 55.5036,29.4184 55.6047,29.4535 59.7795,30.5718 59.9918,30.4811 58.8541,32.3248 59.9464,33.6547 59.9464,30.1789 57.9195,30.1789 56.9763,32.3127 58.3747,32.6773 57.7269)),((35.9663 59.7703,35.1342 59.8447,34.2648 59.6073,34.8636 59.9767,36.2842 59.9615,35.9663 59.7703)),((36.8295 59.693,37.2816 59.9767,38.7324 59.9464,37.2249 59.0621,37.1117 59.6676,36.8295 59.693)),((34.5347 56.6377,35.8287 56.5023,32.4672 54.3493,31.6568 54.6786,34.5347 56.6377)),((36.5672 56.4248,36.6723 56.4138,38.9459 57.2058,33.6753 53.8276,33.1127 54.0851,32.9056 54.17,36.5672 56.4248)),((34.0091 53.173,35.0902 53.173,34.4539 53.4687,36.8394 55.0196,37.3607 54.3795,34.2894 52.2207,32.5968 52.2207,34.0091 53.173)),((31.1809 52.1644,30.6237 52.6993,31.3501 53.173,32.7039 53.173,31.2367 52.1651,31.1809 52.1644)),((30.3667 52.9437,30.1244 53.173,30.7047 53.173,30.3667 52.9437)),((37.9197 58.8184,39.7298 59.9313,44.475 59.8099,44.4214 55.8488,42.8246 56.5836,41.577 56.336,43.0242 57.2553,43.0242 58.0796,39.7202 55.9646,37.6798 55.5522,41.2147 57.7149,42.6928 58.0313,42.2607 58.3313,42.5104 58.4769,41.6943 58.8541,41.5893 58.7925,41.1674 59.0793,41.2107 59.1034,40.9293 59.24,40.891 59.2658,40.8796 59.2641,40.6365 59.3816,40.276 59.1737,37.9197 58.8184)),((35.9939 52.2136,35.4681 52.2021,37.8015 53.8302,37.9906 53.5924,35.9939 52.2136))) diff --git a/tests/queries/0_stateless/01306_polygons_intersection.sql b/tests/queries/0_stateless/01306_polygons_intersection.sql index 0a0aab2233b..105ab978e8c 100644 --- a/tests/queries/0_stateless/01306_polygons_intersection.sql +++ b/tests/queries/0_stateless/01306_polygons_intersection.sql @@ -10,3 +10,15 @@ select polygonsIntersectionSpherical([[[(4.3613577, 50.8651821), (4.349556, 50.8 select '--------'; select polygonsIntersectionMercator([[[(4.346693, 50.858306), (4.367945, 50.852455), (4.366227, 50.840809), (4.344961, 50.833264), (4.338074, 50.848677), (4.346693, 50.858306)]]], [[[(25.0010, 136.9987), (17.7500, 142.5000), (11.3733, 142.5917)]]]); select polygonsIntersectionMercator([[[(4.3613577, 50.8651821), (4.349556, 50.8535879), (4.3602419, 50.8435626), (4.3830299, 50.8428851), (4.3904543, 50.8564867), (4.3613148, 50.8651279)]]], [[[(4.346693, 50.858306), (4.367945, 50.852455), (4.366227, 50.840809), (4.344961, 50.833264), (4.338074, 50.848677), (4.346693, 50.858306)]]]); + +select '-------- MultiPolygon with Polygon'; +select wkt(polygonsIntersectionSpherical([[(29.453587685533865,59.779570356240356),(29.393139070478895,52.276266797422124),(40.636581470703206,59.38168915000267),(41.21084331372543,59.103467777099866),(29.786055068336193,52.146627480315004),(31.23682182965546,52.16517054781818),(41.69443223416517,58.85424941916091),(42.51048853740727,58.47703162291134),(32.59691566839227,52.22075341251539),(34.289476889931414,52.22075341251539),(43.02430176537451,58.07974369546071),(43.02430176537451,57.25537683364851),(35.468224883503325,52.2022335126388),(37.16078610504247,52.23926559241349),(43.02430176537451,56.26136189644947),(43.02430176537451,55.326904361850836),(38.33953409861437,52.16517054781818),(40.09254393520848,52.16517054781818),(44.4146199116388,55.3097062225408),(44.47506852669377,59.80998197603594),(39.72985224487867,59.931351417569715),(30.23941968124846,53.67744677450975),(30.20919537372098,54.63314259659509),(38.73245009647167,59.94649146557819),(37.2816833351524,59.97675082987618),(30.23941968124846,55.2752875586599),(30.33009260383092,56.19415599955667),(36.28428118674541,59.96162460231375),(34.863738732953635,59.97675082987618),(30.178971066193498,56.97640788219866),(30.178971066193498,57.91957806959033),(33.65476643185424,59.94649146557819),(32.32489690064491,59.94649146557819),(30.481214141468342,58.85424941916091),(30.571887064050795,59.99187015036608),(29.453587685533865,59.779570356240356)]], [[[(33.473420586689336,58.85424941916091),(32.23422397806246,58.492830557036),(32.173775363007486,58.03176922751564),(31.508840597402823,57.499784781503735),(31.750635057622702,56.86092686957355),(31.508840597402823,55.941082594334574),(32.20399967053497,55.515591939372456),(31.84130798020516,54.998862226280465),(31.418167674820367,54.422670886434275),(32.47601843828233,53.83826377018255),(32.08310244042503,53.408048308050866),(33.171177511414484,52.82758702113742),(34.77306581037117,52.91880107773494),(34.77306581037117,53.784726518357985),(34.108131044766516,54.17574726780569),(35.07530888564602,54.59813930694554),(34.25925258240394,54.96417435716029),(35.01486027059106,55.361278263643584),(33.50364489421682,55.37845402950552),(32.7480372060297,55.90721384574556),(35.67979503619571,55.68634475630185),(32.83871012861215,56.311688992608396),(34.591719965206266,56.29492065473883),(35.7100193437232,56.311688992608396),(33.83611227701915,56.695333481003644),(32.95960735872209,56.9434497616887),(36.072711034053015,57.091531913901434),(33.171177511414484,57.33702717078384),(36.193608264162954,57.499784781503735),(33.23162612646945,57.77481561306047),(36.43540272438284,58.04776787540811),(33.62454212432676,58.27099811968307),(36.344729801800376,58.54018474404165),(33.83611227701915,58.68186423448108),(34.74284150284369,59.565911441555244),(33.473420586689336,58.85424941916091)]], [[(34.65216858026123,58.91672306881671),(37.19101041256995,58.68186423448108),(36.01226241899805,58.28688958537609),(37.16078610504247,58.04776787540811),(35.74024365125068,57.79092907387934),(37.009664567405046,57.499784781503735),(35.77046795877817,57.25537683364851),(36.979440259877556,57.07510745541089),(34.22902827487645,56.794777197297435),(36.7074214921302,56.210968525786996),(34.712617195316206,56.10998276812964),(36.55629995449277,55.63519693782703),(35.13575750070099,55.53270067649592),(36.43540272438284,55.34409504165558),(34.83351442542614,55.01619492319591),(35.61934642114075,54.49294870011772),(34.89396304048112,54.12264226523038),(35.37755196092087,53.046178687628185),(37.43280487278982,52.95523300597458),(35.92158949641559,53.80257986695776),(36.91899164482259,53.856094327816805),(36.01226241899805,54.75541714463799),(37.765272255592166,55.189110239786885),(36.828318722240134,55.44708256557195),(38.03729102333953,55.652253637168315),(36.64697287707522,55.941082594334574),(38.21863686850443,56.05939028508024),(36.37495410932787,56.64551287174558),(38.30930979108689,56.992876013526654),(37.16078610504247,57.25537683364851),(38.127963945921984,57.516020773674256),(37.43280487278982,57.710289827306724),(38.33953409861437,57.935626886818994),(37.40258056526235,58.31865112960426),(38.58132855883426,58.744648733419496),(37.31190764267989,59.02578062465136),(34.65216858026123,58.91672306881671)]], [[(38.52087994377928,59.11898412389468),(39.54850639971376,58.713270635642914),(38.369758406141855,58.28688958537609),(38.85334732658162,58.06375936407028),(38.33953409861437,57.710289827306724),(38.73245009647167,57.48354156434209),(38.21863686850443,57.271721400459285),(38.97424455669155,56.87744603722649),(37.463029180317314,56.5623320541159),(38.94402024916407,56.05939028508024),(38.18841256097694,55.856355210835915),(38.490655636251795,55.53270067649592),(37.795496563119656,55.39562234093384),(38.30930979108689,55.154587013355666),(36.7074214921302,54.65063295250911),(37.31190764267989,53.92734063371401),(36.979440259877556,53.58783775557231),(37.855945178174615,52.91880107773497),(39.57873070724124,52.69956490610895),(38.33953409861437,53.281741738901104),(40.00187101262603,53.35396273604752),(39.54850639971376,53.58783775557231),(40.24366547284591,53.58783775557231),(39.97164670509855,53.98069568468355),(40.60635716317572,54.03398248547225),(40.39478701048334,54.44025165268903),(39.54850639971376,54.56310590284329),(39.54850639971376,54.87732350170489),(40.39478701048334,54.87732350170489),(40.39478701048334,55.24083903654295),(39.82052516746112,55.2752875586599),(39.760076552406154,55.75443792473942),(40.57613285564824,55.78844000174894),(40.425011318010824,56.19415599955667),(39.82052516746112,56.07626182891758),(39.79030085993364,56.41214455508424),(40.48545993306579,56.495655446714636),(40.33433839542836,56.95993246553937),(39.79030085993364,56.992876013526654),(39.72985224487867,57.46729112028032),(40.33433839542836,57.46729112028032),(40.24366547284591,58.04776787540811),(39.63917932229622,58.04776787540811),(39.63917932229622,58.382088724871295),(40.33433839542836,58.382088724871295),(40.45523562553831,58.9011152358548),(38.52087994377928,59.11898412389468)]]])) format TSV; +select wkt(polygonsIntersectionMercator([[(29.453587685533865,59.779570356240356),(29.393139070478895,52.276266797422124),(40.636581470703206,59.38168915000267),(41.21084331372543,59.103467777099866),(29.786055068336193,52.146627480315004),(31.23682182965546,52.16517054781818),(41.69443223416517,58.85424941916091),(42.51048853740727,58.47703162291134),(32.59691566839227,52.22075341251539),(34.289476889931414,52.22075341251539),(43.02430176537451,58.07974369546071),(43.02430176537451,57.25537683364851),(35.468224883503325,52.2022335126388),(37.16078610504247,52.23926559241349),(43.02430176537451,56.26136189644947),(43.02430176537451,55.326904361850836),(38.33953409861437,52.16517054781818),(40.09254393520848,52.16517054781818),(44.4146199116388,55.3097062225408),(44.47506852669377,59.80998197603594),(39.72985224487867,59.931351417569715),(30.23941968124846,53.67744677450975),(30.20919537372098,54.63314259659509),(38.73245009647167,59.94649146557819),(37.2816833351524,59.97675082987618),(30.23941968124846,55.2752875586599),(30.33009260383092,56.19415599955667),(36.28428118674541,59.96162460231375),(34.863738732953635,59.97675082987618),(30.178971066193498,56.97640788219866),(30.178971066193498,57.91957806959033),(33.65476643185424,59.94649146557819),(32.32489690064491,59.94649146557819),(30.481214141468342,58.85424941916091),(30.571887064050795,59.99187015036608),(29.453587685533865,59.779570356240356)]], [[[(33.473420586689336,58.85424941916091),(32.23422397806246,58.492830557036),(32.173775363007486,58.03176922751564),(31.508840597402823,57.499784781503735),(31.750635057622702,56.86092686957355),(31.508840597402823,55.941082594334574),(32.20399967053497,55.515591939372456),(31.84130798020516,54.998862226280465),(31.418167674820367,54.422670886434275),(32.47601843828233,53.83826377018255),(32.08310244042503,53.408048308050866),(33.171177511414484,52.82758702113742),(34.77306581037117,52.91880107773494),(34.77306581037117,53.784726518357985),(34.108131044766516,54.17574726780569),(35.07530888564602,54.59813930694554),(34.25925258240394,54.96417435716029),(35.01486027059106,55.361278263643584),(33.50364489421682,55.37845402950552),(32.7480372060297,55.90721384574556),(35.67979503619571,55.68634475630185),(32.83871012861215,56.311688992608396),(34.591719965206266,56.29492065473883),(35.7100193437232,56.311688992608396),(33.83611227701915,56.695333481003644),(32.95960735872209,56.9434497616887),(36.072711034053015,57.091531913901434),(33.171177511414484,57.33702717078384),(36.193608264162954,57.499784781503735),(33.23162612646945,57.77481561306047),(36.43540272438284,58.04776787540811),(33.62454212432676,58.27099811968307),(36.344729801800376,58.54018474404165),(33.83611227701915,58.68186423448108),(34.74284150284369,59.565911441555244),(33.473420586689336,58.85424941916091)]], [[(34.65216858026123,58.91672306881671),(37.19101041256995,58.68186423448108),(36.01226241899805,58.28688958537609),(37.16078610504247,58.04776787540811),(35.74024365125068,57.79092907387934),(37.009664567405046,57.499784781503735),(35.77046795877817,57.25537683364851),(36.979440259877556,57.07510745541089),(34.22902827487645,56.794777197297435),(36.7074214921302,56.210968525786996),(34.712617195316206,56.10998276812964),(36.55629995449277,55.63519693782703),(35.13575750070099,55.53270067649592),(36.43540272438284,55.34409504165558),(34.83351442542614,55.01619492319591),(35.61934642114075,54.49294870011772),(34.89396304048112,54.12264226523038),(35.37755196092087,53.046178687628185),(37.43280487278982,52.95523300597458),(35.92158949641559,53.80257986695776),(36.91899164482259,53.856094327816805),(36.01226241899805,54.75541714463799),(37.765272255592166,55.189110239786885),(36.828318722240134,55.44708256557195),(38.03729102333953,55.652253637168315),(36.64697287707522,55.941082594334574),(38.21863686850443,56.05939028508024),(36.37495410932787,56.64551287174558),(38.30930979108689,56.992876013526654),(37.16078610504247,57.25537683364851),(38.127963945921984,57.516020773674256),(37.43280487278982,57.710289827306724),(38.33953409861437,57.935626886818994),(37.40258056526235,58.31865112960426),(38.58132855883426,58.744648733419496),(37.31190764267989,59.02578062465136),(34.65216858026123,58.91672306881671)]], [[(38.52087994377928,59.11898412389468),(39.54850639971376,58.713270635642914),(38.369758406141855,58.28688958537609),(38.85334732658162,58.06375936407028),(38.33953409861437,57.710289827306724),(38.73245009647167,57.48354156434209),(38.21863686850443,57.271721400459285),(38.97424455669155,56.87744603722649),(37.463029180317314,56.5623320541159),(38.94402024916407,56.05939028508024),(38.18841256097694,55.856355210835915),(38.490655636251795,55.53270067649592),(37.795496563119656,55.39562234093384),(38.30930979108689,55.154587013355666),(36.7074214921302,54.65063295250911),(37.31190764267989,53.92734063371401),(36.979440259877556,53.58783775557231),(37.855945178174615,52.91880107773497),(39.57873070724124,52.69956490610895),(38.33953409861437,53.281741738901104),(40.00187101262603,53.35396273604752),(39.54850639971376,53.58783775557231),(40.24366547284591,53.58783775557231),(39.97164670509855,53.98069568468355),(40.60635716317572,54.03398248547225),(40.39478701048334,54.44025165268903),(39.54850639971376,54.56310590284329),(39.54850639971376,54.87732350170489),(40.39478701048334,54.87732350170489),(40.39478701048334,55.24083903654295),(39.82052516746112,55.2752875586599),(39.760076552406154,55.75443792473942),(40.57613285564824,55.78844000174894),(40.425011318010824,56.19415599955667),(39.82052516746112,56.07626182891758),(39.79030085993364,56.41214455508424),(40.48545993306579,56.495655446714636),(40.33433839542836,56.95993246553937),(39.79030085993364,56.992876013526654),(39.72985224487867,57.46729112028032),(40.33433839542836,57.46729112028032),(40.24366547284591,58.04776787540811),(39.63917932229622,58.04776787540811),(39.63917932229622,58.382088724871295),(40.33433839542836,58.382088724871295),(40.45523562553831,58.9011152358548),(38.52087994377928,59.11898412389468)]]])) format TSV; + +select '-------- MultiPolygon with Polygon with Holes'; +select wkt(polygonsIntersectionSpherical([[[(33.473420586689336,58.85424941916091),(32.23422397806246,58.492830557036),(32.173775363007486,58.03176922751564),(31.508840597402823,57.499784781503735),(31.750635057622702,56.86092686957355),(31.508840597402823,55.941082594334574),(32.20399967053497,55.515591939372456),(31.84130798020516,54.998862226280465),(31.418167674820367,54.422670886434275),(32.47601843828233,53.83826377018255),(32.08310244042503,53.408048308050866),(33.171177511414484,52.82758702113742),(34.77306581037117,52.91880107773494),(34.77306581037117,53.784726518357985),(34.108131044766516,54.17574726780569),(35.07530888564602,54.59813930694554),(34.25925258240394,54.96417435716029),(35.01486027059106,55.361278263643584),(33.50364489421682,55.37845402950552),(32.7480372060297,55.90721384574556),(35.67979503619571,55.68634475630185),(32.83871012861215,56.311688992608396),(34.591719965206266,56.29492065473883),(35.7100193437232,56.311688992608396),(33.83611227701915,56.695333481003644),(32.95960735872209,56.9434497616887),(36.072711034053015,57.091531913901434),(33.171177511414484,57.33702717078384),(36.193608264162954,57.499784781503735),(33.23162612646945,57.77481561306047),(36.43540272438284,58.04776787540811),(33.62454212432676,58.27099811968307),(36.344729801800376,58.54018474404165),(33.83611227701915,58.68186423448108),(34.74284150284369,59.565911441555244),(33.473420586689336,58.85424941916091)]], [[(34.65216858026123,58.91672306881671),(37.19101041256995,58.68186423448108),(36.01226241899805,58.28688958537609),(37.16078610504247,58.04776787540811),(35.74024365125068,57.79092907387934),(37.009664567405046,57.499784781503735),(35.77046795877817,57.25537683364851),(36.979440259877556,57.07510745541089),(34.22902827487645,56.794777197297435),(36.7074214921302,56.210968525786996),(34.712617195316206,56.10998276812964),(36.55629995449277,55.63519693782703),(35.13575750070099,55.53270067649592),(36.43540272438284,55.34409504165558),(34.83351442542614,55.01619492319591),(35.61934642114075,54.49294870011772),(34.89396304048112,54.12264226523038),(35.37755196092087,53.046178687628185),(37.43280487278982,52.95523300597458),(35.92158949641559,53.80257986695776),(36.91899164482259,53.856094327816805),(36.01226241899805,54.75541714463799),(37.765272255592166,55.189110239786885),(36.828318722240134,55.44708256557195),(38.03729102333953,55.652253637168315),(36.64697287707522,55.941082594334574),(38.21863686850443,56.05939028508024),(36.37495410932787,56.64551287174558),(38.30930979108689,56.992876013526654),(37.16078610504247,57.25537683364851),(38.127963945921984,57.516020773674256),(37.43280487278982,57.710289827306724),(38.33953409861437,57.935626886818994),(37.40258056526235,58.31865112960426),(38.58132855883426,58.744648733419496),(37.31190764267989,59.02578062465136),(34.65216858026123,58.91672306881671)]], [[(38.52087994377928,59.11898412389468),(39.54850639971376,58.713270635642914),(38.369758406141855,58.28688958537609),(38.85334732658162,58.06375936407028),(38.33953409861437,57.710289827306724),(38.73245009647167,57.48354156434209),(38.21863686850443,57.271721400459285),(38.97424455669155,56.87744603722649),(37.463029180317314,56.5623320541159),(38.94402024916407,56.05939028508024),(38.18841256097694,55.856355210835915),(38.490655636251795,55.53270067649592),(37.795496563119656,55.39562234093384),(38.30930979108689,55.154587013355666),(36.7074214921302,54.65063295250911),(37.31190764267989,53.92734063371401),(36.979440259877556,53.58783775557231),(37.855945178174615,52.91880107773497),(39.57873070724124,52.69956490610895),(38.33953409861437,53.281741738901104),(40.00187101262603,53.35396273604752),(39.54850639971376,53.58783775557231),(40.24366547284591,53.58783775557231),(39.97164670509855,53.98069568468355),(40.60635716317572,54.03398248547225),(40.39478701048334,54.44025165268903),(39.54850639971376,54.56310590284329),(39.54850639971376,54.87732350170489),(40.39478701048334,54.87732350170489),(40.39478701048334,55.24083903654295),(39.82052516746112,55.2752875586599),(39.760076552406154,55.75443792473942),(40.57613285564824,55.78844000174894),(40.425011318010824,56.19415599955667),(39.82052516746112,56.07626182891758),(39.79030085993364,56.41214455508424),(40.48545993306579,56.495655446714636),(40.33433839542836,56.95993246553937),(39.79030085993364,56.992876013526654),(39.72985224487867,57.46729112028032),(40.33433839542836,57.46729112028032),(40.24366547284591,58.04776787540811),(39.63917932229622,58.04776787540811),(39.63917932229622,58.382088724871295),(40.33433839542836,58.382088724871295),(40.45523562553831,58.9011152358548),(38.52087994377928,59.11898412389468)]]], [[(24.367675781249993,61.45977057029751),(19.577636718749993,58.67693767258692),(19.577636718749993,57.492213666700735),(19.445800781249996,55.87531083569678),(19.445800781249996,54.085173420886775),(17.468261718749996,53.014783245859235),(20.017089843749993,51.563412328675895),(21.203613281249993,50.205033264943324),(26.125488281249993,50.40151532278236),(27.22412109374999,48.980216985374994),(32.80517578124999,49.525208341974405),(35.26611328124999,48.74894534343292),(36.93603515624999,49.66762782262194),(42.56103515625,48.77791275550183),(43.92333984374999,49.8096315635631),(47.17529296875,49.152969656170455),(49.28466796875,50.54136296522162),(48.05419921875,51.17934297928929),(51.39404296875,52.48278022207825),(50.64697265625,53.014783245859235),(52.88818359375,53.93021986394004),(51.65771484374999,54.29088164657006),(52.66845703125,55.825973254619015),(50.25146484375,56.145549500679095),(51.92138671875,57.914847767009206),(49.15283203125,58.17070248348605),(49.59228515625,60.086762746260064),(47.043457031249986,59.88893689676584),(43.57177734375,61.37567331572748),(42.64892578125,60.630101766266705),(36.89208984374999,62.000904713685856),(36.01318359374999,61.143235250840576),(31.398925781249993,62.02152819100766),(30.563964843749996,61.05828537037917),(26.872558593749993,61.71070595883174),(26.652832031249993,61.10078883158897),(24.367675781249993,61.45977057029751)], [(24.455566406249993,59.42272750081452),(21.203613281249993,58.49369382056807),(21.335449218749993,56.89700392127261),(21.599121093749993,55.92458580482949),(25.202636718749993,55.998380955359636),(28.850097656249993,57.06463027327854),(27.09228515625,57.844750992890994),(28.806152343749996,59.17592824927138),(26.257324218749993,59.17592824927138),(24.455566406249993,59.42272750081452)], [(35.13427734375,59.84481485969107),(31.970214843749993,58.97266715450152),(33.20068359374999,56.776808316568406),(36.67236328125,56.41390137600675),(39.08935546874999,57.25528054528888),(42.69287109374999,58.03137242177638),(40.89111328124999,59.26588062825809),(37.28759765625,58.722598828043374),(37.11181640624999,59.66774058164964),(35.13427734375,59.84481485969107)], [(29.157714843749993,55.75184939173528),(22.565917968749993,55.128649068488784),(22.565917968749993,53.54030739150019),(22.038574218749996,51.48138289610097),(26.257324218749993,51.42661449707484),(30.124511718749993,50.54136296522162),(32.18994140624999,51.17934297928929),(30.124511718749993,53.173119202640635),(35.09033203124999,53.173119202640635),(33.11279296875,54.085173420886775),(29.597167968749993,55.50374985927513),(29.157714843749993,55.75184939173528)], [(42.82470703125,56.58369172128337),(36.584472656249986,55.329144408405085),(37.99072265625,53.592504809039355),(34.95849609374999,51.48138289610097),(36.54052734374999,50.40151532278236),(39.66064453124999,50.289339253291786),(39.79248046875,52.13348804077148),(41.77001953125,50.68079714532166),(44.49462890624999,51.97134580885171),(47.30712890624999,52.509534770327264),(44.05517578125,53.54030739150019),(46.60400390625,53.696706475303245),(47.61474609375,55.40406982700608),(45.37353515625,55.40406982700608),(42.82470703125,56.58369172128337)]])) format TSV; +select wkt(polygonsIntersectionMercator([[[(33.473420586689336,58.85424941916091),(32.23422397806246,58.492830557036),(32.173775363007486,58.03176922751564),(31.508840597402823,57.499784781503735),(31.750635057622702,56.86092686957355),(31.508840597402823,55.941082594334574),(32.20399967053497,55.515591939372456),(31.84130798020516,54.998862226280465),(31.418167674820367,54.422670886434275),(32.47601843828233,53.83826377018255),(32.08310244042503,53.408048308050866),(33.171177511414484,52.82758702113742),(34.77306581037117,52.91880107773494),(34.77306581037117,53.784726518357985),(34.108131044766516,54.17574726780569),(35.07530888564602,54.59813930694554),(34.25925258240394,54.96417435716029),(35.01486027059106,55.361278263643584),(33.50364489421682,55.37845402950552),(32.7480372060297,55.90721384574556),(35.67979503619571,55.68634475630185),(32.83871012861215,56.311688992608396),(34.591719965206266,56.29492065473883),(35.7100193437232,56.311688992608396),(33.83611227701915,56.695333481003644),(32.95960735872209,56.9434497616887),(36.072711034053015,57.091531913901434),(33.171177511414484,57.33702717078384),(36.193608264162954,57.499784781503735),(33.23162612646945,57.77481561306047),(36.43540272438284,58.04776787540811),(33.62454212432676,58.27099811968307),(36.344729801800376,58.54018474404165),(33.83611227701915,58.68186423448108),(34.74284150284369,59.565911441555244),(33.473420586689336,58.85424941916091)]], [[(34.65216858026123,58.91672306881671),(37.19101041256995,58.68186423448108),(36.01226241899805,58.28688958537609),(37.16078610504247,58.04776787540811),(35.74024365125068,57.79092907387934),(37.009664567405046,57.499784781503735),(35.77046795877817,57.25537683364851),(36.979440259877556,57.07510745541089),(34.22902827487645,56.794777197297435),(36.7074214921302,56.210968525786996),(34.712617195316206,56.10998276812964),(36.55629995449277,55.63519693782703),(35.13575750070099,55.53270067649592),(36.43540272438284,55.34409504165558),(34.83351442542614,55.01619492319591),(35.61934642114075,54.49294870011772),(34.89396304048112,54.12264226523038),(35.37755196092087,53.046178687628185),(37.43280487278982,52.95523300597458),(35.92158949641559,53.80257986695776),(36.91899164482259,53.856094327816805),(36.01226241899805,54.75541714463799),(37.765272255592166,55.189110239786885),(36.828318722240134,55.44708256557195),(38.03729102333953,55.652253637168315),(36.64697287707522,55.941082594334574),(38.21863686850443,56.05939028508024),(36.37495410932787,56.64551287174558),(38.30930979108689,56.992876013526654),(37.16078610504247,57.25537683364851),(38.127963945921984,57.516020773674256),(37.43280487278982,57.710289827306724),(38.33953409861437,57.935626886818994),(37.40258056526235,58.31865112960426),(38.58132855883426,58.744648733419496),(37.31190764267989,59.02578062465136),(34.65216858026123,58.91672306881671)]], [[(38.52087994377928,59.11898412389468),(39.54850639971376,58.713270635642914),(38.369758406141855,58.28688958537609),(38.85334732658162,58.06375936407028),(38.33953409861437,57.710289827306724),(38.73245009647167,57.48354156434209),(38.21863686850443,57.271721400459285),(38.97424455669155,56.87744603722649),(37.463029180317314,56.5623320541159),(38.94402024916407,56.05939028508024),(38.18841256097694,55.856355210835915),(38.490655636251795,55.53270067649592),(37.795496563119656,55.39562234093384),(38.30930979108689,55.154587013355666),(36.7074214921302,54.65063295250911),(37.31190764267989,53.92734063371401),(36.979440259877556,53.58783775557231),(37.855945178174615,52.91880107773497),(39.57873070724124,52.69956490610895),(38.33953409861437,53.281741738901104),(40.00187101262603,53.35396273604752),(39.54850639971376,53.58783775557231),(40.24366547284591,53.58783775557231),(39.97164670509855,53.98069568468355),(40.60635716317572,54.03398248547225),(40.39478701048334,54.44025165268903),(39.54850639971376,54.56310590284329),(39.54850639971376,54.87732350170489),(40.39478701048334,54.87732350170489),(40.39478701048334,55.24083903654295),(39.82052516746112,55.2752875586599),(39.760076552406154,55.75443792473942),(40.57613285564824,55.78844000174894),(40.425011318010824,56.19415599955667),(39.82052516746112,56.07626182891758),(39.79030085993364,56.41214455508424),(40.48545993306579,56.495655446714636),(40.33433839542836,56.95993246553937),(39.79030085993364,56.992876013526654),(39.72985224487867,57.46729112028032),(40.33433839542836,57.46729112028032),(40.24366547284591,58.04776787540811),(39.63917932229622,58.04776787540811),(39.63917932229622,58.382088724871295),(40.33433839542836,58.382088724871295),(40.45523562553831,58.9011152358548),(38.52087994377928,59.11898412389468)]]], [[(24.367675781249993,61.45977057029751),(19.577636718749993,58.67693767258692),(19.577636718749993,57.492213666700735),(19.445800781249996,55.87531083569678),(19.445800781249996,54.085173420886775),(17.468261718749996,53.014783245859235),(20.017089843749993,51.563412328675895),(21.203613281249993,50.205033264943324),(26.125488281249993,50.40151532278236),(27.22412109374999,48.980216985374994),(32.80517578124999,49.525208341974405),(35.26611328124999,48.74894534343292),(36.93603515624999,49.66762782262194),(42.56103515625,48.77791275550183),(43.92333984374999,49.8096315635631),(47.17529296875,49.152969656170455),(49.28466796875,50.54136296522162),(48.05419921875,51.17934297928929),(51.39404296875,52.48278022207825),(50.64697265625,53.014783245859235),(52.88818359375,53.93021986394004),(51.65771484374999,54.29088164657006),(52.66845703125,55.825973254619015),(50.25146484375,56.145549500679095),(51.92138671875,57.914847767009206),(49.15283203125,58.17070248348605),(49.59228515625,60.086762746260064),(47.043457031249986,59.88893689676584),(43.57177734375,61.37567331572748),(42.64892578125,60.630101766266705),(36.89208984374999,62.000904713685856),(36.01318359374999,61.143235250840576),(31.398925781249993,62.02152819100766),(30.563964843749996,61.05828537037917),(26.872558593749993,61.71070595883174),(26.652832031249993,61.10078883158897),(24.367675781249993,61.45977057029751)], [(24.455566406249993,59.42272750081452),(21.203613281249993,58.49369382056807),(21.335449218749993,56.89700392127261),(21.599121093749993,55.92458580482949),(25.202636718749993,55.998380955359636),(28.850097656249993,57.06463027327854),(27.09228515625,57.844750992890994),(28.806152343749996,59.17592824927138),(26.257324218749993,59.17592824927138),(24.455566406249993,59.42272750081452)], [(35.13427734375,59.84481485969107),(31.970214843749993,58.97266715450152),(33.20068359374999,56.776808316568406),(36.67236328125,56.41390137600675),(39.08935546874999,57.25528054528888),(42.69287109374999,58.03137242177638),(40.89111328124999,59.26588062825809),(37.28759765625,58.722598828043374),(37.11181640624999,59.66774058164964),(35.13427734375,59.84481485969107)], [(29.157714843749993,55.75184939173528),(22.565917968749993,55.128649068488784),(22.565917968749993,53.54030739150019),(22.038574218749996,51.48138289610097),(26.257324218749993,51.42661449707484),(30.124511718749993,50.54136296522162),(32.18994140624999,51.17934297928929),(30.124511718749993,53.173119202640635),(35.09033203124999,53.173119202640635),(33.11279296875,54.085173420886775),(29.597167968749993,55.50374985927513),(29.157714843749993,55.75184939173528)], [(42.82470703125,56.58369172128337),(36.584472656249986,55.329144408405085),(37.99072265625,53.592504809039355),(34.95849609374999,51.48138289610097),(36.54052734374999,50.40151532278236),(39.66064453124999,50.289339253291786),(39.79248046875,52.13348804077148),(41.77001953125,50.68079714532166),(44.49462890624999,51.97134580885171),(47.30712890624999,52.509534770327264),(44.05517578125,53.54030739150019),(46.60400390625,53.696706475303245),(47.61474609375,55.40406982700608),(45.37353515625,55.40406982700608),(42.82470703125,56.58369172128337)]])) format TSV; + +select '-------- Polygon with Polygon with Holes'; +select wkt(polygonsIntersectionSpherical([[(29.453587685533865,59.779570356240356),(29.393139070478895,52.276266797422124),(40.636581470703206,59.38168915000267),(41.21084331372543,59.103467777099866),(29.786055068336193,52.146627480315004),(31.23682182965546,52.16517054781818),(41.69443223416517,58.85424941916091),(42.51048853740727,58.47703162291134),(32.59691566839227,52.22075341251539),(34.289476889931414,52.22075341251539),(43.02430176537451,58.07974369546071),(43.02430176537451,57.25537683364851),(35.468224883503325,52.2022335126388),(37.16078610504247,52.23926559241349),(43.02430176537451,56.26136189644947),(43.02430176537451,55.326904361850836),(38.33953409861437,52.16517054781818),(40.09254393520848,52.16517054781818),(44.4146199116388,55.3097062225408),(44.47506852669377,59.80998197603594),(39.72985224487867,59.931351417569715),(30.23941968124846,53.67744677450975),(30.20919537372098,54.63314259659509),(38.73245009647167,59.94649146557819),(37.2816833351524,59.97675082987618),(30.23941968124846,55.2752875586599),(30.33009260383092,56.19415599955667),(36.28428118674541,59.96162460231375),(34.863738732953635,59.97675082987618),(30.178971066193498,56.97640788219866),(30.178971066193498,57.91957806959033),(33.65476643185424,59.94649146557819),(32.32489690064491,59.94649146557819),(30.481214141468342,58.85424941916091),(30.571887064050795,59.99187015036608),(29.453587685533865,59.779570356240356)]], [[(24.367675781249993,61.45977057029751),(19.577636718749993,58.67693767258692),(19.577636718749993,57.492213666700735),(19.445800781249996,55.87531083569678),(19.445800781249996,54.085173420886775),(17.468261718749996,53.014783245859235),(20.017089843749993,51.563412328675895),(21.203613281249993,50.205033264943324),(26.125488281249993,50.40151532278236),(27.22412109374999,48.980216985374994),(32.80517578124999,49.525208341974405),(35.26611328124999,48.74894534343292),(36.93603515624999,49.66762782262194),(42.56103515625,48.77791275550183),(43.92333984374999,49.8096315635631),(47.17529296875,49.152969656170455),(49.28466796875,50.54136296522162),(48.05419921875,51.17934297928929),(51.39404296875,52.48278022207825),(50.64697265625,53.014783245859235),(52.88818359375,53.93021986394004),(51.65771484374999,54.29088164657006),(52.66845703125,55.825973254619015),(50.25146484375,56.145549500679095),(51.92138671875,57.914847767009206),(49.15283203125,58.17070248348605),(49.59228515625,60.086762746260064),(47.043457031249986,59.88893689676584),(43.57177734375,61.37567331572748),(42.64892578125,60.630101766266705),(36.89208984374999,62.000904713685856),(36.01318359374999,61.143235250840576),(31.398925781249993,62.02152819100766),(30.563964843749996,61.05828537037917),(26.872558593749993,61.71070595883174),(26.652832031249993,61.10078883158897),(24.367675781249993,61.45977057029751)], [(24.455566406249993,59.42272750081452),(21.203613281249993,58.49369382056807),(21.335449218749993,56.89700392127261),(21.599121093749993,55.92458580482949),(25.202636718749993,55.998380955359636),(28.850097656249993,57.06463027327854),(27.09228515625,57.844750992890994),(28.806152343749996,59.17592824927138),(26.257324218749993,59.17592824927138),(24.455566406249993,59.42272750081452)], [(35.13427734375,59.84481485969107),(31.970214843749993,58.97266715450152),(33.20068359374999,56.776808316568406),(36.67236328125,56.41390137600675),(39.08935546874999,57.25528054528888),(42.69287109374999,58.03137242177638),(40.89111328124999,59.26588062825809),(37.28759765625,58.722598828043374),(37.11181640624999,59.66774058164964),(35.13427734375,59.84481485969107)], [(29.157714843749993,55.75184939173528),(22.565917968749993,55.128649068488784),(22.565917968749993,53.54030739150019),(22.038574218749996,51.48138289610097),(26.257324218749993,51.42661449707484),(30.124511718749993,50.54136296522162),(32.18994140624999,51.17934297928929),(30.124511718749993,53.173119202640635),(35.09033203124999,53.173119202640635),(33.11279296875,54.085173420886775),(29.597167968749993,55.50374985927513),(29.157714843749993,55.75184939173528)], [(42.82470703125,56.58369172128337),(36.584472656249986,55.329144408405085),(37.99072265625,53.592504809039355),(34.95849609374999,51.48138289610097),(36.54052734374999,50.40151532278236),(39.66064453124999,50.289339253291786),(39.79248046875,52.13348804077148),(41.77001953125,50.68079714532166),(44.49462890624999,51.97134580885171),(47.30712890624999,52.509534770327264),(44.05517578125,53.54030739150019),(46.60400390625,53.696706475303245),(47.61474609375,55.40406982700608),(45.37353515625,55.40406982700608),(42.82470703125,56.58369172128337)]])) format TSV; +select wkt(polygonsIntersectionMercator([[(29.453587685533865,59.779570356240356),(29.393139070478895,52.276266797422124),(40.636581470703206,59.38168915000267),(41.21084331372543,59.103467777099866),(29.786055068336193,52.146627480315004),(31.23682182965546,52.16517054781818),(41.69443223416517,58.85424941916091),(42.51048853740727,58.47703162291134),(32.59691566839227,52.22075341251539),(34.289476889931414,52.22075341251539),(43.02430176537451,58.07974369546071),(43.02430176537451,57.25537683364851),(35.468224883503325,52.2022335126388),(37.16078610504247,52.23926559241349),(43.02430176537451,56.26136189644947),(43.02430176537451,55.326904361850836),(38.33953409861437,52.16517054781818),(40.09254393520848,52.16517054781818),(44.4146199116388,55.3097062225408),(44.47506852669377,59.80998197603594),(39.72985224487867,59.931351417569715),(30.23941968124846,53.67744677450975),(30.20919537372098,54.63314259659509),(38.73245009647167,59.94649146557819),(37.2816833351524,59.97675082987618),(30.23941968124846,55.2752875586599),(30.33009260383092,56.19415599955667),(36.28428118674541,59.96162460231375),(34.863738732953635,59.97675082987618),(30.178971066193498,56.97640788219866),(30.178971066193498,57.91957806959033),(33.65476643185424,59.94649146557819),(32.32489690064491,59.94649146557819),(30.481214141468342,58.85424941916091),(30.571887064050795,59.99187015036608),(29.453587685533865,59.779570356240356)]], [[(24.367675781249993,61.45977057029751),(19.577636718749993,58.67693767258692),(19.577636718749993,57.492213666700735),(19.445800781249996,55.87531083569678),(19.445800781249996,54.085173420886775),(17.468261718749996,53.014783245859235),(20.017089843749993,51.563412328675895),(21.203613281249993,50.205033264943324),(26.125488281249993,50.40151532278236),(27.22412109374999,48.980216985374994),(32.80517578124999,49.525208341974405),(35.26611328124999,48.74894534343292),(36.93603515624999,49.66762782262194),(42.56103515625,48.77791275550183),(43.92333984374999,49.8096315635631),(47.17529296875,49.152969656170455),(49.28466796875,50.54136296522162),(48.05419921875,51.17934297928929),(51.39404296875,52.48278022207825),(50.64697265625,53.014783245859235),(52.88818359375,53.93021986394004),(51.65771484374999,54.29088164657006),(52.66845703125,55.825973254619015),(50.25146484375,56.145549500679095),(51.92138671875,57.914847767009206),(49.15283203125,58.17070248348605),(49.59228515625,60.086762746260064),(47.043457031249986,59.88893689676584),(43.57177734375,61.37567331572748),(42.64892578125,60.630101766266705),(36.89208984374999,62.000904713685856),(36.01318359374999,61.143235250840576),(31.398925781249993,62.02152819100766),(30.563964843749996,61.05828537037917),(26.872558593749993,61.71070595883174),(26.652832031249993,61.10078883158897),(24.367675781249993,61.45977057029751)], [(24.455566406249993,59.42272750081452),(21.203613281249993,58.49369382056807),(21.335449218749993,56.89700392127261),(21.599121093749993,55.92458580482949),(25.202636718749993,55.998380955359636),(28.850097656249993,57.06463027327854),(27.09228515625,57.844750992890994),(28.806152343749996,59.17592824927138),(26.257324218749993,59.17592824927138),(24.455566406249993,59.42272750081452)], [(35.13427734375,59.84481485969107),(31.970214843749993,58.97266715450152),(33.20068359374999,56.776808316568406),(36.67236328125,56.41390137600675),(39.08935546874999,57.25528054528888),(42.69287109374999,58.03137242177638),(40.89111328124999,59.26588062825809),(37.28759765625,58.722598828043374),(37.11181640624999,59.66774058164964),(35.13427734375,59.84481485969107)], [(29.157714843749993,55.75184939173528),(22.565917968749993,55.128649068488784),(22.565917968749993,53.54030739150019),(22.038574218749996,51.48138289610097),(26.257324218749993,51.42661449707484),(30.124511718749993,50.54136296522162),(32.18994140624999,51.17934297928929),(30.124511718749993,53.173119202640635),(35.09033203124999,53.173119202640635),(33.11279296875,54.085173420886775),(29.597167968749993,55.50374985927513),(29.157714843749993,55.75184939173528)], [(42.82470703125,56.58369172128337),(36.584472656249986,55.329144408405085),(37.99072265625,53.592504809039355),(34.95849609374999,51.48138289610097),(36.54052734374999,50.40151532278236),(39.66064453124999,50.289339253291786),(39.79248046875,52.13348804077148),(41.77001953125,50.68079714532166),(44.49462890624999,51.97134580885171),(47.30712890624999,52.509534770327264),(44.05517578125,53.54030739150019),(46.60400390625,53.696706475303245),(47.61474609375,55.40406982700608),(45.37353515625,55.40406982700608),(42.82470703125,56.58369172128337)]])) format TSV; From c81c742877ce76b4f60cb15eab5280c95c814dc1 Mon Sep 17 00:00:00 2001 From: Nikita Mikhaylov Date: Sat, 6 Mar 2021 02:08:38 +0300 Subject: [PATCH 314/544] Revert "more mercator" This reverts commit d4258922d1543211cf8b62f7c827a927712dca42. --- src/Functions/geometryConverters.h | 29 ++++- src/Functions/geometryTypes.h | 33 ------ src/Functions/mercatorConverters.cpp | 111 ------------------ src/Functions/mercatorConverters.h | 85 -------------- src/Functions/polygonArea.cpp | 16 +++ src/Functions/polygonConvexHull.cpp | 16 +++ src/Functions/polygonPerimeter.cpp | 16 +++ src/Functions/polygonsDistance.cpp | 18 +++ src/Functions/polygonsEquals.cpp | 18 +++ src/Functions/polygonsIntersection.cpp | 65 +++++----- src/Functions/polygonsSymDifference.cpp | 62 ++++------ src/Functions/polygonsUnion.cpp | 61 ++++------ src/Functions/polygonsWithin.cpp | 60 +++++----- src/Functions/tests/mercator.cpp | 44 ------- src/Functions/ya.make | 1 - .../01301_polygons_within.reference | 3 + .../0_stateless/01301_polygons_within.sql | 8 +- .../01305_polygons_union.reference | 6 +- .../0_stateless/01305_polygons_union.sql | 10 +- .../01306_polygons_intersection.reference | 5 +- .../01306_polygons_intersection.sql | 10 -- 21 files changed, 235 insertions(+), 442 deletions(-) delete mode 100644 src/Functions/geometryTypes.h delete mode 100644 src/Functions/mercatorConverters.cpp delete mode 100644 src/Functions/mercatorConverters.h diff --git a/src/Functions/geometryConverters.h b/src/Functions/geometryConverters.h index c90e6861b85..f2706c70e3e 100644 --- a/src/Functions/geometryConverters.h +++ b/src/Functions/geometryConverters.h @@ -1,7 +1,12 @@ #pragma once +#include #include +#include +#include +#include + #include #include #include @@ -9,10 +14,12 @@ #include #include #include -#include #include #include +#include +#include + namespace DB { @@ -22,6 +29,26 @@ namespace ErrorCodes extern const int ILLEGAL_TYPE_OF_ARGUMENT; } +template +using Ring = boost::geometry::model::ring; + +template +using Polygon = boost::geometry::model::polygon; + +template +using MultiPolygon = boost::geometry::model::multi_polygon>; + +using CartesianPoint = boost::geometry::model::d2::point_xy; +using CartesianRing = Ring; +using CartesianPolygon = Polygon; +using CartesianMultiPolygon = MultiPolygon; + +/// Latitude, longitude +using SphericalPoint = boost::geometry::model::point>; +using SphericalRing = Ring; +using SphericalPolygon = Polygon; +using SphericalMultiPolygon = MultiPolygon; + /** * Class which takes converts Column with type Tuple(Float64, Float64) to a vector of boost point type. * They are (x,y) in case of cartesian coordinated and (lon,lat) in case of Spherical. diff --git a/src/Functions/geometryTypes.h b/src/Functions/geometryTypes.h deleted file mode 100644 index 20344423db5..00000000000 --- a/src/Functions/geometryTypes.h +++ /dev/null @@ -1,33 +0,0 @@ -#pragma once - -#include - -#include -#include -#include -#include "common/types.h" - -namespace DB -{ - -template -using Ring = boost::geometry::model::ring; - -template -using Polygon = boost::geometry::model::polygon; - -template -using MultiPolygon = boost::geometry::model::multi_polygon>; - -using CartesianPoint = boost::geometry::model::d2::point_xy; -using CartesianRing = Ring; -using CartesianPolygon = Polygon; -using CartesianMultiPolygon = MultiPolygon; - -/// Latitude, longitude -using SphericalPoint = boost::geometry::model::point>; -using SphericalRing = Ring; -using SphericalPolygon = Polygon; -using SphericalMultiPolygon = MultiPolygon; - -} diff --git a/src/Functions/mercatorConverters.cpp b/src/Functions/mercatorConverters.cpp deleted file mode 100644 index 321ad83c16c..00000000000 --- a/src/Functions/mercatorConverters.cpp +++ /dev/null @@ -1,111 +0,0 @@ -#include - -#include - -namespace DB -{ - - -namespace -{ - -constexpr double PI = 3.14159265358979323846; - -constexpr double epsilon = 1e-4; - -/// Convert angle from degrees to radians. -double deg_to_rad(double degree) { - return (degree - epsilon) * (PI / 180.0); -} - -/// Convert angle from radians to degrees. -double rad_to_deg(double radians) { - return radians * (180.0 / PI); -} - -double earth_radius_for_epsg3857 = 6378137.0; -// constexpr double max_coordinate_epsg3857 = 20037508.34; - - -double lon_to_x(double lon) { - return earth_radius_for_epsg3857 * deg_to_rad(lon); -} - -// canonical log(tan()) version -double lat_to_y_with_tan(double lat) { // not constexpr because math functions aren't - return earth_radius_for_epsg3857 * std::log(std::tan(PI/4 + deg_to_rad(lat)/2)); -} - -double x_to_lon(double x) { - return rad_to_deg(x) / earth_radius_for_epsg3857; -} - -double y_to_lat(double y) { // not constexpr because math functions aren't - return rad_to_deg(2 * std::atan(std::exp(y / earth_radius_for_epsg3857)) - PI/2); -} - -} - - -void PointMercatorConverter::forward(CartesianPoint & point) -{ - point.x(lon_to_x(point.template get<0>())); - point.y(lat_to_y_with_tan(point.template get<1>())); -} - - -void RingMercatorConverter::forward(CartesianRing & ring) -{ - for (auto & point : ring) - PointMercatorConverter::forward(point); -} - -void PolygonMercatorConverter::forward(CartesianPolygon & polygon) -{ - RingMercatorConverter::forward(polygon.outer()); - for (auto & hole : polygon.inners()) - RingMercatorConverter::forward(hole); -} - -void MultiPolygonMercatorConverter::forward(CartesianMultiPolygon & multipolygon) -{ - for (auto & polygon : multipolygon) - { - RingMercatorConverter::forward(polygon.outer()); - for (auto & hole : polygon.inners()) - RingMercatorConverter::forward(hole); - } -} - -void PointMercatorConverter::backward(CartesianPoint & point) -{ - point.x(x_to_lon(point.template get<0>())); - point.y(y_to_lat(point.template get<1>())); -} - - -void RingMercatorConverter::backward(CartesianRing & ring) -{ - for (auto & point : ring) - PointMercatorConverter::backward(point); -} - -void PolygonMercatorConverter::backward(CartesianPolygon & polygon) -{ - RingMercatorConverter::backward(polygon.outer()); - for (auto & hole : polygon.inners()) - RingMercatorConverter::backward(hole); -} - -void MultiPolygonMercatorConverter::backward(CartesianMultiPolygon & multipolygon) -{ - for (auto & polygon : multipolygon) - { - RingMercatorConverter::backward(polygon.outer()); - for (auto & hole : polygon.inners()) - RingMercatorConverter::backward(hole); - } -} - -} - diff --git a/src/Functions/mercatorConverters.h b/src/Functions/mercatorConverters.h deleted file mode 100644 index 802370f4c16..00000000000 --- a/src/Functions/mercatorConverters.h +++ /dev/null @@ -1,85 +0,0 @@ -#pragma once - -#include -#include - -namespace ErrorCodes -{ - extern const int LOGICAL_ERROR; -} - -namespace DB -{ - -/// It must work only with CartesianPoint -class PointMercatorConverter -{ -public: - static void forward(CartesianPoint & point); - static void backward(CartesianPoint & point); -}; - - -class RingMercatorConverter -{ -public: - static void forward(CartesianRing & ring); - static void backward(CartesianRing & ring); -}; - - -class PolygonMercatorConverter -{ -public: - static void forward(CartesianPolygon & polygon); - static void backward(CartesianPolygon & polygon); -}; - - - -class MultiPolygonMercatorConverter -{ -public: - static void forward(CartesianMultiPolygon & polygon); - static void backward(CartesianMultiPolygon & polygon); -}; - - -struct PType -{ - using Type = PType; -}; - - -template -void mercatorForward(Geometry & geometry) -{ - if constexpr (std::is_same_v) - return PointMercatorConverter::forward(geometry); - else if constexpr (std::is_same_v) - return RingMercatorConverter::forward(geometry); - else if constexpr (std::is_same_v) - return PolygonMercatorConverter::forward(geometry); - else if constexpr (std::is_same_v) - return MultiPolygonMercatorConverter::forward(geometry); - else - throw Exception("Unknown geometry type", ErrorCodes::LOGICAL_ERROR); -} - - -template -void mercatorBackward(Geometry & geometry) -{ - if constexpr (std::is_same_v) - return PointMercatorConverter::backward(geometry); - else if constexpr (std::is_same_v) - return RingMercatorConverter::backward(geometry); - else if constexpr (std::is_same_v) - return PolygonMercatorConverter::backward(geometry); - else if constexpr (std::is_same_v) - return MultiPolygonMercatorConverter::backward(geometry); - else - throw Exception("Unknown geometry type", ErrorCodes::LOGICAL_ERROR); -} - -} diff --git a/src/Functions/polygonArea.cpp b/src/Functions/polygonArea.cpp index 407d9e96a53..fb1ba7c4a01 100644 --- a/src/Functions/polygonArea.cpp +++ b/src/Functions/polygonArea.cpp @@ -1,6 +1,22 @@ #include #include +#include +#include +#include + +#include + +#include +#include +#include +#include +#include +#include + +#include +#include + namespace DB { diff --git a/src/Functions/polygonConvexHull.cpp b/src/Functions/polygonConvexHull.cpp index 95c6453b84b..5f545cf8ea1 100644 --- a/src/Functions/polygonConvexHull.cpp +++ b/src/Functions/polygonConvexHull.cpp @@ -1,6 +1,22 @@ #include #include +#include +#include +#include + +#include + +#include +#include +#include +#include +#include +#include + +#include +#include + namespace DB { diff --git a/src/Functions/polygonPerimeter.cpp b/src/Functions/polygonPerimeter.cpp index 36af2927ade..c3aadbd187a 100644 --- a/src/Functions/polygonPerimeter.cpp +++ b/src/Functions/polygonPerimeter.cpp @@ -1,6 +1,22 @@ #include #include +#include +#include +#include + +#include + +#include +#include +#include +#include +#include +#include + +#include +#include + namespace DB { namespace ErrorCodes diff --git a/src/Functions/polygonsDistance.cpp b/src/Functions/polygonsDistance.cpp index e05ff3f7f36..074fb5a9798 100644 --- a/src/Functions/polygonsDistance.cpp +++ b/src/Functions/polygonsDistance.cpp @@ -1,6 +1,24 @@ #include #include +#include +#include +#include + +#include + +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include + namespace DB { diff --git a/src/Functions/polygonsEquals.cpp b/src/Functions/polygonsEquals.cpp index b5c3129a616..12f4cf09fa5 100644 --- a/src/Functions/polygonsEquals.cpp +++ b/src/Functions/polygonsEquals.cpp @@ -1,6 +1,24 @@ #include #include +#include +#include +#include + +#include + +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include + namespace DB { namespace ErrorCodes diff --git a/src/Functions/polygonsIntersection.cpp b/src/Functions/polygonsIntersection.cpp index 3f0b67ed722..0de3d023044 100644 --- a/src/Functions/polygonsIntersection.cpp +++ b/src/Functions/polygonsIntersection.cpp @@ -1,6 +1,22 @@ #include #include -#include + +#include +#include +#include + +#include + +#include +#include +#include +#include +#include +#include + +#include +#include +#include namespace DB { @@ -10,29 +26,11 @@ namespace ErrorCodes extern const int ILLEGAL_TYPE_OF_ARGUMENT; } -struct IntersectionCartesian -{ - static inline const char * name = "polygonsIntersectionCartesian"; - using Point = CartesianPoint; -}; - -struct IntersectionSpherical -{ - static inline const char * name = "polygonsIntersectionSpherical"; - using Point = SphericalPoint; -}; - -struct IntersectionMercator -{ - static inline const char * name = "polygonsIntersectionMercator"; - using Point = CartesianPoint; -}; - -template +template class FunctionPolygonsIntersection : public IFunction { public: - static inline const char * name = Holder::name; + static inline const char * name; explicit FunctionPolygonsIntersection() = default; @@ -64,8 +62,6 @@ public: ColumnPtr executeImpl(const ColumnsWithTypeAndName & arguments, const DataTypePtr & /*result_type*/, size_t input_rows_count) const override { - using Point = typename Holder::Point; - MultiPolygonSerializer serializer; callOnTwoGeometryDataTypes(arguments[0].type, arguments[1].type, [&](const auto & left_type, const auto & right_type) @@ -91,19 +87,10 @@ public: boost::geometry::correct(first[i]); boost::geometry::correct(second[i]); - if constexpr (std::is_same_v) - { - mercatorForward(first[i]); - mercatorForward(second[i]); - } - MultiPolygon intersection{}; /// Main work here. boost::geometry::intersection(first[i], second[i], intersection); - if constexpr (std::is_same_v) - mercatorBackward(intersection); - serializer.add(intersection); } } @@ -118,10 +105,18 @@ public: } }; + +template <> +const char * FunctionPolygonsIntersection::name = "polygonsIntersectionCartesian"; + +template <> +const char * FunctionPolygonsIntersection::name = "polygonsIntersectionSpherical"; + + void registerFunctionPolygonsIntersection(FunctionFactory & factory) { - factory.registerFunction>(); - factory.registerFunction>(); - factory.registerFunction>(); + factory.registerFunction>(); + factory.registerFunction>(); } + } diff --git a/src/Functions/polygonsSymDifference.cpp b/src/Functions/polygonsSymDifference.cpp index b7b20f3d6c3..9ab6b79c5f5 100644 --- a/src/Functions/polygonsSymDifference.cpp +++ b/src/Functions/polygonsSymDifference.cpp @@ -1,6 +1,21 @@ #include #include -#include + +#include +#include +#include + +#include + +#include +#include +#include +#include +#include +#include + +#include +#include namespace DB { @@ -11,30 +26,11 @@ namespace ErrorCodes } -struct SymDifferenceCartesian -{ - static inline const char * name = "polygonsSymDifferenceCartesian"; - using Point = CartesianPoint; -}; - -struct SymDifferenceSpherical -{ - static inline const char * name = "polygonsSymDifferenceSpherical"; - using Point = SphericalPoint; -}; - -struct SymDifferenceMercator -{ - static inline const char * name = "polygonsSymDifferenceMercator"; - using Point = CartesianPoint; -}; - - -template +template class FunctionPolygonsSymDifference : public IFunction { public: - static inline const char * name = Holder::name; + static const char * name; explicit FunctionPolygonsSymDifference() = default; @@ -65,8 +61,6 @@ public: ColumnPtr executeImpl(const ColumnsWithTypeAndName & arguments, const DataTypePtr & /*result_type*/, size_t input_rows_count) const override { - using Point = typename Holder::Point; - MultiPolygonSerializer serializer; callOnTwoGeometryDataTypes(arguments[0].type, arguments[1].type, [&](const auto & left_type, const auto & right_type) @@ -90,18 +84,9 @@ public: boost::geometry::correct(first[i]); boost::geometry::correct(second[i]); - if constexpr (std::is_same_v) - { - mercatorForward(first[i]); - mercatorForward(second[i]); - } - MultiPolygon sym_difference{}; boost::geometry::sym_difference(first[i], second[i], sym_difference); - if constexpr (std::is_same_v) - mercatorBackward(sym_difference); - serializer.add(sym_difference); } } @@ -116,11 +101,16 @@ public: } }; +template <> +const char * FunctionPolygonsSymDifference::name = "polygonsSymDifferenceCartesian"; + +template <> +const char * FunctionPolygonsSymDifference::name = "polygonsSymDifferenceSpherical"; + void registerFunctionPolygonsSymDifference(FunctionFactory & factory) { - factory.registerFunction>(); - factory.registerFunction>(); - factory.registerFunction>(); + factory.registerFunction>(); + factory.registerFunction>(); } } diff --git a/src/Functions/polygonsUnion.cpp b/src/Functions/polygonsUnion.cpp index c2dd93d47d4..eab2e2e588f 100644 --- a/src/Functions/polygonsUnion.cpp +++ b/src/Functions/polygonsUnion.cpp @@ -1,6 +1,21 @@ #include #include -#include + +#include +#include +#include + +#include + +#include +#include +#include +#include +#include +#include + +#include +#include namespace DB { @@ -11,30 +26,11 @@ namespace ErrorCodes } -struct UnionCartesian -{ - static inline const char * name = "polygonsUnionCartesian"; - using Point = CartesianPoint; -}; - -struct UnionSpherical -{ - static inline const char * name = "polygonsUnionSpherical"; - using Point = SphericalPoint; -}; - -struct UnionMercator -{ - static inline const char * name = "polygonsUnionMercator"; - using Point = CartesianPoint; -}; - - -template +template class FunctionPolygonsUnion : public IFunction { public: - static inline const char * name = Holder::name ; + static inline const char * name; explicit FunctionPolygonsUnion() = default; @@ -65,7 +61,6 @@ public: ColumnPtr executeImpl(const ColumnsWithTypeAndName & arguments, const DataTypePtr & /*result_type*/, size_t input_rows_count) const override { - using Point = typename Holder::Point; MultiPolygonSerializer serializer; callOnTwoGeometryDataTypes(arguments[0].type, arguments[1].type, [&](const auto & left_type, const auto & right_type) @@ -91,19 +86,10 @@ public: boost::geometry::correct(first[i]); boost::geometry::correct(second[i]); - if constexpr (std::is_same_v) - { - mercatorForward(first[i]); - mercatorForward(second[i]); - } - MultiPolygon polygons_union{}; /// Main work here. boost::geometry::union_(first[i], second[i], polygons_union); - if constexpr (std::is_same_v) - mercatorBackward(polygons_union); - serializer.add(polygons_union); } } @@ -118,12 +104,17 @@ public: } }; +template <> +const char * FunctionPolygonsUnion::name = "polygonsUnionCartesian"; + +template <> +const char * FunctionPolygonsUnion::name = "polygonsUnionSpherical"; + void registerFunctionPolygonsUnion(FunctionFactory & factory) { - factory.registerFunction>(); - factory.registerFunction>(); - factory.registerFunction>(); + factory.registerFunction>(); + factory.registerFunction>(); } } diff --git a/src/Functions/polygonsWithin.cpp b/src/Functions/polygonsWithin.cpp index 47ecdaf2c8e..68db1494a0d 100644 --- a/src/Functions/polygonsWithin.cpp +++ b/src/Functions/polygonsWithin.cpp @@ -1,6 +1,23 @@ #include #include -#include + +#include +#include +#include + +#include + +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include namespace DB { @@ -10,29 +27,12 @@ namespace ErrorCodes extern const int ILLEGAL_TYPE_OF_ARGUMENT; } -struct WithinCartesian -{ - static inline const char * name = "polygonsWithinCartesian"; - using Point = CartesianPoint; -}; -struct WithinSpherical -{ - static inline const char * name = "polygonsWithinSpherical"; - using Point = SphericalPoint; -}; - -struct WithinMercator -{ - static inline const char * name = "polygonsWithinMercator"; - using Point = CartesianPoint; -}; - -template +template class FunctionPolygonsWithin : public IFunction { public: - static inline const char * name = Holder::name; + static inline const char * name; explicit FunctionPolygonsWithin() = default; @@ -63,7 +63,6 @@ public: ColumnPtr executeImpl(const ColumnsWithTypeAndName & arguments, const DataTypePtr & /*result_type*/, size_t input_rows_count) const override { - using Point = typename Holder::Point; auto res_column = ColumnUInt8::create(); auto & res_data = res_column->getData(); res_data.reserve(input_rows_count); @@ -89,12 +88,6 @@ public: boost::geometry::correct(first[i]); boost::geometry::correct(second[i]); - if constexpr (std::is_same_v) - { - mercatorForward(first[i]); - mercatorForward(second[i]); - } - res_data.emplace_back(boost::geometry::within(first[i], second[i])); } } @@ -109,11 +102,18 @@ public: } }; + +template <> +const char * FunctionPolygonsWithin::name = "polygonsWithinCartesian"; + +template <> +const char * FunctionPolygonsWithin::name = "polygonsWithinSpherical"; + + void registerFunctionPolygonsWithin(FunctionFactory & factory) { - factory.registerFunction>(); - factory.registerFunction>(); - factory.registerFunction>(); + factory.registerFunction>(); + factory.registerFunction>(); } } diff --git a/src/Functions/tests/mercator.cpp b/src/Functions/tests/mercator.cpp index 7174cd62550..bc9299fdaf1 100644 --- a/src/Functions/tests/mercator.cpp +++ b/src/Functions/tests/mercator.cpp @@ -1,10 +1,7 @@ -#include #include #include -#include #include -#include constexpr double PI = 3.14159265358979323846; @@ -81,9 +78,6 @@ void printMultiPolygon(DB::CartesianMultiPolygon & multi_polygon) std::cout << "--------------" << std::endl; } - -const char * example = "MULTIPOLYGON (((11.737519 -16.692578, 11.738507 -16.705822, 11.715794 -16.685006, 11.701973 -16.655674, 11.669394 -16.55721, 11.67137 -16.529744, 11.681244 -16.517436, 11.701977 -16.500393, 11.719746 -16.506083, 11.721528 -16.555434, 11.725759 -16.577325, 11.725759 -16.618422, 11.726659 -16.656627, 11.737519 -16.692578)), ((13.98233 -5.853285, 14.006307 -5.866928, 14.02119 -5.872509, 14.140149 -5.863517, 14.163094 -5.866101, 14.222212 -5.886978, 14.241849 -5.889562, 14.346132 -5.891526, 14.436669 -5.893386, 14.482144 -5.885531, 14.545706 -5.901654, 14.576609 -5.904651, 14.703733 -5.881397, 14.759652 -5.880469, 14.797164 -5.879847, 14.888424 -5.878296, 14.979581 -5.876746, 15.070842 -5.875299, 15.162102 -5.873749, 15.253363 -5.872199, 15.34452 -5.870752, 15.435781 -5.869201, 15.527041 -5.867754, 15.618302 -5.866204, 15.709563 -5.864757, 15.80072 -5.863207, 15.892084 -5.861657, 15.983241 -5.860106, 16.074502 -5.858556, 16.100856 -5.858108, 16.165659 -5.857006, 16.257023 -5.855559, 16.315727 -5.854629, 16.341255 -5.859796, 16.358515 -5.870235, 16.374328 -5.882844, 16.396136 -5.894109, 16.412465 -5.896797, 16.427452 -5.894626, 16.459491 -5.885221, 16.479335 -5.882224, 16.497318 -5.882844, 16.515095 -5.887081, 16.552922 -5.902378, 16.56026 -5.906408, 16.597364 -5.924702, 16.594056 -5.933177, 16.580621 -5.94899, 16.580724 -5.955398, 16.590336 -5.968007, 16.595297 -5.978549, 16.597157 -5.98785, 16.601188 -6.050586, 16.604288 -6.061851, 16.610076 -6.061231, 16.619481 -6.05658, 16.62837 -6.05534, 16.632194 -6.065262, 16.61111 -6.089756, 16.630953 -6.09172, 16.643046 -6.100712, 16.660099 -6.123966, 16.671778 -6.130167, 16.68511 -6.134301, 16.695859 -6.140916, 16.700407 -6.154352, 16.717873 -6.174092, 16.726762 -6.189182, 16.724384 -6.202411, 16.717253 -6.215123, 16.714773 -6.231556, 16.715393 -6.304834, 16.716943 -6.313309, 16.720147 -6.32261, 16.722317 -6.332429, 16.720974 -6.342351, 16.715289 -6.345968, 16.706091 -6.346692, 16.697409 -6.349792, 16.693585 -6.359817, 16.692965 -6.398368, 16.694722 -6.402192, 16.700407 -6.411287, 16.707021 -6.415421, 16.713636 -6.413974, 16.718907 -6.415318, 16.720974 -6.42834, 16.715186 -6.441363, 16.706091 -6.455212, 16.706091 -6.465031, 16.727795 -6.465961, 16.721284 -6.480534, 16.721697 -6.491799, 16.728002 -6.501101, 16.738337 -6.510609, 16.747329 -6.521978, 16.746812 -6.53159, 16.734513 -6.552364, 16.730792 -6.56983, 16.733996 -6.586057, 16.748672 -6.619543, 16.753013 -6.637216, 16.755907 -6.674113, 16.760351 -6.691994, 16.768103 -6.708323, 16.823086 -6.785114, 16.82691 -6.799791, 16.828564 -6.814363, 16.832905 -6.827799, 16.84448 -6.838445, 16.855332 -6.842165, 16.879414 -6.844646, 16.890783 -6.847126, 16.914967 -6.864076, 16.926956 -6.888571, 16.934501 -6.945725, 16.932124 -6.949859, 16.92861 -6.959264, 16.928713 -6.968669, 16.937601 -6.97301, 16.949074 -6.975387, 16.955275 -6.981485, 16.959305 -6.989443, 16.964576 -6.997195, 16.973155 -7.014765, 16.969331 -7.030371, 16.962303 -7.047424, 16.961166 -7.069231, 16.929126 -7.060653, 16.927059 -7.085665, 16.947523 -7.151087, 16.945663 -7.204624, 16.950831 -7.213202, 16.960546 -7.219093, 16.972845 -7.232219, 16.989071 -7.254853, 16.991655 -7.270563, 16.991655 -7.286066, 16.996099 -7.297951, 17.022867 -7.306013, 17.056664 -7.329887, 17.082502 -7.34229, 17.093354 -7.352315, 17.097798 -7.367404, 17.097895 -7.378294, 17.098108 -7.402338, 17.103999 -7.416084, 17.118159 -7.425489, 17.12622 -7.411433, 17.132215 -7.409572, 17.138209 -7.413707, 17.146167 -7.418047, 17.167458 -7.423525, 17.176553 -7.428176, 17.180274 -7.435411, 17.1749 -7.441612, 17.163117 -7.445539, 17.151542 -7.451224, 17.146167 -7.463109, 17.148545 -7.472618, 17.154952 -7.479129, 17.163427 -7.485227, 17.172832 -7.493805, 17.186268 -7.516336, 17.202908 -7.564292, 17.217791 -7.585996, 17.266574 -7.610387, 17.286417 -7.625167, 17.282697 -7.643977, 17.287451 -7.660203, 17.286521 -7.683664, 17.288898 -7.699477, 17.303781 -7.692449, 17.305951 -7.700097, 17.308121 -7.701854, 17.308121 -7.702371, 17.303781 -7.706712, 17.32042 -7.713637, 17.320627 -7.722422, 17.314219 -7.731827, 17.310602 -7.740818, 17.315976 -7.753634, 17.334063 -7.769654, 17.337887 -7.778646, 17.342435 -7.78402, 17.365482 -7.787327, 17.375404 -7.792288, 17.376954 -7.801693, 17.376128 -7.814096, 17.380985 -7.823191, 17.400002 -7.823397, 17.400002 -7.830219, 17.3907 -7.83921, 17.397625 -7.841381, 17.410854 -7.841484, 17.420569 -7.843861, 17.423567 -7.855127, 17.42305 -7.867942, 17.426357 -7.875487, 17.44093 -7.871146, 17.452402 -7.883962, 17.448681 -7.89285, 17.43938 -7.902152, 17.434109 -7.915795, 17.437829 -7.922409, 17.446821 -7.928921, 17.468318 -7.939463, 17.463564 -7.946904, 17.463047 -7.949798, 17.465218 -7.952588, 17.468318 -7.959926, 17.471109 -7.955586, 17.477207 -7.950521, 17.481858 -7.945664, 17.490953 -7.977496, 17.496224 -7.987212, 17.506352 -7.996513, 17.528366 -8.007778999999999, 17.536531 -8.015117, 17.540045 -8.032067, 17.528366 -8.060592, 17.536219 -8.075402, 17.536531 -8.075991999999999, 17.544283 -8.084777000000001, 17.600197 -8.098523, 17.628412 -8.098006, 17.713265 -8.068344, 17.74334 -8.070721000000001, 17.77724 -8.08333, 17.789539 -8.081778999999999, 17.851757 -8.047776000000001, 17.867984 -8.044468999999999, 17.901367 -8.048086, 17.930822 -8.062142, 17.981879 -8.105344000000001, 18.008544 -8.111235000000001, 18.098151 -8.109271, 18.091743 -8.084156999999999, 18.09288 -8.056355, 18.102181 -8.033307000000001, 18.120372 -8.022455000000001, 18.135771 -8.018528, 18.176596 -7.999201, 18.192719 -7.996927, 18.267339 -7.999821, 18.316225 -8.001681, 18.353639 -8.015943999999999, 18.366558 -8.016771, 18.381958 -8.012326, 18.403868 -8.000647000000001, 18.420508 -7.99827, 18.481797 -7.99734, 18.487894 -7.99827, 18.507635 -7.997547, 18.50226 -7.960133, 18.504327 -7.940703, 18.516523 -7.932228, 18.588663 -7.932331, 18.666901 -7.932331, 18.738525 -7.932331, 18.74731 -7.937809, 18.752064 -7.965921, 18.758265 -7.97822, 18.773768 -8.001991, 18.818003 -8.001991, 18.945437 -8.001991, 19.073078 -8.001991, 19.200616 -8.001991, 19.328256 -8.001991, 19.355542 -8.001991, 19.334044 -7.92427, 19.330944 -7.898638, 19.333838 -7.867322, 19.348514 -7.821537, 19.358022 -7.791565, 19.377762 -7.73007, 19.38138 -7.703921, 19.380966 -7.679737, 19.373215 -7.653795, 19.361846 -7.627027, 19.353681 -7.600258, 19.355645 -7.57411, 19.424685 -7.579691, 19.443908 -7.575247, 19.458274 -7.560261, 19.468713 -7.52109, 19.482356 -7.50352, 19.483079 -7.5029, 19.483906 -7.502383, 19.484733 -7.50197, 19.513155 -7.479439, 19.513775 -7.448433, 19.487213 -7.381977, 19.479462 -7.347664, 19.476361 -7.315935, 19.487213 -7.153877, 19.497032 -7.121735, 19.508607 -7.103235, 19.536202 -7.068405, 19.54261 -7.052282, 19.540233 -7.041636, 19.521836 -7.001949, 19.57403 -7.001949, 19.616094 -7.001949, 19.658055 -7.001949, 19.700017 -7.001949, 19.741978 -7.001949, 19.784043 -7.001949, 19.826107 -7.001949, 19.868068 -7.001949, 19.91003 -7.001949, 19.952094 -7.001949, 19.963079 -7.001949, 19.994056 -7.001949, 20.03612 -7.001949, 20.078185 -7.001949, 20.120146 -7.001949, 20.162107 -7.001949, 20.204069 -7.001949, 20.246237 -7.001949, 20.294296 -7.001949, 20.294192 -6.987479, 20.298637 -6.95451, 20.303184 -6.940144, 20.311246 -6.925054, 20.319514 -6.918543, 20.331089 -6.916786, 20.383696 -6.916683, 20.467308 -6.916373, 20.553711 -6.916166, 20.611486 -6.915959, 20.588128 -6.976834, 20.560946 -7.047837, 20.545443 -7.088558, 20.524463 -7.143129, 20.518261 -7.189741, 20.519398 -7.23718, 20.520535 -7.286376, 20.567044 -7.286169, 20.639804 -7.285962, 20.712565 -7.285859, 20.785222 -7.285549, 20.857982 -7.285342, 20.930743 -7.285239, 21.003503 -7.284929, 21.076263 -7.284722, 21.148921 -7.284619, 21.221784 -7.284412, 21.294338 -7.284205, 21.367098 -7.283999, 21.439962 -7.283792, 21.512619 -7.283585, 21.585483 -7.283379, 21.658243 -7.283172, 21.730901 -7.282965, 21.76449 -7.282758, 21.784954 -7.283379, 21.784644 -7.286893, 21.788261 -7.29175, 21.794773 -7.294024, 21.802524 -7.298882, 21.808829 -7.306426, 21.831359 -7.354175, 21.836941 -7.360583, 21.842418 -7.369885, 21.839938 -7.378153, 21.834977 -7.385285, 21.83291 -7.390762, 21.838698 -7.402028, 21.845726 -7.409056, 21.851617 -7.416497, 21.854097 -7.428899, 21.85296 -7.434997, 21.847793 -7.444092, 21.846656 -7.449053, 21.848929 -7.454841, 21.858645 -7.464039, 21.860918 -7.469827, 21.859058 -7.482643, 21.849963 -7.5029, 21.846656 -7.514269, 21.845932 -7.526775, 21.848206 -7.532976, 21.851513 -7.538453, 21.854097 -7.548375, 21.854097 -7.585996, 21.85079 -7.593644, 21.836217 -7.60832, 21.83291 -7.614211, 21.831049 -7.632815, 21.826558 -7.645601, 21.819577 -7.665474, 21.807898 -7.699374, 21.795083 -7.736271, 21.787228 -7.758595, 21.776169 -7.790324, 21.770485 -7.806758, 21.764594 -7.850682, 21.770898 -7.876521, 21.772138 -7.8882, 21.768831 -7.899878, 21.754362 -7.919619, 21.751054 -7.929231, 21.753948 -7.99734, 21.758496 -8.015117, 21.768211 -8.023178, 21.79963 -8.040851999999999, 21.806348 -8.052427, 21.808519 -8.076715, 21.815133 -8.096766000000001, 21.825985 -8.114439000000001, 21.851203 -8.143481, 21.857301 -8.156917, 21.860195 -8.173764, 21.862469 -8.216448, 21.873217 -8.264094, 21.881382 -8.282076999999999, 21.910631 -8.310499, 21.920553 -8.329000000000001, 21.913146 -8.333003, 21.905257 -8.337268, 21.899882 -8.348843, 21.911458 -8.374268000000001, 21.935953 -8.413024999999999, 21.943911 -8.455916999999999, 21.939983 -8.499945, 21.916212 -8.576840000000001, 21.921276 -8.578597, 21.924377 -8.580767, 21.928408 -8.582523999999999, 21.935953 -8.583144000000001, 21.917762 -8.615183999999999, 21.90691 -8.645466000000001, 21.895128 -8.718743, 21.884793 -8.782304999999999, 21.877765 -8.78923, 21.870117 -8.822303, 21.859885 -8.846384, 21.85172 -8.925862, 21.854097 -8.977849000000001, 21.839731 -9.092364, 21.851927 -9.192926, 21.851617 -9.233751, 21.841075 -9.275815, 21.810172 -9.338137, 21.803764 -9.372966999999999, 21.793326 -9.393534000000001, 21.791982 -9.406143, 21.79622 -9.417097999999999, 21.813169 -9.437459, 21.819887 -9.447691000000001, 21.832186 -9.488412, 21.843658 -9.576159000000001, 21.854097 -9.61781, 21.878902 -9.657083999999999, 21.94236 -9.72292, 21.95652 -9.76178, 21.962721 -9.758473, 21.978224 -9.752167999999999, 21.984425 -9.748758, 21.986285 -9.781107, 21.998378 -9.809116, 22.055842 -9.875365, 22.065764 -9.881049000000001, 22.072792 -9.878879, 22.079923 -9.871644, 22.090982 -9.877122, 22.107622 -9.891385, 22.146276 -9.915673, 22.159918 -9.931486, 22.167566 -9.953707, 22.175628 -9.988950000000001, 22.196092 -10.043624, 22.204877 -10.085792, 22.209734 -10.098194, 22.20281 -10.130647, 22.221103 -10.176225, 22.245701 -10.221184, 22.263478 -10.268209, 22.286939 -10.309137, 22.295517 -10.339316, 22.31009 -10.356989, 22.313397 -10.368565, 22.31226 -10.379934, 22.307093 -10.399571, 22.305956 -10.409803, 22.301098 -10.424892, 22.278671 -10.460962, 22.271849 -10.481736, 22.27402 -10.503027, 22.284045 -10.516463, 22.296447 -10.529692, 22.305956 -10.550673, 22.30947 -10.592944, 22.308573 -10.612519, 22.307609 -10.633562, 22.31009 -10.668805, 22.32642 -10.694643, 22.321045 -10.700741, 22.319495 -10.703842, 22.322389 -10.742186, 22.319495 -10.761306, 22.309677 -10.769781, 22.29066 -10.772675, 22.265855 -10.779909, 22.223997 -10.797066, 22.206014 -10.809882, 22.182759 -10.830449, 22.165499 -10.85236, 22.165706 -10.868793, 22.174801 -10.884296, 22.196092 -10.954162, 22.196092 -10.992299, 22.199296 -10.997364, 22.213972 -11.007906, 22.217176 -11.012763, 22.220586 -11.030023, 22.23733 -11.059376, 22.243841 -11.078289, 22.25552 -11.166552, 22.252006 -11.208617, 22.23764 -11.249545, 22.263064 -11.248201, 22.281771 -11.236729, 22.325696 -11.190324, 22.340269 -11.180195, 22.440315 -11.146915, 22.469357 -11.122421, 22.500983 -11.042426, 22.53757 -11.037155, 22.613637 -11.06754, 22.650948 -11.08232, 22.688982 -11.092965, 22.727739 -11.096789, 22.764222 -11.090485, 22.785306 -11.078496, 22.806907 -11.063096, 22.829231 -11.052141, 22.853106 -11.053071, 22.87357 -11.063716, 22.91119 -11.088314, 22.931137 -11.096789, 22.969585 -11.102474, 23.014336 -11.102474, 23.114692 -11.085524, 23.196754 -11.071881, 23.306515 -11.01173, 23.373281 -10.975143, 23.415449 -10.963257, 23.455963 -10.960984, 23.494824 -10.964291, 23.557042 -10.981344, 23.641585 -11.004495, 23.674865 -11.006976, 23.714139 -11.003875, 23.752276 -11.008216, 23.79372 -11.021135, 23.833718 -11.028473, 23.867411 -11.016381, 23.874129 -11.006872, 23.881777 -10.984548, 23.887771 -10.974419, 23.89728 -10.966461, 23.918571 -10.954059, 23.927976 -10.945377, 23.936141 -10.925223, 23.940585 -10.901556, 23.948646 -10.881505, 23.967457 -10.872307, 23.974278 -10.921399, 23.980582 -10.938349, 24.000013 -10.967805, 24.003733 -10.982481, 23.997325 -11.001705, 23.993708 -11.019585, 23.994535 -11.074982, 23.990091 -11.113532, 23.995775 -11.127382, 24.015309 -11.130482, 24.011175 -11.272903, 24.016963 -11.298431, 24.061094 -11.394962, 24.061714 -11.406951, 24.052102 -11.42049, 24.020787 -11.444572, 24.009935 -11.459454, 24.007557 -11.470513, 24.007247 -11.483122, 24.009831 -11.507203, 24.009728 -11.52343, 24.005387 -11.535212, 23.977688 -11.577277, 23.959912 -11.617171, 23.954641 -11.636911, 23.954641 -11.662233, 23.962289 -11.68156, 23.972727 -11.700577, 23.981306 -11.724761, 23.985543 -11.799485, 23.990194 -11.824083, 23.98854 -11.834212, 23.967043 -11.882891, 23.961049 -12.011669, 23.954331 -12.151919, 23.959602 -12.19667, 23.981306 -12.227676, 24.006627 -12.253721, 24.016859 -12.278939, 24.020993 -12.340021, 24.030812 -12.385083, 24.028021 -12.402136, 24.019856 -12.419189, 23.98606 -12.467662, 23.940791 -12.532774, 23.928699 -12.561609, 23.910716 -12.631269, 23.891699 -12.705063, 23.872269 -12.750125, 23.865654 -12.789709, 23.874749 -12.821749, 23.895109 -12.849757, 23.949576 -12.904638, 23.971797 -12.93337, 23.988851 -12.965099, 24.000633 -13.001479, 23.968283 -13.001479, 23.844157 -13.001479, 23.719823 -13.001479, 23.595593 -13.001479, 23.471259 -13.001479, 23.346926 -13.001479, 23.222696 -13.001479, 23.098362 -13.001479, 22.974029 -13.001479, 22.849799 -13.001479, 22.725465 -13.001479, 22.601235 -13.001479, 22.477005 -13.001479, 22.352775 -13.001479, 22.228545 -13.001479, 22.104211 -13.001479, 21.979878 -13.001479, 21.979981 -13.152168, 21.980084 -13.302701, 21.980188 -13.453493, 21.980291 -13.604181, 21.980353 -13.694822, 21.980394 -13.754973, 21.980498 -13.905558, 21.980601 -14.05635, 21.980704 -14.206935, 21.980808 -14.357624, 21.980911 -14.508416, 21.981014 -14.659104, 21.981118 -14.809793, 21.981221 -14.960481, 21.981325 -15.11117, 21.981428 -15.261858, 21.981531 -15.412546, 21.981531 -15.452337, 21.981531 -15.473525, 21.981531 -15.503807, 21.981531 -15.547835, 21.981428 -15.649948, 21.981221 -15.751957, 21.980911 -15.853967, 21.980808 -15.956079, 21.980601 -16.001244, 21.981531 -16.004035, 21.981531 -16.067494, 21.981531 -16.128162, 21.981531 -16.144285, 21.983805 -16.165886, 22.010367 -16.198132, 22.045197 -16.252289, 22.019875 -16.253115, 22.01171 -16.252289, 22.022356 -16.266345, 22.025973 -16.278954, 22.02773 -16.291666, 22.032174 -16.306342, 22.036618 -16.312233, 22.048607 -16.322362, 22.052018 -16.326806, 22.053672 -16.336521, 22.054188 -16.358639, 22.055842 -16.364633, 22.089122 -16.371971, 22.105555 -16.379413, 22.103591 -16.391918, 22.094289 -16.404734, 22.086021 -16.422304, 22.08075 -16.441011, 22.079923 -16.457754, 22.084574 -16.470156, 22.101731 -16.497958, 22.107312 -16.512324, 22.106588 -16.525967, 22.104004 -16.536716, 22.108138 -16.54395, 22.127259 -16.546431, 22.138111 -16.552632, 22.142452 -16.567101, 22.145139 -16.584051, 22.15165 -16.597694, 22.237743 -16.665493, 22.251282 -16.670041, 22.259034 -16.669214, 22.274123 -16.663633, 22.283735 -16.661359, 22.287352 -16.658259, 22.290763 -16.656398, 22.295621 -16.659189, 22.303062 -16.66694, 22.306989 -16.669214, 22.333654 -16.673658, 22.343473 -16.683167, 22.350088 -16.696292, 22.365797 -16.71686, 22.372618 -16.728849, 22.377993 -16.734843, 22.383574 -16.73753, 22.399594 -16.740424, 22.408895 -16.745798, 22.416647 -16.75448, 22.489097 -16.865378, 22.499742 -16.89349, 22.508527 -16.906202, 22.522894 -16.914677, 22.554519 -16.924185, 22.567232 -16.936898, 22.569299 -16.944959, 22.569196 -16.962426, 22.573433 -16.971004, 22.579324 -16.975345, 22.592037 -16.975449, 22.651671 -16.998703, 22.665624 -17.008625, 22.710479 -17.055444, 22.730736 -17.081592, 22.744792 -17.108257, 22.755851 -17.154869, 22.765153 -17.169649, 22.778278 -17.180397, 22.809388 -17.196417, 22.849179 -17.23104, 22.876154 -17.248093, 22.936512 -17.273311, 22.984157 -17.285817, 22.998627 -17.293775, 23.040381 -17.33708, 23.046376 -17.348449, 23.054127 -17.375321, 23.073041 -17.405086, 23.097639 -17.432165, 23.121513 -17.450872, 23.165748 -17.467408, 23.176187 -17.478157, 23.179081 -17.494177, 23.17691 -17.509783, 23.177531 -17.524046, 23.189726 -17.536551, 23.206986 -17.541202, 23.224246 -17.539342, 23.241299 -17.535104, 23.258352 -17.532831, 23.290805 -17.535414, 23.305378 -17.539548, 23.320158 -17.54637, 23.340621 -17.560736, 23.359432 -17.582853, 23.382273 -17.601043, 23.375141 -17.615409, 23.375968 -17.628225, 23.381652 -17.641144, 23.29804 -17.656854, 23.093918 -17.695405, 22.889796 -17.734059, 22.685674 -17.772816, 22.481449 -17.811366, 22.230508 -17.857565, 21.979464 -17.903764, 21.728523 -17.949859, 21.477583 -17.996058, 21.405546 -18.009287, 21.386839 -18.014455, 21.381154 -18.012595, 21.364825 -17.992027, 21.335059 -17.977971, 21.278112 -17.958541, 21.23274 -17.93446, 21.216617 -17.930636, 21.175792 -17.93322, 21.161323 -17.930636, 21.143133 -17.933736, 21.12608 -17.941074, 21.10944 -17.945002, 21.09311 -17.938077, 21.000713 -17.962055, 20.986455 -17.965771, 20.908315 -17.986136, 20.894466 -18.012595, 20.8616 -18.018796, 20.831937 -18.029338, 20.806202 -18.031405, 20.784602 -18.012595, 20.754423 -17.997918, 20.737473 -17.993474, 20.729928 -18.002363, 20.722487 -18.006497, 20.706157 -18.00381, 20.68962 -17.996988, 20.682179 -17.988617, 20.666263 -17.980245, 20.630089 -17.976938, 20.590712 -17.978798, 20.565494 -17.985206, 20.552885 -17.979108, 20.523946 -17.958541, 20.489529 -17.942108, 20.479504 -17.934356, 20.460074 -17.913686, 20.437543 -17.894462, 20.412635 -17.884127, 20.386797 -17.890328, 20.347729 -17.858909, 20.336154 -17.854878, 20.302771 -17.860769, 20.288921 -17.861596, 20.278173 -17.854878, 20.243446 -17.882783, 20.219882 -17.873378, 20.193113 -17.872138, 20.169239 -17.878132, 20.154563 -17.890328, 20.147121 -17.882783, 20.132962 -17.888778, 20.119939 -17.890741, 20.092551 -17.890328, 20.03426 -17.895909, 20.024545 -17.894566, 19.989508 -17.882783, 19.951991 -17.862319, 19.793964 -17.862319, 19.78766 -17.866454, 19.775981 -17.885057, 19.77102 -17.890328, 19.755207 -17.889088, 19.732056 -17.88206, 19.711282 -17.872965, 19.702187 -17.86573, 19.693195 -17.847747, 19.672628 -17.842682, 19.649787 -17.844956, 19.59284 -17.857255, 19.421274 -17.859426, 19.262421 -17.81395, 19.246608 -17.804028, 19.201856 -17.807129, 19.17209 -17.801238, 19.160825 -17.800928, 19.139327 -17.805269, 19.105118 -17.818188, 19.020575 -17.822322, 18.89004 -17.799274, 18.800847 -17.756589, 18.761986 -17.747701, 18.74824 -17.736332, 18.728293 -17.710907, 18.670932 -17.653133, 18.66132 -17.646725, 18.64282 -17.638044, 18.633312 -17.628949, 18.627937 -17.619854, 18.622873 -17.60218, 18.619669 -17.594842, 18.567476 -17.553294, 18.554144 -17.54637, 18.55094 -17.535311, 18.516626 -17.471336, 18.492752 -17.464514, 18.489858 -17.462447, 18.488101 -17.452112, 18.471668 -17.414285, 18.465157 -17.40922, 18.458645 -17.405293, 18.455028 -17.395991, 18.453581 -17.389893, 18.445726 -17.389273, 18.392293 -17.389273, 18.135668 -17.38917, 18.000483 -17.389116, 17.879146 -17.389067, 17.622624 -17.388963, 17.366102 -17.388963, 17.278972 -17.388928, 17.109477 -17.38886, 16.852852 -17.388757, 16.59633 -17.388757, 16.339808 -17.388653, 16.08339 -17.388653, 15.826765 -17.388653, 15.570243 -17.38855, 15.559114 -17.388545, 15.313721 -17.388447, 15.057199 -17.388343, 14.800574 -17.388343, 14.543949 -17.38824, 14.287427 -17.388136, 14.219112 -17.388136, 14.218801 -17.388136, 14.207432 -17.388033, 14.206502 -17.393097, 14.197097 -17.412941, 14.174979 -17.416248, 14.130744 -17.40922, 14.123923 -17.411598, 14.108213 -17.420486, 14.097258 -17.423587, 14.085372 -17.423587, 14.069146 -17.418626, 14.047442 -17.416042, 14.029148 -17.410461, 14.017883 -17.40922, 14.008891 -17.411494, 13.992975 -17.421313, 13.980573 -17.423587, 13.957318 -17.419142, 13.942745 -17.408187, 13.896857 -17.349069, 13.884351 -17.338527, 13.790403 -17.288091, 13.69413 -17.236621, 13.606487 -17.167375, 13.521324 -17.1219, 13.530936 -17.093271, 13.522254 -17.076941, 13.507785 -17.063505, 13.494659 -17.024024, 13.479776 -17.010278, 13.458899 -17.001803, 13.435128 -16.999013, 13.417248 -16.993742, 13.382004 -16.970384, 13.363711 -16.964183, 13.345947 -16.968711, 13.321543 -16.974932, 13.315238 -16.974415, 13.308417 -16.970178, 13.293018 -16.973795, 13.267489 -16.98537, 13.267489 -16.977929, 13.257154 -16.98165, 13.245269 -16.98134, 13.222841 -16.977929, 13.212092 -16.972761, 13.205478 -16.963046, 13.19814 -16.957362, 13.184911 -16.964183, 13.166307 -16.951057, 13.1445 -16.952401, 13.121762 -16.959842, 13.014275 -16.977929, 12.961668 -17.007385, 12.930456 -17.014206, 12.911025 -17.023508, 12.887151 -17.029812, 12.882293 -17.039527, 12.880536 -17.050793, 12.876402 -17.059784, 12.867721 -17.065572, 12.849944 -17.070843, 12.842296 -17.074047, 12.833097 -17.081799, 12.824932 -17.096371, 12.818318 -17.104846, 12.784315 -17.115078, 12.739873 -17.135542, 12.704733 -17.164274, 12.685923 -17.173576, 12.660705 -17.17709, 12.63621 -17.185151, 12.591458 -17.222565, 12.56717 -17.234554, 12.554561 -17.235588, 12.519215 -17.227836, 12.46051 -17.223082, 12.4417 -17.216984, 12.417929 -17.203445, 12.407594 -17.204065, 12.393951 -17.21471, 12.379482 -17.220395, 12.314679 -17.218121, 12.242642 -17.224839, 12.239335 -17.220705, 12.236545 -17.2113, 12.23179 -17.201895, 12.222282 -17.197657, 12.21174 -17.195177, 12.200474 -17.189286, 12.190656 -17.182568, 12.185075 -17.17709, 12.181148 -17.169235, 12.179494 -17.161587, 12.177117 -17.154766, 12.170709 -17.149185, 12.16151 -17.146291, 12.156653 -17.149081, 12.151279 -17.153629, 12.13991 -17.156006, 12.105597 -17.146084, 12.095261 -17.139676, 12.08813 -17.139159, 12.082239 -17.14133, 12.075211 -17.142983, 12.028909 -17.148978, 11.98302 -17.161897, 11.942092 -17.180501, 11.894963 -17.214607, 11.853932 -17.233417, 11.835536 -17.245406, 11.829644 -17.253468, 11.827474 -17.260082, 11.82241 -17.264423, 11.80763 -17.26587, 11.796882 -17.263803, 11.779518 -17.254915, 11.766184 -17.252751, 11.766124 -17.252699, 11.766124 -17.245294, 11.757986 -17.23919, 11.752778 -17.229181, 11.750743 -17.216892, 11.751801 -17.203709, 11.758637 -17.165216, 11.759288 -17.032973, 11.772114 -16.947026, 11.780013 -16.869467, 11.768144 -16.79852, 11.768141 -16.768251, 11.785921 -16.76352, 11.79087 -16.806082, 11.80371 -16.824033, 11.820474 -16.781472, 11.817638 -16.704685, 11.82309 -16.678969, 11.813813 -16.583917, 11.821951 -16.479099, 11.779145 -16.099867, 11.779796 -16.080255, 11.784516 -16.06406, 11.806977 -16.018813, 11.784356 -15.972633, 11.736618 -15.900203, 11.735525 -15.85711, 11.751801 -15.799086, 11.76352 -15.789972, 11.784679 -15.777276, 11.804942 -15.76922, 11.813813 -15.774998, 11.81837 -15.792413, 11.829845 -15.797784, 11.844249 -15.793552, 11.894705 -15.746677, 11.903331 -15.731378, 11.906505 -15.71795, 11.907481 -15.690037, 11.910167 -15.676202, 11.930512 -15.652439, 11.992035 -15.617283, 12.005138 -15.594334, 12.008474 -15.565362, 12.028494 -15.490167, 12.040375 -15.463311, 12.03004 -15.436944, 12.033458 -15.407403, 12.046641 -15.350681, 12.056407 -15.233087, 12.061778 -15.217869, 12.070567 -15.204685, 12.104991 -15.172784, 12.112478 -15.170994, 12.127778 -15.178969, 12.135997 -15.178969, 12.145274 -15.168634, 12.14975 -15.151951, 12.148936 -15.13421, 12.142833 -15.120701, 12.12794 -15.113377, 12.117361 -15.116632, 12.112478 -15.115167, 12.115001 -15.093927, 12.124848 -15.062188, 12.151134 -15.002048, 12.156505 -14.967055, 12.160492 -14.953546, 12.170095 -14.942478, 12.181407 -14.932306, 12.190603 -14.922052, 12.196137 -14.908461, 12.205333 -14.853611, 12.205414 -14.840427, 12.207774 -14.833266, 12.21518 -14.824151, 12.221934 -14.823989, 12.228526 -14.82586, 12.235606 -14.823012, 12.244314 -14.809991, 12.252778 -14.782322, 12.272716 -14.750584, 12.272309 -14.732843, 12.268077 -14.714776, 12.266368 -14.696059, 12.268728 -14.683526, 12.276622 -14.66074, 12.292735 -14.587498, 12.299001 -14.507582, 12.303884 -14.494073, 12.319672 -14.468683, 12.331309 -14.433526, 12.342133 -14.36712, 12.345388 -14.302992, 12.342133 -14.27044, 12.331554 -14.236749, 12.324229 -14.220636, 12.322113 -14.213067, 12.321056 -14.202732, 12.319672 -14.197442, 12.315196 -14.187921, 12.31422 -14.182224, 12.315603 -14.174981, 12.319021 -14.175388, 12.323578 -14.177504, 12.332205 -14.173435, 12.344249 -14.171319, 12.348888 -14.168552, 12.353038 -14.160903, 12.355235 -14.151788, 12.355805 -14.142348, 12.355154 -14.133722, 12.347179 -14.119887, 12.334972 -14.106378, 12.328787 -14.092218, 12.344737 -14.066827, 12.349783 -14.045505, 12.369884 -14.021905, 12.374522 -14.007582, 12.377289 -13.992446, 12.383067 -13.977309, 12.399913 -13.951593, 12.402843 -13.941339, 12.404145 -13.906345, 12.407237 -13.890558, 12.415538 -13.878595, 12.439301 -13.87127, 12.445079 -13.866143, 12.450531 -13.862481, 12.458181 -13.863865, 12.471853 -13.874444, 12.478363 -13.878025, 12.485688 -13.87713, 12.502452 -13.855239, 12.508962 -13.819594, 12.516287 -13.715102, 12.512706 -13.616795, 12.51588 -13.604913, 12.530121 -13.578709, 12.533865 -13.565199, 12.523611 -13.551202, 12.522146 -13.544203, 12.523936 -13.539972, 12.526541 -13.536228, 12.527029 -13.50449, 12.524587 -13.489353, 12.514903 -13.457615, 12.512706 -13.442071, 12.51531 -13.425958, 12.522146 -13.410577, 12.53297 -13.399021, 12.547211 -13.394627, 12.568044 -13.392348, 12.57545 -13.385919, 12.580821 -13.375258, 12.594737 -13.360447, 12.611095 -13.351983, 12.624197 -13.349216, 12.634451 -13.343357, 12.642426 -13.325616, 12.64324 -13.316339, 12.64088 -13.308852, 12.637869 -13.302016, 12.636241 -13.29461, 12.638438 -13.288018, 12.647716 -13.27809, 12.650076 -13.271091, 12.656505 -13.255629, 12.685069 -13.240818, 12.69158 -13.229425, 12.694102 -13.218357, 12.699962 -13.217869, 12.706798 -13.221449, 12.712087 -13.222589, 12.730642 -13.210545, 12.731781 -13.208917, 12.749766 -13.201349, 12.761485 -13.192966, 12.827159 -13.109796, 12.855724 -13.089532, 12.865977 -13.078709, 12.883149 -13.052016, 12.944835 -12.982354, 12.958669 -12.955499, 12.958832 -12.929132, 12.949474 -12.902114, 12.934825 -12.873793, 12.927501 -12.842055, 12.939464 -12.816664, 12.985688 -12.770115, 13.009613 -12.756524, 13.075043 -12.695001, 13.101085 -12.68141, 13.109223 -12.674493, 13.126475 -12.644464, 13.132823 -12.636651, 13.161632 -12.611261, 13.178559 -12.601739, 13.198009 -12.599379, 13.198904 -12.603611, 13.221039 -12.607192, 13.232758 -12.613051, 13.264415 -12.582615, 13.276866 -12.578302, 13.294281 -12.58115, 13.311534 -12.594496, 13.32838 -12.599379, 13.364024 -12.592706, 13.392914 -12.570408, 13.437673 -12.523696, 13.459727 -12.507257, 13.469249 -12.494806, 13.472992 -12.479099, 13.474946 -12.437758, 13.479015 -12.419122, 13.486583 -12.400811, 13.505626 -12.371352, 13.568614 -12.311456, 13.57545 -12.311456, 13.57545 -12.318292, 13.564952 -12.322035, 13.558279 -12.328302, 13.556814 -12.336602, 13.561778 -12.346124, 13.577485 -12.330987, 13.597504 -12.298761, 13.630219 -12.263442, 13.639822 -12.248712, 13.663585 -12.16961, 13.69337 -12.110284, 13.705089 -12.078058, 13.71518 -12.008396, 13.722504 -11.992771, 13.73699 -11.977716, 13.74879 -11.961196, 13.758556 -11.942478, 13.766449 -11.920505, 13.782481 -11.820896, 13.792247 -11.790948, 13.794607 -11.770196, 13.77768 -11.589125, 13.781912 -11.481052, 13.785981 -11.458266, 13.794607 -11.440606, 13.786632 -11.431573, 13.787364 -11.420343, 13.79184 -11.407973, 13.794607 -11.395929, 13.7942 -11.341485, 13.795258 -11.332208, 13.800141 -11.321384, 13.819021 -11.295587, 13.821625 -11.283624, 13.821788 -11.25628, 13.846528 -11.113702, 13.847423 -11.055434, 13.848481 -10.98211, 13.843435 -10.945733, 13.828624 -10.91367, 13.739268 -10.78338, 13.731456 -10.764255, 13.73113 -10.747166, 13.739431 -10.730239, 13.769786 -10.694268, 13.771739 -10.677911, 13.764985 -10.662042, 13.75294 -10.646173, 13.711436 -10.612074, 13.705089 -10.601983, 13.700857 -10.590916, 13.60613 -10.478204, 13.542328 -10.42669, 13.527599 -10.406508, 13.522634 -10.387791, 13.520763 -10.320245, 13.517345 -10.298435, 13.508962 -10.277032, 13.455089 -10.201755, 13.442638 -10.179376, 13.431163 -10.130548, 13.415294 -10.11004, 13.37672 -10.070733, 13.317393 -9.976331999999999, 13.319835 -9.961601999999999, 13.329112 -9.947524, 13.335216 -9.926690000000001, 13.327159 -9.885999999999999, 13.305431 -9.842461999999999, 13.278005 -9.801690000000001, 13.198578 -9.708672999999999, 13.191173 -9.690037, 13.196137 -9.672784, 13.218598 -9.649184, 13.225352 -9.63128, 13.223888 -9.614190000000001, 13.217784 -9.594170999999999, 13.209646 -9.577406999999999, 13.201427 -9.570489, 13.202403 -9.559015, 13.169932 -9.466404000000001, 13.168305 -9.447361000000001, 13.170665 -9.402520000000001, 13.166515 -9.38714, 13.147716 -9.352145999999999, 13.143403 -9.330254999999999, 13.120616 -9.313084, 13.045421 -9.18906, 13.016856 -9.122817, 12.996349 -9.093845, 12.992361 -9.038017999999999, 13.030284 -8.959568000000001, 13.086436 -8.892348, 13.136567 -8.871026000000001, 13.114757 -8.902927, 13.042817 -8.962498, 13.006847 -9.045586999999999, 13.000173 -9.072035, 13.013031 -9.083754000000001, 13.023448 -9.074965000000001, 13.0796 -8.980645000000001, 13.081879 -8.970473, 13.086925 -8.959731, 13.098888 -8.956476, 13.112478 -8.956149999999999, 13.122895 -8.954197000000001, 13.137462 -8.940851, 13.148285 -8.907973, 13.160411 -8.887953, 13.185557 -8.861586000000001, 13.191173 -8.850517999999999, 13.192149 -8.839532, 13.190603 -8.816095000000001, 13.194591 -8.806084999999999, 13.224132 -8.773858000000001, 13.241547 -8.760675000000001, 13.259532 -8.754815000000001, 13.209727 -8.804295, 13.198009 -8.823175000000001, 13.212169 -8.814874, 13.245372 -8.800063, 13.263682 -8.788263000000001, 13.271658 -8.786554000000001, 13.278005 -8.782484999999999, 13.280528 -8.771661, 13.281016 -8.764744, 13.28297 -8.759454, 13.28712 -8.756036, 13.2942 -8.754815000000001, 13.35613 -8.763604000000001, 13.380138 -8.75506, 13.393321 -8.704278, 13.408051 -8.666925000000001, 13.410899 -8.651951, 13.406098 -8.63714, 13.37672 -8.596612, 13.365408 -8.566827, 13.352061 -8.499688000000001, 13.342052 -8.467461999999999, 13.355479 -8.464613999999999, 13.365001 -8.465916, 13.372569 -8.464451, 13.379893 -8.45379, 13.382579 -8.444024000000001, 13.383556 -8.368748, 13.378917 -8.347101, 13.369395 -8.323499999999999, 13.276866 -8.178888000000001, 13.264903 -8.169692, 13.250824 -8.124444, 13.242442 -8.107192, 13.234548 -8.095961000000001, 13.222504 -8.069594, 13.215505 -8.059177, 13.198009 -8.02158, 13.189301 -7.975763, 13.122895 -7.857354, 12.986308 -7.550661, 12.939283 -7.435969, 12.911473 -7.335111, 12.883895 -7.305754, 12.869055 -7.288986, 12.849963 -7.265919, 12.855192 -7.231279, 12.849857 -7.205053, 12.833263 -7.005792, 12.823416 -6.975193, 12.8213 -6.959161, 12.819672 -6.954034, 12.816661 -6.950372, 12.815196 -6.946954, 12.817882 -6.942153, 12.824555 -6.934259, 12.827485 -6.929132, 12.829926 -6.918878, 12.832774 -6.915785, 12.833507 -6.912286, 12.828787 -6.904555, 12.799571 -6.923272, 12.779633 -6.914727, 12.706879 -6.814223, 12.620425 -6.734823, 12.550632 -6.632639, 12.458181 -6.465753, 12.421954 -6.380988, 12.395576 -6.324453, 12.368826 -6.287693, 12.344543 -6.254147, 12.319149 -6.22014, 12.279447 -6.147705, 12.27506 -6.114769, 12.290294 -6.096612, 12.312655 -6.090545, 12.333648 -6.078436, 12.327012 -6.093811, 12.324781 -6.122353, 12.346873 -6.120113, 12.382207 -6.110168, 12.432959 -6.108984, 12.496928 -6.094594, 12.527768 -6.074826, 12.549799 -6.067123, 12.580642 -6.062695, 12.632823 -6.034356, 12.645518 -6.028985, 12.669817 -6.02971, 12.690752 -6.037358, 12.716095 -6.045002, 12.742538 -6.049356, 12.77117 -6.041659, 12.817431 -6.03175, 12.861469 -6.016347, 12.882386 -6.000986, 12.927569 -5.981218, 12.94627 -5.955991, 12.966074 -5.931853, 12.97266 -5.913214, 12.98588 -5.90114, 13.011139 -5.895606, 13.029867 -5.893381, 13.046372 -5.885679, 13.064015 -5.887838, 13.100597 -5.891209, 13.111827 -5.890232, 13.129161 -5.883559, 13.169688 -5.861749, 13.183849 -5.856459, 13.183913 -5.856427, 13.184911 -5.856386, 13.26935 -5.863517, 13.311001 -5.873956, 13.342524 -5.890595, 13.353582 -5.882637, 13.37384 -5.860933, 13.380454 -5.856386, 13.41332 -5.857006, 13.493212 -5.858453, 13.522668 -5.867548, 13.53476 -5.867031, 13.553053 -5.859383, 13.562562 -5.857419, 13.641523 -5.865791, 13.807146 -5.852458, 13.943572 -5.841606, 13.98233 -5.853285)), ((12.801058 -4.410014, 12.814804 -4.412908, 12.830307 -4.409704, 12.842296 -4.40433, 12.854285 -4.403089, 12.869891 -4.411978, 12.884464 -4.432131, 12.902447 -4.480191, 12.921981 -4.502308, 12.961565 -4.533831, 13.027194 -4.612172, 13.073703 -4.635323, 13.065331 -4.663952, 13.029571 -4.676871, 12.833201 -4.722656, 12.802918 -4.739813, 12.786175 -4.758416, 12.774806 -4.7795, 12.768295 -4.801928, 12.766228 -4.825182, 12.756926 -4.837275, 12.711968 -4.86332, 12.696982 -4.875619, 12.6883 -4.897013, 12.686233 -4.912826, 12.678895 -4.924298, 12.6544 -4.932566, 12.61585 -4.93546, 12.606961 -4.94223, 12.603964 -4.96259, 12.587738 -5.000314, 12.550427 -5.024188, 12.466505 -5.053127, 12.458133 -5.054161, 12.450692 -5.053127, 12.444387 -5.055091, 12.439426 -5.064806, 12.43891 -5.073281, 12.44077 -5.08403, 12.444697 -5.093538, 12.450382 -5.098809, 12.51198 -5.120513, 12.524072 -5.129091, 12.530687 -5.162681, 12.52769 -5.239472, 12.524382 -5.324738, 12.519731 -5.444214, 12.514977 -5.568755, 12.512497 -5.63366, 12.508983 -5.726264, 12.435499 -5.725541, 12.344445 -5.724611, 12.258972 -5.736703, 12.210555 -5.763465, 12.210541 -5.763442, 12.210297 -5.763116, 12.161794 -5.681248, 12.147227 -5.617934, 12.170177 -5.55462, 12.182384 -5.543227, 12.19337 -5.540623, 12.204845 -5.539809, 12.218598 -5.534112, 12.228282 -5.523696, 12.232188 -5.510919, 12.232188 -5.479587, 12.229259 -5.466892, 12.214122 -5.431248, 12.19639 -5.396097, 12.171362 -5.329088, 12.172091 -5.302095, 12.132349 -5.232529, 12.124612 -5.184828, 12.069737 -5.119783, 12.023123 -5.045994, 12.023285 -5.035252, 12.035151 -5.038469, 12.054771 -5.042354, 12.067823 -5.046253, 12.074346 -5.052106, 12.080892 -5.075509, 12.100527 -5.080022, 12.116815 -5.069629, 12.127259 -5.053366, 12.128561 -5.039711, 12.137715 -5.034497, 12.137054 -5.014996, 12.132476 -5.005899, 12.123331 -5.002655, 12.112885 -5.00201, 12.10505 -5.013067, 12.103085 -5.027376, 12.103738 -5.048836, 12.097859 -5.054043, 12.09067 -5.050798, 12.086108 -5.030636, 12.079588 -5.020232, 12.071752 -5.017634, 12.066529 -5.02674, 12.056727 -5.027397, 12.04496 -5.020902, 12.030556 -5.022867, 12.009608 -5.019631, 12.018072 -5.0086, 12.130815 -4.912929, 12.15717 -4.870554, 12.187555 -4.768028, 12.192206 -4.763481, 12.204092 -4.763481, 12.213497 -4.769165, 12.222592 -4.780637, 12.235201 -4.803995, 12.244089 -4.796347, 12.258559 -4.790766, 12.273752 -4.787149, 12.307651 -4.783635, 12.321914 -4.778157, 12.332353 -4.765858, 12.374107 -4.683176, 12.378862 -4.662298, 12.377621 -4.619407, 12.387026 -4.605454, 12.414105 -4.608865, 12.429091 -4.607521, 12.497201 -4.5853, 12.608098 -4.565043, 12.623808 -4.559255, 12.634867 -4.54768, 12.656571 -4.507372, 12.670523 -4.491353, 12.686853 -4.47833, 12.70494 -4.467995, 12.718479 -4.451872, 12.726541 -4.427997, 12.737909 -4.40495, 12.761681 -4.391204, 12.775426 -4.396165, 12.782656 -4.400071, 12.801058 -4.410014)))"; - void test1() { DB::CartesianPolygon green, blue; @@ -164,43 +158,6 @@ void test3() printMultiPolygon(output); } - -void test4() -{ - DB::CartesianMultiPolygon green, blue; - boost::geometry::read_wkt(example, green); - - boost::geometry::read_wkt(example, blue); - - // boost::geometry::correct(green); - // boost::geometry::correct(blue); - - - // boost::geometry::correct(green); - - for (auto & polygon : green) - mercator(polygon); - - for (auto & polygon: blue) - mercator(polygon); - - std::cout << "----" << std::endl; - std::cout << boost::geometry::wkt(green) << std::endl; - std::cout << "----" << std::endl; - - DB::CartesianMultiPolygon output; - boost::geometry::intersection(green, blue, output); - - reverseMercator(output); - - // boost::geometry::correct(output); - // std::cout << boost::geometry::wkt(output) << std::endl; - - // printMultiPolygon(output); - - // std::cout << std::boolalpha << boost::geometry::equals(output, green) << std::endl; -} - int main(int argc, char ** argv) { (void) argc; @@ -208,6 +165,5 @@ int main(int argc, char ** argv) test1(); test2(); test3(); - test4(); return 0; } diff --git a/src/Functions/ya.make b/src/Functions/ya.make index a373ec88d54..3ac64828b9c 100644 --- a/src/Functions/ya.make +++ b/src/Functions/ya.make @@ -335,7 +335,6 @@ SRCS( map.cpp match.cpp materialize.cpp - mercatorConverters.cpp minus.cpp modulo.cpp moduloOrZero.cpp diff --git a/tests/queries/0_stateless/01301_polygons_within.reference b/tests/queries/0_stateless/01301_polygons_within.reference index 147806964e6..d922e69f725 100644 --- a/tests/queries/0_stateless/01301_polygons_within.reference +++ b/tests/queries/0_stateless/01301_polygons_within.reference @@ -2,6 +2,7 @@ 1 0 1 +<<<<<<< HEAD 0 1 -------- MultiPolygon with Polygon @@ -13,3 +14,5 @@ -------- Polygon with Polygon with Holes 0 0 +======= +>>>>>>> parent of d4258922d1... more mercator diff --git a/tests/queries/0_stateless/01301_polygons_within.sql b/tests/queries/0_stateless/01301_polygons_within.sql index ff8342e783d..901c7909af7 100644 --- a/tests/queries/0_stateless/01301_polygons_within.sql +++ b/tests/queries/0_stateless/01301_polygons_within.sql @@ -4,18 +4,12 @@ select polygonsWithinCartesian([[[(2., 2.), (2., 3.), (3., 3.), (3., 2.)]]], [[[ select polygonsWithinSpherical([[[(4.3613577, 50.8651821), (4.349556, 50.8535879), (4.3602419, 50.8435626), (4.3830299, 50.8428851), (4.3904543, 50.8564867), (4.3613148, 50.8651279)]]], [[[(4.346693, 50.858306), (4.367945, 50.852455), (4.366227, 50.840809), (4.344961, 50.833264), (4.338074, 50.848677), (4.346693, 50.858306)]]]); select polygonsWithinSpherical([[[(4.3501568, 50.8518269), (4.3444920, 50.8439961), (4.3565941, 50.8443213), (4.3501568, 50.8518269)]]], [[[(4.3679450, 50.8524550),(4.3466930, 50.8583060),(4.3380740, 50.8486770),(4.3449610, 50.8332640),(4.3662270, 50.8408090),(4.3679450, 50.8524550)]]]); -select polygonsWithinMercator([[[(4.3613577, 50.8651821), (4.349556, 50.8535879), (4.3602419, 50.8435626), (4.3830299, 50.8428851), (4.3904543, 50.8564867), (4.3613148, 50.8651279)]]], [[[(4.346693, 50.858306), (4.367945, 50.852455), (4.366227, 50.840809), (4.344961, 50.833264), (4.338074, 50.848677), (4.346693, 50.858306)]]]); -select polygonsWithinMercator([[[(4.3501568, 50.8518269), (4.3444920, 50.8439961), (4.3565941, 50.8443213), (4.3501568, 50.8518269)]]], [[[(4.3679450, 50.8524550),(4.3466930, 50.8583060),(4.3380740, 50.8486770),(4.3449610, 50.8332640),(4.3662270, 50.8408090),(4.3679450, 50.8524550)]]]); - - select '-------- MultiPolygon with Polygon'; select polygonsWithinSpherical([[(29.453587685533865,59.779570356240356),(29.393139070478895,52.276266797422124),(40.636581470703206,59.38168915000267),(41.21084331372543,59.103467777099866),(29.786055068336193,52.146627480315004),(31.23682182965546,52.16517054781818),(41.69443223416517,58.85424941916091),(42.51048853740727,58.47703162291134),(32.59691566839227,52.22075341251539),(34.289476889931414,52.22075341251539),(43.02430176537451,58.07974369546071),(43.02430176537451,57.25537683364851),(35.468224883503325,52.2022335126388),(37.16078610504247,52.23926559241349),(43.02430176537451,56.26136189644947),(43.02430176537451,55.326904361850836),(38.33953409861437,52.16517054781818),(40.09254393520848,52.16517054781818),(44.4146199116388,55.3097062225408),(44.47506852669377,59.80998197603594),(39.72985224487867,59.931351417569715),(30.23941968124846,53.67744677450975),(30.20919537372098,54.63314259659509),(38.73245009647167,59.94649146557819),(37.2816833351524,59.97675082987618),(30.23941968124846,55.2752875586599),(30.33009260383092,56.19415599955667),(36.28428118674541,59.96162460231375),(34.863738732953635,59.97675082987618),(30.178971066193498,56.97640788219866),(30.178971066193498,57.91957806959033),(33.65476643185424,59.94649146557819),(32.32489690064491,59.94649146557819),(30.481214141468342,58.85424941916091),(30.571887064050795,59.99187015036608),(29.453587685533865,59.779570356240356)]], [[[(33.473420586689336,58.85424941916091),(32.23422397806246,58.492830557036),(32.173775363007486,58.03176922751564),(31.508840597402823,57.499784781503735),(31.750635057622702,56.86092686957355),(31.508840597402823,55.941082594334574),(32.20399967053497,55.515591939372456),(31.84130798020516,54.998862226280465),(31.418167674820367,54.422670886434275),(32.47601843828233,53.83826377018255),(32.08310244042503,53.408048308050866),(33.171177511414484,52.82758702113742),(34.77306581037117,52.91880107773494),(34.77306581037117,53.784726518357985),(34.108131044766516,54.17574726780569),(35.07530888564602,54.59813930694554),(34.25925258240394,54.96417435716029),(35.01486027059106,55.361278263643584),(33.50364489421682,55.37845402950552),(32.7480372060297,55.90721384574556),(35.67979503619571,55.68634475630185),(32.83871012861215,56.311688992608396),(34.591719965206266,56.29492065473883),(35.7100193437232,56.311688992608396),(33.83611227701915,56.695333481003644),(32.95960735872209,56.9434497616887),(36.072711034053015,57.091531913901434),(33.171177511414484,57.33702717078384),(36.193608264162954,57.499784781503735),(33.23162612646945,57.77481561306047),(36.43540272438284,58.04776787540811),(33.62454212432676,58.27099811968307),(36.344729801800376,58.54018474404165),(33.83611227701915,58.68186423448108),(34.74284150284369,59.565911441555244),(33.473420586689336,58.85424941916091)]], [[(34.65216858026123,58.91672306881671),(37.19101041256995,58.68186423448108),(36.01226241899805,58.28688958537609),(37.16078610504247,58.04776787540811),(35.74024365125068,57.79092907387934),(37.009664567405046,57.499784781503735),(35.77046795877817,57.25537683364851),(36.979440259877556,57.07510745541089),(34.22902827487645,56.794777197297435),(36.7074214921302,56.210968525786996),(34.712617195316206,56.10998276812964),(36.55629995449277,55.63519693782703),(35.13575750070099,55.53270067649592),(36.43540272438284,55.34409504165558),(34.83351442542614,55.01619492319591),(35.61934642114075,54.49294870011772),(34.89396304048112,54.12264226523038),(35.37755196092087,53.046178687628185),(37.43280487278982,52.95523300597458),(35.92158949641559,53.80257986695776),(36.91899164482259,53.856094327816805),(36.01226241899805,54.75541714463799),(37.765272255592166,55.189110239786885),(36.828318722240134,55.44708256557195),(38.03729102333953,55.652253637168315),(36.64697287707522,55.941082594334574),(38.21863686850443,56.05939028508024),(36.37495410932787,56.64551287174558),(38.30930979108689,56.992876013526654),(37.16078610504247,57.25537683364851),(38.127963945921984,57.516020773674256),(37.43280487278982,57.710289827306724),(38.33953409861437,57.935626886818994),(37.40258056526235,58.31865112960426),(38.58132855883426,58.744648733419496),(37.31190764267989,59.02578062465136),(34.65216858026123,58.91672306881671)]], [[(38.52087994377928,59.11898412389468),(39.54850639971376,58.713270635642914),(38.369758406141855,58.28688958537609),(38.85334732658162,58.06375936407028),(38.33953409861437,57.710289827306724),(38.73245009647167,57.48354156434209),(38.21863686850443,57.271721400459285),(38.97424455669155,56.87744603722649),(37.463029180317314,56.5623320541159),(38.94402024916407,56.05939028508024),(38.18841256097694,55.856355210835915),(38.490655636251795,55.53270067649592),(37.795496563119656,55.39562234093384),(38.30930979108689,55.154587013355666),(36.7074214921302,54.65063295250911),(37.31190764267989,53.92734063371401),(36.979440259877556,53.58783775557231),(37.855945178174615,52.91880107773497),(39.57873070724124,52.69956490610895),(38.33953409861437,53.281741738901104),(40.00187101262603,53.35396273604752),(39.54850639971376,53.58783775557231),(40.24366547284591,53.58783775557231),(39.97164670509855,53.98069568468355),(40.60635716317572,54.03398248547225),(40.39478701048334,54.44025165268903),(39.54850639971376,54.56310590284329),(39.54850639971376,54.87732350170489),(40.39478701048334,54.87732350170489),(40.39478701048334,55.24083903654295),(39.82052516746112,55.2752875586599),(39.760076552406154,55.75443792473942),(40.57613285564824,55.78844000174894),(40.425011318010824,56.19415599955667),(39.82052516746112,56.07626182891758),(39.79030085993364,56.41214455508424),(40.48545993306579,56.495655446714636),(40.33433839542836,56.95993246553937),(39.79030085993364,56.992876013526654),(39.72985224487867,57.46729112028032),(40.33433839542836,57.46729112028032),(40.24366547284591,58.04776787540811),(39.63917932229622,58.04776787540811),(39.63917932229622,58.382088724871295),(40.33433839542836,58.382088724871295),(40.45523562553831,58.9011152358548),(38.52087994377928,59.11898412389468)]]]) format TSV; -select polygonsWithinMercator([[(29.453587685533865,59.779570356240356),(29.393139070478895,52.276266797422124),(40.636581470703206,59.38168915000267),(41.21084331372543,59.103467777099866),(29.786055068336193,52.146627480315004),(31.23682182965546,52.16517054781818),(41.69443223416517,58.85424941916091),(42.51048853740727,58.47703162291134),(32.59691566839227,52.22075341251539),(34.289476889931414,52.22075341251539),(43.02430176537451,58.07974369546071),(43.02430176537451,57.25537683364851),(35.468224883503325,52.2022335126388),(37.16078610504247,52.23926559241349),(43.02430176537451,56.26136189644947),(43.02430176537451,55.326904361850836),(38.33953409861437,52.16517054781818),(40.09254393520848,52.16517054781818),(44.4146199116388,55.3097062225408),(44.47506852669377,59.80998197603594),(39.72985224487867,59.931351417569715),(30.23941968124846,53.67744677450975),(30.20919537372098,54.63314259659509),(38.73245009647167,59.94649146557819),(37.2816833351524,59.97675082987618),(30.23941968124846,55.2752875586599),(30.33009260383092,56.19415599955667),(36.28428118674541,59.96162460231375),(34.863738732953635,59.97675082987618),(30.178971066193498,56.97640788219866),(30.178971066193498,57.91957806959033),(33.65476643185424,59.94649146557819),(32.32489690064491,59.94649146557819),(30.481214141468342,58.85424941916091),(30.571887064050795,59.99187015036608),(29.453587685533865,59.779570356240356)]], [[[(33.473420586689336,58.85424941916091),(32.23422397806246,58.492830557036),(32.173775363007486,58.03176922751564),(31.508840597402823,57.499784781503735),(31.750635057622702,56.86092686957355),(31.508840597402823,55.941082594334574),(32.20399967053497,55.515591939372456),(31.84130798020516,54.998862226280465),(31.418167674820367,54.422670886434275),(32.47601843828233,53.83826377018255),(32.08310244042503,53.408048308050866),(33.171177511414484,52.82758702113742),(34.77306581037117,52.91880107773494),(34.77306581037117,53.784726518357985),(34.108131044766516,54.17574726780569),(35.07530888564602,54.59813930694554),(34.25925258240394,54.96417435716029),(35.01486027059106,55.361278263643584),(33.50364489421682,55.37845402950552),(32.7480372060297,55.90721384574556),(35.67979503619571,55.68634475630185),(32.83871012861215,56.311688992608396),(34.591719965206266,56.29492065473883),(35.7100193437232,56.311688992608396),(33.83611227701915,56.695333481003644),(32.95960735872209,56.9434497616887),(36.072711034053015,57.091531913901434),(33.171177511414484,57.33702717078384),(36.193608264162954,57.499784781503735),(33.23162612646945,57.77481561306047),(36.43540272438284,58.04776787540811),(33.62454212432676,58.27099811968307),(36.344729801800376,58.54018474404165),(33.83611227701915,58.68186423448108),(34.74284150284369,59.565911441555244),(33.473420586689336,58.85424941916091)]], [[(34.65216858026123,58.91672306881671),(37.19101041256995,58.68186423448108),(36.01226241899805,58.28688958537609),(37.16078610504247,58.04776787540811),(35.74024365125068,57.79092907387934),(37.009664567405046,57.499784781503735),(35.77046795877817,57.25537683364851),(36.979440259877556,57.07510745541089),(34.22902827487645,56.794777197297435),(36.7074214921302,56.210968525786996),(34.712617195316206,56.10998276812964),(36.55629995449277,55.63519693782703),(35.13575750070099,55.53270067649592),(36.43540272438284,55.34409504165558),(34.83351442542614,55.01619492319591),(35.61934642114075,54.49294870011772),(34.89396304048112,54.12264226523038),(35.37755196092087,53.046178687628185),(37.43280487278982,52.95523300597458),(35.92158949641559,53.80257986695776),(36.91899164482259,53.856094327816805),(36.01226241899805,54.75541714463799),(37.765272255592166,55.189110239786885),(36.828318722240134,55.44708256557195),(38.03729102333953,55.652253637168315),(36.64697287707522,55.941082594334574),(38.21863686850443,56.05939028508024),(36.37495410932787,56.64551287174558),(38.30930979108689,56.992876013526654),(37.16078610504247,57.25537683364851),(38.127963945921984,57.516020773674256),(37.43280487278982,57.710289827306724),(38.33953409861437,57.935626886818994),(37.40258056526235,58.31865112960426),(38.58132855883426,58.744648733419496),(37.31190764267989,59.02578062465136),(34.65216858026123,58.91672306881671)]], [[(38.52087994377928,59.11898412389468),(39.54850639971376,58.713270635642914),(38.369758406141855,58.28688958537609),(38.85334732658162,58.06375936407028),(38.33953409861437,57.710289827306724),(38.73245009647167,57.48354156434209),(38.21863686850443,57.271721400459285),(38.97424455669155,56.87744603722649),(37.463029180317314,56.5623320541159),(38.94402024916407,56.05939028508024),(38.18841256097694,55.856355210835915),(38.490655636251795,55.53270067649592),(37.795496563119656,55.39562234093384),(38.30930979108689,55.154587013355666),(36.7074214921302,54.65063295250911),(37.31190764267989,53.92734063371401),(36.979440259877556,53.58783775557231),(37.855945178174615,52.91880107773497),(39.57873070724124,52.69956490610895),(38.33953409861437,53.281741738901104),(40.00187101262603,53.35396273604752),(39.54850639971376,53.58783775557231),(40.24366547284591,53.58783775557231),(39.97164670509855,53.98069568468355),(40.60635716317572,54.03398248547225),(40.39478701048334,54.44025165268903),(39.54850639971376,54.56310590284329),(39.54850639971376,54.87732350170489),(40.39478701048334,54.87732350170489),(40.39478701048334,55.24083903654295),(39.82052516746112,55.2752875586599),(39.760076552406154,55.75443792473942),(40.57613285564824,55.78844000174894),(40.425011318010824,56.19415599955667),(39.82052516746112,56.07626182891758),(39.79030085993364,56.41214455508424),(40.48545993306579,56.495655446714636),(40.33433839542836,56.95993246553937),(39.79030085993364,56.992876013526654),(39.72985224487867,57.46729112028032),(40.33433839542836,57.46729112028032),(40.24366547284591,58.04776787540811),(39.63917932229622,58.04776787540811),(39.63917932229622,58.382088724871295),(40.33433839542836,58.382088724871295),(40.45523562553831,58.9011152358548),(38.52087994377928,59.11898412389468)]]]) format TSV; select '-------- MultiPolygon with Polygon with Holes'; select polygonsWithinSpherical([[[(33.473420586689336,58.85424941916091),(32.23422397806246,58.492830557036),(32.173775363007486,58.03176922751564),(31.508840597402823,57.499784781503735),(31.750635057622702,56.86092686957355),(31.508840597402823,55.941082594334574),(32.20399967053497,55.515591939372456),(31.84130798020516,54.998862226280465),(31.418167674820367,54.422670886434275),(32.47601843828233,53.83826377018255),(32.08310244042503,53.408048308050866),(33.171177511414484,52.82758702113742),(34.77306581037117,52.91880107773494),(34.77306581037117,53.784726518357985),(34.108131044766516,54.17574726780569),(35.07530888564602,54.59813930694554),(34.25925258240394,54.96417435716029),(35.01486027059106,55.361278263643584),(33.50364489421682,55.37845402950552),(32.7480372060297,55.90721384574556),(35.67979503619571,55.68634475630185),(32.83871012861215,56.311688992608396),(34.591719965206266,56.29492065473883),(35.7100193437232,56.311688992608396),(33.83611227701915,56.695333481003644),(32.95960735872209,56.9434497616887),(36.072711034053015,57.091531913901434),(33.171177511414484,57.33702717078384),(36.193608264162954,57.499784781503735),(33.23162612646945,57.77481561306047),(36.43540272438284,58.04776787540811),(33.62454212432676,58.27099811968307),(36.344729801800376,58.54018474404165),(33.83611227701915,58.68186423448108),(34.74284150284369,59.565911441555244),(33.473420586689336,58.85424941916091)]], [[(34.65216858026123,58.91672306881671),(37.19101041256995,58.68186423448108),(36.01226241899805,58.28688958537609),(37.16078610504247,58.04776787540811),(35.74024365125068,57.79092907387934),(37.009664567405046,57.499784781503735),(35.77046795877817,57.25537683364851),(36.979440259877556,57.07510745541089),(34.22902827487645,56.794777197297435),(36.7074214921302,56.210968525786996),(34.712617195316206,56.10998276812964),(36.55629995449277,55.63519693782703),(35.13575750070099,55.53270067649592),(36.43540272438284,55.34409504165558),(34.83351442542614,55.01619492319591),(35.61934642114075,54.49294870011772),(34.89396304048112,54.12264226523038),(35.37755196092087,53.046178687628185),(37.43280487278982,52.95523300597458),(35.92158949641559,53.80257986695776),(36.91899164482259,53.856094327816805),(36.01226241899805,54.75541714463799),(37.765272255592166,55.189110239786885),(36.828318722240134,55.44708256557195),(38.03729102333953,55.652253637168315),(36.64697287707522,55.941082594334574),(38.21863686850443,56.05939028508024),(36.37495410932787,56.64551287174558),(38.30930979108689,56.992876013526654),(37.16078610504247,57.25537683364851),(38.127963945921984,57.516020773674256),(37.43280487278982,57.710289827306724),(38.33953409861437,57.935626886818994),(37.40258056526235,58.31865112960426),(38.58132855883426,58.744648733419496),(37.31190764267989,59.02578062465136),(34.65216858026123,58.91672306881671)]], [[(38.52087994377928,59.11898412389468),(39.54850639971376,58.713270635642914),(38.369758406141855,58.28688958537609),(38.85334732658162,58.06375936407028),(38.33953409861437,57.710289827306724),(38.73245009647167,57.48354156434209),(38.21863686850443,57.271721400459285),(38.97424455669155,56.87744603722649),(37.463029180317314,56.5623320541159),(38.94402024916407,56.05939028508024),(38.18841256097694,55.856355210835915),(38.490655636251795,55.53270067649592),(37.795496563119656,55.39562234093384),(38.30930979108689,55.154587013355666),(36.7074214921302,54.65063295250911),(37.31190764267989,53.92734063371401),(36.979440259877556,53.58783775557231),(37.855945178174615,52.91880107773497),(39.57873070724124,52.69956490610895),(38.33953409861437,53.281741738901104),(40.00187101262603,53.35396273604752),(39.54850639971376,53.58783775557231),(40.24366547284591,53.58783775557231),(39.97164670509855,53.98069568468355),(40.60635716317572,54.03398248547225),(40.39478701048334,54.44025165268903),(39.54850639971376,54.56310590284329),(39.54850639971376,54.87732350170489),(40.39478701048334,54.87732350170489),(40.39478701048334,55.24083903654295),(39.82052516746112,55.2752875586599),(39.760076552406154,55.75443792473942),(40.57613285564824,55.78844000174894),(40.425011318010824,56.19415599955667),(39.82052516746112,56.07626182891758),(39.79030085993364,56.41214455508424),(40.48545993306579,56.495655446714636),(40.33433839542836,56.95993246553937),(39.79030085993364,56.992876013526654),(39.72985224487867,57.46729112028032),(40.33433839542836,57.46729112028032),(40.24366547284591,58.04776787540811),(39.63917932229622,58.04776787540811),(39.63917932229622,58.382088724871295),(40.33433839542836,58.382088724871295),(40.45523562553831,58.9011152358548),(38.52087994377928,59.11898412389468)]]], [[(24.367675781249993,61.45977057029751),(19.577636718749993,58.67693767258692),(19.577636718749993,57.492213666700735),(19.445800781249996,55.87531083569678),(19.445800781249996,54.085173420886775),(17.468261718749996,53.014783245859235),(20.017089843749993,51.563412328675895),(21.203613281249993,50.205033264943324),(26.125488281249993,50.40151532278236),(27.22412109374999,48.980216985374994),(32.80517578124999,49.525208341974405),(35.26611328124999,48.74894534343292),(36.93603515624999,49.66762782262194),(42.56103515625,48.77791275550183),(43.92333984374999,49.8096315635631),(47.17529296875,49.152969656170455),(49.28466796875,50.54136296522162),(48.05419921875,51.17934297928929),(51.39404296875,52.48278022207825),(50.64697265625,53.014783245859235),(52.88818359375,53.93021986394004),(51.65771484374999,54.29088164657006),(52.66845703125,55.825973254619015),(50.25146484375,56.145549500679095),(51.92138671875,57.914847767009206),(49.15283203125,58.17070248348605),(49.59228515625,60.086762746260064),(47.043457031249986,59.88893689676584),(43.57177734375,61.37567331572748),(42.64892578125,60.630101766266705),(36.89208984374999,62.000904713685856),(36.01318359374999,61.143235250840576),(31.398925781249993,62.02152819100766),(30.563964843749996,61.05828537037917),(26.872558593749993,61.71070595883174),(26.652832031249993,61.10078883158897),(24.367675781249993,61.45977057029751)], [(24.455566406249993,59.42272750081452),(21.203613281249993,58.49369382056807),(21.335449218749993,56.89700392127261),(21.599121093749993,55.92458580482949),(25.202636718749993,55.998380955359636),(28.850097656249993,57.06463027327854),(27.09228515625,57.844750992890994),(28.806152343749996,59.17592824927138),(26.257324218749993,59.17592824927138),(24.455566406249993,59.42272750081452)], [(35.13427734375,59.84481485969107),(31.970214843749993,58.97266715450152),(33.20068359374999,56.776808316568406),(36.67236328125,56.41390137600675),(39.08935546874999,57.25528054528888),(42.69287109374999,58.03137242177638),(40.89111328124999,59.26588062825809),(37.28759765625,58.722598828043374),(37.11181640624999,59.66774058164964),(35.13427734375,59.84481485969107)], [(29.157714843749993,55.75184939173528),(22.565917968749993,55.128649068488784),(22.565917968749993,53.54030739150019),(22.038574218749996,51.48138289610097),(26.257324218749993,51.42661449707484),(30.124511718749993,50.54136296522162),(32.18994140624999,51.17934297928929),(30.124511718749993,53.173119202640635),(35.09033203124999,53.173119202640635),(33.11279296875,54.085173420886775),(29.597167968749993,55.50374985927513),(29.157714843749993,55.75184939173528)], [(42.82470703125,56.58369172128337),(36.584472656249986,55.329144408405085),(37.99072265625,53.592504809039355),(34.95849609374999,51.48138289610097),(36.54052734374999,50.40151532278236),(39.66064453124999,50.289339253291786),(39.79248046875,52.13348804077148),(41.77001953125,50.68079714532166),(44.49462890624999,51.97134580885171),(47.30712890624999,52.509534770327264),(44.05517578125,53.54030739150019),(46.60400390625,53.696706475303245),(47.61474609375,55.40406982700608),(45.37353515625,55.40406982700608),(42.82470703125,56.58369172128337)]]) format TSV; -select polygonsWithinMercator([[[(33.473420586689336,58.85424941916091),(32.23422397806246,58.492830557036),(32.173775363007486,58.03176922751564),(31.508840597402823,57.499784781503735),(31.750635057622702,56.86092686957355),(31.508840597402823,55.941082594334574),(32.20399967053497,55.515591939372456),(31.84130798020516,54.998862226280465),(31.418167674820367,54.422670886434275),(32.47601843828233,53.83826377018255),(32.08310244042503,53.408048308050866),(33.171177511414484,52.82758702113742),(34.77306581037117,52.91880107773494),(34.77306581037117,53.784726518357985),(34.108131044766516,54.17574726780569),(35.07530888564602,54.59813930694554),(34.25925258240394,54.96417435716029),(35.01486027059106,55.361278263643584),(33.50364489421682,55.37845402950552),(32.7480372060297,55.90721384574556),(35.67979503619571,55.68634475630185),(32.83871012861215,56.311688992608396),(34.591719965206266,56.29492065473883),(35.7100193437232,56.311688992608396),(33.83611227701915,56.695333481003644),(32.95960735872209,56.9434497616887),(36.072711034053015,57.091531913901434),(33.171177511414484,57.33702717078384),(36.193608264162954,57.499784781503735),(33.23162612646945,57.77481561306047),(36.43540272438284,58.04776787540811),(33.62454212432676,58.27099811968307),(36.344729801800376,58.54018474404165),(33.83611227701915,58.68186423448108),(34.74284150284369,59.565911441555244),(33.473420586689336,58.85424941916091)]], [[(34.65216858026123,58.91672306881671),(37.19101041256995,58.68186423448108),(36.01226241899805,58.28688958537609),(37.16078610504247,58.04776787540811),(35.74024365125068,57.79092907387934),(37.009664567405046,57.499784781503735),(35.77046795877817,57.25537683364851),(36.979440259877556,57.07510745541089),(34.22902827487645,56.794777197297435),(36.7074214921302,56.210968525786996),(34.712617195316206,56.10998276812964),(36.55629995449277,55.63519693782703),(35.13575750070099,55.53270067649592),(36.43540272438284,55.34409504165558),(34.83351442542614,55.01619492319591),(35.61934642114075,54.49294870011772),(34.89396304048112,54.12264226523038),(35.37755196092087,53.046178687628185),(37.43280487278982,52.95523300597458),(35.92158949641559,53.80257986695776),(36.91899164482259,53.856094327816805),(36.01226241899805,54.75541714463799),(37.765272255592166,55.189110239786885),(36.828318722240134,55.44708256557195),(38.03729102333953,55.652253637168315),(36.64697287707522,55.941082594334574),(38.21863686850443,56.05939028508024),(36.37495410932787,56.64551287174558),(38.30930979108689,56.992876013526654),(37.16078610504247,57.25537683364851),(38.127963945921984,57.516020773674256),(37.43280487278982,57.710289827306724),(38.33953409861437,57.935626886818994),(37.40258056526235,58.31865112960426),(38.58132855883426,58.744648733419496),(37.31190764267989,59.02578062465136),(34.65216858026123,58.91672306881671)]], [[(38.52087994377928,59.11898412389468),(39.54850639971376,58.713270635642914),(38.369758406141855,58.28688958537609),(38.85334732658162,58.06375936407028),(38.33953409861437,57.710289827306724),(38.73245009647167,57.48354156434209),(38.21863686850443,57.271721400459285),(38.97424455669155,56.87744603722649),(37.463029180317314,56.5623320541159),(38.94402024916407,56.05939028508024),(38.18841256097694,55.856355210835915),(38.490655636251795,55.53270067649592),(37.795496563119656,55.39562234093384),(38.30930979108689,55.154587013355666),(36.7074214921302,54.65063295250911),(37.31190764267989,53.92734063371401),(36.979440259877556,53.58783775557231),(37.855945178174615,52.91880107773497),(39.57873070724124,52.69956490610895),(38.33953409861437,53.281741738901104),(40.00187101262603,53.35396273604752),(39.54850639971376,53.58783775557231),(40.24366547284591,53.58783775557231),(39.97164670509855,53.98069568468355),(40.60635716317572,54.03398248547225),(40.39478701048334,54.44025165268903),(39.54850639971376,54.56310590284329),(39.54850639971376,54.87732350170489),(40.39478701048334,54.87732350170489),(40.39478701048334,55.24083903654295),(39.82052516746112,55.2752875586599),(39.760076552406154,55.75443792473942),(40.57613285564824,55.78844000174894),(40.425011318010824,56.19415599955667),(39.82052516746112,56.07626182891758),(39.79030085993364,56.41214455508424),(40.48545993306579,56.495655446714636),(40.33433839542836,56.95993246553937),(39.79030085993364,56.992876013526654),(39.72985224487867,57.46729112028032),(40.33433839542836,57.46729112028032),(40.24366547284591,58.04776787540811),(39.63917932229622,58.04776787540811),(39.63917932229622,58.382088724871295),(40.33433839542836,58.382088724871295),(40.45523562553831,58.9011152358548),(38.52087994377928,59.11898412389468)]]], [[(24.367675781249993,61.45977057029751),(19.577636718749993,58.67693767258692),(19.577636718749993,57.492213666700735),(19.445800781249996,55.87531083569678),(19.445800781249996,54.085173420886775),(17.468261718749996,53.014783245859235),(20.017089843749993,51.563412328675895),(21.203613281249993,50.205033264943324),(26.125488281249993,50.40151532278236),(27.22412109374999,48.980216985374994),(32.80517578124999,49.525208341974405),(35.26611328124999,48.74894534343292),(36.93603515624999,49.66762782262194),(42.56103515625,48.77791275550183),(43.92333984374999,49.8096315635631),(47.17529296875,49.152969656170455),(49.28466796875,50.54136296522162),(48.05419921875,51.17934297928929),(51.39404296875,52.48278022207825),(50.64697265625,53.014783245859235),(52.88818359375,53.93021986394004),(51.65771484374999,54.29088164657006),(52.66845703125,55.825973254619015),(50.25146484375,56.145549500679095),(51.92138671875,57.914847767009206),(49.15283203125,58.17070248348605),(49.59228515625,60.086762746260064),(47.043457031249986,59.88893689676584),(43.57177734375,61.37567331572748),(42.64892578125,60.630101766266705),(36.89208984374999,62.000904713685856),(36.01318359374999,61.143235250840576),(31.398925781249993,62.02152819100766),(30.563964843749996,61.05828537037917),(26.872558593749993,61.71070595883174),(26.652832031249993,61.10078883158897),(24.367675781249993,61.45977057029751)], [(24.455566406249993,59.42272750081452),(21.203613281249993,58.49369382056807),(21.335449218749993,56.89700392127261),(21.599121093749993,55.92458580482949),(25.202636718749993,55.998380955359636),(28.850097656249993,57.06463027327854),(27.09228515625,57.844750992890994),(28.806152343749996,59.17592824927138),(26.257324218749993,59.17592824927138),(24.455566406249993,59.42272750081452)], [(35.13427734375,59.84481485969107),(31.970214843749993,58.97266715450152),(33.20068359374999,56.776808316568406),(36.67236328125,56.41390137600675),(39.08935546874999,57.25528054528888),(42.69287109374999,58.03137242177638),(40.89111328124999,59.26588062825809),(37.28759765625,58.722598828043374),(37.11181640624999,59.66774058164964),(35.13427734375,59.84481485969107)], [(29.157714843749993,55.75184939173528),(22.565917968749993,55.128649068488784),(22.565917968749993,53.54030739150019),(22.038574218749996,51.48138289610097),(26.257324218749993,51.42661449707484),(30.124511718749993,50.54136296522162),(32.18994140624999,51.17934297928929),(30.124511718749993,53.173119202640635),(35.09033203124999,53.173119202640635),(33.11279296875,54.085173420886775),(29.597167968749993,55.50374985927513),(29.157714843749993,55.75184939173528)], [(42.82470703125,56.58369172128337),(36.584472656249986,55.329144408405085),(37.99072265625,53.592504809039355),(34.95849609374999,51.48138289610097),(36.54052734374999,50.40151532278236),(39.66064453124999,50.289339253291786),(39.79248046875,52.13348804077148),(41.77001953125,50.68079714532166),(44.49462890624999,51.97134580885171),(47.30712890624999,52.509534770327264),(44.05517578125,53.54030739150019),(46.60400390625,53.696706475303245),(47.61474609375,55.40406982700608),(45.37353515625,55.40406982700608),(42.82470703125,56.58369172128337)]]) format TSV; select '-------- Polygon with Polygon with Holes'; select polygonsWithinSpherical([[(29.453587685533865,59.779570356240356),(29.393139070478895,52.276266797422124),(40.636581470703206,59.38168915000267),(41.21084331372543,59.103467777099866),(29.786055068336193,52.146627480315004),(31.23682182965546,52.16517054781818),(41.69443223416517,58.85424941916091),(42.51048853740727,58.47703162291134),(32.59691566839227,52.22075341251539),(34.289476889931414,52.22075341251539),(43.02430176537451,58.07974369546071),(43.02430176537451,57.25537683364851),(35.468224883503325,52.2022335126388),(37.16078610504247,52.23926559241349),(43.02430176537451,56.26136189644947),(43.02430176537451,55.326904361850836),(38.33953409861437,52.16517054781818),(40.09254393520848,52.16517054781818),(44.4146199116388,55.3097062225408),(44.47506852669377,59.80998197603594),(39.72985224487867,59.931351417569715),(30.23941968124846,53.67744677450975),(30.20919537372098,54.63314259659509),(38.73245009647167,59.94649146557819),(37.2816833351524,59.97675082987618),(30.23941968124846,55.2752875586599),(30.33009260383092,56.19415599955667),(36.28428118674541,59.96162460231375),(34.863738732953635,59.97675082987618),(30.178971066193498,56.97640788219866),(30.178971066193498,57.91957806959033),(33.65476643185424,59.94649146557819),(32.32489690064491,59.94649146557819),(30.481214141468342,58.85424941916091),(30.571887064050795,59.99187015036608),(29.453587685533865,59.779570356240356)]], [[(24.367675781249993,61.45977057029751),(19.577636718749993,58.67693767258692),(19.577636718749993,57.492213666700735),(19.445800781249996,55.87531083569678),(19.445800781249996,54.085173420886775),(17.468261718749996,53.014783245859235),(20.017089843749993,51.563412328675895),(21.203613281249993,50.205033264943324),(26.125488281249993,50.40151532278236),(27.22412109374999,48.980216985374994),(32.80517578124999,49.525208341974405),(35.26611328124999,48.74894534343292),(36.93603515624999,49.66762782262194),(42.56103515625,48.77791275550183),(43.92333984374999,49.8096315635631),(47.17529296875,49.152969656170455),(49.28466796875,50.54136296522162),(48.05419921875,51.17934297928929),(51.39404296875,52.48278022207825),(50.64697265625,53.014783245859235),(52.88818359375,53.93021986394004),(51.65771484374999,54.29088164657006),(52.66845703125,55.825973254619015),(50.25146484375,56.145549500679095),(51.92138671875,57.914847767009206),(49.15283203125,58.17070248348605),(49.59228515625,60.086762746260064),(47.043457031249986,59.88893689676584),(43.57177734375,61.37567331572748),(42.64892578125,60.630101766266705),(36.89208984374999,62.000904713685856),(36.01318359374999,61.143235250840576),(31.398925781249993,62.02152819100766),(30.563964843749996,61.05828537037917),(26.872558593749993,61.71070595883174),(26.652832031249993,61.10078883158897),(24.367675781249993,61.45977057029751)], [(24.455566406249993,59.42272750081452),(21.203613281249993,58.49369382056807),(21.335449218749993,56.89700392127261),(21.599121093749993,55.92458580482949),(25.202636718749993,55.998380955359636),(28.850097656249993,57.06463027327854),(27.09228515625,57.844750992890994),(28.806152343749996,59.17592824927138),(26.257324218749993,59.17592824927138),(24.455566406249993,59.42272750081452)], [(35.13427734375,59.84481485969107),(31.970214843749993,58.97266715450152),(33.20068359374999,56.776808316568406),(36.67236328125,56.41390137600675),(39.08935546874999,57.25528054528888),(42.69287109374999,58.03137242177638),(40.89111328124999,59.26588062825809),(37.28759765625,58.722598828043374),(37.11181640624999,59.66774058164964),(35.13427734375,59.84481485969107)], [(29.157714843749993,55.75184939173528),(22.565917968749993,55.128649068488784),(22.565917968749993,53.54030739150019),(22.038574218749996,51.48138289610097),(26.257324218749993,51.42661449707484),(30.124511718749993,50.54136296522162),(32.18994140624999,51.17934297928929),(30.124511718749993,53.173119202640635),(35.09033203124999,53.173119202640635),(33.11279296875,54.085173420886775),(29.597167968749993,55.50374985927513),(29.157714843749993,55.75184939173528)], [(42.82470703125,56.58369172128337),(36.584472656249986,55.329144408405085),(37.99072265625,53.592504809039355),(34.95849609374999,51.48138289610097),(36.54052734374999,50.40151532278236),(39.66064453124999,50.289339253291786),(39.79248046875,52.13348804077148),(41.77001953125,50.68079714532166),(44.49462890624999,51.97134580885171),(47.30712890624999,52.509534770327264),(44.05517578125,53.54030739150019),(46.60400390625,53.696706475303245),(47.61474609375,55.40406982700608),(45.37353515625,55.40406982700608),(42.82470703125,56.58369172128337)]]) format TSV; -select polygonsWithinMercator([[(29.453587685533865,59.779570356240356),(29.393139070478895,52.276266797422124),(40.636581470703206,59.38168915000267),(41.21084331372543,59.103467777099866),(29.786055068336193,52.146627480315004),(31.23682182965546,52.16517054781818),(41.69443223416517,58.85424941916091),(42.51048853740727,58.47703162291134),(32.59691566839227,52.22075341251539),(34.289476889931414,52.22075341251539),(43.02430176537451,58.07974369546071),(43.02430176537451,57.25537683364851),(35.468224883503325,52.2022335126388),(37.16078610504247,52.23926559241349),(43.02430176537451,56.26136189644947),(43.02430176537451,55.326904361850836),(38.33953409861437,52.16517054781818),(40.09254393520848,52.16517054781818),(44.4146199116388,55.3097062225408),(44.47506852669377,59.80998197603594),(39.72985224487867,59.931351417569715),(30.23941968124846,53.67744677450975),(30.20919537372098,54.63314259659509),(38.73245009647167,59.94649146557819),(37.2816833351524,59.97675082987618),(30.23941968124846,55.2752875586599),(30.33009260383092,56.19415599955667),(36.28428118674541,59.96162460231375),(34.863738732953635,59.97675082987618),(30.178971066193498,56.97640788219866),(30.178971066193498,57.91957806959033),(33.65476643185424,59.94649146557819),(32.32489690064491,59.94649146557819),(30.481214141468342,58.85424941916091),(30.571887064050795,59.99187015036608),(29.453587685533865,59.779570356240356)]], [[(24.367675781249993,61.45977057029751),(19.577636718749993,58.67693767258692),(19.577636718749993,57.492213666700735),(19.445800781249996,55.87531083569678),(19.445800781249996,54.085173420886775),(17.468261718749996,53.014783245859235),(20.017089843749993,51.563412328675895),(21.203613281249993,50.205033264943324),(26.125488281249993,50.40151532278236),(27.22412109374999,48.980216985374994),(32.80517578124999,49.525208341974405),(35.26611328124999,48.74894534343292),(36.93603515624999,49.66762782262194),(42.56103515625,48.77791275550183),(43.92333984374999,49.8096315635631),(47.17529296875,49.152969656170455),(49.28466796875,50.54136296522162),(48.05419921875,51.17934297928929),(51.39404296875,52.48278022207825),(50.64697265625,53.014783245859235),(52.88818359375,53.93021986394004),(51.65771484374999,54.29088164657006),(52.66845703125,55.825973254619015),(50.25146484375,56.145549500679095),(51.92138671875,57.914847767009206),(49.15283203125,58.17070248348605),(49.59228515625,60.086762746260064),(47.043457031249986,59.88893689676584),(43.57177734375,61.37567331572748),(42.64892578125,60.630101766266705),(36.89208984374999,62.000904713685856),(36.01318359374999,61.143235250840576),(31.398925781249993,62.02152819100766),(30.563964843749996,61.05828537037917),(26.872558593749993,61.71070595883174),(26.652832031249993,61.10078883158897),(24.367675781249993,61.45977057029751)], [(24.455566406249993,59.42272750081452),(21.203613281249993,58.49369382056807),(21.335449218749993,56.89700392127261),(21.599121093749993,55.92458580482949),(25.202636718749993,55.998380955359636),(28.850097656249993,57.06463027327854),(27.09228515625,57.844750992890994),(28.806152343749996,59.17592824927138),(26.257324218749993,59.17592824927138),(24.455566406249993,59.42272750081452)], [(35.13427734375,59.84481485969107),(31.970214843749993,58.97266715450152),(33.20068359374999,56.776808316568406),(36.67236328125,56.41390137600675),(39.08935546874999,57.25528054528888),(42.69287109374999,58.03137242177638),(40.89111328124999,59.26588062825809),(37.28759765625,58.722598828043374),(37.11181640624999,59.66774058164964),(35.13427734375,59.84481485969107)], [(29.157714843749993,55.75184939173528),(22.565917968749993,55.128649068488784),(22.565917968749993,53.54030739150019),(22.038574218749996,51.48138289610097),(26.257324218749993,51.42661449707484),(30.124511718749993,50.54136296522162),(32.18994140624999,51.17934297928929),(30.124511718749993,53.173119202640635),(35.09033203124999,53.173119202640635),(33.11279296875,54.085173420886775),(29.597167968749993,55.50374985927513),(29.157714843749993,55.75184939173528)], [(42.82470703125,56.58369172128337),(36.584472656249986,55.329144408405085),(37.99072265625,53.592504809039355),(34.95849609374999,51.48138289610097),(36.54052734374999,50.40151532278236),(39.66064453124999,50.289339253291786),(39.79248046875,52.13348804077148),(41.77001953125,50.68079714532166),(44.49462890624999,51.97134580885171),(47.30712890624999,52.509534770327264),(44.05517578125,53.54030739150019),(46.60400390625,53.696706475303245),(47.61474609375,55.40406982700608),(45.37353515625,55.40406982700608),(42.82470703125,56.58369172128337)]]) format TSV; + diff --git a/tests/queries/0_stateless/01305_polygons_union.reference b/tests/queries/0_stateless/01305_polygons_union.reference index 8a5c7d1b521..6a72c60c4fa 100644 --- a/tests/queries/0_stateless/01305_polygons_union.reference +++ b/tests/queries/0_stateless/01305_polygons_union.reference @@ -1,8 +1,6 @@ --------- [[[(1,2.9),(1,4),(4,4),(4,1),(2.9,1),(3,0),(0,0),(0,3),(1,2.9)]]] --------- --------- [[[(4.3666052904432435,50.84337386140151),(4.366227,50.840809),(4.344961,50.833264),(4.338074,50.848677),(4.346693,50.858306),(4.3526804582393535,50.856658100365976),(4.3613577,50.8651821),(4.3613148,50.8651279),(4.3904543,50.8564867),(4.3830299,50.8428851),(4.3666052904432435,50.84337386140151)]]] +<<<<<<< HEAD -------- [[[(4.366505261795747,50.843273415405),(4.3661270000000005,50.840709),(4.344861,50.83316399999998),(4.337974,50.84857699999999),(4.346593,50.858205999999996),(4.352580404040105,50.85655765067624),(4.3612577,50.86508209999998),(4.361214800000001,50.8650279),(4.3903543,50.85638669999999),(4.382929900000001,50.842785099999986),(4.366505261795747,50.843273415405)]]] -------- MultiPolygon with Polygon @@ -14,3 +12,5 @@ MULTIPOLYGON(((24.3676 61.4597,26.6527 61.1007,26.8725 61.7106,30.5639 61.0582,3 -------- Polygon with Polygon with Holes MULTIPOLYGON(((24.3677 61.4598,26.6528 61.1008,26.8726 61.7107,30.564 61.0583,31.3989 62.0215,36.0132 61.1432,36.8921 62.0009,42.6489 60.6301,43.5718 61.3757,47.0435 59.8889,49.5923 60.0868,49.1528 58.1707,51.9214 57.9148,50.2515 56.1455,52.6685 55.826,51.6577 54.2909,52.8882 53.9302,50.647 53.0148,51.394 52.4828,48.0542 51.1793,49.2847 50.5414,47.1753 49.153,43.9233 49.8096,42.561 48.7779,36.936 49.6676,35.2661 48.7489,32.8052 49.5252,27.2241 48.9802,26.1255 50.4015,21.2036 50.205,20.0171 51.5634,17.4683 53.0148,19.4458 54.0852,19.4458 55.8753,19.5776 57.4922,19.5776 58.6769,24.3677 61.4598),(24.4556 59.4227,21.2036 58.4937,21.3354 56.897,21.5991 55.9246,25.2026 55.9984,28.8501 57.0646,27.0923 57.8448,28.8062 59.1759,26.2573 59.1759,24.4556 59.4227),(32.6512 57.792,32.9378 57.2699,36.7912 59.6986,35.9475 59.7758,32.6512 57.792),(33.2446 56.7729,34.2635 56.6767,37.6322 58.7797,37.2876 58.7226,37.2102 59.1452,33.2446 56.7729),(36.1815 56.4715,41.168 59.0834,40.9299 59.2404,40.8804 59.2644,40.2079 59.1718,35.4536 56.5531,36.1815 56.4715),(30.7705 55.0525,30.2092 54.6331,30.2394 53.6774,31.5682 54.7333,30.7705 55.0525),(33.8733 53.1922,34.3351 53.53,33.5144 53.9057,32.5603 53.1989,33.8733 53.1922),(31.1968 52.1649,29.7861 52.1466,30.5785 52.7531,30.3098 53.0028,29.3931 52.2763,29.4171 55.606,29.1577 55.7518,22.5659 55.1286,22.5659 53.5403,22.0386 51.4814,26.2573 51.4266,30.1245 50.5414,32.1899 51.1793,31.1968 52.1649),(31.1682 53.1903,32.6907 54.2663,32.2591 54.4483,30.5408 53.1811,31.1682 53.1903),(39.4328 55.9511,37.2766 54.4948,37.7431 53.9104,41.4519 56.3413,39.4328 55.9511),(40.9691 57.677,42.2498 58.3455,41.5887 58.8012,38.1759 56.9472,39.0894 57.2553,40.9691 57.677),(37.1934 55.4694,36.5845 55.3291,36.7219 55.1665,37.1934 55.4694),(32.2964 58.4175,34.2247 59.6064,31.9702 58.9727,32.2964 58.4175),(35.9681 52.2157,34.9585 51.4814,36.5405 50.4015,39.6606 50.2893,39.7925 52.1335,41.77 50.6808,44.4946 51.9713,47.3071 52.5095,44.0552 53.5403,46.604 53.6967,47.6147 55.4041,45.3735 55.4041,44.4212 55.8594,44.4146 55.3097,40.0925 52.1652,38.3395 52.1652,43.0243 55.3269,43.0243 56.2614,37.1608 52.2393,35.9681 52.2157))) MULTIPOLYGON(((24.3676 61.4597,26.6527 61.1007,26.8725 61.7106,30.5639 61.0582,31.3988 62.0214,36.0131 61.1431,36.892 62.0008,42.6488 60.63,43.5717 61.3756,47.0434 59.8888,49.5922 60.0867,49.1527 58.1706,51.9213 57.9147,50.2514 56.1454,52.6684 55.8259,51.6576 54.2908,52.8881 53.9301,50.6469 53.0147,51.3939 52.4827,48.0541 51.1792,49.2846 50.5413,47.1752 49.1529,43.9232 49.8095,42.5609 48.7778,36.9359 49.6675,35.266 48.7488,32.8051 49.5251,27.224 48.9801,26.1254 50.4014,21.2035 50.2049,20.017 51.5633,17.4682 53.0147,19.4457 54.0851,19.4457 55.8752,19.5775 57.4921,19.5775 58.6768,24.3676 61.4597),(24.4555 59.4226,21.2035 58.4936,21.3353 56.8969,21.599 55.9245,25.2025 55.9983,28.85 57.0645,27.0922 57.8447,28.8061 59.1758,26.2572 59.1758,24.4555 59.4226),(32.6773 57.7269,32.9811 57.1783,36.8295 59.693,35.9663 59.7703,32.6773 57.7269),(33.4587 56.7498,34.5347 56.6377,37.9197 58.8184,37.2875 58.7225,37.2249 59.0621,33.4587 56.7498),(36.5672 56.4248,41.1674 59.0793,40.9293 59.24,40.8796 59.2641,40.276 59.1737,35.8287 56.5023,36.5672 56.4248),(30.7975 55.0249,30.2091 54.633,30.2393 53.6773,31.6568 54.6786,30.7975 55.0249),(34.0091 53.173,34.4539 53.4687,33.6753 53.8276,32.7039 53.173,34.0091 53.173),(31.1809 52.1644,29.786 52.1465,30.6237 52.6993,30.3667 52.9437,29.393 52.2762,29.4184 55.6047,29.1576 55.7517,22.5658 55.1285,22.5658 53.5402,22.0385 51.4813,26.2572 51.4265,30.1244 50.5413,32.1898 51.1792,31.1809 52.1644),(31.3501 53.173,32.9056 54.17,32.4672 54.3493,30.7047 53.173,31.3501 53.173),(39.7202 55.9646,37.3607 54.3795,37.8015 53.8302,41.577 56.336,39.7202 55.9646),(41.2147 57.7149,42.2607 58.3313,41.5893 58.7925,38.9459 57.2058,39.0893 57.2552,41.2147 57.7149),(37.6798 55.5522,36.5844 55.329,36.8394 55.0196,37.6798 55.5522),(32.3127 58.3747,34.2648 59.6073,31.9701 58.9726,32.3127 58.3747),(35.9939 52.2136,34.9584 51.4813,36.5404 50.4014,39.6605 50.2892,39.7924 52.1334,41.7699 50.6807,44.4945 51.9712,47.307 52.5094,44.0551 53.5402,46.6039 53.6966,47.6146 55.404,45.3734 55.404,44.4214 55.8488,44.4145 55.3096,40.0924 52.1651,38.3394 52.1651,43.0242 55.3268,43.0242 56.2613,37.1607 52.2392,35.9939 52.2136))) +======= +>>>>>>> parent of d4258922d1... more mercator diff --git a/tests/queries/0_stateless/01305_polygons_union.sql b/tests/queries/0_stateless/01305_polygons_union.sql index e3092dea2c0..01982c21e6e 100644 --- a/tests/queries/0_stateless/01305_polygons_union.sql +++ b/tests/queries/0_stateless/01305_polygons_union.sql @@ -1,23 +1,15 @@ -select '--------'; select polygonsUnionCartesian([[[(0., 0.),(0., 3.),(1., 2.9),(2., 2.6),(2.6, 2.),(2.9, 1),(3., 0.),(0., 0.)]]], [[[(1., 1.),(1., 4.),(4., 4.),(4., 1.),(1., 1.)]]]); -select '--------'; SELECT polygonsUnionCartesian([[[(2., 100.0000991821289), (0., 3.), (1., 2.9), (2., 2.6), (2.6, 2.), (2.9, 1), (3., 0.), (100.0000991821289, 2.)]]], [[[(1., 1.), (1000.0001220703125, nan), (4., 4.), (4., 1.), (1., 1.)]]]); -- { serverError 43 } -select '--------'; select polygonsUnionSpherical([[[(4.3613577, 50.8651821), (4.349556, 50.8535879), (4.3602419, 50.8435626), (4.3830299, 50.8428851), (4.3904543, 50.8564867), (4.3613148, 50.8651279)]]], [[[(4.346693, 50.858306), (4.367945, 50.852455), (4.366227, 50.840809), (4.344961, 50.833264), (4.338074, 50.848677), (4.346693, 50.858306)]]]); -select '--------'; -select polygonsUnionMercator([[[(4.3613577, 50.8651821), (4.349556, 50.8535879), (4.3602419, 50.8435626), (4.3830299, 50.8428851), (4.3904543, 50.8564867), (4.3613148, 50.8651279)]]], [[[(4.346693, 50.858306), (4.367945, 50.852455), (4.366227, 50.840809), (4.344961, 50.833264), (4.338074, 50.848677), (4.346693, 50.858306)]]]); - select '-------- MultiPolygon with Polygon'; select wkt(polygonsUnionSpherical([[(29.453587685533865,59.779570356240356),(29.393139070478895,52.276266797422124),(40.636581470703206,59.38168915000267),(41.21084331372543,59.103467777099866),(29.786055068336193,52.146627480315004),(31.23682182965546,52.16517054781818),(41.69443223416517,58.85424941916091),(42.51048853740727,58.47703162291134),(32.59691566839227,52.22075341251539),(34.289476889931414,52.22075341251539),(43.02430176537451,58.07974369546071),(43.02430176537451,57.25537683364851),(35.468224883503325,52.2022335126388),(37.16078610504247,52.23926559241349),(43.02430176537451,56.26136189644947),(43.02430176537451,55.326904361850836),(38.33953409861437,52.16517054781818),(40.09254393520848,52.16517054781818),(44.4146199116388,55.3097062225408),(44.47506852669377,59.80998197603594),(39.72985224487867,59.931351417569715),(30.23941968124846,53.67744677450975),(30.20919537372098,54.63314259659509),(38.73245009647167,59.94649146557819),(37.2816833351524,59.97675082987618),(30.23941968124846,55.2752875586599),(30.33009260383092,56.19415599955667),(36.28428118674541,59.96162460231375),(34.863738732953635,59.97675082987618),(30.178971066193498,56.97640788219866),(30.178971066193498,57.91957806959033),(33.65476643185424,59.94649146557819),(32.32489690064491,59.94649146557819),(30.481214141468342,58.85424941916091),(30.571887064050795,59.99187015036608),(29.453587685533865,59.779570356240356)]], [[[(33.473420586689336,58.85424941916091),(32.23422397806246,58.492830557036),(32.173775363007486,58.03176922751564),(31.508840597402823,57.499784781503735),(31.750635057622702,56.86092686957355),(31.508840597402823,55.941082594334574),(32.20399967053497,55.515591939372456),(31.84130798020516,54.998862226280465),(31.418167674820367,54.422670886434275),(32.47601843828233,53.83826377018255),(32.08310244042503,53.408048308050866),(33.171177511414484,52.82758702113742),(34.77306581037117,52.91880107773494),(34.77306581037117,53.784726518357985),(34.108131044766516,54.17574726780569),(35.07530888564602,54.59813930694554),(34.25925258240394,54.96417435716029),(35.01486027059106,55.361278263643584),(33.50364489421682,55.37845402950552),(32.7480372060297,55.90721384574556),(35.67979503619571,55.68634475630185),(32.83871012861215,56.311688992608396),(34.591719965206266,56.29492065473883),(35.7100193437232,56.311688992608396),(33.83611227701915,56.695333481003644),(32.95960735872209,56.9434497616887),(36.072711034053015,57.091531913901434),(33.171177511414484,57.33702717078384),(36.193608264162954,57.499784781503735),(33.23162612646945,57.77481561306047),(36.43540272438284,58.04776787540811),(33.62454212432676,58.27099811968307),(36.344729801800376,58.54018474404165),(33.83611227701915,58.68186423448108),(34.74284150284369,59.565911441555244),(33.473420586689336,58.85424941916091)]], [[(34.65216858026123,58.91672306881671),(37.19101041256995,58.68186423448108),(36.01226241899805,58.28688958537609),(37.16078610504247,58.04776787540811),(35.74024365125068,57.79092907387934),(37.009664567405046,57.499784781503735),(35.77046795877817,57.25537683364851),(36.979440259877556,57.07510745541089),(34.22902827487645,56.794777197297435),(36.7074214921302,56.210968525786996),(34.712617195316206,56.10998276812964),(36.55629995449277,55.63519693782703),(35.13575750070099,55.53270067649592),(36.43540272438284,55.34409504165558),(34.83351442542614,55.01619492319591),(35.61934642114075,54.49294870011772),(34.89396304048112,54.12264226523038),(35.37755196092087,53.046178687628185),(37.43280487278982,52.95523300597458),(35.92158949641559,53.80257986695776),(36.91899164482259,53.856094327816805),(36.01226241899805,54.75541714463799),(37.765272255592166,55.189110239786885),(36.828318722240134,55.44708256557195),(38.03729102333953,55.652253637168315),(36.64697287707522,55.941082594334574),(38.21863686850443,56.05939028508024),(36.37495410932787,56.64551287174558),(38.30930979108689,56.992876013526654),(37.16078610504247,57.25537683364851),(38.127963945921984,57.516020773674256),(37.43280487278982,57.710289827306724),(38.33953409861437,57.935626886818994),(37.40258056526235,58.31865112960426),(38.58132855883426,58.744648733419496),(37.31190764267989,59.02578062465136),(34.65216858026123,58.91672306881671)]], [[(38.52087994377928,59.11898412389468),(39.54850639971376,58.713270635642914),(38.369758406141855,58.28688958537609),(38.85334732658162,58.06375936407028),(38.33953409861437,57.710289827306724),(38.73245009647167,57.48354156434209),(38.21863686850443,57.271721400459285),(38.97424455669155,56.87744603722649),(37.463029180317314,56.5623320541159),(38.94402024916407,56.05939028508024),(38.18841256097694,55.856355210835915),(38.490655636251795,55.53270067649592),(37.795496563119656,55.39562234093384),(38.30930979108689,55.154587013355666),(36.7074214921302,54.65063295250911),(37.31190764267989,53.92734063371401),(36.979440259877556,53.58783775557231),(37.855945178174615,52.91880107773497),(39.57873070724124,52.69956490610895),(38.33953409861437,53.281741738901104),(40.00187101262603,53.35396273604752),(39.54850639971376,53.58783775557231),(40.24366547284591,53.58783775557231),(39.97164670509855,53.98069568468355),(40.60635716317572,54.03398248547225),(40.39478701048334,54.44025165268903),(39.54850639971376,54.56310590284329),(39.54850639971376,54.87732350170489),(40.39478701048334,54.87732350170489),(40.39478701048334,55.24083903654295),(39.82052516746112,55.2752875586599),(39.760076552406154,55.75443792473942),(40.57613285564824,55.78844000174894),(40.425011318010824,56.19415599955667),(39.82052516746112,56.07626182891758),(39.79030085993364,56.41214455508424),(40.48545993306579,56.495655446714636),(40.33433839542836,56.95993246553937),(39.79030085993364,56.992876013526654),(39.72985224487867,57.46729112028032),(40.33433839542836,57.46729112028032),(40.24366547284591,58.04776787540811),(39.63917932229622,58.04776787540811),(39.63917932229622,58.382088724871295),(40.33433839542836,58.382088724871295),(40.45523562553831,58.9011152358548),(38.52087994377928,59.11898412389468)]]])) format TSV; -select wkt(polygonsUnionMercator([[(29.453587685533865,59.779570356240356),(29.393139070478895,52.276266797422124),(40.636581470703206,59.38168915000267),(41.21084331372543,59.103467777099866),(29.786055068336193,52.146627480315004),(31.23682182965546,52.16517054781818),(41.69443223416517,58.85424941916091),(42.51048853740727,58.47703162291134),(32.59691566839227,52.22075341251539),(34.289476889931414,52.22075341251539),(43.02430176537451,58.07974369546071),(43.02430176537451,57.25537683364851),(35.468224883503325,52.2022335126388),(37.16078610504247,52.23926559241349),(43.02430176537451,56.26136189644947),(43.02430176537451,55.326904361850836),(38.33953409861437,52.16517054781818),(40.09254393520848,52.16517054781818),(44.4146199116388,55.3097062225408),(44.47506852669377,59.80998197603594),(39.72985224487867,59.931351417569715),(30.23941968124846,53.67744677450975),(30.20919537372098,54.63314259659509),(38.73245009647167,59.94649146557819),(37.2816833351524,59.97675082987618),(30.23941968124846,55.2752875586599),(30.33009260383092,56.19415599955667),(36.28428118674541,59.96162460231375),(34.863738732953635,59.97675082987618),(30.178971066193498,56.97640788219866),(30.178971066193498,57.91957806959033),(33.65476643185424,59.94649146557819),(32.32489690064491,59.94649146557819),(30.481214141468342,58.85424941916091),(30.571887064050795,59.99187015036608),(29.453587685533865,59.779570356240356)]], [[[(33.473420586689336,58.85424941916091),(32.23422397806246,58.492830557036),(32.173775363007486,58.03176922751564),(31.508840597402823,57.499784781503735),(31.750635057622702,56.86092686957355),(31.508840597402823,55.941082594334574),(32.20399967053497,55.515591939372456),(31.84130798020516,54.998862226280465),(31.418167674820367,54.422670886434275),(32.47601843828233,53.83826377018255),(32.08310244042503,53.408048308050866),(33.171177511414484,52.82758702113742),(34.77306581037117,52.91880107773494),(34.77306581037117,53.784726518357985),(34.108131044766516,54.17574726780569),(35.07530888564602,54.59813930694554),(34.25925258240394,54.96417435716029),(35.01486027059106,55.361278263643584),(33.50364489421682,55.37845402950552),(32.7480372060297,55.90721384574556),(35.67979503619571,55.68634475630185),(32.83871012861215,56.311688992608396),(34.591719965206266,56.29492065473883),(35.7100193437232,56.311688992608396),(33.83611227701915,56.695333481003644),(32.95960735872209,56.9434497616887),(36.072711034053015,57.091531913901434),(33.171177511414484,57.33702717078384),(36.193608264162954,57.499784781503735),(33.23162612646945,57.77481561306047),(36.43540272438284,58.04776787540811),(33.62454212432676,58.27099811968307),(36.344729801800376,58.54018474404165),(33.83611227701915,58.68186423448108),(34.74284150284369,59.565911441555244),(33.473420586689336,58.85424941916091)]], [[(34.65216858026123,58.91672306881671),(37.19101041256995,58.68186423448108),(36.01226241899805,58.28688958537609),(37.16078610504247,58.04776787540811),(35.74024365125068,57.79092907387934),(37.009664567405046,57.499784781503735),(35.77046795877817,57.25537683364851),(36.979440259877556,57.07510745541089),(34.22902827487645,56.794777197297435),(36.7074214921302,56.210968525786996),(34.712617195316206,56.10998276812964),(36.55629995449277,55.63519693782703),(35.13575750070099,55.53270067649592),(36.43540272438284,55.34409504165558),(34.83351442542614,55.01619492319591),(35.61934642114075,54.49294870011772),(34.89396304048112,54.12264226523038),(35.37755196092087,53.046178687628185),(37.43280487278982,52.95523300597458),(35.92158949641559,53.80257986695776),(36.91899164482259,53.856094327816805),(36.01226241899805,54.75541714463799),(37.765272255592166,55.189110239786885),(36.828318722240134,55.44708256557195),(38.03729102333953,55.652253637168315),(36.64697287707522,55.941082594334574),(38.21863686850443,56.05939028508024),(36.37495410932787,56.64551287174558),(38.30930979108689,56.992876013526654),(37.16078610504247,57.25537683364851),(38.127963945921984,57.516020773674256),(37.43280487278982,57.710289827306724),(38.33953409861437,57.935626886818994),(37.40258056526235,58.31865112960426),(38.58132855883426,58.744648733419496),(37.31190764267989,59.02578062465136),(34.65216858026123,58.91672306881671)]], [[(38.52087994377928,59.11898412389468),(39.54850639971376,58.713270635642914),(38.369758406141855,58.28688958537609),(38.85334732658162,58.06375936407028),(38.33953409861437,57.710289827306724),(38.73245009647167,57.48354156434209),(38.21863686850443,57.271721400459285),(38.97424455669155,56.87744603722649),(37.463029180317314,56.5623320541159),(38.94402024916407,56.05939028508024),(38.18841256097694,55.856355210835915),(38.490655636251795,55.53270067649592),(37.795496563119656,55.39562234093384),(38.30930979108689,55.154587013355666),(36.7074214921302,54.65063295250911),(37.31190764267989,53.92734063371401),(36.979440259877556,53.58783775557231),(37.855945178174615,52.91880107773497),(39.57873070724124,52.69956490610895),(38.33953409861437,53.281741738901104),(40.00187101262603,53.35396273604752),(39.54850639971376,53.58783775557231),(40.24366547284591,53.58783775557231),(39.97164670509855,53.98069568468355),(40.60635716317572,54.03398248547225),(40.39478701048334,54.44025165268903),(39.54850639971376,54.56310590284329),(39.54850639971376,54.87732350170489),(40.39478701048334,54.87732350170489),(40.39478701048334,55.24083903654295),(39.82052516746112,55.2752875586599),(39.760076552406154,55.75443792473942),(40.57613285564824,55.78844000174894),(40.425011318010824,56.19415599955667),(39.82052516746112,56.07626182891758),(39.79030085993364,56.41214455508424),(40.48545993306579,56.495655446714636),(40.33433839542836,56.95993246553937),(39.79030085993364,56.992876013526654),(39.72985224487867,57.46729112028032),(40.33433839542836,57.46729112028032),(40.24366547284591,58.04776787540811),(39.63917932229622,58.04776787540811),(39.63917932229622,58.382088724871295),(40.33433839542836,58.382088724871295),(40.45523562553831,58.9011152358548),(38.52087994377928,59.11898412389468)]]])) format TSV; select '-------- MultiPolygon with Polygon with Holes'; select wkt(polygonsUnionSpherical([[[(33.473420586689336,58.85424941916091),(32.23422397806246,58.492830557036),(32.173775363007486,58.03176922751564),(31.508840597402823,57.499784781503735),(31.750635057622702,56.86092686957355),(31.508840597402823,55.941082594334574),(32.20399967053497,55.515591939372456),(31.84130798020516,54.998862226280465),(31.418167674820367,54.422670886434275),(32.47601843828233,53.83826377018255),(32.08310244042503,53.408048308050866),(33.171177511414484,52.82758702113742),(34.77306581037117,52.91880107773494),(34.77306581037117,53.784726518357985),(34.108131044766516,54.17574726780569),(35.07530888564602,54.59813930694554),(34.25925258240394,54.96417435716029),(35.01486027059106,55.361278263643584),(33.50364489421682,55.37845402950552),(32.7480372060297,55.90721384574556),(35.67979503619571,55.68634475630185),(32.83871012861215,56.311688992608396),(34.591719965206266,56.29492065473883),(35.7100193437232,56.311688992608396),(33.83611227701915,56.695333481003644),(32.95960735872209,56.9434497616887),(36.072711034053015,57.091531913901434),(33.171177511414484,57.33702717078384),(36.193608264162954,57.499784781503735),(33.23162612646945,57.77481561306047),(36.43540272438284,58.04776787540811),(33.62454212432676,58.27099811968307),(36.344729801800376,58.54018474404165),(33.83611227701915,58.68186423448108),(34.74284150284369,59.565911441555244),(33.473420586689336,58.85424941916091)]], [[(34.65216858026123,58.91672306881671),(37.19101041256995,58.68186423448108),(36.01226241899805,58.28688958537609),(37.16078610504247,58.04776787540811),(35.74024365125068,57.79092907387934),(37.009664567405046,57.499784781503735),(35.77046795877817,57.25537683364851),(36.979440259877556,57.07510745541089),(34.22902827487645,56.794777197297435),(36.7074214921302,56.210968525786996),(34.712617195316206,56.10998276812964),(36.55629995449277,55.63519693782703),(35.13575750070099,55.53270067649592),(36.43540272438284,55.34409504165558),(34.83351442542614,55.01619492319591),(35.61934642114075,54.49294870011772),(34.89396304048112,54.12264226523038),(35.37755196092087,53.046178687628185),(37.43280487278982,52.95523300597458),(35.92158949641559,53.80257986695776),(36.91899164482259,53.856094327816805),(36.01226241899805,54.75541714463799),(37.765272255592166,55.189110239786885),(36.828318722240134,55.44708256557195),(38.03729102333953,55.652253637168315),(36.64697287707522,55.941082594334574),(38.21863686850443,56.05939028508024),(36.37495410932787,56.64551287174558),(38.30930979108689,56.992876013526654),(37.16078610504247,57.25537683364851),(38.127963945921984,57.516020773674256),(37.43280487278982,57.710289827306724),(38.33953409861437,57.935626886818994),(37.40258056526235,58.31865112960426),(38.58132855883426,58.744648733419496),(37.31190764267989,59.02578062465136),(34.65216858026123,58.91672306881671)]], [[(38.52087994377928,59.11898412389468),(39.54850639971376,58.713270635642914),(38.369758406141855,58.28688958537609),(38.85334732658162,58.06375936407028),(38.33953409861437,57.710289827306724),(38.73245009647167,57.48354156434209),(38.21863686850443,57.271721400459285),(38.97424455669155,56.87744603722649),(37.463029180317314,56.5623320541159),(38.94402024916407,56.05939028508024),(38.18841256097694,55.856355210835915),(38.490655636251795,55.53270067649592),(37.795496563119656,55.39562234093384),(38.30930979108689,55.154587013355666),(36.7074214921302,54.65063295250911),(37.31190764267989,53.92734063371401),(36.979440259877556,53.58783775557231),(37.855945178174615,52.91880107773497),(39.57873070724124,52.69956490610895),(38.33953409861437,53.281741738901104),(40.00187101262603,53.35396273604752),(39.54850639971376,53.58783775557231),(40.24366547284591,53.58783775557231),(39.97164670509855,53.98069568468355),(40.60635716317572,54.03398248547225),(40.39478701048334,54.44025165268903),(39.54850639971376,54.56310590284329),(39.54850639971376,54.87732350170489),(40.39478701048334,54.87732350170489),(40.39478701048334,55.24083903654295),(39.82052516746112,55.2752875586599),(39.760076552406154,55.75443792473942),(40.57613285564824,55.78844000174894),(40.425011318010824,56.19415599955667),(39.82052516746112,56.07626182891758),(39.79030085993364,56.41214455508424),(40.48545993306579,56.495655446714636),(40.33433839542836,56.95993246553937),(39.79030085993364,56.992876013526654),(39.72985224487867,57.46729112028032),(40.33433839542836,57.46729112028032),(40.24366547284591,58.04776787540811),(39.63917932229622,58.04776787540811),(39.63917932229622,58.382088724871295),(40.33433839542836,58.382088724871295),(40.45523562553831,58.9011152358548),(38.52087994377928,59.11898412389468)]]], [[(24.367675781249993,61.45977057029751),(19.577636718749993,58.67693767258692),(19.577636718749993,57.492213666700735),(19.445800781249996,55.87531083569678),(19.445800781249996,54.085173420886775),(17.468261718749996,53.014783245859235),(20.017089843749993,51.563412328675895),(21.203613281249993,50.205033264943324),(26.125488281249993,50.40151532278236),(27.22412109374999,48.980216985374994),(32.80517578124999,49.525208341974405),(35.26611328124999,48.74894534343292),(36.93603515624999,49.66762782262194),(42.56103515625,48.77791275550183),(43.92333984374999,49.8096315635631),(47.17529296875,49.152969656170455),(49.28466796875,50.54136296522162),(48.05419921875,51.17934297928929),(51.39404296875,52.48278022207825),(50.64697265625,53.014783245859235),(52.88818359375,53.93021986394004),(51.65771484374999,54.29088164657006),(52.66845703125,55.825973254619015),(50.25146484375,56.145549500679095),(51.92138671875,57.914847767009206),(49.15283203125,58.17070248348605),(49.59228515625,60.086762746260064),(47.043457031249986,59.88893689676584),(43.57177734375,61.37567331572748),(42.64892578125,60.630101766266705),(36.89208984374999,62.000904713685856),(36.01318359374999,61.143235250840576),(31.398925781249993,62.02152819100766),(30.563964843749996,61.05828537037917),(26.872558593749993,61.71070595883174),(26.652832031249993,61.10078883158897),(24.367675781249993,61.45977057029751)], [(24.455566406249993,59.42272750081452),(21.203613281249993,58.49369382056807),(21.335449218749993,56.89700392127261),(21.599121093749993,55.92458580482949),(25.202636718749993,55.998380955359636),(28.850097656249993,57.06463027327854),(27.09228515625,57.844750992890994),(28.806152343749996,59.17592824927138),(26.257324218749993,59.17592824927138),(24.455566406249993,59.42272750081452)], [(35.13427734375,59.84481485969107),(31.970214843749993,58.97266715450152),(33.20068359374999,56.776808316568406),(36.67236328125,56.41390137600675),(39.08935546874999,57.25528054528888),(42.69287109374999,58.03137242177638),(40.89111328124999,59.26588062825809),(37.28759765625,58.722598828043374),(37.11181640624999,59.66774058164964),(35.13427734375,59.84481485969107)], [(29.157714843749993,55.75184939173528),(22.565917968749993,55.128649068488784),(22.565917968749993,53.54030739150019),(22.038574218749996,51.48138289610097),(26.257324218749993,51.42661449707484),(30.124511718749993,50.54136296522162),(32.18994140624999,51.17934297928929),(30.124511718749993,53.173119202640635),(35.09033203124999,53.173119202640635),(33.11279296875,54.085173420886775),(29.597167968749993,55.50374985927513),(29.157714843749993,55.75184939173528)], [(42.82470703125,56.58369172128337),(36.584472656249986,55.329144408405085),(37.99072265625,53.592504809039355),(34.95849609374999,51.48138289610097),(36.54052734374999,50.40151532278236),(39.66064453124999,50.289339253291786),(39.79248046875,52.13348804077148),(41.77001953125,50.68079714532166),(44.49462890624999,51.97134580885171),(47.30712890624999,52.509534770327264),(44.05517578125,53.54030739150019),(46.60400390625,53.696706475303245),(47.61474609375,55.40406982700608),(45.37353515625,55.40406982700608),(42.82470703125,56.58369172128337)]])) format TSV; -select wkt(polygonsUnionMercator([[[(33.473420586689336,58.85424941916091),(32.23422397806246,58.492830557036),(32.173775363007486,58.03176922751564),(31.508840597402823,57.499784781503735),(31.750635057622702,56.86092686957355),(31.508840597402823,55.941082594334574),(32.20399967053497,55.515591939372456),(31.84130798020516,54.998862226280465),(31.418167674820367,54.422670886434275),(32.47601843828233,53.83826377018255),(32.08310244042503,53.408048308050866),(33.171177511414484,52.82758702113742),(34.77306581037117,52.91880107773494),(34.77306581037117,53.784726518357985),(34.108131044766516,54.17574726780569),(35.07530888564602,54.59813930694554),(34.25925258240394,54.96417435716029),(35.01486027059106,55.361278263643584),(33.50364489421682,55.37845402950552),(32.7480372060297,55.90721384574556),(35.67979503619571,55.68634475630185),(32.83871012861215,56.311688992608396),(34.591719965206266,56.29492065473883),(35.7100193437232,56.311688992608396),(33.83611227701915,56.695333481003644),(32.95960735872209,56.9434497616887),(36.072711034053015,57.091531913901434),(33.171177511414484,57.33702717078384),(36.193608264162954,57.499784781503735),(33.23162612646945,57.77481561306047),(36.43540272438284,58.04776787540811),(33.62454212432676,58.27099811968307),(36.344729801800376,58.54018474404165),(33.83611227701915,58.68186423448108),(34.74284150284369,59.565911441555244),(33.473420586689336,58.85424941916091)]], [[(34.65216858026123,58.91672306881671),(37.19101041256995,58.68186423448108),(36.01226241899805,58.28688958537609),(37.16078610504247,58.04776787540811),(35.74024365125068,57.79092907387934),(37.009664567405046,57.499784781503735),(35.77046795877817,57.25537683364851),(36.979440259877556,57.07510745541089),(34.22902827487645,56.794777197297435),(36.7074214921302,56.210968525786996),(34.712617195316206,56.10998276812964),(36.55629995449277,55.63519693782703),(35.13575750070099,55.53270067649592),(36.43540272438284,55.34409504165558),(34.83351442542614,55.01619492319591),(35.61934642114075,54.49294870011772),(34.89396304048112,54.12264226523038),(35.37755196092087,53.046178687628185),(37.43280487278982,52.95523300597458),(35.92158949641559,53.80257986695776),(36.91899164482259,53.856094327816805),(36.01226241899805,54.75541714463799),(37.765272255592166,55.189110239786885),(36.828318722240134,55.44708256557195),(38.03729102333953,55.652253637168315),(36.64697287707522,55.941082594334574),(38.21863686850443,56.05939028508024),(36.37495410932787,56.64551287174558),(38.30930979108689,56.992876013526654),(37.16078610504247,57.25537683364851),(38.127963945921984,57.516020773674256),(37.43280487278982,57.710289827306724),(38.33953409861437,57.935626886818994),(37.40258056526235,58.31865112960426),(38.58132855883426,58.744648733419496),(37.31190764267989,59.02578062465136),(34.65216858026123,58.91672306881671)]], [[(38.52087994377928,59.11898412389468),(39.54850639971376,58.713270635642914),(38.369758406141855,58.28688958537609),(38.85334732658162,58.06375936407028),(38.33953409861437,57.710289827306724),(38.73245009647167,57.48354156434209),(38.21863686850443,57.271721400459285),(38.97424455669155,56.87744603722649),(37.463029180317314,56.5623320541159),(38.94402024916407,56.05939028508024),(38.18841256097694,55.856355210835915),(38.490655636251795,55.53270067649592),(37.795496563119656,55.39562234093384),(38.30930979108689,55.154587013355666),(36.7074214921302,54.65063295250911),(37.31190764267989,53.92734063371401),(36.979440259877556,53.58783775557231),(37.855945178174615,52.91880107773497),(39.57873070724124,52.69956490610895),(38.33953409861437,53.281741738901104),(40.00187101262603,53.35396273604752),(39.54850639971376,53.58783775557231),(40.24366547284591,53.58783775557231),(39.97164670509855,53.98069568468355),(40.60635716317572,54.03398248547225),(40.39478701048334,54.44025165268903),(39.54850639971376,54.56310590284329),(39.54850639971376,54.87732350170489),(40.39478701048334,54.87732350170489),(40.39478701048334,55.24083903654295),(39.82052516746112,55.2752875586599),(39.760076552406154,55.75443792473942),(40.57613285564824,55.78844000174894),(40.425011318010824,56.19415599955667),(39.82052516746112,56.07626182891758),(39.79030085993364,56.41214455508424),(40.48545993306579,56.495655446714636),(40.33433839542836,56.95993246553937),(39.79030085993364,56.992876013526654),(39.72985224487867,57.46729112028032),(40.33433839542836,57.46729112028032),(40.24366547284591,58.04776787540811),(39.63917932229622,58.04776787540811),(39.63917932229622,58.382088724871295),(40.33433839542836,58.382088724871295),(40.45523562553831,58.9011152358548),(38.52087994377928,59.11898412389468)]]], [[(24.367675781249993,61.45977057029751),(19.577636718749993,58.67693767258692),(19.577636718749993,57.492213666700735),(19.445800781249996,55.87531083569678),(19.445800781249996,54.085173420886775),(17.468261718749996,53.014783245859235),(20.017089843749993,51.563412328675895),(21.203613281249993,50.205033264943324),(26.125488281249993,50.40151532278236),(27.22412109374999,48.980216985374994),(32.80517578124999,49.525208341974405),(35.26611328124999,48.74894534343292),(36.93603515624999,49.66762782262194),(42.56103515625,48.77791275550183),(43.92333984374999,49.8096315635631),(47.17529296875,49.152969656170455),(49.28466796875,50.54136296522162),(48.05419921875,51.17934297928929),(51.39404296875,52.48278022207825),(50.64697265625,53.014783245859235),(52.88818359375,53.93021986394004),(51.65771484374999,54.29088164657006),(52.66845703125,55.825973254619015),(50.25146484375,56.145549500679095),(51.92138671875,57.914847767009206),(49.15283203125,58.17070248348605),(49.59228515625,60.086762746260064),(47.043457031249986,59.88893689676584),(43.57177734375,61.37567331572748),(42.64892578125,60.630101766266705),(36.89208984374999,62.000904713685856),(36.01318359374999,61.143235250840576),(31.398925781249993,62.02152819100766),(30.563964843749996,61.05828537037917),(26.872558593749993,61.71070595883174),(26.652832031249993,61.10078883158897),(24.367675781249993,61.45977057029751)], [(24.455566406249993,59.42272750081452),(21.203613281249993,58.49369382056807),(21.335449218749993,56.89700392127261),(21.599121093749993,55.92458580482949),(25.202636718749993,55.998380955359636),(28.850097656249993,57.06463027327854),(27.09228515625,57.844750992890994),(28.806152343749996,59.17592824927138),(26.257324218749993,59.17592824927138),(24.455566406249993,59.42272750081452)], [(35.13427734375,59.84481485969107),(31.970214843749993,58.97266715450152),(33.20068359374999,56.776808316568406),(36.67236328125,56.41390137600675),(39.08935546874999,57.25528054528888),(42.69287109374999,58.03137242177638),(40.89111328124999,59.26588062825809),(37.28759765625,58.722598828043374),(37.11181640624999,59.66774058164964),(35.13427734375,59.84481485969107)], [(29.157714843749993,55.75184939173528),(22.565917968749993,55.128649068488784),(22.565917968749993,53.54030739150019),(22.038574218749996,51.48138289610097),(26.257324218749993,51.42661449707484),(30.124511718749993,50.54136296522162),(32.18994140624999,51.17934297928929),(30.124511718749993,53.173119202640635),(35.09033203124999,53.173119202640635),(33.11279296875,54.085173420886775),(29.597167968749993,55.50374985927513),(29.157714843749993,55.75184939173528)], [(42.82470703125,56.58369172128337),(36.584472656249986,55.329144408405085),(37.99072265625,53.592504809039355),(34.95849609374999,51.48138289610097),(36.54052734374999,50.40151532278236),(39.66064453124999,50.289339253291786),(39.79248046875,52.13348804077148),(41.77001953125,50.68079714532166),(44.49462890624999,51.97134580885171),(47.30712890624999,52.509534770327264),(44.05517578125,53.54030739150019),(46.60400390625,53.696706475303245),(47.61474609375,55.40406982700608),(45.37353515625,55.40406982700608),(42.82470703125,56.58369172128337)]])) format TSV; select '-------- Polygon with Polygon with Holes'; select wkt(polygonsUnionSpherical([[(29.453587685533865,59.779570356240356),(29.393139070478895,52.276266797422124),(40.636581470703206,59.38168915000267),(41.21084331372543,59.103467777099866),(29.786055068336193,52.146627480315004),(31.23682182965546,52.16517054781818),(41.69443223416517,58.85424941916091),(42.51048853740727,58.47703162291134),(32.59691566839227,52.22075341251539),(34.289476889931414,52.22075341251539),(43.02430176537451,58.07974369546071),(43.02430176537451,57.25537683364851),(35.468224883503325,52.2022335126388),(37.16078610504247,52.23926559241349),(43.02430176537451,56.26136189644947),(43.02430176537451,55.326904361850836),(38.33953409861437,52.16517054781818),(40.09254393520848,52.16517054781818),(44.4146199116388,55.3097062225408),(44.47506852669377,59.80998197603594),(39.72985224487867,59.931351417569715),(30.23941968124846,53.67744677450975),(30.20919537372098,54.63314259659509),(38.73245009647167,59.94649146557819),(37.2816833351524,59.97675082987618),(30.23941968124846,55.2752875586599),(30.33009260383092,56.19415599955667),(36.28428118674541,59.96162460231375),(34.863738732953635,59.97675082987618),(30.178971066193498,56.97640788219866),(30.178971066193498,57.91957806959033),(33.65476643185424,59.94649146557819),(32.32489690064491,59.94649146557819),(30.481214141468342,58.85424941916091),(30.571887064050795,59.99187015036608),(29.453587685533865,59.779570356240356)]], [[(24.367675781249993,61.45977057029751),(19.577636718749993,58.67693767258692),(19.577636718749993,57.492213666700735),(19.445800781249996,55.87531083569678),(19.445800781249996,54.085173420886775),(17.468261718749996,53.014783245859235),(20.017089843749993,51.563412328675895),(21.203613281249993,50.205033264943324),(26.125488281249993,50.40151532278236),(27.22412109374999,48.980216985374994),(32.80517578124999,49.525208341974405),(35.26611328124999,48.74894534343292),(36.93603515624999,49.66762782262194),(42.56103515625,48.77791275550183),(43.92333984374999,49.8096315635631),(47.17529296875,49.152969656170455),(49.28466796875,50.54136296522162),(48.05419921875,51.17934297928929),(51.39404296875,52.48278022207825),(50.64697265625,53.014783245859235),(52.88818359375,53.93021986394004),(51.65771484374999,54.29088164657006),(52.66845703125,55.825973254619015),(50.25146484375,56.145549500679095),(51.92138671875,57.914847767009206),(49.15283203125,58.17070248348605),(49.59228515625,60.086762746260064),(47.043457031249986,59.88893689676584),(43.57177734375,61.37567331572748),(42.64892578125,60.630101766266705),(36.89208984374999,62.000904713685856),(36.01318359374999,61.143235250840576),(31.398925781249993,62.02152819100766),(30.563964843749996,61.05828537037917),(26.872558593749993,61.71070595883174),(26.652832031249993,61.10078883158897),(24.367675781249993,61.45977057029751)], [(24.455566406249993,59.42272750081452),(21.203613281249993,58.49369382056807),(21.335449218749993,56.89700392127261),(21.599121093749993,55.92458580482949),(25.202636718749993,55.998380955359636),(28.850097656249993,57.06463027327854),(27.09228515625,57.844750992890994),(28.806152343749996,59.17592824927138),(26.257324218749993,59.17592824927138),(24.455566406249993,59.42272750081452)], [(35.13427734375,59.84481485969107),(31.970214843749993,58.97266715450152),(33.20068359374999,56.776808316568406),(36.67236328125,56.41390137600675),(39.08935546874999,57.25528054528888),(42.69287109374999,58.03137242177638),(40.89111328124999,59.26588062825809),(37.28759765625,58.722598828043374),(37.11181640624999,59.66774058164964),(35.13427734375,59.84481485969107)], [(29.157714843749993,55.75184939173528),(22.565917968749993,55.128649068488784),(22.565917968749993,53.54030739150019),(22.038574218749996,51.48138289610097),(26.257324218749993,51.42661449707484),(30.124511718749993,50.54136296522162),(32.18994140624999,51.17934297928929),(30.124511718749993,53.173119202640635),(35.09033203124999,53.173119202640635),(33.11279296875,54.085173420886775),(29.597167968749993,55.50374985927513),(29.157714843749993,55.75184939173528)], [(42.82470703125,56.58369172128337),(36.584472656249986,55.329144408405085),(37.99072265625,53.592504809039355),(34.95849609374999,51.48138289610097),(36.54052734374999,50.40151532278236),(39.66064453124999,50.289339253291786),(39.79248046875,52.13348804077148),(41.77001953125,50.68079714532166),(44.49462890624999,51.97134580885171),(47.30712890624999,52.509534770327264),(44.05517578125,53.54030739150019),(46.60400390625,53.696706475303245),(47.61474609375,55.40406982700608),(45.37353515625,55.40406982700608),(42.82470703125,56.58369172128337)]])) format TSV; -select wkt(polygonsUnionMercator([[(29.453587685533865,59.779570356240356),(29.393139070478895,52.276266797422124),(40.636581470703206,59.38168915000267),(41.21084331372543,59.103467777099866),(29.786055068336193,52.146627480315004),(31.23682182965546,52.16517054781818),(41.69443223416517,58.85424941916091),(42.51048853740727,58.47703162291134),(32.59691566839227,52.22075341251539),(34.289476889931414,52.22075341251539),(43.02430176537451,58.07974369546071),(43.02430176537451,57.25537683364851),(35.468224883503325,52.2022335126388),(37.16078610504247,52.23926559241349),(43.02430176537451,56.26136189644947),(43.02430176537451,55.326904361850836),(38.33953409861437,52.16517054781818),(40.09254393520848,52.16517054781818),(44.4146199116388,55.3097062225408),(44.47506852669377,59.80998197603594),(39.72985224487867,59.931351417569715),(30.23941968124846,53.67744677450975),(30.20919537372098,54.63314259659509),(38.73245009647167,59.94649146557819),(37.2816833351524,59.97675082987618),(30.23941968124846,55.2752875586599),(30.33009260383092,56.19415599955667),(36.28428118674541,59.96162460231375),(34.863738732953635,59.97675082987618),(30.178971066193498,56.97640788219866),(30.178971066193498,57.91957806959033),(33.65476643185424,59.94649146557819),(32.32489690064491,59.94649146557819),(30.481214141468342,58.85424941916091),(30.571887064050795,59.99187015036608),(29.453587685533865,59.779570356240356)]], [[(24.367675781249993,61.45977057029751),(19.577636718749993,58.67693767258692),(19.577636718749993,57.492213666700735),(19.445800781249996,55.87531083569678),(19.445800781249996,54.085173420886775),(17.468261718749996,53.014783245859235),(20.017089843749993,51.563412328675895),(21.203613281249993,50.205033264943324),(26.125488281249993,50.40151532278236),(27.22412109374999,48.980216985374994),(32.80517578124999,49.525208341974405),(35.26611328124999,48.74894534343292),(36.93603515624999,49.66762782262194),(42.56103515625,48.77791275550183),(43.92333984374999,49.8096315635631),(47.17529296875,49.152969656170455),(49.28466796875,50.54136296522162),(48.05419921875,51.17934297928929),(51.39404296875,52.48278022207825),(50.64697265625,53.014783245859235),(52.88818359375,53.93021986394004),(51.65771484374999,54.29088164657006),(52.66845703125,55.825973254619015),(50.25146484375,56.145549500679095),(51.92138671875,57.914847767009206),(49.15283203125,58.17070248348605),(49.59228515625,60.086762746260064),(47.043457031249986,59.88893689676584),(43.57177734375,61.37567331572748),(42.64892578125,60.630101766266705),(36.89208984374999,62.000904713685856),(36.01318359374999,61.143235250840576),(31.398925781249993,62.02152819100766),(30.563964843749996,61.05828537037917),(26.872558593749993,61.71070595883174),(26.652832031249993,61.10078883158897),(24.367675781249993,61.45977057029751)], [(24.455566406249993,59.42272750081452),(21.203613281249993,58.49369382056807),(21.335449218749993,56.89700392127261),(21.599121093749993,55.92458580482949),(25.202636718749993,55.998380955359636),(28.850097656249993,57.06463027327854),(27.09228515625,57.844750992890994),(28.806152343749996,59.17592824927138),(26.257324218749993,59.17592824927138),(24.455566406249993,59.42272750081452)], [(35.13427734375,59.84481485969107),(31.970214843749993,58.97266715450152),(33.20068359374999,56.776808316568406),(36.67236328125,56.41390137600675),(39.08935546874999,57.25528054528888),(42.69287109374999,58.03137242177638),(40.89111328124999,59.26588062825809),(37.28759765625,58.722598828043374),(37.11181640624999,59.66774058164964),(35.13427734375,59.84481485969107)], [(29.157714843749993,55.75184939173528),(22.565917968749993,55.128649068488784),(22.565917968749993,53.54030739150019),(22.038574218749996,51.48138289610097),(26.257324218749993,51.42661449707484),(30.124511718749993,50.54136296522162),(32.18994140624999,51.17934297928929),(30.124511718749993,53.173119202640635),(35.09033203124999,53.173119202640635),(33.11279296875,54.085173420886775),(29.597167968749993,55.50374985927513),(29.157714843749993,55.75184939173528)], [(42.82470703125,56.58369172128337),(36.584472656249986,55.329144408405085),(37.99072265625,53.592504809039355),(34.95849609374999,51.48138289610097),(36.54052734374999,50.40151532278236),(39.66064453124999,50.289339253291786),(39.79248046875,52.13348804077148),(41.77001953125,50.68079714532166),(44.49462890624999,51.97134580885171),(47.30712890624999,52.509534770327264),(44.05517578125,53.54030739150019),(46.60400390625,53.696706475303245),(47.61474609375,55.40406982700608),(45.37353515625,55.40406982700608),(42.82470703125,56.58369172128337)]])) format TSV; + diff --git a/tests/queries/0_stateless/01306_polygons_intersection.reference b/tests/queries/0_stateless/01306_polygons_intersection.reference index cfec067cbcb..ae23d9f0cda 100644 --- a/tests/queries/0_stateless/01306_polygons_intersection.reference +++ b/tests/queries/0_stateless/01306_polygons_intersection.reference @@ -1,9 +1,8 @@ --------- [[[(1,2.9),(2,2.6),(2.6,2),(2.9,1),(1,1),(1,2.9)]]] [] --------- [] [[[(4.3666052904432435,50.84337386140151),(4.3602419,50.8435626),(4.349556,50.8535879),(4.3526804582393535,50.856658100365976),(4.367945,50.852455),(4.3666052904432435,50.84337386140151)]]] +<<<<<<< HEAD -------- [] [[[(4.366505261795747,50.843273415405),(4.3601419,50.8434626),(4.349456,50.8534879),(4.352580404040105,50.85655765067624),(4.367845,50.85235499999999),(4.366505261795747,50.843273415405)]]] @@ -16,3 +15,5 @@ MULTIPOLYGON(((33.106 56.9503,32.9595 56.9433,33.1372 56.8932,33.2006 56.7767,33 -------- Polygon with Polygon with Holes MULTIPOLYGON(((32.6512 57.792,30.3301 56.1942,30.2394 55.2753,32.9378 57.2699,33.2007 56.7768,33.2446 56.7729,30.7705 55.0525,29.5972 55.5037,29.4171 55.606,29.4536 59.7796,30.5719 59.9919,30.4812 58.8542,32.3249 59.9465,33.6548 59.9465,30.179 57.9196,30.179 56.9764,32.2964 58.4175,32.6512 57.792)),((35.9475 59.7758,35.1343 59.8448,34.2247 59.6064,34.8637 59.9768,36.2843 59.9616,35.9475 59.7758)),((36.7912 59.6986,37.2817 59.9768,38.7325 59.9465,37.2102 59.1452,37.1118 59.6677,36.7912 59.6986)),((34.2635 56.6767,35.4536 56.5531,32.2591 54.4483,31.5682 54.7333,34.2635 56.6767)),((36.1815 56.4715,36.6724 56.4139,38.1759 56.9472,33.5144 53.9057,33.1128 54.0852,32.6907 54.2663,36.1815 56.4715)),((33.8733 53.1922,35.0903 53.1731,34.3351 53.53,36.7219 55.1665,37.2766 54.4948,34.2895 52.2208,32.5969 52.2208,33.8733 53.1922)),((31.1968 52.1649,30.5785 52.7531,31.1682 53.1903,32.5603 53.1989,31.2368 52.1652,31.1968 52.1649)),((30.3098 53.0028,30.1245 53.1731,30.5408 53.1811,30.3098 53.0028)),((37.6322 58.7797,39.7299 59.9314,44.4751 59.81,44.4212 55.8594,42.8247 56.5837,41.4519 56.3413,43.0243 57.2554,43.0243 58.0797,39.4328 55.9511,37.1934 55.4694,40.9691 57.677,42.6929 58.0314,42.2498 58.3455,42.5105 58.477,41.6944 58.8542,41.5887 58.8012,41.168 59.0834,41.2108 59.1035,40.9299 59.2404,40.8911 59.2659,40.8804 59.2644,40.6366 59.3817,40.2079 59.1718,37.6322 58.7797)),((35.9681 52.2157,35.4682 52.2022,37.7431 53.9104,37.9907 53.5925,35.9681 52.2157))) MULTIPOLYGON(((32.6773 57.7269,30.33 56.1941,30.2393 55.2752,32.9811 57.1783,33.2006 56.7767,33.4587 56.7498,30.7975 55.0249,29.5971 55.5036,29.4184 55.6047,29.4535 59.7795,30.5718 59.9918,30.4811 58.8541,32.3248 59.9464,33.6547 59.9464,30.1789 57.9195,30.1789 56.9763,32.3127 58.3747,32.6773 57.7269)),((35.9663 59.7703,35.1342 59.8447,34.2648 59.6073,34.8636 59.9767,36.2842 59.9615,35.9663 59.7703)),((36.8295 59.693,37.2816 59.9767,38.7324 59.9464,37.2249 59.0621,37.1117 59.6676,36.8295 59.693)),((34.5347 56.6377,35.8287 56.5023,32.4672 54.3493,31.6568 54.6786,34.5347 56.6377)),((36.5672 56.4248,36.6723 56.4138,38.9459 57.2058,33.6753 53.8276,33.1127 54.0851,32.9056 54.17,36.5672 56.4248)),((34.0091 53.173,35.0902 53.173,34.4539 53.4687,36.8394 55.0196,37.3607 54.3795,34.2894 52.2207,32.5968 52.2207,34.0091 53.173)),((31.1809 52.1644,30.6237 52.6993,31.3501 53.173,32.7039 53.173,31.2367 52.1651,31.1809 52.1644)),((30.3667 52.9437,30.1244 53.173,30.7047 53.173,30.3667 52.9437)),((37.9197 58.8184,39.7298 59.9313,44.475 59.8099,44.4214 55.8488,42.8246 56.5836,41.577 56.336,43.0242 57.2553,43.0242 58.0796,39.7202 55.9646,37.6798 55.5522,41.2147 57.7149,42.6928 58.0313,42.2607 58.3313,42.5104 58.4769,41.6943 58.8541,41.5893 58.7925,41.1674 59.0793,41.2107 59.1034,40.9293 59.24,40.891 59.2658,40.8796 59.2641,40.6365 59.3816,40.276 59.1737,37.9197 58.8184)),((35.9939 52.2136,35.4681 52.2021,37.8015 53.8302,37.9906 53.5924,35.9939 52.2136))) +======= +>>>>>>> parent of d4258922d1... more mercator diff --git a/tests/queries/0_stateless/01306_polygons_intersection.sql b/tests/queries/0_stateless/01306_polygons_intersection.sql index 105ab978e8c..144408ca0ae 100644 --- a/tests/queries/0_stateless/01306_polygons_intersection.sql +++ b/tests/queries/0_stateless/01306_polygons_intersection.sql @@ -1,24 +1,14 @@ - -select '--------'; select polygonsIntersectionCartesian([[[(0., 0.),(0., 3.),(1., 2.9),(2., 2.6),(2.6, 2.),(2.9, 1.),(3., 0.),(0., 0.)]]], [[[(1., 1.),(1., 4.),(4., 4.),(4., 1.),(1., 1.)]]]); select polygonsIntersectionCartesian([[[(0., 0.),(0., 3.),(1., 2.9),(2., 2.6),(2.6, 2.),(2.9, 1.),(3., 0.),(0., 0.)]]], [[[(3., 3.),(3., 4.),(4., 4.),(4., 3.),(3., 3.)]]]); -select '--------'; select polygonsIntersectionSpherical([[[(4.346693, 50.858306), (4.367945, 50.852455), (4.366227, 50.840809), (4.344961, 50.833264), (4.338074, 50.848677), (4.346693, 50.858306)]]], [[[(25.0010, 136.9987), (17.7500, 142.5000), (11.3733, 142.5917)]]]); select polygonsIntersectionSpherical([[[(4.3613577, 50.8651821), (4.349556, 50.8535879), (4.3602419, 50.8435626), (4.3830299, 50.8428851), (4.3904543, 50.8564867), (4.3613148, 50.8651279)]]], [[[(4.346693, 50.858306), (4.367945, 50.852455), (4.366227, 50.840809), (4.344961, 50.833264), (4.338074, 50.848677), (4.346693, 50.858306)]]]); -select '--------'; -select polygonsIntersectionMercator([[[(4.346693, 50.858306), (4.367945, 50.852455), (4.366227, 50.840809), (4.344961, 50.833264), (4.338074, 50.848677), (4.346693, 50.858306)]]], [[[(25.0010, 136.9987), (17.7500, 142.5000), (11.3733, 142.5917)]]]); -select polygonsIntersectionMercator([[[(4.3613577, 50.8651821), (4.349556, 50.8535879), (4.3602419, 50.8435626), (4.3830299, 50.8428851), (4.3904543, 50.8564867), (4.3613148, 50.8651279)]]], [[[(4.346693, 50.858306), (4.367945, 50.852455), (4.366227, 50.840809), (4.344961, 50.833264), (4.338074, 50.848677), (4.346693, 50.858306)]]]); - select '-------- MultiPolygon with Polygon'; select wkt(polygonsIntersectionSpherical([[(29.453587685533865,59.779570356240356),(29.393139070478895,52.276266797422124),(40.636581470703206,59.38168915000267),(41.21084331372543,59.103467777099866),(29.786055068336193,52.146627480315004),(31.23682182965546,52.16517054781818),(41.69443223416517,58.85424941916091),(42.51048853740727,58.47703162291134),(32.59691566839227,52.22075341251539),(34.289476889931414,52.22075341251539),(43.02430176537451,58.07974369546071),(43.02430176537451,57.25537683364851),(35.468224883503325,52.2022335126388),(37.16078610504247,52.23926559241349),(43.02430176537451,56.26136189644947),(43.02430176537451,55.326904361850836),(38.33953409861437,52.16517054781818),(40.09254393520848,52.16517054781818),(44.4146199116388,55.3097062225408),(44.47506852669377,59.80998197603594),(39.72985224487867,59.931351417569715),(30.23941968124846,53.67744677450975),(30.20919537372098,54.63314259659509),(38.73245009647167,59.94649146557819),(37.2816833351524,59.97675082987618),(30.23941968124846,55.2752875586599),(30.33009260383092,56.19415599955667),(36.28428118674541,59.96162460231375),(34.863738732953635,59.97675082987618),(30.178971066193498,56.97640788219866),(30.178971066193498,57.91957806959033),(33.65476643185424,59.94649146557819),(32.32489690064491,59.94649146557819),(30.481214141468342,58.85424941916091),(30.571887064050795,59.99187015036608),(29.453587685533865,59.779570356240356)]], [[[(33.473420586689336,58.85424941916091),(32.23422397806246,58.492830557036),(32.173775363007486,58.03176922751564),(31.508840597402823,57.499784781503735),(31.750635057622702,56.86092686957355),(31.508840597402823,55.941082594334574),(32.20399967053497,55.515591939372456),(31.84130798020516,54.998862226280465),(31.418167674820367,54.422670886434275),(32.47601843828233,53.83826377018255),(32.08310244042503,53.408048308050866),(33.171177511414484,52.82758702113742),(34.77306581037117,52.91880107773494),(34.77306581037117,53.784726518357985),(34.108131044766516,54.17574726780569),(35.07530888564602,54.59813930694554),(34.25925258240394,54.96417435716029),(35.01486027059106,55.361278263643584),(33.50364489421682,55.37845402950552),(32.7480372060297,55.90721384574556),(35.67979503619571,55.68634475630185),(32.83871012861215,56.311688992608396),(34.591719965206266,56.29492065473883),(35.7100193437232,56.311688992608396),(33.83611227701915,56.695333481003644),(32.95960735872209,56.9434497616887),(36.072711034053015,57.091531913901434),(33.171177511414484,57.33702717078384),(36.193608264162954,57.499784781503735),(33.23162612646945,57.77481561306047),(36.43540272438284,58.04776787540811),(33.62454212432676,58.27099811968307),(36.344729801800376,58.54018474404165),(33.83611227701915,58.68186423448108),(34.74284150284369,59.565911441555244),(33.473420586689336,58.85424941916091)]], [[(34.65216858026123,58.91672306881671),(37.19101041256995,58.68186423448108),(36.01226241899805,58.28688958537609),(37.16078610504247,58.04776787540811),(35.74024365125068,57.79092907387934),(37.009664567405046,57.499784781503735),(35.77046795877817,57.25537683364851),(36.979440259877556,57.07510745541089),(34.22902827487645,56.794777197297435),(36.7074214921302,56.210968525786996),(34.712617195316206,56.10998276812964),(36.55629995449277,55.63519693782703),(35.13575750070099,55.53270067649592),(36.43540272438284,55.34409504165558),(34.83351442542614,55.01619492319591),(35.61934642114075,54.49294870011772),(34.89396304048112,54.12264226523038),(35.37755196092087,53.046178687628185),(37.43280487278982,52.95523300597458),(35.92158949641559,53.80257986695776),(36.91899164482259,53.856094327816805),(36.01226241899805,54.75541714463799),(37.765272255592166,55.189110239786885),(36.828318722240134,55.44708256557195),(38.03729102333953,55.652253637168315),(36.64697287707522,55.941082594334574),(38.21863686850443,56.05939028508024),(36.37495410932787,56.64551287174558),(38.30930979108689,56.992876013526654),(37.16078610504247,57.25537683364851),(38.127963945921984,57.516020773674256),(37.43280487278982,57.710289827306724),(38.33953409861437,57.935626886818994),(37.40258056526235,58.31865112960426),(38.58132855883426,58.744648733419496),(37.31190764267989,59.02578062465136),(34.65216858026123,58.91672306881671)]], [[(38.52087994377928,59.11898412389468),(39.54850639971376,58.713270635642914),(38.369758406141855,58.28688958537609),(38.85334732658162,58.06375936407028),(38.33953409861437,57.710289827306724),(38.73245009647167,57.48354156434209),(38.21863686850443,57.271721400459285),(38.97424455669155,56.87744603722649),(37.463029180317314,56.5623320541159),(38.94402024916407,56.05939028508024),(38.18841256097694,55.856355210835915),(38.490655636251795,55.53270067649592),(37.795496563119656,55.39562234093384),(38.30930979108689,55.154587013355666),(36.7074214921302,54.65063295250911),(37.31190764267989,53.92734063371401),(36.979440259877556,53.58783775557231),(37.855945178174615,52.91880107773497),(39.57873070724124,52.69956490610895),(38.33953409861437,53.281741738901104),(40.00187101262603,53.35396273604752),(39.54850639971376,53.58783775557231),(40.24366547284591,53.58783775557231),(39.97164670509855,53.98069568468355),(40.60635716317572,54.03398248547225),(40.39478701048334,54.44025165268903),(39.54850639971376,54.56310590284329),(39.54850639971376,54.87732350170489),(40.39478701048334,54.87732350170489),(40.39478701048334,55.24083903654295),(39.82052516746112,55.2752875586599),(39.760076552406154,55.75443792473942),(40.57613285564824,55.78844000174894),(40.425011318010824,56.19415599955667),(39.82052516746112,56.07626182891758),(39.79030085993364,56.41214455508424),(40.48545993306579,56.495655446714636),(40.33433839542836,56.95993246553937),(39.79030085993364,56.992876013526654),(39.72985224487867,57.46729112028032),(40.33433839542836,57.46729112028032),(40.24366547284591,58.04776787540811),(39.63917932229622,58.04776787540811),(39.63917932229622,58.382088724871295),(40.33433839542836,58.382088724871295),(40.45523562553831,58.9011152358548),(38.52087994377928,59.11898412389468)]]])) format TSV; -select wkt(polygonsIntersectionMercator([[(29.453587685533865,59.779570356240356),(29.393139070478895,52.276266797422124),(40.636581470703206,59.38168915000267),(41.21084331372543,59.103467777099866),(29.786055068336193,52.146627480315004),(31.23682182965546,52.16517054781818),(41.69443223416517,58.85424941916091),(42.51048853740727,58.47703162291134),(32.59691566839227,52.22075341251539),(34.289476889931414,52.22075341251539),(43.02430176537451,58.07974369546071),(43.02430176537451,57.25537683364851),(35.468224883503325,52.2022335126388),(37.16078610504247,52.23926559241349),(43.02430176537451,56.26136189644947),(43.02430176537451,55.326904361850836),(38.33953409861437,52.16517054781818),(40.09254393520848,52.16517054781818),(44.4146199116388,55.3097062225408),(44.47506852669377,59.80998197603594),(39.72985224487867,59.931351417569715),(30.23941968124846,53.67744677450975),(30.20919537372098,54.63314259659509),(38.73245009647167,59.94649146557819),(37.2816833351524,59.97675082987618),(30.23941968124846,55.2752875586599),(30.33009260383092,56.19415599955667),(36.28428118674541,59.96162460231375),(34.863738732953635,59.97675082987618),(30.178971066193498,56.97640788219866),(30.178971066193498,57.91957806959033),(33.65476643185424,59.94649146557819),(32.32489690064491,59.94649146557819),(30.481214141468342,58.85424941916091),(30.571887064050795,59.99187015036608),(29.453587685533865,59.779570356240356)]], [[[(33.473420586689336,58.85424941916091),(32.23422397806246,58.492830557036),(32.173775363007486,58.03176922751564),(31.508840597402823,57.499784781503735),(31.750635057622702,56.86092686957355),(31.508840597402823,55.941082594334574),(32.20399967053497,55.515591939372456),(31.84130798020516,54.998862226280465),(31.418167674820367,54.422670886434275),(32.47601843828233,53.83826377018255),(32.08310244042503,53.408048308050866),(33.171177511414484,52.82758702113742),(34.77306581037117,52.91880107773494),(34.77306581037117,53.784726518357985),(34.108131044766516,54.17574726780569),(35.07530888564602,54.59813930694554),(34.25925258240394,54.96417435716029),(35.01486027059106,55.361278263643584),(33.50364489421682,55.37845402950552),(32.7480372060297,55.90721384574556),(35.67979503619571,55.68634475630185),(32.83871012861215,56.311688992608396),(34.591719965206266,56.29492065473883),(35.7100193437232,56.311688992608396),(33.83611227701915,56.695333481003644),(32.95960735872209,56.9434497616887),(36.072711034053015,57.091531913901434),(33.171177511414484,57.33702717078384),(36.193608264162954,57.499784781503735),(33.23162612646945,57.77481561306047),(36.43540272438284,58.04776787540811),(33.62454212432676,58.27099811968307),(36.344729801800376,58.54018474404165),(33.83611227701915,58.68186423448108),(34.74284150284369,59.565911441555244),(33.473420586689336,58.85424941916091)]], [[(34.65216858026123,58.91672306881671),(37.19101041256995,58.68186423448108),(36.01226241899805,58.28688958537609),(37.16078610504247,58.04776787540811),(35.74024365125068,57.79092907387934),(37.009664567405046,57.499784781503735),(35.77046795877817,57.25537683364851),(36.979440259877556,57.07510745541089),(34.22902827487645,56.794777197297435),(36.7074214921302,56.210968525786996),(34.712617195316206,56.10998276812964),(36.55629995449277,55.63519693782703),(35.13575750070099,55.53270067649592),(36.43540272438284,55.34409504165558),(34.83351442542614,55.01619492319591),(35.61934642114075,54.49294870011772),(34.89396304048112,54.12264226523038),(35.37755196092087,53.046178687628185),(37.43280487278982,52.95523300597458),(35.92158949641559,53.80257986695776),(36.91899164482259,53.856094327816805),(36.01226241899805,54.75541714463799),(37.765272255592166,55.189110239786885),(36.828318722240134,55.44708256557195),(38.03729102333953,55.652253637168315),(36.64697287707522,55.941082594334574),(38.21863686850443,56.05939028508024),(36.37495410932787,56.64551287174558),(38.30930979108689,56.992876013526654),(37.16078610504247,57.25537683364851),(38.127963945921984,57.516020773674256),(37.43280487278982,57.710289827306724),(38.33953409861437,57.935626886818994),(37.40258056526235,58.31865112960426),(38.58132855883426,58.744648733419496),(37.31190764267989,59.02578062465136),(34.65216858026123,58.91672306881671)]], [[(38.52087994377928,59.11898412389468),(39.54850639971376,58.713270635642914),(38.369758406141855,58.28688958537609),(38.85334732658162,58.06375936407028),(38.33953409861437,57.710289827306724),(38.73245009647167,57.48354156434209),(38.21863686850443,57.271721400459285),(38.97424455669155,56.87744603722649),(37.463029180317314,56.5623320541159),(38.94402024916407,56.05939028508024),(38.18841256097694,55.856355210835915),(38.490655636251795,55.53270067649592),(37.795496563119656,55.39562234093384),(38.30930979108689,55.154587013355666),(36.7074214921302,54.65063295250911),(37.31190764267989,53.92734063371401),(36.979440259877556,53.58783775557231),(37.855945178174615,52.91880107773497),(39.57873070724124,52.69956490610895),(38.33953409861437,53.281741738901104),(40.00187101262603,53.35396273604752),(39.54850639971376,53.58783775557231),(40.24366547284591,53.58783775557231),(39.97164670509855,53.98069568468355),(40.60635716317572,54.03398248547225),(40.39478701048334,54.44025165268903),(39.54850639971376,54.56310590284329),(39.54850639971376,54.87732350170489),(40.39478701048334,54.87732350170489),(40.39478701048334,55.24083903654295),(39.82052516746112,55.2752875586599),(39.760076552406154,55.75443792473942),(40.57613285564824,55.78844000174894),(40.425011318010824,56.19415599955667),(39.82052516746112,56.07626182891758),(39.79030085993364,56.41214455508424),(40.48545993306579,56.495655446714636),(40.33433839542836,56.95993246553937),(39.79030085993364,56.992876013526654),(39.72985224487867,57.46729112028032),(40.33433839542836,57.46729112028032),(40.24366547284591,58.04776787540811),(39.63917932229622,58.04776787540811),(39.63917932229622,58.382088724871295),(40.33433839542836,58.382088724871295),(40.45523562553831,58.9011152358548),(38.52087994377928,59.11898412389468)]]])) format TSV; select '-------- MultiPolygon with Polygon with Holes'; select wkt(polygonsIntersectionSpherical([[[(33.473420586689336,58.85424941916091),(32.23422397806246,58.492830557036),(32.173775363007486,58.03176922751564),(31.508840597402823,57.499784781503735),(31.750635057622702,56.86092686957355),(31.508840597402823,55.941082594334574),(32.20399967053497,55.515591939372456),(31.84130798020516,54.998862226280465),(31.418167674820367,54.422670886434275),(32.47601843828233,53.83826377018255),(32.08310244042503,53.408048308050866),(33.171177511414484,52.82758702113742),(34.77306581037117,52.91880107773494),(34.77306581037117,53.784726518357985),(34.108131044766516,54.17574726780569),(35.07530888564602,54.59813930694554),(34.25925258240394,54.96417435716029),(35.01486027059106,55.361278263643584),(33.50364489421682,55.37845402950552),(32.7480372060297,55.90721384574556),(35.67979503619571,55.68634475630185),(32.83871012861215,56.311688992608396),(34.591719965206266,56.29492065473883),(35.7100193437232,56.311688992608396),(33.83611227701915,56.695333481003644),(32.95960735872209,56.9434497616887),(36.072711034053015,57.091531913901434),(33.171177511414484,57.33702717078384),(36.193608264162954,57.499784781503735),(33.23162612646945,57.77481561306047),(36.43540272438284,58.04776787540811),(33.62454212432676,58.27099811968307),(36.344729801800376,58.54018474404165),(33.83611227701915,58.68186423448108),(34.74284150284369,59.565911441555244),(33.473420586689336,58.85424941916091)]], [[(34.65216858026123,58.91672306881671),(37.19101041256995,58.68186423448108),(36.01226241899805,58.28688958537609),(37.16078610504247,58.04776787540811),(35.74024365125068,57.79092907387934),(37.009664567405046,57.499784781503735),(35.77046795877817,57.25537683364851),(36.979440259877556,57.07510745541089),(34.22902827487645,56.794777197297435),(36.7074214921302,56.210968525786996),(34.712617195316206,56.10998276812964),(36.55629995449277,55.63519693782703),(35.13575750070099,55.53270067649592),(36.43540272438284,55.34409504165558),(34.83351442542614,55.01619492319591),(35.61934642114075,54.49294870011772),(34.89396304048112,54.12264226523038),(35.37755196092087,53.046178687628185),(37.43280487278982,52.95523300597458),(35.92158949641559,53.80257986695776),(36.91899164482259,53.856094327816805),(36.01226241899805,54.75541714463799),(37.765272255592166,55.189110239786885),(36.828318722240134,55.44708256557195),(38.03729102333953,55.652253637168315),(36.64697287707522,55.941082594334574),(38.21863686850443,56.05939028508024),(36.37495410932787,56.64551287174558),(38.30930979108689,56.992876013526654),(37.16078610504247,57.25537683364851),(38.127963945921984,57.516020773674256),(37.43280487278982,57.710289827306724),(38.33953409861437,57.935626886818994),(37.40258056526235,58.31865112960426),(38.58132855883426,58.744648733419496),(37.31190764267989,59.02578062465136),(34.65216858026123,58.91672306881671)]], [[(38.52087994377928,59.11898412389468),(39.54850639971376,58.713270635642914),(38.369758406141855,58.28688958537609),(38.85334732658162,58.06375936407028),(38.33953409861437,57.710289827306724),(38.73245009647167,57.48354156434209),(38.21863686850443,57.271721400459285),(38.97424455669155,56.87744603722649),(37.463029180317314,56.5623320541159),(38.94402024916407,56.05939028508024),(38.18841256097694,55.856355210835915),(38.490655636251795,55.53270067649592),(37.795496563119656,55.39562234093384),(38.30930979108689,55.154587013355666),(36.7074214921302,54.65063295250911),(37.31190764267989,53.92734063371401),(36.979440259877556,53.58783775557231),(37.855945178174615,52.91880107773497),(39.57873070724124,52.69956490610895),(38.33953409861437,53.281741738901104),(40.00187101262603,53.35396273604752),(39.54850639971376,53.58783775557231),(40.24366547284591,53.58783775557231),(39.97164670509855,53.98069568468355),(40.60635716317572,54.03398248547225),(40.39478701048334,54.44025165268903),(39.54850639971376,54.56310590284329),(39.54850639971376,54.87732350170489),(40.39478701048334,54.87732350170489),(40.39478701048334,55.24083903654295),(39.82052516746112,55.2752875586599),(39.760076552406154,55.75443792473942),(40.57613285564824,55.78844000174894),(40.425011318010824,56.19415599955667),(39.82052516746112,56.07626182891758),(39.79030085993364,56.41214455508424),(40.48545993306579,56.495655446714636),(40.33433839542836,56.95993246553937),(39.79030085993364,56.992876013526654),(39.72985224487867,57.46729112028032),(40.33433839542836,57.46729112028032),(40.24366547284591,58.04776787540811),(39.63917932229622,58.04776787540811),(39.63917932229622,58.382088724871295),(40.33433839542836,58.382088724871295),(40.45523562553831,58.9011152358548),(38.52087994377928,59.11898412389468)]]], [[(24.367675781249993,61.45977057029751),(19.577636718749993,58.67693767258692),(19.577636718749993,57.492213666700735),(19.445800781249996,55.87531083569678),(19.445800781249996,54.085173420886775),(17.468261718749996,53.014783245859235),(20.017089843749993,51.563412328675895),(21.203613281249993,50.205033264943324),(26.125488281249993,50.40151532278236),(27.22412109374999,48.980216985374994),(32.80517578124999,49.525208341974405),(35.26611328124999,48.74894534343292),(36.93603515624999,49.66762782262194),(42.56103515625,48.77791275550183),(43.92333984374999,49.8096315635631),(47.17529296875,49.152969656170455),(49.28466796875,50.54136296522162),(48.05419921875,51.17934297928929),(51.39404296875,52.48278022207825),(50.64697265625,53.014783245859235),(52.88818359375,53.93021986394004),(51.65771484374999,54.29088164657006),(52.66845703125,55.825973254619015),(50.25146484375,56.145549500679095),(51.92138671875,57.914847767009206),(49.15283203125,58.17070248348605),(49.59228515625,60.086762746260064),(47.043457031249986,59.88893689676584),(43.57177734375,61.37567331572748),(42.64892578125,60.630101766266705),(36.89208984374999,62.000904713685856),(36.01318359374999,61.143235250840576),(31.398925781249993,62.02152819100766),(30.563964843749996,61.05828537037917),(26.872558593749993,61.71070595883174),(26.652832031249993,61.10078883158897),(24.367675781249993,61.45977057029751)], [(24.455566406249993,59.42272750081452),(21.203613281249993,58.49369382056807),(21.335449218749993,56.89700392127261),(21.599121093749993,55.92458580482949),(25.202636718749993,55.998380955359636),(28.850097656249993,57.06463027327854),(27.09228515625,57.844750992890994),(28.806152343749996,59.17592824927138),(26.257324218749993,59.17592824927138),(24.455566406249993,59.42272750081452)], [(35.13427734375,59.84481485969107),(31.970214843749993,58.97266715450152),(33.20068359374999,56.776808316568406),(36.67236328125,56.41390137600675),(39.08935546874999,57.25528054528888),(42.69287109374999,58.03137242177638),(40.89111328124999,59.26588062825809),(37.28759765625,58.722598828043374),(37.11181640624999,59.66774058164964),(35.13427734375,59.84481485969107)], [(29.157714843749993,55.75184939173528),(22.565917968749993,55.128649068488784),(22.565917968749993,53.54030739150019),(22.038574218749996,51.48138289610097),(26.257324218749993,51.42661449707484),(30.124511718749993,50.54136296522162),(32.18994140624999,51.17934297928929),(30.124511718749993,53.173119202640635),(35.09033203124999,53.173119202640635),(33.11279296875,54.085173420886775),(29.597167968749993,55.50374985927513),(29.157714843749993,55.75184939173528)], [(42.82470703125,56.58369172128337),(36.584472656249986,55.329144408405085),(37.99072265625,53.592504809039355),(34.95849609374999,51.48138289610097),(36.54052734374999,50.40151532278236),(39.66064453124999,50.289339253291786),(39.79248046875,52.13348804077148),(41.77001953125,50.68079714532166),(44.49462890624999,51.97134580885171),(47.30712890624999,52.509534770327264),(44.05517578125,53.54030739150019),(46.60400390625,53.696706475303245),(47.61474609375,55.40406982700608),(45.37353515625,55.40406982700608),(42.82470703125,56.58369172128337)]])) format TSV; -select wkt(polygonsIntersectionMercator([[[(33.473420586689336,58.85424941916091),(32.23422397806246,58.492830557036),(32.173775363007486,58.03176922751564),(31.508840597402823,57.499784781503735),(31.750635057622702,56.86092686957355),(31.508840597402823,55.941082594334574),(32.20399967053497,55.515591939372456),(31.84130798020516,54.998862226280465),(31.418167674820367,54.422670886434275),(32.47601843828233,53.83826377018255),(32.08310244042503,53.408048308050866),(33.171177511414484,52.82758702113742),(34.77306581037117,52.91880107773494),(34.77306581037117,53.784726518357985),(34.108131044766516,54.17574726780569),(35.07530888564602,54.59813930694554),(34.25925258240394,54.96417435716029),(35.01486027059106,55.361278263643584),(33.50364489421682,55.37845402950552),(32.7480372060297,55.90721384574556),(35.67979503619571,55.68634475630185),(32.83871012861215,56.311688992608396),(34.591719965206266,56.29492065473883),(35.7100193437232,56.311688992608396),(33.83611227701915,56.695333481003644),(32.95960735872209,56.9434497616887),(36.072711034053015,57.091531913901434),(33.171177511414484,57.33702717078384),(36.193608264162954,57.499784781503735),(33.23162612646945,57.77481561306047),(36.43540272438284,58.04776787540811),(33.62454212432676,58.27099811968307),(36.344729801800376,58.54018474404165),(33.83611227701915,58.68186423448108),(34.74284150284369,59.565911441555244),(33.473420586689336,58.85424941916091)]], [[(34.65216858026123,58.91672306881671),(37.19101041256995,58.68186423448108),(36.01226241899805,58.28688958537609),(37.16078610504247,58.04776787540811),(35.74024365125068,57.79092907387934),(37.009664567405046,57.499784781503735),(35.77046795877817,57.25537683364851),(36.979440259877556,57.07510745541089),(34.22902827487645,56.794777197297435),(36.7074214921302,56.210968525786996),(34.712617195316206,56.10998276812964),(36.55629995449277,55.63519693782703),(35.13575750070099,55.53270067649592),(36.43540272438284,55.34409504165558),(34.83351442542614,55.01619492319591),(35.61934642114075,54.49294870011772),(34.89396304048112,54.12264226523038),(35.37755196092087,53.046178687628185),(37.43280487278982,52.95523300597458),(35.92158949641559,53.80257986695776),(36.91899164482259,53.856094327816805),(36.01226241899805,54.75541714463799),(37.765272255592166,55.189110239786885),(36.828318722240134,55.44708256557195),(38.03729102333953,55.652253637168315),(36.64697287707522,55.941082594334574),(38.21863686850443,56.05939028508024),(36.37495410932787,56.64551287174558),(38.30930979108689,56.992876013526654),(37.16078610504247,57.25537683364851),(38.127963945921984,57.516020773674256),(37.43280487278982,57.710289827306724),(38.33953409861437,57.935626886818994),(37.40258056526235,58.31865112960426),(38.58132855883426,58.744648733419496),(37.31190764267989,59.02578062465136),(34.65216858026123,58.91672306881671)]], [[(38.52087994377928,59.11898412389468),(39.54850639971376,58.713270635642914),(38.369758406141855,58.28688958537609),(38.85334732658162,58.06375936407028),(38.33953409861437,57.710289827306724),(38.73245009647167,57.48354156434209),(38.21863686850443,57.271721400459285),(38.97424455669155,56.87744603722649),(37.463029180317314,56.5623320541159),(38.94402024916407,56.05939028508024),(38.18841256097694,55.856355210835915),(38.490655636251795,55.53270067649592),(37.795496563119656,55.39562234093384),(38.30930979108689,55.154587013355666),(36.7074214921302,54.65063295250911),(37.31190764267989,53.92734063371401),(36.979440259877556,53.58783775557231),(37.855945178174615,52.91880107773497),(39.57873070724124,52.69956490610895),(38.33953409861437,53.281741738901104),(40.00187101262603,53.35396273604752),(39.54850639971376,53.58783775557231),(40.24366547284591,53.58783775557231),(39.97164670509855,53.98069568468355),(40.60635716317572,54.03398248547225),(40.39478701048334,54.44025165268903),(39.54850639971376,54.56310590284329),(39.54850639971376,54.87732350170489),(40.39478701048334,54.87732350170489),(40.39478701048334,55.24083903654295),(39.82052516746112,55.2752875586599),(39.760076552406154,55.75443792473942),(40.57613285564824,55.78844000174894),(40.425011318010824,56.19415599955667),(39.82052516746112,56.07626182891758),(39.79030085993364,56.41214455508424),(40.48545993306579,56.495655446714636),(40.33433839542836,56.95993246553937),(39.79030085993364,56.992876013526654),(39.72985224487867,57.46729112028032),(40.33433839542836,57.46729112028032),(40.24366547284591,58.04776787540811),(39.63917932229622,58.04776787540811),(39.63917932229622,58.382088724871295),(40.33433839542836,58.382088724871295),(40.45523562553831,58.9011152358548),(38.52087994377928,59.11898412389468)]]], [[(24.367675781249993,61.45977057029751),(19.577636718749993,58.67693767258692),(19.577636718749993,57.492213666700735),(19.445800781249996,55.87531083569678),(19.445800781249996,54.085173420886775),(17.468261718749996,53.014783245859235),(20.017089843749993,51.563412328675895),(21.203613281249993,50.205033264943324),(26.125488281249993,50.40151532278236),(27.22412109374999,48.980216985374994),(32.80517578124999,49.525208341974405),(35.26611328124999,48.74894534343292),(36.93603515624999,49.66762782262194),(42.56103515625,48.77791275550183),(43.92333984374999,49.8096315635631),(47.17529296875,49.152969656170455),(49.28466796875,50.54136296522162),(48.05419921875,51.17934297928929),(51.39404296875,52.48278022207825),(50.64697265625,53.014783245859235),(52.88818359375,53.93021986394004),(51.65771484374999,54.29088164657006),(52.66845703125,55.825973254619015),(50.25146484375,56.145549500679095),(51.92138671875,57.914847767009206),(49.15283203125,58.17070248348605),(49.59228515625,60.086762746260064),(47.043457031249986,59.88893689676584),(43.57177734375,61.37567331572748),(42.64892578125,60.630101766266705),(36.89208984374999,62.000904713685856),(36.01318359374999,61.143235250840576),(31.398925781249993,62.02152819100766),(30.563964843749996,61.05828537037917),(26.872558593749993,61.71070595883174),(26.652832031249993,61.10078883158897),(24.367675781249993,61.45977057029751)], [(24.455566406249993,59.42272750081452),(21.203613281249993,58.49369382056807),(21.335449218749993,56.89700392127261),(21.599121093749993,55.92458580482949),(25.202636718749993,55.998380955359636),(28.850097656249993,57.06463027327854),(27.09228515625,57.844750992890994),(28.806152343749996,59.17592824927138),(26.257324218749993,59.17592824927138),(24.455566406249993,59.42272750081452)], [(35.13427734375,59.84481485969107),(31.970214843749993,58.97266715450152),(33.20068359374999,56.776808316568406),(36.67236328125,56.41390137600675),(39.08935546874999,57.25528054528888),(42.69287109374999,58.03137242177638),(40.89111328124999,59.26588062825809),(37.28759765625,58.722598828043374),(37.11181640624999,59.66774058164964),(35.13427734375,59.84481485969107)], [(29.157714843749993,55.75184939173528),(22.565917968749993,55.128649068488784),(22.565917968749993,53.54030739150019),(22.038574218749996,51.48138289610097),(26.257324218749993,51.42661449707484),(30.124511718749993,50.54136296522162),(32.18994140624999,51.17934297928929),(30.124511718749993,53.173119202640635),(35.09033203124999,53.173119202640635),(33.11279296875,54.085173420886775),(29.597167968749993,55.50374985927513),(29.157714843749993,55.75184939173528)], [(42.82470703125,56.58369172128337),(36.584472656249986,55.329144408405085),(37.99072265625,53.592504809039355),(34.95849609374999,51.48138289610097),(36.54052734374999,50.40151532278236),(39.66064453124999,50.289339253291786),(39.79248046875,52.13348804077148),(41.77001953125,50.68079714532166),(44.49462890624999,51.97134580885171),(47.30712890624999,52.509534770327264),(44.05517578125,53.54030739150019),(46.60400390625,53.696706475303245),(47.61474609375,55.40406982700608),(45.37353515625,55.40406982700608),(42.82470703125,56.58369172128337)]])) format TSV; select '-------- Polygon with Polygon with Holes'; select wkt(polygonsIntersectionSpherical([[(29.453587685533865,59.779570356240356),(29.393139070478895,52.276266797422124),(40.636581470703206,59.38168915000267),(41.21084331372543,59.103467777099866),(29.786055068336193,52.146627480315004),(31.23682182965546,52.16517054781818),(41.69443223416517,58.85424941916091),(42.51048853740727,58.47703162291134),(32.59691566839227,52.22075341251539),(34.289476889931414,52.22075341251539),(43.02430176537451,58.07974369546071),(43.02430176537451,57.25537683364851),(35.468224883503325,52.2022335126388),(37.16078610504247,52.23926559241349),(43.02430176537451,56.26136189644947),(43.02430176537451,55.326904361850836),(38.33953409861437,52.16517054781818),(40.09254393520848,52.16517054781818),(44.4146199116388,55.3097062225408),(44.47506852669377,59.80998197603594),(39.72985224487867,59.931351417569715),(30.23941968124846,53.67744677450975),(30.20919537372098,54.63314259659509),(38.73245009647167,59.94649146557819),(37.2816833351524,59.97675082987618),(30.23941968124846,55.2752875586599),(30.33009260383092,56.19415599955667),(36.28428118674541,59.96162460231375),(34.863738732953635,59.97675082987618),(30.178971066193498,56.97640788219866),(30.178971066193498,57.91957806959033),(33.65476643185424,59.94649146557819),(32.32489690064491,59.94649146557819),(30.481214141468342,58.85424941916091),(30.571887064050795,59.99187015036608),(29.453587685533865,59.779570356240356)]], [[(24.367675781249993,61.45977057029751),(19.577636718749993,58.67693767258692),(19.577636718749993,57.492213666700735),(19.445800781249996,55.87531083569678),(19.445800781249996,54.085173420886775),(17.468261718749996,53.014783245859235),(20.017089843749993,51.563412328675895),(21.203613281249993,50.205033264943324),(26.125488281249993,50.40151532278236),(27.22412109374999,48.980216985374994),(32.80517578124999,49.525208341974405),(35.26611328124999,48.74894534343292),(36.93603515624999,49.66762782262194),(42.56103515625,48.77791275550183),(43.92333984374999,49.8096315635631),(47.17529296875,49.152969656170455),(49.28466796875,50.54136296522162),(48.05419921875,51.17934297928929),(51.39404296875,52.48278022207825),(50.64697265625,53.014783245859235),(52.88818359375,53.93021986394004),(51.65771484374999,54.29088164657006),(52.66845703125,55.825973254619015),(50.25146484375,56.145549500679095),(51.92138671875,57.914847767009206),(49.15283203125,58.17070248348605),(49.59228515625,60.086762746260064),(47.043457031249986,59.88893689676584),(43.57177734375,61.37567331572748),(42.64892578125,60.630101766266705),(36.89208984374999,62.000904713685856),(36.01318359374999,61.143235250840576),(31.398925781249993,62.02152819100766),(30.563964843749996,61.05828537037917),(26.872558593749993,61.71070595883174),(26.652832031249993,61.10078883158897),(24.367675781249993,61.45977057029751)], [(24.455566406249993,59.42272750081452),(21.203613281249993,58.49369382056807),(21.335449218749993,56.89700392127261),(21.599121093749993,55.92458580482949),(25.202636718749993,55.998380955359636),(28.850097656249993,57.06463027327854),(27.09228515625,57.844750992890994),(28.806152343749996,59.17592824927138),(26.257324218749993,59.17592824927138),(24.455566406249993,59.42272750081452)], [(35.13427734375,59.84481485969107),(31.970214843749993,58.97266715450152),(33.20068359374999,56.776808316568406),(36.67236328125,56.41390137600675),(39.08935546874999,57.25528054528888),(42.69287109374999,58.03137242177638),(40.89111328124999,59.26588062825809),(37.28759765625,58.722598828043374),(37.11181640624999,59.66774058164964),(35.13427734375,59.84481485969107)], [(29.157714843749993,55.75184939173528),(22.565917968749993,55.128649068488784),(22.565917968749993,53.54030739150019),(22.038574218749996,51.48138289610097),(26.257324218749993,51.42661449707484),(30.124511718749993,50.54136296522162),(32.18994140624999,51.17934297928929),(30.124511718749993,53.173119202640635),(35.09033203124999,53.173119202640635),(33.11279296875,54.085173420886775),(29.597167968749993,55.50374985927513),(29.157714843749993,55.75184939173528)], [(42.82470703125,56.58369172128337),(36.584472656249986,55.329144408405085),(37.99072265625,53.592504809039355),(34.95849609374999,51.48138289610097),(36.54052734374999,50.40151532278236),(39.66064453124999,50.289339253291786),(39.79248046875,52.13348804077148),(41.77001953125,50.68079714532166),(44.49462890624999,51.97134580885171),(47.30712890624999,52.509534770327264),(44.05517578125,53.54030739150019),(46.60400390625,53.696706475303245),(47.61474609375,55.40406982700608),(45.37353515625,55.40406982700608),(42.82470703125,56.58369172128337)]])) format TSV; -select wkt(polygonsIntersectionMercator([[(29.453587685533865,59.779570356240356),(29.393139070478895,52.276266797422124),(40.636581470703206,59.38168915000267),(41.21084331372543,59.103467777099866),(29.786055068336193,52.146627480315004),(31.23682182965546,52.16517054781818),(41.69443223416517,58.85424941916091),(42.51048853740727,58.47703162291134),(32.59691566839227,52.22075341251539),(34.289476889931414,52.22075341251539),(43.02430176537451,58.07974369546071),(43.02430176537451,57.25537683364851),(35.468224883503325,52.2022335126388),(37.16078610504247,52.23926559241349),(43.02430176537451,56.26136189644947),(43.02430176537451,55.326904361850836),(38.33953409861437,52.16517054781818),(40.09254393520848,52.16517054781818),(44.4146199116388,55.3097062225408),(44.47506852669377,59.80998197603594),(39.72985224487867,59.931351417569715),(30.23941968124846,53.67744677450975),(30.20919537372098,54.63314259659509),(38.73245009647167,59.94649146557819),(37.2816833351524,59.97675082987618),(30.23941968124846,55.2752875586599),(30.33009260383092,56.19415599955667),(36.28428118674541,59.96162460231375),(34.863738732953635,59.97675082987618),(30.178971066193498,56.97640788219866),(30.178971066193498,57.91957806959033),(33.65476643185424,59.94649146557819),(32.32489690064491,59.94649146557819),(30.481214141468342,58.85424941916091),(30.571887064050795,59.99187015036608),(29.453587685533865,59.779570356240356)]], [[(24.367675781249993,61.45977057029751),(19.577636718749993,58.67693767258692),(19.577636718749993,57.492213666700735),(19.445800781249996,55.87531083569678),(19.445800781249996,54.085173420886775),(17.468261718749996,53.014783245859235),(20.017089843749993,51.563412328675895),(21.203613281249993,50.205033264943324),(26.125488281249993,50.40151532278236),(27.22412109374999,48.980216985374994),(32.80517578124999,49.525208341974405),(35.26611328124999,48.74894534343292),(36.93603515624999,49.66762782262194),(42.56103515625,48.77791275550183),(43.92333984374999,49.8096315635631),(47.17529296875,49.152969656170455),(49.28466796875,50.54136296522162),(48.05419921875,51.17934297928929),(51.39404296875,52.48278022207825),(50.64697265625,53.014783245859235),(52.88818359375,53.93021986394004),(51.65771484374999,54.29088164657006),(52.66845703125,55.825973254619015),(50.25146484375,56.145549500679095),(51.92138671875,57.914847767009206),(49.15283203125,58.17070248348605),(49.59228515625,60.086762746260064),(47.043457031249986,59.88893689676584),(43.57177734375,61.37567331572748),(42.64892578125,60.630101766266705),(36.89208984374999,62.000904713685856),(36.01318359374999,61.143235250840576),(31.398925781249993,62.02152819100766),(30.563964843749996,61.05828537037917),(26.872558593749993,61.71070595883174),(26.652832031249993,61.10078883158897),(24.367675781249993,61.45977057029751)], [(24.455566406249993,59.42272750081452),(21.203613281249993,58.49369382056807),(21.335449218749993,56.89700392127261),(21.599121093749993,55.92458580482949),(25.202636718749993,55.998380955359636),(28.850097656249993,57.06463027327854),(27.09228515625,57.844750992890994),(28.806152343749996,59.17592824927138),(26.257324218749993,59.17592824927138),(24.455566406249993,59.42272750081452)], [(35.13427734375,59.84481485969107),(31.970214843749993,58.97266715450152),(33.20068359374999,56.776808316568406),(36.67236328125,56.41390137600675),(39.08935546874999,57.25528054528888),(42.69287109374999,58.03137242177638),(40.89111328124999,59.26588062825809),(37.28759765625,58.722598828043374),(37.11181640624999,59.66774058164964),(35.13427734375,59.84481485969107)], [(29.157714843749993,55.75184939173528),(22.565917968749993,55.128649068488784),(22.565917968749993,53.54030739150019),(22.038574218749996,51.48138289610097),(26.257324218749993,51.42661449707484),(30.124511718749993,50.54136296522162),(32.18994140624999,51.17934297928929),(30.124511718749993,53.173119202640635),(35.09033203124999,53.173119202640635),(33.11279296875,54.085173420886775),(29.597167968749993,55.50374985927513),(29.157714843749993,55.75184939173528)], [(42.82470703125,56.58369172128337),(36.584472656249986,55.329144408405085),(37.99072265625,53.592504809039355),(34.95849609374999,51.48138289610097),(36.54052734374999,50.40151532278236),(39.66064453124999,50.289339253291786),(39.79248046875,52.13348804077148),(41.77001953125,50.68079714532166),(44.49462890624999,51.97134580885171),(47.30712890624999,52.509534770327264),(44.05517578125,53.54030739150019),(46.60400390625,53.696706475303245),(47.61474609375,55.40406982700608),(45.37353515625,55.40406982700608),(42.82470703125,56.58369172128337)]])) format TSV; From 0612d9ed096e357ddafbc1cff7dedcde3bb8a379 Mon Sep 17 00:00:00 2001 From: Nikita Mikhaylov Date: Sat, 6 Mar 2021 02:09:00 +0300 Subject: [PATCH 315/544] Revert "mercator" This reverts commit 27756c21d3691f270bcb36d0de271eed26f4c69a. --- src/Functions/geometryConverters.h | 1 - src/Functions/tests/CMakeLists.txt | 2 - src/Functions/tests/mercator.cpp | 169 ----------------------------- 3 files changed, 172 deletions(-) delete mode 100644 src/Functions/tests/mercator.cpp diff --git a/src/Functions/geometryConverters.h b/src/Functions/geometryConverters.h index f2706c70e3e..283bb1bb7f4 100644 --- a/src/Functions/geometryConverters.h +++ b/src/Functions/geometryConverters.h @@ -43,7 +43,6 @@ using CartesianRing = Ring; using CartesianPolygon = Polygon; using CartesianMultiPolygon = MultiPolygon; -/// Latitude, longitude using SphericalPoint = boost::geometry::model::point>; using SphericalRing = Ring; using SphericalPolygon = Polygon; diff --git a/src/Functions/tests/CMakeLists.txt b/src/Functions/tests/CMakeLists.txt index 9e32d4869be..e69de29bb2d 100644 --- a/src/Functions/tests/CMakeLists.txt +++ b/src/Functions/tests/CMakeLists.txt @@ -1,2 +0,0 @@ -add_executable (mercator mercator.cpp) -target_link_libraries (mercator PRIVATE clickhouse_functions) diff --git a/src/Functions/tests/mercator.cpp b/src/Functions/tests/mercator.cpp deleted file mode 100644 index bc9299fdaf1..00000000000 --- a/src/Functions/tests/mercator.cpp +++ /dev/null @@ -1,169 +0,0 @@ -#include -#include - -#include - - -constexpr double PI = 3.14159265358979323846; - -/// Convert angle from degrees to radians. -inline constexpr double deg_to_rad(double degree) noexcept { - return degree * (PI / 180.0); -} - -/// Convert angle from radians to degrees. -inline constexpr double rad_to_deg(double radians) noexcept { - return radians * (180.0 / PI); -} - -constexpr double earth_radius_for_epsg3857 = 6378137.0; -// constexpr double max_coordinate_epsg3857 = 20037508.34; - -constexpr inline double lon_to_x(double lon) noexcept { - return earth_radius_for_epsg3857 * deg_to_rad(lon); -} - -// canonical log(tan()) version -inline double lat_to_y_with_tan(double lat) { // not constexpr because math functions aren't - return earth_radius_for_epsg3857 * std::log(std::tan(PI/4 + deg_to_rad(lat)/2)); -} - -constexpr inline double x_to_lon(double x) { - return rad_to_deg(x) / earth_radius_for_epsg3857; -} - -inline double y_to_lat(double y) { // not constexpr because math functions aren't - return rad_to_deg(2 * std::atan(std::exp(y / earth_radius_for_epsg3857)) - PI/2); -} - -/// POLYGON((4.346693 50.858306, 4.367945 50.852455, 4.366227 50.840809, 4.344961 50.833264, 4.338074 50.848677, 4.346693 50.858306)) -/// POLYGON((25.0010 136.9987, 17.7500 142.5000, 11.3733 142.5917)) - - -/// POLYGON((4.3613577 50.8651821, 4.349556 50.8535879, 4.3602419 50.8435626, 4.3830299 50.8428851, 4.3904543 50.8564867, 4.3613148 50.8651279)) -/// POLYGON((4.346693 50.858306, 4.367945 50.852455, 4.366227 50.840809, 4.344961 50.833264, 4.338074 50.848677, 4.346693 50.858306)) - -void mercator(DB::CartesianPolygon & polygon) -{ - for (auto & point : polygon.outer()) - { - point.x(lon_to_x(point.x())); - point.y(lat_to_y_with_tan(point.y())); - } -} - -void reverseMercator(DB::CartesianMultiPolygon & multi_polygon) -{ - for (auto & polygon : multi_polygon) - { - for (auto & point : polygon.outer()) - { - point.x(x_to_lon(point.x())); - point.y(y_to_lat(point.y())); - } - } -} - - -void printMultiPolygon(DB::CartesianMultiPolygon & multi_polygon) -{ - std::cout << "--------------" << std::endl; - for (auto & polygon : multi_polygon) - { - for (auto & point : polygon.outer()) - { - std::cout << point.x() << ' ' << point.y() << std::endl; - } - } - std::cout << "--------------" << std::endl; -} - -void test1() -{ - DB::CartesianPolygon green, blue; - boost::geometry::read_wkt( - "POLYGON((4.346693 50.858306, 4.367945 50.852455, 4.366227 50.840809, 4.344961 50.833264, 4.338074 50.848677, 4.346693 50.858306))", green); - - boost::geometry::read_wkt( - "POLYGON((25.0010 136.9987, 17.7500 142.5000, 11.3733 142.5917))", blue); - - mercator(green); - mercator(blue); - - DB::CartesianMultiPolygon output; - boost::geometry::intersection(green, blue, output); - - reverseMercator(output); - - printMultiPolygon(output); -} - - -// 4.3666052904432435, 50.84337386140151 -// 4.3602419 50.8435626 -// 4.349556 50.8535879 -// 4.3526804582393535 50.856658100365976 -// 4.367945 50.852455 -// 4.3666052904432435 50.84337386140151 - - -void test2() -{ - DB::CartesianPolygon green, blue; - boost::geometry::read_wkt( - "POLYGON((4.3613577 50.8651821, 4.349556 50.8535879, 4.3602419 50.8435626, 4.3830299 50.8428851, 4.3904543 50.8564867, 4.3613148 50.8651279))", green); - - boost::geometry::read_wkt( - "POLYGON((4.346693 50.858306, 4.367945 50.852455, 4.366227 50.840809, 4.344961 50.833264, 4.338074 50.848677, 4.346693 50.858306))", blue); - - boost::geometry::correct(green); - boost::geometry::correct(blue); - - mercator(green); - mercator(blue); - - DB::CartesianMultiPolygon output; - boost::geometry::intersection(green, blue, output); - - reverseMercator(output); - - boost::geometry::correct(output); - - printMultiPolygon(output); -} - - -void test3() -{ - DB::CartesianPolygon green, blue; - boost::geometry::read_wkt( - "POLYGON((4.3613577 50.8651821, 4.349556 50.8535879, 4.3602419 50.8435626, 4.3830299 50.8428851, 4.3904543 50.8564867, 4.3613148 50.8651279))", green); - - boost::geometry::read_wkt( - "POLYGON((4.3613577 50.8651821, 4.349556 50.8535879, 4.3602419 50.8435626, 4.3830299 50.8428851, 4.3904543 50.8564867, 4.3613148 50.8651279))", blue); - - boost::geometry::correct(green); - boost::geometry::correct(blue); - - mercator(green); - mercator(blue); - - DB::CartesianMultiPolygon output; - boost::geometry::intersection(green, blue, output); - - reverseMercator(output); - - boost::geometry::correct(output); - - printMultiPolygon(output); -} - -int main(int argc, char ** argv) -{ - (void) argc; - (void) argv; - test1(); - test2(); - test3(); - return 0; -} From 7ecc78e014cccb153d36471b5fbc06d5aad65a9d Mon Sep 17 00:00:00 2001 From: Nikita Mikhaylov Date: Sat, 6 Mar 2021 02:40:21 +0300 Subject: [PATCH 316/544] fix tests after revert --- .../queries/0_stateless/01301_polygons_within.reference | 8 -------- tests/queries/0_stateless/01305_polygons_union.reference | 8 -------- .../0_stateless/01306_polygons_intersection.reference | 9 --------- 3 files changed, 25 deletions(-) diff --git a/tests/queries/0_stateless/01301_polygons_within.reference b/tests/queries/0_stateless/01301_polygons_within.reference index d922e69f725..ee14f826295 100644 --- a/tests/queries/0_stateless/01301_polygons_within.reference +++ b/tests/queries/0_stateless/01301_polygons_within.reference @@ -2,17 +2,9 @@ 1 0 1 -<<<<<<< HEAD -0 -1 -------- MultiPolygon with Polygon 0 -0 -------- MultiPolygon with Polygon with Holes 0 -0 -------- Polygon with Polygon with Holes 0 -0 -======= ->>>>>>> parent of d4258922d1... more mercator diff --git a/tests/queries/0_stateless/01305_polygons_union.reference b/tests/queries/0_stateless/01305_polygons_union.reference index 6a72c60c4fa..f87d03c151c 100644 --- a/tests/queries/0_stateless/01305_polygons_union.reference +++ b/tests/queries/0_stateless/01305_polygons_union.reference @@ -1,16 +1,8 @@ [[[(1,2.9),(1,4),(4,4),(4,1),(2.9,1),(3,0),(0,0),(0,3),(1,2.9)]]] [[[(4.3666052904432435,50.84337386140151),(4.366227,50.840809),(4.344961,50.833264),(4.338074,50.848677),(4.346693,50.858306),(4.3526804582393535,50.856658100365976),(4.3613577,50.8651821),(4.3613148,50.8651279),(4.3904543,50.8564867),(4.3830299,50.8428851),(4.3666052904432435,50.84337386140151)]]] -<<<<<<< HEAD --------- -[[[(4.366505261795747,50.843273415405),(4.3661270000000005,50.840709),(4.344861,50.83316399999998),(4.337974,50.84857699999999),(4.346593,50.858205999999996),(4.352580404040105,50.85655765067624),(4.3612577,50.86508209999998),(4.361214800000001,50.8650279),(4.3903543,50.85638669999999),(4.382929900000001,50.842785099999986),(4.366505261795747,50.843273415405)]]] -------- MultiPolygon with Polygon MULTIPOLYGON(((35.5408 58.9593,37.2817 59.9768,38.7325 59.9465,36.9725 59.0149,37.3119 59.0258,37.8553 58.9075,39.7299 59.9314,44.4751 59.81,44.4146 55.3097,40.0925 52.1652,38.3395 52.1652,39.1456 52.7573,38.0214 52.8989,37.1608 52.2393,35.4682 52.2022,36.5022 53.0008,35.3776 53.0462,35.3645 53.076,34.2895 52.2208,32.5969 52.2208,33.4048 52.8423,33.1712 52.8276,32.5275 53.1741,31.2368 52.1652,29.7861 52.1466,32.2523 53.964,31.8748 54.1736,29.3931 52.2763,29.4536 59.7796,30.5719 59.9919,30.4812 58.8542,32.3249 59.9465,33.6548 59.9465,30.179 57.9196,30.179 56.9764,32.2175 58.3664,32.2342 58.4928,32.5691 58.5924,34.8637 59.9768,36.2843 59.9616,34.0496 58.6717,34.9952 58.6226,35.3712 58.8556,34.6522 58.9167,35.5408 58.9593),(36.4989 58.7512,36.1498 58.553,36.3447 58.5402,36.0877 58.5174,35.4314 58.1349,36.403 58.0507,36.5949 58.1673,36.0123 58.2869,37.191 58.6819,36.4989 58.7512),(34.4816 56.8232,34.8098 57.0409,33.5602 56.9781,33.3418 56.8364,33.8361 56.6953,34.1885 56.6259,34.3867 56.7596,34.229 56.7948,34.4816 56.8232),(35.9179 57.7512,35.7402 57.7909,36.0848 57.855,36.3932 58.0447,35.1134 57.9454,34.6332 57.6538,35.613 57.5595,35.9179 57.7512),(36.8709 53.2765,37.135 53.4711,36.9794 53.5878,37.3119 53.9273,37.0035 54.2999,36.6985 54.0791,36.919 53.8561,36.3552 53.8269,36.1528 53.6763,36.8709 53.2765),(38.1601 55.1091,38.3093 55.1546,38.2609 55.1775,38.1601 55.1091),(38.1688 56.0758,38.4339 56.2361,37.5054 56.5484,37.2281 56.3799,38.1688 56.0758),(38.1319 56.0534,36.647 55.9411,37.6238 55.7402,38.1319 56.0534),(37.2824 55.5258,36.8283 55.4471,37.06 55.3843,37.2824 55.5258),(36.151 54.791,36.0123 54.7554,36.0472 54.7217,36.151 54.791),(34.9611 53.9765,34.894 54.1226,35.6193 54.4929,34.9706 54.9262,34.7231 54.7576,35.0753 54.5981,34.1081 54.1757,34.7279 53.8116,34.9611 53.9765),(38.2312 56.9795,37.565 56.5843,38.9742 56.8774,38.4915 57.1308,38.2699 57.0021,38.3093 56.9929,38.2312 56.9795),(36.5334 56.6753,36.375 56.6455,36.4446 56.6242,36.5334 56.6753),(36.1999 57.0022,36.9794 57.0751,36.4587 57.1544,36.1999 57.0022),(34.6028 58.3749,33.6245 58.271,34.3593 58.2189,34.6028 58.3749),(33.7581 57.8255,33.2316 57.7748,33.6325 57.7419,33.7581 57.8255),(31.6069 56.3194,31.7506 56.8609,31.6514 57.1258,30.3301 56.1942,30.2394 55.2753,31.6069 56.3194),(34.2274 57.4023,34.0208 57.2724,35.0338 57.1875,35.4682 57.4674,34.2274 57.4023),(31.7782 55.7778,30.2092 54.6331,30.2394 53.6774,31.7439 54.8677,31.8413 54.9989,32.204 55.5156,31.7782 55.7778),(33.7222 56.3063,32.8387 56.3117,33.5244 56.1686,33.7222 56.3063),(33.1204 55.8832,32.748 55.9072,32.9547 55.7645,33.1204 55.8832),(35.2275 55.0993,36.4354 55.3441,35.7505 55.4454,35.2275 55.0993),(35.9817 55.5958,36.5563 55.6352,36.193 55.7319,35.9817 55.5958),(35.0954 55.822,35.3188 55.9582,34.7331 56.1049,34.4996 55.9565,35.0954 55.822),(34.9721 55.7463,34.2598 55.8023,33.6125 55.3778,34.3709 55.3709,34.9721 55.7463),(35.6571 56.1619,36.0233 56.3789,35.4083 56.5254,35.2273 56.414,35.71 56.3117,35.0485 56.303,34.744 56.1118,35.6571 56.1619),(40.2143 54.467,40.3948 54.4403,40.6064 54.034,39.9716 53.9807,40.2437 53.5878,39.5485 53.5878,39.9942 53.358,43.0243 55.3269,43.0243 56.2614,40.2143 54.467),(38.5511 53.2922,38.4609 53.226,39.2704 52.8471,39.9877 53.3534,38.5511 53.2922),(40.5716 55.8007,43.0243 57.2554,43.0243 58.0797,40.4543 56.5923,40.4855 56.4957,40.2529 56.4682,39.8102 56.1914,39.8205 56.0763,40.425 56.1942,40.5716 55.8007),(40.5504 55.7875,39.7601 55.7544,39.8151 55.3187,40.5504 55.7875),(39.7863 57.025,42.5105 58.477,41.6944 58.8542,40.1389 58.048,40.2437 58.0478,40.3343 57.4673,39.7299 57.4673,39.7863 57.025),(38.0744 57.5312,38.3737 57.6908,38.3395 57.7103,38.8533 58.0638,38.432 58.2584,38.0535 58.0542,38.3395 57.9356,37.4328 57.7103,38.0744 57.5312),(37.9669 57.4734,37.1608 57.2554,37.4489 57.1909,37.9669 57.4734),(40.4136 58.7241,41.2108 59.1035,40.6366 59.3817,39.8163 58.9766,40.4552 58.9011,40.4136 58.7241),(39.7184 58.3823,39.6392 58.3821,39.6392 58.3427,39.7184 58.3823),(38.7465 58.4255,39.5485 58.7133,39.4085 58.7696,38.7465 58.4255))) -MULTIPOLYGON(((35.677 58.9587,37.2816 59.9767,38.7324 59.9464,37.1527 59.0192,37.3118 59.0257,38.0059 58.8722,39.7298 59.9313,44.475 59.8099,44.4145 55.3096,40.0924 52.1651,38.3394 52.1651,39.1804 52.7502,38.0733 52.8911,37.1607 52.2392,35.4681 52.2021,36.5898 52.9925,35.4415 53.0432,34.2894 52.2207,32.5968 52.2207,33.5229 52.8475,33.1711 52.8275,32.6253 53.1196,31.2367 52.1651,29.786 52.1465,32.4267 53.8656,32.0505 54.0742,29.393 52.2762,29.4535 59.7795,30.5718 59.9918,30.4811 58.8541,32.3248 59.9464,33.6547 59.9464,30.1789 57.9195,30.1789 56.9763,32.2099 58.3086,32.2341 58.4927,32.7181 58.6343,34.8636 59.9767,36.2842 59.9615,34.1607 58.6635,35.1359 58.6084,35.4917 58.8391,34.6521 58.9166,35.677 58.9587),(36.6691 58.7302,35.6455 58.1105,36.4353 58.0477,35.3989 57.9596,34.8546 57.6243,35.8918 57.5278,36.1527 57.6965,35.7401 57.7908,36.5188 57.9318,36.8134 58.1201,36.0122 58.2868,37.1909 58.6818,36.6691 58.7302),(34.8711 56.8603,35.1561 57.0479,33.8307 56.9848,33.5159 56.786,33.836 56.6952,34.4368 56.5727,34.6299 56.7008,34.2289 56.7947,34.8711 56.8603),(36.9376 53.2346,37.2027 53.4183,36.9793 53.5877,37.3118 53.9272,37.0903 54.1938,36.7891 53.9859,36.9189 53.856,36.575 53.8375,36.2567 53.616,36.9376 53.2346),(37.8883 55.6832,38.6515 56.1591,37.7506 56.465,37.4799 56.2952,38.2185 56.0593,36.9557 55.9643,36.8521 55.8985,37.8883 55.6832),(37.7656 55.6061,36.8282 55.447,37.3054 55.3158,37.7656 55.6061),(36.6884 54.9232,36.0122 54.7553,36.1761 54.5942,36.6884 54.9232),(35.0208 53.8426,34.8939 54.1225,35.6192 54.4928,35.153 54.8041,34.9347 54.6613,35.0752 54.598,34.374 54.2922,34.1558 54.1477,34.773 53.7846,34.773 53.6795,35.0208 53.8426),(37.1816 56.7907,36.3749 56.6454,36.7422 56.5294,37.1816 56.7907),(36.7103 57.0477,36.9793 57.075,36.7988 57.102,36.7103 57.0477),(34.7969 58.3872,33.6244 58.2709,34.5125 58.2005,34.7969 58.3872),(33.9633 57.8372,33.2315 57.7747,33.7916 57.7229,33.9633 57.8372),(33.2199 57.3396,33.1711 57.3369,33.211 57.3336,33.2199 57.3396),(31.581 56.2183,31.7505 56.8608,31.6694 57.0765,30.33 56.1941,30.2393 55.2752,31.581 56.2183),(34.5079 57.409,34.2469 57.2461,35.3199 57.1553,35.8173 57.4795,34.5079 57.409),(31.863 55.7247,30.2091 54.633,30.2393 53.6773,31.5514 54.605,31.8412 54.9988,32.2039 55.5155,31.863 55.7247),(34.0285 56.3002,32.8386 56.3116,33.7487 56.1124,34.0285 56.3002),(33.3749 55.86,32.7479 55.9071,33.0915 55.6676,33.3749 55.86),(35.777 55.2096,36.4353 55.344,36.0682 55.3974,35.777 55.2096),(36.4239 55.6256,36.5562 55.6351,36.4727 55.6567,36.4239 55.6256),(35.4341 55.7407,35.6467 55.87,35.0606 56.0207,34.8294 55.8745,35.4341 55.7407),(35.3816 55.7088,34.6546 55.7636,34.0415 55.3722,34.8176 55.3634,35.3816 55.7088),(36.1651 56.1834,36.3512 56.2953,35.7316 56.4418,35.5695 56.3405,35.7099 56.3116,35.5188 56.3087,35.245 56.1369,36.1651 56.1834),(40.3151 54.4517,40.3947 54.4402,40.6063 54.0339,39.9715 53.9806,40.2436 53.5877,39.5484 53.5877,40.0018 53.3539,38.6455 53.2949,38.512 53.2011,39.2985 52.8318,43.0242 55.3268,43.0242 56.2613,40.3151 54.4517),(39.9389 55.2681,43.0242 57.2553,43.0242 58.0796,39.9194 56.0955,40.4249 56.1941,40.576 55.7883,39.76 55.7543,39.8204 55.2752,39.9389 55.2681),(39.9915 56.9806,42.5104 58.4769,41.6943 58.8541,40.2513 57.9982,40.3342 57.4672,39.7298 57.4672,39.7902 56.9928,39.9915 56.9806),(38.1298 56.7016,38.9741 56.8773,38.6704 57.0364,38.1298 56.7016),(37.7436 57.1223,38.5465 57.5909,38.3394 57.7102,38.8532 58.0637,38.591 58.1848,38.2411 57.9759,38.3394 57.9355,38.0568 57.8654,37.6831 57.6403,38.1279 57.5159,37.1607 57.2553,37.7436 57.1223),(40.3959 58.6476,41.2107 59.1034,40.6365 59.3816,39.9119 58.9623,40.4551 58.901,40.3959 58.6476),(39.9258 58.382,39.6391 58.382,39.6391 58.219,39.9258 58.382),(39.3854 58.6545,39.5484 58.7132,39.5108 58.7281,39.3854 58.6545))) -------- MultiPolygon with Polygon with Holes MULTIPOLYGON(((24.3677 61.4598,26.6528 61.1008,26.8726 61.7107,30.564 61.0583,31.3989 62.0215,36.0132 61.1432,36.8921 62.0009,42.6489 60.6301,43.5718 61.3757,47.0435 59.8889,49.5923 60.0868,49.1528 58.1707,51.9214 57.9148,50.2515 56.1455,52.6685 55.826,51.6577 54.2909,52.8882 53.9302,50.647 53.0148,51.394 52.4828,48.0542 51.1793,49.2847 50.5414,47.1753 49.153,43.9233 49.8096,42.561 48.7779,36.936 49.6676,35.2661 48.7489,32.8052 49.5252,27.2241 48.9802,26.1255 50.4015,21.2036 50.205,20.0171 51.5634,17.4683 53.0148,19.4458 54.0852,19.4458 55.8753,19.5776 57.4922,19.5776 58.6769,24.3677 61.4598),(24.4556 59.4227,21.2036 58.4937,21.3354 56.897,21.5991 55.9246,25.2026 55.9984,28.8501 57.0646,27.0923 57.8448,28.8062 59.1759,26.2573 59.1759,24.4556 59.4227),(33.1079 56.9523,33.1392 56.8934,33.7182 56.7292,35.1489 56.5859,34.229 56.7948,36.9794 57.0751,35.7705 57.2554,37.0097 57.4998,35.7402 57.7909,37.1608 58.0478,36.0123 58.2869,37.191 58.6819,34.6522 58.9167,37.2327 59.0233,37.1118 59.6677,35.1343 59.8448,31.9702 58.9727,32.25 58.4976,33.4734 58.8542,34.7428 59.5659,33.8361 58.6819,36.3447 58.5402,33.6245 58.271,36.4354 58.0478,33.2316 57.7748,36.1936 57.4998,33.1712 57.337,36.0727 57.0915,33.1079 56.9523),(37.0604 52.9744,34.9585 51.4814,36.5405 50.4015,39.6606 50.2893,39.7925 52.1335,41.77 50.6808,44.4946 51.9713,47.3071 52.5095,44.0552 53.5403,46.604 53.6967,47.6147 55.4041,45.3735 55.4041,42.8247 56.5837,40.4412 56.1511,40.5761 55.7884,39.7601 55.7544,39.8205 55.2753,40.3948 55.2408,40.3948 54.8773,39.5485 54.8773,39.5485 54.5631,40.3948 54.4403,40.6064 54.034,39.9716 53.9807,40.2437 53.5878,39.5485 53.5878,40.0019 53.354,38.3395 53.2817,39.5787 52.6996,37.8559 52.9188,37.4471 53.2343,37.2165 53.0798,37.4328 52.9552,37.0604 52.9744),(31.627 54.7093,29.5972 55.5037,29.1577 55.7518,22.5659 55.1286,22.5659 53.5403,22.0386 51.4814,26.2573 51.4266,30.1245 50.5414,32.1899 51.1793,30.1245 53.1731,32.4808 53.1989,32.0831 53.408,32.476 53.8383,31.4182 54.4227,31.627 54.7093),(34.7731 53.3243,34.7731 53.1793,35.0903 53.1731,34.7731 53.3243),(36.9508 55.414,37.7653 55.1891,36.8822 54.975,37.0572 54.7635,38.3093 55.1546,37.7955 55.3956,38.4907 55.5327,38.3184 55.7179,38.0262 55.6546,38.0373 55.6523,37.9482 55.6376,36.9508 55.414),(38.3092 56.9929,38.5798 57.0849,38.2186 57.2717,38.7325 57.4835,38.3395 57.7103,38.8533 58.0638,38.3698 58.2869,39.5485 58.7133,38.8838 58.9777,38.0944 58.8545,38.5813 58.7446,37.4026 58.3187,38.3395 57.9356,37.4328 57.7103,38.128 57.516,37.1608 57.2554,38.3092 56.9929),(38.309 56.9928,36.375 56.6455,36.8799 56.4895,38.309 56.9928),(40.3237 57.5365,42.6929 58.0314,40.8911 59.2659,39.2792 59.0373,40.4552 58.9011,40.3343 58.3821,39.6392 58.3821,39.6392 58.0478,40.2437 58.0478,40.3237 57.5365),(40.0149 57.4677,39.7299 57.4673,39.7379 57.4051,40.0149 57.4677))) -MULTIPOLYGON(((24.3676 61.4597,26.6527 61.1007,26.8725 61.7106,30.5639 61.0582,31.3988 62.0214,36.0131 61.1431,36.892 62.0008,42.6488 60.63,43.5717 61.3756,47.0434 59.8888,49.5922 60.0867,49.1527 58.1706,51.9213 57.9147,50.2514 56.1454,52.6684 55.8259,51.6576 54.2908,52.8881 53.9301,50.6469 53.0147,51.3939 52.4827,48.0541 51.1792,49.2846 50.5413,47.1752 49.1529,43.9232 49.8095,42.5609 48.7778,36.9359 49.6675,35.266 48.7488,32.8051 49.5251,27.224 48.9801,26.1254 50.4014,21.2035 50.2049,20.017 51.5633,17.4682 53.0147,19.4457 54.0851,19.4457 55.8752,19.5775 57.4921,19.5775 58.6768,24.3676 61.4597),(24.4555 59.4226,21.2035 58.4936,21.3353 56.8969,21.599 55.9245,25.2025 55.9983,28.85 57.0645,27.0922 57.8447,28.8061 59.1758,26.2572 59.1758,24.4555 59.4226),(33.106 56.9503,33.1372 56.8932,33.7508 56.7194,35.1914 56.569,34.2289 56.7947,36.9793 57.075,35.7704 57.2553,37.0096 57.4997,35.7401 57.7908,37.1607 58.0477,36.0122 58.2868,37.1909 58.6818,34.6521 58.9166,37.2322 59.0224,37.1117 59.6676,35.1342 59.8447,31.9701 58.9726,32.2439 58.4956,33.4733 58.8541,34.7427 59.5658,33.836 58.6818,36.3446 58.5401,33.6244 58.2709,36.4353 58.0477,33.2315 57.7747,36.1935 57.4997,33.1711 57.3369,36.0726 57.0914,33.106 56.9503),(37.0823 52.9707,34.9584 51.4813,36.5404 50.4014,39.6605 50.2892,39.7924 52.1334,41.7699 50.6807,44.4945 51.9712,47.307 52.5094,44.0551 53.5402,46.6039 53.6966,47.6146 55.404,45.3734 55.404,42.8246 56.5836,40.4556 56.1121,40.576 55.7883,39.76 55.7543,39.8204 55.2752,40.3947 55.2407,40.3947 54.8772,39.5484 54.8772,39.5484 54.563,40.3947 54.4402,40.6063 54.0339,39.9715 53.9806,40.2436 53.5877,39.5484 53.5877,40.0018 53.3539,38.3394 53.2816,39.5786 52.6995,37.8558 52.9187,37.4544 53.2264,37.2279 53.0709,37.4327 52.9551,37.0823 52.9707),(34.773 53.173,35.0902 53.173,34.773 53.3207,34.773 53.173),(32.5252 53.173,32.083 53.4079,32.4759 53.8382,31.4181 54.4226,31.6171 54.6946,29.5971 55.5036,29.1576 55.7517,22.5658 55.1285,22.5658 53.5402,22.0385 51.4813,26.2572 51.4265,30.1244 50.5413,32.1898 51.1792,30.1244 53.173,32.5252 53.173),(36.9706 55.4079,37.7652 55.189,36.8796 54.9705,37.0521 54.7595,38.3092 55.1545,37.7954 55.3955,38.4906 55.5326,38.3463 55.6874,36.9706 55.4079),(36.8783 56.4863,38.5847 57.0811,38.2185 57.2716,38.7324 57.4834,38.3394 57.7102,38.8532 58.0637,38.3697 58.2868,39.5484 58.7132,38.9056 58.9675,38.1151 58.848,38.5812 58.7445,37.4025 58.3186,38.3394 57.9355,37.4327 57.7102,38.1279 57.5159,37.1607 57.2553,38.3092 56.9928,36.3749 56.6454,36.8783 56.4863),(40.3255 57.5233,42.6928 58.0313,40.891 59.2658,39.3164 59.0294,40.4551 58.901,40.3342 58.382,39.6391 58.382,39.6391 58.0477,40.2436 58.0477,40.3255 57.5233),(40.066 57.4672,39.7298 57.4672,39.7388 57.3963,40.066 57.4672))) -------- Polygon with Polygon with Holes MULTIPOLYGON(((24.3677 61.4598,26.6528 61.1008,26.8726 61.7107,30.564 61.0583,31.3989 62.0215,36.0132 61.1432,36.8921 62.0009,42.6489 60.6301,43.5718 61.3757,47.0435 59.8889,49.5923 60.0868,49.1528 58.1707,51.9214 57.9148,50.2515 56.1455,52.6685 55.826,51.6577 54.2909,52.8882 53.9302,50.647 53.0148,51.394 52.4828,48.0542 51.1793,49.2847 50.5414,47.1753 49.153,43.9233 49.8096,42.561 48.7779,36.936 49.6676,35.2661 48.7489,32.8052 49.5252,27.2241 48.9802,26.1255 50.4015,21.2036 50.205,20.0171 51.5634,17.4683 53.0148,19.4458 54.0852,19.4458 55.8753,19.5776 57.4922,19.5776 58.6769,24.3677 61.4598),(24.4556 59.4227,21.2036 58.4937,21.3354 56.897,21.5991 55.9246,25.2026 55.9984,28.8501 57.0646,27.0923 57.8448,28.8062 59.1759,26.2573 59.1759,24.4556 59.4227),(32.6512 57.792,32.9378 57.2699,36.7912 59.6986,35.9475 59.7758,32.6512 57.792),(33.2446 56.7729,34.2635 56.6767,37.6322 58.7797,37.2876 58.7226,37.2102 59.1452,33.2446 56.7729),(36.1815 56.4715,41.168 59.0834,40.9299 59.2404,40.8804 59.2644,40.2079 59.1718,35.4536 56.5531,36.1815 56.4715),(30.7705 55.0525,30.2092 54.6331,30.2394 53.6774,31.5682 54.7333,30.7705 55.0525),(33.8733 53.1922,34.3351 53.53,33.5144 53.9057,32.5603 53.1989,33.8733 53.1922),(31.1968 52.1649,29.7861 52.1466,30.5785 52.7531,30.3098 53.0028,29.3931 52.2763,29.4171 55.606,29.1577 55.7518,22.5659 55.1286,22.5659 53.5403,22.0386 51.4814,26.2573 51.4266,30.1245 50.5414,32.1899 51.1793,31.1968 52.1649),(31.1682 53.1903,32.6907 54.2663,32.2591 54.4483,30.5408 53.1811,31.1682 53.1903),(39.4328 55.9511,37.2766 54.4948,37.7431 53.9104,41.4519 56.3413,39.4328 55.9511),(40.9691 57.677,42.2498 58.3455,41.5887 58.8012,38.1759 56.9472,39.0894 57.2553,40.9691 57.677),(37.1934 55.4694,36.5845 55.3291,36.7219 55.1665,37.1934 55.4694),(32.2964 58.4175,34.2247 59.6064,31.9702 58.9727,32.2964 58.4175),(35.9681 52.2157,34.9585 51.4814,36.5405 50.4015,39.6606 50.2893,39.7925 52.1335,41.77 50.6808,44.4946 51.9713,47.3071 52.5095,44.0552 53.5403,46.604 53.6967,47.6147 55.4041,45.3735 55.4041,44.4212 55.8594,44.4146 55.3097,40.0925 52.1652,38.3395 52.1652,43.0243 55.3269,43.0243 56.2614,37.1608 52.2393,35.9681 52.2157))) -MULTIPOLYGON(((24.3676 61.4597,26.6527 61.1007,26.8725 61.7106,30.5639 61.0582,31.3988 62.0214,36.0131 61.1431,36.892 62.0008,42.6488 60.63,43.5717 61.3756,47.0434 59.8888,49.5922 60.0867,49.1527 58.1706,51.9213 57.9147,50.2514 56.1454,52.6684 55.8259,51.6576 54.2908,52.8881 53.9301,50.6469 53.0147,51.3939 52.4827,48.0541 51.1792,49.2846 50.5413,47.1752 49.1529,43.9232 49.8095,42.5609 48.7778,36.9359 49.6675,35.266 48.7488,32.8051 49.5251,27.224 48.9801,26.1254 50.4014,21.2035 50.2049,20.017 51.5633,17.4682 53.0147,19.4457 54.0851,19.4457 55.8752,19.5775 57.4921,19.5775 58.6768,24.3676 61.4597),(24.4555 59.4226,21.2035 58.4936,21.3353 56.8969,21.599 55.9245,25.2025 55.9983,28.85 57.0645,27.0922 57.8447,28.8061 59.1758,26.2572 59.1758,24.4555 59.4226),(32.6773 57.7269,32.9811 57.1783,36.8295 59.693,35.9663 59.7703,32.6773 57.7269),(33.4587 56.7498,34.5347 56.6377,37.9197 58.8184,37.2875 58.7225,37.2249 59.0621,33.4587 56.7498),(36.5672 56.4248,41.1674 59.0793,40.9293 59.24,40.8796 59.2641,40.276 59.1737,35.8287 56.5023,36.5672 56.4248),(30.7975 55.0249,30.2091 54.633,30.2393 53.6773,31.6568 54.6786,30.7975 55.0249),(34.0091 53.173,34.4539 53.4687,33.6753 53.8276,32.7039 53.173,34.0091 53.173),(31.1809 52.1644,29.786 52.1465,30.6237 52.6993,30.3667 52.9437,29.393 52.2762,29.4184 55.6047,29.1576 55.7517,22.5658 55.1285,22.5658 53.5402,22.0385 51.4813,26.2572 51.4265,30.1244 50.5413,32.1898 51.1792,31.1809 52.1644),(31.3501 53.173,32.9056 54.17,32.4672 54.3493,30.7047 53.173,31.3501 53.173),(39.7202 55.9646,37.3607 54.3795,37.8015 53.8302,41.577 56.336,39.7202 55.9646),(41.2147 57.7149,42.2607 58.3313,41.5893 58.7925,38.9459 57.2058,39.0893 57.2552,41.2147 57.7149),(37.6798 55.5522,36.5844 55.329,36.8394 55.0196,37.6798 55.5522),(32.3127 58.3747,34.2648 59.6073,31.9701 58.9726,32.3127 58.3747),(35.9939 52.2136,34.9584 51.4813,36.5404 50.4014,39.6605 50.2892,39.7924 52.1334,41.7699 50.6807,44.4945 51.9712,47.307 52.5094,44.0551 53.5402,46.6039 53.6966,47.6146 55.404,45.3734 55.404,44.4214 55.8488,44.4145 55.3096,40.0924 52.1651,38.3394 52.1651,43.0242 55.3268,43.0242 56.2613,37.1607 52.2392,35.9939 52.2136))) -======= ->>>>>>> parent of d4258922d1... more mercator diff --git a/tests/queries/0_stateless/01306_polygons_intersection.reference b/tests/queries/0_stateless/01306_polygons_intersection.reference index ae23d9f0cda..43ee975913e 100644 --- a/tests/queries/0_stateless/01306_polygons_intersection.reference +++ b/tests/queries/0_stateless/01306_polygons_intersection.reference @@ -2,18 +2,9 @@ [] [] [[[(4.3666052904432435,50.84337386140151),(4.3602419,50.8435626),(4.349556,50.8535879),(4.3526804582393535,50.856658100365976),(4.367945,50.852455),(4.3666052904432435,50.84337386140151)]]] -<<<<<<< HEAD --------- -[] -[[[(4.366505261795747,50.843273415405),(4.3601419,50.8434626),(4.349456,50.8534879),(4.352580404040105,50.85655765067624),(4.367845,50.85235499999999),(4.366505261795747,50.843273415405)]]] -------- MultiPolygon with Polygon MULTIPOLYGON(((35.5408 58.9593,36.9725 59.0149,36.4989 58.7512,35.3712 58.8556,35.5408 58.9593)),((34.4816 56.8232,36.1999 57.0022,35.4083 56.5254,34.3867 56.7596,34.4816 56.8232)),((35.9179 57.7512,36.0848 57.855,37.1608 58.0478,36.5949 58.1673,37.8553 58.9075,38.5813 58.7446,37.4026 58.3187,38.0535 58.0542,36.4587 57.1544,35.7705 57.2554,37.0097 57.4998,35.9179 57.7512)),((36.8709 53.2765,37.4328 52.9552,36.5022 53.0008,36.8709 53.2765)),((36.1528 53.6763,35.3645 53.076,34.9611 53.9765,36.0472 54.7217,36.6985 54.0791,36.3552 53.8269,35.9216 53.8026,36.1528 53.6763)),((37.0035 54.2999,36.7074 54.6506,38.1601 55.1091,37.0035 54.2999)),((38.1688 56.0758,38.2186 56.0594,38.1319 56.0534,38.1688 56.0758)),((37.6238 55.7402,38.0373 55.6523,37.2824 55.5258,37.6238 55.7402)),((37.06 55.3843,37.7653 55.1891,36.151 54.791,37.06 55.3843)),((38.2312 56.9795,36.5334 56.6753,37.4489 57.1909,38.2699 57.0021,38.2312 56.9795)),((37.2281 56.3799,36.193 55.7319,35.3188 55.9582,35.6571 56.1619,36.7074 56.211,36.0233 56.3789,36.4446 56.6242,37.2281 56.3799)),((34.9952 58.6226,36.1498 58.553,36.0877 58.5174,34.6028 58.3749,34.9952 58.6226)),((34.3593 58.2189,35.4314 58.1349,35.1134 57.9454,33.7581 57.8255,34.3593 58.2189)),((33.6325 57.7419,34.6332 57.6538,34.2274 57.4023,33.1712 57.337,34.0208 57.2724,33.5602 56.9781,32.9596 56.9434,33.3418 56.8364,31.7782 55.7778,31.5088 55.9411,31.6069 56.3194,33.6325 57.7419)),((36.403 58.0507,36.4354 58.0478,36.3932 58.0447,36.403 58.0507)),((35.613 57.5595,36.1936 57.4998,35.4682 57.4674,35.613 57.5595)),((35.0338 57.1875,36.0727 57.0915,34.8098 57.0409,35.0338 57.1875)),((34.1885 56.6259,35.2273 56.414,35.0485 56.303,34.5917 56.2949,33.7222 56.3063,34.1885 56.6259)),((33.5244 56.1686,34.4996 55.9565,34.2598 55.8023,33.1204 55.8832,33.5244 56.1686)),((32.9547 55.7645,33.5036 55.3785,33.6125 55.3778,31.8748 54.1736,31.4182 54.4227,31.7439 54.8677,32.9547 55.7645)),((34.7279 53.8116,34.7731 53.7847,34.7731 52.9188,33.4048 52.8423,34.7279 53.8116)),((34.7231 54.7576,32.5275 53.1741,32.0831 53.408,32.476 53.8383,32.2523 53.964,34.3709 55.3709,35.0149 55.3613,34.2593 54.9642,34.7231 54.7576)),((34.9706 54.9262,34.8335 55.0162,35.2275 55.0993,34.9706 54.9262)),((35.7505 55.4454,35.1358 55.5327,35.9817 55.5958,35.7505 55.4454)),((35.0954 55.822,35.6798 55.6863,34.9721 55.7463,35.0954 55.822)),((34.7331 56.1049,34.7126 56.11,34.744 56.1118,34.7331 56.1049)),((40.2143 54.467,38.5511 53.2922,38.3395 53.2817,38.4609 53.226,38.0214 52.8989,37.8559 52.9188,37.135 53.4711,39.8151 55.3187,39.8205 55.2753,40.3948 55.2408,40.3948 54.8773,39.5485 54.8773,39.5485 54.5631,40.2143 54.467)),((40.5716 55.8007,40.5761 55.7884,40.5504 55.7875,40.5716 55.8007)),((40.4543 56.5923,40.2529 56.4682,39.7903 56.4121,39.8102 56.1914,38.2609 55.1775,37.7955 55.3956,38.4907 55.5327,38.1884 55.8564,38.944 56.0594,38.4339 56.2361,39.7863 57.025,39.7903 56.9929,40.3343 56.9599,40.4543 56.5923)),((40.1389 58.048,38.4915 57.1308,38.2186 57.2717,38.7325 57.4835,38.3737 57.6908,39.6392 58.3427,39.6392 58.0478,40.1389 58.048)),((37.5054 56.5484,37.463 56.5623,37.565 56.5843,37.5054 56.5484)),((38.0744 57.5312,38.128 57.516,37.9669 57.4734,38.0744 57.5312)),((40.4136 58.7241,40.3343 58.3821,39.7184 58.3823,40.4136 58.7241)),((39.8163 58.9766,39.4085 58.7696,38.5209 59.119,39.8163 58.9766)),((38.432 58.2584,38.3698 58.2869,38.7465 58.4255,38.432 58.2584)),((32.2175 58.3664,32.5691 58.5924,33.4734 58.8542,34.7428 59.5659,33.8361 58.6819,34.0496 58.6717,31.6514 57.1258,31.5088 57.4998,32.1738 58.0318,32.2175 58.3664)),((39.9942 53.358,40.0019 53.354,39.9877 53.3534,39.9942 53.358)),((39.2704 52.8471,39.5787 52.6996,39.1456 52.7573,39.2704 52.8471))) -MULTIPOLYGON(((35.677 58.9587,37.1527 59.0192,36.6691 58.7302,35.4917 58.8391,35.677 58.9587)),((34.8711 56.8603,36.7103 57.0477,35.7316 56.4418,34.6299 56.7008,34.8711 56.8603)),((36.1527 57.6965,36.5188 57.9318,37.1607 58.0477,36.8134 58.1201,38.0059 58.8722,38.5812 58.7445,37.4025 58.3186,38.2411 57.9759,38.0568 57.8654,37.4327 57.7102,37.6831 57.6403,36.7988 57.102,35.7704 57.2553,37.0096 57.4997,36.1527 57.6965)),((36.9376 53.2346,37.4327 52.9551,36.5898 52.9925,36.9376 53.2346)),((36.2567 53.616,35.4415 53.0432,35.3775 53.0461,35.0208 53.8426,36.1761 54.5942,36.7891 53.9859,36.575 53.8375,35.9215 53.8025,36.2567 53.616)),((37.0903 54.1938,36.7073 54.6505,38.3092 55.1545,37.7954 55.3955,38.4906 55.5326,38.1883 55.8563,38.9439 56.0593,38.6515 56.1591,39.9915 56.9806,40.3342 56.9598,40.4854 56.4956,39.7902 56.412,39.8204 56.0762,39.9194 56.0955,37.0903 54.1938)),((37.8883 55.6832,38.0372 55.6522,37.7656 55.6061,37.8883 55.6832)),((37.3054 55.3158,37.7652 55.189,36.6884 54.9232,37.3054 55.3158)),((37.4799 56.2952,36.9557 55.9643,36.6469 55.941,36.8521 55.8985,36.4727 55.6567,35.6467 55.87,36.1651 56.1834,36.7073 56.2109,36.3512 56.2953,36.7422 56.5294,37.4799 56.2952)),((37.1816 56.7907,37.7436 57.1223,38.3092 56.9928,37.1816 56.7907)),((35.1359 58.6084,36.3446 58.5401,34.7969 58.3872,35.1359 58.6084)),((34.5125 58.2005,35.6455 58.1105,35.3989 57.9596,33.9633 57.8372,34.5125 58.2005)),((33.7916 57.7229,34.8546 57.6243,34.5079 57.409,33.2199 57.3396,33.7916 57.7229)),((33.211 57.3336,34.2469 57.2461,33.8307 56.9848,32.9595 56.9433,33.5159 56.786,31.863 55.7247,31.5087 55.941,31.581 56.2183,33.211 57.3336)),((35.8918 57.5278,36.1935 57.4997,35.8173 57.4795,35.8918 57.5278)),((35.3199 57.1553,36.0726 57.0914,35.1561 57.0479,35.3199 57.1553)),((34.4368 56.5727,35.5695 56.3405,35.5188 56.3087,34.5916 56.2948,34.0285 56.3002,34.4368 56.5727)),((33.7487 56.1124,34.8294 55.8745,34.6546 55.7636,33.3749 55.86,33.7487 56.1124)),((33.0915 55.6676,33.5035 55.3784,34.0415 55.3722,32.0505 54.0742,31.4181 54.4226,31.5514 54.605,33.0915 55.6676)),((34.773 53.6795,34.773 52.9187,33.5229 52.8475,34.773 53.6795)),((34.9347 54.6613,34.374 54.2922,34.108 54.1756,34.1558 54.1477,32.6253 53.1196,32.083 53.4079,32.4759 53.8382,32.4267 53.8656,34.8176 55.3634,35.0148 55.3612,34.2592 54.9641,34.9347 54.6613)),((35.153 54.8041,34.8334 55.0161,35.777 55.2096,35.153 54.8041)),((36.0682 55.3974,35.1357 55.5326,36.4239 55.6256,36.0682 55.3974)),((35.4341 55.7407,35.6797 55.6862,35.3816 55.7088,35.4341 55.7407)),((35.0606 56.0207,34.7125 56.1099,35.245 56.1369,35.0606 56.0207)),((40.3151 54.4517,38.6455 53.2949,38.3394 53.2816,38.512 53.2011,38.0733 52.8911,37.8558 52.9187,37.2027 53.4183,39.9389 55.2681,40.3947 55.2407,40.3947 54.8772,39.5484 54.8772,39.5484 54.563,40.3151 54.4517)),((40.2513 57.9982,38.6704 57.0364,38.2185 57.2716,38.7324 57.4834,38.5465 57.5909,39.6391 58.219,39.6391 58.0477,40.2436 58.0477,40.2513 57.9982)),((37.7506 56.465,37.4629 56.5622,38.1298 56.7016,37.7506 56.465)),((40.3959 58.6476,40.3342 58.382,39.9258 58.382,40.3959 58.6476)),((39.9119 58.9623,39.5108 58.7281,38.5208 59.1189,39.9119 58.9623)),((38.591 58.1848,38.3697 58.2868,39.3854 58.6545,38.591 58.1848)),((32.2099 58.3086,32.7181 58.6343,33.4733 58.8541,34.7427 59.5658,33.836 58.6818,34.1607 58.6635,31.6694 57.0765,31.5087 57.4997,32.1737 58.0317,32.2099 58.3086)),((39.2985 52.8318,39.5786 52.6995,39.1804 52.7502,39.2985 52.8318))) -------- MultiPolygon with Polygon with Holes MULTIPOLYGON(((33.1079 56.9523,32.9596 56.9434,33.1392 56.8934,33.2007 56.7768,33.7182 56.7292,33.8361 56.6953,35.71 56.3117,34.5917 56.2949,32.8387 56.3117,35.6798 55.6863,32.748 55.9072,33.5036 55.3785,35.0149 55.3613,34.2593 54.9642,35.0753 54.5981,34.1081 54.1757,34.7731 53.7847,34.7731 53.3243,33.1128 54.0852,31.627 54.7093,31.8413 54.9989,32.204 55.5156,31.5088 55.9411,31.7506 56.8609,31.5088 57.4998,32.1738 58.0318,32.2342 58.4928,32.25 58.4976,33.1079 56.9523)),((35.1489 56.5859,36.6724 56.4139,36.8799 56.4895,38.2186 56.0594,36.647 55.9411,38.0262 55.6546,37.9482 55.6376,36.8283 55.4471,36.9508 55.414,36.5845 55.3291,36.8822 54.975,36.0123 54.7554,36.919 53.8561,35.9216 53.8026,37.2165 53.0798,37.0604 52.9744,35.3776 53.0462,34.894 54.1226,35.6193 54.4929,34.8335 55.0162,36.4354 55.3441,35.1358 55.5327,36.5563 55.6352,34.7126 56.11,36.7074 56.211,35.1489 56.5859)),((37.2327 59.0233,37.3119 59.0258,38.0944 58.8545,37.2876 58.7226,37.2327 59.0233)),((37.4471 53.2343,36.9794 53.5878,37.3119 53.9273,36.7074 54.6506,37.0572 54.7635,37.9907 53.5925,37.4471 53.2343)),((34.7731 53.1793,34.7731 52.9188,33.1712 52.8276,32.4808 53.1989,34.7731 53.1793)),((40.4412 56.1511,38.3184 55.7179,38.1884 55.8564,38.944 56.0594,37.463 56.5623,38.9742 56.8774,38.5798 57.0849,39.0894 57.2553,39.7379 57.4051,39.7903 56.9929,40.3343 56.9599,40.4855 56.4957,39.7903 56.4121,39.8205 56.0763,40.425 56.1942,40.4412 56.1511)),((38.3092 56.9929,38.3093 56.9929,38.309 56.9928,38.3092 56.9929)),((40.3237 57.5365,40.3343 57.4673,40.0149 57.4677,40.3237 57.5365)),((39.2792 59.0373,38.8838 58.9777,38.5209 59.119,39.2792 59.0373))) -MULTIPOLYGON(((33.106 56.9503,32.9595 56.9433,33.1372 56.8932,33.2006 56.7767,33.7508 56.7194,33.836 56.6952,35.7099 56.3116,34.5916 56.2948,32.8386 56.3116,35.6797 55.6862,32.7479 55.9071,33.5035 55.3784,35.0148 55.3612,34.2592 54.9641,35.0752 54.598,34.108 54.1756,34.773 53.7846,34.773 53.3207,33.1127 54.0851,31.6171 54.6946,31.8412 54.9988,32.2039 55.5155,31.5087 55.941,31.7505 56.8608,31.5087 57.4997,32.1737 58.0317,32.2341 58.4927,32.2439 58.4956,33.106 56.9503)),((35.1914 56.569,36.6723 56.4138,36.8783 56.4863,38.2185 56.0593,36.6469 55.941,38.0372 55.6522,36.8282 55.447,36.9706 55.4079,36.5844 55.329,36.8796 54.9705,36.0122 54.7553,36.9189 53.856,35.9215 53.8025,37.2279 53.0709,37.0823 52.9707,35.3775 53.0461,34.8939 54.1225,35.6192 54.4928,34.8334 55.0161,36.4353 55.344,35.1357 55.5326,36.5562 55.6351,34.7125 56.1099,36.7073 56.2109,35.1914 56.569)),((37.2322 59.0224,37.3118 59.0257,38.1151 58.848,37.2875 58.7225,37.2322 59.0224)),((37.4544 53.2264,36.9793 53.5877,37.3118 53.9272,36.7073 54.6505,37.0521 54.7595,37.9906 53.5924,37.4544 53.2264)),((34.773 53.173,34.773 52.9187,33.1711 52.8275,32.5252 53.173,34.773 53.173)),((40.4556 56.1121,38.3463 55.6874,38.1883 55.8563,38.9439 56.0593,37.4629 56.5622,38.9741 56.8773,38.5847 57.0811,39.0893 57.2552,39.7388 57.3963,39.7902 56.9928,40.3342 56.9598,40.4854 56.4956,39.7902 56.412,39.8204 56.0762,40.4249 56.1941,40.4556 56.1121)),((40.3255 57.5233,40.3342 57.4672,40.066 57.4672,40.3255 57.5233)),((39.3164 59.0294,38.9056 58.9675,38.5208 59.1189,39.3164 59.0294))) -------- Polygon with Polygon with Holes MULTIPOLYGON(((32.6512 57.792,30.3301 56.1942,30.2394 55.2753,32.9378 57.2699,33.2007 56.7768,33.2446 56.7729,30.7705 55.0525,29.5972 55.5037,29.4171 55.606,29.4536 59.7796,30.5719 59.9919,30.4812 58.8542,32.3249 59.9465,33.6548 59.9465,30.179 57.9196,30.179 56.9764,32.2964 58.4175,32.6512 57.792)),((35.9475 59.7758,35.1343 59.8448,34.2247 59.6064,34.8637 59.9768,36.2843 59.9616,35.9475 59.7758)),((36.7912 59.6986,37.2817 59.9768,38.7325 59.9465,37.2102 59.1452,37.1118 59.6677,36.7912 59.6986)),((34.2635 56.6767,35.4536 56.5531,32.2591 54.4483,31.5682 54.7333,34.2635 56.6767)),((36.1815 56.4715,36.6724 56.4139,38.1759 56.9472,33.5144 53.9057,33.1128 54.0852,32.6907 54.2663,36.1815 56.4715)),((33.8733 53.1922,35.0903 53.1731,34.3351 53.53,36.7219 55.1665,37.2766 54.4948,34.2895 52.2208,32.5969 52.2208,33.8733 53.1922)),((31.1968 52.1649,30.5785 52.7531,31.1682 53.1903,32.5603 53.1989,31.2368 52.1652,31.1968 52.1649)),((30.3098 53.0028,30.1245 53.1731,30.5408 53.1811,30.3098 53.0028)),((37.6322 58.7797,39.7299 59.9314,44.4751 59.81,44.4212 55.8594,42.8247 56.5837,41.4519 56.3413,43.0243 57.2554,43.0243 58.0797,39.4328 55.9511,37.1934 55.4694,40.9691 57.677,42.6929 58.0314,42.2498 58.3455,42.5105 58.477,41.6944 58.8542,41.5887 58.8012,41.168 59.0834,41.2108 59.1035,40.9299 59.2404,40.8911 59.2659,40.8804 59.2644,40.6366 59.3817,40.2079 59.1718,37.6322 58.7797)),((35.9681 52.2157,35.4682 52.2022,37.7431 53.9104,37.9907 53.5925,35.9681 52.2157))) -MULTIPOLYGON(((32.6773 57.7269,30.33 56.1941,30.2393 55.2752,32.9811 57.1783,33.2006 56.7767,33.4587 56.7498,30.7975 55.0249,29.5971 55.5036,29.4184 55.6047,29.4535 59.7795,30.5718 59.9918,30.4811 58.8541,32.3248 59.9464,33.6547 59.9464,30.1789 57.9195,30.1789 56.9763,32.3127 58.3747,32.6773 57.7269)),((35.9663 59.7703,35.1342 59.8447,34.2648 59.6073,34.8636 59.9767,36.2842 59.9615,35.9663 59.7703)),((36.8295 59.693,37.2816 59.9767,38.7324 59.9464,37.2249 59.0621,37.1117 59.6676,36.8295 59.693)),((34.5347 56.6377,35.8287 56.5023,32.4672 54.3493,31.6568 54.6786,34.5347 56.6377)),((36.5672 56.4248,36.6723 56.4138,38.9459 57.2058,33.6753 53.8276,33.1127 54.0851,32.9056 54.17,36.5672 56.4248)),((34.0091 53.173,35.0902 53.173,34.4539 53.4687,36.8394 55.0196,37.3607 54.3795,34.2894 52.2207,32.5968 52.2207,34.0091 53.173)),((31.1809 52.1644,30.6237 52.6993,31.3501 53.173,32.7039 53.173,31.2367 52.1651,31.1809 52.1644)),((30.3667 52.9437,30.1244 53.173,30.7047 53.173,30.3667 52.9437)),((37.9197 58.8184,39.7298 59.9313,44.475 59.8099,44.4214 55.8488,42.8246 56.5836,41.577 56.336,43.0242 57.2553,43.0242 58.0796,39.7202 55.9646,37.6798 55.5522,41.2147 57.7149,42.6928 58.0313,42.2607 58.3313,42.5104 58.4769,41.6943 58.8541,41.5893 58.7925,41.1674 59.0793,41.2107 59.1034,40.9293 59.24,40.891 59.2658,40.8796 59.2641,40.6365 59.3816,40.276 59.1737,37.9197 58.8184)),((35.9939 52.2136,35.4681 52.2021,37.8015 53.8302,37.9906 53.5924,35.9939 52.2136))) -======= ->>>>>>> parent of d4258922d1... more mercator From 35d0b3d3ec4bd61b76eafd5d2a2f7807895a7aff Mon Sep 17 00:00:00 2001 From: Nikita Mikhaylov Date: Sat, 6 Mar 2021 02:43:26 +0300 Subject: [PATCH 317/544] better --- .../0_stateless/01304_polygons_sym_difference.reference | 3 --- tests/queries/0_stateless/01304_polygons_sym_difference.sql | 3 --- 2 files changed, 6 deletions(-) diff --git a/tests/queries/0_stateless/01304_polygons_sym_difference.reference b/tests/queries/0_stateless/01304_polygons_sym_difference.reference index aff588052c1..7d16848ac2e 100644 --- a/tests/queries/0_stateless/01304_polygons_sym_difference.reference +++ b/tests/queries/0_stateless/01304_polygons_sym_difference.reference @@ -1,10 +1,7 @@ [[[(1,2.9),(1,1),(2.9,1),(3,0),(0,0),(0,3),(1,2.9)]],[[(1,2.9),(1,4),(4,4),(4,1),(2.9,1),(2.6,2),(2,2.6),(1,2.9)]]] -------- MultiPolygon with Polygon MULTIPOLYGON(((36.9725 59.0149,35.5408 58.9593,37.2817 59.9768,38.7325 59.9465,36.9725 59.0149)),((36.9725 59.0149,37.3119 59.0258,37.8553 58.9075,36.5949 58.1673,36.0123 58.2869,37.191 58.6819,36.4989 58.7512,36.9725 59.0149)),((36.151 54.791,37.7653 55.1891,37.06 55.3843,37.2824 55.5258,38.0373 55.6523,37.6238 55.7402,38.1319 56.0534,38.2186 56.0594,38.1688 56.0758,38.4339 56.2361,38.944 56.0594,38.1884 55.8564,38.4907 55.5327,37.7955 55.3956,38.2609 55.1775,38.1601 55.1091,36.7074 54.6506,37.0035 54.2999,36.6985 54.0791,36.0472 54.7217,36.151 54.791)),((36.151 54.791,36.0123 54.7554,36.0472 54.7217,34.9611 53.9765,34.894 54.1226,35.6193 54.4929,34.9706 54.9262,35.2275 55.0993,36.4354 55.3441,35.7505 55.4454,35.9817 55.5958,36.5563 55.6352,36.193 55.7319,37.2281 56.3799,38.1688 56.0758,38.1319 56.0534,36.647 55.9411,37.6238 55.7402,37.2824 55.5258,36.8283 55.4471,37.06 55.3843,36.151 54.791)),((36.5334 56.6753,38.2312 56.9795,37.565 56.5843,37.463 56.5623,37.5054 56.5484,37.2281 56.3799,36.4446 56.6242,36.5334 56.6753)),((36.5334 56.6753,36.375 56.6455,36.4446 56.6242,36.0233 56.3789,35.4083 56.5254,36.1999 57.0022,36.9794 57.0751,36.4587 57.1544,38.0535 58.0542,38.3395 57.9356,37.4328 57.7103,38.0744 57.5312,37.9669 57.4734,37.1608 57.2554,37.4489 57.1909,36.5334 56.6753)),((36.8709 53.2765,37.135 53.4711,37.8559 52.9188,38.0214 52.8989,37.1608 52.2393,35.4682 52.2022,36.5022 53.0008,37.4328 52.9552,36.8709 53.2765)),((36.8709 53.2765,36.5022 53.0008,35.3776 53.0462,35.3645 53.076,36.1528 53.6763,36.8709 53.2765)),((36.6985 54.0791,36.919 53.8561,36.3552 53.8269,36.6985 54.0791)),((35.5408 58.9593,35.3712 58.8556,34.6522 58.9167,35.5408 58.9593)),((36.0848 57.855,36.3932 58.0447,36.4354 58.0478,36.403 58.0507,36.5949 58.1673,37.1608 58.0478,36.0848 57.855)),((36.0848 57.855,35.9179 57.7512,35.7402 57.7909,36.0848 57.855)),((37.135 53.4711,36.9794 53.5878,37.3119 53.9273,37.0035 54.2999,38.1601 55.1091,38.3093 55.1546,38.2609 55.1775,39.8102 56.1914,39.8205 56.0763,40.425 56.1942,40.5716 55.8007,40.5504 55.7875,39.7601 55.7544,39.8151 55.3187,37.135 53.4711)),((38.2312 56.9795,38.2699 57.0021,38.3093 56.9929,38.2312 56.9795)),((36.4989 58.7512,36.1498 58.553,34.9952 58.6226,35.3712 58.8556,36.4989 58.7512)),((36.4587 57.1544,36.1999 57.0022,34.4816 56.8232,34.8098 57.0409,36.0727 57.0915,35.0338 57.1875,35.4682 57.4674,36.1936 57.4998,35.613 57.5595,35.9179 57.7512,37.0097 57.4998,35.7705 57.2554,36.4587 57.1544)),((38.0535 58.0542,37.4026 58.3187,38.5813 58.7446,37.8553 58.9075,39.7299 59.9314,44.4751 59.81,44.4146 55.3097,40.0925 52.1652,38.3395 52.1652,39.1456 52.7573,39.5787 52.6996,39.2704 52.8471,39.9877 53.3534,40.0019 53.354,39.9942 53.358,43.0243 55.3269,43.0243 56.2614,40.2143 54.467,39.5485 54.5631,39.5485 54.8773,40.3948 54.8773,40.3948 55.2408,39.8205 55.2753,39.8151 55.3187,40.5504 55.7875,40.5761 55.7884,40.5716 55.8007,43.0243 57.2554,43.0243 58.0797,40.4543 56.5923,40.3343 56.9599,39.7903 56.9929,39.7863 57.025,42.5105 58.477,41.6944 58.8542,40.1389 58.048,39.6392 58.0478,39.6392 58.3427,39.7184 58.3823,40.3343 58.3821,40.4136 58.7241,41.2108 59.1035,40.6366 59.3817,39.8163 58.9766,38.5209 59.119,39.4085 58.7696,38.7465 58.4255,38.3698 58.2869,38.432 58.2584,38.0535 58.0542)),((34.4996 55.9565,33.5244 56.1686,33.7222 56.3063,34.5917 56.2949,35.0485 56.303,34.744 56.1118,34.7126 56.11,34.7331 56.1049,34.4996 55.9565)),((34.4996 55.9565,35.0954 55.822,34.9721 55.7463,34.2598 55.8023,34.4996 55.9565)),((31.6069 56.3194,31.5088 55.9411,31.7782 55.7778,30.2092 54.6331,30.2394 53.6774,31.7439 54.8677,31.4182 54.4227,31.8748 54.1736,29.3931 52.2763,29.4536 59.7796,30.5719 59.9919,30.4812 58.8542,32.3249 59.9465,33.6548 59.9465,30.179 57.9196,30.179 56.9764,32.2175 58.3664,32.1738 58.0318,31.5088 57.4998,31.6514 57.1258,30.3301 56.1942,30.2394 55.2753,31.6069 56.3194)),((31.6069 56.3194,31.7506 56.8609,31.6514 57.1258,34.0496 58.6717,34.9952 58.6226,34.6028 58.3749,33.6245 58.271,34.3593 58.2189,33.7581 57.8255,33.2316 57.7748,33.6325 57.7419,31.6069 56.3194)),((33.5244 56.1686,33.1204 55.8832,32.748 55.9072,32.9547 55.7645,31.7439 54.8677,31.8413 54.9989,32.204 55.5156,31.7782 55.7778,33.3418 56.8364,33.8361 56.6953,34.1885 56.6259,33.7222 56.3063,32.8387 56.3117,33.5244 56.1686)),((33.1204 55.8832,34.2598 55.8023,33.6125 55.3778,33.5036 55.3785,32.9547 55.7645,33.1204 55.8832)),((35.3188 55.9582,36.193 55.7319,35.9817 55.5958,35.1358 55.5327,35.7505 55.4454,35.2275 55.0993,34.8335 55.0162,34.9706 54.9262,34.7231 54.7576,34.2593 54.9642,35.0149 55.3613,34.3709 55.3709,34.9721 55.7463,35.6798 55.6863,35.0954 55.822,35.3188 55.9582)),((35.3188 55.9582,34.7331 56.1049,34.744 56.1118,35.6571 56.1619,35.3188 55.9582)),((33.3418 56.8364,32.9596 56.9434,33.5602 56.9781,33.3418 56.8364)),((33.4048 52.8423,34.7731 52.9188,34.7731 53.7847,34.7279 53.8116,34.9611 53.9765,35.3645 53.076,34.2895 52.2208,32.5969 52.2208,33.4048 52.8423)),((33.4048 52.8423,33.1712 52.8276,32.5275 53.1741,34.7231 54.7576,35.0753 54.5981,34.1081 54.1757,34.7279 53.8116,33.4048 52.8423)),((32.2523 53.964,32.476 53.8383,32.0831 53.408,32.5275 53.1741,31.2368 52.1652,29.7861 52.1466,32.2523 53.964)),((32.2523 53.964,31.8748 54.1736,33.6125 55.3778,34.3709 55.3709,32.2523 53.964)),((36.3552 53.8269,36.1528 53.6763,35.9216 53.8026,36.3552 53.8269)),((32.5691 58.5924,34.8637 59.9768,36.2843 59.9616,34.0496 58.6717,33.8361 58.6819,34.7428 59.5659,33.4734 58.8542,32.5691 58.5924)),((32.5691 58.5924,32.2175 58.3664,32.2342 58.4928,32.5691 58.5924)),((33.5602 56.9781,34.0208 57.2724,35.0338 57.1875,34.8098 57.0409,33.5602 56.9781)),((36.3932 58.0447,35.1134 57.9454,35.4314 58.1349,36.403 58.0507,36.3932 58.0447)),((35.1134 57.9454,34.6332 57.6538,33.6325 57.7419,33.7581 57.8255,35.1134 57.9454)),((35.4314 58.1349,34.3593 58.2189,34.6028 58.3749,36.0877 58.5174,35.4314 58.1349)),((35.4682 57.4674,34.2274 57.4023,34.6332 57.6538,35.613 57.5595,35.4682 57.4674)),((34.4816 56.8232,34.3867 56.7596,34.229 56.7948,34.4816 56.8232)),((34.1885 56.6259,34.3867 56.7596,35.4083 56.5254,35.2273 56.414,34.1885 56.6259)),((34.2274 57.4023,34.0208 57.2724,33.1712 57.337,34.2274 57.4023)),((35.0485 56.303,35.2273 56.414,35.71 56.3117,35.0485 56.303)),((35.6571 56.1619,36.0233 56.3789,36.7074 56.211,35.6571 56.1619)),((36.1498 58.553,36.3447 58.5402,36.0877 58.5174,36.1498 58.553)),((40.2143 54.467,40.3948 54.4403,40.6064 54.034,39.9716 53.9807,40.2437 53.5878,39.5485 53.5878,39.9942 53.358,39.9877 53.3534,38.5511 53.2922,40.2143 54.467)),((39.8102 56.1914,39.7903 56.4121,40.2529 56.4682,39.8102 56.1914)),((38.0214 52.8989,38.4609 53.226,39.2704 52.8471,39.1456 52.7573,38.0214 52.8989)),((38.5511 53.2922,38.4609 53.226,38.3395 53.2817,38.5511 53.2922)),((40.4543 56.5923,40.4855 56.4957,40.2529 56.4682,40.4543 56.5923)),((40.1389 58.048,40.2437 58.0478,40.3343 57.4673,39.7299 57.4673,39.7863 57.025,38.4339 56.2361,37.5054 56.5484,37.565 56.5843,38.9742 56.8774,38.4915 57.1308,40.1389 58.048)),((40.4136 58.7241,39.7184 58.3823,39.6392 58.3821,39.6392 58.3427,38.3737 57.6908,38.3395 57.7103,38.8533 58.0638,38.432 58.2584,38.7465 58.4255,39.5485 58.7133,39.4085 58.7696,39.8163 58.9766,40.4552 58.9011,40.4136 58.7241)),((38.3737 57.6908,38.7325 57.4835,38.2186 57.2717,38.4915 57.1308,38.2699 57.0021,37.4489 57.1909,37.9669 57.4734,38.128 57.516,38.0744 57.5312,38.3737 57.6908))) -MULTIPOLYGON(((37.1527 59.0192,35.677 58.9587,37.2816 59.9767,38.7324 59.9464,37.1527 59.0192)),((37.1527 59.0192,37.3118 59.0257,38.0059 58.8722,36.8134 58.1201,36.0122 58.2868,37.1909 58.6818,36.6691 58.7302,37.1527 59.0192)),((36.6884 54.9232,37.7652 55.189,37.3054 55.3158,37.7656 55.6061,38.0372 55.6522,37.8883 55.6832,38.6515 56.1591,38.9439 56.0593,38.1883 55.8563,38.4906 55.5326,37.7954 55.3955,38.3092 55.1545,36.7073 54.6505,37.0903 54.1938,36.7891 53.9859,36.1761 54.5942,36.6884 54.9232)),((36.6884 54.9232,36.0122 54.7553,36.1761 54.5942,35.0208 53.8426,34.8939 54.1225,35.6192 54.4928,35.153 54.8041,35.777 55.2096,36.4353 55.344,36.0682 55.3974,36.4239 55.6256,36.5562 55.6351,36.4727 55.6567,36.8521 55.8985,37.8883 55.6832,37.7656 55.6061,36.8282 55.447,37.3054 55.3158,36.6884 54.9232)),((37.1816 56.7907,38.3092 56.9928,37.7436 57.1223,38.5465 57.5909,38.7324 57.4834,38.2185 57.2716,38.6704 57.0364,38.1298 56.7016,37.4629 56.5622,37.7506 56.465,37.4799 56.2952,36.7422 56.5294,37.1816 56.7907)),((37.1816 56.7907,36.3749 56.6454,36.7422 56.5294,36.3512 56.2953,35.7316 56.4418,36.7103 57.0477,36.9793 57.075,36.7988 57.102,37.6831 57.6403,38.1279 57.5159,37.1607 57.2553,37.7436 57.1223,37.1816 56.7907)),((36.7891 53.9859,36.9189 53.856,36.575 53.8375,36.7891 53.9859)),((36.8521 55.8985,36.6469 55.941,36.9557 55.9643,36.8521 55.8985)),((35.677 58.9587,35.4917 58.8391,34.6521 58.9166,35.677 58.9587)),((36.5188 57.9318,36.8134 58.1201,37.1607 58.0477,36.5188 57.9318)),((36.5188 57.9318,36.1527 57.6965,35.7401 57.7908,36.5188 57.9318)),((37.0903 54.1938,39.9194 56.0955,40.4249 56.1941,40.576 55.7883,39.76 55.7543,39.8204 55.2752,39.9389 55.2681,37.2027 53.4183,36.9793 53.5877,37.3118 53.9272,37.0903 54.1938)),((36.9557 55.9643,37.4799 56.2952,38.2185 56.0593,36.9557 55.9643)),((36.6691 58.7302,35.6455 58.1105,34.5125 58.2005,34.7969 58.3872,36.3446 58.5401,35.1359 58.6084,35.4917 58.8391,36.6691 58.7302)),((36.7988 57.102,36.7103 57.0477,34.8711 56.8603,35.1561 57.0479,36.0726 57.0914,35.3199 57.1553,35.8173 57.4795,36.1935 57.4997,35.8918 57.5278,36.1527 57.6965,37.0096 57.4997,35.7704 57.2553,36.7988 57.102)),((36.5898 52.9925,37.4327 52.9551,36.9376 53.2346,37.2027 53.4183,37.8558 52.9187,38.0733 52.8911,37.1607 52.2392,35.4681 52.2021,36.5898 52.9925)),((36.5898 52.9925,35.4415 53.0432,36.2567 53.616,36.9376 53.2346,36.5898 52.9925)),((37.6831 57.6403,37.4327 57.7102,38.0568 57.8654,37.6831 57.6403)),((35.245 56.1369,34.7125 56.1099,35.0606 56.0207,34.8294 55.8745,33.7487 56.1124,34.0285 56.3002,34.5916 56.2948,35.5188 56.3087,35.245 56.1369)),((35.245 56.1369,36.1651 56.1834,35.6467 55.87,35.0606 56.0207,35.245 56.1369)),((34.0285 56.3002,32.8386 56.3116,33.7487 56.1124,33.3749 55.86,32.7479 55.9071,33.0915 55.6676,31.5514 54.605,31.8412 54.9988,32.2039 55.5155,31.863 55.7247,33.5159 56.786,33.836 56.6952,34.4368 56.5727,34.0285 56.3002)),((36.3512 56.2953,36.7073 56.2109,36.1651 56.1834,36.3512 56.2953)),((31.581 56.2183,31.5087 55.941,31.863 55.7247,30.2091 54.633,30.2393 53.6773,31.5514 54.605,31.4181 54.4226,32.0505 54.0742,29.393 52.2762,29.4535 59.7795,30.5718 59.9918,30.4811 58.8541,32.3248 59.9464,33.6547 59.9464,30.1789 57.9195,30.1789 56.9763,32.2099 58.3086,32.1737 58.0317,31.5087 57.4997,31.6694 57.0765,30.33 56.1941,30.2393 55.2752,31.581 56.2183)),((31.581 56.2183,31.7505 56.8608,31.6694 57.0765,34.1607 58.6635,35.1359 58.6084,34.7969 58.3872,33.6244 58.2709,34.5125 58.2005,33.9633 57.8372,33.2315 57.7747,33.7916 57.7229,33.2199 57.3396,33.1711 57.3369,33.211 57.3336,31.581 56.2183)),((33.5159 56.786,32.9595 56.9433,33.8307 56.9848,33.5159 56.786)),((33.0915 55.6676,33.3749 55.86,34.6546 55.7636,34.0415 55.3722,33.5035 55.3784,33.0915 55.6676)),((33.5229 52.8475,34.773 52.9187,34.773 53.6795,35.0208 53.8426,35.3775 53.0461,35.4415 53.0432,34.2894 52.2207,32.5968 52.2207,33.5229 52.8475)),((33.5229 52.8475,33.1711 52.8275,32.6253 53.1196,34.1558 54.1477,34.773 53.7846,34.773 53.6795,33.5229 52.8475)),((32.4267 53.8656,32.4759 53.8382,32.083 53.4079,32.6253 53.1196,31.2367 52.1651,29.786 52.1465,32.4267 53.8656)),((32.4267 53.8656,32.0505 54.0742,34.0415 55.3722,34.8176 55.3634,32.4267 53.8656)),((34.1558 54.1477,34.108 54.1756,34.374 54.2922,34.1558 54.1477)),((34.374 54.2922,34.9347 54.6613,35.0752 54.598,34.374 54.2922)),((36.575 53.8375,36.2567 53.616,35.9215 53.8025,36.575 53.8375)),((34.8294 55.8745,35.4341 55.7407,35.3816 55.7088,34.6546 55.7636,34.8294 55.8745)),((34.9347 54.6613,34.2592 54.9641,35.0148 55.3612,34.8176 55.3634,35.3816 55.7088,35.6797 55.6862,35.4341 55.7407,35.6467 55.87,36.4727 55.6567,36.4239 55.6256,35.1357 55.5326,36.0682 55.3974,35.777 55.2096,34.8334 55.0161,35.153 54.8041,34.9347 54.6613)),((32.7181 58.6343,34.8636 59.9767,36.2842 59.9615,34.1607 58.6635,33.836 58.6818,34.7427 59.5658,33.4733 58.8541,32.7181 58.6343)),((32.7181 58.6343,32.2099 58.3086,32.2341 58.4927,32.7181 58.6343)),((33.8307 56.9848,34.2469 57.2461,35.3199 57.1553,35.1561 57.0479,33.8307 56.9848)),((35.6455 58.1105,36.4353 58.0477,35.3989 57.9596,35.6455 58.1105)),((33.9633 57.8372,35.3989 57.9596,34.8546 57.6243,33.7916 57.7229,33.9633 57.8372)),((34.8711 56.8603,34.6299 56.7008,34.2289 56.7947,34.8711 56.8603)),((34.8546 57.6243,35.8918 57.5278,35.8173 57.4795,34.5079 57.409,34.8546 57.6243)),((34.4368 56.5727,34.6299 56.7008,35.7316 56.4418,35.5695 56.3405,34.4368 56.5727)),((35.5188 56.3087,35.5695 56.3405,35.7099 56.3116,35.5188 56.3087)),((34.5079 57.409,34.2469 57.2461,33.211 57.3336,33.2199 57.3396,34.5079 57.409)),((39.9194 56.0955,39.8204 56.0762,39.7902 56.412,40.4854 56.4956,40.3342 56.9598,39.9915 56.9806,42.5104 58.4769,41.6943 58.8541,40.2513 57.9982,40.2436 58.0477,39.6391 58.0477,39.6391 58.219,39.9258 58.382,40.3342 58.382,40.3959 58.6476,41.2107 59.1034,40.6365 59.3816,39.9119 58.9623,38.5208 59.1189,39.5108 58.7281,39.3854 58.6545,38.3697 58.2868,38.591 58.1848,38.2411 57.9759,37.4025 58.3186,38.5812 58.7445,38.0059 58.8722,39.7298 59.9313,44.475 59.8099,44.4145 55.3096,40.0924 52.1651,38.3394 52.1651,39.1804 52.7502,39.5786 52.6995,39.2985 52.8318,43.0242 55.3268,43.0242 56.2613,40.3151 54.4517,39.5484 54.563,39.5484 54.8772,40.3947 54.8772,40.3947 55.2407,39.9389 55.2681,43.0242 57.2553,43.0242 58.0796,39.9194 56.0955)),((38.6515 56.1591,37.7506 56.465,38.1298 56.7016,38.9741 56.8773,38.6704 57.0364,40.2513 57.9982,40.3342 57.4672,39.7298 57.4672,39.7902 56.9928,39.9915 56.9806,38.6515 56.1591)),((40.3151 54.4517,40.3947 54.4402,40.6063 54.0339,39.9715 53.9806,40.2436 53.5877,39.5484 53.5877,40.0018 53.3539,38.6455 53.2949,40.3151 54.4517)),((38.0733 52.8911,38.512 53.2011,39.2985 52.8318,39.1804 52.7502,38.0733 52.8911)),((38.6455 53.2949,38.512 53.2011,38.3394 53.2816,38.6455 53.2949)),((40.3959 58.6476,39.9258 58.382,39.6391 58.382,39.6391 58.219,38.5465 57.5909,38.3394 57.7102,38.8532 58.0637,38.591 58.1848,39.3854 58.6545,39.5484 58.7132,39.5108 58.7281,39.9119 58.9623,40.4551 58.901,40.3959 58.6476)),((38.2411 57.9759,38.3394 57.9355,38.0568 57.8654,38.2411 57.9759))) -------- MultiPolygon with Polygon with Holes MULTIPOLYGON(((24.3677 61.4598,26.6528 61.1008,26.8726 61.7107,30.564 61.0583,31.3989 62.0215,36.0132 61.1432,36.8921 62.0009,42.6489 60.6301,43.5718 61.3757,47.0435 59.8889,49.5923 60.0868,49.1528 58.1707,51.9214 57.9148,50.2515 56.1455,52.6685 55.826,51.6577 54.2909,52.8882 53.9302,50.647 53.0148,51.394 52.4828,48.0542 51.1793,49.2847 50.5414,47.1753 49.153,43.9233 49.8096,42.561 48.7779,36.936 49.6676,35.2661 48.7489,32.8052 49.5252,27.2241 48.9802,26.1255 50.4015,21.2036 50.205,20.0171 51.5634,17.4683 53.0148,19.4458 54.0852,19.4458 55.8753,19.5776 57.4922,19.5776 58.6769,24.3677 61.4598),(24.4556 59.4227,21.2036 58.4937,21.3354 56.897,21.5991 55.9246,25.2026 55.9984,28.8501 57.0646,27.0923 57.8448,28.8062 59.1759,26.2573 59.1759,24.4556 59.4227),(35.1489 56.5859,36.7074 56.211,34.7126 56.11,36.5563 55.6352,35.1358 55.5327,36.4354 55.3441,34.8335 55.0162,35.6193 54.4929,34.894 54.1226,35.3776 53.0462,37.0604 52.9744,34.9585 51.4814,36.5405 50.4015,39.6606 50.2893,39.7925 52.1335,41.77 50.6808,44.4946 51.9713,47.3071 52.5095,44.0552 53.5403,46.604 53.6967,47.6147 55.4041,45.3735 55.4041,42.8247 56.5837,40.4412 56.1511,40.425 56.1942,39.8205 56.0763,39.7903 56.4121,40.4855 56.4957,40.3343 56.9599,39.7903 56.9929,39.7379 57.4051,40.0149 57.4677,40.3343 57.4673,40.3237 57.5365,42.6929 58.0314,40.8911 59.2659,39.2792 59.0373,38.5209 59.119,38.8838 58.9777,38.0944 58.8545,37.3119 59.0258,37.2327 59.0233,37.1118 59.6677,35.1343 59.8448,31.9702 58.9727,32.25 58.4976,32.2342 58.4928,32.1738 58.0318,31.5088 57.4998,31.7506 56.8609,31.5088 55.9411,32.204 55.5156,31.8413 54.9989,31.627 54.7093,29.5972 55.5037,29.1577 55.7518,22.5659 55.1286,22.5659 53.5403,22.0386 51.4814,26.2573 51.4266,30.1245 50.5414,32.1899 51.1793,30.1245 53.1731,32.4808 53.1989,33.1712 52.8276,34.7731 52.9188,34.7731 53.1793,35.0903 53.1731,34.7731 53.3243,34.7731 53.7847,34.1081 54.1757,35.0753 54.5981,34.2593 54.9642,35.0149 55.3613,33.5036 55.3785,32.748 55.9072,35.6798 55.6863,32.8387 56.3117,34.5917 56.2949,35.71 56.3117,33.8361 56.6953,33.7182 56.7292,35.1489 56.5859)),((35.1489 56.5859,34.229 56.7948,36.9794 57.0751,35.7705 57.2554,37.0097 57.4998,35.7402 57.7909,37.1608 58.0478,36.0123 58.2869,37.191 58.6819,34.6522 58.9167,37.2327 59.0233,37.2876 58.7226,38.0944 58.8545,38.5813 58.7446,37.4026 58.3187,38.3395 57.9356,37.4328 57.7103,38.128 57.516,37.1608 57.2554,38.3092 56.9929,38.309 56.9928,36.375 56.6455,36.8799 56.4895,36.6724 56.4139,35.1489 56.5859)),((33.1079 56.9523,32.25 58.4976,33.4734 58.8542,34.7428 59.5659,33.8361 58.6819,36.3447 58.5402,33.6245 58.271,36.4354 58.0478,33.2316 57.7748,36.1936 57.4998,33.1712 57.337,36.0727 57.0915,33.1079 56.9523)),((33.1079 56.9523,33.1392 56.8934,32.9596 56.9434,33.1079 56.9523)),((33.7182 56.7292,33.2007 56.7768,33.1392 56.8934,33.7182 56.7292)),((37.0604 52.9744,37.2165 53.0798,37.4328 52.9552,37.0604 52.9744)),((34.7731 53.3243,34.7731 53.1793,32.4808 53.1989,32.0831 53.408,32.476 53.8383,31.4182 54.4227,31.627 54.7093,33.1128 54.0852,34.7731 53.3243)),((36.9508 55.414,37.7653 55.1891,36.8822 54.975,36.5845 55.3291,36.9508 55.414)),((36.9508 55.414,36.8283 55.4471,37.9482 55.6376,36.9508 55.414)),((37.2165 53.0798,35.9216 53.8026,36.919 53.8561,36.0123 54.7554,36.8822 54.975,37.0572 54.7635,36.7074 54.6506,37.3119 53.9273,36.9794 53.5878,37.4471 53.2343,37.2165 53.0798)),((37.0572 54.7635,38.3093 55.1546,37.7955 55.3956,38.4907 55.5327,38.3184 55.7179,40.4412 56.1511,40.5761 55.7884,39.7601 55.7544,39.8205 55.2753,40.3948 55.2408,40.3948 54.8773,39.5485 54.8773,39.5485 54.5631,40.3948 54.4403,40.6064 54.034,39.9716 53.9807,40.2437 53.5878,39.5485 53.5878,40.0019 53.354,38.3395 53.2817,39.5787 52.6996,37.8559 52.9188,37.4471 53.2343,37.9907 53.5925,37.0572 54.7635)),((38.5798 57.0849,38.2186 57.2717,38.7325 57.4835,38.3395 57.7103,38.8533 58.0638,38.3698 58.2869,39.5485 58.7133,38.8838 58.9777,39.2792 59.0373,40.4552 58.9011,40.3343 58.3821,39.6392 58.3821,39.6392 58.0478,40.2437 58.0478,40.3237 57.5365,40.0149 57.4677,39.7299 57.4673,39.7379 57.4051,39.0894 57.2553,38.5798 57.0849)),((38.5798 57.0849,38.9742 56.8774,37.463 56.5623,38.944 56.0594,38.1884 55.8564,38.3184 55.7179,38.0262 55.6546,36.647 55.9411,38.2186 56.0594,36.8799 56.4895,38.309 56.9928,38.3093 56.9929,38.3092 56.9929,38.5798 57.0849)),((37.9482 55.6376,38.0262 55.6546,38.0373 55.6523,37.9482 55.6376))) -MULTIPOLYGON(((24.3676 61.4597,26.6527 61.1007,26.8725 61.7106,30.5639 61.0582,31.3988 62.0214,36.0131 61.1431,36.892 62.0008,42.6488 60.63,43.5717 61.3756,47.0434 59.8888,49.5922 60.0867,49.1527 58.1706,51.9213 57.9147,50.2514 56.1454,52.6684 55.8259,51.6576 54.2908,52.8881 53.9301,50.6469 53.0147,51.3939 52.4827,48.0541 51.1792,49.2846 50.5413,47.1752 49.1529,43.9232 49.8095,42.5609 48.7778,36.9359 49.6675,35.266 48.7488,32.8051 49.5251,27.224 48.9801,26.1254 50.4014,21.2035 50.2049,20.017 51.5633,17.4682 53.0147,19.4457 54.0851,19.4457 55.8752,19.5775 57.4921,19.5775 58.6768,24.3676 61.4597),(24.4555 59.4226,21.2035 58.4936,21.3353 56.8969,21.599 55.9245,25.2025 55.9983,28.85 57.0645,27.0922 57.8447,28.8061 59.1758,26.2572 59.1758,24.4555 59.4226),(35.1914 56.569,36.7073 56.2109,34.7125 56.1099,36.5562 55.6351,35.1357 55.5326,36.4353 55.344,34.8334 55.0161,35.6192 54.4928,34.8939 54.1225,35.3775 53.0461,37.0823 52.9707,34.9584 51.4813,36.5404 50.4014,39.6605 50.2892,39.7924 52.1334,41.7699 50.6807,44.4945 51.9712,47.307 52.5094,44.0551 53.5402,46.6039 53.6966,47.6146 55.404,45.3734 55.404,42.8246 56.5836,40.4556 56.1121,40.4249 56.1941,39.8204 56.0762,39.7902 56.412,40.4854 56.4956,40.3342 56.9598,39.7902 56.9928,39.7388 57.3963,40.066 57.4672,40.3342 57.4672,40.3255 57.5233,42.6928 58.0313,40.891 59.2658,39.3164 59.0294,38.5208 59.1189,38.9056 58.9675,38.1151 58.848,37.3118 59.0257,37.2322 59.0224,37.1117 59.6676,35.1342 59.8447,31.9701 58.9726,32.2439 58.4956,32.2341 58.4927,32.1737 58.0317,31.5087 57.4997,31.7505 56.8608,31.5087 55.941,32.2039 55.5155,31.8412 54.9988,31.6171 54.6946,29.5971 55.5036,29.1576 55.7517,22.5658 55.1285,22.5658 53.5402,22.0385 51.4813,26.2572 51.4265,30.1244 50.5413,32.1898 51.1792,30.1244 53.173,32.5252 53.173,33.1711 52.8275,34.773 52.9187,34.773 53.173,35.0902 53.173,34.773 53.3207,34.773 53.7846,34.108 54.1756,35.0752 54.598,34.2592 54.9641,35.0148 55.3612,33.5035 55.3784,32.7479 55.9071,35.6797 55.6862,32.8386 56.3116,34.5916 56.2948,35.7099 56.3116,33.836 56.6952,33.7508 56.7194,35.1914 56.569)),((35.1914 56.569,34.2289 56.7947,36.9793 57.075,35.7704 57.2553,37.0096 57.4997,35.7401 57.7908,37.1607 58.0477,36.0122 58.2868,37.1909 58.6818,34.6521 58.9166,37.2322 59.0224,37.2875 58.7225,38.1151 58.848,38.5812 58.7445,37.4025 58.3186,38.3394 57.9355,37.4327 57.7102,38.1279 57.5159,37.1607 57.2553,38.3092 56.9928,36.3749 56.6454,36.8783 56.4863,36.6723 56.4138,35.1914 56.569)),((33.106 56.9503,32.2439 58.4956,33.4733 58.8541,34.7427 59.5658,33.836 58.6818,36.3446 58.5401,33.6244 58.2709,36.4353 58.0477,33.2315 57.7747,36.1935 57.4997,33.1711 57.3369,36.0726 57.0914,33.106 56.9503)),((33.106 56.9503,33.1372 56.8932,32.9595 56.9433,33.106 56.9503)),((33.7508 56.7194,33.2006 56.7767,33.1372 56.8932,33.7508 56.7194)),((37.0823 52.9707,37.2279 53.0709,37.4327 52.9551,37.0823 52.9707)),((34.773 53.173,32.5252 53.173,32.083 53.4079,32.4759 53.8382,31.4181 54.4226,31.6171 54.6946,33.1127 54.0851,34.773 53.3207,34.773 53.173)),((40.4556 56.1121,40.576 55.7883,39.76 55.7543,39.8204 55.2752,40.3947 55.2407,40.3947 54.8772,39.5484 54.8772,39.5484 54.563,40.3947 54.4402,40.6063 54.0339,39.9715 53.9806,40.2436 53.5877,39.5484 53.5877,40.0018 53.3539,38.3394 53.2816,39.5786 52.6995,37.8558 52.9187,37.4544 53.2264,37.9906 53.5924,37.0521 54.7595,38.3092 55.1545,37.7954 55.3955,38.4906 55.5326,38.3463 55.6874,40.4556 56.1121)),((37.2279 53.0709,35.9215 53.8025,36.9189 53.856,36.0122 54.7553,36.8796 54.9705,37.0521 54.7595,36.7073 54.6505,37.3118 53.9272,36.9793 53.5877,37.4544 53.2264,37.2279 53.0709)),((38.3463 55.6874,36.9706 55.4079,36.8282 55.447,38.0372 55.6522,36.6469 55.941,38.2185 56.0593,36.8783 56.4863,38.5847 57.0811,38.9741 56.8773,37.4629 56.5622,38.9439 56.0593,38.1883 55.8563,38.3463 55.6874)),((36.8796 54.9705,36.5844 55.329,36.9706 55.4079,37.7652 55.189,36.8796 54.9705)),((39.3164 59.0294,40.4551 58.901,40.3342 58.382,39.6391 58.382,39.6391 58.0477,40.2436 58.0477,40.3255 57.5233,40.066 57.4672,39.7298 57.4672,39.7388 57.3963,39.0893 57.2552,38.5847 57.0811,38.2185 57.2716,38.7324 57.4834,38.3394 57.7102,38.8532 58.0637,38.3697 58.2868,39.5484 58.7132,38.9056 58.9675,39.3164 59.0294))) -------- Polygon with Polygon with Holes MULTIPOLYGON(((24.3677 61.4598,26.6528 61.1008,26.8726 61.7107,30.564 61.0583,31.3989 62.0215,36.0132 61.1432,36.8921 62.0009,42.6489 60.6301,43.5718 61.3757,47.0435 59.8889,49.5923 60.0868,49.1528 58.1707,51.9214 57.9148,50.2515 56.1455,52.6685 55.826,51.6577 54.2909,52.8882 53.9302,50.647 53.0148,51.394 52.4828,48.0542 51.1793,49.2847 50.5414,47.1753 49.153,43.9233 49.8096,42.561 48.7779,36.936 49.6676,35.2661 48.7489,32.8052 49.5252,27.2241 48.9802,26.1255 50.4015,21.2036 50.205,20.0171 51.5634,17.4683 53.0148,19.4458 54.0852,19.4458 55.8753,19.5776 57.4922,19.5776 58.6769,24.3677 61.4598),(24.4556 59.4227,21.2036 58.4937,21.3354 56.897,21.5991 55.9246,25.2026 55.9984,28.8501 57.0646,27.0923 57.8448,28.8062 59.1759,26.2573 59.1759,24.4556 59.4227),(35.9475 59.7758,36.2843 59.9616,34.8637 59.9768,34.2247 59.6064,31.9702 58.9727,32.2964 58.4175,30.179 56.9764,30.179 57.9196,33.6548 59.9465,32.3249 59.9465,30.4812 58.8542,30.5719 59.9919,29.4536 59.7796,29.4171 55.606,29.1577 55.7518,22.5659 55.1286,22.5659 53.5403,22.0386 51.4814,26.2573 51.4266,30.1245 50.5414,32.1899 51.1793,31.1968 52.1649,31.2368 52.1652,32.5603 53.1989,33.8733 53.1922,32.5969 52.2208,34.2895 52.2208,37.2766 54.4948,37.7431 53.9104,35.4682 52.2022,35.9681 52.2157,34.9585 51.4814,36.5405 50.4015,39.6606 50.2893,39.7925 52.1335,41.77 50.6808,44.4946 51.9713,47.3071 52.5095,44.0552 53.5403,46.604 53.6967,47.6147 55.4041,45.3735 55.4041,44.4212 55.8594,44.4751 59.81,39.7299 59.9314,37.6322 58.7797,37.2876 58.7226,37.2102 59.1452,38.7325 59.9465,37.2817 59.9768,36.7912 59.6986,35.9475 59.7758)),((32.6512 57.792,32.2964 58.4175,34.2247 59.6064,35.1343 59.8448,35.9475 59.7758,32.6512 57.792)),((32.6512 57.792,32.9378 57.2699,30.2394 55.2753,30.3301 56.1942,32.6512 57.792)),((33.2446 56.7729,33.2007 56.7768,32.9378 57.2699,36.7912 59.6986,37.1118 59.6677,37.2102 59.1452,33.2446 56.7729)),((33.2446 56.7729,34.2635 56.6767,31.5682 54.7333,30.7705 55.0525,33.2446 56.7729)),((34.2635 56.6767,37.6322 58.7797,40.2079 59.1718,35.4536 56.5531,34.2635 56.6767)),((40.2079 59.1718,40.6366 59.3817,40.8804 59.2644,40.2079 59.1718)),((34.3351 53.53,35.0903 53.1731,33.8733 53.1922,34.3351 53.53)),((34.3351 53.53,33.5144 53.9057,38.1759 56.9472,39.0894 57.2553,40.9691 57.677,37.1934 55.4694,36.5845 55.3291,36.7219 55.1665,34.3351 53.53)),((32.6907 54.2663,33.1128 54.0852,33.5144 53.9057,32.5603 53.1989,31.1682 53.1903,32.6907 54.2663)),((32.6907 54.2663,32.2591 54.4483,35.4536 56.5531,36.1815 56.4715,32.6907 54.2663)),((38.1759 56.9472,36.6724 56.4139,36.1815 56.4715,41.168 59.0834,41.5887 58.8012,38.1759 56.9472)),((37.2766 54.4948,36.7219 55.1665,37.1934 55.4694,39.4328 55.9511,37.2766 54.4948)),((40.9691 57.677,42.2498 58.3455,42.6929 58.0314,40.9691 57.677)),((30.7705 55.0525,30.2092 54.6331,30.2394 53.6774,31.5682 54.7333,32.2591 54.4483,30.5408 53.1811,30.1245 53.1731,30.3098 53.0028,29.3931 52.2763,29.4171 55.606,29.5972 55.5037,30.7705 55.0525)),((30.5408 53.1811,31.1682 53.1903,30.5785 52.7531,30.3098 53.0028,30.5408 53.1811)),((30.5785 52.7531,31.1968 52.1649,29.7861 52.1466,30.5785 52.7531)),((35.9681 52.2157,37.9907 53.5925,37.7431 53.9104,41.4519 56.3413,42.8247 56.5837,44.4212 55.8594,44.4146 55.3097,40.0925 52.1652,38.3395 52.1652,43.0243 55.3269,43.0243 56.2614,37.1608 52.2393,35.9681 52.2157)),((39.4328 55.9511,43.0243 58.0797,43.0243 57.2554,41.4519 56.3413,39.4328 55.9511)),((41.168 59.0834,40.9299 59.2404,41.2108 59.1035,41.168 59.0834)),((41.5887 58.8012,41.6944 58.8542,42.5105 58.477,42.2498 58.3455,41.5887 58.8012)),((40.9299 59.2404,40.8804 59.2644,40.8911 59.2659,40.9299 59.2404))) -MULTIPOLYGON(((24.3676 61.4597,26.6527 61.1007,26.8725 61.7106,30.5639 61.0582,31.3988 62.0214,36.0131 61.1431,36.892 62.0008,42.6488 60.63,43.5717 61.3756,47.0434 59.8888,49.5922 60.0867,49.1527 58.1706,51.9213 57.9147,50.2514 56.1454,52.6684 55.8259,51.6576 54.2908,52.8881 53.9301,50.6469 53.0147,51.3939 52.4827,48.0541 51.1792,49.2846 50.5413,47.1752 49.1529,43.9232 49.8095,42.5609 48.7778,36.9359 49.6675,35.266 48.7488,32.8051 49.5251,27.224 48.9801,26.1254 50.4014,21.2035 50.2049,20.017 51.5633,17.4682 53.0147,19.4457 54.0851,19.4457 55.8752,19.5775 57.4921,19.5775 58.6768,24.3676 61.4597),(24.4555 59.4226,21.2035 58.4936,21.3353 56.8969,21.599 55.9245,25.2025 55.9983,28.85 57.0645,27.0922 57.8447,28.8061 59.1758,26.2572 59.1758,24.4555 59.4226),(35.9663 59.7703,36.2842 59.9615,34.8636 59.9767,34.2648 59.6073,31.9701 58.9726,32.3127 58.3747,30.1789 56.9763,30.1789 57.9195,33.6547 59.9464,32.3248 59.9464,30.4811 58.8541,30.5718 59.9918,29.4535 59.7795,29.4184 55.6047,29.1576 55.7517,22.5658 55.1285,22.5658 53.5402,22.0385 51.4813,26.2572 51.4265,30.1244 50.5413,32.1898 51.1792,31.1809 52.1644,31.2367 52.1651,32.7039 53.173,34.0091 53.173,32.5968 52.2207,34.2894 52.2207,37.3607 54.3795,37.8015 53.8302,35.4681 52.2021,35.9939 52.2136,34.9584 51.4813,36.5404 50.4014,39.6605 50.2892,39.7924 52.1334,41.7699 50.6807,44.4945 51.9712,47.307 52.5094,44.0551 53.5402,46.6039 53.6966,47.6146 55.404,45.3734 55.404,44.4214 55.8488,44.475 59.8099,39.7298 59.9313,37.9197 58.8184,37.2875 58.7225,37.2249 59.0621,38.7324 59.9464,37.2816 59.9767,36.8295 59.693,35.9663 59.7703)),((32.6773 57.7269,32.3127 58.3747,34.2648 59.6073,35.1342 59.8447,35.9663 59.7703,32.6773 57.7269)),((32.6773 57.7269,32.9811 57.1783,30.2393 55.2752,30.33 56.1941,32.6773 57.7269)),((33.4587 56.7498,33.2006 56.7767,32.9811 57.1783,36.8295 59.693,37.1117 59.6676,37.2249 59.0621,33.4587 56.7498)),((33.4587 56.7498,34.5347 56.6377,31.6568 54.6786,30.7975 55.0249,33.4587 56.7498)),((34.5347 56.6377,37.9197 58.8184,40.276 59.1737,35.8287 56.5023,34.5347 56.6377)),((40.276 59.1737,40.6365 59.3816,40.8796 59.2641,40.276 59.1737)),((34.4539 53.4687,35.0902 53.173,34.0091 53.173,34.4539 53.4687)),((34.4539 53.4687,33.6753 53.8276,38.9459 57.2058,39.0893 57.2552,41.2147 57.7149,37.6798 55.5522,36.5844 55.329,36.8394 55.0196,34.4539 53.4687)),((32.9056 54.17,33.1127 54.0851,33.6753 53.8276,32.7039 53.173,31.3501 53.173,32.9056 54.17)),((32.9056 54.17,32.4672 54.3493,35.8287 56.5023,36.5672 56.4248,32.9056 54.17)),((38.9459 57.2058,36.6723 56.4138,36.5672 56.4248,41.1674 59.0793,41.5893 58.7925,38.9459 57.2058)),((37.3607 54.3795,36.8394 55.0196,37.6798 55.5522,39.7202 55.9646,37.3607 54.3795)),((41.2147 57.7149,42.2607 58.3313,42.6928 58.0313,41.2147 57.7149)),((31.6568 54.6786,32.4672 54.3493,30.7047 53.173,30.1244 53.173,30.3667 52.9437,29.393 52.2762,29.4184 55.6047,29.5971 55.5036,30.7975 55.0249,30.2091 54.633,30.2393 53.6773,31.6568 54.6786)),((30.7047 53.173,31.3501 53.173,30.6237 52.6993,30.3667 52.9437,30.7047 53.173)),((30.6237 52.6993,31.1809 52.1644,29.786 52.1465,30.6237 52.6993)),((44.4214 55.8488,44.4145 55.3096,40.0924 52.1651,38.3394 52.1651,43.0242 55.3268,43.0242 56.2613,37.1607 52.2392,35.9939 52.2136,37.9906 53.5924,37.8015 53.8302,41.577 56.336,42.8246 56.5836,44.4214 55.8488)),((39.7202 55.9646,43.0242 58.0796,43.0242 57.2553,41.577 56.336,39.7202 55.9646)),((41.1674 59.0793,40.9293 59.24,41.2107 59.1034,41.1674 59.0793)),((41.5893 58.7925,41.6943 58.8541,42.5104 58.4769,42.2607 58.3313,41.5893 58.7925)),((40.9293 59.24,40.8796 59.2641,40.891 59.2658,40.9293 59.24))) diff --git a/tests/queries/0_stateless/01304_polygons_sym_difference.sql b/tests/queries/0_stateless/01304_polygons_sym_difference.sql index b9f2ef126e6..f4893dd5b33 100644 --- a/tests/queries/0_stateless/01304_polygons_sym_difference.sql +++ b/tests/queries/0_stateless/01304_polygons_sym_difference.sql @@ -2,12 +2,9 @@ select polygonsSymDifferenceCartesian([[[(0, 0),(0, 3),(1, 2.9),(2, 2.6),(2.6, 2 select '-------- MultiPolygon with Polygon'; select wkt(polygonsSymDifferenceSpherical([[(29.453587685533865,59.779570356240356),(29.393139070478895,52.276266797422124),(40.636581470703206,59.38168915000267),(41.21084331372543,59.103467777099866),(29.786055068336193,52.146627480315004),(31.23682182965546,52.16517054781818),(41.69443223416517,58.85424941916091),(42.51048853740727,58.47703162291134),(32.59691566839227,52.22075341251539),(34.289476889931414,52.22075341251539),(43.02430176537451,58.07974369546071),(43.02430176537451,57.25537683364851),(35.468224883503325,52.2022335126388),(37.16078610504247,52.23926559241349),(43.02430176537451,56.26136189644947),(43.02430176537451,55.326904361850836),(38.33953409861437,52.16517054781818),(40.09254393520848,52.16517054781818),(44.4146199116388,55.3097062225408),(44.47506852669377,59.80998197603594),(39.72985224487867,59.931351417569715),(30.23941968124846,53.67744677450975),(30.20919537372098,54.63314259659509),(38.73245009647167,59.94649146557819),(37.2816833351524,59.97675082987618),(30.23941968124846,55.2752875586599),(30.33009260383092,56.19415599955667),(36.28428118674541,59.96162460231375),(34.863738732953635,59.97675082987618),(30.178971066193498,56.97640788219866),(30.178971066193498,57.91957806959033),(33.65476643185424,59.94649146557819),(32.32489690064491,59.94649146557819),(30.481214141468342,58.85424941916091),(30.571887064050795,59.99187015036608),(29.453587685533865,59.779570356240356)]], [[[(33.473420586689336,58.85424941916091),(32.23422397806246,58.492830557036),(32.173775363007486,58.03176922751564),(31.508840597402823,57.499784781503735),(31.750635057622702,56.86092686957355),(31.508840597402823,55.941082594334574),(32.20399967053497,55.515591939372456),(31.84130798020516,54.998862226280465),(31.418167674820367,54.422670886434275),(32.47601843828233,53.83826377018255),(32.08310244042503,53.408048308050866),(33.171177511414484,52.82758702113742),(34.77306581037117,52.91880107773494),(34.77306581037117,53.784726518357985),(34.108131044766516,54.17574726780569),(35.07530888564602,54.59813930694554),(34.25925258240394,54.96417435716029),(35.01486027059106,55.361278263643584),(33.50364489421682,55.37845402950552),(32.7480372060297,55.90721384574556),(35.67979503619571,55.68634475630185),(32.83871012861215,56.311688992608396),(34.591719965206266,56.29492065473883),(35.7100193437232,56.311688992608396),(33.83611227701915,56.695333481003644),(32.95960735872209,56.9434497616887),(36.072711034053015,57.091531913901434),(33.171177511414484,57.33702717078384),(36.193608264162954,57.499784781503735),(33.23162612646945,57.77481561306047),(36.43540272438284,58.04776787540811),(33.62454212432676,58.27099811968307),(36.344729801800376,58.54018474404165),(33.83611227701915,58.68186423448108),(34.74284150284369,59.565911441555244),(33.473420586689336,58.85424941916091)]], [[(34.65216858026123,58.91672306881671),(37.19101041256995,58.68186423448108),(36.01226241899805,58.28688958537609),(37.16078610504247,58.04776787540811),(35.74024365125068,57.79092907387934),(37.009664567405046,57.499784781503735),(35.77046795877817,57.25537683364851),(36.979440259877556,57.07510745541089),(34.22902827487645,56.794777197297435),(36.7074214921302,56.210968525786996),(34.712617195316206,56.10998276812964),(36.55629995449277,55.63519693782703),(35.13575750070099,55.53270067649592),(36.43540272438284,55.34409504165558),(34.83351442542614,55.01619492319591),(35.61934642114075,54.49294870011772),(34.89396304048112,54.12264226523038),(35.37755196092087,53.046178687628185),(37.43280487278982,52.95523300597458),(35.92158949641559,53.80257986695776),(36.91899164482259,53.856094327816805),(36.01226241899805,54.75541714463799),(37.765272255592166,55.189110239786885),(36.828318722240134,55.44708256557195),(38.03729102333953,55.652253637168315),(36.64697287707522,55.941082594334574),(38.21863686850443,56.05939028508024),(36.37495410932787,56.64551287174558),(38.30930979108689,56.992876013526654),(37.16078610504247,57.25537683364851),(38.127963945921984,57.516020773674256),(37.43280487278982,57.710289827306724),(38.33953409861437,57.935626886818994),(37.40258056526235,58.31865112960426),(38.58132855883426,58.744648733419496),(37.31190764267989,59.02578062465136),(34.65216858026123,58.91672306881671)]], [[(38.52087994377928,59.11898412389468),(39.54850639971376,58.713270635642914),(38.369758406141855,58.28688958537609),(38.85334732658162,58.06375936407028),(38.33953409861437,57.710289827306724),(38.73245009647167,57.48354156434209),(38.21863686850443,57.271721400459285),(38.97424455669155,56.87744603722649),(37.463029180317314,56.5623320541159),(38.94402024916407,56.05939028508024),(38.18841256097694,55.856355210835915),(38.490655636251795,55.53270067649592),(37.795496563119656,55.39562234093384),(38.30930979108689,55.154587013355666),(36.7074214921302,54.65063295250911),(37.31190764267989,53.92734063371401),(36.979440259877556,53.58783775557231),(37.855945178174615,52.91880107773497),(39.57873070724124,52.69956490610895),(38.33953409861437,53.281741738901104),(40.00187101262603,53.35396273604752),(39.54850639971376,53.58783775557231),(40.24366547284591,53.58783775557231),(39.97164670509855,53.98069568468355),(40.60635716317572,54.03398248547225),(40.39478701048334,54.44025165268903),(39.54850639971376,54.56310590284329),(39.54850639971376,54.87732350170489),(40.39478701048334,54.87732350170489),(40.39478701048334,55.24083903654295),(39.82052516746112,55.2752875586599),(39.760076552406154,55.75443792473942),(40.57613285564824,55.78844000174894),(40.425011318010824,56.19415599955667),(39.82052516746112,56.07626182891758),(39.79030085993364,56.41214455508424),(40.48545993306579,56.495655446714636),(40.33433839542836,56.95993246553937),(39.79030085993364,56.992876013526654),(39.72985224487867,57.46729112028032),(40.33433839542836,57.46729112028032),(40.24366547284591,58.04776787540811),(39.63917932229622,58.04776787540811),(39.63917932229622,58.382088724871295),(40.33433839542836,58.382088724871295),(40.45523562553831,58.9011152358548),(38.52087994377928,59.11898412389468)]]])) format TSV; -select wkt(polygonsSymDifferenceMercator([[(29.453587685533865,59.779570356240356),(29.393139070478895,52.276266797422124),(40.636581470703206,59.38168915000267),(41.21084331372543,59.103467777099866),(29.786055068336193,52.146627480315004),(31.23682182965546,52.16517054781818),(41.69443223416517,58.85424941916091),(42.51048853740727,58.47703162291134),(32.59691566839227,52.22075341251539),(34.289476889931414,52.22075341251539),(43.02430176537451,58.07974369546071),(43.02430176537451,57.25537683364851),(35.468224883503325,52.2022335126388),(37.16078610504247,52.23926559241349),(43.02430176537451,56.26136189644947),(43.02430176537451,55.326904361850836),(38.33953409861437,52.16517054781818),(40.09254393520848,52.16517054781818),(44.4146199116388,55.3097062225408),(44.47506852669377,59.80998197603594),(39.72985224487867,59.931351417569715),(30.23941968124846,53.67744677450975),(30.20919537372098,54.63314259659509),(38.73245009647167,59.94649146557819),(37.2816833351524,59.97675082987618),(30.23941968124846,55.2752875586599),(30.33009260383092,56.19415599955667),(36.28428118674541,59.96162460231375),(34.863738732953635,59.97675082987618),(30.178971066193498,56.97640788219866),(30.178971066193498,57.91957806959033),(33.65476643185424,59.94649146557819),(32.32489690064491,59.94649146557819),(30.481214141468342,58.85424941916091),(30.571887064050795,59.99187015036608),(29.453587685533865,59.779570356240356)]], [[[(33.473420586689336,58.85424941916091),(32.23422397806246,58.492830557036),(32.173775363007486,58.03176922751564),(31.508840597402823,57.499784781503735),(31.750635057622702,56.86092686957355),(31.508840597402823,55.941082594334574),(32.20399967053497,55.515591939372456),(31.84130798020516,54.998862226280465),(31.418167674820367,54.422670886434275),(32.47601843828233,53.83826377018255),(32.08310244042503,53.408048308050866),(33.171177511414484,52.82758702113742),(34.77306581037117,52.91880107773494),(34.77306581037117,53.784726518357985),(34.108131044766516,54.17574726780569),(35.07530888564602,54.59813930694554),(34.25925258240394,54.96417435716029),(35.01486027059106,55.361278263643584),(33.50364489421682,55.37845402950552),(32.7480372060297,55.90721384574556),(35.67979503619571,55.68634475630185),(32.83871012861215,56.311688992608396),(34.591719965206266,56.29492065473883),(35.7100193437232,56.311688992608396),(33.83611227701915,56.695333481003644),(32.95960735872209,56.9434497616887),(36.072711034053015,57.091531913901434),(33.171177511414484,57.33702717078384),(36.193608264162954,57.499784781503735),(33.23162612646945,57.77481561306047),(36.43540272438284,58.04776787540811),(33.62454212432676,58.27099811968307),(36.344729801800376,58.54018474404165),(33.83611227701915,58.68186423448108),(34.74284150284369,59.565911441555244),(33.473420586689336,58.85424941916091)]], [[(34.65216858026123,58.91672306881671),(37.19101041256995,58.68186423448108),(36.01226241899805,58.28688958537609),(37.16078610504247,58.04776787540811),(35.74024365125068,57.79092907387934),(37.009664567405046,57.499784781503735),(35.77046795877817,57.25537683364851),(36.979440259877556,57.07510745541089),(34.22902827487645,56.794777197297435),(36.7074214921302,56.210968525786996),(34.712617195316206,56.10998276812964),(36.55629995449277,55.63519693782703),(35.13575750070099,55.53270067649592),(36.43540272438284,55.34409504165558),(34.83351442542614,55.01619492319591),(35.61934642114075,54.49294870011772),(34.89396304048112,54.12264226523038),(35.37755196092087,53.046178687628185),(37.43280487278982,52.95523300597458),(35.92158949641559,53.80257986695776),(36.91899164482259,53.856094327816805),(36.01226241899805,54.75541714463799),(37.765272255592166,55.189110239786885),(36.828318722240134,55.44708256557195),(38.03729102333953,55.652253637168315),(36.64697287707522,55.941082594334574),(38.21863686850443,56.05939028508024),(36.37495410932787,56.64551287174558),(38.30930979108689,56.992876013526654),(37.16078610504247,57.25537683364851),(38.127963945921984,57.516020773674256),(37.43280487278982,57.710289827306724),(38.33953409861437,57.935626886818994),(37.40258056526235,58.31865112960426),(38.58132855883426,58.744648733419496),(37.31190764267989,59.02578062465136),(34.65216858026123,58.91672306881671)]], [[(38.52087994377928,59.11898412389468),(39.54850639971376,58.713270635642914),(38.369758406141855,58.28688958537609),(38.85334732658162,58.06375936407028),(38.33953409861437,57.710289827306724),(38.73245009647167,57.48354156434209),(38.21863686850443,57.271721400459285),(38.97424455669155,56.87744603722649),(37.463029180317314,56.5623320541159),(38.94402024916407,56.05939028508024),(38.18841256097694,55.856355210835915),(38.490655636251795,55.53270067649592),(37.795496563119656,55.39562234093384),(38.30930979108689,55.154587013355666),(36.7074214921302,54.65063295250911),(37.31190764267989,53.92734063371401),(36.979440259877556,53.58783775557231),(37.855945178174615,52.91880107773497),(39.57873070724124,52.69956490610895),(38.33953409861437,53.281741738901104),(40.00187101262603,53.35396273604752),(39.54850639971376,53.58783775557231),(40.24366547284591,53.58783775557231),(39.97164670509855,53.98069568468355),(40.60635716317572,54.03398248547225),(40.39478701048334,54.44025165268903),(39.54850639971376,54.56310590284329),(39.54850639971376,54.87732350170489),(40.39478701048334,54.87732350170489),(40.39478701048334,55.24083903654295),(39.82052516746112,55.2752875586599),(39.760076552406154,55.75443792473942),(40.57613285564824,55.78844000174894),(40.425011318010824,56.19415599955667),(39.82052516746112,56.07626182891758),(39.79030085993364,56.41214455508424),(40.48545993306579,56.495655446714636),(40.33433839542836,56.95993246553937),(39.79030085993364,56.992876013526654),(39.72985224487867,57.46729112028032),(40.33433839542836,57.46729112028032),(40.24366547284591,58.04776787540811),(39.63917932229622,58.04776787540811),(39.63917932229622,58.382088724871295),(40.33433839542836,58.382088724871295),(40.45523562553831,58.9011152358548),(38.52087994377928,59.11898412389468)]]])) format TSV; select '-------- MultiPolygon with Polygon with Holes'; select wkt(polygonsSymDifferenceSpherical([[[(33.473420586689336,58.85424941916091),(32.23422397806246,58.492830557036),(32.173775363007486,58.03176922751564),(31.508840597402823,57.499784781503735),(31.750635057622702,56.86092686957355),(31.508840597402823,55.941082594334574),(32.20399967053497,55.515591939372456),(31.84130798020516,54.998862226280465),(31.418167674820367,54.422670886434275),(32.47601843828233,53.83826377018255),(32.08310244042503,53.408048308050866),(33.171177511414484,52.82758702113742),(34.77306581037117,52.91880107773494),(34.77306581037117,53.784726518357985),(34.108131044766516,54.17574726780569),(35.07530888564602,54.59813930694554),(34.25925258240394,54.96417435716029),(35.01486027059106,55.361278263643584),(33.50364489421682,55.37845402950552),(32.7480372060297,55.90721384574556),(35.67979503619571,55.68634475630185),(32.83871012861215,56.311688992608396),(34.591719965206266,56.29492065473883),(35.7100193437232,56.311688992608396),(33.83611227701915,56.695333481003644),(32.95960735872209,56.9434497616887),(36.072711034053015,57.091531913901434),(33.171177511414484,57.33702717078384),(36.193608264162954,57.499784781503735),(33.23162612646945,57.77481561306047),(36.43540272438284,58.04776787540811),(33.62454212432676,58.27099811968307),(36.344729801800376,58.54018474404165),(33.83611227701915,58.68186423448108),(34.74284150284369,59.565911441555244),(33.473420586689336,58.85424941916091)]], [[(34.65216858026123,58.91672306881671),(37.19101041256995,58.68186423448108),(36.01226241899805,58.28688958537609),(37.16078610504247,58.04776787540811),(35.74024365125068,57.79092907387934),(37.009664567405046,57.499784781503735),(35.77046795877817,57.25537683364851),(36.979440259877556,57.07510745541089),(34.22902827487645,56.794777197297435),(36.7074214921302,56.210968525786996),(34.712617195316206,56.10998276812964),(36.55629995449277,55.63519693782703),(35.13575750070099,55.53270067649592),(36.43540272438284,55.34409504165558),(34.83351442542614,55.01619492319591),(35.61934642114075,54.49294870011772),(34.89396304048112,54.12264226523038),(35.37755196092087,53.046178687628185),(37.43280487278982,52.95523300597458),(35.92158949641559,53.80257986695776),(36.91899164482259,53.856094327816805),(36.01226241899805,54.75541714463799),(37.765272255592166,55.189110239786885),(36.828318722240134,55.44708256557195),(38.03729102333953,55.652253637168315),(36.64697287707522,55.941082594334574),(38.21863686850443,56.05939028508024),(36.37495410932787,56.64551287174558),(38.30930979108689,56.992876013526654),(37.16078610504247,57.25537683364851),(38.127963945921984,57.516020773674256),(37.43280487278982,57.710289827306724),(38.33953409861437,57.935626886818994),(37.40258056526235,58.31865112960426),(38.58132855883426,58.744648733419496),(37.31190764267989,59.02578062465136),(34.65216858026123,58.91672306881671)]], [[(38.52087994377928,59.11898412389468),(39.54850639971376,58.713270635642914),(38.369758406141855,58.28688958537609),(38.85334732658162,58.06375936407028),(38.33953409861437,57.710289827306724),(38.73245009647167,57.48354156434209),(38.21863686850443,57.271721400459285),(38.97424455669155,56.87744603722649),(37.463029180317314,56.5623320541159),(38.94402024916407,56.05939028508024),(38.18841256097694,55.856355210835915),(38.490655636251795,55.53270067649592),(37.795496563119656,55.39562234093384),(38.30930979108689,55.154587013355666),(36.7074214921302,54.65063295250911),(37.31190764267989,53.92734063371401),(36.979440259877556,53.58783775557231),(37.855945178174615,52.91880107773497),(39.57873070724124,52.69956490610895),(38.33953409861437,53.281741738901104),(40.00187101262603,53.35396273604752),(39.54850639971376,53.58783775557231),(40.24366547284591,53.58783775557231),(39.97164670509855,53.98069568468355),(40.60635716317572,54.03398248547225),(40.39478701048334,54.44025165268903),(39.54850639971376,54.56310590284329),(39.54850639971376,54.87732350170489),(40.39478701048334,54.87732350170489),(40.39478701048334,55.24083903654295),(39.82052516746112,55.2752875586599),(39.760076552406154,55.75443792473942),(40.57613285564824,55.78844000174894),(40.425011318010824,56.19415599955667),(39.82052516746112,56.07626182891758),(39.79030085993364,56.41214455508424),(40.48545993306579,56.495655446714636),(40.33433839542836,56.95993246553937),(39.79030085993364,56.992876013526654),(39.72985224487867,57.46729112028032),(40.33433839542836,57.46729112028032),(40.24366547284591,58.04776787540811),(39.63917932229622,58.04776787540811),(39.63917932229622,58.382088724871295),(40.33433839542836,58.382088724871295),(40.45523562553831,58.9011152358548),(38.52087994377928,59.11898412389468)]]], [[(24.367675781249993,61.45977057029751),(19.577636718749993,58.67693767258692),(19.577636718749993,57.492213666700735),(19.445800781249996,55.87531083569678),(19.445800781249996,54.085173420886775),(17.468261718749996,53.014783245859235),(20.017089843749993,51.563412328675895),(21.203613281249993,50.205033264943324),(26.125488281249993,50.40151532278236),(27.22412109374999,48.980216985374994),(32.80517578124999,49.525208341974405),(35.26611328124999,48.74894534343292),(36.93603515624999,49.66762782262194),(42.56103515625,48.77791275550183),(43.92333984374999,49.8096315635631),(47.17529296875,49.152969656170455),(49.28466796875,50.54136296522162),(48.05419921875,51.17934297928929),(51.39404296875,52.48278022207825),(50.64697265625,53.014783245859235),(52.88818359375,53.93021986394004),(51.65771484374999,54.29088164657006),(52.66845703125,55.825973254619015),(50.25146484375,56.145549500679095),(51.92138671875,57.914847767009206),(49.15283203125,58.17070248348605),(49.59228515625,60.086762746260064),(47.043457031249986,59.88893689676584),(43.57177734375,61.37567331572748),(42.64892578125,60.630101766266705),(36.89208984374999,62.000904713685856),(36.01318359374999,61.143235250840576),(31.398925781249993,62.02152819100766),(30.563964843749996,61.05828537037917),(26.872558593749993,61.71070595883174),(26.652832031249993,61.10078883158897),(24.367675781249993,61.45977057029751)], [(24.455566406249993,59.42272750081452),(21.203613281249993,58.49369382056807),(21.335449218749993,56.89700392127261),(21.599121093749993,55.92458580482949),(25.202636718749993,55.998380955359636),(28.850097656249993,57.06463027327854),(27.09228515625,57.844750992890994),(28.806152343749996,59.17592824927138),(26.257324218749993,59.17592824927138),(24.455566406249993,59.42272750081452)], [(35.13427734375,59.84481485969107),(31.970214843749993,58.97266715450152),(33.20068359374999,56.776808316568406),(36.67236328125,56.41390137600675),(39.08935546874999,57.25528054528888),(42.69287109374999,58.03137242177638),(40.89111328124999,59.26588062825809),(37.28759765625,58.722598828043374),(37.11181640624999,59.66774058164964),(35.13427734375,59.84481485969107)], [(29.157714843749993,55.75184939173528),(22.565917968749993,55.128649068488784),(22.565917968749993,53.54030739150019),(22.038574218749996,51.48138289610097),(26.257324218749993,51.42661449707484),(30.124511718749993,50.54136296522162),(32.18994140624999,51.17934297928929),(30.124511718749993,53.173119202640635),(35.09033203124999,53.173119202640635),(33.11279296875,54.085173420886775),(29.597167968749993,55.50374985927513),(29.157714843749993,55.75184939173528)], [(42.82470703125,56.58369172128337),(36.584472656249986,55.329144408405085),(37.99072265625,53.592504809039355),(34.95849609374999,51.48138289610097),(36.54052734374999,50.40151532278236),(39.66064453124999,50.289339253291786),(39.79248046875,52.13348804077148),(41.77001953125,50.68079714532166),(44.49462890624999,51.97134580885171),(47.30712890624999,52.509534770327264),(44.05517578125,53.54030739150019),(46.60400390625,53.696706475303245),(47.61474609375,55.40406982700608),(45.37353515625,55.40406982700608),(42.82470703125,56.58369172128337)]])) format TSV; -select wkt(polygonsSymDifferenceMercator([[[(33.473420586689336,58.85424941916091),(32.23422397806246,58.492830557036),(32.173775363007486,58.03176922751564),(31.508840597402823,57.499784781503735),(31.750635057622702,56.86092686957355),(31.508840597402823,55.941082594334574),(32.20399967053497,55.515591939372456),(31.84130798020516,54.998862226280465),(31.418167674820367,54.422670886434275),(32.47601843828233,53.83826377018255),(32.08310244042503,53.408048308050866),(33.171177511414484,52.82758702113742),(34.77306581037117,52.91880107773494),(34.77306581037117,53.784726518357985),(34.108131044766516,54.17574726780569),(35.07530888564602,54.59813930694554),(34.25925258240394,54.96417435716029),(35.01486027059106,55.361278263643584),(33.50364489421682,55.37845402950552),(32.7480372060297,55.90721384574556),(35.67979503619571,55.68634475630185),(32.83871012861215,56.311688992608396),(34.591719965206266,56.29492065473883),(35.7100193437232,56.311688992608396),(33.83611227701915,56.695333481003644),(32.95960735872209,56.9434497616887),(36.072711034053015,57.091531913901434),(33.171177511414484,57.33702717078384),(36.193608264162954,57.499784781503735),(33.23162612646945,57.77481561306047),(36.43540272438284,58.04776787540811),(33.62454212432676,58.27099811968307),(36.344729801800376,58.54018474404165),(33.83611227701915,58.68186423448108),(34.74284150284369,59.565911441555244),(33.473420586689336,58.85424941916091)]], [[(34.65216858026123,58.91672306881671),(37.19101041256995,58.68186423448108),(36.01226241899805,58.28688958537609),(37.16078610504247,58.04776787540811),(35.74024365125068,57.79092907387934),(37.009664567405046,57.499784781503735),(35.77046795877817,57.25537683364851),(36.979440259877556,57.07510745541089),(34.22902827487645,56.794777197297435),(36.7074214921302,56.210968525786996),(34.712617195316206,56.10998276812964),(36.55629995449277,55.63519693782703),(35.13575750070099,55.53270067649592),(36.43540272438284,55.34409504165558),(34.83351442542614,55.01619492319591),(35.61934642114075,54.49294870011772),(34.89396304048112,54.12264226523038),(35.37755196092087,53.046178687628185),(37.43280487278982,52.95523300597458),(35.92158949641559,53.80257986695776),(36.91899164482259,53.856094327816805),(36.01226241899805,54.75541714463799),(37.765272255592166,55.189110239786885),(36.828318722240134,55.44708256557195),(38.03729102333953,55.652253637168315),(36.64697287707522,55.941082594334574),(38.21863686850443,56.05939028508024),(36.37495410932787,56.64551287174558),(38.30930979108689,56.992876013526654),(37.16078610504247,57.25537683364851),(38.127963945921984,57.516020773674256),(37.43280487278982,57.710289827306724),(38.33953409861437,57.935626886818994),(37.40258056526235,58.31865112960426),(38.58132855883426,58.744648733419496),(37.31190764267989,59.02578062465136),(34.65216858026123,58.91672306881671)]], [[(38.52087994377928,59.11898412389468),(39.54850639971376,58.713270635642914),(38.369758406141855,58.28688958537609),(38.85334732658162,58.06375936407028),(38.33953409861437,57.710289827306724),(38.73245009647167,57.48354156434209),(38.21863686850443,57.271721400459285),(38.97424455669155,56.87744603722649),(37.463029180317314,56.5623320541159),(38.94402024916407,56.05939028508024),(38.18841256097694,55.856355210835915),(38.490655636251795,55.53270067649592),(37.795496563119656,55.39562234093384),(38.30930979108689,55.154587013355666),(36.7074214921302,54.65063295250911),(37.31190764267989,53.92734063371401),(36.979440259877556,53.58783775557231),(37.855945178174615,52.91880107773497),(39.57873070724124,52.69956490610895),(38.33953409861437,53.281741738901104),(40.00187101262603,53.35396273604752),(39.54850639971376,53.58783775557231),(40.24366547284591,53.58783775557231),(39.97164670509855,53.98069568468355),(40.60635716317572,54.03398248547225),(40.39478701048334,54.44025165268903),(39.54850639971376,54.56310590284329),(39.54850639971376,54.87732350170489),(40.39478701048334,54.87732350170489),(40.39478701048334,55.24083903654295),(39.82052516746112,55.2752875586599),(39.760076552406154,55.75443792473942),(40.57613285564824,55.78844000174894),(40.425011318010824,56.19415599955667),(39.82052516746112,56.07626182891758),(39.79030085993364,56.41214455508424),(40.48545993306579,56.495655446714636),(40.33433839542836,56.95993246553937),(39.79030085993364,56.992876013526654),(39.72985224487867,57.46729112028032),(40.33433839542836,57.46729112028032),(40.24366547284591,58.04776787540811),(39.63917932229622,58.04776787540811),(39.63917932229622,58.382088724871295),(40.33433839542836,58.382088724871295),(40.45523562553831,58.9011152358548),(38.52087994377928,59.11898412389468)]]], [[(24.367675781249993,61.45977057029751),(19.577636718749993,58.67693767258692),(19.577636718749993,57.492213666700735),(19.445800781249996,55.87531083569678),(19.445800781249996,54.085173420886775),(17.468261718749996,53.014783245859235),(20.017089843749993,51.563412328675895),(21.203613281249993,50.205033264943324),(26.125488281249993,50.40151532278236),(27.22412109374999,48.980216985374994),(32.80517578124999,49.525208341974405),(35.26611328124999,48.74894534343292),(36.93603515624999,49.66762782262194),(42.56103515625,48.77791275550183),(43.92333984374999,49.8096315635631),(47.17529296875,49.152969656170455),(49.28466796875,50.54136296522162),(48.05419921875,51.17934297928929),(51.39404296875,52.48278022207825),(50.64697265625,53.014783245859235),(52.88818359375,53.93021986394004),(51.65771484374999,54.29088164657006),(52.66845703125,55.825973254619015),(50.25146484375,56.145549500679095),(51.92138671875,57.914847767009206),(49.15283203125,58.17070248348605),(49.59228515625,60.086762746260064),(47.043457031249986,59.88893689676584),(43.57177734375,61.37567331572748),(42.64892578125,60.630101766266705),(36.89208984374999,62.000904713685856),(36.01318359374999,61.143235250840576),(31.398925781249993,62.02152819100766),(30.563964843749996,61.05828537037917),(26.872558593749993,61.71070595883174),(26.652832031249993,61.10078883158897),(24.367675781249993,61.45977057029751)], [(24.455566406249993,59.42272750081452),(21.203613281249993,58.49369382056807),(21.335449218749993,56.89700392127261),(21.599121093749993,55.92458580482949),(25.202636718749993,55.998380955359636),(28.850097656249993,57.06463027327854),(27.09228515625,57.844750992890994),(28.806152343749996,59.17592824927138),(26.257324218749993,59.17592824927138),(24.455566406249993,59.42272750081452)], [(35.13427734375,59.84481485969107),(31.970214843749993,58.97266715450152),(33.20068359374999,56.776808316568406),(36.67236328125,56.41390137600675),(39.08935546874999,57.25528054528888),(42.69287109374999,58.03137242177638),(40.89111328124999,59.26588062825809),(37.28759765625,58.722598828043374),(37.11181640624999,59.66774058164964),(35.13427734375,59.84481485969107)], [(29.157714843749993,55.75184939173528),(22.565917968749993,55.128649068488784),(22.565917968749993,53.54030739150019),(22.038574218749996,51.48138289610097),(26.257324218749993,51.42661449707484),(30.124511718749993,50.54136296522162),(32.18994140624999,51.17934297928929),(30.124511718749993,53.173119202640635),(35.09033203124999,53.173119202640635),(33.11279296875,54.085173420886775),(29.597167968749993,55.50374985927513),(29.157714843749993,55.75184939173528)], [(42.82470703125,56.58369172128337),(36.584472656249986,55.329144408405085),(37.99072265625,53.592504809039355),(34.95849609374999,51.48138289610097),(36.54052734374999,50.40151532278236),(39.66064453124999,50.289339253291786),(39.79248046875,52.13348804077148),(41.77001953125,50.68079714532166),(44.49462890624999,51.97134580885171),(47.30712890624999,52.509534770327264),(44.05517578125,53.54030739150019),(46.60400390625,53.696706475303245),(47.61474609375,55.40406982700608),(45.37353515625,55.40406982700608),(42.82470703125,56.58369172128337)]])) format TSV; select '-------- Polygon with Polygon with Holes'; select wkt(polygonsSymDifferenceSpherical([[(29.453587685533865,59.779570356240356),(29.393139070478895,52.276266797422124),(40.636581470703206,59.38168915000267),(41.21084331372543,59.103467777099866),(29.786055068336193,52.146627480315004),(31.23682182965546,52.16517054781818),(41.69443223416517,58.85424941916091),(42.51048853740727,58.47703162291134),(32.59691566839227,52.22075341251539),(34.289476889931414,52.22075341251539),(43.02430176537451,58.07974369546071),(43.02430176537451,57.25537683364851),(35.468224883503325,52.2022335126388),(37.16078610504247,52.23926559241349),(43.02430176537451,56.26136189644947),(43.02430176537451,55.326904361850836),(38.33953409861437,52.16517054781818),(40.09254393520848,52.16517054781818),(44.4146199116388,55.3097062225408),(44.47506852669377,59.80998197603594),(39.72985224487867,59.931351417569715),(30.23941968124846,53.67744677450975),(30.20919537372098,54.63314259659509),(38.73245009647167,59.94649146557819),(37.2816833351524,59.97675082987618),(30.23941968124846,55.2752875586599),(30.33009260383092,56.19415599955667),(36.28428118674541,59.96162460231375),(34.863738732953635,59.97675082987618),(30.178971066193498,56.97640788219866),(30.178971066193498,57.91957806959033),(33.65476643185424,59.94649146557819),(32.32489690064491,59.94649146557819),(30.481214141468342,58.85424941916091),(30.571887064050795,59.99187015036608),(29.453587685533865,59.779570356240356)]], [[(24.367675781249993,61.45977057029751),(19.577636718749993,58.67693767258692),(19.577636718749993,57.492213666700735),(19.445800781249996,55.87531083569678),(19.445800781249996,54.085173420886775),(17.468261718749996,53.014783245859235),(20.017089843749993,51.563412328675895),(21.203613281249993,50.205033264943324),(26.125488281249993,50.40151532278236),(27.22412109374999,48.980216985374994),(32.80517578124999,49.525208341974405),(35.26611328124999,48.74894534343292),(36.93603515624999,49.66762782262194),(42.56103515625,48.77791275550183),(43.92333984374999,49.8096315635631),(47.17529296875,49.152969656170455),(49.28466796875,50.54136296522162),(48.05419921875,51.17934297928929),(51.39404296875,52.48278022207825),(50.64697265625,53.014783245859235),(52.88818359375,53.93021986394004),(51.65771484374999,54.29088164657006),(52.66845703125,55.825973254619015),(50.25146484375,56.145549500679095),(51.92138671875,57.914847767009206),(49.15283203125,58.17070248348605),(49.59228515625,60.086762746260064),(47.043457031249986,59.88893689676584),(43.57177734375,61.37567331572748),(42.64892578125,60.630101766266705),(36.89208984374999,62.000904713685856),(36.01318359374999,61.143235250840576),(31.398925781249993,62.02152819100766),(30.563964843749996,61.05828537037917),(26.872558593749993,61.71070595883174),(26.652832031249993,61.10078883158897),(24.367675781249993,61.45977057029751)], [(24.455566406249993,59.42272750081452),(21.203613281249993,58.49369382056807),(21.335449218749993,56.89700392127261),(21.599121093749993,55.92458580482949),(25.202636718749993,55.998380955359636),(28.850097656249993,57.06463027327854),(27.09228515625,57.844750992890994),(28.806152343749996,59.17592824927138),(26.257324218749993,59.17592824927138),(24.455566406249993,59.42272750081452)], [(35.13427734375,59.84481485969107),(31.970214843749993,58.97266715450152),(33.20068359374999,56.776808316568406),(36.67236328125,56.41390137600675),(39.08935546874999,57.25528054528888),(42.69287109374999,58.03137242177638),(40.89111328124999,59.26588062825809),(37.28759765625,58.722598828043374),(37.11181640624999,59.66774058164964),(35.13427734375,59.84481485969107)], [(29.157714843749993,55.75184939173528),(22.565917968749993,55.128649068488784),(22.565917968749993,53.54030739150019),(22.038574218749996,51.48138289610097),(26.257324218749993,51.42661449707484),(30.124511718749993,50.54136296522162),(32.18994140624999,51.17934297928929),(30.124511718749993,53.173119202640635),(35.09033203124999,53.173119202640635),(33.11279296875,54.085173420886775),(29.597167968749993,55.50374985927513),(29.157714843749993,55.75184939173528)], [(42.82470703125,56.58369172128337),(36.584472656249986,55.329144408405085),(37.99072265625,53.592504809039355),(34.95849609374999,51.48138289610097),(36.54052734374999,50.40151532278236),(39.66064453124999,50.289339253291786),(39.79248046875,52.13348804077148),(41.77001953125,50.68079714532166),(44.49462890624999,51.97134580885171),(47.30712890624999,52.509534770327264),(44.05517578125,53.54030739150019),(46.60400390625,53.696706475303245),(47.61474609375,55.40406982700608),(45.37353515625,55.40406982700608),(42.82470703125,56.58369172128337)]])) format TSV; -select wkt(polygonsSymDifferenceMercator([[(29.453587685533865,59.779570356240356),(29.393139070478895,52.276266797422124),(40.636581470703206,59.38168915000267),(41.21084331372543,59.103467777099866),(29.786055068336193,52.146627480315004),(31.23682182965546,52.16517054781818),(41.69443223416517,58.85424941916091),(42.51048853740727,58.47703162291134),(32.59691566839227,52.22075341251539),(34.289476889931414,52.22075341251539),(43.02430176537451,58.07974369546071),(43.02430176537451,57.25537683364851),(35.468224883503325,52.2022335126388),(37.16078610504247,52.23926559241349),(43.02430176537451,56.26136189644947),(43.02430176537451,55.326904361850836),(38.33953409861437,52.16517054781818),(40.09254393520848,52.16517054781818),(44.4146199116388,55.3097062225408),(44.47506852669377,59.80998197603594),(39.72985224487867,59.931351417569715),(30.23941968124846,53.67744677450975),(30.20919537372098,54.63314259659509),(38.73245009647167,59.94649146557819),(37.2816833351524,59.97675082987618),(30.23941968124846,55.2752875586599),(30.33009260383092,56.19415599955667),(36.28428118674541,59.96162460231375),(34.863738732953635,59.97675082987618),(30.178971066193498,56.97640788219866),(30.178971066193498,57.91957806959033),(33.65476643185424,59.94649146557819),(32.32489690064491,59.94649146557819),(30.481214141468342,58.85424941916091),(30.571887064050795,59.99187015036608),(29.453587685533865,59.779570356240356)]], [[(24.367675781249993,61.45977057029751),(19.577636718749993,58.67693767258692),(19.577636718749993,57.492213666700735),(19.445800781249996,55.87531083569678),(19.445800781249996,54.085173420886775),(17.468261718749996,53.014783245859235),(20.017089843749993,51.563412328675895),(21.203613281249993,50.205033264943324),(26.125488281249993,50.40151532278236),(27.22412109374999,48.980216985374994),(32.80517578124999,49.525208341974405),(35.26611328124999,48.74894534343292),(36.93603515624999,49.66762782262194),(42.56103515625,48.77791275550183),(43.92333984374999,49.8096315635631),(47.17529296875,49.152969656170455),(49.28466796875,50.54136296522162),(48.05419921875,51.17934297928929),(51.39404296875,52.48278022207825),(50.64697265625,53.014783245859235),(52.88818359375,53.93021986394004),(51.65771484374999,54.29088164657006),(52.66845703125,55.825973254619015),(50.25146484375,56.145549500679095),(51.92138671875,57.914847767009206),(49.15283203125,58.17070248348605),(49.59228515625,60.086762746260064),(47.043457031249986,59.88893689676584),(43.57177734375,61.37567331572748),(42.64892578125,60.630101766266705),(36.89208984374999,62.000904713685856),(36.01318359374999,61.143235250840576),(31.398925781249993,62.02152819100766),(30.563964843749996,61.05828537037917),(26.872558593749993,61.71070595883174),(26.652832031249993,61.10078883158897),(24.367675781249993,61.45977057029751)], [(24.455566406249993,59.42272750081452),(21.203613281249993,58.49369382056807),(21.335449218749993,56.89700392127261),(21.599121093749993,55.92458580482949),(25.202636718749993,55.998380955359636),(28.850097656249993,57.06463027327854),(27.09228515625,57.844750992890994),(28.806152343749996,59.17592824927138),(26.257324218749993,59.17592824927138),(24.455566406249993,59.42272750081452)], [(35.13427734375,59.84481485969107),(31.970214843749993,58.97266715450152),(33.20068359374999,56.776808316568406),(36.67236328125,56.41390137600675),(39.08935546874999,57.25528054528888),(42.69287109374999,58.03137242177638),(40.89111328124999,59.26588062825809),(37.28759765625,58.722598828043374),(37.11181640624999,59.66774058164964),(35.13427734375,59.84481485969107)], [(29.157714843749993,55.75184939173528),(22.565917968749993,55.128649068488784),(22.565917968749993,53.54030739150019),(22.038574218749996,51.48138289610097),(26.257324218749993,51.42661449707484),(30.124511718749993,50.54136296522162),(32.18994140624999,51.17934297928929),(30.124511718749993,53.173119202640635),(35.09033203124999,53.173119202640635),(33.11279296875,54.085173420886775),(29.597167968749993,55.50374985927513),(29.157714843749993,55.75184939173528)], [(42.82470703125,56.58369172128337),(36.584472656249986,55.329144408405085),(37.99072265625,53.592504809039355),(34.95849609374999,51.48138289610097),(36.54052734374999,50.40151532278236),(39.66064453124999,50.289339253291786),(39.79248046875,52.13348804077148),(41.77001953125,50.68079714532166),(44.49462890624999,51.97134580885171),(47.30712890624999,52.509534770327264),(44.05517578125,53.54030739150019),(46.60400390625,53.696706475303245),(47.61474609375,55.40406982700608),(45.37353515625,55.40406982700608),(42.82470703125,56.58369172128337)]])) format TSV; From eecc12ff6a4efe0342b6465738501b0d4c99131a Mon Sep 17 00:00:00 2001 From: Nikita Mikhaylov Date: Tue, 9 Mar 2021 14:59:27 +0300 Subject: [PATCH 318/544] suppress warnings and skip tests in arcadia --- src/Functions/CMakeLists.txt | 3 --- src/Functions/pointInPolygon.cpp | 7 ------- tests/queries/0_stateless/01720_country_intersection.sh | 4 ++-- tests/queries/0_stateless/arcadia_skip_list.txt | 5 ++++- tests/ubsan_suppressions.txt | 5 ++++- 5 files changed, 10 insertions(+), 14 deletions(-) diff --git a/src/Functions/CMakeLists.txt b/src/Functions/CMakeLists.txt index 1c3beb2e47d..321aa5e2196 100644 --- a/src/Functions/CMakeLists.txt +++ b/src/Functions/CMakeLists.txt @@ -117,6 +117,3 @@ target_link_libraries(clickhouse_functions PRIVATE clickhouse_functions_array) if (USE_STATS) target_link_libraries(clickhouse_functions PRIVATE stats) endif() - -# Signed integer overflow on user-provided data inside boost::geometry - ignore. -set_source_files_properties("pointInPolygon.cpp" PROPERTIES COMPILE_FLAGS -fno-sanitize=signed-integer-overflow) diff --git a/src/Functions/pointInPolygon.cpp b/src/Functions/pointInPolygon.cpp index 8e4a05bdec5..60904038b61 100644 --- a/src/Functions/pointInPolygon.cpp +++ b/src/Functions/pointInPolygon.cpp @@ -426,14 +426,7 @@ private: { out_container.reserve(end - begin); for (size_t i = begin; i < end; ++i) - { - Int64 result = 0; - if (common::mulOverflow(static_cast(x_data[i]), static_cast(y_data[i]), result)) - throw Exception("The coordinates of the point are such that subsequent calculations cannot be performed correctly. " \ - "Most likely they are very large in modulus.", ErrorCodes::BAD_ARGUMENTS); - out_container.emplace_back(x_data[i], y_data[i]); - } } void parseConstPolygonWithoutHolesFromSingleColumn(const IColumn & column, size_t i, Polygon & out_polygon) const diff --git a/tests/queries/0_stateless/01720_country_intersection.sh b/tests/queries/0_stateless/01720_country_intersection.sh index 77418ef0582..d7e0e67d351 100755 --- a/tests/queries/0_stateless/01720_country_intersection.sh +++ b/tests/queries/0_stateless/01720_country_intersection.sh @@ -7,12 +7,12 @@ CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) ${CLICKHOUSE_CLIENT} -q "drop table if exists country_polygons;" ${CLICKHOUSE_CLIENT} -q "create table country_polygons(name String, p Array(Array(Tuple(Float64, Float64)))) engine=MergeTree() order by tuple();" cat ${CURDIR}/country_polygons.tsv | ${CLICKHOUSE_CLIENT} -q "insert into country_polygons format TSV" -${CLICKHOUSE_CLIENT} -q "SELECT c, d, polygonsIntersectionSpherical(a, b) FROM (SELECT first.p AS a, second.p AS b, first.name AS c, second.name AS d FROM country_polygons AS first CROSS JOIN country_polygons AS second LIMIT 100) format TSV" +${CLICKHOUSE_CLIENT} -q "SELECT c, d, polygonsIntersectionSpherical(a, b) FROM (SELECT first.p AS a, second.p AS b, first.name AS c, second.name AS d FROM country_polygons AS first CROSS JOIN country_polygons AS second LIMIT 100) format TSV" ${CLICKHOUSE_CLIENT} -q "drop table if exists country_polygons;" ${CLICKHOUSE_CLIENT} -q "drop table if exists country_rings;" ${CLICKHOUSE_CLIENT} -q "create table country_rings(name String, p Array(Tuple(Float64, Float64))) engine=MergeTree() order by tuple();" cat ${CURDIR}/country_rings.tsv | ${CLICKHOUSE_CLIENT} -q "insert into country_rings format TSV" -${CLICKHOUSE_CLIENT} -q "SELECT c, d, polygonsIntersectionSpherical(a, b) FROM (SELECT first.p AS a, second.p AS b, first.name AS c, second.name AS d FROM country_rings AS first CROSS JOIN country_rings AS second LIMIT 100) format TSV" +${CLICKHOUSE_CLIENT} -q "SELECT c, d, polygonsIntersectionSpherical(a, b) FROM (SELECT first.p AS a, second.p AS b, first.name AS c, second.name AS d FROM country_rings AS first CROSS JOIN country_rings AS second LIMIT 100) format TSV" ${CLICKHOUSE_CLIENT} -q "drop table if exists country_rings;" \ No newline at end of file diff --git a/tests/queries/0_stateless/arcadia_skip_list.txt b/tests/queries/0_stateless/arcadia_skip_list.txt index 6926f16e027..f28eac03c37 100644 --- a/tests/queries/0_stateless/arcadia_skip_list.txt +++ b/tests/queries/0_stateless/arcadia_skip_list.txt @@ -212,9 +212,12 @@ 01017_uniqCombined_memory_usage 01747_join_view_filter_dictionary 01748_dictionary_table_dot -00950_dict_get, +00950_dict_get 01683_flat_dictionary 01681_cache_dictionary_simple_key 01682_cache_dictionary_complex_key 01684_ssd_cache_dictionary_simple_key 01685_ssd_cache_dictionary_complex_key +01304_polygons_sym_difference +01305_polygons_union +01306_polygons_intersection diff --git a/tests/ubsan_suppressions.txt b/tests/ubsan_suppressions.txt index 6a55155e330..b92a2d9730e 100644 --- a/tests/ubsan_suppressions.txt +++ b/tests/ubsan_suppressions.txt @@ -1 +1,4 @@ -# We have no suppressions! +# https://github.com/llvm-mirror/compiler-rt/blob/master/lib/ubsan/ubsan_checks.inc + +# Some value is outside the range of representable values of type 'long' on user-provided data inside boost::geometry - ignore. +src:*/Functions/pointInPolygon.cpp \ No newline at end of file From 934b5bc48780290f65a8a5487ce219ee652f3bbb Mon Sep 17 00:00:00 2001 From: Nikita Mikhaylov Date: Tue, 9 Mar 2021 15:51:11 +0300 Subject: [PATCH 319/544] Update 01700_point_in_polygon_ubsan.sql --- tests/queries/0_stateless/01700_point_in_polygon_ubsan.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/queries/0_stateless/01700_point_in_polygon_ubsan.sql b/tests/queries/0_stateless/01700_point_in_polygon_ubsan.sql index 3b6ad3b257e..97db40ab65e 100644 --- a/tests/queries/0_stateless/01700_point_in_polygon_ubsan.sql +++ b/tests/queries/0_stateless/01700_point_in_polygon_ubsan.sql @@ -1 +1 @@ -SELECT pointInPolygon((0, 0), [[(0, 0), (10, 10), (256, -9223372036854775808)]]) FORMAT Null; -- { serverError 36 } +SELECT pointInPolygon((0, 0), [[(0, 0), (10, 10), (256, -9223372036854775808)]]) FORMAT Null; From b219b7626e2f7397bc66341f04fe5f53be9af225 Mon Sep 17 00:00:00 2001 From: Nikita Mikhaylov Date: Thu, 11 Mar 2021 15:19:15 +0300 Subject: [PATCH 320/544] better --- src/Functions/pointInPolygon.cpp | 8 +++++++- .../queries/0_stateless/01700_point_in_polygon_ubsan.sql | 2 +- 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/src/Functions/pointInPolygon.cpp b/src/Functions/pointInPolygon.cpp index 60904038b61..63ea3088039 100644 --- a/src/Functions/pointInPolygon.cpp +++ b/src/Functions/pointInPolygon.cpp @@ -425,8 +425,14 @@ private: T & out_container) const { out_container.reserve(end - begin); - for (size_t i = begin; i < end; ++i) + for (size_t i = begin; i < end; ++i) { + Int64 result = 0; + if (common::mulOverflow(static_cast(x_data[i]), static_cast(y_data[i]), result)) + throw Exception("The coordinates of the point are such that subsequent calculations cannot be performed correctly. " \ + "Most likely they are very large in modulus.", ErrorCodes::BAD_ARGUMENTS); + out_container.emplace_back(x_data[i], y_data[i]); + } } void parseConstPolygonWithoutHolesFromSingleColumn(const IColumn & column, size_t i, Polygon & out_polygon) const diff --git a/tests/queries/0_stateless/01700_point_in_polygon_ubsan.sql b/tests/queries/0_stateless/01700_point_in_polygon_ubsan.sql index 97db40ab65e..72317df5439 100644 --- a/tests/queries/0_stateless/01700_point_in_polygon_ubsan.sql +++ b/tests/queries/0_stateless/01700_point_in_polygon_ubsan.sql @@ -1 +1 @@ -SELECT pointInPolygon((0, 0), [[(0, 0), (10, 10), (256, -9223372036854775808)]]) FORMAT Null; +SELECT pointInPolygon((0, 0), [[(0, 0), (10, 10), (256, -9223372036854775808)]]) FORMAT Null ;-- { serverError 36 } From 1194d50e48b4d57956967b16c6bb37274c0ffb56 Mon Sep 17 00:00:00 2001 From: vdimir Date: Thu, 11 Mar 2021 15:16:52 +0300 Subject: [PATCH 321/544] Try to fix test_storage_s3: crash in WriteBufferFromS3 --- src/IO/WriteBufferFromS3.cpp | 9 ++++++++- src/Storages/StorageS3.cpp | 5 +++++ tests/integration/test_storage_s3/test.py | 4 ++-- 3 files changed, 15 insertions(+), 3 deletions(-) diff --git a/src/IO/WriteBufferFromS3.cpp b/src/IO/WriteBufferFromS3.cpp index a6ec60b295f..5edf01a940e 100644 --- a/src/IO/WriteBufferFromS3.cpp +++ b/src/IO/WriteBufferFromS3.cpp @@ -107,7 +107,14 @@ void WriteBufferFromS3::finalizeImpl() WriteBufferFromS3::~WriteBufferFromS3() { - finalizeImpl(); + try + { + finalizeImpl(); + } + catch (...) + { + tryLogCurrentException(__PRETTY_FUNCTION__); + } } void WriteBufferFromS3::createMultipartUpload() diff --git a/src/Storages/StorageS3.cpp b/src/Storages/StorageS3.cpp index a31a7fa0944..e50eb1b4249 100644 --- a/src/Storages/StorageS3.cpp +++ b/src/Storages/StorageS3.cpp @@ -173,6 +173,11 @@ namespace writer->writePrefix(); } + void flush() override + { + writer->flush(); + } + void writeSuffix() override { writer->writeSuffix(); diff --git a/tests/integration/test_storage_s3/test.py b/tests/integration/test_storage_s3/test.py index 3b4c56b524b..8baa1cd64b0 100644 --- a/tests/integration/test_storage_s3/test.py +++ b/tests/integration/test_storage_s3/test.py @@ -281,9 +281,9 @@ def test_put_get_with_globs(cluster): # Test multipart put. @pytest.mark.parametrize("maybe_auth,positive", [ - ("", True) + ("", True), # ("'minio','minio123',",True), Redirect with credentials not working with nginx. - # ("'wrongid','wrongkey',", False) ClickHouse crashes in some time after this test, local integration tests run fails. + ("'wrongid','wrongkey',", False), ]) def test_multipart_put(cluster, maybe_auth, positive): # type: (ClickHouseCluster) -> None From 03bff5c3b5cce70fd5e42603aa8fc1d0a3bc577d Mon Sep 17 00:00:00 2001 From: Vasily Nemkov Date: Thu, 11 Mar 2021 14:43:05 +0200 Subject: [PATCH 322/544] Fixed test flakkiness --- .../01702_system_query_log.reference | 128 +++++++++--------- .../0_stateless/01702_system_query_log.sql | 27 ++-- 2 files changed, 77 insertions(+), 78 deletions(-) diff --git a/tests/queries/0_stateless/01702_system_query_log.reference b/tests/queries/0_stateless/01702_system_query_log.reference index aba0152fcb5..6d8908249bf 100644 --- a/tests/queries/0_stateless/01702_system_query_log.reference +++ b/tests/queries/0_stateless/01702_system_query_log.reference @@ -8,85 +8,85 @@ GRANT queries REVOKE queries Misc queries ACTUAL LOG CONTENT: -Select -- SET ROLE sqllt_role; -- tests are executed by user `default` which is defined in XML and is impossible to update.\n\nSELECT \'ALTER TABLE queries\'; -Select -- SYSTEM RELOAD DICTIONARY sqllt.dictionary; -- temporary out of order: Code: 210, Connection refused (localhost:9001) (version 21.3.1.1)\n-- DROP REPLICA\n-- haha, no\n-- SYSTEM KILL;\n-- SYSTEM SHUTDOWN;\n\n-- Since we don\'t really care about the actual output, suppress it with `FORMAT Null`.\nSELECT \'SHOW queries\'; -Select -- not done, seems to hard, so I\'ve skipped queries of ALTER-X, where X is:\n-- PARTITION\n-- ORDER BY\n-- SAMPLE BY\n-- INDEX\n-- CONSTRAINT\n-- TTL\n-- USER\n-- QUOTA\n-- ROLE\n-- ROW POLICY\n-- SETTINGS PROFILE\n\nSELECT \'SYSTEM queries\'; -Alter ALTER TABLE table ADD COLUMN new_col UInt32 DEFAULT 123456789; -Alter ALTER TABLE table CLEAR COLUMN new_col; -Alter ALTER TABLE table COMMENT COLUMN new_col \'dummy column with a comment\'; -Alter ALTER TABLE table DELETE WHERE i > 65535; -Alter ALTER TABLE table DROP COLUMN the_new_col; -Alter ALTER TABLE table MODIFY COLUMN new_col DateTime DEFAULT \'2015-05-18 07:40:13\'; -Alter ALTER TABLE table MODIFY COLUMN new_col REMOVE COMMENT; -Alter ALTER TABLE table RENAME COLUMN new_col TO the_new_col; -Alter ALTER TABLE table UPDATE i = i + 1 WHERE 1; -Create ATTACH TABLE sqllt.table; - CHECK TABLE sqllt.table FORMAT Null; +Select SELECT \'DROP queries and also a cleanup before the test\'; +Drop DROP DATABASE IF EXISTS sqllt SYNC; + DROP USER IF EXISTS sqllt_user; + DROP ROLE IF EXISTS sqllt_role; + DROP POLICY IF EXISTS sqllt_policy ON sqllt.table, sqllt.view, sqllt.dictionary; + DROP ROW POLICY IF EXISTS sqllt_row_policy ON sqllt.table, sqllt.view, sqllt.dictionary; + DROP QUOTA IF EXISTS sqllt_quota; + DROP SETTINGS PROFILE IF EXISTS sqllt_settings_profile; +Select SELECT \'CREATE queries\'; Create CREATE DATABASE sqllt; +Create CREATE TABLE sqllt.table\n(\n i UInt8, s String\n)\nENGINE = MergeTree PARTITION BY tuple() ORDER BY tuple(); +Create CREATE VIEW sqllt.view AS SELECT i, s FROM sqllt.table; Create CREATE DICTIONARY sqllt.dictionary (key UInt64, value UInt64) PRIMARY KEY key SOURCE(CLICKHOUSE(DB \'sqllt\' TABLE \'table\' HOST \'localhost\' PORT 9001)) LIFETIME(0) LAYOUT(FLAT()); + CREATE USER sqllt_user IDENTIFIED WITH PLAINTEXT_PASSWORD BY \'password\'; + CREATE ROLE sqllt_role; CREATE POLICY sqllt_policy ON sqllt.table, sqllt.view, sqllt.dictionary AS PERMISSIVE TO ALL; CREATE POLICY sqllt_row_policy ON sqllt.table, sqllt.view, sqllt.dictionary AS PERMISSIVE TO ALL; CREATE QUOTA sqllt_quota KEYED BY user_name TO sqllt_role; - CREATE ROLE sqllt_role; CREATE SETTINGS PROFILE sqllt_settings_profile SETTINGS interactive_delay = 200000; -Create CREATE TABLE sqllt.table\n(\n i UInt8, s String\n)\nENGINE = MergeTree PARTITION BY tuple() ORDER BY tuple(); - CREATE USER sqllt_user IDENTIFIED WITH PLAINTEXT_PASSWORD BY \'password\'; -Create CREATE VIEW sqllt.view AS SELECT i, s FROM sqllt.table; - DESCRIBE TABLE sqllt.table FORMAT Null; -Drop DETACH TABLE sqllt.table; -Drop DROP DATABASE IF EXISTS sqllt SYNC; - DROP POLICY IF EXISTS sqllt_policy ON sqllt.table, sqllt.view, sqllt.dictionary; - DROP QUOTA IF EXISTS sqllt_quota; - DROP ROLE IF EXISTS sqllt_role; - DROP ROW POLICY IF EXISTS sqllt_row_policy ON sqllt.table, sqllt.view, sqllt.dictionary; - DROP SETTINGS PROFILE IF EXISTS sqllt_settings_profile; -Drop DROP TABLE sqllt.table SYNC; - DROP USER IF EXISTS sqllt_user; -Grant GRANT DROP ON sqllt.view TO sqllt_user; -Grant GRANT SELECT ON sqllt.table TO sqllt_user; Grant GRANT sqllt_role TO sqllt_user; -Rename RENAME TABLE sqllt.table TO sqllt.table_new; -Rename RENAME TABLE sqllt.table_new TO sqllt.table; -Revoke REVOKE DROP ON sqllt.view FROM sqllt_user; -Revoke REVOKE SELECT ON sqllt.table FROM sqllt_user; -Select SELECT \'CREATE queries\'; -Select SELECT \'DROP queries and also a cleanup before the test\'; -Select SELECT \'GRANT queries\'; -Select SELECT \'Misc queries\'; -Select SELECT \'REVOKE queries\'; Select SELECT \'SET queries\'; - SET DEFAULT ROLE sqllt_role TO sqllt_user; SET log_profile_events=false; - SHOW CREATE DICTIONARY sqllt.dictionary FORMAT Null; - SHOW CREATE POLICY sqllt_policy FORMAT Null; - SHOW CREATE QUOTA sqllt_quota FORMAT Null; - SHOW CREATE ROLE sqllt_role FORMAT Null; - SHOW CREATE ROW POLICY sqllt_row_policy FORMAT Null; - SHOW CREATE SETTINGS PROFILE sqllt_settings_profile FORMAT Null; - SHOW CREATE TABLE sqllt.table FORMAT Null; - SHOW CREATE USER sqllt_user FORMAT Null; - SHOW DATABASES LIKE \'sqllt\' FORMAT Null; - SHOW DICTIONARIES FROM sqllt FORMAT Null; - SHOW GRANTS FOR sqllt_user FORMAT Null; - SHOW GRANTS FORMAT Null; - SHOW TABLES FROM sqllt FORMAT Null; + SET DEFAULT ROLE sqllt_role TO sqllt_user; +Select -- SET ROLE sqllt_role; -- tests are executed by user `default` which is defined in XML and is impossible to update.\n\nSELECT \'ALTER TABLE queries\'; +Alter ALTER TABLE sqllt.table ADD COLUMN new_col UInt32 DEFAULT 123456789; +Alter ALTER TABLE sqllt.table COMMENT COLUMN new_col \'dummy column with a comment\'; +Alter ALTER TABLE sqllt.table CLEAR COLUMN new_col; +Alter ALTER TABLE sqllt.table MODIFY COLUMN new_col DateTime DEFAULT \'2015-05-18 07:40:13\'; +Alter ALTER TABLE sqllt.table MODIFY COLUMN new_col REMOVE COMMENT; +Alter ALTER TABLE sqllt.table RENAME COLUMN new_col TO the_new_col; +Alter ALTER TABLE sqllt.table DROP COLUMN the_new_col; +Alter ALTER TABLE sqllt.table UPDATE i = i + 1 WHERE 1; +Alter ALTER TABLE sqllt.table DELETE WHERE i > 65535; +Select -- not done, seems to hard, so I\'ve skipped queries of ALTER-X, where X is:\n-- PARTITION\n-- ORDER BY\n-- SAMPLE BY\n-- INDEX\n-- CONSTRAINT\n-- TTL\n-- USER\n-- QUOTA\n-- ROLE\n-- ROW POLICY\n-- SETTINGS PROFILE\n\nSELECT \'SYSTEM queries\'; +System SYSTEM RELOAD EMBEDDED DICTIONARIES; +System SYSTEM RELOAD DICTIONARIES; System SYSTEM DROP DNS CACHE; System SYSTEM DROP MARK CACHE; System SYSTEM DROP UNCOMPRESSED CACHE; System SYSTEM FLUSH LOGS; System SYSTEM RELOAD CONFIG; -System SYSTEM RELOAD DICTIONARIES; -System SYSTEM RELOAD EMBEDDED DICTIONARIES; -System SYSTEM START FETCHES; -System SYSTEM START MERGES; -System SYSTEM START MOVES; -System SYSTEM START REPLICATED SENDS; -System SYSTEM START TTL MERGES; -System SYSTEM STOP FETCHES; System SYSTEM STOP MERGES; -System SYSTEM STOP MOVES; -System SYSTEM STOP REPLICATED SENDS; +System SYSTEM START MERGES; System SYSTEM STOP TTL MERGES; +System SYSTEM START TTL MERGES; +System SYSTEM STOP MOVES; +System SYSTEM START MOVES; +System SYSTEM STOP FETCHES; +System SYSTEM START FETCHES; +System SYSTEM STOP REPLICATED SENDS; +System SYSTEM START REPLICATED SENDS; +Select -- SYSTEM RELOAD DICTIONARY sqllt.dictionary; -- temporary out of order: Code: 210, Connection refused (localhost:9001) (version 21.3.1.1)\n-- DROP REPLICA\n-- haha, no\n-- SYSTEM KILL;\n-- SYSTEM SHUTDOWN;\n\n-- Since we don\'t really care about the actual output, suppress it with `FORMAT Null`.\nSELECT \'SHOW queries\'; + SHOW CREATE TABLE sqllt.table FORMAT Null; + SHOW CREATE DICTIONARY sqllt.dictionary FORMAT Null; + SHOW DATABASES LIKE \'sqllt\' FORMAT Null; + SHOW TABLES FROM sqllt FORMAT Null; + SHOW DICTIONARIES FROM sqllt FORMAT Null; + SHOW GRANTS FORMAT Null; + SHOW GRANTS FOR sqllt_user FORMAT Null; + SHOW CREATE USER sqllt_user FORMAT Null; + SHOW CREATE ROLE sqllt_role FORMAT Null; + SHOW CREATE POLICY sqllt_policy FORMAT Null; + SHOW CREATE ROW POLICY sqllt_row_policy FORMAT Null; + SHOW CREATE QUOTA sqllt_quota FORMAT Null; + SHOW CREATE SETTINGS PROFILE sqllt_settings_profile FORMAT Null; +Select SELECT \'GRANT queries\'; +Grant GRANT SELECT ON sqllt.table TO sqllt_user; +Grant GRANT DROP ON sqllt.view TO sqllt_user; +Select SELECT \'REVOKE queries\'; +Revoke REVOKE SELECT ON sqllt.table FROM sqllt_user; +Revoke REVOKE DROP ON sqllt.view FROM sqllt_user; +Select SELECT \'Misc queries\'; + DESCRIBE TABLE sqllt.table FORMAT Null; + CHECK TABLE sqllt.table FORMAT Null; +Drop DETACH TABLE sqllt.table; +Create ATTACH TABLE sqllt.table; +Rename RENAME TABLE sqllt.table TO sqllt.table_new; +Rename RENAME TABLE sqllt.table_new TO sqllt.table; Drop TRUNCATE TABLE sqllt.table; - USE sqllt +Drop DROP TABLE sqllt.table SYNC; + SET log_comment=\'\'; DROP queries and also a cleanup after the test diff --git a/tests/queries/0_stateless/01702_system_query_log.sql b/tests/queries/0_stateless/01702_system_query_log.sql index 979705c2739..f31d8de3577 100644 --- a/tests/queries/0_stateless/01702_system_query_log.sql +++ b/tests/queries/0_stateless/01702_system_query_log.sql @@ -12,7 +12,6 @@ DROP SETTINGS PROFILE IF EXISTS sqllt_settings_profile; SELECT 'CREATE queries'; CREATE DATABASE sqllt; -USE sqllt; CREATE TABLE sqllt.table ( @@ -41,15 +40,15 @@ SET DEFAULT ROLE sqllt_role TO sqllt_user; -- SET ROLE sqllt_role; -- tests are executed by user `default` which is defined in XML and is impossible to update. SELECT 'ALTER TABLE queries'; -ALTER TABLE table ADD COLUMN new_col UInt32 DEFAULT 123456789; -ALTER TABLE table COMMENT COLUMN new_col 'dummy column with a comment'; -ALTER TABLE table CLEAR COLUMN new_col; -ALTER TABLE table MODIFY COLUMN new_col DateTime DEFAULT '2015-05-18 07:40:13'; -ALTER TABLE table MODIFY COLUMN new_col REMOVE COMMENT; -ALTER TABLE table RENAME COLUMN new_col TO the_new_col; -ALTER TABLE table DROP COLUMN the_new_col; -ALTER TABLE table UPDATE i = i + 1 WHERE 1; -ALTER TABLE table DELETE WHERE i > 65535; +ALTER TABLE sqllt.table ADD COLUMN new_col UInt32 DEFAULT 123456789; +ALTER TABLE sqllt.table COMMENT COLUMN new_col 'dummy column with a comment'; +ALTER TABLE sqllt.table CLEAR COLUMN new_col; +ALTER TABLE sqllt.table MODIFY COLUMN new_col DateTime DEFAULT '2015-05-18 07:40:13'; +ALTER TABLE sqllt.table MODIFY COLUMN new_col REMOVE COMMENT; +ALTER TABLE sqllt.table RENAME COLUMN new_col TO the_new_col; +ALTER TABLE sqllt.table DROP COLUMN the_new_col; +ALTER TABLE sqllt.table UPDATE i = i + 1 WHERE 1; +ALTER TABLE sqllt.table DELETE WHERE i > 65535; -- not done, seems to hard, so I've skipped queries of ALTER-X, where X is: -- PARTITION @@ -126,6 +125,7 @@ RENAME TABLE sqllt.table_new TO sqllt.table; TRUNCATE TABLE sqllt.table; DROP TABLE sqllt.table SYNC; +SET log_comment=''; --------------------------------------------------------------------------------------------------- -- Now get all logs related to this test --------------------------------------------------------------------------------------------------- @@ -136,10 +136,9 @@ SELECT 'ACTUAL LOG CONTENT:'; -- Try to filter out all possible previous junk events by excluding old log entries, SELECT query_kind, query FROM system.query_log WHERE - log_comment LIKE '%system.query_log%' AND type == 'QueryStart' AND query_start_time >= now() - 5 - -- this one is to make stylecheck happy and to validate that CREATE\DROP queries of non-db-bound-objects, like USER\ROLS are logged properly. - AND (current_database == currentDatabase() OR current_database != currentDatabase()) -ORDER BY query; + log_comment LIKE '%system.query_log%' AND type == 'QueryStart' AND event_time >= now() - 10 + AND current_database == currentDatabase() +ORDER BY event_time_microseconds; -- cleanup From 083299e549d94fb8cda68a52a36c3a85a06f5033 Mon Sep 17 00:00:00 2001 From: feng lv Date: Thu, 11 Mar 2021 13:54:11 +0000 Subject: [PATCH 323/544] redefine some errorcode --- src/Common/ErrorCodes.cpp | 4 ++-- tests/queries/0_stateless/01684_insert_specify_shard_id.sql | 4 ++-- tests/queries/0_stateless/01753_fix_clickhouse_format.sh | 2 +- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/src/Common/ErrorCodes.cpp b/src/Common/ErrorCodes.cpp index 2e8145144ce..5f8ea4de89c 100644 --- a/src/Common/ErrorCodes.cpp +++ b/src/Common/ErrorCodes.cpp @@ -542,13 +542,13 @@ M(573, EPOLL_ERROR) \ M(574, DISTRIBUTED_TOO_MANY_PENDING_BYTES) \ M(575, UNKNOWN_SNAPSHOT) \ + M(576, INVALID_SHARD_ID) \ + M(577, INVALID_FORMAT_INSERT_QUERY_WITH_DATA) \ \ M(999, KEEPER_EXCEPTION) \ M(1000, POCO_EXCEPTION) \ M(1001, STD_EXCEPTION) \ M(1002, UNKNOWN_EXCEPTION) \ - M(1003, INVALID_SHARD_ID) \ - M(1004, INVALID_FORMAT_INSERT_QUERY_WITH_DATA) /* See END */ diff --git a/tests/queries/0_stateless/01684_insert_specify_shard_id.sql b/tests/queries/0_stateless/01684_insert_specify_shard_id.sql index ce1c7807b59..c51581c7e81 100644 --- a/tests/queries/0_stateless/01684_insert_specify_shard_id.sql +++ b/tests/queries/0_stateless/01684_insert_specify_shard_id.sql @@ -28,8 +28,8 @@ INSERT INTO x_dist SELECT * FROM numbers(10); -- { serverError 55 } INSERT INTO y_dist SELECT * FROM numbers(10); -- { serverError 55 } -- invalid shard id -INSERT INTO x_dist SELECT * FROM numbers(10) settings insert_shard_id = 3; -- { serverError 1003 } -INSERT INTO y_dist SELECT * FROM numbers(10) settings insert_shard_id = 3; -- { serverError 1003 } +INSERT INTO x_dist SELECT * FROM numbers(10) settings insert_shard_id = 3; -- { serverError 576 } +INSERT INTO y_dist SELECT * FROM numbers(10) settings insert_shard_id = 3; -- { serverError 576 } DROP TABLE x; DROP TABLE x_dist; diff --git a/tests/queries/0_stateless/01753_fix_clickhouse_format.sh b/tests/queries/0_stateless/01753_fix_clickhouse_format.sh index 48ce8ded1ad..8bc4e8821fe 100755 --- a/tests/queries/0_stateless/01753_fix_clickhouse_format.sh +++ b/tests/queries/0_stateless/01753_fix_clickhouse_format.sh @@ -8,4 +8,4 @@ echo "select 1; select 1 union all (select 1 union distinct select 1); " | $CL echo "select 1; select 1 union all (select 1 union distinct select 1); -- comment " | $CLICKHOUSE_FORMAT -n; -echo "insert into t values (1); " | $CLICKHOUSE_FORMAT -n 2>&1 \ | grep -F -q "Code: 1004" && echo 'OK' || echo 'FAIL' +echo "insert into t values (1); " | $CLICKHOUSE_FORMAT -n 2>&1 \ | grep -F -q "Code: 577" && echo 'OK' || echo 'FAIL' From 38e1cb41cced59dcf8e0a457817ff7c9fd2480e2 Mon Sep 17 00:00:00 2001 From: Nikita Mikhaylov Date: Thu, 11 Mar 2021 17:04:59 +0300 Subject: [PATCH 324/544] done --- src/IO/Progress.h | 8 ++++---- src/Interpreters/executeQuery.cpp | 4 ++-- .../Formats/Impl/ParallelFormattingOutputFormat.h | 3 +++ 3 files changed, 9 insertions(+), 6 deletions(-) diff --git a/src/IO/Progress.h b/src/IO/Progress.h index a3efb96db98..64bf3a404af 100644 --- a/src/IO/Progress.h +++ b/src/IO/Progress.h @@ -64,12 +64,12 @@ struct Progress std::atomic written_rows {0}; std::atomic written_bytes {0}; - Progress() {} + Progress() = default; Progress(size_t read_rows_, size_t read_bytes_, size_t total_rows_to_read_ = 0) : read_rows(read_rows_), read_bytes(read_bytes_), total_rows_to_read(total_rows_to_read_) {} - Progress(ReadProgress read_progress) + explicit Progress(ReadProgress read_progress) : read_rows(read_progress.read_rows), read_bytes(read_progress.read_bytes), total_rows_to_read(read_progress.total_rows_to_read) {} - Progress(WriteProgress write_progress) + explicit Progress(WriteProgress write_progress) : written_rows(write_progress.written_rows), written_bytes(write_progress.written_bytes) {} void read(ReadBuffer & in, UInt64 server_revision); @@ -86,7 +86,7 @@ struct Progress written_rows += rhs.written_rows; written_bytes += rhs.written_bytes; - return rhs.read_rows || rhs.written_rows ? true : false; + return rhs.read_rows || rhs.written_rows; } void reset() diff --git a/src/Interpreters/executeQuery.cpp b/src/Interpreters/executeQuery.cpp index 1a0aa031d6f..a5c21405ff1 100644 --- a/src/Interpreters/executeQuery.cpp +++ b/src/Interpreters/executeQuery.cpp @@ -1014,7 +1014,7 @@ void executeQuery( ? getIdentifierName(ast_query_with_output->format) : context.getDefaultFormat(); - auto out = context.getOutputStream(format_name, *out_buf, streams.in->getHeader()); + auto out = context.getOutputStreamParallelIfPossible(format_name, *out_buf, streams.in->getHeader()); /// Save previous progress callback if any. TODO Do it more conveniently. auto previous_progress_callback = context.getProgressCallback(); @@ -1059,7 +1059,7 @@ void executeQuery( return std::make_shared(header); }); - auto out = context.getOutputFormat(format_name, *out_buf, pipeline.getHeader()); + auto out = context.getOutputFormatParallelIfPossible(format_name, *out_buf, pipeline.getHeader()); out->setAutoFlush(); /// Save previous progress callback if any. TODO Do it more conveniently. diff --git a/src/Processors/Formats/Impl/ParallelFormattingOutputFormat.h b/src/Processors/Formats/Impl/ParallelFormattingOutputFormat.h index 7e7c44a8aae..2efc369e178 100644 --- a/src/Processors/Formats/Impl/ParallelFormattingOutputFormat.h +++ b/src/Processors/Formats/Impl/ParallelFormattingOutputFormat.h @@ -101,6 +101,9 @@ public: finishAndWait(); } + /// There are no formats which support parallel formatting and progress writing at the same time + void onProgress(const Progress &) override {} + protected: void consume(Chunk chunk) override final { From b0577282eb8e892ff697f69a603139feb6137bda Mon Sep 17 00:00:00 2001 From: Nikita Mikhaylov Date: Thu, 11 Mar 2021 17:19:03 +0300 Subject: [PATCH 325/544] Update CMakeLists.txt --- src/Functions/CMakeLists.txt | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/Functions/CMakeLists.txt b/src/Functions/CMakeLists.txt index 321aa5e2196..1c3beb2e47d 100644 --- a/src/Functions/CMakeLists.txt +++ b/src/Functions/CMakeLists.txt @@ -117,3 +117,6 @@ target_link_libraries(clickhouse_functions PRIVATE clickhouse_functions_array) if (USE_STATS) target_link_libraries(clickhouse_functions PRIVATE stats) endif() + +# Signed integer overflow on user-provided data inside boost::geometry - ignore. +set_source_files_properties("pointInPolygon.cpp" PROPERTIES COMPILE_FLAGS -fno-sanitize=signed-integer-overflow) From 6928f2a409a1755c44d62d2941ba665e14bf8b02 Mon Sep 17 00:00:00 2001 From: Nikita Mikhaylov Date: Thu, 11 Mar 2021 17:20:06 +0300 Subject: [PATCH 326/544] Update pointInPolygon.cpp --- src/Functions/pointInPolygon.cpp | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/Functions/pointInPolygon.cpp b/src/Functions/pointInPolygon.cpp index 63ea3088039..8e4a05bdec5 100644 --- a/src/Functions/pointInPolygon.cpp +++ b/src/Functions/pointInPolygon.cpp @@ -425,7 +425,8 @@ private: T & out_container) const { out_container.reserve(end - begin); - for (size_t i = begin; i < end; ++i) { + for (size_t i = begin; i < end; ++i) + { Int64 result = 0; if (common::mulOverflow(static_cast(x_data[i]), static_cast(y_data[i]), result)) throw Exception("The coordinates of the point are such that subsequent calculations cannot be performed correctly. " \ From 138dedf2df59a1728bd76874d78cd2d3875146e9 Mon Sep 17 00:00:00 2001 From: Nikita Mikhailov Date: Wed, 24 Feb 2021 20:04:37 +0300 Subject: [PATCH 327/544] done --- src/Processors/Formats/IRowInputFormat.h | 1 + .../Formats/Impl/CSVRowInputFormat.cpp | 5 ++- .../Formats/Impl/ParallelParsingInputFormat.h | 3 ++ .../Impl/TabSeparatedRowInputFormat.cpp | 5 +-- ...0161_parallel_parsing_with_names.reference | 8 +++++ .../00161_parallel_parsing_with_names.sh | 31 +++++++++++++++++++ 6 files changed, 50 insertions(+), 3 deletions(-) create mode 100644 tests/queries/1_stateful/00161_parallel_parsing_with_names.reference create mode 100755 tests/queries/1_stateful/00161_parallel_parsing_with_names.sh diff --git a/src/Processors/Formats/IRowInputFormat.h b/src/Processors/Formats/IRowInputFormat.h index b7863704062..b99c3789383 100644 --- a/src/Processors/Formats/IRowInputFormat.h +++ b/src/Processors/Formats/IRowInputFormat.h @@ -53,6 +53,7 @@ public: void resetParser() override; protected: + friend class ParallelParsingInputFormat; /** Read next row and append it to the columns. * If no more rows - return false. */ diff --git a/src/Processors/Formats/Impl/CSVRowInputFormat.cpp b/src/Processors/Formats/Impl/CSVRowInputFormat.cpp index f7f08411dfa..ba3d449de97 100644 --- a/src/Processors/Formats/Impl/CSVRowInputFormat.cpp +++ b/src/Processors/Formats/Impl/CSVRowInputFormat.cpp @@ -8,6 +8,8 @@ #include +#include + namespace DB { @@ -155,7 +157,7 @@ void CSVRowInputFormat::readPrefix() size_t num_columns = data_types.size(); const auto & header = getPort().getHeader(); - if (with_names) + if (with_names && getCurrentUnitNumber() == 0) { /// This CSV file has a header row with column names. Depending on the /// settings, use it or skip it. @@ -492,6 +494,7 @@ static std::pair fileSegmentationEngineCSVImpl(ReadBuffer & in, DB void registerFileSegmentationEngineCSV(FormatFactory & factory) { factory.registerFileSegmentationEngine("CSV", &fileSegmentationEngineCSVImpl); + factory.registerFileSegmentationEngine("CSVWithNames", &fileSegmentationEngineCSVImpl); } } diff --git a/src/Processors/Formats/Impl/ParallelParsingInputFormat.h b/src/Processors/Formats/Impl/ParallelParsingInputFormat.h index 9dda2dfe55d..606842242bc 100644 --- a/src/Processors/Formats/Impl/ParallelParsingInputFormat.h +++ b/src/Processors/Formats/Impl/ParallelParsingInputFormat.h @@ -10,6 +10,7 @@ #include #include #include +#include namespace DB { @@ -97,6 +98,8 @@ public: segmentator_thread = ThreadFromGlobalPool( &ParallelParsingInputFormat::segmentatorThreadFunction, this, CurrentThread::getGroup()); + + LOG_DEBUG(&Poco::Logger::get("ParallelParsingInputFormat"), "Parallel parsing is used"); } ~ParallelParsingInputFormat() override diff --git a/src/Processors/Formats/Impl/TabSeparatedRowInputFormat.cpp b/src/Processors/Formats/Impl/TabSeparatedRowInputFormat.cpp index 96b01a5bd9b..5ce612e6e73 100644 --- a/src/Processors/Formats/Impl/TabSeparatedRowInputFormat.cpp +++ b/src/Processors/Formats/Impl/TabSeparatedRowInputFormat.cpp @@ -136,7 +136,7 @@ void TabSeparatedRowInputFormat::readPrefix() skipBOMIfExists(in); } - if (with_names) + if (with_names && getCurrentUnitNumber() == 0) { if (format_settings.with_names_use_header) { @@ -463,9 +463,10 @@ static std::pair fileSegmentationEngineTabSeparatedImpl(ReadBuffer void registerFileSegmentationEngineTabSeparated(FormatFactory & factory) { // We can use the same segmentation engine for TSKV. - for (const auto * name : {"TabSeparated", "TSV", "TSKV"}) + for (const std::string & name : {"TabSeparated", "TSV", "TSKV"}) { factory.registerFileSegmentationEngine(name, &fileSegmentationEngineTabSeparatedImpl); + factory.registerFileSegmentationEngine(name + "WithNames", &fileSegmentationEngineTabSeparatedImpl); } } diff --git a/tests/queries/1_stateful/00161_parallel_parsing_with_names.reference b/tests/queries/1_stateful/00161_parallel_parsing_with_names.reference new file mode 100644 index 00000000000..4e0017f2d78 --- /dev/null +++ b/tests/queries/1_stateful/00161_parallel_parsing_with_names.reference @@ -0,0 +1,8 @@ +TSVWithNames, false +50000 +TSVWithNames, true +50000 +CSVWithNames, false +50000 +CSVWithNames, true +50000 diff --git a/tests/queries/1_stateful/00161_parallel_parsing_with_names.sh b/tests/queries/1_stateful/00161_parallel_parsing_with_names.sh new file mode 100755 index 00000000000..79a2d5c22c5 --- /dev/null +++ b/tests/queries/1_stateful/00161_parallel_parsing_with_names.sh @@ -0,0 +1,31 @@ +#!/usr/bin/env bash + +CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh +. "$CURDIR"/../shell_config.sh + +FORMATS=('TSVWithNames' 'CSVWithNames') +$CLICKHOUSE_CLIENT -q "DROP TABLE IF EXISTS parsing_with_names" + +for format in "${FORMATS[@]}" +do + $CLICKHOUSE_CLIENT -q "CREATE TABLE parsing_with_names(a DateTime, b String, c FixedString(16)) ENGINE=Memory()" + + echo "$format, false"; + $CLICKHOUSE_CLIENT --output_format_parallel_formatting=false -q \ + "SELECT ClientEventTime as a, MobilePhoneModel as b, ClientIP6 as c FROM test.hits LIMIT 50000 Format $format" | \ + $CLICKHOUSE_CLIENT --input_format_parallel_parsing=false -q "INSERT INTO parsing_with_names FORMAT $format" + + $CLICKHOUSE_CLIENT -q "SELECT count() FROM parsing_with_names;" + $CLICKHOUSE_CLIENT -q "DROP TABLE IF EXISTS parsing_with_names" + + + $CLICKHOUSE_CLIENT -q "CREATE TABLE parsing_with_names(a DateTime, b String, c FixedString(16)) ENGINE=Memory()" + echo "$format, true"; + $CLICKHOUSE_CLIENT --output_format_parallel_formatting=false -q \ + "SELECT ClientEventTime as a, MobilePhoneModel as b, ClientIP6 as c FROM test.hits LIMIT 50000 Format $format" | \ + $CLICKHOUSE_CLIENT --input_format_parallel_parsing=true -q "INSERT INTO parsing_with_names FORMAT $format" + + $CLICKHOUSE_CLIENT -q "SELECT count() FROM parsing_with_names;" + $CLICKHOUSE_CLIENT -q "DROP TABLE IF EXISTS parsing_with_names" +done \ No newline at end of file From 61fdeb58972c0b166710e9f862d327f2e10213d1 Mon Sep 17 00:00:00 2001 From: Nikita Mikhailov Date: Wed, 24 Feb 2021 20:12:22 +0300 Subject: [PATCH 328/544] better --- src/Processors/Formats/IRowInputFormat.h | 1 - src/Processors/Formats/Impl/CSVRowInputFormat.cpp | 4 ++-- src/Processors/Formats/Impl/TabSeparatedRowInputFormat.cpp | 2 ++ 3 files changed, 4 insertions(+), 3 deletions(-) diff --git a/src/Processors/Formats/IRowInputFormat.h b/src/Processors/Formats/IRowInputFormat.h index b99c3789383..b7863704062 100644 --- a/src/Processors/Formats/IRowInputFormat.h +++ b/src/Processors/Formats/IRowInputFormat.h @@ -53,7 +53,6 @@ public: void resetParser() override; protected: - friend class ParallelParsingInputFormat; /** Read next row and append it to the columns. * If no more rows - return false. */ diff --git a/src/Processors/Formats/Impl/CSVRowInputFormat.cpp b/src/Processors/Formats/Impl/CSVRowInputFormat.cpp index ba3d449de97..0f7ca7b9e79 100644 --- a/src/Processors/Formats/Impl/CSVRowInputFormat.cpp +++ b/src/Processors/Formats/Impl/CSVRowInputFormat.cpp @@ -8,8 +8,6 @@ #include -#include - namespace DB { @@ -157,6 +155,8 @@ void CSVRowInputFormat::readPrefix() size_t num_columns = data_types.size(); const auto & header = getPort().getHeader(); + /// This is a bit of abstraction leakage, but we have almost the same code in other places. + /// Thus, we check if this InputFormat is working with the "real" beggining of the data in case of parallel parsing. if (with_names && getCurrentUnitNumber() == 0) { /// This CSV file has a header row with column names. Depending on the diff --git a/src/Processors/Formats/Impl/TabSeparatedRowInputFormat.cpp b/src/Processors/Formats/Impl/TabSeparatedRowInputFormat.cpp index 5ce612e6e73..00926b6e99b 100644 --- a/src/Processors/Formats/Impl/TabSeparatedRowInputFormat.cpp +++ b/src/Processors/Formats/Impl/TabSeparatedRowInputFormat.cpp @@ -136,6 +136,8 @@ void TabSeparatedRowInputFormat::readPrefix() skipBOMIfExists(in); } + /// This is a bit of abstraction leakage, but we have almost the same code in other places. + /// Thus, we check if this InputFormat is working with the "real" beggining of the data in case of parallel parsing. if (with_names && getCurrentUnitNumber() == 0) { if (format_settings.with_names_use_header) From 8b30bb0fb540c1c8db386c0d1f3ca0f6e3a60f1c Mon Sep 17 00:00:00 2001 From: Nikita Mikhailov Date: Wed, 24 Feb 2021 21:03:02 +0300 Subject: [PATCH 329/544] style --- src/Processors/Formats/Impl/CSVRowInputFormat.cpp | 2 +- src/Processors/Formats/Impl/TabSeparatedRowInputFormat.cpp | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/Processors/Formats/Impl/CSVRowInputFormat.cpp b/src/Processors/Formats/Impl/CSVRowInputFormat.cpp index 0f7ca7b9e79..6610a7a0d82 100644 --- a/src/Processors/Formats/Impl/CSVRowInputFormat.cpp +++ b/src/Processors/Formats/Impl/CSVRowInputFormat.cpp @@ -156,7 +156,7 @@ void CSVRowInputFormat::readPrefix() const auto & header = getPort().getHeader(); /// This is a bit of abstraction leakage, but we have almost the same code in other places. - /// Thus, we check if this InputFormat is working with the "real" beggining of the data in case of parallel parsing. + /// Thus, we check if this InputFormat is working with the "real" beginning of the data in case of parallel parsing. if (with_names && getCurrentUnitNumber() == 0) { /// This CSV file has a header row with column names. Depending on the diff --git a/src/Processors/Formats/Impl/TabSeparatedRowInputFormat.cpp b/src/Processors/Formats/Impl/TabSeparatedRowInputFormat.cpp index 00926b6e99b..7dcfd4e930d 100644 --- a/src/Processors/Formats/Impl/TabSeparatedRowInputFormat.cpp +++ b/src/Processors/Formats/Impl/TabSeparatedRowInputFormat.cpp @@ -137,7 +137,7 @@ void TabSeparatedRowInputFormat::readPrefix() } /// This is a bit of abstraction leakage, but we have almost the same code in other places. - /// Thus, we check if this InputFormat is working with the "real" beggining of the data in case of parallel parsing. + /// Thus, we check if this InputFormat is working with the "real" beginning of the data in case of parallel parsing. if (with_names && getCurrentUnitNumber() == 0) { if (format_settings.with_names_use_header) From 91a418b3a9682a4817948264d6cc57a08a117de4 Mon Sep 17 00:00:00 2001 From: Nikita Mikhailov Date: Wed, 24 Feb 2021 21:09:31 +0300 Subject: [PATCH 330/544] better --- src/Processors/Formats/Impl/TabSeparatedRowInputFormat.cpp | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/Processors/Formats/Impl/TabSeparatedRowInputFormat.cpp b/src/Processors/Formats/Impl/TabSeparatedRowInputFormat.cpp index 7dcfd4e930d..510cf67065f 100644 --- a/src/Processors/Formats/Impl/TabSeparatedRowInputFormat.cpp +++ b/src/Processors/Formats/Impl/TabSeparatedRowInputFormat.cpp @@ -465,10 +465,9 @@ static std::pair fileSegmentationEngineTabSeparatedImpl(ReadBuffer void registerFileSegmentationEngineTabSeparated(FormatFactory & factory) { // We can use the same segmentation engine for TSKV. - for (const std::string & name : {"TabSeparated", "TSV", "TSKV"}) + for (const std::string & name : {"TabSeparated", "TSV", "TSKV", "TabSeparatedWithNames", "TSVWithNames"}) { factory.registerFileSegmentationEngine(name, &fileSegmentationEngineTabSeparatedImpl); - factory.registerFileSegmentationEngine(name + "WithNames", &fileSegmentationEngineTabSeparatedImpl); } } From eab35bfc7f6c18c322c291904ae0514b35a3a61a Mon Sep 17 00:00:00 2001 From: alexey-milovidov Date: Sun, 28 Feb 2021 02:37:28 +0300 Subject: [PATCH 331/544] Update ParallelParsingInputFormat.h --- src/Processors/Formats/Impl/ParallelParsingInputFormat.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Processors/Formats/Impl/ParallelParsingInputFormat.h b/src/Processors/Formats/Impl/ParallelParsingInputFormat.h index 606842242bc..f1a290619fa 100644 --- a/src/Processors/Formats/Impl/ParallelParsingInputFormat.h +++ b/src/Processors/Formats/Impl/ParallelParsingInputFormat.h @@ -99,7 +99,7 @@ public: segmentator_thread = ThreadFromGlobalPool( &ParallelParsingInputFormat::segmentatorThreadFunction, this, CurrentThread::getGroup()); - LOG_DEBUG(&Poco::Logger::get("ParallelParsingInputFormat"), "Parallel parsing is used"); + LOG_TRACE(&Poco::Logger::get("ParallelParsingInputFormat"), "Parallel parsing is used"); } ~ParallelParsingInputFormat() override From 369c9da1611a97bc0c0ebf37a8fdba29dd47dc08 Mon Sep 17 00:00:00 2001 From: Nikita Mikhailov Date: Mon, 1 Mar 2021 22:58:55 +0300 Subject: [PATCH 332/544] better --- src/Processors/Formats/IInputFormat.cpp | 1 + src/Processors/Formats/IInputFormat.h | 25 ++++++++ .../Formats/Impl/CSVRowInputFormat.cpp | 50 ++++++++-------- .../Formats/Impl/CSVRowInputFormat.h | 10 ---- .../Impl/ParallelParsingInputFormat.cpp | 9 +++ .../Formats/Impl/ParallelParsingInputFormat.h | 3 + .../Impl/TabSeparatedRowInputFormat.cpp | 58 +++++++++---------- .../Formats/Impl/TabSeparatedRowInputFormat.h | 4 -- src/Processors/ISource.h | 2 +- .../00161_parallel_parsing_with_names.sh | 3 +- 10 files changed, 95 insertions(+), 70 deletions(-) diff --git a/src/Processors/Formats/IInputFormat.cpp b/src/Processors/Formats/IInputFormat.cpp index 0fbc78ea8c0..069d25564b1 100644 --- a/src/Processors/Formats/IInputFormat.cpp +++ b/src/Processors/Formats/IInputFormat.cpp @@ -13,6 +13,7 @@ namespace ErrorCodes IInputFormat::IInputFormat(Block header, ReadBuffer & in_) : ISource(std::move(header)), in(in_) { + column_mapping = std::make_shared(); } void IInputFormat::resetParser() diff --git a/src/Processors/Formats/IInputFormat.h b/src/Processors/Formats/IInputFormat.h index e1537aff6c5..b8ee4d438df 100644 --- a/src/Processors/Formats/IInputFormat.h +++ b/src/Processors/Formats/IInputFormat.h @@ -2,9 +2,26 @@ #include +#include + namespace DB { +/// Used to pass info from header between different InputFormats in ParallelParsing +struct ColumnMapping +{ + /// Maps indexes of columns in the input file to indexes of table columns + using OptionalIndexes = std::vector>; + OptionalIndexes column_indexes_for_input_fields; + + /// Tracks which columns we have read in a single read() call. + /// For columns that are never read, it is initialized to false when we + /// read the file header, and never changed afterwards. + /// For other columns, it is updated on each read() call. + std::vector read_columns; +}; + +using ColumnMappingPtr = std::shared_ptr; class ReadBuffer; @@ -39,9 +56,17 @@ public: return none; } + /// Must be called from ParallelParsingInputFormat after readSuffix + ColumnMappingPtr getColumnMapping() const { return column_mapping; } + /// Must be called from ParallelParsingInputFormat before readPrefix + void setColumnMapping(ColumnMappingPtr column_mapping_ ) { column_mapping = column_mapping_; } + size_t getCurrentUnitNumber() const { return current_unit_number; } void setCurrentUnitNumber(size_t current_unit_number_) { current_unit_number = current_unit_number_; } +protected: + ColumnMappingPtr column_mapping{}; + private: /// Number of currently parsed chunk (if parallel parsing is enabled) size_t current_unit_number = 0; diff --git a/src/Processors/Formats/Impl/CSVRowInputFormat.cpp b/src/Processors/Formats/Impl/CSVRowInputFormat.cpp index 6610a7a0d82..efb8ad93d72 100644 --- a/src/Processors/Formats/Impl/CSVRowInputFormat.cpp +++ b/src/Processors/Formats/Impl/CSVRowInputFormat.cpp @@ -55,13 +55,13 @@ void CSVRowInputFormat::addInputColumn(const String & column_name) { if (format_settings.skip_unknown_fields) { - column_indexes_for_input_fields.push_back(std::nullopt); + column_mapping->column_indexes_for_input_fields.push_back(std::nullopt); return; } throw Exception( "Unknown field found in CSV header: '" + column_name + "' " + - "at position " + std::to_string(column_indexes_for_input_fields.size()) + + "at position " + std::to_string(column_mapping->column_indexes_for_input_fields.size()) + "\nSet the 'input_format_skip_unknown_fields' parameter explicitly to ignore and proceed", ErrorCodes::INCORRECT_DATA ); @@ -69,11 +69,11 @@ void CSVRowInputFormat::addInputColumn(const String & column_name) const auto column_index = column_it->second; - if (read_columns[column_index]) + if (column_mapping->read_columns[column_index]) throw Exception("Duplicate field found while parsing CSV header: " + column_name, ErrorCodes::INCORRECT_DATA); - read_columns[column_index] = true; - column_indexes_for_input_fields.emplace_back(column_index); + column_mapping->read_columns[column_index] = true; + column_mapping->column_indexes_for_input_fields.emplace_back(column_index); } static void skipEndOfLine(ReadBuffer & in) @@ -165,7 +165,7 @@ void CSVRowInputFormat::readPrefix() { /// Look at the file header to see which columns we have there. /// The missing columns are filled with defaults. - read_columns.assign(header.columns(), false); + column_mapping->read_columns.assign(header.columns(), false); do { String column_name; @@ -179,7 +179,7 @@ void CSVRowInputFormat::readPrefix() skipDelimiter(in, format_settings.csv.delimiter, true); - for (auto read_column : read_columns) + for (auto read_column : column_mapping->read_columns) { if (!read_column) { @@ -196,12 +196,12 @@ void CSVRowInputFormat::readPrefix() /// The default: map each column of the file to the column of the table with /// the same index. - read_columns.assign(header.columns(), true); - column_indexes_for_input_fields.resize(header.columns()); + column_mapping->read_columns.assign(header.columns(), true); + column_mapping->column_indexes_for_input_fields.resize(header.columns()); - for (size_t i = 0; i < column_indexes_for_input_fields.size(); ++i) + for (size_t i = 0; i < column_mapping->column_indexes_for_input_fields.size(); ++i) { - column_indexes_for_input_fields[i] = i; + column_mapping->column_indexes_for_input_fields[i] = i; } } @@ -218,12 +218,12 @@ bool CSVRowInputFormat::readRow(MutableColumns & columns, RowReadExtension & ext /// it doesn't have to check it. bool have_default_columns = have_always_default_columns; - ext.read_columns.assign(read_columns.size(), true); + ext.read_columns.assign(column_mapping->read_columns.size(), true); const auto delimiter = format_settings.csv.delimiter; - for (size_t file_column = 0; file_column < column_indexes_for_input_fields.size(); ++file_column) + for (size_t file_column = 0; file_column < column_mapping->column_indexes_for_input_fields.size(); ++file_column) { - const auto & table_column = column_indexes_for_input_fields[file_column]; - const bool is_last_file_column = file_column + 1 == column_indexes_for_input_fields.size(); + const auto & table_column = column_mapping->column_indexes_for_input_fields[file_column]; + const bool is_last_file_column = file_column + 1 == column_mapping->column_indexes_for_input_fields.size(); if (table_column) { @@ -245,9 +245,9 @@ bool CSVRowInputFormat::readRow(MutableColumns & columns, RowReadExtension & ext if (have_default_columns) { - for (size_t i = 0; i < read_columns.size(); i++) + for (size_t i = 0; i < column_mapping->read_columns.size(); i++) { - if (!read_columns[i]) + if (!column_mapping->read_columns[i]) { /// The column value for this row is going to be overwritten /// with default by the caller, but the general assumption is @@ -268,7 +268,7 @@ bool CSVRowInputFormat::parseRowAndPrintDiagnosticInfo(MutableColumns & columns, { const char delimiter = format_settings.csv.delimiter; - for (size_t file_column = 0; file_column < column_indexes_for_input_fields.size(); ++file_column) + for (size_t file_column = 0; file_column < column_mapping->column_indexes_for_input_fields.size(); ++file_column) { if (file_column == 0 && in.eof()) { @@ -277,10 +277,10 @@ bool CSVRowInputFormat::parseRowAndPrintDiagnosticInfo(MutableColumns & columns, } skipWhitespacesAndTabs(in); - if (column_indexes_for_input_fields[file_column].has_value()) + if (column_mapping->column_indexes_for_input_fields[file_column].has_value()) { const auto & header = getPort().getHeader(); - size_t col_idx = column_indexes_for_input_fields[file_column].value(); + size_t col_idx = column_mapping->column_indexes_for_input_fields[file_column].value(); if (!deserializeFieldAndPrintDiagnosticInfo(header.getByPosition(col_idx).name, data_types[col_idx], *columns[col_idx], out, file_column)) return false; @@ -296,7 +296,7 @@ bool CSVRowInputFormat::parseRowAndPrintDiagnosticInfo(MutableColumns & columns, skipWhitespacesAndTabs(in); /// Delimiters - if (file_column + 1 == column_indexes_for_input_fields.size()) + if (file_column + 1 == column_mapping->column_indexes_for_input_fields.size()) { if (in.eof()) return false; @@ -358,9 +358,9 @@ void CSVRowInputFormat::syncAfterError() void CSVRowInputFormat::tryDeserializeField(const DataTypePtr & type, IColumn & column, size_t file_column) { - if (column_indexes_for_input_fields[file_column]) + if (column_mapping->column_indexes_for_input_fields[file_column]) { - const bool is_last_file_column = file_column + 1 == column_indexes_for_input_fields.size(); + const bool is_last_file_column = file_column + 1 == column_mapping->column_indexes_for_input_fields.size(); readField(column, type, is_last_file_column); } else @@ -406,8 +406,8 @@ bool CSVRowInputFormat::readField(IColumn & column, const DataTypePtr & type, bo void CSVRowInputFormat::resetParser() { RowInputFormatWithDiagnosticInfo::resetParser(); - column_indexes_for_input_fields.clear(); - read_columns.clear(); + column_mapping->column_indexes_for_input_fields.clear(); + column_mapping->read_columns.clear(); have_always_default_columns = false; } diff --git a/src/Processors/Formats/Impl/CSVRowInputFormat.h b/src/Processors/Formats/Impl/CSVRowInputFormat.h index c884eb6c3db..3cdafd8ec8d 100644 --- a/src/Processors/Formats/Impl/CSVRowInputFormat.h +++ b/src/Processors/Formats/Impl/CSVRowInputFormat.h @@ -38,16 +38,6 @@ private: using IndexesMap = std::unordered_map; IndexesMap column_indexes_by_names; - /// Maps indexes of columns in the input file to indexes of table columns - using OptionalIndexes = std::vector>; - OptionalIndexes column_indexes_for_input_fields; - - /// Tracks which columns we have read in a single read() call. - /// For columns that are never read, it is initialized to false when we - /// read the file header, and never changed afterwards. - /// For other columns, it is updated on each read() call. - std::vector read_columns; - /// Whether we have any columns that are not read from file at all, /// and must be always initialized with defaults. bool have_always_default_columns = false; diff --git a/src/Processors/Formats/Impl/ParallelParsingInputFormat.cpp b/src/Processors/Formats/Impl/ParallelParsingInputFormat.cpp index d1660b53019..ce041f6636b 100644 --- a/src/Processors/Formats/Impl/ParallelParsingInputFormat.cpp +++ b/src/Processors/Formats/Impl/ParallelParsingInputFormat.cpp @@ -89,6 +89,11 @@ void ParallelParsingInputFormat::parserThreadFunction(ThreadGroupStatusPtr threa unit.chunk_ext.chunk.clear(); unit.chunk_ext.block_missing_values.clear(); + /// Propagate column_mapping to other parsers. + /// Note: column_mapping is used only for *WithNames types + if (current_ticket_number != 0) + input_format->setColumnMapping(column_mapping); + // We don't know how many blocks will be. So we have to read them all // until an empty block occurred. Chunk chunk; @@ -100,6 +105,10 @@ void ParallelParsingInputFormat::parserThreadFunction(ThreadGroupStatusPtr threa unit.chunk_ext.block_missing_values.emplace_back(parser.getMissingValues()); } + /// Extract column_mapping from first parser to propage it to others + if (current_ticket_number == 0) + column_mapping = input_format->getColumnMapping(); + // We suppose we will get at least some blocks for a non-empty buffer, // except at the end of file. Also see a matching assert in readImpl(). assert(unit.is_last || !unit.chunk_ext.chunk.empty() || parsing_finished); diff --git a/src/Processors/Formats/Impl/ParallelParsingInputFormat.h b/src/Processors/Formats/Impl/ParallelParsingInputFormat.h index f1a290619fa..49bb69faeac 100644 --- a/src/Processors/Formats/Impl/ParallelParsingInputFormat.h +++ b/src/Processors/Formats/Impl/ParallelParsingInputFormat.h @@ -253,6 +253,9 @@ private: { parserThreadFunction(group, ticket_number); }); + /// We have to wait here to possibly extract ColumnMappingPtr from the first parser. + if (ticket_number == 0) + pool.wait(); } void finishAndWait() diff --git a/src/Processors/Formats/Impl/TabSeparatedRowInputFormat.cpp b/src/Processors/Formats/Impl/TabSeparatedRowInputFormat.cpp index 510cf67065f..04c6b4c3ee0 100644 --- a/src/Processors/Formats/Impl/TabSeparatedRowInputFormat.cpp +++ b/src/Processors/Formats/Impl/TabSeparatedRowInputFormat.cpp @@ -62,19 +62,19 @@ TabSeparatedRowInputFormat::TabSeparatedRowInputFormat(const Block & header_, Re column_indexes_by_names.emplace(column_info.name, i); } - column_indexes_for_input_fields.reserve(num_columns); - read_columns.assign(num_columns, false); + column_mapping->column_indexes_for_input_fields.reserve(num_columns); + column_mapping->read_columns.assign(num_columns, false); } void TabSeparatedRowInputFormat::setupAllColumnsByTableSchema() { const auto & header = getPort().getHeader(); - read_columns.assign(header.columns(), true); - column_indexes_for_input_fields.resize(header.columns()); + column_mapping->read_columns.assign(header.columns(), true); + column_mapping->column_indexes_for_input_fields.resize(header.columns()); - for (size_t i = 0; i < column_indexes_for_input_fields.size(); ++i) - column_indexes_for_input_fields[i] = i; + for (size_t i = 0; i < column_mapping->column_indexes_for_input_fields.size(); ++i) + column_mapping->column_indexes_for_input_fields[i] = i; } @@ -85,13 +85,13 @@ void TabSeparatedRowInputFormat::addInputColumn(const String & column_name) { if (format_settings.skip_unknown_fields) { - column_indexes_for_input_fields.push_back(std::nullopt); + column_mapping->column_indexes_for_input_fields.push_back(std::nullopt); return; } throw Exception( "Unknown field found in TSV header: '" + column_name + "' " + - "at position " + std::to_string(column_indexes_for_input_fields.size()) + + "at position " + std::to_string(column_mapping->column_indexes_for_input_fields.size()) + "\nSet the 'input_format_skip_unknown_fields' parameter explicitly to ignore and proceed", ErrorCodes::INCORRECT_DATA ); @@ -99,11 +99,11 @@ void TabSeparatedRowInputFormat::addInputColumn(const String & column_name) const auto column_index = column_it->second; - if (read_columns[column_index]) + if (column_mapping->read_columns[column_index]) throw Exception("Duplicate field found while parsing TSV header: " + column_name, ErrorCodes::INCORRECT_DATA); - read_columns[column_index] = true; - column_indexes_for_input_fields.emplace_back(column_index); + column_mapping->read_columns[column_index] = true; + column_mapping->column_indexes_for_input_fields.emplace_back(column_index); } @@ -113,8 +113,8 @@ void TabSeparatedRowInputFormat::fillUnreadColumnsWithDefaults(MutableColumns & if (unlikely(row_num == 1)) { columns_to_fill_with_default_values.clear(); - for (size_t index = 0; index < read_columns.size(); ++index) - if (read_columns[index] == 0) + for (size_t index = 0; index < column_mapping->read_columns.size(); ++index) + if (column_mapping->read_columns[index] == 0) columns_to_fill_with_default_values.push_back(index); } @@ -167,7 +167,7 @@ void TabSeparatedRowInputFormat::readPrefix() else { setupAllColumnsByTableSchema(); - skipTSVRow(in, column_indexes_for_input_fields.size()); + skipTSVRow(in, column_mapping->column_indexes_for_input_fields.size()); } } else @@ -175,7 +175,7 @@ void TabSeparatedRowInputFormat::readPrefix() if (with_types) { - skipTSVRow(in, column_indexes_for_input_fields.size()); + skipTSVRow(in, column_mapping->column_indexes_for_input_fields.size()); } } @@ -187,11 +187,11 @@ bool TabSeparatedRowInputFormat::readRow(MutableColumns & columns, RowReadExtens updateDiagnosticInfo(); - ext.read_columns.assign(read_columns.size(), true); - for (size_t file_column = 0; file_column < column_indexes_for_input_fields.size(); ++file_column) + ext.read_columns.assign(column_mapping->read_columns.size(), true); + for (size_t file_column = 0; file_column < column_mapping->column_indexes_for_input_fields.size(); ++file_column) { - const auto & column_index = column_indexes_for_input_fields[file_column]; - const bool is_last_file_column = file_column + 1 == column_indexes_for_input_fields.size(); + const auto & column_index = column_mapping->column_indexes_for_input_fields[file_column]; + const bool is_last_file_column = file_column + 1 == column_mapping->column_indexes_for_input_fields.size(); if (column_index) { const auto & type = data_types[*column_index]; @@ -204,7 +204,7 @@ bool TabSeparatedRowInputFormat::readRow(MutableColumns & columns, RowReadExtens } /// skip separators - if (file_column + 1 < column_indexes_for_input_fields.size()) + if (file_column + 1 < column_mapping->column_indexes_for_input_fields.size()) { assertChar('\t', in); } @@ -240,7 +240,7 @@ bool TabSeparatedRowInputFormat::readField(IColumn & column, const DataTypePtr & bool TabSeparatedRowInputFormat::parseRowAndPrintDiagnosticInfo(MutableColumns & columns, WriteBuffer & out) { - for (size_t file_column = 0; file_column < column_indexes_for_input_fields.size(); ++file_column) + for (size_t file_column = 0; file_column < column_mapping->column_indexes_for_input_fields.size(); ++file_column) { if (file_column == 0 && in.eof()) { @@ -248,10 +248,10 @@ bool TabSeparatedRowInputFormat::parseRowAndPrintDiagnosticInfo(MutableColumns & return false; } - if (column_indexes_for_input_fields[file_column].has_value()) + if (column_mapping->column_indexes_for_input_fields[file_column].has_value()) { const auto & header = getPort().getHeader(); - size_t col_idx = column_indexes_for_input_fields[file_column].value(); + size_t col_idx = column_mapping->column_indexes_for_input_fields[file_column].value(); if (!deserializeFieldAndPrintDiagnosticInfo(header.getByPosition(col_idx).name, data_types[col_idx], *columns[col_idx], out, file_column)) return false; @@ -266,7 +266,7 @@ bool TabSeparatedRowInputFormat::parseRowAndPrintDiagnosticInfo(MutableColumns & } /// Delimiters - if (file_column + 1 == column_indexes_for_input_fields.size()) + if (file_column + 1 == column_mapping->column_indexes_for_input_fields.size()) { if (!in.eof()) { @@ -332,7 +332,7 @@ bool TabSeparatedRowInputFormat::parseRowAndPrintDiagnosticInfo(MutableColumns & void TabSeparatedRowInputFormat::tryDeserializeField(const DataTypePtr & type, IColumn & column, size_t file_column) { - if (column_indexes_for_input_fields[file_column]) + if (column_mapping->column_indexes_for_input_fields[file_column]) { // check null value for type is not nullable. don't cross buffer bound for simplicity, so maybe missing some case if (!type->isNullable() && !in.eof()) @@ -351,7 +351,7 @@ void TabSeparatedRowInputFormat::tryDeserializeField(const DataTypePtr & type, I } } } - const bool is_last_file_column = file_column + 1 == column_indexes_for_input_fields.size(); + const bool is_last_file_column = file_column + 1 == column_mapping->column_indexes_for_input_fields.size(); readField(column, type, is_last_file_column); } else @@ -370,8 +370,8 @@ void TabSeparatedRowInputFormat::resetParser() { RowInputFormatWithDiagnosticInfo::resetParser(); const auto & sample = getPort().getHeader(); - read_columns.assign(sample.columns(), false); - column_indexes_for_input_fields.clear(); + column_mapping->read_columns.assign(sample.columns(), false); + column_mapping->column_indexes_for_input_fields.clear(); columns_to_fill_with_default_values.clear(); } @@ -465,7 +465,7 @@ static std::pair fileSegmentationEngineTabSeparatedImpl(ReadBuffer void registerFileSegmentationEngineTabSeparated(FormatFactory & factory) { // We can use the same segmentation engine for TSKV. - for (const std::string & name : {"TabSeparated", "TSV", "TSKV", "TabSeparatedWithNames", "TSVWithNames"}) + for (const auto & name : {"TabSeparated", "TSV", "TSKV", "TabSeparatedWithNames", "TSVWithNames"}) { factory.registerFileSegmentationEngine(name, &fileSegmentationEngineTabSeparatedImpl); } diff --git a/src/Processors/Formats/Impl/TabSeparatedRowInputFormat.h b/src/Processors/Formats/Impl/TabSeparatedRowInputFormat.h index 0141d87403a..db70b4d3fea 100644 --- a/src/Processors/Formats/Impl/TabSeparatedRowInputFormat.h +++ b/src/Processors/Formats/Impl/TabSeparatedRowInputFormat.h @@ -41,10 +41,6 @@ private: using IndexesMap = std::unordered_map; IndexesMap column_indexes_by_names; - using OptionalIndexes = std::vector>; - OptionalIndexes column_indexes_for_input_fields; - - std::vector read_columns; std::vector columns_to_fill_with_default_values; void addInputColumn(const String & column_name); diff --git a/src/Processors/ISource.h b/src/Processors/ISource.h index b7e2b5dce8e..db91c0c5bce 100644 --- a/src/Processors/ISource.h +++ b/src/Processors/ISource.h @@ -19,7 +19,7 @@ protected: virtual std::optional tryGenerate(); public: - ISource(Block header); + explicit ISource(Block header); Status prepare() override; void work() override; diff --git a/tests/queries/1_stateful/00161_parallel_parsing_with_names.sh b/tests/queries/1_stateful/00161_parallel_parsing_with_names.sh index 79a2d5c22c5..ab7706e100f 100755 --- a/tests/queries/1_stateful/00161_parallel_parsing_with_names.sh +++ b/tests/queries/1_stateful/00161_parallel_parsing_with_names.sh @@ -9,7 +9,8 @@ $CLICKHOUSE_CLIENT -q "DROP TABLE IF EXISTS parsing_with_names" for format in "${FORMATS[@]}" do - $CLICKHOUSE_CLIENT -q "CREATE TABLE parsing_with_names(a DateTime, b String, c FixedString(16)) ENGINE=Memory()" + # Columns are permuted + $CLICKHOUSE_CLIENT -q "CREATE TABLE parsing_with_names(c FixedString(16), a DateTime, b String) ENGINE=Memory()" echo "$format, false"; $CLICKHOUSE_CLIENT --output_format_parallel_formatting=false -q \ From 3a7d48f3a4c8c1a708ace4b0d9800e5ba3d1b8e6 Mon Sep 17 00:00:00 2001 From: Nikita Mikhailov Date: Tue, 2 Mar 2021 01:32:11 +0300 Subject: [PATCH 333/544] style and tsan fix --- src/Processors/Formats/IInputFormat.h | 4 +++- src/Processors/Formats/Impl/ParallelParsingInputFormat.cpp | 6 +++++- src/Processors/Formats/Impl/TabSeparatedRowInputFormat.cpp | 2 +- 3 files changed, 9 insertions(+), 3 deletions(-) diff --git a/src/Processors/Formats/IInputFormat.h b/src/Processors/Formats/IInputFormat.h index b8ee4d438df..a297f487318 100644 --- a/src/Processors/Formats/IInputFormat.h +++ b/src/Processors/Formats/IInputFormat.h @@ -10,6 +10,8 @@ namespace DB /// Used to pass info from header between different InputFormats in ParallelParsing struct ColumnMapping { + /// Non-atomic because only read access in possible + bool is_set; /// Maps indexes of columns in the input file to indexes of table columns using OptionalIndexes = std::vector>; OptionalIndexes column_indexes_for_input_fields; @@ -59,7 +61,7 @@ public: /// Must be called from ParallelParsingInputFormat after readSuffix ColumnMappingPtr getColumnMapping() const { return column_mapping; } /// Must be called from ParallelParsingInputFormat before readPrefix - void setColumnMapping(ColumnMappingPtr column_mapping_ ) { column_mapping = column_mapping_; } + void setColumnMapping(ColumnMappingPtr column_mapping_) { column_mapping = column_mapping_; } size_t getCurrentUnitNumber() const { return current_unit_number; } void setCurrentUnitNumber(size_t current_unit_number_) { current_unit_number = current_unit_number_; } diff --git a/src/Processors/Formats/Impl/ParallelParsingInputFormat.cpp b/src/Processors/Formats/Impl/ParallelParsingInputFormat.cpp index ce041f6636b..4c2b9df304b 100644 --- a/src/Processors/Formats/Impl/ParallelParsingInputFormat.cpp +++ b/src/Processors/Formats/Impl/ParallelParsingInputFormat.cpp @@ -92,7 +92,11 @@ void ParallelParsingInputFormat::parserThreadFunction(ThreadGroupStatusPtr threa /// Propagate column_mapping to other parsers. /// Note: column_mapping is used only for *WithNames types if (current_ticket_number != 0) + { + column_mapping->is_set = true; input_format->setColumnMapping(column_mapping); + } + // We don't know how many blocks will be. So we have to read them all // until an empty block occurred. @@ -105,7 +109,7 @@ void ParallelParsingInputFormat::parserThreadFunction(ThreadGroupStatusPtr threa unit.chunk_ext.block_missing_values.emplace_back(parser.getMissingValues()); } - /// Extract column_mapping from first parser to propage it to others + /// Extract column_mapping from first parser to propagate it to others if (current_ticket_number == 0) column_mapping = input_format->getColumnMapping(); diff --git a/src/Processors/Formats/Impl/TabSeparatedRowInputFormat.cpp b/src/Processors/Formats/Impl/TabSeparatedRowInputFormat.cpp index 04c6b4c3ee0..ffb1b96f70e 100644 --- a/src/Processors/Formats/Impl/TabSeparatedRowInputFormat.cpp +++ b/src/Processors/Formats/Impl/TabSeparatedRowInputFormat.cpp @@ -170,7 +170,7 @@ void TabSeparatedRowInputFormat::readPrefix() skipTSVRow(in, column_mapping->column_indexes_for_input_fields.size()); } } - else + else if (!column_mapping->is_set) setupAllColumnsByTableSchema(); if (with_types) From 3372dd7b6a69d67abfa658c8518dfe9a9e56d03d Mon Sep 17 00:00:00 2001 From: Nikita Mikhaylov Date: Tue, 2 Mar 2021 16:31:19 +0300 Subject: [PATCH 334/544] fix stupid bug --- src/Processors/Formats/IInputFormat.h | 3 ++- .../Formats/Impl/CSVRowInputFormat.cpp | 22 ++++++++++--------- .../Formats/Impl/CSVRowInputFormat.h | 1 + .../Impl/ParallelParsingInputFormat.cpp | 7 +++--- 4 files changed, 18 insertions(+), 15 deletions(-) diff --git a/src/Processors/Formats/IInputFormat.h b/src/Processors/Formats/IInputFormat.h index a297f487318..95910bf51e5 100644 --- a/src/Processors/Formats/IInputFormat.h +++ b/src/Processors/Formats/IInputFormat.h @@ -10,7 +10,8 @@ namespace DB /// Used to pass info from header between different InputFormats in ParallelParsing struct ColumnMapping { - /// Non-atomic because only read access in possible + /// Non-atomic because there is strict `happens-before` between read and write access + /// See InputFormatParallelParsing bool is_set; /// Maps indexes of columns in the input file to indexes of table columns using OptionalIndexes = std::vector>; diff --git a/src/Processors/Formats/Impl/CSVRowInputFormat.cpp b/src/Processors/Formats/Impl/CSVRowInputFormat.cpp index efb8ad93d72..4cec07f38dc 100644 --- a/src/Processors/Formats/Impl/CSVRowInputFormat.cpp +++ b/src/Processors/Formats/Impl/CSVRowInputFormat.cpp @@ -145,6 +145,16 @@ static void skipRow(ReadBuffer & in, const FormatSettings::CSV & settings, size_ } } +void CSVRowInputFormat::setupAllColumnsByTableSchema() +{ + const auto & header = getPort().getHeader(); + column_mapping->read_columns.assign(header.columns(), true); + column_mapping->column_indexes_for_input_fields.resize(header.columns()); + + for (size_t i = 0; i < column_mapping->column_indexes_for_input_fields.size(); ++i) + column_mapping->column_indexes_for_input_fields[i] = i; +} + void CSVRowInputFormat::readPrefix() { @@ -193,16 +203,8 @@ void CSVRowInputFormat::readPrefix() else skipRow(in, format_settings.csv, num_columns); } - - /// The default: map each column of the file to the column of the table with - /// the same index. - column_mapping->read_columns.assign(header.columns(), true); - column_mapping->column_indexes_for_input_fields.resize(header.columns()); - - for (size_t i = 0; i < column_mapping->column_indexes_for_input_fields.size(); ++i) - { - column_mapping->column_indexes_for_input_fields[i] = i; - } + else if (!column_mapping->is_set) + setupAllColumnsByTableSchema(); } diff --git a/src/Processors/Formats/Impl/CSVRowInputFormat.h b/src/Processors/Formats/Impl/CSVRowInputFormat.h index 3cdafd8ec8d..86e41cf0a43 100644 --- a/src/Processors/Formats/Impl/CSVRowInputFormat.h +++ b/src/Processors/Formats/Impl/CSVRowInputFormat.h @@ -44,6 +44,7 @@ private: void addInputColumn(const String & column_name); + void setupAllColumnsByTableSchema(); bool parseRowAndPrintDiagnosticInfo(MutableColumns & columns, WriteBuffer & out) override; void tryDeserializeField(const DataTypePtr & type, IColumn & column, size_t file_column) override; bool isGarbageAfterField(size_t, ReadBuffer::Position pos) override diff --git a/src/Processors/Formats/Impl/ParallelParsingInputFormat.cpp b/src/Processors/Formats/Impl/ParallelParsingInputFormat.cpp index 4c2b9df304b..a1d115c734c 100644 --- a/src/Processors/Formats/Impl/ParallelParsingInputFormat.cpp +++ b/src/Processors/Formats/Impl/ParallelParsingInputFormat.cpp @@ -92,11 +92,7 @@ void ParallelParsingInputFormat::parserThreadFunction(ThreadGroupStatusPtr threa /// Propagate column_mapping to other parsers. /// Note: column_mapping is used only for *WithNames types if (current_ticket_number != 0) - { - column_mapping->is_set = true; input_format->setColumnMapping(column_mapping); - } - // We don't know how many blocks will be. So we have to read them all // until an empty block occurred. @@ -111,7 +107,10 @@ void ParallelParsingInputFormat::parserThreadFunction(ThreadGroupStatusPtr threa /// Extract column_mapping from first parser to propagate it to others if (current_ticket_number == 0) + { column_mapping = input_format->getColumnMapping(); + column_mapping->is_set = true; + } // We suppose we will get at least some blocks for a non-empty buffer, // except at the end of file. Also see a matching assert in readImpl(). From e7a1398def3dfc553a932af908a4bbe6defba8ba Mon Sep 17 00:00:00 2001 From: Nikita Mikhaylov Date: Tue, 9 Mar 2021 22:02:56 +0300 Subject: [PATCH 335/544] try fix tests --- .../Formats/Impl/ParallelParsingInputFormat.cpp | 1 + .../Formats/Impl/ParallelParsingInputFormat.h | 16 +++++++++++++++- 2 files changed, 16 insertions(+), 1 deletion(-) diff --git a/src/Processors/Formats/Impl/ParallelParsingInputFormat.cpp b/src/Processors/Formats/Impl/ParallelParsingInputFormat.cpp index a1d115c734c..1ead36b99eb 100644 --- a/src/Processors/Formats/Impl/ParallelParsingInputFormat.cpp +++ b/src/Processors/Formats/Impl/ParallelParsingInputFormat.cpp @@ -110,6 +110,7 @@ void ParallelParsingInputFormat::parserThreadFunction(ThreadGroupStatusPtr threa { column_mapping = input_format->getColumnMapping(); column_mapping->is_set = true; + first_parser_finished.Notify(); } // We suppose we will get at least some blocks for a non-empty buffer, diff --git a/src/Processors/Formats/Impl/ParallelParsingInputFormat.h b/src/Processors/Formats/Impl/ParallelParsingInputFormat.h index 49bb69faeac..f42cd3eadd0 100644 --- a/src/Processors/Formats/Impl/ParallelParsingInputFormat.h +++ b/src/Processors/Formats/Impl/ParallelParsingInputFormat.h @@ -12,6 +12,18 @@ #include #include +/// I don't know why, but clang warns about static annotations +/// error: macro name is a reserved identifier [-Werror,-Wreserved-id-macro] +/// #define THREAD_ANNOTATION_ATTRIBUTE__(x) __attribute__((x)) +#ifdef __clang__ +#pragma clang diagnostic push +#pragma clang diagnostic ignored "-Wreserved-id-macro" +#endif +#include +#ifdef __clang__ +#pragma clang diagnostic pop +#endif + namespace DB { @@ -202,6 +214,8 @@ private: std::condition_variable reader_condvar; std::condition_variable segmentator_condvar; + absl::Notification first_parser_finished; + std::atomic parsing_finished{false}; /// There are multiple "parsers", that's why we use thread pool. @@ -255,7 +269,7 @@ private: }); /// We have to wait here to possibly extract ColumnMappingPtr from the first parser. if (ticket_number == 0) - pool.wait(); + first_parser_finished.WaitForNotification(); } void finishAndWait() From f3ee1290745d8a5fdf00ae0ab5fa4e9572f9295c Mon Sep 17 00:00:00 2001 From: Nikita Mikhaylov Date: Wed, 10 Mar 2021 00:22:04 +0300 Subject: [PATCH 336/544] fix grpc --- src/Processors/Formats/Impl/ParallelParsingInputFormat.cpp | 1 + 1 file changed, 1 insertion(+) diff --git a/src/Processors/Formats/Impl/ParallelParsingInputFormat.cpp b/src/Processors/Formats/Impl/ParallelParsingInputFormat.cpp index 1ead36b99eb..0029a7e1084 100644 --- a/src/Processors/Formats/Impl/ParallelParsingInputFormat.cpp +++ b/src/Processors/Formats/Impl/ParallelParsingInputFormat.cpp @@ -123,6 +123,7 @@ void ParallelParsingInputFormat::parserThreadFunction(ThreadGroupStatusPtr threa } catch (...) { + first_parser_finished.Notify(); onBackgroundException(unit.offset); } } From 761b369fa2212dbe4f58fd67622dad0e03c12630 Mon Sep 17 00:00:00 2001 From: Nikita Mikhaylov Date: Wed, 10 Mar 2021 01:20:38 +0300 Subject: [PATCH 337/544] move to poco::event --- .../Formats/Impl/ParallelParsingInputFormat.cpp | 4 ++-- .../Formats/Impl/ParallelParsingInputFormat.h | 17 +++-------------- 2 files changed, 5 insertions(+), 16 deletions(-) diff --git a/src/Processors/Formats/Impl/ParallelParsingInputFormat.cpp b/src/Processors/Formats/Impl/ParallelParsingInputFormat.cpp index 0029a7e1084..1055342cbea 100644 --- a/src/Processors/Formats/Impl/ParallelParsingInputFormat.cpp +++ b/src/Processors/Formats/Impl/ParallelParsingInputFormat.cpp @@ -110,7 +110,7 @@ void ParallelParsingInputFormat::parserThreadFunction(ThreadGroupStatusPtr threa { column_mapping = input_format->getColumnMapping(); column_mapping->is_set = true; - first_parser_finished.Notify(); + first_parser_finished.set(); } // We suppose we will get at least some blocks for a non-empty buffer, @@ -123,7 +123,7 @@ void ParallelParsingInputFormat::parserThreadFunction(ThreadGroupStatusPtr threa } catch (...) { - first_parser_finished.Notify(); + first_parser_finished.set(); onBackgroundException(unit.offset); } } diff --git a/src/Processors/Formats/Impl/ParallelParsingInputFormat.h b/src/Processors/Formats/Impl/ParallelParsingInputFormat.h index f42cd3eadd0..559507055b9 100644 --- a/src/Processors/Formats/Impl/ParallelParsingInputFormat.h +++ b/src/Processors/Formats/Impl/ParallelParsingInputFormat.h @@ -11,18 +11,7 @@ #include #include #include - -/// I don't know why, but clang warns about static annotations -/// error: macro name is a reserved identifier [-Werror,-Wreserved-id-macro] -/// #define THREAD_ANNOTATION_ATTRIBUTE__(x) __attribute__((x)) -#ifdef __clang__ -#pragma clang diagnostic push -#pragma clang diagnostic ignored "-Wreserved-id-macro" -#endif -#include -#ifdef __clang__ -#pragma clang diagnostic pop -#endif +#include namespace DB { @@ -214,7 +203,7 @@ private: std::condition_variable reader_condvar; std::condition_variable segmentator_condvar; - absl::Notification first_parser_finished; + Poco::Event first_parser_finished; std::atomic parsing_finished{false}; @@ -269,7 +258,7 @@ private: }); /// We have to wait here to possibly extract ColumnMappingPtr from the first parser. if (ticket_number == 0) - first_parser_finished.WaitForNotification(); + first_parser_finished.wait(); } void finishAndWait() From b434a2f0bdff85f679bd53b7b9ecbe12a83eabfb Mon Sep 17 00:00:00 2001 From: Nikita Mikhaylov Date: Thu, 11 Mar 2021 02:15:16 +0300 Subject: [PATCH 338/544] disable parallel parsing for grpc tests --- .../Formats/Impl/ParallelParsingInputFormat.cpp | 4 +--- tests/integration/test_grpc_protocol/configs/users.xml | 8 ++++++++ tests/integration/test_grpc_protocol/test.py | 2 +- 3 files changed, 10 insertions(+), 4 deletions(-) create mode 100644 tests/integration/test_grpc_protocol/configs/users.xml diff --git a/src/Processors/Formats/Impl/ParallelParsingInputFormat.cpp b/src/Processors/Formats/Impl/ParallelParsingInputFormat.cpp index 1055342cbea..1ad913a1a59 100644 --- a/src/Processors/Formats/Impl/ParallelParsingInputFormat.cpp +++ b/src/Processors/Formats/Impl/ParallelParsingInputFormat.cpp @@ -123,7 +123,6 @@ void ParallelParsingInputFormat::parserThreadFunction(ThreadGroupStatusPtr threa } catch (...) { - first_parser_finished.set(); onBackgroundException(unit.offset); } } @@ -131,8 +130,6 @@ void ParallelParsingInputFormat::parserThreadFunction(ThreadGroupStatusPtr threa void ParallelParsingInputFormat::onBackgroundException(size_t offset) { - tryLogCurrentException(__PRETTY_FUNCTION__); - std::unique_lock lock(mutex); if (!background_exception) { @@ -143,6 +140,7 @@ void ParallelParsingInputFormat::onBackgroundException(size_t offset) } tryLogCurrentException(__PRETTY_FUNCTION__); parsing_finished = true; + first_parser_finished.set(); reader_condvar.notify_all(); segmentator_condvar.notify_all(); } diff --git a/tests/integration/test_grpc_protocol/configs/users.xml b/tests/integration/test_grpc_protocol/configs/users.xml new file mode 100644 index 00000000000..2ae1a397fe5 --- /dev/null +++ b/tests/integration/test_grpc_protocol/configs/users.xml @@ -0,0 +1,8 @@ + + + + + 0 + + + diff --git a/tests/integration/test_grpc_protocol/test.py b/tests/integration/test_grpc_protocol/test.py index d8604276281..594879427ca 100644 --- a/tests/integration/test_grpc_protocol/test.py +++ b/tests/integration/test_grpc_protocol/test.py @@ -27,7 +27,7 @@ import clickhouse_grpc_pb2_grpc config_dir = os.path.join(SCRIPT_DIR, './configs') cluster = ClickHouseCluster(__file__) -node = cluster.add_instance('node', main_configs=['configs/grpc_config.xml']) +node = cluster.add_instance('node', main_configs=['configs/grpc_config.xml'], user_configs=["configs/users.xml"]) grpc_port = 9100 main_channel = None From 87eb2592deb9dca4eb39fa424493db4bf9d9e403 Mon Sep 17 00:00:00 2001 From: Nikita Mikhaylov Date: Thu, 11 Mar 2021 18:10:06 +0300 Subject: [PATCH 339/544] add input_format_skip_unknown_fields --- .../1_stateful/00161_parallel_parsing_with_names.sh | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/tests/queries/1_stateful/00161_parallel_parsing_with_names.sh b/tests/queries/1_stateful/00161_parallel_parsing_with_names.sh index ab7706e100f..ad7b83c55a4 100755 --- a/tests/queries/1_stateful/00161_parallel_parsing_with_names.sh +++ b/tests/queries/1_stateful/00161_parallel_parsing_with_names.sh @@ -14,8 +14,8 @@ do echo "$format, false"; $CLICKHOUSE_CLIENT --output_format_parallel_formatting=false -q \ - "SELECT ClientEventTime as a, MobilePhoneModel as b, ClientIP6 as c FROM test.hits LIMIT 50000 Format $format" | \ - $CLICKHOUSE_CLIENT --input_format_parallel_parsing=false -q "INSERT INTO parsing_with_names FORMAT $format" + "SELECT URLRegions as d, ClientEventTime as a, MobilePhoneModel as b, ParamPrice as e, ClientIP6 as c FROM test.hits LIMIT 50000 Format $format" | \ + $CLICKHOUSE_CLIENT --input_format_skip_unknown_fields=1 --input_format_parallel_parsing=false -q "INSERT INTO parsing_with_names FORMAT $format" $CLICKHOUSE_CLIENT -q "SELECT count() FROM parsing_with_names;" $CLICKHOUSE_CLIENT -q "DROP TABLE IF EXISTS parsing_with_names" @@ -24,8 +24,8 @@ do $CLICKHOUSE_CLIENT -q "CREATE TABLE parsing_with_names(a DateTime, b String, c FixedString(16)) ENGINE=Memory()" echo "$format, true"; $CLICKHOUSE_CLIENT --output_format_parallel_formatting=false -q \ - "SELECT ClientEventTime as a, MobilePhoneModel as b, ClientIP6 as c FROM test.hits LIMIT 50000 Format $format" | \ - $CLICKHOUSE_CLIENT --input_format_parallel_parsing=true -q "INSERT INTO parsing_with_names FORMAT $format" + "SELECT URLRegions as d, ClientEventTime as a, MobilePhoneModel as b, ParamPrice as e, ClientIP6 as c FROM test.hits LIMIT 50000 Format $format" | \ + $CLICKHOUSE_CLIENT --input_format_skip_unknown_fields=1 --input_format_parallel_parsing=true -q "INSERT INTO parsing_with_names FORMAT $format" $CLICKHOUSE_CLIENT -q "SELECT count() FROM parsing_with_names;" $CLICKHOUSE_CLIENT -q "DROP TABLE IF EXISTS parsing_with_names" From 8e68edebea30779f0e8856dc2e5f000d3bec6387 Mon Sep 17 00:00:00 2001 From: Nikita Mikhaylov Date: Thu, 11 Mar 2021 18:21:58 +0300 Subject: [PATCH 340/544] better test --- .../00161_parallel_parsing_with_names.reference | 8 ++++---- .../1_stateful/00161_parallel_parsing_with_names.sh | 6 +++--- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/tests/queries/1_stateful/00161_parallel_parsing_with_names.reference b/tests/queries/1_stateful/00161_parallel_parsing_with_names.reference index 4e0017f2d78..fb0ba75c148 100644 --- a/tests/queries/1_stateful/00161_parallel_parsing_with_names.reference +++ b/tests/queries/1_stateful/00161_parallel_parsing_with_names.reference @@ -1,8 +1,8 @@ TSVWithNames, false -50000 +29caf86494f169d6339f6c5610b20731 - TSVWithNames, true -50000 +29caf86494f169d6339f6c5610b20731 - CSVWithNames, false -50000 +29caf86494f169d6339f6c5610b20731 - CSVWithNames, true -50000 +29caf86494f169d6339f6c5610b20731 - diff --git a/tests/queries/1_stateful/00161_parallel_parsing_with_names.sh b/tests/queries/1_stateful/00161_parallel_parsing_with_names.sh index ad7b83c55a4..ca9984900e1 100755 --- a/tests/queries/1_stateful/00161_parallel_parsing_with_names.sh +++ b/tests/queries/1_stateful/00161_parallel_parsing_with_names.sh @@ -17,16 +17,16 @@ do "SELECT URLRegions as d, ClientEventTime as a, MobilePhoneModel as b, ParamPrice as e, ClientIP6 as c FROM test.hits LIMIT 50000 Format $format" | \ $CLICKHOUSE_CLIENT --input_format_skip_unknown_fields=1 --input_format_parallel_parsing=false -q "INSERT INTO parsing_with_names FORMAT $format" - $CLICKHOUSE_CLIENT -q "SELECT count() FROM parsing_with_names;" + $CLICKHOUSE_CLIENT -q "SELECT * FROM parsing_with_names;" | md5sum $CLICKHOUSE_CLIENT -q "DROP TABLE IF EXISTS parsing_with_names" - $CLICKHOUSE_CLIENT -q "CREATE TABLE parsing_with_names(a DateTime, b String, c FixedString(16)) ENGINE=Memory()" + $CLICKHOUSE_CLIENT -q "CREATE TABLE parsing_with_names(c FixedString(16), a DateTime, b String) ENGINE=Memory()" echo "$format, true"; $CLICKHOUSE_CLIENT --output_format_parallel_formatting=false -q \ "SELECT URLRegions as d, ClientEventTime as a, MobilePhoneModel as b, ParamPrice as e, ClientIP6 as c FROM test.hits LIMIT 50000 Format $format" | \ $CLICKHOUSE_CLIENT --input_format_skip_unknown_fields=1 --input_format_parallel_parsing=true -q "INSERT INTO parsing_with_names FORMAT $format" - $CLICKHOUSE_CLIENT -q "SELECT count() FROM parsing_with_names;" + $CLICKHOUSE_CLIENT -q "SELECT * FROM parsing_with_names;" | md5sum $CLICKHOUSE_CLIENT -q "DROP TABLE IF EXISTS parsing_with_names" done \ No newline at end of file From 2e2257e26164b1b5f0a22880417cc1548d14f034 Mon Sep 17 00:00:00 2001 From: Alexander Kuzmenkov <36882414+akuzm@users.noreply.github.com> Date: Thu, 11 Mar 2021 19:26:01 +0300 Subject: [PATCH 341/544] Update fuzzing-clickhouse.md --- website/blog/en/2021/fuzzing-clickhouse.md | 39 ++++++++++++++++++---- 1 file changed, 32 insertions(+), 7 deletions(-) diff --git a/website/blog/en/2021/fuzzing-clickhouse.md b/website/blog/en/2021/fuzzing-clickhouse.md index fe859aa0d6a..d15e68760d1 100644 --- a/website/blog/en/2021/fuzzing-clickhouse.md +++ b/website/blog/en/2021/fuzzing-clickhouse.md @@ -1,5 +1,6 @@ --- title: 'Fuzzing ClickHouse' +image: 'https://blog-images.clickhouse.tech/en/2021/fuzzing-clickhouse/some-checks-were-not-successful.png' date: '2021-03-08' author: '[Alexander Kuzmenkov](https://github.com/akuzm)' tags: ['fuzzing', 'testing'] @@ -88,10 +89,34 @@ The core implementation of the fuzzer is relatively small, consisting of about 700 lines of C++ code. A prototype was made in a couple of days, but naturally it took significantly longer to polish it and to start routinely using it in CI. It is very productive and let us find more than 200 bugs already (see the -label [fuzz](https://github.com/ClickHouse/ClickHouse/labels/fuzz) on GitHub). -Some errors it finds are not very interesting, e.g. wrong error messages when a -type of argument doesn't match. But we also found some serious logic errors or -even memory errors. We fix all the errors we find, even not significant ones, +label [fuzz](https://github.com/ClickHouse/ClickHouse/labels/fuzz) on GitHub), some of which are serious logic errors or +even memory errors. When we only started, we could segfault the server or make it enter a never-ending loop with simplest read-only queries such as `SELECT arrayReverseFill(x -> (x < 10), [])` or `SELECT geoDistance(0., 0., -inf, 1.)`. Of course I couldn't resist bringing down our [playground](https://gh-api.clickhouse.tech/play?user=play#LS0gWW91IGNhbiBxdWVyeSB0aGUgR2l0SHViIGhpc3RvcnkgZGF0YSBoZXJlLiBTZWUgaHR0cHM6Ly9naC5jbGlja2hvdXNlLnRlY2gvZXhwbG9yZXIvIGZvciB0aGUgZGVzY3JpcHRpb24gYW5kIGV4YW1wbGUgcXVlcmllcy4Kc2VsZWN0ICdoZWxsbyB3b3JsZCc=) with some of these queries, and was content to see that the server soon restarts correctly. +These queries are actually minified by hand, normally the fuzzer would generate something barely legible such as: +``` +SELECT + (val + 257, + (((tuple(NULL), 10.000100135803223), tuple(-inf)), '-1', (NULL, '0.10', NULL), NULL), + (val + 9223372036854775807) = (rval * 100), + tuple(65535), tuple(NULL), NULL, NULL), + * +FROM +( + SELECT dummy AS val + FROM system.one +) AS s1 +ANY LEFT JOIN +( + SELECT toLowCardinality(toNullable(dummy)) AS rval + FROM system.one +) AS s2 ON (val + 100) = (rval * 7) +``` +In principle, we could add automated test case minification by modifying AST in the +same vein with fuzzing. This is somewhat complicated by the fact that the server dies +after every, excuse my pun, successfully failed query, so we didn't implement it yet. + +Not all errors the fuzzer finds are significant, some of them are pretty boring and +harmless, such as a wrong error code for an +out-of-bounds argument. We still try to fix all of them, because this lets us ensure that under normal operation, the fuzzer doesn't find any errors. This is similar to the approach usually taken with compiler warnings and other optional diagnostics -- it's better to fix or disable every @@ -100,7 +125,7 @@ OK, and it's easy to notice new problems. After fixing the majority of pre-existing error, this fuzzer became efficient for finding errors in new features. Pull requests introducing new features -normally adds an SQL test, and we pay extra attention to the new tests when +normally add an SQL test, and we pay extra attention to the new tests when fuzzing, generating more permutations for them. Even if the coverage of the test is not sufficient, there is a good chance that the fuzzer will find the missing corner cases. So when we see that all the fuzzer runs in different @@ -129,7 +154,7 @@ the client going crazy and running a hundred of random queries instead. All queries from the current session become a source for expressions for fuzzing, so try entering several different queries to get more interesting results. Be careful not to do this in production! When you do this experiment, you'll soon -notice that the fuzzer tends to generate queries that are too long to run. This +notice that the fuzzer tends to generate queries that take very long to run. This is why for the CI fuzzer runs we have to configure the server to limit query execution time, memory usage and so on using the corresponding [server settings](https://clickhouse.tech/docs/en/operations/settings/query-complexity/#:~:text=In%20the%20default%20configuration%20file,query%20within%20a%20single%20server.). @@ -141,7 +166,7 @@ constraints](https://clickhouse.tech/docs/en/operations/settings/constraints-on- The AST-based fuzzer we discussed is only one of the many kinds of fuzzers we have in ClickHouse. There is a [talk](https://www.youtube.com/watch?v=GbmK84ZwSeI&t=4481s) (in Russian, [slides are here](https://presentations.clickhouse.tech/cpp_siberia_2021/)) by Alexey Milovidov that -explores all the fuzzer in greater detail (in Russian). Another interesting +explores all the fuzzers we have. Another interesting recent development is application of pivoted query synthesis technique, implemented in [SQLancer](https://github.com/sqlancer/sqlancer), to ClickHouse. The authors are going to give [a talk about From 4d7f418916ec7991935feb34284dbea3d7745f37 Mon Sep 17 00:00:00 2001 From: Alexander Kuzmenkov Date: Thu, 11 Mar 2021 19:57:28 +0300 Subject: [PATCH 342/544] cleanup --- website/blog/en/2021/fuzzing-clickhouse.md | 158 +++------------------ 1 file changed, 21 insertions(+), 137 deletions(-) diff --git a/website/blog/en/2021/fuzzing-clickhouse.md b/website/blog/en/2021/fuzzing-clickhouse.md index d15e68760d1..d7d01587534 100644 --- a/website/blog/en/2021/fuzzing-clickhouse.md +++ b/website/blog/en/2021/fuzzing-clickhouse.md @@ -1,97 +1,30 @@ --- title: 'Fuzzing ClickHouse' image: 'https://blog-images.clickhouse.tech/en/2021/fuzzing-clickhouse/some-checks-were-not-successful.png' -date: '2021-03-08' +date: '2021-03-11' author: '[Alexander Kuzmenkov](https://github.com/akuzm)' tags: ['fuzzing', 'testing'] --- -Testing is a major problem in software development: there is never enough of -it. It becomes especially true for a database management system, whose task is -to interpret a query language that works on the persistent state managed by the -system in a distributed fashion. Each of these three functions is hard enough -to test even in isolation, and it gets much worse when you combine them. As -ClickHouse developers, we know this from experience. Despite a large amount of -automated testing of all kinds we routinely perform as part of our continuous -integration system, new bugs and regressions are creeping in. We are always -looking for the ways to improve our test coverage, and this article will -describe our recent development in this area -- the AST-based query fuzzer. +Testing is a major problem in software development: there is never enough of it. It becomes especially true for a database management system, whose task is to interpret a query language that works on the persistent state managed by the system in a distributed fashion. Each of these three functions is hard enough to test even in isolation, and it gets much worse when you combine them. As ClickHouse developers, we know this from experience. Despite a large amount of automated testing of all kinds we routinely perform as part of our continuous integration system, new bugs and regressions are creeping in. We are always looking for the ways to improve our test coverage, and this article will describe our recent development in this area — the AST-based query fuzzer. -A natural form of testing for a SQL DBMS is to create a SQL script describing -the test case, and record its reference result. To test, we run the script and -check that the result matches the reference. This is used in many SQL DBMS, -and it is the default kind of a test you are expected to write for any -ClickHouse feature or fix. Currently we have [73k lines of SQL tests -alone](https://github.com/ClickHouse/ClickHouse/tree/master/tests/queries/0_stateless), -that reach the [code coverage of -76%](https://clickhouse-test-reports.s3.yandex.net/0/47d684a5c35410201d4dd4f63f3287bf25cdabb7/coverage_report/test_output/index.html). +## How to Test a SQL DBMS -This form of testing, where a developer writes a few simplified examples of how -the feature can and cannot be used, is sometimes called "example-based -testing". Sadly, the bugs often appear in various corner cases and intersections -of features, and it is not practical to enumerate all of these cases by hand. There is a -technique for automating this process, called "property-based testing". It lets -you write more general tests of the form "for all values matching these specs, -the result of some operation on them should match this other spec". For -example, such a test can check that if you add two positive numbers, the result -is greater than both of them. But you don't specify which numbers exactly, only -these properties. Then, the property testing system randomly generates some -examples with particular numbers that match the specification, and checks that -the result also matches its specification. +A natural form of testing for a SQL DBMS is to create a SQL script describing the test case, and record its reference result. To test, we run the script and check that the result matches the reference. This is used in many SQL DBMS, and it is the default kind of a test you are expected to write for any ClickHouse feature or fix. Currently we have [73k lines of SQL tests alone](https://github.com/ClickHouse/ClickHouse/tree/master/tests/queries/0_stateless), that reach the [code coverage of 76%](https://clickhouse-test-reports.s3.yandex.net/0/47d684a5c35410201d4dd4f63f3287bf25cdabb7/coverage_report/test_output/index.html). -Property-based testing is said to be very efficient, but requires some -developer effort and expertise to write the tests in a special way. There is -another well-known testing technique that is in some sense a corner case of -property-based testing, and that doesn't require much developer time. It is -called fuzzing. When you are fuzzing your program, you feed it random inputs -generated according to some grammar, and the property you are checking is that -your program terminates correctly (no segfaults or assertions or other kinds of -program errors). Most often, the grammar of input for fuzzing is simple -- say, -bit flips and additions, or maybe some dictionary. The space of possible inputs -is huge, so to find interesting paths in it, fuzzing software records the code -paths taken by the program under test for a particular input, and focuses on -the inputs that lead to new code paths that were not seen before. It also -employs some techniques for finding interesting constant values, and so on. In -general, fuzzing allows you to find many interesting corner cases in your -program automatically, without much developer involvement. +This form of testing, where a developer writes a few simplified examples of how the feature can and cannot be used, is sometimes called "example-based testing". Sadly, the bugs often appear in various corner cases and intersections of features, and it is not practical to enumerate all of these cases by hand. There is a technique for automating this process, called "property-based testing". It lets you write more general tests of the form "for all values matching these specs, the result of some operation on them should match this other spec". For example, such a test can check that if you add two positive numbers, the result is greater than both of them. But you don't specify which numbers exactly, only these properties. Then, the property testing system randomly generates some examples with particular numbers that match the specification, and checks that the result also matches its specification. -Generating valid SQL queries with bit flips would take a long time, so there are -systems that generate queries based on the SQL grammar, such as -[SQLSmith](https://github.com/anse1/sqlsmith). They are succesfully used for -finding bugs in databases. It would be interesting to use such a system for -ClickHouse, but it requires some up-front effort to support the ClickHouse SQL -grammar and functions, which may be different from the standard. Also, such -systems don't use any feedback, so while they are much better than systems with -primitive grammar, they still might have a hard time finding interesting -examples. But we already have a big corpus of human-written interesting SQL -queries -- it's in our regression tests. Maybe we can use them as a base for -fuzzing? We tried to do this, and it turned out to be surprisingly simple and -efficient. +Property-based testing is said to be very efficient, but requires some developer effort and expertise to write the tests in a special way. There is another well-known testing technique that is in some sense a corner case of property-based testing, and that doesn't require much developer time. It is called fuzzing. When you are fuzzing your program, you feed it random inputs generated according to some grammar, and the property you are checking is that your program terminates correctly (no segfaults or assertions or other kinds of program errors). Most often, the grammar of input for fuzzing is simple — say, bit flips and additions, or maybe some dictionary. The space of possible inputs is huge, so to find interesting paths in it, fuzzing software records the code paths taken by the program under test for a particular input, and focuses on the inputs that lead to new code paths that were not seen before. It also employs some techniques for finding interesting constant values, and so on. In general, fuzzing allows you to find many interesting corner cases in your program automatically, without much developer involvement. -Consider some SQL query from a regression test. After parsing, it is easy to -mutate the resulting AST (abstract syntax tree, an internal representation of -the parsed query) before execution to introduce random changes into the query. -For strings and arrays, we make random modifications such as inserting a random -character or doubling the string. For numbers, there are well-known Bad Numbers -such as 0, 1, powers of two and nearby, integer limits, `NaN`. `NaN`s proved to -be especially efficient in finding bugs, because you can often have some -alternative branches in your numeric code, but for a `NaN`, both branches hold -(or not) simultaneously, so this leads to nasty effects. +Generating valid SQL queries with bit flips would take a long time, so there are systems that generate queries based on the SQL grammar, such as [SQLSmith](https://github.com/anse1/sqlsmith). They are succesfully used for finding bugs in databases. It would be interesting to use such a system for ClickHouse, but it requires some up-front effort to support the ClickHouse SQL grammar and functions, which may be different from the standard. Also, such systems don't use any feedback, so while they are much better than systems with primitive grammar, they still might have a hard time finding interesting examples. But we already have a big corpus of human-written interesting SQL queries — it's in our regression tests. Maybe we can use them as a base for fuzzing? We tried to do this, and it turned out to be surprisingly simple and efficient. -Another interesting thing we can do is change the arguments of functions, or the list of -expressions in `SELECT`. Naturally, all the interesting arguments can be -taken from other test queries. Same goes for changing the tables used in the -queries. When the fuzzer runs in CI, it runs queries from all the SQL tests in -random order, mixing in the parts of query from different tests, so that we can -eventually test all the possible permutations of our features. +## AST-based Query Fuzzer -The core implementation of the fuzzer is relatively small, consisting of about -700 lines of C++ code. A prototype was made in a couple of days, but naturally -it took significantly longer to polish it and to start routinely using it in -CI. It is very productive and let us find more than 200 bugs already (see the -label [fuzz](https://github.com/ClickHouse/ClickHouse/labels/fuzz) on GitHub), some of which are serious logic errors or -even memory errors. When we only started, we could segfault the server or make it enter a never-ending loop with simplest read-only queries such as `SELECT arrayReverseFill(x -> (x < 10), [])` or `SELECT geoDistance(0., 0., -inf, 1.)`. Of course I couldn't resist bringing down our [playground](https://gh-api.clickhouse.tech/play?user=play#LS0gWW91IGNhbiBxdWVyeSB0aGUgR2l0SHViIGhpc3RvcnkgZGF0YSBoZXJlLiBTZWUgaHR0cHM6Ly9naC5jbGlja2hvdXNlLnRlY2gvZXhwbG9yZXIvIGZvciB0aGUgZGVzY3JpcHRpb24gYW5kIGV4YW1wbGUgcXVlcmllcy4Kc2VsZWN0ICdoZWxsbyB3b3JsZCc=) with some of these queries, and was content to see that the server soon restarts correctly. -These queries are actually minified by hand, normally the fuzzer would generate something barely legible such as: +Consider some SQL query from a regression test. After parsing, it is easy to mutate the resulting AST (abstract syntax tree, an internal representation of the parsed query) before execution to introduce random changes into the query. For strings and arrays, we make random modifications such as inserting a random character or doubling the string. For numbers, there are well-known Bad Numbers such as 0, 1, powers of two and nearby, integer limits, `NaN`. `NaN`s proved to be especially efficient in finding bugs, because you can often have some alternative branches in your numeric code, but for a `NaN`, both branches hold (or not) simultaneously, so this leads to nasty effects. + +Another interesting thing we can do is change the arguments of functions, or the list of expressions in `SELECT`. Naturally, all the interesting arguments can be taken from other test queries. Same goes for changing the tables used in the queries. When the fuzzer runs in CI, it runs queries from all the SQL tests in random order, mixing in the parts of query from different tests, so that we can eventually test all the possible permutations of our features. + +The core implementation of the fuzzer is relatively small, consisting of about 700 lines of C++ code. A prototype was made in a couple of days, but naturally it took significantly longer to polish it and to start routinely using it in CI. It is very productive and let us find more than 200 bugs already (see the label [fuzz](https://github.com/ClickHouse/ClickHouse/labels/fuzz) on GitHub), some of which are serious logic errors or even memory errors. When we only started, we could segfault the server or make it enter a never-ending loop with simplest read-only queries such as `SELECT arrayReverseFill(x -> (x < 10), [])` or `SELECT geoDistance(0., 0., -inf, 1.)`. Of course I couldn't resist bringing down our [public playground](https://gh-api.clickhouse.tech/play?user=play#LS0gWW91IGNhbiBxdWVyeSB0aGUgR2l0SHViIGhpc3RvcnkgZGF0YSBoZXJlLiBTZWUgaHR0cHM6Ly9naC5jbGlja2hvdXNlLnRlY2gvZXhwbG9yZXIvIGZvciB0aGUgZGVzY3JpcHRpb24gYW5kIGV4YW1wbGUgcXVlcmllcy4Kc2VsZWN0ICdoZWxsbyB3b3JsZCc=) with some of these queries, and was content to see that the server soon restarts correctly. These queries are actually minified by hand, normally the fuzzer would generate something barely legible such as: ``` SELECT (val + 257, @@ -110,68 +43,19 @@ ANY LEFT JOIN FROM system.one ) AS s2 ON (val + 100) = (rval * 7) ``` -In principle, we could add automated test case minification by modifying AST in the -same vein with fuzzing. This is somewhat complicated by the fact that the server dies -after every, excuse my pun, successfully failed query, so we didn't implement it yet. +In principle, we could add automated test case minification by modifying AST in the same vein with fuzzing. This is somewhat complicated by the fact that the server dies after every, excuse my pun, successfully failed query, so we didn't implement it yet. -Not all errors the fuzzer finds are significant, some of them are pretty boring and -harmless, such as a wrong error code for an -out-of-bounds argument. We still try to fix all of them, -because this lets us ensure that under normal operation, the fuzzer doesn't -find any errors. This is similar to the approach usually taken with compiler -warnings and other optional diagnostics -- it's better to fix or disable every -single case, so that you can be sure you have no diagnostics if everything is -OK, and it's easy to notice new problems. +Not all errors the fuzzer finds are significant, some of them are pretty boring and harmless, such as a wrong error code for an out-of-bounds argument. We still try to fix all of them, because this lets us ensure that under normal operation, the fuzzer doesn't find any errors. This is similar to the approach usually taken with compiler warnings and other optional diagnostics — it's better to fix or disable every single case, so that you can be sure you have no diagnostics if everything is OK, and it's easy to notice new problems. -After fixing the majority of pre-existing error, this fuzzer became efficient -for finding errors in new features. Pull requests introducing new features -normally add an SQL test, and we pay extra attention to the new tests when -fuzzing, generating more permutations for them. Even if the coverage of the -test is not sufficient, there is a good chance that the fuzzer will find the -missing corner cases. So when we see that all the fuzzer runs in different -configurations have failed for a particular pull request, this almost always -means that it introduces a new bug. When developing a feature that requires -new grammar, it is also helpful to add fuzzing support for it. I did this for -window functions early in the development, and it helped me find several bugs. +After fixing the majority of pre-existing error, this fuzzer became efficient for finding errors in new features. Pull requests introducing new features normally add an SQL test, and we pay extra attention to the new tests when fuzzing, generating more permutations for them. Even if the coverage of the test is not sufficient, there is a good chance that the fuzzer will find the missing corner cases. So when we see that all the fuzzer runs in different configurations have failed for a particular pull request, this almost always means that it introduces a new bug. When developing a feature that requires new grammar, it is also helpful to add fuzzing support for it. I did this for window functions early in the development, and it helped me find several bugs. -A major factor that makes fuzzing really efficient is that we have a lot of -assertions and other checks of program logic in our code. For debug-only -checks, we use the plain `assert` macro from ``. For checks that are -needed even in release mode, we use an exception with a special code -`LOGICAL_ERROR` that signifies an internal program error. We did some work to -ensure that these errors are distinct from errors caused by the wrong user -actions. A user error reported for a randomly generated query is normal (e.g. -it references some non-existent columns), but when we see an internal program -error, we know that it's definitely a bug, same as an assertion. Of course, -even without assertions, you get some checks for memory errors provided by the -OS (segfaults). Various kinds of sanitizers are also very useful in conjunction -with fuzzing. We run this fuzzer under clang's Address, Memory, -UndefinedBehavior and Thread sanitizers, as we do for most of our tests. +A major factor that makes fuzzing really efficient is that we have a lot of assertions and other checks of program logic in our code. For debug-only checks, we use the plain `assert` macro from ``. For checks that are needed even in release mode, we use an exception with a special code `LOGICAL_ERROR` that signifies an internal program error. We did some work to ensure that these errors are distinct from errors caused by the wrong user actions. A user error reported for a randomly generated query is normal (e.g. it references some non-existent columns), but when we see an internal program error, we know that it's definitely a bug, same as an assertion. Of course, even without assertions, you get some checks for memory errors provided by the OS (segfaults). Various kinds of sanitizers are also very useful in conjunction with fuzzing. We run this fuzzer under clang's Address, Memory, UndefinedBehavior and Thread sanitizers, as we do for most of our tests. -To see for yourself how it works, you only need the normal ClickHouse client. -Start `clickhouse-client --query-fuzzer-runs=100`, enter any query, and enjoy -the client going crazy and running a hundred of random queries instead. All -queries from the current session become a source for expressions for fuzzing, -so try entering several different queries to get more interesting results. Be -careful not to do this in production! When you do this experiment, you'll soon -notice that the fuzzer tends to generate queries that take very long to run. This -is why for the CI fuzzer runs we have to configure the server to limit query -execution time, memory usage and so on using the corresponding [server -settings](https://clickhouse.tech/docs/en/operations/settings/query-complexity/#:~:text=In%20the%20default%20configuration%20file,query%20within%20a%20single%20server.). -We had a hilarious situation after that: the fuzzer figured out how to remove -the limits by generating a `SET max_execution_time = 0` query, and then -generated a never-ending query and failed. Thankfully we were able to defeat -its cleverness by using [settings -constraints](https://clickhouse.tech/docs/en/operations/settings/constraints-on-settings/). +To see for yourself how it works, you only need the normal ClickHouse client. Start `clickhouse-client --query-fuzzer-runs=100`, enter any query, and enjoy the client going crazy and running a hundred of random queries instead. All queries from the current session become a source for expressions for fuzzing, so try entering several different queries to get more interesting results. Be careful not to do this in production! When you do this experiment, you'll soon notice that the fuzzer tends to generate queries that take very long to run. This is why for the CI fuzzer runs we have to configure the server to limit query execution time, memory usage and so on using the corresponding [server settings](https://clickhouse.tech/docs/en/operations/settings/query-complexity/#:~:text=In%20the%20default%20configuration%20file,query%20within%20a%20single%20server.). We had a hilarious situation after that: the fuzzer figured out how to remove the limits by generating a `SET max_execution_time = 0` query, and then generated a never-ending query and failed. Thankfully we were able to defeat its cleverness by using [settings constraints](https://clickhouse.tech/docs/en/operations/settings/constraints-on-settings/). -The AST-based fuzzer we discussed is only one of the many kinds of fuzzers we -have in ClickHouse. There is a [talk](https://www.youtube.com/watch?v=GbmK84ZwSeI&t=4481s) (in Russian, [slides are here](https://presentations.clickhouse.tech/cpp_siberia_2021/)) by Alexey Milovidov that -explores all the fuzzers we have. Another interesting -recent development is application of pivoted query synthesis technique, -implemented in [SQLancer](https://github.com/sqlancer/sqlancer), to ClickHouse. -The authors are going to give [a talk about -this](https://heisenbug-piter.ru/2021/spb/talks/nr1cwknssdodjkqgzsbvh/) soon, -so stay tuned. +## Other Fuzzers + +The AST-based fuzzer we discussed is only one of the many kinds of fuzzers we have in ClickHouse. There is a [talk](https://www.youtube.com/watch?v=GbmK84ZwSeI&t=4481s) (in Russian, [slides are here](https://presentations.clickhouse.tech/cpp_siberia_2021/)) by Alexey Milovidov that explores all the fuzzers we have. Another interesting recent development is application of pivoted query synthesis technique, implemented in [SQLancer](https://github.com/sqlancer/sqlancer), to ClickHouse. The authors are going to give [a talk about this](https://heisenbug-piter.ru/2021/spb/talks/nr1cwknssdodjkqgzsbvh/) soon, so stay tuned. 12-08-21 [Alexander Kuzmenkov](https://github.com/akuzm) From 69b2b2a159e18931f3d3f553908a934ce92ad2cd Mon Sep 17 00:00:00 2001 From: Azat Khuzhin Date: Thu, 11 Mar 2021 21:41:27 +0300 Subject: [PATCH 343/544] Fix fsync_part_directory for horizontal merge --- .../MergeTree/MergeTreeDataMergerMutator.cpp | 9 +++++---- .../01643_merge_tree_fsync_smoke.reference | 2 ++ .../0_stateless/01643_merge_tree_fsync_smoke.sql | 13 +++++++++++++ 3 files changed, 20 insertions(+), 4 deletions(-) diff --git a/src/Storages/MergeTree/MergeTreeDataMergerMutator.cpp b/src/Storages/MergeTree/MergeTreeDataMergerMutator.cpp index f2f8172837c..b1a3e108013 100644 --- a/src/Storages/MergeTree/MergeTreeDataMergerMutator.cpp +++ b/src/Storages/MergeTree/MergeTreeDataMergerMutator.cpp @@ -757,6 +757,8 @@ MergeTreeData::MutableDataPartPtr MergeTreeDataMergerMutator::mergePartsToTempor std::unique_ptr rows_sources_write_buf; std::optional column_sizes; + SyncGuardPtr sync_guard; + if (chosen_merge_algorithm == MergeAlgorithm::Vertical) { tmp_disk->createDirectories(new_part_tmp_path); @@ -769,6 +771,9 @@ MergeTreeData::MutableDataPartPtr MergeTreeDataMergerMutator::mergePartsToTempor part->accumulateColumnSizes(merged_column_to_size); column_sizes = ColumnSizeEstimator(merged_column_to_size, merging_column_names, gathering_column_names); + + if (data.getSettings()->fsync_part_directory) + sync_guard = disk->getDirectorySyncGuard(new_part_tmp_path); } else { @@ -778,10 +783,6 @@ MergeTreeData::MutableDataPartPtr MergeTreeDataMergerMutator::mergePartsToTempor gathering_column_names.clear(); } - SyncGuardPtr sync_guard; - if (data.getSettings()->fsync_part_directory) - sync_guard = disk->getDirectorySyncGuard(new_part_tmp_path); - /** Read from all parts, merge and write into a new one. * In passing, we calculate expression for sorting. */ diff --git a/tests/queries/0_stateless/01643_merge_tree_fsync_smoke.reference b/tests/queries/0_stateless/01643_merge_tree_fsync_smoke.reference index 654db9dbc86..f57d5df6efd 100644 --- a/tests/queries/0_stateless/01643_merge_tree_fsync_smoke.reference +++ b/tests/queries/0_stateless/01643_merge_tree_fsync_smoke.reference @@ -10,3 +10,5 @@ wide fsync_after_insert,fsync_part_directory 1 memory in_memory_parts_insert_sync 1 +wide fsync_part_directory,vertical +1 diff --git a/tests/queries/0_stateless/01643_merge_tree_fsync_smoke.sql b/tests/queries/0_stateless/01643_merge_tree_fsync_smoke.sql index 21ebb607693..644cf063a33 100644 --- a/tests/queries/0_stateless/01643_merge_tree_fsync_smoke.sql +++ b/tests/queries/0_stateless/01643_merge_tree_fsync_smoke.sql @@ -4,34 +4,47 @@ select 'default'; create table data_01643 (key Int) engine=MergeTree() order by key; insert into data_01643 values (1); select * from data_01643; +optimize table data_01643 final; drop table data_01643; select 'compact fsync_after_insert'; create table data_01643 (key Int) engine=MergeTree() order by key settings min_rows_for_wide_part=2, fsync_after_insert=1; insert into data_01643 values (1); select * from data_01643; +optimize table data_01643 final; drop table data_01643; select 'compact fsync_after_insert,fsync_part_directory'; create table data_01643 (key Int) engine=MergeTree() order by key settings min_rows_for_wide_part=2, fsync_after_insert=1, fsync_part_directory=1; insert into data_01643 values (1); select * from data_01643; +optimize table data_01643 final; drop table data_01643; select 'wide fsync_after_insert'; create table data_01643 (key Int) engine=MergeTree() order by key settings min_bytes_for_wide_part=0, fsync_after_insert=1; insert into data_01643 values (1); select * from data_01643; +optimize table data_01643 final; drop table data_01643; select 'wide fsync_after_insert,fsync_part_directory'; create table data_01643 (key Int) engine=MergeTree() order by key settings min_bytes_for_wide_part=0, fsync_after_insert=1, fsync_part_directory=1; insert into data_01643 values (1); select * from data_01643; +optimize table data_01643 final; drop table data_01643; select 'memory in_memory_parts_insert_sync'; create table data_01643 (key Int) engine=MergeTree() order by key settings min_rows_for_compact_part=2, in_memory_parts_insert_sync=1, fsync_after_insert=1, fsync_part_directory=1; insert into data_01643 values (1); select * from data_01643; +optimize table data_01643 final; +drop table data_01643; + +select 'wide fsync_part_directory,vertical'; +create table data_01643 (key Int) engine=MergeTree() order by key settings min_bytes_for_wide_part=0, fsync_part_directory=1, enable_vertical_merge_algorithm=1, vertical_merge_algorithm_min_rows_to_activate=1, vertical_merge_algorithm_min_columns_to_activate=1; +insert into data_01643 values (1); +select * from data_01643; +optimize table data_01643 final; drop table data_01643; From 65f90f2ce9ea9e9d4076f06c58ddd981c82cc098 Mon Sep 17 00:00:00 2001 From: Azat Khuzhin Date: Thu, 11 Mar 2021 21:52:10 +0300 Subject: [PATCH 344/544] Fix distributed requests cancellation with async_socket_for_remote=1 Before this patch for distributed queries, that requires cancellation (simple select from multiple shards with limit, i.e. `select * from remote('127.{2,3}', system.numbers) limit 100`) it is very easy to trigger the situation when remote shard is in the middle of sending Data block while the initiator already send Cancel and expecting some new packet, but it will receive not new packet, but part of the Data block that was in the middle of sending before cancellation, and this will lead to some various errors, like: - Unknown packet X from server Y - Unexpected packet from server Y - and a lot more... Fix this, by correctly waiting for the pending packet before cancellation. It is not very easy to write a test, since localhost is too fast. Also note, that it is not possible to get these errors with hedged requests (use_hedged_requests=1) since handle fibers correctly. But it had been disabled by default for 21.3 in #21534, while async_socket_for_remote is enabled by default. --- .../RemoteQueryExecutorReadContext.cpp | 15 +++++++++++---- src/DataStreams/RemoteQueryExecutorReadContext.h | 4 ++-- 2 files changed, 13 insertions(+), 6 deletions(-) diff --git a/src/DataStreams/RemoteQueryExecutorReadContext.cpp b/src/DataStreams/RemoteQueryExecutorReadContext.cpp index c2a65f02d08..11cc2dcd8e4 100644 --- a/src/DataStreams/RemoteQueryExecutorReadContext.cpp +++ b/src/DataStreams/RemoteQueryExecutorReadContext.cpp @@ -104,11 +104,11 @@ void RemoteQueryExecutorReadContext::setConnectionFD(int fd, const Poco::Timespa connection_fd_description = fd_description; } -bool RemoteQueryExecutorReadContext::checkTimeout() const +bool RemoteQueryExecutorReadContext::checkTimeout(bool blocking) const { try { - return checkTimeoutImpl(); + return checkTimeoutImpl(blocking); } catch (DB::Exception & e) { @@ -118,13 +118,13 @@ bool RemoteQueryExecutorReadContext::checkTimeout() const } } -bool RemoteQueryExecutorReadContext::checkTimeoutImpl() const +bool RemoteQueryExecutorReadContext::checkTimeoutImpl(bool blocking) const { /// Wait for epoll will not block if it was polled externally. epoll_event events[3]; events[0].data.fd = events[1].data.fd = events[2].data.fd = -1; - int num_events = epoll.getManyReady(3, events,/* blocking = */ false); + int num_events = epoll.getManyReady(3, events, blocking); bool is_socket_ready = false; bool is_pipe_alarmed = false; @@ -184,9 +184,16 @@ bool RemoteQueryExecutorReadContext::resumeRoutine() void RemoteQueryExecutorReadContext::cancel() { std::lock_guard guard(fiber_lock); + /// It is safe to just destroy fiber - we are not in the process of reading from socket. boost::context::fiber to_destroy = std::move(fiber); + while (is_read_in_progress.load(std::memory_order_relaxed)) + { + checkTimeout(/* blocking= */ true); + to_destroy = std::move(to_destroy).resume(); + } + /// Send something to pipe to cancel executor waiting. uint64_t buf = 0; while (-1 == write(pipe_fd[1], &buf, sizeof(buf))) diff --git a/src/DataStreams/RemoteQueryExecutorReadContext.h b/src/DataStreams/RemoteQueryExecutorReadContext.h index cb6421f78d0..5fbe52469cd 100644 --- a/src/DataStreams/RemoteQueryExecutorReadContext.h +++ b/src/DataStreams/RemoteQueryExecutorReadContext.h @@ -54,8 +54,8 @@ public: explicit RemoteQueryExecutorReadContext(IConnections & connections_); ~RemoteQueryExecutorReadContext(); - bool checkTimeout() const; - bool checkTimeoutImpl() const; + bool checkTimeout(bool blocking = false) const; + bool checkTimeoutImpl(bool blocking) const; void setConnectionFD(int fd, const Poco::Timespan & timeout = 0, const std::string & fd_description = ""); void setTimer() const; From c2372420ee69f9fc3125c6109269f91d9e5aecdb Mon Sep 17 00:00:00 2001 From: Azat Khuzhin Date: Thu, 11 Mar 2021 22:35:49 +0300 Subject: [PATCH 345/544] Add type conversion for StorageJoin (previously led to SIGSEGV) Found with fuzzer under UBsan [1]: [1]: https://clickhouse-test-reports.s3.yandex.net/21579/61d40c3600ba6a1c6d6c0cf4919a3cdaebb3a31f/fuzzer_ubsan/report.html#fail1 --- src/Interpreters/ExpressionAnalyzer.cpp | 7 +++++++ tests/queries/0_stateless/00561_storage_join.reference | 1 + tests/queries/0_stateless/00561_storage_join.sql | 8 ++++++++ 3 files changed, 16 insertions(+) diff --git a/src/Interpreters/ExpressionAnalyzer.cpp b/src/Interpreters/ExpressionAnalyzer.cpp index 2e2a8b9c4d5..03bd370fb8c 100644 --- a/src/Interpreters/ExpressionAnalyzer.cpp +++ b/src/Interpreters/ExpressionAnalyzer.cpp @@ -854,6 +854,13 @@ JoinPtr SelectQueryExpressionAnalyzer::makeTableJoin( return join; } } + else + { + const ColumnsWithTypeAndName & right_sample_columns = subquery_for_join.sample_block.getColumnsWithTypeAndName(); + bool need_convert = syntax->analyzed_join->applyJoinKeyConvert(left_sample_columns, right_sample_columns); + if (need_convert) + subquery_for_join.addJoinActions(std::make_shared(syntax->analyzed_join->rightConvertingActions())); + } return subquery_for_join.join; } diff --git a/tests/queries/0_stateless/00561_storage_join.reference b/tests/queries/0_stateless/00561_storage_join.reference index 867b945ba1c..2fe12a38360 100644 --- a/tests/queries/0_stateless/00561_storage_join.reference +++ b/tests/queries/0_stateless/00561_storage_join.reference @@ -2,3 +2,4 @@ 2 22 92 82 123457 1 11 91 81 123456 2 22 92 82 123457 +11 1 91 81 123456 diff --git a/tests/queries/0_stateless/00561_storage_join.sql b/tests/queries/0_stateless/00561_storage_join.sql index 62ca80d31fe..01e66d0c380 100644 --- a/tests/queries/0_stateless/00561_storage_join.sql +++ b/tests/queries/0_stateless/00561_storage_join.sql @@ -36,5 +36,13 @@ from ( ) js1 SEMI LEFT JOIN joinbug_join using id2; +/* type conversion */ +SELECT * FROM +( + SELECT toUInt32(11) AS id2 +) AS js1 +SEMI LEFT JOIN joinbug_join USING (id2); + + DROP TABLE joinbug; DROP TABLE joinbug_join; From 290a6d273eafcd2b7d7fad8624e7f2d7982a03bf Mon Sep 17 00:00:00 2001 From: Denis Glazachev Date: Fri, 12 Mar 2021 00:41:10 +0400 Subject: [PATCH 346/544] Add Kerberos support for authenticating existing users when accessing over HTTP --- cmake/find/krb5.cmake | 4 +- contrib/krb5-cmake/CMakeLists.txt | 50 +- contrib/krb5-cmake/autoconf_darwin.h | 764 +++++++++++++++++ .../{autoconf.h => autoconf_linux.h} | 0 .../external-authenticators/ldap.md | 8 +- .../en/sql-reference/statements/alter/user.md | 6 +- .../sql-reference/statements/create/user.md | 7 +- programs/client/Client.cpp | 2 +- programs/local/LocalServer.cpp | 2 +- programs/server/config.xml | 21 + programs/server/users.xml | 11 +- src/Access/AccessControlManager.cpp | 4 +- src/Access/AccessControlManager.h | 2 +- src/Access/AllowedClientHosts.h | 2 +- src/Access/Authentication.cpp | 110 ++- src/Access/Authentication.h | 117 ++- src/Access/Credentials.cpp | 86 ++ src/Access/Credentials.h | 55 ++ src/Access/ExternalAuthenticators.cpp | 174 +++- src/Access/ExternalAuthenticators.h | 27 +- src/Access/GSSAcceptor.cpp | 469 ++++++++++ src/Access/GSSAcceptor.h | 66 ++ src/Access/IAccessStorage.cpp | 41 +- src/Access/IAccessStorage.h | 15 +- src/Access/LDAPAccessStorage.cpp | 88 +- src/Access/LDAPAccessStorage.h | 30 +- src/Access/LDAPClient.cpp | 67 +- src/Access/LDAPClient.h | 104 ++- src/Access/LDAPParams.h | 120 --- src/Access/MultipleAccessStorage.cpp | 11 +- src/Access/MultipleAccessStorage.h | 2 +- src/Access/UsersConfigAccessStorage.cpp | 18 +- src/Access/ya.make | 2 + src/CMakeLists.txt | 1 + src/Common/ErrorCodes.cpp | 1 + src/Core/config_core.h.in | 1 + src/Interpreters/Context.cpp | 34 +- src/Interpreters/Context.h | 17 +- src/Parsers/ASTCreateUserQuery.cpp | 27 +- src/Parsers/ASTCreateUserQuery.h | 4 +- src/Parsers/ParserCreateUserQuery.cpp | 36 +- src/Parsers/ParserCreateUserQuery.h | 4 +- src/Server/HTTPHandler.cpp | 215 ++++- src/Server/HTTPHandler.h | 19 + src/Server/HTTPHandlerFactory.cpp | 2 + ...StorageSystemBuildOptions.generated.cpp.in | 1 + src/Storages/System/StorageSystemUsers.cpp | 10 +- tests/testflows/helpers/cluster.py | 52 +- .../kerberos/configs/clickhouse/common.xml | 6 + .../configs/clickhouse/config.d/logs.xml | 17 + .../configs/clickhouse/config.d/ports.xml | 5 + .../configs/clickhouse/config.d/remote.xml | 107 +++ .../configs/clickhouse/config.d/ssl.xml | 17 + .../configs/clickhouse/config.d/storage.xml | 20 + .../configs/clickhouse/config.d/zookeeper.xml | 10 + .../kerberos/configs/clickhouse/config.xml | 440 ++++++++++ .../configs/clickhouse/ssl/dhparam.pem | 8 + .../configs/clickhouse/ssl/server.crt | 19 + .../configs/clickhouse/ssl/server.key | 28 + .../kerberos/configs/clickhouse/users.xml | 133 +++ .../configs/clickhouse1/config.d/kerberos.xml | 5 + .../configs/clickhouse1/config.d/macros.xml | 8 + .../clickhouse1/users.d/kerberos-users.xml | 10 + .../configs/clickhouse2/config.d/kerberos.xml | 5 + .../configs/clickhouse2/config.d/macros.xml | 8 + .../configs/clickhouse3/config.d/macros.xml | 8 + .../kerberos/configs/kerberos/etc/krb5.conf | 38 + .../configs/kerberos/etc/krb5kdc/kdc.conf | 15 + .../configs/kerberos/etc/supervisord.conf | 31 + .../docker-compose/clickhouse-service.yml | 32 + .../docker-compose/docker-compose.yml | 75 ++ .../docker-compose/kerberos-service.yml | 27 + .../docker-compose/zookeeper-service.yml | 18 + tests/testflows/kerberos/regression.py | 38 + .../kerberos/requirements/requirements.md | 281 ++++++ .../kerberos/requirements/requirements.py | 800 ++++++++++++++++++ tests/testflows/kerberos/tests/common.py | 225 +++++ tests/testflows/kerberos/tests/config.py | 163 ++++ tests/testflows/kerberos/tests/generic.py | 332 ++++++++ tests/testflows/kerberos/tests/parallel.py | 204 +++++ .../requirements/requirements.md | 2 +- .../requirements/requirements.py | 4 +- .../ldap/authentication/tests/common.py | 2 +- tests/testflows/regression.py | 1 + 84 files changed, 5577 insertions(+), 474 deletions(-) create mode 100644 contrib/krb5-cmake/autoconf_darwin.h rename contrib/krb5-cmake/{autoconf.h => autoconf_linux.h} (100%) create mode 100644 src/Access/Credentials.cpp create mode 100644 src/Access/Credentials.h create mode 100644 src/Access/GSSAcceptor.cpp create mode 100644 src/Access/GSSAcceptor.h delete mode 100644 src/Access/LDAPParams.h create mode 100644 tests/testflows/kerberos/configs/clickhouse/common.xml create mode 100644 tests/testflows/kerberos/configs/clickhouse/config.d/logs.xml create mode 100644 tests/testflows/kerberos/configs/clickhouse/config.d/ports.xml create mode 100644 tests/testflows/kerberos/configs/clickhouse/config.d/remote.xml create mode 100644 tests/testflows/kerberos/configs/clickhouse/config.d/ssl.xml create mode 100644 tests/testflows/kerberos/configs/clickhouse/config.d/storage.xml create mode 100644 tests/testflows/kerberos/configs/clickhouse/config.d/zookeeper.xml create mode 100644 tests/testflows/kerberos/configs/clickhouse/config.xml create mode 100644 tests/testflows/kerberos/configs/clickhouse/ssl/dhparam.pem create mode 100644 tests/testflows/kerberos/configs/clickhouse/ssl/server.crt create mode 100644 tests/testflows/kerberos/configs/clickhouse/ssl/server.key create mode 100644 tests/testflows/kerberos/configs/clickhouse/users.xml create mode 100644 tests/testflows/kerberos/configs/clickhouse1/config.d/kerberos.xml create mode 100644 tests/testflows/kerberos/configs/clickhouse1/config.d/macros.xml create mode 100644 tests/testflows/kerberos/configs/clickhouse1/users.d/kerberos-users.xml create mode 100644 tests/testflows/kerberos/configs/clickhouse2/config.d/kerberos.xml create mode 100644 tests/testflows/kerberos/configs/clickhouse2/config.d/macros.xml create mode 100644 tests/testflows/kerberos/configs/clickhouse3/config.d/macros.xml create mode 100644 tests/testflows/kerberos/configs/kerberos/etc/krb5.conf create mode 100644 tests/testflows/kerberos/configs/kerberos/etc/krb5kdc/kdc.conf create mode 100644 tests/testflows/kerberos/configs/kerberos/etc/supervisord.conf create mode 100644 tests/testflows/kerberos/docker-compose/clickhouse-service.yml create mode 100644 tests/testflows/kerberos/docker-compose/docker-compose.yml create mode 100644 tests/testflows/kerberos/docker-compose/kerberos-service.yml create mode 100644 tests/testflows/kerberos/docker-compose/zookeeper-service.yml create mode 100644 tests/testflows/kerberos/regression.py create mode 100644 tests/testflows/kerberos/requirements/requirements.md create mode 100644 tests/testflows/kerberos/requirements/requirements.py create mode 100644 tests/testflows/kerberos/tests/common.py create mode 100644 tests/testflows/kerberos/tests/config.py create mode 100644 tests/testflows/kerberos/tests/generic.py create mode 100644 tests/testflows/kerberos/tests/parallel.py diff --git a/cmake/find/krb5.cmake b/cmake/find/krb5.cmake index bd9c8e239cd..49b7462b710 100644 --- a/cmake/find/krb5.cmake +++ b/cmake/find/krb5.cmake @@ -5,8 +5,8 @@ if (NOT EXISTS "${ClickHouse_SOURCE_DIR}/contrib/krb5/README") set (ENABLE_KRB5 0) endif () -if (NOT CMAKE_SYSTEM_NAME MATCHES "Linux") - message (WARNING "krb5 disabled in non-Linux environments") +if (NOT CMAKE_SYSTEM_NAME MATCHES "Linux" AND NOT (CMAKE_SYSTEM_NAME MATCHES "Darwin" AND NOT CMAKE_CROSSCOMPILING)) + message (WARNING "krb5 disabled in non-Linux and non-native-Darwin environments") set (ENABLE_KRB5 0) endif () diff --git a/contrib/krb5-cmake/CMakeLists.txt b/contrib/krb5-cmake/CMakeLists.txt index f88402df1fa..fce7fbc582a 100644 --- a/contrib/krb5-cmake/CMakeLists.txt +++ b/contrib/krb5-cmake/CMakeLists.txt @@ -474,13 +474,6 @@ add_custom_command( WORKING_DIRECTORY "${KRB5_SOURCE_DIR}/util/et" ) -add_custom_target( - CREATE_COMPILE_ET ALL - DEPENDS ${KRB5_SOURCE_DIR}/util/et/compile_et - COMMENT "creating compile_et" - VERBATIM -) - file(GLOB_RECURSE ET_FILES "${KRB5_SOURCE_DIR}/*.et" ) @@ -531,7 +524,7 @@ add_custom_command( add_custom_target( - ERROR_MAP_H ALL + ERROR_MAP_H DEPENDS ${KRB5_SOURCE_DIR}/lib/gssapi/krb5/error_map.h COMMENT "generating error_map.h" VERBATIM @@ -544,14 +537,14 @@ add_custom_command( ) add_custom_target( - ERRMAP_H ALL + ERRMAP_H DEPENDS ${KRB5_SOURCE_DIR}/lib/gssapi/generic/errmap.h COMMENT "generating errmap.h" VERBATIM ) add_custom_target( - KRB_5_H ALL + KRB_5_H DEPENDS ${CMAKE_CURRENT_BINARY_DIR}/include/krb5/krb5.h COMMENT "generating krb5.h" VERBATIM @@ -564,15 +557,19 @@ add_dependencies( ERRMAP_H ERROR_MAP_H KRB_5_H - ) +) preprocess_et(processed_et_files ${ET_FILES}) -add_custom_command( - OUTPUT ${KRB5_SOURCE_DIR}/lib/gssapi/generic/errmap.h - COMMAND perl -w -I../../../util ../../../util/gen.pl bimap errmap.h NAME=mecherrmap LEFT=OM_uint32 RIGHT=struct\ mecherror LEFTPRINT=print_OM_uint32 RIGHTPRINT=mecherror_print LEFTCMP=cmp_OM_uint32 RIGHTCMP=mecherror_cmp - WORKING_DIRECTORY "${KRB5_SOURCE_DIR}/lib/gssapi/generic" -) +if(CMAKE_SYSTEM_NAME MATCHES "Darwin") + add_custom_command( + OUTPUT ${CMAKE_CURRENT_BINARY_DIR}/include_private/kcmrpc.h ${CMAKE_CURRENT_BINARY_DIR}/include_private/kcmrpc.c + COMMAND mig -header kcmrpc.h -user kcmrpc.c -sheader /dev/null -server /dev/null -I${KRB5_SOURCE_DIR}/lib/krb5/ccache ${KRB5_SOURCE_DIR}/lib/krb5/ccache/kcmrpc.defs + WORKING_DIRECTORY "${CMAKE_CURRENT_BINARY_DIR}/include_private" + ) + + list(APPEND ALL_SRCS ${CMAKE_CURRENT_BINARY_DIR}/include_private/kcmrpc.c) +endif() target_sources(${KRB5_LIBRARY} PRIVATE ${ALL_SRCS} @@ -604,6 +601,25 @@ file(COPY ${KRB5_SOURCE_DIR}/util/et/com_err.h DESTINATION ${CMAKE_CURRENT_BINARY_DIR}/include/ ) +file(COPY ${CMAKE_CURRENT_SOURCE_DIR}/osconf.h + DESTINATION ${CMAKE_CURRENT_BINARY_DIR}/include_private/ +) + +file(COPY ${CMAKE_CURRENT_SOURCE_DIR}/profile.h + DESTINATION ${CMAKE_CURRENT_BINARY_DIR}/include_private/ +) + +string(TOLOWER "${CMAKE_SYSTEM_NAME}" _system_name) + +file(COPY ${CMAKE_CURRENT_SOURCE_DIR}/autoconf_${_system_name}.h + DESTINATION ${CMAKE_CURRENT_BINARY_DIR}/include_private/ +) + +file(RENAME + ${CMAKE_CURRENT_BINARY_DIR}/include_private/autoconf_${_system_name}.h + ${CMAKE_CURRENT_BINARY_DIR}/include_private/autoconf.h +) + file(MAKE_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/include/krb5 ) @@ -633,7 +649,7 @@ target_include_directories(${KRB5_LIBRARY} PUBLIC ) target_include_directories(${KRB5_LIBRARY} PRIVATE - ${CMAKE_CURRENT_SOURCE_DIR} #for autoconf.h + ${CMAKE_CURRENT_BINARY_DIR}/include_private # For autoconf.h and other generated headers. ${KRB5_SOURCE_DIR} ${KRB5_SOURCE_DIR}/include ${KRB5_SOURCE_DIR}/lib/gssapi/mechglue diff --git a/contrib/krb5-cmake/autoconf_darwin.h b/contrib/krb5-cmake/autoconf_darwin.h new file mode 100644 index 00000000000..965e2f02997 --- /dev/null +++ b/contrib/krb5-cmake/autoconf_darwin.h @@ -0,0 +1,764 @@ +/* include/autoconf.h. Generated from autoconf.h.in by configure. */ +/* include/autoconf.h.in. Generated from configure.in by autoheader. */ + + +#ifndef KRB5_AUTOCONF_H +#define KRB5_AUTOCONF_H + + +/* Define if AES-NI support is enabled */ +/* #undef AESNI */ + +/* Define if socket can't be bound to 0.0.0.0 */ +/* #undef BROKEN_STREAMS_SOCKETS */ + +/* Define if va_list objects can be simply copied by assignment. */ +/* #undef CAN_COPY_VA_LIST */ + +/* Define to reduce code size even if it means more cpu usage */ +/* #undef CONFIG_SMALL */ + +/* Define if __attribute__((constructor)) works */ +#define CONSTRUCTOR_ATTR_WORKS 1 + +/* Define to default ccache name */ +#define DEFCCNAME "FILE:/tmp/krb5cc_%{uid}" + +/* Define to default client keytab name */ +#define DEFCKTNAME "FILE:/etc/krb5/user/%{euid}/client.keytab" + +/* Define to default keytab name */ +#define DEFKTNAME "FILE:/etc/krb5.keytab" + +/* Define if library initialization should be delayed until first use */ +#define DELAY_INITIALIZER 1 + +/* Define if __attribute__((destructor)) works */ +#define DESTRUCTOR_ATTR_WORKS 1 + +/* Define to disable PKINIT plugin support */ +#define DISABLE_PKINIT 1 + +/* Define if LDAP KDB support within the Kerberos library (mainly ASN.1 code) + should be enabled. */ +/* #undef ENABLE_LDAP */ + +/* Define if translation functions should be used. */ +/* #undef ENABLE_NLS */ + +/* Define if thread support enabled */ +#define ENABLE_THREADS 1 + +/* Define as return type of endrpcent */ +#define ENDRPCENT_TYPE void + +/* Define if Fortuna PRNG is selected */ +#define FORTUNA 1 + +/* Define to the type of elements in the array set by `getgroups'. Usually + this is either `int' or `gid_t'. */ +#define GETGROUPS_T gid_t + +/* Define if gethostbyname_r returns int rather than struct hostent * */ +/* #undef GETHOSTBYNAME_R_RETURNS_INT */ + +/* Type of getpeername second argument. */ +#define GETPEERNAME_ARG3_TYPE GETSOCKNAME_ARG3_TYPE + +/* Define if getpwnam_r exists but takes only 4 arguments (e.g., POSIX draft 6 + implementations like some Solaris releases). */ +/* #undef GETPWNAM_R_4_ARGS */ + +/* Define if getpwnam_r returns an int */ +#define GETPWNAM_R_RETURNS_INT 1 + +/* Define if getpwuid_r exists but takes only 4 arguments (e.g., POSIX draft 6 + implementations like some Solaris releases). */ +/* #undef GETPWUID_R_4_ARGS */ + +/* Define if getservbyname_r returns int rather than struct servent * */ +/* #undef GETSERVBYNAME_R_RETURNS_INT */ + +/* Type of pointer target for argument 3 to getsockname */ +#define GETSOCKNAME_ARG3_TYPE socklen_t + +/* Define if gmtime_r returns int instead of struct tm pointer, as on old + HP-UX systems. */ +/* #undef GMTIME_R_RETURNS_INT */ + +/* Define if va_copy macro or function is available. */ +#define HAS_VA_COPY 1 + +/* Define to 1 if you have the `access' function. */ +#define HAVE_ACCESS 1 + +/* Define to 1 if you have the header file. */ +#define HAVE_ALLOCA_H 1 + +/* Define to 1 if you have the header file. */ +#define HAVE_ARPA_INET_H 1 + +/* Define to 1 if you have the `bswap16' function. */ +/* #undef HAVE_BSWAP16 */ + +/* Define to 1 if you have the `bswap64' function. */ +/* #undef HAVE_BSWAP64 */ + +/* Define to 1 if bswap_16 is available via byteswap.h */ +/* #undef HAVE_BSWAP_16 */ + +/* Define to 1 if bswap_64 is available via byteswap.h */ +/* #undef HAVE_BSWAP_64 */ + +/* Define if bt_rseq is available, for recursive btree traversal. */ +#define HAVE_BT_RSEQ 1 + +/* Define to 1 if you have the header file. */ +/* #undef HAVE_BYTESWAP_H */ + +/* Define to 1 if you have the `chmod' function. */ +#define HAVE_CHMOD 1 + +/* Define if cmocka library is available. */ +/* #undef HAVE_CMOCKA */ + +/* Define to 1 if you have the `compile' function. */ +/* #undef HAVE_COMPILE */ + +/* Define if com_err has compatible gettext support */ +#define HAVE_COM_ERR_INTL 1 + +/* Define to 1 if you have the header file. */ +/* #undef HAVE_CPUID_H */ + +/* Define to 1 if you have the `daemon' function. */ +#define HAVE_DAEMON 1 + +/* Define to 1 if you have the declaration of `strerror_r', and to 0 if you + don't. */ +#define HAVE_DECL_STRERROR_R 1 + +/* Define to 1 if you have the header file, and it defines `DIR'. + */ +#define HAVE_DIRENT_H 1 + +/* Define to 1 if you have the header file. */ +#define HAVE_DLFCN_H 1 + +/* Define to 1 if you have the `dn_skipname' function. */ +#define HAVE_DN_SKIPNAME 1 + +/* Define to 1 if you have the header file. */ +/* #undef HAVE_ENDIAN_H */ + +/* Define to 1 if you have the header file. */ +#define HAVE_ERRNO_H 1 + +/* Define to 1 if you have the `fchmod' function. */ +#define HAVE_FCHMOD 1 + +/* Define to 1 if you have the header file. */ +#define HAVE_FCNTL_H 1 + +/* Define to 1 if you have the `flock' function. */ +#define HAVE_FLOCK 1 + +/* Define to 1 if you have the `fnmatch' function. */ +#define HAVE_FNMATCH 1 + +/* Define to 1 if you have the header file. */ +#define HAVE_FNMATCH_H 1 + +/* Define if you have the getaddrinfo function */ +#define HAVE_GETADDRINFO 1 + +/* Define to 1 if you have the `getcwd' function. */ +#define HAVE_GETCWD 1 + +/* Define to 1 if you have the `getenv' function. */ +#define HAVE_GETENV 1 + +/* Define to 1 if you have the `geteuid' function. */ +#define HAVE_GETEUID 1 + +/* Define if gethostbyname_r exists and its return type is known */ +/* #undef HAVE_GETHOSTBYNAME_R */ + +/* Define to 1 if you have the `getnameinfo' function. */ +#define HAVE_GETNAMEINFO 1 + +/* Define if system getopt should be used. */ +#define HAVE_GETOPT 1 + +/* Define if system getopt_long should be used. */ +#define HAVE_GETOPT_LONG 1 + +/* Define if getpwnam_r is available and useful. */ +#define HAVE_GETPWNAM_R 1 + +/* Define if getpwuid_r is available and useful. */ +#define HAVE_GETPWUID_R 1 + +/* Define if getservbyname_r exists and its return type is known */ +/* #undef HAVE_GETSERVBYNAME_R */ + +/* Have the gettimeofday function */ +#define HAVE_GETTIMEOFDAY 1 + +/* Define to 1 if you have the `getusershell' function. */ +#define HAVE_GETUSERSHELL 1 + +/* Define to 1 if you have the `gmtime_r' function. */ +#define HAVE_GMTIME_R 1 + +/* Define to 1 if you have the header file. */ +#define HAVE_IFADDRS_H 1 + +/* Define to 1 if you have the `inet_ntop' function. */ +#define HAVE_INET_NTOP 1 + +/* Define to 1 if you have the `inet_pton' function. */ +#define HAVE_INET_PTON 1 + +/* Define to 1 if the system has the type `int16_t'. */ +#define HAVE_INT16_T 1 + +/* Define to 1 if the system has the type `int32_t'. */ +#define HAVE_INT32_T 1 + +/* Define to 1 if the system has the type `int8_t'. */ +#define HAVE_INT8_T 1 + +/* Define to 1 if you have the header file. */ +#define HAVE_INTTYPES_H 1 + +/* Define to 1 if you have the header file. */ +/* #undef HAVE_KEYUTILS_H */ + +/* Define to 1 if you have the header file. */ +/* #undef HAVE_LBER_H */ + +/* Define to 1 if you have the header file. */ +/* #undef HAVE_LDAP_H */ + +/* Define to 1 if you have the `crypto' library (-lcrypto). */ +#define HAVE_LIBCRYPTO 1 + +/* Define if building with libedit. */ +/* #undef HAVE_LIBEDIT */ + +/* Define to 1 if you have the `nsl' library (-lnsl). */ +/* #undef HAVE_LIBNSL */ + +/* Define to 1 if you have the `resolv' library (-lresolv). */ +#define HAVE_LIBRESOLV 1 + +/* Define to 1 if you have the `socket' library (-lsocket). */ +/* #undef HAVE_LIBSOCKET */ + +/* Define if the util library is available */ +#define HAVE_LIBUTIL 1 + +/* Define to 1 if you have the header file. */ +#define HAVE_LIMITS_H 1 + +/* Define to 1 if you have the `localtime_r' function. */ +#define HAVE_LOCALTIME_R 1 + +/* Define to 1 if you have the header file. */ +#define HAVE_MACHINE_BYTE_ORDER_H 1 + +/* Define to 1 if you have the header file. */ +#define HAVE_MACHINE_ENDIAN_H 1 + +/* Define to 1 if you have the header file. */ +#define HAVE_MEMORY_H 1 + +/* Define to 1 if you have the `mkstemp' function. */ +#define HAVE_MKSTEMP 1 + +/* Define to 1 if you have the header file, and it defines `DIR'. */ +/* #undef HAVE_NDIR_H */ + +/* Define to 1 if you have the header file. */ +#define HAVE_NETDB_H 1 + +/* Define if netdb.h declares h_errno */ +#define HAVE_NETDB_H_H_ERRNO 1 + +/* Define to 1 if you have the header file. */ +#define HAVE_NETINET_IN_H 1 + +/* Define to 1 if you have the `ns_initparse' function. */ +#define HAVE_NS_INITPARSE 1 + +/* Define to 1 if you have the `ns_name_uncompress' function. */ +#define HAVE_NS_NAME_UNCOMPRESS 1 + +/* Define if OpenSSL supports cms. */ +#define HAVE_OPENSSL_CMS 1 + +/* Define to 1 if you have the header file. */ +#define HAVE_PATHS_H 1 + +/* Define if persistent keyrings are supported */ +/* #undef HAVE_PERSISTENT_KEYRING */ + +/* Define to 1 if you have the header file. */ +#define HAVE_POLL_H 1 + +/* Define if #pragma weak references work */ +/* #undef HAVE_PRAGMA_WEAK_REF */ + +/* Define if you have POSIX threads libraries and header files. */ +#define HAVE_PTHREAD 1 + +/* Define to 1 if you have the `pthread_once' function. */ +#define HAVE_PTHREAD_ONCE 1 + +/* Have PTHREAD_PRIO_INHERIT. */ +#define HAVE_PTHREAD_PRIO_INHERIT 1 + +/* Define to 1 if you have the `pthread_rwlock_init' function. */ +#define HAVE_PTHREAD_RWLOCK_INIT 1 + +/* Define if pthread_rwlock_init is provided in the thread library. */ +#define HAVE_PTHREAD_RWLOCK_INIT_IN_THREAD_LIB 1 + +/* Define to 1 if you have the header file. */ +#define HAVE_PWD_H 1 + +/* Define if building with GNU Readline. */ +/* #undef HAVE_READLINE */ + +/* Define if regcomp exists and functions */ +#define HAVE_REGCOMP 1 + +/* Define to 1 if you have the `regexec' function. */ +#define HAVE_REGEXEC 1 + +/* Define to 1 if you have the header file. */ +/* #undef HAVE_REGEXPR_H */ + +/* Define to 1 if you have the header file. */ +#define HAVE_REGEX_H 1 + +/* Define to 1 if you have the `res_nclose' function. */ +#define HAVE_RES_NCLOSE 1 + +/* Define to 1 if you have the `res_ndestroy' function. */ +#define HAVE_RES_NDESTROY 1 + +/* Define to 1 if you have the `res_ninit' function. */ +#define HAVE_RES_NINIT 1 + +/* Define to 1 if you have the `res_nsearch' function. */ +#define HAVE_RES_NSEARCH 1 + +/* Define to 1 if you have the `res_search' function */ +#define HAVE_RES_SEARCH 1 + +/* Define to 1 if you have the `re_comp' function. */ +/* #undef HAVE_RE_COMP */ + +/* Define to 1 if you have the `re_exec' function. */ +/* #undef HAVE_RE_EXEC */ + +/* Define to 1 if you have the header file. */ +/* #undef HAVE_SASL_SASL_H */ + +/* Define if struct sockaddr contains sa_len */ +#define HAVE_SA_LEN 1 + +/* Define to 1 if you have the `setegid' function. */ +#define HAVE_SETEGID 1 + +/* Define to 1 if you have the `setenv' function. */ +#define HAVE_SETENV 1 + +/* Define to 1 if you have the `seteuid' function. */ +#define HAVE_SETEUID 1 + +/* Define if setluid provided in OSF/1 security library */ +/* #undef HAVE_SETLUID */ + +/* Define to 1 if you have the `setregid' function. */ +#define HAVE_SETREGID 1 + +/* Define to 1 if you have the `setresgid' function. */ +/* #undef HAVE_SETRESGID */ + +/* Define to 1 if you have the `setresuid' function. */ +/* #undef HAVE_SETRESUID */ + +/* Define to 1 if you have the `setreuid' function. */ +#define HAVE_SETREUID 1 + +/* Define to 1 if you have the `setsid' function. */ +#define HAVE_SETSID 1 + +/* Define to 1 if you have the `setvbuf' function. */ +#define HAVE_SETVBUF 1 + +/* Define if there is a socklen_t type. If not, probably use size_t */ +#define HAVE_SOCKLEN_T 1 + +/* Define to 1 if you have the `srand' function. */ +#define HAVE_SRAND 1 + +/* Define to 1 if you have the `srand48' function. */ +#define HAVE_SRAND48 1 + +/* Define to 1 if you have the `srandom' function. */ +#define HAVE_SRANDOM 1 + +/* Define to 1 if the system has the type `ssize_t'. */ +#define HAVE_SSIZE_T 1 + +/* Define to 1 if you have the `stat' function. */ +#define HAVE_STAT 1 + +/* Define to 1 if you have the header file. */ +#define HAVE_STDDEF_H 1 + +/* Define to 1 if you have the header file. */ +#define HAVE_STDINT_H 1 + +/* Define to 1 if you have the header file. */ +#define HAVE_STDLIB_H 1 + +/* Define to 1 if you have the `step' function. */ +/* #undef HAVE_STEP */ + +/* Define to 1 if you have the `strchr' function. */ +#define HAVE_STRCHR 1 + +/* Define to 1 if you have the `strdup' function. */ +#define HAVE_STRDUP 1 + +/* Define to 1 if you have the `strerror' function. */ +#define HAVE_STRERROR 1 + +/* Define to 1 if you have the `strerror_r' function. */ +#define HAVE_STRERROR_R 1 + +/* Define to 1 if you have the header file. */ +#define HAVE_STRINGS_H 1 + +/* Define to 1 if you have the header file. */ +#define HAVE_STRING_H 1 + +/* Define to 1 if you have the `strlcpy' function. */ +#define HAVE_STRLCPY 1 + +/* Define to 1 if you have the `strptime' function. */ +#define HAVE_STRPTIME 1 + +/* Define to 1 if the system has the type `struct cmsghdr'. */ +#define HAVE_STRUCT_CMSGHDR 1 + +/* Define if there is a struct if_laddrconf. */ +/* #undef HAVE_STRUCT_IF_LADDRCONF */ + +/* Define to 1 if the system has the type `struct in6_pktinfo'. */ +#define HAVE_STRUCT_IN6_PKTINFO 1 + +/* Define to 1 if the system has the type `struct in_pktinfo'. */ +#define HAVE_STRUCT_IN_PKTINFO 1 + +/* Define if there is a struct lifconf. */ +/* #undef HAVE_STRUCT_LIFCONF */ + +/* Define to 1 if the system has the type `struct rt_msghdr'. */ +#define HAVE_STRUCT_RT_MSGHDR 1 + +/* Define to 1 if the system has the type `struct sockaddr_storage'. */ +#define HAVE_STRUCT_SOCKADDR_STORAGE 1 + +/* Define to 1 if `st_mtimensec' is a member of `struct stat'. */ +/* #undef HAVE_STRUCT_STAT_ST_MTIMENSEC */ + +/* Define to 1 if `st_mtimespec.tv_nsec' is a member of `struct stat'. */ +#define HAVE_STRUCT_STAT_ST_MTIMESPEC_TV_NSEC 1 + +/* Define to 1 if `st_mtim.tv_nsec' is a member of `struct stat'. */ +/* #undef HAVE_STRUCT_STAT_ST_MTIM_TV_NSEC */ + +/* Define to 1 if you have the header file. */ +/* #undef HAVE_SYS_BSWAP_H */ + +/* Define to 1 if you have the header file, and it defines `DIR'. + */ +/* #undef HAVE_SYS_DIR_H */ + +/* Define if sys_errlist in libc */ +#define HAVE_SYS_ERRLIST 1 + +/* Define to 1 if you have the header file. */ +#define HAVE_SYS_FILE_H 1 + +/* Define to 1 if you have the header file. */ +#define HAVE_SYS_FILIO_H 1 + +/* Define to 1 if you have the header file, and it defines `DIR'. + */ +/* #undef HAVE_SYS_NDIR_H */ + +/* Define to 1 if you have the header file. */ +#define HAVE_SYS_PARAM_H 1 + +/* Define to 1 if you have the header file. */ +#define HAVE_SYS_SELECT_H 1 + +/* Define to 1 if you have the header file. */ +#define HAVE_SYS_SOCKET_H 1 + +/* Define to 1 if you have the header file. */ +#define HAVE_SYS_SOCKIO_H 1 + +/* Define to 1 if you have the header file. */ +#define HAVE_SYS_STAT_H 1 + +/* Define to 1 if you have the header file. */ +#define HAVE_SYS_TIME_H 1 + +/* Define to 1 if you have the header file. */ +#define HAVE_SYS_TYPES_H 1 + +/* Define to 1 if you have the header file. */ +#define HAVE_SYS_UIO_H 1 + +/* Define if tcl.h found */ +/* #undef HAVE_TCL_H */ + +/* Define if tcl/tcl.h found */ +/* #undef HAVE_TCL_TCL_H */ + +/* Define to 1 if you have the `timegm' function. */ +#define HAVE_TIMEGM 1 + +/* Define to 1 if you have the header file. */ +#define HAVE_TIME_H 1 + +/* Define to 1 if you have the header file. */ +#define HAVE_UNISTD_H 1 + +/* Define to 1 if you have the `unsetenv' function. */ +#define HAVE_UNSETENV 1 + +/* Define to 1 if the system has the type `u_char'. */ +#define HAVE_U_CHAR 1 + +/* Define to 1 if the system has the type `u_int'. */ +#define HAVE_U_INT 1 + +/* Define to 1 if the system has the type `u_int16_t'. */ +#define HAVE_U_INT16_T 1 + +/* Define to 1 if the system has the type `u_int32_t'. */ +#define HAVE_U_INT32_T 1 + +/* Define to 1 if the system has the type `u_int8_t'. */ +#define HAVE_U_INT8_T 1 + +/* Define to 1 if the system has the type `u_long'. */ +#define HAVE_U_LONG 1 + +/* Define to 1 if you have the `vasprintf' function. */ +#define HAVE_VASPRINTF 1 + +/* Define to 1 if you have the `vsnprintf' function. */ +#define HAVE_VSNPRINTF 1 + +/* Define to 1 if you have the `vsprintf' function. */ +#define HAVE_VSPRINTF 1 + +/* Define to 1 if the system has the type `__int128_t'. */ +#define HAVE___INT128_T 1 + +/* Define to 1 if the system has the type `__uint128_t'. */ +#define HAVE___UINT128_T 1 + +/* Define if errno.h declares perror */ +/* #undef HDR_HAS_PERROR */ + +/* May need to be defined to enable IPv6 support, for example on IRIX */ +/* #undef INET6 */ + +/* Define if MIT Project Athena default configuration should be used */ +/* #undef KRB5_ATHENA_COMPAT */ + +/* Define for DNS support of locating realms and KDCs */ +#undef KRB5_DNS_LOOKUP + +/* Define to enable DNS lookups of Kerberos realm names */ +/* #undef KRB5_DNS_LOOKUP_REALM */ + +/* Define if the KDC should return only vague error codes to clients */ +/* #undef KRBCONF_VAGUE_ERRORS */ + +/* define if the system header files are missing prototype for daemon() */ +#define NEED_DAEMON_PROTO 1 + +/* Define if in6addr_any is not defined in libc */ +#define NEED_INSIXADDR_ANY 1 + +/* define if the system header files are missing prototype for + ss_execute_command() */ +/* #undef NEED_SS_EXECUTE_COMMAND_PROTO */ + +/* define if the system header files are missing prototype for strptime() */ +/* #undef NEED_STRPTIME_PROTO */ + +/* define if the system header files are missing prototype for swab() */ +/* #undef NEED_SWAB_PROTO */ + +/* Define if need to declare sys_errlist */ +/* #undef NEED_SYS_ERRLIST */ + +/* define if the system header files are missing prototype for vasprintf() */ +/* #undef NEED_VASPRINTF_PROTO */ + +/* Define if the KDC should use no lookaside cache */ +/* #undef NOCACHE */ + +/* Define if references to pthread routines should be non-weak. */ +/* #undef NO_WEAK_PTHREADS */ + +/* Define if lex produes code with yylineno */ +/* #undef NO_YYLINENO */ + +/* Define to the address where bug reports for this package should be sent. */ +#define PACKAGE_BUGREPORT "krb5-bugs@mit.edu" + +/* Define to the full name of this package. */ +#define PACKAGE_NAME "Kerberos 5" + +/* Define to the full name and version of this package. */ +#define PACKAGE_STRING "Kerberos 5 1.17.1" + +/* Define to the one symbol short name of this package. */ +#define PACKAGE_TARNAME "krb5" + +/* Define to the home page for this package. */ +#define PACKAGE_URL "" + +/* Define to the version of this package. */ +#define PACKAGE_VERSION "1.17.1" + +/* Define if setjmp indicates POSIX interface */ +#define POSIX_SETJMP 1 + +/* Define if POSIX signal handling is used */ +#define POSIX_SIGNALS 1 + +/* Define if POSIX signal handlers are used */ +#define POSIX_SIGTYPE 1 + +/* Define if termios.h exists and tcsetattr exists */ +#define POSIX_TERMIOS 1 + +/* Define to necessary symbol if this constant uses a non-standard name on + your system. */ +/* #undef PTHREAD_CREATE_JOINABLE */ + +/* Define as the return type of signal handlers (`int' or `void'). */ +#define RETSIGTYPE void + +/* Define as return type of setrpcent */ +#define SETRPCENT_TYPE void + +/* The size of `size_t', as computed by sizeof. */ +#define SIZEOF_SIZE_T 8 + +/* The size of `time_t', as computed by sizeof. */ +#define SIZEOF_TIME_T 8 + +/* Define to use OpenSSL for SPAKE preauth */ +#define SPAKE_OPENSSL 1 + +/* Define for static plugin linkage */ +/* #undef STATIC_PLUGINS */ + +/* Define to 1 if you have the ANSI C header files. */ +#define STDC_HEADERS 1 + +/* Define to 1 if strerror_r returns char *. */ +/* #undef STRERROR_R_CHAR_P */ + +/* Define if sys_errlist is defined in errno.h */ +#define SYS_ERRLIST_DECLARED 1 + +/* Define to 1 if you can safely include both and . */ +#define TIME_WITH_SYS_TIME 1 + +/* Define if no TLS implementation is selected */ +/* #undef TLS_IMPL_NONE */ + +/* Define if TLS implementation is OpenSSL */ +#define TLS_IMPL_OPENSSL 1 + +/* Define if you have dirent.h functionality */ +#define USE_DIRENT_H 1 + +/* Define if dlopen should be used */ +#define USE_DLOPEN 1 + +/* Define if the keyring ccache should be enabled */ +/* #undef USE_KEYRING_CCACHE */ + +/* Define if link-time options for library finalization will be used */ +/* #undef USE_LINKER_FINI_OPTION */ + +/* Define if link-time options for library initialization will be used */ +/* #undef USE_LINKER_INIT_OPTION */ + +/* Define if sigprocmask should be used */ +#define USE_SIGPROCMASK 1 + +/* Define if wait takes int as a argument */ +#define WAIT_USES_INT 1 + +/* Define to 1 if `lex' declares `yytext' as a `char *' by default, not a + `char[]'. */ +#define YYTEXT_POINTER 1 + +/* Define to enable extensions in glibc */ +#define _GNU_SOURCE 1 + +/* Define to enable C11 extensions */ +#define __STDC_WANT_LIB_EXT1__ 1 + +/* Define to empty if `const' does not conform to ANSI C. */ +/* #undef const */ + +/* Define to `int' if doesn't define. */ +/* #undef gid_t */ + +/* Define to `__inline__' or `__inline' if that's what the C compiler + calls it, or to nothing if 'inline' is not supported under any name. */ +#ifndef __cplusplus +/* #undef inline */ +#endif + +/* Define krb5_sigtype to type of signal handler */ +#define krb5_sigtype void + +/* Define to `int' if does not define. */ +/* #undef mode_t */ + +/* Define to `long int' if does not define. */ +/* #undef off_t */ + +/* Define to `long' if does not define. */ +/* #undef time_t */ + +/* Define to `int' if doesn't define. */ +/* #undef uid_t */ + + +#if defined(__GNUC__) && !defined(inline) +/* Silence gcc pedantic warnings about ANSI C. */ +# define inline __inline__ +#endif +#endif /* KRB5_AUTOCONF_H */ diff --git a/contrib/krb5-cmake/autoconf.h b/contrib/krb5-cmake/autoconf_linux.h similarity index 100% rename from contrib/krb5-cmake/autoconf.h rename to contrib/krb5-cmake/autoconf_linux.h diff --git a/docs/en/operations/external-authenticators/ldap.md b/docs/en/operations/external-authenticators/ldap.md index 36a13227852..5c06ad7daed 100644 --- a/docs/en/operations/external-authenticators/ldap.md +++ b/docs/en/operations/external-authenticators/ldap.md @@ -1,4 +1,4 @@ -# LDAP {#external-authenticators-ldap} +# LDAP {#external-authenticators-ldap} LDAP server can be used to authenticate ClickHouse users. There are two different approaches for doing this: @@ -87,14 +87,13 @@ Note, that user `my_user` refers to `my_ldap_server`. This LDAP server must be c When SQL-driven [Access Control and Account Management](../access-rights.md#access-control) is enabled in ClickHouse, users that are authenticated by LDAP servers can also be created using the [CRATE USER](../../sql-reference/statements/create/user.md#create-user-statement) statement. - ```sql -CREATE USER my_user IDENTIFIED WITH ldap_server BY 'my_ldap_server' +CREATE USER my_user IDENTIFIED WITH ldap SERVER 'my_ldap_server' ``` ## LDAP Exernal User Directory {#ldap-external-user-directory} -In addition to the locally defined users, a remote LDAP server can be used as a source of user definitions. In order to achieve this, specify previously defined LDAP server name (see [LDAP Server Definition](#ldap-server-definition)) in the `ldap` section inside the `users_directories` section of the `config.xml` file. +In addition to the locally defined users, a remote LDAP server can be used as a source of user definitions. In order to achieve this, specify previously defined LDAP server name (see [LDAP Server Definition](#ldap-server-definition)) in an `ldap` section inside the `users_directories` section of the `config.xml` file. At each login attempt, ClickHouse will try to find the user definition locally and authenticate it as usual, but if the user is not defined, ClickHouse will assume it exists in the external LDAP directory, and will try to "bind" to the specified DN at the LDAP server using the provided credentials. If successful, the user will be considered existing and authenticated. The user will be assigned roles from the list specified in the `roles` section. Additionally, LDAP "search" can be performed and results can be transformed and treated as role names and then be assigned to the user if the `role_mapping` section is also configured. All this implies that the SQL-driven [Access Control and Account Management](../access-rights.md#access-control) is enabled and roles are created using the [CREATE ROLE](../../sql-reference/statements/create/role.md#create-role-statement) statement. @@ -153,4 +152,3 @@ Parameters: - `prefix` - prefix, that will be expected to be in front of each string in the original list of strings returned by the LDAP search. Prefix will be removed from the original strings and resulting strings will be treated as local role names. Empty, by default. - diff --git a/docs/en/sql-reference/statements/alter/user.md b/docs/en/sql-reference/statements/alter/user.md index efad6561439..b590bf4887d 100644 --- a/docs/en/sql-reference/statements/alter/user.md +++ b/docs/en/sql-reference/statements/alter/user.md @@ -12,10 +12,10 @@ Syntax: ``` sql ALTER USER [IF EXISTS] name1 [ON CLUSTER cluster_name1] [RENAME TO new_name1] [, name2 [ON CLUSTER cluster_name2] [RENAME TO new_name2] ...] - [IDENTIFIED [WITH {PLAINTEXT_PASSWORD|SHA256_PASSWORD|DOUBLE_SHA1_PASSWORD}] BY {'password'|'hash'}] - [[ADD|DROP] HOST {LOCAL | NAME 'name' | REGEXP 'name_regexp' | IP 'address' | LIKE 'pattern'} [,...] | ANY | NONE] + [NOT IDENTIFIED | IDENTIFIED {[WITH {no_password | plaintext_password | sha256_password | sha256_hash | double_sha1_password | double_sha1_hash}] BY {'password' | 'hash'}} | {WITH ldap SERVER 'server_name'} | {WITH kerberos [REALM 'realm']}] + [[ADD | DROP] HOST {LOCAL | NAME 'name' | REGEXP 'name_regexp' | IP 'address' | LIKE 'pattern'} [,...] | ANY | NONE] [DEFAULT ROLE role [,...] | ALL | ALL EXCEPT role [,...] ] - [SETTINGS variable [= value] [MIN [=] min_value] [MAX [=] max_value] [READONLY|WRITABLE] | PROFILE 'profile_name'] [,...] + [SETTINGS variable [= value] [MIN [=] min_value] [MAX [=] max_value] [READONLY | WRITABLE] | PROFILE 'profile_name'] [,...] ``` To use `ALTER USER` you must have the [ALTER USER](../../../sql-reference/statements/grant.md#grant-access-management) privilege. diff --git a/docs/en/sql-reference/statements/create/user.md b/docs/en/sql-reference/statements/create/user.md index c1a52e3b864..49a4e3813a1 100644 --- a/docs/en/sql-reference/statements/create/user.md +++ b/docs/en/sql-reference/statements/create/user.md @@ -12,10 +12,10 @@ Syntax: ``` sql CREATE USER [IF NOT EXISTS | OR REPLACE] name1 [ON CLUSTER cluster_name1] [, name2 [ON CLUSTER cluster_name2] ...] - [IDENTIFIED [WITH {NO_PASSWORD|PLAINTEXT_PASSWORD|SHA256_PASSWORD|SHA256_HASH|DOUBLE_SHA1_PASSWORD|DOUBLE_SHA1_HASH|LDAP_SERVER}] BY {'password'|'hash'}] + [NOT IDENTIFIED | IDENTIFIED {[WITH {no_password | plaintext_password | sha256_password | sha256_hash | double_sha1_password | double_sha1_hash}] BY {'password' | 'hash'}} | {WITH ldap SERVER 'server_name'} | {WITH kerberos [REALM 'realm']}] [HOST {LOCAL | NAME 'name' | REGEXP 'name_regexp' | IP 'address' | LIKE 'pattern'} [,...] | ANY | NONE] [DEFAULT ROLE role [,...]] - [SETTINGS variable [= value] [MIN [=] min_value] [MAX [=] max_value] [READONLY|WRITABLE] | PROFILE 'profile_name'] [,...] + [SETTINGS variable [= value] [MIN [=] min_value] [MAX [=] max_value] [READONLY | WRITABLE] | PROFILE 'profile_name'] [,...] ``` `ON CLUSTER` clause allows creating users on a cluster, see [Distributed DDL](../../../sql-reference/distributed-ddl.md). @@ -30,7 +30,8 @@ There are multiple ways of user identification: - `IDENTIFIED WITH sha256_hash BY 'hash'` - `IDENTIFIED WITH double_sha1_password BY 'qwerty'` - `IDENTIFIED WITH double_sha1_hash BY 'hash'` -- `IDENTIFIED WITH ldap_server BY 'server'` +- `IDENTIFIED WITH ldap SERVER 'server_name'` +- `IDENTIFIED WITH kerberos` or `IDENTIFIED WITH kerberos REALM 'realm'` ## User Host {#user-host} diff --git a/programs/client/Client.cpp b/programs/client/Client.cpp index 3c27908741c..da7c729a737 100644 --- a/programs/client/Client.cpp +++ b/programs/client/Client.cpp @@ -2472,7 +2472,7 @@ public: /** If "--password [value]" is used but the value is omitted, the bad argument exception will be thrown. * implicit_value is used to avoid this exception (to allow user to type just "--password") * Since currently boost provides no way to check if a value has been set implicitly for an option, - * the "\n" is used to distinguish this case because there is hardly a chance an user would use "\n" + * the "\n" is used to distinguish this case because there is hardly a chance a user would use "\n" * as the password. */ ("password", po::value()->implicit_value("\n", ""), "password") diff --git a/programs/local/LocalServer.cpp b/programs/local/LocalServer.cpp index 5a8d35e204d..7c6b60fbf8e 100644 --- a/programs/local/LocalServer.cpp +++ b/programs/local/LocalServer.cpp @@ -240,7 +240,7 @@ try /// Skip networking - /// Sets external authenticators config (LDAP). + /// Sets external authenticators config (LDAP, Kerberos). global_context->setExternalAuthenticatorsConfig(config()); setupUsers(); diff --git a/programs/server/config.xml b/programs/server/config.xml index b72cf53ca03..715a366af00 100644 --- a/programs/server/config.xml +++ b/programs/server/config.xml @@ -362,6 +362,27 @@ --> + + diff --git a/programs/server/users.xml b/programs/server/users.xml index ef66891a6a0..b33dc0628d1 100644 --- a/programs/server/users.xml +++ b/programs/server/users.xml @@ -41,9 +41,18 @@ If you want to specify double SHA1, place it in 'password_double_sha1_hex' element. Example: e395796d6546b1b65db9d665cd43f0e858dd4303 - If you want to specify a previously defined LDAP server (see 'ldap_servers' in main config) for authentication, place its name in 'server' element inside 'ldap' element. + If you want to specify a previously defined LDAP server (see 'ldap_servers' in the main config) for authentication, + place its name in 'server' element inside 'ldap' element. Example: my_ldap_server + If you want to authenticate the user via Kerberos (assuming Kerberos is enabled, see 'kerberos' in the main config), + place 'kerberos' element instead of 'password' (and similar) elements. + The name part of the canonical principal name of the initiator must match the user name for authentication to succeed. + You can also place 'realm' element inside 'kerberos' element to further restrict authentication to only those requests + whose initiator's realm matches it. + Example: + Example: EXAMPLE.COM + How to generate decent password: Execute: PASSWORD=$(base64 < /dev/urandom | head -c8); echo "$PASSWORD"; echo -n "$PASSWORD" | sha256sum | tr -d '-' In first line will be password and in second - corresponding SHA256. diff --git a/src/Access/AccessControlManager.cpp b/src/Access/AccessControlManager.cpp index e874bda5b69..0e7bf1e56f4 100644 --- a/src/Access/AccessControlManager.cpp +++ b/src/Access/AccessControlManager.cpp @@ -361,9 +361,9 @@ void AccessControlManager::addStoragesFromMainConfig( } -UUID AccessControlManager::login(const String & user_name, const String & password, const Poco::Net::IPAddress & address) const +UUID AccessControlManager::login(const Credentials & credentials, const Poco::Net::IPAddress & address) const { - return MultipleAccessStorage::login(user_name, password, address, *external_authenticators); + return MultipleAccessStorage::login(credentials, address, *external_authenticators); } void AccessControlManager::setExternalAuthenticatorsConfig(const Poco::Util::AbstractConfiguration & config) diff --git a/src/Access/AccessControlManager.h b/src/Access/AccessControlManager.h index 07edfd93475..b4d90a4198e 100644 --- a/src/Access/AccessControlManager.h +++ b/src/Access/AccessControlManager.h @@ -109,7 +109,7 @@ public: bool isSettingNameAllowed(const std::string_view & name) const; void checkSettingNameIsAllowed(const std::string_view & name) const; - UUID login(const String & user_name, const String & password, const Poco::Net::IPAddress & address) const; + UUID login(const Credentials & credentials, const Poco::Net::IPAddress & address) const; void setExternalAuthenticatorsConfig(const Poco::Util::AbstractConfiguration & config); std::shared_ptr getContextAccess( diff --git a/src/Access/AllowedClientHosts.h b/src/Access/AllowedClientHosts.h index 615782d75a2..a6895b120e0 100644 --- a/src/Access/AllowedClientHosts.h +++ b/src/Access/AllowedClientHosts.h @@ -14,7 +14,7 @@ namespace DB using Strings = std::vector; -/// Represents lists of hosts an user is allowed to connect to server from. +/// Represents lists of hosts a user is allowed to connect to server from. class AllowedClientHosts { public: diff --git a/src/Access/Authentication.cpp b/src/Access/Authentication.cpp index 19c40c068b4..54d94d905ae 100644 --- a/src/Access/Authentication.cpp +++ b/src/Access/Authentication.cpp @@ -1,5 +1,8 @@ #include +#include #include +#include +#include #include #include @@ -8,8 +11,8 @@ namespace DB { namespace ErrorCodes { - extern const int BAD_ARGUMENTS; extern const int NOT_IMPLEMENTED; + extern const int LOGICAL_ERROR; } @@ -32,14 +35,13 @@ Authentication::Digest Authentication::getPasswordDoubleSHA1() const return engine.digest(); } - case SHA256_PASSWORD: - throw Exception("Cannot get password double SHA1 for user with 'SHA256_PASSWORD' authentication", ErrorCodes::BAD_ARGUMENTS); - case DOUBLE_SHA1_PASSWORD: return password_hash; - case LDAP_SERVER: - throw Exception("Cannot get password double SHA1 for user with 'LDAP_SERVER' authentication", ErrorCodes::BAD_ARGUMENTS); + case SHA256_PASSWORD: + case LDAP: + case KERBEROS: + throw Exception("Cannot get password double SHA1 hash for authentication type " + toString(type), ErrorCodes::LOGICAL_ERROR); case MAX_TYPE: break; @@ -48,44 +50,76 @@ Authentication::Digest Authentication::getPasswordDoubleSHA1() const } -bool Authentication::isCorrectPassword(const String & user_, const String & password_, const ExternalAuthenticators & external_authenticators) const +bool Authentication::areCredentialsValid(const Credentials & credentials, const ExternalAuthenticators & external_authenticators) const { - switch (type) + if (!credentials.isReady()) + return false; + + if (const auto * gss_acceptor_context = dynamic_cast(&credentials)) { - case NO_PASSWORD: - return true; - - case PLAINTEXT_PASSWORD: + switch (type) { - if (password_ == std::string_view{reinterpret_cast(password_hash.data()), password_hash.size()}) - return true; + case NO_PASSWORD: + case PLAINTEXT_PASSWORD: + case SHA256_PASSWORD: + case DOUBLE_SHA1_PASSWORD: + case LDAP: + throw Require("ClickHouse Basic Authentication"); - // For compatibility with MySQL clients which support only native authentication plugin, SHA1 can be passed instead of password. - auto password_sha1 = encodeSHA1(password_hash); - return password_ == std::string_view{reinterpret_cast(password_sha1.data()), password_sha1.size()}; + case KERBEROS: + return external_authenticators.checkKerberosCredentials(kerberos_realm, *gss_acceptor_context); + + case MAX_TYPE: + break; } - - case SHA256_PASSWORD: - return encodeSHA256(password_) == password_hash; - - case DOUBLE_SHA1_PASSWORD: - { - auto first_sha1 = encodeSHA1(password_); - - /// If it was MySQL compatibility server, then first_sha1 already contains double SHA1. - if (first_sha1 == password_hash) - return true; - - return encodeSHA1(first_sha1) == password_hash; - } - - case LDAP_SERVER: - return external_authenticators.checkLDAPCredentials(server_name, user_, password_); - - case MAX_TYPE: - break; } - throw Exception("Cannot check if the password is correct for authentication type " + toString(type), ErrorCodes::NOT_IMPLEMENTED); + + if (const auto * basic_credentials = dynamic_cast(&credentials)) + { + switch (type) + { + case NO_PASSWORD: + return true; // N.B. even if the password is not empty! + + case PLAINTEXT_PASSWORD: + { + if (basic_credentials->getPassword() == std::string_view{reinterpret_cast(password_hash.data()), password_hash.size()}) + return true; + + // For compatibility with MySQL clients which support only native authentication plugin, SHA1 can be passed instead of password. + const auto password_sha1 = encodeSHA1(password_hash); + return basic_credentials->getPassword() == std::string_view{reinterpret_cast(password_sha1.data()), password_sha1.size()}; + } + + case SHA256_PASSWORD: + return encodeSHA256(basic_credentials->getPassword()) == password_hash; + + case DOUBLE_SHA1_PASSWORD: + { + const auto first_sha1 = encodeSHA1(basic_credentials->getPassword()); + + /// If it was MySQL compatibility server, then first_sha1 already contains double SHA1. + if (first_sha1 == password_hash) + return true; + + return encodeSHA1(first_sha1) == password_hash; + } + + case LDAP: + return external_authenticators.checkLDAPCredentials(ldap_server_name, *basic_credentials); + + case KERBEROS: + throw Require(kerberos_realm); + + case MAX_TYPE: + break; + } + } + + if ([[maybe_unused]] const auto * always_allow_credentials = dynamic_cast(&credentials)) + return true; + + throw Exception("areCredentialsValid(): authentication type " + toString(type) + " not supported", ErrorCodes::NOT_IMPLEMENTED); } } diff --git a/src/Access/Authentication.h b/src/Access/Authentication.h index bd22b7f0ff5..54542cb504a 100644 --- a/src/Access/Authentication.h +++ b/src/Access/Authentication.h @@ -6,8 +6,6 @@ #include #include #include -#include -#include namespace DB @@ -20,13 +18,10 @@ namespace ErrorCodes extern const int NOT_IMPLEMENTED; } +class Credentials; class ExternalAuthenticators; -struct LDAPSearchParams; -using LDAPSearchParamsList = std::vector; -using LDAPSearchResults = std::set; -using LDAPSearchResultsList = std::vector; -/// Authentication type and encrypted password for checking when an user logins. +/// Authentication type and encrypted password for checking when a user logins. class Authentication { public: @@ -46,7 +41,10 @@ public: DOUBLE_SHA1_PASSWORD, /// Password is checked by a [remote] LDAP server. Connection will be made at each authentication attempt. - LDAP_SERVER, + LDAP, + + /// Kerberos authentication performed through GSS-API negotiation loop. + KERBEROS, MAX_TYPE, }; @@ -58,6 +56,18 @@ public: static const TypeInfo & get(Type type_); }; + // A signaling class used to communicate requirements for credentials. + template + class Require : public Exception + { + public: + explicit Require(const String & realm_); + const String & getRealm() const; + + private: + const String realm; + }; + using Digest = std::vector; Authentication(Authentication::Type type_ = NO_PASSWORD) : type(type_) {} @@ -88,14 +98,16 @@ public: /// Allowed to use for Type::NO_PASSWORD, Type::PLAINTEXT_PASSWORD, Type::DOUBLE_SHA1_PASSWORD. Digest getPasswordDoubleSHA1() const; - /// Sets an external authentication server name. - /// When authentication type is LDAP_SERVER, server name is expected to be the name of a preconfigured LDAP server. - const String & getServerName() const; - void setServerName(const String & server_name_); + /// Sets the server name for authentication type LDAP. + const String & getLDAPServerName() const; + void setLDAPServerName(const String & name); - /// Checks if the provided password is correct. Returns false if not. - /// User name and external authenticators are used by the specific authentication types only (e.g., LDAP_SERVER). - bool isCorrectPassword(const String & user_, const String & password_, const ExternalAuthenticators & external_authenticators) const; + /// Sets the realm name for authentication type KERBEROS. + const String & getKerberosRealm() const; + void setKerberosRealm(const String & realm); + + /// Checks the credentials (passwords, readiness, etc.) + bool areCredentialsValid(const Credentials & credentials, const ExternalAuthenticators & external_authenticators) const; friend bool operator ==(const Authentication & lhs, const Authentication & rhs) { return (lhs.type == rhs.type) && (lhs.password_hash == rhs.password_hash); } friend bool operator !=(const Authentication & lhs, const Authentication & rhs) { return !(lhs == rhs); } @@ -109,7 +121,8 @@ private: Type type = Type::NO_PASSWORD; Digest password_hash; - String server_name; + String ldap_server_name; + String kerberos_realm; }; @@ -144,16 +157,35 @@ inline const Authentication::TypeInfo & Authentication::TypeInfo::get(Type type_ static const auto info = make_info("DOUBLE_SHA1_PASSWORD"); return info; } - case LDAP_SERVER: + case LDAP: { - static const auto info = make_info("LDAP_SERVER"); + static const auto info = make_info("LDAP"); return info; } - case MAX_TYPE: break; + case KERBEROS: + { + static const auto info = make_info("KERBEROS"); + return info; + } + case MAX_TYPE: + break; } throw Exception("Unknown authentication type: " + std::to_string(static_cast(type_)), ErrorCodes::LOGICAL_ERROR); } +template +Authentication::Require::Require(const String & realm_) + : Exception("Credentials required", ErrorCodes::BAD_ARGUMENTS) + , realm(realm_) +{ +} + +template +const String & Authentication::Require::getRealm() const +{ + return realm; +} + inline String toString(Authentication::Type type_) { return Authentication::TypeInfo::get(type_).raw_name; @@ -186,9 +218,6 @@ inline void Authentication::setPassword(const String & password_) { switch (type) { - case NO_PASSWORD: - throw Exception("Cannot specify password for the 'NO_PASSWORD' authentication type", ErrorCodes::LOGICAL_ERROR); - case PLAINTEXT_PASSWORD: return setPasswordHashBinary(encodePlainText(password_)); @@ -198,10 +227,13 @@ inline void Authentication::setPassword(const String & password_) case DOUBLE_SHA1_PASSWORD: return setPasswordHashBinary(encodeDoubleSHA1(password_)); - case LDAP_SERVER: - throw Exception("Cannot specify password for the 'LDAP_SERVER' authentication type", ErrorCodes::LOGICAL_ERROR); + case NO_PASSWORD: + case LDAP: + case KERBEROS: + throw Exception("Cannot specify password for authentication type " + toString(type), ErrorCodes::LOGICAL_ERROR); - case MAX_TYPE: break; + case MAX_TYPE: + break; } throw Exception("setPassword(): authentication type " + toString(type) + " not supported", ErrorCodes::NOT_IMPLEMENTED); } @@ -225,8 +257,9 @@ inline void Authentication::setPasswordHashHex(const String & hash) inline String Authentication::getPasswordHashHex() const { - if (type == LDAP_SERVER) - throw Exception("Cannot get password of a user with the 'LDAP_SERVER' authentication type", ErrorCodes::LOGICAL_ERROR); + if (type == LDAP || type == KERBEROS) + throw Exception("Cannot get password hex hash for authentication type " + toString(type), ErrorCodes::LOGICAL_ERROR); + String hex; hex.resize(password_hash.size() * 2); boost::algorithm::hex(password_hash.begin(), password_hash.end(), hex.data()); @@ -238,9 +271,6 @@ inline void Authentication::setPasswordHashBinary(const Digest & hash) { switch (type) { - case NO_PASSWORD: - throw Exception("Cannot specify password for the 'NO_PASSWORD' authentication type", ErrorCodes::LOGICAL_ERROR); - case PLAINTEXT_PASSWORD: { password_hash = hash; @@ -269,22 +299,35 @@ inline void Authentication::setPasswordHashBinary(const Digest & hash) return; } - case LDAP_SERVER: - throw Exception("Cannot specify password for the 'LDAP_SERVER' authentication type", ErrorCodes::LOGICAL_ERROR); + case NO_PASSWORD: + case LDAP: + case KERBEROS: + throw Exception("Cannot specify password binary hash for authentication type " + toString(type), ErrorCodes::LOGICAL_ERROR); - case MAX_TYPE: break; + case MAX_TYPE: + break; } throw Exception("setPasswordHashBinary(): authentication type " + toString(type) + " not supported", ErrorCodes::NOT_IMPLEMENTED); } -inline const String & Authentication::getServerName() const +inline const String & Authentication::getLDAPServerName() const { - return server_name; + return ldap_server_name; } -inline void Authentication::setServerName(const String & server_name_) +inline void Authentication::setLDAPServerName(const String & name) { - server_name = server_name_; + ldap_server_name = name; +} + +inline const String & Authentication::getKerberosRealm() const +{ + return kerberos_realm; +} + +inline void Authentication::setKerberosRealm(const String & realm) +{ + kerberos_realm = realm; } } diff --git a/src/Access/Credentials.cpp b/src/Access/Credentials.cpp new file mode 100644 index 00000000000..c2850ad4d4f --- /dev/null +++ b/src/Access/Credentials.cpp @@ -0,0 +1,86 @@ +#include +#include + + +namespace DB +{ + +namespace ErrorCodes +{ + extern const int LOGICAL_ERROR; +} + +Credentials::Credentials(const String & user_name_) + : user_name(user_name_) +{ +} + +const String & Credentials::getUserName() const +{ + if (!isReady()) + throwNotReady(); + return user_name; +} + +bool Credentials::isReady() const +{ + return is_ready; +} + +void Credentials::throwNotReady() +{ + throw Exception("Credentials are not ready", ErrorCodes::LOGICAL_ERROR); +} + +AlwaysAllowCredentials::AlwaysAllowCredentials() +{ + is_ready = true; +} + +AlwaysAllowCredentials::AlwaysAllowCredentials(const String & user_name_) + : Credentials(user_name_) +{ + is_ready = true; +} + +void AlwaysAllowCredentials::setUserName(const String & user_name_) +{ + user_name = user_name_; +} + +BasicCredentials::BasicCredentials() +{ + is_ready = true; +} + +BasicCredentials::BasicCredentials(const String & user_name_) + : Credentials(user_name_) +{ + is_ready = true; +} + +BasicCredentials::BasicCredentials(const String & user_name_, const String & password_) + : Credentials(user_name_) + , password(password_) +{ + is_ready = true; +} + +void BasicCredentials::setUserName(const String & user_name_) +{ + user_name = user_name_; +} + +void BasicCredentials::setPassword(const String & password_) +{ + password = password_; +} + +const String & BasicCredentials::getPassword() const +{ + if (!isReady()) + throwNotReady(); + return password; +} + +} diff --git a/src/Access/Credentials.h b/src/Access/Credentials.h new file mode 100644 index 00000000000..5e9fd1589e0 --- /dev/null +++ b/src/Access/Credentials.h @@ -0,0 +1,55 @@ +#pragma once + +#include +#include + + +namespace DB +{ + +class Credentials +{ +public: + explicit Credentials() = default; + explicit Credentials(const String & user_name_); + + virtual ~Credentials() = default; + + const String & getUserName() const; + bool isReady() const; + +protected: + [[noreturn]] static void throwNotReady(); + +protected: + bool is_ready = false; + String user_name; +}; + +class AlwaysAllowCredentials + : public Credentials +{ +public: + explicit AlwaysAllowCredentials(); + explicit AlwaysAllowCredentials(const String & user_name_); + + void setUserName(const String & user_name_); +}; + +class BasicCredentials + : public Credentials +{ +public: + explicit BasicCredentials(); + explicit BasicCredentials(const String & user_name_); + explicit BasicCredentials(const String & user_name_, const String & password_); + + void setUserName(const String & user_name_); + void setPassword(const String & password_); + const String & getPassword() const; + +private: + String password; +}; + +} diff --git a/src/Access/ExternalAuthenticators.cpp b/src/Access/ExternalAuthenticators.cpp index 6f66f4303e1..1cade973724 100644 --- a/src/Access/ExternalAuthenticators.cpp +++ b/src/Access/ExternalAuthenticators.cpp @@ -20,14 +20,14 @@ namespace ErrorCodes namespace { -auto parseLDAPServer(const Poco::Util::AbstractConfiguration & config, const String & ldap_server_name) +auto parseLDAPServer(const Poco::Util::AbstractConfiguration & config, const String & name) { - if (ldap_server_name.empty()) + if (name.empty()) throw Exception("LDAP server name cannot be empty", ErrorCodes::BAD_ARGUMENTS); - LDAPServerParams params; + LDAPClient::Params params; - const String ldap_server_config = "ldap_servers." + ldap_server_name; + const String ldap_server_config = "ldap_servers." + name; const bool has_host = config.has(ldap_server_config + ".host"); const bool has_port = config.has(ldap_server_config + ".port"); @@ -75,11 +75,11 @@ auto parseLDAPServer(const Poco::Util::AbstractConfiguration & config, const Str boost::to_lower(enable_tls_lc_str); if (enable_tls_lc_str == "starttls") - params.enable_tls = LDAPServerParams::TLSEnable::YES_STARTTLS; + params.enable_tls = LDAPClient::Params::TLSEnable::YES_STARTTLS; else if (config.getBool(ldap_server_config + ".enable_tls")) - params.enable_tls = LDAPServerParams::TLSEnable::YES; + params.enable_tls = LDAPClient::Params::TLSEnable::YES; else - params.enable_tls = LDAPServerParams::TLSEnable::NO; + params.enable_tls = LDAPClient::Params::TLSEnable::NO; } if (has_tls_minimum_protocol_version) @@ -88,15 +88,15 @@ auto parseLDAPServer(const Poco::Util::AbstractConfiguration & config, const Str boost::to_lower(tls_minimum_protocol_version_lc_str); if (tls_minimum_protocol_version_lc_str == "ssl2") - params.tls_minimum_protocol_version = LDAPServerParams::TLSProtocolVersion::SSL2; + params.tls_minimum_protocol_version = LDAPClient::Params::TLSProtocolVersion::SSL2; else if (tls_minimum_protocol_version_lc_str == "ssl3") - params.tls_minimum_protocol_version = LDAPServerParams::TLSProtocolVersion::SSL3; + params.tls_minimum_protocol_version = LDAPClient::Params::TLSProtocolVersion::SSL3; else if (tls_minimum_protocol_version_lc_str == "tls1.0") - params.tls_minimum_protocol_version = LDAPServerParams::TLSProtocolVersion::TLS1_0; + params.tls_minimum_protocol_version = LDAPClient::Params::TLSProtocolVersion::TLS1_0; else if (tls_minimum_protocol_version_lc_str == "tls1.1") - params.tls_minimum_protocol_version = LDAPServerParams::TLSProtocolVersion::TLS1_1; + params.tls_minimum_protocol_version = LDAPClient::Params::TLSProtocolVersion::TLS1_1; else if (tls_minimum_protocol_version_lc_str == "tls1.2") - params.tls_minimum_protocol_version = LDAPServerParams::TLSProtocolVersion::TLS1_2; + params.tls_minimum_protocol_version = LDAPClient::Params::TLSProtocolVersion::TLS1_2; else throw Exception("Bad value for 'tls_minimum_protocol_version' entry, allowed values are: 'ssl2', 'ssl3', 'tls1.0', 'tls1.1', 'tls1.2'", ErrorCodes::BAD_ARGUMENTS); } @@ -107,13 +107,13 @@ auto parseLDAPServer(const Poco::Util::AbstractConfiguration & config, const Str boost::to_lower(tls_require_cert_lc_str); if (tls_require_cert_lc_str == "never") - params.tls_require_cert = LDAPServerParams::TLSRequireCert::NEVER; + params.tls_require_cert = LDAPClient::Params::TLSRequireCert::NEVER; else if (tls_require_cert_lc_str == "allow") - params.tls_require_cert = LDAPServerParams::TLSRequireCert::ALLOW; + params.tls_require_cert = LDAPClient::Params::TLSRequireCert::ALLOW; else if (tls_require_cert_lc_str == "try") - params.tls_require_cert = LDAPServerParams::TLSRequireCert::TRY; + params.tls_require_cert = LDAPClient::Params::TLSRequireCert::TRY; else if (tls_require_cert_lc_str == "demand") - params.tls_require_cert = LDAPServerParams::TLSRequireCert::DEMAND; + params.tls_require_cert = LDAPClient::Params::TLSRequireCert::DEMAND; else throw Exception("Bad value for 'tls_require_cert' entry, allowed values are: 'never', 'allow', 'try', 'demand'", ErrorCodes::BAD_ARGUMENTS); } @@ -142,7 +142,44 @@ auto parseLDAPServer(const Poco::Util::AbstractConfiguration & config, const Str params.port = port; } else - params.port = (params.enable_tls == LDAPServerParams::TLSEnable::YES ? 636 : 389); + params.port = (params.enable_tls == LDAPClient::Params::TLSEnable::YES ? 636 : 389); + + return params; +} + +auto parseKerberosParams(const Poco::Util::AbstractConfiguration & config) +{ + GSSAcceptorContext::Params params; + + Poco::Util::AbstractConfiguration::Keys keys; + config.keys("kerberos", keys); + + std::size_t reealm_key_count = 0; + std::size_t principal_keys_count = 0; + + for (auto key : keys) + { + const auto bracket_pos = key.find('['); + if (bracket_pos != std::string::npos) + key.resize(bracket_pos); + + boost::algorithm::to_lower(key); + + reealm_key_count += (key == "realm"); + principal_keys_count += (key == "principal"); + } + + if (reealm_key_count > 0 && principal_keys_count > 0) + throw Exception("Realm and principal name cannot be specified simultaneously", ErrorCodes::BAD_ARGUMENTS); + + if (reealm_key_count > 1) + throw Exception("Multiple realm sections are not allowed", ErrorCodes::BAD_ARGUMENTS); + + if (principal_keys_count > 1) + throw Exception("Multiple principal sections are not allowed", ErrorCodes::BAD_ARGUMENTS); + + params.realm = config.getString("kerberos.realm", ""); + params.principal = config.getString("kerberos.principal", ""); return params; } @@ -152,48 +189,82 @@ auto parseLDAPServer(const Poco::Util::AbstractConfiguration & config, const Str void ExternalAuthenticators::reset() { std::scoped_lock lock(mutex); - ldap_server_params.clear(); - ldap_server_caches.clear(); + ldap_client_params_blueprint.clear(); + ldap_caches.clear(); + kerberos_params.reset(); } void ExternalAuthenticators::setConfiguration(const Poco::Util::AbstractConfiguration & config, Poco::Logger * log) { std::scoped_lock lock(mutex); - reset(); + Poco::Util::AbstractConfiguration::Keys all_keys; + config.keys("", all_keys); + + std::size_t ldap_servers_key_count = 0; + std::size_t kerberos_keys_count = 0; + + for (auto key : all_keys) + { + const auto bracket_pos = key.find('['); + if (bracket_pos != std::string::npos) + key.resize(bracket_pos); + + boost::algorithm::to_lower(key); + + ldap_servers_key_count += (key == "ldap_servers"); + kerberos_keys_count += (key == "kerberos"); + } + + if (ldap_servers_key_count > 1) + throw Exception("Multiple ldap_servers sections are not allowed", ErrorCodes::BAD_ARGUMENTS); + + if (kerberos_keys_count > 1) + throw Exception("Multiple kerberos sections are not allowed", ErrorCodes::BAD_ARGUMENTS); + Poco::Util::AbstractConfiguration::Keys ldap_server_names; config.keys("ldap_servers", ldap_server_names); for (const auto & ldap_server_name : ldap_server_names) { try { - ldap_server_params.insert_or_assign(ldap_server_name, parseLDAPServer(config, ldap_server_name)); + ldap_client_params_blueprint.insert_or_assign(ldap_server_name, parseLDAPServer(config, ldap_server_name)); } catch (...) { tryLogCurrentException(log, "Could not parse LDAP server " + backQuote(ldap_server_name)); } } + + try + { + if (kerberos_keys_count > 0) + kerberos_params = parseKerberosParams(config); + } + catch (...) + { + tryLogCurrentException(log, "Could not parse Kerberos section"); + } } -bool ExternalAuthenticators::checkLDAPCredentials(const String & server, const String & user_name, const String & password, - const LDAPSearchParamsList * search_params, LDAPSearchResultsList * search_results) const +bool ExternalAuthenticators::checkLDAPCredentials(const String & server, const BasicCredentials & credentials, + const LDAPClient::SearchParamsList * search_params, LDAPClient::SearchResultsList * search_results) const { - std::optional params; + std::optional params; std::size_t params_hash = 0; { std::scoped_lock lock(mutex); // Retrieve the server parameters. - const auto pit = ldap_server_params.find(server); - if (pit == ldap_server_params.end()) + const auto pit = ldap_client_params_blueprint.find(server); + if (pit == ldap_client_params_blueprint.end()) throw Exception("LDAP server '" + server + "' is not configured", ErrorCodes::BAD_ARGUMENTS); params = pit->second; - params->user = user_name; - params->password = password; + params->user = credentials.getUserName(); + params->password = credentials.getPassword(); params->combineCoreHash(params_hash); if (search_params) @@ -207,12 +278,12 @@ bool ExternalAuthenticators::checkLDAPCredentials(const String & server, const S // Check the cache, but only if the caching is enabled at all. if (params->verification_cooldown > std::chrono::seconds{0}) { - const auto cit = ldap_server_caches.find(server); - if (cit != ldap_server_caches.end()) + const auto cit = ldap_caches.find(server); + if (cit != ldap_caches.end()) { auto & cache = cit->second; - const auto eit = cache.find(user_name); + const auto eit = cache.find(credentials.getUserName()); if (eit != cache.end()) { const auto & entry = eit->second; @@ -249,7 +320,7 @@ bool ExternalAuthenticators::checkLDAPCredentials(const String & server, const S // Erase the cache, if empty. if (cache.empty()) - ldap_server_caches.erase(cit); + ldap_caches.erase(cit); } } } @@ -264,13 +335,13 @@ bool ExternalAuthenticators::checkLDAPCredentials(const String & server, const S std::scoped_lock lock(mutex); // If the server was removed from the config while we were checking the password, we discard the current result. - const auto pit = ldap_server_params.find(server); - if (pit == ldap_server_params.end()) + const auto pit = ldap_client_params_blueprint.find(server); + if (pit == ldap_client_params_blueprint.end()) return false; auto new_params = pit->second; - new_params.user = user_name; - new_params.password = password; + new_params.user = credentials.getUserName(); + new_params.password = credentials.getPassword(); std::size_t new_params_hash = 0; new_params.combineCoreHash(new_params_hash); @@ -286,7 +357,7 @@ bool ExternalAuthenticators::checkLDAPCredentials(const String & server, const S if (params_hash != new_params_hash) return false; - auto & entry = ldap_server_caches[server][user_name]; + auto & entry = ldap_caches[server][credentials.getUserName()]; if (entry.last_successful_authentication_timestamp < current_check_timestamp) { entry.last_successful_params_hash = params_hash; @@ -314,4 +385,33 @@ bool ExternalAuthenticators::checkLDAPCredentials(const String & server, const S return result; } +bool ExternalAuthenticators::checkKerberosCredentials(const String & realm, const GSSAcceptorContext & credentials) const +{ + std::scoped_lock lock(mutex); + + if (!kerberos_params.has_value()) + throw Exception("Kerberos is not enabled", ErrorCodes::BAD_ARGUMENTS); + + if (!credentials.isReady()) + return false; + + if (credentials.isFailed()) + return false; + + if (!realm.empty() && realm != credentials.getRealm()) + return false; + + return true; +} + +GSSAcceptorContext::Params ExternalAuthenticators::getKerberosParams() const +{ + std::scoped_lock lock(mutex); + + if (!kerberos_params.has_value()) + throw Exception("Kerberos is not enabled", ErrorCodes::BAD_ARGUMENTS); + + return kerberos_params.value(); +} + } diff --git a/src/Access/ExternalAuthenticators.h b/src/Access/ExternalAuthenticators.h index abcc8e8d10d..c8feea7eada 100644 --- a/src/Access/ExternalAuthenticators.h +++ b/src/Access/ExternalAuthenticators.h @@ -1,11 +1,14 @@ #pragma once -#include +#include +#include +#include #include #include #include #include +#include #include @@ -28,25 +31,31 @@ class ExternalAuthenticators public: void reset(); void setConfiguration(const Poco::Util::AbstractConfiguration & config, Poco::Logger * log); - bool checkLDAPCredentials(const String & server, const String & user_name, const String & password, - const LDAPSearchParamsList * search_params = nullptr, LDAPSearchResultsList * search_results = nullptr) const; + + // The name and readiness of the credentials must be verified before calling these. + bool checkLDAPCredentials(const String & server, const BasicCredentials & credentials, + const LDAPClient::SearchParamsList * search_params = nullptr, LDAPClient::SearchResultsList * search_results = nullptr) const; + bool checkKerberosCredentials(const String & realm, const GSSAcceptorContext & credentials) const; + + GSSAcceptorContext::Params getKerberosParams() const; private: struct LDAPCacheEntry { std::size_t last_successful_params_hash = 0; std::chrono::steady_clock::time_point last_successful_authentication_timestamp; - LDAPSearchResultsList last_successful_search_results; + LDAPClient::SearchResultsList last_successful_search_results; }; - using LDAPServerCache = std::unordered_map; // user name -> cache entry - using LDAPServerCaches = std::map; // server name -> cache - using LDAPServersParams = std::map; // server name -> params + using LDAPCache = std::unordered_map; // user name -> cache entry + using LDAPCaches = std::map; // server name -> cache + using LDAPParams = std::map; // server name -> params private: mutable std::recursive_mutex mutex; - LDAPServersParams ldap_server_params; - mutable LDAPServerCaches ldap_server_caches; + LDAPParams ldap_client_params_blueprint; + mutable LDAPCaches ldap_caches; + std::optional kerberos_params; }; } diff --git a/src/Access/GSSAcceptor.cpp b/src/Access/GSSAcceptor.cpp new file mode 100644 index 00000000000..49b83cc883c --- /dev/null +++ b/src/Access/GSSAcceptor.cpp @@ -0,0 +1,469 @@ +#include +#include +#include + +#include + +#include +#include + + +namespace DB +{ + +namespace ErrorCodes +{ + extern const int BAD_ARGUMENTS; + extern const int FEATURE_IS_NOT_ENABLED_AT_BUILD_TIME; + extern const int KERBEROS_ERROR; +} + +GSSAcceptorContext::GSSAcceptorContext(const GSSAcceptorContext::Params& params_) + : params(params_) +{ +} + +GSSAcceptorContext::~GSSAcceptorContext() +{ + resetHandles(); +} + +const String & GSSAcceptorContext::getRealm() const +{ + if (!isReady()) + throwNotReady(); + return realm; +} + +bool GSSAcceptorContext::isFailed() const +{ + return is_failed; +} + +#if USE_KRB5 + +namespace +{ + +std::recursive_mutex gss_global_mutex; + +struct PrincipalName +{ + explicit PrincipalName(String principal); +// operator String() const; + + String name; + std::vector instances; + String realm; +}; + +PrincipalName::PrincipalName(String principal) +{ + const auto at_pos = principal.find('@'); + if (at_pos != std::string::npos) + { + realm = principal.substr(at_pos + 1); + principal.resize(at_pos); + } + + Poco::StringTokenizer st(principal, "/"); + auto it = st.begin(); + if (it != st.end()) + { + name = *it; + instances.assign(++it, st.end()); + } +} + +/* +PrincipalName::operator String() const +{ + String principal = name; + + for (const auto & instance : instances) + { + principal += '/'; + principal += instance; + } + + principal += '@'; + principal += realm; + + return principal; +} +*/ + +String bufferToString(const gss_buffer_desc & buf) +{ + String str; + + if (buf.length > 0 && buf.value != nullptr) + { + str.assign(static_cast(buf.value), buf.length); + while (!str.empty() && str.back() == '\0') { str.pop_back(); } + } + + return str; +} + +String extractSpecificStatusMessages(OM_uint32 status_code, int status_type, const gss_OID & mech_type) +{ + std::scoped_lock lock(gss_global_mutex); + + String messages; + OM_uint32 message_context = 0; + + do + { + gss_buffer_desc status_string_buf; + status_string_buf.length = 0; + status_string_buf.value = nullptr; + + SCOPE_EXIT({ + OM_uint32 minor_status = 0; + [[maybe_unused]] OM_uint32 major_status = gss_release_buffer( + &minor_status, + &status_string_buf + ); + }); + + OM_uint32 minor_status = 0; + [[maybe_unused]] OM_uint32 major_status = gss_display_status( + &minor_status, + status_code, + status_type, + mech_type, + &message_context, + &status_string_buf + ); + + const auto message = bufferToString(status_string_buf); + + if (!message.empty()) + { + if (!messages.empty()) + messages += ", "; + + messages += message; + } + } while (message_context != 0); + + return messages; +} + +String extractStatusMessages(OM_uint32 major_status_code, OM_uint32 minor_status_code, const gss_OID & mech_type) +{ + std::scoped_lock lock(gss_global_mutex); + + const auto gss_messages = extractSpecificStatusMessages(major_status_code, GSS_C_GSS_CODE, mech_type); + const auto mech_messages = extractSpecificStatusMessages(minor_status_code, GSS_C_MECH_CODE, mech_type); + + String messages; + + if (!gss_messages.empty()) + messages += "Majors: " + gss_messages; + + if (!mech_messages.empty()) + { + if (!messages.empty()) + messages += "; "; + + messages += "Minors: " + mech_messages; + } + + return messages; +} + +std::pair extractNameAndRealm(const gss_name_t & name) +{ + std::scoped_lock lock(gss_global_mutex); + + gss_buffer_desc name_buf; + name_buf.length = 0; + name_buf.value = nullptr; + + SCOPE_EXIT({ + OM_uint32 minor_status = 0; + [[maybe_unused]] OM_uint32 major_status = gss_release_buffer( + &minor_status, + &name_buf + ); + }); + + OM_uint32 minor_status = 0; + [[maybe_unused]] OM_uint32 major_status = gss_display_name( + &minor_status, + name, + &name_buf, + nullptr + ); + + const PrincipalName principal(bufferToString(name_buf)); + return { principal.name, principal.realm }; +} + +bool equalMechanisms(const String & left_str, const gss_OID & right_oid) +{ + std::scoped_lock lock(gss_global_mutex); + + gss_buffer_desc left_buf; + left_buf.length = left_str.size(); + left_buf.value = const_cast(left_str.c_str()); + + gss_OID left_oid = GSS_C_NO_OID; + + SCOPE_EXIT({ + if (left_oid != GSS_C_NO_OID) + { + OM_uint32 minor_status = 0; + [[maybe_unused]] OM_uint32 major_status = gss_release_oid( + &minor_status, + &left_oid + ); + left_oid = GSS_C_NO_OID; + } + }); + + OM_uint32 minor_status = 0; + OM_uint32 major_status = gss_str_to_oid( + &minor_status, + &left_buf, + &left_oid + ); + + if (GSS_ERROR(major_status)) + return false; + + return gss_oid_equal(left_oid, right_oid); +} + +} + +void GSSAcceptorContext::reset() +{ + is_ready = false; + is_failed = false; + user_name.clear(); + realm.clear(); + initHandles(); +} + +void GSSAcceptorContext::resetHandles() noexcept +{ + std::scoped_lock lock(gss_global_mutex); + + if (acceptor_credentials_handle != GSS_C_NO_CREDENTIAL) + { + OM_uint32 minor_status = 0; + [[maybe_unused]] OM_uint32 major_status = gss_release_cred( + &minor_status, + &acceptor_credentials_handle + ); + acceptor_credentials_handle = GSS_C_NO_CREDENTIAL; + } + + if (context_handle != GSS_C_NO_CONTEXT) + { + OM_uint32 minor_status = 0; + [[maybe_unused]] OM_uint32 major_status = gss_delete_sec_context( + &minor_status, + &context_handle, + GSS_C_NO_BUFFER + ); + context_handle = GSS_C_NO_CONTEXT; + } +} + +void GSSAcceptorContext::initHandles() +{ + std::scoped_lock lock(gss_global_mutex); + + resetHandles(); + + if (!params.principal.empty()) + { + if (!params.realm.empty()) + throw Exception("Realm and principal name cannot be specified simultaneously", ErrorCodes::BAD_ARGUMENTS); + + gss_buffer_desc acceptor_name_buf; + acceptor_name_buf.length = params.principal.size(); + acceptor_name_buf.value = const_cast(params.principal.c_str()); + + gss_name_t acceptor_name = GSS_C_NO_NAME; + + SCOPE_EXIT({ + if (acceptor_name != GSS_C_NO_NAME) + { + OM_uint32 minor_status = 0; + [[maybe_unused]] OM_uint32 major_status = gss_release_name( + &minor_status, + &acceptor_name + ); + acceptor_name = GSS_C_NO_NAME; + } + }); + + OM_uint32 minor_status = 0; + OM_uint32 major_status = gss_import_name( + &minor_status, + &acceptor_name_buf, + GSS_C_NT_HOSTBASED_SERVICE, + &acceptor_name + ); + + if (GSS_ERROR(major_status)) + { + const auto messages = extractStatusMessages(major_status, minor_status, GSS_C_NO_OID); + throw Exception("gss_import_name() failed" + (messages.empty() ? "" : ": " + messages), ErrorCodes::KERBEROS_ERROR); + } + + minor_status = 0; + major_status = gss_acquire_cred( + &minor_status, + acceptor_name, + GSS_C_INDEFINITE, + GSS_C_NO_OID_SET, + GSS_C_ACCEPT, + &acceptor_credentials_handle, + nullptr, + nullptr + ); + + if (GSS_ERROR(major_status)) + { + const auto messages = extractStatusMessages(major_status, minor_status, GSS_C_NO_OID); + throw Exception("gss_acquire_cred() failed" + (messages.empty() ? "" : ": " + messages), ErrorCodes::KERBEROS_ERROR); + } + } +} + +String GSSAcceptorContext::processToken(const String & input_token, Poco::Logger * log) +{ + std::scoped_lock lock(gss_global_mutex); + + String output_token; + + try + { + if (is_ready || is_failed || context_handle == GSS_C_NO_CONTEXT) + reset(); + + gss_buffer_desc input_token_buf; + input_token_buf.length = input_token.size(); + input_token_buf.value = const_cast(input_token.c_str()); + + gss_buffer_desc output_token_buf; + output_token_buf.length = 0; + output_token_buf.value = nullptr; + + gss_name_t initiator_name = GSS_C_NO_NAME; + gss_OID mech_type = GSS_C_NO_OID; + OM_uint32 flags = 0; + + SCOPE_EXIT({ + if (initiator_name != GSS_C_NO_NAME) + { + OM_uint32 minor_status = 0; + [[maybe_unused]] OM_uint32 major_status = gss_release_name( + &minor_status, + &initiator_name + ); + initiator_name = GSS_C_NO_NAME; + } + + OM_uint32 minor_status = 0; + [[maybe_unused]] OM_uint32 major_status = gss_release_buffer( + &minor_status, + &output_token_buf + ); + }); + + OM_uint32 minor_status = 0; + OM_uint32 major_status = gss_accept_sec_context( + &minor_status, + &context_handle, + acceptor_credentials_handle, + &input_token_buf, + GSS_C_NO_CHANNEL_BINDINGS, + &initiator_name, + &mech_type, + &output_token_buf, + &flags, + nullptr, + nullptr + ); + + if (major_status == GSS_S_COMPLETE) + { + if (!params.mechanism.empty() && !equalMechanisms(params.mechanism, mech_type)) + throw Exception("gss_accept_sec_context() succeeded, but: the authentication mechanism is not what was expected", ErrorCodes::KERBEROS_ERROR); + + if (flags & GSS_C_ANON_FLAG) + throw Exception("gss_accept_sec_context() succeeded, but: the initiator does not wish to be authenticated", ErrorCodes::KERBEROS_ERROR); + + std::tie(user_name, realm) = extractNameAndRealm(initiator_name); + + if (user_name.empty()) + throw Exception("gss_accept_sec_context() succeeded, but: the initiator name cannot be extracted", ErrorCodes::KERBEROS_ERROR); + + if (realm.empty()) + throw Exception("gss_accept_sec_context() succeeded, but: the initiator realm cannot be extracted", ErrorCodes::KERBEROS_ERROR); + + if (!params.realm.empty() && params.realm != realm) + throw Exception("gss_accept_sec_context() succeeded, but: the initiator realm is not what was expected (expected: " + params.realm + ", actual: " + realm + ")", ErrorCodes::KERBEROS_ERROR); + + output_token = bufferToString(output_token_buf); + + is_ready = true; + is_failed = false; + + resetHandles(); + } + else if (!GSS_ERROR(major_status) && (major_status & GSS_S_CONTINUE_NEEDED)) + { + output_token = bufferToString(output_token_buf); + + is_ready = false; + is_failed = false; + } + else + { + const auto messages = extractStatusMessages(major_status, minor_status, mech_type); + throw Exception("gss_accept_sec_context() failed" + (messages.empty() ? "" : ": " + messages), ErrorCodes::KERBEROS_ERROR); + } + } + catch (...) + { + tryLogCurrentException(log, "Could not process GSS token"); + + is_ready = true; + is_failed = true; + + resetHandles(); + } + + return output_token; +} + +#else // USE_KRB5 + +void GSSAcceptorContext::reset() +{ +} + +void GSSAcceptorContext::resetHandles() noexcept +{ +} + +void GSSAcceptorContext::initHandles() +{ +} + +String GSSAcceptorContext::processToken(const String &, Poco::Logger *) +{ + throw Exception("ClickHouse was built without GSS-API/Kerberos support", ErrorCodes::FEATURE_IS_NOT_ENABLED_AT_BUILD_TIME); +} + +#endif // USE_KRB5 + +} diff --git a/src/Access/GSSAcceptor.h b/src/Access/GSSAcceptor.h new file mode 100644 index 00000000000..8d207c59c01 --- /dev/null +++ b/src/Access/GSSAcceptor.h @@ -0,0 +1,66 @@ +#pragma once + +#if !defined(ARCADIA_BUILD) +# include "config_core.h" +#endif + +#include +#include +#include + +#if USE_KRB5 +# include +# include +# define MAYBE_NORETURN +#else +# define MAYBE_NORETURN [[noreturn]] +#endif + +namespace Poco { class Logger; } + +namespace DB +{ + +class GSSAcceptorContext + : public Credentials +{ +public: + struct Params + { + String mechanism = "1.2.840.113554.1.2.2"; // OID: krb5 + String principal; + String realm; + }; + + explicit GSSAcceptorContext(const Params& params_); + virtual ~GSSAcceptorContext() override; + + GSSAcceptorContext(const GSSAcceptorContext &) = delete; + GSSAcceptorContext(GSSAcceptorContext &&) = delete; + GSSAcceptorContext & operator= (const GSSAcceptorContext &) = delete; + GSSAcceptorContext & operator= (GSSAcceptorContext &&) = delete; + + const String & getRealm() const; + bool isFailed() const; + MAYBE_NORETURN String processToken(const String & input_token, Poco::Logger * log); + +private: + void reset(); + void resetHandles() noexcept; + void initHandles(); + +private: + const Params params; + + bool is_failed = false; + String realm; + +#if USE_KRB5 + gss_ctx_id_t context_handle = GSS_C_NO_CONTEXT; + gss_cred_id_t acceptor_credentials_handle = GSS_C_NO_CREDENTIAL; +#endif +}; + +} + +#undef MAYBE_NORETURN diff --git a/src/Access/IAccessStorage.cpp b/src/Access/IAccessStorage.cpp index c68f5f55ef5..7b42abb2737 100644 --- a/src/Access/IAccessStorage.cpp +++ b/src/Access/IAccessStorage.cpp @@ -1,5 +1,6 @@ #include #include +#include #include #include #include @@ -417,53 +418,60 @@ void IAccessStorage::notify(const Notifications & notifications) UUID IAccessStorage::login( - const String & user_name, - const String & password, + const Credentials & credentials, const Poco::Net::IPAddress & address, const ExternalAuthenticators & external_authenticators, bool replace_exception_with_cannot_authenticate) const { try { - return loginImpl(user_name, password, address, external_authenticators); + return loginImpl(credentials, address, external_authenticators); } catch (...) { if (!replace_exception_with_cannot_authenticate) throw; - tryLogCurrentException(getLogger(), user_name + ": Authentication failed"); - throwCannotAuthenticate(user_name); + tryLogCurrentException(getLogger(), credentials.getUserName() + ": Authentication failed"); + throwCannotAuthenticate(credentials.getUserName()); } } UUID IAccessStorage::loginImpl( - const String & user_name, - const String & password, + const Credentials & credentials, const Poco::Net::IPAddress & address, const ExternalAuthenticators & external_authenticators) const { - if (auto id = find(user_name)) + if (auto id = find(credentials.getUserName())) { if (auto user = tryRead(*id)) { - if (!isPasswordCorrectImpl(*user, password, external_authenticators)) - throwInvalidPassword(); - if (!isAddressAllowedImpl(*user, address)) throwAddressNotAllowed(address); + if (!areCredentialsValidImpl(*user, credentials, external_authenticators)) + throwInvalidCredentials(); + return *id; } } - throwNotFound(EntityType::USER, user_name); + throwNotFound(EntityType::USER, credentials.getUserName()); } -bool IAccessStorage::isPasswordCorrectImpl(const User & user, const String & password, const ExternalAuthenticators & external_authenticators) const +bool IAccessStorage::areCredentialsValidImpl( + const User & user, + const Credentials & credentials, + const ExternalAuthenticators & external_authenticators) const { - return user.authentication.isCorrectPassword(user.getName(), password, external_authenticators); + if (!credentials.isReady()) + return false; + + if (credentials.getUserName() != user.getName()) + return false; + + return user.authentication.areCredentialsValid(credentials, external_authenticators); } @@ -472,6 +480,7 @@ bool IAccessStorage::isAddressAllowedImpl(const User & user, const Poco::Net::IP return user.allowed_client_hosts.contains(address); } + UUID IAccessStorage::getIDOfLoggedUser(const String & user_name) const { return getIDOfLoggedUserImpl(user_name); @@ -578,9 +587,9 @@ void IAccessStorage::throwAddressNotAllowed(const Poco::Net::IPAddress & address throw Exception("Connections from " + address.toString() + " are not allowed", ErrorCodes::IP_ADDRESS_NOT_ALLOWED); } -void IAccessStorage::throwInvalidPassword() +void IAccessStorage::throwInvalidCredentials() { - throw Exception("Invalid password", ErrorCodes::WRONG_PASSWORD); + throw Exception("Invalid credentials", ErrorCodes::WRONG_PASSWORD); } void IAccessStorage::throwCannotAuthenticate(const String & user_name) diff --git a/src/Access/IAccessStorage.h b/src/Access/IAccessStorage.h index ecf6b260712..2cdd8eabf73 100644 --- a/src/Access/IAccessStorage.h +++ b/src/Access/IAccessStorage.h @@ -16,6 +16,7 @@ namespace Poco::Net { class IPAddress; } namespace DB { struct User; +class Credentials; class ExternalAuthenticators; /// Contains entities, i.e. instances of classes derived from IAccessEntity. @@ -142,11 +143,11 @@ public: bool hasSubscription(EntityType type) const; bool hasSubscription(const UUID & id) const; - /// Finds an user, check its password and returns the ID of the user. - /// Throws an exception if no such user or password is incorrect. - UUID login(const String & user_name, const String & password, const Poco::Net::IPAddress & address, const ExternalAuthenticators & external_authenticators, bool replace_exception_with_cannot_authenticate = true) const; + /// Finds a user, check the provided credentials and returns the ID of the user if they are valid. + /// Throws an exception if no such user or credentials are invalid. + UUID login(const Credentials & credentials, const Poco::Net::IPAddress & address, const ExternalAuthenticators & external_authenticators, bool replace_exception_with_cannot_authenticate = true) const; - /// Returns the ID of an user who has logged in (maybe on another node). + /// Returns the ID of a user who has logged in (maybe on another node). /// The function assumes that the password has been already checked somehow, so we can skip checking it now. UUID getIDOfLoggedUser(const String & user_name) const; @@ -164,8 +165,8 @@ protected: virtual ext::scope_guard subscribeForChangesImpl(EntityType type, const OnChangedHandler & handler) const = 0; virtual bool hasSubscriptionImpl(const UUID & id) const = 0; virtual bool hasSubscriptionImpl(EntityType type) const = 0; - virtual UUID loginImpl(const String & user_name, const String & password, const Poco::Net::IPAddress & address, const ExternalAuthenticators & external_authenticators) const; - virtual bool isPasswordCorrectImpl(const User & user, const String & password, const ExternalAuthenticators & external_authenticators) const; + virtual UUID loginImpl(const Credentials & credentials, const Poco::Net::IPAddress & address, const ExternalAuthenticators & external_authenticators) const; + virtual bool areCredentialsValidImpl(const User & user, const Credentials & credentials, const ExternalAuthenticators & external_authenticators) const; virtual bool isAddressAllowedImpl(const User & user, const Poco::Net::IPAddress & address) const; virtual UUID getIDOfLoggedUserImpl(const String & user_name) const; @@ -183,7 +184,7 @@ protected: [[noreturn]] void throwReadonlyCannotUpdate(EntityType type, const String & name) const; [[noreturn]] void throwReadonlyCannotRemove(EntityType type, const String & name) const; [[noreturn]] static void throwAddressNotAllowed(const Poco::Net::IPAddress & address); - [[noreturn]] static void throwInvalidPassword(); + [[noreturn]] static void throwInvalidCredentials(); [[noreturn]] static void throwCannotAuthenticate(const String & user_name); using Notification = std::tuple; diff --git a/src/Access/LDAPAccessStorage.cpp b/src/Access/LDAPAccessStorage.cpp index 2602422a59a..b69f489ca6b 100644 --- a/src/Access/LDAPAccessStorage.cpp +++ b/src/Access/LDAPAccessStorage.cpp @@ -3,6 +3,7 @@ #include #include #include +#include #include #include #include @@ -35,7 +36,7 @@ LDAPAccessStorage::LDAPAccessStorage(const String & storage_name_, AccessControl String LDAPAccessStorage::getLDAPServerName() const { - return ldap_server; + return ldap_server_name; } @@ -53,8 +54,8 @@ void LDAPAccessStorage::setConfiguration(AccessControlManager * access_control_m if (!has_server) throw Exception("Missing 'server' field for LDAP user directory", ErrorCodes::BAD_ARGUMENTS); - const auto ldap_server_cfg = config.getString(prefix_str + "server"); - if (ldap_server_cfg.empty()) + const auto ldap_server_name_cfg = config.getString(prefix_str + "server"); + if (ldap_server_name_cfg.empty()) throw Exception("Empty 'server' field for LDAP user directory", ErrorCodes::BAD_ARGUMENTS); std::set common_roles_cfg; @@ -67,7 +68,7 @@ void LDAPAccessStorage::setConfiguration(AccessControlManager * access_control_m common_roles_cfg.insert(role_names.begin(), role_names.end()); } - LDAPSearchParamsList role_search_params_cfg; + LDAPClient::SearchParamsList role_search_params_cfg; if (has_role_mapping) { Poco::Util::AbstractConfiguration::Keys all_keys; @@ -89,17 +90,17 @@ void LDAPAccessStorage::setConfiguration(AccessControlManager * access_control_m auto scope = config.getString(rm_prefix_str + "scope", "subtree"); boost::algorithm::to_lower(scope); - if (scope == "base") rm_params.scope = LDAPSearchParams::Scope::BASE; - else if (scope == "one_level") rm_params.scope = LDAPSearchParams::Scope::ONE_LEVEL; - else if (scope == "subtree") rm_params.scope = LDAPSearchParams::Scope::SUBTREE; - else if (scope == "children") rm_params.scope = LDAPSearchParams::Scope::CHILDREN; + if (scope == "base") rm_params.scope = LDAPClient::SearchParams::Scope::BASE; + else if (scope == "one_level") rm_params.scope = LDAPClient::SearchParams::Scope::ONE_LEVEL; + else if (scope == "subtree") rm_params.scope = LDAPClient::SearchParams::Scope::SUBTREE; + else if (scope == "children") rm_params.scope = LDAPClient::SearchParams::Scope::CHILDREN; else throw Exception("Invalid value of 'scope' field in '" + key + "' section of LDAP user directory, must be one of 'base', 'one_level', 'subtree', or 'children'", ErrorCodes::BAD_ARGUMENTS); } } access_control_manager = access_control_manager_; - ldap_server = ldap_server_cfg; + ldap_server_name = ldap_server_name_cfg; role_search_params.swap(role_search_params_cfg); common_role_names.swap(common_roles_cfg); @@ -218,14 +219,14 @@ void LDAPAccessStorage::applyRoleChangeNoLock(bool grant, const UUID & role_id, } -void LDAPAccessStorage::assignRolesNoLock(User & user, const LDAPSearchResultsList & external_roles) const +void LDAPAccessStorage::assignRolesNoLock(User & user, const LDAPClient::SearchResultsList & external_roles) const { - const auto external_roles_hash = boost::hash{}(external_roles); + const auto external_roles_hash = boost::hash{}(external_roles); return assignRolesNoLock(user, external_roles, external_roles_hash); } -void LDAPAccessStorage::assignRolesNoLock(User & user, const LDAPSearchResultsList & external_roles, const std::size_t external_roles_hash) const +void LDAPAccessStorage::assignRolesNoLock(User & user, const LDAPClient::SearchResultsList & external_roles, const std::size_t external_roles_hash) const { const auto & user_name = user.getName(); auto & granted_roles = user.granted_roles.roles; @@ -312,10 +313,10 @@ void LDAPAccessStorage::assignRolesNoLock(User & user, const LDAPSearchResultsLi } -void LDAPAccessStorage::updateAssignedRolesNoLock(const UUID & id, const String & user_name, const LDAPSearchResultsList & external_roles) const +void LDAPAccessStorage::updateAssignedRolesNoLock(const UUID & id, const String & user_name, const LDAPClient::SearchResultsList & external_roles) const { // No need to include common_role_names in this hash each time, since they don't change. - const auto external_roles_hash = boost::hash{}(external_roles); + const auto external_roles_hash = boost::hash{}(external_roles); // Map and grant the roles from scratch only if the list of external role has changed. const auto it = external_role_hashes.find(user_name); @@ -337,7 +338,7 @@ void LDAPAccessStorage::updateAssignedRolesNoLock(const UUID & id, const String } -std::set LDAPAccessStorage::mapExternalRolesNoLock(const LDAPSearchResultsList & external_roles) const +std::set LDAPAccessStorage::mapExternalRolesNoLock(const LDAPClient::SearchResultsList & external_roles) const { std::set role_names; @@ -365,10 +366,19 @@ std::set LDAPAccessStorage::mapExternalRolesNoLock(const LDAPSearchResul } -bool LDAPAccessStorage::isPasswordCorrectLDAPNoLock(const String & user_name, const String & password, - const ExternalAuthenticators & external_authenticators, LDAPSearchResultsList & search_results) const +bool LDAPAccessStorage::areLDAPCredentialsValidNoLock(const User & user, const Credentials & credentials, + const ExternalAuthenticators & external_authenticators, LDAPClient::SearchResultsList & search_results) const { - return external_authenticators.checkLDAPCredentials(ldap_server, user_name, password, &role_search_params, &search_results); + if (!credentials.isReady()) + return false; + + if (credentials.getUserName() != user.getName()) + return false; + + if (const auto * basic_credentials = dynamic_cast(&credentials)) + return external_authenticators.checkLDAPCredentials(ldap_server_name, *basic_credentials, &role_search_params, &search_results); + + return false; } @@ -383,7 +393,7 @@ String LDAPAccessStorage::getStorageParamsJSON() const std::scoped_lock lock(mutex); Poco::JSON::Object params_json; - params_json.set("server", ldap_server); + params_json.set("server", ldap_server_name); Poco::JSON::Array common_role_names_json; for (const auto & role : common_role_names) @@ -405,10 +415,10 @@ String LDAPAccessStorage::getStorageParamsJSON() const String scope; switch (role_mapping.scope) { - case LDAPSearchParams::Scope::BASE: scope = "base"; break; - case LDAPSearchParams::Scope::ONE_LEVEL: scope = "one_level"; break; - case LDAPSearchParams::Scope::SUBTREE: scope = "subtree"; break; - case LDAPSearchParams::Scope::CHILDREN: scope = "children"; break; + case LDAPClient::SearchParams::Scope::BASE: scope = "base"; break; + case LDAPClient::SearchParams::Scope::ONE_LEVEL: scope = "one_level"; break; + case LDAPClient::SearchParams::Scope::SUBTREE: scope = "subtree"; break; + case LDAPClient::SearchParams::Scope::CHILDREN: scope = "children"; break; } role_mapping_json.set("scope", scope); @@ -514,23 +524,23 @@ bool LDAPAccessStorage::hasSubscriptionImpl(EntityType type) const return memory_storage.hasSubscription(type); } -UUID LDAPAccessStorage::loginImpl(const String & user_name, const String & password, const Poco::Net::IPAddress & address, const ExternalAuthenticators & external_authenticators) const +UUID LDAPAccessStorage::loginImpl(const Credentials & credentials, const Poco::Net::IPAddress & address, const ExternalAuthenticators & external_authenticators) const { std::scoped_lock lock(mutex); - LDAPSearchResultsList external_roles; - auto id = memory_storage.find(user_name); + LDAPClient::SearchResultsList external_roles; + auto id = memory_storage.find(credentials.getUserName()); if (id) { auto user = memory_storage.read(*id); - if (!isPasswordCorrectLDAPNoLock(user->getName(), password, external_authenticators, external_roles)) - throwInvalidPassword(); - if (!isAddressAllowedImpl(*user, address)) throwAddressNotAllowed(address); + if (!areLDAPCredentialsValidNoLock(*user, credentials, external_authenticators, external_roles)) + throwInvalidCredentials(); + // Just in case external_roles are changed. This will be no-op if they are not. - updateAssignedRolesNoLock(*id, user_name, external_roles); + updateAssignedRolesNoLock(*id, user->getName(), external_roles); return *id; } @@ -538,16 +548,16 @@ UUID LDAPAccessStorage::loginImpl(const String & user_name, const String & passw { // User does not exist, so we create one, and will add it if authentication is successful. auto user = std::make_shared(); - user->setName(user_name); - user->authentication = Authentication(Authentication::Type::LDAP_SERVER); - user->authentication.setServerName(ldap_server); - - if (!isPasswordCorrectLDAPNoLock(user->getName(), password, external_authenticators, external_roles)) - throwInvalidPassword(); + user->setName(credentials.getUserName()); + user->authentication = Authentication(Authentication::Type::LDAP); + user->authentication.setLDAPServerName(ldap_server_name); if (!isAddressAllowedImpl(*user, address)) throwAddressNotAllowed(address); + if (!areLDAPCredentialsValidNoLock(*user, credentials, external_authenticators, external_roles)) + throwInvalidCredentials(); + assignRolesNoLock(*user, external_roles); return memory_storage.insert(user); @@ -567,10 +577,10 @@ UUID LDAPAccessStorage::getIDOfLoggedUserImpl(const String & user_name) const // User does not exist, so we create one, and add it pretending that the authentication is successful. auto user = std::make_shared(); user->setName(user_name); - user->authentication = Authentication(Authentication::Type::LDAP_SERVER); - user->authentication.setServerName(ldap_server); + user->authentication = Authentication(Authentication::Type::LDAP); + user->authentication.setLDAPServerName(ldap_server_name); - LDAPSearchResultsList external_roles; + LDAPClient::SearchResultsList external_roles; // TODO: mapped external roles are not available here. Without a password we can't authenticate and retrieve roles from LDAP server. diff --git a/src/Access/LDAPAccessStorage.h b/src/Access/LDAPAccessStorage.h index b3d82d1e86b..ea0ab47c225 100644 --- a/src/Access/LDAPAccessStorage.h +++ b/src/Access/LDAPAccessStorage.h @@ -1,7 +1,9 @@ #pragma once #include -#include +#include +#include +#include #include #include #include @@ -21,14 +23,10 @@ namespace Poco namespace DB { class AccessControlManager; -struct LDAPSearchParams; -using LDAPSearchParamsList = std::vector; -using LDAPSearchResults = std::set; -using LDAPSearchResultsList = std::vector; /// Implementation of IAccessStorage which allows attaching users from a remote LDAP server. /// Currently, any user name will be treated as a name of an existing remote user, -/// a user info entity will be created, with LDAP_SERVER authentication type. +/// a user info entity will be created, with LDAP authentication type. class LDAPAccessStorage : public IAccessStorage { public: @@ -57,7 +55,7 @@ private: // IAccessStorage implementations. virtual ext::scope_guard subscribeForChangesImpl(EntityType type, const OnChangedHandler & handler) const override; virtual bool hasSubscriptionImpl(const UUID & id) const override; virtual bool hasSubscriptionImpl(EntityType type) const override; - virtual UUID loginImpl(const String & user_name, const String & password, const Poco::Net::IPAddress & address, const ExternalAuthenticators & external_authenticators) const override; + virtual UUID loginImpl(const Credentials & credentials, const Poco::Net::IPAddress & address, const ExternalAuthenticators & external_authenticators) const override; virtual UUID getIDOfLoggedUserImpl(const String & user_name) const override; private: @@ -65,19 +63,19 @@ private: void processRoleChange(const UUID & id, const AccessEntityPtr & entity); void applyRoleChangeNoLock(bool grant, const UUID & role_id, const String & role_name); - void assignRolesNoLock(User & user, const LDAPSearchResultsList & external_roles) const; - void assignRolesNoLock(User & user, const LDAPSearchResultsList & external_roles, const std::size_t external_roles_hash) const; - void updateAssignedRolesNoLock(const UUID & id, const String & user_name, const LDAPSearchResultsList & external_roles) const; - std::set mapExternalRolesNoLock(const LDAPSearchResultsList & external_roles) const; - bool isPasswordCorrectLDAPNoLock(const String & user_name, const String & password, - const ExternalAuthenticators & external_authenticators, LDAPSearchResultsList & search_results) const; + void assignRolesNoLock(User & user, const LDAPClient::SearchResultsList & external_roles) const; + void assignRolesNoLock(User & user, const LDAPClient::SearchResultsList & external_roles, const std::size_t external_roles_hash) const; + void updateAssignedRolesNoLock(const UUID & id, const String & user_name, const LDAPClient::SearchResultsList & external_roles) const; + std::set mapExternalRolesNoLock(const LDAPClient::SearchResultsList & external_roles) const; + bool areLDAPCredentialsValidNoLock(const User & user, const Credentials & credentials, + const ExternalAuthenticators & external_authenticators, LDAPClient::SearchResultsList & search_results) const; mutable std::recursive_mutex mutex; AccessControlManager * access_control_manager = nullptr; - String ldap_server; - LDAPSearchParamsList role_search_params; + String ldap_server_name; + LDAPClient::SearchParamsList role_search_params; std::set common_role_names; // role name that should be granted to all users at all times - mutable std::map external_role_hashes; // user name -> LDAPSearchResultsList hash (most recently retrieved and processed) + mutable std::map external_role_hashes; // user name -> LDAPClient::SearchResultsList hash (most recently retrieved and processed) mutable std::map> users_per_roles; // role name -> user names (...it should be granted to; may but don't have to exist for common roles) mutable std::map> roles_per_users; // user name -> role names (...that should be granted to it; may but don't have to include common roles) mutable std::map granted_role_names; // (currently granted) role id -> its name diff --git a/src/Access/LDAPClient.cpp b/src/Access/LDAPClient.cpp index 41756aebb9a..5c4b7dd8d99 100644 --- a/src/Access/LDAPClient.cpp +++ b/src/Access/LDAPClient.cpp @@ -5,6 +5,7 @@ #include #include +#include #include #include @@ -25,7 +26,25 @@ namespace ErrorCodes extern const int LDAP_ERROR; } -LDAPClient::LDAPClient(const LDAPServerParams & params_) +void LDAPClient::SearchParams::combineHash(std::size_t & seed) const +{ + boost::hash_combine(seed, base_dn); + boost::hash_combine(seed, static_cast(scope)); + boost::hash_combine(seed, search_filter); + boost::hash_combine(seed, attribute); + boost::hash_combine(seed, prefix); +} + +void LDAPClient::Params::combineCoreHash(std::size_t & seed) const +{ + boost::hash_combine(seed, host); + boost::hash_combine(seed, port); + boost::hash_combine(seed, bind_dn); + boost::hash_combine(seed, user); + boost::hash_combine(seed, password); +} + +LDAPClient::LDAPClient(const Params & params_) : params(params_) { } @@ -143,7 +162,7 @@ void LDAPClient::openConnection() LDAPURLDesc url; std::memset(&url, 0, sizeof(url)); - url.lud_scheme = const_cast(params.enable_tls == LDAPServerParams::TLSEnable::YES ? "ldaps" : "ldap"); + url.lud_scheme = const_cast(params.enable_tls == LDAPClient::Params::TLSEnable::YES ? "ldaps" : "ldap"); url.lud_host = const_cast(params.host.c_str()); url.lud_port = params.port; url.lud_scope = LDAP_SCOPE_DEFAULT; @@ -163,8 +182,8 @@ void LDAPClient::openConnection() int value = 0; switch (params.protocol_version) { - case LDAPServerParams::ProtocolVersion::V2: value = LDAP_VERSION2; break; - case LDAPServerParams::ProtocolVersion::V3: value = LDAP_VERSION3; break; + case LDAPClient::Params::ProtocolVersion::V2: value = LDAP_VERSION2; break; + case LDAPClient::Params::ProtocolVersion::V3: value = LDAP_VERSION3; break; } diag(ldap_set_option(handle, LDAP_OPT_PROTOCOL_VERSION, &value)); } @@ -208,11 +227,11 @@ void LDAPClient::openConnection() int value = 0; switch (params.tls_minimum_protocol_version) { - case LDAPServerParams::TLSProtocolVersion::SSL2: value = LDAP_OPT_X_TLS_PROTOCOL_SSL2; break; - case LDAPServerParams::TLSProtocolVersion::SSL3: value = LDAP_OPT_X_TLS_PROTOCOL_SSL3; break; - case LDAPServerParams::TLSProtocolVersion::TLS1_0: value = LDAP_OPT_X_TLS_PROTOCOL_TLS1_0; break; - case LDAPServerParams::TLSProtocolVersion::TLS1_1: value = LDAP_OPT_X_TLS_PROTOCOL_TLS1_1; break; - case LDAPServerParams::TLSProtocolVersion::TLS1_2: value = LDAP_OPT_X_TLS_PROTOCOL_TLS1_2; break; + case LDAPClient::Params::TLSProtocolVersion::SSL2: value = LDAP_OPT_X_TLS_PROTOCOL_SSL2; break; + case LDAPClient::Params::TLSProtocolVersion::SSL3: value = LDAP_OPT_X_TLS_PROTOCOL_SSL3; break; + case LDAPClient::Params::TLSProtocolVersion::TLS1_0: value = LDAP_OPT_X_TLS_PROTOCOL_TLS1_0; break; + case LDAPClient::Params::TLSProtocolVersion::TLS1_1: value = LDAP_OPT_X_TLS_PROTOCOL_TLS1_1; break; + case LDAPClient::Params::TLSProtocolVersion::TLS1_2: value = LDAP_OPT_X_TLS_PROTOCOL_TLS1_2; break; } diag(ldap_set_option(handle, LDAP_OPT_X_TLS_PROTOCOL_MIN, &value)); } @@ -223,10 +242,10 @@ void LDAPClient::openConnection() int value = 0; switch (params.tls_require_cert) { - case LDAPServerParams::TLSRequireCert::NEVER: value = LDAP_OPT_X_TLS_NEVER; break; - case LDAPServerParams::TLSRequireCert::ALLOW: value = LDAP_OPT_X_TLS_ALLOW; break; - case LDAPServerParams::TLSRequireCert::TRY: value = LDAP_OPT_X_TLS_TRY; break; - case LDAPServerParams::TLSRequireCert::DEMAND: value = LDAP_OPT_X_TLS_DEMAND; break; + case LDAPClient::Params::TLSRequireCert::NEVER: value = LDAP_OPT_X_TLS_NEVER; break; + case LDAPClient::Params::TLSRequireCert::ALLOW: value = LDAP_OPT_X_TLS_ALLOW; break; + case LDAPClient::Params::TLSRequireCert::TRY: value = LDAP_OPT_X_TLS_TRY; break; + case LDAPClient::Params::TLSRequireCert::DEMAND: value = LDAP_OPT_X_TLS_DEMAND; break; } diag(ldap_set_option(handle, LDAP_OPT_X_TLS_REQUIRE_CERT, &value)); } @@ -264,12 +283,12 @@ void LDAPClient::openConnection() } #endif - if (params.enable_tls == LDAPServerParams::TLSEnable::YES_STARTTLS) + if (params.enable_tls == LDAPClient::Params::TLSEnable::YES_STARTTLS) diag(ldap_start_tls_s(handle, nullptr, nullptr)); switch (params.sasl_mechanism) { - case LDAPServerParams::SASLMechanism::SIMPLE: + case LDAPClient::Params::SASLMechanism::SIMPLE: { const auto escaped_user_name = escapeForLDAP(params.user); const auto bind_dn = replacePlaceholders(params.bind_dn, { {"{user_name}", escaped_user_name} }); @@ -299,19 +318,19 @@ void LDAPClient::closeConnection() noexcept handle = nullptr; } -LDAPSearchResults LDAPClient::search(const LDAPSearchParams & search_params) +LDAPClient::SearchResults LDAPClient::search(const SearchParams & search_params) { std::scoped_lock lock(ldap_global_mutex); - LDAPSearchResults result; + SearchResults result; int scope = 0; switch (search_params.scope) { - case LDAPSearchParams::Scope::BASE: scope = LDAP_SCOPE_BASE; break; - case LDAPSearchParams::Scope::ONE_LEVEL: scope = LDAP_SCOPE_ONELEVEL; break; - case LDAPSearchParams::Scope::SUBTREE: scope = LDAP_SCOPE_SUBTREE; break; - case LDAPSearchParams::Scope::CHILDREN: scope = LDAP_SCOPE_CHILDREN; break; + case SearchParams::Scope::BASE: scope = LDAP_SCOPE_BASE; break; + case SearchParams::Scope::ONE_LEVEL: scope = LDAP_SCOPE_ONELEVEL; break; + case SearchParams::Scope::SUBTREE: scope = LDAP_SCOPE_SUBTREE; break; + case SearchParams::Scope::CHILDREN: scope = LDAP_SCOPE_CHILDREN; break; } const auto escaped_user_name = escapeForLDAP(params.user); @@ -452,7 +471,7 @@ LDAPSearchResults LDAPClient::search(const LDAPSearchParams & search_params) return result; } -bool LDAPSimpleAuthClient::authenticate(const LDAPSearchParamsList * search_params, LDAPSearchResultsList * search_results) +bool LDAPSimpleAuthClient::authenticate(const SearchParamsList * search_params, SearchResultsList * search_results) { if (params.user.empty()) throw Exception("LDAP authentication of a user with empty name is not allowed", ErrorCodes::BAD_ARGUMENTS); @@ -508,12 +527,12 @@ void LDAPClient::closeConnection() noexcept { } -LDAPSearchResults LDAPClient::search(const LDAPSearchParams &) +LDAPClient::SearchResults LDAPClient::search(const SearchParams &) { throw Exception("ClickHouse was built without LDAP support", ErrorCodes::FEATURE_IS_NOT_ENABLED_AT_BUILD_TIME); } -bool LDAPSimpleAuthClient::authenticate(const LDAPSearchParamsList *, LDAPSearchResultsList *) +bool LDAPSimpleAuthClient::authenticate(const SearchParamsList *, SearchResultsList *) { throw Exception("ClickHouse was built without LDAP support", ErrorCodes::FEATURE_IS_NOT_ENABLED_AT_BUILD_TIME); } diff --git a/src/Access/LDAPClient.h b/src/Access/LDAPClient.h index f0ace69649b..4fc97bb957b 100644 --- a/src/Access/LDAPClient.h +++ b/src/Access/LDAPClient.h @@ -4,7 +4,6 @@ # include "config_core.h" #endif -#include #include #if USE_LDAP @@ -14,6 +13,10 @@ # define MAYBE_NORETURN [[noreturn]] #endif +#include +#include +#include + namespace DB { @@ -21,7 +24,98 @@ namespace DB class LDAPClient { public: - explicit LDAPClient(const LDAPServerParams & params_); + struct SearchParams + { + enum class Scope + { + BASE, + ONE_LEVEL, + SUBTREE, + CHILDREN + }; + + String base_dn; + Scope scope = Scope::SUBTREE; + String search_filter; + String attribute = "cn"; + String prefix; + + void combineHash(std::size_t & seed) const; + }; + + using SearchParamsList = std::vector; + using SearchResults = std::set; + using SearchResultsList = std::vector; + + struct Params + { + enum class ProtocolVersion + { + V2, + V3 + }; + + enum class TLSEnable + { + NO, + YES_STARTTLS, + YES + }; + + enum class TLSProtocolVersion + { + SSL2, + SSL3, + TLS1_0, + TLS1_1, + TLS1_2 + }; + + enum class TLSRequireCert + { + NEVER, + ALLOW, + TRY, + DEMAND + }; + + enum class SASLMechanism + { + UNKNOWN, + SIMPLE + }; + + ProtocolVersion protocol_version = ProtocolVersion::V3; + + String host; + std::uint16_t port = 636; + + TLSEnable enable_tls = TLSEnable::YES; + TLSProtocolVersion tls_minimum_protocol_version = TLSProtocolVersion::TLS1_2; + TLSRequireCert tls_require_cert = TLSRequireCert::DEMAND; + String tls_cert_file; + String tls_key_file; + String tls_ca_cert_file; + String tls_ca_cert_dir; + String tls_cipher_suite; + + SASLMechanism sasl_mechanism = SASLMechanism::SIMPLE; + + String bind_dn; + String user; + String password; + + std::chrono::seconds verification_cooldown{0}; + + std::chrono::seconds operation_timeout{40}; + std::chrono::seconds network_timeout{30}; + std::chrono::seconds search_timeout{20}; + std::uint32_t search_limit = 100; + + void combineCoreHash(std::size_t & seed) const; + }; + + explicit LDAPClient(const Params & params_); ~LDAPClient(); LDAPClient(const LDAPClient &) = delete; @@ -33,10 +127,10 @@ protected: MAYBE_NORETURN void diag(const int rc, String text = ""); MAYBE_NORETURN void openConnection(); void closeConnection() noexcept; - LDAPSearchResults search(const LDAPSearchParams & search_params); + SearchResults search(const SearchParams & search_params); protected: - const LDAPServerParams params; + const Params params; #if USE_LDAP LDAP * handle = nullptr; #endif @@ -47,7 +141,7 @@ class LDAPSimpleAuthClient { public: using LDAPClient::LDAPClient; - bool authenticate(const LDAPSearchParamsList * search_params, LDAPSearchResultsList * search_results); + bool authenticate(const SearchParamsList * search_params, SearchResultsList * search_results); }; } diff --git a/src/Access/LDAPParams.h b/src/Access/LDAPParams.h deleted file mode 100644 index 5181b2d1621..00000000000 --- a/src/Access/LDAPParams.h +++ /dev/null @@ -1,120 +0,0 @@ -#pragma once - -#include - -#include - -#include -#include -#include - - -namespace DB -{ - -struct LDAPSearchParams -{ - enum class Scope - { - BASE, - ONE_LEVEL, - SUBTREE, - CHILDREN - }; - - String base_dn; - Scope scope = Scope::SUBTREE; - String search_filter; - String attribute = "cn"; - String prefix; - - void combineHash(std::size_t & seed) const - { - boost::hash_combine(seed, base_dn); - boost::hash_combine(seed, static_cast(scope)); - boost::hash_combine(seed, search_filter); - boost::hash_combine(seed, attribute); - boost::hash_combine(seed, prefix); - } -}; - -using LDAPSearchParamsList = std::vector; -using LDAPSearchResults = std::set; -using LDAPSearchResultsList = std::vector; - -struct LDAPServerParams -{ - enum class ProtocolVersion - { - V2, - V3 - }; - - enum class TLSEnable - { - NO, - YES_STARTTLS, - YES - }; - - enum class TLSProtocolVersion - { - SSL2, - SSL3, - TLS1_0, - TLS1_1, - TLS1_2 - }; - - enum class TLSRequireCert - { - NEVER, - ALLOW, - TRY, - DEMAND - }; - - enum class SASLMechanism - { - UNKNOWN, - SIMPLE - }; - - ProtocolVersion protocol_version = ProtocolVersion::V3; - - String host; - std::uint16_t port = 636; - - TLSEnable enable_tls = TLSEnable::YES; - TLSProtocolVersion tls_minimum_protocol_version = TLSProtocolVersion::TLS1_2; - TLSRequireCert tls_require_cert = TLSRequireCert::DEMAND; - String tls_cert_file; - String tls_key_file; - String tls_ca_cert_file; - String tls_ca_cert_dir; - String tls_cipher_suite; - - SASLMechanism sasl_mechanism = SASLMechanism::SIMPLE; - - String bind_dn; - String user; - String password; - - std::chrono::seconds verification_cooldown{0}; - - std::chrono::seconds operation_timeout{40}; - std::chrono::seconds network_timeout{30}; - std::chrono::seconds search_timeout{20}; - std::uint32_t search_limit = 100; - - void combineCoreHash(std::size_t & seed) const - { - boost::hash_combine(seed, host); - boost::hash_combine(seed, port); - boost::hash_combine(seed, bind_dn); - boost::hash_combine(seed, user); - boost::hash_combine(seed, password); - } -}; - -} diff --git a/src/Access/MultipleAccessStorage.cpp b/src/Access/MultipleAccessStorage.cpp index a8ce3f602ed..15281d8c471 100644 --- a/src/Access/MultipleAccessStorage.cpp +++ b/src/Access/MultipleAccessStorage.cpp @@ -1,4 +1,5 @@ #include +#include #include #include #include @@ -382,6 +383,7 @@ void MultipleAccessStorage::updateSubscriptionsToNestedStorages(std::unique_lock /// Lock the mutex again to store added subscriptions to the nested storages. lock.lock(); + for (auto type : ext::range(EntityType::MAX)) { if (!added_subscriptions[static_cast(type)].empty()) @@ -399,25 +401,24 @@ void MultipleAccessStorage::updateSubscriptionsToNestedStorages(std::unique_lock } lock.unlock(); - added_subscriptions->clear(); } -UUID MultipleAccessStorage::loginImpl(const String & user_name, const String & password, const Poco::Net::IPAddress & address, const ExternalAuthenticators & external_authenticators) const +UUID MultipleAccessStorage::loginImpl(const Credentials & credentials, const Poco::Net::IPAddress & address, const ExternalAuthenticators & external_authenticators) const { auto storages = getStoragesInternal(); for (const auto & storage : *storages) { try { - auto id = storage->login(user_name, password, address, external_authenticators, /* replace_exception_with_cannot_authenticate = */ false); + auto id = storage->login(credentials, address, external_authenticators, /* replace_exception_with_cannot_authenticate = */ false); std::lock_guard lock{mutex}; ids_cache.set(id, storage); return id; } catch (...) { - if (!storage->find(EntityType::USER, user_name)) + if (!storage->find(EntityType::USER, credentials.getUserName())) { /// The authentication failed because there no users with such name in the `storage` /// thus we can try to search in other nested storages. @@ -426,7 +427,7 @@ UUID MultipleAccessStorage::loginImpl(const String & user_name, const String & p throw; } } - throwNotFound(EntityType::USER, user_name); + throwNotFound(EntityType::USER, credentials.getUserName()); } diff --git a/src/Access/MultipleAccessStorage.h b/src/Access/MultipleAccessStorage.h index 8844de8c029..610bf2fadcc 100644 --- a/src/Access/MultipleAccessStorage.h +++ b/src/Access/MultipleAccessStorage.h @@ -48,7 +48,7 @@ protected: ext::scope_guard subscribeForChangesImpl(EntityType type, const OnChangedHandler & handler) const override; bool hasSubscriptionImpl(const UUID & id) const override; bool hasSubscriptionImpl(EntityType type) const override; - UUID loginImpl(const String & user_name, const String & password, const Poco::Net::IPAddress & address, const ExternalAuthenticators & external_authenticators) const override; + UUID loginImpl(const Credentials & credentials, const Poco::Net::IPAddress & address, const ExternalAuthenticators & external_authenticators) const override; UUID getIDOfLoggedUserImpl(const String & user_name) const override; private: diff --git a/src/Access/UsersConfigAccessStorage.cpp b/src/Access/UsersConfigAccessStorage.cpp index 33efd71d0d0..407c3679e9f 100644 --- a/src/Access/UsersConfigAccessStorage.cpp +++ b/src/Access/UsersConfigAccessStorage.cpp @@ -62,14 +62,15 @@ namespace bool has_password_sha256_hex = config.has(user_config + ".password_sha256_hex"); bool has_password_double_sha1_hex = config.has(user_config + ".password_double_sha1_hex"); bool has_ldap = config.has(user_config + ".ldap"); + bool has_kerberos = config.has(user_config + ".kerberos"); - size_t num_password_fields = has_no_password + has_password_plaintext + has_password_sha256_hex + has_password_double_sha1_hex + has_ldap; + size_t num_password_fields = has_no_password + has_password_plaintext + has_password_sha256_hex + has_password_double_sha1_hex + has_ldap + has_kerberos; if (num_password_fields > 1) - throw Exception("More than one field of 'password', 'password_sha256_hex', 'password_double_sha1_hex', 'no_password', 'ldap' are used to specify password for user " + user_name + ". Must be only one of them.", + throw Exception("More than one field of 'password', 'password_sha256_hex', 'password_double_sha1_hex', 'no_password', 'ldap', 'kerberos' are used to specify password for user " + user_name + ". Must be only one of them.", ErrorCodes::BAD_ARGUMENTS); if (num_password_fields < 1) - throw Exception("Either 'password' or 'password_sha256_hex' or 'password_double_sha1_hex' or 'no_password' or 'ldap' must be specified for user " + user_name + ".", ErrorCodes::BAD_ARGUMENTS); + throw Exception("Either 'password' or 'password_sha256_hex' or 'password_double_sha1_hex' or 'no_password' or 'ldap' or 'kerberos' must be specified for user " + user_name + ".", ErrorCodes::BAD_ARGUMENTS); if (has_password_plaintext) { @@ -96,8 +97,15 @@ namespace if (ldap_server_name.empty()) throw Exception("LDAP server name cannot be empty for user " + user_name + ".", ErrorCodes::BAD_ARGUMENTS); - user->authentication = Authentication{Authentication::LDAP_SERVER}; - user->authentication.setServerName(ldap_server_name); + user->authentication = Authentication{Authentication::LDAP}; + user->authentication.setLDAPServerName(ldap_server_name); + } + else if (has_kerberos) + { + const auto realm = config.getString(user_config + ".kerberos.realm", ""); + + user->authentication = Authentication{Authentication::KERBEROS}; + user->authentication.setKerberosRealm(realm); } const auto profile_name_config = user_config + ".profile"; diff --git a/src/Access/ya.make b/src/Access/ya.make index b4469aa3167..e8584230538 100644 --- a/src/Access/ya.make +++ b/src/Access/ya.make @@ -15,6 +15,7 @@ SRCS( AllowedClientHosts.cpp Authentication.cpp ContextAccess.cpp + Credentials.cpp DiskAccessStorage.cpp EnabledQuota.cpp EnabledRoles.cpp @@ -22,6 +23,7 @@ SRCS( EnabledRowPolicies.cpp EnabledSettings.cpp ExternalAuthenticators.cpp + GSSAcceptor.cpp GrantedRoles.cpp IAccessEntity.cpp IAccessStorage.cpp diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index b80bcfdf4d4..412da2fc00f 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -316,6 +316,7 @@ if (USE_CYRUS_SASL) endif() if (USE_KRB5) + dbms_target_include_directories(SYSTEM BEFORE PRIVATE ${KRB5_INCLUDE_DIR}) dbms_target_link_libraries(PRIVATE ${KRB5_LIBRARY}) endif() diff --git a/src/Common/ErrorCodes.cpp b/src/Common/ErrorCodes.cpp index 2e8145144ce..04b5b374a56 100644 --- a/src/Common/ErrorCodes.cpp +++ b/src/Common/ErrorCodes.cpp @@ -542,6 +542,7 @@ M(573, EPOLL_ERROR) \ M(574, DISTRIBUTED_TOO_MANY_PENDING_BYTES) \ M(575, UNKNOWN_SNAPSHOT) \ + M(576, KERBEROS_ERROR) \ \ M(999, KEEPER_EXCEPTION) \ M(1000, POCO_EXCEPTION) \ diff --git a/src/Core/config_core.h.in b/src/Core/config_core.h.in index 666ef32efdf..e250e013913 100644 --- a/src/Core/config_core.h.in +++ b/src/Core/config_core.h.in @@ -14,3 +14,4 @@ #cmakedefine01 USE_ROCKSDB #cmakedefine01 USE_LIBPQXX #cmakedefine01 USE_NURAFT +#cmakedefine01 USE_KRB5 diff --git a/src/Interpreters/Context.cpp b/src/Interpreters/Context.cpp index 6f27a6bdb7d..8615cf70343 100644 --- a/src/Interpreters/Context.cpp +++ b/src/Interpreters/Context.cpp @@ -36,8 +36,11 @@ #include #include #include +#include #include #include +#include +#include #include #include #include @@ -669,6 +672,12 @@ void Context::setExternalAuthenticatorsConfig(const Poco::Util::AbstractConfigur shared->access_control_manager.setExternalAuthenticatorsConfig(config); } +std::unique_ptr Context::makeGSSAcceptorContext() const +{ + auto lock = getLock(); + return std::make_unique(shared->access_control_manager.getExternalAuthenticators().getKerberosParams()); +} + void Context::setUsersConfig(const ConfigurationPtr & config) { auto lock = getLock(); @@ -683,29 +692,22 @@ ConfigurationPtr Context::getUsersConfig() } -void Context::setUserImpl(const String & name, const std::optional & password, const Poco::Net::SocketAddress & address) +void Context::setUser(const Credentials & credentials, const Poco::Net::SocketAddress & address) { auto lock = getLock(); - client_info.current_user = name; + client_info.current_user = credentials.getUserName(); client_info.current_address = address; #if defined(ARCADIA_BUILD) /// This is harmful field that is used only in foreign "Arcadia" build. - client_info.current_password = password.value_or(""); + client_info.current_password.clear(); + if (const auto * basic_credentials = dynamic_cast(&credentials)) + client_info.current_password = basic_credentials->getPassword(); #endif - /// Find a user with such name and check the password. - UUID new_user_id; - if (password) - new_user_id = getAccessControlManager().login(name, *password, address.host()); - else - { - /// Access w/o password is done under interserver-secret (remote_servers.secret) - /// So it is okay not to check client's host in this case (since there is trust). - new_user_id = getAccessControlManager().getIDOfLoggedUser(name); - } - + /// Find a user with such name and check the credentials. + auto new_user_id = getAccessControlManager().login(credentials, address.host()); auto new_access = getAccessControlManager().getContextAccess( new_user_id, /* current_roles = */ {}, /* use_default_roles = */ true, settings, current_database, client_info); @@ -720,12 +722,12 @@ void Context::setUserImpl(const String & name, const std::optional & pas void Context::setUser(const String & name, const String & password, const Poco::Net::SocketAddress & address) { - setUserImpl(name, password, address); + setUser(BasicCredentials(name, password), address); } void Context::setUserWithoutCheckingPassword(const String & name, const Poco::Net::SocketAddress & address) { - setUserImpl(name, {} /* no password */, address); + setUser(AlwaysAllowCredentials(name), address); } std::shared_ptr Context::getUser() const diff --git a/src/Interpreters/Context.h b/src/Interpreters/Context.h index 563fb172488..563239cd88d 100644 --- a/src/Interpreters/Context.h +++ b/src/Interpreters/Context.h @@ -93,6 +93,8 @@ using ActionLocksManagerPtr = std::shared_ptr; class ShellCommand; class ICompressionCodec; class AccessControlManager; +class Credentials; +class GSSAcceptorContext; class SettingsConstraints; class RemoteHostFilter; struct StorageID; @@ -322,9 +324,12 @@ public: AccessControlManager & getAccessControlManager(); const AccessControlManager & getAccessControlManager() const; - /// Sets external authenticators config (LDAP). + /// Sets external authenticators config (LDAP, Kerberos). void setExternalAuthenticatorsConfig(const Poco::Util::AbstractConfiguration & config); + /// Creates GSSAcceptorContext instance based on external authenticator params. + std::unique_ptr makeGSSAcceptorContext() const; + /** Take the list of users, quotas and configuration profiles from this config. * The list of users is completely replaced. * The accumulated quota values are not reset if the quota is not deleted. @@ -332,11 +337,12 @@ public: void setUsersConfig(const ConfigurationPtr & config); ConfigurationPtr getUsersConfig(); - /// Sets the current user, checks the password and that the specified host is allowed. - /// Must be called before getClientInfo. + /// Sets the current user, checks the credentials and that the specified host is allowed. + /// Must be called before getClientInfo() can be called. + void setUser(const Credentials & credentials, const Poco::Net::SocketAddress & address); void setUser(const String & name, const String & password, const Poco::Net::SocketAddress & address); - /// Sets the current user, *do not checks the password and that the specified host is allowed*. + /// Sets the current user, *does not check the password/credentials and that the specified host is allowed*. /// Must be called before getClientInfo. /// /// (Used only internally in cluster, if the secret matches) @@ -782,9 +788,6 @@ private: StoragePolicySelectorPtr getStoragePolicySelector(std::lock_guard & lock) const; DiskSelectorPtr getDiskSelector(std::lock_guard & /* lock */) const; - - /// If the password is not set, the password will not be checked - void setUserImpl(const String & name, const std::optional & password, const Poco::Net::SocketAddress & address); }; diff --git a/src/Parsers/ASTCreateUserQuery.cpp b/src/Parsers/ASTCreateUserQuery.cpp index 4b2aa70785a..e2e477fa622 100644 --- a/src/Parsers/ASTCreateUserQuery.cpp +++ b/src/Parsers/ASTCreateUserQuery.cpp @@ -34,9 +34,14 @@ namespace } String authentication_type_name = Authentication::TypeInfo::get(authentication_type).name; + String by_keyword = "BY"; std::optional by_value; - if (show_password || authentication_type == Authentication::LDAP_SERVER) + if ( + show_password || + authentication_type == Authentication::LDAP || + authentication_type == Authentication::KERBEROS + ) { switch (authentication_type) { @@ -57,9 +62,18 @@ namespace by_value = authentication.getPasswordHashHex(); break; } - case Authentication::LDAP_SERVER: + case Authentication::LDAP: { - by_value = authentication.getServerName(); + by_keyword = "SERVER"; + by_value = authentication.getLDAPServerName(); + break; + } + case Authentication::KERBEROS: + { + by_keyword = "REALM"; + const auto & realm = authentication.getKerberosRealm(); + if (!realm.empty()) + by_value = realm; break; } @@ -71,9 +85,12 @@ namespace settings.ostr << (settings.hilite ? IAST::hilite_keyword : "") << " IDENTIFIED WITH " << authentication_type_name << (settings.hilite ? IAST::hilite_none : ""); + if (by_value) - settings.ostr << (settings.hilite ? IAST::hilite_keyword : "") << " BY " << (settings.hilite ? IAST::hilite_none : "") - << quoteString(*by_value); + { + settings.ostr << (settings.hilite ? IAST::hilite_keyword : "") << " " << by_keyword << " " + << (settings.hilite ? IAST::hilite_none : "") << quoteString(*by_value); + } } diff --git a/src/Parsers/ASTCreateUserQuery.h b/src/Parsers/ASTCreateUserQuery.h index 1fbe7eeeb4c..7acfd87909a 100644 --- a/src/Parsers/ASTCreateUserQuery.h +++ b/src/Parsers/ASTCreateUserQuery.h @@ -13,14 +13,14 @@ class ASTRolesOrUsersSet; class ASTSettingsProfileElements; /** CREATE USER [IF NOT EXISTS | OR REPLACE] name - * [NOT IDENTIFIED | IDENTIFIED [WITH {no_password|plaintext_password|sha256_password|sha256_hash|double_sha1_password|double_sha1_hash|ldap_server}] BY {'password'|'hash'|'server_name'}] + * [NOT IDENTIFIED | IDENTIFIED {[WITH {no_password|plaintext_password|sha256_password|sha256_hash|double_sha1_password|double_sha1_hash}] BY {'password'|'hash'}}|{WITH ldap SERVER 'server_name'}|{WITH kerberos [REALM 'realm']}] * [HOST {LOCAL | NAME 'name' | REGEXP 'name_regexp' | IP 'address' | LIKE 'pattern'} [,...] | ANY | NONE] * [DEFAULT ROLE role [,...]] * [SETTINGS variable [= value] [MIN [=] min_value] [MAX [=] max_value] [READONLY|WRITABLE] | PROFILE 'profile_name'] [,...] * * ALTER USER [IF EXISTS] name * [RENAME TO new_name] - * [NOT IDENTIFIED | IDENTIFIED [WITH {no_password|plaintext_password|sha256_password|sha256_hash|double_sha1_password|double_sha1_hash|ldap_server}] BY {'password'|'hash'|'server_name'}] + * [NOT IDENTIFIED | IDENTIFIED {[WITH {no_password|plaintext_password|sha256_password|sha256_hash|double_sha1_password|double_sha1_hash}] BY {'password'|'hash'}}|{WITH ldap SERVER 'server_name'}|{WITH kerberos [REALM 'realm']}] * [[ADD|DROP] HOST {LOCAL | NAME 'name' | REGEXP 'name_regexp' | IP 'address' | LIKE 'pattern'} [,...] | ANY | NONE] * [DEFAULT ROLE role [,...] | ALL | ALL EXCEPT role [,...] ] * [SETTINGS variable [= value] [MIN [=] min_value] [MAX [=] max_value] [READONLY|WRITABLE] | PROFILE 'profile_name'] [,...] diff --git a/src/Parsers/ParserCreateUserQuery.cpp b/src/Parsers/ParserCreateUserQuery.cpp index 98faa0b15eb..16c539d3ebc 100644 --- a/src/Parsers/ParserCreateUserQuery.cpp +++ b/src/Parsers/ParserCreateUserQuery.cpp @@ -49,7 +49,8 @@ namespace std::optional type; bool expect_password = false; bool expect_hash = false; - bool expect_server_name = false; + bool expect_ldap_server_name = false; + bool expect_kerberos_realm = false; if (ParserKeyword{"WITH"}.ignore(pos, expected)) { @@ -59,8 +60,10 @@ namespace { type = check_type; - if (check_type == Authentication::LDAP_SERVER) - expect_server_name = true; + if (check_type == Authentication::LDAP) + expect_ldap_server_name = true; + else if (check_type == Authentication::KERBEROS) + expect_kerberos_realm = true; else if (check_type != Authentication::NO_PASSWORD) expect_password = true; @@ -92,7 +95,7 @@ namespace } String value; - if (expect_password || expect_hash || expect_server_name) + if (expect_password || expect_hash) { ASTPtr ast; if (!ParserKeyword{"BY"}.ignore(pos, expected) || !ParserStringLiteral{}.parse(pos, ast, expected)) @@ -100,14 +103,35 @@ namespace value = ast->as().value.safeGet(); } + else if (expect_ldap_server_name) + { + ASTPtr ast; + if (!ParserKeyword{"SERVER"}.ignore(pos, expected) || !ParserStringLiteral{}.parse(pos, ast, expected)) + return false; + + value = ast->as().value.safeGet(); + } + else if (expect_kerberos_realm) + { + if (ParserKeyword{"REALM"}.ignore(pos, expected)) + { + ASTPtr ast; + if (!ParserStringLiteral{}.parse(pos, ast, expected)) + return false; + + value = ast->as().value.safeGet(); + } + } authentication = Authentication{*type}; if (expect_password) authentication.setPassword(value); else if (expect_hash) authentication.setPasswordHashHex(value); - else if (expect_server_name) - authentication.setServerName(value); + else if (expect_ldap_server_name) + authentication.setLDAPServerName(value); + else if (expect_kerberos_realm) + authentication.setKerberosRealm(value); return true; }); diff --git a/src/Parsers/ParserCreateUserQuery.h b/src/Parsers/ParserCreateUserQuery.h index 69100d1211d..5b83a261fa2 100644 --- a/src/Parsers/ParserCreateUserQuery.h +++ b/src/Parsers/ParserCreateUserQuery.h @@ -7,13 +7,13 @@ namespace DB { /** Parses queries like * CREATE USER [IF NOT EXISTS | OR REPLACE] name - * [NOT IDENTIFIED | IDENTIFIED [WITH {no_password|plaintext_password|sha256_password|sha256_hash|double_sha1_password|double_sha1_hash|ldap_server}] BY {'password'|'hash'|'server_name'}] + * [NOT IDENTIFIED | IDENTIFIED {[WITH {no_password|plaintext_password|sha256_password|sha256_hash|double_sha1_password|double_sha1_hash}] BY {'password'|'hash'}}|{WITH ldap SERVER 'server_name'}|{WITH kerberos [REALM 'realm']}] * [HOST {LOCAL | NAME 'name' | REGEXP 'name_regexp' | IP 'address' | LIKE 'pattern'} [,...] | ANY | NONE] * [SETTINGS variable [= value] [MIN [=] min_value] [MAX [=] max_value] [READONLY|WRITABLE] | PROFILE 'profile_name'] [,...] * * ALTER USER [IF EXISTS] name * [RENAME TO new_name] - * [NOT IDENTIFIED | IDENTIFIED [WITH {no_password|plaintext_password|sha256_password|sha256_hash|double_sha1_password|double_sha1_hash|ldap_server}] BY {'password'|'hash'|'server_name'}] + * [NOT IDENTIFIED | IDENTIFIED {[WITH {no_password|plaintext_password|sha256_password|sha256_hash|double_sha1_password|double_sha1_hash}] BY {'password'|'hash'}}|{WITH ldap SERVER 'server_name'}|{WITH kerberos [REALM 'realm']}] * [[ADD|DROP] HOST {LOCAL | NAME 'name' | REGEXP 'name_regexp' | IP 'address' | LIKE 'pattern'} [,...] | ANY | NONE] * [SETTINGS variable [= value] [MIN [=] min_value] [MAX [=] max_value] [READONLY|WRITABLE] | PROFILE 'profile_name'] [,...] */ diff --git a/src/Server/HTTPHandler.cpp b/src/Server/HTTPHandler.cpp index d200ee7421f..6b4981beae0 100644 --- a/src/Server/HTTPHandler.cpp +++ b/src/Server/HTTPHandler.cpp @@ -1,5 +1,8 @@ #include +#include +#include +#include #include #include #include @@ -34,13 +37,19 @@ # include #endif +#include +#include #include #include #include #include +#include +#include +#include #include #include +#include namespace DB @@ -93,12 +102,32 @@ namespace ErrorCodes extern const int UNKNOWN_USER; extern const int WRONG_PASSWORD; extern const int REQUIRED_PASSWORD; + extern const int AUTHENTICATION_FAILED; extern const int BAD_REQUEST_PARAMETER; extern const int INVALID_SESSION_TIMEOUT; extern const int HTTP_LENGTH_REQUIRED; } +static String base64Decode(const String & encoded) +{ + String decoded; + Poco::MemoryInputStream istr(encoded.data(), encoded.size()); + Poco::Base64Decoder decoder(istr); + Poco::StreamCopier::copyToString(decoder, decoded); + return decoded; +} + +static String base64Encode(const String & decoded) +{ + std::ostringstream ostr; // STYLE_CHECK_ALLOW_STD_STRING_STREAM + ostr.exceptions(std::ios::failbit); + Poco::Base64Encoder encoder(ostr); + encoder.rdbuf()->setLineLength(0); + encoder << decoded; + encoder.close(); + return ostr.str(); +} static Poco::Net::HTTPResponse::HTTPStatus exceptionCodeToHTTPStatus(int exception_code) { @@ -108,6 +137,12 @@ static Poco::Net::HTTPResponse::HTTPStatus exceptionCodeToHTTPStatus(int excepti { return HTTPResponse::HTTP_UNAUTHORIZED; } + else if (exception_code == ErrorCodes::UNKNOWN_USER || + exception_code == ErrorCodes::WRONG_PASSWORD || + exception_code == ErrorCodes::AUTHENTICATION_FAILED) + { + return HTTPResponse::HTTP_FORBIDDEN; + } else if (exception_code == ErrorCodes::CANNOT_PARSE_TEXT || exception_code == ErrorCodes::CANNOT_PARSE_ESCAPE_SEQUENCE || exception_code == ErrorCodes::CANNOT_PARSE_QUOTED_STRING || @@ -233,15 +268,21 @@ HTTPHandler::HTTPHandler(IServer & server_, const std::string & name) } -void HTTPHandler::processQuery( +/// We need d-tor to be present in this translation unit to make it play well with some +/// forward decls in the header. Other than that, the default d-tor would be OK. +HTTPHandler::~HTTPHandler() +{ + (void)this; +} + + +bool HTTPHandler::authenticateUser( Context & context, HTTPServerRequest & request, HTMLForm & params, - HTTPServerResponse & response, - Output & used_output, - std::optional & query_scope) + HTTPServerResponse & response) { - LOG_TRACE(log, "Request URI: {}", request.getURI()); + using namespace Poco::Net; /// The user and password can be passed by headers (similar to X-Auth-*), /// which is used by load balancers to pass authentication information. @@ -249,16 +290,39 @@ void HTTPHandler::processQuery( std::string password = request.get("X-ClickHouse-Key", ""); std::string quota_key = request.get("X-ClickHouse-Quota", ""); + std::string spnego_challenge; + if (user.empty() && password.empty() && quota_key.empty()) { /// User name and password can be passed using query parameters /// or using HTTP Basic auth (both methods are insecure). if (request.hasCredentials()) { - Poco::Net::HTTPBasicCredentials credentials(request); + /// It is prohibited to mix different authorization schemes. + if (params.has("user") || params.has("password")) + throw Exception("Invalid authentication: it is not allowed to use Authorization HTTP header and authentication via parameters simultaneously", ErrorCodes::AUTHENTICATION_FAILED); - user = credentials.getUsername(); - password = credentials.getPassword(); + std::string scheme; + std::string auth_info; + request.getCredentials(scheme, auth_info); + + if (Poco::icompare(scheme, "Basic") == 0) + { + HTTPBasicCredentials credentials(auth_info); + user = credentials.getUsername(); + password = credentials.getPassword(); + } + else if (Poco::icompare(scheme, "Negotiate") == 0) + { + spnego_challenge = auth_info; + + if (spnego_challenge.empty()) + throw Exception("Invalid authentication: SPNEGO challenge is empty", ErrorCodes::AUTHENTICATION_FAILED); + } + else + { + throw Exception("Invalid authentication: '" + scheme + "' HTTP Authorization scheme is not supported", ErrorCodes::AUTHENTICATION_FAILED); + } } else { @@ -271,12 +335,47 @@ void HTTPHandler::processQuery( else { /// It is prohibited to mix different authorization schemes. - if (request.hasCredentials() - || params.has("user") - || params.has("password") - || params.has("quota_key")) + if (request.hasCredentials() || params.has("user") || params.has("password") || params.has("quota_key")) + throw Exception("Invalid authentication: it is not allowed to use X-ClickHouse HTTP headers and other authentication methods simultaneously", ErrorCodes::AUTHENTICATION_FAILED); + } + + if (spnego_challenge.empty()) // I.e., now using user name and password strings ("Basic"). + { + if (!request_credentials) + request_credentials = std::make_unique(); + + auto * basic_credentials = dynamic_cast(request_credentials.get()); + if (!basic_credentials) + throw Exception("Invalid authentication: unexpected 'Basic' HTTP Authorization scheme", ErrorCodes::AUTHENTICATION_FAILED); + + basic_credentials->setUserName(user); + basic_credentials->setPassword(password); + } + else + { + if (!request_credentials) + request_credentials = request_context->makeGSSAcceptorContext(); + + auto * gss_acceptor_context = dynamic_cast(request_credentials.get()); + if (!gss_acceptor_context) + throw Exception("Invalid authentication: unexpected 'Negotiate' HTTP Authorization scheme expected", ErrorCodes::AUTHENTICATION_FAILED); + +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Wunreachable-code" + const auto spnego_response = base64Encode(gss_acceptor_context->processToken(base64Decode(spnego_challenge), log)); +#pragma GCC diagnostic pop + + if (!spnego_response.empty()) + response.set("WWW-Authenticate", "Negotiate " + spnego_response); + + if (!gss_acceptor_context->isFailed() && !gss_acceptor_context->isReady()) { - throw Exception("Invalid authentication: it is not allowed to use X-ClickHouse HTTP headers and other authentication methods simultaneously", ErrorCodes::REQUIRED_PASSWORD); + if (spnego_response.empty()) + throw Exception("Invalid authentication: 'Negotiate' HTTP Authorization failure", ErrorCodes::AUTHENTICATION_FAILED); + + response.setStatusAndReason(HTTPResponse::HTTP_UNAUTHORIZED); + response.send(); + return false; } } @@ -297,8 +396,39 @@ void HTTPHandler::processQuery( client_info.http_referer = request.get("Referer", ""); client_info.forwarded_for = request.get("X-Forwarded-For", ""); - /// This will also set client_info.current_user and current_address - context.setUser(user, password, request.clientAddress()); + try + { + context.setUser(*request_credentials, request.clientAddress()); + } + catch (const Authentication::Require & required_credentials) + { + request_credentials = std::make_unique(); + + if (required_credentials.getRealm().empty()) + response.set("WWW-Authenticate", "Basic"); + else + response.set("WWW-Authenticate", "Basic realm=\"" + required_credentials.getRealm() + "\""); + + response.setStatusAndReason(HTTPResponse::HTTP_UNAUTHORIZED); + response.send(); + return false; + } + catch (const Authentication::Require & required_credentials) + { + request_credentials = request_context->makeGSSAcceptorContext(); + + if (required_credentials.getRealm().empty()) + response.set("WWW-Authenticate", "Negotiate"); + else + response.set("WWW-Authenticate", "Negotiate realm=\"" + required_credentials.getRealm() + "\""); + + response.setStatusAndReason(HTTPResponse::HTTP_UNAUTHORIZED); + response.send(); + return false; + } + + request_credentials.reset(); + if (!quota_key.empty()) context.setQuotaKey(quota_key); @@ -306,6 +436,25 @@ void HTTPHandler::processQuery( client_info.initial_user = client_info.current_user; client_info.initial_address = client_info.current_address; + return true; +} + + +void HTTPHandler::processQuery( + Context & context, + HTTPServerRequest & request, + HTMLForm & params, + HTTPServerResponse & response, + Output & used_output, + std::optional & query_scope) +{ + using namespace Poco::Net; + + LOG_TRACE(log, "Request URI: {}", request.getURI()); + + if (!authenticateUser(context, request, params, response)) + return; // '401 Unauthorized' response with 'Negotiate' has been sent at this point. + /// The user could specify session identifier and session timeout. /// It allows to modify settings, create temporary tables and reuse them in subsequent requests. @@ -355,6 +504,7 @@ void HTTPHandler::processQuery( // Set the query id supplied by the user, if any, and also update the OpenTelemetry fields. context.setCurrentQueryId(params.get("query_id", request.get("X-ClickHouse-Query-Id", ""))); + ClientInfo & client_info = context.getClientInfo(); client_info.initial_query_id = client_info.current_query_id; /// The client can pass a HTTP header indicating supported compression method (gzip or deflate). @@ -400,7 +550,7 @@ void HTTPHandler::processQuery( used_output.out = std::make_shared( response, - request.getMethod() == Poco::Net::HTTPRequest::HTTP_HEAD, + request.getMethod() == HTTPRequest::HTTP_HEAD, keep_alive_timeout, client_supports_http_compression, http_response_compression_method); @@ -659,11 +809,7 @@ void HTTPHandler::trySendExceptionToClient( request.getStream().ignoreAll(); } - bool auth_fail = exception_code == ErrorCodes::UNKNOWN_USER || - exception_code == ErrorCodes::WRONG_PASSWORD || - exception_code == ErrorCodes::REQUIRED_PASSWORD; - - if (auth_fail) + if (exception_code == ErrorCodes::REQUIRED_PASSWORD) { response.requireAuthentication("ClickHouse server HTTP API"); } @@ -720,12 +866,23 @@ void HTTPHandler::handleRequest(HTTPServerRequest & request, HTTPServerResponse setThreadName("HTTPHandler"); ThreadStatus thread_status; - /// Should be initialized before anything, - /// For correct memory accounting. - Context context = server.context(); + SCOPE_EXIT({ + // If there is no request_credentials instance waiting for the next round, then the request is processed, + // so no need to preserve request_context either. + // Needs to be performed with respect to the other destructors in the scope though. + if (!request_credentials) + request_context.reset(); + }); + + if (!request_context) + { + // Context should be initialized before anything, for correct memory accounting. + request_context = std::make_unique(server.context()); + request_credentials.reset(); + } + /// Cannot be set here, since query_id is unknown. std::optional query_scope; - Output used_output; /// In case of exception, send stack trace to client. @@ -750,11 +907,15 @@ void HTTPHandler::handleRequest(HTTPServerRequest & request, HTTPServerResponse ErrorCodes::HTTP_LENGTH_REQUIRED); } - processQuery(context, request, params, response, used_output, query_scope); - LOG_DEBUG(log, "Done processing query"); + processQuery(*request_context, request, params, response, used_output, query_scope); + LOG_DEBUG(log, (request_credentials ? "Authentication in progress..." : "Done processing query")); } catch (...) { + SCOPE_EXIT({ + request_credentials.reset(); // ...so that the next requests on the connection have to always start afresh in case of exceptions. + }); + tryLogCurrentException(log); /** If exception is received from remote server, then stack trace is embedded in message. diff --git a/src/Server/HTTPHandler.h b/src/Server/HTTPHandler.h index e903fbfbff7..0f1d75664bd 100644 --- a/src/Server/HTTPHandler.h +++ b/src/Server/HTTPHandler.h @@ -18,6 +18,8 @@ namespace Poco { class Logger; } namespace DB { +class Context; +class Credentials; class IServer; class WriteBufferFromHTTPServerResponse; @@ -27,6 +29,7 @@ class HTTPHandler : public HTTPRequestHandler { public: HTTPHandler(IServer & server_, const std::string & name); + virtual ~HTTPHandler() override; void handleRequest(HTTPServerRequest & request, HTTPServerResponse & response) override; @@ -69,6 +72,22 @@ private: CurrentMetrics::Increment metric_increment{CurrentMetrics::HTTPConnection}; + // The request_context and the request_credentials instances may outlive a single request/response loop. + // This happens only when the authentication mechanism requires more than a single request/response exchange (e.g., SPNEGO). + std::unique_ptr request_context; + std::unique_ptr request_credentials; + + // Returns true when the user successfully authenticated, + // the request_context instance will be configured accordingly, and the request_credentials instance will be dropped. + // Returns false when the user is not authenticated yet, and the 'Negotiate' response is sent, + // the request_context and request_credentials instances are preserved. + // Throws an exception if authentication failed. + bool authenticateUser( + Context & context, + HTTPServerRequest & request, + HTMLForm & params, + HTTPServerResponse & response); + /// Also initializes 'used_output'. void processQuery( Context & context, diff --git a/src/Server/HTTPHandlerFactory.cpp b/src/Server/HTTPHandlerFactory.cpp index db80750beb8..1e3d02b85ab 100644 --- a/src/Server/HTTPHandlerFactory.cpp +++ b/src/Server/HTTPHandlerFactory.cpp @@ -2,6 +2,8 @@ #include #include +#include +#include #include diff --git a/src/Storages/System/StorageSystemBuildOptions.generated.cpp.in b/src/Storages/System/StorageSystemBuildOptions.generated.cpp.in index f68e40a41b8..8ece4219d0c 100644 --- a/src/Storages/System/StorageSystemBuildOptions.generated.cpp.in +++ b/src/Storages/System/StorageSystemBuildOptions.generated.cpp.in @@ -57,6 +57,7 @@ const char * auto_config_build[] "USE_GRPC", "@USE_GRPC@", "USE_LDAP", "@USE_LDAP@", "TZDATA_VERSION", "@TZDATA_VERSION@", + "USE_KRB5", "@USE_KRB5@", nullptr, nullptr }; diff --git a/src/Storages/System/StorageSystemUsers.cpp b/src/Storages/System/StorageSystemUsers.cpp index 675fee84746..bec94bc388c 100644 --- a/src/Storages/System/StorageSystemUsers.cpp +++ b/src/Storages/System/StorageSystemUsers.cpp @@ -90,11 +90,17 @@ void StorageSystemUsers::fillData(MutableColumns & res_columns, const Context & column_storage.insertData(storage_name.data(), storage_name.length()); column_auth_type.push_back(static_cast(authentication.getType())); - if (authentication.getType() == Authentication::Type::LDAP_SERVER) + if ( + authentication.getType() == Authentication::Type::LDAP || + authentication.getType() == Authentication::Type::KERBEROS + ) { Poco::JSON::Object auth_params_json; - auth_params_json.set("server", authentication.getServerName()); + if (authentication.getType() == Authentication::Type::LDAP) + auth_params_json.set("server", authentication.getLDAPServerName()); + else if (authentication.getType() == Authentication::Type::KERBEROS) + auth_params_json.set("realm", authentication.getKerberosRealm()); std::ostringstream oss; // STYLE_CHECK_ALLOW_STD_STRING_STREAM oss.exceptions(std::ios::failbit); diff --git a/tests/testflows/helpers/cluster.py b/tests/testflows/helpers/cluster.py index 5e8717e7a8e..cc676134024 100755 --- a/tests/testflows/helpers/cluster.py +++ b/tests/testflows/helpers/cluster.py @@ -26,7 +26,7 @@ class Node(object): def repr(self): return f"Node(name='{self.name}')" - def restart(self, timeout=300, retries=5): + def restart(self, timeout=300, retries=5, safe=True): """Restart node. """ with self.cluster.lock: @@ -48,7 +48,7 @@ class Node(object): if r.exitcode == 0: break - def stop(self, timeout=300, retries=5): + def stop(self, timeout=300, retries=5, safe=True): """Stop node. """ with self.cluster.lock: @@ -65,6 +65,44 @@ class Node(object): def command(self, *args, **kwargs): return self.cluster.command(self.name, *args, **kwargs) + def cmd(self, cmd, message=None, exitcode=None, steps=True, shell_command="bash --noediting", no_checks=False, + raise_on_exception=False, step=By, *args, **kwargs): + """Execute and check command. + :param cmd: command + :param message: expected message that should be in the output, default: None + :param exitcode: expected exitcode, default: None + """ + + command = f"{cmd}" + with Step("executing command", description=command, format_description=False) if steps else NullStep(): + try: + r = self.cluster.bash(self.name, command=shell_command)(command, *args, **kwargs) + except ExpectTimeoutError: + self.cluster.close_bash(self.name) + raise + + if no_checks: + return r + + if exitcode is not None: + with Then(f"exitcode should be {exitcode}") if steps else NullStep(): + assert r.exitcode == exitcode, error(r.output) + + if message is not None: + with Then(f"output should contain message", description=message) if steps else NullStep(): + assert message in r.output, error(r.output) + + if message is None or "Exception:" not in message: + with Then("check if output has exception") if steps else NullStep(): + if "Exception:" in r.output: + if raise_on_exception: + raise QueryRuntimeException(r.output) + assert False, error(r.output) + + return r + + + class ClickHouseNode(Node): """Node with ClickHouse server. """ @@ -163,7 +201,7 @@ class ClickHouseNode(Node): echo -e \"{sql[:100]}...\" > {query.name} {command} """ - with step("executing command", description=description, format_description=False) if steps else NullStep(): + with Step("executing command", description=description, format_description=False) if steps else NullStep(): try: r = self.cluster.bash(None)(command, *args, **kwargs) except ExpectTimeoutError: @@ -173,7 +211,7 @@ class ClickHouseNode(Node): for setting in settings: name, value = setting command += f" --{name} \"{value}\"" - with step("executing command", description=command, format_description=False) if steps else NullStep(): + with Step("executing command", description=command, format_description=False) if steps else NullStep(): try: r = self.cluster.bash(self.name)(command, *args, **kwargs) except ExpectTimeoutError: @@ -227,7 +265,7 @@ class Cluster(object): self.configs_dir = caller_configs_dir if not os.path.exists(self.configs_dir): - raise TypeError("configs directory '{self.configs_dir}' does not exist") + raise TypeError(f"configs directory '{self.configs_dir}' does not exist") # auto set docker-compose project directory if docker_compose_project_dir is None: @@ -256,7 +294,7 @@ class Cluster(object): shell.timeout = timeout return shell - def bash(self, node, timeout=300): + def bash(self, node, timeout=300, command="bash --noediting"): """Returns thread-local bash terminal to a specific node. :param node: name of the service @@ -278,7 +316,7 @@ class Cluster(object): self._bash[id] = Shell().__enter__() else: self._bash[id] = Shell(command=[ - "/bin/bash", "--noediting", "-c", f"{self.docker_compose} exec {node} bash --noediting" + "/bin/bash", "--noediting", "-c", f"{self.docker_compose} exec {node} {command}" ], name=node).__enter__() self._bash[id].timeout = timeout diff --git a/tests/testflows/kerberos/configs/clickhouse/common.xml b/tests/testflows/kerberos/configs/clickhouse/common.xml new file mode 100644 index 00000000000..df952b28c82 --- /dev/null +++ b/tests/testflows/kerberos/configs/clickhouse/common.xml @@ -0,0 +1,6 @@ + + Europe/Moscow + 0.0.0.0 + /var/lib/clickhouse/ + /var/lib/clickhouse/tmp/ + diff --git a/tests/testflows/kerberos/configs/clickhouse/config.d/logs.xml b/tests/testflows/kerberos/configs/clickhouse/config.d/logs.xml new file mode 100644 index 00000000000..bdf1bbc11c1 --- /dev/null +++ b/tests/testflows/kerberos/configs/clickhouse/config.d/logs.xml @@ -0,0 +1,17 @@ + + 3 + + trace + /var/log/clickhouse-server/log.log + /var/log/clickhouse-server/log.err.log + 1000M + 10 + /var/log/clickhouse-server/stderr.log + /var/log/clickhouse-server/stdout.log + + + system + part_log
+ 500 +
+
diff --git a/tests/testflows/kerberos/configs/clickhouse/config.d/ports.xml b/tests/testflows/kerberos/configs/clickhouse/config.d/ports.xml new file mode 100644 index 00000000000..fbc6cea74c0 --- /dev/null +++ b/tests/testflows/kerberos/configs/clickhouse/config.d/ports.xml @@ -0,0 +1,5 @@ + + + 8443 + 9440 + \ No newline at end of file diff --git a/tests/testflows/kerberos/configs/clickhouse/config.d/remote.xml b/tests/testflows/kerberos/configs/clickhouse/config.d/remote.xml new file mode 100644 index 00000000000..51be2a6e8e3 --- /dev/null +++ b/tests/testflows/kerberos/configs/clickhouse/config.d/remote.xml @@ -0,0 +1,107 @@ + + + + + + true + + clickhouse1 + 9000 + + + clickhouse2 + 9000 + + + clickhouse3 + 9000 + + + + + + + true + + clickhouse1 + 9440 + 1 + + + clickhouse2 + 9440 + 1 + + + clickhouse3 + 9440 + 1 + + + + + + + clickhouse1 + 9000 + + + + + clickhouse2 + 9000 + + + + + clickhouse3 + 9000 + + + + + + + clickhouse1 + 9440 + 1 + + + + + clickhouse2 + 9440 + 1 + + + + + clickhouse3 + 9440 + 1 + + + + + diff --git a/tests/testflows/kerberos/configs/clickhouse/config.d/ssl.xml b/tests/testflows/kerberos/configs/clickhouse/config.d/ssl.xml new file mode 100644 index 00000000000..ca65ffd5e04 --- /dev/null +++ b/tests/testflows/kerberos/configs/clickhouse/config.d/ssl.xml @@ -0,0 +1,17 @@ + + + + /etc/clickhouse-server/ssl/server.crt + /etc/clickhouse-server/ssl/server.key + none + true + + + true + none + + AcceptCertificateHandler + + + + diff --git a/tests/testflows/kerberos/configs/clickhouse/config.d/storage.xml b/tests/testflows/kerberos/configs/clickhouse/config.d/storage.xml new file mode 100644 index 00000000000..618fd6b6d24 --- /dev/null +++ b/tests/testflows/kerberos/configs/clickhouse/config.d/storage.xml @@ -0,0 +1,20 @@ + + + + + + 1024 + + + + + + + default + + + + + + + diff --git a/tests/testflows/kerberos/configs/clickhouse/config.d/zookeeper.xml b/tests/testflows/kerberos/configs/clickhouse/config.d/zookeeper.xml new file mode 100644 index 00000000000..96270e7b645 --- /dev/null +++ b/tests/testflows/kerberos/configs/clickhouse/config.d/zookeeper.xml @@ -0,0 +1,10 @@ + + + + + zookeeper + 2181 + + 15000 + + diff --git a/tests/testflows/kerberos/configs/clickhouse/config.xml b/tests/testflows/kerberos/configs/clickhouse/config.xml new file mode 100644 index 00000000000..a0133238f25 --- /dev/null +++ b/tests/testflows/kerberos/configs/clickhouse/config.xml @@ -0,0 +1,440 @@ + + + + + + trace + /var/log/clickhouse-server/clickhouse-server.log + /var/log/clickhouse-server/clickhouse-server.err.log + 1000M + 10 + + + + 8123 + 9000 + + + + + + + + + /etc/clickhouse-server/server.crt + /etc/clickhouse-server/server.key + + /etc/clickhouse-server/dhparam.pem + none + true + true + sslv2,sslv3 + true + + + + true + true + sslv2,sslv3 + true + + + + RejectCertificateHandler + + + + + + + + + 9009 + + + + + + + + 0.0.0.0 + + + + + + + + + + + + 4096 + 3 + + + 100 + + + + + + 8589934592 + + + 5368709120 + + + + /var/lib/clickhouse/ + + + /var/lib/clickhouse/tmp/ + + + /var/lib/clickhouse/user_files/ + + + + + + users.xml + + + + /var/lib/clickhouse/access/ + + + + + default + + + + + + default + + + + + + + + + false + + + + + + + + localhost + 9000 + + + + + + + localhost + 9000 + + + + + localhost + 9000 + + + + + + + localhost + 9440 + 1 + + + + + + + localhost + 9000 + + + + + localhost + 1 + + + + + + + + + + + + + + + + + 3600 + + + + 3600 + + + 60 + + + + + + + + + + system + query_log
+ + toYYYYMM(event_date) + + 7500 +
+ + + + system + trace_log
+ + toYYYYMM(event_date) + 7500 +
+ + + + system + query_thread_log
+ toYYYYMM(event_date) + 7500 +
+ + + + + + + + + + + + + + + + *_dictionary.xml + + + + + + + + + + /clickhouse/task_queue/ddl + + + + + + + + + + + + + + + + click_cost + any + + 0 + 3600 + + + 86400 + 60 + + + + max + + 0 + 60 + + + 3600 + 300 + + + 86400 + 3600 + + + + + + /var/lib/clickhouse/format_schemas/ + + + +
diff --git a/tests/testflows/kerberos/configs/clickhouse/ssl/dhparam.pem b/tests/testflows/kerberos/configs/clickhouse/ssl/dhparam.pem new file mode 100644 index 00000000000..2e6cee0798d --- /dev/null +++ b/tests/testflows/kerberos/configs/clickhouse/ssl/dhparam.pem @@ -0,0 +1,8 @@ +-----BEGIN DH PARAMETERS----- +MIIBCAKCAQEAua92DDli13gJ+//ZXyGaggjIuidqB0crXfhUlsrBk9BV1hH3i7fR +XGP9rUdk2ubnB3k2ejBStL5oBrkHm9SzUFSQHqfDjLZjKoUpOEmuDc4cHvX1XTR5 +Pr1vf5cd0yEncJWG5W4zyUB8k++SUdL2qaeslSs+f491HBLDYn/h8zCgRbBvxhxb +9qeho1xcbnWeqkN6Kc9bgGozA16P9NLuuLttNnOblkH+lMBf42BSne/TWt3AlGZf +slKmmZcySUhF8aKfJnLKbkBCFqOtFRh8zBA9a7g+BT/lSANATCDPaAk1YVih2EKb +dpc3briTDbRsiqg2JKMI7+VdULY9bh3EawIBAg== +-----END DH PARAMETERS----- diff --git a/tests/testflows/kerberos/configs/clickhouse/ssl/server.crt b/tests/testflows/kerberos/configs/clickhouse/ssl/server.crt new file mode 100644 index 00000000000..7ade2d96273 --- /dev/null +++ b/tests/testflows/kerberos/configs/clickhouse/ssl/server.crt @@ -0,0 +1,19 @@ +-----BEGIN CERTIFICATE----- +MIIC/TCCAeWgAwIBAgIJANjx1QSR77HBMA0GCSqGSIb3DQEBCwUAMBQxEjAQBgNV +BAMMCWxvY2FsaG9zdDAgFw0xODA3MzAxODE2MDhaGA8yMjkyMDUxNDE4MTYwOFow +FDESMBAGA1UEAwwJbG9jYWxob3N0MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIB +CgKCAQEAs9uSo6lJG8o8pw0fbVGVu0tPOljSWcVSXH9uiJBwlZLQnhN4SFSFohfI +4K8U1tBDTnxPLUo/V1K9yzoLiRDGMkwVj6+4+hE2udS2ePTQv5oaMeJ9wrs+5c9T +4pOtlq3pLAdm04ZMB1nbrEysceVudHRkQbGHzHp6VG29Fw7Ga6YpqyHQihRmEkTU +7UCYNA+Vk7aDPdMS/khweyTpXYZimaK9f0ECU3/VOeG3fH6Sp2X6FN4tUj/aFXEj +sRmU5G2TlYiSIUMF2JPdhSihfk1hJVALrHPTU38SOL+GyyBRWdNcrIwVwbpvsvPg +pryMSNxnpr0AK0dFhjwnupIv5hJIOQIDAQABo1AwTjAdBgNVHQ4EFgQUjPLb3uYC +kcamyZHK4/EV8jAP0wQwHwYDVR0jBBgwFoAUjPLb3uYCkcamyZHK4/EV8jAP0wQw +DAYDVR0TBAUwAwEB/zANBgkqhkiG9w0BAQsFAAOCAQEAM/ocuDvfPus/KpMVD51j +4IdlU8R0vmnYLQ+ygzOAo7+hUWP5j0yvq4ILWNmQX6HNvUggCgFv9bjwDFhb/5Vr +85ieWfTd9+LTjrOzTw4avdGwpX9G+6jJJSSq15tw5ElOIFb/qNA9O4dBiu8vn03C +L/zRSXrARhSqTW5w/tZkUcSTT+M5h28+Lgn9ysx4Ff5vi44LJ1NnrbJbEAIYsAAD ++UA+4MBFKx1r6hHINULev8+lCfkpwIaeS8RL+op4fr6kQPxnULw8wT8gkuc8I4+L +P9gg/xDHB44T3ADGZ5Ib6O0DJaNiToO6rnoaaxs0KkotbvDWvRoxEytSbXKoYjYp +0g== +-----END CERTIFICATE----- diff --git a/tests/testflows/kerberos/configs/clickhouse/ssl/server.key b/tests/testflows/kerberos/configs/clickhouse/ssl/server.key new file mode 100644 index 00000000000..f0fb61ac443 --- /dev/null +++ b/tests/testflows/kerberos/configs/clickhouse/ssl/server.key @@ -0,0 +1,28 @@ +-----BEGIN PRIVATE KEY----- +MIIEvQIBADANBgkqhkiG9w0BAQEFAASCBKcwggSjAgEAAoIBAQCz25KjqUkbyjyn +DR9tUZW7S086WNJZxVJcf26IkHCVktCeE3hIVIWiF8jgrxTW0ENOfE8tSj9XUr3L +OguJEMYyTBWPr7j6ETa51LZ49NC/mhox4n3Cuz7lz1Pik62WreksB2bThkwHWdus +TKxx5W50dGRBsYfMenpUbb0XDsZrpimrIdCKFGYSRNTtQJg0D5WTtoM90xL+SHB7 +JOldhmKZor1/QQJTf9U54bd8fpKnZfoU3i1SP9oVcSOxGZTkbZOViJIhQwXYk92F +KKF+TWElUAusc9NTfxI4v4bLIFFZ01ysjBXBum+y8+CmvIxI3GemvQArR0WGPCe6 +ki/mEkg5AgMBAAECggEATrbIBIxwDJOD2/BoUqWkDCY3dGevF8697vFuZKIiQ7PP +TX9j4vPq0DfsmDjHvAPFkTHiTQXzlroFik3LAp+uvhCCVzImmHq0IrwvZ9xtB43f +7Pkc5P6h1l3Ybo8HJ6zRIY3TuLtLxuPSuiOMTQSGRL0zq3SQ5DKuGwkz+kVjHXUN +MR2TECFwMHKQ5VLrC+7PMpsJYyOMlDAWhRfUalxC55xOXTpaN8TxNnwQ8K2ISVY5 +212Jz/a4hn4LdwxSz3Tiu95PN072K87HLWx3EdT6vW4Ge5P/A3y+smIuNAlanMnu +plHBRtpATLiTxZt/n6npyrfQVbYjSH7KWhB8hBHtaQKBgQDh9Cq1c/KtqDtE0Ccr +/r9tZNTUwBE6VP+3OJeKdEdtsfuxjOCkS1oAjgBJiSDOiWPh1DdoDeVZjPKq6pIu +Mq12OE3Doa8znfCXGbkSzEKOb2unKZMJxzrz99kXt40W5DtrqKPNb24CNqTiY8Aa +CjtcX+3weat82VRXvph6U8ltMwKBgQDLxjiQQzNoY7qvg7CwJCjf9qq8jmLK766g +1FHXopqS+dTxDLM8eJSRrpmxGWJvNeNc1uPhsKsKgotqAMdBUQTf7rSTbt4MyoH5 +bUcRLtr+0QTK9hDWMOOvleqNXha68vATkohWYfCueNsC60qD44o8RZAS6UNy3ENq +cM1cxqe84wKBgQDKkHutWnooJtajlTxY27O/nZKT/HA1bDgniMuKaz4R4Gr1PIez +on3YW3V0d0P7BP6PWRIm7bY79vkiMtLEKdiKUGWeyZdo3eHvhDb/3DCawtau8L2K +GZsHVp2//mS1Lfz7Qh8/L/NedqCQ+L4iWiPnZ3THjjwn3CoZ05ucpvrAMwKBgB54 +nay039MUVq44Owub3KDg+dcIU62U+cAC/9oG7qZbxYPmKkc4oL7IJSNecGHA5SbU +2268RFdl/gLz6tfRjbEOuOHzCjFPdvAdbysanpTMHLNc6FefJ+zxtgk9sJh0C4Jh +vxFrw9nTKKzfEl12gQ1SOaEaUIO0fEBGbe8ZpauRAoGAMAlGV+2/K4ebvAJKOVTa +dKAzQ+TD2SJmeR1HZmKDYddNqwtZlzg3v4ZhCk4eaUmGeC1Bdh8MDuB3QQvXz4Dr +vOIP4UVaOr+uM+7TgAgVnP4/K6IeJGzUDhX93pmpWhODfdu/oojEKVcpCojmEmS1 +KCBtmIrQLqzMpnBpLNuSY+Q= +-----END PRIVATE KEY----- diff --git a/tests/testflows/kerberos/configs/clickhouse/users.xml b/tests/testflows/kerberos/configs/clickhouse/users.xml new file mode 100644 index 00000000000..86b2cd9e1e3 --- /dev/null +++ b/tests/testflows/kerberos/configs/clickhouse/users.xml @@ -0,0 +1,133 @@ + + + + + + + + 10000000000 + + + 0 + + + random + + + + + 1 + + + + + + + + + + + + + ::/0 + + + + default + + + default + + + 1 + + + + + + + + + + + + + + + + + 3600 + + + 0 + 0 + 0 + 0 + 0 + + + + diff --git a/tests/testflows/kerberos/configs/clickhouse1/config.d/kerberos.xml b/tests/testflows/kerberos/configs/clickhouse1/config.d/kerberos.xml new file mode 100644 index 00000000000..ceaa497c561 --- /dev/null +++ b/tests/testflows/kerberos/configs/clickhouse1/config.d/kerberos.xml @@ -0,0 +1,5 @@ + + + EXAMPLE.COM + + \ No newline at end of file diff --git a/tests/testflows/kerberos/configs/clickhouse1/config.d/macros.xml b/tests/testflows/kerberos/configs/clickhouse1/config.d/macros.xml new file mode 100644 index 00000000000..6cdcc1b440c --- /dev/null +++ b/tests/testflows/kerberos/configs/clickhouse1/config.d/macros.xml @@ -0,0 +1,8 @@ + + + + clickhouse1 + 01 + 01 + + diff --git a/tests/testflows/kerberos/configs/clickhouse1/users.d/kerberos-users.xml b/tests/testflows/kerberos/configs/clickhouse1/users.d/kerberos-users.xml new file mode 100644 index 00000000000..7029f20217f --- /dev/null +++ b/tests/testflows/kerberos/configs/clickhouse1/users.d/kerberos-users.xml @@ -0,0 +1,10 @@ + + + + + EXAMPLE.COM + + 1 + + + \ No newline at end of file diff --git a/tests/testflows/kerberos/configs/clickhouse2/config.d/kerberos.xml b/tests/testflows/kerberos/configs/clickhouse2/config.d/kerberos.xml new file mode 100644 index 00000000000..ceaa497c561 --- /dev/null +++ b/tests/testflows/kerberos/configs/clickhouse2/config.d/kerberos.xml @@ -0,0 +1,5 @@ + + + EXAMPLE.COM + + \ No newline at end of file diff --git a/tests/testflows/kerberos/configs/clickhouse2/config.d/macros.xml b/tests/testflows/kerberos/configs/clickhouse2/config.d/macros.xml new file mode 100644 index 00000000000..a114a9ce4ab --- /dev/null +++ b/tests/testflows/kerberos/configs/clickhouse2/config.d/macros.xml @@ -0,0 +1,8 @@ + + + + clickhouse2 + 01 + 02 + + diff --git a/tests/testflows/kerberos/configs/clickhouse3/config.d/macros.xml b/tests/testflows/kerberos/configs/clickhouse3/config.d/macros.xml new file mode 100644 index 00000000000..904a27b0172 --- /dev/null +++ b/tests/testflows/kerberos/configs/clickhouse3/config.d/macros.xml @@ -0,0 +1,8 @@ + + + + clickhouse3 + 01 + 03 + + diff --git a/tests/testflows/kerberos/configs/kerberos/etc/krb5.conf b/tests/testflows/kerberos/configs/kerberos/etc/krb5.conf new file mode 100644 index 00000000000..b963fc25daa --- /dev/null +++ b/tests/testflows/kerberos/configs/kerberos/etc/krb5.conf @@ -0,0 +1,38 @@ +[kdc] + require-preauth = false + +[libdefaults] + default_realm = EXAMPLE.COM + ticket_lifetime = 24000 + dns_lookup_realm = false + dns_lookup_kdc = false + dns_fallback = false + rdns = false + +[realms] + EXAMPLE.COM = { + kdc = kerberos + admin_server = kerberos + } + OTHER.COM = { + kdc = kerberos + admin_server = kerberos + } + +[domain_realm] + docker-compose_default = EXAMPLE.COM + .docker-compose_default = EXAMPLE.COM + +[appdefaults] + validate = false + pam = { + debug = false + ticket_lifetime = 36000 + renew_lifetime = 36000 + forwardable = true + krb4_convert = false + } + +[logging] + kdc = FILE:/var/log/krb5kdc.log + admin_server = FILE:/var/log/kadmin.log diff --git a/tests/testflows/kerberos/configs/kerberos/etc/krb5kdc/kdc.conf b/tests/testflows/kerberos/configs/kerberos/etc/krb5kdc/kdc.conf new file mode 100644 index 00000000000..9afe5351467 --- /dev/null +++ b/tests/testflows/kerberos/configs/kerberos/etc/krb5kdc/kdc.conf @@ -0,0 +1,15 @@ +[kdcdefaults] + kdc_ports = 88 + +[realms] + EXAMPLE.COM = { + kadmind_port = 749 + max_life = 12h 0m 0s + max_renewable_life = 7d 0h 0m 0s + master_key_type = des3-hmac-sha1 + supported_enctypes = des3-hmac-sha1:normal des-cbc-crc:normal des-cbc-crc:v4 + } + +[logging] + kdc = FILE:/usr/local/var/krb5kdc/kdc.log + admin_server = FILE:/usr/local/var/krb5kdc/kadmin.log diff --git a/tests/testflows/kerberos/configs/kerberos/etc/supervisord.conf b/tests/testflows/kerberos/configs/kerberos/etc/supervisord.conf new file mode 100644 index 00000000000..bec4ae41077 --- /dev/null +++ b/tests/testflows/kerberos/configs/kerberos/etc/supervisord.conf @@ -0,0 +1,31 @@ +[inet_http_server] +port=9001 + +[rpcinterface:supervisor] +supervisor.rpcinterface_factory = supervisor.rpcinterface:make_main_rpcinterface + +[supervisorctl] +serverurl = http://127.0.0.1:9001 + +[program:krb5kdc] +command = /usr/sbin/krb5kdc -n +startretries = 1 +startsecs = 5 +stdout_logfile=/dev/stdout +stdout_logfile_maxbytes=0 + +[program:kadmind] +command = /usr/sbin/kadmind -nofork +startretries = 1 +startsecs = 5 +stdout_logfile=/dev/stdout +stdout_logfile_maxbytes=0 + + +[supervisord] +logfile = /tmp/supervisord_zbx_server.log +loglevel = critical +nodaemon = true +user = root +pidfile = /tmp/supervisord_zbx_server.pid +directory = /tmp \ No newline at end of file diff --git a/tests/testflows/kerberos/docker-compose/clickhouse-service.yml b/tests/testflows/kerberos/docker-compose/clickhouse-service.yml new file mode 100644 index 00000000000..14736a264b8 --- /dev/null +++ b/tests/testflows/kerberos/docker-compose/clickhouse-service.yml @@ -0,0 +1,32 @@ +version: '2.3' + +services: + clickhouse: + image: yandex/clickhouse-integration-test:21454 + expose: + - "9000" + - "9009" + - "8123" + volumes: + - "${CLICKHOUSE_TESTS_DIR}/configs/clickhouse/ssl:/etc/clickhouse-server/ssl" + - "${CLICKHOUSE_TESTS_DIR}/configs/clickhouse/config.xml:/etc/clickhouse-server/config.xml" + - "${CLICKHOUSE_TESTS_DIR}/configs/clickhouse/users.xml:/etc/clickhouse-server/users.xml" + - "${CLICKHOUSE_TESTS_SERVER_BIN_PATH:-/usr/bin/clickhouse}:/usr/bin/clickhouse" + - "${CLICKHOUSE_TESTS_ODBC_BRIDGE_BIN_PATH:-/usr/bin/clickhouse-odbc-bridge}:/usr/bin/clickhouse-odbc-bridge" + - "${CLICKHOUSE_TESTS_DIR}/configs/kerberos/etc/krb5.conf:/etc/krb5.conf" + entrypoint: bash -c "clickhouse server --config-file=/etc/clickhouse-server/config.xml --log-file=/var/log/clickhouse-server/clickhouse-server.log --errorlog-file=/var/log/clickhouse-server/clickhouse-server.err.log" + healthcheck: + test: clickhouse client --query='select 1' + interval: 10s + timeout: 10s + retries: 3 + start_period: 300s + + environment: + KRB5_CLIENT_KTNAME: /etc/krb5.keytab + KRB5_KTNAME: /etc/krb5.keytab + + cap_add: + - SYS_PTRACE + security_opt: + - label:disable diff --git a/tests/testflows/kerberos/docker-compose/docker-compose.yml b/tests/testflows/kerberos/docker-compose/docker-compose.yml new file mode 100644 index 00000000000..d1a74662a83 --- /dev/null +++ b/tests/testflows/kerberos/docker-compose/docker-compose.yml @@ -0,0 +1,75 @@ +version: '2.3' + +services: + + zookeeper: + extends: + file: zookeeper-service.yml + service: zookeeper + + kerberos: + extends: + file: kerberos-service.yml + service: kerberos + hostname: kerberos + depends_on: + zookeeper: + condition: service_healthy + + clickhouse1: + extends: + file: clickhouse-service.yml + service: clickhouse + hostname: clickhouse1 + volumes: + - "${CLICKHOUSE_TESTS_DIR}/_instances/clickhouse1/database/:/var/lib/clickhouse/" + - "${CLICKHOUSE_TESTS_DIR}/_instances/clickhouse1/logs/:/var/log/clickhouse-server/" + - "${CLICKHOUSE_TESTS_DIR}/configs/clickhouse1/config.d:/etc/clickhouse-server/config.d" + - "${CLICKHOUSE_TESTS_DIR}/configs/clickhouse1/users.d:/etc/clickhouse-server/users.d" + depends_on: + zookeeper: + condition: service_healthy + + clickhouse2: + extends: + file: clickhouse-service.yml + service: clickhouse + hostname: clickhouse2 + volumes: + - "${CLICKHOUSE_TESTS_DIR}/_instances/clickhouse2/database/:/var/lib/clickhouse/" + - "${CLICKHOUSE_TESTS_DIR}/_instances/clickhouse2/logs/:/var/log/clickhouse-server/" + - "${CLICKHOUSE_TESTS_DIR}/configs/clickhouse2/config.d:/etc/clickhouse-server/config.d" + - "${CLICKHOUSE_TESTS_DIR}/configs/clickhouse2/users.d:/etc/clickhouse-server/users.d" + depends_on: + zookeeper: + condition: service_healthy + + clickhouse3: + extends: + file: clickhouse-service.yml + service: clickhouse + hostname: clickhouse3 + volumes: + - "${CLICKHOUSE_TESTS_DIR}/_instances/clickhouse3/database/:/var/lib/clickhouse/" + - "${CLICKHOUSE_TESTS_DIR}/_instances/clickhouse3/logs/:/var/log/clickhouse-server/" + - "${CLICKHOUSE_TESTS_DIR}/configs/clickhouse3/config.d:/etc/clickhouse-server/config.d" + - "${CLICKHOUSE_TESTS_DIR}/configs/clickhouse3/users.d:/etc/clickhouse-server/users.d" + depends_on: + zookeeper: + condition: service_healthy + + # dummy service which does nothing, but allows to postpone + # 'docker-compose up -d' till all dependecies will go healthy + all_services_ready: + image: hello-world + depends_on: + clickhouse1: + condition: service_healthy + clickhouse2: + condition: service_healthy + clickhouse3: + condition: service_healthy + zookeeper: + condition: service_healthy + kerberos: + condition: service_healthy diff --git a/tests/testflows/kerberos/docker-compose/kerberos-service.yml b/tests/testflows/kerberos/docker-compose/kerberos-service.yml new file mode 100644 index 00000000000..3f21e93e0b6 --- /dev/null +++ b/tests/testflows/kerberos/docker-compose/kerberos-service.yml @@ -0,0 +1,27 @@ +version: '2.3' + +services: + kerberos: + image: zvonand/docker-krb5-server:1.0.0 + restart: always + expose: + - "88" + - "464" + - "749" + healthcheck: + test: echo 1 + interval: 10s + timeout: 10s + retries: 3 + start_period: 300s + environment: + KRB5_PASS: pwd + KRB5_REALM: EXAMPLE.COM + KRB5_KDC: localhost + volumes: + - "${CLICKHOUSE_TESTS_DIR}/configs/kerberos/etc/krb5kdc/kdc.conf:/etc/krb5kdc/kdc.conf" + - "${CLICKHOUSE_TESTS_DIR}/_instances/kerberos/krb5kdc/log/kdc.log:/usr/local/var/krb5kdc/kdc.log" + - "${CLICKHOUSE_TESTS_DIR}/_instances/kerberos/krb5kdc/log/kadmin.log:/usr/local/var/krb5kdc/kadmin.log" + - "${CLICKHOUSE_TESTS_DIR}/_instances/kerberos/var/log:/var/log" + security_opt: + - label:disable diff --git a/tests/testflows/kerberos/docker-compose/zookeeper-service.yml b/tests/testflows/kerberos/docker-compose/zookeeper-service.yml new file mode 100644 index 00000000000..6691a2df31c --- /dev/null +++ b/tests/testflows/kerberos/docker-compose/zookeeper-service.yml @@ -0,0 +1,18 @@ +version: '2.3' + +services: + zookeeper: + image: zookeeper:3.4.12 + expose: + - "2181" + environment: + ZOO_TICK_TIME: 500 + ZOO_MY_ID: 1 + healthcheck: + test: echo stat | nc localhost 2181 + interval: 10s + timeout: 10s + retries: 3 + start_period: 300s + security_opt: + - label:disable diff --git a/tests/testflows/kerberos/regression.py b/tests/testflows/kerberos/regression.py new file mode 100644 index 00000000000..9675ba6e3c8 --- /dev/null +++ b/tests/testflows/kerberos/regression.py @@ -0,0 +1,38 @@ +import sys +from testflows.core import * + +append_path(sys.path, "..") + +from helpers.cluster import Cluster +from helpers.argparser import argparser +from kerberos.requirements.requirements import * + +xfails = { +} + + +@TestModule +@Name("kerberos") +@ArgumentParser(argparser) +@Requirements( + RQ_SRS_016_Kerberos("1.0") +) +@XFails(xfails) +def regression(self, local, clickhouse_binary_path, stress=None, parallel=None): + """ClickHouse Kerberos authentication test regression module. + """ + nodes = { + "clickhouse": ("clickhouse1", "clickhouse2", "clickhouse3"), + "kerberos": ("kerberos", ), + } + + with Cluster(local, clickhouse_binary_path, nodes=nodes) as cluster: + self.context.cluster = cluster + + Feature(run=load("kerberos.tests.generic", "generic"), flags=TE) + Feature(run=load("kerberos.tests.config", "config"), flags=TE) + Feature(run=load("kerberos.tests.parallel", "parallel"), flags=TE) + + +if main(): + regression() diff --git a/tests/testflows/kerberos/requirements/requirements.md b/tests/testflows/kerberos/requirements/requirements.md new file mode 100644 index 00000000000..2121dd343b8 --- /dev/null +++ b/tests/testflows/kerberos/requirements/requirements.md @@ -0,0 +1,281 @@ +# QA-SRS016 ClickHouse Kerberos Authentication +# Software Requirements Specification + +## Table of Contents + +* 1 [Revision History](#revision-history) +* 2 [Introduction](#introduction) +* 3 [Terminology](#terminology) +* 4 [Requirements](#requirements) + * 4.1 [Generic](#generic) + * 4.1.1 [RQ.SRS-016.Kerberos](#rqsrs-016kerberos) + * 4.2 [Configuration](#configuration) + * 4.2.1 [RQ.SRS-016.Kerberos.Configuration.MultipleAuthMethods](#rqsrs-016kerberosconfigurationmultipleauthmethods) + * 4.2.2 [RQ.SRS-016.Kerberos.Configuration.KerberosNotEnabled](#rqsrs-016kerberosconfigurationkerberosnotenabled) + * 4.2.3 [RQ.SRS-016.Kerberos.Configuration.MultipleKerberosSections](#rqsrs-016kerberosconfigurationmultiplekerberossections) + * 4.2.4 [RQ.SRS-016.Kerberos.Configuration.WrongUserRealm](#rqsrs-016kerberosconfigurationwronguserrealm) + * 4.2.5 [RQ.SRS-016.Kerberos.Configuration.PrincipalAndRealmSpecified](#rqsrs-016kerberosconfigurationprincipalandrealmspecified) + * 4.2.6 [RQ.SRS-016.Kerberos.Configuration.MultiplePrincipalSections](#rqsrs-016kerberosconfigurationmultipleprincipalsections) + * 4.2.7 [RQ.SRS-016.Kerberos.Configuration.MultipleRealmSections](#rqsrs-016kerberosconfigurationmultiplerealmsections) + * 4.3 [Valid User](#valid-user) + * 4.3.1 [RQ.SRS-016.Kerberos.ValidUser.XMLConfiguredUser](#rqsrs-016kerberosvaliduserxmlconfigureduser) + * 4.3.2 [RQ.SRS-016.Kerberos.ValidUser.RBACConfiguredUser](#rqsrs-016kerberosvaliduserrbacconfigureduser) + * 4.3.3 [RQ.SRS-016.Kerberos.ValidUser.KerberosNotConfigured](#rqsrs-016kerberosvaliduserkerberosnotconfigured) + * 4.4 [Invalid User](#invalid-user) + * 4.4.1 [RQ.SRS-016.Kerberos.InvalidUser](#rqsrs-016kerberosinvaliduser) + * 4.4.2 [RQ.SRS-016.Kerberos.InvalidUser.UserDeleted](#rqsrs-016kerberosinvaliduseruserdeleted) + * 4.5 [Kerberos Not Available](#kerberos-not-available) + * 4.5.1 [RQ.SRS-016.Kerberos.KerberosNotAvailable.InvalidServerTicket](#rqsrs-016kerberoskerberosnotavailableinvalidserverticket) + * 4.5.2 [RQ.SRS-016.Kerberos.KerberosNotAvailable.InvalidClientTicket](#rqsrs-016kerberoskerberosnotavailableinvalidclientticket) + * 4.5.3 [RQ.SRS-016.Kerberos.KerberosNotAvailable.ValidTickets](#rqsrs-016kerberoskerberosnotavailablevalidtickets) + * 4.6 [Kerberos Restarted](#kerberos-restarted) + * 4.6.1 [RQ.SRS-016.Kerberos.KerberosServerRestarted](#rqsrs-016kerberoskerberosserverrestarted) + * 4.7 [Performance](#performance) + * 4.7.1 [RQ.SRS-016.Kerberos.Performance](#rqsrs-016kerberosperformance) + * 4.8 [Parallel Requests processing](#parallel-requests-processing) + * 4.8.1 [RQ.SRS-016.Kerberos.Parallel](#rqsrs-016kerberosparallel) + * 4.8.2 [RQ.SRS-016.Kerberos.Parallel.ValidRequests.KerberosAndNonKerberos](#rqsrs-016kerberosparallelvalidrequestskerberosandnonkerberos) + * 4.8.3 [RQ.SRS-016.Kerberos.Parallel.ValidRequests.SameCredentials](#rqsrs-016kerberosparallelvalidrequestssamecredentials) + * 4.8.4 [RQ.SRS-016.Kerberos.Parallel.ValidRequests.DifferentCredentials](#rqsrs-016kerberosparallelvalidrequestsdifferentcredentials) + * 4.8.5 [RQ.SRS-016.Kerberos.Parallel.ValidInvalid](#rqsrs-016kerberosparallelvalidinvalid) + * 4.8.6 [RQ.SRS-016.Kerberos.Parallel.Deletion](#rqsrs-016kerberosparalleldeletion) +* 5 [References](#references) + +## Revision History + +This document is stored in an electronic form using [Git] source control management software +hosted in a [GitHub Repository]. +All the updates are tracked using the [Git]'s [Revision History]. + +## Introduction + +This document specifies the behavior for authenticating existing users via [Kerberos] authentication protocol. +Existing [ClickHouse] users, that are properly configured, have an ability to authenticate using [Kerberos]. Kerberos authentication is only supported for HTTP requests, and users configured to authenticate via Kerberos cannot be authenticated by any other means of authentication. + +In order to use Kerberos authentication, Kerberos needs to be properly configured in the environment: Kerberos server must be present and user's and server's credentials must be set up. Configuring the Kerberos environment is outside the scope of this document. + +## Terminology + +* **Principal** - + A unique identity that uses [Kerberos]. + +* **Realm** - + A logical group of resources and identities that use [Kerberos]. + +* **Ticket** - + An encrypted block of data that authenticates principal. + +* **Credentials** - + A Kerberos ticket and a session key. + +* **Kerberized request** - + A HTTP query to ClickHouse server, which uses GSS [SPNEGO] and [Kerberos] to authenticate client. + +* **Unkerberized request** - + A HTTP query to ClickHouse server, which uses any other mean of authentication than GSS [SPNEGO] or [Kerberos]. + +For a more detailed descriprion, visit [Kerberos terminology]. + +## Requirements + +### Generic + +#### RQ.SRS-016.Kerberos +version: 1.0 + +[ClickHouse] SHALL support user authentication using [Kerberos] server. + +### Configuration + +#### RQ.SRS-016.Kerberos.Configuration.MultipleAuthMethods +version: 1.0 + +[ClickHouse] SHALL generate an exception and TERMINATE in case some user in `users.xml` has a `` section specified alongside with any other authentication method's section, e.g. `ldap`, `password`. + +#### RQ.SRS-016.Kerberos.Configuration.KerberosNotEnabled +version: 1.0 + +[ClickHouse] SHALL reject [Kerberos] authentication in case user is properly configured for using Kerberos, but Kerberos itself is not enabled in `config.xml`. For example: + +```xml + + + + +``` +```xml + + + + HTTP/clickhouse.example.com@EXAMPLE.COM + + +``` +```xml + + + + EXAMPLE.COM + + +``` + +#### RQ.SRS-016.Kerberos.Configuration.MultipleKerberosSections +version: 1.0 + +[ClickHouse] SHALL disable [Kerberos] and reject [Kerberos] authentication in case multiple `kerberos` sections are present in `config.xml`. + +#### RQ.SRS-016.Kerberos.Configuration.WrongUserRealm +version: 1.0 + +[ClickHouse] SHALL reject [Kerberos] authentication if user's realm specified in `users.xml` doesn't match the realm of the principal trying to authenticate. + +#### RQ.SRS-016.Kerberos.Configuration.PrincipalAndRealmSpecified +version: 1.0 + +[ClickHouse] SHALL generate an exception and disable [Kerberos] in case both `realm` and `principal` sections are defined in `config.xml`. + +#### RQ.SRS-016.Kerberos.Configuration.MultiplePrincipalSections +version: 1.0 + +[ClickHouse] SHALL generate an exception and disable [Kerberos] in case multiple `principal` sections are specified inside `kerberos` section in `config.xml`. + +#### RQ.SRS-016.Kerberos.Configuration.MultipleRealmSections +version: 1.0 + +[ClickHouse] SHALL generate an exception and disable [Kerberos] in case multiple `realm` sections are specified inside `kerberos` section in `config.xml`. + +### Valid User + +#### RQ.SRS-016.Kerberos.ValidUser.XMLConfiguredUser +version: 1.0 + +[ClickHouse] SHALL accept [Kerberos] authentication for a user that is configured in `users.xml` and has [Kerberos] enabled, i.e.: + +```xml + + + + + + + + EXAMPLE.COM + + + + +``` + +#### RQ.SRS-016.Kerberos.ValidUser.RBACConfiguredUser +version: 1.0 + +[ClickHouse] SHALL accept [Kerberos] authentication if user is configured to authenticate via [Kerberos] using SQL queries + +```sql +CREATE USER my_user IDENTIFIED WITH kerberos REALM 'EXAMPLE.COM' +``` + +or + +```sql +CREATE USER my_user IDENTIFIED WITH kerberos +``` + +#### RQ.SRS-016.Kerberos.ValidUser.KerberosNotConfigured +version: 1.0 + +[ClickHouse] SHALL reject [Kerberos] authentication if username is valid but [ClickHouse] user is not configured to be authenticated using [Kerberos]. + +### Invalid User + +#### RQ.SRS-016.Kerberos.InvalidUser +version: 1.0 + +[ClickHouse] SHALL reject [Kerberos] authentication if name of the principal attempting to authenticate does not translate to a valid [ClickHouse] username configured in `users.xml` or via SQL workflow. + +#### RQ.SRS-016.Kerberos.InvalidUser.UserDeleted +version: 1.0 + +[ClickHouse] SHALL reject [Kerberos] authentication if [ClickHouse] user was removed from the database using an SQL query. + +### Kerberos Not Available + +#### RQ.SRS-016.Kerberos.KerberosNotAvailable.InvalidServerTicket +version: 1.0 + +[ClickHouse] SHALL reject [Kerberos] authentication if [ClickHouse] user is configured to be authenticated using [Kerberos] and [Kerberos] server is unavailable, but [ClickHouse] doesn't have a valid Kerberos ticket or the ticket is expired. + +#### RQ.SRS-016.Kerberos.KerberosNotAvailable.InvalidClientTicket +version: 1.0 + +[ClickHouse] SHALL reject [Kerberos] authentication if [ClickHouse] user is configured to to be authenticated using [Kerberos] and [Kerberos] server is unavailable, but the client doesn't have a valid Kerberos ticket or the ticket is expired. + +#### RQ.SRS-016.Kerberos.KerberosNotAvailable.ValidTickets +version: 1.0 + +[ClickHouse] SHALL accept [Kerberos] authentication if no [Kerberos] server is reachable, but [ClickHouse] is configured to use valid credentials and [ClickHouse] has already processed some valid kerberized request (so it was granted a ticket), and the client has a valid ticket as well. + +### Kerberos Restarted + +#### RQ.SRS-016.Kerberos.KerberosServerRestarted +version: 1.0 + +[ClickHouse] SHALL accept [Kerberos] authentication if [Kerberos] server was restarted. + +### Performance + +#### RQ.SRS-016.Kerberos.Performance +version: 1.0 + +[ClickHouse]'s performance for [Kerberos] authentication SHALL be comparable to regular authentication. + +### Parallel Requests processing + +#### RQ.SRS-016.Kerberos.Parallel +version: 1.0 + +[ClickHouse] SHALL support parallel authentication using [Kerberos]. + +#### RQ.SRS-016.Kerberos.Parallel.ValidRequests.KerberosAndNonKerberos +version: 1.0 + +[ClickHouse] SHALL support processing of simultaneous kerberized (for users configured to authenticate via [Kerberos]) and non-kerberized (for users configured to authenticate with any other means) requests. + +#### RQ.SRS-016.Kerberos.Parallel.ValidRequests.SameCredentials +version: 1.0 + +[ClickHouse] SHALL support processing of simultaneously sent [Kerberos] requests under the same credentials. + +#### RQ.SRS-016.Kerberos.Parallel.ValidRequests.DifferentCredentials +version: 1.0 + +[ClickHouse] SHALL support processing of simultaneously sent [Kerberos] requests under different credentials. + +#### RQ.SRS-016.Kerberos.Parallel.ValidInvalid +version: 1.0 + +[ClickHouse] SHALL support parallel authentication of users using [Kerberos] server, some of which are valid and some invalid. Valid users' authentication should not be affected by invalid users' attempts. + +#### RQ.SRS-016.Kerberos.Parallel.Deletion +version: 1.0 + +[ClickHouse] SHALL not crash when two or more [Kerberos] users are simultaneously deleting one another. + +## References + +* **ClickHouse:** https://clickhouse.tech +* **GitHub Repository:** https://github.com/ClickHouse/ClickHouse/blob/master/tests/testflows/kerberos/requirements/requirements.md +* **Revision History:** https://github.com/ClickHouse/ClickHouse/commits/master/tests/testflows/kerberos/requirements/requirements.md +* **Git:** https://git-scm.com/ +* **Kerberos terminology:** https://web.mit.edu/kerberos/kfw-4.1/kfw-4.1/kfw-4.1-help/html/kerberos_terminology.htm + +[Kerberos]: https://en.wikipedia.org/wiki/Kerberos_(protocol) +[SPNEGO]: https://en.wikipedia.org/wiki/SPNEGO +[ClickHouse]: https://clickhouse.tech +[GitHub]: https://gitlab.com +[GitHub Repository]: https://github.com/ClickHouse/ClickHouse/blob/master/tests/testflows/kerberos/requirements/requirements.md +[Revision History]: https://github.com/ClickHouse/ClickHouse/commits/master/tests/testflows/kerberos/requirements/requirements.md +[Git]: https://git-scm.com/ +[Kerberos terminology]: https://web.mit.edu/kerberos/kfw-4.1/kfw-4.1/kfw-4.1-help/html/kerberos_terminology.htm + diff --git a/tests/testflows/kerberos/requirements/requirements.py b/tests/testflows/kerberos/requirements/requirements.py new file mode 100644 index 00000000000..5c49e7d127f --- /dev/null +++ b/tests/testflows/kerberos/requirements/requirements.py @@ -0,0 +1,800 @@ +# These requirements were auto generated +# from software requirements specification (SRS) +# document by TestFlows v1.6.201216.1172002. +# Do not edit by hand but re-generate instead +# using 'tfs requirements generate' command. +from testflows.core import Specification +from testflows.core import Requirement + +Heading = Specification.Heading + +RQ_SRS_016_Kerberos = Requirement( + name='RQ.SRS-016.Kerberos', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[ClickHouse] SHALL support user authentication using [Kerberos] server.\n' + '\n' + ), + link=None, + level=3, + num='4.1.1') + +RQ_SRS_016_Kerberos_Configuration_MultipleAuthMethods = Requirement( + name='RQ.SRS-016.Kerberos.Configuration.MultipleAuthMethods', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + "[ClickHouse] SHALL generate an exception and TERMINATE in case some user in `users.xml` has a `` section specified alongside with any other authentication method's section, e.g. `ldap`, `password`.\n" + '\n' + ), + link=None, + level=3, + num='4.2.1') + +RQ_SRS_016_Kerberos_Configuration_KerberosNotEnabled = Requirement( + name='RQ.SRS-016.Kerberos.Configuration.KerberosNotEnabled', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[ClickHouse] SHALL reject [Kerberos] authentication in case user is properly configured for using Kerberos, but Kerberos itself is not enabled in `config.xml`. For example:\n' + '\n' + '```xml\n' + '\n' + ' \n' + ' \n' + '\n' + '```\n' + '```xml\n' + '\n' + ' \n' + ' \n' + ' HTTP/clickhouse.example.com@EXAMPLE.COM\n' + ' \n' + '\n' + '```\n' + '```xml\n' + '\n' + ' \n' + ' \n' + ' EXAMPLE.COM\n' + ' \n' + '\n' + '```\n' + '\n' + ), + link=None, + level=3, + num='4.2.2') + +RQ_SRS_016_Kerberos_Configuration_MultipleKerberosSections = Requirement( + name='RQ.SRS-016.Kerberos.Configuration.MultipleKerberosSections', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[ClickHouse] SHALL disable [Kerberos] and reject [Kerberos] authentication in case multiple `kerberos` sections are present in `config.xml`.\n' + '\n' + ), + link=None, + level=3, + num='4.2.3') + +RQ_SRS_016_Kerberos_Configuration_WrongUserRealm = Requirement( + name='RQ.SRS-016.Kerberos.Configuration.WrongUserRealm', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + "[ClickHouse] SHALL reject [Kerberos] authentication if user's realm specified in `users.xml` doesn't match the realm of the principal trying to authenticate.\n" + '\n' + ), + link=None, + level=3, + num='4.2.4') + +RQ_SRS_016_Kerberos_Configuration_PrincipalAndRealmSpecified = Requirement( + name='RQ.SRS-016.Kerberos.Configuration.PrincipalAndRealmSpecified', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[ClickHouse] SHALL generate an exception and disable [Kerberos] in case both `realm` and `principal` sections are defined in `config.xml`.\n' + '\n' + ), + link=None, + level=3, + num='4.2.5') + +RQ_SRS_016_Kerberos_Configuration_MultiplePrincipalSections = Requirement( + name='RQ.SRS-016.Kerberos.Configuration.MultiplePrincipalSections', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[ClickHouse] SHALL generate an exception and disable [Kerberos] in case multiple `principal` sections are specified inside `kerberos` section in `config.xml`.\n' + '\n' + ), + link=None, + level=3, + num='4.2.6') + +RQ_SRS_016_Kerberos_Configuration_MultipleRealmSections = Requirement( + name='RQ.SRS-016.Kerberos.Configuration.MultipleRealmSections', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[ClickHouse] SHALL generate an exception and disable [Kerberos] in case multiple `realm` sections are specified inside `kerberos` section in `config.xml`.\n' + '\n' + ), + link=None, + level=3, + num='4.2.7') + +RQ_SRS_016_Kerberos_ValidUser_XMLConfiguredUser = Requirement( + name='RQ.SRS-016.Kerberos.ValidUser.XMLConfiguredUser', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[ClickHouse] SHALL accept [Kerberos] authentication for a user that is configured in `users.xml` and has [Kerberos] enabled, i.e.:\n' + '\n' + '```xml\n' + '\n' + ' \n' + ' \n' + ' \n' + ' \n' + ' \n' + ' \n' + ' EXAMPLE.COM\n' + ' \n' + ' \n' + ' \n' + '\n' + '```\n' + '\n' + ), + link=None, + level=3, + num='4.3.1') + +RQ_SRS_016_Kerberos_ValidUser_RBACConfiguredUser = Requirement( + name='RQ.SRS-016.Kerberos.ValidUser.RBACConfiguredUser', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[ClickHouse] SHALL accept [Kerberos] authentication if user is configured to authenticate via [Kerberos] using SQL queries\n' + '\n' + '```sql\n' + "CREATE USER my_user IDENTIFIED WITH kerberos REALM 'EXAMPLE.COM'\n" + '```\n' + '\n' + 'or\n' + '\n' + '```sql\n' + 'CREATE USER my_user IDENTIFIED WITH kerberos\n' + '```\n' + '\n' + ), + link=None, + level=3, + num='4.3.2') + +RQ_SRS_016_Kerberos_ValidUser_KerberosNotConfigured = Requirement( + name='RQ.SRS-016.Kerberos.ValidUser.KerberosNotConfigured', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[ClickHouse] SHALL reject [Kerberos] authentication if username is valid but [ClickHouse] user is not configured to be authenticated using [Kerberos].\n' + '\n' + ), + link=None, + level=3, + num='4.3.3') + +RQ_SRS_016_Kerberos_InvalidUser = Requirement( + name='RQ.SRS-016.Kerberos.InvalidUser', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[ClickHouse] SHALL reject [Kerberos] authentication if name of the principal attempting to authenticate does not translate to a valid [ClickHouse] username configured in `users.xml` or via SQL workflow.\n' + '\n' + ), + link=None, + level=3, + num='4.4.1') + +RQ_SRS_016_Kerberos_InvalidUser_UserDeleted = Requirement( + name='RQ.SRS-016.Kerberos.InvalidUser.UserDeleted', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[ClickHouse] SHALL reject [Kerberos] authentication if [ClickHouse] user was removed from the database using an SQL query.\n' + '\n' + ), + link=None, + level=3, + num='4.4.2') + +RQ_SRS_016_Kerberos_KerberosNotAvailable_InvalidServerTicket = Requirement( + name='RQ.SRS-016.Kerberos.KerberosNotAvailable.InvalidServerTicket', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + "[ClickHouse] SHALL reject [Kerberos] authentication if [ClickHouse] user is configured to be authenticated using [Kerberos] and [Kerberos] server is unavailable, but [ClickHouse] doesn't have a valid Kerberos ticket or the ticket is expired.\n" + '\n' + ), + link=None, + level=3, + num='4.5.1') + +RQ_SRS_016_Kerberos_KerberosNotAvailable_InvalidClientTicket = Requirement( + name='RQ.SRS-016.Kerberos.KerberosNotAvailable.InvalidClientTicket', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + "[ClickHouse] SHALL reject [Kerberos] authentication if [ClickHouse] user is configured to to be authenticated using [Kerberos] and [Kerberos] server is unavailable, but the client doesn't have a valid Kerberos ticket or the ticket is expired.\n" + '\n' + ), + link=None, + level=3, + num='4.5.2') + +RQ_SRS_016_Kerberos_KerberosNotAvailable_ValidTickets = Requirement( + name='RQ.SRS-016.Kerberos.KerberosNotAvailable.ValidTickets', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[ClickHouse] SHALL accept [Kerberos] authentication if no [Kerberos] server is reachable, but [ClickHouse] is configured to use valid credentials and [ClickHouse] has already processed some valid kerberized request (so it was granted a ticket), and the client has a valid ticket as well.\n' + '\n' + ), + link=None, + level=3, + num='4.5.3') + +RQ_SRS_016_Kerberos_KerberosServerRestarted = Requirement( + name='RQ.SRS-016.Kerberos.KerberosServerRestarted', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[ClickHouse] SHALL accept [Kerberos] authentication if [Kerberos] server was restarted.\n' + '\n' + ), + link=None, + level=3, + num='4.6.1') + +RQ_SRS_016_Kerberos_Performance = Requirement( + name='RQ.SRS-016.Kerberos.Performance', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + "[ClickHouse]'s performance for [Kerberos] authentication SHALL be comparable to regular authentication.\n" + '\n' + ), + link=None, + level=3, + num='4.7.1') + +RQ_SRS_016_Kerberos_Parallel = Requirement( + name='RQ.SRS-016.Kerberos.Parallel', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[ClickHouse] SHALL support parallel authentication using [Kerberos].\n' + '\n' + ), + link=None, + level=3, + num='4.8.1') + +RQ_SRS_016_Kerberos_Parallel_ValidRequests_KerberosAndNonKerberos = Requirement( + name='RQ.SRS-016.Kerberos.Parallel.ValidRequests.KerberosAndNonKerberos', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[ClickHouse] SHALL support processing of simultaneous kerberized (for users configured to authenticate via [Kerberos]) and non-kerberized (for users configured to authenticate with any other means) requests.\n' + '\n' + ), + link=None, + level=3, + num='4.8.2') + +RQ_SRS_016_Kerberos_Parallel_ValidRequests_SameCredentials = Requirement( + name='RQ.SRS-016.Kerberos.Parallel.ValidRequests.SameCredentials', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[ClickHouse] SHALL support processing of simultaneously sent [Kerberos] requests under the same credentials.\n' + '\n' + ), + link=None, + level=3, + num='4.8.3') + +RQ_SRS_016_Kerberos_Parallel_ValidRequests_DifferentCredentials = Requirement( + name='RQ.SRS-016.Kerberos.Parallel.ValidRequests.DifferentCredentials', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[ClickHouse] SHALL support processing of simultaneously sent [Kerberos] requests under different credentials.\n' + '\n' + ), + link=None, + level=3, + num='4.8.4') + +RQ_SRS_016_Kerberos_Parallel_ValidInvalid = Requirement( + name='RQ.SRS-016.Kerberos.Parallel.ValidInvalid', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + "[ClickHouse] SHALL support parallel authentication of users using [Kerberos] server, some of which are valid and some invalid. Valid users' authentication should not be affected by invalid users' attempts.\n" + '\n' + ), + link=None, + level=3, + num='4.8.5') + +RQ_SRS_016_Kerberos_Parallel_Deletion = Requirement( + name='RQ.SRS-016.Kerberos.Parallel.Deletion', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + '[ClickHouse] SHALL not crash when two or more [Kerberos] users are simultaneously deleting one another.\n' + '\n' + ), + link=None, + level=3, + num='4.8.6') + +QA_SRS016_ClickHouse_Kerberos_Authentication = Specification( + name='QA-SRS016 ClickHouse Kerberos Authentication', + description=None, + author='Andrey Zvonov', + date='December 14, 2020', + status='-', + approved_by='-', + approved_date='-', + approved_version='-', + version=None, + group=None, + type=None, + link=None, + uid=None, + parent=None, + children=None, + headings=( + Heading(name='Revision History', level=1, num='1'), + Heading(name='Introduction', level=1, num='2'), + Heading(name='Terminology', level=1, num='3'), + Heading(name='Requirements', level=1, num='4'), + Heading(name='Generic', level=2, num='4.1'), + Heading(name='RQ.SRS-016.Kerberos', level=3, num='4.1.1'), + Heading(name='Configuration', level=2, num='4.2'), + Heading(name='RQ.SRS-016.Kerberos.Configuration.MultipleAuthMethods', level=3, num='4.2.1'), + Heading(name='RQ.SRS-016.Kerberos.Configuration.KerberosNotEnabled', level=3, num='4.2.2'), + Heading(name='RQ.SRS-016.Kerberos.Configuration.MultipleKerberosSections', level=3, num='4.2.3'), + Heading(name='RQ.SRS-016.Kerberos.Configuration.WrongUserRealm', level=3, num='4.2.4'), + Heading(name='RQ.SRS-016.Kerberos.Configuration.PrincipalAndRealmSpecified', level=3, num='4.2.5'), + Heading(name='RQ.SRS-016.Kerberos.Configuration.MultiplePrincipalSections', level=3, num='4.2.6'), + Heading(name='RQ.SRS-016.Kerberos.Configuration.MultipleRealmSections', level=3, num='4.2.7'), + Heading(name='Valid User', level=2, num='4.3'), + Heading(name='RQ.SRS-016.Kerberos.ValidUser.XMLConfiguredUser', level=3, num='4.3.1'), + Heading(name='RQ.SRS-016.Kerberos.ValidUser.RBACConfiguredUser', level=3, num='4.3.2'), + Heading(name='RQ.SRS-016.Kerberos.ValidUser.KerberosNotConfigured', level=3, num='4.3.3'), + Heading(name='Invalid User', level=2, num='4.4'), + Heading(name='RQ.SRS-016.Kerberos.InvalidUser', level=3, num='4.4.1'), + Heading(name='RQ.SRS-016.Kerberos.InvalidUser.UserDeleted', level=3, num='4.4.2'), + Heading(name='Kerberos Not Available', level=2, num='4.5'), + Heading(name='RQ.SRS-016.Kerberos.KerberosNotAvailable.InvalidServerTicket', level=3, num='4.5.1'), + Heading(name='RQ.SRS-016.Kerberos.KerberosNotAvailable.InvalidClientTicket', level=3, num='4.5.2'), + Heading(name='RQ.SRS-016.Kerberos.KerberosNotAvailable.ValidTickets', level=3, num='4.5.3'), + Heading(name='Kerberos Restarted', level=2, num='4.6'), + Heading(name='RQ.SRS-016.Kerberos.KerberosServerRestarted', level=3, num='4.6.1'), + Heading(name='Performance', level=2, num='4.7'), + Heading(name='RQ.SRS-016.Kerberos.Performance', level=3, num='4.7.1'), + Heading(name='Parallel Requests processing', level=2, num='4.8'), + Heading(name='RQ.SRS-016.Kerberos.Parallel', level=3, num='4.8.1'), + Heading(name='RQ.SRS-016.Kerberos.Parallel.ValidRequests.KerberosAndNonKerberos', level=3, num='4.8.2'), + Heading(name='RQ.SRS-016.Kerberos.Parallel.ValidRequests.SameCredentials', level=3, num='4.8.3'), + Heading(name='RQ.SRS-016.Kerberos.Parallel.ValidRequests.DifferentCredentials', level=3, num='4.8.4'), + Heading(name='RQ.SRS-016.Kerberos.Parallel.ValidInvalid', level=3, num='4.8.5'), + Heading(name='RQ.SRS-016.Kerberos.Parallel.Deletion', level=3, num='4.8.6'), + Heading(name='References', level=1, num='5'), + ), + requirements=( + RQ_SRS_016_Kerberos, + RQ_SRS_016_Kerberos_Configuration_MultipleAuthMethods, + RQ_SRS_016_Kerberos_Configuration_KerberosNotEnabled, + RQ_SRS_016_Kerberos_Configuration_MultipleKerberosSections, + RQ_SRS_016_Kerberos_Configuration_WrongUserRealm, + RQ_SRS_016_Kerberos_Configuration_PrincipalAndRealmSpecified, + RQ_SRS_016_Kerberos_Configuration_MultiplePrincipalSections, + RQ_SRS_016_Kerberos_Configuration_MultipleRealmSections, + RQ_SRS_016_Kerberos_ValidUser_XMLConfiguredUser, + RQ_SRS_016_Kerberos_ValidUser_RBACConfiguredUser, + RQ_SRS_016_Kerberos_ValidUser_KerberosNotConfigured, + RQ_SRS_016_Kerberos_InvalidUser, + RQ_SRS_016_Kerberos_InvalidUser_UserDeleted, + RQ_SRS_016_Kerberos_KerberosNotAvailable_InvalidServerTicket, + RQ_SRS_016_Kerberos_KerberosNotAvailable_InvalidClientTicket, + RQ_SRS_016_Kerberos_KerberosNotAvailable_ValidTickets, + RQ_SRS_016_Kerberos_KerberosServerRestarted, + RQ_SRS_016_Kerberos_Performance, + RQ_SRS_016_Kerberos_Parallel, + RQ_SRS_016_Kerberos_Parallel_ValidRequests_KerberosAndNonKerberos, + RQ_SRS_016_Kerberos_Parallel_ValidRequests_SameCredentials, + RQ_SRS_016_Kerberos_Parallel_ValidRequests_DifferentCredentials, + RQ_SRS_016_Kerberos_Parallel_ValidInvalid, + RQ_SRS_016_Kerberos_Parallel_Deletion, + ), + content=''' +# QA-SRS016 ClickHouse Kerberos Authentication +# Software Requirements Specification + +(c) 2020 Altinity LTD. All Rights Reserved. + +**Document status:** Confidential + +**Author:** Andrey Zvonov + +**Date:** December 14, 2020 + +## Approval + +**Status:** - + +**Version:** - + +**Approved by:** - + +**Date:** - + + +## Table of Contents + +* 1 [Revision History](#revision-history) +* 2 [Introduction](#introduction) +* 3 [Terminology](#terminology) +* 4 [Requirements](#requirements) + * 4.1 [Generic](#generic) + * 4.1.1 [RQ.SRS-016.Kerberos](#rqsrs-016kerberos) + * 4.2 [Configuration](#configuration) + * 4.2.1 [RQ.SRS-016.Kerberos.Configuration.MultipleAuthMethods](#rqsrs-016kerberosconfigurationmultipleauthmethods) + * 4.2.2 [RQ.SRS-016.Kerberos.Configuration.KerberosNotEnabled](#rqsrs-016kerberosconfigurationkerberosnotenabled) + * 4.2.3 [RQ.SRS-016.Kerberos.Configuration.MultipleKerberosSections](#rqsrs-016kerberosconfigurationmultiplekerberossections) + * 4.2.4 [RQ.SRS-016.Kerberos.Configuration.WrongUserRealm](#rqsrs-016kerberosconfigurationwronguserrealm) + * 4.2.5 [RQ.SRS-016.Kerberos.Configuration.PrincipalAndRealmSpecified](#rqsrs-016kerberosconfigurationprincipalandrealmspecified) + * 4.2.6 [RQ.SRS-016.Kerberos.Configuration.MultiplePrincipalSections](#rqsrs-016kerberosconfigurationmultipleprincipalsections) + * 4.2.7 [RQ.SRS-016.Kerberos.Configuration.MultipleRealmSections](#rqsrs-016kerberosconfigurationmultiplerealmsections) + * 4.3 [Valid User](#valid-user) + * 4.3.1 [RQ.SRS-016.Kerberos.ValidUser.XMLConfiguredUser](#rqsrs-016kerberosvaliduserxmlconfigureduser) + * 4.3.2 [RQ.SRS-016.Kerberos.ValidUser.RBACConfiguredUser](#rqsrs-016kerberosvaliduserrbacconfigureduser) + * 4.3.3 [RQ.SRS-016.Kerberos.ValidUser.KerberosNotConfigured](#rqsrs-016kerberosvaliduserkerberosnotconfigured) + * 4.4 [Invalid User](#invalid-user) + * 4.4.1 [RQ.SRS-016.Kerberos.InvalidUser](#rqsrs-016kerberosinvaliduser) + * 4.4.2 [RQ.SRS-016.Kerberos.InvalidUser.UserDeleted](#rqsrs-016kerberosinvaliduseruserdeleted) + * 4.5 [Kerberos Not Available](#kerberos-not-available) + * 4.5.1 [RQ.SRS-016.Kerberos.KerberosNotAvailable.InvalidServerTicket](#rqsrs-016kerberoskerberosnotavailableinvalidserverticket) + * 4.5.2 [RQ.SRS-016.Kerberos.KerberosNotAvailable.InvalidClientTicket](#rqsrs-016kerberoskerberosnotavailableinvalidclientticket) + * 4.5.3 [RQ.SRS-016.Kerberos.KerberosNotAvailable.ValidTickets](#rqsrs-016kerberoskerberosnotavailablevalidtickets) + * 4.6 [Kerberos Restarted](#kerberos-restarted) + * 4.6.1 [RQ.SRS-016.Kerberos.KerberosServerRestarted](#rqsrs-016kerberoskerberosserverrestarted) + * 4.7 [Performance](#performance) + * 4.7.1 [RQ.SRS-016.Kerberos.Performance](#rqsrs-016kerberosperformance) + * 4.8 [Parallel Requests processing](#parallel-requests-processing) + * 4.8.1 [RQ.SRS-016.Kerberos.Parallel](#rqsrs-016kerberosparallel) + * 4.8.2 [RQ.SRS-016.Kerberos.Parallel.ValidRequests.KerberosAndNonKerberos](#rqsrs-016kerberosparallelvalidrequestskerberosandnonkerberos) + * 4.8.3 [RQ.SRS-016.Kerberos.Parallel.ValidRequests.SameCredentials](#rqsrs-016kerberosparallelvalidrequestssamecredentials) + * 4.8.4 [RQ.SRS-016.Kerberos.Parallel.ValidRequests.DifferentCredentials](#rqsrs-016kerberosparallelvalidrequestsdifferentcredentials) + * 4.8.5 [RQ.SRS-016.Kerberos.Parallel.ValidInvalid](#rqsrs-016kerberosparallelvalidinvalid) + * 4.8.6 [RQ.SRS-016.Kerberos.Parallel.Deletion](#rqsrs-016kerberosparalleldeletion) +* 5 [References](#references) + +## Revision History + +This document is stored in an electronic form using [Git] source control management software +hosted in a [GitLab Repository]. +All the updates are tracked using the [Git]'s [Revision History]. + +## Introduction + +This document specifies the behavior for authenticating existing users using [Kerberos] authentication protocol. +Existing [ClickHouse] users, that are properly configured, have an ability to authenticate using [Kerberos]. Kerberos authentication is only supported for HTTP requests, and users configured to authenticate via Kerberos cannot be authenticated by any other means of authentication. + +In order to use Kerberos authentication, Kerberos needs to be properly configured in the environment: Kerberos server must be present and user's and server's credentials must be set up. Configuring the Kerberos environment is outside the scope of this document. + +## Terminology + +* **Principal** - + A unique identity that uses [Kerberos]. + +* **Realm** - + A logical group of resources and identities that use [Kerberos]. + +* **Ticket** - + An encrypted block of data that authenticates principal. + +* **Credentials** - + A Kerberos ticket and a session key. + +* **Kerberized request** - + A HTTP query to ClickHouse server, which uses GSS [SPNEGO] and [Kerberos] to authenticate client. + +* **Unkerberized request** - + A HTTP query to ClickHouse server, which uses any other mean of authentication than GSS [SPNEGO] or [Kerberos]. + +For a more detailed descriprion, visit [Kerberos terminology]. + +## Requirements + +### Generic + +#### RQ.SRS-016.Kerberos +version: 1.0 + +[ClickHouse] SHALL support user authentication using [Kerberos] server. + +### Configuration + +#### RQ.SRS-016.Kerberos.Configuration.MultipleAuthMethods +version: 1.0 + +[ClickHouse] SHALL generate an exception and TERMINATE in case some user in `users.xml` has a `` section specified alongside with any other authentication method's section, e.g. `ldap`, `password`. + +#### RQ.SRS-016.Kerberos.Configuration.KerberosNotEnabled +version: 1.0 + +[ClickHouse] SHALL reject [Kerberos] authentication in case user is properly configured for using Kerberos, but Kerberos itself is not enabled in `config.xml`. For example: + +```xml + + + + +``` +```xml + + + + HTTP/clickhouse.example.com@EXAMPLE.COM + + +``` +```xml + + + + EXAMPLE.COM + + +``` + +#### RQ.SRS-016.Kerberos.Configuration.MultipleKerberosSections +version: 1.0 + +[ClickHouse] SHALL disable [Kerberos] and reject [Kerberos] authentication in case multiple `kerberos` sections are present in `config.xml`. + +#### RQ.SRS-016.Kerberos.Configuration.WrongUserRealm +version: 1.0 + +[ClickHouse] SHALL reject [Kerberos] authentication if user's realm specified in `users.xml` doesn't match the realm of the principal trying to authenticate. + +#### RQ.SRS-016.Kerberos.Configuration.PrincipalAndRealmSpecified +version: 1.0 + +[ClickHouse] SHALL generate an exception and disable [Kerberos] in case both `realm` and `principal` sections are defined in `config.xml`. + +#### RQ.SRS-016.Kerberos.Configuration.MultiplePrincipalSections +version: 1.0 + +[ClickHouse] SHALL generate an exception and disable [Kerberos] in case multiple `principal` sections are specified inside `kerberos` section in `config.xml`. + +#### RQ.SRS-016.Kerberos.Configuration.MultipleRealmSections +version: 1.0 + +[ClickHouse] SHALL generate an exception and disable [Kerberos] in case multiple `realm` sections are specified inside `kerberos` section in `config.xml`. + +### Valid User + +#### RQ.SRS-016.Kerberos.ValidUser.XMLConfiguredUser +version: 1.0 + +[ClickHouse] SHALL accept [Kerberos] authentication for a user that is configured in `users.xml` and has [Kerberos] enabled, i.e.: + +```xml + + + + + + + + EXAMPLE.COM + + + + +``` + +#### RQ.SRS-016.Kerberos.ValidUser.RBACConfiguredUser +version: 1.0 + +[ClickHouse] SHALL accept [Kerberos] authentication if user is configured to authenticate via [Kerberos] using SQL queries + +```sql +CREATE USER my_user IDENTIFIED WITH kerberos REALM 'EXAMPLE.COM' +``` + +or + +```sql +CREATE USER my_user IDENTIFIED WITH kerberos +``` + +#### RQ.SRS-016.Kerberos.ValidUser.KerberosNotConfigured +version: 1.0 + +[ClickHouse] SHALL reject [Kerberos] authentication if username is valid but [ClickHouse] user is not configured to be authenticated using [Kerberos]. + +### Invalid User + +#### RQ.SRS-016.Kerberos.InvalidUser +version: 1.0 + +[ClickHouse] SHALL reject [Kerberos] authentication if name of the principal attempting to authenticate does not translate to a valid [ClickHouse] username configured in `users.xml` or via SQL workflow. + +#### RQ.SRS-016.Kerberos.InvalidUser.UserDeleted +version: 1.0 + +[ClickHouse] SHALL reject [Kerberos] authentication if [ClickHouse] user was removed from the database using an SQL query. + +### Kerberos Not Available + +#### RQ.SRS-016.Kerberos.KerberosNotAvailable.InvalidServerTicket +version: 1.0 + +[ClickHouse] SHALL reject [Kerberos] authentication if [ClickHouse] user is configured to be authenticated using [Kerberos] and [Kerberos] server is unavailable, but [ClickHouse] doesn't have a valid Kerberos ticket or the ticket is expired. + +#### RQ.SRS-016.Kerberos.KerberosNotAvailable.InvalidClientTicket +version: 1.0 + +[ClickHouse] SHALL reject [Kerberos] authentication if [ClickHouse] user is configured to to be authenticated using [Kerberos] and [Kerberos] server is unavailable, but the client doesn't have a valid Kerberos ticket or the ticket is expired. + +#### RQ.SRS-016.Kerberos.KerberosNotAvailable.ValidTickets +version: 1.0 + +[ClickHouse] SHALL accept [Kerberos] authentication if no [Kerberos] server is reachable, but [ClickHouse] is configured to use valid credentials and [ClickHouse] has already processed some valid kerberized request (so it was granted a ticket), and the client has a valid ticket as well. + +### Kerberos Restarted + +#### RQ.SRS-016.Kerberos.KerberosServerRestarted +version: 1.0 + +[ClickHouse] SHALL accept [Kerberos] authentication if [Kerberos] server was restarted. + +### Performance + +#### RQ.SRS-016.Kerberos.Performance +version: 1.0 + +[ClickHouse]'s performance for [Kerberos] authentication SHALL be comparable to regular authentication. + +### Parallel Requests processing + +#### RQ.SRS-016.Kerberos.Parallel +version: 1.0 + +[ClickHouse] SHALL support parallel authentication using [Kerberos]. + +#### RQ.SRS-016.Kerberos.Parallel.ValidRequests.KerberosAndNonKerberos +version: 1.0 + +[ClickHouse] SHALL support processing of simultaneous kerberized (for users configured to authenticate via [Kerberos]) and non-kerberized (for users configured to authenticate with any other means) requests. + +#### RQ.SRS-016.Kerberos.Parallel.ValidRequests.SameCredentials +version: 1.0 + +[ClickHouse] SHALL support processing of simultaneously sent [Kerberos] requests under the same credentials. + +#### RQ.SRS-016.Kerberos.Parallel.ValidRequests.DifferentCredentials +version: 1.0 + +[ClickHouse] SHALL support processing of simultaneously sent [Kerberos] requests under different credentials. + +#### RQ.SRS-016.Kerberos.Parallel.ValidInvalid +version: 1.0 + +[ClickHouse] SHALL support parallel authentication of users using [Kerberos] server, some of which are valid and some invalid. Valid users' authentication should not be affected by invalid users' attempts. + +#### RQ.SRS-016.Kerberos.Parallel.Deletion +version: 1.0 + +[ClickHouse] SHALL not crash when two or more [Kerberos] users are simultaneously deleting one another. + +## References + +* **ClickHouse:** https://clickhouse.tech +* **Gitlab Repository:** https://gitlab.com/altinity-qa/documents/qa-srs016-clickhouse-kerberos-authentication/-/blob/master/QA_SRS016_ClickHouse_Kerberos_Authentication.md +* **Revision History:** https://gitlab.com/altinity-qa/documents/qa-srs016-clickhouse-kerberos-authentication/-/commits/master/QA_SRS016_ClickHouse_Kerberos_Authentication.md +* **Git:** https://git-scm.com/ +* **Kerberos terminology:** https://web.mit.edu/kerberos/kfw-4.1/kfw-4.1/kfw-4.1-help/html/kerberos_terminology.htm + +[Kerberos]: https://en.wikipedia.org/wiki/Kerberos_(protocol) +[SPNEGO]: https://en.wikipedia.org/wiki/SPNEGO +[ClickHouse]: https://clickhouse.tech +[GitLab]: https://gitlab.com +[GitLab Repository]: https://gitlab.com/altinity-qa/documents/qa-srs016-clickhouse-kerberos-authentication/-/blob/master/QA_SRS016_ClickHouse_Kerberos_Authentication.md +[Revision History]: https://gitlab.com/altinity-qa/documents/qa-srs016-clickhouse-kerberos-authentication/-/commits/master/QA_SRS016_ClickHouse_Kerberos_Authentication.md +[Git]: https://git-scm.com/ +[Kerberos terminology]: https://web.mit.edu/kerberos/kfw-4.1/kfw-4.1/kfw-4.1-help/html/kerberos_terminology.htm +''') diff --git a/tests/testflows/kerberos/tests/common.py b/tests/testflows/kerberos/tests/common.py new file mode 100644 index 00000000000..e768a78cad5 --- /dev/null +++ b/tests/testflows/kerberos/tests/common.py @@ -0,0 +1,225 @@ +from testflows.core import * +from testflows.asserts import error +from contextlib import contextmanager +import xml.etree.ElementTree as xmltree + +import time +import uuid + + +def getuid(): + return str(uuid.uuid1()).replace('-', '_') + + +def xml_append(root, tag, text=Null): + element = xmltree.Element(tag) + if text: + element.text = text + root.append(element) + + +def xml_write(data, filename): + strdata = xmltree.tostring(data) + with open(filename, "wb") as f: + f.write(strdata) + + +def xml_parse_file(filename): + return xmltree.parse(filename).getroot() + + +def create_default_config(filename): + contents = "" + if "kerberos_users.xml" in filename: + contents = "EXAMPLE.COM" \ + "" + elif "kerberos.xml" in filename: + contents = "EXAMPLE.COM" + + with open(filename, "w") as f: + f.write(contents) + + +def test_select_query(node, krb_auth=True, req="SELECT currentUser()"): + """ Helper forming a HTTP query to ClickHouse server + """ + if krb_auth: + return f"echo '{req}' | curl --negotiate -u : 'http://{node.name}:8123/' --data-binary @-" + else: + return f"echo '{req}' | curl 'http://{node.name}:8123/' --data-binary @-" + + +@TestStep(Given) +def kinit_no_keytab(self, node, principal="kerberos_user", lifetime_option="-l 10:00"): + """ Helper for obtaining Kerberos ticket for client + """ + try: + node.cmd("echo pwd | kinit admin/admin") + node.cmd(f"kadmin -w pwd -q \"add_principal -pw pwd {principal}\"") + node.cmd(f"echo pwd | kinit {lifetime_option} {principal}") + yield + finally: + node.cmd("kdestroy") + + +@TestStep(Given) +def create_server_principal(self, node): + """ Helper for obtaining Kerberos ticket for server + """ + try: + node.cmd("echo pwd | kinit admin/admin") + node.cmd(f"kadmin -w pwd -q \"add_principal -randkey HTTP/docker-compose_{node.name}_1.docker-compose_default\"") + node.cmd(f"kadmin -w pwd -q \"ktadd -k /etc/krb5.keytab HTTP/docker-compose_{node.name}_1.docker-compose_default\"") + yield + finally: + node.cmd("kdestroy") + node.cmd("rm /etc/krb5.keytab") + + +@TestStep(Given) +def save_file_state(self, node, filename): + """ Save current file and then restore it, restarting the node + """ + try: + with When("I save file state"): + with open(filename, 'r') as f: + a = f.read() + yield + finally: + with Finally("I restore initial state"): + with open(filename, 'w') as f: + f.write(a) + node.restart() + + +@TestStep(Given) +def temp_erase(self, node, filename=None): + """ Temporary erasing config file and restarting the node + """ + if filename is None: + filename = f"kerberos/configs/{node.name}/config.d/kerberos.xml" + with When("I save file state"): + with open(filename, 'r') as f: + a = f.read() + try: + with Then("I overwrite file to be dummy"): + with open(filename, 'w') as f: + f.write("\n") + node.restart() + yield + finally: + with Finally("I restore initial file state"): + with open(filename, 'w') as f: + f.write(a) + node.restart() + + +def restart(node, config_path, safe=False, timeout=60): + """Restart ClickHouse server and wait for config to be reloaded. + """ + + filename = '/etc/clickhouse-server/config.xml' if 'config.d' in config_path else '/etc/clickhouse-server/users.xml' + with When("I restart ClickHouse server node"): + with node.cluster.shell(node.name) as bash: + bash.expect(bash.prompt) + + with By("closing terminal to the node to be restarted"): + bash.close() + + with And("getting current log size"): + logsize = \ + node.command("stat --format=%s /var/log/clickhouse-server/clickhouse-server.log").output.split(" ")[0].strip() + + with And("restarting ClickHouse server"): + node.restart(safe=safe) + + with Then("tailing the log file from using previous log size as the offset"): + bash.prompt = bash.__class__.prompt + bash.open() + bash.send(f"tail -c +{logsize} -f /var/log/clickhouse-server/clickhouse-server.log") + + with And("waiting for config reload message in the log file"): + bash.expect( + f"ConfigReloader: Loaded config '{filename}', performed update on configuration", + timeout=timeout) + + +@TestStep +def check_wrong_config(self, node, client, config_path, modify_file, log_error="", output="", + tail=120, timeout=60, healthy_on_restart=True): + """Check that ClickHouse errors when trying to load invalid configuration file. + """ + preprocessed_name = "config.xml" if "config.d" in config_path else "users.xml" + + full_config_path = "/etc/clickhouse-server/config.d/kerberos.xml" if "config.d" in config_path else "/etc/clickhouse-server/users.d/kerberos-users.xml" + + uid = getuid() + + try: + with Given("I save config file to restore it later"): + with open(config_path, 'r') as f: + initial_contents = f.read() + + with And("I prepare the error log by writing empty lines into it"): + node.command("echo -e \"%s\" > /var/log/clickhouse-server/clickhouse-server.err.log" % ("-\\n" * tail)) + + with When("I modify xml file"): + root = xml_parse_file(config_path) + root = modify_file(root) + root.append(xmltree.fromstring(f"{uid}")) + config_contents = xmltree.tostring(root, encoding='utf8', method='xml').decode('utf-8') + command = f"cat < {full_config_path}\n{config_contents}\nHEREDOC" + node.command(command, steps=False, exitcode=0) + # time.sleep(1) + + with Then(f"{preprocessed_name} should be updated", description=f"timeout {timeout}"): + started = time.time() + command = f"cat /var/lib/clickhouse/preprocessed_configs/{preprocessed_name} | grep {uid} > /dev/null" + while time.time() - started < timeout: + exitcode = node.command(command, steps=False).exitcode + if exitcode == 0: + break + time.sleep(1) + assert exitcode == 0, error() + + with When("I restart ClickHouse to apply the config changes"): + if output: + node.restart(safe=False, wait_healthy=True) + else: + node.restart(safe=False, wait_healthy=False) + + if output != "": + with Then(f"check {output} is in output"): + time.sleep(5) + started = time.time() + while time.time() - started < timeout: + kinit_no_keytab(node=client) + create_server_principal(node=node) + r = client.cmd(test_select_query(node=node), no_checks=True) + if output in r.output: + assert True, error() + break + time.sleep(1) + else: + assert False, error() + + finally: + with Finally("I restore original config"): + with By("restoring the (correct) config file"): + with open(config_path, 'w') as f: + f.write(initial_contents) + with And("restarting the node"): + node.restart(safe=False) + + if log_error != "": + with Then("error log should contain the expected error message"): + started = time.time() + command = f"tail -n {tail} /var/log/clickhouse-server/clickhouse-server.err.log | grep \"{log_error}\"" + while time.time() - started < timeout: + exitcode = node.command(command, steps=False).exitcode + if exitcode == 0: + break + time.sleep(1) + assert exitcode == 0, error() + + diff --git a/tests/testflows/kerberos/tests/config.py b/tests/testflows/kerberos/tests/config.py new file mode 100644 index 00000000000..3f4bf15deb5 --- /dev/null +++ b/tests/testflows/kerberos/tests/config.py @@ -0,0 +1,163 @@ +from testflows.core import * +from kerberos.tests.common import * +from kerberos.requirements.requirements import * + +import time +import datetime +import itertools + + +@TestScenario +@Requirements( + RQ_SRS_016_Kerberos_Configuration_KerberosNotEnabled("1.0") +) +def kerberos_not_enabled(self): + """ClickHouse SHALL reject Kerberos authentication if user is properly configured for Kerberos, + but Kerberos itself is not enabled in config.xml. + """ + ch_nodes = self.context.ch_nodes + config_path = f"kerberos/configs/{ch_nodes[0].name}/config.d/kerberos.xml" + + def modify_file(root): + return xmltree.fromstring("") + + check_wrong_config(node=ch_nodes[0], client=ch_nodes[2], config_path=config_path, modify_file=modify_file, + output="Kerberos is not enabled") + + +@TestScenario +@Requirements( + RQ_SRS_016_Kerberos_Configuration_MultipleKerberosSections("1.0") +) +def multiple_kerberos(self): + """ClickHouse SHALL disable Kerberos authentication if more than one kerberos sections specified in config.xml. + """ + ch_nodes = self.context.ch_nodes + config_path = f"kerberos/configs/{ch_nodes[0].name}/config.d/kerberos.xml" + + def modify_file(root): + second_section = "EXAM.COM" + root.append(xmltree.fromstring(second_section)) + return root + + check_wrong_config(node=ch_nodes[0], client=ch_nodes[2], config_path=config_path, modify_file=modify_file, + log_error="Multiple kerberos sections are not allowed", healthy_on_restart=False) + + +@TestScenario +@Requirements( + RQ_SRS_016_Kerberos_Configuration_WrongUserRealm("1.0") +) +def wrong_user_realm(self): + """ClickHouse SHALL reject Kerberos authentication if user's realm specified in users.xml + doesn't match the realm of the principal trying to authenticate. + """ + + ch_nodes = self.context.ch_nodes + config_path = f"kerberos/configs/{ch_nodes[0].name}/users.d/kerberos-users.xml" + + def modify_file(root): + krb = root.find('users').find('kerberos_user') + krb.find('kerberos').find('realm').text = "OTHER.COM" + return root + + check_wrong_config(node=ch_nodes[0], client=ch_nodes[2], config_path=config_path, modify_file=modify_file, + output="Authentication failed") + + +@TestScenario +@Requirements( + RQ_SRS_016_Kerberos_Configuration_MultipleAuthMethods("1.0") +) +def multiple_auth_methods(self): + """ClickHouse SHALL reject Kerberos authentication if other + auth method is specified for user alongside with Kerberos. + """ + ch_nodes = self.context.ch_nodes + config_path = f"kerberos/configs/{ch_nodes[0].name}/users.d/kerberos-users.xml" + + def modify_file(root): + krb = root.find('users').find('kerberos_user') + xml_append(krb, 'password', 'qwerty') + return root + + check_wrong_config(node=ch_nodes[0], client=ch_nodes[2], config_path=config_path, modify_file=modify_file, + log_error="More than one field of", healthy_on_restart=False) + + +@TestScenario +@Requirements( + RQ_SRS_016_Kerberos_Configuration_PrincipalAndRealmSpecified("1.0") +) +def principal_and_realm_specified(self): + """ClickHouse SHALL drop an exception if both realm and principal fields are specified in config.xml. + """ + ch_nodes = self.context.ch_nodes + config_path = f"kerberos/configs/{ch_nodes[0].name}/config.d/kerberos.xml" + + def modify_file(root): + krb = root.find('kerberos') + xml_append(krb, 'principal', 'HTTP/srv1@EXAMPLE.COM') + return root + + check_wrong_config(node=ch_nodes[0], client=ch_nodes[2], config_path=config_path, modify_file=modify_file, + log_error="Realm and principal name cannot be specified simultaneously", + output="Kerberos is not enabled") + + +@TestScenario +@Requirements( + RQ_SRS_016_Kerberos_Configuration_MultipleRealmSections("1.0") +) +def multiple_realm(self): + """ClickHouse SHALL throw an exception and disable Kerberos if more than one realm is specified in config.xml. + """ + ch_nodes = self.context.ch_nodes + config_path = f"kerberos/configs/{ch_nodes[0].name}/config.d/kerberos.xml" + + def modify_file(root): + krb = root.find('kerberos') + xml_append(krb, 'realm', 'EXAM.COM') + return root + + check_wrong_config(node=ch_nodes[0], client=ch_nodes[2], config_path=config_path, modify_file=modify_file, + log_error="Multiple realm sections are not allowed") + + +@TestScenario +@Requirements( + RQ_SRS_016_Kerberos_Configuration_MultiplePrincipalSections("1.0") +) +def multiple_principal(self): + """ClickHouse SHALL throw an exception and disable Kerberos if more than one principal is specified in config.xml. + """ + ch_nodes = self.context.ch_nodes + config_path = f"kerberos/configs/{ch_nodes[0].name}/config.d/kerberos.xml" + + def modify_file(root): + krb = root.find('kerberos') + krb.remove(krb.find('realm')) + xml_append(krb, 'principal', 'HTTP/s1@EXAMPLE.COM') + xml_append(krb, 'principal', 'HTTP/s2@EXAMPLE.COM') + return root + + check_wrong_config(node=ch_nodes[0], client=ch_nodes[2], config_path=config_path, modify_file=modify_file, + log_error="Multiple principal sections are not allowed") + + + + + + + +@TestFeature +def config(self): + """Perform ClickHouse Kerberos authentication testing for incorrect configuration files + """ + + self.context.ch_nodes = [self.context.cluster.node(f"clickhouse{i}") for i in range(1, 4)] + self.context.krb_server = self.context.cluster.node("kerberos") + self.context.clients = [self.context.cluster.node(f"krb-client{i}") for i in range(1, 6)] + + for scenario in loads(current_module(), Scenario, Suite): + Scenario(run=scenario, flags=TE) diff --git a/tests/testflows/kerberos/tests/generic.py b/tests/testflows/kerberos/tests/generic.py new file mode 100644 index 00000000000..3276fd5ec5f --- /dev/null +++ b/tests/testflows/kerberos/tests/generic.py @@ -0,0 +1,332 @@ +from testflows.core import * +from kerberos.tests.common import * +from kerberos.requirements.requirements import * + +import time +import datetime +import itertools + + +@TestScenario +@Requirements( + RQ_SRS_016_Kerberos_ValidUser_XMLConfiguredUser("1.0") +) +def xml_configured_user(self): + """ClickHouse SHALL accept Kerberos authentication for valid XML-configured user + """ + ch_nodes = self.context.ch_nodes + + with Given("kinit for client"): + kinit_no_keytab(node=ch_nodes[2]) + + with And("kinit for server"): + create_server_principal(node=ch_nodes[0]) + + with When("I attempt to authenticate"): + r = ch_nodes[2].cmd(test_select_query(node=ch_nodes[0])) + + with Then(f"I expect 'kerberos_user'"): + assert r.output == "kerberos_user", error() + + +@TestScenario +@Requirements( + RQ_SRS_016_Kerberos_ValidUser_RBACConfiguredUser("1.0") +) +def rbac_configured_user(self): + """ClickHouse SHALL accept Kerberos authentication for valid RBAC-configured user + """ + ch_nodes = self.context.ch_nodes + + with Given("kinit for client"): + kinit_no_keytab(node=ch_nodes[2], principal="krb_rbac") + + with And("kinit for server"): + create_server_principal(node=ch_nodes[0]) + + with When("I create a RBAC user"): + ch_nodes[0].query("CREATE USER krb_rbac IDENTIFIED WITH kerberos REALM 'EXAMPLE.COM'") + + with When("I attempt to authenticate"): + r = ch_nodes[2].cmd(test_select_query(node=ch_nodes[0])) + + with Then("I restore server original state"): + ch_nodes[0].query("DROP USER krb_rbac") + + with Finally("I expect 'krb_rbac'"): + assert r.output == "krb_rbac", error() + + +@TestScenario +@Requirements( + RQ_SRS_016_Kerberos_KerberosNotAvailable_InvalidServerTicket("1.0") +) +def invalid_server_ticket(self): + """ClickHouse SHALL reject Kerberos authentication no Kerberos server is reachable + and CH-server has no valid ticket (or the existing ticket is outdated). + """ + ch_nodes = self.context.ch_nodes + + with Given("kinit for client"): + kinit_no_keytab(node=ch_nodes[2]) + + with And("setting up server principal"): + create_server_principal(node=ch_nodes[0]) + + with And("I kill kerberos-server"): + self.context.krb_server.stop() + + with When("I attempt to authenticate as kerberos_user"): + r = ch_nodes[2].cmd(test_select_query(node=ch_nodes[0])) + + with Then("I start kerberos server again"): + self.context.krb_server.start() + ch_nodes[2].cmd("kdestroy") + while True: + kinit_no_keytab(node=ch_nodes[2]) + if ch_nodes[2].cmd(test_select_query(node=ch_nodes[0])).output == "kerberos_user": + break + ch_nodes[2].cmd("kdestroy") + + with And("I expect the user to be default"): + assert r.output == "default", error() + + +@TestScenario +@Requirements( + RQ_SRS_016_Kerberos_KerberosNotAvailable_InvalidClientTicket("1.0") +) +def invalid_client_ticket(self): + """ClickHouse SHALL reject Kerberos authentication no Kerberos server is reachable + and client has no valid ticket (or the existing ticket is outdated). + """ + ch_nodes = self.context.ch_nodes + + with Given("kinit for client"): + kinit_no_keytab(node=ch_nodes[2], lifetime_option="-l 00:00:05") + + with And("setting up server principal"): + create_server_principal(node=ch_nodes[0]) + + with And("I kill kerberos-server"): + self.context.krb_server.stop() + + with And("I wait until client ticket is expired"): + time.sleep(10) + + with When("I attempt to authenticate as kerberos_user"): + r = ch_nodes[2].cmd(test_select_query(node=ch_nodes[0])) + + with Then("I expect the user to be default"): + assert r.output == "default", error() + + with Finally("I start kerberos server again"): + self.context.krb_server.start() + ch_nodes[2].cmd("kdestroy") + while True: + kinit_no_keytab(node=ch_nodes[2]) + if ch_nodes[2].cmd(test_select_query(node=ch_nodes[0])).output == "kerberos_user": + break + ch_nodes[2].cmd("kdestroy") + + +@TestScenario +@Requirements( + RQ_SRS_016_Kerberos_KerberosNotAvailable_ValidTickets("1.0") +) +def kerberos_unreachable_valid_tickets(self): + """ClickHouse SHALL accept Kerberos authentication if no Kerberos server is reachable + but both CH-server and client have valid tickets. + """ + ch_nodes = self.context.ch_nodes + + with Given("kinit for client"): + kinit_no_keytab(node=ch_nodes[2]) + + with And("setting up server principal"): + create_server_principal(node=ch_nodes[0]) + + with And("make sure server obtained ticket"): + ch_nodes[2].cmd(test_select_query(node=ch_nodes[0])) + + with And("I kill kerberos-server"): + self.context.krb_server.stop() + + with When("I attempt to authenticate as kerberos_user"): + r = ch_nodes[2].cmd(test_select_query(node=ch_nodes[0])) + + with Then("I expect the user to be default"): + assert r.output == "kerberos_user", error() + + with Finally("I start kerberos server again"): + self.context.krb_server.start() + ch_nodes[2].cmd("kdestroy") + while True: + kinit_no_keytab(node=ch_nodes[2]) + if ch_nodes[2].cmd(test_select_query(node=ch_nodes[0])).output == "kerberos_user": + break + ch_nodes[2].cmd("kdestroy") + + +@TestScenario +@Requirements( + RQ_SRS_016_Kerberos_ValidUser_KerberosNotConfigured("1.0") +) +def kerberos_not_configured(self): + """ClickHouse SHALL reject Kerberos authentication if user is not a kerberos-auth user. + """ + ch_nodes = self.context.ch_nodes + + with Given("kinit for client"): + kinit_no_keytab(node=ch_nodes[2], principal="unkerberized") + + with And('Kinit for server'): + create_server_principal(node=ch_nodes[0]) + + with By("I add non-Kerberos user to ClickHouse"): + ch_nodes[0].query("CREATE USER unkerberized IDENTIFIED WITH plaintext_password BY 'qwerty'") + + with When("I attempt to authenticate"): + r = ch_nodes[2].cmd(test_select_query(node=ch_nodes[0]), no_checks=True) + + with Then("I expect authentication failure"): + assert "Authentication failed" in r.output, error() + + with Finally("I drop the user"): + ch_nodes[0].query("DROP USER unkerberized") + + +@TestScenario +@Requirements( + RQ_SRS_016_Kerberos_KerberosServerRestarted("1.0") +) +def kerberos_server_restarted(self): + """ClickHouse SHALL accept Kerberos authentication if Kerberos server was restarted. + """ + ch_nodes = self.context.ch_nodes + krb_server = self.context.krb_server + + with Given("I obtain keytab for user"): + kinit_no_keytab(node=ch_nodes[2]) + with And("I create server principal"): + create_server_principal(node=ch_nodes[0]) + with And("I obtain server ticket"): + ch_nodes[2].cmd(test_select_query(node=ch_nodes[0]), no_checks=True) + with By("I dump, restart and restore kerberos server"): + krb_server.cmd("kdb5_util dump dump.dmp", shell_command="/bin/sh") + krb_server.restart() + krb_server.cmd("kdb5_util load dump.dmp", shell_command="/bin/sh") + + with When("I attempt to authenticate"): + r = ch_nodes[2].cmd(test_select_query(node=ch_nodes[0])) + + with And("I wait for kerberos to be healthy"): + ch_nodes[2].cmd("kdestroy") + while True: + kinit_no_keytab(node=ch_nodes[2]) + if ch_nodes[2].cmd(test_select_query(node=ch_nodes[0])).output == "kerberos_user": + break + + with Then(f"I expect kerberos_user"): + assert r.output == "kerberos_user", error() + + +@TestScenario +@Requirements( + RQ_SRS_016_Kerberos_InvalidUser("1.0") +) +def invalid_user(self): + """ClickHouse SHALL reject Kerberos authentication for invalid principal + """ + ch_nodes = self.context.ch_nodes + + with Given("I obtain keytab for invalid user"): + kinit_no_keytab(node=ch_nodes[2], principal="invalid") + + with And("I create server principal"): + create_server_principal(node=ch_nodes[0]) + + with When("I attempt to authenticate"): + r = ch_nodes[2].cmd(test_select_query(node=ch_nodes[0]), no_checks=True) + + with Then(f"I expect default"): + assert "Authentication failed: password is incorrect or there is no user with such name" in r.output, error() + + +@TestScenario +@Requirements( + RQ_SRS_016_Kerberos_InvalidUser_UserDeleted("1.0") +) +def user_deleted(self): + """ClickHouse SHALL reject Kerberos authentication if Kerberos user was deleted prior to query. + """ + ch_nodes = self.context.ch_nodes + + with Given("I obtain keytab for a user"): + kinit_no_keytab(node=ch_nodes[2], principal="krb_rbac") + + with And("I create server principal"): + create_server_principal(node=ch_nodes[0]) + + with And("I create and then delete kerberized user"): + ch_nodes[0].query("CREATE USER krb_rbac IDENTIFIED WITH kerberos REALM 'EXAMPLE.COM'") + ch_nodes[0].query("DROP USER krb_rbac") + + with When("I attempt to authenticate"): + r = ch_nodes[2].cmd(test_select_query(node=ch_nodes[0]), no_checks=True) + + with Then(f"I expect error"): + assert "Authentication failed: password is incorrect or there is no user with such name" in r.output, error() + + +@TestScenario +@Requirements( + RQ_SRS_016_Kerberos_Performance("1.0") +) +def authentication_performance(self): + """ClickHouse's performance for Kerberos authentication SHALL shall be comparable to regular authentication. + """ + ch_nodes = self.context.ch_nodes + + with Given("I obtain keytab for a user"): + kinit_no_keytab(node=ch_nodes[2]) + + with And("I create server principal"): + create_server_principal(node=ch_nodes[0]) + + with And("I create a password-identified user"): + ch_nodes[0].query("CREATE USER pwd_user IDENTIFIED WITH plaintext_password BY 'pwd'") + + with When("I measure kerberos auth time"): + start_time_krb = time.time() + for i in range(100): + ch_nodes[2].cmd(test_select_query(node=ch_nodes[0])) + krb_time = (time.time() - start_time_krb) / 100 + + with And("I measure password auth time"): + start_time_usual = time.time() + for i in range(100): + ch_nodes[2].cmd(f"echo 'SELECT 1' | curl 'http://pwd_user:pwd@clickhouse1:8123/' -d @-") + usual_time = (time.time() - start_time_usual) / 100 + + with Then("measuring the performance compared to password auth"): + metric("percentage_improvement", units="%", value=100*(krb_time - usual_time)/usual_time) + + with Finally("I drop pwd_user"): + ch_nodes[0].query("DROP USER pwd_user") + + + + + +@TestFeature +def generic(self): + """Perform ClickHouse Kerberos authentication testing + """ + + self.context.ch_nodes = [self.context.cluster.node(f"clickhouse{i}") for i in range(1, 4)] + self.context.krb_server = self.context.cluster.node("kerberos") + self.context.clients = [self.context.cluster.node(f"krb-client{i}") for i in range(1, 6)] + + for scenario in loads(current_module(), Scenario, Suite): + Scenario(run=scenario, flags=TE) diff --git a/tests/testflows/kerberos/tests/parallel.py b/tests/testflows/kerberos/tests/parallel.py new file mode 100644 index 00000000000..694245e524c --- /dev/null +++ b/tests/testflows/kerberos/tests/parallel.py @@ -0,0 +1,204 @@ +from testflows.core import * +from kerberos.tests.common import * +from kerberos.requirements.requirements import * +from multiprocessing.dummy import Pool + +import time +import datetime +import itertools + + +@TestScenario +@Requirements( + RQ_SRS_016_Kerberos_Parallel_ValidRequests_SameCredentials("1.0") +) +def valid_requests_same_credentials(self): + """ClickHouse should be able to process parallel requests sent under the same credentials. + """ + ch_nodes = self.context.ch_nodes + + with Given("kinit for clients"): + kinit_no_keytab(node=ch_nodes[1]) + kinit_no_keytab(node=ch_nodes[2]) + + with And('create server principal'): + create_server_principal(node=ch_nodes[0]) + + def helper(cmd): + return cmd(test_select_query(node=ch_nodes[0])) + + for i in range(15): + pool = Pool(2) + tasks = [] + with When("I try simultaneous authentication"): + tasks.append(pool.apply_async(helper, (ch_nodes[1].cmd, ))) + tasks.append(pool.apply_async(helper, (ch_nodes[2].cmd, ))) + tasks[0].wait(timeout=200) + tasks[1].wait(timeout=200) + + with Then(f"I expect requests to success"): + assert tasks[0].get(timeout=300).output == "kerberos_user", error() + assert tasks[1].get(timeout=300).output == "kerberos_user", error() + + +@TestScenario +@Requirements( + RQ_SRS_016_Kerberos_Parallel_ValidRequests_DifferentCredentials("1.0") +) +def valid_requests_different_credentials(self): + """ClickHouse should be able to process parallel requests by different users. + """ + ch_nodes = self.context.ch_nodes + + with Given("kinit for clients"): + kinit_no_keytab(node=ch_nodes[1], principal="krb1") + kinit_no_keytab(node=ch_nodes[2], principal="krb2") + + with And("create server principal"): + create_server_principal(node=ch_nodes[0]) + + def helper(cmd): + return cmd(test_select_query(node=ch_nodes[0])) + + for i in range(15): + pool = Pool(2) + tasks = [] + + with And("add 2 kerberos users via RBAC"): + ch_nodes[0].query("CREATE USER krb1 IDENTIFIED WITH kerberos REALM 'EXAMPLE.COM'") + ch_nodes[0].query("CREATE USER krb2 IDENTIFIED WITH kerberos REALM 'EXAMPLE.COM'") + + with When("I try simultaneous authentication for valid and invalid"): + tasks.append(pool.apply_async(helper, (ch_nodes[1].cmd, ))) + tasks.append(pool.apply_async(helper, (ch_nodes[2].cmd, ))) + tasks[0].wait(timeout=200) + tasks[1].wait(timeout=200) + + with Then(f"I expect have auth failure"): + assert tasks[1].get(timeout=300).output == "krb2", error() + assert tasks[0].get(timeout=300).output == "krb1", error() + + with Finally("I make sure both users are removed"): + ch_nodes[0].query("DROP USER krb1", no_checks=True) + ch_nodes[0].query("DROP USER krb2", no_checks=True) + + +@TestScenario +@Requirements( + RQ_SRS_016_Kerberos_Parallel_ValidInvalid("1.0") +) +def valid_invalid(self): + """Valid users' Kerberos authentication should not be affected by invalid users' attempts. + """ + ch_nodes = self.context.ch_nodes + + with Given("kinit for clients"): + kinit_no_keytab(node=ch_nodes[2]) + kinit_no_keytab(node=ch_nodes[1], principal="invalid_user") + + with And('create server principal'): + create_server_principal(node=ch_nodes[0]) + + def helper(cmd): + return cmd(test_select_query(node=ch_nodes[0]), no_checks=True) + + for i in range(15): + pool = Pool(2) + tasks = [] + with When("I try simultaneous authentication for valid and invalid"): + tasks.append(pool.apply_async(helper, (ch_nodes[1].cmd, ))) # invalid + tasks.append(pool.apply_async(helper, (ch_nodes[2].cmd, ))) # valid + + with Then(f"I expect have auth failure"): + assert tasks[1].get(timeout=300).output == "kerberos_user", error() + assert tasks[0].get(timeout=300).output != "kerberos_user", error() + + +@TestScenario +@Requirements( + RQ_SRS_016_Kerberos_Parallel_Deletion("1.0") +) +def deletion(self): + """ClickHouse SHALL NOT crash when 2 Kerberos users are simultaneously deleting one another. + """ + ch_nodes = self.context.ch_nodes + + with Given("kinit for clients"): + kinit_no_keytab(node=ch_nodes[1], principal="krb1") + kinit_no_keytab(node=ch_nodes[2], principal="krb2") + + with And("create server principal"): + create_server_principal(node=ch_nodes[0]) + + def helper(cmd, todel): + return cmd(test_select_query(node=ch_nodes[0], req=f"DROP USER {todel}"), no_checks=True) + + for i in range(15): + pool = Pool(2) + tasks = [] + + with And("add 2 kerberos users via RBAC"): + ch_nodes[0].query("CREATE USER krb1 IDENTIFIED WITH kerberos REALM 'EXAMPLE.COM'") + ch_nodes[0].query("CREATE USER krb2 IDENTIFIED WITH kerberos REALM 'EXAMPLE.COM'") + ch_nodes[0].query("GRANT ACCESS MANAGEMENT ON *.* TO krb1") + ch_nodes[0].query("GRANT ACCESS MANAGEMENT ON *.* TO krb2") + + + with When("I try simultaneous authentication for valid and invalid"): + tasks.append(pool.apply_async(helper, (ch_nodes[1].cmd, "krb2"))) + tasks.append(pool.apply_async(helper, (ch_nodes[2].cmd, "krb1"))) + tasks[0].wait(timeout=200) + tasks[1].wait(timeout=200) + + with Then(f"I check CH is alive"): + assert ch_nodes[0].query("SELECT 1").output == "1", error() + + with Finally("I make sure both users are removed"): + ch_nodes[0].query("DROP USER krb1", no_checks=True) + ch_nodes[0].query("DROP USER krb2", no_checks=True) + + +@TestScenario +@Requirements( + RQ_SRS_016_Kerberos_Parallel_ValidRequests_KerberosAndNonKerberos("1.0") +) +def kerberos_and_nonkerberos(self): + """ClickHouse SHALL support processing of simultaneous kerberized and non-kerberized requests. + """ + ch_nodes = self.context.ch_nodes + + with Given("kinit for clients"): + kinit_no_keytab(node=ch_nodes[2]) + + with And('create server principal'): + create_server_principal(node=ch_nodes[0]) + + def helper(cmd, krb_auth): + return cmd(test_select_query(node=ch_nodes[0], krb_auth=krb_auth), no_checks=True) + + for i in range(15): + pool = Pool(2) + tasks = [] + with When("I try simultaneous authentication for valid and invalid"): + tasks.append(pool.apply_async(helper, (ch_nodes[1].cmd, False))) # non-kerberos + tasks.append(pool.apply_async(helper, (ch_nodes[2].cmd, True))) # kerberos + + with Then(f"I expect have auth failure"): + assert tasks[1].get(timeout=300).output == "kerberos_user", error() + assert tasks[0].get(timeout=300).output == "default", error() + + +@TestFeature +@Requirements( + RQ_SRS_016_Kerberos_Parallel("1.0") +) +def parallel(self): + """Perform ClickHouse Kerberos authentication testing for incorrect configuration files + """ + + self.context.ch_nodes = [self.context.cluster.node(f"clickhouse{i}") for i in range(1, 4)] + self.context.krb_server = self.context.cluster.node("kerberos") + self.context.clients = [self.context.cluster.node(f"krb-client{i}") for i in range(1, 6)] + + for scenario in loads(current_module(), Scenario, Suite): + Scenario(run=scenario, flags=TE) diff --git a/tests/testflows/ldap/authentication/requirements/requirements.md b/tests/testflows/ldap/authentication/requirements/requirements.md index 27ce8c921a0..e27a0ca14f7 100644 --- a/tests/testflows/ldap/authentication/requirements/requirements.md +++ b/tests/testflows/ldap/authentication/requirements/requirements.md @@ -468,7 +468,7 @@ version: 1.0 the following RBAC command ```sql -CREATE USER name IDENTIFIED WITH ldap_server BY 'server_name' +CREATE USER name IDENTIFIED WITH ldap SERVER 'server_name' ``` #### RQ.SRS-007.LDAP.Configuration.User.Syntax diff --git a/tests/testflows/ldap/authentication/requirements/requirements.py b/tests/testflows/ldap/authentication/requirements/requirements.py index 25b943d18c2..2437711dd22 100644 --- a/tests/testflows/ldap/authentication/requirements/requirements.py +++ b/tests/testflows/ldap/authentication/requirements/requirements.py @@ -903,7 +903,7 @@ RQ_SRS_007_LDAP_Configuration_User_RBAC = Requirement( 'the following RBAC command\n' '\n' '```sql\n' - "CREATE USER name IDENTIFIED WITH ldap_server BY 'server_name'\n" + "CREATE USER name IDENTIFIED WITH ldap SERVER 'server_name'\n" '```\n' '\n' ), @@ -1841,7 +1841,7 @@ version: 1.0 the following RBAC command ```sql -CREATE USER name IDENTIFIED WITH ldap_server BY 'server_name' +CREATE USER name IDENTIFIED WITH ldap SERVER 'server_name' ``` #### RQ.SRS-007.LDAP.Configuration.User.Syntax diff --git a/tests/testflows/ldap/authentication/tests/common.py b/tests/testflows/ldap/authentication/tests/common.py index 7f9f16e827c..6b6ce0413a4 100644 --- a/tests/testflows/ldap/authentication/tests/common.py +++ b/tests/testflows/ldap/authentication/tests/common.py @@ -248,7 +248,7 @@ def add_users_identified_with_ldap(*users): try: with Given("I create users"): for user in users: - node.query(f"CREATE USER '{user['username']}' IDENTIFIED WITH ldap_server BY '{user['server']}'") + node.query(f"CREATE USER '{user['username']}' IDENTIFIED WITH ldap SERVER '{user['server']}'") yield finally: with Finally("I remove users"): diff --git a/tests/testflows/regression.py b/tests/testflows/regression.py index 0e9a821cae0..05fec3ea985 100755 --- a/tests/testflows/regression.py +++ b/tests/testflows/regression.py @@ -18,6 +18,7 @@ def regression(self, local, clickhouse_binary_path, stress=None, parallel=None): Feature(test=load("ldap.regression", "regression"))(**args) Feature(test=load("rbac.regression", "regression"))(**args) Feature(test=load("aes_encryption.regression", "regression"))(**args) + # Feature(test=load("kerberos.regression", "regression"))(**args) if main(): regression() From 9f8e61b734ef496aa11fd318e1b84c5492cbd6e9 Mon Sep 17 00:00:00 2001 From: Alexander Kuzmenkov <36882414+akuzm@users.noreply.github.com> Date: Thu, 11 Mar 2021 23:57:11 +0300 Subject: [PATCH 347/544] Update fuzzing-clickhouse.md --- website/blog/en/2021/fuzzing-clickhouse.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/website/blog/en/2021/fuzzing-clickhouse.md b/website/blog/en/2021/fuzzing-clickhouse.md index d7d01587534..c1759f11f89 100644 --- a/website/blog/en/2021/fuzzing-clickhouse.md +++ b/website/blog/en/2021/fuzzing-clickhouse.md @@ -57,5 +57,5 @@ To see for yourself how it works, you only need the normal ClickHouse client. S The AST-based fuzzer we discussed is only one of the many kinds of fuzzers we have in ClickHouse. There is a [talk](https://www.youtube.com/watch?v=GbmK84ZwSeI&t=4481s) (in Russian, [slides are here](https://presentations.clickhouse.tech/cpp_siberia_2021/)) by Alexey Milovidov that explores all the fuzzers we have. Another interesting recent development is application of pivoted query synthesis technique, implemented in [SQLancer](https://github.com/sqlancer/sqlancer), to ClickHouse. The authors are going to give [a talk about this](https://heisenbug-piter.ru/2021/spb/talks/nr1cwknssdodjkqgzsbvh/) soon, so stay tuned. -12-08-21 [Alexander Kuzmenkov](https://github.com/akuzm) +_2021-03-11 [Alexander Kuzmenkov](https://github.com/akuzm)_ From 6cfac215fec7d9ea59c0d3ad1e204685a601d851 Mon Sep 17 00:00:00 2001 From: Alexander Kuzmenkov <36882414+akuzm@users.noreply.github.com> Date: Fri, 12 Mar 2021 01:11:46 +0300 Subject: [PATCH 348/544] Update fuzzing-clickhouse.md --- website/blog/en/2021/fuzzing-clickhouse.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/website/blog/en/2021/fuzzing-clickhouse.md b/website/blog/en/2021/fuzzing-clickhouse.md index c1759f11f89..47146c38d50 100644 --- a/website/blog/en/2021/fuzzing-clickhouse.md +++ b/website/blog/en/2021/fuzzing-clickhouse.md @@ -22,7 +22,7 @@ Generating valid SQL queries with bit flips would take a long time, so there are Consider some SQL query from a regression test. After parsing, it is easy to mutate the resulting AST (abstract syntax tree, an internal representation of the parsed query) before execution to introduce random changes into the query. For strings and arrays, we make random modifications such as inserting a random character or doubling the string. For numbers, there are well-known Bad Numbers such as 0, 1, powers of two and nearby, integer limits, `NaN`. `NaN`s proved to be especially efficient in finding bugs, because you can often have some alternative branches in your numeric code, but for a `NaN`, both branches hold (or not) simultaneously, so this leads to nasty effects. -Another interesting thing we can do is change the arguments of functions, or the list of expressions in `SELECT`. Naturally, all the interesting arguments can be taken from other test queries. Same goes for changing the tables used in the queries. When the fuzzer runs in CI, it runs queries from all the SQL tests in random order, mixing in the parts of query from different tests, so that we can eventually test all the possible permutations of our features. +Another interesting thing we can do is change the arguments of functions, or the list of expressions in `SELECT`, `ORDER BY` and so on. Naturally, all the interesting arguments can be taken from other test queries. Same goes for changing the tables used in the queries. When the fuzzer runs in CI, it runs queries from all the SQL tests in random order, mixing into them some parts of queries it has seen previously. This process can eventually cover all the possible permutations of our features. The core implementation of the fuzzer is relatively small, consisting of about 700 lines of C++ code. A prototype was made in a couple of days, but naturally it took significantly longer to polish it and to start routinely using it in CI. It is very productive and let us find more than 200 bugs already (see the label [fuzz](https://github.com/ClickHouse/ClickHouse/labels/fuzz) on GitHub), some of which are serious logic errors or even memory errors. When we only started, we could segfault the server or make it enter a never-ending loop with simplest read-only queries such as `SELECT arrayReverseFill(x -> (x < 10), [])` or `SELECT geoDistance(0., 0., -inf, 1.)`. Of course I couldn't resist bringing down our [public playground](https://gh-api.clickhouse.tech/play?user=play#LS0gWW91IGNhbiBxdWVyeSB0aGUgR2l0SHViIGhpc3RvcnkgZGF0YSBoZXJlLiBTZWUgaHR0cHM6Ly9naC5jbGlja2hvdXNlLnRlY2gvZXhwbG9yZXIvIGZvciB0aGUgZGVzY3JpcHRpb24gYW5kIGV4YW1wbGUgcXVlcmllcy4Kc2VsZWN0ICdoZWxsbyB3b3JsZCc=) with some of these queries, and was content to see that the server soon restarts correctly. These queries are actually minified by hand, normally the fuzzer would generate something barely legible such as: ``` @@ -49,9 +49,9 @@ Not all errors the fuzzer finds are significant, some of them are pretty boring After fixing the majority of pre-existing error, this fuzzer became efficient for finding errors in new features. Pull requests introducing new features normally add an SQL test, and we pay extra attention to the new tests when fuzzing, generating more permutations for them. Even if the coverage of the test is not sufficient, there is a good chance that the fuzzer will find the missing corner cases. So when we see that all the fuzzer runs in different configurations have failed for a particular pull request, this almost always means that it introduces a new bug. When developing a feature that requires new grammar, it is also helpful to add fuzzing support for it. I did this for window functions early in the development, and it helped me find several bugs. -A major factor that makes fuzzing really efficient is that we have a lot of assertions and other checks of program logic in our code. For debug-only checks, we use the plain `assert` macro from ``. For checks that are needed even in release mode, we use an exception with a special code `LOGICAL_ERROR` that signifies an internal program error. We did some work to ensure that these errors are distinct from errors caused by the wrong user actions. A user error reported for a randomly generated query is normal (e.g. it references some non-existent columns), but when we see an internal program error, we know that it's definitely a bug, same as an assertion. Of course, even without assertions, you get some checks for memory errors provided by the OS (segfaults). Various kinds of sanitizers are also very useful in conjunction with fuzzing. We run this fuzzer under clang's Address, Memory, UndefinedBehavior and Thread sanitizers, as we do for most of our tests. +A major factor that makes fuzzing really efficient for us is that we have a lot of assertions and other checks of program logic in our code. For debug-only checks, we use the plain `assert` macro from ``. For checks that are needed even in release mode, we use an exception with a special code `LOGICAL_ERROR` that signifies an internal program error. We did some work to ensure that these errors are distinct from errors caused by the wrong user actions. A user error reported for a randomly generated query is normal (e.g. it references some non-existent columns), but when we see an internal program error, we know that it's definitely a bug, same as an assertion. Of course, even without assertions, you get some checks for memory errors provided by the OS (segfaults). Another way to add runtime checks to your program is to use some kind of sanitizer. We already run most of our tests under clang's Address, Memory, UndefinedBehavior and Thread sanitizers. Using them in conjunction with this fuzzer also proved to be very efficient. -To see for yourself how it works, you only need the normal ClickHouse client. Start `clickhouse-client --query-fuzzer-runs=100`, enter any query, and enjoy the client going crazy and running a hundred of random queries instead. All queries from the current session become a source for expressions for fuzzing, so try entering several different queries to get more interesting results. Be careful not to do this in production! When you do this experiment, you'll soon notice that the fuzzer tends to generate queries that take very long to run. This is why for the CI fuzzer runs we have to configure the server to limit query execution time, memory usage and so on using the corresponding [server settings](https://clickhouse.tech/docs/en/operations/settings/query-complexity/#:~:text=In%20the%20default%20configuration%20file,query%20within%20a%20single%20server.). We had a hilarious situation after that: the fuzzer figured out how to remove the limits by generating a `SET max_execution_time = 0` query, and then generated a never-ending query and failed. Thankfully we were able to defeat its cleverness by using [settings constraints](https://clickhouse.tech/docs/en/operations/settings/constraints-on-settings/). +To see for yourself how it works, you only need the normal ClickHouse client. Start `clickhouse-client --query-fuzzer-runs=100`, enter any query, and enjoy the client going crazy and running a hundred of random queries instead. All queries from the current session become a source for expressions for fuzzing, so try entering several different queries to get more interesting results. Be careful not to do this in production! When you do this experiment, you'll soon notice that the fuzzer tends to generate queries that take very long to run. This is why for the CI fuzzer runs we have to configure the server to limit query execution time, memory usage and so on using the corresponding [server settings](https://clickhouse.tech/docs/en/operations/settings/query-complexity/#:~:text=In%20the%20default%20configuration%20file,query%20within%20a%20single%20server.). We had a hilarious situation after that: the fuzzer figured out how to remove the limits by generating a `SET max_execution_time = 0` query, and then generated a never-ending query and failed. Thankfully we were able to defeat its cleverness by using [settings constraints](https://clickhouse.tech/docs/en/operations/settings/constraints-on-settings/). ## Other Fuzzers From 2827042787cb88ad1901f3f22f7fc7c8bcd4c3cc Mon Sep 17 00:00:00 2001 From: Alexander Kuzmenkov <36882414+akuzm@users.noreply.github.com> Date: Fri, 12 Mar 2021 02:34:48 +0300 Subject: [PATCH 349/544] Update fuzzing-clickhouse.md --- website/blog/en/2021/fuzzing-clickhouse.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/website/blog/en/2021/fuzzing-clickhouse.md b/website/blog/en/2021/fuzzing-clickhouse.md index 47146c38d50..b6852dcce15 100644 --- a/website/blog/en/2021/fuzzing-clickhouse.md +++ b/website/blog/en/2021/fuzzing-clickhouse.md @@ -24,7 +24,7 @@ Consider some SQL query from a regression test. After parsing, it is easy to mut Another interesting thing we can do is change the arguments of functions, or the list of expressions in `SELECT`, `ORDER BY` and so on. Naturally, all the interesting arguments can be taken from other test queries. Same goes for changing the tables used in the queries. When the fuzzer runs in CI, it runs queries from all the SQL tests in random order, mixing into them some parts of queries it has seen previously. This process can eventually cover all the possible permutations of our features. -The core implementation of the fuzzer is relatively small, consisting of about 700 lines of C++ code. A prototype was made in a couple of days, but naturally it took significantly longer to polish it and to start routinely using it in CI. It is very productive and let us find more than 200 bugs already (see the label [fuzz](https://github.com/ClickHouse/ClickHouse/labels/fuzz) on GitHub), some of which are serious logic errors or even memory errors. When we only started, we could segfault the server or make it enter a never-ending loop with simplest read-only queries such as `SELECT arrayReverseFill(x -> (x < 10), [])` or `SELECT geoDistance(0., 0., -inf, 1.)`. Of course I couldn't resist bringing down our [public playground](https://gh-api.clickhouse.tech/play?user=play#LS0gWW91IGNhbiBxdWVyeSB0aGUgR2l0SHViIGhpc3RvcnkgZGF0YSBoZXJlLiBTZWUgaHR0cHM6Ly9naC5jbGlja2hvdXNlLnRlY2gvZXhwbG9yZXIvIGZvciB0aGUgZGVzY3JpcHRpb24gYW5kIGV4YW1wbGUgcXVlcmllcy4Kc2VsZWN0ICdoZWxsbyB3b3JsZCc=) with some of these queries, and was content to see that the server soon restarts correctly. These queries are actually minified by hand, normally the fuzzer would generate something barely legible such as: +The core implementation of the fuzzer is relatively small, consisting of about 700 lines of C++ code. A prototype was made in a couple of days, but naturally it took significantly longer to polish it and to start routinely using it in CI. It is very productive and let us find more than 200 bugs already (see the label [fuzz](https://github.com/ClickHouse/ClickHouse/labels/fuzz) on GitHub), some of which are serious logic errors or even memory errors. When we only started, we could segfault the server or make it enter a never-ending loop with simplest read-only queries such as `SELECT arrayReverseFill(x -> (x < 10), [])` or `SELECT geoDistance(0., 0., -inf, 1.)`. Of course I couldn't resist bringing down our [public playground](https://gh-api.clickhouse.tech/play?user=play#LS0gWW91IGNhbiBxdWVyeSB0aGUgR2l0SHViIGhpc3RvcnkgZGF0YSBoZXJlLiBTZWUgaHR0cHM6Ly9naC5jbGlja2hvdXNlLnRlY2gvZXhwbG9yZXIvIGZvciB0aGUgZGVzY3JpcHRpb24gYW5kIGV4YW1wbGUgcXVlcmllcy4Kc2VsZWN0ICdoZWxsbyB3b3JsZCc=) with some of these queries, and was content to see that the server soon restarts correctly. These queries are actually minified by hand, normally the fuzzer would generate something barely intelligible such as: ``` SELECT (val + 257, @@ -51,7 +51,7 @@ After fixing the majority of pre-existing error, this fuzzer became efficient fo A major factor that makes fuzzing really efficient for us is that we have a lot of assertions and other checks of program logic in our code. For debug-only checks, we use the plain `assert` macro from ``. For checks that are needed even in release mode, we use an exception with a special code `LOGICAL_ERROR` that signifies an internal program error. We did some work to ensure that these errors are distinct from errors caused by the wrong user actions. A user error reported for a randomly generated query is normal (e.g. it references some non-existent columns), but when we see an internal program error, we know that it's definitely a bug, same as an assertion. Of course, even without assertions, you get some checks for memory errors provided by the OS (segfaults). Another way to add runtime checks to your program is to use some kind of sanitizer. We already run most of our tests under clang's Address, Memory, UndefinedBehavior and Thread sanitizers. Using them in conjunction with this fuzzer also proved to be very efficient. -To see for yourself how it works, you only need the normal ClickHouse client. Start `clickhouse-client --query-fuzzer-runs=100`, enter any query, and enjoy the client going crazy and running a hundred of random queries instead. All queries from the current session become a source for expressions for fuzzing, so try entering several different queries to get more interesting results. Be careful not to do this in production! When you do this experiment, you'll soon notice that the fuzzer tends to generate queries that take very long to run. This is why for the CI fuzzer runs we have to configure the server to limit query execution time, memory usage and so on using the corresponding [server settings](https://clickhouse.tech/docs/en/operations/settings/query-complexity/#:~:text=In%20the%20default%20configuration%20file,query%20within%20a%20single%20server.). We had a hilarious situation after that: the fuzzer figured out how to remove the limits by generating a `SET max_execution_time = 0` query, and then generated a never-ending query and failed. Thankfully we were able to defeat its cleverness by using [settings constraints](https://clickhouse.tech/docs/en/operations/settings/constraints-on-settings/). +To see for yourself how the fuzzer works, you only need the normal ClickHouse client. Start `clickhouse-client --query-fuzzer-runs=100`, enter any query, and enjoy the client going crazy and running a hundred of random queries instead. All queries from the current session become a source for expressions for fuzzing, so try entering several different queries to get more interesting results. Be careful not to do this in production! When you do this experiment, you'll soon notice that the fuzzer tends to generate queries that take very long to run. This is why for the CI fuzzer runs we have to configure the server to limit query execution time, memory usage and so on using the corresponding [server settings](https://clickhouse.tech/docs/en/operations/settings/query-complexity/#:~:text=In%20the%20default%20configuration%20file,query%20within%20a%20single%20server.). We had a hilarious situation after that: the fuzzer figured out how to remove the limits by generating a `SET max_execution_time = 0` query, and then generated a never-ending query and failed. Thankfully we were able to defeat its cleverness by using [settings constraints](https://clickhouse.tech/docs/en/operations/settings/constraints-on-settings/). ## Other Fuzzers From 183868dfcb133ec521f3b6ce4b11ddabc244bac6 Mon Sep 17 00:00:00 2001 From: olgarev <56617294+olgarev@users.noreply.github.com> Date: Fri, 12 Mar 2021 05:12:28 +0300 Subject: [PATCH 350/544] Apply suggestions from code review Co-authored-by: Nikita Mikhaylov --- docs/en/sql-reference/functions/math-functions.md | 2 +- .../table-engines/mergetree-family/mergetree.md | 10 +++++----- docs/ru/sql-reference/functions/math-functions.md | 8 ++++---- 3 files changed, 10 insertions(+), 10 deletions(-) diff --git a/docs/en/sql-reference/functions/math-functions.md b/docs/en/sql-reference/functions/math-functions.md index eceffae6b97..bfe973e3d96 100644 --- a/docs/en/sql-reference/functions/math-functions.md +++ b/docs/en/sql-reference/functions/math-functions.md @@ -415,7 +415,7 @@ Result: ## sign(x) {#signx} -Extracts the sign of a real number. +Returns the sign of a real number. **Syntax** diff --git a/docs/ru/engines/table-engines/mergetree-family/mergetree.md b/docs/ru/engines/table-engines/mergetree-family/mergetree.md index cb633aa1e8a..9cf44ae5e34 100644 --- a/docs/ru/engines/table-engines/mergetree-family/mergetree.md +++ b/docs/ru/engines/table-engines/mergetree-family/mergetree.md @@ -691,7 +691,7 @@ SETTINGS storage_policy = 'moving_from_ssd_to_hdd' ## Использование сервиса S3 для хранения данных {#table_engine-mergetree-s3} -Таблицы семейства `MergeTree` могут использовать для хранения данных сервиса [Amazon S3](https://aws.amazon.com/s3/) с диском типа `s3`. +Таблицы семейства `MergeTree` могут хранить данные в сервисе [S3](https://aws.amazon.com/s3/) при использовании диска типа `s3`. Конфигурация: @@ -725,7 +725,7 @@ SETTINGS storage_policy = 'moving_from_ssd_to_hdd' Обязательные параметры: -- `endpoint` — URL конечной точки S3 в запросах `path` или `virtual hosted` [styles](https://docs.aws.amazon.com/AmazonS3/latest/dev/VirtualHosting.html). URL конечной точки должен содержать бакет и путь к корневой директории на сервере, где хранятся данные. +- `endpoint` — URL точки приема запроса на стороне S3 в [форматах](https://docs.aws.amazon.com/AmazonS3/latest/userguide/VirtualHosting.html) `path` или `virtual hosted`. URL точки должен содержать бакет и путь к корневой директории на сервере, где хранятся данные. - `access_key_id` — id ключа доступа к S3. - `secret_access_key` — секретный ключ доступа к S3. @@ -739,9 +739,9 @@ SETTINGS storage_policy = 'moving_from_ssd_to_hdd' - `retry_attempts` — число попыток выполнения запроса в случае возникновения ошибки. Значение по умолчанию: `10`. - `min_bytes_for_seek` — минимальное количество байтов, которые используются для операций поиска вместо последовательного чтения. Значение по умолчанию: 1 МБайт. - `metadata_path` — путь к локальному файловому хранилищу для хранения файлов с метаданными для S3. Значение по умолчанию: `/var/lib/clickhouse/disks//`. -- `cache_enabled` — признак, разрешено ли хранение меток кеша и индексных файлов в локальном файловом хранилище. Значение по умолчанию: `true`. -- `cache_path` — путь к локальному файловому хранилищу, где хранятся метки кеша и индексные файлы. Значение по умолчанию: `/var/lib/clickhouse/disks//cache/`. -- `skip_access_check` — признак, выполнять ли проверку доступа к диску при запуске диска. Если установлено значение `true`, то проверка не выполняется. Значение по умолчанию: `false`. +- `cache_enabled` — признак, разрешено ли хранение кэша засечек и файлов индекса в локальной файловой системе. Значение по умолчанию: `true`. +- `cache_path` — путь в локальной файловой системе, где будут храниться кэш засечек и файлы индекса. Значение по умолчанию: `/var/lib/clickhouse/disks//cache/`. +- `skip_access_check` — признак, выполнять ли проверку доступов при запуске диска. Если установлено значение `true`, то проверка не выполняется. Значение по умолчанию: `false`. Диск S3 может быть сконфигурирован как `main` или `cold`: diff --git a/docs/ru/sql-reference/functions/math-functions.md b/docs/ru/sql-reference/functions/math-functions.md index 9748cbf7b28..aa6f1f6813b 100644 --- a/docs/ru/sql-reference/functions/math-functions.md +++ b/docs/ru/sql-reference/functions/math-functions.md @@ -407,7 +407,7 @@ SELECT log1p(0); ## sign(x) {#signx} -Выделяет знак действительного числа. +Возвращает знак действительного числа. **Синтаксис** @@ -427,7 +427,7 @@ sign(x) **Примеры** -Знак для нулевого значения: +Результат sign() для нуля: ``` sql SELECT sign(0); @@ -440,7 +440,7 @@ SELECT sign(0); └─────────┘ ``` -Знак для положительного значения: +Результат sign() для положительного аргумента: ``` sql SELECT sign(1); @@ -454,7 +454,7 @@ SELECT sign(1); └─────────┘ ``` -Знак для отрицательного значения: +Результат sign() для отрицательного аргумента: ``` sql SELECT sign(-1); From 8cf08281171e1eb23679afdc087b99c2b89cf8c2 Mon Sep 17 00:00:00 2001 From: Xiang Zhou Date: Fri, 12 Mar 2021 10:48:23 +0800 Subject: [PATCH 351/544] Update cloud doc zh. --- docs/zh/commercial/cloud.md | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/docs/zh/commercial/cloud.md b/docs/zh/commercial/cloud.md index 9cca8776d14..da08895c56e 100644 --- a/docs/zh/commercial/cloud.md +++ b/docs/zh/commercial/cloud.md @@ -29,6 +29,18 @@ toc_title: 云 - 跨可用区扩展以实现性能和高可用性 - 内置监控和SQL查询编辑器 +## 阿里云 {#alibaba-cloud} + +阿里云的 ClickHouse 托管服务 [中国站](https://www.aliyun.com/product/clickhouse) [国际站 (2021年5月初)] 提供以下主要功能: + +- 高可靠存储引擎:基于阿里飞天分布式系统的云盘存储,充分利用云平台存储可靠性特性 +- 扩展性强:提供数据库扩容管理模块,能便捷地按需对集群进行扩容,无需手动进行数据搬迁 +- 架构灵活:支持单节点、单副本、多节点、多副本多种架构,支持冷热数据分层 +- 数据安全:访问白名单和一键恢复,多层网络安全防护,云盘加密 +- 数据闭环流通:与云上日志系统、数据库、数据应用工具无缝集成,能便捷地进行数据同步,降低数据迁移工作量 +- 便捷运维:专业监控和数据库管理平台 +- 专业服务能力:阿里云提供专业的数据库专家,给用户提供技术支持和服务 + ## 腾讯云 {#tencent-cloud} [腾讯云的 ClickHouse 托管服务](https://cloud.tencent.com/product/cdwch)提供以下主要功能: From 56d615ea15ea269fdac8d4080d0adbb9a5d27dec Mon Sep 17 00:00:00 2001 From: Xiang Zhou Date: Fri, 12 Mar 2021 11:16:02 +0800 Subject: [PATCH 352/544] Add cloud MD in zh and en. --- docs/en/commercial/cloud.md | 11 +++++++++++ docs/zh/commercial/cloud.md | 16 ++++++++-------- 2 files changed, 19 insertions(+), 8 deletions(-) diff --git a/docs/en/commercial/cloud.md b/docs/en/commercial/cloud.md index 0490881c622..85f6ae7284c 100644 --- a/docs/en/commercial/cloud.md +++ b/docs/en/commercial/cloud.md @@ -29,6 +29,17 @@ toc_title: Cloud - Cross-AZ scaling for performance and high availability - Built-in monitoring and SQL query editor +## Alibaba Cloud {#alibaba-cloud} + +Alibaba Cloud Managed Service for ClickHouse [China Site](https://www.aliyun.com/product/clickhouse) (Will be available at international site at May, 2021) provides the following key features: +- Highly reliable cloud disk storage engine based on Alibaba Cloud Apsara distributed system +- Expand capacity on demand without manual data migration +- Support single-node, single-replica, multi-node, and multi-replica architectures, and support hot and cold data tiering +- Support access whitelist, one-key recovery, multi-layer network security protection, cloud disk encryption +- Seamless integration with cloud log systems, databases, and data application tools +- Built-in monitoring and database management platform +- Professional database expert technical support and service + ## Tencent Cloud {#tencent-cloud} [Tencent Managed Service for ClickHouse](https://cloud.tencent.com/product/cdwch) provides the following key features: diff --git a/docs/zh/commercial/cloud.md b/docs/zh/commercial/cloud.md index da08895c56e..c74ffa93e9a 100644 --- a/docs/zh/commercial/cloud.md +++ b/docs/zh/commercial/cloud.md @@ -31,15 +31,15 @@ toc_title: 云 ## 阿里云 {#alibaba-cloud} -阿里云的 ClickHouse 托管服务 [中国站](https://www.aliyun.com/product/clickhouse) [国际站 (2021年5月初)] 提供以下主要功能: +阿里云的 ClickHouse 托管服务 [中国站](https://www.aliyun.com/product/clickhouse) (国际站于2021年5月初开放) 提供以下主要功能: -- 高可靠存储引擎:基于阿里飞天分布式系统的云盘存储,充分利用云平台存储可靠性特性 -- 扩展性强:提供数据库扩容管理模块,能便捷地按需对集群进行扩容,无需手动进行数据搬迁 -- 架构灵活:支持单节点、单副本、多节点、多副本多种架构,支持冷热数据分层 -- 数据安全:访问白名单和一键恢复,多层网络安全防护,云盘加密 -- 数据闭环流通:与云上日志系统、数据库、数据应用工具无缝集成,能便捷地进行数据同步,降低数据迁移工作量 -- 便捷运维:专业监控和数据库管理平台 -- 专业服务能力:阿里云提供专业的数据库专家,给用户提供技术支持和服务 +- 基于阿里飞天分布式系统的高可靠云盘存储引擎 +- 按需扩容,无需手动进行数据搬迁 +- 支持单节点、单副本、多节点、多副本多种架构,支持冷热数据分层 +- 支持访问白名单和一键恢复,多层网络安全防护,云盘加密 +- 与云上日志系统、数据库、数据应用工具无缝集成 +- 内置监控和数据库管理平台 +- 专业的数据库专家技术支持和服务 ## 腾讯云 {#tencent-cloud} From ceae3f841cc1c856b0c9b328a05745e4347689f7 Mon Sep 17 00:00:00 2001 From: Xiang Zhou Date: Fri, 12 Mar 2021 11:30:47 +0800 Subject: [PATCH 353/544] Add cloud MD in jp. --- docs/ja/commercial/cloud.md | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/docs/ja/commercial/cloud.md b/docs/ja/commercial/cloud.md index 403b34d198c..84f58e46cdb 100644 --- a/docs/ja/commercial/cloud.md +++ b/docs/ja/commercial/cloud.md @@ -20,4 +20,16 @@ toc_title: "\u30AF\u30E9\u30A6\u30C9" - 暗号化と分離 - 自動メンテナンス +## Alibaba Cloud {#alibaba-cloud} + +ClickHouseのためのAlibaba Cloudの管理サービス [中国サイト](https://www.aliyun.com/product/clickhouse) (2021年5月に国際サイトで利用可能になります) 次の主な機能を提供します: + +- Alibaba Cloud Apsara分散システムをベースにした信頼性の高いクラウドディスクストレージエンジン +- 手動でのデータ移行を必要とせずに、オン・デマンドで容量を拡張 +- シングル・ノード、シングル・レプリカ、マルチ・ノード、マルチ・レプリカ・アーキテクチャをサポートし、ホット・データとコールド・データの階層化をサポート +- アクセスホワイトリスト、OneKey Recovery、マルチレイヤーネットワークセキュリティ保護、クラウドディスク暗号化をサポート +- クラウドログシステム、データベース、およびデータアプリケーションツールとのシームレスな統合 +- 組み込み型の監視およびデータベース管理プラットフォーム +- プロフェッショナルデータベースエキスパートによるテクニカル・サポートとサービス + {## [元の記事](https://clickhouse.tech/docs/en/commercial/cloud/) ##} From 7a0ca6dddd4d874ed92dbe62fc220000a29e5676 Mon Sep 17 00:00:00 2001 From: alexey-milovidov Date: Fri, 12 Mar 2021 08:36:03 +0300 Subject: [PATCH 354/544] Update cloud.md --- docs/en/commercial/cloud.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/en/commercial/cloud.md b/docs/en/commercial/cloud.md index 85f6ae7284c..91d2061c0af 100644 --- a/docs/en/commercial/cloud.md +++ b/docs/en/commercial/cloud.md @@ -35,7 +35,7 @@ Alibaba Cloud Managed Service for ClickHouse [China Site](https://www.aliyun.com - Highly reliable cloud disk storage engine based on Alibaba Cloud Apsara distributed system - Expand capacity on demand without manual data migration - Support single-node, single-replica, multi-node, and multi-replica architectures, and support hot and cold data tiering -- Support access whitelist, one-key recovery, multi-layer network security protection, cloud disk encryption +- Support access allow-list, one-key recovery, multi-layer network security protection, cloud disk encryption - Seamless integration with cloud log systems, databases, and data application tools - Built-in monitoring and database management platform - Professional database expert technical support and service From b030bfedd7db4603034643fd183cf7784a820193 Mon Sep 17 00:00:00 2001 From: alexey-milovidov Date: Fri, 12 Mar 2021 11:34:55 +0300 Subject: [PATCH 355/544] Update CHANGELOG.md --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 5d9b901999d..e697b65b331 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,7 +5,7 @@ #### Backward Incompatible Change * Now it's not allowed to create MergeTree tables in old syntax with table TTL because it's just ignored. Attach of old tables is still possible. [#20282](https://github.com/ClickHouse/ClickHouse/pull/20282) ([alesapin](https://github.com/alesapin)). -* Now all case-insensitive function names will be lower-cased during query analysis. This is needed for projection query routing. [#20174](https://github.com/ClickHouse/ClickHouse/pull/20174) ([Amos Bird](https://github.com/amosbird)). +* Now all case-insensitive function names will be rewritten to their canonical representations. This is needed for projection query routing (the upcoming feature). [#20174](https://github.com/ClickHouse/ClickHouse/pull/20174) ([Amos Bird](https://github.com/amosbird)). * Fix creation of `TTL` in cases, when its expression is a function and it is the same as `ORDER BY` key. Now it's allowed to set custom aggregation to primary key columns in `TTL` with `GROUP BY`. Backward incompatible: For primary key columns, which are not in `GROUP BY` and aren't set explicitly now is applied function `any` instead of `max`, when TTL is expired. Also if you use TTL with `WHERE` or `GROUP BY` you can see exceptions at merges, while making rolling update. [#15450](https://github.com/ClickHouse/ClickHouse/pull/15450) ([Anton Popov](https://github.com/CurtizJ)). #### New Feature From 4f35cbf937e8755990426378f518e2359b1f7a0d Mon Sep 17 00:00:00 2001 From: alexey-milovidov Date: Fri, 12 Mar 2021 11:37:56 +0300 Subject: [PATCH 356/544] Update CHANGELOG.md --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index e697b65b331..a27cb1840c0 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -87,6 +87,7 @@ #### Bug Fix +* Fix redundant reconnects to ZooKeeper and the possibility of two active sessions for a single clickhouse server. Both problems introduced in #14678. [#21264](https://github.com/ClickHouse/ClickHouse/pull/21264) ([alesapin](https://github.com/alesapin)). * Fix a deadlock in `ALTER DELETE` mutations for non replicated MergeTree table engines when the predicate contains the table itself. Fixes [#20558](https://github.com/ClickHouse/ClickHouse/issues/20558). [#21477](https://github.com/ClickHouse/ClickHouse/pull/21477) ([alesapin](https://github.com/alesapin)). * Fix SIGSEGV for distributed queries on failures. [#21434](https://github.com/ClickHouse/ClickHouse/pull/21434) ([Azat Khuzhin](https://github.com/azat)). * Now `ALTER MODIFY COLUMN` queries will correctly affect changes in partition key, skip indices, TTLs, and so on. Fixes [#13675](https://github.com/ClickHouse/ClickHouse/issues/13675). [#21334](https://github.com/ClickHouse/ClickHouse/pull/21334) ([alesapin](https://github.com/alesapin)). From 87bce6dfa7e655cfcb55637e67e53d54769d3887 Mon Sep 17 00:00:00 2001 From: alexey-milovidov Date: Fri, 12 Mar 2021 11:39:04 +0300 Subject: [PATCH 357/544] Update CHANGELOG.md --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index a27cb1840c0..9ffaeccd1ea 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -88,6 +88,7 @@ #### Bug Fix * Fix redundant reconnects to ZooKeeper and the possibility of two active sessions for a single clickhouse server. Both problems introduced in #14678. [#21264](https://github.com/ClickHouse/ClickHouse/pull/21264) ([alesapin](https://github.com/alesapin)). +* Fix error `Bad cast from type ... to DB::ColumnLowCardinality` while inserting into table with `LowCardinality` column from `Values` format. Fixes #21140 [#21357](https://github.com/ClickHouse/ClickHouse/pull/21357) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). * Fix a deadlock in `ALTER DELETE` mutations for non replicated MergeTree table engines when the predicate contains the table itself. Fixes [#20558](https://github.com/ClickHouse/ClickHouse/issues/20558). [#21477](https://github.com/ClickHouse/ClickHouse/pull/21477) ([alesapin](https://github.com/alesapin)). * Fix SIGSEGV for distributed queries on failures. [#21434](https://github.com/ClickHouse/ClickHouse/pull/21434) ([Azat Khuzhin](https://github.com/azat)). * Now `ALTER MODIFY COLUMN` queries will correctly affect changes in partition key, skip indices, TTLs, and so on. Fixes [#13675](https://github.com/ClickHouse/ClickHouse/issues/13675). [#21334](https://github.com/ClickHouse/ClickHouse/pull/21334) ([alesapin](https://github.com/alesapin)). From bf933cf39da7b9cf7f09bb3e82b6e114d546fb95 Mon Sep 17 00:00:00 2001 From: alexey-milovidov Date: Fri, 12 Mar 2021 11:39:58 +0300 Subject: [PATCH 358/544] Update CHANGELOG.md --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 9ffaeccd1ea..fb3bceb4048 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -50,6 +50,7 @@ #### Improvement +* Case-insensitive compression methods for table functions. Also fixed LZMA compression method which was checked in upper case. [#21416](https://github.com/ClickHouse/ClickHouse/pull/21416) ([Vladimir Chebotarev](https://github.com/excitoon)). * Add two settings to delay or throw error during insertion when there are too many inactive parts. This is useful when server fails to clean up parts quickly enough. [#20178](https://github.com/ClickHouse/ClickHouse/pull/20178) ([Amos Bird](https://github.com/amosbird)). * Provide better compatibility for mysql clients. 1. mysql jdbc 2. mycli. [#21367](https://github.com/ClickHouse/ClickHouse/pull/21367) ([Amos Bird](https://github.com/amosbird)). * Forbid to drop a column if it's referenced by materialized view. Closes [#21164](https://github.com/ClickHouse/ClickHouse/issues/21164). [#21303](https://github.com/ClickHouse/ClickHouse/pull/21303) ([flynn](https://github.com/ucasFL)). From 59e38d77fdd19820f9c46fa48ecef289c86303b6 Mon Sep 17 00:00:00 2001 From: alexey-milovidov Date: Fri, 12 Mar 2021 11:41:20 +0300 Subject: [PATCH 359/544] Update CHANGELOG.md --- CHANGELOG.md | 1 - 1 file changed, 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index fb3bceb4048..928991dc937 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -43,7 +43,6 @@ * Add back `intDiv`/`modulo` specializations for better performance. This fixes [#21293](https://github.com/ClickHouse/ClickHouse/issues/21293) . The regression was introduced in https://github.com/ClickHouse/ClickHouse/pull/18145 . [#21307](https://github.com/ClickHouse/ClickHouse/pull/21307) ([Amos Bird](https://github.com/amosbird)). * Do not squash blocks too much on INSERT SELECT if inserting into Memory table. In previous versions inefficient data representation was created in Memory table after INSERT SELECT. This closes [#13052](https://github.com/ClickHouse/ClickHouse/issues/13052). [#20169](https://github.com/ClickHouse/ClickHouse/pull/20169) ([alexey-milovidov](https://github.com/alexey-milovidov)). * Fix at least one case when DataType parser may have exponential complexity (found by fuzzer). This closes [#20096](https://github.com/ClickHouse/ClickHouse/issues/20096). [#20132](https://github.com/ClickHouse/ClickHouse/pull/20132) ([alexey-milovidov](https://github.com/alexey-milovidov)). -* Improved performance of bitmap columns during joins. [#19407](https://github.com/ClickHouse/ClickHouse/pull/19407) ([templarzq](https://github.com/templarzq)). * Parallelize SELECT with FINAL for single part with level > 0 when `do_not_merge_across_partitions_select_final` setting is 1. [#19375](https://github.com/ClickHouse/ClickHouse/pull/19375) ([Kruglov Pavel](https://github.com/Avogar)). * Fill only requested columns when querying `system.parts` and `system.parts_columns`. Closes [#19570](https://github.com/ClickHouse/ClickHouse/issues/19570). [#21035](https://github.com/ClickHouse/ClickHouse/pull/21035) ([Anmol Arora](https://github.com/anmolarora)). * Perform algebraic optimizations of arithmetic expressions inside `avg` aggregate function. close [#20092](https://github.com/ClickHouse/ClickHouse/issues/20092). [#20183](https://github.com/ClickHouse/ClickHouse/pull/20183) ([flynn](https://github.com/ucasFL)). From 1688b1a8ada6284cf112e67204d9adcd8043e69c Mon Sep 17 00:00:00 2001 From: Roman Bug Date: Fri, 12 Mar 2021 11:49:32 +0300 Subject: [PATCH 360/544] Update docs/ru/sql-reference/aggregate-functions/reference/avg.md Co-authored-by: olgarev <56617294+olgarev@users.noreply.github.com> --- docs/ru/sql-reference/aggregate-functions/reference/avg.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/ru/sql-reference/aggregate-functions/reference/avg.md b/docs/ru/sql-reference/aggregate-functions/reference/avg.md index 7f9ba6c4e6d..8027674e68c 100644 --- a/docs/ru/sql-reference/aggregate-functions/reference/avg.md +++ b/docs/ru/sql-reference/aggregate-functions/reference/avg.md @@ -12,7 +12,7 @@ toc_priority: 5 avg(x) ``` -**Параметры** +**Аргументы** - `x` — входное значение в формате [Integer](../../../sql-reference/data-types/int-uint.md), [Float](../../../sql-reference/data-types/float.md) или [Decimal](../../../sql-reference/data-types/decimal.md). From 637aacdaa424b89264afd07a246b418fd2157573 Mon Sep 17 00:00:00 2001 From: Roman Bug Date: Fri, 12 Mar 2021 11:49:38 +0300 Subject: [PATCH 361/544] Update docs/en/sql-reference/aggregate-functions/reference/avg.md Co-authored-by: olgarev <56617294+olgarev@users.noreply.github.com> --- docs/en/sql-reference/aggregate-functions/reference/avg.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/en/sql-reference/aggregate-functions/reference/avg.md b/docs/en/sql-reference/aggregate-functions/reference/avg.md index b9aebab2233..cbd409ccab6 100644 --- a/docs/en/sql-reference/aggregate-functions/reference/avg.md +++ b/docs/en/sql-reference/aggregate-functions/reference/avg.md @@ -63,4 +63,4 @@ Result: └────────┘ ``` -[Оригинальная статья](https://clickhouse.tech/docs/en/sql-reference/aggregate-functions/reference/avg/) +[Original article](https://clickhouse.tech/docs/en/sql-reference/aggregate-functions/reference/avg/) From 2cf39d3ac4f6e9106cc39bca6d8ca27daaf796c9 Mon Sep 17 00:00:00 2001 From: Vasily Nemkov Date: Fri, 12 Mar 2021 11:25:01 +0200 Subject: [PATCH 362/544] Excluded 01702_system_query_log from arcadia runs --- tests/queries/0_stateless/arcadia_skip_list.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/queries/0_stateless/arcadia_skip_list.txt b/tests/queries/0_stateless/arcadia_skip_list.txt index 6926f16e027..30a2ad33e3e 100644 --- a/tests/queries/0_stateless/arcadia_skip_list.txt +++ b/tests/queries/0_stateless/arcadia_skip_list.txt @@ -218,3 +218,4 @@ 01682_cache_dictionary_complex_key 01684_ssd_cache_dictionary_simple_key 01685_ssd_cache_dictionary_complex_key +01702_system_query_log From 339c12f510dca633d663d8fde6e3f18729c9e7ce Mon Sep 17 00:00:00 2001 From: Nikita Mikhaylov Date: Fri, 12 Mar 2021 12:39:40 +0300 Subject: [PATCH 363/544] suppress boost geometry UBSan --- tests/ubsan_suppressions.txt | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/ubsan_suppressions.txt b/tests/ubsan_suppressions.txt index b92a2d9730e..8d10b4f73dd 100644 --- a/tests/ubsan_suppressions.txt +++ b/tests/ubsan_suppressions.txt @@ -1,4 +1,5 @@ # https://github.com/llvm-mirror/compiler-rt/blob/master/lib/ubsan/ubsan_checks.inc # Some value is outside the range of representable values of type 'long' on user-provided data inside boost::geometry - ignore. -src:*/Functions/pointInPolygon.cpp \ No newline at end of file +src:*/Functions/pointInPolygon.cpp +src:*/contrib/boost/boost/geometry/* From 0eba776fa605d9d302b555629d6a111e8b2dd301 Mon Sep 17 00:00:00 2001 From: vdimir Date: Fri, 12 Mar 2021 12:56:43 +0300 Subject: [PATCH 364/544] Fix ambigous column error in joins_in_memory --- src/Interpreters/CrossToInnerJoinVisitor.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Interpreters/CrossToInnerJoinVisitor.cpp b/src/Interpreters/CrossToInnerJoinVisitor.cpp index c4d330831bb..2fcd75b1f23 100644 --- a/src/Interpreters/CrossToInnerJoinVisitor.cpp +++ b/src/Interpreters/CrossToInnerJoinVisitor.cpp @@ -136,7 +136,7 @@ std::optional getIdentMembership(const ASTIdentifier & ident, const std: std::optional table_pos = IdentifierSemantic::getMembership(ident); if (table_pos) return table_pos; - return IdentifierSemantic::chooseTableColumnMatch(ident, tables); + return IdentifierSemantic::chooseTableColumnMatch(ident, tables, true); } std::optional getIdentsMembership(const ASTPtr ast, From 58264321725bd6b27088b2a5affec19d3b4006a8 Mon Sep 17 00:00:00 2001 From: Olga Revyakina Date: Fri, 12 Mar 2021 13:00:46 +0300 Subject: [PATCH 365/544] Headers order changed --- .../mergetree-family/mergetree.md | 52 +++++++++---------- .../mergetree-family/mergetree.md | 45 ++++++++-------- 2 files changed, 49 insertions(+), 48 deletions(-) diff --git a/docs/en/engines/table-engines/mergetree-family/mergetree.md b/docs/en/engines/table-engines/mergetree-family/mergetree.md index 753859b46d2..52d9111dc90 100644 --- a/docs/en/engines/table-engines/mergetree-family/mergetree.md +++ b/docs/en/engines/table-engines/mergetree-family/mergetree.md @@ -701,6 +701,32 @@ The `default` storage policy implies using only one volume, which consists of on The number of threads performing background moves of data parts can be changed by [background_move_pool_size](../../../operations/settings/settings.md#background_move_pool_size) setting. +### Details {#details} + +In the case of `MergeTree` tables, data is getting to disk in different ways: + +- As a result of an insert (`INSERT` query). +- During background merges and [mutations](../../../sql-reference/statements/alter/index.md#alter-mutations). +- When downloading from another replica. +- As a result of partition freezing [ALTER TABLE … FREEZE PARTITION](../../../sql-reference/statements/alter/partition.md#alter_freeze-partition). + +In all these cases except for mutations and partition freezing, a part is stored on a volume and a disk according to the given storage policy: + +1. The first volume (in the order of definition) that has enough disk space for storing a part (`unreserved_space > current_part_size`) and allows for storing parts of a given size (`max_data_part_size_bytes > current_part_size`) is chosen. +2. Within this volume, that disk is chosen that follows the one, which was used for storing the previous chunk of data, and that has free space more than the part size (`unreserved_space - keep_free_space_bytes > current_part_size`). + +Under the hood, mutations and partition freezing make use of [hard links](https://en.wikipedia.org/wiki/Hard_link). Hard links between different disks are not supported, therefore in such cases the resulting parts are stored on the same disks as the initial ones. + +In the background, parts are moved between volumes on the basis of the amount of free space (`move_factor` parameter) according to the order the volumes are declared in the configuration file. +Data is never transferred from the last one and into the first one. One may use system tables [system.part_log](../../../operations/system-tables/part_log.md#system_tables-part-log) (field `type = MOVE_PART`) and [system.parts](../../../operations/system-tables/parts.md#system_tables-parts) (fields `path` and `disk`) to monitor background moves. Also, the detailed information can be found in server logs. + +User can force moving a part or a partition from one volume to another using the query [ALTER TABLE … MOVE PART\|PARTITION … TO VOLUME\|DISK …](../../../sql-reference/statements/alter/partition.md#alter_move-partition), all the restrictions for background operations are taken into account. The query initiates a move on its own and does not wait for background operations to be completed. User will get an error message if not enough free space is available or if any of the required conditions are not met. + +Moving data does not interfere with data replication. Therefore, different storage policies can be specified for the same table on different replicas. + +After the completion of background merges and mutations, old parts are removed only after a certain amount of time (`old_parts_lifetime`). +During this time, they are not moved to other volumes or disks. Therefore, until the parts are finally removed, they are still taken into account for evaluation of the occupied disk space. + ## Using S3 for Data Storage {#table_engine-mergetree-s3} `MergeTree` family table engines is able to store data to [S3](https://aws.amazon.com/s3/) using a disk with type `s3`. @@ -793,30 +819,4 @@ S3 disk can be configured as `main` or `cold` storage: In case of `cold` option a data can be moved to S3 if local disk free size will be smaller than `move_factor * disk_size` or by TTL move rule. -### Details {#details} - -In the case of `MergeTree` tables, data is getting to disk in different ways: - -- As a result of an insert (`INSERT` query). -- During background merges and [mutations](../../../sql-reference/statements/alter/index.md#alter-mutations). -- When downloading from another replica. -- As a result of partition freezing [ALTER TABLE … FREEZE PARTITION](../../../sql-reference/statements/alter/partition.md#alter_freeze-partition). - -In all these cases except for mutations and partition freezing, a part is stored on a volume and a disk according to the given storage policy: - -1. The first volume (in the order of definition) that has enough disk space for storing a part (`unreserved_space > current_part_size`) and allows for storing parts of a given size (`max_data_part_size_bytes > current_part_size`) is chosen. -2. Within this volume, that disk is chosen that follows the one, which was used for storing the previous chunk of data, and that has free space more than the part size (`unreserved_space - keep_free_space_bytes > current_part_size`). - -Under the hood, mutations and partition freezing make use of [hard links](https://en.wikipedia.org/wiki/Hard_link). Hard links between different disks are not supported, therefore in such cases the resulting parts are stored on the same disks as the initial ones. - -In the background, parts are moved between volumes on the basis of the amount of free space (`move_factor` parameter) according to the order the volumes are declared in the configuration file. -Data is never transferred from the last one and into the first one. One may use system tables [system.part_log](../../../operations/system-tables/part_log.md#system_tables-part-log) (field `type = MOVE_PART`) and [system.parts](../../../operations/system-tables/parts.md#system_tables-parts) (fields `path` and `disk`) to monitor background moves. Also, the detailed information can be found in server logs. - -User can force moving a part or a partition from one volume to another using the query [ALTER TABLE … MOVE PART\|PARTITION … TO VOLUME\|DISK …](../../../sql-reference/statements/alter/partition.md#alter_move-partition), all the restrictions for background operations are taken into account. The query initiates a move on its own and does not wait for background operations to be completed. User will get an error message if not enough free space is available or if any of the required conditions are not met. - -Moving data does not interfere with data replication. Therefore, different storage policies can be specified for the same table on different replicas. - -After the completion of background merges and mutations, old parts are removed only after a certain amount of time (`old_parts_lifetime`). -During this time, they are not moved to other volumes or disks. Therefore, until the parts are finally removed, they are still taken into account for evaluation of the occupied disk space. - [Original article](https://clickhouse.tech/docs/ru/operations/table_engines/mergetree/) diff --git a/docs/ru/engines/table-engines/mergetree-family/mergetree.md b/docs/ru/engines/table-engines/mergetree-family/mergetree.md index cb633aa1e8a..b2f619564f0 100644 --- a/docs/ru/engines/table-engines/mergetree-family/mergetree.md +++ b/docs/ru/engines/table-engines/mergetree-family/mergetree.md @@ -689,6 +689,29 @@ SETTINGS storage_policy = 'moving_from_ssd_to_hdd' Количество потоков для фоновых перемещений кусков между дисками можно изменить с помощью настройки [background_move_pool_size](../../../operations/settings/settings.md#background_move_pool_size) +### Особенности работы {#details} + +В таблицах `MergeTree` данные попадают на диск несколькими способами: + +- В результате вставки (запрос `INSERT`). +- В фоновых операциях слияний и [мутаций](../../../sql-reference/statements/alter/index.md#mutations). +- При скачивании данных с другой реплики. +- В результате заморозки партиций [ALTER TABLE … FREEZE PARTITION](../../../engines/table-engines/mergetree-family/mergetree.md#alter_freeze-partition). + +Во всех случаях, кроме мутаций и заморозки партиций, при записи куска выбирается том и диск в соответствии с указанной конфигурацией хранилища: + +1. Выбирается первый по порядку том, на котором есть свободное место для записи куска (`unreserved_space > current_part_size`) и который позволяет записывать куски требуемого размера `max_data_part_size_bytes > current_part_size`. +2. Внутри тома выбирается следующий диск после того, на который была предыдущая запись и на котором свободного места больше чем размер куска (`unreserved_space - keep_free_space_bytes > current_part_size`) + +Мутации и запросы заморозки партиций в реализации используют [жесткие ссылки](https://ru.wikipedia.org/wiki/%D0%96%D1%91%D1%81%D1%82%D0%BA%D0%B0%D1%8F_%D1%81%D1%81%D1%8B%D0%BB%D0%BA%D0%B0). Жесткие ссылки между различными дисками не поддерживаются, поэтому в случае таких операций куски размещаются на тех же дисках, что и исходные. + +В фоне куски перемещаются между томами на основе информации о занятом месте (настройка `move_factor`) по порядку, в котором указаны тома в конфигурации. Данные никогда не перемещаются с последнего тома и на первый том. Следить за фоновыми перемещениями можно с помощью системных таблиц [system.part_log](../../../engines/table-engines/mergetree-family/mergetree.md#system_tables-part-log) (поле `type = MOVE_PART`) и [system.parts](../../../engines/table-engines/mergetree-family/mergetree.md#system_tables-parts) (поля `path` и `disk`). Также подробная информация о перемещениях доступна в логах сервера. +С помощью запроса [ALTER TABLE … MOVE PART\|PARTITION … TO VOLUME\|DISK …](../../../engines/table-engines/mergetree-family/mergetree.md#alter_move-partition) пользователь может принудительно перенести кусок или партицию с одного раздела на другой. При этом учитываются все ограничения, указанные для фоновых операций. Запрос самостоятельно инициирует процесс перемещения не дожидаясь фоновых операций. В случае недостатка места или неудовлетворения ограничениям пользователь получит сообщение об ошибке. + +Перемещения данных не взаимодействуют с репликацией данных, поэтому на разных репликах одной и той же таблицы могут быть указаны разные политики хранения. + +После выполнения фоновых слияний или мутаций старые куски не удаляются сразу, а через некоторое время (табличная настройка `old_parts_lifetime`). Также они не перемещаются на другие тома или диски, поэтому до момента удаления они продолжают учитываться при подсчёте занятого дискового пространства. + ## Использование сервиса S3 для хранения данных {#table_engine-mergetree-s3} Таблицы семейства `MergeTree` могут использовать для хранения данных сервиса [Amazon S3](https://aws.amazon.com/s3/) с диском типа `s3`. @@ -783,27 +806,5 @@ SETTINGS storage_policy = 'moving_from_ssd_to_hdd' Если диск сконфигурирован как `cold`, данные будут переноситься в S3 при срабатывании правил TTL или когда свободное место на локальном диске станет меньше порогового значения, которое определяется как `move_factor * disk_size`. -### Особенности работы {#details} - -В таблицах `MergeTree` данные попадают на диск несколькими способами: - -- В результате вставки (запрос `INSERT`). -- В фоновых операциях слияний и [мутаций](../../../sql-reference/statements/alter/index.md#mutations). -- При скачивании данных с другой реплики. -- В результате заморозки партиций [ALTER TABLE … FREEZE PARTITION](../../../engines/table-engines/mergetree-family/mergetree.md#alter_freeze-partition). - -Во всех случаях, кроме мутаций и заморозки партиций, при записи куска выбирается том и диск в соответствии с указанной конфигурацией хранилища: - -1. Выбирается первый по порядку том, на котором есть свободное место для записи куска (`unreserved_space > current_part_size`) и который позволяет записывать куски требуемого размера `max_data_part_size_bytes > current_part_size`. -2. Внутри тома выбирается следующий диск после того, на который была предыдущая запись и на котором свободного места больше чем размер куска (`unreserved_space - keep_free_space_bytes > current_part_size`) - -Мутации и запросы заморозки партиций в реализации используют [жесткие ссылки](https://ru.wikipedia.org/wiki/%D0%96%D1%91%D1%81%D1%82%D0%BA%D0%B0%D1%8F_%D1%81%D1%81%D1%8B%D0%BB%D0%BA%D0%B0). Жесткие ссылки между различными дисками не поддерживаются, поэтому в случае таких операций куски размещаются на тех же дисках, что и исходные. - -В фоне куски перемещаются между томами на основе информации о занятом месте (настройка `move_factor`) по порядку, в котором указаны тома в конфигурации. Данные никогда не перемещаются с последнего тома и на первый том. Следить за фоновыми перемещениями можно с помощью системных таблиц [system.part_log](../../../engines/table-engines/mergetree-family/mergetree.md#system_tables-part-log) (поле `type = MOVE_PART`) и [system.parts](../../../engines/table-engines/mergetree-family/mergetree.md#system_tables-parts) (поля `path` и `disk`). Также подробная информация о перемещениях доступна в логах сервера. -С помощью запроса [ALTER TABLE … MOVE PART\|PARTITION … TO VOLUME\|DISK …](../../../engines/table-engines/mergetree-family/mergetree.md#alter_move-partition) пользователь может принудительно перенести кусок или партицию с одного раздела на другой. При этом учитываются все ограничения, указанные для фоновых операций. Запрос самостоятельно инициирует процесс перемещения не дожидаясь фоновых операций. В случае недостатка места или неудовлетворения ограничениям пользователь получит сообщение об ошибке. - -Перемещения данных не взаимодействуют с репликацией данных, поэтому на разных репликах одной и той же таблицы могут быть указаны разные политики хранения. - -После выполнения фоновых слияний или мутаций старые куски не удаляются сразу, а через некоторое время (табличная настройка `old_parts_lifetime`). Также они не перемещаются на другие тома или диски, поэтому до момента удаления они продолжают учитываться при подсчёте занятого дискового пространства. [Оригинальная статья](https://clickhouse.tech/docs/ru/engines/table-engines/mergetree-family/mergetree/) From e4b4665ff556f7da153dd0bfb8afe210e1fc7d24 Mon Sep 17 00:00:00 2001 From: Nikita Mikhaylov Date: Fri, 12 Mar 2021 13:58:03 +0300 Subject: [PATCH 366/544] better --- programs/client/Client.cpp | 8 ++++---- .../Formats/Impl/ParallelFormattingOutputFormat.h | 9 +++++++++ src/Server/HTTP/WriteBufferFromHTTPServerResponse.cpp | 2 ++ .../00416_pocopatch_progress_in_http_headers.sh | 2 +- 4 files changed, 16 insertions(+), 5 deletions(-) diff --git a/programs/client/Client.cpp b/programs/client/Client.cpp index 3c27908741c..c878a3071c4 100644 --- a/programs/client/Client.cpp +++ b/programs/client/Client.cpp @@ -2096,10 +2096,10 @@ private: current_format = "Vertical"; /// It is not clear how to write progress with parallel formatting. It may increase code complexity significantly. - if (!need_render_progress) - block_out_stream = context.getOutputStreamParallelIfPossible(current_format, *out_buf, block); - else - block_out_stream = context.getOutputStream(current_format, *out_buf, block); + // if (!need_render_progress) + // block_out_stream = context.getOutputStreamParallelIfPossible(current_format, *out_buf, block); + // else + block_out_stream = context.getOutputStream(current_format, *out_buf, block); block_out_stream->writePrefix(); } diff --git a/src/Processors/Formats/Impl/ParallelFormattingOutputFormat.h b/src/Processors/Formats/Impl/ParallelFormattingOutputFormat.h index 2efc369e178..3fcd1f0aadf 100644 --- a/src/Processors/Formats/Impl/ParallelFormattingOutputFormat.h +++ b/src/Processors/Formats/Impl/ParallelFormattingOutputFormat.h @@ -6,6 +6,9 @@ #include #include #include +#include "IO/ReadBuffer.h" +#include "IO/ReadBufferFromString.h" +#include "IO/WriteBufferFromString.h" #include #include #include @@ -104,6 +107,12 @@ public: /// There are no formats which support parallel formatting and progress writing at the same time void onProgress(const Progress &) override {} + String getContentType() const override + { + WriteBufferFromOwnString buffer; + return internal_formatter_creator(buffer)->getContentType(); + } + protected: void consume(Chunk chunk) override final { diff --git a/src/Server/HTTP/WriteBufferFromHTTPServerResponse.cpp b/src/Server/HTTP/WriteBufferFromHTTPServerResponse.cpp index 355af038da9..7ded716b34e 100644 --- a/src/Server/HTTP/WriteBufferFromHTTPServerResponse.cpp +++ b/src/Server/HTTP/WriteBufferFromHTTPServerResponse.cpp @@ -149,6 +149,8 @@ void WriteBufferFromHTTPServerResponse::onProgress(const Progress & progress) { std::lock_guard lock(mutex); + std::cout << StackTrace().toString() << std::endl; + /// Cannot add new headers if body was started to send. if (headers_finished_sending) return; diff --git a/tests/queries/0_stateless/00416_pocopatch_progress_in_http_headers.sh b/tests/queries/0_stateless/00416_pocopatch_progress_in_http_headers.sh index 5d9cd12e4bf..6e9814cbca8 100755 --- a/tests/queries/0_stateless/00416_pocopatch_progress_in_http_headers.sh +++ b/tests/queries/0_stateless/00416_pocopatch_progress_in_http_headers.sh @@ -7,7 +7,7 @@ CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) ${CLICKHOUSE_CURL} -vsS "${CLICKHOUSE_URL}&max_block_size=5&send_progress_in_http_headers=1&http_headers_progress_interval_ms=0" -d 'SELECT max(number) FROM numbers(10)' 2>&1 | grep -E 'Content-Encoding|X-ClickHouse-Progress|^[0-9]' # This test will fail with external poco (progress not supported) -${CLICKHOUSE_CURL} -vsS "${CLICKHOUSE_URL}&max_block_size=1&send_progress_in_http_headers=1&http_headers_progress_interval_ms=0" -d 'SELECT number FROM numbers(10)' 2>&1 | grep -E 'Content-Encoding|X-ClickHouse-Progress|^[0-9]' +${CLICKHOUSE_CURL} -vsS "${CLICKHOUSE_URL}&max_block_size=1&send_progress_in_http_headers=1&http_headers_progress_interval_ms=0&output_format_parallel_formatting=0" -d 'SELECT number FROM numbers(10)' 2>&1 | grep -E 'Content-Encoding|X-ClickHouse-Progress|^[0-9]' ${CLICKHOUSE_CURL} -sS "${CLICKHOUSE_URL}&max_block_size=1&send_progress_in_http_headers=1&http_headers_progress_interval_ms=0&enable_http_compression=1" -H 'Accept-Encoding: gzip' -d 'SELECT number FROM system.numbers LIMIT 10' | gzip -d # 'send_progress_in_http_headers' is false by default From b83564bdadf0b092e1aa517d4e1fb3f1633ced8a Mon Sep 17 00:00:00 2001 From: Pavel Kruglov Date: Fri, 12 Mar 2021 15:08:00 +0300 Subject: [PATCH 367/544] Add test for path as a query parameter in system.zookeeper --- .../01753_system_zookeeper_query_param_path.reference | 1 + .../01753_system_zookeeper_query_param_path.sh | 9 +++++++++ 2 files changed, 10 insertions(+) create mode 100644 tests/queries/0_stateless/01753_system_zookeeper_query_param_path.reference create mode 100755 tests/queries/0_stateless/01753_system_zookeeper_query_param_path.sh diff --git a/tests/queries/0_stateless/01753_system_zookeeper_query_param_path.reference b/tests/queries/0_stateless/01753_system_zookeeper_query_param_path.reference new file mode 100644 index 00000000000..938ba95c63a --- /dev/null +++ b/tests/queries/0_stateless/01753_system_zookeeper_query_param_path.reference @@ -0,0 +1 @@ +clickhouse diff --git a/tests/queries/0_stateless/01753_system_zookeeper_query_param_path.sh b/tests/queries/0_stateless/01753_system_zookeeper_query_param_path.sh new file mode 100755 index 00000000000..9e98e6d125e --- /dev/null +++ b/tests/queries/0_stateless/01753_system_zookeeper_query_param_path.sh @@ -0,0 +1,9 @@ +#!/usr/bin/env bash + +CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh +. "$CUR_DIR"/../shell_config.sh + + +${CLICKHOUSE_CLIENT} --query="SELECT name FROM system.zookeeper WHERE path = {path:String}" --param_path "/" + From 8529b6a5bb92956e5de7a17878ca36aab22404ed Mon Sep 17 00:00:00 2001 From: Alexander Kuzmenkov <36882414+akuzm@users.noreply.github.com> Date: Fri, 12 Mar 2021 15:23:38 +0300 Subject: [PATCH 368/544] Update fuzzing-clickhouse.md --- website/blog/en/2021/fuzzing-clickhouse.md | 1 + 1 file changed, 1 insertion(+) diff --git a/website/blog/en/2021/fuzzing-clickhouse.md b/website/blog/en/2021/fuzzing-clickhouse.md index b6852dcce15..05b070e98a4 100644 --- a/website/blog/en/2021/fuzzing-clickhouse.md +++ b/website/blog/en/2021/fuzzing-clickhouse.md @@ -6,6 +6,7 @@ author: '[Alexander Kuzmenkov](https://github.com/akuzm)' tags: ['fuzzing', 'testing'] --- + Testing is a major problem in software development: there is never enough of it. It becomes especially true for a database management system, whose task is to interpret a query language that works on the persistent state managed by the system in a distributed fashion. Each of these three functions is hard enough to test even in isolation, and it gets much worse when you combine them. As ClickHouse developers, we know this from experience. Despite a large amount of automated testing of all kinds we routinely perform as part of our continuous integration system, new bugs and regressions are creeping in. We are always looking for the ways to improve our test coverage, and this article will describe our recent development in this area — the AST-based query fuzzer. ## How to Test a SQL DBMS From 1b908cad88e58079904effcf49070f70f3b5511a Mon Sep 17 00:00:00 2001 From: Maksim Kita Date: Fri, 12 Mar 2021 15:37:56 +0300 Subject: [PATCH 369/544] ExecutablePool fix default max execution time setting --- src/Dictionaries/ExecutablePoolDictionarySource.cpp | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/Dictionaries/ExecutablePoolDictionarySource.cpp b/src/Dictionaries/ExecutablePoolDictionarySource.cpp index 4bafeeecf7e..0c9ca4ce714 100644 --- a/src/Dictionaries/ExecutablePoolDictionarySource.cpp +++ b/src/Dictionaries/ExecutablePoolDictionarySource.cpp @@ -221,7 +221,9 @@ BlockInputStreamPtr ExecutablePoolDictionarySource::getStreamForBlock(const Bloc }, configuration.max_command_execution_time * 10000); if (!result) - throw Exception(ErrorCodes::TIMEOUT_EXCEEDED, "Could not get process from pool, max command execution timeout exceeded"); + throw Exception(ErrorCodes::TIMEOUT_EXCEEDED, + "Could not get process from pool, max command execution timeout exceeded ({}) seconds", + configuration.max_command_execution_time); size_t rows_to_read = block.rows(); auto read_stream = context.getInputFormat(configuration.format, process->out, sample_block, rows_to_read); @@ -298,7 +300,7 @@ void registerDictionarySourceExecutablePool(DictionarySourceFactory & factory) size_t max_command_execution_time = config.getUInt64(configuration_config_prefix + ".max_command_execution_time", 10); size_t max_execution_time_seconds = static_cast(context.getSettings().max_execution_time.totalSeconds()); - if (max_command_execution_time > max_execution_time_seconds) + if (max_execution_time_seconds != 0 && max_command_execution_time > max_execution_time_seconds) max_command_execution_time = max_execution_time_seconds; ExecutablePoolDictionarySource::Configuration configuration From 97a58777b88ddcd607ddc84c82234b184411146a Mon Sep 17 00:00:00 2001 From: Maksim Kita Date: Fri, 12 Mar 2021 16:36:42 +0300 Subject: [PATCH 370/544] DictionaryStructure fix non unique attribute names --- src/Dictionaries/DictionaryStructure.cpp | 9 ++++++ ...ictionary_unique_attribute_names.reference | 3 ++ ...1759_dictionary_unique_attribute_names.sql | 32 +++++++++++++++++++ 3 files changed, 44 insertions(+) create mode 100644 tests/queries/0_stateless/01759_dictionary_unique_attribute_names.reference create mode 100644 tests/queries/0_stateless/01759_dictionary_unique_attribute_names.sql diff --git a/src/Dictionaries/DictionaryStructure.cpp b/src/Dictionaries/DictionaryStructure.cpp index 25e29d7e0e8..3465eafda3b 100644 --- a/src/Dictionaries/DictionaryStructure.cpp +++ b/src/Dictionaries/DictionaryStructure.cpp @@ -354,6 +354,7 @@ std::vector DictionaryStructure::getAttributes( config.keys(config_prefix, config_elems); auto has_hierarchy = false; + std::unordered_set attribute_names; std::vector res_attributes; const FormatSettings format_settings; @@ -376,6 +377,14 @@ std::vector DictionaryStructure::getAttributes( if ((range_min && name == range_min->name) || (range_max && name == range_max->name)) continue; + if (attribute_names.find(name) != attribute_names.end()) + throw Exception( + ErrorCodes::BAD_ARGUMENTS, + "Dictionary attributes names must be unique. Attribute name ({}) is not unique", + name); + + attribute_names.insert(name); + const auto type_string = config.getString(prefix + "type"); const auto initial_type = DataTypeFactory::instance().get(type_string); auto type = initial_type; diff --git a/tests/queries/0_stateless/01759_dictionary_unique_attribute_names.reference b/tests/queries/0_stateless/01759_dictionary_unique_attribute_names.reference new file mode 100644 index 00000000000..bb08bbbe0b5 --- /dev/null +++ b/tests/queries/0_stateless/01759_dictionary_unique_attribute_names.reference @@ -0,0 +1,3 @@ +0 2 3 +1 5 6 +2 8 9 diff --git a/tests/queries/0_stateless/01759_dictionary_unique_attribute_names.sql b/tests/queries/0_stateless/01759_dictionary_unique_attribute_names.sql new file mode 100644 index 00000000000..11a52976716 --- /dev/null +++ b/tests/queries/0_stateless/01759_dictionary_unique_attribute_names.sql @@ -0,0 +1,32 @@ +DROP DATABASE IF EXISTS 01759_db; +CREATE DATABASE 01759_db; + +DROP TABLE IF EXISTS 01759_db.dictionary_source_table; +CREATE TABLE 01759_db.dictionary_source_table +( + key UInt64, + value1 UInt64, + value2 UInt64 +) +ENGINE = TinyLog; + +INSERT INTO 01759_db.dictionary_source_table VALUES (0, 2, 3), (1, 5, 6), (2, 8, 9); + +DROP DICTIONARY IF EXISTS 01759_db.test_dictionary; + +CREATE DICTIONARY 01759_db.test_dictionary(key UInt64, value1 UInt64, value1 UInt64) +PRIMARY KEY key +SOURCE(CLICKHOUSE(HOST 'localhost' PORT tcpPort() USER 'default' TABLE 'dictionary_source_table' DB '01759_db')) +LAYOUT(COMPLEX_KEY_DIRECT()); -- {serverError 36} + +CREATE DICTIONARY 01759_db.test_dictionary(key UInt64, value1 UInt64, value2 UInt64) +PRIMARY KEY key +SOURCE(CLICKHOUSE(HOST 'localhost' PORT tcpPort() USER 'default' TABLE 'dictionary_source_table' DB '01759_db')) +LAYOUT(COMPLEX_KEY_DIRECT()); + +SELECT number, dictGet('01759_db.test_dictionary', 'value1', tuple(number)) as value1, + dictGet('01759_db.test_dictionary', 'value2', tuple(number)) as value2 FROM system.numbers LIMIT 3; + +DROP TABLE 01759_db.dictionary_source_table; + +DROP DATABASE 01759_db; From ede0031dc493f6e95dc0abad779d90c9d41bf376 Mon Sep 17 00:00:00 2001 From: robot-clickhouse Date: Fri, 12 Mar 2021 18:08:46 +0300 Subject: [PATCH 371/544] Update version_date.tsv after release 21.3.2.5 --- utils/list-versions/version_date.tsv | 1 + 1 file changed, 1 insertion(+) diff --git a/utils/list-versions/version_date.tsv b/utils/list-versions/version_date.tsv index 3e63f8898c0..5edbc4bca1a 100644 --- a/utils/list-versions/version_date.tsv +++ b/utils/list-versions/version_date.tsv @@ -1,3 +1,4 @@ +v21.3.2.5-lts 2021-03-12 v21.2.5.5-stable 2021-03-02 v21.2.4.6-stable 2021-02-20 v21.2.3.15-stable 2021-02-14 From 5e5e54af309f97708eb350d0990dd48c3367e998 Mon Sep 17 00:00:00 2001 From: Alexander Kuzmenkov <36882414+akuzm@users.noreply.github.com> Date: Fri, 12 Mar 2021 18:13:21 +0300 Subject: [PATCH 372/544] Update fuzzing-clickhouse.md --- website/blog/en/2021/fuzzing-clickhouse.md | 1 - 1 file changed, 1 deletion(-) diff --git a/website/blog/en/2021/fuzzing-clickhouse.md b/website/blog/en/2021/fuzzing-clickhouse.md index 05b070e98a4..b6852dcce15 100644 --- a/website/blog/en/2021/fuzzing-clickhouse.md +++ b/website/blog/en/2021/fuzzing-clickhouse.md @@ -6,7 +6,6 @@ author: '[Alexander Kuzmenkov](https://github.com/akuzm)' tags: ['fuzzing', 'testing'] --- - Testing is a major problem in software development: there is never enough of it. It becomes especially true for a database management system, whose task is to interpret a query language that works on the persistent state managed by the system in a distributed fashion. Each of these three functions is hard enough to test even in isolation, and it gets much worse when you combine them. As ClickHouse developers, we know this from experience. Despite a large amount of automated testing of all kinds we routinely perform as part of our continuous integration system, new bugs and regressions are creeping in. We are always looking for the ways to improve our test coverage, and this article will describe our recent development in this area — the AST-based query fuzzer. ## How to Test a SQL DBMS From 8443ff563bc76a80c960febb4eee0942b74628a0 Mon Sep 17 00:00:00 2001 From: Maksim Kita Date: Fri, 12 Mar 2021 18:22:04 +0300 Subject: [PATCH 373/544] Fixed code review issues --- src/Dictionaries/DictionaryStructure.cpp | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/src/Dictionaries/DictionaryStructure.cpp b/src/Dictionaries/DictionaryStructure.cpp index 3465eafda3b..5e9d81ad415 100644 --- a/src/Dictionaries/DictionaryStructure.cpp +++ b/src/Dictionaries/DictionaryStructure.cpp @@ -377,14 +377,15 @@ std::vector DictionaryStructure::getAttributes( if ((range_min && name == range_min->name) || (range_max && name == range_max->name)) continue; - if (attribute_names.find(name) != attribute_names.end()) + auto insert_result = attribute_names.insert(name); + bool inserted = insert_result.second; + + if (!inserted) throw Exception( ErrorCodes::BAD_ARGUMENTS, "Dictionary attributes names must be unique. Attribute name ({}) is not unique", name); - attribute_names.insert(name); - const auto type_string = config.getString(prefix + "type"); const auto initial_type = DataTypeFactory::instance().get(type_string); auto type = initial_type; From 7f2d03b12802c392f114b00ca667e278d9016761 Mon Sep 17 00:00:00 2001 From: Pavel Kruglov Date: Fri, 12 Mar 2021 18:24:59 +0300 Subject: [PATCH 374/544] Change test --- .../01753_system_zookeeper_query_param_path.reference | 2 +- .../0_stateless/01753_system_zookeeper_query_param_path.sh | 7 ++++++- 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/tests/queries/0_stateless/01753_system_zookeeper_query_param_path.reference b/tests/queries/0_stateless/01753_system_zookeeper_query_param_path.reference index 938ba95c63a..9daeafb9864 100644 --- a/tests/queries/0_stateless/01753_system_zookeeper_query_param_path.reference +++ b/tests/queries/0_stateless/01753_system_zookeeper_query_param_path.reference @@ -1 +1 @@ -clickhouse +test diff --git a/tests/queries/0_stateless/01753_system_zookeeper_query_param_path.sh b/tests/queries/0_stateless/01753_system_zookeeper_query_param_path.sh index 9e98e6d125e..75a8e1b95ac 100755 --- a/tests/queries/0_stateless/01753_system_zookeeper_query_param_path.sh +++ b/tests/queries/0_stateless/01753_system_zookeeper_query_param_path.sh @@ -5,5 +5,10 @@ CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) . "$CUR_DIR"/../shell_config.sh -${CLICKHOUSE_CLIENT} --query="SELECT name FROM system.zookeeper WHERE path = {path:String}" --param_path "/" +${CLICKHOUSE_CLIENT} --query="DROP TABLE IF EXISTS test_01753"; +${CLICKHOUSE_CLIENT} --query="CREATE TABLE test_01753 (n Int8) ENGINE=ReplicatedMergeTree('/test_01753/test', 'r') ORDER BY n" +${CLICKHOUSE_CLIENT} --query="SELECT name FROM system.zookeeper WHERE path = {path:String}" --param_path "/test_01753" + + +${CLICKHOUSE_CLIENT} --query="DROP TABLE test_01753 SYNC"; From a10c4c9ded2477ef26d55f8499df302ac1308fd9 Mon Sep 17 00:00:00 2001 From: tavplubix Date: Fri, 12 Mar 2021 18:26:35 +0300 Subject: [PATCH 375/544] Update Dockerfile --- docker/test/sqlancer/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/test/sqlancer/Dockerfile b/docker/test/sqlancer/Dockerfile index 32012efb064..6bcdc3df5cd 100644 --- a/docker/test/sqlancer/Dockerfile +++ b/docker/test/sqlancer/Dockerfile @@ -1,7 +1,7 @@ # docker build -t yandex/clickhouse-sqlancer-test . FROM ubuntu:20.04 -RUN apt-get update --yes && env DEBIAN_FRONTEND=noninteractive apt-get install wget unzip git openjdk-14-jdk maven --yes --no-install-recommends +RUN apt-get update --yes && env DEBIAN_FRONTEND=noninteractive apt-get install wget unzip git openjdk-14-jdk maven python3 --yes --no-install-recommends RUN wget https://github.com/sqlancer/sqlancer/archive/master.zip -O /sqlancer.zip RUN mkdir /sqlancer && \ From b3f8ac796c24dfebb0eacfbb0c51d0c0ca094750 Mon Sep 17 00:00:00 2001 From: gyuton <40863448+gyuton@users.noreply.github.com> Date: Fri, 12 Mar 2021 19:09:55 +0300 Subject: [PATCH 376/544] Update docs/en/sql-reference/statements/detach.md Co-authored-by: olgarev <56617294+olgarev@users.noreply.github.com> --- docs/en/sql-reference/statements/detach.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/en/sql-reference/statements/detach.md b/docs/en/sql-reference/statements/detach.md index f3e2f88ffc3..6da49c4d3d4 100644 --- a/docs/en/sql-reference/statements/detach.md +++ b/docs/en/sql-reference/statements/detach.md @@ -5,7 +5,7 @@ toc_title: DETACH # DETACH Statement {#detach} -Deletes information about the table or view from the server. The server stops knowing about their existence. +Deletes information about the table or materialized view from the server. The server stops knowing about their existence. Syntax: From ed87521fcfc3c6a8bc81eb105d0a2e1e9167a7f2 Mon Sep 17 00:00:00 2001 From: gyuton <40863448+gyuton@users.noreply.github.com> Date: Fri, 12 Mar 2021 19:10:01 +0300 Subject: [PATCH 377/544] Update docs/en/sql-reference/statements/detach.md Co-authored-by: olgarev <56617294+olgarev@users.noreply.github.com> --- docs/en/sql-reference/statements/detach.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/en/sql-reference/statements/detach.md b/docs/en/sql-reference/statements/detach.md index 6da49c4d3d4..47490f3e0f9 100644 --- a/docs/en/sql-reference/statements/detach.md +++ b/docs/en/sql-reference/statements/detach.md @@ -13,7 +13,7 @@ Syntax: DETACH TABLE|VIEW [IF EXISTS] [db.]name [PERMANENTLY] [ON CLUSTER cluster] ``` -Detaching does not delete the data or metadata for the table or view. If the table or view was not detached `PERMANENTLY`, on the next server launch the server will read the metadata and recall the table/view again. If the table or view was detached `PERMANENTLY`, there will be no automatic recall. +Detaching does not delete the data or metadata for the table or materialized view. If the table or view was not detached `PERMANENTLY`, on the next server launch the server will read the metadata and recall the table/view again. If the table or view was detached `PERMANENTLY`, there will be no automatic recall. Whether the table was detached permanently or not, in both cases you can reattach it using the [ATTACH](../../sql-reference/statements/attach.md). System log tables can be also attached back (e.g. `query_log`, `text_log`, etc). Other system tables can't be reattached. On the next server launch the server will recall those tables again. From ca5e20a57dc576cb1a03d5a952b76732ed6c6309 Mon Sep 17 00:00:00 2001 From: gyuton <40863448+gyuton@users.noreply.github.com> Date: Fri, 12 Mar 2021 19:11:27 +0300 Subject: [PATCH 378/544] Update docs/en/sql-reference/statements/detach.md Co-authored-by: olgarev <56617294+olgarev@users.noreply.github.com> --- docs/en/sql-reference/statements/detach.md | 19 ++----------------- 1 file changed, 2 insertions(+), 17 deletions(-) diff --git a/docs/en/sql-reference/statements/detach.md b/docs/en/sql-reference/statements/detach.md index 47490f3e0f9..ae97d716a5e 100644 --- a/docs/en/sql-reference/statements/detach.md +++ b/docs/en/sql-reference/statements/detach.md @@ -25,25 +25,10 @@ Also you can not [DROP](../../sql-reference/statements/drop.md#drop-table) the d **Example** -Query: +Creating a table: ``` sql CREATE TABLE test ENGINE = Log AS SELECT * FROM numbers(10); - -DETACH TABLE test; - -SELECT * FROM TEST; -``` - -Result: - -``` text -Ok. - -Ok. - -Received exception from server (version 21.3.1): -Code: 60. DB::Exception: Received from localhost:9000. DB::Exception: Table default.TEST doesn't exist. -``` +SELECT * FROM test; [Original article](https://clickhouse.tech/docs/en/sql-reference/statements/detach/) From 53fd7cb8a840719af7a4316b216d931a5a1806b3 Mon Sep 17 00:00:00 2001 From: George Date: Fri, 12 Mar 2021 19:14:40 +0300 Subject: [PATCH 379/544] Updated example --- docs/en/sql-reference/statements/detach.md | 36 ++++++++++++++++++++++ 1 file changed, 36 insertions(+) diff --git a/docs/en/sql-reference/statements/detach.md b/docs/en/sql-reference/statements/detach.md index ae97d716a5e..cb0d7cf7b66 100644 --- a/docs/en/sql-reference/statements/detach.md +++ b/docs/en/sql-reference/statements/detach.md @@ -27,8 +27,44 @@ Also you can not [DROP](../../sql-reference/statements/drop.md#drop-table) the d Creating a table: +Query: + ``` sql CREATE TABLE test ENGINE = Log AS SELECT * FROM numbers(10); SELECT * FROM test; +``` + +Result: + +``` text +┌─number─┐ +│ 0 │ +│ 1 │ +│ 2 │ +│ 3 │ +│ 4 │ +│ 5 │ +│ 6 │ +│ 7 │ +│ 8 │ +│ 9 │ +└────────┘ +``` + +Detaching the table: + +Query: + +``` sql +DETACH TABLE test; +SELECT * FROM test; +``` + +Result: + +``` text +Received exception from server (version 21.4.1): +Code: 60. DB::Exception: Received from localhost:9000. DB::Exception: Table default.test doesn't exist. +``` [Original article](https://clickhouse.tech/docs/en/sql-reference/statements/detach/) From f89247d7bd6d790d5d59679add762b082e905329 Mon Sep 17 00:00:00 2001 From: George Date: Fri, 12 Mar 2021 19:28:04 +0300 Subject: [PATCH 380/544] updated attach.md --- docs/en/sql-reference/statements/attach.md | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/docs/en/sql-reference/statements/attach.md b/docs/en/sql-reference/statements/attach.md index 035441ef5f1..ffb577a8839 100644 --- a/docs/en/sql-reference/statements/attach.md +++ b/docs/en/sql-reference/statements/attach.md @@ -17,4 +17,8 @@ If the table was previously detached ([DETACH](../../sql-reference/statements/de ATTACH TABLE [IF NOT EXISTS] [db.]name [ON CLUSTER cluster] ``` -This query is used when starting the server. The server stores table metadata as files with `ATTACH` queries, which it simply runs at launch (with the exception of system tables, which are explicitly created on the server). +This query is used when starting the server. The server stores table metadata as files with `ATTACH` queries, which it simply runs at launch (with the exception of some system tables, which are explicitly created on the server). + +If the table was detached permanently, it won't be reattached at the server start, so you need to use `ATTACH` query explicitly. + +[Original article](https://clickhouse.tech/docs/en/sql-reference/statements/attach/) From 1c0c5f44410d9b45336fec8514dfa09f8efb1cd9 Mon Sep 17 00:00:00 2001 From: Roman Bug Date: Fri, 12 Mar 2021 19:44:36 +0300 Subject: [PATCH 381/544] Update avg.md --- docs/ru/sql-reference/aggregate-functions/reference/avg.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/ru/sql-reference/aggregate-functions/reference/avg.md b/docs/ru/sql-reference/aggregate-functions/reference/avg.md index 8027674e68c..c032199aa32 100644 --- a/docs/ru/sql-reference/aggregate-functions/reference/avg.md +++ b/docs/ru/sql-reference/aggregate-functions/reference/avg.md @@ -14,12 +14,12 @@ avg(x) **Аргументы** -- `x` — входное значение в формате [Integer](../../../sql-reference/data-types/int-uint.md), [Float](../../../sql-reference/data-types/float.md) или [Decimal](../../../sql-reference/data-types/decimal.md). +- `x` — входное значение типа [Integer](../../../sql-reference/data-types/int-uint.md), [Float](../../../sql-reference/data-types/float.md) или [Decimal](../../../sql-reference/data-types/decimal.md). **Возвращаемое значение** -- среднее арифметическое, всегда в формате [Float64](../../../sql-reference/data-types/float.md). -- `NaN`, если не указано входное значение `x`. +- среднее арифметическое, всегда типа [Float64](../../../sql-reference/data-types/float.md). +- `NaN`, если входное значение `x` — пустое. **Пример** From 3269e1d3316bf2be3a1f3f6ea74111004d8db703 Mon Sep 17 00:00:00 2001 From: Vitaliy Zakaznikov Date: Fri, 12 Mar 2021 12:32:32 -0500 Subject: [PATCH 382/544] Updating TestFlows to 1.6.74 --- docker/test/testflows/runner/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/test/testflows/runner/Dockerfile b/docker/test/testflows/runner/Dockerfile index 10014851a82..bd7eee4c166 100644 --- a/docker/test/testflows/runner/Dockerfile +++ b/docker/test/testflows/runner/Dockerfile @@ -35,7 +35,7 @@ RUN apt-get update \ ENV TZ=Europe/Moscow RUN ln -snf /usr/share/zoneinfo/$TZ /etc/localtime && echo $TZ > /etc/timezone -RUN pip3 install urllib3 testflows==1.6.72 docker-compose docker dicttoxml kazoo tzlocal +RUN pip3 install urllib3 testflows==1.6.74 docker-compose docker dicttoxml kazoo tzlocal ENV DOCKER_CHANNEL stable ENV DOCKER_VERSION 17.09.1-ce From 5af5c90b8baa2a701f3c688023f7d78fa61c4551 Mon Sep 17 00:00:00 2001 From: Azat Khuzhin Date: Fri, 12 Mar 2021 21:10:16 +0300 Subject: [PATCH 383/544] Fix test_replace_partition flakiness MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit test_replace_partition/test.py::test_replace_after_replace_failover statistics: ```sql ┌──────ymd─┬─success─┬─failure─┐ │ 20200613 │ 40 │ 2 │ │ 20200614 │ 98 │ 2 │ │ 20200615 │ 126 │ 9 │ │ 20200616 │ 141 │ 10 │ │ 20200617 │ 119 │ 7 │ │ 20200618 │ 87 │ 12 │ │ 20200619 │ 81 │ 9 │ ... │ 20210305 │ 60 │ 28 │ │ 20210306 │ 69 │ 7 │ │ 20210307 │ 42 │ 0 │ │ 20210308 │ 53 │ 8 │ │ 20210309 │ 137 │ 23 │ │ 20210310 │ 84 │ 14 │ │ 20210311 │ 81 │ 20 │ └──────────┴─────────┴─────────┘ ``` --- .../test_replace_partition/test.py | 23 +++++++++++-------- 1 file changed, 14 insertions(+), 9 deletions(-) diff --git a/tests/integration/test_replace_partition/test.py b/tests/integration/test_replace_partition/test.py index 06e7f4be82b..c87a650fce7 100644 --- a/tests/integration/test_replace_partition/test.py +++ b/tests/integration/test_replace_partition/test.py @@ -1,3 +1,4 @@ +import time import pytest from helpers.cluster import ClickHouseCluster @@ -97,12 +98,13 @@ def test_drop_failover(drop_failover): # Drop partition on source node node3.query("ALTER TABLE test_table DROP PARTITION 201706") - # connection restored + # Wait few seconds for connection to zookeeper to be restored + time.sleep(5) - node4.query_with_retry("select last_exception from system.replication_queue where type = 'REPLACE_RANGE'", - check_callback=lambda x: 'Not found part' not in x, sleep_time=1) - assert 'Not found part' not in node4.query( - "select last_exception from system.replication_queue where type = 'REPLACE_RANGE'") + msg = node4.query_with_retry( + "select last_exception from system.replication_queue where type = 'REPLACE_RANGE'", + check_callback=lambda x: 'Not found part' not in x, sleep_time=1) + assert 'Not found part' not in msg assert_eq_with_retry(node4, "SELECT id FROM test_table order by id", '') @@ -151,8 +153,11 @@ def test_replace_after_replace_failover(replace_after_replace_failover): assert_eq_with_retry(node5, "SELECT id FROM test_table order by id", '333') - node6.query_with_retry("select last_exception from system.replication_queue where type = 'REPLACE_RANGE'", - check_callback=lambda x: 'Not found part' not in x, sleep_time=1) - assert 'Not found part' not in node6.query( - "select last_exception from system.replication_queue where type = 'REPLACE_RANGE'") + # Wait few seconds for connection to zookeeper to be restored + time.sleep(5) + + msg = node6.query_with_retry( + "select last_exception from system.replication_queue where type = 'REPLACE_RANGE'", + check_callback=lambda x: 'Not found part' not in x, sleep_time=1) + assert 'Not found part' not in msg assert_eq_with_retry(node6, "SELECT id FROM test_table order by id", '333') From 2e99dad56213413a72969d47650050b42cbcc073 Mon Sep 17 00:00:00 2001 From: Azat Khuzhin Date: Fri, 12 Mar 2021 21:10:16 +0300 Subject: [PATCH 384/544] Tiny "machine" cleanup of test_replace_partition --- tests/integration/test_replace_partition/test.py | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/tests/integration/test_replace_partition/test.py b/tests/integration/test_replace_partition/test.py index c87a650fce7..d30a038825f 100644 --- a/tests/integration/test_replace_partition/test.py +++ b/tests/integration/test_replace_partition/test.py @@ -1,3 +1,7 @@ +# pylint: disable=line-too-long +# pylint: disable=unused-argument +# pylint: disable=redefined-outer-name: + import time import pytest @@ -14,13 +18,13 @@ def _fill_nodes(nodes, shard): node.query( ''' CREATE DATABASE test; - + CREATE TABLE real_table(date Date, id UInt32, dummy UInt32) ENGINE = MergeTree(date, id, 8192); - + CREATE TABLE other_table(date Date, id UInt32, dummy UInt32) ENGINE = MergeTree(date, id, 8192); - + CREATE TABLE test_table(date Date, id UInt32, dummy UInt32) ENGINE = ReplicatedMergeTree('/clickhouse/tables/test{shard}/replicated', '{replica}', date, id, 8192); '''.format(shard=shard, replica=node.name)) From 34915d031145cfab1ca6f5f365c59b4e3acc77f3 Mon Sep 17 00:00:00 2001 From: Michael Monashev Date: Fri, 12 Mar 2021 21:13:20 +0300 Subject: [PATCH 385/544] Fix ORDER BY syntax --- docs/ru/engines/table-engines/mergetree-family/mergetree.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/ru/engines/table-engines/mergetree-family/mergetree.md b/docs/ru/engines/table-engines/mergetree-family/mergetree.md index bc74b2592b9..0615613533b 100644 --- a/docs/ru/engines/table-engines/mergetree-family/mergetree.md +++ b/docs/ru/engines/table-engines/mergetree-family/mergetree.md @@ -517,7 +517,7 @@ CREATE TABLE table_for_aggregation y Int ) ENGINE = MergeTree -ORDER BY k1, k2 +ORDER BY (k1, k2) TTL d + INTERVAL 1 MONTH GROUP BY k1, k2 SET x = max(x), y = min(y); ``` From 12e411f8b979d1e67b0e1765a43d421b3cb50990 Mon Sep 17 00:00:00 2001 From: Maksim Kita Date: Fri, 12 Mar 2021 21:38:52 +0300 Subject: [PATCH 386/544] PODArray swap fix --- src/Common/PODArray.h | 42 ++++++++++++++++++++++++++---------------- 1 file changed, 26 insertions(+), 16 deletions(-) diff --git a/src/Common/PODArray.h b/src/Common/PODArray.h index 26be7fe82ba..c88a0fa16ff 100644 --- a/src/Common/PODArray.h +++ b/src/Common/PODArray.h @@ -568,7 +568,7 @@ public: /// arr1 takes ownership of the heap memory of arr2. arr1.c_start = arr2.c_start; - arr1.c_end_of_storage = arr1.c_start + heap_allocated - arr1.pad_right; + arr1.c_end_of_storage = arr1.c_start + heap_allocated - arr2.pad_right - arr2.pad_left; arr1.c_end = arr1.c_start + this->byte_size(heap_size); /// Allocate stack space for arr2. @@ -585,7 +585,7 @@ public: dest.dealloc(); dest.alloc(src.allocated_bytes(), std::forward(allocator_params)...); memcpy(dest.c_start, src.c_start, this->byte_size(src.size())); - dest.c_end = dest.c_start + (src.c_end - src.c_start); + dest.c_end = dest.c_start + this->byte_size(src.size()); src.c_start = Base::null; src.c_end = Base::null; @@ -597,6 +597,11 @@ public: std::swap(dest.c_end, src.c_end); std::swap(dest.c_end_of_storage, src.c_end_of_storage); } + +#ifndef NDEBUG + this->protect(); + rhs.protect(); +#endif }; if (!this->isInitialized() && !rhs.isInitialized()) @@ -639,8 +644,8 @@ public: size_t rhs_size = rhs.size(); size_t rhs_allocated = rhs.allocated_bytes(); - this->c_end_of_storage = this->c_start + rhs_allocated - Base::pad_right; - rhs.c_end_of_storage = rhs.c_start + lhs_allocated - Base::pad_right; + this->c_end_of_storage = this->c_start + rhs_allocated - Base::pad_right - Base::pad_left; + rhs.c_end_of_storage = rhs.c_start + lhs_allocated - Base::pad_right - Base::pad_left; this->c_end = this->c_start + this->byte_size(rhs_size); rhs.c_end = rhs.c_start + this->byte_size(lhs_size); @@ -659,6 +664,11 @@ public: std::swap(this->c_end, rhs.c_end); std::swap(this->c_end_of_storage, rhs.c_end_of_storage); } + +#ifndef NDEBUG + this->protect(); + rhs.protect(); +#endif } template @@ -693,34 +703,34 @@ public: } - bool operator== (const PODArray & other) const + bool operator== (const PODArray & rhs) const { - if (this->size() != other.size()) + if (this->size() != rhs.size()) return false; - const_iterator this_it = begin(); - const_iterator that_it = other.begin(); + const_iterator lhs_it = begin(); + const_iterator rhs_it = rhs.begin(); - while (this_it != end()) + while (lhs_it != end()) { - if (*this_it != *that_it) + if (*lhs_it != *rhs_it) return false; - ++this_it; - ++that_it; + ++lhs_it; + ++rhs_it; } return true; } - bool operator!= (const PODArray & other) const + bool operator!= (const PODArray & rhs) const { - return !operator==(other); + return !operator==(rhs); } }; -template -void swap(PODArray & lhs, PODArray & rhs) +template +void swap(PODArray & lhs, PODArray & rhs) { lhs.swap(rhs); } From ff1cb65f0b3f6c8a0dc12818a92d8884aaa4d8d1 Mon Sep 17 00:00:00 2001 From: Azat Khuzhin Date: Fri, 12 Mar 2021 21:44:59 +0300 Subject: [PATCH 387/544] Start accepting connections after DDLWorker and dictionaries initialization Found by integration tests [1]: Code: 139. DB::Exception: Received from 172.18.0.6:9000. DB::Exception: DDL background thread is not initialized. Stack trace: [1]: https://clickhouse-test-reports.s3.yandex.net/21643/65f90f2ce9ea9e9d4076f06c58ddd981c82cc098/integration_tests_(thread).html#fail1 --- programs/server/Server.cpp | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/programs/server/Server.cpp b/programs/server/Server.cpp index 9889b08828b..57d9257df5d 100644 --- a/programs/server/Server.cpp +++ b/programs/server/Server.cpp @@ -1280,9 +1280,6 @@ int Server::main(const std::vector & /*args*/) async_metrics.start(); global_context->enableNamedSessions(); - for (auto & server : *servers) - server.start(); - { String level_str = config().getString("text_log.level", ""); int level = level_str.empty() ? INT_MAX : Poco::Logger::parseLevel(level_str); @@ -1334,6 +1331,8 @@ int Server::main(const std::vector & /*args*/) "distributed_ddl", "DDLWorker", &CurrentMetrics::MaxDDLEntryID)); } + for (auto & server : *servers) + server.start(); LOG_INFO(log, "Ready for connections."); SCOPE_EXIT({ From 388f5d51442ba304cde1374ae6f2027aec6bddec Mon Sep 17 00:00:00 2001 From: Alexander Kuzmenkov <36882414+akuzm@users.noreply.github.com> Date: Fri, 12 Mar 2021 22:38:43 +0300 Subject: [PATCH 388/544] Update fuzzing-clickhouse.md --- website/blog/en/2021/fuzzing-clickhouse.md | 1 + 1 file changed, 1 insertion(+) diff --git a/website/blog/en/2021/fuzzing-clickhouse.md b/website/blog/en/2021/fuzzing-clickhouse.md index b6852dcce15..05b070e98a4 100644 --- a/website/blog/en/2021/fuzzing-clickhouse.md +++ b/website/blog/en/2021/fuzzing-clickhouse.md @@ -6,6 +6,7 @@ author: '[Alexander Kuzmenkov](https://github.com/akuzm)' tags: ['fuzzing', 'testing'] --- + Testing is a major problem in software development: there is never enough of it. It becomes especially true for a database management system, whose task is to interpret a query language that works on the persistent state managed by the system in a distributed fashion. Each of these three functions is hard enough to test even in isolation, and it gets much worse when you combine them. As ClickHouse developers, we know this from experience. Despite a large amount of automated testing of all kinds we routinely perform as part of our continuous integration system, new bugs and regressions are creeping in. We are always looking for the ways to improve our test coverage, and this article will describe our recent development in this area — the AST-based query fuzzer. ## How to Test a SQL DBMS From 58e415c7d42066ff478c1409e9bd7b84c34d044e Mon Sep 17 00:00:00 2001 From: Pavel Kruglov Date: Sat, 13 Mar 2021 00:17:19 +0300 Subject: [PATCH 389/544] Update clusters only if their configs were updated --- .../AbstractConfigurationComparison.cpp | 4 + .../Config/AbstractConfigurationComparison.h | 5 + src/Interpreters/Cluster.cpp | 38 ++- src/Interpreters/Cluster.h | 2 +- src/Interpreters/Context.cpp | 9 +- .../configs/remote_servers.xml | 30 +++ .../test_reload_clusters_config/test.py | 235 ++++++++++++++++++ 7 files changed, 315 insertions(+), 8 deletions(-) create mode 100644 tests/integration/test_reload_clusters_config/configs/remote_servers.xml create mode 100644 tests/integration/test_reload_clusters_config/test.py diff --git a/src/Common/Config/AbstractConfigurationComparison.cpp b/src/Common/Config/AbstractConfigurationComparison.cpp index 0e603cb1056..59c0c895a89 100644 --- a/src/Common/Config/AbstractConfigurationComparison.cpp +++ b/src/Common/Config/AbstractConfigurationComparison.cpp @@ -26,6 +26,10 @@ bool isSameConfiguration(const Poco::Util::AbstractConfiguration & left, const P return isSameConfiguration(left, String(), right, String()); } +bool isSameConfiguration(const Poco::Util::AbstractConfiguration & left, const Poco::Util::AbstractConfiguration & right, const String & key) +{ + return isSameConfiguration(left, key, right, key); +} bool isSameConfiguration(const Poco::Util::AbstractConfiguration & left, const String & left_key, const Poco::Util::AbstractConfiguration & right, const String & right_key) diff --git a/src/Common/Config/AbstractConfigurationComparison.h b/src/Common/Config/AbstractConfigurationComparison.h index f825ad4e53d..795fca2af8e 100644 --- a/src/Common/Config/AbstractConfigurationComparison.h +++ b/src/Common/Config/AbstractConfigurationComparison.h @@ -13,6 +13,11 @@ namespace DB bool isSameConfiguration(const Poco::Util::AbstractConfiguration & left, const Poco::Util::AbstractConfiguration & right); + /// Returns true if the specified subview of the two configurations contains the same keys and values. + bool isSameConfiguration(const Poco::Util::AbstractConfiguration & left, + const Poco::Util::AbstractConfiguration & right, + const String & key); + /// Returns true if specified subviews of the two configurations contains the same keys and values. bool isSameConfiguration(const Poco::Util::AbstractConfiguration & left, const String & left_key, const Poco::Util::AbstractConfiguration & right, const String & right_key); diff --git a/src/Interpreters/Cluster.cpp b/src/Interpreters/Cluster.cpp index fb9788e84c4..b77d5019d48 100644 --- a/src/Interpreters/Cluster.cpp +++ b/src/Interpreters/Cluster.cpp @@ -5,6 +5,7 @@ #include #include #include +#include #include #include #include @@ -265,20 +266,45 @@ void Clusters::setCluster(const String & cluster_name, const std::shared_ptrkeys(config_prefix, old_config_keys); + std::sort(old_config_keys.begin(), old_config_keys.end()); + + std::set_difference( + old_config_keys.begin(), old_config_keys.end(), new_config_keys.begin(), new_config_keys.end(), std::back_inserter(deleted_keys)); + } std::lock_guard lock(mutex); - impl.clear(); - for (const auto & key : config_keys) + /// If old congig is set, remove deleted clusters from impl, otherwise just clear it. + if (old_config) + { + for (const auto & key : deleted_keys) + impl.erase(key); + } + else + impl.clear(); + + for (const auto & key : new_config_keys) { if (key.find('.') != String::npos) throw Exception("Cluster names with dots are not supported: '" + key + "'", ErrorCodes::SYNTAX_ERROR); - impl.emplace(key, std::make_shared(config, settings, config_prefix, key)); + /// If old config is set and cluster config wasn't changed, don't update this cluster. + if (!old_config || !isSameConfiguration(new_config, *old_config, config_prefix + "." + key)) + impl[key] = std::make_shared(new_config, settings, config_prefix, key); } } diff --git a/src/Interpreters/Cluster.h b/src/Interpreters/Cluster.h index c64d52724e5..a047f199204 100644 --- a/src/Interpreters/Cluster.h +++ b/src/Interpreters/Cluster.h @@ -276,7 +276,7 @@ public: ClusterPtr getCluster(const std::string & cluster_name) const; void setCluster(const String & cluster_name, const ClusterPtr & cluster); - void updateClusters(const Poco::Util::AbstractConfiguration & config, const Settings & settings, const String & config_prefix); + void updateClusters(const Poco::Util::AbstractConfiguration & new_config, const Settings & settings, const String & config_prefix, Poco::Util::AbstractConfiguration * old_config = nullptr); public: using Impl = std::map; diff --git a/src/Interpreters/Context.cpp b/src/Interpreters/Context.cpp index 8615cf70343..bb2d553b8e8 100644 --- a/src/Interpreters/Context.cpp +++ b/src/Interpreters/Context.cpp @@ -60,6 +60,7 @@ #include #include #include +#include #include #include #include @@ -1833,12 +1834,18 @@ void Context::setClustersConfig(const ConfigurationPtr & config, const String & { std::lock_guard lock(shared->clusters_mutex); + /// Do not update clusters if this part of config wasn't changed. + if (shared->clusters && isSameConfiguration(*config, *shared->clusters_config, config_name)) { + return; + } + + auto old_clusters_config = shared->clusters_config; shared->clusters_config = config; if (!shared->clusters) shared->clusters = std::make_unique(*shared->clusters_config, settings, config_name); else - shared->clusters->updateClusters(*shared->clusters_config, settings, config_name); + shared->clusters->updateClusters(*shared->clusters_config, settings, config_name, old_clusters_config); } diff --git a/tests/integration/test_reload_clusters_config/configs/remote_servers.xml b/tests/integration/test_reload_clusters_config/configs/remote_servers.xml new file mode 100644 index 00000000000..b827fce02be --- /dev/null +++ b/tests/integration/test_reload_clusters_config/configs/remote_servers.xml @@ -0,0 +1,30 @@ + + + + + true + + node_1 + 9000 + + + node_2 + 9000 + + + + + + true + + node_1 + 9000 + + + node_2 + 9000 + + + + + diff --git a/tests/integration/test_reload_clusters_config/test.py b/tests/integration/test_reload_clusters_config/test.py new file mode 100644 index 00000000000..f1fb0d820d4 --- /dev/null +++ b/tests/integration/test_reload_clusters_config/test.py @@ -0,0 +1,235 @@ +import os +import sys +import time + +import pytest + +sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) + +from helpers.cluster import ClickHouseCluster +from helpers.network import PartitionManager +from helpers.test_tools import TSV + +cluster = ClickHouseCluster(__file__) +node = cluster.add_instance('node', with_zookeeper=True, main_configs=['configs/remote_servers.xml']) +node_1 = cluster.add_instance('node_1', with_zookeeper=True) +node_2 = cluster.add_instance('node_2', with_zookeeper=True) + +@pytest.fixture(scope="module") +def started_cluster(): + try: + cluster.start() + + node.query('''CREATE TABLE distributed (id UInt32) ENGINE = + Distributed('test_cluster', 'default', 'replicated')''') + + node.query('''CREATE TABLE distributed2 (id UInt32) ENGINE = + Distributed('test_cluster2', 'default', 'replicated')''') + + cluster.pause_container('node_1') + cluster.pause_container('node_2') + + yield cluster + + finally: + cluster.shutdown() + + +base_config = ''' + + + + + true + + node_1 + 9000 + + + node_2 + 9000 + + + + + + true + + node_1 + 9000 + + + node_2 + 9000 + + + + + +''' + +test_config1 = ''' + + + + + true + + node_1 + 9000 + + + + + + true + + node_1 + 9000 + + + node_2 + 9000 + + + + + +''' + +test_config2 = ''' + + + + + true + + node_1 + 9000 + + + node_2 + 9000 + + + + + +''' + +test_config3 = ''' + + + + + true + + node_1 + 9000 + + + node_2 + 9000 + + + + + + true + + node_1 + 9000 + + + node_2 + 9000 + + + + + + true + + node_1 + 9000 + + + + + +''' + + +def send_repeated_query(table, count=5): + for i in range(count): + node.query_and_get_error("SELECT count() FROM {} SETTINGS receive_timeout=1".format(table)) + + +def get_errors_count(cluster, host_name="node_1"): + return int(node.query("SELECT errors_count FROM system.clusters WHERE cluster='{}' and host_name='{}'".format(cluster, host_name))) + + +def set_config(config): + node.replace_config("/etc/clickhouse-server/config.d/remote_servers.xml", config) + node.query("SYSTEM RELOAD CONFIG") + + +def test_simple_reload(started_cluster): + send_repeated_query("distributed") + + assert get_errors_count("test_cluster") > 0 + + node.query("SYSTEM RELOAD CONFIG") + + assert get_errors_count("test_cluster") > 0 + + +def test_update_one_cluster(started_cluster): + send_repeated_query("distributed") + send_repeated_query("distributed2") + + assert get_errors_count("test_cluster") > 0 + assert get_errors_count("test_cluster2") > 0 + + set_config(test_config1) + + assert get_errors_count("test_cluster") == 0 + assert get_errors_count("test_cluster2") > 0 + + set_config(base_config) + + +def test_delete_cluster(started_cluster): + send_repeated_query("distributed") + send_repeated_query("distributed2") + + assert get_errors_count("test_cluster") > 0 + assert get_errors_count("test_cluster2") > 0 + + set_config(test_config2) + + assert get_errors_count("test_cluster") > 0 + + result = node.query("SELECT * FROM system.clusters WHERE cluster='test_cluster2'") + assert result == '' + + set_config(base_config) + + +def test_add_cluster(started_cluster): + send_repeated_query("distributed") + send_repeated_query("distributed2") + + assert get_errors_count("test_cluster") > 0 + assert get_errors_count("test_cluster2") > 0 + + set_config(test_config3) + + assert get_errors_count("test_cluster") > 0 + assert get_errors_count("test_cluster2") > 0 + + result = node.query("SELECT * FROM system.clusters WHERE cluster='test_cluster3'") + assert result != '' + + set_config(base_config) + From 8312553d24fd59d2b134f9e60a9efd2e27ae46a7 Mon Sep 17 00:00:00 2001 From: Maksim Kita Date: Sat, 13 Mar 2021 00:30:59 +0300 Subject: [PATCH 390/544] LibraryDictionarySource fix possible leak --- src/Dictionaries/LibraryDictionarySource.cpp | 25 ++++++++++++++------ 1 file changed, 18 insertions(+), 7 deletions(-) diff --git a/src/Dictionaries/LibraryDictionarySource.cpp b/src/Dictionaries/LibraryDictionarySource.cpp index 6d763444b54..0e692a5893a 100644 --- a/src/Dictionaries/LibraryDictionarySource.cpp +++ b/src/Dictionaries/LibraryDictionarySource.cpp @@ -84,9 +84,7 @@ namespace + (columns_received->error_string ? columns_received->error_string : ""), ErrorCodes::EXTERNAL_LIBRARY_ERROR); - MutableColumns columns(sample_block.columns()); - for (const auto i : ext::range(0, columns.size())) - columns[i] = sample_block.getByPosition(i).column->cloneEmpty(); + MutableColumns columns = sample_block.cloneEmptyColumns(); for (size_t col_n = 0; col_n < columns_received->size; ++col_n) { @@ -151,6 +149,7 @@ LibraryDictionarySource::LibraryDictionarySource( #endif ); settings = std::make_shared(getLibSettings(config, config_prefix + lib_config_settings)); + if (auto lib_new = library->tryGetstrings), decltype(&ClickHouseLibrary::log))>( "ClickHouseDictionary_v3_libNew")) lib_data = lib_new(&settings->strings, ClickHouseLibrary::log); @@ -193,15 +192,20 @@ BlockInputStreamPtr LibraryDictionarySource::loadAll() columns.data[i] = a.name.c_str(); ++i; } + void * data_ptr = nullptr; /// Get function pointer before dataNew call because library->get may throw. auto func_load_all = library->getstrings), decltype(&columns))>("ClickHouseDictionary_v3_loadAll"); + data_ptr = library->get("ClickHouseDictionary_v3_dataNew")(lib_data); - auto * data = func_load_all(data_ptr, &settings->strings, &columns); - auto block = dataToBlock(description.sample_block, data); SCOPE_EXIT(library->get("ClickHouseDictionary_v3_dataDelete")(lib_data, data_ptr)); + + auto * data = func_load_all(data_ptr, &settings->strings, &columns); + + auto block = dataToBlock(description.sample_block, data); + return std::make_shared(block); } @@ -219,16 +223,20 @@ BlockInputStreamPtr LibraryDictionarySource::loadIds(const std::vector & columns_pass.data[i] = a.name.c_str(); ++i; } + void * data_ptr = nullptr; /// Get function pointer before dataNew call because library->get may throw. auto func_load_ids = library->getstrings), decltype(&columns_pass), decltype(&ids_data))>( "ClickHouseDictionary_v3_loadIds"); + data_ptr = library->get("ClickHouseDictionary_v3_dataNew")(lib_data); + SCOPE_EXIT(library->get("ClickHouseDictionary_v3_dataDelete")(lib_data, data_ptr)); + auto * data = func_load_ids(data_ptr, &settings->strings, &columns_pass, &ids_data); auto block = dataToBlock(description.sample_block, data); - SCOPE_EXIT(library->get("ClickHouseDictionary_v3_dataDelete")(lib_data, data_ptr)); + return std::make_shared(block); } @@ -258,10 +266,13 @@ BlockInputStreamPtr LibraryDictionarySource::loadKeys(const Columns & key_column /// Get function pointer before dataNew call because library->get may throw. auto func_load_keys = library->getstrings), decltype(&request_cols))>( "ClickHouseDictionary_v3_loadKeys"); + data_ptr = library->get("ClickHouseDictionary_v3_dataNew")(lib_data); + SCOPE_EXIT(library->get("ClickHouseDictionary_v3_dataDelete")(lib_data, data_ptr)); + auto * data = func_load_keys(data_ptr, &settings->strings, &request_cols); auto block = dataToBlock(description.sample_block, data); - SCOPE_EXIT(library->get("ClickHouseDictionary_v3_dataDelete")(lib_data, data_ptr)); + return std::make_shared(block); } From eb3710c164b991b8d4f86b1435a65f9eceb8f1f5 Mon Sep 17 00:00:00 2001 From: Azat Khuzhin Date: Sat, 13 Mar 2021 08:39:08 +0300 Subject: [PATCH 391/544] Fix error message in clickhouse-test {} was not replaced: Cannot get server pid with {}, got {}: {} lsof -i tcp:9000 -s tcp:LISTEN -Fp | awk '/^p[0-9]+$/{print substr($0, 2)}' b'306\n1025011\n' invalid literal for int() with base 10: b'306\n1025011\n' --- tests/clickhouse-test | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/clickhouse-test b/tests/clickhouse-test index 198c8bbe0c9..b7ffa3a8d6d 100755 --- a/tests/clickhouse-test +++ b/tests/clickhouse-test @@ -245,11 +245,11 @@ def get_server_pid(server_tcp_port): output = None for cmd in commands: try: - output = subprocess.check_output(cmd, shell=True, stderr=subprocess.STDOUT) + output = subprocess.check_output(cmd, shell=True, stderr=subprocess.STDOUT, universal_newlines=True) if output: return int(output) except Exception as e: - print("Cannot get server pid with {}, got {}: {}", cmd, output, e) + print("Cannot get server pid with {}, got {}: {}".format(cmd, output, e)) return None # most likely server dead From 481d897cadb6b1a309478f24a46efe506b7108d6 Mon Sep 17 00:00:00 2001 From: Azat Khuzhin Date: Sat, 13 Mar 2021 09:05:35 +0300 Subject: [PATCH 392/544] Fix SIGSEGV on not existing attributes from ip_trie with access_to_key_from_attributes Found with fuzzer [1]: [1]: https://clickhouse-test-reports.s3.yandex.net/21674/2e99dad56213413a72969d47650050b42cbcc073/fuzzer_asan/report.html#fail1 --- src/Dictionaries/DictionaryStructure.cpp | 2 ++ tests/queries/0_stateless/01018_ip_dictionary.sql | 3 +++ 2 files changed, 5 insertions(+) diff --git a/src/Dictionaries/DictionaryStructure.cpp b/src/Dictionaries/DictionaryStructure.cpp index 25e29d7e0e8..166f33b9a00 100644 --- a/src/Dictionaries/DictionaryStructure.cpp +++ b/src/Dictionaries/DictionaryStructure.cpp @@ -241,6 +241,8 @@ const DictionaryAttribute & DictionaryStructure::getAttribute(const std::string for (const auto & key_attribute : *key) if (key_attribute.name == attribute_name) return key_attribute; + + throw Exception{"No such attribute '" + attribute_name + "'", ErrorCodes::BAD_ARGUMENTS}; } size_t attribute_index = it->second; diff --git a/tests/queries/0_stateless/01018_ip_dictionary.sql b/tests/queries/0_stateless/01018_ip_dictionary.sql index 5df1afcd559..2abd51cc9fe 100644 --- a/tests/queries/0_stateless/01018_ip_dictionary.sql +++ b/tests/queries/0_stateless/01018_ip_dictionary.sql @@ -41,6 +41,9 @@ SOURCE(CLICKHOUSE(host 'localhost' port 9000 user 'default' db 'database_for_dic LAYOUT(IP_TRIE()) LIFETIME(MIN 10 MAX 100); +-- fuzzer +SELECT '127.0.0.0/24' = dictGetString('database_for_dict.dict_ipv4_trie', 'prefixprefixprefixprefix', tuple(IPv4StringToNum('127.0.0.0127.0.0.0'))); -- { serverError 36 } + SELECT 0 == dictGetUInt32('database_for_dict.dict_ipv4_trie', 'asn', tuple(IPv4StringToNum('0.0.0.0'))); SELECT 1 == dictGetUInt32('database_for_dict.dict_ipv4_trie', 'asn', tuple(IPv4StringToNum('128.0.0.0'))); SELECT 2 == dictGetUInt32('database_for_dict.dict_ipv4_trie', 'asn', tuple(IPv4StringToNum('192.0.0.0'))); From 1ce9293e9773669be48b03a3f5ea4542b253aa78 Mon Sep 17 00:00:00 2001 From: Azat Khuzhin Date: Sat, 13 Mar 2021 10:31:19 +0300 Subject: [PATCH 393/544] Fix 01035_lc_empty_part_bug under stress tests Found in [1]: 2021.03.12 23:32:58.897190 [ 436606 ] {97a08a8f-4630-44f8-ba18-a94c6fabbded} test_13.lc_empty_part_bug: Waiting mutation: mutation_3.txt ... 2021.03.12 23:32:58.900444 [ 374 ] {} test_13.lc_empty_part_bug: Current max source part size for mutation is 0 but part size 1. Will not mutate part all_1_1_0_2. Max size depends not only on available space, but also on settings 'number_of_free_entries_in_pool_to_execute_mutation' and 'background_pool_size' ... Hang check: query: alter table lc_empty_part_bug modify column s LowCardinality(String); elapsed: 2156.396613506 [1]: https://clickhouse-test-reports.s3.yandex.net/21511/d7d9638ceb6bd702f34a88ee54f8f83197e90af5/stress_test_(address).html#fail1 --- tests/queries/0_stateless/01035_lc_empty_part_bug.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/queries/0_stateless/01035_lc_empty_part_bug.sh b/tests/queries/0_stateless/01035_lc_empty_part_bug.sh index b65cf87d1ca..185c4ef4a4e 100755 --- a/tests/queries/0_stateless/01035_lc_empty_part_bug.sh +++ b/tests/queries/0_stateless/01035_lc_empty_part_bug.sh @@ -8,7 +8,7 @@ CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) ${CLICKHOUSE_CLIENT} --multiquery --query=" DROP TABLE IF EXISTS lc_empty_part_bug; - create table lc_empty_part_bug (id UInt64, s String) Engine=MergeTree ORDER BY id; + create table lc_empty_part_bug (id UInt64, s String) Engine=MergeTree ORDER BY id SETTINGS number_of_free_entries_in_pool_to_execute_mutation=0; insert into lc_empty_part_bug select number as id, toString(rand()) from numbers(100); alter table lc_empty_part_bug delete where id < 100; " --mutations_sync=1 From a301aa64dae1c879f121ee02dd64e6c0b3bae4a9 Mon Sep 17 00:00:00 2001 From: Azat Khuzhin Date: Sat, 13 Mar 2021 10:37:05 +0300 Subject: [PATCH 394/544] Do not use system-wide SYSTEM STOP TTL MERGES, use per-table instead Fixes 01294_lazy_database_concurrent_recreate_reattach_and_show_tables --- ...y_database_concurrent_recreate_reattach_and_show_tables.sh | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/tests/queries/0_stateless/01294_lazy_database_concurrent_recreate_reattach_and_show_tables.sh b/tests/queries/0_stateless/01294_lazy_database_concurrent_recreate_reattach_and_show_tables.sh index d8f72c7837d..f5a4a1adac0 100755 --- a/tests/queries/0_stateless/01294_lazy_database_concurrent_recreate_reattach_and_show_tables.sh +++ b/tests/queries/0_stateless/01294_lazy_database_concurrent_recreate_reattach_and_show_tables.sh @@ -70,8 +70,10 @@ function recreate_lazy_func4() function test_func() { while true; do - $CLICKHOUSE_CLIENT -q "SYSTEM STOP TTL MERGES"; + for table in log tlog slog tlog2; do + $CLICKHOUSE_CLIENT -q "SYSTEM STOP TTL MERGES $CURR_DATABASE.$table" >& /dev/null done + done } From 6d91881f6af68efe6e9ceaa8f82b11ab518782fd Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Sat, 13 Mar 2021 10:38:35 +0300 Subject: [PATCH 395/544] Remove currently unused code --- base/glibc-compatibility/memcpy/memcpy.h | 113 +---------------------- programs/main.cpp | 9 -- 2 files changed, 5 insertions(+), 117 deletions(-) diff --git a/base/glibc-compatibility/memcpy/memcpy.h b/base/glibc-compatibility/memcpy/memcpy.h index b651e6c3b62..3e1ef793f81 100644 --- a/base/glibc-compatibility/memcpy/memcpy.h +++ b/base/glibc-compatibility/memcpy/memcpy.h @@ -2,27 +2,13 @@ #include -#if defined(__clang__) && defined(__has_feature) -# define ch_has_feature __has_feature -#endif -#if !defined(MEMORY_SANITIZER) -# if defined(ch_has_feature) -# if ch_has_feature(memory_sanitizer) -# define MEMORY_SANITIZER 1 -# endif -# elif defined(__MEMORY_SANITIZER__) -# define MEMORY_SANITIZER 1 -# endif -#endif - -//extern bool have_avx; -void init_memcpy(); - - -static inline char * inline_memcpy(char * __restrict dst, const char * __restrict src, size_t size) +static inline void * inline_memcpy(void * __restrict dst_, const void * __restrict src_, size_t size) { - char * ret = dst; + char * __restrict dst = reinterpret_cast(dst_); + const char * __restrict src = reinterpret_cast(src_); + + void * ret = dst; tail: if (size <= 16) @@ -47,95 +33,6 @@ tail: *dst = *src; } } -/* -#if !defined(MEMORY_SANITIZER) /// Asm code is not instrumented by MSan, skip this branch - else if (have_avx) - { - if (size <= 32) - { - __builtin_memcpy(dst, src, 8); - __builtin_memcpy(dst + 8, src + 8, 8); - - dst += 16; - src += 16; - size -= 16; - - goto tail; - } - - if (size <= 256) - { - __asm__( - "vmovups -0x20(%[s],%[size],1), %%ymm0\n" - "vmovups %%ymm0, -0x20(%[d],%[size],1)\n" - : [d]"+r"(dst), [s]"+r"(src) - : [size]"r"(size) - : "ymm0", "memory"); - - while (size > 32) - { - __asm__( - "vmovups (%[s]), %%ymm0\n" - "vmovups %%ymm0, (%[d])\n" - : [d]"+r"(dst), [s]"+r"(src) - : - : "ymm0", "memory"); - - dst += 32; - src += 32; - size -= 32; - } - } - else - { - size_t padding = (32 - (reinterpret_cast(dst) & 31)) & 31; - - if (padding > 0) - { - __asm__( - "vmovups (%[s]), %%ymm0\n" - "vmovups %%ymm0, (%[d])\n" - : [d]"+r"(dst), [s]"+r"(src) - : - : "ymm0", "memory"); - - dst += padding; - src += padding; - size -= padding; - } - - while (size >= 256) - { - __asm__( - "vmovups (%[s]), %%ymm0\n" - "vmovups 0x20(%[s]), %%ymm1\n" - "vmovups 0x40(%[s]), %%ymm2\n" - "vmovups 0x60(%[s]), %%ymm3\n" - "vmovups 0x80(%[s]), %%ymm4\n" - "vmovups 0xa0(%[s]), %%ymm5\n" - "vmovups 0xc0(%[s]), %%ymm6\n" - "vmovups 0xe0(%[s]), %%ymm7\n" - "add $0x100,%[s]\n" - "vmovaps %%ymm0, (%[d])\n" - "vmovaps %%ymm1, 0x20(%[d])\n" - "vmovaps %%ymm2, 0x40(%[d])\n" - "vmovaps %%ymm3, 0x60(%[d])\n" - "vmovaps %%ymm4, 0x80(%[d])\n" - "vmovaps %%ymm5, 0xa0(%[d])\n" - "vmovaps %%ymm6, 0xc0(%[d])\n" - "vmovaps %%ymm7, 0xe0(%[d])\n" - "add $0x100, %[d]\n" - : [d]"+r"(dst), [s]"+r"(src) - : - : "ymm0", "ymm1", "ymm2", "ymm3", "ymm4", "ymm5", "ymm6", "ymm7", "memory"); - - size -= 256; - } - - goto tail; - } - } -#endif*/ else { if (size <= 128) diff --git a/programs/main.cpp b/programs/main.cpp index 447a3518dcc..cbb22b7a87b 100644 --- a/programs/main.cpp +++ b/programs/main.cpp @@ -23,13 +23,6 @@ #include #include -/// Custom memcpy - only for x86_64 and not for Arcadia. -#if defined(__x86_64__) && !defined(ARCADIA_BUILD) -#include // Y_IGNORE -#else -void init_memcpy() {} -#endif - /// Universal executable for various clickhouse applications #if ENABLE_CLICKHOUSE_SERVER @@ -349,8 +342,6 @@ int main(int argc_, char ** argv_) inside_main = true; SCOPE_EXIT({ inside_main = false; }); - init_memcpy(); - /// Reset new handler to default (that throws std::bad_alloc) /// It is needed because LLVM library clobbers it. std::set_new_handler(nullptr); From a446612e8aea837f0aa49dc20a2d8c1af5d575c9 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Sat, 13 Mar 2021 10:39:43 +0300 Subject: [PATCH 396/544] Remove currently unused code --- base/glibc-compatibility/memcpy/memcpy.cpp | 18 +----------------- 1 file changed, 1 insertion(+), 17 deletions(-) diff --git a/base/glibc-compatibility/memcpy/memcpy.cpp b/base/glibc-compatibility/memcpy/memcpy.cpp index c67e4eb54ca..ec43a2c3649 100644 --- a/base/glibc-compatibility/memcpy/memcpy.cpp +++ b/base/glibc-compatibility/memcpy/memcpy.cpp @@ -1,22 +1,6 @@ #include "memcpy.h" -#include -#include - - -bool have_avx = false; - -void init_memcpy() -{ - uint32_t eax; - uint32_t ebx; - uint32_t ecx; - uint32_t edx; - __cpuid(1, eax, ebx, ecx, edx); - have_avx = (ecx >> 28) & 1; -} - extern "C" void * memcpy(void * __restrict dst, const void * __restrict src, size_t size) { - return inline_memcpy(reinterpret_cast(dst), reinterpret_cast(src), size); + return inline_memcpy(dst, src, size); } From 68dff2d954bb02495c66d36c1e5a8ee8eeba7948 Mon Sep 17 00:00:00 2001 From: Azat Khuzhin Date: Sat, 13 Mar 2021 11:06:49 +0300 Subject: [PATCH 397/544] Mark 01018_ip_dictionary as long https://clickhouse-test-reports.s3.yandex.net/21692/481d897cadb6b1a309478f24a46efe506b7108d6/functional_stateless_tests_flaky_check_(address).html#fail1 --- ..._dictionary.reference => 01018_ip_dictionary_long.reference} | 0 .../{01018_ip_dictionary.sql => 01018_ip_dictionary_long.sql} | 0 tests/queries/0_stateless/arcadia_skip_list.txt | 2 +- tests/queries/query_test.py | 2 +- tests/queries/skip_list.json | 2 +- 5 files changed, 3 insertions(+), 3 deletions(-) rename tests/queries/0_stateless/{01018_ip_dictionary.reference => 01018_ip_dictionary_long.reference} (100%) rename tests/queries/0_stateless/{01018_ip_dictionary.sql => 01018_ip_dictionary_long.sql} (100%) diff --git a/tests/queries/0_stateless/01018_ip_dictionary.reference b/tests/queries/0_stateless/01018_ip_dictionary_long.reference similarity index 100% rename from tests/queries/0_stateless/01018_ip_dictionary.reference rename to tests/queries/0_stateless/01018_ip_dictionary_long.reference diff --git a/tests/queries/0_stateless/01018_ip_dictionary.sql b/tests/queries/0_stateless/01018_ip_dictionary_long.sql similarity index 100% rename from tests/queries/0_stateless/01018_ip_dictionary.sql rename to tests/queries/0_stateless/01018_ip_dictionary_long.sql diff --git a/tests/queries/0_stateless/arcadia_skip_list.txt b/tests/queries/0_stateless/arcadia_skip_list.txt index f28eac03c37..57994712e15 100644 --- a/tests/queries/0_stateless/arcadia_skip_list.txt +++ b/tests/queries/0_stateless/arcadia_skip_list.txt @@ -176,7 +176,7 @@ 01560_timeseriesgroupsum_segfault 00976_ttl_with_old_parts 01584_distributed_buffer_cannot_find_column -01018_ip_dictionary +01018_ip_dictionary_long 01582_distinct_subquery_groupby 01558_ttest 01558_ttest_scipy diff --git a/tests/queries/query_test.py b/tests/queries/query_test.py index 417a51fe523..6db183b9684 100644 --- a/tests/queries/query_test.py +++ b/tests/queries/query_test.py @@ -26,7 +26,7 @@ SKIP_LIST = [ "00990_metric_log_table_not_empty", "01014_lazy_database_concurrent_recreate_reattach_and_show_tables", "01018_Distributed__shard_num", - "01018_ip_dictionary", + "01018_ip_dictionary_long", "01050_clickhouse_dict_source_with_subquery", "01053_ssd_dictionary", "01054_cache_dictionary_overflow_cell", diff --git a/tests/queries/skip_list.json b/tests/queries/skip_list.json index cc34b828d36..92a32e98727 100644 --- a/tests/queries/skip_list.json +++ b/tests/queries/skip_list.json @@ -592,7 +592,7 @@ "01018_ddl_dictionaries_select", "01018_ddl_dictionaries_special", "01018_dictionaries_from_dictionaries", - "01018_ip_dictionary", + "01018_ip_dictionary_long", "01021_only_tuple_columns", "01023_materialized_view_query_context", "01031_mutations_interpreter_and_context", From 2d66a5c393cf7b42309b82050e197ad2f1709934 Mon Sep 17 00:00:00 2001 From: Azat Khuzhin Date: Sat, 13 Mar 2021 11:29:40 +0300 Subject: [PATCH 398/544] Fix hang check for 00966_live_view_watch_events_http If INSERT was failed then WATCH EVENTS will hang: 2021.03.12 23:45:33.666223 [ 1901 ] {} DynamicQueryHandler: Request URI: /?allow_experimental_live_view=1&query=WATCH%20test.lv%20EVENTS ... 2021.03.12 23:45:34.119465 [ 35492 ] {55f2c8ec-df9f-4e97-9acc-385ed5b5fde2} executeQuery: Code: 241, e.displayText() = DB::Exception: Memory limit (for user) exceeded: would use 153.99 MiB (attempt to allocate chunk of 4194729 bytes), maximum: 150.00 MiB (version 21.4.1.6239) (from [::1]:43910) (in query: INSERT INTO test.mt VALUES ), Stack trace (when copying this message, always include the lines below): ... Hung check: elapsed: 597.835824299 is_cancelled: 0 query: WATCH test.lv EVENTS thread_ids: [1901] gdb: Thread 133 (Thread 0x7f26ac95b700 (LWP 1901)): 0 0x00007f2793d447b1 in pthread_cond_timedwait@@GLIBC_2.3.2 () from /lib/x86_64-linux-gnu/libpthread.so.0 1 0x0000000008cbd5c5 in __tsan::call_pthread_cancel_with_cleanup(int (*)(void*, void*, void*), void*, void*, void*, void (*)(void*), void*) () 2 0x0000000008c4f134 in cond_wait(__tsan::ThreadState*, unsigned long, __tsan::ScopedInterceptor*, int (*)(void*, void*, void*), void*, void*, void*) () 3 0x0000000008c4f339 in pthread_cond_timedwait () 4 0x000000001831c8b6 in std::__1::__libcpp_condvar_timedwait (__cv=0x7b5801053520, __m=0x189, __ts=0x7f26ac9132b8) at ../contrib/libcxx/include/__threading_support:442 5 std::__1::condition_variable::__do_timed_wait (this=0x7b5801053520, lk=..., tp=...) at ../contrib/libcxx/src/condition_variable.cpp:74 6 0x00000000134fe7f2 in std::__1::condition_variable::wait_for > (this=this@entry=0x7b5801053520, __lk=..., __d=...) at ../contrib/libcxx/include/__mutex_base:462 7 0x00000000134fe2c9 in DB::LiveViewEventsBlockInputStream::tryReadImpl (this=this@entry=0x7b5c0070ff98, blocking=false) at ../src/Storages/LiveView/LiveViewEventsBlockInputStream.h:180 8 0x00000000134fd6e4 in DB::LiveViewEventsBlockInputStream::readImpl (this=0x189) at ../src/Storages/LiveView/LiveViewEventsBlockInputStream.h:115 9 0x0000000012285593 in DB::IBlockInputStream::read (this=0x7b5c0070ff98) at ../src/DataStreams/IBlockInputStream.cpp:58 10 0x00000000122afc7e in DB::copyDataImpl const&, std::__1::function const&>(DB::IBlockInputStream&, DB::IBlockOutputStream&, std::__1::function const&, std::__1::function const&) (from=..., to=..., is_cancelled=..., progress=...) at ../src/DataStreams/copyData.cpp:26 11 0x00000000122af8e1 in DB::copyData(DB::IBlockInputStream&, DB::IBlockOutputStream&, std::__1::function const&, std::__1::function const&) (from=..., to=..., is_cancelled=..., progress=...) at ../src/DataStreams/copyData.cpp:73 12 0x0000000012cfc5e2 in DB::executeQuery(DB::ReadBuffer&, DB::WriteBuffer&, bool, DB::Context&, std::__1::function, std::__1::allocator > const&, std::__1::basic_string, std::__1::allocator > const&, std::__1::basic_string, std::__1::allocator > const&, std::__1::basic_string, std::__1::allocator > const&)>) (istr=..., ostr=..., allow_into_outfile=false, context=..., set_result_details=...) at ../src/Interpreters/executeQuery.cpp:1033 13 0x000000001357941d in DB::HTTPHandler::processQuery (this=this@entry=0x7b1c006f41b0, context=..., request=..., params=..., response=..., used_output=..., query_scope=...) at ../src/Server/HTTPHandler.cpp:772 14 0x000000001357c507 in DB::HTTPHandler::handleRequest (this=, request=..., response=...) at ../src/Server/HTTPHandler.cpp:910 15 0x00000000135f3081 in DB::HTTPServerConnection::run (this=) at ../src/Server/HTTP/HTTPServerConnection.cpp:48 16 0x00000000161f9e73 in Poco::Net::TCPServerConnection::start (this=0x7b8c01554000) at ../contrib/poco/Net/src/TCPServerConnection.cpp:43 17 0x00000000161fa59f in Poco::Net::TCPServerDispatcher::run (this=) at ../contrib/poco/Net/src/TCPServerDispatcher.cpp:113 18 0x0000000016361732 in Poco::PooledThread::run (this=) at ../contrib/poco/Foundation/src/ThreadPool.cpp:199 19 0x000000001635fcd0 in Poco::(anonymous namespace)::RunnableHolder::run (this=) at ../contrib/poco/Foundation/src/Thread.cpp:55 20 0x000000001635e4d8 in Poco::ThreadImpl::runnableEntry (pThread=0x7b5400082538) at ../contrib/poco/Foundation/src/Thread_POSIX.cpp:345 21 0x0000000008c4e36d in __tsan_thread_start_func () 22 0x00007f2793d3d609 in start_thread () from /lib/x86_64-linux-gnu/libpthread.so.0 23 0x00007f2793c64293 in clone () from /lib/x86_64-linux-gnu/libc.so.6 Cc: @vzakaznikov --- .../00966_live_view_watch_events_http.py | 19 +++++++++--------- .../0_stateless/00967_live_view_watch_http.py | 20 +++++++++---------- 2 files changed, 20 insertions(+), 19 deletions(-) diff --git a/tests/queries/0_stateless/00966_live_view_watch_events_http.py b/tests/queries/0_stateless/00966_live_view_watch_events_http.py index 3d407ec5602..1c00a5d1236 100755 --- a/tests/queries/0_stateless/00966_live_view_watch_events_http.py +++ b/tests/queries/0_stateless/00966_live_view_watch_events_http.py @@ -28,13 +28,14 @@ with client(name='client1>', log=log) as client1: client1.expect(prompt) - with http_client({'method':'GET', 'url': '/?allow_experimental_live_view=1&query=WATCH%20test.lv%20EVENTS'}, name='client2>', log=log) as client2: - client2.expect('.*1\n') - client1.send('INSERT INTO test.mt VALUES (1),(2),(3)') + try: + with http_client({'method':'GET', 'url': '/?allow_experimental_live_view=1&query=WATCH%20test.lv%20EVENTS'}, name='client2>', log=log) as client2: + client2.expect('.*1\n') + client1.send('INSERT INTO test.mt VALUES (1),(2),(3)') + client1.expect(prompt) + client2.expect('.*2\n') + finally: + client1.send('DROP TABLE test.lv') + client1.expect(prompt) + client1.send('DROP TABLE test.mt') client1.expect(prompt) - client2.expect('.*2\n') - - client1.send('DROP TABLE test.lv') - client1.expect(prompt) - client1.send('DROP TABLE test.mt') - client1.expect(prompt) diff --git a/tests/queries/0_stateless/00967_live_view_watch_http.py b/tests/queries/0_stateless/00967_live_view_watch_http.py index d26bb5402e7..c41b9f0c861 100755 --- a/tests/queries/0_stateless/00967_live_view_watch_http.py +++ b/tests/queries/0_stateless/00967_live_view_watch_http.py @@ -27,14 +27,14 @@ with client(name='client1>', log=log) as client1: client1.send('CREATE LIVE VIEW test.lv AS SELECT sum(a) FROM test.mt') client1.expect(prompt) - - with http_client({'method':'GET', 'url':'/?allow_experimental_live_view=1&query=WATCH%20test.lv'}, name='client2>', log=log) as client2: - client2.expect('.*0\t1\n') - client1.send('INSERT INTO test.mt VALUES (1),(2),(3)') + try: + with http_client({'method':'GET', 'url':'/?allow_experimental_live_view=1&query=WATCH%20test.lv'}, name='client2>', log=log) as client2: + client2.expect('.*0\t1\n') + client1.send('INSERT INTO test.mt VALUES (1),(2),(3)') + client1.expect(prompt) + client2.expect('.*6\t2\n') + finally: + client1.send('DROP TABLE test.lv') + client1.expect(prompt) + client1.send('DROP TABLE test.mt') client1.expect(prompt) - client2.expect('.*6\t2\n') - - client1.send('DROP TABLE test.lv') - client1.expect(prompt) - client1.send('DROP TABLE test.mt') - client1.expect(prompt) From 0c1b2f6d59609254f0d4209e367a4a4bc352204a Mon Sep 17 00:00:00 2001 From: Azat Khuzhin Date: Sat, 13 Mar 2021 12:52:56 +0300 Subject: [PATCH 399/544] Update exception message for lack of attribute with access_to_key_from_attributes --- src/Dictionaries/DictionaryStructure.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Dictionaries/DictionaryStructure.cpp b/src/Dictionaries/DictionaryStructure.cpp index 166f33b9a00..537698a2015 100644 --- a/src/Dictionaries/DictionaryStructure.cpp +++ b/src/Dictionaries/DictionaryStructure.cpp @@ -242,7 +242,7 @@ const DictionaryAttribute & DictionaryStructure::getAttribute(const std::string if (key_attribute.name == attribute_name) return key_attribute; - throw Exception{"No such attribute '" + attribute_name + "'", ErrorCodes::BAD_ARGUMENTS}; + throw Exception{"No such attribute '" + attribute_name + "' in keys", ErrorCodes::BAD_ARGUMENTS}; } size_t attribute_index = it->second; From 092b0b49f02bda79fe21b8ffd1b65b76ab2e51b3 Mon Sep 17 00:00:00 2001 From: Azat Khuzhin Date: Sat, 13 Mar 2021 08:56:54 +0300 Subject: [PATCH 400/544] Set SOCK_CLOEXEC for sockets (hardcoded via poco update) Found this in [1]: [Detaching after fork from child process 184152] ... Cannot get server pid with {}, got {}: {} lsof -i tcp:9000 -s tcp:LISTEN -Fp | awk '/^p[0-9]+$/{print substr($0, 2)}' b'301\n184152\n' invalid literal for int() with base 10: b'301\n184152\n' But the major idea is that parent process should not export any file descriptors to childrens, otherwise this may create security and other breaches (like extra pipe endpoints, ...) [1]: https://clickhouse-test-reports.s3.yandex.net/21511/d7d9638ceb6bd702f34a88ee54f8f83197e90af5/stress_test_(address).html#fail1 Refs: https://github.com/ClickHouse-Extras/poco/pull/35 --- contrib/poco | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/contrib/poco b/contrib/poco index c55b91f394e..83beecccb09 160000 --- a/contrib/poco +++ b/contrib/poco @@ -1 +1 @@ -Subproject commit c55b91f394efa9c238c33957682501681ef9b716 +Subproject commit 83beecccb09eec0c9fd2669cacea03ede1d9f138 From 833dbf122572f209a79ddeae9bcf3a626be69915 Mon Sep 17 00:00:00 2001 From: Azat Khuzhin Date: Sat, 13 Mar 2021 13:42:47 +0300 Subject: [PATCH 401/544] Mark 01294_lazy_database_concurrent_recreate_reattach_and_show_tables as long --- ...ncurrent_recreate_reattach_and_show_tables_long.reference} | 0 ...base_concurrent_recreate_reattach_and_show_tables_long.sh} | 0 tests/queries/query_test.py | 2 +- tests/queries/skip_list.json | 4 ++-- 4 files changed, 3 insertions(+), 3 deletions(-) rename tests/queries/0_stateless/{01294_lazy_database_concurrent_recreate_reattach_and_show_tables.reference => 01294_lazy_database_concurrent_recreate_reattach_and_show_tables_long.reference} (100%) rename tests/queries/0_stateless/{01294_lazy_database_concurrent_recreate_reattach_and_show_tables.sh => 01294_lazy_database_concurrent_recreate_reattach_and_show_tables_long.sh} (100%) diff --git a/tests/queries/0_stateless/01294_lazy_database_concurrent_recreate_reattach_and_show_tables.reference b/tests/queries/0_stateless/01294_lazy_database_concurrent_recreate_reattach_and_show_tables_long.reference similarity index 100% rename from tests/queries/0_stateless/01294_lazy_database_concurrent_recreate_reattach_and_show_tables.reference rename to tests/queries/0_stateless/01294_lazy_database_concurrent_recreate_reattach_and_show_tables_long.reference diff --git a/tests/queries/0_stateless/01294_lazy_database_concurrent_recreate_reattach_and_show_tables.sh b/tests/queries/0_stateless/01294_lazy_database_concurrent_recreate_reattach_and_show_tables_long.sh similarity index 100% rename from tests/queries/0_stateless/01294_lazy_database_concurrent_recreate_reattach_and_show_tables.sh rename to tests/queries/0_stateless/01294_lazy_database_concurrent_recreate_reattach_and_show_tables_long.sh diff --git a/tests/queries/query_test.py b/tests/queries/query_test.py index 417a51fe523..adb26820478 100644 --- a/tests/queries/query_test.py +++ b/tests/queries/query_test.py @@ -46,7 +46,7 @@ SKIP_LIST = [ "01293_client_interactive_vertical_singleline", # expect-test "01293_system_distribution_queue", # FLAKY "01293_show_clusters", - "01294_lazy_database_concurrent_recreate_reattach_and_show_tables", + "01294_lazy_database_concurrent_recreate_reattach_and_show_tables_long", "01294_system_distributed_on_cluster", "01300_client_save_history_when_terminated", # expect-test "01304_direct_io", diff --git a/tests/queries/skip_list.json b/tests/queries/skip_list.json index cc34b828d36..a24ed327fbd 100644 --- a/tests/queries/skip_list.json +++ b/tests/queries/skip_list.json @@ -396,7 +396,7 @@ "01293_pretty_max_value_width", "01293_show_settings", "01294_create_settings_profile", - "01294_lazy_database_concurrent_recreate_reattach_and_show_tables", + "01294_lazy_database_concurrent_recreate_reattach_and_show_tables_long", "01295_create_row_policy", "01296_create_row_policy_in_current_database", "01297_create_quota", @@ -681,7 +681,7 @@ "01281_unsucceeded_insert_select_queries_counter", "01293_system_distribution_queue", "01294_lazy_database_concurrent", - "01294_lazy_database_concurrent_recreate_reattach_and_show_tables", + "01294_lazy_database_concurrent_recreate_reattach_and_show_tables_long", "01294_system_distributed_on_cluster", "01296_create_row_policy_in_current_database", "01297_create_quota", From 6be5c271ae0aea1797ef04ba08ca0d7b76a7dad4 Mon Sep 17 00:00:00 2001 From: Maksim Kita Date: Sat, 13 Mar 2021 14:09:56 +0300 Subject: [PATCH 402/544] Added explicit function interface --- src/Dictionaries/LibraryDictionarySource.cpp | 66 ++++++++----------- .../LibraryDictionarySourceExternal.cpp | 21 +++++- .../LibraryDictionarySourceExternal.h | 46 +++++++++++++ 3 files changed, 94 insertions(+), 39 deletions(-) diff --git a/src/Dictionaries/LibraryDictionarySource.cpp b/src/Dictionaries/LibraryDictionarySource.cpp index 0e692a5893a..0632dd3e30f 100644 --- a/src/Dictionaries/LibraryDictionarySource.cpp +++ b/src/Dictionaries/LibraryDictionarySource.cpp @@ -72,7 +72,7 @@ namespace } - Block dataToBlock(const Block & sample_block, const void * data) + Block dataToBlock(const Block & sample_block, const ClickHouseLibrary::RawClickHouseLibraryTable data) { if (!data) throw Exception("LibraryDictionarySource: No data returned", ErrorCodes::EXTERNAL_LIBRARY_ERROR); @@ -150,8 +150,7 @@ LibraryDictionarySource::LibraryDictionarySource( ); settings = std::make_shared(getLibSettings(config, config_prefix + lib_config_settings)); - if (auto lib_new = library->tryGetstrings), decltype(&ClickHouseLibrary::log))>( - "ClickHouseDictionary_v3_libNew")) + if (auto lib_new = library->tryGet(ClickHouseLibrary::LIBRARY_CREATE_NEW_FUNC_NAME)) lib_data = lib_new(&settings->strings, ClickHouseLibrary::log); } @@ -165,17 +164,15 @@ LibraryDictionarySource::LibraryDictionarySource(const LibraryDictionarySource & , description{other.description} , settings{other.settings} { - if (auto lib_clone = library->tryGet("ClickHouseDictionary_v3_libClone")) + if (auto lib_clone = library->tryGet(ClickHouseLibrary::LIBRARY_CLONE_FUNC_NAME)) lib_data = lib_clone(other.lib_data); - else if ( - auto lib_new = library->tryGetstrings), decltype(&ClickHouseLibrary::log))>( - "ClickHouseDictionary_v3_libNew")) + else if (auto lib_new = library->tryGet(ClickHouseLibrary::LIBRARY_CREATE_NEW_FUNC_NAME)) lib_data = lib_new(&settings->strings, ClickHouseLibrary::log); } LibraryDictionarySource::~LibraryDictionarySource() { - if (auto lib_delete = library->tryGet("ClickHouseDictionary_v3_libDelete")) + if (auto lib_delete = library->tryGet(ClickHouseLibrary::LIBRARY_DELETE_FUNC_NAME)) lib_delete(lib_data); } @@ -193,17 +190,14 @@ BlockInputStreamPtr LibraryDictionarySource::loadAll() ++i; } - void * data_ptr = nullptr; + auto load_all_func = library->get(ClickHouseLibrary::LIBRARY_LOAD_ALL_FUNC_NAME); + auto data_new_func = library->get(ClickHouseLibrary::LIBRARY_DATA_NEW_FUNC_NAME); + auto data_delete_func = library->get(ClickHouseLibrary::LIBRARY_DATA_DELETE_FUNC_NAME); - /// Get function pointer before dataNew call because library->get may throw. - auto func_load_all - = library->getstrings), decltype(&columns))>("ClickHouseDictionary_v3_loadAll"); - - data_ptr = library->get("ClickHouseDictionary_v3_dataNew")(lib_data); - SCOPE_EXIT(library->get("ClickHouseDictionary_v3_dataDelete")(lib_data, data_ptr)); - - auto * data = func_load_all(data_ptr, &settings->strings, &columns); + ClickHouseLibrary::LibraryData data_ptr = data_new_func(lib_data); + SCOPE_EXIT(data_delete_func(lib_data, data_ptr)); + ClickHouseLibrary::RawClickHouseLibraryTable data = load_all_func(data_ptr, &settings->strings, &columns); auto block = dataToBlock(description.sample_block, data); return std::make_shared(block); @@ -224,17 +218,14 @@ BlockInputStreamPtr LibraryDictionarySource::loadIds(const std::vector & ++i; } - void * data_ptr = nullptr; + auto load_ids_func = library->get(ClickHouseLibrary::LIBRARY_LOAD_IDS_FUNC_NAME); + auto data_new_func = library->get(ClickHouseLibrary::LIBRARY_DATA_NEW_FUNC_NAME); + auto data_delete_func = library->get(ClickHouseLibrary::LIBRARY_DATA_DELETE_FUNC_NAME); - /// Get function pointer before dataNew call because library->get may throw. - auto func_load_ids - = library->getstrings), decltype(&columns_pass), decltype(&ids_data))>( - "ClickHouseDictionary_v3_loadIds"); + ClickHouseLibrary::LibraryData data_ptr = data_new_func(lib_data); + SCOPE_EXIT(data_delete_func(lib_data, data_ptr)); - data_ptr = library->get("ClickHouseDictionary_v3_dataNew")(lib_data); - SCOPE_EXIT(library->get("ClickHouseDictionary_v3_dataDelete")(lib_data, data_ptr)); - - auto * data = func_load_ids(data_ptr, &settings->strings, &columns_pass, &ids_data); + ClickHouseLibrary::RawClickHouseLibraryTable data = load_ids_func(data_ptr, &settings->strings, &columns_pass, &ids_data); auto block = dataToBlock(description.sample_block, data); return std::make_shared(block); @@ -262,15 +253,14 @@ BlockInputStreamPtr LibraryDictionarySource::loadKeys(const Columns & key_column ClickHouseLibrary::Table request_cols{.data = static_cast(holder.get()), .size = key_columns.size()}; - void * data_ptr = nullptr; - /// Get function pointer before dataNew call because library->get may throw. - auto func_load_keys = library->getstrings), decltype(&request_cols))>( - "ClickHouseDictionary_v3_loadKeys"); + auto load_keys_func = library->get(ClickHouseLibrary::LIBRARY_LOAD_KEYS_FUNC_NAME); + auto data_new_func = library->get(ClickHouseLibrary::LIBRARY_DATA_NEW_FUNC_NAME); + auto data_delete_func = library->get(ClickHouseLibrary::LIBRARY_DATA_DELETE_FUNC_NAME); - data_ptr = library->get("ClickHouseDictionary_v3_dataNew")(lib_data); - SCOPE_EXIT(library->get("ClickHouseDictionary_v3_dataDelete")(lib_data, data_ptr)); + ClickHouseLibrary::LibraryData data_ptr = data_new_func(lib_data); + SCOPE_EXIT(data_delete_func(lib_data, data_ptr)); - auto * data = func_load_keys(data_ptr, &settings->strings, &request_cols); + ClickHouseLibrary::RawClickHouseLibraryTable data = load_keys_func(data_ptr, &settings->strings, &request_cols); auto block = dataToBlock(description.sample_block, data); return std::make_shared(block); @@ -278,17 +268,19 @@ BlockInputStreamPtr LibraryDictionarySource::loadKeys(const Columns & key_column bool LibraryDictionarySource::isModified() const { - if (auto func_is_modified - = library->tryGetstrings))>("ClickHouseDictionary_v3_isModified")) + if (auto func_is_modified = library->tryGet( + ClickHouseLibrary::LIBRARY_IS_MODIFIED_FUNC_NAME)) return func_is_modified(lib_data, &settings->strings); + return true; } bool LibraryDictionarySource::supportsSelectiveLoad() const { - if (auto func_supports_selective_load - = library->tryGetstrings))>("ClickHouseDictionary_v3_supportsSelectiveLoad")) + if (auto func_supports_selective_load = library->tryGet( + ClickHouseLibrary::LIBRARY_SUPPORTS_SELECTIVE_LOAD_FUNC_NAME)) return func_supports_selective_load(lib_data, &settings->strings); + return true; } diff --git a/src/Dictionaries/LibraryDictionarySourceExternal.cpp b/src/Dictionaries/LibraryDictionarySourceExternal.cpp index 2e944056283..eba088c2c55 100644 --- a/src/Dictionaries/LibraryDictionarySourceExternal.cpp +++ b/src/Dictionaries/LibraryDictionarySourceExternal.cpp @@ -6,10 +6,25 @@ namespace const char DICT_LOGGER_NAME[] = "LibraryDictionarySourceExternal"; } -void ClickHouseLibrary::log(ClickHouseLibrary::LogLevel level, ClickHouseLibrary::CString msg) +namespace ClickHouseLibrary { - using ClickHouseLibrary::LogLevel; +std::string LIBRARY_CREATE_NEW_FUNC_NAME = "ClickHouseDictionary_v3_libNew"; +std::string LIBRARY_CLONE_FUNC_NAME = "ClickHouseDictionary_v3_libClone"; +std::string LIBRARY_DELETE_FUNC_NAME = "ClickHouseDictionary_v3_libDelete"; + +std::string LIBRARY_DATA_NEW_FUNC_NAME = "ClickHouseDictionary_v3_dataNew"; +std::string LIBRARY_DATA_DELETE_FUNC_NAME = "ClickHouseDictionary_v3_dataDelete"; + +std::string LIBRARY_LOAD_ALL_FUNC_NAME = "ClickHouseDictionary_v3_loadAll"; +std::string LIBRARY_LOAD_IDS_FUNC_NAME = "ClickHouseDictionary_v3_loadIds"; +std::string LIBRARY_LOAD_KEYS_FUNC_NAME = "ClickHouseDictionary_v3_loadKeys"; + +std::string LIBRARY_IS_MODIFIED_FUNC_NAME = "ClickHouseDictionary_v3_isModified"; +std::string LIBRARY_SUPPORTS_SELECTIVE_LOAD_FUNC_NAME = "ClickHouseDictionary_v3_supportsSelectiveLoad"; + +void log(LogLevel level, CString msg) +{ auto & logger = Poco::Logger::get(DICT_LOGGER_NAME); switch (level) { @@ -47,3 +62,5 @@ void ClickHouseLibrary::log(ClickHouseLibrary::LogLevel level, ClickHouseLibrary break; } } + +} diff --git a/src/Dictionaries/LibraryDictionarySourceExternal.h b/src/Dictionaries/LibraryDictionarySourceExternal.h index 7a031cdb315..64a5f678578 100644 --- a/src/Dictionaries/LibraryDictionarySourceExternal.h +++ b/src/Dictionaries/LibraryDictionarySourceExternal.h @@ -1,6 +1,7 @@ #pragma once #include +#include #define CLICKHOUSE_DICTIONARY_LIBRARY_API 1 @@ -61,4 +62,49 @@ enum LogLevel }; void log(LogLevel level, CString msg); + +extern std::string LIBRARY_CREATE_NEW_FUNC_NAME; +extern std::string LIBRARY_CLONE_FUNC_NAME; +extern std::string LIBRARY_DELETE_FUNC_NAME; + +extern std::string LIBRARY_DATA_NEW_FUNC_NAME; +extern std::string LIBRARY_DATA_DELETE_FUNC_NAME; + +extern std::string LIBRARY_LOAD_ALL_FUNC_NAME; +extern std::string LIBRARY_LOAD_IDS_FUNC_NAME; +extern std::string LIBRARY_LOAD_KEYS_FUNC_NAME; + +extern std::string LIBRARY_IS_MODIFIED_FUNC_NAME; +extern std::string LIBRARY_SUPPORTS_SELECTIVE_LOAD_FUNC_NAME; + +using LibraryContext = void *; + +using LibraryLoggerFunc = void (*)(LogLevel, CString /* message */); + +using LibrarySettings = CStrings *; + +using LibraryNewFunc = LibraryContext (*)(LibrarySettings, LibraryLoggerFunc); +using LibraryCloneFunc = LibraryContext (*)(LibraryContext); +using LibraryDeleteFunc = void (*)(LibraryContext); + +using LibraryData = void *; +using LibraryDataNewFunc = LibraryData (*)(LibraryContext); +using LibraryDataDeleteFunc = void (*)(LibraryContext, LibraryData); + +/// Can be safely casted into const Table * with static_cast +using RawClickHouseLibraryTable = void *; +using RequestedColumnsNames = CStrings *; + +using LibraryLoadAllFunc = RawClickHouseLibraryTable (*)(LibraryData, LibrarySettings, RequestedColumnsNames); + +using RequestedIds = const VectorUInt64 *; +using LibraryLoadIdsFunc = RawClickHouseLibraryTable (*)(LibraryData, LibrarySettings, RequestedColumnsNames, RequestedIds); + +using RequestedKeys = Table *; +/// There is no requested columns names for load keys func +using LibraryLoadKeysFunc = RawClickHouseLibraryTable (*)(LibraryData, LibrarySettings, RequestedKeys); + +using LibraryIsModifiedFunc = bool (*)(LibraryContext, LibrarySettings); +using LibrarySupportsSelectiveLoadFunc = bool (*)(LibraryContext, LibrarySettings); + } From 468ab258bf8401efc1c3247fce668dad3d4866f9 Mon Sep 17 00:00:00 2001 From: Maksim Kita Date: Sat, 13 Mar 2021 14:55:56 +0300 Subject: [PATCH 403/544] Remove protect --- src/Common/PODArray.h | 10 ---------- 1 file changed, 10 deletions(-) diff --git a/src/Common/PODArray.h b/src/Common/PODArray.h index c88a0fa16ff..163a6503d2e 100644 --- a/src/Common/PODArray.h +++ b/src/Common/PODArray.h @@ -597,11 +597,6 @@ public: std::swap(dest.c_end, src.c_end); std::swap(dest.c_end_of_storage, src.c_end_of_storage); } - -#ifndef NDEBUG - this->protect(); - rhs.protect(); -#endif }; if (!this->isInitialized() && !rhs.isInitialized()) @@ -664,11 +659,6 @@ public: std::swap(this->c_end, rhs.c_end); std::swap(this->c_end_of_storage, rhs.c_end_of_storage); } - -#ifndef NDEBUG - this->protect(); - rhs.protect(); -#endif } template From 854bfaf36606f84b1434b08fc5122c29e9d15080 Mon Sep 17 00:00:00 2001 From: Maksim Kita Date: Fri, 29 Jan 2021 17:48:13 +0300 Subject: [PATCH 404/544] RangeHashed dictionary support has function --- src/Dictionaries/RangeHashedDictionary.cpp | 73 +++++++++++++- src/Dictionaries/RangeHashedDictionary.h | 6 ++ src/Functions/FunctionsExternalDictionaries.h | 32 ++++++- .../01676_range_hashed_dictionary.reference | 32 +++++++ .../01676_range_hashed_dictionary.sql | 96 +++++++++++++++++++ 5 files changed, 230 insertions(+), 9 deletions(-) create mode 100644 tests/queries/0_stateless/01676_range_hashed_dictionary.reference create mode 100644 tests/queries/0_stateless/01676_range_hashed_dictionary.sql diff --git a/src/Dictionaries/RangeHashedDictionary.cpp b/src/Dictionaries/RangeHashedDictionary.cpp index f5be04c120d..c87da2c740d 100644 --- a/src/Dictionaries/RangeHashedDictionary.cpp +++ b/src/Dictionaries/RangeHashedDictionary.cpp @@ -52,7 +52,6 @@ namespace ErrorCodes extern const int DICTIONARY_IS_EMPTY; extern const int TYPE_MISMATCH; extern const int UNSUPPORTED_METHOD; - extern const int NOT_IMPLEMENTED; } bool RangeHashedDictionary::Range::isCorrectDate(const RangeStorageType & date) @@ -178,10 +177,76 @@ ColumnPtr RangeHashedDictionary::getColumn( return result; } -ColumnUInt8::Ptr RangeHashedDictionary::hasKeys(const Columns &, const DataTypes &) const +ColumnUInt8::Ptr RangeHashedDictionary::hasKeys(const Columns & key_columns, const DataTypes & key_types) const { - throw Exception(ErrorCodes::NOT_IMPLEMENTED, - "Has not supported", getDictionaryID().getNameForLogs()); + auto range_storage_column = key_columns[1]; + ColumnWithTypeAndName column_to_cast = {range_storage_column->convertToFullColumnIfConst(), key_types[1], ""}; + + auto range_column_storage_type = std::make_shared(); + auto range_column_updated = castColumnAccurate(column_to_cast, range_column_storage_type); + + PaddedPODArray key_backup_storage; + PaddedPODArray range_backup_storage; + + const PaddedPODArray & ids = getColumnVectorData(this, key_columns[0], key_backup_storage); + const PaddedPODArray & dates = getColumnVectorData(this, range_column_updated, range_backup_storage); + + const auto & attribute = attributes.front(); + + ColumnUInt8::Ptr result; + + auto type_call = [&](const auto & dictionary_attribute_type) + { + using Type = std::decay_t; + using AttributeType = typename Type::AttributeType; + using ValueType = DictionaryValueType; + result = hasKeysImpl(attribute, ids, dates); + }; + + callOnDictionaryAttributeType(attribute.type, type_call); + + query_count.fetch_add(ids.size(), std::memory_order_relaxed); + + return result; +} + +template +ColumnUInt8::Ptr RangeHashedDictionary::hasKeysImpl( + const Attribute & attribute, + const PaddedPODArray & ids, + const PaddedPODArray & dates) const +{ + auto result = ColumnUInt8::create(ids.size()); + auto& out = result->getData(); + + const auto & attr = *std::get>(attribute.maps); + + for (const auto row : ext::range(0, ids.size())) + { + const auto it = attr.find(ids[row]); + + if (it) + { + const auto date = dates[row]; + const auto & ranges_and_values = it->getMapped(); + const auto val_it = std::find_if( + std::begin(ranges_and_values), + std::end(ranges_and_values), + [date](const Value & v) + { + return v.range.contains(date); + }); + + if (val_it != std::end(ranges_and_values)) + out[row] = true; + else + out[row] = false; + } + else + out[row] = false; + } + + return result; } void RangeHashedDictionary::createAttributes() diff --git a/src/Dictionaries/RangeHashedDictionary.h b/src/Dictionaries/RangeHashedDictionary.h index 1f93fa75775..7ac2d33c63b 100644 --- a/src/Dictionaries/RangeHashedDictionary.h +++ b/src/Dictionaries/RangeHashedDictionary.h @@ -159,6 +159,12 @@ private: ValueSetter && set_value, DefaultValueExtractor & default_value_extractor) const; + template + ColumnUInt8::Ptr hasKeysImpl( + const Attribute & attribute, + const PaddedPODArray & ids, + const PaddedPODArray & dates) const; + template static void setAttributeValueImpl(Attribute & attribute, const Key id, const Range & range, const Field & value); diff --git a/src/Functions/FunctionsExternalDictionaries.h b/src/Functions/FunctionsExternalDictionaries.h index acf158d42ef..4177d686f57 100644 --- a/src/Functions/FunctionsExternalDictionaries.h +++ b/src/Functions/FunctionsExternalDictionaries.h @@ -54,7 +54,6 @@ namespace ErrorCodes extern const int ILLEGAL_COLUMN; extern const int BAD_ARGUMENTS; extern const int TYPE_MISMATCH; - extern const int NOT_IMPLEMENTED; } @@ -154,13 +153,20 @@ public: String getName() const override { return name; } private: - size_t getNumberOfArguments() const override { return 2; } + size_t getNumberOfArguments() const override { return 0; } + bool isVariadic() const override { return true; } + + bool isDeterministic() const override { return false; } bool useDefaultImplementationForConstants() const final { return true; } + ColumnNumbers getArgumentsThatAreAlwaysConstant() const final { return {0}; } DataTypePtr getReturnTypeImpl(const DataTypes & arguments) const override { + if (arguments.size() < 2) + throw Exception{"Wrong argument count for function " + getName(), ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH}; + if (!isString(arguments[0])) throw Exception{"Illegal type " + arguments[0]->getName() + " of first argument of function " + getName() + ", expected a string.", ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT}; @@ -173,8 +179,6 @@ private: return std::make_shared(); } - bool isDeterministic() const override { return false; } - ColumnPtr executeImpl(const ColumnsWithTypeAndName & arguments, const DataTypePtr & result_type, size_t input_rows_count) const override { /** Do not require existence of the dictionary if the function is called for empty columns. @@ -194,6 +198,24 @@ private: const auto key_column = key_column_with_type.column; const auto key_column_type = WhichDataType(key_column_with_type.type); + ColumnPtr range_col = nullptr; + DataTypePtr range_col_type = nullptr; + + if (dictionary_key_type == DictionaryKeyType::range) + { + if (arguments.size() != 3) + throw Exception{"Wrong argument count for function " + getName() + + " when dictionary has key type range", ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH}; + + range_col = arguments[2].column; + range_col_type = arguments[2].type; + + if (!(range_col_type->isValueRepresentedByInteger() && range_col_type->getSizeOfValueInMemory() <= sizeof(Int64))) + throw Exception{"Illegal type " + range_col_type->getName() + " of fourth argument of function " + + getName() + " must be convertible to Int64.", + ErrorCodes::ILLEGAL_COLUMN}; + } + if (dictionary_key_type == DictionaryKeyType::simple) { if (!key_column_type.isUInt64()) @@ -217,7 +239,7 @@ private: return dictionary->hasKeys(key_columns, key_types); } else - throw Exception(ErrorCodes::NOT_IMPLEMENTED, "Has not supported for range dictionary", dictionary->getDictionaryID().getNameForLogs()); + return dictionary->hasKeys({key_column, range_col}, {std::make_shared(), range_col_type}); } mutable FunctionDictHelper helper; diff --git a/tests/queries/0_stateless/01676_range_hashed_dictionary.reference b/tests/queries/0_stateless/01676_range_hashed_dictionary.reference new file mode 100644 index 00000000000..7b19ddda8cb --- /dev/null +++ b/tests/queries/0_stateless/01676_range_hashed_dictionary.reference @@ -0,0 +1,32 @@ +Dictionary not nullable +dictGet +0.33 +0.42 +0.46 +0.2 +0.4 +dictHas +1 +1 +1 +0 +select columns from dictionary +1 2019-05-05 2019-05-20 0.33 +1 2019-05-21 2019-05-30 0.42 +2 2019-05-21 2019-05-30 0.46 +Dictionary nullable +dictGet +0.33 +0.42 +\N +0.2 +0.4 +dictHas +1 +1 +1 +0 +select columns from dictionary +1 2019-05-05 2019-05-20 0.33 +1 2019-05-21 2019-05-30 0.42 +2 2019-05-21 2019-05-30 \N diff --git a/tests/queries/0_stateless/01676_range_hashed_dictionary.sql b/tests/queries/0_stateless/01676_range_hashed_dictionary.sql new file mode 100644 index 00000000000..a4d56e188c4 --- /dev/null +++ b/tests/queries/0_stateless/01676_range_hashed_dictionary.sql @@ -0,0 +1,96 @@ +DROP DATABASE IF EXISTS database_for_range_dict; + +CREATE DATABASE database_for_range_dict; + +CREATE TABLE database_for_range_dict.date_table +( + CountryID UInt64, + StartDate Date, + EndDate Date, + Tax Float64 +) +ENGINE = MergeTree() +ORDER BY CountryID; + +INSERT INTO database_for_range_dict.date_table VALUES(1, toDate('2019-05-05'), toDate('2019-05-20'), 0.33); +INSERT INTO database_for_range_dict.date_table VALUES(1, toDate('2019-05-21'), toDate('2019-05-30'), 0.42); +INSERT INTO database_for_range_dict.date_table VALUES(2, toDate('2019-05-21'), toDate('2019-05-30'), 0.46); + +CREATE DICTIONARY database_for_range_dict.range_dictionary +( + CountryID UInt64, + StartDate Date, + EndDate Date, + Tax Float64 DEFAULT 0.2 +) +PRIMARY KEY CountryID +SOURCE(CLICKHOUSE(HOST 'localhost' PORT tcpPort() USER 'default' TABLE 'date_table' DB 'database_for_range_dict')) +LIFETIME(MIN 1 MAX 1000) +LAYOUT(RANGE_HASHED()) +RANGE(MIN StartDate MAX EndDate); + +SELECT 'Dictionary not nullable'; +SELECT 'dictGet'; +SELECT dictGet('database_for_range_dict.range_dictionary', 'Tax', toUInt64(1), toDate('2019-05-15')); +SELECT dictGet('database_for_range_dict.range_dictionary', 'Tax', toUInt64(1), toDate('2019-05-29')); +SELECT dictGet('database_for_range_dict.range_dictionary', 'Tax', toUInt64(2), toDate('2019-05-29')); +SELECT dictGet('database_for_range_dict.range_dictionary', 'Tax', toUInt64(2), toDate('2019-05-31')); +SELECT dictGetOrDefault('database_for_range_dict.range_dictionary', 'Tax', toUInt64(2), toDate('2019-05-31'), 0.4); +SELECT 'dictHas'; +SELECT dictHas('database_for_range_dict.range_dictionary', toUInt64(1), toDate('2019-05-15')); +SELECT dictHas('database_for_range_dict.range_dictionary', toUInt64(1), toDate('2019-05-29')); +SELECT dictHas('database_for_range_dict.range_dictionary', toUInt64(2), toDate('2019-05-29')); +SELECT dictHas('database_for_range_dict.range_dictionary', toUInt64(2), toDate('2019-05-31')); +SELECT 'select columns from dictionary'; +SELECT * FROM database_for_range_dict.range_dictionary; + +DROP TABLE database_for_range_dict.date_table; +DROP DICTIONARY database_for_range_dict.range_dictionary; + +CREATE TABLE database_for_range_dict.date_table +( + CountryID UInt64, + StartDate Date, + EndDate Date, + Tax Nullable(Float64) +) +ENGINE = MergeTree() +ORDER BY CountryID; + +INSERT INTO database_for_range_dict.date_table VALUES(1, toDate('2019-05-05'), toDate('2019-05-20'), 0.33); +INSERT INTO database_for_range_dict.date_table VALUES(1, toDate('2019-05-21'), toDate('2019-05-30'), 0.42); +INSERT INTO database_for_range_dict.date_table VALUES(2, toDate('2019-05-21'), toDate('2019-05-30'), NULL); + +CREATE DICTIONARY database_for_range_dict.range_dictionary_nullable +( + CountryID UInt64, + StartDate Date, + EndDate Date, + Tax Nullable(Float64) DEFAULT 0.2 +) +PRIMARY KEY CountryID +SOURCE(CLICKHOUSE(HOST 'localhost' PORT tcpPort() USER 'default' TABLE 'date_table' DB 'database_for_range_dict')) +LIFETIME(MIN 1 MAX 1000) +LAYOUT(RANGE_HASHED()) +RANGE(MIN StartDate MAX EndDate); + +SELECT 'Dictionary nullable'; +SELECT 'dictGet'; +SELECT dictGet('database_for_range_dict.range_dictionary_nullable', 'Tax', toUInt64(1), toDate('2019-05-15')); +SELECT dictGet('database_for_range_dict.range_dictionary_nullable', 'Tax', toUInt64(1), toDate('2019-05-29')); +SELECT dictGet('database_for_range_dict.range_dictionary_nullable', 'Tax', toUInt64(2), toDate('2019-05-29')); +SELECT dictGet('database_for_range_dict.range_dictionary_nullable', 'Tax', toUInt64(2), toDate('2019-05-31')); +SELECT dictGetOrDefault('database_for_range_dict.range_dictionary_nullable', 'Tax', toUInt64(2), toDate('2019-05-31'), 0.4); +SELECT 'dictHas'; +SELECT dictHas('database_for_range_dict.range_dictionary_nullable', toUInt64(1), toDate('2019-05-15')); +SELECT dictHas('database_for_range_dict.range_dictionary_nullable', toUInt64(1), toDate('2019-05-29')); +SELECT dictHas('database_for_range_dict.range_dictionary_nullable', toUInt64(2), toDate('2019-05-29')); +SELECT dictHas('database_for_range_dict.range_dictionary_nullable', toUInt64(2), toDate('2019-05-31')); +SELECT 'select columns from dictionary'; +SELECT * FROM database_for_range_dict.range_dictionary_nullable; + +DROP TABLE database_for_range_dict.date_table; +DROP DICTIONARY database_for_range_dict.range_dictionary_nullable; + +DROP DATABASE database_for_range_dict; + From add444f71c63e3bb88361c55a8a7d0aec83006ab Mon Sep 17 00:00:00 2001 From: Maksim Kita Date: Fri, 29 Jan 2021 23:14:41 +0300 Subject: [PATCH 405/544] Fixed broken test --- tests/queries/0_stateless/01280_ssd_complex_key_dictionary.sql | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/queries/0_stateless/01280_ssd_complex_key_dictionary.sql b/tests/queries/0_stateless/01280_ssd_complex_key_dictionary.sql index 8c304818602..3f351f14863 100644 --- a/tests/queries/0_stateless/01280_ssd_complex_key_dictionary.sql +++ b/tests/queries/0_stateless/01280_ssd_complex_key_dictionary.sql @@ -42,7 +42,6 @@ LAYOUT(COMPLEX_KEY_SSD_CACHE(FILE_SIZE 8192 PATH '/var/lib/clickhouse/clickhouse SELECT 'TEST_SMALL'; SELECT 'VALUE FROM RAM BUFFER'; --- NUMBER_OF_ARGUMENTS_DOESNT_MATCH SELECT dictHas('01280_db.ssd_dict', 'a', tuple('1')); -- { serverError 42 } SELECT dictGetUInt64('01280_db.ssd_dict', 'a', tuple('1', toInt32(3))); From ee8c51f65ebbe0dcfb52270387bf1e958509ffec Mon Sep 17 00:00:00 2001 From: Maksim Kita Date: Sun, 31 Jan 2021 18:14:26 +0300 Subject: [PATCH 406/544] Fixed tests --- .../RangeDictionaryBlockInputStream.h | 98 ++++++++----------- src/Dictionaries/RangeHashedDictionary.cpp | 29 +++--- src/Dictionaries/RangeHashedDictionary.h | 4 +- .../01676_range_hashed_dictionary.reference | 26 +++++ .../01676_range_hashed_dictionary.sql | 14 +++ 5 files changed, 100 insertions(+), 71 deletions(-) diff --git a/src/Dictionaries/RangeDictionaryBlockInputStream.h b/src/Dictionaries/RangeDictionaryBlockInputStream.h index ccd77d49e0f..6531f5cba9d 100644 --- a/src/Dictionaries/RangeDictionaryBlockInputStream.h +++ b/src/Dictionaries/RangeDictionaryBlockInputStream.h @@ -17,14 +17,14 @@ namespace DB * BlockInputStream implementation for external dictionaries * read() returns single block consisting of the in-memory contents of the dictionaries */ -template +template class RangeDictionaryBlockInputStream : public DictionaryBlockInputStreamBase { public: - using DictionaryPtr = std::shared_ptr; + using Key = UInt64; RangeDictionaryBlockInputStream( - DictionaryPtr dictionary, + std::shared_ptr dictionary, size_t max_block_size, const Names & column_names, PaddedPODArray && ids_to_fill, @@ -40,35 +40,26 @@ private: template ColumnPtr getColumnFromPODArray(const PaddedPODArray & array) const; - template - void addSpecialColumn( - const std::optional & attribute, - DataTypePtr type, - const std::string & default_name, - const std::unordered_set & column_names_set, - const PaddedPODArray & values, - ColumnsWithTypeAndName & columns, - bool force = false) const; - Block fillBlock( const PaddedPODArray & ids_to_fill, const PaddedPODArray & block_start_dates, const PaddedPODArray & block_end_dates) const; - PaddedPODArray - makeDateKey(const PaddedPODArray & block_start_dates, const PaddedPODArray & block_end_dates) const; + PaddedPODArray makeDateKey( + const PaddedPODArray & block_start_dates, + const PaddedPODArray & block_end_dates) const; - DictionaryPtr dictionary; - Names column_names; + std::shared_ptr dictionary; + NameSet column_names; PaddedPODArray ids; PaddedPODArray start_dates; PaddedPODArray end_dates; }; -template -RangeDictionaryBlockInputStream::RangeDictionaryBlockInputStream( - DictionaryPtr dictionary_, +template +RangeDictionaryBlockInputStream::RangeDictionaryBlockInputStream( + std::shared_ptr dictionary_, size_t max_block_size_, const Names & column_names_, PaddedPODArray && ids_, @@ -76,15 +67,15 @@ RangeDictionaryBlockInputStream::RangeDictionary PaddedPODArray && block_end_dates) : DictionaryBlockInputStreamBase(ids_.size(), max_block_size_) , dictionary(dictionary_) - , column_names(column_names_) + , column_names(column_names_.begin(), column_names_.end()) , ids(std::move(ids_)) , start_dates(std::move(block_start_dates)) , end_dates(std::move(block_end_dates)) { } -template -Block RangeDictionaryBlockInputStream::getBlock(size_t start, size_t length) const +template +Block RangeDictionaryBlockInputStream::getBlock(size_t start, size_t length) const { PaddedPODArray block_ids; PaddedPODArray block_start_dates; @@ -103,38 +94,19 @@ Block RangeDictionaryBlockInputStream::getBlock( return fillBlock(block_ids, block_start_dates, block_end_dates); } -template +template template -ColumnPtr RangeDictionaryBlockInputStream::getColumnFromPODArray(const PaddedPODArray & array) const +ColumnPtr RangeDictionaryBlockInputStream::getColumnFromPODArray(const PaddedPODArray & array) const { auto column_vector = ColumnVector::create(); column_vector->getData().reserve(array.size()); - for (T value : array) - column_vector->insertValue(value); + column_vector->getData().insert(array.begin(), array.end()); + return column_vector; } -template -template -void RangeDictionaryBlockInputStream::addSpecialColumn( - const std::optional & attribute, - DataTypePtr type, - const std::string & default_name, - const std::unordered_set & column_names_set, - const PaddedPODArray & values, - ColumnsWithTypeAndName & columns, - bool force) const -{ - std::string name = default_name; - if (attribute) - name = attribute->name; - - if (force || column_names_set.find(name) != column_names_set.end()) - columns.emplace_back(getColumnFromPODArray(values), type, name); -} - -template -PaddedPODArray RangeDictionaryBlockInputStream::makeDateKey( +template +PaddedPODArray RangeDictionaryBlockInputStream::makeDateKey( const PaddedPODArray & block_start_dates, const PaddedPODArray & block_end_dates) const { PaddedPODArray key(block_start_dates.size()); @@ -150,8 +122,8 @@ PaddedPODArray RangeDictionaryBlockInputStream -Block RangeDictionaryBlockInputStream::fillBlock( +template +Block RangeDictionaryBlockInputStream::fillBlock( const PaddedPODArray & ids_to_fill, const PaddedPODArray & block_start_dates, const PaddedPODArray & block_end_dates) const @@ -159,20 +131,32 @@ Block RangeDictionaryBlockInputStream::fillBlock ColumnsWithTypeAndName columns; const DictionaryStructure & structure = dictionary->getStructure(); - std::unordered_set names(column_names.begin(), column_names.end()); - - addSpecialColumn(structure.id, std::make_shared(), "ID", names, ids_to_fill, columns, true); - auto ids_column = columns.back().column; - addSpecialColumn(structure.range_min, structure.range_max->type, "Range Start", names, block_start_dates, columns); - addSpecialColumn(structure.range_max, structure.range_max->type, "Range End", names, block_end_dates, columns); + auto ids_column = getColumnFromPODArray(ids_to_fill); + const std::string & id_column_name = structure.id->name; + if (column_names.find(id_column_name) != column_names.end()) + columns.emplace_back(ids_column, std::make_shared(), id_column_name); auto date_key = makeDateKey(block_start_dates, block_end_dates); auto date_column = getColumnFromPODArray(date_key); + const std::string & range_min_column_name = structure.range_min->name; + if (column_names.find(range_min_column_name) != column_names.end()) + { + auto range_min_column = getColumnFromPODArray(block_start_dates); + columns.emplace_back(range_min_column, structure.range_max->type, range_min_column_name); + } + + const std::string & range_max_column_name = structure.range_max->name; + if (column_names.find(range_max_column_name) != column_names.end()) + { + auto range_max_column = getColumnFromPODArray(block_end_dates); + columns.emplace_back(range_max_column, structure.range_max->type, range_max_column_name); + } + for (const auto idx : ext::range(0, structure.attributes.size())) { const DictionaryAttribute & attribute = structure.attributes[idx]; - if (names.find(attribute.name) != names.end()) + if (column_names.find(attribute.name) != column_names.end()) { ColumnPtr column = dictionary->getColumn( attribute.name, diff --git a/src/Dictionaries/RangeHashedDictionary.cpp b/src/Dictionaries/RangeHashedDictionary.cpp index c87da2c740d..52b6f219bac 100644 --- a/src/Dictionaries/RangeHashedDictionary.cpp +++ b/src/Dictionaries/RangeHashedDictionary.cpp @@ -515,7 +515,9 @@ RangeHashedDictionary::getAttributeWithType(const std::string & attribute_name, template void RangeHashedDictionary::getIdsAndDates( - PaddedPODArray & ids, PaddedPODArray & start_dates, PaddedPODArray & end_dates) const + PaddedPODArray & ids, + PaddedPODArray & start_dates, + PaddedPODArray & end_dates) const { const auto & attribute = attributes.front(); @@ -523,11 +525,9 @@ void RangeHashedDictionary::getIdsAndDates( { using Type = std::decay_t; using AttributeType = typename Type::AttributeType; + using ValueType = DictionaryValueType; - if constexpr (std::is_same_v) - getIdsAndDates(attribute, ids, start_dates, end_dates); - else - getIdsAndDates(attribute, ids, start_dates, end_dates); + getIdsAndDates(attribute, ids, start_dates, end_dates); }; callOnDictionaryAttributeType(attribute.type, type_call); @@ -571,13 +571,20 @@ BlockInputStreamPtr RangeHashedDictionary::getBlockInputStreamImpl(const Names & PaddedPODArray end_dates; getIdsAndDates(ids, start_dates, end_dates); - using BlockInputStreamType = RangeDictionaryBlockInputStream; - auto dict_ptr = std::static_pointer_cast(shared_from_this()); - return std::make_shared( - dict_ptr, max_block_size, column_names, std::move(ids), std::move(start_dates), std::move(end_dates)); + using BlockInputStreamType = RangeDictionaryBlockInputStream; + + auto stream = std::make_shared( + shared_from_this(), + max_block_size, + column_names, + std::move(ids), + std::move(start_dates), + std::move(end_dates)); + + return stream; } -struct RangeHashedDIctionaryCallGetBlockInputStreamImpl +struct RangeHashedDictionaryCallGetBlockInputStreamImpl { BlockInputStreamPtr stream; const RangeHashedDictionary * dict; @@ -597,7 +604,7 @@ BlockInputStreamPtr RangeHashedDictionary::getBlockInputStream(const Names & col { using ListType = TypeList; - RangeHashedDIctionaryCallGetBlockInputStreamImpl callable; + RangeHashedDictionaryCallGetBlockInputStreamImpl callable; callable.dict = this; callable.column_names = &column_names; callable.max_block_size = max_block_size; diff --git a/src/Dictionaries/RangeHashedDictionary.h b/src/Dictionaries/RangeHashedDictionary.h index 7ac2d33c63b..f2b24e52dfc 100644 --- a/src/Dictionaries/RangeHashedDictionary.h +++ b/src/Dictionaries/RangeHashedDictionary.h @@ -93,8 +93,6 @@ private: template using Ptr = std::unique_ptr>; - using NullableSet = HashSet>; - struct Attribute final { public: @@ -187,7 +185,7 @@ private: template BlockInputStreamPtr getBlockInputStreamImpl(const Names & column_names, size_t max_block_size) const; - friend struct RangeHashedDIctionaryCallGetBlockInputStreamImpl; + friend struct RangeHashedDictionaryCallGetBlockInputStreamImpl; const DictionaryStructure dict_struct; const DictionarySourcePtr source_ptr; diff --git a/tests/queries/0_stateless/01676_range_hashed_dictionary.reference b/tests/queries/0_stateless/01676_range_hashed_dictionary.reference index 7b19ddda8cb..23a5180d99c 100644 --- a/tests/queries/0_stateless/01676_range_hashed_dictionary.reference +++ b/tests/queries/0_stateless/01676_range_hashed_dictionary.reference @@ -11,9 +11,22 @@ dictHas 1 0 select columns from dictionary +allColumns 1 2019-05-05 2019-05-20 0.33 1 2019-05-21 2019-05-30 0.42 2 2019-05-21 2019-05-30 0.46 +noColumns +1 +1 +1 +onlySpecificColumns +1 2019-05-05 0.33 +1 2019-05-21 0.42 +2 2019-05-21 0.46 +onlySpecificColumn +0.33 +0.42 +0.46 Dictionary nullable dictGet 0.33 @@ -27,6 +40,19 @@ dictHas 1 0 select columns from dictionary +allColumns 1 2019-05-05 2019-05-20 0.33 1 2019-05-21 2019-05-30 0.42 2 2019-05-21 2019-05-30 \N +noColumns +1 +1 +1 +onlySpecificColumns +1 2019-05-05 0.33 +1 2019-05-21 0.42 +2 2019-05-21 \N +onlySpecificColumn +0.33 +0.42 +\N diff --git a/tests/queries/0_stateless/01676_range_hashed_dictionary.sql b/tests/queries/0_stateless/01676_range_hashed_dictionary.sql index a4d56e188c4..455e850b239 100644 --- a/tests/queries/0_stateless/01676_range_hashed_dictionary.sql +++ b/tests/queries/0_stateless/01676_range_hashed_dictionary.sql @@ -42,7 +42,14 @@ SELECT dictHas('database_for_range_dict.range_dictionary', toUInt64(1), toDate(' SELECT dictHas('database_for_range_dict.range_dictionary', toUInt64(2), toDate('2019-05-29')); SELECT dictHas('database_for_range_dict.range_dictionary', toUInt64(2), toDate('2019-05-31')); SELECT 'select columns from dictionary'; +SELECT 'allColumns'; SELECT * FROM database_for_range_dict.range_dictionary; +SELECT 'noColumns'; +SELECT 1 FROM database_for_range_dict.range_dictionary; +SELECT 'onlySpecificColumns'; +SELECT CountryID, StartDate, Tax FROM database_for_range_dict.range_dictionary; +SELECT 'onlySpecificColumn'; +SELECT Tax FROM database_for_range_dict.range_dictionary; DROP TABLE database_for_range_dict.date_table; DROP DICTIONARY database_for_range_dict.range_dictionary; @@ -87,7 +94,14 @@ SELECT dictHas('database_for_range_dict.range_dictionary_nullable', toUInt64(1), SELECT dictHas('database_for_range_dict.range_dictionary_nullable', toUInt64(2), toDate('2019-05-29')); SELECT dictHas('database_for_range_dict.range_dictionary_nullable', toUInt64(2), toDate('2019-05-31')); SELECT 'select columns from dictionary'; +SELECT 'allColumns'; SELECT * FROM database_for_range_dict.range_dictionary_nullable; +SELECT 'noColumns'; +SELECT 1 FROM database_for_range_dict.range_dictionary_nullable; +SELECT 'onlySpecificColumns'; +SELECT CountryID, StartDate, Tax FROM database_for_range_dict.range_dictionary_nullable; +SELECT 'onlySpecificColumn'; +SELECT Tax FROM database_for_range_dict.range_dictionary_nullable; DROP TABLE database_for_range_dict.date_table; DROP DICTIONARY database_for_range_dict.range_dictionary_nullable; From 6495de66fcd770f4a4d378bfcd8079706afb5185 Mon Sep 17 00:00:00 2001 From: Alexander Kuzmenkov <36882414+akuzm@users.noreply.github.com> Date: Sat, 13 Mar 2021 14:59:19 +0300 Subject: [PATCH 407/544] boop the CI --- website/blog/en/2021/fuzzing-clickhouse.md | 1 - 1 file changed, 1 deletion(-) diff --git a/website/blog/en/2021/fuzzing-clickhouse.md b/website/blog/en/2021/fuzzing-clickhouse.md index 05b070e98a4..b6852dcce15 100644 --- a/website/blog/en/2021/fuzzing-clickhouse.md +++ b/website/blog/en/2021/fuzzing-clickhouse.md @@ -6,7 +6,6 @@ author: '[Alexander Kuzmenkov](https://github.com/akuzm)' tags: ['fuzzing', 'testing'] --- - Testing is a major problem in software development: there is never enough of it. It becomes especially true for a database management system, whose task is to interpret a query language that works on the persistent state managed by the system in a distributed fashion. Each of these three functions is hard enough to test even in isolation, and it gets much worse when you combine them. As ClickHouse developers, we know this from experience. Despite a large amount of automated testing of all kinds we routinely perform as part of our continuous integration system, new bugs and regressions are creeping in. We are always looking for the ways to improve our test coverage, and this article will describe our recent development in this area — the AST-based query fuzzer. ## How to Test a SQL DBMS From 9474414c1766219cd68a5b1d0f107bbbbcde3c7d Mon Sep 17 00:00:00 2001 From: Alexander Kuzmenkov <36882414+akuzm@users.noreply.github.com> Date: Sat, 13 Mar 2021 15:15:58 +0300 Subject: [PATCH 408/544] boop the ci --- website/blog/en/2021/fuzzing-clickhouse.md | 1 + 1 file changed, 1 insertion(+) diff --git a/website/blog/en/2021/fuzzing-clickhouse.md b/website/blog/en/2021/fuzzing-clickhouse.md index b6852dcce15..05b070e98a4 100644 --- a/website/blog/en/2021/fuzzing-clickhouse.md +++ b/website/blog/en/2021/fuzzing-clickhouse.md @@ -6,6 +6,7 @@ author: '[Alexander Kuzmenkov](https://github.com/akuzm)' tags: ['fuzzing', 'testing'] --- + Testing is a major problem in software development: there is never enough of it. It becomes especially true for a database management system, whose task is to interpret a query language that works on the persistent state managed by the system in a distributed fashion. Each of these three functions is hard enough to test even in isolation, and it gets much worse when you combine them. As ClickHouse developers, we know this from experience. Despite a large amount of automated testing of all kinds we routinely perform as part of our continuous integration system, new bugs and regressions are creeping in. We are always looking for the ways to improve our test coverage, and this article will describe our recent development in this area — the AST-based query fuzzer. ## How to Test a SQL DBMS From c4798dd45fdd7ba7695ca131fbb5e5be166a5918 Mon Sep 17 00:00:00 2001 From: Alexander Kuzmenkov <36882414+akuzm@users.noreply.github.com> Date: Sat, 13 Mar 2021 15:43:02 +0300 Subject: [PATCH 409/544] boop the ci --- website/blog/en/2021/fuzzing-clickhouse.md | 1 - 1 file changed, 1 deletion(-) diff --git a/website/blog/en/2021/fuzzing-clickhouse.md b/website/blog/en/2021/fuzzing-clickhouse.md index 05b070e98a4..b6852dcce15 100644 --- a/website/blog/en/2021/fuzzing-clickhouse.md +++ b/website/blog/en/2021/fuzzing-clickhouse.md @@ -6,7 +6,6 @@ author: '[Alexander Kuzmenkov](https://github.com/akuzm)' tags: ['fuzzing', 'testing'] --- - Testing is a major problem in software development: there is never enough of it. It becomes especially true for a database management system, whose task is to interpret a query language that works on the persistent state managed by the system in a distributed fashion. Each of these three functions is hard enough to test even in isolation, and it gets much worse when you combine them. As ClickHouse developers, we know this from experience. Despite a large amount of automated testing of all kinds we routinely perform as part of our continuous integration system, new bugs and regressions are creeping in. We are always looking for the ways to improve our test coverage, and this article will describe our recent development in this area — the AST-based query fuzzer. ## How to Test a SQL DBMS From 334520557c6cd5140fcc9fc54a34d855cf205e7b Mon Sep 17 00:00:00 2001 From: Azat Khuzhin Date: Sat, 13 Mar 2021 18:05:24 +0300 Subject: [PATCH 410/544] Fix log_comment for *.sh in clickhouse-test setdefault will set it only once --- tests/clickhouse-test | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/clickhouse-test b/tests/clickhouse-test index 198c8bbe0c9..f01f909f62e 100755 --- a/tests/clickhouse-test +++ b/tests/clickhouse-test @@ -138,7 +138,7 @@ def run_single_test(args, ext, server_logs_level, client_options, case_file, std os.environ["CLICKHOUSE_DATABASE"] = database # This is for .sh tests - os.environ.setdefault("CLICKHOUSE_LOG_COMMENT", case_file) + os.environ["CLICKHOUSE_LOG_COMMENT"] = case_file params = { 'client': args.client + ' --database=' + database, From aad98b368e5f6a738e5f42abb462443f63669cdd Mon Sep 17 00:00:00 2001 From: Dmitriy Date: Sat, 13 Mar 2021 21:18:45 +0300 Subject: [PATCH 411/544] Edit and translate to Russian MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Поправил шаблоны в английской и русской версиях. --- .../parametric-functions.md | 10 +- .../aggregate-functions/reference/count.md | 5 +- .../reference/grouparrayinsertat.md | 6 +- .../reference/groupbitmapor.md | 2 +- .../reference/groupbitmapxor.md | 2 +- .../reference/groupbitor.md | 2 +- .../reference/initializeAggregation.md | 2 +- .../aggregate-functions/reference/kurtpop.md | 2 +- .../aggregate-functions/reference/kurtsamp.md | 2 +- .../reference/mannwhitneyutest.md | 2 +- .../aggregate-functions/reference/skewpop.md | 2 +- .../aggregate-functions/reference/skewsamp.md | 2 +- .../reference/studentttest.md | 4 +- .../aggregate-functions/reference/topk.md | 4 +- .../reference/topkweighted.md | 2 +- .../reference/welchttest.md | 4 +- .../functions/array-functions.md | 39 ++--- .../sql-reference/functions/bit-functions.md | 26 ++-- .../functions/bitmap-functions.md | 90 ++++++----- .../functions/conditional-functions.md | 8 +- .../functions/date-time-functions.md | 2 +- .../functions/encoding-functions.md | 2 +- .../functions/functions-for-nulls.md | 24 +-- .../en/sql-reference/functions/geo/geohash.md | 7 +- docs/en/sql-reference/functions/geo/h3.md | 17 +- .../sql-reference/functions/hash-functions.md | 28 ++-- .../sql-reference/functions/introspection.md | 19 +-- .../functions/ip-address-functions.md | 12 +- .../sql-reference/functions/json-functions.md | 12 +- .../functions/machine-learning-functions.md | 14 +- .../sql-reference/functions/math-functions.md | 2 +- .../functions/other-functions.md | 1 - .../functions/rounding-functions.md | 2 +- .../functions/string-functions.md | 29 ++-- .../functions/string-search-functions.md | 56 +++---- .../functions/tuple-functions.md | 2 +- .../functions/tuple-map-functions.md | 1 - .../functions/type-conversion-functions.md | 2 +- .../sql-reference/functions/url-functions.md | 4 +- .../functions/ym-dict-functions.md | 2 +- .../sql-reference/table-functions/generate.md | 2 +- docs/en/sql-reference/table-functions/view.md | 9 +- docs/ru/getting-started/tutorial.md | 2 +- .../aggregate-functions/combinators.md | 14 +- .../parametric-functions.md | 38 +++-- .../aggregate-functions/reference/argmax.md | 2 +- .../aggregate-functions/reference/argmin.md | 2 +- .../reference/avgweighted.md | 6 +- .../aggregate-functions/reference/count.md | 6 +- .../reference/grouparrayinsertat.md | 18 +-- .../reference/grouparraymovingavg.md | 8 +- .../reference/grouparraymovingsum.md | 2 +- .../reference/grouparraysample.md | 2 +- .../reference/groupbitand.md | 2 +- .../reference/groupbitmap.md | 2 +- .../reference/groupbitor.md | 2 +- .../reference/groupbitxor.md | 2 +- .../reference/initializeAggregation.md | 4 +- .../aggregate-functions/reference/kurtpop.md | 6 +- .../aggregate-functions/reference/kurtsamp.md | 6 +- .../reference/mannwhitneyutest.md | 10 +- .../aggregate-functions/reference/quantile.md | 6 +- .../reference/quantiledeterministic.md | 8 +- .../reference/quantileexact.md | 18 +-- .../reference/quantileexactweighted.md | 8 +- .../reference/quantiletdigest.md | 6 +- .../reference/quantiletdigestweighted.md | 8 +- .../reference/quantiletiming.md | 6 +- .../reference/quantiletimingweighted.md | 8 +- .../aggregate-functions/reference/rankCorr.md | 6 +- .../aggregate-functions/reference/skewpop.md | 6 +- .../aggregate-functions/reference/skewsamp.md | 6 +- .../reference/studentttest.md | 2 +- .../aggregate-functions/reference/topk.md | 4 +- .../reference/topkweighted.md | 6 +- .../aggregate-functions/reference/uniq.md | 2 +- .../reference/uniqcombined.md | 2 +- .../reference/uniqexact.md | 2 +- .../reference/uniqhll12.md | 2 +- .../reference/welchttest.md | 2 +- .../functions/array-functions.md | 147 +++++++++--------- .../sql-reference/functions/bit-functions.md | 38 ++--- .../functions/bitmap-functions.md | 59 ++++--- .../functions/conditional-functions.md | 24 +-- .../functions/date-time-functions.md | 24 +-- .../functions/encoding-functions.md | 14 +- .../functions/encryption-functions.md | 10 +- .../functions/ext-dict-functions.md | 12 +- .../functions/functions-for-nulls.md | 38 ++--- .../ru/sql-reference/functions/geo/geohash.md | 17 +- docs/ru/sql-reference/functions/geo/h3.md | 29 ++-- .../sql-reference/functions/hash-functions.md | 72 ++++----- .../sql-reference/functions/introspection.md | 35 +++-- .../functions/ip-address-functions.md | 14 +- .../sql-reference/functions/json-functions.md | 29 ++-- .../functions/machine-learning-functions.md | 16 +- .../sql-reference/functions/math-functions.md | 18 +-- .../functions/other-functions.md | 135 ++++++++-------- .../functions/random-functions.md | 6 +- .../functions/rounding-functions.md | 18 +-- .../functions/splitting-merging-functions.md | 14 +- .../functions/string-functions.md | 72 ++++----- .../functions/string-search-functions.md | 84 +++++----- .../functions/tuple-functions.md | 4 +- .../functions/tuple-map-functions.md | 21 +-- .../functions/type-conversion-functions.md | 48 +++--- .../sql-reference/functions/url-functions.md | 16 +- .../functions/ym-dict-functions.md | 8 +- .../sql-reference/table-functions/generate.md | 5 +- .../ru/sql-reference/table-functions/mysql.md | 4 +- docs/ru/sql-reference/table-functions/view.md | 11 +- 111 files changed, 897 insertions(+), 852 deletions(-) diff --git a/docs/en/sql-reference/aggregate-functions/parametric-functions.md b/docs/en/sql-reference/aggregate-functions/parametric-functions.md index c6c97b5428b..f7db3c7fb50 100644 --- a/docs/en/sql-reference/aggregate-functions/parametric-functions.md +++ b/docs/en/sql-reference/aggregate-functions/parametric-functions.md @@ -254,8 +254,8 @@ windowFunnel(window, [mode])(timestamp, cond1, cond2, ..., condN) **Parameters** - `window` — Length of the sliding window. The unit of `window` depends on the `timestamp` itself and varies. Determined using the expression `timestamp of cond2 <= timestamp of cond1 + window`. -- `mode` - It is an optional argument. - - `'strict'` - When the `'strict'` is set, the windowFunnel() applies conditions only for the unique values. +- `mode` — It is an optional argument. + - `'strict'` — When the `'strict'` is set, the windowFunnel() applies conditions only for the unique values. **Returned value** @@ -336,14 +336,14 @@ retention(cond1, cond2, ..., cond32); **Arguments** -- `cond` — an expression that returns a `UInt8` result (1 or 0). +- `cond` — An expression that returns a `UInt8` result (1 or 0). **Returned value** The array of 1 or 0. -- 1 — condition was met for the event. -- 0 — condition wasn’t met for the event. +- 1 — Condition was met for the event. +- 0 — Condition wasn’t met for the event. Type: `UInt8`. diff --git a/docs/en/sql-reference/aggregate-functions/reference/count.md b/docs/en/sql-reference/aggregate-functions/reference/count.md index 0a5aef2fe97..48c6f3f8c05 100644 --- a/docs/en/sql-reference/aggregate-functions/reference/count.md +++ b/docs/en/sql-reference/aggregate-functions/reference/count.md @@ -7,8 +7,9 @@ toc_priority: 1 Counts the number of rows or not-NULL values. ClickHouse supports the following syntaxes for `count`: -- `count(expr)` or `COUNT(DISTINCT expr)`. -- `count()` or `COUNT(*)`. The `count()` syntax is ClickHouse-specific. + +- `count(expr)` or `COUNT(DISTINCT expr)`. +- `count()` or `COUNT(*)`. The `count()` syntax is ClickHouse-specific. **Arguments** diff --git a/docs/en/sql-reference/aggregate-functions/reference/grouparrayinsertat.md b/docs/en/sql-reference/aggregate-functions/reference/grouparrayinsertat.md index 68456bf7844..d29550b007e 100644 --- a/docs/en/sql-reference/aggregate-functions/reference/grouparrayinsertat.md +++ b/docs/en/sql-reference/aggregate-functions/reference/grouparrayinsertat.md @@ -9,7 +9,7 @@ Inserts a value into the array at the specified position. **Syntax** ``` sql -groupArrayInsertAt(default_x, size)(x, pos); +groupArrayInsertAt(default_x, size)(x, pos) ``` If in one query several values are inserted into the same position, the function behaves in the following ways: @@ -21,8 +21,8 @@ If in one query several values are inserted into the same position, the function - `x` — Value to be inserted. [Expression](../../../sql-reference/syntax.md#syntax-expressions) resulting in one of the [supported data types](../../../sql-reference/data-types/index.md). - `pos` — Position at which the specified element `x` is to be inserted. Index numbering in the array starts from zero. [UInt32](../../../sql-reference/data-types/int-uint.md#uint-ranges). -- `default_x`— Default value for substituting in empty positions. Optional parameter. [Expression](../../../sql-reference/syntax.md#syntax-expressions) resulting in the data type configured for the `x` parameter. If `default_x` is not defined, the [default values](../../../sql-reference/statements/create/table.md#create-default-values) are used. -- `size`— Length of the resulting array. Optional parameter. When using this parameter, the default value `default_x` must be specified. [UInt32](../../../sql-reference/data-types/int-uint.md#uint-ranges). +- `default_x` — Default value for substituting in empty positions. Optional parameter. [Expression](../../../sql-reference/syntax.md#syntax-expressions) resulting in the data type configured for the `x` parameter. If `default_x` is not defined, the [default values](../../../sql-reference/statements/create/table.md#create-default-values) are used. +- `size` — Length of the resulting array. Optional parameter. When using this parameter, the default value `default_x` must be specified. [UInt32](../../../sql-reference/data-types/int-uint.md#uint-ranges). **Returned value** diff --git a/docs/en/sql-reference/aggregate-functions/reference/groupbitmapor.md b/docs/en/sql-reference/aggregate-functions/reference/groupbitmapor.md index a4d99fd29e3..d3f40f63f65 100644 --- a/docs/en/sql-reference/aggregate-functions/reference/groupbitmapor.md +++ b/docs/en/sql-reference/aggregate-functions/reference/groupbitmapor.md @@ -14,7 +14,7 @@ groupBitmapOr(expr) `expr` – An expression that results in `AggregateFunction(groupBitmap, UInt*)` type. -**Return value** +**Returned value** Value of the `UInt64` type. diff --git a/docs/en/sql-reference/aggregate-functions/reference/groupbitmapxor.md b/docs/en/sql-reference/aggregate-functions/reference/groupbitmapxor.md index 834f088d02f..cbe01e08145 100644 --- a/docs/en/sql-reference/aggregate-functions/reference/groupbitmapxor.md +++ b/docs/en/sql-reference/aggregate-functions/reference/groupbitmapxor.md @@ -14,7 +14,7 @@ groupBitmapOr(expr) `expr` – An expression that results in `AggregateFunction(groupBitmap, UInt*)` type. -**Return value** +**Returned value** Value of the `UInt64` type. diff --git a/docs/en/sql-reference/aggregate-functions/reference/groupbitor.md b/docs/en/sql-reference/aggregate-functions/reference/groupbitor.md index e427a9ad970..24077de0adc 100644 --- a/docs/en/sql-reference/aggregate-functions/reference/groupbitor.md +++ b/docs/en/sql-reference/aggregate-functions/reference/groupbitor.md @@ -14,7 +14,7 @@ groupBitOr(expr) `expr` – An expression that results in `UInt*` type. -**Return value** +**Returned value** Value of the `UInt*` type. diff --git a/docs/en/sql-reference/aggregate-functions/reference/initializeAggregation.md b/docs/en/sql-reference/aggregate-functions/reference/initializeAggregation.md index 313d6bf81f5..c8fb535089b 100644 --- a/docs/en/sql-reference/aggregate-functions/reference/initializeAggregation.md +++ b/docs/en/sql-reference/aggregate-functions/reference/initializeAggregation.md @@ -10,7 +10,7 @@ Use it for tests or to process columns of types `AggregateFunction` and `Aggrega **Syntax** ``` sql -initializeAggregation (aggregate_function, column_1, column_2); +initializeAggregation (aggregate_function, column_1, column_2) ``` **Arguments** diff --git a/docs/en/sql-reference/aggregate-functions/reference/kurtpop.md b/docs/en/sql-reference/aggregate-functions/reference/kurtpop.md index db402c99663..c51c4b92e74 100644 --- a/docs/en/sql-reference/aggregate-functions/reference/kurtpop.md +++ b/docs/en/sql-reference/aggregate-functions/reference/kurtpop.md @@ -21,5 +21,5 @@ The kurtosis of the given distribution. Type — [Float64](../../../sql-referenc **Example** ``` sql -SELECT kurtPop(value) FROM series_with_value_column +SELECT kurtPop(value) FROM series_with_value_column; ``` diff --git a/docs/en/sql-reference/aggregate-functions/reference/kurtsamp.md b/docs/en/sql-reference/aggregate-functions/reference/kurtsamp.md index 4bb9f76763b..0ee40138adc 100644 --- a/docs/en/sql-reference/aggregate-functions/reference/kurtsamp.md +++ b/docs/en/sql-reference/aggregate-functions/reference/kurtsamp.md @@ -23,5 +23,5 @@ The kurtosis of the given distribution. Type — [Float64](../../../sql-referenc **Example** ``` sql -SELECT kurtSamp(value) FROM series_with_value_column +SELECT kurtSamp(value) FROM series_with_value_column; ``` diff --git a/docs/en/sql-reference/aggregate-functions/reference/mannwhitneyutest.md b/docs/en/sql-reference/aggregate-functions/reference/mannwhitneyutest.md index dc5fc45b878..34e8188299c 100644 --- a/docs/en/sql-reference/aggregate-functions/reference/mannwhitneyutest.md +++ b/docs/en/sql-reference/aggregate-functions/reference/mannwhitneyutest.md @@ -27,7 +27,7 @@ The null hypothesis is that two populations are stochastically equal. Also one-s - `'two-sided'`; - `'greater'`; - `'less'`. -- `continuity_correction` - if not 0 then continuity correction in the normal approximation for the p-value is applied. (Optional, default: 1.) [UInt64](../../../sql-reference/data-types/int-uint.md). +- `continuity_correction` — if not 0 then continuity correction in the normal approximation for the p-value is applied. (Optional, default: 1.) [UInt64](../../../sql-reference/data-types/int-uint.md). **Returned values** diff --git a/docs/en/sql-reference/aggregate-functions/reference/skewpop.md b/docs/en/sql-reference/aggregate-functions/reference/skewpop.md index b9dfc390f9d..f84f8897a35 100644 --- a/docs/en/sql-reference/aggregate-functions/reference/skewpop.md +++ b/docs/en/sql-reference/aggregate-functions/reference/skewpop.md @@ -21,5 +21,5 @@ The skewness of the given distribution. Type — [Float64](../../../sql-referenc **Example** ``` sql -SELECT skewPop(value) FROM series_with_value_column +SELECT skewPop(value) FROM series_with_value_column; ``` diff --git a/docs/en/sql-reference/aggregate-functions/reference/skewsamp.md b/docs/en/sql-reference/aggregate-functions/reference/skewsamp.md index f7a6df8f507..48a049ca69d 100644 --- a/docs/en/sql-reference/aggregate-functions/reference/skewsamp.md +++ b/docs/en/sql-reference/aggregate-functions/reference/skewsamp.md @@ -23,5 +23,5 @@ The skewness of the given distribution. Type — [Float64](../../../sql-referenc **Example** ``` sql -SELECT skewSamp(value) FROM series_with_value_column +SELECT skewSamp(value) FROM series_with_value_column; ``` diff --git a/docs/en/sql-reference/aggregate-functions/reference/studentttest.md b/docs/en/sql-reference/aggregate-functions/reference/studentttest.md index a1d7ae33fe1..3398fc1ca8c 100644 --- a/docs/en/sql-reference/aggregate-functions/reference/studentttest.md +++ b/docs/en/sql-reference/aggregate-functions/reference/studentttest.md @@ -18,8 +18,8 @@ The null hypothesis is that means of populations are equal. Normal distribution **Arguments** -- `sample_data` — sample data. [Integer](../../../sql-reference/data-types/int-uint.md), [Float](../../../sql-reference/data-types/float.md) or [Decimal](../../../sql-reference/data-types/decimal.md). -- `sample_index` — sample index. [Integer](../../../sql-reference/data-types/int-uint.md). +- `sample_data` — Sample data. [Integer](../../../sql-reference/data-types/int-uint.md), [Float](../../../sql-reference/data-types/float.md) or [Decimal](../../../sql-reference/data-types/decimal.md). +- `sample_index` — Sample index. [Integer](../../../sql-reference/data-types/int-uint.md). **Returned values** diff --git a/docs/en/sql-reference/aggregate-functions/reference/topk.md b/docs/en/sql-reference/aggregate-functions/reference/topk.md index b3e79803ba1..b9bea013ea8 100644 --- a/docs/en/sql-reference/aggregate-functions/reference/topk.md +++ b/docs/en/sql-reference/aggregate-functions/reference/topk.md @@ -18,13 +18,13 @@ We recommend using the `N < 10` value; performance is reduced with large `N` val **Arguments** -- ‘N’ is the number of elements to return. +- `N` – The number of elements to return. If the parameter is omitted, default value 10 is used. **Arguments** -- ’ x ’ – The value to calculate frequency. +- `x` – The value to calculate frequency. **Example** diff --git a/docs/en/sql-reference/aggregate-functions/reference/topkweighted.md b/docs/en/sql-reference/aggregate-functions/reference/topkweighted.md index 02b9f77ea6f..8562336c829 100644 --- a/docs/en/sql-reference/aggregate-functions/reference/topkweighted.md +++ b/docs/en/sql-reference/aggregate-functions/reference/topkweighted.md @@ -18,7 +18,7 @@ topKWeighted(N)(x, weight) **Arguments** -- `x` – The value. +- `x` — The value. - `weight` — The weight. [UInt8](../../../sql-reference/data-types/int-uint.md). **Returned value** diff --git a/docs/en/sql-reference/aggregate-functions/reference/welchttest.md b/docs/en/sql-reference/aggregate-functions/reference/welchttest.md index b391fb1d979..02238de42ef 100644 --- a/docs/en/sql-reference/aggregate-functions/reference/welchttest.md +++ b/docs/en/sql-reference/aggregate-functions/reference/welchttest.md @@ -18,8 +18,8 @@ The null hypothesis is that means of populations are equal. Normal distribution **Arguments** -- `sample_data` — sample data. [Integer](../../../sql-reference/data-types/int-uint.md), [Float](../../../sql-reference/data-types/float.md) or [Decimal](../../../sql-reference/data-types/decimal.md). -- `sample_index` — sample index. [Integer](../../../sql-reference/data-types/int-uint.md). +- `sample_data` — Sample data. [Integer](../../../sql-reference/data-types/int-uint.md), [Float](../../../sql-reference/data-types/float.md) or [Decimal](../../../sql-reference/data-types/decimal.md). +- `sample_index` — Sample index. [Integer](../../../sql-reference/data-types/int-uint.md). **Returned values** diff --git a/docs/en/sql-reference/functions/array-functions.md b/docs/en/sql-reference/functions/array-functions.md index c9c418d57a4..69124827255 100644 --- a/docs/en/sql-reference/functions/array-functions.md +++ b/docs/en/sql-reference/functions/array-functions.md @@ -376,7 +376,7 @@ arrayPopBack(array) **Example** ``` sql -SELECT arrayPopBack([1, 2, 3]) AS res +SELECT arrayPopBack([1, 2, 3]) AS res; ``` ``` text @@ -400,7 +400,7 @@ arrayPopFront(array) **Example** ``` sql -SELECT arrayPopFront([1, 2, 3]) AS res +SELECT arrayPopFront([1, 2, 3]) AS res; ``` ``` text @@ -425,7 +425,7 @@ arrayPushBack(array, single_value) **Example** ``` sql -SELECT arrayPushBack(['a'], 'b') AS res +SELECT arrayPushBack(['a'], 'b') AS res; ``` ``` text @@ -450,7 +450,7 @@ arrayPushFront(array, single_value) **Example** ``` sql -SELECT arrayPushFront(['b'], 'a') AS res +SELECT arrayPushFront(['b'], 'a') AS res; ``` ``` text @@ -482,7 +482,7 @@ An array of length `size`. **Examples of calls** ``` sql -SELECT arrayResize([1], 3) +SELECT arrayResize([1], 3); ``` ``` text @@ -492,7 +492,7 @@ SELECT arrayResize([1], 3) ``` ``` sql -SELECT arrayResize([1], 3, NULL) +SELECT arrayResize([1], 3, NULL); ``` ``` text @@ -513,12 +513,12 @@ arraySlice(array, offset[, length]) - `array` – Array of data. - `offset` – Indent from the edge of the array. A positive value indicates an offset on the left, and a negative value is an indent on the right. Numbering of the array items begins with 1. -- `length` - The length of the required slice. If you specify a negative value, the function returns an open slice `[offset, array_length - length)`. If you omit the value, the function returns the slice `[offset, the_end_of_array]`. +- `length` – The length of the required slice. If you specify a negative value, the function returns an open slice `[offset, array_length - length)`. If you omit the value, the function returns the slice `[offset, the_end_of_array]`. **Example** ``` sql -SELECT arraySlice([1, 2, NULL, 4, 5], 2, 3) AS res +SELECT arraySlice([1, 2, NULL, 4, 5], 2, 3) AS res; ``` ``` text @@ -766,7 +766,7 @@ Type: [UInt\*](https://clickhouse.tech/docs/en/data_types/int_uint/#uint-ranges) Query: ``` sql -SELECT arrayDifference([1, 2, 3, 4]) +SELECT arrayDifference([1, 2, 3, 4]); ``` Result: @@ -782,7 +782,7 @@ Example of the overflow due to result type Int64: Query: ``` sql -SELECT arrayDifference([0, 10000000000000000000]) +SELECT arrayDifference([0, 10000000000000000000]); ``` Result: @@ -816,7 +816,7 @@ Returns an array containing the distinct elements. Query: ``` sql -SELECT arrayDistinct([1, 2, 2, 3, 1]) +SELECT arrayDistinct([1, 2, 2, 3, 1]); ``` Result: @@ -883,7 +883,7 @@ arrayReduce(agg_func, arr1, arr2, ..., arrN) Query: ``` sql -SELECT arrayReduce('max', [1, 2, 3]) +SELECT arrayReduce('max', [1, 2, 3]); ``` Result: @@ -899,7 +899,7 @@ If an aggregate function takes multiple arguments, then this function must be ap Query: ``` sql -SELECT arrayReduce('maxIf', [3, 5], [1, 0]) +SELECT arrayReduce('maxIf', [3, 5], [1, 0]); ``` Result: @@ -915,7 +915,7 @@ Example with a parametric aggregate function: Query: ``` sql -SELECT arrayReduce('uniqUpTo(3)', [1, 2, 3, 4, 5, 6, 7, 8, 9, 10]) +SELECT arrayReduce('uniqUpTo(3)', [1, 2, 3, 4, 5, 6, 7, 8, 9, 10]); ``` Result: @@ -1014,7 +1014,7 @@ Alias: `flatten`. **Examples** ``` sql -SELECT flatten([[[1]], [[2], [3]]]) +SELECT flatten([[[1]], [[2], [3]]]); ``` ``` text @@ -1048,7 +1048,7 @@ Type: `Array`. Query: ``` sql -SELECT arrayCompact([1, 1, nan, nan, 2, 3, 3, 3]) +SELECT arrayCompact([1, 1, nan, nan, 2, 3, 3, 3]); ``` Result: @@ -1086,7 +1086,7 @@ Type: [Array](../../sql-reference/data-types/array.md). Query: ``` sql -SELECT arrayZip(['a', 'b', 'c'], [5, 2, 1]) +SELECT arrayZip(['a', 'b', 'c'], [5, 2, 1]); ``` Result: @@ -1108,17 +1108,20 @@ arrayAUC(arr_scores, arr_labels) ``` **Arguments** + - `arr_scores` — scores prediction model gives. - `arr_labels` — labels of samples, usually 1 for positive sample and 0 for negtive sample. **Returned value** + Returns AUC value with type Float64. **Example** + Query: ``` sql -select arrayAUC([0.1, 0.4, 0.35, 0.8], [0, 0, 1, 1]) +select arrayAUC([0.1, 0.4, 0.35, 0.8], [0, 0, 1, 1]); ``` Result: diff --git a/docs/en/sql-reference/functions/bit-functions.md b/docs/en/sql-reference/functions/bit-functions.md index a3d0c82d8ab..ce08c569bab 100644 --- a/docs/en/sql-reference/functions/bit-functions.md +++ b/docs/en/sql-reference/functions/bit-functions.md @@ -37,8 +37,8 @@ SELECT bitTest(number, index) **Arguments** -- `number` – integer number. -- `index` – position of bit. +- `number` – Integer number. +- `index` – Position of bit. **Returned values** @@ -53,7 +53,7 @@ For example, the number 43 in base-2 (binary) numeral system is 101011. Query: ``` sql -SELECT bitTest(43, 1) +SELECT bitTest(43, 1); ``` Result: @@ -69,7 +69,7 @@ Another example: Query: ``` sql -SELECT bitTest(43, 2) +SELECT bitTest(43, 2); ``` Result: @@ -102,8 +102,8 @@ SELECT bitTestAll(number, index1, index2, index3, index4, ...) **Arguments** -- `number` – integer number. -- `index1`, `index2`, `index3`, `index4` – positions of bit. For example, for set of positions (`index1`, `index2`, `index3`, `index4`) is true if and only if all of its positions are true (`index1` ⋀ `index2`, ⋀ `index3` ⋀ `index4`). +- `number` – Integer number. +- `index1`, `index2`, `index3`, `index4` – Positions of bit. For example, for set of positions (`index1`, `index2`, `index3`, `index4`) is true if and only if all of its positions are true (`index1` ⋀ `index2`, ⋀ `index3` ⋀ `index4`). **Returned values** @@ -118,7 +118,7 @@ For example, the number 43 in base-2 (binary) numeral system is 101011. Query: ``` sql -SELECT bitTestAll(43, 0, 1, 3, 5) +SELECT bitTestAll(43, 0, 1, 3, 5); ``` Result: @@ -134,7 +134,7 @@ Another example: Query: ``` sql -SELECT bitTestAll(43, 0, 1, 3, 5, 2) +SELECT bitTestAll(43, 0, 1, 3, 5, 2); ``` Result: @@ -167,8 +167,8 @@ SELECT bitTestAny(number, index1, index2, index3, index4, ...) **Arguments** -- `number` – integer number. -- `index1`, `index2`, `index3`, `index4` – positions of bit. +- `number` – Integer number. +- `index1`, `index2`, `index3`, `index4` – Positions of bit. **Returned values** @@ -183,7 +183,7 @@ For example, the number 43 in base-2 (binary) numeral system is 101011. Query: ``` sql -SELECT bitTestAny(43, 0, 2) +SELECT bitTestAny(43, 0, 2); ``` Result: @@ -199,7 +199,7 @@ Another example: Query: ``` sql -SELECT bitTestAny(43, 4, 2) +SELECT bitTestAny(43, 4, 2); ``` Result: @@ -239,7 +239,7 @@ Take for example the number 333. Its binary representation: 0000000101001101. Query: ``` sql -SELECT bitCount(333) +SELECT bitCount(333); ``` Result: diff --git a/docs/en/sql-reference/functions/bitmap-functions.md b/docs/en/sql-reference/functions/bitmap-functions.md index bfff70576f2..c809aee85fe 100644 --- a/docs/en/sql-reference/functions/bitmap-functions.md +++ b/docs/en/sql-reference/functions/bitmap-functions.md @@ -23,12 +23,12 @@ bitmapBuild(array) **Arguments** -- `array` – unsigned integer array. +- `array` – Unsigned integer array. **Example** ``` sql -SELECT bitmapBuild([1, 2, 3, 4, 5]) AS res, toTypeName(res) +SELECT bitmapBuild([1, 2, 3, 4, 5]) AS res, toTypeName(res); ``` ``` text @@ -47,12 +47,12 @@ bitmapToArray(bitmap) **Arguments** -- `bitmap` – bitmap object. +- `bitmap` – Bitmap object. **Example** ``` sql -SELECT bitmapToArray(bitmapBuild([1, 2, 3, 4, 5])) AS res +SELECT bitmapToArray(bitmapBuild([1, 2, 3, 4, 5])) AS res; ``` ``` text @@ -72,13 +72,13 @@ bitmapSubsetInRange(bitmap, range_start, range_end) **Arguments** - `bitmap` – [Bitmap object](#bitmap_functions-bitmapbuild). -- `range_start` – range start point. Type: [UInt32](../../sql-reference/data-types/int-uint.md). -- `range_end` – range end point(excluded). Type: [UInt32](../../sql-reference/data-types/int-uint.md). +- `range_start` – Range start point. Type: [UInt32](../../sql-reference/data-types/int-uint.md). +- `range_end` – Range end point(excluded). Type: [UInt32](../../sql-reference/data-types/int-uint.md). **Example** ``` sql -SELECT bitmapToArray(bitmapSubsetInRange(bitmapBuild([0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30,31,32,33,100,200,500]), toUInt32(30), toUInt32(200))) AS res +SELECT bitmapToArray(bitmapSubsetInRange(bitmapBuild([0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30,31,32,33,100,200,500]), toUInt32(30), toUInt32(200))) AS res; ``` ``` text @@ -114,7 +114,7 @@ Type: `Bitmap object`. Query: ``` sql -SELECT bitmapToArray(bitmapSubsetLimit(bitmapBuild([0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30,31,32,33,100,200,500]), toUInt32(30), toUInt32(200))) AS res +SELECT bitmapToArray(bitmapSubsetLimit(bitmapBuild([0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30,31,32,33,100,200,500]), toUInt32(30), toUInt32(200))) AS res; ``` Result: @@ -148,7 +148,7 @@ Type: `UInt8`. **Example** ``` sql -SELECT bitmapContains(bitmapBuild([1,5,7,9]), toUInt32(9)) AS res +SELECT bitmapContains(bitmapBuild([1,5,7,9]), toUInt32(9)) AS res; ``` ``` text @@ -169,7 +169,7 @@ If you are sure that `bitmap2` contains strictly one element, consider using the **Arguments** -- `bitmap*` – bitmap object. +- `bitmap*` – Bitmap object. **Return values** @@ -179,7 +179,7 @@ If you are sure that `bitmap2` contains strictly one element, consider using the **Example** ``` sql -SELECT bitmapHasAny(bitmapBuild([1,2,3]),bitmapBuild([3,4,5])) AS res +SELECT bitmapHasAny(bitmapBuild([1,2,3]),bitmapBuild([3,4,5])) AS res; ``` ``` text @@ -199,12 +199,12 @@ bitmapHasAll(bitmap,bitmap) **Arguments** -- `bitmap` – bitmap object. +- `bitmap` – Bitmap object. **Example** ``` sql -SELECT bitmapHasAll(bitmapBuild([1,2,3]),bitmapBuild([3,4,5])) AS res +SELECT bitmapHasAll(bitmapBuild([1,2,3]),bitmapBuild([3,4,5])) AS res; ``` ``` text @@ -223,12 +223,12 @@ bitmapCardinality(bitmap) **Arguments** -- `bitmap` – bitmap object. +- `bitmap` – Bitmap object. **Example** ``` sql -SELECT bitmapCardinality(bitmapBuild([1, 2, 3, 4, 5])) AS res +SELECT bitmapCardinality(bitmapBuild([1, 2, 3, 4, 5])) AS res; ``` ``` text @@ -245,17 +245,19 @@ Retrun the smallest value of type UInt64 in the set, UINT32_MAX if the set is em **Arguments** -- `bitmap` – bitmap object. +- `bitmap` – Bitmap object. **Example** ``` sql -SELECT bitmapMin(bitmapBuild([1, 2, 3, 4, 5])) AS res +SELECT bitmapMin(bitmapBuild([1, 2, 3, 4, 5])) AS res; ``` - ┌─res─┐ - │ 1 │ - └─────┘ +``` text + ┌─res─┐ + │ 1 │ + └─────┘ +``` ## bitmapMax {#bitmapmax} @@ -265,17 +267,19 @@ Retrun the greatest value of type UInt64 in the set, 0 if the set is empty. **Arguments** -- `bitmap` – bitmap object. +- `bitmap` – Bitmap object. **Example** ``` sql -SELECT bitmapMax(bitmapBuild([1, 2, 3, 4, 5])) AS res +SELECT bitmapMax(bitmapBuild([1, 2, 3, 4, 5])) AS res; ``` - ┌─res─┐ - │ 5 │ - └─────┘ +``` text + ┌─res─┐ + │ 5 │ + └─────┘ +``` ## bitmapTransform {#bitmaptransform} @@ -285,19 +289,21 @@ Transform an array of values in a bitmap to another array of values, the result **Arguments** -- `bitmap` – bitmap object. +- `bitmap` – Bitmap object. - `from_array` – UInt32 array. For idx in range \[0, from_array.size()), if bitmap contains from_array\[idx\], then replace it with to_array\[idx\]. Note that the result depends on array ordering if there are common elements between from_array and to_array. - `to_array` – UInt32 array, its size shall be the same to from_array. **Example** ``` sql -SELECT bitmapToArray(bitmapTransform(bitmapBuild([1, 2, 3, 4, 5, 6, 7, 8, 9, 10]), cast([5,999,2] as Array(UInt32)), cast([2,888,20] as Array(UInt32)))) AS res +SELECT bitmapToArray(bitmapTransform(bitmapBuild([1, 2, 3, 4, 5, 6, 7, 8, 9, 10]), cast([5,999,2] as Array(UInt32)), cast([2,888,20] as Array(UInt32)))) AS res; ``` - ┌─res───────────────────┐ - │ [1,3,4,6,7,8,9,10,20] │ - └───────────────────────┘ +``` text + ┌─res───────────────────┐ + │ [1,3,4,6,7,8,9,10,20] │ + └───────────────────────┘ +``` ## bitmapAnd {#bitmapand} @@ -309,12 +315,12 @@ bitmapAnd(bitmap,bitmap) **Arguments** -- `bitmap` – bitmap object. +- `bitmap` – Bitmap object. **Example** ``` sql -SELECT bitmapToArray(bitmapAnd(bitmapBuild([1,2,3]),bitmapBuild([3,4,5]))) AS res +SELECT bitmapToArray(bitmapAnd(bitmapBuild([1,2,3]),bitmapBuild([3,4,5]))) AS res; ``` ``` text @@ -333,12 +339,12 @@ bitmapOr(bitmap,bitmap) **Arguments** -- `bitmap` – bitmap object. +- `bitmap` – Bitmap object. **Example** ``` sql -SELECT bitmapToArray(bitmapOr(bitmapBuild([1,2,3]),bitmapBuild([3,4,5]))) AS res +SELECT bitmapToArray(bitmapOr(bitmapBuild([1,2,3]),bitmapBuild([3,4,5]))) AS res; ``` ``` text @@ -357,12 +363,12 @@ bitmapXor(bitmap,bitmap) **Arguments** -- `bitmap` – bitmap object. +- `bitmap` – Bitmap object. **Example** ``` sql -SELECT bitmapToArray(bitmapXor(bitmapBuild([1,2,3]),bitmapBuild([3,4,5]))) AS res +SELECT bitmapToArray(bitmapXor(bitmapBuild([1,2,3]),bitmapBuild([3,4,5]))) AS res; ``` ``` text @@ -381,12 +387,12 @@ bitmapAndnot(bitmap,bitmap) **Arguments** -- `bitmap` – bitmap object. +- `bitmap` – Bitmap object. **Example** ``` sql -SELECT bitmapToArray(bitmapAndnot(bitmapBuild([1,2,3]),bitmapBuild([3,4,5]))) AS res +SELECT bitmapToArray(bitmapAndnot(bitmapBuild([1,2,3]),bitmapBuild([3,4,5]))) AS res; ``` ``` text @@ -405,7 +411,7 @@ bitmapAndCardinality(bitmap,bitmap) **Arguments** -- `bitmap` – bitmap object. +- `bitmap` – Bitmap object. **Example** @@ -429,7 +435,7 @@ bitmapOrCardinality(bitmap,bitmap) **Arguments** -- `bitmap` – bitmap object. +- `bitmap` – Bitmap object. **Example** @@ -453,7 +459,7 @@ bitmapXorCardinality(bitmap,bitmap) **Arguments** -- `bitmap` – bitmap object. +- `bitmap` – Bitmap object. **Example** @@ -477,7 +483,7 @@ bitmapAndnotCardinality(bitmap,bitmap) **Arguments** -- `bitmap` – bitmap object. +- `bitmap` – Bitmap object. **Example** diff --git a/docs/en/sql-reference/functions/conditional-functions.md b/docs/en/sql-reference/functions/conditional-functions.md index 2d57cbb3bd5..462307012b5 100644 --- a/docs/en/sql-reference/functions/conditional-functions.md +++ b/docs/en/sql-reference/functions/conditional-functions.md @@ -20,8 +20,8 @@ If the condition `cond` evaluates to a non-zero value, returns the result of the **Arguments** - `cond` – The condition for evaluation that can be zero or not. The type is UInt8, Nullable(UInt8) or NULL. -- `then` - The expression to return if condition is met. -- `else` - The expression to return if condition is not met. +- `then` – The expression to return if condition is met. +- `else` – The expression to return if condition is not met. **Returned values** @@ -32,7 +32,7 @@ The function executes `then` and `else` expressions and returns its result, depe Query: ``` sql -SELECT if(1, plus(2, 2), plus(2, 6)) +SELECT if(1, plus(2, 2), plus(2, 6)); ``` Result: @@ -46,7 +46,7 @@ Result: Query: ``` sql -SELECT if(0, plus(2, 2), plus(2, 6)) +SELECT if(0, plus(2, 2), plus(2, 6)); ``` Result: diff --git a/docs/en/sql-reference/functions/date-time-functions.md b/docs/en/sql-reference/functions/date-time-functions.md index 01c8ae59e02..77a64eee41f 100644 --- a/docs/en/sql-reference/functions/date-time-functions.md +++ b/docs/en/sql-reference/functions/date-time-functions.md @@ -470,7 +470,7 @@ Aliases: `dateAdd`, `DATE_ADD`. - `unit` — The type of interval to add. [String](../../sql-reference/data-types/string.md). Supported values: second, minute, hour, day, week, month, quarter, year. -- `value` - Value in specified unit - [Int](../../sql-reference/data-types/int-uint.md) +- `value` — Value in specified unit - [Int](../../sql-reference/data-types/int-uint.md) - `date` — [Date](../../sql-reference/data-types/date.md) or [DateTime](../../sql-reference/data-types/datetime.md). diff --git a/docs/en/sql-reference/functions/encoding-functions.md b/docs/en/sql-reference/functions/encoding-functions.md index c1013ebb0e1..3779e5accc5 100644 --- a/docs/en/sql-reference/functions/encoding-functions.md +++ b/docs/en/sql-reference/functions/encoding-functions.md @@ -30,7 +30,7 @@ Type: `String`. Query: ``` sql -SELECT char(104.1, 101, 108.9, 108.9, 111) AS hello +SELECT char(104.1, 101, 108.9, 108.9, 111) AS hello; ``` Result: diff --git a/docs/en/sql-reference/functions/functions-for-nulls.md b/docs/en/sql-reference/functions/functions-for-nulls.md index f57f0f7e27d..43e38f459d0 100644 --- a/docs/en/sql-reference/functions/functions-for-nulls.md +++ b/docs/en/sql-reference/functions/functions-for-nulls.md @@ -38,7 +38,7 @@ Input table Query ``` sql -SELECT x FROM t_null WHERE isNull(y) +SELECT x FROM t_null WHERE isNull(y); ``` ``` text @@ -78,7 +78,7 @@ Input table Query ``` sql -SELECT x FROM t_null WHERE isNotNull(y) +SELECT x FROM t_null WHERE isNotNull(y); ``` ``` text @@ -120,7 +120,7 @@ The `mail` and `phone` fields are of type String, but the `icq` field is `UInt32 Get the first available contact method for the customer from the contact list: ``` sql -SELECT coalesce(mail, phone, CAST(icq,'Nullable(String)')) FROM aBook +SELECT coalesce(mail, phone, CAST(icq,'Nullable(String)')) FROM aBook; ``` ``` text @@ -151,7 +151,7 @@ ifNull(x,alt) **Example** ``` sql -SELECT ifNull('a', 'b') +SELECT ifNull('a', 'b'); ``` ``` text @@ -161,7 +161,7 @@ SELECT ifNull('a', 'b') ``` ``` sql -SELECT ifNull(NULL, 'b') +SELECT ifNull(NULL, 'b'); ``` ``` text @@ -190,7 +190,7 @@ nullIf(x, y) **Example** ``` sql -SELECT nullIf(1, 1) +SELECT nullIf(1, 1); ``` ``` text @@ -200,7 +200,7 @@ SELECT nullIf(1, 1) ``` ``` sql -SELECT nullIf(1, 2) +SELECT nullIf(1, 2); ``` ``` text @@ -231,7 +231,7 @@ assumeNotNull(x) Consider the `t_null` table. ``` sql -SHOW CREATE TABLE t_null +SHOW CREATE TABLE t_null; ``` ``` text @@ -250,7 +250,7 @@ SHOW CREATE TABLE t_null Apply the `assumeNotNull` function to the `y` column. ``` sql -SELECT assumeNotNull(y) FROM t_null +SELECT assumeNotNull(y) FROM t_null; ``` ``` text @@ -261,7 +261,7 @@ SELECT assumeNotNull(y) FROM t_null ``` ``` sql -SELECT toTypeName(assumeNotNull(y)) FROM t_null +SELECT toTypeName(assumeNotNull(y)) FROM t_null; ``` ``` text @@ -290,7 +290,7 @@ toNullable(x) **Example** ``` sql -SELECT toTypeName(10) +SELECT toTypeName(10); ``` ``` text @@ -300,7 +300,7 @@ SELECT toTypeName(10) ``` ``` sql -SELECT toTypeName(toNullable(10)) +SELECT toTypeName(toNullable(10)); ``` ``` text diff --git a/docs/en/sql-reference/functions/geo/geohash.md b/docs/en/sql-reference/functions/geo/geohash.md index c27eab0b421..cfe35746809 100644 --- a/docs/en/sql-reference/functions/geo/geohash.md +++ b/docs/en/sql-reference/functions/geo/geohash.md @@ -29,7 +29,7 @@ geohashEncode(longitude, latitude, [precision]) **Example** ``` sql -SELECT geohashEncode(-5.60302734375, 42.593994140625, 0) AS res +SELECT geohashEncode(-5.60302734375, 42.593994140625, 0) AS res; ``` ``` text @@ -53,7 +53,7 @@ Decodes any [geohash](#geohash)-encoded string into longitude and latitude. **Example** ``` sql -SELECT geohashDecode('ezs42') AS res +SELECT geohashDecode('ezs42') AS res; ``` ``` text @@ -98,8 +98,9 @@ Type: [Array](../../../sql-reference/data-types/array.md)([String](../../../sql- Query: ``` sql -SELECT geohashesInBox(24.48, 40.56, 24.785, 40.81, 4) AS thasos +SELECT geohashesInBox(24.48, 40.56, 24.785, 40.81, 4) AS thasos; ``` + Result: ``` text diff --git a/docs/en/sql-reference/functions/geo/h3.md b/docs/en/sql-reference/functions/geo/h3.md index 9dda947b3a7..20dc7b29902 100644 --- a/docs/en/sql-reference/functions/geo/h3.md +++ b/docs/en/sql-reference/functions/geo/h3.md @@ -40,8 +40,9 @@ Type: [UInt8](../../../sql-reference/data-types/int-uint.md). Query: ``` sql -SELECT h3IsValid(630814730351855103) as h3IsValid +SELECT h3IsValid(630814730351855103) as h3IsValid; ``` + Result: ``` text @@ -76,8 +77,9 @@ Type: [UInt8](../../../sql-reference/data-types/int-uint.md). Query: ``` sql -SELECT h3GetResolution(639821929606596015) as resolution +SELECT h3GetResolution(639821929606596015) as resolution; ``` + Result: ``` text @@ -109,8 +111,9 @@ h3EdgeAngle(resolution) Query: ``` sql -SELECT h3EdgeAngle(10) as edgeAngle +SELECT h3EdgeAngle(10) as edgeAngle; ``` + Result: ``` text @@ -142,8 +145,9 @@ h3EdgeLengthM(resolution) Query: ``` sql -SELECT h3EdgeLengthM(15) as edgeLengthM +SELECT h3EdgeLengthM(15) as edgeLengthM; ``` + Result: ``` text @@ -180,7 +184,7 @@ Type: [UInt64](../../../sql-reference/data-types/int-uint.md). Query: ``` sql -SELECT geoToH3(37.79506683, 55.71290588, 15) as h3Index +SELECT geoToH3(37.79506683, 55.71290588, 15) as h3Index; ``` Result: @@ -217,8 +221,9 @@ Type: [Array](../../../sql-reference/data-types/array.md)([UInt64](../../../sql- Query: ``` sql -SELECT arrayJoin(h3kRing(644325529233966508, 1)) AS h3index +SELECT arrayJoin(h3kRing(644325529233966508, 1)) AS h3index; ``` + Result: ``` text diff --git a/docs/en/sql-reference/functions/hash-functions.md b/docs/en/sql-reference/functions/hash-functions.md index 465ad01527f..cb850101311 100644 --- a/docs/en/sql-reference/functions/hash-functions.md +++ b/docs/en/sql-reference/functions/hash-functions.md @@ -29,7 +29,7 @@ A [UInt64](../../sql-reference/data-types/int-uint.md) data type hash value. **Example** ``` sql -SELECT halfMD5(array('e','x','a'), 'mple', 10, toDateTime('2019-06-15 23:00:00')) AS halfMD5hash, toTypeName(halfMD5hash) AS type +SELECT halfMD5(array('e','x','a'), 'mple', 10, toDateTime('2019-06-15 23:00:00')) AS halfMD5hash, toTypeName(halfMD5hash) AS type; ``` ``` text @@ -72,7 +72,7 @@ A [UInt64](../../sql-reference/data-types/int-uint.md) data type hash value. **Example** ``` sql -SELECT sipHash64(array('e','x','a'), 'mple', 10, toDateTime('2019-06-15 23:00:00')) AS SipHash, toTypeName(SipHash) AS type +SELECT sipHash64(array('e','x','a'), 'mple', 10, toDateTime('2019-06-15 23:00:00')) AS SipHash, toTypeName(SipHash) AS type; ``` ``` text @@ -110,7 +110,7 @@ A [UInt64](../../sql-reference/data-types/int-uint.md) data type hash value. Call example: ``` sql -SELECT cityHash64(array('e','x','a'), 'mple', 10, toDateTime('2019-06-15 23:00:00')) AS CityHash, toTypeName(CityHash) AS type +SELECT cityHash64(array('e','x','a'), 'mple', 10, toDateTime('2019-06-15 23:00:00')) AS CityHash, toTypeName(CityHash) AS type; ``` ``` text @@ -177,7 +177,7 @@ A [UInt64](../../sql-reference/data-types/int-uint.md) data type hash value. **Example** ``` sql -SELECT farmHash64(array('e','x','a'), 'mple', 10, toDateTime('2019-06-15 23:00:00')) AS FarmHash, toTypeName(FarmHash) AS type +SELECT farmHash64(array('e','x','a'), 'mple', 10, toDateTime('2019-06-15 23:00:00')) AS FarmHash, toTypeName(FarmHash) AS type; ``` ``` text @@ -193,7 +193,7 @@ Calculates [JavaHash](http://hg.openjdk.java.net/jdk8u/jdk8u/jdk/file/478a4add97 **Syntax** ``` sql -SELECT javaHash(''); +SELECT javaHash('') ``` **Returned value** @@ -241,7 +241,7 @@ Correct query with UTF-16LE encoded string. Query: ``` sql -SELECT javaHashUTF16LE(convertCharset('test', 'utf-8', 'utf-16le')) +SELECT javaHashUTF16LE(convertCharset('test', 'utf-8', 'utf-16le')); ``` Result: @@ -257,7 +257,7 @@ Result: Calculates `HiveHash` from a string. ``` sql -SELECT hiveHash(''); +SELECT hiveHash('') ``` This is just [JavaHash](#hash_functions-javahash) with zeroed out sign bit. This function is used in [Apache Hive](https://en.wikipedia.org/wiki/Apache_Hive) for versions before 3.0. This hash function is neither fast nor having a good quality. The only reason to use it is when this algorithm is already used in another system and you have to calculate exactly the same result. @@ -303,7 +303,7 @@ A [UInt64](../../sql-reference/data-types/int-uint.md) data type hash value. **Example** ``` sql -SELECT metroHash64(array('e','x','a'), 'mple', 10, toDateTime('2019-06-15 23:00:00')) AS MetroHash, toTypeName(MetroHash) AS type +SELECT metroHash64(array('e','x','a'), 'mple', 10, toDateTime('2019-06-15 23:00:00')) AS MetroHash, toTypeName(MetroHash) AS type; ``` ``` text @@ -339,7 +339,7 @@ Both functions take a variable number of input parameters. Arguments can be any **Example** ``` sql -SELECT murmurHash2_64(array('e','x','a'), 'mple', 10, toDateTime('2019-06-15 23:00:00')) AS MurmurHash2, toTypeName(MurmurHash2) AS type +SELECT murmurHash2_64(array('e','x','a'), 'mple', 10, toDateTime('2019-06-15 23:00:00')) AS MurmurHash2, toTypeName(MurmurHash2) AS type; ``` ``` text @@ -355,7 +355,7 @@ Calculates a 64-bit [MurmurHash2](https://github.com/aappleby/smhasher) hash val **Syntax** ``` sql -gccMurmurHash(par1, ...); +gccMurmurHash(par1, ...) ``` **Arguments** @@ -407,7 +407,7 @@ Both functions take a variable number of input parameters. Arguments can be any **Example** ``` sql -SELECT murmurHash3_32(array('e','x','a'), 'mple', 10, toDateTime('2019-06-15 23:00:00')) AS MurmurHash3, toTypeName(MurmurHash3) AS type +SELECT murmurHash3_32(array('e','x','a'), 'mple', 10, toDateTime('2019-06-15 23:00:00')) AS MurmurHash3, toTypeName(MurmurHash3) AS type; ``` ``` text @@ -435,7 +435,7 @@ A [FixedString(16)](../../sql-reference/data-types/fixedstring.md) data type has **Example** ``` sql -SELECT murmurHash3_128('example_string') AS MurmurHash3, toTypeName(MurmurHash3) AS type +SELECT murmurHash3_128('example_string') AS MurmurHash3, toTypeName(MurmurHash3) AS type; ``` ``` text @@ -449,11 +449,11 @@ SELECT murmurHash3_128('example_string') AS MurmurHash3, toTypeName(MurmurHash3) Calculates `xxHash` from a string. It is proposed in two flavors, 32 and 64 bits. ``` sql -SELECT xxHash32(''); +SELECT xxHash32('') OR -SELECT xxHash64(''); +SELECT xxHash64('') ``` **Returned value** diff --git a/docs/en/sql-reference/functions/introspection.md b/docs/en/sql-reference/functions/introspection.md index 964265a461b..823fd4dfebe 100644 --- a/docs/en/sql-reference/functions/introspection.md +++ b/docs/en/sql-reference/functions/introspection.md @@ -53,13 +53,13 @@ Type: [String](../../sql-reference/data-types/string.md). Enabling introspection functions: ``` sql -SET allow_introspection_functions=1 +SET allow_introspection_functions=1; ``` Selecting the first string from the `trace_log` system table: ``` sql -SELECT * FROM system.trace_log LIMIT 1 \G +SELECT * FROM system.trace_log LIMIT 1 \G; ``` ``` text @@ -79,7 +79,7 @@ The `trace` field contains the stack trace at the moment of sampling. Getting the source code filename and the line number for a single address: ``` sql -SELECT addressToLine(94784076370703) \G +SELECT addressToLine(94784076370703) \G; ``` ``` text @@ -139,13 +139,13 @@ Type: [String](../../sql-reference/data-types/string.md). Enabling introspection functions: ``` sql -SET allow_introspection_functions=1 +SET allow_introspection_functions=1; ``` Selecting the first string from the `trace_log` system table: ``` sql -SELECT * FROM system.trace_log LIMIT 1 \G +SELECT * FROM system.trace_log LIMIT 1 \G; ``` ``` text @@ -165,7 +165,7 @@ The `trace` field contains the stack trace at the moment of sampling. Getting a symbol for a single address: ``` sql -SELECT addressToSymbol(94138803686098) \G +SELECT addressToSymbol(94138803686098) \G; ``` ``` text @@ -236,13 +236,13 @@ Type: [String](../../sql-reference/data-types/string.md). Enabling introspection functions: ``` sql -SET allow_introspection_functions=1 +SET allow_introspection_functions=1; ``` Selecting the first string from the `trace_log` system table: ``` sql -SELECT * FROM system.trace_log LIMIT 1 \G +SELECT * FROM system.trace_log LIMIT 1 \G; ``` ``` text @@ -262,7 +262,7 @@ The `trace` field contains the stack trace at the moment of sampling. Getting a function name for a single address: ``` sql -SELECT demangle(addressToSymbol(94138803686098)) \G +SELECT demangle(addressToSymbol(94138803686098)) \G; ``` ``` text @@ -335,6 +335,7 @@ Result: │ 3878 │ └───────┘ ``` + ## logTrace {#logtrace} Emits trace log message to server log for each [Block](https://clickhouse.tech/docs/en/development/architecture/#block). diff --git a/docs/en/sql-reference/functions/ip-address-functions.md b/docs/en/sql-reference/functions/ip-address-functions.md index 64457627cce..9217838cd35 100644 --- a/docs/en/sql-reference/functions/ip-address-functions.md +++ b/docs/en/sql-reference/functions/ip-address-functions.md @@ -60,7 +60,7 @@ Alias: `INET6_NTOA`. Examples: ``` sql -SELECT IPv6NumToString(toFixedString(unhex('2A0206B8000000000000000000000011'), 16)) AS addr +SELECT IPv6NumToString(toFixedString(unhex('2A0206B8000000000000000000000011'), 16)) AS addr; ``` ``` text @@ -164,7 +164,7 @@ Result: └────────────┴──────────────────────────────────────┘ ``` -**See also** +**See Also** - [cutIPv6](#cutipv6x-bytestocutforipv6-bytestocutforipv4). @@ -173,7 +173,7 @@ Result: Takes a `UInt32` number. Interprets it as an IPv4 address in [big endian](https://en.wikipedia.org/wiki/Endianness). Returns a `FixedString(16)` value containing the IPv6 address in binary format. Examples: ``` sql -SELECT IPv6NumToString(IPv4ToIPv6(IPv4StringToNum('192.168.0.1'))) AS addr +SELECT IPv6NumToString(IPv4ToIPv6(IPv4StringToNum('192.168.0.1'))) AS addr; ``` ``` text @@ -206,7 +206,7 @@ SELECT Accepts an IPv4 and an UInt8 value containing the [CIDR](https://en.wikipedia.org/wiki/Classless_Inter-Domain_Routing). Return a tuple with two IPv4 containing the lower range and the higher range of the subnet. ``` sql -SELECT IPv4CIDRToRange(toIPv4('192.168.5.2'), 16) +SELECT IPv4CIDRToRange(toIPv4('192.168.5.2'), 16); ``` ``` text @@ -342,7 +342,7 @@ Type: [UInt8](../../sql-reference/data-types/int-uint.md). Query: ```sql -SELECT addr, isIPv4String(addr) FROM ( SELECT ['0.0.0.0', '127.0.0.1', '::ffff:127.0.0.1'] AS addr ) ARRAY JOIN addr +SELECT addr, isIPv4String(addr) FROM ( SELECT ['0.0.0.0', '127.0.0.1', '::ffff:127.0.0.1'] AS addr ) ARRAY JOIN addr; ``` Result: @@ -380,7 +380,7 @@ Type: [UInt8](../../sql-reference/data-types/int-uint.md). Query: ``` sql -SELECT addr, isIPv6String(addr) FROM ( SELECT ['::', '1111::ffff', '::ffff:127.0.0.1', '127.0.0.1'] AS addr ) ARRAY JOIN addr +SELECT addr, isIPv6String(addr) FROM ( SELECT ['::', '1111::ffff', '::ffff:127.0.0.1', '127.0.0.1'] AS addr ) ARRAY JOIN addr; ``` Result: diff --git a/docs/en/sql-reference/functions/json-functions.md b/docs/en/sql-reference/functions/json-functions.md index edee048eb77..6c2a807492d 100644 --- a/docs/en/sql-reference/functions/json-functions.md +++ b/docs/en/sql-reference/functions/json-functions.md @@ -199,7 +199,7 @@ Parses key-value pairs from a JSON where the values are of the given ClickHouse Example: ``` sql -SELECT JSONExtractKeysAndValues('{"x": {"a": 5, "b": 7, "c": 11}}', 'x', 'Int8') = [('a',5),('b',7),('c',11)] +SELECT JSONExtractKeysAndValues('{"x": {"a": 5, "b": 7, "c": 11}}', 'x', 'Int8') = [('a',5),('b',7),('c',11)]; ``` ## JSONExtractRaw(json\[, indices_or_keys\]…) {#jsonextractrawjson-indices-or-keys} @@ -211,7 +211,7 @@ If the part does not exist or has a wrong type, an empty string will be returned Example: ``` sql -SELECT JSONExtractRaw('{"a": "hello", "b": [-100, 200.0, 300]}', 'b') = '[-100, 200.0, 300]' +SELECT JSONExtractRaw('{"a": "hello", "b": [-100, 200.0, 300]}', 'b') = '[-100, 200.0, 300]'; ``` ## JSONExtractArrayRaw(json\[, indices_or_keys…\]) {#jsonextractarrayrawjson-indices-or-keys} @@ -223,7 +223,7 @@ If the part does not exist or isn’t array, an empty array will be returned. Example: ``` sql -SELECT JSONExtractArrayRaw('{"a": "hello", "b": [-100, 200.0, "hello"]}', 'b') = ['-100', '200.0', '"hello"']' +SELECT JSONExtractArrayRaw('{"a": "hello", "b": [-100, 200.0, "hello"]}', 'b') = ['-100', '200.0', '"hello"']'; ``` ## JSONExtractKeysAndValuesRaw {#json-extract-keys-and-values-raw} @@ -253,7 +253,7 @@ Type: [Array](../../sql-reference/data-types/array.md)([Tuple](../../sql-referen Query: ``` sql -SELECT JSONExtractKeysAndValuesRaw('{"a": [-100, 200.0], "b":{"c": {"d": "hello", "f": "world"}}}') +SELECT JSONExtractKeysAndValuesRaw('{"a": [-100, 200.0], "b":{"c": {"d": "hello", "f": "world"}}}'); ``` Result: @@ -267,7 +267,7 @@ Result: Query: ``` sql -SELECT JSONExtractKeysAndValuesRaw('{"a": [-100, 200.0], "b":{"c": {"d": "hello", "f": "world"}}}', 'b') +SELECT JSONExtractKeysAndValuesRaw('{"a": [-100, 200.0], "b":{"c": {"d": "hello", "f": "world"}}}', 'b'); ``` Result: @@ -281,7 +281,7 @@ Result: Query: ``` sql -SELECT JSONExtractKeysAndValuesRaw('{"a": [-100, 200.0], "b":{"c": {"d": "hello", "f": "world"}}}', -1, 'c') +SELECT JSONExtractKeysAndValuesRaw('{"a": [-100, 200.0], "b":{"c": {"d": "hello", "f": "world"}}}', -1, 'c'); ``` Result: diff --git a/docs/en/sql-reference/functions/machine-learning-functions.md b/docs/en/sql-reference/functions/machine-learning-functions.md index f103a4ea421..d1bb66993b2 100644 --- a/docs/en/sql-reference/functions/machine-learning-functions.md +++ b/docs/en/sql-reference/functions/machine-learning-functions.md @@ -36,14 +36,14 @@ bayesAB(distribution_name, higher_is_better, variant_names, x, y) - `higher_is_better` — Boolean flag. [Boolean](../../sql-reference/data-types/boolean.md). Possible values: - - `0` - lower values are considered to be better than higher - - `1` - higher values are considered to be better than lower + - `0` — lower values are considered to be better than higher + - `1` — higher values are considered to be better than lower -- `variant_names` - Variant names. [Array](../../sql-reference/data-types/array.md)([String](../../sql-reference/data-types/string.md)). +- `variant_names` — Variant names. [Array](../../sql-reference/data-types/array.md)([String](../../sql-reference/data-types/string.md)). -- `x` - Numbers of tests for the corresponding variants. [Array](../../sql-reference/data-types/array.md)([Float64](../../sql-reference/data-types/float.md)). +- `x` — Numbers of tests for the corresponding variants. [Array](../../sql-reference/data-types/array.md)([Float64](../../sql-reference/data-types/float.md)). -- `y` - Numbers of successful tests for the corresponding variants. [Array](../../sql-reference/data-types/array.md)([Float64](../../sql-reference/data-types/float.md)). +- `y` — Numbers of successful tests for the corresponding variants. [Array](../../sql-reference/data-types/array.md)([Float64](../../sql-reference/data-types/float.md)). !!! note "Note" All three arrays must have the same size. All `x` and `y` values must be non-negative constant numbers. `y` cannot be larger than `x`. @@ -51,8 +51,8 @@ bayesAB(distribution_name, higher_is_better, variant_names, x, y) **Returned values** For each variant the function calculates: -- `beats_control` - long-term probability to out-perform the first (control) variant -- `to_be_best` - long-term probability to out-perform all other variants +- `beats_control` — long-term probability to out-perform the first (control) variant +- `to_be_best` — long-term probability to out-perform all other variants Type: JSON. diff --git a/docs/en/sql-reference/functions/math-functions.md b/docs/en/sql-reference/functions/math-functions.md index f56a721c0c0..843c1e52454 100644 --- a/docs/en/sql-reference/functions/math-functions.md +++ b/docs/en/sql-reference/functions/math-functions.md @@ -54,7 +54,7 @@ If ‘x’ is non-negative, then `erf(x / σ√2)` is the probability that a ran Example (three sigma rule): ``` sql -SELECT erf(3 / sqrt(2)) +SELECT erf(3 / sqrt(2)); ``` ``` text diff --git a/docs/en/sql-reference/functions/other-functions.md b/docs/en/sql-reference/functions/other-functions.md index 2c7f8da881e..d99d671418f 100644 --- a/docs/en/sql-reference/functions/other-functions.md +++ b/docs/en/sql-reference/functions/other-functions.md @@ -1762,7 +1762,6 @@ Result: ``` - ## randomStringUTF8 {#randomstringutf8} Generates a random string of a specified length. Result string contains valid UTF-8 code points. The value of code points may be outside of the range of assigned Unicode. diff --git a/docs/en/sql-reference/functions/rounding-functions.md b/docs/en/sql-reference/functions/rounding-functions.md index 83db1975366..102a1fff5a0 100644 --- a/docs/en/sql-reference/functions/rounding-functions.md +++ b/docs/en/sql-reference/functions/rounding-functions.md @@ -35,7 +35,7 @@ The function returns the nearest number of the specified order. In case when giv round(expression [, decimal_places]) ``` -**Arguments:** +**Arguments** - `expression` — A number to be rounded. Can be any [expression](../../sql-reference/syntax.md#syntax-expressions) returning the numeric [data type](../../sql-reference/data-types/index.md#data_types). - `decimal-places` — An integer value. diff --git a/docs/en/sql-reference/functions/string-functions.md b/docs/en/sql-reference/functions/string-functions.md index 2c08fa3acb7..40481975886 100644 --- a/docs/en/sql-reference/functions/string-functions.md +++ b/docs/en/sql-reference/functions/string-functions.md @@ -73,19 +73,19 @@ Returns 1, if the set of bytes is valid UTF-8 encoded, otherwise 0. Replaces invalid UTF-8 characters by the `�` (U+FFFD) character. All running in a row invalid characters are collapsed into the one replacement character. ``` sql -toValidUTF8( input_string ) +toValidUTF8(input_string) ``` **Arguments** -- input_string — Any set of bytes represented as the [String](../../sql-reference/data-types/string.md) data type object. +- `input_string` — Any set of bytes represented as the [String](../../sql-reference/data-types/string.md) data type object. Returned value: Valid UTF-8 string. **Example** ``` sql -SELECT toValidUTF8('\x61\xF0\x80\x80\x80b') +SELECT toValidUTF8('\x61\xF0\x80\x80\x80b'); ``` ``` text @@ -122,7 +122,7 @@ Type: `String`. Query: ``` sql -SELECT repeat('abc', 10) +SELECT repeat('abc', 10); ``` Result: @@ -190,7 +190,7 @@ If any of argument values is `NULL`, `concat` returns `NULL`. Query: ``` sql -SELECT concat('Hello, ', 'World!') +SELECT concat('Hello, ', 'World!'); ``` Result: @@ -245,7 +245,7 @@ SELECT * from key_val; Query: ``` sql -SELECT concat(key1, key2), sum(value) FROM key_val GROUP BY concatAssumeInjective(key1, key2) +SELECT concat(key1, key2), sum(value) FROM key_val GROUP BY concatAssumeInjective(key1, key2); ``` Result: @@ -336,8 +336,8 @@ trim([[LEADING|TRAILING|BOTH] trim_character FROM] input_string) **Arguments** -- `trim_character` — specified characters for trim. [String](../../sql-reference/data-types/string.md). -- `input_string` — string for trim. [String](../../sql-reference/data-types/string.md). +- `trim_character` — Specified characters for trim. [String](../../sql-reference/data-types/string.md). +- `input_string` — String for trim. [String](../../sql-reference/data-types/string.md). **Returned value** @@ -350,7 +350,7 @@ Type: `String`. Query: ``` sql -SELECT trim(BOTH ' ()' FROM '( Hello, world! )') +SELECT trim(BOTH ' ()' FROM '( Hello, world! )'); ``` Result: @@ -388,7 +388,7 @@ Type: `String`. Query: ``` sql -SELECT trimLeft(' Hello, world! ') +SELECT trimLeft(' Hello, world! '); ``` Result: @@ -426,7 +426,7 @@ Type: `String`. Query: ``` sql -SELECT trimRight(' Hello, world! ') +SELECT trimRight(' Hello, world! '); ``` Result: @@ -464,7 +464,7 @@ Type: `String`. Query: ``` sql -SELECT trimBoth(' Hello, world! ') +SELECT trimBoth(' Hello, world! '); ``` Result: @@ -497,7 +497,8 @@ The result type is UInt64. Replaces literals, sequences of literals and complex aliases with placeholders. -**Syntax** +**Syntax** + ``` sql normalizeQuery(x) ``` @@ -617,7 +618,7 @@ This function also replaces numeric character references with Unicode characters decodeXMLComponent(x) ``` -**Parameters** +**Arguments** - `x` — A sequence of characters. [String](../../sql-reference/data-types/string.md). diff --git a/docs/en/sql-reference/functions/string-search-functions.md b/docs/en/sql-reference/functions/string-search-functions.md index 83b0edea438..6ca7473cdc2 100644 --- a/docs/en/sql-reference/functions/string-search-functions.md +++ b/docs/en/sql-reference/functions/string-search-functions.md @@ -26,9 +26,9 @@ Alias: `locate(haystack, needle[, start_pos])`. **Arguments** -- `haystack` — string, in which substring will to be searched. [String](../../sql-reference/syntax.md#syntax-string-literal). -- `needle` — substring to be searched. [String](../../sql-reference/syntax.md#syntax-string-literal). -- `start_pos` – Optional parameter, position of the first character in the string to start search. [UInt](../../sql-reference/data-types/int-uint.md) +- `haystack` — String, in which substring will to be searched. [String](../../sql-reference/syntax.md#syntax-string-literal). +- `needle` — Substring to be searched. [String](../../sql-reference/syntax.md#syntax-string-literal). +- `start_pos` — Optional parameter, position of the first character in the string to start search. [UInt](../../sql-reference/data-types/int-uint.md). **Returned values** @@ -44,7 +44,7 @@ The phrase “Hello, world!” contains a set of bytes representing a single-byt Query: ``` sql -SELECT position('Hello, world!', '!') +SELECT position('Hello, world!', '!'); ``` Result: @@ -72,7 +72,7 @@ The same phrase in Russian contains characters which can’t be represented usin Query: ``` sql -SELECT position('Привет, мир!', '!') +SELECT position('Привет, мир!', '!'); ``` Result: @@ -97,9 +97,9 @@ positionCaseInsensitive(haystack, needle[, start_pos]) **Arguments** -- `haystack` — string, in which substring will to be searched. [String](../../sql-reference/syntax.md#syntax-string-literal). -- `needle` — substring to be searched. [String](../../sql-reference/syntax.md#syntax-string-literal). -- `start_pos` – Optional parameter, position of the first character in the string to start search. [UInt](../../sql-reference/data-types/int-uint.md) +- `haystack` — String, in which substring will to be searched. [String](../../sql-reference/syntax.md#syntax-string-literal). +- `needle` — Substring to be searched. [String](../../sql-reference/syntax.md#syntax-string-literal). +- `start_pos` — Optional parameter, position of the first character in the string to start search. [UInt](../../sql-reference/data-types/int-uint.md). **Returned values** @@ -113,7 +113,7 @@ Type: `Integer`. Query: ``` sql -SELECT positionCaseInsensitive('Hello, world!', 'hello') +SELECT positionCaseInsensitive('Hello, world!', 'hello'); ``` Result: @@ -140,9 +140,9 @@ positionUTF8(haystack, needle[, start_pos]) **Arguments** -- `haystack` — string, in which substring will to be searched. [String](../../sql-reference/syntax.md#syntax-string-literal). -- `needle` — substring to be searched. [String](../../sql-reference/syntax.md#syntax-string-literal). -- `start_pos` – Optional parameter, position of the first character in the string to start search. [UInt](../../sql-reference/data-types/int-uint.md) +- `haystack` — String, in which substring will to be searched. [String](../../sql-reference/syntax.md#syntax-string-literal). +- `needle` — Substring to be searched. [String](../../sql-reference/syntax.md#syntax-string-literal). +- `start_pos` — Optional parameter, position of the first character in the string to start search. [UInt](../../sql-reference/data-types/int-uint.md) **Returned values** @@ -158,7 +158,7 @@ The phrase “Hello, world!” in Russian contains a set of Unicode points repre Query: ``` sql -SELECT positionUTF8('Привет, мир!', '!') +SELECT positionUTF8('Привет, мир!', '!'); ``` Result: @@ -174,7 +174,7 @@ The phrase “Salut, étudiante!”, where character `é` can be represented usi Query for the letter `é`, which is represented one Unicode point `U+00E9`: ``` sql -SELECT positionUTF8('Salut, étudiante!', '!') +SELECT positionUTF8('Salut, étudiante!', '!'); ``` Result: @@ -188,7 +188,7 @@ Result: Query for the letter `é`, which is represented two Unicode points `U+0065U+0301`: ``` sql -SELECT positionUTF8('Salut, étudiante!', '!') +SELECT positionUTF8('Salut, étudiante!', '!'); ``` Result: @@ -213,9 +213,9 @@ positionCaseInsensitiveUTF8(haystack, needle[, start_pos]) **Arguments** -- `haystack` — string, in which substring will to be searched. [String](../../sql-reference/syntax.md#syntax-string-literal). -- `needle` — substring to be searched. [String](../../sql-reference/syntax.md#syntax-string-literal). -- `start_pos` – Optional parameter, position of the first character in the string to start search. [UInt](../../sql-reference/data-types/int-uint.md) +- `haystack` — String, in which substring will to be searched. [String](../../sql-reference/syntax.md#syntax-string-literal). +- `needle` — Substring to be searched. [String](../../sql-reference/syntax.md#syntax-string-literal). +- `start_pos` — Optional parameter, position of the first character in the string to start search. [UInt](../../sql-reference/data-types/int-uint.md) **Returned value** @@ -229,7 +229,7 @@ Type: `Integer`. Query: ``` sql -SELECT positionCaseInsensitiveUTF8('Привет, мир!', 'Мир') +SELECT positionCaseInsensitiveUTF8('Привет, мир!', 'Мир'); ``` Result: @@ -258,8 +258,8 @@ multiSearchAllPositions(haystack, [needle1, needle2, ..., needlen]) **Arguments** -- `haystack` — string, in which substring will to be searched. [String](../../sql-reference/syntax.md#syntax-string-literal). -- `needle` — substring to be searched. [String](../../sql-reference/syntax.md#syntax-string-literal). +- `haystack` — String, in which substring will to be searched. [String](../../sql-reference/syntax.md#syntax-string-literal). +- `needle` — Substring to be searched. [String](../../sql-reference/syntax.md#syntax-string-literal). **Returned values** @@ -270,7 +270,7 @@ multiSearchAllPositions(haystack, [needle1, needle2, ..., needlen]) Query: ``` sql -SELECT multiSearchAllPositions('Hello, World!', ['hello', '!', 'world']) +SELECT multiSearchAllPositions('Hello, World!', ['hello', '!', 'world']); ``` Result: @@ -387,7 +387,7 @@ If `haystack` doesn’t match the `pattern` regex, an array of empty arrays is r Query: ``` sql -SELECT extractAllGroupsHorizontal('abc=111, def=222, ghi=333', '("[^"]+"|\\w+)=("[^"]+"|\\w+)') +SELECT extractAllGroupsHorizontal('abc=111, def=222, ghi=333', '("[^"]+"|\\w+)=("[^"]+"|\\w+)'); ``` Result: @@ -428,7 +428,7 @@ If `haystack` doesn’t match the `pattern` regex, an empty array is returned. Query: ``` sql -SELECT extractAllGroupsVertical('abc=111, def=222, ghi=333', '("[^"]+"|\\w+)=("[^"]+"|\\w+)') +SELECT extractAllGroupsVertical('abc=111, def=222, ghi=333', '("[^"]+"|\\w+)=("[^"]+"|\\w+)'); ``` Result: @@ -506,7 +506,7 @@ Input table: Query: ``` sql -SELECT * FROM Months WHERE ilike(name, '%j%') +SELECT * FROM Months WHERE ilike(name, '%j%'); ``` Result: @@ -618,7 +618,7 @@ countSubstringsCaseInsensitive(haystack, needle[, start_pos]) - `haystack` — The string to search in. [String](../../sql-reference/syntax.md#syntax-string-literal). - `needle` — The substring to search for. [String](../../sql-reference/syntax.md#syntax-string-literal). -- `start_pos` – Position of the first character in the string to start search. Optional. [UInt](../../sql-reference/data-types/int-uint.md). +- `start_pos` — Position of the first character in the string to start search. Optional. [UInt](../../sql-reference/data-types/int-uint.md). **Returned values** @@ -631,7 +631,7 @@ Type: [UInt64](../../sql-reference/data-types/int-uint.md). Query: ``` sql -select countSubstringsCaseInsensitive('aba', 'B'); +SELECT countSubstringsCaseInsensitive('aba', 'B'); ``` Result: @@ -684,7 +684,7 @@ SELECT countSubstringsCaseInsensitiveUTF8(haystack, needle[, start_pos]) - `haystack` — The string to search in. [String](../../sql-reference/syntax.md#syntax-string-literal). - `needle` — The substring to search for. [String](../../sql-reference/syntax.md#syntax-string-literal). -- `start_pos` – Position of the first character in the string to start search. Optional. [UInt](../../sql-reference/data-types/int-uint.md). +- `start_pos` — Position of the first character in the string to start search. Optional. [UInt](../../sql-reference/data-types/int-uint.md). **Returned values** diff --git a/docs/en/sql-reference/functions/tuple-functions.md b/docs/en/sql-reference/functions/tuple-functions.md index 1006b68b8ee..884e1ef754f 100644 --- a/docs/en/sql-reference/functions/tuple-functions.md +++ b/docs/en/sql-reference/functions/tuple-functions.md @@ -47,7 +47,7 @@ You can use the `EXCEPT` expression to skip columns as a result of the query. **Arguments** -- `x` - A `tuple` function, column, or tuple of elements. [Tuple](../../sql-reference/data-types/tuple.md). +- `x` — A `tuple` function, column, or tuple of elements. [Tuple](../../sql-reference/data-types/tuple.md). **Returned value** diff --git a/docs/en/sql-reference/functions/tuple-map-functions.md b/docs/en/sql-reference/functions/tuple-map-functions.md index 1d4839cbbf9..8b0710c0182 100644 --- a/docs/en/sql-reference/functions/tuple-map-functions.md +++ b/docs/en/sql-reference/functions/tuple-map-functions.md @@ -66,7 +66,6 @@ Result: - [Map(key, value)](../../sql-reference/data-types/map.md) data type - ## mapAdd {#function-mapadd} Collect all the keys and sum corresponding values. diff --git a/docs/en/sql-reference/functions/type-conversion-functions.md b/docs/en/sql-reference/functions/type-conversion-functions.md index 8a793b99ac9..8d2d253046b 100644 --- a/docs/en/sql-reference/functions/type-conversion-functions.md +++ b/docs/en/sql-reference/functions/type-conversion-functions.md @@ -381,7 +381,7 @@ This function accepts 16 bytes string, and returns UUID containing bytes represe reinterpretAsUUID(fixed_string) ``` -**Parameters** +**Arguments** - `fixed_string` — Big-endian byte string. [FixedString](../../sql-reference/data-types/fixedstring.md#fixedstring). diff --git a/docs/en/sql-reference/functions/url-functions.md b/docs/en/sql-reference/functions/url-functions.md index 9e79ef2d0cb..54ec463ae66 100644 --- a/docs/en/sql-reference/functions/url-functions.md +++ b/docs/en/sql-reference/functions/url-functions.md @@ -55,7 +55,7 @@ Type: `String`. **Example** ``` sql -SELECT domain('svn+ssh://some.svn-hosting.com:80/repo/trunk') +SELECT domain('svn+ssh://some.svn-hosting.com:80/repo/trunk'); ``` ``` text @@ -98,7 +98,7 @@ Type: `String`. **Example** ``` sql -SELECT topLevelDomain('svn+ssh://www.some.svn-hosting.com:80/repo/trunk') +SELECT topLevelDomain('svn+ssh://www.some.svn-hosting.com:80/repo/trunk'); ``` ``` text diff --git a/docs/en/sql-reference/functions/ym-dict-functions.md b/docs/en/sql-reference/functions/ym-dict-functions.md index 56530b5e83b..d42d44f8336 100644 --- a/docs/en/sql-reference/functions/ym-dict-functions.md +++ b/docs/en/sql-reference/functions/ym-dict-functions.md @@ -112,7 +112,7 @@ Finds the highest continent in the hierarchy for the region. **Syntax** ``` sql -regionToTopContinent(id[, geobase]); +regionToTopContinent(id[, geobase]) ``` **Arguments** diff --git a/docs/en/sql-reference/table-functions/generate.md b/docs/en/sql-reference/table-functions/generate.md index be6ba2b8bc4..6bd82d4e88b 100644 --- a/docs/en/sql-reference/table-functions/generate.md +++ b/docs/en/sql-reference/table-functions/generate.md @@ -10,7 +10,7 @@ Allows to populate test tables with data. Supports all data types that can be stored in table except `LowCardinality` and `AggregateFunction`. ``` sql -generateRandom('name TypeName[, name TypeName]...', [, 'random_seed'[, 'max_string_length'[, 'max_array_length']]]); +generateRandom('name TypeName[, name TypeName]...', [, 'random_seed'[, 'max_string_length'[, 'max_array_length']]]) ``` **Arguments** diff --git a/docs/en/sql-reference/table-functions/view.md b/docs/en/sql-reference/table-functions/view.md index 08096c2b019..73618dd2c96 100644 --- a/docs/en/sql-reference/table-functions/view.md +++ b/docs/en/sql-reference/table-functions/view.md @@ -37,7 +37,7 @@ Input table: Query: ``` sql -SELECT * FROM view(SELECT name FROM months) +SELECT * FROM view(SELECT name FROM months); ``` Result: @@ -54,14 +54,15 @@ Result: You can use the `view` function as a parameter of the [remote](https://clickhouse.tech/docs/en/sql-reference/table-functions/remote/#remote-remotesecure) and [cluster](https://clickhouse.tech/docs/en/sql-reference/table-functions/cluster/#cluster-clusterallreplicas) table functions: ``` sql -SELECT * FROM remote(`127.0.0.1`, view(SELECT a, b, c FROM table_name)) +SELECT * FROM remote(`127.0.0.1`, view(SELECT a, b, c FROM table_name)); ``` ``` sql -SELECT * FROM cluster(`cluster_name`, view(SELECT a, b, c FROM table_name)) +SELECT * FROM cluster(`cluster_name`, view(SELECT a, b, c FROM table_name)); ``` **See Also** - [View Table Engine](https://clickhouse.tech/docs/en/engines/table-engines/special/view/) -[Original article](https://clickhouse.tech/docs/en/query_language/table_functions/view/) \ No newline at end of file + +[Original article](https://clickhouse.tech/docs/en/query_language/table_functions/view/) diff --git a/docs/ru/getting-started/tutorial.md b/docs/ru/getting-started/tutorial.md index f5455ba2b9a..68b3e4dbae7 100644 --- a/docs/ru/getting-started/tutorial.md +++ b/docs/ru/getting-started/tutorial.md @@ -644,7 +644,7 @@ If there are no replicas at the moment on replicated table creation, a new first ``` sql CREATE TABLE tutorial.hits_replica (...) -ENGINE = ReplcatedMergeTree( +ENGINE = ReplicatedMergeTree( '/clickhouse_perftest/tables/{shard}/hits', '{replica}' ) diff --git a/docs/ru/sql-reference/aggregate-functions/combinators.md b/docs/ru/sql-reference/aggregate-functions/combinators.md index 592c61f87ff..5270a269111 100644 --- a/docs/ru/sql-reference/aggregate-functions/combinators.md +++ b/docs/ru/sql-reference/aggregate-functions/combinators.md @@ -70,9 +70,9 @@ toc_title: "\u041a\u043e\u043c\u0431\u0438\u043d\u0430\u0442\u043e\u0440\u044b\u OrDefault(x) ``` -**Параметры** +**Аргументы** -- `x` — Параметры агрегатной функции. +- `x` — параметры агрегатной функции. **Возращаемые зачения** @@ -131,14 +131,14 @@ FROM OrNull(x) ``` -**Параметры** +**Аргументы** -- `x` — Параметры агрегатной функции. +- `x` — параметры агрегатной функции. **Возвращаемые значения** -- Результат агрегатной функции, преобразованный в тип данных `Nullable`. -- `NULL`, если у агрегатной функции нет входных данных. +- Результат агрегатной функции, преобразованный в тип данных `Nullable`. +- `NULL`, если у агрегатной функции нет входных данных. Тип: `Nullable(aggregate function return type)`. @@ -188,7 +188,7 @@ FROM Resample(start, end, step)(, resampling_key) ``` -**Параметры** +**Аргументы** - `start` — начальное значение для интервала значений `resampling_key`. - `stop` — конечное значение для интервала значений `resampling_key`. Интервал не включает значение `stop` (`[start, stop)`). diff --git a/docs/ru/sql-reference/aggregate-functions/parametric-functions.md b/docs/ru/sql-reference/aggregate-functions/parametric-functions.md index d96f7a13bcc..83742fb3dc4 100644 --- a/docs/ru/sql-reference/aggregate-functions/parametric-functions.md +++ b/docs/ru/sql-reference/aggregate-functions/parametric-functions.md @@ -11,14 +11,19 @@ toc_title: "\u041f\u0430\u0440\u0430\u043c\u0435\u0442\u0440\u0438\u0447\u0435\u Рассчитывает адаптивную гистограмму. Не гарантирует точного результата. - histogram(number_of_bins)(values) +``` sql +histogram(number_of_bins)(values) +``` Функция использует [A Streaming Parallel Decision Tree Algorithm](http://jmlr.org/papers/volume11/ben-haim10a/ben-haim10a.pdf). Границы столбцов устанавливаются по мере поступления новых данных в функцию. В общем случае столбцы имею разную ширину. +**Аргументы** + +`values` — [выражение](../syntax.md#syntax-expressions), предоставляющее входные значения. + **Параметры** `number_of_bins` — максимальное количество корзин в гистограмме. Функция автоматически вычисляет количество корзин. Она пытается получить указанное количество корзин, но если не получилось, то в результате корзин будет меньше. -`values` — [выражение](../syntax.md#syntax-expressions), предоставляющее входные значения. **Возвращаемые значения** @@ -87,14 +92,16 @@ sequenceMatch(pattern)(timestamp, cond1, cond2, ...) !!! warning "Предупреждение" События, произошедшие в одну и ту же секунду, располагаются в последовательности в неопределенном порядке, что может повлиять на результат работы функции. -**Параметры** - -- `pattern` — строка с шаблоном. Смотрите [Синтаксис шаблонов](#sequence-function-pattern-syntax). +**Аргументы** - `timestamp` — столбец, содержащий метки времени. Типичный тип данных столбца — `Date` или `DateTime`. Также можно использовать любой из поддержанных типов данных [UInt](../../sql-reference/aggregate-functions/parametric-functions.md). - `cond1`, `cond2` — условия, описывающие цепочку событий. Тип данных — `UInt8`. Можно использовать до 32 условий. Функция учитывает только те события, которые указаны в условиях. Функция пропускает данные из последовательности, если они не описаны ни в одном из условий. +**Параметры** + +- `pattern` — строка с шаблоном. Смотрите [Синтаксис шаблонов](#sequence-function-pattern-syntax). + **Возвращаемые значения** - 1, если цепочка событий, соответствующая шаблону найдена. @@ -174,14 +181,16 @@ SELECT sequenceMatch('(?1)(?2)')(time, number = 1, number = 2, number = 4) FROM sequenceCount(pattern)(timestamp, cond1, cond2, ...) ``` -**Параметры** - -- `pattern` — строка с шаблоном. Смотрите [Синтаксис шаблонов](#sequence-function-pattern-syntax). +**Аргументы** - `timestamp` — столбец, содержащий метки времени. Типичный тип данных столбца — `Date` или `DateTime`. Также можно использовать любой из поддержанных типов данных [UInt](../../sql-reference/aggregate-functions/parametric-functions.md). - `cond1`, `cond2` — условия, описывающие цепочку событий. Тип данных — `UInt8`. Можно использовать до 32 условий. Функция учитывает только те события, которые указаны в условиях. Функция пропускает данные из последовательности, если они не описаны ни в одном из условий. +**Параметры** + +- `pattern` — строка с шаблоном. Смотрите [Синтаксис шаблонов](#sequence-function-pattern-syntax). + **Возвращаемое значение** - Число непересекающихся цепочек событий, соответствущих шаблону. @@ -237,12 +246,15 @@ SELECT sequenceCount('(?1).*(?2)')(time, number = 1, number = 2) FROM t windowFunnel(window, [mode])(timestamp, cond1, cond2, ..., condN) ``` +**Аргументы** + +- `timestamp` — имя столбца, содержащего временные отметки. [Date](../../sql-reference/aggregate-functions/parametric-functions.md), [DateTime](../../sql-reference/aggregate-functions/parametric-functions.md#data_type-datetime) и другие параметры с типом `Integer`. В случае хранения меток времени в столбцах с типом `UInt64`, максимально допустимое значение соответствует ограничению для типа `Int64`, т.е. равно `2^63-1`. +- `cond` — условия или данные, описывающие цепочку событий. [UInt8](../../sql-reference/aggregate-functions/parametric-functions.md). + **Параметры** - `window` — ширина скользящего окна по времени. Единица измерения зависит от `timestamp` и может варьироваться. Должно соблюдаться условие `timestamp события cond2 <= timestamp события cond1 + window`. -- `mode` - необязательный параметр. Если установлено значение `'strict'`, то функция `windowFunnel()` применяет условия только для уникальных значений. -- `timestamp` — имя столбца, содержащего временные отметки. [Date](../../sql-reference/aggregate-functions/parametric-functions.md), [DateTime](../../sql-reference/aggregate-functions/parametric-functions.md#data_type-datetime) и другие параметры с типом `Integer`. В случае хранения меток времени в столбцах с типом `UInt64`, максимально допустимое значение соответствует ограничению для типа `Int64`, т.е. равно `2^63-1`. -- `cond` — условия или данные, описывающие цепочку событий. [UInt8](../../sql-reference/aggregate-functions/parametric-functions.md). +- `mode` — необязательный параметр. Если установлено значение `'strict'`, то функция `windowFunnel()` применяет условия только для уникальных значений. **Возвращаемое значение** @@ -306,7 +318,7 @@ ORDER BY level ASC Функция принимает набор (от 1 до 32) логических условий, как в [WHERE](../../sql-reference/statements/select/where.md#select-where), и применяет их к заданному набору данных. -Условия, кроме первого, применяются попарно: результат второго будет истинным, если истинно первое и второе, третьего - если истинно первое и третье и т. д. +Условия, кроме первого, применяются попарно: результат второго будет истинным, если истинно первое и второе, третьего - если истинно первое и третье и т.д. **Синтаксис** @@ -314,7 +326,7 @@ ORDER BY level ASC retention(cond1, cond2, ..., cond32) ``` -**Параметры** +**Аргументы** - `cond` — вычисляемое условие или выражение, которое возвращает `UInt8` результат (1/0). diff --git a/docs/ru/sql-reference/aggregate-functions/reference/argmax.md b/docs/ru/sql-reference/aggregate-functions/reference/argmax.md index f44e65831a9..c5585d58485 100644 --- a/docs/ru/sql-reference/aggregate-functions/reference/argmax.md +++ b/docs/ru/sql-reference/aggregate-functions/reference/argmax.md @@ -20,7 +20,7 @@ argMax(arg, val) argMax(tuple(arg, val)) ``` -**Параметры** +**Аргументы** - `arg` — аргумент. - `val` — значение. diff --git a/docs/ru/sql-reference/aggregate-functions/reference/argmin.md b/docs/ru/sql-reference/aggregate-functions/reference/argmin.md index 8c25b79f92a..d508042d474 100644 --- a/docs/ru/sql-reference/aggregate-functions/reference/argmin.md +++ b/docs/ru/sql-reference/aggregate-functions/reference/argmin.md @@ -20,7 +20,7 @@ argMin(arg, val) argMin(tuple(arg, val)) ``` -**Параметры** +**Аргументы** - `arg` — аргумент. - `val` — значение. diff --git a/docs/ru/sql-reference/aggregate-functions/reference/avgweighted.md b/docs/ru/sql-reference/aggregate-functions/reference/avgweighted.md index 72e6ca5c88c..0275fb77bbb 100644 --- a/docs/ru/sql-reference/aggregate-functions/reference/avgweighted.md +++ b/docs/ru/sql-reference/aggregate-functions/reference/avgweighted.md @@ -12,10 +12,10 @@ toc_priority: 107 avgWeighted(x, weight) ``` -**Параметры** +**Аргументы** -- `x` — Значения. [Целые числа](../../../sql-reference/data-types/int-uint.md) или [числа с плавающей запятой](../../../sql-reference/data-types/float.md). -- `weight` — Веса отдельных значений. [Целые числа](../../../sql-reference/data-types/int-uint.md) или [числа с плавающей запятой](../../../sql-reference/data-types/float.md). +- `x` — значения. [Целые числа](../../../sql-reference/data-types/int-uint.md) или [числа с плавающей запятой](../../../sql-reference/data-types/float.md). +- `weight` — веса отдельных значений. [Целые числа](../../../sql-reference/data-types/int-uint.md) или [числа с плавающей запятой](../../../sql-reference/data-types/float.md). Типы параметров должны совпадать. diff --git a/docs/ru/sql-reference/aggregate-functions/reference/count.md b/docs/ru/sql-reference/aggregate-functions/reference/count.md index d99c3b2aeb2..a9135a35f20 100644 --- a/docs/ru/sql-reference/aggregate-functions/reference/count.md +++ b/docs/ru/sql-reference/aggregate-functions/reference/count.md @@ -4,14 +4,14 @@ toc_priority: 1 # count {#agg_function-count} -Вычисляет количество строк или не NULL значений . +Вычисляет количество строк или не NULL значений. ClickHouse поддерживает следующие виды синтаксиса для `count`: - `count(expr)` или `COUNT(DISTINCT expr)`. - `count()` или `COUNT(*)`. Синтаксис `count()` специфичен для ClickHouse. -**Параметры** +**Аргументы** Функция может принимать: @@ -21,7 +21,7 @@ ClickHouse поддерживает следующие виды синтакси **Возвращаемое значение** - Если функция вызывается без параметров, она вычисляет количество строк. -- Если передаётся [выражение](../../syntax.md#syntax-expressions) , то функция вычисляет количество раз, когда выражение возвращает не NULL. Если выражение возвращает значение типа [Nullable](../../../sql-reference/data-types/nullable.md), то результат `count` не становится `Nullable`. Функция возвращает 0, если выражение возвращает `NULL` для всех строк. +- Если передаётся [выражение](../../syntax.md#syntax-expressions), то функция вычисляет количество раз, когда выражение возвращает не NULL. Если выражение возвращает значение типа [Nullable](../../../sql-reference/data-types/nullable.md), то результат `count` не становится `Nullable`. Функция возвращает 0, если выражение возвращает `NULL` для всех строк. В обоих случаях тип возвращаемого значения [UInt64](../../../sql-reference/data-types/int-uint.md). diff --git a/docs/ru/sql-reference/aggregate-functions/reference/grouparrayinsertat.md b/docs/ru/sql-reference/aggregate-functions/reference/grouparrayinsertat.md index 5c73bccc2bb..37447189155 100644 --- a/docs/ru/sql-reference/aggregate-functions/reference/grouparrayinsertat.md +++ b/docs/ru/sql-reference/aggregate-functions/reference/grouparrayinsertat.md @@ -9,24 +9,24 @@ toc_priority: 112 **Синтаксис** ```sql -groupArrayInsertAt(default_x, size)(x, pos); +groupArrayInsertAt(default_x, size)(x, pos) ``` Если запрос вставляет вставляется несколько значений в одну и ту же позицию, то функция ведет себя следующим образом: -- Если запрос выполняется в одном потоке, то используется первое из вставляемых значений. -- Если запрос выполняется в нескольких потоках, то в результирующем массиве может оказаться любое из вставляемых значений. +- Если запрос выполняется в одном потоке, то используется первое из вставляемых значений. +- Если запрос выполняется в нескольких потоках, то в результирующем массиве может оказаться любое из вставляемых значений. -**Параметры** +**Аргументы** -- `x` — Значение, которое будет вставлено. [Выражение](../../syntax.md#syntax-expressions), возвращающее значение одного из [поддерживаемых типов данных](../../../sql-reference/data-types/index.md#data_types). -- `pos` — Позиция, в которую вставляется заданный элемент `x`. Нумерация индексов в массиве начинается с нуля. [UInt32](../../../sql-reference/data-types/int-uint.md#uint8-uint16-uint32-uint64-int8-int16-int32-int64). -- `default_x` — Значение по умолчанию для подстановки на пустые позиции. Опциональный параметр. [Выражение](../../syntax.md#syntax-expressions), возвращающее значение с типом параметра `x`. Если `default_x` не определен, используются [значения по умолчанию](../../../sql-reference/statements/create/table.md#create-default-values). -- `size`— Длина результирующего массива. Опциональный параметр. При использовании этого параметра должно быть указано значение по умолчанию `default_x`. [UInt32](../../../sql-reference/data-types/int-uint.md#uint-ranges). +- `x` — значение, которое будет вставлено. [Выражение](../../syntax.md#syntax-expressions), возвращающее значение одного из [поддерживаемых типов данных](../../../sql-reference/data-types/index.md#data_types). +- `pos` — позиция, в которую вставляется заданный элемент `x`. Нумерация индексов в массиве начинается с нуля. [UInt32](../../../sql-reference/data-types/int-uint.md#uint8-uint16-uint32-uint64-int8-int16-int32-int64). +- `default_x` — значение по умолчанию для подстановки на пустые позиции. Опциональный параметр. [Выражение](../../syntax.md#syntax-expressions), возвращающее значение с типом параметра `x`. Если `default_x` не определен, используются [значения по умолчанию](../../../sql-reference/statements/create/table.md#create-default-values). +- `size` — длина результирующего массива. Опциональный параметр. При использовании этого параметра должно быть указано значение по умолчанию `default_x`. [UInt32](../../../sql-reference/data-types/int-uint.md#uint-ranges). **Возвращаемое значение** -- Массив со вставленными значениями. +- Массив со вставленными значениями. Тип: [Array](../../../sql-reference/data-types/array.md#data-type-array). diff --git a/docs/ru/sql-reference/aggregate-functions/reference/grouparraymovingavg.md b/docs/ru/sql-reference/aggregate-functions/reference/grouparraymovingavg.md index 6307189c440..a66dacd64f4 100644 --- a/docs/ru/sql-reference/aggregate-functions/reference/grouparraymovingavg.md +++ b/docs/ru/sql-reference/aggregate-functions/reference/grouparraymovingavg.md @@ -6,12 +6,14 @@ toc_priority: 114 Вычисляет скользящее среднее для входных значений. - groupArrayMovingAvg(numbers_for_summing) - groupArrayMovingAvg(window_size)(numbers_for_summing) +``` sql +groupArrayMovingAvg(numbers_for_summing) +groupArrayMovingAvg(window_size)(numbers_for_summing) +``` Функция может принимать размер окна в качестве параметра. Если окно не указано, то функция использует размер окна, равный количеству строк в столбце. -**Параметры** +**Аргументы** - `numbers_for_summing` — [выражение](../../syntax.md#syntax-expressions), возвращающее значение числового типа. - `window_size` — размер окна. diff --git a/docs/ru/sql-reference/aggregate-functions/reference/grouparraymovingsum.md b/docs/ru/sql-reference/aggregate-functions/reference/grouparraymovingsum.md index c95f1b0b0eb..fb825703c9d 100644 --- a/docs/ru/sql-reference/aggregate-functions/reference/grouparraymovingsum.md +++ b/docs/ru/sql-reference/aggregate-functions/reference/grouparraymovingsum.md @@ -13,7 +13,7 @@ groupArrayMovingSum(window_size)(numbers_for_summing) Функция может принимать размер окна в качестве параметра. Если окно не указано, то функция использует размер окна, равный количеству строк в столбце. -**Параметры** +**Аргументы** - `numbers_for_summing` — [выражение](../../syntax.md#syntax-expressions), возвращающее значение числового типа. - `window_size` — размер окна. diff --git a/docs/ru/sql-reference/aggregate-functions/reference/grouparraysample.md b/docs/ru/sql-reference/aggregate-functions/reference/grouparraysample.md index 4c2dafe1a3c..1d58b3397ab 100644 --- a/docs/ru/sql-reference/aggregate-functions/reference/grouparraysample.md +++ b/docs/ru/sql-reference/aggregate-functions/reference/grouparraysample.md @@ -12,7 +12,7 @@ toc_priority: 114 groupArraySample(max_size[, seed])(x) ``` -**Параметры** +**Аргументы** - `max_size` — максимальное количество элементов в возвращаемом массиве. [UInt64](../../data-types/int-uint.md). - `seed` — состояние генератора случайных чисел. Необязательный параметр. [UInt64](../../data-types/int-uint.md). Значение по умолчанию: `123456`. diff --git a/docs/ru/sql-reference/aggregate-functions/reference/groupbitand.md b/docs/ru/sql-reference/aggregate-functions/reference/groupbitand.md index 03aff64fecf..7f381ca2906 100644 --- a/docs/ru/sql-reference/aggregate-functions/reference/groupbitand.md +++ b/docs/ru/sql-reference/aggregate-functions/reference/groupbitand.md @@ -10,7 +10,7 @@ toc_priority: 125 groupBitAnd(expr) ``` -**Параметры** +**Аргументы** `expr` – выражение, результат которого имеет тип данных `UInt*`. diff --git a/docs/ru/sql-reference/aggregate-functions/reference/groupbitmap.md b/docs/ru/sql-reference/aggregate-functions/reference/groupbitmap.md index a4be18b75ec..6aeb492add0 100644 --- a/docs/ru/sql-reference/aggregate-functions/reference/groupbitmap.md +++ b/docs/ru/sql-reference/aggregate-functions/reference/groupbitmap.md @@ -10,7 +10,7 @@ Bitmap или агрегатные вычисления для столбца с groupBitmap(expr) ``` -**Параметры** +**Аргументы** `expr` – выражение, результат которого имеет тип данных `UInt*`. diff --git a/docs/ru/sql-reference/aggregate-functions/reference/groupbitor.md b/docs/ru/sql-reference/aggregate-functions/reference/groupbitor.md index e1afced014f..9f7165286ed 100644 --- a/docs/ru/sql-reference/aggregate-functions/reference/groupbitor.md +++ b/docs/ru/sql-reference/aggregate-functions/reference/groupbitor.md @@ -10,7 +10,7 @@ toc_priority: 126 groupBitOr(expr) ``` -**Параметры** +**Аргументы** `expr` – выражение, результат которого имеет тип данных `UInt*`. diff --git a/docs/ru/sql-reference/aggregate-functions/reference/groupbitxor.md b/docs/ru/sql-reference/aggregate-functions/reference/groupbitxor.md index a80f86b2a5f..7d43d4a06f4 100644 --- a/docs/ru/sql-reference/aggregate-functions/reference/groupbitxor.md +++ b/docs/ru/sql-reference/aggregate-functions/reference/groupbitxor.md @@ -10,7 +10,7 @@ toc_priority: 127 groupBitXor(expr) ``` -**Параметры** +**Аргументы** `expr` – выражение, результат которого имеет тип данных `UInt*`. diff --git a/docs/ru/sql-reference/aggregate-functions/reference/initializeAggregation.md b/docs/ru/sql-reference/aggregate-functions/reference/initializeAggregation.md index a2e3764193e..3565115d8de 100644 --- a/docs/ru/sql-reference/aggregate-functions/reference/initializeAggregation.md +++ b/docs/ru/sql-reference/aggregate-functions/reference/initializeAggregation.md @@ -10,10 +10,10 @@ toc_priority: 150 **Синтаксис** ``` sql -initializeAggregation (aggregate_function, column_1, column_2); +initializeAggregation (aggregate_function, column_1, column_2) ``` -**Параметры** +**Аргументы** - `aggregate_function` — название функции агрегации, состояние которой нужно создать. [String](../../../sql-reference/data-types/string.md#string). - `column_n` — столбец, который передается в функцию агрегации как аргумент. [String](../../../sql-reference/data-types/string.md#string). diff --git a/docs/ru/sql-reference/aggregate-functions/reference/kurtpop.md b/docs/ru/sql-reference/aggregate-functions/reference/kurtpop.md index a00dae51ed6..4c10ecb5abc 100644 --- a/docs/ru/sql-reference/aggregate-functions/reference/kurtpop.md +++ b/docs/ru/sql-reference/aggregate-functions/reference/kurtpop.md @@ -10,9 +10,9 @@ toc_priority: 153 kurtPop(expr) ``` -**Параметры** +**Аргументы** -`expr` — [Выражение](../../syntax.md#syntax-expressions), возвращающее число. +`expr` — [выражение](../../syntax.md#syntax-expressions), возвращающее число. **Возвращаемое значение** @@ -21,7 +21,7 @@ kurtPop(expr) **Пример** ``` sql -SELECT kurtPop(value) FROM series_with_value_column +SELECT kurtPop(value) FROM series_with_value_column; ``` [Оригинальная статья](https://clickhouse.tech/docs/en/sql-reference/aggregate-functions/reference/kurtpop/) diff --git a/docs/ru/sql-reference/aggregate-functions/reference/kurtsamp.md b/docs/ru/sql-reference/aggregate-functions/reference/kurtsamp.md index 379d74ec0c3..4562a193b3e 100644 --- a/docs/ru/sql-reference/aggregate-functions/reference/kurtsamp.md +++ b/docs/ru/sql-reference/aggregate-functions/reference/kurtsamp.md @@ -12,9 +12,9 @@ toc_priority: 154 kurtSamp(expr) ``` -**Параметры** +**Аргументы** -`expr` — [Выражение](../../syntax.md#syntax-expressions), возвращающее число. +`expr` — [выражение](../../syntax.md#syntax-expressions), возвращающее число. **Возвращаемое значение** @@ -23,7 +23,7 @@ kurtSamp(expr) **Пример** ``` sql -SELECT kurtSamp(value) FROM series_with_value_column +SELECT kurtSamp(value) FROM series_with_value_column; ``` [Оригинальная статья](https://clickhouse.tech/docs/en/sql-reference/aggregate-functions/reference/kurtsamp/) diff --git a/docs/ru/sql-reference/aggregate-functions/reference/mannwhitneyutest.md b/docs/ru/sql-reference/aggregate-functions/reference/mannwhitneyutest.md index a4647ecfb34..068ff9990b4 100644 --- a/docs/ru/sql-reference/aggregate-functions/reference/mannwhitneyutest.md +++ b/docs/ru/sql-reference/aggregate-functions/reference/mannwhitneyutest.md @@ -17,16 +17,18 @@ mannWhitneyUTest[(alternative[, continuity_correction])](sample_data, sample_ind Проверяется нулевая гипотеза, что генеральные совокупности стохастически равны. Наряду с двусторонней гипотезой могут быть проверены и односторонние. Для применения U-критерия Манна — Уитни закон распределения генеральных совокупностей не обязан быть нормальным. +**Аргументы** + +- `sample_data` — данные выборок. [Integer](../../../sql-reference/data-types/int-uint.md), [Float](../../../sql-reference/data-types/float.md) или [Decimal](../../../sql-reference/data-types/decimal.md). +- `sample_index` — индексы выборок. [Integer](../../../sql-reference/data-types/int-uint.md). + **Параметры** - `alternative` — альтернативная гипотеза. (Необязательный параметр, по умолчанию: `'two-sided'`.) [String](../../../sql-reference/data-types/string.md). - `'two-sided'`; - `'greater'`; - `'less'`. -- `continuity_correction` - если не 0, то при вычислении p-значения применяется коррекция непрерывности. (Необязательный параметр, по умолчанию: 1.) [UInt64](../../../sql-reference/data-types/int-uint.md). -- `sample_data` — данные выборок. [Integer](../../../sql-reference/data-types/int-uint.md), [Float](../../../sql-reference/data-types/float.md) or [Decimal](../../../sql-reference/data-types/decimal.md). -- `sample_index` — индексы выборок. [Integer](../../../sql-reference/data-types/int-uint.md). - +- `continuity_correction` — если не 0, то при вычислении p-значения применяется коррекция непрерывности. (Необязательный параметр, по умолчанию: 1.) [UInt64](../../../sql-reference/data-types/int-uint.md). **Возвращаемые значения** diff --git a/docs/ru/sql-reference/aggregate-functions/reference/quantile.md b/docs/ru/sql-reference/aggregate-functions/reference/quantile.md index 10fec16ab94..7cc4f8c7aef 100644 --- a/docs/ru/sql-reference/aggregate-functions/reference/quantile.md +++ b/docs/ru/sql-reference/aggregate-functions/reference/quantile.md @@ -18,10 +18,10 @@ quantile(level)(expr) Алиас: `median`. -**Параметры** +**Аргументы** -- `level` — Уровень квантили. Опционально. Константное значение с плавающей запятой от 0 до 1. Мы рекомендуем использовать значение `level` из диапазона `[0.01, 0.99]`. Значение по умолчанию: 0.5. При `level=0.5` функция вычисляет [медиану](https://ru.wikipedia.org/wiki/Медиана_(статистика)). -- `expr` — Выражение над значениями столбца, которое возвращает данные [числовых типов](../../../sql-reference/data-types/index.md#data_types) или типов [Date](../../../sql-reference/data-types/date.md), [DateTime](../../../sql-reference/data-types/datetime.md). +- `level` — уровень квантили. Опционально. Константное значение с плавающей запятой от 0 до 1. Мы рекомендуем использовать значение `level` из диапазона `[0.01, 0.99]`. Значение по умолчанию: 0.5. При `level=0.5` функция вычисляет [медиану](https://ru.wikipedia.org/wiki/Медиана_(статистика)). +- `expr` — выражение над значениями столбца, которое возвращает данные [числовых типов](../../../sql-reference/data-types/index.md#data_types) или типов [Date](../../../sql-reference/data-types/date.md), [DateTime](../../../sql-reference/data-types/datetime.md). **Возвращаемое значение** diff --git a/docs/ru/sql-reference/aggregate-functions/reference/quantiledeterministic.md b/docs/ru/sql-reference/aggregate-functions/reference/quantiledeterministic.md index fdbcda821f6..3c03c356ab9 100644 --- a/docs/ru/sql-reference/aggregate-functions/reference/quantiledeterministic.md +++ b/docs/ru/sql-reference/aggregate-functions/reference/quantiledeterministic.md @@ -18,11 +18,11 @@ quantileDeterministic(level)(expr, determinator) Алиас: `medianDeterministic`. -**Параметры** +**Аргументы** -- `level` — Уровень квантили. Опционально. Константное значение с плавающей запятой от 0 до 1. Мы рекомендуем использовать значение `level` из диапазона `[0.01, 0.99]`. Значение по умолчанию: 0.5. При `level=0.5` функция вычисляет [медиану](https://ru.wikipedia.org/wiki/Медиана_(статистика)). -- `expr` — Выражение над значениями столбца, которое возвращает данные [числовых типов](../../../sql-reference/data-types/index.md#data_types) или типов [Date](../../../sql-reference/data-types/date.md), [DateTime](../../../sql-reference/data-types/datetime.md). -- `determinator` — Число, хэш которого используется при сэмплировании в алгоритме reservoir sampling, чтобы сделать результат детерминированным. В качестве детерминатора можно использовать любое определённое положительное число, например, идентификатор пользователя или события. Если одно и то же значение детерминатора попадается в выборке слишком часто, то функция выдаёт некорректный результат. +- `level` — уровень квантили. Опционально. Константное значение с плавающей запятой от 0 до 1. Мы рекомендуем использовать значение `level` из диапазона `[0.01, 0.99]`. Значение по умолчанию: 0.5. При `level=0.5` функция вычисляет [медиану](https://ru.wikipedia.org/wiki/Медиана_(статистика)). +- `expr` — выражение над значениями столбца, которое возвращает данные [числовых типов](../../../sql-reference/data-types/index.md#data_types) или типов [Date](../../../sql-reference/data-types/date.md), [DateTime](../../../sql-reference/data-types/datetime.md). +- `determinator` — число, хэш которого используется при сэмплировании в алгоритме reservoir sampling, чтобы сделать результат детерминированным. В качестве детерминатора можно использовать любое определённое положительное число, например, идентификатор пользователя или события. Если одно и то же значение детерминатора попадается в выборке слишком часто, то функция выдаёт некорректный результат. **Возвращаемое значение** diff --git a/docs/ru/sql-reference/aggregate-functions/reference/quantileexact.md b/docs/ru/sql-reference/aggregate-functions/reference/quantileexact.md index 4ee815a94fb..452b810cf03 100644 --- a/docs/ru/sql-reference/aggregate-functions/reference/quantileexact.md +++ b/docs/ru/sql-reference/aggregate-functions/reference/quantileexact.md @@ -18,10 +18,10 @@ quantileExact(level)(expr) Алиас: `medianExact`. -**Параметры** +**Аргументы** -- `level` — Уровень квантили. Опционально. Константное значение с плавающей запятой от 0 до 1. Мы рекомендуем использовать значение `level` из диапазона `[0.01, 0.99]`. Значение по умолчанию: 0.5. При `level=0.5` функция вычисляет [медиану](https://ru.wikipedia.org/wiki/Медиана_(статистика)). -- `expr` — Выражение над значениями столбца, которое возвращает данные [числовых типов](../../../sql-reference/data-types/index.md#data_types) или типов [Date](../../../sql-reference/data-types/date.md), [DateTime](../../../sql-reference/data-types/datetime.md). +- `level` — уровень квантили. Опционально. Константное значение с плавающей запятой от 0 до 1. Мы рекомендуем использовать значение `level` из диапазона `[0.01, 0.99]`. Значение по умолчанию: 0.5. При `level=0.5` функция вычисляет [медиану](https://ru.wikipedia.org/wiki/Медиана_(статистика)). +- `expr` — выражение над значениями столбца, которое возвращает данные [числовых типов](../../../sql-reference/data-types/index.md#data_types) или типов [Date](../../../sql-reference/data-types/date.md), [DateTime](../../../sql-reference/data-types/datetime.md). **Возвращаемое значение** @@ -77,10 +77,10 @@ quantileExact(level)(expr) Алиас: `medianExactLow`. -**Параметры** +**Аргументы** -- `level` — Уровень квантили. Опциональный параметр. Константное занчение с плавающей запятой от 0 до 1. Мы рекомендуем использовать значение `level` из диапазона `[0.01, 0.99]`. Значение по умолчанию: 0.5. При `level=0.5` функция вычисляет [медиану](https://en.wikipedia.org/wiki/Median). -- `expr` — Выражение над значениями столбца, которое возвращает данные [числовых типов](../../../sql-reference/data-types/index.md#data_types), [Date](../../../sql-reference/data-types/date.md) или [DateTime](../../../sql-reference/data-types/datetime.md). +- `level` — уровень квантили. Опциональный параметр. Константное занчение с плавающей запятой от 0 до 1. Мы рекомендуем использовать значение `level` из диапазона `[0.01, 0.99]`. Значение по умолчанию: 0.5. При `level=0.5` функция вычисляет [медиану](https://en.wikipedia.org/wiki/Median). +- `expr` — выражение над значениями столбца, которое возвращает данные [числовых типов](../../../sql-reference/data-types/index.md#data_types), [Date](../../../sql-reference/data-types/date.md) или [DateTime](../../../sql-reference/data-types/datetime.md). **Возвращаемое значение** @@ -127,10 +127,10 @@ quantileExactHigh(level)(expr) Алиас: `medianExactHigh`. -**Параметры** +**Аргументы** -- `level` — Уровень квантили. Опциональный параметр. Константное занчение с плавающей запятой от 0 до 1. Мы рекомендуем использовать значение `level` из диапазона `[0.01, 0.99]`. Значение по умолчанию: 0.5. При `level=0.5` функция вычисляет [медиану](https://en.wikipedia.org/wiki/Median). -- `expr` — Выражение над значениями столбца, которое возвращает данные [числовых типов](../../../sql-reference/data-types/index.md#data_types), [Date](../../../sql-reference/data-types/date.md) или [DateTime](../../../sql-reference/data-types/datetime.md). +- `level` — уровень квантили. Опциональный параметр. Константное занчение с плавающей запятой от 0 до 1. Мы рекомендуем использовать значение `level` из диапазона `[0.01, 0.99]`. Значение по умолчанию: 0.5. При `level=0.5` функция вычисляет [медиану](https://en.wikipedia.org/wiki/Median). +- `expr` — выражение над значениями столбца, которое возвращает данные [числовых типов](../../../sql-reference/data-types/index.md#data_types), [Date](../../../sql-reference/data-types/date.md) или [DateTime](../../../sql-reference/data-types/datetime.md). **Возвращаемое значение** diff --git a/docs/ru/sql-reference/aggregate-functions/reference/quantileexactweighted.md b/docs/ru/sql-reference/aggregate-functions/reference/quantileexactweighted.md index f6982d4566f..ee55aaec121 100644 --- a/docs/ru/sql-reference/aggregate-functions/reference/quantileexactweighted.md +++ b/docs/ru/sql-reference/aggregate-functions/reference/quantileexactweighted.md @@ -18,11 +18,11 @@ quantileExactWeighted(level)(expr, weight) Алиас: `medianExactWeighted`. -**Параметры** +**Аргументы** -- `level` — Уровень квантили. Опционально. Константное значение с плавающей запятой от 0 до 1. Мы рекомендуем использовать значение `level` из диапазона `[0.01, 0.99]`. Значение по умолчанию: 0.5. При `level=0.5` функция вычисляет [медиану](https://ru.wikipedia.org/wiki/Медиана_(статистика)). -- `expr` — Выражение над значениями столбца, которое возвращает данные [числовых типов](../../../sql-reference/data-types/index.md#data_types) или типов [Date](../../../sql-reference/data-types/date.md), [DateTime](../../../sql-reference/data-types/datetime.md). -- `weight` — Столбец с весам элементов последовательности. Вес — это количество повторений элемента в последовательности. +- `level` — уровень квантили. Опционально. Константное значение с плавающей запятой от 0 до 1. Мы рекомендуем использовать значение `level` из диапазона `[0.01, 0.99]`. Значение по умолчанию: 0.5. При `level=0.5` функция вычисляет [медиану](https://ru.wikipedia.org/wiki/Медиана_(статистика)). +- `expr` — выражение над значениями столбца, которое возвращает данные [числовых типов](../../../sql-reference/data-types/index.md#data_types) или типов [Date](../../../sql-reference/data-types/date.md), [DateTime](../../../sql-reference/data-types/datetime.md). +- `weight` — столбец с весам элементов последовательности. Вес — это количество повторений элемента в последовательности. **Возвращаемое значение** diff --git a/docs/ru/sql-reference/aggregate-functions/reference/quantiletdigest.md b/docs/ru/sql-reference/aggregate-functions/reference/quantiletdigest.md index f372e308e73..a119ca940b6 100644 --- a/docs/ru/sql-reference/aggregate-functions/reference/quantiletdigest.md +++ b/docs/ru/sql-reference/aggregate-functions/reference/quantiletdigest.md @@ -20,10 +20,10 @@ quantileTDigest(level)(expr) Алиас: `medianTDigest`. -**Параметры** +**Аргументы** -- `level` — Уровень квантили. Опционально. Константное значение с плавающей запятой от 0 до 1. Мы рекомендуем использовать значение `level` из диапазона `[0.01, 0.99]`. Значение по умолчанию: 0.5. При `level=0.5` функция вычисляет [медиану](https://ru.wikipedia.org/wiki/Медиана_(статистика)). -- `expr` — Выражение над значениями столбца, которое возвращает данные [числовых типов](../../../sql-reference/data-types/index.md#data_types) или типов [Date](../../../sql-reference/data-types/date.md), [DateTime](../../../sql-reference/data-types/datetime.md). +- `level` — уровень квантили. Опционально. Константное значение с плавающей запятой от 0 до 1. Мы рекомендуем использовать значение `level` из диапазона `[0.01, 0.99]`. Значение по умолчанию: 0.5. При `level=0.5` функция вычисляет [медиану](https://ru.wikipedia.org/wiki/Медиана_(статистика)). +- `expr` — выражение над значениями столбца, которое возвращает данные [числовых типов](../../../sql-reference/data-types/index.md#data_types) или типов [Date](../../../sql-reference/data-types/date.md), [DateTime](../../../sql-reference/data-types/datetime.md). **Возвращаемое значение** diff --git a/docs/ru/sql-reference/aggregate-functions/reference/quantiletdigestweighted.md b/docs/ru/sql-reference/aggregate-functions/reference/quantiletdigestweighted.md index b6dd846967b..a00c1f8af58 100644 --- a/docs/ru/sql-reference/aggregate-functions/reference/quantiletdigestweighted.md +++ b/docs/ru/sql-reference/aggregate-functions/reference/quantiletdigestweighted.md @@ -20,11 +20,11 @@ quantileTDigestWeighted(level)(expr, weight) Алиас: `medianTDigest`. -**Параметры** +**Аргументы** -- `level` — Уровень квантили. Опционально. Константное значение с плавающей запятой от 0 до 1. Мы рекомендуем использовать значение `level` из диапазона `[0.01, 0.99]`. Значение по умолчанию: 0.5. При `level=0.5` функция вычисляет [медиану](https://ru.wikipedia.org/wiki/Медиана_(статистика)). -- `expr` — Выражение над значениями столбца, которое возвращает данные [числовых типов](../../../sql-reference/data-types/index.md#data_types) или типов [Date](../../../sql-reference/data-types/date.md), [DateTime](../../../sql-reference/data-types/datetime.md). -- `weight` — Столбец с весам элементов последовательности. Вес — это количество повторений элемента в последовательности. +- `level` — уровень квантили. Опционально. Константное значение с плавающей запятой от 0 до 1. Мы рекомендуем использовать значение `level` из диапазона `[0.01, 0.99]`. Значение по умолчанию: 0.5. При `level=0.5` функция вычисляет [медиану](https://ru.wikipedia.org/wiki/Медиана_(статистика)). +- `expr` — выражение над значениями столбца, которое возвращает данные [числовых типов](../../../sql-reference/data-types/index.md#data_types) или типов [Date](../../../sql-reference/data-types/date.md), [DateTime](../../../sql-reference/data-types/datetime.md). +- `weight` — столбец с весам элементов последовательности. Вес — это количество повторений элемента в последовательности. **Возвращаемое значение** diff --git a/docs/ru/sql-reference/aggregate-functions/reference/quantiletiming.md b/docs/ru/sql-reference/aggregate-functions/reference/quantiletiming.md index 32e5e6ce31b..6131f2035cb 100644 --- a/docs/ru/sql-reference/aggregate-functions/reference/quantiletiming.md +++ b/docs/ru/sql-reference/aggregate-functions/reference/quantiletiming.md @@ -18,11 +18,11 @@ quantileTiming(level)(expr) Алиас: `medianTiming`. -**Параметры** +**Аргументы** -- `level` — Уровень квантили. Опционально. Константное значение с плавающей запятой от 0 до 1. Мы рекомендуем использовать значение `level` из диапазона `[0.01, 0.99]`. Значение по умолчанию: 0.5. При `level=0.5` функция вычисляет [медиану](https://ru.wikipedia.org/wiki/Медиана_(статистика)). +- `level` — уровень квантили. Опционально. Константное значение с плавающей запятой от 0 до 1. Мы рекомендуем использовать значение `level` из диапазона `[0.01, 0.99]`. Значение по умолчанию: 0.5. При `level=0.5` функция вычисляет [медиану](https://ru.wikipedia.org/wiki/Медиана_(статистика)). -- `expr` — [Выражение](../../syntax.md#syntax-expressions) над значения столбца, которые возвращают данные типа [Float\*](../../../sql-reference/data-types/float.md). +- `expr` — [выражение](../../syntax.md#syntax-expressions) над значения столбца, которые возвращают данные типа [Float\*](../../../sql-reference/data-types/float.md). - Если в функцию передать отрицательные значения, то её поведение не определено. - Если значение больше, чем 30 000 (например, время загрузки страницы превышает 30 секунд), то оно приравнивается к 30 000. diff --git a/docs/ru/sql-reference/aggregate-functions/reference/quantiletimingweighted.md b/docs/ru/sql-reference/aggregate-functions/reference/quantiletimingweighted.md index 4a7fcc666d5..58a0a4599f9 100644 --- a/docs/ru/sql-reference/aggregate-functions/reference/quantiletimingweighted.md +++ b/docs/ru/sql-reference/aggregate-functions/reference/quantiletimingweighted.md @@ -18,16 +18,16 @@ quantileTimingWeighted(level)(expr, weight) Алиас: `medianTimingWeighted`. -**Параметры** +**Аргументы** -- `level` — Уровень квантили. Опционально. Константное значение с плавающей запятой от 0 до 1. Мы рекомендуем использовать значение `level` из диапазона `[0.01, 0.99]`. Значение по умолчанию: 0.5. При `level=0.5` функция вычисляет [медиану](https://ru.wikipedia.org/wiki/Медиана_(статистика)). +- `level` — уровень квантили. Опционально. Константное значение с плавающей запятой от 0 до 1. Мы рекомендуем использовать значение `level` из диапазона `[0.01, 0.99]`. Значение по умолчанию: 0.5. При `level=0.5` функция вычисляет [медиану](https://ru.wikipedia.org/wiki/Медиана_(статистика)). -- `expr` — [Выражение](../../syntax.md#syntax-expressions) над значения столбца, которые возвращают данные типа [Float\*](../../../sql-reference/data-types/float.md). +- `expr` — [выражение](../../syntax.md#syntax-expressions) над значения столбца, которые возвращают данные типа [Float\*](../../../sql-reference/data-types/float.md). - Если в функцию передать отрицательные значения, то её поведение не определено. - Если значение больше, чем 30 000 (например, время загрузки страницы превышает 30 секунд), то оно приравнивается к 30 000. -- `weight` — Столбец с весам элементов последовательности. Вес — это количество повторений элемента в последовательности. +- `weight` — столбец с весам элементов последовательности. Вес — это количество повторений элемента в последовательности. **Точность** diff --git a/docs/ru/sql-reference/aggregate-functions/reference/rankCorr.md b/docs/ru/sql-reference/aggregate-functions/reference/rankCorr.md index 48a19e87c52..c98e7b88bcf 100644 --- a/docs/ru/sql-reference/aggregate-functions/reference/rankCorr.md +++ b/docs/ru/sql-reference/aggregate-functions/reference/rankCorr.md @@ -8,10 +8,10 @@ rankCorr(x, y) ``` -**Параметры** +**Аргументы** -- `x` — Произвольное значение. [Float32](../../../sql-reference/data-types/float.md#float32-float64) или [Float64](../../../sql-reference/data-types/float.md#float32-float64). -- `y` — Произвольное значение. [Float32](../../../sql-reference/data-types/float.md#float32-float64) или [Float64](../../../sql-reference/data-types/float.md#float32-float64). +- `x` — произвольное значение. [Float32](../../../sql-reference/data-types/float.md#float32-float64) или [Float64](../../../sql-reference/data-types/float.md#float32-float64). +- `y` — произвольное значение. [Float32](../../../sql-reference/data-types/float.md#float32-float64) или [Float64](../../../sql-reference/data-types/float.md#float32-float64). **Возвращаемое значение** diff --git a/docs/ru/sql-reference/aggregate-functions/reference/skewpop.md b/docs/ru/sql-reference/aggregate-functions/reference/skewpop.md index a6dee5dc5ef..03fec8bd895 100644 --- a/docs/ru/sql-reference/aggregate-functions/reference/skewpop.md +++ b/docs/ru/sql-reference/aggregate-functions/reference/skewpop.md @@ -10,9 +10,9 @@ toc_priority: 150 skewPop(expr) ``` -**Параметры** +**Аргументы** -`expr` — [Выражение](../../syntax.md#syntax-expressions), возвращающее число. +`expr` — [выражение](../../syntax.md#syntax-expressions), возвращающее число. **Возвращаемое значение** @@ -21,7 +21,7 @@ skewPop(expr) **Пример** ``` sql -SELECT skewPop(value) FROM series_with_value_column +SELECT skewPop(value) FROM series_with_value_column; ``` [Оригинальная статья](https://clickhouse.tech/docs/en/sql-reference/aggregate-functions/reference/skewpop/) diff --git a/docs/ru/sql-reference/aggregate-functions/reference/skewsamp.md b/docs/ru/sql-reference/aggregate-functions/reference/skewsamp.md index 171eb5e304a..2be9e338d00 100644 --- a/docs/ru/sql-reference/aggregate-functions/reference/skewsamp.md +++ b/docs/ru/sql-reference/aggregate-functions/reference/skewsamp.md @@ -12,9 +12,9 @@ toc_priority: 151 skewSamp(expr) ``` -**Параметры** +**Аргументы** -`expr` — [Выражение](../../syntax.md#syntax-expressions), возвращающее число. +`expr` — [выражение](../../syntax.md#syntax-expressions), возвращающее число. **Возвращаемое значение** @@ -23,7 +23,7 @@ skewSamp(expr) **Пример** ``` sql -SELECT skewSamp(value) FROM series_with_value_column +SELECT skewSamp(value) FROM series_with_value_column; ``` [Оригинальная статья](https://clickhouse.tech/docs/en/sql-reference/aggregate-functions/reference/skewsamp/) diff --git a/docs/ru/sql-reference/aggregate-functions/reference/studentttest.md b/docs/ru/sql-reference/aggregate-functions/reference/studentttest.md index 77378de95d1..d2e680e5ed9 100644 --- a/docs/ru/sql-reference/aggregate-functions/reference/studentttest.md +++ b/docs/ru/sql-reference/aggregate-functions/reference/studentttest.md @@ -16,7 +16,7 @@ studentTTest(sample_data, sample_index) Значения выборок берутся из столбца `sample_data`. Если `sample_index` равно 0, то значение из этой строки принадлежит первой выборке. Во всех остальных случаях значение принадлежит второй выборке. Проверяется нулевая гипотеза, что средние значения генеральных совокупностей совпадают. Для применения t-критерия Стьюдента распределение в генеральных совокупностях должно быть нормальным и дисперсии должны совпадать. -**Параметры** +**Аргументы** - `sample_data` — данные выборок. [Integer](../../../sql-reference/data-types/int-uint.md), [Float](../../../sql-reference/data-types/float.md) or [Decimal](../../../sql-reference/data-types/decimal.md). - `sample_index` — индексы выборок. [Integer](../../../sql-reference/data-types/int-uint.md). diff --git a/docs/ru/sql-reference/aggregate-functions/reference/topk.md b/docs/ru/sql-reference/aggregate-functions/reference/topk.md index 6aefd38bf34..86b9f5fe5b3 100644 --- a/docs/ru/sql-reference/aggregate-functions/reference/topk.md +++ b/docs/ru/sql-reference/aggregate-functions/reference/topk.md @@ -18,8 +18,8 @@ topK(N)(column) **Аргументы** -- ‘N’ - Количество значений. -- ‘x’ – Столбец. +- `N` – количество значений. +- `x` – столбец. **Пример** diff --git a/docs/ru/sql-reference/aggregate-functions/reference/topkweighted.md b/docs/ru/sql-reference/aggregate-functions/reference/topkweighted.md index 20bd3ee85ff..cc0e433cfb3 100644 --- a/docs/ru/sql-reference/aggregate-functions/reference/topkweighted.md +++ b/docs/ru/sql-reference/aggregate-functions/reference/topkweighted.md @@ -12,13 +12,13 @@ toc_priority: 109 topKWeighted(N)(x, weight) ``` -**Параметры** +**Аргументы** -- `N` — Количество элементов для выдачи. +- `N` — количество элементов для выдачи. **Аргументы** -- `x` – значение. +- `x` — значение. - `weight` — вес. [UInt8](../../../sql-reference/data-types/int-uint.md). **Возвращаемое значение** diff --git a/docs/ru/sql-reference/aggregate-functions/reference/uniq.md b/docs/ru/sql-reference/aggregate-functions/reference/uniq.md index f5f3f198139..f9b4fad7d59 100644 --- a/docs/ru/sql-reference/aggregate-functions/reference/uniq.md +++ b/docs/ru/sql-reference/aggregate-functions/reference/uniq.md @@ -10,7 +10,7 @@ toc_priority: 190 uniq(x[, ...]) ``` -**Параметры** +**Аргументы** Функция принимает переменное число входных параметров. Параметры могут быть числовых типов, а также `Tuple`, `Array`, `Date`, `DateTime`, `String`. diff --git a/docs/ru/sql-reference/aggregate-functions/reference/uniqcombined.md b/docs/ru/sql-reference/aggregate-functions/reference/uniqcombined.md index 751dc1a8c98..323454f7778 100644 --- a/docs/ru/sql-reference/aggregate-functions/reference/uniqcombined.md +++ b/docs/ru/sql-reference/aggregate-functions/reference/uniqcombined.md @@ -12,7 +12,7 @@ uniqCombined(HLL_precision)(x[, ...]) Функция `uniqCombined` — это хороший выбор для вычисления количества различных значений. -**Параметры** +**Аргументы** Функция принимает переменное число входных параметров. Параметры могут быть числовых типов, а также `Tuple`, `Array`, `Date`, `DateTime`, `String`. diff --git a/docs/ru/sql-reference/aggregate-functions/reference/uniqexact.md b/docs/ru/sql-reference/aggregate-functions/reference/uniqexact.md index 3dd22b2b4bc..20c05ab20f9 100644 --- a/docs/ru/sql-reference/aggregate-functions/reference/uniqexact.md +++ b/docs/ru/sql-reference/aggregate-functions/reference/uniqexact.md @@ -14,7 +14,7 @@ uniqExact(x[, ...]) Функция `uniqExact` расходует больше оперативной памяти, чем функция `uniq`, так как размер состояния неограниченно растёт по мере роста количества различных значений. -**Параметры** +**Аргументы** Функция принимает переменное число входных параметров. Параметры могут быть числовых типов, а также `Tuple`, `Array`, `Date`, `DateTime`, `String`. diff --git a/docs/ru/sql-reference/aggregate-functions/reference/uniqhll12.md b/docs/ru/sql-reference/aggregate-functions/reference/uniqhll12.md index 09e52ac6833..290dd65ad8b 100644 --- a/docs/ru/sql-reference/aggregate-functions/reference/uniqhll12.md +++ b/docs/ru/sql-reference/aggregate-functions/reference/uniqhll12.md @@ -10,7 +10,7 @@ toc_priority: 194 uniqHLL12(x[, ...]) ``` -**Параметры** +**Аргументы** Функция принимает переменное число входных параметров. Параметры могут быть числовых типов, а также `Tuple`, `Array`, `Date`, `DateTime`, `String`. diff --git a/docs/ru/sql-reference/aggregate-functions/reference/welchttest.md b/docs/ru/sql-reference/aggregate-functions/reference/welchttest.md index 16c122d1b49..0af5d865dae 100644 --- a/docs/ru/sql-reference/aggregate-functions/reference/welchttest.md +++ b/docs/ru/sql-reference/aggregate-functions/reference/welchttest.md @@ -16,7 +16,7 @@ welchTTest(sample_data, sample_index) Значения выборок берутся из столбца `sample_data`. Если `sample_index` равно 0, то значение из этой строки принадлежит первой выборке. Во всех остальных случаях значение принадлежит второй выборке. Проверяется нулевая гипотеза, что средние значения генеральных совокупностей совпадают. Для применения t-критерия Уэлча распределение в генеральных совокупностях должно быть нормальным. Дисперсии могут не совпадать. -**Параметры** +**Аргументы** - `sample_data` — данные выборок. [Integer](../../../sql-reference/data-types/int-uint.md), [Float](../../../sql-reference/data-types/float.md) or [Decimal](../../../sql-reference/data-types/decimal.md). - `sample_index` — индексы выборок. [Integer](../../../sql-reference/data-types/int-uint.md). diff --git a/docs/ru/sql-reference/functions/array-functions.md b/docs/ru/sql-reference/functions/array-functions.md index fe216b1aed1..3c7e2ec619a 100644 --- a/docs/ru/sql-reference/functions/array-functions.md +++ b/docs/ru/sql-reference/functions/array-functions.md @@ -58,7 +58,7 @@ toc_title: "\u041c\u0430\u0441\u0441\u0438\u0432\u044b" arrayConcat(arrays) ``` -**Параметры** +**Аргументы** - `arrays` – произвольное количество элементов типа [Array](../../sql-reference/functions/array-functions.md) **Пример** @@ -108,7 +108,7 @@ SELECT has([1, 2, NULL], NULL) hasAll(set, subset) ``` -**Параметры** +**Аргументы** - `set` – массив любого типа с набором элементов. - `subset` – массив любого типа со значениями, которые проверяются на вхождение в `set`. @@ -146,7 +146,7 @@ hasAll(set, subset) hasAny(array1, array2) ``` -**Параметры** +**Аргументы** - `array1` – массив любого типа с набором элементов. - `array2` – массив любого типа с набором элементов. @@ -320,21 +320,21 @@ SELECT arrayEnumerateUniq([1, 1, 1, 2, 2, 2], [1, 1, 2, 1, 1, 2]) AS res arrayPopBack(array) ``` -**Параметры** +**Аргументы** -- `array` - Массив. +- `array` – массив. **Пример** ``` sql -SELECT arrayPopBack([1, 2, 3]) AS res +SELECT arrayPopBack([1, 2, 3]) AS res; ``` -text - - ┌─res───┐ - │ [1,2] │ - └───────┘ +``` text +┌─res───┐ +│ [1,2] │ +└───────┘ +``` ## arrayPopFront {#arraypopfront} @@ -344,14 +344,14 @@ text arrayPopFront(array) ``` -**Параметры** +**Аргументы** -- `array` - Массив. +- `array` – массив. **Пример** ``` sql -SELECT arrayPopFront([1, 2, 3]) AS res +SELECT arrayPopFront([1, 2, 3]) AS res; ``` ``` text @@ -368,15 +368,15 @@ SELECT arrayPopFront([1, 2, 3]) AS res arrayPushBack(array, single_value) ``` -**Параметры** +**Аргументы** -- `array` - Массив. -- `single_value` - Одиночное значение. В массив с числам можно добавить только числа, в массив со строками только строки. При добавлении чисел ClickHouse автоматически приводит тип `single_value` к типу данных массива. Подробнее о типах данных в ClickHouse читайте в разделе «[Типы данных](../../sql-reference/functions/array-functions.md#data_types)». Может быть равно `NULL`. Функция добавит элемент `NULL` в массив, а тип элементов массива преобразует в `Nullable`. +- `array` – массив. +- `single_value` – одиночное значение. В массив с числам можно добавить только числа, в массив со строками только строки. При добавлении чисел ClickHouse автоматически приводит тип `single_value` к типу данных массива. Подробнее о типах данных в ClickHouse читайте в разделе «[Типы данных](../../sql-reference/functions/array-functions.md#data_types)». Может быть равно `NULL`. Функция добавит элемент `NULL` в массив, а тип элементов массива преобразует в `Nullable`. **Пример** ``` sql -SELECT arrayPushBack(['a'], 'b') AS res +SELECT arrayPushBack(['a'], 'b') AS res; ``` ``` text @@ -393,15 +393,15 @@ SELECT arrayPushBack(['a'], 'b') AS res arrayPushFront(array, single_value) ``` -**Параметры** +**Аргументы** -- `array` - Массив. -- `single_value` - Одиночное значение. В массив с числам можно добавить только числа, в массив со строками только строки. При добавлении чисел ClickHouse автоматически приводит тип `single_value` к типу данных массива. Подробнее о типах данных в ClickHouse читайте в разделе «[Типы данных](../../sql-reference/functions/array-functions.md#data_types)». Может быть равно `NULL`. Функция добавит элемент `NULL` в массив, а тип элементов массива преобразует в `Nullable`. +- `array` – массив. +- `single_value` – одиночное значение. В массив с числам можно добавить только числа, в массив со строками только строки. При добавлении чисел ClickHouse автоматически приводит тип `single_value` к типу данных массива. Подробнее о типах данных в ClickHouse читайте в разделе «[Типы данных](../../sql-reference/functions/array-functions.md#data_types)». Может быть равно `NULL`. Функция добавит элемент `NULL` в массив, а тип элементов массива преобразует в `Nullable`. **Пример** ``` sql -SELECT arrayPushFront(['b'], 'a') AS res +SELECT arrayPushFront(['b'], 'a') AS res; ``` ``` text @@ -418,7 +418,7 @@ SELECT arrayPushFront(['b'], 'a') AS res arrayResize(array, size[, extender]) ``` -**Параметры** +**Аргументы** - `array` — массив. - `size` — необходимая длина массива. @@ -433,7 +433,7 @@ arrayResize(array, size[, extender]) **Примеры вызовов** ``` sql -SELECT arrayResize([1], 3) +SELECT arrayResize([1], 3); ``` ``` text @@ -443,7 +443,7 @@ SELECT arrayResize([1], 3) ``` ``` sql -SELECT arrayResize([1], 3, NULL) +SELECT arrayResize([1], 3, NULL); ``` ``` text @@ -460,16 +460,16 @@ SELECT arrayResize([1], 3, NULL) arraySlice(array, offset[, length]) ``` -**Параметры** +**Аргументы** -- `array` - Массив данных. -- `offset` - Отступ от края массива. Положительное значение - отступ слева, отрицательное значение - отступ справа. Отсчет элементов массива начинается с 1. -- `length` - Длина необходимого среза. Если указать отрицательное значение, то функция вернёт открытый срез `[offset, array_length - length)`. Если не указать значение, то функция вернёт срез `[offset, the_end_of_array]`. +- `array` – массив данных. +- `offset` – отступ от края массива. Положительное значение - отступ слева, отрицательное значение - отступ справа. Отсчет элементов массива начинается с 1. +- `length` – длина необходимого среза. Если указать отрицательное значение, то функция вернёт открытый срез `[offset, array_length - length)`. Если не указать значение, то функция вернёт срез `[offset, the_end_of_array]`. **Пример** ``` sql -SELECT arraySlice([1, 2, NULL, 4, 5], 2, 3) AS res +SELECT arraySlice([1, 2, NULL, 4, 5], 2, 3) AS res; ``` ``` text @@ -702,9 +702,9 @@ SELECT arrayReverseSort((x, y) -> -y, [4, 3, 5], [1, 2, 3]) AS res; arrayDifference(array) ``` -**Параметры** +**Аргументы** -- `array` – [Массив](https://clickhouse.tech/docs/ru/data_types/array/). +- `array` – [массив](https://clickhouse.tech/docs/ru/data_types/array/). **Возвращаемое значение** @@ -715,10 +715,10 @@ arrayDifference(array) Запрос: ``` sql -SELECT arrayDifference([1, 2, 3, 4]) +SELECT arrayDifference([1, 2, 3, 4]); ``` -Ответ: +Результат: ``` text ┌─arrayDifference([1, 2, 3, 4])─┐ @@ -731,10 +731,10 @@ SELECT arrayDifference([1, 2, 3, 4]) Запрос: ``` sql -SELECT arrayDifference([0, 10000000000000000000]) +SELECT arrayDifference([0, 10000000000000000000]); ``` -Ответ: +Результат: ``` text ┌─arrayDifference([0, 10000000000000000000])─┐ @@ -752,9 +752,9 @@ SELECT arrayDifference([0, 10000000000000000000]) arrayDistinct(array) ``` -**Параметры** +**Аргументы** -- `array` – [Массив](https://clickhouse.tech/docs/ru/data_types/array/). +- `array` – [массив](https://clickhouse.tech/docs/ru/data_types/array/). **Возвращаемое значение** @@ -765,7 +765,7 @@ arrayDistinct(array) Запрос: ``` sql -SELECT arrayDistinct([1, 2, 2, 3, 1]) +SELECT arrayDistinct([1, 2, 2, 3, 1]); ``` Ответ: @@ -820,7 +820,7 @@ SELECT arrayReduce(agg_func, arr1, arr2, ..., arrN) ``` -**Параметры** +**Аргументы** - `agg_func` — Имя агрегатной функции, которая должна быть константой [string](../../sql-reference/data-types/string.md). - `arr` — Любое количество столбцов типа [array](../../sql-reference/data-types/array.md) в качестве параметров агрегатной функции. @@ -832,10 +832,10 @@ arrayReduce(agg_func, arr1, arr2, ..., arrN) Запрос: ```sql -SELECT arrayReduce('max', [1, 2, 3]) +SELECT arrayReduce('max', [1, 2, 3]); ``` -Ответ: +Результат: ```text ┌─arrayReduce('max', [1, 2, 3])─┐ @@ -850,10 +850,10 @@ SELECT arrayReduce('max', [1, 2, 3]) Запрос: ```sql -SELECT arrayReduce('maxIf', [3, 5], [1, 0]) +SELECT arrayReduce('maxIf', [3, 5], [1, 0]); ``` -Ответ: +Результат: ```text ┌─arrayReduce('maxIf', [3, 5], [1, 0])─┐ @@ -866,10 +866,10 @@ SELECT arrayReduce('maxIf', [3, 5], [1, 0]) Запрос: ```sql -SELECT arrayReduce('uniqUpTo(3)', [1, 2, 3, 4, 5, 6, 7, 8, 9, 10]) +SELECT arrayReduce('uniqUpTo(3)', [1, 2, 3, 4, 5, 6, 7, 8, 9, 10]); ``` -Ответ: +Результат: ```text ┌─arrayReduce('uniqUpTo(3)', [1, 2, 3, 4, 5, 6, 7, 8, 9, 10])─┐ @@ -887,15 +887,15 @@ SELECT arrayReduce('uniqUpTo(3)', [1, 2, 3, 4, 5, 6, 7, 8, 9, 10]) arrayReduceInRanges(agg_func, ranges, arr1, arr2, ..., arrN) ``` -**Параметры** +**Аргументы** -- `agg_func` — Имя агрегатной функции, которая должна быть [строковой](../../sql-reference/data-types/string.md) константой. -- `ranges` — Диапазоны для агрегирования, которые должны быть [массивом](../../sql-reference/data-types/array.md) of [кортежей](../../sql-reference/data-types/tuple.md) который содержит индекс и длину каждого диапазона. -- `arr` — Любое количество столбцов типа [Array](../../sql-reference/data-types/array.md) в качестве параметров агрегатной функции. +- `agg_func` — имя агрегатной функции, которая должна быть [строковой](../../sql-reference/data-types/string.md) константой. +- `ranges` — диапазоны для агрегирования, которые должны быть [массивом](../../sql-reference/data-types/array.md) of [кортежей](../../sql-reference/data-types/tuple.md) который содержит индекс и длину каждого диапазона. +- `arr` — любое количество столбцов типа [Array](../../sql-reference/data-types/array.md) в качестве параметров агрегатной функции. **Возвращаемое значение** -- Массив, содержащий результаты агрегатной функции для указанных диапазонов. +- Массив, содержащий результаты агрегатной функции для указанных диапазонов. Тип: [Array](../../sql-reference/data-types/array.md). @@ -911,7 +911,7 @@ SELECT arrayReduceInRanges( ) AS res ``` -Ответ: +Результат: ```text ┌─res─────────────────────────┐ @@ -958,14 +958,14 @@ flatten(array_of_arrays) Синоним: `flatten`. -**Параметры** +**Аргументы** -- `array_of_arrays` — [Массив](../../sql-reference/functions/array-functions.md) массивов. Например, `[[1,2,3], [4,5]]`. +- `array_of_arrays` — [массив](../../sql-reference/functions/array-functions.md) массивов. Например, `[[1,2,3], [4,5]]`. **Примеры** ``` sql -SELECT flatten([[[1]], [[2], [3]]]) +SELECT flatten([[[1]], [[2], [3]]]); ``` ``` text @@ -984,9 +984,9 @@ SELECT flatten([[[1]], [[2], [3]]]) arrayCompact(arr) ``` -**Параметры** +**Аргументы** -`arr` — [Массив](../../sql-reference/functions/array-functions.md) для обхода. +`arr` — [массив](../../sql-reference/functions/array-functions.md) для обхода. **Возвращаемое значение** @@ -999,10 +999,10 @@ arrayCompact(arr) Запрос: ``` sql -SELECT arrayCompact([1, 1, nan, nan, 2, 3, 3, 3]) +SELECT arrayCompact([1, 1, nan, nan, 2, 3, 3, 3]); ``` -Ответ: +Результат: ``` text ┌─arrayCompact([1, 1, nan, nan, 2, 3, 3, 3])─┐ @@ -1020,9 +1020,9 @@ SELECT arrayCompact([1, 1, nan, nan, 2, 3, 3, 3]) arrayZip(arr1, arr2, ..., arrN) ``` -**Параметры** +**Аргументы** -- `arrN` — [Массив](../data-types/array.md). +- `arrN` — [массив](../data-types/array.md). Функция принимает любое количество массивов, которые могут быть различных типов. Все массивы должны иметь одинаковую длину. @@ -1037,10 +1037,10 @@ arrayZip(arr1, arr2, ..., arrN) Запрос: ``` sql -SELECT arrayZip(['a', 'b', 'c'], [5, 2, 1]) +SELECT arrayZip(['a', 'b', 'c'], [5, 2, 1]); ``` -Ответ: +Результат: ``` text ┌─arrayZip(['a', 'b', 'c'], [5, 2, 1])─┐ @@ -1067,7 +1067,7 @@ SELECT arrayMap(x -> (x + 2), [1, 2, 3]) as res; Следующий пример показывает, как создать кортежи из элементов разных массивов: ``` sql -SELECT arrayMap((x, y) -> (x, y), [1, 2, 3], [4, 5, 6]) AS res +SELECT arrayMap((x, y) -> (x, y), [1, 2, 3], [4, 5, 6]) AS res; ``` ``` text @@ -1149,7 +1149,7 @@ SELECT arrayMin([func,] arr) ``` -**Параметры** +**Аргументы** - `func` — функция. [Expression](../../sql-reference/data-types/special-data-types/expression.md). - `arr` — массив. [Array](../../sql-reference/data-types/array.md). @@ -1204,7 +1204,7 @@ SELECT arrayMin(x -> (-x), [1, 2, 4]) AS res; arrayMax([func,] arr) ``` -**Параметры** +**Аргументы** - `func` — функция. [Expression](../../sql-reference/data-types/special-data-types/expression.md). - `arr` — массив. [Array](../../sql-reference/data-types/array.md). @@ -1259,7 +1259,7 @@ SELECT arrayMax(x -> (-x), [1, 2, 4]) AS res; arraySum([func,] arr) ``` -**Параметры** +**Аргументы** - `func` — функция. [Expression](../../sql-reference/data-types/special-data-types/expression.md). - `arr` — массив. [Array](../../sql-reference/data-types/array.md). @@ -1314,7 +1314,7 @@ SELECT arraySum(x -> x*x, [2, 3]) AS res; arrayAvg([func,] arr) ``` -**Параметры** +**Аргументы** - `func` — функция. [Expression](../../sql-reference/data-types/special-data-types/expression.md). - `arr` — массив. [Array](../../sql-reference/data-types/array.md). @@ -1367,9 +1367,9 @@ arraySum(arr) Тип: [Int](../../sql-reference/data-types/int-uint.md) или [Float](../../sql-reference/data-types/float.md). -**Параметры** +**Аргументы** -- `arr` — [Массив](../../sql-reference/data-types/array.md). +- `arr` — [массив](../../sql-reference/data-types/array.md). **Примеры** @@ -1429,7 +1429,8 @@ SELECT arrayCumSum([1, 1, 1, 1]) AS res arrayAUC(arr_scores, arr_labels) ``` -**Параметры** +**Аргументы** + - `arr_scores` — оценка, которую дает модель предсказания. - `arr_labels` — ярлыки выборок, обычно 1 для содержательных выборок и 0 для бессодержательных выборок. @@ -1444,10 +1445,10 @@ arrayAUC(arr_scores, arr_labels) Запрос: ``` sql -select arrayAUC([0.1, 0.4, 0.35, 0.8], [0, 0, 1, 1]) +SELECT arrayAUC([0.1, 0.4, 0.35, 0.8], [0, 0, 1, 1]); ``` -Ответ: +Результат: ``` text ┌─arrayAUC([0.1, 0.4, 0.35, 0.8], [0, 0, 1, 1])─┐ diff --git a/docs/ru/sql-reference/functions/bit-functions.md b/docs/ru/sql-reference/functions/bit-functions.md index 8c7808437a5..9f9808bf0d2 100644 --- a/docs/ru/sql-reference/functions/bit-functions.md +++ b/docs/ru/sql-reference/functions/bit-functions.md @@ -31,10 +31,10 @@ toc_title: "\u0411\u0438\u0442\u043e\u0432\u044b\u0435\u0020\u0444\u0443\u043d\u SELECT bitTest(number, index) ``` -**Параметры** +**Аргументы** - `number` – целое число. -- `index` – position of bit. +- `index` – позиция бита. **Возвращаемое значение** @@ -49,10 +49,10 @@ SELECT bitTest(number, index) Запрос: ``` sql -SELECT bitTest(43, 1) +SELECT bitTest(43, 1); ``` -Ответ: +Результат: ``` text ┌─bitTest(43, 1)─┐ @@ -65,10 +65,10 @@ SELECT bitTest(43, 1) Запрос: ``` sql -SELECT bitTest(43, 2) +SELECT bitTest(43, 2); ``` -Ответ: +Результат: ``` text ┌─bitTest(43, 2)─┐ @@ -93,7 +93,7 @@ SELECT bitTest(43, 2) SELECT bitTestAll(number, index1, index2, index3, index4, ...) ``` -**Параметры** +**Аргументы** - `number` – целое число. - `index1`, `index2`, `index3`, `index4` – позиция бита. Например, конъюнкция для набора позиций `index1`, `index2`, `index3`, `index4` является истинной, если все его позиции истинны `index1` ⋀ `index2` ⋀ `index3` ⋀ `index4`. @@ -111,10 +111,10 @@ SELECT bitTestAll(number, index1, index2, index3, index4, ...) Запрос: ``` sql -SELECT bitTestAll(43, 0, 1, 3, 5) +SELECT bitTestAll(43, 0, 1, 3, 5); ``` -Ответ: +Результат: ``` text ┌─bitTestAll(43, 0, 1, 3, 5)─┐ @@ -127,10 +127,10 @@ SELECT bitTestAll(43, 0, 1, 3, 5) Запрос: ``` sql -SELECT bitTestAll(43, 0, 1, 3, 5, 2) +SELECT bitTestAll(43, 0, 1, 3, 5, 2); ``` -Ответ: +Результат: ``` text ┌─bitTestAll(43, 0, 1, 3, 5, 2)─┐ @@ -155,7 +155,7 @@ SELECT bitTestAll(43, 0, 1, 3, 5, 2) SELECT bitTestAny(number, index1, index2, index3, index4, ...) ``` -**Параметры** +**Аргументы** - `number` – целое число. - `index1`, `index2`, `index3`, `index4` – позиции бита. @@ -173,10 +173,10 @@ SELECT bitTestAny(number, index1, index2, index3, index4, ...) Запрос: ``` sql -SELECT bitTestAny(43, 0, 2) +SELECT bitTestAny(43, 0, 2); ``` -Ответ: +Результат: ``` text ┌─bitTestAny(43, 0, 2)─┐ @@ -189,10 +189,10 @@ SELECT bitTestAny(43, 0, 2) Запрос: ``` sql -SELECT bitTestAny(43, 4, 2) +SELECT bitTestAny(43, 4, 2); ``` -Ответ: +Результат: ``` text ┌─bitTestAny(43, 4, 2)─┐ @@ -210,9 +210,9 @@ SELECT bitTestAny(43, 4, 2) bitCount(x) ``` -**Параметры** +**Аргументы** -- `x` — [Целое число](../../sql-reference/functions/bit-functions.md) или [число с плавающей запятой](../../sql-reference/functions/bit-functions.md). Функция использует представление числа в памяти, что позволяет поддержать числа с плавающей запятой. +- `x` — [целое число](../../sql-reference/functions/bit-functions.md) или [число с плавающей запятой](../../sql-reference/functions/bit-functions.md). Функция использует представление числа в памяти, что позволяет поддержать числа с плавающей запятой. **Возвращаемое значение** @@ -229,7 +229,7 @@ bitCount(x) Запрос: ``` sql -SELECT bitCount(333) +SELECT bitCount(333); ``` Результат: diff --git a/docs/ru/sql-reference/functions/bitmap-functions.md b/docs/ru/sql-reference/functions/bitmap-functions.md index b21ddea94e4..949fa29acb3 100644 --- a/docs/ru/sql-reference/functions/bitmap-functions.md +++ b/docs/ru/sql-reference/functions/bitmap-functions.md @@ -13,14 +13,14 @@ toc_title: "\u0424\u0443\u043d\u043a\u0446\u0438\u0438\u0020\u0434\u043b\u044f\u bitmapBuild(array) ``` -**Параметры** +**Аргументы** - `array` – массив типа `UInt*`. **Пример** ``` sql -SELECT bitmapBuild([1, 2, 3, 4, 5]) AS res, toTypeName(res) +SELECT bitmapBuild([1, 2, 3, 4, 5]) AS res, toTypeName(res); ``` ``` text @@ -37,14 +37,14 @@ SELECT bitmapBuild([1, 2, 3, 4, 5]) AS res, toTypeName(res) bitmapToArray(bitmap) ``` -**Параметры** +**Аргументы** - `bitmap` – битовый массив. **Пример** ``` sql -SELECT bitmapToArray(bitmapBuild([1, 2, 3, 4, 5])) AS res +SELECT bitmapToArray(bitmapBuild([1, 2, 3, 4, 5])) AS res; ``` ``` text @@ -63,11 +63,11 @@ SELECT bitmapToArray(bitmapBuild([1, 2, 3, 4, 5])) AS res bitmapSubsetLimit(bitmap, range_start, cardinality_limit) ``` -**Параметры** +**Аргументы** -- `bitmap` – Битмап. [Bitmap object](#bitmap_functions-bitmapbuild). +- `bitmap` – битмап. [Bitmap object](#bitmap_functions-bitmapbuild). -- `range_start` – Начальная точка подмножества. [UInt32](../../sql-reference/functions/bitmap-functions.md#bitmap-functions). +- `range_start` – начальная точка подмножества. [UInt32](../../sql-reference/functions/bitmap-functions.md#bitmap-functions). - `cardinality_limit` – Верхний предел подмножества. [UInt32](../../sql-reference/functions/bitmap-functions.md#bitmap-functions). **Возвращаемое значение** @@ -81,10 +81,10 @@ bitmapSubsetLimit(bitmap, range_start, cardinality_limit) Запрос: ``` sql -SELECT bitmapToArray(bitmapSubsetLimit(bitmapBuild([0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30,31,32,33,100,200,500]), toUInt32(30), toUInt32(200))) AS res +SELECT bitmapToArray(bitmapSubsetLimit(bitmapBuild([0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30,31,32,33,100,200,500]), toUInt32(30), toUInt32(200))) AS res; ``` -Ответ: +Результат: ``` text ┌─res───────────────────────┐ @@ -100,12 +100,11 @@ SELECT bitmapToArray(bitmapSubsetLimit(bitmapBuild([0,1,2,3,4,5,6,7,8,9,10,11,12 bitmapContains(haystack, needle) ``` -**Параметры** +**Аргументы** - `haystack` – [объект Bitmap](#bitmap_functions-bitmapbuild), в котором функция ищет значение. - `needle` – значение, которое функция ищет. Тип — [UInt32](../../sql-reference/data-types/int-uint.md). - **Возвращаемые значения** - 0 — если в `haystack` нет `needle`. @@ -116,7 +115,7 @@ bitmapContains(haystack, needle) **Пример** ``` sql -SELECT bitmapContains(bitmapBuild([1,5,7,9]), toUInt32(9)) AS res +SELECT bitmapContains(bitmapBuild([1,5,7,9]), toUInt32(9)) AS res; ``` ``` text @@ -135,7 +134,7 @@ bitmapHasAny(bitmap1, bitmap2) Если вы уверены, что `bitmap2` содержит строго один элемент, используйте функцию [bitmapContains](#bitmap_functions-bitmapcontains). Она работает эффективнее. -**Параметры** +**Аргументы** - `bitmap*` – массив любого типа с набором элементов. @@ -147,7 +146,7 @@ bitmapHasAny(bitmap1, bitmap2) **Пример** ``` sql -SELECT bitmapHasAny(bitmapBuild([1,2,3]),bitmapBuild([3,4,5])) AS res +SELECT bitmapHasAny(bitmapBuild([1,2,3]),bitmapBuild([3,4,5])) AS res; ``` ``` text @@ -165,14 +164,14 @@ SELECT bitmapHasAny(bitmapBuild([1,2,3]),bitmapBuild([3,4,5])) AS res bitmapHasAll(bitmap,bitmap) ``` -**Параметры** +**Аргументы** - `bitmap` – битовый массив. **Пример** ``` sql -SELECT bitmapHasAll(bitmapBuild([1,2,3]),bitmapBuild([3,4,5])) AS res +SELECT bitmapHasAll(bitmapBuild([1,2,3]),bitmapBuild([3,4,5])) AS res; ``` ``` text @@ -189,14 +188,14 @@ SELECT bitmapHasAll(bitmapBuild([1,2,3]),bitmapBuild([3,4,5])) AS res bitmapAnd(bitmap,bitmap) ``` -**Параметры** +**Аргументы** - `bitmap` – битовый массив. **Пример** ``` sql -SELECT bitmapToArray(bitmapAnd(bitmapBuild([1,2,3]),bitmapBuild([3,4,5]))) AS res +SELECT bitmapToArray(bitmapAnd(bitmapBuild([1,2,3]),bitmapBuild([3,4,5]))) AS res; ``` ``` text @@ -213,14 +212,14 @@ SELECT bitmapToArray(bitmapAnd(bitmapBuild([1,2,3]),bitmapBuild([3,4,5]))) AS re bitmapOr(bitmap,bitmap) ``` -**Параметры** +**Аргументы** - `bitmap` – битовый массив. **Пример** ``` sql -SELECT bitmapToArray(bitmapOr(bitmapBuild([1,2,3]),bitmapBuild([3,4,5]))) AS res +SELECT bitmapToArray(bitmapOr(bitmapBuild([1,2,3]),bitmapBuild([3,4,5]))) AS res; ``` ``` text @@ -237,14 +236,14 @@ SELECT bitmapToArray(bitmapOr(bitmapBuild([1,2,3]),bitmapBuild([3,4,5]))) AS res bitmapXor(bitmap,bitmap) ``` -**Параметры** +**Аргументы** - `bitmap` – битовый массив. **Пример** ``` sql -SELECT bitmapToArray(bitmapXor(bitmapBuild([1,2,3]),bitmapBuild([3,4,5]))) AS res +SELECT bitmapToArray(bitmapXor(bitmapBuild([1,2,3]),bitmapBuild([3,4,5]))) AS res; ``` ``` text @@ -261,14 +260,14 @@ SELECT bitmapToArray(bitmapXor(bitmapBuild([1,2,3]),bitmapBuild([3,4,5]))) AS re bitmapAndnot(bitmap,bitmap) ``` -**Параметры** +**Аргументы** - `bitmap` – битовый массив. **Пример** ``` sql -SELECT bitmapToArray(bitmapAndnot(bitmapBuild([1,2,3]),bitmapBuild([3,4,5]))) AS res +SELECT bitmapToArray(bitmapAndnot(bitmapBuild([1,2,3]),bitmapBuild([3,4,5]))) AS res; ``` ``` text @@ -285,14 +284,14 @@ SELECT bitmapToArray(bitmapAndnot(bitmapBuild([1,2,3]),bitmapBuild([3,4,5]))) AS bitmapCardinality(bitmap) ``` -**Параметры** +**Аргументы** - `bitmap` – битовый массив. **Пример** ``` sql -SELECT bitmapCardinality(bitmapBuild([1, 2, 3, 4, 5])) AS res +SELECT bitmapCardinality(bitmapBuild([1, 2, 3, 4, 5])) AS res; ``` ``` text @@ -309,7 +308,7 @@ SELECT bitmapCardinality(bitmapBuild([1, 2, 3, 4, 5])) AS res bitmapAndCardinality(bitmap,bitmap) ``` -**Параметры** +**Аргументы** - `bitmap` – битовый массив. @@ -333,7 +332,7 @@ SELECT bitmapAndCardinality(bitmapBuild([1,2,3]),bitmapBuild([3,4,5])) AS res; bitmapOrCardinality(bitmap,bitmap) ``` -**Параметры** +**Аргументы** - `bitmap` – битовый массив. @@ -357,7 +356,7 @@ SELECT bitmapOrCardinality(bitmapBuild([1,2,3]),bitmapBuild([3,4,5])) AS res; bitmapXorCardinality(bitmap,bitmap) ``` -**Параметры** +**Аргументы** - `bitmap` – битовый массив. @@ -381,7 +380,7 @@ SELECT bitmapXorCardinality(bitmapBuild([1,2,3]),bitmapBuild([3,4,5])) AS res; bitmapAndnotCardinality(bitmap,bitmap) ``` -**Параметры** +**Аргументы** - `bitmap` – битовый массив. diff --git a/docs/ru/sql-reference/functions/conditional-functions.md b/docs/ru/sql-reference/functions/conditional-functions.md index 83268b68959..823c0489de2 100644 --- a/docs/ru/sql-reference/functions/conditional-functions.md +++ b/docs/ru/sql-reference/functions/conditional-functions.md @@ -17,11 +17,11 @@ SELECT if(cond, then, else) Если условие `cond` не равно нулю, то возвращается результат выражения `then`. Если условие `cond` равно нулю или является NULL, то результат выражения `then` пропускается и возвращается результат выражения `else`. -**Параметры** +**Аргументы** -- `cond` – Условие, которое может быть равно 0 или нет. Может быть [UInt8](../../sql-reference/functions/conditional-functions.md) или `NULL`. -- `then` - Возвращается результат выражения, если условие `cond` истинно. -- `else` - Возвращается результат выражения, если условие `cond` ложно. +- `cond` – условие, которое может быть равно 0 или нет. Может быть [UInt8](../../sql-reference/functions/conditional-functions.md) или `NULL`. +- `then` – возвращается результат выражения, если условие `cond` истинно. +- `else` – возвращается результат выражения, если условие `cond` ложно. **Возвращаемые значения** @@ -32,10 +32,10 @@ SELECT if(cond, then, else) Запрос: ``` sql -SELECT if(1, plus(2, 2), plus(2, 6)) +SELECT if(1, plus(2, 2), plus(2, 6)); ``` -Ответ: +Результат: ``` text ┌─plus(2, 2)─┐ @@ -46,10 +46,10 @@ SELECT if(1, plus(2, 2), plus(2, 6)) Запрос: ``` sql -SELECT if(0, plus(2, 2), plus(2, 6)) +SELECT if(0, plus(2, 2), plus(2, 6)); ``` -Ответ: +Результат: ``` text ┌─plus(2, 6)─┐ @@ -79,11 +79,11 @@ SELECT if(0, plus(2, 2), plus(2, 6)) multiIf(cond_1, then_1, cond_2, then_2...else) -**Параметры** +**Аргументы** -- `cond_N` — Условие, при выполнении которого функция вернёт `then_N`. -- `then_N` — Результат функции при выполнении. -- `else` — Результат функции, если ни одно из условий не выполнено. +- `cond_N` — условие, при выполнении которого функция вернёт `then_N`. +- `then_N` — результат функции при выполнении. +- `else` — результат функции, если ни одно из условий не выполнено. Функция принимает `2N+1` параметров. diff --git a/docs/ru/sql-reference/functions/date-time-functions.md b/docs/ru/sql-reference/functions/date-time-functions.md index 1cd5ec74540..3e78a5d294a 100644 --- a/docs/ru/sql-reference/functions/date-time-functions.md +++ b/docs/ru/sql-reference/functions/date-time-functions.md @@ -136,7 +136,7 @@ toUnixTimestamp(str, [timezone]) Запрос: ``` sql -SELECT toUnixTimestamp('2017-11-05 08:07:47', 'Asia/Tokyo') AS unix_timestamp +SELECT toUnixTimestamp('2017-11-05 08:07:47', 'Asia/Tokyo') AS unix_timestamp; ``` Результат: @@ -162,6 +162,7 @@ SELECT toUnixTimestamp('2017-11-05 08:07:47', 'Asia/Tokyo') AS unix_timestamp ```sql SELECT toStartOfISOYear(toDate('2017-01-01')) AS ISOYear20170101; ``` + ```text ┌─ISOYear20170101─┐ │ 2016-01-04 │ @@ -215,14 +216,14 @@ SELECT toStartOfISOYear(toDate('2017-01-01')) AS ISOYear20170101; toStartOfSecond(value[, timezone]) ``` -**Параметры** +**Аргументы** -- `value` — Дата и время. [DateTime64](../data-types/datetime64.md). -- `timezone` — [Часовой пояс](../../operations/server-configuration-parameters/settings.md#server_configuration_parameters-timezone) для возвращаемого значения (необязательно). Если параметр не задан, используется часовой пояс параметра `value`. [String](../data-types/string.md). +- `value` — дата и время. [DateTime64](../data-types/datetime64.md). +- `timezone` — [часовой пояс](../../operations/server-configuration-parameters/settings.md#server_configuration_parameters-timezone) для возвращаемого значения (необязательно). Если параметр не задан, используется часовой пояс параметра `value`. [String](../data-types/string.md). **Возвращаемое значение** -- Входное значение с отсеченными долями секунды. +- Входное значение с отсеченными долями секунды. Тип: [DateTime64](../data-types/datetime64.md). @@ -256,9 +257,9 @@ WITH toDateTime64('2020-01-01 10:20:30.999', 3) AS dt64 SELECT toStartOfSecond(d └────────────────────────────────────────┘ ``` -**См. также** +**Смотрите также** -- Часовая зона сервера, конфигурационный параметр [timezone](../../operations/server-configuration-parameters/settings.md#server_configuration_parameters-timezone). +- Часовая зона сервера, конфигурационный параметр [timezone](../../operations/server-configuration-parameters/settings.md#server_configuration_parameters-timezone). ## toStartOfFiveMinute {#tostartoffiveminute} @@ -497,7 +498,7 @@ SELECT now(), date_trunc('hour', now(), 'Europe/Moscow'); └─────────────────────┴────────────────────────────────────────────┘ ``` -**См. также** +**Смотрите также** - [toStartOfInterval](#tostartofintervaltime-or-data-interval-x-unit-time-zone) @@ -511,7 +512,7 @@ SELECT now(), date_trunc('hour', now(), 'Europe/Moscow'); now([timezone]) ``` -**Параметры** +**Аргументы** - `timezone` — [часовой пояс](../../operations/server-configuration-parameters/settings.md#server_configuration_parameters-timezone) для возвращаемого значения (необязательно). [String](../../sql-reference/data-types/string.md) @@ -571,9 +572,9 @@ SELECT now('Europe/Moscow'); dateDiff('unit', startdate, enddate, [timezone]) ``` -**Параметры** +**Аргументы** -- `unit` — Единица измерения времени, в которой будет вычислена разница между `startdate` и `enddate`. [String](../syntax.md#syntax-string-literal). +- `unit` — единица измерения времени, в которой будет вычислена разница между `startdate` и `enddate`. [String](../syntax.md#syntax-string-literal). Поддерживаемые значения: @@ -706,6 +707,7 @@ formatDateTime(Time, Format\[, Timezone\]) Возвращает значение времени и даты в определенном вами формате. **Поля подстановки** + Используйте поля подстановки для того, чтобы определить шаблон для выводимой строки. В колонке «Пример» результат работы функции для времени `2018-01-02 22:33:44`. | Поле | Описание | Пример | diff --git a/docs/ru/sql-reference/functions/encoding-functions.md b/docs/ru/sql-reference/functions/encoding-functions.md index 8c3065e5a77..53a6e126f29 100644 --- a/docs/ru/sql-reference/functions/encoding-functions.md +++ b/docs/ru/sql-reference/functions/encoding-functions.md @@ -15,13 +15,13 @@ toc_title: "\u0424\u0443\u043d\u043a\u0446\u0438\u0438\u0020\u043a\u043e\u0434\u char(number_1, [number_2, ..., number_n]); ``` -**Параметры** +**Аргументы** -- `number_1, number_2, ..., number_n` — Числовые аргументы, которые интерпретируются как целые числа. Типы: [Int](../../sql-reference/functions/encoding-functions.md), [Float](../../sql-reference/functions/encoding-functions.md). +- `number_1, number_2, ..., number_n` — числовые аргументы, которые интерпретируются как целые числа. Типы: [Int](../../sql-reference/functions/encoding-functions.md), [Float](../../sql-reference/functions/encoding-functions.md). **Возвращаемое значение** -- строка из соответствующих байт. +- Строка из соответствующих байт. Тип: `String`. @@ -30,10 +30,10 @@ char(number_1, [number_2, ..., number_n]); Запрос: ``` sql -SELECT char(104.1, 101, 108.9, 108.9, 111) AS hello +SELECT char(104.1, 101, 108.9, 108.9, 111) AS hello; ``` -Ответ: +Результат: ``` text ┌─hello─┐ @@ -49,7 +49,7 @@ SELECT char(104.1, 101, 108.9, 108.9, 111) AS hello SELECT char(0xD0, 0xBF, 0xD1, 0x80, 0xD0, 0xB8, 0xD0, 0xB2, 0xD0, 0xB5, 0xD1, 0x82) AS hello; ``` -Ответ: +Результат: ``` text ┌─hello──┐ @@ -63,7 +63,7 @@ SELECT char(0xD0, 0xBF, 0xD1, 0x80, 0xD0, 0xB8, 0xD0, 0xB2, 0xD0, 0xB5, 0xD1, 0x SELECT char(0xE4, 0xBD, 0xA0, 0xE5, 0xA5, 0xBD) AS hello; ``` -Ответ: +Результат: ``` text ┌─hello─┐ diff --git a/docs/ru/sql-reference/functions/encryption-functions.md b/docs/ru/sql-reference/functions/encryption-functions.md index 0216a6b2356..2309aa85478 100644 --- a/docs/ru/sql-reference/functions/encryption-functions.md +++ b/docs/ru/sql-reference/functions/encryption-functions.md @@ -31,7 +31,7 @@ toc_title: "\u0424\u0443\u043d\u043a\u0446\u0438\u0438 \u0434\u043b\u044f \u0448 encrypt('mode', 'plaintext', 'key' [, iv, aad]) ``` -**Параметры** +**Аргументы** - `mode` — режим шифрования. [String](../../sql-reference/data-types/string.md#string). - `plaintext` — текст, который будет зашифрован. [String](../../sql-reference/data-types/string.md#string). @@ -127,7 +127,7 @@ SELECT comment, hex(secret) FROM encryption_test WHERE comment LIKE '%gcm%'; aes_encrypt_mysql('mode', 'plaintext', 'key' [, iv]) ``` -**Параметры** +**Аргументы** - `mode` — режим шифрования. [String](../../sql-reference/data-types/string.md#string). - `plaintext` — текст, который будет зашифрован. [String](../../sql-reference/data-types/string.md#string). @@ -236,13 +236,13 @@ mysql> SELECT aes_encrypt('Secret', '123456789101213141516171819202122', 'iviviv decrypt('mode', 'ciphertext', 'key' [, iv, aad]) ``` -**Параметры** +**Аргументы** - `mode` — режим шифрования. [String](../../sql-reference/data-types/string.md#string). - `ciphertext` — зашифрованный текст, который будет расшифрован. [String](../../sql-reference/data-types/string.md#string). - `key` — ключ шифрования. [String](../../sql-reference/data-types/string.md#string). - `iv` — инициализирующий вектор. Обязателен для `-gcm` режимов, для остальных режимов опциональный. [String](../../sql-reference/data-types/string.md#string). -- `aad` — дополнительные аутентифицированные данные. Текст не будет расшифрован, если это значение неверно. Работает только с `-gcm` режимами. Для остальных вызовет исключение. [String](../../sql-reference/data-types/string.md#string). +- `aad` — дополнительные аутентифицированные данные. Текст не будет расшифрован, если это значение неверно. Работает только с `-gcm` режимами. Для остальных вызовет исключение. [String](../../sql-reference/data-types/string.md#string). **Возвращаемое значение** @@ -316,7 +316,7 @@ SELECT comment, decrypt('aes-256-cfb128', secret, '12345678910121314151617181920 aes_decrypt_mysql('mode', 'ciphertext', 'key' [, iv]) ``` -**Параметры** +**Аргументы** - `mode` — режим шифрования. [String](../../sql-reference/data-types/string.md#string). - `ciphertext` — зашифрованный текст, который будет расшифрован. [String](../../sql-reference/data-types/string.md#string). diff --git a/docs/ru/sql-reference/functions/ext-dict-functions.md b/docs/ru/sql-reference/functions/ext-dict-functions.md index 6054ed141d4..edc6282108b 100644 --- a/docs/ru/sql-reference/functions/ext-dict-functions.md +++ b/docs/ru/sql-reference/functions/ext-dict-functions.md @@ -16,7 +16,7 @@ dictGet('dict_name', 'attr_name', id_expr) dictGetOrDefault('dict_name', 'attr_name', id_expr, default_value_expr) ``` -**Параметры** +**Аргументы** - `dict_name` — имя словаря. [Строковый литерал](../syntax.md#syntax-string-literal). - `attr_name` — имя столбца словаря. [Строковый литерал](../syntax.md#syntax-string-literal). @@ -105,7 +105,7 @@ LIMIT 3 dictHas('dict_name', id) ``` -**Параметры** +**Аргументы** - `dict_name` — имя словаря. [Строковый литерал](../syntax.md#syntax-string-literal). - `id_expr` — значение ключа словаря. [Выражение](../syntax.md#syntax-expressions), возвращающее значение типа [UInt64](../../sql-reference/functions/ext-dict-functions.md) или [Tuple](../../sql-reference/functions/ext-dict-functions.md) в зависимости от конфигурации словаря. @@ -127,7 +127,7 @@ dictHas('dict_name', id) dictGetHierarchy('dict_name', key) ``` -**Параметры** +**Аргументы** - `dict_name` — имя словаря. [Строковый литерал](../syntax.md#syntax-string-literal). - `key` — значение ключа. [Выражение](../syntax.md#syntax-expressions), возвращающее значение типа [UInt64](../../sql-reference/functions/ext-dict-functions.md). @@ -144,7 +144,7 @@ Type: [Array(UInt64)](../../sql-reference/functions/ext-dict-functions.md). `dictIsIn ('dict_name', child_id_expr, ancestor_id_expr)` -**Параметры** +**Аргументы** - `dict_name` — имя словаря. [Строковый литерал](../syntax.md#syntax-string-literal). - `child_id_expr` — ключ для проверки. [Выражение](../syntax.md#syntax-expressions), возвращающее значение типа [UInt64](../../sql-reference/functions/ext-dict-functions.md). @@ -180,12 +180,12 @@ dictGet[Type]('dict_name', 'attr_name', id_expr) dictGet[Type]OrDefault('dict_name', 'attr_name', id_expr, default_value_expr) ``` -**Параметры** +**Аргументы** - `dict_name` — имя словаря. [Строковый литерал](../syntax.md#syntax-string-literal). - `attr_name` — имя столбца словаря. [Строковый литерал](../syntax.md#syntax-string-literal). - `id_expr` — значение ключа словаря. [Выражение](../syntax.md#syntax-expressions), возвращающее значение типа [UInt64](../../sql-reference/functions/ext-dict-functions.md) или [Tuple](../../sql-reference/functions/ext-dict-functions.md) в зависимости от конфигурации словаря. -- `default_value_expr` — значение, возвращаемое в том случае, когда словарь не содержит строки с заданным ключом `id_expr`. [Выражение](../syntax.md#syntax-expressions) возвращающее значение с типом данных, сконфигурированным для атрибута `attr_name`. +- `default_value_expr` — значение, возвращаемое в том случае, когда словарь не содержит строки с заданным ключом `id_expr`. [Выражение](../syntax.md#syntax-expressions), возвращающее значение с типом данных, сконфигурированным для атрибута `attr_name`. **Возвращаемое значение** diff --git a/docs/ru/sql-reference/functions/functions-for-nulls.md b/docs/ru/sql-reference/functions/functions-for-nulls.md index 0db55847631..ed91af85473 100644 --- a/docs/ru/sql-reference/functions/functions-for-nulls.md +++ b/docs/ru/sql-reference/functions/functions-for-nulls.md @@ -15,7 +15,7 @@ isNull(x) Синоним: `ISNULL`. -**Параметры** +**Аргументы** - `x` — значение с не составным типом данных. @@ -38,7 +38,7 @@ isNull(x) Запрос ``` sql -SELECT x FROM t_null WHERE isNull(y) +SELECT x FROM t_null WHERE isNull(y); ``` ``` text @@ -55,7 +55,7 @@ SELECT x FROM t_null WHERE isNull(y) isNotNull(x) ``` -**Параметры** +**Аргументы** - `x` — значение с не составным типом данных. @@ -78,7 +78,7 @@ isNotNull(x) Запрос ``` sql -SELECT x FROM t_null WHERE isNotNull(y) +SELECT x FROM t_null WHERE isNotNull(y); ``` ``` text @@ -95,7 +95,7 @@ SELECT x FROM t_null WHERE isNotNull(y) coalesce(x,...) ``` -**Параметры** +**Аргументы** - Произвольное количество параметров не составного типа. Все параметры должны быть совместимы по типу данных. @@ -120,7 +120,7 @@ coalesce(x,...) Получим из адресной книги первый доступный способ связаться с клиентом: ``` sql -SELECT coalesce(mail, phone, CAST(icq,'Nullable(String)')) FROM aBook +SELECT coalesce(mail, phone, CAST(icq,'Nullable(String)')) FROM aBook; ``` ``` text @@ -138,7 +138,7 @@ SELECT coalesce(mail, phone, CAST(icq,'Nullable(String)')) FROM aBook ifNull(x,alt) ``` -**Параметры** +**Аргументы** - `x` — значение для проверки на `NULL`, - `alt` — значение, которое функция вернёт, если `x` — `NULL`. @@ -151,7 +151,7 @@ ifNull(x,alt) **Пример** ``` sql -SELECT ifNull('a', 'b') +SELECT ifNull('a', 'b'); ``` ``` text @@ -161,7 +161,7 @@ SELECT ifNull('a', 'b') ``` ``` sql -SELECT ifNull(NULL, 'b') +SELECT ifNull(NULL, 'b'); ``` ``` text @@ -178,7 +178,7 @@ SELECT ifNull(NULL, 'b') nullIf(x, y) ``` -**Параметры** +**Аргументы** `x`, `y` — значения для сравнивания. Они должны быть совместимых типов, иначе ClickHouse сгенерирует исключение. @@ -190,7 +190,7 @@ nullIf(x, y) **Пример** ``` sql -SELECT nullIf(1, 1) +SELECT nullIf(1, 1); ``` ``` text @@ -200,7 +200,7 @@ SELECT nullIf(1, 1) ``` ``` sql -SELECT nullIf(1, 2) +SELECT nullIf(1, 2); ``` ``` text @@ -217,7 +217,7 @@ SELECT nullIf(1, 2) assumeNotNull(x) ``` -**Параметры** +**Аргументы** - `x` — исходное значение. @@ -231,7 +231,7 @@ assumeNotNull(x) Рассмотрим таблицу `t_null`. ``` sql -SHOW CREATE TABLE t_null +SHOW CREATE TABLE t_null; ``` ``` text @@ -250,7 +250,7 @@ SHOW CREATE TABLE t_null Применим функцию `assumeNotNull` к столбцу `y`. ``` sql -SELECT assumeNotNull(y) FROM t_null +SELECT assumeNotNull(y) FROM t_null; ``` ``` text @@ -261,7 +261,7 @@ SELECT assumeNotNull(y) FROM t_null ``` ``` sql -SELECT toTypeName(assumeNotNull(y)) FROM t_null +SELECT toTypeName(assumeNotNull(y)) FROM t_null; ``` ``` text @@ -279,7 +279,7 @@ SELECT toTypeName(assumeNotNull(y)) FROM t_null toNullable(x) ``` -**Параметры** +**Аргументы** - `x` — значение произвольного не составного типа. @@ -290,7 +290,7 @@ toNullable(x) **Пример** ``` sql -SELECT toTypeName(10) +SELECT toTypeName(10); ``` ``` text @@ -300,7 +300,7 @@ SELECT toTypeName(10) ``` ``` sql -SELECT toTypeName(toNullable(10)) +SELECT toTypeName(toNullable(10)); ``` ``` text diff --git a/docs/ru/sql-reference/functions/geo/geohash.md b/docs/ru/sql-reference/functions/geo/geohash.md index 38c64f11b10..2559e26fcd2 100644 --- a/docs/ru/sql-reference/functions/geo/geohash.md +++ b/docs/ru/sql-reference/functions/geo/geohash.md @@ -29,7 +29,7 @@ geohashEncode(longitude, latitude, [precision]) **Пример** ``` sql -SELECT geohashEncode(-5.60302734375, 42.593994140625, 0) AS res +SELECT geohashEncode(-5.60302734375, 42.593994140625, 0) AS res; ``` ``` text @@ -57,7 +57,7 @@ geohashDecode(geohash_string) **Пример** ``` sql -SELECT geohashDecode('ezs42') AS res +SELECT geohashDecode('ezs42') AS res; ``` ``` text @@ -76,13 +76,13 @@ SELECT geohashDecode('ezs42') AS res geohashesInBox(longitude_min, latitude_min, longitude_max, latitude_max, precision) ``` -**Параметры** +**Аргументы** - `longitude_min` — минимальная долгота. Диапазон возможных значений: `[-180°, 180°]`. Тип данных: [Float](../../../sql-reference/data-types/float.md)). -- `latitude_min` - минимальная широта. Диапазон возможных значений: `[-90°, 90°]`. Тип данных: [Float](../../../sql-reference/data-types/float.md). -- `longitude_max` - максимальная долгота. Диапазон возможных значений: `[-180°, 180°]`. Тип данных: [Float](../../../sql-reference/data-types/float.md). -- `latitude_max` - максимальная широта. Диапазон возможных значений: `[-90°, 90°]`. Тип данных: [Float](../../../sql-reference/data-types/float.md). -- `precision` - точность geohash. Диапазон возможных значений: `[1, 12]`. Тип данных: [UInt8](../../../sql-reference/data-types/int-uint.md). +- `latitude_min` — минимальная широта. Диапазон возможных значений: `[-90°, 90°]`. Тип данных: [Float](../../../sql-reference/data-types/float.md). +- `longitude_max` — максимальная долгота. Диапазон возможных значений: `[-180°, 180°]`. Тип данных: [Float](../../../sql-reference/data-types/float.md). +- `latitude_max` — максимальная широта. Диапазон возможных значений: `[-90°, 90°]`. Тип данных: [Float](../../../sql-reference/data-types/float.md). +- `precision` — точность geohash. Диапазон возможных значений: `[1, 12]`. Тип данных: [UInt8](../../../sql-reference/data-types/int-uint.md). !!! info "Замечание" Все передаваемые координаты должны быть одного и того же типа: либо `Float32`, либо `Float64`. @@ -102,8 +102,9 @@ geohashesInBox(longitude_min, latitude_min, longitude_max, latitude_max, precisi Запрос: ``` sql -SELECT geohashesInBox(24.48, 40.56, 24.785, 40.81, 4) AS thasos +SELECT geohashesInBox(24.48, 40.56, 24.785, 40.81, 4) AS thasos; ``` + Результат: ``` text diff --git a/docs/ru/sql-reference/functions/geo/h3.md b/docs/ru/sql-reference/functions/geo/h3.md index 69d06b5dfa6..55da8864cf2 100644 --- a/docs/ru/sql-reference/functions/geo/h3.md +++ b/docs/ru/sql-reference/functions/geo/h3.md @@ -38,8 +38,9 @@ h3IsValid(h3index) Запрос: ``` sql -SELECT h3IsValid(630814730351855103) as h3IsValid +SELECT h3IsValid(630814730351855103) as h3IsValid; ``` + Результат: ``` text @@ -74,8 +75,9 @@ h3GetResolution(h3index) Запрос: ``` sql -SELECT h3GetResolution(639821929606596015) as resolution +SELECT h3GetResolution(639821929606596015) as resolution; ``` + Результат: ``` text @@ -107,8 +109,9 @@ h3EdgeAngle(resolution) Запрос: ``` sql -SELECT h3EdgeAngle(10) as edgeAngle +SELECT h3EdgeAngle(10) as edgeAngle; ``` + Результат: ``` text @@ -140,8 +143,9 @@ h3EdgeLengthM(resolution) Запрос: ``` sql -SELECT h3EdgeLengthM(15) as edgeLengthM +SELECT h3EdgeLengthM(15) as edgeLengthM; ``` + Результат: ``` text @@ -160,7 +164,7 @@ SELECT h3EdgeLengthM(15) as edgeLengthM geoToH3(lon, lat, resolution) ``` -**Параметры** +**Аргументы** - `lon` — географическая долгота. Тип данных — [Float64](../../../sql-reference/data-types/float.md). - `lat` — географическая широта. Тип данных — [Float64](../../../sql-reference/data-types/float.md). @@ -178,10 +182,10 @@ geoToH3(lon, lat, resolution) Запрос: ``` sql -SELECT geoToH3(37.79506683, 55.71290588, 15) as h3Index +SELECT geoToH3(37.79506683, 55.71290588, 15) as h3Index; ``` -Ответ: +Результат: ``` text ┌────────────h3Index─┐ @@ -199,7 +203,7 @@ SELECT geoToH3(37.79506683, 55.71290588, 15) as h3Index h3kRing(h3index, k) ``` -**Параметры** +**Аргументы** - `h3index` — идентификатор шестигранника. Тип данных: [UInt64](../../../sql-reference/data-types/int-uint.md). - `k` — радиус. Тип данных: [целое число](../../../sql-reference/data-types/int-uint.md) @@ -215,8 +219,9 @@ h3kRing(h3index, k) Запрос: ``` sql -SELECT arrayJoin(h3kRing(644325529233966508, 1)) AS h3index +SELECT arrayJoin(h3kRing(644325529233966508, 1)) AS h3index; ``` + Результат: ``` text @@ -311,7 +316,7 @@ SELECT h3HexAreaM2(13) as area; h3IndexesAreNeighbors(index1, index2) ``` -**Параметры** +**Аргументы** - `index1` — индекс шестиугольной ячейки. Тип: [UInt64](../../../sql-reference/data-types/int-uint.md). - `index2` — индекс шестиугольной ячейки. Тип: [UInt64](../../../sql-reference/data-types/int-uint.md). @@ -349,7 +354,7 @@ SELECT h3IndexesAreNeighbors(617420388351344639, 617420388352655359) AS n; h3ToChildren(index, resolution) ``` -**Параметры** +**Аргументы** - `index` — индекс шестиугольной ячейки. Тип: [UInt64](../../../sql-reference/data-types/int-uint.md). - `resolution` — разрешение. Диапазон: `[0, 15]`. Тип: [UInt8](../../../sql-reference/data-types/int-uint.md). @@ -386,7 +391,7 @@ SELECT h3ToChildren(599405990164561919, 6) AS children; h3ToParent(index, resolution) ``` -**Параметры** +**Аргументы** - `index` — индекс шестиугольной ячейки. Тип: [UInt64](../../../sql-reference/data-types/int-uint.md). - `resolution` — разрешение. Диапазон: `[0, 15]`. Тип: [UInt8](../../../sql-reference/data-types/int-uint.md). diff --git a/docs/ru/sql-reference/functions/hash-functions.md b/docs/ru/sql-reference/functions/hash-functions.md index f7820889ea9..31229f49889 100644 --- a/docs/ru/sql-reference/functions/hash-functions.md +++ b/docs/ru/sql-reference/functions/hash-functions.md @@ -18,9 +18,9 @@ halfMD5(par1, ...) Функция относительно медленная (5 миллионов коротких строк в секунду на ядро процессора). По возможности, используйте функцию [sipHash64](#hash_functions-siphash64) вместо неё. -**Параметры** +**Аргументы** -Функция принимает переменное число входных параметров. Параметры могут быть любого [поддерживаемого типа данных](../../sql-reference/functions/hash-functions.md). +Функция принимает переменное число входных параметров. Аргументы могут быть любого [поддерживаемого типа данных](../../sql-reference/functions/hash-functions.md). **Возвращаемое значение** @@ -29,7 +29,7 @@ halfMD5(par1, ...) **Пример** ``` sql -SELECT halfMD5(array('e','x','a'), 'mple', 10, toDateTime('2019-06-15 23:00:00')) AS halfMD5hash, toTypeName(halfMD5hash) AS type +SELECT halfMD5(array('e','x','a'), 'mple', 10, toDateTime('2019-06-15 23:00:00')) AS halfMD5hash, toTypeName(halfMD5hash) AS type; ``` ``` text @@ -61,9 +61,9 @@ sipHash64(par1,...) 3. Затем функция принимает хэш-значение, вычисленное на предыдущем шаге, и третий элемент исходного хэш-массива, и вычисляет хэш для массива из них. 4. Предыдущий шаг повторяется для всех остальных элементов исходного хэш-массива. -**Параметры** +**Аргументы** -Функция принимает переменное число входных параметров. Параметры могут быть любого [поддерживаемого типа данных](../../sql-reference/functions/hash-functions.md). +Функция принимает переменное число входных параметров. Аргументы могут быть любого [поддерживаемого типа данных](../../sql-reference/functions/hash-functions.md). **Возвращаемое значение** @@ -72,7 +72,7 @@ sipHash64(par1,...) **Пример** ``` sql -SELECT sipHash64(array('e','x','a'), 'mple', 10, toDateTime('2019-06-15 23:00:00')) AS SipHash, toTypeName(SipHash) AS type +SELECT sipHash64(array('e','x','a'), 'mple', 10, toDateTime('2019-06-15 23:00:00')) AS SipHash, toTypeName(SipHash) AS type; ``` ``` text @@ -97,9 +97,9 @@ cityHash64(par1,...) Это не криптографическая хэш-функция. Она использует CityHash алгоритм для строковых параметров и зависящую от реализации быструю некриптографическую хэш-функцию для параметров с другими типами данных. Функция использует комбинатор CityHash для получения конечных результатов. -**Параметры** +**Аргументы** -Функция принимает переменное число входных параметров. Параметры могут быть любого [поддерживаемого типа данных](../../sql-reference/functions/hash-functions.md). +Функция принимает переменное число входных параметров. Аргументы могут быть любого [поддерживаемого типа данных](../../sql-reference/functions/hash-functions.md). **Возвращаемое значение** @@ -110,7 +110,7 @@ cityHash64(par1,...) Пример вызова: ``` sql -SELECT cityHash64(array('e','x','a'), 'mple', 10, toDateTime('2019-06-15 23:00:00')) AS CityHash, toTypeName(CityHash) AS type +SELECT cityHash64(array('e','x','a'), 'mple', 10, toDateTime('2019-06-15 23:00:00')) AS CityHash, toTypeName(CityHash) AS type; ``` ``` text @@ -166,9 +166,9 @@ farmHash64(par1, ...) Эти функции используют методы `Fingerprint64` и `Hash64` из всех [доступных методов](https://github.com/google/farmhash/blob/master/src/farmhash.h). -**Параметры** +**Аргументы** -Функция принимает переменное число входных параметров. Параметры могут быть любого [поддерживаемого типа данных](../../sql-reference/functions/hash-functions.md). +Функция принимает переменное число входных параметров. Аргументы могут быть любого [поддерживаемого типа данных](../../sql-reference/functions/hash-functions.md). **Возвращаемое значение** @@ -177,7 +177,7 @@ farmHash64(par1, ...) **Пример** ``` sql -SELECT farmHash64(array('e','x','a'), 'mple', 10, toDateTime('2019-06-15 23:00:00')) AS FarmHash, toTypeName(FarmHash) AS type +SELECT farmHash64(array('e','x','a'), 'mple', 10, toDateTime('2019-06-15 23:00:00')) AS FarmHash, toTypeName(FarmHash) AS type; ``` ``` text @@ -191,7 +191,7 @@ SELECT farmHash64(array('e','x','a'), 'mple', 10, toDateTime('2019-06-15 23:00:0 Вычисляет [JavaHash](http://hg.openjdk.java.net/jdk8u/jdk8u/jdk/file/478a4add975b/src/share/classes/java/lang/String.java#l1452) от строки. `JavaHash` не отличается ни скоростью, ни качеством, поэтому эту функцию следует считать устаревшей. Используйте эту функцию, если вам необходимо получить значение хэша по такому же алгоритму. ``` sql -SELECT javaHash(''); +SELECT javaHash('') ``` **Возвращаемое значение** @@ -208,7 +208,7 @@ SELECT javaHash(''); SELECT javaHash('Hello, world!'); ``` -Ответ: +Результат: ``` text ┌─javaHash('Hello, world!')─┐ @@ -226,7 +226,7 @@ SELECT javaHash('Hello, world!'); javaHashUTF16LE(stringUtf16le) ``` -**Параметры** +**Аргументы** - `stringUtf16le` — строка в `UTF-16LE`. @@ -243,10 +243,10 @@ javaHashUTF16LE(stringUtf16le) Запрос: ``` sql -SELECT javaHashUTF16LE(convertCharset('test', 'utf-8', 'utf-16le')) +SELECT javaHashUTF16LE(convertCharset('test', 'utf-8', 'utf-16le')); ``` -Ответ: +Результат: ``` text ┌─javaHashUTF16LE(convertCharset('test', 'utf-8', 'utf-16le'))─┐ @@ -259,7 +259,7 @@ SELECT javaHashUTF16LE(convertCharset('test', 'utf-8', 'utf-16le')) Вычисляет `HiveHash` от строки. ``` sql -SELECT hiveHash(''); +SELECT hiveHash('') ``` `HiveHash` — это результат [JavaHash](#hash_functions-javahash) с обнулённым битом знака числа. Функция используется в [Apache Hive](https://en.wikipedia.org/wiki/Apache_Hive) вплоть до версии 3.0. @@ -278,7 +278,7 @@ SELECT hiveHash(''); SELECT hiveHash('Hello, world!'); ``` -Ответ: +Результат: ``` text ┌─hiveHash('Hello, world!')─┐ @@ -294,9 +294,9 @@ SELECT hiveHash('Hello, world!'); metroHash64(par1, ...) ``` -**Параметры** +**Аргументы** -Функция принимает переменное число входных параметров. Параметры могут быть любого [поддерживаемого типа данных](../../sql-reference/functions/hash-functions.md). +Функция принимает переменное число входных параметров. Аргументы могут быть любого [поддерживаемого типа данных](../../sql-reference/functions/hash-functions.md). **Возвращаемое значение** @@ -305,7 +305,7 @@ metroHash64(par1, ...) **Пример** ``` sql -SELECT metroHash64(array('e','x','a'), 'mple', 10, toDateTime('2019-06-15 23:00:00')) AS MetroHash, toTypeName(MetroHash) AS type +SELECT metroHash64(array('e','x','a'), 'mple', 10, toDateTime('2019-06-15 23:00:00')) AS MetroHash, toTypeName(MetroHash) AS type; ``` ``` text @@ -329,9 +329,9 @@ murmurHash2_32(par1, ...) murmurHash2_64(par1, ...) ``` -**Параметры** +**Аргументы** -Обе функции принимают переменное число входных параметров. Параметры могут быть любого [поддерживаемого типа данных](../../sql-reference/functions/hash-functions.md). +Обе функции принимают переменное число входных параметров. Аргументы могут быть любого [поддерживаемого типа данных](../../sql-reference/functions/hash-functions.md). **Возвращаемое значение** @@ -341,7 +341,7 @@ murmurHash2_64(par1, ...) **Пример** ``` sql -SELECT murmurHash2_64(array('e','x','a'), 'mple', 10, toDateTime('2019-06-15 23:00:00')) AS MurmurHash2, toTypeName(MurmurHash2) AS type +SELECT murmurHash2_64(array('e','x','a'), 'mple', 10, toDateTime('2019-06-15 23:00:00')) AS MurmurHash2, toTypeName(MurmurHash2) AS type; ``` ``` text @@ -360,9 +360,9 @@ SELECT murmurHash2_64(array('e','x','a'), 'mple', 10, toDateTime('2019-06-15 23: gccMurmurHash(par1, ...); ``` -**Параметры** +**Аргументы** -- `par1, ...` — Переменное число параметров. Каждый параметр может быть любого из [поддерживаемых типов данных](../../sql-reference/data-types/index.md). +- `par1, ...` — переменное число параметров. Каждый параметр может быть любого из [поддерживаемых типов данных](../../sql-reference/data-types/index.md). **Возвращаемое значение** @@ -397,9 +397,9 @@ murmurHash3_32(par1, ...) murmurHash3_64(par1, ...) ``` -**Параметры** +**Аргументы** -Обе функции принимают переменное число входных параметров. Параметры могут быть любого [поддерживаемого типа данных](../../sql-reference/functions/hash-functions.md). +Обе функции принимают переменное число входных параметров. Аргументы могут быть любого [поддерживаемого типа данных](../../sql-reference/functions/hash-functions.md). **Возвращаемое значение** @@ -409,7 +409,7 @@ murmurHash3_64(par1, ...) **Пример** ``` sql -SELECT murmurHash3_32(array('e','x','a'), 'mple', 10, toDateTime('2019-06-15 23:00:00')) AS MurmurHash3, toTypeName(MurmurHash3) AS type +SELECT murmurHash3_32(array('e','x','a'), 'mple', 10, toDateTime('2019-06-15 23:00:00')) AS MurmurHash3, toTypeName(MurmurHash3) AS type; ``` ``` text @@ -426,9 +426,9 @@ SELECT murmurHash3_32(array('e','x','a'), 'mple', 10, toDateTime('2019-06-15 23: murmurHash3_128( expr ) ``` -**Параметры** +**Аргументы** -- `expr` — [выражение](../syntax.md#syntax-expressions) возвращающее значение типа[String](../../sql-reference/functions/hash-functions.md). +- `expr` — [выражение](../syntax.md#syntax-expressions), возвращающее значение типа[String](../../sql-reference/functions/hash-functions.md). **Возвращаемое значение** @@ -437,7 +437,7 @@ murmurHash3_128( expr ) **Пример** ``` sql -SELECT murmurHash3_128('example_string') AS MurmurHash3, toTypeName(MurmurHash3) AS type +SELECT murmurHash3_128('example_string') AS MurmurHash3, toTypeName(MurmurHash3) AS type; ``` ``` text @@ -451,11 +451,11 @@ SELECT murmurHash3_128('example_string') AS MurmurHash3, toTypeName(MurmurHash3) Вычисляет `xxHash` от строки. Предлагается в двух вариантах: 32 и 64 бита. ``` sql -SELECT xxHash32(''); +SELECT xxHash32('') OR -SELECT xxHash64(''); +SELECT xxHash64('') ``` **Возвращаемое значение** @@ -472,7 +472,7 @@ SELECT xxHash64(''); SELECT xxHash32('Hello, world!'); ``` -Ответ: +Результат: ``` text ┌─xxHash32('Hello, world!')─┐ diff --git a/docs/ru/sql-reference/functions/introspection.md b/docs/ru/sql-reference/functions/introspection.md index 00dd660bc16..7177f947f8a 100644 --- a/docs/ru/sql-reference/functions/introspection.md +++ b/docs/ru/sql-reference/functions/introspection.md @@ -32,7 +32,7 @@ ClickHouse сохраняет отчеты профилировщика в [жу addressToLine(address_of_binary_instruction) ``` -**Параметры** +**Аргументы** - `address_of_binary_instruction` ([Тип UInt64](../../sql-reference/functions/introspection.md))- Адрес инструкции в запущенном процессе. @@ -53,13 +53,13 @@ addressToLine(address_of_binary_instruction) Включение функций самоанализа: ``` sql -SET allow_introspection_functions=1 +SET allow_introspection_functions=1; ``` Выбор первой строки из списка `trace_log` системная таблица: ``` sql -SELECT * FROM system.trace_log LIMIT 1 \G +SELECT * FROM system.trace_log LIMIT 1 \G; ``` ``` text @@ -79,7 +79,7 @@ trace: [140658411141617,94784174532828,94784076370703,94784076 Получение имени файла исходного кода и номера строки для одного адреса: ``` sql -SELECT addressToLine(94784076370703) \G +SELECT addressToLine(94784076370703) \G; ``` ``` text @@ -123,9 +123,9 @@ trace_source_code_lines: /lib/x86_64-linux-gnu/libpthread-2.27.so addressToSymbol(address_of_binary_instruction) ``` -**Параметры** +**Аргументы** -- `address_of_binary_instruction` ([Тип uint64](../../sql-reference/functions/introspection.md)) — Адрес инструкции в запущенном процессе. +- `address_of_binary_instruction` ([Тип uint64](../../sql-reference/functions/introspection.md)) — адрес инструкции в запущенном процессе. **Возвращаемое значение** @@ -139,13 +139,13 @@ addressToSymbol(address_of_binary_instruction) Включение функций самоанализа: ``` sql -SET allow_introspection_functions=1 +SET allow_introspection_functions=1; ``` Выбор первой строки из списка `trace_log` системная таблица: ``` sql -SELECT * FROM system.trace_log LIMIT 1 \G +SELECT * FROM system.trace_log LIMIT 1 \G; ``` ``` text @@ -165,7 +165,7 @@ trace: [94138803686098,94138815010911,94138815096522,94138815101224,9413 Получение символа для одного адреса: ``` sql -SELECT addressToSymbol(94138803686098) \G +SELECT addressToSymbol(94138803686098) \G; ``` ``` text @@ -220,9 +220,9 @@ clone demangle(symbol) ``` -**Параметры** +**Аргументы** -- `symbol` ([Строка](../../sql-reference/functions/introspection.md)) - Символ из объектного файла. +- `symbol` ([Строка](../../sql-reference/functions/introspection.md)) - символ из объектного файла. **Возвращаемое значение** @@ -236,13 +236,13 @@ demangle(symbol) Включение функций самоанализа: ``` sql -SET allow_introspection_functions=1 +SET allow_introspection_functions=1; ``` Выбор первой строки из списка `trace_log` системная таблица: ``` sql -SELECT * FROM system.trace_log LIMIT 1 \G +SELECT * FROM system.trace_log LIMIT 1 \G; ``` ``` text @@ -262,7 +262,7 @@ trace: [94138803686098,94138815010911,94138815096522,94138815101224,9413 Получение имени функции для одного адреса: ``` sql -SELECT demangle(addressToSymbol(94138803686098)) \G +SELECT demangle(addressToSymbol(94138803686098)) \G; ``` ``` text @@ -336,6 +336,7 @@ SELECT tid(); │ 3878 │ └───────┘ ``` + ## logTrace {#logtrace} Выводит сообщение в лог сервера для каждого [Block](https://clickhouse.tech/docs/ru/development/architecture/#block). @@ -346,7 +347,7 @@ SELECT tid(); logTrace('message') ``` -**Параметры** +**Аргументы** - `message` — сообщение, которое отправляется в серверный лог. [String](../../sql-reference/data-types/string.md#string). @@ -354,7 +355,7 @@ logTrace('message') - Всегда возвращает 0. -**Example** +**Пример** Запрос: @@ -370,4 +371,4 @@ SELECT logTrace('logTrace message'); └──────────────────────────────┘ ``` -[Original article](https://clickhouse.tech/docs/en/query_language/functions/introspection/) \ No newline at end of file +[Original article](https://clickhouse.tech/docs/en/query_language/functions/introspection/) diff --git a/docs/ru/sql-reference/functions/ip-address-functions.md b/docs/ru/sql-reference/functions/ip-address-functions.md index bc48419473d..aa0ff54069c 100644 --- a/docs/ru/sql-reference/functions/ip-address-functions.md +++ b/docs/ru/sql-reference/functions/ip-address-functions.md @@ -174,7 +174,7 @@ SELECT addr, cutIPv6(IPv6StringToNum(addr), 0, 0) FROM (SELECT ['notaddress', '1 Принимает число типа `UInt32`. Интерпретирует его, как IPv4-адрес в [big endian](https://en.wikipedia.org/wiki/Endianness). Возвращает значение `FixedString(16)`, содержащее адрес IPv6 в двоичном формате. Примеры: ``` sql -SELECT IPv6NumToString(IPv4ToIPv6(IPv4StringToNum('192.168.0.1'))) AS addr +SELECT IPv6NumToString(IPv4ToIPv6(IPv4StringToNum('192.168.0.1'))) AS addr; ``` ``` text @@ -207,7 +207,7 @@ SELECT Принимает на вход IPv4 и значение `UInt8`, содержащее [CIDR](https://ru.wikipedia.org/wiki/Бесклассовая_адресация). Возвращает кортеж с двумя IPv4, содержащими нижний и более высокий диапазон подсети. ``` sql -SELECT IPv4CIDRToRange(toIPv4('192.168.5.2'), 16) +SELECT IPv4CIDRToRange(toIPv4('192.168.5.2'), 16); ``` ``` text @@ -221,7 +221,7 @@ SELECT IPv4CIDRToRange(toIPv4('192.168.5.2'), 16) Принимает на вход IPv6 и значение `UInt8`, содержащее CIDR. Возвращает кортеж с двумя IPv6, содержащими нижний и более высокий диапазон подсети. ``` sql -SELECT IPv6CIDRToRange(toIPv6('2001:0db8:0000:85a3:0000:0000:ac1f:8001'), 32) +SELECT IPv6CIDRToRange(toIPv6('2001:0db8:0000:85a3:0000:0000:ac1f:8001'), 32); ``` ``` text @@ -328,7 +328,7 @@ SELECT toIPv6('127.0.0.1'); isIPv4String(string) ``` -**Параметры** +**Аргументы** - `string` — IP адрес. [String](../../sql-reference/data-types/string.md). @@ -343,7 +343,7 @@ isIPv4String(string) Запрос: ```sql -SELECT addr, isIPv4String(addr) FROM ( SELECT ['0.0.0.0', '127.0.0.1', '::ffff:127.0.0.1'] AS addr ) ARRAY JOIN addr +SELECT addr, isIPv4String(addr) FROM ( SELECT ['0.0.0.0', '127.0.0.1', '::ffff:127.0.0.1'] AS addr ) ARRAY JOIN addr; ``` Результат: @@ -366,7 +366,7 @@ SELECT addr, isIPv4String(addr) FROM ( SELECT ['0.0.0.0', '127.0.0.1', '::ffff:1 isIPv6String(string) ``` -**Параметры** +**Аргументы** - `string` — IP адрес. [String](../../sql-reference/data-types/string.md). @@ -381,7 +381,7 @@ isIPv6String(string) Запрос: ``` sql -SELECT addr, isIPv6String(addr) FROM ( SELECT ['::', '1111::ffff', '::ffff:127.0.0.1', '127.0.0.1'] AS addr ) ARRAY JOIN addr +SELECT addr, isIPv6String(addr) FROM ( SELECT ['::', '1111::ffff', '::ffff:127.0.0.1', '127.0.0.1'] AS addr ) ARRAY JOIN addr; ``` Результат: diff --git a/docs/ru/sql-reference/functions/json-functions.md b/docs/ru/sql-reference/functions/json-functions.md index 69b8f8f98f5..770b8dec37c 100644 --- a/docs/ru/sql-reference/functions/json-functions.md +++ b/docs/ru/sql-reference/functions/json-functions.md @@ -211,7 +211,7 @@ SELECT JSONExtractKeysAndValues('{"x": {"a": 5, "b": 7, "c": 11}}', 'x', 'Int8') Пример: ``` sql -SELECT JSONExtractRaw('{"a": "hello", "b": [-100, 200.0, 300]}', 'b') = '[-100, 200.0, 300]' +SELECT JSONExtractRaw('{"a": "hello", "b": [-100, 200.0, 300]}', 'b') = '[-100, 200.0, 300]'; ``` ## JSONExtractArrayRaw(json\[, indices_or_keys\]…) {#jsonextractarrayrawjson-indices-or-keys} @@ -223,7 +223,7 @@ SELECT JSONExtractRaw('{"a": "hello", "b": [-100, 200.0, 300]}', 'b') = '[-100, Пример: ``` sql -SELECT JSONExtractArrayRaw('{"a": "hello", "b": [-100, 200.0, "hello"]}', 'b') = ['-100', '200.0', '"hello"']' +SELECT JSONExtractArrayRaw('{"a": "hello", "b": [-100, 200.0, "hello"]}', 'b') = ['-100', '200.0', '"hello"']'; ``` ## JSONExtractKeysAndValuesRaw {#json-extract-keys-and-values-raw} @@ -236,29 +236,28 @@ SELECT JSONExtractArrayRaw('{"a": "hello", "b": [-100, 200.0, "hello"]}', 'b') = JSONExtractKeysAndValuesRaw(json[, p, a, t, h]) ``` -**Параметры** +**Аргументы** -- `json` — [Строка](../data-types/string.md), содержащая валидный JSON. -- `p, a, t, h` — Индексы или ключи, разделенные запятыми, которые указывают путь к внутреннему полю во вложенном объекте JSON. Каждый аргумент может быть либо [строкой](../data-types/string.md) для получения поля по ключу, либо [целым числом](../data-types/int-uint.md) для получения N-го поля (индексирование начинается с 1, отрицательные числа используются для отсчета с конца). Если параметр не задан, весь JSON парсится как объект верхнего уровня. Необязательный параметр. +- `json` — [строка](../data-types/string.md), содержащая валидный JSON. +- `p, a, t, h` — индексы или ключи, разделенные запятыми, которые указывают путь к внутреннему полю во вложенном объекте JSON. Каждый аргумент может быть либо [строкой](../data-types/string.md) для получения поля по ключу, либо [целым числом](../data-types/int-uint.md) для получения N-го поля (индексирование начинается с 1, отрицательные числа используются для отсчета с конца). Если параметр не задан, весь JSON парсится как объект верхнего уровня. Необязательный параметр. **Возвращаемые значения** -- Массив с кортежами `('key', 'value')`. Члены кортежа — строки. +- Массив с кортежами `('key', 'value')`. Члены кортежа — строки. -- Пустой массив, если заданный объект не существует или входные данные не валидный JSON. +- Пустой массив, если заданный объект не существует или входные данные не валидный JSON. -Тип: Type: [Array](../data-types/array.md)([Tuple](../data-types/tuple.md)([String](../data-types/string.md), [String](../data-types/string.md)). -. +Тип: [Array](../data-types/array.md)([Tuple](../data-types/tuple.md)([String](../data-types/string.md), [String](../data-types/string.md)). **Примеры** Запрос: ``` sql -SELECT JSONExtractKeysAndValuesRaw('{"a": [-100, 200.0], "b":{"c": {"d": "hello", "f": "world"}}}') +SELECT JSONExtractKeysAndValuesRaw('{"a": [-100, 200.0], "b":{"c": {"d": "hello", "f": "world"}}}'); ``` -Ответ: +Результат: ``` text ┌─JSONExtractKeysAndValuesRaw('{"a": [-100, 200.0], "b":{"c": {"d": "hello", "f": "world"}}}')─┐ @@ -269,10 +268,10 @@ SELECT JSONExtractKeysAndValuesRaw('{"a": [-100, 200.0], "b":{"c": {"d": "hello" Запрос: ``` sql -SELECT JSONExtractKeysAndValuesRaw('{"a": [-100, 200.0], "b":{"c": {"d": "hello", "f": "world"}}}', 'b') +SELECT JSONExtractKeysAndValuesRaw('{"a": [-100, 200.0], "b":{"c": {"d": "hello", "f": "world"}}}', 'b'); ``` -Ответ: +Результат: ``` text ┌─JSONExtractKeysAndValuesRaw('{"a": [-100, 200.0], "b":{"c": {"d": "hello", "f": "world"}}}', 'b')─┐ @@ -283,10 +282,10 @@ SELECT JSONExtractKeysAndValuesRaw('{"a": [-100, 200.0], "b":{"c": {"d": "hello" Запрос: ``` sql -SELECT JSONExtractKeysAndValuesRaw('{"a": [-100, 200.0], "b":{"c": {"d": "hello", "f": "world"}}}', -1, 'c') +SELECT JSONExtractKeysAndValuesRaw('{"a": [-100, 200.0], "b":{"c": {"d": "hello", "f": "world"}}}', -1, 'c'); ``` -Ответ: +Результат: ``` text ┌─JSONExtractKeysAndValuesRaw('{"a": [-100, 200.0], "b":{"c": {"d": "hello", "f": "world"}}}', -1, 'c')─┐ diff --git a/docs/ru/sql-reference/functions/machine-learning-functions.md b/docs/ru/sql-reference/functions/machine-learning-functions.md index 2ffdfd05613..7cba6ffccc2 100644 --- a/docs/ru/sql-reference/functions/machine-learning-functions.md +++ b/docs/ru/sql-reference/functions/machine-learning-functions.md @@ -27,7 +27,7 @@ toc_title: "\u0424\u0443\u043d\u043a\u0446\u0438\u0438\u0020\u043c\u0430\u0448\u bayesAB(distribution_name, higher_is_better, variant_names, x, y) ``` -**Параметры** +**Аргументы** - `distribution_name` — вероятностное распределение. [String](../../sql-reference/data-types/string.md). Возможные значения: @@ -36,14 +36,14 @@ bayesAB(distribution_name, higher_is_better, variant_names, x, y) - `higher_is_better` — способ определения предпочтений. [Boolean](../../sql-reference/data-types/boolean.md). Возможные значения: - - `0` - чем меньше значение, тем лучше - - `1` - чем больше значение, тем лучше + - `0` — чем меньше значение, тем лучше + - `1` — чем больше значение, тем лучше -- `variant_names` - массив, содержащий названия вариантов. [Array](../../sql-reference/data-types/array.md)([String](../../sql-reference/data-types/string.md)). +- `variant_names` — массив, содержащий названия вариантов. [Array](../../sql-reference/data-types/array.md)([String](../../sql-reference/data-types/string.md)). -- `x` - массив, содержащий число проведенных тестов (испытаний) для каждого варианта. [Array](../../sql-reference/data-types/array.md)([Float64](../../sql-reference/data-types/float.md)). +- `x` — массив, содержащий число проведенных тестов (испытаний) для каждого варианта. [Array](../../sql-reference/data-types/array.md)([Float64](../../sql-reference/data-types/float.md)). -- `y` - массив, содержащий число успешных тестов (испытаний) для каждого варианта. [Array](../../sql-reference/data-types/array.md)([Float64](../../sql-reference/data-types/float.md)). +- `y` — массив, содержащий число успешных тестов (испытаний) для каждого варианта. [Array](../../sql-reference/data-types/array.md)([Float64](../../sql-reference/data-types/float.md)). !!! note "Замечание" Все три массива должны иметь одинаковый размер. Все значения `x` и `y` должны быть неотрицательными числами (константами). Значение `y` не может превышать соответствующее значение `x`. @@ -51,8 +51,8 @@ bayesAB(distribution_name, higher_is_better, variant_names, x, y) **Возвращаемые значения** Для каждого варианта рассчитываются: -- `beats_control` - вероятность, что данный вариант превосходит контрольный в долгосрочной перспективе -- `to_be_best` - вероятность, что данный вариант является лучшим в долгосрочной перспективе +- `beats_control` — вероятность, что данный вариант превосходит контрольный в долгосрочной перспективе +- `to_be_best` — вероятность, что данный вариант является лучшим в долгосрочной перспективе Тип: JSON. diff --git a/docs/ru/sql-reference/functions/math-functions.md b/docs/ru/sql-reference/functions/math-functions.md index 2e57aca6a0a..b78fb7223a9 100644 --- a/docs/ru/sql-reference/functions/math-functions.md +++ b/docs/ru/sql-reference/functions/math-functions.md @@ -54,7 +54,7 @@ toc_title: "\u041c\u0430\u0442\u0435\u043c\u0430\u0442\u0438\u0447\u0435\u0441\u Пример (правило трёх сигм): ``` sql -SELECT erf(3 / sqrt(2)) +SELECT erf(3 / sqrt(2)); ``` ``` text @@ -113,7 +113,7 @@ SELECT erf(3 / sqrt(2)) cosh(x) ``` -**Параметры** +**Аргументы** - `x` — угол в радианах. Значения из интервала: `-∞ < x < +∞`. [Float64](../../sql-reference/data-types/float.md#float32-float64). @@ -149,7 +149,7 @@ SELECT cosh(0); acosh(x) ``` -**Параметры** +**Аргументы** - `x` — гиперболический косинус угла. Значения из интервала: `1 <= x < +∞`. [Float64](../../sql-reference/data-types/float.md#float32-float64). @@ -189,7 +189,7 @@ SELECT acosh(1); sinh(x) ``` -**Параметры** +**Аргументы** - `x` — угол в радианах. Значения из интервала: `-∞ < x < +∞`. [Float64](../../sql-reference/data-types/float.md#float32-float64). @@ -225,7 +225,7 @@ SELECT sinh(0); asinh(x) ``` -**Параметры** +**Аргументы** - `x` — гиперболический синус угла. Значения из интервала: `-∞ < x < +∞`. [Float64](../../sql-reference/data-types/float.md#float32-float64). @@ -265,7 +265,7 @@ SELECT asinh(0); atanh(x) ``` -**Параметры** +**Аргументы** - `x` — гиперболический тангенс угла. Значения из интервала: `–1 < x < 1`. [Float64](../../sql-reference/data-types/float.md#float32-float64). @@ -301,7 +301,7 @@ SELECT atanh(0); atan2(y, x) ``` -**Параметры** +**Аргументы** - `y` — координата y точки, в которую проведена линия. [Float64](../../sql-reference/data-types/float.md#float32-float64). - `x` — координата х точки, в которую проведена линия. [Float64](../../sql-reference/data-types/float.md#float32-float64). @@ -338,7 +338,7 @@ SELECT atan2(1, 1); hypot(x, y) ``` -**Параметры** +**Аргументы** - `x` — первый катет прямоугольного треугольника. [Float64](../../sql-reference/data-types/float.md#float32-float64). - `y` — второй катет прямоугольного треугольника. [Float64](../../sql-reference/data-types/float.md#float32-float64). @@ -375,7 +375,7 @@ SELECT hypot(1, 1); log1p(x) ``` -**Параметры** +**Аргументы** - `x` — значения из интервала: `-1 < x < +∞`. [Float64](../../sql-reference/data-types/float.md#float32-float64). diff --git a/docs/ru/sql-reference/functions/other-functions.md b/docs/ru/sql-reference/functions/other-functions.md index 19494c0aa10..061824498dc 100644 --- a/docs/ru/sql-reference/functions/other-functions.md +++ b/docs/ru/sql-reference/functions/other-functions.md @@ -16,16 +16,16 @@ toc_title: "\u041f\u0440\u043e\u0447\u0438\u0435\u0020\u0444\u0443\u043d\u043a\u **Синтаксис** ```sql -getMacro(name); +getMacro(name) ``` -**Параметры** +**Аргументы** -- `name` — Имя, которое необходимо получить из секции `macros`. [String](../../sql-reference/data-types/string.md#string). +- `name` — имя, которое необходимо получить из секции `macros`. [String](../../sql-reference/data-types/string.md#string). **Возвращаемое значение** -- Значение по указанному имени. +- Значение по указанному имени. Тип: [String](../../sql-reference/data-types/string.md). @@ -66,7 +66,6 @@ WHERE macro = 'test' └───────┴──────────────┘ ``` - ## FQDN {#fqdn} Возвращает полное имя домена. @@ -74,7 +73,7 @@ WHERE macro = 'test' **Синтаксис** ``` sql -fqdn(); +fqdn() ``` Эта функция регистронезависимая. @@ -93,7 +92,7 @@ fqdn(); SELECT FQDN(); ``` -Ответ: +Результат: ``` text ┌─FQDN()──────────────────────────┐ @@ -109,9 +108,9 @@ SELECT FQDN(); basename( expr ) ``` -**Параметры** +**Аргументы** -- `expr` — Выражение, возвращающее значение типа [String](../../sql-reference/functions/other-functions.md). В результирующем значении все бэкслэши должны быть экранированы. +- `expr` — выражение, возвращающее значение типа [String](../../sql-reference/functions/other-functions.md). В результирующем значении все бэкслэши должны быть экранированы. **Возвращаемое значение** @@ -126,7 +125,7 @@ basename( expr ) **Пример** ``` sql -SELECT 'some/long/path/to/file' AS a, basename(a) +SELECT 'some/long/path/to/file' AS a, basename(a); ``` ``` text @@ -136,7 +135,7 @@ SELECT 'some/long/path/to/file' AS a, basename(a) ``` ``` sql -SELECT 'some\\long\\path\\to\\file' AS a, basename(a) +SELECT 'some\\long\\path\\to\\file' AS a, basename(a); ``` ``` text @@ -146,7 +145,7 @@ SELECT 'some\\long\\path\\to\\file' AS a, basename(a) ``` ``` sql -SELECT 'some-file-name' AS a, basename(a) +SELECT 'some-file-name' AS a, basename(a); ``` ``` text @@ -193,7 +192,7 @@ SELECT visibleWidth(NULL) byteSize(argument [, ...]) ``` -**Параметры** +**Аргументы** - `argument` — значение. @@ -246,7 +245,7 @@ INSERT INTO test VALUES(1, 8, 16, 32, 64, -8, -16, -32, -64, 32.32, 64.64); SELECT key, byteSize(u8) AS `byteSize(UInt8)`, byteSize(u16) AS `byteSize(UInt16)`, byteSize(u32) AS `byteSize(UInt32)`, byteSize(u64) AS `byteSize(UInt64)`, byteSize(i8) AS `byteSize(Int8)`, byteSize(i16) AS `byteSize(Int16)`, byteSize(i32) AS `byteSize(Int32)`, byteSize(i64) AS `byteSize(Int64)`, byteSize(f32) AS `byteSize(Float32)`, byteSize(f64) AS `byteSize(Float64)` FROM test ORDER BY key ASC FORMAT Vertical; ``` -Result: +Результат: ``` text Row 1: @@ -324,7 +323,7 @@ SELECT currentUser(); SELECT currentUser(); ``` -Ответ: +Результат: ``` text ┌─currentUser()─┐ @@ -346,14 +345,14 @@ SELECT currentUser(); isConstant(x) ``` -**Параметры** +**Аргументы** -- `x` — Выражение для проверки. +- `x` — выражение для проверки. **Возвращаемые значения** -- `1` — Выражение `x` является константным. -- `0` — Выражение `x` не является константным. +- `1` — выражение `x` является константным. +- `0` — выражение `x` не является константным. Тип: [UInt8](../data-types/int-uint.md). @@ -362,7 +361,7 @@ isConstant(x) Запрос: ```sql -SELECT isConstant(x + 1) FROM (SELECT 43 AS x) +SELECT isConstant(x + 1) FROM (SELECT 43 AS x); ``` Результат: @@ -376,7 +375,7 @@ SELECT isConstant(x + 1) FROM (SELECT 43 AS x) Запрос: ```sql -WITH 3.14 AS pi SELECT isConstant(cos(pi)) +WITH 3.14 AS pi SELECT isConstant(cos(pi)); ``` Результат: @@ -413,10 +412,10 @@ SELECT isConstant(number) FROM numbers(1) ifNotFinite(x,y) -**Параметры** +**Аргументы** -- `x` — Значение, которое нужно проверить на бесконечность. Тип: [Float\*](../../sql-reference/functions/other-functions.md). -- `y` — Запасное значение. Тип: [Float\*](../../sql-reference/functions/other-functions.md). +- `x` — значение, которое нужно проверить на бесконечность. Тип: [Float\*](../../sql-reference/functions/other-functions.md). +- `y` — запасное значение. Тип: [Float\*](../../sql-reference/functions/other-functions.md). **Возвращаемые значения** @@ -458,7 +457,7 @@ SELECT isConstant(number) FROM numbers(1) `bar(x, min, max, width)` рисует полосу ширины пропорциональной `(x - min)` и равной `width` символов при `x = max`. -Параметры: +Аргументы: - `x` — Величина для отображения. - `min, max` — Целочисленные константы, значение должно помещаться в `Int64`. @@ -675,11 +674,11 @@ neighbor(column, offset[, default_value]) Порядок строк, используемый при вычислении функции `neighbor`, может отличаться от порядка строк, возвращаемых пользователю. Чтобы этого не случилось, вы можете сделать подзапрос с [ORDER BY](../../sql-reference/statements/select/order-by.md) и вызвать функцию изне подзапроса. -**Параметры** +**Аргументы** -- `column` — Имя столбца или скалярное выражение. -- `offset` - Смещение от текущей строки `column`. [Int64](../../sql-reference/functions/other-functions.md). -- `default_value` - Опциональный параметр. Значение, которое будет возвращено, если смещение выходит за пределы блока данных. +- `column` — имя столбца или скалярное выражение. +- `offset` — смещение от текущей строки `column`. [Int64](../../sql-reference/functions/other-functions.md). +- `default_value` — опциональный параметр. Значение, которое будет возвращено, если смещение выходит за пределы блока данных. **Возвращаемое значение** @@ -696,7 +695,7 @@ neighbor(column, offset[, default_value]) SELECT number, neighbor(number, 2) FROM system.numbers LIMIT 10; ``` -Ответ: +Результат: ``` text ┌─number─┬─neighbor(number, 2)─┐ @@ -719,7 +718,7 @@ SELECT number, neighbor(number, 2) FROM system.numbers LIMIT 10; SELECT number, neighbor(number, 2, 999) FROM system.numbers LIMIT 10; ``` -Ответ: +Результат: ``` text ┌─number─┬─neighbor(number, 2, 999)─┐ @@ -750,7 +749,7 @@ SELECT FROM numbers(16) ``` -Ответ: +Результат: ``` text ┌──────month─┬─money─┬─prev_year─┬─year_over_year─┐ @@ -872,9 +871,9 @@ WHERE diff != 1 getSizeOfEnumType(value) ``` -**Параметры** +**Аргументы** -- `value` — Значение типа `Enum`. +- `value` — значение типа `Enum`. **Возвращаемые значения** @@ -901,9 +900,9 @@ SELECT getSizeOfEnumType( CAST('a' AS Enum8('a' = 1, 'b' = 2) ) ) AS x blockSerializedSize(value[, value[, ...]]) ``` -**Параметры** +**Аргументы** -- `value` — Значение произвольного типа. +- `value` — значение произвольного типа. **Возвращаемые значения** @@ -933,9 +932,9 @@ SELECT blockSerializedSize(maxState(1)) as x toColumnTypeName(value) ``` -**Параметры** +**Аргументы** -- `value` — Значение произвольного типа. +- `value` — значение произвольного типа. **Возвращаемые значения** @@ -973,9 +972,9 @@ SELECT toColumnTypeName(CAST('2018-01-01 01:02:03' AS DateTime)) dumpColumnStructure(value) ``` -**Параметры** +**Аргументы** -- `value` — Значение произвольного типа. +- `value` — значение произвольного типа. **Возвращаемые значения** @@ -1003,9 +1002,9 @@ SELECT dumpColumnStructure(CAST('2018-01-01 01:02:03', 'DateTime')) defaultValueOfArgumentType(expression) ``` -**Параметры** +**Аргументы** -- `expression` — Значение произвольного типа или выражение, результатом которого является значение произвольного типа. +- `expression` — значение произвольного типа или выражение, результатом которого является значение произвольного типа. **Возвращаемые значения** @@ -1045,7 +1044,7 @@ SELECT defaultValueOfArgumentType( CAST(1 AS Nullable(Int8) ) ) defaultValueOfTypeName(type) ``` -**Параметры:** +**Аргументы** - `type` — тип данных. @@ -1087,10 +1086,10 @@ SELECT defaultValueOfTypeName('Nullable(Int8)') SELECT replicate(x, arr); ``` -**Параметры** +**Аргументы** -- `arr` — Исходный массив. ClickHouse создаёт новый массив такой же длины как исходный и заполняет его значением `x`. -- `x` — Значение, которым будет заполнен результирующий массив. +- `arr` — исходный массив. ClickHouse создаёт новый массив такой же длины как исходный и заполняет его значением `x`. +- `x` — значение, которым будет заполнен результирующий массив. **Возвращаемое значение** @@ -1170,7 +1169,7 @@ filesystemFree() SELECT formatReadableSize(filesystemFree()) AS "Free space", toTypeName(filesystemFree()) AS "Type"; ``` -Ответ: +Результат: ``` text ┌─Free space─┬─Type───┐ @@ -1202,7 +1201,7 @@ filesystemCapacity() SELECT formatReadableSize(filesystemCapacity()) AS "Capacity", toTypeName(filesystemCapacity()) AS "Type" ``` -Ответ: +Результат: ``` text ┌─Capacity──┬─Type───┐ @@ -1220,7 +1219,7 @@ SELECT formatReadableSize(filesystemCapacity()) AS "Capacity", toTypeName(filesy finalizeAggregation(state) ``` -**Параметры** +**Аргументы** - `state` — состояние агрегатной функции. [AggregateFunction](../../sql-reference/data-types/aggregatefunction.md#data-type-aggregatefunction). @@ -1321,17 +1320,17 @@ FROM numbers(10); **Синтаксис** ```sql -runningAccumulate(agg_state[, grouping]); +runningAccumulate(agg_state[, grouping]) ``` -**Параметры** +**Аргументы** -- `agg_state` — Состояние агрегатной функции. [AggregateFunction](../../sql-reference/data-types/aggregatefunction.md#data-type-aggregatefunction). -- `grouping` — Ключ группировки. Опциональный параметр. Состояние функции обнуляется, если значение `grouping` меняется. Параметр может быть любого [поддерживаемого типа данных](../../sql-reference/data-types/index.md), для которого определен оператор равенства. +- `agg_state` — состояние агрегатной функции. [AggregateFunction](../../sql-reference/data-types/aggregatefunction.md#data-type-aggregatefunction). +- `grouping` — ключ группировки. Опциональный параметр. Состояние функции обнуляется, если значение `grouping` меняется. Параметр может быть любого [поддерживаемого типа данных](../../sql-reference/data-types/index.md), для которого определен оператор равенства. **Возвращаемое значение** -- Каждая результирующая строка содержит результат агрегатной функции, накопленный для всех входных строк от 0 до текущей позиции. `runningAccumulate` обнуляет состояния для каждого нового блока данных или при изменении значения `grouping`. +- Каждая результирующая строка содержит результат агрегатной функции, накопленный для всех входных строк от 0 до текущей позиции. `runningAccumulate` обнуляет состояния для каждого нового блока данных или при изменении значения `grouping`. Тип зависит от используемой агрегатной функции. @@ -1430,7 +1429,7 @@ FROM joinGet(join_storage_table_name, `value_column`, join_keys) ``` -**Параметры** +**Аргументы** - `join_storage_table_name` — [идентификатор](../syntax.md#syntax-identifiers), который указывает, откуда производится выборка данных. Поиск по идентификатору осуществляется в базе данных по умолчанию (см. конфигурацию `default_database`). Чтобы переопределить базу данных по умолчанию, используйте команду `USE db_name`, или укажите базу данных и таблицу через разделитель `db_name.db_table`, см. пример. - `value_column` — столбец, из которого нужно произвести выборку данных. @@ -1535,9 +1534,9 @@ SELECT identity(42) randomPrintableASCII(length) ``` -**Параметры** +**Аргументы** -- `length` — Длина результирующей строки. Положительное целое число. +- `length` — длина результирующей строки. Положительное целое число. Если передать `length < 0`, то поведение функции не определено. @@ -1571,7 +1570,7 @@ SELECT number, randomPrintableASCII(30) as str, length(str) FROM system.numbers randomString(length) ``` -**Параметры** +**Аргументы** - `length` — длина строки. Положительное целое число. @@ -1619,11 +1618,11 @@ len: 30 randomFixedString(length); ``` -**Параметры** +**Аргументы** -- `length` — Длина строки в байтах. [UInt64](../../sql-reference/data-types/int-uint.md). +- `length` — длина строки в байтах. [UInt64](../../sql-reference/data-types/int-uint.md). -**Returned value(s)** +**Возвращаемое значение** - Строка, заполненная случайными байтами. @@ -1653,12 +1652,12 @@ SELECT randomFixedString(13) as rnd, toTypeName(rnd) **Синтаксис** ``` sql -randomStringUTF8(length); +randomStringUTF8(length) ``` -**Параметры** +**Аргументы** -- `length` — Длина итоговой строки в кодовых точках. [UInt64](../../sql-reference/data-types/int-uint.md). +- `length` — длина итоговой строки в кодовых точках. [UInt64](../../sql-reference/data-types/int-uint.md). **Возвращаемое значение** @@ -1690,7 +1689,7 @@ SELECT randomStringUTF8(13) **Синтаксис** ```sql -getSetting('custom_setting'); +getSetting('custom_setting') ``` **Параметр** @@ -1728,7 +1727,7 @@ SELECT getSetting('custom_a'); isDecimalOverflow(d, [p]) ``` -**Параметры** +**Аргументы** - `d` — число. [Decimal](../../sql-reference/data-types/decimal.md). - `p` — точность. Необязательный параметр. Если опущен, используется исходная точность первого аргумента. Использование этого параметра может быть полезно для извлечения данных в другую СУБД или файл. [UInt8](../../sql-reference/data-types/int-uint.md#uint-ranges). @@ -1765,7 +1764,7 @@ SELECT isDecimalOverflow(toDecimal32(1000000000, 0), 9), countDigits(x) ``` -**Параметры** +**Аргументы** - `x` — [целое](../../sql-reference/data-types/int-uint.md#uint8-uint16-uint32-uint64-int8-int16-int32-int64) или [дробное](../../sql-reference/data-types/decimal.md) число. @@ -1824,7 +1823,7 @@ UNSUPPORTED_METHOD tcpPort() ``` -**Параметры** +**Аргументы** - Нет. diff --git a/docs/ru/sql-reference/functions/random-functions.md b/docs/ru/sql-reference/functions/random-functions.md index f3889504fa6..0bbd46c6018 100644 --- a/docs/ru/sql-reference/functions/random-functions.md +++ b/docs/ru/sql-reference/functions/random-functions.md @@ -31,9 +31,9 @@ toc_title: "\u0424\u0443\u043d\u043a\u0446\u0438\u0438\u0020\u0433\u0435\u043d\u randConstant([x]) ``` -**Параметры** +**Аргументы** -- `x` — [Выражение](../syntax.md#syntax-expressions), возвращающее значение одного из [поддерживаемых типов данных](../data-types/index.md#data_types). Значение используется, чтобы избежать [склейки одинаковых выражений](index.md#common-subexpression-elimination), если функция вызывается несколько раз в одном запросе. Необязательный параметр. +- `x` — [выражение](../syntax.md#syntax-expressions), возвращающее значение одного из [поддерживаемых типов данных](../data-types/index.md#data_types). Значение используется, чтобы избежать [склейки одинаковых выражений](index.md#common-subexpression-elimination), если функция вызывается несколько раз в одном запросе. Необязательный параметр. **Возвращаемое значение** @@ -79,7 +79,7 @@ fuzzBits([s], [prob]) ``` Инвертирует каждый бит `s` с вероятностью `prob`. -**Параметры** +**Аргументы** - `s` — `String` or `FixedString` - `prob` — constant `Float32/64` diff --git a/docs/ru/sql-reference/functions/rounding-functions.md b/docs/ru/sql-reference/functions/rounding-functions.md index 78033160396..cfbc8b46960 100644 --- a/docs/ru/sql-reference/functions/rounding-functions.md +++ b/docs/ru/sql-reference/functions/rounding-functions.md @@ -33,10 +33,10 @@ N может быть отрицательным. round(expression [, decimal_places]) ``` -**Параметры:** +**Аргументы** -- `expression` — Число для округления. Может быть любым [выражением](../syntax.md#syntax-expressions), возвращающим числовой [тип данных](../../sql-reference/functions/rounding-functions.md#data_types). -- `decimal-places` — Целое значение. +- `expression` — число для округления. Может быть любым [выражением](../syntax.md#syntax-expressions), возвращающим числовой [тип данных](../../sql-reference/functions/rounding-functions.md#data_types). +- `decimal-places` — целое значение. - Если `decimal-places > 0`, то функция округляет значение справа от запятой. - Если `decimal-places < 0` то функция округляет значение слева от запятой. - Если `decimal-places = 0`, то функция округляет значение до целого. В этом случае аргумент можно опустить. @@ -112,13 +112,13 @@ round(3.65, 1) = 3.6 roundBankers(expression [, decimal_places]) ``` -**Параметры** +**Аргументы** -- `expression` — Число для округления. Может быть любым [выражением](../syntax.md#syntax-expressions), возвращающим числовой [тип данных](../../sql-reference/functions/rounding-functions.md#data_types). -- `decimal-places` — Десятичный разряд. Целое число. - - `decimal-places > 0` — Функция округляет значение выражения до ближайшего чётного числа на соответствующей позиции справа от запятой. Например, `roundBankers(3.55, 1) = 3.6`. - - `decimal-places < 0` — Функция округляет значение выражения до ближайшего чётного числа на соответствующей позиции слева от запятой. Например, `roundBankers(24.55, -1) = 20`. - - `decimal-places = 0` — Функция округляет значение до целого. В этом случае аргумент можно не передавать. Например, `roundBankers(2.5) = 2`. +- `expression` — число для округления. Может быть любым [выражением](../syntax.md#syntax-expressions), возвращающим числовой [тип данных](../../sql-reference/functions/rounding-functions.md#data_types). +- `decimal-places` — десятичный разряд. Целое число. + - `decimal-places > 0` — функция округляет значение выражения до ближайшего чётного числа на соответствующей позиции справа от запятой. Например, `roundBankers(3.55, 1) = 3.6`. + - `decimal-places < 0` — функция округляет значение выражения до ближайшего чётного числа на соответствующей позиции слева от запятой. Например, `roundBankers(24.55, -1) = 20`. + - `decimal-places = 0` — функция округляет значение до целого. В этом случае аргумент можно не передавать. Например, `roundBankers(2.5) = 2`. **Возвращаемое значение** diff --git a/docs/ru/sql-reference/functions/splitting-merging-functions.md b/docs/ru/sql-reference/functions/splitting-merging-functions.md index d451eabc407..8f8fd9925b2 100644 --- a/docs/ru/sql-reference/functions/splitting-merging-functions.md +++ b/docs/ru/sql-reference/functions/splitting-merging-functions.md @@ -17,10 +17,10 @@ separator должен быть константной строкой из ро splitByChar(, ) ``` -**Параметры** +**Аргументы** -- `separator` — Разделитель, состоящий из одного символа. [String](../../sql-reference/data-types/string.md). -- `s` — Разбиваемая строка. [String](../../sql-reference/data-types/string.md). +- `separator` — разделитель, состоящий из одного символа. [String](../../sql-reference/data-types/string.md). +- `s` — разбиваемая строка. [String](../../sql-reference/data-types/string.md). **Возвращаемые значения** @@ -54,10 +54,10 @@ SELECT splitByChar(',', '1,2,3,abcde') splitByString(separator, s) ``` -**Параметры** +**Аргументы** -- `separator` — Разделитель. [String](../../sql-reference/data-types/string.md). -- `s` — Разбиваемая строка. [String](../../sql-reference/data-types/string.md). +- `separator` — разделитель. [String](../../sql-reference/data-types/string.md). +- `s` — разбиваемая строка. [String](../../sql-reference/data-types/string.md). **Возвращаемые значения** @@ -67,7 +67,7 @@ splitByString(separator, s) - Задано несколько последовательных разделителей; - Исходная строка `s` пуста. -Type: [Array](../../sql-reference/data-types/array.md) of [String](../../sql-reference/data-types/string.md). +Тип: [Array](../../sql-reference/data-types/array.md) of [String](../../sql-reference/data-types/string.md). **Примеры** diff --git a/docs/ru/sql-reference/functions/string-functions.md b/docs/ru/sql-reference/functions/string-functions.md index 1159a1f5823..8c794a920e3 100644 --- a/docs/ru/sql-reference/functions/string-functions.md +++ b/docs/ru/sql-reference/functions/string-functions.md @@ -70,19 +70,19 @@ toc_title: "\u0424\u0443\u043d\u043a\u0446\u0438\u0438\u0020\u0434\u043b\u044f\u Заменяет некорректные символы UTF-8 на символ `�` (U+FFFD). Все идущие подряд некорректные символы схлопываются в один заменяющий символ. ``` sql -toValidUTF8( input_string ) +toValidUTF8(input_string) ``` -Параметры: +**Аргументы** -- input_string — произвольный набор байтов, представленный как объект типа [String](../../sql-reference/functions/string-functions.md). +- `input_string` — произвольный набор байтов, представленный как объект типа [String](../../sql-reference/functions/string-functions.md). Возвращаемое значение: Корректная строка UTF-8. **Пример** ``` sql -SELECT toValidUTF8('\x61\xF0\x80\x80\x80b') +SELECT toValidUTF8('\x61\xF0\x80\x80\x80b'); ``` ``` text @@ -103,10 +103,10 @@ SELECT toValidUTF8('\x61\xF0\x80\x80\x80b') repeat(s, n) ``` -**Параметры** +**Аргументы** -- `s` — Строка для повторения. [String](../../sql-reference/functions/string-functions.md). -- `n` — Количество повторов. [UInt](../../sql-reference/functions/string-functions.md). +- `s` — строка для повторения. [String](../../sql-reference/functions/string-functions.md). +- `n` — количество повторов. [UInt](../../sql-reference/functions/string-functions.md). **Возвращаемое значение** @@ -119,10 +119,10 @@ repeat(s, n) Запрос: ``` sql -SELECT repeat('abc', 10) +SELECT repeat('abc', 10); ``` -Ответ: +Результат: ``` text ┌─repeat('abc', 10)──────────────┐ @@ -172,7 +172,7 @@ SELECT format('{} {}', 'Hello', 'World') concat(s1, s2, ...) ``` -**Параметры** +**Аргументы** Значения типа String или FixedString. @@ -187,10 +187,10 @@ concat(s1, s2, ...) Запрос: ``` sql -SELECT concat('Hello, ', 'World!') +SELECT concat('Hello, ', 'World!'); ``` -Ответ: +Результат: ``` text ┌─concat('Hello, ', 'World!')─┐ @@ -210,7 +210,7 @@ SELECT concat('Hello, ', 'World!') concatAssumeInjective(s1, s2, ...) ``` -**Параметры** +**Аргументы** Значения типа String или FixedString. @@ -242,10 +242,10 @@ SELECT * from key_val Запрос: ``` sql -SELECT concat(key1, key2), sum(value) FROM key_val GROUP BY (key1, key2) +SELECT concat(key1, key2), sum(value) FROM key_val GROUP BY (key1, key2); ``` -Ответ: +Результат: ``` text ┌─concat(key1, key2)─┬─sum(value)─┐ @@ -312,7 +312,7 @@ SELECT startsWith('Spider-Man', 'Spi'); SELECT startsWith('Hello, world!', 'He'); ``` -Ответ: +Результат: ``` text ┌─startsWith('Hello, world!', 'He')─┐ @@ -331,7 +331,7 @@ SELECT startsWith('Hello, world!', 'He'); trim([[LEADING|TRAILING|BOTH] trim_character FROM] input_string) ``` -**Параметры** +**Аргументы** - `trim_character` — один или несколько символов, подлежащие удалению. [String](../../sql-reference/functions/string-functions.md). - `input_string` — строка для обрезки. [String](../../sql-reference/functions/string-functions.md). @@ -347,10 +347,10 @@ trim([[LEADING|TRAILING|BOTH] trim_character FROM] input_string) Запрос: ``` sql -SELECT trim(BOTH ' ()' FROM '( Hello, world! )') +SELECT trim(BOTH ' ()' FROM '( Hello, world! )'); ``` -Ответ: +Результат: ``` text ┌─trim(BOTH ' ()' FROM '( Hello, world! )')─┐ @@ -370,7 +370,7 @@ trimLeft(input_string) Алиас: `ltrim(input_string)`. -**Параметры** +**Аргументы** - `input_string` — строка для обрезки. [String](../../sql-reference/functions/string-functions.md). @@ -385,10 +385,10 @@ trimLeft(input_string) Запрос: ``` sql -SELECT trimLeft(' Hello, world! ') +SELECT trimLeft(' Hello, world! '); ``` -Ответ: +Результат: ``` text ┌─trimLeft(' Hello, world! ')─┐ @@ -408,7 +408,7 @@ trimRight(input_string) Алиас: `rtrim(input_string)`. -**Параметры** +**Аргументы** - `input_string` — строка для обрезки. [String](../../sql-reference/functions/string-functions.md). @@ -423,10 +423,10 @@ trimRight(input_string) Запрос: ``` sql -SELECT trimRight(' Hello, world! ') +SELECT trimRight(' Hello, world! '); ``` -Ответ: +Результат: ``` text ┌─trimRight(' Hello, world! ')─┐ @@ -446,7 +446,7 @@ trimBoth(input_string) Алиас: `trim(input_string)`. -**Параметры** +**Аргументы** - `input_string` — строка для обрезки. [String](../../sql-reference/functions/string-functions.md). @@ -461,10 +461,10 @@ trimBoth(input_string) Запрос: ``` sql -SELECT trimBoth(' Hello, world! ') +SELECT trimBoth(' Hello, world! '); ``` -Ответ: +Результат: ``` text ┌─trimBoth(' Hello, world! ')─┐ @@ -494,14 +494,15 @@ SELECT trimBoth(' Hello, world! ') Заменяет литералы, последовательности литералов и сложные псевдонимы заполнителями. -**Синтаксис** +**Синтаксис** + ``` sql normalizeQuery(x) ``` -**Параметры** +**Аргументы** -- `x` — Последовательность символов. [String](../../sql-reference/data-types/string.md). +- `x` — последовательность символов. [String](../../sql-reference/data-types/string.md). **Возвращаемое значение** @@ -535,9 +536,9 @@ SELECT normalizeQuery('[1, 2, 3, x]') AS query; normalizedQueryHash(x) ``` -**Параметры** +**Аргументы** -- `x` — Последовательность символов. [String](../../sql-reference/data-types/string.md). +- `x` — последовательность символов. [String](../../sql-reference/data-types/string.md). **Возвращаемое значение** @@ -573,7 +574,7 @@ SELECT normalizedQueryHash('SELECT 1 AS `xyz`') != normalizedQueryHash('SELECT 1 encodeXMLComponent(x) ``` -**Параметры** +**Аргументы** - `x` — последовательность символов. [String](../../sql-reference/data-types/string.md). @@ -603,7 +604,6 @@ Hello, "world"! 'foo' ``` - ## decodeXMLComponent {#decode-xml-component} Заменяет символами предопределенные мнемоники XML: `"` `&` `'` `>` `<` @@ -615,7 +615,7 @@ Hello, "world"! decodeXMLComponent(x) ``` -**Параметры** +**Аргументы** - `x` — последовательность символов. [String](../../sql-reference/data-types/string.md). diff --git a/docs/ru/sql-reference/functions/string-search-functions.md b/docs/ru/sql-reference/functions/string-search-functions.md index b7193da6f33..509e28a009e 100644 --- a/docs/ru/sql-reference/functions/string-search-functions.md +++ b/docs/ru/sql-reference/functions/string-search-functions.md @@ -23,11 +23,11 @@ position(haystack, needle[, start_pos]) Алиас: `locate(haystack, needle[, start_pos])`. -**Параметры** +**Аргументы** - `haystack` — строка, по которой выполняется поиск. [Строка](../syntax.md#syntax-string-literal). - `needle` — подстрока, которую необходимо найти. [Строка](../syntax.md#syntax-string-literal). -- `start_pos` – Опциональный параметр, позиция символа в строке, с которого начинается поиск. [UInt](../../sql-reference/data-types/int-uint.md) +- `start_pos` — опциональный параметр, позиция символа в строке, с которого начинается поиск. [UInt](../../sql-reference/data-types/int-uint.md). **Возвращаемые значения** @@ -43,10 +43,10 @@ position(haystack, needle[, start_pos]) Запрос: ``` sql -SELECT position('Hello, world!', '!') +SELECT position('Hello, world!', '!'); ``` -Ответ: +Результат: ``` text ┌─position('Hello, world!', '!')─┐ @@ -59,10 +59,10 @@ SELECT position('Hello, world!', '!') Запрос: ``` sql -SELECT position('Привет, мир!', '!') +SELECT position('Привет, мир!', '!'); ``` -Ответ: +Результат: ``` text ┌─position('Привет, мир!', '!')─┐ @@ -82,11 +82,11 @@ SELECT position('Привет, мир!', '!') positionCaseInsensitive(haystack, needle[, start_pos]) ``` -**Параметры** +**Аргументы** - `haystack` — строка, по которой выполняется поиск. [Строка](../syntax.md#syntax-string-literal). - `needle` — подстрока, которую необходимо найти. [Строка](../syntax.md#syntax-string-literal). -- `start_pos` – Опциональный параметр, позиция символа в строке, с которого начинается поиск. [UInt](../../sql-reference/data-types/int-uint.md) +- `start_pos` — опциональный параметр, позиция символа в строке, с которого начинается поиск. [UInt](../../sql-reference/data-types/int-uint.md). **Возвращаемые значения** @@ -100,10 +100,10 @@ positionCaseInsensitive(haystack, needle[, start_pos]) Запрос: ``` sql -SELECT positionCaseInsensitive('Hello, world!', 'hello') +SELECT positionCaseInsensitive('Hello, world!', 'hello'); ``` -Ответ: +Результат: ``` text ┌─positionCaseInsensitive('Hello, world!', 'hello')─┐ @@ -125,11 +125,11 @@ SELECT positionCaseInsensitive('Hello, world!', 'hello') positionUTF8(haystack, needle[, start_pos]) ``` -**Параметры** +**Аргументы** - `haystack` — строка, по которой выполняется поиск. [Строка](../syntax.md#syntax-string-literal). - `needle` — подстрока, которую необходимо найти. [Строка](../syntax.md#syntax-string-literal). -- `start_pos` – Опциональный параметр, позиция символа в строке, с которого начинается поиск. [UInt](../../sql-reference/data-types/int-uint.md) +- `start_pos` — опциональный параметр, позиция символа в строке, с которого начинается поиск. [UInt](../../sql-reference/data-types/int-uint.md). **Возвращаемые значения** @@ -145,10 +145,10 @@ positionUTF8(haystack, needle[, start_pos]) Запрос: ``` sql -SELECT positionUTF8('Привет, мир!', '!') +SELECT positionUTF8('Привет, мир!', '!'); ``` -Ответ: +Результат: ``` text ┌─positionUTF8('Привет, мир!', '!')─┐ @@ -161,7 +161,7 @@ SELECT positionUTF8('Привет, мир!', '!') Запрос для символа `é`, который представлен одной кодовой точкой `U+00E9`: ``` sql -SELECT positionUTF8('Salut, étudiante!', '!') +SELECT positionUTF8('Salut, étudiante!', '!'); ``` Result: @@ -175,10 +175,10 @@ Result: Запрос для символа `é`, который представлен двумя кодовыми точками `U+0065U+0301`: ``` sql -SELECT positionUTF8('Salut, étudiante!', '!') +SELECT positionUTF8('Salut, étudiante!', '!'); ``` -Ответ: +Результат: ``` text ┌─positionUTF8('Salut, étudiante!', '!')─┐ @@ -198,11 +198,11 @@ SELECT positionUTF8('Salut, étudiante!', '!') positionCaseInsensitiveUTF8(haystack, needle[, start_pos]) ``` -**Параметры** +**Аргументы** - `haystack` — строка, по которой выполняется поиск. [Строка](../syntax.md#syntax-string-literal). - `needle` — подстрока, которую необходимо найти. [Строка](../syntax.md#syntax-string-literal). -- `start_pos` – Опциональный параметр, позиция символа в строке, с которого начинается поиск. [UInt](../../sql-reference/data-types/int-uint.md) +- `start_pos` — опциональный параметр, позиция символа в строке, с которого начинается поиск. [UInt](../../sql-reference/data-types/int-uint.md). **Возвращаемые значения** @@ -216,10 +216,10 @@ positionCaseInsensitiveUTF8(haystack, needle[, start_pos]) Запрос: ``` sql -SELECT positionCaseInsensitiveUTF8('Привет, мир!', 'Мир') +SELECT positionCaseInsensitiveUTF8('Привет, мир!', 'Мир'); ``` -Ответ: +Результат: ``` text ┌─positionCaseInsensitiveUTF8('Привет, мир!', 'Мир')─┐ @@ -257,7 +257,7 @@ multiSearchAllPositions(haystack, [needle1, needle2, ..., needlen]) Query: ``` sql -SELECT multiSearchAllPositions('Hello, World!', ['hello', '!', 'world']) +SELECT multiSearchAllPositions('Hello, World!', ['hello', '!', 'world']); ``` Result: @@ -357,7 +357,7 @@ Result: extractAllGroupsHorizontal(haystack, pattern) ``` -**Параметры** +**Аргументы** - `haystack` — строка для разбора. Тип: [String](../../sql-reference/data-types/string.md). - `pattern` — регулярное выражение, построенное по синтаксическим правилам [re2](https://github.com/google/re2/wiki/Syntax). Выражение должно содержать группы, заключенные в круглые скобки. Если выражение не содержит групп, генерируется исключение. Тип: [String](../../sql-reference/data-types/string.md). @@ -373,7 +373,7 @@ extractAllGroupsHorizontal(haystack, pattern) Запрос: ``` sql -SELECT extractAllGroupsHorizontal('abc=111, def=222, ghi=333', '("[^"]+"|\\w+)=("[^"]+"|\\w+)') +SELECT extractAllGroupsHorizontal('abc=111, def=222, ghi=333', '("[^"]+"|\\w+)=("[^"]+"|\\w+)'); ``` Результат: @@ -384,8 +384,9 @@ SELECT extractAllGroupsHorizontal('abc=111, def=222, ghi=333', '("[^"]+"|\\w+)=( └──────────────────────────────────────────────────────────────────────────────────────────┘ ``` -**См. также** -- функция [extractAllGroupsVertical](#extractallgroups-vertical) +**Смотрите также** + +- Функция [extractAllGroupsVertical](#extractallgroups-vertical) ## extractAllGroupsVertical {#extractallgroups-vertical} @@ -397,7 +398,7 @@ SELECT extractAllGroupsHorizontal('abc=111, def=222, ghi=333', '("[^"]+"|\\w+)=( extractAllGroupsVertical(haystack, pattern) ``` -**Параметры** +**Аргументы** - `haystack` — строка для разбора. Тип: [String](../../sql-reference/data-types/string.md). - `pattern` — регулярное выражение, построенное по синтаксическим правилам [re2](https://github.com/google/re2/wiki/Syntax). Выражение должно содержать группы, заключенные в круглые скобки. Если выражение не содержит групп, генерируется исключение. Тип: [String](../../sql-reference/data-types/string.md). @@ -413,7 +414,7 @@ extractAllGroupsVertical(haystack, pattern) Запрос: ``` sql -SELECT extractAllGroupsVertical('abc=111, def=222, ghi=333', '("[^"]+"|\\w+)=("[^"]+"|\\w+)') +SELECT extractAllGroupsVertical('abc=111, def=222, ghi=333', '("[^"]+"|\\w+)=("[^"]+"|\\w+)'); ``` Результат: @@ -424,8 +425,9 @@ SELECT extractAllGroupsVertical('abc=111, def=222, ghi=333', '("[^"]+"|\\w+)=("[ └────────────────────────────────────────────────────────────────────────────────────────┘ ``` -**См. также** -- функция [extractAllGroupsHorizontal](#extractallgroups-horizontal) +**Смотрите также** + +- Функция [extractAllGroupsHorizontal](#extractallgroups-horizontal) ## like(haystack, pattern), оператор haystack LIKE pattern {#function-like} @@ -455,10 +457,10 @@ SELECT extractAllGroupsVertical('abc=111, def=222, ghi=333', '("[^"]+"|\\w+)=("[ ilike(haystack, pattern) ``` -**Параметры** +**Аргументы** -- `haystack` — Входная строка. [String](../../sql-reference/syntax.md#syntax-string-literal). -- `pattern` — Если `pattern` не содержит процента или нижнего подчеркивания, тогда `pattern` представляет саму строку. Нижнее подчеркивание (`_`) в `pattern` обозначает любой отдельный символ. Знак процента (`%`) соответствует последовательности из любого количества символов: от нуля и более. +- `haystack` — входная строка. [String](../../sql-reference/syntax.md#syntax-string-literal). +- `pattern` — если `pattern` не содержит процента или нижнего подчеркивания, тогда `pattern` представляет саму строку. Нижнее подчеркивание (`_`) в `pattern` обозначает любой отдельный символ. Знак процента (`%`) соответствует последовательности из любого количества символов: от нуля и более. Некоторые примеры `pattern`: @@ -490,7 +492,7 @@ ilike(haystack, pattern) Запрос: ``` sql -SELECT * FROM Months WHERE ilike(name, '%j%') +SELECT * FROM Months WHERE ilike(name, '%j%'); ``` Результат: @@ -530,7 +532,7 @@ SELECT * FROM Months WHERE ilike(name, '%j%') countMatches(haystack, pattern) ``` -**Параметры** +**Аргументы** - `haystack` — строка, по которой выполняется поиск. [String](../../sql-reference/syntax.md#syntax-string-literal). - `pattern` — регулярное выражение, построенное по синтаксическим правилам [re2](https://github.com/google/re2/wiki/Syntax). [String](../../sql-reference/data-types/string.md). @@ -583,11 +585,11 @@ SELECT countMatches('aaaa', 'aa'); countSubstrings(haystack, needle[, start_pos]) ``` -**Параметры** +**Аргументы** - `haystack` — строка, в которой ведется поиск. [String](../../sql-reference/syntax.md#syntax-string-literal). - `needle` — искомая подстрока. [String](../../sql-reference/syntax.md#syntax-string-literal). -- `start_pos` – позиция первого символа в строке, с которого начнется поиск. Необязательный параметр. [UInt](../../sql-reference/data-types/int-uint.md). +- `start_pos` — позиция первого символа в строке, с которого начнется поиск. Необязательный параметр. [UInt](../../sql-reference/data-types/int-uint.md). **Возвращаемые значения** @@ -649,11 +651,11 @@ SELECT countSubstrings('abc___abc', 'abc', 4); countSubstringsCaseInsensitive(haystack, needle[, start_pos]) ``` -**Параметры** +**Аргументы** - `haystack` — строка, в которой ведется поиск. [String](../../sql-reference/syntax.md#syntax-string-literal). - `needle` — искомая подстрока. [String](../../sql-reference/syntax.md#syntax-string-literal). -- `start_pos` – позиция первого символа в строке, с которого начнется поиск. Необязательный параметр. [UInt](../../sql-reference/data-types/int-uint.md). +- `start_pos` — позиция первого символа в строке, с которого начнется поиск. Необязательный параметр. [UInt](../../sql-reference/data-types/int-uint.md). **Возвращаемые значения** @@ -715,11 +717,11 @@ SELECT countSubstringsCaseInsensitive('abC___abC', 'aBc', 2); SELECT countSubstringsCaseInsensitiveUTF8(haystack, needle[, start_pos]) ``` -**Параметры** +**Аргументы** - `haystack` — строка, в которой ведется поиск. [String](../../sql-reference/syntax.md#syntax-string-literal). - `needle` — искомая подстрока. [String](../../sql-reference/syntax.md#syntax-string-literal). -- `start_pos` – позиция первого символа в строке, с которого начнется поиск. Необязательный параметр. [UInt](../../sql-reference/data-types/int-uint.md). +- `start_pos` — позиция первого символа в строке, с которого начнется поиск. Необязательный параметр. [UInt](../../sql-reference/data-types/int-uint.md). **Возвращаемые значения** diff --git a/docs/ru/sql-reference/functions/tuple-functions.md b/docs/ru/sql-reference/functions/tuple-functions.md index f88886ec6f1..244998b15d0 100644 --- a/docs/ru/sql-reference/functions/tuple-functions.md +++ b/docs/ru/sql-reference/functions/tuple-functions.md @@ -45,9 +45,9 @@ untuple(x) Чтобы пропустить некоторые столбцы в результате запроса, вы можете использовать выражение `EXCEPT`. -**Параметры** +**Аргументы** -- `x` - функция `tuple`, столбец или кортеж элементов. [Tuple](../../sql-reference/data-types/tuple.md). +- `x` — функция `tuple`, столбец или кортеж элементов. [Tuple](../../sql-reference/data-types/tuple.md). **Возвращаемое значение** diff --git a/docs/ru/sql-reference/functions/tuple-map-functions.md b/docs/ru/sql-reference/functions/tuple-map-functions.md index 696fdb9e5ae..4d36f2ef18b 100644 --- a/docs/ru/sql-reference/functions/tuple-map-functions.md +++ b/docs/ru/sql-reference/functions/tuple-map-functions.md @@ -15,7 +15,7 @@ toc_title: Работа с контейнерами map map(key1, value1[, key2, value2, ...]) ``` -**Параметры** +**Аргументы** - `key` — ключ. [String](../../sql-reference/data-types/string.md) или [Integer](../../sql-reference/data-types/int-uint.md). - `value` — значение. [String](../../sql-reference/data-types/string.md), [Integer](../../sql-reference/data-types/int-uint.md) или [Array](../../sql-reference/data-types/array.md). @@ -62,9 +62,10 @@ SELECT a['key2'] FROM table_map; └─────────────────────────┘ ``` -**См. также** +**Смотрите также** - тип данных [Map(key, value)](../../sql-reference/data-types/map.md) + ## mapAdd {#function-mapadd} Собирает все ключи и суммирует соответствующие значения. @@ -75,7 +76,7 @@ SELECT a['key2'] FROM table_map; mapAdd(Tuple(Array, Array), Tuple(Array, Array) [, ...]) ``` -**Параметры** +**Аргументы** Аргументами являются [кортежи](../../sql-reference/data-types/tuple.md#tuplet1-t2) из двух [массивов](../../sql-reference/data-types/array.md#data-type-array), где элементы в первом массиве представляют ключи, а второй массив содержит значения для каждого ключа. Все массивы ключей должны иметь один и тот же тип, а все массивы значений должны содержать элементы, которые можно приводить к одному типу ([Int64](../../sql-reference/data-types/int-uint.md#int-ranges), [UInt64](../../sql-reference/data-types/int-uint.md#uint-ranges) или [Float64](../../sql-reference/data-types/float.md#float32-float64)). @@ -111,7 +112,7 @@ SELECT mapAdd(([toUInt8(1), 2], [1, 1]), ([toUInt8(1), 2], [1, 1])) as res, toTy mapSubtract(Tuple(Array, Array), Tuple(Array, Array) [, ...]) ``` -**Параметры** +**Аргументы** Аргументами являются [кортежи](../../sql-reference/data-types/tuple.md#tuplet1-t2) из двух [массивов](../../sql-reference/data-types/array.md#data-type-array), где элементы в первом массиве представляют ключи, а второй массив содержит значения для каждого ключа. Все массивы ключей должны иметь один и тот же тип, а все массивы значений должны содержать элементы, которые можно приводить к одному типу ([Int64](../../sql-reference/data-types/int-uint.md#int-ranges), [UInt64](../../sql-reference/data-types/int-uint.md#uint-ranges) или [Float64](../../sql-reference/data-types/float.md#float32-float64)). @@ -151,10 +152,10 @@ mapPopulateSeries(keys, values[, max]) Количество элементов в `keys` и `values` должно быть одинаковым для каждой строки. -**Параметры** +**Аргументы** -- `keys` — Массив ключей [Array](../../sql-reference/data-types/array.md#data-type-array)([Int](../../sql-reference/data-types/int-uint.md#int-ranges)). -- `values` — Массив значений. [Array](../../sql-reference/data-types/array.md#data-type-array)([Int](../../sql-reference/data-types/int-uint.md#int-ranges)). +- `keys` — массив ключей [Array](../../sql-reference/data-types/array.md#data-type-array)([Int](../../sql-reference/data-types/int-uint.md#int-ranges)). +- `values` — массив значений. [Array](../../sql-reference/data-types/array.md#data-type-array)([Int](../../sql-reference/data-types/int-uint.md#int-ranges)). **Возвращаемое значение** @@ -186,7 +187,7 @@ select mapPopulateSeries([1,2,4], [11,22,44], 5) as res, toTypeName(res) as type mapContains(map, key) ``` -**Параметры** +**Аргументы** - `map` — контейнер Map. [Map](../../sql-reference/data-types/map.md). - `key` — ключ. Тип соответстует типу ключей параметра `map`. @@ -229,7 +230,7 @@ SELECT mapContains(a, 'name') FROM test; mapKeys(map) ``` -**Параметры** +**Аргументы** - `map` — контейнер Map. [Map](../../sql-reference/data-types/map.md). @@ -270,7 +271,7 @@ SELECT mapKeys(a) FROM test; mapKeys(map) ``` -**Параметры** +**Аргументы** - `map` — контейнер Map. [Map](../../sql-reference/data-types/map.md). diff --git a/docs/ru/sql-reference/functions/type-conversion-functions.md b/docs/ru/sql-reference/functions/type-conversion-functions.md index 53e7bc1300e..7fea524a233 100644 --- a/docs/ru/sql-reference/functions/type-conversion-functions.md +++ b/docs/ru/sql-reference/functions/type-conversion-functions.md @@ -22,7 +22,7 @@ toc_title: "\u0424\u0443\u043d\u043a\u0446\u0438\u0438\u0020\u043f\u0440\u0435\u - `toInt128(expr)` — возвращает значение типа `Int128`. - `toInt256(expr)` — возвращает значение типа `Int256`. -**Параметры** +**Аргументы** - `expr` — [выражение](../syntax.md#syntax-expressions) возвращающее число или строку с десятичным представление числа. Бинарное, восьмеричное и шестнадцатеричное представление числа не поддержаны. Ведущие нули обрезаются. @@ -100,7 +100,7 @@ SELECT toInt64OrNull('123123'), toInt8OrNull('123qwe123'); - `toUInt64(expr)` — возвращает значение типа `UInt64`. - `toUInt256(expr)` — возвращает значение типа `UInt256`. -**Параметры** +**Аргументы** - `expr` — [выражение](../syntax.md#syntax-expressions) возвращающее число или строку с десятичным представление числа. Бинарное, восьмеричное и шестнадцатеричное представление числа не поддержаны. Ведущие нули обрезаются. @@ -172,7 +172,7 @@ Cиноним: `DATE`. Эти функции следует использовать вместо функций `toDecimal*()`, если при ошибке обработки входного значения вы хотите получать `NULL` вместо исключения. -**Параметры** +**Аргументы** - `expr` — [выражение](../syntax.md#syntax-expressions), возвращающее значение типа [String](../../sql-reference/functions/type-conversion-functions.md). ClickHouse ожидает текстовое представление десятичного числа. Например, `'1.111'`. - `S` — количество десятичных знаков в результирующем значении. @@ -225,7 +225,7 @@ SELECT toDecimal32OrNull(toString(-1.111), 2) AS val, toTypeName(val); Эти функции следует использовать вместо функций `toDecimal*()`, если при ошибке обработки входного значения вы хотите получать `0` вместо исключения. -**Параметры** +**Аргументы** - `expr` — [выражение](../syntax.md#syntax-expressions), возвращающее значение типа [String](../../sql-reference/functions/type-conversion-functions.md). ClickHouse ожидает текстовое представление десятичного числа. Например, `'1.111'`. - `S` — количество десятичных знаков в результирующем значении. @@ -377,7 +377,7 @@ SELECT toFixedString('foo\0bar', 8) AS s, toStringCutToZero(s) AS s_cut; reinterpretAsUUID(fixed_string) ``` -**Параметры** +**Аргументы** - `fixed_string` — cтрока с big-endian порядком байтов. [FixedString](../../sql-reference/data-types/fixedstring.md#fixedstring). @@ -488,7 +488,7 @@ SELECT toTypeName(CAST(x, 'Nullable(UInt16)')) FROM t_null; └─────────────────────────────────────────┘ ``` -**См. также** +**Смотрите также** - Настройка [cast_keep_nullable](../../operations/settings/settings.md#cast_keep_nullable) @@ -537,7 +537,7 @@ Code: 70. DB::Exception: Received from localhost:9000. DB::Exception: Value in c accurateCastOrNull(x, T) ``` -**Параметры** +**Аргументы** - `x` — входное значение. - `T` — имя возвращаемого типа данных. @@ -596,7 +596,7 @@ toIntervalQuarter(number) toIntervalYear(number) ``` -**Параметры** +**Аргументы** - `number` — длительность интервала. Положительное целое число. @@ -638,7 +638,7 @@ SELECT parseDateTimeBestEffort(time_string[, time_zone]) ``` -**Параметры** +**Аргументы** - `time_string` — строка, содержащая дату и время для преобразования. [String](../../sql-reference/functions/type-conversion-functions.md). - `time_zone` — часовой пояс. Функция анализирует `time_string` в соответствии с заданным часовым поясом. [String](../../sql-reference/functions/type-conversion-functions.md). @@ -733,7 +733,7 @@ SELECT parseDateTimeBestEffort('10 20:19'); └─────────────────────────────────────┘ ``` -**См. также** +**Смотрите также** - [Информация о формате ISO 8601 от @xkcd](https://xkcd.com/1179/) - [RFC 1123](https://tools.ietf.org/html/rfc1123) @@ -750,7 +750,7 @@ SELECT parseDateTimeBestEffort('10 20:19'); parseDateTimeBestEffortUS(time_string [, time_zone]) ``` -**Параметры** +**Аргументы** - `time_string` — строка, содержащая дату и время для преобразования. [String](../../sql-reference/data-types/string.md). - `time_zone` — часовой пояс. Функция анализирует `time_string` в соответствии с часовым поясом. [String](../../sql-reference/data-types/string.md). @@ -824,7 +824,7 @@ AS parseDateTimeBestEffortUS; parseDateTimeBestEffortUSOrNull(time_string[, time_zone]) ``` -**Параметры** +**Аргументы** - `time_string` — строка, содержащая дату или дату со временем для преобразования. Дата должна быть в американском формате (`MM/DD/YYYY` и т.д.). [String](../../sql-reference/data-types/string.md). - `time_zone` — [часовой пояс](../../operations/server-configuration-parameters/settings.md#server_configuration_parameters-timezone). Функция анализирует `time_string` в соответствии с заданным часовым поясом. Опциональный параметр. [String](../../sql-reference/data-types/string.md). @@ -910,7 +910,7 @@ SELECT parseDateTimeBestEffortUSOrNull('10.2021') AS parseDateTimeBestEffortUSOr parseDateTimeBestEffortUSOrZero(time_string[, time_zone]) ``` -**Параметры** +**Аргументы** - `time_string` — строка, содержащая дату или дату со временем для преобразования. Дата должна быть в американском формате (`MM/DD/YYYY` и т.д.). [String](../../sql-reference/data-types/string.md). - `time_zone` — [часовой пояс](../../operations/server-configuration-parameters/settings.md#server_configuration_parameters-timezone). Функция анализирует `time_string` в соответствии с заданным часовым поясом. Опциональный параметр. [String](../../sql-reference/data-types/string.md). @@ -999,7 +999,7 @@ SELECT parseDateTimeBestEffortUSOrZero('02.2021') AS parseDateTimeBestEffortUSOr toUnixTimestamp64Milli(value) ``` -**Параметры** +**Аргументы** - `value` — значение `DateTime64` с любой точностью. @@ -1051,7 +1051,7 @@ SELECT toUnixTimestamp64Nano(dt64); fromUnixTimestamp64Milli(value [, ti]) ``` -**Параметры** +**Аргументы** - `value` — значение типы `Int64` с любой точностью. - `timezone` — (не обязательный параметр) часовой пояс в формате `String` для возвращаемого результата. @@ -1089,14 +1089,14 @@ SELECT fromUnixTimestamp64Milli(i64, 'UTC'); toLowCardinality(expr) ``` -**Параметры** +**Аргументы** -- `expr` — [Выражение](../syntax.md#syntax-expressions), которое в результате преобразуется в один из [поддерживаемых типов данных](../data-types/index.md#data_types). +- `expr` — [выражение](../syntax.md#syntax-expressions), которое в результате преобразуется в один из [поддерживаемых типов данных](../data-types/index.md#data_types). **Возвращаемое значение** -- Результат преобразования `expr`. +- Результат преобразования `expr`. Тип: `LowCardinality(expr_result_type)` @@ -1126,10 +1126,10 @@ SELECT toLowCardinality('1'); formatRow(format, x, y, ...) ``` -**Параметры** +**Аргументы** -- `format` — Текстовый формат. Например, [CSV](../../interfaces/formats.md#csv), [TSV](../../interfaces/formats.md#tabseparated). -- `x`,`y`, ... — Выражения. +- `format` — текстовый формат. Например, [CSV](../../interfaces/formats.md#csv), [TSV](../../interfaces/formats.md#tabseparated). +- `x`,`y`, ... — выражения. **Возвращаемое значение** @@ -1167,10 +1167,10 @@ FROM numbers(3); formatRowNoNewline(format, x, y, ...) ``` -**Параметры** +**Аргументы** -- `format` — Текстовый формат. Например, [CSV](../../interfaces/formats.md#csv), [TSV](../../interfaces/formats.md#tabseparated). -- `x`,`y`, ... — Выражения. +- `format` — текстовый формат. Например, [CSV](../../interfaces/formats.md#csv), [TSV](../../interfaces/formats.md#tabseparated). +- `x`,`y`, ... — выражения. **Возвращаемое значение** diff --git a/docs/ru/sql-reference/functions/url-functions.md b/docs/ru/sql-reference/functions/url-functions.md index 7541e16bed4..bd1f9987291 100644 --- a/docs/ru/sql-reference/functions/url-functions.md +++ b/docs/ru/sql-reference/functions/url-functions.md @@ -23,7 +23,7 @@ toc_title: "\u0424\u0443\u043d\u043a\u0446\u0438\u0438\u0020\u0434\u043b\u044f\u domain(url) ``` -**Параметры** +**Аргументы** - `url` — URL. Тип — [String](../../sql-reference/functions/url-functions.md). @@ -53,7 +53,7 @@ yandex.com **Пример** ``` sql -SELECT domain('svn+ssh://some.svn-hosting.com:80/repo/trunk') +SELECT domain('svn+ssh://some.svn-hosting.com:80/repo/trunk'); ``` ``` text @@ -74,7 +74,7 @@ SELECT domain('svn+ssh://some.svn-hosting.com:80/repo/trunk') topLevelDomain(url) ``` -**Параметры** +**Аргументы** - `url` — URL. Тип — [String](../../sql-reference/functions/url-functions.md). @@ -96,7 +96,7 @@ https://yandex.com/time/ **Пример** ``` sql -SELECT topLevelDomain('svn+ssh://www.some.svn-hosting.com:80/repo/trunk') +SELECT topLevelDomain('svn+ssh://www.some.svn-hosting.com:80/repo/trunk'); ``` ``` text @@ -138,7 +138,7 @@ SELECT topLevelDomain('svn+ssh://www.some.svn-hosting.com:80/repo/trunk') cutToFirstSignificantSubdomain(URL, TLD) ``` -**Parameters** +**Аргументы** - `URL` — URL. [String](../../sql-reference/data-types/string.md). - `TLD` — имя пользовательского списка доменов верхнего уровня. [String](../../sql-reference/data-types/string.md). @@ -192,7 +192,7 @@ SELECT cutToFirstSignificantSubdomainCustom('bar.foo.there-is-no-such-domain', ' cutToFirstSignificantSubdomainCustomWithWWW(URL, TLD) ``` -**Параметры** +**Аргументы** - `URL` — URL. [String](../../sql-reference/data-types/string.md). - `TLD` — имя пользовательского списка доменов верхнего уровня. [String](../../sql-reference/data-types/string.md). @@ -246,7 +246,7 @@ SELECT cutToFirstSignificantSubdomainCustomWithWWW('www.foo', 'public_suffix_lis firstSignificantSubdomainCustom(URL, TLD) ``` -**Параметры** +**Аргументы** - `URL` — URL. [String](../../sql-reference/data-types/string.md). - `TLD` — имя пользовательского списка доменов верхнего уровня. [String](../../sql-reference/data-types/string.md). @@ -355,7 +355,7 @@ SELECT decodeURLComponent('http://127.0.0.1:8123/?query=SELECT%201%3B') AS Decod netloc(URL) ``` -**Параметры** +**Аргументы** - `url` — URL. Тип — [String](../../sql-reference/data-types/string.md). diff --git a/docs/ru/sql-reference/functions/ym-dict-functions.md b/docs/ru/sql-reference/functions/ym-dict-functions.md index c3b04e4ab66..63ea76907f9 100644 --- a/docs/ru/sql-reference/functions/ym-dict-functions.md +++ b/docs/ru/sql-reference/functions/ym-dict-functions.md @@ -113,13 +113,13 @@ LIMIT 15 **Синтаксис** ``` sql -regionToTopContinent(id[, geobase]); +regionToTopContinent(id[, geobase]) ``` -**Параметры** +**Аргументы** -- `id` — Идентификатор региона из геобазы Яндекса. [UInt32](../../sql-reference/functions/ym-dict-functions.md). -- `geobase` — Ключ словаря. Смотрите [Множественные геобазы](#multiple-geobases). [String](../../sql-reference/functions/ym-dict-functions.md). Опциональный параметр. +- `id` — идентификатор региона из геобазы Яндекса. [UInt32](../../sql-reference/functions/ym-dict-functions.md). +- `geobase` — ключ словаря. Смотрите [Множественные геобазы](#multiple-geobases). [String](../../sql-reference/functions/ym-dict-functions.md). Опциональный параметр. **Возвращаемое значение** diff --git a/docs/ru/sql-reference/table-functions/generate.md b/docs/ru/sql-reference/table-functions/generate.md index 47b7e43bc86..cb1bcda01b3 100644 --- a/docs/ru/sql-reference/table-functions/generate.md +++ b/docs/ru/sql-reference/table-functions/generate.md @@ -10,10 +10,11 @@ toc_title: generateRandom Поддерживает все типы данных, которые могут храниться в таблице, за исключением `LowCardinality` и `AggregateFunction`. ``` sql -generateRandom('name TypeName[, name TypeName]...', [, 'random_seed'[, 'max_string_length'[, 'max_array_length']]]); +generateRandom('name TypeName[, name TypeName]...', [, 'random_seed'[, 'max_string_length'[, 'max_array_length']]]) ``` -**Входные параметры** +**Аргументы** + - `name` — название соответствующего столбца. - `TypeName` — тип соответствующего столбца. - `max_array_length` — максимальная длина массива для всех сгенерированных массивов. По умолчанию `10`. diff --git a/docs/ru/sql-reference/table-functions/mysql.md b/docs/ru/sql-reference/table-functions/mysql.md index 18b34d0bf6c..9af46d61e45 100644 --- a/docs/ru/sql-reference/table-functions/mysql.md +++ b/docs/ru/sql-reference/table-functions/mysql.md @@ -10,10 +10,10 @@ toc_title: mysql **Синтаксис** ``` sql -mysql('host:port', 'database', 'table', 'user', 'password'[, replace_query, 'on_duplicate_clause']); +mysql('host:port', 'database', 'table', 'user', 'password'[, replace_query, 'on_duplicate_clause']) ``` -**Параметры** +**Аргументы** - `host:port` — адрес сервера MySQL. diff --git a/docs/ru/sql-reference/table-functions/view.md b/docs/ru/sql-reference/table-functions/view.md index 8a97253d048..91b68b31653 100644 --- a/docs/ru/sql-reference/table-functions/view.md +++ b/docs/ru/sql-reference/table-functions/view.md @@ -8,7 +8,7 @@ view(subquery) ``` -**Входные параметры** +**Аргументы** - `subquery` — запрос `SELECT`. @@ -32,7 +32,7 @@ view(subquery) Запрос: ``` sql -SELECT * FROM view(SELECT name FROM months) +SELECT * FROM view(SELECT name FROM months); ``` Результат: @@ -49,14 +49,15 @@ SELECT * FROM view(SELECT name FROM months) Вы можете использовать функцию `view` как параметр табличных функций [remote](https://clickhouse.tech/docs/ru/sql-reference/table-functions/remote/#remote-remotesecure) и [cluster](https://clickhouse.tech/docs/ru/sql-reference/table-functions/cluster/#cluster-clusterallreplicas): ``` sql -SELECT * FROM remote(`127.0.0.1`, view(SELECT a, b, c FROM table_name)) +SELECT * FROM remote(`127.0.0.1`, view(SELECT a, b, c FROM table_name)); ``` ``` sql -SELECT * FROM cluster(`cluster_name`, view(SELECT a, b, c FROM table_name)) +SELECT * FROM cluster(`cluster_name`, view(SELECT a, b, c FROM table_name)); ``` **Смотрите также** - [view](https://clickhouse.tech/docs/ru/engines/table-engines/special/view/#table_engines-view) -[Оригинальная статья](https://clickhouse.tech/docs/ru/query_language/table_functions/view/) \ No newline at end of file + +[Оригинальная статья](https://clickhouse.tech/docs/ru/query_language/table_functions/view/) From 37cc2fe9770ad7850fe4e1f947f3b35653cf8f3c Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Sat, 13 Mar 2021 22:39:10 +0300 Subject: [PATCH 412/544] Use custom memcpy only for AArch64 --- base/glibc-compatibility/CMakeLists.txt | 7 +++++-- base/glibc-compatibility/memcpy/CMakeLists.txt | 10 ++++++---- 2 files changed, 11 insertions(+), 6 deletions(-) diff --git a/base/glibc-compatibility/CMakeLists.txt b/base/glibc-compatibility/CMakeLists.txt index cdd5ec61833..e785e2ab2ce 100644 --- a/base/glibc-compatibility/CMakeLists.txt +++ b/base/glibc-compatibility/CMakeLists.txt @@ -1,5 +1,8 @@ if (GLIBC_COMPATIBILITY) add_subdirectory(memcpy) + if(TARGET memcpy) + set(MEMCPY_LIBRARY memcpy) + endif() enable_language(ASM) include(CheckIncludeFile) @@ -44,10 +47,10 @@ if (GLIBC_COMPATIBILITY) target_compile_options(glibc-compatibility PRIVATE -fPIC) endif () - target_link_libraries(global-libs INTERFACE glibc-compatibility memcpy) + target_link_libraries(global-libs INTERFACE glibc-compatibility ${MEMCPY_LIBRARY}) install( - TARGETS glibc-compatibility memcpy + TARGETS glibc-compatibility ${MEMCPY_LIBRARY} EXPORT global ARCHIVE DESTINATION lib ) diff --git a/base/glibc-compatibility/memcpy/CMakeLists.txt b/base/glibc-compatibility/memcpy/CMakeLists.txt index e8de76151bc..133995d9b96 100644 --- a/base/glibc-compatibility/memcpy/CMakeLists.txt +++ b/base/glibc-compatibility/memcpy/CMakeLists.txt @@ -1,6 +1,8 @@ -add_library(memcpy STATIC memcpy.cpp) +if (ARCH_AMD64) + add_library(memcpy STATIC memcpy.cpp) -# We allow to include memcpy.h from user code for better inlining. -target_include_directories(memcpy PUBLIC $) + # We allow to include memcpy.h from user code for better inlining. + target_include_directories(memcpy PUBLIC $) -target_compile_options(memcpy PRIVATE -fno-builtin-memcpy) + target_compile_options(memcpy PRIVATE -fno-builtin-memcpy) +endif () From ee483d89e5887d6e8494c37f8896082d9af048d0 Mon Sep 17 00:00:00 2001 From: Pavel Kruglov Date: Sat, 13 Mar 2021 23:16:24 +0300 Subject: [PATCH 413/544] Fix style --- src/Interpreters/Context.cpp | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/Interpreters/Context.cpp b/src/Interpreters/Context.cpp index bb2d553b8e8..1f2da5c3946 100644 --- a/src/Interpreters/Context.cpp +++ b/src/Interpreters/Context.cpp @@ -1835,9 +1835,8 @@ void Context::setClustersConfig(const ConfigurationPtr & config, const String & std::lock_guard lock(shared->clusters_mutex); /// Do not update clusters if this part of config wasn't changed. - if (shared->clusters && isSameConfiguration(*config, *shared->clusters_config, config_name)) { + if (shared->clusters && isSameConfiguration(*config, *shared->clusters_config, config_name)) return; - } auto old_clusters_config = shared->clusters_config; shared->clusters_config = config; From b1398e6ffa37cfb793695876dc361b0ca756ec9e Mon Sep 17 00:00:00 2001 From: Maksim Kita Date: Sat, 13 Mar 2021 23:26:01 +0300 Subject: [PATCH 414/544] Updated test --- .../0_stateless/01280_ssd_complex_key_dictionary.sql | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/queries/0_stateless/01280_ssd_complex_key_dictionary.sql b/tests/queries/0_stateless/01280_ssd_complex_key_dictionary.sql index 3f351f14863..c63f858c59c 100644 --- a/tests/queries/0_stateless/01280_ssd_complex_key_dictionary.sql +++ b/tests/queries/0_stateless/01280_ssd_complex_key_dictionary.sql @@ -37,12 +37,12 @@ CREATE DICTIONARY 01280_db.ssd_dict PRIMARY KEY k1, k2 SOURCE(CLICKHOUSE(HOST 'localhost' PORT tcpPort() USER 'default' TABLE 'table_for_dict' PASSWORD '' DB '01280_db')) LIFETIME(MIN 1000 MAX 2000) -LAYOUT(COMPLEX_KEY_SSD_CACHE(FILE_SIZE 8192 PATH '/var/lib/clickhouse/clickhouse_dicts/0d')); +LAYOUT(COMPLEX_KEY_SSD_CACHE(FILE_SIZE 8192 PATH '/home/yetti/Documents/ClickHouseClang/build_debug/programs/0d')); SELECT 'TEST_SMALL'; SELECT 'VALUE FROM RAM BUFFER'; -SELECT dictHas('01280_db.ssd_dict', 'a', tuple('1')); -- { serverError 42 } +SELECT dictHas('01280_db.ssd_dict', 'a', tuple('1')); -- { serverError 43 } SELECT dictGetUInt64('01280_db.ssd_dict', 'a', tuple('1', toInt32(3))); SELECT dictGetInt32('01280_db.ssd_dict', 'b', tuple('1', toInt32(3))); @@ -98,7 +98,7 @@ CREATE DICTIONARY 01280_db.ssd_dict PRIMARY KEY k1, k2 SOURCE(CLICKHOUSE(HOST 'localhost' PORT tcpPort() USER 'default' TABLE 'table_for_dict' PASSWORD '' DB '01280_db')) LIFETIME(MIN 1000 MAX 2000) -LAYOUT(COMPLEX_KEY_SSD_CACHE(FILE_SIZE 8192 PATH '/var/lib/clickhouse/clickhouse_dicts/1d' BLOCK_SIZE 512 WRITE_BUFFER_SIZE 4096 MAX_STORED_KEYS 1000000)); +LAYOUT(COMPLEX_KEY_SSD_CACHE(FILE_SIZE 8192 PATH '/home/yetti/Documents/ClickHouseClang/build_debug/programs/1d' BLOCK_SIZE 512 WRITE_BUFFER_SIZE 4096 MAX_STORED_KEYS 1000000)); SELECT 'UPDATE DICTIONARY'; -- 118 From d83b4d3ab6c342bf369e441e3e442386f2309daf Mon Sep 17 00:00:00 2001 From: Pavel Kruglov Date: Sat, 13 Mar 2021 23:43:33 +0300 Subject: [PATCH 415/544] Try fix flacky test --- .../0_stateless/01753_system_zookeeper_query_param_path.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/queries/0_stateless/01753_system_zookeeper_query_param_path.sh b/tests/queries/0_stateless/01753_system_zookeeper_query_param_path.sh index 75a8e1b95ac..33d979d4bb7 100755 --- a/tests/queries/0_stateless/01753_system_zookeeper_query_param_path.sh +++ b/tests/queries/0_stateless/01753_system_zookeeper_query_param_path.sh @@ -6,9 +6,9 @@ CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) ${CLICKHOUSE_CLIENT} --query="DROP TABLE IF EXISTS test_01753"; -${CLICKHOUSE_CLIENT} --query="CREATE TABLE test_01753 (n Int8) ENGINE=ReplicatedMergeTree('/test_01753/test', 'r') ORDER BY n" +${CLICKHOUSE_CLIENT} --query="CREATE TABLE test_01753 (n Int8) ENGINE=ReplicatedMergeTree('/"$CLICKHOUSE_DATABASE"/test_01753/test', '1') ORDER BY n" -${CLICKHOUSE_CLIENT} --query="SELECT name FROM system.zookeeper WHERE path = {path:String}" --param_path "/test_01753" +${CLICKHOUSE_CLIENT} --query="SELECT name FROM system.zookeeper WHERE path = {path:String}" --param_path $CLICKHOUSE_DATABASE"/test_01753" ${CLICKHOUSE_CLIENT} --query="DROP TABLE test_01753 SYNC"; From 36a41220d3e9b9e052b3a26ce4538c73e47d7c6e Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Sun, 14 Mar 2021 00:46:47 +0300 Subject: [PATCH 416/544] Add comments and documentation --- base/glibc-compatibility/memcpy/memcpy.h | 118 ++++++++++++++++++++++- 1 file changed, 117 insertions(+), 1 deletion(-) diff --git a/base/glibc-compatibility/memcpy/memcpy.h b/base/glibc-compatibility/memcpy/memcpy.h index 3e1ef793f81..b069bcf9653 100644 --- a/base/glibc-compatibility/memcpy/memcpy.h +++ b/base/glibc-compatibility/memcpy/memcpy.h @@ -3,42 +3,152 @@ #include +/** Custom memcpy implementation for ClickHouse. + * It has the following benefits over using glibc's implementation: + * 1. Avoiding dependency on specific version of glibc's symbol, like memcpy@@GLIBC_2.14 for portability. + * 2. Avoiding indirect call via PLT due to shared linking, that can be less efficient. + * 3. It's possible to include this header and call inline_memcpy directly for better inlining or interprocedural analysis. + * 4. Better results on our performance tests on current CPUs: up to 25% on some queries and up to 0.7%..1% in average across all queries. + * + * Writing our own memcpy is extremely difficult for the following reasons: + * 1. The optimal variant depends on the specific CPU model. + * 2. The optimal variant depends on the distribution of size arguments. + * 3. It depends on the number of threads copying data concurrently. + * 4. It also depends on how the calling code is using the copied data and how the different memcpy calls are related to each other. + * Due to vast range of scenarios it makes proper testing especially difficult. + * When writing our own memcpy there is a risk to overoptimize it + * on non-representative microbenchmarks while making real-world use cases actually worse. + * + * Most of the benchmarks for memcpy on the internet are wrong. + * + * Let's look at the details: + * + * For small size, the order of branches in code is important. + * There are variants with specific order of branches (like here or in glibc) + * or with jump table (in asm code see example from Cosmopolitan libc: + * https://github.com/jart/cosmopolitan/blob/de09bec215675e9b0beb722df89c6f794da74f3f/libc/nexgen32e/memcpy.S#L61) + * or with Duff device in C (see https://github.com/skywind3000/FastMemcpy/) + * + * It's also important how to copy uneven sizes. + * Almost every implementation, including this, is using two overlapping movs. + * + * It is important to disable -ftree-loop-distribute-patterns when compiling memcpy implementation, + * otherwise the compiler can replace internal loops to a call to memcpy that will lead to infinite recursion. + * + * For larger sizes it's important to choose the instructions used: + * - SSE or AVX or AVX-512; + * - rep movsb; + * Performance will depend on the size threshold, on the CPU model, on the "erms" flag + * ("Enhansed Rep MovS" - it indicates that performance of "rep movsb" is decent for large sizes) + * + * Using AVX-512 can be bad due to throttling. + * Using AVX can be bad if most code is using SSE due to switching penalty + * (it also depends on the usage of "vzeroupper" instruction). + * But in some cases AVX gives a win. + * + * It also depends on how many times the loop will be unrolled. + * We are unrolling the loop 8 times (by the number of available registers), but it not always the best. + * + * It also depends on the usage of aligned or unaligned loads/stores. + * We are using unaligned loads and aligned stores. + * + * It also depends on the usage of prefetch instructions. It makes sense on some Intel CPUs but can slow down performance on AMD. + * Setting up correct offset for prefetching is non-obvious. + * + * Non-temporary (cache bypassing) stores can be used for very large sizes (more than a half of L3 cache). + * But the exact threshold is unclear - when doing memcpy from multiple threads the optimal threshold can be lower, + * because L3 cache is shared (and L2 cache is partially shared). + * + * Very large size of memcpy typically indicates suboptimal (not cache friendly) algorithms in code or unrealistic scenarios, + * so we don't pay attention to it. + * + * On recent Intel CPUs, the presense of "erms" makes "rep movsb" the most benefitial, + * even comparing to non-temporary aligned unrolled stores even with the most wide registers. + * + * memcpy can be written in asm, C or C++. The latter can also use inline asm. + * The asm implementation can be better to make sure that compiler won't make the code worse, + * to ensure the order of branches, the code layout, the usage of all required registers. + * But if it is located in separate translation unit, inlining will not be possible + * (inline asm can be used to overcome this limitation). + * Sometimes C or C++ code can be further optimized by compiler. + * For example, clang is capable replacing SSE intrinsics to AVX code if -mavx is used. + * + * Please note that compiler can replace plain code to memcpy and vice versa. + * - memcpy with compile-time known small size is replaced to simple instructions without a call to memcpy; + * it is controlled by -fbuiltin-memcpy and can be manually ensured by calling __builtin_memcpy. + * This is often used to implement unaligned load/store without undefined behaviour in C++. + * - a loop with copying bytes can be recognized and replaced by a call to memcpy; + * it is controlled by -ftree-loop-distribute-patterns. + * - also note that a loop with copying bytes can be unrolled, peeled and vectorized that will give you + * inline code somewhat similar to a decent implementation of memcpy. + * + * This description is up to date as of Mar 2021. + * + * How to test the memcpy implementation for performance: + * 1. Test on real production workload. + * 2. For synthetic test, see utils/memcpy-bench, but make sure you will do the best to exhaust the wide range of scenarios. + */ + + static inline void * inline_memcpy(void * __restrict dst_, const void * __restrict src_, size_t size) { + /// We will use pointer arithmetic, so char pointer will be used. + /// Note that __restrict makes sense (otherwise compiler will reload data from memory + /// instead of using the value of registers due to possible aliasing). char * __restrict dst = reinterpret_cast(dst_); const char * __restrict src = reinterpret_cast(src_); + /// Standard memcpy returns the original value of dst. It is rarely used but we have to do it. + /// If you use memcpy with small but non-constant sizes, you can call inline_memcpy directly + /// for inlining and removing this single instruction. void * ret = dst; tail: + /// Small sizes and tails after the loop for large sizes. + /// The order of branches is important but in fact the optimal order depends on the distribution of sizes in your application. + /// This order of branches is from the disassembly of glibc's code. + /// We copy chunks of possibly uneven size with two overlapping movs. + /// Example: to copy 5 bytes [0, 1, 2, 3, 4] we will copy tail [1, 2, 3, 4] first and then head [0, 1, 2, 3]. if (size <= 16) { if (size >= 8) { + /// Chunks of 8..16 bytes. __builtin_memcpy(dst + size - 8, src + size - 8, 8); __builtin_memcpy(dst, src, 8); } else if (size >= 4) { + /// Chunks of 4..7 bytes. __builtin_memcpy(dst + size - 4, src + size - 4, 4); __builtin_memcpy(dst, src, 4); } else if (size >= 2) { + /// Chunks of 2..3 bytes. __builtin_memcpy(dst + size - 2, src + size - 2, 2); __builtin_memcpy(dst, src, 2); } else if (size >= 1) { + /// A single byte. *dst = *src; } + /// Zero bytes. } else { + /// Medium and large sizes. if (size <= 128) { + /// Medium size, not enough for full loop unrolling. + + /// We will copy the last 16 bytes. _mm_storeu_si128(reinterpret_cast<__m128i *>(dst + size - 16), _mm_loadu_si128(reinterpret_cast(src + size - 16))); + /// Then we will copy every 16 bytes from the beginning in a loop. + /// The last loop iteration will possibly overwrite some part of already copied last 16 bytes. + /// This is Ok, similar to the code for small sizes above. while (size > 16) { _mm_storeu_si128(reinterpret_cast<__m128i *>(dst), _mm_loadu_si128(reinterpret_cast(src))); @@ -49,9 +159,12 @@ tail: } else { + /// Large size with fully unrolled loop. + /// Align destination to 16 bytes boundary. size_t padding = (16 - (reinterpret_cast(dst) & 15)) & 15; + /// If not aligned - we will copy first 16 bytes with unaligned stores. if (padding > 0) { __m128i head = _mm_loadu_si128(reinterpret_cast(src)); @@ -61,7 +174,9 @@ tail: size -= padding; } - /// Aligned unrolled copy. + /// Aligned unrolled copy. We will use all available SSE registers. + /// It's not possible to have both src and dst aligned. + /// So, we will use aligned stores and unaligned loads. __m128i c0, c1, c2, c3, c4, c5, c6, c7; while (size >= 128) @@ -88,6 +203,7 @@ tail: size -= 128; } + /// The latest remaining 0..127 bytes will be processed as usual. goto tail; } } From b4cae2c8589ff8ef0155ccd90d3b7ffee1446961 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Sun, 14 Mar 2021 00:48:08 +0300 Subject: [PATCH 417/544] Add TODO --- base/glibc-compatibility/memcpy/memcpy.h | 3 +++ 1 file changed, 3 insertions(+) diff --git a/base/glibc-compatibility/memcpy/memcpy.h b/base/glibc-compatibility/memcpy/memcpy.h index b069bcf9653..c9a9b3a7772 100644 --- a/base/glibc-compatibility/memcpy/memcpy.h +++ b/base/glibc-compatibility/memcpy/memcpy.h @@ -87,6 +87,9 @@ * How to test the memcpy implementation for performance: * 1. Test on real production workload. * 2. For synthetic test, see utils/memcpy-bench, but make sure you will do the best to exhaust the wide range of scenarios. + * + * TODO: Add self-tuning memcpy with bayesian bandits algorithm for large sizes. + * See https://habr.com/en/company/yandex/blog/457612/ */ From 32e66baa0a9acb30b44f836321cd3216da851774 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Sun, 14 Mar 2021 00:48:44 +0300 Subject: [PATCH 418/544] Add comments and documentation --- base/glibc-compatibility/memcpy/memcpy.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/base/glibc-compatibility/memcpy/memcpy.h b/base/glibc-compatibility/memcpy/memcpy.h index c9a9b3a7772..67b705d930a 100644 --- a/base/glibc-compatibility/memcpy/memcpy.h +++ b/base/glibc-compatibility/memcpy/memcpy.h @@ -137,7 +137,7 @@ tail: /// A single byte. *dst = *src; } - /// Zero bytes. + /// No bytes remaining. } else { From a1ebd52d09a15f4e5215bf4f8706a02ef5f2719a Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Sun, 14 Mar 2021 00:51:13 +0300 Subject: [PATCH 419/544] Add comments and documentation --- base/glibc-compatibility/memcpy/memcpy.h | 1 + 1 file changed, 1 insertion(+) diff --git a/base/glibc-compatibility/memcpy/memcpy.h b/base/glibc-compatibility/memcpy/memcpy.h index 67b705d930a..1fd522fd4c3 100644 --- a/base/glibc-compatibility/memcpy/memcpy.h +++ b/base/glibc-compatibility/memcpy/memcpy.h @@ -40,6 +40,7 @@ * - rep movsb; * Performance will depend on the size threshold, on the CPU model, on the "erms" flag * ("Enhansed Rep MovS" - it indicates that performance of "rep movsb" is decent for large sizes) + * https://stackoverflow.com/questions/43343231/enhanced-rep-movsb-for-memcpy * * Using AVX-512 can be bad due to throttling. * Using AVX can be bad if most code is using SSE due to switching penalty From bab924620ac804b51cc7c8514fc19e937ea7ffcb Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Sun, 14 Mar 2021 00:53:00 +0300 Subject: [PATCH 420/544] Add comments and documentation --- base/glibc-compatibility/memcpy/memcpy.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/base/glibc-compatibility/memcpy/memcpy.h b/base/glibc-compatibility/memcpy/memcpy.h index 1fd522fd4c3..60303e4f544 100644 --- a/base/glibc-compatibility/memcpy/memcpy.h +++ b/base/glibc-compatibility/memcpy/memcpy.h @@ -61,7 +61,7 @@ * because L3 cache is shared (and L2 cache is partially shared). * * Very large size of memcpy typically indicates suboptimal (not cache friendly) algorithms in code or unrealistic scenarios, - * so we don't pay attention to it. + * so we don't pay attention to using non-temporary stores. * * On recent Intel CPUs, the presense of "erms" makes "rep movsb" the most benefitial, * even comparing to non-temporary aligned unrolled stores even with the most wide registers. From 98192f12343046fa4dacf9c2054ffcb9b9996325 Mon Sep 17 00:00:00 2001 From: Maksim Kita Date: Sun, 14 Mar 2021 01:28:53 +0300 Subject: [PATCH 421/544] Update 01280_ssd_complex_key_dictionary.sql --- .../queries/0_stateless/01280_ssd_complex_key_dictionary.sql | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/queries/0_stateless/01280_ssd_complex_key_dictionary.sql b/tests/queries/0_stateless/01280_ssd_complex_key_dictionary.sql index c63f858c59c..50b34c4b18f 100644 --- a/tests/queries/0_stateless/01280_ssd_complex_key_dictionary.sql +++ b/tests/queries/0_stateless/01280_ssd_complex_key_dictionary.sql @@ -37,7 +37,7 @@ CREATE DICTIONARY 01280_db.ssd_dict PRIMARY KEY k1, k2 SOURCE(CLICKHOUSE(HOST 'localhost' PORT tcpPort() USER 'default' TABLE 'table_for_dict' PASSWORD '' DB '01280_db')) LIFETIME(MIN 1000 MAX 2000) -LAYOUT(COMPLEX_KEY_SSD_CACHE(FILE_SIZE 8192 PATH '/home/yetti/Documents/ClickHouseClang/build_debug/programs/0d')); +LAYOUT(COMPLEX_KEY_SSD_CACHE(FILE_SIZE 8192 PATH '/var/lib/clickhouse/clickhouse_dicts/0d')); SELECT 'TEST_SMALL'; SELECT 'VALUE FROM RAM BUFFER'; @@ -98,7 +98,7 @@ CREATE DICTIONARY 01280_db.ssd_dict PRIMARY KEY k1, k2 SOURCE(CLICKHOUSE(HOST 'localhost' PORT tcpPort() USER 'default' TABLE 'table_for_dict' PASSWORD '' DB '01280_db')) LIFETIME(MIN 1000 MAX 2000) -LAYOUT(COMPLEX_KEY_SSD_CACHE(FILE_SIZE 8192 PATH '/home/yetti/Documents/ClickHouseClang/build_debug/programs/1d' BLOCK_SIZE 512 WRITE_BUFFER_SIZE 4096 MAX_STORED_KEYS 1000000)); +LAYOUT(COMPLEX_KEY_SSD_CACHE(FILE_SIZE 8192 PATH '/var/lib/clickhouse/clickhouse_dicts/1d' BLOCK_SIZE 512 WRITE_BUFFER_SIZE 4096 MAX_STORED_KEYS 1000000)); SELECT 'UPDATE DICTIONARY'; -- 118 From 3e3b5c64bf33c6777d4df0113fda0d9beeaac6f4 Mon Sep 17 00:00:00 2001 From: Pavel Kruglov Date: Sun, 14 Mar 2021 02:34:32 +0300 Subject: [PATCH 422/544] add init file --- tests/integration/test_reload_clusters_config/__init__.py | 0 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 tests/integration/test_reload_clusters_config/__init__.py diff --git a/tests/integration/test_reload_clusters_config/__init__.py b/tests/integration/test_reload_clusters_config/__init__.py new file mode 100644 index 00000000000..e69de29bb2d From e116e75d2cb52c09ca31b5851e22bf100e36c07b Mon Sep 17 00:00:00 2001 From: Pavel Kruglov Date: Sun, 14 Mar 2021 02:36:40 +0300 Subject: [PATCH 423/544] Fix style --- .../0_stateless/01753_system_zookeeper_query_param_path.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/queries/0_stateless/01753_system_zookeeper_query_param_path.sh b/tests/queries/0_stateless/01753_system_zookeeper_query_param_path.sh index 33d979d4bb7..1f4ba412a19 100755 --- a/tests/queries/0_stateless/01753_system_zookeeper_query_param_path.sh +++ b/tests/queries/0_stateless/01753_system_zookeeper_query_param_path.sh @@ -6,9 +6,9 @@ CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) ${CLICKHOUSE_CLIENT} --query="DROP TABLE IF EXISTS test_01753"; -${CLICKHOUSE_CLIENT} --query="CREATE TABLE test_01753 (n Int8) ENGINE=ReplicatedMergeTree('/"$CLICKHOUSE_DATABASE"/test_01753/test', '1') ORDER BY n" +${CLICKHOUSE_CLIENT} --query="CREATE TABLE test_01753 (n Int8) ENGINE=ReplicatedMergeTree('/$CLICKHOUSE_DATABASE/test_01753/test', '1') ORDER BY n" -${CLICKHOUSE_CLIENT} --query="SELECT name FROM system.zookeeper WHERE path = {path:String}" --param_path $CLICKHOUSE_DATABASE"/test_01753" +${CLICKHOUSE_CLIENT} --query="SELECT name FROM system.zookeeper WHERE path = {path:String}" --param_path "$CLICKHOUSE_DATABASE/test_01753" ${CLICKHOUSE_CLIENT} --query="DROP TABLE test_01753 SYNC"; From 8eea6a87e991eec1a196ea5291b64ec8293ab583 Mon Sep 17 00:00:00 2001 From: kssenii Date: Sun, 14 Mar 2021 08:35:07 +0000 Subject: [PATCH 424/544] Support replicas priority for postgres dictionary source --- .../compose/docker_compose_postgres.yml | 7 ++ src/Common/ErrorCodes.cpp | 1 + .../PostgreSQL/DatabasePostgreSQL.cpp | 2 +- .../PostgreSQLDictionarySource.cpp | 31 +++---- src/Dictionaries/PostgreSQLDictionarySource.h | 6 +- .../PostgreSQL/PostgreSQLConnection.cpp | 44 +++++++++- .../PostgreSQL/PostgreSQLConnection.h | 18 ++-- .../PostgreSQLReplicaConnection.cpp | 77 +++++++++++++++++ .../PostgreSQL/PostgreSQLReplicaConnection.h | 35 ++++++++ .../configs/dictionaries/postgres_dict.xml | 83 +++++++++++++++++++ .../configs/postgres_dict.xml | 37 --------- .../test_dictionaries_postgresql/test.py | 76 +++++++++++++---- 12 files changed, 332 insertions(+), 85 deletions(-) create mode 100644 src/Storages/PostgreSQL/PostgreSQLReplicaConnection.cpp create mode 100644 src/Storages/PostgreSQL/PostgreSQLReplicaConnection.h create mode 100644 tests/integration/test_dictionaries_postgresql/configs/dictionaries/postgres_dict.xml delete mode 100644 tests/integration/test_dictionaries_postgresql/configs/postgres_dict.xml diff --git a/docker/test/integration/runner/compose/docker_compose_postgres.yml b/docker/test/integration/runner/compose/docker_compose_postgres.yml index 5657352e1b3..58ed97251fb 100644 --- a/docker/test/integration/runner/compose/docker_compose_postgres.yml +++ b/docker/test/integration/runner/compose/docker_compose_postgres.yml @@ -11,3 +11,10 @@ services: default: aliases: - postgre-sql.local + postgres2: + image: postgres + restart: always + environment: + POSTGRES_PASSWORD: mysecretpassword + ports: + - 5441:5432 diff --git a/src/Common/ErrorCodes.cpp b/src/Common/ErrorCodes.cpp index f44cfd938d6..3a0fccc5358 100644 --- a/src/Common/ErrorCodes.cpp +++ b/src/Common/ErrorCodes.cpp @@ -546,6 +546,7 @@ M(577, INVALID_SHARD_ID) \ M(578, INVALID_FORMAT_INSERT_QUERY_WITH_DATA) \ \ + M(998, POSTGRESQL_CONNECTION_FAILURE) \ M(999, KEEPER_EXCEPTION) \ M(1000, POCO_EXCEPTION) \ M(1001, STD_EXCEPTION) \ diff --git a/src/Databases/PostgreSQL/DatabasePostgreSQL.cpp b/src/Databases/PostgreSQL/DatabasePostgreSQL.cpp index 722b9c64edb..511cf8d847c 100644 --- a/src/Databases/PostgreSQL/DatabasePostgreSQL.cpp +++ b/src/Databases/PostgreSQL/DatabasePostgreSQL.cpp @@ -168,7 +168,7 @@ StoragePtr DatabasePostgreSQL::fetchTable(const String & table_name, const Conte return StoragePtr{}; auto storage = StoragePostgreSQL::create( - StorageID(database_name, table_name), table_name, std::make_shared(connection->conn_str()), + StorageID(database_name, table_name), table_name, std::make_shared(*connection), ColumnsDescription{*columns}, ConstraintsDescription{}, context); if (cache_tables) diff --git a/src/Dictionaries/PostgreSQLDictionarySource.cpp b/src/Dictionaries/PostgreSQLDictionarySource.cpp index aa852404750..5b71ad61120 100644 --- a/src/Dictionaries/PostgreSQLDictionarySource.cpp +++ b/src/Dictionaries/PostgreSQLDictionarySource.cpp @@ -8,7 +8,6 @@ #include #include #include -#include #include "readInvalidateQuery.h" #endif @@ -29,11 +28,10 @@ PostgreSQLDictionarySource::PostgreSQLDictionarySource( const DictionaryStructure & dict_struct_, const Poco::Util::AbstractConfiguration & config_, const std::string & config_prefix, - PostgreSQLConnectionPtr connection_, const Block & sample_block_) : dict_struct{dict_struct_} , sample_block(sample_block_) - , connection(std::move(connection_)) + , connection(std::make_shared(config_, config_prefix)) , log(&Poco::Logger::get("PostgreSQLDictionarySource")) , db(config_.getString(fmt::format("{}.db", config_prefix), "")) , table(config_.getString(fmt::format("{}.table", config_prefix), "")) @@ -50,7 +48,7 @@ PostgreSQLDictionarySource::PostgreSQLDictionarySource( PostgreSQLDictionarySource::PostgreSQLDictionarySource(const PostgreSQLDictionarySource & other) : dict_struct(other.dict_struct) , sample_block(other.sample_block) - , connection(std::make_shared(other.connection->conn_str())) + , connection(other.connection) , log(&Poco::Logger::get("PostgreSQLDictionarySource")) , db(other.db) , table(other.table) @@ -68,8 +66,7 @@ PostgreSQLDictionarySource::PostgreSQLDictionarySource(const PostgreSQLDictionar BlockInputStreamPtr PostgreSQLDictionarySource::loadAll() { LOG_TRACE(log, load_all_query); - return std::make_shared( - connection->conn(), load_all_query, sample_block, max_block_size); + return loadBase(load_all_query); } @@ -77,23 +74,28 @@ BlockInputStreamPtr PostgreSQLDictionarySource::loadUpdatedAll() { auto load_update_query = getUpdateFieldAndDate(); LOG_TRACE(log, load_update_query); - return std::make_shared(connection->conn(), load_update_query, sample_block, max_block_size); + return loadBase(load_update_query); } BlockInputStreamPtr PostgreSQLDictionarySource::loadIds(const std::vector & ids) { const auto query = query_builder.composeLoadIdsQuery(ids); - return std::make_shared(connection->conn(), query, sample_block, max_block_size); + return loadBase(query); } BlockInputStreamPtr PostgreSQLDictionarySource::loadKeys(const Columns & key_columns, const std::vector & requested_rows) { const auto query = query_builder.composeLoadKeysQuery(key_columns, requested_rows, ExternalQueryBuilder::AND_OR_CHAIN); - return std::make_shared(connection->conn(), query, sample_block, max_block_size); + return loadBase(query); } +BlockInputStreamPtr PostgreSQLDictionarySource::loadBase(const String & query) +{ + return std::make_shared(connection->get(), query, sample_block, max_block_size); +} + bool PostgreSQLDictionarySource::isModified() const { if (!invalidate_query.empty()) @@ -112,7 +114,7 @@ std::string PostgreSQLDictionarySource::doInvalidateQuery(const std::string & re Block invalidate_sample_block; ColumnPtr column(ColumnString::create()); invalidate_sample_block.insert(ColumnWithTypeAndName(column, std::make_shared(), "Sample Block")); - PostgreSQLBlockInputStream block_input_stream(connection->conn(), request, invalidate_sample_block, 1); + PostgreSQLBlockInputStream block_input_stream(connection->get(), request, invalidate_sample_block, 1); return readInvalidateQuery(block_input_stream); } @@ -172,15 +174,8 @@ void registerDictionarySourcePostgreSQL(DictionarySourceFactory & factory) { #if USE_LIBPQXX const auto config_prefix = root_config_prefix + ".postgresql"; - auto connection = std::make_shared( - config.getString(fmt::format("{}.db", config_prefix), ""), - config.getString(fmt::format("{}.host", config_prefix), ""), - config.getUInt(fmt::format("{}.port", config_prefix), 0), - config.getString(fmt::format("{}.user", config_prefix), ""), - config.getString(fmt::format("{}.password", config_prefix), "")); - return std::make_unique( - dict_struct, config, config_prefix, connection, sample_block); + dict_struct, config, config_prefix, sample_block); #else (void)dict_struct; (void)config; diff --git a/src/Dictionaries/PostgreSQLDictionarySource.h b/src/Dictionaries/PostgreSQLDictionarySource.h index a826ff15f4f..dd2d35db83a 100644 --- a/src/Dictionaries/PostgreSQLDictionarySource.h +++ b/src/Dictionaries/PostgreSQLDictionarySource.h @@ -11,7 +11,7 @@ #include #include #include -#include +#include #include @@ -26,7 +26,6 @@ public: const DictionaryStructure & dict_struct_, const Poco::Util::AbstractConfiguration & config_, const std::string & config_prefix, - PostgreSQLConnectionPtr connection_, const Block & sample_block_); /// copy-constructor is provided in order to support cloneability @@ -48,10 +47,11 @@ public: private: std::string getUpdateFieldAndDate(); std::string doInvalidateQuery(const std::string & request) const; + BlockInputStreamPtr loadBase(const String & query); const DictionaryStructure dict_struct; Block sample_block; - PostgreSQLConnectionPtr connection; + PostgreSQLReplicaConnectionPtr connection; Poco::Logger * log; const std::string db; diff --git a/src/Storages/PostgreSQL/PostgreSQLConnection.cpp b/src/Storages/PostgreSQL/PostgreSQLConnection.cpp index 668550ec721..58eb7192eb9 100644 --- a/src/Storages/PostgreSQL/PostgreSQLConnection.cpp +++ b/src/Storages/PostgreSQL/PostgreSQLConnection.cpp @@ -6,23 +6,63 @@ #include #include #include +#include namespace DB { +PostgreSQLConnection::PostgreSQLConnection(std::string dbname, std::string host, UInt16 port, std::string user, std::string password) +{ + address = host + ':' + std::to_string(port); + connection_str = formatConnectionString(std::move(dbname), std::move(host), port, std::move(user), std::move(password)); +} + + +PostgreSQLConnection::PostgreSQLConnection(const PostgreSQLConnection & other) + : connection_str(other.connection_str) + , address(other.address) +{ +} + + PostgreSQLConnection::ConnectionPtr PostgreSQLConnection::conn() { - checkUpdateConnection(); + connect(); return connection; } -void PostgreSQLConnection::checkUpdateConnection() + +void PostgreSQLConnection::connect() { if (!connection || !connection->is_open()) connection = std::make_unique(connection_str); } + +bool PostgreSQLConnection::tryConnect() +{ + try + { + connect(); + } + catch (const pqxx::broken_connection & pqxx_error) + { + LOG_ERROR( + &Poco::Logger::get("PostgreSQLConnection"), + "Unable to setup connection to {}, reason: {}", + getAddress(), pqxx_error.what()); + return false; + } + catch (...) + { + throw; + } + + return true; +} + + std::string PostgreSQLConnection::formatConnectionString( std::string dbname, std::string host, UInt16 port, std::string user, std::string password) { diff --git a/src/Storages/PostgreSQL/PostgreSQLConnection.h b/src/Storages/PostgreSQL/PostgreSQLConnection.h index ae79a3436e0..f23308ddef9 100644 --- a/src/Storages/PostgreSQL/PostgreSQLConnection.h +++ b/src/Storages/PostgreSQL/PostgreSQLConnection.h @@ -16,29 +16,31 @@ namespace DB /// Connection is not made until actually used. class PostgreSQLConnection { +public: using ConnectionPtr = std::shared_ptr; -public: - PostgreSQLConnection(std::string dbname, std::string host, UInt16 port, std::string user, std::string password) - : connection_str(formatConnectionString(std::move(dbname), std::move(host), port, std::move(user), std::move(password))) {} + PostgreSQLConnection(std::string dbname, std::string host, UInt16 port, std::string user, std::string password); - PostgreSQLConnection(const std::string & connection_str_) : connection_str(connection_str_) {} + PostgreSQLConnection(const PostgreSQLConnection & other); - PostgreSQLConnection(const PostgreSQLConnection &) = delete; PostgreSQLConnection operator =(const PostgreSQLConnection &) = delete; + bool tryConnect(); + ConnectionPtr conn(); + const std::string & getAddress() { return address; } + std::string & conn_str() { return connection_str; } private: - ConnectionPtr connection; - std::string connection_str; + void connect(); static std::string formatConnectionString( std::string dbname, std::string host, UInt16 port, std::string user, std::string password); - void checkUpdateConnection(); + ConnectionPtr connection; + std::string connection_str, address; }; using PostgreSQLConnectionPtr = std::shared_ptr; diff --git a/src/Storages/PostgreSQL/PostgreSQLReplicaConnection.cpp b/src/Storages/PostgreSQL/PostgreSQLReplicaConnection.cpp new file mode 100644 index 00000000000..0c1efc16e05 --- /dev/null +++ b/src/Storages/PostgreSQL/PostgreSQLReplicaConnection.cpp @@ -0,0 +1,77 @@ +#include "PostgreSQLReplicaConnection.h" +#include + + +namespace DB +{ + +namespace ErrorCodes +{ + extern const int POSTGRESQL_CONNECTION_FAILURE; +} + + +PostgreSQLReplicaConnection::PostgreSQLReplicaConnection( + const Poco::Util::AbstractConfiguration & config, + const String & config_prefix, + const size_t num_retries_) + : log(&Poco::Logger::get("PostgreSQLConnection")) + , num_retries(num_retries_) +{ + auto db = config.getString(config_prefix + ".db", ""); + auto host = config.getString(config_prefix + ".host", ""); + auto port = config.getUInt(config_prefix + ".port", 0); + auto user = config.getString(config_prefix + ".user", ""); + auto password = config.getString(config_prefix + ".password", ""); + + if (config.has(config_prefix + ".replica")) + { + Poco::Util::AbstractConfiguration::Keys config_keys; + config.keys(config_prefix, config_keys); + + for (const auto & config_key : config_keys) + { + if (config_key.starts_with("replica")) + { + std::string replica_name = config_prefix + "." + config_key; + size_t priority = config.getInt(replica_name + ".priority", 0); + + auto replica_host = config.getString(replica_name + ".host", host); + auto replica_port = config.getUInt(replica_name + ".port", port); + auto replica_user = config.getString(replica_name + ".user", user); + auto replica_password = config.getString(replica_name + ".password", password); + + replicas[priority] = std::make_shared(db, replica_host, replica_port, replica_user, replica_password); + } + } + } + else + { + replicas[0] = std::make_shared(db, host, port, user, password); + } +} + + +PostgreSQLReplicaConnection::PostgreSQLReplicaConnection(const PostgreSQLReplicaConnection & other) + : log(&Poco::Logger::get("PostgreSQLConnection")) + , replicas(other.replicas) + , num_retries(other.num_retries) +{ +} + + +PostgreSQLConnection::ConnectionPtr PostgreSQLReplicaConnection::get() +{ + for (size_t i = 0; i < num_retries; ++i) + { + for (auto & replica : replicas) + { + if (replica.second->tryConnect()) + return replica.second->conn(); + } + } + + throw Exception(ErrorCodes::POSTGRESQL_CONNECTION_FAILURE, "Unable to connect to any of the replicas"); +} + +} diff --git a/src/Storages/PostgreSQL/PostgreSQLReplicaConnection.h b/src/Storages/PostgreSQL/PostgreSQLReplicaConnection.h new file mode 100644 index 00000000000..9b5dec5e9a5 --- /dev/null +++ b/src/Storages/PostgreSQL/PostgreSQLReplicaConnection.h @@ -0,0 +1,35 @@ +#include "PostgreSQLConnection.h" +#include +#include +#include + +namespace DB +{ + +class PostgreSQLReplicaConnection +{ + +public: + static constexpr inline auto POSTGRESQL_CONNECTION_DEFAULT_RETRIES_NUM = 5; + + PostgreSQLReplicaConnection( + const Poco::Util::AbstractConfiguration & config, + const String & config_name, + const size_t num_retries = POSTGRESQL_CONNECTION_DEFAULT_RETRIES_NUM); + + PostgreSQLReplicaConnection(const PostgreSQLReplicaConnection & other); + + PostgreSQLConnection::ConnectionPtr get(); + + +private: + using ReplicasByPriority = std::map; + + Poco::Logger * log; + ReplicasByPriority replicas; + size_t num_retries; +}; + +using PostgreSQLReplicaConnectionPtr = std::shared_ptr; + +} diff --git a/tests/integration/test_dictionaries_postgresql/configs/dictionaries/postgres_dict.xml b/tests/integration/test_dictionaries_postgresql/configs/dictionaries/postgres_dict.xml new file mode 100644 index 00000000000..4ee07d0972a --- /dev/null +++ b/tests/integration/test_dictionaries_postgresql/configs/dictionaries/postgres_dict.xml @@ -0,0 +1,83 @@ + + + + dict0 + + + clickhouse + postgres1 + 5432 + postgres + mysecretpassword + test0
+ SELECT value FROM test0 WHERE id = 0 +
+ + + + + + + id + UInt32 + + + id + UInt32 + + + + value + UInt32 + + + + 1 +
+ + dict1 + + + clickhouse + postgres + mysecretpassword + test1
+ + postgres1 + 3 + 5432 + + + postgres2 + 5433 + 1 + + + postgres2 + 5432 + 2 + +
+ + + + + + + id + UInt32 + + + id + UInt32 + + + + value + UInt32 + + + + 1 +
+
diff --git a/tests/integration/test_dictionaries_postgresql/configs/postgres_dict.xml b/tests/integration/test_dictionaries_postgresql/configs/postgres_dict.xml deleted file mode 100644 index 2572930a798..00000000000 --- a/tests/integration/test_dictionaries_postgresql/configs/postgres_dict.xml +++ /dev/null @@ -1,37 +0,0 @@ - - - - dict0 - - - clickhouse - postgres1 - 5432 - postgres - mysecretpassword - test0
- SELECT value FROM test0 WHERE id = 0 -
- - - - - - - id - UInt32 - - - id - UInt32 - - - - value - UInt32 - - - - 1 -
-
diff --git a/tests/integration/test_dictionaries_postgresql/test.py b/tests/integration/test_dictionaries_postgresql/test.py index b83c00409af..1ca3c89a5ee 100644 --- a/tests/integration/test_dictionaries_postgresql/test.py +++ b/tests/integration/test_dictionaries_postgresql/test.py @@ -6,7 +6,10 @@ from helpers.cluster import ClickHouseCluster from psycopg2.extensions import ISOLATION_LEVEL_AUTOCOMMIT cluster = ClickHouseCluster(__file__) -node1 = cluster.add_instance('node1', main_configs=['configs/config.xml', 'configs/postgres_dict.xml', 'configs/log_conf.xml'], with_postgres=True) +node1 = cluster.add_instance('node1', main_configs=[ + 'configs/config.xml', + 'configs/dictionaries/postgres_dict.xml', + 'configs/log_conf.xml'], with_postgres=True) postgres_dict_table_template = """ CREATE TABLE IF NOT EXISTS {} ( @@ -18,11 +21,12 @@ click_dict_table_template = """ ) ENGINE = Dictionary({}) """ -def get_postgres_conn(database=False): +def get_postgres_conn(port=5432, database=False): if database == True: - conn_string = "host='localhost' dbname='clickhouse' user='postgres' password='mysecretpassword'" + conn_string = "host='localhost' port={} dbname='clickhouse' user='postgres' password='mysecretpassword'".format(port) else: - conn_string = "host='localhost' user='postgres' password='mysecretpassword'" + conn_string = "host='localhost' port={} user='postgres' password='mysecretpassword'".format(port) + conn = psycopg2.connect(conn_string) conn.set_isolation_level(ISOLATION_LEVEL_AUTOCOMMIT) conn.autocommit = True @@ -32,15 +36,13 @@ def create_postgres_db(conn, name): cursor = conn.cursor() cursor.execute("CREATE DATABASE {}".format(name)) -def create_postgres_table(conn, table_name): - cursor = conn.cursor() +def create_postgres_table(cursor, table_name): cursor.execute(postgres_dict_table_template.format(table_name)) -def create_and_fill_postgres_table(table_name): - conn = get_postgres_conn(True) - create_postgres_table(conn, table_name) +def create_and_fill_postgres_table(cursor, table_name, host='postgres1', port=5432): + create_postgres_table(cursor, table_name) # Fill postgres table using clickhouse postgres table function and check - table_func = '''postgresql('postgres1:5432', 'clickhouse', '{}', 'postgres', 'mysecretpassword')'''.format(table_name) + table_func = '''postgresql('{}:{}', 'clickhouse', '{}', 'postgres', 'mysecretpassword')'''.format(host, port, table_name) node1.query('''INSERT INTO TABLE FUNCTION {} SELECT number, number from numbers(10000) '''.format(table_func, table_name)) result = node1.query("SELECT count() FROM {}".format(table_func)) @@ -54,10 +56,16 @@ def create_dict(table_name, index=0): def started_cluster(): try: cluster.start() - postgres_conn = get_postgres_conn() node1.query("CREATE DATABASE IF NOT EXISTS test") - print("postgres connected") + + postgres_conn = get_postgres_conn(port=5432) + print("postgres1 connected") create_postgres_db(postgres_conn, 'clickhouse') + + postgres_conn = get_postgres_conn(port=5441) + print("postgres2 connected") + create_postgres_db(postgres_conn, 'clickhouse') + yield cluster finally: @@ -65,10 +73,10 @@ def started_cluster(): def test_load_dictionaries(started_cluster): - conn = get_postgres_conn(True) + conn = get_postgres_conn(database=True) cursor = conn.cursor() table_name = 'test0' - create_and_fill_postgres_table(table_name) + create_and_fill_postgres_table(cursor, table_name) create_dict(table_name) dict_name = 'dict0' @@ -76,14 +84,17 @@ def test_load_dictionaries(started_cluster): assert node1.query("SELECT count() FROM `test`.`dict_table_{}`".format(table_name)).rstrip() == '10000' assert node1.query("SELECT dictGetUInt32('{}', 'id', toUInt64(0))".format(dict_name)) == '0\n' assert node1.query("SELECT dictGetUInt32('{}', 'value', toUInt64(9999))".format(dict_name)) == '9999\n' + cursor.execute("DROP TABLE IF EXISTS {}".format(table_name)) + node1.query("DROP TABLE IF EXISTS {}".format(table_name)) + node1.query("DROP DICTIONARY IF EXISTS {}".format(dict_name)) def test_invalidate_query(started_cluster): - conn = get_postgres_conn(True) + conn = get_postgres_conn(database=True) cursor = conn.cursor() table_name = 'test0' - create_and_fill_postgres_table(table_name) + create_and_fill_postgres_table(cursor, table_name) # invalidate query: SELECT value FROM test0 WHERE id = 0 dict_name = 'dict0' @@ -112,6 +123,39 @@ def test_invalidate_query(started_cluster): assert node1.query("SELECT dictGetUInt32('{}', 'value', toUInt64(0))".format(dict_name)) == '2\n' assert node1.query("SELECT dictGetUInt32('{}', 'value', toUInt64(1))".format(dict_name)) == '2\n' + node1.query("DROP TABLE IF EXISTS {}".format(table_name)) + node1.query("DROP DICTIONARY IF EXISTS {}".format(dict_name)) + + +def test_dictionary_with_replicas(started_cluster): + conn1 = get_postgres_conn(port=5432, database=True) + cursor1 = conn1.cursor() + conn2 = get_postgres_conn(port=5441, database=True) + cursor2 = conn2.cursor() + + create_postgres_table(cursor1, 'test1') + create_postgres_table(cursor2, 'test1') + + cursor1.execute('INSERT INTO test1 select i, i from generate_series(0, 99) as t(i);'); + cursor2.execute('INSERT INTO test1 select i, i from generate_series(100, 199) as t(i);'); + + create_dict('test1', 1) + result = node1.query("SELECT * FROM `test`.`dict_table_test1` ORDER BY id") + + # priority 0 - non running port + assert node1.contains_in_log('Unable to setup connection to postgres2:5433*') + + # priority 1 - postgres2, table contains rows with values 100-200 + # priority 2 - postgres1, table contains rows with values 0-100 + expected = node1.query("SELECT number, number FROM numbers(100, 100)") + assert(result == expected) + + cursor1.execute("DROP TABLE IF EXISTS test1") + cursor2.execute("DROP TABLE IF EXISTS test1") + + node1.query("DROP TABLE IF EXISTS test1") + node1.query("DROP DICTIONARY IF EXISTS dict1") + if __name__ == '__main__': cluster.start() From 1606c7e3f3aafd041009b9686a224e3a063b50c7 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Sun, 14 Mar 2021 12:24:17 +0300 Subject: [PATCH 425/544] Fix typo --- base/glibc-compatibility/memcpy/memcpy.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/base/glibc-compatibility/memcpy/memcpy.h b/base/glibc-compatibility/memcpy/memcpy.h index 60303e4f544..f9f81bcb0fe 100644 --- a/base/glibc-compatibility/memcpy/memcpy.h +++ b/base/glibc-compatibility/memcpy/memcpy.h @@ -63,7 +63,7 @@ * Very large size of memcpy typically indicates suboptimal (not cache friendly) algorithms in code or unrealistic scenarios, * so we don't pay attention to using non-temporary stores. * - * On recent Intel CPUs, the presense of "erms" makes "rep movsb" the most benefitial, + * On recent Intel CPUs, the presence of "erms" makes "rep movsb" the most benefitial, * even comparing to non-temporary aligned unrolled stores even with the most wide registers. * * memcpy can be written in asm, C or C++. The latter can also use inline asm. From 797063ae0fae95c3a79e8ccab1d3b19bee8e843d Mon Sep 17 00:00:00 2001 From: kssenii Date: Sun, 14 Mar 2021 10:35:10 +0000 Subject: [PATCH 426/544] Support non-default table schema for postgres storage/table-function --- src/Storages/StoragePostgreSQL.cpp | 20 ++++++++++++------ src/Storages/StoragePostgreSQL.h | 4 +++- .../TableFunctionPostgreSQL.cpp | 16 +++++++++----- src/TableFunctions/TableFunctionPostgreSQL.h | 2 +- .../test_storage_postgresql/test.py | 21 +++++++++++++++++++ 5 files changed, 50 insertions(+), 13 deletions(-) diff --git a/src/Storages/StoragePostgreSQL.cpp b/src/Storages/StoragePostgreSQL.cpp index 78ec8c34e41..3e122ed2fc7 100644 --- a/src/Storages/StoragePostgreSQL.cpp +++ b/src/Storages/StoragePostgreSQL.cpp @@ -45,9 +45,11 @@ StoragePostgreSQL::StoragePostgreSQL( PostgreSQLConnectionPtr connection_, const ColumnsDescription & columns_, const ConstraintsDescription & constraints_, - const Context & context_) + const Context & context_, + const String & remote_table_schema_) : IStorage(table_id_) , remote_table_name(remote_table_name_) + , remote_table_schema(remote_table_schema_) , global_context(context_) , connection(std::move(connection_)) { @@ -69,9 +71,11 @@ Pipe StoragePostgreSQL::read( { metadata_snapshot->check(column_names_, getVirtuals(), getStorageID()); + /// Connection is already made to the needed database, so it should not be present in the query; + /// remote_table_schema is empty if it is not specified, will access only table_name. String query = transformQueryForExternalDatabase( query_info_, metadata_snapshot->getColumns().getOrdinary(), - IdentifierQuotingStyle::DoubleQuotes, "", remote_table_name, context_); + IdentifierQuotingStyle::DoubleQuotes, remote_table_schema, remote_table_name, context_); Block sample_block; for (const String & column_name : column_names_) @@ -293,9 +297,9 @@ void registerStoragePostgreSQL(StorageFactory & factory) { ASTs & engine_args = args.engine_args; - if (engine_args.size() != 5) - throw Exception("Storage PostgreSQL requires 5 parameters: " - "PostgreSQL('host:port', 'database', 'table', 'username', 'password'.", + if (engine_args.size() < 5 || engine_args.size() > 6) + throw Exception("Storage PostgreSQL requires 5-6 parameters: " + "PostgreSQL('host:port', 'database', 'table', 'username', 'password' [, 'schema']", ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH); for (auto & engine_arg : engine_args) @@ -304,6 +308,10 @@ void registerStoragePostgreSQL(StorageFactory & factory) auto parsed_host_port = parseAddress(engine_args[0]->as().value.safeGet(), 5432); const String & remote_table = engine_args[2]->as().value.safeGet(); + String remote_table_schema; + if (engine_args.size() == 6) + remote_table_schema = engine_args[5]->as().value.safeGet(); + auto connection = std::make_shared( engine_args[1]->as().value.safeGet(), parsed_host_port.first, @@ -312,7 +320,7 @@ void registerStoragePostgreSQL(StorageFactory & factory) engine_args[4]->as().value.safeGet()); return StoragePostgreSQL::create( - args.table_id, remote_table, connection, args.columns, args.constraints, args.context); + args.table_id, remote_table, connection, args.columns, args.constraints, args.context, remote_table_schema); }, { .source_access_type = AccessType::POSTGRES, diff --git a/src/Storages/StoragePostgreSQL.h b/src/Storages/StoragePostgreSQL.h index 8aebae5896b..0d574c9e98e 100644 --- a/src/Storages/StoragePostgreSQL.h +++ b/src/Storages/StoragePostgreSQL.h @@ -28,7 +28,8 @@ public: PostgreSQLConnectionPtr connection_, const ColumnsDescription & columns_, const ConstraintsDescription & constraints_, - const Context & context_); + const Context & context_, + const std::string & remote_table_schema_ = ""); String getName() const override { return "PostgreSQL"; } @@ -47,6 +48,7 @@ private: friend class PostgreSQLBlockOutputStream; String remote_table_name; + String remote_table_schema; Context global_context; PostgreSQLConnectionPtr connection; }; diff --git a/src/TableFunctions/TableFunctionPostgreSQL.cpp b/src/TableFunctions/TableFunctionPostgreSQL.cpp index eefdff1fa87..0e3f1c5da24 100644 --- a/src/TableFunctions/TableFunctionPostgreSQL.cpp +++ b/src/TableFunctions/TableFunctionPostgreSQL.cpp @@ -29,7 +29,7 @@ StoragePtr TableFunctionPostgreSQL::executeImpl(const ASTPtr & /*ast_function*/, auto columns = getActualTableStructure(context); auto result = std::make_shared( StorageID(getDatabaseName(), table_name), remote_table_name, - connection, columns, ConstraintsDescription{}, context); + connection, columns, ConstraintsDescription{}, context, remote_table_schema); result->startup(); return result; @@ -39,7 +39,10 @@ StoragePtr TableFunctionPostgreSQL::executeImpl(const ASTPtr & /*ast_function*/, ColumnsDescription TableFunctionPostgreSQL::getActualTableStructure(const Context & context) const { const bool use_nulls = context.getSettingsRef().external_table_functions_use_nulls; - auto columns = fetchPostgreSQLTableStructure(connection->conn(), remote_table_name, use_nulls); + auto columns = fetchPostgreSQLTableStructure( + connection->conn(), + remote_table_schema.empty() ? remote_table_name : remote_table_schema + '.' + remote_table_name, + use_nulls); return ColumnsDescription{*columns}; } @@ -54,9 +57,9 @@ void TableFunctionPostgreSQL::parseArguments(const ASTPtr & ast_function, const ASTs & args = func_args.arguments->children; - if (args.size() != 5) - throw Exception("Table function 'PostgreSQL' requires 5 parameters: " - "PostgreSQL('host:port', 'database', 'table', 'user', 'password').", + if (args.size() < 5 || args.size() > 6) + throw Exception("Table function 'PostgreSQL' requires 5-6 parameters: " + "PostgreSQL('host:port', 'database', 'table', 'user', 'password', [, 'schema']).", ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH); for (auto & arg : args) @@ -65,6 +68,9 @@ void TableFunctionPostgreSQL::parseArguments(const ASTPtr & ast_function, const auto parsed_host_port = parseAddress(args[0]->as().value.safeGet(), 5432); remote_table_name = args[2]->as().value.safeGet(); + if (args.size() == 6) + remote_table_schema = args[5]->as().value.safeGet(); + connection = std::make_shared( args[1]->as().value.safeGet(), parsed_host_port.first, diff --git a/src/TableFunctions/TableFunctionPostgreSQL.h b/src/TableFunctions/TableFunctionPostgreSQL.h index e625cbd9bf6..92e061e18ca 100644 --- a/src/TableFunctions/TableFunctionPostgreSQL.h +++ b/src/TableFunctions/TableFunctionPostgreSQL.h @@ -30,7 +30,7 @@ private: void parseArguments(const ASTPtr & ast_function, const Context & context) override; String connection_str; - String remote_table_name; + String remote_table_name, remote_table_schema; PostgreSQLConnectionPtr connection; }; diff --git a/tests/integration/test_storage_postgresql/test.py b/tests/integration/test_storage_postgresql/test.py index cee495438a2..58f3233bacc 100644 --- a/tests/integration/test_storage_postgresql/test.py +++ b/tests/integration/test_storage_postgresql/test.py @@ -132,6 +132,27 @@ def test_postgres_conversions(started_cluster): assert(result == expected) +def test_non_default_scema(started_cluster): + conn = get_postgres_conn(True) + cursor = conn.cursor() + cursor.execute('CREATE SCHEMA test_schema') + cursor.execute('CREATE TABLE test_schema.test_table (a integer)') + cursor.execute('INSERT INTO test_schema.test_table SELECT i FROM generate_series(0, 99) as t(i)') + + node1.query(''' + CREATE TABLE test_pg_table_schema (a UInt32) + ENGINE PostgreSQL('postgres1:5432', 'clickhouse', 'test_table', 'postgres', 'mysecretpassword', 'test_schema'); + ''') + + result = node1.query('SELECT * FROM test_pg_table_schema') + expected = node1.query('SELECT number FROM numbers(100)') + assert(result == expected) + + table_function = '''postgresql('postgres1:5432', 'clickhouse', 'test_table', 'postgres', 'mysecretpassword', 'test_schema')''' + result = node1.query('SELECT * FROM {}'.format(table_function)) + assert(result == expected) + + if __name__ == '__main__': cluster.start() input("Cluster created, press any key to destroy...") From 6e69299191e10588500b32510dc0f1e9ac4d1471 Mon Sep 17 00:00:00 2001 From: kssenii Date: Sun, 14 Mar 2021 10:43:23 +0000 Subject: [PATCH 427/544] Fix style --- src/Storages/PostgreSQL/PostgreSQLReplicaConnection.h | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/Storages/PostgreSQL/PostgreSQLReplicaConnection.h b/src/Storages/PostgreSQL/PostgreSQLReplicaConnection.h index 9b5dec5e9a5..289183d8451 100644 --- a/src/Storages/PostgreSQL/PostgreSQLReplicaConnection.h +++ b/src/Storages/PostgreSQL/PostgreSQLReplicaConnection.h @@ -1,3 +1,5 @@ +#pragma once + #include "PostgreSQLConnection.h" #include #include From 583c78143de1367afea36ebb8aafd4f78b4a8e92 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Sun, 14 Mar 2021 14:08:08 +0300 Subject: [PATCH 428/544] Remove useless CMake option --- CMakeLists.txt | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 7c3571f1118..d310f7c298c 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -240,9 +240,7 @@ else() message(STATUS "Disabling compiler -pipe option (have only ${AVAILABLE_PHYSICAL_MEMORY} mb of memory)") endif() -if(NOT DISABLE_CPU_OPTIMIZE) - include(cmake/cpu_features.cmake) -endif() +include(cmake/cpu_features.cmake) option(ARCH_NATIVE "Add -march=native compiler flag") From d273fa241a750d264c8964ead464e5f575977a7b Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Sun, 14 Mar 2021 14:24:03 +0300 Subject: [PATCH 429/544] Remove trash --- website/locale/fa/LC_MESSAGES/messages.mo | Bin 424 -> 0 bytes website/locale/fa/LC_MESSAGES/messages.po | 325 --------------------- website/locale/tr/LC_MESSAGES/messages.mo | Bin 431 -> 0 bytes website/locale/tr/LC_MESSAGES/messages.po | 326 ---------------------- 4 files changed, 651 deletions(-) delete mode 100644 website/locale/fa/LC_MESSAGES/messages.mo delete mode 100644 website/locale/fa/LC_MESSAGES/messages.po delete mode 100644 website/locale/tr/LC_MESSAGES/messages.mo delete mode 100644 website/locale/tr/LC_MESSAGES/messages.po diff --git a/website/locale/fa/LC_MESSAGES/messages.mo b/website/locale/fa/LC_MESSAGES/messages.mo deleted file mode 100644 index 89c73f3fea4a8629ec01407c3ee36a9301bbd7fd..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 424 zcmaJ+%TB{E5G)cWA31aIfg5abf&`VTO7sy>P(!0M>fKGYA;`5O+d+i?;P?0z91xTf zE3LG%JDQpOI5~bF_&V}^>iet*f9k@|Z~h4VE;afY3kPut8#K1oiqkY(Jw)@IZlY`* zuO3N;jW!MvTh%4Z+R8$%IgOHOyu6(*7Fo1jlXR8C+@Nq?0T#j`ry&a&uxkizXb|#{ zowF;(JP|T{se7k?c}rm(lZ@Gj#jA9c$<pLlFHTy33uA;tzR`?&`yFe{T?TbBykcQ{tt$X%(Zf;e1fj=$_`)MWg|tc#_SuTQ!|1Cc6pS*wcE(YAKv{wF8QI!@=DZQzXRVyk7wE9*#x>4I!G9e`f# z<)#s}96<3Tv>_X|c7^?&uM4~)jgc%=S<8w;e_31a{wFSuwj4%cNeu)SeMjhh0%yQ| AGynhq diff --git a/website/locale/tr/LC_MESSAGES/messages.po b/website/locale/tr/LC_MESSAGES/messages.po deleted file mode 100644 index 710ebbdf120..00000000000 --- a/website/locale/tr/LC_MESSAGES/messages.po +++ /dev/null @@ -1,326 +0,0 @@ -# Translations template for PROJECT. -# Copyright (C) 2020 ORGANIZATION -# This file is distributed under the same license as the PROJECT project. -# Automatically generated, 2020. -# -msgid "" -msgstr "" -"Project-Id-Version: PROJECT VERSION\n" -"Report-Msgid-Bugs-To: EMAIL@ADDRESS\n" -"POT-Creation-Date: 2020-06-17 12:20+0300\n" -"PO-Revision-Date: 2020-06-17 12:20+0300\n" -"Last-Translator: Automatically generated\n" -"Language-Team: none\n" -"MIME-Version: 1.0\n" -"Content-Type: text/plain; charset=UTF-8\n" -"Content-Transfer-Encoding: 8bit\n" -"Generated-By: Babel 2.8.0\n" -"Language: tr\n" -"Plural-Forms: nplurals=2; plural=(n != 1);\n" - -#: templates/common_meta.html:1 -msgid "" -"ClickHouse is a fast open-source column-oriented database management system " -"that allows generating analytical data reports in real-time using SQL queries" -msgstr "" - -#: templates/common_meta.html:6 -msgid "ClickHouse - fast open-source OLAP DBMS" -msgstr "" - -#: templates/common_meta.html:10 -msgid "ClickHouse DBMS" -msgstr "" - -#: templates/common_meta.html:32 -msgid "open-source" -msgstr "" - -#: templates/common_meta.html:32 -msgid "relational" -msgstr "" - -#: templates/common_meta.html:32 -msgid "analytics" -msgstr "" - -#: templates/common_meta.html:32 -msgid "analytical" -msgstr "" - -#: templates/common_meta.html:32 -msgid "Big Data" -msgstr "" - -#: templates/common_meta.html:32 -msgid "web-analytics" -msgstr "" - -#: templates/footer.html:8 -msgid "ClickHouse source code is published under the Apache 2.0 License." -msgstr "" - -#: templates/footer.html:8 -msgid "" -"Software is distributed on an \"AS IS\" BASIS, WITHOUT WARRANTIES OR " -"CONDITIONS OF ANY KIND, either express or implied." -msgstr "" - -#: templates/footer.html:11 -msgid "Yandex LLC" -msgstr "" - -#: templates/blog/content.html:20 templates/blog/content.html:25 -#: templates/blog/content.html:30 -msgid "Share on" -msgstr "" - -#: templates/blog/content.html:37 -msgid "Published date" -msgstr "" - -#: templates/blog/nav.html:20 -msgid "New post" -msgstr "" - -#: templates/blog/nav.html:25 -msgid "Documentation" -msgstr "" - -#: templates/docs/footer.html:3 -msgid "Rating" -msgstr "" - -#: templates/docs/footer.html:3 -msgid "votes" -msgstr "" - -#: templates/docs/footer.html:4 -msgid "Article Rating" -msgstr "" - -#: templates/docs/footer.html:4 -msgid "Was this content helpful?" -msgstr "" - -#: templates/docs/footer.html:7 -msgid "Unusable" -msgstr "" - -#: templates/docs/footer.html:7 -msgid "Poor" -msgstr "" - -#: templates/docs/footer.html:7 -msgid "Good" -msgstr "" - -#: templates/docs/footer.html:7 -msgid "Excellent" -msgstr "" - -#: templates/docs/footer.html:8 -msgid "documentation" -msgstr "" - -#: templates/docs/footer.html:15 -msgid "Built from" -msgstr "" - -#: templates/docs/footer.html:15 -msgid "published on" -msgstr "" - -#: templates/docs/footer.html:15 -msgid "modified on" -msgstr "" - -#: templates/docs/machine-translated.html:3 -msgid "Help wanted!" -msgstr "" - -#: templates/docs/machine-translated.html:4 -msgid "" -"The following content of this documentation page has been machine-" -"translated. But unlike other websites, it is not done on the fly. This " -"translated text lives on GitHub repository alongside main ClickHouse " -"codebase and waits for fellow native speakers to make it more human-readable." -msgstr "" - -#: templates/docs/machine-translated.html:4 -msgid "You can also use the original English version as a reference." -msgstr "" - -#: templates/docs/machine-translated.html:7 -msgid "Help ClickHouse documentation by editing this page" -msgstr "" - -#: templates/docs/sidebar.html:3 -msgid "Multi-page or single-page" -msgstr "" - -#: templates/docs/sidebar.html:5 -msgid "Multi-page version" -msgstr "" - -#: templates/docs/sidebar.html:8 -msgid "Single-page version" -msgstr "" - -#: templates/docs/sidebar.html:13 -msgid "Version" -msgstr "" - -#: templates/docs/sidebar.html:13 templates/docs/sidebar.html:19 -msgid "latest" -msgstr "" - -#: templates/docs/sidebar.html:36 -msgid "PDF version" -msgstr "" - -#: templates/docs/toc.html:8 -msgid "Table of Contents" -msgstr "" - -#: templates/index/community.html:4 -msgid "ClickHouse community" -msgstr "" - -#: templates/index/community.html:13 templates/index/community.html:14 -msgid "ClickHouse YouTube Channel" -msgstr "" - -#: templates/index/community.html:25 templates/index/community.html:26 -msgid "ClickHouse Official Twitter Account" -msgstr "" - -#: templates/index/community.html:36 templates/index/community.html:37 -msgid "ClickHouse at Telegram" -msgstr "" - -#: templates/index/community.html:41 -msgid "Chat with real users in " -msgstr "" - -#: templates/index/community.html:44 templates/index/community.html:116 -msgid "English" -msgstr "" - -#: templates/index/community.html:45 -msgid "or in" -msgstr "" - -#: templates/index/community.html:47 templates/index/community.html:117 -msgid "Russian" -msgstr "" - -#: templates/index/community.html:65 -msgid "Open GitHub issue to ask for help or to file a feature request" -msgstr "" - -#: templates/index/community.html:76 templates/index/community.html:77 -msgid "ClickHouse Slack Workspace" -msgstr "" - -#: templates/index/community.html:82 -msgid "Multipurpose public hangout" -msgstr "" - -#: templates/index/community.html:101 -msgid "Ask any questions" -msgstr "" - -#: templates/index/community.html:115 -msgid "ClickHouse Blog" -msgstr "" - -#: templates/index/community.html:116 -msgid "in" -msgstr "" - -#: templates/index/community.html:128 templates/index/community.html:129 -msgid "ClickHouse at Google Groups" -msgstr "" - -#: templates/index/community.html:133 -msgid "Email discussions" -msgstr "" - -#: templates/index/community.html:142 -msgid "Like ClickHouse?" -msgstr "" - -#: templates/index/community.html:143 -msgid "Help to spread the word about it via" -msgstr "" - -#: templates/index/community.html:144 -msgid "and" -msgstr "" - -#: templates/index/community.html:153 -msgid "Hosting ClickHouse Meetups" -msgstr "" - -#: templates/index/community.html:157 -msgid "" -"ClickHouse meetups are essential for strengthening community worldwide, but " -"they couldn't be possible without the help of local organizers. Please, fill " -"this form if you want to become one or want to meet ClickHouse core team for " -"any other reason." -msgstr "" - -#: templates/index/community.html:159 -msgid "ClickHouse Meetup" -msgstr "" - -#: templates/index/community.html:165 -msgid "Name" -msgstr "" - -#: templates/index/community.html:168 -msgid "Email" -msgstr "" - -#: templates/index/community.html:171 -msgid "Company" -msgstr "" - -#: templates/index/community.html:174 -msgid "City" -msgstr "" - -#: templates/index/community.html:179 -msgid "We'd like to host a public ClickHouse Meetup" -msgstr "" - -#: templates/index/community.html:185 -msgid "We'd like to invite Yandex ClickHouse team to our office" -msgstr "" - -#: templates/index/community.html:191 -msgid "We'd like to invite Yandex ClickHouse team to another event we organize" -msgstr "" - -#: templates/index/community.html:197 -msgid "We're interested in commercial consulting, support or managed service" -msgstr "" - -#: templates/index/community.html:201 -msgid "Additional comments" -msgstr "" - -#: templates/index/community.html:203 -msgid "Send" -msgstr "" - -#: templates/index/community.html:212 -msgid "" -"If you have any more thoughts or questions, feel free to contact Yandex " -"ClickHouse team directly at" -msgstr "" - -#: templates/index/community.html:213 -msgid "turn on JavaScript to see email address" -msgstr "" From 1ac62970c40d50f54f404855950ebfac5587c3f4 Mon Sep 17 00:00:00 2001 From: Maksim Kita Date: Sun, 14 Mar 2021 14:29:02 +0300 Subject: [PATCH 430/544] Modified SharedLibrary interface --- src/Common/SharedLibrary.cpp | 13 ++++++------ src/Common/SharedLibrary.h | 8 ++++---- .../LibraryDictionarySourceExternal.cpp | 20 +++++++++---------- .../LibraryDictionarySourceExternal.h | 20 +++++++++---------- 4 files changed, 31 insertions(+), 30 deletions(-) diff --git a/src/Common/SharedLibrary.cpp b/src/Common/SharedLibrary.cpp index 689179be7d8..9b81f74eb23 100644 --- a/src/Common/SharedLibrary.cpp +++ b/src/Common/SharedLibrary.cpp @@ -13,11 +13,11 @@ namespace ErrorCodes extern const int CANNOT_DLSYM; } -SharedLibrary::SharedLibrary(const std::string & path, int flags) +SharedLibrary::SharedLibrary(const std::string_view & path, int flags) { - handle = dlopen(path.c_str(), flags); + handle = dlopen(path.data(), flags); if (!handle) - throw Exception(std::string("Cannot dlopen: ") + dlerror(), ErrorCodes::CANNOT_DLOPEN); + throw Exception(ErrorCodes::CANNOT_DLOPEN, "Cannot dlopen: ({})", dlerror()); updatePHDRCache(); @@ -31,17 +31,18 @@ SharedLibrary::~SharedLibrary() std::terminate(); } -void * SharedLibrary::getImpl(const std::string & name, bool no_throw) +void * SharedLibrary::getImpl(const std::string_view & name, bool no_throw) { dlerror(); - auto * res = dlsym(handle, name.c_str()); + auto * res = dlsym(handle, name.data()); if (char * error = dlerror()) { if (no_throw) return nullptr; - throw Exception(std::string("Cannot dlsym: ") + error, ErrorCodes::CANNOT_DLSYM); + + throw Exception(ErrorCodes::CANNOT_DLSYM, "Cannot dlsym: ({})", error); } return res; diff --git a/src/Common/SharedLibrary.h b/src/Common/SharedLibrary.h index 9d2b9bc7843..e665c335c6f 100644 --- a/src/Common/SharedLibrary.h +++ b/src/Common/SharedLibrary.h @@ -14,23 +14,23 @@ namespace DB class SharedLibrary : private boost::noncopyable { public: - explicit SharedLibrary(const std::string & path, int flags = RTLD_LAZY); + explicit SharedLibrary(const std::string_view & path, int flags = RTLD_LAZY); ~SharedLibrary(); template - Func get(const std::string & name) + Func get(const std::string_view & name) { return reinterpret_cast(getImpl(name)); } template - Func tryGet(const std::string & name) + Func tryGet(const std::string_view & name) { return reinterpret_cast(getImpl(name, true)); } private: - void * getImpl(const std::string & name, bool no_throw = false); + void * getImpl(const std::string_view & name, bool no_throw = false); void * handle = nullptr; }; diff --git a/src/Dictionaries/LibraryDictionarySourceExternal.cpp b/src/Dictionaries/LibraryDictionarySourceExternal.cpp index eba088c2c55..259d0a2846a 100644 --- a/src/Dictionaries/LibraryDictionarySourceExternal.cpp +++ b/src/Dictionaries/LibraryDictionarySourceExternal.cpp @@ -9,19 +9,19 @@ const char DICT_LOGGER_NAME[] = "LibraryDictionarySourceExternal"; namespace ClickHouseLibrary { -std::string LIBRARY_CREATE_NEW_FUNC_NAME = "ClickHouseDictionary_v3_libNew"; -std::string LIBRARY_CLONE_FUNC_NAME = "ClickHouseDictionary_v3_libClone"; -std::string LIBRARY_DELETE_FUNC_NAME = "ClickHouseDictionary_v3_libDelete"; +std::string_view LIBRARY_CREATE_NEW_FUNC_NAME = "ClickHouseDictionary_v3_libNew"; +std::string_view LIBRARY_CLONE_FUNC_NAME = "ClickHouseDictionary_v3_libClone"; +std::string_view LIBRARY_DELETE_FUNC_NAME = "ClickHouseDictionary_v3_libDelete"; -std::string LIBRARY_DATA_NEW_FUNC_NAME = "ClickHouseDictionary_v3_dataNew"; -std::string LIBRARY_DATA_DELETE_FUNC_NAME = "ClickHouseDictionary_v3_dataDelete"; +std::string_view LIBRARY_DATA_NEW_FUNC_NAME = "ClickHouseDictionary_v3_dataNew"; +std::string_view LIBRARY_DATA_DELETE_FUNC_NAME = "ClickHouseDictionary_v3_dataDelete"; -std::string LIBRARY_LOAD_ALL_FUNC_NAME = "ClickHouseDictionary_v3_loadAll"; -std::string LIBRARY_LOAD_IDS_FUNC_NAME = "ClickHouseDictionary_v3_loadIds"; -std::string LIBRARY_LOAD_KEYS_FUNC_NAME = "ClickHouseDictionary_v3_loadKeys"; +std::string_view LIBRARY_LOAD_ALL_FUNC_NAME = "ClickHouseDictionary_v3_loadAll"; +std::string_view LIBRARY_LOAD_IDS_FUNC_NAME = "ClickHouseDictionary_v3_loadIds"; +std::string_view LIBRARY_LOAD_KEYS_FUNC_NAME = "ClickHouseDictionary_v3_loadKeys"; -std::string LIBRARY_IS_MODIFIED_FUNC_NAME = "ClickHouseDictionary_v3_isModified"; -std::string LIBRARY_SUPPORTS_SELECTIVE_LOAD_FUNC_NAME = "ClickHouseDictionary_v3_supportsSelectiveLoad"; +std::string_view LIBRARY_IS_MODIFIED_FUNC_NAME = "ClickHouseDictionary_v3_isModified"; +std::string_view LIBRARY_SUPPORTS_SELECTIVE_LOAD_FUNC_NAME = "ClickHouseDictionary_v3_supportsSelectiveLoad"; void log(LogLevel level, CString msg) { diff --git a/src/Dictionaries/LibraryDictionarySourceExternal.h b/src/Dictionaries/LibraryDictionarySourceExternal.h index 64a5f678578..3b92707d091 100644 --- a/src/Dictionaries/LibraryDictionarySourceExternal.h +++ b/src/Dictionaries/LibraryDictionarySourceExternal.h @@ -63,19 +63,19 @@ enum LogLevel void log(LogLevel level, CString msg); -extern std::string LIBRARY_CREATE_NEW_FUNC_NAME; -extern std::string LIBRARY_CLONE_FUNC_NAME; -extern std::string LIBRARY_DELETE_FUNC_NAME; +extern std::string_view LIBRARY_CREATE_NEW_FUNC_NAME; +extern std::string_view LIBRARY_CLONE_FUNC_NAME; +extern std::string_view LIBRARY_DELETE_FUNC_NAME; -extern std::string LIBRARY_DATA_NEW_FUNC_NAME; -extern std::string LIBRARY_DATA_DELETE_FUNC_NAME; +extern std::string_view LIBRARY_DATA_NEW_FUNC_NAME; +extern std::string_view LIBRARY_DATA_DELETE_FUNC_NAME; -extern std::string LIBRARY_LOAD_ALL_FUNC_NAME; -extern std::string LIBRARY_LOAD_IDS_FUNC_NAME; -extern std::string LIBRARY_LOAD_KEYS_FUNC_NAME; +extern std::string_view LIBRARY_LOAD_ALL_FUNC_NAME; +extern std::string_view LIBRARY_LOAD_IDS_FUNC_NAME; +extern std::string_view LIBRARY_LOAD_KEYS_FUNC_NAME; -extern std::string LIBRARY_IS_MODIFIED_FUNC_NAME; -extern std::string LIBRARY_SUPPORTS_SELECTIVE_LOAD_FUNC_NAME; +extern std::string_view LIBRARY_IS_MODIFIED_FUNC_NAME; +extern std::string_view LIBRARY_SUPPORTS_SELECTIVE_LOAD_FUNC_NAME; using LibraryContext = void *; From 0dcf8b4d55d0b3e91ada48c920f75d6b04cea69d Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Sun, 14 Mar 2021 14:39:14 +0300 Subject: [PATCH 431/544] Add .gitignore --- docs/tools/.gitignore | 2 ++ 1 file changed, 2 insertions(+) create mode 100644 docs/tools/.gitignore diff --git a/docs/tools/.gitignore b/docs/tools/.gitignore new file mode 100644 index 00000000000..7d75298b357 --- /dev/null +++ b/docs/tools/.gitignore @@ -0,0 +1,2 @@ +build +__pycache__ From ef4913d184d07bab697016fb6dc49657cf7084e6 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Sun, 14 Mar 2021 14:41:04 +0300 Subject: [PATCH 432/544] Removed harmful code that was added due to misconduct --- docs/tools/translate/add_meta_flag.py | 12 -- docs/tools/translate/babel-mapping.ini | 3 - docs/tools/translate/filter.py | 199 ------------------ docs/tools/translate/normalize-markdown.sh | 13 -- .../remove_machine_translated_meta.py | 21 -- .../translate/replace-with-translation.sh | 17 -- docs/tools/translate/requirements.txt | 12 -- docs/tools/translate/split_meta.py | 35 --- docs/tools/translate/translate.py | 80 ------- docs/tools/translate/translate.sh | 29 --- docs/tools/translate/typograph_ru.py | 45 ---- .../update-all-machine-translated.sh | 26 --- docs/tools/translate/update-po.sh | 22 -- docs/tools/translate/util.py | 1 - 14 files changed, 515 deletions(-) delete mode 100755 docs/tools/translate/add_meta_flag.py delete mode 100644 docs/tools/translate/babel-mapping.ini delete mode 100755 docs/tools/translate/filter.py delete mode 100755 docs/tools/translate/normalize-markdown.sh delete mode 100755 docs/tools/translate/remove_machine_translated_meta.py delete mode 100755 docs/tools/translate/replace-with-translation.sh delete mode 100644 docs/tools/translate/requirements.txt delete mode 100755 docs/tools/translate/split_meta.py delete mode 100755 docs/tools/translate/translate.py delete mode 100755 docs/tools/translate/translate.sh delete mode 100644 docs/tools/translate/typograph_ru.py delete mode 100755 docs/tools/translate/update-all-machine-translated.sh delete mode 100755 docs/tools/translate/update-po.sh delete mode 120000 docs/tools/translate/util.py diff --git a/docs/tools/translate/add_meta_flag.py b/docs/tools/translate/add_meta_flag.py deleted file mode 100755 index d87aa044faf..00000000000 --- a/docs/tools/translate/add_meta_flag.py +++ /dev/null @@ -1,12 +0,0 @@ -#!/usr/bin/env python3 - -import sys - -import util - -if __name__ == '__main__': - flag_name = sys.argv[1] - path = sys.argv[2] - meta, content = util.read_md_file(path) - meta[flag_name] = True - util.write_md_file(path, meta, content) diff --git a/docs/tools/translate/babel-mapping.ini b/docs/tools/translate/babel-mapping.ini deleted file mode 100644 index 6a9a3e5c073..00000000000 --- a/docs/tools/translate/babel-mapping.ini +++ /dev/null @@ -1,3 +0,0 @@ -[python: **.py] -[jinja2: **/templates/**.html] -extensions=jinja2.ext.i18n,jinja2.ext.autoescape,jinja2.ext.with_ diff --git a/docs/tools/translate/filter.py b/docs/tools/translate/filter.py deleted file mode 100755 index 61e1104d345..00000000000 --- a/docs/tools/translate/filter.py +++ /dev/null @@ -1,199 +0,0 @@ -#!/usr/bin/env python3 - -import os -import sys -import json.decoder - -import pandocfilters -import slugify - -import translate -import util - - -is_debug = os.environ.get('DEBUG') is not None - -filename = os.getenv('INPUT') - - -def debug(*args): - if is_debug: - print(*args, file=sys.stderr) - - -def process_buffer(buffer, new_value, item=None, is_header=False): - if buffer: - text = ''.join(buffer) - - try: - translated_text = translate.translate(text) - except TypeError: - translated_text = text - except json.decoder.JSONDecodeError as e: - print('Failed to translate', str(e), file=sys.stderr) - sys.exit(1) - - debug(f'Translate: "{text}" -> "{translated_text}"') - - if text and text[0].isupper() and not translated_text[0].isupper(): - translated_text = translated_text[0].upper() + translated_text[1:] - - if text.startswith(' ') and not translated_text.startswith(' '): - translated_text = ' ' + translated_text - - if text.endswith(' ') and not translated_text.endswith(' '): - translated_text = translated_text + ' ' - - if is_header and translated_text.endswith('.'): - translated_text = translated_text.rstrip('.') - - title_case = is_header and translate.default_target_language == 'en' and text[0].isupper() - title_case_whitelist = { - 'a', 'an', 'the', 'and', 'or', 'that', - 'of', 'on', 'for', 'from', 'with', 'to', 'in' - } - is_first_iteration = True - for token in translated_text.split(' '): - if title_case and token.isascii() and not token.isupper(): - if len(token) > 1 and token.lower() not in title_case_whitelist: - token = token[0].upper() + token[1:] - elif not is_first_iteration: - token = token.lower() - is_first_iteration = False - - new_value.append(pandocfilters.Str(token)) - new_value.append(pandocfilters.Space()) - - if item is None and len(new_value): - new_value.pop(len(new_value) - 1) - else: - new_value[-1] = item - elif item: - new_value.append(item) - - -def process_sentence(value, is_header=False): - new_value = [] - buffer = [] - for item in value: - if isinstance(item, list): - new_value.append([process_sentence(subitem, is_header) for subitem in item]) - continue - elif isinstance(item, dict): - t = item.get('t') - c = item.get('c') - if t == 'Str': - buffer.append(c) - elif t == 'Space': - buffer.append(' ') - elif t == 'DoubleQuote': - buffer.append('"') - else: - process_buffer(buffer, new_value, item, is_header) - buffer = [] - else: - new_value.append(item) - process_buffer(buffer, new_value, is_header=is_header) - return new_value - - -def translate_filter(key, value, _format, _): - if key not in ['Space', 'Str']: - debug(key, value) - try: - cls = getattr(pandocfilters, key) - except AttributeError: - return - - if key == 'Para' and value: - marker = value[0].get('c') - if isinstance(marker, str) and marker.startswith('!!!') and len(value) > 2: - # Admonition case - if marker != '!!!': - # Lost space after !!! case - value.insert(1, pandocfilters.Str(marker[3:])) - value.insert(1, pandocfilters.Space()) - value[0]['c'] = '!!!' - admonition_value = [] - remaining_para_value = [] - in_admonition = True - break_value = [pandocfilters.LineBreak(), pandocfilters.Str(' ' * 4)] - for item in value: - if in_admonition: - if item.get('t') == 'SoftBreak': - in_admonition = False - else: - admonition_value.append(item) - else: - if item.get('t') == 'SoftBreak': - remaining_para_value += break_value - else: - remaining_para_value.append(item) - - if admonition_value[-1].get('t') == 'Quoted': - text = process_sentence(admonition_value[-1]['c'][-1]) - text[0]['c'] = '"' + text[0]['c'] - text[-1]['c'] = text[-1]['c'] + '"' - admonition_value.pop(-1) - admonition_value += text - else: - text = admonition_value[-1].get('c') - if text: - text = translate.translate(text[0].upper() + text[1:]) - admonition_value.append(pandocfilters.Space()) - admonition_value.append(pandocfilters.Str(f'"{text}"')) - - return cls(admonition_value + break_value + process_sentence(remaining_para_value)) - else: - return cls(process_sentence(value)) - elif key == 'Plain' or key == 'Strong' or key == 'Emph': - return cls(process_sentence(value)) - elif key == 'Link': - try: - # Plain links case - if value[2][0] == value[1][0].get('c'): - return pandocfilters.Str(value[2][0]) - except IndexError: - pass - - value[1] = process_sentence(value[1]) - href = value[2][0] - if not (href.startswith('http') or href.startswith('#')): - anchor = None - attempts = 10 - if '#' in href: - href, anchor = href.split('#', 1) - if href.endswith('.md') and not href.startswith('/'): - parts = [part for part in os.environ['INPUT'].split('/') if len(part) == 2] - lang = parts[-1] - script_path = os.path.dirname(__file__) - base_path = os.path.abspath(f'{script_path}/../../{lang}') - href = os.path.join( - os.path.relpath(base_path, os.path.dirname(os.environ['INPUT'])), - os.path.relpath(href, base_path) - ) - if anchor: - href = f'{href}#{anchor}' - value[2][0] = href - return cls(*value) - elif key == 'Header': - if value[1][0].islower() and '_' not in value[1][0]: # Preserve some manually specified anchors - value[1][0] = slugify.slugify(value[1][0], separator='-', word_boundary=True, save_order=True) - - # TODO: title case header in en - value[2] = process_sentence(value[2], is_header=True) - return cls(*value) - elif key == 'SoftBreak': - return pandocfilters.LineBreak() - - return - - -if __name__ == "__main__": - os.environ['INPUT'] = os.path.abspath(os.environ['INPUT']) - pwd = os.path.dirname(filename or '.') - if pwd: - with util.cd(pwd): - pandocfilters.toJSONFilter(translate_filter) - else: - pandocfilters.toJSONFilter(translate_filter) diff --git a/docs/tools/translate/normalize-markdown.sh b/docs/tools/translate/normalize-markdown.sh deleted file mode 100755 index 7850fa34b1d..00000000000 --- a/docs/tools/translate/normalize-markdown.sh +++ /dev/null @@ -1,13 +0,0 @@ -#!/usr/bin/env bash -# Usage: normalize-en-markdown.sh -set -e -BASE_DIR=$(dirname $(readlink -f $0)) -TEMP_FILE=$(mktemp) -trap 'rm -f -- "${TEMP_FILE}"' INT TERM HUP EXIT -INPUT="$1" -if [[ ! -L "${INPUT}" ]] -then - export INPUT - cat "${INPUT}" > "${TEMP_FILE}" - "${BASE_DIR}/translate.sh" "en" "${TEMP_FILE}" "${INPUT}" -fi diff --git a/docs/tools/translate/remove_machine_translated_meta.py b/docs/tools/translate/remove_machine_translated_meta.py deleted file mode 100755 index 26cfde97f1e..00000000000 --- a/docs/tools/translate/remove_machine_translated_meta.py +++ /dev/null @@ -1,21 +0,0 @@ -#!/usr/bin/env python3 -import os -import sys -sys.path.insert(0, os.path.join(os.path.dirname(__file__), '..')) -import convert_toc -import util - - -if __name__ == '__main__': - path = sys.argv[1][2:] - convert_toc.init_redirects() - try: - path = convert_toc.redirects[path] - except KeyError: - pass - meta, content = util.read_md_file(path) - if 'machine_translated' in meta: - del meta['machine_translated'] - if 'machine_translated_rev' in meta: - del meta['machine_translated_rev'] - util.write_md_file(path, meta, content) diff --git a/docs/tools/translate/replace-with-translation.sh b/docs/tools/translate/replace-with-translation.sh deleted file mode 100755 index 922ac65a921..00000000000 --- a/docs/tools/translate/replace-with-translation.sh +++ /dev/null @@ -1,17 +0,0 @@ -#!/usr/bin/env bash -# Usage: replace-with-translation.sh -set -e -BASE_DIR=$(dirname $(readlink -f $0)) -TEMP_FILE=$(mktemp) -trap 'rm -f -- "${TEMP_FILE}"' INT TERM HUP EXIT -TARGET_LANGUAGE="$1" -export INPUT="$2" -cat "${INPUT}" > "${TEMP_FILE}" -if [[ ! -z $SLEEP ]] -then - sleep $[ ( $RANDOM % 20 ) + 1 ]s -fi -rm -f "${INPUT}" -mkdir -p $(dirname "${INPUT}") || true -YANDEX=1 "${BASE_DIR}/translate.sh" "${TARGET_LANGUAGE}" "${TEMP_FILE}" "${INPUT}" -git add "${INPUT}" diff --git a/docs/tools/translate/requirements.txt b/docs/tools/translate/requirements.txt deleted file mode 100644 index 1bbd119b823..00000000000 --- a/docs/tools/translate/requirements.txt +++ /dev/null @@ -1,12 +0,0 @@ -Babel==2.8.0 -certifi==2020.6.20 -chardet==3.0.4 -googletrans==3.0.0 -idna==2.10 -Jinja2==2.11.2 -pandocfilters==1.4.2 -python-slugify==4.0.1 -PyYAML==5.3.1 -requests==2.24.0 -text-unidecode==1.3 -urllib3==1.25.10 diff --git a/docs/tools/translate/split_meta.py b/docs/tools/translate/split_meta.py deleted file mode 100755 index b38b93e10b4..00000000000 --- a/docs/tools/translate/split_meta.py +++ /dev/null @@ -1,35 +0,0 @@ -#!/usr/bin/env python3 -import os -import subprocess -import sys - -import translate -import util - - -if __name__ == '__main__': - path = sys.argv[1] - content_path = f'{path}.content' - meta_path = f'{path}.meta' - meta, content = util.read_md_file(path) - - target_language = os.getenv('TARGET_LANGUAGE') - if target_language is not None and target_language != 'en': - rev = subprocess.check_output( - 'git rev-parse HEAD', shell=True - ).decode('utf-8').strip() - meta['machine_translated'] = True - meta['machine_translated_rev'] = rev - title = meta.get('toc_title') - if title: - meta['toc_title'] = translate.translate(title, target_language) - folder_title = meta.get('toc_folder_title') - if folder_title: - meta['toc_folder_title'] = translate.translate(folder_title, target_language) - if 'en_copy' in meta: - del meta['en_copy'] - - with open(content_path, 'w') as f: - print(content, file=f) - - util.write_md_file(meta_path, meta, '') diff --git a/docs/tools/translate/translate.py b/docs/tools/translate/translate.py deleted file mode 100755 index 605ff78f424..00000000000 --- a/docs/tools/translate/translate.py +++ /dev/null @@ -1,80 +0,0 @@ -#!/usr/bin/env python3 - -import os -import random -import re -import sys -import time -import urllib.parse - -import googletrans -import requests -import yaml - - -translator = googletrans.Translator() -default_target_language = os.environ.get('TARGET_LANGUAGE', 'ru') -curly_braces_re = re.compile('({[^}]+})') - -is_yandex = os.environ.get('YANDEX') is not None - - -def translate_impl(text, target_language=None): - target_language = target_language or default_target_language - if target_language == 'en': - return text - elif is_yandex: - text = text.replace('‘', '\'') - text = text.replace('’', '\'') - has_alpha = any([char.isalpha() for char in text]) - if text.isascii() and has_alpha and not text.isupper(): - text = urllib.parse.quote(text) - url = f'http://translate.yandex.net/api/v1/tr.json/translate?srv=docs&lang=en-{target_language}&text={text}' - result = requests.get(url).json() - if result.get('code') == 200: - return result['text'][0] - else: - result = str(result) - print(f'Failed to translate "{text}": {result}', file=sys.stderr) - sys.exit(1) - else: - return text - else: - time.sleep(random.random()) - return translator.translate(text, target_language).text - - -def translate(text, target_language=None): - return "".join( - [ - part - if part.startswith("{") and part.endswith("}") - else translate_impl(part, target_language=target_language) - for part in re.split(curly_braces_re, text) - ] - ) - - -def translate_po(): - import babel.messages.pofile - base_dir = os.path.join(os.path.dirname(__file__), '..', '..', '..', 'website', 'locale') - for lang in ['en', 'zh', 'es', 'fr', 'ru', 'ja']: - po_path = os.path.join(base_dir, lang, 'LC_MESSAGES', 'messages.po') - with open(po_path, 'r') as f: - po_file = babel.messages.pofile.read_po(f, locale=lang, domain='messages') - for item in po_file: - if not item.string: - global is_yandex - is_yandex = True - item.string = translate(item.id, lang) - with open(po_path, 'wb') as f: - babel.messages.pofile.write_po(f, po_file) - - -if __name__ == '__main__': - target_language = sys.argv[1] - if target_language == 'po': - translate_po() - else: - result = translate_toc(yaml.full_load(sys.stdin.read())['nav'], sys.argv[1]) - print(yaml.dump({'nav': result})) diff --git a/docs/tools/translate/translate.sh b/docs/tools/translate/translate.sh deleted file mode 100755 index 1acf645eb81..00000000000 --- a/docs/tools/translate/translate.sh +++ /dev/null @@ -1,29 +0,0 @@ -#!/usr/bin/env bash -# Usage: translate.sh -set -e -BASE_DIR=$(dirname $(readlink -f $0)) -OUTPUT=${3:-/dev/stdout} -export TARGET_LANGUAGE="$1" -export DEBUG -TEMP_FILE=$(mktemp) -export INPUT_PATH="$2" -INPUT_META="${INPUT_PATH}.meta" -INPUT_CONTENT="${INPUT_PATH}.content" - -trap 'rm -f -- "${TEMP_FILE}" "${INPUT_META}" "${INPUT_CONTENT}"' INT TERM HUP EXIT -source "${BASE_DIR}/venv/bin/activate" - -${BASE_DIR}/split_meta.py "${INPUT_PATH}" - -pandoc "${INPUT_CONTENT}" --filter "${BASE_DIR}/filter.py" -o "${TEMP_FILE}" \ - -f "markdown-space_in_atx_header" -t "markdown_strict+pipe_tables+markdown_attribute+all_symbols_escapable+backtick_code_blocks+autolink_bare_uris-link_attributes+markdown_attribute+mmd_link_attributes-raw_attribute+header_attributes-grid_tables+definition_lists" \ - --atx-headers --wrap=none --columns=99999 --tab-stop=4 -perl -pi -e 's/{\\#\\#/{##/g' "${TEMP_FILE}" -perl -pi -e 's/\\#\\#}/##}/g' "${TEMP_FILE}" -perl -pi -e 's/ *$//gg' "${TEMP_FILE}" -if [[ "${TARGET_LANGUAGE}" == "ru" ]] -then - perl -pi -e 's/“/«/gg' "${TEMP_FILE}" - perl -pi -e 's/”/»/gg' "${TEMP_FILE}" -fi -cat "${INPUT_META}" "${TEMP_FILE}" > "${OUTPUT}" diff --git a/docs/tools/translate/typograph_ru.py b/docs/tools/translate/typograph_ru.py deleted file mode 100644 index 2d970cf2a2e..00000000000 --- a/docs/tools/translate/typograph_ru.py +++ /dev/null @@ -1,45 +0,0 @@ -import requests - -class TypographError(Exception): - pass - - -def typograph(text): - text = text.replace('&', '&') - text = text.replace('<', '<') - text = text.replace('>', '>') - template = f''' - - - - {text} - 3 - 0 - 0 - 0 - - - - ''' - result = requests.post( - url='http://typograf.artlebedev.ru/webservices/typograf.asmx', - data=template.encode('utf-8'), - headers={ - 'Content-Type': 'text/xml', - 'SOAPAction': 'http://typograf.artlebedev.ru/webservices/ProcessText' - } - ) - if result.ok and 'ProcessTextResult' in result.text: - result_text = result.text.split('')[1].split('')[0].rstrip() - result_text = result_text.replace('&', '&') - result_text = result_text.replace('<', '<') - result_text = result_text.replace('>', '>') - return result_text - else: - raise TypographError(result.text) - - -if __name__ == '__main__': - import sys - print((typograph(sys.stdin.read()))) diff --git a/docs/tools/translate/update-all-machine-translated.sh b/docs/tools/translate/update-all-machine-translated.sh deleted file mode 100755 index fae2aae787f..00000000000 --- a/docs/tools/translate/update-all-machine-translated.sh +++ /dev/null @@ -1,26 +0,0 @@ -#!/usr/bin/env bash -BASE_DIR=$(dirname $(readlink -f $0)) - -function translate() { - set -x - LANGUAGE=$1 - DOCS_ROOT="${BASE_DIR}/../../" - REV="$(git rev-parse HEAD)" - for FILENAME in $(find "${DOCS_ROOT}${LANGUAGE}" -name "*.md" -type f) - do - HAS_MT_TAG=$(grep -c "machine_translated: true" "${FILENAME}") - IS_UP_TO_DATE=$(grep -c "machine_translated_rev: \"${REV}\"" "${FILENAME}") - if [ "${HAS_MT_TAG}" -eq "1" ] && [ "${IS_UP_TO_DATE}" -eq "0" ] - then - set -e - EN_FILENAME=${FILENAME/\/${LANGUAGE}\///en/} - rm "${FILENAME}" || true - cp "${EN_FILENAME}" "${FILENAME}" - DEBUG=1 SLEEP=1 ${BASE_DIR}/replace-with-translation.sh ${LANGUAGE} "${FILENAME}" - set +e - fi - done -} -export BASE_DIR -export -f translate -parallel translate ::: es fr zh ja fa tr diff --git a/docs/tools/translate/update-po.sh b/docs/tools/translate/update-po.sh deleted file mode 100755 index f2f4039bcb8..00000000000 --- a/docs/tools/translate/update-po.sh +++ /dev/null @@ -1,22 +0,0 @@ -#!/usr/bin/env bash -# Usage: update-po.sh -set -ex -BASE_DIR=$(dirname $(readlink -f $0)) -WEBSITE_DIR="${BASE_DIR}/../../../website" -LOCALE_DIR="${WEBSITE_DIR}/locale" -MESSAGES_POT="${LOCALE_DIR}/messages.pot" -BABEL_INI="${BASE_DIR}/babel-mapping.ini" -LANGS="en zh es fr ru ja tr fa" -source "${BASE_DIR}/venv/bin/activate" -cd "${WEBSITE_DIR}" -pybabel extract "." -o "${MESSAGES_POT}" -F "${BABEL_INI}" -for L in ${LANGS} -do - pybabel update -d locale -l "${L}" -i "${MESSAGES_POT}" || \ - pybabel init -d locale -l "${L}" -i "${MESSAGES_POT}" -done -python3 "${BASE_DIR}/translate.py" po -for L in ${LANGS} -do - pybabel compile -d locale -l "${L}" -done diff --git a/docs/tools/translate/util.py b/docs/tools/translate/util.py deleted file mode 120000 index 7f16d68497e..00000000000 --- a/docs/tools/translate/util.py +++ /dev/null @@ -1 +0,0 @@ -../util.py \ No newline at end of file From b65cc841e41dd034c61f1be30b1f1ec557ce16bf Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Sun, 14 Mar 2021 14:45:47 +0300 Subject: [PATCH 433/544] Remove a test that cannot be fixed - less garbage in CI check output --- docs/tools/test.py | 15 ++++++--------- 1 file changed, 6 insertions(+), 9 deletions(-) diff --git a/docs/tools/test.py b/docs/tools/test.py index 7d11157c986..00d1d47137f 100755 --- a/docs/tools/test.py +++ b/docs/tools/test.py @@ -68,17 +68,17 @@ def test_single_page(input_path, lang): f, features='html.parser' ) + anchor_points = set() + duplicate_anchor_points = 0 links_to_nowhere = 0 + for tag in soup.find_all(): for anchor_point in [tag.attrs.get('name'), tag.attrs.get('id')]: if anchor_point: - if anchor_point in anchor_points: - duplicate_anchor_points += 1 - logging.info('Duplicate anchor point: %s' % anchor_point) - else: - anchor_points.add(anchor_point) + anchor_points.add(anchor_point) + for tag in soup.find_all(): href = tag.attrs.get('href') if href and href.startswith('#') and href != '#': @@ -87,11 +87,8 @@ def test_single_page(input_path, lang): logging.info("Tag %s", tag) logging.info('Link to nowhere: %s' % href) - if duplicate_anchor_points: - logging.warning('Found %d duplicate anchor points' % duplicate_anchor_points) - if links_to_nowhere: - if lang == 'en' or lang == 'ru': # TODO: check all languages again + if lang == 'en' or lang == 'ru': logging.error(f'Found {links_to_nowhere} links to nowhere in {lang}') sys.exit(1) else: From 78fba611e0880eca9dee8709df2f435889069359 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Sun, 14 Mar 2021 14:48:36 +0300 Subject: [PATCH 434/544] Remove trash --- docs/tools/make_links.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/tools/make_links.sh b/docs/tools/make_links.sh index 743d4eebf16..c1194901f8f 100755 --- a/docs/tools/make_links.sh +++ b/docs/tools/make_links.sh @@ -8,7 +8,7 @@ BASE_DIR=$(dirname $(readlink -f $0)) function do_make_links() { set -x - langs=(en es zh fr ru ja tr fa) + langs=(en es zh fr ru ja) src_file="$1" for lang in "${langs[@]}" do From cb865ebe604c0c3c930988825f75ea4cdbdd56ec Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Sun, 14 Mar 2021 14:49:53 +0300 Subject: [PATCH 435/544] Removed strange file --- docs/tools/output.md | 204 ------------------------------------------- 1 file changed, 204 deletions(-) delete mode 100644 docs/tools/output.md diff --git a/docs/tools/output.md b/docs/tools/output.md deleted file mode 100644 index 91ec6e75999..00000000000 --- a/docs/tools/output.md +++ /dev/null @@ -1,204 +0,0 @@ -# What is ClickHouse? {#what-is-clickhouse} - -ClickHouse is a column-oriented database management system (DBMS) for -online analytical processing of queries (OLAP). - -In a “normal” row-oriented DBMS, data is stored in this order: - - Row WatchID JavaEnable Title GoodEvent EventTime - ----- ------------- ------------ -------------------- ----------- --------------------- - #0 89354350662 1 Investor Relations 1 2016-05-18 05:19:20 - #1 90329509958 0 Contact us 1 2016-05-18 08:10:20 - #2 89953706054 1 Mission 1 2016-05-18 07:38:00 - #N ... ... ... ... ... - -In other words, all the values related to a row are physically stored -next to each other. - -Examples of a row-oriented DBMS are MySQL, Postgres, and MS SQL Server. -{: .grey } - -In a column-oriented DBMS, data is stored like this: - - Row: #0 #1 #2 #N - ------------- --------------------- --------------------- --------------------- ----- - WatchID: 89354350662 90329509958 89953706054 ... - JavaEnable: 1 0 1 ... - Title: Investor Relations Contact us Mission ... - GoodEvent: 1 1 1 ... - EventTime: 2016-05-18 05:19:20 2016-05-18 08:10:20 2016-05-18 07:38:00 ... - -These examples only show the order that data is arranged in. The values -from different columns are stored separately, and data from the same -column is stored together. - -Examples of a column-oriented DBMS: Vertica, Paraccel (Actian Matrix and -Amazon Redshift), Sybase IQ, Exasol, Infobright, InfiniDB, MonetDB -(VectorWise and Actian Vector), LucidDB, SAP HANA, Google Dremel, Google -PowerDrill, Druid, and kdb+. {: .grey } - -Different orders for storing data are better suited to different -scenarios. The data access scenario refers to what queries are made, how -often, and in what proportion; how much data is read for each type of -query – rows, columns, and bytes; the relationship between reading and -updating data; the working size of the data and how locally it is used; -whether transactions are used, and how isolated they are; requirements -for data replication and logical integrity; requirements for latency and -throughput for each type of query, and so on. - -The higher the load on the system, the more important it is to customize -the system set up to match the requirements of the usage scenario, and -the more fine grained this customization becomes. There is no system -that is equally well-suited to significantly different scenarios. If a -system is adaptable to a wide set of scenarios, under a high load, the -system will handle all the scenarios equally poorly, or will work well -for just one or few of possible scenarios. - -## Key Properties of the OLAP scenario {#key-properties-of-the-olap-scenario} - -- The vast majority of requests are for read access. -- Data is updated in fairly large batches (\> 1000 rows), not by - single rows; or it is not updated at all. -- Data is added to the DB but is not modified. -- For reads, quite a large number of rows are extracted from the DB, - but only a small subset of columns. -- Tables are “wide,” meaning they contain a large number of columns. -- Queries are relatively rare (usually hundreds of queries per server - or less per second). -- For simple queries, latencies around 50 ms are allowed. -- Column values are fairly small: numbers and short strings (for - example, 60 bytes per URL). -- Requires high throughput when processing a single query (up to - billions of rows per second per server). -- Transactions are not necessary. -- Low requirements for data consistency. -- There is one large table per query. All tables are small, except for - one. -- A query result is significantly smaller than the source data. In - other words, data is filtered or aggregated, so the result fits in a - single server’s RAM. - -It is easy to see that the OLAP scenario is very different from other -popular scenarios (such as OLTP or Key-Value access). So it doesn’t make -sense to try to use OLTP or a Key-Value DB for processing analytical -queries if you want to get decent performance. For example, if you try -to use MongoDB or Redis for analytics, you will get very poor -performance compared to OLAP databases. - -## Why Column-Oriented Databases Work Better in the OLAP Scenario {#why-column-oriented-databases-work-better-in-the-olap-scenario} - -Column-oriented databases are better suited to OLAP scenarios: they are -at least 100 times faster in processing most queries. The reasons are -explained in detail below, but the fact is easier to demonstrate -visually: - -**Row-oriented DBMS** - -![Row-oriented](images/row_oriented.gif#) - -**Column-oriented DBMS** - -![Column-oriented](images/column_oriented.gif#) - -See the difference? - -### Input/output {#inputoutput} - -1. For an analytical query, only a small number of table columns need - to be read. In a column-oriented database, you can read just the - data you need. For example, if you need 5 columns out of 100, you - can expect a 20-fold reduction in I/O. -2. Since data is read in packets, it is easier to compress. Data in - columns is also easier to compress. This further reduces the I/O - volume. -3. Due to the reduced I/O, more data fits in the system cache. - -For example, the query “count the number of records for each advertising -platform” requires reading one “advertising platform ID” column, which -takes up 1 byte uncompressed. If most of the traffic was not from -advertising platforms, you can expect at least 10-fold compression of -this column. When using a quick compression algorithm, data -decompression is possible at a speed of at least several gigabytes of -uncompressed data per second. In other words, this query can be -processed at a speed of approximately several billion rows per second on -a single server. This speed is actually achieved in practice. - -
- -Example - - $ clickhouse-client - ClickHouse client version 0.0.52053. - Connecting to localhost:9000. - Connected to ClickHouse server version 0.0.52053. - - :) SELECT CounterID, count() FROM hits GROUP BY CounterID ORDER BY count() DESC LIMIT 20 - - SELECT - CounterID, - count() - FROM hits - GROUP BY CounterID - ORDER BY count() DESC - LIMIT 20 - - ┌─CounterID─┬──count()─┐ - │ 114208 │ 56057344 │ - │ 115080 │ 51619590 │ - │ 3228 │ 44658301 │ - │ 38230 │ 42045932 │ - │ 145263 │ 42042158 │ - │ 91244 │ 38297270 │ - │ 154139 │ 26647572 │ - │ 150748 │ 24112755 │ - │ 242232 │ 21302571 │ - │ 338158 │ 13507087 │ - │ 62180 │ 12229491 │ - │ 82264 │ 12187441 │ - │ 232261 │ 12148031 │ - │ 146272 │ 11438516 │ - │ 168777 │ 11403636 │ - │ 4120072 │ 11227824 │ - │ 10938808 │ 10519739 │ - │ 74088 │ 9047015 │ - │ 115079 │ 8837972 │ - │ 337234 │ 8205961 │ - └───────────┴──────────┘ - - 20 rows in set. Elapsed: 0.153 sec. Processed 1.00 billion rows, 4.00 GB (6.53 billion rows/s., 26.10 GB/s.) - - :) - -
- -### CPU {#cpu} - -Since executing a query requires processing a large number of rows, it -helps to dispatch all operations for entire vectors instead of for -separate rows, or to implement the query engine so that there is almost -no dispatching cost. If you don’t do this, with any half-decent disk -subsystem, the query interpreter inevitably stalls the CPU. It makes -sense to both store data in columns and process it, when possible, by -columns. - -There are two ways to do this: - -1. A vector engine. All operations are written for vectors, instead of - for separate values. This means you don’t need to call operations - very often, and dispatching costs are negligible. Operation code - contains an optimized internal cycle. - -2. Code generation. The code generated for the query has all the - indirect calls in it. - -This is not done in “normal” databases, because it doesn’t make sense -when running simple queries. However, there are exceptions. For example, -MemSQL uses code generation to reduce latency when processing SQL -queries. (For comparison, analytical DBMSs require optimization of -throughput, not latency.) - -Note that for CPU efficiency, the query language must be declarative -(SQL or MDX), or at least a vector (J, K). The query should only contain -implicit loops, allowing for optimization. - -[Original article](https://clickhouse.tech/docs/en/) From 4820498231d7597be25fb04279ba5988c4712104 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Sun, 14 Mar 2021 14:53:27 +0300 Subject: [PATCH 436/544] Update gitignore --- docs/tools/.gitignore | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/tools/.gitignore b/docs/tools/.gitignore index 7d75298b357..443cee8638c 100644 --- a/docs/tools/.gitignore +++ b/docs/tools/.gitignore @@ -1,2 +1,3 @@ build __pycache__ +*.pyc From 3f35e686edc730220f31a5310d2327b59ae2585b Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Sun, 14 Mar 2021 15:01:03 +0300 Subject: [PATCH 437/544] Add .gitignore --- docs/.gitignore | 1 + website/README.md | 6 ++++++ 2 files changed, 7 insertions(+) create mode 100644 docs/.gitignore diff --git a/docs/.gitignore b/docs/.gitignore new file mode 100644 index 00000000000..378eac25d31 --- /dev/null +++ b/docs/.gitignore @@ -0,0 +1 @@ +build diff --git a/website/README.md b/website/README.md index c4383bea24c..a09a00379d1 100644 --- a/website/README.md +++ b/website/README.md @@ -22,3 +22,9 @@ virtualenv build ``` ./build.py --skip-multi-page --skip-single-page --skip-amp --skip-pdf --skip-git-log --skip-docs --skip-test-templates --livereload 8080 ``` + +# How to quickly test the ugly annoying broken links in docs + +``` +./build.py --skip-multi-page --skip-amp --skip-pdf --skip-blog --skip-git-log --skip-test-templates --lang en --livereload 8080 +``` From bab7e9be9afed99bc1659a456427fab957a1ac57 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Sun, 14 Mar 2021 15:18:10 +0300 Subject: [PATCH 438/544] Fix some broken links --- docs/en/getting-started/playground.md | 8 ++++---- docs/en/interfaces/formats.md | 12 ++++++------ 2 files changed, 10 insertions(+), 10 deletions(-) diff --git a/docs/en/getting-started/playground.md b/docs/en/getting-started/playground.md index 7838dad14ea..9adf0423cf3 100644 --- a/docs/en/getting-started/playground.md +++ b/docs/en/getting-started/playground.md @@ -38,10 +38,10 @@ The queries are executed as a read-only user. It implies some limitations: The following settings are also enforced: -- [max_result_bytes=10485760](../operations/settings/query_complexity/#max-result-bytes) -- [max_result_rows=2000](../operations/settings/query_complexity/#setting-max_result_rows) -- [result_overflow_mode=break](../operations/settings/query_complexity/#result-overflow-mode) -- [max_execution_time=60000](../operations/settings/query_complexity/#max-execution-time) +- [max_result_bytes=10485760](../operations/settings/query-complexity/#max-result-bytes) +- [max_result_rows=2000](../operations/settings/query-complexity/#setting-max_result_rows) +- [result_overflow_mode=break](../operations/settings/query-complexity/#result-overflow-mode) +- [max_execution_time=60000](../operations/settings/query-complexity/#max-execution-time) ## Examples {#examples} diff --git a/docs/en/interfaces/formats.md b/docs/en/interfaces/formats.md index 33bf90a8b52..ee2235b7861 100644 --- a/docs/en/interfaces/formats.md +++ b/docs/en/interfaces/formats.md @@ -1254,7 +1254,7 @@ ClickHouse supports configurable precision of `Decimal` type. The `INSERT` query Unsupported Parquet data types: `DATE32`, `TIME32`, `FIXED_SIZE_BINARY`, `JSON`, `UUID`, `ENUM`. -Data types of ClickHouse table columns can differ from the corresponding fields of the Parquet data inserted. When inserting data, ClickHouse interprets data types according to the table above and then [cast](../query_language/functions/type_conversion_functions/#type_conversion_function-cast) the data to that data type which is set for the ClickHouse table column. +Data types of ClickHouse table columns can differ from the corresponding fields of the Parquet data inserted. When inserting data, ClickHouse interprets data types according to the table above and then [cast](../sql-reference/functions/type-conversion-functions/#type_conversion_function-cast) the data to that data type which is set for the ClickHouse table column. ### Inserting and Selecting Data {#inserting-and-selecting-data} @@ -1359,15 +1359,15 @@ When working with the `Regexp` format, you can use the following settings: - Escaped (similarly to [TSV](#tabseparated)) - Quoted (similarly to [Values](#data-format-values)) - Raw (extracts subpatterns as a whole, no escaping rules) -- `format_regexp_skip_unmatched` — [UInt8](../sql-reference/data-types/int-uint.md). Defines the need to throw an exeption in case the `format_regexp` expression does not match the imported data. Can be set to `0` or `1`. +- `format_regexp_skip_unmatched` — [UInt8](../sql-reference/data-types/int-uint.md). Defines the need to throw an exeption in case the `format_regexp` expression does not match the imported data. Can be set to `0` or `1`. -**Usage** +**Usage** -The regular expression from `format_regexp` setting is applied to every line of imported data. The number of subpatterns in the regular expression must be equal to the number of columns in imported dataset. +The regular expression from `format_regexp` setting is applied to every line of imported data. The number of subpatterns in the regular expression must be equal to the number of columns in imported dataset. -Lines of the imported data must be separated by newline character `'\n'` or DOS-style newline `"\r\n"`. +Lines of the imported data must be separated by newline character `'\n'` or DOS-style newline `"\r\n"`. -The content of every matched subpattern is parsed with the method of corresponding data type, according to `format_regexp_escaping_rule` setting. +The content of every matched subpattern is parsed with the method of corresponding data type, according to `format_regexp_escaping_rule` setting. If the regular expression does not match the line and `format_regexp_skip_unmatched` is set to 1, the line is silently skipped. If `format_regexp_skip_unmatched` is set to 0, exception is thrown. From 8c9bd09e6c8ba8033fb08a6b3bb72a392f4e62df Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Sun, 14 Mar 2021 15:19:04 +0300 Subject: [PATCH 439/544] Remove broken "Original article" --- docs/en/sql-reference/aggregate-functions/combinators.md | 1 - docs/en/sql-reference/aggregate-functions/index.md | 1 - .../en/sql-reference/aggregate-functions/parametric-functions.md | 1 - .../external-dictionaries/external-dicts-dict-hierarchical.md | 1 - .../external-dictionaries/external-dicts-dict-layout.md | 1 - .../external-dictionaries/external-dicts-dict-lifetime.md | 1 - .../external-dictionaries/external-dicts-dict-structure.md | 1 - .../dictionaries/external-dictionaries/external-dicts-dict.md | 1 - .../dictionaries/external-dictionaries/external-dicts.md | 1 - docs/en/sql-reference/dictionaries/index.md | 1 - docs/en/sql-reference/dictionaries/internal-dicts.md | 1 - docs/en/sql-reference/functions/arithmetic-functions.md | 1 - docs/en/sql-reference/functions/array-functions.md | 1 - docs/en/sql-reference/functions/array-join.md | 1 - docs/en/sql-reference/functions/bit-functions.md | 1 - docs/en/sql-reference/functions/bitmap-functions.md | 1 - docs/en/sql-reference/functions/comparison-functions.md | 1 - docs/en/sql-reference/functions/conditional-functions.md | 1 - docs/en/sql-reference/functions/date-time-functions.md | 1 - docs/en/sql-reference/functions/encoding-functions.md | 1 - docs/en/sql-reference/functions/ext-dict-functions.md | 1 - docs/en/sql-reference/functions/functions-for-nulls.md | 1 - docs/en/sql-reference/functions/hash-functions.md | 1 - docs/en/sql-reference/functions/in-functions.md | 1 - docs/en/sql-reference/functions/index.md | 1 - docs/en/sql-reference/functions/introspection.md | 1 - docs/en/sql-reference/functions/ip-address-functions.md | 1 - docs/en/sql-reference/functions/json-functions.md | 1 - docs/en/sql-reference/functions/logical-functions.md | 1 - docs/en/sql-reference/functions/machine-learning-functions.md | 1 - docs/en/sql-reference/functions/math-functions.md | 1 - docs/en/sql-reference/functions/other-functions.md | 1 - docs/en/sql-reference/functions/random-functions.md | 1 - docs/en/sql-reference/functions/rounding-functions.md | 1 - docs/en/sql-reference/functions/splitting-merging-functions.md | 1 - docs/en/sql-reference/functions/string-functions.md | 1 - docs/en/sql-reference/functions/string-replace-functions.md | 1 - docs/en/sql-reference/functions/string-search-functions.md | 1 - docs/en/sql-reference/functions/type-conversion-functions.md | 1 - docs/en/sql-reference/functions/url-functions.md | 1 - docs/en/sql-reference/functions/uuid-functions.md | 1 - docs/en/sql-reference/functions/ym-dict-functions.md | 1 - docs/en/sql-reference/operators/index.md | 1 - docs/en/sql-reference/statements/alter/index.md | 1 - docs/en/sql-reference/statements/grant.md | 1 - docs/en/sql-reference/statements/insert-into.md | 1 - docs/en/sql-reference/statements/system.md | 1 - docs/en/sql-reference/table-functions/generate.md | 1 - docs/en/sql-reference/table-functions/hdfs.md | 1 - docs/en/sql-reference/table-functions/input.md | 1 - docs/en/sql-reference/table-functions/jdbc.md | 1 - docs/en/sql-reference/table-functions/merge.md | 1 - docs/en/sql-reference/table-functions/numbers.md | 1 - docs/en/sql-reference/table-functions/s3.md | 1 - 54 files changed, 54 deletions(-) diff --git a/docs/en/sql-reference/aggregate-functions/combinators.md b/docs/en/sql-reference/aggregate-functions/combinators.md index 015c90e90c7..cddef68d49c 100644 --- a/docs/en/sql-reference/aggregate-functions/combinators.md +++ b/docs/en/sql-reference/aggregate-functions/combinators.md @@ -250,4 +250,3 @@ FROM people ``` -[Original article](https://clickhouse.tech/docs/en/query_language/agg_functions/combinators/) diff --git a/docs/en/sql-reference/aggregate-functions/index.md b/docs/en/sql-reference/aggregate-functions/index.md index 543a5d3fed8..d2b46f6de53 100644 --- a/docs/en/sql-reference/aggregate-functions/index.md +++ b/docs/en/sql-reference/aggregate-functions/index.md @@ -59,4 +59,3 @@ SELECT groupArray(y) FROM t_null_big `groupArray` does not include `NULL` in the resulting array. -[Original article](https://clickhouse.tech/docs/en/query_language/agg_functions/) diff --git a/docs/en/sql-reference/aggregate-functions/parametric-functions.md b/docs/en/sql-reference/aggregate-functions/parametric-functions.md index c6c97b5428b..d059e0fc744 100644 --- a/docs/en/sql-reference/aggregate-functions/parametric-functions.md +++ b/docs/en/sql-reference/aggregate-functions/parametric-functions.md @@ -500,7 +500,6 @@ Problem: Generate a report that shows only keywords that produced at least 5 uni Solution: Write in the GROUP BY query SearchPhrase HAVING uniqUpTo(4)(UserID) >= 5 ``` -[Original article](https://clickhouse.tech/docs/en/query_language/agg_functions/parametric_functions/) ## sumMapFiltered(keys_to_keep)(keys, values) {#summapfilteredkeys-to-keepkeys-values} diff --git a/docs/en/sql-reference/dictionaries/external-dictionaries/external-dicts-dict-hierarchical.md b/docs/en/sql-reference/dictionaries/external-dictionaries/external-dicts-dict-hierarchical.md index a5e105d2e13..08d3b8d8ad0 100644 --- a/docs/en/sql-reference/dictionaries/external-dictionaries/external-dicts-dict-hierarchical.md +++ b/docs/en/sql-reference/dictionaries/external-dictionaries/external-dicts-dict-hierarchical.md @@ -65,4 +65,3 @@ For our example, the structure of dictionary can be the following: ``` -[Original article](https://clickhouse.tech/docs/en/query_language/dicts/external_dicts_dict_hierarchical/) diff --git a/docs/en/sql-reference/dictionaries/external-dictionaries/external-dicts-dict-layout.md b/docs/en/sql-reference/dictionaries/external-dictionaries/external-dicts-dict-layout.md index efef91b4b09..337586a2e10 100644 --- a/docs/en/sql-reference/dictionaries/external-dictionaries/external-dicts-dict-layout.md +++ b/docs/en/sql-reference/dictionaries/external-dictionaries/external-dicts-dict-layout.md @@ -445,4 +445,3 @@ Other types are not supported yet. The function returns the attribute for the pr Data must completely fit into RAM. -[Original article](https://clickhouse.tech/docs/en/query_language/dicts/external_dicts_dict_layout/) diff --git a/docs/en/sql-reference/dictionaries/external-dictionaries/external-dicts-dict-lifetime.md b/docs/en/sql-reference/dictionaries/external-dictionaries/external-dicts-dict-lifetime.md index 32763e27ddd..081cc5b0b69 100644 --- a/docs/en/sql-reference/dictionaries/external-dictionaries/external-dicts-dict-lifetime.md +++ b/docs/en/sql-reference/dictionaries/external-dictionaries/external-dicts-dict-lifetime.md @@ -86,4 +86,3 @@ SOURCE(ODBC(... invalidate_query 'SELECT update_time FROM dictionary_source wher ... ``` -[Original article](https://clickhouse.tech/docs/en/query_language/dicts/external_dicts_dict_lifetime/) diff --git a/docs/en/sql-reference/dictionaries/external-dictionaries/external-dicts-dict-structure.md b/docs/en/sql-reference/dictionaries/external-dictionaries/external-dicts-dict-structure.md index e25b3ab78c3..dbf2fa67ac5 100644 --- a/docs/en/sql-reference/dictionaries/external-dictionaries/external-dicts-dict-structure.md +++ b/docs/en/sql-reference/dictionaries/external-dictionaries/external-dicts-dict-structure.md @@ -170,4 +170,3 @@ Configuration fields: - [Functions for working with external dictionaries](../../../sql-reference/functions/ext-dict-functions.md). -[Original article](https://clickhouse.tech/docs/en/query_language/dicts/external_dicts_dict_structure/) diff --git a/docs/en/sql-reference/dictionaries/external-dictionaries/external-dicts-dict.md b/docs/en/sql-reference/dictionaries/external-dictionaries/external-dicts-dict.md index 17ad110aa19..e15d944130e 100644 --- a/docs/en/sql-reference/dictionaries/external-dictionaries/external-dicts-dict.md +++ b/docs/en/sql-reference/dictionaries/external-dictionaries/external-dicts-dict.md @@ -48,4 +48,3 @@ LIFETIME(...) -- Lifetime of dictionary in memory - [structure](../../../sql-reference/dictionaries/external-dictionaries/external-dicts-dict-structure.md) — Structure of the dictionary . A key and attributes that can be retrieved by this key. - [lifetime](../../../sql-reference/dictionaries/external-dictionaries/external-dicts-dict-lifetime.md) — Frequency of dictionary updates. -[Original article](https://clickhouse.tech/docs/en/query_language/dicts/external_dicts_dict/) diff --git a/docs/en/sql-reference/dictionaries/external-dictionaries/external-dicts.md b/docs/en/sql-reference/dictionaries/external-dictionaries/external-dicts.md index 99a62002822..8217fb8da3a 100644 --- a/docs/en/sql-reference/dictionaries/external-dictionaries/external-dicts.md +++ b/docs/en/sql-reference/dictionaries/external-dictionaries/external-dicts.md @@ -57,4 +57,3 @@ You can [configure](../../../sql-reference/dictionaries/external-dictionaries/ex - [Dictionary Key and Fields](../../../sql-reference/dictionaries/external-dictionaries/external-dicts-dict-structure.md) - [Functions for Working with External Dictionaries](../../../sql-reference/functions/ext-dict-functions.md) -[Original article](https://clickhouse.tech/docs/en/query_language/dicts/external_dicts/) diff --git a/docs/en/sql-reference/dictionaries/index.md b/docs/en/sql-reference/dictionaries/index.md index 420182642bb..fa127dab103 100644 --- a/docs/en/sql-reference/dictionaries/index.md +++ b/docs/en/sql-reference/dictionaries/index.md @@ -17,4 +17,3 @@ ClickHouse supports: - [Built-in dictionaries](../../sql-reference/dictionaries/internal-dicts.md#internal_dicts) with a specific [set of functions](../../sql-reference/functions/ym-dict-functions.md). - [Plug-in (external) dictionaries](../../sql-reference/dictionaries/external-dictionaries/external-dicts.md#dicts-external-dicts) with a [set of functions](../../sql-reference/functions/ext-dict-functions.md). -[Original article](https://clickhouse.tech/docs/en/query_language/dicts/) diff --git a/docs/en/sql-reference/dictionaries/internal-dicts.md b/docs/en/sql-reference/dictionaries/internal-dicts.md index 7d657d4177f..472351a19a4 100644 --- a/docs/en/sql-reference/dictionaries/internal-dicts.md +++ b/docs/en/sql-reference/dictionaries/internal-dicts.md @@ -50,4 +50,3 @@ We recommend periodically updating the dictionaries with the geobase. During an There are also functions for working with OS identifiers and Yandex.Metrica search engines, but they shouldn’t be used. -[Original article](https://clickhouse.tech/docs/en/query_language/dicts/internal_dicts/) diff --git a/docs/en/sql-reference/functions/arithmetic-functions.md b/docs/en/sql-reference/functions/arithmetic-functions.md index c4b151f59ce..faa03dfc9d3 100644 --- a/docs/en/sql-reference/functions/arithmetic-functions.md +++ b/docs/en/sql-reference/functions/arithmetic-functions.md @@ -82,4 +82,3 @@ An exception is thrown when dividing by zero or when dividing a minimal negative Returns the least common multiple of the numbers. An exception is thrown when dividing by zero or when dividing a minimal negative number by minus one. -[Original article](https://clickhouse.tech/docs/en/query_language/functions/arithmetic_functions/) diff --git a/docs/en/sql-reference/functions/array-functions.md b/docs/en/sql-reference/functions/array-functions.md index c9c418d57a4..9c2d37a0abb 100644 --- a/docs/en/sql-reference/functions/array-functions.md +++ b/docs/en/sql-reference/functions/array-functions.md @@ -1541,4 +1541,3 @@ SELECT arrayCumSumNonNegative([1, 1, -4, 1]) AS res ``` Note that the `arraySumNonNegative` is a [higher-order function](../../sql-reference/functions/index.md#higher-order-functions). You can pass a lambda function to it as the first argument. -[Original article](https://clickhouse.tech/docs/en/query_language/functions/array_functions/) diff --git a/docs/en/sql-reference/functions/array-join.md b/docs/en/sql-reference/functions/array-join.md index f1f9a545366..f35e0d10117 100644 --- a/docs/en/sql-reference/functions/array-join.md +++ b/docs/en/sql-reference/functions/array-join.md @@ -32,4 +32,3 @@ SELECT arrayJoin([1, 2, 3] AS src) AS dst, 'Hello', src └─────┴───────────┴─────────┘ ``` -[Original article](https://clickhouse.tech/docs/en/query_language/functions/array_join/) diff --git a/docs/en/sql-reference/functions/bit-functions.md b/docs/en/sql-reference/functions/bit-functions.md index a3d0c82d8ab..0a1ef737f2a 100644 --- a/docs/en/sql-reference/functions/bit-functions.md +++ b/docs/en/sql-reference/functions/bit-functions.md @@ -250,4 +250,3 @@ Result: └───────────────┘ ``` -[Original article](https://clickhouse.tech/docs/en/query_language/functions/bit_functions/) diff --git a/docs/en/sql-reference/functions/bitmap-functions.md b/docs/en/sql-reference/functions/bitmap-functions.md index bfff70576f2..16ae053f715 100644 --- a/docs/en/sql-reference/functions/bitmap-functions.md +++ b/docs/en/sql-reference/functions/bitmap-functions.md @@ -491,4 +491,3 @@ SELECT bitmapAndnotCardinality(bitmapBuild([1,2,3]),bitmapBuild([3,4,5])) AS res └─────┘ ``` -[Original article](https://clickhouse.tech/docs/en/query_language/functions/bitmap_functions/) diff --git a/docs/en/sql-reference/functions/comparison-functions.md b/docs/en/sql-reference/functions/comparison-functions.md index 0b6d8b6e36e..edaf0a01c73 100644 --- a/docs/en/sql-reference/functions/comparison-functions.md +++ b/docs/en/sql-reference/functions/comparison-functions.md @@ -32,4 +32,3 @@ Strings are compared by bytes. A shorter string is smaller than all strings that ## greaterOrEquals, \>= operator {#function-greaterorequals} -[Original article](https://clickhouse.tech/docs/en/query_language/functions/comparison_functions/) diff --git a/docs/en/sql-reference/functions/conditional-functions.md b/docs/en/sql-reference/functions/conditional-functions.md index 2d57cbb3bd5..70eba4156c8 100644 --- a/docs/en/sql-reference/functions/conditional-functions.md +++ b/docs/en/sql-reference/functions/conditional-functions.md @@ -202,4 +202,3 @@ FROM LEFT_RIGHT └──────┴───────┴──────────────────┘ ``` -[Original article](https://clickhouse.tech/docs/en/query_language/functions/conditional_functions/) diff --git a/docs/en/sql-reference/functions/date-time-functions.md b/docs/en/sql-reference/functions/date-time-functions.md index 304371f44eb..1ef116be617 100644 --- a/docs/en/sql-reference/functions/date-time-functions.md +++ b/docs/en/sql-reference/functions/date-time-functions.md @@ -1070,4 +1070,3 @@ Result: └────────────────────────────────────┘ ``` -[Original article](https://clickhouse.tech/docs/en/query_language/functions/date_time_functions/) diff --git a/docs/en/sql-reference/functions/encoding-functions.md b/docs/en/sql-reference/functions/encoding-functions.md index c1013ebb0e1..29286750240 100644 --- a/docs/en/sql-reference/functions/encoding-functions.md +++ b/docs/en/sql-reference/functions/encoding-functions.md @@ -172,4 +172,3 @@ Accepts an integer. Returns a string containing the list of powers of two that t Accepts an integer. Returns an array of UInt64 numbers containing the list of powers of two that total the source number when summed. Numbers in the array are in ascending order. -[Original article](https://clickhouse.tech/docs/en/query_language/functions/encoding_functions/) diff --git a/docs/en/sql-reference/functions/ext-dict-functions.md b/docs/en/sql-reference/functions/ext-dict-functions.md index 834fcdf8282..5fc146f603f 100644 --- a/docs/en/sql-reference/functions/ext-dict-functions.md +++ b/docs/en/sql-reference/functions/ext-dict-functions.md @@ -203,4 +203,3 @@ dictGet[Type]OrDefault('dict_name', 'attr_name', id_expr, default_value_expr) ClickHouse throws an exception if it cannot parse the value of the attribute or the value doesn’t match the attribute data type. -[Original article](https://clickhouse.tech/docs/en/query_language/functions/ext_dict_functions/) diff --git a/docs/en/sql-reference/functions/functions-for-nulls.md b/docs/en/sql-reference/functions/functions-for-nulls.md index f57f0f7e27d..dde0ef8ba93 100644 --- a/docs/en/sql-reference/functions/functions-for-nulls.md +++ b/docs/en/sql-reference/functions/functions-for-nulls.md @@ -309,4 +309,3 @@ SELECT toTypeName(toNullable(10)) └────────────────────────────┘ ``` -[Original article](https://clickhouse.tech/docs/en/query_language/functions/functions_for_nulls/) diff --git a/docs/en/sql-reference/functions/hash-functions.md b/docs/en/sql-reference/functions/hash-functions.md index 465ad01527f..2e78d5f4105 100644 --- a/docs/en/sql-reference/functions/hash-functions.md +++ b/docs/en/sql-reference/functions/hash-functions.md @@ -482,4 +482,3 @@ Result: - [xxHash](http://cyan4973.github.io/xxHash/). -[Original article](https://clickhouse.tech/docs/en/query_language/functions/hash_functions/) diff --git a/docs/en/sql-reference/functions/in-functions.md b/docs/en/sql-reference/functions/in-functions.md index dd3c1900fdc..c8936e74954 100644 --- a/docs/en/sql-reference/functions/in-functions.md +++ b/docs/en/sql-reference/functions/in-functions.md @@ -9,4 +9,3 @@ toc_title: IN Operator See the section [IN operators](../../sql-reference/operators/in.md#select-in-operators). -[Original article](https://clickhouse.tech/docs/en/query_language/functions/in_functions/) diff --git a/docs/en/sql-reference/functions/index.md b/docs/en/sql-reference/functions/index.md index 1a0b9d83b5f..32408759b98 100644 --- a/docs/en/sql-reference/functions/index.md +++ b/docs/en/sql-reference/functions/index.md @@ -84,4 +84,3 @@ Another example is the `hostName` function, which returns the name of the server If a function in a query is performed on the requestor server, but you need to perform it on remote servers, you can wrap it in an ‘any’ aggregate function or add it to a key in `GROUP BY`. -[Original article](https://clickhouse.tech/docs/en/query_language/functions/) diff --git a/docs/en/sql-reference/functions/introspection.md b/docs/en/sql-reference/functions/introspection.md index 964265a461b..29752ae00bf 100644 --- a/docs/en/sql-reference/functions/introspection.md +++ b/docs/en/sql-reference/functions/introspection.md @@ -369,4 +369,3 @@ Result: └──────────────────────────────┘ ``` -[Original article](https://clickhouse.tech/docs/en/query_language/functions/introspection/) diff --git a/docs/en/sql-reference/functions/ip-address-functions.md b/docs/en/sql-reference/functions/ip-address-functions.md index 64457627cce..ef17654295c 100644 --- a/docs/en/sql-reference/functions/ip-address-functions.md +++ b/docs/en/sql-reference/functions/ip-address-functions.md @@ -394,4 +394,3 @@ Result: └──────────────────┴────────────────────┘ ``` -[Original article](https://clickhouse.tech/docs/en/query_language/functions/ip_address_functions/) diff --git a/docs/en/sql-reference/functions/json-functions.md b/docs/en/sql-reference/functions/json-functions.md index edee048eb77..2b274ee912e 100644 --- a/docs/en/sql-reference/functions/json-functions.md +++ b/docs/en/sql-reference/functions/json-functions.md @@ -292,4 +292,3 @@ Result: └───────────────────────────────────────────────────────────────────────────────────────────────────────┘ ``` -[Original article](https://clickhouse.tech/docs/en/query_language/functions/json_functions/) diff --git a/docs/en/sql-reference/functions/logical-functions.md b/docs/en/sql-reference/functions/logical-functions.md index 13452f88a85..6cce0e4fff5 100644 --- a/docs/en/sql-reference/functions/logical-functions.md +++ b/docs/en/sql-reference/functions/logical-functions.md @@ -17,4 +17,3 @@ Zero as an argument is considered “false,” while any non-zero value is consi ## xor {#xor} -[Original article](https://clickhouse.tech/docs/en/query_language/functions/logical_functions/) diff --git a/docs/en/sql-reference/functions/machine-learning-functions.md b/docs/en/sql-reference/functions/machine-learning-functions.md index f103a4ea421..630a4465c82 100644 --- a/docs/en/sql-reference/functions/machine-learning-functions.md +++ b/docs/en/sql-reference/functions/machine-learning-functions.md @@ -94,4 +94,3 @@ Result: } ``` -[Original article](https://clickhouse.tech/docs/en/query_language/functions/machine-learning-functions/) diff --git a/docs/en/sql-reference/functions/math-functions.md b/docs/en/sql-reference/functions/math-functions.md index bfe973e3d96..54fbc03f7ae 100644 --- a/docs/en/sql-reference/functions/math-functions.md +++ b/docs/en/sql-reference/functions/math-functions.md @@ -477,4 +477,3 @@ Result: └──────────┘ ``` -[Original article](https://clickhouse.tech/docs/en/query_language/functions/math_functions/) diff --git a/docs/en/sql-reference/functions/other-functions.md b/docs/en/sql-reference/functions/other-functions.md index 2c7f8da881e..ce9d3a13221 100644 --- a/docs/en/sql-reference/functions/other-functions.md +++ b/docs/en/sql-reference/functions/other-functions.md @@ -1971,4 +1971,3 @@ Result: - [tcp_port](../../operations/server-configuration-parameters/settings.md#server_configuration_parameters-tcp_port) -[Original article](https://clickhouse.tech/docs/en/query_language/functions/other_functions/) diff --git a/docs/en/sql-reference/functions/random-functions.md b/docs/en/sql-reference/functions/random-functions.md index 2b9846344e4..aab9483de45 100644 --- a/docs/en/sql-reference/functions/random-functions.md +++ b/docs/en/sql-reference/functions/random-functions.md @@ -102,4 +102,3 @@ FROM numbers(3) │ aeca2A │ └───────────────────────────────────────┘ -[Original article](https://clickhouse.tech/docs/en/query_language/functions/random_functions/) diff --git a/docs/en/sql-reference/functions/rounding-functions.md b/docs/en/sql-reference/functions/rounding-functions.md index 83db1975366..4fb077f0be3 100644 --- a/docs/en/sql-reference/functions/rounding-functions.md +++ b/docs/en/sql-reference/functions/rounding-functions.md @@ -185,4 +185,3 @@ Accepts a number. If the number is less than 18, it returns 0. Otherwise, it rou Accepts a number and rounds it down to an element in the specified array. If the value is less than the lowest bound, the lowest bound is returned. -[Original article](https://clickhouse.tech/docs/en/query_language/functions/rounding_functions/) diff --git a/docs/en/sql-reference/functions/splitting-merging-functions.md b/docs/en/sql-reference/functions/splitting-merging-functions.md index c70ee20f076..bd7e209549c 100644 --- a/docs/en/sql-reference/functions/splitting-merging-functions.md +++ b/docs/en/sql-reference/functions/splitting-merging-functions.md @@ -150,4 +150,3 @@ Result: └───────────────────────────────────────────────────────────────────────┘ ``` -[Original article](https://clickhouse.tech/docs/en/query_language/functions/splitting_merging_functions/) diff --git a/docs/en/sql-reference/functions/string-functions.md b/docs/en/sql-reference/functions/string-functions.md index 2c08fa3acb7..91020ae795e 100644 --- a/docs/en/sql-reference/functions/string-functions.md +++ b/docs/en/sql-reference/functions/string-functions.md @@ -648,4 +648,3 @@ Result: - [List of XML and HTML character entity references](https://en.wikipedia.org/wiki/List_of_XML_and_HTML_character_entity_references) -[Original article](https://clickhouse.tech/docs/en/query_language/functions/string_functions/) diff --git a/docs/en/sql-reference/functions/string-replace-functions.md b/docs/en/sql-reference/functions/string-replace-functions.md index 8905500995c..144b4fbc1da 100644 --- a/docs/en/sql-reference/functions/string-replace-functions.md +++ b/docs/en/sql-reference/functions/string-replace-functions.md @@ -92,4 +92,3 @@ Predefined characters: `\0`, `\\`, `|`, `(`, `)`, `^`, `$`, `.`, `[`, `]`, `?`, This implementation slightly differs from re2::RE2::QuoteMeta. It escapes zero byte as `\0` instead of `\x00` and it escapes only required characters. For more information, see the link: [RE2](https://github.com/google/re2/blob/master/re2/re2.cc#L473) -[Original article](https://clickhouse.tech/docs/en/query_language/functions/string_replace_functions/) diff --git a/docs/en/sql-reference/functions/string-search-functions.md b/docs/en/sql-reference/functions/string-search-functions.md index 83b0edea438..050234be19b 100644 --- a/docs/en/sql-reference/functions/string-search-functions.md +++ b/docs/en/sql-reference/functions/string-search-functions.md @@ -773,4 +773,3 @@ Result: └───────────────────────────────┘ ``` -[Original article](https://clickhouse.tech/docs/en/query_language/functions/string_search_functions/) diff --git a/docs/en/sql-reference/functions/type-conversion-functions.md b/docs/en/sql-reference/functions/type-conversion-functions.md index 8a793b99ac9..ff4e8da6697 100644 --- a/docs/en/sql-reference/functions/type-conversion-functions.md +++ b/docs/en/sql-reference/functions/type-conversion-functions.md @@ -1210,4 +1210,3 @@ Result: └───────────────────────────────────────────┘ ``` -[Original article](https://clickhouse.tech/docs/en/query_language/functions/type_conversion_functions/) diff --git a/docs/en/sql-reference/functions/url-functions.md b/docs/en/sql-reference/functions/url-functions.md index 9e79ef2d0cb..f352fb4f74b 100644 --- a/docs/en/sql-reference/functions/url-functions.md +++ b/docs/en/sql-reference/functions/url-functions.md @@ -420,4 +420,3 @@ Removes the query string and fragment identifier. The question mark and number s Removes the ‘name’ URL parameter, if present. This function works under the assumption that the parameter name is encoded in the URL exactly the same way as in the passed argument. -[Original article](https://clickhouse.tech/docs/en/query_language/functions/url_functions/) diff --git a/docs/en/sql-reference/functions/uuid-functions.md b/docs/en/sql-reference/functions/uuid-functions.md index 01a61c65b67..e7e55c699cd 100644 --- a/docs/en/sql-reference/functions/uuid-functions.md +++ b/docs/en/sql-reference/functions/uuid-functions.md @@ -165,4 +165,3 @@ SELECT - [dictGetUUID](../../sql-reference/functions/ext-dict-functions.md#ext_dict_functions-other) -[Original article](https://clickhouse.tech/docs/en/query_language/functions/uuid_function/) diff --git a/docs/en/sql-reference/functions/ym-dict-functions.md b/docs/en/sql-reference/functions/ym-dict-functions.md index 56530b5e83b..9dff9a8cba8 100644 --- a/docs/en/sql-reference/functions/ym-dict-functions.md +++ b/docs/en/sql-reference/functions/ym-dict-functions.md @@ -150,4 +150,3 @@ Accepts a UInt32 number – the region ID from the Yandex geobase. A string with `ua` and `uk` both mean Ukrainian. -[Original article](https://clickhouse.tech/docs/en/query_language/functions/ym_dict_functions/) diff --git a/docs/en/sql-reference/operators/index.md b/docs/en/sql-reference/operators/index.md index 274f7269bc8..e073d5f23f0 100644 --- a/docs/en/sql-reference/operators/index.md +++ b/docs/en/sql-reference/operators/index.md @@ -296,4 +296,3 @@ SELECT * FROM t_null WHERE y IS NOT NULL └───┴───┘ ``` -[Original article](https://clickhouse.tech/docs/en/query_language/operators/) diff --git a/docs/en/sql-reference/statements/alter/index.md b/docs/en/sql-reference/statements/alter/index.md index 30603122096..71333e6fcce 100644 --- a/docs/en/sql-reference/statements/alter/index.md +++ b/docs/en/sql-reference/statements/alter/index.md @@ -47,4 +47,3 @@ For `ALTER ... ATTACH|DETACH|DROP` queries, you can use the `replication_alter_p For `ALTER TABLE ... UPDATE|DELETE` queries the synchronicity is defined by the [mutations_sync](../../../operations/settings/settings.md#mutations_sync) setting. -[Original article](https://clickhouse.tech/docs/en/query_language/alter/) diff --git a/docs/en/sql-reference/statements/grant.md b/docs/en/sql-reference/statements/grant.md index f3829de2fbb..0afc9b5b95f 100644 --- a/docs/en/sql-reference/statements/grant.md +++ b/docs/en/sql-reference/statements/grant.md @@ -473,4 +473,3 @@ Doesn’t grant any privileges. The `ADMIN OPTION` privilege allows a user to grant their role to another user. -[Original article](https://clickhouse.tech/docs/en/query_language/grant/) diff --git a/docs/en/sql-reference/statements/insert-into.md b/docs/en/sql-reference/statements/insert-into.md index c517a515ab7..66effcccc3f 100644 --- a/docs/en/sql-reference/statements/insert-into.md +++ b/docs/en/sql-reference/statements/insert-into.md @@ -117,4 +117,3 @@ Performance will not decrease if: - Data is added in real time. - You upload data that is usually sorted by time. -[Original article](https://clickhouse.tech/docs/en/query_language/insert_into/) diff --git a/docs/en/sql-reference/statements/system.md b/docs/en/sql-reference/statements/system.md index bb279703cc2..725024efe0c 100644 --- a/docs/en/sql-reference/statements/system.md +++ b/docs/en/sql-reference/statements/system.md @@ -277,4 +277,3 @@ SYSTEM RESTART REPLICA [db.]replicated_merge_tree_family_table_name Provides possibility to reinitialize Zookeeper sessions state for all `ReplicatedMergeTree` tables, will compare current state with Zookeeper as source of true and add tasks to Zookeeper queue if needed -[Original article](https://clickhouse.tech/docs/en/query_language/system/) diff --git a/docs/en/sql-reference/table-functions/generate.md b/docs/en/sql-reference/table-functions/generate.md index be6ba2b8bc4..fee2c80cc8f 100644 --- a/docs/en/sql-reference/table-functions/generate.md +++ b/docs/en/sql-reference/table-functions/generate.md @@ -39,4 +39,3 @@ SELECT * FROM generateRandom('a Array(Int8), d Decimal32(4), c Tuple(DateTime64( └──────────┴──────────────┴────────────────────────────────────────────────────────────────────┘ ``` -[Original article](https://clickhouse.tech/docs/en/query_language/table_functions/generate/) diff --git a/docs/en/sql-reference/table-functions/hdfs.md b/docs/en/sql-reference/table-functions/hdfs.md index 31e2000b22d..a7c3baca299 100644 --- a/docs/en/sql-reference/table-functions/hdfs.md +++ b/docs/en/sql-reference/table-functions/hdfs.md @@ -99,4 +99,3 @@ FROM hdfs('hdfs://hdfs1:9000/big_dir/file{0..9}{0..9}{0..9}', 'CSV', 'name Strin - [Virtual columns](../../engines/table-engines/index.md#table_engines-virtual_columns) -[Original article](https://clickhouse.tech/docs/en/query_language/table_functions/hdfs/) diff --git a/docs/en/sql-reference/table-functions/input.md b/docs/en/sql-reference/table-functions/input.md index 40f9f4f7f6f..17707b798d6 100644 --- a/docs/en/sql-reference/table-functions/input.md +++ b/docs/en/sql-reference/table-functions/input.md @@ -42,4 +42,3 @@ $ cat data.csv | clickhouse-client --query="INSERT INTO test FORMAT CSV" $ cat data.csv | clickhouse-client --query="INSERT INTO test SELECT * FROM input('test_structure') FORMAT CSV" ``` -[Original article](https://clickhouse.tech/docs/en/query_language/table_functions/input/) diff --git a/docs/en/sql-reference/table-functions/jdbc.md b/docs/en/sql-reference/table-functions/jdbc.md index 6fd53b0e794..c6df022c342 100644 --- a/docs/en/sql-reference/table-functions/jdbc.md +++ b/docs/en/sql-reference/table-functions/jdbc.md @@ -24,4 +24,3 @@ SELECT * FROM jdbc('mysql://localhost:3306/?user=root&password=root', 'schema', SELECT * FROM jdbc('datasource://mysql-local', 'schema', 'table') ``` -[Original article](https://clickhouse.tech/docs/en/query_language/table_functions/jdbc/) diff --git a/docs/en/sql-reference/table-functions/merge.md b/docs/en/sql-reference/table-functions/merge.md index 7b3d88f6266..a5c74b71069 100644 --- a/docs/en/sql-reference/table-functions/merge.md +++ b/docs/en/sql-reference/table-functions/merge.md @@ -9,4 +9,3 @@ toc_title: merge The table structure is taken from the first table encountered that matches the regular expression. -[Original article](https://clickhouse.tech/docs/en/query_language/table_functions/merge/) diff --git a/docs/en/sql-reference/table-functions/numbers.md b/docs/en/sql-reference/table-functions/numbers.md index 53e4e42a2f8..f9735056b05 100644 --- a/docs/en/sql-reference/table-functions/numbers.md +++ b/docs/en/sql-reference/table-functions/numbers.md @@ -25,4 +25,3 @@ Examples: select toDate('2010-01-01') + number as d FROM numbers(365); ``` -[Original article](https://clickhouse.tech/docs/en/query_language/table_functions/numbers/) diff --git a/docs/en/sql-reference/table-functions/s3.md b/docs/en/sql-reference/table-functions/s3.md index ea5dde707b8..2427f0f863c 100644 --- a/docs/en/sql-reference/table-functions/s3.md +++ b/docs/en/sql-reference/table-functions/s3.md @@ -166,4 +166,3 @@ Security consideration: if malicious user can specify arbitrary S3 URLs, `s3_max - [Virtual columns](../../engines/table-engines/index.md#table_engines-virtual_columns) -[Original article](https://clickhouse.tech/docs/en/query_language/table_functions/s3/) From ecb686b0be84da52979d7574d4fdb70d4452a9c4 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Sun, 14 Mar 2021 15:29:23 +0300 Subject: [PATCH 440/544] Fix some broken links --- .../system-tables/replication_queue.md | 4 ++-- docs/en/sql-reference/statements/alter/ttl.md | 6 +++--- .../sql-reference/statements/create/table.md | 18 ++++++++---------- docs/en/sql-reference/table-functions/url.md | 2 +- 4 files changed, 14 insertions(+), 16 deletions(-) diff --git a/docs/en/operations/system-tables/replication_queue.md b/docs/en/operations/system-tables/replication_queue.md index aa379caa46c..d1c74a771c6 100644 --- a/docs/en/operations/system-tables/replication_queue.md +++ b/docs/en/operations/system-tables/replication_queue.md @@ -70,12 +70,12 @@ num_tries: 36 last_exception: Code: 226, e.displayText() = DB::Exception: Marks file '/opt/clickhouse/data/merge/visits_v2/tmp_fetch_20201130_121373_121384_2/CounterID.mrk' doesn't exist (version 20.8.7.15 (official build)) last_attempt_time: 2020-12-08 17:35:54 num_postponed: 0 -postpone_reason: +postpone_reason: last_postpone_time: 1970-01-01 03:00:00 ``` **See Also** -- [Managing ReplicatedMergeTree Tables](../../sql-reference/statements/system.md/#query-language-system-replicated) +- [Managing ReplicatedMergeTree Tables](../../sql-reference/statements/system.md#query-language-system-replicated) [Original article](https://clickhouse.tech/docs/en/operations/system_tables/replication_queue) diff --git a/docs/en/sql-reference/statements/alter/ttl.md b/docs/en/sql-reference/statements/alter/ttl.md index e8bfb78ec68..e740bfe173e 100644 --- a/docs/en/sql-reference/statements/alter/ttl.md +++ b/docs/en/sql-reference/statements/alter/ttl.md @@ -18,7 +18,7 @@ ALTER TABLE table_name MODIFY TTL ttl_expression; TTL-property can be removed from table with the following query: ```sql -ALTER TABLE table_name REMOVE TTL +ALTER TABLE table_name REMOVE TTL ``` **Example** @@ -81,5 +81,5 @@ The `TTL` is no longer there, so the second row is not deleted: ### See Also -- More about the [TTL-expression](../../../../sql-reference/statements/create/table#ttl-expression). -- Modify column [with TTL](../../../../sql-reference/statements/alter/column#alter_modify-column). +- More about the [TTL-expression](../../../../sql-reference/statements/create/table.md#ttl-expression). +- Modify column [with TTL](../../../../sql-reference/statements/alter/column.md#alter_modify-column). diff --git a/docs/en/sql-reference/statements/create/table.md b/docs/en/sql-reference/statements/create/table.md index 0090eec14b7..60ec40a60ba 100644 --- a/docs/en/sql-reference/statements/create/table.md +++ b/docs/en/sql-reference/statements/create/table.md @@ -59,7 +59,7 @@ There can be other clauses after the `ENGINE` clause in the query. See detailed ## NULL Or NOT NULL Modifiers {#null-modifiers} -`NULL` and `NOT NULL` modifiers after data type in column definition allow or do not allow it to be [Nullable](../../../sql-reference/data-types/nullable.md#data_type-nullable). +`NULL` and `NOT NULL` modifiers after data type in column definition allow or do not allow it to be [Nullable](../../../sql-reference/data-types/nullable.md#data_type-nullable). If the type is not `Nullable` and if `NULL` is specified, it will be treated as `Nullable`; if `NOT NULL` is specified, then no. For example, `INT NULL` is the same as `Nullable(INT)`. If the type is `Nullable` and `NULL` or `NOT NULL` modifiers are specified, the exception will be thrown. @@ -109,16 +109,16 @@ It is not possible to set default values for elements in nested data structures. ## Primary Key {#primary-key} -You can define a [primary key](../../../engines/table-engines/mergetree-family/mergetree.md#primary-keys-and-indexes-in-queries) when creating a table. Primary key can be specified in two ways: +You can define a [primary key](../../../engines/table-engines/mergetree-family/mergetree.md#primary-keys-and-indexes-in-queries) when creating a table. Primary key can be specified in two ways: - Inside the column list ``` sql -CREATE TABLE db.table_name -( - name1 type1, name2 type2, ..., +CREATE TABLE db.table_name +( + name1 type1, name2 type2, ..., PRIMARY KEY(expr1[, expr2,...])] -) +) ENGINE = engine; ``` @@ -126,9 +126,9 @@ ENGINE = engine; ``` sql CREATE TABLE db.table_name -( +( name1 type1, name2 type2, ... -) +) ENGINE = engine PRIMARY KEY(expr1[, expr2,...]); ``` @@ -333,5 +333,3 @@ SELECT * FROM base.t1; │ 3 │ └───┘ ``` - - [Original article](https://clickhouse.tech/docs/en/sql-reference/statements/create/table) diff --git a/docs/en/sql-reference/table-functions/url.md b/docs/en/sql-reference/table-functions/url.md index 63b0ff0e152..2192b69d006 100644 --- a/docs/en/sql-reference/table-functions/url.md +++ b/docs/en/sql-reference/table-functions/url.md @@ -27,7 +27,7 @@ A table with the specified format and structure and with data from the defined ` **Examples** -Getting the first 3 lines of a table that contains columns of `String` and [UInt32](../../sql-reference/data-types/int-uint.md) type from HTTP-server which answers in [CSV](../../interfaces/formats.md/#csv) format. +Getting the first 3 lines of a table that contains columns of `String` and [UInt32](../../sql-reference/data-types/int-uint.md) type from HTTP-server which answers in [CSV](../../interfaces/formats.md#csv) format. ``` sql SELECT * FROM url('http://127.0.0.1:12345/', CSV, 'column1 String, column2 UInt32') LIMIT 3; From 037cc92433ba200eb2772cc6af83636a62f5e972 Mon Sep 17 00:00:00 2001 From: Maksim Kita Date: Sun, 14 Mar 2021 15:45:39 +0300 Subject: [PATCH 441/544] Reverted changes InterpreterSelectWithUnionQuery --- src/Interpreters/InterpreterSelectWithUnionQuery.cpp | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/src/Interpreters/InterpreterSelectWithUnionQuery.cpp b/src/Interpreters/InterpreterSelectWithUnionQuery.cpp index 1d5b05ddd6b..b894db79c7b 100644 --- a/src/Interpreters/InterpreterSelectWithUnionQuery.cpp +++ b/src/Interpreters/InterpreterSelectWithUnionQuery.cpp @@ -43,7 +43,7 @@ struct CustomizeASTSelectWithUnionQueryNormalize return; } - selects.push_back(ast_select); + selects.push_back(std::move(ast_select)); } void visit(ASTSelectWithUnionQuery & ast, ASTPtr &) const @@ -76,10 +76,10 @@ struct CustomizeASTSelectWithUnionQueryNormalize for (auto child = inner_union->list_of_selects->children.rbegin(); child != inner_union->list_of_selects->children.rend(); ++child) - selects.push_back(*child); + selects.push_back(std::move(*child)); } else - selects.push_back(select_list[i + 1]); + selects.push_back(std::move(select_list[i + 1])); } /// flatten all left nodes and current node to a UNION DISTINCT list else if (union_modes[i] == ASTSelectWithUnionQuery::Mode::DISTINCT) @@ -108,10 +108,10 @@ struct CustomizeASTSelectWithUnionQueryNormalize /// Inner_union is an UNION ALL list, just lift it up for (auto child = inner_union->list_of_selects->children.rbegin(); child != inner_union->list_of_selects->children.rend(); ++child) - selects.push_back(*child); + selects.push_back(std::move(*child)); } else - selects.push_back(select_list[0]); + selects.push_back(std::move(select_list[0])); } // reverse children list From f617976571e9bb20b1f0821dcd64b520606d0395 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Sun, 14 Mar 2021 15:55:11 +0300 Subject: [PATCH 442/544] Fix broken links --- docs/en/operations/system-tables/data_type_families.md | 2 +- docs/en/sql-reference/statements/alter/ttl.md | 4 ++-- docs/en/sql-reference/statements/watch.md | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/docs/en/operations/system-tables/data_type_families.md b/docs/en/operations/system-tables/data_type_families.md index ddda91ed151..4e439f13aa5 100644 --- a/docs/en/operations/system-tables/data_type_families.md +++ b/docs/en/operations/system-tables/data_type_families.md @@ -1,6 +1,6 @@ # system.data_type_families {#system_tables-data_type_families} -Contains information about supported [data types](../../sql-reference/data-types/). +Contains information about supported [data types](../../sql-reference/data-types/index.md). Columns: diff --git a/docs/en/sql-reference/statements/alter/ttl.md b/docs/en/sql-reference/statements/alter/ttl.md index e740bfe173e..aa7ee838e10 100644 --- a/docs/en/sql-reference/statements/alter/ttl.md +++ b/docs/en/sql-reference/statements/alter/ttl.md @@ -81,5 +81,5 @@ The `TTL` is no longer there, so the second row is not deleted: ### See Also -- More about the [TTL-expression](../../../../sql-reference/statements/create/table.md#ttl-expression). -- Modify column [with TTL](../../../../sql-reference/statements/alter/column.md#alter_modify-column). +- More about the [TTL-expression](../../../sql-reference/statements/create/table.md#ttl-expression). +- Modify column [with TTL](../../../sql-reference/statements/alter/column.md#alter_modify-column). diff --git a/docs/en/sql-reference/statements/watch.md b/docs/en/sql-reference/statements/watch.md index 761bc8a041e..4da586259d2 100644 --- a/docs/en/sql-reference/statements/watch.md +++ b/docs/en/sql-reference/statements/watch.md @@ -102,5 +102,5 @@ WATCH lv EVENTS LIMIT 1 The `FORMAT` clause works the same way as for the [SELECT](../../sql-reference/statements/select/format.md#format-clause). !!! info "Note" - The [JSONEachRowWithProgress](../../../interfaces/formats/#jsoneachrowwithprogress) format should be used when watching [live view](./create/view.md#live-view) tables over the HTTP interface. The progress messages will be added to the output to keep the long-lived HTTP connection alive until the query result changes. The interval between progress messages is controlled using the [live_view_heartbeat_interval](./create/view.md#live-view-settings) setting. + The [JSONEachRowWithProgress](../../interfaces/formats/#jsoneachrowwithprogress) format should be used when watching [live view](./create/view.md#live-view) tables over the HTTP interface. The progress messages will be added to the output to keep the long-lived HTTP connection alive until the query result changes. The interval between progress messages is controlled using the [live_view_heartbeat_interval](./create/view.md#live-view-settings) setting. From fd766a351917ec7207faa2471367e0371adb8ea2 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Sun, 14 Mar 2021 16:31:16 +0300 Subject: [PATCH 443/544] Better than nothing --- docs/tools/single_page.py | 80 ++++++++++++++++++++++++--------------- docs/tools/util.py | 9 ----- 2 files changed, 50 insertions(+), 39 deletions(-) diff --git a/docs/tools/single_page.py b/docs/tools/single_page.py index 05d50e768e2..27628962271 100644 --- a/docs/tools/single_page.py +++ b/docs/tools/single_page.py @@ -24,55 +24,71 @@ def recursive_values(item): yield item +anchor_not_allowed_chars = re.compile(r'[^\w\-]') +def generate_anchor_from_path(path): + return re.sub(anchor_not_allowed_chars, '-', path) + + +def replace_link(match, path): + link = match.group(1) + if link.endswith('/'): + link = link[0:-1] + '.md' + + return '(#{})'.format(generate_anchor_from_path(os.path.normpath(os.path.join(os.path.dirname(path), link)))) + + +# Concatenates Markdown files to a single file. def concatenate(lang, docs_path, single_page_file, nav): lang_path = os.path.join(docs_path, lang) - az_re = re.compile(r'[a-z]') proj_config = f'{docs_path}/toc_{lang}.yml' if os.path.exists(proj_config): with open(proj_config) as cfg_file: nav = yaml.full_load(cfg_file.read())['nav'] + files_to_concatenate = list(recursive_values(nav)) files_count = len(files_to_concatenate) logging.info(f'{files_count} files will be concatenated into single md-file for {lang}.') logging.debug('Concatenating: ' + ', '.join(files_to_concatenate)) assert files_count > 0, f'Empty single-page for {lang}' + # (../anything) or (../anything#anchor) or (xyz-abc.md) or (xyz-abc.md#anchor) + relative_link_regexp = re.compile(r'\((\.\./[^)#]+|[\w\-]+\.md)(?:#[^\)]*)?\)') + for path in files_to_concatenate: - if path.endswith('introduction/info.md'): - continue try: with open(os.path.join(lang_path, path)) as f: - anchors = set() - tmp_path = path.replace('/index.md', '/').replace('.md', '/') - prefixes = ['', '../', '../../', '../../../'] - parts = tmp_path.split('/') - anchors.add(parts[-2] + '/') - anchors.add('/'.join(parts[1:])) - - for part in parts[0:-2] if len(parts) > 2 else parts: - for prefix in prefixes: - anchor = prefix + tmp_path - if anchor: - anchors.add(anchor) - anchors.add('../' + anchor) - anchors.add('../../' + anchor) - tmp_path = tmp_path.replace(part, '..') - - for anchor in anchors: - if re.search(az_re, anchor): - single_page_file.write('' % anchor) - - single_page_file.write('\n') + # Insert a horizontal ruler. Then insert an anchor that we will link to. Its name will be a path to the .md file. + single_page_file.write('\n______\n\n' % generate_anchor_from_path(path)) in_metadata = False - for l in f: - if l.startswith('---'): + for line in f: + # Skip YAML metadata. + if line == '---\n': in_metadata = not in_metadata - if l.startswith('#'): - l = '#' + l + continue + if not in_metadata: - single_page_file.write(l) + # Increase the level of headers. + if line.startswith('#'): + line = '#' + line + + # Replace links within the docs. + + if re.search(relative_link_regexp, line): + line = re.sub( + relative_link_regexp, + lambda match: replace_link(match, path), + line) + + # If failed to replace the relative link, print to log + if '../' in line: + logging.info('Failed to resolve relative link:') + logging.info(path) + logging.info(line) + + single_page_file.write(line) + except IOError as e: logging.warning(str(e)) @@ -86,7 +102,7 @@ def build_single_page_version(lang, args, nav, cfg): extra['single_page'] = True extra['is_amp'] = False - with util.autoremoved_file(os.path.join(args.docs_dir, lang, 'single.md')) as single_md: + with open(os.path.join(args.docs_dir, lang, 'single.md'), 'w') as single_md: concatenate(lang, args.docs_dir, single_md, nav) with util.temp_dir() as site_temp: @@ -123,11 +139,14 @@ def build_single_page_version(lang, args, nav, cfg): single_page_index_html = os.path.join(single_page_output_path, 'index.html') single_page_content_js = os.path.join(single_page_output_path, 'content.js') + with open(single_page_index_html, 'r') as f: sp_prefix, sp_js, sp_suffix = f.read().split('') + with open(single_page_index_html, 'w') as f: f.write(sp_prefix) f.write(sp_suffix) + with open(single_page_content_js, 'w') as f: if args.minify: import jsmin @@ -151,6 +170,7 @@ def build_single_page_version(lang, args, nav, cfg): js_in = ' '.join(website.get_js_in(args)) subprocess.check_call(f'cat {css_in} > {test_dir}/css/base.css', shell=True) subprocess.check_call(f'cat {js_in} > {test_dir}/js/base.js', shell=True) + if args.save_raw_single_page: shutil.copytree(test_dir, args.save_raw_single_page) diff --git a/docs/tools/util.py b/docs/tools/util.py index b840dc1168a..25961561f99 100644 --- a/docs/tools/util.py +++ b/docs/tools/util.py @@ -22,15 +22,6 @@ def temp_dir(): shutil.rmtree(path) -@contextlib.contextmanager -def autoremoved_file(path): - try: - with open(path, 'w') as handle: - yield handle - finally: - os.unlink(path) - - @contextlib.contextmanager def cd(new_cwd): old_cwd = os.getcwd() From dfd2068350b7cc6b58d14556e7ee9938565ceb1e Mon Sep 17 00:00:00 2001 From: Maksim Kita Date: Sun, 14 Mar 2021 17:19:48 +0300 Subject: [PATCH 444/544] Updated SharedLibrary string_view interface --- src/Common/SharedLibrary.cpp | 4 ++-- src/Common/SharedLibrary.h | 9 +++++---- 2 files changed, 7 insertions(+), 6 deletions(-) diff --git a/src/Common/SharedLibrary.cpp b/src/Common/SharedLibrary.cpp index 9b81f74eb23..37da308d5af 100644 --- a/src/Common/SharedLibrary.cpp +++ b/src/Common/SharedLibrary.cpp @@ -13,7 +13,7 @@ namespace ErrorCodes extern const int CANNOT_DLSYM; } -SharedLibrary::SharedLibrary(const std::string_view & path, int flags) +SharedLibrary::SharedLibrary(std::string_view path, int flags) { handle = dlopen(path.data(), flags); if (!handle) @@ -31,7 +31,7 @@ SharedLibrary::~SharedLibrary() std::terminate(); } -void * SharedLibrary::getImpl(const std::string_view & name, bool no_throw) +void * SharedLibrary::getImpl(std::string_view name, bool no_throw) { dlerror(); diff --git a/src/Common/SharedLibrary.h b/src/Common/SharedLibrary.h index e665c335c6f..866e60fbd33 100644 --- a/src/Common/SharedLibrary.h +++ b/src/Common/SharedLibrary.h @@ -14,23 +14,24 @@ namespace DB class SharedLibrary : private boost::noncopyable { public: - explicit SharedLibrary(const std::string_view & path, int flags = RTLD_LAZY); + explicit SharedLibrary(std::string_view path, int flags = RTLD_LAZY); ~SharedLibrary(); template - Func get(const std::string_view & name) + Func get(std::string_view name) { return reinterpret_cast(getImpl(name)); } + template - Func tryGet(const std::string_view & name) + Func tryGet(std::string_view name) { return reinterpret_cast(getImpl(name, true)); } private: - void * getImpl(const std::string_view & name, bool no_throw = false); + void * getImpl(std::string_view name, bool no_throw = false); void * handle = nullptr; }; From b3d29480815d31900692521d6c00ea877230b906 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Sun, 14 Mar 2021 17:31:07 +0300 Subject: [PATCH 445/544] Fix UBSan report in modulo by constant --- src/Functions/modulo.cpp | 8 ++++++++ tests/queries/0_stateless/01760_modulo_negative.reference | 0 tests/queries/0_stateless/01760_modulo_negative.sql | 1 + 3 files changed, 9 insertions(+) create mode 100644 tests/queries/0_stateless/01760_modulo_negative.reference create mode 100644 tests/queries/0_stateless/01760_modulo_negative.sql diff --git a/src/Functions/modulo.cpp b/src/Functions/modulo.cpp index d9bf74ccaf5..fe215851bb6 100644 --- a/src/Functions/modulo.cpp +++ b/src/Functions/modulo.cpp @@ -70,6 +70,14 @@ struct ModuloByConstantImpl if (unlikely(static_cast(b) == 0)) throw Exception("Division by zero", ErrorCodes::ILLEGAL_DIVISION); + /// Division by min negative value. + if (std::is_signed_v && b == std::numeric_limits::lowest()) + throw Exception("Division by the most negative number", ErrorCodes::ILLEGAL_DIVISION); + + /// Modulo of division by negative number is the same as the positive number. + if (b < 0) + b = -b; + libdivide::divider divider(b); /// Here we failed to make the SSE variant from libdivide give an advantage. diff --git a/tests/queries/0_stateless/01760_modulo_negative.reference b/tests/queries/0_stateless/01760_modulo_negative.reference new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tests/queries/0_stateless/01760_modulo_negative.sql b/tests/queries/0_stateless/01760_modulo_negative.sql new file mode 100644 index 00000000000..dbea06cc100 --- /dev/null +++ b/tests/queries/0_stateless/01760_modulo_negative.sql @@ -0,0 +1 @@ +SELECT -number % -9223372036854775808 FROM system.numbers; -- { serverError 153 } From c64892cdc83ff56e676652f8d4ade1be3a8a5fdc Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Sun, 14 Mar 2021 17:36:45 +0300 Subject: [PATCH 446/544] Fix broken links --- docs/ru/sql-reference/table-functions/index.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/ru/sql-reference/table-functions/index.md b/docs/ru/sql-reference/table-functions/index.md index 52fb1bd2737..ca90306bbd5 100644 --- a/docs/ru/sql-reference/table-functions/index.md +++ b/docs/ru/sql-reference/table-functions/index.md @@ -33,6 +33,6 @@ toc_title: "Введение" | [jdbc](../../sql-reference/table-functions/jdbc.md) | Создаёт таблицу с движком [JDBC](../../engines/table-engines/integrations/jdbc.md). | | [odbc](../../sql-reference/table-functions/odbc.md) | Создаёт таблицу с движком [ODBC](../../engines/table-engines/integrations/odbc.md). | | [hdfs](../../sql-reference/table-functions/hdfs.md) | Создаёт таблицу с движком [HDFS](../../engines/table-engines/integrations/hdfs.md). | -| [s3](../../sql-reference/table-functions/s3.md) | Создаёт таблицу с движком [S3](../../engines/table-engines/integrations/s3.md). | +| [s3](../../sql-reference/table-functions/s3.md) | Создаёт таблицу с движком [S3](../../engines/table-engines/integrations/s3.md). | [Оригинальная статья](https://clickhouse.tech/docs/ru/query_language/table_functions/) From d6e0342c3041024bd0386b297ca5bff47cc11207 Mon Sep 17 00:00:00 2001 From: Vitaly Baranov Date: Sat, 27 Feb 2021 01:37:00 +0300 Subject: [PATCH 447/544] Improvements in implementations of the classes AccessRights and GrantedRoles. --- src/Access/AccessControlManager.cpp | 8 +- src/Access/AccessControlManager.h | 6 +- src/Access/AccessRights.cpp | 103 +++++-- src/Access/AccessRights.h | 15 +- src/Access/AccessRightsElement.cpp | 233 +++++++-------- src/Access/AccessRightsElement.h | 142 +++------ src/Access/ContextAccess.cpp | 188 ++++++------ src/Access/ContextAccess.h | 47 +-- src/Access/GrantedRoles.cpp | 144 +++++++-- src/Access/GrantedRoles.h | 46 ++- src/Access/LDAPAccessStorage.cpp | 13 +- src/Access/RoleCache.cpp | 10 +- src/Access/RoleCache.h | 3 +- src/Access/RolesOrUsersSet.cpp | 64 +--- src/Access/RolesOrUsersSet.h | 5 +- src/Interpreters/Context.cpp | 2 +- src/Interpreters/Context.h | 4 +- .../InterpreterCreateQuotaQuery.cpp | 2 +- .../InterpreterCreateRowPolicyQuery.cpp | 4 +- .../InterpreterCreateSettingsProfileQuery.cpp | 2 +- .../InterpreterDropAccessEntityQuery.cpp | 2 +- src/Interpreters/InterpreterGrantQuery.cpp | 278 ++++++++++-------- src/Interpreters/InterpreterSetRoleQuery.cpp | 12 +- .../InterpreterShowAccessEntitiesQuery.cpp | 2 +- ...InterpreterShowCreateAccessEntityQuery.cpp | 2 +- .../InterpreterShowGrantsQuery.cpp | 44 ++- src/Interpreters/executeDDLQueryOnCluster.cpp | 11 +- src/Interpreters/executeDDLQueryOnCluster.h | 4 +- src/Parsers/ASTCreateQuotaQuery.cpp | 4 +- src/Parsers/ASTCreateQuotaQuery.h | 2 +- src/Parsers/ASTCreateRowPolicyQuery.cpp | 8 +- src/Parsers/ASTCreateRowPolicyQuery.h | 4 +- src/Parsers/ASTCreateSettingsProfileQuery.cpp | 4 +- src/Parsers/ASTCreateSettingsProfileQuery.h | 2 +- src/Parsers/ASTCreateUserQuery.h | 11 +- src/Parsers/ASTDropAccessEntityQuery.cpp | 4 +- src/Parsers/ASTDropAccessEntityQuery.h | 2 +- src/Parsers/ASTGrantQuery.cpp | 75 +++-- src/Parsers/ASTGrantQuery.h | 12 +- src/Parsers/ASTRolesOrUsersSet.cpp | 13 +- src/Parsers/ASTRolesOrUsersSet.h | 13 +- src/Parsers/ASTRowPolicyName.cpp | 4 +- src/Parsers/ASTRowPolicyName.h | 4 +- src/Parsers/ASTShowAccessEntitiesQuery.cpp | 2 +- src/Parsers/ASTShowAccessEntitiesQuery.h | 2 +- .../ASTShowCreateAccessEntityQuery.cpp | 4 +- src/Parsers/ASTShowCreateAccessEntityQuery.h | 2 +- src/Parsers/ParserCreateQuotaQuery.cpp | 2 +- src/Parsers/ParserCreateRowPolicyQuery.cpp | 2 +- .../ParserCreateSettingsProfileQuery.cpp | 2 +- src/Parsers/ParserCreateUserQuery.cpp | 4 +- src/Parsers/ParserCreateUserQuery.h | 2 + src/Parsers/ParserGrantQuery.cpp | 91 +++--- src/Parsers/ParserRolesOrUsersSet.cpp | 29 +- src/Parsers/ParserRolesOrUsersSet.h | 11 +- src/Parsers/ParserSetRoleQuery.cpp | 8 +- src/Parsers/ParserShowGrantsQuery.cpp | 2 +- src/Storages/System/StorageSystemGrants.cpp | 11 +- .../System/StorageSystemRoleGrants.cpp | 16 +- .../rbac/tests/syntax/revoke_role.py | 7 +- 60 files changed, 944 insertions(+), 811 deletions(-) diff --git a/src/Access/AccessControlManager.cpp b/src/Access/AccessControlManager.cpp index 0e7bf1e56f4..66023c1c0ea 100644 --- a/src/Access/AccessControlManager.cpp +++ b/src/Access/AccessControlManager.cpp @@ -403,7 +403,7 @@ void AccessControlManager::checkSettingNameIsAllowed(const std::string_view & se std::shared_ptr AccessControlManager::getContextAccess( const UUID & user_id, - const boost::container::flat_set & current_roles, + const std::vector & current_roles, bool use_default_roles, const Settings & settings, const String & current_database, @@ -411,7 +411,7 @@ std::shared_ptr AccessControlManager::getContextAccess( { ContextAccessParams params; params.user_id = user_id; - params.current_roles = current_roles; + params.current_roles.insert(current_roles.begin(), current_roles.end()); params.use_default_roles = use_default_roles; params.current_database = current_database; params.readonly = settings.readonly; @@ -444,8 +444,8 @@ std::shared_ptr AccessControlManager::getContextAccess(cons std::shared_ptr AccessControlManager::getEnabledRoles( - const boost::container::flat_set & current_roles, - const boost::container::flat_set & current_roles_with_admin_option) const + const std::vector & current_roles, + const std::vector & current_roles_with_admin_option) const { return role_cache->getEnabledRoles(current_roles, current_roles_with_admin_option); } diff --git a/src/Access/AccessControlManager.h b/src/Access/AccessControlManager.h index b4d90a4198e..789c33af1c1 100644 --- a/src/Access/AccessControlManager.h +++ b/src/Access/AccessControlManager.h @@ -114,7 +114,7 @@ public: std::shared_ptr getContextAccess( const UUID & user_id, - const boost::container::flat_set & current_roles, + const std::vector & current_roles, bool use_default_roles, const Settings & settings, const String & current_database, @@ -123,8 +123,8 @@ public: std::shared_ptr getContextAccess(const ContextAccessParams & params) const; std::shared_ptr getEnabledRoles( - const boost::container::flat_set & current_roles, - const boost::container::flat_set & current_roles_with_admin_option) const; + const std::vector & current_roles, + const std::vector & current_roles_with_admin_option) const; std::shared_ptr getEnabledRowPolicies( const UUID & user_id, diff --git a/src/Access/AccessRights.cpp b/src/Access/AccessRights.cpp index 8ce71dd8da8..f9c1d23350d 100644 --- a/src/Access/AccessRights.cpp +++ b/src/Access/AccessRights.cpp @@ -7,16 +7,19 @@ namespace DB { +namespace ErrorCodes +{ + extern const int BAD_ARGUMENTS; +} + namespace { - using Kind = AccessRightsElementWithOptions::Kind; - struct ProtoElement { AccessFlags access_flags; boost::container::small_vector full_name; bool grant_option = false; - Kind kind = Kind::GRANT; + bool is_partial_revoke = false; friend bool operator<(const ProtoElement & left, const ProtoElement & right) { @@ -43,8 +46,8 @@ namespace if (int cmp = compare_name(left.full_name, right.full_name, 1)) return cmp < 0; - if (left.kind != right.kind) - return (left.kind == Kind::GRANT); + if (left.is_partial_revoke != right.is_partial_revoke) + return right.is_partial_revoke; if (left.grant_option != right.grant_option) return right.grant_option; @@ -55,12 +58,12 @@ namespace return (left.access_flags < right.access_flags); } - AccessRightsElementWithOptions getResult() const + AccessRightsElement getResult() const { - AccessRightsElementWithOptions res; + AccessRightsElement res; res.access_flags = access_flags; res.grant_option = grant_option; - res.kind = kind; + res.is_partial_revoke = is_partial_revoke; switch (full_name.size()) { case 0: @@ -105,11 +108,11 @@ namespace class ProtoElements : public std::vector { public: - AccessRightsElementsWithOptions getResult() const + AccessRightsElements getResult() const { ProtoElements sorted = *this; boost::range::sort(sorted); - AccessRightsElementsWithOptions res; + AccessRightsElements res; res.reserve(sorted.size()); for (size_t i = 0; i != sorted.size();) @@ -144,7 +147,7 @@ namespace { return (element.full_name.size() != 3) || (element.full_name[0] != start_element.full_name[0]) || (element.full_name[1] != start_element.full_name[1]) || (element.grant_option != start_element.grant_option) - || (element.kind != start_element.kind); + || (element.is_partial_revoke != start_element.is_partial_revoke); }); return it - (begin() + start); @@ -153,7 +156,7 @@ namespace /// Collects columns together to write multiple columns into one AccessRightsElement. /// That procedure allows to output access rights in more compact way, /// e.g. "SELECT(x, y)" instead of "SELECT(x), SELECT(y)". - void appendResultWithElementsWithDifferenceInColumnOnly(size_t start, size_t count, AccessRightsElementsWithOptions & res) const + void appendResultWithElementsWithDifferenceInColumnOnly(size_t start, size_t count, AccessRightsElements & res) const { const auto * pbegin = data() + start; const auto * pend = pbegin + count; @@ -180,7 +183,7 @@ namespace res.emplace_back(); auto & back = res.back(); back.grant_option = pbegin->grant_option; - back.kind = pbegin->kind; + back.is_partial_revoke = pbegin->is_partial_revoke; back.any_database = false; back.database = pbegin->full_name[0]; back.any_table = false; @@ -515,10 +518,10 @@ private: auto grants = flags - parent_fl; if (revokes) - res.push_back(ProtoElement{revokes, full_name, false, Kind::REVOKE}); + res.push_back(ProtoElement{revokes, full_name, false, true}); if (grants) - res.push_back(ProtoElement{grants, full_name, false, Kind::GRANT}); + res.push_back(ProtoElement{grants, full_name, false, false}); if (node.children) { @@ -550,16 +553,16 @@ private: auto grants = flags - parent_fl - grants_go; if (revokes) - res.push_back(ProtoElement{revokes, full_name, false, Kind::REVOKE}); + res.push_back(ProtoElement{revokes, full_name, false, true}); if (revokes_go) - res.push_back(ProtoElement{revokes_go, full_name, true, Kind::REVOKE}); + res.push_back(ProtoElement{revokes_go, full_name, true, true}); if (grants) - res.push_back(ProtoElement{grants, full_name, false, Kind::GRANT}); + res.push_back(ProtoElement{grants, full_name, false, false}); if (grants_go) - res.push_back(ProtoElement{grants_go, full_name, true, Kind::GRANT}); + res.push_back(ProtoElement{grants_go, full_name, true, false}); if (node && node->children) { @@ -774,8 +777,10 @@ void AccessRights::grantImpl(const AccessFlags & flags, const Args &... args) } template -void AccessRights::grantImpl(const AccessRightsElement & element) +void AccessRights::grantImplHelper(const AccessRightsElement & element) { + assert(!element.is_partial_revoke); + assert(!element.grant_option || with_grant_option); if (element.any_database) grantImpl(element.access_flags); else if (element.any_table) @@ -786,6 +791,24 @@ void AccessRights::grantImpl(const AccessRightsElement & element) grantImpl(element.access_flags, element.database, element.table, element.columns); } +template +void AccessRights::grantImpl(const AccessRightsElement & element) +{ + if (element.is_partial_revoke) + throw Exception("A partial revoke should be revoked, not granted", ErrorCodes::BAD_ARGUMENTS); + if constexpr (with_grant_option) + { + grantImplHelper(element); + } + else + { + if (element.grant_option) + grantImplHelper(element); + else + grantImplHelper(element); + } +} + template void AccessRights::grantImpl(const AccessRightsElements & elements) { @@ -830,8 +853,9 @@ void AccessRights::revokeImpl(const AccessFlags & flags, const Args &... args) } template -void AccessRights::revokeImpl(const AccessRightsElement & element) +void AccessRights::revokeImplHelper(const AccessRightsElement & element) { + assert(!element.grant_option || grant_option); if (element.any_database) revokeImpl(element.access_flags); else if (element.any_table) @@ -842,6 +866,22 @@ void AccessRights::revokeImpl(const AccessRightsElement & element) revokeImpl(element.access_flags, element.database, element.table, element.columns); } +template +void AccessRights::revokeImpl(const AccessRightsElement & element) +{ + if constexpr (grant_option) + { + revokeImplHelper(element); + } + else + { + if (element.grant_option) + revokeImplHelper(element); + else + revokeImplHelper(element); + } +} + template void AccessRights::revokeImpl(const AccessRightsElements & elements) { @@ -868,7 +908,7 @@ void AccessRights::revokeGrantOption(const AccessRightsElement & element) { revo void AccessRights::revokeGrantOption(const AccessRightsElements & elements) { revokeImpl(elements); } -AccessRightsElementsWithOptions AccessRights::getElements() const +AccessRightsElements AccessRights::getElements() const { #if 0 logTree(); @@ -903,8 +943,9 @@ bool AccessRights::isGrantedImpl(const AccessFlags & flags, const Args &... args } template -bool AccessRights::isGrantedImpl(const AccessRightsElement & element) const +bool AccessRights::isGrantedImplHelper(const AccessRightsElement & element) const { + assert(!element.grant_option || grant_option); if (element.any_database) return isGrantedImpl(element.access_flags); else if (element.any_table) @@ -915,6 +956,22 @@ bool AccessRights::isGrantedImpl(const AccessRightsElement & element) const return isGrantedImpl(element.access_flags, element.database, element.table, element.columns); } +template +bool AccessRights::isGrantedImpl(const AccessRightsElement & element) const +{ + if constexpr (grant_option) + { + return isGrantedImplHelper(element); + } + else + { + if (element.grant_option) + return isGrantedImplHelper(element); + else + return isGrantedImplHelper(element); + } +} + template bool AccessRights::isGrantedImpl(const AccessRightsElements & elements) const { diff --git a/src/Access/AccessRights.h b/src/Access/AccessRights.h index c610795ab45..a90616ea27f 100644 --- a/src/Access/AccessRights.h +++ b/src/Access/AccessRights.h @@ -30,7 +30,7 @@ public: String toString() const; /// Returns the information about all the access granted. - AccessRightsElementsWithOptions getElements() const; + AccessRightsElements getElements() const; /// Grants access on a specified database/table/column. /// Does nothing if the specified access has been already granted. @@ -119,12 +119,15 @@ private: template void grantImpl(const AccessFlags & flags, const Args &... args); - template + template void grantImpl(const AccessRightsElement & element); - template + template void grantImpl(const AccessRightsElements & elements); + template + void grantImplHelper(const AccessRightsElement & element); + template void revokeImpl(const AccessFlags & flags, const Args &... args); @@ -134,6 +137,9 @@ private: template void revokeImpl(const AccessRightsElements & elements); + template + void revokeImplHelper(const AccessRightsElement & element); + template bool isGrantedImpl(const AccessFlags & flags, const Args &... args) const; @@ -143,6 +149,9 @@ private: template bool isGrantedImpl(const AccessRightsElements & elements) const; + template + bool isGrantedImplHelper(const AccessRightsElement & element) const; + void logTree() const; struct Node; diff --git a/src/Access/AccessRightsElement.cpp b/src/Access/AccessRightsElement.cpp index e69fb6d3b74..823019ffebd 100644 --- a/src/Access/AccessRightsElement.cpp +++ b/src/Access/AccessRightsElement.cpp @@ -1,169 +1,162 @@ #include -#include #include -#include -#include -#include #include -#include namespace DB { namespace { - using Kind = AccessRightsElementWithOptions::Kind; - - String formatOptions(bool grant_option, Kind kind, const String & inner_part) + void formatColumnNames(const Strings & columns, String & result) { - if (kind == Kind::REVOKE) + result += "("; + bool need_comma = false; + for (const auto & column : columns) { - if (grant_option) - return "REVOKE GRANT OPTION " + inner_part; - else - return "REVOKE " + inner_part; - } - else - { - if (grant_option) - return "GRANT " + inner_part + " WITH GRANT OPTION"; - else - return "GRANT " + inner_part; + if (need_comma) + result += ", "; + need_comma = true; + result += backQuoteIfNeed(column); } + result += ")"; } - - String formatONClause(const String & database, bool any_database, const String & table, bool any_table) + void formatONClause(const String & database, bool any_database, const String & table, bool any_table, String & result) { - String msg = "ON "; - + result += "ON "; if (any_database) - msg += "*."; - else if (!database.empty()) - msg += backQuoteIfNeed(database) + "."; - - if (any_table) - msg += "*"; + { + result += "*.*"; + } else - msg += backQuoteIfNeed(table); - return msg; + { + if (!database.empty()) + { + result += backQuoteIfNeed(database); + result += "."; + } + if (any_table) + result += "*"; + else + result += backQuoteIfNeed(table); + } } - - String formatAccessFlagsWithColumns(const AccessFlags & access_flags, const Strings & columns, bool any_column) + void formatOptions(bool grant_option, bool is_partial_revoke, String & result) { - String columns_in_parentheses; + if (is_partial_revoke) + { + if (grant_option) + result.insert(0, "REVOKE GRANT OPTION "); + else + result.insert(0, "REVOKE "); + } + else + { + if (grant_option) + result.insert(0, "GRANT ").append(" WITH GRANT OPTION"); + else + result.insert(0, "GRANT "); + } + } + + void formatAccessFlagsWithColumns(const AccessFlags & access_flags, const Strings & columns, bool any_column, String & result) + { + String columns_as_str; if (!any_column) { if (columns.empty()) - return "USAGE"; - for (const auto & column : columns) { - columns_in_parentheses += columns_in_parentheses.empty() ? "(" : ", "; - columns_in_parentheses += backQuoteIfNeed(column); + result += "USAGE"; + return; } - columns_in_parentheses += ")"; + formatColumnNames(columns, columns_as_str); } auto keywords = access_flags.toKeywords(); if (keywords.empty()) - return "USAGE"; + { + result += "USAGE"; + return; + } - String msg; + bool need_comma = false; for (const std::string_view & keyword : keywords) { - if (!msg.empty()) - msg += ", "; - msg += String{keyword} + columns_in_parentheses; + if (need_comma) + result.append(", "); + need_comma = true; + result += keyword; + result += columns_as_str; } - return msg; } -} - -String AccessRightsElement::toString() const -{ - return formatAccessFlagsWithColumns(access_flags, columns, any_column) + " " + formatONClause(database, any_database, table, any_table); -} - -String AccessRightsElementWithOptions::toString() const -{ - return formatOptions(grant_option, kind, AccessRightsElement::toString()); -} - -String AccessRightsElements::toString() const -{ - if (empty()) - return "USAGE ON *.*"; - - String res; - String inner_part; - - for (size_t i = 0; i != size(); ++i) + String toStringImpl(const AccessRightsElement & element, bool with_options) { - const auto & element = (*this)[i]; - - if (!inner_part.empty()) - inner_part += ", "; - inner_part += formatAccessFlagsWithColumns(element.access_flags, element.columns, element.any_column); - - bool next_element_uses_same_table = false; - if (i != size() - 1) - { - const auto & next_element = (*this)[i + 1]; - if (element.sameDatabaseAndTable(next_element)) - next_element_uses_same_table = true; - } - - if (!next_element_uses_same_table) - { - if (!res.empty()) - res += ", "; - res += inner_part + " " + formatONClause(element.database, element.any_database, element.table, element.any_table); - inner_part.clear(); - } + String result; + formatAccessFlagsWithColumns(element.access_flags, element.columns, element.any_column, result); + result += " "; + formatONClause(element.database, element.any_database, element.table, element.any_table, result); + if (with_options) + formatOptions(element.grant_option, element.is_partial_revoke, result); + return result; } - return res; -} - -String AccessRightsElementsWithOptions::toString() const -{ - if (empty()) - return "GRANT USAGE ON *.*"; - - String res; - String inner_part; - - for (size_t i = 0; i != size(); ++i) + String toStringImpl(const AccessRightsElements & elements, bool with_options) { - const auto & element = (*this)[i]; + if (elements.empty()) + return with_options ? "GRANT USAGE ON *.*" : "USAGE ON *.*"; - if (!inner_part.empty()) - inner_part += ", "; - inner_part += formatAccessFlagsWithColumns(element.access_flags, element.columns, element.any_column); + String result; + String part; - bool next_element_uses_same_mode_and_table = false; - if (i != size() - 1) + for (size_t i = 0; i != elements.size(); ++i) { - const auto & next_element = (*this)[i + 1]; - if (element.sameDatabaseAndTable(next_element) && element.sameOptions(next_element)) - next_element_uses_same_mode_and_table = true; + const auto & element = elements[i]; + + if (!part.empty()) + part += ", "; + formatAccessFlagsWithColumns(element.access_flags, element.columns, element.any_column, part); + + bool next_element_uses_same_table_and_options = false; + if (i != elements.size() - 1) + { + const auto & next_element = elements[i + 1]; + if (element.sameDatabaseAndTable(next_element) && element.sameOptions(next_element)) + next_element_uses_same_table_and_options = true; + } + + if (!next_element_uses_same_table_and_options) + { + part += " "; + formatONClause(element.database, element.any_database, element.table, element.any_table, part); + if (with_options) + formatOptions(element.grant_option, element.is_partial_revoke, part); + if (result.empty()) + result = std::move(part); + else + result.append(", ").append(part); + part.clear(); + } } - if (!next_element_uses_same_mode_and_table) - { - if (!res.empty()) - res += ", "; - res += formatOptions( - element.grant_option, - element.kind, - inner_part + " " + formatONClause(element.database, element.any_database, element.table, element.any_table)); - inner_part.clear(); - } + return result; } +} - return res; + +String AccessRightsElement::toString() const { return toStringImpl(*this, true); } +String AccessRightsElement::toStringWithoutOptions() const { return toStringImpl(*this, false); } +String AccessRightsElements::toString() const { return toStringImpl(*this, true); } +String AccessRightsElements::toStringWithoutOptions() const { return toStringImpl(*this, false); } + +void AccessRightsElements::eraseNonGrantable() +{ + boost::range::remove_erase_if(*this, [](AccessRightsElement & element) + { + element.eraseNonGrantable(); + return element.empty(); + }); } } diff --git a/src/Access/AccessRightsElement.h b/src/Access/AccessRightsElement.h index 36cb64e6eba..c76f019bc61 100644 --- a/src/Access/AccessRightsElement.h +++ b/src/Access/AccessRightsElement.h @@ -16,6 +16,8 @@ struct AccessRightsElement bool any_database = true; bool any_table = true; bool any_column = true; + bool grant_option = false; + bool is_partial_revoke = false; AccessRightsElement() = default; AccessRightsElement(const AccessRightsElement &) = default; @@ -73,7 +75,7 @@ struct AccessRightsElement bool empty() const { return !access_flags || (!any_column && columns.empty()); } - auto toTuple() const { return std::tie(access_flags, any_database, database, any_table, table, any_column, columns); } + auto toTuple() const { return std::tie(access_flags, any_database, database, any_table, table, any_column, columns, grant_option, is_partial_revoke); } friend bool operator==(const AccessRightsElement & left, const AccessRightsElement & right) { return left.toTuple() == right.toTuple(); } friend bool operator!=(const AccessRightsElement & left, const AccessRightsElement & right) { return !(left == right); } @@ -83,44 +85,36 @@ struct AccessRightsElement && (any_table == other.any_table); } - bool isEmptyDatabase() const { return !any_database && database.empty(); } - - /// If the database is empty, replaces it with `new_database`. Otherwise does nothing. - void replaceEmptyDatabase(const String & new_database); - - /// Resets flags which cannot be granted. - void removeNonGrantableFlags(); - - /// Returns a human-readable representation like "SELECT, UPDATE(x, y) ON db.table". - String toString() const; -}; - - -struct AccessRightsElementWithOptions : public AccessRightsElement -{ - bool grant_option = false; - - enum class Kind + bool sameOptions(const AccessRightsElement & other) const { - GRANT, - REVOKE, - }; - Kind kind = Kind::GRANT; - - bool sameOptions(const AccessRightsElementWithOptions & other) const - { - return (grant_option == other.grant_option) && (kind == other.kind); + return (grant_option == other.grant_option) && (is_partial_revoke == other.is_partial_revoke); } - auto toTuple() const { return std::tie(access_flags, any_database, database, any_table, table, any_column, columns, grant_option, kind); } - friend bool operator==(const AccessRightsElementWithOptions & left, const AccessRightsElementWithOptions & right) { return left.toTuple() == right.toTuple(); } - friend bool operator!=(const AccessRightsElementWithOptions & left, const AccessRightsElementWithOptions & right) { return !(left == right); } - /// Resets flags which cannot be granted. - void removeNonGrantableFlags(); + void eraseNonGrantable() + { + if (!any_column) + access_flags &= AccessFlags::allFlagsGrantableOnColumnLevel(); + else if (!any_table) + access_flags &= AccessFlags::allFlagsGrantableOnTableLevel(); + else if (!any_database) + access_flags &= AccessFlags::allFlagsGrantableOnDatabaseLevel(); + else + access_flags &= AccessFlags::allFlagsGrantableOnGlobalLevel(); + } + + bool isEmptyDatabase() const { return !any_database && database.empty(); } + + /// If the database is empty, replaces it with `current_database`. Otherwise does nothing. + void replaceEmptyDatabase(const String & current_database) + { + if (isEmptyDatabase()) + database = current_database; + } /// Returns a human-readable representation like "GRANT SELECT, UPDATE(x, y) ON db.table". String toString() const; + String toStringWithoutOptions() const; }; @@ -130,77 +124,29 @@ class AccessRightsElements : public std::vector public: bool empty() const { return std::all_of(begin(), end(), [](const AccessRightsElement & e) { return e.empty(); }); } - /// Replaces the empty database with `new_database`. - void replaceEmptyDatabase(const String & new_database); + bool sameDatabaseAndTable() const + { + return (size() < 2) || std::all_of(std::next(begin()), end(), [this](const AccessRightsElement & e) { return e.sameDatabaseAndTable(front()); }); + } + + bool sameOptions() const + { + return (size() < 2) || std::all_of(std::next(begin()), end(), [this](const AccessRightsElement & e) { return e.sameOptions(front()); }); + } /// Resets flags which cannot be granted. - void removeNonGrantableFlags(); + void eraseNonGrantable(); + + /// If the database is empty, replaces it with `current_database`. Otherwise does nothing. + void replaceEmptyDatabase(const String & current_database) + { + for (auto & element : *this) + element.replaceEmptyDatabase(current_database); + } /// Returns a human-readable representation like "GRANT SELECT, UPDATE(x, y) ON db.table". String toString() const; + String toStringWithoutOptions() const; }; - -class AccessRightsElementsWithOptions : public std::vector -{ -public: - /// Replaces the empty database with `new_database`. - void replaceEmptyDatabase(const String & new_database); - - /// Resets flags which cannot be granted. - void removeNonGrantableFlags(); - - /// Returns a human-readable representation like "GRANT SELECT, UPDATE(x, y) ON db.table". - String toString() const; -}; - - -inline void AccessRightsElement::replaceEmptyDatabase(const String & new_database) -{ - if (isEmptyDatabase()) - database = new_database; -} - -inline void AccessRightsElements::replaceEmptyDatabase(const String & new_database) -{ - for (auto & element : *this) - element.replaceEmptyDatabase(new_database); -} - -inline void AccessRightsElementsWithOptions::replaceEmptyDatabase(const String & new_database) -{ - for (auto & element : *this) - element.replaceEmptyDatabase(new_database); -} - -inline void AccessRightsElement::removeNonGrantableFlags() -{ - if (!any_column) - access_flags &= AccessFlags::allFlagsGrantableOnColumnLevel(); - else if (!any_table) - access_flags &= AccessFlags::allFlagsGrantableOnTableLevel(); - else if (!any_database) - access_flags &= AccessFlags::allFlagsGrantableOnDatabaseLevel(); - else - access_flags &= AccessFlags::allFlagsGrantableOnGlobalLevel(); -} - -inline void AccessRightsElementWithOptions::removeNonGrantableFlags() -{ - if (kind == Kind::GRANT) - AccessRightsElement::removeNonGrantableFlags(); -} - -inline void AccessRightsElements::removeNonGrantableFlags() -{ - for (auto & element : *this) - element.removeNonGrantableFlags(); -} - -inline void AccessRightsElementsWithOptions::removeNonGrantableFlags() -{ - for (auto & element : *this) - element.removeNonGrantableFlags(); -} - } diff --git a/src/Access/ContextAccess.cpp b/src/Access/ContextAccess.cpp index 494da4eaeae..0bcaef1e441 100644 --- a/src/Access/ContextAccess.cpp +++ b/src/Access/ContextAccess.cpp @@ -177,28 +177,18 @@ void ContextAccess::setUser(const UserPtr & user_) const user_name = user->getName(); trace_log = &Poco::Logger::get("ContextAccess (" + user_name + ")"); - boost::container::flat_set current_roles, current_roles_with_admin_option; + std::vector current_roles, current_roles_with_admin_option; if (params.use_default_roles) { - for (const UUID & id : user->granted_roles.roles) - { - if (user->default_roles.match(id)) - current_roles.emplace(id); - } + current_roles = user->granted_roles.findGranted(user->default_roles); + current_roles_with_admin_option = user->granted_roles.findGrantedWithAdminOption(user->default_roles); } else { - boost::range::set_intersection( - params.current_roles, - user->granted_roles.roles, - std::inserter(current_roles, current_roles.end())); + current_roles = user->granted_roles.findGranted(params.current_roles); + current_roles_with_admin_option = user->granted_roles.findGrantedWithAdminOption(params.current_roles); } - boost::range::set_intersection( - current_roles, - user->granted_roles.roles_with_admin_option, - std::inserter(current_roles_with_admin_option, current_roles_with_admin_option.end())); - subscription_for_roles_changes = {}; enabled_roles = manager->getEnabledRoles(current_roles, current_roles_with_admin_option); subscription_for_roles_changes = enabled_roles->subscribeForChanges([this](const std::shared_ptr & roles_info_) @@ -331,47 +321,13 @@ std::shared_ptr ContextAccess::getAccessRightsWithImplicit() } -template -bool ContextAccess::checkAccessImpl(const AccessFlags & flags) const -{ - return checkAccessImpl2(flags); -} - template -bool ContextAccess::checkAccessImpl(const AccessFlags & flags, const std::string_view & database, const Args &... args) const -{ - return checkAccessImpl2(flags, database.empty() ? params.current_database : database, args...); -} - -template -bool ContextAccess::checkAccessImpl(const AccessRightsElement & element) const -{ - if (element.any_database) - return checkAccessImpl(element.access_flags); - else if (element.any_table) - return checkAccessImpl(element.access_flags, element.database); - else if (element.any_column) - return checkAccessImpl(element.access_flags, element.database, element.table); - else - return checkAccessImpl(element.access_flags, element.database, element.table, element.columns); -} - -template -bool ContextAccess::checkAccessImpl(const AccessRightsElements & elements) const -{ - for (const auto & element : elements) - if (!checkAccessImpl(element)) - return false; - return true; -} - -template -bool ContextAccess::checkAccessImpl2(const AccessFlags & flags, const Args &... args) const +bool ContextAccess::checkAccessImplHelper(const AccessFlags & flags, const Args &... args) const { auto access_granted = [&] { if (trace_log) - LOG_TRACE(trace_log, "Access granted: {}{}", (AccessRightsElement{flags, args...}.toString()), + LOG_TRACE(trace_log, "Access granted: {}{}", (AccessRightsElement{flags, args...}.toStringWithoutOptions()), (grant_option ? " WITH GRANT OPTION" : "")); return true; }; @@ -379,7 +335,7 @@ bool ContextAccess::checkAccessImpl2(const AccessFlags & flags, const Args &... auto access_denied = [&](const String & error_msg, int error_code [[maybe_unused]]) { if (trace_log) - LOG_TRACE(trace_log, "Access denied: {}{}", (AccessRightsElement{flags, args...}.toString()), + LOG_TRACE(trace_log, "Access denied: {}{}", (AccessRightsElement{flags, args...}.toStringWithoutOptions()), (grant_option ? " WITH GRANT OPTION" : "")); if constexpr (throw_if_denied) throw Exception(getUserName() + ": " + error_msg, error_code); @@ -415,13 +371,13 @@ bool ContextAccess::checkAccessImpl2(const AccessFlags & flags, const Args &... "Not enough privileges. " "The required privileges have been granted, but without grant option. " "To execute this query it's necessary to have grant " - + AccessRightsElement{flags, args...}.toString() + " WITH GRANT OPTION", + + AccessRightsElement{flags, args...}.toStringWithoutOptions() + " WITH GRANT OPTION", ErrorCodes::ACCESS_DENIED); } return access_denied( "Not enough privileges. To execute this query it's necessary to have grant " - + AccessRightsElement{flags, args...}.toString() + (grant_option ? " WITH GRANT OPTION" : ""), + + AccessRightsElement{flags, args...}.toStringWithoutOptions() + (grant_option ? " WITH GRANT OPTION" : ""), ErrorCodes::ACCESS_DENIED); } @@ -478,6 +434,56 @@ bool ContextAccess::checkAccessImpl2(const AccessFlags & flags, const Args &... return access_granted(); } +template +bool ContextAccess::checkAccessImpl(const AccessFlags & flags) const +{ + return checkAccessImplHelper(flags); +} + +template +bool ContextAccess::checkAccessImpl(const AccessFlags & flags, const std::string_view & database, const Args &... args) const +{ + return checkAccessImplHelper(flags, database.empty() ? params.current_database : database, args...); +} + +template +bool ContextAccess::checkAccessImplHelper(const AccessRightsElement & element) const +{ + assert(!element.grant_option || grant_option); + if (element.any_database) + return checkAccessImpl(element.access_flags); + else if (element.any_table) + return checkAccessImpl(element.access_flags, element.database); + else if (element.any_column) + return checkAccessImpl(element.access_flags, element.database, element.table); + else + return checkAccessImpl(element.access_flags, element.database, element.table, element.columns); +} + +template +bool ContextAccess::checkAccessImpl(const AccessRightsElement & element) const +{ + if constexpr (grant_option) + { + return checkAccessImplHelper(element); + } + else + { + if (element.grant_option) + return checkAccessImplHelper(element); + else + return checkAccessImplHelper(element); + } +} + +template +bool ContextAccess::checkAccessImpl(const AccessRightsElements & elements) const +{ + for (const auto & element : elements) + if (!checkAccessImpl(element)) + return false; + return true; +} bool ContextAccess::isGranted(const AccessFlags & flags) const { return checkAccessImpl(flags); } bool ContextAccess::isGranted(const AccessFlags & flags, const std::string_view & database) const { return checkAccessImpl(flags, database); } @@ -516,44 +522,8 @@ void ContextAccess::checkGrantOption(const AccessRightsElement & element) const void ContextAccess::checkGrantOption(const AccessRightsElements & elements) const { checkAccessImpl(elements); } -template -bool ContextAccess::checkAdminOptionImpl(const UUID & role_id) const -{ - return checkAdminOptionImpl2(to_array(role_id), [this](const UUID & id, size_t) { return manager->tryReadName(id); }); -} - -template -bool ContextAccess::checkAdminOptionImpl(const UUID & role_id, const String & role_name) const -{ - return checkAdminOptionImpl2(to_array(role_id), [&role_name](const UUID &, size_t) { return std::optional{role_name}; }); -} - -template -bool ContextAccess::checkAdminOptionImpl(const UUID & role_id, const std::unordered_map & names_of_roles) const -{ - return checkAdminOptionImpl2(to_array(role_id), [&names_of_roles](const UUID & id, size_t) { auto it = names_of_roles.find(id); return (it != names_of_roles.end()) ? it->second : std::optional{}; }); -} - -template -bool ContextAccess::checkAdminOptionImpl(const std::vector & role_ids) const -{ - return checkAdminOptionImpl2(role_ids, [this](const UUID & id, size_t) { return manager->tryReadName(id); }); -} - -template -bool ContextAccess::checkAdminOptionImpl(const std::vector & role_ids, const Strings & names_of_roles) const -{ - return checkAdminOptionImpl2(role_ids, [&names_of_roles](const UUID &, size_t i) { return std::optional{names_of_roles[i]}; }); -} - -template -bool ContextAccess::checkAdminOptionImpl(const std::vector & role_ids, const std::unordered_map & names_of_roles) const -{ - return checkAdminOptionImpl2(role_ids, [&names_of_roles](const UUID & id, size_t) { auto it = names_of_roles.find(id); return (it != names_of_roles.end()) ? it->second : std::optional{}; }); -} - template -bool ContextAccess::checkAdminOptionImpl2(const Container & role_ids, const GetNameFunction & get_name_function) const +bool ContextAccess::checkAdminOptionImplHelper(const Container & role_ids, const GetNameFunction & get_name_function) const { if (!std::size(role_ids) || is_full_access) return true; @@ -605,6 +575,42 @@ bool ContextAccess::checkAdminOptionImpl2(const Container & role_ids, const GetN return true; } +template +bool ContextAccess::checkAdminOptionImpl(const UUID & role_id) const +{ + return checkAdminOptionImplHelper(to_array(role_id), [this](const UUID & id, size_t) { return manager->tryReadName(id); }); +} + +template +bool ContextAccess::checkAdminOptionImpl(const UUID & role_id, const String & role_name) const +{ + return checkAdminOptionImplHelper(to_array(role_id), [&role_name](const UUID &, size_t) { return std::optional{role_name}; }); +} + +template +bool ContextAccess::checkAdminOptionImpl(const UUID & role_id, const std::unordered_map & names_of_roles) const +{ + return checkAdminOptionImplHelper(to_array(role_id), [&names_of_roles](const UUID & id, size_t) { auto it = names_of_roles.find(id); return (it != names_of_roles.end()) ? it->second : std::optional{}; }); +} + +template +bool ContextAccess::checkAdminOptionImpl(const std::vector & role_ids) const +{ + return checkAdminOptionImplHelper(role_ids, [this](const UUID & id, size_t) { return manager->tryReadName(id); }); +} + +template +bool ContextAccess::checkAdminOptionImpl(const std::vector & role_ids, const Strings & names_of_roles) const +{ + return checkAdminOptionImplHelper(role_ids, [&names_of_roles](const UUID &, size_t i) { return std::optional{names_of_roles[i]}; }); +} + +template +bool ContextAccess::checkAdminOptionImpl(const std::vector & role_ids, const std::unordered_map & names_of_roles) const +{ + return checkAdminOptionImplHelper(role_ids, [&names_of_roles](const UUID & id, size_t) { auto it = names_of_roles.find(id); return (it != names_of_roles.end()) ? it->second : std::optional{}; }); +} + bool ContextAccess::hasAdminOption(const UUID & role_id) const { return checkAdminOptionImpl(role_id); } bool ContextAccess::hasAdminOption(const UUID & role_id, const String & role_name) const { return checkAdminOptionImpl(role_id, role_name); } bool ContextAccess::hasAdminOption(const UUID & role_id, const std::unordered_map & names_of_roles) const { return checkAdminOptionImpl(role_id, names_of_roles); } diff --git a/src/Access/ContextAccess.h b/src/Access/ContextAccess.h index 43e9f60a4c6..320c2566769 100644 --- a/src/Access/ContextAccess.h +++ b/src/Access/ContextAccess.h @@ -99,25 +99,6 @@ public: std::shared_ptr getAccessRights() const; std::shared_ptr getAccessRightsWithImplicit() const; - /// Checks if a specified access is granted. - bool isGranted(const AccessFlags & flags) const; - bool isGranted(const AccessFlags & flags, const std::string_view & database) const; - bool isGranted(const AccessFlags & flags, const std::string_view & database, const std::string_view & table) const; - bool isGranted(const AccessFlags & flags, const std::string_view & database, const std::string_view & table, const std::string_view & column) const; - bool isGranted(const AccessFlags & flags, const std::string_view & database, const std::string_view & table, const std::vector & columns) const; - bool isGranted(const AccessFlags & flags, const std::string_view & database, const std::string_view & table, const Strings & columns) const; - bool isGranted(const AccessRightsElement & element) const; - bool isGranted(const AccessRightsElements & elements) const; - - bool hasGrantOption(const AccessFlags & flags) const; - bool hasGrantOption(const AccessFlags & flags, const std::string_view & database) const; - bool hasGrantOption(const AccessFlags & flags, const std::string_view & database, const std::string_view & table) const; - bool hasGrantOption(const AccessFlags & flags, const std::string_view & database, const std::string_view & table, const std::string_view & column) const; - bool hasGrantOption(const AccessFlags & flags, const std::string_view & database, const std::string_view & table, const std::vector & columns) const; - bool hasGrantOption(const AccessFlags & flags, const std::string_view & database, const std::string_view & table, const Strings & columns) const; - bool hasGrantOption(const AccessRightsElement & element) const; - bool hasGrantOption(const AccessRightsElements & elements) const; - /// Checks if a specified access is granted, and throws an exception if not. /// Empty database means the current database. void checkAccess(const AccessFlags & flags) const; @@ -138,6 +119,26 @@ public: void checkGrantOption(const AccessRightsElement & element) const; void checkGrantOption(const AccessRightsElements & elements) const; + /// Checks if a specified access is granted, and returns false if not. + /// Empty database means the current database. + bool isGranted(const AccessFlags & flags) const; + bool isGranted(const AccessFlags & flags, const std::string_view & database) const; + bool isGranted(const AccessFlags & flags, const std::string_view & database, const std::string_view & table) const; + bool isGranted(const AccessFlags & flags, const std::string_view & database, const std::string_view & table, const std::string_view & column) const; + bool isGranted(const AccessFlags & flags, const std::string_view & database, const std::string_view & table, const std::vector & columns) const; + bool isGranted(const AccessFlags & flags, const std::string_view & database, const std::string_view & table, const Strings & columns) const; + bool isGranted(const AccessRightsElement & element) const; + bool isGranted(const AccessRightsElements & elements) const; + + bool hasGrantOption(const AccessFlags & flags) const; + bool hasGrantOption(const AccessFlags & flags, const std::string_view & database) const; + bool hasGrantOption(const AccessFlags & flags, const std::string_view & database, const std::string_view & table) const; + bool hasGrantOption(const AccessFlags & flags, const std::string_view & database, const std::string_view & table, const std::string_view & column) const; + bool hasGrantOption(const AccessFlags & flags, const std::string_view & database, const std::string_view & table, const std::vector & columns) const; + bool hasGrantOption(const AccessFlags & flags, const std::string_view & database, const std::string_view & table, const Strings & columns) const; + bool hasGrantOption(const AccessRightsElement & element) const; + bool hasGrantOption(const AccessRightsElements & elements) const; + /// Checks if a specified role is granted with admin option, and throws an exception if not. void checkAdminOption(const UUID & role_id) const; void checkAdminOption(const UUID & role_id, const String & role_name) const; @@ -146,6 +147,7 @@ public: void checkAdminOption(const std::vector & role_ids, const Strings & names_of_roles) const; void checkAdminOption(const std::vector & role_ids, const std::unordered_map & names_of_roles) const; + /// Checks if a specified role is granted with admin option, and returns false if not. bool hasAdminOption(const UUID & role_id) const; bool hasAdminOption(const UUID & role_id, const String & role_name) const; bool hasAdminOption(const UUID & role_id, const std::unordered_map & names_of_roles) const; @@ -180,7 +182,10 @@ private: bool checkAccessImpl(const AccessRightsElements & elements) const; template - bool checkAccessImpl2(const AccessFlags & flags, const Args &... args) const; + bool checkAccessImplHelper(const AccessFlags & flags, const Args &... args) const; + + template + bool checkAccessImplHelper(const AccessRightsElement & element) const; template bool checkAdminOptionImpl(const UUID & role_id) const; @@ -201,7 +206,7 @@ private: bool checkAdminOptionImpl(const std::vector & role_ids, const std::unordered_map & names_of_roles) const; template - bool checkAdminOptionImpl2(const Container & role_ids, const GetNameFunction & get_name_function) const; + bool checkAdminOptionImplHelper(const Container & role_ids, const GetNameFunction & get_name_function) const; const AccessControlManager * manager = nullptr; const Params params; diff --git a/src/Access/GrantedRoles.cpp b/src/Access/GrantedRoles.cpp index 4d7007c4db6..7930b56e44d 100644 --- a/src/Access/GrantedRoles.cpp +++ b/src/Access/GrantedRoles.cpp @@ -1,37 +1,38 @@ #include +#include #include +#include namespace DB { -void GrantedRoles::grant(const UUID & role) +void GrantedRoles::grant(const UUID & role_) { - roles.insert(role); + roles.insert(role_); } void GrantedRoles::grant(const std::vector & roles_) { - for (const UUID & role : roles_) - grant(role); + roles.insert(roles_.begin(), roles_.end()); } -void GrantedRoles::grantWithAdminOption(const UUID & role) +void GrantedRoles::grantWithAdminOption(const UUID & role_) { - roles.insert(role); - roles_with_admin_option.insert(role); + roles.insert(role_); + roles_with_admin_option.insert(role_); } void GrantedRoles::grantWithAdminOption(const std::vector & roles_) { - for (const UUID & role : roles_) - grantWithAdminOption(role); + roles.insert(roles_.begin(), roles_.end()); + roles_with_admin_option.insert(roles_.begin(), roles_.end()); } -void GrantedRoles::revoke(const UUID & role) +void GrantedRoles::revoke(const UUID & role_) { - roles.erase(role); - roles_with_admin_option.erase(role); + roles.erase(role_); + roles_with_admin_option.erase(role_); } void GrantedRoles::revoke(const std::vector & roles_) @@ -40,9 +41,9 @@ void GrantedRoles::revoke(const std::vector & roles_) revoke(role); } -void GrantedRoles::revokeAdminOption(const UUID & role) +void GrantedRoles::revokeAdminOption(const UUID & role_) { - roles_with_admin_option.erase(role); + roles_with_admin_option.erase(role_); } void GrantedRoles::revokeAdminOption(const std::vector & roles_) @@ -52,13 +53,118 @@ void GrantedRoles::revokeAdminOption(const std::vector & roles_) } -GrantedRoles::Grants GrantedRoles::getGrants() const +bool GrantedRoles::isGranted(const UUID & role_) const { - Grants res; - res.grants_with_admin_option.insert(res.grants_with_admin_option.end(), roles_with_admin_option.begin(), roles_with_admin_option.end()); - res.grants.reserve(roles.size() - roles_with_admin_option.size()); - boost::range::set_difference(roles, roles_with_admin_option, std::back_inserter(res.grants)); + return roles.count(role_); +} + +bool GrantedRoles::isGrantedWithAdminOption(const UUID & role_) const +{ + return roles_with_admin_option.count(role_); +} + + +std::vector GrantedRoles::findGranted(const std::vector & ids) const +{ + std::vector res; + res.reserve(ids.size()); + for (const UUID & id : ids) + { + if (isGranted(id)) + res.push_back(id); + } return res; } +std::vector GrantedRoles::findGranted(const boost::container::flat_set & ids) const +{ + std::vector res; + res.reserve(ids.size()); + boost::range::set_difference(ids, roles, std::back_inserter(res)); + return res; +} + +std::vector GrantedRoles::findGranted(const RolesOrUsersSet & ids) const +{ + std::vector res; + for (const UUID & id : roles) + { + if (ids.match(id)) + res.emplace_back(id); + } + return res; +} + +std::vector GrantedRoles::findGrantedWithAdminOption(const std::vector & ids) const +{ + std::vector res; + res.reserve(ids.size()); + for (const UUID & id : ids) + { + if (isGrantedWithAdminOption(id)) + res.push_back(id); + } + return res; +} + +std::vector GrantedRoles::findGrantedWithAdminOption(const boost::container::flat_set & ids) const +{ + std::vector res; + res.reserve(ids.size()); + boost::range::set_difference(ids, roles_with_admin_option, std::back_inserter(res)); + return res; +} + +std::vector GrantedRoles::findGrantedWithAdminOption(const RolesOrUsersSet & ids) const +{ + std::vector res; + for (const UUID & id : roles_with_admin_option) + { + if (ids.match(id)) + res.emplace_back(id); + } + return res; +} + + +GrantedRoles::Elements GrantedRoles::getElements() const +{ + Elements elements; + + Element element; + element.ids.reserve(roles.size()); + boost::range::set_difference(roles, roles_with_admin_option, std::back_inserter(element.ids)); + if (!element.empty()) + { + element.admin_option = false; + elements.emplace_back(std::move(element)); + } + + if (!roles_with_admin_option.empty()) + { + element = {}; + element.ids.insert(element.ids.end(), roles_with_admin_option.begin(), roles_with_admin_option.end()); + element.admin_option = true; + elements.emplace_back(std::move(element)); + } + + return elements; +} + + +void GrantedRoles::makeUnion(const GrantedRoles & other) +{ + roles.insert(other.roles.begin(), other.roles.end()); + roles_with_admin_option.insert(other.roles_with_admin_option.begin(), other.roles_with_admin_option.end()); +} + +void GrantedRoles::makeIntersection(const GrantedRoles & other) +{ + boost::range::remove_erase_if(roles, [&other](const UUID & id) { return other.roles.find(id) == other.roles.end(); }); + + boost::range::remove_erase_if(roles_with_admin_option, [&other](const UUID & id) + { + return other.roles_with_admin_option.find(id) == other.roles_with_admin_option.end(); + }); +} } diff --git a/src/Access/GrantedRoles.h b/src/Access/GrantedRoles.h index fd091755a80..75ea56aba96 100644 --- a/src/Access/GrantedRoles.h +++ b/src/Access/GrantedRoles.h @@ -7,33 +7,55 @@ namespace DB { +struct RolesOrUsersSet; + /// Roles when they are granted to a role or user. /// Stores both the roles themselves and the roles with admin option. -struct GrantedRoles +class GrantedRoles { - boost::container::flat_set roles; - boost::container::flat_set roles_with_admin_option; - - void grant(const UUID & role); +public: + void grant(const UUID & role_); void grant(const std::vector & roles_); - void grantWithAdminOption(const UUID & role); + void grantWithAdminOption(const UUID & role_); void grantWithAdminOption(const std::vector & roles_); - void revoke(const UUID & role); + void revoke(const UUID & role_); void revoke(const std::vector & roles_); - void revokeAdminOption(const UUID & role); + void revokeAdminOption(const UUID & role_); void revokeAdminOption(const std::vector & roles_); - struct Grants + bool isGranted(const UUID & role_) const; + bool isGrantedWithAdminOption(const UUID & role_) const; + + const boost::container::flat_set & getGranted() const { return roles; } + const boost::container::flat_set & getGrantedWithAdminOption() const { return roles_with_admin_option; } + + std::vector findGranted(const std::vector & ids) const; + std::vector findGranted(const boost::container::flat_set & ids) const; + std::vector findGranted(const RolesOrUsersSet & ids) const; + std::vector findGrantedWithAdminOption(const std::vector & ids) const; + std::vector findGrantedWithAdminOption(const boost::container::flat_set & ids) const; + std::vector findGrantedWithAdminOption(const RolesOrUsersSet & ids) const; + + struct Element { - std::vector grants; - std::vector grants_with_admin_option; + std::vector ids; + bool admin_option = false; + bool empty() const { return ids.empty(); } }; + using Elements = std::vector; /// Retrieves the information about grants. - Grants getGrants() const; + Elements getElements() const; + + void makeUnion(const GrantedRoles & other); + void makeIntersection(const GrantedRoles & other); friend bool operator ==(const GrantedRoles & left, const GrantedRoles & right) { return (left.roles == right.roles) && (left.roles_with_admin_option == right.roles_with_admin_option); } friend bool operator !=(const GrantedRoles & left, const GrantedRoles & right) { return !(left == right); } + +private: + boost::container::flat_set roles; + boost::container::flat_set roles_with_admin_option; }; } diff --git a/src/Access/LDAPAccessStorage.cpp b/src/Access/LDAPAccessStorage.cpp index b69f489ca6b..b47a9b3e041 100644 --- a/src/Access/LDAPAccessStorage.cpp +++ b/src/Access/LDAPAccessStorage.cpp @@ -187,13 +187,10 @@ void LDAPAccessStorage::applyRoleChangeNoLock(bool grant, const UUID & role_id, if (auto user = typeid_cast>(entity_)) { auto changed_user = typeid_cast>(user->clone()); - auto & granted_roles = changed_user->granted_roles.roles; - if (grant) - granted_roles.insert(role_id); + changed_user->granted_roles.grant(role_id); else - granted_roles.erase(role_id); - + changed_user->granted_roles.revoke(role_id); return changed_user; } return entity_; @@ -229,7 +226,7 @@ void LDAPAccessStorage::assignRolesNoLock(User & user, const LDAPClient::SearchR void LDAPAccessStorage::assignRolesNoLock(User & user, const LDAPClient::SearchResultsList & external_roles, const std::size_t external_roles_hash) const { const auto & user_name = user.getName(); - auto & granted_roles = user.granted_roles.roles; + auto & granted_roles = user.granted_roles; const auto local_role_names = mapExternalRolesNoLock(external_roles); auto grant_role = [this, &user_name, &granted_roles] (const String & role_name, const bool common) @@ -247,7 +244,7 @@ void LDAPAccessStorage::assignRolesNoLock(User & user, const LDAPClient::SearchR if (it != granted_role_ids.end()) { const auto & role_id = it->second; - granted_roles.insert(role_id); + granted_roles.grant(role_id); } else { @@ -256,7 +253,7 @@ void LDAPAccessStorage::assignRolesNoLock(User & user, const LDAPClient::SearchR }; external_role_hashes.erase(user_name); - granted_roles.clear(); + granted_roles = {}; const auto old_role_names = std::move(roles_per_users[user_name]); // Grant the common roles first. diff --git a/src/Access/RoleCache.cpp b/src/Access/RoleCache.cpp index f386044bbf7..8fa3d51f867 100644 --- a/src/Access/RoleCache.cpp +++ b/src/Access/RoleCache.cpp @@ -46,10 +46,10 @@ namespace roles_info.access.makeUnion(role->access); roles_info.settings_from_enabled_roles.merge(role->settings); - for (const auto & granted_role : role->granted_roles.roles) + for (const auto & granted_role : role->granted_roles.getGranted()) collectRoles(roles_info, skip_ids, get_role_function, granted_role, false, false); - for (const auto & granted_role : role->granted_roles.roles_with_admin_option) + for (const auto & granted_role : role->granted_roles.getGrantedWithAdminOption()) collectRoles(roles_info, skip_ids, get_role_function, granted_role, false, true); } } @@ -63,15 +63,15 @@ RoleCache::~RoleCache() = default; std::shared_ptr -RoleCache::getEnabledRoles(const boost::container::flat_set & roles, const boost::container::flat_set & roles_with_admin_option) +RoleCache::getEnabledRoles(const std::vector & roles, const std::vector & roles_with_admin_option) { /// Declared before `lock` to send notifications after the mutex will be unlocked. ext::scope_guard notifications; std::lock_guard lock{mutex}; EnabledRoles::Params params; - params.current_roles = roles; - params.current_roles_with_admin_option = roles_with_admin_option; + params.current_roles.insert(roles.begin(), roles.end()); + params.current_roles_with_admin_option.insert(roles_with_admin_option.begin(), roles_with_admin_option.end()); auto it = enabled_roles.find(params); if (it != enabled_roles.end()) { diff --git a/src/Access/RoleCache.h b/src/Access/RoleCache.h index cc6c8599f27..b3f426debcb 100644 --- a/src/Access/RoleCache.h +++ b/src/Access/RoleCache.h @@ -20,7 +20,8 @@ public: ~RoleCache(); std::shared_ptr getEnabledRoles( - const boost::container::flat_set & current_roles, const boost::container::flat_set & current_roles_with_admin_option); + const std::vector & current_roles, + const std::vector & current_roles_with_admin_option); private: void collectEnabledRoles(ext::scope_guard & notifications); diff --git a/src/Access/RolesOrUsersSet.cpp b/src/Access/RolesOrUsersSet.cpp index cb0beb42700..ebd4f0f7a40 100644 --- a/src/Access/RolesOrUsersSet.cpp +++ b/src/Access/RolesOrUsersSet.cpp @@ -72,20 +72,20 @@ void RolesOrUsersSet::init(const ASTRolesOrUsersSet & ast, const AccessControlMa if (ast.id_mode) return parse(name); assert(manager); - if (ast.allow_user_names && ast.allow_role_names) + if (ast.allow_users && ast.allow_roles) { auto id = manager->find(name); if (id) return *id; return manager->getID(name); } - else if (ast.allow_user_names) + else if (ast.allow_users) { return manager->getID(name); } else { - assert(ast.allow_role_names); + assert(ast.allow_roles); return manager->getID(name); } }; @@ -106,8 +106,8 @@ void RolesOrUsersSet::init(const ASTRolesOrUsersSet & ast, const AccessControlMa if (!ast.except_names.empty()) { except_ids.reserve(ast.except_names.size()); - for (const String & except_name : ast.except_names) - except_ids.insert(name_to_id(except_name)); + for (const String & name : ast.except_names) + except_ids.insert(name_to_id(name)); } if (ast.except_current_user) @@ -116,8 +116,8 @@ void RolesOrUsersSet::init(const ASTRolesOrUsersSet & ast, const AccessControlMa except_ids.insert(*current_user_id); } - for (const UUID & except_id : except_ids) - ids.erase(except_id); + for (const UUID & id : except_ids) + ids.erase(id); } @@ -127,7 +127,7 @@ std::shared_ptr RolesOrUsersSet::toAST() const ast->id_mode = true; ast->all = all; - if (!ids.empty()) + if (!ids.empty() && !all) { ast->names.reserve(ids.size()); for (const UUID & id : ids) @@ -152,7 +152,7 @@ std::shared_ptr RolesOrUsersSet::toASTWithNames(const Access auto ast = std::make_shared(); ast->all = all; - if (!ids.empty()) + if (!ids.empty() && !all) { ast->names.reserve(ids.size()); for (const UUID & id : ids) @@ -194,44 +194,6 @@ String RolesOrUsersSet::toStringWithNames(const AccessControlManager & manager) } -Strings RolesOrUsersSet::toStringsWithNames(const AccessControlManager & manager) const -{ - if (!all && ids.empty()) - return {}; - - Strings res; - res.reserve(ids.size() + except_ids.size()); - - if (all) - res.emplace_back("ALL"); - else - { - for (const UUID & id : ids) - { - auto name = manager.tryReadName(id); - if (name) - res.emplace_back(std::move(*name)); - } - std::sort(res.begin(), res.end()); - } - - if (!except_ids.empty()) - { - res.emplace_back("EXCEPT"); - size_t old_size = res.size(); - for (const UUID & id : except_ids) - { - auto name = manager.tryReadName(id); - if (name) - res.emplace_back(std::move(*name)); - } - std::sort(res.begin() + old_size, res.end()); - } - - return res; -} - - bool RolesOrUsersSet::empty() const { return ids.empty() && !all; @@ -248,14 +210,18 @@ void RolesOrUsersSet::clear() void RolesOrUsersSet::add(const UUID & id) { - ids.insert(id); + if (!all) + ids.insert(id); + except_ids.erase(id); } void RolesOrUsersSet::add(const std::vector & ids_) { + if (!all) + ids.insert(ids_.begin(), ids_.end()); for (const auto & id : ids_) - add(id); + except_ids.erase(id); } diff --git a/src/Access/RolesOrUsersSet.h b/src/Access/RolesOrUsersSet.h index bae7f52a574..0d8983c2ec3 100644 --- a/src/Access/RolesOrUsersSet.h +++ b/src/Access/RolesOrUsersSet.h @@ -13,7 +13,8 @@ class AccessControlManager; /// Represents a set of users/roles like -/// {user_name | role_name | CURRENT_USER} [,...] | NONE | ALL | ALL EXCEPT {user_name | role_name | CURRENT_USER} [,...] +/// {user_name | role_name | CURRENT_USER | ALL | NONE} [,...] +/// [EXCEPT {user_name | role_name | CURRENT_USER | ALL | NONE} [,...]] /// Similar to ASTRolesOrUsersSet, but with IDs instead of names. struct RolesOrUsersSet { @@ -60,8 +61,8 @@ struct RolesOrUsersSet friend bool operator ==(const RolesOrUsersSet & lhs, const RolesOrUsersSet & rhs); friend bool operator !=(const RolesOrUsersSet & lhs, const RolesOrUsersSet & rhs) { return !(lhs == rhs); } - boost::container::flat_set ids; bool all = false; + boost::container::flat_set ids; boost::container::flat_set except_ids; private: diff --git a/src/Interpreters/Context.cpp b/src/Interpreters/Context.cpp index 8615cf70343..62a3d8a279d 100644 --- a/src/Interpreters/Context.cpp +++ b/src/Interpreters/Context.cpp @@ -753,7 +753,7 @@ std::optional Context::getUserID() const } -void Context::setCurrentRoles(const boost::container::flat_set & current_roles_) +void Context::setCurrentRoles(const std::vector & current_roles_) { auto lock = getLock(); if (current_roles == current_roles_ && !use_default_roles) diff --git a/src/Interpreters/Context.h b/src/Interpreters/Context.h index 563239cd88d..d89799e9d97 100644 --- a/src/Interpreters/Context.h +++ b/src/Interpreters/Context.h @@ -181,7 +181,7 @@ private: InputBlocksReader input_blocks_reader; std::optional user_id; - boost::container::flat_set current_roles; + std::vector current_roles; bool use_default_roles = false; std::shared_ptr access; std::shared_ptr initial_row_policy; @@ -354,7 +354,7 @@ public: String getUserName() const; std::optional getUserID() const; - void setCurrentRoles(const boost::container::flat_set & current_roles_); + void setCurrentRoles(const std::vector & current_roles_); void setCurrentRolesDefault(); boost::container::flat_set getCurrentRoles() const; boost::container::flat_set getEnabledRoles() const; diff --git a/src/Interpreters/InterpreterCreateQuotaQuery.cpp b/src/Interpreters/InterpreterCreateQuotaQuery.cpp index ff30a2fff47..4c35aaf573b 100644 --- a/src/Interpreters/InterpreterCreateQuotaQuery.cpp +++ b/src/Interpreters/InterpreterCreateQuotaQuery.cpp @@ -78,7 +78,7 @@ BlockIO InterpreterCreateQuotaQuery::execute() if (!query.cluster.empty()) { - query.replaceCurrentUserTagWithName(context.getUserName()); + query.replaceCurrentUserTag(context.getUserName()); return executeDDLQueryOnCluster(query_ptr, context); } diff --git a/src/Interpreters/InterpreterCreateRowPolicyQuery.cpp b/src/Interpreters/InterpreterCreateRowPolicyQuery.cpp index 8f1c5b061e0..0932f74cdc5 100644 --- a/src/Interpreters/InterpreterCreateRowPolicyQuery.cpp +++ b/src/Interpreters/InterpreterCreateRowPolicyQuery.cpp @@ -49,7 +49,7 @@ BlockIO InterpreterCreateRowPolicyQuery::execute() if (!query.cluster.empty()) { - query.replaceCurrentUserTagWithName(context.getUserName()); + query.replaceCurrentUserTag(context.getUserName()); return executeDDLQueryOnCluster(query_ptr, context); } @@ -58,7 +58,7 @@ BlockIO InterpreterCreateRowPolicyQuery::execute() if (query.roles) roles_from_query = RolesOrUsersSet{*query.roles, access_control, context.getUserID()}; - query.replaceEmptyDatabaseWithCurrent(context.getCurrentDatabase()); + query.replaceEmptyDatabase(context.getCurrentDatabase()); if (query.alter) { diff --git a/src/Interpreters/InterpreterCreateSettingsProfileQuery.cpp b/src/Interpreters/InterpreterCreateSettingsProfileQuery.cpp index b65225db16c..0931b48e723 100644 --- a/src/Interpreters/InterpreterCreateSettingsProfileQuery.cpp +++ b/src/Interpreters/InterpreterCreateSettingsProfileQuery.cpp @@ -50,7 +50,7 @@ BlockIO InterpreterCreateSettingsProfileQuery::execute() if (!query.cluster.empty()) { - query.replaceCurrentUserTagWithName(context.getUserName()); + query.replaceCurrentUserTag(context.getUserName()); return executeDDLQueryOnCluster(query_ptr, context); } diff --git a/src/Interpreters/InterpreterDropAccessEntityQuery.cpp b/src/Interpreters/InterpreterDropAccessEntityQuery.cpp index e86f8361100..e02c047e75d 100644 --- a/src/Interpreters/InterpreterDropAccessEntityQuery.cpp +++ b/src/Interpreters/InterpreterDropAccessEntityQuery.cpp @@ -31,7 +31,7 @@ BlockIO InterpreterDropAccessEntityQuery::execute() if (!query.cluster.empty()) return executeDDLQueryOnCluster(query_ptr, context); - query.replaceEmptyDatabaseWithCurrent(context.getCurrentDatabase()); + query.replaceEmptyDatabase(context.getCurrentDatabase()); auto do_drop = [&](const Strings & names) { diff --git a/src/Interpreters/InterpreterGrantQuery.cpp b/src/Interpreters/InterpreterGrantQuery.cpp index 034ebcec050..ac37266bcfb 100644 --- a/src/Interpreters/InterpreterGrantQuery.cpp +++ b/src/Interpreters/InterpreterGrantQuery.cpp @@ -12,13 +12,15 @@ #include #include - namespace DB { +namespace ErrorCodes +{ + extern const int LOGICAL_ERROR; +} + namespace { - using Kind = ASTGrantQuery::Kind; - template void updateFromQueryTemplate( T & grantee, @@ -27,38 +29,28 @@ namespace { if (!query.access_rights_elements.empty()) { - if (query.kind == Kind::GRANT) - { - if (query.grant_option) - grantee.access.grantWithGrantOption(query.access_rights_elements); - else - grantee.access.grant(query.access_rights_elements); - } + if (query.is_revoke) + grantee.access.revoke(query.access_rights_elements); else - { - if (query.grant_option) - grantee.access.revokeGrantOption(query.access_rights_elements); - else - grantee.access.revoke(query.access_rights_elements); - } + grantee.access.grant(query.access_rights_elements); } if (!roles_to_grant_or_revoke.empty()) { - if (query.kind == Kind::GRANT) - { - if (query.admin_option) - grantee.granted_roles.grantWithAdminOption(roles_to_grant_or_revoke); - else - grantee.granted_roles.grant(roles_to_grant_or_revoke); - } - else + if (query.is_revoke) { if (query.admin_option) grantee.granted_roles.revokeAdminOption(roles_to_grant_or_revoke); else grantee.granted_roles.revoke(roles_to_grant_or_revoke); } + else + { + if (query.admin_option) + grantee.granted_roles.grantWithAdminOption(roles_to_grant_or_revoke); + else + grantee.granted_roles.grant(roles_to_grant_or_revoke); + } } } @@ -72,122 +64,166 @@ namespace else if (auto * role = typeid_cast(&grantee)) updateFromQueryTemplate(*role, query, roles_to_grant_or_revoke); } + + void checkGrantOption( + const AccessControlManager & access_control, + const ContextAccess & access, + const ASTGrantQuery & query, + const std::vector & grantees_from_query) + { + const auto & elements = query.access_rights_elements; + if (elements.empty()) + return; + + /// To execute the command GRANT the current user needs to have the access granted + /// with GRANT OPTION. + if (!query.is_revoke) + { + access.checkGrantOption(elements); + return; + } + + if (access.hasGrantOption(elements)) + return; + + /// Special case for the command REVOKE: it's possible that the current user doesn't have + /// the access granted with GRANT OPTION but it's still ok because the roles or users + /// from whom the access rights will be revoked don't have the specified access granted either. + /// + /// For example, to execute + /// GRANT ALL ON mydb.* TO role1 + /// REVOKE ALL ON *.* FROM role1 + /// the current user needs to have grants only on the 'mydb' database. + AccessRights all_granted_access; + for (const auto & id : grantees_from_query) + { + auto entity = access_control.tryRead(id); + if (auto role = typeid_cast(entity)) + all_granted_access.makeUnion(role->access); + else if (auto user = typeid_cast(entity)) + all_granted_access.makeUnion(user->access); + } + + AccessRights required_access; + if (elements[0].is_partial_revoke) + { + AccessRightsElements non_revoke_elements = elements; + std::for_each(non_revoke_elements.begin(), non_revoke_elements.end(), [&](AccessRightsElement & element) { element.is_partial_revoke = false; }); + required_access.grant(non_revoke_elements); + } + else + { + required_access.grant(elements); + } + required_access.makeIntersection(all_granted_access); + + for (auto & required_access_element : required_access.getElements()) + { + if (!required_access_element.is_partial_revoke && (required_access_element.grant_option || !elements[0].grant_option)) + access.checkGrantOption(required_access_element); + } + } + + + std::vector getRoleIDsAndCheckAdminOption( + const AccessControlManager & access_control, + const ContextAccess & access, + const ASTGrantQuery & query, + const RolesOrUsersSet & roles_from_query, + const std::vector & grantees_from_query) + { + std::vector matching_ids; + + if (!query.is_revoke) + { + matching_ids = roles_from_query.getMatchingIDs(access_control); + access.checkAdminOption(matching_ids); + return matching_ids; + } + + if (!roles_from_query.all) + { + matching_ids = roles_from_query.getMatchingIDs(); + if (access.hasAdminOption(matching_ids)) + return matching_ids; + } + + /// Special case for the command REVOKE: it's possible that the current user doesn't have the admin option + /// for some of the specified roles but it's still ok because the roles or users from whom the roles will be + /// revoked from don't have the specified roles granted either. + /// + /// For example, to execute + /// GRANT role2 TO role1 + /// REVOKE ALL FROM role1 + /// the current user needs to have only 'role2' to be granted with admin option (not all the roles). + GrantedRoles all_granted_roles; + for (const auto & id : grantees_from_query) + { + auto entity = access_control.tryRead(id); + if (auto role = typeid_cast(entity)) + all_granted_roles.makeUnion(role->granted_roles); + else if (auto user = typeid_cast(entity)) + all_granted_roles.makeUnion(user->granted_roles); + } + + const auto & all_granted_roles_set = query.admin_option ? all_granted_roles.getGrantedWithAdminOption() : all_granted_roles.getGranted(); + if (roles_from_query.all) + boost::range::set_difference(all_granted_roles_set, roles_from_query.except_ids, std::back_inserter(matching_ids)); + else + boost::range::remove_erase_if(matching_ids, [&](const UUID & id) { return !all_granted_roles_set.count(id); }); + access.checkAdminOption(matching_ids); + return matching_ids; + } } BlockIO InterpreterGrantQuery::execute() { auto & query = query_ptr->as(); - query.replaceCurrentUserTagWithName(context.getUserName()); - if (!query.cluster.empty()) - return executeDDLQueryOnCluster(query_ptr, context, query.access_rights_elements, true); + query.replaceCurrentUserTag(context.getUserName()); + query.access_rights_elements.eraseNonGrantable(); + + if (!query.access_rights_elements.sameOptions()) + throw Exception("Elements of an ASTGrantQuery are expected to have the same options", ErrorCodes::LOGICAL_ERROR); + if (!query.access_rights_elements.empty() && query.access_rights_elements[0].is_partial_revoke && !query.is_revoke) + throw Exception("A partial revoke should be revoked, not granted", ErrorCodes::LOGICAL_ERROR); - auto access = context.getAccess(); auto & access_control = context.getAccessControlManager(); - query.replaceEmptyDatabaseWithCurrent(context.getCurrentDatabase()); - - RolesOrUsersSet roles_set; + std::optional roles_set; if (query.roles) roles_set = RolesOrUsersSet{*query.roles, access_control}; - std::vector to_roles = RolesOrUsersSet{*query.to_roles, access_control, context.getUserID()}.getMatchingIDs(access_control); + std::vector grantees = RolesOrUsersSet{*query.grantees, access_control, context.getUserID()}.getMatchingIDs(access_control); + + /// Check if the current user has corresponding roles granted with admin option. + std::vector roles; + if (roles_set) + roles = getRoleIDsAndCheckAdminOption(access_control, *context.getAccess(), query, *roles_set, grantees); + + if (!query.cluster.empty()) + { + /// To execute the command GRANT the current user needs to have the access granted with GRANT OPTION. + auto required_access = query.access_rights_elements; + std::for_each(required_access.begin(), required_access.end(), [&](AccessRightsElement & element) { element.grant_option = true; }); + return executeDDLQueryOnCluster(query_ptr, context, std::move(required_access)); + } + + query.replaceEmptyDatabase(context.getCurrentDatabase()); /// Check if the current user has corresponding access rights with grant option. if (!query.access_rights_elements.empty()) - { - query.access_rights_elements.removeNonGrantableFlags(); + checkGrantOption(access_control, *context.getAccess(), query, grantees); - /// Special case for REVOKE: it's possible that the current user doesn't have the grant option for all - /// the specified access rights and that's ok because the roles or users which the access rights - /// will be revoked from don't have the specified access rights either. - /// - /// For example, to execute - /// GRANT ALL ON mydb.* TO role1 - /// REVOKE ALL ON *.* FROM role1 - /// the current user needs to have access rights only for the 'mydb' database. - if ((query.kind == Kind::REVOKE) && !access->hasGrantOption(query.access_rights_elements)) - { - AccessRights max_access; - for (const auto & id : to_roles) - { - auto entity = access_control.tryRead(id); - if (auto role = typeid_cast(entity)) - max_access.makeUnion(role->access); - else if (auto user = typeid_cast(entity)) - max_access.makeUnion(user->access); - } - AccessRights access_to_revoke; - if (query.grant_option) - access_to_revoke.grantWithGrantOption(query.access_rights_elements); - else - access_to_revoke.grant(query.access_rights_elements); - access_to_revoke.makeIntersection(max_access); - AccessRightsElements filtered_access_to_revoke; - for (auto & element : access_to_revoke.getElements()) - { - if ((element.kind == Kind::GRANT) && (element.grant_option || !query.grant_option)) - filtered_access_to_revoke.emplace_back(std::move(element)); - } - query.access_rights_elements = std::move(filtered_access_to_revoke); - } - - access->checkGrantOption(query.access_rights_elements); - } - - /// Check if the current user has corresponding roles granted with admin option. - std::vector roles_to_grant_or_revoke; - if (!roles_set.empty()) - { - bool all = roles_set.all; - if (!all) - roles_to_grant_or_revoke = roles_set.getMatchingIDs(); - - /// Special case for REVOKE: it's possible that the current user doesn't have the admin option for all - /// the specified roles and that's ok because the roles or users which the roles will be revoked from - /// don't have the specified roles granted either. - /// - /// For example, to execute - /// GRANT role2 TO role1 - /// REVOKE ALL FROM role1 - /// the current user needs to have only 'role2' to be granted with admin option (not all the roles). - if ((query.kind == Kind::REVOKE) && (roles_set.all || !access->hasAdminOption(roles_to_grant_or_revoke))) - { - auto & roles_to_revoke = roles_to_grant_or_revoke; - boost::container::flat_set max_roles; - for (const auto & id : to_roles) - { - auto entity = access_control.tryRead(id); - auto add_to_max_roles = [&](const GrantedRoles & granted_roles) - { - if (query.admin_option) - max_roles.insert(granted_roles.roles_with_admin_option.begin(), granted_roles.roles_with_admin_option.end()); - else - max_roles.insert(granted_roles.roles.begin(), granted_roles.roles.end()); - }; - if (auto role = typeid_cast(entity)) - add_to_max_roles(role->granted_roles); - else if (auto user = typeid_cast(entity)) - add_to_max_roles(user->granted_roles); - } - if (roles_set.all) - boost::range::set_difference(max_roles, roles_set.except_ids, std::back_inserter(roles_to_revoke)); - else - boost::range::remove_erase_if(roles_to_revoke, [&](const UUID & id) { return !max_roles.count(id); }); - } - - access->checkAdminOption(roles_to_grant_or_revoke); - } - - /// Update roles and users listed in `to_roles`. + /// Update roles and users listed in `grantees`. auto update_func = [&](const AccessEntityPtr & entity) -> AccessEntityPtr { auto clone = entity->clone(); - updateFromQueryImpl(*clone, query, roles_to_grant_or_revoke); + updateFromQueryImpl(*clone, query, roles); return clone; }; - access_control.update(to_roles, update_func); + access_control.update(grantees, update_func); return {}; } @@ -213,10 +249,10 @@ void InterpreterGrantQuery::updateRoleFromQuery(Role & role, const ASTGrantQuery void InterpreterGrantQuery::extendQueryLogElemImpl(QueryLogElement & elem, const ASTPtr & /*ast*/, const Context &) const { auto & query = query_ptr->as(); - if (query.kind == Kind::GRANT) - elem.query_kind = "Grant"; - else if (query.kind == Kind::REVOKE) + if (query.is_revoke) elem.query_kind = "Revoke"; + else + elem.query_kind = "Grant"; } } diff --git a/src/Interpreters/InterpreterSetRoleQuery.cpp b/src/Interpreters/InterpreterSetRoleQuery.cpp index f955c881b2e..8d314606329 100644 --- a/src/Interpreters/InterpreterSetRoleQuery.cpp +++ b/src/Interpreters/InterpreterSetRoleQuery.cpp @@ -39,20 +39,18 @@ void InterpreterSetRoleQuery::setRole(const ASTSetRoleQuery & query) else { RolesOrUsersSet roles_from_query{*query.roles, access_control}; - boost::container::flat_set new_current_roles; + std::vector new_current_roles; if (roles_from_query.all) { - for (const auto & id : user->granted_roles.roles) - if (roles_from_query.match(id)) - new_current_roles.emplace(id); + new_current_roles = user->granted_roles.findGranted(roles_from_query); } else { for (const auto & id : roles_from_query.getMatchingIDs()) { - if (!user->granted_roles.roles.count(id)) + if (!user->granted_roles.isGranted(id)) throw Exception("Role should be granted to set current", ErrorCodes::SET_NON_GRANTED_ROLE); - new_current_roles.emplace(id); + new_current_roles.emplace_back(id); } } session_context.setCurrentRoles(new_current_roles); @@ -85,7 +83,7 @@ void InterpreterSetRoleQuery::updateUserSetDefaultRoles(User & user, const Roles { for (const auto & id : roles_from_query.getMatchingIDs()) { - if (!user.granted_roles.roles.count(id)) + if (!user.granted_roles.isGranted(id)) throw Exception("Role should be granted to set default", ErrorCodes::SET_NON_GRANTED_ROLE); } } diff --git a/src/Interpreters/InterpreterShowAccessEntitiesQuery.cpp b/src/Interpreters/InterpreterShowAccessEntitiesQuery.cpp index 009b9c580d3..31c1ef874df 100644 --- a/src/Interpreters/InterpreterShowAccessEntitiesQuery.cpp +++ b/src/Interpreters/InterpreterShowAccessEntitiesQuery.cpp @@ -32,7 +32,7 @@ BlockIO InterpreterShowAccessEntitiesQuery::execute() String InterpreterShowAccessEntitiesQuery::getRewrittenQuery() const { auto & query = query_ptr->as(); - query.replaceEmptyDatabaseWithCurrent(context.getCurrentDatabase()); + query.replaceEmptyDatabase(context.getCurrentDatabase()); String origin; String expr = "*"; String filter, order; diff --git a/src/Interpreters/InterpreterShowCreateAccessEntityQuery.cpp b/src/Interpreters/InterpreterShowCreateAccessEntityQuery.cpp index 3135b0cfdf2..4e391035d5d 100644 --- a/src/Interpreters/InterpreterShowCreateAccessEntityQuery.cpp +++ b/src/Interpreters/InterpreterShowCreateAccessEntityQuery.cpp @@ -263,7 +263,7 @@ std::vector InterpreterShowCreateAccessEntityQuery::getEntities auto & show_query = query_ptr->as(); const auto & access_control = context.getAccessControlManager(); context.checkAccess(getRequiredAccess()); - show_query.replaceEmptyDatabaseWithCurrent(context.getCurrentDatabase()); + show_query.replaceEmptyDatabase(context.getCurrentDatabase()); std::vector entities; if (show_query.all) diff --git a/src/Interpreters/InterpreterShowGrantsQuery.cpp b/src/Interpreters/InterpreterShowGrantsQuery.cpp index a2ddc5eec27..bda5ef0f8b1 100644 --- a/src/Interpreters/InterpreterShowGrantsQuery.cpp +++ b/src/Interpreters/InterpreterShowGrantsQuery.cpp @@ -32,56 +32,50 @@ namespace { ASTs res; - std::shared_ptr to_roles = std::make_shared(); - to_roles->names.push_back(grantee.getName()); + std::shared_ptr grantees = std::make_shared(); + grantees->names.push_back(grantee.getName()); std::shared_ptr current_query = nullptr; - auto elements = grantee.access.getElements(); - for (const auto & element : elements) + for (const auto & element : grantee.access.getElements()) { + if (element.empty()) + continue; + if (current_query) { const auto & prev_element = current_query->access_rights_elements.back(); - bool continue_using_current_query = (element.database == prev_element.database) - && (element.any_database == prev_element.any_database) && (element.table == prev_element.table) - && (element.any_table == prev_element.any_table) && (element.grant_option == current_query->grant_option) - && (element.kind == current_query->kind); - if (!continue_using_current_query) + bool continue_with_current_query = element.sameDatabaseAndTable(prev_element) && element.sameOptions(prev_element); + if (!continue_with_current_query) current_query = nullptr; } if (!current_query) { current_query = std::make_shared(); - current_query->kind = element.kind; - current_query->attach = attach_mode; - current_query->grant_option = element.grant_option; - current_query->to_roles = to_roles; + current_query->grantees = grantees; + current_query->attach_mode = attach_mode; + if (element.is_partial_revoke) + current_query->is_revoke = true; res.push_back(current_query); } current_query->access_rights_elements.emplace_back(std::move(element)); } - auto grants_roles = grantee.granted_roles.getGrants(); - - for (bool admin_option : {false, true}) + for (const auto & element : grantee.granted_roles.getElements()) { - const auto & roles = admin_option ? grants_roles.grants_with_admin_option : grants_roles.grants; - if (roles.empty()) + if (element.empty()) continue; auto grant_query = std::make_shared(); - using Kind = ASTGrantQuery::Kind; - grant_query->kind = Kind::GRANT; - grant_query->attach = attach_mode; - grant_query->admin_option = admin_option; - grant_query->to_roles = to_roles; + grant_query->grantees = grantees; + grant_query->admin_option = element.admin_option; + grant_query->attach_mode = attach_mode; if (attach_mode) - grant_query->roles = RolesOrUsersSet{roles}.toAST(); + grant_query->roles = RolesOrUsersSet{element.ids}.toAST(); else - grant_query->roles = RolesOrUsersSet{roles}.toASTWithNames(*manager); + grant_query->roles = RolesOrUsersSet{element.ids}.toASTWithNames(*manager); res.push_back(std::move(grant_query)); } diff --git a/src/Interpreters/executeDDLQueryOnCluster.cpp b/src/Interpreters/executeDDLQueryOnCluster.cpp index 1937fbaf905..c498eb21379 100644 --- a/src/Interpreters/executeDDLQueryOnCluster.cpp +++ b/src/Interpreters/executeDDLQueryOnCluster.cpp @@ -50,12 +50,12 @@ BlockIO executeDDLQueryOnCluster(const ASTPtr & query_ptr_, const Context & cont return executeDDLQueryOnCluster(query_ptr_, context, {}); } -BlockIO executeDDLQueryOnCluster(const ASTPtr & query_ptr, const Context & context, const AccessRightsElements & query_requires_access, bool query_requires_grant_option) +BlockIO executeDDLQueryOnCluster(const ASTPtr & query_ptr, const Context & context, const AccessRightsElements & query_requires_access) { - return executeDDLQueryOnCluster(query_ptr, context, AccessRightsElements{query_requires_access}, query_requires_grant_option); + return executeDDLQueryOnCluster(query_ptr, context, AccessRightsElements{query_requires_access}); } -BlockIO executeDDLQueryOnCluster(const ASTPtr & query_ptr_, const Context & context, AccessRightsElements && query_requires_access, bool query_requires_grant_option) +BlockIO executeDDLQueryOnCluster(const ASTPtr & query_ptr_, const Context & context, AccessRightsElements && query_requires_access) { /// Remove FORMAT and INTO OUTFILE if exists ASTPtr query_ptr = query_ptr_->clone(); @@ -154,10 +154,7 @@ BlockIO executeDDLQueryOnCluster(const ASTPtr & query_ptr_, const Context & cont visitor.visitDDL(query_ptr); /// Check access rights, assume that all servers have the same users config - if (query_requires_grant_option) - context.getAccess()->checkGrantOption(query_requires_access); - else - context.checkAccess(query_requires_access); + context.checkAccess(query_requires_access); DDLLogEntry entry; entry.hosts = std::move(hosts); diff --git a/src/Interpreters/executeDDLQueryOnCluster.h b/src/Interpreters/executeDDLQueryOnCluster.h index 2b272d3b0da..1bcbff36178 100644 --- a/src/Interpreters/executeDDLQueryOnCluster.h +++ b/src/Interpreters/executeDDLQueryOnCluster.h @@ -21,8 +21,8 @@ bool isSupportedAlterType(int type); /// Pushes distributed DDL query to the queue. /// Returns DDLQueryStatusInputStream, which reads results of query execution on each host in the cluster. BlockIO executeDDLQueryOnCluster(const ASTPtr & query_ptr, const Context & context); -BlockIO executeDDLQueryOnCluster(const ASTPtr & query_ptr, const Context & context, const AccessRightsElements & query_requires_access, bool query_requires_grant_option = false); -BlockIO executeDDLQueryOnCluster(const ASTPtr & query_ptr, const Context & context, AccessRightsElements && query_requires_access, bool query_requires_grant_option = false); +BlockIO executeDDLQueryOnCluster(const ASTPtr & query_ptr, const Context & context, const AccessRightsElements & query_requires_access); +BlockIO executeDDLQueryOnCluster(const ASTPtr & query_ptr, const Context & context, AccessRightsElements && query_requires_access); class DDLQueryStatusInputStream final : public IBlockInputStream diff --git a/src/Parsers/ASTCreateQuotaQuery.cpp b/src/Parsers/ASTCreateQuotaQuery.cpp index 7e570b889e3..18f72d61319 100644 --- a/src/Parsers/ASTCreateQuotaQuery.cpp +++ b/src/Parsers/ASTCreateQuotaQuery.cpp @@ -185,10 +185,10 @@ void ASTCreateQuotaQuery::formatImpl(const FormatSettings & settings, FormatStat } -void ASTCreateQuotaQuery::replaceCurrentUserTagWithName(const String & current_user_name) const +void ASTCreateQuotaQuery::replaceCurrentUserTag(const String & current_user_name) const { if (roles) - roles->replaceCurrentUserTagWithName(current_user_name); + roles->replaceCurrentUserTag(current_user_name); } } diff --git a/src/Parsers/ASTCreateQuotaQuery.h b/src/Parsers/ASTCreateQuotaQuery.h index a1269afafa6..00984d4b4c9 100644 --- a/src/Parsers/ASTCreateQuotaQuery.h +++ b/src/Parsers/ASTCreateQuotaQuery.h @@ -56,7 +56,7 @@ public: String getID(char) const override; ASTPtr clone() const override; void formatImpl(const FormatSettings & settings, FormatState &, FormatStateStacked) const override; - void replaceCurrentUserTagWithName(const String & current_user_name) const; + void replaceCurrentUserTag(const String & current_user_name) const; ASTPtr getRewrittenASTWithoutOnCluster(const std::string &) const override { return removeOnCluster(clone()); } }; } diff --git a/src/Parsers/ASTCreateRowPolicyQuery.cpp b/src/Parsers/ASTCreateRowPolicyQuery.cpp index 30b001feeca..3b4c2484acf 100644 --- a/src/Parsers/ASTCreateRowPolicyQuery.cpp +++ b/src/Parsers/ASTCreateRowPolicyQuery.cpp @@ -169,15 +169,15 @@ void ASTCreateRowPolicyQuery::formatImpl(const FormatSettings & settings, Format } -void ASTCreateRowPolicyQuery::replaceCurrentUserTagWithName(const String & current_user_name) const +void ASTCreateRowPolicyQuery::replaceCurrentUserTag(const String & current_user_name) const { if (roles) - roles->replaceCurrentUserTagWithName(current_user_name); + roles->replaceCurrentUserTag(current_user_name); } -void ASTCreateRowPolicyQuery::replaceEmptyDatabaseWithCurrent(const String & current_database) const +void ASTCreateRowPolicyQuery::replaceEmptyDatabase(const String & current_database) const { if (names) - names->replaceEmptyDatabaseWithCurrent(current_database); + names->replaceEmptyDatabase(current_database); } } diff --git a/src/Parsers/ASTCreateRowPolicyQuery.h b/src/Parsers/ASTCreateRowPolicyQuery.h index 9d0e2fcce7b..46a7578726e 100644 --- a/src/Parsers/ASTCreateRowPolicyQuery.h +++ b/src/Parsers/ASTCreateRowPolicyQuery.h @@ -49,7 +49,7 @@ public: void formatImpl(const FormatSettings & settings, FormatState &, FormatStateStacked) const override; ASTPtr getRewrittenASTWithoutOnCluster(const std::string &) const override { return removeOnCluster(clone()); } - void replaceCurrentUserTagWithName(const String & current_user_name) const; - void replaceEmptyDatabaseWithCurrent(const String & current_database) const; + void replaceCurrentUserTag(const String & current_user_name) const; + void replaceEmptyDatabase(const String & current_database) const; }; } diff --git a/src/Parsers/ASTCreateSettingsProfileQuery.cpp b/src/Parsers/ASTCreateSettingsProfileQuery.cpp index 84f8309462e..e99c40ca681 100644 --- a/src/Parsers/ASTCreateSettingsProfileQuery.cpp +++ b/src/Parsers/ASTCreateSettingsProfileQuery.cpp @@ -86,9 +86,9 @@ void ASTCreateSettingsProfileQuery::formatImpl(const FormatSettings & format, Fo } -void ASTCreateSettingsProfileQuery::replaceCurrentUserTagWithName(const String & current_user_name) const +void ASTCreateSettingsProfileQuery::replaceCurrentUserTag(const String & current_user_name) const { if (to_roles) - to_roles->replaceCurrentUserTagWithName(current_user_name); + to_roles->replaceCurrentUserTag(current_user_name); } } diff --git a/src/Parsers/ASTCreateSettingsProfileQuery.h b/src/Parsers/ASTCreateSettingsProfileQuery.h index 119019093b2..df0a11456bc 100644 --- a/src/Parsers/ASTCreateSettingsProfileQuery.h +++ b/src/Parsers/ASTCreateSettingsProfileQuery.h @@ -39,7 +39,7 @@ public: String getID(char) const override; ASTPtr clone() const override; void formatImpl(const FormatSettings & format, FormatState &, FormatStateStacked) const override; - void replaceCurrentUserTagWithName(const String & current_user_name) const; + void replaceCurrentUserTag(const String & current_user_name) const; ASTPtr getRewrittenASTWithoutOnCluster(const std::string &) const override { return removeOnCluster(clone()); } }; } diff --git a/src/Parsers/ASTCreateUserQuery.h b/src/Parsers/ASTCreateUserQuery.h index 7acfd87909a..22992b2c408 100644 --- a/src/Parsers/ASTCreateUserQuery.h +++ b/src/Parsers/ASTCreateUserQuery.h @@ -19,11 +19,11 @@ class ASTSettingsProfileElements; * [SETTINGS variable [= value] [MIN [=] min_value] [MAX [=] max_value] [READONLY|WRITABLE] | PROFILE 'profile_name'] [,...] * * ALTER USER [IF EXISTS] name - * [RENAME TO new_name] - * [NOT IDENTIFIED | IDENTIFIED {[WITH {no_password|plaintext_password|sha256_password|sha256_hash|double_sha1_password|double_sha1_hash}] BY {'password'|'hash'}}|{WITH ldap SERVER 'server_name'}|{WITH kerberos [REALM 'realm']}] - * [[ADD|DROP] HOST {LOCAL | NAME 'name' | REGEXP 'name_regexp' | IP 'address' | LIKE 'pattern'} [,...] | ANY | NONE] - * [DEFAULT ROLE role [,...] | ALL | ALL EXCEPT role [,...] ] - * [SETTINGS variable [= value] [MIN [=] min_value] [MAX [=] max_value] [READONLY|WRITABLE] | PROFILE 'profile_name'] [,...] + * [RENAME TO new_name] + * [NOT IDENTIFIED | IDENTIFIED {[WITH {no_password|plaintext_password|sha256_password|sha256_hash|double_sha1_password|double_sha1_hash}] BY {'password'|'hash'}}|{WITH ldap SERVER 'server_name'}|{WITH kerberos [REALM 'realm']}] + * [[ADD|DROP] HOST {LOCAL | NAME 'name' | REGEXP 'name_regexp' | IP 'address' | LIKE 'pattern'} [,...] | ANY | NONE] + * [DEFAULT ROLE role [,...] | ALL | ALL EXCEPT role [,...] ] + * [SETTINGS variable [= value] [MIN [=] min_value] [MAX [=] max_value] [READONLY|WRITABLE] | PROFILE 'profile_name'] [,...] */ class ASTCreateUserQuery : public IAST, public ASTQueryWithOnCluster { @@ -46,7 +46,6 @@ public: std::optional remove_hosts; std::shared_ptr default_roles; - std::shared_ptr settings; String getID(char) const override; diff --git a/src/Parsers/ASTDropAccessEntityQuery.cpp b/src/Parsers/ASTDropAccessEntityQuery.cpp index 1df176c24ec..6c19c9f8af3 100644 --- a/src/Parsers/ASTDropAccessEntityQuery.cpp +++ b/src/Parsers/ASTDropAccessEntityQuery.cpp @@ -54,9 +54,9 @@ void ASTDropAccessEntityQuery::formatImpl(const FormatSettings & settings, Forma } -void ASTDropAccessEntityQuery::replaceEmptyDatabaseWithCurrent(const String & current_database) const +void ASTDropAccessEntityQuery::replaceEmptyDatabase(const String & current_database) const { if (row_policy_names) - row_policy_names->replaceEmptyDatabaseWithCurrent(current_database); + row_policy_names->replaceEmptyDatabase(current_database); } } diff --git a/src/Parsers/ASTDropAccessEntityQuery.h b/src/Parsers/ASTDropAccessEntityQuery.h index 76a5f450566..df78acef6f4 100644 --- a/src/Parsers/ASTDropAccessEntityQuery.h +++ b/src/Parsers/ASTDropAccessEntityQuery.h @@ -30,6 +30,6 @@ public: void formatImpl(const FormatSettings & settings, FormatState &, FormatStateStacked) const override; ASTPtr getRewrittenASTWithoutOnCluster(const std::string &) const override { return removeOnCluster(clone()); } - void replaceEmptyDatabaseWithCurrent(const String & current_database) const; + void replaceEmptyDatabase(const String & current_database) const; }; } diff --git a/src/Parsers/ASTGrantQuery.cpp b/src/Parsers/ASTGrantQuery.cpp index 2610836c759..aca53868226 100644 --- a/src/Parsers/ASTGrantQuery.cpp +++ b/src/Parsers/ASTGrantQuery.cpp @@ -27,7 +27,26 @@ namespace } - void formatAccessRightsElements(const AccessRightsElements & elements, const IAST::FormatSettings & settings) + void formatONClause(const String & database, bool any_database, const String & table, bool any_table, const IAST::FormatSettings & settings) + { + settings.ostr << (settings.hilite ? IAST::hilite_keyword : "") << "ON " << (settings.hilite ? IAST::hilite_none : ""); + if (any_database) + { + settings.ostr << "*.*"; + } + else + { + if (!database.empty()) + settings.ostr << backQuoteIfNeed(database) << "."; + if (any_table) + settings.ostr << "*"; + else + settings.ostr << backQuoteIfNeed(table); + } + } + + + void formatElementsWithoutOptions(const AccessRightsElements & elements, const IAST::FormatSettings & settings) { bool no_output = true; for (size_t i = 0; i != elements.size(); ++i) @@ -58,31 +77,14 @@ namespace if (!next_element_on_same_db_and_table) { - settings.ostr << (settings.hilite ? IAST::hilite_keyword : "") << " ON " << (settings.hilite ? IAST::hilite_none : ""); - if (element.any_database) - settings.ostr << "*."; - else if (!element.database.empty()) - settings.ostr << backQuoteIfNeed(element.database) + "."; - - if (element.any_table) - settings.ostr << "*"; - else - settings.ostr << backQuoteIfNeed(element.table); + settings.ostr << " "; + formatONClause(element.database, element.any_database, element.table, element.any_table, settings); } } if (no_output) settings.ostr << (settings.hilite ? IAST::hilite_keyword : "") << "USAGE ON " << (settings.hilite ? IAST::hilite_none : "") << "*.*"; } - - - void formatToRoles(const ASTRolesOrUsersSet & to_roles, ASTGrantQuery::Kind kind, const IAST::FormatSettings & settings) - { - using Kind = ASTGrantQuery::Kind; - settings.ostr << (settings.hilite ? IAST::hilite_keyword : "") << ((kind == Kind::GRANT) ? " TO " : " FROM ") - << (settings.hilite ? IAST::hilite_none : ""); - to_roles.format(settings); - } } @@ -100,12 +102,18 @@ ASTPtr ASTGrantQuery::clone() const void ASTGrantQuery::formatImpl(const FormatSettings & settings, FormatState &, FormatStateStacked) const { - settings.ostr << (settings.hilite ? IAST::hilite_keyword : "") << (attach ? "ATTACH " : "") << ((kind == Kind::GRANT) ? "GRANT" : "REVOKE") + settings.ostr << (settings.hilite ? IAST::hilite_keyword : "") << (attach_mode ? "ATTACH " : "") << (is_revoke ? "REVOKE" : "GRANT") << (settings.hilite ? IAST::hilite_none : ""); + if (!access_rights_elements.sameOptions()) + throw Exception("Elements of an ASTGrantQuery are expected to have the same options", ErrorCodes::LOGICAL_ERROR); + if (!access_rights_elements.empty() && access_rights_elements[0].is_partial_revoke && !is_revoke) + throw Exception("A partial revoke should be revoked, not granted", ErrorCodes::LOGICAL_ERROR); + bool grant_option = !access_rights_elements.empty() && access_rights_elements[0].grant_option; + formatOnCluster(settings); - if (kind == Kind::REVOKE) + if (is_revoke) { if (grant_option) settings.ostr << (settings.hilite ? hilite_keyword : "") << " GRANT OPTION FOR" << (settings.hilite ? hilite_none : ""); @@ -113,18 +121,21 @@ void ASTGrantQuery::formatImpl(const FormatSettings & settings, FormatState &, F settings.ostr << (settings.hilite ? hilite_keyword : "") << " ADMIN OPTION FOR" << (settings.hilite ? hilite_none : ""); } - if (roles && !access_rights_elements.empty()) - throw Exception("Either roles or access rights elements should be set", ErrorCodes::LOGICAL_ERROR); - settings.ostr << " "; if (roles) + { roles->format(settings); + if (!access_rights_elements.empty()) + throw Exception("ASTGrantQuery can contain either roles or access rights elements to grant or revoke, not both of them", ErrorCodes::LOGICAL_ERROR); + } else - formatAccessRightsElements(access_rights_elements, settings); + formatElementsWithoutOptions(access_rights_elements, settings); - formatToRoles(*to_roles, kind, settings); + settings.ostr << (settings.hilite ? IAST::hilite_keyword : "") << (is_revoke ? " FROM " : " TO ") + << (settings.hilite ? IAST::hilite_none : ""); + grantees->format(settings); - if (kind == Kind::GRANT) + if (!is_revoke) { if (grant_option) settings.ostr << (settings.hilite ? hilite_keyword : "") << " WITH GRANT OPTION" << (settings.hilite ? hilite_none : ""); @@ -134,16 +145,16 @@ void ASTGrantQuery::formatImpl(const FormatSettings & settings, FormatState &, F } -void ASTGrantQuery::replaceEmptyDatabaseWithCurrent(const String & current_database) +void ASTGrantQuery::replaceEmptyDatabase(const String & current_database) { access_rights_elements.replaceEmptyDatabase(current_database); } -void ASTGrantQuery::replaceCurrentUserTagWithName(const String & current_user_name) const +void ASTGrantQuery::replaceCurrentUserTag(const String & current_user_name) const { - if (to_roles) - to_roles->replaceCurrentUserTagWithName(current_user_name); + if (grantees) + grantees->replaceCurrentUserTag(current_user_name); } } diff --git a/src/Parsers/ASTGrantQuery.h b/src/Parsers/ASTGrantQuery.h index c36e42689a5..833c4db8ec6 100644 --- a/src/Parsers/ASTGrantQuery.h +++ b/src/Parsers/ASTGrantQuery.h @@ -19,20 +19,18 @@ class ASTRolesOrUsersSet; class ASTGrantQuery : public IAST, public ASTQueryWithOnCluster { public: - using Kind = AccessRightsElementWithOptions::Kind; - Kind kind = Kind::GRANT; - bool attach = false; + bool attach_mode = false; + bool is_revoke = false; AccessRightsElements access_rights_elements; std::shared_ptr roles; - std::shared_ptr to_roles; - bool grant_option = false; bool admin_option = false; + std::shared_ptr grantees; String getID(char) const override; ASTPtr clone() const override; void formatImpl(const FormatSettings & settings, FormatState &, FormatStateStacked) const override; - void replaceEmptyDatabaseWithCurrent(const String & current_database); - void replaceCurrentUserTagWithName(const String & current_user_name) const; + void replaceEmptyDatabase(const String & current_database); + void replaceCurrentUserTag(const String & current_user_name) const; ASTPtr getRewrittenASTWithoutOnCluster(const std::string &) const override { return removeOnCluster(clone()); } }; } diff --git a/src/Parsers/ASTRolesOrUsersSet.cpp b/src/Parsers/ASTRolesOrUsersSet.cpp index 1e7cd79f527..9eb83cf895e 100644 --- a/src/Parsers/ASTRolesOrUsersSet.cpp +++ b/src/Parsers/ASTRolesOrUsersSet.cpp @@ -7,7 +7,7 @@ namespace DB { namespace { - void formatRoleNameOrID(const String & str, bool is_id, const IAST::FormatSettings & settings) + void formatNameOrID(const String & str, bool is_id, const IAST::FormatSettings & settings) { if (is_id) { @@ -30,6 +30,7 @@ void ASTRolesOrUsersSet::formatImpl(const FormatSettings & settings, FormatState } bool need_comma = false; + if (all) { if (std::exchange(need_comma, true)) @@ -38,11 +39,11 @@ void ASTRolesOrUsersSet::formatImpl(const FormatSettings & settings, FormatState } else { - for (const auto & role : names) + for (const auto & name : names) { if (std::exchange(need_comma, true)) settings.ostr << ", "; - formatRoleNameOrID(role, id_mode, settings); + formatNameOrID(name, id_mode, settings); } if (current_user) @@ -58,11 +59,11 @@ void ASTRolesOrUsersSet::formatImpl(const FormatSettings & settings, FormatState settings.ostr << (settings.hilite ? IAST::hilite_keyword : "") << " EXCEPT " << (settings.hilite ? IAST::hilite_none : ""); need_comma = false; - for (const auto & except_role : except_names) + for (const auto & name : except_names) { if (std::exchange(need_comma, true)) settings.ostr << ", "; - formatRoleNameOrID(except_role, id_mode, settings); + formatNameOrID(name, id_mode, settings); } if (except_current_user) @@ -75,7 +76,7 @@ void ASTRolesOrUsersSet::formatImpl(const FormatSettings & settings, FormatState } -void ASTRolesOrUsersSet::replaceCurrentUserTagWithName(const String & current_user_name) +void ASTRolesOrUsersSet::replaceCurrentUserTag(const String & current_user_name) { if (current_user) { diff --git a/src/Parsers/ASTRolesOrUsersSet.h b/src/Parsers/ASTRolesOrUsersSet.h index f18aa0bdd73..0f78f67d35b 100644 --- a/src/Parsers/ASTRolesOrUsersSet.h +++ b/src/Parsers/ASTRolesOrUsersSet.h @@ -9,22 +9,23 @@ namespace DB using Strings = std::vector; /// Represents a set of users/roles like -/// {user_name | role_name | CURRENT_USER} [,...] | NONE | ALL | ALL EXCEPT {user_name | role_name | CURRENT_USER} [,...] +/// {user_name | role_name | CURRENT_USER | ALL | NONE} [,...] +/// [EXCEPT {user_name | role_name | CURRENT_USER | ALL | NONE} [,...]] class ASTRolesOrUsersSet : public IAST { public: + bool all = false; Strings names; bool current_user = false; - bool all = false; Strings except_names; bool except_current_user = false; - bool id_mode = false; /// true if `names` and `except_names` keep UUIDs, not names. - bool allow_role_names = true; /// true if this set can contain names of roles. - bool allow_user_names = true; /// true if this set can contain names of users. + bool allow_users = true; /// whether this set can contain names of users + bool allow_roles = true; /// whether this set can contain names of roles + bool id_mode = false; /// whether this set keep UUIDs instead of names bool empty() const { return names.empty() && !current_user && !all; } - void replaceCurrentUserTagWithName(const String & current_user_name); + void replaceCurrentUserTag(const String & current_user_name); String getID(char) const override { return "RolesOrUsersSet"; } ASTPtr clone() const override { return std::make_shared(*this); } diff --git a/src/Parsers/ASTRowPolicyName.cpp b/src/Parsers/ASTRowPolicyName.cpp index 3d1ac5621db..0b69c1a46b3 100644 --- a/src/Parsers/ASTRowPolicyName.cpp +++ b/src/Parsers/ASTRowPolicyName.cpp @@ -23,7 +23,7 @@ void ASTRowPolicyName::formatImpl(const FormatSettings & settings, FormatState & } -void ASTRowPolicyName::replaceEmptyDatabaseWithCurrent(const String & current_database) +void ASTRowPolicyName::replaceEmptyDatabase(const String & current_database) { if (name_parts.database.empty()) name_parts.database = current_database; @@ -125,7 +125,7 @@ Strings ASTRowPolicyNames::toStrings() const } -void ASTRowPolicyNames::replaceEmptyDatabaseWithCurrent(const String & current_database) +void ASTRowPolicyNames::replaceEmptyDatabase(const String & current_database) { for (auto & np : name_parts) if (np.database.empty()) diff --git a/src/Parsers/ASTRowPolicyName.h b/src/Parsers/ASTRowPolicyName.h index ac2f84f5d8b..b195596225b 100644 --- a/src/Parsers/ASTRowPolicyName.h +++ b/src/Parsers/ASTRowPolicyName.h @@ -22,7 +22,7 @@ public: void formatImpl(const FormatSettings & settings, FormatState &, FormatStateStacked) const override; ASTPtr getRewrittenASTWithoutOnCluster(const std::string &) const override { return removeOnCluster(clone()); } - void replaceEmptyDatabaseWithCurrent(const String & current_database); + void replaceEmptyDatabase(const String & current_database); }; @@ -44,6 +44,6 @@ public: void formatImpl(const FormatSettings & settings, FormatState &, FormatStateStacked) const override; ASTPtr getRewrittenASTWithoutOnCluster(const std::string &) const override { return removeOnCluster(clone()); } - void replaceEmptyDatabaseWithCurrent(const String & current_database); + void replaceEmptyDatabase(const String & current_database); }; } diff --git a/src/Parsers/ASTShowAccessEntitiesQuery.cpp b/src/Parsers/ASTShowAccessEntitiesQuery.cpp index bacde098640..6dd53fd5cde 100644 --- a/src/Parsers/ASTShowAccessEntitiesQuery.cpp +++ b/src/Parsers/ASTShowAccessEntitiesQuery.cpp @@ -43,7 +43,7 @@ void ASTShowAccessEntitiesQuery::formatQueryImpl(const FormatSettings & settings } -void ASTShowAccessEntitiesQuery::replaceEmptyDatabaseWithCurrent(const String & current_database) +void ASTShowAccessEntitiesQuery::replaceEmptyDatabase(const String & current_database) { if (database_and_table_name) { diff --git a/src/Parsers/ASTShowAccessEntitiesQuery.h b/src/Parsers/ASTShowAccessEntitiesQuery.h index 7ccd76bfe5e..2be1e0b92f0 100644 --- a/src/Parsers/ASTShowAccessEntitiesQuery.h +++ b/src/Parsers/ASTShowAccessEntitiesQuery.h @@ -31,7 +31,7 @@ public: String getID(char) const override; ASTPtr clone() const override { return std::make_shared(*this); } - void replaceEmptyDatabaseWithCurrent(const String & current_database); + void replaceEmptyDatabase(const String & current_database); protected: void formatQueryImpl(const FormatSettings & settings, FormatState &, FormatStateStacked) const override; diff --git a/src/Parsers/ASTShowCreateAccessEntityQuery.cpp b/src/Parsers/ASTShowCreateAccessEntityQuery.cpp index f870c98071c..5ff51a47002 100644 --- a/src/Parsers/ASTShowCreateAccessEntityQuery.cpp +++ b/src/Parsers/ASTShowCreateAccessEntityQuery.cpp @@ -72,10 +72,10 @@ void ASTShowCreateAccessEntityQuery::formatQueryImpl(const FormatSettings & sett } -void ASTShowCreateAccessEntityQuery::replaceEmptyDatabaseWithCurrent(const String & current_database) +void ASTShowCreateAccessEntityQuery::replaceEmptyDatabase(const String & current_database) { if (row_policy_names) - row_policy_names->replaceEmptyDatabaseWithCurrent(current_database); + row_policy_names->replaceEmptyDatabase(current_database); if (database_and_table_name) { diff --git a/src/Parsers/ASTShowCreateAccessEntityQuery.h b/src/Parsers/ASTShowCreateAccessEntityQuery.h index 10c4c0ca511..e20bb4f022e 100644 --- a/src/Parsers/ASTShowCreateAccessEntityQuery.h +++ b/src/Parsers/ASTShowCreateAccessEntityQuery.h @@ -40,7 +40,7 @@ public: String getID(char) const override; ASTPtr clone() const override; - void replaceEmptyDatabaseWithCurrent(const String & current_database); + void replaceEmptyDatabase(const String & current_database); protected: String getKeyword() const; diff --git a/src/Parsers/ParserCreateQuotaQuery.cpp b/src/Parsers/ParserCreateQuotaQuery.cpp index 68c53d2fc1d..a8779a68600 100644 --- a/src/Parsers/ParserCreateQuotaQuery.cpp +++ b/src/Parsers/ParserCreateQuotaQuery.cpp @@ -226,7 +226,7 @@ namespace { ASTPtr node; ParserRolesOrUsersSet roles_p; - roles_p.allowAll().allowRoleNames().allowUserNames().allowCurrentUser().useIDMode(id_mode); + roles_p.allowAll().allowRoles().allowUsers().allowCurrentUser().useIDMode(id_mode); if (!ParserKeyword{"TO"}.ignore(pos, expected) || !roles_p.parse(pos, node, expected)) return false; diff --git a/src/Parsers/ParserCreateRowPolicyQuery.cpp b/src/Parsers/ParserCreateRowPolicyQuery.cpp index fae5bd35b43..534f781a273 100644 --- a/src/Parsers/ParserCreateRowPolicyQuery.cpp +++ b/src/Parsers/ParserCreateRowPolicyQuery.cpp @@ -187,7 +187,7 @@ namespace return false; ParserRolesOrUsersSet roles_p; - roles_p.allowAll().allowRoleNames().allowUserNames().allowCurrentUser().useIDMode(id_mode); + roles_p.allowAll().allowRoles().allowUsers().allowCurrentUser().useIDMode(id_mode); if (!roles_p.parse(pos, ast, expected)) return false; diff --git a/src/Parsers/ParserCreateSettingsProfileQuery.cpp b/src/Parsers/ParserCreateSettingsProfileQuery.cpp index 797379509e4..2d1e6824b50 100644 --- a/src/Parsers/ParserCreateSettingsProfileQuery.cpp +++ b/src/Parsers/ParserCreateSettingsProfileQuery.cpp @@ -53,7 +53,7 @@ namespace return false; ParserRolesOrUsersSet roles_p; - roles_p.allowAll().allowRoleNames().allowUserNames().allowCurrentUser().useIDMode(id_mode); + roles_p.allowAll().allowRoles().allowUsers().allowCurrentUser().useIDMode(id_mode); if (!roles_p.parse(pos, ast, expected)) return false; diff --git a/src/Parsers/ParserCreateUserQuery.cpp b/src/Parsers/ParserCreateUserQuery.cpp index 16c539d3ebc..b856b03ab9b 100644 --- a/src/Parsers/ParserCreateUserQuery.cpp +++ b/src/Parsers/ParserCreateUserQuery.cpp @@ -246,12 +246,12 @@ namespace ASTPtr ast; ParserRolesOrUsersSet default_roles_p; - default_roles_p.allowAll().allowRoleNames().useIDMode(id_mode); + default_roles_p.allowAll().allowRoles().useIDMode(id_mode); if (!default_roles_p.parse(pos, ast, expected)) return false; default_roles = typeid_cast>(ast); - default_roles->allow_user_names = false; + default_roles->allow_users = false; return true; }); } diff --git a/src/Parsers/ParserCreateUserQuery.h b/src/Parsers/ParserCreateUserQuery.h index 5b83a261fa2..0eee522979f 100644 --- a/src/Parsers/ParserCreateUserQuery.h +++ b/src/Parsers/ParserCreateUserQuery.h @@ -9,12 +9,14 @@ namespace DB * CREATE USER [IF NOT EXISTS | OR REPLACE] name * [NOT IDENTIFIED | IDENTIFIED {[WITH {no_password|plaintext_password|sha256_password|sha256_hash|double_sha1_password|double_sha1_hash}] BY {'password'|'hash'}}|{WITH ldap SERVER 'server_name'}|{WITH kerberos [REALM 'realm']}] * [HOST {LOCAL | NAME 'name' | REGEXP 'name_regexp' | IP 'address' | LIKE 'pattern'} [,...] | ANY | NONE] + * [DEFAULT ROLE role [,...]] * [SETTINGS variable [= value] [MIN [=] min_value] [MAX [=] max_value] [READONLY|WRITABLE] | PROFILE 'profile_name'] [,...] * * ALTER USER [IF EXISTS] name * [RENAME TO new_name] * [NOT IDENTIFIED | IDENTIFIED {[WITH {no_password|plaintext_password|sha256_password|sha256_hash|double_sha1_password|double_sha1_hash}] BY {'password'|'hash'}}|{WITH ldap SERVER 'server_name'}|{WITH kerberos [REALM 'realm']}] * [[ADD|DROP] HOST {LOCAL | NAME 'name' | REGEXP 'name_regexp' | IP 'address' | LIKE 'pattern'} [,...] | ANY | NONE] + * [DEFAULT ROLE role [,...] | ALL | ALL EXCEPT role [,...] ] * [SETTINGS variable [= value] [MIN [=] min_value] [MAX [=] max_value] [READONLY|WRITABLE] | PROFILE 'profile_name'] [,...] */ class ParserCreateUserQuery : public IParserBase diff --git a/src/Parsers/ParserGrantQuery.cpp b/src/Parsers/ParserGrantQuery.cpp index 7dd721c9af2..d3aa62e73da 100644 --- a/src/Parsers/ParserGrantQuery.cpp +++ b/src/Parsers/ParserGrantQuery.cpp @@ -8,6 +8,7 @@ #include #include #include +#include namespace DB @@ -20,8 +21,6 @@ namespace ErrorCodes namespace { - using Kind = ASTGrantQuery::Kind; - bool parseAccessFlags(IParser::Pos & pos, Expected & expected, AccessFlags & access_flags) { static constexpr auto is_one_of_access_type_words = [](IParser::Pos & pos_) @@ -87,7 +86,7 @@ namespace }); } - bool parseAccessTypesWithColumns(IParser::Pos & pos, Expected & expected, + bool parseAccessFlagsWithColumns(IParser::Pos & pos, Expected & expected, std::vector> & access_and_columns) { std::vector> res; @@ -112,7 +111,7 @@ namespace } - bool parseAccessRightsElements(IParser::Pos & pos, Expected & expected, AccessRightsElements & elements) + bool parseElementsWithoutOptions(IParser::Pos & pos, Expected & expected, AccessRightsElements & elements) { return IParserBase::wrapParseImpl(pos, [&] { @@ -121,7 +120,7 @@ namespace auto parse_around_on = [&] { std::vector> access_and_columns; - if (!parseAccessTypesWithColumns(pos, expected, access_and_columns)) + if (!parseAccessFlagsWithColumns(pos, expected, access_and_columns)) return false; if (!ParserKeyword{"ON"}.ignore(pos, expected)) @@ -157,16 +156,16 @@ namespace } - void removeNonGrantableFlags(AccessRightsElements & elements) + void eraseNonGrantable(AccessRightsElements & elements) { - for (auto & element : elements) + boost::range::remove_erase_if(elements, [](AccessRightsElement & element) { if (element.empty()) - continue; + return true; auto old_flags = element.access_flags; - element.removeNonGrantableFlags(); + element.eraseNonGrantable(); if (!element.empty()) - continue; + return false; if (!element.any_column) throw Exception(old_flags.toString() + " cannot be granted on the column level", ErrorCodes::INVALID_GRANT); @@ -176,17 +175,17 @@ namespace throw Exception(old_flags.toString() + " cannot be granted on the database level", ErrorCodes::INVALID_GRANT); else throw Exception(old_flags.toString() + " cannot be granted", ErrorCodes::INVALID_GRANT); - } + }); } - bool parseRoles(IParser::Pos & pos, Expected & expected, Kind kind, bool id_mode, std::shared_ptr & roles) + bool parseRoles(IParser::Pos & pos, Expected & expected, bool is_revoke, bool id_mode, std::shared_ptr & roles) { return IParserBase::wrapParseImpl(pos, [&] { ParserRolesOrUsersSet roles_p; - roles_p.allowRoleNames().useIDMode(id_mode); - if (kind == Kind::REVOKE) + roles_p.allowRoles().useIDMode(id_mode); + if (is_revoke) roles_p.allowAll(); ASTPtr ast; @@ -199,28 +198,20 @@ namespace } - bool parseToRoles(IParser::Pos & pos, Expected & expected, ASTGrantQuery::Kind kind, std::shared_ptr & to_roles) + bool parseToGrantees(IParser::Pos & pos, Expected & expected, bool is_revoke, std::shared_ptr & grantees) { return IParserBase::wrapParseImpl(pos, [&] { - if (kind == Kind::GRANT) - { - if (!ParserKeyword{"TO"}.ignore(pos, expected)) - return false; - } - else - { - if (!ParserKeyword{"FROM"}.ignore(pos, expected)) - return false; - } + if (!ParserKeyword{is_revoke ? "FROM" : "TO"}.ignore(pos, expected)) + return false; ASTPtr ast; ParserRolesOrUsersSet roles_p; - roles_p.allowRoleNames().allowUserNames().allowCurrentUser().allowAll(kind == Kind::REVOKE); + roles_p.allowRoles().allowUsers().allowCurrentUser().allowAll(is_revoke); if (!roles_p.parse(pos, ast, expected)) return false; - to_roles = typeid_cast>(ast); + grantees = typeid_cast>(ast); return true; }); } @@ -237,20 +228,13 @@ namespace bool ParserGrantQuery::parseImpl(Pos & pos, ASTPtr & node, Expected & expected) { - bool attach = false; - if (attach_mode) - { - if (!ParserKeyword{"ATTACH"}.ignore(pos, expected)) - return false; - attach = true; - } + if (attach_mode && !ParserKeyword{"ATTACH"}.ignore(pos, expected)) + return false; - Kind kind; - if (ParserKeyword{"GRANT"}.ignore(pos, expected)) - kind = Kind::GRANT; - else if (ParserKeyword{"REVOKE"}.ignore(pos, expected)) - kind = Kind::REVOKE; - else + bool is_revoke = false; + if (ParserKeyword{"REVOKE"}.ignore(pos, expected)) + is_revoke = true; + else if (!ParserKeyword{"GRANT"}.ignore(pos, expected)) return false; String cluster; @@ -259,7 +243,7 @@ bool ParserGrantQuery::parseImpl(Pos & pos, ASTPtr & node, Expected & expected) bool grant_option = false; bool admin_option = false; - if (kind == Kind::REVOKE) + if (is_revoke) { if (ParserKeyword{"GRANT OPTION FOR"}.ignore(pos, expected)) grant_option = true; @@ -269,20 +253,20 @@ bool ParserGrantQuery::parseImpl(Pos & pos, ASTPtr & node, Expected & expected) AccessRightsElements elements; std::shared_ptr roles; - if (!parseAccessRightsElements(pos, expected, elements) && !parseRoles(pos, expected, kind, attach, roles)) + if (!parseElementsWithoutOptions(pos, expected, elements) && !parseRoles(pos, expected, is_revoke, attach_mode, roles)) return false; if (cluster.empty()) parseOnCluster(pos, expected, cluster); - std::shared_ptr to_roles; - if (!parseToRoles(pos, expected, kind, to_roles)) + std::shared_ptr grantees; + if (!parseToGrantees(pos, expected, is_revoke, grantees)) return false; if (cluster.empty()) parseOnCluster(pos, expected, cluster); - if (kind == Kind::GRANT) + if (!is_revoke) { if (ParserKeyword{"WITH GRANT OPTION"}.ignore(pos, expected)) grant_option = true; @@ -298,19 +282,24 @@ bool ParserGrantQuery::parseImpl(Pos & pos, ASTPtr & node, Expected & expected) if (admin_option && !elements.empty()) throw Exception("ADMIN OPTION should be specified for roles", ErrorCodes::SYNTAX_ERROR); - if (kind == Kind::GRANT) - removeNonGrantableFlags(elements); + if (grant_option) + { + for (auto & element : elements) + element.grant_option = true; + } + + if (!is_revoke) + eraseNonGrantable(elements); auto query = std::make_shared(); node = query; - query->kind = kind; - query->attach = attach; + query->is_revoke = is_revoke; + query->attach_mode = attach_mode; query->cluster = std::move(cluster); query->access_rights_elements = std::move(elements); query->roles = std::move(roles); - query->to_roles = std::move(to_roles); - query->grant_option = grant_option; + query->grantees = std::move(grantees); query->admin_option = admin_option; return true; diff --git a/src/Parsers/ParserRolesOrUsersSet.cpp b/src/Parsers/ParserRolesOrUsersSet.cpp index 0f3ba3f0f84..701c5c2f9d5 100644 --- a/src/Parsers/ParserRolesOrUsersSet.cpp +++ b/src/Parsers/ParserRolesOrUsersSet.cpp @@ -12,11 +12,7 @@ namespace DB { namespace { - bool parseRoleNameOrID( - IParserBase::Pos & pos, - Expected & expected, - bool id_mode, - String & res) + bool parseNameOrID(IParserBase::Pos & pos, Expected & expected, bool id_mode, String & res) { return IParserBase::wrapParseImpl(pos, [&] { @@ -39,20 +35,20 @@ namespace }); } - bool parseBeforeExcept( IParserBase::Pos & pos, Expected & expected, bool id_mode, bool allow_all, bool allow_current_user, - Strings & names, bool & all, + Strings & names, bool & current_user) { bool res_all = false; - bool res_current_user = false; Strings res_names; + bool res_current_user = false; + Strings res_with_roles_names; auto parse_element = [&] { @@ -72,7 +68,7 @@ namespace } String name; - if (parseRoleNameOrID(pos, expected, id_mode, name)) + if (parseNameOrID(pos, expected, id_mode, name)) { res_names.emplace_back(std::move(name)); return true; @@ -85,8 +81,8 @@ namespace return false; names = std::move(res_names); - all = res_all; current_user = res_current_user; + all = res_all; return true; } @@ -98,13 +94,12 @@ namespace Strings & except_names, bool & except_current_user) { - return IParserBase::wrapParseImpl(pos, [&] - { + return IParserBase::wrapParseImpl(pos, [&] { if (!ParserKeyword{"EXCEPT"}.ignore(pos, expected)) return false; bool unused; - return parseBeforeExcept(pos, expected, id_mode, false, allow_current_user, except_names, unused, except_current_user); + return parseBeforeExcept(pos, expected, id_mode, false, allow_current_user, unused, except_names, except_current_user); }); } } @@ -112,13 +107,13 @@ namespace bool ParserRolesOrUsersSet::parseImpl(Pos & pos, ASTPtr & node, Expected & expected) { + bool all = false; Strings names; bool current_user = false; - bool all = false; Strings except_names; bool except_current_user = false; - if (!parseBeforeExcept(pos, expected, id_mode, allow_all, allow_current_user, names, all, current_user)) + if (!parseBeforeExcept(pos, expected, id_mode, allow_all, allow_current_user, all, names, current_user)) return false; parseExceptAndAfterExcept(pos, expected, id_mode, allow_current_user, except_names, except_current_user); @@ -132,9 +127,9 @@ bool ParserRolesOrUsersSet::parseImpl(Pos & pos, ASTPtr & node, Expected & expec result->all = all; result->except_names = std::move(except_names); result->except_current_user = except_current_user; + result->allow_users = allow_users; + result->allow_roles = allow_roles; result->id_mode = id_mode; - result->allow_user_names = allow_user_names; - result->allow_role_names = allow_role_names; node = result; return true; } diff --git a/src/Parsers/ParserRolesOrUsersSet.h b/src/Parsers/ParserRolesOrUsersSet.h index c71012e874c..d63c045e7a0 100644 --- a/src/Parsers/ParserRolesOrUsersSet.h +++ b/src/Parsers/ParserRolesOrUsersSet.h @@ -6,15 +6,16 @@ namespace DB { /** Parses a string like this: - * {role|CURRENT_USER} [,...] | NONE | ALL | ALL EXCEPT {role|CURRENT_USER} [,...] + * {user_name | role_name | CURRENT_USER | ALL | NONE} [,...] + * [EXCEPT {user_name | role_name | CURRENT_USER | ALL | NONE} [,...]] */ class ParserRolesOrUsersSet : public IParserBase { public: ParserRolesOrUsersSet & allowAll(bool allow_all_ = true) { allow_all = allow_all_; return *this; } - ParserRolesOrUsersSet & allowUserNames(bool allow_user_names_ = true) { allow_user_names = allow_user_names_; return *this; } - ParserRolesOrUsersSet & allowRoleNames(bool allow_role_names_ = true) { allow_role_names = allow_role_names_; return *this; } + ParserRolesOrUsersSet & allowUsers(bool allow_users_ = true) { allow_users = allow_users_; return *this; } ParserRolesOrUsersSet & allowCurrentUser(bool allow_current_user_ = true) { allow_current_user = allow_current_user_; return *this; } + ParserRolesOrUsersSet & allowRoles(bool allow_roles_ = true) { allow_roles = allow_roles_; return *this; } ParserRolesOrUsersSet & useIDMode(bool id_mode_ = true) { id_mode = id_mode_; return *this; } protected: @@ -23,9 +24,9 @@ protected: private: bool allow_all = false; - bool allow_user_names = false; - bool allow_role_names = false; + bool allow_users = false; bool allow_current_user = false; + bool allow_roles = false; bool id_mode = false; }; diff --git a/src/Parsers/ParserSetRoleQuery.cpp b/src/Parsers/ParserSetRoleQuery.cpp index e8734f8dfc1..678474af040 100644 --- a/src/Parsers/ParserSetRoleQuery.cpp +++ b/src/Parsers/ParserSetRoleQuery.cpp @@ -15,12 +15,12 @@ namespace { ASTPtr ast; ParserRolesOrUsersSet roles_p; - roles_p.allowRoleNames().allowAll(); + roles_p.allowRoles().allowAll(); if (!roles_p.parse(pos, ast, expected)) return false; roles = typeid_cast>(ast); - roles->allow_user_names = false; + roles->allow_users = false; return true; }); } @@ -34,12 +34,12 @@ namespace ASTPtr ast; ParserRolesOrUsersSet users_p; - users_p.allowUserNames().allowCurrentUser(); + users_p.allowUsers().allowCurrentUser(); if (!users_p.parse(pos, ast, expected)) return false; to_users = typeid_cast>(ast); - to_users->allow_role_names = false; + to_users->allow_roles = false; return true; }); } diff --git a/src/Parsers/ParserShowGrantsQuery.cpp b/src/Parsers/ParserShowGrantsQuery.cpp index d25527754be..bd9e4012771 100644 --- a/src/Parsers/ParserShowGrantsQuery.cpp +++ b/src/Parsers/ParserShowGrantsQuery.cpp @@ -19,7 +19,7 @@ bool ParserShowGrantsQuery::parseImpl(Pos & pos, ASTPtr & node, Expected & expec { ASTPtr for_roles_ast; ParserRolesOrUsersSet for_roles_p; - for_roles_p.allowUserNames().allowRoleNames().allowAll().allowCurrentUser(); + for_roles_p.allowUsers().allowRoles().allowAll().allowCurrentUser(); if (!for_roles_p.parse(pos, for_roles_ast, expected)) return false; diff --git a/src/Storages/System/StorageSystemGrants.cpp b/src/Storages/System/StorageSystemGrants.cpp index 360256c1f45..0c06ad99b22 100644 --- a/src/Storages/System/StorageSystemGrants.cpp +++ b/src/Storages/System/StorageSystemGrants.cpp @@ -18,7 +18,6 @@ namespace DB { using EntityType = IAccessEntity::Type; -using Kind = AccessRightsElementWithOptions::Kind; NamesAndTypesList StorageSystemGrants::getNamesAndTypes() { @@ -64,7 +63,7 @@ void StorageSystemGrants::fillData(MutableColumns & res_columns, const Context & const String * database, const String * table, const String * column, - Kind kind, + bool is_partial_revoke, bool grant_option) { if (grantee_type == EntityType::USER) @@ -119,13 +118,13 @@ void StorageSystemGrants::fillData(MutableColumns & res_columns, const Context & column_column_null_map.push_back(true); } - column_is_partial_revoke.push_back(kind == Kind::REVOKE); + column_is_partial_revoke.push_back(is_partial_revoke); column_grant_option.push_back(grant_option); }; auto add_rows = [&](const String & grantee_name, IAccessEntity::Type grantee_type, - const AccessRightsElementsWithOptions & elements) + const AccessRightsElements & elements) { for (const auto & element : elements) { @@ -139,13 +138,13 @@ void StorageSystemGrants::fillData(MutableColumns & res_columns, const Context & if (element.any_column) { for (const auto & access_type : access_types) - add_row(grantee_name, grantee_type, access_type, database, table, nullptr, element.kind, element.grant_option); + add_row(grantee_name, grantee_type, access_type, database, table, nullptr, element.is_partial_revoke, element.grant_option); } else { for (const auto & access_type : access_types) for (const auto & column : element.columns) - add_row(grantee_name, grantee_type, access_type, database, table, &column, element.kind, element.grant_option); + add_row(grantee_name, grantee_type, access_type, database, table, &column, element.is_partial_revoke, element.grant_option); } } }; diff --git a/src/Storages/System/StorageSystemRoleGrants.cpp b/src/Storages/System/StorageSystemRoleGrants.cpp index 0f0fcd831d9..cf0fad8f8ce 100644 --- a/src/Storages/System/StorageSystemRoleGrants.cpp +++ b/src/Storages/System/StorageSystemRoleGrants.cpp @@ -80,15 +80,17 @@ void StorageSystemRoleGrants::fillData(MutableColumns & res_columns, const Conte const GrantedRoles & granted_roles, const RolesOrUsersSet * default_roles) { - for (const auto & role_id : granted_roles.roles) + for (const auto & element : granted_roles.getElements()) { - auto role_name = access_control.tryReadName(role_id); - if (!role_name) - continue; + for (const auto & role_id : element.ids) + { + auto role_name = access_control.tryReadName(role_id); + if (!role_name) + continue; - bool is_default = !default_roles || default_roles->match(role_id); - bool with_admin_option = granted_roles.roles_with_admin_option.count(role_id); - add_row(grantee_name, grantee_type, *role_name, is_default, with_admin_option); + bool is_default = !default_roles || default_roles->match(role_id); + add_row(grantee_name, grantee_type, *role_name, is_default, element.admin_option); + } } }; diff --git a/tests/testflows/rbac/tests/syntax/revoke_role.py b/tests/testflows/rbac/tests/syntax/revoke_role.py index 4acdf127cec..ea8b874ff51 100755 --- a/tests/testflows/rbac/tests/syntax/revoke_role.py +++ b/tests/testflows/rbac/tests/syntax/revoke_role.py @@ -166,9 +166,10 @@ def feature(self, node="clickhouse1"): with Scenario("I revoke a role on fake cluster, throws exception", requirements=[ RQ_SRS_006_RBAC_Revoke_Role_Cluster("1.0")]): - with When("I revoke a role from user on a cluster"): - exitcode, message = errors.cluster_not_found("fake_cluster") - node.query("REVOKE ON CLUSTER fake_cluster role0 FROM user0", exitcode=exitcode, message=message) + with setup(): + with When("I revoke a role from user on a cluster"): + exitcode, message = errors.cluster_not_found("fake_cluster") + node.query("REVOKE ON CLUSTER fake_cluster role0 FROM user0", exitcode=exitcode, message=message) with Scenario("I revoke multiple roles from multiple users on cluster", requirements=[ RQ_SRS_006_RBAC_Revoke_Role("1.0"), From 37ce6e26d365787ac68933db782dca1c5180d571 Mon Sep 17 00:00:00 2001 From: Vitaly Baranov Date: Thu, 11 Mar 2021 16:44:00 +0300 Subject: [PATCH 448/544] Add a new clause GRANTEES to commands CREATE USER and ALTER USER. --- src/Access/User.cpp | 2 +- src/Access/User.h | 1 + .../InterpreterCreateUserQuery.cpp | 37 ++++++++++++--- src/Interpreters/InterpreterGrantQuery.cpp | 45 +++++++++++++++++++ ...InterpreterShowCreateAccessEntityQuery.cpp | 9 ++++ src/Parsers/ASTCreateUserQuery.cpp | 10 +++++ src/Parsers/ASTCreateUserQuery.h | 3 ++ src/Parsers/ASTRolesOrUsersSet.cpp | 3 +- src/Parsers/ASTRolesOrUsersSet.h | 7 +-- src/Parsers/ParserCreateUserQuery.cpp | 23 ++++++++++ src/Parsers/ParserCreateUserQuery.h | 2 + src/Parsers/ParserRolesOrUsersSet.cpp | 12 ++++- src/Parsers/ParserRolesOrUsersSet.h | 2 + src/Storages/System/StorageSystemUsers.cpp | 28 +++++++++--- .../integration/test_grant_and_revoke/test.py | 42 ++++++++++++++++- 15 files changed, 205 insertions(+), 21 deletions(-) diff --git a/src/Access/User.cpp b/src/Access/User.cpp index f57ec7c1359..016f378e83f 100644 --- a/src/Access/User.cpp +++ b/src/Access/User.cpp @@ -11,7 +11,7 @@ bool User::equal(const IAccessEntity & other) const const auto & other_user = typeid_cast(other); return (authentication == other_user.authentication) && (allowed_client_hosts == other_user.allowed_client_hosts) && (access == other_user.access) && (granted_roles == other_user.granted_roles) && (default_roles == other_user.default_roles) - && (settings == other_user.settings); + && (settings == other_user.settings) && (grantees == other_user.grantees); } } diff --git a/src/Access/User.h b/src/Access/User.h index 13f1e532015..5b10d953fc0 100644 --- a/src/Access/User.h +++ b/src/Access/User.h @@ -21,6 +21,7 @@ struct User : public IAccessEntity GrantedRoles granted_roles; RolesOrUsersSet default_roles = RolesOrUsersSet::AllTag{}; SettingsProfileElements settings; + RolesOrUsersSet grantees = RolesOrUsersSet::AllTag{}; bool equal(const IAccessEntity & other) const override; std::shared_ptr clone() const override { return cloneImpl(); } diff --git a/src/Interpreters/InterpreterCreateUserQuery.cpp b/src/Interpreters/InterpreterCreateUserQuery.cpp index c9b087de5b4..777cf7de297 100644 --- a/src/Interpreters/InterpreterCreateUserQuery.cpp +++ b/src/Interpreters/InterpreterCreateUserQuery.cpp @@ -20,7 +20,8 @@ namespace const ASTCreateUserQuery & query, const std::shared_ptr & override_name, const std::optional & override_default_roles, - const std::optional & override_settings) + const std::optional & override_settings, + const std::optional & override_grantees) { if (override_name) user.setName(override_name->toString()); @@ -62,6 +63,11 @@ namespace user.settings = *override_settings; else if (query.settings) user.settings = *query.settings; + + if (override_grantees) + user.grantees = *override_grantees; + else if (query.grantees) + user.grantees = *query.grantees; } } @@ -93,12 +99,17 @@ BlockIO InterpreterCreateUserQuery::execute() if (query.alter) { + std::optional grantees_from_query; + if (query.grantees) + grantees_from_query = RolesOrUsersSet{*query.grantees, access_control}; + auto update_func = [&](const AccessEntityPtr & entity) -> AccessEntityPtr { auto updated_user = typeid_cast>(entity->clone()); - updateUserFromQueryImpl(*updated_user, query, {}, default_roles_from_query, settings_from_query); + updateUserFromQueryImpl(*updated_user, query, {}, default_roles_from_query, settings_from_query, grantees_from_query); return updated_user; }; + Strings names = query.names->toStrings(); if (query.if_exists) { @@ -114,16 +125,28 @@ BlockIO InterpreterCreateUserQuery::execute() for (const auto & name : *query.names) { auto new_user = std::make_shared(); - updateUserFromQueryImpl(*new_user, query, name, default_roles_from_query, settings_from_query); + updateUserFromQueryImpl(*new_user, query, name, default_roles_from_query, settings_from_query, RolesOrUsersSet::AllTag{}); new_users.emplace_back(std::move(new_user)); } + std::vector ids; if (query.if_not_exists) - access_control.tryInsert(new_users); + ids = access_control.tryInsert(new_users); else if (query.or_replace) - access_control.insertOrReplace(new_users); + ids = access_control.insertOrReplace(new_users); else - access_control.insert(new_users); + ids = access_control.insert(new_users); + + if (query.grantees) + { + RolesOrUsersSet grantees_from_query = RolesOrUsersSet{*query.grantees, access_control}; + access_control.update(ids, [&](const AccessEntityPtr & entity) -> AccessEntityPtr + { + auto updated_user = typeid_cast>(entity->clone()); + updated_user->grantees = grantees_from_query; + return updated_user; + }); + } } return {}; @@ -132,7 +155,7 @@ BlockIO InterpreterCreateUserQuery::execute() void InterpreterCreateUserQuery::updateUserFromQuery(User & user, const ASTCreateUserQuery & query) { - updateUserFromQueryImpl(user, query, {}, {}, {}); + updateUserFromQueryImpl(user, query, {}, {}, {}, {}); } } diff --git a/src/Interpreters/InterpreterGrantQuery.cpp b/src/Interpreters/InterpreterGrantQuery.cpp index ac37266bcfb..b518178f6d0 100644 --- a/src/Interpreters/InterpreterGrantQuery.cpp +++ b/src/Interpreters/InterpreterGrantQuery.cpp @@ -16,6 +16,7 @@ namespace DB { namespace ErrorCodes { + extern const int ACCESS_DENIED; extern const int LOGICAL_ERROR; } @@ -65,6 +66,29 @@ namespace updateFromQueryTemplate(*role, query, roles_to_grant_or_revoke); } + void checkGranteeIsAllowed(const ContextAccess & access, const UUID & grantee_id, const IAccessEntity & grantee) + { + auto current_user = access.getUser(); + if (current_user && !current_user->grantees.match(grantee_id)) + throw Exception(grantee.outputTypeAndName() + " is not allowed as grantee", ErrorCodes::ACCESS_DENIED); + } + + void checkGranteesAreAllowed(const AccessControlManager & access_control, const ContextAccess & access, const std::vector & grantee_ids) + { + auto current_user = access.getUser(); + if (!current_user || (current_user->grantees == RolesOrUsersSet::AllTag{})) + return; + + for (const auto & id : grantee_ids) + { + auto entity = access_control.tryRead(id); + if (auto role = typeid_cast(entity)) + checkGranteeIsAllowed(access, id, *role); + else if (auto user = typeid_cast(entity)) + checkGranteeIsAllowed(access, id, *user); + } + } + void checkGrantOption( const AccessControlManager & access_control, const ContextAccess & access, @@ -80,11 +104,15 @@ namespace if (!query.is_revoke) { access.checkGrantOption(elements); + checkGranteesAreAllowed(access_control, access, grantees_from_query); return; } if (access.hasGrantOption(elements)) + { + checkGranteesAreAllowed(access_control, access, grantees_from_query); return; + } /// Special case for the command REVOKE: it's possible that the current user doesn't have /// the access granted with GRANT OPTION but it's still ok because the roles or users @@ -99,9 +127,15 @@ namespace { auto entity = access_control.tryRead(id); if (auto role = typeid_cast(entity)) + { + checkGranteeIsAllowed(access, id, *role); all_granted_access.makeUnion(role->access); + } else if (auto user = typeid_cast(entity)) + { + checkGranteeIsAllowed(access, id, *user); all_granted_access.makeUnion(user->access); + } } AccessRights required_access; @@ -138,6 +172,7 @@ namespace { matching_ids = roles_from_query.getMatchingIDs(access_control); access.checkAdminOption(matching_ids); + checkGranteesAreAllowed(access_control, access, grantees_from_query); return matching_ids; } @@ -145,7 +180,10 @@ namespace { matching_ids = roles_from_query.getMatchingIDs(); if (access.hasAdminOption(matching_ids)) + { + checkGranteesAreAllowed(access_control, access, grantees_from_query); return matching_ids; + } } /// Special case for the command REVOKE: it's possible that the current user doesn't have the admin option @@ -161,9 +199,15 @@ namespace { auto entity = access_control.tryRead(id); if (auto role = typeid_cast(entity)) + { + checkGranteeIsAllowed(access, id, *role); all_granted_roles.makeUnion(role->granted_roles); + } else if (auto user = typeid_cast(entity)) + { + checkGranteeIsAllowed(access, id, *user); all_granted_roles.makeUnion(user->granted_roles); + } } const auto & all_granted_roles_set = query.admin_option ? all_granted_roles.getGrantedWithAdminOption() : all_granted_roles.getGranted(); @@ -206,6 +250,7 @@ BlockIO InterpreterGrantQuery::execute() /// To execute the command GRANT the current user needs to have the access granted with GRANT OPTION. auto required_access = query.access_rights_elements; std::for_each(required_access.begin(), required_access.end(), [&](AccessRightsElement & element) { element.grant_option = true; }); + checkGranteesAreAllowed(access_control, *context.getAccess(), grantees); return executeDDLQueryOnCluster(query_ptr, context, std::move(required_access)); } diff --git a/src/Interpreters/InterpreterShowCreateAccessEntityQuery.cpp b/src/Interpreters/InterpreterShowCreateAccessEntityQuery.cpp index 4e391035d5d..c39fed8fb62 100644 --- a/src/Interpreters/InterpreterShowCreateAccessEntityQuery.cpp +++ b/src/Interpreters/InterpreterShowCreateAccessEntityQuery.cpp @@ -73,6 +73,15 @@ namespace query->settings = user.settings.toASTWithNames(*manager); } + if (user.grantees != RolesOrUsersSet::AllTag{}) + { + if (attach_mode) + query->grantees = user.grantees.toAST(); + else + query->grantees = user.grantees.toASTWithNames(*manager); + query->grantees->use_keyword_any = true; + } + return query; } diff --git a/src/Parsers/ASTCreateUserQuery.cpp b/src/Parsers/ASTCreateUserQuery.cpp index e2e477fa622..696b88ea9c1 100644 --- a/src/Parsers/ASTCreateUserQuery.cpp +++ b/src/Parsers/ASTCreateUserQuery.cpp @@ -203,6 +203,13 @@ namespace format.ostr << (format.hilite ? IAST::hilite_keyword : "") << " SETTINGS " << (format.hilite ? IAST::hilite_none : ""); settings.format(format); } + + + void formatGrantees(const ASTRolesOrUsersSet & grantees, const IAST::FormatSettings & settings) + { + settings.ostr << (settings.hilite ? IAST::hilite_keyword : "") << " GRANTEES " << (settings.hilite ? IAST::hilite_none : ""); + grantees.format(settings); + } } @@ -260,5 +267,8 @@ void ASTCreateUserQuery::formatImpl(const FormatSettings & format, FormatState & if (settings && (!settings->empty() || alter)) formatSettings(*settings, format); + + if (grantees) + formatGrantees(*grantees, format); } } diff --git a/src/Parsers/ASTCreateUserQuery.h b/src/Parsers/ASTCreateUserQuery.h index 22992b2c408..1612c213f34 100644 --- a/src/Parsers/ASTCreateUserQuery.h +++ b/src/Parsers/ASTCreateUserQuery.h @@ -17,6 +17,7 @@ class ASTSettingsProfileElements; * [HOST {LOCAL | NAME 'name' | REGEXP 'name_regexp' | IP 'address' | LIKE 'pattern'} [,...] | ANY | NONE] * [DEFAULT ROLE role [,...]] * [SETTINGS variable [= value] [MIN [=] min_value] [MAX [=] max_value] [READONLY|WRITABLE] | PROFILE 'profile_name'] [,...] + * [GRANTEES {user | role | ANY | NONE} [,...] [EXCEPT {user | role} [,...]]] * * ALTER USER [IF EXISTS] name * [RENAME TO new_name] @@ -24,6 +25,7 @@ class ASTSettingsProfileElements; * [[ADD|DROP] HOST {LOCAL | NAME 'name' | REGEXP 'name_regexp' | IP 'address' | LIKE 'pattern'} [,...] | ANY | NONE] * [DEFAULT ROLE role [,...] | ALL | ALL EXCEPT role [,...] ] * [SETTINGS variable [= value] [MIN [=] min_value] [MAX [=] max_value] [READONLY|WRITABLE] | PROFILE 'profile_name'] [,...] + * [GRANTEES {user | role | ANY | NONE} [,...] [EXCEPT {user | role} [,...]]] */ class ASTCreateUserQuery : public IAST, public ASTQueryWithOnCluster { @@ -47,6 +49,7 @@ public: std::shared_ptr default_roles; std::shared_ptr settings; + std::shared_ptr grantees; String getID(char) const override; ASTPtr clone() const override; diff --git a/src/Parsers/ASTRolesOrUsersSet.cpp b/src/Parsers/ASTRolesOrUsersSet.cpp index 9eb83cf895e..fc5385e4a58 100644 --- a/src/Parsers/ASTRolesOrUsersSet.cpp +++ b/src/Parsers/ASTRolesOrUsersSet.cpp @@ -35,7 +35,8 @@ void ASTRolesOrUsersSet::formatImpl(const FormatSettings & settings, FormatState { if (std::exchange(need_comma, true)) settings.ostr << ", "; - settings.ostr << (settings.hilite ? IAST::hilite_keyword : "") << "ALL" << (settings.hilite ? IAST::hilite_none : ""); + settings.ostr << (settings.hilite ? IAST::hilite_keyword : "") << (use_keyword_any ? "ANY" : "ALL") + << (settings.hilite ? IAST::hilite_none : ""); } else { diff --git a/src/Parsers/ASTRolesOrUsersSet.h b/src/Parsers/ASTRolesOrUsersSet.h index 0f78f67d35b..15d42ee39a0 100644 --- a/src/Parsers/ASTRolesOrUsersSet.h +++ b/src/Parsers/ASTRolesOrUsersSet.h @@ -20,9 +20,10 @@ public: Strings except_names; bool except_current_user = false; - bool allow_users = true; /// whether this set can contain names of users - bool allow_roles = true; /// whether this set can contain names of roles - bool id_mode = false; /// whether this set keep UUIDs instead of names + bool allow_users = true; /// whether this set can contain names of users + bool allow_roles = true; /// whether this set can contain names of roles + bool id_mode = false; /// whether this set keep UUIDs instead of names + bool use_keyword_any = false; /// whether the keyword ANY should be used instead of the keyword ALL bool empty() const { return names.empty() && !current_user && !all; } void replaceCurrentUserTag(const String & current_user_name); diff --git a/src/Parsers/ParserCreateUserQuery.cpp b/src/Parsers/ParserCreateUserQuery.cpp index b856b03ab9b..84bf60d56d3 100644 --- a/src/Parsers/ParserCreateUserQuery.cpp +++ b/src/Parsers/ParserCreateUserQuery.cpp @@ -275,6 +275,24 @@ namespace }); } + bool parseGrantees(IParserBase::Pos & pos, Expected & expected, bool id_mode, std::shared_ptr & grantees) + { + return IParserBase::wrapParseImpl(pos, [&] + { + if (!ParserKeyword{"GRANTEES"}.ignore(pos, expected)) + return false; + + ASTPtr ast; + ParserRolesOrUsersSet grantees_p; + grantees_p.allowAny().allowUsers().allowCurrentUser().allowRoles().useIDMode(id_mode); + if (!grantees_p.parse(pos, ast, expected)) + return false; + + grantees = typeid_cast>(ast); + return true; + }); + } + bool parseOnCluster(IParserBase::Pos & pos, Expected & expected, String & cluster) { return IParserBase::wrapParseImpl(pos, [&] @@ -330,6 +348,7 @@ bool ParserCreateUserQuery::parseImpl(Pos & pos, ASTPtr & node, Expected & expec std::optional remove_hosts; std::shared_ptr default_roles; std::shared_ptr settings; + std::shared_ptr grantees; String cluster; while (true) @@ -368,6 +387,9 @@ bool ParserCreateUserQuery::parseImpl(Pos & pos, ASTPtr & node, Expected & expec if (cluster.empty() && parseOnCluster(pos, expected, cluster)) continue; + if (!grantees && parseGrantees(pos, expected, attach_mode, grantees)) + continue; + if (alter) { if (new_name.empty() && (names->size() == 1) && parseRenameTo(pos, expected, new_name)) @@ -422,6 +444,7 @@ bool ParserCreateUserQuery::parseImpl(Pos & pos, ASTPtr & node, Expected & expec query->remove_hosts = std::move(remove_hosts); query->default_roles = std::move(default_roles); query->settings = std::move(settings); + query->grantees = std::move(grantees); return true; } diff --git a/src/Parsers/ParserCreateUserQuery.h b/src/Parsers/ParserCreateUserQuery.h index 0eee522979f..215133a777c 100644 --- a/src/Parsers/ParserCreateUserQuery.h +++ b/src/Parsers/ParserCreateUserQuery.h @@ -11,6 +11,7 @@ namespace DB * [HOST {LOCAL | NAME 'name' | REGEXP 'name_regexp' | IP 'address' | LIKE 'pattern'} [,...] | ANY | NONE] * [DEFAULT ROLE role [,...]] * [SETTINGS variable [= value] [MIN [=] min_value] [MAX [=] max_value] [READONLY|WRITABLE] | PROFILE 'profile_name'] [,...] + * [GRANTEES {user | role | ANY | NONE} [,...] [EXCEPT {user | role} [,...]]] * * ALTER USER [IF EXISTS] name * [RENAME TO new_name] @@ -18,6 +19,7 @@ namespace DB * [[ADD|DROP] HOST {LOCAL | NAME 'name' | REGEXP 'name_regexp' | IP 'address' | LIKE 'pattern'} [,...] | ANY | NONE] * [DEFAULT ROLE role [,...] | ALL | ALL EXCEPT role [,...] ] * [SETTINGS variable [= value] [MIN [=] min_value] [MAX [=] max_value] [READONLY|WRITABLE] | PROFILE 'profile_name'] [,...] + * [GRANTEES {user | role | ANY | NONE} [,...] [EXCEPT {user | role} [,...]]] */ class ParserCreateUserQuery : public IParserBase { diff --git a/src/Parsers/ParserRolesOrUsersSet.cpp b/src/Parsers/ParserRolesOrUsersSet.cpp index 701c5c2f9d5..41e9ee6501d 100644 --- a/src/Parsers/ParserRolesOrUsersSet.cpp +++ b/src/Parsers/ParserRolesOrUsersSet.cpp @@ -40,6 +40,7 @@ namespace Expected & expected, bool id_mode, bool allow_all, + bool allow_any, bool allow_current_user, bool & all, Strings & names, @@ -61,6 +62,12 @@ namespace return true; } + if (allow_any && ParserKeyword{"ANY"}.ignore(pos, expected)) + { + res_all = true; + return true; + } + if (allow_current_user && parseCurrentUserTag(pos, expected)) { res_current_user = true; @@ -99,7 +106,7 @@ namespace return false; bool unused; - return parseBeforeExcept(pos, expected, id_mode, false, allow_current_user, unused, except_names, except_current_user); + return parseBeforeExcept(pos, expected, id_mode, false, false, allow_current_user, unused, except_names, except_current_user); }); } } @@ -113,7 +120,7 @@ bool ParserRolesOrUsersSet::parseImpl(Pos & pos, ASTPtr & node, Expected & expec Strings except_names; bool except_current_user = false; - if (!parseBeforeExcept(pos, expected, id_mode, allow_all, allow_current_user, all, names, current_user)) + if (!parseBeforeExcept(pos, expected, id_mode, allow_all, allow_any, allow_current_user, all, names, current_user)) return false; parseExceptAndAfterExcept(pos, expected, id_mode, allow_current_user, except_names, except_current_user); @@ -130,6 +137,7 @@ bool ParserRolesOrUsersSet::parseImpl(Pos & pos, ASTPtr & node, Expected & expec result->allow_users = allow_users; result->allow_roles = allow_roles; result->id_mode = id_mode; + result->use_keyword_any = all && allow_any && !allow_all; node = result; return true; } diff --git a/src/Parsers/ParserRolesOrUsersSet.h b/src/Parsers/ParserRolesOrUsersSet.h index d63c045e7a0..9ae9937e784 100644 --- a/src/Parsers/ParserRolesOrUsersSet.h +++ b/src/Parsers/ParserRolesOrUsersSet.h @@ -13,6 +13,7 @@ class ParserRolesOrUsersSet : public IParserBase { public: ParserRolesOrUsersSet & allowAll(bool allow_all_ = true) { allow_all = allow_all_; return *this; } + ParserRolesOrUsersSet & allowAny(bool allow_any_ = true) { allow_any = allow_any_; return *this; } ParserRolesOrUsersSet & allowUsers(bool allow_users_ = true) { allow_users = allow_users_; return *this; } ParserRolesOrUsersSet & allowCurrentUser(bool allow_current_user_ = true) { allow_current_user = allow_current_user_; return *this; } ParserRolesOrUsersSet & allowRoles(bool allow_roles_ = true) { allow_roles = allow_roles_; return *this; } @@ -24,6 +25,7 @@ protected: private: bool allow_all = false; + bool allow_any = false; bool allow_users = false; bool allow_current_user = false; bool allow_roles = false; diff --git a/src/Storages/System/StorageSystemUsers.cpp b/src/Storages/System/StorageSystemUsers.cpp index bec94bc388c..eaebf759a85 100644 --- a/src/Storages/System/StorageSystemUsers.cpp +++ b/src/Storages/System/StorageSystemUsers.cpp @@ -47,6 +47,9 @@ NamesAndTypesList StorageSystemUsers::getNamesAndTypes() {"default_roles_all", std::make_shared()}, {"default_roles_list", std::make_shared(std::make_shared())}, {"default_roles_except", std::make_shared(std::make_shared())}, + {"grantees_any", std::make_shared()}, + {"grantees_list", std::make_shared(std::make_shared())}, + {"grantees_except", std::make_shared(std::make_shared())}, }; return names_and_types; } @@ -77,13 +80,19 @@ void StorageSystemUsers::fillData(MutableColumns & res_columns, const Context & auto & column_default_roles_list_offsets = assert_cast(*res_columns[column_index++]).getOffsets(); auto & column_default_roles_except = assert_cast(assert_cast(*res_columns[column_index]).getData()); auto & column_default_roles_except_offsets = assert_cast(*res_columns[column_index++]).getOffsets(); + auto & column_grantees_any = assert_cast(*res_columns[column_index++]).getData(); + auto & column_grantees_list = assert_cast(assert_cast(*res_columns[column_index]).getData()); + auto & column_grantees_list_offsets = assert_cast(*res_columns[column_index++]).getOffsets(); + auto & column_grantees_except = assert_cast(assert_cast(*res_columns[column_index]).getData()); + auto & column_grantees_except_offsets = assert_cast(*res_columns[column_index++]).getOffsets(); auto add_row = [&](const String & name, const UUID & id, const String & storage_name, const Authentication & authentication, const AllowedClientHosts & allowed_hosts, - const RolesOrUsersSet & default_roles) + const RolesOrUsersSet & default_roles, + const RolesOrUsersSet & grantees) { column_name.insertData(name.data(), name.length()); column_id.push_back(id); @@ -156,14 +165,21 @@ void StorageSystemUsers::fillData(MutableColumns & res_columns, const Context & auto default_roles_ast = default_roles.toASTWithNames(access_control); column_default_roles_all.push_back(default_roles_ast->all); - for (const auto & role_name : default_roles_ast->names) column_default_roles_list.insertData(role_name.data(), role_name.length()); column_default_roles_list_offsets.push_back(column_default_roles_list.size()); - - for (const auto & role_name : default_roles_ast->except_names) - column_default_roles_except.insertData(role_name.data(), role_name.length()); + for (const auto & except_name : default_roles_ast->except_names) + column_default_roles_except.insertData(except_name.data(), except_name.length()); column_default_roles_except_offsets.push_back(column_default_roles_except.size()); + + auto grantees_ast = grantees.toASTWithNames(access_control); + column_grantees_any.push_back(grantees_ast->all); + for (const auto & grantee_name : grantees_ast->names) + column_grantees_list.insertData(grantee_name.data(), grantee_name.length()); + column_grantees_list_offsets.push_back(column_grantees_list.size()); + for (const auto & except_name : grantees_ast->except_names) + column_grantees_except.insertData(except_name.data(), except_name.length()); + column_grantees_except_offsets.push_back(column_grantees_except.size()); }; for (const auto & id : ids) @@ -176,7 +192,7 @@ void StorageSystemUsers::fillData(MutableColumns & res_columns, const Context & if (!storage) continue; - add_row(user->getName(), id, storage->getStorageName(), user->authentication, user->allowed_client_hosts, user->default_roles); + add_row(user->getName(), id, storage->getStorageName(), user->authentication, user->allowed_client_hosts, user->default_roles, user->grantees); } } diff --git a/tests/integration/test_grant_and_revoke/test.py b/tests/integration/test_grant_and_revoke/test.py index e29d63c9e0b..c1be16fe17d 100644 --- a/tests/integration/test_grant_and_revoke/test.py +++ b/tests/integration/test_grant_and_revoke/test.py @@ -26,7 +26,7 @@ def cleanup_after_test(): try: yield finally: - instance.query("DROP USER IF EXISTS A, B") + instance.query("DROP USER IF EXISTS A, B, C") instance.query("DROP TABLE IF EXISTS test.view_1") @@ -106,6 +106,46 @@ def test_revoke_requires_grant_option(): assert instance.query("SHOW GRANTS FOR B") == "" +def test_allowed_grantees(): + instance.query("CREATE USER A") + instance.query("CREATE USER B") + + instance.query('GRANT SELECT ON test.table TO A WITH GRANT OPTION') + instance.query("GRANT SELECT ON test.table TO B", user='A') + assert instance.query("SELECT * FROM test.table", user='B') == "1\t5\n2\t10\n" + instance.query("REVOKE SELECT ON test.table FROM B", user='A') + + instance.query('ALTER USER A GRANTEES NONE') + expected_error = "user `B` is not allowed as grantee" + assert expected_error in instance.query_and_get_error("GRANT SELECT ON test.table TO B", user='A') + + instance.query('ALTER USER A GRANTEES ANY EXCEPT B') + assert instance.query('SHOW CREATE USER A') == "CREATE USER A GRANTEES ANY EXCEPT B\n" + expected_error = "user `B` is not allowed as grantee" + assert expected_error in instance.query_and_get_error("GRANT SELECT ON test.table TO B", user='A') + + instance.query('ALTER USER A GRANTEES B') + instance.query("GRANT SELECT ON test.table TO B", user='A') + assert instance.query("SELECT * FROM test.table", user='B') == "1\t5\n2\t10\n" + instance.query("REVOKE SELECT ON test.table FROM B", user='A') + + instance.query('ALTER USER A GRANTEES ANY') + assert instance.query('SHOW CREATE USER A') == "CREATE USER A\n" + instance.query("GRANT SELECT ON test.table TO B", user='A') + assert instance.query("SELECT * FROM test.table", user='B') == "1\t5\n2\t10\n" + + instance.query('ALTER USER A GRANTEES NONE') + expected_error = "user `B` is not allowed as grantee" + assert expected_error in instance.query_and_get_error("REVOKE SELECT ON test.table FROM B", user='A') + + instance.query("CREATE USER C GRANTEES ANY EXCEPT C") + assert instance.query('SHOW CREATE USER C') == "CREATE USER C GRANTEES ANY EXCEPT C\n" + instance.query('GRANT SELECT ON test.table TO C WITH GRANT OPTION') + assert instance.query("SELECT * FROM test.table", user='C') == "1\t5\n2\t10\n" + expected_error = "user `C` is not allowed as grantee" + assert expected_error in instance.query_and_get_error("REVOKE SELECT ON test.table FROM C", user='C') + + def test_grant_all_on_table(): instance.query("CREATE USER A, B") instance.query("GRANT ALL ON test.table TO A WITH GRANT OPTION") From 1bc21789d239cd3f90703711629b6d15905d86c1 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Sun, 14 Mar 2021 19:52:51 +0300 Subject: [PATCH 449/544] Add more variants --- utils/memcpy-bench/CMakeLists.txt | 23 +- utils/memcpy-bench/FastMemcpy.cpp | 1 + utils/memcpy-bench/FastMemcpy.h | 2 +- utils/memcpy-bench/FastMemcpy_Avx.cpp | 1 + utils/memcpy-bench/glibc/asm-syntax.h | 24 + utils/memcpy-bench/glibc/dwarf2.h | 590 +++ utils/memcpy-bench/glibc/memcpy-ssse3-back.S | 3182 +++++++++++++++++ utils/memcpy-bench/glibc/memcpy-ssse3.S | 3152 ++++++++++++++++ .../glibc/memmove-avx-unaligned-erms.S | 12 + .../glibc/memmove-avx512-no-vzeroupper.S | 419 +++ .../glibc/memmove-avx512-unaligned-erms.S | 12 + .../glibc/memmove-sse2-unaligned-erms.S | 33 + .../glibc/memmove-vec-unaligned-erms.S | 559 +++ utils/memcpy-bench/glibc/memmove.S | 71 + utils/memcpy-bench/glibc/sysdep.h | 129 + utils/memcpy-bench/glibc/sysdep_generic.h | 113 + utils/memcpy-bench/glibc/sysdep_x86.h | 113 + utils/memcpy-bench/memcpy-bench.cpp | 208 +- 18 files changed, 8585 insertions(+), 59 deletions(-) create mode 100644 utils/memcpy-bench/FastMemcpy.cpp create mode 100644 utils/memcpy-bench/FastMemcpy_Avx.cpp create mode 100644 utils/memcpy-bench/glibc/asm-syntax.h create mode 100644 utils/memcpy-bench/glibc/dwarf2.h create mode 100644 utils/memcpy-bench/glibc/memcpy-ssse3-back.S create mode 100644 utils/memcpy-bench/glibc/memcpy-ssse3.S create mode 100644 utils/memcpy-bench/glibc/memmove-avx-unaligned-erms.S create mode 100644 utils/memcpy-bench/glibc/memmove-avx512-no-vzeroupper.S create mode 100644 utils/memcpy-bench/glibc/memmove-avx512-unaligned-erms.S create mode 100644 utils/memcpy-bench/glibc/memmove-sse2-unaligned-erms.S create mode 100644 utils/memcpy-bench/glibc/memmove-vec-unaligned-erms.S create mode 100644 utils/memcpy-bench/glibc/memmove.S create mode 100644 utils/memcpy-bench/glibc/sysdep.h create mode 100644 utils/memcpy-bench/glibc/sysdep_generic.h create mode 100644 utils/memcpy-bench/glibc/sysdep_x86.h diff --git a/utils/memcpy-bench/CMakeLists.txt b/utils/memcpy-bench/CMakeLists.txt index 54dd0398912..5fcde231688 100644 --- a/utils/memcpy-bench/CMakeLists.txt +++ b/utils/memcpy-bench/CMakeLists.txt @@ -1,5 +1,22 @@ enable_language(ASM) -add_executable (memcpy-bench memcpy-bench.cpp memcpy_jart.S) -#target_compile_options(memcpy-bench PRIVATE -mavx) -target_link_libraries(memcpy-bench PRIVATE dbms) + +add_executable (memcpy-bench + memcpy-bench.cpp + FastMemcpy.cpp + FastMemcpy_Avx.cpp + memcpy_jart.S + glibc/memcpy-ssse3.S + glibc/memcpy-ssse3-back.S + glibc/memmove-sse2-unaligned-erms.S + glibc/memmove-avx-unaligned-erms.S + glibc/memmove-avx512-unaligned-erms.S + glibc/memmove-avx512-no-vzeroupper.S + ) + +add_compile_options(memcpy-bench PRIVATE -fno-tree-loop-distribute-patterns) + +set_source_files_properties(FastMemcpy.cpp PROPERTIES COMPILE_FLAGS "-Wno-old-style-cast") +set_source_files_properties(FastMemcpy_Avx.cpp PROPERTIES COMPILE_FLAGS "-mavx -Wno-old-style-cast -Wno-cast-qual -Wno-cast-align") + +target_link_libraries(memcpy-bench PRIVATE dbms boost::program_options) diff --git a/utils/memcpy-bench/FastMemcpy.cpp b/utils/memcpy-bench/FastMemcpy.cpp new file mode 100644 index 00000000000..9a50caba2b1 --- /dev/null +++ b/utils/memcpy-bench/FastMemcpy.cpp @@ -0,0 +1 @@ +#include "FastMemcpy.h" diff --git a/utils/memcpy-bench/FastMemcpy.h b/utils/memcpy-bench/FastMemcpy.h index 9c37524443a..85d09c5f53e 100644 --- a/utils/memcpy-bench/FastMemcpy.h +++ b/utils/memcpy-bench/FastMemcpy.h @@ -93,7 +93,7 @@ static INLINE void memcpy_sse2_128(void * __restrict dst, const void * __restric /// Attribute is used to avoid an error with undefined behaviour sanitizer /// ../contrib/FastMemcpy/FastMemcpy.h:91:56: runtime error: applying zero offset to null pointer /// Found by 01307_orc_output_format.sh, cause - ORCBlockInputFormat and external ORC library. -__attribute__((__no_sanitize__("undefined"))) static INLINE void *memcpy_tiny(void * __restrict dst, const void * __restrict src, size_t size) +__attribute__((__no_sanitize__("undefined"))) inline void *memcpy_tiny(void * __restrict dst, const void * __restrict src, size_t size) { unsigned char *dd = ((unsigned char*)dst) + size; const unsigned char *ss = ((const unsigned char*)src) + size; diff --git a/utils/memcpy-bench/FastMemcpy_Avx.cpp b/utils/memcpy-bench/FastMemcpy_Avx.cpp new file mode 100644 index 00000000000..8cef0f89507 --- /dev/null +++ b/utils/memcpy-bench/FastMemcpy_Avx.cpp @@ -0,0 +1 @@ +#include "FastMemcpy_Avx.h" diff --git a/utils/memcpy-bench/glibc/asm-syntax.h b/utils/memcpy-bench/glibc/asm-syntax.h new file mode 100644 index 00000000000..6e299c1fec2 --- /dev/null +++ b/utils/memcpy-bench/glibc/asm-syntax.h @@ -0,0 +1,24 @@ +/* Definitions for x86 syntax variations. + Copyright (C) 1992-2020 Free Software Foundation, Inc. + This file is part of the GNU C Library. Its master source is NOT part of + the C library, however. The master source lives in the GNU MP Library. + + The GNU C Library is free software; you can redistribute it and/or + modify it under the terms of the GNU Lesser General Public + License as published by the Free Software Foundation; either + version 2.1 of the License, or (at your option) any later version. + + The GNU C Library is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + Lesser General Public License for more details. + + You should have received a copy of the GNU Lesser General Public + License along with the GNU C Library; if not, see + . */ + +#undef ALIGN +#define ALIGN(log) .align 1<. */ + +#ifndef _DWARF2_H +#define _DWARF2_H 1 + +/* This file is derived from the DWARF specification (a public document) + Revision 2.0.0 (July 27, 1993) developed by the UNIX International + Programming Languages Special Interest Group (UI/PLSIG) and distributed + by UNIX International. Copies of this specification are available from + UNIX International, 20 Waterview Boulevard, Parsippany, NJ, 07054. */ + +/* This file is shared between GCC and GDB, and should not contain + prototypes. */ + +#ifndef __ASSEMBLER__ +/* Tag names and codes. */ + +enum dwarf_tag + { + DW_TAG_padding = 0x00, + DW_TAG_array_type = 0x01, + DW_TAG_class_type = 0x02, + DW_TAG_entry_point = 0x03, + DW_TAG_enumeration_type = 0x04, + DW_TAG_formal_parameter = 0x05, + DW_TAG_imported_declaration = 0x08, + DW_TAG_label = 0x0a, + DW_TAG_lexical_block = 0x0b, + DW_TAG_member = 0x0d, + DW_TAG_pointer_type = 0x0f, + DW_TAG_reference_type = 0x10, + DW_TAG_compile_unit = 0x11, + DW_TAG_string_type = 0x12, + DW_TAG_structure_type = 0x13, + DW_TAG_subroutine_type = 0x15, + DW_TAG_typedef = 0x16, + DW_TAG_union_type = 0x17, + DW_TAG_unspecified_parameters = 0x18, + DW_TAG_variant = 0x19, + DW_TAG_common_block = 0x1a, + DW_TAG_common_inclusion = 0x1b, + DW_TAG_inheritance = 0x1c, + DW_TAG_inlined_subroutine = 0x1d, + DW_TAG_module = 0x1e, + DW_TAG_ptr_to_member_type = 0x1f, + DW_TAG_set_type = 0x20, + DW_TAG_subrange_type = 0x21, + DW_TAG_with_stmt = 0x22, + DW_TAG_access_declaration = 0x23, + DW_TAG_base_type = 0x24, + DW_TAG_catch_block = 0x25, + DW_TAG_const_type = 0x26, + DW_TAG_constant = 0x27, + DW_TAG_enumerator = 0x28, + DW_TAG_file_type = 0x29, + DW_TAG_friend = 0x2a, + DW_TAG_namelist = 0x2b, + DW_TAG_namelist_item = 0x2c, + DW_TAG_packed_type = 0x2d, + DW_TAG_subprogram = 0x2e, + DW_TAG_template_type_param = 0x2f, + DW_TAG_template_value_param = 0x30, + DW_TAG_thrown_type = 0x31, + DW_TAG_try_block = 0x32, + DW_TAG_variant_part = 0x33, + DW_TAG_variable = 0x34, + DW_TAG_volatile_type = 0x35, + /* SGI/MIPS Extensions */ + DW_TAG_MIPS_loop = 0x4081, + /* GNU extensions */ + DW_TAG_format_label = 0x4101, /* for FORTRAN 77 and Fortran 90 */ + DW_TAG_function_template = 0x4102, /* for C++ */ + DW_TAG_class_template = 0x4103, /* for C++ */ + DW_TAG_GNU_BINCL = 0x4104, + DW_TAG_GNU_EINCL = 0x4105 + }; + +#define DW_TAG_lo_user 0x4080 +#define DW_TAG_hi_user 0xffff + +/* flag that tells whether entry has a child or not */ +#define DW_children_no 0 +#define DW_children_yes 1 + +/* Form names and codes. */ +enum dwarf_form + { + DW_FORM_addr = 0x01, + DW_FORM_block2 = 0x03, + DW_FORM_block4 = 0x04, + DW_FORM_data2 = 0x05, + DW_FORM_data4 = 0x06, + DW_FORM_data8 = 0x07, + DW_FORM_string = 0x08, + DW_FORM_block = 0x09, + DW_FORM_block1 = 0x0a, + DW_FORM_data1 = 0x0b, + DW_FORM_flag = 0x0c, + DW_FORM_sdata = 0x0d, + DW_FORM_strp = 0x0e, + DW_FORM_udata = 0x0f, + DW_FORM_ref_addr = 0x10, + DW_FORM_ref1 = 0x11, + DW_FORM_ref2 = 0x12, + DW_FORM_ref4 = 0x13, + DW_FORM_ref8 = 0x14, + DW_FORM_ref_udata = 0x15, + DW_FORM_indirect = 0x16 + }; + +/* Attribute names and codes. */ + +enum dwarf_attribute + { + DW_AT_sibling = 0x01, + DW_AT_location = 0x02, + DW_AT_name = 0x03, + DW_AT_ordering = 0x09, + DW_AT_subscr_data = 0x0a, + DW_AT_byte_size = 0x0b, + DW_AT_bit_offset = 0x0c, + DW_AT_bit_size = 0x0d, + DW_AT_element_list = 0x0f, + DW_AT_stmt_list = 0x10, + DW_AT_low_pc = 0x11, + DW_AT_high_pc = 0x12, + DW_AT_language = 0x13, + DW_AT_member = 0x14, + DW_AT_discr = 0x15, + DW_AT_discr_value = 0x16, + DW_AT_visibility = 0x17, + DW_AT_import = 0x18, + DW_AT_string_length = 0x19, + DW_AT_common_reference = 0x1a, + DW_AT_comp_dir = 0x1b, + DW_AT_const_value = 0x1c, + DW_AT_containing_type = 0x1d, + DW_AT_default_value = 0x1e, + DW_AT_inline = 0x20, + DW_AT_is_optional = 0x21, + DW_AT_lower_bound = 0x22, + DW_AT_producer = 0x25, + DW_AT_prototyped = 0x27, + DW_AT_return_addr = 0x2a, + DW_AT_start_scope = 0x2c, + DW_AT_stride_size = 0x2e, + DW_AT_upper_bound = 0x2f, + DW_AT_abstract_origin = 0x31, + DW_AT_accessibility = 0x32, + DW_AT_address_class = 0x33, + DW_AT_artificial = 0x34, + DW_AT_base_types = 0x35, + DW_AT_calling_convention = 0x36, + DW_AT_count = 0x37, + DW_AT_data_member_location = 0x38, + DW_AT_decl_column = 0x39, + DW_AT_decl_file = 0x3a, + DW_AT_decl_line = 0x3b, + DW_AT_declaration = 0x3c, + DW_AT_discr_list = 0x3d, + DW_AT_encoding = 0x3e, + DW_AT_external = 0x3f, + DW_AT_frame_base = 0x40, + DW_AT_friend = 0x41, + DW_AT_identifier_case = 0x42, + DW_AT_macro_info = 0x43, + DW_AT_namelist_items = 0x44, + DW_AT_priority = 0x45, + DW_AT_segment = 0x46, + DW_AT_specification = 0x47, + DW_AT_static_link = 0x48, + DW_AT_type = 0x49, + DW_AT_use_location = 0x4a, + DW_AT_variable_parameter = 0x4b, + DW_AT_virtuality = 0x4c, + DW_AT_vtable_elem_location = 0x4d, + /* SGI/MIPS Extensions */ + DW_AT_MIPS_fde = 0x2001, + DW_AT_MIPS_loop_begin = 0x2002, + DW_AT_MIPS_tail_loop_begin = 0x2003, + DW_AT_MIPS_epilog_begin = 0x2004, + DW_AT_MIPS_loop_unroll_factor = 0x2005, + DW_AT_MIPS_software_pipeline_depth = 0x2006, + DW_AT_MIPS_linkage_name = 0x2007, + DW_AT_MIPS_stride = 0x2008, + DW_AT_MIPS_abstract_name = 0x2009, + DW_AT_MIPS_clone_origin = 0x200a, + DW_AT_MIPS_has_inlines = 0x200b, + /* GNU extensions. */ + DW_AT_sf_names = 0x2101, + DW_AT_src_info = 0x2102, + DW_AT_mac_info = 0x2103, + DW_AT_src_coords = 0x2104, + DW_AT_body_begin = 0x2105, + DW_AT_body_end = 0x2106 + }; + +#define DW_AT_lo_user 0x2000 /* implementation-defined range start */ +#define DW_AT_hi_user 0x3ff0 /* implementation-defined range end */ + +/* Location atom names and codes. */ + +enum dwarf_location_atom + { + DW_OP_addr = 0x03, + DW_OP_deref = 0x06, + DW_OP_const1u = 0x08, + DW_OP_const1s = 0x09, + DW_OP_const2u = 0x0a, + DW_OP_const2s = 0x0b, + DW_OP_const4u = 0x0c, + DW_OP_const4s = 0x0d, + DW_OP_const8u = 0x0e, + DW_OP_const8s = 0x0f, + DW_OP_constu = 0x10, + DW_OP_consts = 0x11, + DW_OP_dup = 0x12, + DW_OP_drop = 0x13, + DW_OP_over = 0x14, + DW_OP_pick = 0x15, + DW_OP_swap = 0x16, + DW_OP_rot = 0x17, + DW_OP_xderef = 0x18, + DW_OP_abs = 0x19, + DW_OP_and = 0x1a, + DW_OP_div = 0x1b, + DW_OP_minus = 0x1c, + DW_OP_mod = 0x1d, + DW_OP_mul = 0x1e, + DW_OP_neg = 0x1f, + DW_OP_not = 0x20, + DW_OP_or = 0x21, + DW_OP_plus = 0x22, + DW_OP_plus_uconst = 0x23, + DW_OP_shl = 0x24, + DW_OP_shr = 0x25, + DW_OP_shra = 0x26, + DW_OP_xor = 0x27, + DW_OP_bra = 0x28, + DW_OP_eq = 0x29, + DW_OP_ge = 0x2a, + DW_OP_gt = 0x2b, + DW_OP_le = 0x2c, + DW_OP_lt = 0x2d, + DW_OP_ne = 0x2e, + DW_OP_skip = 0x2f, + DW_OP_lit0 = 0x30, + DW_OP_lit1 = 0x31, + DW_OP_lit2 = 0x32, + DW_OP_lit3 = 0x33, + DW_OP_lit4 = 0x34, + DW_OP_lit5 = 0x35, + DW_OP_lit6 = 0x36, + DW_OP_lit7 = 0x37, + DW_OP_lit8 = 0x38, + DW_OP_lit9 = 0x39, + DW_OP_lit10 = 0x3a, + DW_OP_lit11 = 0x3b, + DW_OP_lit12 = 0x3c, + DW_OP_lit13 = 0x3d, + DW_OP_lit14 = 0x3e, + DW_OP_lit15 = 0x3f, + DW_OP_lit16 = 0x40, + DW_OP_lit17 = 0x41, + DW_OP_lit18 = 0x42, + DW_OP_lit19 = 0x43, + DW_OP_lit20 = 0x44, + DW_OP_lit21 = 0x45, + DW_OP_lit22 = 0x46, + DW_OP_lit23 = 0x47, + DW_OP_lit24 = 0x48, + DW_OP_lit25 = 0x49, + DW_OP_lit26 = 0x4a, + DW_OP_lit27 = 0x4b, + DW_OP_lit28 = 0x4c, + DW_OP_lit29 = 0x4d, + DW_OP_lit30 = 0x4e, + DW_OP_lit31 = 0x4f, + DW_OP_reg0 = 0x50, + DW_OP_reg1 = 0x51, + DW_OP_reg2 = 0x52, + DW_OP_reg3 = 0x53, + DW_OP_reg4 = 0x54, + DW_OP_reg5 = 0x55, + DW_OP_reg6 = 0x56, + DW_OP_reg7 = 0x57, + DW_OP_reg8 = 0x58, + DW_OP_reg9 = 0x59, + DW_OP_reg10 = 0x5a, + DW_OP_reg11 = 0x5b, + DW_OP_reg12 = 0x5c, + DW_OP_reg13 = 0x5d, + DW_OP_reg14 = 0x5e, + DW_OP_reg15 = 0x5f, + DW_OP_reg16 = 0x60, + DW_OP_reg17 = 0x61, + DW_OP_reg18 = 0x62, + DW_OP_reg19 = 0x63, + DW_OP_reg20 = 0x64, + DW_OP_reg21 = 0x65, + DW_OP_reg22 = 0x66, + DW_OP_reg23 = 0x67, + DW_OP_reg24 = 0x68, + DW_OP_reg25 = 0x69, + DW_OP_reg26 = 0x6a, + DW_OP_reg27 = 0x6b, + DW_OP_reg28 = 0x6c, + DW_OP_reg29 = 0x6d, + DW_OP_reg30 = 0x6e, + DW_OP_reg31 = 0x6f, + DW_OP_breg0 = 0x70, + DW_OP_breg1 = 0x71, + DW_OP_breg2 = 0x72, + DW_OP_breg3 = 0x73, + DW_OP_breg4 = 0x74, + DW_OP_breg5 = 0x75, + DW_OP_breg6 = 0x76, + DW_OP_breg7 = 0x77, + DW_OP_breg8 = 0x78, + DW_OP_breg9 = 0x79, + DW_OP_breg10 = 0x7a, + DW_OP_breg11 = 0x7b, + DW_OP_breg12 = 0x7c, + DW_OP_breg13 = 0x7d, + DW_OP_breg14 = 0x7e, + DW_OP_breg15 = 0x7f, + DW_OP_breg16 = 0x80, + DW_OP_breg17 = 0x81, + DW_OP_breg18 = 0x82, + DW_OP_breg19 = 0x83, + DW_OP_breg20 = 0x84, + DW_OP_breg21 = 0x85, + DW_OP_breg22 = 0x86, + DW_OP_breg23 = 0x87, + DW_OP_breg24 = 0x88, + DW_OP_breg25 = 0x89, + DW_OP_breg26 = 0x8a, + DW_OP_breg27 = 0x8b, + DW_OP_breg28 = 0x8c, + DW_OP_breg29 = 0x8d, + DW_OP_breg30 = 0x8e, + DW_OP_breg31 = 0x8f, + DW_OP_regx = 0x90, + DW_OP_fbreg = 0x91, + DW_OP_bregx = 0x92, + DW_OP_piece = 0x93, + DW_OP_deref_size = 0x94, + DW_OP_xderef_size = 0x95, + DW_OP_nop = 0x96 + }; + +#define DW_OP_lo_user 0x80 /* implementation-defined range start */ +#define DW_OP_hi_user 0xff /* implementation-defined range end */ + +/* Type encodings. */ + +enum dwarf_type + { + DW_ATE_void = 0x0, + DW_ATE_address = 0x1, + DW_ATE_boolean = 0x2, + DW_ATE_complex_float = 0x3, + DW_ATE_float = 0x4, + DW_ATE_signed = 0x5, + DW_ATE_signed_char = 0x6, + DW_ATE_unsigned = 0x7, + DW_ATE_unsigned_char = 0x8 + }; + +#define DW_ATE_lo_user 0x80 +#define DW_ATE_hi_user 0xff + +/* Array ordering names and codes. */ +enum dwarf_array_dim_ordering + { + DW_ORD_row_major = 0, + DW_ORD_col_major = 1 + }; + +/* access attribute */ +enum dwarf_access_attribute + { + DW_ACCESS_public = 1, + DW_ACCESS_protected = 2, + DW_ACCESS_private = 3 + }; + +/* visibility */ +enum dwarf_visibility_attribute + { + DW_VIS_local = 1, + DW_VIS_exported = 2, + DW_VIS_qualified = 3 + }; + +/* virtuality */ +enum dwarf_virtuality_attribute + { + DW_VIRTUALITY_none = 0, + DW_VIRTUALITY_virtual = 1, + DW_VIRTUALITY_pure_virtual = 2 + }; + +/* case sensitivity */ +enum dwarf_id_case + { + DW_ID_case_sensitive = 0, + DW_ID_up_case = 1, + DW_ID_down_case = 2, + DW_ID_case_insensitive = 3 + }; + +/* calling convention */ +enum dwarf_calling_convention + { + DW_CC_normal = 0x1, + DW_CC_program = 0x2, + DW_CC_nocall = 0x3 + }; + +#define DW_CC_lo_user 0x40 +#define DW_CC_hi_user 0xff + +/* inline attribute */ +enum dwarf_inline_attribute + { + DW_INL_not_inlined = 0, + DW_INL_inlined = 1, + DW_INL_declared_not_inlined = 2, + DW_INL_declared_inlined = 3 + }; + +/* discriminant lists */ +enum dwarf_discrim_list + { + DW_DSC_label = 0, + DW_DSC_range = 1 + }; + +/* line number opcodes */ +enum dwarf_line_number_ops + { + DW_LNS_extended_op = 0, + DW_LNS_copy = 1, + DW_LNS_advance_pc = 2, + DW_LNS_advance_line = 3, + DW_LNS_set_file = 4, + DW_LNS_set_column = 5, + DW_LNS_negate_stmt = 6, + DW_LNS_set_basic_block = 7, + DW_LNS_const_add_pc = 8, + DW_LNS_fixed_advance_pc = 9 + }; + +/* line number extended opcodes */ +enum dwarf_line_number_x_ops + { + DW_LNE_end_sequence = 1, + DW_LNE_set_address = 2, + DW_LNE_define_file = 3 + }; + +/* call frame information */ +enum dwarf_call_frame_info + { + DW_CFA_advance_loc = 0x40, + DW_CFA_offset = 0x80, + DW_CFA_restore = 0xc0, + DW_CFA_nop = 0x00, + DW_CFA_set_loc = 0x01, + DW_CFA_advance_loc1 = 0x02, + DW_CFA_advance_loc2 = 0x03, + DW_CFA_advance_loc4 = 0x04, + DW_CFA_offset_extended = 0x05, + DW_CFA_restore_extended = 0x06, + DW_CFA_undefined = 0x07, + DW_CFA_same_value = 0x08, + DW_CFA_register = 0x09, + DW_CFA_remember_state = 0x0a, + DW_CFA_restore_state = 0x0b, + DW_CFA_def_cfa = 0x0c, + DW_CFA_def_cfa_register = 0x0d, + DW_CFA_def_cfa_offset = 0x0e, + DW_CFA_def_cfa_expression = 0x0f, + DW_CFA_expression = 0x10, + /* Dwarf 2.1 */ + DW_CFA_offset_extended_sf = 0x11, + DW_CFA_def_cfa_sf = 0x12, + DW_CFA_def_cfa_offset_sf = 0x13, + + /* SGI/MIPS specific */ + DW_CFA_MIPS_advance_loc8 = 0x1d, + + /* GNU extensions */ + DW_CFA_GNU_window_save = 0x2d, + DW_CFA_GNU_args_size = 0x2e, + DW_CFA_GNU_negative_offset_extended = 0x2f + }; + +#define DW_CIE_ID 0xffffffff +#define DW_CIE_VERSION 1 + +#define DW_CFA_extended 0 +#define DW_CFA_low_user 0x1c +#define DW_CFA_high_user 0x3f + +#define DW_CHILDREN_no 0x00 +#define DW_CHILDREN_yes 0x01 + +#define DW_ADDR_none 0 + +/* Source language names and codes. */ + +enum dwarf_source_language + { + DW_LANG_C89 = 0x0001, + DW_LANG_C = 0x0002, + DW_LANG_Ada83 = 0x0003, + DW_LANG_C_plus_plus = 0x0004, + DW_LANG_Cobol74 = 0x0005, + DW_LANG_Cobol85 = 0x0006, + DW_LANG_Fortran77 = 0x0007, + DW_LANG_Fortran90 = 0x0008, + DW_LANG_Pascal83 = 0x0009, + DW_LANG_Modula2 = 0x000a, + DW_LANG_Java = 0x000b, + DW_LANG_Mips_Assembler = 0x8001 + }; + + +#define DW_LANG_lo_user 0x8000 /* implementation-defined range start */ +#define DW_LANG_hi_user 0xffff /* implementation-defined range start */ + +/* Names and codes for macro information. */ + +enum dwarf_macinfo_record_type + { + DW_MACINFO_define = 1, + DW_MACINFO_undef = 2, + DW_MACINFO_start_file = 3, + DW_MACINFO_end_file = 4, + DW_MACINFO_vendor_ext = 255 + }; + +#endif /* !ASSEMBLER */ + +/* @@@ For use with GNU frame unwind information. */ + +#define DW_EH_PE_absptr 0x00 +#define DW_EH_PE_omit 0xff + +#define DW_EH_PE_uleb128 0x01 +#define DW_EH_PE_udata2 0x02 +#define DW_EH_PE_udata4 0x03 +#define DW_EH_PE_udata8 0x04 +#define DW_EH_PE_sleb128 0x09 +#define DW_EH_PE_sdata2 0x0A +#define DW_EH_PE_sdata4 0x0B +#define DW_EH_PE_sdata8 0x0C +#define DW_EH_PE_signed 0x08 + +#define DW_EH_PE_pcrel 0x10 +#define DW_EH_PE_textrel 0x20 +#define DW_EH_PE_datarel 0x30 +#define DW_EH_PE_funcrel 0x40 +#define DW_EH_PE_aligned 0x50 + +#define DW_EH_PE_indirect 0x80 + +#endif /* dwarf2.h */ diff --git a/utils/memcpy-bench/glibc/memcpy-ssse3-back.S b/utils/memcpy-bench/glibc/memcpy-ssse3-back.S new file mode 100644 index 00000000000..1492dd38e73 --- /dev/null +++ b/utils/memcpy-bench/glibc/memcpy-ssse3-back.S @@ -0,0 +1,3182 @@ +/* memcpy with SSSE3 and REP string + Copyright (C) 2010-2020 Free Software Foundation, Inc. + Contributed by Intel Corporation. + This file is part of the GNU C Library. + + The GNU C Library is free software; you can redistribute it and/or + modify it under the terms of the GNU Lesser General Public + License as published by the Free Software Foundation; either + version 2.1 of the License, or (at your option) any later version. + + The GNU C Library is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + Lesser General Public License for more details. + + You should have received a copy of the GNU Lesser General Public + License along with the GNU C Library; if not, see + . */ + +#include "sysdep.h" + +#if 1 + +#include "asm-syntax.h" + +#ifndef MEMCPY +# define MEMCPY __memcpy_ssse3_back +# define MEMCPY_CHK __memcpy_chk_ssse3_back +# define MEMPCPY __mempcpy_ssse3_back +# define MEMPCPY_CHK __mempcpy_chk_ssse3_back +#endif + +#define JMPTBL(I, B) I - B + +/* Branch to an entry in a jump table. TABLE is a jump table with + relative offsets. INDEX is a register contains the index into the + jump table. SCALE is the scale of INDEX. */ +#define BRANCH_TO_JMPTBL_ENTRY(TABLE, INDEX, SCALE) \ + lea TABLE(%rip), %r11; \ + movslq (%r11, INDEX, SCALE), INDEX; \ + lea (%r11, INDEX), INDEX; \ + _CET_NOTRACK jmp *INDEX; \ + ud2 + + .section .text.ssse3,"ax",@progbits +#if !defined USE_AS_MEMPCPY && !defined USE_AS_MEMMOVE +ENTRY (MEMPCPY_CHK) + cmp %RDX_LP, %RCX_LP + jb HIDDEN_JUMPTARGET (__chk_fail) +END (MEMPCPY_CHK) + +ENTRY (MEMPCPY) + mov %RDI_LP, %RAX_LP + add %RDX_LP, %RAX_LP + jmp L(start) +END (MEMPCPY) +#endif + +#if !defined USE_AS_BCOPY +ENTRY (MEMCPY_CHK) + cmp %RDX_LP, %RCX_LP + jb HIDDEN_JUMPTARGET (__chk_fail) +END (MEMCPY_CHK) +#endif + +ENTRY (MEMCPY) + mov %RDI_LP, %RAX_LP +#ifdef USE_AS_MEMPCPY + add %RDX_LP, %RAX_LP +#endif + +#ifdef __ILP32__ + /* Clear the upper 32 bits. */ + mov %edx, %edx +#endif + +#ifdef USE_AS_MEMMOVE + cmp %rsi, %rdi + jb L(copy_forward) + je L(bwd_write_0bytes) + cmp $144, %rdx + jae L(copy_backward) + BRANCH_TO_JMPTBL_ENTRY (L(table_144_bytes_bwd), %rdx, 4) +L(copy_forward): +#endif +L(start): + cmp $144, %rdx + jae L(144bytesormore) + +L(fwd_write_less32bytes): +#ifndef USE_AS_MEMMOVE + cmp %dil, %sil + jbe L(bk_write) +#endif + add %rdx, %rsi + add %rdx, %rdi + BRANCH_TO_JMPTBL_ENTRY (L(table_144_bytes_fwd), %rdx, 4) +#ifndef USE_AS_MEMMOVE +L(bk_write): + + BRANCH_TO_JMPTBL_ENTRY (L(table_144_bytes_bwd), %rdx, 4) +#endif + + .p2align 4 +L(144bytesormore): + +#ifndef USE_AS_MEMMOVE + cmp %dil, %sil + jle L(copy_backward) +#endif + movdqu (%rsi), %xmm0 + mov %rdi, %r8 + and $-16, %rdi + add $16, %rdi + mov %rdi, %r9 + sub %r8, %r9 + sub %r9, %rdx + add %r9, %rsi + mov %rsi, %r9 + and $0xf, %r9 + jz L(shl_0) +#ifdef DATA_CACHE_SIZE + mov $DATA_CACHE_SIZE, %RCX_LP +#else + mov __x86_data_cache_size(%rip), %RCX_LP +#endif + cmp %rcx, %rdx + jae L(gobble_mem_fwd) + lea L(shl_table_fwd)(%rip), %r11 + sub $0x80, %rdx + movslq (%r11, %r9, 4), %r9 + add %r11, %r9 + _CET_NOTRACK jmp *%r9 + ud2 + + .p2align 4 +L(copy_backward): +#ifdef DATA_CACHE_SIZE + mov $DATA_CACHE_SIZE, %RCX_LP +#else + mov __x86_data_cache_size(%rip), %RCX_LP +#endif + shl $1, %rcx + cmp %rcx, %rdx + ja L(gobble_mem_bwd) + + add %rdx, %rdi + add %rdx, %rsi + movdqu -16(%rsi), %xmm0 + lea -16(%rdi), %r8 + mov %rdi, %r9 + and $0xf, %r9 + xor %r9, %rdi + sub %r9, %rsi + sub %r9, %rdx + mov %rsi, %r9 + and $0xf, %r9 + jz L(shl_0_bwd) + lea L(shl_table_bwd)(%rip), %r11 + sub $0x80, %rdx + movslq (%r11, %r9, 4), %r9 + add %r11, %r9 + _CET_NOTRACK jmp *%r9 + ud2 + + .p2align 4 +L(shl_0): + + mov %rdx, %r9 + shr $8, %r9 + add %rdx, %r9 +#ifdef DATA_CACHE_SIZE + cmp $DATA_CACHE_SIZE_HALF, %R9_LP +#else + cmp __x86_data_cache_size_half(%rip), %R9_LP +#endif + jae L(gobble_mem_fwd) + sub $0x80, %rdx + .p2align 4 +L(shl_0_loop): + movdqa (%rsi), %xmm1 + movdqa %xmm1, (%rdi) + movaps 0x10(%rsi), %xmm2 + movaps %xmm2, 0x10(%rdi) + movaps 0x20(%rsi), %xmm3 + movaps %xmm3, 0x20(%rdi) + movaps 0x30(%rsi), %xmm4 + movaps %xmm4, 0x30(%rdi) + movaps 0x40(%rsi), %xmm1 + movaps %xmm1, 0x40(%rdi) + movaps 0x50(%rsi), %xmm2 + movaps %xmm2, 0x50(%rdi) + movaps 0x60(%rsi), %xmm3 + movaps %xmm3, 0x60(%rdi) + movaps 0x70(%rsi), %xmm4 + movaps %xmm4, 0x70(%rdi) + sub $0x80, %rdx + lea 0x80(%rsi), %rsi + lea 0x80(%rdi), %rdi + jae L(shl_0_loop) + movdqu %xmm0, (%r8) + add $0x80, %rdx + add %rdx, %rsi + add %rdx, %rdi + BRANCH_TO_JMPTBL_ENTRY (L(table_144_bytes_fwd), %rdx, 4) + + .p2align 4 +L(shl_0_bwd): + sub $0x80, %rdx +L(copy_backward_loop): + movaps -0x10(%rsi), %xmm1 + movaps %xmm1, -0x10(%rdi) + movaps -0x20(%rsi), %xmm2 + movaps %xmm2, -0x20(%rdi) + movaps -0x30(%rsi), %xmm3 + movaps %xmm3, -0x30(%rdi) + movaps -0x40(%rsi), %xmm4 + movaps %xmm4, -0x40(%rdi) + movaps -0x50(%rsi), %xmm5 + movaps %xmm5, -0x50(%rdi) + movaps -0x60(%rsi), %xmm5 + movaps %xmm5, -0x60(%rdi) + movaps -0x70(%rsi), %xmm5 + movaps %xmm5, -0x70(%rdi) + movaps -0x80(%rsi), %xmm5 + movaps %xmm5, -0x80(%rdi) + sub $0x80, %rdx + lea -0x80(%rdi), %rdi + lea -0x80(%rsi), %rsi + jae L(copy_backward_loop) + + movdqu %xmm0, (%r8) + add $0x80, %rdx + sub %rdx, %rdi + sub %rdx, %rsi + BRANCH_TO_JMPTBL_ENTRY (L(table_144_bytes_bwd), %rdx, 4) + + .p2align 4 +L(shl_1): + sub $0x80, %rdx + movaps -0x01(%rsi), %xmm1 + movaps 0x0f(%rsi), %xmm2 + movaps 0x1f(%rsi), %xmm3 + movaps 0x2f(%rsi), %xmm4 + movaps 0x3f(%rsi), %xmm5 + movaps 0x4f(%rsi), %xmm6 + movaps 0x5f(%rsi), %xmm7 + movaps 0x6f(%rsi), %xmm8 + movaps 0x7f(%rsi), %xmm9 + lea 0x80(%rsi), %rsi + palignr $1, %xmm8, %xmm9 + movaps %xmm9, 0x70(%rdi) + palignr $1, %xmm7, %xmm8 + movaps %xmm8, 0x60(%rdi) + palignr $1, %xmm6, %xmm7 + movaps %xmm7, 0x50(%rdi) + palignr $1, %xmm5, %xmm6 + movaps %xmm6, 0x40(%rdi) + palignr $1, %xmm4, %xmm5 + movaps %xmm5, 0x30(%rdi) + palignr $1, %xmm3, %xmm4 + movaps %xmm4, 0x20(%rdi) + palignr $1, %xmm2, %xmm3 + movaps %xmm3, 0x10(%rdi) + palignr $1, %xmm1, %xmm2 + movaps %xmm2, (%rdi) + lea 0x80(%rdi), %rdi + jae L(shl_1) + movdqu %xmm0, (%r8) + add $0x80, %rdx + add %rdx, %rdi + add %rdx, %rsi + BRANCH_TO_JMPTBL_ENTRY (L(table_144_bytes_fwd), %rdx, 4) + + .p2align 4 +L(shl_1_bwd): + movaps -0x01(%rsi), %xmm1 + + movaps -0x11(%rsi), %xmm2 + palignr $1, %xmm2, %xmm1 + movaps %xmm1, -0x10(%rdi) + + movaps -0x21(%rsi), %xmm3 + palignr $1, %xmm3, %xmm2 + movaps %xmm2, -0x20(%rdi) + + movaps -0x31(%rsi), %xmm4 + palignr $1, %xmm4, %xmm3 + movaps %xmm3, -0x30(%rdi) + + movaps -0x41(%rsi), %xmm5 + palignr $1, %xmm5, %xmm4 + movaps %xmm4, -0x40(%rdi) + + movaps -0x51(%rsi), %xmm6 + palignr $1, %xmm6, %xmm5 + movaps %xmm5, -0x50(%rdi) + + movaps -0x61(%rsi), %xmm7 + palignr $1, %xmm7, %xmm6 + movaps %xmm6, -0x60(%rdi) + + movaps -0x71(%rsi), %xmm8 + palignr $1, %xmm8, %xmm7 + movaps %xmm7, -0x70(%rdi) + + movaps -0x81(%rsi), %xmm9 + palignr $1, %xmm9, %xmm8 + movaps %xmm8, -0x80(%rdi) + + sub $0x80, %rdx + lea -0x80(%rdi), %rdi + lea -0x80(%rsi), %rsi + jae L(shl_1_bwd) + movdqu %xmm0, (%r8) + add $0x80, %rdx + sub %rdx, %rdi + sub %rdx, %rsi + BRANCH_TO_JMPTBL_ENTRY (L(table_144_bytes_bwd), %rdx, 4) + + .p2align 4 +L(shl_2): + sub $0x80, %rdx + movaps -0x02(%rsi), %xmm1 + movaps 0x0e(%rsi), %xmm2 + movaps 0x1e(%rsi), %xmm3 + movaps 0x2e(%rsi), %xmm4 + movaps 0x3e(%rsi), %xmm5 + movaps 0x4e(%rsi), %xmm6 + movaps 0x5e(%rsi), %xmm7 + movaps 0x6e(%rsi), %xmm8 + movaps 0x7e(%rsi), %xmm9 + lea 0x80(%rsi), %rsi + palignr $2, %xmm8, %xmm9 + movaps %xmm9, 0x70(%rdi) + palignr $2, %xmm7, %xmm8 + movaps %xmm8, 0x60(%rdi) + palignr $2, %xmm6, %xmm7 + movaps %xmm7, 0x50(%rdi) + palignr $2, %xmm5, %xmm6 + movaps %xmm6, 0x40(%rdi) + palignr $2, %xmm4, %xmm5 + movaps %xmm5, 0x30(%rdi) + palignr $2, %xmm3, %xmm4 + movaps %xmm4, 0x20(%rdi) + palignr $2, %xmm2, %xmm3 + movaps %xmm3, 0x10(%rdi) + palignr $2, %xmm1, %xmm2 + movaps %xmm2, (%rdi) + lea 0x80(%rdi), %rdi + jae L(shl_2) + movdqu %xmm0, (%r8) + add $0x80, %rdx + add %rdx, %rdi + add %rdx, %rsi + BRANCH_TO_JMPTBL_ENTRY (L(table_144_bytes_fwd), %rdx, 4) + + .p2align 4 +L(shl_2_bwd): + movaps -0x02(%rsi), %xmm1 + + movaps -0x12(%rsi), %xmm2 + palignr $2, %xmm2, %xmm1 + movaps %xmm1, -0x10(%rdi) + + movaps -0x22(%rsi), %xmm3 + palignr $2, %xmm3, %xmm2 + movaps %xmm2, -0x20(%rdi) + + movaps -0x32(%rsi), %xmm4 + palignr $2, %xmm4, %xmm3 + movaps %xmm3, -0x30(%rdi) + + movaps -0x42(%rsi), %xmm5 + palignr $2, %xmm5, %xmm4 + movaps %xmm4, -0x40(%rdi) + + movaps -0x52(%rsi), %xmm6 + palignr $2, %xmm6, %xmm5 + movaps %xmm5, -0x50(%rdi) + + movaps -0x62(%rsi), %xmm7 + palignr $2, %xmm7, %xmm6 + movaps %xmm6, -0x60(%rdi) + + movaps -0x72(%rsi), %xmm8 + palignr $2, %xmm8, %xmm7 + movaps %xmm7, -0x70(%rdi) + + movaps -0x82(%rsi), %xmm9 + palignr $2, %xmm9, %xmm8 + movaps %xmm8, -0x80(%rdi) + + sub $0x80, %rdx + lea -0x80(%rdi), %rdi + lea -0x80(%rsi), %rsi + jae L(shl_2_bwd) + movdqu %xmm0, (%r8) + add $0x80, %rdx + sub %rdx, %rdi + sub %rdx, %rsi + BRANCH_TO_JMPTBL_ENTRY (L(table_144_bytes_bwd), %rdx, 4) + + .p2align 4 +L(shl_3): + sub $0x80, %rdx + movaps -0x03(%rsi), %xmm1 + movaps 0x0d(%rsi), %xmm2 + movaps 0x1d(%rsi), %xmm3 + movaps 0x2d(%rsi), %xmm4 + movaps 0x3d(%rsi), %xmm5 + movaps 0x4d(%rsi), %xmm6 + movaps 0x5d(%rsi), %xmm7 + movaps 0x6d(%rsi), %xmm8 + movaps 0x7d(%rsi), %xmm9 + lea 0x80(%rsi), %rsi + palignr $3, %xmm8, %xmm9 + movaps %xmm9, 0x70(%rdi) + palignr $3, %xmm7, %xmm8 + movaps %xmm8, 0x60(%rdi) + palignr $3, %xmm6, %xmm7 + movaps %xmm7, 0x50(%rdi) + palignr $3, %xmm5, %xmm6 + movaps %xmm6, 0x40(%rdi) + palignr $3, %xmm4, %xmm5 + movaps %xmm5, 0x30(%rdi) + palignr $3, %xmm3, %xmm4 + movaps %xmm4, 0x20(%rdi) + palignr $3, %xmm2, %xmm3 + movaps %xmm3, 0x10(%rdi) + palignr $3, %xmm1, %xmm2 + movaps %xmm2, (%rdi) + lea 0x80(%rdi), %rdi + jae L(shl_3) + movdqu %xmm0, (%r8) + add $0x80, %rdx + add %rdx, %rdi + add %rdx, %rsi + BRANCH_TO_JMPTBL_ENTRY (L(table_144_bytes_fwd), %rdx, 4) + + .p2align 4 +L(shl_3_bwd): + movaps -0x03(%rsi), %xmm1 + + movaps -0x13(%rsi), %xmm2 + palignr $3, %xmm2, %xmm1 + movaps %xmm1, -0x10(%rdi) + + movaps -0x23(%rsi), %xmm3 + palignr $3, %xmm3, %xmm2 + movaps %xmm2, -0x20(%rdi) + + movaps -0x33(%rsi), %xmm4 + palignr $3, %xmm4, %xmm3 + movaps %xmm3, -0x30(%rdi) + + movaps -0x43(%rsi), %xmm5 + palignr $3, %xmm5, %xmm4 + movaps %xmm4, -0x40(%rdi) + + movaps -0x53(%rsi), %xmm6 + palignr $3, %xmm6, %xmm5 + movaps %xmm5, -0x50(%rdi) + + movaps -0x63(%rsi), %xmm7 + palignr $3, %xmm7, %xmm6 + movaps %xmm6, -0x60(%rdi) + + movaps -0x73(%rsi), %xmm8 + palignr $3, %xmm8, %xmm7 + movaps %xmm7, -0x70(%rdi) + + movaps -0x83(%rsi), %xmm9 + palignr $3, %xmm9, %xmm8 + movaps %xmm8, -0x80(%rdi) + + sub $0x80, %rdx + lea -0x80(%rdi), %rdi + lea -0x80(%rsi), %rsi + jae L(shl_3_bwd) + movdqu %xmm0, (%r8) + add $0x80, %rdx + sub %rdx, %rdi + sub %rdx, %rsi + BRANCH_TO_JMPTBL_ENTRY (L(table_144_bytes_bwd), %rdx, 4) + + .p2align 4 +L(shl_4): + sub $0x80, %rdx + movaps -0x04(%rsi), %xmm1 + movaps 0x0c(%rsi), %xmm2 + movaps 0x1c(%rsi), %xmm3 + movaps 0x2c(%rsi), %xmm4 + movaps 0x3c(%rsi), %xmm5 + movaps 0x4c(%rsi), %xmm6 + movaps 0x5c(%rsi), %xmm7 + movaps 0x6c(%rsi), %xmm8 + movaps 0x7c(%rsi), %xmm9 + lea 0x80(%rsi), %rsi + palignr $4, %xmm8, %xmm9 + movaps %xmm9, 0x70(%rdi) + palignr $4, %xmm7, %xmm8 + movaps %xmm8, 0x60(%rdi) + palignr $4, %xmm6, %xmm7 + movaps %xmm7, 0x50(%rdi) + palignr $4, %xmm5, %xmm6 + movaps %xmm6, 0x40(%rdi) + palignr $4, %xmm4, %xmm5 + movaps %xmm5, 0x30(%rdi) + palignr $4, %xmm3, %xmm4 + movaps %xmm4, 0x20(%rdi) + palignr $4, %xmm2, %xmm3 + movaps %xmm3, 0x10(%rdi) + palignr $4, %xmm1, %xmm2 + movaps %xmm2, (%rdi) + lea 0x80(%rdi), %rdi + jae L(shl_4) + movdqu %xmm0, (%r8) + add $0x80, %rdx + add %rdx, %rdi + add %rdx, %rsi + BRANCH_TO_JMPTBL_ENTRY (L(table_144_bytes_fwd), %rdx, 4) + + .p2align 4 +L(shl_4_bwd): + movaps -0x04(%rsi), %xmm1 + + movaps -0x14(%rsi), %xmm2 + palignr $4, %xmm2, %xmm1 + movaps %xmm1, -0x10(%rdi) + + movaps -0x24(%rsi), %xmm3 + palignr $4, %xmm3, %xmm2 + movaps %xmm2, -0x20(%rdi) + + movaps -0x34(%rsi), %xmm4 + palignr $4, %xmm4, %xmm3 + movaps %xmm3, -0x30(%rdi) + + movaps -0x44(%rsi), %xmm5 + palignr $4, %xmm5, %xmm4 + movaps %xmm4, -0x40(%rdi) + + movaps -0x54(%rsi), %xmm6 + palignr $4, %xmm6, %xmm5 + movaps %xmm5, -0x50(%rdi) + + movaps -0x64(%rsi), %xmm7 + palignr $4, %xmm7, %xmm6 + movaps %xmm6, -0x60(%rdi) + + movaps -0x74(%rsi), %xmm8 + palignr $4, %xmm8, %xmm7 + movaps %xmm7, -0x70(%rdi) + + movaps -0x84(%rsi), %xmm9 + palignr $4, %xmm9, %xmm8 + movaps %xmm8, -0x80(%rdi) + + sub $0x80, %rdx + lea -0x80(%rdi), %rdi + lea -0x80(%rsi), %rsi + jae L(shl_4_bwd) + movdqu %xmm0, (%r8) + add $0x80, %rdx + sub %rdx, %rdi + sub %rdx, %rsi + BRANCH_TO_JMPTBL_ENTRY (L(table_144_bytes_bwd), %rdx, 4) + + .p2align 4 +L(shl_5): + sub $0x80, %rdx + movaps -0x05(%rsi), %xmm1 + movaps 0x0b(%rsi), %xmm2 + movaps 0x1b(%rsi), %xmm3 + movaps 0x2b(%rsi), %xmm4 + movaps 0x3b(%rsi), %xmm5 + movaps 0x4b(%rsi), %xmm6 + movaps 0x5b(%rsi), %xmm7 + movaps 0x6b(%rsi), %xmm8 + movaps 0x7b(%rsi), %xmm9 + lea 0x80(%rsi), %rsi + palignr $5, %xmm8, %xmm9 + movaps %xmm9, 0x70(%rdi) + palignr $5, %xmm7, %xmm8 + movaps %xmm8, 0x60(%rdi) + palignr $5, %xmm6, %xmm7 + movaps %xmm7, 0x50(%rdi) + palignr $5, %xmm5, %xmm6 + movaps %xmm6, 0x40(%rdi) + palignr $5, %xmm4, %xmm5 + movaps %xmm5, 0x30(%rdi) + palignr $5, %xmm3, %xmm4 + movaps %xmm4, 0x20(%rdi) + palignr $5, %xmm2, %xmm3 + movaps %xmm3, 0x10(%rdi) + palignr $5, %xmm1, %xmm2 + movaps %xmm2, (%rdi) + lea 0x80(%rdi), %rdi + jae L(shl_5) + movdqu %xmm0, (%r8) + add $0x80, %rdx + add %rdx, %rdi + add %rdx, %rsi + BRANCH_TO_JMPTBL_ENTRY (L(table_144_bytes_fwd), %rdx, 4) + + .p2align 4 +L(shl_5_bwd): + movaps -0x05(%rsi), %xmm1 + + movaps -0x15(%rsi), %xmm2 + palignr $5, %xmm2, %xmm1 + movaps %xmm1, -0x10(%rdi) + + movaps -0x25(%rsi), %xmm3 + palignr $5, %xmm3, %xmm2 + movaps %xmm2, -0x20(%rdi) + + movaps -0x35(%rsi), %xmm4 + palignr $5, %xmm4, %xmm3 + movaps %xmm3, -0x30(%rdi) + + movaps -0x45(%rsi), %xmm5 + palignr $5, %xmm5, %xmm4 + movaps %xmm4, -0x40(%rdi) + + movaps -0x55(%rsi), %xmm6 + palignr $5, %xmm6, %xmm5 + movaps %xmm5, -0x50(%rdi) + + movaps -0x65(%rsi), %xmm7 + palignr $5, %xmm7, %xmm6 + movaps %xmm6, -0x60(%rdi) + + movaps -0x75(%rsi), %xmm8 + palignr $5, %xmm8, %xmm7 + movaps %xmm7, -0x70(%rdi) + + movaps -0x85(%rsi), %xmm9 + palignr $5, %xmm9, %xmm8 + movaps %xmm8, -0x80(%rdi) + + sub $0x80, %rdx + lea -0x80(%rdi), %rdi + lea -0x80(%rsi), %rsi + jae L(shl_5_bwd) + movdqu %xmm0, (%r8) + add $0x80, %rdx + sub %rdx, %rdi + sub %rdx, %rsi + BRANCH_TO_JMPTBL_ENTRY (L(table_144_bytes_bwd), %rdx, 4) + + .p2align 4 +L(shl_6): + sub $0x80, %rdx + movaps -0x06(%rsi), %xmm1 + movaps 0x0a(%rsi), %xmm2 + movaps 0x1a(%rsi), %xmm3 + movaps 0x2a(%rsi), %xmm4 + movaps 0x3a(%rsi), %xmm5 + movaps 0x4a(%rsi), %xmm6 + movaps 0x5a(%rsi), %xmm7 + movaps 0x6a(%rsi), %xmm8 + movaps 0x7a(%rsi), %xmm9 + lea 0x80(%rsi), %rsi + palignr $6, %xmm8, %xmm9 + movaps %xmm9, 0x70(%rdi) + palignr $6, %xmm7, %xmm8 + movaps %xmm8, 0x60(%rdi) + palignr $6, %xmm6, %xmm7 + movaps %xmm7, 0x50(%rdi) + palignr $6, %xmm5, %xmm6 + movaps %xmm6, 0x40(%rdi) + palignr $6, %xmm4, %xmm5 + movaps %xmm5, 0x30(%rdi) + palignr $6, %xmm3, %xmm4 + movaps %xmm4, 0x20(%rdi) + palignr $6, %xmm2, %xmm3 + movaps %xmm3, 0x10(%rdi) + palignr $6, %xmm1, %xmm2 + movaps %xmm2, (%rdi) + lea 0x80(%rdi), %rdi + jae L(shl_6) + movdqu %xmm0, (%r8) + add $0x80, %rdx + add %rdx, %rdi + add %rdx, %rsi + BRANCH_TO_JMPTBL_ENTRY (L(table_144_bytes_fwd), %rdx, 4) + + .p2align 4 +L(shl_6_bwd): + movaps -0x06(%rsi), %xmm1 + + movaps -0x16(%rsi), %xmm2 + palignr $6, %xmm2, %xmm1 + movaps %xmm1, -0x10(%rdi) + + movaps -0x26(%rsi), %xmm3 + palignr $6, %xmm3, %xmm2 + movaps %xmm2, -0x20(%rdi) + + movaps -0x36(%rsi), %xmm4 + palignr $6, %xmm4, %xmm3 + movaps %xmm3, -0x30(%rdi) + + movaps -0x46(%rsi), %xmm5 + palignr $6, %xmm5, %xmm4 + movaps %xmm4, -0x40(%rdi) + + movaps -0x56(%rsi), %xmm6 + palignr $6, %xmm6, %xmm5 + movaps %xmm5, -0x50(%rdi) + + movaps -0x66(%rsi), %xmm7 + palignr $6, %xmm7, %xmm6 + movaps %xmm6, -0x60(%rdi) + + movaps -0x76(%rsi), %xmm8 + palignr $6, %xmm8, %xmm7 + movaps %xmm7, -0x70(%rdi) + + movaps -0x86(%rsi), %xmm9 + palignr $6, %xmm9, %xmm8 + movaps %xmm8, -0x80(%rdi) + + sub $0x80, %rdx + lea -0x80(%rdi), %rdi + lea -0x80(%rsi), %rsi + jae L(shl_6_bwd) + movdqu %xmm0, (%r8) + add $0x80, %rdx + sub %rdx, %rdi + sub %rdx, %rsi + BRANCH_TO_JMPTBL_ENTRY (L(table_144_bytes_bwd), %rdx, 4) + + .p2align 4 +L(shl_7): + sub $0x80, %rdx + movaps -0x07(%rsi), %xmm1 + movaps 0x09(%rsi), %xmm2 + movaps 0x19(%rsi), %xmm3 + movaps 0x29(%rsi), %xmm4 + movaps 0x39(%rsi), %xmm5 + movaps 0x49(%rsi), %xmm6 + movaps 0x59(%rsi), %xmm7 + movaps 0x69(%rsi), %xmm8 + movaps 0x79(%rsi), %xmm9 + lea 0x80(%rsi), %rsi + palignr $7, %xmm8, %xmm9 + movaps %xmm9, 0x70(%rdi) + palignr $7, %xmm7, %xmm8 + movaps %xmm8, 0x60(%rdi) + palignr $7, %xmm6, %xmm7 + movaps %xmm7, 0x50(%rdi) + palignr $7, %xmm5, %xmm6 + movaps %xmm6, 0x40(%rdi) + palignr $7, %xmm4, %xmm5 + movaps %xmm5, 0x30(%rdi) + palignr $7, %xmm3, %xmm4 + movaps %xmm4, 0x20(%rdi) + palignr $7, %xmm2, %xmm3 + movaps %xmm3, 0x10(%rdi) + palignr $7, %xmm1, %xmm2 + movaps %xmm2, (%rdi) + lea 0x80(%rdi), %rdi + jae L(shl_7) + movdqu %xmm0, (%r8) + add $0x80, %rdx + add %rdx, %rdi + add %rdx, %rsi + BRANCH_TO_JMPTBL_ENTRY (L(table_144_bytes_fwd), %rdx, 4) + + .p2align 4 +L(shl_7_bwd): + movaps -0x07(%rsi), %xmm1 + + movaps -0x17(%rsi), %xmm2 + palignr $7, %xmm2, %xmm1 + movaps %xmm1, -0x10(%rdi) + + movaps -0x27(%rsi), %xmm3 + palignr $7, %xmm3, %xmm2 + movaps %xmm2, -0x20(%rdi) + + movaps -0x37(%rsi), %xmm4 + palignr $7, %xmm4, %xmm3 + movaps %xmm3, -0x30(%rdi) + + movaps -0x47(%rsi), %xmm5 + palignr $7, %xmm5, %xmm4 + movaps %xmm4, -0x40(%rdi) + + movaps -0x57(%rsi), %xmm6 + palignr $7, %xmm6, %xmm5 + movaps %xmm5, -0x50(%rdi) + + movaps -0x67(%rsi), %xmm7 + palignr $7, %xmm7, %xmm6 + movaps %xmm6, -0x60(%rdi) + + movaps -0x77(%rsi), %xmm8 + palignr $7, %xmm8, %xmm7 + movaps %xmm7, -0x70(%rdi) + + movaps -0x87(%rsi), %xmm9 + palignr $7, %xmm9, %xmm8 + movaps %xmm8, -0x80(%rdi) + + sub $0x80, %rdx + lea -0x80(%rdi), %rdi + lea -0x80(%rsi), %rsi + jae L(shl_7_bwd) + movdqu %xmm0, (%r8) + add $0x80, %rdx + sub %rdx, %rdi + sub %rdx, %rsi + BRANCH_TO_JMPTBL_ENTRY (L(table_144_bytes_bwd), %rdx, 4) + + .p2align 4 +L(shl_8): + sub $0x80, %rdx + movaps -0x08(%rsi), %xmm1 + movaps 0x08(%rsi), %xmm2 + movaps 0x18(%rsi), %xmm3 + movaps 0x28(%rsi), %xmm4 + movaps 0x38(%rsi), %xmm5 + movaps 0x48(%rsi), %xmm6 + movaps 0x58(%rsi), %xmm7 + movaps 0x68(%rsi), %xmm8 + movaps 0x78(%rsi), %xmm9 + lea 0x80(%rsi), %rsi + palignr $8, %xmm8, %xmm9 + movaps %xmm9, 0x70(%rdi) + palignr $8, %xmm7, %xmm8 + movaps %xmm8, 0x60(%rdi) + palignr $8, %xmm6, %xmm7 + movaps %xmm7, 0x50(%rdi) + palignr $8, %xmm5, %xmm6 + movaps %xmm6, 0x40(%rdi) + palignr $8, %xmm4, %xmm5 + movaps %xmm5, 0x30(%rdi) + palignr $8, %xmm3, %xmm4 + movaps %xmm4, 0x20(%rdi) + palignr $8, %xmm2, %xmm3 + movaps %xmm3, 0x10(%rdi) + palignr $8, %xmm1, %xmm2 + movaps %xmm2, (%rdi) + lea 0x80(%rdi), %rdi + jae L(shl_8) + movdqu %xmm0, (%r8) + add $0x80, %rdx + add %rdx, %rdi + add %rdx, %rsi + BRANCH_TO_JMPTBL_ENTRY (L(table_144_bytes_fwd), %rdx, 4) + + .p2align 4 +L(shl_8_bwd): + movaps -0x08(%rsi), %xmm1 + + movaps -0x18(%rsi), %xmm2 + palignr $8, %xmm2, %xmm1 + movaps %xmm1, -0x10(%rdi) + + movaps -0x28(%rsi), %xmm3 + palignr $8, %xmm3, %xmm2 + movaps %xmm2, -0x20(%rdi) + + movaps -0x38(%rsi), %xmm4 + palignr $8, %xmm4, %xmm3 + movaps %xmm3, -0x30(%rdi) + + movaps -0x48(%rsi), %xmm5 + palignr $8, %xmm5, %xmm4 + movaps %xmm4, -0x40(%rdi) + + movaps -0x58(%rsi), %xmm6 + palignr $8, %xmm6, %xmm5 + movaps %xmm5, -0x50(%rdi) + + movaps -0x68(%rsi), %xmm7 + palignr $8, %xmm7, %xmm6 + movaps %xmm6, -0x60(%rdi) + + movaps -0x78(%rsi), %xmm8 + palignr $8, %xmm8, %xmm7 + movaps %xmm7, -0x70(%rdi) + + movaps -0x88(%rsi), %xmm9 + palignr $8, %xmm9, %xmm8 + movaps %xmm8, -0x80(%rdi) + + sub $0x80, %rdx + lea -0x80(%rdi), %rdi + lea -0x80(%rsi), %rsi + jae L(shl_8_bwd) +L(shl_8_end_bwd): + movdqu %xmm0, (%r8) + add $0x80, %rdx + sub %rdx, %rdi + sub %rdx, %rsi + BRANCH_TO_JMPTBL_ENTRY (L(table_144_bytes_bwd), %rdx, 4) + + .p2align 4 +L(shl_9): + sub $0x80, %rdx + movaps -0x09(%rsi), %xmm1 + movaps 0x07(%rsi), %xmm2 + movaps 0x17(%rsi), %xmm3 + movaps 0x27(%rsi), %xmm4 + movaps 0x37(%rsi), %xmm5 + movaps 0x47(%rsi), %xmm6 + movaps 0x57(%rsi), %xmm7 + movaps 0x67(%rsi), %xmm8 + movaps 0x77(%rsi), %xmm9 + lea 0x80(%rsi), %rsi + palignr $9, %xmm8, %xmm9 + movaps %xmm9, 0x70(%rdi) + palignr $9, %xmm7, %xmm8 + movaps %xmm8, 0x60(%rdi) + palignr $9, %xmm6, %xmm7 + movaps %xmm7, 0x50(%rdi) + palignr $9, %xmm5, %xmm6 + movaps %xmm6, 0x40(%rdi) + palignr $9, %xmm4, %xmm5 + movaps %xmm5, 0x30(%rdi) + palignr $9, %xmm3, %xmm4 + movaps %xmm4, 0x20(%rdi) + palignr $9, %xmm2, %xmm3 + movaps %xmm3, 0x10(%rdi) + palignr $9, %xmm1, %xmm2 + movaps %xmm2, (%rdi) + lea 0x80(%rdi), %rdi + jae L(shl_9) + movdqu %xmm0, (%r8) + add $0x80, %rdx + add %rdx, %rdi + add %rdx, %rsi + BRANCH_TO_JMPTBL_ENTRY (L(table_144_bytes_fwd), %rdx, 4) + + .p2align 4 +L(shl_9_bwd): + movaps -0x09(%rsi), %xmm1 + + movaps -0x19(%rsi), %xmm2 + palignr $9, %xmm2, %xmm1 + movaps %xmm1, -0x10(%rdi) + + movaps -0x29(%rsi), %xmm3 + palignr $9, %xmm3, %xmm2 + movaps %xmm2, -0x20(%rdi) + + movaps -0x39(%rsi), %xmm4 + palignr $9, %xmm4, %xmm3 + movaps %xmm3, -0x30(%rdi) + + movaps -0x49(%rsi), %xmm5 + palignr $9, %xmm5, %xmm4 + movaps %xmm4, -0x40(%rdi) + + movaps -0x59(%rsi), %xmm6 + palignr $9, %xmm6, %xmm5 + movaps %xmm5, -0x50(%rdi) + + movaps -0x69(%rsi), %xmm7 + palignr $9, %xmm7, %xmm6 + movaps %xmm6, -0x60(%rdi) + + movaps -0x79(%rsi), %xmm8 + palignr $9, %xmm8, %xmm7 + movaps %xmm7, -0x70(%rdi) + + movaps -0x89(%rsi), %xmm9 + palignr $9, %xmm9, %xmm8 + movaps %xmm8, -0x80(%rdi) + + sub $0x80, %rdx + lea -0x80(%rdi), %rdi + lea -0x80(%rsi), %rsi + jae L(shl_9_bwd) + movdqu %xmm0, (%r8) + add $0x80, %rdx + sub %rdx, %rdi + sub %rdx, %rsi + BRANCH_TO_JMPTBL_ENTRY (L(table_144_bytes_bwd), %rdx, 4) + + .p2align 4 +L(shl_10): + sub $0x80, %rdx + movaps -0x0a(%rsi), %xmm1 + movaps 0x06(%rsi), %xmm2 + movaps 0x16(%rsi), %xmm3 + movaps 0x26(%rsi), %xmm4 + movaps 0x36(%rsi), %xmm5 + movaps 0x46(%rsi), %xmm6 + movaps 0x56(%rsi), %xmm7 + movaps 0x66(%rsi), %xmm8 + movaps 0x76(%rsi), %xmm9 + lea 0x80(%rsi), %rsi + palignr $10, %xmm8, %xmm9 + movaps %xmm9, 0x70(%rdi) + palignr $10, %xmm7, %xmm8 + movaps %xmm8, 0x60(%rdi) + palignr $10, %xmm6, %xmm7 + movaps %xmm7, 0x50(%rdi) + palignr $10, %xmm5, %xmm6 + movaps %xmm6, 0x40(%rdi) + palignr $10, %xmm4, %xmm5 + movaps %xmm5, 0x30(%rdi) + palignr $10, %xmm3, %xmm4 + movaps %xmm4, 0x20(%rdi) + palignr $10, %xmm2, %xmm3 + movaps %xmm3, 0x10(%rdi) + palignr $10, %xmm1, %xmm2 + movaps %xmm2, (%rdi) + lea 0x80(%rdi), %rdi + jae L(shl_10) + movdqu %xmm0, (%r8) + add $0x80, %rdx + add %rdx, %rdi + add %rdx, %rsi + BRANCH_TO_JMPTBL_ENTRY (L(table_144_bytes_fwd), %rdx, 4) + + .p2align 4 +L(shl_10_bwd): + movaps -0x0a(%rsi), %xmm1 + + movaps -0x1a(%rsi), %xmm2 + palignr $10, %xmm2, %xmm1 + movaps %xmm1, -0x10(%rdi) + + movaps -0x2a(%rsi), %xmm3 + palignr $10, %xmm3, %xmm2 + movaps %xmm2, -0x20(%rdi) + + movaps -0x3a(%rsi), %xmm4 + palignr $10, %xmm4, %xmm3 + movaps %xmm3, -0x30(%rdi) + + movaps -0x4a(%rsi), %xmm5 + palignr $10, %xmm5, %xmm4 + movaps %xmm4, -0x40(%rdi) + + movaps -0x5a(%rsi), %xmm6 + palignr $10, %xmm6, %xmm5 + movaps %xmm5, -0x50(%rdi) + + movaps -0x6a(%rsi), %xmm7 + palignr $10, %xmm7, %xmm6 + movaps %xmm6, -0x60(%rdi) + + movaps -0x7a(%rsi), %xmm8 + palignr $10, %xmm8, %xmm7 + movaps %xmm7, -0x70(%rdi) + + movaps -0x8a(%rsi), %xmm9 + palignr $10, %xmm9, %xmm8 + movaps %xmm8, -0x80(%rdi) + + sub $0x80, %rdx + lea -0x80(%rdi), %rdi + lea -0x80(%rsi), %rsi + jae L(shl_10_bwd) + movdqu %xmm0, (%r8) + add $0x80, %rdx + sub %rdx, %rdi + sub %rdx, %rsi + BRANCH_TO_JMPTBL_ENTRY (L(table_144_bytes_bwd), %rdx, 4) + + .p2align 4 +L(shl_11): + sub $0x80, %rdx + movaps -0x0b(%rsi), %xmm1 + movaps 0x05(%rsi), %xmm2 + movaps 0x15(%rsi), %xmm3 + movaps 0x25(%rsi), %xmm4 + movaps 0x35(%rsi), %xmm5 + movaps 0x45(%rsi), %xmm6 + movaps 0x55(%rsi), %xmm7 + movaps 0x65(%rsi), %xmm8 + movaps 0x75(%rsi), %xmm9 + lea 0x80(%rsi), %rsi + palignr $11, %xmm8, %xmm9 + movaps %xmm9, 0x70(%rdi) + palignr $11, %xmm7, %xmm8 + movaps %xmm8, 0x60(%rdi) + palignr $11, %xmm6, %xmm7 + movaps %xmm7, 0x50(%rdi) + palignr $11, %xmm5, %xmm6 + movaps %xmm6, 0x40(%rdi) + palignr $11, %xmm4, %xmm5 + movaps %xmm5, 0x30(%rdi) + palignr $11, %xmm3, %xmm4 + movaps %xmm4, 0x20(%rdi) + palignr $11, %xmm2, %xmm3 + movaps %xmm3, 0x10(%rdi) + palignr $11, %xmm1, %xmm2 + movaps %xmm2, (%rdi) + lea 0x80(%rdi), %rdi + jae L(shl_11) + movdqu %xmm0, (%r8) + add $0x80, %rdx + add %rdx, %rdi + add %rdx, %rsi + BRANCH_TO_JMPTBL_ENTRY (L(table_144_bytes_fwd), %rdx, 4) + + .p2align 4 +L(shl_11_bwd): + movaps -0x0b(%rsi), %xmm1 + + movaps -0x1b(%rsi), %xmm2 + palignr $11, %xmm2, %xmm1 + movaps %xmm1, -0x10(%rdi) + + movaps -0x2b(%rsi), %xmm3 + palignr $11, %xmm3, %xmm2 + movaps %xmm2, -0x20(%rdi) + + movaps -0x3b(%rsi), %xmm4 + palignr $11, %xmm4, %xmm3 + movaps %xmm3, -0x30(%rdi) + + movaps -0x4b(%rsi), %xmm5 + palignr $11, %xmm5, %xmm4 + movaps %xmm4, -0x40(%rdi) + + movaps -0x5b(%rsi), %xmm6 + palignr $11, %xmm6, %xmm5 + movaps %xmm5, -0x50(%rdi) + + movaps -0x6b(%rsi), %xmm7 + palignr $11, %xmm7, %xmm6 + movaps %xmm6, -0x60(%rdi) + + movaps -0x7b(%rsi), %xmm8 + palignr $11, %xmm8, %xmm7 + movaps %xmm7, -0x70(%rdi) + + movaps -0x8b(%rsi), %xmm9 + palignr $11, %xmm9, %xmm8 + movaps %xmm8, -0x80(%rdi) + + sub $0x80, %rdx + lea -0x80(%rdi), %rdi + lea -0x80(%rsi), %rsi + jae L(shl_11_bwd) + movdqu %xmm0, (%r8) + add $0x80, %rdx + sub %rdx, %rdi + sub %rdx, %rsi + BRANCH_TO_JMPTBL_ENTRY (L(table_144_bytes_bwd), %rdx, 4) + + .p2align 4 +L(shl_12): + sub $0x80, %rdx + movdqa -0x0c(%rsi), %xmm1 + movaps 0x04(%rsi), %xmm2 + movaps 0x14(%rsi), %xmm3 + movaps 0x24(%rsi), %xmm4 + movaps 0x34(%rsi), %xmm5 + movaps 0x44(%rsi), %xmm6 + movaps 0x54(%rsi), %xmm7 + movaps 0x64(%rsi), %xmm8 + movaps 0x74(%rsi), %xmm9 + lea 0x80(%rsi), %rsi + palignr $12, %xmm8, %xmm9 + movaps %xmm9, 0x70(%rdi) + palignr $12, %xmm7, %xmm8 + movaps %xmm8, 0x60(%rdi) + palignr $12, %xmm6, %xmm7 + movaps %xmm7, 0x50(%rdi) + palignr $12, %xmm5, %xmm6 + movaps %xmm6, 0x40(%rdi) + palignr $12, %xmm4, %xmm5 + movaps %xmm5, 0x30(%rdi) + palignr $12, %xmm3, %xmm4 + movaps %xmm4, 0x20(%rdi) + palignr $12, %xmm2, %xmm3 + movaps %xmm3, 0x10(%rdi) + palignr $12, %xmm1, %xmm2 + movaps %xmm2, (%rdi) + + lea 0x80(%rdi), %rdi + jae L(shl_12) + movdqu %xmm0, (%r8) + add $0x80, %rdx + add %rdx, %rdi + add %rdx, %rsi + BRANCH_TO_JMPTBL_ENTRY (L(table_144_bytes_fwd), %rdx, 4) + + .p2align 4 +L(shl_12_bwd): + movaps -0x0c(%rsi), %xmm1 + + movaps -0x1c(%rsi), %xmm2 + palignr $12, %xmm2, %xmm1 + movaps %xmm1, -0x10(%rdi) + + movaps -0x2c(%rsi), %xmm3 + palignr $12, %xmm3, %xmm2 + movaps %xmm2, -0x20(%rdi) + + movaps -0x3c(%rsi), %xmm4 + palignr $12, %xmm4, %xmm3 + movaps %xmm3, -0x30(%rdi) + + movaps -0x4c(%rsi), %xmm5 + palignr $12, %xmm5, %xmm4 + movaps %xmm4, -0x40(%rdi) + + movaps -0x5c(%rsi), %xmm6 + palignr $12, %xmm6, %xmm5 + movaps %xmm5, -0x50(%rdi) + + movaps -0x6c(%rsi), %xmm7 + palignr $12, %xmm7, %xmm6 + movaps %xmm6, -0x60(%rdi) + + movaps -0x7c(%rsi), %xmm8 + palignr $12, %xmm8, %xmm7 + movaps %xmm7, -0x70(%rdi) + + movaps -0x8c(%rsi), %xmm9 + palignr $12, %xmm9, %xmm8 + movaps %xmm8, -0x80(%rdi) + + sub $0x80, %rdx + lea -0x80(%rdi), %rdi + lea -0x80(%rsi), %rsi + jae L(shl_12_bwd) + movdqu %xmm0, (%r8) + add $0x80, %rdx + sub %rdx, %rdi + sub %rdx, %rsi + BRANCH_TO_JMPTBL_ENTRY (L(table_144_bytes_bwd), %rdx, 4) + + .p2align 4 +L(shl_13): + sub $0x80, %rdx + movaps -0x0d(%rsi), %xmm1 + movaps 0x03(%rsi), %xmm2 + movaps 0x13(%rsi), %xmm3 + movaps 0x23(%rsi), %xmm4 + movaps 0x33(%rsi), %xmm5 + movaps 0x43(%rsi), %xmm6 + movaps 0x53(%rsi), %xmm7 + movaps 0x63(%rsi), %xmm8 + movaps 0x73(%rsi), %xmm9 + lea 0x80(%rsi), %rsi + palignr $13, %xmm8, %xmm9 + movaps %xmm9, 0x70(%rdi) + palignr $13, %xmm7, %xmm8 + movaps %xmm8, 0x60(%rdi) + palignr $13, %xmm6, %xmm7 + movaps %xmm7, 0x50(%rdi) + palignr $13, %xmm5, %xmm6 + movaps %xmm6, 0x40(%rdi) + palignr $13, %xmm4, %xmm5 + movaps %xmm5, 0x30(%rdi) + palignr $13, %xmm3, %xmm4 + movaps %xmm4, 0x20(%rdi) + palignr $13, %xmm2, %xmm3 + movaps %xmm3, 0x10(%rdi) + palignr $13, %xmm1, %xmm2 + movaps %xmm2, (%rdi) + lea 0x80(%rdi), %rdi + jae L(shl_13) + movdqu %xmm0, (%r8) + add $0x80, %rdx + add %rdx, %rdi + add %rdx, %rsi + BRANCH_TO_JMPTBL_ENTRY (L(table_144_bytes_fwd), %rdx, 4) + + .p2align 4 +L(shl_13_bwd): + movaps -0x0d(%rsi), %xmm1 + + movaps -0x1d(%rsi), %xmm2 + palignr $13, %xmm2, %xmm1 + movaps %xmm1, -0x10(%rdi) + + movaps -0x2d(%rsi), %xmm3 + palignr $13, %xmm3, %xmm2 + movaps %xmm2, -0x20(%rdi) + + movaps -0x3d(%rsi), %xmm4 + palignr $13, %xmm4, %xmm3 + movaps %xmm3, -0x30(%rdi) + + movaps -0x4d(%rsi), %xmm5 + palignr $13, %xmm5, %xmm4 + movaps %xmm4, -0x40(%rdi) + + movaps -0x5d(%rsi), %xmm6 + palignr $13, %xmm6, %xmm5 + movaps %xmm5, -0x50(%rdi) + + movaps -0x6d(%rsi), %xmm7 + palignr $13, %xmm7, %xmm6 + movaps %xmm6, -0x60(%rdi) + + movaps -0x7d(%rsi), %xmm8 + palignr $13, %xmm8, %xmm7 + movaps %xmm7, -0x70(%rdi) + + movaps -0x8d(%rsi), %xmm9 + palignr $13, %xmm9, %xmm8 + movaps %xmm8, -0x80(%rdi) + + sub $0x80, %rdx + lea -0x80(%rdi), %rdi + lea -0x80(%rsi), %rsi + jae L(shl_13_bwd) + movdqu %xmm0, (%r8) + add $0x80, %rdx + sub %rdx, %rdi + sub %rdx, %rsi + BRANCH_TO_JMPTBL_ENTRY (L(table_144_bytes_bwd), %rdx, 4) + + .p2align 4 +L(shl_14): + sub $0x80, %rdx + movaps -0x0e(%rsi), %xmm1 + movaps 0x02(%rsi), %xmm2 + movaps 0x12(%rsi), %xmm3 + movaps 0x22(%rsi), %xmm4 + movaps 0x32(%rsi), %xmm5 + movaps 0x42(%rsi), %xmm6 + movaps 0x52(%rsi), %xmm7 + movaps 0x62(%rsi), %xmm8 + movaps 0x72(%rsi), %xmm9 + lea 0x80(%rsi), %rsi + palignr $14, %xmm8, %xmm9 + movaps %xmm9, 0x70(%rdi) + palignr $14, %xmm7, %xmm8 + movaps %xmm8, 0x60(%rdi) + palignr $14, %xmm6, %xmm7 + movaps %xmm7, 0x50(%rdi) + palignr $14, %xmm5, %xmm6 + movaps %xmm6, 0x40(%rdi) + palignr $14, %xmm4, %xmm5 + movaps %xmm5, 0x30(%rdi) + palignr $14, %xmm3, %xmm4 + movaps %xmm4, 0x20(%rdi) + palignr $14, %xmm2, %xmm3 + movaps %xmm3, 0x10(%rdi) + palignr $14, %xmm1, %xmm2 + movaps %xmm2, (%rdi) + lea 0x80(%rdi), %rdi + jae L(shl_14) + movdqu %xmm0, (%r8) + add $0x80, %rdx + add %rdx, %rdi + add %rdx, %rsi + BRANCH_TO_JMPTBL_ENTRY (L(table_144_bytes_fwd), %rdx, 4) + + .p2align 4 +L(shl_14_bwd): + movaps -0x0e(%rsi), %xmm1 + + movaps -0x1e(%rsi), %xmm2 + palignr $14, %xmm2, %xmm1 + movaps %xmm1, -0x10(%rdi) + + movaps -0x2e(%rsi), %xmm3 + palignr $14, %xmm3, %xmm2 + movaps %xmm2, -0x20(%rdi) + + movaps -0x3e(%rsi), %xmm4 + palignr $14, %xmm4, %xmm3 + movaps %xmm3, -0x30(%rdi) + + movaps -0x4e(%rsi), %xmm5 + palignr $14, %xmm5, %xmm4 + movaps %xmm4, -0x40(%rdi) + + movaps -0x5e(%rsi), %xmm6 + palignr $14, %xmm6, %xmm5 + movaps %xmm5, -0x50(%rdi) + + movaps -0x6e(%rsi), %xmm7 + palignr $14, %xmm7, %xmm6 + movaps %xmm6, -0x60(%rdi) + + movaps -0x7e(%rsi), %xmm8 + palignr $14, %xmm8, %xmm7 + movaps %xmm7, -0x70(%rdi) + + movaps -0x8e(%rsi), %xmm9 + palignr $14, %xmm9, %xmm8 + movaps %xmm8, -0x80(%rdi) + + sub $0x80, %rdx + lea -0x80(%rdi), %rdi + lea -0x80(%rsi), %rsi + jae L(shl_14_bwd) + movdqu %xmm0, (%r8) + add $0x80, %rdx + sub %rdx, %rdi + sub %rdx, %rsi + BRANCH_TO_JMPTBL_ENTRY (L(table_144_bytes_bwd), %rdx, 4) + + .p2align 4 +L(shl_15): + sub $0x80, %rdx + movaps -0x0f(%rsi), %xmm1 + movaps 0x01(%rsi), %xmm2 + movaps 0x11(%rsi), %xmm3 + movaps 0x21(%rsi), %xmm4 + movaps 0x31(%rsi), %xmm5 + movaps 0x41(%rsi), %xmm6 + movaps 0x51(%rsi), %xmm7 + movaps 0x61(%rsi), %xmm8 + movaps 0x71(%rsi), %xmm9 + lea 0x80(%rsi), %rsi + palignr $15, %xmm8, %xmm9 + movaps %xmm9, 0x70(%rdi) + palignr $15, %xmm7, %xmm8 + movaps %xmm8, 0x60(%rdi) + palignr $15, %xmm6, %xmm7 + movaps %xmm7, 0x50(%rdi) + palignr $15, %xmm5, %xmm6 + movaps %xmm6, 0x40(%rdi) + palignr $15, %xmm4, %xmm5 + movaps %xmm5, 0x30(%rdi) + palignr $15, %xmm3, %xmm4 + movaps %xmm4, 0x20(%rdi) + palignr $15, %xmm2, %xmm3 + movaps %xmm3, 0x10(%rdi) + palignr $15, %xmm1, %xmm2 + movaps %xmm2, (%rdi) + lea 0x80(%rdi), %rdi + jae L(shl_15) + movdqu %xmm0, (%r8) + add $0x80, %rdx + add %rdx, %rdi + add %rdx, %rsi + BRANCH_TO_JMPTBL_ENTRY (L(table_144_bytes_fwd), %rdx, 4) + + .p2align 4 +L(shl_15_bwd): + movaps -0x0f(%rsi), %xmm1 + + movaps -0x1f(%rsi), %xmm2 + palignr $15, %xmm2, %xmm1 + movaps %xmm1, -0x10(%rdi) + + movaps -0x2f(%rsi), %xmm3 + palignr $15, %xmm3, %xmm2 + movaps %xmm2, -0x20(%rdi) + + movaps -0x3f(%rsi), %xmm4 + palignr $15, %xmm4, %xmm3 + movaps %xmm3, -0x30(%rdi) + + movaps -0x4f(%rsi), %xmm5 + palignr $15, %xmm5, %xmm4 + movaps %xmm4, -0x40(%rdi) + + movaps -0x5f(%rsi), %xmm6 + palignr $15, %xmm6, %xmm5 + movaps %xmm5, -0x50(%rdi) + + movaps -0x6f(%rsi), %xmm7 + palignr $15, %xmm7, %xmm6 + movaps %xmm6, -0x60(%rdi) + + movaps -0x7f(%rsi), %xmm8 + palignr $15, %xmm8, %xmm7 + movaps %xmm7, -0x70(%rdi) + + movaps -0x8f(%rsi), %xmm9 + palignr $15, %xmm9, %xmm8 + movaps %xmm8, -0x80(%rdi) + + sub $0x80, %rdx + lea -0x80(%rdi), %rdi + lea -0x80(%rsi), %rsi + jae L(shl_15_bwd) + movdqu %xmm0, (%r8) + add $0x80, %rdx + sub %rdx, %rdi + sub %rdx, %rsi + BRANCH_TO_JMPTBL_ENTRY (L(table_144_bytes_bwd), %rdx, 4) + + .p2align 4 +L(gobble_mem_fwd): + movdqu (%rsi), %xmm1 + movdqu %xmm0, (%r8) + movdqa %xmm1, (%rdi) + sub $16, %rdx + add $16, %rsi + add $16, %rdi + +#ifdef SHARED_CACHE_SIZE_HALF + mov $SHARED_CACHE_SIZE_HALF, %RCX_LP +#else + mov __x86_shared_cache_size_half(%rip), %RCX_LP +#endif +#ifdef USE_AS_MEMMOVE + mov %rsi, %r9 + sub %rdi, %r9 + cmp %rdx, %r9 + jae L(memmove_is_memcpy_fwd) + cmp %rcx, %r9 + jbe L(ll_cache_copy_fwd_start) +L(memmove_is_memcpy_fwd): +#endif + cmp %rcx, %rdx + ja L(bigger_in_fwd) + mov %rdx, %rcx +L(bigger_in_fwd): + sub %rcx, %rdx + cmp $0x1000, %rdx + jbe L(ll_cache_copy_fwd) + + mov %rcx, %r9 + shl $3, %r9 + cmp %r9, %rdx + jbe L(2steps_copy_fwd) + add %rcx, %rdx + xor %rcx, %rcx +L(2steps_copy_fwd): + sub $0x80, %rdx +L(gobble_mem_fwd_loop): + sub $0x80, %rdx + prefetcht0 0x200(%rsi) + prefetcht0 0x300(%rsi) + movdqu (%rsi), %xmm0 + movdqu 0x10(%rsi), %xmm1 + movdqu 0x20(%rsi), %xmm2 + movdqu 0x30(%rsi), %xmm3 + movdqu 0x40(%rsi), %xmm4 + movdqu 0x50(%rsi), %xmm5 + movdqu 0x60(%rsi), %xmm6 + movdqu 0x70(%rsi), %xmm7 + lfence + movntdq %xmm0, (%rdi) + movntdq %xmm1, 0x10(%rdi) + movntdq %xmm2, 0x20(%rdi) + movntdq %xmm3, 0x30(%rdi) + movntdq %xmm4, 0x40(%rdi) + movntdq %xmm5, 0x50(%rdi) + movntdq %xmm6, 0x60(%rdi) + movntdq %xmm7, 0x70(%rdi) + lea 0x80(%rsi), %rsi + lea 0x80(%rdi), %rdi + jae L(gobble_mem_fwd_loop) + sfence + cmp $0x80, %rcx + jb L(gobble_mem_fwd_end) + add $0x80, %rdx +L(ll_cache_copy_fwd): + add %rcx, %rdx +L(ll_cache_copy_fwd_start): + sub $0x80, %rdx +L(gobble_ll_loop_fwd): + prefetchnta 0x1c0(%rsi) + prefetchnta 0x280(%rsi) + prefetchnta 0x1c0(%rdi) + prefetchnta 0x280(%rdi) + sub $0x80, %rdx + movdqu (%rsi), %xmm0 + movdqu 0x10(%rsi), %xmm1 + movdqu 0x20(%rsi), %xmm2 + movdqu 0x30(%rsi), %xmm3 + movdqu 0x40(%rsi), %xmm4 + movdqu 0x50(%rsi), %xmm5 + movdqu 0x60(%rsi), %xmm6 + movdqu 0x70(%rsi), %xmm7 + movdqa %xmm0, (%rdi) + movdqa %xmm1, 0x10(%rdi) + movdqa %xmm2, 0x20(%rdi) + movdqa %xmm3, 0x30(%rdi) + movdqa %xmm4, 0x40(%rdi) + movdqa %xmm5, 0x50(%rdi) + movdqa %xmm6, 0x60(%rdi) + movdqa %xmm7, 0x70(%rdi) + lea 0x80(%rsi), %rsi + lea 0x80(%rdi), %rdi + jae L(gobble_ll_loop_fwd) +L(gobble_mem_fwd_end): + add $0x80, %rdx + add %rdx, %rsi + add %rdx, %rdi + BRANCH_TO_JMPTBL_ENTRY (L(table_144_bytes_fwd), %rdx, 4) + + .p2align 4 +L(gobble_mem_bwd): + add %rdx, %rsi + add %rdx, %rdi + + movdqu -16(%rsi), %xmm0 + lea -16(%rdi), %r8 + mov %rdi, %r9 + and $-16, %rdi + sub %rdi, %r9 + sub %r9, %rsi + sub %r9, %rdx + + +#ifdef SHARED_CACHE_SIZE_HALF + mov $SHARED_CACHE_SIZE_HALF, %RCX_LP +#else + mov __x86_shared_cache_size_half(%rip), %RCX_LP +#endif +#ifdef USE_AS_MEMMOVE + mov %rdi, %r9 + sub %rsi, %r9 + cmp %rdx, %r9 + jae L(memmove_is_memcpy_bwd) + cmp %rcx, %r9 + jbe L(ll_cache_copy_bwd_start) +L(memmove_is_memcpy_bwd): +#endif + cmp %rcx, %rdx + ja L(bigger) + mov %rdx, %rcx +L(bigger): + sub %rcx, %rdx + cmp $0x1000, %rdx + jbe L(ll_cache_copy) + + mov %rcx, %r9 + shl $3, %r9 + cmp %r9, %rdx + jbe L(2steps_copy) + add %rcx, %rdx + xor %rcx, %rcx +L(2steps_copy): + sub $0x80, %rdx +L(gobble_mem_bwd_loop): + sub $0x80, %rdx + prefetcht0 -0x200(%rsi) + prefetcht0 -0x300(%rsi) + movdqu -0x10(%rsi), %xmm1 + movdqu -0x20(%rsi), %xmm2 + movdqu -0x30(%rsi), %xmm3 + movdqu -0x40(%rsi), %xmm4 + movdqu -0x50(%rsi), %xmm5 + movdqu -0x60(%rsi), %xmm6 + movdqu -0x70(%rsi), %xmm7 + movdqu -0x80(%rsi), %xmm8 + lfence + movntdq %xmm1, -0x10(%rdi) + movntdq %xmm2, -0x20(%rdi) + movntdq %xmm3, -0x30(%rdi) + movntdq %xmm4, -0x40(%rdi) + movntdq %xmm5, -0x50(%rdi) + movntdq %xmm6, -0x60(%rdi) + movntdq %xmm7, -0x70(%rdi) + movntdq %xmm8, -0x80(%rdi) + lea -0x80(%rsi), %rsi + lea -0x80(%rdi), %rdi + jae L(gobble_mem_bwd_loop) + sfence + cmp $0x80, %rcx + jb L(gobble_mem_bwd_end) + add $0x80, %rdx +L(ll_cache_copy): + add %rcx, %rdx +L(ll_cache_copy_bwd_start): + sub $0x80, %rdx +L(gobble_ll_loop): + prefetchnta -0x1c0(%rsi) + prefetchnta -0x280(%rsi) + prefetchnta -0x1c0(%rdi) + prefetchnta -0x280(%rdi) + sub $0x80, %rdx + movdqu -0x10(%rsi), %xmm1 + movdqu -0x20(%rsi), %xmm2 + movdqu -0x30(%rsi), %xmm3 + movdqu -0x40(%rsi), %xmm4 + movdqu -0x50(%rsi), %xmm5 + movdqu -0x60(%rsi), %xmm6 + movdqu -0x70(%rsi), %xmm7 + movdqu -0x80(%rsi), %xmm8 + movdqa %xmm1, -0x10(%rdi) + movdqa %xmm2, -0x20(%rdi) + movdqa %xmm3, -0x30(%rdi) + movdqa %xmm4, -0x40(%rdi) + movdqa %xmm5, -0x50(%rdi) + movdqa %xmm6, -0x60(%rdi) + movdqa %xmm7, -0x70(%rdi) + movdqa %xmm8, -0x80(%rdi) + lea -0x80(%rsi), %rsi + lea -0x80(%rdi), %rdi + jae L(gobble_ll_loop) +L(gobble_mem_bwd_end): + movdqu %xmm0, (%r8) + add $0x80, %rdx + sub %rdx, %rsi + sub %rdx, %rdi + BRANCH_TO_JMPTBL_ENTRY (L(table_144_bytes_bwd), %rdx, 4) + + .p2align 4 +L(fwd_write_128bytes): + lddqu -128(%rsi), %xmm0 + movdqu %xmm0, -128(%rdi) +L(fwd_write_112bytes): + lddqu -112(%rsi), %xmm0 + movdqu %xmm0, -112(%rdi) +L(fwd_write_96bytes): + lddqu -96(%rsi), %xmm0 + movdqu %xmm0, -96(%rdi) +L(fwd_write_80bytes): + lddqu -80(%rsi), %xmm0 + movdqu %xmm0, -80(%rdi) +L(fwd_write_64bytes): + lddqu -64(%rsi), %xmm0 + movdqu %xmm0, -64(%rdi) +L(fwd_write_48bytes): + lddqu -48(%rsi), %xmm0 + movdqu %xmm0, -48(%rdi) +L(fwd_write_32bytes): + lddqu -32(%rsi), %xmm0 + movdqu %xmm0, -32(%rdi) +L(fwd_write_16bytes): + lddqu -16(%rsi), %xmm0 + movdqu %xmm0, -16(%rdi) +L(fwd_write_0bytes): + ret + + + .p2align 4 +L(fwd_write_143bytes): + lddqu -143(%rsi), %xmm0 + movdqu %xmm0, -143(%rdi) +L(fwd_write_127bytes): + lddqu -127(%rsi), %xmm0 + movdqu %xmm0, -127(%rdi) +L(fwd_write_111bytes): + lddqu -111(%rsi), %xmm0 + movdqu %xmm0, -111(%rdi) +L(fwd_write_95bytes): + lddqu -95(%rsi), %xmm0 + movdqu %xmm0, -95(%rdi) +L(fwd_write_79bytes): + lddqu -79(%rsi), %xmm0 + movdqu %xmm0, -79(%rdi) +L(fwd_write_63bytes): + lddqu -63(%rsi), %xmm0 + movdqu %xmm0, -63(%rdi) +L(fwd_write_47bytes): + lddqu -47(%rsi), %xmm0 + movdqu %xmm0, -47(%rdi) +L(fwd_write_31bytes): + lddqu -31(%rsi), %xmm0 + lddqu -16(%rsi), %xmm1 + movdqu %xmm0, -31(%rdi) + movdqu %xmm1, -16(%rdi) + ret + + .p2align 4 +L(fwd_write_15bytes): + mov -15(%rsi), %rdx + mov -8(%rsi), %rcx + mov %rdx, -15(%rdi) + mov %rcx, -8(%rdi) + ret + + .p2align 4 +L(fwd_write_142bytes): + lddqu -142(%rsi), %xmm0 + movdqu %xmm0, -142(%rdi) +L(fwd_write_126bytes): + lddqu -126(%rsi), %xmm0 + movdqu %xmm0, -126(%rdi) +L(fwd_write_110bytes): + lddqu -110(%rsi), %xmm0 + movdqu %xmm0, -110(%rdi) +L(fwd_write_94bytes): + lddqu -94(%rsi), %xmm0 + movdqu %xmm0, -94(%rdi) +L(fwd_write_78bytes): + lddqu -78(%rsi), %xmm0 + movdqu %xmm0, -78(%rdi) +L(fwd_write_62bytes): + lddqu -62(%rsi), %xmm0 + movdqu %xmm0, -62(%rdi) +L(fwd_write_46bytes): + lddqu -46(%rsi), %xmm0 + movdqu %xmm0, -46(%rdi) +L(fwd_write_30bytes): + lddqu -30(%rsi), %xmm0 + lddqu -16(%rsi), %xmm1 + movdqu %xmm0, -30(%rdi) + movdqu %xmm1, -16(%rdi) + ret + + .p2align 4 +L(fwd_write_14bytes): + mov -14(%rsi), %rdx + mov -8(%rsi), %rcx + mov %rdx, -14(%rdi) + mov %rcx, -8(%rdi) + ret + + .p2align 4 +L(fwd_write_141bytes): + lddqu -141(%rsi), %xmm0 + movdqu %xmm0, -141(%rdi) +L(fwd_write_125bytes): + lddqu -125(%rsi), %xmm0 + movdqu %xmm0, -125(%rdi) +L(fwd_write_109bytes): + lddqu -109(%rsi), %xmm0 + movdqu %xmm0, -109(%rdi) +L(fwd_write_93bytes): + lddqu -93(%rsi), %xmm0 + movdqu %xmm0, -93(%rdi) +L(fwd_write_77bytes): + lddqu -77(%rsi), %xmm0 + movdqu %xmm0, -77(%rdi) +L(fwd_write_61bytes): + lddqu -61(%rsi), %xmm0 + movdqu %xmm0, -61(%rdi) +L(fwd_write_45bytes): + lddqu -45(%rsi), %xmm0 + movdqu %xmm0, -45(%rdi) +L(fwd_write_29bytes): + lddqu -29(%rsi), %xmm0 + lddqu -16(%rsi), %xmm1 + movdqu %xmm0, -29(%rdi) + movdqu %xmm1, -16(%rdi) + ret + + .p2align 4 +L(fwd_write_13bytes): + mov -13(%rsi), %rdx + mov -8(%rsi), %rcx + mov %rdx, -13(%rdi) + mov %rcx, -8(%rdi) + ret + + .p2align 4 +L(fwd_write_140bytes): + lddqu -140(%rsi), %xmm0 + movdqu %xmm0, -140(%rdi) +L(fwd_write_124bytes): + lddqu -124(%rsi), %xmm0 + movdqu %xmm0, -124(%rdi) +L(fwd_write_108bytes): + lddqu -108(%rsi), %xmm0 + movdqu %xmm0, -108(%rdi) +L(fwd_write_92bytes): + lddqu -92(%rsi), %xmm0 + movdqu %xmm0, -92(%rdi) +L(fwd_write_76bytes): + lddqu -76(%rsi), %xmm0 + movdqu %xmm0, -76(%rdi) +L(fwd_write_60bytes): + lddqu -60(%rsi), %xmm0 + movdqu %xmm0, -60(%rdi) +L(fwd_write_44bytes): + lddqu -44(%rsi), %xmm0 + movdqu %xmm0, -44(%rdi) +L(fwd_write_28bytes): + lddqu -28(%rsi), %xmm0 + lddqu -16(%rsi), %xmm1 + movdqu %xmm0, -28(%rdi) + movdqu %xmm1, -16(%rdi) + ret + + .p2align 4 +L(fwd_write_12bytes): + mov -12(%rsi), %rdx + mov -4(%rsi), %ecx + mov %rdx, -12(%rdi) + mov %ecx, -4(%rdi) + ret + + .p2align 4 +L(fwd_write_139bytes): + lddqu -139(%rsi), %xmm0 + movdqu %xmm0, -139(%rdi) +L(fwd_write_123bytes): + lddqu -123(%rsi), %xmm0 + movdqu %xmm0, -123(%rdi) +L(fwd_write_107bytes): + lddqu -107(%rsi), %xmm0 + movdqu %xmm0, -107(%rdi) +L(fwd_write_91bytes): + lddqu -91(%rsi), %xmm0 + movdqu %xmm0, -91(%rdi) +L(fwd_write_75bytes): + lddqu -75(%rsi), %xmm0 + movdqu %xmm0, -75(%rdi) +L(fwd_write_59bytes): + lddqu -59(%rsi), %xmm0 + movdqu %xmm0, -59(%rdi) +L(fwd_write_43bytes): + lddqu -43(%rsi), %xmm0 + movdqu %xmm0, -43(%rdi) +L(fwd_write_27bytes): + lddqu -27(%rsi), %xmm0 + lddqu -16(%rsi), %xmm1 + movdqu %xmm0, -27(%rdi) + movdqu %xmm1, -16(%rdi) + ret + + .p2align 4 +L(fwd_write_11bytes): + mov -11(%rsi), %rdx + mov -4(%rsi), %ecx + mov %rdx, -11(%rdi) + mov %ecx, -4(%rdi) + ret + + .p2align 4 +L(fwd_write_138bytes): + lddqu -138(%rsi), %xmm0 + movdqu %xmm0, -138(%rdi) +L(fwd_write_122bytes): + lddqu -122(%rsi), %xmm0 + movdqu %xmm0, -122(%rdi) +L(fwd_write_106bytes): + lddqu -106(%rsi), %xmm0 + movdqu %xmm0, -106(%rdi) +L(fwd_write_90bytes): + lddqu -90(%rsi), %xmm0 + movdqu %xmm0, -90(%rdi) +L(fwd_write_74bytes): + lddqu -74(%rsi), %xmm0 + movdqu %xmm0, -74(%rdi) +L(fwd_write_58bytes): + lddqu -58(%rsi), %xmm0 + movdqu %xmm0, -58(%rdi) +L(fwd_write_42bytes): + lddqu -42(%rsi), %xmm0 + movdqu %xmm0, -42(%rdi) +L(fwd_write_26bytes): + lddqu -26(%rsi), %xmm0 + lddqu -16(%rsi), %xmm1 + movdqu %xmm0, -26(%rdi) + movdqu %xmm1, -16(%rdi) + ret + + .p2align 4 +L(fwd_write_10bytes): + mov -10(%rsi), %rdx + mov -4(%rsi), %ecx + mov %rdx, -10(%rdi) + mov %ecx, -4(%rdi) + ret + + .p2align 4 +L(fwd_write_137bytes): + lddqu -137(%rsi), %xmm0 + movdqu %xmm0, -137(%rdi) +L(fwd_write_121bytes): + lddqu -121(%rsi), %xmm0 + movdqu %xmm0, -121(%rdi) +L(fwd_write_105bytes): + lddqu -105(%rsi), %xmm0 + movdqu %xmm0, -105(%rdi) +L(fwd_write_89bytes): + lddqu -89(%rsi), %xmm0 + movdqu %xmm0, -89(%rdi) +L(fwd_write_73bytes): + lddqu -73(%rsi), %xmm0 + movdqu %xmm0, -73(%rdi) +L(fwd_write_57bytes): + lddqu -57(%rsi), %xmm0 + movdqu %xmm0, -57(%rdi) +L(fwd_write_41bytes): + lddqu -41(%rsi), %xmm0 + movdqu %xmm0, -41(%rdi) +L(fwd_write_25bytes): + lddqu -25(%rsi), %xmm0 + lddqu -16(%rsi), %xmm1 + movdqu %xmm0, -25(%rdi) + movdqu %xmm1, -16(%rdi) + ret + + .p2align 4 +L(fwd_write_9bytes): + mov -9(%rsi), %rdx + mov -4(%rsi), %ecx + mov %rdx, -9(%rdi) + mov %ecx, -4(%rdi) + ret + + .p2align 4 +L(fwd_write_136bytes): + lddqu -136(%rsi), %xmm0 + movdqu %xmm0, -136(%rdi) +L(fwd_write_120bytes): + lddqu -120(%rsi), %xmm0 + movdqu %xmm0, -120(%rdi) +L(fwd_write_104bytes): + lddqu -104(%rsi), %xmm0 + movdqu %xmm0, -104(%rdi) +L(fwd_write_88bytes): + lddqu -88(%rsi), %xmm0 + movdqu %xmm0, -88(%rdi) +L(fwd_write_72bytes): + lddqu -72(%rsi), %xmm0 + movdqu %xmm0, -72(%rdi) +L(fwd_write_56bytes): + lddqu -56(%rsi), %xmm0 + movdqu %xmm0, -56(%rdi) +L(fwd_write_40bytes): + lddqu -40(%rsi), %xmm0 + movdqu %xmm0, -40(%rdi) +L(fwd_write_24bytes): + lddqu -24(%rsi), %xmm0 + lddqu -16(%rsi), %xmm1 + movdqu %xmm0, -24(%rdi) + movdqu %xmm1, -16(%rdi) + ret + + .p2align 4 +L(fwd_write_8bytes): + mov -8(%rsi), %rdx + mov %rdx, -8(%rdi) + ret + + .p2align 4 +L(fwd_write_135bytes): + lddqu -135(%rsi), %xmm0 + movdqu %xmm0, -135(%rdi) +L(fwd_write_119bytes): + lddqu -119(%rsi), %xmm0 + movdqu %xmm0, -119(%rdi) +L(fwd_write_103bytes): + lddqu -103(%rsi), %xmm0 + movdqu %xmm0, -103(%rdi) +L(fwd_write_87bytes): + lddqu -87(%rsi), %xmm0 + movdqu %xmm0, -87(%rdi) +L(fwd_write_71bytes): + lddqu -71(%rsi), %xmm0 + movdqu %xmm0, -71(%rdi) +L(fwd_write_55bytes): + lddqu -55(%rsi), %xmm0 + movdqu %xmm0, -55(%rdi) +L(fwd_write_39bytes): + lddqu -39(%rsi), %xmm0 + movdqu %xmm0, -39(%rdi) +L(fwd_write_23bytes): + lddqu -23(%rsi), %xmm0 + lddqu -16(%rsi), %xmm1 + movdqu %xmm0, -23(%rdi) + movdqu %xmm1, -16(%rdi) + ret + + .p2align 4 +L(fwd_write_7bytes): + mov -7(%rsi), %edx + mov -4(%rsi), %ecx + mov %edx, -7(%rdi) + mov %ecx, -4(%rdi) + ret + + .p2align 4 +L(fwd_write_134bytes): + lddqu -134(%rsi), %xmm0 + movdqu %xmm0, -134(%rdi) +L(fwd_write_118bytes): + lddqu -118(%rsi), %xmm0 + movdqu %xmm0, -118(%rdi) +L(fwd_write_102bytes): + lddqu -102(%rsi), %xmm0 + movdqu %xmm0, -102(%rdi) +L(fwd_write_86bytes): + lddqu -86(%rsi), %xmm0 + movdqu %xmm0, -86(%rdi) +L(fwd_write_70bytes): + lddqu -70(%rsi), %xmm0 + movdqu %xmm0, -70(%rdi) +L(fwd_write_54bytes): + lddqu -54(%rsi), %xmm0 + movdqu %xmm0, -54(%rdi) +L(fwd_write_38bytes): + lddqu -38(%rsi), %xmm0 + movdqu %xmm0, -38(%rdi) +L(fwd_write_22bytes): + lddqu -22(%rsi), %xmm0 + lddqu -16(%rsi), %xmm1 + movdqu %xmm0, -22(%rdi) + movdqu %xmm1, -16(%rdi) + ret + + .p2align 4 +L(fwd_write_6bytes): + mov -6(%rsi), %edx + mov -4(%rsi), %ecx + mov %edx, -6(%rdi) + mov %ecx, -4(%rdi) + ret + + .p2align 4 +L(fwd_write_133bytes): + lddqu -133(%rsi), %xmm0 + movdqu %xmm0, -133(%rdi) +L(fwd_write_117bytes): + lddqu -117(%rsi), %xmm0 + movdqu %xmm0, -117(%rdi) +L(fwd_write_101bytes): + lddqu -101(%rsi), %xmm0 + movdqu %xmm0, -101(%rdi) +L(fwd_write_85bytes): + lddqu -85(%rsi), %xmm0 + movdqu %xmm0, -85(%rdi) +L(fwd_write_69bytes): + lddqu -69(%rsi), %xmm0 + movdqu %xmm0, -69(%rdi) +L(fwd_write_53bytes): + lddqu -53(%rsi), %xmm0 + movdqu %xmm0, -53(%rdi) +L(fwd_write_37bytes): + lddqu -37(%rsi), %xmm0 + movdqu %xmm0, -37(%rdi) +L(fwd_write_21bytes): + lddqu -21(%rsi), %xmm0 + lddqu -16(%rsi), %xmm1 + movdqu %xmm0, -21(%rdi) + movdqu %xmm1, -16(%rdi) + ret + + .p2align 4 +L(fwd_write_5bytes): + mov -5(%rsi), %edx + mov -4(%rsi), %ecx + mov %edx, -5(%rdi) + mov %ecx, -4(%rdi) + ret + + .p2align 4 +L(fwd_write_132bytes): + lddqu -132(%rsi), %xmm0 + movdqu %xmm0, -132(%rdi) +L(fwd_write_116bytes): + lddqu -116(%rsi), %xmm0 + movdqu %xmm0, -116(%rdi) +L(fwd_write_100bytes): + lddqu -100(%rsi), %xmm0 + movdqu %xmm0, -100(%rdi) +L(fwd_write_84bytes): + lddqu -84(%rsi), %xmm0 + movdqu %xmm0, -84(%rdi) +L(fwd_write_68bytes): + lddqu -68(%rsi), %xmm0 + movdqu %xmm0, -68(%rdi) +L(fwd_write_52bytes): + lddqu -52(%rsi), %xmm0 + movdqu %xmm0, -52(%rdi) +L(fwd_write_36bytes): + lddqu -36(%rsi), %xmm0 + movdqu %xmm0, -36(%rdi) +L(fwd_write_20bytes): + lddqu -20(%rsi), %xmm0 + lddqu -16(%rsi), %xmm1 + movdqu %xmm0, -20(%rdi) + movdqu %xmm1, -16(%rdi) + ret + + .p2align 4 +L(fwd_write_4bytes): + mov -4(%rsi), %edx + mov %edx, -4(%rdi) + ret + + .p2align 4 +L(fwd_write_131bytes): + lddqu -131(%rsi), %xmm0 + movdqu %xmm0, -131(%rdi) +L(fwd_write_115bytes): + lddqu -115(%rsi), %xmm0 + movdqu %xmm0, -115(%rdi) +L(fwd_write_99bytes): + lddqu -99(%rsi), %xmm0 + movdqu %xmm0, -99(%rdi) +L(fwd_write_83bytes): + lddqu -83(%rsi), %xmm0 + movdqu %xmm0, -83(%rdi) +L(fwd_write_67bytes): + lddqu -67(%rsi), %xmm0 + movdqu %xmm0, -67(%rdi) +L(fwd_write_51bytes): + lddqu -51(%rsi), %xmm0 + movdqu %xmm0, -51(%rdi) +L(fwd_write_35bytes): + lddqu -35(%rsi), %xmm0 + movdqu %xmm0, -35(%rdi) +L(fwd_write_19bytes): + lddqu -19(%rsi), %xmm0 + lddqu -16(%rsi), %xmm1 + movdqu %xmm0, -19(%rdi) + movdqu %xmm1, -16(%rdi) + ret + + .p2align 4 +L(fwd_write_3bytes): + mov -3(%rsi), %dx + mov -2(%rsi), %cx + mov %dx, -3(%rdi) + mov %cx, -2(%rdi) + ret + + .p2align 4 +L(fwd_write_130bytes): + lddqu -130(%rsi), %xmm0 + movdqu %xmm0, -130(%rdi) +L(fwd_write_114bytes): + lddqu -114(%rsi), %xmm0 + movdqu %xmm0, -114(%rdi) +L(fwd_write_98bytes): + lddqu -98(%rsi), %xmm0 + movdqu %xmm0, -98(%rdi) +L(fwd_write_82bytes): + lddqu -82(%rsi), %xmm0 + movdqu %xmm0, -82(%rdi) +L(fwd_write_66bytes): + lddqu -66(%rsi), %xmm0 + movdqu %xmm0, -66(%rdi) +L(fwd_write_50bytes): + lddqu -50(%rsi), %xmm0 + movdqu %xmm0, -50(%rdi) +L(fwd_write_34bytes): + lddqu -34(%rsi), %xmm0 + movdqu %xmm0, -34(%rdi) +L(fwd_write_18bytes): + lddqu -18(%rsi), %xmm0 + lddqu -16(%rsi), %xmm1 + movdqu %xmm0, -18(%rdi) + movdqu %xmm1, -16(%rdi) + ret + + .p2align 4 +L(fwd_write_2bytes): + movzwl -2(%rsi), %edx + mov %dx, -2(%rdi) + ret + + .p2align 4 +L(fwd_write_129bytes): + lddqu -129(%rsi), %xmm0 + movdqu %xmm0, -129(%rdi) +L(fwd_write_113bytes): + lddqu -113(%rsi), %xmm0 + movdqu %xmm0, -113(%rdi) +L(fwd_write_97bytes): + lddqu -97(%rsi), %xmm0 + movdqu %xmm0, -97(%rdi) +L(fwd_write_81bytes): + lddqu -81(%rsi), %xmm0 + movdqu %xmm0, -81(%rdi) +L(fwd_write_65bytes): + lddqu -65(%rsi), %xmm0 + movdqu %xmm0, -65(%rdi) +L(fwd_write_49bytes): + lddqu -49(%rsi), %xmm0 + movdqu %xmm0, -49(%rdi) +L(fwd_write_33bytes): + lddqu -33(%rsi), %xmm0 + movdqu %xmm0, -33(%rdi) +L(fwd_write_17bytes): + lddqu -17(%rsi), %xmm0 + lddqu -16(%rsi), %xmm1 + movdqu %xmm0, -17(%rdi) + movdqu %xmm1, -16(%rdi) + ret + + .p2align 4 +L(fwd_write_1bytes): + movzbl -1(%rsi), %edx + mov %dl, -1(%rdi) + ret + + .p2align 4 +L(bwd_write_128bytes): + lddqu 112(%rsi), %xmm0 + movdqu %xmm0, 112(%rdi) +L(bwd_write_112bytes): + lddqu 96(%rsi), %xmm0 + movdqu %xmm0, 96(%rdi) +L(bwd_write_96bytes): + lddqu 80(%rsi), %xmm0 + movdqu %xmm0, 80(%rdi) +L(bwd_write_80bytes): + lddqu 64(%rsi), %xmm0 + movdqu %xmm0, 64(%rdi) +L(bwd_write_64bytes): + lddqu 48(%rsi), %xmm0 + movdqu %xmm0, 48(%rdi) +L(bwd_write_48bytes): + lddqu 32(%rsi), %xmm0 + movdqu %xmm0, 32(%rdi) +L(bwd_write_32bytes): + lddqu 16(%rsi), %xmm0 + movdqu %xmm0, 16(%rdi) +L(bwd_write_16bytes): + lddqu (%rsi), %xmm0 + movdqu %xmm0, (%rdi) +L(bwd_write_0bytes): + ret + + .p2align 4 +L(bwd_write_143bytes): + lddqu 127(%rsi), %xmm0 + movdqu %xmm0, 127(%rdi) +L(bwd_write_127bytes): + lddqu 111(%rsi), %xmm0 + movdqu %xmm0, 111(%rdi) +L(bwd_write_111bytes): + lddqu 95(%rsi), %xmm0 + movdqu %xmm0, 95(%rdi) +L(bwd_write_95bytes): + lddqu 79(%rsi), %xmm0 + movdqu %xmm0, 79(%rdi) +L(bwd_write_79bytes): + lddqu 63(%rsi), %xmm0 + movdqu %xmm0, 63(%rdi) +L(bwd_write_63bytes): + lddqu 47(%rsi), %xmm0 + movdqu %xmm0, 47(%rdi) +L(bwd_write_47bytes): + lddqu 31(%rsi), %xmm0 + movdqu %xmm0, 31(%rdi) +L(bwd_write_31bytes): + lddqu 15(%rsi), %xmm0 + lddqu (%rsi), %xmm1 + movdqu %xmm0, 15(%rdi) + movdqu %xmm1, (%rdi) + ret + + + .p2align 4 +L(bwd_write_15bytes): + mov 7(%rsi), %rdx + mov (%rsi), %rcx + mov %rdx, 7(%rdi) + mov %rcx, (%rdi) + ret + + .p2align 4 +L(bwd_write_142bytes): + lddqu 126(%rsi), %xmm0 + movdqu %xmm0, 126(%rdi) +L(bwd_write_126bytes): + lddqu 110(%rsi), %xmm0 + movdqu %xmm0, 110(%rdi) +L(bwd_write_110bytes): + lddqu 94(%rsi), %xmm0 + movdqu %xmm0, 94(%rdi) +L(bwd_write_94bytes): + lddqu 78(%rsi), %xmm0 + movdqu %xmm0, 78(%rdi) +L(bwd_write_78bytes): + lddqu 62(%rsi), %xmm0 + movdqu %xmm0, 62(%rdi) +L(bwd_write_62bytes): + lddqu 46(%rsi), %xmm0 + movdqu %xmm0, 46(%rdi) +L(bwd_write_46bytes): + lddqu 30(%rsi), %xmm0 + movdqu %xmm0, 30(%rdi) +L(bwd_write_30bytes): + lddqu 14(%rsi), %xmm0 + lddqu (%rsi), %xmm1 + movdqu %xmm0, 14(%rdi) + movdqu %xmm1, (%rdi) + ret + + .p2align 4 +L(bwd_write_14bytes): + mov 6(%rsi), %rdx + mov (%rsi), %rcx + mov %rdx, 6(%rdi) + mov %rcx, (%rdi) + ret + + .p2align 4 +L(bwd_write_141bytes): + lddqu 125(%rsi), %xmm0 + movdqu %xmm0, 125(%rdi) +L(bwd_write_125bytes): + lddqu 109(%rsi), %xmm0 + movdqu %xmm0, 109(%rdi) +L(bwd_write_109bytes): + lddqu 93(%rsi), %xmm0 + movdqu %xmm0, 93(%rdi) +L(bwd_write_93bytes): + lddqu 77(%rsi), %xmm0 + movdqu %xmm0, 77(%rdi) +L(bwd_write_77bytes): + lddqu 61(%rsi), %xmm0 + movdqu %xmm0, 61(%rdi) +L(bwd_write_61bytes): + lddqu 45(%rsi), %xmm0 + movdqu %xmm0, 45(%rdi) +L(bwd_write_45bytes): + lddqu 29(%rsi), %xmm0 + movdqu %xmm0, 29(%rdi) +L(bwd_write_29bytes): + lddqu 13(%rsi), %xmm0 + lddqu (%rsi), %xmm1 + movdqu %xmm0, 13(%rdi) + movdqu %xmm1, (%rdi) + ret + + .p2align 4 +L(bwd_write_13bytes): + mov 5(%rsi), %rdx + mov (%rsi), %rcx + mov %rdx, 5(%rdi) + mov %rcx, (%rdi) + ret + + .p2align 4 +L(bwd_write_140bytes): + lddqu 124(%rsi), %xmm0 + movdqu %xmm0, 124(%rdi) +L(bwd_write_124bytes): + lddqu 108(%rsi), %xmm0 + movdqu %xmm0, 108(%rdi) +L(bwd_write_108bytes): + lddqu 92(%rsi), %xmm0 + movdqu %xmm0, 92(%rdi) +L(bwd_write_92bytes): + lddqu 76(%rsi), %xmm0 + movdqu %xmm0, 76(%rdi) +L(bwd_write_76bytes): + lddqu 60(%rsi), %xmm0 + movdqu %xmm0, 60(%rdi) +L(bwd_write_60bytes): + lddqu 44(%rsi), %xmm0 + movdqu %xmm0, 44(%rdi) +L(bwd_write_44bytes): + lddqu 28(%rsi), %xmm0 + movdqu %xmm0, 28(%rdi) +L(bwd_write_28bytes): + lddqu 12(%rsi), %xmm0 + lddqu (%rsi), %xmm1 + movdqu %xmm0, 12(%rdi) + movdqu %xmm1, (%rdi) + ret + + .p2align 4 +L(bwd_write_12bytes): + mov 4(%rsi), %rdx + mov (%rsi), %rcx + mov %rdx, 4(%rdi) + mov %rcx, (%rdi) + ret + + .p2align 4 +L(bwd_write_139bytes): + lddqu 123(%rsi), %xmm0 + movdqu %xmm0, 123(%rdi) +L(bwd_write_123bytes): + lddqu 107(%rsi), %xmm0 + movdqu %xmm0, 107(%rdi) +L(bwd_write_107bytes): + lddqu 91(%rsi), %xmm0 + movdqu %xmm0, 91(%rdi) +L(bwd_write_91bytes): + lddqu 75(%rsi), %xmm0 + movdqu %xmm0, 75(%rdi) +L(bwd_write_75bytes): + lddqu 59(%rsi), %xmm0 + movdqu %xmm0, 59(%rdi) +L(bwd_write_59bytes): + lddqu 43(%rsi), %xmm0 + movdqu %xmm0, 43(%rdi) +L(bwd_write_43bytes): + lddqu 27(%rsi), %xmm0 + movdqu %xmm0, 27(%rdi) +L(bwd_write_27bytes): + lddqu 11(%rsi), %xmm0 + lddqu (%rsi), %xmm1 + movdqu %xmm0, 11(%rdi) + movdqu %xmm1, (%rdi) + ret + + .p2align 4 +L(bwd_write_11bytes): + mov 3(%rsi), %rdx + mov (%rsi), %rcx + mov %rdx, 3(%rdi) + mov %rcx, (%rdi) + ret + + .p2align 4 +L(bwd_write_138bytes): + lddqu 122(%rsi), %xmm0 + movdqu %xmm0, 122(%rdi) +L(bwd_write_122bytes): + lddqu 106(%rsi), %xmm0 + movdqu %xmm0, 106(%rdi) +L(bwd_write_106bytes): + lddqu 90(%rsi), %xmm0 + movdqu %xmm0, 90(%rdi) +L(bwd_write_90bytes): + lddqu 74(%rsi), %xmm0 + movdqu %xmm0, 74(%rdi) +L(bwd_write_74bytes): + lddqu 58(%rsi), %xmm0 + movdqu %xmm0, 58(%rdi) +L(bwd_write_58bytes): + lddqu 42(%rsi), %xmm0 + movdqu %xmm0, 42(%rdi) +L(bwd_write_42bytes): + lddqu 26(%rsi), %xmm0 + movdqu %xmm0, 26(%rdi) +L(bwd_write_26bytes): + lddqu 10(%rsi), %xmm0 + lddqu (%rsi), %xmm1 + movdqu %xmm0, 10(%rdi) + movdqu %xmm1, (%rdi) + ret + + .p2align 4 +L(bwd_write_10bytes): + mov 2(%rsi), %rdx + mov (%rsi), %rcx + mov %rdx, 2(%rdi) + mov %rcx, (%rdi) + ret + + .p2align 4 +L(bwd_write_137bytes): + lddqu 121(%rsi), %xmm0 + movdqu %xmm0, 121(%rdi) +L(bwd_write_121bytes): + lddqu 105(%rsi), %xmm0 + movdqu %xmm0, 105(%rdi) +L(bwd_write_105bytes): + lddqu 89(%rsi), %xmm0 + movdqu %xmm0, 89(%rdi) +L(bwd_write_89bytes): + lddqu 73(%rsi), %xmm0 + movdqu %xmm0, 73(%rdi) +L(bwd_write_73bytes): + lddqu 57(%rsi), %xmm0 + movdqu %xmm0, 57(%rdi) +L(bwd_write_57bytes): + lddqu 41(%rsi), %xmm0 + movdqu %xmm0, 41(%rdi) +L(bwd_write_41bytes): + lddqu 25(%rsi), %xmm0 + movdqu %xmm0, 25(%rdi) +L(bwd_write_25bytes): + lddqu 9(%rsi), %xmm0 + lddqu (%rsi), %xmm1 + movdqu %xmm0, 9(%rdi) + movdqu %xmm1, (%rdi) + ret + + .p2align 4 +L(bwd_write_9bytes): + mov 1(%rsi), %rdx + mov (%rsi), %rcx + mov %rdx, 1(%rdi) + mov %rcx, (%rdi) + ret + + .p2align 4 +L(bwd_write_136bytes): + lddqu 120(%rsi), %xmm0 + movdqu %xmm0, 120(%rdi) +L(bwd_write_120bytes): + lddqu 104(%rsi), %xmm0 + movdqu %xmm0, 104(%rdi) +L(bwd_write_104bytes): + lddqu 88(%rsi), %xmm0 + movdqu %xmm0, 88(%rdi) +L(bwd_write_88bytes): + lddqu 72(%rsi), %xmm0 + movdqu %xmm0, 72(%rdi) +L(bwd_write_72bytes): + lddqu 56(%rsi), %xmm0 + movdqu %xmm0, 56(%rdi) +L(bwd_write_56bytes): + lddqu 40(%rsi), %xmm0 + movdqu %xmm0, 40(%rdi) +L(bwd_write_40bytes): + lddqu 24(%rsi), %xmm0 + movdqu %xmm0, 24(%rdi) +L(bwd_write_24bytes): + lddqu 8(%rsi), %xmm0 + lddqu (%rsi), %xmm1 + movdqu %xmm0, 8(%rdi) + movdqu %xmm1, (%rdi) + ret + + .p2align 4 +L(bwd_write_8bytes): + mov (%rsi), %rdx + mov %rdx, (%rdi) + ret + + .p2align 4 +L(bwd_write_135bytes): + lddqu 119(%rsi), %xmm0 + movdqu %xmm0, 119(%rdi) +L(bwd_write_119bytes): + lddqu 103(%rsi), %xmm0 + movdqu %xmm0, 103(%rdi) +L(bwd_write_103bytes): + lddqu 87(%rsi), %xmm0 + movdqu %xmm0, 87(%rdi) +L(bwd_write_87bytes): + lddqu 71(%rsi), %xmm0 + movdqu %xmm0, 71(%rdi) +L(bwd_write_71bytes): + lddqu 55(%rsi), %xmm0 + movdqu %xmm0, 55(%rdi) +L(bwd_write_55bytes): + lddqu 39(%rsi), %xmm0 + movdqu %xmm0, 39(%rdi) +L(bwd_write_39bytes): + lddqu 23(%rsi), %xmm0 + movdqu %xmm0, 23(%rdi) +L(bwd_write_23bytes): + lddqu 7(%rsi), %xmm0 + lddqu (%rsi), %xmm1 + movdqu %xmm0, 7(%rdi) + movdqu %xmm1, (%rdi) + ret + + .p2align 4 +L(bwd_write_7bytes): + mov 3(%rsi), %edx + mov (%rsi), %ecx + mov %edx, 3(%rdi) + mov %ecx, (%rdi) + ret + + .p2align 4 +L(bwd_write_134bytes): + lddqu 118(%rsi), %xmm0 + movdqu %xmm0, 118(%rdi) +L(bwd_write_118bytes): + lddqu 102(%rsi), %xmm0 + movdqu %xmm0, 102(%rdi) +L(bwd_write_102bytes): + lddqu 86(%rsi), %xmm0 + movdqu %xmm0, 86(%rdi) +L(bwd_write_86bytes): + lddqu 70(%rsi), %xmm0 + movdqu %xmm0, 70(%rdi) +L(bwd_write_70bytes): + lddqu 54(%rsi), %xmm0 + movdqu %xmm0, 54(%rdi) +L(bwd_write_54bytes): + lddqu 38(%rsi), %xmm0 + movdqu %xmm0, 38(%rdi) +L(bwd_write_38bytes): + lddqu 22(%rsi), %xmm0 + movdqu %xmm0, 22(%rdi) +L(bwd_write_22bytes): + lddqu 6(%rsi), %xmm0 + lddqu (%rsi), %xmm1 + movdqu %xmm0, 6(%rdi) + movdqu %xmm1, (%rdi) + ret + + .p2align 4 +L(bwd_write_6bytes): + mov 2(%rsi), %edx + mov (%rsi), %ecx + mov %edx, 2(%rdi) + mov %ecx, (%rdi) + ret + + .p2align 4 +L(bwd_write_133bytes): + lddqu 117(%rsi), %xmm0 + movdqu %xmm0, 117(%rdi) +L(bwd_write_117bytes): + lddqu 101(%rsi), %xmm0 + movdqu %xmm0, 101(%rdi) +L(bwd_write_101bytes): + lddqu 85(%rsi), %xmm0 + movdqu %xmm0, 85(%rdi) +L(bwd_write_85bytes): + lddqu 69(%rsi), %xmm0 + movdqu %xmm0, 69(%rdi) +L(bwd_write_69bytes): + lddqu 53(%rsi), %xmm0 + movdqu %xmm0, 53(%rdi) +L(bwd_write_53bytes): + lddqu 37(%rsi), %xmm0 + movdqu %xmm0, 37(%rdi) +L(bwd_write_37bytes): + lddqu 21(%rsi), %xmm0 + movdqu %xmm0, 21(%rdi) +L(bwd_write_21bytes): + lddqu 5(%rsi), %xmm0 + lddqu (%rsi), %xmm1 + movdqu %xmm0, 5(%rdi) + movdqu %xmm1, (%rdi) + ret + + .p2align 4 +L(bwd_write_5bytes): + mov 1(%rsi), %edx + mov (%rsi), %ecx + mov %edx, 1(%rdi) + mov %ecx, (%rdi) + ret + + .p2align 4 +L(bwd_write_132bytes): + lddqu 116(%rsi), %xmm0 + movdqu %xmm0, 116(%rdi) +L(bwd_write_116bytes): + lddqu 100(%rsi), %xmm0 + movdqu %xmm0, 100(%rdi) +L(bwd_write_100bytes): + lddqu 84(%rsi), %xmm0 + movdqu %xmm0, 84(%rdi) +L(bwd_write_84bytes): + lddqu 68(%rsi), %xmm0 + movdqu %xmm0, 68(%rdi) +L(bwd_write_68bytes): + lddqu 52(%rsi), %xmm0 + movdqu %xmm0, 52(%rdi) +L(bwd_write_52bytes): + lddqu 36(%rsi), %xmm0 + movdqu %xmm0, 36(%rdi) +L(bwd_write_36bytes): + lddqu 20(%rsi), %xmm0 + movdqu %xmm0, 20(%rdi) +L(bwd_write_20bytes): + lddqu 4(%rsi), %xmm0 + lddqu (%rsi), %xmm1 + movdqu %xmm0, 4(%rdi) + movdqu %xmm1, (%rdi) + ret + + .p2align 4 +L(bwd_write_4bytes): + mov (%rsi), %edx + mov %edx, (%rdi) + ret + + .p2align 4 +L(bwd_write_131bytes): + lddqu 115(%rsi), %xmm0 + movdqu %xmm0, 115(%rdi) +L(bwd_write_115bytes): + lddqu 99(%rsi), %xmm0 + movdqu %xmm0, 99(%rdi) +L(bwd_write_99bytes): + lddqu 83(%rsi), %xmm0 + movdqu %xmm0, 83(%rdi) +L(bwd_write_83bytes): + lddqu 67(%rsi), %xmm0 + movdqu %xmm0, 67(%rdi) +L(bwd_write_67bytes): + lddqu 51(%rsi), %xmm0 + movdqu %xmm0, 51(%rdi) +L(bwd_write_51bytes): + lddqu 35(%rsi), %xmm0 + movdqu %xmm0, 35(%rdi) +L(bwd_write_35bytes): + lddqu 19(%rsi), %xmm0 + movdqu %xmm0, 19(%rdi) +L(bwd_write_19bytes): + lddqu 3(%rsi), %xmm0 + lddqu (%rsi), %xmm1 + movdqu %xmm0, 3(%rdi) + movdqu %xmm1, (%rdi) + ret + + .p2align 4 +L(bwd_write_3bytes): + mov 1(%rsi), %dx + mov (%rsi), %cx + mov %dx, 1(%rdi) + mov %cx, (%rdi) + ret + + .p2align 4 +L(bwd_write_130bytes): + lddqu 114(%rsi), %xmm0 + movdqu %xmm0, 114(%rdi) +L(bwd_write_114bytes): + lddqu 98(%rsi), %xmm0 + movdqu %xmm0, 98(%rdi) +L(bwd_write_98bytes): + lddqu 82(%rsi), %xmm0 + movdqu %xmm0, 82(%rdi) +L(bwd_write_82bytes): + lddqu 66(%rsi), %xmm0 + movdqu %xmm0, 66(%rdi) +L(bwd_write_66bytes): + lddqu 50(%rsi), %xmm0 + movdqu %xmm0, 50(%rdi) +L(bwd_write_50bytes): + lddqu 34(%rsi), %xmm0 + movdqu %xmm0, 34(%rdi) +L(bwd_write_34bytes): + lddqu 18(%rsi), %xmm0 + movdqu %xmm0, 18(%rdi) +L(bwd_write_18bytes): + lddqu 2(%rsi), %xmm0 + lddqu (%rsi), %xmm1 + movdqu %xmm0, 2(%rdi) + movdqu %xmm1, (%rdi) + ret + + .p2align 4 +L(bwd_write_2bytes): + movzwl (%rsi), %edx + mov %dx, (%rdi) + ret + + .p2align 4 +L(bwd_write_129bytes): + lddqu 113(%rsi), %xmm0 + movdqu %xmm0, 113(%rdi) +L(bwd_write_113bytes): + lddqu 97(%rsi), %xmm0 + movdqu %xmm0, 97(%rdi) +L(bwd_write_97bytes): + lddqu 81(%rsi), %xmm0 + movdqu %xmm0, 81(%rdi) +L(bwd_write_81bytes): + lddqu 65(%rsi), %xmm0 + movdqu %xmm0, 65(%rdi) +L(bwd_write_65bytes): + lddqu 49(%rsi), %xmm0 + movdqu %xmm0, 49(%rdi) +L(bwd_write_49bytes): + lddqu 33(%rsi), %xmm0 + movdqu %xmm0, 33(%rdi) +L(bwd_write_33bytes): + lddqu 17(%rsi), %xmm0 + movdqu %xmm0, 17(%rdi) +L(bwd_write_17bytes): + lddqu 1(%rsi), %xmm0 + lddqu (%rsi), %xmm1 + movdqu %xmm0, 1(%rdi) + movdqu %xmm1, (%rdi) + ret + + .p2align 4 +L(bwd_write_1bytes): + movzbl (%rsi), %edx + mov %dl, (%rdi) + ret + +END (MEMCPY) + + .section .rodata.ssse3,"a",@progbits + .p2align 3 +L(table_144_bytes_bwd): + .int JMPTBL (L(bwd_write_0bytes), L(table_144_bytes_bwd)) + .int JMPTBL (L(bwd_write_1bytes), L(table_144_bytes_bwd)) + .int JMPTBL (L(bwd_write_2bytes), L(table_144_bytes_bwd)) + .int JMPTBL (L(bwd_write_3bytes), L(table_144_bytes_bwd)) + .int JMPTBL (L(bwd_write_4bytes), L(table_144_bytes_bwd)) + .int JMPTBL (L(bwd_write_5bytes), L(table_144_bytes_bwd)) + .int JMPTBL (L(bwd_write_6bytes), L(table_144_bytes_bwd)) + .int JMPTBL (L(bwd_write_7bytes), L(table_144_bytes_bwd)) + .int JMPTBL (L(bwd_write_8bytes), L(table_144_bytes_bwd)) + .int JMPTBL (L(bwd_write_9bytes), L(table_144_bytes_bwd)) + .int JMPTBL (L(bwd_write_10bytes), L(table_144_bytes_bwd)) + .int JMPTBL (L(bwd_write_11bytes), L(table_144_bytes_bwd)) + .int JMPTBL (L(bwd_write_12bytes), L(table_144_bytes_bwd)) + .int JMPTBL (L(bwd_write_13bytes), L(table_144_bytes_bwd)) + .int JMPTBL (L(bwd_write_14bytes), L(table_144_bytes_bwd)) + .int JMPTBL (L(bwd_write_15bytes), L(table_144_bytes_bwd)) + .int JMPTBL (L(bwd_write_16bytes), L(table_144_bytes_bwd)) + .int JMPTBL (L(bwd_write_17bytes), L(table_144_bytes_bwd)) + .int JMPTBL (L(bwd_write_18bytes), L(table_144_bytes_bwd)) + .int JMPTBL (L(bwd_write_19bytes), L(table_144_bytes_bwd)) + .int JMPTBL (L(bwd_write_20bytes), L(table_144_bytes_bwd)) + .int JMPTBL (L(bwd_write_21bytes), L(table_144_bytes_bwd)) + .int JMPTBL (L(bwd_write_22bytes), L(table_144_bytes_bwd)) + .int JMPTBL (L(bwd_write_23bytes), L(table_144_bytes_bwd)) + .int JMPTBL (L(bwd_write_24bytes), L(table_144_bytes_bwd)) + .int JMPTBL (L(bwd_write_25bytes), L(table_144_bytes_bwd)) + .int JMPTBL (L(bwd_write_26bytes), L(table_144_bytes_bwd)) + .int JMPTBL (L(bwd_write_27bytes), L(table_144_bytes_bwd)) + .int JMPTBL (L(bwd_write_28bytes), L(table_144_bytes_bwd)) + .int JMPTBL (L(bwd_write_29bytes), L(table_144_bytes_bwd)) + .int JMPTBL (L(bwd_write_30bytes), L(table_144_bytes_bwd)) + .int JMPTBL (L(bwd_write_31bytes), L(table_144_bytes_bwd)) + .int JMPTBL (L(bwd_write_32bytes), L(table_144_bytes_bwd)) + .int JMPTBL (L(bwd_write_33bytes), L(table_144_bytes_bwd)) + .int JMPTBL (L(bwd_write_34bytes), L(table_144_bytes_bwd)) + .int JMPTBL (L(bwd_write_35bytes), L(table_144_bytes_bwd)) + .int JMPTBL (L(bwd_write_36bytes), L(table_144_bytes_bwd)) + .int JMPTBL (L(bwd_write_37bytes), L(table_144_bytes_bwd)) + .int JMPTBL (L(bwd_write_38bytes), L(table_144_bytes_bwd)) + .int JMPTBL (L(bwd_write_39bytes), L(table_144_bytes_bwd)) + .int JMPTBL (L(bwd_write_40bytes), L(table_144_bytes_bwd)) + .int JMPTBL (L(bwd_write_41bytes), L(table_144_bytes_bwd)) + .int JMPTBL (L(bwd_write_42bytes), L(table_144_bytes_bwd)) + .int JMPTBL (L(bwd_write_43bytes), L(table_144_bytes_bwd)) + .int JMPTBL (L(bwd_write_44bytes), L(table_144_bytes_bwd)) + .int JMPTBL (L(bwd_write_45bytes), L(table_144_bytes_bwd)) + .int JMPTBL (L(bwd_write_46bytes), L(table_144_bytes_bwd)) + .int JMPTBL (L(bwd_write_47bytes), L(table_144_bytes_bwd)) + .int JMPTBL (L(bwd_write_48bytes), L(table_144_bytes_bwd)) + .int JMPTBL (L(bwd_write_49bytes), L(table_144_bytes_bwd)) + .int JMPTBL (L(bwd_write_50bytes), L(table_144_bytes_bwd)) + .int JMPTBL (L(bwd_write_51bytes), L(table_144_bytes_bwd)) + .int JMPTBL (L(bwd_write_52bytes), L(table_144_bytes_bwd)) + .int JMPTBL (L(bwd_write_53bytes), L(table_144_bytes_bwd)) + .int JMPTBL (L(bwd_write_54bytes), L(table_144_bytes_bwd)) + .int JMPTBL (L(bwd_write_55bytes), L(table_144_bytes_bwd)) + .int JMPTBL (L(bwd_write_56bytes), L(table_144_bytes_bwd)) + .int JMPTBL (L(bwd_write_57bytes), L(table_144_bytes_bwd)) + .int JMPTBL (L(bwd_write_58bytes), L(table_144_bytes_bwd)) + .int JMPTBL (L(bwd_write_59bytes), L(table_144_bytes_bwd)) + .int JMPTBL (L(bwd_write_60bytes), L(table_144_bytes_bwd)) + .int JMPTBL (L(bwd_write_61bytes), L(table_144_bytes_bwd)) + .int JMPTBL (L(bwd_write_62bytes), L(table_144_bytes_bwd)) + .int JMPTBL (L(bwd_write_63bytes), L(table_144_bytes_bwd)) + .int JMPTBL (L(bwd_write_64bytes), L(table_144_bytes_bwd)) + .int JMPTBL (L(bwd_write_65bytes), L(table_144_bytes_bwd)) + .int JMPTBL (L(bwd_write_66bytes), L(table_144_bytes_bwd)) + .int JMPTBL (L(bwd_write_67bytes), L(table_144_bytes_bwd)) + .int JMPTBL (L(bwd_write_68bytes), L(table_144_bytes_bwd)) + .int JMPTBL (L(bwd_write_69bytes), L(table_144_bytes_bwd)) + .int JMPTBL (L(bwd_write_70bytes), L(table_144_bytes_bwd)) + .int JMPTBL (L(bwd_write_71bytes), L(table_144_bytes_bwd)) + .int JMPTBL (L(bwd_write_72bytes), L(table_144_bytes_bwd)) + .int JMPTBL (L(bwd_write_73bytes), L(table_144_bytes_bwd)) + .int JMPTBL (L(bwd_write_74bytes), L(table_144_bytes_bwd)) + .int JMPTBL (L(bwd_write_75bytes), L(table_144_bytes_bwd)) + .int JMPTBL (L(bwd_write_76bytes), L(table_144_bytes_bwd)) + .int JMPTBL (L(bwd_write_77bytes), L(table_144_bytes_bwd)) + .int JMPTBL (L(bwd_write_78bytes), L(table_144_bytes_bwd)) + .int JMPTBL (L(bwd_write_79bytes), L(table_144_bytes_bwd)) + .int JMPTBL (L(bwd_write_80bytes), L(table_144_bytes_bwd)) + .int JMPTBL (L(bwd_write_81bytes), L(table_144_bytes_bwd)) + .int JMPTBL (L(bwd_write_82bytes), L(table_144_bytes_bwd)) + .int JMPTBL (L(bwd_write_83bytes), L(table_144_bytes_bwd)) + .int JMPTBL (L(bwd_write_84bytes), L(table_144_bytes_bwd)) + .int JMPTBL (L(bwd_write_85bytes), L(table_144_bytes_bwd)) + .int JMPTBL (L(bwd_write_86bytes), L(table_144_bytes_bwd)) + .int JMPTBL (L(bwd_write_87bytes), L(table_144_bytes_bwd)) + .int JMPTBL (L(bwd_write_88bytes), L(table_144_bytes_bwd)) + .int JMPTBL (L(bwd_write_89bytes), L(table_144_bytes_bwd)) + .int JMPTBL (L(bwd_write_90bytes), L(table_144_bytes_bwd)) + .int JMPTBL (L(bwd_write_91bytes), L(table_144_bytes_bwd)) + .int JMPTBL (L(bwd_write_92bytes), L(table_144_bytes_bwd)) + .int JMPTBL (L(bwd_write_93bytes), L(table_144_bytes_bwd)) + .int JMPTBL (L(bwd_write_94bytes), L(table_144_bytes_bwd)) + .int JMPTBL (L(bwd_write_95bytes), L(table_144_bytes_bwd)) + .int JMPTBL (L(bwd_write_96bytes), L(table_144_bytes_bwd)) + .int JMPTBL (L(bwd_write_97bytes), L(table_144_bytes_bwd)) + .int JMPTBL (L(bwd_write_98bytes), L(table_144_bytes_bwd)) + .int JMPTBL (L(bwd_write_99bytes), L(table_144_bytes_bwd)) + .int JMPTBL (L(bwd_write_100bytes), L(table_144_bytes_bwd)) + .int JMPTBL (L(bwd_write_101bytes), L(table_144_bytes_bwd)) + .int JMPTBL (L(bwd_write_102bytes), L(table_144_bytes_bwd)) + .int JMPTBL (L(bwd_write_103bytes), L(table_144_bytes_bwd)) + .int JMPTBL (L(bwd_write_104bytes), L(table_144_bytes_bwd)) + .int JMPTBL (L(bwd_write_105bytes), L(table_144_bytes_bwd)) + .int JMPTBL (L(bwd_write_106bytes), L(table_144_bytes_bwd)) + .int JMPTBL (L(bwd_write_107bytes), L(table_144_bytes_bwd)) + .int JMPTBL (L(bwd_write_108bytes), L(table_144_bytes_bwd)) + .int JMPTBL (L(bwd_write_109bytes), L(table_144_bytes_bwd)) + .int JMPTBL (L(bwd_write_110bytes), L(table_144_bytes_bwd)) + .int JMPTBL (L(bwd_write_111bytes), L(table_144_bytes_bwd)) + .int JMPTBL (L(bwd_write_112bytes), L(table_144_bytes_bwd)) + .int JMPTBL (L(bwd_write_113bytes), L(table_144_bytes_bwd)) + .int JMPTBL (L(bwd_write_114bytes), L(table_144_bytes_bwd)) + .int JMPTBL (L(bwd_write_115bytes), L(table_144_bytes_bwd)) + .int JMPTBL (L(bwd_write_116bytes), L(table_144_bytes_bwd)) + .int JMPTBL (L(bwd_write_117bytes), L(table_144_bytes_bwd)) + .int JMPTBL (L(bwd_write_118bytes), L(table_144_bytes_bwd)) + .int JMPTBL (L(bwd_write_119bytes), L(table_144_bytes_bwd)) + .int JMPTBL (L(bwd_write_120bytes), L(table_144_bytes_bwd)) + .int JMPTBL (L(bwd_write_121bytes), L(table_144_bytes_bwd)) + .int JMPTBL (L(bwd_write_122bytes), L(table_144_bytes_bwd)) + .int JMPTBL (L(bwd_write_123bytes), L(table_144_bytes_bwd)) + .int JMPTBL (L(bwd_write_124bytes), L(table_144_bytes_bwd)) + .int JMPTBL (L(bwd_write_125bytes), L(table_144_bytes_bwd)) + .int JMPTBL (L(bwd_write_126bytes), L(table_144_bytes_bwd)) + .int JMPTBL (L(bwd_write_127bytes), L(table_144_bytes_bwd)) + .int JMPTBL (L(bwd_write_128bytes), L(table_144_bytes_bwd)) + .int JMPTBL (L(bwd_write_129bytes), L(table_144_bytes_bwd)) + .int JMPTBL (L(bwd_write_130bytes), L(table_144_bytes_bwd)) + .int JMPTBL (L(bwd_write_131bytes), L(table_144_bytes_bwd)) + .int JMPTBL (L(bwd_write_132bytes), L(table_144_bytes_bwd)) + .int JMPTBL (L(bwd_write_133bytes), L(table_144_bytes_bwd)) + .int JMPTBL (L(bwd_write_134bytes), L(table_144_bytes_bwd)) + .int JMPTBL (L(bwd_write_135bytes), L(table_144_bytes_bwd)) + .int JMPTBL (L(bwd_write_136bytes), L(table_144_bytes_bwd)) + .int JMPTBL (L(bwd_write_137bytes), L(table_144_bytes_bwd)) + .int JMPTBL (L(bwd_write_138bytes), L(table_144_bytes_bwd)) + .int JMPTBL (L(bwd_write_139bytes), L(table_144_bytes_bwd)) + .int JMPTBL (L(bwd_write_140bytes), L(table_144_bytes_bwd)) + .int JMPTBL (L(bwd_write_141bytes), L(table_144_bytes_bwd)) + .int JMPTBL (L(bwd_write_142bytes), L(table_144_bytes_bwd)) + .int JMPTBL (L(bwd_write_143bytes), L(table_144_bytes_bwd)) + + .p2align 3 +L(table_144_bytes_fwd): + .int JMPTBL (L(fwd_write_0bytes), L(table_144_bytes_fwd)) + .int JMPTBL (L(fwd_write_1bytes), L(table_144_bytes_fwd)) + .int JMPTBL (L(fwd_write_2bytes), L(table_144_bytes_fwd)) + .int JMPTBL (L(fwd_write_3bytes), L(table_144_bytes_fwd)) + .int JMPTBL (L(fwd_write_4bytes), L(table_144_bytes_fwd)) + .int JMPTBL (L(fwd_write_5bytes), L(table_144_bytes_fwd)) + .int JMPTBL (L(fwd_write_6bytes), L(table_144_bytes_fwd)) + .int JMPTBL (L(fwd_write_7bytes), L(table_144_bytes_fwd)) + .int JMPTBL (L(fwd_write_8bytes), L(table_144_bytes_fwd)) + .int JMPTBL (L(fwd_write_9bytes), L(table_144_bytes_fwd)) + .int JMPTBL (L(fwd_write_10bytes), L(table_144_bytes_fwd)) + .int JMPTBL (L(fwd_write_11bytes), L(table_144_bytes_fwd)) + .int JMPTBL (L(fwd_write_12bytes), L(table_144_bytes_fwd)) + .int JMPTBL (L(fwd_write_13bytes), L(table_144_bytes_fwd)) + .int JMPTBL (L(fwd_write_14bytes), L(table_144_bytes_fwd)) + .int JMPTBL (L(fwd_write_15bytes), L(table_144_bytes_fwd)) + .int JMPTBL (L(fwd_write_16bytes), L(table_144_bytes_fwd)) + .int JMPTBL (L(fwd_write_17bytes), L(table_144_bytes_fwd)) + .int JMPTBL (L(fwd_write_18bytes), L(table_144_bytes_fwd)) + .int JMPTBL (L(fwd_write_19bytes), L(table_144_bytes_fwd)) + .int JMPTBL (L(fwd_write_20bytes), L(table_144_bytes_fwd)) + .int JMPTBL (L(fwd_write_21bytes), L(table_144_bytes_fwd)) + .int JMPTBL (L(fwd_write_22bytes), L(table_144_bytes_fwd)) + .int JMPTBL (L(fwd_write_23bytes), L(table_144_bytes_fwd)) + .int JMPTBL (L(fwd_write_24bytes), L(table_144_bytes_fwd)) + .int JMPTBL (L(fwd_write_25bytes), L(table_144_bytes_fwd)) + .int JMPTBL (L(fwd_write_26bytes), L(table_144_bytes_fwd)) + .int JMPTBL (L(fwd_write_27bytes), L(table_144_bytes_fwd)) + .int JMPTBL (L(fwd_write_28bytes), L(table_144_bytes_fwd)) + .int JMPTBL (L(fwd_write_29bytes), L(table_144_bytes_fwd)) + .int JMPTBL (L(fwd_write_30bytes), L(table_144_bytes_fwd)) + .int JMPTBL (L(fwd_write_31bytes), L(table_144_bytes_fwd)) + .int JMPTBL (L(fwd_write_32bytes), L(table_144_bytes_fwd)) + .int JMPTBL (L(fwd_write_33bytes), L(table_144_bytes_fwd)) + .int JMPTBL (L(fwd_write_34bytes), L(table_144_bytes_fwd)) + .int JMPTBL (L(fwd_write_35bytes), L(table_144_bytes_fwd)) + .int JMPTBL (L(fwd_write_36bytes), L(table_144_bytes_fwd)) + .int JMPTBL (L(fwd_write_37bytes), L(table_144_bytes_fwd)) + .int JMPTBL (L(fwd_write_38bytes), L(table_144_bytes_fwd)) + .int JMPTBL (L(fwd_write_39bytes), L(table_144_bytes_fwd)) + .int JMPTBL (L(fwd_write_40bytes), L(table_144_bytes_fwd)) + .int JMPTBL (L(fwd_write_41bytes), L(table_144_bytes_fwd)) + .int JMPTBL (L(fwd_write_42bytes), L(table_144_bytes_fwd)) + .int JMPTBL (L(fwd_write_43bytes), L(table_144_bytes_fwd)) + .int JMPTBL (L(fwd_write_44bytes), L(table_144_bytes_fwd)) + .int JMPTBL (L(fwd_write_45bytes), L(table_144_bytes_fwd)) + .int JMPTBL (L(fwd_write_46bytes), L(table_144_bytes_fwd)) + .int JMPTBL (L(fwd_write_47bytes), L(table_144_bytes_fwd)) + .int JMPTBL (L(fwd_write_48bytes), L(table_144_bytes_fwd)) + .int JMPTBL (L(fwd_write_49bytes), L(table_144_bytes_fwd)) + .int JMPTBL (L(fwd_write_50bytes), L(table_144_bytes_fwd)) + .int JMPTBL (L(fwd_write_51bytes), L(table_144_bytes_fwd)) + .int JMPTBL (L(fwd_write_52bytes), L(table_144_bytes_fwd)) + .int JMPTBL (L(fwd_write_53bytes), L(table_144_bytes_fwd)) + .int JMPTBL (L(fwd_write_54bytes), L(table_144_bytes_fwd)) + .int JMPTBL (L(fwd_write_55bytes), L(table_144_bytes_fwd)) + .int JMPTBL (L(fwd_write_56bytes), L(table_144_bytes_fwd)) + .int JMPTBL (L(fwd_write_57bytes), L(table_144_bytes_fwd)) + .int JMPTBL (L(fwd_write_58bytes), L(table_144_bytes_fwd)) + .int JMPTBL (L(fwd_write_59bytes), L(table_144_bytes_fwd)) + .int JMPTBL (L(fwd_write_60bytes), L(table_144_bytes_fwd)) + .int JMPTBL (L(fwd_write_61bytes), L(table_144_bytes_fwd)) + .int JMPTBL (L(fwd_write_62bytes), L(table_144_bytes_fwd)) + .int JMPTBL (L(fwd_write_63bytes), L(table_144_bytes_fwd)) + .int JMPTBL (L(fwd_write_64bytes), L(table_144_bytes_fwd)) + .int JMPTBL (L(fwd_write_65bytes), L(table_144_bytes_fwd)) + .int JMPTBL (L(fwd_write_66bytes), L(table_144_bytes_fwd)) + .int JMPTBL (L(fwd_write_67bytes), L(table_144_bytes_fwd)) + .int JMPTBL (L(fwd_write_68bytes), L(table_144_bytes_fwd)) + .int JMPTBL (L(fwd_write_69bytes), L(table_144_bytes_fwd)) + .int JMPTBL (L(fwd_write_70bytes), L(table_144_bytes_fwd)) + .int JMPTBL (L(fwd_write_71bytes), L(table_144_bytes_fwd)) + .int JMPTBL (L(fwd_write_72bytes), L(table_144_bytes_fwd)) + .int JMPTBL (L(fwd_write_73bytes), L(table_144_bytes_fwd)) + .int JMPTBL (L(fwd_write_74bytes), L(table_144_bytes_fwd)) + .int JMPTBL (L(fwd_write_75bytes), L(table_144_bytes_fwd)) + .int JMPTBL (L(fwd_write_76bytes), L(table_144_bytes_fwd)) + .int JMPTBL (L(fwd_write_77bytes), L(table_144_bytes_fwd)) + .int JMPTBL (L(fwd_write_78bytes), L(table_144_bytes_fwd)) + .int JMPTBL (L(fwd_write_79bytes), L(table_144_bytes_fwd)) + .int JMPTBL (L(fwd_write_80bytes), L(table_144_bytes_fwd)) + .int JMPTBL (L(fwd_write_81bytes), L(table_144_bytes_fwd)) + .int JMPTBL (L(fwd_write_82bytes), L(table_144_bytes_fwd)) + .int JMPTBL (L(fwd_write_83bytes), L(table_144_bytes_fwd)) + .int JMPTBL (L(fwd_write_84bytes), L(table_144_bytes_fwd)) + .int JMPTBL (L(fwd_write_85bytes), L(table_144_bytes_fwd)) + .int JMPTBL (L(fwd_write_86bytes), L(table_144_bytes_fwd)) + .int JMPTBL (L(fwd_write_87bytes), L(table_144_bytes_fwd)) + .int JMPTBL (L(fwd_write_88bytes), L(table_144_bytes_fwd)) + .int JMPTBL (L(fwd_write_89bytes), L(table_144_bytes_fwd)) + .int JMPTBL (L(fwd_write_90bytes), L(table_144_bytes_fwd)) + .int JMPTBL (L(fwd_write_91bytes), L(table_144_bytes_fwd)) + .int JMPTBL (L(fwd_write_92bytes), L(table_144_bytes_fwd)) + .int JMPTBL (L(fwd_write_93bytes), L(table_144_bytes_fwd)) + .int JMPTBL (L(fwd_write_94bytes), L(table_144_bytes_fwd)) + .int JMPTBL (L(fwd_write_95bytes), L(table_144_bytes_fwd)) + .int JMPTBL (L(fwd_write_96bytes), L(table_144_bytes_fwd)) + .int JMPTBL (L(fwd_write_97bytes), L(table_144_bytes_fwd)) + .int JMPTBL (L(fwd_write_98bytes), L(table_144_bytes_fwd)) + .int JMPTBL (L(fwd_write_99bytes), L(table_144_bytes_fwd)) + .int JMPTBL (L(fwd_write_100bytes), L(table_144_bytes_fwd)) + .int JMPTBL (L(fwd_write_101bytes), L(table_144_bytes_fwd)) + .int JMPTBL (L(fwd_write_102bytes), L(table_144_bytes_fwd)) + .int JMPTBL (L(fwd_write_103bytes), L(table_144_bytes_fwd)) + .int JMPTBL (L(fwd_write_104bytes), L(table_144_bytes_fwd)) + .int JMPTBL (L(fwd_write_105bytes), L(table_144_bytes_fwd)) + .int JMPTBL (L(fwd_write_106bytes), L(table_144_bytes_fwd)) + .int JMPTBL (L(fwd_write_107bytes), L(table_144_bytes_fwd)) + .int JMPTBL (L(fwd_write_108bytes), L(table_144_bytes_fwd)) + .int JMPTBL (L(fwd_write_109bytes), L(table_144_bytes_fwd)) + .int JMPTBL (L(fwd_write_110bytes), L(table_144_bytes_fwd)) + .int JMPTBL (L(fwd_write_111bytes), L(table_144_bytes_fwd)) + .int JMPTBL (L(fwd_write_112bytes), L(table_144_bytes_fwd)) + .int JMPTBL (L(fwd_write_113bytes), L(table_144_bytes_fwd)) + .int JMPTBL (L(fwd_write_114bytes), L(table_144_bytes_fwd)) + .int JMPTBL (L(fwd_write_115bytes), L(table_144_bytes_fwd)) + .int JMPTBL (L(fwd_write_116bytes), L(table_144_bytes_fwd)) + .int JMPTBL (L(fwd_write_117bytes), L(table_144_bytes_fwd)) + .int JMPTBL (L(fwd_write_118bytes), L(table_144_bytes_fwd)) + .int JMPTBL (L(fwd_write_119bytes), L(table_144_bytes_fwd)) + .int JMPTBL (L(fwd_write_120bytes), L(table_144_bytes_fwd)) + .int JMPTBL (L(fwd_write_121bytes), L(table_144_bytes_fwd)) + .int JMPTBL (L(fwd_write_122bytes), L(table_144_bytes_fwd)) + .int JMPTBL (L(fwd_write_123bytes), L(table_144_bytes_fwd)) + .int JMPTBL (L(fwd_write_124bytes), L(table_144_bytes_fwd)) + .int JMPTBL (L(fwd_write_125bytes), L(table_144_bytes_fwd)) + .int JMPTBL (L(fwd_write_126bytes), L(table_144_bytes_fwd)) + .int JMPTBL (L(fwd_write_127bytes), L(table_144_bytes_fwd)) + .int JMPTBL (L(fwd_write_128bytes), L(table_144_bytes_fwd)) + .int JMPTBL (L(fwd_write_129bytes), L(table_144_bytes_fwd)) + .int JMPTBL (L(fwd_write_130bytes), L(table_144_bytes_fwd)) + .int JMPTBL (L(fwd_write_131bytes), L(table_144_bytes_fwd)) + .int JMPTBL (L(fwd_write_132bytes), L(table_144_bytes_fwd)) + .int JMPTBL (L(fwd_write_133bytes), L(table_144_bytes_fwd)) + .int JMPTBL (L(fwd_write_134bytes), L(table_144_bytes_fwd)) + .int JMPTBL (L(fwd_write_135bytes), L(table_144_bytes_fwd)) + .int JMPTBL (L(fwd_write_136bytes), L(table_144_bytes_fwd)) + .int JMPTBL (L(fwd_write_137bytes), L(table_144_bytes_fwd)) + .int JMPTBL (L(fwd_write_138bytes), L(table_144_bytes_fwd)) + .int JMPTBL (L(fwd_write_139bytes), L(table_144_bytes_fwd)) + .int JMPTBL (L(fwd_write_140bytes), L(table_144_bytes_fwd)) + .int JMPTBL (L(fwd_write_141bytes), L(table_144_bytes_fwd)) + .int JMPTBL (L(fwd_write_142bytes), L(table_144_bytes_fwd)) + .int JMPTBL (L(fwd_write_143bytes), L(table_144_bytes_fwd)) + + .p2align 3 +L(shl_table_fwd): + .int JMPTBL (L(shl_0), L(shl_table_fwd)) + .int JMPTBL (L(shl_1), L(shl_table_fwd)) + .int JMPTBL (L(shl_2), L(shl_table_fwd)) + .int JMPTBL (L(shl_3), L(shl_table_fwd)) + .int JMPTBL (L(shl_4), L(shl_table_fwd)) + .int JMPTBL (L(shl_5), L(shl_table_fwd)) + .int JMPTBL (L(shl_6), L(shl_table_fwd)) + .int JMPTBL (L(shl_7), L(shl_table_fwd)) + .int JMPTBL (L(shl_8), L(shl_table_fwd)) + .int JMPTBL (L(shl_9), L(shl_table_fwd)) + .int JMPTBL (L(shl_10), L(shl_table_fwd)) + .int JMPTBL (L(shl_11), L(shl_table_fwd)) + .int JMPTBL (L(shl_12), L(shl_table_fwd)) + .int JMPTBL (L(shl_13), L(shl_table_fwd)) + .int JMPTBL (L(shl_14), L(shl_table_fwd)) + .int JMPTBL (L(shl_15), L(shl_table_fwd)) + + .p2align 3 +L(shl_table_bwd): + .int JMPTBL (L(shl_0_bwd), L(shl_table_bwd)) + .int JMPTBL (L(shl_1_bwd), L(shl_table_bwd)) + .int JMPTBL (L(shl_2_bwd), L(shl_table_bwd)) + .int JMPTBL (L(shl_3_bwd), L(shl_table_bwd)) + .int JMPTBL (L(shl_4_bwd), L(shl_table_bwd)) + .int JMPTBL (L(shl_5_bwd), L(shl_table_bwd)) + .int JMPTBL (L(shl_6_bwd), L(shl_table_bwd)) + .int JMPTBL (L(shl_7_bwd), L(shl_table_bwd)) + .int JMPTBL (L(shl_8_bwd), L(shl_table_bwd)) + .int JMPTBL (L(shl_9_bwd), L(shl_table_bwd)) + .int JMPTBL (L(shl_10_bwd), L(shl_table_bwd)) + .int JMPTBL (L(shl_11_bwd), L(shl_table_bwd)) + .int JMPTBL (L(shl_12_bwd), L(shl_table_bwd)) + .int JMPTBL (L(shl_13_bwd), L(shl_table_bwd)) + .int JMPTBL (L(shl_14_bwd), L(shl_table_bwd)) + .int JMPTBL (L(shl_15_bwd), L(shl_table_bwd)) + +#endif diff --git a/utils/memcpy-bench/glibc/memcpy-ssse3.S b/utils/memcpy-bench/glibc/memcpy-ssse3.S new file mode 100644 index 00000000000..2fd26651645 --- /dev/null +++ b/utils/memcpy-bench/glibc/memcpy-ssse3.S @@ -0,0 +1,3152 @@ +/* memcpy with SSSE3 + Copyright (C) 2010-2020 Free Software Foundation, Inc. + Contributed by Intel Corporation. + This file is part of the GNU C Library. + + The GNU C Library is free software; you can redistribute it and/or + modify it under the terms of the GNU Lesser General Public + License as published by the Free Software Foundation; either + version 2.1 of the License, or (at your option) any later version. + + The GNU C Library is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + Lesser General Public License for more details. + + You should have received a copy of the GNU Lesser General Public + License along with the GNU C Library; if not, see + . */ + +#include "sysdep.h" + +#if 1 + +#include "asm-syntax.h" + +#ifndef MEMCPY +# define MEMCPY __memcpy_ssse3 +# define MEMCPY_CHK __memcpy_chk_ssse3 +# define MEMPCPY __mempcpy_ssse3 +# define MEMPCPY_CHK __mempcpy_chk_ssse3 +#endif + +#define JMPTBL(I, B) I - B + +/* Branch to an entry in a jump table. TABLE is a jump table with + relative offsets. INDEX is a register contains the index into the + jump table. SCALE is the scale of INDEX. */ +#define BRANCH_TO_JMPTBL_ENTRY(TABLE, INDEX, SCALE) \ + lea TABLE(%rip), %r11; \ + movslq (%r11, INDEX, SCALE), INDEX; \ + lea (%r11, INDEX), INDEX; \ + _CET_NOTRACK jmp *INDEX; \ + ud2 + + .section .text.ssse3,"ax",@progbits +#if !defined USE_AS_MEMPCPY && !defined USE_AS_MEMMOVE +ENTRY (MEMPCPY_CHK) + cmp %RDX_LP, %RCX_LP + jb HIDDEN_JUMPTARGET (__chk_fail) +END (MEMPCPY_CHK) + +ENTRY (MEMPCPY) + mov %RDI_LP, %RAX_LP + add %RDX_LP, %RAX_LP + jmp L(start) +END (MEMPCPY) +#endif + +#if !defined USE_AS_BCOPY +ENTRY (MEMCPY_CHK) + cmp %RDX_LP, %RCX_LP + jb HIDDEN_JUMPTARGET (__chk_fail) +END (MEMCPY_CHK) +#endif + +ENTRY (MEMCPY) + mov %RDI_LP, %RAX_LP +#ifdef USE_AS_MEMPCPY + add %RDX_LP, %RAX_LP +#endif + +#ifdef __ILP32__ + /* Clear the upper 32 bits. */ + mov %edx, %edx +#endif + +#ifdef USE_AS_MEMMOVE + cmp %rsi, %rdi + jb L(copy_forward) + je L(write_0bytes) + cmp $79, %rdx + jbe L(copy_forward) + jmp L(copy_backward) +L(copy_forward): +#endif +L(start): + cmp $79, %rdx + lea L(table_less_80bytes)(%rip), %r11 + ja L(80bytesormore) + movslq (%r11, %rdx, 4), %r9 + add %rdx, %rsi + add %rdx, %rdi + add %r11, %r9 + _CET_NOTRACK jmp *%r9 + ud2 + + .p2align 4 +L(80bytesormore): +#ifndef USE_AS_MEMMOVE + cmp %dil, %sil + jle L(copy_backward) +#endif + + movdqu (%rsi), %xmm0 + mov %rdi, %rcx + and $-16, %rdi + add $16, %rdi + mov %rcx, %r8 + sub %rdi, %rcx + add %rcx, %rdx + sub %rcx, %rsi + +#ifdef SHARED_CACHE_SIZE_HALF + mov $SHARED_CACHE_SIZE_HALF, %RCX_LP +#else + mov __x86_shared_cache_size_half(%rip), %RCX_LP +#endif + cmp %rcx, %rdx + mov %rsi, %r9 + ja L(large_page_fwd) + and $0xf, %r9 + jz L(shl_0) +#ifdef DATA_CACHE_SIZE_HALF + mov $DATA_CACHE_SIZE_HALF, %RCX_LP +#else + mov __x86_data_cache_size_half(%rip), %RCX_LP +#endif + BRANCH_TO_JMPTBL_ENTRY (L(shl_table), %r9, 4) + + .p2align 4 +L(copy_backward): + movdqu -16(%rsi, %rdx), %xmm0 + add %rdx, %rsi + lea -16(%rdi, %rdx), %r8 + add %rdx, %rdi + + mov %rdi, %rcx + and $0xf, %rcx + xor %rcx, %rdi + sub %rcx, %rdx + sub %rcx, %rsi + +#ifdef SHARED_CACHE_SIZE_HALF + mov $SHARED_CACHE_SIZE_HALF, %RCX_LP +#else + mov __x86_shared_cache_size_half(%rip), %RCX_LP +#endif + + cmp %rcx, %rdx + mov %rsi, %r9 + ja L(large_page_bwd) + and $0xf, %r9 + jz L(shl_0_bwd) +#ifdef DATA_CACHE_SIZE_HALF + mov $DATA_CACHE_SIZE_HALF, %RCX_LP +#else + mov __x86_data_cache_size_half(%rip), %RCX_LP +#endif + BRANCH_TO_JMPTBL_ENTRY (L(shl_table_bwd), %r9, 4) + + .p2align 4 +L(shl_0): + sub $16, %rdx + movdqa (%rsi), %xmm1 + add $16, %rsi + movdqa %xmm1, (%rdi) + add $16, %rdi + cmp $128, %rdx + movdqu %xmm0, (%r8) + ja L(shl_0_gobble) + cmp $64, %rdx + jb L(shl_0_less_64bytes) + movaps (%rsi), %xmm4 + movaps 16(%rsi), %xmm1 + movaps 32(%rsi), %xmm2 + movaps 48(%rsi), %xmm3 + movaps %xmm4, (%rdi) + movaps %xmm1, 16(%rdi) + movaps %xmm2, 32(%rdi) + movaps %xmm3, 48(%rdi) + sub $64, %rdx + add $64, %rsi + add $64, %rdi +L(shl_0_less_64bytes): + add %rdx, %rsi + add %rdx, %rdi + BRANCH_TO_JMPTBL_ENTRY (L(table_less_80bytes), %rdx, 4) + + .p2align 4 +L(shl_0_gobble): +#ifdef DATA_CACHE_SIZE_HALF + cmp $DATA_CACHE_SIZE_HALF, %RDX_LP +#else + cmp __x86_data_cache_size_half(%rip), %RDX_LP +#endif + lea -128(%rdx), %rdx + jae L(shl_0_gobble_mem_loop) +L(shl_0_gobble_cache_loop): + movdqa (%rsi), %xmm4 + movaps 0x10(%rsi), %xmm1 + movaps 0x20(%rsi), %xmm2 + movaps 0x30(%rsi), %xmm3 + + movdqa %xmm4, (%rdi) + movaps %xmm1, 0x10(%rdi) + movaps %xmm2, 0x20(%rdi) + movaps %xmm3, 0x30(%rdi) + + sub $128, %rdx + movaps 0x40(%rsi), %xmm4 + movaps 0x50(%rsi), %xmm5 + movaps 0x60(%rsi), %xmm6 + movaps 0x70(%rsi), %xmm7 + lea 0x80(%rsi), %rsi + movaps %xmm4, 0x40(%rdi) + movaps %xmm5, 0x50(%rdi) + movaps %xmm6, 0x60(%rdi) + movaps %xmm7, 0x70(%rdi) + lea 0x80(%rdi), %rdi + + jae L(shl_0_gobble_cache_loop) + cmp $-0x40, %rdx + lea 0x80(%rdx), %rdx + jl L(shl_0_cache_less_64bytes) + + movdqa (%rsi), %xmm4 + sub $0x40, %rdx + movdqa 0x10(%rsi), %xmm1 + + movdqa %xmm4, (%rdi) + movdqa %xmm1, 0x10(%rdi) + + movdqa 0x20(%rsi), %xmm4 + movdqa 0x30(%rsi), %xmm1 + add $0x40, %rsi + + movdqa %xmm4, 0x20(%rdi) + movdqa %xmm1, 0x30(%rdi) + add $0x40, %rdi +L(shl_0_cache_less_64bytes): + add %rdx, %rsi + add %rdx, %rdi + BRANCH_TO_JMPTBL_ENTRY (L(table_less_80bytes), %rdx, 4) + + .p2align 4 +L(shl_0_gobble_mem_loop): + prefetcht0 0x1c0(%rsi) + prefetcht0 0x280(%rsi) + + movdqa (%rsi), %xmm0 + movdqa 0x10(%rsi), %xmm1 + movdqa 0x20(%rsi), %xmm2 + movdqa 0x30(%rsi), %xmm3 + movdqa 0x40(%rsi), %xmm4 + movdqa 0x50(%rsi), %xmm5 + movdqa 0x60(%rsi), %xmm6 + movdqa 0x70(%rsi), %xmm7 + lea 0x80(%rsi), %rsi + sub $0x80, %rdx + movdqa %xmm0, (%rdi) + movdqa %xmm1, 0x10(%rdi) + movdqa %xmm2, 0x20(%rdi) + movdqa %xmm3, 0x30(%rdi) + movdqa %xmm4, 0x40(%rdi) + movdqa %xmm5, 0x50(%rdi) + movdqa %xmm6, 0x60(%rdi) + movdqa %xmm7, 0x70(%rdi) + lea 0x80(%rdi), %rdi + + jae L(shl_0_gobble_mem_loop) + cmp $-0x40, %rdx + lea 0x80(%rdx), %rdx + jl L(shl_0_mem_less_64bytes) + + movdqa (%rsi), %xmm0 + sub $0x40, %rdx + movdqa 0x10(%rsi), %xmm1 + + movdqa %xmm0, (%rdi) + movdqa %xmm1, 0x10(%rdi) + + movdqa 0x20(%rsi), %xmm0 + movdqa 0x30(%rsi), %xmm1 + add $0x40, %rsi + + movdqa %xmm0, 0x20(%rdi) + movdqa %xmm1, 0x30(%rdi) + add $0x40, %rdi +L(shl_0_mem_less_64bytes): + cmp $0x20, %rdx + jb L(shl_0_mem_less_32bytes) + movdqa (%rsi), %xmm0 + sub $0x20, %rdx + movdqa 0x10(%rsi), %xmm1 + add $0x20, %rsi + movdqa %xmm0, (%rdi) + movdqa %xmm1, 0x10(%rdi) + add $0x20, %rdi +L(shl_0_mem_less_32bytes): + add %rdx, %rdi + add %rdx, %rsi + BRANCH_TO_JMPTBL_ENTRY (L(table_less_80bytes), %rdx, 4) + + .p2align 4 +L(shl_0_bwd): + sub $16, %rdx + movdqa -0x10(%rsi), %xmm1 + sub $16, %rsi + movdqa %xmm1, -0x10(%rdi) + sub $16, %rdi + cmp $0x80, %rdx + movdqu %xmm0, (%r8) + ja L(shl_0_gobble_bwd) + cmp $64, %rdx + jb L(shl_0_less_64bytes_bwd) + movaps -0x10(%rsi), %xmm0 + movaps -0x20(%rsi), %xmm1 + movaps -0x30(%rsi), %xmm2 + movaps -0x40(%rsi), %xmm3 + movaps %xmm0, -0x10(%rdi) + movaps %xmm1, -0x20(%rdi) + movaps %xmm2, -0x30(%rdi) + movaps %xmm3, -0x40(%rdi) + sub $64, %rdx + sub $0x40, %rsi + sub $0x40, %rdi +L(shl_0_less_64bytes_bwd): + BRANCH_TO_JMPTBL_ENTRY (L(table_less_80bytes), %rdx, 4) + + .p2align 4 +L(shl_0_gobble_bwd): +#ifdef DATA_CACHE_SIZE_HALF + cmp $DATA_CACHE_SIZE_HALF, %RDX_LP +#else + cmp __x86_data_cache_size_half(%rip), %RDX_LP +#endif + lea -128(%rdx), %rdx + jae L(shl_0_gobble_mem_bwd_loop) +L(shl_0_gobble_bwd_loop): + movdqa -0x10(%rsi), %xmm0 + movaps -0x20(%rsi), %xmm1 + movaps -0x30(%rsi), %xmm2 + movaps -0x40(%rsi), %xmm3 + + movdqa %xmm0, -0x10(%rdi) + movaps %xmm1, -0x20(%rdi) + movaps %xmm2, -0x30(%rdi) + movaps %xmm3, -0x40(%rdi) + + sub $0x80, %rdx + movaps -0x50(%rsi), %xmm4 + movaps -0x60(%rsi), %xmm5 + movaps -0x70(%rsi), %xmm6 + movaps -0x80(%rsi), %xmm7 + lea -0x80(%rsi), %rsi + movaps %xmm4, -0x50(%rdi) + movaps %xmm5, -0x60(%rdi) + movaps %xmm6, -0x70(%rdi) + movaps %xmm7, -0x80(%rdi) + lea -0x80(%rdi), %rdi + + jae L(shl_0_gobble_bwd_loop) + cmp $-0x40, %rdx + lea 0x80(%rdx), %rdx + jl L(shl_0_gobble_bwd_less_64bytes) + + movdqa -0x10(%rsi), %xmm0 + sub $0x40, %rdx + movdqa -0x20(%rsi), %xmm1 + + movdqa %xmm0, -0x10(%rdi) + movdqa %xmm1, -0x20(%rdi) + + movdqa -0x30(%rsi), %xmm0 + movdqa -0x40(%rsi), %xmm1 + sub $0x40, %rsi + + movdqa %xmm0, -0x30(%rdi) + movdqa %xmm1, -0x40(%rdi) + sub $0x40, %rdi +L(shl_0_gobble_bwd_less_64bytes): + BRANCH_TO_JMPTBL_ENTRY (L(table_less_80bytes), %rdx, 4) + + .p2align 4 +L(shl_0_gobble_mem_bwd_loop): + prefetcht0 -0x1c0(%rsi) + prefetcht0 -0x280(%rsi) + movdqa -0x10(%rsi), %xmm0 + movdqa -0x20(%rsi), %xmm1 + movdqa -0x30(%rsi), %xmm2 + movdqa -0x40(%rsi), %xmm3 + movdqa -0x50(%rsi), %xmm4 + movdqa -0x60(%rsi), %xmm5 + movdqa -0x70(%rsi), %xmm6 + movdqa -0x80(%rsi), %xmm7 + lea -0x80(%rsi), %rsi + sub $0x80, %rdx + movdqa %xmm0, -0x10(%rdi) + movdqa %xmm1, -0x20(%rdi) + movdqa %xmm2, -0x30(%rdi) + movdqa %xmm3, -0x40(%rdi) + movdqa %xmm4, -0x50(%rdi) + movdqa %xmm5, -0x60(%rdi) + movdqa %xmm6, -0x70(%rdi) + movdqa %xmm7, -0x80(%rdi) + lea -0x80(%rdi), %rdi + + jae L(shl_0_gobble_mem_bwd_loop) + cmp $-0x40, %rdx + lea 0x80(%rdx), %rdx + jl L(shl_0_mem_bwd_less_64bytes) + + movdqa -0x10(%rsi), %xmm0 + sub $0x40, %rdx + movdqa -0x20(%rsi), %xmm1 + + movdqa %xmm0, -0x10(%rdi) + movdqa %xmm1, -0x20(%rdi) + + movdqa -0x30(%rsi), %xmm0 + movdqa -0x40(%rsi), %xmm1 + sub $0x40, %rsi + + movdqa %xmm0, -0x30(%rdi) + movdqa %xmm1, -0x40(%rdi) + sub $0x40, %rdi +L(shl_0_mem_bwd_less_64bytes): + cmp $0x20, %rdx + jb L(shl_0_mem_bwd_less_32bytes) + movdqa -0x10(%rsi), %xmm0 + sub $0x20, %rdx + movdqa -0x20(%rsi), %xmm1 + sub $0x20, %rsi + movdqa %xmm0, -0x10(%rdi) + movdqa %xmm1, -0x20(%rdi) + sub $0x20, %rdi +L(shl_0_mem_bwd_less_32bytes): + BRANCH_TO_JMPTBL_ENTRY (L(table_less_80bytes), %rdx, 4) + + .p2align 4 +L(shl_1): + lea (L(shl_1_loop_L1)-L(shl_1))(%r9), %r9 + cmp %rcx, %rdx + movaps -0x01(%rsi), %xmm1 + jb L(L1_fwd) + lea (L(shl_1_loop_L2)-L(shl_1_loop_L1))(%r9), %r9 +L(L1_fwd): + lea -64(%rdx), %rdx + _CET_NOTRACK jmp *%r9 + ud2 +L(shl_1_loop_L2): + prefetchnta 0x1c0(%rsi) +L(shl_1_loop_L1): + sub $64, %rdx + movaps 0x0f(%rsi), %xmm2 + movaps 0x1f(%rsi), %xmm3 + movaps 0x2f(%rsi), %xmm4 + movaps 0x3f(%rsi), %xmm5 + movdqa %xmm5, %xmm6 + palignr $1, %xmm4, %xmm5 + lea 64(%rsi), %rsi + palignr $1, %xmm3, %xmm4 + palignr $1, %xmm2, %xmm3 + lea 64(%rdi), %rdi + palignr $1, %xmm1, %xmm2 + movdqa %xmm6, %xmm1 + movdqa %xmm2, -0x40(%rdi) + movaps %xmm3, -0x30(%rdi) + jb L(shl_1_end) + movaps %xmm4, -0x20(%rdi) + movaps %xmm5, -0x10(%rdi) + _CET_NOTRACK jmp *%r9 + ud2 +L(shl_1_end): + movaps %xmm4, -0x20(%rdi) + lea 64(%rdx), %rdx + movaps %xmm5, -0x10(%rdi) + add %rdx, %rdi + movdqu %xmm0, (%r8) + add %rdx, %rsi + BRANCH_TO_JMPTBL_ENTRY(L(table_less_80bytes), %rdx, 4) + + .p2align 4 +L(shl_1_bwd): + lea (L(shl_1_bwd_loop_L1)-L(shl_1_bwd))(%r9), %r9 + cmp %rcx, %rdx + movaps -0x01(%rsi), %xmm1 + jb L(L1_bwd) + lea (L(shl_1_bwd_loop_L2)-L(shl_1_bwd_loop_L1))(%r9), %r9 +L(L1_bwd): + lea -64(%rdx), %rdx + _CET_NOTRACK jmp *%r9 + ud2 +L(shl_1_bwd_loop_L2): + prefetchnta -0x1c0(%rsi) +L(shl_1_bwd_loop_L1): + movaps -0x11(%rsi), %xmm2 + sub $0x40, %rdx + movaps -0x21(%rsi), %xmm3 + movaps -0x31(%rsi), %xmm4 + movaps -0x41(%rsi), %xmm5 + lea -0x40(%rsi), %rsi + palignr $1, %xmm2, %xmm1 + palignr $1, %xmm3, %xmm2 + palignr $1, %xmm4, %xmm3 + palignr $1, %xmm5, %xmm4 + + movaps %xmm1, -0x10(%rdi) + movaps %xmm5, %xmm1 + + movaps %xmm2, -0x20(%rdi) + lea -0x40(%rdi), %rdi + + movaps %xmm3, 0x10(%rdi) + jb L(shl_1_bwd_end) + movaps %xmm4, (%rdi) + _CET_NOTRACK jmp *%r9 + ud2 +L(shl_1_bwd_end): + movaps %xmm4, (%rdi) + lea 64(%rdx), %rdx + movdqu %xmm0, (%r8) + BRANCH_TO_JMPTBL_ENTRY(L(table_less_80bytes), %rdx, 4) + + .p2align 4 +L(shl_2): + lea (L(shl_2_loop_L1)-L(shl_2))(%r9), %r9 + cmp %rcx, %rdx + movaps -0x02(%rsi), %xmm1 + jb L(L2_fwd) + lea (L(shl_2_loop_L2)-L(shl_2_loop_L1))(%r9), %r9 +L(L2_fwd): + lea -64(%rdx), %rdx + _CET_NOTRACK jmp *%r9 + ud2 +L(shl_2_loop_L2): + prefetchnta 0x1c0(%rsi) +L(shl_2_loop_L1): + sub $64, %rdx + movaps 0x0e(%rsi), %xmm2 + movaps 0x1e(%rsi), %xmm3 + movaps 0x2e(%rsi), %xmm4 + movaps 0x3e(%rsi), %xmm5 + movdqa %xmm5, %xmm6 + palignr $2, %xmm4, %xmm5 + lea 64(%rsi), %rsi + palignr $2, %xmm3, %xmm4 + palignr $2, %xmm2, %xmm3 + lea 64(%rdi), %rdi + palignr $2, %xmm1, %xmm2 + movdqa %xmm6, %xmm1 + movdqa %xmm2, -0x40(%rdi) + movaps %xmm3, -0x30(%rdi) + jb L(shl_2_end) + movaps %xmm4, -0x20(%rdi) + movaps %xmm5, -0x10(%rdi) + _CET_NOTRACK jmp *%r9 + ud2 +L(shl_2_end): + movaps %xmm4, -0x20(%rdi) + lea 64(%rdx), %rdx + movaps %xmm5, -0x10(%rdi) + add %rdx, %rdi + movdqu %xmm0, (%r8) + add %rdx, %rsi + BRANCH_TO_JMPTBL_ENTRY(L(table_less_80bytes), %rdx, 4) + + .p2align 4 +L(shl_2_bwd): + lea (L(shl_2_bwd_loop_L1)-L(shl_2_bwd))(%r9), %r9 + cmp %rcx, %rdx + movaps -0x02(%rsi), %xmm1 + jb L(L2_bwd) + lea (L(shl_2_bwd_loop_L2)-L(shl_2_bwd_loop_L1))(%r9), %r9 +L(L2_bwd): + lea -64(%rdx), %rdx + _CET_NOTRACK jmp *%r9 + ud2 +L(shl_2_bwd_loop_L2): + prefetchnta -0x1c0(%rsi) +L(shl_2_bwd_loop_L1): + movaps -0x12(%rsi), %xmm2 + sub $0x40, %rdx + movaps -0x22(%rsi), %xmm3 + movaps -0x32(%rsi), %xmm4 + movaps -0x42(%rsi), %xmm5 + lea -0x40(%rsi), %rsi + palignr $2, %xmm2, %xmm1 + palignr $2, %xmm3, %xmm2 + palignr $2, %xmm4, %xmm3 + palignr $2, %xmm5, %xmm4 + + movaps %xmm1, -0x10(%rdi) + movaps %xmm5, %xmm1 + + movaps %xmm2, -0x20(%rdi) + lea -0x40(%rdi), %rdi + + movaps %xmm3, 0x10(%rdi) + jb L(shl_2_bwd_end) + movaps %xmm4, (%rdi) + _CET_NOTRACK jmp *%r9 + ud2 +L(shl_2_bwd_end): + movaps %xmm4, (%rdi) + lea 64(%rdx), %rdx + movdqu %xmm0, (%r8) + BRANCH_TO_JMPTBL_ENTRY(L(table_less_80bytes), %rdx, 4) + + .p2align 4 +L(shl_3): + lea (L(shl_3_loop_L1)-L(shl_3))(%r9), %r9 + cmp %rcx, %rdx + movaps -0x03(%rsi), %xmm1 + jb L(L3_fwd) + lea (L(shl_3_loop_L2)-L(shl_3_loop_L1))(%r9), %r9 +L(L3_fwd): + lea -64(%rdx), %rdx + _CET_NOTRACK jmp *%r9 + ud2 +L(shl_3_loop_L2): + prefetchnta 0x1c0(%rsi) +L(shl_3_loop_L1): + sub $64, %rdx + movaps 0x0d(%rsi), %xmm2 + movaps 0x1d(%rsi), %xmm3 + movaps 0x2d(%rsi), %xmm4 + movaps 0x3d(%rsi), %xmm5 + movdqa %xmm5, %xmm6 + palignr $3, %xmm4, %xmm5 + lea 64(%rsi), %rsi + palignr $3, %xmm3, %xmm4 + palignr $3, %xmm2, %xmm3 + lea 64(%rdi), %rdi + palignr $3, %xmm1, %xmm2 + movdqa %xmm6, %xmm1 + movdqa %xmm2, -0x40(%rdi) + movaps %xmm3, -0x30(%rdi) + jb L(shl_3_end) + movaps %xmm4, -0x20(%rdi) + movaps %xmm5, -0x10(%rdi) + _CET_NOTRACK jmp *%r9 + ud2 +L(shl_3_end): + movaps %xmm4, -0x20(%rdi) + lea 64(%rdx), %rdx + movaps %xmm5, -0x10(%rdi) + add %rdx, %rdi + movdqu %xmm0, (%r8) + add %rdx, %rsi + BRANCH_TO_JMPTBL_ENTRY(L(table_less_80bytes), %rdx, 4) + + .p2align 4 +L(shl_3_bwd): + lea (L(shl_3_bwd_loop_L1)-L(shl_3_bwd))(%r9), %r9 + cmp %rcx, %rdx + movaps -0x03(%rsi), %xmm1 + jb L(L3_bwd) + lea (L(shl_3_bwd_loop_L2)-L(shl_3_bwd_loop_L1))(%r9), %r9 +L(L3_bwd): + lea -64(%rdx), %rdx + _CET_NOTRACK jmp *%r9 + ud2 +L(shl_3_bwd_loop_L2): + prefetchnta -0x1c0(%rsi) +L(shl_3_bwd_loop_L1): + movaps -0x13(%rsi), %xmm2 + sub $0x40, %rdx + movaps -0x23(%rsi), %xmm3 + movaps -0x33(%rsi), %xmm4 + movaps -0x43(%rsi), %xmm5 + lea -0x40(%rsi), %rsi + palignr $3, %xmm2, %xmm1 + palignr $3, %xmm3, %xmm2 + palignr $3, %xmm4, %xmm3 + palignr $3, %xmm5, %xmm4 + + movaps %xmm1, -0x10(%rdi) + movaps %xmm5, %xmm1 + + movaps %xmm2, -0x20(%rdi) + lea -0x40(%rdi), %rdi + + movaps %xmm3, 0x10(%rdi) + jb L(shl_3_bwd_end) + movaps %xmm4, (%rdi) + _CET_NOTRACK jmp *%r9 + ud2 +L(shl_3_bwd_end): + movaps %xmm4, (%rdi) + lea 64(%rdx), %rdx + movdqu %xmm0, (%r8) + BRANCH_TO_JMPTBL_ENTRY(L(table_less_80bytes), %rdx, 4) + + .p2align 4 +L(shl_4): + lea (L(shl_4_loop_L1)-L(shl_4))(%r9), %r9 + cmp %rcx, %rdx + movaps -0x04(%rsi), %xmm1 + jb L(L4_fwd) + lea (L(shl_4_loop_L2)-L(shl_4_loop_L1))(%r9), %r9 +L(L4_fwd): + lea -64(%rdx), %rdx + _CET_NOTRACK jmp *%r9 + ud2 +L(shl_4_loop_L2): + prefetchnta 0x1c0(%rsi) +L(shl_4_loop_L1): + sub $64, %rdx + movaps 0x0c(%rsi), %xmm2 + movaps 0x1c(%rsi), %xmm3 + movaps 0x2c(%rsi), %xmm4 + movaps 0x3c(%rsi), %xmm5 + movdqa %xmm5, %xmm6 + palignr $4, %xmm4, %xmm5 + lea 64(%rsi), %rsi + palignr $4, %xmm3, %xmm4 + palignr $4, %xmm2, %xmm3 + lea 64(%rdi), %rdi + palignr $4, %xmm1, %xmm2 + movdqa %xmm6, %xmm1 + movdqa %xmm2, -0x40(%rdi) + movaps %xmm3, -0x30(%rdi) + jb L(shl_4_end) + movaps %xmm4, -0x20(%rdi) + movaps %xmm5, -0x10(%rdi) + _CET_NOTRACK jmp *%r9 + ud2 +L(shl_4_end): + movaps %xmm4, -0x20(%rdi) + lea 64(%rdx), %rdx + movaps %xmm5, -0x10(%rdi) + add %rdx, %rdi + movdqu %xmm0, (%r8) + add %rdx, %rsi + BRANCH_TO_JMPTBL_ENTRY(L(table_less_80bytes), %rdx, 4) + + .p2align 4 +L(shl_4_bwd): + lea (L(shl_4_bwd_loop_L1)-L(shl_4_bwd))(%r9), %r9 + cmp %rcx, %rdx + movaps -0x04(%rsi), %xmm1 + jb L(L4_bwd) + lea (L(shl_4_bwd_loop_L2)-L(shl_4_bwd_loop_L1))(%r9), %r9 +L(L4_bwd): + lea -64(%rdx), %rdx + _CET_NOTRACK jmp *%r9 + ud2 +L(shl_4_bwd_loop_L2): + prefetchnta -0x1c0(%rsi) +L(shl_4_bwd_loop_L1): + movaps -0x14(%rsi), %xmm2 + sub $0x40, %rdx + movaps -0x24(%rsi), %xmm3 + movaps -0x34(%rsi), %xmm4 + movaps -0x44(%rsi), %xmm5 + lea -0x40(%rsi), %rsi + palignr $4, %xmm2, %xmm1 + palignr $4, %xmm3, %xmm2 + palignr $4, %xmm4, %xmm3 + palignr $4, %xmm5, %xmm4 + + movaps %xmm1, -0x10(%rdi) + movaps %xmm5, %xmm1 + + movaps %xmm2, -0x20(%rdi) + lea -0x40(%rdi), %rdi + + movaps %xmm3, 0x10(%rdi) + jb L(shl_4_bwd_end) + movaps %xmm4, (%rdi) + _CET_NOTRACK jmp *%r9 + ud2 +L(shl_4_bwd_end): + movaps %xmm4, (%rdi) + lea 64(%rdx), %rdx + movdqu %xmm0, (%r8) + BRANCH_TO_JMPTBL_ENTRY(L(table_less_80bytes), %rdx, 4) + + .p2align 4 +L(shl_5): + lea (L(shl_5_loop_L1)-L(shl_5))(%r9), %r9 + cmp %rcx, %rdx + movaps -0x05(%rsi), %xmm1 + jb L(L5_fwd) + lea (L(shl_5_loop_L2)-L(shl_5_loop_L1))(%r9), %r9 +L(L5_fwd): + lea -64(%rdx), %rdx + _CET_NOTRACK jmp *%r9 + ud2 +L(shl_5_loop_L2): + prefetchnta 0x1c0(%rsi) +L(shl_5_loop_L1): + sub $64, %rdx + movaps 0x0b(%rsi), %xmm2 + movaps 0x1b(%rsi), %xmm3 + movaps 0x2b(%rsi), %xmm4 + movaps 0x3b(%rsi), %xmm5 + movdqa %xmm5, %xmm6 + palignr $5, %xmm4, %xmm5 + lea 64(%rsi), %rsi + palignr $5, %xmm3, %xmm4 + palignr $5, %xmm2, %xmm3 + lea 64(%rdi), %rdi + palignr $5, %xmm1, %xmm2 + movdqa %xmm6, %xmm1 + movdqa %xmm2, -0x40(%rdi) + movaps %xmm3, -0x30(%rdi) + jb L(shl_5_end) + movaps %xmm4, -0x20(%rdi) + movaps %xmm5, -0x10(%rdi) + _CET_NOTRACK jmp *%r9 + ud2 +L(shl_5_end): + movaps %xmm4, -0x20(%rdi) + lea 64(%rdx), %rdx + movaps %xmm5, -0x10(%rdi) + add %rdx, %rdi + movdqu %xmm0, (%r8) + add %rdx, %rsi + BRANCH_TO_JMPTBL_ENTRY(L(table_less_80bytes), %rdx, 4) + + .p2align 4 +L(shl_5_bwd): + lea (L(shl_5_bwd_loop_L1)-L(shl_5_bwd))(%r9), %r9 + cmp %rcx, %rdx + movaps -0x05(%rsi), %xmm1 + jb L(L5_bwd) + lea (L(shl_5_bwd_loop_L2)-L(shl_5_bwd_loop_L1))(%r9), %r9 +L(L5_bwd): + lea -64(%rdx), %rdx + _CET_NOTRACK jmp *%r9 + ud2 +L(shl_5_bwd_loop_L2): + prefetchnta -0x1c0(%rsi) +L(shl_5_bwd_loop_L1): + movaps -0x15(%rsi), %xmm2 + sub $0x40, %rdx + movaps -0x25(%rsi), %xmm3 + movaps -0x35(%rsi), %xmm4 + movaps -0x45(%rsi), %xmm5 + lea -0x40(%rsi), %rsi + palignr $5, %xmm2, %xmm1 + palignr $5, %xmm3, %xmm2 + palignr $5, %xmm4, %xmm3 + palignr $5, %xmm5, %xmm4 + + movaps %xmm1, -0x10(%rdi) + movaps %xmm5, %xmm1 + + movaps %xmm2, -0x20(%rdi) + lea -0x40(%rdi), %rdi + + movaps %xmm3, 0x10(%rdi) + jb L(shl_5_bwd_end) + movaps %xmm4, (%rdi) + _CET_NOTRACK jmp *%r9 + ud2 +L(shl_5_bwd_end): + movaps %xmm4, (%rdi) + lea 64(%rdx), %rdx + movdqu %xmm0, (%r8) + BRANCH_TO_JMPTBL_ENTRY(L(table_less_80bytes), %rdx, 4) + + .p2align 4 +L(shl_6): + lea (L(shl_6_loop_L1)-L(shl_6))(%r9), %r9 + cmp %rcx, %rdx + movaps -0x06(%rsi), %xmm1 + jb L(L6_fwd) + lea (L(shl_6_loop_L2)-L(shl_6_loop_L1))(%r9), %r9 +L(L6_fwd): + lea -64(%rdx), %rdx + _CET_NOTRACK jmp *%r9 + ud2 +L(shl_6_loop_L2): + prefetchnta 0x1c0(%rsi) +L(shl_6_loop_L1): + sub $64, %rdx + movaps 0x0a(%rsi), %xmm2 + movaps 0x1a(%rsi), %xmm3 + movaps 0x2a(%rsi), %xmm4 + movaps 0x3a(%rsi), %xmm5 + movdqa %xmm5, %xmm6 + palignr $6, %xmm4, %xmm5 + lea 64(%rsi), %rsi + palignr $6, %xmm3, %xmm4 + palignr $6, %xmm2, %xmm3 + lea 64(%rdi), %rdi + palignr $6, %xmm1, %xmm2 + movdqa %xmm6, %xmm1 + movdqa %xmm2, -0x40(%rdi) + movaps %xmm3, -0x30(%rdi) + jb L(shl_6_end) + movaps %xmm4, -0x20(%rdi) + movaps %xmm5, -0x10(%rdi) + _CET_NOTRACK jmp *%r9 + ud2 +L(shl_6_end): + movaps %xmm4, -0x20(%rdi) + lea 64(%rdx), %rdx + movaps %xmm5, -0x10(%rdi) + add %rdx, %rdi + movdqu %xmm0, (%r8) + add %rdx, %rsi + BRANCH_TO_JMPTBL_ENTRY(L(table_less_80bytes), %rdx, 4) + + .p2align 4 +L(shl_6_bwd): + lea (L(shl_6_bwd_loop_L1)-L(shl_6_bwd))(%r9), %r9 + cmp %rcx, %rdx + movaps -0x06(%rsi), %xmm1 + jb L(L6_bwd) + lea (L(shl_6_bwd_loop_L2)-L(shl_6_bwd_loop_L1))(%r9), %r9 +L(L6_bwd): + lea -64(%rdx), %rdx + _CET_NOTRACK jmp *%r9 + ud2 +L(shl_6_bwd_loop_L2): + prefetchnta -0x1c0(%rsi) +L(shl_6_bwd_loop_L1): + movaps -0x16(%rsi), %xmm2 + sub $0x40, %rdx + movaps -0x26(%rsi), %xmm3 + movaps -0x36(%rsi), %xmm4 + movaps -0x46(%rsi), %xmm5 + lea -0x40(%rsi), %rsi + palignr $6, %xmm2, %xmm1 + palignr $6, %xmm3, %xmm2 + palignr $6, %xmm4, %xmm3 + palignr $6, %xmm5, %xmm4 + + movaps %xmm1, -0x10(%rdi) + movaps %xmm5, %xmm1 + + movaps %xmm2, -0x20(%rdi) + lea -0x40(%rdi), %rdi + + movaps %xmm3, 0x10(%rdi) + jb L(shl_6_bwd_end) + movaps %xmm4, (%rdi) + _CET_NOTRACK jmp *%r9 + ud2 +L(shl_6_bwd_end): + movaps %xmm4, (%rdi) + lea 64(%rdx), %rdx + movdqu %xmm0, (%r8) + BRANCH_TO_JMPTBL_ENTRY(L(table_less_80bytes), %rdx, 4) + + .p2align 4 +L(shl_7): + lea (L(shl_7_loop_L1)-L(shl_7))(%r9), %r9 + cmp %rcx, %rdx + movaps -0x07(%rsi), %xmm1 + jb L(L7_fwd) + lea (L(shl_7_loop_L2)-L(shl_7_loop_L1))(%r9), %r9 +L(L7_fwd): + lea -64(%rdx), %rdx + _CET_NOTRACK jmp *%r9 + ud2 +L(shl_7_loop_L2): + prefetchnta 0x1c0(%rsi) +L(shl_7_loop_L1): + sub $64, %rdx + movaps 0x09(%rsi), %xmm2 + movaps 0x19(%rsi), %xmm3 + movaps 0x29(%rsi), %xmm4 + movaps 0x39(%rsi), %xmm5 + movdqa %xmm5, %xmm6 + palignr $7, %xmm4, %xmm5 + lea 64(%rsi), %rsi + palignr $7, %xmm3, %xmm4 + palignr $7, %xmm2, %xmm3 + lea 64(%rdi), %rdi + palignr $7, %xmm1, %xmm2 + movdqa %xmm6, %xmm1 + movdqa %xmm2, -0x40(%rdi) + movaps %xmm3, -0x30(%rdi) + jb L(shl_7_end) + movaps %xmm4, -0x20(%rdi) + movaps %xmm5, -0x10(%rdi) + _CET_NOTRACK jmp *%r9 + ud2 +L(shl_7_end): + movaps %xmm4, -0x20(%rdi) + lea 64(%rdx), %rdx + movaps %xmm5, -0x10(%rdi) + add %rdx, %rdi + movdqu %xmm0, (%r8) + add %rdx, %rsi + BRANCH_TO_JMPTBL_ENTRY(L(table_less_80bytes), %rdx, 4) + + .p2align 4 +L(shl_7_bwd): + lea (L(shl_7_bwd_loop_L1)-L(shl_7_bwd))(%r9), %r9 + cmp %rcx, %rdx + movaps -0x07(%rsi), %xmm1 + jb L(L7_bwd) + lea (L(shl_7_bwd_loop_L2)-L(shl_7_bwd_loop_L1))(%r9), %r9 +L(L7_bwd): + lea -64(%rdx), %rdx + _CET_NOTRACK jmp *%r9 + ud2 +L(shl_7_bwd_loop_L2): + prefetchnta -0x1c0(%rsi) +L(shl_7_bwd_loop_L1): + movaps -0x17(%rsi), %xmm2 + sub $0x40, %rdx + movaps -0x27(%rsi), %xmm3 + movaps -0x37(%rsi), %xmm4 + movaps -0x47(%rsi), %xmm5 + lea -0x40(%rsi), %rsi + palignr $7, %xmm2, %xmm1 + palignr $7, %xmm3, %xmm2 + palignr $7, %xmm4, %xmm3 + palignr $7, %xmm5, %xmm4 + + movaps %xmm1, -0x10(%rdi) + movaps %xmm5, %xmm1 + + movaps %xmm2, -0x20(%rdi) + lea -0x40(%rdi), %rdi + + movaps %xmm3, 0x10(%rdi) + jb L(shl_7_bwd_end) + movaps %xmm4, (%rdi) + _CET_NOTRACK jmp *%r9 + ud2 +L(shl_7_bwd_end): + movaps %xmm4, (%rdi) + lea 64(%rdx), %rdx + movdqu %xmm0, (%r8) + BRANCH_TO_JMPTBL_ENTRY(L(table_less_80bytes), %rdx, 4) + + .p2align 4 +L(shl_8): + lea (L(shl_8_loop_L1)-L(shl_8))(%r9), %r9 + cmp %rcx, %rdx + movaps -0x08(%rsi), %xmm1 + jb L(L8_fwd) + lea (L(shl_8_loop_L2)-L(shl_8_loop_L1))(%r9), %r9 +L(L8_fwd): + lea -64(%rdx), %rdx + _CET_NOTRACK jmp *%r9 +L(shl_8_loop_L2): + prefetchnta 0x1c0(%rsi) +L(shl_8_loop_L1): + sub $64, %rdx + movaps 0x08(%rsi), %xmm2 + movaps 0x18(%rsi), %xmm3 + movaps 0x28(%rsi), %xmm4 + movaps 0x38(%rsi), %xmm5 + movdqa %xmm5, %xmm6 + palignr $8, %xmm4, %xmm5 + lea 64(%rsi), %rsi + palignr $8, %xmm3, %xmm4 + palignr $8, %xmm2, %xmm3 + lea 64(%rdi), %rdi + palignr $8, %xmm1, %xmm2 + movdqa %xmm6, %xmm1 + movdqa %xmm2, -0x40(%rdi) + movaps %xmm3, -0x30(%rdi) + jb L(shl_8_end) + movaps %xmm4, -0x20(%rdi) + movaps %xmm5, -0x10(%rdi) + _CET_NOTRACK jmp *%r9 + ud2 + .p2align 4 +L(shl_8_end): + lea 64(%rdx), %rdx + movaps %xmm4, -0x20(%rdi) + add %rdx, %rsi + movaps %xmm5, -0x10(%rdi) + add %rdx, %rdi + movdqu %xmm0, (%r8) + BRANCH_TO_JMPTBL_ENTRY(L(table_less_80bytes), %rdx, 4) + + .p2align 4 +L(shl_8_bwd): + lea (L(shl_8_bwd_loop_L1)-L(shl_8_bwd))(%r9), %r9 + cmp %rcx, %rdx + movaps -0x08(%rsi), %xmm1 + jb L(L8_bwd) + lea (L(shl_8_bwd_loop_L2)-L(shl_8_bwd_loop_L1))(%r9), %r9 +L(L8_bwd): + lea -64(%rdx), %rdx + _CET_NOTRACK jmp *%r9 + ud2 +L(shl_8_bwd_loop_L2): + prefetchnta -0x1c0(%rsi) +L(shl_8_bwd_loop_L1): + movaps -0x18(%rsi), %xmm2 + sub $0x40, %rdx + movaps -0x28(%rsi), %xmm3 + movaps -0x38(%rsi), %xmm4 + movaps -0x48(%rsi), %xmm5 + lea -0x40(%rsi), %rsi + palignr $8, %xmm2, %xmm1 + palignr $8, %xmm3, %xmm2 + palignr $8, %xmm4, %xmm3 + palignr $8, %xmm5, %xmm4 + + movaps %xmm1, -0x10(%rdi) + movaps %xmm5, %xmm1 + + movaps %xmm2, -0x20(%rdi) + lea -0x40(%rdi), %rdi + + movaps %xmm3, 0x10(%rdi) + jb L(shl_8_bwd_end) + movaps %xmm4, (%rdi) + _CET_NOTRACK jmp *%r9 + ud2 +L(shl_8_bwd_end): + movaps %xmm4, (%rdi) + lea 64(%rdx), %rdx + movdqu %xmm0, (%r8) + BRANCH_TO_JMPTBL_ENTRY(L(table_less_80bytes), %rdx, 4) + + .p2align 4 +L(shl_9): + lea (L(shl_9_loop_L1)-L(shl_9))(%r9), %r9 + cmp %rcx, %rdx + movaps -0x09(%rsi), %xmm1 + jb L(L9_fwd) + lea (L(shl_9_loop_L2)-L(shl_9_loop_L1))(%r9), %r9 +L(L9_fwd): + lea -64(%rdx), %rdx + _CET_NOTRACK jmp *%r9 + ud2 +L(shl_9_loop_L2): + prefetchnta 0x1c0(%rsi) +L(shl_9_loop_L1): + sub $64, %rdx + movaps 0x07(%rsi), %xmm2 + movaps 0x17(%rsi), %xmm3 + movaps 0x27(%rsi), %xmm4 + movaps 0x37(%rsi), %xmm5 + movdqa %xmm5, %xmm6 + palignr $9, %xmm4, %xmm5 + lea 64(%rsi), %rsi + palignr $9, %xmm3, %xmm4 + palignr $9, %xmm2, %xmm3 + lea 64(%rdi), %rdi + palignr $9, %xmm1, %xmm2 + movdqa %xmm6, %xmm1 + movdqa %xmm2, -0x40(%rdi) + movaps %xmm3, -0x30(%rdi) + jb L(shl_9_end) + movaps %xmm4, -0x20(%rdi) + movaps %xmm5, -0x10(%rdi) + _CET_NOTRACK jmp *%r9 + ud2 +L(shl_9_end): + movaps %xmm4, -0x20(%rdi) + lea 64(%rdx), %rdx + movaps %xmm5, -0x10(%rdi) + add %rdx, %rdi + movdqu %xmm0, (%r8) + add %rdx, %rsi + BRANCH_TO_JMPTBL_ENTRY(L(table_less_80bytes), %rdx, 4) + + .p2align 4 +L(shl_9_bwd): + lea (L(shl_9_bwd_loop_L1)-L(shl_9_bwd))(%r9), %r9 + cmp %rcx, %rdx + movaps -0x09(%rsi), %xmm1 + jb L(L9_bwd) + lea (L(shl_9_bwd_loop_L2)-L(shl_9_bwd_loop_L1))(%r9), %r9 +L(L9_bwd): + lea -64(%rdx), %rdx + _CET_NOTRACK jmp *%r9 + ud2 +L(shl_9_bwd_loop_L2): + prefetchnta -0x1c0(%rsi) +L(shl_9_bwd_loop_L1): + movaps -0x19(%rsi), %xmm2 + sub $0x40, %rdx + movaps -0x29(%rsi), %xmm3 + movaps -0x39(%rsi), %xmm4 + movaps -0x49(%rsi), %xmm5 + lea -0x40(%rsi), %rsi + palignr $9, %xmm2, %xmm1 + palignr $9, %xmm3, %xmm2 + palignr $9, %xmm4, %xmm3 + palignr $9, %xmm5, %xmm4 + + movaps %xmm1, -0x10(%rdi) + movaps %xmm5, %xmm1 + + movaps %xmm2, -0x20(%rdi) + lea -0x40(%rdi), %rdi + + movaps %xmm3, 0x10(%rdi) + jb L(shl_9_bwd_end) + movaps %xmm4, (%rdi) + _CET_NOTRACK jmp *%r9 + ud2 +L(shl_9_bwd_end): + movaps %xmm4, (%rdi) + lea 64(%rdx), %rdx + movdqu %xmm0, (%r8) + BRANCH_TO_JMPTBL_ENTRY(L(table_less_80bytes), %rdx, 4) + + .p2align 4 +L(shl_10): + lea (L(shl_10_loop_L1)-L(shl_10))(%r9), %r9 + cmp %rcx, %rdx + movaps -0x0a(%rsi), %xmm1 + jb L(L10_fwd) + lea (L(shl_10_loop_L2)-L(shl_10_loop_L1))(%r9), %r9 +L(L10_fwd): + lea -64(%rdx), %rdx + _CET_NOTRACK jmp *%r9 + ud2 +L(shl_10_loop_L2): + prefetchnta 0x1c0(%rsi) +L(shl_10_loop_L1): + sub $64, %rdx + movaps 0x06(%rsi), %xmm2 + movaps 0x16(%rsi), %xmm3 + movaps 0x26(%rsi), %xmm4 + movaps 0x36(%rsi), %xmm5 + movdqa %xmm5, %xmm6 + palignr $10, %xmm4, %xmm5 + lea 64(%rsi), %rsi + palignr $10, %xmm3, %xmm4 + palignr $10, %xmm2, %xmm3 + lea 64(%rdi), %rdi + palignr $10, %xmm1, %xmm2 + movdqa %xmm6, %xmm1 + movdqa %xmm2, -0x40(%rdi) + movaps %xmm3, -0x30(%rdi) + jb L(shl_10_end) + movaps %xmm4, -0x20(%rdi) + movaps %xmm5, -0x10(%rdi) + _CET_NOTRACK jmp *%r9 + ud2 +L(shl_10_end): + movaps %xmm4, -0x20(%rdi) + lea 64(%rdx), %rdx + movaps %xmm5, -0x10(%rdi) + add %rdx, %rdi + movdqu %xmm0, (%r8) + add %rdx, %rsi + BRANCH_TO_JMPTBL_ENTRY(L(table_less_80bytes), %rdx, 4) + + .p2align 4 +L(shl_10_bwd): + lea (L(shl_10_bwd_loop_L1)-L(shl_10_bwd))(%r9), %r9 + cmp %rcx, %rdx + movaps -0x0a(%rsi), %xmm1 + jb L(L10_bwd) + lea (L(shl_10_bwd_loop_L2)-L(shl_10_bwd_loop_L1))(%r9), %r9 +L(L10_bwd): + lea -64(%rdx), %rdx + _CET_NOTRACK jmp *%r9 + ud2 +L(shl_10_bwd_loop_L2): + prefetchnta -0x1c0(%rsi) +L(shl_10_bwd_loop_L1): + movaps -0x1a(%rsi), %xmm2 + sub $0x40, %rdx + movaps -0x2a(%rsi), %xmm3 + movaps -0x3a(%rsi), %xmm4 + movaps -0x4a(%rsi), %xmm5 + lea -0x40(%rsi), %rsi + palignr $10, %xmm2, %xmm1 + palignr $10, %xmm3, %xmm2 + palignr $10, %xmm4, %xmm3 + palignr $10, %xmm5, %xmm4 + + movaps %xmm1, -0x10(%rdi) + movaps %xmm5, %xmm1 + + movaps %xmm2, -0x20(%rdi) + lea -0x40(%rdi), %rdi + + movaps %xmm3, 0x10(%rdi) + jb L(shl_10_bwd_end) + movaps %xmm4, (%rdi) + _CET_NOTRACK jmp *%r9 + ud2 +L(shl_10_bwd_end): + movaps %xmm4, (%rdi) + lea 64(%rdx), %rdx + movdqu %xmm0, (%r8) + BRANCH_TO_JMPTBL_ENTRY(L(table_less_80bytes), %rdx, 4) + + .p2align 4 +L(shl_11): + lea (L(shl_11_loop_L1)-L(shl_11))(%r9), %r9 + cmp %rcx, %rdx + movaps -0x0b(%rsi), %xmm1 + jb L(L11_fwd) + lea (L(shl_11_loop_L2)-L(shl_11_loop_L1))(%r9), %r9 +L(L11_fwd): + lea -64(%rdx), %rdx + _CET_NOTRACK jmp *%r9 + ud2 +L(shl_11_loop_L2): + prefetchnta 0x1c0(%rsi) +L(shl_11_loop_L1): + sub $64, %rdx + movaps 0x05(%rsi), %xmm2 + movaps 0x15(%rsi), %xmm3 + movaps 0x25(%rsi), %xmm4 + movaps 0x35(%rsi), %xmm5 + movdqa %xmm5, %xmm6 + palignr $11, %xmm4, %xmm5 + lea 64(%rsi), %rsi + palignr $11, %xmm3, %xmm4 + palignr $11, %xmm2, %xmm3 + lea 64(%rdi), %rdi + palignr $11, %xmm1, %xmm2 + movdqa %xmm6, %xmm1 + movdqa %xmm2, -0x40(%rdi) + movaps %xmm3, -0x30(%rdi) + jb L(shl_11_end) + movaps %xmm4, -0x20(%rdi) + movaps %xmm5, -0x10(%rdi) + _CET_NOTRACK jmp *%r9 + ud2 +L(shl_11_end): + movaps %xmm4, -0x20(%rdi) + lea 64(%rdx), %rdx + movaps %xmm5, -0x10(%rdi) + add %rdx, %rdi + movdqu %xmm0, (%r8) + add %rdx, %rsi + BRANCH_TO_JMPTBL_ENTRY(L(table_less_80bytes), %rdx, 4) + + .p2align 4 +L(shl_11_bwd): + lea (L(shl_11_bwd_loop_L1)-L(shl_11_bwd))(%r9), %r9 + cmp %rcx, %rdx + movaps -0x0b(%rsi), %xmm1 + jb L(L11_bwd) + lea (L(shl_11_bwd_loop_L2)-L(shl_11_bwd_loop_L1))(%r9), %r9 +L(L11_bwd): + lea -64(%rdx), %rdx + _CET_NOTRACK jmp *%r9 + ud2 +L(shl_11_bwd_loop_L2): + prefetchnta -0x1c0(%rsi) +L(shl_11_bwd_loop_L1): + movaps -0x1b(%rsi), %xmm2 + sub $0x40, %rdx + movaps -0x2b(%rsi), %xmm3 + movaps -0x3b(%rsi), %xmm4 + movaps -0x4b(%rsi), %xmm5 + lea -0x40(%rsi), %rsi + palignr $11, %xmm2, %xmm1 + palignr $11, %xmm3, %xmm2 + palignr $11, %xmm4, %xmm3 + palignr $11, %xmm5, %xmm4 + + movaps %xmm1, -0x10(%rdi) + movaps %xmm5, %xmm1 + + movaps %xmm2, -0x20(%rdi) + lea -0x40(%rdi), %rdi + + movaps %xmm3, 0x10(%rdi) + jb L(shl_11_bwd_end) + movaps %xmm4, (%rdi) + _CET_NOTRACK jmp *%r9 + ud2 +L(shl_11_bwd_end): + movaps %xmm4, (%rdi) + lea 64(%rdx), %rdx + movdqu %xmm0, (%r8) + BRANCH_TO_JMPTBL_ENTRY(L(table_less_80bytes), %rdx, 4) + + .p2align 4 +L(shl_12): + lea (L(shl_12_loop_L1)-L(shl_12))(%r9), %r9 + cmp %rcx, %rdx + movaps -0x0c(%rsi), %xmm1 + jb L(L12_fwd) + lea (L(shl_12_loop_L2)-L(shl_12_loop_L1))(%r9), %r9 +L(L12_fwd): + lea -64(%rdx), %rdx + _CET_NOTRACK jmp *%r9 + ud2 +L(shl_12_loop_L2): + prefetchnta 0x1c0(%rsi) +L(shl_12_loop_L1): + sub $64, %rdx + movaps 0x04(%rsi), %xmm2 + movaps 0x14(%rsi), %xmm3 + movaps 0x24(%rsi), %xmm4 + movaps 0x34(%rsi), %xmm5 + movdqa %xmm5, %xmm6 + palignr $12, %xmm4, %xmm5 + lea 64(%rsi), %rsi + palignr $12, %xmm3, %xmm4 + palignr $12, %xmm2, %xmm3 + lea 64(%rdi), %rdi + palignr $12, %xmm1, %xmm2 + movdqa %xmm6, %xmm1 + movdqa %xmm2, -0x40(%rdi) + movaps %xmm3, -0x30(%rdi) + jb L(shl_12_end) + movaps %xmm4, -0x20(%rdi) + movaps %xmm5, -0x10(%rdi) + _CET_NOTRACK jmp *%r9 + ud2 +L(shl_12_end): + movaps %xmm4, -0x20(%rdi) + lea 64(%rdx), %rdx + movaps %xmm5, -0x10(%rdi) + add %rdx, %rdi + movdqu %xmm0, (%r8) + add %rdx, %rsi + BRANCH_TO_JMPTBL_ENTRY(L(table_less_80bytes), %rdx, 4) + + .p2align 4 +L(shl_12_bwd): + lea (L(shl_12_bwd_loop_L1)-L(shl_12_bwd))(%r9), %r9 + cmp %rcx, %rdx + movaps -0x0c(%rsi), %xmm1 + jb L(L12_bwd) + lea (L(shl_12_bwd_loop_L2)-L(shl_12_bwd_loop_L1))(%r9), %r9 +L(L12_bwd): + lea -64(%rdx), %rdx + _CET_NOTRACK jmp *%r9 + ud2 +L(shl_12_bwd_loop_L2): + prefetchnta -0x1c0(%rsi) +L(shl_12_bwd_loop_L1): + movaps -0x1c(%rsi), %xmm2 + sub $0x40, %rdx + movaps -0x2c(%rsi), %xmm3 + movaps -0x3c(%rsi), %xmm4 + movaps -0x4c(%rsi), %xmm5 + lea -0x40(%rsi), %rsi + palignr $12, %xmm2, %xmm1 + palignr $12, %xmm3, %xmm2 + palignr $12, %xmm4, %xmm3 + palignr $12, %xmm5, %xmm4 + + movaps %xmm1, -0x10(%rdi) + movaps %xmm5, %xmm1 + + movaps %xmm2, -0x20(%rdi) + lea -0x40(%rdi), %rdi + + movaps %xmm3, 0x10(%rdi) + jb L(shl_12_bwd_end) + movaps %xmm4, (%rdi) + _CET_NOTRACK jmp *%r9 + ud2 +L(shl_12_bwd_end): + movaps %xmm4, (%rdi) + lea 64(%rdx), %rdx + movdqu %xmm0, (%r8) + BRANCH_TO_JMPTBL_ENTRY(L(table_less_80bytes), %rdx, 4) + + .p2align 4 +L(shl_13): + lea (L(shl_13_loop_L1)-L(shl_13))(%r9), %r9 + cmp %rcx, %rdx + movaps -0x0d(%rsi), %xmm1 + jb L(L13_fwd) + lea (L(shl_13_loop_L2)-L(shl_13_loop_L1))(%r9), %r9 +L(L13_fwd): + lea -64(%rdx), %rdx + _CET_NOTRACK jmp *%r9 + ud2 +L(shl_13_loop_L2): + prefetchnta 0x1c0(%rsi) +L(shl_13_loop_L1): + sub $64, %rdx + movaps 0x03(%rsi), %xmm2 + movaps 0x13(%rsi), %xmm3 + movaps 0x23(%rsi), %xmm4 + movaps 0x33(%rsi), %xmm5 + movdqa %xmm5, %xmm6 + palignr $13, %xmm4, %xmm5 + lea 64(%rsi), %rsi + palignr $13, %xmm3, %xmm4 + palignr $13, %xmm2, %xmm3 + lea 64(%rdi), %rdi + palignr $13, %xmm1, %xmm2 + movdqa %xmm6, %xmm1 + movdqa %xmm2, -0x40(%rdi) + movaps %xmm3, -0x30(%rdi) + jb L(shl_13_end) + movaps %xmm4, -0x20(%rdi) + movaps %xmm5, -0x10(%rdi) + _CET_NOTRACK jmp *%r9 + ud2 +L(shl_13_end): + movaps %xmm4, -0x20(%rdi) + lea 64(%rdx), %rdx + movaps %xmm5, -0x10(%rdi) + add %rdx, %rdi + movdqu %xmm0, (%r8) + add %rdx, %rsi + BRANCH_TO_JMPTBL_ENTRY(L(table_less_80bytes), %rdx, 4) + + .p2align 4 +L(shl_13_bwd): + lea (L(shl_13_bwd_loop_L1)-L(shl_13_bwd))(%r9), %r9 + cmp %rcx, %rdx + movaps -0x0d(%rsi), %xmm1 + jb L(L13_bwd) + lea (L(shl_13_bwd_loop_L2)-L(shl_13_bwd_loop_L1))(%r9), %r9 +L(L13_bwd): + lea -64(%rdx), %rdx + _CET_NOTRACK jmp *%r9 + ud2 +L(shl_13_bwd_loop_L2): + prefetchnta -0x1c0(%rsi) +L(shl_13_bwd_loop_L1): + movaps -0x1d(%rsi), %xmm2 + sub $0x40, %rdx + movaps -0x2d(%rsi), %xmm3 + movaps -0x3d(%rsi), %xmm4 + movaps -0x4d(%rsi), %xmm5 + lea -0x40(%rsi), %rsi + palignr $13, %xmm2, %xmm1 + palignr $13, %xmm3, %xmm2 + palignr $13, %xmm4, %xmm3 + palignr $13, %xmm5, %xmm4 + + movaps %xmm1, -0x10(%rdi) + movaps %xmm5, %xmm1 + + movaps %xmm2, -0x20(%rdi) + lea -0x40(%rdi), %rdi + + movaps %xmm3, 0x10(%rdi) + jb L(shl_13_bwd_end) + movaps %xmm4, (%rdi) + _CET_NOTRACK jmp *%r9 + ud2 +L(shl_13_bwd_end): + movaps %xmm4, (%rdi) + lea 64(%rdx), %rdx + movdqu %xmm0, (%r8) + BRANCH_TO_JMPTBL_ENTRY(L(table_less_80bytes), %rdx, 4) + + .p2align 4 +L(shl_14): + lea (L(shl_14_loop_L1)-L(shl_14))(%r9), %r9 + cmp %rcx, %rdx + movaps -0x0e(%rsi), %xmm1 + jb L(L14_fwd) + lea (L(shl_14_loop_L2)-L(shl_14_loop_L1))(%r9), %r9 +L(L14_fwd): + lea -64(%rdx), %rdx + _CET_NOTRACK jmp *%r9 + ud2 +L(shl_14_loop_L2): + prefetchnta 0x1c0(%rsi) +L(shl_14_loop_L1): + sub $64, %rdx + movaps 0x02(%rsi), %xmm2 + movaps 0x12(%rsi), %xmm3 + movaps 0x22(%rsi), %xmm4 + movaps 0x32(%rsi), %xmm5 + movdqa %xmm5, %xmm6 + palignr $14, %xmm4, %xmm5 + lea 64(%rsi), %rsi + palignr $14, %xmm3, %xmm4 + palignr $14, %xmm2, %xmm3 + lea 64(%rdi), %rdi + palignr $14, %xmm1, %xmm2 + movdqa %xmm6, %xmm1 + movdqa %xmm2, -0x40(%rdi) + movaps %xmm3, -0x30(%rdi) + jb L(shl_14_end) + movaps %xmm4, -0x20(%rdi) + movaps %xmm5, -0x10(%rdi) + _CET_NOTRACK jmp *%r9 + ud2 +L(shl_14_end): + movaps %xmm4, -0x20(%rdi) + lea 64(%rdx), %rdx + movaps %xmm5, -0x10(%rdi) + add %rdx, %rdi + movdqu %xmm0, (%r8) + add %rdx, %rsi + BRANCH_TO_JMPTBL_ENTRY(L(table_less_80bytes), %rdx, 4) + + .p2align 4 +L(shl_14_bwd): + lea (L(shl_14_bwd_loop_L1)-L(shl_14_bwd))(%r9), %r9 + cmp %rcx, %rdx + movaps -0x0e(%rsi), %xmm1 + jb L(L14_bwd) + lea (L(shl_14_bwd_loop_L2)-L(shl_14_bwd_loop_L1))(%r9), %r9 +L(L14_bwd): + lea -64(%rdx), %rdx + _CET_NOTRACK jmp *%r9 + ud2 +L(shl_14_bwd_loop_L2): + prefetchnta -0x1c0(%rsi) +L(shl_14_bwd_loop_L1): + movaps -0x1e(%rsi), %xmm2 + sub $0x40, %rdx + movaps -0x2e(%rsi), %xmm3 + movaps -0x3e(%rsi), %xmm4 + movaps -0x4e(%rsi), %xmm5 + lea -0x40(%rsi), %rsi + palignr $14, %xmm2, %xmm1 + palignr $14, %xmm3, %xmm2 + palignr $14, %xmm4, %xmm3 + palignr $14, %xmm5, %xmm4 + + movaps %xmm1, -0x10(%rdi) + movaps %xmm5, %xmm1 + + movaps %xmm2, -0x20(%rdi) + lea -0x40(%rdi), %rdi + + movaps %xmm3, 0x10(%rdi) + jb L(shl_14_bwd_end) + movaps %xmm4, (%rdi) + _CET_NOTRACK jmp *%r9 + ud2 +L(shl_14_bwd_end): + movaps %xmm4, (%rdi) + lea 64(%rdx), %rdx + movdqu %xmm0, (%r8) + BRANCH_TO_JMPTBL_ENTRY(L(table_less_80bytes), %rdx, 4) + + .p2align 4 +L(shl_15): + lea (L(shl_15_loop_L1)-L(shl_15))(%r9), %r9 + cmp %rcx, %rdx + movaps -0x0f(%rsi), %xmm1 + jb L(L15_fwd) + lea (L(shl_15_loop_L2)-L(shl_15_loop_L1))(%r9), %r9 +L(L15_fwd): + lea -64(%rdx), %rdx + _CET_NOTRACK jmp *%r9 + ud2 +L(shl_15_loop_L2): + prefetchnta 0x1c0(%rsi) +L(shl_15_loop_L1): + sub $64, %rdx + movaps 0x01(%rsi), %xmm2 + movaps 0x11(%rsi), %xmm3 + movaps 0x21(%rsi), %xmm4 + movaps 0x31(%rsi), %xmm5 + movdqa %xmm5, %xmm6 + palignr $15, %xmm4, %xmm5 + lea 64(%rsi), %rsi + palignr $15, %xmm3, %xmm4 + palignr $15, %xmm2, %xmm3 + lea 64(%rdi), %rdi + palignr $15, %xmm1, %xmm2 + movdqa %xmm6, %xmm1 + movdqa %xmm2, -0x40(%rdi) + movaps %xmm3, -0x30(%rdi) + jb L(shl_15_end) + movaps %xmm4, -0x20(%rdi) + movaps %xmm5, -0x10(%rdi) + _CET_NOTRACK jmp *%r9 + ud2 +L(shl_15_end): + movaps %xmm4, -0x20(%rdi) + lea 64(%rdx), %rdx + movaps %xmm5, -0x10(%rdi) + add %rdx, %rdi + movdqu %xmm0, (%r8) + add %rdx, %rsi + BRANCH_TO_JMPTBL_ENTRY(L(table_less_80bytes), %rdx, 4) + + .p2align 4 +L(shl_15_bwd): + lea (L(shl_15_bwd_loop_L1)-L(shl_15_bwd))(%r9), %r9 + cmp %rcx, %rdx + movaps -0x0f(%rsi), %xmm1 + jb L(L15_bwd) + lea (L(shl_15_bwd_loop_L2)-L(shl_15_bwd_loop_L1))(%r9), %r9 +L(L15_bwd): + lea -64(%rdx), %rdx + _CET_NOTRACK jmp *%r9 + ud2 +L(shl_15_bwd_loop_L2): + prefetchnta -0x1c0(%rsi) +L(shl_15_bwd_loop_L1): + movaps -0x1f(%rsi), %xmm2 + sub $0x40, %rdx + movaps -0x2f(%rsi), %xmm3 + movaps -0x3f(%rsi), %xmm4 + movaps -0x4f(%rsi), %xmm5 + lea -0x40(%rsi), %rsi + palignr $15, %xmm2, %xmm1 + palignr $15, %xmm3, %xmm2 + palignr $15, %xmm4, %xmm3 + palignr $15, %xmm5, %xmm4 + + movaps %xmm1, -0x10(%rdi) + movaps %xmm5, %xmm1 + + movaps %xmm2, -0x20(%rdi) + lea -0x40(%rdi), %rdi + + movaps %xmm3, 0x10(%rdi) + jb L(shl_15_bwd_end) + movaps %xmm4, (%rdi) + _CET_NOTRACK jmp *%r9 + ud2 +L(shl_15_bwd_end): + movaps %xmm4, (%rdi) + lea 64(%rdx), %rdx + movdqu %xmm0, (%r8) + BRANCH_TO_JMPTBL_ENTRY(L(table_less_80bytes), %rdx, 4) + + .p2align 4 +L(write_72bytes): + movdqu -72(%rsi), %xmm0 + movdqu -56(%rsi), %xmm1 + mov -40(%rsi), %r8 + mov -32(%rsi), %r9 + mov -24(%rsi), %r10 + mov -16(%rsi), %r11 + mov -8(%rsi), %rcx + movdqu %xmm0, -72(%rdi) + movdqu %xmm1, -56(%rdi) + mov %r8, -40(%rdi) + mov %r9, -32(%rdi) + mov %r10, -24(%rdi) + mov %r11, -16(%rdi) + mov %rcx, -8(%rdi) + ret + + .p2align 4 +L(write_64bytes): + movdqu -64(%rsi), %xmm0 + mov -48(%rsi), %rcx + mov -40(%rsi), %r8 + mov -32(%rsi), %r9 + mov -24(%rsi), %r10 + mov -16(%rsi), %r11 + mov -8(%rsi), %rdx + movdqu %xmm0, -64(%rdi) + mov %rcx, -48(%rdi) + mov %r8, -40(%rdi) + mov %r9, -32(%rdi) + mov %r10, -24(%rdi) + mov %r11, -16(%rdi) + mov %rdx, -8(%rdi) + ret + + .p2align 4 +L(write_56bytes): + movdqu -56(%rsi), %xmm0 + mov -40(%rsi), %r8 + mov -32(%rsi), %r9 + mov -24(%rsi), %r10 + mov -16(%rsi), %r11 + mov -8(%rsi), %rcx + movdqu %xmm0, -56(%rdi) + mov %r8, -40(%rdi) + mov %r9, -32(%rdi) + mov %r10, -24(%rdi) + mov %r11, -16(%rdi) + mov %rcx, -8(%rdi) + ret + + .p2align 4 +L(write_48bytes): + mov -48(%rsi), %rcx + mov -40(%rsi), %r8 + mov -32(%rsi), %r9 + mov -24(%rsi), %r10 + mov -16(%rsi), %r11 + mov -8(%rsi), %rdx + mov %rcx, -48(%rdi) + mov %r8, -40(%rdi) + mov %r9, -32(%rdi) + mov %r10, -24(%rdi) + mov %r11, -16(%rdi) + mov %rdx, -8(%rdi) + ret + + .p2align 4 +L(write_40bytes): + mov -40(%rsi), %r8 + mov -32(%rsi), %r9 + mov -24(%rsi), %r10 + mov -16(%rsi), %r11 + mov -8(%rsi), %rdx + mov %r8, -40(%rdi) + mov %r9, -32(%rdi) + mov %r10, -24(%rdi) + mov %r11, -16(%rdi) + mov %rdx, -8(%rdi) + ret + + .p2align 4 +L(write_32bytes): + mov -32(%rsi), %r9 + mov -24(%rsi), %r10 + mov -16(%rsi), %r11 + mov -8(%rsi), %rdx + mov %r9, -32(%rdi) + mov %r10, -24(%rdi) + mov %r11, -16(%rdi) + mov %rdx, -8(%rdi) + ret + + .p2align 4 +L(write_24bytes): + mov -24(%rsi), %r10 + mov -16(%rsi), %r11 + mov -8(%rsi), %rdx + mov %r10, -24(%rdi) + mov %r11, -16(%rdi) + mov %rdx, -8(%rdi) + ret + + .p2align 4 +L(write_16bytes): + mov -16(%rsi), %r11 + mov -8(%rsi), %rdx + mov %r11, -16(%rdi) + mov %rdx, -8(%rdi) + ret + + .p2align 4 +L(write_8bytes): + mov -8(%rsi), %rdx + mov %rdx, -8(%rdi) +L(write_0bytes): + ret + + .p2align 4 +L(write_73bytes): + movdqu -73(%rsi), %xmm0 + movdqu -57(%rsi), %xmm1 + mov -41(%rsi), %rcx + mov -33(%rsi), %r9 + mov -25(%rsi), %r10 + mov -17(%rsi), %r11 + mov -9(%rsi), %r8 + mov -4(%rsi), %edx + movdqu %xmm0, -73(%rdi) + movdqu %xmm1, -57(%rdi) + mov %rcx, -41(%rdi) + mov %r9, -33(%rdi) + mov %r10, -25(%rdi) + mov %r11, -17(%rdi) + mov %r8, -9(%rdi) + mov %edx, -4(%rdi) + ret + + .p2align 4 +L(write_65bytes): + movdqu -65(%rsi), %xmm0 + movdqu -49(%rsi), %xmm1 + mov -33(%rsi), %r9 + mov -25(%rsi), %r10 + mov -17(%rsi), %r11 + mov -9(%rsi), %rcx + mov -4(%rsi), %edx + movdqu %xmm0, -65(%rdi) + movdqu %xmm1, -49(%rdi) + mov %r9, -33(%rdi) + mov %r10, -25(%rdi) + mov %r11, -17(%rdi) + mov %rcx, -9(%rdi) + mov %edx, -4(%rdi) + ret + + .p2align 4 +L(write_57bytes): + movdqu -57(%rsi), %xmm0 + mov -41(%rsi), %r8 + mov -33(%rsi), %r9 + mov -25(%rsi), %r10 + mov -17(%rsi), %r11 + mov -9(%rsi), %rcx + mov -4(%rsi), %edx + movdqu %xmm0, -57(%rdi) + mov %r8, -41(%rdi) + mov %r9, -33(%rdi) + mov %r10, -25(%rdi) + mov %r11, -17(%rdi) + mov %rcx, -9(%rdi) + mov %edx, -4(%rdi) + ret + + .p2align 4 +L(write_49bytes): + movdqu -49(%rsi), %xmm0 + mov -33(%rsi), %r9 + mov -25(%rsi), %r10 + mov -17(%rsi), %r11 + mov -9(%rsi), %rcx + mov -4(%rsi), %edx + movdqu %xmm0, -49(%rdi) + mov %r9, -33(%rdi) + mov %r10, -25(%rdi) + mov %r11, -17(%rdi) + mov %rcx, -9(%rdi) + mov %edx, -4(%rdi) + ret + + .p2align 4 +L(write_41bytes): + mov -41(%rsi), %r8 + mov -33(%rsi), %r9 + mov -25(%rsi), %r10 + mov -17(%rsi), %r11 + mov -9(%rsi), %rcx + mov -1(%rsi), %dl + mov %r8, -41(%rdi) + mov %r9, -33(%rdi) + mov %r10, -25(%rdi) + mov %r11, -17(%rdi) + mov %rcx, -9(%rdi) + mov %dl, -1(%rdi) + ret + + .p2align 4 +L(write_33bytes): + mov -33(%rsi), %r9 + mov -25(%rsi), %r10 + mov -17(%rsi), %r11 + mov -9(%rsi), %rcx + mov -1(%rsi), %dl + mov %r9, -33(%rdi) + mov %r10, -25(%rdi) + mov %r11, -17(%rdi) + mov %rcx, -9(%rdi) + mov %dl, -1(%rdi) + ret + + .p2align 4 +L(write_25bytes): + mov -25(%rsi), %r10 + mov -17(%rsi), %r11 + mov -9(%rsi), %rcx + mov -1(%rsi), %dl + mov %r10, -25(%rdi) + mov %r11, -17(%rdi) + mov %rcx, -9(%rdi) + mov %dl, -1(%rdi) + ret + + .p2align 4 +L(write_17bytes): + mov -17(%rsi), %r11 + mov -9(%rsi), %rcx + mov -4(%rsi), %edx + mov %r11, -17(%rdi) + mov %rcx, -9(%rdi) + mov %edx, -4(%rdi) + ret + + .p2align 4 +L(write_9bytes): + mov -9(%rsi), %rcx + mov -4(%rsi), %edx + mov %rcx, -9(%rdi) + mov %edx, -4(%rdi) + ret + + .p2align 4 +L(write_1bytes): + mov -1(%rsi), %dl + mov %dl, -1(%rdi) + ret + + .p2align 4 +L(write_74bytes): + movdqu -74(%rsi), %xmm0 + movdqu -58(%rsi), %xmm1 + mov -42(%rsi), %r8 + mov -34(%rsi), %r9 + mov -26(%rsi), %r10 + mov -18(%rsi), %r11 + mov -10(%rsi), %rcx + mov -4(%rsi), %edx + movdqu %xmm0, -74(%rdi) + movdqu %xmm1, -58(%rdi) + mov %r8, -42(%rdi) + mov %r9, -34(%rdi) + mov %r10, -26(%rdi) + mov %r11, -18(%rdi) + mov %rcx, -10(%rdi) + mov %edx, -4(%rdi) + ret + + .p2align 4 +L(write_66bytes): + movdqu -66(%rsi), %xmm0 + movdqu -50(%rsi), %xmm1 + mov -42(%rsi), %r8 + mov -34(%rsi), %r9 + mov -26(%rsi), %r10 + mov -18(%rsi), %r11 + mov -10(%rsi), %rcx + mov -4(%rsi), %edx + movdqu %xmm0, -66(%rdi) + movdqu %xmm1, -50(%rdi) + mov %r8, -42(%rdi) + mov %r9, -34(%rdi) + mov %r10, -26(%rdi) + mov %r11, -18(%rdi) + mov %rcx, -10(%rdi) + mov %edx, -4(%rdi) + ret + + .p2align 4 +L(write_58bytes): + movdqu -58(%rsi), %xmm1 + mov -42(%rsi), %r8 + mov -34(%rsi), %r9 + mov -26(%rsi), %r10 + mov -18(%rsi), %r11 + mov -10(%rsi), %rcx + mov -4(%rsi), %edx + movdqu %xmm1, -58(%rdi) + mov %r8, -42(%rdi) + mov %r9, -34(%rdi) + mov %r10, -26(%rdi) + mov %r11, -18(%rdi) + mov %rcx, -10(%rdi) + mov %edx, -4(%rdi) + ret + + .p2align 4 +L(write_50bytes): + movdqu -50(%rsi), %xmm0 + mov -34(%rsi), %r9 + mov -26(%rsi), %r10 + mov -18(%rsi), %r11 + mov -10(%rsi), %rcx + mov -4(%rsi), %edx + movdqu %xmm0, -50(%rdi) + mov %r9, -34(%rdi) + mov %r10, -26(%rdi) + mov %r11, -18(%rdi) + mov %rcx, -10(%rdi) + mov %edx, -4(%rdi) + ret + + .p2align 4 +L(write_42bytes): + mov -42(%rsi), %r8 + mov -34(%rsi), %r9 + mov -26(%rsi), %r10 + mov -18(%rsi), %r11 + mov -10(%rsi), %rcx + mov -4(%rsi), %edx + mov %r8, -42(%rdi) + mov %r9, -34(%rdi) + mov %r10, -26(%rdi) + mov %r11, -18(%rdi) + mov %rcx, -10(%rdi) + mov %edx, -4(%rdi) + ret + + .p2align 4 +L(write_34bytes): + mov -34(%rsi), %r9 + mov -26(%rsi), %r10 + mov -18(%rsi), %r11 + mov -10(%rsi), %rcx + mov -4(%rsi), %edx + mov %r9, -34(%rdi) + mov %r10, -26(%rdi) + mov %r11, -18(%rdi) + mov %rcx, -10(%rdi) + mov %edx, -4(%rdi) + ret + + .p2align 4 +L(write_26bytes): + mov -26(%rsi), %r10 + mov -18(%rsi), %r11 + mov -10(%rsi), %rcx + mov -4(%rsi), %edx + mov %r10, -26(%rdi) + mov %r11, -18(%rdi) + mov %rcx, -10(%rdi) + mov %edx, -4(%rdi) + ret + + .p2align 4 +L(write_18bytes): + mov -18(%rsi), %r11 + mov -10(%rsi), %rcx + mov -4(%rsi), %edx + mov %r11, -18(%rdi) + mov %rcx, -10(%rdi) + mov %edx, -4(%rdi) + ret + + .p2align 4 +L(write_10bytes): + mov -10(%rsi), %rcx + mov -4(%rsi), %edx + mov %rcx, -10(%rdi) + mov %edx, -4(%rdi) + ret + + .p2align 4 +L(write_2bytes): + mov -2(%rsi), %dx + mov %dx, -2(%rdi) + ret + + .p2align 4 +L(write_75bytes): + movdqu -75(%rsi), %xmm0 + movdqu -59(%rsi), %xmm1 + mov -43(%rsi), %r8 + mov -35(%rsi), %r9 + mov -27(%rsi), %r10 + mov -19(%rsi), %r11 + mov -11(%rsi), %rcx + mov -4(%rsi), %edx + movdqu %xmm0, -75(%rdi) + movdqu %xmm1, -59(%rdi) + mov %r8, -43(%rdi) + mov %r9, -35(%rdi) + mov %r10, -27(%rdi) + mov %r11, -19(%rdi) + mov %rcx, -11(%rdi) + mov %edx, -4(%rdi) + ret + + .p2align 4 +L(write_67bytes): + movdqu -67(%rsi), %xmm0 + movdqu -59(%rsi), %xmm1 + mov -43(%rsi), %r8 + mov -35(%rsi), %r9 + mov -27(%rsi), %r10 + mov -19(%rsi), %r11 + mov -11(%rsi), %rcx + mov -4(%rsi), %edx + movdqu %xmm0, -67(%rdi) + movdqu %xmm1, -59(%rdi) + mov %r8, -43(%rdi) + mov %r9, -35(%rdi) + mov %r10, -27(%rdi) + mov %r11, -19(%rdi) + mov %rcx, -11(%rdi) + mov %edx, -4(%rdi) + ret + + .p2align 4 +L(write_59bytes): + movdqu -59(%rsi), %xmm0 + mov -43(%rsi), %r8 + mov -35(%rsi), %r9 + mov -27(%rsi), %r10 + mov -19(%rsi), %r11 + mov -11(%rsi), %rcx + mov -4(%rsi), %edx + movdqu %xmm0, -59(%rdi) + mov %r8, -43(%rdi) + mov %r9, -35(%rdi) + mov %r10, -27(%rdi) + mov %r11, -19(%rdi) + mov %rcx, -11(%rdi) + mov %edx, -4(%rdi) + ret + + .p2align 4 +L(write_51bytes): + movdqu -51(%rsi), %xmm0 + mov -35(%rsi), %r9 + mov -27(%rsi), %r10 + mov -19(%rsi), %r11 + mov -11(%rsi), %rcx + mov -4(%rsi), %edx + movdqu %xmm0, -51(%rdi) + mov %r9, -35(%rdi) + mov %r10, -27(%rdi) + mov %r11, -19(%rdi) + mov %rcx, -11(%rdi) + mov %edx, -4(%rdi) + ret + + .p2align 4 +L(write_43bytes): + mov -43(%rsi), %r8 + mov -35(%rsi), %r9 + mov -27(%rsi), %r10 + mov -19(%rsi), %r11 + mov -11(%rsi), %rcx + mov -4(%rsi), %edx + mov %r8, -43(%rdi) + mov %r9, -35(%rdi) + mov %r10, -27(%rdi) + mov %r11, -19(%rdi) + mov %rcx, -11(%rdi) + mov %edx, -4(%rdi) + ret + + .p2align 4 +L(write_35bytes): + mov -35(%rsi), %r9 + mov -27(%rsi), %r10 + mov -19(%rsi), %r11 + mov -11(%rsi), %rcx + mov -4(%rsi), %edx + mov %r9, -35(%rdi) + mov %r10, -27(%rdi) + mov %r11, -19(%rdi) + mov %rcx, -11(%rdi) + mov %edx, -4(%rdi) + ret + + .p2align 4 +L(write_27bytes): + mov -27(%rsi), %r10 + mov -19(%rsi), %r11 + mov -11(%rsi), %rcx + mov -4(%rsi), %edx + mov %r10, -27(%rdi) + mov %r11, -19(%rdi) + mov %rcx, -11(%rdi) + mov %edx, -4(%rdi) + ret + + .p2align 4 +L(write_19bytes): + mov -19(%rsi), %r11 + mov -11(%rsi), %rcx + mov -4(%rsi), %edx + mov %r11, -19(%rdi) + mov %rcx, -11(%rdi) + mov %edx, -4(%rdi) + ret + + .p2align 4 +L(write_11bytes): + mov -11(%rsi), %rcx + mov -4(%rsi), %edx + mov %rcx, -11(%rdi) + mov %edx, -4(%rdi) + ret + + .p2align 4 +L(write_3bytes): + mov -3(%rsi), %dx + mov -2(%rsi), %cx + mov %dx, -3(%rdi) + mov %cx, -2(%rdi) + ret + + .p2align 4 +L(write_76bytes): + movdqu -76(%rsi), %xmm0 + movdqu -60(%rsi), %xmm1 + mov -44(%rsi), %r8 + mov -36(%rsi), %r9 + mov -28(%rsi), %r10 + mov -20(%rsi), %r11 + mov -12(%rsi), %rcx + mov -4(%rsi), %edx + movdqu %xmm0, -76(%rdi) + movdqu %xmm1, -60(%rdi) + mov %r8, -44(%rdi) + mov %r9, -36(%rdi) + mov %r10, -28(%rdi) + mov %r11, -20(%rdi) + mov %rcx, -12(%rdi) + mov %edx, -4(%rdi) + ret + + .p2align 4 +L(write_68bytes): + movdqu -68(%rsi), %xmm0 + movdqu -52(%rsi), %xmm1 + mov -36(%rsi), %r9 + mov -28(%rsi), %r10 + mov -20(%rsi), %r11 + mov -12(%rsi), %rcx + mov -4(%rsi), %edx + movdqu %xmm0, -68(%rdi) + movdqu %xmm1, -52(%rdi) + mov %r9, -36(%rdi) + mov %r10, -28(%rdi) + mov %r11, -20(%rdi) + mov %rcx, -12(%rdi) + mov %edx, -4(%rdi) + ret + + .p2align 4 +L(write_60bytes): + movdqu -60(%rsi), %xmm0 + mov -44(%rsi), %r8 + mov -36(%rsi), %r9 + mov -28(%rsi), %r10 + mov -20(%rsi), %r11 + mov -12(%rsi), %rcx + mov -4(%rsi), %edx + movdqu %xmm0, -60(%rdi) + mov %r8, -44(%rdi) + mov %r9, -36(%rdi) + mov %r10, -28(%rdi) + mov %r11, -20(%rdi) + mov %rcx, -12(%rdi) + mov %edx, -4(%rdi) + ret + + .p2align 4 +L(write_52bytes): + movdqu -52(%rsi), %xmm0 + mov -36(%rsi), %r9 + mov -28(%rsi), %r10 + mov -20(%rsi), %r11 + mov -12(%rsi), %rcx + mov -4(%rsi), %edx + movdqu %xmm0, -52(%rdi) + mov %r9, -36(%rdi) + mov %r10, -28(%rdi) + mov %r11, -20(%rdi) + mov %rcx, -12(%rdi) + mov %edx, -4(%rdi) + ret + + .p2align 4 +L(write_44bytes): + mov -44(%rsi), %r8 + mov -36(%rsi), %r9 + mov -28(%rsi), %r10 + mov -20(%rsi), %r11 + mov -12(%rsi), %rcx + mov -4(%rsi), %edx + mov %r8, -44(%rdi) + mov %r9, -36(%rdi) + mov %r10, -28(%rdi) + mov %r11, -20(%rdi) + mov %rcx, -12(%rdi) + mov %edx, -4(%rdi) + ret + + .p2align 4 +L(write_36bytes): + mov -36(%rsi), %r9 + mov -28(%rsi), %r10 + mov -20(%rsi), %r11 + mov -12(%rsi), %rcx + mov -4(%rsi), %edx + mov %r9, -36(%rdi) + mov %r10, -28(%rdi) + mov %r11, -20(%rdi) + mov %rcx, -12(%rdi) + mov %edx, -4(%rdi) + ret + + .p2align 4 +L(write_28bytes): + mov -28(%rsi), %r10 + mov -20(%rsi), %r11 + mov -12(%rsi), %rcx + mov -4(%rsi), %edx + mov %r10, -28(%rdi) + mov %r11, -20(%rdi) + mov %rcx, -12(%rdi) + mov %edx, -4(%rdi) + ret + + .p2align 4 +L(write_20bytes): + mov -20(%rsi), %r11 + mov -12(%rsi), %rcx + mov -4(%rsi), %edx + mov %r11, -20(%rdi) + mov %rcx, -12(%rdi) + mov %edx, -4(%rdi) + ret + + .p2align 4 +L(write_12bytes): + mov -12(%rsi), %rcx + mov -4(%rsi), %edx + mov %rcx, -12(%rdi) + mov %edx, -4(%rdi) + ret + + .p2align 4 +L(write_4bytes): + mov -4(%rsi), %edx + mov %edx, -4(%rdi) + ret + + .p2align 4 +L(write_77bytes): + movdqu -77(%rsi), %xmm0 + movdqu -61(%rsi), %xmm1 + mov -45(%rsi), %r8 + mov -37(%rsi), %r9 + mov -29(%rsi), %r10 + mov -21(%rsi), %r11 + mov -13(%rsi), %rcx + mov -8(%rsi), %rdx + movdqu %xmm0, -77(%rdi) + movdqu %xmm1, -61(%rdi) + mov %r8, -45(%rdi) + mov %r9, -37(%rdi) + mov %r10, -29(%rdi) + mov %r11, -21(%rdi) + mov %rcx, -13(%rdi) + mov %rdx, -8(%rdi) + ret + + .p2align 4 +L(write_69bytes): + movdqu -69(%rsi), %xmm0 + movdqu -53(%rsi), %xmm1 + mov -37(%rsi), %r9 + mov -29(%rsi), %r10 + mov -21(%rsi), %r11 + mov -13(%rsi), %rcx + mov -8(%rsi), %rdx + movdqu %xmm0, -69(%rdi) + movdqu %xmm1, -53(%rdi) + mov %r9, -37(%rdi) + mov %r10, -29(%rdi) + mov %r11, -21(%rdi) + mov %rcx, -13(%rdi) + mov %rdx, -8(%rdi) + ret + + .p2align 4 +L(write_61bytes): + movdqu -61(%rsi), %xmm0 + mov -45(%rsi), %r8 + mov -37(%rsi), %r9 + mov -29(%rsi), %r10 + mov -21(%rsi), %r11 + mov -13(%rsi), %rcx + mov -8(%rsi), %rdx + movdqu %xmm0, -61(%rdi) + mov %r8, -45(%rdi) + mov %r9, -37(%rdi) + mov %r10, -29(%rdi) + mov %r11, -21(%rdi) + mov %rcx, -13(%rdi) + mov %rdx, -8(%rdi) + ret + + .p2align 4 +L(write_53bytes): + movdqu -53(%rsi), %xmm0 + mov -45(%rsi), %r8 + mov -37(%rsi), %r9 + mov -29(%rsi), %r10 + mov -21(%rsi), %r11 + mov -13(%rsi), %rcx + mov -8(%rsi), %rdx + movdqu %xmm0, -53(%rdi) + mov %r9, -37(%rdi) + mov %r10, -29(%rdi) + mov %r11, -21(%rdi) + mov %rcx, -13(%rdi) + mov %rdx, -8(%rdi) + ret + + .p2align 4 +L(write_45bytes): + mov -45(%rsi), %r8 + mov -37(%rsi), %r9 + mov -29(%rsi), %r10 + mov -21(%rsi), %r11 + mov -13(%rsi), %rcx + mov -8(%rsi), %rdx + mov %r8, -45(%rdi) + mov %r9, -37(%rdi) + mov %r10, -29(%rdi) + mov %r11, -21(%rdi) + mov %rcx, -13(%rdi) + mov %rdx, -8(%rdi) + ret + + .p2align 4 +L(write_37bytes): + mov -37(%rsi), %r9 + mov -29(%rsi), %r10 + mov -21(%rsi), %r11 + mov -13(%rsi), %rcx + mov -8(%rsi), %rdx + mov %r9, -37(%rdi) + mov %r10, -29(%rdi) + mov %r11, -21(%rdi) + mov %rcx, -13(%rdi) + mov %rdx, -8(%rdi) + ret + + .p2align 4 +L(write_29bytes): + mov -29(%rsi), %r10 + mov -21(%rsi), %r11 + mov -13(%rsi), %rcx + mov -8(%rsi), %rdx + mov %r10, -29(%rdi) + mov %r11, -21(%rdi) + mov %rcx, -13(%rdi) + mov %rdx, -8(%rdi) + ret + + .p2align 4 +L(write_21bytes): + mov -21(%rsi), %r11 + mov -13(%rsi), %rcx + mov -8(%rsi), %rdx + mov %r11, -21(%rdi) + mov %rcx, -13(%rdi) + mov %rdx, -8(%rdi) + ret + + .p2align 4 +L(write_13bytes): + mov -13(%rsi), %rcx + mov -8(%rsi), %rdx + mov %rcx, -13(%rdi) + mov %rdx, -8(%rdi) + ret + + .p2align 4 +L(write_5bytes): + mov -5(%rsi), %edx + mov -4(%rsi), %ecx + mov %edx, -5(%rdi) + mov %ecx, -4(%rdi) + ret + + .p2align 4 +L(write_78bytes): + movdqu -78(%rsi), %xmm0 + movdqu -62(%rsi), %xmm1 + mov -46(%rsi), %r8 + mov -38(%rsi), %r9 + mov -30(%rsi), %r10 + mov -22(%rsi), %r11 + mov -14(%rsi), %rcx + mov -8(%rsi), %rdx + movdqu %xmm0, -78(%rdi) + movdqu %xmm1, -62(%rdi) + mov %r8, -46(%rdi) + mov %r9, -38(%rdi) + mov %r10, -30(%rdi) + mov %r11, -22(%rdi) + mov %rcx, -14(%rdi) + mov %rdx, -8(%rdi) + ret + + .p2align 4 +L(write_70bytes): + movdqu -70(%rsi), %xmm0 + movdqu -54(%rsi), %xmm1 + mov -38(%rsi), %r9 + mov -30(%rsi), %r10 + mov -22(%rsi), %r11 + mov -14(%rsi), %rcx + mov -8(%rsi), %rdx + movdqu %xmm0, -70(%rdi) + movdqu %xmm1, -54(%rdi) + mov %r9, -38(%rdi) + mov %r10, -30(%rdi) + mov %r11, -22(%rdi) + mov %rcx, -14(%rdi) + mov %rdx, -8(%rdi) + ret + + .p2align 4 +L(write_62bytes): + movdqu -62(%rsi), %xmm0 + mov -46(%rsi), %r8 + mov -38(%rsi), %r9 + mov -30(%rsi), %r10 + mov -22(%rsi), %r11 + mov -14(%rsi), %rcx + mov -8(%rsi), %rdx + movdqu %xmm0, -62(%rdi) + mov %r8, -46(%rdi) + mov %r9, -38(%rdi) + mov %r10, -30(%rdi) + mov %r11, -22(%rdi) + mov %rcx, -14(%rdi) + mov %rdx, -8(%rdi) + ret + + .p2align 4 +L(write_54bytes): + movdqu -54(%rsi), %xmm0 + mov -38(%rsi), %r9 + mov -30(%rsi), %r10 + mov -22(%rsi), %r11 + mov -14(%rsi), %rcx + mov -8(%rsi), %rdx + movdqu %xmm0, -54(%rdi) + mov %r9, -38(%rdi) + mov %r10, -30(%rdi) + mov %r11, -22(%rdi) + mov %rcx, -14(%rdi) + mov %rdx, -8(%rdi) + ret + + .p2align 4 +L(write_46bytes): + mov -46(%rsi), %r8 + mov -38(%rsi), %r9 + mov -30(%rsi), %r10 + mov -22(%rsi), %r11 + mov -14(%rsi), %rcx + mov -8(%rsi), %rdx + mov %r8, -46(%rdi) + mov %r9, -38(%rdi) + mov %r10, -30(%rdi) + mov %r11, -22(%rdi) + mov %rcx, -14(%rdi) + mov %rdx, -8(%rdi) + ret + + .p2align 4 +L(write_38bytes): + mov -38(%rsi), %r9 + mov -30(%rsi), %r10 + mov -22(%rsi), %r11 + mov -14(%rsi), %rcx + mov -8(%rsi), %rdx + mov %r9, -38(%rdi) + mov %r10, -30(%rdi) + mov %r11, -22(%rdi) + mov %rcx, -14(%rdi) + mov %rdx, -8(%rdi) + ret + + .p2align 4 +L(write_30bytes): + mov -30(%rsi), %r10 + mov -22(%rsi), %r11 + mov -14(%rsi), %rcx + mov -8(%rsi), %rdx + mov %r10, -30(%rdi) + mov %r11, -22(%rdi) + mov %rcx, -14(%rdi) + mov %rdx, -8(%rdi) + ret + + .p2align 4 +L(write_22bytes): + mov -22(%rsi), %r11 + mov -14(%rsi), %rcx + mov -8(%rsi), %rdx + mov %r11, -22(%rdi) + mov %rcx, -14(%rdi) + mov %rdx, -8(%rdi) + ret + + .p2align 4 +L(write_14bytes): + mov -14(%rsi), %rcx + mov -8(%rsi), %rdx + mov %rcx, -14(%rdi) + mov %rdx, -8(%rdi) + ret + + .p2align 4 +L(write_6bytes): + mov -6(%rsi), %edx + mov -4(%rsi), %ecx + mov %edx, -6(%rdi) + mov %ecx, -4(%rdi) + ret + + .p2align 4 +L(write_79bytes): + movdqu -79(%rsi), %xmm0 + movdqu -63(%rsi), %xmm1 + mov -47(%rsi), %r8 + mov -39(%rsi), %r9 + mov -31(%rsi), %r10 + mov -23(%rsi), %r11 + mov -15(%rsi), %rcx + mov -8(%rsi), %rdx + movdqu %xmm0, -79(%rdi) + movdqu %xmm1, -63(%rdi) + mov %r8, -47(%rdi) + mov %r9, -39(%rdi) + mov %r10, -31(%rdi) + mov %r11, -23(%rdi) + mov %rcx, -15(%rdi) + mov %rdx, -8(%rdi) + ret + + .p2align 4 +L(write_71bytes): + movdqu -71(%rsi), %xmm0 + movdqu -55(%rsi), %xmm1 + mov -39(%rsi), %r9 + mov -31(%rsi), %r10 + mov -23(%rsi), %r11 + mov -15(%rsi), %rcx + mov -8(%rsi), %rdx + movdqu %xmm0, -71(%rdi) + movdqu %xmm1, -55(%rdi) + mov %r9, -39(%rdi) + mov %r10, -31(%rdi) + mov %r11, -23(%rdi) + mov %rcx, -15(%rdi) + mov %rdx, -8(%rdi) + ret + + .p2align 4 +L(write_63bytes): + movdqu -63(%rsi), %xmm0 + mov -47(%rsi), %r8 + mov -39(%rsi), %r9 + mov -31(%rsi), %r10 + mov -23(%rsi), %r11 + mov -15(%rsi), %rcx + mov -8(%rsi), %rdx + movdqu %xmm0, -63(%rdi) + mov %r8, -47(%rdi) + mov %r9, -39(%rdi) + mov %r10, -31(%rdi) + mov %r11, -23(%rdi) + mov %rcx, -15(%rdi) + mov %rdx, -8(%rdi) + ret + + .p2align 4 +L(write_55bytes): + movdqu -55(%rsi), %xmm0 + mov -39(%rsi), %r9 + mov -31(%rsi), %r10 + mov -23(%rsi), %r11 + mov -15(%rsi), %rcx + mov -8(%rsi), %rdx + movdqu %xmm0, -55(%rdi) + mov %r9, -39(%rdi) + mov %r10, -31(%rdi) + mov %r11, -23(%rdi) + mov %rcx, -15(%rdi) + mov %rdx, -8(%rdi) + ret + + .p2align 4 +L(write_47bytes): + mov -47(%rsi), %r8 + mov -39(%rsi), %r9 + mov -31(%rsi), %r10 + mov -23(%rsi), %r11 + mov -15(%rsi), %rcx + mov -8(%rsi), %rdx + mov %r8, -47(%rdi) + mov %r9, -39(%rdi) + mov %r10, -31(%rdi) + mov %r11, -23(%rdi) + mov %rcx, -15(%rdi) + mov %rdx, -8(%rdi) + ret + + .p2align 4 +L(write_39bytes): + mov -39(%rsi), %r9 + mov -31(%rsi), %r10 + mov -23(%rsi), %r11 + mov -15(%rsi), %rcx + mov -8(%rsi), %rdx + mov %r9, -39(%rdi) + mov %r10, -31(%rdi) + mov %r11, -23(%rdi) + mov %rcx, -15(%rdi) + mov %rdx, -8(%rdi) + ret + + .p2align 4 +L(write_31bytes): + mov -31(%rsi), %r10 + mov -23(%rsi), %r11 + mov -15(%rsi), %rcx + mov -8(%rsi), %rdx + mov %r10, -31(%rdi) + mov %r11, -23(%rdi) + mov %rcx, -15(%rdi) + mov %rdx, -8(%rdi) + ret + + .p2align 4 +L(write_23bytes): + mov -23(%rsi), %r11 + mov -15(%rsi), %rcx + mov -8(%rsi), %rdx + mov %r11, -23(%rdi) + mov %rcx, -15(%rdi) + mov %rdx, -8(%rdi) + ret + + .p2align 4 +L(write_15bytes): + mov -15(%rsi), %rcx + mov -8(%rsi), %rdx + mov %rcx, -15(%rdi) + mov %rdx, -8(%rdi) + ret + + .p2align 4 +L(write_7bytes): + mov -7(%rsi), %edx + mov -4(%rsi), %ecx + mov %edx, -7(%rdi) + mov %ecx, -4(%rdi) + ret + + .p2align 4 +L(large_page_fwd): + movdqu (%rsi), %xmm1 + lea 16(%rsi), %rsi + movdqu %xmm0, (%r8) + movntdq %xmm1, (%rdi) + lea 16(%rdi), %rdi + lea -0x90(%rdx), %rdx +#ifdef USE_AS_MEMMOVE + mov %rsi, %r9 + sub %rdi, %r9 + cmp %rdx, %r9 + jae L(memmove_is_memcpy_fwd) + shl $2, %rcx + cmp %rcx, %rdx + jb L(ll_cache_copy_fwd_start) +L(memmove_is_memcpy_fwd): +#endif +L(large_page_loop): + movdqu (%rsi), %xmm0 + movdqu 0x10(%rsi), %xmm1 + movdqu 0x20(%rsi), %xmm2 + movdqu 0x30(%rsi), %xmm3 + movdqu 0x40(%rsi), %xmm4 + movdqu 0x50(%rsi), %xmm5 + movdqu 0x60(%rsi), %xmm6 + movdqu 0x70(%rsi), %xmm7 + lea 0x80(%rsi), %rsi + + sub $0x80, %rdx + movntdq %xmm0, (%rdi) + movntdq %xmm1, 0x10(%rdi) + movntdq %xmm2, 0x20(%rdi) + movntdq %xmm3, 0x30(%rdi) + movntdq %xmm4, 0x40(%rdi) + movntdq %xmm5, 0x50(%rdi) + movntdq %xmm6, 0x60(%rdi) + movntdq %xmm7, 0x70(%rdi) + lea 0x80(%rdi), %rdi + jae L(large_page_loop) + cmp $-0x40, %rdx + lea 0x80(%rdx), %rdx + jl L(large_page_less_64bytes) + + movdqu (%rsi), %xmm0 + movdqu 0x10(%rsi), %xmm1 + movdqu 0x20(%rsi), %xmm2 + movdqu 0x30(%rsi), %xmm3 + lea 0x40(%rsi), %rsi + + movntdq %xmm0, (%rdi) + movntdq %xmm1, 0x10(%rdi) + movntdq %xmm2, 0x20(%rdi) + movntdq %xmm3, 0x30(%rdi) + lea 0x40(%rdi), %rdi + sub $0x40, %rdx +L(large_page_less_64bytes): + add %rdx, %rsi + add %rdx, %rdi + sfence + BRANCH_TO_JMPTBL_ENTRY (L(table_less_80bytes), %rdx, 4) + +#ifdef USE_AS_MEMMOVE + .p2align 4 +L(ll_cache_copy_fwd_start): + prefetcht0 0x1c0(%rsi) + prefetcht0 0x200(%rsi) + movdqu (%rsi), %xmm0 + movdqu 0x10(%rsi), %xmm1 + movdqu 0x20(%rsi), %xmm2 + movdqu 0x30(%rsi), %xmm3 + movdqu 0x40(%rsi), %xmm4 + movdqu 0x50(%rsi), %xmm5 + movdqu 0x60(%rsi), %xmm6 + movdqu 0x70(%rsi), %xmm7 + lea 0x80(%rsi), %rsi + + sub $0x80, %rdx + movaps %xmm0, (%rdi) + movaps %xmm1, 0x10(%rdi) + movaps %xmm2, 0x20(%rdi) + movaps %xmm3, 0x30(%rdi) + movaps %xmm4, 0x40(%rdi) + movaps %xmm5, 0x50(%rdi) + movaps %xmm6, 0x60(%rdi) + movaps %xmm7, 0x70(%rdi) + lea 0x80(%rdi), %rdi + jae L(ll_cache_copy_fwd_start) + cmp $-0x40, %rdx + lea 0x80(%rdx), %rdx + jl L(large_page_ll_less_fwd_64bytes) + + movdqu (%rsi), %xmm0 + movdqu 0x10(%rsi), %xmm1 + movdqu 0x20(%rsi), %xmm2 + movdqu 0x30(%rsi), %xmm3 + lea 0x40(%rsi), %rsi + + movaps %xmm0, (%rdi) + movaps %xmm1, 0x10(%rdi) + movaps %xmm2, 0x20(%rdi) + movaps %xmm3, 0x30(%rdi) + lea 0x40(%rdi), %rdi + sub $0x40, %rdx +L(large_page_ll_less_fwd_64bytes): + add %rdx, %rsi + add %rdx, %rdi + BRANCH_TO_JMPTBL_ENTRY (L(table_less_80bytes), %rdx, 4) + +#endif + .p2align 4 +L(large_page_bwd): + movdqu -0x10(%rsi), %xmm1 + lea -16(%rsi), %rsi + movdqu %xmm0, (%r8) + movdqa %xmm1, -0x10(%rdi) + lea -16(%rdi), %rdi + lea -0x90(%rdx), %rdx +#ifdef USE_AS_MEMMOVE + mov %rdi, %r9 + sub %rsi, %r9 + cmp %rdx, %r9 + jae L(memmove_is_memcpy_bwd) + cmp %rcx, %r9 + jb L(ll_cache_copy_bwd_start) +L(memmove_is_memcpy_bwd): +#endif +L(large_page_bwd_loop): + movdqu -0x10(%rsi), %xmm0 + movdqu -0x20(%rsi), %xmm1 + movdqu -0x30(%rsi), %xmm2 + movdqu -0x40(%rsi), %xmm3 + movdqu -0x50(%rsi), %xmm4 + movdqu -0x60(%rsi), %xmm5 + movdqu -0x70(%rsi), %xmm6 + movdqu -0x80(%rsi), %xmm7 + lea -0x80(%rsi), %rsi + + sub $0x80, %rdx + movntdq %xmm0, -0x10(%rdi) + movntdq %xmm1, -0x20(%rdi) + movntdq %xmm2, -0x30(%rdi) + movntdq %xmm3, -0x40(%rdi) + movntdq %xmm4, -0x50(%rdi) + movntdq %xmm5, -0x60(%rdi) + movntdq %xmm6, -0x70(%rdi) + movntdq %xmm7, -0x80(%rdi) + lea -0x80(%rdi), %rdi + jae L(large_page_bwd_loop) + cmp $-0x40, %rdx + lea 0x80(%rdx), %rdx + jl L(large_page_less_bwd_64bytes) + + movdqu -0x10(%rsi), %xmm0 + movdqu -0x20(%rsi), %xmm1 + movdqu -0x30(%rsi), %xmm2 + movdqu -0x40(%rsi), %xmm3 + lea -0x40(%rsi), %rsi + + movntdq %xmm0, -0x10(%rdi) + movntdq %xmm1, -0x20(%rdi) + movntdq %xmm2, -0x30(%rdi) + movntdq %xmm3, -0x40(%rdi) + lea -0x40(%rdi), %rdi + sub $0x40, %rdx +L(large_page_less_bwd_64bytes): + sfence + BRANCH_TO_JMPTBL_ENTRY (L(table_less_80bytes), %rdx, 4) + +#ifdef USE_AS_MEMMOVE + .p2align 4 +L(ll_cache_copy_bwd_start): + prefetcht0 -0x1c0(%rsi) + prefetcht0 -0x200(%rsi) + movdqu -0x10(%rsi), %xmm0 + movdqu -0x20(%rsi), %xmm1 + movdqu -0x30(%rsi), %xmm2 + movdqu -0x40(%rsi), %xmm3 + movdqu -0x50(%rsi), %xmm4 + movdqu -0x60(%rsi), %xmm5 + movdqu -0x70(%rsi), %xmm6 + movdqu -0x80(%rsi), %xmm7 + lea -0x80(%rsi), %rsi + + sub $0x80, %rdx + movaps %xmm0, -0x10(%rdi) + movaps %xmm1, -0x20(%rdi) + movaps %xmm2, -0x30(%rdi) + movaps %xmm3, -0x40(%rdi) + movaps %xmm4, -0x50(%rdi) + movaps %xmm5, -0x60(%rdi) + movaps %xmm6, -0x70(%rdi) + movaps %xmm7, -0x80(%rdi) + lea -0x80(%rdi), %rdi + jae L(ll_cache_copy_bwd_start) + cmp $-0x40, %rdx + lea 0x80(%rdx), %rdx + jl L(large_page_ll_less_bwd_64bytes) + + movdqu -0x10(%rsi), %xmm0 + movdqu -0x20(%rsi), %xmm1 + movdqu -0x30(%rsi), %xmm2 + movdqu -0x40(%rsi), %xmm3 + lea -0x40(%rsi), %rsi + + movaps %xmm0, -0x10(%rdi) + movaps %xmm1, -0x20(%rdi) + movaps %xmm2, -0x30(%rdi) + movaps %xmm3, -0x40(%rdi) + lea -0x40(%rdi), %rdi + sub $0x40, %rdx +L(large_page_ll_less_bwd_64bytes): + BRANCH_TO_JMPTBL_ENTRY (L(table_less_80bytes), %rdx, 4) +#endif + +END (MEMCPY) + + .section .rodata.ssse3,"a",@progbits + .p2align 3 +L(table_less_80bytes): + .int JMPTBL (L(write_0bytes), L(table_less_80bytes)) + .int JMPTBL (L(write_1bytes), L(table_less_80bytes)) + .int JMPTBL (L(write_2bytes), L(table_less_80bytes)) + .int JMPTBL (L(write_3bytes), L(table_less_80bytes)) + .int JMPTBL (L(write_4bytes), L(table_less_80bytes)) + .int JMPTBL (L(write_5bytes), L(table_less_80bytes)) + .int JMPTBL (L(write_6bytes), L(table_less_80bytes)) + .int JMPTBL (L(write_7bytes), L(table_less_80bytes)) + .int JMPTBL (L(write_8bytes), L(table_less_80bytes)) + .int JMPTBL (L(write_9bytes), L(table_less_80bytes)) + .int JMPTBL (L(write_10bytes), L(table_less_80bytes)) + .int JMPTBL (L(write_11bytes), L(table_less_80bytes)) + .int JMPTBL (L(write_12bytes), L(table_less_80bytes)) + .int JMPTBL (L(write_13bytes), L(table_less_80bytes)) + .int JMPTBL (L(write_14bytes), L(table_less_80bytes)) + .int JMPTBL (L(write_15bytes), L(table_less_80bytes)) + .int JMPTBL (L(write_16bytes), L(table_less_80bytes)) + .int JMPTBL (L(write_17bytes), L(table_less_80bytes)) + .int JMPTBL (L(write_18bytes), L(table_less_80bytes)) + .int JMPTBL (L(write_19bytes), L(table_less_80bytes)) + .int JMPTBL (L(write_20bytes), L(table_less_80bytes)) + .int JMPTBL (L(write_21bytes), L(table_less_80bytes)) + .int JMPTBL (L(write_22bytes), L(table_less_80bytes)) + .int JMPTBL (L(write_23bytes), L(table_less_80bytes)) + .int JMPTBL (L(write_24bytes), L(table_less_80bytes)) + .int JMPTBL (L(write_25bytes), L(table_less_80bytes)) + .int JMPTBL (L(write_26bytes), L(table_less_80bytes)) + .int JMPTBL (L(write_27bytes), L(table_less_80bytes)) + .int JMPTBL (L(write_28bytes), L(table_less_80bytes)) + .int JMPTBL (L(write_29bytes), L(table_less_80bytes)) + .int JMPTBL (L(write_30bytes), L(table_less_80bytes)) + .int JMPTBL (L(write_31bytes), L(table_less_80bytes)) + .int JMPTBL (L(write_32bytes), L(table_less_80bytes)) + .int JMPTBL (L(write_33bytes), L(table_less_80bytes)) + .int JMPTBL (L(write_34bytes), L(table_less_80bytes)) + .int JMPTBL (L(write_35bytes), L(table_less_80bytes)) + .int JMPTBL (L(write_36bytes), L(table_less_80bytes)) + .int JMPTBL (L(write_37bytes), L(table_less_80bytes)) + .int JMPTBL (L(write_38bytes), L(table_less_80bytes)) + .int JMPTBL (L(write_39bytes), L(table_less_80bytes)) + .int JMPTBL (L(write_40bytes), L(table_less_80bytes)) + .int JMPTBL (L(write_41bytes), L(table_less_80bytes)) + .int JMPTBL (L(write_42bytes), L(table_less_80bytes)) + .int JMPTBL (L(write_43bytes), L(table_less_80bytes)) + .int JMPTBL (L(write_44bytes), L(table_less_80bytes)) + .int JMPTBL (L(write_45bytes), L(table_less_80bytes)) + .int JMPTBL (L(write_46bytes), L(table_less_80bytes)) + .int JMPTBL (L(write_47bytes), L(table_less_80bytes)) + .int JMPTBL (L(write_48bytes), L(table_less_80bytes)) + .int JMPTBL (L(write_49bytes), L(table_less_80bytes)) + .int JMPTBL (L(write_50bytes), L(table_less_80bytes)) + .int JMPTBL (L(write_51bytes), L(table_less_80bytes)) + .int JMPTBL (L(write_52bytes), L(table_less_80bytes)) + .int JMPTBL (L(write_53bytes), L(table_less_80bytes)) + .int JMPTBL (L(write_54bytes), L(table_less_80bytes)) + .int JMPTBL (L(write_55bytes), L(table_less_80bytes)) + .int JMPTBL (L(write_56bytes), L(table_less_80bytes)) + .int JMPTBL (L(write_57bytes), L(table_less_80bytes)) + .int JMPTBL (L(write_58bytes), L(table_less_80bytes)) + .int JMPTBL (L(write_59bytes), L(table_less_80bytes)) + .int JMPTBL (L(write_60bytes), L(table_less_80bytes)) + .int JMPTBL (L(write_61bytes), L(table_less_80bytes)) + .int JMPTBL (L(write_62bytes), L(table_less_80bytes)) + .int JMPTBL (L(write_63bytes), L(table_less_80bytes)) + .int JMPTBL (L(write_64bytes), L(table_less_80bytes)) + .int JMPTBL (L(write_65bytes), L(table_less_80bytes)) + .int JMPTBL (L(write_66bytes), L(table_less_80bytes)) + .int JMPTBL (L(write_67bytes), L(table_less_80bytes)) + .int JMPTBL (L(write_68bytes), L(table_less_80bytes)) + .int JMPTBL (L(write_69bytes), L(table_less_80bytes)) + .int JMPTBL (L(write_70bytes), L(table_less_80bytes)) + .int JMPTBL (L(write_71bytes), L(table_less_80bytes)) + .int JMPTBL (L(write_72bytes), L(table_less_80bytes)) + .int JMPTBL (L(write_73bytes), L(table_less_80bytes)) + .int JMPTBL (L(write_74bytes), L(table_less_80bytes)) + .int JMPTBL (L(write_75bytes), L(table_less_80bytes)) + .int JMPTBL (L(write_76bytes), L(table_less_80bytes)) + .int JMPTBL (L(write_77bytes), L(table_less_80bytes)) + .int JMPTBL (L(write_78bytes), L(table_less_80bytes)) + .int JMPTBL (L(write_79bytes), L(table_less_80bytes)) + + .p2align 3 +L(shl_table): + .int JMPTBL (L(shl_0), L(shl_table)) + .int JMPTBL (L(shl_1), L(shl_table)) + .int JMPTBL (L(shl_2), L(shl_table)) + .int JMPTBL (L(shl_3), L(shl_table)) + .int JMPTBL (L(shl_4), L(shl_table)) + .int JMPTBL (L(shl_5), L(shl_table)) + .int JMPTBL (L(shl_6), L(shl_table)) + .int JMPTBL (L(shl_7), L(shl_table)) + .int JMPTBL (L(shl_8), L(shl_table)) + .int JMPTBL (L(shl_9), L(shl_table)) + .int JMPTBL (L(shl_10), L(shl_table)) + .int JMPTBL (L(shl_11), L(shl_table)) + .int JMPTBL (L(shl_12), L(shl_table)) + .int JMPTBL (L(shl_13), L(shl_table)) + .int JMPTBL (L(shl_14), L(shl_table)) + .int JMPTBL (L(shl_15), L(shl_table)) + + .p2align 3 +L(shl_table_bwd): + .int JMPTBL (L(shl_0_bwd), L(shl_table_bwd)) + .int JMPTBL (L(shl_1_bwd), L(shl_table_bwd)) + .int JMPTBL (L(shl_2_bwd), L(shl_table_bwd)) + .int JMPTBL (L(shl_3_bwd), L(shl_table_bwd)) + .int JMPTBL (L(shl_4_bwd), L(shl_table_bwd)) + .int JMPTBL (L(shl_5_bwd), L(shl_table_bwd)) + .int JMPTBL (L(shl_6_bwd), L(shl_table_bwd)) + .int JMPTBL (L(shl_7_bwd), L(shl_table_bwd)) + .int JMPTBL (L(shl_8_bwd), L(shl_table_bwd)) + .int JMPTBL (L(shl_9_bwd), L(shl_table_bwd)) + .int JMPTBL (L(shl_10_bwd), L(shl_table_bwd)) + .int JMPTBL (L(shl_11_bwd), L(shl_table_bwd)) + .int JMPTBL (L(shl_12_bwd), L(shl_table_bwd)) + .int JMPTBL (L(shl_13_bwd), L(shl_table_bwd)) + .int JMPTBL (L(shl_14_bwd), L(shl_table_bwd)) + .int JMPTBL (L(shl_15_bwd), L(shl_table_bwd)) + +#endif diff --git a/utils/memcpy-bench/glibc/memmove-avx-unaligned-erms.S b/utils/memcpy-bench/glibc/memmove-avx-unaligned-erms.S new file mode 100644 index 00000000000..9ee6f0a71c3 --- /dev/null +++ b/utils/memcpy-bench/glibc/memmove-avx-unaligned-erms.S @@ -0,0 +1,12 @@ +#if 1 +# define VEC_SIZE 32 +# define VEC(i) ymm##i +# define VMOVNT vmovntdq +# define VMOVU vmovdqu +# define VMOVA vmovdqa + +# define SECTION(p) p##.avx +# define MEMMOVE_SYMBOL(p,s) p##_avx_##s + +# include "memmove-vec-unaligned-erms.S" +#endif diff --git a/utils/memcpy-bench/glibc/memmove-avx512-no-vzeroupper.S b/utils/memcpy-bench/glibc/memmove-avx512-no-vzeroupper.S new file mode 100644 index 00000000000..b14d92fd6a8 --- /dev/null +++ b/utils/memcpy-bench/glibc/memmove-avx512-no-vzeroupper.S @@ -0,0 +1,419 @@ +/* memmove/memcpy/mempcpy optimized with AVX512 for KNL hardware. + Copyright (C) 2016-2020 Free Software Foundation, Inc. + This file is part of the GNU C Library. + + The GNU C Library is free software; you can redistribute it and/or + modify it under the terms of the GNU Lesser General Public + License as published by the Free Software Foundation; either + version 2.1 of the License, or (at your option) any later version. + + The GNU C Library is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + Lesser General Public License for more details. + + You should have received a copy of the GNU Lesser General Public + License along with the GNU C Library; if not, see + . */ + +#include "sysdep.h" + +#if 1 + +# include "asm-syntax.h" + + .section .text.avx512,"ax",@progbits +ENTRY (__mempcpy_chk_avx512_no_vzeroupper) + cmp %RDX_LP, %RCX_LP + jb HIDDEN_JUMPTARGET (__chk_fail) +END (__mempcpy_chk_avx512_no_vzeroupper) + +ENTRY (__mempcpy_avx512_no_vzeroupper) + mov %RDI_LP, %RAX_LP + add %RDX_LP, %RAX_LP + jmp L(start) +END (__mempcpy_avx512_no_vzeroupper) + +ENTRY (__memmove_chk_avx512_no_vzeroupper) + cmp %RDX_LP, %RCX_LP + jb HIDDEN_JUMPTARGET (__chk_fail) +END (__memmove_chk_avx512_no_vzeroupper) + +ENTRY (__memmove_avx512_no_vzeroupper) + mov %RDI_LP, %RAX_LP +# ifdef USE_AS_MEMPCPY + add %RDX_LP, %RAX_LP +# endif +L(start): +# ifdef __ILP32__ + /* Clear the upper 32 bits. */ + mov %edx, %edx +# endif + lea (%rsi, %rdx), %rcx + lea (%rdi, %rdx), %r9 + cmp $512, %rdx + ja L(512bytesormore) + +L(check): + cmp $16, %rdx + jbe L(less_16bytes) + cmp $256, %rdx + jb L(less_256bytes) + vmovups (%rsi), %zmm0 + vmovups 0x40(%rsi), %zmm1 + vmovups 0x80(%rsi), %zmm2 + vmovups 0xC0(%rsi), %zmm3 + vmovups -0x100(%rcx), %zmm4 + vmovups -0xC0(%rcx), %zmm5 + vmovups -0x80(%rcx), %zmm6 + vmovups -0x40(%rcx), %zmm7 + vmovups %zmm0, (%rdi) + vmovups %zmm1, 0x40(%rdi) + vmovups %zmm2, 0x80(%rdi) + vmovups %zmm3, 0xC0(%rdi) + vmovups %zmm4, -0x100(%r9) + vmovups %zmm5, -0xC0(%r9) + vmovups %zmm6, -0x80(%r9) + vmovups %zmm7, -0x40(%r9) + ret + +L(less_256bytes): + cmp $128, %dl + jb L(less_128bytes) + vmovups (%rsi), %zmm0 + vmovups 0x40(%rsi), %zmm1 + vmovups -0x80(%rcx), %zmm2 + vmovups -0x40(%rcx), %zmm3 + vmovups %zmm0, (%rdi) + vmovups %zmm1, 0x40(%rdi) + vmovups %zmm2, -0x80(%r9) + vmovups %zmm3, -0x40(%r9) + ret + +L(less_128bytes): + cmp $64, %dl + jb L(less_64bytes) + vmovdqu (%rsi), %ymm0 + vmovdqu 0x20(%rsi), %ymm1 + vmovdqu -0x40(%rcx), %ymm2 + vmovdqu -0x20(%rcx), %ymm3 + vmovdqu %ymm0, (%rdi) + vmovdqu %ymm1, 0x20(%rdi) + vmovdqu %ymm2, -0x40(%r9) + vmovdqu %ymm3, -0x20(%r9) + ret + +L(less_64bytes): + cmp $32, %dl + jb L(less_32bytes) + vmovdqu (%rsi), %ymm0 + vmovdqu -0x20(%rcx), %ymm1 + vmovdqu %ymm0, (%rdi) + vmovdqu %ymm1, -0x20(%r9) + ret + +L(less_32bytes): + vmovdqu (%rsi), %xmm0 + vmovdqu -0x10(%rcx), %xmm1 + vmovdqu %xmm0, (%rdi) + vmovdqu %xmm1, -0x10(%r9) + ret + +L(less_16bytes): + cmp $8, %dl + jb L(less_8bytes) + movq (%rsi), %rsi + movq -0x8(%rcx), %rcx + movq %rsi, (%rdi) + movq %rcx, -0x8(%r9) + ret + +L(less_8bytes): + cmp $4, %dl + jb L(less_4bytes) + mov (%rsi), %esi + mov -0x4(%rcx), %ecx + mov %esi, (%rdi) + mov %ecx, -0x4(%r9) + ret + +L(less_4bytes): + cmp $2, %dl + jb L(less_2bytes) + mov (%rsi), %si + mov -0x2(%rcx), %cx + mov %si, (%rdi) + mov %cx, -0x2(%r9) + ret + +L(less_2bytes): + cmp $1, %dl + jb L(less_1bytes) + mov (%rsi), %cl + mov %cl, (%rdi) +L(less_1bytes): + ret + +L(512bytesormore): +# ifdef SHARED_CACHE_SIZE_HALF + mov $SHARED_CACHE_SIZE_HALF, %r8 +# else + mov __x86_shared_cache_size_half(%rip), %r8 +# endif + cmp %r8, %rdx + jae L(preloop_large) + cmp $1024, %rdx + ja L(1024bytesormore) + prefetcht1 (%rsi) + prefetcht1 0x40(%rsi) + prefetcht1 0x80(%rsi) + prefetcht1 0xC0(%rsi) + prefetcht1 0x100(%rsi) + prefetcht1 0x140(%rsi) + prefetcht1 0x180(%rsi) + prefetcht1 0x1C0(%rsi) + prefetcht1 -0x200(%rcx) + prefetcht1 -0x1C0(%rcx) + prefetcht1 -0x180(%rcx) + prefetcht1 -0x140(%rcx) + prefetcht1 -0x100(%rcx) + prefetcht1 -0xC0(%rcx) + prefetcht1 -0x80(%rcx) + prefetcht1 -0x40(%rcx) + vmovups (%rsi), %zmm0 + vmovups 0x40(%rsi), %zmm1 + vmovups 0x80(%rsi), %zmm2 + vmovups 0xC0(%rsi), %zmm3 + vmovups 0x100(%rsi), %zmm4 + vmovups 0x140(%rsi), %zmm5 + vmovups 0x180(%rsi), %zmm6 + vmovups 0x1C0(%rsi), %zmm7 + vmovups -0x200(%rcx), %zmm8 + vmovups -0x1C0(%rcx), %zmm9 + vmovups -0x180(%rcx), %zmm10 + vmovups -0x140(%rcx), %zmm11 + vmovups -0x100(%rcx), %zmm12 + vmovups -0xC0(%rcx), %zmm13 + vmovups -0x80(%rcx), %zmm14 + vmovups -0x40(%rcx), %zmm15 + vmovups %zmm0, (%rdi) + vmovups %zmm1, 0x40(%rdi) + vmovups %zmm2, 0x80(%rdi) + vmovups %zmm3, 0xC0(%rdi) + vmovups %zmm4, 0x100(%rdi) + vmovups %zmm5, 0x140(%rdi) + vmovups %zmm6, 0x180(%rdi) + vmovups %zmm7, 0x1C0(%rdi) + vmovups %zmm8, -0x200(%r9) + vmovups %zmm9, -0x1C0(%r9) + vmovups %zmm10, -0x180(%r9) + vmovups %zmm11, -0x140(%r9) + vmovups %zmm12, -0x100(%r9) + vmovups %zmm13, -0xC0(%r9) + vmovups %zmm14, -0x80(%r9) + vmovups %zmm15, -0x40(%r9) + ret + +L(1024bytesormore): + cmp %rsi, %rdi + ja L(1024bytesormore_bkw) + sub $512, %r9 + vmovups -0x200(%rcx), %zmm8 + vmovups -0x1C0(%rcx), %zmm9 + vmovups -0x180(%rcx), %zmm10 + vmovups -0x140(%rcx), %zmm11 + vmovups -0x100(%rcx), %zmm12 + vmovups -0xC0(%rcx), %zmm13 + vmovups -0x80(%rcx), %zmm14 + vmovups -0x40(%rcx), %zmm15 + prefetcht1 (%rsi) + prefetcht1 0x40(%rsi) + prefetcht1 0x80(%rsi) + prefetcht1 0xC0(%rsi) + prefetcht1 0x100(%rsi) + prefetcht1 0x140(%rsi) + prefetcht1 0x180(%rsi) + prefetcht1 0x1C0(%rsi) + +/* Loop with unaligned memory access. */ +L(gobble_512bytes_loop): + vmovups (%rsi), %zmm0 + vmovups 0x40(%rsi), %zmm1 + vmovups 0x80(%rsi), %zmm2 + vmovups 0xC0(%rsi), %zmm3 + vmovups 0x100(%rsi), %zmm4 + vmovups 0x140(%rsi), %zmm5 + vmovups 0x180(%rsi), %zmm6 + vmovups 0x1C0(%rsi), %zmm7 + add $512, %rsi + prefetcht1 (%rsi) + prefetcht1 0x40(%rsi) + prefetcht1 0x80(%rsi) + prefetcht1 0xC0(%rsi) + prefetcht1 0x100(%rsi) + prefetcht1 0x140(%rsi) + prefetcht1 0x180(%rsi) + prefetcht1 0x1C0(%rsi) + vmovups %zmm0, (%rdi) + vmovups %zmm1, 0x40(%rdi) + vmovups %zmm2, 0x80(%rdi) + vmovups %zmm3, 0xC0(%rdi) + vmovups %zmm4, 0x100(%rdi) + vmovups %zmm5, 0x140(%rdi) + vmovups %zmm6, 0x180(%rdi) + vmovups %zmm7, 0x1C0(%rdi) + add $512, %rdi + cmp %r9, %rdi + jb L(gobble_512bytes_loop) + vmovups %zmm8, (%r9) + vmovups %zmm9, 0x40(%r9) + vmovups %zmm10, 0x80(%r9) + vmovups %zmm11, 0xC0(%r9) + vmovups %zmm12, 0x100(%r9) + vmovups %zmm13, 0x140(%r9) + vmovups %zmm14, 0x180(%r9) + vmovups %zmm15, 0x1C0(%r9) + ret + +L(1024bytesormore_bkw): + add $512, %rdi + vmovups 0x1C0(%rsi), %zmm8 + vmovups 0x180(%rsi), %zmm9 + vmovups 0x140(%rsi), %zmm10 + vmovups 0x100(%rsi), %zmm11 + vmovups 0xC0(%rsi), %zmm12 + vmovups 0x80(%rsi), %zmm13 + vmovups 0x40(%rsi), %zmm14 + vmovups (%rsi), %zmm15 + prefetcht1 -0x40(%rcx) + prefetcht1 -0x80(%rcx) + prefetcht1 -0xC0(%rcx) + prefetcht1 -0x100(%rcx) + prefetcht1 -0x140(%rcx) + prefetcht1 -0x180(%rcx) + prefetcht1 -0x1C0(%rcx) + prefetcht1 -0x200(%rcx) + +/* Backward loop with unaligned memory access. */ +L(gobble_512bytes_loop_bkw): + vmovups -0x40(%rcx), %zmm0 + vmovups -0x80(%rcx), %zmm1 + vmovups -0xC0(%rcx), %zmm2 + vmovups -0x100(%rcx), %zmm3 + vmovups -0x140(%rcx), %zmm4 + vmovups -0x180(%rcx), %zmm5 + vmovups -0x1C0(%rcx), %zmm6 + vmovups -0x200(%rcx), %zmm7 + sub $512, %rcx + prefetcht1 -0x40(%rcx) + prefetcht1 -0x80(%rcx) + prefetcht1 -0xC0(%rcx) + prefetcht1 -0x100(%rcx) + prefetcht1 -0x140(%rcx) + prefetcht1 -0x180(%rcx) + prefetcht1 -0x1C0(%rcx) + prefetcht1 -0x200(%rcx) + vmovups %zmm0, -0x40(%r9) + vmovups %zmm1, -0x80(%r9) + vmovups %zmm2, -0xC0(%r9) + vmovups %zmm3, -0x100(%r9) + vmovups %zmm4, -0x140(%r9) + vmovups %zmm5, -0x180(%r9) + vmovups %zmm6, -0x1C0(%r9) + vmovups %zmm7, -0x200(%r9) + sub $512, %r9 + cmp %rdi, %r9 + ja L(gobble_512bytes_loop_bkw) + vmovups %zmm8, -0x40(%rdi) + vmovups %zmm9, -0x80(%rdi) + vmovups %zmm10, -0xC0(%rdi) + vmovups %zmm11, -0x100(%rdi) + vmovups %zmm12, -0x140(%rdi) + vmovups %zmm13, -0x180(%rdi) + vmovups %zmm14, -0x1C0(%rdi) + vmovups %zmm15, -0x200(%rdi) + ret + +L(preloop_large): + cmp %rsi, %rdi + ja L(preloop_large_bkw) + vmovups (%rsi), %zmm4 + vmovups 0x40(%rsi), %zmm5 + + mov %rdi, %r11 +/* Align destination for access with non-temporal stores in the loop. */ + mov %rdi, %r8 + and $-0x80, %rdi + add $0x80, %rdi + sub %rdi, %r8 + sub %r8, %rsi + add %r8, %rdx +L(gobble_256bytes_nt_loop): + prefetcht1 0x200(%rsi) + prefetcht1 0x240(%rsi) + prefetcht1 0x280(%rsi) + prefetcht1 0x2C0(%rsi) + prefetcht1 0x300(%rsi) + prefetcht1 0x340(%rsi) + prefetcht1 0x380(%rsi) + prefetcht1 0x3C0(%rsi) + vmovdqu64 (%rsi), %zmm0 + vmovdqu64 0x40(%rsi), %zmm1 + vmovdqu64 0x80(%rsi), %zmm2 + vmovdqu64 0xC0(%rsi), %zmm3 + vmovntdq %zmm0, (%rdi) + vmovntdq %zmm1, 0x40(%rdi) + vmovntdq %zmm2, 0x80(%rdi) + vmovntdq %zmm3, 0xC0(%rdi) + sub $256, %rdx + add $256, %rsi + add $256, %rdi + cmp $256, %rdx + ja L(gobble_256bytes_nt_loop) + sfence + vmovups %zmm4, (%r11) + vmovups %zmm5, 0x40(%r11) + jmp L(check) + +L(preloop_large_bkw): + vmovups -0x80(%rcx), %zmm4 + vmovups -0x40(%rcx), %zmm5 + +/* Align end of destination for access with non-temporal stores. */ + mov %r9, %r8 + and $-0x80, %r9 + sub %r9, %r8 + sub %r8, %rcx + sub %r8, %rdx + add %r9, %r8 +L(gobble_256bytes_nt_loop_bkw): + prefetcht1 -0x400(%rcx) + prefetcht1 -0x3C0(%rcx) + prefetcht1 -0x380(%rcx) + prefetcht1 -0x340(%rcx) + prefetcht1 -0x300(%rcx) + prefetcht1 -0x2C0(%rcx) + prefetcht1 -0x280(%rcx) + prefetcht1 -0x240(%rcx) + vmovdqu64 -0x100(%rcx), %zmm0 + vmovdqu64 -0xC0(%rcx), %zmm1 + vmovdqu64 -0x80(%rcx), %zmm2 + vmovdqu64 -0x40(%rcx), %zmm3 + vmovntdq %zmm0, -0x100(%r9) + vmovntdq %zmm1, -0xC0(%r9) + vmovntdq %zmm2, -0x80(%r9) + vmovntdq %zmm3, -0x40(%r9) + sub $256, %rdx + sub $256, %rcx + sub $256, %r9 + cmp $256, %rdx + ja L(gobble_256bytes_nt_loop_bkw) + sfence + vmovups %zmm4, -0x80(%r8) + vmovups %zmm5, -0x40(%r8) + jmp L(check) +END (__memmove_avx512_no_vzeroupper) + +strong_alias (__memmove_avx512_no_vzeroupper, __memcpy_avx512_no_vzeroupper) +strong_alias (__memmove_chk_avx512_no_vzeroupper, __memcpy_chk_avx512_no_vzeroupper) +#endif diff --git a/utils/memcpy-bench/glibc/memmove-avx512-unaligned-erms.S b/utils/memcpy-bench/glibc/memmove-avx512-unaligned-erms.S new file mode 100644 index 00000000000..db70fdf1b4e --- /dev/null +++ b/utils/memcpy-bench/glibc/memmove-avx512-unaligned-erms.S @@ -0,0 +1,12 @@ +#if 1 +# define VEC_SIZE 64 +# define VEC(i) zmm##i +# define VMOVNT vmovntdq +# define VMOVU vmovdqu64 +# define VMOVA vmovdqa64 + +# define SECTION(p) p##.avx512 +# define MEMMOVE_SYMBOL(p,s) p##_avx512_##s + +# include "memmove-vec-unaligned-erms.S" +#endif diff --git a/utils/memcpy-bench/glibc/memmove-sse2-unaligned-erms.S b/utils/memcpy-bench/glibc/memmove-sse2-unaligned-erms.S new file mode 100644 index 00000000000..17b4f861621 --- /dev/null +++ b/utils/memcpy-bench/glibc/memmove-sse2-unaligned-erms.S @@ -0,0 +1,33 @@ +/* memmove with SSE2. + Copyright (C) 2017-2020 Free Software Foundation, Inc. + This file is part of the GNU C Library. + + The GNU C Library is free software; you can redistribute it and/or + modify it under the terms of the GNU Lesser General Public + License as published by the Free Software Foundation; either + version 2.1 of the License, or (at your option) any later version. + + The GNU C Library is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + Lesser General Public License for more details. + + You should have received a copy of the GNU Lesser General Public + License along with the GNU C Library; if not, see + . */ + +#if 1 +# define MEMMOVE_SYMBOL(p,s) p##_sse2_##s +#else +weak_alias (__mempcpy, mempcpy) +#endif + +#include "memmove.S" + +#if defined SHARED +# include +# if SHLIB_COMPAT (libc, GLIBC_2_2_5, GLIBC_2_14) +/* Use __memmove_sse2_unaligned to support overlapping addresses. */ +compat_symbol (libc, __memmove_sse2_unaligned, memcpy, GLIBC_2_2_5); +# endif +#endif diff --git a/utils/memcpy-bench/glibc/memmove-vec-unaligned-erms.S b/utils/memcpy-bench/glibc/memmove-vec-unaligned-erms.S new file mode 100644 index 00000000000..21be351b4e7 --- /dev/null +++ b/utils/memcpy-bench/glibc/memmove-vec-unaligned-erms.S @@ -0,0 +1,559 @@ +/* memmove/memcpy/mempcpy with unaligned load/store and rep movsb + Copyright (C) 2016-2020 Free Software Foundation, Inc. + This file is part of the GNU C Library. + + The GNU C Library is free software; you can redistribute it and/or + modify it under the terms of the GNU Lesser General Public + License as published by the Free Software Foundation; either + version 2.1 of the License, or (at your option) any later version. + + The GNU C Library is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + Lesser General Public License for more details. + + You should have received a copy of the GNU Lesser General Public + License along with the GNU C Library; if not, see + . */ + +/* memmove/memcpy/mempcpy is implemented as: + 1. Use overlapping load and store to avoid branch. + 2. Load all sources into registers and store them together to avoid + possible address overlap between source and destination. + 3. If size is 8 * VEC_SIZE or less, load all sources into registers + and store them together. + 4. If address of destination > address of source, backward copy + 4 * VEC_SIZE at a time with unaligned load and aligned store. + Load the first 4 * VEC and last VEC before the loop and store + them after the loop to support overlapping addresses. + 5. Otherwise, forward copy 4 * VEC_SIZE at a time with unaligned + load and aligned store. Load the last 4 * VEC and first VEC + before the loop and store them after the loop to support + overlapping addresses. + 6. If size >= __x86_shared_non_temporal_threshold and there is no + overlap between destination and source, use non-temporal store + instead of aligned store. */ + +#include "sysdep.h" + +#ifndef MEMCPY_SYMBOL +# define MEMCPY_SYMBOL(p,s) MEMMOVE_SYMBOL(p, s) +#endif + +#ifndef MEMPCPY_SYMBOL +# define MEMPCPY_SYMBOL(p,s) MEMMOVE_SYMBOL(p, s) +#endif + +#ifndef MEMMOVE_CHK_SYMBOL +# define MEMMOVE_CHK_SYMBOL(p,s) MEMMOVE_SYMBOL(p, s) +#endif + +#ifndef VZEROUPPER +# if VEC_SIZE > 16 +# define VZEROUPPER vzeroupper +# else +# define VZEROUPPER +# endif +#endif + +#ifndef PREFETCH +# define PREFETCH(addr) prefetcht0 addr +#endif + +/* Assume 64-byte prefetch size. */ +#ifndef PREFETCH_SIZE +# define PREFETCH_SIZE 64 +#endif + +#define PREFETCHED_LOAD_SIZE (VEC_SIZE * 4) + +#if PREFETCH_SIZE == 64 +# if PREFETCHED_LOAD_SIZE == PREFETCH_SIZE +# define PREFETCH_ONE_SET(dir, base, offset) \ + PREFETCH ((offset)base) +# elif PREFETCHED_LOAD_SIZE == 2 * PREFETCH_SIZE +# define PREFETCH_ONE_SET(dir, base, offset) \ + PREFETCH ((offset)base); \ + PREFETCH ((offset + dir * PREFETCH_SIZE)base) +# elif PREFETCHED_LOAD_SIZE == 4 * PREFETCH_SIZE +# define PREFETCH_ONE_SET(dir, base, offset) \ + PREFETCH ((offset)base); \ + PREFETCH ((offset + dir * PREFETCH_SIZE)base); \ + PREFETCH ((offset + dir * PREFETCH_SIZE * 2)base); \ + PREFETCH ((offset + dir * PREFETCH_SIZE * 3)base) +# else +# error Unsupported PREFETCHED_LOAD_SIZE! +# endif +#else +# error Unsupported PREFETCH_SIZE! +#endif + +#ifndef SECTION +# error SECTION is not defined! +#endif + + .section SECTION(.text),"ax",@progbits +#if defined SHARED +ENTRY (MEMMOVE_CHK_SYMBOL (__mempcpy_chk, unaligned)) + cmp %RDX_LP, %RCX_LP + jb HIDDEN_JUMPTARGET (__chk_fail) +END (MEMMOVE_CHK_SYMBOL (__mempcpy_chk, unaligned)) +#endif + +ENTRY (MEMPCPY_SYMBOL (__mempcpy, unaligned)) + mov %RDI_LP, %RAX_LP + add %RDX_LP, %RAX_LP + jmp L(start) +END (MEMPCPY_SYMBOL (__mempcpy, unaligned)) + +#if defined SHARED +ENTRY (MEMMOVE_CHK_SYMBOL (__memmove_chk, unaligned)) + cmp %RDX_LP, %RCX_LP + jb HIDDEN_JUMPTARGET (__chk_fail) +END (MEMMOVE_CHK_SYMBOL (__memmove_chk, unaligned)) +#endif + +ENTRY (MEMMOVE_SYMBOL (__memmove, unaligned)) + movq %rdi, %rax +L(start): +# ifdef __ILP32__ + /* Clear the upper 32 bits. */ + movl %edx, %edx +# endif + cmp $VEC_SIZE, %RDX_LP + jb L(less_vec) + cmp $(VEC_SIZE * 2), %RDX_LP + ja L(more_2x_vec) +#if !defined USE_MULTIARCH +L(last_2x_vec): +#endif + /* From VEC and to 2 * VEC. No branch when size == VEC_SIZE. */ + VMOVU (%rsi), %VEC(0) + VMOVU -VEC_SIZE(%rsi,%rdx), %VEC(1) + VMOVU %VEC(0), (%rdi) + VMOVU %VEC(1), -VEC_SIZE(%rdi,%rdx) + VZEROUPPER +#if !defined USE_MULTIARCH +L(nop): +#endif + ret +#if defined USE_MULTIARCH +END (MEMMOVE_SYMBOL (__memmove, unaligned)) + +# if VEC_SIZE == 16 +ENTRY (__mempcpy_chk_erms) + cmp %RDX_LP, %RCX_LP + jb HIDDEN_JUMPTARGET (__chk_fail) +END (__mempcpy_chk_erms) + +/* Only used to measure performance of REP MOVSB. */ +ENTRY (__mempcpy_erms) + mov %RDI_LP, %RAX_LP + /* Skip zero length. */ + test %RDX_LP, %RDX_LP + jz 2f + add %RDX_LP, %RAX_LP + jmp L(start_movsb) +END (__mempcpy_erms) + +ENTRY (__memmove_chk_erms) + cmp %RDX_LP, %RCX_LP + jb HIDDEN_JUMPTARGET (__chk_fail) +END (__memmove_chk_erms) + +ENTRY (__memmove_erms) + movq %rdi, %rax + /* Skip zero length. */ + test %RDX_LP, %RDX_LP + jz 2f +L(start_movsb): + mov %RDX_LP, %RCX_LP + cmp %RSI_LP, %RDI_LP + jb 1f + /* Source == destination is less common. */ + je 2f + lea (%rsi,%rcx), %RDX_LP + cmp %RDX_LP, %RDI_LP + jb L(movsb_backward) +1: + rep movsb +2: + ret +L(movsb_backward): + leaq -1(%rdi,%rcx), %rdi + leaq -1(%rsi,%rcx), %rsi + std + rep movsb + cld + ret +END (__memmove_erms) +strong_alias (__memmove_erms, __memcpy_erms) +strong_alias (__memmove_chk_erms, __memcpy_chk_erms) +# endif + +# ifdef SHARED +ENTRY (MEMMOVE_CHK_SYMBOL (__mempcpy_chk, unaligned_erms)) + cmp %RDX_LP, %RCX_LP + jb HIDDEN_JUMPTARGET (__chk_fail) +END (MEMMOVE_CHK_SYMBOL (__mempcpy_chk, unaligned_erms)) +# endif + +ENTRY (MEMMOVE_SYMBOL (__mempcpy, unaligned_erms)) + mov %RDI_LP, %RAX_LP + add %RDX_LP, %RAX_LP + jmp L(start_erms) +END (MEMMOVE_SYMBOL (__mempcpy, unaligned_erms)) + +# ifdef SHARED +ENTRY (MEMMOVE_CHK_SYMBOL (__memmove_chk, unaligned_erms)) + cmp %RDX_LP, %RCX_LP + jb HIDDEN_JUMPTARGET (__chk_fail) +END (MEMMOVE_CHK_SYMBOL (__memmove_chk, unaligned_erms)) +# endif + +ENTRY (MEMMOVE_SYMBOL (__memmove, unaligned_erms)) + movq %rdi, %rax +L(start_erms): +# ifdef __ILP32__ + /* Clear the upper 32 bits. */ + movl %edx, %edx +# endif + cmp $VEC_SIZE, %RDX_LP + jb L(less_vec) + cmp $(VEC_SIZE * 2), %RDX_LP + ja L(movsb_more_2x_vec) +L(last_2x_vec): + /* From VEC and to 2 * VEC. No branch when size == VEC_SIZE. */ + VMOVU (%rsi), %VEC(0) + VMOVU -VEC_SIZE(%rsi,%rdx), %VEC(1) + VMOVU %VEC(0), (%rdi) + VMOVU %VEC(1), -VEC_SIZE(%rdi,%rdx) +L(return): + VZEROUPPER + ret + +L(movsb): + cmp $SHARED_NON_TEMPORAL_THRESHOLD, %RDX_LP + jae L(more_8x_vec) + cmpq %rsi, %rdi + jb 1f + /* Source == destination is less common. */ + je L(nop) + leaq (%rsi,%rdx), %r9 + cmpq %r9, %rdi + /* Avoid slow backward REP MOVSB. */ + jb L(more_8x_vec_backward) +1: + mov %RDX_LP, %RCX_LP + rep movsb +L(nop): + ret +#endif + +L(less_vec): + /* Less than 1 VEC. */ +#if VEC_SIZE != 16 && VEC_SIZE != 32 && VEC_SIZE != 64 +# error Unsupported VEC_SIZE! +#endif +#if VEC_SIZE > 32 + cmpb $32, %dl + jae L(between_32_63) +#endif +#if VEC_SIZE > 16 + cmpb $16, %dl + jae L(between_16_31) +#endif + cmpb $8, %dl + jae L(between_8_15) + cmpb $4, %dl + jae L(between_4_7) + cmpb $1, %dl + ja L(between_2_3) + jb 1f + movzbl (%rsi), %ecx + movb %cl, (%rdi) +1: + ret +#if VEC_SIZE > 32 +L(between_32_63): + /* From 32 to 63. No branch when size == 32. */ + vmovdqu (%rsi), %ymm0 + vmovdqu -32(%rsi,%rdx), %ymm1 + vmovdqu %ymm0, (%rdi) + vmovdqu %ymm1, -32(%rdi,%rdx) + VZEROUPPER + ret +#endif +#if VEC_SIZE > 16 + /* From 16 to 31. No branch when size == 16. */ +L(between_16_31): + vmovdqu (%rsi), %xmm0 + vmovdqu -16(%rsi,%rdx), %xmm1 + vmovdqu %xmm0, (%rdi) + vmovdqu %xmm1, -16(%rdi,%rdx) + ret +#endif +L(between_8_15): + /* From 8 to 15. No branch when size == 8. */ + movq -8(%rsi,%rdx), %rcx + movq (%rsi), %rsi + movq %rcx, -8(%rdi,%rdx) + movq %rsi, (%rdi) + ret +L(between_4_7): + /* From 4 to 7. No branch when size == 4. */ + movl -4(%rsi,%rdx), %ecx + movl (%rsi), %esi + movl %ecx, -4(%rdi,%rdx) + movl %esi, (%rdi) + ret +L(between_2_3): + /* From 2 to 3. No branch when size == 2. */ + movzwl -2(%rsi,%rdx), %ecx + movzwl (%rsi), %esi + movw %cx, -2(%rdi,%rdx) + movw %si, (%rdi) + ret + +#if defined USE_MULTIARCH +L(movsb_more_2x_vec): + cmp $REP_MOSB_THRESHOLD, %RDX_LP + ja L(movsb) +#endif +L(more_2x_vec): + /* More than 2 * VEC and there may be overlap between destination + and source. */ + cmpq $(VEC_SIZE * 8), %rdx + ja L(more_8x_vec) + cmpq $(VEC_SIZE * 4), %rdx + jb L(last_4x_vec) + /* Copy from 4 * VEC to 8 * VEC, inclusively. */ + VMOVU (%rsi), %VEC(0) + VMOVU VEC_SIZE(%rsi), %VEC(1) + VMOVU (VEC_SIZE * 2)(%rsi), %VEC(2) + VMOVU (VEC_SIZE * 3)(%rsi), %VEC(3) + VMOVU -VEC_SIZE(%rsi,%rdx), %VEC(4) + VMOVU -(VEC_SIZE * 2)(%rsi,%rdx), %VEC(5) + VMOVU -(VEC_SIZE * 3)(%rsi,%rdx), %VEC(6) + VMOVU -(VEC_SIZE * 4)(%rsi,%rdx), %VEC(7) + VMOVU %VEC(0), (%rdi) + VMOVU %VEC(1), VEC_SIZE(%rdi) + VMOVU %VEC(2), (VEC_SIZE * 2)(%rdi) + VMOVU %VEC(3), (VEC_SIZE * 3)(%rdi) + VMOVU %VEC(4), -VEC_SIZE(%rdi,%rdx) + VMOVU %VEC(5), -(VEC_SIZE * 2)(%rdi,%rdx) + VMOVU %VEC(6), -(VEC_SIZE * 3)(%rdi,%rdx) + VMOVU %VEC(7), -(VEC_SIZE * 4)(%rdi,%rdx) + VZEROUPPER + ret +L(last_4x_vec): + /* Copy from 2 * VEC to 4 * VEC. */ + VMOVU (%rsi), %VEC(0) + VMOVU VEC_SIZE(%rsi), %VEC(1) + VMOVU -VEC_SIZE(%rsi,%rdx), %VEC(2) + VMOVU -(VEC_SIZE * 2)(%rsi,%rdx), %VEC(3) + VMOVU %VEC(0), (%rdi) + VMOVU %VEC(1), VEC_SIZE(%rdi) + VMOVU %VEC(2), -VEC_SIZE(%rdi,%rdx) + VMOVU %VEC(3), -(VEC_SIZE * 2)(%rdi,%rdx) + VZEROUPPER + ret + +L(more_8x_vec): + cmpq %rsi, %rdi + ja L(more_8x_vec_backward) + /* Source == destination is less common. */ + je L(nop) + /* Load the first VEC and last 4 * VEC to support overlapping + addresses. */ + VMOVU (%rsi), %VEC(4) + VMOVU -VEC_SIZE(%rsi, %rdx), %VEC(5) + VMOVU -(VEC_SIZE * 2)(%rsi, %rdx), %VEC(6) + VMOVU -(VEC_SIZE * 3)(%rsi, %rdx), %VEC(7) + VMOVU -(VEC_SIZE * 4)(%rsi, %rdx), %VEC(8) + /* Save start and stop of the destination buffer. */ + movq %rdi, %r11 + leaq -VEC_SIZE(%rdi, %rdx), %rcx + /* Align destination for aligned stores in the loop. Compute + how much destination is misaligned. */ + movq %rdi, %r8 + andq $(VEC_SIZE - 1), %r8 + /* Get the negative of offset for alignment. */ + subq $VEC_SIZE, %r8 + /* Adjust source. */ + subq %r8, %rsi + /* Adjust destination which should be aligned now. */ + subq %r8, %rdi + /* Adjust length. */ + addq %r8, %rdx +#if (defined USE_MULTIARCH || VEC_SIZE == 16) + /* Check non-temporal store threshold. */ + cmp $SHARED_NON_TEMPORAL_THRESHOLD, %RDX_LP + ja L(large_forward) +#endif +L(loop_4x_vec_forward): + /* Copy 4 * VEC a time forward. */ + VMOVU (%rsi), %VEC(0) + VMOVU VEC_SIZE(%rsi), %VEC(1) + VMOVU (VEC_SIZE * 2)(%rsi), %VEC(2) + VMOVU (VEC_SIZE * 3)(%rsi), %VEC(3) + addq $(VEC_SIZE * 4), %rsi + subq $(VEC_SIZE * 4), %rdx + VMOVA %VEC(0), (%rdi) + VMOVA %VEC(1), VEC_SIZE(%rdi) + VMOVA %VEC(2), (VEC_SIZE * 2)(%rdi) + VMOVA %VEC(3), (VEC_SIZE * 3)(%rdi) + addq $(VEC_SIZE * 4), %rdi + cmpq $(VEC_SIZE * 4), %rdx + ja L(loop_4x_vec_forward) + /* Store the last 4 * VEC. */ + VMOVU %VEC(5), (%rcx) + VMOVU %VEC(6), -VEC_SIZE(%rcx) + VMOVU %VEC(7), -(VEC_SIZE * 2)(%rcx) + VMOVU %VEC(8), -(VEC_SIZE * 3)(%rcx) + /* Store the first VEC. */ + VMOVU %VEC(4), (%r11) + VZEROUPPER + ret + +L(more_8x_vec_backward): + /* Load the first 4 * VEC and last VEC to support overlapping + addresses. */ + VMOVU (%rsi), %VEC(4) + VMOVU VEC_SIZE(%rsi), %VEC(5) + VMOVU (VEC_SIZE * 2)(%rsi), %VEC(6) + VMOVU (VEC_SIZE * 3)(%rsi), %VEC(7) + VMOVU -VEC_SIZE(%rsi,%rdx), %VEC(8) + /* Save stop of the destination buffer. */ + leaq -VEC_SIZE(%rdi, %rdx), %r11 + /* Align destination end for aligned stores in the loop. Compute + how much destination end is misaligned. */ + leaq -VEC_SIZE(%rsi, %rdx), %rcx + movq %r11, %r9 + movq %r11, %r8 + andq $(VEC_SIZE - 1), %r8 + /* Adjust source. */ + subq %r8, %rcx + /* Adjust the end of destination which should be aligned now. */ + subq %r8, %r9 + /* Adjust length. */ + subq %r8, %rdx +#if (defined USE_MULTIARCH || VEC_SIZE == 16) + /* Check non-temporal store threshold. */ + cmp $SHARED_NON_TEMPORAL_THRESHOLD, %RDX_LP + ja L(large_backward) +#endif +L(loop_4x_vec_backward): + /* Copy 4 * VEC a time backward. */ + VMOVU (%rcx), %VEC(0) + VMOVU -VEC_SIZE(%rcx), %VEC(1) + VMOVU -(VEC_SIZE * 2)(%rcx), %VEC(2) + VMOVU -(VEC_SIZE * 3)(%rcx), %VEC(3) + subq $(VEC_SIZE * 4), %rcx + subq $(VEC_SIZE * 4), %rdx + VMOVA %VEC(0), (%r9) + VMOVA %VEC(1), -VEC_SIZE(%r9) + VMOVA %VEC(2), -(VEC_SIZE * 2)(%r9) + VMOVA %VEC(3), -(VEC_SIZE * 3)(%r9) + subq $(VEC_SIZE * 4), %r9 + cmpq $(VEC_SIZE * 4), %rdx + ja L(loop_4x_vec_backward) + /* Store the first 4 * VEC. */ + VMOVU %VEC(4), (%rdi) + VMOVU %VEC(5), VEC_SIZE(%rdi) + VMOVU %VEC(6), (VEC_SIZE * 2)(%rdi) + VMOVU %VEC(7), (VEC_SIZE * 3)(%rdi) + /* Store the last VEC. */ + VMOVU %VEC(8), (%r11) + VZEROUPPER + ret + +#if (defined USE_MULTIARCH || VEC_SIZE == 16) +L(large_forward): + /* Don't use non-temporal store if there is overlap between + destination and source since destination may be in cache + when source is loaded. */ + leaq (%rdi, %rdx), %r10 + cmpq %r10, %rsi + jb L(loop_4x_vec_forward) +L(loop_large_forward): + /* Copy 4 * VEC a time forward with non-temporal stores. */ + PREFETCH_ONE_SET (1, (%rsi), PREFETCHED_LOAD_SIZE * 2) + PREFETCH_ONE_SET (1, (%rsi), PREFETCHED_LOAD_SIZE * 3) + VMOVU (%rsi), %VEC(0) + VMOVU VEC_SIZE(%rsi), %VEC(1) + VMOVU (VEC_SIZE * 2)(%rsi), %VEC(2) + VMOVU (VEC_SIZE * 3)(%rsi), %VEC(3) + addq $PREFETCHED_LOAD_SIZE, %rsi + subq $PREFETCHED_LOAD_SIZE, %rdx + VMOVNT %VEC(0), (%rdi) + VMOVNT %VEC(1), VEC_SIZE(%rdi) + VMOVNT %VEC(2), (VEC_SIZE * 2)(%rdi) + VMOVNT %VEC(3), (VEC_SIZE * 3)(%rdi) + addq $PREFETCHED_LOAD_SIZE, %rdi + cmpq $PREFETCHED_LOAD_SIZE, %rdx + ja L(loop_large_forward) + sfence + /* Store the last 4 * VEC. */ + VMOVU %VEC(5), (%rcx) + VMOVU %VEC(6), -VEC_SIZE(%rcx) + VMOVU %VEC(7), -(VEC_SIZE * 2)(%rcx) + VMOVU %VEC(8), -(VEC_SIZE * 3)(%rcx) + /* Store the first VEC. */ + VMOVU %VEC(4), (%r11) + VZEROUPPER + ret + +L(large_backward): + /* Don't use non-temporal store if there is overlap between + destination and source since destination may be in cache + when source is loaded. */ + leaq (%rcx, %rdx), %r10 + cmpq %r10, %r9 + jb L(loop_4x_vec_backward) +L(loop_large_backward): + /* Copy 4 * VEC a time backward with non-temporal stores. */ + PREFETCH_ONE_SET (-1, (%rcx), -PREFETCHED_LOAD_SIZE * 2) + PREFETCH_ONE_SET (-1, (%rcx), -PREFETCHED_LOAD_SIZE * 3) + VMOVU (%rcx), %VEC(0) + VMOVU -VEC_SIZE(%rcx), %VEC(1) + VMOVU -(VEC_SIZE * 2)(%rcx), %VEC(2) + VMOVU -(VEC_SIZE * 3)(%rcx), %VEC(3) + subq $PREFETCHED_LOAD_SIZE, %rcx + subq $PREFETCHED_LOAD_SIZE, %rdx + VMOVNT %VEC(0), (%r9) + VMOVNT %VEC(1), -VEC_SIZE(%r9) + VMOVNT %VEC(2), -(VEC_SIZE * 2)(%r9) + VMOVNT %VEC(3), -(VEC_SIZE * 3)(%r9) + subq $PREFETCHED_LOAD_SIZE, %r9 + cmpq $PREFETCHED_LOAD_SIZE, %rdx + ja L(loop_large_backward) + sfence + /* Store the first 4 * VEC. */ + VMOVU %VEC(4), (%rdi) + VMOVU %VEC(5), VEC_SIZE(%rdi) + VMOVU %VEC(6), (VEC_SIZE * 2)(%rdi) + VMOVU %VEC(7), (VEC_SIZE * 3)(%rdi) + /* Store the last VEC. */ + VMOVU %VEC(8), (%r11) + VZEROUPPER + ret +#endif +END (MEMMOVE_SYMBOL (__memmove, unaligned_erms)) + +#if 1 +# ifdef USE_MULTIARCH +strong_alias (MEMMOVE_SYMBOL (__memmove, unaligned_erms), + MEMMOVE_SYMBOL (__memcpy, unaligned_erms)) +# ifdef SHARED +strong_alias (MEMMOVE_SYMBOL (__memmove_chk, unaligned_erms), + MEMMOVE_SYMBOL (__memcpy_chk, unaligned_erms)) +# endif +# endif +# ifdef SHARED +strong_alias (MEMMOVE_CHK_SYMBOL (__memmove_chk, unaligned), + MEMMOVE_CHK_SYMBOL (__memcpy_chk, unaligned)) +# endif +#endif +strong_alias (MEMMOVE_SYMBOL (__memmove, unaligned), + MEMCPY_SYMBOL (__memcpy, unaligned)) diff --git a/utils/memcpy-bench/glibc/memmove.S b/utils/memcpy-bench/glibc/memmove.S new file mode 100644 index 00000000000..97e735facff --- /dev/null +++ b/utils/memcpy-bench/glibc/memmove.S @@ -0,0 +1,71 @@ +/* Optimized memmove for x86-64. + Copyright (C) 2016-2020 Free Software Foundation, Inc. + This file is part of the GNU C Library. + + The GNU C Library is free software; you can redistribute it and/or + modify it under the terms of the GNU Lesser General Public + License as published by the Free Software Foundation; either + version 2.1 of the License, or (at your option) any later version. + + The GNU C Library is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + Lesser General Public License for more details. + + You should have received a copy of the GNU Lesser General Public + License along with the GNU C Library; if not, see + . */ + +#include "sysdep.h" + +#define VEC_SIZE 16 +#define VEC(i) xmm##i +#define PREFETCHNT prefetchnta +#define VMOVNT movntdq +/* Use movups and movaps for smaller code sizes. */ +#define VMOVU movups +#define VMOVA movaps + +#define SECTION(p) p + +#ifdef USE_MULTIARCH +# if 0 +# define MEMCPY_SYMBOL(p,s) memcpy +# endif +#else +# if defined SHARED +# define MEMCPY_SYMBOL(p,s) __memcpy +# else +# define MEMCPY_SYMBOL(p,s) memcpy +# endif +#endif +#if !defined USE_MULTIARCH +# define MEMPCPY_SYMBOL(p,s) __mempcpy +#endif +#ifndef MEMMOVE_SYMBOL +# define MEMMOVE_CHK_SYMBOL(p,s) p +# define MEMMOVE_SYMBOL(p,s) memmove +#endif + +#include "memmove-vec-unaligned-erms.S" + +#ifndef USE_MULTIARCH +libc_hidden_builtin_def (memmove) +# if defined SHARED && IS_IN (libc) +strong_alias (memmove, __memcpy) +libc_hidden_ver (memmove, memcpy) +# endif +libc_hidden_def (__mempcpy) +weak_alias (__mempcpy, mempcpy) +libc_hidden_builtin_def (mempcpy) + +# if defined SHARED && IS_IN (libc) +# undef memcpy +# include +versioned_symbol (libc, __memcpy, memcpy, GLIBC_2_14); + +# if SHLIB_COMPAT (libc, GLIBC_2_2_5, GLIBC_2_14) +compat_symbol (libc, memmove, memcpy, GLIBC_2_2_5); +# endif +# endif +#endif diff --git a/utils/memcpy-bench/glibc/sysdep.h b/utils/memcpy-bench/glibc/sysdep.h new file mode 100644 index 00000000000..099134b2a2f --- /dev/null +++ b/utils/memcpy-bench/glibc/sysdep.h @@ -0,0 +1,129 @@ +/* Assembler macros for x86-64. + Copyright (C) 2001-2020 Free Software Foundation, Inc. + This file is part of the GNU C Library. + + The GNU C Library is free software; you can redistribute it and/or + modify it under the terms of the GNU Lesser General Public + License as published by the Free Software Foundation; either + version 2.1 of the License, or (at your option) any later version. + + The GNU C Library is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + Lesser General Public License for more details. + + You should have received a copy of the GNU Lesser General Public + License along with the GNU C Library; if not, see + . */ + +#ifndef _X86_64_SYSDEP_H +#define _X86_64_SYSDEP_H 1 + +#include "sysdep_x86.h" + +#ifdef __ASSEMBLER__ + +/* Syntactic details of assembler. */ + +/* This macro is for setting proper CFI with DW_CFA_expression describing + the register as saved relative to %rsp instead of relative to the CFA. + Expression is DW_OP_drop, DW_OP_breg7 (%rsp is register 7), sleb128 offset + from %rsp. */ +#define cfi_offset_rel_rsp(regn, off) .cfi_escape 0x10, regn, 0x4, 0x13, \ + 0x77, off & 0x7F | 0x80, off >> 7 + +/* If compiled for profiling, call `mcount' at the start of each function. */ +#ifdef PROF +/* The mcount code relies on a normal frame pointer being on the stack + to locate our caller, so push one just for its benefit. */ +#define CALL_MCOUNT \ + pushq %rbp; \ + cfi_adjust_cfa_offset(8); \ + movq %rsp, %rbp; \ + cfi_def_cfa_register(%rbp); \ + call JUMPTARGET(mcount); \ + popq %rbp; \ + cfi_def_cfa(rsp,8); +#else +#define CALL_MCOUNT /* Do nothing. */ +#endif + +#define PSEUDO(name, syscall_name, args) \ +lose: \ + jmp JUMPTARGET(syscall_error) \ + .globl syscall_error; \ + ENTRY (name) \ + DO_CALL (syscall_name, args); \ + jb lose + +#undef JUMPTARGET +#ifdef SHARED +# ifdef BIND_NOW +# define JUMPTARGET(name) *name##@GOTPCREL(%rip) +# else +# define JUMPTARGET(name) name##@PLT +# endif +#else +/* For static archives, branch to target directly. */ +# define JUMPTARGET(name) name +#endif + +/* Long and pointer size in bytes. */ +#define LP_SIZE 8 + +/* Instruction to operate on long and pointer. */ +#define LP_OP(insn) insn##q + +/* Assembler address directive. */ +#define ASM_ADDR .quad + +/* Registers to hold long and pointer. */ +#define RAX_LP rax +#define RBP_LP rbp +#define RBX_LP rbx +#define RCX_LP rcx +#define RDI_LP rdi +#define RDX_LP rdx +#define RSI_LP rsi +#define RSP_LP rsp +#define R8_LP r8 +#define R9_LP r9 +#define R10_LP r10 +#define R11_LP r11 +#define R12_LP r12 +#define R13_LP r13 +#define R14_LP r14 +#define R15_LP r15 + +#else /* __ASSEMBLER__ */ + +/* Long and pointer size in bytes. */ +#define LP_SIZE "8" + +/* Instruction to operate on long and pointer. */ +#define LP_OP(insn) #insn "q" + +/* Assembler address directive. */ +#define ASM_ADDR ".quad" + +/* Registers to hold long and pointer. */ +#define RAX_LP "rax" +#define RBP_LP "rbp" +#define RBX_LP "rbx" +#define RCX_LP "rcx" +#define RDI_LP "rdi" +#define RDX_LP "rdx" +#define RSI_LP "rsi" +#define RSP_LP "rsp" +#define R8_LP "r8" +#define R9_LP "r9" +#define R10_LP "r10" +#define R11_LP "r11" +#define R12_LP "r12" +#define R13_LP "r13" +#define R14_LP "r14" +#define R15_LP "r15" + +#endif /* __ASSEMBLER__ */ + +#endif /* _X86_64_SYSDEP_H */ diff --git a/utils/memcpy-bench/glibc/sysdep_generic.h b/utils/memcpy-bench/glibc/sysdep_generic.h new file mode 100644 index 00000000000..91f78e1b04d --- /dev/null +++ b/utils/memcpy-bench/glibc/sysdep_generic.h @@ -0,0 +1,113 @@ +/* Generic asm macros used on many machines. + Copyright (C) 1991-2020 Free Software Foundation, Inc. + This file is part of the GNU C Library. + + The GNU C Library is free software; you can redistribute it and/or + modify it under the terms of the GNU Lesser General Public + License as published by the Free Software Foundation; either + version 2.1 of the License, or (at your option) any later version. + + The GNU C Library is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + Lesser General Public License for more details. + + You should have received a copy of the GNU Lesser General Public + License along with the GNU C Library; if not, see + . */ + +#define C_SYMBOL_NAME(name) name +#define HIDDEN_JUMPTARGET(name) 0x0 +#define SHARED_CACHE_SIZE_HALF (1024*1024) +#define DATA_CACHE_SIZE_HALF (1024*32/2) +#define DATA_CACHE_SIZE (1024*32) +#define SHARED_NON_TEMPORAL_THRESHOLD (1024*1024*4) +#define REP_MOSB_THRESHOLD 1024 + +#define USE_MULTIARCH + +#define ASM_LINE_SEP ; + +#define strong_alias(original, alias) \ + .globl C_SYMBOL_NAME (alias) ASM_LINE_SEP \ + C_SYMBOL_NAME (alias) = C_SYMBOL_NAME (original) + +#ifndef C_LABEL + +/* Define a macro we can use to construct the asm name for a C symbol. */ +# define C_LABEL(name) name##: + +#endif + +#ifdef __ASSEMBLER__ +/* Mark the end of function named SYM. This is used on some platforms + to generate correct debugging information. */ +# ifndef END +# define END(sym) +# endif + +# ifndef JUMPTARGET +# define JUMPTARGET(sym) sym +# endif +#endif + +/* Makros to generate eh_frame unwind information. */ +#ifdef __ASSEMBLER__ +# define cfi_startproc .cfi_startproc +# define cfi_endproc .cfi_endproc +# define cfi_def_cfa(reg, off) .cfi_def_cfa reg, off +# define cfi_def_cfa_register(reg) .cfi_def_cfa_register reg +# define cfi_def_cfa_offset(off) .cfi_def_cfa_offset off +# define cfi_adjust_cfa_offset(off) .cfi_adjust_cfa_offset off +# define cfi_offset(reg, off) .cfi_offset reg, off +# define cfi_rel_offset(reg, off) .cfi_rel_offset reg, off +# define cfi_register(r1, r2) .cfi_register r1, r2 +# define cfi_return_column(reg) .cfi_return_column reg +# define cfi_restore(reg) .cfi_restore reg +# define cfi_same_value(reg) .cfi_same_value reg +# define cfi_undefined(reg) .cfi_undefined reg +# define cfi_remember_state .cfi_remember_state +# define cfi_restore_state .cfi_restore_state +# define cfi_window_save .cfi_window_save +# define cfi_personality(enc, exp) .cfi_personality enc, exp +# define cfi_lsda(enc, exp) .cfi_lsda enc, exp + +#else /* ! ASSEMBLER */ + +# define CFI_STRINGIFY(Name) CFI_STRINGIFY2 (Name) +# define CFI_STRINGIFY2(Name) #Name +# define CFI_STARTPROC ".cfi_startproc" +# define CFI_ENDPROC ".cfi_endproc" +# define CFI_DEF_CFA(reg, off) \ + ".cfi_def_cfa " CFI_STRINGIFY(reg) "," CFI_STRINGIFY(off) +# define CFI_DEF_CFA_REGISTER(reg) \ + ".cfi_def_cfa_register " CFI_STRINGIFY(reg) +# define CFI_DEF_CFA_OFFSET(off) \ + ".cfi_def_cfa_offset " CFI_STRINGIFY(off) +# define CFI_ADJUST_CFA_OFFSET(off) \ + ".cfi_adjust_cfa_offset " CFI_STRINGIFY(off) +# define CFI_OFFSET(reg, off) \ + ".cfi_offset " CFI_STRINGIFY(reg) "," CFI_STRINGIFY(off) +# define CFI_REL_OFFSET(reg, off) \ + ".cfi_rel_offset " CFI_STRINGIFY(reg) "," CFI_STRINGIFY(off) +# define CFI_REGISTER(r1, r2) \ + ".cfi_register " CFI_STRINGIFY(r1) "," CFI_STRINGIFY(r2) +# define CFI_RETURN_COLUMN(reg) \ + ".cfi_return_column " CFI_STRINGIFY(reg) +# define CFI_RESTORE(reg) \ + ".cfi_restore " CFI_STRINGIFY(reg) +# define CFI_UNDEFINED(reg) \ + ".cfi_undefined " CFI_STRINGIFY(reg) +# define CFI_REMEMBER_STATE \ + ".cfi_remember_state" +# define CFI_RESTORE_STATE \ + ".cfi_restore_state" +# define CFI_WINDOW_SAVE \ + ".cfi_window_save" +# define CFI_PERSONALITY(enc, exp) \ + ".cfi_personality " CFI_STRINGIFY(enc) "," CFI_STRINGIFY(exp) +# define CFI_LSDA(enc, exp) \ + ".cfi_lsda " CFI_STRINGIFY(enc) "," CFI_STRINGIFY(exp) +#endif + +#include "dwarf2.h" diff --git a/utils/memcpy-bench/glibc/sysdep_x86.h b/utils/memcpy-bench/glibc/sysdep_x86.h new file mode 100644 index 00000000000..a3fecd01268 --- /dev/null +++ b/utils/memcpy-bench/glibc/sysdep_x86.h @@ -0,0 +1,113 @@ +/* Assembler macros for x86. + Copyright (C) 2017-2020 Free Software Foundation, Inc. + This file is part of the GNU C Library. + + The GNU C Library is free software; you can redistribute it and/or + modify it under the terms of the GNU Lesser General Public + License as published by the Free Software Foundation; either + version 2.1 of the License, or (at your option) any later version. + + The GNU C Library is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + Lesser General Public License for more details. + + You should have received a copy of the GNU Lesser General Public + License along with the GNU C Library; if not, see + . */ + +#ifndef _X86_SYSDEP_H +#define _X86_SYSDEP_H 1 + +#include "sysdep_generic.h" + +/* __CET__ is defined by GCC with Control-Flow Protection values: + +enum cf_protection_level +{ + CF_NONE = 0, + CF_BRANCH = 1 << 0, + CF_RETURN = 1 << 1, + CF_FULL = CF_BRANCH | CF_RETURN, + CF_SET = 1 << 2 +}; +*/ + +/* Set if CF_BRANCH (IBT) is enabled. */ +#define X86_FEATURE_1_IBT (1U << 0) +/* Set if CF_RETURN (SHSTK) is enabled. */ +#define X86_FEATURE_1_SHSTK (1U << 1) + +#ifdef __CET__ +# define CET_ENABLED 1 +# define IBT_ENABLED (__CET__ & X86_FEATURE_1_IBT) +# define SHSTK_ENABLED (__CET__ & X86_FEATURE_1_SHSTK) +#else +# define CET_ENABLED 0 +# define IBT_ENABLED 0 +# define SHSTK_ENABLED 0 +#endif + +/* Offset for fxsave/xsave area used by _dl_runtime_resolve. Also need + space to preserve RCX, RDX, RSI, RDI, R8, R9 and RAX. It must be + aligned to 16 bytes for fxsave and 64 bytes for xsave. */ +#define STATE_SAVE_OFFSET (8 * 7 + 8) + +/* Save SSE, AVX, AVX512, mask and bound registers. */ +#define STATE_SAVE_MASK \ + ((1 << 1) | (1 << 2) | (1 << 3) | (1 << 5) | (1 << 6) | (1 << 7)) + +#ifdef __ASSEMBLER__ + +/* Syntactic details of assembler. */ + +#ifdef _CET_ENDBR +# define _CET_NOTRACK notrack +#else +# define _CET_ENDBR +# define _CET_NOTRACK +#endif + +/* ELF uses byte-counts for .align, most others use log2 of count of bytes. */ +#define ALIGNARG(log2) 1< #include +#include #include #include #include @@ -14,15 +15,11 @@ #include -#pragma GCC diagnostic ignored "-Wold-style-cast" -#pragma GCC diagnostic ignored "-Wcast-align" -#pragma GCC diagnostic ignored "-Wcast-qual" -#include "FastMemcpy.h" -//#include "FastMemcpy_Avx.h" - #include #include +#include + template void NO_INLINE loop(uint8_t * dst, uint8_t * src, size_t size, F && chunk_size_distribution, MemcpyImpl && impl) @@ -47,7 +44,7 @@ size_t generatorUniform(RNG & rng) { return rng() % N; }; template -void test(uint8_t * dst, uint8_t * src, size_t size, size_t iterations, size_t num_threads, F && generator, MemcpyImpl && impl) +uint64_t test(uint8_t * dst, uint8_t * src, size_t size, size_t iterations, size_t num_threads, F && generator, MemcpyImpl && impl, const char * name) { Stopwatch watch; @@ -76,15 +73,22 @@ void test(uint8_t * dst, uint8_t * src, size_t size, size_t iterations, size_t n for (auto & thread : threads) thread.join(); - double elapsed_ns = watch.elapsed(); + uint64_t elapsed_ns = watch.elapsed(); /// Validation size_t sum = 0; + size_t reference = 0; for (size_t i = 0; i < size; ++i) + { sum += dst[i]; + reference += uint8_t(i); + } - std::cerr << std::fixed << std::setprecision(3) - << "Processed in " << (elapsed_ns / 1e9) << "sec, " << (size * iterations * 1.0 / elapsed_ns) << " GB/sec (sum = " << sum << ")\n"; + if (sum != reference) + throw std::logic_error("Incorrect result"); + + std::cout << name; + return elapsed_ns; } @@ -101,9 +105,30 @@ static void * memcpy_erms(void * dst, const void * src, size_t size) return dst; } +static void * memcpy_trivial(void * __restrict dst_, const void * __restrict src_, size_t size) +{ + char * __restrict dst = reinterpret_cast(dst_); + const char * __restrict src = reinterpret_cast(src_); + void * ret = dst; + + while (size > 0) + { + *dst = *src; + ++dst; + ++src; + --size; + } + + return ret; +} + extern "C" void * memcpy_jart(void * dst, const void * src, size_t size); extern "C" void MemCpy(void * dst, const void * src, size_t size); +void * memcpy_fast_sse(void * dst, const void * src, size_t size); +void * memcpy_fast_avx(void * dst, const void * src, size_t size); +void * memcpy_tiny(void * dst, const void * src, size_t size); + static void * memcpySSE2(void * __restrict destination, const void * __restrict source, size_t size) { @@ -329,7 +354,7 @@ void memcpy_my_medium_avx(uint8_t * __restrict & __restrict dst, const uint8_t * if (padding > 0) { __m256i head = _mm256_loadu_si256(reinterpret_cast(src)); - _mm256_storeu_si256((__m256i*)dst, head); + _mm256_storeu_si256(reinterpret_cast<__m256i*>(dst), head); dst += padding; src += padding; size -= padding; @@ -539,70 +564,125 @@ tail: return ret; } +extern "C" void * __memcpy_erms(void * __restrict destination, const void * __restrict source, size_t size); +extern "C" void * __memcpy_sse2_unaligned(void * __restrict destination, const void * __restrict source, size_t size); +extern "C" void * __memcpy_ssse3(void * __restrict destination, const void * __restrict source, size_t size); +extern "C" void * __memcpy_ssse3_back(void * __restrict destination, const void * __restrict source, size_t size); +extern "C" void * __memcpy_avx_unaligned(void * __restrict destination, const void * __restrict source, size_t size); +extern "C" void * __memcpy_avx_unaligned_erms(void * __restrict destination, const void * __restrict source, size_t size); +extern "C" void * __memcpy_avx512_unaligned(void * __restrict destination, const void * __restrict source, size_t size); +extern "C" void * __memcpy_avx512_unaligned_erms(void * __restrict destination, const void * __restrict source, size_t size); +extern "C" void * __memcpy_avx512_no_vzeroupper(void * __restrict destination, const void * __restrict source, size_t size); + + +#define VARIANT(N, NAME) \ + if (memcpy_variant == N) \ + return test(dst, src, size, iterations, num_threads, std::forward(generator), NAME, #NAME); template -void dispatchMemcpyVariants(size_t memcpy_variant, uint8_t * dst, uint8_t * src, size_t size, size_t iterations, size_t num_threads, F && generator) +uint64_t dispatchMemcpyVariants(size_t memcpy_variant, uint8_t * dst, uint8_t * src, size_t size, size_t iterations, size_t num_threads, F && generator) { - memcpy_type memcpy_libc = reinterpret_cast(dlsym(RTLD_NEXT, "memcpy")); + memcpy_type memcpy_libc_old = reinterpret_cast(dlsym(RTLD_NEXT, "memcpy")); - if (memcpy_variant == 1) - test(dst, src, size, iterations, num_threads, std::forward(generator), memcpy); - if (memcpy_variant == 2) - test(dst, src, size, iterations, num_threads, std::forward(generator), memcpy_libc); - if (memcpy_variant == 3) - test(dst, src, size, iterations, num_threads, std::forward(generator), memcpy_erms); - if (memcpy_variant == 4) - test(dst, src, size, iterations, num_threads, std::forward(generator), MemCpy); - if (memcpy_variant == 5) - test(dst, src, size, iterations, num_threads, std::forward(generator), memcpySSE2); - if (memcpy_variant == 6) - test(dst, src, size, iterations, num_threads, std::forward(generator), memcpySSE2Unrolled2); - if (memcpy_variant == 7) - test(dst, src, size, iterations, num_threads, std::forward(generator), memcpySSE2Unrolled4); - if (memcpy_variant == 8) - test(dst, src, size, iterations, num_threads, std::forward(generator), memcpySSE2Unrolled8); -// if (memcpy_variant == 9) -// test(dst, src, size, iterations, num_threads, std::forward(generator), memcpy_fast_avx); - if (memcpy_variant == 10) - test(dst, src, size, iterations, num_threads, std::forward(generator), memcpy_my); + VARIANT(1, memcpy) + VARIANT(2, memcpy_trivial) + VARIANT(3, memcpy_libc_old) + VARIANT(4, memcpy_erms) + VARIANT(5, MemCpy) + VARIANT(6, memcpySSE2) + VARIANT(7, memcpySSE2Unrolled2) + VARIANT(8, memcpySSE2Unrolled4) + VARIANT(9, memcpySSE2Unrolled8) + VARIANT(10, memcpy_fast_sse) + VARIANT(11, memcpy_fast_avx) + VARIANT(12, memcpy_my) + + VARIANT(21, __memcpy_erms) + VARIANT(22, __memcpy_sse2_unaligned) + VARIANT(23, __memcpy_ssse3) + VARIANT(24, __memcpy_ssse3_back) + VARIANT(25, __memcpy_avx_unaligned) + VARIANT(26, __memcpy_avx_unaligned_erms) + VARIANT(27, __memcpy_avx512_unaligned) + VARIANT(28, __memcpy_avx512_unaligned_erms) + VARIANT(29, __memcpy_avx512_no_vzeroupper) + + return 0; } -void dispatchVariants(size_t memcpy_variant, size_t generator_variant, uint8_t * dst, uint8_t * src, size_t size, size_t iterations, size_t num_threads) +uint64_t dispatchVariants( + size_t memcpy_variant, size_t generator_variant, uint8_t * dst, uint8_t * src, size_t size, size_t iterations, size_t num_threads) { if (generator_variant == 1) - dispatchMemcpyVariants(memcpy_variant, dst, src, size, iterations, num_threads, generatorUniform<16>); + return dispatchMemcpyVariants(memcpy_variant, dst, src, size, iterations, num_threads, generatorUniform<16>); if (generator_variant == 2) - dispatchMemcpyVariants(memcpy_variant, dst, src, size, iterations, num_threads, generatorUniform<256>); + return dispatchMemcpyVariants(memcpy_variant, dst, src, size, iterations, num_threads, generatorUniform<256>); if (generator_variant == 3) - dispatchMemcpyVariants(memcpy_variant, dst, src, size, iterations, num_threads, generatorUniform<4096>); + return dispatchMemcpyVariants(memcpy_variant, dst, src, size, iterations, num_threads, generatorUniform<4096>); if (generator_variant == 4) - dispatchMemcpyVariants(memcpy_variant, dst, src, size, iterations, num_threads, generatorUniform<65536>); + return dispatchMemcpyVariants(memcpy_variant, dst, src, size, iterations, num_threads, generatorUniform<65536>); if (generator_variant == 5) - dispatchMemcpyVariants(memcpy_variant, dst, src, size, iterations, num_threads, generatorUniform<1048576>); + return dispatchMemcpyVariants(memcpy_variant, dst, src, size, iterations, num_threads, generatorUniform<1048576>); + + return 0; } int main(int argc, char ** argv) { - size_t size = 1000000000; - if (argc >= 2) - size = std::stoull(argv[1]); + boost::program_options::options_description desc("Allowed options"); + desc.add_options()("help,h", "produce help message") + ("size", boost::program_options::value()->default_value(1000000), "Bytes to copy on every iteration") + ("iterations", boost::program_options::value(), "Number of iterations") + ("threads", boost::program_options::value()->default_value(1), "Number of copying threads") + ("distribution", boost::program_options::value()->default_value(4), "Distribution of chunk sizes to perform copy") + ("variant", boost::program_options::value(), "Variant of memcpy implementation") + ("tsv", "Print result in tab-separated format") + ; - size_t iterations = 10; - if (argc >= 3) - iterations = std::stoull(argv[2]); + boost::program_options::variables_map options; + boost::program_options::store(boost::program_options::parse_command_line(argc, argv, desc), options); - size_t num_threads = 1; - if (argc >= 4) - num_threads = std::stoull(argv[3]); + if (options.count("help") || !options.count("variant")) + { + std::cout << R"(Usage: - size_t memcpy_variant = 1; - if (argc >= 5) - memcpy_variant = std::stoull(argv[4]); +for size in 4096 16384 50000 65536 100000 1000000 10000000 100000000; do + for threads in 1 2 4 $(($(nproc) / 2)) $(nproc); do + for distribution in 1 2 3 4 5; do + for variant in {1..12} {21..29}; do + for i in {1..10}; do + ./memcpy-bench --tsv --size $size --variant $variant --threads $threads --distribution $distribution; + done; + done; + done; + done; +done | tee result.tsv - size_t generator_variant = 1; - if (argc >= 6) - generator_variant = std::stoull(argv[5]); +)" << std::endl; + std::cout << desc << std::endl; + return 1; + } + + size_t size = options["size"].as(); + size_t num_threads = options["threads"].as(); + size_t memcpy_variant = options["variant"].as(); + size_t generator_variant = options["distribution"].as(); + + size_t iterations; + if (options.count("iterations")) + { + iterations = options["iterations"].as(); + } + else + { + iterations = 10000000000ULL * num_threads / size; + + if (generator_variant == 1) + iterations /= 100; + if (generator_variant == 2) + iterations /= 10; + } std::unique_ptr src(new uint8_t[size]); std::unique_ptr dst(new uint8_t[size]); @@ -614,7 +694,25 @@ int main(int argc, char ** argv) /// Fill dst to avoid page faults. memset(dst.get(), 0, size); - dispatchVariants(memcpy_variant, generator_variant, dst.get(), src.get(), size, iterations, num_threads); + uint64_t elapsed_ns = dispatchVariants(memcpy_variant, generator_variant, dst.get(), src.get(), size, iterations, num_threads); + + std::cout << std::fixed << std::setprecision(3); + + if (options.count("tsv")) + { + std::cout + << '\t' << size + << '\t' << iterations + << '\t' << num_threads + << '\t' << generator_variant + << '\t' << memcpy_variant + << '\t' << elapsed_ns + << '\n'; + } + else + { + std::cout << ": processed in " << (elapsed_ns / 1e9) << " sec, " << (size * iterations * 1.0 / elapsed_ns) << " GB/sec\n"; + } return 0; } From 402bf77783cbda48a9ee1b748bfce3c52ef8fe11 Mon Sep 17 00:00:00 2001 From: Azat Khuzhin Date: Sat, 13 Mar 2021 18:05:54 +0300 Subject: [PATCH 450/544] Fix concurrent OPTIMIZE and DROP for ReplicatedMergeTree Found with fuzzer [1] for 00992_system_parts_race_condition_zookeeper: 2021.03.13 11:12:30.385188 [ 42042 ] {2d3a8e17-26be-47c1-974f-bd2c9fc7c3af} executeQuery: (from [::1]:58192, using production parser) (comment: '/usr/share/clickhouse-test/queries/1_stateful/00153_aggregate_arena_race.sql') CREATE TABLE alter_tabl e (a UInt8, b Int16, c Float32, d String, e Array(UInt8), f Nullable(UUID), g Tuple(UInt8, UInt16)) ENGINE = ReplicatedMergeTree('/clickhouse/tables/test_3.alter_table', 'r1') ORDER BY a PARTITION BY b % 10 SETTINGS old_parts_lifetime = 1, cleanup_delay_p eriod = 1, cleanup_delay_period_random_add = 0; ... 2021.03.13 11:12:30.678387 [ 42042 ] {528cafc5-a02b-4df8-a531-a9a98e37b478} executeQuery: (from [::1]:58192, using production parser) (comment: '/usr/share/clickhouse-test/queries/1_stateful/00153_aggregate_arena_race.sql') CREATE TABLE alter_table2 (a UInt8, b Int16, c Float32, d String, e Array(UInt8), f Nullable(UUID), g Tuple(UInt8, UInt16)) ENGINE = ReplicatedMergeTree('/clickhouse/tables/test_3.alter_table', 'r2') ORDER BY a PARTITION BY b % 10 SETTINGS old_parts_lifetime = 1, cleanup_delay_period = 1, cleanup_delay_period_random_add = 0; ... 2021.03.13 11:12:40.671994 [ 4193 ] {d96ee93c-69b0-4e89-b411-16c382ae27a8} executeQuery: (from [::1]:59714, using production parser) (comment: '/usr/share/clickhouse-test/queries/1_stateful/00153_aggregate_arena_race.sql') OPTIMIZE TABLE alter_table FINAL ... 2021.03.13 11:12:40.990174 [ 2298 ] {a80f9306-3a73-4778-a921-db53249247e3} executeQuery: (from [::1]:59768, using production parser) (comment: '/usr/share/clickhouse-test/queries/1_stateful/00153_aggregate_arena_race.sql') DROP TABLE alter_table; ... 2021.03.13 11:12:41.333054 [ 2298 ] {a80f9306-3a73-4778-a921-db53249247e3} test_3.alter_table (d4fedaca-e0f6-4c22-9a4f-9f4d11b6b705): Removing part from filesystem 7_0_0_0 ... 2021.03.13 11:12:41.335380 [ 2298 ] {a80f9306-3a73-4778-a921-db53249247e3} DatabaseCatalog: Waiting for table d4fedaca-e0f6-4c22-9a4f-9f4d11b6b705 to be finally dropped ... 2021.03.13 11:12:41.781032 [ 4193 ] {d96ee93c-69b0-4e89-b411-16c382ae27a8} test_3.alter_table (d4fedaca-e0f6-4c22-9a4f-9f4d11b6b705): Waiting for queue-0000000085 to disappear from r2 queue ... 2021.03.13 11:12:41.900039 [ 371 ] {} test_3.alter_table2 (ReplicatedMergeTreeQueue): Not executing log entry queue-0000000085 of type MERGE_PARTS for part 7_0_0_1 because part 7_0_0_0 is not ready yet (log entry for that part is being processed). 2021.03.13 11:12:41.900213 [ 365 ] {} test_3.alter_table2 (ReplicatedMergeTreeQueue): Cannot execute alter metadata queue-0000000056 with version 22 because another alter 21 must be executed before 2021.03.13 11:12:41.900231 [ 13762 ] {} test_3.alter_table2 (ae877c49-0d30-416d-9afe-27fd457d8fc4): Executing log entry to merge parts -7_0_0_0 to -7_0_0_1 2021.03.13 11:12:41.900330 [ 13762 ] {} test_3.alter_table2 (ae877c49-0d30-416d-9afe-27fd457d8fc4): Don't have all parts for merge -7_0_0_1; will try to fetch it instead ... [1]: https://clickhouse-test-reports.s3.yandex.net/21691/eb3710c164b991b8d4f86b1435a65f9eceb8f1f5/stress_test_(address).html#fail1 --- src/Storages/StorageReplicatedMergeTree.cpp | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/Storages/StorageReplicatedMergeTree.cpp b/src/Storages/StorageReplicatedMergeTree.cpp index d08c5b6ad7c..2a280508364 100644 --- a/src/Storages/StorageReplicatedMergeTree.cpp +++ b/src/Storages/StorageReplicatedMergeTree.cpp @@ -4151,6 +4151,10 @@ bool StorageReplicatedMergeTree::optimize( const Names & deduplicate_by_columns, const Context & query_context) { + /// NOTE: exclusive lock cannot be used here, since this may lead to deadlock (see comments below), + /// but it should be safe to use non-exclusive to avoid dropping parts that may be required for processing queue. + auto table_lock = lockForShare(query_context.getCurrentQueryId(), query_context.getSettingsRef().lock_acquire_timeout); + assertNotReadonly(); if (!is_leader) From ce364955b7d070808b4bf862dcabec04c5d4d50d Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Sun, 14 Mar 2021 23:19:53 +0300 Subject: [PATCH 451/544] Fix broken link --- docs/ru/sql-reference/table-functions/s3.md | 168 ++++++++++++++++++++ 1 file changed, 168 insertions(+) create mode 100644 docs/ru/sql-reference/table-functions/s3.md diff --git a/docs/ru/sql-reference/table-functions/s3.md b/docs/ru/sql-reference/table-functions/s3.md new file mode 100644 index 00000000000..2427f0f863c --- /dev/null +++ b/docs/ru/sql-reference/table-functions/s3.md @@ -0,0 +1,168 @@ +--- +toc_priority: 45 +toc_title: s3 +--- + +# s3 {#s3} + +Provides table-like interface to select/insert files in S3. This table function is similar to [hdfs](../../sql-reference/table-functions/hdfs.md). + +``` sql +s3(path, [aws_access_key_id, aws_secret_access_key,] format, structure, [compression]) +``` + +**Input parameters** + +- `path` — Bucket url with path to file. Supports following wildcards in readonly mode: *, ?, {abc,def} and {N..M} where N, M — numbers, `’abc’, ‘def’ — strings. +- `format` — The [format](../../interfaces/formats.md#formats) of the file. +- `structure` — Structure of the table. Format `'column1_name column1_type, column2_name column2_type, ...'`. +- `compression` — Parameter is optional. Supported values: none, gzip/gz, brotli/br, xz/LZMA, zstd/zst. By default, it will autodetect compression by file extension. + +**Returned value** + +A table with the specified structure for reading or writing data in the specified file. + +**Example** + +Table from S3 file `https://storage.yandexcloud.net/my-test-bucket-768/data.csv` and selection of the first two rows from it: + +``` sql +SELECT * +FROM s3('https://storage.yandexcloud.net/my-test-bucket-768/data.csv', 'CSV', 'column1 UInt32, column2 UInt32, column3 UInt32') +LIMIT 2 +``` + +``` text +┌─column1─┬─column2─┬─column3─┐ +│ 1 │ 2 │ 3 │ +│ 3 │ 2 │ 1 │ +└─────────┴─────────┴─────────┘ +``` + +The similar but from file with `gzip` compression: + +``` sql +SELECT * +FROM s3('https://storage.yandexcloud.net/my-test-bucket-768/data.csv.gz', 'CSV', 'column1 UInt32, column2 UInt32, column3 UInt32', 'gzip') +LIMIT 2 +``` + +``` text +┌─column1─┬─column2─┬─column3─┐ +│ 1 │ 2 │ 3 │ +│ 3 │ 2 │ 1 │ +└─────────┴─────────┴─────────┘ +``` + +**Globs in path** + +Multiple path components can have globs. For being processed file should exists and matches to the whole path pattern (not only suffix or prefix). + +- `*` — Substitutes any number of any characters except `/` including empty string. +- `?` — Substitutes any single character. +- `{some_string,another_string,yet_another_one}` — Substitutes any of strings `'some_string', 'another_string', 'yet_another_one'`. +- `{N..M}` — Substitutes any number in range from N to M including both borders. N and M can have leading zeroes e.g. `000..078`. + +Constructions with `{}` are similar to the [remote table function](../../sql-reference/table-functions/remote.md)). + +**Example** + +1. Suppose that we have several files with following URIs on S3: + +- ‘https://storage.yandexcloud.net/my-test-bucket-768/some_prefix/some_file_1.csv’ +- ‘https://storage.yandexcloud.net/my-test-bucket-768/some_prefix/some_file_2.csv’ +- ‘https://storage.yandexcloud.net/my-test-bucket-768/some_prefix/some_file_3.csv’ +- ‘https://storage.yandexcloud.net/my-test-bucket-768/some_prefix/some_file_4.csv’ +- ‘https://storage.yandexcloud.net/my-test-bucket-768/another_prefix/some_file_1.csv’ +- ‘https://storage.yandexcloud.net/my-test-bucket-768/another_prefix/some_file_2.csv’ +- ‘https://storage.yandexcloud.net/my-test-bucket-768/another_prefix/some_file_3.csv’ +- ‘https://storage.yandexcloud.net/my-test-bucket-768/another_prefix/some_file_4.csv’ + +2. Query the amount of rows in files end with number from 1 to 3: + + + +``` sql +SELECT count(*) +FROM s3('https://storage.yandexcloud.net/my-test-bucket-768/{some,another}_prefix/some_file_{1..3}.csv', 'CSV', 'name String, value UInt32') +``` + +``` text +┌─count()─┐ +│ 18 │ +└─────────┘ +``` + +3. Query the amount of rows in all files of these two directories: + + + +``` sql +SELECT count(*) +FROM s3('https://storage.yandexcloud.net/my-test-bucket-768/{some,another}_prefix/*', 'CSV', 'name String, value UInt32') +``` + +``` text +┌─count()─┐ +│ 24 │ +└─────────┘ +``` + + +!!! warning "Warning" + If your listing of files contains number ranges with leading zeros, use the construction with braces for each digit separately or use `?`. + +**Example** + +Query the data from files named `file-000.csv`, `file-001.csv`, … , `file-999.csv`: + +``` sql +SELECT count(*) +FROM s3('https://storage.yandexcloud.net/my-test-bucket-768/big_prefix/file-{000..999}.csv', 'CSV', 'name String, value UInt32') +``` + +``` text +┌─count()─┐ +│ 12 │ +└─────────┘ +``` + +**Data insert** + +The S3 table function may be used for data insert as well. + +**Example** + +Insert a data into file `test-data.csv.gz`: + +``` sql +INSERT INTO s3('https://storage.yandexcloud.net/my-test-bucket-768/test-data.csv.gz', 'CSV', 'name String, value UInt32', 'gzip') +VALUES ('test-data', 1), ('test-data-2', 2) +``` + +Insert a data into file `test-data.csv.gz` from existing table: + +``` sql +INSERT INTO s3('https://storage.yandexcloud.net/my-test-bucket-768/test-data.csv.gz', 'CSV', 'name String, value UInt32', 'gzip') +SELECT name, value FROM existing_table +``` + +## Virtual Columns {#virtual-columns} + +- `_path` — Path to the file. +- `_file` — Name of the file. + +## S3-related settings {#settings} + +The following settings can be set before query execution or placed into configuration file. + +- `s3_max_single_part_upload_size` — Default value is `64Mb`. The maximum size of object to upload using singlepart upload to S3. +- `s3_min_upload_part_size` — Default value is `512Mb`. The minimum size of part to upload during multipart upload to [S3 Multipart upload](https://docs.aws.amazon.com/AmazonS3/latest/dev/uploadobjusingmpu.html). +- `s3_max_redirects` — Default value is `10`. Max number of S3 redirects hops allowed. + +Security consideration: if malicious user can specify arbitrary S3 URLs, `s3_max_redirects` must be set to zero to avoid [SSRF](https://en.wikipedia.org/wiki/Server-side_request_forgery) attacks; or alternatively, `remote_host_filter` must be specified in server configuration. + +**See Also** + +- [Virtual columns](../../engines/table-engines/index.md#table_engines-virtual_columns) + From 145116bfb64a9a135cd60a8d5b5ebdd6d8310676 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Sun, 14 Mar 2021 23:22:12 +0300 Subject: [PATCH 452/544] Fix style --- utils/memcpy-bench/glibc/dwarf2.h | 74 +- utils/memcpy-bench/glibc/memcpy-ssse3-back.S | 4988 ++++++++-------- utils/memcpy-bench/glibc/memcpy-ssse3.S | 5176 ++++++++--------- .../glibc/memmove-avx-unaligned-erms.S | 14 +- .../glibc/memmove-avx512-no-vzeroupper.S | 662 +-- .../glibc/memmove-avx512-unaligned-erms.S | 14 +- .../glibc/memmove-sse2-unaligned-erms.S | 2 +- .../glibc/memmove-vec-unaligned-erms.S | 694 +-- utils/memcpy-bench/glibc/memmove.S | 26 +- utils/memcpy-bench/glibc/sysdep.h | 100 +- utils/memcpy-bench/glibc/sysdep_generic.h | 52 +- utils/memcpy-bench/glibc/sysdep_x86.h | 52 +- 12 files changed, 5927 insertions(+), 5927 deletions(-) diff --git a/utils/memcpy-bench/glibc/dwarf2.h b/utils/memcpy-bench/glibc/dwarf2.h index 4c7de0d8737..2be827f00ae 100644 --- a/utils/memcpy-bench/glibc/dwarf2.h +++ b/utils/memcpy-bench/glibc/dwarf2.h @@ -21,7 +21,7 @@ . */ #ifndef _DWARF2_H -#define _DWARF2_H 1 +#define _DWARF2_H 1 /* This file is derived from the DWARF specification (a public document) Revision 2.0.0 (July 27, 1993) developed by the UNIX International @@ -88,19 +88,19 @@ enum dwarf_tag /* SGI/MIPS Extensions */ DW_TAG_MIPS_loop = 0x4081, /* GNU extensions */ - DW_TAG_format_label = 0x4101, /* for FORTRAN 77 and Fortran 90 */ - DW_TAG_function_template = 0x4102, /* for C++ */ - DW_TAG_class_template = 0x4103, /* for C++ */ + DW_TAG_format_label = 0x4101, /* for FORTRAN 77 and Fortran 90 */ + DW_TAG_function_template = 0x4102, /* for C++ */ + DW_TAG_class_template = 0x4103, /* for C++ */ DW_TAG_GNU_BINCL = 0x4104, DW_TAG_GNU_EINCL = 0x4105 }; -#define DW_TAG_lo_user 0x4080 -#define DW_TAG_hi_user 0xffff +#define DW_TAG_lo_user 0x4080 +#define DW_TAG_hi_user 0xffff /* flag that tells whether entry has a child or not */ #define DW_children_no 0 -#define DW_children_yes 1 +#define DW_children_yes 1 /* Form names and codes. */ enum dwarf_form @@ -215,8 +215,8 @@ enum dwarf_attribute DW_AT_body_end = 0x2106 }; -#define DW_AT_lo_user 0x2000 /* implementation-defined range start */ -#define DW_AT_hi_user 0x3ff0 /* implementation-defined range end */ +#define DW_AT_lo_user 0x2000 /* implementation-defined range start */ +#define DW_AT_hi_user 0x3ff0 /* implementation-defined range end */ /* Location atom names and codes. */ @@ -369,8 +369,8 @@ enum dwarf_location_atom DW_OP_nop = 0x96 }; -#define DW_OP_lo_user 0x80 /* implementation-defined range start */ -#define DW_OP_hi_user 0xff /* implementation-defined range end */ +#define DW_OP_lo_user 0x80 /* implementation-defined range start */ +#define DW_OP_hi_user 0xff /* implementation-defined range end */ /* Type encodings. */ @@ -387,8 +387,8 @@ enum dwarf_type DW_ATE_unsigned_char = 0x8 }; -#define DW_ATE_lo_user 0x80 -#define DW_ATE_hi_user 0xff +#define DW_ATE_lo_user 0x80 +#define DW_ATE_hi_user 0xff /* Array ordering names and codes. */ enum dwarf_array_dim_ordering @@ -517,17 +517,17 @@ enum dwarf_call_frame_info DW_CFA_GNU_negative_offset_extended = 0x2f }; -#define DW_CIE_ID 0xffffffff -#define DW_CIE_VERSION 1 +#define DW_CIE_ID 0xffffffff +#define DW_CIE_VERSION 1 #define DW_CFA_extended 0 #define DW_CFA_low_user 0x1c #define DW_CFA_high_user 0x3f -#define DW_CHILDREN_no 0x00 -#define DW_CHILDREN_yes 0x01 +#define DW_CHILDREN_no 0x00 +#define DW_CHILDREN_yes 0x01 -#define DW_ADDR_none 0 +#define DW_ADDR_none 0 /* Source language names and codes. */ @@ -548,8 +548,8 @@ enum dwarf_source_language }; -#define DW_LANG_lo_user 0x8000 /* implementation-defined range start */ -#define DW_LANG_hi_user 0xffff /* implementation-defined range start */ +#define DW_LANG_lo_user 0x8000 /* implementation-defined range start */ +#define DW_LANG_hi_user 0xffff /* implementation-defined range start */ /* Names and codes for macro information. */ @@ -566,25 +566,25 @@ enum dwarf_macinfo_record_type /* @@@ For use with GNU frame unwind information. */ -#define DW_EH_PE_absptr 0x00 -#define DW_EH_PE_omit 0xff +#define DW_EH_PE_absptr 0x00 +#define DW_EH_PE_omit 0xff -#define DW_EH_PE_uleb128 0x01 -#define DW_EH_PE_udata2 0x02 -#define DW_EH_PE_udata4 0x03 -#define DW_EH_PE_udata8 0x04 -#define DW_EH_PE_sleb128 0x09 -#define DW_EH_PE_sdata2 0x0A -#define DW_EH_PE_sdata4 0x0B -#define DW_EH_PE_sdata8 0x0C -#define DW_EH_PE_signed 0x08 +#define DW_EH_PE_uleb128 0x01 +#define DW_EH_PE_udata2 0x02 +#define DW_EH_PE_udata4 0x03 +#define DW_EH_PE_udata8 0x04 +#define DW_EH_PE_sleb128 0x09 +#define DW_EH_PE_sdata2 0x0A +#define DW_EH_PE_sdata4 0x0B +#define DW_EH_PE_sdata8 0x0C +#define DW_EH_PE_signed 0x08 -#define DW_EH_PE_pcrel 0x10 -#define DW_EH_PE_textrel 0x20 -#define DW_EH_PE_datarel 0x30 -#define DW_EH_PE_funcrel 0x40 -#define DW_EH_PE_aligned 0x50 +#define DW_EH_PE_pcrel 0x10 +#define DW_EH_PE_textrel 0x20 +#define DW_EH_PE_datarel 0x30 +#define DW_EH_PE_funcrel 0x40 +#define DW_EH_PE_aligned 0x50 -#define DW_EH_PE_indirect 0x80 +#define DW_EH_PE_indirect 0x80 #endif /* dwarf2.h */ diff --git a/utils/memcpy-bench/glibc/memcpy-ssse3-back.S b/utils/memcpy-bench/glibc/memcpy-ssse3-back.S index 1492dd38e73..c5257592efa 100644 --- a/utils/memcpy-bench/glibc/memcpy-ssse3-back.S +++ b/utils/memcpy-bench/glibc/memcpy-ssse3-back.S @@ -24,3159 +24,3159 @@ #include "asm-syntax.h" #ifndef MEMCPY -# define MEMCPY __memcpy_ssse3_back -# define MEMCPY_CHK __memcpy_chk_ssse3_back -# define MEMPCPY __mempcpy_ssse3_back -# define MEMPCPY_CHK __mempcpy_chk_ssse3_back +# define MEMCPY __memcpy_ssse3_back +# define MEMCPY_CHK __memcpy_chk_ssse3_back +# define MEMPCPY __mempcpy_ssse3_back +# define MEMPCPY_CHK __mempcpy_chk_ssse3_back #endif -#define JMPTBL(I, B) I - B +#define JMPTBL(I, B) I - B /* Branch to an entry in a jump table. TABLE is a jump table with relative offsets. INDEX is a register contains the index into the jump table. SCALE is the scale of INDEX. */ -#define BRANCH_TO_JMPTBL_ENTRY(TABLE, INDEX, SCALE) \ - lea TABLE(%rip), %r11; \ - movslq (%r11, INDEX, SCALE), INDEX; \ - lea (%r11, INDEX), INDEX; \ - _CET_NOTRACK jmp *INDEX; \ +#define BRANCH_TO_JMPTBL_ENTRY(TABLE, INDEX, SCALE) \ + lea TABLE(%rip), %r11; \ + movslq (%r11, INDEX, SCALE), INDEX; \ + lea (%r11, INDEX), INDEX; \ + _CET_NOTRACK jmp *INDEX; \ ud2 - .section .text.ssse3,"ax",@progbits + .section .text.ssse3,"ax",@progbits #if !defined USE_AS_MEMPCPY && !defined USE_AS_MEMMOVE ENTRY (MEMPCPY_CHK) - cmp %RDX_LP, %RCX_LP - jb HIDDEN_JUMPTARGET (__chk_fail) + cmp %RDX_LP, %RCX_LP + jb HIDDEN_JUMPTARGET (__chk_fail) END (MEMPCPY_CHK) ENTRY (MEMPCPY) - mov %RDI_LP, %RAX_LP - add %RDX_LP, %RAX_LP - jmp L(start) + mov %RDI_LP, %RAX_LP + add %RDX_LP, %RAX_LP + jmp L(start) END (MEMPCPY) #endif #if !defined USE_AS_BCOPY ENTRY (MEMCPY_CHK) - cmp %RDX_LP, %RCX_LP - jb HIDDEN_JUMPTARGET (__chk_fail) + cmp %RDX_LP, %RCX_LP + jb HIDDEN_JUMPTARGET (__chk_fail) END (MEMCPY_CHK) #endif ENTRY (MEMCPY) - mov %RDI_LP, %RAX_LP + mov %RDI_LP, %RAX_LP #ifdef USE_AS_MEMPCPY - add %RDX_LP, %RAX_LP + add %RDX_LP, %RAX_LP #endif #ifdef __ILP32__ - /* Clear the upper 32 bits. */ - mov %edx, %edx + /* Clear the upper 32 bits. */ + mov %edx, %edx #endif #ifdef USE_AS_MEMMOVE - cmp %rsi, %rdi - jb L(copy_forward) - je L(bwd_write_0bytes) - cmp $144, %rdx - jae L(copy_backward) - BRANCH_TO_JMPTBL_ENTRY (L(table_144_bytes_bwd), %rdx, 4) + cmp %rsi, %rdi + jb L(copy_forward) + je L(bwd_write_0bytes) + cmp $144, %rdx + jae L(copy_backward) + BRANCH_TO_JMPTBL_ENTRY (L(table_144_bytes_bwd), %rdx, 4) L(copy_forward): #endif L(start): - cmp $144, %rdx - jae L(144bytesormore) + cmp $144, %rdx + jae L(144bytesormore) L(fwd_write_less32bytes): #ifndef USE_AS_MEMMOVE - cmp %dil, %sil - jbe L(bk_write) + cmp %dil, %sil + jbe L(bk_write) #endif - add %rdx, %rsi - add %rdx, %rdi - BRANCH_TO_JMPTBL_ENTRY (L(table_144_bytes_fwd), %rdx, 4) + add %rdx, %rsi + add %rdx, %rdi + BRANCH_TO_JMPTBL_ENTRY (L(table_144_bytes_fwd), %rdx, 4) #ifndef USE_AS_MEMMOVE L(bk_write): - BRANCH_TO_JMPTBL_ENTRY (L(table_144_bytes_bwd), %rdx, 4) + BRANCH_TO_JMPTBL_ENTRY (L(table_144_bytes_bwd), %rdx, 4) #endif - .p2align 4 + .p2align 4 L(144bytesormore): #ifndef USE_AS_MEMMOVE - cmp %dil, %sil - jle L(copy_backward) + cmp %dil, %sil + jle L(copy_backward) #endif - movdqu (%rsi), %xmm0 - mov %rdi, %r8 - and $-16, %rdi - add $16, %rdi - mov %rdi, %r9 - sub %r8, %r9 - sub %r9, %rdx - add %r9, %rsi - mov %rsi, %r9 - and $0xf, %r9 - jz L(shl_0) + movdqu (%rsi), %xmm0 + mov %rdi, %r8 + and $-16, %rdi + add $16, %rdi + mov %rdi, %r9 + sub %r8, %r9 + sub %r9, %rdx + add %r9, %rsi + mov %rsi, %r9 + and $0xf, %r9 + jz L(shl_0) #ifdef DATA_CACHE_SIZE - mov $DATA_CACHE_SIZE, %RCX_LP + mov $DATA_CACHE_SIZE, %RCX_LP #else - mov __x86_data_cache_size(%rip), %RCX_LP + mov __x86_data_cache_size(%rip), %RCX_LP #endif - cmp %rcx, %rdx - jae L(gobble_mem_fwd) - lea L(shl_table_fwd)(%rip), %r11 - sub $0x80, %rdx - movslq (%r11, %r9, 4), %r9 - add %r11, %r9 - _CET_NOTRACK jmp *%r9 - ud2 + cmp %rcx, %rdx + jae L(gobble_mem_fwd) + lea L(shl_table_fwd)(%rip), %r11 + sub $0x80, %rdx + movslq (%r11, %r9, 4), %r9 + add %r11, %r9 + _CET_NOTRACK jmp *%r9 + ud2 - .p2align 4 + .p2align 4 L(copy_backward): #ifdef DATA_CACHE_SIZE - mov $DATA_CACHE_SIZE, %RCX_LP + mov $DATA_CACHE_SIZE, %RCX_LP #else - mov __x86_data_cache_size(%rip), %RCX_LP + mov __x86_data_cache_size(%rip), %RCX_LP #endif - shl $1, %rcx - cmp %rcx, %rdx - ja L(gobble_mem_bwd) + shl $1, %rcx + cmp %rcx, %rdx + ja L(gobble_mem_bwd) - add %rdx, %rdi - add %rdx, %rsi - movdqu -16(%rsi), %xmm0 - lea -16(%rdi), %r8 - mov %rdi, %r9 - and $0xf, %r9 - xor %r9, %rdi - sub %r9, %rsi - sub %r9, %rdx - mov %rsi, %r9 - and $0xf, %r9 - jz L(shl_0_bwd) - lea L(shl_table_bwd)(%rip), %r11 - sub $0x80, %rdx - movslq (%r11, %r9, 4), %r9 - add %r11, %r9 - _CET_NOTRACK jmp *%r9 - ud2 + add %rdx, %rdi + add %rdx, %rsi + movdqu -16(%rsi), %xmm0 + lea -16(%rdi), %r8 + mov %rdi, %r9 + and $0xf, %r9 + xor %r9, %rdi + sub %r9, %rsi + sub %r9, %rdx + mov %rsi, %r9 + and $0xf, %r9 + jz L(shl_0_bwd) + lea L(shl_table_bwd)(%rip), %r11 + sub $0x80, %rdx + movslq (%r11, %r9, 4), %r9 + add %r11, %r9 + _CET_NOTRACK jmp *%r9 + ud2 - .p2align 4 + .p2align 4 L(shl_0): - mov %rdx, %r9 - shr $8, %r9 - add %rdx, %r9 + mov %rdx, %r9 + shr $8, %r9 + add %rdx, %r9 #ifdef DATA_CACHE_SIZE - cmp $DATA_CACHE_SIZE_HALF, %R9_LP + cmp $DATA_CACHE_SIZE_HALF, %R9_LP #else - cmp __x86_data_cache_size_half(%rip), %R9_LP + cmp __x86_data_cache_size_half(%rip), %R9_LP #endif - jae L(gobble_mem_fwd) - sub $0x80, %rdx - .p2align 4 + jae L(gobble_mem_fwd) + sub $0x80, %rdx + .p2align 4 L(shl_0_loop): - movdqa (%rsi), %xmm1 - movdqa %xmm1, (%rdi) - movaps 0x10(%rsi), %xmm2 - movaps %xmm2, 0x10(%rdi) - movaps 0x20(%rsi), %xmm3 - movaps %xmm3, 0x20(%rdi) - movaps 0x30(%rsi), %xmm4 - movaps %xmm4, 0x30(%rdi) - movaps 0x40(%rsi), %xmm1 - movaps %xmm1, 0x40(%rdi) - movaps 0x50(%rsi), %xmm2 - movaps %xmm2, 0x50(%rdi) - movaps 0x60(%rsi), %xmm3 - movaps %xmm3, 0x60(%rdi) - movaps 0x70(%rsi), %xmm4 - movaps %xmm4, 0x70(%rdi) - sub $0x80, %rdx - lea 0x80(%rsi), %rsi - lea 0x80(%rdi), %rdi - jae L(shl_0_loop) - movdqu %xmm0, (%r8) - add $0x80, %rdx - add %rdx, %rsi - add %rdx, %rdi - BRANCH_TO_JMPTBL_ENTRY (L(table_144_bytes_fwd), %rdx, 4) + movdqa (%rsi), %xmm1 + movdqa %xmm1, (%rdi) + movaps 0x10(%rsi), %xmm2 + movaps %xmm2, 0x10(%rdi) + movaps 0x20(%rsi), %xmm3 + movaps %xmm3, 0x20(%rdi) + movaps 0x30(%rsi), %xmm4 + movaps %xmm4, 0x30(%rdi) + movaps 0x40(%rsi), %xmm1 + movaps %xmm1, 0x40(%rdi) + movaps 0x50(%rsi), %xmm2 + movaps %xmm2, 0x50(%rdi) + movaps 0x60(%rsi), %xmm3 + movaps %xmm3, 0x60(%rdi) + movaps 0x70(%rsi), %xmm4 + movaps %xmm4, 0x70(%rdi) + sub $0x80, %rdx + lea 0x80(%rsi), %rsi + lea 0x80(%rdi), %rdi + jae L(shl_0_loop) + movdqu %xmm0, (%r8) + add $0x80, %rdx + add %rdx, %rsi + add %rdx, %rdi + BRANCH_TO_JMPTBL_ENTRY (L(table_144_bytes_fwd), %rdx, 4) - .p2align 4 + .p2align 4 L(shl_0_bwd): - sub $0x80, %rdx + sub $0x80, %rdx L(copy_backward_loop): - movaps -0x10(%rsi), %xmm1 - movaps %xmm1, -0x10(%rdi) - movaps -0x20(%rsi), %xmm2 - movaps %xmm2, -0x20(%rdi) - movaps -0x30(%rsi), %xmm3 - movaps %xmm3, -0x30(%rdi) - movaps -0x40(%rsi), %xmm4 - movaps %xmm4, -0x40(%rdi) - movaps -0x50(%rsi), %xmm5 - movaps %xmm5, -0x50(%rdi) - movaps -0x60(%rsi), %xmm5 - movaps %xmm5, -0x60(%rdi) - movaps -0x70(%rsi), %xmm5 - movaps %xmm5, -0x70(%rdi) - movaps -0x80(%rsi), %xmm5 - movaps %xmm5, -0x80(%rdi) - sub $0x80, %rdx - lea -0x80(%rdi), %rdi - lea -0x80(%rsi), %rsi - jae L(copy_backward_loop) + movaps -0x10(%rsi), %xmm1 + movaps %xmm1, -0x10(%rdi) + movaps -0x20(%rsi), %xmm2 + movaps %xmm2, -0x20(%rdi) + movaps -0x30(%rsi), %xmm3 + movaps %xmm3, -0x30(%rdi) + movaps -0x40(%rsi), %xmm4 + movaps %xmm4, -0x40(%rdi) + movaps -0x50(%rsi), %xmm5 + movaps %xmm5, -0x50(%rdi) + movaps -0x60(%rsi), %xmm5 + movaps %xmm5, -0x60(%rdi) + movaps -0x70(%rsi), %xmm5 + movaps %xmm5, -0x70(%rdi) + movaps -0x80(%rsi), %xmm5 + movaps %xmm5, -0x80(%rdi) + sub $0x80, %rdx + lea -0x80(%rdi), %rdi + lea -0x80(%rsi), %rsi + jae L(copy_backward_loop) - movdqu %xmm0, (%r8) - add $0x80, %rdx - sub %rdx, %rdi - sub %rdx, %rsi - BRANCH_TO_JMPTBL_ENTRY (L(table_144_bytes_bwd), %rdx, 4) + movdqu %xmm0, (%r8) + add $0x80, %rdx + sub %rdx, %rdi + sub %rdx, %rsi + BRANCH_TO_JMPTBL_ENTRY (L(table_144_bytes_bwd), %rdx, 4) - .p2align 4 + .p2align 4 L(shl_1): - sub $0x80, %rdx - movaps -0x01(%rsi), %xmm1 - movaps 0x0f(%rsi), %xmm2 - movaps 0x1f(%rsi), %xmm3 - movaps 0x2f(%rsi), %xmm4 - movaps 0x3f(%rsi), %xmm5 - movaps 0x4f(%rsi), %xmm6 - movaps 0x5f(%rsi), %xmm7 - movaps 0x6f(%rsi), %xmm8 - movaps 0x7f(%rsi), %xmm9 - lea 0x80(%rsi), %rsi - palignr $1, %xmm8, %xmm9 - movaps %xmm9, 0x70(%rdi) - palignr $1, %xmm7, %xmm8 - movaps %xmm8, 0x60(%rdi) - palignr $1, %xmm6, %xmm7 - movaps %xmm7, 0x50(%rdi) - palignr $1, %xmm5, %xmm6 - movaps %xmm6, 0x40(%rdi) - palignr $1, %xmm4, %xmm5 - movaps %xmm5, 0x30(%rdi) - palignr $1, %xmm3, %xmm4 - movaps %xmm4, 0x20(%rdi) - palignr $1, %xmm2, %xmm3 - movaps %xmm3, 0x10(%rdi) - palignr $1, %xmm1, %xmm2 - movaps %xmm2, (%rdi) - lea 0x80(%rdi), %rdi - jae L(shl_1) - movdqu %xmm0, (%r8) - add $0x80, %rdx - add %rdx, %rdi - add %rdx, %rsi - BRANCH_TO_JMPTBL_ENTRY (L(table_144_bytes_fwd), %rdx, 4) + sub $0x80, %rdx + movaps -0x01(%rsi), %xmm1 + movaps 0x0f(%rsi), %xmm2 + movaps 0x1f(%rsi), %xmm3 + movaps 0x2f(%rsi), %xmm4 + movaps 0x3f(%rsi), %xmm5 + movaps 0x4f(%rsi), %xmm6 + movaps 0x5f(%rsi), %xmm7 + movaps 0x6f(%rsi), %xmm8 + movaps 0x7f(%rsi), %xmm9 + lea 0x80(%rsi), %rsi + palignr $1, %xmm8, %xmm9 + movaps %xmm9, 0x70(%rdi) + palignr $1, %xmm7, %xmm8 + movaps %xmm8, 0x60(%rdi) + palignr $1, %xmm6, %xmm7 + movaps %xmm7, 0x50(%rdi) + palignr $1, %xmm5, %xmm6 + movaps %xmm6, 0x40(%rdi) + palignr $1, %xmm4, %xmm5 + movaps %xmm5, 0x30(%rdi) + palignr $1, %xmm3, %xmm4 + movaps %xmm4, 0x20(%rdi) + palignr $1, %xmm2, %xmm3 + movaps %xmm3, 0x10(%rdi) + palignr $1, %xmm1, %xmm2 + movaps %xmm2, (%rdi) + lea 0x80(%rdi), %rdi + jae L(shl_1) + movdqu %xmm0, (%r8) + add $0x80, %rdx + add %rdx, %rdi + add %rdx, %rsi + BRANCH_TO_JMPTBL_ENTRY (L(table_144_bytes_fwd), %rdx, 4) - .p2align 4 + .p2align 4 L(shl_1_bwd): - movaps -0x01(%rsi), %xmm1 + movaps -0x01(%rsi), %xmm1 - movaps -0x11(%rsi), %xmm2 - palignr $1, %xmm2, %xmm1 - movaps %xmm1, -0x10(%rdi) + movaps -0x11(%rsi), %xmm2 + palignr $1, %xmm2, %xmm1 + movaps %xmm1, -0x10(%rdi) - movaps -0x21(%rsi), %xmm3 - palignr $1, %xmm3, %xmm2 - movaps %xmm2, -0x20(%rdi) + movaps -0x21(%rsi), %xmm3 + palignr $1, %xmm3, %xmm2 + movaps %xmm2, -0x20(%rdi) - movaps -0x31(%rsi), %xmm4 - palignr $1, %xmm4, %xmm3 - movaps %xmm3, -0x30(%rdi) + movaps -0x31(%rsi), %xmm4 + palignr $1, %xmm4, %xmm3 + movaps %xmm3, -0x30(%rdi) - movaps -0x41(%rsi), %xmm5 - palignr $1, %xmm5, %xmm4 - movaps %xmm4, -0x40(%rdi) + movaps -0x41(%rsi), %xmm5 + palignr $1, %xmm5, %xmm4 + movaps %xmm4, -0x40(%rdi) - movaps -0x51(%rsi), %xmm6 - palignr $1, %xmm6, %xmm5 - movaps %xmm5, -0x50(%rdi) + movaps -0x51(%rsi), %xmm6 + palignr $1, %xmm6, %xmm5 + movaps %xmm5, -0x50(%rdi) - movaps -0x61(%rsi), %xmm7 - palignr $1, %xmm7, %xmm6 - movaps %xmm6, -0x60(%rdi) + movaps -0x61(%rsi), %xmm7 + palignr $1, %xmm7, %xmm6 + movaps %xmm6, -0x60(%rdi) - movaps -0x71(%rsi), %xmm8 - palignr $1, %xmm8, %xmm7 - movaps %xmm7, -0x70(%rdi) + movaps -0x71(%rsi), %xmm8 + palignr $1, %xmm8, %xmm7 + movaps %xmm7, -0x70(%rdi) - movaps -0x81(%rsi), %xmm9 - palignr $1, %xmm9, %xmm8 - movaps %xmm8, -0x80(%rdi) + movaps -0x81(%rsi), %xmm9 + palignr $1, %xmm9, %xmm8 + movaps %xmm8, -0x80(%rdi) - sub $0x80, %rdx - lea -0x80(%rdi), %rdi - lea -0x80(%rsi), %rsi - jae L(shl_1_bwd) - movdqu %xmm0, (%r8) - add $0x80, %rdx - sub %rdx, %rdi - sub %rdx, %rsi - BRANCH_TO_JMPTBL_ENTRY (L(table_144_bytes_bwd), %rdx, 4) + sub $0x80, %rdx + lea -0x80(%rdi), %rdi + lea -0x80(%rsi), %rsi + jae L(shl_1_bwd) + movdqu %xmm0, (%r8) + add $0x80, %rdx + sub %rdx, %rdi + sub %rdx, %rsi + BRANCH_TO_JMPTBL_ENTRY (L(table_144_bytes_bwd), %rdx, 4) - .p2align 4 + .p2align 4 L(shl_2): - sub $0x80, %rdx - movaps -0x02(%rsi), %xmm1 - movaps 0x0e(%rsi), %xmm2 - movaps 0x1e(%rsi), %xmm3 - movaps 0x2e(%rsi), %xmm4 - movaps 0x3e(%rsi), %xmm5 - movaps 0x4e(%rsi), %xmm6 - movaps 0x5e(%rsi), %xmm7 - movaps 0x6e(%rsi), %xmm8 - movaps 0x7e(%rsi), %xmm9 - lea 0x80(%rsi), %rsi - palignr $2, %xmm8, %xmm9 - movaps %xmm9, 0x70(%rdi) - palignr $2, %xmm7, %xmm8 - movaps %xmm8, 0x60(%rdi) - palignr $2, %xmm6, %xmm7 - movaps %xmm7, 0x50(%rdi) - palignr $2, %xmm5, %xmm6 - movaps %xmm6, 0x40(%rdi) - palignr $2, %xmm4, %xmm5 - movaps %xmm5, 0x30(%rdi) - palignr $2, %xmm3, %xmm4 - movaps %xmm4, 0x20(%rdi) - palignr $2, %xmm2, %xmm3 - movaps %xmm3, 0x10(%rdi) - palignr $2, %xmm1, %xmm2 - movaps %xmm2, (%rdi) - lea 0x80(%rdi), %rdi - jae L(shl_2) - movdqu %xmm0, (%r8) - add $0x80, %rdx - add %rdx, %rdi - add %rdx, %rsi - BRANCH_TO_JMPTBL_ENTRY (L(table_144_bytes_fwd), %rdx, 4) + sub $0x80, %rdx + movaps -0x02(%rsi), %xmm1 + movaps 0x0e(%rsi), %xmm2 + movaps 0x1e(%rsi), %xmm3 + movaps 0x2e(%rsi), %xmm4 + movaps 0x3e(%rsi), %xmm5 + movaps 0x4e(%rsi), %xmm6 + movaps 0x5e(%rsi), %xmm7 + movaps 0x6e(%rsi), %xmm8 + movaps 0x7e(%rsi), %xmm9 + lea 0x80(%rsi), %rsi + palignr $2, %xmm8, %xmm9 + movaps %xmm9, 0x70(%rdi) + palignr $2, %xmm7, %xmm8 + movaps %xmm8, 0x60(%rdi) + palignr $2, %xmm6, %xmm7 + movaps %xmm7, 0x50(%rdi) + palignr $2, %xmm5, %xmm6 + movaps %xmm6, 0x40(%rdi) + palignr $2, %xmm4, %xmm5 + movaps %xmm5, 0x30(%rdi) + palignr $2, %xmm3, %xmm4 + movaps %xmm4, 0x20(%rdi) + palignr $2, %xmm2, %xmm3 + movaps %xmm3, 0x10(%rdi) + palignr $2, %xmm1, %xmm2 + movaps %xmm2, (%rdi) + lea 0x80(%rdi), %rdi + jae L(shl_2) + movdqu %xmm0, (%r8) + add $0x80, %rdx + add %rdx, %rdi + add %rdx, %rsi + BRANCH_TO_JMPTBL_ENTRY (L(table_144_bytes_fwd), %rdx, 4) - .p2align 4 + .p2align 4 L(shl_2_bwd): - movaps -0x02(%rsi), %xmm1 + movaps -0x02(%rsi), %xmm1 - movaps -0x12(%rsi), %xmm2 - palignr $2, %xmm2, %xmm1 - movaps %xmm1, -0x10(%rdi) + movaps -0x12(%rsi), %xmm2 + palignr $2, %xmm2, %xmm1 + movaps %xmm1, -0x10(%rdi) - movaps -0x22(%rsi), %xmm3 - palignr $2, %xmm3, %xmm2 - movaps %xmm2, -0x20(%rdi) + movaps -0x22(%rsi), %xmm3 + palignr $2, %xmm3, %xmm2 + movaps %xmm2, -0x20(%rdi) - movaps -0x32(%rsi), %xmm4 - palignr $2, %xmm4, %xmm3 - movaps %xmm3, -0x30(%rdi) + movaps -0x32(%rsi), %xmm4 + palignr $2, %xmm4, %xmm3 + movaps %xmm3, -0x30(%rdi) - movaps -0x42(%rsi), %xmm5 - palignr $2, %xmm5, %xmm4 - movaps %xmm4, -0x40(%rdi) + movaps -0x42(%rsi), %xmm5 + palignr $2, %xmm5, %xmm4 + movaps %xmm4, -0x40(%rdi) - movaps -0x52(%rsi), %xmm6 - palignr $2, %xmm6, %xmm5 - movaps %xmm5, -0x50(%rdi) + movaps -0x52(%rsi), %xmm6 + palignr $2, %xmm6, %xmm5 + movaps %xmm5, -0x50(%rdi) - movaps -0x62(%rsi), %xmm7 - palignr $2, %xmm7, %xmm6 - movaps %xmm6, -0x60(%rdi) + movaps -0x62(%rsi), %xmm7 + palignr $2, %xmm7, %xmm6 + movaps %xmm6, -0x60(%rdi) - movaps -0x72(%rsi), %xmm8 - palignr $2, %xmm8, %xmm7 - movaps %xmm7, -0x70(%rdi) + movaps -0x72(%rsi), %xmm8 + palignr $2, %xmm8, %xmm7 + movaps %xmm7, -0x70(%rdi) - movaps -0x82(%rsi), %xmm9 - palignr $2, %xmm9, %xmm8 - movaps %xmm8, -0x80(%rdi) + movaps -0x82(%rsi), %xmm9 + palignr $2, %xmm9, %xmm8 + movaps %xmm8, -0x80(%rdi) - sub $0x80, %rdx - lea -0x80(%rdi), %rdi - lea -0x80(%rsi), %rsi - jae L(shl_2_bwd) - movdqu %xmm0, (%r8) - add $0x80, %rdx - sub %rdx, %rdi - sub %rdx, %rsi - BRANCH_TO_JMPTBL_ENTRY (L(table_144_bytes_bwd), %rdx, 4) + sub $0x80, %rdx + lea -0x80(%rdi), %rdi + lea -0x80(%rsi), %rsi + jae L(shl_2_bwd) + movdqu %xmm0, (%r8) + add $0x80, %rdx + sub %rdx, %rdi + sub %rdx, %rsi + BRANCH_TO_JMPTBL_ENTRY (L(table_144_bytes_bwd), %rdx, 4) - .p2align 4 + .p2align 4 L(shl_3): - sub $0x80, %rdx - movaps -0x03(%rsi), %xmm1 - movaps 0x0d(%rsi), %xmm2 - movaps 0x1d(%rsi), %xmm3 - movaps 0x2d(%rsi), %xmm4 - movaps 0x3d(%rsi), %xmm5 - movaps 0x4d(%rsi), %xmm6 - movaps 0x5d(%rsi), %xmm7 - movaps 0x6d(%rsi), %xmm8 - movaps 0x7d(%rsi), %xmm9 - lea 0x80(%rsi), %rsi - palignr $3, %xmm8, %xmm9 - movaps %xmm9, 0x70(%rdi) - palignr $3, %xmm7, %xmm8 - movaps %xmm8, 0x60(%rdi) - palignr $3, %xmm6, %xmm7 - movaps %xmm7, 0x50(%rdi) - palignr $3, %xmm5, %xmm6 - movaps %xmm6, 0x40(%rdi) - palignr $3, %xmm4, %xmm5 - movaps %xmm5, 0x30(%rdi) - palignr $3, %xmm3, %xmm4 - movaps %xmm4, 0x20(%rdi) - palignr $3, %xmm2, %xmm3 - movaps %xmm3, 0x10(%rdi) - palignr $3, %xmm1, %xmm2 - movaps %xmm2, (%rdi) - lea 0x80(%rdi), %rdi - jae L(shl_3) - movdqu %xmm0, (%r8) - add $0x80, %rdx - add %rdx, %rdi - add %rdx, %rsi - BRANCH_TO_JMPTBL_ENTRY (L(table_144_bytes_fwd), %rdx, 4) + sub $0x80, %rdx + movaps -0x03(%rsi), %xmm1 + movaps 0x0d(%rsi), %xmm2 + movaps 0x1d(%rsi), %xmm3 + movaps 0x2d(%rsi), %xmm4 + movaps 0x3d(%rsi), %xmm5 + movaps 0x4d(%rsi), %xmm6 + movaps 0x5d(%rsi), %xmm7 + movaps 0x6d(%rsi), %xmm8 + movaps 0x7d(%rsi), %xmm9 + lea 0x80(%rsi), %rsi + palignr $3, %xmm8, %xmm9 + movaps %xmm9, 0x70(%rdi) + palignr $3, %xmm7, %xmm8 + movaps %xmm8, 0x60(%rdi) + palignr $3, %xmm6, %xmm7 + movaps %xmm7, 0x50(%rdi) + palignr $3, %xmm5, %xmm6 + movaps %xmm6, 0x40(%rdi) + palignr $3, %xmm4, %xmm5 + movaps %xmm5, 0x30(%rdi) + palignr $3, %xmm3, %xmm4 + movaps %xmm4, 0x20(%rdi) + palignr $3, %xmm2, %xmm3 + movaps %xmm3, 0x10(%rdi) + palignr $3, %xmm1, %xmm2 + movaps %xmm2, (%rdi) + lea 0x80(%rdi), %rdi + jae L(shl_3) + movdqu %xmm0, (%r8) + add $0x80, %rdx + add %rdx, %rdi + add %rdx, %rsi + BRANCH_TO_JMPTBL_ENTRY (L(table_144_bytes_fwd), %rdx, 4) - .p2align 4 + .p2align 4 L(shl_3_bwd): - movaps -0x03(%rsi), %xmm1 + movaps -0x03(%rsi), %xmm1 - movaps -0x13(%rsi), %xmm2 - palignr $3, %xmm2, %xmm1 - movaps %xmm1, -0x10(%rdi) + movaps -0x13(%rsi), %xmm2 + palignr $3, %xmm2, %xmm1 + movaps %xmm1, -0x10(%rdi) - movaps -0x23(%rsi), %xmm3 - palignr $3, %xmm3, %xmm2 - movaps %xmm2, -0x20(%rdi) + movaps -0x23(%rsi), %xmm3 + palignr $3, %xmm3, %xmm2 + movaps %xmm2, -0x20(%rdi) - movaps -0x33(%rsi), %xmm4 - palignr $3, %xmm4, %xmm3 - movaps %xmm3, -0x30(%rdi) + movaps -0x33(%rsi), %xmm4 + palignr $3, %xmm4, %xmm3 + movaps %xmm3, -0x30(%rdi) - movaps -0x43(%rsi), %xmm5 - palignr $3, %xmm5, %xmm4 - movaps %xmm4, -0x40(%rdi) + movaps -0x43(%rsi), %xmm5 + palignr $3, %xmm5, %xmm4 + movaps %xmm4, -0x40(%rdi) - movaps -0x53(%rsi), %xmm6 - palignr $3, %xmm6, %xmm5 - movaps %xmm5, -0x50(%rdi) + movaps -0x53(%rsi), %xmm6 + palignr $3, %xmm6, %xmm5 + movaps %xmm5, -0x50(%rdi) - movaps -0x63(%rsi), %xmm7 - palignr $3, %xmm7, %xmm6 - movaps %xmm6, -0x60(%rdi) + movaps -0x63(%rsi), %xmm7 + palignr $3, %xmm7, %xmm6 + movaps %xmm6, -0x60(%rdi) - movaps -0x73(%rsi), %xmm8 - palignr $3, %xmm8, %xmm7 - movaps %xmm7, -0x70(%rdi) + movaps -0x73(%rsi), %xmm8 + palignr $3, %xmm8, %xmm7 + movaps %xmm7, -0x70(%rdi) - movaps -0x83(%rsi), %xmm9 - palignr $3, %xmm9, %xmm8 - movaps %xmm8, -0x80(%rdi) + movaps -0x83(%rsi), %xmm9 + palignr $3, %xmm9, %xmm8 + movaps %xmm8, -0x80(%rdi) - sub $0x80, %rdx - lea -0x80(%rdi), %rdi - lea -0x80(%rsi), %rsi - jae L(shl_3_bwd) - movdqu %xmm0, (%r8) - add $0x80, %rdx - sub %rdx, %rdi - sub %rdx, %rsi - BRANCH_TO_JMPTBL_ENTRY (L(table_144_bytes_bwd), %rdx, 4) + sub $0x80, %rdx + lea -0x80(%rdi), %rdi + lea -0x80(%rsi), %rsi + jae L(shl_3_bwd) + movdqu %xmm0, (%r8) + add $0x80, %rdx + sub %rdx, %rdi + sub %rdx, %rsi + BRANCH_TO_JMPTBL_ENTRY (L(table_144_bytes_bwd), %rdx, 4) - .p2align 4 + .p2align 4 L(shl_4): - sub $0x80, %rdx - movaps -0x04(%rsi), %xmm1 - movaps 0x0c(%rsi), %xmm2 - movaps 0x1c(%rsi), %xmm3 - movaps 0x2c(%rsi), %xmm4 - movaps 0x3c(%rsi), %xmm5 - movaps 0x4c(%rsi), %xmm6 - movaps 0x5c(%rsi), %xmm7 - movaps 0x6c(%rsi), %xmm8 - movaps 0x7c(%rsi), %xmm9 - lea 0x80(%rsi), %rsi - palignr $4, %xmm8, %xmm9 - movaps %xmm9, 0x70(%rdi) - palignr $4, %xmm7, %xmm8 - movaps %xmm8, 0x60(%rdi) - palignr $4, %xmm6, %xmm7 - movaps %xmm7, 0x50(%rdi) - palignr $4, %xmm5, %xmm6 - movaps %xmm6, 0x40(%rdi) - palignr $4, %xmm4, %xmm5 - movaps %xmm5, 0x30(%rdi) - palignr $4, %xmm3, %xmm4 - movaps %xmm4, 0x20(%rdi) - palignr $4, %xmm2, %xmm3 - movaps %xmm3, 0x10(%rdi) - palignr $4, %xmm1, %xmm2 - movaps %xmm2, (%rdi) - lea 0x80(%rdi), %rdi - jae L(shl_4) - movdqu %xmm0, (%r8) - add $0x80, %rdx - add %rdx, %rdi - add %rdx, %rsi - BRANCH_TO_JMPTBL_ENTRY (L(table_144_bytes_fwd), %rdx, 4) + sub $0x80, %rdx + movaps -0x04(%rsi), %xmm1 + movaps 0x0c(%rsi), %xmm2 + movaps 0x1c(%rsi), %xmm3 + movaps 0x2c(%rsi), %xmm4 + movaps 0x3c(%rsi), %xmm5 + movaps 0x4c(%rsi), %xmm6 + movaps 0x5c(%rsi), %xmm7 + movaps 0x6c(%rsi), %xmm8 + movaps 0x7c(%rsi), %xmm9 + lea 0x80(%rsi), %rsi + palignr $4, %xmm8, %xmm9 + movaps %xmm9, 0x70(%rdi) + palignr $4, %xmm7, %xmm8 + movaps %xmm8, 0x60(%rdi) + palignr $4, %xmm6, %xmm7 + movaps %xmm7, 0x50(%rdi) + palignr $4, %xmm5, %xmm6 + movaps %xmm6, 0x40(%rdi) + palignr $4, %xmm4, %xmm5 + movaps %xmm5, 0x30(%rdi) + palignr $4, %xmm3, %xmm4 + movaps %xmm4, 0x20(%rdi) + palignr $4, %xmm2, %xmm3 + movaps %xmm3, 0x10(%rdi) + palignr $4, %xmm1, %xmm2 + movaps %xmm2, (%rdi) + lea 0x80(%rdi), %rdi + jae L(shl_4) + movdqu %xmm0, (%r8) + add $0x80, %rdx + add %rdx, %rdi + add %rdx, %rsi + BRANCH_TO_JMPTBL_ENTRY (L(table_144_bytes_fwd), %rdx, 4) - .p2align 4 + .p2align 4 L(shl_4_bwd): - movaps -0x04(%rsi), %xmm1 + movaps -0x04(%rsi), %xmm1 - movaps -0x14(%rsi), %xmm2 - palignr $4, %xmm2, %xmm1 - movaps %xmm1, -0x10(%rdi) + movaps -0x14(%rsi), %xmm2 + palignr $4, %xmm2, %xmm1 + movaps %xmm1, -0x10(%rdi) - movaps -0x24(%rsi), %xmm3 - palignr $4, %xmm3, %xmm2 - movaps %xmm2, -0x20(%rdi) + movaps -0x24(%rsi), %xmm3 + palignr $4, %xmm3, %xmm2 + movaps %xmm2, -0x20(%rdi) - movaps -0x34(%rsi), %xmm4 - palignr $4, %xmm4, %xmm3 - movaps %xmm3, -0x30(%rdi) + movaps -0x34(%rsi), %xmm4 + palignr $4, %xmm4, %xmm3 + movaps %xmm3, -0x30(%rdi) - movaps -0x44(%rsi), %xmm5 - palignr $4, %xmm5, %xmm4 - movaps %xmm4, -0x40(%rdi) + movaps -0x44(%rsi), %xmm5 + palignr $4, %xmm5, %xmm4 + movaps %xmm4, -0x40(%rdi) - movaps -0x54(%rsi), %xmm6 - palignr $4, %xmm6, %xmm5 - movaps %xmm5, -0x50(%rdi) + movaps -0x54(%rsi), %xmm6 + palignr $4, %xmm6, %xmm5 + movaps %xmm5, -0x50(%rdi) - movaps -0x64(%rsi), %xmm7 - palignr $4, %xmm7, %xmm6 - movaps %xmm6, -0x60(%rdi) + movaps -0x64(%rsi), %xmm7 + palignr $4, %xmm7, %xmm6 + movaps %xmm6, -0x60(%rdi) - movaps -0x74(%rsi), %xmm8 - palignr $4, %xmm8, %xmm7 - movaps %xmm7, -0x70(%rdi) + movaps -0x74(%rsi), %xmm8 + palignr $4, %xmm8, %xmm7 + movaps %xmm7, -0x70(%rdi) - movaps -0x84(%rsi), %xmm9 - palignr $4, %xmm9, %xmm8 - movaps %xmm8, -0x80(%rdi) + movaps -0x84(%rsi), %xmm9 + palignr $4, %xmm9, %xmm8 + movaps %xmm8, -0x80(%rdi) - sub $0x80, %rdx - lea -0x80(%rdi), %rdi - lea -0x80(%rsi), %rsi - jae L(shl_4_bwd) - movdqu %xmm0, (%r8) - add $0x80, %rdx - sub %rdx, %rdi - sub %rdx, %rsi - BRANCH_TO_JMPTBL_ENTRY (L(table_144_bytes_bwd), %rdx, 4) + sub $0x80, %rdx + lea -0x80(%rdi), %rdi + lea -0x80(%rsi), %rsi + jae L(shl_4_bwd) + movdqu %xmm0, (%r8) + add $0x80, %rdx + sub %rdx, %rdi + sub %rdx, %rsi + BRANCH_TO_JMPTBL_ENTRY (L(table_144_bytes_bwd), %rdx, 4) - .p2align 4 + .p2align 4 L(shl_5): - sub $0x80, %rdx - movaps -0x05(%rsi), %xmm1 - movaps 0x0b(%rsi), %xmm2 - movaps 0x1b(%rsi), %xmm3 - movaps 0x2b(%rsi), %xmm4 - movaps 0x3b(%rsi), %xmm5 - movaps 0x4b(%rsi), %xmm6 - movaps 0x5b(%rsi), %xmm7 - movaps 0x6b(%rsi), %xmm8 - movaps 0x7b(%rsi), %xmm9 - lea 0x80(%rsi), %rsi - palignr $5, %xmm8, %xmm9 - movaps %xmm9, 0x70(%rdi) - palignr $5, %xmm7, %xmm8 - movaps %xmm8, 0x60(%rdi) - palignr $5, %xmm6, %xmm7 - movaps %xmm7, 0x50(%rdi) - palignr $5, %xmm5, %xmm6 - movaps %xmm6, 0x40(%rdi) - palignr $5, %xmm4, %xmm5 - movaps %xmm5, 0x30(%rdi) - palignr $5, %xmm3, %xmm4 - movaps %xmm4, 0x20(%rdi) - palignr $5, %xmm2, %xmm3 - movaps %xmm3, 0x10(%rdi) - palignr $5, %xmm1, %xmm2 - movaps %xmm2, (%rdi) - lea 0x80(%rdi), %rdi - jae L(shl_5) - movdqu %xmm0, (%r8) - add $0x80, %rdx - add %rdx, %rdi - add %rdx, %rsi - BRANCH_TO_JMPTBL_ENTRY (L(table_144_bytes_fwd), %rdx, 4) + sub $0x80, %rdx + movaps -0x05(%rsi), %xmm1 + movaps 0x0b(%rsi), %xmm2 + movaps 0x1b(%rsi), %xmm3 + movaps 0x2b(%rsi), %xmm4 + movaps 0x3b(%rsi), %xmm5 + movaps 0x4b(%rsi), %xmm6 + movaps 0x5b(%rsi), %xmm7 + movaps 0x6b(%rsi), %xmm8 + movaps 0x7b(%rsi), %xmm9 + lea 0x80(%rsi), %rsi + palignr $5, %xmm8, %xmm9 + movaps %xmm9, 0x70(%rdi) + palignr $5, %xmm7, %xmm8 + movaps %xmm8, 0x60(%rdi) + palignr $5, %xmm6, %xmm7 + movaps %xmm7, 0x50(%rdi) + palignr $5, %xmm5, %xmm6 + movaps %xmm6, 0x40(%rdi) + palignr $5, %xmm4, %xmm5 + movaps %xmm5, 0x30(%rdi) + palignr $5, %xmm3, %xmm4 + movaps %xmm4, 0x20(%rdi) + palignr $5, %xmm2, %xmm3 + movaps %xmm3, 0x10(%rdi) + palignr $5, %xmm1, %xmm2 + movaps %xmm2, (%rdi) + lea 0x80(%rdi), %rdi + jae L(shl_5) + movdqu %xmm0, (%r8) + add $0x80, %rdx + add %rdx, %rdi + add %rdx, %rsi + BRANCH_TO_JMPTBL_ENTRY (L(table_144_bytes_fwd), %rdx, 4) - .p2align 4 + .p2align 4 L(shl_5_bwd): - movaps -0x05(%rsi), %xmm1 + movaps -0x05(%rsi), %xmm1 - movaps -0x15(%rsi), %xmm2 - palignr $5, %xmm2, %xmm1 - movaps %xmm1, -0x10(%rdi) + movaps -0x15(%rsi), %xmm2 + palignr $5, %xmm2, %xmm1 + movaps %xmm1, -0x10(%rdi) - movaps -0x25(%rsi), %xmm3 - palignr $5, %xmm3, %xmm2 - movaps %xmm2, -0x20(%rdi) + movaps -0x25(%rsi), %xmm3 + palignr $5, %xmm3, %xmm2 + movaps %xmm2, -0x20(%rdi) - movaps -0x35(%rsi), %xmm4 - palignr $5, %xmm4, %xmm3 - movaps %xmm3, -0x30(%rdi) + movaps -0x35(%rsi), %xmm4 + palignr $5, %xmm4, %xmm3 + movaps %xmm3, -0x30(%rdi) - movaps -0x45(%rsi), %xmm5 - palignr $5, %xmm5, %xmm4 - movaps %xmm4, -0x40(%rdi) + movaps -0x45(%rsi), %xmm5 + palignr $5, %xmm5, %xmm4 + movaps %xmm4, -0x40(%rdi) - movaps -0x55(%rsi), %xmm6 - palignr $5, %xmm6, %xmm5 - movaps %xmm5, -0x50(%rdi) + movaps -0x55(%rsi), %xmm6 + palignr $5, %xmm6, %xmm5 + movaps %xmm5, -0x50(%rdi) - movaps -0x65(%rsi), %xmm7 - palignr $5, %xmm7, %xmm6 - movaps %xmm6, -0x60(%rdi) + movaps -0x65(%rsi), %xmm7 + palignr $5, %xmm7, %xmm6 + movaps %xmm6, -0x60(%rdi) - movaps -0x75(%rsi), %xmm8 - palignr $5, %xmm8, %xmm7 - movaps %xmm7, -0x70(%rdi) + movaps -0x75(%rsi), %xmm8 + palignr $5, %xmm8, %xmm7 + movaps %xmm7, -0x70(%rdi) - movaps -0x85(%rsi), %xmm9 - palignr $5, %xmm9, %xmm8 - movaps %xmm8, -0x80(%rdi) + movaps -0x85(%rsi), %xmm9 + palignr $5, %xmm9, %xmm8 + movaps %xmm8, -0x80(%rdi) - sub $0x80, %rdx - lea -0x80(%rdi), %rdi - lea -0x80(%rsi), %rsi - jae L(shl_5_bwd) - movdqu %xmm0, (%r8) - add $0x80, %rdx - sub %rdx, %rdi - sub %rdx, %rsi - BRANCH_TO_JMPTBL_ENTRY (L(table_144_bytes_bwd), %rdx, 4) + sub $0x80, %rdx + lea -0x80(%rdi), %rdi + lea -0x80(%rsi), %rsi + jae L(shl_5_bwd) + movdqu %xmm0, (%r8) + add $0x80, %rdx + sub %rdx, %rdi + sub %rdx, %rsi + BRANCH_TO_JMPTBL_ENTRY (L(table_144_bytes_bwd), %rdx, 4) - .p2align 4 + .p2align 4 L(shl_6): - sub $0x80, %rdx - movaps -0x06(%rsi), %xmm1 - movaps 0x0a(%rsi), %xmm2 - movaps 0x1a(%rsi), %xmm3 - movaps 0x2a(%rsi), %xmm4 - movaps 0x3a(%rsi), %xmm5 - movaps 0x4a(%rsi), %xmm6 - movaps 0x5a(%rsi), %xmm7 - movaps 0x6a(%rsi), %xmm8 - movaps 0x7a(%rsi), %xmm9 - lea 0x80(%rsi), %rsi - palignr $6, %xmm8, %xmm9 - movaps %xmm9, 0x70(%rdi) - palignr $6, %xmm7, %xmm8 - movaps %xmm8, 0x60(%rdi) - palignr $6, %xmm6, %xmm7 - movaps %xmm7, 0x50(%rdi) - palignr $6, %xmm5, %xmm6 - movaps %xmm6, 0x40(%rdi) - palignr $6, %xmm4, %xmm5 - movaps %xmm5, 0x30(%rdi) - palignr $6, %xmm3, %xmm4 - movaps %xmm4, 0x20(%rdi) - palignr $6, %xmm2, %xmm3 - movaps %xmm3, 0x10(%rdi) - palignr $6, %xmm1, %xmm2 - movaps %xmm2, (%rdi) - lea 0x80(%rdi), %rdi - jae L(shl_6) - movdqu %xmm0, (%r8) - add $0x80, %rdx - add %rdx, %rdi - add %rdx, %rsi - BRANCH_TO_JMPTBL_ENTRY (L(table_144_bytes_fwd), %rdx, 4) + sub $0x80, %rdx + movaps -0x06(%rsi), %xmm1 + movaps 0x0a(%rsi), %xmm2 + movaps 0x1a(%rsi), %xmm3 + movaps 0x2a(%rsi), %xmm4 + movaps 0x3a(%rsi), %xmm5 + movaps 0x4a(%rsi), %xmm6 + movaps 0x5a(%rsi), %xmm7 + movaps 0x6a(%rsi), %xmm8 + movaps 0x7a(%rsi), %xmm9 + lea 0x80(%rsi), %rsi + palignr $6, %xmm8, %xmm9 + movaps %xmm9, 0x70(%rdi) + palignr $6, %xmm7, %xmm8 + movaps %xmm8, 0x60(%rdi) + palignr $6, %xmm6, %xmm7 + movaps %xmm7, 0x50(%rdi) + palignr $6, %xmm5, %xmm6 + movaps %xmm6, 0x40(%rdi) + palignr $6, %xmm4, %xmm5 + movaps %xmm5, 0x30(%rdi) + palignr $6, %xmm3, %xmm4 + movaps %xmm4, 0x20(%rdi) + palignr $6, %xmm2, %xmm3 + movaps %xmm3, 0x10(%rdi) + palignr $6, %xmm1, %xmm2 + movaps %xmm2, (%rdi) + lea 0x80(%rdi), %rdi + jae L(shl_6) + movdqu %xmm0, (%r8) + add $0x80, %rdx + add %rdx, %rdi + add %rdx, %rsi + BRANCH_TO_JMPTBL_ENTRY (L(table_144_bytes_fwd), %rdx, 4) - .p2align 4 + .p2align 4 L(shl_6_bwd): - movaps -0x06(%rsi), %xmm1 + movaps -0x06(%rsi), %xmm1 - movaps -0x16(%rsi), %xmm2 - palignr $6, %xmm2, %xmm1 - movaps %xmm1, -0x10(%rdi) + movaps -0x16(%rsi), %xmm2 + palignr $6, %xmm2, %xmm1 + movaps %xmm1, -0x10(%rdi) - movaps -0x26(%rsi), %xmm3 - palignr $6, %xmm3, %xmm2 - movaps %xmm2, -0x20(%rdi) + movaps -0x26(%rsi), %xmm3 + palignr $6, %xmm3, %xmm2 + movaps %xmm2, -0x20(%rdi) - movaps -0x36(%rsi), %xmm4 - palignr $6, %xmm4, %xmm3 - movaps %xmm3, -0x30(%rdi) + movaps -0x36(%rsi), %xmm4 + palignr $6, %xmm4, %xmm3 + movaps %xmm3, -0x30(%rdi) - movaps -0x46(%rsi), %xmm5 - palignr $6, %xmm5, %xmm4 - movaps %xmm4, -0x40(%rdi) + movaps -0x46(%rsi), %xmm5 + palignr $6, %xmm5, %xmm4 + movaps %xmm4, -0x40(%rdi) - movaps -0x56(%rsi), %xmm6 - palignr $6, %xmm6, %xmm5 - movaps %xmm5, -0x50(%rdi) + movaps -0x56(%rsi), %xmm6 + palignr $6, %xmm6, %xmm5 + movaps %xmm5, -0x50(%rdi) - movaps -0x66(%rsi), %xmm7 - palignr $6, %xmm7, %xmm6 - movaps %xmm6, -0x60(%rdi) + movaps -0x66(%rsi), %xmm7 + palignr $6, %xmm7, %xmm6 + movaps %xmm6, -0x60(%rdi) - movaps -0x76(%rsi), %xmm8 - palignr $6, %xmm8, %xmm7 - movaps %xmm7, -0x70(%rdi) + movaps -0x76(%rsi), %xmm8 + palignr $6, %xmm8, %xmm7 + movaps %xmm7, -0x70(%rdi) - movaps -0x86(%rsi), %xmm9 - palignr $6, %xmm9, %xmm8 - movaps %xmm8, -0x80(%rdi) + movaps -0x86(%rsi), %xmm9 + palignr $6, %xmm9, %xmm8 + movaps %xmm8, -0x80(%rdi) - sub $0x80, %rdx - lea -0x80(%rdi), %rdi - lea -0x80(%rsi), %rsi - jae L(shl_6_bwd) - movdqu %xmm0, (%r8) - add $0x80, %rdx - sub %rdx, %rdi - sub %rdx, %rsi - BRANCH_TO_JMPTBL_ENTRY (L(table_144_bytes_bwd), %rdx, 4) + sub $0x80, %rdx + lea -0x80(%rdi), %rdi + lea -0x80(%rsi), %rsi + jae L(shl_6_bwd) + movdqu %xmm0, (%r8) + add $0x80, %rdx + sub %rdx, %rdi + sub %rdx, %rsi + BRANCH_TO_JMPTBL_ENTRY (L(table_144_bytes_bwd), %rdx, 4) - .p2align 4 + .p2align 4 L(shl_7): - sub $0x80, %rdx - movaps -0x07(%rsi), %xmm1 - movaps 0x09(%rsi), %xmm2 - movaps 0x19(%rsi), %xmm3 - movaps 0x29(%rsi), %xmm4 - movaps 0x39(%rsi), %xmm5 - movaps 0x49(%rsi), %xmm6 - movaps 0x59(%rsi), %xmm7 - movaps 0x69(%rsi), %xmm8 - movaps 0x79(%rsi), %xmm9 - lea 0x80(%rsi), %rsi - palignr $7, %xmm8, %xmm9 - movaps %xmm9, 0x70(%rdi) - palignr $7, %xmm7, %xmm8 - movaps %xmm8, 0x60(%rdi) - palignr $7, %xmm6, %xmm7 - movaps %xmm7, 0x50(%rdi) - palignr $7, %xmm5, %xmm6 - movaps %xmm6, 0x40(%rdi) - palignr $7, %xmm4, %xmm5 - movaps %xmm5, 0x30(%rdi) - palignr $7, %xmm3, %xmm4 - movaps %xmm4, 0x20(%rdi) - palignr $7, %xmm2, %xmm3 - movaps %xmm3, 0x10(%rdi) - palignr $7, %xmm1, %xmm2 - movaps %xmm2, (%rdi) - lea 0x80(%rdi), %rdi - jae L(shl_7) - movdqu %xmm0, (%r8) - add $0x80, %rdx - add %rdx, %rdi - add %rdx, %rsi - BRANCH_TO_JMPTBL_ENTRY (L(table_144_bytes_fwd), %rdx, 4) + sub $0x80, %rdx + movaps -0x07(%rsi), %xmm1 + movaps 0x09(%rsi), %xmm2 + movaps 0x19(%rsi), %xmm3 + movaps 0x29(%rsi), %xmm4 + movaps 0x39(%rsi), %xmm5 + movaps 0x49(%rsi), %xmm6 + movaps 0x59(%rsi), %xmm7 + movaps 0x69(%rsi), %xmm8 + movaps 0x79(%rsi), %xmm9 + lea 0x80(%rsi), %rsi + palignr $7, %xmm8, %xmm9 + movaps %xmm9, 0x70(%rdi) + palignr $7, %xmm7, %xmm8 + movaps %xmm8, 0x60(%rdi) + palignr $7, %xmm6, %xmm7 + movaps %xmm7, 0x50(%rdi) + palignr $7, %xmm5, %xmm6 + movaps %xmm6, 0x40(%rdi) + palignr $7, %xmm4, %xmm5 + movaps %xmm5, 0x30(%rdi) + palignr $7, %xmm3, %xmm4 + movaps %xmm4, 0x20(%rdi) + palignr $7, %xmm2, %xmm3 + movaps %xmm3, 0x10(%rdi) + palignr $7, %xmm1, %xmm2 + movaps %xmm2, (%rdi) + lea 0x80(%rdi), %rdi + jae L(shl_7) + movdqu %xmm0, (%r8) + add $0x80, %rdx + add %rdx, %rdi + add %rdx, %rsi + BRANCH_TO_JMPTBL_ENTRY (L(table_144_bytes_fwd), %rdx, 4) - .p2align 4 + .p2align 4 L(shl_7_bwd): - movaps -0x07(%rsi), %xmm1 + movaps -0x07(%rsi), %xmm1 - movaps -0x17(%rsi), %xmm2 - palignr $7, %xmm2, %xmm1 - movaps %xmm1, -0x10(%rdi) + movaps -0x17(%rsi), %xmm2 + palignr $7, %xmm2, %xmm1 + movaps %xmm1, -0x10(%rdi) - movaps -0x27(%rsi), %xmm3 - palignr $7, %xmm3, %xmm2 - movaps %xmm2, -0x20(%rdi) + movaps -0x27(%rsi), %xmm3 + palignr $7, %xmm3, %xmm2 + movaps %xmm2, -0x20(%rdi) - movaps -0x37(%rsi), %xmm4 - palignr $7, %xmm4, %xmm3 - movaps %xmm3, -0x30(%rdi) + movaps -0x37(%rsi), %xmm4 + palignr $7, %xmm4, %xmm3 + movaps %xmm3, -0x30(%rdi) - movaps -0x47(%rsi), %xmm5 - palignr $7, %xmm5, %xmm4 - movaps %xmm4, -0x40(%rdi) + movaps -0x47(%rsi), %xmm5 + palignr $7, %xmm5, %xmm4 + movaps %xmm4, -0x40(%rdi) - movaps -0x57(%rsi), %xmm6 - palignr $7, %xmm6, %xmm5 - movaps %xmm5, -0x50(%rdi) + movaps -0x57(%rsi), %xmm6 + palignr $7, %xmm6, %xmm5 + movaps %xmm5, -0x50(%rdi) - movaps -0x67(%rsi), %xmm7 - palignr $7, %xmm7, %xmm6 - movaps %xmm6, -0x60(%rdi) + movaps -0x67(%rsi), %xmm7 + palignr $7, %xmm7, %xmm6 + movaps %xmm6, -0x60(%rdi) - movaps -0x77(%rsi), %xmm8 - palignr $7, %xmm8, %xmm7 - movaps %xmm7, -0x70(%rdi) + movaps -0x77(%rsi), %xmm8 + palignr $7, %xmm8, %xmm7 + movaps %xmm7, -0x70(%rdi) - movaps -0x87(%rsi), %xmm9 - palignr $7, %xmm9, %xmm8 - movaps %xmm8, -0x80(%rdi) + movaps -0x87(%rsi), %xmm9 + palignr $7, %xmm9, %xmm8 + movaps %xmm8, -0x80(%rdi) - sub $0x80, %rdx - lea -0x80(%rdi), %rdi - lea -0x80(%rsi), %rsi - jae L(shl_7_bwd) - movdqu %xmm0, (%r8) - add $0x80, %rdx - sub %rdx, %rdi - sub %rdx, %rsi - BRANCH_TO_JMPTBL_ENTRY (L(table_144_bytes_bwd), %rdx, 4) + sub $0x80, %rdx + lea -0x80(%rdi), %rdi + lea -0x80(%rsi), %rsi + jae L(shl_7_bwd) + movdqu %xmm0, (%r8) + add $0x80, %rdx + sub %rdx, %rdi + sub %rdx, %rsi + BRANCH_TO_JMPTBL_ENTRY (L(table_144_bytes_bwd), %rdx, 4) - .p2align 4 + .p2align 4 L(shl_8): - sub $0x80, %rdx - movaps -0x08(%rsi), %xmm1 - movaps 0x08(%rsi), %xmm2 - movaps 0x18(%rsi), %xmm3 - movaps 0x28(%rsi), %xmm4 - movaps 0x38(%rsi), %xmm5 - movaps 0x48(%rsi), %xmm6 - movaps 0x58(%rsi), %xmm7 - movaps 0x68(%rsi), %xmm8 - movaps 0x78(%rsi), %xmm9 - lea 0x80(%rsi), %rsi - palignr $8, %xmm8, %xmm9 - movaps %xmm9, 0x70(%rdi) - palignr $8, %xmm7, %xmm8 - movaps %xmm8, 0x60(%rdi) - palignr $8, %xmm6, %xmm7 - movaps %xmm7, 0x50(%rdi) - palignr $8, %xmm5, %xmm6 - movaps %xmm6, 0x40(%rdi) - palignr $8, %xmm4, %xmm5 - movaps %xmm5, 0x30(%rdi) - palignr $8, %xmm3, %xmm4 - movaps %xmm4, 0x20(%rdi) - palignr $8, %xmm2, %xmm3 - movaps %xmm3, 0x10(%rdi) - palignr $8, %xmm1, %xmm2 - movaps %xmm2, (%rdi) - lea 0x80(%rdi), %rdi - jae L(shl_8) - movdqu %xmm0, (%r8) - add $0x80, %rdx - add %rdx, %rdi - add %rdx, %rsi - BRANCH_TO_JMPTBL_ENTRY (L(table_144_bytes_fwd), %rdx, 4) + sub $0x80, %rdx + movaps -0x08(%rsi), %xmm1 + movaps 0x08(%rsi), %xmm2 + movaps 0x18(%rsi), %xmm3 + movaps 0x28(%rsi), %xmm4 + movaps 0x38(%rsi), %xmm5 + movaps 0x48(%rsi), %xmm6 + movaps 0x58(%rsi), %xmm7 + movaps 0x68(%rsi), %xmm8 + movaps 0x78(%rsi), %xmm9 + lea 0x80(%rsi), %rsi + palignr $8, %xmm8, %xmm9 + movaps %xmm9, 0x70(%rdi) + palignr $8, %xmm7, %xmm8 + movaps %xmm8, 0x60(%rdi) + palignr $8, %xmm6, %xmm7 + movaps %xmm7, 0x50(%rdi) + palignr $8, %xmm5, %xmm6 + movaps %xmm6, 0x40(%rdi) + palignr $8, %xmm4, %xmm5 + movaps %xmm5, 0x30(%rdi) + palignr $8, %xmm3, %xmm4 + movaps %xmm4, 0x20(%rdi) + palignr $8, %xmm2, %xmm3 + movaps %xmm3, 0x10(%rdi) + palignr $8, %xmm1, %xmm2 + movaps %xmm2, (%rdi) + lea 0x80(%rdi), %rdi + jae L(shl_8) + movdqu %xmm0, (%r8) + add $0x80, %rdx + add %rdx, %rdi + add %rdx, %rsi + BRANCH_TO_JMPTBL_ENTRY (L(table_144_bytes_fwd), %rdx, 4) - .p2align 4 + .p2align 4 L(shl_8_bwd): - movaps -0x08(%rsi), %xmm1 + movaps -0x08(%rsi), %xmm1 - movaps -0x18(%rsi), %xmm2 - palignr $8, %xmm2, %xmm1 - movaps %xmm1, -0x10(%rdi) + movaps -0x18(%rsi), %xmm2 + palignr $8, %xmm2, %xmm1 + movaps %xmm1, -0x10(%rdi) - movaps -0x28(%rsi), %xmm3 - palignr $8, %xmm3, %xmm2 - movaps %xmm2, -0x20(%rdi) + movaps -0x28(%rsi), %xmm3 + palignr $8, %xmm3, %xmm2 + movaps %xmm2, -0x20(%rdi) - movaps -0x38(%rsi), %xmm4 - palignr $8, %xmm4, %xmm3 - movaps %xmm3, -0x30(%rdi) + movaps -0x38(%rsi), %xmm4 + palignr $8, %xmm4, %xmm3 + movaps %xmm3, -0x30(%rdi) - movaps -0x48(%rsi), %xmm5 - palignr $8, %xmm5, %xmm4 - movaps %xmm4, -0x40(%rdi) + movaps -0x48(%rsi), %xmm5 + palignr $8, %xmm5, %xmm4 + movaps %xmm4, -0x40(%rdi) - movaps -0x58(%rsi), %xmm6 - palignr $8, %xmm6, %xmm5 - movaps %xmm5, -0x50(%rdi) + movaps -0x58(%rsi), %xmm6 + palignr $8, %xmm6, %xmm5 + movaps %xmm5, -0x50(%rdi) - movaps -0x68(%rsi), %xmm7 - palignr $8, %xmm7, %xmm6 - movaps %xmm6, -0x60(%rdi) + movaps -0x68(%rsi), %xmm7 + palignr $8, %xmm7, %xmm6 + movaps %xmm6, -0x60(%rdi) - movaps -0x78(%rsi), %xmm8 - palignr $8, %xmm8, %xmm7 - movaps %xmm7, -0x70(%rdi) + movaps -0x78(%rsi), %xmm8 + palignr $8, %xmm8, %xmm7 + movaps %xmm7, -0x70(%rdi) - movaps -0x88(%rsi), %xmm9 - palignr $8, %xmm9, %xmm8 - movaps %xmm8, -0x80(%rdi) + movaps -0x88(%rsi), %xmm9 + palignr $8, %xmm9, %xmm8 + movaps %xmm8, -0x80(%rdi) - sub $0x80, %rdx - lea -0x80(%rdi), %rdi - lea -0x80(%rsi), %rsi - jae L(shl_8_bwd) + sub $0x80, %rdx + lea -0x80(%rdi), %rdi + lea -0x80(%rsi), %rsi + jae L(shl_8_bwd) L(shl_8_end_bwd): - movdqu %xmm0, (%r8) - add $0x80, %rdx - sub %rdx, %rdi - sub %rdx, %rsi - BRANCH_TO_JMPTBL_ENTRY (L(table_144_bytes_bwd), %rdx, 4) + movdqu %xmm0, (%r8) + add $0x80, %rdx + sub %rdx, %rdi + sub %rdx, %rsi + BRANCH_TO_JMPTBL_ENTRY (L(table_144_bytes_bwd), %rdx, 4) - .p2align 4 + .p2align 4 L(shl_9): - sub $0x80, %rdx - movaps -0x09(%rsi), %xmm1 - movaps 0x07(%rsi), %xmm2 - movaps 0x17(%rsi), %xmm3 - movaps 0x27(%rsi), %xmm4 - movaps 0x37(%rsi), %xmm5 - movaps 0x47(%rsi), %xmm6 - movaps 0x57(%rsi), %xmm7 - movaps 0x67(%rsi), %xmm8 - movaps 0x77(%rsi), %xmm9 - lea 0x80(%rsi), %rsi - palignr $9, %xmm8, %xmm9 - movaps %xmm9, 0x70(%rdi) - palignr $9, %xmm7, %xmm8 - movaps %xmm8, 0x60(%rdi) - palignr $9, %xmm6, %xmm7 - movaps %xmm7, 0x50(%rdi) - palignr $9, %xmm5, %xmm6 - movaps %xmm6, 0x40(%rdi) - palignr $9, %xmm4, %xmm5 - movaps %xmm5, 0x30(%rdi) - palignr $9, %xmm3, %xmm4 - movaps %xmm4, 0x20(%rdi) - palignr $9, %xmm2, %xmm3 - movaps %xmm3, 0x10(%rdi) - palignr $9, %xmm1, %xmm2 - movaps %xmm2, (%rdi) - lea 0x80(%rdi), %rdi - jae L(shl_9) - movdqu %xmm0, (%r8) - add $0x80, %rdx - add %rdx, %rdi - add %rdx, %rsi - BRANCH_TO_JMPTBL_ENTRY (L(table_144_bytes_fwd), %rdx, 4) + sub $0x80, %rdx + movaps -0x09(%rsi), %xmm1 + movaps 0x07(%rsi), %xmm2 + movaps 0x17(%rsi), %xmm3 + movaps 0x27(%rsi), %xmm4 + movaps 0x37(%rsi), %xmm5 + movaps 0x47(%rsi), %xmm6 + movaps 0x57(%rsi), %xmm7 + movaps 0x67(%rsi), %xmm8 + movaps 0x77(%rsi), %xmm9 + lea 0x80(%rsi), %rsi + palignr $9, %xmm8, %xmm9 + movaps %xmm9, 0x70(%rdi) + palignr $9, %xmm7, %xmm8 + movaps %xmm8, 0x60(%rdi) + palignr $9, %xmm6, %xmm7 + movaps %xmm7, 0x50(%rdi) + palignr $9, %xmm5, %xmm6 + movaps %xmm6, 0x40(%rdi) + palignr $9, %xmm4, %xmm5 + movaps %xmm5, 0x30(%rdi) + palignr $9, %xmm3, %xmm4 + movaps %xmm4, 0x20(%rdi) + palignr $9, %xmm2, %xmm3 + movaps %xmm3, 0x10(%rdi) + palignr $9, %xmm1, %xmm2 + movaps %xmm2, (%rdi) + lea 0x80(%rdi), %rdi + jae L(shl_9) + movdqu %xmm0, (%r8) + add $0x80, %rdx + add %rdx, %rdi + add %rdx, %rsi + BRANCH_TO_JMPTBL_ENTRY (L(table_144_bytes_fwd), %rdx, 4) - .p2align 4 + .p2align 4 L(shl_9_bwd): - movaps -0x09(%rsi), %xmm1 + movaps -0x09(%rsi), %xmm1 - movaps -0x19(%rsi), %xmm2 - palignr $9, %xmm2, %xmm1 - movaps %xmm1, -0x10(%rdi) + movaps -0x19(%rsi), %xmm2 + palignr $9, %xmm2, %xmm1 + movaps %xmm1, -0x10(%rdi) - movaps -0x29(%rsi), %xmm3 - palignr $9, %xmm3, %xmm2 - movaps %xmm2, -0x20(%rdi) + movaps -0x29(%rsi), %xmm3 + palignr $9, %xmm3, %xmm2 + movaps %xmm2, -0x20(%rdi) - movaps -0x39(%rsi), %xmm4 - palignr $9, %xmm4, %xmm3 - movaps %xmm3, -0x30(%rdi) + movaps -0x39(%rsi), %xmm4 + palignr $9, %xmm4, %xmm3 + movaps %xmm3, -0x30(%rdi) - movaps -0x49(%rsi), %xmm5 - palignr $9, %xmm5, %xmm4 - movaps %xmm4, -0x40(%rdi) + movaps -0x49(%rsi), %xmm5 + palignr $9, %xmm5, %xmm4 + movaps %xmm4, -0x40(%rdi) - movaps -0x59(%rsi), %xmm6 - palignr $9, %xmm6, %xmm5 - movaps %xmm5, -0x50(%rdi) + movaps -0x59(%rsi), %xmm6 + palignr $9, %xmm6, %xmm5 + movaps %xmm5, -0x50(%rdi) - movaps -0x69(%rsi), %xmm7 - palignr $9, %xmm7, %xmm6 - movaps %xmm6, -0x60(%rdi) + movaps -0x69(%rsi), %xmm7 + palignr $9, %xmm7, %xmm6 + movaps %xmm6, -0x60(%rdi) - movaps -0x79(%rsi), %xmm8 - palignr $9, %xmm8, %xmm7 - movaps %xmm7, -0x70(%rdi) + movaps -0x79(%rsi), %xmm8 + palignr $9, %xmm8, %xmm7 + movaps %xmm7, -0x70(%rdi) - movaps -0x89(%rsi), %xmm9 - palignr $9, %xmm9, %xmm8 - movaps %xmm8, -0x80(%rdi) + movaps -0x89(%rsi), %xmm9 + palignr $9, %xmm9, %xmm8 + movaps %xmm8, -0x80(%rdi) - sub $0x80, %rdx - lea -0x80(%rdi), %rdi - lea -0x80(%rsi), %rsi - jae L(shl_9_bwd) - movdqu %xmm0, (%r8) - add $0x80, %rdx - sub %rdx, %rdi - sub %rdx, %rsi - BRANCH_TO_JMPTBL_ENTRY (L(table_144_bytes_bwd), %rdx, 4) + sub $0x80, %rdx + lea -0x80(%rdi), %rdi + lea -0x80(%rsi), %rsi + jae L(shl_9_bwd) + movdqu %xmm0, (%r8) + add $0x80, %rdx + sub %rdx, %rdi + sub %rdx, %rsi + BRANCH_TO_JMPTBL_ENTRY (L(table_144_bytes_bwd), %rdx, 4) - .p2align 4 + .p2align 4 L(shl_10): - sub $0x80, %rdx - movaps -0x0a(%rsi), %xmm1 - movaps 0x06(%rsi), %xmm2 - movaps 0x16(%rsi), %xmm3 - movaps 0x26(%rsi), %xmm4 - movaps 0x36(%rsi), %xmm5 - movaps 0x46(%rsi), %xmm6 - movaps 0x56(%rsi), %xmm7 - movaps 0x66(%rsi), %xmm8 - movaps 0x76(%rsi), %xmm9 - lea 0x80(%rsi), %rsi - palignr $10, %xmm8, %xmm9 - movaps %xmm9, 0x70(%rdi) - palignr $10, %xmm7, %xmm8 - movaps %xmm8, 0x60(%rdi) - palignr $10, %xmm6, %xmm7 - movaps %xmm7, 0x50(%rdi) - palignr $10, %xmm5, %xmm6 - movaps %xmm6, 0x40(%rdi) - palignr $10, %xmm4, %xmm5 - movaps %xmm5, 0x30(%rdi) - palignr $10, %xmm3, %xmm4 - movaps %xmm4, 0x20(%rdi) - palignr $10, %xmm2, %xmm3 - movaps %xmm3, 0x10(%rdi) - palignr $10, %xmm1, %xmm2 - movaps %xmm2, (%rdi) - lea 0x80(%rdi), %rdi - jae L(shl_10) - movdqu %xmm0, (%r8) - add $0x80, %rdx - add %rdx, %rdi - add %rdx, %rsi - BRANCH_TO_JMPTBL_ENTRY (L(table_144_bytes_fwd), %rdx, 4) + sub $0x80, %rdx + movaps -0x0a(%rsi), %xmm1 + movaps 0x06(%rsi), %xmm2 + movaps 0x16(%rsi), %xmm3 + movaps 0x26(%rsi), %xmm4 + movaps 0x36(%rsi), %xmm5 + movaps 0x46(%rsi), %xmm6 + movaps 0x56(%rsi), %xmm7 + movaps 0x66(%rsi), %xmm8 + movaps 0x76(%rsi), %xmm9 + lea 0x80(%rsi), %rsi + palignr $10, %xmm8, %xmm9 + movaps %xmm9, 0x70(%rdi) + palignr $10, %xmm7, %xmm8 + movaps %xmm8, 0x60(%rdi) + palignr $10, %xmm6, %xmm7 + movaps %xmm7, 0x50(%rdi) + palignr $10, %xmm5, %xmm6 + movaps %xmm6, 0x40(%rdi) + palignr $10, %xmm4, %xmm5 + movaps %xmm5, 0x30(%rdi) + palignr $10, %xmm3, %xmm4 + movaps %xmm4, 0x20(%rdi) + palignr $10, %xmm2, %xmm3 + movaps %xmm3, 0x10(%rdi) + palignr $10, %xmm1, %xmm2 + movaps %xmm2, (%rdi) + lea 0x80(%rdi), %rdi + jae L(shl_10) + movdqu %xmm0, (%r8) + add $0x80, %rdx + add %rdx, %rdi + add %rdx, %rsi + BRANCH_TO_JMPTBL_ENTRY (L(table_144_bytes_fwd), %rdx, 4) - .p2align 4 + .p2align 4 L(shl_10_bwd): - movaps -0x0a(%rsi), %xmm1 + movaps -0x0a(%rsi), %xmm1 - movaps -0x1a(%rsi), %xmm2 - palignr $10, %xmm2, %xmm1 - movaps %xmm1, -0x10(%rdi) + movaps -0x1a(%rsi), %xmm2 + palignr $10, %xmm2, %xmm1 + movaps %xmm1, -0x10(%rdi) - movaps -0x2a(%rsi), %xmm3 - palignr $10, %xmm3, %xmm2 - movaps %xmm2, -0x20(%rdi) + movaps -0x2a(%rsi), %xmm3 + palignr $10, %xmm3, %xmm2 + movaps %xmm2, -0x20(%rdi) - movaps -0x3a(%rsi), %xmm4 - palignr $10, %xmm4, %xmm3 - movaps %xmm3, -0x30(%rdi) + movaps -0x3a(%rsi), %xmm4 + palignr $10, %xmm4, %xmm3 + movaps %xmm3, -0x30(%rdi) - movaps -0x4a(%rsi), %xmm5 - palignr $10, %xmm5, %xmm4 - movaps %xmm4, -0x40(%rdi) + movaps -0x4a(%rsi), %xmm5 + palignr $10, %xmm5, %xmm4 + movaps %xmm4, -0x40(%rdi) - movaps -0x5a(%rsi), %xmm6 - palignr $10, %xmm6, %xmm5 - movaps %xmm5, -0x50(%rdi) + movaps -0x5a(%rsi), %xmm6 + palignr $10, %xmm6, %xmm5 + movaps %xmm5, -0x50(%rdi) - movaps -0x6a(%rsi), %xmm7 - palignr $10, %xmm7, %xmm6 - movaps %xmm6, -0x60(%rdi) + movaps -0x6a(%rsi), %xmm7 + palignr $10, %xmm7, %xmm6 + movaps %xmm6, -0x60(%rdi) - movaps -0x7a(%rsi), %xmm8 - palignr $10, %xmm8, %xmm7 - movaps %xmm7, -0x70(%rdi) + movaps -0x7a(%rsi), %xmm8 + palignr $10, %xmm8, %xmm7 + movaps %xmm7, -0x70(%rdi) - movaps -0x8a(%rsi), %xmm9 - palignr $10, %xmm9, %xmm8 - movaps %xmm8, -0x80(%rdi) + movaps -0x8a(%rsi), %xmm9 + palignr $10, %xmm9, %xmm8 + movaps %xmm8, -0x80(%rdi) - sub $0x80, %rdx - lea -0x80(%rdi), %rdi - lea -0x80(%rsi), %rsi - jae L(shl_10_bwd) - movdqu %xmm0, (%r8) - add $0x80, %rdx - sub %rdx, %rdi - sub %rdx, %rsi - BRANCH_TO_JMPTBL_ENTRY (L(table_144_bytes_bwd), %rdx, 4) + sub $0x80, %rdx + lea -0x80(%rdi), %rdi + lea -0x80(%rsi), %rsi + jae L(shl_10_bwd) + movdqu %xmm0, (%r8) + add $0x80, %rdx + sub %rdx, %rdi + sub %rdx, %rsi + BRANCH_TO_JMPTBL_ENTRY (L(table_144_bytes_bwd), %rdx, 4) - .p2align 4 + .p2align 4 L(shl_11): - sub $0x80, %rdx - movaps -0x0b(%rsi), %xmm1 - movaps 0x05(%rsi), %xmm2 - movaps 0x15(%rsi), %xmm3 - movaps 0x25(%rsi), %xmm4 - movaps 0x35(%rsi), %xmm5 - movaps 0x45(%rsi), %xmm6 - movaps 0x55(%rsi), %xmm7 - movaps 0x65(%rsi), %xmm8 - movaps 0x75(%rsi), %xmm9 - lea 0x80(%rsi), %rsi - palignr $11, %xmm8, %xmm9 - movaps %xmm9, 0x70(%rdi) - palignr $11, %xmm7, %xmm8 - movaps %xmm8, 0x60(%rdi) - palignr $11, %xmm6, %xmm7 - movaps %xmm7, 0x50(%rdi) - palignr $11, %xmm5, %xmm6 - movaps %xmm6, 0x40(%rdi) - palignr $11, %xmm4, %xmm5 - movaps %xmm5, 0x30(%rdi) - palignr $11, %xmm3, %xmm4 - movaps %xmm4, 0x20(%rdi) - palignr $11, %xmm2, %xmm3 - movaps %xmm3, 0x10(%rdi) - palignr $11, %xmm1, %xmm2 - movaps %xmm2, (%rdi) - lea 0x80(%rdi), %rdi - jae L(shl_11) - movdqu %xmm0, (%r8) - add $0x80, %rdx - add %rdx, %rdi - add %rdx, %rsi - BRANCH_TO_JMPTBL_ENTRY (L(table_144_bytes_fwd), %rdx, 4) + sub $0x80, %rdx + movaps -0x0b(%rsi), %xmm1 + movaps 0x05(%rsi), %xmm2 + movaps 0x15(%rsi), %xmm3 + movaps 0x25(%rsi), %xmm4 + movaps 0x35(%rsi), %xmm5 + movaps 0x45(%rsi), %xmm6 + movaps 0x55(%rsi), %xmm7 + movaps 0x65(%rsi), %xmm8 + movaps 0x75(%rsi), %xmm9 + lea 0x80(%rsi), %rsi + palignr $11, %xmm8, %xmm9 + movaps %xmm9, 0x70(%rdi) + palignr $11, %xmm7, %xmm8 + movaps %xmm8, 0x60(%rdi) + palignr $11, %xmm6, %xmm7 + movaps %xmm7, 0x50(%rdi) + palignr $11, %xmm5, %xmm6 + movaps %xmm6, 0x40(%rdi) + palignr $11, %xmm4, %xmm5 + movaps %xmm5, 0x30(%rdi) + palignr $11, %xmm3, %xmm4 + movaps %xmm4, 0x20(%rdi) + palignr $11, %xmm2, %xmm3 + movaps %xmm3, 0x10(%rdi) + palignr $11, %xmm1, %xmm2 + movaps %xmm2, (%rdi) + lea 0x80(%rdi), %rdi + jae L(shl_11) + movdqu %xmm0, (%r8) + add $0x80, %rdx + add %rdx, %rdi + add %rdx, %rsi + BRANCH_TO_JMPTBL_ENTRY (L(table_144_bytes_fwd), %rdx, 4) - .p2align 4 + .p2align 4 L(shl_11_bwd): - movaps -0x0b(%rsi), %xmm1 + movaps -0x0b(%rsi), %xmm1 - movaps -0x1b(%rsi), %xmm2 - palignr $11, %xmm2, %xmm1 - movaps %xmm1, -0x10(%rdi) + movaps -0x1b(%rsi), %xmm2 + palignr $11, %xmm2, %xmm1 + movaps %xmm1, -0x10(%rdi) - movaps -0x2b(%rsi), %xmm3 - palignr $11, %xmm3, %xmm2 - movaps %xmm2, -0x20(%rdi) + movaps -0x2b(%rsi), %xmm3 + palignr $11, %xmm3, %xmm2 + movaps %xmm2, -0x20(%rdi) - movaps -0x3b(%rsi), %xmm4 - palignr $11, %xmm4, %xmm3 - movaps %xmm3, -0x30(%rdi) + movaps -0x3b(%rsi), %xmm4 + palignr $11, %xmm4, %xmm3 + movaps %xmm3, -0x30(%rdi) - movaps -0x4b(%rsi), %xmm5 - palignr $11, %xmm5, %xmm4 - movaps %xmm4, -0x40(%rdi) + movaps -0x4b(%rsi), %xmm5 + palignr $11, %xmm5, %xmm4 + movaps %xmm4, -0x40(%rdi) - movaps -0x5b(%rsi), %xmm6 - palignr $11, %xmm6, %xmm5 - movaps %xmm5, -0x50(%rdi) + movaps -0x5b(%rsi), %xmm6 + palignr $11, %xmm6, %xmm5 + movaps %xmm5, -0x50(%rdi) - movaps -0x6b(%rsi), %xmm7 - palignr $11, %xmm7, %xmm6 - movaps %xmm6, -0x60(%rdi) + movaps -0x6b(%rsi), %xmm7 + palignr $11, %xmm7, %xmm6 + movaps %xmm6, -0x60(%rdi) - movaps -0x7b(%rsi), %xmm8 - palignr $11, %xmm8, %xmm7 - movaps %xmm7, -0x70(%rdi) + movaps -0x7b(%rsi), %xmm8 + palignr $11, %xmm8, %xmm7 + movaps %xmm7, -0x70(%rdi) - movaps -0x8b(%rsi), %xmm9 - palignr $11, %xmm9, %xmm8 - movaps %xmm8, -0x80(%rdi) + movaps -0x8b(%rsi), %xmm9 + palignr $11, %xmm9, %xmm8 + movaps %xmm8, -0x80(%rdi) - sub $0x80, %rdx - lea -0x80(%rdi), %rdi - lea -0x80(%rsi), %rsi - jae L(shl_11_bwd) - movdqu %xmm0, (%r8) - add $0x80, %rdx - sub %rdx, %rdi - sub %rdx, %rsi - BRANCH_TO_JMPTBL_ENTRY (L(table_144_bytes_bwd), %rdx, 4) + sub $0x80, %rdx + lea -0x80(%rdi), %rdi + lea -0x80(%rsi), %rsi + jae L(shl_11_bwd) + movdqu %xmm0, (%r8) + add $0x80, %rdx + sub %rdx, %rdi + sub %rdx, %rsi + BRANCH_TO_JMPTBL_ENTRY (L(table_144_bytes_bwd), %rdx, 4) - .p2align 4 + .p2align 4 L(shl_12): - sub $0x80, %rdx - movdqa -0x0c(%rsi), %xmm1 - movaps 0x04(%rsi), %xmm2 - movaps 0x14(%rsi), %xmm3 - movaps 0x24(%rsi), %xmm4 - movaps 0x34(%rsi), %xmm5 - movaps 0x44(%rsi), %xmm6 - movaps 0x54(%rsi), %xmm7 - movaps 0x64(%rsi), %xmm8 - movaps 0x74(%rsi), %xmm9 - lea 0x80(%rsi), %rsi - palignr $12, %xmm8, %xmm9 - movaps %xmm9, 0x70(%rdi) - palignr $12, %xmm7, %xmm8 - movaps %xmm8, 0x60(%rdi) - palignr $12, %xmm6, %xmm7 - movaps %xmm7, 0x50(%rdi) - palignr $12, %xmm5, %xmm6 - movaps %xmm6, 0x40(%rdi) - palignr $12, %xmm4, %xmm5 - movaps %xmm5, 0x30(%rdi) - palignr $12, %xmm3, %xmm4 - movaps %xmm4, 0x20(%rdi) - palignr $12, %xmm2, %xmm3 - movaps %xmm3, 0x10(%rdi) - palignr $12, %xmm1, %xmm2 - movaps %xmm2, (%rdi) + sub $0x80, %rdx + movdqa -0x0c(%rsi), %xmm1 + movaps 0x04(%rsi), %xmm2 + movaps 0x14(%rsi), %xmm3 + movaps 0x24(%rsi), %xmm4 + movaps 0x34(%rsi), %xmm5 + movaps 0x44(%rsi), %xmm6 + movaps 0x54(%rsi), %xmm7 + movaps 0x64(%rsi), %xmm8 + movaps 0x74(%rsi), %xmm9 + lea 0x80(%rsi), %rsi + palignr $12, %xmm8, %xmm9 + movaps %xmm9, 0x70(%rdi) + palignr $12, %xmm7, %xmm8 + movaps %xmm8, 0x60(%rdi) + palignr $12, %xmm6, %xmm7 + movaps %xmm7, 0x50(%rdi) + palignr $12, %xmm5, %xmm6 + movaps %xmm6, 0x40(%rdi) + palignr $12, %xmm4, %xmm5 + movaps %xmm5, 0x30(%rdi) + palignr $12, %xmm3, %xmm4 + movaps %xmm4, 0x20(%rdi) + palignr $12, %xmm2, %xmm3 + movaps %xmm3, 0x10(%rdi) + palignr $12, %xmm1, %xmm2 + movaps %xmm2, (%rdi) - lea 0x80(%rdi), %rdi - jae L(shl_12) - movdqu %xmm0, (%r8) - add $0x80, %rdx - add %rdx, %rdi - add %rdx, %rsi - BRANCH_TO_JMPTBL_ENTRY (L(table_144_bytes_fwd), %rdx, 4) + lea 0x80(%rdi), %rdi + jae L(shl_12) + movdqu %xmm0, (%r8) + add $0x80, %rdx + add %rdx, %rdi + add %rdx, %rsi + BRANCH_TO_JMPTBL_ENTRY (L(table_144_bytes_fwd), %rdx, 4) - .p2align 4 + .p2align 4 L(shl_12_bwd): - movaps -0x0c(%rsi), %xmm1 + movaps -0x0c(%rsi), %xmm1 - movaps -0x1c(%rsi), %xmm2 - palignr $12, %xmm2, %xmm1 - movaps %xmm1, -0x10(%rdi) + movaps -0x1c(%rsi), %xmm2 + palignr $12, %xmm2, %xmm1 + movaps %xmm1, -0x10(%rdi) - movaps -0x2c(%rsi), %xmm3 - palignr $12, %xmm3, %xmm2 - movaps %xmm2, -0x20(%rdi) + movaps -0x2c(%rsi), %xmm3 + palignr $12, %xmm3, %xmm2 + movaps %xmm2, -0x20(%rdi) - movaps -0x3c(%rsi), %xmm4 - palignr $12, %xmm4, %xmm3 - movaps %xmm3, -0x30(%rdi) + movaps -0x3c(%rsi), %xmm4 + palignr $12, %xmm4, %xmm3 + movaps %xmm3, -0x30(%rdi) - movaps -0x4c(%rsi), %xmm5 - palignr $12, %xmm5, %xmm4 - movaps %xmm4, -0x40(%rdi) + movaps -0x4c(%rsi), %xmm5 + palignr $12, %xmm5, %xmm4 + movaps %xmm4, -0x40(%rdi) - movaps -0x5c(%rsi), %xmm6 - palignr $12, %xmm6, %xmm5 - movaps %xmm5, -0x50(%rdi) + movaps -0x5c(%rsi), %xmm6 + palignr $12, %xmm6, %xmm5 + movaps %xmm5, -0x50(%rdi) - movaps -0x6c(%rsi), %xmm7 - palignr $12, %xmm7, %xmm6 - movaps %xmm6, -0x60(%rdi) + movaps -0x6c(%rsi), %xmm7 + palignr $12, %xmm7, %xmm6 + movaps %xmm6, -0x60(%rdi) - movaps -0x7c(%rsi), %xmm8 - palignr $12, %xmm8, %xmm7 - movaps %xmm7, -0x70(%rdi) + movaps -0x7c(%rsi), %xmm8 + palignr $12, %xmm8, %xmm7 + movaps %xmm7, -0x70(%rdi) - movaps -0x8c(%rsi), %xmm9 - palignr $12, %xmm9, %xmm8 - movaps %xmm8, -0x80(%rdi) + movaps -0x8c(%rsi), %xmm9 + palignr $12, %xmm9, %xmm8 + movaps %xmm8, -0x80(%rdi) - sub $0x80, %rdx - lea -0x80(%rdi), %rdi - lea -0x80(%rsi), %rsi - jae L(shl_12_bwd) - movdqu %xmm0, (%r8) - add $0x80, %rdx - sub %rdx, %rdi - sub %rdx, %rsi - BRANCH_TO_JMPTBL_ENTRY (L(table_144_bytes_bwd), %rdx, 4) + sub $0x80, %rdx + lea -0x80(%rdi), %rdi + lea -0x80(%rsi), %rsi + jae L(shl_12_bwd) + movdqu %xmm0, (%r8) + add $0x80, %rdx + sub %rdx, %rdi + sub %rdx, %rsi + BRANCH_TO_JMPTBL_ENTRY (L(table_144_bytes_bwd), %rdx, 4) - .p2align 4 + .p2align 4 L(shl_13): - sub $0x80, %rdx - movaps -0x0d(%rsi), %xmm1 - movaps 0x03(%rsi), %xmm2 - movaps 0x13(%rsi), %xmm3 - movaps 0x23(%rsi), %xmm4 - movaps 0x33(%rsi), %xmm5 - movaps 0x43(%rsi), %xmm6 - movaps 0x53(%rsi), %xmm7 - movaps 0x63(%rsi), %xmm8 - movaps 0x73(%rsi), %xmm9 - lea 0x80(%rsi), %rsi - palignr $13, %xmm8, %xmm9 - movaps %xmm9, 0x70(%rdi) - palignr $13, %xmm7, %xmm8 - movaps %xmm8, 0x60(%rdi) - palignr $13, %xmm6, %xmm7 - movaps %xmm7, 0x50(%rdi) - palignr $13, %xmm5, %xmm6 - movaps %xmm6, 0x40(%rdi) - palignr $13, %xmm4, %xmm5 - movaps %xmm5, 0x30(%rdi) - palignr $13, %xmm3, %xmm4 - movaps %xmm4, 0x20(%rdi) - palignr $13, %xmm2, %xmm3 - movaps %xmm3, 0x10(%rdi) - palignr $13, %xmm1, %xmm2 - movaps %xmm2, (%rdi) - lea 0x80(%rdi), %rdi - jae L(shl_13) - movdqu %xmm0, (%r8) - add $0x80, %rdx - add %rdx, %rdi - add %rdx, %rsi - BRANCH_TO_JMPTBL_ENTRY (L(table_144_bytes_fwd), %rdx, 4) + sub $0x80, %rdx + movaps -0x0d(%rsi), %xmm1 + movaps 0x03(%rsi), %xmm2 + movaps 0x13(%rsi), %xmm3 + movaps 0x23(%rsi), %xmm4 + movaps 0x33(%rsi), %xmm5 + movaps 0x43(%rsi), %xmm6 + movaps 0x53(%rsi), %xmm7 + movaps 0x63(%rsi), %xmm8 + movaps 0x73(%rsi), %xmm9 + lea 0x80(%rsi), %rsi + palignr $13, %xmm8, %xmm9 + movaps %xmm9, 0x70(%rdi) + palignr $13, %xmm7, %xmm8 + movaps %xmm8, 0x60(%rdi) + palignr $13, %xmm6, %xmm7 + movaps %xmm7, 0x50(%rdi) + palignr $13, %xmm5, %xmm6 + movaps %xmm6, 0x40(%rdi) + palignr $13, %xmm4, %xmm5 + movaps %xmm5, 0x30(%rdi) + palignr $13, %xmm3, %xmm4 + movaps %xmm4, 0x20(%rdi) + palignr $13, %xmm2, %xmm3 + movaps %xmm3, 0x10(%rdi) + palignr $13, %xmm1, %xmm2 + movaps %xmm2, (%rdi) + lea 0x80(%rdi), %rdi + jae L(shl_13) + movdqu %xmm0, (%r8) + add $0x80, %rdx + add %rdx, %rdi + add %rdx, %rsi + BRANCH_TO_JMPTBL_ENTRY (L(table_144_bytes_fwd), %rdx, 4) - .p2align 4 + .p2align 4 L(shl_13_bwd): - movaps -0x0d(%rsi), %xmm1 + movaps -0x0d(%rsi), %xmm1 - movaps -0x1d(%rsi), %xmm2 - palignr $13, %xmm2, %xmm1 - movaps %xmm1, -0x10(%rdi) + movaps -0x1d(%rsi), %xmm2 + palignr $13, %xmm2, %xmm1 + movaps %xmm1, -0x10(%rdi) - movaps -0x2d(%rsi), %xmm3 - palignr $13, %xmm3, %xmm2 - movaps %xmm2, -0x20(%rdi) + movaps -0x2d(%rsi), %xmm3 + palignr $13, %xmm3, %xmm2 + movaps %xmm2, -0x20(%rdi) - movaps -0x3d(%rsi), %xmm4 - palignr $13, %xmm4, %xmm3 - movaps %xmm3, -0x30(%rdi) + movaps -0x3d(%rsi), %xmm4 + palignr $13, %xmm4, %xmm3 + movaps %xmm3, -0x30(%rdi) - movaps -0x4d(%rsi), %xmm5 - palignr $13, %xmm5, %xmm4 - movaps %xmm4, -0x40(%rdi) + movaps -0x4d(%rsi), %xmm5 + palignr $13, %xmm5, %xmm4 + movaps %xmm4, -0x40(%rdi) - movaps -0x5d(%rsi), %xmm6 - palignr $13, %xmm6, %xmm5 - movaps %xmm5, -0x50(%rdi) + movaps -0x5d(%rsi), %xmm6 + palignr $13, %xmm6, %xmm5 + movaps %xmm5, -0x50(%rdi) - movaps -0x6d(%rsi), %xmm7 - palignr $13, %xmm7, %xmm6 - movaps %xmm6, -0x60(%rdi) + movaps -0x6d(%rsi), %xmm7 + palignr $13, %xmm7, %xmm6 + movaps %xmm6, -0x60(%rdi) - movaps -0x7d(%rsi), %xmm8 - palignr $13, %xmm8, %xmm7 - movaps %xmm7, -0x70(%rdi) + movaps -0x7d(%rsi), %xmm8 + palignr $13, %xmm8, %xmm7 + movaps %xmm7, -0x70(%rdi) - movaps -0x8d(%rsi), %xmm9 - palignr $13, %xmm9, %xmm8 - movaps %xmm8, -0x80(%rdi) + movaps -0x8d(%rsi), %xmm9 + palignr $13, %xmm9, %xmm8 + movaps %xmm8, -0x80(%rdi) - sub $0x80, %rdx - lea -0x80(%rdi), %rdi - lea -0x80(%rsi), %rsi - jae L(shl_13_bwd) - movdqu %xmm0, (%r8) - add $0x80, %rdx - sub %rdx, %rdi - sub %rdx, %rsi - BRANCH_TO_JMPTBL_ENTRY (L(table_144_bytes_bwd), %rdx, 4) + sub $0x80, %rdx + lea -0x80(%rdi), %rdi + lea -0x80(%rsi), %rsi + jae L(shl_13_bwd) + movdqu %xmm0, (%r8) + add $0x80, %rdx + sub %rdx, %rdi + sub %rdx, %rsi + BRANCH_TO_JMPTBL_ENTRY (L(table_144_bytes_bwd), %rdx, 4) - .p2align 4 + .p2align 4 L(shl_14): - sub $0x80, %rdx - movaps -0x0e(%rsi), %xmm1 - movaps 0x02(%rsi), %xmm2 - movaps 0x12(%rsi), %xmm3 - movaps 0x22(%rsi), %xmm4 - movaps 0x32(%rsi), %xmm5 - movaps 0x42(%rsi), %xmm6 - movaps 0x52(%rsi), %xmm7 - movaps 0x62(%rsi), %xmm8 - movaps 0x72(%rsi), %xmm9 - lea 0x80(%rsi), %rsi - palignr $14, %xmm8, %xmm9 - movaps %xmm9, 0x70(%rdi) - palignr $14, %xmm7, %xmm8 - movaps %xmm8, 0x60(%rdi) - palignr $14, %xmm6, %xmm7 - movaps %xmm7, 0x50(%rdi) - palignr $14, %xmm5, %xmm6 - movaps %xmm6, 0x40(%rdi) - palignr $14, %xmm4, %xmm5 - movaps %xmm5, 0x30(%rdi) - palignr $14, %xmm3, %xmm4 - movaps %xmm4, 0x20(%rdi) - palignr $14, %xmm2, %xmm3 - movaps %xmm3, 0x10(%rdi) - palignr $14, %xmm1, %xmm2 - movaps %xmm2, (%rdi) - lea 0x80(%rdi), %rdi - jae L(shl_14) - movdqu %xmm0, (%r8) - add $0x80, %rdx - add %rdx, %rdi - add %rdx, %rsi - BRANCH_TO_JMPTBL_ENTRY (L(table_144_bytes_fwd), %rdx, 4) + sub $0x80, %rdx + movaps -0x0e(%rsi), %xmm1 + movaps 0x02(%rsi), %xmm2 + movaps 0x12(%rsi), %xmm3 + movaps 0x22(%rsi), %xmm4 + movaps 0x32(%rsi), %xmm5 + movaps 0x42(%rsi), %xmm6 + movaps 0x52(%rsi), %xmm7 + movaps 0x62(%rsi), %xmm8 + movaps 0x72(%rsi), %xmm9 + lea 0x80(%rsi), %rsi + palignr $14, %xmm8, %xmm9 + movaps %xmm9, 0x70(%rdi) + palignr $14, %xmm7, %xmm8 + movaps %xmm8, 0x60(%rdi) + palignr $14, %xmm6, %xmm7 + movaps %xmm7, 0x50(%rdi) + palignr $14, %xmm5, %xmm6 + movaps %xmm6, 0x40(%rdi) + palignr $14, %xmm4, %xmm5 + movaps %xmm5, 0x30(%rdi) + palignr $14, %xmm3, %xmm4 + movaps %xmm4, 0x20(%rdi) + palignr $14, %xmm2, %xmm3 + movaps %xmm3, 0x10(%rdi) + palignr $14, %xmm1, %xmm2 + movaps %xmm2, (%rdi) + lea 0x80(%rdi), %rdi + jae L(shl_14) + movdqu %xmm0, (%r8) + add $0x80, %rdx + add %rdx, %rdi + add %rdx, %rsi + BRANCH_TO_JMPTBL_ENTRY (L(table_144_bytes_fwd), %rdx, 4) - .p2align 4 + .p2align 4 L(shl_14_bwd): - movaps -0x0e(%rsi), %xmm1 + movaps -0x0e(%rsi), %xmm1 - movaps -0x1e(%rsi), %xmm2 - palignr $14, %xmm2, %xmm1 - movaps %xmm1, -0x10(%rdi) + movaps -0x1e(%rsi), %xmm2 + palignr $14, %xmm2, %xmm1 + movaps %xmm1, -0x10(%rdi) - movaps -0x2e(%rsi), %xmm3 - palignr $14, %xmm3, %xmm2 - movaps %xmm2, -0x20(%rdi) + movaps -0x2e(%rsi), %xmm3 + palignr $14, %xmm3, %xmm2 + movaps %xmm2, -0x20(%rdi) - movaps -0x3e(%rsi), %xmm4 - palignr $14, %xmm4, %xmm3 - movaps %xmm3, -0x30(%rdi) + movaps -0x3e(%rsi), %xmm4 + palignr $14, %xmm4, %xmm3 + movaps %xmm3, -0x30(%rdi) - movaps -0x4e(%rsi), %xmm5 - palignr $14, %xmm5, %xmm4 - movaps %xmm4, -0x40(%rdi) + movaps -0x4e(%rsi), %xmm5 + palignr $14, %xmm5, %xmm4 + movaps %xmm4, -0x40(%rdi) - movaps -0x5e(%rsi), %xmm6 - palignr $14, %xmm6, %xmm5 - movaps %xmm5, -0x50(%rdi) + movaps -0x5e(%rsi), %xmm6 + palignr $14, %xmm6, %xmm5 + movaps %xmm5, -0x50(%rdi) - movaps -0x6e(%rsi), %xmm7 - palignr $14, %xmm7, %xmm6 - movaps %xmm6, -0x60(%rdi) + movaps -0x6e(%rsi), %xmm7 + palignr $14, %xmm7, %xmm6 + movaps %xmm6, -0x60(%rdi) - movaps -0x7e(%rsi), %xmm8 - palignr $14, %xmm8, %xmm7 - movaps %xmm7, -0x70(%rdi) + movaps -0x7e(%rsi), %xmm8 + palignr $14, %xmm8, %xmm7 + movaps %xmm7, -0x70(%rdi) - movaps -0x8e(%rsi), %xmm9 - palignr $14, %xmm9, %xmm8 - movaps %xmm8, -0x80(%rdi) + movaps -0x8e(%rsi), %xmm9 + palignr $14, %xmm9, %xmm8 + movaps %xmm8, -0x80(%rdi) - sub $0x80, %rdx - lea -0x80(%rdi), %rdi - lea -0x80(%rsi), %rsi - jae L(shl_14_bwd) - movdqu %xmm0, (%r8) - add $0x80, %rdx - sub %rdx, %rdi - sub %rdx, %rsi - BRANCH_TO_JMPTBL_ENTRY (L(table_144_bytes_bwd), %rdx, 4) + sub $0x80, %rdx + lea -0x80(%rdi), %rdi + lea -0x80(%rsi), %rsi + jae L(shl_14_bwd) + movdqu %xmm0, (%r8) + add $0x80, %rdx + sub %rdx, %rdi + sub %rdx, %rsi + BRANCH_TO_JMPTBL_ENTRY (L(table_144_bytes_bwd), %rdx, 4) - .p2align 4 + .p2align 4 L(shl_15): - sub $0x80, %rdx - movaps -0x0f(%rsi), %xmm1 - movaps 0x01(%rsi), %xmm2 - movaps 0x11(%rsi), %xmm3 - movaps 0x21(%rsi), %xmm4 - movaps 0x31(%rsi), %xmm5 - movaps 0x41(%rsi), %xmm6 - movaps 0x51(%rsi), %xmm7 - movaps 0x61(%rsi), %xmm8 - movaps 0x71(%rsi), %xmm9 - lea 0x80(%rsi), %rsi - palignr $15, %xmm8, %xmm9 - movaps %xmm9, 0x70(%rdi) - palignr $15, %xmm7, %xmm8 - movaps %xmm8, 0x60(%rdi) - palignr $15, %xmm6, %xmm7 - movaps %xmm7, 0x50(%rdi) - palignr $15, %xmm5, %xmm6 - movaps %xmm6, 0x40(%rdi) - palignr $15, %xmm4, %xmm5 - movaps %xmm5, 0x30(%rdi) - palignr $15, %xmm3, %xmm4 - movaps %xmm4, 0x20(%rdi) - palignr $15, %xmm2, %xmm3 - movaps %xmm3, 0x10(%rdi) - palignr $15, %xmm1, %xmm2 - movaps %xmm2, (%rdi) - lea 0x80(%rdi), %rdi - jae L(shl_15) - movdqu %xmm0, (%r8) - add $0x80, %rdx - add %rdx, %rdi - add %rdx, %rsi - BRANCH_TO_JMPTBL_ENTRY (L(table_144_bytes_fwd), %rdx, 4) + sub $0x80, %rdx + movaps -0x0f(%rsi), %xmm1 + movaps 0x01(%rsi), %xmm2 + movaps 0x11(%rsi), %xmm3 + movaps 0x21(%rsi), %xmm4 + movaps 0x31(%rsi), %xmm5 + movaps 0x41(%rsi), %xmm6 + movaps 0x51(%rsi), %xmm7 + movaps 0x61(%rsi), %xmm8 + movaps 0x71(%rsi), %xmm9 + lea 0x80(%rsi), %rsi + palignr $15, %xmm8, %xmm9 + movaps %xmm9, 0x70(%rdi) + palignr $15, %xmm7, %xmm8 + movaps %xmm8, 0x60(%rdi) + palignr $15, %xmm6, %xmm7 + movaps %xmm7, 0x50(%rdi) + palignr $15, %xmm5, %xmm6 + movaps %xmm6, 0x40(%rdi) + palignr $15, %xmm4, %xmm5 + movaps %xmm5, 0x30(%rdi) + palignr $15, %xmm3, %xmm4 + movaps %xmm4, 0x20(%rdi) + palignr $15, %xmm2, %xmm3 + movaps %xmm3, 0x10(%rdi) + palignr $15, %xmm1, %xmm2 + movaps %xmm2, (%rdi) + lea 0x80(%rdi), %rdi + jae L(shl_15) + movdqu %xmm0, (%r8) + add $0x80, %rdx + add %rdx, %rdi + add %rdx, %rsi + BRANCH_TO_JMPTBL_ENTRY (L(table_144_bytes_fwd), %rdx, 4) - .p2align 4 + .p2align 4 L(shl_15_bwd): - movaps -0x0f(%rsi), %xmm1 + movaps -0x0f(%rsi), %xmm1 - movaps -0x1f(%rsi), %xmm2 - palignr $15, %xmm2, %xmm1 - movaps %xmm1, -0x10(%rdi) + movaps -0x1f(%rsi), %xmm2 + palignr $15, %xmm2, %xmm1 + movaps %xmm1, -0x10(%rdi) - movaps -0x2f(%rsi), %xmm3 - palignr $15, %xmm3, %xmm2 - movaps %xmm2, -0x20(%rdi) + movaps -0x2f(%rsi), %xmm3 + palignr $15, %xmm3, %xmm2 + movaps %xmm2, -0x20(%rdi) - movaps -0x3f(%rsi), %xmm4 - palignr $15, %xmm4, %xmm3 - movaps %xmm3, -0x30(%rdi) + movaps -0x3f(%rsi), %xmm4 + palignr $15, %xmm4, %xmm3 + movaps %xmm3, -0x30(%rdi) - movaps -0x4f(%rsi), %xmm5 - palignr $15, %xmm5, %xmm4 - movaps %xmm4, -0x40(%rdi) + movaps -0x4f(%rsi), %xmm5 + palignr $15, %xmm5, %xmm4 + movaps %xmm4, -0x40(%rdi) - movaps -0x5f(%rsi), %xmm6 - palignr $15, %xmm6, %xmm5 - movaps %xmm5, -0x50(%rdi) + movaps -0x5f(%rsi), %xmm6 + palignr $15, %xmm6, %xmm5 + movaps %xmm5, -0x50(%rdi) - movaps -0x6f(%rsi), %xmm7 - palignr $15, %xmm7, %xmm6 - movaps %xmm6, -0x60(%rdi) + movaps -0x6f(%rsi), %xmm7 + palignr $15, %xmm7, %xmm6 + movaps %xmm6, -0x60(%rdi) - movaps -0x7f(%rsi), %xmm8 - palignr $15, %xmm8, %xmm7 - movaps %xmm7, -0x70(%rdi) + movaps -0x7f(%rsi), %xmm8 + palignr $15, %xmm8, %xmm7 + movaps %xmm7, -0x70(%rdi) - movaps -0x8f(%rsi), %xmm9 - palignr $15, %xmm9, %xmm8 - movaps %xmm8, -0x80(%rdi) + movaps -0x8f(%rsi), %xmm9 + palignr $15, %xmm9, %xmm8 + movaps %xmm8, -0x80(%rdi) - sub $0x80, %rdx - lea -0x80(%rdi), %rdi - lea -0x80(%rsi), %rsi - jae L(shl_15_bwd) - movdqu %xmm0, (%r8) - add $0x80, %rdx - sub %rdx, %rdi - sub %rdx, %rsi - BRANCH_TO_JMPTBL_ENTRY (L(table_144_bytes_bwd), %rdx, 4) + sub $0x80, %rdx + lea -0x80(%rdi), %rdi + lea -0x80(%rsi), %rsi + jae L(shl_15_bwd) + movdqu %xmm0, (%r8) + add $0x80, %rdx + sub %rdx, %rdi + sub %rdx, %rsi + BRANCH_TO_JMPTBL_ENTRY (L(table_144_bytes_bwd), %rdx, 4) - .p2align 4 + .p2align 4 L(gobble_mem_fwd): - movdqu (%rsi), %xmm1 - movdqu %xmm0, (%r8) - movdqa %xmm1, (%rdi) - sub $16, %rdx - add $16, %rsi - add $16, %rdi + movdqu (%rsi), %xmm1 + movdqu %xmm0, (%r8) + movdqa %xmm1, (%rdi) + sub $16, %rdx + add $16, %rsi + add $16, %rdi #ifdef SHARED_CACHE_SIZE_HALF - mov $SHARED_CACHE_SIZE_HALF, %RCX_LP + mov $SHARED_CACHE_SIZE_HALF, %RCX_LP #else - mov __x86_shared_cache_size_half(%rip), %RCX_LP + mov __x86_shared_cache_size_half(%rip), %RCX_LP #endif #ifdef USE_AS_MEMMOVE - mov %rsi, %r9 - sub %rdi, %r9 - cmp %rdx, %r9 - jae L(memmove_is_memcpy_fwd) - cmp %rcx, %r9 - jbe L(ll_cache_copy_fwd_start) + mov %rsi, %r9 + sub %rdi, %r9 + cmp %rdx, %r9 + jae L(memmove_is_memcpy_fwd) + cmp %rcx, %r9 + jbe L(ll_cache_copy_fwd_start) L(memmove_is_memcpy_fwd): #endif - cmp %rcx, %rdx - ja L(bigger_in_fwd) - mov %rdx, %rcx + cmp %rcx, %rdx + ja L(bigger_in_fwd) + mov %rdx, %rcx L(bigger_in_fwd): - sub %rcx, %rdx - cmp $0x1000, %rdx - jbe L(ll_cache_copy_fwd) + sub %rcx, %rdx + cmp $0x1000, %rdx + jbe L(ll_cache_copy_fwd) - mov %rcx, %r9 - shl $3, %r9 - cmp %r9, %rdx - jbe L(2steps_copy_fwd) - add %rcx, %rdx - xor %rcx, %rcx + mov %rcx, %r9 + shl $3, %r9 + cmp %r9, %rdx + jbe L(2steps_copy_fwd) + add %rcx, %rdx + xor %rcx, %rcx L(2steps_copy_fwd): - sub $0x80, %rdx + sub $0x80, %rdx L(gobble_mem_fwd_loop): - sub $0x80, %rdx - prefetcht0 0x200(%rsi) - prefetcht0 0x300(%rsi) - movdqu (%rsi), %xmm0 - movdqu 0x10(%rsi), %xmm1 - movdqu 0x20(%rsi), %xmm2 - movdqu 0x30(%rsi), %xmm3 - movdqu 0x40(%rsi), %xmm4 - movdqu 0x50(%rsi), %xmm5 - movdqu 0x60(%rsi), %xmm6 - movdqu 0x70(%rsi), %xmm7 - lfence - movntdq %xmm0, (%rdi) - movntdq %xmm1, 0x10(%rdi) - movntdq %xmm2, 0x20(%rdi) - movntdq %xmm3, 0x30(%rdi) - movntdq %xmm4, 0x40(%rdi) - movntdq %xmm5, 0x50(%rdi) - movntdq %xmm6, 0x60(%rdi) - movntdq %xmm7, 0x70(%rdi) - lea 0x80(%rsi), %rsi - lea 0x80(%rdi), %rdi - jae L(gobble_mem_fwd_loop) - sfence - cmp $0x80, %rcx - jb L(gobble_mem_fwd_end) - add $0x80, %rdx + sub $0x80, %rdx + prefetcht0 0x200(%rsi) + prefetcht0 0x300(%rsi) + movdqu (%rsi), %xmm0 + movdqu 0x10(%rsi), %xmm1 + movdqu 0x20(%rsi), %xmm2 + movdqu 0x30(%rsi), %xmm3 + movdqu 0x40(%rsi), %xmm4 + movdqu 0x50(%rsi), %xmm5 + movdqu 0x60(%rsi), %xmm6 + movdqu 0x70(%rsi), %xmm7 + lfence + movntdq %xmm0, (%rdi) + movntdq %xmm1, 0x10(%rdi) + movntdq %xmm2, 0x20(%rdi) + movntdq %xmm3, 0x30(%rdi) + movntdq %xmm4, 0x40(%rdi) + movntdq %xmm5, 0x50(%rdi) + movntdq %xmm6, 0x60(%rdi) + movntdq %xmm7, 0x70(%rdi) + lea 0x80(%rsi), %rsi + lea 0x80(%rdi), %rdi + jae L(gobble_mem_fwd_loop) + sfence + cmp $0x80, %rcx + jb L(gobble_mem_fwd_end) + add $0x80, %rdx L(ll_cache_copy_fwd): - add %rcx, %rdx + add %rcx, %rdx L(ll_cache_copy_fwd_start): - sub $0x80, %rdx + sub $0x80, %rdx L(gobble_ll_loop_fwd): - prefetchnta 0x1c0(%rsi) - prefetchnta 0x280(%rsi) - prefetchnta 0x1c0(%rdi) - prefetchnta 0x280(%rdi) - sub $0x80, %rdx - movdqu (%rsi), %xmm0 - movdqu 0x10(%rsi), %xmm1 - movdqu 0x20(%rsi), %xmm2 - movdqu 0x30(%rsi), %xmm3 - movdqu 0x40(%rsi), %xmm4 - movdqu 0x50(%rsi), %xmm5 - movdqu 0x60(%rsi), %xmm6 - movdqu 0x70(%rsi), %xmm7 - movdqa %xmm0, (%rdi) - movdqa %xmm1, 0x10(%rdi) - movdqa %xmm2, 0x20(%rdi) - movdqa %xmm3, 0x30(%rdi) - movdqa %xmm4, 0x40(%rdi) - movdqa %xmm5, 0x50(%rdi) - movdqa %xmm6, 0x60(%rdi) - movdqa %xmm7, 0x70(%rdi) - lea 0x80(%rsi), %rsi - lea 0x80(%rdi), %rdi - jae L(gobble_ll_loop_fwd) + prefetchnta 0x1c0(%rsi) + prefetchnta 0x280(%rsi) + prefetchnta 0x1c0(%rdi) + prefetchnta 0x280(%rdi) + sub $0x80, %rdx + movdqu (%rsi), %xmm0 + movdqu 0x10(%rsi), %xmm1 + movdqu 0x20(%rsi), %xmm2 + movdqu 0x30(%rsi), %xmm3 + movdqu 0x40(%rsi), %xmm4 + movdqu 0x50(%rsi), %xmm5 + movdqu 0x60(%rsi), %xmm6 + movdqu 0x70(%rsi), %xmm7 + movdqa %xmm0, (%rdi) + movdqa %xmm1, 0x10(%rdi) + movdqa %xmm2, 0x20(%rdi) + movdqa %xmm3, 0x30(%rdi) + movdqa %xmm4, 0x40(%rdi) + movdqa %xmm5, 0x50(%rdi) + movdqa %xmm6, 0x60(%rdi) + movdqa %xmm7, 0x70(%rdi) + lea 0x80(%rsi), %rsi + lea 0x80(%rdi), %rdi + jae L(gobble_ll_loop_fwd) L(gobble_mem_fwd_end): - add $0x80, %rdx - add %rdx, %rsi - add %rdx, %rdi - BRANCH_TO_JMPTBL_ENTRY (L(table_144_bytes_fwd), %rdx, 4) + add $0x80, %rdx + add %rdx, %rsi + add %rdx, %rdi + BRANCH_TO_JMPTBL_ENTRY (L(table_144_bytes_fwd), %rdx, 4) - .p2align 4 + .p2align 4 L(gobble_mem_bwd): - add %rdx, %rsi - add %rdx, %rdi + add %rdx, %rsi + add %rdx, %rdi - movdqu -16(%rsi), %xmm0 - lea -16(%rdi), %r8 - mov %rdi, %r9 - and $-16, %rdi - sub %rdi, %r9 - sub %r9, %rsi - sub %r9, %rdx + movdqu -16(%rsi), %xmm0 + lea -16(%rdi), %r8 + mov %rdi, %r9 + and $-16, %rdi + sub %rdi, %r9 + sub %r9, %rsi + sub %r9, %rdx #ifdef SHARED_CACHE_SIZE_HALF - mov $SHARED_CACHE_SIZE_HALF, %RCX_LP + mov $SHARED_CACHE_SIZE_HALF, %RCX_LP #else - mov __x86_shared_cache_size_half(%rip), %RCX_LP + mov __x86_shared_cache_size_half(%rip), %RCX_LP #endif #ifdef USE_AS_MEMMOVE - mov %rdi, %r9 - sub %rsi, %r9 - cmp %rdx, %r9 - jae L(memmove_is_memcpy_bwd) - cmp %rcx, %r9 - jbe L(ll_cache_copy_bwd_start) + mov %rdi, %r9 + sub %rsi, %r9 + cmp %rdx, %r9 + jae L(memmove_is_memcpy_bwd) + cmp %rcx, %r9 + jbe L(ll_cache_copy_bwd_start) L(memmove_is_memcpy_bwd): #endif - cmp %rcx, %rdx - ja L(bigger) - mov %rdx, %rcx + cmp %rcx, %rdx + ja L(bigger) + mov %rdx, %rcx L(bigger): - sub %rcx, %rdx - cmp $0x1000, %rdx - jbe L(ll_cache_copy) + sub %rcx, %rdx + cmp $0x1000, %rdx + jbe L(ll_cache_copy) - mov %rcx, %r9 - shl $3, %r9 - cmp %r9, %rdx - jbe L(2steps_copy) - add %rcx, %rdx - xor %rcx, %rcx + mov %rcx, %r9 + shl $3, %r9 + cmp %r9, %rdx + jbe L(2steps_copy) + add %rcx, %rdx + xor %rcx, %rcx L(2steps_copy): - sub $0x80, %rdx + sub $0x80, %rdx L(gobble_mem_bwd_loop): - sub $0x80, %rdx - prefetcht0 -0x200(%rsi) - prefetcht0 -0x300(%rsi) - movdqu -0x10(%rsi), %xmm1 - movdqu -0x20(%rsi), %xmm2 - movdqu -0x30(%rsi), %xmm3 - movdqu -0x40(%rsi), %xmm4 - movdqu -0x50(%rsi), %xmm5 - movdqu -0x60(%rsi), %xmm6 - movdqu -0x70(%rsi), %xmm7 - movdqu -0x80(%rsi), %xmm8 - lfence - movntdq %xmm1, -0x10(%rdi) - movntdq %xmm2, -0x20(%rdi) - movntdq %xmm3, -0x30(%rdi) - movntdq %xmm4, -0x40(%rdi) - movntdq %xmm5, -0x50(%rdi) - movntdq %xmm6, -0x60(%rdi) - movntdq %xmm7, -0x70(%rdi) - movntdq %xmm8, -0x80(%rdi) - lea -0x80(%rsi), %rsi - lea -0x80(%rdi), %rdi - jae L(gobble_mem_bwd_loop) - sfence - cmp $0x80, %rcx - jb L(gobble_mem_bwd_end) - add $0x80, %rdx + sub $0x80, %rdx + prefetcht0 -0x200(%rsi) + prefetcht0 -0x300(%rsi) + movdqu -0x10(%rsi), %xmm1 + movdqu -0x20(%rsi), %xmm2 + movdqu -0x30(%rsi), %xmm3 + movdqu -0x40(%rsi), %xmm4 + movdqu -0x50(%rsi), %xmm5 + movdqu -0x60(%rsi), %xmm6 + movdqu -0x70(%rsi), %xmm7 + movdqu -0x80(%rsi), %xmm8 + lfence + movntdq %xmm1, -0x10(%rdi) + movntdq %xmm2, -0x20(%rdi) + movntdq %xmm3, -0x30(%rdi) + movntdq %xmm4, -0x40(%rdi) + movntdq %xmm5, -0x50(%rdi) + movntdq %xmm6, -0x60(%rdi) + movntdq %xmm7, -0x70(%rdi) + movntdq %xmm8, -0x80(%rdi) + lea -0x80(%rsi), %rsi + lea -0x80(%rdi), %rdi + jae L(gobble_mem_bwd_loop) + sfence + cmp $0x80, %rcx + jb L(gobble_mem_bwd_end) + add $0x80, %rdx L(ll_cache_copy): - add %rcx, %rdx + add %rcx, %rdx L(ll_cache_copy_bwd_start): - sub $0x80, %rdx + sub $0x80, %rdx L(gobble_ll_loop): - prefetchnta -0x1c0(%rsi) - prefetchnta -0x280(%rsi) - prefetchnta -0x1c0(%rdi) - prefetchnta -0x280(%rdi) - sub $0x80, %rdx - movdqu -0x10(%rsi), %xmm1 - movdqu -0x20(%rsi), %xmm2 - movdqu -0x30(%rsi), %xmm3 - movdqu -0x40(%rsi), %xmm4 - movdqu -0x50(%rsi), %xmm5 - movdqu -0x60(%rsi), %xmm6 - movdqu -0x70(%rsi), %xmm7 - movdqu -0x80(%rsi), %xmm8 - movdqa %xmm1, -0x10(%rdi) - movdqa %xmm2, -0x20(%rdi) - movdqa %xmm3, -0x30(%rdi) - movdqa %xmm4, -0x40(%rdi) - movdqa %xmm5, -0x50(%rdi) - movdqa %xmm6, -0x60(%rdi) - movdqa %xmm7, -0x70(%rdi) - movdqa %xmm8, -0x80(%rdi) - lea -0x80(%rsi), %rsi - lea -0x80(%rdi), %rdi - jae L(gobble_ll_loop) + prefetchnta -0x1c0(%rsi) + prefetchnta -0x280(%rsi) + prefetchnta -0x1c0(%rdi) + prefetchnta -0x280(%rdi) + sub $0x80, %rdx + movdqu -0x10(%rsi), %xmm1 + movdqu -0x20(%rsi), %xmm2 + movdqu -0x30(%rsi), %xmm3 + movdqu -0x40(%rsi), %xmm4 + movdqu -0x50(%rsi), %xmm5 + movdqu -0x60(%rsi), %xmm6 + movdqu -0x70(%rsi), %xmm7 + movdqu -0x80(%rsi), %xmm8 + movdqa %xmm1, -0x10(%rdi) + movdqa %xmm2, -0x20(%rdi) + movdqa %xmm3, -0x30(%rdi) + movdqa %xmm4, -0x40(%rdi) + movdqa %xmm5, -0x50(%rdi) + movdqa %xmm6, -0x60(%rdi) + movdqa %xmm7, -0x70(%rdi) + movdqa %xmm8, -0x80(%rdi) + lea -0x80(%rsi), %rsi + lea -0x80(%rdi), %rdi + jae L(gobble_ll_loop) L(gobble_mem_bwd_end): - movdqu %xmm0, (%r8) - add $0x80, %rdx - sub %rdx, %rsi - sub %rdx, %rdi - BRANCH_TO_JMPTBL_ENTRY (L(table_144_bytes_bwd), %rdx, 4) + movdqu %xmm0, (%r8) + add $0x80, %rdx + sub %rdx, %rsi + sub %rdx, %rdi + BRANCH_TO_JMPTBL_ENTRY (L(table_144_bytes_bwd), %rdx, 4) - .p2align 4 + .p2align 4 L(fwd_write_128bytes): - lddqu -128(%rsi), %xmm0 - movdqu %xmm0, -128(%rdi) + lddqu -128(%rsi), %xmm0 + movdqu %xmm0, -128(%rdi) L(fwd_write_112bytes): - lddqu -112(%rsi), %xmm0 - movdqu %xmm0, -112(%rdi) + lddqu -112(%rsi), %xmm0 + movdqu %xmm0, -112(%rdi) L(fwd_write_96bytes): - lddqu -96(%rsi), %xmm0 - movdqu %xmm0, -96(%rdi) + lddqu -96(%rsi), %xmm0 + movdqu %xmm0, -96(%rdi) L(fwd_write_80bytes): - lddqu -80(%rsi), %xmm0 - movdqu %xmm0, -80(%rdi) + lddqu -80(%rsi), %xmm0 + movdqu %xmm0, -80(%rdi) L(fwd_write_64bytes): - lddqu -64(%rsi), %xmm0 - movdqu %xmm0, -64(%rdi) + lddqu -64(%rsi), %xmm0 + movdqu %xmm0, -64(%rdi) L(fwd_write_48bytes): - lddqu -48(%rsi), %xmm0 - movdqu %xmm0, -48(%rdi) + lddqu -48(%rsi), %xmm0 + movdqu %xmm0, -48(%rdi) L(fwd_write_32bytes): - lddqu -32(%rsi), %xmm0 - movdqu %xmm0, -32(%rdi) + lddqu -32(%rsi), %xmm0 + movdqu %xmm0, -32(%rdi) L(fwd_write_16bytes): - lddqu -16(%rsi), %xmm0 - movdqu %xmm0, -16(%rdi) + lddqu -16(%rsi), %xmm0 + movdqu %xmm0, -16(%rdi) L(fwd_write_0bytes): - ret + ret - .p2align 4 + .p2align 4 L(fwd_write_143bytes): - lddqu -143(%rsi), %xmm0 - movdqu %xmm0, -143(%rdi) + lddqu -143(%rsi), %xmm0 + movdqu %xmm0, -143(%rdi) L(fwd_write_127bytes): - lddqu -127(%rsi), %xmm0 - movdqu %xmm0, -127(%rdi) + lddqu -127(%rsi), %xmm0 + movdqu %xmm0, -127(%rdi) L(fwd_write_111bytes): - lddqu -111(%rsi), %xmm0 - movdqu %xmm0, -111(%rdi) + lddqu -111(%rsi), %xmm0 + movdqu %xmm0, -111(%rdi) L(fwd_write_95bytes): - lddqu -95(%rsi), %xmm0 - movdqu %xmm0, -95(%rdi) + lddqu -95(%rsi), %xmm0 + movdqu %xmm0, -95(%rdi) L(fwd_write_79bytes): - lddqu -79(%rsi), %xmm0 - movdqu %xmm0, -79(%rdi) + lddqu -79(%rsi), %xmm0 + movdqu %xmm0, -79(%rdi) L(fwd_write_63bytes): - lddqu -63(%rsi), %xmm0 - movdqu %xmm0, -63(%rdi) + lddqu -63(%rsi), %xmm0 + movdqu %xmm0, -63(%rdi) L(fwd_write_47bytes): - lddqu -47(%rsi), %xmm0 - movdqu %xmm0, -47(%rdi) + lddqu -47(%rsi), %xmm0 + movdqu %xmm0, -47(%rdi) L(fwd_write_31bytes): - lddqu -31(%rsi), %xmm0 - lddqu -16(%rsi), %xmm1 - movdqu %xmm0, -31(%rdi) - movdqu %xmm1, -16(%rdi) - ret + lddqu -31(%rsi), %xmm0 + lddqu -16(%rsi), %xmm1 + movdqu %xmm0, -31(%rdi) + movdqu %xmm1, -16(%rdi) + ret - .p2align 4 + .p2align 4 L(fwd_write_15bytes): - mov -15(%rsi), %rdx - mov -8(%rsi), %rcx - mov %rdx, -15(%rdi) - mov %rcx, -8(%rdi) - ret + mov -15(%rsi), %rdx + mov -8(%rsi), %rcx + mov %rdx, -15(%rdi) + mov %rcx, -8(%rdi) + ret - .p2align 4 + .p2align 4 L(fwd_write_142bytes): - lddqu -142(%rsi), %xmm0 - movdqu %xmm0, -142(%rdi) + lddqu -142(%rsi), %xmm0 + movdqu %xmm0, -142(%rdi) L(fwd_write_126bytes): - lddqu -126(%rsi), %xmm0 - movdqu %xmm0, -126(%rdi) + lddqu -126(%rsi), %xmm0 + movdqu %xmm0, -126(%rdi) L(fwd_write_110bytes): - lddqu -110(%rsi), %xmm0 - movdqu %xmm0, -110(%rdi) + lddqu -110(%rsi), %xmm0 + movdqu %xmm0, -110(%rdi) L(fwd_write_94bytes): - lddqu -94(%rsi), %xmm0 - movdqu %xmm0, -94(%rdi) + lddqu -94(%rsi), %xmm0 + movdqu %xmm0, -94(%rdi) L(fwd_write_78bytes): - lddqu -78(%rsi), %xmm0 - movdqu %xmm0, -78(%rdi) + lddqu -78(%rsi), %xmm0 + movdqu %xmm0, -78(%rdi) L(fwd_write_62bytes): - lddqu -62(%rsi), %xmm0 - movdqu %xmm0, -62(%rdi) + lddqu -62(%rsi), %xmm0 + movdqu %xmm0, -62(%rdi) L(fwd_write_46bytes): - lddqu -46(%rsi), %xmm0 - movdqu %xmm0, -46(%rdi) + lddqu -46(%rsi), %xmm0 + movdqu %xmm0, -46(%rdi) L(fwd_write_30bytes): - lddqu -30(%rsi), %xmm0 - lddqu -16(%rsi), %xmm1 - movdqu %xmm0, -30(%rdi) - movdqu %xmm1, -16(%rdi) - ret + lddqu -30(%rsi), %xmm0 + lddqu -16(%rsi), %xmm1 + movdqu %xmm0, -30(%rdi) + movdqu %xmm1, -16(%rdi) + ret - .p2align 4 + .p2align 4 L(fwd_write_14bytes): - mov -14(%rsi), %rdx - mov -8(%rsi), %rcx - mov %rdx, -14(%rdi) - mov %rcx, -8(%rdi) - ret + mov -14(%rsi), %rdx + mov -8(%rsi), %rcx + mov %rdx, -14(%rdi) + mov %rcx, -8(%rdi) + ret - .p2align 4 + .p2align 4 L(fwd_write_141bytes): - lddqu -141(%rsi), %xmm0 - movdqu %xmm0, -141(%rdi) + lddqu -141(%rsi), %xmm0 + movdqu %xmm0, -141(%rdi) L(fwd_write_125bytes): - lddqu -125(%rsi), %xmm0 - movdqu %xmm0, -125(%rdi) + lddqu -125(%rsi), %xmm0 + movdqu %xmm0, -125(%rdi) L(fwd_write_109bytes): - lddqu -109(%rsi), %xmm0 - movdqu %xmm0, -109(%rdi) + lddqu -109(%rsi), %xmm0 + movdqu %xmm0, -109(%rdi) L(fwd_write_93bytes): - lddqu -93(%rsi), %xmm0 - movdqu %xmm0, -93(%rdi) + lddqu -93(%rsi), %xmm0 + movdqu %xmm0, -93(%rdi) L(fwd_write_77bytes): - lddqu -77(%rsi), %xmm0 - movdqu %xmm0, -77(%rdi) + lddqu -77(%rsi), %xmm0 + movdqu %xmm0, -77(%rdi) L(fwd_write_61bytes): - lddqu -61(%rsi), %xmm0 - movdqu %xmm0, -61(%rdi) + lddqu -61(%rsi), %xmm0 + movdqu %xmm0, -61(%rdi) L(fwd_write_45bytes): - lddqu -45(%rsi), %xmm0 - movdqu %xmm0, -45(%rdi) + lddqu -45(%rsi), %xmm0 + movdqu %xmm0, -45(%rdi) L(fwd_write_29bytes): - lddqu -29(%rsi), %xmm0 - lddqu -16(%rsi), %xmm1 - movdqu %xmm0, -29(%rdi) - movdqu %xmm1, -16(%rdi) - ret + lddqu -29(%rsi), %xmm0 + lddqu -16(%rsi), %xmm1 + movdqu %xmm0, -29(%rdi) + movdqu %xmm1, -16(%rdi) + ret - .p2align 4 + .p2align 4 L(fwd_write_13bytes): - mov -13(%rsi), %rdx - mov -8(%rsi), %rcx - mov %rdx, -13(%rdi) - mov %rcx, -8(%rdi) - ret + mov -13(%rsi), %rdx + mov -8(%rsi), %rcx + mov %rdx, -13(%rdi) + mov %rcx, -8(%rdi) + ret - .p2align 4 + .p2align 4 L(fwd_write_140bytes): - lddqu -140(%rsi), %xmm0 - movdqu %xmm0, -140(%rdi) + lddqu -140(%rsi), %xmm0 + movdqu %xmm0, -140(%rdi) L(fwd_write_124bytes): - lddqu -124(%rsi), %xmm0 - movdqu %xmm0, -124(%rdi) + lddqu -124(%rsi), %xmm0 + movdqu %xmm0, -124(%rdi) L(fwd_write_108bytes): - lddqu -108(%rsi), %xmm0 - movdqu %xmm0, -108(%rdi) + lddqu -108(%rsi), %xmm0 + movdqu %xmm0, -108(%rdi) L(fwd_write_92bytes): - lddqu -92(%rsi), %xmm0 - movdqu %xmm0, -92(%rdi) + lddqu -92(%rsi), %xmm0 + movdqu %xmm0, -92(%rdi) L(fwd_write_76bytes): - lddqu -76(%rsi), %xmm0 - movdqu %xmm0, -76(%rdi) + lddqu -76(%rsi), %xmm0 + movdqu %xmm0, -76(%rdi) L(fwd_write_60bytes): - lddqu -60(%rsi), %xmm0 - movdqu %xmm0, -60(%rdi) + lddqu -60(%rsi), %xmm0 + movdqu %xmm0, -60(%rdi) L(fwd_write_44bytes): - lddqu -44(%rsi), %xmm0 - movdqu %xmm0, -44(%rdi) + lddqu -44(%rsi), %xmm0 + movdqu %xmm0, -44(%rdi) L(fwd_write_28bytes): - lddqu -28(%rsi), %xmm0 - lddqu -16(%rsi), %xmm1 - movdqu %xmm0, -28(%rdi) - movdqu %xmm1, -16(%rdi) - ret + lddqu -28(%rsi), %xmm0 + lddqu -16(%rsi), %xmm1 + movdqu %xmm0, -28(%rdi) + movdqu %xmm1, -16(%rdi) + ret - .p2align 4 + .p2align 4 L(fwd_write_12bytes): - mov -12(%rsi), %rdx - mov -4(%rsi), %ecx - mov %rdx, -12(%rdi) - mov %ecx, -4(%rdi) - ret + mov -12(%rsi), %rdx + mov -4(%rsi), %ecx + mov %rdx, -12(%rdi) + mov %ecx, -4(%rdi) + ret - .p2align 4 + .p2align 4 L(fwd_write_139bytes): - lddqu -139(%rsi), %xmm0 - movdqu %xmm0, -139(%rdi) + lddqu -139(%rsi), %xmm0 + movdqu %xmm0, -139(%rdi) L(fwd_write_123bytes): - lddqu -123(%rsi), %xmm0 - movdqu %xmm0, -123(%rdi) + lddqu -123(%rsi), %xmm0 + movdqu %xmm0, -123(%rdi) L(fwd_write_107bytes): - lddqu -107(%rsi), %xmm0 - movdqu %xmm0, -107(%rdi) + lddqu -107(%rsi), %xmm0 + movdqu %xmm0, -107(%rdi) L(fwd_write_91bytes): - lddqu -91(%rsi), %xmm0 - movdqu %xmm0, -91(%rdi) + lddqu -91(%rsi), %xmm0 + movdqu %xmm0, -91(%rdi) L(fwd_write_75bytes): - lddqu -75(%rsi), %xmm0 - movdqu %xmm0, -75(%rdi) + lddqu -75(%rsi), %xmm0 + movdqu %xmm0, -75(%rdi) L(fwd_write_59bytes): - lddqu -59(%rsi), %xmm0 - movdqu %xmm0, -59(%rdi) + lddqu -59(%rsi), %xmm0 + movdqu %xmm0, -59(%rdi) L(fwd_write_43bytes): - lddqu -43(%rsi), %xmm0 - movdqu %xmm0, -43(%rdi) + lddqu -43(%rsi), %xmm0 + movdqu %xmm0, -43(%rdi) L(fwd_write_27bytes): - lddqu -27(%rsi), %xmm0 - lddqu -16(%rsi), %xmm1 - movdqu %xmm0, -27(%rdi) - movdqu %xmm1, -16(%rdi) - ret + lddqu -27(%rsi), %xmm0 + lddqu -16(%rsi), %xmm1 + movdqu %xmm0, -27(%rdi) + movdqu %xmm1, -16(%rdi) + ret - .p2align 4 + .p2align 4 L(fwd_write_11bytes): - mov -11(%rsi), %rdx - mov -4(%rsi), %ecx - mov %rdx, -11(%rdi) - mov %ecx, -4(%rdi) - ret + mov -11(%rsi), %rdx + mov -4(%rsi), %ecx + mov %rdx, -11(%rdi) + mov %ecx, -4(%rdi) + ret - .p2align 4 + .p2align 4 L(fwd_write_138bytes): - lddqu -138(%rsi), %xmm0 - movdqu %xmm0, -138(%rdi) + lddqu -138(%rsi), %xmm0 + movdqu %xmm0, -138(%rdi) L(fwd_write_122bytes): - lddqu -122(%rsi), %xmm0 - movdqu %xmm0, -122(%rdi) + lddqu -122(%rsi), %xmm0 + movdqu %xmm0, -122(%rdi) L(fwd_write_106bytes): - lddqu -106(%rsi), %xmm0 - movdqu %xmm0, -106(%rdi) + lddqu -106(%rsi), %xmm0 + movdqu %xmm0, -106(%rdi) L(fwd_write_90bytes): - lddqu -90(%rsi), %xmm0 - movdqu %xmm0, -90(%rdi) + lddqu -90(%rsi), %xmm0 + movdqu %xmm0, -90(%rdi) L(fwd_write_74bytes): - lddqu -74(%rsi), %xmm0 - movdqu %xmm0, -74(%rdi) + lddqu -74(%rsi), %xmm0 + movdqu %xmm0, -74(%rdi) L(fwd_write_58bytes): - lddqu -58(%rsi), %xmm0 - movdqu %xmm0, -58(%rdi) + lddqu -58(%rsi), %xmm0 + movdqu %xmm0, -58(%rdi) L(fwd_write_42bytes): - lddqu -42(%rsi), %xmm0 - movdqu %xmm0, -42(%rdi) + lddqu -42(%rsi), %xmm0 + movdqu %xmm0, -42(%rdi) L(fwd_write_26bytes): - lddqu -26(%rsi), %xmm0 - lddqu -16(%rsi), %xmm1 - movdqu %xmm0, -26(%rdi) - movdqu %xmm1, -16(%rdi) - ret + lddqu -26(%rsi), %xmm0 + lddqu -16(%rsi), %xmm1 + movdqu %xmm0, -26(%rdi) + movdqu %xmm1, -16(%rdi) + ret - .p2align 4 + .p2align 4 L(fwd_write_10bytes): - mov -10(%rsi), %rdx - mov -4(%rsi), %ecx - mov %rdx, -10(%rdi) - mov %ecx, -4(%rdi) - ret + mov -10(%rsi), %rdx + mov -4(%rsi), %ecx + mov %rdx, -10(%rdi) + mov %ecx, -4(%rdi) + ret - .p2align 4 + .p2align 4 L(fwd_write_137bytes): - lddqu -137(%rsi), %xmm0 - movdqu %xmm0, -137(%rdi) + lddqu -137(%rsi), %xmm0 + movdqu %xmm0, -137(%rdi) L(fwd_write_121bytes): - lddqu -121(%rsi), %xmm0 - movdqu %xmm0, -121(%rdi) + lddqu -121(%rsi), %xmm0 + movdqu %xmm0, -121(%rdi) L(fwd_write_105bytes): - lddqu -105(%rsi), %xmm0 - movdqu %xmm0, -105(%rdi) + lddqu -105(%rsi), %xmm0 + movdqu %xmm0, -105(%rdi) L(fwd_write_89bytes): - lddqu -89(%rsi), %xmm0 - movdqu %xmm0, -89(%rdi) + lddqu -89(%rsi), %xmm0 + movdqu %xmm0, -89(%rdi) L(fwd_write_73bytes): - lddqu -73(%rsi), %xmm0 - movdqu %xmm0, -73(%rdi) + lddqu -73(%rsi), %xmm0 + movdqu %xmm0, -73(%rdi) L(fwd_write_57bytes): - lddqu -57(%rsi), %xmm0 - movdqu %xmm0, -57(%rdi) + lddqu -57(%rsi), %xmm0 + movdqu %xmm0, -57(%rdi) L(fwd_write_41bytes): - lddqu -41(%rsi), %xmm0 - movdqu %xmm0, -41(%rdi) + lddqu -41(%rsi), %xmm0 + movdqu %xmm0, -41(%rdi) L(fwd_write_25bytes): - lddqu -25(%rsi), %xmm0 - lddqu -16(%rsi), %xmm1 - movdqu %xmm0, -25(%rdi) - movdqu %xmm1, -16(%rdi) - ret + lddqu -25(%rsi), %xmm0 + lddqu -16(%rsi), %xmm1 + movdqu %xmm0, -25(%rdi) + movdqu %xmm1, -16(%rdi) + ret - .p2align 4 + .p2align 4 L(fwd_write_9bytes): - mov -9(%rsi), %rdx - mov -4(%rsi), %ecx - mov %rdx, -9(%rdi) - mov %ecx, -4(%rdi) - ret + mov -9(%rsi), %rdx + mov -4(%rsi), %ecx + mov %rdx, -9(%rdi) + mov %ecx, -4(%rdi) + ret - .p2align 4 + .p2align 4 L(fwd_write_136bytes): - lddqu -136(%rsi), %xmm0 - movdqu %xmm0, -136(%rdi) + lddqu -136(%rsi), %xmm0 + movdqu %xmm0, -136(%rdi) L(fwd_write_120bytes): - lddqu -120(%rsi), %xmm0 - movdqu %xmm0, -120(%rdi) + lddqu -120(%rsi), %xmm0 + movdqu %xmm0, -120(%rdi) L(fwd_write_104bytes): - lddqu -104(%rsi), %xmm0 - movdqu %xmm0, -104(%rdi) + lddqu -104(%rsi), %xmm0 + movdqu %xmm0, -104(%rdi) L(fwd_write_88bytes): - lddqu -88(%rsi), %xmm0 - movdqu %xmm0, -88(%rdi) + lddqu -88(%rsi), %xmm0 + movdqu %xmm0, -88(%rdi) L(fwd_write_72bytes): - lddqu -72(%rsi), %xmm0 - movdqu %xmm0, -72(%rdi) + lddqu -72(%rsi), %xmm0 + movdqu %xmm0, -72(%rdi) L(fwd_write_56bytes): - lddqu -56(%rsi), %xmm0 - movdqu %xmm0, -56(%rdi) + lddqu -56(%rsi), %xmm0 + movdqu %xmm0, -56(%rdi) L(fwd_write_40bytes): - lddqu -40(%rsi), %xmm0 - movdqu %xmm0, -40(%rdi) + lddqu -40(%rsi), %xmm0 + movdqu %xmm0, -40(%rdi) L(fwd_write_24bytes): - lddqu -24(%rsi), %xmm0 - lddqu -16(%rsi), %xmm1 - movdqu %xmm0, -24(%rdi) - movdqu %xmm1, -16(%rdi) - ret + lddqu -24(%rsi), %xmm0 + lddqu -16(%rsi), %xmm1 + movdqu %xmm0, -24(%rdi) + movdqu %xmm1, -16(%rdi) + ret - .p2align 4 + .p2align 4 L(fwd_write_8bytes): - mov -8(%rsi), %rdx - mov %rdx, -8(%rdi) - ret + mov -8(%rsi), %rdx + mov %rdx, -8(%rdi) + ret - .p2align 4 + .p2align 4 L(fwd_write_135bytes): - lddqu -135(%rsi), %xmm0 - movdqu %xmm0, -135(%rdi) + lddqu -135(%rsi), %xmm0 + movdqu %xmm0, -135(%rdi) L(fwd_write_119bytes): - lddqu -119(%rsi), %xmm0 - movdqu %xmm0, -119(%rdi) + lddqu -119(%rsi), %xmm0 + movdqu %xmm0, -119(%rdi) L(fwd_write_103bytes): - lddqu -103(%rsi), %xmm0 - movdqu %xmm0, -103(%rdi) + lddqu -103(%rsi), %xmm0 + movdqu %xmm0, -103(%rdi) L(fwd_write_87bytes): - lddqu -87(%rsi), %xmm0 - movdqu %xmm0, -87(%rdi) + lddqu -87(%rsi), %xmm0 + movdqu %xmm0, -87(%rdi) L(fwd_write_71bytes): - lddqu -71(%rsi), %xmm0 - movdqu %xmm0, -71(%rdi) + lddqu -71(%rsi), %xmm0 + movdqu %xmm0, -71(%rdi) L(fwd_write_55bytes): - lddqu -55(%rsi), %xmm0 - movdqu %xmm0, -55(%rdi) + lddqu -55(%rsi), %xmm0 + movdqu %xmm0, -55(%rdi) L(fwd_write_39bytes): - lddqu -39(%rsi), %xmm0 - movdqu %xmm0, -39(%rdi) + lddqu -39(%rsi), %xmm0 + movdqu %xmm0, -39(%rdi) L(fwd_write_23bytes): - lddqu -23(%rsi), %xmm0 - lddqu -16(%rsi), %xmm1 - movdqu %xmm0, -23(%rdi) - movdqu %xmm1, -16(%rdi) - ret + lddqu -23(%rsi), %xmm0 + lddqu -16(%rsi), %xmm1 + movdqu %xmm0, -23(%rdi) + movdqu %xmm1, -16(%rdi) + ret - .p2align 4 + .p2align 4 L(fwd_write_7bytes): - mov -7(%rsi), %edx - mov -4(%rsi), %ecx - mov %edx, -7(%rdi) - mov %ecx, -4(%rdi) - ret + mov -7(%rsi), %edx + mov -4(%rsi), %ecx + mov %edx, -7(%rdi) + mov %ecx, -4(%rdi) + ret - .p2align 4 + .p2align 4 L(fwd_write_134bytes): - lddqu -134(%rsi), %xmm0 - movdqu %xmm0, -134(%rdi) + lddqu -134(%rsi), %xmm0 + movdqu %xmm0, -134(%rdi) L(fwd_write_118bytes): - lddqu -118(%rsi), %xmm0 - movdqu %xmm0, -118(%rdi) + lddqu -118(%rsi), %xmm0 + movdqu %xmm0, -118(%rdi) L(fwd_write_102bytes): - lddqu -102(%rsi), %xmm0 - movdqu %xmm0, -102(%rdi) + lddqu -102(%rsi), %xmm0 + movdqu %xmm0, -102(%rdi) L(fwd_write_86bytes): - lddqu -86(%rsi), %xmm0 - movdqu %xmm0, -86(%rdi) + lddqu -86(%rsi), %xmm0 + movdqu %xmm0, -86(%rdi) L(fwd_write_70bytes): - lddqu -70(%rsi), %xmm0 - movdqu %xmm0, -70(%rdi) + lddqu -70(%rsi), %xmm0 + movdqu %xmm0, -70(%rdi) L(fwd_write_54bytes): - lddqu -54(%rsi), %xmm0 - movdqu %xmm0, -54(%rdi) + lddqu -54(%rsi), %xmm0 + movdqu %xmm0, -54(%rdi) L(fwd_write_38bytes): - lddqu -38(%rsi), %xmm0 - movdqu %xmm0, -38(%rdi) + lddqu -38(%rsi), %xmm0 + movdqu %xmm0, -38(%rdi) L(fwd_write_22bytes): - lddqu -22(%rsi), %xmm0 - lddqu -16(%rsi), %xmm1 - movdqu %xmm0, -22(%rdi) - movdqu %xmm1, -16(%rdi) - ret + lddqu -22(%rsi), %xmm0 + lddqu -16(%rsi), %xmm1 + movdqu %xmm0, -22(%rdi) + movdqu %xmm1, -16(%rdi) + ret - .p2align 4 + .p2align 4 L(fwd_write_6bytes): - mov -6(%rsi), %edx - mov -4(%rsi), %ecx - mov %edx, -6(%rdi) - mov %ecx, -4(%rdi) - ret + mov -6(%rsi), %edx + mov -4(%rsi), %ecx + mov %edx, -6(%rdi) + mov %ecx, -4(%rdi) + ret - .p2align 4 + .p2align 4 L(fwd_write_133bytes): - lddqu -133(%rsi), %xmm0 - movdqu %xmm0, -133(%rdi) + lddqu -133(%rsi), %xmm0 + movdqu %xmm0, -133(%rdi) L(fwd_write_117bytes): - lddqu -117(%rsi), %xmm0 - movdqu %xmm0, -117(%rdi) + lddqu -117(%rsi), %xmm0 + movdqu %xmm0, -117(%rdi) L(fwd_write_101bytes): - lddqu -101(%rsi), %xmm0 - movdqu %xmm0, -101(%rdi) + lddqu -101(%rsi), %xmm0 + movdqu %xmm0, -101(%rdi) L(fwd_write_85bytes): - lddqu -85(%rsi), %xmm0 - movdqu %xmm0, -85(%rdi) + lddqu -85(%rsi), %xmm0 + movdqu %xmm0, -85(%rdi) L(fwd_write_69bytes): - lddqu -69(%rsi), %xmm0 - movdqu %xmm0, -69(%rdi) + lddqu -69(%rsi), %xmm0 + movdqu %xmm0, -69(%rdi) L(fwd_write_53bytes): - lddqu -53(%rsi), %xmm0 - movdqu %xmm0, -53(%rdi) + lddqu -53(%rsi), %xmm0 + movdqu %xmm0, -53(%rdi) L(fwd_write_37bytes): - lddqu -37(%rsi), %xmm0 - movdqu %xmm0, -37(%rdi) + lddqu -37(%rsi), %xmm0 + movdqu %xmm0, -37(%rdi) L(fwd_write_21bytes): - lddqu -21(%rsi), %xmm0 - lddqu -16(%rsi), %xmm1 - movdqu %xmm0, -21(%rdi) - movdqu %xmm1, -16(%rdi) - ret + lddqu -21(%rsi), %xmm0 + lddqu -16(%rsi), %xmm1 + movdqu %xmm0, -21(%rdi) + movdqu %xmm1, -16(%rdi) + ret - .p2align 4 + .p2align 4 L(fwd_write_5bytes): - mov -5(%rsi), %edx - mov -4(%rsi), %ecx - mov %edx, -5(%rdi) - mov %ecx, -4(%rdi) - ret + mov -5(%rsi), %edx + mov -4(%rsi), %ecx + mov %edx, -5(%rdi) + mov %ecx, -4(%rdi) + ret - .p2align 4 + .p2align 4 L(fwd_write_132bytes): - lddqu -132(%rsi), %xmm0 - movdqu %xmm0, -132(%rdi) + lddqu -132(%rsi), %xmm0 + movdqu %xmm0, -132(%rdi) L(fwd_write_116bytes): - lddqu -116(%rsi), %xmm0 - movdqu %xmm0, -116(%rdi) + lddqu -116(%rsi), %xmm0 + movdqu %xmm0, -116(%rdi) L(fwd_write_100bytes): - lddqu -100(%rsi), %xmm0 - movdqu %xmm0, -100(%rdi) + lddqu -100(%rsi), %xmm0 + movdqu %xmm0, -100(%rdi) L(fwd_write_84bytes): - lddqu -84(%rsi), %xmm0 - movdqu %xmm0, -84(%rdi) + lddqu -84(%rsi), %xmm0 + movdqu %xmm0, -84(%rdi) L(fwd_write_68bytes): - lddqu -68(%rsi), %xmm0 - movdqu %xmm0, -68(%rdi) + lddqu -68(%rsi), %xmm0 + movdqu %xmm0, -68(%rdi) L(fwd_write_52bytes): - lddqu -52(%rsi), %xmm0 - movdqu %xmm0, -52(%rdi) + lddqu -52(%rsi), %xmm0 + movdqu %xmm0, -52(%rdi) L(fwd_write_36bytes): - lddqu -36(%rsi), %xmm0 - movdqu %xmm0, -36(%rdi) + lddqu -36(%rsi), %xmm0 + movdqu %xmm0, -36(%rdi) L(fwd_write_20bytes): - lddqu -20(%rsi), %xmm0 - lddqu -16(%rsi), %xmm1 - movdqu %xmm0, -20(%rdi) - movdqu %xmm1, -16(%rdi) - ret + lddqu -20(%rsi), %xmm0 + lddqu -16(%rsi), %xmm1 + movdqu %xmm0, -20(%rdi) + movdqu %xmm1, -16(%rdi) + ret - .p2align 4 + .p2align 4 L(fwd_write_4bytes): - mov -4(%rsi), %edx - mov %edx, -4(%rdi) - ret + mov -4(%rsi), %edx + mov %edx, -4(%rdi) + ret - .p2align 4 + .p2align 4 L(fwd_write_131bytes): - lddqu -131(%rsi), %xmm0 - movdqu %xmm0, -131(%rdi) + lddqu -131(%rsi), %xmm0 + movdqu %xmm0, -131(%rdi) L(fwd_write_115bytes): - lddqu -115(%rsi), %xmm0 - movdqu %xmm0, -115(%rdi) + lddqu -115(%rsi), %xmm0 + movdqu %xmm0, -115(%rdi) L(fwd_write_99bytes): - lddqu -99(%rsi), %xmm0 - movdqu %xmm0, -99(%rdi) + lddqu -99(%rsi), %xmm0 + movdqu %xmm0, -99(%rdi) L(fwd_write_83bytes): - lddqu -83(%rsi), %xmm0 - movdqu %xmm0, -83(%rdi) + lddqu -83(%rsi), %xmm0 + movdqu %xmm0, -83(%rdi) L(fwd_write_67bytes): - lddqu -67(%rsi), %xmm0 - movdqu %xmm0, -67(%rdi) + lddqu -67(%rsi), %xmm0 + movdqu %xmm0, -67(%rdi) L(fwd_write_51bytes): - lddqu -51(%rsi), %xmm0 - movdqu %xmm0, -51(%rdi) + lddqu -51(%rsi), %xmm0 + movdqu %xmm0, -51(%rdi) L(fwd_write_35bytes): - lddqu -35(%rsi), %xmm0 - movdqu %xmm0, -35(%rdi) + lddqu -35(%rsi), %xmm0 + movdqu %xmm0, -35(%rdi) L(fwd_write_19bytes): - lddqu -19(%rsi), %xmm0 - lddqu -16(%rsi), %xmm1 - movdqu %xmm0, -19(%rdi) - movdqu %xmm1, -16(%rdi) - ret + lddqu -19(%rsi), %xmm0 + lddqu -16(%rsi), %xmm1 + movdqu %xmm0, -19(%rdi) + movdqu %xmm1, -16(%rdi) + ret - .p2align 4 + .p2align 4 L(fwd_write_3bytes): - mov -3(%rsi), %dx - mov -2(%rsi), %cx - mov %dx, -3(%rdi) - mov %cx, -2(%rdi) - ret + mov -3(%rsi), %dx + mov -2(%rsi), %cx + mov %dx, -3(%rdi) + mov %cx, -2(%rdi) + ret - .p2align 4 + .p2align 4 L(fwd_write_130bytes): - lddqu -130(%rsi), %xmm0 - movdqu %xmm0, -130(%rdi) + lddqu -130(%rsi), %xmm0 + movdqu %xmm0, -130(%rdi) L(fwd_write_114bytes): - lddqu -114(%rsi), %xmm0 - movdqu %xmm0, -114(%rdi) + lddqu -114(%rsi), %xmm0 + movdqu %xmm0, -114(%rdi) L(fwd_write_98bytes): - lddqu -98(%rsi), %xmm0 - movdqu %xmm0, -98(%rdi) + lddqu -98(%rsi), %xmm0 + movdqu %xmm0, -98(%rdi) L(fwd_write_82bytes): - lddqu -82(%rsi), %xmm0 - movdqu %xmm0, -82(%rdi) + lddqu -82(%rsi), %xmm0 + movdqu %xmm0, -82(%rdi) L(fwd_write_66bytes): - lddqu -66(%rsi), %xmm0 - movdqu %xmm0, -66(%rdi) + lddqu -66(%rsi), %xmm0 + movdqu %xmm0, -66(%rdi) L(fwd_write_50bytes): - lddqu -50(%rsi), %xmm0 - movdqu %xmm0, -50(%rdi) + lddqu -50(%rsi), %xmm0 + movdqu %xmm0, -50(%rdi) L(fwd_write_34bytes): - lddqu -34(%rsi), %xmm0 - movdqu %xmm0, -34(%rdi) + lddqu -34(%rsi), %xmm0 + movdqu %xmm0, -34(%rdi) L(fwd_write_18bytes): - lddqu -18(%rsi), %xmm0 - lddqu -16(%rsi), %xmm1 - movdqu %xmm0, -18(%rdi) - movdqu %xmm1, -16(%rdi) - ret + lddqu -18(%rsi), %xmm0 + lddqu -16(%rsi), %xmm1 + movdqu %xmm0, -18(%rdi) + movdqu %xmm1, -16(%rdi) + ret - .p2align 4 + .p2align 4 L(fwd_write_2bytes): - movzwl -2(%rsi), %edx - mov %dx, -2(%rdi) - ret + movzwl -2(%rsi), %edx + mov %dx, -2(%rdi) + ret - .p2align 4 + .p2align 4 L(fwd_write_129bytes): - lddqu -129(%rsi), %xmm0 - movdqu %xmm0, -129(%rdi) + lddqu -129(%rsi), %xmm0 + movdqu %xmm0, -129(%rdi) L(fwd_write_113bytes): - lddqu -113(%rsi), %xmm0 - movdqu %xmm0, -113(%rdi) + lddqu -113(%rsi), %xmm0 + movdqu %xmm0, -113(%rdi) L(fwd_write_97bytes): - lddqu -97(%rsi), %xmm0 - movdqu %xmm0, -97(%rdi) + lddqu -97(%rsi), %xmm0 + movdqu %xmm0, -97(%rdi) L(fwd_write_81bytes): - lddqu -81(%rsi), %xmm0 - movdqu %xmm0, -81(%rdi) + lddqu -81(%rsi), %xmm0 + movdqu %xmm0, -81(%rdi) L(fwd_write_65bytes): - lddqu -65(%rsi), %xmm0 - movdqu %xmm0, -65(%rdi) + lddqu -65(%rsi), %xmm0 + movdqu %xmm0, -65(%rdi) L(fwd_write_49bytes): - lddqu -49(%rsi), %xmm0 - movdqu %xmm0, -49(%rdi) + lddqu -49(%rsi), %xmm0 + movdqu %xmm0, -49(%rdi) L(fwd_write_33bytes): - lddqu -33(%rsi), %xmm0 - movdqu %xmm0, -33(%rdi) + lddqu -33(%rsi), %xmm0 + movdqu %xmm0, -33(%rdi) L(fwd_write_17bytes): - lddqu -17(%rsi), %xmm0 - lddqu -16(%rsi), %xmm1 - movdqu %xmm0, -17(%rdi) - movdqu %xmm1, -16(%rdi) - ret + lddqu -17(%rsi), %xmm0 + lddqu -16(%rsi), %xmm1 + movdqu %xmm0, -17(%rdi) + movdqu %xmm1, -16(%rdi) + ret - .p2align 4 + .p2align 4 L(fwd_write_1bytes): - movzbl -1(%rsi), %edx - mov %dl, -1(%rdi) - ret + movzbl -1(%rsi), %edx + mov %dl, -1(%rdi) + ret - .p2align 4 + .p2align 4 L(bwd_write_128bytes): - lddqu 112(%rsi), %xmm0 - movdqu %xmm0, 112(%rdi) + lddqu 112(%rsi), %xmm0 + movdqu %xmm0, 112(%rdi) L(bwd_write_112bytes): - lddqu 96(%rsi), %xmm0 - movdqu %xmm0, 96(%rdi) + lddqu 96(%rsi), %xmm0 + movdqu %xmm0, 96(%rdi) L(bwd_write_96bytes): - lddqu 80(%rsi), %xmm0 - movdqu %xmm0, 80(%rdi) + lddqu 80(%rsi), %xmm0 + movdqu %xmm0, 80(%rdi) L(bwd_write_80bytes): - lddqu 64(%rsi), %xmm0 - movdqu %xmm0, 64(%rdi) + lddqu 64(%rsi), %xmm0 + movdqu %xmm0, 64(%rdi) L(bwd_write_64bytes): - lddqu 48(%rsi), %xmm0 - movdqu %xmm0, 48(%rdi) + lddqu 48(%rsi), %xmm0 + movdqu %xmm0, 48(%rdi) L(bwd_write_48bytes): - lddqu 32(%rsi), %xmm0 - movdqu %xmm0, 32(%rdi) + lddqu 32(%rsi), %xmm0 + movdqu %xmm0, 32(%rdi) L(bwd_write_32bytes): - lddqu 16(%rsi), %xmm0 - movdqu %xmm0, 16(%rdi) + lddqu 16(%rsi), %xmm0 + movdqu %xmm0, 16(%rdi) L(bwd_write_16bytes): - lddqu (%rsi), %xmm0 - movdqu %xmm0, (%rdi) + lddqu (%rsi), %xmm0 + movdqu %xmm0, (%rdi) L(bwd_write_0bytes): - ret + ret - .p2align 4 + .p2align 4 L(bwd_write_143bytes): - lddqu 127(%rsi), %xmm0 - movdqu %xmm0, 127(%rdi) + lddqu 127(%rsi), %xmm0 + movdqu %xmm0, 127(%rdi) L(bwd_write_127bytes): - lddqu 111(%rsi), %xmm0 - movdqu %xmm0, 111(%rdi) + lddqu 111(%rsi), %xmm0 + movdqu %xmm0, 111(%rdi) L(bwd_write_111bytes): - lddqu 95(%rsi), %xmm0 - movdqu %xmm0, 95(%rdi) + lddqu 95(%rsi), %xmm0 + movdqu %xmm0, 95(%rdi) L(bwd_write_95bytes): - lddqu 79(%rsi), %xmm0 - movdqu %xmm0, 79(%rdi) + lddqu 79(%rsi), %xmm0 + movdqu %xmm0, 79(%rdi) L(bwd_write_79bytes): - lddqu 63(%rsi), %xmm0 - movdqu %xmm0, 63(%rdi) + lddqu 63(%rsi), %xmm0 + movdqu %xmm0, 63(%rdi) L(bwd_write_63bytes): - lddqu 47(%rsi), %xmm0 - movdqu %xmm0, 47(%rdi) + lddqu 47(%rsi), %xmm0 + movdqu %xmm0, 47(%rdi) L(bwd_write_47bytes): - lddqu 31(%rsi), %xmm0 - movdqu %xmm0, 31(%rdi) + lddqu 31(%rsi), %xmm0 + movdqu %xmm0, 31(%rdi) L(bwd_write_31bytes): - lddqu 15(%rsi), %xmm0 - lddqu (%rsi), %xmm1 - movdqu %xmm0, 15(%rdi) - movdqu %xmm1, (%rdi) - ret + lddqu 15(%rsi), %xmm0 + lddqu (%rsi), %xmm1 + movdqu %xmm0, 15(%rdi) + movdqu %xmm1, (%rdi) + ret - .p2align 4 + .p2align 4 L(bwd_write_15bytes): - mov 7(%rsi), %rdx - mov (%rsi), %rcx - mov %rdx, 7(%rdi) - mov %rcx, (%rdi) - ret + mov 7(%rsi), %rdx + mov (%rsi), %rcx + mov %rdx, 7(%rdi) + mov %rcx, (%rdi) + ret - .p2align 4 + .p2align 4 L(bwd_write_142bytes): - lddqu 126(%rsi), %xmm0 - movdqu %xmm0, 126(%rdi) + lddqu 126(%rsi), %xmm0 + movdqu %xmm0, 126(%rdi) L(bwd_write_126bytes): - lddqu 110(%rsi), %xmm0 - movdqu %xmm0, 110(%rdi) + lddqu 110(%rsi), %xmm0 + movdqu %xmm0, 110(%rdi) L(bwd_write_110bytes): - lddqu 94(%rsi), %xmm0 - movdqu %xmm0, 94(%rdi) + lddqu 94(%rsi), %xmm0 + movdqu %xmm0, 94(%rdi) L(bwd_write_94bytes): - lddqu 78(%rsi), %xmm0 - movdqu %xmm0, 78(%rdi) + lddqu 78(%rsi), %xmm0 + movdqu %xmm0, 78(%rdi) L(bwd_write_78bytes): - lddqu 62(%rsi), %xmm0 - movdqu %xmm0, 62(%rdi) + lddqu 62(%rsi), %xmm0 + movdqu %xmm0, 62(%rdi) L(bwd_write_62bytes): - lddqu 46(%rsi), %xmm0 - movdqu %xmm0, 46(%rdi) + lddqu 46(%rsi), %xmm0 + movdqu %xmm0, 46(%rdi) L(bwd_write_46bytes): - lddqu 30(%rsi), %xmm0 - movdqu %xmm0, 30(%rdi) + lddqu 30(%rsi), %xmm0 + movdqu %xmm0, 30(%rdi) L(bwd_write_30bytes): - lddqu 14(%rsi), %xmm0 - lddqu (%rsi), %xmm1 - movdqu %xmm0, 14(%rdi) - movdqu %xmm1, (%rdi) - ret + lddqu 14(%rsi), %xmm0 + lddqu (%rsi), %xmm1 + movdqu %xmm0, 14(%rdi) + movdqu %xmm1, (%rdi) + ret - .p2align 4 + .p2align 4 L(bwd_write_14bytes): - mov 6(%rsi), %rdx - mov (%rsi), %rcx - mov %rdx, 6(%rdi) - mov %rcx, (%rdi) - ret + mov 6(%rsi), %rdx + mov (%rsi), %rcx + mov %rdx, 6(%rdi) + mov %rcx, (%rdi) + ret - .p2align 4 + .p2align 4 L(bwd_write_141bytes): - lddqu 125(%rsi), %xmm0 - movdqu %xmm0, 125(%rdi) + lddqu 125(%rsi), %xmm0 + movdqu %xmm0, 125(%rdi) L(bwd_write_125bytes): - lddqu 109(%rsi), %xmm0 - movdqu %xmm0, 109(%rdi) + lddqu 109(%rsi), %xmm0 + movdqu %xmm0, 109(%rdi) L(bwd_write_109bytes): - lddqu 93(%rsi), %xmm0 - movdqu %xmm0, 93(%rdi) + lddqu 93(%rsi), %xmm0 + movdqu %xmm0, 93(%rdi) L(bwd_write_93bytes): - lddqu 77(%rsi), %xmm0 - movdqu %xmm0, 77(%rdi) + lddqu 77(%rsi), %xmm0 + movdqu %xmm0, 77(%rdi) L(bwd_write_77bytes): - lddqu 61(%rsi), %xmm0 - movdqu %xmm0, 61(%rdi) + lddqu 61(%rsi), %xmm0 + movdqu %xmm0, 61(%rdi) L(bwd_write_61bytes): - lddqu 45(%rsi), %xmm0 - movdqu %xmm0, 45(%rdi) + lddqu 45(%rsi), %xmm0 + movdqu %xmm0, 45(%rdi) L(bwd_write_45bytes): - lddqu 29(%rsi), %xmm0 - movdqu %xmm0, 29(%rdi) + lddqu 29(%rsi), %xmm0 + movdqu %xmm0, 29(%rdi) L(bwd_write_29bytes): - lddqu 13(%rsi), %xmm0 - lddqu (%rsi), %xmm1 - movdqu %xmm0, 13(%rdi) - movdqu %xmm1, (%rdi) - ret + lddqu 13(%rsi), %xmm0 + lddqu (%rsi), %xmm1 + movdqu %xmm0, 13(%rdi) + movdqu %xmm1, (%rdi) + ret - .p2align 4 + .p2align 4 L(bwd_write_13bytes): - mov 5(%rsi), %rdx - mov (%rsi), %rcx - mov %rdx, 5(%rdi) - mov %rcx, (%rdi) - ret + mov 5(%rsi), %rdx + mov (%rsi), %rcx + mov %rdx, 5(%rdi) + mov %rcx, (%rdi) + ret - .p2align 4 + .p2align 4 L(bwd_write_140bytes): - lddqu 124(%rsi), %xmm0 - movdqu %xmm0, 124(%rdi) + lddqu 124(%rsi), %xmm0 + movdqu %xmm0, 124(%rdi) L(bwd_write_124bytes): - lddqu 108(%rsi), %xmm0 - movdqu %xmm0, 108(%rdi) + lddqu 108(%rsi), %xmm0 + movdqu %xmm0, 108(%rdi) L(bwd_write_108bytes): - lddqu 92(%rsi), %xmm0 - movdqu %xmm0, 92(%rdi) + lddqu 92(%rsi), %xmm0 + movdqu %xmm0, 92(%rdi) L(bwd_write_92bytes): - lddqu 76(%rsi), %xmm0 - movdqu %xmm0, 76(%rdi) + lddqu 76(%rsi), %xmm0 + movdqu %xmm0, 76(%rdi) L(bwd_write_76bytes): - lddqu 60(%rsi), %xmm0 - movdqu %xmm0, 60(%rdi) + lddqu 60(%rsi), %xmm0 + movdqu %xmm0, 60(%rdi) L(bwd_write_60bytes): - lddqu 44(%rsi), %xmm0 - movdqu %xmm0, 44(%rdi) + lddqu 44(%rsi), %xmm0 + movdqu %xmm0, 44(%rdi) L(bwd_write_44bytes): - lddqu 28(%rsi), %xmm0 - movdqu %xmm0, 28(%rdi) + lddqu 28(%rsi), %xmm0 + movdqu %xmm0, 28(%rdi) L(bwd_write_28bytes): - lddqu 12(%rsi), %xmm0 - lddqu (%rsi), %xmm1 - movdqu %xmm0, 12(%rdi) - movdqu %xmm1, (%rdi) - ret + lddqu 12(%rsi), %xmm0 + lddqu (%rsi), %xmm1 + movdqu %xmm0, 12(%rdi) + movdqu %xmm1, (%rdi) + ret - .p2align 4 + .p2align 4 L(bwd_write_12bytes): - mov 4(%rsi), %rdx - mov (%rsi), %rcx - mov %rdx, 4(%rdi) - mov %rcx, (%rdi) - ret + mov 4(%rsi), %rdx + mov (%rsi), %rcx + mov %rdx, 4(%rdi) + mov %rcx, (%rdi) + ret - .p2align 4 + .p2align 4 L(bwd_write_139bytes): - lddqu 123(%rsi), %xmm0 - movdqu %xmm0, 123(%rdi) + lddqu 123(%rsi), %xmm0 + movdqu %xmm0, 123(%rdi) L(bwd_write_123bytes): - lddqu 107(%rsi), %xmm0 - movdqu %xmm0, 107(%rdi) + lddqu 107(%rsi), %xmm0 + movdqu %xmm0, 107(%rdi) L(bwd_write_107bytes): - lddqu 91(%rsi), %xmm0 - movdqu %xmm0, 91(%rdi) + lddqu 91(%rsi), %xmm0 + movdqu %xmm0, 91(%rdi) L(bwd_write_91bytes): - lddqu 75(%rsi), %xmm0 - movdqu %xmm0, 75(%rdi) + lddqu 75(%rsi), %xmm0 + movdqu %xmm0, 75(%rdi) L(bwd_write_75bytes): - lddqu 59(%rsi), %xmm0 - movdqu %xmm0, 59(%rdi) + lddqu 59(%rsi), %xmm0 + movdqu %xmm0, 59(%rdi) L(bwd_write_59bytes): - lddqu 43(%rsi), %xmm0 - movdqu %xmm0, 43(%rdi) + lddqu 43(%rsi), %xmm0 + movdqu %xmm0, 43(%rdi) L(bwd_write_43bytes): - lddqu 27(%rsi), %xmm0 - movdqu %xmm0, 27(%rdi) + lddqu 27(%rsi), %xmm0 + movdqu %xmm0, 27(%rdi) L(bwd_write_27bytes): - lddqu 11(%rsi), %xmm0 - lddqu (%rsi), %xmm1 - movdqu %xmm0, 11(%rdi) - movdqu %xmm1, (%rdi) - ret + lddqu 11(%rsi), %xmm0 + lddqu (%rsi), %xmm1 + movdqu %xmm0, 11(%rdi) + movdqu %xmm1, (%rdi) + ret - .p2align 4 + .p2align 4 L(bwd_write_11bytes): - mov 3(%rsi), %rdx - mov (%rsi), %rcx - mov %rdx, 3(%rdi) - mov %rcx, (%rdi) - ret + mov 3(%rsi), %rdx + mov (%rsi), %rcx + mov %rdx, 3(%rdi) + mov %rcx, (%rdi) + ret - .p2align 4 + .p2align 4 L(bwd_write_138bytes): - lddqu 122(%rsi), %xmm0 - movdqu %xmm0, 122(%rdi) + lddqu 122(%rsi), %xmm0 + movdqu %xmm0, 122(%rdi) L(bwd_write_122bytes): - lddqu 106(%rsi), %xmm0 - movdqu %xmm0, 106(%rdi) + lddqu 106(%rsi), %xmm0 + movdqu %xmm0, 106(%rdi) L(bwd_write_106bytes): - lddqu 90(%rsi), %xmm0 - movdqu %xmm0, 90(%rdi) + lddqu 90(%rsi), %xmm0 + movdqu %xmm0, 90(%rdi) L(bwd_write_90bytes): - lddqu 74(%rsi), %xmm0 - movdqu %xmm0, 74(%rdi) + lddqu 74(%rsi), %xmm0 + movdqu %xmm0, 74(%rdi) L(bwd_write_74bytes): - lddqu 58(%rsi), %xmm0 - movdqu %xmm0, 58(%rdi) + lddqu 58(%rsi), %xmm0 + movdqu %xmm0, 58(%rdi) L(bwd_write_58bytes): - lddqu 42(%rsi), %xmm0 - movdqu %xmm0, 42(%rdi) + lddqu 42(%rsi), %xmm0 + movdqu %xmm0, 42(%rdi) L(bwd_write_42bytes): - lddqu 26(%rsi), %xmm0 - movdqu %xmm0, 26(%rdi) + lddqu 26(%rsi), %xmm0 + movdqu %xmm0, 26(%rdi) L(bwd_write_26bytes): - lddqu 10(%rsi), %xmm0 - lddqu (%rsi), %xmm1 - movdqu %xmm0, 10(%rdi) - movdqu %xmm1, (%rdi) - ret + lddqu 10(%rsi), %xmm0 + lddqu (%rsi), %xmm1 + movdqu %xmm0, 10(%rdi) + movdqu %xmm1, (%rdi) + ret - .p2align 4 + .p2align 4 L(bwd_write_10bytes): - mov 2(%rsi), %rdx - mov (%rsi), %rcx - mov %rdx, 2(%rdi) - mov %rcx, (%rdi) - ret + mov 2(%rsi), %rdx + mov (%rsi), %rcx + mov %rdx, 2(%rdi) + mov %rcx, (%rdi) + ret - .p2align 4 + .p2align 4 L(bwd_write_137bytes): - lddqu 121(%rsi), %xmm0 - movdqu %xmm0, 121(%rdi) + lddqu 121(%rsi), %xmm0 + movdqu %xmm0, 121(%rdi) L(bwd_write_121bytes): - lddqu 105(%rsi), %xmm0 - movdqu %xmm0, 105(%rdi) + lddqu 105(%rsi), %xmm0 + movdqu %xmm0, 105(%rdi) L(bwd_write_105bytes): - lddqu 89(%rsi), %xmm0 - movdqu %xmm0, 89(%rdi) + lddqu 89(%rsi), %xmm0 + movdqu %xmm0, 89(%rdi) L(bwd_write_89bytes): - lddqu 73(%rsi), %xmm0 - movdqu %xmm0, 73(%rdi) + lddqu 73(%rsi), %xmm0 + movdqu %xmm0, 73(%rdi) L(bwd_write_73bytes): - lddqu 57(%rsi), %xmm0 - movdqu %xmm0, 57(%rdi) + lddqu 57(%rsi), %xmm0 + movdqu %xmm0, 57(%rdi) L(bwd_write_57bytes): - lddqu 41(%rsi), %xmm0 - movdqu %xmm0, 41(%rdi) + lddqu 41(%rsi), %xmm0 + movdqu %xmm0, 41(%rdi) L(bwd_write_41bytes): - lddqu 25(%rsi), %xmm0 - movdqu %xmm0, 25(%rdi) + lddqu 25(%rsi), %xmm0 + movdqu %xmm0, 25(%rdi) L(bwd_write_25bytes): - lddqu 9(%rsi), %xmm0 - lddqu (%rsi), %xmm1 - movdqu %xmm0, 9(%rdi) - movdqu %xmm1, (%rdi) - ret + lddqu 9(%rsi), %xmm0 + lddqu (%rsi), %xmm1 + movdqu %xmm0, 9(%rdi) + movdqu %xmm1, (%rdi) + ret - .p2align 4 + .p2align 4 L(bwd_write_9bytes): - mov 1(%rsi), %rdx - mov (%rsi), %rcx - mov %rdx, 1(%rdi) - mov %rcx, (%rdi) - ret + mov 1(%rsi), %rdx + mov (%rsi), %rcx + mov %rdx, 1(%rdi) + mov %rcx, (%rdi) + ret - .p2align 4 + .p2align 4 L(bwd_write_136bytes): - lddqu 120(%rsi), %xmm0 - movdqu %xmm0, 120(%rdi) + lddqu 120(%rsi), %xmm0 + movdqu %xmm0, 120(%rdi) L(bwd_write_120bytes): - lddqu 104(%rsi), %xmm0 - movdqu %xmm0, 104(%rdi) + lddqu 104(%rsi), %xmm0 + movdqu %xmm0, 104(%rdi) L(bwd_write_104bytes): - lddqu 88(%rsi), %xmm0 - movdqu %xmm0, 88(%rdi) + lddqu 88(%rsi), %xmm0 + movdqu %xmm0, 88(%rdi) L(bwd_write_88bytes): - lddqu 72(%rsi), %xmm0 - movdqu %xmm0, 72(%rdi) + lddqu 72(%rsi), %xmm0 + movdqu %xmm0, 72(%rdi) L(bwd_write_72bytes): - lddqu 56(%rsi), %xmm0 - movdqu %xmm0, 56(%rdi) + lddqu 56(%rsi), %xmm0 + movdqu %xmm0, 56(%rdi) L(bwd_write_56bytes): - lddqu 40(%rsi), %xmm0 - movdqu %xmm0, 40(%rdi) + lddqu 40(%rsi), %xmm0 + movdqu %xmm0, 40(%rdi) L(bwd_write_40bytes): - lddqu 24(%rsi), %xmm0 - movdqu %xmm0, 24(%rdi) + lddqu 24(%rsi), %xmm0 + movdqu %xmm0, 24(%rdi) L(bwd_write_24bytes): - lddqu 8(%rsi), %xmm0 - lddqu (%rsi), %xmm1 - movdqu %xmm0, 8(%rdi) - movdqu %xmm1, (%rdi) - ret + lddqu 8(%rsi), %xmm0 + lddqu (%rsi), %xmm1 + movdqu %xmm0, 8(%rdi) + movdqu %xmm1, (%rdi) + ret - .p2align 4 + .p2align 4 L(bwd_write_8bytes): - mov (%rsi), %rdx - mov %rdx, (%rdi) - ret + mov (%rsi), %rdx + mov %rdx, (%rdi) + ret - .p2align 4 + .p2align 4 L(bwd_write_135bytes): - lddqu 119(%rsi), %xmm0 - movdqu %xmm0, 119(%rdi) + lddqu 119(%rsi), %xmm0 + movdqu %xmm0, 119(%rdi) L(bwd_write_119bytes): - lddqu 103(%rsi), %xmm0 - movdqu %xmm0, 103(%rdi) + lddqu 103(%rsi), %xmm0 + movdqu %xmm0, 103(%rdi) L(bwd_write_103bytes): - lddqu 87(%rsi), %xmm0 - movdqu %xmm0, 87(%rdi) + lddqu 87(%rsi), %xmm0 + movdqu %xmm0, 87(%rdi) L(bwd_write_87bytes): - lddqu 71(%rsi), %xmm0 - movdqu %xmm0, 71(%rdi) + lddqu 71(%rsi), %xmm0 + movdqu %xmm0, 71(%rdi) L(bwd_write_71bytes): - lddqu 55(%rsi), %xmm0 - movdqu %xmm0, 55(%rdi) + lddqu 55(%rsi), %xmm0 + movdqu %xmm0, 55(%rdi) L(bwd_write_55bytes): - lddqu 39(%rsi), %xmm0 - movdqu %xmm0, 39(%rdi) + lddqu 39(%rsi), %xmm0 + movdqu %xmm0, 39(%rdi) L(bwd_write_39bytes): - lddqu 23(%rsi), %xmm0 - movdqu %xmm0, 23(%rdi) + lddqu 23(%rsi), %xmm0 + movdqu %xmm0, 23(%rdi) L(bwd_write_23bytes): - lddqu 7(%rsi), %xmm0 - lddqu (%rsi), %xmm1 - movdqu %xmm0, 7(%rdi) - movdqu %xmm1, (%rdi) - ret + lddqu 7(%rsi), %xmm0 + lddqu (%rsi), %xmm1 + movdqu %xmm0, 7(%rdi) + movdqu %xmm1, (%rdi) + ret - .p2align 4 + .p2align 4 L(bwd_write_7bytes): - mov 3(%rsi), %edx - mov (%rsi), %ecx - mov %edx, 3(%rdi) - mov %ecx, (%rdi) - ret + mov 3(%rsi), %edx + mov (%rsi), %ecx + mov %edx, 3(%rdi) + mov %ecx, (%rdi) + ret - .p2align 4 + .p2align 4 L(bwd_write_134bytes): - lddqu 118(%rsi), %xmm0 - movdqu %xmm0, 118(%rdi) + lddqu 118(%rsi), %xmm0 + movdqu %xmm0, 118(%rdi) L(bwd_write_118bytes): - lddqu 102(%rsi), %xmm0 - movdqu %xmm0, 102(%rdi) + lddqu 102(%rsi), %xmm0 + movdqu %xmm0, 102(%rdi) L(bwd_write_102bytes): - lddqu 86(%rsi), %xmm0 - movdqu %xmm0, 86(%rdi) + lddqu 86(%rsi), %xmm0 + movdqu %xmm0, 86(%rdi) L(bwd_write_86bytes): - lddqu 70(%rsi), %xmm0 - movdqu %xmm0, 70(%rdi) + lddqu 70(%rsi), %xmm0 + movdqu %xmm0, 70(%rdi) L(bwd_write_70bytes): - lddqu 54(%rsi), %xmm0 - movdqu %xmm0, 54(%rdi) + lddqu 54(%rsi), %xmm0 + movdqu %xmm0, 54(%rdi) L(bwd_write_54bytes): - lddqu 38(%rsi), %xmm0 - movdqu %xmm0, 38(%rdi) + lddqu 38(%rsi), %xmm0 + movdqu %xmm0, 38(%rdi) L(bwd_write_38bytes): - lddqu 22(%rsi), %xmm0 - movdqu %xmm0, 22(%rdi) + lddqu 22(%rsi), %xmm0 + movdqu %xmm0, 22(%rdi) L(bwd_write_22bytes): - lddqu 6(%rsi), %xmm0 - lddqu (%rsi), %xmm1 - movdqu %xmm0, 6(%rdi) - movdqu %xmm1, (%rdi) - ret + lddqu 6(%rsi), %xmm0 + lddqu (%rsi), %xmm1 + movdqu %xmm0, 6(%rdi) + movdqu %xmm1, (%rdi) + ret - .p2align 4 + .p2align 4 L(bwd_write_6bytes): - mov 2(%rsi), %edx - mov (%rsi), %ecx - mov %edx, 2(%rdi) - mov %ecx, (%rdi) - ret + mov 2(%rsi), %edx + mov (%rsi), %ecx + mov %edx, 2(%rdi) + mov %ecx, (%rdi) + ret - .p2align 4 + .p2align 4 L(bwd_write_133bytes): - lddqu 117(%rsi), %xmm0 - movdqu %xmm0, 117(%rdi) + lddqu 117(%rsi), %xmm0 + movdqu %xmm0, 117(%rdi) L(bwd_write_117bytes): - lddqu 101(%rsi), %xmm0 - movdqu %xmm0, 101(%rdi) + lddqu 101(%rsi), %xmm0 + movdqu %xmm0, 101(%rdi) L(bwd_write_101bytes): - lddqu 85(%rsi), %xmm0 - movdqu %xmm0, 85(%rdi) + lddqu 85(%rsi), %xmm0 + movdqu %xmm0, 85(%rdi) L(bwd_write_85bytes): - lddqu 69(%rsi), %xmm0 - movdqu %xmm0, 69(%rdi) + lddqu 69(%rsi), %xmm0 + movdqu %xmm0, 69(%rdi) L(bwd_write_69bytes): - lddqu 53(%rsi), %xmm0 - movdqu %xmm0, 53(%rdi) + lddqu 53(%rsi), %xmm0 + movdqu %xmm0, 53(%rdi) L(bwd_write_53bytes): - lddqu 37(%rsi), %xmm0 - movdqu %xmm0, 37(%rdi) + lddqu 37(%rsi), %xmm0 + movdqu %xmm0, 37(%rdi) L(bwd_write_37bytes): - lddqu 21(%rsi), %xmm0 - movdqu %xmm0, 21(%rdi) + lddqu 21(%rsi), %xmm0 + movdqu %xmm0, 21(%rdi) L(bwd_write_21bytes): - lddqu 5(%rsi), %xmm0 - lddqu (%rsi), %xmm1 - movdqu %xmm0, 5(%rdi) - movdqu %xmm1, (%rdi) - ret + lddqu 5(%rsi), %xmm0 + lddqu (%rsi), %xmm1 + movdqu %xmm0, 5(%rdi) + movdqu %xmm1, (%rdi) + ret - .p2align 4 + .p2align 4 L(bwd_write_5bytes): - mov 1(%rsi), %edx - mov (%rsi), %ecx - mov %edx, 1(%rdi) - mov %ecx, (%rdi) - ret + mov 1(%rsi), %edx + mov (%rsi), %ecx + mov %edx, 1(%rdi) + mov %ecx, (%rdi) + ret - .p2align 4 + .p2align 4 L(bwd_write_132bytes): - lddqu 116(%rsi), %xmm0 - movdqu %xmm0, 116(%rdi) + lddqu 116(%rsi), %xmm0 + movdqu %xmm0, 116(%rdi) L(bwd_write_116bytes): - lddqu 100(%rsi), %xmm0 - movdqu %xmm0, 100(%rdi) + lddqu 100(%rsi), %xmm0 + movdqu %xmm0, 100(%rdi) L(bwd_write_100bytes): - lddqu 84(%rsi), %xmm0 - movdqu %xmm0, 84(%rdi) + lddqu 84(%rsi), %xmm0 + movdqu %xmm0, 84(%rdi) L(bwd_write_84bytes): - lddqu 68(%rsi), %xmm0 - movdqu %xmm0, 68(%rdi) + lddqu 68(%rsi), %xmm0 + movdqu %xmm0, 68(%rdi) L(bwd_write_68bytes): - lddqu 52(%rsi), %xmm0 - movdqu %xmm0, 52(%rdi) + lddqu 52(%rsi), %xmm0 + movdqu %xmm0, 52(%rdi) L(bwd_write_52bytes): - lddqu 36(%rsi), %xmm0 - movdqu %xmm0, 36(%rdi) + lddqu 36(%rsi), %xmm0 + movdqu %xmm0, 36(%rdi) L(bwd_write_36bytes): - lddqu 20(%rsi), %xmm0 - movdqu %xmm0, 20(%rdi) + lddqu 20(%rsi), %xmm0 + movdqu %xmm0, 20(%rdi) L(bwd_write_20bytes): - lddqu 4(%rsi), %xmm0 - lddqu (%rsi), %xmm1 - movdqu %xmm0, 4(%rdi) - movdqu %xmm1, (%rdi) - ret + lddqu 4(%rsi), %xmm0 + lddqu (%rsi), %xmm1 + movdqu %xmm0, 4(%rdi) + movdqu %xmm1, (%rdi) + ret - .p2align 4 + .p2align 4 L(bwd_write_4bytes): - mov (%rsi), %edx - mov %edx, (%rdi) - ret + mov (%rsi), %edx + mov %edx, (%rdi) + ret - .p2align 4 + .p2align 4 L(bwd_write_131bytes): - lddqu 115(%rsi), %xmm0 - movdqu %xmm0, 115(%rdi) + lddqu 115(%rsi), %xmm0 + movdqu %xmm0, 115(%rdi) L(bwd_write_115bytes): - lddqu 99(%rsi), %xmm0 - movdqu %xmm0, 99(%rdi) + lddqu 99(%rsi), %xmm0 + movdqu %xmm0, 99(%rdi) L(bwd_write_99bytes): - lddqu 83(%rsi), %xmm0 - movdqu %xmm0, 83(%rdi) + lddqu 83(%rsi), %xmm0 + movdqu %xmm0, 83(%rdi) L(bwd_write_83bytes): - lddqu 67(%rsi), %xmm0 - movdqu %xmm0, 67(%rdi) + lddqu 67(%rsi), %xmm0 + movdqu %xmm0, 67(%rdi) L(bwd_write_67bytes): - lddqu 51(%rsi), %xmm0 - movdqu %xmm0, 51(%rdi) + lddqu 51(%rsi), %xmm0 + movdqu %xmm0, 51(%rdi) L(bwd_write_51bytes): - lddqu 35(%rsi), %xmm0 - movdqu %xmm0, 35(%rdi) + lddqu 35(%rsi), %xmm0 + movdqu %xmm0, 35(%rdi) L(bwd_write_35bytes): - lddqu 19(%rsi), %xmm0 - movdqu %xmm0, 19(%rdi) + lddqu 19(%rsi), %xmm0 + movdqu %xmm0, 19(%rdi) L(bwd_write_19bytes): - lddqu 3(%rsi), %xmm0 - lddqu (%rsi), %xmm1 - movdqu %xmm0, 3(%rdi) - movdqu %xmm1, (%rdi) - ret + lddqu 3(%rsi), %xmm0 + lddqu (%rsi), %xmm1 + movdqu %xmm0, 3(%rdi) + movdqu %xmm1, (%rdi) + ret - .p2align 4 + .p2align 4 L(bwd_write_3bytes): - mov 1(%rsi), %dx - mov (%rsi), %cx - mov %dx, 1(%rdi) - mov %cx, (%rdi) - ret + mov 1(%rsi), %dx + mov (%rsi), %cx + mov %dx, 1(%rdi) + mov %cx, (%rdi) + ret - .p2align 4 + .p2align 4 L(bwd_write_130bytes): - lddqu 114(%rsi), %xmm0 - movdqu %xmm0, 114(%rdi) + lddqu 114(%rsi), %xmm0 + movdqu %xmm0, 114(%rdi) L(bwd_write_114bytes): - lddqu 98(%rsi), %xmm0 - movdqu %xmm0, 98(%rdi) + lddqu 98(%rsi), %xmm0 + movdqu %xmm0, 98(%rdi) L(bwd_write_98bytes): - lddqu 82(%rsi), %xmm0 - movdqu %xmm0, 82(%rdi) + lddqu 82(%rsi), %xmm0 + movdqu %xmm0, 82(%rdi) L(bwd_write_82bytes): - lddqu 66(%rsi), %xmm0 - movdqu %xmm0, 66(%rdi) + lddqu 66(%rsi), %xmm0 + movdqu %xmm0, 66(%rdi) L(bwd_write_66bytes): - lddqu 50(%rsi), %xmm0 - movdqu %xmm0, 50(%rdi) + lddqu 50(%rsi), %xmm0 + movdqu %xmm0, 50(%rdi) L(bwd_write_50bytes): - lddqu 34(%rsi), %xmm0 - movdqu %xmm0, 34(%rdi) + lddqu 34(%rsi), %xmm0 + movdqu %xmm0, 34(%rdi) L(bwd_write_34bytes): - lddqu 18(%rsi), %xmm0 - movdqu %xmm0, 18(%rdi) + lddqu 18(%rsi), %xmm0 + movdqu %xmm0, 18(%rdi) L(bwd_write_18bytes): - lddqu 2(%rsi), %xmm0 - lddqu (%rsi), %xmm1 - movdqu %xmm0, 2(%rdi) - movdqu %xmm1, (%rdi) - ret + lddqu 2(%rsi), %xmm0 + lddqu (%rsi), %xmm1 + movdqu %xmm0, 2(%rdi) + movdqu %xmm1, (%rdi) + ret - .p2align 4 + .p2align 4 L(bwd_write_2bytes): - movzwl (%rsi), %edx - mov %dx, (%rdi) - ret + movzwl (%rsi), %edx + mov %dx, (%rdi) + ret - .p2align 4 + .p2align 4 L(bwd_write_129bytes): - lddqu 113(%rsi), %xmm0 - movdqu %xmm0, 113(%rdi) + lddqu 113(%rsi), %xmm0 + movdqu %xmm0, 113(%rdi) L(bwd_write_113bytes): - lddqu 97(%rsi), %xmm0 - movdqu %xmm0, 97(%rdi) + lddqu 97(%rsi), %xmm0 + movdqu %xmm0, 97(%rdi) L(bwd_write_97bytes): - lddqu 81(%rsi), %xmm0 - movdqu %xmm0, 81(%rdi) + lddqu 81(%rsi), %xmm0 + movdqu %xmm0, 81(%rdi) L(bwd_write_81bytes): - lddqu 65(%rsi), %xmm0 - movdqu %xmm0, 65(%rdi) + lddqu 65(%rsi), %xmm0 + movdqu %xmm0, 65(%rdi) L(bwd_write_65bytes): - lddqu 49(%rsi), %xmm0 - movdqu %xmm0, 49(%rdi) + lddqu 49(%rsi), %xmm0 + movdqu %xmm0, 49(%rdi) L(bwd_write_49bytes): - lddqu 33(%rsi), %xmm0 - movdqu %xmm0, 33(%rdi) + lddqu 33(%rsi), %xmm0 + movdqu %xmm0, 33(%rdi) L(bwd_write_33bytes): - lddqu 17(%rsi), %xmm0 - movdqu %xmm0, 17(%rdi) + lddqu 17(%rsi), %xmm0 + movdqu %xmm0, 17(%rdi) L(bwd_write_17bytes): - lddqu 1(%rsi), %xmm0 - lddqu (%rsi), %xmm1 - movdqu %xmm0, 1(%rdi) - movdqu %xmm1, (%rdi) - ret + lddqu 1(%rsi), %xmm0 + lddqu (%rsi), %xmm1 + movdqu %xmm0, 1(%rdi) + movdqu %xmm1, (%rdi) + ret - .p2align 4 + .p2align 4 L(bwd_write_1bytes): - movzbl (%rsi), %edx - mov %dl, (%rdi) - ret + movzbl (%rsi), %edx + mov %dl, (%rdi) + ret END (MEMCPY) - .section .rodata.ssse3,"a",@progbits - .p2align 3 + .section .rodata.ssse3,"a",@progbits + .p2align 3 L(table_144_bytes_bwd): - .int JMPTBL (L(bwd_write_0bytes), L(table_144_bytes_bwd)) - .int JMPTBL (L(bwd_write_1bytes), L(table_144_bytes_bwd)) - .int JMPTBL (L(bwd_write_2bytes), L(table_144_bytes_bwd)) - .int JMPTBL (L(bwd_write_3bytes), L(table_144_bytes_bwd)) - .int JMPTBL (L(bwd_write_4bytes), L(table_144_bytes_bwd)) - .int JMPTBL (L(bwd_write_5bytes), L(table_144_bytes_bwd)) - .int JMPTBL (L(bwd_write_6bytes), L(table_144_bytes_bwd)) - .int JMPTBL (L(bwd_write_7bytes), L(table_144_bytes_bwd)) - .int JMPTBL (L(bwd_write_8bytes), L(table_144_bytes_bwd)) - .int JMPTBL (L(bwd_write_9bytes), L(table_144_bytes_bwd)) - .int JMPTBL (L(bwd_write_10bytes), L(table_144_bytes_bwd)) - .int JMPTBL (L(bwd_write_11bytes), L(table_144_bytes_bwd)) - .int JMPTBL (L(bwd_write_12bytes), L(table_144_bytes_bwd)) - .int JMPTBL (L(bwd_write_13bytes), L(table_144_bytes_bwd)) - .int JMPTBL (L(bwd_write_14bytes), L(table_144_bytes_bwd)) - .int JMPTBL (L(bwd_write_15bytes), L(table_144_bytes_bwd)) - .int JMPTBL (L(bwd_write_16bytes), L(table_144_bytes_bwd)) - .int JMPTBL (L(bwd_write_17bytes), L(table_144_bytes_bwd)) - .int JMPTBL (L(bwd_write_18bytes), L(table_144_bytes_bwd)) - .int JMPTBL (L(bwd_write_19bytes), L(table_144_bytes_bwd)) - .int JMPTBL (L(bwd_write_20bytes), L(table_144_bytes_bwd)) - .int JMPTBL (L(bwd_write_21bytes), L(table_144_bytes_bwd)) - .int JMPTBL (L(bwd_write_22bytes), L(table_144_bytes_bwd)) - .int JMPTBL (L(bwd_write_23bytes), L(table_144_bytes_bwd)) - .int JMPTBL (L(bwd_write_24bytes), L(table_144_bytes_bwd)) - .int JMPTBL (L(bwd_write_25bytes), L(table_144_bytes_bwd)) - .int JMPTBL (L(bwd_write_26bytes), L(table_144_bytes_bwd)) - .int JMPTBL (L(bwd_write_27bytes), L(table_144_bytes_bwd)) - .int JMPTBL (L(bwd_write_28bytes), L(table_144_bytes_bwd)) - .int JMPTBL (L(bwd_write_29bytes), L(table_144_bytes_bwd)) - .int JMPTBL (L(bwd_write_30bytes), L(table_144_bytes_bwd)) - .int JMPTBL (L(bwd_write_31bytes), L(table_144_bytes_bwd)) - .int JMPTBL (L(bwd_write_32bytes), L(table_144_bytes_bwd)) - .int JMPTBL (L(bwd_write_33bytes), L(table_144_bytes_bwd)) - .int JMPTBL (L(bwd_write_34bytes), L(table_144_bytes_bwd)) - .int JMPTBL (L(bwd_write_35bytes), L(table_144_bytes_bwd)) - .int JMPTBL (L(bwd_write_36bytes), L(table_144_bytes_bwd)) - .int JMPTBL (L(bwd_write_37bytes), L(table_144_bytes_bwd)) - .int JMPTBL (L(bwd_write_38bytes), L(table_144_bytes_bwd)) - .int JMPTBL (L(bwd_write_39bytes), L(table_144_bytes_bwd)) - .int JMPTBL (L(bwd_write_40bytes), L(table_144_bytes_bwd)) - .int JMPTBL (L(bwd_write_41bytes), L(table_144_bytes_bwd)) - .int JMPTBL (L(bwd_write_42bytes), L(table_144_bytes_bwd)) - .int JMPTBL (L(bwd_write_43bytes), L(table_144_bytes_bwd)) - .int JMPTBL (L(bwd_write_44bytes), L(table_144_bytes_bwd)) - .int JMPTBL (L(bwd_write_45bytes), L(table_144_bytes_bwd)) - .int JMPTBL (L(bwd_write_46bytes), L(table_144_bytes_bwd)) - .int JMPTBL (L(bwd_write_47bytes), L(table_144_bytes_bwd)) - .int JMPTBL (L(bwd_write_48bytes), L(table_144_bytes_bwd)) - .int JMPTBL (L(bwd_write_49bytes), L(table_144_bytes_bwd)) - .int JMPTBL (L(bwd_write_50bytes), L(table_144_bytes_bwd)) - .int JMPTBL (L(bwd_write_51bytes), L(table_144_bytes_bwd)) - .int JMPTBL (L(bwd_write_52bytes), L(table_144_bytes_bwd)) - .int JMPTBL (L(bwd_write_53bytes), L(table_144_bytes_bwd)) - .int JMPTBL (L(bwd_write_54bytes), L(table_144_bytes_bwd)) - .int JMPTBL (L(bwd_write_55bytes), L(table_144_bytes_bwd)) - .int JMPTBL (L(bwd_write_56bytes), L(table_144_bytes_bwd)) - .int JMPTBL (L(bwd_write_57bytes), L(table_144_bytes_bwd)) - .int JMPTBL (L(bwd_write_58bytes), L(table_144_bytes_bwd)) - .int JMPTBL (L(bwd_write_59bytes), L(table_144_bytes_bwd)) - .int JMPTBL (L(bwd_write_60bytes), L(table_144_bytes_bwd)) - .int JMPTBL (L(bwd_write_61bytes), L(table_144_bytes_bwd)) - .int JMPTBL (L(bwd_write_62bytes), L(table_144_bytes_bwd)) - .int JMPTBL (L(bwd_write_63bytes), L(table_144_bytes_bwd)) - .int JMPTBL (L(bwd_write_64bytes), L(table_144_bytes_bwd)) - .int JMPTBL (L(bwd_write_65bytes), L(table_144_bytes_bwd)) - .int JMPTBL (L(bwd_write_66bytes), L(table_144_bytes_bwd)) - .int JMPTBL (L(bwd_write_67bytes), L(table_144_bytes_bwd)) - .int JMPTBL (L(bwd_write_68bytes), L(table_144_bytes_bwd)) - .int JMPTBL (L(bwd_write_69bytes), L(table_144_bytes_bwd)) - .int JMPTBL (L(bwd_write_70bytes), L(table_144_bytes_bwd)) - .int JMPTBL (L(bwd_write_71bytes), L(table_144_bytes_bwd)) - .int JMPTBL (L(bwd_write_72bytes), L(table_144_bytes_bwd)) - .int JMPTBL (L(bwd_write_73bytes), L(table_144_bytes_bwd)) - .int JMPTBL (L(bwd_write_74bytes), L(table_144_bytes_bwd)) - .int JMPTBL (L(bwd_write_75bytes), L(table_144_bytes_bwd)) - .int JMPTBL (L(bwd_write_76bytes), L(table_144_bytes_bwd)) - .int JMPTBL (L(bwd_write_77bytes), L(table_144_bytes_bwd)) - .int JMPTBL (L(bwd_write_78bytes), L(table_144_bytes_bwd)) - .int JMPTBL (L(bwd_write_79bytes), L(table_144_bytes_bwd)) - .int JMPTBL (L(bwd_write_80bytes), L(table_144_bytes_bwd)) - .int JMPTBL (L(bwd_write_81bytes), L(table_144_bytes_bwd)) - .int JMPTBL (L(bwd_write_82bytes), L(table_144_bytes_bwd)) - .int JMPTBL (L(bwd_write_83bytes), L(table_144_bytes_bwd)) - .int JMPTBL (L(bwd_write_84bytes), L(table_144_bytes_bwd)) - .int JMPTBL (L(bwd_write_85bytes), L(table_144_bytes_bwd)) - .int JMPTBL (L(bwd_write_86bytes), L(table_144_bytes_bwd)) - .int JMPTBL (L(bwd_write_87bytes), L(table_144_bytes_bwd)) - .int JMPTBL (L(bwd_write_88bytes), L(table_144_bytes_bwd)) - .int JMPTBL (L(bwd_write_89bytes), L(table_144_bytes_bwd)) - .int JMPTBL (L(bwd_write_90bytes), L(table_144_bytes_bwd)) - .int JMPTBL (L(bwd_write_91bytes), L(table_144_bytes_bwd)) - .int JMPTBL (L(bwd_write_92bytes), L(table_144_bytes_bwd)) - .int JMPTBL (L(bwd_write_93bytes), L(table_144_bytes_bwd)) - .int JMPTBL (L(bwd_write_94bytes), L(table_144_bytes_bwd)) - .int JMPTBL (L(bwd_write_95bytes), L(table_144_bytes_bwd)) - .int JMPTBL (L(bwd_write_96bytes), L(table_144_bytes_bwd)) - .int JMPTBL (L(bwd_write_97bytes), L(table_144_bytes_bwd)) - .int JMPTBL (L(bwd_write_98bytes), L(table_144_bytes_bwd)) - .int JMPTBL (L(bwd_write_99bytes), L(table_144_bytes_bwd)) - .int JMPTBL (L(bwd_write_100bytes), L(table_144_bytes_bwd)) - .int JMPTBL (L(bwd_write_101bytes), L(table_144_bytes_bwd)) - .int JMPTBL (L(bwd_write_102bytes), L(table_144_bytes_bwd)) - .int JMPTBL (L(bwd_write_103bytes), L(table_144_bytes_bwd)) - .int JMPTBL (L(bwd_write_104bytes), L(table_144_bytes_bwd)) - .int JMPTBL (L(bwd_write_105bytes), L(table_144_bytes_bwd)) - .int JMPTBL (L(bwd_write_106bytes), L(table_144_bytes_bwd)) - .int JMPTBL (L(bwd_write_107bytes), L(table_144_bytes_bwd)) - .int JMPTBL (L(bwd_write_108bytes), L(table_144_bytes_bwd)) - .int JMPTBL (L(bwd_write_109bytes), L(table_144_bytes_bwd)) - .int JMPTBL (L(bwd_write_110bytes), L(table_144_bytes_bwd)) - .int JMPTBL (L(bwd_write_111bytes), L(table_144_bytes_bwd)) - .int JMPTBL (L(bwd_write_112bytes), L(table_144_bytes_bwd)) - .int JMPTBL (L(bwd_write_113bytes), L(table_144_bytes_bwd)) - .int JMPTBL (L(bwd_write_114bytes), L(table_144_bytes_bwd)) - .int JMPTBL (L(bwd_write_115bytes), L(table_144_bytes_bwd)) - .int JMPTBL (L(bwd_write_116bytes), L(table_144_bytes_bwd)) - .int JMPTBL (L(bwd_write_117bytes), L(table_144_bytes_bwd)) - .int JMPTBL (L(bwd_write_118bytes), L(table_144_bytes_bwd)) - .int JMPTBL (L(bwd_write_119bytes), L(table_144_bytes_bwd)) - .int JMPTBL (L(bwd_write_120bytes), L(table_144_bytes_bwd)) - .int JMPTBL (L(bwd_write_121bytes), L(table_144_bytes_bwd)) - .int JMPTBL (L(bwd_write_122bytes), L(table_144_bytes_bwd)) - .int JMPTBL (L(bwd_write_123bytes), L(table_144_bytes_bwd)) - .int JMPTBL (L(bwd_write_124bytes), L(table_144_bytes_bwd)) - .int JMPTBL (L(bwd_write_125bytes), L(table_144_bytes_bwd)) - .int JMPTBL (L(bwd_write_126bytes), L(table_144_bytes_bwd)) - .int JMPTBL (L(bwd_write_127bytes), L(table_144_bytes_bwd)) - .int JMPTBL (L(bwd_write_128bytes), L(table_144_bytes_bwd)) - .int JMPTBL (L(bwd_write_129bytes), L(table_144_bytes_bwd)) - .int JMPTBL (L(bwd_write_130bytes), L(table_144_bytes_bwd)) - .int JMPTBL (L(bwd_write_131bytes), L(table_144_bytes_bwd)) - .int JMPTBL (L(bwd_write_132bytes), L(table_144_bytes_bwd)) - .int JMPTBL (L(bwd_write_133bytes), L(table_144_bytes_bwd)) - .int JMPTBL (L(bwd_write_134bytes), L(table_144_bytes_bwd)) - .int JMPTBL (L(bwd_write_135bytes), L(table_144_bytes_bwd)) - .int JMPTBL (L(bwd_write_136bytes), L(table_144_bytes_bwd)) - .int JMPTBL (L(bwd_write_137bytes), L(table_144_bytes_bwd)) - .int JMPTBL (L(bwd_write_138bytes), L(table_144_bytes_bwd)) - .int JMPTBL (L(bwd_write_139bytes), L(table_144_bytes_bwd)) - .int JMPTBL (L(bwd_write_140bytes), L(table_144_bytes_bwd)) - .int JMPTBL (L(bwd_write_141bytes), L(table_144_bytes_bwd)) - .int JMPTBL (L(bwd_write_142bytes), L(table_144_bytes_bwd)) - .int JMPTBL (L(bwd_write_143bytes), L(table_144_bytes_bwd)) + .int JMPTBL (L(bwd_write_0bytes), L(table_144_bytes_bwd)) + .int JMPTBL (L(bwd_write_1bytes), L(table_144_bytes_bwd)) + .int JMPTBL (L(bwd_write_2bytes), L(table_144_bytes_bwd)) + .int JMPTBL (L(bwd_write_3bytes), L(table_144_bytes_bwd)) + .int JMPTBL (L(bwd_write_4bytes), L(table_144_bytes_bwd)) + .int JMPTBL (L(bwd_write_5bytes), L(table_144_bytes_bwd)) + .int JMPTBL (L(bwd_write_6bytes), L(table_144_bytes_bwd)) + .int JMPTBL (L(bwd_write_7bytes), L(table_144_bytes_bwd)) + .int JMPTBL (L(bwd_write_8bytes), L(table_144_bytes_bwd)) + .int JMPTBL (L(bwd_write_9bytes), L(table_144_bytes_bwd)) + .int JMPTBL (L(bwd_write_10bytes), L(table_144_bytes_bwd)) + .int JMPTBL (L(bwd_write_11bytes), L(table_144_bytes_bwd)) + .int JMPTBL (L(bwd_write_12bytes), L(table_144_bytes_bwd)) + .int JMPTBL (L(bwd_write_13bytes), L(table_144_bytes_bwd)) + .int JMPTBL (L(bwd_write_14bytes), L(table_144_bytes_bwd)) + .int JMPTBL (L(bwd_write_15bytes), L(table_144_bytes_bwd)) + .int JMPTBL (L(bwd_write_16bytes), L(table_144_bytes_bwd)) + .int JMPTBL (L(bwd_write_17bytes), L(table_144_bytes_bwd)) + .int JMPTBL (L(bwd_write_18bytes), L(table_144_bytes_bwd)) + .int JMPTBL (L(bwd_write_19bytes), L(table_144_bytes_bwd)) + .int JMPTBL (L(bwd_write_20bytes), L(table_144_bytes_bwd)) + .int JMPTBL (L(bwd_write_21bytes), L(table_144_bytes_bwd)) + .int JMPTBL (L(bwd_write_22bytes), L(table_144_bytes_bwd)) + .int JMPTBL (L(bwd_write_23bytes), L(table_144_bytes_bwd)) + .int JMPTBL (L(bwd_write_24bytes), L(table_144_bytes_bwd)) + .int JMPTBL (L(bwd_write_25bytes), L(table_144_bytes_bwd)) + .int JMPTBL (L(bwd_write_26bytes), L(table_144_bytes_bwd)) + .int JMPTBL (L(bwd_write_27bytes), L(table_144_bytes_bwd)) + .int JMPTBL (L(bwd_write_28bytes), L(table_144_bytes_bwd)) + .int JMPTBL (L(bwd_write_29bytes), L(table_144_bytes_bwd)) + .int JMPTBL (L(bwd_write_30bytes), L(table_144_bytes_bwd)) + .int JMPTBL (L(bwd_write_31bytes), L(table_144_bytes_bwd)) + .int JMPTBL (L(bwd_write_32bytes), L(table_144_bytes_bwd)) + .int JMPTBL (L(bwd_write_33bytes), L(table_144_bytes_bwd)) + .int JMPTBL (L(bwd_write_34bytes), L(table_144_bytes_bwd)) + .int JMPTBL (L(bwd_write_35bytes), L(table_144_bytes_bwd)) + .int JMPTBL (L(bwd_write_36bytes), L(table_144_bytes_bwd)) + .int JMPTBL (L(bwd_write_37bytes), L(table_144_bytes_bwd)) + .int JMPTBL (L(bwd_write_38bytes), L(table_144_bytes_bwd)) + .int JMPTBL (L(bwd_write_39bytes), L(table_144_bytes_bwd)) + .int JMPTBL (L(bwd_write_40bytes), L(table_144_bytes_bwd)) + .int JMPTBL (L(bwd_write_41bytes), L(table_144_bytes_bwd)) + .int JMPTBL (L(bwd_write_42bytes), L(table_144_bytes_bwd)) + .int JMPTBL (L(bwd_write_43bytes), L(table_144_bytes_bwd)) + .int JMPTBL (L(bwd_write_44bytes), L(table_144_bytes_bwd)) + .int JMPTBL (L(bwd_write_45bytes), L(table_144_bytes_bwd)) + .int JMPTBL (L(bwd_write_46bytes), L(table_144_bytes_bwd)) + .int JMPTBL (L(bwd_write_47bytes), L(table_144_bytes_bwd)) + .int JMPTBL (L(bwd_write_48bytes), L(table_144_bytes_bwd)) + .int JMPTBL (L(bwd_write_49bytes), L(table_144_bytes_bwd)) + .int JMPTBL (L(bwd_write_50bytes), L(table_144_bytes_bwd)) + .int JMPTBL (L(bwd_write_51bytes), L(table_144_bytes_bwd)) + .int JMPTBL (L(bwd_write_52bytes), L(table_144_bytes_bwd)) + .int JMPTBL (L(bwd_write_53bytes), L(table_144_bytes_bwd)) + .int JMPTBL (L(bwd_write_54bytes), L(table_144_bytes_bwd)) + .int JMPTBL (L(bwd_write_55bytes), L(table_144_bytes_bwd)) + .int JMPTBL (L(bwd_write_56bytes), L(table_144_bytes_bwd)) + .int JMPTBL (L(bwd_write_57bytes), L(table_144_bytes_bwd)) + .int JMPTBL (L(bwd_write_58bytes), L(table_144_bytes_bwd)) + .int JMPTBL (L(bwd_write_59bytes), L(table_144_bytes_bwd)) + .int JMPTBL (L(bwd_write_60bytes), L(table_144_bytes_bwd)) + .int JMPTBL (L(bwd_write_61bytes), L(table_144_bytes_bwd)) + .int JMPTBL (L(bwd_write_62bytes), L(table_144_bytes_bwd)) + .int JMPTBL (L(bwd_write_63bytes), L(table_144_bytes_bwd)) + .int JMPTBL (L(bwd_write_64bytes), L(table_144_bytes_bwd)) + .int JMPTBL (L(bwd_write_65bytes), L(table_144_bytes_bwd)) + .int JMPTBL (L(bwd_write_66bytes), L(table_144_bytes_bwd)) + .int JMPTBL (L(bwd_write_67bytes), L(table_144_bytes_bwd)) + .int JMPTBL (L(bwd_write_68bytes), L(table_144_bytes_bwd)) + .int JMPTBL (L(bwd_write_69bytes), L(table_144_bytes_bwd)) + .int JMPTBL (L(bwd_write_70bytes), L(table_144_bytes_bwd)) + .int JMPTBL (L(bwd_write_71bytes), L(table_144_bytes_bwd)) + .int JMPTBL (L(bwd_write_72bytes), L(table_144_bytes_bwd)) + .int JMPTBL (L(bwd_write_73bytes), L(table_144_bytes_bwd)) + .int JMPTBL (L(bwd_write_74bytes), L(table_144_bytes_bwd)) + .int JMPTBL (L(bwd_write_75bytes), L(table_144_bytes_bwd)) + .int JMPTBL (L(bwd_write_76bytes), L(table_144_bytes_bwd)) + .int JMPTBL (L(bwd_write_77bytes), L(table_144_bytes_bwd)) + .int JMPTBL (L(bwd_write_78bytes), L(table_144_bytes_bwd)) + .int JMPTBL (L(bwd_write_79bytes), L(table_144_bytes_bwd)) + .int JMPTBL (L(bwd_write_80bytes), L(table_144_bytes_bwd)) + .int JMPTBL (L(bwd_write_81bytes), L(table_144_bytes_bwd)) + .int JMPTBL (L(bwd_write_82bytes), L(table_144_bytes_bwd)) + .int JMPTBL (L(bwd_write_83bytes), L(table_144_bytes_bwd)) + .int JMPTBL (L(bwd_write_84bytes), L(table_144_bytes_bwd)) + .int JMPTBL (L(bwd_write_85bytes), L(table_144_bytes_bwd)) + .int JMPTBL (L(bwd_write_86bytes), L(table_144_bytes_bwd)) + .int JMPTBL (L(bwd_write_87bytes), L(table_144_bytes_bwd)) + .int JMPTBL (L(bwd_write_88bytes), L(table_144_bytes_bwd)) + .int JMPTBL (L(bwd_write_89bytes), L(table_144_bytes_bwd)) + .int JMPTBL (L(bwd_write_90bytes), L(table_144_bytes_bwd)) + .int JMPTBL (L(bwd_write_91bytes), L(table_144_bytes_bwd)) + .int JMPTBL (L(bwd_write_92bytes), L(table_144_bytes_bwd)) + .int JMPTBL (L(bwd_write_93bytes), L(table_144_bytes_bwd)) + .int JMPTBL (L(bwd_write_94bytes), L(table_144_bytes_bwd)) + .int JMPTBL (L(bwd_write_95bytes), L(table_144_bytes_bwd)) + .int JMPTBL (L(bwd_write_96bytes), L(table_144_bytes_bwd)) + .int JMPTBL (L(bwd_write_97bytes), L(table_144_bytes_bwd)) + .int JMPTBL (L(bwd_write_98bytes), L(table_144_bytes_bwd)) + .int JMPTBL (L(bwd_write_99bytes), L(table_144_bytes_bwd)) + .int JMPTBL (L(bwd_write_100bytes), L(table_144_bytes_bwd)) + .int JMPTBL (L(bwd_write_101bytes), L(table_144_bytes_bwd)) + .int JMPTBL (L(bwd_write_102bytes), L(table_144_bytes_bwd)) + .int JMPTBL (L(bwd_write_103bytes), L(table_144_bytes_bwd)) + .int JMPTBL (L(bwd_write_104bytes), L(table_144_bytes_bwd)) + .int JMPTBL (L(bwd_write_105bytes), L(table_144_bytes_bwd)) + .int JMPTBL (L(bwd_write_106bytes), L(table_144_bytes_bwd)) + .int JMPTBL (L(bwd_write_107bytes), L(table_144_bytes_bwd)) + .int JMPTBL (L(bwd_write_108bytes), L(table_144_bytes_bwd)) + .int JMPTBL (L(bwd_write_109bytes), L(table_144_bytes_bwd)) + .int JMPTBL (L(bwd_write_110bytes), L(table_144_bytes_bwd)) + .int JMPTBL (L(bwd_write_111bytes), L(table_144_bytes_bwd)) + .int JMPTBL (L(bwd_write_112bytes), L(table_144_bytes_bwd)) + .int JMPTBL (L(bwd_write_113bytes), L(table_144_bytes_bwd)) + .int JMPTBL (L(bwd_write_114bytes), L(table_144_bytes_bwd)) + .int JMPTBL (L(bwd_write_115bytes), L(table_144_bytes_bwd)) + .int JMPTBL (L(bwd_write_116bytes), L(table_144_bytes_bwd)) + .int JMPTBL (L(bwd_write_117bytes), L(table_144_bytes_bwd)) + .int JMPTBL (L(bwd_write_118bytes), L(table_144_bytes_bwd)) + .int JMPTBL (L(bwd_write_119bytes), L(table_144_bytes_bwd)) + .int JMPTBL (L(bwd_write_120bytes), L(table_144_bytes_bwd)) + .int JMPTBL (L(bwd_write_121bytes), L(table_144_bytes_bwd)) + .int JMPTBL (L(bwd_write_122bytes), L(table_144_bytes_bwd)) + .int JMPTBL (L(bwd_write_123bytes), L(table_144_bytes_bwd)) + .int JMPTBL (L(bwd_write_124bytes), L(table_144_bytes_bwd)) + .int JMPTBL (L(bwd_write_125bytes), L(table_144_bytes_bwd)) + .int JMPTBL (L(bwd_write_126bytes), L(table_144_bytes_bwd)) + .int JMPTBL (L(bwd_write_127bytes), L(table_144_bytes_bwd)) + .int JMPTBL (L(bwd_write_128bytes), L(table_144_bytes_bwd)) + .int JMPTBL (L(bwd_write_129bytes), L(table_144_bytes_bwd)) + .int JMPTBL (L(bwd_write_130bytes), L(table_144_bytes_bwd)) + .int JMPTBL (L(bwd_write_131bytes), L(table_144_bytes_bwd)) + .int JMPTBL (L(bwd_write_132bytes), L(table_144_bytes_bwd)) + .int JMPTBL (L(bwd_write_133bytes), L(table_144_bytes_bwd)) + .int JMPTBL (L(bwd_write_134bytes), L(table_144_bytes_bwd)) + .int JMPTBL (L(bwd_write_135bytes), L(table_144_bytes_bwd)) + .int JMPTBL (L(bwd_write_136bytes), L(table_144_bytes_bwd)) + .int JMPTBL (L(bwd_write_137bytes), L(table_144_bytes_bwd)) + .int JMPTBL (L(bwd_write_138bytes), L(table_144_bytes_bwd)) + .int JMPTBL (L(bwd_write_139bytes), L(table_144_bytes_bwd)) + .int JMPTBL (L(bwd_write_140bytes), L(table_144_bytes_bwd)) + .int JMPTBL (L(bwd_write_141bytes), L(table_144_bytes_bwd)) + .int JMPTBL (L(bwd_write_142bytes), L(table_144_bytes_bwd)) + .int JMPTBL (L(bwd_write_143bytes), L(table_144_bytes_bwd)) - .p2align 3 + .p2align 3 L(table_144_bytes_fwd): - .int JMPTBL (L(fwd_write_0bytes), L(table_144_bytes_fwd)) - .int JMPTBL (L(fwd_write_1bytes), L(table_144_bytes_fwd)) - .int JMPTBL (L(fwd_write_2bytes), L(table_144_bytes_fwd)) - .int JMPTBL (L(fwd_write_3bytes), L(table_144_bytes_fwd)) - .int JMPTBL (L(fwd_write_4bytes), L(table_144_bytes_fwd)) - .int JMPTBL (L(fwd_write_5bytes), L(table_144_bytes_fwd)) - .int JMPTBL (L(fwd_write_6bytes), L(table_144_bytes_fwd)) - .int JMPTBL (L(fwd_write_7bytes), L(table_144_bytes_fwd)) - .int JMPTBL (L(fwd_write_8bytes), L(table_144_bytes_fwd)) - .int JMPTBL (L(fwd_write_9bytes), L(table_144_bytes_fwd)) - .int JMPTBL (L(fwd_write_10bytes), L(table_144_bytes_fwd)) - .int JMPTBL (L(fwd_write_11bytes), L(table_144_bytes_fwd)) - .int JMPTBL (L(fwd_write_12bytes), L(table_144_bytes_fwd)) - .int JMPTBL (L(fwd_write_13bytes), L(table_144_bytes_fwd)) - .int JMPTBL (L(fwd_write_14bytes), L(table_144_bytes_fwd)) - .int JMPTBL (L(fwd_write_15bytes), L(table_144_bytes_fwd)) - .int JMPTBL (L(fwd_write_16bytes), L(table_144_bytes_fwd)) - .int JMPTBL (L(fwd_write_17bytes), L(table_144_bytes_fwd)) - .int JMPTBL (L(fwd_write_18bytes), L(table_144_bytes_fwd)) - .int JMPTBL (L(fwd_write_19bytes), L(table_144_bytes_fwd)) - .int JMPTBL (L(fwd_write_20bytes), L(table_144_bytes_fwd)) - .int JMPTBL (L(fwd_write_21bytes), L(table_144_bytes_fwd)) - .int JMPTBL (L(fwd_write_22bytes), L(table_144_bytes_fwd)) - .int JMPTBL (L(fwd_write_23bytes), L(table_144_bytes_fwd)) - .int JMPTBL (L(fwd_write_24bytes), L(table_144_bytes_fwd)) - .int JMPTBL (L(fwd_write_25bytes), L(table_144_bytes_fwd)) - .int JMPTBL (L(fwd_write_26bytes), L(table_144_bytes_fwd)) - .int JMPTBL (L(fwd_write_27bytes), L(table_144_bytes_fwd)) - .int JMPTBL (L(fwd_write_28bytes), L(table_144_bytes_fwd)) - .int JMPTBL (L(fwd_write_29bytes), L(table_144_bytes_fwd)) - .int JMPTBL (L(fwd_write_30bytes), L(table_144_bytes_fwd)) - .int JMPTBL (L(fwd_write_31bytes), L(table_144_bytes_fwd)) - .int JMPTBL (L(fwd_write_32bytes), L(table_144_bytes_fwd)) - .int JMPTBL (L(fwd_write_33bytes), L(table_144_bytes_fwd)) - .int JMPTBL (L(fwd_write_34bytes), L(table_144_bytes_fwd)) - .int JMPTBL (L(fwd_write_35bytes), L(table_144_bytes_fwd)) - .int JMPTBL (L(fwd_write_36bytes), L(table_144_bytes_fwd)) - .int JMPTBL (L(fwd_write_37bytes), L(table_144_bytes_fwd)) - .int JMPTBL (L(fwd_write_38bytes), L(table_144_bytes_fwd)) - .int JMPTBL (L(fwd_write_39bytes), L(table_144_bytes_fwd)) - .int JMPTBL (L(fwd_write_40bytes), L(table_144_bytes_fwd)) - .int JMPTBL (L(fwd_write_41bytes), L(table_144_bytes_fwd)) - .int JMPTBL (L(fwd_write_42bytes), L(table_144_bytes_fwd)) - .int JMPTBL (L(fwd_write_43bytes), L(table_144_bytes_fwd)) - .int JMPTBL (L(fwd_write_44bytes), L(table_144_bytes_fwd)) - .int JMPTBL (L(fwd_write_45bytes), L(table_144_bytes_fwd)) - .int JMPTBL (L(fwd_write_46bytes), L(table_144_bytes_fwd)) - .int JMPTBL (L(fwd_write_47bytes), L(table_144_bytes_fwd)) - .int JMPTBL (L(fwd_write_48bytes), L(table_144_bytes_fwd)) - .int JMPTBL (L(fwd_write_49bytes), L(table_144_bytes_fwd)) - .int JMPTBL (L(fwd_write_50bytes), L(table_144_bytes_fwd)) - .int JMPTBL (L(fwd_write_51bytes), L(table_144_bytes_fwd)) - .int JMPTBL (L(fwd_write_52bytes), L(table_144_bytes_fwd)) - .int JMPTBL (L(fwd_write_53bytes), L(table_144_bytes_fwd)) - .int JMPTBL (L(fwd_write_54bytes), L(table_144_bytes_fwd)) - .int JMPTBL (L(fwd_write_55bytes), L(table_144_bytes_fwd)) - .int JMPTBL (L(fwd_write_56bytes), L(table_144_bytes_fwd)) - .int JMPTBL (L(fwd_write_57bytes), L(table_144_bytes_fwd)) - .int JMPTBL (L(fwd_write_58bytes), L(table_144_bytes_fwd)) - .int JMPTBL (L(fwd_write_59bytes), L(table_144_bytes_fwd)) - .int JMPTBL (L(fwd_write_60bytes), L(table_144_bytes_fwd)) - .int JMPTBL (L(fwd_write_61bytes), L(table_144_bytes_fwd)) - .int JMPTBL (L(fwd_write_62bytes), L(table_144_bytes_fwd)) - .int JMPTBL (L(fwd_write_63bytes), L(table_144_bytes_fwd)) - .int JMPTBL (L(fwd_write_64bytes), L(table_144_bytes_fwd)) - .int JMPTBL (L(fwd_write_65bytes), L(table_144_bytes_fwd)) - .int JMPTBL (L(fwd_write_66bytes), L(table_144_bytes_fwd)) - .int JMPTBL (L(fwd_write_67bytes), L(table_144_bytes_fwd)) - .int JMPTBL (L(fwd_write_68bytes), L(table_144_bytes_fwd)) - .int JMPTBL (L(fwd_write_69bytes), L(table_144_bytes_fwd)) - .int JMPTBL (L(fwd_write_70bytes), L(table_144_bytes_fwd)) - .int JMPTBL (L(fwd_write_71bytes), L(table_144_bytes_fwd)) - .int JMPTBL (L(fwd_write_72bytes), L(table_144_bytes_fwd)) - .int JMPTBL (L(fwd_write_73bytes), L(table_144_bytes_fwd)) - .int JMPTBL (L(fwd_write_74bytes), L(table_144_bytes_fwd)) - .int JMPTBL (L(fwd_write_75bytes), L(table_144_bytes_fwd)) - .int JMPTBL (L(fwd_write_76bytes), L(table_144_bytes_fwd)) - .int JMPTBL (L(fwd_write_77bytes), L(table_144_bytes_fwd)) - .int JMPTBL (L(fwd_write_78bytes), L(table_144_bytes_fwd)) - .int JMPTBL (L(fwd_write_79bytes), L(table_144_bytes_fwd)) - .int JMPTBL (L(fwd_write_80bytes), L(table_144_bytes_fwd)) - .int JMPTBL (L(fwd_write_81bytes), L(table_144_bytes_fwd)) - .int JMPTBL (L(fwd_write_82bytes), L(table_144_bytes_fwd)) - .int JMPTBL (L(fwd_write_83bytes), L(table_144_bytes_fwd)) - .int JMPTBL (L(fwd_write_84bytes), L(table_144_bytes_fwd)) - .int JMPTBL (L(fwd_write_85bytes), L(table_144_bytes_fwd)) - .int JMPTBL (L(fwd_write_86bytes), L(table_144_bytes_fwd)) - .int JMPTBL (L(fwd_write_87bytes), L(table_144_bytes_fwd)) - .int JMPTBL (L(fwd_write_88bytes), L(table_144_bytes_fwd)) - .int JMPTBL (L(fwd_write_89bytes), L(table_144_bytes_fwd)) - .int JMPTBL (L(fwd_write_90bytes), L(table_144_bytes_fwd)) - .int JMPTBL (L(fwd_write_91bytes), L(table_144_bytes_fwd)) - .int JMPTBL (L(fwd_write_92bytes), L(table_144_bytes_fwd)) - .int JMPTBL (L(fwd_write_93bytes), L(table_144_bytes_fwd)) - .int JMPTBL (L(fwd_write_94bytes), L(table_144_bytes_fwd)) - .int JMPTBL (L(fwd_write_95bytes), L(table_144_bytes_fwd)) - .int JMPTBL (L(fwd_write_96bytes), L(table_144_bytes_fwd)) - .int JMPTBL (L(fwd_write_97bytes), L(table_144_bytes_fwd)) - .int JMPTBL (L(fwd_write_98bytes), L(table_144_bytes_fwd)) - .int JMPTBL (L(fwd_write_99bytes), L(table_144_bytes_fwd)) - .int JMPTBL (L(fwd_write_100bytes), L(table_144_bytes_fwd)) - .int JMPTBL (L(fwd_write_101bytes), L(table_144_bytes_fwd)) - .int JMPTBL (L(fwd_write_102bytes), L(table_144_bytes_fwd)) - .int JMPTBL (L(fwd_write_103bytes), L(table_144_bytes_fwd)) - .int JMPTBL (L(fwd_write_104bytes), L(table_144_bytes_fwd)) - .int JMPTBL (L(fwd_write_105bytes), L(table_144_bytes_fwd)) - .int JMPTBL (L(fwd_write_106bytes), L(table_144_bytes_fwd)) - .int JMPTBL (L(fwd_write_107bytes), L(table_144_bytes_fwd)) - .int JMPTBL (L(fwd_write_108bytes), L(table_144_bytes_fwd)) - .int JMPTBL (L(fwd_write_109bytes), L(table_144_bytes_fwd)) - .int JMPTBL (L(fwd_write_110bytes), L(table_144_bytes_fwd)) - .int JMPTBL (L(fwd_write_111bytes), L(table_144_bytes_fwd)) - .int JMPTBL (L(fwd_write_112bytes), L(table_144_bytes_fwd)) - .int JMPTBL (L(fwd_write_113bytes), L(table_144_bytes_fwd)) - .int JMPTBL (L(fwd_write_114bytes), L(table_144_bytes_fwd)) - .int JMPTBL (L(fwd_write_115bytes), L(table_144_bytes_fwd)) - .int JMPTBL (L(fwd_write_116bytes), L(table_144_bytes_fwd)) - .int JMPTBL (L(fwd_write_117bytes), L(table_144_bytes_fwd)) - .int JMPTBL (L(fwd_write_118bytes), L(table_144_bytes_fwd)) - .int JMPTBL (L(fwd_write_119bytes), L(table_144_bytes_fwd)) - .int JMPTBL (L(fwd_write_120bytes), L(table_144_bytes_fwd)) - .int JMPTBL (L(fwd_write_121bytes), L(table_144_bytes_fwd)) - .int JMPTBL (L(fwd_write_122bytes), L(table_144_bytes_fwd)) - .int JMPTBL (L(fwd_write_123bytes), L(table_144_bytes_fwd)) - .int JMPTBL (L(fwd_write_124bytes), L(table_144_bytes_fwd)) - .int JMPTBL (L(fwd_write_125bytes), L(table_144_bytes_fwd)) - .int JMPTBL (L(fwd_write_126bytes), L(table_144_bytes_fwd)) - .int JMPTBL (L(fwd_write_127bytes), L(table_144_bytes_fwd)) - .int JMPTBL (L(fwd_write_128bytes), L(table_144_bytes_fwd)) - .int JMPTBL (L(fwd_write_129bytes), L(table_144_bytes_fwd)) - .int JMPTBL (L(fwd_write_130bytes), L(table_144_bytes_fwd)) - .int JMPTBL (L(fwd_write_131bytes), L(table_144_bytes_fwd)) - .int JMPTBL (L(fwd_write_132bytes), L(table_144_bytes_fwd)) - .int JMPTBL (L(fwd_write_133bytes), L(table_144_bytes_fwd)) - .int JMPTBL (L(fwd_write_134bytes), L(table_144_bytes_fwd)) - .int JMPTBL (L(fwd_write_135bytes), L(table_144_bytes_fwd)) - .int JMPTBL (L(fwd_write_136bytes), L(table_144_bytes_fwd)) - .int JMPTBL (L(fwd_write_137bytes), L(table_144_bytes_fwd)) - .int JMPTBL (L(fwd_write_138bytes), L(table_144_bytes_fwd)) - .int JMPTBL (L(fwd_write_139bytes), L(table_144_bytes_fwd)) - .int JMPTBL (L(fwd_write_140bytes), L(table_144_bytes_fwd)) - .int JMPTBL (L(fwd_write_141bytes), L(table_144_bytes_fwd)) - .int JMPTBL (L(fwd_write_142bytes), L(table_144_bytes_fwd)) - .int JMPTBL (L(fwd_write_143bytes), L(table_144_bytes_fwd)) + .int JMPTBL (L(fwd_write_0bytes), L(table_144_bytes_fwd)) + .int JMPTBL (L(fwd_write_1bytes), L(table_144_bytes_fwd)) + .int JMPTBL (L(fwd_write_2bytes), L(table_144_bytes_fwd)) + .int JMPTBL (L(fwd_write_3bytes), L(table_144_bytes_fwd)) + .int JMPTBL (L(fwd_write_4bytes), L(table_144_bytes_fwd)) + .int JMPTBL (L(fwd_write_5bytes), L(table_144_bytes_fwd)) + .int JMPTBL (L(fwd_write_6bytes), L(table_144_bytes_fwd)) + .int JMPTBL (L(fwd_write_7bytes), L(table_144_bytes_fwd)) + .int JMPTBL (L(fwd_write_8bytes), L(table_144_bytes_fwd)) + .int JMPTBL (L(fwd_write_9bytes), L(table_144_bytes_fwd)) + .int JMPTBL (L(fwd_write_10bytes), L(table_144_bytes_fwd)) + .int JMPTBL (L(fwd_write_11bytes), L(table_144_bytes_fwd)) + .int JMPTBL (L(fwd_write_12bytes), L(table_144_bytes_fwd)) + .int JMPTBL (L(fwd_write_13bytes), L(table_144_bytes_fwd)) + .int JMPTBL (L(fwd_write_14bytes), L(table_144_bytes_fwd)) + .int JMPTBL (L(fwd_write_15bytes), L(table_144_bytes_fwd)) + .int JMPTBL (L(fwd_write_16bytes), L(table_144_bytes_fwd)) + .int JMPTBL (L(fwd_write_17bytes), L(table_144_bytes_fwd)) + .int JMPTBL (L(fwd_write_18bytes), L(table_144_bytes_fwd)) + .int JMPTBL (L(fwd_write_19bytes), L(table_144_bytes_fwd)) + .int JMPTBL (L(fwd_write_20bytes), L(table_144_bytes_fwd)) + .int JMPTBL (L(fwd_write_21bytes), L(table_144_bytes_fwd)) + .int JMPTBL (L(fwd_write_22bytes), L(table_144_bytes_fwd)) + .int JMPTBL (L(fwd_write_23bytes), L(table_144_bytes_fwd)) + .int JMPTBL (L(fwd_write_24bytes), L(table_144_bytes_fwd)) + .int JMPTBL (L(fwd_write_25bytes), L(table_144_bytes_fwd)) + .int JMPTBL (L(fwd_write_26bytes), L(table_144_bytes_fwd)) + .int JMPTBL (L(fwd_write_27bytes), L(table_144_bytes_fwd)) + .int JMPTBL (L(fwd_write_28bytes), L(table_144_bytes_fwd)) + .int JMPTBL (L(fwd_write_29bytes), L(table_144_bytes_fwd)) + .int JMPTBL (L(fwd_write_30bytes), L(table_144_bytes_fwd)) + .int JMPTBL (L(fwd_write_31bytes), L(table_144_bytes_fwd)) + .int JMPTBL (L(fwd_write_32bytes), L(table_144_bytes_fwd)) + .int JMPTBL (L(fwd_write_33bytes), L(table_144_bytes_fwd)) + .int JMPTBL (L(fwd_write_34bytes), L(table_144_bytes_fwd)) + .int JMPTBL (L(fwd_write_35bytes), L(table_144_bytes_fwd)) + .int JMPTBL (L(fwd_write_36bytes), L(table_144_bytes_fwd)) + .int JMPTBL (L(fwd_write_37bytes), L(table_144_bytes_fwd)) + .int JMPTBL (L(fwd_write_38bytes), L(table_144_bytes_fwd)) + .int JMPTBL (L(fwd_write_39bytes), L(table_144_bytes_fwd)) + .int JMPTBL (L(fwd_write_40bytes), L(table_144_bytes_fwd)) + .int JMPTBL (L(fwd_write_41bytes), L(table_144_bytes_fwd)) + .int JMPTBL (L(fwd_write_42bytes), L(table_144_bytes_fwd)) + .int JMPTBL (L(fwd_write_43bytes), L(table_144_bytes_fwd)) + .int JMPTBL (L(fwd_write_44bytes), L(table_144_bytes_fwd)) + .int JMPTBL (L(fwd_write_45bytes), L(table_144_bytes_fwd)) + .int JMPTBL (L(fwd_write_46bytes), L(table_144_bytes_fwd)) + .int JMPTBL (L(fwd_write_47bytes), L(table_144_bytes_fwd)) + .int JMPTBL (L(fwd_write_48bytes), L(table_144_bytes_fwd)) + .int JMPTBL (L(fwd_write_49bytes), L(table_144_bytes_fwd)) + .int JMPTBL (L(fwd_write_50bytes), L(table_144_bytes_fwd)) + .int JMPTBL (L(fwd_write_51bytes), L(table_144_bytes_fwd)) + .int JMPTBL (L(fwd_write_52bytes), L(table_144_bytes_fwd)) + .int JMPTBL (L(fwd_write_53bytes), L(table_144_bytes_fwd)) + .int JMPTBL (L(fwd_write_54bytes), L(table_144_bytes_fwd)) + .int JMPTBL (L(fwd_write_55bytes), L(table_144_bytes_fwd)) + .int JMPTBL (L(fwd_write_56bytes), L(table_144_bytes_fwd)) + .int JMPTBL (L(fwd_write_57bytes), L(table_144_bytes_fwd)) + .int JMPTBL (L(fwd_write_58bytes), L(table_144_bytes_fwd)) + .int JMPTBL (L(fwd_write_59bytes), L(table_144_bytes_fwd)) + .int JMPTBL (L(fwd_write_60bytes), L(table_144_bytes_fwd)) + .int JMPTBL (L(fwd_write_61bytes), L(table_144_bytes_fwd)) + .int JMPTBL (L(fwd_write_62bytes), L(table_144_bytes_fwd)) + .int JMPTBL (L(fwd_write_63bytes), L(table_144_bytes_fwd)) + .int JMPTBL (L(fwd_write_64bytes), L(table_144_bytes_fwd)) + .int JMPTBL (L(fwd_write_65bytes), L(table_144_bytes_fwd)) + .int JMPTBL (L(fwd_write_66bytes), L(table_144_bytes_fwd)) + .int JMPTBL (L(fwd_write_67bytes), L(table_144_bytes_fwd)) + .int JMPTBL (L(fwd_write_68bytes), L(table_144_bytes_fwd)) + .int JMPTBL (L(fwd_write_69bytes), L(table_144_bytes_fwd)) + .int JMPTBL (L(fwd_write_70bytes), L(table_144_bytes_fwd)) + .int JMPTBL (L(fwd_write_71bytes), L(table_144_bytes_fwd)) + .int JMPTBL (L(fwd_write_72bytes), L(table_144_bytes_fwd)) + .int JMPTBL (L(fwd_write_73bytes), L(table_144_bytes_fwd)) + .int JMPTBL (L(fwd_write_74bytes), L(table_144_bytes_fwd)) + .int JMPTBL (L(fwd_write_75bytes), L(table_144_bytes_fwd)) + .int JMPTBL (L(fwd_write_76bytes), L(table_144_bytes_fwd)) + .int JMPTBL (L(fwd_write_77bytes), L(table_144_bytes_fwd)) + .int JMPTBL (L(fwd_write_78bytes), L(table_144_bytes_fwd)) + .int JMPTBL (L(fwd_write_79bytes), L(table_144_bytes_fwd)) + .int JMPTBL (L(fwd_write_80bytes), L(table_144_bytes_fwd)) + .int JMPTBL (L(fwd_write_81bytes), L(table_144_bytes_fwd)) + .int JMPTBL (L(fwd_write_82bytes), L(table_144_bytes_fwd)) + .int JMPTBL (L(fwd_write_83bytes), L(table_144_bytes_fwd)) + .int JMPTBL (L(fwd_write_84bytes), L(table_144_bytes_fwd)) + .int JMPTBL (L(fwd_write_85bytes), L(table_144_bytes_fwd)) + .int JMPTBL (L(fwd_write_86bytes), L(table_144_bytes_fwd)) + .int JMPTBL (L(fwd_write_87bytes), L(table_144_bytes_fwd)) + .int JMPTBL (L(fwd_write_88bytes), L(table_144_bytes_fwd)) + .int JMPTBL (L(fwd_write_89bytes), L(table_144_bytes_fwd)) + .int JMPTBL (L(fwd_write_90bytes), L(table_144_bytes_fwd)) + .int JMPTBL (L(fwd_write_91bytes), L(table_144_bytes_fwd)) + .int JMPTBL (L(fwd_write_92bytes), L(table_144_bytes_fwd)) + .int JMPTBL (L(fwd_write_93bytes), L(table_144_bytes_fwd)) + .int JMPTBL (L(fwd_write_94bytes), L(table_144_bytes_fwd)) + .int JMPTBL (L(fwd_write_95bytes), L(table_144_bytes_fwd)) + .int JMPTBL (L(fwd_write_96bytes), L(table_144_bytes_fwd)) + .int JMPTBL (L(fwd_write_97bytes), L(table_144_bytes_fwd)) + .int JMPTBL (L(fwd_write_98bytes), L(table_144_bytes_fwd)) + .int JMPTBL (L(fwd_write_99bytes), L(table_144_bytes_fwd)) + .int JMPTBL (L(fwd_write_100bytes), L(table_144_bytes_fwd)) + .int JMPTBL (L(fwd_write_101bytes), L(table_144_bytes_fwd)) + .int JMPTBL (L(fwd_write_102bytes), L(table_144_bytes_fwd)) + .int JMPTBL (L(fwd_write_103bytes), L(table_144_bytes_fwd)) + .int JMPTBL (L(fwd_write_104bytes), L(table_144_bytes_fwd)) + .int JMPTBL (L(fwd_write_105bytes), L(table_144_bytes_fwd)) + .int JMPTBL (L(fwd_write_106bytes), L(table_144_bytes_fwd)) + .int JMPTBL (L(fwd_write_107bytes), L(table_144_bytes_fwd)) + .int JMPTBL (L(fwd_write_108bytes), L(table_144_bytes_fwd)) + .int JMPTBL (L(fwd_write_109bytes), L(table_144_bytes_fwd)) + .int JMPTBL (L(fwd_write_110bytes), L(table_144_bytes_fwd)) + .int JMPTBL (L(fwd_write_111bytes), L(table_144_bytes_fwd)) + .int JMPTBL (L(fwd_write_112bytes), L(table_144_bytes_fwd)) + .int JMPTBL (L(fwd_write_113bytes), L(table_144_bytes_fwd)) + .int JMPTBL (L(fwd_write_114bytes), L(table_144_bytes_fwd)) + .int JMPTBL (L(fwd_write_115bytes), L(table_144_bytes_fwd)) + .int JMPTBL (L(fwd_write_116bytes), L(table_144_bytes_fwd)) + .int JMPTBL (L(fwd_write_117bytes), L(table_144_bytes_fwd)) + .int JMPTBL (L(fwd_write_118bytes), L(table_144_bytes_fwd)) + .int JMPTBL (L(fwd_write_119bytes), L(table_144_bytes_fwd)) + .int JMPTBL (L(fwd_write_120bytes), L(table_144_bytes_fwd)) + .int JMPTBL (L(fwd_write_121bytes), L(table_144_bytes_fwd)) + .int JMPTBL (L(fwd_write_122bytes), L(table_144_bytes_fwd)) + .int JMPTBL (L(fwd_write_123bytes), L(table_144_bytes_fwd)) + .int JMPTBL (L(fwd_write_124bytes), L(table_144_bytes_fwd)) + .int JMPTBL (L(fwd_write_125bytes), L(table_144_bytes_fwd)) + .int JMPTBL (L(fwd_write_126bytes), L(table_144_bytes_fwd)) + .int JMPTBL (L(fwd_write_127bytes), L(table_144_bytes_fwd)) + .int JMPTBL (L(fwd_write_128bytes), L(table_144_bytes_fwd)) + .int JMPTBL (L(fwd_write_129bytes), L(table_144_bytes_fwd)) + .int JMPTBL (L(fwd_write_130bytes), L(table_144_bytes_fwd)) + .int JMPTBL (L(fwd_write_131bytes), L(table_144_bytes_fwd)) + .int JMPTBL (L(fwd_write_132bytes), L(table_144_bytes_fwd)) + .int JMPTBL (L(fwd_write_133bytes), L(table_144_bytes_fwd)) + .int JMPTBL (L(fwd_write_134bytes), L(table_144_bytes_fwd)) + .int JMPTBL (L(fwd_write_135bytes), L(table_144_bytes_fwd)) + .int JMPTBL (L(fwd_write_136bytes), L(table_144_bytes_fwd)) + .int JMPTBL (L(fwd_write_137bytes), L(table_144_bytes_fwd)) + .int JMPTBL (L(fwd_write_138bytes), L(table_144_bytes_fwd)) + .int JMPTBL (L(fwd_write_139bytes), L(table_144_bytes_fwd)) + .int JMPTBL (L(fwd_write_140bytes), L(table_144_bytes_fwd)) + .int JMPTBL (L(fwd_write_141bytes), L(table_144_bytes_fwd)) + .int JMPTBL (L(fwd_write_142bytes), L(table_144_bytes_fwd)) + .int JMPTBL (L(fwd_write_143bytes), L(table_144_bytes_fwd)) - .p2align 3 + .p2align 3 L(shl_table_fwd): - .int JMPTBL (L(shl_0), L(shl_table_fwd)) - .int JMPTBL (L(shl_1), L(shl_table_fwd)) - .int JMPTBL (L(shl_2), L(shl_table_fwd)) - .int JMPTBL (L(shl_3), L(shl_table_fwd)) - .int JMPTBL (L(shl_4), L(shl_table_fwd)) - .int JMPTBL (L(shl_5), L(shl_table_fwd)) - .int JMPTBL (L(shl_6), L(shl_table_fwd)) - .int JMPTBL (L(shl_7), L(shl_table_fwd)) - .int JMPTBL (L(shl_8), L(shl_table_fwd)) - .int JMPTBL (L(shl_9), L(shl_table_fwd)) - .int JMPTBL (L(shl_10), L(shl_table_fwd)) - .int JMPTBL (L(shl_11), L(shl_table_fwd)) - .int JMPTBL (L(shl_12), L(shl_table_fwd)) - .int JMPTBL (L(shl_13), L(shl_table_fwd)) - .int JMPTBL (L(shl_14), L(shl_table_fwd)) - .int JMPTBL (L(shl_15), L(shl_table_fwd)) + .int JMPTBL (L(shl_0), L(shl_table_fwd)) + .int JMPTBL (L(shl_1), L(shl_table_fwd)) + .int JMPTBL (L(shl_2), L(shl_table_fwd)) + .int JMPTBL (L(shl_3), L(shl_table_fwd)) + .int JMPTBL (L(shl_4), L(shl_table_fwd)) + .int JMPTBL (L(shl_5), L(shl_table_fwd)) + .int JMPTBL (L(shl_6), L(shl_table_fwd)) + .int JMPTBL (L(shl_7), L(shl_table_fwd)) + .int JMPTBL (L(shl_8), L(shl_table_fwd)) + .int JMPTBL (L(shl_9), L(shl_table_fwd)) + .int JMPTBL (L(shl_10), L(shl_table_fwd)) + .int JMPTBL (L(shl_11), L(shl_table_fwd)) + .int JMPTBL (L(shl_12), L(shl_table_fwd)) + .int JMPTBL (L(shl_13), L(shl_table_fwd)) + .int JMPTBL (L(shl_14), L(shl_table_fwd)) + .int JMPTBL (L(shl_15), L(shl_table_fwd)) - .p2align 3 + .p2align 3 L(shl_table_bwd): - .int JMPTBL (L(shl_0_bwd), L(shl_table_bwd)) - .int JMPTBL (L(shl_1_bwd), L(shl_table_bwd)) - .int JMPTBL (L(shl_2_bwd), L(shl_table_bwd)) - .int JMPTBL (L(shl_3_bwd), L(shl_table_bwd)) - .int JMPTBL (L(shl_4_bwd), L(shl_table_bwd)) - .int JMPTBL (L(shl_5_bwd), L(shl_table_bwd)) - .int JMPTBL (L(shl_6_bwd), L(shl_table_bwd)) - .int JMPTBL (L(shl_7_bwd), L(shl_table_bwd)) - .int JMPTBL (L(shl_8_bwd), L(shl_table_bwd)) - .int JMPTBL (L(shl_9_bwd), L(shl_table_bwd)) - .int JMPTBL (L(shl_10_bwd), L(shl_table_bwd)) - .int JMPTBL (L(shl_11_bwd), L(shl_table_bwd)) - .int JMPTBL (L(shl_12_bwd), L(shl_table_bwd)) - .int JMPTBL (L(shl_13_bwd), L(shl_table_bwd)) - .int JMPTBL (L(shl_14_bwd), L(shl_table_bwd)) - .int JMPTBL (L(shl_15_bwd), L(shl_table_bwd)) + .int JMPTBL (L(shl_0_bwd), L(shl_table_bwd)) + .int JMPTBL (L(shl_1_bwd), L(shl_table_bwd)) + .int JMPTBL (L(shl_2_bwd), L(shl_table_bwd)) + .int JMPTBL (L(shl_3_bwd), L(shl_table_bwd)) + .int JMPTBL (L(shl_4_bwd), L(shl_table_bwd)) + .int JMPTBL (L(shl_5_bwd), L(shl_table_bwd)) + .int JMPTBL (L(shl_6_bwd), L(shl_table_bwd)) + .int JMPTBL (L(shl_7_bwd), L(shl_table_bwd)) + .int JMPTBL (L(shl_8_bwd), L(shl_table_bwd)) + .int JMPTBL (L(shl_9_bwd), L(shl_table_bwd)) + .int JMPTBL (L(shl_10_bwd), L(shl_table_bwd)) + .int JMPTBL (L(shl_11_bwd), L(shl_table_bwd)) + .int JMPTBL (L(shl_12_bwd), L(shl_table_bwd)) + .int JMPTBL (L(shl_13_bwd), L(shl_table_bwd)) + .int JMPTBL (L(shl_14_bwd), L(shl_table_bwd)) + .int JMPTBL (L(shl_15_bwd), L(shl_table_bwd)) #endif diff --git a/utils/memcpy-bench/glibc/memcpy-ssse3.S b/utils/memcpy-bench/glibc/memcpy-ssse3.S index 2fd26651645..11cb6559a8b 100644 --- a/utils/memcpy-bench/glibc/memcpy-ssse3.S +++ b/utils/memcpy-bench/glibc/memcpy-ssse3.S @@ -24,3129 +24,3129 @@ #include "asm-syntax.h" #ifndef MEMCPY -# define MEMCPY __memcpy_ssse3 -# define MEMCPY_CHK __memcpy_chk_ssse3 -# define MEMPCPY __mempcpy_ssse3 -# define MEMPCPY_CHK __mempcpy_chk_ssse3 +# define MEMCPY __memcpy_ssse3 +# define MEMCPY_CHK __memcpy_chk_ssse3 +# define MEMPCPY __mempcpy_ssse3 +# define MEMPCPY_CHK __mempcpy_chk_ssse3 #endif -#define JMPTBL(I, B) I - B +#define JMPTBL(I, B) I - B /* Branch to an entry in a jump table. TABLE is a jump table with relative offsets. INDEX is a register contains the index into the jump table. SCALE is the scale of INDEX. */ -#define BRANCH_TO_JMPTBL_ENTRY(TABLE, INDEX, SCALE) \ - lea TABLE(%rip), %r11; \ - movslq (%r11, INDEX, SCALE), INDEX; \ - lea (%r11, INDEX), INDEX; \ - _CET_NOTRACK jmp *INDEX; \ +#define BRANCH_TO_JMPTBL_ENTRY(TABLE, INDEX, SCALE) \ + lea TABLE(%rip), %r11; \ + movslq (%r11, INDEX, SCALE), INDEX; \ + lea (%r11, INDEX), INDEX; \ + _CET_NOTRACK jmp *INDEX; \ ud2 - .section .text.ssse3,"ax",@progbits + .section .text.ssse3,"ax",@progbits #if !defined USE_AS_MEMPCPY && !defined USE_AS_MEMMOVE ENTRY (MEMPCPY_CHK) - cmp %RDX_LP, %RCX_LP - jb HIDDEN_JUMPTARGET (__chk_fail) + cmp %RDX_LP, %RCX_LP + jb HIDDEN_JUMPTARGET (__chk_fail) END (MEMPCPY_CHK) ENTRY (MEMPCPY) - mov %RDI_LP, %RAX_LP - add %RDX_LP, %RAX_LP - jmp L(start) + mov %RDI_LP, %RAX_LP + add %RDX_LP, %RAX_LP + jmp L(start) END (MEMPCPY) #endif #if !defined USE_AS_BCOPY ENTRY (MEMCPY_CHK) - cmp %RDX_LP, %RCX_LP - jb HIDDEN_JUMPTARGET (__chk_fail) + cmp %RDX_LP, %RCX_LP + jb HIDDEN_JUMPTARGET (__chk_fail) END (MEMCPY_CHK) #endif ENTRY (MEMCPY) - mov %RDI_LP, %RAX_LP + mov %RDI_LP, %RAX_LP #ifdef USE_AS_MEMPCPY - add %RDX_LP, %RAX_LP + add %RDX_LP, %RAX_LP #endif #ifdef __ILP32__ - /* Clear the upper 32 bits. */ - mov %edx, %edx + /* Clear the upper 32 bits. */ + mov %edx, %edx #endif #ifdef USE_AS_MEMMOVE - cmp %rsi, %rdi - jb L(copy_forward) - je L(write_0bytes) - cmp $79, %rdx - jbe L(copy_forward) - jmp L(copy_backward) + cmp %rsi, %rdi + jb L(copy_forward) + je L(write_0bytes) + cmp $79, %rdx + jbe L(copy_forward) + jmp L(copy_backward) L(copy_forward): #endif L(start): - cmp $79, %rdx - lea L(table_less_80bytes)(%rip), %r11 - ja L(80bytesormore) - movslq (%r11, %rdx, 4), %r9 - add %rdx, %rsi - add %rdx, %rdi - add %r11, %r9 - _CET_NOTRACK jmp *%r9 - ud2 + cmp $79, %rdx + lea L(table_less_80bytes)(%rip), %r11 + ja L(80bytesormore) + movslq (%r11, %rdx, 4), %r9 + add %rdx, %rsi + add %rdx, %rdi + add %r11, %r9 + _CET_NOTRACK jmp *%r9 + ud2 - .p2align 4 + .p2align 4 L(80bytesormore): #ifndef USE_AS_MEMMOVE - cmp %dil, %sil - jle L(copy_backward) + cmp %dil, %sil + jle L(copy_backward) #endif - movdqu (%rsi), %xmm0 - mov %rdi, %rcx - and $-16, %rdi - add $16, %rdi - mov %rcx, %r8 - sub %rdi, %rcx - add %rcx, %rdx - sub %rcx, %rsi + movdqu (%rsi), %xmm0 + mov %rdi, %rcx + and $-16, %rdi + add $16, %rdi + mov %rcx, %r8 + sub %rdi, %rcx + add %rcx, %rdx + sub %rcx, %rsi #ifdef SHARED_CACHE_SIZE_HALF - mov $SHARED_CACHE_SIZE_HALF, %RCX_LP + mov $SHARED_CACHE_SIZE_HALF, %RCX_LP #else - mov __x86_shared_cache_size_half(%rip), %RCX_LP + mov __x86_shared_cache_size_half(%rip), %RCX_LP #endif - cmp %rcx, %rdx - mov %rsi, %r9 - ja L(large_page_fwd) - and $0xf, %r9 - jz L(shl_0) + cmp %rcx, %rdx + mov %rsi, %r9 + ja L(large_page_fwd) + and $0xf, %r9 + jz L(shl_0) #ifdef DATA_CACHE_SIZE_HALF - mov $DATA_CACHE_SIZE_HALF, %RCX_LP + mov $DATA_CACHE_SIZE_HALF, %RCX_LP #else - mov __x86_data_cache_size_half(%rip), %RCX_LP + mov __x86_data_cache_size_half(%rip), %RCX_LP #endif - BRANCH_TO_JMPTBL_ENTRY (L(shl_table), %r9, 4) + BRANCH_TO_JMPTBL_ENTRY (L(shl_table), %r9, 4) - .p2align 4 + .p2align 4 L(copy_backward): - movdqu -16(%rsi, %rdx), %xmm0 - add %rdx, %rsi - lea -16(%rdi, %rdx), %r8 - add %rdx, %rdi + movdqu -16(%rsi, %rdx), %xmm0 + add %rdx, %rsi + lea -16(%rdi, %rdx), %r8 + add %rdx, %rdi - mov %rdi, %rcx - and $0xf, %rcx - xor %rcx, %rdi - sub %rcx, %rdx - sub %rcx, %rsi + mov %rdi, %rcx + and $0xf, %rcx + xor %rcx, %rdi + sub %rcx, %rdx + sub %rcx, %rsi #ifdef SHARED_CACHE_SIZE_HALF - mov $SHARED_CACHE_SIZE_HALF, %RCX_LP + mov $SHARED_CACHE_SIZE_HALF, %RCX_LP #else - mov __x86_shared_cache_size_half(%rip), %RCX_LP + mov __x86_shared_cache_size_half(%rip), %RCX_LP #endif - cmp %rcx, %rdx - mov %rsi, %r9 - ja L(large_page_bwd) - and $0xf, %r9 - jz L(shl_0_bwd) + cmp %rcx, %rdx + mov %rsi, %r9 + ja L(large_page_bwd) + and $0xf, %r9 + jz L(shl_0_bwd) #ifdef DATA_CACHE_SIZE_HALF - mov $DATA_CACHE_SIZE_HALF, %RCX_LP + mov $DATA_CACHE_SIZE_HALF, %RCX_LP #else - mov __x86_data_cache_size_half(%rip), %RCX_LP + mov __x86_data_cache_size_half(%rip), %RCX_LP #endif - BRANCH_TO_JMPTBL_ENTRY (L(shl_table_bwd), %r9, 4) + BRANCH_TO_JMPTBL_ENTRY (L(shl_table_bwd), %r9, 4) - .p2align 4 + .p2align 4 L(shl_0): - sub $16, %rdx - movdqa (%rsi), %xmm1 - add $16, %rsi - movdqa %xmm1, (%rdi) - add $16, %rdi - cmp $128, %rdx - movdqu %xmm0, (%r8) - ja L(shl_0_gobble) - cmp $64, %rdx - jb L(shl_0_less_64bytes) - movaps (%rsi), %xmm4 - movaps 16(%rsi), %xmm1 - movaps 32(%rsi), %xmm2 - movaps 48(%rsi), %xmm3 - movaps %xmm4, (%rdi) - movaps %xmm1, 16(%rdi) - movaps %xmm2, 32(%rdi) - movaps %xmm3, 48(%rdi) - sub $64, %rdx - add $64, %rsi - add $64, %rdi + sub $16, %rdx + movdqa (%rsi), %xmm1 + add $16, %rsi + movdqa %xmm1, (%rdi) + add $16, %rdi + cmp $128, %rdx + movdqu %xmm0, (%r8) + ja L(shl_0_gobble) + cmp $64, %rdx + jb L(shl_0_less_64bytes) + movaps (%rsi), %xmm4 + movaps 16(%rsi), %xmm1 + movaps 32(%rsi), %xmm2 + movaps 48(%rsi), %xmm3 + movaps %xmm4, (%rdi) + movaps %xmm1, 16(%rdi) + movaps %xmm2, 32(%rdi) + movaps %xmm3, 48(%rdi) + sub $64, %rdx + add $64, %rsi + add $64, %rdi L(shl_0_less_64bytes): - add %rdx, %rsi - add %rdx, %rdi - BRANCH_TO_JMPTBL_ENTRY (L(table_less_80bytes), %rdx, 4) + add %rdx, %rsi + add %rdx, %rdi + BRANCH_TO_JMPTBL_ENTRY (L(table_less_80bytes), %rdx, 4) - .p2align 4 + .p2align 4 L(shl_0_gobble): #ifdef DATA_CACHE_SIZE_HALF - cmp $DATA_CACHE_SIZE_HALF, %RDX_LP + cmp $DATA_CACHE_SIZE_HALF, %RDX_LP #else - cmp __x86_data_cache_size_half(%rip), %RDX_LP + cmp __x86_data_cache_size_half(%rip), %RDX_LP #endif - lea -128(%rdx), %rdx - jae L(shl_0_gobble_mem_loop) + lea -128(%rdx), %rdx + jae L(shl_0_gobble_mem_loop) L(shl_0_gobble_cache_loop): - movdqa (%rsi), %xmm4 - movaps 0x10(%rsi), %xmm1 - movaps 0x20(%rsi), %xmm2 - movaps 0x30(%rsi), %xmm3 + movdqa (%rsi), %xmm4 + movaps 0x10(%rsi), %xmm1 + movaps 0x20(%rsi), %xmm2 + movaps 0x30(%rsi), %xmm3 - movdqa %xmm4, (%rdi) - movaps %xmm1, 0x10(%rdi) - movaps %xmm2, 0x20(%rdi) - movaps %xmm3, 0x30(%rdi) + movdqa %xmm4, (%rdi) + movaps %xmm1, 0x10(%rdi) + movaps %xmm2, 0x20(%rdi) + movaps %xmm3, 0x30(%rdi) - sub $128, %rdx - movaps 0x40(%rsi), %xmm4 - movaps 0x50(%rsi), %xmm5 - movaps 0x60(%rsi), %xmm6 - movaps 0x70(%rsi), %xmm7 - lea 0x80(%rsi), %rsi - movaps %xmm4, 0x40(%rdi) - movaps %xmm5, 0x50(%rdi) - movaps %xmm6, 0x60(%rdi) - movaps %xmm7, 0x70(%rdi) - lea 0x80(%rdi), %rdi + sub $128, %rdx + movaps 0x40(%rsi), %xmm4 + movaps 0x50(%rsi), %xmm5 + movaps 0x60(%rsi), %xmm6 + movaps 0x70(%rsi), %xmm7 + lea 0x80(%rsi), %rsi + movaps %xmm4, 0x40(%rdi) + movaps %xmm5, 0x50(%rdi) + movaps %xmm6, 0x60(%rdi) + movaps %xmm7, 0x70(%rdi) + lea 0x80(%rdi), %rdi - jae L(shl_0_gobble_cache_loop) - cmp $-0x40, %rdx - lea 0x80(%rdx), %rdx - jl L(shl_0_cache_less_64bytes) + jae L(shl_0_gobble_cache_loop) + cmp $-0x40, %rdx + lea 0x80(%rdx), %rdx + jl L(shl_0_cache_less_64bytes) - movdqa (%rsi), %xmm4 - sub $0x40, %rdx - movdqa 0x10(%rsi), %xmm1 + movdqa (%rsi), %xmm4 + sub $0x40, %rdx + movdqa 0x10(%rsi), %xmm1 - movdqa %xmm4, (%rdi) - movdqa %xmm1, 0x10(%rdi) + movdqa %xmm4, (%rdi) + movdqa %xmm1, 0x10(%rdi) - movdqa 0x20(%rsi), %xmm4 - movdqa 0x30(%rsi), %xmm1 - add $0x40, %rsi + movdqa 0x20(%rsi), %xmm4 + movdqa 0x30(%rsi), %xmm1 + add $0x40, %rsi - movdqa %xmm4, 0x20(%rdi) - movdqa %xmm1, 0x30(%rdi) - add $0x40, %rdi + movdqa %xmm4, 0x20(%rdi) + movdqa %xmm1, 0x30(%rdi) + add $0x40, %rdi L(shl_0_cache_less_64bytes): - add %rdx, %rsi - add %rdx, %rdi - BRANCH_TO_JMPTBL_ENTRY (L(table_less_80bytes), %rdx, 4) + add %rdx, %rsi + add %rdx, %rdi + BRANCH_TO_JMPTBL_ENTRY (L(table_less_80bytes), %rdx, 4) - .p2align 4 + .p2align 4 L(shl_0_gobble_mem_loop): - prefetcht0 0x1c0(%rsi) - prefetcht0 0x280(%rsi) + prefetcht0 0x1c0(%rsi) + prefetcht0 0x280(%rsi) - movdqa (%rsi), %xmm0 - movdqa 0x10(%rsi), %xmm1 - movdqa 0x20(%rsi), %xmm2 - movdqa 0x30(%rsi), %xmm3 - movdqa 0x40(%rsi), %xmm4 - movdqa 0x50(%rsi), %xmm5 - movdqa 0x60(%rsi), %xmm6 - movdqa 0x70(%rsi), %xmm7 - lea 0x80(%rsi), %rsi - sub $0x80, %rdx - movdqa %xmm0, (%rdi) - movdqa %xmm1, 0x10(%rdi) - movdqa %xmm2, 0x20(%rdi) - movdqa %xmm3, 0x30(%rdi) - movdqa %xmm4, 0x40(%rdi) - movdqa %xmm5, 0x50(%rdi) - movdqa %xmm6, 0x60(%rdi) - movdqa %xmm7, 0x70(%rdi) - lea 0x80(%rdi), %rdi + movdqa (%rsi), %xmm0 + movdqa 0x10(%rsi), %xmm1 + movdqa 0x20(%rsi), %xmm2 + movdqa 0x30(%rsi), %xmm3 + movdqa 0x40(%rsi), %xmm4 + movdqa 0x50(%rsi), %xmm5 + movdqa 0x60(%rsi), %xmm6 + movdqa 0x70(%rsi), %xmm7 + lea 0x80(%rsi), %rsi + sub $0x80, %rdx + movdqa %xmm0, (%rdi) + movdqa %xmm1, 0x10(%rdi) + movdqa %xmm2, 0x20(%rdi) + movdqa %xmm3, 0x30(%rdi) + movdqa %xmm4, 0x40(%rdi) + movdqa %xmm5, 0x50(%rdi) + movdqa %xmm6, 0x60(%rdi) + movdqa %xmm7, 0x70(%rdi) + lea 0x80(%rdi), %rdi - jae L(shl_0_gobble_mem_loop) - cmp $-0x40, %rdx - lea 0x80(%rdx), %rdx - jl L(shl_0_mem_less_64bytes) + jae L(shl_0_gobble_mem_loop) + cmp $-0x40, %rdx + lea 0x80(%rdx), %rdx + jl L(shl_0_mem_less_64bytes) - movdqa (%rsi), %xmm0 - sub $0x40, %rdx - movdqa 0x10(%rsi), %xmm1 + movdqa (%rsi), %xmm0 + sub $0x40, %rdx + movdqa 0x10(%rsi), %xmm1 - movdqa %xmm0, (%rdi) - movdqa %xmm1, 0x10(%rdi) + movdqa %xmm0, (%rdi) + movdqa %xmm1, 0x10(%rdi) - movdqa 0x20(%rsi), %xmm0 - movdqa 0x30(%rsi), %xmm1 - add $0x40, %rsi + movdqa 0x20(%rsi), %xmm0 + movdqa 0x30(%rsi), %xmm1 + add $0x40, %rsi - movdqa %xmm0, 0x20(%rdi) - movdqa %xmm1, 0x30(%rdi) - add $0x40, %rdi + movdqa %xmm0, 0x20(%rdi) + movdqa %xmm1, 0x30(%rdi) + add $0x40, %rdi L(shl_0_mem_less_64bytes): - cmp $0x20, %rdx - jb L(shl_0_mem_less_32bytes) - movdqa (%rsi), %xmm0 - sub $0x20, %rdx - movdqa 0x10(%rsi), %xmm1 - add $0x20, %rsi - movdqa %xmm0, (%rdi) - movdqa %xmm1, 0x10(%rdi) - add $0x20, %rdi + cmp $0x20, %rdx + jb L(shl_0_mem_less_32bytes) + movdqa (%rsi), %xmm0 + sub $0x20, %rdx + movdqa 0x10(%rsi), %xmm1 + add $0x20, %rsi + movdqa %xmm0, (%rdi) + movdqa %xmm1, 0x10(%rdi) + add $0x20, %rdi L(shl_0_mem_less_32bytes): - add %rdx, %rdi - add %rdx, %rsi - BRANCH_TO_JMPTBL_ENTRY (L(table_less_80bytes), %rdx, 4) + add %rdx, %rdi + add %rdx, %rsi + BRANCH_TO_JMPTBL_ENTRY (L(table_less_80bytes), %rdx, 4) - .p2align 4 + .p2align 4 L(shl_0_bwd): - sub $16, %rdx - movdqa -0x10(%rsi), %xmm1 - sub $16, %rsi - movdqa %xmm1, -0x10(%rdi) - sub $16, %rdi - cmp $0x80, %rdx - movdqu %xmm0, (%r8) - ja L(shl_0_gobble_bwd) - cmp $64, %rdx - jb L(shl_0_less_64bytes_bwd) - movaps -0x10(%rsi), %xmm0 - movaps -0x20(%rsi), %xmm1 - movaps -0x30(%rsi), %xmm2 - movaps -0x40(%rsi), %xmm3 - movaps %xmm0, -0x10(%rdi) - movaps %xmm1, -0x20(%rdi) - movaps %xmm2, -0x30(%rdi) - movaps %xmm3, -0x40(%rdi) - sub $64, %rdx - sub $0x40, %rsi - sub $0x40, %rdi + sub $16, %rdx + movdqa -0x10(%rsi), %xmm1 + sub $16, %rsi + movdqa %xmm1, -0x10(%rdi) + sub $16, %rdi + cmp $0x80, %rdx + movdqu %xmm0, (%r8) + ja L(shl_0_gobble_bwd) + cmp $64, %rdx + jb L(shl_0_less_64bytes_bwd) + movaps -0x10(%rsi), %xmm0 + movaps -0x20(%rsi), %xmm1 + movaps -0x30(%rsi), %xmm2 + movaps -0x40(%rsi), %xmm3 + movaps %xmm0, -0x10(%rdi) + movaps %xmm1, -0x20(%rdi) + movaps %xmm2, -0x30(%rdi) + movaps %xmm3, -0x40(%rdi) + sub $64, %rdx + sub $0x40, %rsi + sub $0x40, %rdi L(shl_0_less_64bytes_bwd): - BRANCH_TO_JMPTBL_ENTRY (L(table_less_80bytes), %rdx, 4) + BRANCH_TO_JMPTBL_ENTRY (L(table_less_80bytes), %rdx, 4) - .p2align 4 + .p2align 4 L(shl_0_gobble_bwd): #ifdef DATA_CACHE_SIZE_HALF - cmp $DATA_CACHE_SIZE_HALF, %RDX_LP + cmp $DATA_CACHE_SIZE_HALF, %RDX_LP #else - cmp __x86_data_cache_size_half(%rip), %RDX_LP + cmp __x86_data_cache_size_half(%rip), %RDX_LP #endif - lea -128(%rdx), %rdx - jae L(shl_0_gobble_mem_bwd_loop) + lea -128(%rdx), %rdx + jae L(shl_0_gobble_mem_bwd_loop) L(shl_0_gobble_bwd_loop): - movdqa -0x10(%rsi), %xmm0 - movaps -0x20(%rsi), %xmm1 - movaps -0x30(%rsi), %xmm2 - movaps -0x40(%rsi), %xmm3 + movdqa -0x10(%rsi), %xmm0 + movaps -0x20(%rsi), %xmm1 + movaps -0x30(%rsi), %xmm2 + movaps -0x40(%rsi), %xmm3 - movdqa %xmm0, -0x10(%rdi) - movaps %xmm1, -0x20(%rdi) - movaps %xmm2, -0x30(%rdi) - movaps %xmm3, -0x40(%rdi) + movdqa %xmm0, -0x10(%rdi) + movaps %xmm1, -0x20(%rdi) + movaps %xmm2, -0x30(%rdi) + movaps %xmm3, -0x40(%rdi) - sub $0x80, %rdx - movaps -0x50(%rsi), %xmm4 - movaps -0x60(%rsi), %xmm5 - movaps -0x70(%rsi), %xmm6 - movaps -0x80(%rsi), %xmm7 - lea -0x80(%rsi), %rsi - movaps %xmm4, -0x50(%rdi) - movaps %xmm5, -0x60(%rdi) - movaps %xmm6, -0x70(%rdi) - movaps %xmm7, -0x80(%rdi) - lea -0x80(%rdi), %rdi + sub $0x80, %rdx + movaps -0x50(%rsi), %xmm4 + movaps -0x60(%rsi), %xmm5 + movaps -0x70(%rsi), %xmm6 + movaps -0x80(%rsi), %xmm7 + lea -0x80(%rsi), %rsi + movaps %xmm4, -0x50(%rdi) + movaps %xmm5, -0x60(%rdi) + movaps %xmm6, -0x70(%rdi) + movaps %xmm7, -0x80(%rdi) + lea -0x80(%rdi), %rdi - jae L(shl_0_gobble_bwd_loop) - cmp $-0x40, %rdx - lea 0x80(%rdx), %rdx - jl L(shl_0_gobble_bwd_less_64bytes) + jae L(shl_0_gobble_bwd_loop) + cmp $-0x40, %rdx + lea 0x80(%rdx), %rdx + jl L(shl_0_gobble_bwd_less_64bytes) - movdqa -0x10(%rsi), %xmm0 - sub $0x40, %rdx - movdqa -0x20(%rsi), %xmm1 + movdqa -0x10(%rsi), %xmm0 + sub $0x40, %rdx + movdqa -0x20(%rsi), %xmm1 - movdqa %xmm0, -0x10(%rdi) - movdqa %xmm1, -0x20(%rdi) + movdqa %xmm0, -0x10(%rdi) + movdqa %xmm1, -0x20(%rdi) - movdqa -0x30(%rsi), %xmm0 - movdqa -0x40(%rsi), %xmm1 - sub $0x40, %rsi + movdqa -0x30(%rsi), %xmm0 + movdqa -0x40(%rsi), %xmm1 + sub $0x40, %rsi - movdqa %xmm0, -0x30(%rdi) - movdqa %xmm1, -0x40(%rdi) - sub $0x40, %rdi + movdqa %xmm0, -0x30(%rdi) + movdqa %xmm1, -0x40(%rdi) + sub $0x40, %rdi L(shl_0_gobble_bwd_less_64bytes): - BRANCH_TO_JMPTBL_ENTRY (L(table_less_80bytes), %rdx, 4) + BRANCH_TO_JMPTBL_ENTRY (L(table_less_80bytes), %rdx, 4) - .p2align 4 + .p2align 4 L(shl_0_gobble_mem_bwd_loop): - prefetcht0 -0x1c0(%rsi) - prefetcht0 -0x280(%rsi) - movdqa -0x10(%rsi), %xmm0 - movdqa -0x20(%rsi), %xmm1 - movdqa -0x30(%rsi), %xmm2 - movdqa -0x40(%rsi), %xmm3 - movdqa -0x50(%rsi), %xmm4 - movdqa -0x60(%rsi), %xmm5 - movdqa -0x70(%rsi), %xmm6 - movdqa -0x80(%rsi), %xmm7 - lea -0x80(%rsi), %rsi - sub $0x80, %rdx - movdqa %xmm0, -0x10(%rdi) - movdqa %xmm1, -0x20(%rdi) - movdqa %xmm2, -0x30(%rdi) - movdqa %xmm3, -0x40(%rdi) - movdqa %xmm4, -0x50(%rdi) - movdqa %xmm5, -0x60(%rdi) - movdqa %xmm6, -0x70(%rdi) - movdqa %xmm7, -0x80(%rdi) - lea -0x80(%rdi), %rdi + prefetcht0 -0x1c0(%rsi) + prefetcht0 -0x280(%rsi) + movdqa -0x10(%rsi), %xmm0 + movdqa -0x20(%rsi), %xmm1 + movdqa -0x30(%rsi), %xmm2 + movdqa -0x40(%rsi), %xmm3 + movdqa -0x50(%rsi), %xmm4 + movdqa -0x60(%rsi), %xmm5 + movdqa -0x70(%rsi), %xmm6 + movdqa -0x80(%rsi), %xmm7 + lea -0x80(%rsi), %rsi + sub $0x80, %rdx + movdqa %xmm0, -0x10(%rdi) + movdqa %xmm1, -0x20(%rdi) + movdqa %xmm2, -0x30(%rdi) + movdqa %xmm3, -0x40(%rdi) + movdqa %xmm4, -0x50(%rdi) + movdqa %xmm5, -0x60(%rdi) + movdqa %xmm6, -0x70(%rdi) + movdqa %xmm7, -0x80(%rdi) + lea -0x80(%rdi), %rdi - jae L(shl_0_gobble_mem_bwd_loop) - cmp $-0x40, %rdx - lea 0x80(%rdx), %rdx - jl L(shl_0_mem_bwd_less_64bytes) + jae L(shl_0_gobble_mem_bwd_loop) + cmp $-0x40, %rdx + lea 0x80(%rdx), %rdx + jl L(shl_0_mem_bwd_less_64bytes) - movdqa -0x10(%rsi), %xmm0 - sub $0x40, %rdx - movdqa -0x20(%rsi), %xmm1 + movdqa -0x10(%rsi), %xmm0 + sub $0x40, %rdx + movdqa -0x20(%rsi), %xmm1 - movdqa %xmm0, -0x10(%rdi) - movdqa %xmm1, -0x20(%rdi) + movdqa %xmm0, -0x10(%rdi) + movdqa %xmm1, -0x20(%rdi) - movdqa -0x30(%rsi), %xmm0 - movdqa -0x40(%rsi), %xmm1 - sub $0x40, %rsi + movdqa -0x30(%rsi), %xmm0 + movdqa -0x40(%rsi), %xmm1 + sub $0x40, %rsi - movdqa %xmm0, -0x30(%rdi) - movdqa %xmm1, -0x40(%rdi) - sub $0x40, %rdi + movdqa %xmm0, -0x30(%rdi) + movdqa %xmm1, -0x40(%rdi) + sub $0x40, %rdi L(shl_0_mem_bwd_less_64bytes): - cmp $0x20, %rdx - jb L(shl_0_mem_bwd_less_32bytes) - movdqa -0x10(%rsi), %xmm0 - sub $0x20, %rdx - movdqa -0x20(%rsi), %xmm1 - sub $0x20, %rsi - movdqa %xmm0, -0x10(%rdi) - movdqa %xmm1, -0x20(%rdi) - sub $0x20, %rdi + cmp $0x20, %rdx + jb L(shl_0_mem_bwd_less_32bytes) + movdqa -0x10(%rsi), %xmm0 + sub $0x20, %rdx + movdqa -0x20(%rsi), %xmm1 + sub $0x20, %rsi + movdqa %xmm0, -0x10(%rdi) + movdqa %xmm1, -0x20(%rdi) + sub $0x20, %rdi L(shl_0_mem_bwd_less_32bytes): - BRANCH_TO_JMPTBL_ENTRY (L(table_less_80bytes), %rdx, 4) + BRANCH_TO_JMPTBL_ENTRY (L(table_less_80bytes), %rdx, 4) - .p2align 4 + .p2align 4 L(shl_1): - lea (L(shl_1_loop_L1)-L(shl_1))(%r9), %r9 - cmp %rcx, %rdx - movaps -0x01(%rsi), %xmm1 - jb L(L1_fwd) - lea (L(shl_1_loop_L2)-L(shl_1_loop_L1))(%r9), %r9 + lea (L(shl_1_loop_L1)-L(shl_1))(%r9), %r9 + cmp %rcx, %rdx + movaps -0x01(%rsi), %xmm1 + jb L(L1_fwd) + lea (L(shl_1_loop_L2)-L(shl_1_loop_L1))(%r9), %r9 L(L1_fwd): - lea -64(%rdx), %rdx - _CET_NOTRACK jmp *%r9 - ud2 + lea -64(%rdx), %rdx + _CET_NOTRACK jmp *%r9 + ud2 L(shl_1_loop_L2): - prefetchnta 0x1c0(%rsi) + prefetchnta 0x1c0(%rsi) L(shl_1_loop_L1): - sub $64, %rdx - movaps 0x0f(%rsi), %xmm2 - movaps 0x1f(%rsi), %xmm3 - movaps 0x2f(%rsi), %xmm4 - movaps 0x3f(%rsi), %xmm5 - movdqa %xmm5, %xmm6 - palignr $1, %xmm4, %xmm5 - lea 64(%rsi), %rsi - palignr $1, %xmm3, %xmm4 - palignr $1, %xmm2, %xmm3 - lea 64(%rdi), %rdi - palignr $1, %xmm1, %xmm2 - movdqa %xmm6, %xmm1 - movdqa %xmm2, -0x40(%rdi) - movaps %xmm3, -0x30(%rdi) - jb L(shl_1_end) - movaps %xmm4, -0x20(%rdi) - movaps %xmm5, -0x10(%rdi) - _CET_NOTRACK jmp *%r9 - ud2 + sub $64, %rdx + movaps 0x0f(%rsi), %xmm2 + movaps 0x1f(%rsi), %xmm3 + movaps 0x2f(%rsi), %xmm4 + movaps 0x3f(%rsi), %xmm5 + movdqa %xmm5, %xmm6 + palignr $1, %xmm4, %xmm5 + lea 64(%rsi), %rsi + palignr $1, %xmm3, %xmm4 + palignr $1, %xmm2, %xmm3 + lea 64(%rdi), %rdi + palignr $1, %xmm1, %xmm2 + movdqa %xmm6, %xmm1 + movdqa %xmm2, -0x40(%rdi) + movaps %xmm3, -0x30(%rdi) + jb L(shl_1_end) + movaps %xmm4, -0x20(%rdi) + movaps %xmm5, -0x10(%rdi) + _CET_NOTRACK jmp *%r9 + ud2 L(shl_1_end): - movaps %xmm4, -0x20(%rdi) - lea 64(%rdx), %rdx - movaps %xmm5, -0x10(%rdi) - add %rdx, %rdi - movdqu %xmm0, (%r8) - add %rdx, %rsi - BRANCH_TO_JMPTBL_ENTRY(L(table_less_80bytes), %rdx, 4) + movaps %xmm4, -0x20(%rdi) + lea 64(%rdx), %rdx + movaps %xmm5, -0x10(%rdi) + add %rdx, %rdi + movdqu %xmm0, (%r8) + add %rdx, %rsi + BRANCH_TO_JMPTBL_ENTRY(L(table_less_80bytes), %rdx, 4) - .p2align 4 + .p2align 4 L(shl_1_bwd): - lea (L(shl_1_bwd_loop_L1)-L(shl_1_bwd))(%r9), %r9 - cmp %rcx, %rdx - movaps -0x01(%rsi), %xmm1 - jb L(L1_bwd) - lea (L(shl_1_bwd_loop_L2)-L(shl_1_bwd_loop_L1))(%r9), %r9 + lea (L(shl_1_bwd_loop_L1)-L(shl_1_bwd))(%r9), %r9 + cmp %rcx, %rdx + movaps -0x01(%rsi), %xmm1 + jb L(L1_bwd) + lea (L(shl_1_bwd_loop_L2)-L(shl_1_bwd_loop_L1))(%r9), %r9 L(L1_bwd): - lea -64(%rdx), %rdx - _CET_NOTRACK jmp *%r9 - ud2 + lea -64(%rdx), %rdx + _CET_NOTRACK jmp *%r9 + ud2 L(shl_1_bwd_loop_L2): - prefetchnta -0x1c0(%rsi) + prefetchnta -0x1c0(%rsi) L(shl_1_bwd_loop_L1): - movaps -0x11(%rsi), %xmm2 - sub $0x40, %rdx - movaps -0x21(%rsi), %xmm3 - movaps -0x31(%rsi), %xmm4 - movaps -0x41(%rsi), %xmm5 - lea -0x40(%rsi), %rsi - palignr $1, %xmm2, %xmm1 - palignr $1, %xmm3, %xmm2 - palignr $1, %xmm4, %xmm3 - palignr $1, %xmm5, %xmm4 + movaps -0x11(%rsi), %xmm2 + sub $0x40, %rdx + movaps -0x21(%rsi), %xmm3 + movaps -0x31(%rsi), %xmm4 + movaps -0x41(%rsi), %xmm5 + lea -0x40(%rsi), %rsi + palignr $1, %xmm2, %xmm1 + palignr $1, %xmm3, %xmm2 + palignr $1, %xmm4, %xmm3 + palignr $1, %xmm5, %xmm4 - movaps %xmm1, -0x10(%rdi) - movaps %xmm5, %xmm1 + movaps %xmm1, -0x10(%rdi) + movaps %xmm5, %xmm1 - movaps %xmm2, -0x20(%rdi) - lea -0x40(%rdi), %rdi + movaps %xmm2, -0x20(%rdi) + lea -0x40(%rdi), %rdi - movaps %xmm3, 0x10(%rdi) - jb L(shl_1_bwd_end) - movaps %xmm4, (%rdi) - _CET_NOTRACK jmp *%r9 - ud2 + movaps %xmm3, 0x10(%rdi) + jb L(shl_1_bwd_end) + movaps %xmm4, (%rdi) + _CET_NOTRACK jmp *%r9 + ud2 L(shl_1_bwd_end): - movaps %xmm4, (%rdi) - lea 64(%rdx), %rdx - movdqu %xmm0, (%r8) - BRANCH_TO_JMPTBL_ENTRY(L(table_less_80bytes), %rdx, 4) + movaps %xmm4, (%rdi) + lea 64(%rdx), %rdx + movdqu %xmm0, (%r8) + BRANCH_TO_JMPTBL_ENTRY(L(table_less_80bytes), %rdx, 4) - .p2align 4 + .p2align 4 L(shl_2): - lea (L(shl_2_loop_L1)-L(shl_2))(%r9), %r9 - cmp %rcx, %rdx - movaps -0x02(%rsi), %xmm1 - jb L(L2_fwd) - lea (L(shl_2_loop_L2)-L(shl_2_loop_L1))(%r9), %r9 + lea (L(shl_2_loop_L1)-L(shl_2))(%r9), %r9 + cmp %rcx, %rdx + movaps -0x02(%rsi), %xmm1 + jb L(L2_fwd) + lea (L(shl_2_loop_L2)-L(shl_2_loop_L1))(%r9), %r9 L(L2_fwd): - lea -64(%rdx), %rdx - _CET_NOTRACK jmp *%r9 - ud2 + lea -64(%rdx), %rdx + _CET_NOTRACK jmp *%r9 + ud2 L(shl_2_loop_L2): - prefetchnta 0x1c0(%rsi) + prefetchnta 0x1c0(%rsi) L(shl_2_loop_L1): - sub $64, %rdx - movaps 0x0e(%rsi), %xmm2 - movaps 0x1e(%rsi), %xmm3 - movaps 0x2e(%rsi), %xmm4 - movaps 0x3e(%rsi), %xmm5 - movdqa %xmm5, %xmm6 - palignr $2, %xmm4, %xmm5 - lea 64(%rsi), %rsi - palignr $2, %xmm3, %xmm4 - palignr $2, %xmm2, %xmm3 - lea 64(%rdi), %rdi - palignr $2, %xmm1, %xmm2 - movdqa %xmm6, %xmm1 - movdqa %xmm2, -0x40(%rdi) - movaps %xmm3, -0x30(%rdi) - jb L(shl_2_end) - movaps %xmm4, -0x20(%rdi) - movaps %xmm5, -0x10(%rdi) - _CET_NOTRACK jmp *%r9 - ud2 + sub $64, %rdx + movaps 0x0e(%rsi), %xmm2 + movaps 0x1e(%rsi), %xmm3 + movaps 0x2e(%rsi), %xmm4 + movaps 0x3e(%rsi), %xmm5 + movdqa %xmm5, %xmm6 + palignr $2, %xmm4, %xmm5 + lea 64(%rsi), %rsi + palignr $2, %xmm3, %xmm4 + palignr $2, %xmm2, %xmm3 + lea 64(%rdi), %rdi + palignr $2, %xmm1, %xmm2 + movdqa %xmm6, %xmm1 + movdqa %xmm2, -0x40(%rdi) + movaps %xmm3, -0x30(%rdi) + jb L(shl_2_end) + movaps %xmm4, -0x20(%rdi) + movaps %xmm5, -0x10(%rdi) + _CET_NOTRACK jmp *%r9 + ud2 L(shl_2_end): - movaps %xmm4, -0x20(%rdi) - lea 64(%rdx), %rdx - movaps %xmm5, -0x10(%rdi) - add %rdx, %rdi - movdqu %xmm0, (%r8) - add %rdx, %rsi - BRANCH_TO_JMPTBL_ENTRY(L(table_less_80bytes), %rdx, 4) + movaps %xmm4, -0x20(%rdi) + lea 64(%rdx), %rdx + movaps %xmm5, -0x10(%rdi) + add %rdx, %rdi + movdqu %xmm0, (%r8) + add %rdx, %rsi + BRANCH_TO_JMPTBL_ENTRY(L(table_less_80bytes), %rdx, 4) - .p2align 4 + .p2align 4 L(shl_2_bwd): - lea (L(shl_2_bwd_loop_L1)-L(shl_2_bwd))(%r9), %r9 - cmp %rcx, %rdx - movaps -0x02(%rsi), %xmm1 - jb L(L2_bwd) - lea (L(shl_2_bwd_loop_L2)-L(shl_2_bwd_loop_L1))(%r9), %r9 + lea (L(shl_2_bwd_loop_L1)-L(shl_2_bwd))(%r9), %r9 + cmp %rcx, %rdx + movaps -0x02(%rsi), %xmm1 + jb L(L2_bwd) + lea (L(shl_2_bwd_loop_L2)-L(shl_2_bwd_loop_L1))(%r9), %r9 L(L2_bwd): - lea -64(%rdx), %rdx - _CET_NOTRACK jmp *%r9 - ud2 + lea -64(%rdx), %rdx + _CET_NOTRACK jmp *%r9 + ud2 L(shl_2_bwd_loop_L2): - prefetchnta -0x1c0(%rsi) + prefetchnta -0x1c0(%rsi) L(shl_2_bwd_loop_L1): - movaps -0x12(%rsi), %xmm2 - sub $0x40, %rdx - movaps -0x22(%rsi), %xmm3 - movaps -0x32(%rsi), %xmm4 - movaps -0x42(%rsi), %xmm5 - lea -0x40(%rsi), %rsi - palignr $2, %xmm2, %xmm1 - palignr $2, %xmm3, %xmm2 - palignr $2, %xmm4, %xmm3 - palignr $2, %xmm5, %xmm4 + movaps -0x12(%rsi), %xmm2 + sub $0x40, %rdx + movaps -0x22(%rsi), %xmm3 + movaps -0x32(%rsi), %xmm4 + movaps -0x42(%rsi), %xmm5 + lea -0x40(%rsi), %rsi + palignr $2, %xmm2, %xmm1 + palignr $2, %xmm3, %xmm2 + palignr $2, %xmm4, %xmm3 + palignr $2, %xmm5, %xmm4 - movaps %xmm1, -0x10(%rdi) - movaps %xmm5, %xmm1 + movaps %xmm1, -0x10(%rdi) + movaps %xmm5, %xmm1 - movaps %xmm2, -0x20(%rdi) - lea -0x40(%rdi), %rdi + movaps %xmm2, -0x20(%rdi) + lea -0x40(%rdi), %rdi - movaps %xmm3, 0x10(%rdi) - jb L(shl_2_bwd_end) - movaps %xmm4, (%rdi) - _CET_NOTRACK jmp *%r9 - ud2 + movaps %xmm3, 0x10(%rdi) + jb L(shl_2_bwd_end) + movaps %xmm4, (%rdi) + _CET_NOTRACK jmp *%r9 + ud2 L(shl_2_bwd_end): - movaps %xmm4, (%rdi) - lea 64(%rdx), %rdx - movdqu %xmm0, (%r8) - BRANCH_TO_JMPTBL_ENTRY(L(table_less_80bytes), %rdx, 4) + movaps %xmm4, (%rdi) + lea 64(%rdx), %rdx + movdqu %xmm0, (%r8) + BRANCH_TO_JMPTBL_ENTRY(L(table_less_80bytes), %rdx, 4) - .p2align 4 + .p2align 4 L(shl_3): - lea (L(shl_3_loop_L1)-L(shl_3))(%r9), %r9 - cmp %rcx, %rdx - movaps -0x03(%rsi), %xmm1 - jb L(L3_fwd) - lea (L(shl_3_loop_L2)-L(shl_3_loop_L1))(%r9), %r9 + lea (L(shl_3_loop_L1)-L(shl_3))(%r9), %r9 + cmp %rcx, %rdx + movaps -0x03(%rsi), %xmm1 + jb L(L3_fwd) + lea (L(shl_3_loop_L2)-L(shl_3_loop_L1))(%r9), %r9 L(L3_fwd): - lea -64(%rdx), %rdx - _CET_NOTRACK jmp *%r9 - ud2 + lea -64(%rdx), %rdx + _CET_NOTRACK jmp *%r9 + ud2 L(shl_3_loop_L2): - prefetchnta 0x1c0(%rsi) + prefetchnta 0x1c0(%rsi) L(shl_3_loop_L1): - sub $64, %rdx - movaps 0x0d(%rsi), %xmm2 - movaps 0x1d(%rsi), %xmm3 - movaps 0x2d(%rsi), %xmm4 - movaps 0x3d(%rsi), %xmm5 - movdqa %xmm5, %xmm6 - palignr $3, %xmm4, %xmm5 - lea 64(%rsi), %rsi - palignr $3, %xmm3, %xmm4 - palignr $3, %xmm2, %xmm3 - lea 64(%rdi), %rdi - palignr $3, %xmm1, %xmm2 - movdqa %xmm6, %xmm1 - movdqa %xmm2, -0x40(%rdi) - movaps %xmm3, -0x30(%rdi) - jb L(shl_3_end) - movaps %xmm4, -0x20(%rdi) - movaps %xmm5, -0x10(%rdi) - _CET_NOTRACK jmp *%r9 - ud2 + sub $64, %rdx + movaps 0x0d(%rsi), %xmm2 + movaps 0x1d(%rsi), %xmm3 + movaps 0x2d(%rsi), %xmm4 + movaps 0x3d(%rsi), %xmm5 + movdqa %xmm5, %xmm6 + palignr $3, %xmm4, %xmm5 + lea 64(%rsi), %rsi + palignr $3, %xmm3, %xmm4 + palignr $3, %xmm2, %xmm3 + lea 64(%rdi), %rdi + palignr $3, %xmm1, %xmm2 + movdqa %xmm6, %xmm1 + movdqa %xmm2, -0x40(%rdi) + movaps %xmm3, -0x30(%rdi) + jb L(shl_3_end) + movaps %xmm4, -0x20(%rdi) + movaps %xmm5, -0x10(%rdi) + _CET_NOTRACK jmp *%r9 + ud2 L(shl_3_end): - movaps %xmm4, -0x20(%rdi) - lea 64(%rdx), %rdx - movaps %xmm5, -0x10(%rdi) - add %rdx, %rdi - movdqu %xmm0, (%r8) - add %rdx, %rsi - BRANCH_TO_JMPTBL_ENTRY(L(table_less_80bytes), %rdx, 4) + movaps %xmm4, -0x20(%rdi) + lea 64(%rdx), %rdx + movaps %xmm5, -0x10(%rdi) + add %rdx, %rdi + movdqu %xmm0, (%r8) + add %rdx, %rsi + BRANCH_TO_JMPTBL_ENTRY(L(table_less_80bytes), %rdx, 4) - .p2align 4 + .p2align 4 L(shl_3_bwd): - lea (L(shl_3_bwd_loop_L1)-L(shl_3_bwd))(%r9), %r9 - cmp %rcx, %rdx - movaps -0x03(%rsi), %xmm1 - jb L(L3_bwd) - lea (L(shl_3_bwd_loop_L2)-L(shl_3_bwd_loop_L1))(%r9), %r9 + lea (L(shl_3_bwd_loop_L1)-L(shl_3_bwd))(%r9), %r9 + cmp %rcx, %rdx + movaps -0x03(%rsi), %xmm1 + jb L(L3_bwd) + lea (L(shl_3_bwd_loop_L2)-L(shl_3_bwd_loop_L1))(%r9), %r9 L(L3_bwd): - lea -64(%rdx), %rdx - _CET_NOTRACK jmp *%r9 - ud2 + lea -64(%rdx), %rdx + _CET_NOTRACK jmp *%r9 + ud2 L(shl_3_bwd_loop_L2): - prefetchnta -0x1c0(%rsi) + prefetchnta -0x1c0(%rsi) L(shl_3_bwd_loop_L1): - movaps -0x13(%rsi), %xmm2 - sub $0x40, %rdx - movaps -0x23(%rsi), %xmm3 - movaps -0x33(%rsi), %xmm4 - movaps -0x43(%rsi), %xmm5 - lea -0x40(%rsi), %rsi - palignr $3, %xmm2, %xmm1 - palignr $3, %xmm3, %xmm2 - palignr $3, %xmm4, %xmm3 - palignr $3, %xmm5, %xmm4 + movaps -0x13(%rsi), %xmm2 + sub $0x40, %rdx + movaps -0x23(%rsi), %xmm3 + movaps -0x33(%rsi), %xmm4 + movaps -0x43(%rsi), %xmm5 + lea -0x40(%rsi), %rsi + palignr $3, %xmm2, %xmm1 + palignr $3, %xmm3, %xmm2 + palignr $3, %xmm4, %xmm3 + palignr $3, %xmm5, %xmm4 - movaps %xmm1, -0x10(%rdi) - movaps %xmm5, %xmm1 + movaps %xmm1, -0x10(%rdi) + movaps %xmm5, %xmm1 - movaps %xmm2, -0x20(%rdi) - lea -0x40(%rdi), %rdi + movaps %xmm2, -0x20(%rdi) + lea -0x40(%rdi), %rdi - movaps %xmm3, 0x10(%rdi) - jb L(shl_3_bwd_end) - movaps %xmm4, (%rdi) - _CET_NOTRACK jmp *%r9 - ud2 + movaps %xmm3, 0x10(%rdi) + jb L(shl_3_bwd_end) + movaps %xmm4, (%rdi) + _CET_NOTRACK jmp *%r9 + ud2 L(shl_3_bwd_end): - movaps %xmm4, (%rdi) - lea 64(%rdx), %rdx - movdqu %xmm0, (%r8) - BRANCH_TO_JMPTBL_ENTRY(L(table_less_80bytes), %rdx, 4) + movaps %xmm4, (%rdi) + lea 64(%rdx), %rdx + movdqu %xmm0, (%r8) + BRANCH_TO_JMPTBL_ENTRY(L(table_less_80bytes), %rdx, 4) - .p2align 4 + .p2align 4 L(shl_4): - lea (L(shl_4_loop_L1)-L(shl_4))(%r9), %r9 - cmp %rcx, %rdx - movaps -0x04(%rsi), %xmm1 - jb L(L4_fwd) - lea (L(shl_4_loop_L2)-L(shl_4_loop_L1))(%r9), %r9 + lea (L(shl_4_loop_L1)-L(shl_4))(%r9), %r9 + cmp %rcx, %rdx + movaps -0x04(%rsi), %xmm1 + jb L(L4_fwd) + lea (L(shl_4_loop_L2)-L(shl_4_loop_L1))(%r9), %r9 L(L4_fwd): - lea -64(%rdx), %rdx - _CET_NOTRACK jmp *%r9 - ud2 + lea -64(%rdx), %rdx + _CET_NOTRACK jmp *%r9 + ud2 L(shl_4_loop_L2): - prefetchnta 0x1c0(%rsi) + prefetchnta 0x1c0(%rsi) L(shl_4_loop_L1): - sub $64, %rdx - movaps 0x0c(%rsi), %xmm2 - movaps 0x1c(%rsi), %xmm3 - movaps 0x2c(%rsi), %xmm4 - movaps 0x3c(%rsi), %xmm5 - movdqa %xmm5, %xmm6 - palignr $4, %xmm4, %xmm5 - lea 64(%rsi), %rsi - palignr $4, %xmm3, %xmm4 - palignr $4, %xmm2, %xmm3 - lea 64(%rdi), %rdi - palignr $4, %xmm1, %xmm2 - movdqa %xmm6, %xmm1 - movdqa %xmm2, -0x40(%rdi) - movaps %xmm3, -0x30(%rdi) - jb L(shl_4_end) - movaps %xmm4, -0x20(%rdi) - movaps %xmm5, -0x10(%rdi) - _CET_NOTRACK jmp *%r9 - ud2 + sub $64, %rdx + movaps 0x0c(%rsi), %xmm2 + movaps 0x1c(%rsi), %xmm3 + movaps 0x2c(%rsi), %xmm4 + movaps 0x3c(%rsi), %xmm5 + movdqa %xmm5, %xmm6 + palignr $4, %xmm4, %xmm5 + lea 64(%rsi), %rsi + palignr $4, %xmm3, %xmm4 + palignr $4, %xmm2, %xmm3 + lea 64(%rdi), %rdi + palignr $4, %xmm1, %xmm2 + movdqa %xmm6, %xmm1 + movdqa %xmm2, -0x40(%rdi) + movaps %xmm3, -0x30(%rdi) + jb L(shl_4_end) + movaps %xmm4, -0x20(%rdi) + movaps %xmm5, -0x10(%rdi) + _CET_NOTRACK jmp *%r9 + ud2 L(shl_4_end): - movaps %xmm4, -0x20(%rdi) - lea 64(%rdx), %rdx - movaps %xmm5, -0x10(%rdi) - add %rdx, %rdi - movdqu %xmm0, (%r8) - add %rdx, %rsi - BRANCH_TO_JMPTBL_ENTRY(L(table_less_80bytes), %rdx, 4) + movaps %xmm4, -0x20(%rdi) + lea 64(%rdx), %rdx + movaps %xmm5, -0x10(%rdi) + add %rdx, %rdi + movdqu %xmm0, (%r8) + add %rdx, %rsi + BRANCH_TO_JMPTBL_ENTRY(L(table_less_80bytes), %rdx, 4) - .p2align 4 + .p2align 4 L(shl_4_bwd): - lea (L(shl_4_bwd_loop_L1)-L(shl_4_bwd))(%r9), %r9 - cmp %rcx, %rdx - movaps -0x04(%rsi), %xmm1 - jb L(L4_bwd) - lea (L(shl_4_bwd_loop_L2)-L(shl_4_bwd_loop_L1))(%r9), %r9 + lea (L(shl_4_bwd_loop_L1)-L(shl_4_bwd))(%r9), %r9 + cmp %rcx, %rdx + movaps -0x04(%rsi), %xmm1 + jb L(L4_bwd) + lea (L(shl_4_bwd_loop_L2)-L(shl_4_bwd_loop_L1))(%r9), %r9 L(L4_bwd): - lea -64(%rdx), %rdx - _CET_NOTRACK jmp *%r9 - ud2 + lea -64(%rdx), %rdx + _CET_NOTRACK jmp *%r9 + ud2 L(shl_4_bwd_loop_L2): - prefetchnta -0x1c0(%rsi) + prefetchnta -0x1c0(%rsi) L(shl_4_bwd_loop_L1): - movaps -0x14(%rsi), %xmm2 - sub $0x40, %rdx - movaps -0x24(%rsi), %xmm3 - movaps -0x34(%rsi), %xmm4 - movaps -0x44(%rsi), %xmm5 - lea -0x40(%rsi), %rsi - palignr $4, %xmm2, %xmm1 - palignr $4, %xmm3, %xmm2 - palignr $4, %xmm4, %xmm3 - palignr $4, %xmm5, %xmm4 + movaps -0x14(%rsi), %xmm2 + sub $0x40, %rdx + movaps -0x24(%rsi), %xmm3 + movaps -0x34(%rsi), %xmm4 + movaps -0x44(%rsi), %xmm5 + lea -0x40(%rsi), %rsi + palignr $4, %xmm2, %xmm1 + palignr $4, %xmm3, %xmm2 + palignr $4, %xmm4, %xmm3 + palignr $4, %xmm5, %xmm4 - movaps %xmm1, -0x10(%rdi) - movaps %xmm5, %xmm1 + movaps %xmm1, -0x10(%rdi) + movaps %xmm5, %xmm1 - movaps %xmm2, -0x20(%rdi) - lea -0x40(%rdi), %rdi + movaps %xmm2, -0x20(%rdi) + lea -0x40(%rdi), %rdi - movaps %xmm3, 0x10(%rdi) - jb L(shl_4_bwd_end) - movaps %xmm4, (%rdi) - _CET_NOTRACK jmp *%r9 - ud2 + movaps %xmm3, 0x10(%rdi) + jb L(shl_4_bwd_end) + movaps %xmm4, (%rdi) + _CET_NOTRACK jmp *%r9 + ud2 L(shl_4_bwd_end): - movaps %xmm4, (%rdi) - lea 64(%rdx), %rdx - movdqu %xmm0, (%r8) - BRANCH_TO_JMPTBL_ENTRY(L(table_less_80bytes), %rdx, 4) + movaps %xmm4, (%rdi) + lea 64(%rdx), %rdx + movdqu %xmm0, (%r8) + BRANCH_TO_JMPTBL_ENTRY(L(table_less_80bytes), %rdx, 4) - .p2align 4 + .p2align 4 L(shl_5): - lea (L(shl_5_loop_L1)-L(shl_5))(%r9), %r9 - cmp %rcx, %rdx - movaps -0x05(%rsi), %xmm1 - jb L(L5_fwd) - lea (L(shl_5_loop_L2)-L(shl_5_loop_L1))(%r9), %r9 + lea (L(shl_5_loop_L1)-L(shl_5))(%r9), %r9 + cmp %rcx, %rdx + movaps -0x05(%rsi), %xmm1 + jb L(L5_fwd) + lea (L(shl_5_loop_L2)-L(shl_5_loop_L1))(%r9), %r9 L(L5_fwd): - lea -64(%rdx), %rdx - _CET_NOTRACK jmp *%r9 - ud2 + lea -64(%rdx), %rdx + _CET_NOTRACK jmp *%r9 + ud2 L(shl_5_loop_L2): - prefetchnta 0x1c0(%rsi) + prefetchnta 0x1c0(%rsi) L(shl_5_loop_L1): - sub $64, %rdx - movaps 0x0b(%rsi), %xmm2 - movaps 0x1b(%rsi), %xmm3 - movaps 0x2b(%rsi), %xmm4 - movaps 0x3b(%rsi), %xmm5 - movdqa %xmm5, %xmm6 - palignr $5, %xmm4, %xmm5 - lea 64(%rsi), %rsi - palignr $5, %xmm3, %xmm4 - palignr $5, %xmm2, %xmm3 - lea 64(%rdi), %rdi - palignr $5, %xmm1, %xmm2 - movdqa %xmm6, %xmm1 - movdqa %xmm2, -0x40(%rdi) - movaps %xmm3, -0x30(%rdi) - jb L(shl_5_end) - movaps %xmm4, -0x20(%rdi) - movaps %xmm5, -0x10(%rdi) - _CET_NOTRACK jmp *%r9 - ud2 + sub $64, %rdx + movaps 0x0b(%rsi), %xmm2 + movaps 0x1b(%rsi), %xmm3 + movaps 0x2b(%rsi), %xmm4 + movaps 0x3b(%rsi), %xmm5 + movdqa %xmm5, %xmm6 + palignr $5, %xmm4, %xmm5 + lea 64(%rsi), %rsi + palignr $5, %xmm3, %xmm4 + palignr $5, %xmm2, %xmm3 + lea 64(%rdi), %rdi + palignr $5, %xmm1, %xmm2 + movdqa %xmm6, %xmm1 + movdqa %xmm2, -0x40(%rdi) + movaps %xmm3, -0x30(%rdi) + jb L(shl_5_end) + movaps %xmm4, -0x20(%rdi) + movaps %xmm5, -0x10(%rdi) + _CET_NOTRACK jmp *%r9 + ud2 L(shl_5_end): - movaps %xmm4, -0x20(%rdi) - lea 64(%rdx), %rdx - movaps %xmm5, -0x10(%rdi) - add %rdx, %rdi - movdqu %xmm0, (%r8) - add %rdx, %rsi - BRANCH_TO_JMPTBL_ENTRY(L(table_less_80bytes), %rdx, 4) + movaps %xmm4, -0x20(%rdi) + lea 64(%rdx), %rdx + movaps %xmm5, -0x10(%rdi) + add %rdx, %rdi + movdqu %xmm0, (%r8) + add %rdx, %rsi + BRANCH_TO_JMPTBL_ENTRY(L(table_less_80bytes), %rdx, 4) - .p2align 4 + .p2align 4 L(shl_5_bwd): - lea (L(shl_5_bwd_loop_L1)-L(shl_5_bwd))(%r9), %r9 - cmp %rcx, %rdx - movaps -0x05(%rsi), %xmm1 - jb L(L5_bwd) - lea (L(shl_5_bwd_loop_L2)-L(shl_5_bwd_loop_L1))(%r9), %r9 + lea (L(shl_5_bwd_loop_L1)-L(shl_5_bwd))(%r9), %r9 + cmp %rcx, %rdx + movaps -0x05(%rsi), %xmm1 + jb L(L5_bwd) + lea (L(shl_5_bwd_loop_L2)-L(shl_5_bwd_loop_L1))(%r9), %r9 L(L5_bwd): - lea -64(%rdx), %rdx - _CET_NOTRACK jmp *%r9 - ud2 + lea -64(%rdx), %rdx + _CET_NOTRACK jmp *%r9 + ud2 L(shl_5_bwd_loop_L2): - prefetchnta -0x1c0(%rsi) + prefetchnta -0x1c0(%rsi) L(shl_5_bwd_loop_L1): - movaps -0x15(%rsi), %xmm2 - sub $0x40, %rdx - movaps -0x25(%rsi), %xmm3 - movaps -0x35(%rsi), %xmm4 - movaps -0x45(%rsi), %xmm5 - lea -0x40(%rsi), %rsi - palignr $5, %xmm2, %xmm1 - palignr $5, %xmm3, %xmm2 - palignr $5, %xmm4, %xmm3 - palignr $5, %xmm5, %xmm4 + movaps -0x15(%rsi), %xmm2 + sub $0x40, %rdx + movaps -0x25(%rsi), %xmm3 + movaps -0x35(%rsi), %xmm4 + movaps -0x45(%rsi), %xmm5 + lea -0x40(%rsi), %rsi + palignr $5, %xmm2, %xmm1 + palignr $5, %xmm3, %xmm2 + palignr $5, %xmm4, %xmm3 + palignr $5, %xmm5, %xmm4 - movaps %xmm1, -0x10(%rdi) - movaps %xmm5, %xmm1 + movaps %xmm1, -0x10(%rdi) + movaps %xmm5, %xmm1 - movaps %xmm2, -0x20(%rdi) - lea -0x40(%rdi), %rdi + movaps %xmm2, -0x20(%rdi) + lea -0x40(%rdi), %rdi - movaps %xmm3, 0x10(%rdi) - jb L(shl_5_bwd_end) - movaps %xmm4, (%rdi) - _CET_NOTRACK jmp *%r9 - ud2 + movaps %xmm3, 0x10(%rdi) + jb L(shl_5_bwd_end) + movaps %xmm4, (%rdi) + _CET_NOTRACK jmp *%r9 + ud2 L(shl_5_bwd_end): - movaps %xmm4, (%rdi) - lea 64(%rdx), %rdx - movdqu %xmm0, (%r8) - BRANCH_TO_JMPTBL_ENTRY(L(table_less_80bytes), %rdx, 4) + movaps %xmm4, (%rdi) + lea 64(%rdx), %rdx + movdqu %xmm0, (%r8) + BRANCH_TO_JMPTBL_ENTRY(L(table_less_80bytes), %rdx, 4) - .p2align 4 + .p2align 4 L(shl_6): - lea (L(shl_6_loop_L1)-L(shl_6))(%r9), %r9 - cmp %rcx, %rdx - movaps -0x06(%rsi), %xmm1 - jb L(L6_fwd) - lea (L(shl_6_loop_L2)-L(shl_6_loop_L1))(%r9), %r9 + lea (L(shl_6_loop_L1)-L(shl_6))(%r9), %r9 + cmp %rcx, %rdx + movaps -0x06(%rsi), %xmm1 + jb L(L6_fwd) + lea (L(shl_6_loop_L2)-L(shl_6_loop_L1))(%r9), %r9 L(L6_fwd): - lea -64(%rdx), %rdx - _CET_NOTRACK jmp *%r9 - ud2 + lea -64(%rdx), %rdx + _CET_NOTRACK jmp *%r9 + ud2 L(shl_6_loop_L2): - prefetchnta 0x1c0(%rsi) + prefetchnta 0x1c0(%rsi) L(shl_6_loop_L1): - sub $64, %rdx - movaps 0x0a(%rsi), %xmm2 - movaps 0x1a(%rsi), %xmm3 - movaps 0x2a(%rsi), %xmm4 - movaps 0x3a(%rsi), %xmm5 - movdqa %xmm5, %xmm6 - palignr $6, %xmm4, %xmm5 - lea 64(%rsi), %rsi - palignr $6, %xmm3, %xmm4 - palignr $6, %xmm2, %xmm3 - lea 64(%rdi), %rdi - palignr $6, %xmm1, %xmm2 - movdqa %xmm6, %xmm1 - movdqa %xmm2, -0x40(%rdi) - movaps %xmm3, -0x30(%rdi) - jb L(shl_6_end) - movaps %xmm4, -0x20(%rdi) - movaps %xmm5, -0x10(%rdi) - _CET_NOTRACK jmp *%r9 - ud2 + sub $64, %rdx + movaps 0x0a(%rsi), %xmm2 + movaps 0x1a(%rsi), %xmm3 + movaps 0x2a(%rsi), %xmm4 + movaps 0x3a(%rsi), %xmm5 + movdqa %xmm5, %xmm6 + palignr $6, %xmm4, %xmm5 + lea 64(%rsi), %rsi + palignr $6, %xmm3, %xmm4 + palignr $6, %xmm2, %xmm3 + lea 64(%rdi), %rdi + palignr $6, %xmm1, %xmm2 + movdqa %xmm6, %xmm1 + movdqa %xmm2, -0x40(%rdi) + movaps %xmm3, -0x30(%rdi) + jb L(shl_6_end) + movaps %xmm4, -0x20(%rdi) + movaps %xmm5, -0x10(%rdi) + _CET_NOTRACK jmp *%r9 + ud2 L(shl_6_end): - movaps %xmm4, -0x20(%rdi) - lea 64(%rdx), %rdx - movaps %xmm5, -0x10(%rdi) - add %rdx, %rdi - movdqu %xmm0, (%r8) - add %rdx, %rsi - BRANCH_TO_JMPTBL_ENTRY(L(table_less_80bytes), %rdx, 4) + movaps %xmm4, -0x20(%rdi) + lea 64(%rdx), %rdx + movaps %xmm5, -0x10(%rdi) + add %rdx, %rdi + movdqu %xmm0, (%r8) + add %rdx, %rsi + BRANCH_TO_JMPTBL_ENTRY(L(table_less_80bytes), %rdx, 4) - .p2align 4 + .p2align 4 L(shl_6_bwd): - lea (L(shl_6_bwd_loop_L1)-L(shl_6_bwd))(%r9), %r9 - cmp %rcx, %rdx - movaps -0x06(%rsi), %xmm1 - jb L(L6_bwd) - lea (L(shl_6_bwd_loop_L2)-L(shl_6_bwd_loop_L1))(%r9), %r9 + lea (L(shl_6_bwd_loop_L1)-L(shl_6_bwd))(%r9), %r9 + cmp %rcx, %rdx + movaps -0x06(%rsi), %xmm1 + jb L(L6_bwd) + lea (L(shl_6_bwd_loop_L2)-L(shl_6_bwd_loop_L1))(%r9), %r9 L(L6_bwd): - lea -64(%rdx), %rdx - _CET_NOTRACK jmp *%r9 - ud2 + lea -64(%rdx), %rdx + _CET_NOTRACK jmp *%r9 + ud2 L(shl_6_bwd_loop_L2): - prefetchnta -0x1c0(%rsi) + prefetchnta -0x1c0(%rsi) L(shl_6_bwd_loop_L1): - movaps -0x16(%rsi), %xmm2 - sub $0x40, %rdx - movaps -0x26(%rsi), %xmm3 - movaps -0x36(%rsi), %xmm4 - movaps -0x46(%rsi), %xmm5 - lea -0x40(%rsi), %rsi - palignr $6, %xmm2, %xmm1 - palignr $6, %xmm3, %xmm2 - palignr $6, %xmm4, %xmm3 - palignr $6, %xmm5, %xmm4 + movaps -0x16(%rsi), %xmm2 + sub $0x40, %rdx + movaps -0x26(%rsi), %xmm3 + movaps -0x36(%rsi), %xmm4 + movaps -0x46(%rsi), %xmm5 + lea -0x40(%rsi), %rsi + palignr $6, %xmm2, %xmm1 + palignr $6, %xmm3, %xmm2 + palignr $6, %xmm4, %xmm3 + palignr $6, %xmm5, %xmm4 - movaps %xmm1, -0x10(%rdi) - movaps %xmm5, %xmm1 + movaps %xmm1, -0x10(%rdi) + movaps %xmm5, %xmm1 - movaps %xmm2, -0x20(%rdi) - lea -0x40(%rdi), %rdi + movaps %xmm2, -0x20(%rdi) + lea -0x40(%rdi), %rdi - movaps %xmm3, 0x10(%rdi) - jb L(shl_6_bwd_end) - movaps %xmm4, (%rdi) - _CET_NOTRACK jmp *%r9 - ud2 + movaps %xmm3, 0x10(%rdi) + jb L(shl_6_bwd_end) + movaps %xmm4, (%rdi) + _CET_NOTRACK jmp *%r9 + ud2 L(shl_6_bwd_end): - movaps %xmm4, (%rdi) - lea 64(%rdx), %rdx - movdqu %xmm0, (%r8) - BRANCH_TO_JMPTBL_ENTRY(L(table_less_80bytes), %rdx, 4) + movaps %xmm4, (%rdi) + lea 64(%rdx), %rdx + movdqu %xmm0, (%r8) + BRANCH_TO_JMPTBL_ENTRY(L(table_less_80bytes), %rdx, 4) - .p2align 4 + .p2align 4 L(shl_7): - lea (L(shl_7_loop_L1)-L(shl_7))(%r9), %r9 - cmp %rcx, %rdx - movaps -0x07(%rsi), %xmm1 - jb L(L7_fwd) - lea (L(shl_7_loop_L2)-L(shl_7_loop_L1))(%r9), %r9 + lea (L(shl_7_loop_L1)-L(shl_7))(%r9), %r9 + cmp %rcx, %rdx + movaps -0x07(%rsi), %xmm1 + jb L(L7_fwd) + lea (L(shl_7_loop_L2)-L(shl_7_loop_L1))(%r9), %r9 L(L7_fwd): - lea -64(%rdx), %rdx - _CET_NOTRACK jmp *%r9 - ud2 + lea -64(%rdx), %rdx + _CET_NOTRACK jmp *%r9 + ud2 L(shl_7_loop_L2): - prefetchnta 0x1c0(%rsi) + prefetchnta 0x1c0(%rsi) L(shl_7_loop_L1): - sub $64, %rdx - movaps 0x09(%rsi), %xmm2 - movaps 0x19(%rsi), %xmm3 - movaps 0x29(%rsi), %xmm4 - movaps 0x39(%rsi), %xmm5 - movdqa %xmm5, %xmm6 - palignr $7, %xmm4, %xmm5 - lea 64(%rsi), %rsi - palignr $7, %xmm3, %xmm4 - palignr $7, %xmm2, %xmm3 - lea 64(%rdi), %rdi - palignr $7, %xmm1, %xmm2 - movdqa %xmm6, %xmm1 - movdqa %xmm2, -0x40(%rdi) - movaps %xmm3, -0x30(%rdi) - jb L(shl_7_end) - movaps %xmm4, -0x20(%rdi) - movaps %xmm5, -0x10(%rdi) - _CET_NOTRACK jmp *%r9 - ud2 + sub $64, %rdx + movaps 0x09(%rsi), %xmm2 + movaps 0x19(%rsi), %xmm3 + movaps 0x29(%rsi), %xmm4 + movaps 0x39(%rsi), %xmm5 + movdqa %xmm5, %xmm6 + palignr $7, %xmm4, %xmm5 + lea 64(%rsi), %rsi + palignr $7, %xmm3, %xmm4 + palignr $7, %xmm2, %xmm3 + lea 64(%rdi), %rdi + palignr $7, %xmm1, %xmm2 + movdqa %xmm6, %xmm1 + movdqa %xmm2, -0x40(%rdi) + movaps %xmm3, -0x30(%rdi) + jb L(shl_7_end) + movaps %xmm4, -0x20(%rdi) + movaps %xmm5, -0x10(%rdi) + _CET_NOTRACK jmp *%r9 + ud2 L(shl_7_end): - movaps %xmm4, -0x20(%rdi) - lea 64(%rdx), %rdx - movaps %xmm5, -0x10(%rdi) - add %rdx, %rdi - movdqu %xmm0, (%r8) - add %rdx, %rsi - BRANCH_TO_JMPTBL_ENTRY(L(table_less_80bytes), %rdx, 4) + movaps %xmm4, -0x20(%rdi) + lea 64(%rdx), %rdx + movaps %xmm5, -0x10(%rdi) + add %rdx, %rdi + movdqu %xmm0, (%r8) + add %rdx, %rsi + BRANCH_TO_JMPTBL_ENTRY(L(table_less_80bytes), %rdx, 4) - .p2align 4 + .p2align 4 L(shl_7_bwd): - lea (L(shl_7_bwd_loop_L1)-L(shl_7_bwd))(%r9), %r9 - cmp %rcx, %rdx - movaps -0x07(%rsi), %xmm1 - jb L(L7_bwd) - lea (L(shl_7_bwd_loop_L2)-L(shl_7_bwd_loop_L1))(%r9), %r9 + lea (L(shl_7_bwd_loop_L1)-L(shl_7_bwd))(%r9), %r9 + cmp %rcx, %rdx + movaps -0x07(%rsi), %xmm1 + jb L(L7_bwd) + lea (L(shl_7_bwd_loop_L2)-L(shl_7_bwd_loop_L1))(%r9), %r9 L(L7_bwd): - lea -64(%rdx), %rdx - _CET_NOTRACK jmp *%r9 - ud2 + lea -64(%rdx), %rdx + _CET_NOTRACK jmp *%r9 + ud2 L(shl_7_bwd_loop_L2): - prefetchnta -0x1c0(%rsi) + prefetchnta -0x1c0(%rsi) L(shl_7_bwd_loop_L1): - movaps -0x17(%rsi), %xmm2 - sub $0x40, %rdx - movaps -0x27(%rsi), %xmm3 - movaps -0x37(%rsi), %xmm4 - movaps -0x47(%rsi), %xmm5 - lea -0x40(%rsi), %rsi - palignr $7, %xmm2, %xmm1 - palignr $7, %xmm3, %xmm2 - palignr $7, %xmm4, %xmm3 - palignr $7, %xmm5, %xmm4 + movaps -0x17(%rsi), %xmm2 + sub $0x40, %rdx + movaps -0x27(%rsi), %xmm3 + movaps -0x37(%rsi), %xmm4 + movaps -0x47(%rsi), %xmm5 + lea -0x40(%rsi), %rsi + palignr $7, %xmm2, %xmm1 + palignr $7, %xmm3, %xmm2 + palignr $7, %xmm4, %xmm3 + palignr $7, %xmm5, %xmm4 - movaps %xmm1, -0x10(%rdi) - movaps %xmm5, %xmm1 + movaps %xmm1, -0x10(%rdi) + movaps %xmm5, %xmm1 - movaps %xmm2, -0x20(%rdi) - lea -0x40(%rdi), %rdi + movaps %xmm2, -0x20(%rdi) + lea -0x40(%rdi), %rdi - movaps %xmm3, 0x10(%rdi) - jb L(shl_7_bwd_end) - movaps %xmm4, (%rdi) - _CET_NOTRACK jmp *%r9 - ud2 + movaps %xmm3, 0x10(%rdi) + jb L(shl_7_bwd_end) + movaps %xmm4, (%rdi) + _CET_NOTRACK jmp *%r9 + ud2 L(shl_7_bwd_end): - movaps %xmm4, (%rdi) - lea 64(%rdx), %rdx - movdqu %xmm0, (%r8) - BRANCH_TO_JMPTBL_ENTRY(L(table_less_80bytes), %rdx, 4) + movaps %xmm4, (%rdi) + lea 64(%rdx), %rdx + movdqu %xmm0, (%r8) + BRANCH_TO_JMPTBL_ENTRY(L(table_less_80bytes), %rdx, 4) - .p2align 4 + .p2align 4 L(shl_8): - lea (L(shl_8_loop_L1)-L(shl_8))(%r9), %r9 - cmp %rcx, %rdx - movaps -0x08(%rsi), %xmm1 - jb L(L8_fwd) - lea (L(shl_8_loop_L2)-L(shl_8_loop_L1))(%r9), %r9 + lea (L(shl_8_loop_L1)-L(shl_8))(%r9), %r9 + cmp %rcx, %rdx + movaps -0x08(%rsi), %xmm1 + jb L(L8_fwd) + lea (L(shl_8_loop_L2)-L(shl_8_loop_L1))(%r9), %r9 L(L8_fwd): - lea -64(%rdx), %rdx - _CET_NOTRACK jmp *%r9 + lea -64(%rdx), %rdx + _CET_NOTRACK jmp *%r9 L(shl_8_loop_L2): - prefetchnta 0x1c0(%rsi) + prefetchnta 0x1c0(%rsi) L(shl_8_loop_L1): - sub $64, %rdx - movaps 0x08(%rsi), %xmm2 - movaps 0x18(%rsi), %xmm3 - movaps 0x28(%rsi), %xmm4 - movaps 0x38(%rsi), %xmm5 - movdqa %xmm5, %xmm6 - palignr $8, %xmm4, %xmm5 - lea 64(%rsi), %rsi - palignr $8, %xmm3, %xmm4 - palignr $8, %xmm2, %xmm3 - lea 64(%rdi), %rdi - palignr $8, %xmm1, %xmm2 - movdqa %xmm6, %xmm1 - movdqa %xmm2, -0x40(%rdi) - movaps %xmm3, -0x30(%rdi) - jb L(shl_8_end) - movaps %xmm4, -0x20(%rdi) - movaps %xmm5, -0x10(%rdi) - _CET_NOTRACK jmp *%r9 - ud2 - .p2align 4 + sub $64, %rdx + movaps 0x08(%rsi), %xmm2 + movaps 0x18(%rsi), %xmm3 + movaps 0x28(%rsi), %xmm4 + movaps 0x38(%rsi), %xmm5 + movdqa %xmm5, %xmm6 + palignr $8, %xmm4, %xmm5 + lea 64(%rsi), %rsi + palignr $8, %xmm3, %xmm4 + palignr $8, %xmm2, %xmm3 + lea 64(%rdi), %rdi + palignr $8, %xmm1, %xmm2 + movdqa %xmm6, %xmm1 + movdqa %xmm2, -0x40(%rdi) + movaps %xmm3, -0x30(%rdi) + jb L(shl_8_end) + movaps %xmm4, -0x20(%rdi) + movaps %xmm5, -0x10(%rdi) + _CET_NOTRACK jmp *%r9 + ud2 + .p2align 4 L(shl_8_end): - lea 64(%rdx), %rdx - movaps %xmm4, -0x20(%rdi) - add %rdx, %rsi - movaps %xmm5, -0x10(%rdi) - add %rdx, %rdi - movdqu %xmm0, (%r8) - BRANCH_TO_JMPTBL_ENTRY(L(table_less_80bytes), %rdx, 4) + lea 64(%rdx), %rdx + movaps %xmm4, -0x20(%rdi) + add %rdx, %rsi + movaps %xmm5, -0x10(%rdi) + add %rdx, %rdi + movdqu %xmm0, (%r8) + BRANCH_TO_JMPTBL_ENTRY(L(table_less_80bytes), %rdx, 4) - .p2align 4 + .p2align 4 L(shl_8_bwd): - lea (L(shl_8_bwd_loop_L1)-L(shl_8_bwd))(%r9), %r9 - cmp %rcx, %rdx - movaps -0x08(%rsi), %xmm1 - jb L(L8_bwd) - lea (L(shl_8_bwd_loop_L2)-L(shl_8_bwd_loop_L1))(%r9), %r9 + lea (L(shl_8_bwd_loop_L1)-L(shl_8_bwd))(%r9), %r9 + cmp %rcx, %rdx + movaps -0x08(%rsi), %xmm1 + jb L(L8_bwd) + lea (L(shl_8_bwd_loop_L2)-L(shl_8_bwd_loop_L1))(%r9), %r9 L(L8_bwd): - lea -64(%rdx), %rdx - _CET_NOTRACK jmp *%r9 - ud2 + lea -64(%rdx), %rdx + _CET_NOTRACK jmp *%r9 + ud2 L(shl_8_bwd_loop_L2): - prefetchnta -0x1c0(%rsi) + prefetchnta -0x1c0(%rsi) L(shl_8_bwd_loop_L1): - movaps -0x18(%rsi), %xmm2 - sub $0x40, %rdx - movaps -0x28(%rsi), %xmm3 - movaps -0x38(%rsi), %xmm4 - movaps -0x48(%rsi), %xmm5 - lea -0x40(%rsi), %rsi - palignr $8, %xmm2, %xmm1 - palignr $8, %xmm3, %xmm2 - palignr $8, %xmm4, %xmm3 - palignr $8, %xmm5, %xmm4 + movaps -0x18(%rsi), %xmm2 + sub $0x40, %rdx + movaps -0x28(%rsi), %xmm3 + movaps -0x38(%rsi), %xmm4 + movaps -0x48(%rsi), %xmm5 + lea -0x40(%rsi), %rsi + palignr $8, %xmm2, %xmm1 + palignr $8, %xmm3, %xmm2 + palignr $8, %xmm4, %xmm3 + palignr $8, %xmm5, %xmm4 - movaps %xmm1, -0x10(%rdi) - movaps %xmm5, %xmm1 + movaps %xmm1, -0x10(%rdi) + movaps %xmm5, %xmm1 - movaps %xmm2, -0x20(%rdi) - lea -0x40(%rdi), %rdi + movaps %xmm2, -0x20(%rdi) + lea -0x40(%rdi), %rdi - movaps %xmm3, 0x10(%rdi) - jb L(shl_8_bwd_end) - movaps %xmm4, (%rdi) - _CET_NOTRACK jmp *%r9 - ud2 + movaps %xmm3, 0x10(%rdi) + jb L(shl_8_bwd_end) + movaps %xmm4, (%rdi) + _CET_NOTRACK jmp *%r9 + ud2 L(shl_8_bwd_end): - movaps %xmm4, (%rdi) - lea 64(%rdx), %rdx - movdqu %xmm0, (%r8) - BRANCH_TO_JMPTBL_ENTRY(L(table_less_80bytes), %rdx, 4) + movaps %xmm4, (%rdi) + lea 64(%rdx), %rdx + movdqu %xmm0, (%r8) + BRANCH_TO_JMPTBL_ENTRY(L(table_less_80bytes), %rdx, 4) - .p2align 4 + .p2align 4 L(shl_9): - lea (L(shl_9_loop_L1)-L(shl_9))(%r9), %r9 - cmp %rcx, %rdx - movaps -0x09(%rsi), %xmm1 - jb L(L9_fwd) - lea (L(shl_9_loop_L2)-L(shl_9_loop_L1))(%r9), %r9 + lea (L(shl_9_loop_L1)-L(shl_9))(%r9), %r9 + cmp %rcx, %rdx + movaps -0x09(%rsi), %xmm1 + jb L(L9_fwd) + lea (L(shl_9_loop_L2)-L(shl_9_loop_L1))(%r9), %r9 L(L9_fwd): - lea -64(%rdx), %rdx - _CET_NOTRACK jmp *%r9 - ud2 + lea -64(%rdx), %rdx + _CET_NOTRACK jmp *%r9 + ud2 L(shl_9_loop_L2): - prefetchnta 0x1c0(%rsi) + prefetchnta 0x1c0(%rsi) L(shl_9_loop_L1): - sub $64, %rdx - movaps 0x07(%rsi), %xmm2 - movaps 0x17(%rsi), %xmm3 - movaps 0x27(%rsi), %xmm4 - movaps 0x37(%rsi), %xmm5 - movdqa %xmm5, %xmm6 - palignr $9, %xmm4, %xmm5 - lea 64(%rsi), %rsi - palignr $9, %xmm3, %xmm4 - palignr $9, %xmm2, %xmm3 - lea 64(%rdi), %rdi - palignr $9, %xmm1, %xmm2 - movdqa %xmm6, %xmm1 - movdqa %xmm2, -0x40(%rdi) - movaps %xmm3, -0x30(%rdi) - jb L(shl_9_end) - movaps %xmm4, -0x20(%rdi) - movaps %xmm5, -0x10(%rdi) - _CET_NOTRACK jmp *%r9 - ud2 + sub $64, %rdx + movaps 0x07(%rsi), %xmm2 + movaps 0x17(%rsi), %xmm3 + movaps 0x27(%rsi), %xmm4 + movaps 0x37(%rsi), %xmm5 + movdqa %xmm5, %xmm6 + palignr $9, %xmm4, %xmm5 + lea 64(%rsi), %rsi + palignr $9, %xmm3, %xmm4 + palignr $9, %xmm2, %xmm3 + lea 64(%rdi), %rdi + palignr $9, %xmm1, %xmm2 + movdqa %xmm6, %xmm1 + movdqa %xmm2, -0x40(%rdi) + movaps %xmm3, -0x30(%rdi) + jb L(shl_9_end) + movaps %xmm4, -0x20(%rdi) + movaps %xmm5, -0x10(%rdi) + _CET_NOTRACK jmp *%r9 + ud2 L(shl_9_end): - movaps %xmm4, -0x20(%rdi) - lea 64(%rdx), %rdx - movaps %xmm5, -0x10(%rdi) - add %rdx, %rdi - movdqu %xmm0, (%r8) - add %rdx, %rsi - BRANCH_TO_JMPTBL_ENTRY(L(table_less_80bytes), %rdx, 4) + movaps %xmm4, -0x20(%rdi) + lea 64(%rdx), %rdx + movaps %xmm5, -0x10(%rdi) + add %rdx, %rdi + movdqu %xmm0, (%r8) + add %rdx, %rsi + BRANCH_TO_JMPTBL_ENTRY(L(table_less_80bytes), %rdx, 4) - .p2align 4 + .p2align 4 L(shl_9_bwd): - lea (L(shl_9_bwd_loop_L1)-L(shl_9_bwd))(%r9), %r9 - cmp %rcx, %rdx - movaps -0x09(%rsi), %xmm1 - jb L(L9_bwd) - lea (L(shl_9_bwd_loop_L2)-L(shl_9_bwd_loop_L1))(%r9), %r9 + lea (L(shl_9_bwd_loop_L1)-L(shl_9_bwd))(%r9), %r9 + cmp %rcx, %rdx + movaps -0x09(%rsi), %xmm1 + jb L(L9_bwd) + lea (L(shl_9_bwd_loop_L2)-L(shl_9_bwd_loop_L1))(%r9), %r9 L(L9_bwd): - lea -64(%rdx), %rdx - _CET_NOTRACK jmp *%r9 - ud2 + lea -64(%rdx), %rdx + _CET_NOTRACK jmp *%r9 + ud2 L(shl_9_bwd_loop_L2): - prefetchnta -0x1c0(%rsi) + prefetchnta -0x1c0(%rsi) L(shl_9_bwd_loop_L1): - movaps -0x19(%rsi), %xmm2 - sub $0x40, %rdx - movaps -0x29(%rsi), %xmm3 - movaps -0x39(%rsi), %xmm4 - movaps -0x49(%rsi), %xmm5 - lea -0x40(%rsi), %rsi - palignr $9, %xmm2, %xmm1 - palignr $9, %xmm3, %xmm2 - palignr $9, %xmm4, %xmm3 - palignr $9, %xmm5, %xmm4 + movaps -0x19(%rsi), %xmm2 + sub $0x40, %rdx + movaps -0x29(%rsi), %xmm3 + movaps -0x39(%rsi), %xmm4 + movaps -0x49(%rsi), %xmm5 + lea -0x40(%rsi), %rsi + palignr $9, %xmm2, %xmm1 + palignr $9, %xmm3, %xmm2 + palignr $9, %xmm4, %xmm3 + palignr $9, %xmm5, %xmm4 - movaps %xmm1, -0x10(%rdi) - movaps %xmm5, %xmm1 + movaps %xmm1, -0x10(%rdi) + movaps %xmm5, %xmm1 - movaps %xmm2, -0x20(%rdi) - lea -0x40(%rdi), %rdi + movaps %xmm2, -0x20(%rdi) + lea -0x40(%rdi), %rdi - movaps %xmm3, 0x10(%rdi) - jb L(shl_9_bwd_end) - movaps %xmm4, (%rdi) - _CET_NOTRACK jmp *%r9 - ud2 + movaps %xmm3, 0x10(%rdi) + jb L(shl_9_bwd_end) + movaps %xmm4, (%rdi) + _CET_NOTRACK jmp *%r9 + ud2 L(shl_9_bwd_end): - movaps %xmm4, (%rdi) - lea 64(%rdx), %rdx - movdqu %xmm0, (%r8) - BRANCH_TO_JMPTBL_ENTRY(L(table_less_80bytes), %rdx, 4) + movaps %xmm4, (%rdi) + lea 64(%rdx), %rdx + movdqu %xmm0, (%r8) + BRANCH_TO_JMPTBL_ENTRY(L(table_less_80bytes), %rdx, 4) - .p2align 4 + .p2align 4 L(shl_10): - lea (L(shl_10_loop_L1)-L(shl_10))(%r9), %r9 - cmp %rcx, %rdx - movaps -0x0a(%rsi), %xmm1 - jb L(L10_fwd) - lea (L(shl_10_loop_L2)-L(shl_10_loop_L1))(%r9), %r9 + lea (L(shl_10_loop_L1)-L(shl_10))(%r9), %r9 + cmp %rcx, %rdx + movaps -0x0a(%rsi), %xmm1 + jb L(L10_fwd) + lea (L(shl_10_loop_L2)-L(shl_10_loop_L1))(%r9), %r9 L(L10_fwd): - lea -64(%rdx), %rdx - _CET_NOTRACK jmp *%r9 - ud2 + lea -64(%rdx), %rdx + _CET_NOTRACK jmp *%r9 + ud2 L(shl_10_loop_L2): - prefetchnta 0x1c0(%rsi) + prefetchnta 0x1c0(%rsi) L(shl_10_loop_L1): - sub $64, %rdx - movaps 0x06(%rsi), %xmm2 - movaps 0x16(%rsi), %xmm3 - movaps 0x26(%rsi), %xmm4 - movaps 0x36(%rsi), %xmm5 - movdqa %xmm5, %xmm6 - palignr $10, %xmm4, %xmm5 - lea 64(%rsi), %rsi - palignr $10, %xmm3, %xmm4 - palignr $10, %xmm2, %xmm3 - lea 64(%rdi), %rdi - palignr $10, %xmm1, %xmm2 - movdqa %xmm6, %xmm1 - movdqa %xmm2, -0x40(%rdi) - movaps %xmm3, -0x30(%rdi) - jb L(shl_10_end) - movaps %xmm4, -0x20(%rdi) - movaps %xmm5, -0x10(%rdi) - _CET_NOTRACK jmp *%r9 - ud2 + sub $64, %rdx + movaps 0x06(%rsi), %xmm2 + movaps 0x16(%rsi), %xmm3 + movaps 0x26(%rsi), %xmm4 + movaps 0x36(%rsi), %xmm5 + movdqa %xmm5, %xmm6 + palignr $10, %xmm4, %xmm5 + lea 64(%rsi), %rsi + palignr $10, %xmm3, %xmm4 + palignr $10, %xmm2, %xmm3 + lea 64(%rdi), %rdi + palignr $10, %xmm1, %xmm2 + movdqa %xmm6, %xmm1 + movdqa %xmm2, -0x40(%rdi) + movaps %xmm3, -0x30(%rdi) + jb L(shl_10_end) + movaps %xmm4, -0x20(%rdi) + movaps %xmm5, -0x10(%rdi) + _CET_NOTRACK jmp *%r9 + ud2 L(shl_10_end): - movaps %xmm4, -0x20(%rdi) - lea 64(%rdx), %rdx - movaps %xmm5, -0x10(%rdi) - add %rdx, %rdi - movdqu %xmm0, (%r8) - add %rdx, %rsi - BRANCH_TO_JMPTBL_ENTRY(L(table_less_80bytes), %rdx, 4) + movaps %xmm4, -0x20(%rdi) + lea 64(%rdx), %rdx + movaps %xmm5, -0x10(%rdi) + add %rdx, %rdi + movdqu %xmm0, (%r8) + add %rdx, %rsi + BRANCH_TO_JMPTBL_ENTRY(L(table_less_80bytes), %rdx, 4) - .p2align 4 + .p2align 4 L(shl_10_bwd): - lea (L(shl_10_bwd_loop_L1)-L(shl_10_bwd))(%r9), %r9 - cmp %rcx, %rdx - movaps -0x0a(%rsi), %xmm1 - jb L(L10_bwd) - lea (L(shl_10_bwd_loop_L2)-L(shl_10_bwd_loop_L1))(%r9), %r9 + lea (L(shl_10_bwd_loop_L1)-L(shl_10_bwd))(%r9), %r9 + cmp %rcx, %rdx + movaps -0x0a(%rsi), %xmm1 + jb L(L10_bwd) + lea (L(shl_10_bwd_loop_L2)-L(shl_10_bwd_loop_L1))(%r9), %r9 L(L10_bwd): - lea -64(%rdx), %rdx - _CET_NOTRACK jmp *%r9 - ud2 + lea -64(%rdx), %rdx + _CET_NOTRACK jmp *%r9 + ud2 L(shl_10_bwd_loop_L2): - prefetchnta -0x1c0(%rsi) + prefetchnta -0x1c0(%rsi) L(shl_10_bwd_loop_L1): - movaps -0x1a(%rsi), %xmm2 - sub $0x40, %rdx - movaps -0x2a(%rsi), %xmm3 - movaps -0x3a(%rsi), %xmm4 - movaps -0x4a(%rsi), %xmm5 - lea -0x40(%rsi), %rsi - palignr $10, %xmm2, %xmm1 - palignr $10, %xmm3, %xmm2 - palignr $10, %xmm4, %xmm3 - palignr $10, %xmm5, %xmm4 + movaps -0x1a(%rsi), %xmm2 + sub $0x40, %rdx + movaps -0x2a(%rsi), %xmm3 + movaps -0x3a(%rsi), %xmm4 + movaps -0x4a(%rsi), %xmm5 + lea -0x40(%rsi), %rsi + palignr $10, %xmm2, %xmm1 + palignr $10, %xmm3, %xmm2 + palignr $10, %xmm4, %xmm3 + palignr $10, %xmm5, %xmm4 - movaps %xmm1, -0x10(%rdi) - movaps %xmm5, %xmm1 + movaps %xmm1, -0x10(%rdi) + movaps %xmm5, %xmm1 - movaps %xmm2, -0x20(%rdi) - lea -0x40(%rdi), %rdi + movaps %xmm2, -0x20(%rdi) + lea -0x40(%rdi), %rdi - movaps %xmm3, 0x10(%rdi) - jb L(shl_10_bwd_end) - movaps %xmm4, (%rdi) - _CET_NOTRACK jmp *%r9 - ud2 + movaps %xmm3, 0x10(%rdi) + jb L(shl_10_bwd_end) + movaps %xmm4, (%rdi) + _CET_NOTRACK jmp *%r9 + ud2 L(shl_10_bwd_end): - movaps %xmm4, (%rdi) - lea 64(%rdx), %rdx - movdqu %xmm0, (%r8) - BRANCH_TO_JMPTBL_ENTRY(L(table_less_80bytes), %rdx, 4) + movaps %xmm4, (%rdi) + lea 64(%rdx), %rdx + movdqu %xmm0, (%r8) + BRANCH_TO_JMPTBL_ENTRY(L(table_less_80bytes), %rdx, 4) - .p2align 4 + .p2align 4 L(shl_11): - lea (L(shl_11_loop_L1)-L(shl_11))(%r9), %r9 - cmp %rcx, %rdx - movaps -0x0b(%rsi), %xmm1 - jb L(L11_fwd) - lea (L(shl_11_loop_L2)-L(shl_11_loop_L1))(%r9), %r9 + lea (L(shl_11_loop_L1)-L(shl_11))(%r9), %r9 + cmp %rcx, %rdx + movaps -0x0b(%rsi), %xmm1 + jb L(L11_fwd) + lea (L(shl_11_loop_L2)-L(shl_11_loop_L1))(%r9), %r9 L(L11_fwd): - lea -64(%rdx), %rdx - _CET_NOTRACK jmp *%r9 - ud2 + lea -64(%rdx), %rdx + _CET_NOTRACK jmp *%r9 + ud2 L(shl_11_loop_L2): - prefetchnta 0x1c0(%rsi) + prefetchnta 0x1c0(%rsi) L(shl_11_loop_L1): - sub $64, %rdx - movaps 0x05(%rsi), %xmm2 - movaps 0x15(%rsi), %xmm3 - movaps 0x25(%rsi), %xmm4 - movaps 0x35(%rsi), %xmm5 - movdqa %xmm5, %xmm6 - palignr $11, %xmm4, %xmm5 - lea 64(%rsi), %rsi - palignr $11, %xmm3, %xmm4 - palignr $11, %xmm2, %xmm3 - lea 64(%rdi), %rdi - palignr $11, %xmm1, %xmm2 - movdqa %xmm6, %xmm1 - movdqa %xmm2, -0x40(%rdi) - movaps %xmm3, -0x30(%rdi) - jb L(shl_11_end) - movaps %xmm4, -0x20(%rdi) - movaps %xmm5, -0x10(%rdi) - _CET_NOTRACK jmp *%r9 - ud2 + sub $64, %rdx + movaps 0x05(%rsi), %xmm2 + movaps 0x15(%rsi), %xmm3 + movaps 0x25(%rsi), %xmm4 + movaps 0x35(%rsi), %xmm5 + movdqa %xmm5, %xmm6 + palignr $11, %xmm4, %xmm5 + lea 64(%rsi), %rsi + palignr $11, %xmm3, %xmm4 + palignr $11, %xmm2, %xmm3 + lea 64(%rdi), %rdi + palignr $11, %xmm1, %xmm2 + movdqa %xmm6, %xmm1 + movdqa %xmm2, -0x40(%rdi) + movaps %xmm3, -0x30(%rdi) + jb L(shl_11_end) + movaps %xmm4, -0x20(%rdi) + movaps %xmm5, -0x10(%rdi) + _CET_NOTRACK jmp *%r9 + ud2 L(shl_11_end): - movaps %xmm4, -0x20(%rdi) - lea 64(%rdx), %rdx - movaps %xmm5, -0x10(%rdi) - add %rdx, %rdi - movdqu %xmm0, (%r8) - add %rdx, %rsi - BRANCH_TO_JMPTBL_ENTRY(L(table_less_80bytes), %rdx, 4) + movaps %xmm4, -0x20(%rdi) + lea 64(%rdx), %rdx + movaps %xmm5, -0x10(%rdi) + add %rdx, %rdi + movdqu %xmm0, (%r8) + add %rdx, %rsi + BRANCH_TO_JMPTBL_ENTRY(L(table_less_80bytes), %rdx, 4) - .p2align 4 + .p2align 4 L(shl_11_bwd): - lea (L(shl_11_bwd_loop_L1)-L(shl_11_bwd))(%r9), %r9 - cmp %rcx, %rdx - movaps -0x0b(%rsi), %xmm1 - jb L(L11_bwd) - lea (L(shl_11_bwd_loop_L2)-L(shl_11_bwd_loop_L1))(%r9), %r9 + lea (L(shl_11_bwd_loop_L1)-L(shl_11_bwd))(%r9), %r9 + cmp %rcx, %rdx + movaps -0x0b(%rsi), %xmm1 + jb L(L11_bwd) + lea (L(shl_11_bwd_loop_L2)-L(shl_11_bwd_loop_L1))(%r9), %r9 L(L11_bwd): - lea -64(%rdx), %rdx - _CET_NOTRACK jmp *%r9 - ud2 + lea -64(%rdx), %rdx + _CET_NOTRACK jmp *%r9 + ud2 L(shl_11_bwd_loop_L2): - prefetchnta -0x1c0(%rsi) + prefetchnta -0x1c0(%rsi) L(shl_11_bwd_loop_L1): - movaps -0x1b(%rsi), %xmm2 - sub $0x40, %rdx - movaps -0x2b(%rsi), %xmm3 - movaps -0x3b(%rsi), %xmm4 - movaps -0x4b(%rsi), %xmm5 - lea -0x40(%rsi), %rsi - palignr $11, %xmm2, %xmm1 - palignr $11, %xmm3, %xmm2 - palignr $11, %xmm4, %xmm3 - palignr $11, %xmm5, %xmm4 + movaps -0x1b(%rsi), %xmm2 + sub $0x40, %rdx + movaps -0x2b(%rsi), %xmm3 + movaps -0x3b(%rsi), %xmm4 + movaps -0x4b(%rsi), %xmm5 + lea -0x40(%rsi), %rsi + palignr $11, %xmm2, %xmm1 + palignr $11, %xmm3, %xmm2 + palignr $11, %xmm4, %xmm3 + palignr $11, %xmm5, %xmm4 - movaps %xmm1, -0x10(%rdi) - movaps %xmm5, %xmm1 + movaps %xmm1, -0x10(%rdi) + movaps %xmm5, %xmm1 - movaps %xmm2, -0x20(%rdi) - lea -0x40(%rdi), %rdi + movaps %xmm2, -0x20(%rdi) + lea -0x40(%rdi), %rdi - movaps %xmm3, 0x10(%rdi) - jb L(shl_11_bwd_end) - movaps %xmm4, (%rdi) - _CET_NOTRACK jmp *%r9 - ud2 + movaps %xmm3, 0x10(%rdi) + jb L(shl_11_bwd_end) + movaps %xmm4, (%rdi) + _CET_NOTRACK jmp *%r9 + ud2 L(shl_11_bwd_end): - movaps %xmm4, (%rdi) - lea 64(%rdx), %rdx - movdqu %xmm0, (%r8) - BRANCH_TO_JMPTBL_ENTRY(L(table_less_80bytes), %rdx, 4) + movaps %xmm4, (%rdi) + lea 64(%rdx), %rdx + movdqu %xmm0, (%r8) + BRANCH_TO_JMPTBL_ENTRY(L(table_less_80bytes), %rdx, 4) - .p2align 4 + .p2align 4 L(shl_12): - lea (L(shl_12_loop_L1)-L(shl_12))(%r9), %r9 - cmp %rcx, %rdx - movaps -0x0c(%rsi), %xmm1 - jb L(L12_fwd) - lea (L(shl_12_loop_L2)-L(shl_12_loop_L1))(%r9), %r9 + lea (L(shl_12_loop_L1)-L(shl_12))(%r9), %r9 + cmp %rcx, %rdx + movaps -0x0c(%rsi), %xmm1 + jb L(L12_fwd) + lea (L(shl_12_loop_L2)-L(shl_12_loop_L1))(%r9), %r9 L(L12_fwd): - lea -64(%rdx), %rdx - _CET_NOTRACK jmp *%r9 - ud2 + lea -64(%rdx), %rdx + _CET_NOTRACK jmp *%r9 + ud2 L(shl_12_loop_L2): - prefetchnta 0x1c0(%rsi) + prefetchnta 0x1c0(%rsi) L(shl_12_loop_L1): - sub $64, %rdx - movaps 0x04(%rsi), %xmm2 - movaps 0x14(%rsi), %xmm3 - movaps 0x24(%rsi), %xmm4 - movaps 0x34(%rsi), %xmm5 - movdqa %xmm5, %xmm6 - palignr $12, %xmm4, %xmm5 - lea 64(%rsi), %rsi - palignr $12, %xmm3, %xmm4 - palignr $12, %xmm2, %xmm3 - lea 64(%rdi), %rdi - palignr $12, %xmm1, %xmm2 - movdqa %xmm6, %xmm1 - movdqa %xmm2, -0x40(%rdi) - movaps %xmm3, -0x30(%rdi) - jb L(shl_12_end) - movaps %xmm4, -0x20(%rdi) - movaps %xmm5, -0x10(%rdi) - _CET_NOTRACK jmp *%r9 - ud2 + sub $64, %rdx + movaps 0x04(%rsi), %xmm2 + movaps 0x14(%rsi), %xmm3 + movaps 0x24(%rsi), %xmm4 + movaps 0x34(%rsi), %xmm5 + movdqa %xmm5, %xmm6 + palignr $12, %xmm4, %xmm5 + lea 64(%rsi), %rsi + palignr $12, %xmm3, %xmm4 + palignr $12, %xmm2, %xmm3 + lea 64(%rdi), %rdi + palignr $12, %xmm1, %xmm2 + movdqa %xmm6, %xmm1 + movdqa %xmm2, -0x40(%rdi) + movaps %xmm3, -0x30(%rdi) + jb L(shl_12_end) + movaps %xmm4, -0x20(%rdi) + movaps %xmm5, -0x10(%rdi) + _CET_NOTRACK jmp *%r9 + ud2 L(shl_12_end): - movaps %xmm4, -0x20(%rdi) - lea 64(%rdx), %rdx - movaps %xmm5, -0x10(%rdi) - add %rdx, %rdi - movdqu %xmm0, (%r8) - add %rdx, %rsi - BRANCH_TO_JMPTBL_ENTRY(L(table_less_80bytes), %rdx, 4) + movaps %xmm4, -0x20(%rdi) + lea 64(%rdx), %rdx + movaps %xmm5, -0x10(%rdi) + add %rdx, %rdi + movdqu %xmm0, (%r8) + add %rdx, %rsi + BRANCH_TO_JMPTBL_ENTRY(L(table_less_80bytes), %rdx, 4) - .p2align 4 + .p2align 4 L(shl_12_bwd): - lea (L(shl_12_bwd_loop_L1)-L(shl_12_bwd))(%r9), %r9 - cmp %rcx, %rdx - movaps -0x0c(%rsi), %xmm1 - jb L(L12_bwd) - lea (L(shl_12_bwd_loop_L2)-L(shl_12_bwd_loop_L1))(%r9), %r9 + lea (L(shl_12_bwd_loop_L1)-L(shl_12_bwd))(%r9), %r9 + cmp %rcx, %rdx + movaps -0x0c(%rsi), %xmm1 + jb L(L12_bwd) + lea (L(shl_12_bwd_loop_L2)-L(shl_12_bwd_loop_L1))(%r9), %r9 L(L12_bwd): - lea -64(%rdx), %rdx - _CET_NOTRACK jmp *%r9 - ud2 + lea -64(%rdx), %rdx + _CET_NOTRACK jmp *%r9 + ud2 L(shl_12_bwd_loop_L2): - prefetchnta -0x1c0(%rsi) + prefetchnta -0x1c0(%rsi) L(shl_12_bwd_loop_L1): - movaps -0x1c(%rsi), %xmm2 - sub $0x40, %rdx - movaps -0x2c(%rsi), %xmm3 - movaps -0x3c(%rsi), %xmm4 - movaps -0x4c(%rsi), %xmm5 - lea -0x40(%rsi), %rsi - palignr $12, %xmm2, %xmm1 - palignr $12, %xmm3, %xmm2 - palignr $12, %xmm4, %xmm3 - palignr $12, %xmm5, %xmm4 + movaps -0x1c(%rsi), %xmm2 + sub $0x40, %rdx + movaps -0x2c(%rsi), %xmm3 + movaps -0x3c(%rsi), %xmm4 + movaps -0x4c(%rsi), %xmm5 + lea -0x40(%rsi), %rsi + palignr $12, %xmm2, %xmm1 + palignr $12, %xmm3, %xmm2 + palignr $12, %xmm4, %xmm3 + palignr $12, %xmm5, %xmm4 - movaps %xmm1, -0x10(%rdi) - movaps %xmm5, %xmm1 + movaps %xmm1, -0x10(%rdi) + movaps %xmm5, %xmm1 - movaps %xmm2, -0x20(%rdi) - lea -0x40(%rdi), %rdi + movaps %xmm2, -0x20(%rdi) + lea -0x40(%rdi), %rdi - movaps %xmm3, 0x10(%rdi) - jb L(shl_12_bwd_end) - movaps %xmm4, (%rdi) - _CET_NOTRACK jmp *%r9 - ud2 + movaps %xmm3, 0x10(%rdi) + jb L(shl_12_bwd_end) + movaps %xmm4, (%rdi) + _CET_NOTRACK jmp *%r9 + ud2 L(shl_12_bwd_end): - movaps %xmm4, (%rdi) - lea 64(%rdx), %rdx - movdqu %xmm0, (%r8) - BRANCH_TO_JMPTBL_ENTRY(L(table_less_80bytes), %rdx, 4) + movaps %xmm4, (%rdi) + lea 64(%rdx), %rdx + movdqu %xmm0, (%r8) + BRANCH_TO_JMPTBL_ENTRY(L(table_less_80bytes), %rdx, 4) - .p2align 4 + .p2align 4 L(shl_13): - lea (L(shl_13_loop_L1)-L(shl_13))(%r9), %r9 - cmp %rcx, %rdx - movaps -0x0d(%rsi), %xmm1 - jb L(L13_fwd) - lea (L(shl_13_loop_L2)-L(shl_13_loop_L1))(%r9), %r9 + lea (L(shl_13_loop_L1)-L(shl_13))(%r9), %r9 + cmp %rcx, %rdx + movaps -0x0d(%rsi), %xmm1 + jb L(L13_fwd) + lea (L(shl_13_loop_L2)-L(shl_13_loop_L1))(%r9), %r9 L(L13_fwd): - lea -64(%rdx), %rdx - _CET_NOTRACK jmp *%r9 - ud2 + lea -64(%rdx), %rdx + _CET_NOTRACK jmp *%r9 + ud2 L(shl_13_loop_L2): - prefetchnta 0x1c0(%rsi) + prefetchnta 0x1c0(%rsi) L(shl_13_loop_L1): - sub $64, %rdx - movaps 0x03(%rsi), %xmm2 - movaps 0x13(%rsi), %xmm3 - movaps 0x23(%rsi), %xmm4 - movaps 0x33(%rsi), %xmm5 - movdqa %xmm5, %xmm6 - palignr $13, %xmm4, %xmm5 - lea 64(%rsi), %rsi - palignr $13, %xmm3, %xmm4 - palignr $13, %xmm2, %xmm3 - lea 64(%rdi), %rdi - palignr $13, %xmm1, %xmm2 - movdqa %xmm6, %xmm1 - movdqa %xmm2, -0x40(%rdi) - movaps %xmm3, -0x30(%rdi) - jb L(shl_13_end) - movaps %xmm4, -0x20(%rdi) - movaps %xmm5, -0x10(%rdi) - _CET_NOTRACK jmp *%r9 - ud2 + sub $64, %rdx + movaps 0x03(%rsi), %xmm2 + movaps 0x13(%rsi), %xmm3 + movaps 0x23(%rsi), %xmm4 + movaps 0x33(%rsi), %xmm5 + movdqa %xmm5, %xmm6 + palignr $13, %xmm4, %xmm5 + lea 64(%rsi), %rsi + palignr $13, %xmm3, %xmm4 + palignr $13, %xmm2, %xmm3 + lea 64(%rdi), %rdi + palignr $13, %xmm1, %xmm2 + movdqa %xmm6, %xmm1 + movdqa %xmm2, -0x40(%rdi) + movaps %xmm3, -0x30(%rdi) + jb L(shl_13_end) + movaps %xmm4, -0x20(%rdi) + movaps %xmm5, -0x10(%rdi) + _CET_NOTRACK jmp *%r9 + ud2 L(shl_13_end): - movaps %xmm4, -0x20(%rdi) - lea 64(%rdx), %rdx - movaps %xmm5, -0x10(%rdi) - add %rdx, %rdi - movdqu %xmm0, (%r8) - add %rdx, %rsi - BRANCH_TO_JMPTBL_ENTRY(L(table_less_80bytes), %rdx, 4) + movaps %xmm4, -0x20(%rdi) + lea 64(%rdx), %rdx + movaps %xmm5, -0x10(%rdi) + add %rdx, %rdi + movdqu %xmm0, (%r8) + add %rdx, %rsi + BRANCH_TO_JMPTBL_ENTRY(L(table_less_80bytes), %rdx, 4) - .p2align 4 + .p2align 4 L(shl_13_bwd): - lea (L(shl_13_bwd_loop_L1)-L(shl_13_bwd))(%r9), %r9 - cmp %rcx, %rdx - movaps -0x0d(%rsi), %xmm1 - jb L(L13_bwd) - lea (L(shl_13_bwd_loop_L2)-L(shl_13_bwd_loop_L1))(%r9), %r9 + lea (L(shl_13_bwd_loop_L1)-L(shl_13_bwd))(%r9), %r9 + cmp %rcx, %rdx + movaps -0x0d(%rsi), %xmm1 + jb L(L13_bwd) + lea (L(shl_13_bwd_loop_L2)-L(shl_13_bwd_loop_L1))(%r9), %r9 L(L13_bwd): - lea -64(%rdx), %rdx - _CET_NOTRACK jmp *%r9 - ud2 + lea -64(%rdx), %rdx + _CET_NOTRACK jmp *%r9 + ud2 L(shl_13_bwd_loop_L2): - prefetchnta -0x1c0(%rsi) + prefetchnta -0x1c0(%rsi) L(shl_13_bwd_loop_L1): - movaps -0x1d(%rsi), %xmm2 - sub $0x40, %rdx - movaps -0x2d(%rsi), %xmm3 - movaps -0x3d(%rsi), %xmm4 - movaps -0x4d(%rsi), %xmm5 - lea -0x40(%rsi), %rsi - palignr $13, %xmm2, %xmm1 - palignr $13, %xmm3, %xmm2 - palignr $13, %xmm4, %xmm3 - palignr $13, %xmm5, %xmm4 + movaps -0x1d(%rsi), %xmm2 + sub $0x40, %rdx + movaps -0x2d(%rsi), %xmm3 + movaps -0x3d(%rsi), %xmm4 + movaps -0x4d(%rsi), %xmm5 + lea -0x40(%rsi), %rsi + palignr $13, %xmm2, %xmm1 + palignr $13, %xmm3, %xmm2 + palignr $13, %xmm4, %xmm3 + palignr $13, %xmm5, %xmm4 - movaps %xmm1, -0x10(%rdi) - movaps %xmm5, %xmm1 + movaps %xmm1, -0x10(%rdi) + movaps %xmm5, %xmm1 - movaps %xmm2, -0x20(%rdi) - lea -0x40(%rdi), %rdi + movaps %xmm2, -0x20(%rdi) + lea -0x40(%rdi), %rdi - movaps %xmm3, 0x10(%rdi) - jb L(shl_13_bwd_end) - movaps %xmm4, (%rdi) - _CET_NOTRACK jmp *%r9 - ud2 + movaps %xmm3, 0x10(%rdi) + jb L(shl_13_bwd_end) + movaps %xmm4, (%rdi) + _CET_NOTRACK jmp *%r9 + ud2 L(shl_13_bwd_end): - movaps %xmm4, (%rdi) - lea 64(%rdx), %rdx - movdqu %xmm0, (%r8) - BRANCH_TO_JMPTBL_ENTRY(L(table_less_80bytes), %rdx, 4) + movaps %xmm4, (%rdi) + lea 64(%rdx), %rdx + movdqu %xmm0, (%r8) + BRANCH_TO_JMPTBL_ENTRY(L(table_less_80bytes), %rdx, 4) - .p2align 4 + .p2align 4 L(shl_14): - lea (L(shl_14_loop_L1)-L(shl_14))(%r9), %r9 - cmp %rcx, %rdx - movaps -0x0e(%rsi), %xmm1 - jb L(L14_fwd) - lea (L(shl_14_loop_L2)-L(shl_14_loop_L1))(%r9), %r9 + lea (L(shl_14_loop_L1)-L(shl_14))(%r9), %r9 + cmp %rcx, %rdx + movaps -0x0e(%rsi), %xmm1 + jb L(L14_fwd) + lea (L(shl_14_loop_L2)-L(shl_14_loop_L1))(%r9), %r9 L(L14_fwd): - lea -64(%rdx), %rdx - _CET_NOTRACK jmp *%r9 - ud2 + lea -64(%rdx), %rdx + _CET_NOTRACK jmp *%r9 + ud2 L(shl_14_loop_L2): - prefetchnta 0x1c0(%rsi) + prefetchnta 0x1c0(%rsi) L(shl_14_loop_L1): - sub $64, %rdx - movaps 0x02(%rsi), %xmm2 - movaps 0x12(%rsi), %xmm3 - movaps 0x22(%rsi), %xmm4 - movaps 0x32(%rsi), %xmm5 - movdqa %xmm5, %xmm6 - palignr $14, %xmm4, %xmm5 - lea 64(%rsi), %rsi - palignr $14, %xmm3, %xmm4 - palignr $14, %xmm2, %xmm3 - lea 64(%rdi), %rdi - palignr $14, %xmm1, %xmm2 - movdqa %xmm6, %xmm1 - movdqa %xmm2, -0x40(%rdi) - movaps %xmm3, -0x30(%rdi) - jb L(shl_14_end) - movaps %xmm4, -0x20(%rdi) - movaps %xmm5, -0x10(%rdi) - _CET_NOTRACK jmp *%r9 - ud2 + sub $64, %rdx + movaps 0x02(%rsi), %xmm2 + movaps 0x12(%rsi), %xmm3 + movaps 0x22(%rsi), %xmm4 + movaps 0x32(%rsi), %xmm5 + movdqa %xmm5, %xmm6 + palignr $14, %xmm4, %xmm5 + lea 64(%rsi), %rsi + palignr $14, %xmm3, %xmm4 + palignr $14, %xmm2, %xmm3 + lea 64(%rdi), %rdi + palignr $14, %xmm1, %xmm2 + movdqa %xmm6, %xmm1 + movdqa %xmm2, -0x40(%rdi) + movaps %xmm3, -0x30(%rdi) + jb L(shl_14_end) + movaps %xmm4, -0x20(%rdi) + movaps %xmm5, -0x10(%rdi) + _CET_NOTRACK jmp *%r9 + ud2 L(shl_14_end): - movaps %xmm4, -0x20(%rdi) - lea 64(%rdx), %rdx - movaps %xmm5, -0x10(%rdi) - add %rdx, %rdi - movdqu %xmm0, (%r8) - add %rdx, %rsi - BRANCH_TO_JMPTBL_ENTRY(L(table_less_80bytes), %rdx, 4) + movaps %xmm4, -0x20(%rdi) + lea 64(%rdx), %rdx + movaps %xmm5, -0x10(%rdi) + add %rdx, %rdi + movdqu %xmm0, (%r8) + add %rdx, %rsi + BRANCH_TO_JMPTBL_ENTRY(L(table_less_80bytes), %rdx, 4) - .p2align 4 + .p2align 4 L(shl_14_bwd): - lea (L(shl_14_bwd_loop_L1)-L(shl_14_bwd))(%r9), %r9 - cmp %rcx, %rdx - movaps -0x0e(%rsi), %xmm1 - jb L(L14_bwd) - lea (L(shl_14_bwd_loop_L2)-L(shl_14_bwd_loop_L1))(%r9), %r9 + lea (L(shl_14_bwd_loop_L1)-L(shl_14_bwd))(%r9), %r9 + cmp %rcx, %rdx + movaps -0x0e(%rsi), %xmm1 + jb L(L14_bwd) + lea (L(shl_14_bwd_loop_L2)-L(shl_14_bwd_loop_L1))(%r9), %r9 L(L14_bwd): - lea -64(%rdx), %rdx - _CET_NOTRACK jmp *%r9 - ud2 + lea -64(%rdx), %rdx + _CET_NOTRACK jmp *%r9 + ud2 L(shl_14_bwd_loop_L2): - prefetchnta -0x1c0(%rsi) + prefetchnta -0x1c0(%rsi) L(shl_14_bwd_loop_L1): - movaps -0x1e(%rsi), %xmm2 - sub $0x40, %rdx - movaps -0x2e(%rsi), %xmm3 - movaps -0x3e(%rsi), %xmm4 - movaps -0x4e(%rsi), %xmm5 - lea -0x40(%rsi), %rsi - palignr $14, %xmm2, %xmm1 - palignr $14, %xmm3, %xmm2 - palignr $14, %xmm4, %xmm3 - palignr $14, %xmm5, %xmm4 + movaps -0x1e(%rsi), %xmm2 + sub $0x40, %rdx + movaps -0x2e(%rsi), %xmm3 + movaps -0x3e(%rsi), %xmm4 + movaps -0x4e(%rsi), %xmm5 + lea -0x40(%rsi), %rsi + palignr $14, %xmm2, %xmm1 + palignr $14, %xmm3, %xmm2 + palignr $14, %xmm4, %xmm3 + palignr $14, %xmm5, %xmm4 - movaps %xmm1, -0x10(%rdi) - movaps %xmm5, %xmm1 + movaps %xmm1, -0x10(%rdi) + movaps %xmm5, %xmm1 - movaps %xmm2, -0x20(%rdi) - lea -0x40(%rdi), %rdi + movaps %xmm2, -0x20(%rdi) + lea -0x40(%rdi), %rdi - movaps %xmm3, 0x10(%rdi) - jb L(shl_14_bwd_end) - movaps %xmm4, (%rdi) - _CET_NOTRACK jmp *%r9 - ud2 + movaps %xmm3, 0x10(%rdi) + jb L(shl_14_bwd_end) + movaps %xmm4, (%rdi) + _CET_NOTRACK jmp *%r9 + ud2 L(shl_14_bwd_end): - movaps %xmm4, (%rdi) - lea 64(%rdx), %rdx - movdqu %xmm0, (%r8) - BRANCH_TO_JMPTBL_ENTRY(L(table_less_80bytes), %rdx, 4) + movaps %xmm4, (%rdi) + lea 64(%rdx), %rdx + movdqu %xmm0, (%r8) + BRANCH_TO_JMPTBL_ENTRY(L(table_less_80bytes), %rdx, 4) - .p2align 4 + .p2align 4 L(shl_15): - lea (L(shl_15_loop_L1)-L(shl_15))(%r9), %r9 - cmp %rcx, %rdx - movaps -0x0f(%rsi), %xmm1 - jb L(L15_fwd) - lea (L(shl_15_loop_L2)-L(shl_15_loop_L1))(%r9), %r9 + lea (L(shl_15_loop_L1)-L(shl_15))(%r9), %r9 + cmp %rcx, %rdx + movaps -0x0f(%rsi), %xmm1 + jb L(L15_fwd) + lea (L(shl_15_loop_L2)-L(shl_15_loop_L1))(%r9), %r9 L(L15_fwd): - lea -64(%rdx), %rdx - _CET_NOTRACK jmp *%r9 - ud2 + lea -64(%rdx), %rdx + _CET_NOTRACK jmp *%r9 + ud2 L(shl_15_loop_L2): - prefetchnta 0x1c0(%rsi) + prefetchnta 0x1c0(%rsi) L(shl_15_loop_L1): - sub $64, %rdx - movaps 0x01(%rsi), %xmm2 - movaps 0x11(%rsi), %xmm3 - movaps 0x21(%rsi), %xmm4 - movaps 0x31(%rsi), %xmm5 - movdqa %xmm5, %xmm6 - palignr $15, %xmm4, %xmm5 - lea 64(%rsi), %rsi - palignr $15, %xmm3, %xmm4 - palignr $15, %xmm2, %xmm3 - lea 64(%rdi), %rdi - palignr $15, %xmm1, %xmm2 - movdqa %xmm6, %xmm1 - movdqa %xmm2, -0x40(%rdi) - movaps %xmm3, -0x30(%rdi) - jb L(shl_15_end) - movaps %xmm4, -0x20(%rdi) - movaps %xmm5, -0x10(%rdi) - _CET_NOTRACK jmp *%r9 - ud2 + sub $64, %rdx + movaps 0x01(%rsi), %xmm2 + movaps 0x11(%rsi), %xmm3 + movaps 0x21(%rsi), %xmm4 + movaps 0x31(%rsi), %xmm5 + movdqa %xmm5, %xmm6 + palignr $15, %xmm4, %xmm5 + lea 64(%rsi), %rsi + palignr $15, %xmm3, %xmm4 + palignr $15, %xmm2, %xmm3 + lea 64(%rdi), %rdi + palignr $15, %xmm1, %xmm2 + movdqa %xmm6, %xmm1 + movdqa %xmm2, -0x40(%rdi) + movaps %xmm3, -0x30(%rdi) + jb L(shl_15_end) + movaps %xmm4, -0x20(%rdi) + movaps %xmm5, -0x10(%rdi) + _CET_NOTRACK jmp *%r9 + ud2 L(shl_15_end): - movaps %xmm4, -0x20(%rdi) - lea 64(%rdx), %rdx - movaps %xmm5, -0x10(%rdi) - add %rdx, %rdi - movdqu %xmm0, (%r8) - add %rdx, %rsi - BRANCH_TO_JMPTBL_ENTRY(L(table_less_80bytes), %rdx, 4) + movaps %xmm4, -0x20(%rdi) + lea 64(%rdx), %rdx + movaps %xmm5, -0x10(%rdi) + add %rdx, %rdi + movdqu %xmm0, (%r8) + add %rdx, %rsi + BRANCH_TO_JMPTBL_ENTRY(L(table_less_80bytes), %rdx, 4) - .p2align 4 + .p2align 4 L(shl_15_bwd): - lea (L(shl_15_bwd_loop_L1)-L(shl_15_bwd))(%r9), %r9 - cmp %rcx, %rdx - movaps -0x0f(%rsi), %xmm1 - jb L(L15_bwd) - lea (L(shl_15_bwd_loop_L2)-L(shl_15_bwd_loop_L1))(%r9), %r9 + lea (L(shl_15_bwd_loop_L1)-L(shl_15_bwd))(%r9), %r9 + cmp %rcx, %rdx + movaps -0x0f(%rsi), %xmm1 + jb L(L15_bwd) + lea (L(shl_15_bwd_loop_L2)-L(shl_15_bwd_loop_L1))(%r9), %r9 L(L15_bwd): - lea -64(%rdx), %rdx - _CET_NOTRACK jmp *%r9 - ud2 + lea -64(%rdx), %rdx + _CET_NOTRACK jmp *%r9 + ud2 L(shl_15_bwd_loop_L2): - prefetchnta -0x1c0(%rsi) + prefetchnta -0x1c0(%rsi) L(shl_15_bwd_loop_L1): - movaps -0x1f(%rsi), %xmm2 - sub $0x40, %rdx - movaps -0x2f(%rsi), %xmm3 - movaps -0x3f(%rsi), %xmm4 - movaps -0x4f(%rsi), %xmm5 - lea -0x40(%rsi), %rsi - palignr $15, %xmm2, %xmm1 - palignr $15, %xmm3, %xmm2 - palignr $15, %xmm4, %xmm3 - palignr $15, %xmm5, %xmm4 + movaps -0x1f(%rsi), %xmm2 + sub $0x40, %rdx + movaps -0x2f(%rsi), %xmm3 + movaps -0x3f(%rsi), %xmm4 + movaps -0x4f(%rsi), %xmm5 + lea -0x40(%rsi), %rsi + palignr $15, %xmm2, %xmm1 + palignr $15, %xmm3, %xmm2 + palignr $15, %xmm4, %xmm3 + palignr $15, %xmm5, %xmm4 - movaps %xmm1, -0x10(%rdi) - movaps %xmm5, %xmm1 + movaps %xmm1, -0x10(%rdi) + movaps %xmm5, %xmm1 - movaps %xmm2, -0x20(%rdi) - lea -0x40(%rdi), %rdi + movaps %xmm2, -0x20(%rdi) + lea -0x40(%rdi), %rdi - movaps %xmm3, 0x10(%rdi) - jb L(shl_15_bwd_end) - movaps %xmm4, (%rdi) - _CET_NOTRACK jmp *%r9 - ud2 + movaps %xmm3, 0x10(%rdi) + jb L(shl_15_bwd_end) + movaps %xmm4, (%rdi) + _CET_NOTRACK jmp *%r9 + ud2 L(shl_15_bwd_end): - movaps %xmm4, (%rdi) - lea 64(%rdx), %rdx - movdqu %xmm0, (%r8) - BRANCH_TO_JMPTBL_ENTRY(L(table_less_80bytes), %rdx, 4) + movaps %xmm4, (%rdi) + lea 64(%rdx), %rdx + movdqu %xmm0, (%r8) + BRANCH_TO_JMPTBL_ENTRY(L(table_less_80bytes), %rdx, 4) - .p2align 4 + .p2align 4 L(write_72bytes): - movdqu -72(%rsi), %xmm0 - movdqu -56(%rsi), %xmm1 - mov -40(%rsi), %r8 - mov -32(%rsi), %r9 - mov -24(%rsi), %r10 - mov -16(%rsi), %r11 - mov -8(%rsi), %rcx - movdqu %xmm0, -72(%rdi) - movdqu %xmm1, -56(%rdi) - mov %r8, -40(%rdi) - mov %r9, -32(%rdi) - mov %r10, -24(%rdi) - mov %r11, -16(%rdi) - mov %rcx, -8(%rdi) - ret + movdqu -72(%rsi), %xmm0 + movdqu -56(%rsi), %xmm1 + mov -40(%rsi), %r8 + mov -32(%rsi), %r9 + mov -24(%rsi), %r10 + mov -16(%rsi), %r11 + mov -8(%rsi), %rcx + movdqu %xmm0, -72(%rdi) + movdqu %xmm1, -56(%rdi) + mov %r8, -40(%rdi) + mov %r9, -32(%rdi) + mov %r10, -24(%rdi) + mov %r11, -16(%rdi) + mov %rcx, -8(%rdi) + ret - .p2align 4 + .p2align 4 L(write_64bytes): - movdqu -64(%rsi), %xmm0 - mov -48(%rsi), %rcx - mov -40(%rsi), %r8 - mov -32(%rsi), %r9 - mov -24(%rsi), %r10 - mov -16(%rsi), %r11 - mov -8(%rsi), %rdx - movdqu %xmm0, -64(%rdi) - mov %rcx, -48(%rdi) - mov %r8, -40(%rdi) - mov %r9, -32(%rdi) - mov %r10, -24(%rdi) - mov %r11, -16(%rdi) - mov %rdx, -8(%rdi) - ret + movdqu -64(%rsi), %xmm0 + mov -48(%rsi), %rcx + mov -40(%rsi), %r8 + mov -32(%rsi), %r9 + mov -24(%rsi), %r10 + mov -16(%rsi), %r11 + mov -8(%rsi), %rdx + movdqu %xmm0, -64(%rdi) + mov %rcx, -48(%rdi) + mov %r8, -40(%rdi) + mov %r9, -32(%rdi) + mov %r10, -24(%rdi) + mov %r11, -16(%rdi) + mov %rdx, -8(%rdi) + ret - .p2align 4 + .p2align 4 L(write_56bytes): - movdqu -56(%rsi), %xmm0 - mov -40(%rsi), %r8 - mov -32(%rsi), %r9 - mov -24(%rsi), %r10 - mov -16(%rsi), %r11 - mov -8(%rsi), %rcx - movdqu %xmm0, -56(%rdi) - mov %r8, -40(%rdi) - mov %r9, -32(%rdi) - mov %r10, -24(%rdi) - mov %r11, -16(%rdi) - mov %rcx, -8(%rdi) - ret + movdqu -56(%rsi), %xmm0 + mov -40(%rsi), %r8 + mov -32(%rsi), %r9 + mov -24(%rsi), %r10 + mov -16(%rsi), %r11 + mov -8(%rsi), %rcx + movdqu %xmm0, -56(%rdi) + mov %r8, -40(%rdi) + mov %r9, -32(%rdi) + mov %r10, -24(%rdi) + mov %r11, -16(%rdi) + mov %rcx, -8(%rdi) + ret - .p2align 4 + .p2align 4 L(write_48bytes): - mov -48(%rsi), %rcx - mov -40(%rsi), %r8 - mov -32(%rsi), %r9 - mov -24(%rsi), %r10 - mov -16(%rsi), %r11 - mov -8(%rsi), %rdx - mov %rcx, -48(%rdi) - mov %r8, -40(%rdi) - mov %r9, -32(%rdi) - mov %r10, -24(%rdi) - mov %r11, -16(%rdi) - mov %rdx, -8(%rdi) - ret + mov -48(%rsi), %rcx + mov -40(%rsi), %r8 + mov -32(%rsi), %r9 + mov -24(%rsi), %r10 + mov -16(%rsi), %r11 + mov -8(%rsi), %rdx + mov %rcx, -48(%rdi) + mov %r8, -40(%rdi) + mov %r9, -32(%rdi) + mov %r10, -24(%rdi) + mov %r11, -16(%rdi) + mov %rdx, -8(%rdi) + ret - .p2align 4 + .p2align 4 L(write_40bytes): - mov -40(%rsi), %r8 - mov -32(%rsi), %r9 - mov -24(%rsi), %r10 - mov -16(%rsi), %r11 - mov -8(%rsi), %rdx - mov %r8, -40(%rdi) - mov %r9, -32(%rdi) - mov %r10, -24(%rdi) - mov %r11, -16(%rdi) - mov %rdx, -8(%rdi) - ret + mov -40(%rsi), %r8 + mov -32(%rsi), %r9 + mov -24(%rsi), %r10 + mov -16(%rsi), %r11 + mov -8(%rsi), %rdx + mov %r8, -40(%rdi) + mov %r9, -32(%rdi) + mov %r10, -24(%rdi) + mov %r11, -16(%rdi) + mov %rdx, -8(%rdi) + ret - .p2align 4 + .p2align 4 L(write_32bytes): - mov -32(%rsi), %r9 - mov -24(%rsi), %r10 - mov -16(%rsi), %r11 - mov -8(%rsi), %rdx - mov %r9, -32(%rdi) - mov %r10, -24(%rdi) - mov %r11, -16(%rdi) - mov %rdx, -8(%rdi) - ret + mov -32(%rsi), %r9 + mov -24(%rsi), %r10 + mov -16(%rsi), %r11 + mov -8(%rsi), %rdx + mov %r9, -32(%rdi) + mov %r10, -24(%rdi) + mov %r11, -16(%rdi) + mov %rdx, -8(%rdi) + ret - .p2align 4 + .p2align 4 L(write_24bytes): - mov -24(%rsi), %r10 - mov -16(%rsi), %r11 - mov -8(%rsi), %rdx - mov %r10, -24(%rdi) - mov %r11, -16(%rdi) - mov %rdx, -8(%rdi) - ret + mov -24(%rsi), %r10 + mov -16(%rsi), %r11 + mov -8(%rsi), %rdx + mov %r10, -24(%rdi) + mov %r11, -16(%rdi) + mov %rdx, -8(%rdi) + ret - .p2align 4 + .p2align 4 L(write_16bytes): - mov -16(%rsi), %r11 - mov -8(%rsi), %rdx - mov %r11, -16(%rdi) - mov %rdx, -8(%rdi) - ret + mov -16(%rsi), %r11 + mov -8(%rsi), %rdx + mov %r11, -16(%rdi) + mov %rdx, -8(%rdi) + ret - .p2align 4 + .p2align 4 L(write_8bytes): - mov -8(%rsi), %rdx - mov %rdx, -8(%rdi) + mov -8(%rsi), %rdx + mov %rdx, -8(%rdi) L(write_0bytes): - ret + ret - .p2align 4 + .p2align 4 L(write_73bytes): - movdqu -73(%rsi), %xmm0 - movdqu -57(%rsi), %xmm1 - mov -41(%rsi), %rcx - mov -33(%rsi), %r9 - mov -25(%rsi), %r10 - mov -17(%rsi), %r11 - mov -9(%rsi), %r8 - mov -4(%rsi), %edx - movdqu %xmm0, -73(%rdi) - movdqu %xmm1, -57(%rdi) - mov %rcx, -41(%rdi) - mov %r9, -33(%rdi) - mov %r10, -25(%rdi) - mov %r11, -17(%rdi) - mov %r8, -9(%rdi) - mov %edx, -4(%rdi) - ret + movdqu -73(%rsi), %xmm0 + movdqu -57(%rsi), %xmm1 + mov -41(%rsi), %rcx + mov -33(%rsi), %r9 + mov -25(%rsi), %r10 + mov -17(%rsi), %r11 + mov -9(%rsi), %r8 + mov -4(%rsi), %edx + movdqu %xmm0, -73(%rdi) + movdqu %xmm1, -57(%rdi) + mov %rcx, -41(%rdi) + mov %r9, -33(%rdi) + mov %r10, -25(%rdi) + mov %r11, -17(%rdi) + mov %r8, -9(%rdi) + mov %edx, -4(%rdi) + ret - .p2align 4 + .p2align 4 L(write_65bytes): - movdqu -65(%rsi), %xmm0 - movdqu -49(%rsi), %xmm1 - mov -33(%rsi), %r9 - mov -25(%rsi), %r10 - mov -17(%rsi), %r11 - mov -9(%rsi), %rcx - mov -4(%rsi), %edx - movdqu %xmm0, -65(%rdi) - movdqu %xmm1, -49(%rdi) - mov %r9, -33(%rdi) - mov %r10, -25(%rdi) - mov %r11, -17(%rdi) - mov %rcx, -9(%rdi) - mov %edx, -4(%rdi) - ret + movdqu -65(%rsi), %xmm0 + movdqu -49(%rsi), %xmm1 + mov -33(%rsi), %r9 + mov -25(%rsi), %r10 + mov -17(%rsi), %r11 + mov -9(%rsi), %rcx + mov -4(%rsi), %edx + movdqu %xmm0, -65(%rdi) + movdqu %xmm1, -49(%rdi) + mov %r9, -33(%rdi) + mov %r10, -25(%rdi) + mov %r11, -17(%rdi) + mov %rcx, -9(%rdi) + mov %edx, -4(%rdi) + ret - .p2align 4 + .p2align 4 L(write_57bytes): - movdqu -57(%rsi), %xmm0 - mov -41(%rsi), %r8 - mov -33(%rsi), %r9 - mov -25(%rsi), %r10 - mov -17(%rsi), %r11 - mov -9(%rsi), %rcx - mov -4(%rsi), %edx - movdqu %xmm0, -57(%rdi) - mov %r8, -41(%rdi) - mov %r9, -33(%rdi) - mov %r10, -25(%rdi) - mov %r11, -17(%rdi) - mov %rcx, -9(%rdi) - mov %edx, -4(%rdi) - ret + movdqu -57(%rsi), %xmm0 + mov -41(%rsi), %r8 + mov -33(%rsi), %r9 + mov -25(%rsi), %r10 + mov -17(%rsi), %r11 + mov -9(%rsi), %rcx + mov -4(%rsi), %edx + movdqu %xmm0, -57(%rdi) + mov %r8, -41(%rdi) + mov %r9, -33(%rdi) + mov %r10, -25(%rdi) + mov %r11, -17(%rdi) + mov %rcx, -9(%rdi) + mov %edx, -4(%rdi) + ret - .p2align 4 + .p2align 4 L(write_49bytes): - movdqu -49(%rsi), %xmm0 - mov -33(%rsi), %r9 - mov -25(%rsi), %r10 - mov -17(%rsi), %r11 - mov -9(%rsi), %rcx - mov -4(%rsi), %edx - movdqu %xmm0, -49(%rdi) - mov %r9, -33(%rdi) - mov %r10, -25(%rdi) - mov %r11, -17(%rdi) - mov %rcx, -9(%rdi) - mov %edx, -4(%rdi) - ret + movdqu -49(%rsi), %xmm0 + mov -33(%rsi), %r9 + mov -25(%rsi), %r10 + mov -17(%rsi), %r11 + mov -9(%rsi), %rcx + mov -4(%rsi), %edx + movdqu %xmm0, -49(%rdi) + mov %r9, -33(%rdi) + mov %r10, -25(%rdi) + mov %r11, -17(%rdi) + mov %rcx, -9(%rdi) + mov %edx, -4(%rdi) + ret - .p2align 4 + .p2align 4 L(write_41bytes): - mov -41(%rsi), %r8 - mov -33(%rsi), %r9 - mov -25(%rsi), %r10 - mov -17(%rsi), %r11 - mov -9(%rsi), %rcx - mov -1(%rsi), %dl - mov %r8, -41(%rdi) - mov %r9, -33(%rdi) - mov %r10, -25(%rdi) - mov %r11, -17(%rdi) - mov %rcx, -9(%rdi) - mov %dl, -1(%rdi) - ret + mov -41(%rsi), %r8 + mov -33(%rsi), %r9 + mov -25(%rsi), %r10 + mov -17(%rsi), %r11 + mov -9(%rsi), %rcx + mov -1(%rsi), %dl + mov %r8, -41(%rdi) + mov %r9, -33(%rdi) + mov %r10, -25(%rdi) + mov %r11, -17(%rdi) + mov %rcx, -9(%rdi) + mov %dl, -1(%rdi) + ret - .p2align 4 + .p2align 4 L(write_33bytes): - mov -33(%rsi), %r9 - mov -25(%rsi), %r10 - mov -17(%rsi), %r11 - mov -9(%rsi), %rcx - mov -1(%rsi), %dl - mov %r9, -33(%rdi) - mov %r10, -25(%rdi) - mov %r11, -17(%rdi) - mov %rcx, -9(%rdi) - mov %dl, -1(%rdi) - ret + mov -33(%rsi), %r9 + mov -25(%rsi), %r10 + mov -17(%rsi), %r11 + mov -9(%rsi), %rcx + mov -1(%rsi), %dl + mov %r9, -33(%rdi) + mov %r10, -25(%rdi) + mov %r11, -17(%rdi) + mov %rcx, -9(%rdi) + mov %dl, -1(%rdi) + ret - .p2align 4 + .p2align 4 L(write_25bytes): - mov -25(%rsi), %r10 - mov -17(%rsi), %r11 - mov -9(%rsi), %rcx - mov -1(%rsi), %dl - mov %r10, -25(%rdi) - mov %r11, -17(%rdi) - mov %rcx, -9(%rdi) - mov %dl, -1(%rdi) - ret + mov -25(%rsi), %r10 + mov -17(%rsi), %r11 + mov -9(%rsi), %rcx + mov -1(%rsi), %dl + mov %r10, -25(%rdi) + mov %r11, -17(%rdi) + mov %rcx, -9(%rdi) + mov %dl, -1(%rdi) + ret - .p2align 4 + .p2align 4 L(write_17bytes): - mov -17(%rsi), %r11 - mov -9(%rsi), %rcx - mov -4(%rsi), %edx - mov %r11, -17(%rdi) - mov %rcx, -9(%rdi) - mov %edx, -4(%rdi) - ret + mov -17(%rsi), %r11 + mov -9(%rsi), %rcx + mov -4(%rsi), %edx + mov %r11, -17(%rdi) + mov %rcx, -9(%rdi) + mov %edx, -4(%rdi) + ret - .p2align 4 + .p2align 4 L(write_9bytes): - mov -9(%rsi), %rcx - mov -4(%rsi), %edx - mov %rcx, -9(%rdi) - mov %edx, -4(%rdi) - ret + mov -9(%rsi), %rcx + mov -4(%rsi), %edx + mov %rcx, -9(%rdi) + mov %edx, -4(%rdi) + ret - .p2align 4 + .p2align 4 L(write_1bytes): - mov -1(%rsi), %dl - mov %dl, -1(%rdi) - ret + mov -1(%rsi), %dl + mov %dl, -1(%rdi) + ret - .p2align 4 + .p2align 4 L(write_74bytes): - movdqu -74(%rsi), %xmm0 - movdqu -58(%rsi), %xmm1 - mov -42(%rsi), %r8 - mov -34(%rsi), %r9 - mov -26(%rsi), %r10 - mov -18(%rsi), %r11 - mov -10(%rsi), %rcx - mov -4(%rsi), %edx - movdqu %xmm0, -74(%rdi) - movdqu %xmm1, -58(%rdi) - mov %r8, -42(%rdi) - mov %r9, -34(%rdi) - mov %r10, -26(%rdi) - mov %r11, -18(%rdi) - mov %rcx, -10(%rdi) - mov %edx, -4(%rdi) - ret + movdqu -74(%rsi), %xmm0 + movdqu -58(%rsi), %xmm1 + mov -42(%rsi), %r8 + mov -34(%rsi), %r9 + mov -26(%rsi), %r10 + mov -18(%rsi), %r11 + mov -10(%rsi), %rcx + mov -4(%rsi), %edx + movdqu %xmm0, -74(%rdi) + movdqu %xmm1, -58(%rdi) + mov %r8, -42(%rdi) + mov %r9, -34(%rdi) + mov %r10, -26(%rdi) + mov %r11, -18(%rdi) + mov %rcx, -10(%rdi) + mov %edx, -4(%rdi) + ret - .p2align 4 + .p2align 4 L(write_66bytes): - movdqu -66(%rsi), %xmm0 - movdqu -50(%rsi), %xmm1 - mov -42(%rsi), %r8 - mov -34(%rsi), %r9 - mov -26(%rsi), %r10 - mov -18(%rsi), %r11 - mov -10(%rsi), %rcx - mov -4(%rsi), %edx - movdqu %xmm0, -66(%rdi) - movdqu %xmm1, -50(%rdi) - mov %r8, -42(%rdi) - mov %r9, -34(%rdi) - mov %r10, -26(%rdi) - mov %r11, -18(%rdi) - mov %rcx, -10(%rdi) - mov %edx, -4(%rdi) - ret + movdqu -66(%rsi), %xmm0 + movdqu -50(%rsi), %xmm1 + mov -42(%rsi), %r8 + mov -34(%rsi), %r9 + mov -26(%rsi), %r10 + mov -18(%rsi), %r11 + mov -10(%rsi), %rcx + mov -4(%rsi), %edx + movdqu %xmm0, -66(%rdi) + movdqu %xmm1, -50(%rdi) + mov %r8, -42(%rdi) + mov %r9, -34(%rdi) + mov %r10, -26(%rdi) + mov %r11, -18(%rdi) + mov %rcx, -10(%rdi) + mov %edx, -4(%rdi) + ret - .p2align 4 + .p2align 4 L(write_58bytes): - movdqu -58(%rsi), %xmm1 - mov -42(%rsi), %r8 - mov -34(%rsi), %r9 - mov -26(%rsi), %r10 - mov -18(%rsi), %r11 - mov -10(%rsi), %rcx - mov -4(%rsi), %edx - movdqu %xmm1, -58(%rdi) - mov %r8, -42(%rdi) - mov %r9, -34(%rdi) - mov %r10, -26(%rdi) - mov %r11, -18(%rdi) - mov %rcx, -10(%rdi) - mov %edx, -4(%rdi) - ret + movdqu -58(%rsi), %xmm1 + mov -42(%rsi), %r8 + mov -34(%rsi), %r9 + mov -26(%rsi), %r10 + mov -18(%rsi), %r11 + mov -10(%rsi), %rcx + mov -4(%rsi), %edx + movdqu %xmm1, -58(%rdi) + mov %r8, -42(%rdi) + mov %r9, -34(%rdi) + mov %r10, -26(%rdi) + mov %r11, -18(%rdi) + mov %rcx, -10(%rdi) + mov %edx, -4(%rdi) + ret - .p2align 4 + .p2align 4 L(write_50bytes): - movdqu -50(%rsi), %xmm0 - mov -34(%rsi), %r9 - mov -26(%rsi), %r10 - mov -18(%rsi), %r11 - mov -10(%rsi), %rcx - mov -4(%rsi), %edx - movdqu %xmm0, -50(%rdi) - mov %r9, -34(%rdi) - mov %r10, -26(%rdi) - mov %r11, -18(%rdi) - mov %rcx, -10(%rdi) - mov %edx, -4(%rdi) - ret + movdqu -50(%rsi), %xmm0 + mov -34(%rsi), %r9 + mov -26(%rsi), %r10 + mov -18(%rsi), %r11 + mov -10(%rsi), %rcx + mov -4(%rsi), %edx + movdqu %xmm0, -50(%rdi) + mov %r9, -34(%rdi) + mov %r10, -26(%rdi) + mov %r11, -18(%rdi) + mov %rcx, -10(%rdi) + mov %edx, -4(%rdi) + ret - .p2align 4 + .p2align 4 L(write_42bytes): - mov -42(%rsi), %r8 - mov -34(%rsi), %r9 - mov -26(%rsi), %r10 - mov -18(%rsi), %r11 - mov -10(%rsi), %rcx - mov -4(%rsi), %edx - mov %r8, -42(%rdi) - mov %r9, -34(%rdi) - mov %r10, -26(%rdi) - mov %r11, -18(%rdi) - mov %rcx, -10(%rdi) - mov %edx, -4(%rdi) - ret + mov -42(%rsi), %r8 + mov -34(%rsi), %r9 + mov -26(%rsi), %r10 + mov -18(%rsi), %r11 + mov -10(%rsi), %rcx + mov -4(%rsi), %edx + mov %r8, -42(%rdi) + mov %r9, -34(%rdi) + mov %r10, -26(%rdi) + mov %r11, -18(%rdi) + mov %rcx, -10(%rdi) + mov %edx, -4(%rdi) + ret - .p2align 4 + .p2align 4 L(write_34bytes): - mov -34(%rsi), %r9 - mov -26(%rsi), %r10 - mov -18(%rsi), %r11 - mov -10(%rsi), %rcx - mov -4(%rsi), %edx - mov %r9, -34(%rdi) - mov %r10, -26(%rdi) - mov %r11, -18(%rdi) - mov %rcx, -10(%rdi) - mov %edx, -4(%rdi) - ret + mov -34(%rsi), %r9 + mov -26(%rsi), %r10 + mov -18(%rsi), %r11 + mov -10(%rsi), %rcx + mov -4(%rsi), %edx + mov %r9, -34(%rdi) + mov %r10, -26(%rdi) + mov %r11, -18(%rdi) + mov %rcx, -10(%rdi) + mov %edx, -4(%rdi) + ret - .p2align 4 + .p2align 4 L(write_26bytes): - mov -26(%rsi), %r10 - mov -18(%rsi), %r11 - mov -10(%rsi), %rcx - mov -4(%rsi), %edx - mov %r10, -26(%rdi) - mov %r11, -18(%rdi) - mov %rcx, -10(%rdi) - mov %edx, -4(%rdi) - ret + mov -26(%rsi), %r10 + mov -18(%rsi), %r11 + mov -10(%rsi), %rcx + mov -4(%rsi), %edx + mov %r10, -26(%rdi) + mov %r11, -18(%rdi) + mov %rcx, -10(%rdi) + mov %edx, -4(%rdi) + ret - .p2align 4 + .p2align 4 L(write_18bytes): - mov -18(%rsi), %r11 - mov -10(%rsi), %rcx - mov -4(%rsi), %edx - mov %r11, -18(%rdi) - mov %rcx, -10(%rdi) - mov %edx, -4(%rdi) - ret + mov -18(%rsi), %r11 + mov -10(%rsi), %rcx + mov -4(%rsi), %edx + mov %r11, -18(%rdi) + mov %rcx, -10(%rdi) + mov %edx, -4(%rdi) + ret - .p2align 4 + .p2align 4 L(write_10bytes): - mov -10(%rsi), %rcx - mov -4(%rsi), %edx - mov %rcx, -10(%rdi) - mov %edx, -4(%rdi) - ret + mov -10(%rsi), %rcx + mov -4(%rsi), %edx + mov %rcx, -10(%rdi) + mov %edx, -4(%rdi) + ret - .p2align 4 + .p2align 4 L(write_2bytes): - mov -2(%rsi), %dx - mov %dx, -2(%rdi) - ret + mov -2(%rsi), %dx + mov %dx, -2(%rdi) + ret - .p2align 4 + .p2align 4 L(write_75bytes): - movdqu -75(%rsi), %xmm0 - movdqu -59(%rsi), %xmm1 - mov -43(%rsi), %r8 - mov -35(%rsi), %r9 - mov -27(%rsi), %r10 - mov -19(%rsi), %r11 - mov -11(%rsi), %rcx - mov -4(%rsi), %edx - movdqu %xmm0, -75(%rdi) - movdqu %xmm1, -59(%rdi) - mov %r8, -43(%rdi) - mov %r9, -35(%rdi) - mov %r10, -27(%rdi) - mov %r11, -19(%rdi) - mov %rcx, -11(%rdi) - mov %edx, -4(%rdi) - ret + movdqu -75(%rsi), %xmm0 + movdqu -59(%rsi), %xmm1 + mov -43(%rsi), %r8 + mov -35(%rsi), %r9 + mov -27(%rsi), %r10 + mov -19(%rsi), %r11 + mov -11(%rsi), %rcx + mov -4(%rsi), %edx + movdqu %xmm0, -75(%rdi) + movdqu %xmm1, -59(%rdi) + mov %r8, -43(%rdi) + mov %r9, -35(%rdi) + mov %r10, -27(%rdi) + mov %r11, -19(%rdi) + mov %rcx, -11(%rdi) + mov %edx, -4(%rdi) + ret - .p2align 4 + .p2align 4 L(write_67bytes): - movdqu -67(%rsi), %xmm0 - movdqu -59(%rsi), %xmm1 - mov -43(%rsi), %r8 - mov -35(%rsi), %r9 - mov -27(%rsi), %r10 - mov -19(%rsi), %r11 - mov -11(%rsi), %rcx - mov -4(%rsi), %edx - movdqu %xmm0, -67(%rdi) - movdqu %xmm1, -59(%rdi) - mov %r8, -43(%rdi) - mov %r9, -35(%rdi) - mov %r10, -27(%rdi) - mov %r11, -19(%rdi) - mov %rcx, -11(%rdi) - mov %edx, -4(%rdi) - ret + movdqu -67(%rsi), %xmm0 + movdqu -59(%rsi), %xmm1 + mov -43(%rsi), %r8 + mov -35(%rsi), %r9 + mov -27(%rsi), %r10 + mov -19(%rsi), %r11 + mov -11(%rsi), %rcx + mov -4(%rsi), %edx + movdqu %xmm0, -67(%rdi) + movdqu %xmm1, -59(%rdi) + mov %r8, -43(%rdi) + mov %r9, -35(%rdi) + mov %r10, -27(%rdi) + mov %r11, -19(%rdi) + mov %rcx, -11(%rdi) + mov %edx, -4(%rdi) + ret - .p2align 4 + .p2align 4 L(write_59bytes): - movdqu -59(%rsi), %xmm0 - mov -43(%rsi), %r8 - mov -35(%rsi), %r9 - mov -27(%rsi), %r10 - mov -19(%rsi), %r11 - mov -11(%rsi), %rcx - mov -4(%rsi), %edx - movdqu %xmm0, -59(%rdi) - mov %r8, -43(%rdi) - mov %r9, -35(%rdi) - mov %r10, -27(%rdi) - mov %r11, -19(%rdi) - mov %rcx, -11(%rdi) - mov %edx, -4(%rdi) - ret + movdqu -59(%rsi), %xmm0 + mov -43(%rsi), %r8 + mov -35(%rsi), %r9 + mov -27(%rsi), %r10 + mov -19(%rsi), %r11 + mov -11(%rsi), %rcx + mov -4(%rsi), %edx + movdqu %xmm0, -59(%rdi) + mov %r8, -43(%rdi) + mov %r9, -35(%rdi) + mov %r10, -27(%rdi) + mov %r11, -19(%rdi) + mov %rcx, -11(%rdi) + mov %edx, -4(%rdi) + ret - .p2align 4 + .p2align 4 L(write_51bytes): - movdqu -51(%rsi), %xmm0 - mov -35(%rsi), %r9 - mov -27(%rsi), %r10 - mov -19(%rsi), %r11 - mov -11(%rsi), %rcx - mov -4(%rsi), %edx - movdqu %xmm0, -51(%rdi) - mov %r9, -35(%rdi) - mov %r10, -27(%rdi) - mov %r11, -19(%rdi) - mov %rcx, -11(%rdi) - mov %edx, -4(%rdi) - ret + movdqu -51(%rsi), %xmm0 + mov -35(%rsi), %r9 + mov -27(%rsi), %r10 + mov -19(%rsi), %r11 + mov -11(%rsi), %rcx + mov -4(%rsi), %edx + movdqu %xmm0, -51(%rdi) + mov %r9, -35(%rdi) + mov %r10, -27(%rdi) + mov %r11, -19(%rdi) + mov %rcx, -11(%rdi) + mov %edx, -4(%rdi) + ret - .p2align 4 + .p2align 4 L(write_43bytes): - mov -43(%rsi), %r8 - mov -35(%rsi), %r9 - mov -27(%rsi), %r10 - mov -19(%rsi), %r11 - mov -11(%rsi), %rcx - mov -4(%rsi), %edx - mov %r8, -43(%rdi) - mov %r9, -35(%rdi) - mov %r10, -27(%rdi) - mov %r11, -19(%rdi) - mov %rcx, -11(%rdi) - mov %edx, -4(%rdi) - ret + mov -43(%rsi), %r8 + mov -35(%rsi), %r9 + mov -27(%rsi), %r10 + mov -19(%rsi), %r11 + mov -11(%rsi), %rcx + mov -4(%rsi), %edx + mov %r8, -43(%rdi) + mov %r9, -35(%rdi) + mov %r10, -27(%rdi) + mov %r11, -19(%rdi) + mov %rcx, -11(%rdi) + mov %edx, -4(%rdi) + ret - .p2align 4 + .p2align 4 L(write_35bytes): - mov -35(%rsi), %r9 - mov -27(%rsi), %r10 - mov -19(%rsi), %r11 - mov -11(%rsi), %rcx - mov -4(%rsi), %edx - mov %r9, -35(%rdi) - mov %r10, -27(%rdi) - mov %r11, -19(%rdi) - mov %rcx, -11(%rdi) - mov %edx, -4(%rdi) - ret + mov -35(%rsi), %r9 + mov -27(%rsi), %r10 + mov -19(%rsi), %r11 + mov -11(%rsi), %rcx + mov -4(%rsi), %edx + mov %r9, -35(%rdi) + mov %r10, -27(%rdi) + mov %r11, -19(%rdi) + mov %rcx, -11(%rdi) + mov %edx, -4(%rdi) + ret - .p2align 4 + .p2align 4 L(write_27bytes): - mov -27(%rsi), %r10 - mov -19(%rsi), %r11 - mov -11(%rsi), %rcx - mov -4(%rsi), %edx - mov %r10, -27(%rdi) - mov %r11, -19(%rdi) - mov %rcx, -11(%rdi) - mov %edx, -4(%rdi) - ret + mov -27(%rsi), %r10 + mov -19(%rsi), %r11 + mov -11(%rsi), %rcx + mov -4(%rsi), %edx + mov %r10, -27(%rdi) + mov %r11, -19(%rdi) + mov %rcx, -11(%rdi) + mov %edx, -4(%rdi) + ret - .p2align 4 + .p2align 4 L(write_19bytes): - mov -19(%rsi), %r11 - mov -11(%rsi), %rcx - mov -4(%rsi), %edx - mov %r11, -19(%rdi) - mov %rcx, -11(%rdi) - mov %edx, -4(%rdi) - ret + mov -19(%rsi), %r11 + mov -11(%rsi), %rcx + mov -4(%rsi), %edx + mov %r11, -19(%rdi) + mov %rcx, -11(%rdi) + mov %edx, -4(%rdi) + ret - .p2align 4 + .p2align 4 L(write_11bytes): - mov -11(%rsi), %rcx - mov -4(%rsi), %edx - mov %rcx, -11(%rdi) - mov %edx, -4(%rdi) - ret + mov -11(%rsi), %rcx + mov -4(%rsi), %edx + mov %rcx, -11(%rdi) + mov %edx, -4(%rdi) + ret - .p2align 4 + .p2align 4 L(write_3bytes): - mov -3(%rsi), %dx - mov -2(%rsi), %cx - mov %dx, -3(%rdi) - mov %cx, -2(%rdi) - ret + mov -3(%rsi), %dx + mov -2(%rsi), %cx + mov %dx, -3(%rdi) + mov %cx, -2(%rdi) + ret - .p2align 4 + .p2align 4 L(write_76bytes): - movdqu -76(%rsi), %xmm0 - movdqu -60(%rsi), %xmm1 - mov -44(%rsi), %r8 - mov -36(%rsi), %r9 - mov -28(%rsi), %r10 - mov -20(%rsi), %r11 - mov -12(%rsi), %rcx - mov -4(%rsi), %edx - movdqu %xmm0, -76(%rdi) - movdqu %xmm1, -60(%rdi) - mov %r8, -44(%rdi) - mov %r9, -36(%rdi) - mov %r10, -28(%rdi) - mov %r11, -20(%rdi) - mov %rcx, -12(%rdi) - mov %edx, -4(%rdi) - ret + movdqu -76(%rsi), %xmm0 + movdqu -60(%rsi), %xmm1 + mov -44(%rsi), %r8 + mov -36(%rsi), %r9 + mov -28(%rsi), %r10 + mov -20(%rsi), %r11 + mov -12(%rsi), %rcx + mov -4(%rsi), %edx + movdqu %xmm0, -76(%rdi) + movdqu %xmm1, -60(%rdi) + mov %r8, -44(%rdi) + mov %r9, -36(%rdi) + mov %r10, -28(%rdi) + mov %r11, -20(%rdi) + mov %rcx, -12(%rdi) + mov %edx, -4(%rdi) + ret - .p2align 4 + .p2align 4 L(write_68bytes): - movdqu -68(%rsi), %xmm0 - movdqu -52(%rsi), %xmm1 - mov -36(%rsi), %r9 - mov -28(%rsi), %r10 - mov -20(%rsi), %r11 - mov -12(%rsi), %rcx - mov -4(%rsi), %edx - movdqu %xmm0, -68(%rdi) - movdqu %xmm1, -52(%rdi) - mov %r9, -36(%rdi) - mov %r10, -28(%rdi) - mov %r11, -20(%rdi) - mov %rcx, -12(%rdi) - mov %edx, -4(%rdi) - ret + movdqu -68(%rsi), %xmm0 + movdqu -52(%rsi), %xmm1 + mov -36(%rsi), %r9 + mov -28(%rsi), %r10 + mov -20(%rsi), %r11 + mov -12(%rsi), %rcx + mov -4(%rsi), %edx + movdqu %xmm0, -68(%rdi) + movdqu %xmm1, -52(%rdi) + mov %r9, -36(%rdi) + mov %r10, -28(%rdi) + mov %r11, -20(%rdi) + mov %rcx, -12(%rdi) + mov %edx, -4(%rdi) + ret - .p2align 4 + .p2align 4 L(write_60bytes): - movdqu -60(%rsi), %xmm0 - mov -44(%rsi), %r8 - mov -36(%rsi), %r9 - mov -28(%rsi), %r10 - mov -20(%rsi), %r11 - mov -12(%rsi), %rcx - mov -4(%rsi), %edx - movdqu %xmm0, -60(%rdi) - mov %r8, -44(%rdi) - mov %r9, -36(%rdi) - mov %r10, -28(%rdi) - mov %r11, -20(%rdi) - mov %rcx, -12(%rdi) - mov %edx, -4(%rdi) - ret + movdqu -60(%rsi), %xmm0 + mov -44(%rsi), %r8 + mov -36(%rsi), %r9 + mov -28(%rsi), %r10 + mov -20(%rsi), %r11 + mov -12(%rsi), %rcx + mov -4(%rsi), %edx + movdqu %xmm0, -60(%rdi) + mov %r8, -44(%rdi) + mov %r9, -36(%rdi) + mov %r10, -28(%rdi) + mov %r11, -20(%rdi) + mov %rcx, -12(%rdi) + mov %edx, -4(%rdi) + ret - .p2align 4 + .p2align 4 L(write_52bytes): - movdqu -52(%rsi), %xmm0 - mov -36(%rsi), %r9 - mov -28(%rsi), %r10 - mov -20(%rsi), %r11 - mov -12(%rsi), %rcx - mov -4(%rsi), %edx - movdqu %xmm0, -52(%rdi) - mov %r9, -36(%rdi) - mov %r10, -28(%rdi) - mov %r11, -20(%rdi) - mov %rcx, -12(%rdi) - mov %edx, -4(%rdi) - ret + movdqu -52(%rsi), %xmm0 + mov -36(%rsi), %r9 + mov -28(%rsi), %r10 + mov -20(%rsi), %r11 + mov -12(%rsi), %rcx + mov -4(%rsi), %edx + movdqu %xmm0, -52(%rdi) + mov %r9, -36(%rdi) + mov %r10, -28(%rdi) + mov %r11, -20(%rdi) + mov %rcx, -12(%rdi) + mov %edx, -4(%rdi) + ret - .p2align 4 + .p2align 4 L(write_44bytes): - mov -44(%rsi), %r8 - mov -36(%rsi), %r9 - mov -28(%rsi), %r10 - mov -20(%rsi), %r11 - mov -12(%rsi), %rcx - mov -4(%rsi), %edx - mov %r8, -44(%rdi) - mov %r9, -36(%rdi) - mov %r10, -28(%rdi) - mov %r11, -20(%rdi) - mov %rcx, -12(%rdi) - mov %edx, -4(%rdi) - ret + mov -44(%rsi), %r8 + mov -36(%rsi), %r9 + mov -28(%rsi), %r10 + mov -20(%rsi), %r11 + mov -12(%rsi), %rcx + mov -4(%rsi), %edx + mov %r8, -44(%rdi) + mov %r9, -36(%rdi) + mov %r10, -28(%rdi) + mov %r11, -20(%rdi) + mov %rcx, -12(%rdi) + mov %edx, -4(%rdi) + ret - .p2align 4 + .p2align 4 L(write_36bytes): - mov -36(%rsi), %r9 - mov -28(%rsi), %r10 - mov -20(%rsi), %r11 - mov -12(%rsi), %rcx - mov -4(%rsi), %edx - mov %r9, -36(%rdi) - mov %r10, -28(%rdi) - mov %r11, -20(%rdi) - mov %rcx, -12(%rdi) - mov %edx, -4(%rdi) - ret + mov -36(%rsi), %r9 + mov -28(%rsi), %r10 + mov -20(%rsi), %r11 + mov -12(%rsi), %rcx + mov -4(%rsi), %edx + mov %r9, -36(%rdi) + mov %r10, -28(%rdi) + mov %r11, -20(%rdi) + mov %rcx, -12(%rdi) + mov %edx, -4(%rdi) + ret - .p2align 4 + .p2align 4 L(write_28bytes): - mov -28(%rsi), %r10 - mov -20(%rsi), %r11 - mov -12(%rsi), %rcx - mov -4(%rsi), %edx - mov %r10, -28(%rdi) - mov %r11, -20(%rdi) - mov %rcx, -12(%rdi) - mov %edx, -4(%rdi) - ret + mov -28(%rsi), %r10 + mov -20(%rsi), %r11 + mov -12(%rsi), %rcx + mov -4(%rsi), %edx + mov %r10, -28(%rdi) + mov %r11, -20(%rdi) + mov %rcx, -12(%rdi) + mov %edx, -4(%rdi) + ret - .p2align 4 + .p2align 4 L(write_20bytes): - mov -20(%rsi), %r11 - mov -12(%rsi), %rcx - mov -4(%rsi), %edx - mov %r11, -20(%rdi) - mov %rcx, -12(%rdi) - mov %edx, -4(%rdi) - ret + mov -20(%rsi), %r11 + mov -12(%rsi), %rcx + mov -4(%rsi), %edx + mov %r11, -20(%rdi) + mov %rcx, -12(%rdi) + mov %edx, -4(%rdi) + ret - .p2align 4 + .p2align 4 L(write_12bytes): - mov -12(%rsi), %rcx - mov -4(%rsi), %edx - mov %rcx, -12(%rdi) - mov %edx, -4(%rdi) - ret + mov -12(%rsi), %rcx + mov -4(%rsi), %edx + mov %rcx, -12(%rdi) + mov %edx, -4(%rdi) + ret - .p2align 4 + .p2align 4 L(write_4bytes): - mov -4(%rsi), %edx - mov %edx, -4(%rdi) - ret + mov -4(%rsi), %edx + mov %edx, -4(%rdi) + ret - .p2align 4 + .p2align 4 L(write_77bytes): - movdqu -77(%rsi), %xmm0 - movdqu -61(%rsi), %xmm1 - mov -45(%rsi), %r8 - mov -37(%rsi), %r9 - mov -29(%rsi), %r10 - mov -21(%rsi), %r11 - mov -13(%rsi), %rcx - mov -8(%rsi), %rdx - movdqu %xmm0, -77(%rdi) - movdqu %xmm1, -61(%rdi) - mov %r8, -45(%rdi) - mov %r9, -37(%rdi) - mov %r10, -29(%rdi) - mov %r11, -21(%rdi) - mov %rcx, -13(%rdi) - mov %rdx, -8(%rdi) - ret + movdqu -77(%rsi), %xmm0 + movdqu -61(%rsi), %xmm1 + mov -45(%rsi), %r8 + mov -37(%rsi), %r9 + mov -29(%rsi), %r10 + mov -21(%rsi), %r11 + mov -13(%rsi), %rcx + mov -8(%rsi), %rdx + movdqu %xmm0, -77(%rdi) + movdqu %xmm1, -61(%rdi) + mov %r8, -45(%rdi) + mov %r9, -37(%rdi) + mov %r10, -29(%rdi) + mov %r11, -21(%rdi) + mov %rcx, -13(%rdi) + mov %rdx, -8(%rdi) + ret - .p2align 4 + .p2align 4 L(write_69bytes): - movdqu -69(%rsi), %xmm0 - movdqu -53(%rsi), %xmm1 - mov -37(%rsi), %r9 - mov -29(%rsi), %r10 - mov -21(%rsi), %r11 - mov -13(%rsi), %rcx - mov -8(%rsi), %rdx - movdqu %xmm0, -69(%rdi) - movdqu %xmm1, -53(%rdi) - mov %r9, -37(%rdi) - mov %r10, -29(%rdi) - mov %r11, -21(%rdi) - mov %rcx, -13(%rdi) - mov %rdx, -8(%rdi) - ret + movdqu -69(%rsi), %xmm0 + movdqu -53(%rsi), %xmm1 + mov -37(%rsi), %r9 + mov -29(%rsi), %r10 + mov -21(%rsi), %r11 + mov -13(%rsi), %rcx + mov -8(%rsi), %rdx + movdqu %xmm0, -69(%rdi) + movdqu %xmm1, -53(%rdi) + mov %r9, -37(%rdi) + mov %r10, -29(%rdi) + mov %r11, -21(%rdi) + mov %rcx, -13(%rdi) + mov %rdx, -8(%rdi) + ret - .p2align 4 + .p2align 4 L(write_61bytes): - movdqu -61(%rsi), %xmm0 - mov -45(%rsi), %r8 - mov -37(%rsi), %r9 - mov -29(%rsi), %r10 - mov -21(%rsi), %r11 - mov -13(%rsi), %rcx - mov -8(%rsi), %rdx - movdqu %xmm0, -61(%rdi) - mov %r8, -45(%rdi) - mov %r9, -37(%rdi) - mov %r10, -29(%rdi) - mov %r11, -21(%rdi) - mov %rcx, -13(%rdi) - mov %rdx, -8(%rdi) - ret + movdqu -61(%rsi), %xmm0 + mov -45(%rsi), %r8 + mov -37(%rsi), %r9 + mov -29(%rsi), %r10 + mov -21(%rsi), %r11 + mov -13(%rsi), %rcx + mov -8(%rsi), %rdx + movdqu %xmm0, -61(%rdi) + mov %r8, -45(%rdi) + mov %r9, -37(%rdi) + mov %r10, -29(%rdi) + mov %r11, -21(%rdi) + mov %rcx, -13(%rdi) + mov %rdx, -8(%rdi) + ret - .p2align 4 + .p2align 4 L(write_53bytes): - movdqu -53(%rsi), %xmm0 - mov -45(%rsi), %r8 - mov -37(%rsi), %r9 - mov -29(%rsi), %r10 - mov -21(%rsi), %r11 - mov -13(%rsi), %rcx - mov -8(%rsi), %rdx - movdqu %xmm0, -53(%rdi) - mov %r9, -37(%rdi) - mov %r10, -29(%rdi) - mov %r11, -21(%rdi) - mov %rcx, -13(%rdi) - mov %rdx, -8(%rdi) - ret + movdqu -53(%rsi), %xmm0 + mov -45(%rsi), %r8 + mov -37(%rsi), %r9 + mov -29(%rsi), %r10 + mov -21(%rsi), %r11 + mov -13(%rsi), %rcx + mov -8(%rsi), %rdx + movdqu %xmm0, -53(%rdi) + mov %r9, -37(%rdi) + mov %r10, -29(%rdi) + mov %r11, -21(%rdi) + mov %rcx, -13(%rdi) + mov %rdx, -8(%rdi) + ret - .p2align 4 + .p2align 4 L(write_45bytes): - mov -45(%rsi), %r8 - mov -37(%rsi), %r9 - mov -29(%rsi), %r10 - mov -21(%rsi), %r11 - mov -13(%rsi), %rcx - mov -8(%rsi), %rdx - mov %r8, -45(%rdi) - mov %r9, -37(%rdi) - mov %r10, -29(%rdi) - mov %r11, -21(%rdi) - mov %rcx, -13(%rdi) - mov %rdx, -8(%rdi) - ret + mov -45(%rsi), %r8 + mov -37(%rsi), %r9 + mov -29(%rsi), %r10 + mov -21(%rsi), %r11 + mov -13(%rsi), %rcx + mov -8(%rsi), %rdx + mov %r8, -45(%rdi) + mov %r9, -37(%rdi) + mov %r10, -29(%rdi) + mov %r11, -21(%rdi) + mov %rcx, -13(%rdi) + mov %rdx, -8(%rdi) + ret - .p2align 4 + .p2align 4 L(write_37bytes): - mov -37(%rsi), %r9 - mov -29(%rsi), %r10 - mov -21(%rsi), %r11 - mov -13(%rsi), %rcx - mov -8(%rsi), %rdx - mov %r9, -37(%rdi) - mov %r10, -29(%rdi) - mov %r11, -21(%rdi) - mov %rcx, -13(%rdi) - mov %rdx, -8(%rdi) - ret + mov -37(%rsi), %r9 + mov -29(%rsi), %r10 + mov -21(%rsi), %r11 + mov -13(%rsi), %rcx + mov -8(%rsi), %rdx + mov %r9, -37(%rdi) + mov %r10, -29(%rdi) + mov %r11, -21(%rdi) + mov %rcx, -13(%rdi) + mov %rdx, -8(%rdi) + ret - .p2align 4 + .p2align 4 L(write_29bytes): - mov -29(%rsi), %r10 - mov -21(%rsi), %r11 - mov -13(%rsi), %rcx - mov -8(%rsi), %rdx - mov %r10, -29(%rdi) - mov %r11, -21(%rdi) - mov %rcx, -13(%rdi) - mov %rdx, -8(%rdi) - ret + mov -29(%rsi), %r10 + mov -21(%rsi), %r11 + mov -13(%rsi), %rcx + mov -8(%rsi), %rdx + mov %r10, -29(%rdi) + mov %r11, -21(%rdi) + mov %rcx, -13(%rdi) + mov %rdx, -8(%rdi) + ret - .p2align 4 + .p2align 4 L(write_21bytes): - mov -21(%rsi), %r11 - mov -13(%rsi), %rcx - mov -8(%rsi), %rdx - mov %r11, -21(%rdi) - mov %rcx, -13(%rdi) - mov %rdx, -8(%rdi) - ret + mov -21(%rsi), %r11 + mov -13(%rsi), %rcx + mov -8(%rsi), %rdx + mov %r11, -21(%rdi) + mov %rcx, -13(%rdi) + mov %rdx, -8(%rdi) + ret - .p2align 4 + .p2align 4 L(write_13bytes): - mov -13(%rsi), %rcx - mov -8(%rsi), %rdx - mov %rcx, -13(%rdi) - mov %rdx, -8(%rdi) - ret + mov -13(%rsi), %rcx + mov -8(%rsi), %rdx + mov %rcx, -13(%rdi) + mov %rdx, -8(%rdi) + ret - .p2align 4 + .p2align 4 L(write_5bytes): - mov -5(%rsi), %edx - mov -4(%rsi), %ecx - mov %edx, -5(%rdi) - mov %ecx, -4(%rdi) - ret + mov -5(%rsi), %edx + mov -4(%rsi), %ecx + mov %edx, -5(%rdi) + mov %ecx, -4(%rdi) + ret - .p2align 4 + .p2align 4 L(write_78bytes): - movdqu -78(%rsi), %xmm0 - movdqu -62(%rsi), %xmm1 - mov -46(%rsi), %r8 - mov -38(%rsi), %r9 - mov -30(%rsi), %r10 - mov -22(%rsi), %r11 - mov -14(%rsi), %rcx - mov -8(%rsi), %rdx - movdqu %xmm0, -78(%rdi) - movdqu %xmm1, -62(%rdi) - mov %r8, -46(%rdi) - mov %r9, -38(%rdi) - mov %r10, -30(%rdi) - mov %r11, -22(%rdi) - mov %rcx, -14(%rdi) - mov %rdx, -8(%rdi) - ret + movdqu -78(%rsi), %xmm0 + movdqu -62(%rsi), %xmm1 + mov -46(%rsi), %r8 + mov -38(%rsi), %r9 + mov -30(%rsi), %r10 + mov -22(%rsi), %r11 + mov -14(%rsi), %rcx + mov -8(%rsi), %rdx + movdqu %xmm0, -78(%rdi) + movdqu %xmm1, -62(%rdi) + mov %r8, -46(%rdi) + mov %r9, -38(%rdi) + mov %r10, -30(%rdi) + mov %r11, -22(%rdi) + mov %rcx, -14(%rdi) + mov %rdx, -8(%rdi) + ret - .p2align 4 + .p2align 4 L(write_70bytes): - movdqu -70(%rsi), %xmm0 - movdqu -54(%rsi), %xmm1 - mov -38(%rsi), %r9 - mov -30(%rsi), %r10 - mov -22(%rsi), %r11 - mov -14(%rsi), %rcx - mov -8(%rsi), %rdx - movdqu %xmm0, -70(%rdi) - movdqu %xmm1, -54(%rdi) - mov %r9, -38(%rdi) - mov %r10, -30(%rdi) - mov %r11, -22(%rdi) - mov %rcx, -14(%rdi) - mov %rdx, -8(%rdi) - ret + movdqu -70(%rsi), %xmm0 + movdqu -54(%rsi), %xmm1 + mov -38(%rsi), %r9 + mov -30(%rsi), %r10 + mov -22(%rsi), %r11 + mov -14(%rsi), %rcx + mov -8(%rsi), %rdx + movdqu %xmm0, -70(%rdi) + movdqu %xmm1, -54(%rdi) + mov %r9, -38(%rdi) + mov %r10, -30(%rdi) + mov %r11, -22(%rdi) + mov %rcx, -14(%rdi) + mov %rdx, -8(%rdi) + ret - .p2align 4 + .p2align 4 L(write_62bytes): - movdqu -62(%rsi), %xmm0 - mov -46(%rsi), %r8 - mov -38(%rsi), %r9 - mov -30(%rsi), %r10 - mov -22(%rsi), %r11 - mov -14(%rsi), %rcx - mov -8(%rsi), %rdx - movdqu %xmm0, -62(%rdi) - mov %r8, -46(%rdi) - mov %r9, -38(%rdi) - mov %r10, -30(%rdi) - mov %r11, -22(%rdi) - mov %rcx, -14(%rdi) - mov %rdx, -8(%rdi) - ret + movdqu -62(%rsi), %xmm0 + mov -46(%rsi), %r8 + mov -38(%rsi), %r9 + mov -30(%rsi), %r10 + mov -22(%rsi), %r11 + mov -14(%rsi), %rcx + mov -8(%rsi), %rdx + movdqu %xmm0, -62(%rdi) + mov %r8, -46(%rdi) + mov %r9, -38(%rdi) + mov %r10, -30(%rdi) + mov %r11, -22(%rdi) + mov %rcx, -14(%rdi) + mov %rdx, -8(%rdi) + ret - .p2align 4 + .p2align 4 L(write_54bytes): - movdqu -54(%rsi), %xmm0 - mov -38(%rsi), %r9 - mov -30(%rsi), %r10 - mov -22(%rsi), %r11 - mov -14(%rsi), %rcx - mov -8(%rsi), %rdx - movdqu %xmm0, -54(%rdi) - mov %r9, -38(%rdi) - mov %r10, -30(%rdi) - mov %r11, -22(%rdi) - mov %rcx, -14(%rdi) - mov %rdx, -8(%rdi) - ret + movdqu -54(%rsi), %xmm0 + mov -38(%rsi), %r9 + mov -30(%rsi), %r10 + mov -22(%rsi), %r11 + mov -14(%rsi), %rcx + mov -8(%rsi), %rdx + movdqu %xmm0, -54(%rdi) + mov %r9, -38(%rdi) + mov %r10, -30(%rdi) + mov %r11, -22(%rdi) + mov %rcx, -14(%rdi) + mov %rdx, -8(%rdi) + ret - .p2align 4 + .p2align 4 L(write_46bytes): - mov -46(%rsi), %r8 - mov -38(%rsi), %r9 - mov -30(%rsi), %r10 - mov -22(%rsi), %r11 - mov -14(%rsi), %rcx - mov -8(%rsi), %rdx - mov %r8, -46(%rdi) - mov %r9, -38(%rdi) - mov %r10, -30(%rdi) - mov %r11, -22(%rdi) - mov %rcx, -14(%rdi) - mov %rdx, -8(%rdi) - ret + mov -46(%rsi), %r8 + mov -38(%rsi), %r9 + mov -30(%rsi), %r10 + mov -22(%rsi), %r11 + mov -14(%rsi), %rcx + mov -8(%rsi), %rdx + mov %r8, -46(%rdi) + mov %r9, -38(%rdi) + mov %r10, -30(%rdi) + mov %r11, -22(%rdi) + mov %rcx, -14(%rdi) + mov %rdx, -8(%rdi) + ret - .p2align 4 + .p2align 4 L(write_38bytes): - mov -38(%rsi), %r9 - mov -30(%rsi), %r10 - mov -22(%rsi), %r11 - mov -14(%rsi), %rcx - mov -8(%rsi), %rdx - mov %r9, -38(%rdi) - mov %r10, -30(%rdi) - mov %r11, -22(%rdi) - mov %rcx, -14(%rdi) - mov %rdx, -8(%rdi) - ret + mov -38(%rsi), %r9 + mov -30(%rsi), %r10 + mov -22(%rsi), %r11 + mov -14(%rsi), %rcx + mov -8(%rsi), %rdx + mov %r9, -38(%rdi) + mov %r10, -30(%rdi) + mov %r11, -22(%rdi) + mov %rcx, -14(%rdi) + mov %rdx, -8(%rdi) + ret - .p2align 4 + .p2align 4 L(write_30bytes): - mov -30(%rsi), %r10 - mov -22(%rsi), %r11 - mov -14(%rsi), %rcx - mov -8(%rsi), %rdx - mov %r10, -30(%rdi) - mov %r11, -22(%rdi) - mov %rcx, -14(%rdi) - mov %rdx, -8(%rdi) - ret + mov -30(%rsi), %r10 + mov -22(%rsi), %r11 + mov -14(%rsi), %rcx + mov -8(%rsi), %rdx + mov %r10, -30(%rdi) + mov %r11, -22(%rdi) + mov %rcx, -14(%rdi) + mov %rdx, -8(%rdi) + ret - .p2align 4 + .p2align 4 L(write_22bytes): - mov -22(%rsi), %r11 - mov -14(%rsi), %rcx - mov -8(%rsi), %rdx - mov %r11, -22(%rdi) - mov %rcx, -14(%rdi) - mov %rdx, -8(%rdi) - ret + mov -22(%rsi), %r11 + mov -14(%rsi), %rcx + mov -8(%rsi), %rdx + mov %r11, -22(%rdi) + mov %rcx, -14(%rdi) + mov %rdx, -8(%rdi) + ret - .p2align 4 + .p2align 4 L(write_14bytes): - mov -14(%rsi), %rcx - mov -8(%rsi), %rdx - mov %rcx, -14(%rdi) - mov %rdx, -8(%rdi) - ret + mov -14(%rsi), %rcx + mov -8(%rsi), %rdx + mov %rcx, -14(%rdi) + mov %rdx, -8(%rdi) + ret - .p2align 4 + .p2align 4 L(write_6bytes): - mov -6(%rsi), %edx - mov -4(%rsi), %ecx - mov %edx, -6(%rdi) - mov %ecx, -4(%rdi) - ret + mov -6(%rsi), %edx + mov -4(%rsi), %ecx + mov %edx, -6(%rdi) + mov %ecx, -4(%rdi) + ret - .p2align 4 + .p2align 4 L(write_79bytes): - movdqu -79(%rsi), %xmm0 - movdqu -63(%rsi), %xmm1 - mov -47(%rsi), %r8 - mov -39(%rsi), %r9 - mov -31(%rsi), %r10 - mov -23(%rsi), %r11 - mov -15(%rsi), %rcx - mov -8(%rsi), %rdx - movdqu %xmm0, -79(%rdi) - movdqu %xmm1, -63(%rdi) - mov %r8, -47(%rdi) - mov %r9, -39(%rdi) - mov %r10, -31(%rdi) - mov %r11, -23(%rdi) - mov %rcx, -15(%rdi) - mov %rdx, -8(%rdi) - ret + movdqu -79(%rsi), %xmm0 + movdqu -63(%rsi), %xmm1 + mov -47(%rsi), %r8 + mov -39(%rsi), %r9 + mov -31(%rsi), %r10 + mov -23(%rsi), %r11 + mov -15(%rsi), %rcx + mov -8(%rsi), %rdx + movdqu %xmm0, -79(%rdi) + movdqu %xmm1, -63(%rdi) + mov %r8, -47(%rdi) + mov %r9, -39(%rdi) + mov %r10, -31(%rdi) + mov %r11, -23(%rdi) + mov %rcx, -15(%rdi) + mov %rdx, -8(%rdi) + ret - .p2align 4 + .p2align 4 L(write_71bytes): - movdqu -71(%rsi), %xmm0 - movdqu -55(%rsi), %xmm1 - mov -39(%rsi), %r9 - mov -31(%rsi), %r10 - mov -23(%rsi), %r11 - mov -15(%rsi), %rcx - mov -8(%rsi), %rdx - movdqu %xmm0, -71(%rdi) - movdqu %xmm1, -55(%rdi) - mov %r9, -39(%rdi) - mov %r10, -31(%rdi) - mov %r11, -23(%rdi) - mov %rcx, -15(%rdi) - mov %rdx, -8(%rdi) - ret + movdqu -71(%rsi), %xmm0 + movdqu -55(%rsi), %xmm1 + mov -39(%rsi), %r9 + mov -31(%rsi), %r10 + mov -23(%rsi), %r11 + mov -15(%rsi), %rcx + mov -8(%rsi), %rdx + movdqu %xmm0, -71(%rdi) + movdqu %xmm1, -55(%rdi) + mov %r9, -39(%rdi) + mov %r10, -31(%rdi) + mov %r11, -23(%rdi) + mov %rcx, -15(%rdi) + mov %rdx, -8(%rdi) + ret - .p2align 4 + .p2align 4 L(write_63bytes): - movdqu -63(%rsi), %xmm0 - mov -47(%rsi), %r8 - mov -39(%rsi), %r9 - mov -31(%rsi), %r10 - mov -23(%rsi), %r11 - mov -15(%rsi), %rcx - mov -8(%rsi), %rdx - movdqu %xmm0, -63(%rdi) - mov %r8, -47(%rdi) - mov %r9, -39(%rdi) - mov %r10, -31(%rdi) - mov %r11, -23(%rdi) - mov %rcx, -15(%rdi) - mov %rdx, -8(%rdi) - ret + movdqu -63(%rsi), %xmm0 + mov -47(%rsi), %r8 + mov -39(%rsi), %r9 + mov -31(%rsi), %r10 + mov -23(%rsi), %r11 + mov -15(%rsi), %rcx + mov -8(%rsi), %rdx + movdqu %xmm0, -63(%rdi) + mov %r8, -47(%rdi) + mov %r9, -39(%rdi) + mov %r10, -31(%rdi) + mov %r11, -23(%rdi) + mov %rcx, -15(%rdi) + mov %rdx, -8(%rdi) + ret - .p2align 4 + .p2align 4 L(write_55bytes): - movdqu -55(%rsi), %xmm0 - mov -39(%rsi), %r9 - mov -31(%rsi), %r10 - mov -23(%rsi), %r11 - mov -15(%rsi), %rcx - mov -8(%rsi), %rdx - movdqu %xmm0, -55(%rdi) - mov %r9, -39(%rdi) - mov %r10, -31(%rdi) - mov %r11, -23(%rdi) - mov %rcx, -15(%rdi) - mov %rdx, -8(%rdi) - ret + movdqu -55(%rsi), %xmm0 + mov -39(%rsi), %r9 + mov -31(%rsi), %r10 + mov -23(%rsi), %r11 + mov -15(%rsi), %rcx + mov -8(%rsi), %rdx + movdqu %xmm0, -55(%rdi) + mov %r9, -39(%rdi) + mov %r10, -31(%rdi) + mov %r11, -23(%rdi) + mov %rcx, -15(%rdi) + mov %rdx, -8(%rdi) + ret - .p2align 4 + .p2align 4 L(write_47bytes): - mov -47(%rsi), %r8 - mov -39(%rsi), %r9 - mov -31(%rsi), %r10 - mov -23(%rsi), %r11 - mov -15(%rsi), %rcx - mov -8(%rsi), %rdx - mov %r8, -47(%rdi) - mov %r9, -39(%rdi) - mov %r10, -31(%rdi) - mov %r11, -23(%rdi) - mov %rcx, -15(%rdi) - mov %rdx, -8(%rdi) - ret + mov -47(%rsi), %r8 + mov -39(%rsi), %r9 + mov -31(%rsi), %r10 + mov -23(%rsi), %r11 + mov -15(%rsi), %rcx + mov -8(%rsi), %rdx + mov %r8, -47(%rdi) + mov %r9, -39(%rdi) + mov %r10, -31(%rdi) + mov %r11, -23(%rdi) + mov %rcx, -15(%rdi) + mov %rdx, -8(%rdi) + ret - .p2align 4 + .p2align 4 L(write_39bytes): - mov -39(%rsi), %r9 - mov -31(%rsi), %r10 - mov -23(%rsi), %r11 - mov -15(%rsi), %rcx - mov -8(%rsi), %rdx - mov %r9, -39(%rdi) - mov %r10, -31(%rdi) - mov %r11, -23(%rdi) - mov %rcx, -15(%rdi) - mov %rdx, -8(%rdi) - ret + mov -39(%rsi), %r9 + mov -31(%rsi), %r10 + mov -23(%rsi), %r11 + mov -15(%rsi), %rcx + mov -8(%rsi), %rdx + mov %r9, -39(%rdi) + mov %r10, -31(%rdi) + mov %r11, -23(%rdi) + mov %rcx, -15(%rdi) + mov %rdx, -8(%rdi) + ret - .p2align 4 + .p2align 4 L(write_31bytes): - mov -31(%rsi), %r10 - mov -23(%rsi), %r11 - mov -15(%rsi), %rcx - mov -8(%rsi), %rdx - mov %r10, -31(%rdi) - mov %r11, -23(%rdi) - mov %rcx, -15(%rdi) - mov %rdx, -8(%rdi) - ret + mov -31(%rsi), %r10 + mov -23(%rsi), %r11 + mov -15(%rsi), %rcx + mov -8(%rsi), %rdx + mov %r10, -31(%rdi) + mov %r11, -23(%rdi) + mov %rcx, -15(%rdi) + mov %rdx, -8(%rdi) + ret - .p2align 4 + .p2align 4 L(write_23bytes): - mov -23(%rsi), %r11 - mov -15(%rsi), %rcx - mov -8(%rsi), %rdx - mov %r11, -23(%rdi) - mov %rcx, -15(%rdi) - mov %rdx, -8(%rdi) - ret + mov -23(%rsi), %r11 + mov -15(%rsi), %rcx + mov -8(%rsi), %rdx + mov %r11, -23(%rdi) + mov %rcx, -15(%rdi) + mov %rdx, -8(%rdi) + ret - .p2align 4 + .p2align 4 L(write_15bytes): - mov -15(%rsi), %rcx - mov -8(%rsi), %rdx - mov %rcx, -15(%rdi) - mov %rdx, -8(%rdi) - ret + mov -15(%rsi), %rcx + mov -8(%rsi), %rdx + mov %rcx, -15(%rdi) + mov %rdx, -8(%rdi) + ret - .p2align 4 + .p2align 4 L(write_7bytes): - mov -7(%rsi), %edx - mov -4(%rsi), %ecx - mov %edx, -7(%rdi) - mov %ecx, -4(%rdi) - ret + mov -7(%rsi), %edx + mov -4(%rsi), %ecx + mov %edx, -7(%rdi) + mov %ecx, -4(%rdi) + ret - .p2align 4 + .p2align 4 L(large_page_fwd): - movdqu (%rsi), %xmm1 - lea 16(%rsi), %rsi - movdqu %xmm0, (%r8) - movntdq %xmm1, (%rdi) - lea 16(%rdi), %rdi - lea -0x90(%rdx), %rdx + movdqu (%rsi), %xmm1 + lea 16(%rsi), %rsi + movdqu %xmm0, (%r8) + movntdq %xmm1, (%rdi) + lea 16(%rdi), %rdi + lea -0x90(%rdx), %rdx #ifdef USE_AS_MEMMOVE - mov %rsi, %r9 - sub %rdi, %r9 - cmp %rdx, %r9 - jae L(memmove_is_memcpy_fwd) - shl $2, %rcx - cmp %rcx, %rdx - jb L(ll_cache_copy_fwd_start) + mov %rsi, %r9 + sub %rdi, %r9 + cmp %rdx, %r9 + jae L(memmove_is_memcpy_fwd) + shl $2, %rcx + cmp %rcx, %rdx + jb L(ll_cache_copy_fwd_start) L(memmove_is_memcpy_fwd): #endif L(large_page_loop): - movdqu (%rsi), %xmm0 - movdqu 0x10(%rsi), %xmm1 - movdqu 0x20(%rsi), %xmm2 - movdqu 0x30(%rsi), %xmm3 - movdqu 0x40(%rsi), %xmm4 - movdqu 0x50(%rsi), %xmm5 - movdqu 0x60(%rsi), %xmm6 - movdqu 0x70(%rsi), %xmm7 - lea 0x80(%rsi), %rsi + movdqu (%rsi), %xmm0 + movdqu 0x10(%rsi), %xmm1 + movdqu 0x20(%rsi), %xmm2 + movdqu 0x30(%rsi), %xmm3 + movdqu 0x40(%rsi), %xmm4 + movdqu 0x50(%rsi), %xmm5 + movdqu 0x60(%rsi), %xmm6 + movdqu 0x70(%rsi), %xmm7 + lea 0x80(%rsi), %rsi - sub $0x80, %rdx - movntdq %xmm0, (%rdi) - movntdq %xmm1, 0x10(%rdi) - movntdq %xmm2, 0x20(%rdi) - movntdq %xmm3, 0x30(%rdi) - movntdq %xmm4, 0x40(%rdi) - movntdq %xmm5, 0x50(%rdi) - movntdq %xmm6, 0x60(%rdi) - movntdq %xmm7, 0x70(%rdi) - lea 0x80(%rdi), %rdi - jae L(large_page_loop) - cmp $-0x40, %rdx - lea 0x80(%rdx), %rdx - jl L(large_page_less_64bytes) + sub $0x80, %rdx + movntdq %xmm0, (%rdi) + movntdq %xmm1, 0x10(%rdi) + movntdq %xmm2, 0x20(%rdi) + movntdq %xmm3, 0x30(%rdi) + movntdq %xmm4, 0x40(%rdi) + movntdq %xmm5, 0x50(%rdi) + movntdq %xmm6, 0x60(%rdi) + movntdq %xmm7, 0x70(%rdi) + lea 0x80(%rdi), %rdi + jae L(large_page_loop) + cmp $-0x40, %rdx + lea 0x80(%rdx), %rdx + jl L(large_page_less_64bytes) - movdqu (%rsi), %xmm0 - movdqu 0x10(%rsi), %xmm1 - movdqu 0x20(%rsi), %xmm2 - movdqu 0x30(%rsi), %xmm3 - lea 0x40(%rsi), %rsi + movdqu (%rsi), %xmm0 + movdqu 0x10(%rsi), %xmm1 + movdqu 0x20(%rsi), %xmm2 + movdqu 0x30(%rsi), %xmm3 + lea 0x40(%rsi), %rsi - movntdq %xmm0, (%rdi) - movntdq %xmm1, 0x10(%rdi) - movntdq %xmm2, 0x20(%rdi) - movntdq %xmm3, 0x30(%rdi) - lea 0x40(%rdi), %rdi - sub $0x40, %rdx + movntdq %xmm0, (%rdi) + movntdq %xmm1, 0x10(%rdi) + movntdq %xmm2, 0x20(%rdi) + movntdq %xmm3, 0x30(%rdi) + lea 0x40(%rdi), %rdi + sub $0x40, %rdx L(large_page_less_64bytes): - add %rdx, %rsi - add %rdx, %rdi - sfence - BRANCH_TO_JMPTBL_ENTRY (L(table_less_80bytes), %rdx, 4) + add %rdx, %rsi + add %rdx, %rdi + sfence + BRANCH_TO_JMPTBL_ENTRY (L(table_less_80bytes), %rdx, 4) #ifdef USE_AS_MEMMOVE - .p2align 4 + .p2align 4 L(ll_cache_copy_fwd_start): - prefetcht0 0x1c0(%rsi) - prefetcht0 0x200(%rsi) - movdqu (%rsi), %xmm0 - movdqu 0x10(%rsi), %xmm1 - movdqu 0x20(%rsi), %xmm2 - movdqu 0x30(%rsi), %xmm3 - movdqu 0x40(%rsi), %xmm4 - movdqu 0x50(%rsi), %xmm5 - movdqu 0x60(%rsi), %xmm6 - movdqu 0x70(%rsi), %xmm7 - lea 0x80(%rsi), %rsi + prefetcht0 0x1c0(%rsi) + prefetcht0 0x200(%rsi) + movdqu (%rsi), %xmm0 + movdqu 0x10(%rsi), %xmm1 + movdqu 0x20(%rsi), %xmm2 + movdqu 0x30(%rsi), %xmm3 + movdqu 0x40(%rsi), %xmm4 + movdqu 0x50(%rsi), %xmm5 + movdqu 0x60(%rsi), %xmm6 + movdqu 0x70(%rsi), %xmm7 + lea 0x80(%rsi), %rsi - sub $0x80, %rdx - movaps %xmm0, (%rdi) - movaps %xmm1, 0x10(%rdi) - movaps %xmm2, 0x20(%rdi) - movaps %xmm3, 0x30(%rdi) - movaps %xmm4, 0x40(%rdi) - movaps %xmm5, 0x50(%rdi) - movaps %xmm6, 0x60(%rdi) - movaps %xmm7, 0x70(%rdi) - lea 0x80(%rdi), %rdi - jae L(ll_cache_copy_fwd_start) - cmp $-0x40, %rdx - lea 0x80(%rdx), %rdx - jl L(large_page_ll_less_fwd_64bytes) + sub $0x80, %rdx + movaps %xmm0, (%rdi) + movaps %xmm1, 0x10(%rdi) + movaps %xmm2, 0x20(%rdi) + movaps %xmm3, 0x30(%rdi) + movaps %xmm4, 0x40(%rdi) + movaps %xmm5, 0x50(%rdi) + movaps %xmm6, 0x60(%rdi) + movaps %xmm7, 0x70(%rdi) + lea 0x80(%rdi), %rdi + jae L(ll_cache_copy_fwd_start) + cmp $-0x40, %rdx + lea 0x80(%rdx), %rdx + jl L(large_page_ll_less_fwd_64bytes) - movdqu (%rsi), %xmm0 - movdqu 0x10(%rsi), %xmm1 - movdqu 0x20(%rsi), %xmm2 - movdqu 0x30(%rsi), %xmm3 - lea 0x40(%rsi), %rsi + movdqu (%rsi), %xmm0 + movdqu 0x10(%rsi), %xmm1 + movdqu 0x20(%rsi), %xmm2 + movdqu 0x30(%rsi), %xmm3 + lea 0x40(%rsi), %rsi - movaps %xmm0, (%rdi) - movaps %xmm1, 0x10(%rdi) - movaps %xmm2, 0x20(%rdi) - movaps %xmm3, 0x30(%rdi) - lea 0x40(%rdi), %rdi - sub $0x40, %rdx + movaps %xmm0, (%rdi) + movaps %xmm1, 0x10(%rdi) + movaps %xmm2, 0x20(%rdi) + movaps %xmm3, 0x30(%rdi) + lea 0x40(%rdi), %rdi + sub $0x40, %rdx L(large_page_ll_less_fwd_64bytes): - add %rdx, %rsi - add %rdx, %rdi - BRANCH_TO_JMPTBL_ENTRY (L(table_less_80bytes), %rdx, 4) + add %rdx, %rsi + add %rdx, %rdi + BRANCH_TO_JMPTBL_ENTRY (L(table_less_80bytes), %rdx, 4) #endif - .p2align 4 + .p2align 4 L(large_page_bwd): - movdqu -0x10(%rsi), %xmm1 - lea -16(%rsi), %rsi - movdqu %xmm0, (%r8) - movdqa %xmm1, -0x10(%rdi) - lea -16(%rdi), %rdi - lea -0x90(%rdx), %rdx + movdqu -0x10(%rsi), %xmm1 + lea -16(%rsi), %rsi + movdqu %xmm0, (%r8) + movdqa %xmm1, -0x10(%rdi) + lea -16(%rdi), %rdi + lea -0x90(%rdx), %rdx #ifdef USE_AS_MEMMOVE - mov %rdi, %r9 - sub %rsi, %r9 - cmp %rdx, %r9 - jae L(memmove_is_memcpy_bwd) - cmp %rcx, %r9 - jb L(ll_cache_copy_bwd_start) + mov %rdi, %r9 + sub %rsi, %r9 + cmp %rdx, %r9 + jae L(memmove_is_memcpy_bwd) + cmp %rcx, %r9 + jb L(ll_cache_copy_bwd_start) L(memmove_is_memcpy_bwd): #endif L(large_page_bwd_loop): - movdqu -0x10(%rsi), %xmm0 - movdqu -0x20(%rsi), %xmm1 - movdqu -0x30(%rsi), %xmm2 - movdqu -0x40(%rsi), %xmm3 - movdqu -0x50(%rsi), %xmm4 - movdqu -0x60(%rsi), %xmm5 - movdqu -0x70(%rsi), %xmm6 - movdqu -0x80(%rsi), %xmm7 - lea -0x80(%rsi), %rsi + movdqu -0x10(%rsi), %xmm0 + movdqu -0x20(%rsi), %xmm1 + movdqu -0x30(%rsi), %xmm2 + movdqu -0x40(%rsi), %xmm3 + movdqu -0x50(%rsi), %xmm4 + movdqu -0x60(%rsi), %xmm5 + movdqu -0x70(%rsi), %xmm6 + movdqu -0x80(%rsi), %xmm7 + lea -0x80(%rsi), %rsi - sub $0x80, %rdx - movntdq %xmm0, -0x10(%rdi) - movntdq %xmm1, -0x20(%rdi) - movntdq %xmm2, -0x30(%rdi) - movntdq %xmm3, -0x40(%rdi) - movntdq %xmm4, -0x50(%rdi) - movntdq %xmm5, -0x60(%rdi) - movntdq %xmm6, -0x70(%rdi) - movntdq %xmm7, -0x80(%rdi) - lea -0x80(%rdi), %rdi - jae L(large_page_bwd_loop) - cmp $-0x40, %rdx - lea 0x80(%rdx), %rdx - jl L(large_page_less_bwd_64bytes) + sub $0x80, %rdx + movntdq %xmm0, -0x10(%rdi) + movntdq %xmm1, -0x20(%rdi) + movntdq %xmm2, -0x30(%rdi) + movntdq %xmm3, -0x40(%rdi) + movntdq %xmm4, -0x50(%rdi) + movntdq %xmm5, -0x60(%rdi) + movntdq %xmm6, -0x70(%rdi) + movntdq %xmm7, -0x80(%rdi) + lea -0x80(%rdi), %rdi + jae L(large_page_bwd_loop) + cmp $-0x40, %rdx + lea 0x80(%rdx), %rdx + jl L(large_page_less_bwd_64bytes) - movdqu -0x10(%rsi), %xmm0 - movdqu -0x20(%rsi), %xmm1 - movdqu -0x30(%rsi), %xmm2 - movdqu -0x40(%rsi), %xmm3 - lea -0x40(%rsi), %rsi + movdqu -0x10(%rsi), %xmm0 + movdqu -0x20(%rsi), %xmm1 + movdqu -0x30(%rsi), %xmm2 + movdqu -0x40(%rsi), %xmm3 + lea -0x40(%rsi), %rsi - movntdq %xmm0, -0x10(%rdi) - movntdq %xmm1, -0x20(%rdi) - movntdq %xmm2, -0x30(%rdi) - movntdq %xmm3, -0x40(%rdi) - lea -0x40(%rdi), %rdi - sub $0x40, %rdx + movntdq %xmm0, -0x10(%rdi) + movntdq %xmm1, -0x20(%rdi) + movntdq %xmm2, -0x30(%rdi) + movntdq %xmm3, -0x40(%rdi) + lea -0x40(%rdi), %rdi + sub $0x40, %rdx L(large_page_less_bwd_64bytes): - sfence - BRANCH_TO_JMPTBL_ENTRY (L(table_less_80bytes), %rdx, 4) + sfence + BRANCH_TO_JMPTBL_ENTRY (L(table_less_80bytes), %rdx, 4) #ifdef USE_AS_MEMMOVE - .p2align 4 + .p2align 4 L(ll_cache_copy_bwd_start): - prefetcht0 -0x1c0(%rsi) - prefetcht0 -0x200(%rsi) - movdqu -0x10(%rsi), %xmm0 - movdqu -0x20(%rsi), %xmm1 - movdqu -0x30(%rsi), %xmm2 - movdqu -0x40(%rsi), %xmm3 - movdqu -0x50(%rsi), %xmm4 - movdqu -0x60(%rsi), %xmm5 - movdqu -0x70(%rsi), %xmm6 - movdqu -0x80(%rsi), %xmm7 - lea -0x80(%rsi), %rsi + prefetcht0 -0x1c0(%rsi) + prefetcht0 -0x200(%rsi) + movdqu -0x10(%rsi), %xmm0 + movdqu -0x20(%rsi), %xmm1 + movdqu -0x30(%rsi), %xmm2 + movdqu -0x40(%rsi), %xmm3 + movdqu -0x50(%rsi), %xmm4 + movdqu -0x60(%rsi), %xmm5 + movdqu -0x70(%rsi), %xmm6 + movdqu -0x80(%rsi), %xmm7 + lea -0x80(%rsi), %rsi - sub $0x80, %rdx - movaps %xmm0, -0x10(%rdi) - movaps %xmm1, -0x20(%rdi) - movaps %xmm2, -0x30(%rdi) - movaps %xmm3, -0x40(%rdi) - movaps %xmm4, -0x50(%rdi) - movaps %xmm5, -0x60(%rdi) - movaps %xmm6, -0x70(%rdi) - movaps %xmm7, -0x80(%rdi) - lea -0x80(%rdi), %rdi - jae L(ll_cache_copy_bwd_start) - cmp $-0x40, %rdx - lea 0x80(%rdx), %rdx - jl L(large_page_ll_less_bwd_64bytes) + sub $0x80, %rdx + movaps %xmm0, -0x10(%rdi) + movaps %xmm1, -0x20(%rdi) + movaps %xmm2, -0x30(%rdi) + movaps %xmm3, -0x40(%rdi) + movaps %xmm4, -0x50(%rdi) + movaps %xmm5, -0x60(%rdi) + movaps %xmm6, -0x70(%rdi) + movaps %xmm7, -0x80(%rdi) + lea -0x80(%rdi), %rdi + jae L(ll_cache_copy_bwd_start) + cmp $-0x40, %rdx + lea 0x80(%rdx), %rdx + jl L(large_page_ll_less_bwd_64bytes) - movdqu -0x10(%rsi), %xmm0 - movdqu -0x20(%rsi), %xmm1 - movdqu -0x30(%rsi), %xmm2 - movdqu -0x40(%rsi), %xmm3 - lea -0x40(%rsi), %rsi + movdqu -0x10(%rsi), %xmm0 + movdqu -0x20(%rsi), %xmm1 + movdqu -0x30(%rsi), %xmm2 + movdqu -0x40(%rsi), %xmm3 + lea -0x40(%rsi), %rsi - movaps %xmm0, -0x10(%rdi) - movaps %xmm1, -0x20(%rdi) - movaps %xmm2, -0x30(%rdi) - movaps %xmm3, -0x40(%rdi) - lea -0x40(%rdi), %rdi - sub $0x40, %rdx + movaps %xmm0, -0x10(%rdi) + movaps %xmm1, -0x20(%rdi) + movaps %xmm2, -0x30(%rdi) + movaps %xmm3, -0x40(%rdi) + lea -0x40(%rdi), %rdi + sub $0x40, %rdx L(large_page_ll_less_bwd_64bytes): - BRANCH_TO_JMPTBL_ENTRY (L(table_less_80bytes), %rdx, 4) + BRANCH_TO_JMPTBL_ENTRY (L(table_less_80bytes), %rdx, 4) #endif END (MEMCPY) - .section .rodata.ssse3,"a",@progbits - .p2align 3 + .section .rodata.ssse3,"a",@progbits + .p2align 3 L(table_less_80bytes): - .int JMPTBL (L(write_0bytes), L(table_less_80bytes)) - .int JMPTBL (L(write_1bytes), L(table_less_80bytes)) - .int JMPTBL (L(write_2bytes), L(table_less_80bytes)) - .int JMPTBL (L(write_3bytes), L(table_less_80bytes)) - .int JMPTBL (L(write_4bytes), L(table_less_80bytes)) - .int JMPTBL (L(write_5bytes), L(table_less_80bytes)) - .int JMPTBL (L(write_6bytes), L(table_less_80bytes)) - .int JMPTBL (L(write_7bytes), L(table_less_80bytes)) - .int JMPTBL (L(write_8bytes), L(table_less_80bytes)) - .int JMPTBL (L(write_9bytes), L(table_less_80bytes)) - .int JMPTBL (L(write_10bytes), L(table_less_80bytes)) - .int JMPTBL (L(write_11bytes), L(table_less_80bytes)) - .int JMPTBL (L(write_12bytes), L(table_less_80bytes)) - .int JMPTBL (L(write_13bytes), L(table_less_80bytes)) - .int JMPTBL (L(write_14bytes), L(table_less_80bytes)) - .int JMPTBL (L(write_15bytes), L(table_less_80bytes)) - .int JMPTBL (L(write_16bytes), L(table_less_80bytes)) - .int JMPTBL (L(write_17bytes), L(table_less_80bytes)) - .int JMPTBL (L(write_18bytes), L(table_less_80bytes)) - .int JMPTBL (L(write_19bytes), L(table_less_80bytes)) - .int JMPTBL (L(write_20bytes), L(table_less_80bytes)) - .int JMPTBL (L(write_21bytes), L(table_less_80bytes)) - .int JMPTBL (L(write_22bytes), L(table_less_80bytes)) - .int JMPTBL (L(write_23bytes), L(table_less_80bytes)) - .int JMPTBL (L(write_24bytes), L(table_less_80bytes)) - .int JMPTBL (L(write_25bytes), L(table_less_80bytes)) - .int JMPTBL (L(write_26bytes), L(table_less_80bytes)) - .int JMPTBL (L(write_27bytes), L(table_less_80bytes)) - .int JMPTBL (L(write_28bytes), L(table_less_80bytes)) - .int JMPTBL (L(write_29bytes), L(table_less_80bytes)) - .int JMPTBL (L(write_30bytes), L(table_less_80bytes)) - .int JMPTBL (L(write_31bytes), L(table_less_80bytes)) - .int JMPTBL (L(write_32bytes), L(table_less_80bytes)) - .int JMPTBL (L(write_33bytes), L(table_less_80bytes)) - .int JMPTBL (L(write_34bytes), L(table_less_80bytes)) - .int JMPTBL (L(write_35bytes), L(table_less_80bytes)) - .int JMPTBL (L(write_36bytes), L(table_less_80bytes)) - .int JMPTBL (L(write_37bytes), L(table_less_80bytes)) - .int JMPTBL (L(write_38bytes), L(table_less_80bytes)) - .int JMPTBL (L(write_39bytes), L(table_less_80bytes)) - .int JMPTBL (L(write_40bytes), L(table_less_80bytes)) - .int JMPTBL (L(write_41bytes), L(table_less_80bytes)) - .int JMPTBL (L(write_42bytes), L(table_less_80bytes)) - .int JMPTBL (L(write_43bytes), L(table_less_80bytes)) - .int JMPTBL (L(write_44bytes), L(table_less_80bytes)) - .int JMPTBL (L(write_45bytes), L(table_less_80bytes)) - .int JMPTBL (L(write_46bytes), L(table_less_80bytes)) - .int JMPTBL (L(write_47bytes), L(table_less_80bytes)) - .int JMPTBL (L(write_48bytes), L(table_less_80bytes)) - .int JMPTBL (L(write_49bytes), L(table_less_80bytes)) - .int JMPTBL (L(write_50bytes), L(table_less_80bytes)) - .int JMPTBL (L(write_51bytes), L(table_less_80bytes)) - .int JMPTBL (L(write_52bytes), L(table_less_80bytes)) - .int JMPTBL (L(write_53bytes), L(table_less_80bytes)) - .int JMPTBL (L(write_54bytes), L(table_less_80bytes)) - .int JMPTBL (L(write_55bytes), L(table_less_80bytes)) - .int JMPTBL (L(write_56bytes), L(table_less_80bytes)) - .int JMPTBL (L(write_57bytes), L(table_less_80bytes)) - .int JMPTBL (L(write_58bytes), L(table_less_80bytes)) - .int JMPTBL (L(write_59bytes), L(table_less_80bytes)) - .int JMPTBL (L(write_60bytes), L(table_less_80bytes)) - .int JMPTBL (L(write_61bytes), L(table_less_80bytes)) - .int JMPTBL (L(write_62bytes), L(table_less_80bytes)) - .int JMPTBL (L(write_63bytes), L(table_less_80bytes)) - .int JMPTBL (L(write_64bytes), L(table_less_80bytes)) - .int JMPTBL (L(write_65bytes), L(table_less_80bytes)) - .int JMPTBL (L(write_66bytes), L(table_less_80bytes)) - .int JMPTBL (L(write_67bytes), L(table_less_80bytes)) - .int JMPTBL (L(write_68bytes), L(table_less_80bytes)) - .int JMPTBL (L(write_69bytes), L(table_less_80bytes)) - .int JMPTBL (L(write_70bytes), L(table_less_80bytes)) - .int JMPTBL (L(write_71bytes), L(table_less_80bytes)) - .int JMPTBL (L(write_72bytes), L(table_less_80bytes)) - .int JMPTBL (L(write_73bytes), L(table_less_80bytes)) - .int JMPTBL (L(write_74bytes), L(table_less_80bytes)) - .int JMPTBL (L(write_75bytes), L(table_less_80bytes)) - .int JMPTBL (L(write_76bytes), L(table_less_80bytes)) - .int JMPTBL (L(write_77bytes), L(table_less_80bytes)) - .int JMPTBL (L(write_78bytes), L(table_less_80bytes)) - .int JMPTBL (L(write_79bytes), L(table_less_80bytes)) + .int JMPTBL (L(write_0bytes), L(table_less_80bytes)) + .int JMPTBL (L(write_1bytes), L(table_less_80bytes)) + .int JMPTBL (L(write_2bytes), L(table_less_80bytes)) + .int JMPTBL (L(write_3bytes), L(table_less_80bytes)) + .int JMPTBL (L(write_4bytes), L(table_less_80bytes)) + .int JMPTBL (L(write_5bytes), L(table_less_80bytes)) + .int JMPTBL (L(write_6bytes), L(table_less_80bytes)) + .int JMPTBL (L(write_7bytes), L(table_less_80bytes)) + .int JMPTBL (L(write_8bytes), L(table_less_80bytes)) + .int JMPTBL (L(write_9bytes), L(table_less_80bytes)) + .int JMPTBL (L(write_10bytes), L(table_less_80bytes)) + .int JMPTBL (L(write_11bytes), L(table_less_80bytes)) + .int JMPTBL (L(write_12bytes), L(table_less_80bytes)) + .int JMPTBL (L(write_13bytes), L(table_less_80bytes)) + .int JMPTBL (L(write_14bytes), L(table_less_80bytes)) + .int JMPTBL (L(write_15bytes), L(table_less_80bytes)) + .int JMPTBL (L(write_16bytes), L(table_less_80bytes)) + .int JMPTBL (L(write_17bytes), L(table_less_80bytes)) + .int JMPTBL (L(write_18bytes), L(table_less_80bytes)) + .int JMPTBL (L(write_19bytes), L(table_less_80bytes)) + .int JMPTBL (L(write_20bytes), L(table_less_80bytes)) + .int JMPTBL (L(write_21bytes), L(table_less_80bytes)) + .int JMPTBL (L(write_22bytes), L(table_less_80bytes)) + .int JMPTBL (L(write_23bytes), L(table_less_80bytes)) + .int JMPTBL (L(write_24bytes), L(table_less_80bytes)) + .int JMPTBL (L(write_25bytes), L(table_less_80bytes)) + .int JMPTBL (L(write_26bytes), L(table_less_80bytes)) + .int JMPTBL (L(write_27bytes), L(table_less_80bytes)) + .int JMPTBL (L(write_28bytes), L(table_less_80bytes)) + .int JMPTBL (L(write_29bytes), L(table_less_80bytes)) + .int JMPTBL (L(write_30bytes), L(table_less_80bytes)) + .int JMPTBL (L(write_31bytes), L(table_less_80bytes)) + .int JMPTBL (L(write_32bytes), L(table_less_80bytes)) + .int JMPTBL (L(write_33bytes), L(table_less_80bytes)) + .int JMPTBL (L(write_34bytes), L(table_less_80bytes)) + .int JMPTBL (L(write_35bytes), L(table_less_80bytes)) + .int JMPTBL (L(write_36bytes), L(table_less_80bytes)) + .int JMPTBL (L(write_37bytes), L(table_less_80bytes)) + .int JMPTBL (L(write_38bytes), L(table_less_80bytes)) + .int JMPTBL (L(write_39bytes), L(table_less_80bytes)) + .int JMPTBL (L(write_40bytes), L(table_less_80bytes)) + .int JMPTBL (L(write_41bytes), L(table_less_80bytes)) + .int JMPTBL (L(write_42bytes), L(table_less_80bytes)) + .int JMPTBL (L(write_43bytes), L(table_less_80bytes)) + .int JMPTBL (L(write_44bytes), L(table_less_80bytes)) + .int JMPTBL (L(write_45bytes), L(table_less_80bytes)) + .int JMPTBL (L(write_46bytes), L(table_less_80bytes)) + .int JMPTBL (L(write_47bytes), L(table_less_80bytes)) + .int JMPTBL (L(write_48bytes), L(table_less_80bytes)) + .int JMPTBL (L(write_49bytes), L(table_less_80bytes)) + .int JMPTBL (L(write_50bytes), L(table_less_80bytes)) + .int JMPTBL (L(write_51bytes), L(table_less_80bytes)) + .int JMPTBL (L(write_52bytes), L(table_less_80bytes)) + .int JMPTBL (L(write_53bytes), L(table_less_80bytes)) + .int JMPTBL (L(write_54bytes), L(table_less_80bytes)) + .int JMPTBL (L(write_55bytes), L(table_less_80bytes)) + .int JMPTBL (L(write_56bytes), L(table_less_80bytes)) + .int JMPTBL (L(write_57bytes), L(table_less_80bytes)) + .int JMPTBL (L(write_58bytes), L(table_less_80bytes)) + .int JMPTBL (L(write_59bytes), L(table_less_80bytes)) + .int JMPTBL (L(write_60bytes), L(table_less_80bytes)) + .int JMPTBL (L(write_61bytes), L(table_less_80bytes)) + .int JMPTBL (L(write_62bytes), L(table_less_80bytes)) + .int JMPTBL (L(write_63bytes), L(table_less_80bytes)) + .int JMPTBL (L(write_64bytes), L(table_less_80bytes)) + .int JMPTBL (L(write_65bytes), L(table_less_80bytes)) + .int JMPTBL (L(write_66bytes), L(table_less_80bytes)) + .int JMPTBL (L(write_67bytes), L(table_less_80bytes)) + .int JMPTBL (L(write_68bytes), L(table_less_80bytes)) + .int JMPTBL (L(write_69bytes), L(table_less_80bytes)) + .int JMPTBL (L(write_70bytes), L(table_less_80bytes)) + .int JMPTBL (L(write_71bytes), L(table_less_80bytes)) + .int JMPTBL (L(write_72bytes), L(table_less_80bytes)) + .int JMPTBL (L(write_73bytes), L(table_less_80bytes)) + .int JMPTBL (L(write_74bytes), L(table_less_80bytes)) + .int JMPTBL (L(write_75bytes), L(table_less_80bytes)) + .int JMPTBL (L(write_76bytes), L(table_less_80bytes)) + .int JMPTBL (L(write_77bytes), L(table_less_80bytes)) + .int JMPTBL (L(write_78bytes), L(table_less_80bytes)) + .int JMPTBL (L(write_79bytes), L(table_less_80bytes)) - .p2align 3 + .p2align 3 L(shl_table): - .int JMPTBL (L(shl_0), L(shl_table)) - .int JMPTBL (L(shl_1), L(shl_table)) - .int JMPTBL (L(shl_2), L(shl_table)) - .int JMPTBL (L(shl_3), L(shl_table)) - .int JMPTBL (L(shl_4), L(shl_table)) - .int JMPTBL (L(shl_5), L(shl_table)) - .int JMPTBL (L(shl_6), L(shl_table)) - .int JMPTBL (L(shl_7), L(shl_table)) - .int JMPTBL (L(shl_8), L(shl_table)) - .int JMPTBL (L(shl_9), L(shl_table)) - .int JMPTBL (L(shl_10), L(shl_table)) - .int JMPTBL (L(shl_11), L(shl_table)) - .int JMPTBL (L(shl_12), L(shl_table)) - .int JMPTBL (L(shl_13), L(shl_table)) - .int JMPTBL (L(shl_14), L(shl_table)) - .int JMPTBL (L(shl_15), L(shl_table)) + .int JMPTBL (L(shl_0), L(shl_table)) + .int JMPTBL (L(shl_1), L(shl_table)) + .int JMPTBL (L(shl_2), L(shl_table)) + .int JMPTBL (L(shl_3), L(shl_table)) + .int JMPTBL (L(shl_4), L(shl_table)) + .int JMPTBL (L(shl_5), L(shl_table)) + .int JMPTBL (L(shl_6), L(shl_table)) + .int JMPTBL (L(shl_7), L(shl_table)) + .int JMPTBL (L(shl_8), L(shl_table)) + .int JMPTBL (L(shl_9), L(shl_table)) + .int JMPTBL (L(shl_10), L(shl_table)) + .int JMPTBL (L(shl_11), L(shl_table)) + .int JMPTBL (L(shl_12), L(shl_table)) + .int JMPTBL (L(shl_13), L(shl_table)) + .int JMPTBL (L(shl_14), L(shl_table)) + .int JMPTBL (L(shl_15), L(shl_table)) - .p2align 3 + .p2align 3 L(shl_table_bwd): - .int JMPTBL (L(shl_0_bwd), L(shl_table_bwd)) - .int JMPTBL (L(shl_1_bwd), L(shl_table_bwd)) - .int JMPTBL (L(shl_2_bwd), L(shl_table_bwd)) - .int JMPTBL (L(shl_3_bwd), L(shl_table_bwd)) - .int JMPTBL (L(shl_4_bwd), L(shl_table_bwd)) - .int JMPTBL (L(shl_5_bwd), L(shl_table_bwd)) - .int JMPTBL (L(shl_6_bwd), L(shl_table_bwd)) - .int JMPTBL (L(shl_7_bwd), L(shl_table_bwd)) - .int JMPTBL (L(shl_8_bwd), L(shl_table_bwd)) - .int JMPTBL (L(shl_9_bwd), L(shl_table_bwd)) - .int JMPTBL (L(shl_10_bwd), L(shl_table_bwd)) - .int JMPTBL (L(shl_11_bwd), L(shl_table_bwd)) - .int JMPTBL (L(shl_12_bwd), L(shl_table_bwd)) - .int JMPTBL (L(shl_13_bwd), L(shl_table_bwd)) - .int JMPTBL (L(shl_14_bwd), L(shl_table_bwd)) - .int JMPTBL (L(shl_15_bwd), L(shl_table_bwd)) + .int JMPTBL (L(shl_0_bwd), L(shl_table_bwd)) + .int JMPTBL (L(shl_1_bwd), L(shl_table_bwd)) + .int JMPTBL (L(shl_2_bwd), L(shl_table_bwd)) + .int JMPTBL (L(shl_3_bwd), L(shl_table_bwd)) + .int JMPTBL (L(shl_4_bwd), L(shl_table_bwd)) + .int JMPTBL (L(shl_5_bwd), L(shl_table_bwd)) + .int JMPTBL (L(shl_6_bwd), L(shl_table_bwd)) + .int JMPTBL (L(shl_7_bwd), L(shl_table_bwd)) + .int JMPTBL (L(shl_8_bwd), L(shl_table_bwd)) + .int JMPTBL (L(shl_9_bwd), L(shl_table_bwd)) + .int JMPTBL (L(shl_10_bwd), L(shl_table_bwd)) + .int JMPTBL (L(shl_11_bwd), L(shl_table_bwd)) + .int JMPTBL (L(shl_12_bwd), L(shl_table_bwd)) + .int JMPTBL (L(shl_13_bwd), L(shl_table_bwd)) + .int JMPTBL (L(shl_14_bwd), L(shl_table_bwd)) + .int JMPTBL (L(shl_15_bwd), L(shl_table_bwd)) #endif diff --git a/utils/memcpy-bench/glibc/memmove-avx-unaligned-erms.S b/utils/memcpy-bench/glibc/memmove-avx-unaligned-erms.S index 9ee6f0a71c3..2de73b29a85 100644 --- a/utils/memcpy-bench/glibc/memmove-avx-unaligned-erms.S +++ b/utils/memcpy-bench/glibc/memmove-avx-unaligned-erms.S @@ -1,12 +1,12 @@ #if 1 -# define VEC_SIZE 32 -# define VEC(i) ymm##i -# define VMOVNT vmovntdq -# define VMOVU vmovdqu -# define VMOVA vmovdqa +# define VEC_SIZE 32 +# define VEC(i) ymm##i +# define VMOVNT vmovntdq +# define VMOVU vmovdqu +# define VMOVA vmovdqa -# define SECTION(p) p##.avx -# define MEMMOVE_SYMBOL(p,s) p##_avx_##s +# define SECTION(p) p##.avx +# define MEMMOVE_SYMBOL(p,s) p##_avx_##s # include "memmove-vec-unaligned-erms.S" #endif diff --git a/utils/memcpy-bench/glibc/memmove-avx512-no-vzeroupper.S b/utils/memcpy-bench/glibc/memmove-avx512-no-vzeroupper.S index b14d92fd6a8..3effa845274 100644 --- a/utils/memcpy-bench/glibc/memmove-avx512-no-vzeroupper.S +++ b/utils/memcpy-bench/glibc/memmove-avx512-no-vzeroupper.S @@ -22,396 +22,396 @@ # include "asm-syntax.h" - .section .text.avx512,"ax",@progbits + .section .text.avx512,"ax",@progbits ENTRY (__mempcpy_chk_avx512_no_vzeroupper) - cmp %RDX_LP, %RCX_LP - jb HIDDEN_JUMPTARGET (__chk_fail) + cmp %RDX_LP, %RCX_LP + jb HIDDEN_JUMPTARGET (__chk_fail) END (__mempcpy_chk_avx512_no_vzeroupper) ENTRY (__mempcpy_avx512_no_vzeroupper) - mov %RDI_LP, %RAX_LP - add %RDX_LP, %RAX_LP - jmp L(start) + mov %RDI_LP, %RAX_LP + add %RDX_LP, %RAX_LP + jmp L(start) END (__mempcpy_avx512_no_vzeroupper) ENTRY (__memmove_chk_avx512_no_vzeroupper) - cmp %RDX_LP, %RCX_LP - jb HIDDEN_JUMPTARGET (__chk_fail) + cmp %RDX_LP, %RCX_LP + jb HIDDEN_JUMPTARGET (__chk_fail) END (__memmove_chk_avx512_no_vzeroupper) ENTRY (__memmove_avx512_no_vzeroupper) - mov %RDI_LP, %RAX_LP + mov %RDI_LP, %RAX_LP # ifdef USE_AS_MEMPCPY - add %RDX_LP, %RAX_LP + add %RDX_LP, %RAX_LP # endif L(start): # ifdef __ILP32__ - /* Clear the upper 32 bits. */ - mov %edx, %edx + /* Clear the upper 32 bits. */ + mov %edx, %edx # endif - lea (%rsi, %rdx), %rcx - lea (%rdi, %rdx), %r9 - cmp $512, %rdx - ja L(512bytesormore) + lea (%rsi, %rdx), %rcx + lea (%rdi, %rdx), %r9 + cmp $512, %rdx + ja L(512bytesormore) L(check): - cmp $16, %rdx - jbe L(less_16bytes) - cmp $256, %rdx - jb L(less_256bytes) - vmovups (%rsi), %zmm0 - vmovups 0x40(%rsi), %zmm1 - vmovups 0x80(%rsi), %zmm2 - vmovups 0xC0(%rsi), %zmm3 - vmovups -0x100(%rcx), %zmm4 - vmovups -0xC0(%rcx), %zmm5 - vmovups -0x80(%rcx), %zmm6 - vmovups -0x40(%rcx), %zmm7 - vmovups %zmm0, (%rdi) - vmovups %zmm1, 0x40(%rdi) - vmovups %zmm2, 0x80(%rdi) - vmovups %zmm3, 0xC0(%rdi) - vmovups %zmm4, -0x100(%r9) - vmovups %zmm5, -0xC0(%r9) - vmovups %zmm6, -0x80(%r9) - vmovups %zmm7, -0x40(%r9) - ret + cmp $16, %rdx + jbe L(less_16bytes) + cmp $256, %rdx + jb L(less_256bytes) + vmovups (%rsi), %zmm0 + vmovups 0x40(%rsi), %zmm1 + vmovups 0x80(%rsi), %zmm2 + vmovups 0xC0(%rsi), %zmm3 + vmovups -0x100(%rcx), %zmm4 + vmovups -0xC0(%rcx), %zmm5 + vmovups -0x80(%rcx), %zmm6 + vmovups -0x40(%rcx), %zmm7 + vmovups %zmm0, (%rdi) + vmovups %zmm1, 0x40(%rdi) + vmovups %zmm2, 0x80(%rdi) + vmovups %zmm3, 0xC0(%rdi) + vmovups %zmm4, -0x100(%r9) + vmovups %zmm5, -0xC0(%r9) + vmovups %zmm6, -0x80(%r9) + vmovups %zmm7, -0x40(%r9) + ret L(less_256bytes): - cmp $128, %dl - jb L(less_128bytes) - vmovups (%rsi), %zmm0 - vmovups 0x40(%rsi), %zmm1 - vmovups -0x80(%rcx), %zmm2 - vmovups -0x40(%rcx), %zmm3 - vmovups %zmm0, (%rdi) - vmovups %zmm1, 0x40(%rdi) - vmovups %zmm2, -0x80(%r9) - vmovups %zmm3, -0x40(%r9) - ret + cmp $128, %dl + jb L(less_128bytes) + vmovups (%rsi), %zmm0 + vmovups 0x40(%rsi), %zmm1 + vmovups -0x80(%rcx), %zmm2 + vmovups -0x40(%rcx), %zmm3 + vmovups %zmm0, (%rdi) + vmovups %zmm1, 0x40(%rdi) + vmovups %zmm2, -0x80(%r9) + vmovups %zmm3, -0x40(%r9) + ret L(less_128bytes): - cmp $64, %dl - jb L(less_64bytes) - vmovdqu (%rsi), %ymm0 - vmovdqu 0x20(%rsi), %ymm1 - vmovdqu -0x40(%rcx), %ymm2 - vmovdqu -0x20(%rcx), %ymm3 - vmovdqu %ymm0, (%rdi) - vmovdqu %ymm1, 0x20(%rdi) - vmovdqu %ymm2, -0x40(%r9) - vmovdqu %ymm3, -0x20(%r9) - ret + cmp $64, %dl + jb L(less_64bytes) + vmovdqu (%rsi), %ymm0 + vmovdqu 0x20(%rsi), %ymm1 + vmovdqu -0x40(%rcx), %ymm2 + vmovdqu -0x20(%rcx), %ymm3 + vmovdqu %ymm0, (%rdi) + vmovdqu %ymm1, 0x20(%rdi) + vmovdqu %ymm2, -0x40(%r9) + vmovdqu %ymm3, -0x20(%r9) + ret L(less_64bytes): - cmp $32, %dl - jb L(less_32bytes) - vmovdqu (%rsi), %ymm0 - vmovdqu -0x20(%rcx), %ymm1 - vmovdqu %ymm0, (%rdi) - vmovdqu %ymm1, -0x20(%r9) - ret + cmp $32, %dl + jb L(less_32bytes) + vmovdqu (%rsi), %ymm0 + vmovdqu -0x20(%rcx), %ymm1 + vmovdqu %ymm0, (%rdi) + vmovdqu %ymm1, -0x20(%r9) + ret L(less_32bytes): - vmovdqu (%rsi), %xmm0 - vmovdqu -0x10(%rcx), %xmm1 - vmovdqu %xmm0, (%rdi) - vmovdqu %xmm1, -0x10(%r9) - ret + vmovdqu (%rsi), %xmm0 + vmovdqu -0x10(%rcx), %xmm1 + vmovdqu %xmm0, (%rdi) + vmovdqu %xmm1, -0x10(%r9) + ret L(less_16bytes): - cmp $8, %dl - jb L(less_8bytes) - movq (%rsi), %rsi - movq -0x8(%rcx), %rcx - movq %rsi, (%rdi) - movq %rcx, -0x8(%r9) - ret + cmp $8, %dl + jb L(less_8bytes) + movq (%rsi), %rsi + movq -0x8(%rcx), %rcx + movq %rsi, (%rdi) + movq %rcx, -0x8(%r9) + ret L(less_8bytes): - cmp $4, %dl - jb L(less_4bytes) - mov (%rsi), %esi - mov -0x4(%rcx), %ecx - mov %esi, (%rdi) - mov %ecx, -0x4(%r9) - ret + cmp $4, %dl + jb L(less_4bytes) + mov (%rsi), %esi + mov -0x4(%rcx), %ecx + mov %esi, (%rdi) + mov %ecx, -0x4(%r9) + ret L(less_4bytes): - cmp $2, %dl - jb L(less_2bytes) - mov (%rsi), %si - mov -0x2(%rcx), %cx - mov %si, (%rdi) - mov %cx, -0x2(%r9) - ret + cmp $2, %dl + jb L(less_2bytes) + mov (%rsi), %si + mov -0x2(%rcx), %cx + mov %si, (%rdi) + mov %cx, -0x2(%r9) + ret L(less_2bytes): - cmp $1, %dl - jb L(less_1bytes) - mov (%rsi), %cl - mov %cl, (%rdi) + cmp $1, %dl + jb L(less_1bytes) + mov (%rsi), %cl + mov %cl, (%rdi) L(less_1bytes): - ret + ret L(512bytesormore): # ifdef SHARED_CACHE_SIZE_HALF - mov $SHARED_CACHE_SIZE_HALF, %r8 + mov $SHARED_CACHE_SIZE_HALF, %r8 # else - mov __x86_shared_cache_size_half(%rip), %r8 + mov __x86_shared_cache_size_half(%rip), %r8 # endif - cmp %r8, %rdx - jae L(preloop_large) - cmp $1024, %rdx - ja L(1024bytesormore) - prefetcht1 (%rsi) - prefetcht1 0x40(%rsi) - prefetcht1 0x80(%rsi) - prefetcht1 0xC0(%rsi) - prefetcht1 0x100(%rsi) - prefetcht1 0x140(%rsi) - prefetcht1 0x180(%rsi) - prefetcht1 0x1C0(%rsi) - prefetcht1 -0x200(%rcx) - prefetcht1 -0x1C0(%rcx) - prefetcht1 -0x180(%rcx) - prefetcht1 -0x140(%rcx) - prefetcht1 -0x100(%rcx) - prefetcht1 -0xC0(%rcx) - prefetcht1 -0x80(%rcx) - prefetcht1 -0x40(%rcx) - vmovups (%rsi), %zmm0 - vmovups 0x40(%rsi), %zmm1 - vmovups 0x80(%rsi), %zmm2 - vmovups 0xC0(%rsi), %zmm3 - vmovups 0x100(%rsi), %zmm4 - vmovups 0x140(%rsi), %zmm5 - vmovups 0x180(%rsi), %zmm6 - vmovups 0x1C0(%rsi), %zmm7 - vmovups -0x200(%rcx), %zmm8 - vmovups -0x1C0(%rcx), %zmm9 - vmovups -0x180(%rcx), %zmm10 - vmovups -0x140(%rcx), %zmm11 - vmovups -0x100(%rcx), %zmm12 - vmovups -0xC0(%rcx), %zmm13 - vmovups -0x80(%rcx), %zmm14 - vmovups -0x40(%rcx), %zmm15 - vmovups %zmm0, (%rdi) - vmovups %zmm1, 0x40(%rdi) - vmovups %zmm2, 0x80(%rdi) - vmovups %zmm3, 0xC0(%rdi) - vmovups %zmm4, 0x100(%rdi) - vmovups %zmm5, 0x140(%rdi) - vmovups %zmm6, 0x180(%rdi) - vmovups %zmm7, 0x1C0(%rdi) - vmovups %zmm8, -0x200(%r9) - vmovups %zmm9, -0x1C0(%r9) - vmovups %zmm10, -0x180(%r9) - vmovups %zmm11, -0x140(%r9) - vmovups %zmm12, -0x100(%r9) - vmovups %zmm13, -0xC0(%r9) - vmovups %zmm14, -0x80(%r9) - vmovups %zmm15, -0x40(%r9) - ret + cmp %r8, %rdx + jae L(preloop_large) + cmp $1024, %rdx + ja L(1024bytesormore) + prefetcht1 (%rsi) + prefetcht1 0x40(%rsi) + prefetcht1 0x80(%rsi) + prefetcht1 0xC0(%rsi) + prefetcht1 0x100(%rsi) + prefetcht1 0x140(%rsi) + prefetcht1 0x180(%rsi) + prefetcht1 0x1C0(%rsi) + prefetcht1 -0x200(%rcx) + prefetcht1 -0x1C0(%rcx) + prefetcht1 -0x180(%rcx) + prefetcht1 -0x140(%rcx) + prefetcht1 -0x100(%rcx) + prefetcht1 -0xC0(%rcx) + prefetcht1 -0x80(%rcx) + prefetcht1 -0x40(%rcx) + vmovups (%rsi), %zmm0 + vmovups 0x40(%rsi), %zmm1 + vmovups 0x80(%rsi), %zmm2 + vmovups 0xC0(%rsi), %zmm3 + vmovups 0x100(%rsi), %zmm4 + vmovups 0x140(%rsi), %zmm5 + vmovups 0x180(%rsi), %zmm6 + vmovups 0x1C0(%rsi), %zmm7 + vmovups -0x200(%rcx), %zmm8 + vmovups -0x1C0(%rcx), %zmm9 + vmovups -0x180(%rcx), %zmm10 + vmovups -0x140(%rcx), %zmm11 + vmovups -0x100(%rcx), %zmm12 + vmovups -0xC0(%rcx), %zmm13 + vmovups -0x80(%rcx), %zmm14 + vmovups -0x40(%rcx), %zmm15 + vmovups %zmm0, (%rdi) + vmovups %zmm1, 0x40(%rdi) + vmovups %zmm2, 0x80(%rdi) + vmovups %zmm3, 0xC0(%rdi) + vmovups %zmm4, 0x100(%rdi) + vmovups %zmm5, 0x140(%rdi) + vmovups %zmm6, 0x180(%rdi) + vmovups %zmm7, 0x1C0(%rdi) + vmovups %zmm8, -0x200(%r9) + vmovups %zmm9, -0x1C0(%r9) + vmovups %zmm10, -0x180(%r9) + vmovups %zmm11, -0x140(%r9) + vmovups %zmm12, -0x100(%r9) + vmovups %zmm13, -0xC0(%r9) + vmovups %zmm14, -0x80(%r9) + vmovups %zmm15, -0x40(%r9) + ret L(1024bytesormore): - cmp %rsi, %rdi - ja L(1024bytesormore_bkw) - sub $512, %r9 - vmovups -0x200(%rcx), %zmm8 - vmovups -0x1C0(%rcx), %zmm9 - vmovups -0x180(%rcx), %zmm10 - vmovups -0x140(%rcx), %zmm11 - vmovups -0x100(%rcx), %zmm12 - vmovups -0xC0(%rcx), %zmm13 - vmovups -0x80(%rcx), %zmm14 - vmovups -0x40(%rcx), %zmm15 - prefetcht1 (%rsi) - prefetcht1 0x40(%rsi) - prefetcht1 0x80(%rsi) - prefetcht1 0xC0(%rsi) - prefetcht1 0x100(%rsi) - prefetcht1 0x140(%rsi) - prefetcht1 0x180(%rsi) - prefetcht1 0x1C0(%rsi) + cmp %rsi, %rdi + ja L(1024bytesormore_bkw) + sub $512, %r9 + vmovups -0x200(%rcx), %zmm8 + vmovups -0x1C0(%rcx), %zmm9 + vmovups -0x180(%rcx), %zmm10 + vmovups -0x140(%rcx), %zmm11 + vmovups -0x100(%rcx), %zmm12 + vmovups -0xC0(%rcx), %zmm13 + vmovups -0x80(%rcx), %zmm14 + vmovups -0x40(%rcx), %zmm15 + prefetcht1 (%rsi) + prefetcht1 0x40(%rsi) + prefetcht1 0x80(%rsi) + prefetcht1 0xC0(%rsi) + prefetcht1 0x100(%rsi) + prefetcht1 0x140(%rsi) + prefetcht1 0x180(%rsi) + prefetcht1 0x1C0(%rsi) /* Loop with unaligned memory access. */ L(gobble_512bytes_loop): - vmovups (%rsi), %zmm0 - vmovups 0x40(%rsi), %zmm1 - vmovups 0x80(%rsi), %zmm2 - vmovups 0xC0(%rsi), %zmm3 - vmovups 0x100(%rsi), %zmm4 - vmovups 0x140(%rsi), %zmm5 - vmovups 0x180(%rsi), %zmm6 - vmovups 0x1C0(%rsi), %zmm7 - add $512, %rsi - prefetcht1 (%rsi) - prefetcht1 0x40(%rsi) - prefetcht1 0x80(%rsi) - prefetcht1 0xC0(%rsi) - prefetcht1 0x100(%rsi) - prefetcht1 0x140(%rsi) - prefetcht1 0x180(%rsi) - prefetcht1 0x1C0(%rsi) - vmovups %zmm0, (%rdi) - vmovups %zmm1, 0x40(%rdi) - vmovups %zmm2, 0x80(%rdi) - vmovups %zmm3, 0xC0(%rdi) - vmovups %zmm4, 0x100(%rdi) - vmovups %zmm5, 0x140(%rdi) - vmovups %zmm6, 0x180(%rdi) - vmovups %zmm7, 0x1C0(%rdi) - add $512, %rdi - cmp %r9, %rdi - jb L(gobble_512bytes_loop) - vmovups %zmm8, (%r9) - vmovups %zmm9, 0x40(%r9) - vmovups %zmm10, 0x80(%r9) - vmovups %zmm11, 0xC0(%r9) - vmovups %zmm12, 0x100(%r9) - vmovups %zmm13, 0x140(%r9) - vmovups %zmm14, 0x180(%r9) - vmovups %zmm15, 0x1C0(%r9) - ret + vmovups (%rsi), %zmm0 + vmovups 0x40(%rsi), %zmm1 + vmovups 0x80(%rsi), %zmm2 + vmovups 0xC0(%rsi), %zmm3 + vmovups 0x100(%rsi), %zmm4 + vmovups 0x140(%rsi), %zmm5 + vmovups 0x180(%rsi), %zmm6 + vmovups 0x1C0(%rsi), %zmm7 + add $512, %rsi + prefetcht1 (%rsi) + prefetcht1 0x40(%rsi) + prefetcht1 0x80(%rsi) + prefetcht1 0xC0(%rsi) + prefetcht1 0x100(%rsi) + prefetcht1 0x140(%rsi) + prefetcht1 0x180(%rsi) + prefetcht1 0x1C0(%rsi) + vmovups %zmm0, (%rdi) + vmovups %zmm1, 0x40(%rdi) + vmovups %zmm2, 0x80(%rdi) + vmovups %zmm3, 0xC0(%rdi) + vmovups %zmm4, 0x100(%rdi) + vmovups %zmm5, 0x140(%rdi) + vmovups %zmm6, 0x180(%rdi) + vmovups %zmm7, 0x1C0(%rdi) + add $512, %rdi + cmp %r9, %rdi + jb L(gobble_512bytes_loop) + vmovups %zmm8, (%r9) + vmovups %zmm9, 0x40(%r9) + vmovups %zmm10, 0x80(%r9) + vmovups %zmm11, 0xC0(%r9) + vmovups %zmm12, 0x100(%r9) + vmovups %zmm13, 0x140(%r9) + vmovups %zmm14, 0x180(%r9) + vmovups %zmm15, 0x1C0(%r9) + ret L(1024bytesormore_bkw): - add $512, %rdi - vmovups 0x1C0(%rsi), %zmm8 - vmovups 0x180(%rsi), %zmm9 - vmovups 0x140(%rsi), %zmm10 - vmovups 0x100(%rsi), %zmm11 - vmovups 0xC0(%rsi), %zmm12 - vmovups 0x80(%rsi), %zmm13 - vmovups 0x40(%rsi), %zmm14 - vmovups (%rsi), %zmm15 - prefetcht1 -0x40(%rcx) - prefetcht1 -0x80(%rcx) - prefetcht1 -0xC0(%rcx) - prefetcht1 -0x100(%rcx) - prefetcht1 -0x140(%rcx) - prefetcht1 -0x180(%rcx) - prefetcht1 -0x1C0(%rcx) - prefetcht1 -0x200(%rcx) + add $512, %rdi + vmovups 0x1C0(%rsi), %zmm8 + vmovups 0x180(%rsi), %zmm9 + vmovups 0x140(%rsi), %zmm10 + vmovups 0x100(%rsi), %zmm11 + vmovups 0xC0(%rsi), %zmm12 + vmovups 0x80(%rsi), %zmm13 + vmovups 0x40(%rsi), %zmm14 + vmovups (%rsi), %zmm15 + prefetcht1 -0x40(%rcx) + prefetcht1 -0x80(%rcx) + prefetcht1 -0xC0(%rcx) + prefetcht1 -0x100(%rcx) + prefetcht1 -0x140(%rcx) + prefetcht1 -0x180(%rcx) + prefetcht1 -0x1C0(%rcx) + prefetcht1 -0x200(%rcx) /* Backward loop with unaligned memory access. */ L(gobble_512bytes_loop_bkw): - vmovups -0x40(%rcx), %zmm0 - vmovups -0x80(%rcx), %zmm1 - vmovups -0xC0(%rcx), %zmm2 - vmovups -0x100(%rcx), %zmm3 - vmovups -0x140(%rcx), %zmm4 - vmovups -0x180(%rcx), %zmm5 - vmovups -0x1C0(%rcx), %zmm6 - vmovups -0x200(%rcx), %zmm7 - sub $512, %rcx - prefetcht1 -0x40(%rcx) - prefetcht1 -0x80(%rcx) - prefetcht1 -0xC0(%rcx) - prefetcht1 -0x100(%rcx) - prefetcht1 -0x140(%rcx) - prefetcht1 -0x180(%rcx) - prefetcht1 -0x1C0(%rcx) - prefetcht1 -0x200(%rcx) - vmovups %zmm0, -0x40(%r9) - vmovups %zmm1, -0x80(%r9) - vmovups %zmm2, -0xC0(%r9) - vmovups %zmm3, -0x100(%r9) - vmovups %zmm4, -0x140(%r9) - vmovups %zmm5, -0x180(%r9) - vmovups %zmm6, -0x1C0(%r9) - vmovups %zmm7, -0x200(%r9) - sub $512, %r9 - cmp %rdi, %r9 - ja L(gobble_512bytes_loop_bkw) - vmovups %zmm8, -0x40(%rdi) - vmovups %zmm9, -0x80(%rdi) - vmovups %zmm10, -0xC0(%rdi) - vmovups %zmm11, -0x100(%rdi) - vmovups %zmm12, -0x140(%rdi) - vmovups %zmm13, -0x180(%rdi) - vmovups %zmm14, -0x1C0(%rdi) - vmovups %zmm15, -0x200(%rdi) - ret + vmovups -0x40(%rcx), %zmm0 + vmovups -0x80(%rcx), %zmm1 + vmovups -0xC0(%rcx), %zmm2 + vmovups -0x100(%rcx), %zmm3 + vmovups -0x140(%rcx), %zmm4 + vmovups -0x180(%rcx), %zmm5 + vmovups -0x1C0(%rcx), %zmm6 + vmovups -0x200(%rcx), %zmm7 + sub $512, %rcx + prefetcht1 -0x40(%rcx) + prefetcht1 -0x80(%rcx) + prefetcht1 -0xC0(%rcx) + prefetcht1 -0x100(%rcx) + prefetcht1 -0x140(%rcx) + prefetcht1 -0x180(%rcx) + prefetcht1 -0x1C0(%rcx) + prefetcht1 -0x200(%rcx) + vmovups %zmm0, -0x40(%r9) + vmovups %zmm1, -0x80(%r9) + vmovups %zmm2, -0xC0(%r9) + vmovups %zmm3, -0x100(%r9) + vmovups %zmm4, -0x140(%r9) + vmovups %zmm5, -0x180(%r9) + vmovups %zmm6, -0x1C0(%r9) + vmovups %zmm7, -0x200(%r9) + sub $512, %r9 + cmp %rdi, %r9 + ja L(gobble_512bytes_loop_bkw) + vmovups %zmm8, -0x40(%rdi) + vmovups %zmm9, -0x80(%rdi) + vmovups %zmm10, -0xC0(%rdi) + vmovups %zmm11, -0x100(%rdi) + vmovups %zmm12, -0x140(%rdi) + vmovups %zmm13, -0x180(%rdi) + vmovups %zmm14, -0x1C0(%rdi) + vmovups %zmm15, -0x200(%rdi) + ret L(preloop_large): - cmp %rsi, %rdi - ja L(preloop_large_bkw) - vmovups (%rsi), %zmm4 - vmovups 0x40(%rsi), %zmm5 + cmp %rsi, %rdi + ja L(preloop_large_bkw) + vmovups (%rsi), %zmm4 + vmovups 0x40(%rsi), %zmm5 - mov %rdi, %r11 + mov %rdi, %r11 /* Align destination for access with non-temporal stores in the loop. */ - mov %rdi, %r8 - and $-0x80, %rdi - add $0x80, %rdi - sub %rdi, %r8 - sub %r8, %rsi - add %r8, %rdx + mov %rdi, %r8 + and $-0x80, %rdi + add $0x80, %rdi + sub %rdi, %r8 + sub %r8, %rsi + add %r8, %rdx L(gobble_256bytes_nt_loop): - prefetcht1 0x200(%rsi) - prefetcht1 0x240(%rsi) - prefetcht1 0x280(%rsi) - prefetcht1 0x2C0(%rsi) - prefetcht1 0x300(%rsi) - prefetcht1 0x340(%rsi) - prefetcht1 0x380(%rsi) - prefetcht1 0x3C0(%rsi) - vmovdqu64 (%rsi), %zmm0 - vmovdqu64 0x40(%rsi), %zmm1 - vmovdqu64 0x80(%rsi), %zmm2 - vmovdqu64 0xC0(%rsi), %zmm3 - vmovntdq %zmm0, (%rdi) - vmovntdq %zmm1, 0x40(%rdi) - vmovntdq %zmm2, 0x80(%rdi) - vmovntdq %zmm3, 0xC0(%rdi) - sub $256, %rdx - add $256, %rsi - add $256, %rdi - cmp $256, %rdx - ja L(gobble_256bytes_nt_loop) - sfence - vmovups %zmm4, (%r11) - vmovups %zmm5, 0x40(%r11) - jmp L(check) + prefetcht1 0x200(%rsi) + prefetcht1 0x240(%rsi) + prefetcht1 0x280(%rsi) + prefetcht1 0x2C0(%rsi) + prefetcht1 0x300(%rsi) + prefetcht1 0x340(%rsi) + prefetcht1 0x380(%rsi) + prefetcht1 0x3C0(%rsi) + vmovdqu64 (%rsi), %zmm0 + vmovdqu64 0x40(%rsi), %zmm1 + vmovdqu64 0x80(%rsi), %zmm2 + vmovdqu64 0xC0(%rsi), %zmm3 + vmovntdq %zmm0, (%rdi) + vmovntdq %zmm1, 0x40(%rdi) + vmovntdq %zmm2, 0x80(%rdi) + vmovntdq %zmm3, 0xC0(%rdi) + sub $256, %rdx + add $256, %rsi + add $256, %rdi + cmp $256, %rdx + ja L(gobble_256bytes_nt_loop) + sfence + vmovups %zmm4, (%r11) + vmovups %zmm5, 0x40(%r11) + jmp L(check) L(preloop_large_bkw): - vmovups -0x80(%rcx), %zmm4 - vmovups -0x40(%rcx), %zmm5 + vmovups -0x80(%rcx), %zmm4 + vmovups -0x40(%rcx), %zmm5 /* Align end of destination for access with non-temporal stores. */ - mov %r9, %r8 - and $-0x80, %r9 - sub %r9, %r8 - sub %r8, %rcx - sub %r8, %rdx - add %r9, %r8 + mov %r9, %r8 + and $-0x80, %r9 + sub %r9, %r8 + sub %r8, %rcx + sub %r8, %rdx + add %r9, %r8 L(gobble_256bytes_nt_loop_bkw): - prefetcht1 -0x400(%rcx) - prefetcht1 -0x3C0(%rcx) - prefetcht1 -0x380(%rcx) - prefetcht1 -0x340(%rcx) - prefetcht1 -0x300(%rcx) - prefetcht1 -0x2C0(%rcx) - prefetcht1 -0x280(%rcx) - prefetcht1 -0x240(%rcx) - vmovdqu64 -0x100(%rcx), %zmm0 - vmovdqu64 -0xC0(%rcx), %zmm1 - vmovdqu64 -0x80(%rcx), %zmm2 - vmovdqu64 -0x40(%rcx), %zmm3 - vmovntdq %zmm0, -0x100(%r9) - vmovntdq %zmm1, -0xC0(%r9) - vmovntdq %zmm2, -0x80(%r9) - vmovntdq %zmm3, -0x40(%r9) - sub $256, %rdx - sub $256, %rcx - sub $256, %r9 - cmp $256, %rdx - ja L(gobble_256bytes_nt_loop_bkw) - sfence - vmovups %zmm4, -0x80(%r8) - vmovups %zmm5, -0x40(%r8) - jmp L(check) + prefetcht1 -0x400(%rcx) + prefetcht1 -0x3C0(%rcx) + prefetcht1 -0x380(%rcx) + prefetcht1 -0x340(%rcx) + prefetcht1 -0x300(%rcx) + prefetcht1 -0x2C0(%rcx) + prefetcht1 -0x280(%rcx) + prefetcht1 -0x240(%rcx) + vmovdqu64 -0x100(%rcx), %zmm0 + vmovdqu64 -0xC0(%rcx), %zmm1 + vmovdqu64 -0x80(%rcx), %zmm2 + vmovdqu64 -0x40(%rcx), %zmm3 + vmovntdq %zmm0, -0x100(%r9) + vmovntdq %zmm1, -0xC0(%r9) + vmovntdq %zmm2, -0x80(%r9) + vmovntdq %zmm3, -0x40(%r9) + sub $256, %rdx + sub $256, %rcx + sub $256, %r9 + cmp $256, %rdx + ja L(gobble_256bytes_nt_loop_bkw) + sfence + vmovups %zmm4, -0x80(%r8) + vmovups %zmm5, -0x40(%r8) + jmp L(check) END (__memmove_avx512_no_vzeroupper) strong_alias (__memmove_avx512_no_vzeroupper, __memcpy_avx512_no_vzeroupper) diff --git a/utils/memcpy-bench/glibc/memmove-avx512-unaligned-erms.S b/utils/memcpy-bench/glibc/memmove-avx512-unaligned-erms.S index db70fdf1b4e..9666b05f1c5 100644 --- a/utils/memcpy-bench/glibc/memmove-avx512-unaligned-erms.S +++ b/utils/memcpy-bench/glibc/memmove-avx512-unaligned-erms.S @@ -1,12 +1,12 @@ #if 1 -# define VEC_SIZE 64 -# define VEC(i) zmm##i -# define VMOVNT vmovntdq -# define VMOVU vmovdqu64 -# define VMOVA vmovdqa64 +# define VEC_SIZE 64 +# define VEC(i) zmm##i +# define VMOVNT vmovntdq +# define VMOVU vmovdqu64 +# define VMOVA vmovdqa64 -# define SECTION(p) p##.avx512 -# define MEMMOVE_SYMBOL(p,s) p##_avx512_##s +# define SECTION(p) p##.avx512 +# define MEMMOVE_SYMBOL(p,s) p##_avx512_##s # include "memmove-vec-unaligned-erms.S" #endif diff --git a/utils/memcpy-bench/glibc/memmove-sse2-unaligned-erms.S b/utils/memcpy-bench/glibc/memmove-sse2-unaligned-erms.S index 17b4f861621..ad405be479e 100644 --- a/utils/memcpy-bench/glibc/memmove-sse2-unaligned-erms.S +++ b/utils/memcpy-bench/glibc/memmove-sse2-unaligned-erms.S @@ -17,7 +17,7 @@ . */ #if 1 -# define MEMMOVE_SYMBOL(p,s) p##_sse2_##s +# define MEMMOVE_SYMBOL(p,s) p##_sse2_##s #else weak_alias (__mempcpy, mempcpy) #endif diff --git a/utils/memcpy-bench/glibc/memmove-vec-unaligned-erms.S b/utils/memcpy-bench/glibc/memmove-vec-unaligned-erms.S index 21be351b4e7..097ff6ca617 100644 --- a/utils/memcpy-bench/glibc/memmove-vec-unaligned-erms.S +++ b/utils/memcpy-bench/glibc/memmove-vec-unaligned-erms.S @@ -37,15 +37,15 @@ #include "sysdep.h" #ifndef MEMCPY_SYMBOL -# define MEMCPY_SYMBOL(p,s) MEMMOVE_SYMBOL(p, s) +# define MEMCPY_SYMBOL(p,s) MEMMOVE_SYMBOL(p, s) #endif #ifndef MEMPCPY_SYMBOL -# define MEMPCPY_SYMBOL(p,s) MEMMOVE_SYMBOL(p, s) +# define MEMPCPY_SYMBOL(p,s) MEMMOVE_SYMBOL(p, s) #endif #ifndef MEMMOVE_CHK_SYMBOL -# define MEMMOVE_CHK_SYMBOL(p,s) MEMMOVE_SYMBOL(p, s) +# define MEMMOVE_CHK_SYMBOL(p,s) MEMMOVE_SYMBOL(p, s) #endif #ifndef VZEROUPPER @@ -70,17 +70,17 @@ #if PREFETCH_SIZE == 64 # if PREFETCHED_LOAD_SIZE == PREFETCH_SIZE # define PREFETCH_ONE_SET(dir, base, offset) \ - PREFETCH ((offset)base) + PREFETCH ((offset)base) # elif PREFETCHED_LOAD_SIZE == 2 * PREFETCH_SIZE # define PREFETCH_ONE_SET(dir, base, offset) \ - PREFETCH ((offset)base); \ - PREFETCH ((offset + dir * PREFETCH_SIZE)base) + PREFETCH ((offset)base); \ + PREFETCH ((offset + dir * PREFETCH_SIZE)base) # elif PREFETCHED_LOAD_SIZE == 4 * PREFETCH_SIZE # define PREFETCH_ONE_SET(dir, base, offset) \ - PREFETCH ((offset)base); \ - PREFETCH ((offset + dir * PREFETCH_SIZE)base); \ - PREFETCH ((offset + dir * PREFETCH_SIZE * 2)base); \ - PREFETCH ((offset + dir * PREFETCH_SIZE * 3)base) + PREFETCH ((offset)base); \ + PREFETCH ((offset + dir * PREFETCH_SIZE)base); \ + PREFETCH ((offset + dir * PREFETCH_SIZE * 2)base); \ + PREFETCH ((offset + dir * PREFETCH_SIZE * 3)base) # else # error Unsupported PREFETCHED_LOAD_SIZE! # endif @@ -92,100 +92,100 @@ # error SECTION is not defined! #endif - .section SECTION(.text),"ax",@progbits + .section SECTION(.text),"ax",@progbits #if defined SHARED ENTRY (MEMMOVE_CHK_SYMBOL (__mempcpy_chk, unaligned)) - cmp %RDX_LP, %RCX_LP - jb HIDDEN_JUMPTARGET (__chk_fail) + cmp %RDX_LP, %RCX_LP + jb HIDDEN_JUMPTARGET (__chk_fail) END (MEMMOVE_CHK_SYMBOL (__mempcpy_chk, unaligned)) #endif ENTRY (MEMPCPY_SYMBOL (__mempcpy, unaligned)) - mov %RDI_LP, %RAX_LP - add %RDX_LP, %RAX_LP - jmp L(start) + mov %RDI_LP, %RAX_LP + add %RDX_LP, %RAX_LP + jmp L(start) END (MEMPCPY_SYMBOL (__mempcpy, unaligned)) #if defined SHARED ENTRY (MEMMOVE_CHK_SYMBOL (__memmove_chk, unaligned)) - cmp %RDX_LP, %RCX_LP - jb HIDDEN_JUMPTARGET (__chk_fail) + cmp %RDX_LP, %RCX_LP + jb HIDDEN_JUMPTARGET (__chk_fail) END (MEMMOVE_CHK_SYMBOL (__memmove_chk, unaligned)) #endif ENTRY (MEMMOVE_SYMBOL (__memmove, unaligned)) - movq %rdi, %rax + movq %rdi, %rax L(start): # ifdef __ILP32__ - /* Clear the upper 32 bits. */ - movl %edx, %edx + /* Clear the upper 32 bits. */ + movl %edx, %edx # endif - cmp $VEC_SIZE, %RDX_LP - jb L(less_vec) - cmp $(VEC_SIZE * 2), %RDX_LP - ja L(more_2x_vec) + cmp $VEC_SIZE, %RDX_LP + jb L(less_vec) + cmp $(VEC_SIZE * 2), %RDX_LP + ja L(more_2x_vec) #if !defined USE_MULTIARCH L(last_2x_vec): #endif - /* From VEC and to 2 * VEC. No branch when size == VEC_SIZE. */ - VMOVU (%rsi), %VEC(0) - VMOVU -VEC_SIZE(%rsi,%rdx), %VEC(1) - VMOVU %VEC(0), (%rdi) - VMOVU %VEC(1), -VEC_SIZE(%rdi,%rdx) - VZEROUPPER + /* From VEC and to 2 * VEC. No branch when size == VEC_SIZE. */ + VMOVU (%rsi), %VEC(0) + VMOVU -VEC_SIZE(%rsi,%rdx), %VEC(1) + VMOVU %VEC(0), (%rdi) + VMOVU %VEC(1), -VEC_SIZE(%rdi,%rdx) + VZEROUPPER #if !defined USE_MULTIARCH L(nop): #endif - ret + ret #if defined USE_MULTIARCH END (MEMMOVE_SYMBOL (__memmove, unaligned)) # if VEC_SIZE == 16 ENTRY (__mempcpy_chk_erms) - cmp %RDX_LP, %RCX_LP - jb HIDDEN_JUMPTARGET (__chk_fail) + cmp %RDX_LP, %RCX_LP + jb HIDDEN_JUMPTARGET (__chk_fail) END (__mempcpy_chk_erms) /* Only used to measure performance of REP MOVSB. */ ENTRY (__mempcpy_erms) - mov %RDI_LP, %RAX_LP - /* Skip zero length. */ - test %RDX_LP, %RDX_LP - jz 2f - add %RDX_LP, %RAX_LP - jmp L(start_movsb) + mov %RDI_LP, %RAX_LP + /* Skip zero length. */ + test %RDX_LP, %RDX_LP + jz 2f + add %RDX_LP, %RAX_LP + jmp L(start_movsb) END (__mempcpy_erms) ENTRY (__memmove_chk_erms) - cmp %RDX_LP, %RCX_LP - jb HIDDEN_JUMPTARGET (__chk_fail) + cmp %RDX_LP, %RCX_LP + jb HIDDEN_JUMPTARGET (__chk_fail) END (__memmove_chk_erms) ENTRY (__memmove_erms) - movq %rdi, %rax - /* Skip zero length. */ - test %RDX_LP, %RDX_LP - jz 2f + movq %rdi, %rax + /* Skip zero length. */ + test %RDX_LP, %RDX_LP + jz 2f L(start_movsb): - mov %RDX_LP, %RCX_LP - cmp %RSI_LP, %RDI_LP - jb 1f - /* Source == destination is less common. */ - je 2f - lea (%rsi,%rcx), %RDX_LP - cmp %RDX_LP, %RDI_LP - jb L(movsb_backward) + mov %RDX_LP, %RCX_LP + cmp %RSI_LP, %RDI_LP + jb 1f + /* Source == destination is less common. */ + je 2f + lea (%rsi,%rcx), %RDX_LP + cmp %RDX_LP, %RDI_LP + jb L(movsb_backward) 1: - rep movsb + rep movsb 2: - ret + ret L(movsb_backward): - leaq -1(%rdi,%rcx), %rdi - leaq -1(%rsi,%rcx), %rsi - std - rep movsb - cld - ret + leaq -1(%rdi,%rcx), %rdi + leaq -1(%rsi,%rcx), %rsi + std + rep movsb + cld + ret END (__memmove_erms) strong_alias (__memmove_erms, __memcpy_erms) strong_alias (__memmove_chk_erms, __memcpy_chk_erms) @@ -193,367 +193,367 @@ strong_alias (__memmove_chk_erms, __memcpy_chk_erms) # ifdef SHARED ENTRY (MEMMOVE_CHK_SYMBOL (__mempcpy_chk, unaligned_erms)) - cmp %RDX_LP, %RCX_LP - jb HIDDEN_JUMPTARGET (__chk_fail) + cmp %RDX_LP, %RCX_LP + jb HIDDEN_JUMPTARGET (__chk_fail) END (MEMMOVE_CHK_SYMBOL (__mempcpy_chk, unaligned_erms)) # endif ENTRY (MEMMOVE_SYMBOL (__mempcpy, unaligned_erms)) - mov %RDI_LP, %RAX_LP - add %RDX_LP, %RAX_LP - jmp L(start_erms) + mov %RDI_LP, %RAX_LP + add %RDX_LP, %RAX_LP + jmp L(start_erms) END (MEMMOVE_SYMBOL (__mempcpy, unaligned_erms)) # ifdef SHARED ENTRY (MEMMOVE_CHK_SYMBOL (__memmove_chk, unaligned_erms)) - cmp %RDX_LP, %RCX_LP - jb HIDDEN_JUMPTARGET (__chk_fail) + cmp %RDX_LP, %RCX_LP + jb HIDDEN_JUMPTARGET (__chk_fail) END (MEMMOVE_CHK_SYMBOL (__memmove_chk, unaligned_erms)) # endif ENTRY (MEMMOVE_SYMBOL (__memmove, unaligned_erms)) - movq %rdi, %rax + movq %rdi, %rax L(start_erms): # ifdef __ILP32__ - /* Clear the upper 32 bits. */ - movl %edx, %edx + /* Clear the upper 32 bits. */ + movl %edx, %edx # endif - cmp $VEC_SIZE, %RDX_LP - jb L(less_vec) - cmp $(VEC_SIZE * 2), %RDX_LP - ja L(movsb_more_2x_vec) + cmp $VEC_SIZE, %RDX_LP + jb L(less_vec) + cmp $(VEC_SIZE * 2), %RDX_LP + ja L(movsb_more_2x_vec) L(last_2x_vec): - /* From VEC and to 2 * VEC. No branch when size == VEC_SIZE. */ - VMOVU (%rsi), %VEC(0) - VMOVU -VEC_SIZE(%rsi,%rdx), %VEC(1) - VMOVU %VEC(0), (%rdi) - VMOVU %VEC(1), -VEC_SIZE(%rdi,%rdx) + /* From VEC and to 2 * VEC. No branch when size == VEC_SIZE. */ + VMOVU (%rsi), %VEC(0) + VMOVU -VEC_SIZE(%rsi,%rdx), %VEC(1) + VMOVU %VEC(0), (%rdi) + VMOVU %VEC(1), -VEC_SIZE(%rdi,%rdx) L(return): - VZEROUPPER - ret + VZEROUPPER + ret L(movsb): - cmp $SHARED_NON_TEMPORAL_THRESHOLD, %RDX_LP - jae L(more_8x_vec) - cmpq %rsi, %rdi - jb 1f - /* Source == destination is less common. */ - je L(nop) - leaq (%rsi,%rdx), %r9 - cmpq %r9, %rdi - /* Avoid slow backward REP MOVSB. */ - jb L(more_8x_vec_backward) + cmp $SHARED_NON_TEMPORAL_THRESHOLD, %RDX_LP + jae L(more_8x_vec) + cmpq %rsi, %rdi + jb 1f + /* Source == destination is less common. */ + je L(nop) + leaq (%rsi,%rdx), %r9 + cmpq %r9, %rdi + /* Avoid slow backward REP MOVSB. */ + jb L(more_8x_vec_backward) 1: - mov %RDX_LP, %RCX_LP - rep movsb + mov %RDX_LP, %RCX_LP + rep movsb L(nop): - ret + ret #endif L(less_vec): - /* Less than 1 VEC. */ + /* Less than 1 VEC. */ #if VEC_SIZE != 16 && VEC_SIZE != 32 && VEC_SIZE != 64 # error Unsupported VEC_SIZE! #endif #if VEC_SIZE > 32 - cmpb $32, %dl - jae L(between_32_63) + cmpb $32, %dl + jae L(between_32_63) #endif #if VEC_SIZE > 16 - cmpb $16, %dl - jae L(between_16_31) + cmpb $16, %dl + jae L(between_16_31) #endif - cmpb $8, %dl - jae L(between_8_15) - cmpb $4, %dl - jae L(between_4_7) - cmpb $1, %dl - ja L(between_2_3) - jb 1f - movzbl (%rsi), %ecx - movb %cl, (%rdi) + cmpb $8, %dl + jae L(between_8_15) + cmpb $4, %dl + jae L(between_4_7) + cmpb $1, %dl + ja L(between_2_3) + jb 1f + movzbl (%rsi), %ecx + movb %cl, (%rdi) 1: - ret + ret #if VEC_SIZE > 32 L(between_32_63): - /* From 32 to 63. No branch when size == 32. */ - vmovdqu (%rsi), %ymm0 - vmovdqu -32(%rsi,%rdx), %ymm1 - vmovdqu %ymm0, (%rdi) - vmovdqu %ymm1, -32(%rdi,%rdx) - VZEROUPPER - ret + /* From 32 to 63. No branch when size == 32. */ + vmovdqu (%rsi), %ymm0 + vmovdqu -32(%rsi,%rdx), %ymm1 + vmovdqu %ymm0, (%rdi) + vmovdqu %ymm1, -32(%rdi,%rdx) + VZEROUPPER + ret #endif #if VEC_SIZE > 16 - /* From 16 to 31. No branch when size == 16. */ + /* From 16 to 31. No branch when size == 16. */ L(between_16_31): - vmovdqu (%rsi), %xmm0 - vmovdqu -16(%rsi,%rdx), %xmm1 - vmovdqu %xmm0, (%rdi) - vmovdqu %xmm1, -16(%rdi,%rdx) - ret + vmovdqu (%rsi), %xmm0 + vmovdqu -16(%rsi,%rdx), %xmm1 + vmovdqu %xmm0, (%rdi) + vmovdqu %xmm1, -16(%rdi,%rdx) + ret #endif L(between_8_15): - /* From 8 to 15. No branch when size == 8. */ - movq -8(%rsi,%rdx), %rcx - movq (%rsi), %rsi - movq %rcx, -8(%rdi,%rdx) - movq %rsi, (%rdi) - ret + /* From 8 to 15. No branch when size == 8. */ + movq -8(%rsi,%rdx), %rcx + movq (%rsi), %rsi + movq %rcx, -8(%rdi,%rdx) + movq %rsi, (%rdi) + ret L(between_4_7): - /* From 4 to 7. No branch when size == 4. */ - movl -4(%rsi,%rdx), %ecx - movl (%rsi), %esi - movl %ecx, -4(%rdi,%rdx) - movl %esi, (%rdi) - ret + /* From 4 to 7. No branch when size == 4. */ + movl -4(%rsi,%rdx), %ecx + movl (%rsi), %esi + movl %ecx, -4(%rdi,%rdx) + movl %esi, (%rdi) + ret L(between_2_3): - /* From 2 to 3. No branch when size == 2. */ - movzwl -2(%rsi,%rdx), %ecx - movzwl (%rsi), %esi - movw %cx, -2(%rdi,%rdx) - movw %si, (%rdi) - ret + /* From 2 to 3. No branch when size == 2. */ + movzwl -2(%rsi,%rdx), %ecx + movzwl (%rsi), %esi + movw %cx, -2(%rdi,%rdx) + movw %si, (%rdi) + ret #if defined USE_MULTIARCH L(movsb_more_2x_vec): - cmp $REP_MOSB_THRESHOLD, %RDX_LP - ja L(movsb) + cmp $REP_MOSB_THRESHOLD, %RDX_LP + ja L(movsb) #endif L(more_2x_vec): - /* More than 2 * VEC and there may be overlap between destination - and source. */ - cmpq $(VEC_SIZE * 8), %rdx - ja L(more_8x_vec) - cmpq $(VEC_SIZE * 4), %rdx - jb L(last_4x_vec) - /* Copy from 4 * VEC to 8 * VEC, inclusively. */ - VMOVU (%rsi), %VEC(0) - VMOVU VEC_SIZE(%rsi), %VEC(1) - VMOVU (VEC_SIZE * 2)(%rsi), %VEC(2) - VMOVU (VEC_SIZE * 3)(%rsi), %VEC(3) - VMOVU -VEC_SIZE(%rsi,%rdx), %VEC(4) - VMOVU -(VEC_SIZE * 2)(%rsi,%rdx), %VEC(5) - VMOVU -(VEC_SIZE * 3)(%rsi,%rdx), %VEC(6) - VMOVU -(VEC_SIZE * 4)(%rsi,%rdx), %VEC(7) - VMOVU %VEC(0), (%rdi) - VMOVU %VEC(1), VEC_SIZE(%rdi) - VMOVU %VEC(2), (VEC_SIZE * 2)(%rdi) - VMOVU %VEC(3), (VEC_SIZE * 3)(%rdi) - VMOVU %VEC(4), -VEC_SIZE(%rdi,%rdx) - VMOVU %VEC(5), -(VEC_SIZE * 2)(%rdi,%rdx) - VMOVU %VEC(6), -(VEC_SIZE * 3)(%rdi,%rdx) - VMOVU %VEC(7), -(VEC_SIZE * 4)(%rdi,%rdx) - VZEROUPPER - ret + /* More than 2 * VEC and there may be overlap between destination + and source. */ + cmpq $(VEC_SIZE * 8), %rdx + ja L(more_8x_vec) + cmpq $(VEC_SIZE * 4), %rdx + jb L(last_4x_vec) + /* Copy from 4 * VEC to 8 * VEC, inclusively. */ + VMOVU (%rsi), %VEC(0) + VMOVU VEC_SIZE(%rsi), %VEC(1) + VMOVU (VEC_SIZE * 2)(%rsi), %VEC(2) + VMOVU (VEC_SIZE * 3)(%rsi), %VEC(3) + VMOVU -VEC_SIZE(%rsi,%rdx), %VEC(4) + VMOVU -(VEC_SIZE * 2)(%rsi,%rdx), %VEC(5) + VMOVU -(VEC_SIZE * 3)(%rsi,%rdx), %VEC(6) + VMOVU -(VEC_SIZE * 4)(%rsi,%rdx), %VEC(7) + VMOVU %VEC(0), (%rdi) + VMOVU %VEC(1), VEC_SIZE(%rdi) + VMOVU %VEC(2), (VEC_SIZE * 2)(%rdi) + VMOVU %VEC(3), (VEC_SIZE * 3)(%rdi) + VMOVU %VEC(4), -VEC_SIZE(%rdi,%rdx) + VMOVU %VEC(5), -(VEC_SIZE * 2)(%rdi,%rdx) + VMOVU %VEC(6), -(VEC_SIZE * 3)(%rdi,%rdx) + VMOVU %VEC(7), -(VEC_SIZE * 4)(%rdi,%rdx) + VZEROUPPER + ret L(last_4x_vec): - /* Copy from 2 * VEC to 4 * VEC. */ - VMOVU (%rsi), %VEC(0) - VMOVU VEC_SIZE(%rsi), %VEC(1) - VMOVU -VEC_SIZE(%rsi,%rdx), %VEC(2) - VMOVU -(VEC_SIZE * 2)(%rsi,%rdx), %VEC(3) - VMOVU %VEC(0), (%rdi) - VMOVU %VEC(1), VEC_SIZE(%rdi) - VMOVU %VEC(2), -VEC_SIZE(%rdi,%rdx) - VMOVU %VEC(3), -(VEC_SIZE * 2)(%rdi,%rdx) - VZEROUPPER - ret + /* Copy from 2 * VEC to 4 * VEC. */ + VMOVU (%rsi), %VEC(0) + VMOVU VEC_SIZE(%rsi), %VEC(1) + VMOVU -VEC_SIZE(%rsi,%rdx), %VEC(2) + VMOVU -(VEC_SIZE * 2)(%rsi,%rdx), %VEC(3) + VMOVU %VEC(0), (%rdi) + VMOVU %VEC(1), VEC_SIZE(%rdi) + VMOVU %VEC(2), -VEC_SIZE(%rdi,%rdx) + VMOVU %VEC(3), -(VEC_SIZE * 2)(%rdi,%rdx) + VZEROUPPER + ret L(more_8x_vec): - cmpq %rsi, %rdi - ja L(more_8x_vec_backward) - /* Source == destination is less common. */ - je L(nop) - /* Load the first VEC and last 4 * VEC to support overlapping - addresses. */ - VMOVU (%rsi), %VEC(4) - VMOVU -VEC_SIZE(%rsi, %rdx), %VEC(5) - VMOVU -(VEC_SIZE * 2)(%rsi, %rdx), %VEC(6) - VMOVU -(VEC_SIZE * 3)(%rsi, %rdx), %VEC(7) - VMOVU -(VEC_SIZE * 4)(%rsi, %rdx), %VEC(8) - /* Save start and stop of the destination buffer. */ - movq %rdi, %r11 - leaq -VEC_SIZE(%rdi, %rdx), %rcx - /* Align destination for aligned stores in the loop. Compute - how much destination is misaligned. */ - movq %rdi, %r8 - andq $(VEC_SIZE - 1), %r8 - /* Get the negative of offset for alignment. */ - subq $VEC_SIZE, %r8 - /* Adjust source. */ - subq %r8, %rsi - /* Adjust destination which should be aligned now. */ - subq %r8, %rdi - /* Adjust length. */ - addq %r8, %rdx + cmpq %rsi, %rdi + ja L(more_8x_vec_backward) + /* Source == destination is less common. */ + je L(nop) + /* Load the first VEC and last 4 * VEC to support overlapping + addresses. */ + VMOVU (%rsi), %VEC(4) + VMOVU -VEC_SIZE(%rsi, %rdx), %VEC(5) + VMOVU -(VEC_SIZE * 2)(%rsi, %rdx), %VEC(6) + VMOVU -(VEC_SIZE * 3)(%rsi, %rdx), %VEC(7) + VMOVU -(VEC_SIZE * 4)(%rsi, %rdx), %VEC(8) + /* Save start and stop of the destination buffer. */ + movq %rdi, %r11 + leaq -VEC_SIZE(%rdi, %rdx), %rcx + /* Align destination for aligned stores in the loop. Compute + how much destination is misaligned. */ + movq %rdi, %r8 + andq $(VEC_SIZE - 1), %r8 + /* Get the negative of offset for alignment. */ + subq $VEC_SIZE, %r8 + /* Adjust source. */ + subq %r8, %rsi + /* Adjust destination which should be aligned now. */ + subq %r8, %rdi + /* Adjust length. */ + addq %r8, %rdx #if (defined USE_MULTIARCH || VEC_SIZE == 16) - /* Check non-temporal store threshold. */ - cmp $SHARED_NON_TEMPORAL_THRESHOLD, %RDX_LP - ja L(large_forward) + /* Check non-temporal store threshold. */ + cmp $SHARED_NON_TEMPORAL_THRESHOLD, %RDX_LP + ja L(large_forward) #endif L(loop_4x_vec_forward): - /* Copy 4 * VEC a time forward. */ - VMOVU (%rsi), %VEC(0) - VMOVU VEC_SIZE(%rsi), %VEC(1) - VMOVU (VEC_SIZE * 2)(%rsi), %VEC(2) - VMOVU (VEC_SIZE * 3)(%rsi), %VEC(3) - addq $(VEC_SIZE * 4), %rsi - subq $(VEC_SIZE * 4), %rdx - VMOVA %VEC(0), (%rdi) - VMOVA %VEC(1), VEC_SIZE(%rdi) - VMOVA %VEC(2), (VEC_SIZE * 2)(%rdi) - VMOVA %VEC(3), (VEC_SIZE * 3)(%rdi) - addq $(VEC_SIZE * 4), %rdi - cmpq $(VEC_SIZE * 4), %rdx - ja L(loop_4x_vec_forward) - /* Store the last 4 * VEC. */ - VMOVU %VEC(5), (%rcx) - VMOVU %VEC(6), -VEC_SIZE(%rcx) - VMOVU %VEC(7), -(VEC_SIZE * 2)(%rcx) - VMOVU %VEC(8), -(VEC_SIZE * 3)(%rcx) - /* Store the first VEC. */ - VMOVU %VEC(4), (%r11) - VZEROUPPER - ret + /* Copy 4 * VEC a time forward. */ + VMOVU (%rsi), %VEC(0) + VMOVU VEC_SIZE(%rsi), %VEC(1) + VMOVU (VEC_SIZE * 2)(%rsi), %VEC(2) + VMOVU (VEC_SIZE * 3)(%rsi), %VEC(3) + addq $(VEC_SIZE * 4), %rsi + subq $(VEC_SIZE * 4), %rdx + VMOVA %VEC(0), (%rdi) + VMOVA %VEC(1), VEC_SIZE(%rdi) + VMOVA %VEC(2), (VEC_SIZE * 2)(%rdi) + VMOVA %VEC(3), (VEC_SIZE * 3)(%rdi) + addq $(VEC_SIZE * 4), %rdi + cmpq $(VEC_SIZE * 4), %rdx + ja L(loop_4x_vec_forward) + /* Store the last 4 * VEC. */ + VMOVU %VEC(5), (%rcx) + VMOVU %VEC(6), -VEC_SIZE(%rcx) + VMOVU %VEC(7), -(VEC_SIZE * 2)(%rcx) + VMOVU %VEC(8), -(VEC_SIZE * 3)(%rcx) + /* Store the first VEC. */ + VMOVU %VEC(4), (%r11) + VZEROUPPER + ret L(more_8x_vec_backward): - /* Load the first 4 * VEC and last VEC to support overlapping - addresses. */ - VMOVU (%rsi), %VEC(4) - VMOVU VEC_SIZE(%rsi), %VEC(5) - VMOVU (VEC_SIZE * 2)(%rsi), %VEC(6) - VMOVU (VEC_SIZE * 3)(%rsi), %VEC(7) - VMOVU -VEC_SIZE(%rsi,%rdx), %VEC(8) - /* Save stop of the destination buffer. */ - leaq -VEC_SIZE(%rdi, %rdx), %r11 - /* Align destination end for aligned stores in the loop. Compute - how much destination end is misaligned. */ - leaq -VEC_SIZE(%rsi, %rdx), %rcx - movq %r11, %r9 - movq %r11, %r8 - andq $(VEC_SIZE - 1), %r8 - /* Adjust source. */ - subq %r8, %rcx - /* Adjust the end of destination which should be aligned now. */ - subq %r8, %r9 - /* Adjust length. */ - subq %r8, %rdx + /* Load the first 4 * VEC and last VEC to support overlapping + addresses. */ + VMOVU (%rsi), %VEC(4) + VMOVU VEC_SIZE(%rsi), %VEC(5) + VMOVU (VEC_SIZE * 2)(%rsi), %VEC(6) + VMOVU (VEC_SIZE * 3)(%rsi), %VEC(7) + VMOVU -VEC_SIZE(%rsi,%rdx), %VEC(8) + /* Save stop of the destination buffer. */ + leaq -VEC_SIZE(%rdi, %rdx), %r11 + /* Align destination end for aligned stores in the loop. Compute + how much destination end is misaligned. */ + leaq -VEC_SIZE(%rsi, %rdx), %rcx + movq %r11, %r9 + movq %r11, %r8 + andq $(VEC_SIZE - 1), %r8 + /* Adjust source. */ + subq %r8, %rcx + /* Adjust the end of destination which should be aligned now. */ + subq %r8, %r9 + /* Adjust length. */ + subq %r8, %rdx #if (defined USE_MULTIARCH || VEC_SIZE == 16) - /* Check non-temporal store threshold. */ - cmp $SHARED_NON_TEMPORAL_THRESHOLD, %RDX_LP - ja L(large_backward) + /* Check non-temporal store threshold. */ + cmp $SHARED_NON_TEMPORAL_THRESHOLD, %RDX_LP + ja L(large_backward) #endif L(loop_4x_vec_backward): - /* Copy 4 * VEC a time backward. */ - VMOVU (%rcx), %VEC(0) - VMOVU -VEC_SIZE(%rcx), %VEC(1) - VMOVU -(VEC_SIZE * 2)(%rcx), %VEC(2) - VMOVU -(VEC_SIZE * 3)(%rcx), %VEC(3) - subq $(VEC_SIZE * 4), %rcx - subq $(VEC_SIZE * 4), %rdx - VMOVA %VEC(0), (%r9) - VMOVA %VEC(1), -VEC_SIZE(%r9) - VMOVA %VEC(2), -(VEC_SIZE * 2)(%r9) - VMOVA %VEC(3), -(VEC_SIZE * 3)(%r9) - subq $(VEC_SIZE * 4), %r9 - cmpq $(VEC_SIZE * 4), %rdx - ja L(loop_4x_vec_backward) - /* Store the first 4 * VEC. */ - VMOVU %VEC(4), (%rdi) - VMOVU %VEC(5), VEC_SIZE(%rdi) - VMOVU %VEC(6), (VEC_SIZE * 2)(%rdi) - VMOVU %VEC(7), (VEC_SIZE * 3)(%rdi) - /* Store the last VEC. */ - VMOVU %VEC(8), (%r11) - VZEROUPPER - ret + /* Copy 4 * VEC a time backward. */ + VMOVU (%rcx), %VEC(0) + VMOVU -VEC_SIZE(%rcx), %VEC(1) + VMOVU -(VEC_SIZE * 2)(%rcx), %VEC(2) + VMOVU -(VEC_SIZE * 3)(%rcx), %VEC(3) + subq $(VEC_SIZE * 4), %rcx + subq $(VEC_SIZE * 4), %rdx + VMOVA %VEC(0), (%r9) + VMOVA %VEC(1), -VEC_SIZE(%r9) + VMOVA %VEC(2), -(VEC_SIZE * 2)(%r9) + VMOVA %VEC(3), -(VEC_SIZE * 3)(%r9) + subq $(VEC_SIZE * 4), %r9 + cmpq $(VEC_SIZE * 4), %rdx + ja L(loop_4x_vec_backward) + /* Store the first 4 * VEC. */ + VMOVU %VEC(4), (%rdi) + VMOVU %VEC(5), VEC_SIZE(%rdi) + VMOVU %VEC(6), (VEC_SIZE * 2)(%rdi) + VMOVU %VEC(7), (VEC_SIZE * 3)(%rdi) + /* Store the last VEC. */ + VMOVU %VEC(8), (%r11) + VZEROUPPER + ret #if (defined USE_MULTIARCH || VEC_SIZE == 16) L(large_forward): - /* Don't use non-temporal store if there is overlap between - destination and source since destination may be in cache - when source is loaded. */ - leaq (%rdi, %rdx), %r10 - cmpq %r10, %rsi - jb L(loop_4x_vec_forward) + /* Don't use non-temporal store if there is overlap between + destination and source since destination may be in cache + when source is loaded. */ + leaq (%rdi, %rdx), %r10 + cmpq %r10, %rsi + jb L(loop_4x_vec_forward) L(loop_large_forward): - /* Copy 4 * VEC a time forward with non-temporal stores. */ - PREFETCH_ONE_SET (1, (%rsi), PREFETCHED_LOAD_SIZE * 2) - PREFETCH_ONE_SET (1, (%rsi), PREFETCHED_LOAD_SIZE * 3) - VMOVU (%rsi), %VEC(0) - VMOVU VEC_SIZE(%rsi), %VEC(1) - VMOVU (VEC_SIZE * 2)(%rsi), %VEC(2) - VMOVU (VEC_SIZE * 3)(%rsi), %VEC(3) - addq $PREFETCHED_LOAD_SIZE, %rsi - subq $PREFETCHED_LOAD_SIZE, %rdx - VMOVNT %VEC(0), (%rdi) - VMOVNT %VEC(1), VEC_SIZE(%rdi) - VMOVNT %VEC(2), (VEC_SIZE * 2)(%rdi) - VMOVNT %VEC(3), (VEC_SIZE * 3)(%rdi) - addq $PREFETCHED_LOAD_SIZE, %rdi - cmpq $PREFETCHED_LOAD_SIZE, %rdx - ja L(loop_large_forward) - sfence - /* Store the last 4 * VEC. */ - VMOVU %VEC(5), (%rcx) - VMOVU %VEC(6), -VEC_SIZE(%rcx) - VMOVU %VEC(7), -(VEC_SIZE * 2)(%rcx) - VMOVU %VEC(8), -(VEC_SIZE * 3)(%rcx) - /* Store the first VEC. */ - VMOVU %VEC(4), (%r11) - VZEROUPPER - ret + /* Copy 4 * VEC a time forward with non-temporal stores. */ + PREFETCH_ONE_SET (1, (%rsi), PREFETCHED_LOAD_SIZE * 2) + PREFETCH_ONE_SET (1, (%rsi), PREFETCHED_LOAD_SIZE * 3) + VMOVU (%rsi), %VEC(0) + VMOVU VEC_SIZE(%rsi), %VEC(1) + VMOVU (VEC_SIZE * 2)(%rsi), %VEC(2) + VMOVU (VEC_SIZE * 3)(%rsi), %VEC(3) + addq $PREFETCHED_LOAD_SIZE, %rsi + subq $PREFETCHED_LOAD_SIZE, %rdx + VMOVNT %VEC(0), (%rdi) + VMOVNT %VEC(1), VEC_SIZE(%rdi) + VMOVNT %VEC(2), (VEC_SIZE * 2)(%rdi) + VMOVNT %VEC(3), (VEC_SIZE * 3)(%rdi) + addq $PREFETCHED_LOAD_SIZE, %rdi + cmpq $PREFETCHED_LOAD_SIZE, %rdx + ja L(loop_large_forward) + sfence + /* Store the last 4 * VEC. */ + VMOVU %VEC(5), (%rcx) + VMOVU %VEC(6), -VEC_SIZE(%rcx) + VMOVU %VEC(7), -(VEC_SIZE * 2)(%rcx) + VMOVU %VEC(8), -(VEC_SIZE * 3)(%rcx) + /* Store the first VEC. */ + VMOVU %VEC(4), (%r11) + VZEROUPPER + ret L(large_backward): - /* Don't use non-temporal store if there is overlap between - destination and source since destination may be in cache - when source is loaded. */ - leaq (%rcx, %rdx), %r10 - cmpq %r10, %r9 - jb L(loop_4x_vec_backward) + /* Don't use non-temporal store if there is overlap between + destination and source since destination may be in cache + when source is loaded. */ + leaq (%rcx, %rdx), %r10 + cmpq %r10, %r9 + jb L(loop_4x_vec_backward) L(loop_large_backward): - /* Copy 4 * VEC a time backward with non-temporal stores. */ - PREFETCH_ONE_SET (-1, (%rcx), -PREFETCHED_LOAD_SIZE * 2) - PREFETCH_ONE_SET (-1, (%rcx), -PREFETCHED_LOAD_SIZE * 3) - VMOVU (%rcx), %VEC(0) - VMOVU -VEC_SIZE(%rcx), %VEC(1) - VMOVU -(VEC_SIZE * 2)(%rcx), %VEC(2) - VMOVU -(VEC_SIZE * 3)(%rcx), %VEC(3) - subq $PREFETCHED_LOAD_SIZE, %rcx - subq $PREFETCHED_LOAD_SIZE, %rdx - VMOVNT %VEC(0), (%r9) - VMOVNT %VEC(1), -VEC_SIZE(%r9) - VMOVNT %VEC(2), -(VEC_SIZE * 2)(%r9) - VMOVNT %VEC(3), -(VEC_SIZE * 3)(%r9) - subq $PREFETCHED_LOAD_SIZE, %r9 - cmpq $PREFETCHED_LOAD_SIZE, %rdx - ja L(loop_large_backward) - sfence - /* Store the first 4 * VEC. */ - VMOVU %VEC(4), (%rdi) - VMOVU %VEC(5), VEC_SIZE(%rdi) - VMOVU %VEC(6), (VEC_SIZE * 2)(%rdi) - VMOVU %VEC(7), (VEC_SIZE * 3)(%rdi) - /* Store the last VEC. */ - VMOVU %VEC(8), (%r11) - VZEROUPPER - ret + /* Copy 4 * VEC a time backward with non-temporal stores. */ + PREFETCH_ONE_SET (-1, (%rcx), -PREFETCHED_LOAD_SIZE * 2) + PREFETCH_ONE_SET (-1, (%rcx), -PREFETCHED_LOAD_SIZE * 3) + VMOVU (%rcx), %VEC(0) + VMOVU -VEC_SIZE(%rcx), %VEC(1) + VMOVU -(VEC_SIZE * 2)(%rcx), %VEC(2) + VMOVU -(VEC_SIZE * 3)(%rcx), %VEC(3) + subq $PREFETCHED_LOAD_SIZE, %rcx + subq $PREFETCHED_LOAD_SIZE, %rdx + VMOVNT %VEC(0), (%r9) + VMOVNT %VEC(1), -VEC_SIZE(%r9) + VMOVNT %VEC(2), -(VEC_SIZE * 2)(%r9) + VMOVNT %VEC(3), -(VEC_SIZE * 3)(%r9) + subq $PREFETCHED_LOAD_SIZE, %r9 + cmpq $PREFETCHED_LOAD_SIZE, %rdx + ja L(loop_large_backward) + sfence + /* Store the first 4 * VEC. */ + VMOVU %VEC(4), (%rdi) + VMOVU %VEC(5), VEC_SIZE(%rdi) + VMOVU %VEC(6), (VEC_SIZE * 2)(%rdi) + VMOVU %VEC(7), (VEC_SIZE * 3)(%rdi) + /* Store the last VEC. */ + VMOVU %VEC(8), (%r11) + VZEROUPPER + ret #endif END (MEMMOVE_SYMBOL (__memmove, unaligned_erms)) #if 1 # ifdef USE_MULTIARCH strong_alias (MEMMOVE_SYMBOL (__memmove, unaligned_erms), - MEMMOVE_SYMBOL (__memcpy, unaligned_erms)) + MEMMOVE_SYMBOL (__memcpy, unaligned_erms)) # ifdef SHARED strong_alias (MEMMOVE_SYMBOL (__memmove_chk, unaligned_erms), - MEMMOVE_SYMBOL (__memcpy_chk, unaligned_erms)) + MEMMOVE_SYMBOL (__memcpy_chk, unaligned_erms)) # endif # endif # ifdef SHARED strong_alias (MEMMOVE_CHK_SYMBOL (__memmove_chk, unaligned), - MEMMOVE_CHK_SYMBOL (__memcpy_chk, unaligned)) + MEMMOVE_CHK_SYMBOL (__memcpy_chk, unaligned)) # endif #endif strong_alias (MEMMOVE_SYMBOL (__memmove, unaligned), - MEMCPY_SYMBOL (__memcpy, unaligned)) + MEMCPY_SYMBOL (__memcpy, unaligned)) diff --git a/utils/memcpy-bench/glibc/memmove.S b/utils/memcpy-bench/glibc/memmove.S index 97e735facff..7bd47b9a03f 100644 --- a/utils/memcpy-bench/glibc/memmove.S +++ b/utils/memcpy-bench/glibc/memmove.S @@ -18,33 +18,33 @@ #include "sysdep.h" -#define VEC_SIZE 16 -#define VEC(i) xmm##i -#define PREFETCHNT prefetchnta -#define VMOVNT movntdq +#define VEC_SIZE 16 +#define VEC(i) xmm##i +#define PREFETCHNT prefetchnta +#define VMOVNT movntdq /* Use movups and movaps for smaller code sizes. */ -#define VMOVU movups -#define VMOVA movaps +#define VMOVU movups +#define VMOVA movaps -#define SECTION(p) p +#define SECTION(p) p #ifdef USE_MULTIARCH # if 0 -# define MEMCPY_SYMBOL(p,s) memcpy +# define MEMCPY_SYMBOL(p,s) memcpy # endif #else # if defined SHARED -# define MEMCPY_SYMBOL(p,s) __memcpy +# define MEMCPY_SYMBOL(p,s) __memcpy # else -# define MEMCPY_SYMBOL(p,s) memcpy +# define MEMCPY_SYMBOL(p,s) memcpy # endif #endif #if !defined USE_MULTIARCH -# define MEMPCPY_SYMBOL(p,s) __mempcpy +# define MEMPCPY_SYMBOL(p,s) __mempcpy #endif #ifndef MEMMOVE_SYMBOL -# define MEMMOVE_CHK_SYMBOL(p,s) p -# define MEMMOVE_SYMBOL(p,s) memmove +# define MEMMOVE_CHK_SYMBOL(p,s) p +# define MEMMOVE_SYMBOL(p,s) memmove #endif #include "memmove-vec-unaligned-erms.S" diff --git a/utils/memcpy-bench/glibc/sysdep.h b/utils/memcpy-bench/glibc/sysdep.h index 099134b2a2f..e255e7488da 100644 --- a/utils/memcpy-bench/glibc/sysdep.h +++ b/utils/memcpy-bench/glibc/sysdep.h @@ -21,7 +21,7 @@ #include "sysdep_x86.h" -#ifdef __ASSEMBLER__ +#ifdef __ASSEMBLER__ /* Syntactic details of assembler. */ @@ -29,11 +29,11 @@ the register as saved relative to %rsp instead of relative to the CFA. Expression is DW_OP_drop, DW_OP_breg7 (%rsp is register 7), sleb128 offset from %rsp. */ -#define cfi_offset_rel_rsp(regn, off) .cfi_escape 0x10, regn, 0x4, 0x13, \ - 0x77, off & 0x7F | 0x80, off >> 7 +#define cfi_offset_rel_rsp(regn, off) .cfi_escape 0x10, regn, 0x4, 0x13, \ + 0x77, off & 0x7F | 0x80, off >> 7 /* If compiled for profiling, call `mcount' at the start of each function. */ -#ifdef PROF +#ifdef PROF /* The mcount code relies on a normal frame pointer being on the stack to locate our caller, so push one just for its benefit. */ #define CALL_MCOUNT \ @@ -45,31 +45,31 @@ popq %rbp; \ cfi_def_cfa(rsp,8); #else -#define CALL_MCOUNT /* Do nothing. */ +#define CALL_MCOUNT /* Do nothing. */ #endif -#define PSEUDO(name, syscall_name, args) \ -lose: \ - jmp JUMPTARGET(syscall_error) \ - .globl syscall_error; \ - ENTRY (name) \ - DO_CALL (syscall_name, args); \ +#define PSEUDO(name, syscall_name, args) \ +lose: \ + jmp JUMPTARGET(syscall_error) \ + .globl syscall_error; \ + ENTRY (name) \ + DO_CALL (syscall_name, args); \ jb lose #undef JUMPTARGET #ifdef SHARED # ifdef BIND_NOW -# define JUMPTARGET(name) *name##@GOTPCREL(%rip) +# define JUMPTARGET(name) *name##@GOTPCREL(%rip) # else -# define JUMPTARGET(name) name##@PLT +# define JUMPTARGET(name) name##@PLT # endif #else /* For static archives, branch to target directly. */ -# define JUMPTARGET(name) name +# define JUMPTARGET(name) name #endif /* Long and pointer size in bytes. */ -#define LP_SIZE 8 +#define LP_SIZE 8 /* Instruction to operate on long and pointer. */ #define LP_OP(insn) insn##q @@ -78,24 +78,24 @@ lose: \ #define ASM_ADDR .quad /* Registers to hold long and pointer. */ -#define RAX_LP rax -#define RBP_LP rbp -#define RBX_LP rbx -#define RCX_LP rcx -#define RDI_LP rdi -#define RDX_LP rdx -#define RSI_LP rsi -#define RSP_LP rsp -#define R8_LP r8 -#define R9_LP r9 -#define R10_LP r10 -#define R11_LP r11 -#define R12_LP r12 -#define R13_LP r13 -#define R14_LP r14 -#define R15_LP r15 +#define RAX_LP rax +#define RBP_LP rbp +#define RBX_LP rbx +#define RCX_LP rcx +#define RDI_LP rdi +#define RDX_LP rdx +#define RSI_LP rsi +#define RSP_LP rsp +#define R8_LP r8 +#define R9_LP r9 +#define R10_LP r10 +#define R11_LP r11 +#define R12_LP r12 +#define R13_LP r13 +#define R14_LP r14 +#define R15_LP r15 -#else /* __ASSEMBLER__ */ +#else /* __ASSEMBLER__ */ /* Long and pointer size in bytes. */ #define LP_SIZE "8" @@ -107,23 +107,23 @@ lose: \ #define ASM_ADDR ".quad" /* Registers to hold long and pointer. */ -#define RAX_LP "rax" -#define RBP_LP "rbp" -#define RBX_LP "rbx" -#define RCX_LP "rcx" -#define RDI_LP "rdi" -#define RDX_LP "rdx" -#define RSI_LP "rsi" -#define RSP_LP "rsp" -#define R8_LP "r8" -#define R9_LP "r9" -#define R10_LP "r10" -#define R11_LP "r11" -#define R12_LP "r12" -#define R13_LP "r13" -#define R14_LP "r14" -#define R15_LP "r15" +#define RAX_LP "rax" +#define RBP_LP "rbp" +#define RBX_LP "rbx" +#define RCX_LP "rcx" +#define RDI_LP "rdi" +#define RDX_LP "rdx" +#define RSI_LP "rsi" +#define RSP_LP "rsp" +#define R8_LP "r8" +#define R9_LP "r9" +#define R10_LP "r10" +#define R11_LP "r11" +#define R12_LP "r12" +#define R13_LP "r13" +#define R14_LP "r14" +#define R15_LP "r15" -#endif /* __ASSEMBLER__ */ +#endif /* __ASSEMBLER__ */ -#endif /* _X86_64_SYSDEP_H */ +#endif /* _X86_64_SYSDEP_H */ diff --git a/utils/memcpy-bench/glibc/sysdep_generic.h b/utils/memcpy-bench/glibc/sysdep_generic.h index 91f78e1b04d..afecea8c356 100644 --- a/utils/memcpy-bench/glibc/sysdep_generic.h +++ b/utils/memcpy-bench/glibc/sysdep_generic.h @@ -28,14 +28,14 @@ #define ASM_LINE_SEP ; -#define strong_alias(original, alias) \ - .globl C_SYMBOL_NAME (alias) ASM_LINE_SEP \ +#define strong_alias(original, alias) \ + .globl C_SYMBOL_NAME (alias) ASM_LINE_SEP \ C_SYMBOL_NAME (alias) = C_SYMBOL_NAME (original) #ifndef C_LABEL /* Define a macro we can use to construct the asm name for a C symbol. */ -# define C_LABEL(name) name##: +# define C_LABEL(name) name##: #endif @@ -47,38 +47,38 @@ # endif # ifndef JUMPTARGET -# define JUMPTARGET(sym) sym +# define JUMPTARGET(sym) sym # endif #endif -/* Makros to generate eh_frame unwind information. */ +/* Macros to generate eh_frame unwind information. */ #ifdef __ASSEMBLER__ -# define cfi_startproc .cfi_startproc -# define cfi_endproc .cfi_endproc -# define cfi_def_cfa(reg, off) .cfi_def_cfa reg, off -# define cfi_def_cfa_register(reg) .cfi_def_cfa_register reg -# define cfi_def_cfa_offset(off) .cfi_def_cfa_offset off -# define cfi_adjust_cfa_offset(off) .cfi_adjust_cfa_offset off -# define cfi_offset(reg, off) .cfi_offset reg, off -# define cfi_rel_offset(reg, off) .cfi_rel_offset reg, off -# define cfi_register(r1, r2) .cfi_register r1, r2 -# define cfi_return_column(reg) .cfi_return_column reg -# define cfi_restore(reg) .cfi_restore reg -# define cfi_same_value(reg) .cfi_same_value reg -# define cfi_undefined(reg) .cfi_undefined reg -# define cfi_remember_state .cfi_remember_state -# define cfi_restore_state .cfi_restore_state -# define cfi_window_save .cfi_window_save -# define cfi_personality(enc, exp) .cfi_personality enc, exp -# define cfi_lsda(enc, exp) .cfi_lsda enc, exp +# define cfi_startproc .cfi_startproc +# define cfi_endproc .cfi_endproc +# define cfi_def_cfa(reg, off) .cfi_def_cfa reg, off +# define cfi_def_cfa_register(reg) .cfi_def_cfa_register reg +# define cfi_def_cfa_offset(off) .cfi_def_cfa_offset off +# define cfi_adjust_cfa_offset(off) .cfi_adjust_cfa_offset off +# define cfi_offset(reg, off) .cfi_offset reg, off +# define cfi_rel_offset(reg, off) .cfi_rel_offset reg, off +# define cfi_register(r1, r2) .cfi_register r1, r2 +# define cfi_return_column(reg) .cfi_return_column reg +# define cfi_restore(reg) .cfi_restore reg +# define cfi_same_value(reg) .cfi_same_value reg +# define cfi_undefined(reg) .cfi_undefined reg +# define cfi_remember_state .cfi_remember_state +# define cfi_restore_state .cfi_restore_state +# define cfi_window_save .cfi_window_save +# define cfi_personality(enc, exp) .cfi_personality enc, exp +# define cfi_lsda(enc, exp) .cfi_lsda enc, exp #else /* ! ASSEMBLER */ # define CFI_STRINGIFY(Name) CFI_STRINGIFY2 (Name) # define CFI_STRINGIFY2(Name) #Name -# define CFI_STARTPROC ".cfi_startproc" -# define CFI_ENDPROC ".cfi_endproc" -# define CFI_DEF_CFA(reg, off) \ +# define CFI_STARTPROC ".cfi_startproc" +# define CFI_ENDPROC ".cfi_endproc" +# define CFI_DEF_CFA(reg, off) \ ".cfi_def_cfa " CFI_STRINGIFY(reg) "," CFI_STRINGIFY(off) # define CFI_DEF_CFA_REGISTER(reg) \ ".cfi_def_cfa_register " CFI_STRINGIFY(reg) diff --git a/utils/memcpy-bench/glibc/sysdep_x86.h b/utils/memcpy-bench/glibc/sysdep_x86.h index a3fecd01268..7abb350242f 100644 --- a/utils/memcpy-bench/glibc/sysdep_x86.h +++ b/utils/memcpy-bench/glibc/sysdep_x86.h @@ -34,18 +34,18 @@ enum cf_protection_level */ /* Set if CF_BRANCH (IBT) is enabled. */ -#define X86_FEATURE_1_IBT (1U << 0) +#define X86_FEATURE_1_IBT (1U << 0) /* Set if CF_RETURN (SHSTK) is enabled. */ -#define X86_FEATURE_1_SHSTK (1U << 1) +#define X86_FEATURE_1_SHSTK (1U << 1) #ifdef __CET__ -# define CET_ENABLED 1 -# define IBT_ENABLED (__CET__ & X86_FEATURE_1_IBT) -# define SHSTK_ENABLED (__CET__ & X86_FEATURE_1_SHSTK) +# define CET_ENABLED 1 +# define IBT_ENABLED (__CET__ & X86_FEATURE_1_IBT) +# define SHSTK_ENABLED (__CET__ & X86_FEATURE_1_SHSTK) #else -# define CET_ENABLED 0 -# define IBT_ENABLED 0 -# define SHSTK_ENABLED 0 +# define CET_ENABLED 0 +# define IBT_ENABLED 0 +# define SHSTK_ENABLED 0 #endif /* Offset for fxsave/xsave area used by _dl_runtime_resolve. Also need @@ -57,7 +57,7 @@ enum cf_protection_level #define STATE_SAVE_MASK \ ((1 << 1) | (1 << 2) | (1 << 3) | (1 << 5) | (1 << 6) | (1 << 7)) -#ifdef __ASSEMBLER__ +#ifdef __ASSEMBLER__ /* Syntactic details of assembler. */ @@ -73,18 +73,18 @@ enum cf_protection_level #define ASM_SIZE_DIRECTIVE(name) .size name,.-name; /* Define an entry point visible from C. */ -#define ENTRY(name) \ - .globl C_SYMBOL_NAME(name); \ - .type C_SYMBOL_NAME(name),@function; \ - .align ALIGNARG(4); \ - C_LABEL(name) \ - cfi_startproc; \ - _CET_ENDBR; \ +#define ENTRY(name) \ + .globl C_SYMBOL_NAME(name); \ + .type C_SYMBOL_NAME(name),@function; \ + .align ALIGNARG(4); \ + C_LABEL(name) \ + cfi_startproc; \ + _CET_ENDBR; \ CALL_MCOUNT -#undef END -#define END(name) \ - cfi_endproc; \ +#undef END +#define END(name) \ + cfi_endproc; \ ASM_SIZE_DIRECTIVE(name) #define ENTRY_CHK(name) ENTRY (name) @@ -93,21 +93,21 @@ enum cf_protection_level /* Since C identifiers are not normally prefixed with an underscore on this system, the asm identifier `syscall_error' intrudes on the C name space. Make sure we use an innocuous name. */ -#define syscall_error __syscall_error -#define mcount _mcount +#define syscall_error __syscall_error +#define mcount _mcount -#undef PSEUDO_END -#define PSEUDO_END(name) \ +#undef PSEUDO_END +#define PSEUDO_END(name) \ END (name) /* Local label name for asm code. */ #ifndef L /* ELF-like local names start with `.L'. */ -# define L(name) .L##name +# define L(name) .L##name #endif #define atom_text_section .section ".text.atom", "ax" -#endif /* __ASSEMBLER__ */ +#endif /* __ASSEMBLER__ */ -#endif /* _X86_SYSDEP_H */ +#endif /* _X86_SYSDEP_H */ From 8c2d65242a81b68f9ca520cf015e53933a52eaca Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Sun, 14 Mar 2021 23:24:22 +0300 Subject: [PATCH 453/544] Fix style --- utils/memcpy-bench/glibc/asm-syntax.h | 2 ++ utils/memcpy-bench/glibc/dwarf2.h | 4 +++- utils/memcpy-bench/glibc/sysdep.h | 2 ++ utils/memcpy-bench/glibc/sysdep_generic.h | 2 ++ utils/memcpy-bench/glibc/sysdep_x86.h | 2 ++ 5 files changed, 11 insertions(+), 1 deletion(-) diff --git a/utils/memcpy-bench/glibc/asm-syntax.h b/utils/memcpy-bench/glibc/asm-syntax.h index 6e299c1fec2..9d65213ba30 100644 --- a/utils/memcpy-bench/glibc/asm-syntax.h +++ b/utils/memcpy-bench/glibc/asm-syntax.h @@ -1,3 +1,5 @@ +#pragma once + /* Definitions for x86 syntax variations. Copyright (C) 1992-2020 Free Software Foundation, Inc. This file is part of the GNU C Library. Its master source is NOT part of diff --git a/utils/memcpy-bench/glibc/dwarf2.h b/utils/memcpy-bench/glibc/dwarf2.h index 2be827f00ae..b0536c97e5e 100644 --- a/utils/memcpy-bench/glibc/dwarf2.h +++ b/utils/memcpy-bench/glibc/dwarf2.h @@ -1,3 +1,5 @@ +#pragma once + /* Declarations and definitions of codes relating to the DWARF2 symbolic debugging information format. Copyright (C) 1992-2020 Free Software Foundation, Inc. @@ -563,7 +565,7 @@ enum dwarf_macinfo_record_type }; #endif /* !ASSEMBLER */ - + /* @@@ For use with GNU frame unwind information. */ #define DW_EH_PE_absptr 0x00 diff --git a/utils/memcpy-bench/glibc/sysdep.h b/utils/memcpy-bench/glibc/sysdep.h index e255e7488da..2f43d688df9 100644 --- a/utils/memcpy-bench/glibc/sysdep.h +++ b/utils/memcpy-bench/glibc/sysdep.h @@ -1,3 +1,5 @@ +#pragma once + /* Assembler macros for x86-64. Copyright (C) 2001-2020 Free Software Foundation, Inc. This file is part of the GNU C Library. diff --git a/utils/memcpy-bench/glibc/sysdep_generic.h b/utils/memcpy-bench/glibc/sysdep_generic.h index afecea8c356..0cb5bca4102 100644 --- a/utils/memcpy-bench/glibc/sysdep_generic.h +++ b/utils/memcpy-bench/glibc/sysdep_generic.h @@ -1,3 +1,5 @@ +#pragma once + /* Generic asm macros used on many machines. Copyright (C) 1991-2020 Free Software Foundation, Inc. This file is part of the GNU C Library. diff --git a/utils/memcpy-bench/glibc/sysdep_x86.h b/utils/memcpy-bench/glibc/sysdep_x86.h index 7abb350242f..4469ed2e885 100644 --- a/utils/memcpy-bench/glibc/sysdep_x86.h +++ b/utils/memcpy-bench/glibc/sysdep_x86.h @@ -1,3 +1,5 @@ +#pragma once + /* Assembler macros for x86. Copyright (C) 2017-2020 Free Software Foundation, Inc. This file is part of the GNU C Library. From 4be01d927dcb81fc824f3a9b6367456809432029 Mon Sep 17 00:00:00 2001 From: Kseniia Sumarokova <54203879+kssenii@users.noreply.github.com> Date: Sun, 14 Mar 2021 23:57:31 +0300 Subject: [PATCH 454/544] Bump CI From 5b7ef512f58f5697af0820a5c470a9c7c06b03d4 Mon Sep 17 00:00:00 2001 From: Kseniia Sumarokova <54203879+kssenii@users.noreply.github.com> Date: Mon, 15 Mar 2021 09:14:30 +0300 Subject: [PATCH 455/544] Update PostgreSQLReplicaConnection.h --- src/Storages/PostgreSQL/PostgreSQLReplicaConnection.h | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/Storages/PostgreSQL/PostgreSQLReplicaConnection.h b/src/Storages/PostgreSQL/PostgreSQLReplicaConnection.h index 289183d8451..9465d4a119b 100644 --- a/src/Storages/PostgreSQL/PostgreSQLReplicaConnection.h +++ b/src/Storages/PostgreSQL/PostgreSQLReplicaConnection.h @@ -16,8 +16,8 @@ public: PostgreSQLReplicaConnection( const Poco::Util::AbstractConfiguration & config, - const String & config_name, - const size_t num_retries = POSTGRESQL_CONNECTION_DEFAULT_RETRIES_NUM); + const String & config_prefix, + const size_t num_retries_ = POSTGRESQL_CONNECTION_DEFAULT_RETRIES_NUM); PostgreSQLReplicaConnection(const PostgreSQLReplicaConnection & other); From 679b32ee547ee4b43f6807476b9fe6ae6345711c Mon Sep 17 00:00:00 2001 From: Pysaoke Date: Mon, 15 Mar 2021 16:11:57 +0800 Subject: [PATCH 456/544] Update query_log.md Improve the translation of `query_log.md` in Chinese documents --- docs/zh/operations/system-tables/query_log.md | 125 +++++++++--------- 1 file changed, 63 insertions(+), 62 deletions(-) diff --git a/docs/zh/operations/system-tables/query_log.md b/docs/zh/operations/system-tables/query_log.md index 6d8d7a39699..aa954fc4845 100644 --- a/docs/zh/operations/system-tables/query_log.md +++ b/docs/zh/operations/system-tables/query_log.md @@ -5,86 +5,87 @@ machine_translated_rev: 5decc73b5dc60054f19087d3690c4eb99446a6c3 # system.query_log {#system_tables-query_log} -包含有关已执行查询的信息,例如,开始时间、处理持续时间、错误消息。 +包含已执行查询的相关信息,例如:开始时间、处理持续时间、错误消息。 !!! note "注" 此表不包含以下内容的摄取数据 `INSERT` 查询。 -您可以更改查询日志记录的设置 [query_log](../../operations/server-configuration-parameters/settings.md#server_configuration_parameters-query-log) 服务器配置部分。 +您可以更改query_log的设置,在服务器配置的 [query_log](../../operations/server-configuration-parameters/settings.md#server_configuration_parameters-query-log) 部分。 -您可以通过设置禁用查询日志记录 [log_queries=0](../../operations/settings/settings.md#settings-log-queries). 我们不建议关闭日志记录,因为此表中的信息对于解决问题很重要。 +您可以通过设置 [log_queries=0](../../operations/settings/settings.md#settings-log-queries)来禁用query_log. 我们不建议关闭此日志,因为此表中的信息对于解决问题很重要。 -数据的冲洗周期设置在 `flush_interval_milliseconds` 的参数 [query_log](../../operations/server-configuration-parameters/settings.md#server_configuration_parameters-query-log) 服务器设置部分。 要强制冲洗,请使用 [SYSTEM FLUSH LOGS](../../sql-reference/statements/system.md#query_language-system-flush_logs) 查询。 +数据刷新的周期可通过 `flush_interval_milliseconds` 参数来设置 [query_log](../../operations/server-configuration-parameters/settings.md#server_configuration_parameters-query-log) 。 要强制刷新,请使用 [SYSTEM FLUSH LOGS](../../sql-reference/statements/system.md#query_language-system-flush_logs)。 -ClickHouse不会自动从表中删除数据。 看 [导言](../../operations/system-tables/index.md#system-tables-introduction) 欲了解更多详情。 +ClickHouse不会自动从表中删除数据。更多详情请看 [introduction](../../operations/system-tables/index.md#system-tables-introduction) 。 -该 `system.query_log` 表注册两种查询: +`system.query_log` 表注册两种查询: 1. 客户端直接运行的初始查询。 2. 由其他查询启动的子查询(用于分布式查询执行)。 对于这些类型的查询,有关父查询的信息显示在 `initial_*` 列。 -每个查询创建一个或两个行中 `query_log` 表,这取决于状态(见 `type` 列)的查询: +每个查询在`query_log` 表中创建一或两行记录,这取决于查询的状态(见 `type` 列): -1. 如果查询执行成功,则两行具有 `QueryStart` 和 `QueryFinish` 创建类型。 -2. 如果在查询处理过程中发生错误,两个事件与 `QueryStart` 和 `ExceptionWhileProcessing` 创建类型。 -3. 如果在启动查询之前发生错误,则单个事件具有 `ExceptionBeforeStart` 创建类型。 +1. 如果查询执行成功,会创建type分别为`QueryStart` 和 `QueryFinish` 的两行记录。 +2. 如果在查询处理过程中发生错误,会创建type分别为`QueryStart` 和 `ExceptionWhileProcessing` 的两行记录。 +3. 如果在启动查询之前发生错误,则创建一行type为`ExceptionBeforeStart` 的记录。 列: -- `type` ([枚举8](../../sql-reference/data-types/enum.md)) — Type of an event that occurred when executing the query. Values: - - `'QueryStart' = 1` — Successful start of query execution. - - `'QueryFinish' = 2` — Successful end of query execution. - - `'ExceptionBeforeStart' = 3` — Exception before the start of query execution. - - `'ExceptionWhileProcessing' = 4` — Exception during the query execution. -- `event_date` ([日期](../../sql-reference/data-types/date.md)) — Query starting date. -- `event_time` ([日期时间](../../sql-reference/data-types/datetime.md)) — Query starting time. -- `query_start_time` ([日期时间](../../sql-reference/data-types/datetime.md)) — Start time of query execution. -- `query_duration_ms` ([UInt64](../../sql-reference/data-types/int-uint.md#uint-ranges)) — Duration of query execution in milliseconds. -- `read_rows` ([UInt64](../../sql-reference/data-types/int-uint.md#uint-ranges)) — Total number or rows read from all tables and table functions participated in query. It includes usual subqueries, subqueries for `IN` 和 `JOIN`. 对于分布式查询 `read_rows` 包括在所有副本上读取的行总数。 每个副本发送它的 `read_rows` 值,并且查询的服务器-发起方汇总所有接收到的和本地的值。 缓存卷不会影响此值。 -- `read_bytes` ([UInt64](../../sql-reference/data-types/int-uint.md#uint-ranges)) — Total number or bytes read from all tables and table functions participated in query. It includes usual subqueries, subqueries for `IN` 和 `JOIN`. 对于分布式查询 `read_bytes` 包括在所有副本上读取的行总数。 每个副本发送它的 `read_bytes` 值,并且查询的服务器-发起方汇总所有接收到的和本地的值。 缓存卷不会影响此值。 -- `written_rows` ([UInt64](../../sql-reference/data-types/int-uint.md#uint-ranges)) — For `INSERT` 查询,写入的行数。 对于其他查询,列值为0。 -- `written_bytes` ([UInt64](../../sql-reference/data-types/int-uint.md#uint-ranges)) — For `INSERT` 查询时,写入的字节数。 对于其他查询,列值为0。 -- `result_rows` ([UInt64](../../sql-reference/data-types/int-uint.md#uint-ranges)) — Number of rows in a result of the `SELECT` 查询,或者在一些行 `INSERT` 查询。 -- `result_bytes` ([UInt64](../../sql-reference/data-types/int-uint.md#uint-ranges)) — RAM volume in bytes used to store a query result. -- `memory_usage` ([UInt64](../../sql-reference/data-types/int-uint.md#uint-ranges)) — Memory consumption by the query. -- `query` ([字符串](../../sql-reference/data-types/string.md)) — Query string. -- `exception` ([字符串](../../sql-reference/data-types/string.md)) — Exception message. -- `exception_code` ([Int32](../../sql-reference/data-types/int-uint.md)) — Code of an exception. -- `stack_trace` ([字符串](../../sql-reference/data-types/string.md)) — [堆栈跟踪](https://en.wikipedia.org/wiki/Stack_trace). 如果查询成功完成,则为空字符串。 -- `is_initial_query` ([UInt8](../../sql-reference/data-types/int-uint.md)) — Query type. Possible values: - - 1 — Query was initiated by the client. - - 0 — Query was initiated by another query as part of distributed query execution. -- `user` ([字符串](../../sql-reference/data-types/string.md)) — Name of the user who initiated the current query. -- `query_id` ([字符串](../../sql-reference/data-types/string.md)) — ID of the query. -- `address` ([IPv6](../../sql-reference/data-types/domains/ipv6.md)) — IP address that was used to make the query. -- `port` ([UInt16](../../sql-reference/data-types/int-uint.md)) — The client port that was used to make the query. -- `initial_user` ([字符串](../../sql-reference/data-types/string.md)) — Name of the user who ran the initial query (for distributed query execution). -- `initial_query_id` ([字符串](../../sql-reference/data-types/string.md)) — ID of the initial query (for distributed query execution). -- `initial_address` ([IPv6](../../sql-reference/data-types/domains/ipv6.md)) — IP address that the parent query was launched from. -- `initial_port` ([UInt16](../../sql-reference/data-types/int-uint.md)) — The client port that was used to make the parent query. -- `interface` ([UInt8](../../sql-reference/data-types/int-uint.md)) — Interface that the query was initiated from. Possible values: +- `type` ([Enum8](../../sql-reference/data-types/enum.md)) — 执行查询时的事件类型. 值: + - `'QueryStart' = 1` — 查询成功启动. + - `'QueryFinish' = 2` — 查询成功完成. + - `'ExceptionBeforeStart' = 3` — 查询执行前有异常. + - `'ExceptionWhileProcessing' = 4` — 查询执行期间有异常. +- `event_date` ([Date](../../sql-reference/data-types/date.md)) — 查询开始日期. +- `event_time` ([DateTime](../../sql-reference/data-types/datetime.md)) — 查询开始时间. +- `event_time_microseconds` ([DateTime64](../../sql-reference/data-types/datetime64.md)) — 查询开始时间(毫秒精度). +- `query_start_time` ([DateTime](../../sql-reference/data-types/datetime.md)) — 查询执行的开始时间. +- `query_start_time_microseconds` (DateTime64) — 查询执行的开始时间(毫秒精度). +- `query_duration_ms` ([UInt64](../../sql-reference/data-types/int-uint.md#uint-ranges)) — 查询消耗的时间(毫秒). +- `read_rows` ([UInt64](../../sql-reference/data-types/int-uint.md#uint-ranges)) — 从参与了查询的所有表和表函数读取的总行数. 包括:普通的子查询, `IN` 和 `JOIN`的子查询. 对于分布式查询 `read_rows` 包括在所有副本上读取的行总数。 每个副本发送它的 `read_rows` 值,并且查询的服务器-发起方汇总所有接收到的和本地的值。 缓存卷不会影响此值。 +- `read_bytes` ([UInt64](../../sql-reference/data-types/int-uint.md#uint-ranges)) — 从参与了查询的所有表和表函数读取的总字节数. 包括:普通的子查询, `IN` 和 `JOIN`的子查询. 对于分布式查询 `read_bytes` 包括在所有副本上读取的字节总数。 每个副本发送它的 `read_bytes` 值,并且查询的服务器-发起方汇总所有接收到的和本地的值。 缓存卷不会影响此值。 +- `written_rows` ([UInt64](../../sql-reference/data-types/int-uint.md#uint-ranges)) — 对于 `INSERT` 查询,为写入的行数。 对于其他查询,值为0。 +- `written_bytes` ([UInt64](../../sql-reference/data-types/int-uint.md#uint-ranges)) — 对于 `INSERT` 查询时,为写入的字节数。 对于其他查询,值为0。 +- `result_rows` ([UInt64](../../sql-reference/data-types/int-uint.md#uint-ranges)) — `SELECT` 查询结果的行数,或`INSERT` 的行数。 +- `result_bytes` ([UInt64](../../sql-reference/data-types/int-uint.md#uint-ranges)) — 存储查询结果的RAM量. +- `memory_usage` ([UInt64](../../sql-reference/data-types/int-uint.md#uint-ranges)) — 查询使用的内存. +- `query` ([String](../../sql-reference/data-types/string.md)) — 查询语句. +- `exception` ([String](../../sql-reference/data-types/string.md)) — 异常信息. +- `exception_code` ([Int32](../../sql-reference/data-types/int-uint.md)) — 异常码. +- `stack_trace` ([String](../../sql-reference/data-types/string.md)) — [Stack Trace](https://en.wikipedia.org/wiki/Stack_trace). 如果查询成功完成,则为空字符串。 +- `is_initial_query` ([UInt8](../../sql-reference/data-types/int-uint.md)) — 查询类型. 可能的值: + - 1 — 客户端发起的查询. + - 0 — 由另一个查询发起的,作为分布式查询的一部分. +- `user` ([String](../../sql-reference/data-types/string.md)) — 发起查询的用户. +- `query_id` ([String](../../sql-reference/data-types/string.md)) — 查询ID. +- `address` ([IPv6](../../sql-reference/data-types/domains/ipv6.md)) — 发起查询的客户端IP地址. +- `port` ([UInt16](../../sql-reference/data-types/int-uint.md)) — 发起查询的客户端端口. +- `initial_user` ([String](../../sql-reference/data-types/string.md)) — 初始查询的用户名(用于分布式查询执行). +- `initial_query_id` ([String](../../sql-reference/data-types/string.md)) — 运行初始查询的ID(用于分布式查询执行). +- `initial_address` ([IPv6](../../sql-reference/data-types/domains/ipv6.md)) — 运行父查询的IP地址. +- `initial_port` ([UInt16](../../sql-reference/data-types/int-uint.md)) — 发起父查询的客户端端口. +- `interface` ([UInt8](../../sql-reference/data-types/int-uint.md)) — 发起查询的接口. 可能的值: - 1 — TCP. - 2 — HTTP. -- `os_user` ([字符串](../../sql-reference/data-types/string.md)) — Operating system username who runs [ツ环板clientョツ嘉ッツ偲](../../interfaces/cli.md). -- `client_hostname` ([字符串](../../sql-reference/data-types/string.md)) — Hostname of the client machine where the [ツ环板clientョツ嘉ッツ偲](../../interfaces/cli.md) 或者运行另一个TCP客户端。 -- `client_name` ([字符串](../../sql-reference/data-types/string.md)) — The [ツ环板clientョツ嘉ッツ偲](../../interfaces/cli.md) 或另一个TCP客户端名称。 -- `client_revision` ([UInt32](../../sql-reference/data-types/int-uint.md)) — Revision of the [ツ环板clientョツ嘉ッツ偲](../../interfaces/cli.md) 或另一个TCP客户端。 -- `client_version_major` ([UInt32](../../sql-reference/data-types/int-uint.md)) — Major version of the [ツ环板clientョツ嘉ッツ偲](../../interfaces/cli.md) 或另一个TCP客户端。 -- `client_version_minor` ([UInt32](../../sql-reference/data-types/int-uint.md)) — Minor version of the [ツ环板clientョツ嘉ッツ偲](../../interfaces/cli.md) 或另一个TCP客户端。 -- `client_version_patch` ([UInt32](../../sql-reference/data-types/int-uint.md)) — Patch component of the [ツ环板clientョツ嘉ッツ偲](../../interfaces/cli.md) 或另一个TCP客户端版本。 -- `http_method` (UInt8) — HTTP method that initiated the query. Possible values: - - 0 — The query was launched from the TCP interface. - - 1 — `GET` 方法被使用。 - - 2 — `POST` 方法被使用。 -- `http_user_agent` ([字符串](../../sql-reference/data-types/string.md)) — The `UserAgent` http请求中传递的标头。 -- `quota_key` ([字符串](../../sql-reference/data-types/string.md)) — The “quota key” 在指定 [配额](../../operations/quotas.md) 设置(见 `keyed`). +- `os_user` ([String](../../sql-reference/data-types/string.md)) — 运行 [clickhouse-client](../../interfaces/cli.md)的操作系统用户名. +- `client_hostname` ([String](../../sql-reference/data-types/string.md)) — 运行[clickhouse-client](../../interfaces/cli.md) 或其他TCP客户端的机器的主机名。 +- `client_name` ([String](../../sql-reference/data-types/string.md)) — [clickhouse-client](../../interfaces/cli.md) 或其他TCP客户端的名称。 +- `client_revision` ([UInt32](../../sql-reference/data-types/int-uint.md)) — [clickhouse-client](../../interfaces/cli.md) 或其他TCP客户端的Revision。 +- `client_version_major` ([UInt32](../../sql-reference/data-types/int-uint.md)) — [clickhouse-client](../../interfaces/cli.md) 或其他TCP客户端的Major version。 +- `client_version_minor` ([UInt32](../../sql-reference/data-types/int-uint.md)) — [clickhouse-client](../../interfaces/cli.md) 或其他TCP客户端的Minor version。 +- `client_version_patch` ([UInt32](../../sql-reference/data-types/int-uint.md)) — [clickhouse-client](../../interfaces/cli.md) 或其他TCP客户端的Patch component。 +- `http_method` (UInt8) — 发起查询的HTTP方法. 可能值: + - 0 — TCP接口的查询. + - 1 — `GET` + - 2 — `POST` +- `http_user_agent` ([String](../../sql-reference/data-types/string.md)) — The `UserAgent` The UserAgent header passed in the HTTP request。 +- `quota_key` ([String](../../sql-reference/data-types/string.md)) — 在[quotas](../../operations/quotas.md) 配置里设置的“quota key” (见 `keyed`). - `revision` ([UInt32](../../sql-reference/data-types/int-uint.md)) — ClickHouse revision. -- `thread_numbers` ([数组(UInt32)](../../sql-reference/data-types/array.md)) — Number of threads that are participating in query execution. -- `ProfileEvents.Names` ([数组(字符串)](../../sql-reference/data-types/array.md)) — Counters that measure different metrics. The description of them could be found in the table [系统。活动](../../operations/system-tables/events.md#system_tables-events) -- `ProfileEvents.Values` ([数组(UInt64)](../../sql-reference/data-types/array.md)) — Values of metrics that are listed in the `ProfileEvents.Names` 列。 -- `Settings.Names` ([数组(字符串)](../../sql-reference/data-types/array.md)) — Names of settings that were changed when the client ran the query. To enable logging changes to settings, set the `log_query_settings` 参数为1。 -- `Settings.Values` ([数组(字符串)](../../sql-reference/data-types/array.md)) — Values of settings that are listed in the `Settings.Names` 列。 - +- `thread_numbers` ([Array(UInt32)](../../sql-reference/data-types/array.md)) — 参与查询的线程数. +- `ProfileEvents.Names` ([Array(String)](../../sql-reference/data-types/array.md)) — 衡量不同指标的计数器。 可以在[system.events](../../operations/system-tables/events.md#system_tables-events)中找到它们的描述。 +- `ProfileEvents.Values` ([Array(UInt64)](../../sql-reference/data-types/array.md)) — `ProfileEvents.Names` 列中列出的指标的值。 +- `Settings.Names` ([Array(String)](../../sql-reference/data-types/array.md)) — 客户端运行查询时更改的设置的名称。 要启用对设置的日志记录更改,请将log_query_settings参数设置为1。 +- `Settings.Values` ([Array(String)](../../sql-reference/data-types/array.md)) — `Settings.Names` 列中列出的设置的值。 **示例** ``` sql @@ -140,4 +141,4 @@ Settings.Values: ['0','random','1','10000000000'] **另请参阅** -- [system.query_thread_log](../../operations/system-tables/query_thread_log.md#system_tables-query_thread_log) — This table contains information about each query execution thread. +- [system.query_thread_log](../../operations/system-tables/query_thread_log.md#system_tables-query_thread_log) — 这个表包含了每个查询执行线程的信息 From 637f6a29a649ee46360848c5a8013fb040050589 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Mon, 15 Mar 2021 11:16:15 +0300 Subject: [PATCH 457/544] Add penalty --- utils/memcpy-bench/memcpy-bench.cpp | 18 ++++++------------ 1 file changed, 6 insertions(+), 12 deletions(-) diff --git a/utils/memcpy-bench/memcpy-bench.cpp b/utils/memcpy-bench/memcpy-bench.cpp index 365abe1f01e..dc510af0dbf 100644 --- a/utils/memcpy-bench/memcpy-bench.cpp +++ b/utils/memcpy-bench/memcpy-bench.cpp @@ -33,6 +33,9 @@ void NO_INLINE loop(uint8_t * dst, uint8_t * src, size_t size, F && chunk_size_d dst += bytes_to_copy; src += bytes_to_copy; size -= bytes_to_copy; + + /// Execute at least one SSE instruction as a penalty after running AVX code. + __asm__ volatile ("pxor %%xmm7, %%xmm7" ::: "xmm7"); } } @@ -76,16 +79,9 @@ uint64_t test(uint8_t * dst, uint8_t * src, size_t size, size_t iterations, size uint64_t elapsed_ns = watch.elapsed(); /// Validation - size_t sum = 0; - size_t reference = 0; for (size_t i = 0; i < size; ++i) - { - sum += dst[i]; - reference += uint8_t(i); - } - - if (sum != reference) - throw std::logic_error("Incorrect result"); + if (dst[i] != uint8_t(i)) + throw std::logic_error("Incorrect result"); std::cout << name; return elapsed_ns; @@ -676,11 +672,9 @@ done | tee result.tsv } else { - iterations = 10000000000ULL * num_threads / size; + iterations = 10000000000ULL / size; if (generator_variant == 1) - iterations /= 100; - if (generator_variant == 2) iterations /= 10; } From 1f6b05cd85d34c2d6f71b057c16d95b83f7d8853 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Mon, 15 Mar 2021 11:18:11 +0300 Subject: [PATCH 458/544] Add example --- utils/memcpy-bench/memcpy-bench.cpp | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/utils/memcpy-bench/memcpy-bench.cpp b/utils/memcpy-bench/memcpy-bench.cpp index dc510af0dbf..cd769640017 100644 --- a/utils/memcpy-bench/memcpy-bench.cpp +++ b/utils/memcpy-bench/memcpy-bench.cpp @@ -655,6 +655,24 @@ for size in 4096 16384 50000 65536 100000 1000000 10000000 100000000; do done; done | tee result.tsv +clickhouse-local --structure ' + name String, + size UInt64, + iterations UInt64, + threads UInt16, + generator UInt8, + memcpy UInt8, + elapsed UInt64 +' --query " + SELECT + size, name, + avg(1000 * elapsed / size / iterations) AS s, + count() AS c + FROM table + GROUP BY size, name + ORDER BY size ASC, s DESC +" --output-format PrettyCompact < result.tsv + )" << std::endl; std::cout << desc << std::endl; return 1; From 69b8ded5786ce341e842aec7d1967d6b6661761a Mon Sep 17 00:00:00 2001 From: alesapin Date: Mon, 15 Mar 2021 11:22:15 +0300 Subject: [PATCH 459/544] Fix alter modify for decimal columns when type size doesn't change --- src/Storages/StorageReplicatedMergeTree.cpp | 74 +++++++++---------- .../01761_alter_decimal_zookeeper.reference | 9 +++ .../01761_alter_decimal_zookeeper.sql | 31 ++++++++ 3 files changed, 75 insertions(+), 39 deletions(-) create mode 100644 tests/queries/0_stateless/01761_alter_decimal_zookeeper.reference create mode 100644 tests/queries/0_stateless/01761_alter_decimal_zookeeper.sql diff --git a/src/Storages/StorageReplicatedMergeTree.cpp b/src/Storages/StorageReplicatedMergeTree.cpp index d08c5b6ad7c..391d685be78 100644 --- a/src/Storages/StorageReplicatedMergeTree.cpp +++ b/src/Storages/StorageReplicatedMergeTree.cpp @@ -907,8 +907,7 @@ void StorageReplicatedMergeTree::setTableStructure( StorageInMemoryMetadata new_metadata = getInMemoryMetadata(); StorageInMemoryMetadata old_metadata = getInMemoryMetadata(); - if (new_columns != new_metadata.columns) - new_metadata.columns = new_columns; + new_metadata.columns = new_columns; if (!metadata_diff.empty()) { @@ -976,46 +975,43 @@ void StorageReplicatedMergeTree::setTableStructure( } /// Changes in columns may affect following metadata fields - if (new_metadata.columns != old_metadata.columns) + new_metadata.column_ttls_by_name.clear(); + for (const auto & [name, ast] : new_metadata.columns.getColumnTTLs()) { - new_metadata.column_ttls_by_name.clear(); - for (const auto & [name, ast] : new_metadata.columns.getColumnTTLs()) - { - auto new_ttl_entry = TTLDescription::getTTLFromAST(ast, new_metadata.columns, global_context, new_metadata.primary_key); - new_metadata.column_ttls_by_name[name] = new_ttl_entry; - } - - if (new_metadata.partition_key.definition_ast != nullptr) - new_metadata.partition_key.recalculateWithNewColumns(new_metadata.columns, global_context); - - if (!metadata_diff.sorting_key_changed) /// otherwise already updated - new_metadata.sorting_key.recalculateWithNewColumns(new_metadata.columns, global_context); - - /// Primary key is special, it exists even if not defined - if (new_metadata.primary_key.definition_ast != nullptr) - { - new_metadata.primary_key.recalculateWithNewColumns(new_metadata.columns, global_context); - } - else - { - new_metadata.primary_key = KeyDescription::getKeyFromAST(new_metadata.sorting_key.definition_ast, new_metadata.columns, global_context); - new_metadata.primary_key.definition_ast = nullptr; - } - - if (!metadata_diff.sampling_expression_changed && new_metadata.sampling_key.definition_ast != nullptr) - new_metadata.sampling_key.recalculateWithNewColumns(new_metadata.columns, global_context); - - if (!metadata_diff.skip_indices_changed) /// otherwise already updated - { - for (auto & index : new_metadata.secondary_indices) - index.recalculateWithNewColumns(new_metadata.columns, global_context); - } - - if (!metadata_diff.ttl_table_changed && new_metadata.table_ttl.definition_ast != nullptr) - new_metadata.table_ttl = TTLTableDescription::getTTLForTableFromAST( - new_metadata.table_ttl.definition_ast, new_metadata.columns, global_context, new_metadata.primary_key); + auto new_ttl_entry = TTLDescription::getTTLFromAST(ast, new_metadata.columns, global_context, new_metadata.primary_key); + new_metadata.column_ttls_by_name[name] = new_ttl_entry; } + if (new_metadata.partition_key.definition_ast != nullptr) + new_metadata.partition_key.recalculateWithNewColumns(new_metadata.columns, global_context); + + if (!metadata_diff.sorting_key_changed) /// otherwise already updated + new_metadata.sorting_key.recalculateWithNewColumns(new_metadata.columns, global_context); + + /// Primary key is special, it exists even if not defined + if (new_metadata.primary_key.definition_ast != nullptr) + { + new_metadata.primary_key.recalculateWithNewColumns(new_metadata.columns, global_context); + } + else + { + new_metadata.primary_key = KeyDescription::getKeyFromAST(new_metadata.sorting_key.definition_ast, new_metadata.columns, global_context); + new_metadata.primary_key.definition_ast = nullptr; + } + + if (!metadata_diff.sampling_expression_changed && new_metadata.sampling_key.definition_ast != nullptr) + new_metadata.sampling_key.recalculateWithNewColumns(new_metadata.columns, global_context); + + if (!metadata_diff.skip_indices_changed) /// otherwise already updated + { + for (auto & index : new_metadata.secondary_indices) + index.recalculateWithNewColumns(new_metadata.columns, global_context); + } + + if (!metadata_diff.ttl_table_changed && new_metadata.table_ttl.definition_ast != nullptr) + new_metadata.table_ttl = TTLTableDescription::getTTLForTableFromAST( + new_metadata.table_ttl.definition_ast, new_metadata.columns, global_context, new_metadata.primary_key); + /// Even if the primary/sorting/partition keys didn't change we must reinitialize it /// because primary/partition key column types might have changed. checkTTLExpressions(new_metadata, old_metadata); diff --git a/tests/queries/0_stateless/01761_alter_decimal_zookeeper.reference b/tests/queries/0_stateless/01761_alter_decimal_zookeeper.reference new file mode 100644 index 00000000000..5dcc95fd7b7 --- /dev/null +++ b/tests/queries/0_stateless/01761_alter_decimal_zookeeper.reference @@ -0,0 +1,9 @@ +1 5.00000000 +2 6.00000000 +CREATE TABLE default.test_alter_decimal\n(\n `n` UInt64,\n `d` Decimal(18, 8)\n)\nENGINE = ReplicatedMergeTree(\'/clickhouse/01761_alter_decimal_zookeeper\', \'r1\')\nORDER BY tuple()\nSETTINGS index_granularity = 8192 +1 5.00000000 +2 6.00000000 +CREATE TABLE default.test_alter_decimal\n(\n `n` UInt64,\n `d` Decimal(18, 8)\n)\nENGINE = ReplicatedMergeTree(\'/clickhouse/01761_alter_decimal_zookeeper\', \'r1\')\nORDER BY tuple()\nSETTINGS index_granularity = 8192 +1 5.00000000 +2 6.00000000 +3 7.00000000 diff --git a/tests/queries/0_stateless/01761_alter_decimal_zookeeper.sql b/tests/queries/0_stateless/01761_alter_decimal_zookeeper.sql new file mode 100644 index 00000000000..01766f0d6c2 --- /dev/null +++ b/tests/queries/0_stateless/01761_alter_decimal_zookeeper.sql @@ -0,0 +1,31 @@ +DROP TABLE IF EXISTS test_alter_decimal; + +CREATE TABLE test_alter_decimal +(n UInt64, d Decimal(15, 8)) +ENGINE = ReplicatedMergeTree('/clickhouse/01761_alter_decimal_zookeeper', 'r1') +ORDER BY tuple(); + +INSERT INTO test_alter_decimal VALUES (1, toDecimal32(5, 5)); + +INSERT INTO test_alter_decimal VALUES (2, toDecimal32(6, 6)); + +SELECT * FROM test_alter_decimal ORDER BY n; + +ALTER TABLE test_alter_decimal MODIFY COLUMN d Decimal(18, 8); + +SHOW CREATE TABLE test_alter_decimal; + +SELECT * FROM test_alter_decimal ORDER BY n; + +DETACH TABLE test_alter_decimal; +ATTACH TABLE test_alter_decimal; + +SHOW CREATE TABLE test_alter_decimal; + +INSERT INTO test_alter_decimal VALUES (3, toDecimal32(7, 7)); + +OPTIMIZE TABLE test_alter_decimal FINAL; + +SELECT * FROM test_alter_decimal ORDER BY n; + +DROP TABLE IF EXISTS test_alter_decimal; From 9bea10d9f94206671c89b8faf196725ed47e0d5e Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Mon, 15 Mar 2021 11:49:56 +0300 Subject: [PATCH 460/544] Fix style --- utils/memcpy-bench/glibc/asm-syntax.h | 28 +++--- utils/memcpy-bench/glibc/dwarf2.h | 114 +++++++++++----------- utils/memcpy-bench/glibc/sysdep.h | 58 +++++------ utils/memcpy-bench/glibc/sysdep_generic.h | 62 ++++++------ utils/memcpy-bench/glibc/sysdep_x86.h | 66 ++++++------- 5 files changed, 164 insertions(+), 164 deletions(-) diff --git a/utils/memcpy-bench/glibc/asm-syntax.h b/utils/memcpy-bench/glibc/asm-syntax.h index 9d65213ba30..0879f2606c7 100644 --- a/utils/memcpy-bench/glibc/asm-syntax.h +++ b/utils/memcpy-bench/glibc/asm-syntax.h @@ -1,23 +1,23 @@ #pragma once /* Definitions for x86 syntax variations. - Copyright (C) 1992-2020 Free Software Foundation, Inc. - This file is part of the GNU C Library. Its master source is NOT part of - the C library, however. The master source lives in the GNU MP Library. + Copyright (C) 1992-2020 Free Software Foundation, Inc. + This file is part of the GNU C Library. Its master source is NOT part of + the C library, however. The master source lives in the GNU MP Library. - The GNU C Library is free software; you can redistribute it and/or - modify it under the terms of the GNU Lesser General Public - License as published by the Free Software Foundation; either - version 2.1 of the License, or (at your option) any later version. + The GNU C Library is free software; you can redistribute it and/or + modify it under the terms of the GNU Lesser General Public + License as published by the Free Software Foundation; either + version 2.1 of the License, or (at your option) any later version. - The GNU C Library is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU - Lesser General Public License for more details. + The GNU C Library is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + Lesser General Public License for more details. - You should have received a copy of the GNU Lesser General Public - License along with the GNU C Library; if not, see - . */ + You should have received a copy of the GNU Lesser General Public + License along with the GNU C Library; if not, see + . */ #undef ALIGN #define ALIGN(log) .align 1<. */ + You should have received a copy of the GNU Lesser General Public + License along with the GNU C Library; if not, see + . */ #ifndef _DWARF2_H #define _DWARF2_H 1 /* This file is derived from the DWARF specification (a public document) - Revision 2.0.0 (July 27, 1993) developed by the UNIX International - Programming Languages Special Interest Group (UI/PLSIG) and distributed - by UNIX International. Copies of this specification are available from - UNIX International, 20 Waterview Boulevard, Parsippany, NJ, 07054. */ + Revision 2.0.0 (July 27, 1993) developed by the UNIX International + Programming Languages Special Interest Group (UI/PLSIG) and distributed + by UNIX International. Copies of this specification are available from + UNIX International, 20 Waterview Boulevard, Parsippany, NJ, 07054. */ /* This file is shared between GCC and GDB, and should not contain - prototypes. */ + prototypes. */ #ifndef __ASSEMBLER__ /* Tag names and codes. */ enum dwarf_tag - { + { DW_TAG_padding = 0x00, DW_TAG_array_type = 0x01, DW_TAG_class_type = 0x02, @@ -95,7 +95,7 @@ enum dwarf_tag DW_TAG_class_template = 0x4103, /* for C++ */ DW_TAG_GNU_BINCL = 0x4104, DW_TAG_GNU_EINCL = 0x4105 - }; + }; #define DW_TAG_lo_user 0x4080 #define DW_TAG_hi_user 0xffff @@ -106,7 +106,7 @@ enum dwarf_tag /* Form names and codes. */ enum dwarf_form - { + { DW_FORM_addr = 0x01, DW_FORM_block2 = 0x03, DW_FORM_block4 = 0x04, @@ -128,12 +128,12 @@ enum dwarf_form DW_FORM_ref8 = 0x14, DW_FORM_ref_udata = 0x15, DW_FORM_indirect = 0x16 - }; + }; /* Attribute names and codes. */ enum dwarf_attribute - { + { DW_AT_sibling = 0x01, DW_AT_location = 0x02, DW_AT_name = 0x03, @@ -215,7 +215,7 @@ enum dwarf_attribute DW_AT_src_coords = 0x2104, DW_AT_body_begin = 0x2105, DW_AT_body_end = 0x2106 - }; + }; #define DW_AT_lo_user 0x2000 /* implementation-defined range start */ #define DW_AT_hi_user 0x3ff0 /* implementation-defined range end */ @@ -223,7 +223,7 @@ enum dwarf_attribute /* Location atom names and codes. */ enum dwarf_location_atom - { + { DW_OP_addr = 0x03, DW_OP_deref = 0x06, DW_OP_const1u = 0x08, @@ -369,7 +369,7 @@ enum dwarf_location_atom DW_OP_deref_size = 0x94, DW_OP_xderef_size = 0x95, DW_OP_nop = 0x96 - }; + }; #define DW_OP_lo_user 0x80 /* implementation-defined range start */ #define DW_OP_hi_user 0xff /* implementation-defined range end */ @@ -377,7 +377,7 @@ enum dwarf_location_atom /* Type encodings. */ enum dwarf_type - { + { DW_ATE_void = 0x0, DW_ATE_address = 0x1, DW_ATE_boolean = 0x2, @@ -387,81 +387,81 @@ enum dwarf_type DW_ATE_signed_char = 0x6, DW_ATE_unsigned = 0x7, DW_ATE_unsigned_char = 0x8 - }; + }; #define DW_ATE_lo_user 0x80 #define DW_ATE_hi_user 0xff /* Array ordering names and codes. */ enum dwarf_array_dim_ordering - { + { DW_ORD_row_major = 0, DW_ORD_col_major = 1 - }; + }; /* access attribute */ enum dwarf_access_attribute - { + { DW_ACCESS_public = 1, DW_ACCESS_protected = 2, DW_ACCESS_private = 3 - }; + }; /* visibility */ enum dwarf_visibility_attribute - { + { DW_VIS_local = 1, DW_VIS_exported = 2, DW_VIS_qualified = 3 - }; + }; /* virtuality */ enum dwarf_virtuality_attribute - { + { DW_VIRTUALITY_none = 0, DW_VIRTUALITY_virtual = 1, DW_VIRTUALITY_pure_virtual = 2 - }; + }; /* case sensitivity */ enum dwarf_id_case - { + { DW_ID_case_sensitive = 0, DW_ID_up_case = 1, DW_ID_down_case = 2, DW_ID_case_insensitive = 3 - }; + }; /* calling convention */ enum dwarf_calling_convention - { + { DW_CC_normal = 0x1, DW_CC_program = 0x2, DW_CC_nocall = 0x3 - }; + }; #define DW_CC_lo_user 0x40 #define DW_CC_hi_user 0xff /* inline attribute */ enum dwarf_inline_attribute - { + { DW_INL_not_inlined = 0, DW_INL_inlined = 1, DW_INL_declared_not_inlined = 2, DW_INL_declared_inlined = 3 - }; + }; /* discriminant lists */ enum dwarf_discrim_list - { + { DW_DSC_label = 0, DW_DSC_range = 1 - }; + }; /* line number opcodes */ enum dwarf_line_number_ops - { + { DW_LNS_extended_op = 0, DW_LNS_copy = 1, DW_LNS_advance_pc = 2, @@ -472,19 +472,19 @@ enum dwarf_line_number_ops DW_LNS_set_basic_block = 7, DW_LNS_const_add_pc = 8, DW_LNS_fixed_advance_pc = 9 - }; + }; /* line number extended opcodes */ enum dwarf_line_number_x_ops - { + { DW_LNE_end_sequence = 1, DW_LNE_set_address = 2, DW_LNE_define_file = 3 - }; + }; /* call frame information */ enum dwarf_call_frame_info - { + { DW_CFA_advance_loc = 0x40, DW_CFA_offset = 0x80, DW_CFA_restore = 0xc0, @@ -517,7 +517,7 @@ enum dwarf_call_frame_info DW_CFA_GNU_window_save = 0x2d, DW_CFA_GNU_args_size = 0x2e, DW_CFA_GNU_negative_offset_extended = 0x2f - }; + }; #define DW_CIE_ID 0xffffffff #define DW_CIE_VERSION 1 @@ -534,7 +534,7 @@ enum dwarf_call_frame_info /* Source language names and codes. */ enum dwarf_source_language - { + { DW_LANG_C89 = 0x0001, DW_LANG_C = 0x0002, DW_LANG_Ada83 = 0x0003, @@ -547,7 +547,7 @@ enum dwarf_source_language DW_LANG_Modula2 = 0x000a, DW_LANG_Java = 0x000b, DW_LANG_Mips_Assembler = 0x8001 - }; + }; #define DW_LANG_lo_user 0x8000 /* implementation-defined range start */ @@ -556,13 +556,13 @@ enum dwarf_source_language /* Names and codes for macro information. */ enum dwarf_macinfo_record_type - { + { DW_MACINFO_define = 1, DW_MACINFO_undef = 2, DW_MACINFO_start_file = 3, DW_MACINFO_end_file = 4, DW_MACINFO_vendor_ext = 255 - }; + }; #endif /* !ASSEMBLER */ diff --git a/utils/memcpy-bench/glibc/sysdep.h b/utils/memcpy-bench/glibc/sysdep.h index 2f43d688df9..82b1e747fbe 100644 --- a/utils/memcpy-bench/glibc/sysdep.h +++ b/utils/memcpy-bench/glibc/sysdep.h @@ -1,22 +1,22 @@ #pragma once /* Assembler macros for x86-64. - Copyright (C) 2001-2020 Free Software Foundation, Inc. - This file is part of the GNU C Library. + Copyright (C) 2001-2020 Free Software Foundation, Inc. + This file is part of the GNU C Library. - The GNU C Library is free software; you can redistribute it and/or - modify it under the terms of the GNU Lesser General Public - License as published by the Free Software Foundation; either - version 2.1 of the License, or (at your option) any later version. + The GNU C Library is free software; you can redistribute it and/or + modify it under the terms of the GNU Lesser General Public + License as published by the Free Software Foundation; either + version 2.1 of the License, or (at your option) any later version. - The GNU C Library is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU - Lesser General Public License for more details. + The GNU C Library is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + Lesser General Public License for more details. - You should have received a copy of the GNU Lesser General Public - License along with the GNU C Library; if not, see - . */ + You should have received a copy of the GNU Lesser General Public + License along with the GNU C Library; if not, see + . */ #ifndef _X86_64_SYSDEP_H #define _X86_64_SYSDEP_H 1 @@ -28,35 +28,35 @@ /* Syntactic details of assembler. */ /* This macro is for setting proper CFI with DW_CFA_expression describing - the register as saved relative to %rsp instead of relative to the CFA. - Expression is DW_OP_drop, DW_OP_breg7 (%rsp is register 7), sleb128 offset - from %rsp. */ + the register as saved relative to %rsp instead of relative to the CFA. + Expression is DW_OP_drop, DW_OP_breg7 (%rsp is register 7), sleb128 offset + from %rsp. */ #define cfi_offset_rel_rsp(regn, off) .cfi_escape 0x10, regn, 0x4, 0x13, \ 0x77, off & 0x7F | 0x80, off >> 7 /* If compiled for profiling, call `mcount' at the start of each function. */ #ifdef PROF /* The mcount code relies on a normal frame pointer being on the stack - to locate our caller, so push one just for its benefit. */ + to locate our caller, so push one just for its benefit. */ #define CALL_MCOUNT \ - pushq %rbp; \ - cfi_adjust_cfa_offset(8); \ - movq %rsp, %rbp; \ - cfi_def_cfa_register(%rbp); \ - call JUMPTARGET(mcount); \ - popq %rbp; \ - cfi_def_cfa(rsp,8); + pushq %rbp; \ + cfi_adjust_cfa_offset(8); \ + movq %rsp, %rbp; \ + cfi_def_cfa_register(%rbp); \ + call JUMPTARGET(mcount); \ + popq %rbp; \ + cfi_def_cfa(rsp,8); #else #define CALL_MCOUNT /* Do nothing. */ #endif #define PSEUDO(name, syscall_name, args) \ lose: \ - jmp JUMPTARGET(syscall_error) \ - .globl syscall_error; \ - ENTRY (name) \ - DO_CALL (syscall_name, args); \ - jb lose + jmp JUMPTARGET(syscall_error) \ + .globl syscall_error; \ + ENTRY (name) \ + DO_CALL (syscall_name, args); \ + jb lose #undef JUMPTARGET #ifdef SHARED diff --git a/utils/memcpy-bench/glibc/sysdep_generic.h b/utils/memcpy-bench/glibc/sysdep_generic.h index 0cb5bca4102..e6183d72792 100644 --- a/utils/memcpy-bench/glibc/sysdep_generic.h +++ b/utils/memcpy-bench/glibc/sysdep_generic.h @@ -1,22 +1,22 @@ #pragma once /* Generic asm macros used on many machines. - Copyright (C) 1991-2020 Free Software Foundation, Inc. - This file is part of the GNU C Library. + Copyright (C) 1991-2020 Free Software Foundation, Inc. + This file is part of the GNU C Library. - The GNU C Library is free software; you can redistribute it and/or - modify it under the terms of the GNU Lesser General Public - License as published by the Free Software Foundation; either - version 2.1 of the License, or (at your option) any later version. + The GNU C Library is free software; you can redistribute it and/or + modify it under the terms of the GNU Lesser General Public + License as published by the Free Software Foundation; either + version 2.1 of the License, or (at your option) any later version. - The GNU C Library is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU - Lesser General Public License for more details. + The GNU C Library is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + Lesser General Public License for more details. - You should have received a copy of the GNU Lesser General Public - License along with the GNU C Library; if not, see - . */ + You should have received a copy of the GNU Lesser General Public + License along with the GNU C Library; if not, see + . */ #define C_SYMBOL_NAME(name) name #define HIDDEN_JUMPTARGET(name) 0x0 @@ -31,8 +31,8 @@ #define ASM_LINE_SEP ; #define strong_alias(original, alias) \ - .globl C_SYMBOL_NAME (alias) ASM_LINE_SEP \ - C_SYMBOL_NAME (alias) = C_SYMBOL_NAME (original) + .globl C_SYMBOL_NAME (alias) ASM_LINE_SEP \ + C_SYMBOL_NAME (alias) = C_SYMBOL_NAME (original) #ifndef C_LABEL @@ -43,7 +43,7 @@ #ifdef __ASSEMBLER__ /* Mark the end of function named SYM. This is used on some platforms - to generate correct debugging information. */ + to generate correct debugging information. */ # ifndef END # define END(sym) # endif @@ -81,35 +81,35 @@ # define CFI_STARTPROC ".cfi_startproc" # define CFI_ENDPROC ".cfi_endproc" # define CFI_DEF_CFA(reg, off) \ - ".cfi_def_cfa " CFI_STRINGIFY(reg) "," CFI_STRINGIFY(off) + ".cfi_def_cfa " CFI_STRINGIFY(reg) "," CFI_STRINGIFY(off) # define CFI_DEF_CFA_REGISTER(reg) \ - ".cfi_def_cfa_register " CFI_STRINGIFY(reg) + ".cfi_def_cfa_register " CFI_STRINGIFY(reg) # define CFI_DEF_CFA_OFFSET(off) \ - ".cfi_def_cfa_offset " CFI_STRINGIFY(off) + ".cfi_def_cfa_offset " CFI_STRINGIFY(off) # define CFI_ADJUST_CFA_OFFSET(off) \ - ".cfi_adjust_cfa_offset " CFI_STRINGIFY(off) + ".cfi_adjust_cfa_offset " CFI_STRINGIFY(off) # define CFI_OFFSET(reg, off) \ - ".cfi_offset " CFI_STRINGIFY(reg) "," CFI_STRINGIFY(off) + ".cfi_offset " CFI_STRINGIFY(reg) "," CFI_STRINGIFY(off) # define CFI_REL_OFFSET(reg, off) \ - ".cfi_rel_offset " CFI_STRINGIFY(reg) "," CFI_STRINGIFY(off) + ".cfi_rel_offset " CFI_STRINGIFY(reg) "," CFI_STRINGIFY(off) # define CFI_REGISTER(r1, r2) \ - ".cfi_register " CFI_STRINGIFY(r1) "," CFI_STRINGIFY(r2) + ".cfi_register " CFI_STRINGIFY(r1) "," CFI_STRINGIFY(r2) # define CFI_RETURN_COLUMN(reg) \ - ".cfi_return_column " CFI_STRINGIFY(reg) + ".cfi_return_column " CFI_STRINGIFY(reg) # define CFI_RESTORE(reg) \ - ".cfi_restore " CFI_STRINGIFY(reg) + ".cfi_restore " CFI_STRINGIFY(reg) # define CFI_UNDEFINED(reg) \ - ".cfi_undefined " CFI_STRINGIFY(reg) + ".cfi_undefined " CFI_STRINGIFY(reg) # define CFI_REMEMBER_STATE \ - ".cfi_remember_state" + ".cfi_remember_state" # define CFI_RESTORE_STATE \ - ".cfi_restore_state" + ".cfi_restore_state" # define CFI_WINDOW_SAVE \ - ".cfi_window_save" + ".cfi_window_save" # define CFI_PERSONALITY(enc, exp) \ - ".cfi_personality " CFI_STRINGIFY(enc) "," CFI_STRINGIFY(exp) + ".cfi_personality " CFI_STRINGIFY(enc) "," CFI_STRINGIFY(exp) # define CFI_LSDA(enc, exp) \ - ".cfi_lsda " CFI_STRINGIFY(enc) "," CFI_STRINGIFY(exp) + ".cfi_lsda " CFI_STRINGIFY(enc) "," CFI_STRINGIFY(exp) #endif #include "dwarf2.h" diff --git a/utils/memcpy-bench/glibc/sysdep_x86.h b/utils/memcpy-bench/glibc/sysdep_x86.h index 4469ed2e885..1c482cfabb7 100644 --- a/utils/memcpy-bench/glibc/sysdep_x86.h +++ b/utils/memcpy-bench/glibc/sysdep_x86.h @@ -1,22 +1,22 @@ #pragma once /* Assembler macros for x86. - Copyright (C) 2017-2020 Free Software Foundation, Inc. - This file is part of the GNU C Library. + Copyright (C) 2017-2020 Free Software Foundation, Inc. + This file is part of the GNU C Library. - The GNU C Library is free software; you can redistribute it and/or - modify it under the terms of the GNU Lesser General Public - License as published by the Free Software Foundation; either - version 2.1 of the License, or (at your option) any later version. + The GNU C Library is free software; you can redistribute it and/or + modify it under the terms of the GNU Lesser General Public + License as published by the Free Software Foundation; either + version 2.1 of the License, or (at your option) any later version. - The GNU C Library is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU - Lesser General Public License for more details. + The GNU C Library is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + Lesser General Public License for more details. - You should have received a copy of the GNU Lesser General Public - License along with the GNU C Library; if not, see - . */ + You should have received a copy of the GNU Lesser General Public + License along with the GNU C Library; if not, see + . */ #ifndef _X86_SYSDEP_H #define _X86_SYSDEP_H 1 @@ -27,11 +27,11 @@ enum cf_protection_level { - CF_NONE = 0, - CF_BRANCH = 1 << 0, - CF_RETURN = 1 << 1, - CF_FULL = CF_BRANCH | CF_RETURN, - CF_SET = 1 << 2 + CF_NONE = 0, + CF_BRANCH = 1 << 0, + CF_RETURN = 1 << 1, + CF_FULL = CF_BRANCH | CF_RETURN, + CF_SET = 1 << 2 }; */ @@ -51,13 +51,13 @@ enum cf_protection_level #endif /* Offset for fxsave/xsave area used by _dl_runtime_resolve. Also need - space to preserve RCX, RDX, RSI, RDI, R8, R9 and RAX. It must be - aligned to 16 bytes for fxsave and 64 bytes for xsave. */ + space to preserve RCX, RDX, RSI, RDI, R8, R9 and RAX. It must be + aligned to 16 bytes for fxsave and 64 bytes for xsave. */ #define STATE_SAVE_OFFSET (8 * 7 + 8) /* Save SSE, AVX, AVX512, mask and bound registers. */ #define STATE_SAVE_MASK \ - ((1 << 1) | (1 << 2) | (1 << 3) | (1 << 5) | (1 << 6) | (1 << 7)) + ((1 << 1) | (1 << 2) | (1 << 3) | (1 << 5) | (1 << 6) | (1 << 7)) #ifdef __ASSEMBLER__ @@ -76,31 +76,31 @@ enum cf_protection_level /* Define an entry point visible from C. */ #define ENTRY(name) \ - .globl C_SYMBOL_NAME(name); \ - .type C_SYMBOL_NAME(name),@function; \ - .align ALIGNARG(4); \ - C_LABEL(name) \ - cfi_startproc; \ - _CET_ENDBR; \ - CALL_MCOUNT + .globl C_SYMBOL_NAME(name); \ + .type C_SYMBOL_NAME(name),@function; \ + .align ALIGNARG(4); \ + C_LABEL(name) \ + cfi_startproc; \ + _CET_ENDBR; \ + CALL_MCOUNT #undef END #define END(name) \ - cfi_endproc; \ - ASM_SIZE_DIRECTIVE(name) + cfi_endproc; \ + ASM_SIZE_DIRECTIVE(name) #define ENTRY_CHK(name) ENTRY (name) #define END_CHK(name) END (name) /* Since C identifiers are not normally prefixed with an underscore - on this system, the asm identifier `syscall_error' intrudes on the - C name space. Make sure we use an innocuous name. */ + on this system, the asm identifier `syscall_error' intrudes on the + C name space. Make sure we use an innocuous name. */ #define syscall_error __syscall_error #define mcount _mcount #undef PSEUDO_END #define PSEUDO_END(name) \ - END (name) + END (name) /* Local label name for asm code. */ #ifndef L From caccc6da3787d4f328149ccae528328c165d4810 Mon Sep 17 00:00:00 2001 From: alesapin Date: Mon, 15 Mar 2021 12:01:26 +0300 Subject: [PATCH 461/544] Add missing tests --- .../test_optimize_on_insert/__init__.py | 1 + .../test_optimize_on_insert/test.py | 48 +++++++++++++++++++ ...560_optimize_on_insert_zookeeper.reference | 1 + .../01560_optimize_on_insert_zookeeper.sql | 36 ++++++++++++++ 4 files changed, 86 insertions(+) create mode 100644 tests/integration/test_optimize_on_insert/__init__.py create mode 100644 tests/integration/test_optimize_on_insert/test.py create mode 100644 tests/queries/0_stateless/01560_optimize_on_insert_zookeeper.reference create mode 100644 tests/queries/0_stateless/01560_optimize_on_insert_zookeeper.sql diff --git a/tests/integration/test_optimize_on_insert/__init__.py b/tests/integration/test_optimize_on_insert/__init__.py new file mode 100644 index 00000000000..e5a0d9b4834 --- /dev/null +++ b/tests/integration/test_optimize_on_insert/__init__.py @@ -0,0 +1 @@ +#!/usr/bin/env python3 diff --git a/tests/integration/test_optimize_on_insert/test.py b/tests/integration/test_optimize_on_insert/test.py new file mode 100644 index 00000000000..da4e20edf0c --- /dev/null +++ b/tests/integration/test_optimize_on_insert/test.py @@ -0,0 +1,48 @@ +#!/usr/bin/env python3 + +import pytest +from helpers.client import QueryRuntimeException +from helpers.cluster import ClickHouseCluster + +cluster = ClickHouseCluster(__file__) +node1 = cluster.add_instance('node1', with_zookeeper=True) +node2 = cluster.add_instance('node2', with_zookeeper=True) + +@pytest.fixture(scope="module") +def start_cluster(): + try: + cluster.start() + + yield cluster + + finally: + cluster.shutdown() + + +def get_data_files_for_table(node, table_name): + raw_output = node.exec_in_container(["bash", "-c", "ls /var/lib/clickhouse/data/default/{}".format(table_name)]) + return raw_output.strip().split("\n") + +def test_empty_parts_optimize(start_cluster): + for n, node in enumerate([node1, node2]): + node.query(""" + CREATE TABLE empty (key UInt32, val UInt32, date Datetime) + ENGINE=ReplicatedSummingMergeTree('/clickhouse/01560_optimize_on_insert', '{}', val) + PARTITION BY date ORDER BY key; + """.format(n+1)) + + node1.query("INSERT INTO empty VALUES (1, 1, '2020-01-01'), (1, 1, '2020-01-01'), (1, -2, '2020-01-01')") + + node2.query("SYSTEM SYNC REPLICA empty", timeout=15) + + assert node1.query("SELECT * FROM empty") == "" + assert node2.query("SELECT * FROM empty") == "" + + # No other tmp files exists + assert set(get_data_files_for_table(node1, "empty")) == {"detached", "format_version.txt"} + assert set(get_data_files_for_table(node2, "empty")) == {"detached", "format_version.txt"} + + node1.query("INSERT INTO empty VALUES (1, 1, '2020-02-01'), (1, 1, '2020-02-01'), (1, -2, '2020-02-01')", settings={"insert_quorum": 2}) + + assert node1.query("SELECT * FROM empty") == "" + assert node2.query("SELECT * FROM empty") == "" diff --git a/tests/queries/0_stateless/01560_optimize_on_insert_zookeeper.reference b/tests/queries/0_stateless/01560_optimize_on_insert_zookeeper.reference new file mode 100644 index 00000000000..e89c6201fb7 --- /dev/null +++ b/tests/queries/0_stateless/01560_optimize_on_insert_zookeeper.reference @@ -0,0 +1 @@ +Check creating empty parts diff --git a/tests/queries/0_stateless/01560_optimize_on_insert_zookeeper.sql b/tests/queries/0_stateless/01560_optimize_on_insert_zookeeper.sql new file mode 100644 index 00000000000..a98818b2195 --- /dev/null +++ b/tests/queries/0_stateless/01560_optimize_on_insert_zookeeper.sql @@ -0,0 +1,36 @@ +DROP TABLE IF EXISTS empty1; +DROP TABLE IF EXISTS empty2; + +SELECT 'Check creating empty parts'; + +CREATE TABLE empty1 (key UInt32, val UInt32, date Datetime) +ENGINE=ReplicatedSummingMergeTree('/clickhouse/01560_optimize_on_insert', '1', val) +PARTITION BY date ORDER BY key; + +CREATE TABLE empty2 (key UInt32, val UInt32, date Datetime) +ENGINE=ReplicatedSummingMergeTree('/clickhouse/01560_optimize_on_insert', '2', val) +PARTITION BY date ORDER BY key; + +INSERT INTO empty2 VALUES (1, 1, '2020-01-01'), (1, 1, '2020-01-01'), (1, -2, '2020-01-01'); + +SYSTEM SYNC REPLICA empty1; + +SELECT * FROM empty1 ORDER BY key; +SELECT * FROM empty2 ORDER BY key; + +SELECT table, partition, active FROM system.parts where table = 'empty1' and database=currentDatabase() and active = 1; +SELECT table, partition, active FROM system.parts where table = 'empty2' and database=currentDatabase() and active = 1; + +DETACH table empty1; +DETACH table empty2; +ATTACH table empty1; +ATTACH table empty2; + +SELECT * FROM empty1 ORDER BY key; +SELECT * FROM empty2 ORDER BY key; + +SELECT table, partition, active FROM system.parts where table = 'empty1' and database=currentDatabase() and active = 1; +SELECT table, partition, active FROM system.parts where table = 'empty2' and database=currentDatabase() and active = 1; + +DROP TABLE IF EXISTS empty1; +DROP TABLE IF EXISTS empty2; From 1e495705ada1957c64437d0a0387675f01536a2d Mon Sep 17 00:00:00 2001 From: robot-clickhouse Date: Mon, 15 Mar 2021 12:44:13 +0300 Subject: [PATCH 462/544] Update version_date.tsv after release 21.2.6.1 --- utils/list-versions/version_date.tsv | 1 + 1 file changed, 1 insertion(+) diff --git a/utils/list-versions/version_date.tsv b/utils/list-versions/version_date.tsv index 5edbc4bca1a..b13f651f10a 100644 --- a/utils/list-versions/version_date.tsv +++ b/utils/list-versions/version_date.tsv @@ -1,4 +1,5 @@ v21.3.2.5-lts 2021-03-12 +v21.2.6.1-stable 2021-03-15 v21.2.5.5-stable 2021-03-02 v21.2.4.6-stable 2021-02-20 v21.2.3.15-stable 2021-02-14 From 1ede69e27707998cf6ce39cb9ec5925e6717d6f4 Mon Sep 17 00:00:00 2001 From: robot-clickhouse Date: Mon, 15 Mar 2021 14:15:06 +0300 Subject: [PATCH 463/544] Update version_date.tsv after release 21.1.7.1 --- utils/list-versions/version_date.tsv | 1 + 1 file changed, 1 insertion(+) diff --git a/utils/list-versions/version_date.tsv b/utils/list-versions/version_date.tsv index b13f651f10a..687dbc24000 100644 --- a/utils/list-versions/version_date.tsv +++ b/utils/list-versions/version_date.tsv @@ -4,6 +4,7 @@ v21.2.5.5-stable 2021-03-02 v21.2.4.6-stable 2021-02-20 v21.2.3.15-stable 2021-02-14 v21.2.2.8-stable 2021-02-07 +v21.1.7.1-stable 2021-03-15 v21.1.6.13-stable 2021-03-02 v21.1.5.4-stable 2021-02-20 v21.1.4.46-stable 2021-02-14 From 35e0d0f672321d8ab2120dd873f562a39d6310e6 Mon Sep 17 00:00:00 2001 From: Vladimir Chebotarev Date: Mon, 15 Mar 2021 14:17:16 +0300 Subject: [PATCH 464/544] Reverted S3 connection pools. --- .../mergetree-family/mergetree.md | 2 -- .../mergetree-family/mergetree.md | 2 -- src/IO/S3/PocoHTTPClient.cpp | 26 +++++++++++-------- src/IO/S3/PocoHTTPClient.h | 3 +-- 4 files changed, 16 insertions(+), 17 deletions(-) diff --git a/docs/en/engines/table-engines/mergetree-family/mergetree.md b/docs/en/engines/table-engines/mergetree-family/mergetree.md index 52d9111dc90..70cf7f2212e 100644 --- a/docs/en/engines/table-engines/mergetree-family/mergetree.md +++ b/docs/en/engines/table-engines/mergetree-family/mergetree.md @@ -748,7 +748,6 @@ Configuration markup: 10000 5000 - 100 10 1000 /var/lib/clickhouse/disks/s3/ @@ -771,7 +770,6 @@ Optional parameters: - `proxy` — Proxy configuration for S3 endpoint. Each `uri` element inside `proxy` block should contain a proxy URL. - `connect_timeout_ms` — Socket connect timeout in milliseconds. Default value is `10 seconds`. - `request_timeout_ms` — Request timeout in milliseconds. Default value is `5 seconds`. -- `max_connections` — S3 connections pool size. Default value is `100`. - `retry_attempts` — Number of retry attempts in case of failed request. Default value is `10`. - `min_bytes_for_seek` — Minimal number of bytes to use seek operation instead of sequential read. Default value is `1 Mb`. - `metadata_path` — Path on local FS to store metadata files for S3. Default value is `/var/lib/clickhouse/disks//`. diff --git a/docs/ru/engines/table-engines/mergetree-family/mergetree.md b/docs/ru/engines/table-engines/mergetree-family/mergetree.md index bc74b2592b9..b10087e0697 100644 --- a/docs/ru/engines/table-engines/mergetree-family/mergetree.md +++ b/docs/ru/engines/table-engines/mergetree-family/mergetree.md @@ -733,7 +733,6 @@ SETTINGS storage_policy = 'moving_from_ssd_to_hdd' 10000 5000 - 100 10 1000 /var/lib/clickhouse/disks/s3/ @@ -758,7 +757,6 @@ SETTINGS storage_policy = 'moving_from_ssd_to_hdd' - `proxy` — конфигурация прокси-сервера для конечной точки S3. Каждый элемент `uri` внутри блока `proxy` должен содержать URL прокси-сервера. - `connect_timeout_ms` — таймаут подключения к сокету в миллисекундах. Значение по умолчанию: 10 секунд. - `request_timeout_ms` — таймаут выполнения запроса в миллисекундах. Значение по умолчанию: 5 секунд. -- `max_connections` — размер пула соединений S3. Значение по умолчанию: `100`. - `retry_attempts` — число попыток выполнения запроса в случае возникновения ошибки. Значение по умолчанию: `10`. - `min_bytes_for_seek` — минимальное количество байтов, которые используются для операций поиска вместо последовательного чтения. Значение по умолчанию: 1 МБайт. - `metadata_path` — путь к локальному файловому хранилищу для хранения файлов с метаданными для S3. Значение по умолчанию: `/var/lib/clickhouse/disks//`. diff --git a/src/IO/S3/PocoHTTPClient.cpp b/src/IO/S3/PocoHTTPClient.cpp index 471044dd08c..c31c12a1899 100644 --- a/src/IO/S3/PocoHTTPClient.cpp +++ b/src/IO/S3/PocoHTTPClient.cpp @@ -86,7 +86,6 @@ PocoHTTPClient::PocoHTTPClient(const PocoHTTPClientConfiguration & clientConfigu )) , remote_host_filter(clientConfiguration.remote_host_filter) , s3_max_redirects(clientConfiguration.s3_max_redirects) - , max_connections(clientConfiguration.maxConnections) { } @@ -156,19 +155,24 @@ void PocoHTTPClient::makeRequestInternal( for (unsigned int attempt = 0; attempt <= s3_max_redirects; ++attempt) { Poco::URI target_uri(uri); - Poco::URI proxy_uri; - - auto request_configuration = per_request_configuration(request); - if (!request_configuration.proxyHost.empty()) - { - proxy_uri.setScheme(Aws::Http::SchemeMapper::ToString(request_configuration.proxyScheme)); - proxy_uri.setHost(request_configuration.proxyHost); - proxy_uri.setPort(request_configuration.proxyPort); - } /// Reverse proxy can replace host header with resolved ip address instead of host name. /// This can lead to request signature difference on S3 side. - auto session = makePooledHTTPSession(target_uri, proxy_uri, timeouts, max_connections, false); + auto session = makeHTTPSession(target_uri, timeouts, false); + + auto request_configuration = per_request_configuration(request); + + if (!request_configuration.proxyHost.empty()) + { + bool use_tunnel = request_configuration.proxyScheme == Aws::Http::Scheme::HTTP && target_uri.getScheme() == "https"; + + session->setProxy( + request_configuration.proxyHost, + request_configuration.proxyPort, + Aws::Http::SchemeMapper::ToString(request_configuration.proxyScheme), + use_tunnel + ); + } Poco::Net::HTTPRequest poco_request(Poco::Net::HTTPRequest::HTTP_1_1); diff --git a/src/IO/S3/PocoHTTPClient.h b/src/IO/S3/PocoHTTPClient.h index 918943a413c..da6c4dd5985 100644 --- a/src/IO/S3/PocoHTTPClient.h +++ b/src/IO/S3/PocoHTTPClient.h @@ -40,7 +40,7 @@ private: class PocoHTTPResponse : public Aws::Http::Standard::StandardHttpResponse { public: - using SessionPtr = PooledHTTPSessionPtr; + using SessionPtr = HTTPSessionPtr; PocoHTTPResponse(const std::shared_ptr request) : Aws::Http::Standard::StandardHttpResponse(request) @@ -91,7 +91,6 @@ private: ConnectionTimeouts timeouts; const RemoteHostFilter & remote_host_filter; unsigned int s3_max_redirects; - unsigned int max_connections; }; } From 307b89f6b0101268507a284ffa35f6a2682a569d Mon Sep 17 00:00:00 2001 From: alesapin Date: Mon, 15 Mar 2021 14:20:11 +0300 Subject: [PATCH 465/544] fix skip list --- tests/queries/skip_list.json | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tests/queries/skip_list.json b/tests/queries/skip_list.json index 974ef48ef3c..5dc0aef0081 100644 --- a/tests/queries/skip_list.json +++ b/tests/queries/skip_list.json @@ -111,6 +111,7 @@ "memory_tracking", "memory_usage", "live_view", + "01761_alter_decimal_zookeeper", "01720_type_map_and_casts", "01413_alter_update_supertype", "01149_zookeeper_mutation_stuck_after_replace_partition", @@ -753,6 +754,7 @@ "01700_system_zookeeper_path_in", "01715_background_checker_blather_zookeeper", "01747_alter_partition_key_enum_zookeeper", + "01761_alter_decimal_zookeeper", "attach", "ddl_dictionaries", "dictionary", From 31ec24255e74098aa8e39723cfd14781770d9ce9 Mon Sep 17 00:00:00 2001 From: alesapin Date: Mon, 15 Mar 2021 14:24:52 +0300 Subject: [PATCH 466/544] Fix skip list --- tests/queries/skip_list.json | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tests/queries/skip_list.json b/tests/queries/skip_list.json index 974ef48ef3c..90564d354e5 100644 --- a/tests/queries/skip_list.json +++ b/tests/queries/skip_list.json @@ -111,6 +111,7 @@ "memory_tracking", "memory_usage", "live_view", + "01560_optimize_on_insert_zookeeper", "01720_type_map_and_casts", "01413_alter_update_supertype", "01149_zookeeper_mutation_stuck_after_replace_partition", @@ -736,6 +737,7 @@ "01530_drop_database_atomic_sync", "01541_max_memory_usage_for_user_long", "01542_dictionary_load_exception_race", + "01560_optimize_on_insert_zookeeper", "01575_disable_detach_table_of_dictionary", "01593_concurrent_alter_mutations_kill", "01593_concurrent_alter_mutations_kill_many_replicas", From c2b398b39a5bada456e2c8d0d2fd84b7c1290a8a Mon Sep 17 00:00:00 2001 From: Vasily Nemkov Date: Mon, 15 Mar 2021 13:25:22 +0200 Subject: [PATCH 467/544] Documnentation on OPTIMIZE DEDUPLICATE BY expression. --- docs/en/sql-reference/statements/optimize.md | 66 ++++++++++++++++++- .../sql-reference/statements/select/index.md | 3 + 2 files changed, 67 insertions(+), 2 deletions(-) diff --git a/docs/en/sql-reference/statements/optimize.md b/docs/en/sql-reference/statements/optimize.md index 9b16a12d2e2..ea1fac90466 100644 --- a/docs/en/sql-reference/statements/optimize.md +++ b/docs/en/sql-reference/statements/optimize.md @@ -6,7 +6,7 @@ toc_title: OPTIMIZE # OPTIMIZE Statement {#misc_operations-optimize} ``` sql -OPTIMIZE TABLE [db.]name [ON CLUSTER cluster] [PARTITION partition | PARTITION ID 'partition_id'] [FINAL] [DEDUPLICATE] +OPTIMIZE TABLE [db.]name [ON CLUSTER cluster] [PARTITION partition | PARTITION ID 'partition_id'] [FINAL] [DEDUPLICATE [BY expression]] ``` This query tries to initialize an unscheduled merge of data parts for tables with a table engine from the [MergeTree](../../engines/table-engines/mergetree-family/mergetree.md) family. @@ -18,7 +18,69 @@ When `OPTIMIZE` is used with the [ReplicatedMergeTree](../../engines/table-engin - If `OPTIMIZE` doesn’t perform a merge for any reason, it doesn’t notify the client. To enable notifications, use the [optimize_throw_if_noop](../../operations/settings/settings.md#setting-optimize_throw_if_noop) setting. - If you specify a `PARTITION`, only the specified partition is optimized. [How to set partition expression](../../sql-reference/statements/alter/index.md#alter-how-to-specify-part-expr). - If you specify `FINAL`, optimization is performed even when all the data is already in one part. Also merge is forced even if concurrent merges are performed. -- If you specify `DEDUPLICATE`, then completely identical rows will be deduplicated (all columns are compared), it makes sense only for the MergeTree engine. +- If you specify `DEDUPLICATE`, then completely identical rows (unless by-clause is specified) will be deduplicated (all columns are compared), it makes sense only for the MergeTree engine. + + +### BY expression {#by-expression} + +If you want to perform deduplication on custom set of columns rather than on all, you can specify list of columns explicitly or use any combination of [`*`](../../../sql-reference/statements/select/index.md#asterisk), [`COLUMNS`](../../../sql-reference/statements/select/index.md#columns-expression) or [`EXCEPT`](../../../sql-reference/statements/select/index.md#except-modifier) expressions. The explictly written or implicitly expanded list of columns must include all columns specified in row ordering expression (both primary and sorting keys) and partitioning expression (partitioning key). + +Note that `*` behaves just like in `SELECT`: `MATERIALIZED`, and `ALIAS` columns are not used for expansion. +Also, it is an error to specify empty list of columns, or write an expression that results in an empty list of columns, or deduplicate by an ALIAS column. + +``` sql +OPTIMIZE TABLE table DEDUPLICATE; -- the old one +OPTIMIZE TABLE table DEDUPLICATE BY *; -- not the same as the old one, excludes MATERIALIZED columns (see the note above) +OPTIMIZE TABLE table DEDUPLICATE BY * EXCEPT colX; +OPTIMIZE TABLE table DEDUPLICATE BY * EXCEPT (colX, colY); +OPTIMIZE TABLE table DEDUPLICATE BY col1,col2,col3; +OPTIMIZE TABLE table DEDUPLICATE BY COLUMNS('column-matched-by-regex'); +OPTIMIZE TABLE table DEDUPLICATE BY COLUMNS('column-matched-by-regex') EXCEPT colX; +OPTIMIZE TABLE table DEDUPLICATE BY COLUMNS('column-matched-by-regex') EXCEPT (colX, colY); +``` + +**Example:** + +A silly synthetic table. +``` sql +CREATE TABLE example ( + primary_key Int32, + secondary_key Int32, + value UInt32, + partition_key UInt32, + materialized_value UInt32 MATERIALIZED 12345, + aliased_value UInt32 ALIAS 2, + PRIMARY KEY primary_key +) ENGINE=MergeTree +PARTITION BY partition_key +ORDER BY (primary_key, secondary_key); +``` + +``` sql +-- The 'old' deduplicate, all columns are taken into account, i.e. row is removed only if all values in all columns are equal to corresponding values in previous row. +OPTIMIZE TABLE example FINAL DEDUPLICATE; +``` + +``` sql +-- Deduplicate by all columns that are not `ALIAS` or `MATERIALIZED`: `primary_key`, `secondary_key`, `value`, `partition_key`, and `materialized_value` columns. +OPTIMIZE TABLE example FINAL DEDUPLICATE BY *; +``` + +``` sql +-- Deduplicate by all columns that are not `ALIAS` or `MATERIALIZED` and explicitly not `materialized_value`: `primary_key`, `secondary_key`, `value`, and `partition_key` columns. +OPTIMIZE TABLE example FINAL DEDUPLICATE BY * EXCEPT materialized_value; +``` + +``` sql +-- Deduplicate explicitly by `primary_key`, `secondary_key`, and `partition_key` columns. +OPTIMIZE TABLE example FINAL DEDUPLICATE BY primary_key, secondary_key, partition_key; +``` + +``` sql +-- Deduplicate by any column matching a regex: `primary_key`, `secondary_key`, and `partition_key` columns. +OPTIMIZE TABLE example FINAL DEDUPLICATE BY COLUMNS('.*_key'); +``` + !!! warning "Warning" `OPTIMIZE` can’t fix the “Too many parts” error. diff --git a/docs/en/sql-reference/statements/select/index.md b/docs/en/sql-reference/statements/select/index.md index e99ebef838c..ada4699c224 100644 --- a/docs/en/sql-reference/statements/select/index.md +++ b/docs/en/sql-reference/statements/select/index.md @@ -57,6 +57,9 @@ Specifics of each optional clause are covered in separate sections, which are li If you want to include all columns in the result, use the asterisk (`*`) symbol. For example, `SELECT * FROM ...`. + +### COLUMNS expression {#columns-expression} + To match some columns in the result with a [re2](https://en.wikipedia.org/wiki/RE2_(software)) regular expression, you can use the `COLUMNS` expression. ``` sql From 63bf066e2d5164671a5c6ff492ea008e86eaadc3 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Mon, 15 Mar 2021 14:27:49 +0300 Subject: [PATCH 468/544] Improve links matching --- docs/tools/single_page.py | 17 +++++++++++------ 1 file changed, 11 insertions(+), 6 deletions(-) diff --git a/docs/tools/single_page.py b/docs/tools/single_page.py index 27628962271..af38c3fb8d7 100644 --- a/docs/tools/single_page.py +++ b/docs/tools/single_page.py @@ -30,11 +30,17 @@ def generate_anchor_from_path(path): def replace_link(match, path): - link = match.group(1) + title = match.group(1) + link = match.group(2) + + # Not a relative link + if link.startswith('http'): + return match.group(0) + if link.endswith('/'): link = link[0:-1] + '.md' - return '(#{})'.format(generate_anchor_from_path(os.path.normpath(os.path.join(os.path.dirname(path), link)))) + return '[{}](#{})'.format(title, generate_anchor_from_path(os.path.normpath(os.path.join(os.path.dirname(path), link)))) # Concatenates Markdown files to a single file. @@ -52,8 +58,7 @@ def concatenate(lang, docs_path, single_page_file, nav): logging.debug('Concatenating: ' + ', '.join(files_to_concatenate)) assert files_count > 0, f'Empty single-page for {lang}' - # (../anything) or (../anything#anchor) or (xyz-abc.md) or (xyz-abc.md#anchor) - relative_link_regexp = re.compile(r'\((\.\./[^)#]+|[\w\-]+\.md)(?:#[^\)]*)?\)') + link_regexp = re.compile(r'(\[[^\]]+\])\(([^)#]+)(?:#[^\)]+)?\)') for path in files_to_concatenate: try: @@ -75,9 +80,9 @@ def concatenate(lang, docs_path, single_page_file, nav): # Replace links within the docs. - if re.search(relative_link_regexp, line): + if re.search(link_regexp, line): line = re.sub( - relative_link_regexp, + link_regexp, lambda match: replace_link(match, path), line) From 4ad6cc331374fe93332949a20ea1be777503194b Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Mon, 15 Mar 2021 14:28:37 +0300 Subject: [PATCH 469/544] Improve links matching --- docs/tools/single_page.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/tools/single_page.py b/docs/tools/single_page.py index af38c3fb8d7..f885a84ec89 100644 --- a/docs/tools/single_page.py +++ b/docs/tools/single_page.py @@ -40,7 +40,7 @@ def replace_link(match, path): if link.endswith('/'): link = link[0:-1] + '.md' - return '[{}](#{})'.format(title, generate_anchor_from_path(os.path.normpath(os.path.join(os.path.dirname(path), link)))) + return '{}(#{})'.format(title, generate_anchor_from_path(os.path.normpath(os.path.join(os.path.dirname(path), link)))) # Concatenates Markdown files to a single file. From 0a6a80fa5293ee282e34c1cd326f96f0e122d65b Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Mon, 15 Mar 2021 14:30:56 +0300 Subject: [PATCH 470/544] Fix broken links --- docs/ru/getting-started/playground.md | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/docs/ru/getting-started/playground.md b/docs/ru/getting-started/playground.md index 86a5cd5272c..b51a9b2b436 100644 --- a/docs/ru/getting-started/playground.md +++ b/docs/ru/getting-started/playground.md @@ -36,10 +36,10 @@ ClickHouse Playground дает возможность поработать с [ - запрещены INSERT запросы Также установлены следующие опции: -- [max_result_bytes=10485760](../operations/settings/query_complexity/#max-result-bytes) -- [max_result_rows=2000](../operations/settings/query_complexity/#setting-max_result_rows) -- [result_overflow_mode=break](../operations/settings/query_complexity/#result-overflow-mode) -- [max_execution_time=60000](../operations/settings/query_complexity/#max-execution-time) +- [max_result_bytes=10485760](../operations/settings/query-complexity.md#max-result-bytes) +- [max_result_rows=2000](../operations/settings/query-complexity.md#setting-max_result_rows) +- [result_overflow_mode=break](../operations/settings/query-complexity.md#result-overflow-mode) +- [max_execution_time=60000](../operations/settings/query-complexity.md#max-execution-time) ## Примеры {#examples} From 8850856276a42aaa2ed4810009cd4f63af06d4f7 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Mon, 15 Mar 2021 14:32:26 +0300 Subject: [PATCH 471/544] Remove "Original article" (must be automated instead) --- docs/ru/commercial/cloud.md | 1 - docs/ru/development/style.md | 1 - docs/ru/engines/database-engines/index.md | 1 - docs/ru/engines/database-engines/lazy.md | 1 - docs/ru/engines/database-engines/materialize-mysql.md | 1 - docs/ru/engines/table-engines/index.md | 1 - docs/ru/engines/table-engines/integrations/embedded-rocksdb.md | 1 - docs/ru/engines/table-engines/integrations/hdfs.md | 1 - docs/ru/engines/table-engines/integrations/index.md | 1 - docs/ru/engines/table-engines/integrations/jdbc.md | 1 - docs/ru/engines/table-engines/integrations/kafka.md | 1 - docs/ru/engines/table-engines/integrations/mysql.md | 1 - docs/ru/engines/table-engines/integrations/odbc.md | 1 - docs/ru/engines/table-engines/integrations/postgresql.md | 1 - docs/ru/engines/table-engines/integrations/rabbitmq.md | 1 - docs/ru/engines/table-engines/integrations/s3.md | 1 - docs/ru/engines/table-engines/log-family/index.md | 1 - docs/ru/engines/table-engines/log-family/log.md | 1 - docs/ru/engines/table-engines/log-family/stripelog.md | 1 - docs/ru/engines/table-engines/log-family/tinylog.md | 1 - .../table-engines/mergetree-family/aggregatingmergetree.md | 1 - .../table-engines/mergetree-family/collapsingmergetree.md | 1 - .../table-engines/mergetree-family/custom-partitioning-key.md | 1 - .../engines/table-engines/mergetree-family/graphitemergetree.md | 1 - docs/ru/engines/table-engines/mergetree-family/mergetree.md | 1 - .../engines/table-engines/mergetree-family/replacingmergetree.md | 1 - docs/ru/engines/table-engines/mergetree-family/replication.md | 1 - .../engines/table-engines/mergetree-family/summingmergetree.md | 1 - .../mergetree-family/versionedcollapsingmergetree.md | 1 - docs/ru/engines/table-engines/special/buffer.md | 1 - docs/ru/engines/table-engines/special/dictionary.md | 1 - docs/ru/engines/table-engines/special/distributed.md | 1 - docs/ru/engines/table-engines/special/external-data.md | 1 - docs/ru/engines/table-engines/special/file.md | 1 - docs/ru/engines/table-engines/special/index.md | 1 - docs/ru/engines/table-engines/special/join.md | 1 - docs/ru/engines/table-engines/special/materializedview.md | 1 - docs/ru/engines/table-engines/special/memory.md | 1 - docs/ru/engines/table-engines/special/merge.md | 1 - docs/ru/engines/table-engines/special/null.md | 1 - docs/ru/engines/table-engines/special/set.md | 1 - docs/ru/engines/table-engines/special/url.md | 1 - docs/ru/engines/table-engines/special/view.md | 1 - docs/ru/getting-started/example-datasets/amplab-benchmark.md | 1 - docs/ru/getting-started/example-datasets/brown-benchmark.md | 1 - docs/ru/getting-started/example-datasets/criteo.md | 1 - docs/ru/getting-started/example-datasets/index.md | 1 - docs/ru/getting-started/example-datasets/nyc-taxi.md | 1 - docs/ru/getting-started/example-datasets/ontime.md | 1 - docs/ru/getting-started/example-datasets/wikistat.md | 1 - docs/ru/getting-started/index.md | 1 - docs/ru/getting-started/install.md | 1 - docs/ru/index.md | 1 - docs/ru/interfaces/cli.md | 1 - docs/ru/interfaces/cpp.md | 1 - docs/ru/interfaces/formats.md | 1 - docs/ru/interfaces/http.md | 1 - docs/ru/interfaces/index.md | 1 - docs/ru/interfaces/jdbc.md | 1 - docs/ru/interfaces/odbc.md | 1 - docs/ru/interfaces/tcp.md | 1 - docs/ru/interfaces/third-party/client-libraries.md | 1 - docs/ru/interfaces/third-party/gui.md | 1 - docs/ru/interfaces/third-party/index.md | 1 - docs/ru/interfaces/third-party/integrations.md | 1 - docs/ru/interfaces/third-party/proxy.md | 1 - docs/ru/introduction/distinctive-features.md | 1 - docs/ru/introduction/history.md | 1 - docs/ru/introduction/info.md | 1 - docs/ru/introduction/performance.md | 1 - docs/ru/operations/access-rights.md | 1 - docs/ru/operations/backup.md | 1 - docs/ru/operations/caches.md | 1 - docs/ru/operations/configuration-files.md | 1 - docs/ru/operations/index.md | 1 - docs/ru/operations/monitoring.md | 1 - docs/ru/operations/opentelemetry.md | 1 - docs/ru/operations/quotas.md | 1 - docs/ru/operations/server-configuration-parameters/index.md | 1 - docs/ru/operations/server-configuration-parameters/settings.md | 1 - docs/ru/operations/settings/constraints-on-settings.md | 1 - docs/ru/operations/settings/index.md | 1 - docs/ru/operations/settings/permissions-for-queries.md | 1 - docs/ru/operations/settings/query-complexity.md | 1 - docs/ru/operations/settings/settings-profiles.md | 1 - docs/ru/operations/settings/settings-users.md | 1 - docs/ru/operations/settings/settings.md | 1 - docs/ru/operations/system-tables/asynchronous_metric_log.md | 1 - docs/ru/operations/system-tables/asynchronous_metrics.md | 1 - docs/ru/operations/system-tables/clusters.md | 1 - docs/ru/operations/system-tables/columns.md | 1 - docs/ru/operations/system-tables/contributors.md | 1 - docs/ru/operations/system-tables/current-roles.md | 1 - docs/ru/operations/system-tables/data_type_families.md | 1 - docs/ru/operations/system-tables/databases.md | 1 - docs/ru/operations/system-tables/detached_parts.md | 1 - docs/ru/operations/system-tables/dictionaries.md | 1 - docs/ru/operations/system-tables/disks.md | 1 - docs/ru/operations/system-tables/distributed_ddl_queue.md | 1 - docs/ru/operations/system-tables/distribution_queue.md | 1 - docs/ru/operations/system-tables/enabled-roles.md | 1 - docs/ru/operations/system-tables/events.md | 1 - docs/ru/operations/system-tables/functions.md | 1 - docs/ru/operations/system-tables/grants.md | 1 - docs/ru/operations/system-tables/graphite_retentions.md | 1 - docs/ru/operations/system-tables/index.md | 1 - docs/ru/operations/system-tables/licenses.md | 1 - docs/ru/operations/system-tables/merges.md | 1 - docs/ru/operations/system-tables/metric_log.md | 1 - docs/ru/operations/system-tables/metrics.md | 1 - docs/ru/operations/system-tables/mutations.md | 1 - docs/ru/operations/system-tables/numbers.md | 1 - docs/ru/operations/system-tables/numbers_mt.md | 1 - docs/ru/operations/system-tables/one.md | 1 - docs/ru/operations/system-tables/opentelemetry_span_log.md | 1 - docs/ru/operations/system-tables/part_log.md | 1 - docs/ru/operations/system-tables/parts.md | 1 - docs/ru/operations/system-tables/parts_columns.md | 1 - docs/ru/operations/system-tables/processes.md | 1 - docs/ru/operations/system-tables/query_log.md | 1 - docs/ru/operations/system-tables/query_thread_log.md | 1 - docs/ru/operations/system-tables/quota_limits.md | 1 - docs/ru/operations/system-tables/quota_usage.md | 1 - docs/ru/operations/system-tables/quotas.md | 1 - docs/ru/operations/system-tables/quotas_usage.md | 1 - docs/ru/operations/system-tables/replicas.md | 1 - docs/ru/operations/system-tables/replicated_fetches.md | 1 - docs/ru/operations/system-tables/replication_queue.md | 1 - docs/ru/operations/system-tables/role-grants.md | 1 - docs/ru/operations/system-tables/roles.md | 1 - docs/ru/operations/system-tables/row_policies.md | 1 - docs/ru/operations/system-tables/settings.md | 1 - docs/ru/operations/system-tables/settings_profile_elements.md | 1 - docs/ru/operations/system-tables/settings_profiles.md | 1 - docs/ru/operations/system-tables/stack_trace.md | 1 - docs/ru/operations/system-tables/storage_policies.md | 1 - docs/ru/operations/system-tables/table_engines.md | 1 - docs/ru/operations/system-tables/tables.md | 1 - docs/ru/operations/system-tables/text_log.md | 1 - docs/ru/operations/system-tables/trace_log.md | 1 - docs/ru/operations/system-tables/users.md | 1 - docs/ru/operations/system-tables/zookeeper.md | 1 - docs/ru/operations/tips.md | 1 - docs/ru/operations/utilities/clickhouse-benchmark.md | 1 - docs/ru/operations/utilities/clickhouse-copier.md | 1 - docs/ru/operations/utilities/clickhouse-local.md | 1 - docs/ru/operations/utilities/index.md | 1 - docs/ru/sql-reference/aggregate-functions/combinators.md | 1 - docs/ru/sql-reference/aggregate-functions/index.md | 1 - .../ru/sql-reference/aggregate-functions/parametric-functions.md | 1 - docs/ru/sql-reference/aggregate-functions/reference/any.md | 1 - docs/ru/sql-reference/aggregate-functions/reference/anyheavy.md | 1 - docs/ru/sql-reference/aggregate-functions/reference/anylast.md | 1 - docs/ru/sql-reference/aggregate-functions/reference/argmax.md | 1 - docs/ru/sql-reference/aggregate-functions/reference/argmin.md | 1 - docs/ru/sql-reference/aggregate-functions/reference/avg.md | 1 - .../sql-reference/aggregate-functions/reference/avgweighted.md | 1 - docs/ru/sql-reference/aggregate-functions/reference/corr.md | 1 - docs/ru/sql-reference/aggregate-functions/reference/count.md | 1 - docs/ru/sql-reference/aggregate-functions/reference/covarpop.md | 1 - docs/ru/sql-reference/aggregate-functions/reference/covarsamp.md | 1 - .../ru/sql-reference/aggregate-functions/reference/grouparray.md | 1 - .../aggregate-functions/reference/grouparrayinsertat.md | 1 - .../aggregate-functions/reference/grouparraymovingavg.md | 1 - .../aggregate-functions/reference/grouparraymovingsum.md | 1 - .../sql-reference/aggregate-functions/reference/groupbitand.md | 1 - .../sql-reference/aggregate-functions/reference/groupbitmap.md | 1 - .../ru/sql-reference/aggregate-functions/reference/groupbitor.md | 1 - .../sql-reference/aggregate-functions/reference/groupbitxor.md | 1 - .../aggregate-functions/reference/groupuniqarray.md | 1 - docs/ru/sql-reference/aggregate-functions/reference/index.md | 1 - docs/ru/sql-reference/aggregate-functions/reference/kurtpop.md | 1 - docs/ru/sql-reference/aggregate-functions/reference/kurtsamp.md | 1 - .../aggregate-functions/reference/mannwhitneyutest.md | 1 - docs/ru/sql-reference/aggregate-functions/reference/max.md | 1 - docs/ru/sql-reference/aggregate-functions/reference/median.md | 1 - docs/ru/sql-reference/aggregate-functions/reference/min.md | 1 - docs/ru/sql-reference/aggregate-functions/reference/quantile.md | 1 - .../aggregate-functions/reference/quantiledeterministic.md | 1 - .../sql-reference/aggregate-functions/reference/quantileexact.md | 1 - .../aggregate-functions/reference/quantileexactweighted.md | 1 - docs/ru/sql-reference/aggregate-functions/reference/quantiles.md | 1 - .../aggregate-functions/reference/quantiletdigest.md | 1 - .../aggregate-functions/reference/quantiletdigestweighted.md | 1 - .../aggregate-functions/reference/quantiletiming.md | 1 - .../aggregate-functions/reference/quantiletimingweighted.md | 1 - .../aggregate-functions/reference/simplelinearregression.md | 1 - docs/ru/sql-reference/aggregate-functions/reference/skewpop.md | 1 - docs/ru/sql-reference/aggregate-functions/reference/skewsamp.md | 1 - docs/ru/sql-reference/aggregate-functions/reference/stddevpop.md | 1 - .../ru/sql-reference/aggregate-functions/reference/stddevsamp.md | 1 - .../aggregate-functions/reference/stochasticlinearregression.md | 1 - .../reference/stochasticlogisticregression.md | 1 - .../sql-reference/aggregate-functions/reference/studentttest.md | 1 - docs/ru/sql-reference/aggregate-functions/reference/sum.md | 1 - docs/ru/sql-reference/aggregate-functions/reference/summap.md | 1 - .../aggregate-functions/reference/sumwithoverflow.md | 1 - docs/ru/sql-reference/aggregate-functions/reference/topk.md | 1 - .../sql-reference/aggregate-functions/reference/topkweighted.md | 1 - docs/ru/sql-reference/aggregate-functions/reference/uniq.md | 1 - .../sql-reference/aggregate-functions/reference/uniqcombined.md | 1 - .../aggregate-functions/reference/uniqcombined64.md | 1 - docs/ru/sql-reference/aggregate-functions/reference/uniqexact.md | 1 - docs/ru/sql-reference/aggregate-functions/reference/uniqhll12.md | 1 - docs/ru/sql-reference/aggregate-functions/reference/varpop.md | 1 - docs/ru/sql-reference/aggregate-functions/reference/varsamp.md | 1 - .../ru/sql-reference/aggregate-functions/reference/welchttest.md | 1 - docs/ru/sql-reference/data-types/aggregatefunction.md | 1 - docs/ru/sql-reference/data-types/array.md | 1 - docs/ru/sql-reference/data-types/boolean.md | 1 - docs/ru/sql-reference/data-types/date.md | 1 - docs/ru/sql-reference/data-types/datetime.md | 1 - docs/ru/sql-reference/data-types/decimal.md | 1 - docs/ru/sql-reference/data-types/domains/index.md | 1 - docs/ru/sql-reference/data-types/domains/ipv4.md | 1 - docs/ru/sql-reference/data-types/domains/ipv6.md | 1 - docs/ru/sql-reference/data-types/enum.md | 1 - docs/ru/sql-reference/data-types/fixedstring.md | 1 - docs/ru/sql-reference/data-types/float.md | 1 - docs/ru/sql-reference/data-types/geo.md | 1 - docs/ru/sql-reference/data-types/index.md | 1 - docs/ru/sql-reference/data-types/int-uint.md | 1 - docs/ru/sql-reference/data-types/lowcardinality.md | 1 - docs/ru/sql-reference/data-types/multiword-types.md | 1 - docs/ru/sql-reference/data-types/nested-data-structures/index.md | 1 - .../ru/sql-reference/data-types/nested-data-structures/nested.md | 1 - docs/ru/sql-reference/data-types/nullable.md | 1 - docs/ru/sql-reference/data-types/simpleaggregatefunction.md | 1 - .../ru/sql-reference/data-types/special-data-types/expression.md | 1 - docs/ru/sql-reference/data-types/special-data-types/index.md | 1 - docs/ru/sql-reference/data-types/special-data-types/nothing.md | 1 - docs/ru/sql-reference/data-types/special-data-types/set.md | 1 - docs/ru/sql-reference/data-types/string.md | 1 - docs/ru/sql-reference/data-types/tuple.md | 1 - .../external-dictionaries/external-dicts-dict-hierarchical.md | 1 - .../external-dictionaries/external-dicts-dict-layout.md | 1 - .../external-dictionaries/external-dicts-dict-lifetime.md | 1 - .../external-dictionaries/external-dicts-dict-sources.md | 1 - .../external-dictionaries/external-dicts-dict-structure.md | 1 - .../dictionaries/external-dictionaries/external-dicts-dict.md | 1 - .../dictionaries/external-dictionaries/external-dicts.md | 1 - docs/ru/sql-reference/dictionaries/index.md | 1 - docs/ru/sql-reference/dictionaries/internal-dicts.md | 1 - docs/ru/sql-reference/distributed-ddl.md | 1 - docs/ru/sql-reference/functions/arithmetic-functions.md | 1 - docs/ru/sql-reference/functions/array-functions.md | 1 - docs/ru/sql-reference/functions/array-join.md | 1 - docs/ru/sql-reference/functions/bit-functions.md | 1 - docs/ru/sql-reference/functions/bitmap-functions.md | 1 - docs/ru/sql-reference/functions/comparison-functions.md | 1 - docs/ru/sql-reference/functions/conditional-functions.md | 1 - docs/ru/sql-reference/functions/date-time-functions.md | 1 - docs/ru/sql-reference/functions/encoding-functions.md | 1 - docs/ru/sql-reference/functions/ext-dict-functions.md | 1 - docs/ru/sql-reference/functions/functions-for-nulls.md | 1 - docs/ru/sql-reference/functions/geo/coordinates.md | 1 - docs/ru/sql-reference/functions/geo/geohash.md | 1 - docs/ru/sql-reference/functions/geo/h3.md | 1 - docs/ru/sql-reference/functions/geo/index.md | 1 - docs/ru/sql-reference/functions/hash-functions.md | 1 - docs/ru/sql-reference/functions/in-functions.md | 1 - docs/ru/sql-reference/functions/index.md | 1 - docs/ru/sql-reference/functions/ip-address-functions.md | 1 - docs/ru/sql-reference/functions/json-functions.md | 1 - docs/ru/sql-reference/functions/logical-functions.md | 1 - docs/ru/sql-reference/functions/math-functions.md | 1 - docs/ru/sql-reference/functions/other-functions.md | 1 - docs/ru/sql-reference/functions/random-functions.md | 1 - docs/ru/sql-reference/functions/rounding-functions.md | 1 - docs/ru/sql-reference/functions/splitting-merging-functions.md | 1 - docs/ru/sql-reference/functions/string-functions.md | 1 - docs/ru/sql-reference/functions/string-replace-functions.md | 1 - docs/ru/sql-reference/functions/string-search-functions.md | 1 - docs/ru/sql-reference/functions/tuple-functions.md | 1 - docs/ru/sql-reference/functions/tuple-map-functions.md | 1 - docs/ru/sql-reference/functions/type-conversion-functions.md | 1 - docs/ru/sql-reference/functions/url-functions.md | 1 - docs/ru/sql-reference/functions/ym-dict-functions.md | 1 - docs/ru/sql-reference/index.md | 1 - docs/ru/sql-reference/operators/index.md | 1 - docs/ru/sql-reference/statements/alter/column.md | 1 - docs/ru/sql-reference/statements/alter/constraint.md | 1 - docs/ru/sql-reference/statements/alter/delete.md | 1 - docs/ru/sql-reference/statements/alter/index.md | 1 - docs/ru/sql-reference/statements/alter/index/index.md | 1 - docs/ru/sql-reference/statements/alter/order-by.md | 1 - docs/ru/sql-reference/statements/alter/partition.md | 1 - docs/ru/sql-reference/statements/alter/quota.md | 1 - docs/ru/sql-reference/statements/alter/role.md | 1 - docs/ru/sql-reference/statements/alter/row-policy.md | 1 - docs/ru/sql-reference/statements/alter/settings-profile.md | 1 - docs/ru/sql-reference/statements/alter/ttl.md | 1 - docs/ru/sql-reference/statements/alter/update.md | 1 - docs/ru/sql-reference/statements/alter/user.md | 1 - docs/ru/sql-reference/statements/attach.md | 1 - docs/ru/sql-reference/statements/check-table.md | 1 - docs/ru/sql-reference/statements/create/database.md | 1 - docs/ru/sql-reference/statements/create/dictionary.md | 1 - docs/ru/sql-reference/statements/create/index.md | 1 - docs/ru/sql-reference/statements/create/quota.md | 1 - docs/ru/sql-reference/statements/create/role.md | 1 - docs/ru/sql-reference/statements/create/row-policy.md | 1 - docs/ru/sql-reference/statements/create/settings-profile.md | 1 - docs/ru/sql-reference/statements/create/table.md | 1 - docs/ru/sql-reference/statements/create/user.md | 1 - docs/ru/sql-reference/statements/create/view.md | 1 - docs/ru/sql-reference/statements/describe-table.md | 1 - docs/ru/sql-reference/statements/detach.md | 1 - docs/ru/sql-reference/statements/drop.md | 1 - docs/ru/sql-reference/statements/exists.md | 1 - docs/ru/sql-reference/statements/grant.md | 1 - docs/ru/sql-reference/statements/insert-into.md | 1 - docs/ru/sql-reference/statements/kill.md | 1 - docs/ru/sql-reference/statements/misc.md | 1 - docs/ru/sql-reference/statements/optimize.md | 1 - docs/ru/sql-reference/statements/rename.md | 1 - docs/ru/sql-reference/statements/revoke.md | 1 - docs/ru/sql-reference/statements/select/all.md | 1 - docs/ru/sql-reference/statements/select/index.md | 1 - docs/ru/sql-reference/statements/select/order-by.md | 1 - docs/ru/sql-reference/statements/select/union.md | 1 - docs/ru/sql-reference/statements/select/with.md | 1 - docs/ru/sql-reference/statements/set-role.md | 1 - docs/ru/sql-reference/statements/set.md | 1 - docs/ru/sql-reference/statements/show.md | 1 - docs/ru/sql-reference/statements/system.md | 1 - docs/ru/sql-reference/statements/truncate.md | 1 - docs/ru/sql-reference/statements/use.md | 1 - docs/ru/sql-reference/syntax.md | 1 - docs/ru/sql-reference/table-functions/file.md | 1 - docs/ru/sql-reference/table-functions/generate.md | 1 - docs/ru/sql-reference/table-functions/hdfs.md | 1 - docs/ru/sql-reference/table-functions/index.md | 1 - docs/ru/sql-reference/table-functions/input.md | 1 - docs/ru/sql-reference/table-functions/jdbc.md | 1 - docs/ru/sql-reference/table-functions/merge.md | 1 - docs/ru/sql-reference/table-functions/mysql.md | 1 - docs/ru/sql-reference/table-functions/numbers.md | 1 - docs/ru/sql-reference/table-functions/odbc.md | 1 - docs/ru/sql-reference/table-functions/remote.md | 1 - docs/ru/sql-reference/table-functions/url.md | 1 - docs/ru/sql-reference/table-functions/view.md | 1 - docs/ru/whats-new/extended-roadmap.md | 1 - docs/ru/whats-new/security-changelog.md | 1 - 344 files changed, 344 deletions(-) diff --git a/docs/ru/commercial/cloud.md b/docs/ru/commercial/cloud.md index 8023f738c70..610f0f00a99 100644 --- a/docs/ru/commercial/cloud.md +++ b/docs/ru/commercial/cloud.md @@ -29,4 +29,3 @@ toc_title: "Поставщики облачных услуг ClickHouse" - cross-az масштабирование для повышения производительности и обеспечения высокой доступности - встроенный мониторинг и редактор SQL-запросов -{## [Оригинальная статья](https://clickhouse.tech/docs/ru/commercial/cloud/) ##} diff --git a/docs/ru/development/style.md b/docs/ru/development/style.md index 72607ca6bad..f08ecc3c4c7 100644 --- a/docs/ru/development/style.md +++ b/docs/ru/development/style.md @@ -911,4 +911,3 @@ function( size_t limit) ``` -[Оригинальная статья](https://clickhouse.tech/docs/ru/development/style/) diff --git a/docs/ru/engines/database-engines/index.md b/docs/ru/engines/database-engines/index.md index e06c032a636..e56faad39ef 100644 --- a/docs/ru/engines/database-engines/index.md +++ b/docs/ru/engines/database-engines/index.md @@ -18,4 +18,3 @@ toc_title: "Введение" - [Lazy](../../engines/database-engines/lazy.md) -[Оригинальная статья](https://clickhouse.tech/docs/ru/database_engines/) diff --git a/docs/ru/engines/database-engines/lazy.md b/docs/ru/engines/database-engines/lazy.md index c01aae0284e..140a67be761 100644 --- a/docs/ru/engines/database-engines/lazy.md +++ b/docs/ru/engines/database-engines/lazy.md @@ -15,4 +15,3 @@ toc_title: Lazy CREATE DATABASE testlazy ENGINE = Lazy(expiration_time_in_seconds); ``` -[Оригинальная статья](https://clickhouse.tech/docs/ru/database_engines/lazy/) diff --git a/docs/ru/engines/database-engines/materialize-mysql.md b/docs/ru/engines/database-engines/materialize-mysql.md index 3022542e294..2067dfecca0 100644 --- a/docs/ru/engines/database-engines/materialize-mysql.md +++ b/docs/ru/engines/database-engines/materialize-mysql.md @@ -157,4 +157,3 @@ SELECT * FROM mysql.test; └───┴─────┴──────┘ ``` -[Оригинальная статья](https://clickhouse.tech/docs/ru/engines/database-engines/materialize-mysql/) diff --git a/docs/ru/engines/table-engines/index.md b/docs/ru/engines/table-engines/index.md index 05236eb5b33..6c11011a307 100644 --- a/docs/ru/engines/table-engines/index.md +++ b/docs/ru/engines/table-engines/index.md @@ -80,4 +80,3 @@ toc_title: "Введение" При создании таблицы со столбцом, имя которого совпадает с именем одного из виртуальных столбцов таблицы, виртуальный столбец становится недоступным. Не делайте так. Чтобы помочь избежать конфликтов, имена виртуальных столбцов обычно предваряются подчеркиванием. -[Оригинальная статья](https://clickhouse.tech/docs/ru/operations/table_engines/) diff --git a/docs/ru/engines/table-engines/integrations/embedded-rocksdb.md b/docs/ru/engines/table-engines/integrations/embedded-rocksdb.md index 7bd1420dfab..f66e789a392 100644 --- a/docs/ru/engines/table-engines/integrations/embedded-rocksdb.md +++ b/docs/ru/engines/table-engines/integrations/embedded-rocksdb.md @@ -41,4 +41,3 @@ ENGINE = EmbeddedRocksDB PRIMARY KEY key; ``` -[Оригинальная статья](https://clickhouse.tech/docs/ru/engines/table-engines/integrations/embedded-rocksdb/) \ No newline at end of file diff --git a/docs/ru/engines/table-engines/integrations/hdfs.md b/docs/ru/engines/table-engines/integrations/hdfs.md index 449d7c9a20c..3d9cb388a01 100644 --- a/docs/ru/engines/table-engines/integrations/hdfs.md +++ b/docs/ru/engines/table-engines/integrations/hdfs.md @@ -202,4 +202,3 @@ CREATE TABLE big_table (name String, value UInt32) ENGINE = HDFS('hdfs://hdfs1:9 - [Виртуальные колонки](../../../engines/table-engines/index.md#table_engines-virtual_columns) -[Оригинальная статья](https://clickhouse.tech/docs/ru/engines/table-engines/integrations/hdfs/) diff --git a/docs/ru/engines/table-engines/integrations/index.md b/docs/ru/engines/table-engines/integrations/index.md index c7004d104f8..cb217270129 100644 --- a/docs/ru/engines/table-engines/integrations/index.md +++ b/docs/ru/engines/table-engines/integrations/index.md @@ -20,4 +20,3 @@ toc_priority: 30 - [RabbitMQ](../../../engines/table-engines/integrations/rabbitmq.md) - [PostgreSQL](../../../engines/table-engines/integrations/postgresql.md) -[Оригинальная статья](https://clickhouse.tech/docs/ru/engines/table-engines/integrations/) diff --git a/docs/ru/engines/table-engines/integrations/jdbc.md b/docs/ru/engines/table-engines/integrations/jdbc.md index 8ead5abb277..e2db6fac0b2 100644 --- a/docs/ru/engines/table-engines/integrations/jdbc.md +++ b/docs/ru/engines/table-engines/integrations/jdbc.md @@ -89,4 +89,3 @@ FROM jdbc_table - [Табличная функция JDBC](../../../engines/table-engines/integrations/jdbc.md). -[Оригинальная статья](https://clickhouse.tech/docs/ru/engines/table-engines/integrations/jdbc/) diff --git a/docs/ru/engines/table-engines/integrations/kafka.md b/docs/ru/engines/table-engines/integrations/kafka.md index 06a0d4df180..f053b80aebd 100644 --- a/docs/ru/engines/table-engines/integrations/kafka.md +++ b/docs/ru/engines/table-engines/integrations/kafka.md @@ -193,4 +193,3 @@ ClickHouse может поддерживать учетные данные Kerbe - [Виртуальные столбцы](index.md#table_engines-virtual_columns) - [background_schedule_pool_size](../../../operations/settings/settings.md#background_schedule_pool_size) -[Оригинальная статья](https://clickhouse.tech/docs/ru/engines/table-engines/integrations/kafka/) diff --git a/docs/ru/engines/table-engines/integrations/mysql.md b/docs/ru/engines/table-engines/integrations/mysql.md index bc53e0f1fbb..9152a57d122 100644 --- a/docs/ru/engines/table-engines/integrations/mysql.md +++ b/docs/ru/engines/table-engines/integrations/mysql.md @@ -101,4 +101,3 @@ SELECT * FROM mysql_table - [Табличная функция ‘mysql’](../../../engines/table-engines/integrations/mysql.md) - [Использование MySQL в качестве источника для внешнего словаря](../../../engines/table-engines/integrations/mysql.md#dicts-external_dicts_dict_sources-mysql) -[Оригинальная статья](https://clickhouse.tech/docs/engines/table-engines/integrations/mysql/) diff --git a/docs/ru/engines/table-engines/integrations/odbc.md b/docs/ru/engines/table-engines/integrations/odbc.md index ee34be302bc..b2faa9b1e9e 100644 --- a/docs/ru/engines/table-engines/integrations/odbc.md +++ b/docs/ru/engines/table-engines/integrations/odbc.md @@ -128,4 +128,3 @@ SELECT * FROM odbc_t - [Внешние словари ODBC](../../../engines/table-engines/integrations/odbc.md#dicts-external_dicts_dict_sources-odbc) - [Табличная функция odbc](../../../engines/table-engines/integrations/odbc.md) -[Оригинальная статья](https://clickhouse.tech/docs/ru/engines/table-engines/integrations/odbc/) diff --git a/docs/ru/engines/table-engines/integrations/postgresql.md b/docs/ru/engines/table-engines/integrations/postgresql.md index bc26899f55b..ecf431830f8 100644 --- a/docs/ru/engines/table-engines/integrations/postgresql.md +++ b/docs/ru/engines/table-engines/integrations/postgresql.md @@ -103,4 +103,3 @@ SELECT * FROM postgresql_table WHERE str IN ('test') - [Табличная функция ‘postgresql’](../../../sql-reference/table-functions/postgresql.md) - [Использование PostgreSQL в качестве истояника для внешнего словаря](../../../sql-reference/dictionaries/external-dictionaries/external-dicts-dict-sources.md#dicts-external_dicts_dict_sources-postgresql) -[Оригинальная статья](https://clickhouse.tech/docs/ru/engines/table-engines/integrations/postgresql/) diff --git a/docs/ru/engines/table-engines/integrations/rabbitmq.md b/docs/ru/engines/table-engines/integrations/rabbitmq.md index 1865cb16fcc..ef8a58c4c82 100644 --- a/docs/ru/engines/table-engines/integrations/rabbitmq.md +++ b/docs/ru/engines/table-engines/integrations/rabbitmq.md @@ -156,4 +156,3 @@ Example: - `_message_id` - значение поля `messageID` полученного сообщения. Данное поле непусто, если указано в параметрах при отправке сообщения. - `_timestamp` - значение поля `timestamp` полученного сообщения. Данное поле непусто, если указано в параметрах при отправке сообщения. -[Оригинальная статья](https://clickhouse.tech/docs/ru/engines/table-engines/integrations/rabbitmq/) diff --git a/docs/ru/engines/table-engines/integrations/s3.md b/docs/ru/engines/table-engines/integrations/s3.md index f1b2e78b0ba..4eaf2d5b05c 100644 --- a/docs/ru/engines/table-engines/integrations/s3.md +++ b/docs/ru/engines/table-engines/integrations/s3.md @@ -153,4 +153,3 @@ CREATE TABLE big_table (name String, value UInt32) ENGINE = S3('https://storage. ``` -[Оригинальная статья](https://clickhouse.tech/docs/ru/engines/table-engines/integrations/s3/) diff --git a/docs/ru/engines/table-engines/log-family/index.md b/docs/ru/engines/table-engines/log-family/index.md index b2a56f650f4..7737eac2f43 100644 --- a/docs/ru/engines/table-engines/log-family/index.md +++ b/docs/ru/engines/table-engines/log-family/index.md @@ -42,4 +42,3 @@ toc_priority: 29 Движки `Log` и `StripeLog` поддерживают параллельное чтение. При чтении данных, ClickHouse использует множество потоков. Каждый поток обрабатывает отдельный блок данных. Движок `Log` сохраняет каждый столбец таблицы в отдельном файле. Движок `StripeLog` хранит все данные в одном файле. Таким образом, движок `StripeLog` использует меньше дескрипторов в операционной системе, а движок `Log` обеспечивает более эффективное считывание данных. -[Оригинальная статья](https://clickhouse.tech/docs/ru/operations/table_engines/log_family/) diff --git a/docs/ru/engines/table-engines/log-family/log.md b/docs/ru/engines/table-engines/log-family/log.md index fad331454c7..6c5bf2221f8 100644 --- a/docs/ru/engines/table-engines/log-family/log.md +++ b/docs/ru/engines/table-engines/log-family/log.md @@ -11,4 +11,3 @@ toc_title: Log При конкурентном доступе к данным, чтения могут выполняться одновременно, а записи блокируют чтения и друг друга. Движок Log не поддерживает индексы. Также, если при записи в таблицу произошёл сбой, то таблица станет битой, и чтения из неё будут возвращать ошибку. Движок Log подходит для временных данных, write-once таблиц, а также для тестовых и демонстрационных целей. -[Оригинальная статья](https://clickhouse.tech/docs/ru/operations/table_engines/log/) diff --git a/docs/ru/engines/table-engines/log-family/stripelog.md b/docs/ru/engines/table-engines/log-family/stripelog.md index e505aae4c52..2f4b228f894 100644 --- a/docs/ru/engines/table-engines/log-family/stripelog.md +++ b/docs/ru/engines/table-engines/log-family/stripelog.md @@ -90,4 +90,3 @@ SELECT * FROM stripe_log_table ORDER BY timestamp └─────────────────────┴──────────────┴────────────────────────────┘ ``` -[Оригинальная статья](https://clickhouse.tech/docs/ru/operations/table_engines/stripelog/) diff --git a/docs/ru/engines/table-engines/log-family/tinylog.md b/docs/ru/engines/table-engines/log-family/tinylog.md index d5c24d41ca4..721355d8702 100644 --- a/docs/ru/engines/table-engines/log-family/tinylog.md +++ b/docs/ru/engines/table-engines/log-family/tinylog.md @@ -11,4 +11,3 @@ toc_title: TinyLog Запросы выполняются в один поток. То есть, этот движок предназначен для сравнительно маленьких таблиц (до 1 000 000 строк). Этот движок таблиц имеет смысл использовать в том случае, когда у вас есть много маленьких таблиц, так как он проще, чем движок [Log](log.md) (требуется открывать меньше файлов). -[Оригинальная статья](https://clickhouse.tech/docs/ru/operations/table_engines/tinylog/) diff --git a/docs/ru/engines/table-engines/mergetree-family/aggregatingmergetree.md b/docs/ru/engines/table-engines/mergetree-family/aggregatingmergetree.md index 99b4ec06765..6e01cc2bcac 100644 --- a/docs/ru/engines/table-engines/mergetree-family/aggregatingmergetree.md +++ b/docs/ru/engines/table-engines/mergetree-family/aggregatingmergetree.md @@ -97,4 +97,3 @@ GROUP BY StartDate ORDER BY StartDate; ``` -[Оригинальная статья](https://clickhouse.tech/docs/ru/operations/table_engines/aggregatingmergetree/) diff --git a/docs/ru/engines/table-engines/mergetree-family/collapsingmergetree.md b/docs/ru/engines/table-engines/mergetree-family/collapsingmergetree.md index 8ea3a5a7c92..424fcbb5873 100644 --- a/docs/ru/engines/table-engines/mergetree-family/collapsingmergetree.md +++ b/docs/ru/engines/table-engines/mergetree-family/collapsingmergetree.md @@ -304,4 +304,3 @@ select * FROM UAct └─────────────────────┴───────────┴──────────┴──────┘ ``` -[Оригинальная статья](https://clickhouse.tech/docs/ru/operations/table_engines/collapsingmergetree/) diff --git a/docs/ru/engines/table-engines/mergetree-family/custom-partitioning-key.md b/docs/ru/engines/table-engines/mergetree-family/custom-partitioning-key.md index 00d850b01c3..9a09618e508 100644 --- a/docs/ru/engines/table-engines/mergetree-family/custom-partitioning-key.md +++ b/docs/ru/engines/table-engines/mergetree-family/custom-partitioning-key.md @@ -129,4 +129,3 @@ drwxr-xr-x 2 clickhouse clickhouse 4096 Feb 1 16:48 detached ClickHouse позволяет производить различные манипуляции с кусками: удалять, копировать из одной таблицы в другую или создавать их резервные копии. Подробнее см. в разделе [Манипуляции с партициями и кусками](../../../engines/table-engines/mergetree-family/custom-partitioning-key.md#alter_manipulations-with-partitions). -[Оригинальная статья:](https://clickhouse.tech/docs/ru/operations/table_engines/custom_partitioning_key/) diff --git a/docs/ru/engines/table-engines/mergetree-family/graphitemergetree.md b/docs/ru/engines/table-engines/mergetree-family/graphitemergetree.md index e47c9127711..f3e915a413b 100644 --- a/docs/ru/engines/table-engines/mergetree-family/graphitemergetree.md +++ b/docs/ru/engines/table-engines/mergetree-family/graphitemergetree.md @@ -171,4 +171,3 @@ default ``` -[Оригинальная статья](https://clickhouse.tech/docs/ru/operations/table_engines/graphitemergetree/) diff --git a/docs/ru/engines/table-engines/mergetree-family/mergetree.md b/docs/ru/engines/table-engines/mergetree-family/mergetree.md index bc74b2592b9..0975544263b 100644 --- a/docs/ru/engines/table-engines/mergetree-family/mergetree.md +++ b/docs/ru/engines/table-engines/mergetree-family/mergetree.md @@ -807,4 +807,3 @@ SETTINGS storage_policy = 'moving_from_ssd_to_hdd' Если диск сконфигурирован как `cold`, данные будут переноситься в S3 при срабатывании правил TTL или когда свободное место на локальном диске станет меньше порогового значения, которое определяется как `move_factor * disk_size`. -[Оригинальная статья](https://clickhouse.tech/docs/ru/engines/table-engines/mergetree-family/mergetree/) diff --git a/docs/ru/engines/table-engines/mergetree-family/replacingmergetree.md b/docs/ru/engines/table-engines/mergetree-family/replacingmergetree.md index a4e47b161ad..ec0b339e8c9 100644 --- a/docs/ru/engines/table-engines/mergetree-family/replacingmergetree.md +++ b/docs/ru/engines/table-engines/mergetree-family/replacingmergetree.md @@ -66,4 +66,3 @@ CREATE TABLE [IF NOT EXISTS] [db.]table_name [ON CLUSTER cluster] -[Оригинальная статья](https://clickhouse.tech/docs/ru/operations/table_engines/replacingmergetree/) diff --git a/docs/ru/engines/table-engines/mergetree-family/replication.md b/docs/ru/engines/table-engines/mergetree-family/replication.md index 1735a02cf4c..848adbee4da 100644 --- a/docs/ru/engines/table-engines/mergetree-family/replication.md +++ b/docs/ru/engines/table-engines/mergetree-family/replication.md @@ -251,4 +251,3 @@ $ sudo -u clickhouse touch /var/lib/clickhouse/flags/force_restore_data - [background_schedule_pool_size](../../../operations/settings/settings.md#background_schedule_pool_size) - [execute_merges_on_single_replica_time_threshold](../../../operations/settings/settings.md#execute-merges-on-single-replica-time-threshold) -[Оригинальная статья](https://clickhouse.tech/docs/ru/operations/table_engines/replication/) diff --git a/docs/ru/engines/table-engines/mergetree-family/summingmergetree.md b/docs/ru/engines/table-engines/mergetree-family/summingmergetree.md index 7b9c11adc2e..adb40037319 100644 --- a/docs/ru/engines/table-engines/mergetree-family/summingmergetree.md +++ b/docs/ru/engines/table-engines/mergetree-family/summingmergetree.md @@ -136,4 +136,3 @@ ClickHouse может слить куски данных таким образо Для вложенной структуры данных не нужно указывать её столбцы в кортеже столбцов для суммирования. -[Оригинальная статья](https://clickhouse.tech/docs/ru/operations/table_engines/summingmergetree/) diff --git a/docs/ru/engines/table-engines/mergetree-family/versionedcollapsingmergetree.md b/docs/ru/engines/table-engines/mergetree-family/versionedcollapsingmergetree.md index 2adb8cc0d77..61688b1f00f 100644 --- a/docs/ru/engines/table-engines/mergetree-family/versionedcollapsingmergetree.md +++ b/docs/ru/engines/table-engines/mergetree-family/versionedcollapsingmergetree.md @@ -233,4 +233,3 @@ SELECT * FROM UAct FINAL Это очень неэффективный способ выбора данных. Не используйте его для больших таблиц. -[Оригинальная статья](https://clickhouse.tech/docs/ru/operations/table_engines/versionedcollapsingmergetree/) diff --git a/docs/ru/engines/table-engines/special/buffer.md b/docs/ru/engines/table-engines/special/buffer.md index 75ce12f50fa..ba865b72b78 100644 --- a/docs/ru/engines/table-engines/special/buffer.md +++ b/docs/ru/engines/table-engines/special/buffer.md @@ -66,4 +66,3 @@ CREATE TABLE merge.hits_buffer AS merge.hits ENGINE = Buffer(merge, hits, 16, 10 Заметим, что даже для таблиц типа Buffer не имеет смысла вставлять данные по одной строке, так как таким образом будет достигнута скорость всего лишь в несколько тысяч строк в секунду, тогда как при вставке более крупными блоками, достижимо более миллиона строк в секунду (смотрите раздел [«Производительность»](../../../introduction/performance/). -[Оригинальная статья](https://clickhouse.tech/docs/ru/operations/table_engines/buffer/) diff --git a/docs/ru/engines/table-engines/special/dictionary.md b/docs/ru/engines/table-engines/special/dictionary.md index 048da157b2d..243fd5395c0 100644 --- a/docs/ru/engines/table-engines/special/dictionary.md +++ b/docs/ru/engines/table-engines/special/dictionary.md @@ -90,4 +90,3 @@ select * from products limit 1; └───────────────┴─────────────────┘ ``` -[Оригинальная статья](https://clickhouse.tech/docs/ru/operations/table_engines/dictionary/) diff --git a/docs/ru/engines/table-engines/special/distributed.md b/docs/ru/engines/table-engines/special/distributed.md index 7ab0b916337..86eef35ebbc 100644 --- a/docs/ru/engines/table-engines/special/distributed.md +++ b/docs/ru/engines/table-engines/special/distributed.md @@ -136,4 +136,3 @@ logs - имя кластера в конфигурационном файле с При выставлении опции max_parallel_replicas выполнение запроса распараллеливается по всем репликам внутри одного шарда. Подробнее смотрите раздел [max_parallel_replicas](../../../operations/settings/settings.md#settings-max_parallel_replicas). -[Оригинальная статья](https://clickhouse.tech/docs/ru/operations/table_engines/distributed/) diff --git a/docs/ru/engines/table-engines/special/external-data.md b/docs/ru/engines/table-engines/special/external-data.md index da9e132dd4f..29075837aba 100644 --- a/docs/ru/engines/table-engines/special/external-data.md +++ b/docs/ru/engines/table-engines/special/external-data.md @@ -65,4 +65,3 @@ $ curl -F 'passwd=@passwd.tsv;' 'http://localhost:8123/?query=SELECT+shell,+coun При распределённой обработке запроса, временные таблицы передаются на все удалённые серверы. -[Оригинальная статья](https://clickhouse.tech/docs/ru/operations/table_engines/external_data/) diff --git a/docs/ru/engines/table-engines/special/file.md b/docs/ru/engines/table-engines/special/file.md index 9be09fd33e6..6f1c723d2a7 100644 --- a/docs/ru/engines/table-engines/special/file.md +++ b/docs/ru/engines/table-engines/special/file.md @@ -81,4 +81,3 @@ $ echo -e "1,2\n3,4" | clickhouse-local -q "CREATE TABLE table (a Int64, b Int64 - индексы; - репликация. -[Оригинальная статья](https://clickhouse.tech/docs/ru/operations/table_engines/file/) diff --git a/docs/ru/engines/table-engines/special/index.md b/docs/ru/engines/table-engines/special/index.md index 0300d3ad641..231bf2979ed 100644 --- a/docs/ru/engines/table-engines/special/index.md +++ b/docs/ru/engines/table-engines/special/index.md @@ -13,4 +13,3 @@ toc_priority: 31 Остальные движки таблиц уникальны по своему назначению и еще не сгруппированы в семейства, поэтому они помещены в эту специальную категорию. -[Оригинальная статья](https://clickhouse.tech/docs/ru/engines/table-engines/special/) diff --git a/docs/ru/engines/table-engines/special/join.md b/docs/ru/engines/table-engines/special/join.md index 8cb7acd91e1..ef27ac3f10f 100644 --- a/docs/ru/engines/table-engines/special/join.md +++ b/docs/ru/engines/table-engines/special/join.md @@ -107,4 +107,3 @@ SELECT joinGet('id_val_join', 'val', toUInt32(1)) При аварийном перезапуске сервера блок данных на диске может быть потерян или повреждён. В последнем случае, может потребоваться вручную удалить файл с повреждёнными данными. -[Оригинальная статья](https://clickhouse.tech/docs/ru/operations/table_engines/join/) diff --git a/docs/ru/engines/table-engines/special/materializedview.md b/docs/ru/engines/table-engines/special/materializedview.md index 1281d1db9ab..6b82f95df92 100644 --- a/docs/ru/engines/table-engines/special/materializedview.md +++ b/docs/ru/engines/table-engines/special/materializedview.md @@ -7,4 +7,3 @@ toc_title: MaterializedView Используется для реализации материализованных представлений (подробнее см. запрос [CREATE TABLE](../../../sql-reference/statements/create/table.md#create-table-query)). Для хранения данных, использует другой движок, который был указан при создании представления. При чтении из таблицы, просто использует этот движок. -[Оригинальная статья](https://clickhouse.tech/docs/ru/operations/table_engines/materializedview/) diff --git a/docs/ru/engines/table-engines/special/memory.md b/docs/ru/engines/table-engines/special/memory.md index 9ca189ef3b2..5a242238a02 100644 --- a/docs/ru/engines/table-engines/special/memory.md +++ b/docs/ru/engines/table-engines/special/memory.md @@ -14,4 +14,3 @@ toc_title: Memory Движок Memory используется системой для временных таблиц - внешних данных запроса (смотрите раздел «Внешние данные для обработки запроса»), для реализации `GLOBAL IN` (смотрите раздел «Операторы IN»). -[Оригинальная статья](https://clickhouse.tech/docs/ru/operations/table_engines/memory/) diff --git a/docs/ru/engines/table-engines/special/merge.md b/docs/ru/engines/table-engines/special/merge.md index 656aa7cfd6b..714b087c201 100644 --- a/docs/ru/engines/table-engines/special/merge.md +++ b/docs/ru/engines/table-engines/special/merge.md @@ -65,4 +65,3 @@ FROM WatchLog - [Виртуальные столбцы](index.md#table_engines-virtual_columns) -[Оригинальная статья](https://clickhouse.tech/docs/ru/operations/table_engines/merge/) diff --git a/docs/ru/engines/table-engines/special/null.md b/docs/ru/engines/table-engines/special/null.md index 2c3af1ce11e..05f5c88bacb 100644 --- a/docs/ru/engines/table-engines/special/null.md +++ b/docs/ru/engines/table-engines/special/null.md @@ -7,4 +7,3 @@ toc_title: 'Null' Тем не менее, есть возможность создать материализованное представление над таблицей типа Null. Тогда данные, записываемые в таблицу, будут попадать в представление. -[Оригинальная статья](https://clickhouse.tech/docs/ru/operations/table_engines/null/) diff --git a/docs/ru/engines/table-engines/special/set.md b/docs/ru/engines/table-engines/special/set.md index 14b7f123a34..ced9abf55dc 100644 --- a/docs/ru/engines/table-engines/special/set.md +++ b/docs/ru/engines/table-engines/special/set.md @@ -20,4 +20,3 @@ toc_title: Set - [persistent](../../../operations/settings/settings.md#persistent) -[Оригинальная статья](https://clickhouse.tech/docs/ru/operations/table_engines/set/) diff --git a/docs/ru/engines/table-engines/special/url.md b/docs/ru/engines/table-engines/special/url.md index cdb5afddf75..b8fcd27204f 100644 --- a/docs/ru/engines/table-engines/special/url.md +++ b/docs/ru/engines/table-engines/special/url.md @@ -77,4 +77,3 @@ SELECT * FROM url_engine_table - индексы; - репликация. -[Оригинальная статья](https://clickhouse.tech/docs/ru/operations/table_engines/url/) diff --git a/docs/ru/engines/table-engines/special/view.md b/docs/ru/engines/table-engines/special/view.md index 18813a55da2..45aeb55cd85 100644 --- a/docs/ru/engines/table-engines/special/view.md +++ b/docs/ru/engines/table-engines/special/view.md @@ -7,4 +7,3 @@ toc_title: View Используется для реализации представлений (подробнее см. запрос `CREATE VIEW`). Не хранит данные, а хранит только указанный запрос `SELECT`. При чтении из таблицы, выполняет его (с удалением из запроса всех ненужных столбцов). -[Оригинальная статья](https://clickhouse.tech/docs/ru/operations/table_engines/view/) diff --git a/docs/ru/getting-started/example-datasets/amplab-benchmark.md b/docs/ru/getting-started/example-datasets/amplab-benchmark.md index bc59672ab26..8a75852aad9 100644 --- a/docs/ru/getting-started/example-datasets/amplab-benchmark.md +++ b/docs/ru/getting-started/example-datasets/amplab-benchmark.md @@ -125,4 +125,3 @@ ORDER BY totalRevenue DESC LIMIT 1 ``` -[Оригинальная статья](https://clickhouse.tech/docs/ru/getting_started/example_datasets/amplab_benchmark/) diff --git a/docs/ru/getting-started/example-datasets/brown-benchmark.md b/docs/ru/getting-started/example-datasets/brown-benchmark.md index 23702e07fcd..f1aad06b743 100644 --- a/docs/ru/getting-started/example-datasets/brown-benchmark.md +++ b/docs/ru/getting-started/example-datasets/brown-benchmark.md @@ -413,4 +413,3 @@ ORDER BY yr, Данные также доступны для работы с интерактивными запросами через [Playground](https://gh-api.clickhouse.tech/play?user=play), [пример](https://gh-api.clickhouse.tech/play?user=play#U0VMRUNUIG1hY2hpbmVfbmFtZSwKICAgICAgIE1JTihjcHUpIEFTIGNwdV9taW4sCiAgICAgICBNQVgoY3B1KSBBUyBjcHVfbWF4LAogICAgICAgQVZHKGNwdSkgQVMgY3B1X2F2ZywKICAgICAgIE1JTihuZXRfaW4pIEFTIG5ldF9pbl9taW4sCiAgICAgICBNQVgobmV0X2luKSBBUyBuZXRfaW5fbWF4LAogICAgICAgQVZHKG5ldF9pbikgQVMgbmV0X2luX2F2ZywKICAgICAgIE1JTihuZXRfb3V0KSBBUyBuZXRfb3V0X21pbiwKICAgICAgIE1BWChuZXRfb3V0KSBBUyBuZXRfb3V0X21heCwKICAgICAgIEFWRyhuZXRfb3V0KSBBUyBuZXRfb3V0X2F2ZwpGUk9NICgKICBTRUxFQ1QgbWFjaGluZV9uYW1lLAogICAgICAgICBDT0FMRVNDRShjcHVfdXNlciwgMC4wKSBBUyBjcHUsCiAgICAgICAgIENPQUxFU0NFKGJ5dGVzX2luLCAwLjApIEFTIG5ldF9pbiwKICAgICAgICAgQ09BTEVTQ0UoYnl0ZXNfb3V0LCAwLjApIEFTIG5ldF9vdXQKICBGUk9NIG1nYmVuY2gubG9nczEKICBXSEVSRSBtYWNoaW5lX25hbWUgSU4gKCdhbmFuc2knLCdhcmFnb2cnLCd1cmQnKQogICAgQU5EIGxvZ190aW1lID49IFRJTUVTVEFNUCAnMjAxNy0wMS0xMSAwMDowMDowMCcKKSBBUyByCkdST1VQIEJZIG1hY2hpbmVfbmFtZQ==). -[Оригинальная статья](https://clickhouse.tech/docs/ru/getting_started/example_datasets/brown-benchmark/) diff --git a/docs/ru/getting-started/example-datasets/criteo.md b/docs/ru/getting-started/example-datasets/criteo.md index ecdc5f5fa41..bfa428a0e1c 100644 --- a/docs/ru/getting-started/example-datasets/criteo.md +++ b/docs/ru/getting-started/example-datasets/criteo.md @@ -76,4 +76,3 @@ INSERT INTO criteo SELECT date, clicked, int1, int2, int3, int4, int5, int6, int DROP TABLE criteo_log; ``` -[Оригинальная статья](https://clickhouse.tech/docs/ru/getting_started/example_datasets/criteo/) diff --git a/docs/ru/getting-started/example-datasets/index.md b/docs/ru/getting-started/example-datasets/index.md index fd89bb122e3..f590300adda 100644 --- a/docs/ru/getting-started/example-datasets/index.md +++ b/docs/ru/getting-started/example-datasets/index.md @@ -17,4 +17,3 @@ toc_title: "Введение" - [Данные о такси в Нью-Йорке](nyc-taxi.md) - [OnTime](ontime.md) -[Оригинальная статья](https://clickhouse.tech/docs/en/getting_started/example_datasets) diff --git a/docs/ru/getting-started/example-datasets/nyc-taxi.md b/docs/ru/getting-started/example-datasets/nyc-taxi.md index 891a92e2fa7..38a60ed1b2d 100644 --- a/docs/ru/getting-started/example-datasets/nyc-taxi.md +++ b/docs/ru/getting-started/example-datasets/nyc-taxi.md @@ -390,4 +390,3 @@ Q4: 0.072 sec. | 3 | 0.212 | 0.438 | 0.733 | 1.241 | | 140 | 0.028 | 0.043 | 0.051 | 0.072 | -[Оригинальная статья](https://clickhouse.tech/docs/ru/getting_started/example_datasets/nyc_taxi/) diff --git a/docs/ru/getting-started/example-datasets/ontime.md b/docs/ru/getting-started/example-datasets/ontime.md index 41a1c0d3142..be5b1cd1b70 100644 --- a/docs/ru/getting-started/example-datasets/ontime.md +++ b/docs/ru/getting-started/example-datasets/ontime.md @@ -407,4 +407,3 @@ LIMIT 10; - https://www.percona.com/blog/2016/01/07/apache-spark-with-air-ontime-performance-data/ - http://nickmakos.blogspot.ru/2012/08/analyzing-air-traffic-performance-with.html -[Оригинальная статья](https://clickhouse.tech/docs/ru/getting_started/example_datasets/ontime/) diff --git a/docs/ru/getting-started/example-datasets/wikistat.md b/docs/ru/getting-started/example-datasets/wikistat.md index c5a877ff8fd..f224c24e6ac 100644 --- a/docs/ru/getting-started/example-datasets/wikistat.md +++ b/docs/ru/getting-started/example-datasets/wikistat.md @@ -30,4 +30,3 @@ $ cat links.txt | while read link; do wget http://dumps.wikimedia.org/other/page $ ls -1 /opt/wikistat/ | grep gz | while read i; do echo $i; gzip -cd /opt/wikistat/$i | ./wikistat-loader --time="$(echo -n $i | sed -r 's/pagecounts-([0-9]{4})([0-9]{2})([0-9]{2})-([0-9]{2})([0-9]{2})([0-9]{2})\.gz/\1-\2-\3 \4-00-00/')" | clickhouse-client --query="INSERT INTO wikistat FORMAT TabSeparated"; done ``` -[Оригинальная статья](https://clickhouse.tech/docs/ru/getting_started/example_datasets/wikistat/) diff --git a/docs/ru/getting-started/index.md b/docs/ru/getting-started/index.md index 78b56092740..599cb8b9434 100644 --- a/docs/ru/getting-started/index.md +++ b/docs/ru/getting-started/index.md @@ -14,4 +14,3 @@ toc_title: hidden - [Пройти подробное руководство для начинающих](tutorial.md) - [Поэкспериментировать с тестовыми наборами данных](example-datasets/ontime.md) -[Оригинальная статья](https://clickhouse.tech/docs/ru/getting_started/) diff --git a/docs/ru/getting-started/install.md b/docs/ru/getting-started/install.md index aa5e8d77512..4ae27a910ea 100644 --- a/docs/ru/getting-started/install.md +++ b/docs/ru/getting-started/install.md @@ -173,4 +173,3 @@ SELECT 1 Для дальнейших экспериментов можно попробовать загрузить один из тестовых наборов данных или пройти [пошаговое руководство для начинающих](https://clickhouse.tech/tutorial.html). -[Оригинальная статья](https://clickhouse.tech/docs/ru/getting_started/install/) diff --git a/docs/ru/index.md b/docs/ru/index.md index 26d7dc3bf21..e16f2afed82 100644 --- a/docs/ru/index.md +++ b/docs/ru/index.md @@ -97,4 +97,3 @@ ClickHouse - столбцовая система управления базам Стоит заметить, что для эффективности по CPU требуется, чтобы язык запросов был декларативным (SQL, MDX) или хотя бы векторным (J, K). То есть, чтобы запрос содержал циклы только в неявном виде, открывая возможности для оптимизации. -[Оригинальная статья](https://clickhouse.tech/docs/ru/) diff --git a/docs/ru/interfaces/cli.md b/docs/ru/interfaces/cli.md index 3f6b288fc2b..96ec36be79f 100644 --- a/docs/ru/interfaces/cli.md +++ b/docs/ru/interfaces/cli.md @@ -153,4 +153,3 @@ $ clickhouse-client --param_tbl="numbers" --param_db="system" --param_col="numbe ``` -[Оригинальная статья](https://clickhouse.tech/docs/ru/interfaces/cli/) diff --git a/docs/ru/interfaces/cpp.md b/docs/ru/interfaces/cpp.md index 018f4e22e34..f0691453fe6 100644 --- a/docs/ru/interfaces/cpp.md +++ b/docs/ru/interfaces/cpp.md @@ -7,4 +7,3 @@ toc_title: "C++ клиентская библиотека" См. README в репозитории [clickhouse-cpp](https://github.com/ClickHouse/clickhouse-cpp). -[Оригинальная статья](https://clickhouse.tech/docs/ru/interfaces/cpp/) diff --git a/docs/ru/interfaces/formats.md b/docs/ru/interfaces/formats.md index edea533b642..3a61d789e75 100644 --- a/docs/ru/interfaces/formats.md +++ b/docs/ru/interfaces/formats.md @@ -1390,4 +1390,3 @@ $ clickhouse-client --query "SELECT * FROM {some_table} FORMAT RawBLOB" | md5sum f9725a22f9191e064120d718e26862a9 - ``` -[Оригинальная статья](https://clickhouse.tech/docs/ru/interfaces/formats/) diff --git a/docs/ru/interfaces/http.md b/docs/ru/interfaces/http.md index 5cb50d8f168..9e553c12dc0 100644 --- a/docs/ru/interfaces/http.md +++ b/docs/ru/interfaces/http.md @@ -635,4 +635,3 @@ $ curl -vv -H 'XXX:xxx' 'http://localhost:8123/get_relative_path_static_handler' * Connection #0 to host localhost left intact ``` -[Оригинальная статья](https://clickhouse.tech/docs/ru/interfaces/http_interface/) diff --git a/docs/ru/interfaces/index.md b/docs/ru/interfaces/index.md index fc8743b3c1e..12e8853823e 100644 --- a/docs/ru/interfaces/index.md +++ b/docs/ru/interfaces/index.md @@ -24,4 +24,3 @@ ClickHouse предоставляет два сетевых интерфейса - [Библиотеки для интеграции](third-party/integrations.md); - [Визуальные интерфейсы](third-party/gui.md). -[Оригинальная статья](https://clickhouse.tech/docs/ru/interfaces/) diff --git a/docs/ru/interfaces/jdbc.md b/docs/ru/interfaces/jdbc.md index ac86375c74f..30270322f7a 100644 --- a/docs/ru/interfaces/jdbc.md +++ b/docs/ru/interfaces/jdbc.md @@ -10,4 +10,3 @@ toc_title: "JDBC-драйвер" - [ClickHouse-Native-JDBC](https://github.com/housepower/ClickHouse-Native-JDBC) - [clickhouse4j](https://github.com/blynkkk/clickhouse4j) -[Оригинальная статья](https://clickhouse.tech/docs/ru/interfaces/jdbc/) diff --git a/docs/ru/interfaces/odbc.md b/docs/ru/interfaces/odbc.md index 7843d3cb943..22153865298 100644 --- a/docs/ru/interfaces/odbc.md +++ b/docs/ru/interfaces/odbc.md @@ -8,4 +8,3 @@ toc_title: "ODBC-драйвер" - [Официальный драйвер](https://github.com/ClickHouse/clickhouse-odbc). -[Оригинальная статья](https://clickhouse.tech/docs/ru/interfaces/odbc/) diff --git a/docs/ru/interfaces/tcp.md b/docs/ru/interfaces/tcp.md index ea8c170009d..5261e1eafef 100644 --- a/docs/ru/interfaces/tcp.md +++ b/docs/ru/interfaces/tcp.md @@ -7,4 +7,3 @@ toc_title: "Родной интерфейс (TCP)" Нативный протокол используется в [клиенте командной строки](cli.md), для взаимодействия между серверами во время обработки распределенных запросов, а также в других программах на C++. К сожалению, у родного протокола ClickHouse пока нет формальной спецификации, но в нем можно разобраться с использованием исходного кода ClickHouse (начиная с [примерно этого места](https://github.com/ClickHouse/ClickHouse/tree/master/src/Client)) и/или путем перехвата и анализа TCP трафика. -[Оригинальная статья](https://clickhouse.tech/docs/ru/interfaces/tcp/) diff --git a/docs/ru/interfaces/third-party/client-libraries.md b/docs/ru/interfaces/third-party/client-libraries.md index 65e93731300..411475f0aaa 100644 --- a/docs/ru/interfaces/third-party/client-libraries.md +++ b/docs/ru/interfaces/third-party/client-libraries.md @@ -58,4 +58,3 @@ toc_title: "Клиентские библиотеки от сторонних р - Nim - [nim-clickhouse](https://github.com/leonardoce/nim-clickhouse) -[Оригинальная статья](https://clickhouse.tech/docs/ru/interfaces/third-party/client_libraries/) diff --git a/docs/ru/interfaces/third-party/gui.md b/docs/ru/interfaces/third-party/gui.md index c02c32e08f4..ba455df312c 100644 --- a/docs/ru/interfaces/third-party/gui.md +++ b/docs/ru/interfaces/third-party/gui.md @@ -146,7 +146,6 @@ toc_title: "Визуальные интерфейсы от сторонних р - Подготовка данных и возможности ETL. - Моделирование данных с помощью SQL для их реляционного отображения. -[Оригинальная статья](https://clickhouse.tech/docs/ru/interfaces/third-party/gui/) ### Looker {#looker} diff --git a/docs/ru/interfaces/third-party/index.md b/docs/ru/interfaces/third-party/index.md index 8b59bb5fd28..bbf5a237000 100644 --- a/docs/ru/interfaces/third-party/index.md +++ b/docs/ru/interfaces/third-party/index.md @@ -15,4 +15,3 @@ toc_priority: 24 !!! note "Примечание" С ClickHouse работают также универсальные инструменты, поддерживающие общий API, такие как [ODBC](../../interfaces/odbc.md) или [JDBC](../../interfaces/jdbc.md). -[Оригинальная статья](https://clickhouse.tech/docs/ru/interfaces/third-party/) diff --git a/docs/ru/interfaces/third-party/integrations.md b/docs/ru/interfaces/third-party/integrations.md index 84d5b93f92f..6da1459c34b 100644 --- a/docs/ru/interfaces/third-party/integrations.md +++ b/docs/ru/interfaces/third-party/integrations.md @@ -105,4 +105,3 @@ toc_title: "Библиотеки для интеграции от сторонн - [GraphQL](https://github.com/graphql) - [activecube-graphql](https://github.com/bitquery/activecube-graphql) -[Оригинальная статья](https://clickhouse.tech/docs/ru/interfaces/third-party/integrations/) diff --git a/docs/ru/interfaces/third-party/proxy.md b/docs/ru/interfaces/third-party/proxy.md index 48853cb352e..6d85c960c0e 100644 --- a/docs/ru/interfaces/third-party/proxy.md +++ b/docs/ru/interfaces/third-party/proxy.md @@ -41,4 +41,3 @@ toc_title: "Прокси-серверы от сторонних разработ Реализован на Go. -[Оригинальная статья](https://clickhouse.tech/docs/ru/interfaces/third-party/proxy/) diff --git a/docs/ru/introduction/distinctive-features.md b/docs/ru/introduction/distinctive-features.md index 852f5cecd5b..dedb1412dbf 100644 --- a/docs/ru/introduction/distinctive-features.md +++ b/docs/ru/introduction/distinctive-features.md @@ -73,4 +73,3 @@ ClickHouse предоставляет различные способы разм 3. Разреженный индекс делает ClickHouse плохо пригодным для точечных чтений одиночных строк по своим ключам. -[Оригинальная статья](https://clickhouse.tech/docs/ru/introduction/distinctive_features/) diff --git a/docs/ru/introduction/history.md b/docs/ru/introduction/history.md index ad17b2be27d..dc4aa935c27 100644 --- a/docs/ru/introduction/history.md +++ b/docs/ru/introduction/history.md @@ -52,4 +52,3 @@ OLAPServer хорошо подходил для неагрегированных Чтобы снять ограничения OLAPServer-а и решить задачу работы с неагрегированными данными для всех отчётов, разработана СУБД ClickHouse. -[Оригинальная статья](https://clickhouse.tech/docs/ru/introduction/ya_metrika_task/) diff --git a/docs/ru/introduction/info.md b/docs/ru/introduction/info.md index a9398b8c9cd..a5e7efffc7e 100644 --- a/docs/ru/introduction/info.md +++ b/docs/ru/introduction/info.md @@ -9,4 +9,3 @@ toc_priority: 100 - Адрес электронной почты: - Телефон: +7-495-780-6510 -[Оригинальная статья](https://clickhouse.tech/docs/ru/introduction/info/) diff --git a/docs/ru/introduction/performance.md b/docs/ru/introduction/performance.md index dd92d3df9f5..eec1dcf4d0a 100644 --- a/docs/ru/introduction/performance.md +++ b/docs/ru/introduction/performance.md @@ -27,4 +27,3 @@ toc_title: "Производительность" Данные рекомендуется вставлять пачками не менее 1000 строк или не более одного запроса в секунду. При вставке в таблицу типа MergeTree из tab-separated дампа, скорость вставки будет в районе 50-200 МБ/сек. Если вставляются строчки размером около 1 КБ, то скорость будет в районе 50 000 - 200 000 строчек в секунду. Если строчки маленькие - производительность в строчках в секунду будет выше (на данных БК - `>` 500 000 строк в секунду, на данных Graphite - `>` 1 000 000 строк в секунду). Для увеличения производительности, можно производить несколько запросов INSERT параллельно - при этом производительность растёт линейно. -[Оригинальная статья](https://clickhouse.tech/docs/ru/introduction/performance/) diff --git a/docs/ru/operations/access-rights.md b/docs/ru/operations/access-rights.md index 9aa4e5f2561..a0ad7664131 100644 --- a/docs/ru/operations/access-rights.md +++ b/docs/ru/operations/access-rights.md @@ -146,4 +146,3 @@ ClickHouse поддерживает управление доступом на По умолчанию управление доступом на основе SQL выключено для всех пользователей. Вам необходимо настроить хотя бы одного пользователя в файле конфигурации `users.xml` и присвоить значение 1 параметру [access_management](settings/settings-users.md#access_management-user-setting). -[Оригинальная статья](https://clickhouse.tech/docs/ru/operations/access_rights/) diff --git a/docs/ru/operations/backup.md b/docs/ru/operations/backup.md index 703217e8547..ed0adeb5e6f 100644 --- a/docs/ru/operations/backup.md +++ b/docs/ru/operations/backup.md @@ -36,4 +36,3 @@ ClickHouse позволяет использовать запрос `ALTER TABLE Для автоматизации этого подхода доступен инструмент от сторонних разработчиков: [clickhouse-backup](https://github.com/AlexAkulov/clickhouse-backup). -[Оригинальная статья](https://clickhouse.tech/docs/ru/operations/backup/) diff --git a/docs/ru/operations/caches.md b/docs/ru/operations/caches.md index 7744c596cd9..a0b71d1782a 100644 --- a/docs/ru/operations/caches.md +++ b/docs/ru/operations/caches.md @@ -26,4 +26,3 @@ toc_title: Кеши Чтобы очистить кеш, используйте выражение [SYSTEM DROP ... CACHE](../sql-reference/statements/system.md). -[Оригинальная статья](https://clickhouse.tech/docs/ru/operations/caches/) diff --git a/docs/ru/operations/configuration-files.md b/docs/ru/operations/configuration-files.md index 84b26d0ba2a..11a01d1e6d2 100644 --- a/docs/ru/operations/configuration-files.md +++ b/docs/ru/operations/configuration-files.md @@ -52,4 +52,3 @@ $ cat /etc/clickhouse-server/users.d/alice.xml Сервер следит за изменениями конфигурационных файлов, а также файлов и ZooKeeper-узлов, которые были использованы при выполнении подстановок и переопределений, и перезагружает настройки пользователей и кластеров на лету. То есть, можно изменять кластера, пользователей и их настройки без перезапуска сервера. -[Оригинальная статья](https://clickhouse.tech/docs/ru/operations/configuration_files/) diff --git a/docs/ru/operations/index.md b/docs/ru/operations/index.md index 99dcf652891..88212e6804f 100644 --- a/docs/ru/operations/index.md +++ b/docs/ru/operations/index.md @@ -23,4 +23,3 @@ toc_title: "Эксплуатация" - [Настройки](settings/index.md#settings) - [Утилиты](utilities/index.md) -[Оригинальная статья](https://clickhouse.tech/docs/ru/operations/) diff --git a/docs/ru/operations/monitoring.md b/docs/ru/operations/monitoring.md index 7656b04d011..da51d27ded2 100644 --- a/docs/ru/operations/monitoring.md +++ b/docs/ru/operations/monitoring.md @@ -43,4 +43,3 @@ ClickHouse собирает: Для мониторинга серверов в кластерной конфигурации необходимо установить параметр [max_replica_delay_for_distributed_queries](settings/settings.md#settings-max_replica_delay_for_distributed_queries) и использовать HTTP ресурс `/replicas_status`. Если реплика доступна и не отстаёт от других реплик, то запрос к `/replicas_status` возвращает `200 OK`. Если реплика отстаёт, то запрос возвращает `503 HTTP_SERVICE_UNAVAILABLE`, включая информацию о размере отставания. -[Оригинальная статья](https://clickhouse.tech/docs/ru/operations/system_tables/monitoring) diff --git a/docs/ru/operations/opentelemetry.md b/docs/ru/operations/opentelemetry.md index a60f1b3e085..073e7c67e9c 100644 --- a/docs/ru/operations/opentelemetry.md +++ b/docs/ru/operations/opentelemetry.md @@ -34,4 +34,3 @@ ClickHouse создает `trace spans` для каждого запроса и Теги или атрибуты сохраняются в виде двух параллельных массивов, содержащих ключи и значения. Для работы с ними используйте [ARRAY JOIN](../sql-reference/statements/select/array-join.md). -[Оригинальная статья](https://clickhouse.tech/docs/ru/operations/opentelemetry/) diff --git a/docs/ru/operations/quotas.md b/docs/ru/operations/quotas.md index 31f3a66a1c3..bf531c0b35a 100644 --- a/docs/ru/operations/quotas.md +++ b/docs/ru/operations/quotas.md @@ -107,4 +107,3 @@ toc_title: "Квоты" При перезапуске сервера, квоты сбрасываются. -[Оригинальная статья](https://clickhouse.tech/docs/ru/operations/quotas/) diff --git a/docs/ru/operations/server-configuration-parameters/index.md b/docs/ru/operations/server-configuration-parameters/index.md index f511955ebc4..503c5d32163 100644 --- a/docs/ru/operations/server-configuration-parameters/index.md +++ b/docs/ru/operations/server-configuration-parameters/index.md @@ -14,4 +14,3 @@ toc_title: "Введение" Перед изучением настроек ознакомьтесь с разделом [Конфигурационные файлы](../configuration-files.md#configuration_files), обратите внимание на использование подстановок (атрибуты `incl` и `optional`). -[Оригинальная статья](https://clickhouse.tech/docs/ru/operations/server_configuration_parameters/) diff --git a/docs/ru/operations/server-configuration-parameters/settings.md b/docs/ru/operations/server-configuration-parameters/settings.md index f46d899a3b7..b50347f6196 100644 --- a/docs/ru/operations/server-configuration-parameters/settings.md +++ b/docs/ru/operations/server-configuration-parameters/settings.md @@ -1160,4 +1160,3 @@ ClickHouse использует ZooKeeper для хранения метадан ``` -[Оригинальная статья](https://clickhouse.tech/docs/ru/operations/server_configuration_parameters/settings/) diff --git a/docs/ru/operations/settings/constraints-on-settings.md b/docs/ru/operations/settings/constraints-on-settings.md index a4c1876574d..754d6cbba8a 100644 --- a/docs/ru/operations/settings/constraints-on-settings.md +++ b/docs/ru/operations/settings/constraints-on-settings.md @@ -71,4 +71,3 @@ Code: 452, e.displayText() = DB::Exception: Setting force_index_by_date should n **Примечание:** профиль с именем `default` обрабатывается специальным образом: все ограничения на изменение настроек из этого профиля становятся дефолтными и влияют на всех пользователей, кроме тех, где эти ограничения явно переопределены. -[Оригинальная статья](https://clickhouse.tech/docs/ru/operations/settings/constraints_on_settings/) diff --git a/docs/ru/operations/settings/index.md b/docs/ru/operations/settings/index.md index 2ef1d4730a3..050df975b47 100644 --- a/docs/ru/operations/settings/index.md +++ b/docs/ru/operations/settings/index.md @@ -54,4 +54,3 @@ SELECT getSetting('custom_a'); - [Конфигурационные параметры сервера](../../operations/server-configuration-parameters/settings.md) -[Оригинальная статья](https://clickhouse.tech/docs/ru/operations/settings/) diff --git a/docs/ru/operations/settings/permissions-for-queries.md b/docs/ru/operations/settings/permissions-for-queries.md index 571f56fc3bd..8cd5a2570ca 100644 --- a/docs/ru/operations/settings/permissions-for-queries.md +++ b/docs/ru/operations/settings/permissions-for-queries.md @@ -59,4 +59,3 @@ toc_title: "Разрешения для запросов" 1 -[Оригинальная статья](https://clickhouse.tech/docs/ru/operations/settings/permissions_for_queries/) diff --git a/docs/ru/operations/settings/query-complexity.md b/docs/ru/operations/settings/query-complexity.md index c6e580a2209..c2e00302d18 100644 --- a/docs/ru/operations/settings/query-complexity.md +++ b/docs/ru/operations/settings/query-complexity.md @@ -314,4 +314,3 @@ FORMAT Null; > «Too many partitions for single INSERT block (more than» + toString(max_parts) + «). The limit is controlled by ‘max_partitions_per_insert_block’ setting. Large number of partitions is a common misconception. It will lead to severe negative performance impact, including slow server startup, slow INSERT queries and slow SELECT queries. Recommended total number of partitions for a table is under 1000..10000. Please note, that partitioning is not intended to speed up SELECT queries (ORDER BY key is sufficient to make range queries fast). Partitions are intended for data manipulation (DROP PARTITION, etc).» -[Оригинальная статья](https://clickhouse.tech/docs/ru/operations/settings/query_complexity/) diff --git a/docs/ru/operations/settings/settings-profiles.md b/docs/ru/operations/settings/settings-profiles.md index e8082919d89..d3b3d29db94 100644 --- a/docs/ru/operations/settings/settings-profiles.md +++ b/docs/ru/operations/settings/settings-profiles.md @@ -77,4 +77,3 @@ SET profile = 'web' Профиль `web` — обычный профиль, который может быть установлен с помощью запроса `SET` или параметра URL при запросе по HTTP. -[Оригинальная статья](https://clickhouse.tech/docs/ru/operations/settings/settings_profiles/) diff --git a/docs/ru/operations/settings/settings-users.md b/docs/ru/operations/settings/settings-users.md index 21cd78569df..6a10e518817 100644 --- a/docs/ru/operations/settings/settings-users.md +++ b/docs/ru/operations/settings/settings-users.md @@ -162,4 +162,3 @@ toc_title: "Настройки пользователей" Элемент `filter` содержать любое выражение, возвращающее значение типа [UInt8](../../sql-reference/data-types/int-uint.md). Обычно он содержит сравнения и логические операторы. Строки `database_name.table1`, для которых фильтр возвращает 0 не выдаются пользователю. Фильтрация несовместима с операциями `PREWHERE` и отключает оптимизацию `WHERE→PREWHERE`. -[Оригинальная статья](https://clickhouse.tech/docs/ru/operations/settings/settings_users/) diff --git a/docs/ru/operations/settings/settings.md b/docs/ru/operations/settings/settings.md index 663821158bd..ab24b7f3a44 100644 --- a/docs/ru/operations/settings/settings.md +++ b/docs/ru/operations/settings/settings.md @@ -2563,4 +2563,3 @@ SELECT * FROM test2; Значение по умолчанию: `0`. -[Оригинальная статья](https://clickhouse.tech/docs/ru/operations/settings/settings/) diff --git a/docs/ru/operations/system-tables/asynchronous_metric_log.md b/docs/ru/operations/system-tables/asynchronous_metric_log.md index 2fe617e48af..979b63f0cc8 100644 --- a/docs/ru/operations/system-tables/asynchronous_metric_log.md +++ b/docs/ru/operations/system-tables/asynchronous_metric_log.md @@ -34,4 +34,3 @@ SELECT * FROM system.asynchronous_metric_log LIMIT 10 - [system.asynchronous_metrics](#system_tables-asynchronous_metrics) — Содержит метрики, которые периодически вычисляются в фоновом режиме. - [system.metric_log](#system_tables-metric_log) — таблица фиксирующая историю значений метрик из `system.metrics` и `system.events`. -[Оригинальная статья](https://clickhouse.tech/docs/ru/operations/system_tables/asynchronous_metric_log) diff --git a/docs/ru/operations/system-tables/asynchronous_metrics.md b/docs/ru/operations/system-tables/asynchronous_metrics.md index 5ff010bc79f..9d12a119c43 100644 --- a/docs/ru/operations/system-tables/asynchronous_metrics.md +++ b/docs/ru/operations/system-tables/asynchronous_metrics.md @@ -35,5 +35,4 @@ SELECT * FROM system.asynchronous_metrics LIMIT 10 - [system.events](#system_tables-events) — таблица с количеством произошедших событий. - [system.metric_log](#system_tables-metric_log) — таблица фиксирующая историю значений метрик из `system.metrics` и `system.events`. - [Оригинальная статья](https://clickhouse.tech/docs/ru/operations/system_tables/asynchronous_metrics) \ No newline at end of file diff --git a/docs/ru/operations/system-tables/clusters.md b/docs/ru/operations/system-tables/clusters.md index 9cf84ea5f02..ddc6849b44d 100644 --- a/docs/ru/operations/system-tables/clusters.md +++ b/docs/ru/operations/system-tables/clusters.md @@ -13,4 +13,3 @@ - `port` (UInt16) — порт, на который обращаться для соединения с сервером. - `user` (String) — имя пользователя, которого использовать для соединения с сервером. -[Оригинальная статья](https://clickhouse.tech/docs/ru/operations/system_tables/clusters) diff --git a/docs/ru/operations/system-tables/columns.md b/docs/ru/operations/system-tables/columns.md index 8cb9408e7d8..af4cff85439 100644 --- a/docs/ru/operations/system-tables/columns.md +++ b/docs/ru/operations/system-tables/columns.md @@ -23,4 +23,3 @@ - `is_in_sampling_key` ([UInt8](../../sql-reference/data-types/int-uint.md)) — флаг, показывающий включение столбца в ключ выборки. - `compression_codec` ([String](../../sql-reference/data-types/string.md)) — имя кодека сжатия. -[Оригинальная статья](https://clickhouse.tech/docs/ru/operations/system_tables/columns) diff --git a/docs/ru/operations/system-tables/contributors.md b/docs/ru/operations/system-tables/contributors.md index 64c9a863bc3..6e11219e044 100644 --- a/docs/ru/operations/system-tables/contributors.md +++ b/docs/ru/operations/system-tables/contributors.md @@ -39,4 +39,3 @@ SELECT * FROM system.contributors WHERE name='Olga Khvostikova' └──────────────────┘ ``` -[Оригинальная статья](https://clickhouse.tech/docs/ru/operations/system_tables/contributors) diff --git a/docs/ru/operations/system-tables/current-roles.md b/docs/ru/operations/system-tables/current-roles.md index a948b7b1e97..42ed4260fde 100644 --- a/docs/ru/operations/system-tables/current-roles.md +++ b/docs/ru/operations/system-tables/current-roles.md @@ -8,4 +8,3 @@ - `with_admin_option` ([UInt8](../../sql-reference/data-types/int-uint.md#uint-ranges)) — Флаг, который показывает, обладает ли `current_role` роль привилегией `ADMIN OPTION`. - `is_default` ([UInt8](../../sql-reference/data-types/int-uint.md#uint-ranges)) — Флаг, который показывает, является ли `current_role` ролью по умолчанию. - [Оригинальная статья](https://clickhouse.tech/docs/ru/operations/system_tables/current-roles) diff --git a/docs/ru/operations/system-tables/data_type_families.md b/docs/ru/operations/system-tables/data_type_families.md index d8d0b5e1074..3a9a4a3413a 100644 --- a/docs/ru/operations/system-tables/data_type_families.md +++ b/docs/ru/operations/system-tables/data_type_families.md @@ -33,4 +33,3 @@ SELECT * FROM system.data_type_families WHERE alias_to = 'String' - [Синтаксис](../../sql-reference/syntax.md) — поддерживаемый SQL синтаксис. -[Оригинальная статья](https://clickhouse.tech/docs/ru/operations/system_tables/data_type_families) diff --git a/docs/ru/operations/system-tables/databases.md b/docs/ru/operations/system-tables/databases.md index 00a4b543717..026f49c0d5d 100644 --- a/docs/ru/operations/system-tables/databases.md +++ b/docs/ru/operations/system-tables/databases.md @@ -4,4 +4,3 @@ Для каждой базы данных, о которой знает сервер, будет присутствовать соответствующая запись в таблице. Эта системная таблица используется для реализации запроса `SHOW DATABASES`. -[Оригинальная статья](https://clickhouse.tech/docs/ru/operations/system_tables/databases) \ No newline at end of file diff --git a/docs/ru/operations/system-tables/detached_parts.md b/docs/ru/operations/system-tables/detached_parts.md index c59daa3985c..23fd4882c44 100644 --- a/docs/ru/operations/system-tables/detached_parts.md +++ b/docs/ru/operations/system-tables/detached_parts.md @@ -4,4 +4,3 @@ Такие куски могут быть присоединены с помощью [ALTER TABLE ATTACH PARTITION\|PART](../../sql_reference/alter/#alter_attach-partition). Остальные столбцы описаны в [system.parts](#system_tables-parts). Если имя куска некорректно, значения некоторых столбцов могут быть `NULL`. Такие куски могут быть удалены с помощью [ALTER TABLE DROP DETACHED PART](../../sql_reference/alter/#alter_drop-detached). -[Оригинальная статья](https://clickhouse.tech/docs/ru/operations/system_tables/detached_parts) diff --git a/docs/ru/operations/system-tables/dictionaries.md b/docs/ru/operations/system-tables/dictionaries.md index cd1a4acab72..6a49904aae9 100644 --- a/docs/ru/operations/system-tables/dictionaries.md +++ b/docs/ru/operations/system-tables/dictionaries.md @@ -59,4 +59,3 @@ SELECT * FROM system.dictionaries └──────────┴──────┴────────┴─────────────┴──────┴────────┴──────────────────────────────────────┴─────────────────────┴─────────────────┴─────────────┴──────────┴───────────────┴───────────────────────┴────────────────────────────┴──────────────┴──────────────┴─────────────────────┴──────────────────────────────┘───────────────────────┴────────────────┘ ``` -[Оригинальная статья](https://clickhouse.tech/docs/ru/operations/system_tables/dictionaries) \ No newline at end of file diff --git a/docs/ru/operations/system-tables/disks.md b/docs/ru/operations/system-tables/disks.md index 2832e7a1a32..186dfbd7819 100644 --- a/docs/ru/operations/system-tables/disks.md +++ b/docs/ru/operations/system-tables/disks.md @@ -10,4 +10,3 @@ Cодержит информацию о дисках, заданных в [ко - `total_space` ([UInt64](../../sql-reference/data-types/int-uint.md)) — объём диска в байтах. - `keep_free_space` ([UInt64](../../sql-reference/data-types/int-uint.md)) — место, которое должно остаться свободным на диске в байтах. Задаётся значением параметра `keep_free_space_bytes` конфигурации дисков. -[Оригинальная статья](https://clickhouse.tech/docs/ru/operations/system_tables/disks) diff --git a/docs/ru/operations/system-tables/distributed_ddl_queue.md b/docs/ru/operations/system-tables/distributed_ddl_queue.md index 71be69e98d7..99d92574a0b 100644 --- a/docs/ru/operations/system-tables/distributed_ddl_queue.md +++ b/docs/ru/operations/system-tables/distributed_ddl_queue.md @@ -61,5 +61,4 @@ exception_code: ZOK 2 rows in set. Elapsed: 0.025 sec. ``` -[Оригинальная статья](https://clickhouse.tech/docs/ru/operations/system_tables/distributed_ddl_queuedistributed_ddl_queue.md) \ No newline at end of file diff --git a/docs/ru/operations/system-tables/distribution_queue.md b/docs/ru/operations/system-tables/distribution_queue.md index 18346b34e04..5b811ab2be8 100644 --- a/docs/ru/operations/system-tables/distribution_queue.md +++ b/docs/ru/operations/system-tables/distribution_queue.md @@ -43,4 +43,3 @@ last_exception: - [Движок таблиц Distributed](../../engines/table-engines/special/distributed.md) -[Оригинальная статья](https://clickhouse.tech/docs/ru/operations/system_tables/distribution_queue) diff --git a/docs/ru/operations/system-tables/enabled-roles.md b/docs/ru/operations/system-tables/enabled-roles.md index cd3b0846718..a3f5ba179b3 100644 --- a/docs/ru/operations/system-tables/enabled-roles.md +++ b/docs/ru/operations/system-tables/enabled-roles.md @@ -9,4 +9,3 @@ - `is_current` ([UInt8](../../sql-reference/data-types/int-uint.md#uint-ranges)) — Флаг, который показывает, является ли `enabled_role` текущей ролью текущего пользователя. - `is_default` ([UInt8](../../sql-reference/data-types/int-uint.md#uint-ranges)) — Флаг, который показывает, является ли `enabled_role` ролью по умолчанию. -[Оригинальная статья](https://clickhouse.tech/docs/ru/operations/system_tables/enabled-roles) \ No newline at end of file diff --git a/docs/ru/operations/system-tables/events.md b/docs/ru/operations/system-tables/events.md index 0a48617bb5c..c05be74eea6 100644 --- a/docs/ru/operations/system-tables/events.md +++ b/docs/ru/operations/system-tables/events.md @@ -31,4 +31,3 @@ SELECT * FROM system.events LIMIT 5 - [system.metric_log](#system_tables-metric_log) — таблица фиксирующая историю значений метрик из `system.metrics` и `system.events`. - [Мониторинг](../../operations/monitoring.md) — основы мониторинга в ClickHouse. -[Оригинальная статья](https://clickhouse.tech/docs/ru/operations/system_tables/events) diff --git a/docs/ru/operations/system-tables/functions.md b/docs/ru/operations/system-tables/functions.md index c51adb2c109..de752e2018c 100644 --- a/docs/ru/operations/system-tables/functions.md +++ b/docs/ru/operations/system-tables/functions.md @@ -7,4 +7,3 @@ - `name` (`String`) – Имя функции. - `is_aggregate` (`UInt8`) – Признак, является ли функция агрегатной. -[Оригинальная статья](https://clickhouse.tech/docs/ru/operations/system_tables/functions) diff --git a/docs/ru/operations/system-tables/grants.md b/docs/ru/operations/system-tables/grants.md index 58d8a9e1e06..76a014f62dd 100644 --- a/docs/ru/operations/system-tables/grants.md +++ b/docs/ru/operations/system-tables/grants.md @@ -21,4 +21,3 @@ - `grant_option` ([UInt8](../../sql-reference/data-types/int-uint.md#uint-ranges)) — Разрешение предоставлено с опцией `WITH GRANT OPTION`, подробнее см. [GRANT](../../sql-reference/statements/grant.md#grant-privigele-syntax). -[Оригинальная статья](https://clickhouse.tech/docs/ru/operations/system_tables/grants) diff --git a/docs/ru/operations/system-tables/graphite_retentions.md b/docs/ru/operations/system-tables/graphite_retentions.md index 66fca7ba299..1098a29aac6 100644 --- a/docs/ru/operations/system-tables/graphite_retentions.md +++ b/docs/ru/operations/system-tables/graphite_retentions.md @@ -14,4 +14,3 @@ - `Tables.database` (Array(String)) - Массив имён баз данных таблиц, использующих параметр `config_name`. - `Tables.table` (Array(String)) - Массив имён таблиц, использующих параметр `config_name`. -[Оригинальная статья](https://clickhouse.tech/docs/ru/operations/system_tables/graphite_retentions) diff --git a/docs/ru/operations/system-tables/index.md b/docs/ru/operations/system-tables/index.md index e4b6f5beb9d..2760938add2 100644 --- a/docs/ru/operations/system-tables/index.md +++ b/docs/ru/operations/system-tables/index.md @@ -70,4 +70,3 @@ toc_title: "Системные таблицы" - `OSReadBytes` - `OSWriteBytes` -[Оригинальная статья](https://clickhouse.tech/docs/ru/operations/system-tables/) diff --git a/docs/ru/operations/system-tables/licenses.md b/docs/ru/operations/system-tables/licenses.md index a6a49d5e0be..598da1e72ee 100644 --- a/docs/ru/operations/system-tables/licenses.md +++ b/docs/ru/operations/system-tables/licenses.md @@ -36,4 +36,3 @@ SELECT library_name, license_type, license_path FROM system.licenses LIMIT 15 ``` -[Оригинальная статья](https://clickhouse.tech/docs/ru/operations/system_tables/licenses) diff --git a/docs/ru/operations/system-tables/merges.md b/docs/ru/operations/system-tables/merges.md index 021a95981e6..f48f0d1ac27 100644 --- a/docs/ru/operations/system-tables/merges.md +++ b/docs/ru/operations/system-tables/merges.md @@ -18,4 +18,3 @@ - `bytes_written_uncompressed UInt64` — Количество записанных байт, несжатых. - `rows_written UInt64` — Количество записанных строк. -[Оригинальная статья](https://clickhouse.tech/docs/ru/operations/system_tables/merges) diff --git a/docs/ru/operations/system-tables/metric_log.md b/docs/ru/operations/system-tables/metric_log.md index 2458c93da59..5160b32927b 100644 --- a/docs/ru/operations/system-tables/metric_log.md +++ b/docs/ru/operations/system-tables/metric_log.md @@ -48,4 +48,3 @@ CurrentMetric_ReplicatedChecks: 0 - [system.metrics](#system_tables-metrics) — таблица с мгновенно вычисляемыми метриками. - [Мониторинг](../../operations/monitoring.md) — основы мониторинга в ClickHouse. -[Оригинальная статья](https://clickhouse.tech/docs/ru/operations/system_tables/metric_log) diff --git a/docs/ru/operations/system-tables/metrics.md b/docs/ru/operations/system-tables/metrics.md index db4016687d6..13d5fbc750a 100644 --- a/docs/ru/operations/system-tables/metrics.md +++ b/docs/ru/operations/system-tables/metrics.md @@ -38,4 +38,3 @@ SELECT * FROM system.metrics LIMIT 10 - [system.metric_log](#system_tables-metric_log) — таблица фиксирующая историю значений метрик из `system.metrics` и `system.events`. - [Мониторинг](../../operations/monitoring.md) — основы мониторинга в ClickHouse. -[Оригинальная статья](https://clickhouse.tech/docs/ru/operations/system_tables/metrics) diff --git a/docs/ru/operations/system-tables/mutations.md b/docs/ru/operations/system-tables/mutations.md index 044677030ba..4370ab593e7 100644 --- a/docs/ru/operations/system-tables/mutations.md +++ b/docs/ru/operations/system-tables/mutations.md @@ -45,4 +45,3 @@ - [Движок MergeTree](../../engines/table-engines/mergetree-family/mergetree.md) - [Репликация данных](../../engines/table-engines/mergetree-family/replication.md) (семейство ReplicatedMergeTree) -[Оригинальная статья](https://clickhouse.tech/docs/ru/operations/system_tables/mutations) diff --git a/docs/ru/operations/system-tables/numbers.md b/docs/ru/operations/system-tables/numbers.md index 02192184aa1..0be4a4ce05d 100644 --- a/docs/ru/operations/system-tables/numbers.md +++ b/docs/ru/operations/system-tables/numbers.md @@ -4,4 +4,3 @@ Эту таблицу можно использовать для тестов, а также если вам нужно сделать перебор. Чтения из этой таблицы не распараллеливаются. -[Оригинальная статья](https://clickhouse.tech/docs/ru/operations/system_tables/numbers) diff --git a/docs/ru/operations/system-tables/numbers_mt.md b/docs/ru/operations/system-tables/numbers_mt.md index 12409d831a1..d66c4515ddb 100644 --- a/docs/ru/operations/system-tables/numbers_mt.md +++ b/docs/ru/operations/system-tables/numbers_mt.md @@ -3,4 +3,3 @@ То же самое, что и [system.numbers](../../operations/system-tables/numbers.md), но чтение распараллеливается. Числа могут возвращаться в произвольном порядке. Используется для тестов. -[Оригинальная статья](https://clickhouse.tech/docs/ru/operations/system_tables/numbers_mt) diff --git a/docs/ru/operations/system-tables/one.md b/docs/ru/operations/system-tables/one.md index 4231277ffe4..5cb297f06d4 100644 --- a/docs/ru/operations/system-tables/one.md +++ b/docs/ru/operations/system-tables/one.md @@ -4,4 +4,3 @@ Эта таблица используется, если в `SELECT` запросе не указана секция `FROM`. То есть, это - аналог таблицы `DUAL`, которую можно найти в других СУБД. -[Оригинальная статья](https://clickhouse.tech/docs/ru/operations/system_tables/one) diff --git a/docs/ru/operations/system-tables/opentelemetry_span_log.md b/docs/ru/operations/system-tables/opentelemetry_span_log.md index 96555064b0e..c421a602300 100644 --- a/docs/ru/operations/system-tables/opentelemetry_span_log.md +++ b/docs/ru/operations/system-tables/opentelemetry_span_log.md @@ -46,4 +46,3 @@ attribute.names: [] attribute.values: [] ``` -[Оригинальная статья](https://clickhouse.tech/docs/ru/operations/system_tables/opentelemetry_span_log) diff --git a/docs/ru/operations/system-tables/part_log.md b/docs/ru/operations/system-tables/part_log.md index 4157cd41bff..a8d892f3b67 100644 --- a/docs/ru/operations/system-tables/part_log.md +++ b/docs/ru/operations/system-tables/part_log.md @@ -66,4 +66,3 @@ error: 0 exception: ``` -[Оригинальная статья](https://clickhouse.tech/docs/ru/operations/system_tables/part_log) diff --git a/docs/ru/operations/system-tables/parts.md b/docs/ru/operations/system-tables/parts.md index 950e652332d..1c7f0ad2e9a 100644 --- a/docs/ru/operations/system-tables/parts.md +++ b/docs/ru/operations/system-tables/parts.md @@ -155,4 +155,3 @@ move_ttl_info.max: [] - [Движок MergeTree](../../engines/table-engines/mergetree-family/mergetree.md) - [TTL для столбцов и таблиц](../../engines/table-engines/mergetree-family/mergetree.md#table_engine-mergetree-ttl) -[Оригинальная статья](https://clickhouse.tech/docs/ru/operations/system_tables/parts) diff --git a/docs/ru/operations/system-tables/parts_columns.md b/docs/ru/operations/system-tables/parts_columns.md index db4d453e8f1..5640929d810 100644 --- a/docs/ru/operations/system-tables/parts_columns.md +++ b/docs/ru/operations/system-tables/parts_columns.md @@ -145,4 +145,3 @@ column_marks_bytes: 48 - [Движок MergeTree](../../engines/table-engines/mergetree-family/mergetree.md) -[Оригинальная статья](https://clickhouse.tech/docs/en/operations/system_tables/parts_columns) diff --git a/docs/ru/operations/system-tables/processes.md b/docs/ru/operations/system-tables/processes.md index c9216e162b3..682b174c483 100644 --- a/docs/ru/operations/system-tables/processes.md +++ b/docs/ru/operations/system-tables/processes.md @@ -14,4 +14,3 @@ - `query` (String) – текст запроса. Для запросов `INSERT` не содержит встаявляемые данные. - `query_id` (String) – идентификатор запроса, если был задан. -[Оригинальная статья](https://clickhouse.tech/docs/ru/operations/system_tables/processes) diff --git a/docs/ru/operations/system-tables/query_log.md b/docs/ru/operations/system-tables/query_log.md index 39f685288d8..2d9c5f3eaab 100644 --- a/docs/ru/operations/system-tables/query_log.md +++ b/docs/ru/operations/system-tables/query_log.md @@ -142,5 +142,4 @@ Settings.Values: ['0','random','1','10000000000','1'] - [system.query_thread_log](../../operations/system-tables/query_thread_log.md#system_tables-query_thread_log) — в этой таблице содержится информация о цепочке каждого выполненного запроса. -[Оригинальная статья](https://clickhouse.tech/docs/ru/operations/system_tables/query_log) diff --git a/docs/ru/operations/system-tables/query_thread_log.md b/docs/ru/operations/system-tables/query_thread_log.md index 052baf98035..0292a321524 100644 --- a/docs/ru/operations/system-tables/query_thread_log.md +++ b/docs/ru/operations/system-tables/query_thread_log.md @@ -114,4 +114,3 @@ ProfileEvents.Values: [1,1,11,11,591,148,3,71,29,6533808,1,11,72,18,47, - [system.query_log](../../operations/system-tables/query_log.md#system_tables-query_log) — описание системной таблицы `query_log`, которая содержит общую информацию о выполненных запросах. -[Оригинальная статья](https://clickhouse.tech/docs/ru/operations/system_tables/query_thread_log) diff --git a/docs/ru/operations/system-tables/quota_limits.md b/docs/ru/operations/system-tables/quota_limits.md index a9ab87055d4..4327dd2f29d 100644 --- a/docs/ru/operations/system-tables/quota_limits.md +++ b/docs/ru/operations/system-tables/quota_limits.md @@ -17,4 +17,3 @@ - `max_read_bytes` ([Nullable](../../sql-reference/data-types/nullable.md)([UInt64](../../sql-reference/data-types/int-uint.md))) — Максимальное количество байтов, считываемых из всех таблиц и табличных функций, участвующих в запросе. - `max_execution_time` ([Nullable](../../sql-reference/data-types/nullable.md)([Float64](../../sql-reference/data-types/float.md))) — Максимальное время выполнения запроса, в секундах. -[Оригинальная статья](https://clickhouse.tech/docs/ru/operations/system_tables/quota_limits) diff --git a/docs/ru/operations/system-tables/quota_usage.md b/docs/ru/operations/system-tables/quota_usage.md index cea3c4b2daa..f777b336f23 100644 --- a/docs/ru/operations/system-tables/quota_usage.md +++ b/docs/ru/operations/system-tables/quota_usage.md @@ -28,4 +28,3 @@ - [SHOW QUOTA](../../sql-reference/statements/show.md#show-quota-statement) -[Оригинальная статья](https://clickhouse.tech/docs/ru/operations/system_tables/quota_usage) diff --git a/docs/ru/operations/system-tables/quotas.md b/docs/ru/operations/system-tables/quotas.md index 15bb41a85bf..fe6b78cc44b 100644 --- a/docs/ru/operations/system-tables/quotas.md +++ b/docs/ru/operations/system-tables/quotas.md @@ -25,5 +25,4 @@ - [SHOW QUOTAS](../../sql-reference/statements/show.md#show-quotas-statement) -[Оригинальная статья](https://clickhouse.tech/docs/ru/operations/system_tables/quotas) diff --git a/docs/ru/operations/system-tables/quotas_usage.md b/docs/ru/operations/system-tables/quotas_usage.md index 9d6d339c434..3baecee8ece 100644 --- a/docs/ru/operations/system-tables/quotas_usage.md +++ b/docs/ru/operations/system-tables/quotas_usage.md @@ -29,4 +29,3 @@ - [SHOW QUOTA](../../sql-reference/statements/show.md#show-quota-statement) -[Оригинальная статья](https://clickhouse.tech/docs/ru/operations/system_tables/quotas_usage) diff --git a/docs/ru/operations/system-tables/replicas.md b/docs/ru/operations/system-tables/replicas.md index 8d4eb60c56a..7879ee707a4 100644 --- a/docs/ru/operations/system-tables/replicas.md +++ b/docs/ru/operations/system-tables/replicas.md @@ -120,5 +120,4 @@ WHERE Если этот запрос ничего не возвращает - значит всё хорошо. -[Оригинальная статья](https://clickhouse.tech/docs/ru/operations/system_tables/replicas) diff --git a/docs/ru/operations/system-tables/replicated_fetches.md b/docs/ru/operations/system-tables/replicated_fetches.md index 94584f390ee..31d5a5cfe08 100644 --- a/docs/ru/operations/system-tables/replicated_fetches.md +++ b/docs/ru/operations/system-tables/replicated_fetches.md @@ -67,4 +67,3 @@ thread_id: 54 - [Управление таблицами ReplicatedMergeTree](../../sql-reference/statements/system/#query-language-system-replicated) -[Оригинальная статья](https://clickhouse.tech/docs/en/operations/system_tables/replicated_fetches) diff --git a/docs/ru/operations/system-tables/replication_queue.md b/docs/ru/operations/system-tables/replication_queue.md index 47f64aea55d..2851551955a 100644 --- a/docs/ru/operations/system-tables/replication_queue.md +++ b/docs/ru/operations/system-tables/replication_queue.md @@ -78,4 +78,3 @@ last_postpone_time: 1970-01-01 03:00:00 - [Управление таблицами ReplicatedMergeTree](../../sql-reference/statements/system.md/#query-language-system-replicated) -[Оригинальная статья](https://clickhouse.tech/docs/ru/operations/system_tables/replication_queue) diff --git a/docs/ru/operations/system-tables/role-grants.md b/docs/ru/operations/system-tables/role-grants.md index f014af1fe3d..2c80a597857 100644 --- a/docs/ru/operations/system-tables/role-grants.md +++ b/docs/ru/operations/system-tables/role-grants.md @@ -14,4 +14,3 @@ - 1 — Роль обладает привилегией `ADMIN OPTION`. - 0 — Роль не обладает привилегией `ADMIN OPTION`. -[Оригинальная статья](https://clickhouse.tech/docs/ru/operations/system_tables/role-grants) \ No newline at end of file diff --git a/docs/ru/operations/system-tables/roles.md b/docs/ru/operations/system-tables/roles.md index 1b548e85be2..c2b94214012 100644 --- a/docs/ru/operations/system-tables/roles.md +++ b/docs/ru/operations/system-tables/roles.md @@ -14,4 +14,3 @@ - [SHOW ROLES](../../sql-reference/statements/show.md#show-roles-statement) -[Оригинальная статья](https://clickhouse.tech/docs/ru/operations/system_tables/roles) diff --git a/docs/ru/operations/system-tables/row_policies.md b/docs/ru/operations/system-tables/row_policies.md index 7d0a490f01c..f1e84a201cb 100644 --- a/docs/ru/operations/system-tables/row_policies.md +++ b/docs/ru/operations/system-tables/row_policies.md @@ -31,4 +31,3 @@ - [SHOW POLICIES](../../sql-reference/statements/show.md#show-policies-statement) -[Оригинальная статья](https://clickhouse.tech/docs/ru/operations/system_tables/row_policies) diff --git a/docs/ru/operations/system-tables/settings.md b/docs/ru/operations/system-tables/settings.md index 50ccac684c4..c9d63d336b6 100644 --- a/docs/ru/operations/system-tables/settings.md +++ b/docs/ru/operations/system-tables/settings.md @@ -50,4 +50,3 @@ SELECT * FROM system.settings WHERE changed AND name='load_balancing' - [Ограничения для значений настроек](../settings/constraints-on-settings.md) - Выражение [SHOW SETTINGS](../../sql-reference/statements/show.md#show-settings) -[Оригинальная статья](https://clickhouse.tech/docs/ru/operations/system_tables/settings) diff --git a/docs/ru/operations/system-tables/settings_profile_elements.md b/docs/ru/operations/system-tables/settings_profile_elements.md index cd801468e21..8a1461c6bb0 100644 --- a/docs/ru/operations/system-tables/settings_profile_elements.md +++ b/docs/ru/operations/system-tables/settings_profile_elements.md @@ -27,4 +27,3 @@ - `inherit_profile` ([Nullable](../../sql-reference/data-types/nullable.md)([String](../../sql-reference/data-types/string.md))) — Родительский профиль для данного профиля настроек. `NULL` если не задано. Профиль настроек может наследовать все значения и ограничения настроек (`min`, `max`, `readonly`) от своего родительского профиля. -[Оригинальная статья](https://clickhouse.tech/docs/ru/operations/system_tables/settings_profile_elements) diff --git a/docs/ru/operations/system-tables/settings_profiles.md b/docs/ru/operations/system-tables/settings_profiles.md index e1401553a4a..f8101fb0cb7 100644 --- a/docs/ru/operations/system-tables/settings_profiles.md +++ b/docs/ru/operations/system-tables/settings_profiles.md @@ -21,4 +21,3 @@ - [SHOW PROFILES](../../sql-reference/statements/show.md#show-profiles-statement) -[Оригинальная статья](https://clickhouse.tech/docs/ru/operations/system_tables/settings_profiles) diff --git a/docs/ru/operations/system-tables/stack_trace.md b/docs/ru/operations/system-tables/stack_trace.md index 0689e15c35c..58d0a1c4b6a 100644 --- a/docs/ru/operations/system-tables/stack_trace.md +++ b/docs/ru/operations/system-tables/stack_trace.md @@ -85,4 +85,3 @@ res: /lib/x86_64-linux-gnu/libc-2.27.so - [arrayMap](../../sql-reference/functions/array-functions.md#array-map) — Описание и пример использования функции `arrayMap`. - [arrayFilter](../../sql-reference/functions/array-functions.md#array-filter) — Описание и пример использования функции `arrayFilter`. -[Оригинальная статья](https://clickhouse.tech/docs/ru/operations/system_tables/stack_trace) diff --git a/docs/ru/operations/system-tables/storage_policies.md b/docs/ru/operations/system-tables/storage_policies.md index e62266af131..b2005d5f31e 100644 --- a/docs/ru/operations/system-tables/storage_policies.md +++ b/docs/ru/operations/system-tables/storage_policies.md @@ -14,4 +14,3 @@ Если политика хранения содержит несколько томов, то каждому тому соответствует отдельная запись в таблице. -[Оригинальная статья](https://clickhouse.tech/docs/ru/operations/system_tables/storage_policies) diff --git a/docs/ru/operations/system-tables/table_engines.md b/docs/ru/operations/system-tables/table_engines.md index eb198475e43..6af29753bbf 100644 --- a/docs/ru/operations/system-tables/table_engines.md +++ b/docs/ru/operations/system-tables/table_engines.md @@ -34,4 +34,3 @@ WHERE name in ('Kafka', 'MergeTree', 'ReplicatedCollapsingMergeTree') - [Настройки](../../engines/table-engines/integrations/kafka.md#table_engine-kafka-creating-a-table) Kafka - [Настройки](../../engines/table-engines/special/join.md#join-limitations-and-settings) Join -[Оригинальная статья](https://clickhouse.tech/docs/ru/operations/system_tables/table_engines) diff --git a/docs/ru/operations/system-tables/tables.md b/docs/ru/operations/system-tables/tables.md index 52de10871b2..42e55b1f6b7 100644 --- a/docs/ru/operations/system-tables/tables.md +++ b/docs/ru/operations/system-tables/tables.md @@ -37,4 +37,3 @@ Таблица `system.tables` используется при выполнении запроса `SHOW TABLES`. -[Оригинальная статья](https://clickhouse.tech/docs/ru/operations/system_tables/tables) diff --git a/docs/ru/operations/system-tables/text_log.md b/docs/ru/operations/system-tables/text_log.md index 141c3680c07..97c6ef9e2cd 100644 --- a/docs/ru/operations/system-tables/text_log.md +++ b/docs/ru/operations/system-tables/text_log.md @@ -50,4 +50,3 @@ source_file: /ClickHouse/src/Interpreters/DNSCacheUpdater.cpp; void source_line: 45 ``` -[Оригинальная статья](https://clickhouse.tech/docs/ru/operations/system_tables/text_log) diff --git a/docs/ru/operations/system-tables/trace_log.md b/docs/ru/operations/system-tables/trace_log.md index 88f4b29651b..3d22e4eabfd 100644 --- a/docs/ru/operations/system-tables/trace_log.md +++ b/docs/ru/operations/system-tables/trace_log.md @@ -50,4 +50,3 @@ trace: [371912858,371912789,371798468,371799717,371801313,3717 size: 5244400 ``` -[Оригинальная статья](https://clickhouse.tech/docs/ru/operations/system-tables/trace_log) diff --git a/docs/ru/operations/system-tables/users.md b/docs/ru/operations/system-tables/users.md index c12b91f445f..2a523ae4a9a 100644 --- a/docs/ru/operations/system-tables/users.md +++ b/docs/ru/operations/system-tables/users.md @@ -31,4 +31,3 @@ - [SHOW USERS](../../sql-reference/statements/show.md#show-users-statement) -[Оригинальная статья](https://clickhouse.tech/docs/ru/operations/system_tables/users) diff --git a/docs/ru/operations/system-tables/zookeeper.md b/docs/ru/operations/system-tables/zookeeper.md index 9a2b781d8f3..a6ce62a9d4e 100644 --- a/docs/ru/operations/system-tables/zookeeper.md +++ b/docs/ru/operations/system-tables/zookeeper.md @@ -69,4 +69,3 @@ pzxid: 987021252247 path: /clickhouse/tables/01-08/visits/replicas ``` -[Оригинальная статья](https://clickhouse.tech/docs/ru/operations/system_tables/zookeeper) diff --git a/docs/ru/operations/tips.md b/docs/ru/operations/tips.md index 0a2ca5ecac1..4535767e8e0 100644 --- a/docs/ru/operations/tips.md +++ b/docs/ru/operations/tips.md @@ -246,4 +246,3 @@ script end script ``` -[Оригинальная статья](https://clickhouse.tech/docs/ru/operations/tips/) diff --git a/docs/ru/operations/utilities/clickhouse-benchmark.md b/docs/ru/operations/utilities/clickhouse-benchmark.md index 2a883cf3bb5..b4769b17818 100644 --- a/docs/ru/operations/utilities/clickhouse-benchmark.md +++ b/docs/ru/operations/utilities/clickhouse-benchmark.md @@ -160,4 +160,3 @@ localhost:9000, queries 10, QPS: 6.082, RPS: 121959604.568, MiB/s: 930.478, resu 99.990% 0.172 sec. ``` -[Оригинальная статья](https://clickhouse.tech/docs/ru/operations/utilities/clickhouse-benchmark.md) diff --git a/docs/ru/operations/utilities/clickhouse-copier.md b/docs/ru/operations/utilities/clickhouse-copier.md index 243ad7f379b..aa4fd68f8e8 100644 --- a/docs/ru/operations/utilities/clickhouse-copier.md +++ b/docs/ru/operations/utilities/clickhouse-copier.md @@ -181,4 +181,3 @@ $ clickhouse-copier --daemon --config zookeeper.xml --task-path /task/path --bas `clickhouse-copier` отслеживает изменения `/task/path/description` и применяет их «на лету». Если вы поменяете, например, значение `max_workers`, то количество процессов, выполняющих задания, также изменится. -[Оригинальная статья](https://clickhouse.tech/docs/ru/operations/utils/clickhouse-copier/) diff --git a/docs/ru/operations/utilities/clickhouse-local.md b/docs/ru/operations/utilities/clickhouse-local.md index 137472fa993..682dc0b5ace 100644 --- a/docs/ru/operations/utilities/clickhouse-local.md +++ b/docs/ru/operations/utilities/clickhouse-local.md @@ -110,4 +110,3 @@ Read 186 rows, 4.15 KiB in 0.035 sec., 5302 rows/sec., 118.34 KiB/sec. ... ``` -[Оригинальная статья](https://clickhouse.tech/docs/ru/operations/utils/clickhouse-local/) diff --git a/docs/ru/operations/utilities/index.md b/docs/ru/operations/utilities/index.md index 8b533c29ff5..fa257fb4b1a 100644 --- a/docs/ru/operations/utilities/index.md +++ b/docs/ru/operations/utilities/index.md @@ -9,4 +9,3 @@ toc_title: "Обзор" - [clickhouse-local](clickhouse-local.md) - [clickhouse-copier](clickhouse-copier.md) - копирует (и перешардирует) данные с одного кластера на другой. -[Оригинальная статья](https://clickhouse.tech/docs/ru/operations/utils/) diff --git a/docs/ru/sql-reference/aggregate-functions/combinators.md b/docs/ru/sql-reference/aggregate-functions/combinators.md index 3b35716ec27..aaf03428d7c 100644 --- a/docs/ru/sql-reference/aggregate-functions/combinators.md +++ b/docs/ru/sql-reference/aggregate-functions/combinators.md @@ -248,4 +248,3 @@ FROM people └────────┴───────────────────────────┘ ``` -[Оригинальная статья](https://clickhouse.tech/docs/ru/query_language/agg_functions/combinators/) diff --git a/docs/ru/sql-reference/aggregate-functions/index.md b/docs/ru/sql-reference/aggregate-functions/index.md index 3c931222f58..7afb6a374a7 100644 --- a/docs/ru/sql-reference/aggregate-functions/index.md +++ b/docs/ru/sql-reference/aggregate-functions/index.md @@ -57,4 +57,3 @@ SELECT groupArray(y) FROM t_null_big `groupArray` не включает `NULL` в результирующий массив. -[Оригинальная статья](https://clickhouse.tech/docs/ru/sql-reference/aggregate-functions/) diff --git a/docs/ru/sql-reference/aggregate-functions/parametric-functions.md b/docs/ru/sql-reference/aggregate-functions/parametric-functions.md index 61518cb6f02..806d0140a9d 100644 --- a/docs/ru/sql-reference/aggregate-functions/parametric-functions.md +++ b/docs/ru/sql-reference/aggregate-functions/parametric-functions.md @@ -481,4 +481,3 @@ FROM Решение: пишем в запросе GROUP BY SearchPhrase HAVING uniqUpTo(4)(UserID) >= 5 ``` -[Оригинальная статья](https://clickhouse.tech/docs/ru/query_language/agg_functions/parametric_functions/) diff --git a/docs/ru/sql-reference/aggregate-functions/reference/any.md b/docs/ru/sql-reference/aggregate-functions/reference/any.md index 38c412813ab..6142b9a2092 100644 --- a/docs/ru/sql-reference/aggregate-functions/reference/any.md +++ b/docs/ru/sql-reference/aggregate-functions/reference/any.md @@ -12,4 +12,3 @@ toc_priority: 6 При наличии в запросе `SELECT` секции `GROUP BY` или хотя бы одной агрегатной функции, ClickHouse (в отличие от, например, MySQL) требует, чтобы все выражения в секциях `SELECT`, `HAVING`, `ORDER BY` вычислялись из ключей или из агрегатных функций. То есть, каждый выбираемый из таблицы столбец, должен использоваться либо в ключах, либо внутри агрегатных функций. Чтобы получить поведение, как в MySQL, вы можете поместить остальные столбцы в агрегатную функцию `any`. -[Оригинальная статья](https://clickhouse.tech/docs/en/sql-reference/aggregate-functions/reference/any/) diff --git a/docs/ru/sql-reference/aggregate-functions/reference/anyheavy.md b/docs/ru/sql-reference/aggregate-functions/reference/anyheavy.md index 19fda7f64b7..bb7a01a47f3 100644 --- a/docs/ru/sql-reference/aggregate-functions/reference/anyheavy.md +++ b/docs/ru/sql-reference/aggregate-functions/reference/anyheavy.md @@ -29,4 +29,3 @@ FROM ontime └───────┘ ``` -[Оригинальная статья](https://clickhouse.tech/docs/en/sql-reference/aggregate-functions/reference/anyheavy/) diff --git a/docs/ru/sql-reference/aggregate-functions/reference/anylast.md b/docs/ru/sql-reference/aggregate-functions/reference/anylast.md index da68c926d43..7be380461f7 100644 --- a/docs/ru/sql-reference/aggregate-functions/reference/anylast.md +++ b/docs/ru/sql-reference/aggregate-functions/reference/anylast.md @@ -7,4 +7,3 @@ toc_priority: 104 Выбирает последнее попавшееся значение. Результат так же недетерминирован, как и для функции [any](../../../sql-reference/aggregate-functions/reference/any.md). -[Оригинальная статья](https://clickhouse.tech/docs/en/sql-reference/aggregate-functions/reference/anylast/) diff --git a/docs/ru/sql-reference/aggregate-functions/reference/argmax.md b/docs/ru/sql-reference/aggregate-functions/reference/argmax.md index dd2df23e1cd..246bfcfba9d 100644 --- a/docs/ru/sql-reference/aggregate-functions/reference/argmax.md +++ b/docs/ru/sql-reference/aggregate-functions/reference/argmax.md @@ -63,4 +63,3 @@ SELECT argMax(user, salary), argMax(tuple(user, salary), salary), argMax(tuple(u └──────────────────────┴─────────────────────────────────────┴─────────────────────────────┘ ``` -[Оригинальная статья](https://clickhouse.tech/docs/ru/sql-reference/aggregate-functions/reference/argmax/) diff --git a/docs/ru/sql-reference/aggregate-functions/reference/argmin.md b/docs/ru/sql-reference/aggregate-functions/reference/argmin.md index 8c25b79f92a..811f3706d2b 100644 --- a/docs/ru/sql-reference/aggregate-functions/reference/argmin.md +++ b/docs/ru/sql-reference/aggregate-functions/reference/argmin.md @@ -63,4 +63,3 @@ SELECT argMin(user, salary), argMin(tuple(user, salary)) FROM salary; └──────────────────────┴─────────────────────────────┘ ``` -[Оригинальная статья](https://clickhouse.tech/docs/ru/sql-reference/aggregate-functions/reference/argmin/) diff --git a/docs/ru/sql-reference/aggregate-functions/reference/avg.md b/docs/ru/sql-reference/aggregate-functions/reference/avg.md index c032199aa32..c5e1dec14e0 100644 --- a/docs/ru/sql-reference/aggregate-functions/reference/avg.md +++ b/docs/ru/sql-reference/aggregate-functions/reference/avg.md @@ -61,4 +61,3 @@ SELECT avg(t) FROM test; └────────┘ ``` -[Оригинальная статья](https://clickhouse.tech/docs/ru/sql-reference/aggregate-functions/reference/avg/) diff --git a/docs/ru/sql-reference/aggregate-functions/reference/avgweighted.md b/docs/ru/sql-reference/aggregate-functions/reference/avgweighted.md index 72e6ca5c88c..5e63ed61c6d 100644 --- a/docs/ru/sql-reference/aggregate-functions/reference/avgweighted.md +++ b/docs/ru/sql-reference/aggregate-functions/reference/avgweighted.md @@ -43,4 +43,3 @@ FROM values('x Int8, w Int8', (4, 1), (1, 0), (10, 2)) └────────────────────────┘ ``` -[Оригинальная статья](https://clickhouse.tech/docs/en/sql-reference/aggregate-functions/reference/avgweighted/) diff --git a/docs/ru/sql-reference/aggregate-functions/reference/corr.md b/docs/ru/sql-reference/aggregate-functions/reference/corr.md index 6d631241f6a..7522dcebd0b 100644 --- a/docs/ru/sql-reference/aggregate-functions/reference/corr.md +++ b/docs/ru/sql-reference/aggregate-functions/reference/corr.md @@ -11,4 +11,3 @@ toc_priority: 107 !!! note "Примечание" Функция использует вычислительно неустойчивый алгоритм. Если для ваших расчётов необходима [вычислительная устойчивость](https://ru.wikipedia.org/wiki/Вычислительная_устойчивость), используйте функцию `corrStable`. Она работает медленнее, но обеспечивает меньшую вычислительную ошибку. -[Оригинальная статья](https://clickhouse.tech/docs/en/sql-reference/aggregate-functions/reference/corr/) diff --git a/docs/ru/sql-reference/aggregate-functions/reference/count.md b/docs/ru/sql-reference/aggregate-functions/reference/count.md index d99c3b2aeb2..762acce5602 100644 --- a/docs/ru/sql-reference/aggregate-functions/reference/count.md +++ b/docs/ru/sql-reference/aggregate-functions/reference/count.md @@ -69,4 +69,3 @@ SELECT count(DISTINCT num) FROM t Этот пример показывает, что `count(DISTINCT num)` выполняется с помощью функции `uniqExact` в соответствии со значением настройки `count_distinct_implementation`. -[Оригинальная статья](https://clickhouse.tech/docs/en/sql-reference/aggregate-functions/reference/count/) diff --git a/docs/ru/sql-reference/aggregate-functions/reference/covarpop.md b/docs/ru/sql-reference/aggregate-functions/reference/covarpop.md index e30b19924f9..1438fefbd8e 100644 --- a/docs/ru/sql-reference/aggregate-functions/reference/covarpop.md +++ b/docs/ru/sql-reference/aggregate-functions/reference/covarpop.md @@ -11,4 +11,3 @@ toc_priority: 36 !!! note "Примечание" Функция использует вычислительно неустойчивый алгоритм. Если для ваших расчётов необходима [вычислительная устойчивость](https://ru.wikipedia.org/wiki/Вычислительная_устойчивость), используйте функцию `covarPopStable`. Она работает медленнее, но обеспечивает меньшую вычислительную ошибку. -[Оригинальная статья](https://clickhouse.tech/docs/en/sql-reference/aggregate-functions/reference/covarpop/) diff --git a/docs/ru/sql-reference/aggregate-functions/reference/covarsamp.md b/docs/ru/sql-reference/aggregate-functions/reference/covarsamp.md index 7fa9a1d3f2c..b4cea16f4c0 100644 --- a/docs/ru/sql-reference/aggregate-functions/reference/covarsamp.md +++ b/docs/ru/sql-reference/aggregate-functions/reference/covarsamp.md @@ -13,4 +13,3 @@ toc_priority: 37 !!! note "Примечание" Функция использует вычислительно неустойчивый алгоритм. Если для ваших расчётов необходима [вычислительная устойчивость](https://ru.wikipedia.org/wiki/Вычислительная_устойчивость), используйте функцию `covarSampStable`. Она работает медленнее, но обеспечивает меньшую вычислительную ошибку. -[Оригинальная статья](https://clickhouse.tech/docs/en/sql-reference/aggregate-functions/reference/covarsamp/) diff --git a/docs/ru/sql-reference/aggregate-functions/reference/grouparray.md b/docs/ru/sql-reference/aggregate-functions/reference/grouparray.md index 7640795fc51..370190dbb3c 100644 --- a/docs/ru/sql-reference/aggregate-functions/reference/grouparray.md +++ b/docs/ru/sql-reference/aggregate-functions/reference/grouparray.md @@ -14,4 +14,3 @@ toc_priority: 110 В некоторых случаях, вы всё же можете рассчитывать на порядок выполнения запроса. Это — случаи, когда `SELECT` идёт из подзапроса, в котором используется `ORDER BY`. -[Оригинальная статья](https://clickhouse.tech/docs/en/sql-reference/aggregate-functions/reference/grouparray/) diff --git a/docs/ru/sql-reference/aggregate-functions/reference/grouparrayinsertat.md b/docs/ru/sql-reference/aggregate-functions/reference/grouparrayinsertat.md index 5c73bccc2bb..6933f17aabf 100644 --- a/docs/ru/sql-reference/aggregate-functions/reference/grouparrayinsertat.md +++ b/docs/ru/sql-reference/aggregate-functions/reference/grouparrayinsertat.md @@ -90,4 +90,3 @@ SELECT groupArrayInsertAt(number, 0) FROM numbers_mt(10) SETTINGS max_block_size └───────────────────────────────┘ ``` -[Оригинальная статья](https://clickhouse.tech/docs/en/sql-reference/aggregate-functions/reference/grouparrayinsertat/) diff --git a/docs/ru/sql-reference/aggregate-functions/reference/grouparraymovingavg.md b/docs/ru/sql-reference/aggregate-functions/reference/grouparraymovingavg.md index 6307189c440..5417cafa91f 100644 --- a/docs/ru/sql-reference/aggregate-functions/reference/grouparraymovingavg.md +++ b/docs/ru/sql-reference/aggregate-functions/reference/grouparraymovingavg.md @@ -75,4 +75,3 @@ FROM t └───────────┴──────────────────────────────────┴───────────────────────┘ ``` -[Оригинальная статья](https://clickhouse.tech/docs/en/sql-reference/aggregate-functions/reference/grouparraymovingavg/) diff --git a/docs/ru/sql-reference/aggregate-functions/reference/grouparraymovingsum.md b/docs/ru/sql-reference/aggregate-functions/reference/grouparraymovingsum.md index c95f1b0b0eb..97eae78bd5b 100644 --- a/docs/ru/sql-reference/aggregate-functions/reference/grouparraymovingsum.md +++ b/docs/ru/sql-reference/aggregate-functions/reference/grouparraymovingsum.md @@ -75,4 +75,3 @@ FROM t └────────────┴─────────────────────────────────┴────────────────────────┘ ``` -[Оригинальная статья](https://clickhouse.tech/docs/en/sql-reference/aggregate-functions/reference/grouparraymovingsum/) diff --git a/docs/ru/sql-reference/aggregate-functions/reference/groupbitand.md b/docs/ru/sql-reference/aggregate-functions/reference/groupbitand.md index 03aff64fecf..8ec7a98c2ef 100644 --- a/docs/ru/sql-reference/aggregate-functions/reference/groupbitand.md +++ b/docs/ru/sql-reference/aggregate-functions/reference/groupbitand.md @@ -45,4 +45,3 @@ binary decimal 00000100 = 4 ``` -[Оригинальная статья](https://clickhouse.tech/docs/en/sql-reference/aggregate-functions/reference/groupbitand/) diff --git a/docs/ru/sql-reference/aggregate-functions/reference/groupbitmap.md b/docs/ru/sql-reference/aggregate-functions/reference/groupbitmap.md index a4be18b75ec..d42960f8b80 100644 --- a/docs/ru/sql-reference/aggregate-functions/reference/groupbitmap.md +++ b/docs/ru/sql-reference/aggregate-functions/reference/groupbitmap.md @@ -43,4 +43,3 @@ num 3 ``` -[Оригинальная статья](https://clickhouse.tech/docs/en/sql-reference/aggregate-functions/reference/groupbitmap/) diff --git a/docs/ru/sql-reference/aggregate-functions/reference/groupbitor.md b/docs/ru/sql-reference/aggregate-functions/reference/groupbitor.md index e1afced014f..12cff73f575 100644 --- a/docs/ru/sql-reference/aggregate-functions/reference/groupbitor.md +++ b/docs/ru/sql-reference/aggregate-functions/reference/groupbitor.md @@ -45,4 +45,3 @@ binary decimal 01111101 = 125 ``` -[Оригинальная статья](https://clickhouse.tech/docs/en/sql-reference/aggregate-functions/reference/groupbitor/) diff --git a/docs/ru/sql-reference/aggregate-functions/reference/groupbitxor.md b/docs/ru/sql-reference/aggregate-functions/reference/groupbitxor.md index a80f86b2a5f..a0ec75fa8db 100644 --- a/docs/ru/sql-reference/aggregate-functions/reference/groupbitxor.md +++ b/docs/ru/sql-reference/aggregate-functions/reference/groupbitxor.md @@ -45,4 +45,3 @@ binary decimal 01101000 = 104 ``` -[Оригинальная статья](https://clickhouse.tech/docs/en/sql-reference/aggregate-functions/reference/groupbitxor/) diff --git a/docs/ru/sql-reference/aggregate-functions/reference/groupuniqarray.md b/docs/ru/sql-reference/aggregate-functions/reference/groupuniqarray.md index cecc63aef22..7d64b13a203 100644 --- a/docs/ru/sql-reference/aggregate-functions/reference/groupuniqarray.md +++ b/docs/ru/sql-reference/aggregate-functions/reference/groupuniqarray.md @@ -10,4 +10,3 @@ toc_priority: 111 Функция `groupUniqArray(max_size)(x)` ограничивает размер результирующего массива до `max_size` элементов. Например, `groupUniqArray(1)(x)` равнозначно `[any(x)]`. -[Оригинальная статья](https://clickhouse.tech/docs/en/sql-reference/aggregate-functions/reference/groupuniqarray/) diff --git a/docs/ru/sql-reference/aggregate-functions/reference/index.md b/docs/ru/sql-reference/aggregate-functions/reference/index.md index e496893a771..1af07623ade 100644 --- a/docs/ru/sql-reference/aggregate-functions/reference/index.md +++ b/docs/ru/sql-reference/aggregate-functions/reference/index.md @@ -65,4 +65,3 @@ toc_hidden: true - [stochasticLinearRegression](../../../sql-reference/aggregate-functions/reference/stochasticlinearregression.md) - [stochasticLogisticRegression](../../../sql-reference/aggregate-functions/reference/stochasticlogisticregression.md) -[Оригинальная статья](https://clickhouse.tech/docs/ru/sql-reference/aggregate-functions/reference) diff --git a/docs/ru/sql-reference/aggregate-functions/reference/kurtpop.md b/docs/ru/sql-reference/aggregate-functions/reference/kurtpop.md index a00dae51ed6..4220195dc2f 100644 --- a/docs/ru/sql-reference/aggregate-functions/reference/kurtpop.md +++ b/docs/ru/sql-reference/aggregate-functions/reference/kurtpop.md @@ -24,4 +24,3 @@ kurtPop(expr) SELECT kurtPop(value) FROM series_with_value_column ``` -[Оригинальная статья](https://clickhouse.tech/docs/en/sql-reference/aggregate-functions/reference/kurtpop/) diff --git a/docs/ru/sql-reference/aggregate-functions/reference/kurtsamp.md b/docs/ru/sql-reference/aggregate-functions/reference/kurtsamp.md index 379d74ec0c3..41668d7ee17 100644 --- a/docs/ru/sql-reference/aggregate-functions/reference/kurtsamp.md +++ b/docs/ru/sql-reference/aggregate-functions/reference/kurtsamp.md @@ -26,4 +26,3 @@ kurtSamp(expr) SELECT kurtSamp(value) FROM series_with_value_column ``` -[Оригинальная статья](https://clickhouse.tech/docs/en/sql-reference/aggregate-functions/reference/kurtsamp/) diff --git a/docs/ru/sql-reference/aggregate-functions/reference/mannwhitneyutest.md b/docs/ru/sql-reference/aggregate-functions/reference/mannwhitneyutest.md index a4647ecfb34..5e101b89c4e 100644 --- a/docs/ru/sql-reference/aggregate-functions/reference/mannwhitneyutest.md +++ b/docs/ru/sql-reference/aggregate-functions/reference/mannwhitneyutest.md @@ -69,4 +69,3 @@ SELECT mannWhitneyUTest('greater')(sample_data, sample_index) FROM mww_ttest; - [U-критерий Манна — Уитни](https://ru.wikipedia.org/wiki/U-%D0%BA%D1%80%D0%B8%D1%82%D0%B5%D1%80%D0%B8%D0%B9_%D0%9C%D0%B0%D0%BD%D0%BD%D0%B0_%E2%80%94_%D0%A3%D0%B8%D1%82%D0%BD%D0%B8) -[Оригинальная статья](https://clickhouse.tech/docs/ru/sql-reference/aggregate-functions/reference/mannwhitneyutest/) diff --git a/docs/ru/sql-reference/aggregate-functions/reference/max.md b/docs/ru/sql-reference/aggregate-functions/reference/max.md index 4ee577471ea..4f61ecd051d 100644 --- a/docs/ru/sql-reference/aggregate-functions/reference/max.md +++ b/docs/ru/sql-reference/aggregate-functions/reference/max.md @@ -6,4 +6,3 @@ toc_priority: 3 Вычисляет максимум. -[Оригинальная статья](https://clickhouse.tech/docs/en/sql-reference/aggregate-functions/reference/max/) diff --git a/docs/ru/sql-reference/aggregate-functions/reference/median.md b/docs/ru/sql-reference/aggregate-functions/reference/median.md index 803b2309665..a208c21dd21 100644 --- a/docs/ru/sql-reference/aggregate-functions/reference/median.md +++ b/docs/ru/sql-reference/aggregate-functions/reference/median.md @@ -40,4 +40,3 @@ SELECT medianDeterministic(val, 1) FROM t └─────────────────────────────┘ ``` -[Оригинальная статья](https://clickhouse.tech/docs/en/sql-reference/aggregate-functions/reference/median/) diff --git a/docs/ru/sql-reference/aggregate-functions/reference/min.md b/docs/ru/sql-reference/aggregate-functions/reference/min.md index 7b56de3aed4..16dd577e790 100644 --- a/docs/ru/sql-reference/aggregate-functions/reference/min.md +++ b/docs/ru/sql-reference/aggregate-functions/reference/min.md @@ -6,4 +6,3 @@ toc_priority: 2 Вычисляет минимум. -[Оригинальная статья](https://clickhouse.tech/docs/en/sql-reference/aggregate-functions/reference/min/) diff --git a/docs/ru/sql-reference/aggregate-functions/reference/quantile.md b/docs/ru/sql-reference/aggregate-functions/reference/quantile.md index 10fec16ab94..05446856a8a 100644 --- a/docs/ru/sql-reference/aggregate-functions/reference/quantile.md +++ b/docs/ru/sql-reference/aggregate-functions/reference/quantile.md @@ -65,4 +65,3 @@ SELECT quantile(val) FROM t - [median](../../../sql-reference/aggregate-functions/reference/median.md#median) - [quantiles](../../../sql-reference/aggregate-functions/reference/quantiles.md#quantiles) -[Оригинальная статья](https://clickhouse.tech/docs/en/sql-reference/aggregate-functions/reference/quantile/) diff --git a/docs/ru/sql-reference/aggregate-functions/reference/quantiledeterministic.md b/docs/ru/sql-reference/aggregate-functions/reference/quantiledeterministic.md index fdbcda821f6..02c50c58ae8 100644 --- a/docs/ru/sql-reference/aggregate-functions/reference/quantiledeterministic.md +++ b/docs/ru/sql-reference/aggregate-functions/reference/quantiledeterministic.md @@ -65,4 +65,3 @@ SELECT quantileDeterministic(val, 1) FROM t - [median](../../../sql-reference/aggregate-functions/reference/median.md#median) - [quantiles](../../../sql-reference/aggregate-functions/reference/quantiles.md#quantiles) -[Оригинальная статья](https://clickhouse.tech/docs/en/sql-reference/aggregate-functions/reference/qurntiledeterministic/) diff --git a/docs/ru/sql-reference/aggregate-functions/reference/quantileexact.md b/docs/ru/sql-reference/aggregate-functions/reference/quantileexact.md index 4ee815a94fb..7347318fae0 100644 --- a/docs/ru/sql-reference/aggregate-functions/reference/quantileexact.md +++ b/docs/ru/sql-reference/aggregate-functions/reference/quantileexact.md @@ -163,4 +163,3 @@ SELECT quantileExactHigh(number) FROM numbers(10) - [median](../../../sql-reference/aggregate-functions/reference/median.md#median) - [quantiles](../../../sql-reference/aggregate-functions/reference/quantiles.md#quantiles) -[Оригинальная статья](https://clickhouse.tech/docs/en/sql-reference/aggregate-functions/reference/quantileexact/) diff --git a/docs/ru/sql-reference/aggregate-functions/reference/quantileexactweighted.md b/docs/ru/sql-reference/aggregate-functions/reference/quantileexactweighted.md index f6982d4566f..baef55c818d 100644 --- a/docs/ru/sql-reference/aggregate-functions/reference/quantileexactweighted.md +++ b/docs/ru/sql-reference/aggregate-functions/reference/quantileexactweighted.md @@ -66,4 +66,3 @@ SELECT quantileExactWeighted(n, val) FROM t - [median](../../../sql-reference/aggregate-functions/reference/median.md#median) - [quantiles](../../../sql-reference/aggregate-functions/reference/quantiles.md#quantiles) -[Оригинальная статья](https://clickhouse.tech/docs/en/sql-reference/aggregate-functions/reference/quantileexactweited/) diff --git a/docs/ru/sql-reference/aggregate-functions/reference/quantiles.md b/docs/ru/sql-reference/aggregate-functions/reference/quantiles.md index 82e806b67fa..671cbc1fc4d 100644 --- a/docs/ru/sql-reference/aggregate-functions/reference/quantiles.md +++ b/docs/ru/sql-reference/aggregate-functions/reference/quantiles.md @@ -8,4 +8,3 @@ Syntax: `quantiles(level1, level2, …)(x)` All the quantile functions also have corresponding quantiles functions: `quantiles`, `quantilesDeterministic`, `quantilesTiming`, `quantilesTimingWeighted`, `quantilesExact`, `quantilesExactWeighted`, `quantilesTDigest`. These functions calculate all the quantiles of the listed levels in one pass, and return an array of the resulting values. -[Оригинальная статья](https://clickhouse.tech/docs/en/sql-reference/aggregate-functions/reference/quantiles/) diff --git a/docs/ru/sql-reference/aggregate-functions/reference/quantiletdigest.md b/docs/ru/sql-reference/aggregate-functions/reference/quantiletdigest.md index f372e308e73..fbd1abadb54 100644 --- a/docs/ru/sql-reference/aggregate-functions/reference/quantiletdigest.md +++ b/docs/ru/sql-reference/aggregate-functions/reference/quantiletdigest.md @@ -56,4 +56,3 @@ SELECT quantileTDigest(number) FROM numbers(10) - [median](../../../sql-reference/aggregate-functions/reference/median.md#median) - [quantiles](../../../sql-reference/aggregate-functions/reference/quantiles.md#quantiles) -[Оригинальная статья](https://clickhouse.tech/docs/en/sql-reference/aggregate-functions/reference/qurntiledigest/) diff --git a/docs/ru/sql-reference/aggregate-functions/reference/quantiletdigestweighted.md b/docs/ru/sql-reference/aggregate-functions/reference/quantiletdigestweighted.md index b6dd846967b..6d943f568fc 100644 --- a/docs/ru/sql-reference/aggregate-functions/reference/quantiletdigestweighted.md +++ b/docs/ru/sql-reference/aggregate-functions/reference/quantiletdigestweighted.md @@ -57,4 +57,3 @@ SELECT quantileTDigestWeighted(number, 1) FROM numbers(10) - [median](../../../sql-reference/aggregate-functions/reference/median.md#median) - [quantiles](../../../sql-reference/aggregate-functions/reference/quantiles.md#quantiles) -[Оригинальная статья](https://clickhouse.tech/docs/en/sql-reference/aggregate-functions/reference/quantiledigestweighted/) diff --git a/docs/ru/sql-reference/aggregate-functions/reference/quantiletiming.md b/docs/ru/sql-reference/aggregate-functions/reference/quantiletiming.md index 32e5e6ce31b..c71332b0471 100644 --- a/docs/ru/sql-reference/aggregate-functions/reference/quantiletiming.md +++ b/docs/ru/sql-reference/aggregate-functions/reference/quantiletiming.md @@ -85,4 +85,3 @@ SELECT quantileTiming(response_time) FROM t - [median](../../../sql-reference/aggregate-functions/reference/median.md#median) - [quantiles](../../../sql-reference/aggregate-functions/reference/quantiles.md#quantiles) -[Оригинальная статья](https://clickhouse.tech/docs/en/sql-reference/aggregate-functions/reference/quantiletiming/) diff --git a/docs/ru/sql-reference/aggregate-functions/reference/quantiletimingweighted.md b/docs/ru/sql-reference/aggregate-functions/reference/quantiletimingweighted.md index 4a7fcc666d5..c732e6a9bb8 100644 --- a/docs/ru/sql-reference/aggregate-functions/reference/quantiletimingweighted.md +++ b/docs/ru/sql-reference/aggregate-functions/reference/quantiletimingweighted.md @@ -84,4 +84,3 @@ SELECT quantileTimingWeighted(response_time, weight) FROM t - [median](../../../sql-reference/aggregate-functions/reference/median.md#median) - [quantiles](../../../sql-reference/aggregate-functions/reference/quantiles.md#quantiles) -[Оригинальная статья](https://clickhouse.tech/docs/en/sql-reference/aggregate-functions/reference/quantiletiming weighted/) diff --git a/docs/ru/sql-reference/aggregate-functions/reference/simplelinearregression.md b/docs/ru/sql-reference/aggregate-functions/reference/simplelinearregression.md index 370b1bde8d2..f634e553738 100644 --- a/docs/ru/sql-reference/aggregate-functions/reference/simplelinearregression.md +++ b/docs/ru/sql-reference/aggregate-functions/reference/simplelinearregression.md @@ -41,4 +41,3 @@ SELECT arrayReduce('simpleLinearRegression', [0, 1, 2, 3], [3, 4, 5, 6]) └───────────────────────────────────────────────────────────────────┘ ``` -[Оригинальная статья](https://clickhouse.tech/docs/en/sql-reference/aggregate-functions/reference/simplelinearregression/) diff --git a/docs/ru/sql-reference/aggregate-functions/reference/skewpop.md b/docs/ru/sql-reference/aggregate-functions/reference/skewpop.md index a6dee5dc5ef..298aa6b76fc 100644 --- a/docs/ru/sql-reference/aggregate-functions/reference/skewpop.md +++ b/docs/ru/sql-reference/aggregate-functions/reference/skewpop.md @@ -24,4 +24,3 @@ skewPop(expr) SELECT skewPop(value) FROM series_with_value_column ``` -[Оригинальная статья](https://clickhouse.tech/docs/en/sql-reference/aggregate-functions/reference/skewpop/) diff --git a/docs/ru/sql-reference/aggregate-functions/reference/skewsamp.md b/docs/ru/sql-reference/aggregate-functions/reference/skewsamp.md index 171eb5e304a..872f0fee875 100644 --- a/docs/ru/sql-reference/aggregate-functions/reference/skewsamp.md +++ b/docs/ru/sql-reference/aggregate-functions/reference/skewsamp.md @@ -26,4 +26,3 @@ skewSamp(expr) SELECT skewSamp(value) FROM series_with_value_column ``` -[Оригинальная статья](https://clickhouse.tech/docs/en/sql-reference/aggregate-functions/reference/skewsamp/) diff --git a/docs/ru/sql-reference/aggregate-functions/reference/stddevpop.md b/docs/ru/sql-reference/aggregate-functions/reference/stddevpop.md index ada8b8884cd..66d63147586 100644 --- a/docs/ru/sql-reference/aggregate-functions/reference/stddevpop.md +++ b/docs/ru/sql-reference/aggregate-functions/reference/stddevpop.md @@ -9,4 +9,3 @@ toc_priority: 30 !!! note "Примечание" Функция использует вычислительно неустойчивый алгоритм. Если для ваших расчётов необходима [вычислительная устойчивость](https://ru.wikipedia.org/wiki/Вычислительная_устойчивость), используйте функцию `stddevPopStable`. Она работает медленнее, но обеспечивает меньшую вычислительную ошибку. -[Оригинальная статья](https://clickhouse.tech/docs/en/sql-reference/aggregate-functions/reference/stddevpop/) diff --git a/docs/ru/sql-reference/aggregate-functions/reference/stddevsamp.md b/docs/ru/sql-reference/aggregate-functions/reference/stddevsamp.md index 952b6bcde68..5fbf438e894 100644 --- a/docs/ru/sql-reference/aggregate-functions/reference/stddevsamp.md +++ b/docs/ru/sql-reference/aggregate-functions/reference/stddevsamp.md @@ -9,4 +9,3 @@ toc_priority: 31 !!! note "Примечание" Функция использует вычислительно неустойчивый алгоритм. Если для ваших расчётов необходима [вычислительная устойчивость](https://ru.wikipedia.org/wiki/Вычислительная_устойчивость), используйте функцию `stddevSampStable`. Она работает медленнее, но обеспечивает меньшую вычислительную ошибку. -[Оригинальная статья](https://clickhouse.tech/docs/en/sql-reference/aggregate-functions/reference/stddevsamp/) diff --git a/docs/ru/sql-reference/aggregate-functions/reference/stochasticlinearregression.md b/docs/ru/sql-reference/aggregate-functions/reference/stochasticlinearregression.md index 0b268e9ea1b..6da0f6caacd 100644 --- a/docs/ru/sql-reference/aggregate-functions/reference/stochasticlinearregression.md +++ b/docs/ru/sql-reference/aggregate-functions/reference/stochasticlinearregression.md @@ -86,4 +86,3 @@ evalMLMethod(model, param1, param2) FROM test_data - [stochasticLogisticRegression](../../../sql-reference/aggregate-functions/reference/stochasticlinearregression.md#agg_functions-stochasticlogisticregression) - [Отличие линейной от логистической регрессии.](https://stackoverflow.com/questions/12146914/what-is-the-difference-between-linear-regression-and-logistic-regression) -[Оригинальная статья](https://clickhouse.tech/docs/en/sql-reference/aggregate-functions/reference/stochasticlinearregression/) diff --git a/docs/ru/sql-reference/aggregate-functions/reference/stochasticlogisticregression.md b/docs/ru/sql-reference/aggregate-functions/reference/stochasticlogisticregression.md index 01d3a0797bd..67454aa2c1b 100644 --- a/docs/ru/sql-reference/aggregate-functions/reference/stochasticlogisticregression.md +++ b/docs/ru/sql-reference/aggregate-functions/reference/stochasticlogisticregression.md @@ -54,4 +54,3 @@ stochasticLogisticRegression(1.0, 1.0, 10, 'SGD') - [stochasticLinearRegression](../../../sql-reference/aggregate-functions/reference/stochasticlinearregression.md#agg_functions-stochasticlinearregression) - [Отличие линейной от логистической регрессии](https://moredez.ru/q/51225972/) -[Оригинальная статья](https://clickhouse.tech/docs/en/sql-reference/aggregate-functions/reference/stochasticlogisticregression/) diff --git a/docs/ru/sql-reference/aggregate-functions/reference/studentttest.md b/docs/ru/sql-reference/aggregate-functions/reference/studentttest.md index 77378de95d1..a9ce46a51ba 100644 --- a/docs/ru/sql-reference/aggregate-functions/reference/studentttest.md +++ b/docs/ru/sql-reference/aggregate-functions/reference/studentttest.md @@ -63,4 +63,3 @@ SELECT studentTTest(sample_data, sample_index) FROM student_ttest; - [t-критерий Стьюдента](https://ru.wikipedia.org/wiki/T-%D0%BA%D1%80%D0%B8%D1%82%D0%B5%D1%80%D0%B8%D0%B9_%D0%A1%D1%82%D1%8C%D1%8E%D0%B4%D0%B5%D0%BD%D1%82%D0%B0) - [welchTTest](welchttest.md#welchttest) -[Оригинальная статья](https://clickhouse.tech/docs/ru/sql-reference/aggregate-functions/reference/studentttest/) diff --git a/docs/ru/sql-reference/aggregate-functions/reference/sum.md b/docs/ru/sql-reference/aggregate-functions/reference/sum.md index 5fa769f3479..487313c006b 100644 --- a/docs/ru/sql-reference/aggregate-functions/reference/sum.md +++ b/docs/ru/sql-reference/aggregate-functions/reference/sum.md @@ -7,4 +7,3 @@ toc_priority: 4 Вычисляет сумму. Работает только для чисел. -[Оригинальная статья](https://clickhouse.tech/docs/en/sql-reference/aggregate-functions/reference/sum/) diff --git a/docs/ru/sql-reference/aggregate-functions/reference/summap.md b/docs/ru/sql-reference/aggregate-functions/reference/summap.md index 460fc078893..3cfe4c26fcc 100644 --- a/docs/ru/sql-reference/aggregate-functions/reference/summap.md +++ b/docs/ru/sql-reference/aggregate-functions/reference/summap.md @@ -42,4 +42,3 @@ GROUP BY timeslot └─────────────────────┴──────────────────────────────────────────────┴────────────────────────────────┘ ``` -[Оригинальная статья](https://clickhouse.tech/docs/en/sql-reference/aggregate-functions/reference/summap/) diff --git a/docs/ru/sql-reference/aggregate-functions/reference/sumwithoverflow.md b/docs/ru/sql-reference/aggregate-functions/reference/sumwithoverflow.md index 845adc510f2..1e1962babbe 100644 --- a/docs/ru/sql-reference/aggregate-functions/reference/sumwithoverflow.md +++ b/docs/ru/sql-reference/aggregate-functions/reference/sumwithoverflow.md @@ -8,4 +8,3 @@ toc_priority: 140 Работает только для чисел. -[Оригинальная статья](https://clickhouse.tech/docs/en/sql-reference/aggregate-functions/reference/sumwithoverflow/) diff --git a/docs/ru/sql-reference/aggregate-functions/reference/topk.md b/docs/ru/sql-reference/aggregate-functions/reference/topk.md index 6aefd38bf34..929b49c35e9 100644 --- a/docs/ru/sql-reference/aggregate-functions/reference/topk.md +++ b/docs/ru/sql-reference/aggregate-functions/reference/topk.md @@ -36,4 +36,3 @@ FROM ontime └─────────────────────┘ ``` -[Оригинальная статья](https://clickhouse.tech/docs/en/sql-reference/aggregate-functions/reference/topk/) diff --git a/docs/ru/sql-reference/aggregate-functions/reference/topkweighted.md b/docs/ru/sql-reference/aggregate-functions/reference/topkweighted.md index 20bd3ee85ff..e8a41e7f05f 100644 --- a/docs/ru/sql-reference/aggregate-functions/reference/topkweighted.md +++ b/docs/ru/sql-reference/aggregate-functions/reference/topkweighted.md @@ -41,4 +41,3 @@ SELECT topKWeighted(10)(number, number) FROM numbers(1000) └───────────────────────────────────────────┘ ``` -[Оригинальная статья](https://clickhouse.tech/docs/en/sql-reference/aggregate-functions/reference/topkweighted/) diff --git a/docs/ru/sql-reference/aggregate-functions/reference/uniq.md b/docs/ru/sql-reference/aggregate-functions/reference/uniq.md index f5f3f198139..cb92545a7fe 100644 --- a/docs/ru/sql-reference/aggregate-functions/reference/uniq.md +++ b/docs/ru/sql-reference/aggregate-functions/reference/uniq.md @@ -39,4 +39,3 @@ uniq(x[, ...]) - [uniqHLL12](../../../sql-reference/aggregate-functions/reference/uniqhll12.md#agg_function-uniqhll12) - [uniqExact](../../../sql-reference/aggregate-functions/reference/uniqexact.md#agg_function-uniqexact) -[Оригинальная статья](https://clickhouse.tech/docs/en/sql-reference/aggregate-functions/reference/uniq/) diff --git a/docs/ru/sql-reference/aggregate-functions/reference/uniqcombined.md b/docs/ru/sql-reference/aggregate-functions/reference/uniqcombined.md index 751dc1a8c98..44783b8aca4 100644 --- a/docs/ru/sql-reference/aggregate-functions/reference/uniqcombined.md +++ b/docs/ru/sql-reference/aggregate-functions/reference/uniqcombined.md @@ -50,4 +50,3 @@ uniqCombined(HLL_precision)(x[, ...]) - [uniqHLL12](../../../sql-reference/aggregate-functions/reference/uniqhll12.md#agg_function-uniqhll12) - [uniqExact](../../../sql-reference/aggregate-functions/reference/uniqexact.md#agg_function-uniqexact) -[Оригинальная статья](https://clickhouse.tech/docs/en/sql-reference/aggregate-functions/reference/uniqcombined/) diff --git a/docs/ru/sql-reference/aggregate-functions/reference/uniqcombined64.md b/docs/ru/sql-reference/aggregate-functions/reference/uniqcombined64.md index 5db27fb301d..6fde16b4b0c 100644 --- a/docs/ru/sql-reference/aggregate-functions/reference/uniqcombined64.md +++ b/docs/ru/sql-reference/aggregate-functions/reference/uniqcombined64.md @@ -6,4 +6,3 @@ toc_priority: 193 Использует 64-битный хэш для всех типов, в отличие от [uniqCombined](../../../sql-reference/aggregate-functions/reference/uniqcombined.md#agg_function-uniqcombined). -[Оригинальная статья](https://clickhouse.tech/docs/en/sql-reference/aggregate-functions/reference/uniqcombined64/) diff --git a/docs/ru/sql-reference/aggregate-functions/reference/uniqexact.md b/docs/ru/sql-reference/aggregate-functions/reference/uniqexact.md index 3dd22b2b4bc..3e8ef4480be 100644 --- a/docs/ru/sql-reference/aggregate-functions/reference/uniqexact.md +++ b/docs/ru/sql-reference/aggregate-functions/reference/uniqexact.md @@ -24,4 +24,3 @@ uniqExact(x[, ...]) - [uniqCombined](../../../sql-reference/aggregate-functions/reference/uniq.md#agg_function-uniqcombined) - [uniqHLL12](../../../sql-reference/aggregate-functions/reference/uniq.md#agg_function-uniqhll12) -[Оригинальная статья](https://clickhouse.tech/docs/en/sql-reference/aggregate-functions/reference/uniqexact/) diff --git a/docs/ru/sql-reference/aggregate-functions/reference/uniqhll12.md b/docs/ru/sql-reference/aggregate-functions/reference/uniqhll12.md index 09e52ac6833..b220121e85f 100644 --- a/docs/ru/sql-reference/aggregate-functions/reference/uniqhll12.md +++ b/docs/ru/sql-reference/aggregate-functions/reference/uniqhll12.md @@ -38,4 +38,3 @@ uniqHLL12(x[, ...]) - [uniqExact](../../../sql-reference/aggregate-functions/reference/uniqexact.md#agg_function-uniqexact) -[Оригинальная статья](https://clickhouse.tech/docs/en/sql-reference/aggregate-functions/reference/uniqhll12/) diff --git a/docs/ru/sql-reference/aggregate-functions/reference/varpop.md b/docs/ru/sql-reference/aggregate-functions/reference/varpop.md index 9615e03673b..0a78b3cbb76 100644 --- a/docs/ru/sql-reference/aggregate-functions/reference/varpop.md +++ b/docs/ru/sql-reference/aggregate-functions/reference/varpop.md @@ -11,4 +11,3 @@ toc_priority: 32 !!! note "Примечание" Функция использует вычислительно неустойчивый алгоритм. Если для ваших расчётов необходима [вычислительная устойчивость](https://ru.wikipedia.org/wiki/Вычислительная_устойчивость), используйте функцию `varPopStable`. Она работает медленнее, но обеспечивает меньшую вычислительную ошибку. -[Оригинальная статья](https://clickhouse.tech/docs/en/sql-reference/aggregate-functions/reference/varpop/) diff --git a/docs/ru/sql-reference/aggregate-functions/reference/varsamp.md b/docs/ru/sql-reference/aggregate-functions/reference/varsamp.md index 31aaac68e7b..e18b858b7e2 100644 --- a/docs/ru/sql-reference/aggregate-functions/reference/varsamp.md +++ b/docs/ru/sql-reference/aggregate-functions/reference/varsamp.md @@ -13,4 +13,3 @@ toc_priority: 33 !!! note "Примечание" Функция использует вычислительно неустойчивый алгоритм. Если для ваших расчётов необходима [вычислительная устойчивость](https://ru.wikipedia.org/wiki/Вычислительная_устойчивость), используйте функцию `varSampStable`. Она работает медленнее, но обеспечиват меньшую вычислительную ошибку. -[Оригинальная статья](https://clickhouse.tech/docs/en/sql-reference/aggregate-functions/reference/vasamp/) diff --git a/docs/ru/sql-reference/aggregate-functions/reference/welchttest.md b/docs/ru/sql-reference/aggregate-functions/reference/welchttest.md index 16c122d1b49..a37d08fc671 100644 --- a/docs/ru/sql-reference/aggregate-functions/reference/welchttest.md +++ b/docs/ru/sql-reference/aggregate-functions/reference/welchttest.md @@ -63,4 +63,3 @@ SELECT welchTTest(sample_data, sample_index) FROM welch_ttest; - [t-критерий Уэлча](https://ru.wikipedia.org/wiki/T-%D0%BA%D1%80%D0%B8%D1%82%D0%B5%D1%80%D0%B8%D0%B9_%D0%A3%D1%8D%D0%BB%D1%87%D0%B0) - [studentTTest](studentttest.md#studentttest) -[Оригинальная статья](https://clickhouse.tech/docs/ru/sql-reference/aggregate-functions/reference/welchTTest/) diff --git a/docs/ru/sql-reference/data-types/aggregatefunction.md b/docs/ru/sql-reference/data-types/aggregatefunction.md index 018d38d825e..6ca6879cf6c 100644 --- a/docs/ru/sql-reference/data-types/aggregatefunction.md +++ b/docs/ru/sql-reference/data-types/aggregatefunction.md @@ -65,4 +65,3 @@ SELECT uniqMerge(state) FROM (SELECT uniqState(UserID) AS state FROM table GROUP Смотрите в описании движка [AggregatingMergeTree](../../sql-reference/data-types/aggregatefunction.md). -[Оригинальная статья](https://clickhouse.tech/docs/ru/data_types/nested_data_structures/aggregatefunction/) diff --git a/docs/ru/sql-reference/data-types/array.md b/docs/ru/sql-reference/data-types/array.md index 86a23ed041b..30952d6e126 100644 --- a/docs/ru/sql-reference/data-types/array.md +++ b/docs/ru/sql-reference/data-types/array.md @@ -76,4 +76,3 @@ Received exception from server (version 1.1.54388): Code: 386. DB::Exception: Received from localhost:9000, 127.0.0.1. DB::Exception: There is no supertype for types UInt8, String because some of them are String/FixedString and some of them are not. ``` -[Оригинальная статья](https://clickhouse.tech/docs/ru/data_types/array/) diff --git a/docs/ru/sql-reference/data-types/boolean.md b/docs/ru/sql-reference/data-types/boolean.md index b0fad6d7446..dff35777ff9 100644 --- a/docs/ru/sql-reference/data-types/boolean.md +++ b/docs/ru/sql-reference/data-types/boolean.md @@ -7,4 +7,3 @@ toc_title: "Булевы значения" Отдельного типа для булевых значений нет. Для них используется тип UInt8, в котором используются только значения 0 и 1. -[Оригинальная статья](https://clickhouse.tech/docs/ru/data_types/boolean/) diff --git a/docs/ru/sql-reference/data-types/date.md b/docs/ru/sql-reference/data-types/date.md index 490bc5c28b4..50508de96a3 100644 --- a/docs/ru/sql-reference/data-types/date.md +++ b/docs/ru/sql-reference/data-types/date.md @@ -44,4 +44,3 @@ SELECT * FROM dt; - [Тип данных `DateTime`](../../sql-reference/data-types/datetime.md) -[Оригинальная статья](https://clickhouse.tech/docs/ru/data_types/date/) diff --git a/docs/ru/sql-reference/data-types/datetime.md b/docs/ru/sql-reference/data-types/datetime.md index ffdf83e5bd0..ebd780d0d7d 100644 --- a/docs/ru/sql-reference/data-types/datetime.md +++ b/docs/ru/sql-reference/data-types/datetime.md @@ -126,4 +126,3 @@ FROM dt - [Тип данных `Date`](date.md) - [Тип данных `DateTime64`](datetime64.md) -[Оригинальная статья](https://clickhouse.tech/docs/ru/data_types/datetime/) diff --git a/docs/ru/sql-reference/data-types/decimal.md b/docs/ru/sql-reference/data-types/decimal.md index bdcd3c767b9..8524e8ea132 100644 --- a/docs/ru/sql-reference/data-types/decimal.md +++ b/docs/ru/sql-reference/data-types/decimal.md @@ -112,4 +112,3 @@ DB::Exception: Can't compare. - [countDigits](../../sql-reference/functions/other-functions.md#count-digits) -[Оригинальная статья](https://clickhouse.tech/docs/ru/data_types/decimal/) diff --git a/docs/ru/sql-reference/data-types/domains/index.md b/docs/ru/sql-reference/data-types/domains/index.md index 6a968a76ff6..35f8149112f 100644 --- a/docs/ru/sql-reference/data-types/domains/index.md +++ b/docs/ru/sql-reference/data-types/domains/index.md @@ -30,4 +30,3 @@ toc_priority: 56 - Невозможно неявно преобразовывать строковые значение в значения с доменным типом данных при вставке данных из другого столбца или таблицы. - Домен не добавляет ограничения на хранимые значения. -[Оригинальная статья](https://clickhouse.tech/docs/ru/data_types/domains/overview) diff --git a/docs/ru/sql-reference/data-types/domains/ipv4.md b/docs/ru/sql-reference/data-types/domains/ipv4.md index 57d6f12ab17..af5f8261fae 100644 --- a/docs/ru/sql-reference/data-types/domains/ipv4.md +++ b/docs/ru/sql-reference/data-types/domains/ipv4.md @@ -81,4 +81,3 @@ SELECT toTypeName(i), CAST(from AS UInt32) AS i FROM hits LIMIT 1; └──────────────────────────────────┴────────────┘ ``` -[Оригинальная статья](https://clickhouse.tech/docs/ru/data_types/domains/ipv4) diff --git a/docs/ru/sql-reference/data-types/domains/ipv6.md b/docs/ru/sql-reference/data-types/domains/ipv6.md index 04c5fd0d491..5b3c17feceb 100644 --- a/docs/ru/sql-reference/data-types/domains/ipv6.md +++ b/docs/ru/sql-reference/data-types/domains/ipv6.md @@ -81,4 +81,3 @@ SELECT toTypeName(i), CAST(from AS FixedString(16)) AS i FROM hits LIMIT 1; └───────────────────────────────────────────┴─────────┘ ``` -[Оригинальная статья](https://clickhouse.tech/docs/ru/data_types/domains/ipv6) diff --git a/docs/ru/sql-reference/data-types/enum.md b/docs/ru/sql-reference/data-types/enum.md index b86d15c19a8..95c053bed2c 100644 --- a/docs/ru/sql-reference/data-types/enum.md +++ b/docs/ru/sql-reference/data-types/enum.md @@ -126,4 +126,3 @@ INSERT INTO t_enum_nullable Values('hello'),('world'),(NULL) При ALTER, есть возможность поменять Enum8 на Enum16 и обратно - так же, как можно поменять Int8 на Int16. -[Оригинальная статья](https://clickhouse.tech/docs/ru/data_types/enum/) diff --git a/docs/ru/sql-reference/data-types/fixedstring.md b/docs/ru/sql-reference/data-types/fixedstring.md index 21115418e30..ef73dadaddf 100644 --- a/docs/ru/sql-reference/data-types/fixedstring.md +++ b/docs/ru/sql-reference/data-types/fixedstring.md @@ -58,4 +58,3 @@ WHERE a = 'b\0' Обратите внимание, что длина значения `FixedString(N)` постоянна. Функция [length](../../sql-reference/data-types/fixedstring.md#array_functions-length) возвращает `N` даже если значение `FixedString(N)` заполнено только нулевыми байтами, однако функция [empty](../../sql-reference/data-types/fixedstring.md#empty) в этом же случае возвращает `1`. -[Оригинальная статья](https://clickhouse.tech/docs/ru/data_types/fixedstring/) diff --git a/docs/ru/sql-reference/data-types/float.md b/docs/ru/sql-reference/data-types/float.md index 0e861f170b7..89ac00ab62f 100644 --- a/docs/ru/sql-reference/data-types/float.md +++ b/docs/ru/sql-reference/data-types/float.md @@ -89,4 +89,3 @@ SELECT 0 / 0 Смотрите правила сортировки `NaN` в разделе [Секция ORDER BY ](../../sql-reference/statements/select/order-by.md). -[Оригинальная статья](https://clickhouse.tech/docs/ru/data_types/float/) diff --git a/docs/ru/sql-reference/data-types/geo.md b/docs/ru/sql-reference/data-types/geo.md index 23293b30927..23b47f38d05 100644 --- a/docs/ru/sql-reference/data-types/geo.md +++ b/docs/ru/sql-reference/data-types/geo.md @@ -103,4 +103,3 @@ Result: └─────────────────────────────────────────────────────────────────────────────────────────────────┴─────────────────┘ ``` -[Оригинальная статья](https://clickhouse.tech/docs/ru/data-types/geo/) diff --git a/docs/ru/sql-reference/data-types/index.md b/docs/ru/sql-reference/data-types/index.md index 53c983a147a..2b29ee1bc19 100644 --- a/docs/ru/sql-reference/data-types/index.md +++ b/docs/ru/sql-reference/data-types/index.md @@ -11,4 +11,3 @@ ClickHouse может сохранять в ячейках таблиц данн Зависимость имен типов данных от регистра можно проверить в системной таблице [system.data_type_families](../../operations/system-tables/data_type_families.md#system_tables-data_type_families). Раздел содержит описания поддерживаемых типов данных и специфику их использования и/или реализации, если таковые имеются. -[Оригинальная статья](https://clickhouse.tech/docs/ru/data_types/) diff --git a/docs/ru/sql-reference/data-types/int-uint.md b/docs/ru/sql-reference/data-types/int-uint.md index d3c342e467a..c026f5fc4a5 100644 --- a/docs/ru/sql-reference/data-types/int-uint.md +++ b/docs/ru/sql-reference/data-types/int-uint.md @@ -35,4 +35,3 @@ toc_title: UInt8, UInt16, UInt32, UInt64, Int8, Int16, Int32, Int64 `UInt128` пока не реализован. -[Оригинальная статья](https://clickhouse.tech/docs/ru/data_types/int_uint/) diff --git a/docs/ru/sql-reference/data-types/lowcardinality.md b/docs/ru/sql-reference/data-types/lowcardinality.md index 52713e2d747..fe9118b1e14 100644 --- a/docs/ru/sql-reference/data-types/lowcardinality.md +++ b/docs/ru/sql-reference/data-types/lowcardinality.md @@ -58,4 +58,3 @@ ORDER BY id - [Reducing Clickhouse Storage Cost with the Low Cardinality Type – Lessons from an Instana Engineer](https://www.instana.com/blog/reducing-clickhouse-storage-cost-with-the-low-cardinality-type-lessons-from-an-instana-engineer/). - [String Optimization (video presentation in Russian)](https://youtu.be/rqf-ILRgBdY?list=PL0Z2YDlm0b3iwXCpEFiOOYmwXzVmjJfEt). [Slides in English](https://github.com/yandex/clickhouse-presentations/raw/master/meetup19/string_optimization.pdf). -[Оригинальная статья](https://clickhouse.tech/docs/ru/sql-reference/data-types/lowcardinality/) diff --git a/docs/ru/sql-reference/data-types/multiword-types.md b/docs/ru/sql-reference/data-types/multiword-types.md index 559755ef989..0a8afff448d 100644 --- a/docs/ru/sql-reference/data-types/multiword-types.md +++ b/docs/ru/sql-reference/data-types/multiword-types.md @@ -26,4 +26,3 @@ toc_title: Составные типы | BINARY LARGE OBJECT | [String](../../sql-reference/data-types/string.md) | | BINARY VARYING | [String](../../sql-reference/data-types/string.md) | -[Оригинальная статья](https://clickhouse.tech/docs/ru/sql-reference/data-types/multiword-types/) diff --git a/docs/ru/sql-reference/data-types/nested-data-structures/index.md b/docs/ru/sql-reference/data-types/nested-data-structures/index.md index db214b90c03..78262347bac 100644 --- a/docs/ru/sql-reference/data-types/nested-data-structures/index.md +++ b/docs/ru/sql-reference/data-types/nested-data-structures/index.md @@ -7,4 +7,3 @@ toc_title: hidden # Вложенные структуры данных {#vlozhennye-struktury-dannykh} -[Оригинальная статья](https://clickhouse.tech/docs/ru/data_types/nested_data_structures/) diff --git a/docs/ru/sql-reference/data-types/nested-data-structures/nested.md b/docs/ru/sql-reference/data-types/nested-data-structures/nested.md index 0e43383b283..199d141a191 100644 --- a/docs/ru/sql-reference/data-types/nested-data-structures/nested.md +++ b/docs/ru/sql-reference/data-types/nested-data-structures/nested.md @@ -96,4 +96,3 @@ LIMIT 10 Работоспособность запроса ALTER для элементов вложенных структур данных, является сильно ограниченной. -[Оригинальная статья](https://clickhouse.tech/docs/ru/data_types/nested_data_structures/nested/) diff --git a/docs/ru/sql-reference/data-types/nullable.md b/docs/ru/sql-reference/data-types/nullable.md index 71e1f7a37a0..3f33c4b2540 100644 --- a/docs/ru/sql-reference/data-types/nullable.md +++ b/docs/ru/sql-reference/data-types/nullable.md @@ -48,4 +48,3 @@ SELECT x + y from t_null └────────────┘ ``` -[Оригинальная статья](https://clickhouse.tech/docs/ru/data_types/nullable/) diff --git a/docs/ru/sql-reference/data-types/simpleaggregatefunction.md b/docs/ru/sql-reference/data-types/simpleaggregatefunction.md index 668b579ff78..454add05e8a 100644 --- a/docs/ru/sql-reference/data-types/simpleaggregatefunction.md +++ b/docs/ru/sql-reference/data-types/simpleaggregatefunction.md @@ -36,4 +36,3 @@ CREATE TABLE simple (id UInt64, val SimpleAggregateFunction(sum, Double)) ENGINE=AggregatingMergeTree ORDER BY id; ``` -[Оригинальная статья](https://clickhouse.tech/docs/en/sql-reference/data-types/simpleaggregatefunction/) diff --git a/docs/ru/sql-reference/data-types/special-data-types/expression.md b/docs/ru/sql-reference/data-types/special-data-types/expression.md index 718fcc886a6..f11f66a40c7 100644 --- a/docs/ru/sql-reference/data-types/special-data-types/expression.md +++ b/docs/ru/sql-reference/data-types/special-data-types/expression.md @@ -7,4 +7,3 @@ toc_title: Expression Используется для представления лямбда-выражений в функциях высшего порядка. -[Оригинальная статья](https://clickhouse.tech/docs/ru/data_types/special_data_types/expression/) diff --git a/docs/ru/sql-reference/data-types/special-data-types/index.md b/docs/ru/sql-reference/data-types/special-data-types/index.md index e6d9fa8b011..823a84e2e43 100644 --- a/docs/ru/sql-reference/data-types/special-data-types/index.md +++ b/docs/ru/sql-reference/data-types/special-data-types/index.md @@ -9,4 +9,3 @@ toc_title: hidden Значения служебных типов данных не могут сохраняться в таблицу и выводиться в качестве результата, а возникают как промежуточный результат выполнения запроса. -[Оригинальная статья](https://clickhouse.tech/docs/ru/data_types/special_data_types/) diff --git a/docs/ru/sql-reference/data-types/special-data-types/nothing.md b/docs/ru/sql-reference/data-types/special-data-types/nothing.md index c6a9cb868d8..30d425461e1 100644 --- a/docs/ru/sql-reference/data-types/special-data-types/nothing.md +++ b/docs/ru/sql-reference/data-types/special-data-types/nothing.md @@ -19,4 +19,3 @@ SELECT toTypeName(Array()) └─────────────────────┘ ``` -[Оригинальная статья](https://clickhouse.tech/docs/ru/data_types/special_data_types/nothing/) diff --git a/docs/ru/sql-reference/data-types/special-data-types/set.md b/docs/ru/sql-reference/data-types/special-data-types/set.md index 4c2f4ed2c66..5867df3c947 100644 --- a/docs/ru/sql-reference/data-types/special-data-types/set.md +++ b/docs/ru/sql-reference/data-types/special-data-types/set.md @@ -7,4 +7,3 @@ toc_title: Set Используется для представления правой части выражения IN. -[Оригинальная статья](https://clickhouse.tech/docs/ru/data_types/special_data_types/set/) diff --git a/docs/ru/sql-reference/data-types/string.md b/docs/ru/sql-reference/data-types/string.md index 6a07f7e51de..9470f523629 100644 --- a/docs/ru/sql-reference/data-types/string.md +++ b/docs/ru/sql-reference/data-types/string.md @@ -17,4 +17,3 @@ toc_title: String Также, некоторые функции по работе со строками, имеют отдельные варианты, которые работают при допущении, что строка содержит набор байт, представляющий текст в кодировке UTF-8. Например, функция length вычисляет длину строки в байтах, а функция lengthUTF8 - длину строки в кодовых точках Unicode, при допущении, что значение в кодировке UTF-8. -[Оригинальная статья](https://clickhouse.tech/docs/ru/data_types/string/) diff --git a/docs/ru/sql-reference/data-types/tuple.md b/docs/ru/sql-reference/data-types/tuple.md index e2a1450b47f..702b5962f7b 100644 --- a/docs/ru/sql-reference/data-types/tuple.md +++ b/docs/ru/sql-reference/data-types/tuple.md @@ -47,4 +47,3 @@ SELECT tuple(1,NULL) AS x, toTypeName(x) └──────────┴─────────────────────────────────┘ ``` -[Оригинальная статья](https://clickhouse.tech/docs/ru/data_types/tuple/) diff --git a/docs/ru/sql-reference/dictionaries/external-dictionaries/external-dicts-dict-hierarchical.md b/docs/ru/sql-reference/dictionaries/external-dictionaries/external-dicts-dict-hierarchical.md index 9c0b731bc7d..da8492e7cc0 100644 --- a/docs/ru/sql-reference/dictionaries/external-dictionaries/external-dicts-dict-hierarchical.md +++ b/docs/ru/sql-reference/dictionaries/external-dictionaries/external-dicts-dict-hierarchical.md @@ -65,4 +65,3 @@ ClickHouse поддерживает свойство [hierarchical](external-dic ``` -[Оригинальная статья](https://clickhouse.tech/docs/ru/query_language/dicts/external_dicts_dict_hierarchical/) diff --git a/docs/ru/sql-reference/dictionaries/external-dictionaries/external-dicts-dict-layout.md b/docs/ru/sql-reference/dictionaries/external-dictionaries/external-dicts-dict-layout.md index 0fd4a85c46f..1d1e46250e2 100644 --- a/docs/ru/sql-reference/dictionaries/external-dictionaries/external-dicts-dict-layout.md +++ b/docs/ru/sql-reference/dictionaries/external-dictionaries/external-dicts-dict-layout.md @@ -443,4 +443,3 @@ dictGetString('prefix', 'asn', tuple(IPv6StringToNum('2001:db8::1'))) Данные должны полностью помещаться в оперативной памяти. -[Оригинальная статья](https://clickhouse.tech/docs/ru/query_language/dicts/external_dicts_dict_layout/) diff --git a/docs/ru/sql-reference/dictionaries/external-dictionaries/external-dicts-dict-lifetime.md b/docs/ru/sql-reference/dictionaries/external-dictionaries/external-dicts-dict-lifetime.md index 48d891b2042..9589353649d 100644 --- a/docs/ru/sql-reference/dictionaries/external-dictionaries/external-dicts-dict-lifetime.md +++ b/docs/ru/sql-reference/dictionaries/external-dictionaries/external-dicts-dict-lifetime.md @@ -86,4 +86,3 @@ SOURCE(ODBC(... invalidate_query 'SELECT update_time FROM dictionary_source wher ... ``` -[Оригинальная статья](https://clickhouse.tech/docs/ru/query_language/dicts/external_dicts_dict_lifetime/) diff --git a/docs/ru/sql-reference/dictionaries/external-dictionaries/external-dicts-dict-sources.md b/docs/ru/sql-reference/dictionaries/external-dictionaries/external-dicts-dict-sources.md index 9735e91c8b6..e3816e78547 100644 --- a/docs/ru/sql-reference/dictionaries/external-dictionaries/external-dicts-dict-sources.md +++ b/docs/ru/sql-reference/dictionaries/external-dictionaries/external-dicts-dict-sources.md @@ -725,4 +725,3 @@ Setting fields: - `invalidate_query` – Запрос для проверки условия загрузки словаря. Необязательный параметр. Читайте больше в разделе [Обновление словарей](../../../sql-reference/dictionaries/external-dictionaries/external-dicts-dict-lifetime.md). -[Оригинальная статья](https://clickhouse.tech/docs/ru/query_language/dicts/external_dicts_dict_sources/) diff --git a/docs/ru/sql-reference/dictionaries/external-dictionaries/external-dicts-dict-structure.md b/docs/ru/sql-reference/dictionaries/external-dictionaries/external-dicts-dict-structure.md index 6efbe706110..57f53390d1c 100644 --- a/docs/ru/sql-reference/dictionaries/external-dictionaries/external-dicts-dict-structure.md +++ b/docs/ru/sql-reference/dictionaries/external-dictionaries/external-dicts-dict-structure.md @@ -169,4 +169,3 @@ CREATE DICTIONARY somename ( - [Функции для работы с внешними словарями](../../../sql-reference/functions/ext-dict-functions.md). -[Оригинальная статья](https://clickhouse.tech/docs/ru/query_language/dicts/external_dicts_dict_structure/) diff --git a/docs/ru/sql-reference/dictionaries/external-dictionaries/external-dicts-dict.md b/docs/ru/sql-reference/dictionaries/external-dictionaries/external-dicts-dict.md index 7e35f59609d..4dc74200093 100644 --- a/docs/ru/sql-reference/dictionaries/external-dictionaries/external-dicts-dict.md +++ b/docs/ru/sql-reference/dictionaries/external-dictionaries/external-dicts-dict.md @@ -48,4 +48,3 @@ LIFETIME(...) -- Lifetime of dictionary in memory - [structure](external-dicts-dict-structure.md) — Структура словаря. Ключ и атрибуты, которые можно получить по ключу. - [lifetime](external-dicts-dict-lifetime.md) — Периодичность обновления словарей. -[Оригинальная статья](https://clickhouse.tech/docs/ru/query_language/dicts/external_dicts_dict/) diff --git a/docs/ru/sql-reference/dictionaries/external-dictionaries/external-dicts.md b/docs/ru/sql-reference/dictionaries/external-dictionaries/external-dicts.md index 6467b5f82e4..04ef24b68c5 100644 --- a/docs/ru/sql-reference/dictionaries/external-dictionaries/external-dicts.md +++ b/docs/ru/sql-reference/dictionaries/external-dictionaries/external-dicts.md @@ -61,4 +61,3 @@ ClickHouse: - [Ключ и поля словаря](external-dicts-dict-structure.md) - [Функции для работы с внешними словарями](../../../sql-reference/functions/ext-dict-functions.md) -[Оригинальная статья](https://clickhouse.tech/docs/ru/query_language/dicts/external_dicts/) diff --git a/docs/ru/sql-reference/dictionaries/index.md b/docs/ru/sql-reference/dictionaries/index.md index 238aa244967..bd432497be8 100644 --- a/docs/ru/sql-reference/dictionaries/index.md +++ b/docs/ru/sql-reference/dictionaries/index.md @@ -17,4 +17,3 @@ ClickHouse поддерживает: - [Встроенные словари](internal-dicts.md#internal_dicts) со специфическим [набором функций](../../sql-reference/dictionaries/external-dictionaries/index.md). - [Подключаемые (внешние) словари](external-dictionaries/external-dicts.md#dicts-external-dicts) с [набором функций](../../sql-reference/dictionaries/external-dictionaries/index.md). -[Оригинальная статья](https://clickhouse.tech/docs/ru/query_language/dicts/) diff --git a/docs/ru/sql-reference/dictionaries/internal-dicts.md b/docs/ru/sql-reference/dictionaries/internal-dicts.md index af7f13f7133..34e407ceacd 100644 --- a/docs/ru/sql-reference/dictionaries/internal-dicts.md +++ b/docs/ru/sql-reference/dictionaries/internal-dicts.md @@ -50,4 +50,3 @@ ClickHouse содержит встроенную возможность рабо Также имеются функции для работы с идентификаторами операционных систем и поисковых систем Яндекс.Метрики, пользоваться которыми не нужно. -[Оригинальная статья](https://clickhouse.tech/docs/ru/query_language/dicts/internal_dicts/) diff --git a/docs/ru/sql-reference/distributed-ddl.md b/docs/ru/sql-reference/distributed-ddl.md index 17c38cfe820..e03ecb893bc 100644 --- a/docs/ru/sql-reference/distributed-ddl.md +++ b/docs/ru/sql-reference/distributed-ddl.md @@ -15,5 +15,4 @@ CREATE TABLE IF NOT EXISTS all_hits ON CLUSTER cluster (p Date, i Int32) ENGINE Для корректного выполнения таких запросов необходимо на каждом хосте иметь одинаковое определение кластера (для упрощения синхронизации конфигов можете использовать подстановки из ZooKeeper). Также необходимо подключение к ZooKeeper серверам. Локальная версия запроса в конечном итоге будет выполнена на каждом хосте кластера, даже если некоторые хосты в данный момент не доступны. Гарантируется упорядоченность выполнения запросов в рамках одного хоста. -[Оригинальная статья](https://clickhouse.tech/docs/ru/sql-reference/distributed-ddl) \ No newline at end of file diff --git a/docs/ru/sql-reference/functions/arithmetic-functions.md b/docs/ru/sql-reference/functions/arithmetic-functions.md index 779e0a9fe4a..f587b7b5b5d 100644 --- a/docs/ru/sql-reference/functions/arithmetic-functions.md +++ b/docs/ru/sql-reference/functions/arithmetic-functions.md @@ -83,4 +83,3 @@ SELECT toTypeName(0), toTypeName(0 + 0), toTypeName(0 + 0 + 0), toTypeName(0 + 0 Вычисляет наименьшее общее кратное чисел. При делении на ноль или при делении минимального отрицательного числа на минус единицу, кидается исключение. -[Оригинальная статья](https://clickhouse.tech/docs/ru/query_language/functions/arithmetic_functions/) diff --git a/docs/ru/sql-reference/functions/array-functions.md b/docs/ru/sql-reference/functions/array-functions.md index dca645888a9..eb62fdd70cb 100644 --- a/docs/ru/sql-reference/functions/array-functions.md +++ b/docs/ru/sql-reference/functions/array-functions.md @@ -1455,4 +1455,3 @@ select arrayAUC([0.1, 0.4, 0.35, 0.8], [0, 0, 1, 1]) └────────────────────────────────────────---──┘ ``` -[Оригинальная статья](https://clickhouse.tech/docs/ru/query_language/functions/array_functions/) diff --git a/docs/ru/sql-reference/functions/array-join.md b/docs/ru/sql-reference/functions/array-join.md index ed67d30062b..3e3cf5c4011 100644 --- a/docs/ru/sql-reference/functions/array-join.md +++ b/docs/ru/sql-reference/functions/array-join.md @@ -32,4 +32,3 @@ SELECT arrayJoin([1, 2, 3] AS src) AS dst, 'Hello', src └─────┴───────────┴─────────┘ ``` -[Оригинальная статья](https://clickhouse.tech/docs/ru/query_language/functions/array_join/) diff --git a/docs/ru/sql-reference/functions/bit-functions.md b/docs/ru/sql-reference/functions/bit-functions.md index 79ea05f4bd7..da34c7f3aba 100644 --- a/docs/ru/sql-reference/functions/bit-functions.md +++ b/docs/ru/sql-reference/functions/bit-functions.md @@ -240,4 +240,3 @@ SELECT bitCount(333) └───────────────┘ ``` -[Оригинальная статья](https://clickhouse.tech/docs/ru/query_language/functions/bit_functions/) diff --git a/docs/ru/sql-reference/functions/bitmap-functions.md b/docs/ru/sql-reference/functions/bitmap-functions.md index cd0ddee01a6..af28be67116 100644 --- a/docs/ru/sql-reference/functions/bitmap-functions.md +++ b/docs/ru/sql-reference/functions/bitmap-functions.md @@ -397,4 +397,3 @@ SELECT bitmapAndnotCardinality(bitmapBuild([1,2,3]),bitmapBuild([3,4,5])) AS res └─────┘ ``` -[Оригинальная статья](https://clickhouse.tech/docs/ru/query_language/functions/bitmap_functions/) diff --git a/docs/ru/sql-reference/functions/comparison-functions.md b/docs/ru/sql-reference/functions/comparison-functions.md index 179df5c2ed5..b7301bde275 100644 --- a/docs/ru/sql-reference/functions/comparison-functions.md +++ b/docs/ru/sql-reference/functions/comparison-functions.md @@ -34,4 +34,3 @@ toc_title: "Функции сравнения" ## greaterOrEquals, оператор `>=` {#function-greaterorequals} -[Оригинальная статья](https://clickhouse.tech/docs/ru/query_language/functions/comparison_functions/) diff --git a/docs/ru/sql-reference/functions/conditional-functions.md b/docs/ru/sql-reference/functions/conditional-functions.md index 888e9427a79..537ac9dc11e 100644 --- a/docs/ru/sql-reference/functions/conditional-functions.md +++ b/docs/ru/sql-reference/functions/conditional-functions.md @@ -111,4 +111,3 @@ SELECT if(0, plus(2, 2), plus(2, 6)) └────────────────────────────────────────────┘ ``` -[Оригинальная статья](https://clickhouse.tech/docs/ru/query_language/functions/conditional_functions/) diff --git a/docs/ru/sql-reference/functions/date-time-functions.md b/docs/ru/sql-reference/functions/date-time-functions.md index 9f3df92922f..f2cc051e9bd 100644 --- a/docs/ru/sql-reference/functions/date-time-functions.md +++ b/docs/ru/sql-reference/functions/date-time-functions.md @@ -941,4 +941,3 @@ SELECT FROM_UNIXTIME(1234334543, '%Y-%m-%d %R:%S') AS DateTime; └─────────────────────┘ ``` -[Оригинальная статья](https://clickhouse.tech/docs/ru/query_language/functions/date_time_functions/) diff --git a/docs/ru/sql-reference/functions/encoding-functions.md b/docs/ru/sql-reference/functions/encoding-functions.md index 951c6c60e38..f0f0a5a1aaf 100644 --- a/docs/ru/sql-reference/functions/encoding-functions.md +++ b/docs/ru/sql-reference/functions/encoding-functions.md @@ -172,4 +172,3 @@ If you want to convert the result to a number, you can use the ‘reverse’ and Принимает целое число. Возвращает массив чисел типа UInt64, содержащий степени двойки, в сумме дающих исходное число; числа в массиве идут по возрастанию. -[Оригинальная статья](https://clickhouse.tech/docs/ru/query_language/functions/encoding_functions/) diff --git a/docs/ru/sql-reference/functions/ext-dict-functions.md b/docs/ru/sql-reference/functions/ext-dict-functions.md index 8d018e8e9ac..0e7ee78e52c 100644 --- a/docs/ru/sql-reference/functions/ext-dict-functions.md +++ b/docs/ru/sql-reference/functions/ext-dict-functions.md @@ -198,4 +198,3 @@ dictGet[Type]OrDefault('dict_name', 'attr_name', id_expr, default_value_expr) Если значение атрибута не удалось обработать или оно не соответствует типу данных атрибута, то ClickHouse генерирует исключение. -[Оригинальная статья](https://clickhouse.tech/docs/ru/query_language/functions/ext_dict_functions/) diff --git a/docs/ru/sql-reference/functions/functions-for-nulls.md b/docs/ru/sql-reference/functions/functions-for-nulls.md index f0277a59699..34de3acf5ca 100644 --- a/docs/ru/sql-reference/functions/functions-for-nulls.md +++ b/docs/ru/sql-reference/functions/functions-for-nulls.md @@ -309,4 +309,3 @@ SELECT toTypeName(toNullable(10)) └────────────────────────────┘ ``` -[Оригинальная статья](https://clickhouse.tech/docs/ru/query_language/functions/functions_for_nulls/) diff --git a/docs/ru/sql-reference/functions/geo/coordinates.md b/docs/ru/sql-reference/functions/geo/coordinates.md index 09e2d7d01bf..2605dc7a82f 100644 --- a/docs/ru/sql-reference/functions/geo/coordinates.md +++ b/docs/ru/sql-reference/functions/geo/coordinates.md @@ -133,4 +133,3 @@ SELECT pointInPolygon((3., 3.), [(6, 0), (8, 4), (5, 8), (0, 2)]) AS res └─────┘ ``` -[Оригинальная статья](https://clickhouse.tech/docs/ru/sql-reference/functions/geo/coordinates) diff --git a/docs/ru/sql-reference/functions/geo/geohash.md b/docs/ru/sql-reference/functions/geo/geohash.md index 2dd3f83ddf1..01193eab543 100644 --- a/docs/ru/sql-reference/functions/geo/geohash.md +++ b/docs/ru/sql-reference/functions/geo/geohash.md @@ -112,4 +112,3 @@ SELECT geohashesInBox(24.48, 40.56, 24.785, 40.81, 4) AS thasos └─────────────────────────────────────────────┘ ``` -[Оригинальная статья](https://clickhouse.tech/docs/ru/sql-reference/functions/geo/geohash) diff --git a/docs/ru/sql-reference/functions/geo/h3.md b/docs/ru/sql-reference/functions/geo/h3.md index 7046833f7ec..41ff9f1a4f9 100644 --- a/docs/ru/sql-reference/functions/geo/h3.md +++ b/docs/ru/sql-reference/functions/geo/h3.md @@ -520,4 +520,3 @@ SELECT h3GetResolution(617420388352917503) as res; └─────┘ ``` -[Оригинальная статья](https://clickhouse.tech/docs/ru/sql-reference/functions/geo/h3) diff --git a/docs/ru/sql-reference/functions/geo/index.md b/docs/ru/sql-reference/functions/geo/index.md index 6b9a14e4d02..4d3bdfcd468 100644 --- a/docs/ru/sql-reference/functions/geo/index.md +++ b/docs/ru/sql-reference/functions/geo/index.md @@ -5,4 +5,3 @@ toc_title: hidden --- -[Оригинальная статья](https://clickhouse.tech/docs/ru/sql-reference/functions/geo/) diff --git a/docs/ru/sql-reference/functions/hash-functions.md b/docs/ru/sql-reference/functions/hash-functions.md index 1742abe5b56..5a6d82d2738 100644 --- a/docs/ru/sql-reference/functions/hash-functions.md +++ b/docs/ru/sql-reference/functions/hash-functions.md @@ -484,4 +484,3 @@ SELECT xxHash32('Hello, world!'); - [xxHash](http://cyan4973.github.io/xxHash/). -[Оригинальная статья](https://clickhouse.tech/docs/ru/query_language/functions/hash_functions/) diff --git a/docs/ru/sql-reference/functions/in-functions.md b/docs/ru/sql-reference/functions/in-functions.md index 7326d087610..2bdb71d5f93 100644 --- a/docs/ru/sql-reference/functions/in-functions.md +++ b/docs/ru/sql-reference/functions/in-functions.md @@ -9,4 +9,3 @@ toc_title: "Функции для реализации оператора IN" Смотрите раздел [Операторы IN](../operators/in.md#select-in-operators). -[Оригинальная статья](https://clickhouse.tech/docs/ru/query_language/functions/in_functions/) diff --git a/docs/ru/sql-reference/functions/index.md b/docs/ru/sql-reference/functions/index.md index ae3879b6c96..1eefd4d9f73 100644 --- a/docs/ru/sql-reference/functions/index.md +++ b/docs/ru/sql-reference/functions/index.md @@ -82,4 +82,3 @@ str -> str != Referer Если функция в запросе выполняется на сервере-инициаторе запроса, а вам нужно, чтобы она выполнялась на удалённых серверах, вы можете обернуть её в агрегатную функцию any или добавить в ключ в `GROUP BY`. -[Оригинальная статья](https://clickhouse.tech/docs/ru/query_language/functions/) diff --git a/docs/ru/sql-reference/functions/ip-address-functions.md b/docs/ru/sql-reference/functions/ip-address-functions.md index a2a08b1938e..501e42d78db 100644 --- a/docs/ru/sql-reference/functions/ip-address-functions.md +++ b/docs/ru/sql-reference/functions/ip-address-functions.md @@ -395,4 +395,3 @@ SELECT addr, isIPv6String(addr) FROM ( SELECT ['::', '1111::ffff', '::ffff:127.0 └──────────────────┴────────────────────┘ ``` -[Оригинальная статья](https://clickhouse.tech/docs/ru/query_language/functions/ip_address_functions/) diff --git a/docs/ru/sql-reference/functions/json-functions.md b/docs/ru/sql-reference/functions/json-functions.md index 69b8f8f98f5..704979b9dc6 100644 --- a/docs/ru/sql-reference/functions/json-functions.md +++ b/docs/ru/sql-reference/functions/json-functions.md @@ -294,4 +294,3 @@ SELECT JSONExtractKeysAndValuesRaw('{"a": [-100, 200.0], "b":{"c": {"d": "hello" └───────────────────────────────────────────────────────────────────────────────────────────────────────┘ ``` -[Оригинальная статья](https://clickhouse.tech/docs/ru/query_language/functions/json_functions/) diff --git a/docs/ru/sql-reference/functions/logical-functions.md b/docs/ru/sql-reference/functions/logical-functions.md index 2d71c60a509..8566657d2eb 100644 --- a/docs/ru/sql-reference/functions/logical-functions.md +++ b/docs/ru/sql-reference/functions/logical-functions.md @@ -17,4 +17,3 @@ toc_title: "Логические функции" ## xor {#xor} -[Оригинальная статья](https://clickhouse.tech/docs/ru/query_language/functions/logical_functions/) diff --git a/docs/ru/sql-reference/functions/math-functions.md b/docs/ru/sql-reference/functions/math-functions.md index a5ba01f6282..eb369d476a0 100644 --- a/docs/ru/sql-reference/functions/math-functions.md +++ b/docs/ru/sql-reference/functions/math-functions.md @@ -468,4 +468,3 @@ SELECT sign(-1); └──────────┘ ``` -[Оригинальная статья](https://clickhouse.tech/docs/ru/query_language/functions/math_functions/) diff --git a/docs/ru/sql-reference/functions/other-functions.md b/docs/ru/sql-reference/functions/other-functions.md index 595d2458ca9..60c867a2158 100644 --- a/docs/ru/sql-reference/functions/other-functions.md +++ b/docs/ru/sql-reference/functions/other-functions.md @@ -1854,4 +1854,3 @@ SELECT tcpPort(); - [tcp_port](../../operations/server-configuration-parameters/settings.md#server_configuration_parameters-tcp_port) -[Оригинальная статья](https://clickhouse.tech/docs/ru/query_language/functions/other_functions/) diff --git a/docs/ru/sql-reference/functions/random-functions.md b/docs/ru/sql-reference/functions/random-functions.md index a09f5159309..efe5259c44a 100644 --- a/docs/ru/sql-reference/functions/random-functions.md +++ b/docs/ru/sql-reference/functions/random-functions.md @@ -107,4 +107,3 @@ FROM numbers(3) └───────────────────────────────────────┘ ``` -[Оригинальная статья](https://clickhouse.tech/docs/ru/query_language/functions/random_functions/) diff --git a/docs/ru/sql-reference/functions/rounding-functions.md b/docs/ru/sql-reference/functions/rounding-functions.md index 704e7f5dd52..9eb59d955fc 100644 --- a/docs/ru/sql-reference/functions/rounding-functions.md +++ b/docs/ru/sql-reference/functions/rounding-functions.md @@ -177,4 +177,3 @@ roundBankers(10.755, 2) = 11,76 Принимает число. Если число меньше 18 - возвращает 0. Иначе округляет число вниз до чисел из набора: 18, 25, 35, 45, 55. Эта функция специфична для Яндекс.Метрики и предназначена для реализации отчёта по возрасту посетителей. -[Оригинальная статья](https://clickhouse.tech/docs/ru/query_language/functions/rounding_functions/) diff --git a/docs/ru/sql-reference/functions/splitting-merging-functions.md b/docs/ru/sql-reference/functions/splitting-merging-functions.md index cacce5f4ba2..f393b220e1f 100644 --- a/docs/ru/sql-reference/functions/splitting-merging-functions.md +++ b/docs/ru/sql-reference/functions/splitting-merging-functions.md @@ -115,4 +115,3 @@ SELECT alphaTokens('abca1abc') └─────────────────────────┘ ``` -[Оригинальная статья](https://clickhouse.tech/docs/ru/query_language/functions/splitting_merging_functions/) diff --git a/docs/ru/sql-reference/functions/string-functions.md b/docs/ru/sql-reference/functions/string-functions.md index 65a1cd63563..c222a7bd4f9 100644 --- a/docs/ru/sql-reference/functions/string-functions.md +++ b/docs/ru/sql-reference/functions/string-functions.md @@ -645,4 +645,3 @@ SELECT decodeXMLComponent('< Σ >'); - [Мнемоники в HTML](https://ru.wikipedia.org/wiki/%D0%9C%D0%BD%D0%B5%D0%BC%D0%BE%D0%BD%D0%B8%D0%BA%D0%B8_%D0%B2_HTML) -[Оригинальная статья](https://clickhouse.tech/docs/ru/query_language/functions/string_functions/) diff --git a/docs/ru/sql-reference/functions/string-replace-functions.md b/docs/ru/sql-reference/functions/string-replace-functions.md index f00a06d1560..9426e8685b0 100644 --- a/docs/ru/sql-reference/functions/string-replace-functions.md +++ b/docs/ru/sql-reference/functions/string-replace-functions.md @@ -83,4 +83,3 @@ SELECT replaceRegexpAll('Hello, World!', '^', 'here: ') AS res └─────────────────────┘ ``` -[Оригинальная статья](https://clickhouse.tech/docs/ru/query_language/functions/string_replace_functions/) diff --git a/docs/ru/sql-reference/functions/string-search-functions.md b/docs/ru/sql-reference/functions/string-search-functions.md index 95ac922a4a8..51ec90bbeff 100644 --- a/docs/ru/sql-reference/functions/string-search-functions.md +++ b/docs/ru/sql-reference/functions/string-search-functions.md @@ -757,4 +757,3 @@ SELECT countSubstringsCaseInsensitiveUTF8('аБв__АбВ__абв', 'Абв'); └────────────────────────────────────────────────────────────┘ ``` -[Оригинальная статья](https://clickhouse.tech/docs/ru/query_language/functions/string_search_functions/) diff --git a/docs/ru/sql-reference/functions/tuple-functions.md b/docs/ru/sql-reference/functions/tuple-functions.md index f88886ec6f1..eb7a873072b 100644 --- a/docs/ru/sql-reference/functions/tuple-functions.md +++ b/docs/ru/sql-reference/functions/tuple-functions.md @@ -111,4 +111,3 @@ SELECT untuple((* EXCEPT (v2, v3),)) FROM kv; - [Tuple](../../sql-reference/data-types/tuple.md) -[Оригинальная статья](https://clickhouse.tech/docs/ru/sql-reference/functions/tuple-functions/) diff --git a/docs/ru/sql-reference/functions/tuple-map-functions.md b/docs/ru/sql-reference/functions/tuple-map-functions.md index 696fdb9e5ae..23aca717c05 100644 --- a/docs/ru/sql-reference/functions/tuple-map-functions.md +++ b/docs/ru/sql-reference/functions/tuple-map-functions.md @@ -301,4 +301,3 @@ SELECT mapValues(a) FROM test; └──────────────────┘ ``` -[Оригинальная статья](https://clickhouse.tech/docs/ru/sql-reference/functions/tuple-map-functions/) diff --git a/docs/ru/sql-reference/functions/type-conversion-functions.md b/docs/ru/sql-reference/functions/type-conversion-functions.md index f312f9f5847..c1aa982f051 100644 --- a/docs/ru/sql-reference/functions/type-conversion-functions.md +++ b/docs/ru/sql-reference/functions/type-conversion-functions.md @@ -1195,4 +1195,3 @@ FROM numbers(3); └───────────────────────────────────────────┘ ``` -[Оригинальная статья](https://clickhouse.tech/docs/ru/query_language/functions/type_conversion_functions/) diff --git a/docs/ru/sql-reference/functions/url-functions.md b/docs/ru/sql-reference/functions/url-functions.md index 83f7fd32f6c..0d2b3030f9b 100644 --- a/docs/ru/sql-reference/functions/url-functions.md +++ b/docs/ru/sql-reference/functions/url-functions.md @@ -405,4 +405,3 @@ SELECT netloc('http://paul@www.example.com:80/'); Удаляет параметр URL с именем name, если такой есть. Функция работает при допущении, что имя параметра закодировано в URL в точности таким же образом, что и в переданном аргументе. -[Оригинальная статья](https://clickhouse.tech/docs/ru/query_language/functions/url_functions/) diff --git a/docs/ru/sql-reference/functions/ym-dict-functions.md b/docs/ru/sql-reference/functions/ym-dict-functions.md index f6d02e553a0..aa2be3e1892 100644 --- a/docs/ru/sql-reference/functions/ym-dict-functions.md +++ b/docs/ru/sql-reference/functions/ym-dict-functions.md @@ -151,4 +151,3 @@ regionToTopContinent(id[, geobase]); `ua` и `uk` обозначают одно и то же - украинский язык. -[Оригинальная статья](https://clickhouse.tech/docs/ru/query_language/functions/ym_dict_functions/) diff --git a/docs/ru/sql-reference/index.md b/docs/ru/sql-reference/index.md index 7aea530c7ee..62d6a9cecde 100644 --- a/docs/ru/sql-reference/index.md +++ b/docs/ru/sql-reference/index.md @@ -13,4 +13,3 @@ toc_title: hidden - [ALTER](statements/alter/index.md#query_language_queries_alter) - [Прочие виды запросов](statements/misc.md) -[Оригинальная статья](https://clickhouse.tech/docs/ru/query_language/) diff --git a/docs/ru/sql-reference/operators/index.md b/docs/ru/sql-reference/operators/index.md index 691c398ce4c..b7cacaf7a03 100644 --- a/docs/ru/sql-reference/operators/index.md +++ b/docs/ru/sql-reference/operators/index.md @@ -297,4 +297,3 @@ SELECT * FROM t_null WHERE y IS NOT NULL └───┴───┘ ``` -[Оригинальная статья](https://clickhouse.tech/docs/ru/query_language/operators/) diff --git a/docs/ru/sql-reference/statements/alter/column.md b/docs/ru/sql-reference/statements/alter/column.md index 35a1952d842..d1a117e4dd1 100644 --- a/docs/ru/sql-reference/statements/alter/column.md +++ b/docs/ru/sql-reference/statements/alter/column.md @@ -170,4 +170,3 @@ ALTER TABLE table_with_ttl MODIFY COLUMN column_ttl REMOVE TTL; Для таблиц, которые не хранят данные самостоятельно (типа [Merge](../../../sql-reference/statements/alter/index.md) и [Distributed](../../../sql-reference/statements/alter/index.md)), `ALTER` всего лишь меняет структуру таблицы, но не меняет структуру подчинённых таблиц. Для примера, при ALTER-е таблицы типа `Distributed`, вам также потребуется выполнить запрос `ALTER` для таблиц на всех удалённых серверах. -[Оригинальная статья](https://clickhouse.tech/docs/ru/query_language/alter/column/) diff --git a/docs/ru/sql-reference/statements/alter/constraint.md b/docs/ru/sql-reference/statements/alter/constraint.md index 13396f33621..452bf649415 100644 --- a/docs/ru/sql-reference/statements/alter/constraint.md +++ b/docs/ru/sql-reference/statements/alter/constraint.md @@ -20,4 +20,3 @@ ALTER TABLE [db].name DROP CONSTRAINT constraint_name; Запрос на изменение ограничений для Replicated таблиц реплицируется, сохраняя новые метаданные в ZooKeeper и применяя изменения на всех репликах. -[Оригинальная статья](https://clickhouse.tech/docs/ru/query_language/alter/constraint/) \ No newline at end of file diff --git a/docs/ru/sql-reference/statements/alter/delete.md b/docs/ru/sql-reference/statements/alter/delete.md index ee5f03d9d95..70a411dab83 100644 --- a/docs/ru/sql-reference/statements/alter/delete.md +++ b/docs/ru/sql-reference/statements/alter/delete.md @@ -26,4 +26,3 @@ ALTER TABLE [db.]table [ON CLUSTER cluster] DELETE WHERE filter_expr - [Синхронность запросов ALTER](../../../sql-reference/statements/alter/index.md#synchronicity-of-alter-queries) - [mutations_sync](../../../operations/settings/settings.md#mutations_sync) setting -[Оригинальная статья](https://clickhouse.tech/docs/ru/query_language/alter/delete/) \ No newline at end of file diff --git a/docs/ru/sql-reference/statements/alter/index.md b/docs/ru/sql-reference/statements/alter/index.md index 830c4a5745b..648fb7e7c5c 100644 --- a/docs/ru/sql-reference/statements/alter/index.md +++ b/docs/ru/sql-reference/statements/alter/index.md @@ -69,4 +69,3 @@ ALTER TABLE [db.]table MATERIALIZE INDEX name IN PARTITION partition_name Для запросов `ALTER TABLE ... UPDATE|DELETE` синхронность выполнения определяется настройкой [mutations_sync](../../../operations/settings/settings.md#mutations_sync). -[Оригинальная статья](https://clickhouse.tech/docs/ru/query_language/alter/index/) \ No newline at end of file diff --git a/docs/ru/sql-reference/statements/alter/index/index.md b/docs/ru/sql-reference/statements/alter/index/index.md index a42bccd7b47..862def5cc04 100644 --- a/docs/ru/sql-reference/statements/alter/index/index.md +++ b/docs/ru/sql-reference/statements/alter/index/index.md @@ -21,4 +21,3 @@ ALTER TABLE [db].name DROP INDEX name Запрос на изменение индексов реплицируется, сохраняя новые метаданные в ZooKeeper и применяя изменения на всех репликах. -[Оригинальная статья](https://clickhouse.tech/docs/ru/query_language/alter/index/index/) \ No newline at end of file diff --git a/docs/ru/sql-reference/statements/alter/order-by.md b/docs/ru/sql-reference/statements/alter/order-by.md index 32c0e382445..f0a9bfe3730 100644 --- a/docs/ru/sql-reference/statements/alter/order-by.md +++ b/docs/ru/sql-reference/statements/alter/order-by.md @@ -19,4 +19,3 @@ MODIFY ORDER BY new_expression сортировки, разрешено добавлять в ключ только новые столбцы (т.е. столбцы, добавляемые командой `ADD COLUMN` в том же запросе `ALTER`), у которых нет выражения по умолчанию. -[Оригинальная статья](https://clickhouse.tech/docs/ru/query_language/alter/order-by/) \ No newline at end of file diff --git a/docs/ru/sql-reference/statements/alter/partition.md b/docs/ru/sql-reference/statements/alter/partition.md index 8776c70c89e..3e7b069b066 100644 --- a/docs/ru/sql-reference/statements/alter/partition.md +++ b/docs/ru/sql-reference/statements/alter/partition.md @@ -306,4 +306,3 @@ OPTIMIZE TABLE table_not_partitioned PARTITION tuple() FINAL; Примеры запросов `ALTER ... PARTITION` можно посмотреть в тестах: [`00502_custom_partitioning_local`](https://github.com/ClickHouse/ClickHouse/blob/master/tests/queries/0_stateless/00502_custom_partitioning_local.sql) и [`00502_custom_partitioning_replicated_zookeeper`](https://github.com/ClickHouse/ClickHouse/blob/master/tests/queries/0_stateless/00502_custom_partitioning_replicated_zookeeper.sql). -[Оригинальная статья](https://clickhouse.tech/docs/ru/query_language/alter/partition/) diff --git a/docs/ru/sql-reference/statements/alter/quota.md b/docs/ru/sql-reference/statements/alter/quota.md index 0bdac1381da..4dadac1f473 100644 --- a/docs/ru/sql-reference/statements/alter/quota.md +++ b/docs/ru/sql-reference/statements/alter/quota.md @@ -39,4 +39,3 @@ ALTER QUOTA IF EXISTS qA FOR INTERVAL 15 month MAX queries = 123 TO CURRENT_USER ALTER QUOTA IF EXISTS qB FOR INTERVAL 30 minute MAX execution_time = 0.5, FOR INTERVAL 5 quarter MAX queries = 321, errors = 10 TO default; ``` -[Оригинальная статья](https://clickhouse.tech/docs/ru/sql-reference/alter/quota/) diff --git a/docs/ru/sql-reference/statements/alter/role.md b/docs/ru/sql-reference/statements/alter/role.md index 69f7c5828c5..e9ce62c58d5 100644 --- a/docs/ru/sql-reference/statements/alter/role.md +++ b/docs/ru/sql-reference/statements/alter/role.md @@ -15,4 +15,3 @@ ALTER ROLE [IF EXISTS] name1 [ON CLUSTER cluster_name1] [RENAME TO new_name1] [SETTINGS variable [= value] [MIN [=] min_value] [MAX [=] max_value] [READONLY|WRITABLE] | PROFILE 'profile_name'] [,...] ``` -[Оригинальная статья](https://clickhouse.tech/docs/ru/sql-reference/alter/role/) diff --git a/docs/ru/sql-reference/statements/alter/row-policy.md b/docs/ru/sql-reference/statements/alter/row-policy.md index e2d23cda3ff..cff4d4e497a 100644 --- a/docs/ru/sql-reference/statements/alter/row-policy.md +++ b/docs/ru/sql-reference/statements/alter/row-policy.md @@ -18,4 +18,3 @@ ALTER [ROW] POLICY [IF EXISTS] name1 [ON CLUSTER cluster_name1] ON [database1.]t [TO {role [,...] | ALL | ALL EXCEPT role [,...]}] ``` -[Оригинальная статья](https://clickhouse.tech/docs/ru/sql-reference/alter/row-policy/) \ No newline at end of file diff --git a/docs/ru/sql-reference/statements/alter/settings-profile.md b/docs/ru/sql-reference/statements/alter/settings-profile.md index 54502901837..9b8646919ca 100644 --- a/docs/ru/sql-reference/statements/alter/settings-profile.md +++ b/docs/ru/sql-reference/statements/alter/settings-profile.md @@ -15,4 +15,3 @@ ALTER SETTINGS PROFILE [IF EXISTS] TO name1 [ON CLUSTER cluster_name1] [RENAME T [SETTINGS variable [= value] [MIN [=] min_value] [MAX [=] max_value] [READONLY|WRITABLE] | INHERIT 'profile_name'] [,...] ``` -[Оригинальная статья](https://clickhouse.tech/docs/ru/sql-reference/alter/settings-profile) \ No newline at end of file diff --git a/docs/ru/sql-reference/statements/alter/ttl.md b/docs/ru/sql-reference/statements/alter/ttl.md index 5721ec9cf27..32b35da8627 100644 --- a/docs/ru/sql-reference/statements/alter/ttl.md +++ b/docs/ru/sql-reference/statements/alter/ttl.md @@ -83,4 +83,3 @@ SELECT * FROM table_with_ttl; - Подробнее о [свойстве TTL](../../../engines/table-engines/mergetree-family/mergetree#table_engine-mergetree-ttl). -[Оригинальная статья](https://clickhouse.tech/docs/ru/query_language/alter/ttl/) diff --git a/docs/ru/sql-reference/statements/alter/update.md b/docs/ru/sql-reference/statements/alter/update.md index e3d6725419a..206412d4be9 100644 --- a/docs/ru/sql-reference/statements/alter/update.md +++ b/docs/ru/sql-reference/statements/alter/update.md @@ -26,4 +26,3 @@ ALTER TABLE [db.]table UPDATE column1 = expr1 [, ...] WHERE filter_expr - [Синхронность запросов ALTER](../../../sql-reference/statements/alter/index.md#synchronicity-of-alter-queries) - [mutations_sync](../../../operations/settings/settings.md#mutations_sync) setting -[Оригинальная статья](https://clickhouse.tech/docs/ru/query_language/alter/update/) \ No newline at end of file diff --git a/docs/ru/sql-reference/statements/alter/user.md b/docs/ru/sql-reference/statements/alter/user.md index 41574f74200..604eff9de15 100644 --- a/docs/ru/sql-reference/statements/alter/user.md +++ b/docs/ru/sql-reference/statements/alter/user.md @@ -44,4 +44,3 @@ ALTER USER user DEFAULT ROLE ALL ALTER USER user DEFAULT ROLE ALL EXCEPT role1, role2 ``` -[Оригинальная статья](https://clickhouse.tech/docs/ru/query_language/alter/user/) \ No newline at end of file diff --git a/docs/ru/sql-reference/statements/attach.md b/docs/ru/sql-reference/statements/attach.md index 259ab893e63..b0d0a31ba4a 100644 --- a/docs/ru/sql-reference/statements/attach.md +++ b/docs/ru/sql-reference/statements/attach.md @@ -19,5 +19,4 @@ ATTACH TABLE [IF NOT EXISTS] [db.]name [ON CLUSTER cluster] Этот запрос используется при старте сервера. Сервер хранит метаданные таблиц в виде файлов с запросами `ATTACH`, которые он просто исполняет при запуске (за исключением системных таблиц, которые явно создаются на сервере). -[Оригинальная статья](https://clickhouse.tech/docs/ru/sql-reference/statements/attach/) diff --git a/docs/ru/sql-reference/statements/check-table.md b/docs/ru/sql-reference/statements/check-table.md index 3dc135d87c6..10336f821d0 100644 --- a/docs/ru/sql-reference/statements/check-table.md +++ b/docs/ru/sql-reference/statements/check-table.md @@ -41,4 +41,3 @@ CHECK TABLE [db.]name 4. Перезапустите `clickhouse-client`, чтобы вернуть предыдущее значение параметра `max_threads`. -[Оригинальная статья](https://clickhouse.tech/docs/ru/sql-reference/statements/check-table/) \ No newline at end of file diff --git a/docs/ru/sql-reference/statements/create/database.md b/docs/ru/sql-reference/statements/create/database.md index 0e880517134..7d19f3e8f17 100644 --- a/docs/ru/sql-reference/statements/create/database.md +++ b/docs/ru/sql-reference/statements/create/database.md @@ -31,5 +31,4 @@ CREATE DATABASE [IF NOT EXISTS] db_name [ON CLUSTER cluster] [ENGINE = engine(.. По умолчанию ClickHouse использует собственный движок баз данных. -[Оригинальная статья](https://clickhouse.tech/docs/ru/sql-reference/statements/create/database) diff --git a/docs/ru/sql-reference/statements/create/dictionary.md b/docs/ru/sql-reference/statements/create/dictionary.md index dba2aa61ca1..a41b2cb9ad5 100644 --- a/docs/ru/sql-reference/statements/create/dictionary.md +++ b/docs/ru/sql-reference/statements/create/dictionary.md @@ -27,5 +27,4 @@ LIFETIME({MIN min_val MAX max_val | max_val}) Смотрите [Внешние словари](../../../sql-reference/dictionaries/external-dictionaries/external-dicts.md). -[Оригинальная статья](https://clickhouse.tech/docs/ru/sql-reference/statements/create/dictionary) \ No newline at end of file diff --git a/docs/ru/sql-reference/statements/create/index.md b/docs/ru/sql-reference/statements/create/index.md index 70961e4f404..dfa5c28fff7 100644 --- a/docs/ru/sql-reference/statements/create/index.md +++ b/docs/ru/sql-reference/statements/create/index.md @@ -18,4 +18,3 @@ toc_title: "Обзор" - [QUOTA](../../../sql-reference/statements/create/quota.md) - [SETTINGS PROFILE](../../../sql-reference/statements/create/settings-profile.md) -[Оригинальная статья](https://clickhouse.tech/docs/en/sql-reference/statements/create/) diff --git a/docs/ru/sql-reference/statements/create/quota.md b/docs/ru/sql-reference/statements/create/quota.md index f5ac0df010e..8b07b51337a 100644 --- a/docs/ru/sql-reference/statements/create/quota.md +++ b/docs/ru/sql-reference/statements/create/quota.md @@ -37,5 +37,4 @@ CREATE QUOTA qA FOR INTERVAL 15 month MAX queries = 123 TO CURRENT_USER; CREATE QUOTA qB FOR INTERVAL 30 minute MAX execution_time = 0.5, FOR INTERVAL 5 quarter MAX queries = 321, errors = 10 TO default; ``` -[Оригинальная статья](https://clickhouse.tech/docs/ru/sql-reference/statements/create/quota) \ No newline at end of file diff --git a/docs/ru/sql-reference/statements/create/role.md b/docs/ru/sql-reference/statements/create/role.md index 8592f263156..16450b41126 100644 --- a/docs/ru/sql-reference/statements/create/role.md +++ b/docs/ru/sql-reference/statements/create/role.md @@ -46,5 +46,4 @@ SET ROLE accountant; SELECT * FROM db.*; ``` -[Оригинальная статья](https://clickhouse.tech/docs/ru/sql-reference/statements/create/role) \ No newline at end of file diff --git a/docs/ru/sql-reference/statements/create/row-policy.md b/docs/ru/sql-reference/statements/create/row-policy.md index 75f6fdfd2e1..88709598906 100644 --- a/docs/ru/sql-reference/statements/create/row-policy.md +++ b/docs/ru/sql-reference/statements/create/row-policy.md @@ -42,5 +42,4 @@ CREATE [ROW] POLICY [IF NOT EXISTS | OR REPLACE] policy_name1 [ON CLUSTER cluste `CREATE ROW POLICY filter ON mydb.mytable FOR SELECT USING a<1000 TO ALL EXCEPT mira` -[Оригинальная статья](https://clickhouse.tech/docs/ru/sql-reference/statements/create/row-policy) \ No newline at end of file diff --git a/docs/ru/sql-reference/statements/create/settings-profile.md b/docs/ru/sql-reference/statements/create/settings-profile.md index 5838ddc9153..522caf04c80 100644 --- a/docs/ru/sql-reference/statements/create/settings-profile.md +++ b/docs/ru/sql-reference/statements/create/settings-profile.md @@ -25,5 +25,4 @@ CREATE SETTINGS PROFILE [IF NOT EXISTS | OR REPLACE] TO name1 [ON CLUSTER cluste CREATE SETTINGS PROFILE max_memory_usage_profile SETTINGS max_memory_usage = 100000001 MIN 90000000 MAX 110000000 TO robin ``` -[Оригинальная статья](https://clickhouse.tech/docs/ru/sql-reference/statements/create/settings-profile) \ No newline at end of file diff --git a/docs/ru/sql-reference/statements/create/table.md b/docs/ru/sql-reference/statements/create/table.md index 8e2c471e548..a210c3687ef 100644 --- a/docs/ru/sql-reference/statements/create/table.md +++ b/docs/ru/sql-reference/statements/create/table.md @@ -243,5 +243,4 @@ CREATE TEMPORARY TABLE [IF NOT EXISTS] table_name -[Оригинальная статья](https://clickhouse.tech/docs/ru/sql-reference/statements/create/table) diff --git a/docs/ru/sql-reference/statements/create/user.md b/docs/ru/sql-reference/statements/create/user.md index ac9547691e6..68277d67052 100644 --- a/docs/ru/sql-reference/statements/create/user.md +++ b/docs/ru/sql-reference/statements/create/user.md @@ -81,5 +81,4 @@ CREATE USER user DEFAULT ROLE ALL CREATE USER john DEFAULT ROLE ALL EXCEPT role1, role2 ``` -[Оригинальная статья](https://clickhouse.tech/docs/ru/sql-reference/statements/create/user) \ No newline at end of file diff --git a/docs/ru/sql-reference/statements/create/view.md b/docs/ru/sql-reference/statements/create/view.md index da021059a8e..f867fc18de2 100644 --- a/docs/ru/sql-reference/statements/create/view.md +++ b/docs/ru/sql-reference/statements/create/view.md @@ -62,4 +62,3 @@ CREATE MATERIALIZED VIEW [IF NOT EXISTS] [db.]table_name [ON CLUSTER] [TO[db.]na Отсутствует отдельный запрос для удаления представлений. Чтобы удалить представление, следует использовать `DROP TABLE`. -[Оригинальная статья](https://clickhouse.tech/docs/ru/sql-reference/statements/create/view) diff --git a/docs/ru/sql-reference/statements/describe-table.md b/docs/ru/sql-reference/statements/describe-table.md index 64ed61de232..c66dbb66521 100644 --- a/docs/ru/sql-reference/statements/describe-table.md +++ b/docs/ru/sql-reference/statements/describe-table.md @@ -21,4 +21,3 @@ DESC|DESCRIBE TABLE [db.]table [INTO OUTFILE filename] [FORMAT format] Вложенные структуры данных выводятся в «развёрнутом» виде. То есть, каждый столбец - по отдельности, с именем через точку. -[Оригинальная статья](https://clickhouse.tech/docs/ru/sql-reference/statements/describe-table/) diff --git a/docs/ru/sql-reference/statements/detach.md b/docs/ru/sql-reference/statements/detach.md index 00d0a4b20c6..1dd5cd97643 100644 --- a/docs/ru/sql-reference/statements/detach.md +++ b/docs/ru/sql-reference/statements/detach.md @@ -16,4 +16,3 @@ DETACH TABLE [IF EXISTS] [db.]name [ON CLUSTER cluster] Запроса `DETACH DATABASE` нет. -[Оригинальная статья](https://clickhouse.tech/docs/ru/sql-reference/statements/detach/) diff --git a/docs/ru/sql-reference/statements/drop.md b/docs/ru/sql-reference/statements/drop.md index 514a92db91f..118f8eb923a 100644 --- a/docs/ru/sql-reference/statements/drop.md +++ b/docs/ru/sql-reference/statements/drop.md @@ -97,4 +97,3 @@ DROP [SETTINGS] PROFILE [IF EXISTS] name [,...] [ON CLUSTER cluster_name] DROP VIEW [IF EXISTS] [db.]name [ON CLUSTER cluster] ``` -[Оригинальная статья](https://clickhouse.tech/docs/ru/sql-reference/statements/drop/) \ No newline at end of file diff --git a/docs/ru/sql-reference/statements/exists.md b/docs/ru/sql-reference/statements/exists.md index 0b2fd69273c..d4f1f707e79 100644 --- a/docs/ru/sql-reference/statements/exists.md +++ b/docs/ru/sql-reference/statements/exists.md @@ -12,4 +12,3 @@ EXISTS [TEMPORARY] TABLE [db.]name [INTO OUTFILE filename] [FORMAT format] Возвращает один столбец типа `UInt8`, содержащий одно значение - `0`, если таблицы или БД не существует и `1`, если таблица в указанной БД существует. -[Оригинальная статья](https://clickhouse.tech/docs/ru/sql-reference/statements/exists/) \ No newline at end of file diff --git a/docs/ru/sql-reference/statements/grant.md b/docs/ru/sql-reference/statements/grant.md index d38e2ea38a0..7b2d26902ef 100644 --- a/docs/ru/sql-reference/statements/grant.md +++ b/docs/ru/sql-reference/statements/grant.md @@ -483,4 +483,3 @@ GRANT INSERT(x,y) ON db.table TO john Привилегия `ADMIN OPTION` разрешает пользователю назначать свои роли другому пользователю. -[Оригинальная статья](https://clickhouse.tech/docs/ru/sql-reference/statements/grant/) diff --git a/docs/ru/sql-reference/statements/insert-into.md b/docs/ru/sql-reference/statements/insert-into.md index 0ad85ed0166..bbd330962cf 100644 --- a/docs/ru/sql-reference/statements/insert-into.md +++ b/docs/ru/sql-reference/statements/insert-into.md @@ -119,4 +119,3 @@ INSERT INTO [db.]table [(c1, c2, c3)] SELECT ... - Данные поступают в режиме реального времени. - Вы загружаете данные, которые как правило отсортированы по времени. -[Оригинальная статья](https://clickhouse.tech/docs/ru/query_language/insert_into/) diff --git a/docs/ru/sql-reference/statements/kill.md b/docs/ru/sql-reference/statements/kill.md index e2556a7f782..6981d630dd8 100644 --- a/docs/ru/sql-reference/statements/kill.md +++ b/docs/ru/sql-reference/statements/kill.md @@ -70,4 +70,3 @@ KILL MUTATION WHERE database = 'default' AND table = 'table' AND mutation_id = ' Данные, уже изменённые мутацией, остаются в таблице (отката на старую версию данных не происходит). -[Оригинальная статья](https://clickhouse.tech/docs/ru/sql-reference/statements/kill/) \ No newline at end of file diff --git a/docs/ru/sql-reference/statements/misc.md b/docs/ru/sql-reference/statements/misc.md index e9ceece8b2c..cedf52b7a34 100644 --- a/docs/ru/sql-reference/statements/misc.md +++ b/docs/ru/sql-reference/statements/misc.md @@ -19,4 +19,3 @@ toc_priority: 41 - [TRUNCATE](../../sql-reference/statements/truncate.md) - [USE](../../sql-reference/statements/use.md) -[Оригинальная статья](https://clickhouse.tech/docs/ru/sql-reference/statements/misc/) \ No newline at end of file diff --git a/docs/ru/sql-reference/statements/optimize.md b/docs/ru/sql-reference/statements/optimize.md index 8b1d72fed80..44101910a6c 100644 --- a/docs/ru/sql-reference/statements/optimize.md +++ b/docs/ru/sql-reference/statements/optimize.md @@ -21,4 +21,3 @@ OPTIMIZE TABLE [db.]name [ON CLUSTER cluster] [PARTITION partition | PARTITION I !!! warning "Внимание" Запрос `OPTIMIZE` не может устранить причину появления ошибки «Too many parts». -[Оригинальная статья](https://clickhouse.tech/docs/ru/sql-reference/statements/optimize/) diff --git a/docs/ru/sql-reference/statements/rename.md b/docs/ru/sql-reference/statements/rename.md index 94bf3c682a1..104918c1a73 100644 --- a/docs/ru/sql-reference/statements/rename.md +++ b/docs/ru/sql-reference/statements/rename.md @@ -14,4 +14,3 @@ RENAME TABLE [db11.]name11 TO [db12.]name12, [db21.]name21 TO [db22.]name22, ... Переименовывание таблицы является лёгкой операцией. Если вы указали после `TO` другую базу данных, то таблица будет перенесена в эту базу данных. При этом, директории с базами данных должны быть расположены в одной файловой системе (иначе возвращается ошибка). В случае переименования нескольких таблиц в одном запросе — это неатомарная операция, может выполнится частично, запросы в других сессиях могут получить ошибку `Table ... doesn't exist...`. -[Оригинальная статья](https://clickhouse.tech/docs/ru/sql-reference/statements/rename/) diff --git a/docs/ru/sql-reference/statements/revoke.md b/docs/ru/sql-reference/statements/revoke.md index 339746b8591..a3a282d6e5c 100644 --- a/docs/ru/sql-reference/statements/revoke.md +++ b/docs/ru/sql-reference/statements/revoke.md @@ -45,4 +45,3 @@ GRANT SELECT ON accounts.staff TO mira; REVOKE SELECT(wage) ON accounts.staff FROM mira; ``` -[Оригинальная статья](https://clickhouse.tech/docs/en/operations/settings/settings/) diff --git a/docs/ru/sql-reference/statements/select/all.md b/docs/ru/sql-reference/statements/select/all.md index 4049d77a173..d36a23ca54e 100644 --- a/docs/ru/sql-reference/statements/select/all.md +++ b/docs/ru/sql-reference/statements/select/all.md @@ -19,4 +19,3 @@ SELECT sum(ALL number) FROM numbers(10); SELECT sum(number) FROM numbers(10); ``` -[Оригинальная статья](https://clickhouse.tech/docs/ru/sql-reference/statements/select/all) diff --git a/docs/ru/sql-reference/statements/select/index.md b/docs/ru/sql-reference/statements/select/index.md index a548a988a89..886952ea5cf 100644 --- a/docs/ru/sql-reference/statements/select/index.md +++ b/docs/ru/sql-reference/statements/select/index.md @@ -280,4 +280,3 @@ SELECT * REPLACE(i + 1 AS i) EXCEPT (j) APPLY(sum) from columns_transformers; SELECT * FROM some_table SETTINGS optimize_read_in_order=1, cast_keep_nullable=1; ``` -[Оригинальная статья](https://clickhouse.tech/docs/ru/sql-reference/statements/select/) diff --git a/docs/ru/sql-reference/statements/select/order-by.md b/docs/ru/sql-reference/statements/select/order-by.md index f8b838cbd15..9ddec923701 100644 --- a/docs/ru/sql-reference/statements/select/order-by.md +++ b/docs/ru/sql-reference/statements/select/order-by.md @@ -473,4 +473,3 @@ SELECT * FROM test_fetch ORDER BY a OFFSET 3 ROW FETCH FIRST 3 ROWS WITH TIES; └───┴───┘ ``` -[Оригинальная статья](https://clickhouse.tech/docs/ru/sql-reference/statements/select/order-by/) diff --git a/docs/ru/sql-reference/statements/select/union.md b/docs/ru/sql-reference/statements/select/union.md index 8f1dc11c802..de8a9b0e4ea 100644 --- a/docs/ru/sql-reference/statements/select/union.md +++ b/docs/ru/sql-reference/statements/select/union.md @@ -78,4 +78,3 @@ SELECT 1 UNION SELECT 2 UNION SELECT 3 UNION SELECT 2; Запросы, которые являются частью `UNION/UNION ALL/UNION DISTINCT`, выполняются параллельно, и их результаты могут быть смешаны вместе. -[Оригинальная статья](https://clickhouse.tech/docs/ru/sql-reference/statements/select/union/) diff --git a/docs/ru/sql-reference/statements/select/with.md b/docs/ru/sql-reference/statements/select/with.md index 328b28c27ef..7e09d94770a 100644 --- a/docs/ru/sql-reference/statements/select/with.md +++ b/docs/ru/sql-reference/statements/select/with.md @@ -67,4 +67,3 @@ WITH test1 AS (SELECT i + 1, j + 1 FROM test1) SELECT * FROM test1; ``` -[Оригинальная статья](https://clickhouse.tech/docs/en/sql-reference/statements/select/with/) diff --git a/docs/ru/sql-reference/statements/set-role.md b/docs/ru/sql-reference/statements/set-role.md index ccbef41aa9b..b21a9ec8319 100644 --- a/docs/ru/sql-reference/statements/set-role.md +++ b/docs/ru/sql-reference/statements/set-role.md @@ -54,4 +54,3 @@ SET DEFAULT ROLE ALL EXCEPT role1, role2 TO user ``` -[Оригинальная статья](https://clickhouse.tech/docs/ru/sql-reference/statements/set-role/) \ No newline at end of file diff --git a/docs/ru/sql-reference/statements/set.md b/docs/ru/sql-reference/statements/set.md index b60dfcf8324..fa96c3c2a1b 100644 --- a/docs/ru/sql-reference/statements/set.md +++ b/docs/ru/sql-reference/statements/set.md @@ -19,4 +19,3 @@ SET profile = 'profile-name-from-the-settings-file' Подробности смотрите в разделе [Настройки](../../operations/settings/settings.md). -[Оригинальная статья](https://clickhouse.tech/docs/ru/sql-reference/statements/set/) \ No newline at end of file diff --git a/docs/ru/sql-reference/statements/show.md b/docs/ru/sql-reference/statements/show.md index b214f0072e3..6d39bab4990 100644 --- a/docs/ru/sql-reference/statements/show.md +++ b/docs/ru/sql-reference/statements/show.md @@ -427,4 +427,3 @@ SHOW CHANGED SETTINGS ILIKE '%MEMORY%' - Таблица [system.settings](../../operations/system-tables/settings.md) -[Оригинальная статья](https://clickhouse.tech/docs/ru/sql-reference/statements/show/) diff --git a/docs/ru/sql-reference/statements/system.md b/docs/ru/sql-reference/statements/system.md index a6a6c5047af..ab68033d4f3 100644 --- a/docs/ru/sql-reference/statements/system.md +++ b/docs/ru/sql-reference/statements/system.md @@ -265,4 +265,3 @@ SYSTEM RESTART REPLICA [db.]replicated_merge_tree_family_table_name ### RESTART REPLICAS {#query_language-system-restart-replicas} Реинициализация состояния Zookeeper сессий для всех `ReplicatedMergeTree` таблиц, сравнивает текущее состояние с тем что хранится в Zookeeper как источник правды и добавляет задачи Zookeeper очередь если необходимо -[Оригинальная статья](https://clickhouse.tech/docs/ru/query_language/system/) diff --git a/docs/ru/sql-reference/statements/truncate.md b/docs/ru/sql-reference/statements/truncate.md index 4909d349658..b23d96d5b08 100644 --- a/docs/ru/sql-reference/statements/truncate.md +++ b/docs/ru/sql-reference/statements/truncate.md @@ -14,4 +14,3 @@ TRUNCATE TABLE [IF EXISTS] [db.]name [ON CLUSTER cluster] Запрос `TRUNCATE` не поддерживается для следующих движков: [View](../../engines/table-engines/special/view.md), [File](../../engines/table-engines/special/file.md), [URL](../../engines/table-engines/special/url.md) и [Null](../../engines/table-engines/special/null.md). -[Оригинальная статья](https://clickhouse.tech/docs/ru/sql-reference/statements/truncate/) \ No newline at end of file diff --git a/docs/ru/sql-reference/statements/use.md b/docs/ru/sql-reference/statements/use.md index c84329ea5ff..0d40870c23a 100644 --- a/docs/ru/sql-reference/statements/use.md +++ b/docs/ru/sql-reference/statements/use.md @@ -13,4 +13,3 @@ USE db Текущая база данных используется для поиска таблиц, если база данных не указана в запросе явно через точку перед именем таблицы. При использовании HTTP протокола запрос не может быть выполнен, так как понятия сессии не существует. -[Оригинальная статья](https://clickhouse.tech/docs/ru/sql-reference/statements/use/) \ No newline at end of file diff --git a/docs/ru/sql-reference/syntax.md b/docs/ru/sql-reference/syntax.md index d8eaa4f1731..6a923fd6b58 100644 --- a/docs/ru/sql-reference/syntax.md +++ b/docs/ru/sql-reference/syntax.md @@ -181,4 +181,3 @@ Code: 184. DB::Exception: Received from localhost:9000, 127.0.0.1. DB::Exception Список выражений - одно выражение или несколько выражений через запятую. Функции и операторы, в свою очередь, в качестве аргументов, могут иметь произвольные выражения. -[Оригинальная статья](https://clickhouse.tech/docs/ru/query_language/syntax/) diff --git a/docs/ru/sql-reference/table-functions/file.md b/docs/ru/sql-reference/table-functions/file.md index f9bdf902ad8..1d8604528be 100644 --- a/docs/ru/sql-reference/table-functions/file.md +++ b/docs/ru/sql-reference/table-functions/file.md @@ -126,4 +126,3 @@ SELECT count(*) FROM file('big_dir/file{0..9}{0..9}{0..9}', 'CSV', 'name String, - [Виртуальные столбцы](index.md#table_engines-virtual_columns) -[Оригинальная статья](https://clickhouse.tech/docs/ru/sql-reference/table-functions/file/) diff --git a/docs/ru/sql-reference/table-functions/generate.md b/docs/ru/sql-reference/table-functions/generate.md index 47b7e43bc86..91b8847be8f 100644 --- a/docs/ru/sql-reference/table-functions/generate.md +++ b/docs/ru/sql-reference/table-functions/generate.md @@ -38,4 +38,3 @@ SELECT * FROM generateRandom('a Array(Int8), d Decimal32(4), c Tuple(DateTime64( └──────────┴──────────────┴────────────────────────────────────────────────────────────────────┘ ``` -[Оригинальная статья](https://clickhouse.tech/docs/ru/query_language/table_functions/generate/) diff --git a/docs/ru/sql-reference/table-functions/hdfs.md b/docs/ru/sql-reference/table-functions/hdfs.md index 6edd70b7b1b..56aaeae487c 100644 --- a/docs/ru/sql-reference/table-functions/hdfs.md +++ b/docs/ru/sql-reference/table-functions/hdfs.md @@ -61,4 +61,3 @@ LIMIT 2 - [Виртуальные столбцы](index.md#table_engines-virtual_columns) -[Оригинальная статья](https://clickhouse.tech/docs/ru/query_language/table_functions/hdfs/) diff --git a/docs/ru/sql-reference/table-functions/index.md b/docs/ru/sql-reference/table-functions/index.md index ca90306bbd5..52eaad13507 100644 --- a/docs/ru/sql-reference/table-functions/index.md +++ b/docs/ru/sql-reference/table-functions/index.md @@ -35,4 +35,3 @@ toc_title: "Введение" | [hdfs](../../sql-reference/table-functions/hdfs.md) | Создаёт таблицу с движком [HDFS](../../engines/table-engines/integrations/hdfs.md). | | [s3](../../sql-reference/table-functions/s3.md) | Создаёт таблицу с движком [S3](../../engines/table-engines/integrations/s3.md). | -[Оригинальная статья](https://clickhouse.tech/docs/ru/query_language/table_functions/) diff --git a/docs/ru/sql-reference/table-functions/input.md b/docs/ru/sql-reference/table-functions/input.md index 96cf7515d52..0f5f621a247 100644 --- a/docs/ru/sql-reference/table-functions/input.md +++ b/docs/ru/sql-reference/table-functions/input.md @@ -43,4 +43,3 @@ $ cat data.csv | clickhouse-client --query="INSERT INTO test FORMAT CSV" $ cat data.csv | clickhouse-client --query="INSERT INTO test SELECT * FROM input('test_structure') FORMAT CSV" ``` -[Оригинальная статья](https://clickhouse.tech/docs/ru/query_language/table_functions/input/) diff --git a/docs/ru/sql-reference/table-functions/jdbc.md b/docs/ru/sql-reference/table-functions/jdbc.md index d388262606f..4fc237f940d 100644 --- a/docs/ru/sql-reference/table-functions/jdbc.md +++ b/docs/ru/sql-reference/table-functions/jdbc.md @@ -24,4 +24,3 @@ SELECT * FROM jdbc('mysql://localhost:3306/?user=root&password=root', 'schema', SELECT * FROM jdbc('datasource://mysql-local', 'schema', 'table') ``` -[Оригинальная статья](https://clickhouse.tech/docs/ru/query_language/table_functions/jdbc/) diff --git a/docs/ru/sql-reference/table-functions/merge.md b/docs/ru/sql-reference/table-functions/merge.md index 0822fdfe535..5b33f458468 100644 --- a/docs/ru/sql-reference/table-functions/merge.md +++ b/docs/ru/sql-reference/table-functions/merge.md @@ -9,4 +9,3 @@ toc_title: merge Структура таблицы берётся из первой попавшейся таблицы, подходящей под регулярное выражение. -[Оригинальная статья](https://clickhouse.tech/docs/ru/query_language/table_functions/merge/) diff --git a/docs/ru/sql-reference/table-functions/mysql.md b/docs/ru/sql-reference/table-functions/mysql.md index 18b34d0bf6c..b90aad22fdf 100644 --- a/docs/ru/sql-reference/table-functions/mysql.md +++ b/docs/ru/sql-reference/table-functions/mysql.md @@ -96,4 +96,3 @@ SELECT * FROM mysql('localhost:3306', 'test', 'test', 'bayonet', '123'); - [Движок таблиц ‘MySQL’](../../sql-reference/table-functions/mysql.md) - [Использование MySQL как источника данных для внешнего словаря](../../sql-reference/table-functions/mysql.md#dicts-external_dicts_dict_sources-mysql) -[Оригинальная статья](https://clickhouse.tech/docs/ru/sql-reference/table_functions/mysql/) diff --git a/docs/ru/sql-reference/table-functions/numbers.md b/docs/ru/sql-reference/table-functions/numbers.md index 005f400e082..71f63078415 100644 --- a/docs/ru/sql-reference/table-functions/numbers.md +++ b/docs/ru/sql-reference/table-functions/numbers.md @@ -25,4 +25,3 @@ SELECT * FROM system.numbers LIMIT 10; select toDate('2010-01-01') + number as d FROM numbers(365); ``` -[Оригинальная статья](https://clickhouse.tech/docs/ru/query_language/table_functions/numbers/) diff --git a/docs/ru/sql-reference/table-functions/odbc.md b/docs/ru/sql-reference/table-functions/odbc.md index 19203123840..557e7d2a15b 100644 --- a/docs/ru/sql-reference/table-functions/odbc.md +++ b/docs/ru/sql-reference/table-functions/odbc.md @@ -103,4 +103,3 @@ SELECT * FROM odbc('DSN=mysqlconn', 'test', 'test') - [Внешние словари ODBC](../../sql-reference/table-functions/odbc.md#dicts-external_dicts_dict_sources-odbc) - [Движок таблиц ODBC](../../sql-reference/table-functions/odbc.md). -[Оригинальная статья](https://clickhouse.tech/docs/ru/query_language/table_functions/jdbc/) diff --git a/docs/ru/sql-reference/table-functions/remote.md b/docs/ru/sql-reference/table-functions/remote.md index 83b3687f61d..00179abb207 100644 --- a/docs/ru/sql-reference/table-functions/remote.md +++ b/docs/ru/sql-reference/table-functions/remote.md @@ -106,4 +106,3 @@ INSERT INTO FUNCTION remote('127.0.0.1', currentDatabase(), 'remote_table') VALU SELECT * FROM remote_table; ``` -[Оригинальная статья](https://clickhouse.tech/docs/ru/sql-reference/table-functions/remote/) diff --git a/docs/ru/sql-reference/table-functions/url.md b/docs/ru/sql-reference/table-functions/url.md index 043a9231e75..0552aa8e076 100644 --- a/docs/ru/sql-reference/table-functions/url.md +++ b/docs/ru/sql-reference/table-functions/url.md @@ -41,4 +41,3 @@ INSERT INTO FUNCTION url('http://127.0.0.1:8123/?query=INSERT+INTO+test_table+FO SELECT * FROM test_table; ``` -[Оригинальная статья](https://clickhouse.tech/docs/ru/sql-reference/table-functions/url/) diff --git a/docs/ru/sql-reference/table-functions/view.md b/docs/ru/sql-reference/table-functions/view.md index 8a97253d048..c081ec12747 100644 --- a/docs/ru/sql-reference/table-functions/view.md +++ b/docs/ru/sql-reference/table-functions/view.md @@ -59,4 +59,3 @@ SELECT * FROM cluster(`cluster_name`, view(SELECT a, b, c FROM table_name)) **Смотрите также** - [view](https://clickhouse.tech/docs/ru/engines/table-engines/special/view/#table_engines-view) -[Оригинальная статья](https://clickhouse.tech/docs/ru/query_language/table_functions/view/) \ No newline at end of file diff --git a/docs/ru/whats-new/extended-roadmap.md b/docs/ru/whats-new/extended-roadmap.md index 16c7709ec28..7b317d424f1 100644 --- a/docs/ru/whats-new/extended-roadmap.md +++ b/docs/ru/whats-new/extended-roadmap.md @@ -7,4 +7,3 @@ toc_title: Roadmap Планы развития на 2021 год опубликованы для обсуждения [здесь](https://github.com/ClickHouse/ClickHouse/issues/17623). -[Оригинальная статья](https://clickhouse.tech/docs/ru/roadmap/) diff --git a/docs/ru/whats-new/security-changelog.md b/docs/ru/whats-new/security-changelog.md index 1f46535833d..e3d26e772c4 100644 --- a/docs/ru/whats-new/security-changelog.md +++ b/docs/ru/whats-new/security-changelog.md @@ -73,4 +73,3 @@ unixODBC позволял указать путь для подключения Обнаружено благодаря: the UK’s National Cyber Security Centre (NCSC) -{## [Оригинальная статья](https://clickhouse.tech/docs/ru/security_changelog/) ##} From 10a885c9b2e90f37a896d02b74e5d8f9474ea8a7 Mon Sep 17 00:00:00 2001 From: Nikita Mikhaylov Date: Mon, 15 Mar 2021 14:36:01 +0300 Subject: [PATCH 472/544] return back formatting in client --- programs/client/Client.cpp | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/programs/client/Client.cpp b/programs/client/Client.cpp index c878a3071c4..3c27908741c 100644 --- a/programs/client/Client.cpp +++ b/programs/client/Client.cpp @@ -2096,10 +2096,10 @@ private: current_format = "Vertical"; /// It is not clear how to write progress with parallel formatting. It may increase code complexity significantly. - // if (!need_render_progress) - // block_out_stream = context.getOutputStreamParallelIfPossible(current_format, *out_buf, block); - // else - block_out_stream = context.getOutputStream(current_format, *out_buf, block); + if (!need_render_progress) + block_out_stream = context.getOutputStreamParallelIfPossible(current_format, *out_buf, block); + else + block_out_stream = context.getOutputStream(current_format, *out_buf, block); block_out_stream->writePrefix(); } From 6d6633eca442a6364e9cc6a61f1af3bed2da3dd4 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Mon, 15 Mar 2021 14:38:28 +0300 Subject: [PATCH 473/544] Fix garbage --- docs/ru/interfaces/formats.md | 16 ++++++++-------- .../data-types/simpleaggregatefunction.md | 4 ++-- 2 files changed, 10 insertions(+), 10 deletions(-) diff --git a/docs/ru/interfaces/formats.md b/docs/ru/interfaces/formats.md index 3a61d789e75..67cc80f5cd8 100644 --- a/docs/ru/interfaces/formats.md +++ b/docs/ru/interfaces/formats.md @@ -1173,7 +1173,7 @@ ClickHouse поддерживает настраиваемую точность Неподдержанные типы данных Parquet: `DATE32`, `TIME32`, `FIXED_SIZE_BINARY`, `JSON`, `UUID`, `ENUM`. -Типы данных столбцов в ClickHouse могут отличаться от типов данных соответствующих полей файла в формате Parquet. При вставке данных, ClickHouse интерпретирует типы данных в соответствии с таблицей выше, а затем [приводит](../query_language/functions/type_conversion_functions/#type_conversion_function-cast) данные к тому типу, который установлен для столбца таблицы. +Типы данных столбцов в ClickHouse могут отличаться от типов данных соответствующих полей файла в формате Parquet. При вставке данных, ClickHouse интерпретирует типы данных в соответствии с таблицей выше, а затем [приводит](../sql-reference/functions/type-conversion-functions/#type_conversion_function-cast) данные к тому типу, который установлен для столбца таблицы. ### Вставка и выборка данных {#vstavka-i-vyborka-dannykh} @@ -1230,7 +1230,7 @@ ClickHouse поддерживает настраиваемую точность Неподдержанные типы данных ORC: `DATE32`, `TIME32`, `FIXED_SIZE_BINARY`, `JSON`, `UUID`, `ENUM`. -Типы данных столбцов в таблицах ClickHouse могут отличаться от типов данных для соответствующих полей ORC. При вставке данных, ClickHouse интерпретирует типы данных ORC согласно таблице соответствия, а затем [приводит](../query_language/functions/type_conversion_functions/#type_conversion_function-cast) данные к типу, установленному для столбца таблицы ClickHouse. +Типы данных столбцов в таблицах ClickHouse могут отличаться от типов данных для соответствующих полей ORC. При вставке данных, ClickHouse интерпретирует типы данных ORC согласно таблице соответствия, а затем [приводит](../sql-reference/functions/type-conversion-functions/#type_conversion_function-cast) данные к типу, установленному для столбца таблицы ClickHouse. ### Вставка данных {#vstavka-dannykh-1} @@ -1268,7 +1268,7 @@ SELECT * FROM line_as_string; ## Regexp {#data-format-regexp} -Каждая строка импортируемых данных разбирается в соответствии с регулярным выражением. +Каждая строка импортируемых данных разбирается в соответствии с регулярным выражением. При работе с форматом `Regexp` можно использовать следующие параметры: @@ -1279,15 +1279,15 @@ SELECT * FROM line_as_string; - Escaped (как в [TSV](#tabseparated)) - Quoted (как в [Values](#data-format-values)) - Raw (данные импортируются как есть, без сериализации) -- `format_regexp_skip_unmatched` — [UInt8](../sql-reference/data-types/int-uint.md). Признак, будет ли генерироваться исключение в случае, если импортируемые данные не соответствуют регулярному выражению `format_regexp`. Может принимать значение `0` или `1`. +- `format_regexp_skip_unmatched` — [UInt8](../sql-reference/data-types/int-uint.md). Признак, будет ли генерироваться исключение в случае, если импортируемые данные не соответствуют регулярному выражению `format_regexp`. Может принимать значение `0` или `1`. -**Использование** +**Использование** -Регулярное выражение (шаблон) из параметра `format_regexp` применяется к каждой строке импортируемых данных. Количество частей в шаблоне (подшаблонов) должно соответствовать количеству колонок в импортируемых данных. +Регулярное выражение (шаблон) из параметра `format_regexp` применяется к каждой строке импортируемых данных. Количество частей в шаблоне (подшаблонов) должно соответствовать количеству колонок в импортируемых данных. -Строки импортируемых данных должны разделяться символом новой строки `'\n'` или символами `"\r\n"` (перенос строки в формате DOS). +Строки импортируемых данных должны разделяться символом новой строки `'\n'` или символами `"\r\n"` (перенос строки в формате DOS). -Данные, выделенные по подшаблонам, интерпретируются в соответствии с типом, указанным в параметре `format_regexp_escaping_rule`. +Данные, выделенные по подшаблонам, интерпретируются в соответствии с типом, указанным в параметре `format_regexp_escaping_rule`. Если строка импортируемых данных не соответствует регулярному выражению и параметр `format_regexp_skip_unmatched` равен 1, строка просто игнорируется. Если же параметр `format_regexp_skip_unmatched` равен 0, генерируется исключение. diff --git a/docs/ru/sql-reference/data-types/simpleaggregatefunction.md b/docs/ru/sql-reference/data-types/simpleaggregatefunction.md index 454add05e8a..0948153362b 100644 --- a/docs/ru/sql-reference/data-types/simpleaggregatefunction.md +++ b/docs/ru/sql-reference/data-types/simpleaggregatefunction.md @@ -21,8 +21,8 @@ - [`maxMap`](../../sql-reference/aggregate-functions/reference/maxmap.md#agg_functions-maxmap) !!! note "Примечание" - Значения `SimpleAggregateFunction(func, Type)` отображаются и хранятся так же, как и `Type`, поэтому комбинаторы [-Merge](../../sql-reference/aggregate-functions/combinators.md#aggregate_functions_combinators-merge) и [-State]((../../sql-reference/aggregate-functions/combinators.md#agg-functions-combinator-state) не требуются. - + Значения `SimpleAggregateFunction(func, Type)` отображаются и хранятся так же, как и `Type`, поэтому комбинаторы [-Merge](../../sql-reference/aggregate-functions/combinators.md#aggregate_functions_combinators-merge) и [-State](../../sql-reference/aggregate-functions/combinators.md#agg-functions-combinator-state) не требуются. + `SimpleAggregateFunction` имеет лучшую производительность, чем `AggregateFunction` с той же агрегатной функцией. **Параметры** From c506356cc6e5d93bc67b263ae5b394932a7159e4 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Mon, 15 Mar 2021 14:41:20 +0300 Subject: [PATCH 474/544] Fix trash links --- docs/ru/operations/system-tables/replication_queue.md | 4 ++-- docs/ru/sql-reference/statements/alter/ttl.md | 6 +++--- docs/ru/sql-reference/table-functions/url.md | 4 ++-- 3 files changed, 7 insertions(+), 7 deletions(-) diff --git a/docs/ru/operations/system-tables/replication_queue.md b/docs/ru/operations/system-tables/replication_queue.md index 2851551955a..56e8c695a21 100644 --- a/docs/ru/operations/system-tables/replication_queue.md +++ b/docs/ru/operations/system-tables/replication_queue.md @@ -70,11 +70,11 @@ num_tries: 36 last_exception: Code: 226, e.displayText() = DB::Exception: Marks file '/opt/clickhouse/data/merge/visits_v2/tmp_fetch_20201130_121373_121384_2/CounterID.mrk' doesn't exist (version 20.8.7.15 (official build)) last_attempt_time: 2020-12-08 17:35:54 num_postponed: 0 -postpone_reason: +postpone_reason: last_postpone_time: 1970-01-01 03:00:00 ``` **Смотрите также** -- [Управление таблицами ReplicatedMergeTree](../../sql-reference/statements/system.md/#query-language-system-replicated) +- [Управление таблицами ReplicatedMergeTree](../../sql-reference/statements/system.md#query-language-system-replicated) diff --git a/docs/ru/sql-reference/statements/alter/ttl.md b/docs/ru/sql-reference/statements/alter/ttl.md index 32b35da8627..e949c992bbe 100644 --- a/docs/ru/sql-reference/statements/alter/ttl.md +++ b/docs/ru/sql-reference/statements/alter/ttl.md @@ -18,7 +18,7 @@ ALTER TABLE table-name MODIFY TTL ttl-expression Удалить табличный TTL можно запросом следующего вида: ```sql -ALTER TABLE table_name REMOVE TTL +ALTER TABLE table_name REMOVE TTL ``` **Пример** @@ -64,7 +64,7 @@ ALTER TABLE table_with_ttl REMOVE TTL; Заново вставляем удаленную строку и снова принудительно запускаем очистку по `TTL` с помощью `OPTIMIZE`: -```sql +```sql INSERT INTO table_with_ttl VALUES (now() - INTERVAL 4 MONTH, 2, 'username2'); OPTIMIZE TABLE table_with_ttl FINAL; SELECT * FROM table_with_ttl; @@ -81,5 +81,5 @@ SELECT * FROM table_with_ttl; ### Смотрите также -- Подробнее о [свойстве TTL](../../../engines/table-engines/mergetree-family/mergetree#table_engine-mergetree-ttl). +- Подробнее о [свойстве TTL](../../../engines/table-engines/mergetree-family/mergetree.md#mergetree-column-ttl). diff --git a/docs/ru/sql-reference/table-functions/url.md b/docs/ru/sql-reference/table-functions/url.md index 0552aa8e076..a41a1f53cde 100644 --- a/docs/ru/sql-reference/table-functions/url.md +++ b/docs/ru/sql-reference/table-functions/url.md @@ -5,7 +5,7 @@ toc_title: url # url {#url} -Функция `url` берет данные по указанному адресу `URL` и создает из них таблицу указанной структуры со столбцами указанного формата. +Функция `url` берет данные по указанному адресу `URL` и создает из них таблицу указанной структуры со столбцами указанного формата. Функция `url` может быть использована в запросах `SELECT` и `INSERT` с таблицами на движке [URL](../../engines/table-engines/special/url.md). @@ -27,7 +27,7 @@ url(URL, format, structure) **Примеры** -Получение с HTTP-сервера первых 3 строк таблицы с данными в формате [CSV](../../interfaces/formats.md/#csv), содержащей столбцы типа [String](../../sql-reference/data-types/string.md) и [UInt32](../../sql-reference/data-types/int-uint.md). +Получение с HTTP-сервера первых 3 строк таблицы с данными в формате [CSV](../../interfaces/formats.md#csv), содержащей столбцы типа [String](../../sql-reference/data-types/string.md) и [UInt32](../../sql-reference/data-types/int-uint.md). ``` sql SELECT * FROM url('http://127.0.0.1:12345/', CSV, 'column1 String, column2 UInt32') LIMIT 3; From 588f3ee11ec5cd3f753ceda50fca06650b04a1e9 Mon Sep 17 00:00:00 2001 From: Nikita Mikhaylov Date: Mon, 15 Mar 2021 14:43:57 +0300 Subject: [PATCH 475/544] better[2] --- src/Processors/Formats/Impl/ParallelFormattingOutputFormat.h | 2 -- src/Server/HTTP/WriteBufferFromHTTPServerResponse.cpp | 2 -- 2 files changed, 4 deletions(-) diff --git a/src/Processors/Formats/Impl/ParallelFormattingOutputFormat.h b/src/Processors/Formats/Impl/ParallelFormattingOutputFormat.h index 3fcd1f0aadf..584aa364d27 100644 --- a/src/Processors/Formats/Impl/ParallelFormattingOutputFormat.h +++ b/src/Processors/Formats/Impl/ParallelFormattingOutputFormat.h @@ -6,8 +6,6 @@ #include #include #include -#include "IO/ReadBuffer.h" -#include "IO/ReadBufferFromString.h" #include "IO/WriteBufferFromString.h" #include #include diff --git a/src/Server/HTTP/WriteBufferFromHTTPServerResponse.cpp b/src/Server/HTTP/WriteBufferFromHTTPServerResponse.cpp index 7ded716b34e..355af038da9 100644 --- a/src/Server/HTTP/WriteBufferFromHTTPServerResponse.cpp +++ b/src/Server/HTTP/WriteBufferFromHTTPServerResponse.cpp @@ -149,8 +149,6 @@ void WriteBufferFromHTTPServerResponse::onProgress(const Progress & progress) { std::lock_guard lock(mutex); - std::cout << StackTrace().toString() << std::endl; - /// Cannot add new headers if body was started to send. if (headers_finished_sending) return; From 7978320cc2013c5f82284d8e51bad96a83d7cc44 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Mon, 15 Mar 2021 14:49:38 +0300 Subject: [PATCH 476/544] Fix tons of trash --- docs/ru/operations/settings/settings.md | 36 +++++++++---------- .../system-tables/data_type_families.md | 2 +- .../system-tables/detached_parts.md | 4 +-- .../operations/system-tables/table_engines.md | 4 +-- 4 files changed, 23 insertions(+), 23 deletions(-) diff --git a/docs/ru/operations/settings/settings.md b/docs/ru/operations/settings/settings.md index ab24b7f3a44..d35b40e6556 100644 --- a/docs/ru/operations/settings/settings.md +++ b/docs/ru/operations/settings/settings.md @@ -134,7 +134,7 @@ ClickHouse применяет настройку в тех случаях, ко ## max_http_get_redirects {#setting-max_http_get_redirects} -Ограничивает максимальное количество переходов по редиректам в таблицах с движком [URL](../../engines/table-engines/special/url.md) при выполнении HTTP запросов методом GET. Настройка применяется для обоих типов таблиц: созданных запросом [CREATE TABLE](../../sql_reference/create/#create-table-query) и с помощью табличной функции [url](../../sql-reference/table-functions/url.md). +Ограничивает максимальное количество переходов по редиректам в таблицах с движком [URL](../../engines/table-engines/special/url.md) при выполнении HTTP запросов методом GET. Настройка применяется для обоих типов таблиц: созданных запросом [CREATE TABLE](../../sql_reference/statements/create/table.md#create-table-query) и с помощью табличной функции [url](../../sql-reference/table-functions/url.md). Возможные значения: @@ -306,7 +306,7 @@ INSERT INTO test VALUES (lower('Hello')), (lower('world')), (lower('INSERT')), ( CREATE TABLE table_with_enum_column_for_tsv_insert (Id Int32,Value Enum('first' = 1, 'second' = 2)) ENGINE=Memory(); ``` -При включенной настройке `input_format_tsv_enum_as_number`: +При включенной настройке `input_format_tsv_enum_as_number`: ```sql SET input_format_tsv_enum_as_number = 1; @@ -556,7 +556,7 @@ ClickHouse может парсить только базовый формат `Y Возможные значения: -- 0 — Устаревшее поведение отключено. +- 0 — Устаревшее поведение отключено. - 1 — Устаревшее поведение включено. Значение по умолчанию: 0. @@ -1236,7 +1236,7 @@ SELECT area/period FROM account_orders FORMAT JSON; CREATE TABLE table_with_enum_column_for_csv_insert (Id Int32,Value Enum('first' = 1, 'second' = 2)) ENGINE=Memory(); ``` -При включенной настройке `input_format_csv_enum_as_number`: +При включенной настройке `input_format_csv_enum_as_number`: ```sql SET input_format_csv_enum_as_number = 1; @@ -1731,7 +1731,7 @@ ClickHouse генерирует исключение Включает или отключает режим синхронного добавления данных в распределенные таблицы (таблицы с движком [Distributed](../../engines/table-engines/special/distributed.md#distributed)). -По умолчанию ClickHouse вставляет данные в распределённую таблицу в асинхронном режиме. Если `insert_distributed_sync=1`, то данные вставляются сихронно, а запрос `INSERT` считается выполненным успешно, когда данные записаны на все шарды (по крайней мере на одну реплику для каждого шарда, если `internal_replication = true`). +По умолчанию ClickHouse вставляет данные в распределённую таблицу в асинхронном режиме. Если `insert_distributed_sync=1`, то данные вставляются сихронно, а запрос `INSERT` считается выполненным успешно, когда данные записаны на все шарды (по крайней мере на одну реплику для каждого шарда, если `internal_replication = true`). Возможные значения: @@ -2067,11 +2067,11 @@ SELECT * FROM a; ## ttl_only_drop_parts {#ttl_only_drop_parts} -Для таблиц [MergeTree](../../engines/table-engines/mergetree-family/mergetree.md) включает или отключает возможность полного удаления кусков данных, в которых все записи устарели. +Для таблиц [MergeTree](../../engines/table-engines/mergetree-family/mergetree.md) включает или отключает возможность полного удаления кусков данных, в которых все записи устарели. -Когда настройка `ttl_only_drop_parts` отключена (т.е. по умолчанию), сервер лишь удаляет устаревшие записи в соответствии с их временем жизни (TTL). +Когда настройка `ttl_only_drop_parts` отключена (т.е. по умолчанию), сервер лишь удаляет устаревшие записи в соответствии с их временем жизни (TTL). -Когда настройка `ttl_only_drop_parts` включена, сервер целиком удаляет куски данных, в которых все записи устарели. +Когда настройка `ttl_only_drop_parts` включена, сервер целиком удаляет куски данных, в которых все записи устарели. Удаление целых кусков данных вместо удаления отдельных записей позволяет устанавливать меньший таймаут `merge_with_ttl_timeout` и уменьшает нагрузку на сервер, что способствует росту производительности. @@ -2082,18 +2082,18 @@ SELECT * FROM a; Значение по умолчанию: `0`. -**См. также** +**См. также** - [Секции и настройки запроса CREATE TABLE](../../engines/table-engines/mergetree-family/mergetree.md#mergetree-query-clauses) (настройка `merge_with_ttl_timeout`) - [Table TTL](../../engines/table-engines/mergetree-family/mergetree.md#mergetree-table-ttl) ## output_format_pretty_max_value_width {#output_format_pretty_max_value_width} -Ограничивает длину значения, выводимого в формате [Pretty](../../interfaces/formats.md#pretty). Если значение длиннее указанного количества символов, оно обрезается. +Ограничивает длину значения, выводимого в формате [Pretty](../../interfaces/formats.md#pretty). Если значение длиннее указанного количества символов, оно обрезается. Возможные значения: -- Положительное целое число. +- Положительное целое число. - 0 — значение обрезается полностью. Значение по умолчанию: `10000` символов. @@ -2242,17 +2242,17 @@ SELECT * FROM system.events WHERE event='QueryMemoryLimitExceeded'; Включает или отключает сохранение типа `Nullable` для аргумента функции [CAST](../../sql-reference/functions/type-conversion-functions.md#type_conversion_function-cast). -Если настройка включена, то когда в функцию `CAST` передается аргумент с типом `Nullable`, функция возвращает результат, также преобразованный к типу `Nullable`. -Если настройка отключена, то функция `CAST` всегда возвращает результат строго указанного типа. +Если настройка включена, то когда в функцию `CAST` передается аргумент с типом `Nullable`, функция возвращает результат, также преобразованный к типу `Nullable`. +Если настройка отключена, то функция `CAST` всегда возвращает результат строго указанного типа. Возможные значения: - 0 — функция `CAST` преобразует аргумент строго к указанному типу. -- 1 — если аргумент имеет тип `Nullable`, то функция `CAST` преобразует его к типу `Nullable` для указанного типа. +- 1 — если аргумент имеет тип `Nullable`, то функция `CAST` преобразует его к типу `Nullable` для указанного типа. Значение по умолчанию: `0`. -**Примеры** +**Примеры** Запрос возвращает аргумент, преобразованный строго к указанному типу: @@ -2284,9 +2284,9 @@ SELECT CAST(toNullable(toInt32(0)) AS Int32) as x, toTypeName(x); └───┴───────────────────────────────────────────────────┘ ``` -**См. также** +**См. также** -- Функция [CAST](../../sql-reference/functions/type-conversion-functions.md#type_conversion_function-cast) +- Функция [CAST](../../sql-reference/functions/type-conversion-functions.md#type_conversion_function-cast) ## persistent {#persistent} @@ -2364,7 +2364,7 @@ SELECT number FROM numbers(3) FORMAT JSONEachRow; [ {"number":"0"}, {"number":"1"}, -{"number":"2"} +{"number":"2"} ] ``` diff --git a/docs/ru/operations/system-tables/data_type_families.md b/docs/ru/operations/system-tables/data_type_families.md index 3a9a4a3413a..ba4e5e64ec3 100644 --- a/docs/ru/operations/system-tables/data_type_families.md +++ b/docs/ru/operations/system-tables/data_type_families.md @@ -1,6 +1,6 @@ # system.data_type_families {#system_tables-data_type_families} -Содержит информацию о поддерживаемых [типах данных](../../sql-reference/data-types/). +Содержит информацию о поддерживаемых [типах данных](../../sql-reference/data-types/index.md). Столбцы: diff --git a/docs/ru/operations/system-tables/detached_parts.md b/docs/ru/operations/system-tables/detached_parts.md index 23fd4882c44..5e0aa29001f 100644 --- a/docs/ru/operations/system-tables/detached_parts.md +++ b/docs/ru/operations/system-tables/detached_parts.md @@ -1,6 +1,6 @@ # system.detached_parts {#system_tables-detached_parts} Содержит информацию об отсоединённых кусках таблиц семейства [MergeTree](../../engines/table-engines/mergetree-family/mergetree.md). Столбец `reason` содержит причину, по которой кусок был отсоединён. Для кусов, отсоединённых пользователем, `reason` содержит пустую строку. -Такие куски могут быть присоединены с помощью [ALTER TABLE ATTACH PARTITION\|PART](../../sql_reference/alter/#alter_attach-partition). Остальные столбцы описаны в [system.parts](#system_tables-parts). -Если имя куска некорректно, значения некоторых столбцов могут быть `NULL`. Такие куски могут быть удалены с помощью [ALTER TABLE DROP DETACHED PART](../../sql_reference/alter/#alter_drop-detached). +Такие куски могут быть присоединены с помощью [ALTER TABLE ATTACH PARTITION|PART](../../sql_reference/statements/alter/index.md#alter_attach-partition). Остальные столбцы описаны в [system.parts](#system_tables-parts). +Если имя куска некорректно, значения некоторых столбцов могут быть `NULL`. Такие куски могут быть удалены с помощью [ALTER TABLE DROP DETACHED PART](../../sql_reference/statements/alter/index.md#alter_drop-detached). diff --git a/docs/ru/operations/system-tables/table_engines.md b/docs/ru/operations/system-tables/table_engines.md index 6af29753bbf..b6f6d3decc2 100644 --- a/docs/ru/operations/system-tables/table_engines.md +++ b/docs/ru/operations/system-tables/table_engines.md @@ -6,8 +6,8 @@ - `name` (String) — имя движка. - `supports_settings` (UInt8) — флаг, показывающий поддержку секции `SETTINGS`. -- `supports_skipping_indices` (UInt8) — флаг, показывающий поддержку [индексов пропуска данных](table_engines/mergetree/#table_engine-mergetree-data_skipping-indexes). -- `supports_ttl` (UInt8) — флаг, показывающий поддержку [TTL](table_engines/mergetree/#table_engine-mergetree-ttl). +- `supports_skipping_indices` (UInt8) — флаг, показывающий поддержку [индексов пропуска данных](../../engines/table-engines/mergetree-family/mergetree.md#table_engine-mergetree-data_skipping-indexes). +- `supports_ttl` (UInt8) — флаг, показывающий поддержку [TTL](../../engines/table-engines/mergetree-family/mergetree.md#table_engine-mergetree-ttl). - `supports_sort_order` (UInt8) — флаг, показывающий поддержку секций `PARTITION_BY`, `PRIMARY_KEY`, `ORDER_BY` и `SAMPLE_BY`. - `supports_replication` (UInt8) — флаг, показывающий поддержку [репликации](../../engines/table-engines/mergetree-family/replication.md). - `supports_deduplication` (UInt8) — флаг, показывающий наличие в движке дедупликации данных. From 1f92c8ce581bfb8c820a7220a63ae005ed13316c Mon Sep 17 00:00:00 2001 From: Nikita Mikhaylov Date: Mon, 15 Mar 2021 14:51:24 +0300 Subject: [PATCH 477/544] parallel formatting everywhere --- programs/obfuscator/Obfuscator.cpp | 2 +- programs/odbc-bridge/MainHandler.cpp | 2 +- src/Dictionaries/HTTPDictionarySource.cpp | 4 ++-- src/Storages/HDFS/StorageHDFS.cpp | 2 +- src/Storages/StorageS3.cpp | 2 +- src/Storages/StorageURL.cpp | 2 +- 6 files changed, 7 insertions(+), 7 deletions(-) diff --git a/programs/obfuscator/Obfuscator.cpp b/programs/obfuscator/Obfuscator.cpp index 950db4e4f05..3ccbfd44357 100644 --- a/programs/obfuscator/Obfuscator.cpp +++ b/programs/obfuscator/Obfuscator.cpp @@ -1180,7 +1180,7 @@ try file_in.seek(0, SEEK_SET); BlockInputStreamPtr input = context.getInputFormat(input_format, file_in, header, max_block_size); - BlockOutputStreamPtr output = context.getOutputStream(output_format, file_out, header); + BlockOutputStreamPtr output = context.getOutputStreamParallelIfPossible(output_format, file_out, header); if (processed_rows + source_rows > limit) input = std::make_shared(input, limit - processed_rows, 0); diff --git a/programs/odbc-bridge/MainHandler.cpp b/programs/odbc-bridge/MainHandler.cpp index 4fcc9deea6a..079fc371ab4 100644 --- a/programs/odbc-bridge/MainHandler.cpp +++ b/programs/odbc-bridge/MainHandler.cpp @@ -176,7 +176,7 @@ void ODBCHandler::handleRequest(HTTPServerRequest & request, HTTPServerResponse std::string query = params.get("query"); LOG_TRACE(log, "Query: {}", query); - BlockOutputStreamPtr writer = FormatFactory::instance().getOutputStream(format, out, *sample_block, context); + BlockOutputStreamPtr writer = FormatFactory::instance().getOutputStreamParallelIfPossible(format, out, *sample_block, context); auto pool = getPool(connection_string); ODBCBlockInputStream inp(pool->get(), query, *sample_block, max_block_size); copyData(inp, *writer); diff --git a/src/Dictionaries/HTTPDictionarySource.cpp b/src/Dictionaries/HTTPDictionarySource.cpp index ddcac117e58..62bf478afc4 100644 --- a/src/Dictionaries/HTTPDictionarySource.cpp +++ b/src/Dictionaries/HTTPDictionarySource.cpp @@ -136,7 +136,7 @@ BlockInputStreamPtr HTTPDictionarySource::loadIds(const std::vector & id ReadWriteBufferFromHTTP::OutStreamCallback out_stream_callback = [block, this](std::ostream & ostr) { WriteBufferFromOStream out_buffer(ostr); - auto output_stream = context.getOutputStream(format, out_buffer, sample_block); + auto output_stream = context.getOutputStreamParallelIfPossible(format, out_buffer, sample_block); formatBlock(output_stream, block); }; @@ -157,7 +157,7 @@ BlockInputStreamPtr HTTPDictionarySource::loadKeys(const Columns & key_columns, ReadWriteBufferFromHTTP::OutStreamCallback out_stream_callback = [block, this](std::ostream & ostr) { WriteBufferFromOStream out_buffer(ostr); - auto output_stream = context.getOutputStream(format, out_buffer, sample_block); + auto output_stream = context.getOutputStreamParallelIfPossible(format, out_buffer, sample_block); formatBlock(output_stream, block); }; diff --git a/src/Storages/HDFS/StorageHDFS.cpp b/src/Storages/HDFS/StorageHDFS.cpp index f7afd4a497d..e26d3375c33 100644 --- a/src/Storages/HDFS/StorageHDFS.cpp +++ b/src/Storages/HDFS/StorageHDFS.cpp @@ -183,7 +183,7 @@ public: : sample_block(sample_block_) { write_buf = wrapWriteBufferWithCompressionMethod(std::make_unique(uri, context.getGlobalContext().getConfigRef()), compression_method, 3); - writer = FormatFactory::instance().getOutputStream(format, *write_buf, sample_block, context); + writer = FormatFactory::instance().getOutputStreamParallelIfPossible(format, *write_buf, sample_block, context); } Block getHeader() const override diff --git a/src/Storages/StorageS3.cpp b/src/Storages/StorageS3.cpp index a31a7fa0944..1cbbe14d09f 100644 --- a/src/Storages/StorageS3.cpp +++ b/src/Storages/StorageS3.cpp @@ -155,7 +155,7 @@ namespace { write_buf = wrapWriteBufferWithCompressionMethod( std::make_unique(client, bucket, key, min_upload_part_size, max_single_part_upload_size), compression_method, 3); - writer = FormatFactory::instance().getOutputStream(format, *write_buf, sample_block, context); + writer = FormatFactory::instance().getOutputStreamParallelIfPossible(format, *write_buf, sample_block, context); } Block getHeader() const override diff --git a/src/Storages/StorageURL.cpp b/src/Storages/StorageURL.cpp index ca984f9ece9..8b16a08b957 100644 --- a/src/Storages/StorageURL.cpp +++ b/src/Storages/StorageURL.cpp @@ -155,7 +155,7 @@ StorageURLBlockOutputStream::StorageURLBlockOutputStream(const Poco::URI & uri, write_buf = wrapWriteBufferWithCompressionMethod( std::make_unique(uri, Poco::Net::HTTPRequest::HTTP_POST, timeouts), compression_method, 3); - writer = FormatFactory::instance().getOutputStream(format, *write_buf, sample_block, + writer = FormatFactory::instance().getOutputStreamParallelIfPossible(format, *write_buf, sample_block, context, {} /* write callback */, format_settings); } From da08d299552b73d5d07b3dbc704bbb7fdac79457 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Mon, 15 Mar 2021 14:56:04 +0300 Subject: [PATCH 478/544] Remove trash and fix broken links --- docs/ru/operations/settings/settings.md | 2 +- docs/ru/operations/system-tables/detached_parts.md | 4 ++-- docs/tools/single_page.py | 4 +++- 3 files changed, 6 insertions(+), 4 deletions(-) diff --git a/docs/ru/operations/settings/settings.md b/docs/ru/operations/settings/settings.md index d35b40e6556..c937a5f7112 100644 --- a/docs/ru/operations/settings/settings.md +++ b/docs/ru/operations/settings/settings.md @@ -134,7 +134,7 @@ ClickHouse применяет настройку в тех случаях, ко ## max_http_get_redirects {#setting-max_http_get_redirects} -Ограничивает максимальное количество переходов по редиректам в таблицах с движком [URL](../../engines/table-engines/special/url.md) при выполнении HTTP запросов методом GET. Настройка применяется для обоих типов таблиц: созданных запросом [CREATE TABLE](../../sql_reference/statements/create/table.md#create-table-query) и с помощью табличной функции [url](../../sql-reference/table-functions/url.md). +Ограничивает максимальное количество переходов по редиректам в таблицах с движком [URL](../../engines/table-engines/special/url.md) при выполнении HTTP запросов методом GET. Настройка применяется для обоих типов таблиц: созданных запросом [CREATE TABLE](../../sql-reference/statements/create/table.md#create-table-query) и с помощью табличной функции [url](../../sql-reference/table-functions/url.md). Возможные значения: diff --git a/docs/ru/operations/system-tables/detached_parts.md b/docs/ru/operations/system-tables/detached_parts.md index 5e0aa29001f..7abed6500aa 100644 --- a/docs/ru/operations/system-tables/detached_parts.md +++ b/docs/ru/operations/system-tables/detached_parts.md @@ -1,6 +1,6 @@ # system.detached_parts {#system_tables-detached_parts} Содержит информацию об отсоединённых кусках таблиц семейства [MergeTree](../../engines/table-engines/mergetree-family/mergetree.md). Столбец `reason` содержит причину, по которой кусок был отсоединён. Для кусов, отсоединённых пользователем, `reason` содержит пустую строку. -Такие куски могут быть присоединены с помощью [ALTER TABLE ATTACH PARTITION|PART](../../sql_reference/statements/alter/index.md#alter_attach-partition). Остальные столбцы описаны в [system.parts](#system_tables-parts). -Если имя куска некорректно, значения некоторых столбцов могут быть `NULL`. Такие куски могут быть удалены с помощью [ALTER TABLE DROP DETACHED PART](../../sql_reference/statements/alter/index.md#alter_drop-detached). +Такие куски могут быть присоединены с помощью [ALTER TABLE ATTACH PARTITION|PART](../../sql-reference/statements/alter/index.md#alter_attach-partition). Остальные столбцы описаны в [system.parts](#system_tables-parts). +Если имя куска некорректно, значения некоторых столбцов могут быть `NULL`. Такие куски могут быть удалены с помощью [ALTER TABLE DROP DETACHED PART](../../sql-reference/statements/alter/index.md#alter_drop-detached). diff --git a/docs/tools/single_page.py b/docs/tools/single_page.py index f885a84ec89..b88df5a03cb 100644 --- a/docs/tools/single_page.py +++ b/docs/tools/single_page.py @@ -28,13 +28,15 @@ anchor_not_allowed_chars = re.compile(r'[^\w\-]') def generate_anchor_from_path(path): return re.sub(anchor_not_allowed_chars, '-', path) +absolute_link = re.compile(r'^https?://') + def replace_link(match, path): title = match.group(1) link = match.group(2) # Not a relative link - if link.startswith('http'): + if re.search(absolute_link, link): return match.group(0) if link.endswith('/'): From c92d8c776cb650f96d3303e1982d04cdfae7ee56 Mon Sep 17 00:00:00 2001 From: Nikita Mikhaylov Date: Mon, 15 Mar 2021 15:14:58 +0300 Subject: [PATCH 479/544] try to avoid race in grpc server --- src/Server/GRPCServer.cpp | 6 ++++-- tests/integration/test_grpc_protocol/configs/users.xml | 8 -------- tests/integration/test_grpc_protocol/test.py | 2 +- 3 files changed, 5 insertions(+), 11 deletions(-) delete mode 100644 tests/integration/test_grpc_protocol/configs/users.xml diff --git a/src/Server/GRPCServer.cpp b/src/Server/GRPCServer.cpp index ede9bbff063..52a2c106488 100644 --- a/src/Server/GRPCServer.cpp +++ b/src/Server/GRPCServer.cpp @@ -783,8 +783,6 @@ namespace if (!io.out) return; - initializeBlockInputStream(io.out->getHeader()); - bool has_data_to_insert = (insert_query && insert_query->data) || !query_info.input_data().empty() || query_info.next_query_info(); if (!has_data_to_insert) @@ -795,6 +793,10 @@ namespace throw Exception("No data to insert", ErrorCodes::NO_DATA_TO_INSERT); } + /// This is significant, because parallel parsing may be used. + /// So we mustn't touch the input stream from other thread. + initializeBlockInputStream(io.out->getHeader()); + block_input_stream->readPrefix(); io.out->writePrefix(); diff --git a/tests/integration/test_grpc_protocol/configs/users.xml b/tests/integration/test_grpc_protocol/configs/users.xml deleted file mode 100644 index 2ae1a397fe5..00000000000 --- a/tests/integration/test_grpc_protocol/configs/users.xml +++ /dev/null @@ -1,8 +0,0 @@ - - - - - 0 - - - diff --git a/tests/integration/test_grpc_protocol/test.py b/tests/integration/test_grpc_protocol/test.py index 594879427ca..d8604276281 100644 --- a/tests/integration/test_grpc_protocol/test.py +++ b/tests/integration/test_grpc_protocol/test.py @@ -27,7 +27,7 @@ import clickhouse_grpc_pb2_grpc config_dir = os.path.join(SCRIPT_DIR, './configs') cluster = ClickHouseCluster(__file__) -node = cluster.add_instance('node', main_configs=['configs/grpc_config.xml'], user_configs=["configs/users.xml"]) +node = cluster.add_instance('node', main_configs=['configs/grpc_config.xml']) grpc_port = 9100 main_channel = None From 9820ab3558bf896f54c2232e102f84f674753ea7 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Mon, 15 Mar 2021 15:46:19 +0300 Subject: [PATCH 480/544] Fix not working links on single-page docs --- website/js/base.js | 17 ----------------- 1 file changed, 17 deletions(-) diff --git a/website/js/base.js b/website/js/base.js index 6cec8313bd4..aca6f407d24 100644 --- a/website/js/base.js +++ b/website/js/base.js @@ -16,23 +16,6 @@ if (target_id && target_id.startsWith('logo-')) { selector = '#'; } - if (selector && selector.startsWith('#') && !is_tab && !is_collapse && !is_rating) { - event.preventDefault(); - var dst = window.location.href.replace(window.location.hash, ''); - var offset = 0; - - if (selector !== '#') { - var destination = $(selector); - if (destination.length) { - offset = destination.offset().top - $('#top-nav').height() * 1.5; - dst += selector; - } - } - $('html, body').animate({ - scrollTop: offset - }, 500); - window.history.replaceState('', document.title, dst); - } }); var top_nav = $('#top-nav.sticky-top'); From 4028461c77067c509950145f2a31c55001d778e3 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Mon, 15 Mar 2021 15:54:44 +0300 Subject: [PATCH 481/544] Whitespaces --- docs/ru/introduction/index.md | 2 -- 1 file changed, 2 deletions(-) diff --git a/docs/ru/introduction/index.md b/docs/ru/introduction/index.md index c37cde09060..99f8aad0531 100644 --- a/docs/ru/introduction/index.md +++ b/docs/ru/introduction/index.md @@ -2,5 +2,3 @@ toc_folder_title: "Введение" toc_priority: 1 --- - - From 67a24de661ec0c95cf5e506c433cd37331f83081 Mon Sep 17 00:00:00 2001 From: Vladimir Date: Mon, 15 Mar 2021 15:54:51 +0300 Subject: [PATCH 482/544] Apply suggestions from code review --- docs/en/sql-reference/functions/bitmap-functions.md | 2 +- docs/ru/sql-reference/aggregate-functions/combinators.md | 4 ++-- .../sql-reference/aggregate-functions/reference/count.md | 2 +- .../aggregate-functions/reference/quantile.md | 2 +- .../reference/quantiledeterministic.md | 4 ++-- .../aggregate-functions/reference/quantileexact.md | 9 ++++++--- .../reference/quantileexactweighted.md | 2 +- .../aggregate-functions/reference/quantiletdigest.md | 2 +- .../reference/quantiletdigestweighted.md | 2 +- .../aggregate-functions/reference/quantiletiming.md | 2 +- .../reference/quantiletimingweighted.md | 2 +- docs/ru/sql-reference/functions/array-functions.md | 6 +++--- docs/ru/sql-reference/functions/conditional-functions.md | 2 +- 13 files changed, 22 insertions(+), 19 deletions(-) diff --git a/docs/en/sql-reference/functions/bitmap-functions.md b/docs/en/sql-reference/functions/bitmap-functions.md index c809aee85fe..36675a37e61 100644 --- a/docs/en/sql-reference/functions/bitmap-functions.md +++ b/docs/en/sql-reference/functions/bitmap-functions.md @@ -73,7 +73,7 @@ bitmapSubsetInRange(bitmap, range_start, range_end) - `bitmap` – [Bitmap object](#bitmap_functions-bitmapbuild). - `range_start` – Range start point. Type: [UInt32](../../sql-reference/data-types/int-uint.md). -- `range_end` – Range end point(excluded). Type: [UInt32](../../sql-reference/data-types/int-uint.md). +- `range_end` – Range end point (excluded). Type: [UInt32](../../sql-reference/data-types/int-uint.md). **Example** diff --git a/docs/ru/sql-reference/aggregate-functions/combinators.md b/docs/ru/sql-reference/aggregate-functions/combinators.md index 7d56fe7bafc..4310009ad09 100644 --- a/docs/ru/sql-reference/aggregate-functions/combinators.md +++ b/docs/ru/sql-reference/aggregate-functions/combinators.md @@ -72,7 +72,7 @@ toc_title: "Комбинаторы агрегатных функций" **Аргументы** -- `x` — параметры агрегатной функции. +- `x` — аргументы агрегатной функции. **Возращаемые зачения** @@ -133,7 +133,7 @@ FROM **Аргументы** -- `x` — параметры агрегатной функции. +- `x` — аргументы агрегатной функции. **Возвращаемые значения** diff --git a/docs/ru/sql-reference/aggregate-functions/reference/count.md b/docs/ru/sql-reference/aggregate-functions/reference/count.md index a9135a35f20..ab6de9379b2 100644 --- a/docs/ru/sql-reference/aggregate-functions/reference/count.md +++ b/docs/ru/sql-reference/aggregate-functions/reference/count.md @@ -21,7 +21,7 @@ ClickHouse поддерживает следующие виды синтакси **Возвращаемое значение** - Если функция вызывается без параметров, она вычисляет количество строк. -- Если передаётся [выражение](../../syntax.md#syntax-expressions), то функция вычисляет количество раз, когда выражение возвращает не NULL. Если выражение возвращает значение типа [Nullable](../../../sql-reference/data-types/nullable.md), то результат `count` не становится `Nullable`. Функция возвращает 0, если выражение возвращает `NULL` для всех строк. +- Если передаётся [выражение](../../syntax.md#syntax-expressions), то функция подсчитывает количество раз, когда выражение не равно NULL. Если выражение имеет тип [Nullable](../../../sql-reference/data-types/nullable.md), то результат `count` не становится `Nullable`. Функция возвращает 0, если выражение равно `NULL` для всех строк. В обоих случаях тип возвращаемого значения [UInt64](../../../sql-reference/data-types/int-uint.md). diff --git a/docs/ru/sql-reference/aggregate-functions/reference/quantile.md b/docs/ru/sql-reference/aggregate-functions/reference/quantile.md index 7cc4f8c7aef..a73e8c3abad 100644 --- a/docs/ru/sql-reference/aggregate-functions/reference/quantile.md +++ b/docs/ru/sql-reference/aggregate-functions/reference/quantile.md @@ -21,7 +21,7 @@ quantile(level)(expr) **Аргументы** - `level` — уровень квантили. Опционально. Константное значение с плавающей запятой от 0 до 1. Мы рекомендуем использовать значение `level` из диапазона `[0.01, 0.99]`. Значение по умолчанию: 0.5. При `level=0.5` функция вычисляет [медиану](https://ru.wikipedia.org/wiki/Медиана_(статистика)). -- `expr` — выражение над значениями столбца, которое возвращает данные [числовых типов](../../../sql-reference/data-types/index.md#data_types) или типов [Date](../../../sql-reference/data-types/date.md), [DateTime](../../../sql-reference/data-types/datetime.md). +- `expr` — выражение, зависящее от значений столбцов, возвращающее данные [числовых типов](../../../sql-reference/data-types/index.md#data_types) или типов [Date](../../../sql-reference/data-types/date.md), [DateTime](../../../sql-reference/data-types/datetime.md). **Возвращаемое значение** diff --git a/docs/ru/sql-reference/aggregate-functions/reference/quantiledeterministic.md b/docs/ru/sql-reference/aggregate-functions/reference/quantiledeterministic.md index 3c03c356ab9..b7ab7429fb3 100644 --- a/docs/ru/sql-reference/aggregate-functions/reference/quantiledeterministic.md +++ b/docs/ru/sql-reference/aggregate-functions/reference/quantiledeterministic.md @@ -21,8 +21,8 @@ quantileDeterministic(level)(expr, determinator) **Аргументы** - `level` — уровень квантили. Опционально. Константное значение с плавающей запятой от 0 до 1. Мы рекомендуем использовать значение `level` из диапазона `[0.01, 0.99]`. Значение по умолчанию: 0.5. При `level=0.5` функция вычисляет [медиану](https://ru.wikipedia.org/wiki/Медиана_(статистика)). -- `expr` — выражение над значениями столбца, которое возвращает данные [числовых типов](../../../sql-reference/data-types/index.md#data_types) или типов [Date](../../../sql-reference/data-types/date.md), [DateTime](../../../sql-reference/data-types/datetime.md). -- `determinator` — число, хэш которого используется при сэмплировании в алгоритме reservoir sampling, чтобы сделать результат детерминированным. В качестве детерминатора можно использовать любое определённое положительное число, например, идентификатор пользователя или события. Если одно и то же значение детерминатора попадается в выборке слишком часто, то функция выдаёт некорректный результат. +- `expr` — выражение, зависящее от значений столбцов, возвращающее данные [числовых типов](../../../sql-reference/data-types/index.md#data_types) или типов [Date](../../../sql-reference/data-types/date.md), [DateTime](../../../sql-reference/data-types/datetime.md). +- `determinator` — число, хэш которого используется при сэмплировании в алгоритме «Reservoir sampling», чтобы сделать результат детерминированным. В качестве значения можно использовать любое определённое положительное число, например, идентификатор пользователя или события. Если одно и то же значение попадается в выборке слишком часто, то функция выдаёт некорректный результат. **Возвращаемое значение** diff --git a/docs/ru/sql-reference/aggregate-functions/reference/quantileexact.md b/docs/ru/sql-reference/aggregate-functions/reference/quantileexact.md index 452b810cf03..82cf687a55c 100644 --- a/docs/ru/sql-reference/aggregate-functions/reference/quantileexact.md +++ b/docs/ru/sql-reference/aggregate-functions/reference/quantileexact.md @@ -21,7 +21,8 @@ quantileExact(level)(expr) **Аргументы** - `level` — уровень квантили. Опционально. Константное значение с плавающей запятой от 0 до 1. Мы рекомендуем использовать значение `level` из диапазона `[0.01, 0.99]`. Значение по умолчанию: 0.5. При `level=0.5` функция вычисляет [медиану](https://ru.wikipedia.org/wiki/Медиана_(статистика)). -- `expr` — выражение над значениями столбца, которое возвращает данные [числовых типов](../../../sql-reference/data-types/index.md#data_types) или типов [Date](../../../sql-reference/data-types/date.md), [DateTime](../../../sql-reference/data-types/datetime.md). +- `expr` — выражение, зависящее от значений столбцов, возвращающее данные [числовых типов](../../../sql-reference/data-types/index.md#data_types) или типов [Date](../../../sql-reference/data-types/date.md), [DateTime](../../../sql-reference/data-types/datetime.md). + **Возвращаемое значение** @@ -80,7 +81,8 @@ quantileExact(level)(expr) **Аргументы** - `level` — уровень квантили. Опциональный параметр. Константное занчение с плавающей запятой от 0 до 1. Мы рекомендуем использовать значение `level` из диапазона `[0.01, 0.99]`. Значение по умолчанию: 0.5. При `level=0.5` функция вычисляет [медиану](https://en.wikipedia.org/wiki/Median). -- `expr` — выражение над значениями столбца, которое возвращает данные [числовых типов](../../../sql-reference/data-types/index.md#data_types), [Date](../../../sql-reference/data-types/date.md) или [DateTime](../../../sql-reference/data-types/datetime.md). +- `expr` — выражение, зависящее от значений столбцов, возвращающее данные [числовых типов](../../../sql-reference/data-types/index.md#data_types), [Date](../../../sql-reference/data-types/date.md) или [DateTime](../../../sql-reference/data-types/datetime.md). + **Возвращаемое значение** @@ -130,7 +132,8 @@ quantileExactHigh(level)(expr) **Аргументы** - `level` — уровень квантили. Опциональный параметр. Константное занчение с плавающей запятой от 0 до 1. Мы рекомендуем использовать значение `level` из диапазона `[0.01, 0.99]`. Значение по умолчанию: 0.5. При `level=0.5` функция вычисляет [медиану](https://en.wikipedia.org/wiki/Median). -- `expr` — выражение над значениями столбца, которое возвращает данные [числовых типов](../../../sql-reference/data-types/index.md#data_types), [Date](../../../sql-reference/data-types/date.md) или [DateTime](../../../sql-reference/data-types/datetime.md). +- `expr` — выражение, зависящее от значений столбцов, возвращающее данные [числовых типов](../../../sql-reference/data-types/index.md#data_types), [Date](../../../sql-reference/data-types/date.md) или [DateTime](../../../sql-reference/data-types/datetime.md). + **Возвращаемое значение** diff --git a/docs/ru/sql-reference/aggregate-functions/reference/quantileexactweighted.md b/docs/ru/sql-reference/aggregate-functions/reference/quantileexactweighted.md index ee55aaec121..7a7a1f1d412 100644 --- a/docs/ru/sql-reference/aggregate-functions/reference/quantileexactweighted.md +++ b/docs/ru/sql-reference/aggregate-functions/reference/quantileexactweighted.md @@ -21,7 +21,7 @@ quantileExactWeighted(level)(expr, weight) **Аргументы** - `level` — уровень квантили. Опционально. Константное значение с плавающей запятой от 0 до 1. Мы рекомендуем использовать значение `level` из диапазона `[0.01, 0.99]`. Значение по умолчанию: 0.5. При `level=0.5` функция вычисляет [медиану](https://ru.wikipedia.org/wiki/Медиана_(статистика)). -- `expr` — выражение над значениями столбца, которое возвращает данные [числовых типов](../../../sql-reference/data-types/index.md#data_types) или типов [Date](../../../sql-reference/data-types/date.md), [DateTime](../../../sql-reference/data-types/datetime.md). +- `expr` — выражение, зависящее от значений столбцов, возвращающее данные [числовых типов](../../../sql-reference/data-types/index.md#data_types) или типов [Date](../../../sql-reference/data-types/date.md), [DateTime](../../../sql-reference/data-types/datetime.md). - `weight` — столбец с весам элементов последовательности. Вес — это количество повторений элемента в последовательности. **Возвращаемое значение** diff --git a/docs/ru/sql-reference/aggregate-functions/reference/quantiletdigest.md b/docs/ru/sql-reference/aggregate-functions/reference/quantiletdigest.md index a119ca940b6..63593c4f468 100644 --- a/docs/ru/sql-reference/aggregate-functions/reference/quantiletdigest.md +++ b/docs/ru/sql-reference/aggregate-functions/reference/quantiletdigest.md @@ -23,7 +23,7 @@ quantileTDigest(level)(expr) **Аргументы** - `level` — уровень квантили. Опционально. Константное значение с плавающей запятой от 0 до 1. Мы рекомендуем использовать значение `level` из диапазона `[0.01, 0.99]`. Значение по умолчанию: 0.5. При `level=0.5` функция вычисляет [медиану](https://ru.wikipedia.org/wiki/Медиана_(статистика)). -- `expr` — выражение над значениями столбца, которое возвращает данные [числовых типов](../../../sql-reference/data-types/index.md#data_types) или типов [Date](../../../sql-reference/data-types/date.md), [DateTime](../../../sql-reference/data-types/datetime.md). +- `expr` — выражение, зависящее от значений столбцов, возвращающее данные [числовых типов](../../../sql-reference/data-types/index.md#data_types) или типов [Date](../../../sql-reference/data-types/date.md), [DateTime](../../../sql-reference/data-types/datetime.md). **Возвращаемое значение** diff --git a/docs/ru/sql-reference/aggregate-functions/reference/quantiletdigestweighted.md b/docs/ru/sql-reference/aggregate-functions/reference/quantiletdigestweighted.md index a00c1f8af58..3dee2ffdc27 100644 --- a/docs/ru/sql-reference/aggregate-functions/reference/quantiletdigestweighted.md +++ b/docs/ru/sql-reference/aggregate-functions/reference/quantiletdigestweighted.md @@ -23,7 +23,7 @@ quantileTDigestWeighted(level)(expr, weight) **Аргументы** - `level` — уровень квантили. Опционально. Константное значение с плавающей запятой от 0 до 1. Мы рекомендуем использовать значение `level` из диапазона `[0.01, 0.99]`. Значение по умолчанию: 0.5. При `level=0.5` функция вычисляет [медиану](https://ru.wikipedia.org/wiki/Медиана_(статистика)). -- `expr` — выражение над значениями столбца, которое возвращает данные [числовых типов](../../../sql-reference/data-types/index.md#data_types) или типов [Date](../../../sql-reference/data-types/date.md), [DateTime](../../../sql-reference/data-types/datetime.md). +- `expr` — выражение, зависящее от значений столбцов, возвращающее данные [числовых типов](../../../sql-reference/data-types/index.md#data_types) или типов [Date](../../../sql-reference/data-types/date.md), [DateTime](../../../sql-reference/data-types/datetime.md). - `weight` — столбец с весам элементов последовательности. Вес — это количество повторений элемента в последовательности. **Возвращаемое значение** diff --git a/docs/ru/sql-reference/aggregate-functions/reference/quantiletiming.md b/docs/ru/sql-reference/aggregate-functions/reference/quantiletiming.md index 6131f2035cb..8921fc7766b 100644 --- a/docs/ru/sql-reference/aggregate-functions/reference/quantiletiming.md +++ b/docs/ru/sql-reference/aggregate-functions/reference/quantiletiming.md @@ -22,7 +22,7 @@ quantileTiming(level)(expr) - `level` — уровень квантили. Опционально. Константное значение с плавающей запятой от 0 до 1. Мы рекомендуем использовать значение `level` из диапазона `[0.01, 0.99]`. Значение по умолчанию: 0.5. При `level=0.5` функция вычисляет [медиану](https://ru.wikipedia.org/wiki/Медиана_(статистика)). -- `expr` — [выражение](../../syntax.md#syntax-expressions) над значения столбца, которые возвращают данные типа [Float\*](../../../sql-reference/data-types/float.md). +- `expr` — [выражение](../../syntax.md#syntax-expressions), зависящее от значений столбцов, возвращающее данные типа [Float\*](../../../sql-reference/data-types/float.md). - Если в функцию передать отрицательные значения, то её поведение не определено. - Если значение больше, чем 30 000 (например, время загрузки страницы превышает 30 секунд), то оно приравнивается к 30 000. diff --git a/docs/ru/sql-reference/aggregate-functions/reference/quantiletimingweighted.md b/docs/ru/sql-reference/aggregate-functions/reference/quantiletimingweighted.md index 58a0a4599f9..79a709181fe 100644 --- a/docs/ru/sql-reference/aggregate-functions/reference/quantiletimingweighted.md +++ b/docs/ru/sql-reference/aggregate-functions/reference/quantiletimingweighted.md @@ -22,7 +22,7 @@ quantileTimingWeighted(level)(expr, weight) - `level` — уровень квантили. Опционально. Константное значение с плавающей запятой от 0 до 1. Мы рекомендуем использовать значение `level` из диапазона `[0.01, 0.99]`. Значение по умолчанию: 0.5. При `level=0.5` функция вычисляет [медиану](https://ru.wikipedia.org/wiki/Медиана_(статистика)). -- `expr` — [выражение](../../syntax.md#syntax-expressions) над значения столбца, которые возвращают данные типа [Float\*](../../../sql-reference/data-types/float.md). +- `expr` — [выражение](../../syntax.md#syntax-expressions), зависящее от значений столбцов, возвращающее данные типа [Float\*](../../../sql-reference/data-types/float.md). - Если в функцию передать отрицательные значения, то её поведение не определено. - Если значение больше, чем 30 000 (например, время загрузки страницы превышает 30 секунд), то оно приравнивается к 30 000. diff --git a/docs/ru/sql-reference/functions/array-functions.md b/docs/ru/sql-reference/functions/array-functions.md index 965f8f725d7..51a34f9700f 100644 --- a/docs/ru/sql-reference/functions/array-functions.md +++ b/docs/ru/sql-reference/functions/array-functions.md @@ -371,7 +371,7 @@ arrayPushBack(array, single_value) **Аргументы** - `array` – массив. -- `single_value` – одиночное значение. В массив с числам можно добавить только числа, в массив со строками только строки. При добавлении чисел ClickHouse автоматически приводит тип `single_value` к типу данных массива. Подробнее о типах данных в ClickHouse читайте в разделе «[Типы данных](../../sql-reference/functions/array-functions.md#data_types)». Может быть равно `NULL`. Функция добавит элемент `NULL` в массив, а тип элементов массива преобразует в `Nullable`. +- `single_value` – значение добавляемого элемента. В массив с числам можно добавить только числа, в массив со строками только строки. При добавлении чисел ClickHouse автоматически приводит тип `single_value` к типу данных массива. Подробнее о типах данных в ClickHouse читайте в разделе «[Типы данных](../../sql-reference/functions/array-functions.md#data_types)». Может быть равно `NULL`, в этом случае функция добавит элемент `NULL` в массив, а тип элементов массива преобразует в `Nullable`. **Пример** @@ -396,7 +396,7 @@ arrayPushFront(array, single_value) **Аргументы** - `array` – массив. -- `single_value` – одиночное значение. В массив с числам можно добавить только числа, в массив со строками только строки. При добавлении чисел ClickHouse автоматически приводит тип `single_value` к типу данных массива. Подробнее о типах данных в ClickHouse читайте в разделе «[Типы данных](../../sql-reference/functions/array-functions.md#data_types)». Может быть равно `NULL`. Функция добавит элемент `NULL` в массив, а тип элементов массива преобразует в `Nullable`. +- `single_value` – значение добавляемого элемента. В массив с числам можно добавить только числа, в массив со строками только строки. При добавлении чисел ClickHouse автоматически приводит тип `single_value` к типу данных массива. Подробнее о типах данных в ClickHouse читайте в разделе «[Типы данных](../../sql-reference/functions/array-functions.md#data_types)». Может быть равно `NULL`, в этом случае функция добавит элемент `NULL` в массив, а тип элементов массива преобразует в `Nullable`. **Пример** @@ -890,7 +890,7 @@ arrayReduceInRanges(agg_func, ranges, arr1, arr2, ..., arrN) **Аргументы** - `agg_func` — имя агрегатной функции, которая должна быть [строковой](../../sql-reference/data-types/string.md) константой. -- `ranges` — диапазоны для агрегирования, которые должны быть [массивом](../../sql-reference/data-types/array.md) of [кортежей](../../sql-reference/data-types/tuple.md) который содержит индекс и длину каждого диапазона. +- `ranges` — диапазоны для агрегирования, которые должны быть [массивом](../../sql-reference/data-types/array.md) of [кортежей](../../sql-reference/data-types/tuple.md) содержащих индекс и длину каждого диапазона. - `arr` — любое количество столбцов типа [Array](../../sql-reference/data-types/array.md) в качестве параметров агрегатной функции. **Возвращаемое значение** diff --git a/docs/ru/sql-reference/functions/conditional-functions.md b/docs/ru/sql-reference/functions/conditional-functions.md index 1ca57ea8eb1..e70e4e2c02d 100644 --- a/docs/ru/sql-reference/functions/conditional-functions.md +++ b/docs/ru/sql-reference/functions/conditional-functions.md @@ -19,7 +19,7 @@ SELECT if(cond, then, else) **Аргументы** -- `cond` – условие, которое может быть равно 0 или нет. Может быть [UInt8](../../sql-reference/functions/conditional-functions.md) или `NULL`. +- `cond` – проверяемое условие. Может быть [UInt8](../../sql-reference/functions/conditional-functions.md) или `NULL`. - `then` – возвращается результат выражения, если условие `cond` истинно. - `else` – возвращается результат выражения, если условие `cond` ложно. From d7c43291b14092c7a393b5d3bfc2f9c8dd636c0b Mon Sep 17 00:00:00 2001 From: Alexander Tokmakov Date: Mon, 15 Mar 2021 17:23:04 +0300 Subject: [PATCH 483/544] improve hung check in stress test --- docker/test/stress/run.sh | 1 + docker/test/stress/stress | 12 +++++++++++- 2 files changed, 12 insertions(+), 1 deletion(-) diff --git a/docker/test/stress/run.sh b/docker/test/stress/run.sh index 9ddf7421934..3594eead992 100755 --- a/docker/test/stress/run.sh +++ b/docker/test/stress/run.sh @@ -130,6 +130,7 @@ zgrep -Fa "########################################" /test_output/* > /dev/null pigz < /var/log/clickhouse-server/clickhouse-server.log > /test_output/clickhouse-server.log.gz tar -chf /test_output/coordination.tar /var/lib/clickhouse/coordination ||: mv /var/log/clickhouse-server/stderr.log /test_output/ +tar -chf /test_output/query_log_dump.tar /var/lib/clickhouse/data/system/query_log ||: # Write check result into check_status.tsv clickhouse-local --structure "test String, res String" -q "SELECT 'failure', test FROM table WHERE res != 'OK' order by (lower(test) like '%hung%') LIMIT 1" < /test_output/test_results.tsv > /test_output/check_status.tsv diff --git a/docker/test/stress/stress b/docker/test/stress/stress index a6f1ae19303..25a705ecbd1 100755 --- a/docker/test/stress/stress +++ b/docker/test/stress/stress @@ -67,6 +67,10 @@ def prepare_for_hung_check(): logging.info("Will terminate gdb (if any)") call("kill -TERM $(pidof gdb)", shell=True, stderr=STDOUT) + # Some tests set too low memory limit for default user and forget to reset in back. + # It may cause SYSTEM queries to fail, let's disable memory limit. + call("clickhouse client --max_memory_usage_for_user=0 -q 'SELECT 1 FORMAT Null'", shell=True, stderr=STDOUT) + # Some tests execute SYSTEM STOP MERGES or similar queries. # It may cause some ALTERs to hang. # Possibly we should fix tests and forbid to use such queries without specifying table. @@ -78,7 +82,13 @@ def prepare_for_hung_check(): call("clickhouse client -q 'SYSTEM START REPLICATED SENDS'", shell=True, stderr=STDOUT) call("clickhouse client -q 'SYSTEM START REPLICATION QUEUES'", shell=True, stderr=STDOUT) - time.sleep(30) + # Issue #21004, live views are experimental, so let's just suppress it + call("""clickhouse client -q "KILL QUERY WHERE upper(query) LIKE 'WATCH %'" """, shell=True, stderr=STDOUT) + + # Wait for last queries to finish if any, not longer than 120 seconds + call("""clickhouse client -q "select sleepEachRow(( + select maxOrDefault(120 - elapsed) + 1 from system.processes where query not like '%from system.processes%' and elapsed < 120 + ) / 120) from numbers(120) format Null" """, shell=True, stderr=STDOUT) if __name__ == "__main__": logging.basicConfig(level=logging.INFO, format='%(asctime)s %(message)s') From 23a1c634602c94fa010c238088c0c0706d4d2b80 Mon Sep 17 00:00:00 2001 From: Nikita Mikhaylov Date: Mon, 15 Mar 2021 17:40:51 +0300 Subject: [PATCH 484/544] Update StorageURL.cpp --- src/Storages/StorageURL.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Storages/StorageURL.cpp b/src/Storages/StorageURL.cpp index 8b16a08b957..ca984f9ece9 100644 --- a/src/Storages/StorageURL.cpp +++ b/src/Storages/StorageURL.cpp @@ -155,7 +155,7 @@ StorageURLBlockOutputStream::StorageURLBlockOutputStream(const Poco::URI & uri, write_buf = wrapWriteBufferWithCompressionMethod( std::make_unique(uri, Poco::Net::HTTPRequest::HTTP_POST, timeouts), compression_method, 3); - writer = FormatFactory::instance().getOutputStreamParallelIfPossible(format, *write_buf, sample_block, + writer = FormatFactory::instance().getOutputStream(format, *write_buf, sample_block, context, {} /* write callback */, format_settings); } From 4e35cf55098ea8df3f68542f46eaed4420f76838 Mon Sep 17 00:00:00 2001 From: Kseniia Sumarokova <54203879+kssenii@users.noreply.github.com> Date: Mon, 15 Mar 2021 19:26:44 +0300 Subject: [PATCH 485/544] Update src/Storages/StoragePostgreSQL.cpp Co-authored-by: tavplubix --- src/Storages/StoragePostgreSQL.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Storages/StoragePostgreSQL.cpp b/src/Storages/StoragePostgreSQL.cpp index 3e122ed2fc7..ac1a5569293 100644 --- a/src/Storages/StoragePostgreSQL.cpp +++ b/src/Storages/StoragePostgreSQL.cpp @@ -298,7 +298,7 @@ void registerStoragePostgreSQL(StorageFactory & factory) ASTs & engine_args = args.engine_args; if (engine_args.size() < 5 || engine_args.size() > 6) - throw Exception("Storage PostgreSQL requires 5-6 parameters: " + throw Exception("Storage PostgreSQL requires from 5 to 6 parameters: " "PostgreSQL('host:port', 'database', 'table', 'username', 'password' [, 'schema']", ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH); From 9e8ebb5e2bbf9a740d269c4e251280aec4578125 Mon Sep 17 00:00:00 2001 From: tavplubix Date: Mon, 15 Mar 2021 20:07:36 +0300 Subject: [PATCH 486/544] Update TableFunctionPostgreSQL.cpp --- src/TableFunctions/TableFunctionPostgreSQL.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/TableFunctions/TableFunctionPostgreSQL.cpp b/src/TableFunctions/TableFunctionPostgreSQL.cpp index 0e3f1c5da24..0cf7c36b5ae 100644 --- a/src/TableFunctions/TableFunctionPostgreSQL.cpp +++ b/src/TableFunctions/TableFunctionPostgreSQL.cpp @@ -58,7 +58,7 @@ void TableFunctionPostgreSQL::parseArguments(const ASTPtr & ast_function, const ASTs & args = func_args.arguments->children; if (args.size() < 5 || args.size() > 6) - throw Exception("Table function 'PostgreSQL' requires 5-6 parameters: " + throw Exception("Table function 'PostgreSQL' requires from 5 to 6 parameters: " "PostgreSQL('host:port', 'database', 'table', 'user', 'password', [, 'schema']).", ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH); From badd5165dac3e864efeb863da3324a57ebabc6ee Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Mon, 15 Mar 2021 20:36:49 +0300 Subject: [PATCH 487/544] Fix UBSan report in rounding to years intervals --- base/common/DateLUTImpl.h | 9 ++++++++- .../0_stateless/01761_round_year_bounds.reference | 0 tests/queries/0_stateless/01761_round_year_bounds.sql | 1 + 3 files changed, 9 insertions(+), 1 deletion(-) create mode 100644 tests/queries/0_stateless/01761_round_year_bounds.reference create mode 100644 tests/queries/0_stateless/01761_round_year_bounds.sql diff --git a/base/common/DateLUTImpl.h b/base/common/DateLUTImpl.h index 6e968a0cd50..fc30ee9c6a0 100644 --- a/base/common/DateLUTImpl.h +++ b/base/common/DateLUTImpl.h @@ -807,7 +807,14 @@ public: return toFirstDayNumOfYear(v); const LUTIndex i = toLUTIndex(v); - return toDayNum(years_lut[lut[i].year / years * years - DATE_LUT_MIN_YEAR]); + + UInt16 year = lut[i].year / years * years; + + /// For example, rounding down 1925 to 100 years will be 1900, but it's less than min supported year. + if (unlikely(year < DATE_LUT_MIN_YEAR)) + year = DATE_LUT_MIN_YEAR; + + return toDayNum(years_lut[year - DATE_LUT_MIN_YEAR]); } inline ExtendedDayNum toStartOfQuarterInterval(ExtendedDayNum d, UInt64 quarters) const diff --git a/tests/queries/0_stateless/01761_round_year_bounds.reference b/tests/queries/0_stateless/01761_round_year_bounds.reference new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tests/queries/0_stateless/01761_round_year_bounds.sql b/tests/queries/0_stateless/01761_round_year_bounds.sql new file mode 100644 index 00000000000..fed12c55568 --- /dev/null +++ b/tests/queries/0_stateless/01761_round_year_bounds.sql @@ -0,0 +1 @@ +SELECT toStartOfInterval(toDateTime(-9223372036854775808), toIntervalYear(100), 'Europe/Moscow') FORMAT Null; From 845f4afbf4c7287a586b42874150f7b6ef039b80 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Mon, 15 Mar 2021 21:09:05 +0300 Subject: [PATCH 488/544] Fix parsing of pre-epoch time --- base/common/DateLUTImpl.h | 7 +------ .../01762_datetime64_extended_parsing.reference | 1 + .../0_stateless/01762_datetime64_extended_parsing.sql | 1 + 3 files changed, 3 insertions(+), 6 deletions(-) create mode 100644 tests/queries/0_stateless/01762_datetime64_extended_parsing.reference create mode 100644 tests/queries/0_stateless/01762_datetime64_extended_parsing.sql diff --git a/base/common/DateLUTImpl.h b/base/common/DateLUTImpl.h index fc30ee9c6a0..867862ad51e 100644 --- a/base/common/DateLUTImpl.h +++ b/base/common/DateLUTImpl.h @@ -913,12 +913,7 @@ public: if (time_offset >= lut[index].time_at_offset_change()) time_offset -= lut[index].amount_of_offset_change(); - UInt32 res = lut[index].date + time_offset; - - if (unlikely(res > DATE_LUT_MAX)) - return 0; - - return res; + return lut[index].date + time_offset; } template diff --git a/tests/queries/0_stateless/01762_datetime64_extended_parsing.reference b/tests/queries/0_stateless/01762_datetime64_extended_parsing.reference new file mode 100644 index 00000000000..531b6f8bf13 --- /dev/null +++ b/tests/queries/0_stateless/01762_datetime64_extended_parsing.reference @@ -0,0 +1 @@ +1925-01-02 03:04:05.678901 diff --git a/tests/queries/0_stateless/01762_datetime64_extended_parsing.sql b/tests/queries/0_stateless/01762_datetime64_extended_parsing.sql new file mode 100644 index 00000000000..a7ad447b215 --- /dev/null +++ b/tests/queries/0_stateless/01762_datetime64_extended_parsing.sql @@ -0,0 +1 @@ +SELECT toDateTime64('1925-01-02 03:04:05.678901', 6); From 60f125c2b5b9c002e12238ff73f80f6814448699 Mon Sep 17 00:00:00 2001 From: George Date: Mon, 15 Mar 2021 21:36:42 +0300 Subject: [PATCH 489/544] translation draft --- docs/ru/sql-reference/statements/attach.md | 5 +- docs/ru/sql-reference/statements/detach.md | 61 ++++++++++++++++++++-- 2 files changed, 59 insertions(+), 7 deletions(-) diff --git a/docs/ru/sql-reference/statements/attach.md b/docs/ru/sql-reference/statements/attach.md index 259ab893e63..be5b0b6d44a 100644 --- a/docs/ru/sql-reference/statements/attach.md +++ b/docs/ru/sql-reference/statements/attach.md @@ -10,14 +10,15 @@ toc_title: ATTACH - вместо слова `CREATE` используется слово `ATTACH`; - запрос не создаёт данные на диске, а предполагает, что данные уже лежат в соответствующих местах, и всего лишь добавляет информацию о таблице на сервер. После выполнения запроса `ATTACH` сервер будет знать о существовании таблицы. -Если таблица перед этим была отсоединена (`DETACH`), т.е. её структура известна, можно использовать сокращенную форму записи без определения структуры. +Если таблица перед этим была откреплена ([DETACH](../../sql-reference/statements/detach.md)), т.е. её структура известна, можно использовать сокращенную форму записи без определения структуры. ``` sql ATTACH TABLE [IF NOT EXISTS] [db.]name [ON CLUSTER cluster] ``` -Этот запрос используется при старте сервера. Сервер хранит метаданные таблиц в виде файлов с запросами `ATTACH`, которые он просто исполняет при запуске (за исключением системных таблиц, которые явно создаются на сервере). +Этот запрос используется при старте сервера. Сервер хранит метаданные таблиц в виде файлов с запросами `ATTACH`, которые он просто исполняет при запуске (за исключением некоторых системных таблиц, которые явно создаются на сервере). +Если таблица была откреплена перманентно, она не будет прикреплена обратно во время старта сервера, так что нужно явно использовать запрос `ATTACH`, чтобы прикрепить ее. [Оригинальная статья](https://clickhouse.tech/docs/ru/sql-reference/statements/attach/) diff --git a/docs/ru/sql-reference/statements/detach.md b/docs/ru/sql-reference/statements/detach.md index 00d0a4b20c6..bec8f4c5ff7 100644 --- a/docs/ru/sql-reference/statements/detach.md +++ b/docs/ru/sql-reference/statements/detach.md @@ -5,15 +5,66 @@ toc_title: DETACH # DETACH {#detach-statement} -Удаляет из сервера информацию о таблице name. Сервер перестаёт знать о существовании таблицы. +Удаляет из сервера информацию о таблице или материализованном представлении. Сервер перестаёт знать о существовании таблицы. + +Синтаксис: ``` sql -DETACH TABLE [IF EXISTS] [db.]name [ON CLUSTER cluster] +DETACH TABLE|VIEW [IF EXISTS] [db.]name [PERMANENTLY] [ON CLUSTER cluster] ``` -Но ни данные, ни метаданные таблицы не удаляются. При следующем запуске сервера, сервер прочитает метаданные и снова узнает о таблице. -Также, «отцепленную» таблицу можно прицепить заново запросом `ATTACH` (за исключением системных таблиц, для которых метаданные не хранятся). +Но ни данные, ни метаданные таблицы или материализованного представления не удаляются. При следующем запуске сервера, если не было использовано `PERMANENTLY`, сервер прочитает метаданные и снова узнает о таблице/представлении. Если таблица или представление были откреплено перманентно, сервер не прикрепит их обратно автоматически. -Запроса `DETACH DATABASE` нет. +Независимо от того, каким способом таблица была откреплена, ее можно прикрепить обратно с помощью запроса [ATTACH](../../sql-reference/statements/attach.md). Системные log таблицы также могут быть прикреплены обратно (к примеру `query_log`, `text_log` и др.) Другие системные таблицы не могут быть прикреплены обратно, но на следующем запуске сервер снова вспомнит об этих таблицах. + +`ATTACH MATERIALIZED VIEW` не может быть использован с кратким синтаксисом (без `SELECT`), но можно прикрепить представление с помощью запроса `ATTACH TABLE`. + +Обратите внимание, что нельзя перманентно открепить таблицу, которая уже временно откреплена. Для этого ее сначала надо прикрепить обратно, а затем снова открепить перманентно. + +Также нельзя использовать [DROP](../../sql-reference/statements/drop.md#drop-table) с открепленной таблицей или создавать таблицу с помощью [CREATE TABLE](../../sql-reference/statements/create/table.md) с таким же именем, как уже открепленная таблица. Еще нельзя заменить открепленную таблицу другой с помощью запроса [RENAME TABLE](../../sql-reference/statements/rename.md). + +**Пример** + +Создание таблицы: + +Запрос: + +``` sql +CREATE TABLE test ENGINE = Log AS SELECT * FROM numbers(10); +SELECT * FROM test; +``` + +Результат: + +``` text +┌─number─┐ +│ 0 │ +│ 1 │ +│ 2 │ +│ 3 │ +│ 4 │ +│ 5 │ +│ 6 │ +│ 7 │ +│ 8 │ +│ 9 │ +└────────┘ +``` + +Открепление таблицы: + +Запрос: + +``` sql +DETACH TABLE test; +SELECT * FROM test; +``` + +Результат: + +``` text +Received exception from server (version 21.4.1): +Code: 60. DB::Exception: Received from localhost:9000. DB::Exception: Table default.test doesn't exist. +``` [Оригинальная статья](https://clickhouse.tech/docs/ru/sql-reference/statements/detach/) From 671395e8c81e79f079334650112737eca6e3f6c7 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Mon, 15 Mar 2021 22:23:27 +0300 Subject: [PATCH 490/544] Most likely improve performance --- base/common/DateLUTImpl.h | 145 +++++++++++++----- base/common/LocalDate.h | 20 --- base/common/LocalDateTime.h | 57 ++----- programs/git-import/git-import.cpp | 2 +- .../PostgreSQLBlockInputStream.cpp | 15 +- .../ClickHouseDictionarySource.cpp | 5 +- src/Dictionaries/MySQLDictionarySource.cpp | 7 +- .../PostgreSQLDictionarySource.cpp | 5 +- src/Dictionaries/RedisBlockInputStream.cpp | 7 +- src/Dictionaries/XDBCDictionarySource.cpp | 5 +- src/Formats/MySQLBlockInputStream.cpp | 8 +- src/IO/WriteHelpers.h | 12 +- .../MergeTree/MergeTreeMutationEntry.cpp | 4 +- .../MergeTree/ReplicatedMergeTreeLogEntry.cpp | 4 +- .../ReplicatedMergeTreeMutationEntry.cpp | 4 +- src/Storages/StorageReplicatedMergeTree.cpp | 2 +- 16 files changed, 163 insertions(+), 139 deletions(-) diff --git a/base/common/DateLUTImpl.h b/base/common/DateLUTImpl.h index 867862ad51e..6bac9bd5126 100644 --- a/base/common/DateLUTImpl.h +++ b/base/common/DateLUTImpl.h @@ -944,16 +944,70 @@ public: } + struct DateComponents + { + uint16_t year; + uint8_t month; + uint8_t day; + }; + + struct TimeComponents + { + uint8_t hour; + uint8_t minute; + uint8_t second; + }; + + struct DateTimeComponents + { + DateComponents date; + TimeComponents time; + }; + + inline DateComponents toDateComponents(time_t t) const + { + const Values & values = getValues(t); + return { values.year, values.month, values.day_of_month }; + } + + inline DateTimeComponents toDateTimeComponents(time_t t) const + { + const LUTIndex index = findIndex(t); + const Values & values = lut[index]; + + DateTimeComponents res; + + res.date.year = values.year; + res.date.month = values.month; + res.date.day = values.day_of_month; + + time_t time = t - values.date; + if (time >= values.time_at_offset_change()) + time += values.amount_of_offset_change(); + + res.time.second = time % 60; + res.time.minute = time / 60 % 60; + res.time.hour = time / 3600; + + /// In case time was changed backwards at the start of next day, we will repeat the hour 23. + if (unlikely(res.time.hour > 23)) + res.time.hour = 23; + + return res; + } + + inline UInt64 toNumYYYYMMDDhhmmss(time_t t) const { - const Values & values = find(t); + DateTimeComponents components = toDateTimeComponents(t); + return - toSecond(t) - + toMinute(t) * 100 - + toHour(t) * 10000 - + UInt64(values.day_of_month) * 1000000 - + UInt64(values.month) * 100000000 - + UInt64(values.year) * 10000000000; + components.time.second + + components.time.minute * 100 + + components.time.hour * 10000 + + UInt64(components.date.day) * 1000000 + + UInt64(components.date.month) * 100000000 + + UInt64(components.date.year) * 10000000000; } inline time_t YYYYMMDDhhmmssToTime(UInt64 num) const @@ -972,16 +1026,19 @@ public: inline NO_SANITIZE_UNDEFINED time_t addDays(time_t t, Int64 delta) const { - LUTIndex index = findIndex(t); - time_t time_offset = toHour(t) * 3600 + toMinute(t) * 60 + toSecond(t); + const LUTIndex index = findIndex(t); + const Values & values = lut[index]; - index += delta; - index &= date_lut_mask; + time_t time = t - values.date; + if (time >= values.time_at_offset_change()) + time += values.amount_of_offset_change(); - if (time_offset >= lut[index].time_at_offset_change()) - time_offset -= lut[index].amount_of_offset_change(); + const LUTIndex new_index = index + delta; - return lut[index].date + time_offset; + if (time >= lut[new_index].time_at_offset_change()) + time -= lut[new_index].amount_of_offset_change(); + + return lut[new_index].date + time; } inline NO_SANITIZE_UNDEFINED time_t addWeeks(time_t t, Int64 delta) const @@ -1033,12 +1090,17 @@ public: { const auto result_day = addMonthsIndex(t, delta); - time_t time_offset = toHour(t) * 3600 + toMinute(t) * 60 + toSecond(t); + const LUTIndex index = findIndex(t); + const Values & values = lut[index]; - if (time_offset >= lut[result_day].time_at_offset_change()) - time_offset -= lut[result_day].amount_of_offset_change(); + time_t time = t - values.date; + if (time >= values.time_at_offset_change()) + time += values.amount_of_offset_change(); - return lut[result_day].date + time_offset; + if (time >= lut[result_day].time_at_offset_change()) + time -= lut[result_day].amount_of_offset_change(); + + return lut[result_day].date + time; } inline ExtendedDayNum NO_SANITIZE_UNDEFINED addMonths(ExtendedDayNum d, Int64 delta) const @@ -1077,12 +1139,17 @@ public: { auto result_day = addYearsIndex(t, delta); - time_t time_offset = toHour(t) * 3600 + toMinute(t) * 60 + toSecond(t); + const LUTIndex index = findIndex(t); + const Values & values = lut[index]; - if (time_offset >= lut[result_day].time_at_offset_change()) - time_offset -= lut[result_day].amount_of_offset_change(); + time_t time = t - values.date; + if (time >= values.time_at_offset_change()) + time += values.amount_of_offset_change(); - return lut[result_day].date + time_offset; + if (time >= lut[result_day].time_at_offset_change()) + time -= lut[result_day].amount_of_offset_change(); + + return lut[result_day].date + time; } inline ExtendedDayNum addYears(ExtendedDayNum d, Int64 delta) const @@ -1093,29 +1160,25 @@ public: inline std::string timeToString(time_t t) const { - const Values & values = getValues(t); + DateTimeComponents components = toDateTimeComponents(t); std::string s {"0000-00-00 00:00:00"}; - s[0] += values.year / 1000; - s[1] += (values.year / 100) % 10; - s[2] += (values.year / 10) % 10; - s[3] += values.year % 10; - s[5] += values.month / 10; - s[6] += values.month % 10; - s[8] += values.day_of_month / 10; - s[9] += values.day_of_month % 10; + s[0] += components.date.year / 1000; + s[1] += (components.date.year / 100) % 10; + s[2] += (components.date.year / 10) % 10; + s[3] += components.date.year % 10; + s[5] += components.date.month / 10; + s[6] += components.date.month % 10; + s[8] += components.date.day / 10; + s[9] += components.date.day % 10; - auto hour = toHour(t); - auto minute = toMinute(t); - auto second = toSecond(t); - - s[11] += hour / 10; - s[12] += hour % 10; - s[14] += minute / 10; - s[15] += minute % 10; - s[17] += second / 10; - s[18] += second % 10; + s[11] += components.time.hour / 10; + s[12] += components.time.hour % 10; + s[14] += components.time.minute / 10; + s[15] += components.time.minute % 10; + s[17] += components.time.second / 10; + s[18] += components.time.second % 10; return s; } diff --git a/base/common/LocalDate.h b/base/common/LocalDate.h index 7e1260c1385..b1e6eeb907c 100644 --- a/base/common/LocalDate.h +++ b/base/common/LocalDate.h @@ -92,17 +92,6 @@ public: LocalDate(const LocalDate &) noexcept = default; LocalDate & operator= (const LocalDate &) noexcept = default; - LocalDate & operator= (time_t time) - { - init(time); - return *this; - } - - operator time_t() const - { - return DateLUT::instance().makeDate(m_year, m_month, m_day); - } - DayNum getDayNum() const { const auto & lut = DateLUT::instance(); @@ -167,12 +156,3 @@ public: }; static_assert(sizeof(LocalDate) == 4); - - -namespace std -{ -inline string to_string(const LocalDate & date) -{ - return date.toString(); -} -} diff --git a/base/common/LocalDateTime.h b/base/common/LocalDateTime.h index 0e237789bd1..4c2cf0e637d 100644 --- a/base/common/LocalDateTime.h +++ b/base/common/LocalDateTime.h @@ -29,29 +29,16 @@ private: /// NOTE We may use attribute packed instead, but it is less portable. unsigned char pad = 0; - void init(time_t time) + void init(time_t time, const DateLUTImpl & time_zone) { - if (unlikely(time > DATE_LUT_MAX || time == 0)) - { - m_year = 0; - m_month = 0; - m_day = 0; - m_hour = 0; - m_minute = 0; - m_second = 0; + DateLUTImpl::DateTimeComponents components = time_zone.toDateTimeComponents(time); - return; - } - - const auto & date_lut = DateLUT::instance(); - const auto & values = date_lut.getValues(time); - - m_year = values.year; - m_month = values.month; - m_day = values.day_of_month; - m_hour = date_lut.toHour(time); - m_minute = date_lut.toMinute(time); - m_second = date_lut.toSecond(time); + m_year = components.date.year; + m_month = components.date.month; + m_day = components.date.day; + m_hour = components.time.hour; + m_minute = components.time.minute; + m_second = components.time.second; (void)pad; /// Suppress unused private field warning. } @@ -73,9 +60,9 @@ private: } public: - explicit LocalDateTime(time_t time) + explicit LocalDateTime(time_t time, const DateLUTImpl & time_zone = DateLUT::instance()) { - init(time); + init(time, time_zone); } LocalDateTime(unsigned short year_, unsigned char month_, unsigned char day_, @@ -104,19 +91,6 @@ public: LocalDateTime(const LocalDateTime &) noexcept = default; LocalDateTime & operator= (const LocalDateTime &) noexcept = default; - LocalDateTime & operator= (time_t time) - { - init(time); - return *this; - } - - operator time_t() const - { - return m_year == 0 - ? 0 - : DateLUT::instance().makeDateTime(m_year, m_month, m_day, m_hour, m_minute, m_second); - } - unsigned short year() const { return m_year; } unsigned char month() const { return m_month; } unsigned char day() const { return m_day; } @@ -167,14 +141,3 @@ public: }; static_assert(sizeof(LocalDateTime) == 8); - - -namespace std -{ -inline string to_string(const LocalDateTime & datetime) -{ - stringstream str; - str << datetime; - return str.str(); -} -} diff --git a/programs/git-import/git-import.cpp b/programs/git-import/git-import.cpp index ae8b55e2aff..b07435dcf78 100644 --- a/programs/git-import/git-import.cpp +++ b/programs/git-import/git-import.cpp @@ -1064,7 +1064,7 @@ void processCommit( time_t commit_time; readText(commit_time, in); - commit.time = commit_time; + commit.time = LocalDateTime(commit_time); assertChar('\0', in); readNullTerminated(commit.author, in); std::string parent_hash; diff --git a/src/DataStreams/PostgreSQLBlockInputStream.cpp b/src/DataStreams/PostgreSQLBlockInputStream.cpp index da6a83fb930..4cf2d942885 100644 --- a/src/DataStreams/PostgreSQLBlockInputStream.cpp +++ b/src/DataStreams/PostgreSQLBlockInputStream.cpp @@ -160,8 +160,13 @@ void PostgreSQLBlockInputStream::insertValue(IColumn & column, std::string_view assert_cast(column).insertValue(UInt16{LocalDate{std::string(value)}.getDayNum()}); break; case ValueType::vtDateTime: - assert_cast(column).insertValue(time_t{LocalDateTime{std::string(value)}}); + { + ReadBufferFromString in(value); + time_t time = 0; + readDateTimeText(time, in); + assert_cast(column).insertValue(time); break; + } case ValueType::vtDateTime64:[[fallthrough]]; case ValueType::vtDecimal32: [[fallthrough]]; case ValueType::vtDecimal64: [[fallthrough]]; @@ -257,7 +262,13 @@ void PostgreSQLBlockInputStream::prepareArrayInfo(size_t column_idx, const DataT else if (which.isDate()) parser = [](std::string & field) -> Field { return UInt16{LocalDate{field}.getDayNum()}; }; else if (which.isDateTime()) - parser = [](std::string & field) -> Field { return time_t{LocalDateTime{field}}; }; + parser = [](std::string & field) -> Field + { + ReadBufferFromString in(field); + time_t time = 0; + readDateTimeText(time, in); + return time; + }; else if (which.isDecimal32()) parser = [nested](std::string & field) -> Field { diff --git a/src/Dictionaries/ClickHouseDictionarySource.cpp b/src/Dictionaries/ClickHouseDictionarySource.cpp index f4c17884afa..5e69df2c8a0 100644 --- a/src/Dictionaries/ClickHouseDictionarySource.cpp +++ b/src/Dictionaries/ClickHouseDictionarySource.cpp @@ -118,10 +118,9 @@ std::string ClickHouseDictionarySource::getUpdateFieldAndDate() { if (update_time != std::chrono::system_clock::from_time_t(0)) { - auto tmp_time = update_time; + time_t hr_time = std::chrono::system_clock::to_time_t(update_time) - 1; + std::string str_time = DateLUT::instance().timeToString(hr_time); update_time = std::chrono::system_clock::now(); - time_t hr_time = std::chrono::system_clock::to_time_t(tmp_time) - 1; - std::string str_time = std::to_string(LocalDateTime(hr_time)); return query_builder.composeUpdateQuery(update_field, str_time); } else diff --git a/src/Dictionaries/MySQLDictionarySource.cpp b/src/Dictionaries/MySQLDictionarySource.cpp index 572080aee1e..a1d0cbe7ffc 100644 --- a/src/Dictionaries/MySQLDictionarySource.cpp +++ b/src/Dictionaries/MySQLDictionarySource.cpp @@ -107,10 +107,9 @@ std::string MySQLDictionarySource::getUpdateFieldAndDate() { if (update_time != std::chrono::system_clock::from_time_t(0)) { - auto tmp_time = update_time; + time_t hr_time = std::chrono::system_clock::to_time_t(update_time) - 1; + std::string str_time = DateLUT::instance().timeToString(hr_time); update_time = std::chrono::system_clock::now(); - time_t hr_time = std::chrono::system_clock::to_time_t(tmp_time) - 1; - std::string str_time = std::to_string(LocalDateTime(hr_time)); return query_builder.composeUpdateQuery(update_field, str_time); } else @@ -262,7 +261,7 @@ LocalDateTime MySQLDictionarySource::getLastModification(mysqlxx::Pool::Entry & if (!update_time_value.isNull()) { modification_time = update_time_value.getDateTime(); - LOG_TRACE(log, "Got modification time: {}", modification_time); + LOG_TRACE(log, "Got modification time: {}", update_time_value.getString()); } /// fetch remaining rows to avoid "commands out of sync" error diff --git a/src/Dictionaries/PostgreSQLDictionarySource.cpp b/src/Dictionaries/PostgreSQLDictionarySource.cpp index aa852404750..93a57383380 100644 --- a/src/Dictionaries/PostgreSQLDictionarySource.cpp +++ b/src/Dictionaries/PostgreSQLDictionarySource.cpp @@ -127,10 +127,9 @@ std::string PostgreSQLDictionarySource::getUpdateFieldAndDate() { if (update_time != std::chrono::system_clock::from_time_t(0)) { - auto tmp_time = update_time; + time_t hr_time = std::chrono::system_clock::to_time_t(update_time) - 1; + std::string str_time = DateLUT::instance().timeToString(hr_time); update_time = std::chrono::system_clock::now(); - time_t hr_time = std::chrono::system_clock::to_time_t(tmp_time) - 1; - std::string str_time = std::to_string(LocalDateTime(hr_time)); return query_builder.composeUpdateQuery(update_field, str_time); } else diff --git a/src/Dictionaries/RedisBlockInputStream.cpp b/src/Dictionaries/RedisBlockInputStream.cpp index a5514d14155..6a2072f7f6a 100644 --- a/src/Dictionaries/RedisBlockInputStream.cpp +++ b/src/Dictionaries/RedisBlockInputStream.cpp @@ -99,8 +99,13 @@ namespace DB assert_cast(column).insertValue(parse(string_value).getDayNum()); break; case ValueType::vtDateTime: - assert_cast(column).insertValue(static_cast(parse(string_value))); + { + ReadBufferFromString in(string_value); + time_t time = 0; + readDateTimeText(time, in); + assert_cast(column).insertValue(time); break; + } case ValueType::vtUUID: assert_cast(column).insertValue(parse(string_value)); break; diff --git a/src/Dictionaries/XDBCDictionarySource.cpp b/src/Dictionaries/XDBCDictionarySource.cpp index 3615f72605f..37d54f3549c 100644 --- a/src/Dictionaries/XDBCDictionarySource.cpp +++ b/src/Dictionaries/XDBCDictionarySource.cpp @@ -156,10 +156,9 @@ std::string XDBCDictionarySource::getUpdateFieldAndDate() { if (update_time != std::chrono::system_clock::from_time_t(0)) { - auto tmp_time = update_time; + time_t hr_time = std::chrono::system_clock::to_time_t(update_time) - 1; + std::string str_time = DateLUT::instance().timeToString(hr_time); update_time = std::chrono::system_clock::now(); - time_t hr_time = std::chrono::system_clock::to_time_t(tmp_time) - 1; - std::string str_time = std::to_string(LocalDateTime(hr_time)); return query_builder.composeUpdateQuery(update_field, str_time); } else diff --git a/src/Formats/MySQLBlockInputStream.cpp b/src/Formats/MySQLBlockInputStream.cpp index 87df0c1f4b1..bf7cf266e91 100644 --- a/src/Formats/MySQLBlockInputStream.cpp +++ b/src/Formats/MySQLBlockInputStream.cpp @@ -11,6 +11,7 @@ # include # include # include +# include # include # include # include @@ -97,8 +98,13 @@ namespace assert_cast(column).insertValue(UInt16(value.getDate().getDayNum())); break; case ValueType::vtDateTime: - assert_cast(column).insertValue(UInt32(value.getDateTime())); + { + ReadBufferFromString in(value); + time_t time = 0; + readDateTimeText(time, in); + assert_cast(column).insertValue(time); break; + } case ValueType::vtUUID: assert_cast(column).insert(parse(value.data(), value.size())); break; diff --git a/src/IO/WriteHelpers.h b/src/IO/WriteHelpers.h index 72c6b69114f..b9497b6f87e 100644 --- a/src/IO/WriteHelpers.h +++ b/src/IO/WriteHelpers.h @@ -821,24 +821,18 @@ inline void writeDateTimeText(const LocalDateTime & datetime, WriteBuffer & buf) template inline void writeDateTimeText(time_t datetime, WriteBuffer & buf, const DateLUTImpl & time_zone = DateLUT::instance()) { - const auto & values = time_zone.getValues(datetime); - writeDateTimeText( - LocalDateTime(values.year, values.month, values.day_of_month, - time_zone.toHour(datetime), time_zone.toMinute(datetime), time_zone.toSecond(datetime)), buf); + writeDateTimeText(LocalDateTime(datetime, time_zone), buf); } /// In the format YYYY-MM-DD HH:MM:SS.NNNNNNNNN, according to the specified time zone. template -inline void writeDateTimeText(DateTime64 datetime64, UInt32 scale, WriteBuffer & buf, const DateLUTImpl & date_lut = DateLUT::instance()) +inline void writeDateTimeText(DateTime64 datetime64, UInt32 scale, WriteBuffer & buf, const DateLUTImpl & time_zone = DateLUT::instance()) { static constexpr UInt32 MaxScale = DecimalUtils::max_precision; scale = scale > MaxScale ? MaxScale : scale; auto components = DecimalUtils::split(datetime64, scale); - const auto & values = date_lut.getValues(components.whole); - writeDateTimeText( - LocalDateTime(values.year, values.month, values.day_of_month, - date_lut.toHour(components.whole), date_lut.toMinute(components.whole), date_lut.toSecond(components.whole)), buf); + writeDateTimeText(LocalDateTime(components.whole, time_zone), buf); if (scale > 0) { diff --git a/src/Storages/MergeTree/MergeTreeMutationEntry.cpp b/src/Storages/MergeTree/MergeTreeMutationEntry.cpp index 44c4b3c4d10..49c4e93eb1d 100644 --- a/src/Storages/MergeTree/MergeTreeMutationEntry.cpp +++ b/src/Storages/MergeTree/MergeTreeMutationEntry.cpp @@ -75,7 +75,9 @@ MergeTreeMutationEntry::MergeTreeMutationEntry(DiskPtr disk_, const String & pat LocalDateTime create_time_dt; *buf >> "create time: " >> create_time_dt >> "\n"; - create_time = create_time_dt; + create_time = DateLUT::instance().makeDateTime( + create_time_dt.year(), create_time_dt.month(), create_time_dt.day(), + create_time_dt.hour(), create_time_dt.minute(), create_time_dt.second()); *buf >> "commands: "; commands.readText(*buf); diff --git a/src/Storages/MergeTree/ReplicatedMergeTreeLogEntry.cpp b/src/Storages/MergeTree/ReplicatedMergeTreeLogEntry.cpp index 9a9f25fd470..07c64d9c95c 100644 --- a/src/Storages/MergeTree/ReplicatedMergeTreeLogEntry.cpp +++ b/src/Storages/MergeTree/ReplicatedMergeTreeLogEntry.cpp @@ -162,7 +162,9 @@ void ReplicatedMergeTreeLogEntryData::readText(ReadBuffer & in) { LocalDateTime create_time_dt; in >> "create_time: " >> create_time_dt >> "\n"; - create_time = create_time_dt; + create_time = DateLUT::instance().makeDateTime( + create_time_dt.year(), create_time_dt.month(), create_time_dt.day(), + create_time_dt.hour(), create_time_dt.minute(), create_time_dt.second()); } in >> "source replica: " >> source_replica >> "\n"; diff --git a/src/Storages/MergeTree/ReplicatedMergeTreeMutationEntry.cpp b/src/Storages/MergeTree/ReplicatedMergeTreeMutationEntry.cpp index b2299b2cbbd..c617befe9c4 100644 --- a/src/Storages/MergeTree/ReplicatedMergeTreeMutationEntry.cpp +++ b/src/Storages/MergeTree/ReplicatedMergeTreeMutationEntry.cpp @@ -37,7 +37,9 @@ void ReplicatedMergeTreeMutationEntry::readText(ReadBuffer & in) LocalDateTime create_time_dt; in >> "create time: " >> create_time_dt >> "\n"; - create_time = create_time_dt; + create_time = DateLUT::instance().makeDateTime( + create_time_dt.year(), create_time_dt.month(), create_time_dt.day(), + create_time_dt.hour(), create_time_dt.minute(), create_time_dt.second()); in >> "source replica: " >> source_replica >> "\n"; diff --git a/src/Storages/StorageReplicatedMergeTree.cpp b/src/Storages/StorageReplicatedMergeTree.cpp index 3490e9918c5..a8de89a8bb0 100644 --- a/src/Storages/StorageReplicatedMergeTree.cpp +++ b/src/Storages/StorageReplicatedMergeTree.cpp @@ -1450,7 +1450,7 @@ bool StorageReplicatedMergeTree::tryExecuteMerge(const LogEntry & entry) { LOG_INFO(log, "Will try to fetch part {} until '{}' because this part assigned to recompression merge. " "Source replica {} will try to merge this part first", entry.new_part_name, - LocalDateTime(entry.create_time + storage_settings_ptr->try_fetch_recompressed_part_timeout.totalSeconds()), entry.source_replica); + DateLUT::instance().timeToString(entry.create_time + storage_settings_ptr->try_fetch_recompressed_part_timeout.totalSeconds()), entry.source_replica); return false; } From f48bf2aaba9f38a02a830991922e1cd97fe9c0f5 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Mon, 15 Mar 2021 23:29:35 +0300 Subject: [PATCH 491/544] Fix issue --- base/common/DateLUTImpl.h | 15 ++++++++++++--- ...1702_toDateTime_from_string_clamping.reference | 2 +- 2 files changed, 13 insertions(+), 4 deletions(-) diff --git a/base/common/DateLUTImpl.h b/base/common/DateLUTImpl.h index 6bac9bd5126..98b767ccbcc 100644 --- a/base/common/DateLUTImpl.h +++ b/base/common/DateLUTImpl.h @@ -985,9 +985,18 @@ public: if (time >= values.time_at_offset_change()) time += values.amount_of_offset_change(); - res.time.second = time % 60; - res.time.minute = time / 60 % 60; - res.time.hour = time / 3600; + if (unlikely(time < 0)) + { + res.time.second = 0; + res.time.minute = 0; + res.time.hour = 0; + } + else + { + res.time.second = time % 60; + res.time.minute = time / 60 % 60; + res.time.hour = time / 3600; + } /// In case time was changed backwards at the start of next day, we will repeat the hour 23. if (unlikely(res.time.hour > 23)) diff --git a/tests/queries/0_stateless/01702_toDateTime_from_string_clamping.reference b/tests/queries/0_stateless/01702_toDateTime_from_string_clamping.reference index 77da114be68..7e8307d66a6 100644 --- a/tests/queries/0_stateless/01702_toDateTime_from_string_clamping.reference +++ b/tests/queries/0_stateless/01702_toDateTime_from_string_clamping.reference @@ -6,4 +6,4 @@ SELECT toString(toDateTime('9922337203.6854775808', 1)); SELECT toDateTime64(CAST('10000000000.1' AS Decimal64(1)), 1); 2283-11-11 23:46:40.1 SELECT toDateTime64(CAST('-10000000000.1' AS Decimal64(1)), 1); -1925-01-01 23:09:20.1 +1925-01-01 00:00:00.1 From 31d7f10a94c5c9505c8ca11b3cee311a313e9948 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Mon, 15 Mar 2021 23:34:42 +0300 Subject: [PATCH 492/544] Remove bad code --- src/DataTypes/DataTypeDateTime.cpp | 25 +++++++++++++++---------- 1 file changed, 15 insertions(+), 10 deletions(-) diff --git a/src/DataTypes/DataTypeDateTime.cpp b/src/DataTypes/DataTypeDateTime.cpp index d2bbb4a1efa..d8ae7f688ae 100644 --- a/src/DataTypes/DataTypeDateTime.cpp +++ b/src/DataTypes/DataTypeDateTime.cpp @@ -12,10 +12,14 @@ #include #include +namespace DB +{ + namespace { -using namespace DB; -inline void readText(time_t & x, ReadBuffer & istr, const FormatSettings & settings, const DateLUTImpl & time_zone, const DateLUTImpl & utc_time_zone) + +inline void readTextHelper( + time_t & x, ReadBuffer & istr, const FormatSettings & settings, const DateLUTImpl & time_zone, const DateLUTImpl & utc_time_zone) { switch (settings.date_time_input_format) { @@ -27,16 +31,16 @@ inline void readText(time_t & x, ReadBuffer & istr, const FormatSettings & setti return; } } + } -namespace DB -{ TimezoneMixin::TimezoneMixin(const String & time_zone_name) : has_explicit_time_zone(!time_zone_name.empty()), time_zone(DateLUT::instance(time_zone_name)), utc_time_zone(DateLUT::instance("UTC")) -{} +{ +} DataTypeDateTime::DataTypeDateTime(const String & time_zone_name) : TimezoneMixin(time_zone_name) @@ -45,7 +49,8 @@ DataTypeDateTime::DataTypeDateTime(const String & time_zone_name) DataTypeDateTime::DataTypeDateTime(const TimezoneMixin & time_zone_) : TimezoneMixin(time_zone_) -{} +{ +} String DataTypeDateTime::doGetName() const { @@ -87,7 +92,7 @@ void DataTypeDateTime::deserializeWholeText(IColumn & column, ReadBuffer & istr, void DataTypeDateTime::deserializeTextEscaped(IColumn & column, ReadBuffer & istr, const FormatSettings & settings) const { time_t x; - ::readText(x, istr, settings, time_zone, utc_time_zone); + readTextHelper(x, istr, settings, time_zone, utc_time_zone); assert_cast(column).getData().push_back(x); } @@ -103,7 +108,7 @@ void DataTypeDateTime::deserializeTextQuoted(IColumn & column, ReadBuffer & istr time_t x; if (checkChar('\'', istr)) /// Cases: '2017-08-31 18:36:48' or '1504193808' { - ::readText(x, istr, settings, time_zone, utc_time_zone); + readTextHelper(x, istr, settings, time_zone, utc_time_zone); assertChar('\'', istr); } else /// Just 1504193808 or 01504193808 @@ -125,7 +130,7 @@ void DataTypeDateTime::deserializeTextJSON(IColumn & column, ReadBuffer & istr, time_t x; if (checkChar('"', istr)) { - ::readText(x, istr, settings, time_zone, utc_time_zone); + readTextHelper(x, istr, settings, time_zone, utc_time_zone); assertChar('"', istr); } else @@ -154,7 +159,7 @@ void DataTypeDateTime::deserializeTextCSV(IColumn & column, ReadBuffer & istr, c if (maybe_quote == '\'' || maybe_quote == '\"') ++istr.position(); - ::readText(x, istr, settings, time_zone, utc_time_zone); + readTextHelper(x, istr, settings, time_zone, utc_time_zone); if (maybe_quote == '\'' || maybe_quote == '\"') assertChar(maybe_quote, istr); From 3f67f4f47b79e101b8fc70e7501a5bc07c0a3c03 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Mon, 15 Mar 2021 23:40:33 +0300 Subject: [PATCH 493/544] Saturation for DateTime --- src/DataStreams/PostgreSQLBlockInputStream.cpp | 2 ++ src/DataTypes/DataTypeDateTime.cpp | 11 +++++++++++ src/Dictionaries/RedisBlockInputStream.cpp | 2 ++ src/Formats/MySQLBlockInputStream.cpp | 2 ++ src/Formats/ProtobufSerializer.cpp | 2 ++ src/Functions/FunctionsConversion.h | 8 +++++--- 6 files changed, 24 insertions(+), 3 deletions(-) diff --git a/src/DataStreams/PostgreSQLBlockInputStream.cpp b/src/DataStreams/PostgreSQLBlockInputStream.cpp index 4cf2d942885..8350dc86849 100644 --- a/src/DataStreams/PostgreSQLBlockInputStream.cpp +++ b/src/DataStreams/PostgreSQLBlockInputStream.cpp @@ -164,6 +164,8 @@ void PostgreSQLBlockInputStream::insertValue(IColumn & column, std::string_view ReadBufferFromString in(value); time_t time = 0; readDateTimeText(time, in); + if (time < 0) + time = 0; assert_cast(column).insertValue(time); break; } diff --git a/src/DataTypes/DataTypeDateTime.cpp b/src/DataTypes/DataTypeDateTime.cpp index d8ae7f688ae..09dcb5f3e2e 100644 --- a/src/DataTypes/DataTypeDateTime.cpp +++ b/src/DataTypes/DataTypeDateTime.cpp @@ -93,6 +93,8 @@ void DataTypeDateTime::deserializeTextEscaped(IColumn & column, ReadBuffer & ist { time_t x; readTextHelper(x, istr, settings, time_zone, utc_time_zone); + if (x < 0) + x = 0; assert_cast(column).getData().push_back(x); } @@ -115,6 +117,8 @@ void DataTypeDateTime::deserializeTextQuoted(IColumn & column, ReadBuffer & istr { readIntText(x, istr); } + if (x < 0) + x = 0; assert_cast(column).getData().push_back(x); /// It's important to do this at the end - for exception safety. } @@ -137,6 +141,10 @@ void DataTypeDateTime::deserializeTextJSON(IColumn & column, ReadBuffer & istr, { readIntText(x, istr); } + + if (x < 0) + x = 0; + assert_cast(column).getData().push_back(x); } @@ -164,6 +172,9 @@ void DataTypeDateTime::deserializeTextCSV(IColumn & column, ReadBuffer & istr, c if (maybe_quote == '\'' || maybe_quote == '\"') assertChar(maybe_quote, istr); + if (x < 0) + x = 0; + assert_cast(column).getData().push_back(x); } diff --git a/src/Dictionaries/RedisBlockInputStream.cpp b/src/Dictionaries/RedisBlockInputStream.cpp index 6a2072f7f6a..39cc2d610df 100644 --- a/src/Dictionaries/RedisBlockInputStream.cpp +++ b/src/Dictionaries/RedisBlockInputStream.cpp @@ -103,6 +103,8 @@ namespace DB ReadBufferFromString in(string_value); time_t time = 0; readDateTimeText(time, in); + if (time < 0) + time = 0; assert_cast(column).insertValue(time); break; } diff --git a/src/Formats/MySQLBlockInputStream.cpp b/src/Formats/MySQLBlockInputStream.cpp index bf7cf266e91..29cf749de3b 100644 --- a/src/Formats/MySQLBlockInputStream.cpp +++ b/src/Formats/MySQLBlockInputStream.cpp @@ -102,6 +102,8 @@ namespace ReadBufferFromString in(value); time_t time = 0; readDateTimeText(time, in); + if (time < 0) + time = 0; assert_cast(column).insertValue(time); break; } diff --git a/src/Formats/ProtobufSerializer.cpp b/src/Formats/ProtobufSerializer.cpp index 2ab8e11a854..f99888975bc 100644 --- a/src/Formats/ProtobufSerializer.cpp +++ b/src/Formats/ProtobufSerializer.cpp @@ -1486,6 +1486,8 @@ namespace ReadBufferFromString buf{str}; time_t tm = 0; readDateTimeText(tm, buf); + if (tm < 0) + tm = 0; return tm; } diff --git a/src/Functions/FunctionsConversion.h b/src/Functions/FunctionsConversion.h index fb3c2787c96..ef74f7778b1 100644 --- a/src/Functions/FunctionsConversion.h +++ b/src/Functions/FunctionsConversion.h @@ -769,9 +769,11 @@ inline void parseImpl(DataTypeDate::FieldType & x, ReadBuffer & rb template <> inline void parseImpl(DataTypeDateTime::FieldType & x, ReadBuffer & rb, const DateLUTImpl * time_zone) { - time_t tmp = 0; - readDateTimeText(tmp, rb, *time_zone); - x = tmp; + time_t time = 0; + readDateTimeText(time, rb, *time_zone); + if (time < 0) + time = 0; + x = time; } From c1a2745bfbdf54906ede3411c02b6f320107d052 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Tue, 16 Mar 2021 00:04:03 +0300 Subject: [PATCH 494/544] Add one more variant to memcpy benchmark --- base/glibc-compatibility/memcpy/memcpy.h | 2 +- utils/memcpy-bench/memcpy-bench.cpp | 172 ++++++++++++++++++++++- 2 files changed, 172 insertions(+), 2 deletions(-) diff --git a/base/glibc-compatibility/memcpy/memcpy.h b/base/glibc-compatibility/memcpy/memcpy.h index f9f81bcb0fe..211d144cecb 100644 --- a/base/glibc-compatibility/memcpy/memcpy.h +++ b/base/glibc-compatibility/memcpy/memcpy.h @@ -178,7 +178,7 @@ tail: size -= padding; } - /// Aligned unrolled copy. We will use all available SSE registers. + /// Aligned unrolled copy. We will use half of available SSE registers. /// It's not possible to have both src and dst aligned. /// So, we will use aligned stores and unaligned loads. __m128i c0, c1, c2, c3, c4, c5, c6, c7; diff --git a/utils/memcpy-bench/memcpy-bench.cpp b/utils/memcpy-bench/memcpy-bench.cpp index cd769640017..5c664a76fe2 100644 --- a/utils/memcpy-bench/memcpy-bench.cpp +++ b/utils/memcpy-bench/memcpy-bench.cpp @@ -35,7 +35,7 @@ void NO_INLINE loop(uint8_t * dst, uint8_t * src, size_t size, F && chunk_size_d size -= bytes_to_copy; /// Execute at least one SSE instruction as a penalty after running AVX code. - __asm__ volatile ("pxor %%xmm7, %%xmm7" ::: "xmm7"); + __asm__ __volatile__ ("pxor %%xmm15, %%xmm15" ::: "xmm15"); } } @@ -385,6 +385,7 @@ void memcpy_my_medium_avx(uint8_t * __restrict & __restrict dst, const uint8_t * bool have_avx = true; + static uint8_t * memcpy_my(uint8_t * __restrict dst, const uint8_t * __restrict src, size_t size) { uint8_t * ret = dst; @@ -560,6 +561,174 @@ tail: return ret; } + +static uint8_t * memcpy_my2(uint8_t * __restrict dst, const uint8_t * __restrict src, size_t size) +{ + uint8_t * ret = dst; + + if (size <= 16) + { + if (size >= 8) + { + __builtin_memcpy(dst + size - 8, src + size - 8, 8); + __builtin_memcpy(dst, src, 8); + } + else if (size >= 4) + { + __builtin_memcpy(dst + size - 4, src + size - 4, 4); + __builtin_memcpy(dst, src, 4); + } + else if (size >= 2) + { + __builtin_memcpy(dst + size - 2, src + size - 2, 2); + __builtin_memcpy(dst, src, 2); + } + else if (size >= 1) + { + *dst = *src; + } + } + else + { + if (size <= 128) + { + _mm_storeu_si128(reinterpret_cast<__m128i *>(dst + size - 16), _mm_loadu_si128(reinterpret_cast(src + size - 16))); + + while (size > 16) + { + _mm_storeu_si128(reinterpret_cast<__m128i *>(dst), _mm_loadu_si128(reinterpret_cast(src))); + dst += 16; + src += 16; + size -= 16; + } + } + else if (size < 30000 || !have_avx) + { + /// Align destination to 16 bytes boundary. + size_t padding = (16 - (reinterpret_cast(dst) & 15)) & 15; + + if (padding > 0) + { + __m128i head = _mm_loadu_si128(reinterpret_cast(src)); + _mm_storeu_si128(reinterpret_cast<__m128i*>(dst), head); + dst += padding; + src += padding; + size -= padding; + } + + /// Aligned unrolled copy. + __m128i c0, c1, c2, c3, c4, c5, c6, c7; + + while (size >= 128) + { + c0 = _mm_loadu_si128(reinterpret_cast(src) + 0); + c1 = _mm_loadu_si128(reinterpret_cast(src) + 1); + c2 = _mm_loadu_si128(reinterpret_cast(src) + 2); + c3 = _mm_loadu_si128(reinterpret_cast(src) + 3); + c4 = _mm_loadu_si128(reinterpret_cast(src) + 4); + c5 = _mm_loadu_si128(reinterpret_cast(src) + 5); + c6 = _mm_loadu_si128(reinterpret_cast(src) + 6); + c7 = _mm_loadu_si128(reinterpret_cast(src) + 7); + src += 128; + _mm_store_si128((reinterpret_cast<__m128i*>(dst) + 0), c0); + _mm_store_si128((reinterpret_cast<__m128i*>(dst) + 1), c1); + _mm_store_si128((reinterpret_cast<__m128i*>(dst) + 2), c2); + _mm_store_si128((reinterpret_cast<__m128i*>(dst) + 3), c3); + _mm_store_si128((reinterpret_cast<__m128i*>(dst) + 4), c4); + _mm_store_si128((reinterpret_cast<__m128i*>(dst) + 5), c5); + _mm_store_si128((reinterpret_cast<__m128i*>(dst) + 6), c6); + _mm_store_si128((reinterpret_cast<__m128i*>(dst) + 7), c7); + dst += 128; + + size -= 128; + } + + _mm_storeu_si128(reinterpret_cast<__m128i *>(dst + size - 16), _mm_loadu_si128(reinterpret_cast(src + size - 16))); + + while (size > 16) + { + _mm_storeu_si128(reinterpret_cast<__m128i *>(dst), _mm_loadu_si128(reinterpret_cast(src))); + dst += 16; + src += 16; + size -= 16; + } + } + else + { + size_t padding = (32 - (reinterpret_cast(dst) & 31)) & 31; + + if (padding > 0) + { + __asm__( + "vmovups (%[s]), %%ymm0\n" + "vmovups %%ymm0, (%[d])\n" + : [d]"+r"(dst), [s]"+r"(src) + : + : "ymm0", "memory"); + + dst += padding; + src += padding; + size -= padding; + } + + while (size >= 256) + { + __asm__( + "vmovups (%[s]), %%ymm0\n" + "vmovups 0x20(%[s]), %%ymm1\n" + "vmovups 0x40(%[s]), %%ymm2\n" + "vmovups 0x60(%[s]), %%ymm3\n" + "vmovups 0x80(%[s]), %%ymm4\n" + "vmovups 0xa0(%[s]), %%ymm5\n" + "vmovups 0xc0(%[s]), %%ymm6\n" + "vmovups 0xe0(%[s]), %%ymm7\n" + "add $0x100,%[s]\n" + "vmovaps %%ymm0, (%[d])\n" + "vmovaps %%ymm1, 0x20(%[d])\n" + "vmovaps %%ymm2, 0x40(%[d])\n" + "vmovaps %%ymm3, 0x60(%[d])\n" + "vmovaps %%ymm4, 0x80(%[d])\n" + "vmovaps %%ymm5, 0xa0(%[d])\n" + "vmovaps %%ymm6, 0xc0(%[d])\n" + "vmovaps %%ymm7, 0xe0(%[d])\n" + "add $0x100, %[d]\n" + : [d]"+r"(dst), [s]"+r"(src) + : + : "ymm0", "ymm1", "ymm2", "ymm3", "ymm4", "ymm5", "ymm6", "ymm7", "memory"); + + size -= 256; + } + + __asm__( + "vmovups -0x20(%[s],%[size],1), %%ymm0\n" + "vmovups %%ymm0, -0x20(%[d],%[size],1)\n" + : [d]"+r"(dst), [s]"+r"(src) + : [size]"r"(size) + : "ymm0", "memory"); + + while (size > 32) + { + __asm__( + "vmovups (%[s]), %%ymm0\n" + "vmovups %%ymm0, (%[d])\n" + : [d]"+r"(dst), [s]"+r"(src) + : + : "ymm0", "memory"); + + dst += 32; + src += 32; + size -= 32; + } + + __asm__ __volatile__ ("vzeroupper" + ::: "ymm0", "ymm1", "ymm2", "ymm3", "ymm4", "ymm5", "ymm6", "ymm7", + "ymm8", "ymm9", "ymm10", "ymm11", "ymm12", "ymm13", "ymm14", "ymm15"); + } + } + + return ret; +} + extern "C" void * __memcpy_erms(void * __restrict destination, const void * __restrict source, size_t size); extern "C" void * __memcpy_sse2_unaligned(void * __restrict destination, const void * __restrict source, size_t size); extern "C" void * __memcpy_ssse3(void * __restrict destination, const void * __restrict source, size_t size); @@ -592,6 +761,7 @@ uint64_t dispatchMemcpyVariants(size_t memcpy_variant, uint8_t * dst, uint8_t * VARIANT(10, memcpy_fast_sse) VARIANT(11, memcpy_fast_avx) VARIANT(12, memcpy_my) + VARIANT(13, memcpy_my2) VARIANT(21, __memcpy_erms) VARIANT(22, __memcpy_sse2_unaligned) From b9ae9c9cd2f55e494a2dd0dfa181e6406a2be936 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Tue, 16 Mar 2021 00:05:30 +0300 Subject: [PATCH 495/544] Add one more variant to memcpy benchmark --- utils/memcpy-bench/memcpy-bench.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/utils/memcpy-bench/memcpy-bench.cpp b/utils/memcpy-bench/memcpy-bench.cpp index 5c664a76fe2..704ea04a184 100644 --- a/utils/memcpy-bench/memcpy-bench.cpp +++ b/utils/memcpy-bench/memcpy-bench.cpp @@ -816,7 +816,7 @@ int main(int argc, char ** argv) for size in 4096 16384 50000 65536 100000 1000000 10000000 100000000; do for threads in 1 2 4 $(($(nproc) / 2)) $(nproc); do for distribution in 1 2 3 4 5; do - for variant in {1..12} {21..29}; do + for variant in {1..13} {21..29}; do for i in {1..10}; do ./memcpy-bench --tsv --size $size --variant $variant --threads $threads --distribution $distribution; done; From fe371d9cfceb5b97f324565351d6f75ced34fbc1 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Tue, 16 Mar 2021 01:13:27 +0300 Subject: [PATCH 496/544] Minor modifications --- utils/memcpy-bench/memcpy-bench.cpp | 320 +++++++++++++++++----------- 1 file changed, 191 insertions(+), 129 deletions(-) diff --git a/utils/memcpy-bench/memcpy-bench.cpp b/utils/memcpy-bench/memcpy-bench.cpp index 704ea04a184..b607c45370d 100644 --- a/utils/memcpy-bench/memcpy-bench.cpp +++ b/utils/memcpy-bench/memcpy-bench.cpp @@ -588,142 +588,203 @@ static uint8_t * memcpy_my2(uint8_t * __restrict dst, const uint8_t * __restrict *dst = *src; } } + else if (size <= 128) + { + _mm_storeu_si128(reinterpret_cast<__m128i *>(dst + size - 16), _mm_loadu_si128(reinterpret_cast(src + size - 16))); + + while (size > 16) + { + _mm_storeu_si128(reinterpret_cast<__m128i *>(dst), _mm_loadu_si128(reinterpret_cast(src))); + dst += 16; + src += 16; + size -= 16; + } + } + else if (size < 30000 || !have_avx) + { + /// Align destination to 16 bytes boundary. + size_t padding = (16 - (reinterpret_cast(dst) & 15)) & 15; + + if (padding > 0) + { + __m128i head = _mm_loadu_si128(reinterpret_cast(src)); + _mm_storeu_si128(reinterpret_cast<__m128i*>(dst), head); + dst += padding; + src += padding; + size -= padding; + } + + /// Aligned unrolled copy. + __m128i c0, c1, c2, c3, c4, c5, c6, c7; + + while (size >= 128) + { + c0 = _mm_loadu_si128(reinterpret_cast(src) + 0); + c1 = _mm_loadu_si128(reinterpret_cast(src) + 1); + c2 = _mm_loadu_si128(reinterpret_cast(src) + 2); + c3 = _mm_loadu_si128(reinterpret_cast(src) + 3); + c4 = _mm_loadu_si128(reinterpret_cast(src) + 4); + c5 = _mm_loadu_si128(reinterpret_cast(src) + 5); + c6 = _mm_loadu_si128(reinterpret_cast(src) + 6); + c7 = _mm_loadu_si128(reinterpret_cast(src) + 7); + src += 128; + _mm_store_si128((reinterpret_cast<__m128i*>(dst) + 0), c0); + _mm_store_si128((reinterpret_cast<__m128i*>(dst) + 1), c1); + _mm_store_si128((reinterpret_cast<__m128i*>(dst) + 2), c2); + _mm_store_si128((reinterpret_cast<__m128i*>(dst) + 3), c3); + _mm_store_si128((reinterpret_cast<__m128i*>(dst) + 4), c4); + _mm_store_si128((reinterpret_cast<__m128i*>(dst) + 5), c5); + _mm_store_si128((reinterpret_cast<__m128i*>(dst) + 6), c6); + _mm_store_si128((reinterpret_cast<__m128i*>(dst) + 7), c7); + dst += 128; + + size -= 128; + } + + _mm_storeu_si128(reinterpret_cast<__m128i *>(dst + size - 16), _mm_loadu_si128(reinterpret_cast(src + size - 16))); + + while (size > 16) + { + _mm_storeu_si128(reinterpret_cast<__m128i *>(dst), _mm_loadu_si128(reinterpret_cast(src))); + dst += 16; + src += 16; + size -= 16; + } + } else { - if (size <= 128) + size_t padding = (32 - (reinterpret_cast(dst) & 31)) & 31; + + if (padding > 0) { - _mm_storeu_si128(reinterpret_cast<__m128i *>(dst + size - 16), _mm_loadu_si128(reinterpret_cast(src + size - 16))); - - while (size > 16) - { - _mm_storeu_si128(reinterpret_cast<__m128i *>(dst), _mm_loadu_si128(reinterpret_cast(src))); - dst += 16; - src += 16; - size -= 16; - } - } - else if (size < 30000 || !have_avx) - { - /// Align destination to 16 bytes boundary. - size_t padding = (16 - (reinterpret_cast(dst) & 15)) & 15; - - if (padding > 0) - { - __m128i head = _mm_loadu_si128(reinterpret_cast(src)); - _mm_storeu_si128(reinterpret_cast<__m128i*>(dst), head); - dst += padding; - src += padding; - size -= padding; - } - - /// Aligned unrolled copy. - __m128i c0, c1, c2, c3, c4, c5, c6, c7; - - while (size >= 128) - { - c0 = _mm_loadu_si128(reinterpret_cast(src) + 0); - c1 = _mm_loadu_si128(reinterpret_cast(src) + 1); - c2 = _mm_loadu_si128(reinterpret_cast(src) + 2); - c3 = _mm_loadu_si128(reinterpret_cast(src) + 3); - c4 = _mm_loadu_si128(reinterpret_cast(src) + 4); - c5 = _mm_loadu_si128(reinterpret_cast(src) + 5); - c6 = _mm_loadu_si128(reinterpret_cast(src) + 6); - c7 = _mm_loadu_si128(reinterpret_cast(src) + 7); - src += 128; - _mm_store_si128((reinterpret_cast<__m128i*>(dst) + 0), c0); - _mm_store_si128((reinterpret_cast<__m128i*>(dst) + 1), c1); - _mm_store_si128((reinterpret_cast<__m128i*>(dst) + 2), c2); - _mm_store_si128((reinterpret_cast<__m128i*>(dst) + 3), c3); - _mm_store_si128((reinterpret_cast<__m128i*>(dst) + 4), c4); - _mm_store_si128((reinterpret_cast<__m128i*>(dst) + 5), c5); - _mm_store_si128((reinterpret_cast<__m128i*>(dst) + 6), c6); - _mm_store_si128((reinterpret_cast<__m128i*>(dst) + 7), c7); - dst += 128; - - size -= 128; - } - - _mm_storeu_si128(reinterpret_cast<__m128i *>(dst + size - 16), _mm_loadu_si128(reinterpret_cast(src + size - 16))); - - while (size > 16) - { - _mm_storeu_si128(reinterpret_cast<__m128i *>(dst), _mm_loadu_si128(reinterpret_cast(src))); - dst += 16; - src += 16; - size -= 16; - } - } - else - { - size_t padding = (32 - (reinterpret_cast(dst) & 31)) & 31; - - if (padding > 0) - { - __asm__( - "vmovups (%[s]), %%ymm0\n" - "vmovups %%ymm0, (%[d])\n" - : [d]"+r"(dst), [s]"+r"(src) - : - : "ymm0", "memory"); - - dst += padding; - src += padding; - size -= padding; - } - - while (size >= 256) - { - __asm__( - "vmovups (%[s]), %%ymm0\n" - "vmovups 0x20(%[s]), %%ymm1\n" - "vmovups 0x40(%[s]), %%ymm2\n" - "vmovups 0x60(%[s]), %%ymm3\n" - "vmovups 0x80(%[s]), %%ymm4\n" - "vmovups 0xa0(%[s]), %%ymm5\n" - "vmovups 0xc0(%[s]), %%ymm6\n" - "vmovups 0xe0(%[s]), %%ymm7\n" - "add $0x100,%[s]\n" - "vmovaps %%ymm0, (%[d])\n" - "vmovaps %%ymm1, 0x20(%[d])\n" - "vmovaps %%ymm2, 0x40(%[d])\n" - "vmovaps %%ymm3, 0x60(%[d])\n" - "vmovaps %%ymm4, 0x80(%[d])\n" - "vmovaps %%ymm5, 0xa0(%[d])\n" - "vmovaps %%ymm6, 0xc0(%[d])\n" - "vmovaps %%ymm7, 0xe0(%[d])\n" - "add $0x100, %[d]\n" - : [d]"+r"(dst), [s]"+r"(src) - : - : "ymm0", "ymm1", "ymm2", "ymm3", "ymm4", "ymm5", "ymm6", "ymm7", "memory"); - - size -= 256; - } - __asm__( - "vmovups -0x20(%[s],%[size],1), %%ymm0\n" - "vmovups %%ymm0, -0x20(%[d],%[size],1)\n" + "vmovups (%[s]), %%ymm0\n" + "vmovups %%ymm0, (%[d])\n" : [d]"+r"(dst), [s]"+r"(src) - : [size]"r"(size) + : : "ymm0", "memory"); - while (size > 32) - { - __asm__( - "vmovups (%[s]), %%ymm0\n" - "vmovups %%ymm0, (%[d])\n" - : [d]"+r"(dst), [s]"+r"(src) - : - : "ymm0", "memory"); - - dst += 32; - src += 32; - size -= 32; - } - - __asm__ __volatile__ ("vzeroupper" - ::: "ymm0", "ymm1", "ymm2", "ymm3", "ymm4", "ymm5", "ymm6", "ymm7", - "ymm8", "ymm9", "ymm10", "ymm11", "ymm12", "ymm13", "ymm14", "ymm15"); + dst += padding; + src += padding; + size -= padding; } + + while (size >= 512) + { + __asm__( + "vmovups (%[s]), %%ymm0\n" + "vmovups 0x20(%[s]), %%ymm1\n" + "vmovups 0x40(%[s]), %%ymm2\n" + "vmovups 0x60(%[s]), %%ymm3\n" + "vmovups 0x80(%[s]), %%ymm4\n" + "vmovups 0xa0(%[s]), %%ymm5\n" + "vmovups 0xc0(%[s]), %%ymm6\n" + "vmovups 0xe0(%[s]), %%ymm7\n" + "vmovups 0x100(%[s]), %%ymm8\n" + "vmovups 0x120(%[s]), %%ymm9\n" + "vmovups 0x140(%[s]), %%ymm10\n" + "vmovups 0x160(%[s]), %%ymm11\n" + "vmovups 0x180(%[s]), %%ymm12\n" + "vmovups 0x1a0(%[s]), %%ymm13\n" + "vmovups 0x1c0(%[s]), %%ymm14\n" + "vmovups 0x1e0(%[s]), %%ymm15\n" + "add $0x200, %[s]\n" + "sub $0x200, %[size]\n" + "vmovaps %%ymm0, (%[d])\n" + "vmovaps %%ymm1, 0x20(%[d])\n" + "vmovaps %%ymm2, 0x40(%[d])\n" + "vmovaps %%ymm3, 0x60(%[d])\n" + "vmovaps %%ymm4, 0x80(%[d])\n" + "vmovaps %%ymm5, 0xa0(%[d])\n" + "vmovaps %%ymm6, 0xc0(%[d])\n" + "vmovaps %%ymm7, 0xe0(%[d])\n" + "vmovaps %%ymm8, 0x100(%[d])\n" + "vmovaps %%ymm9, 0x120(%[d])\n" + "vmovaps %%ymm10, 0x140(%[d])\n" + "vmovaps %%ymm11, 0x160(%[d])\n" + "vmovaps %%ymm12, 0x180(%[d])\n" + "vmovaps %%ymm13, 0x1a0(%[d])\n" + "vmovaps %%ymm14, 0x1c0(%[d])\n" + "vmovaps %%ymm15, 0x1e0(%[d])\n" + "add $0x200, %[d]\n" + : [d]"+r"(dst), [s]"+r"(src), [size]"+r"(size) + : + : "ymm0", "ymm1", "ymm2", "ymm3", "ymm4", "ymm5", "ymm6", "ymm7", + "ymm8", "ymm9", "ymm10", "ymm11", "ymm12", "ymm13", "ymm14", "ymm15", + "memory"); + } + + /*while (size >= 256) + { + __asm__( + "vmovups (%[s]), %%ymm0\n" + "vmovups 0x20(%[s]), %%ymm1\n" + "vmovups 0x40(%[s]), %%ymm2\n" + "vmovups 0x60(%[s]), %%ymm3\n" + "vmovups 0x80(%[s]), %%ymm4\n" + "vmovups 0xa0(%[s]), %%ymm5\n" + "vmovups 0xc0(%[s]), %%ymm6\n" + "vmovups 0xe0(%[s]), %%ymm7\n" + "add $0x100,%[s]\n" + "vmovaps %%ymm0, (%[d])\n" + "vmovaps %%ymm1, 0x20(%[d])\n" + "vmovaps %%ymm2, 0x40(%[d])\n" + "vmovaps %%ymm3, 0x60(%[d])\n" + "vmovaps %%ymm4, 0x80(%[d])\n" + "vmovaps %%ymm5, 0xa0(%[d])\n" + "vmovaps %%ymm6, 0xc0(%[d])\n" + "vmovaps %%ymm7, 0xe0(%[d])\n" + "add $0x100, %[d]\n" + : [d]"+r"(dst), [s]"+r"(src) + : + : "ymm0", "ymm1", "ymm2", "ymm3", "ymm4", "ymm5", "ymm6", "ymm7", "memory"); + + size -= 256; + }*/ + + /*while (size > 128) + { + __asm__( + "vmovups (%[s]), %%ymm0\n" + "vmovups 0x20(%[s]), %%ymm1\n" + "vmovups 0x40(%[s]), %%ymm2\n" + "vmovups 0x60(%[s]), %%ymm3\n" + "add $0x80, %[s]\n" + "sub $0x80, %[size]\n" + "vmovaps %%ymm0, (%[d])\n" + "vmovaps %%ymm1, 0x20(%[d])\n" + "vmovaps %%ymm2, 0x40(%[d])\n" + "vmovaps %%ymm3, 0x60(%[d])\n" + "add $0x80, %[d]\n" + : [d]"+r"(dst), [s]"+r"(src), [size]"+r"(size) + : + : "ymm0", "ymm1", "ymm2", "ymm3", "memory"); + }*/ + + __asm__( + "vmovups -0x20(%[s],%[size],1), %%ymm0\n" + "vmovups %%ymm0, -0x20(%[d],%[size],1)\n" + : [d]"+r"(dst), [s]"+r"(src) + : [size]"r"(size) + : "ymm0", "memory"); + + while (size > 32) + { + __asm__( + "vmovups (%[s]), %%ymm0\n" + "vmovups %%ymm0, (%[d])\n" + : [d]"+r"(dst), [s]"+r"(src) + : + : "ymm0", "memory"); + + dst += 32; + src += 32; + size -= 32; + } + + __asm__ __volatile__ ("vzeroupper" + ::: "ymm0", "ymm1", "ymm2", "ymm3", "ymm4", "ymm5", "ymm6", "ymm7", + "ymm8", "ymm9", "ymm10", "ymm11", "ymm12", "ymm13", "ymm14", "ymm15"); } return ret; @@ -893,7 +954,8 @@ clickhouse-local --structure ' } else { - std::cout << ": processed in " << (elapsed_ns / 1e9) << " sec, " << (size * iterations * 1.0 / elapsed_ns) << " GB/sec\n"; + std::cout << ": " << num_threads << " threads, " << "size: " << size << ", distribution " << generator_variant + << ", processed in " << (elapsed_ns / 1e9) << " sec, " << (size * iterations * 1.0 / elapsed_ns) << " GB/sec\n"; } return 0; From 54f124aae499d55779632d0f0a2641a4ec41a0c0 Mon Sep 17 00:00:00 2001 From: Azat Khuzhin Date: Mon, 15 Mar 2021 07:55:59 +0300 Subject: [PATCH 497/544] Cleanup 00626_replace_partition_from_table_zookeeper --- .../00626_replace_partition_from_table_zookeeper.sh | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/tests/queries/0_stateless/00626_replace_partition_from_table_zookeeper.sh b/tests/queries/0_stateless/00626_replace_partition_from_table_zookeeper.sh index 5aa445858db..f5f667d084b 100755 --- a/tests/queries/0_stateless/00626_replace_partition_from_table_zookeeper.sh +++ b/tests/queries/0_stateless/00626_replace_partition_from_table_zookeeper.sh @@ -12,19 +12,22 @@ CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) function query_with_retry { - retry=0 + local query="$1" && shift + + local retry=0 until [ $retry -ge 5 ] do - result=$($CLICKHOUSE_CLIENT $2 --query="$1" 2>&1) + local result + result="$($CLICKHOUSE_CLIENT "$@" --query="$query" 2>&1)" if [ "$?" == 0 ]; then echo -n "$result" return else - retry=$(($retry + 1)) + retry=$((retry + 1)) sleep 3 fi done - echo "Query '$1' failed with '$result'" + echo "Query '$query' failed with '$result'" } $CLICKHOUSE_CLIENT --query="DROP TABLE IF EXISTS src;" @@ -139,7 +142,7 @@ $CLICKHOUSE_CLIENT --query="DROP TABLE src;" $CLICKHOUSE_CLIENT --query="SELECT count(), sum(d), uniqExact(_part) FROM dst_r1;" $CLICKHOUSE_CLIENT --query="SYSTEM SYNC REPLICA dst_r1;" -query_with_retry "OPTIMIZE TABLE dst_r1 PARTITION 1;" "--replication_alter_partitions_sync=0 --optimize_throw_if_noop=1" +query_with_retry "OPTIMIZE TABLE dst_r1 PARTITION 1;" --replication_alter_partitions_sync=0 --optimize_throw_if_noop=1 $CLICKHOUSE_CLIENT --query="SYSTEM SYNC REPLICA dst_r1;" $CLICKHOUSE_CLIENT --query="SELECT count(), sum(d), uniqExact(_part) FROM dst_r1;" From e6b9740a6957ecf94ce92090e233d8441147c7d7 Mon Sep 17 00:00:00 2001 From: Azat Khuzhin Date: Mon, 15 Mar 2021 08:29:05 +0300 Subject: [PATCH 498/544] Add $CLICKHOUSE_TEST_ZOOKEEPER_PREFIX (for *.sh stateless tests) --- tests/queries/shell_config.sh | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/tests/queries/shell_config.sh b/tests/queries/shell_config.sh index d20b5669cc5..5b942a95d02 100644 --- a/tests/queries/shell_config.sh +++ b/tests/queries/shell_config.sh @@ -5,6 +5,13 @@ export ASAN_OPTIONS=detect_odr_violation=0 export CLICKHOUSE_DATABASE=${CLICKHOUSE_DATABASE:="test"} export CLICKHOUSE_CLIENT_SERVER_LOGS_LEVEL=${CLICKHOUSE_CLIENT_SERVER_LOGS_LEVEL:="warning"} + +# Unique zookeeper path (based on test name and current database) to avoid overlaps +export CLICKHOUSE_TEST_PATH="${BASH_SOURCE[1]}" +CLICKHOUSE_TEST_NAME="$(basename "$CLICKHOUSE_TEST_PATH" .sh)" +export CLICKHOUSE_TEST_NAME +export CLICKHOUSE_TEST_ZOOKEEPER_PREFIX="${CLICKHOUSE_TEST_NAME}_${CLICKHOUSE_DATABASE}" + [ -v CLICKHOUSE_CONFIG_CLIENT ] && CLICKHOUSE_CLIENT_OPT0+=" --config-file=${CLICKHOUSE_CONFIG_CLIENT} " [ -v CLICKHOUSE_HOST ] && CLICKHOUSE_CLIENT_OPT0+=" --host=${CLICKHOUSE_HOST} " [ -v CLICKHOUSE_PORT_TCP ] && CLICKHOUSE_CLIENT_OPT0+=" --port=${CLICKHOUSE_PORT_TCP} " From 6c2622d4519fee3207426582c8cdafd85b204b1b Mon Sep 17 00:00:00 2001 From: Azat Khuzhin Date: Mon, 15 Mar 2021 08:14:28 +0300 Subject: [PATCH 499/544] Add style check for ReplicatedMergeTree path --- utils/check-style/check-style | 30 ++++++++++++++++++++++++++++++ 1 file changed, 30 insertions(+) diff --git a/utils/check-style/check-style b/utils/check-style/check-style index f8926a9af2f..db6b33a569b 100755 --- a/utils/check-style/check-style +++ b/utils/check-style/check-style @@ -97,6 +97,36 @@ for test_case in "${tests_with_query_log[@]}"; do grep -qE current_database.*currentDatabase "$test_case" || echo "Queries to system.query_log/system.query_thread_log does not have current_database = currentDatabase() condition in $test_case" done +# Queries with ReplicatedMergeTree +# NOTE: it is not that accuate, but at least something. +tests_with_replicated_merge_tree=( $( + find $ROOT_PATH/tests/queries -iname '*.sql' -or -iname '*.sh' -or -iname '*.py' | + grep -vP $EXCLUDE_DIRS | + xargs grep --with-filename -e ReplicatedMergeTree | cut -d: -f1 | sort -u +) ) +for test_case in "${tests_with_replicated_merge_tree[@]}"; do + case "$test_case" in + *.sh) + test_case_zk_prefix="\$CLICKHOUSE_TEST_ZOOKEEPER_PREFIX" + grep -q -e "ReplicatedMergeTree.*$test_case_zk_prefix" "$test_case" || echo "ReplicatedMergeTree should contain '$test_case_zk_prefix' in zookeeper path to avoid overlaps ($test_case)" + ;; + *.sql) + # NOTE: *.sql is not supported because it is not possible right now, because: + # - ReplicatedMergeTree supports only ASTLiteral for zookeeper path + # (and adding support of other nodes, with evaluating them are not that easy, due to zk_prefix is "optional") + # - Hence concat(currentDatabase(), 'foo') + # - Also params cannot be used, because the are wrapped with CAST() + # + # But hopefully they will not be a problem + # (since they do not do any "stressing" and overlap probability should be lower). + ;; + *.py) + # Right now there is not such tests anyway + echo "No ReplicatedMergeTree style check for *.py ($test_case)" + ;; + esac +done + # All the submodules should be from https://github.com/ find $ROOT_PATH -name '.gitmodules' | while read i; do grep -F 'url = ' $i | grep -v -F 'https://github.com/' && echo 'All the submodules should be from https://github.com/'; done From 05a8c73eb948ad2e225af3e69ba7c493d2ae4b8a Mon Sep 17 00:00:00 2001 From: Azat Khuzhin Date: Mon, 15 Mar 2021 07:51:28 +0300 Subject: [PATCH 500/544] Do not overlap zookeeper path for ReplicatedMergeTree in stateless *.sh tests Found with stress tests for 00626_replace_partition_from_table_zookeeper [1]: 2021.03.15 00:59:48.200106 [ 27417 ] {0f47dbeb-938a-4560-8408-a7cc7b3bafb8} ContextAccess (default): Access granted: CREATE TABLE ON test_31.dst_r1 ... 2021.03.15 00:59:48.403227 [ 27417 ] {0f47dbeb-938a-4560-8408-a7cc7b3bafb8} test_31.dst_r1: This table /clickhouse/test_00626/dst_1 is already created, will add new replica 2021.03.15 00:59:48.736450 [ 83006 ] {b2db1355-3ec3-4e3a-9c79-f93f27c6e658} ContextAccess (default): Access granted: CREATE TABLE ON test_31.dst_r2 ... 2021.03.15 00:59:48.851768 [ 83006 ] {b2db1355-3ec3-4e3a-9c79-f93f27c6e658} test_31.dst_r2: This table /clickhouse/test_00626/dst_1 is already created, will add new replica ... 2021.03.15 00:59:48.919059 [ 366 ] {} test_31.dst_r2 (ReplicatedMergeTreeQueue): Loading queue from /clickhouse/test_00626/dst_1/replicas/2/queue 2021.03.15 00:59:48.919948 [ 366 ] {} test_31.dst_r2 (ReplicatedMergeTreeQueue): Having 3 queue entries to load, 0 entries already loaded. 2021.03.15 00:59:48.921833 [ 366 ] {} test_31.dst_r2 (ReplicatedMergeTreeQueue): Loaded queue ... 2021.03.15 00:59:51.904230 [ 246952 ] {59753eea-3896-45ca-8625-fdaa094ee9ef} ContextAccess (default): Access granted: SYSTEM SYNC REPLICA ON test_31.dst_r2 ... 2021.03.15 01:04:51.913683 [ 246952 ] {59753eea-3896-45ca-8625-fdaa094ee9ef} InterpreterSystemQuery: SYNC REPLICA test_31.dst_r2: Timed out! [1]: https://clickhouse-test-reports.s3.yandex.net/21716/402bf77783cbda48a9ee1b748bfce3c52ef8fe11/stress_test_(memory)/test_run.txt.out.log But the problem is more generic, so fix all tests. --- .../00029_test_zookeeper_optimize_exception.sh | 3 +-- ...ear_column_in_partition_concurrent_zookeeper.sh | 4 ++-- ...00626_replace_partition_from_table_zookeeper.sh | 4 ++-- ...terialized_view_and_too_many_parts_zookeeper.sh | 10 +++++----- ...licated_mutations_default_database_zookeeper.sh | 2 +- .../00652_replicated_mutations_zookeeper.sh | 8 ++++---- ...00715_fetch_merged_or_mutated_part_zookeeper.sh | 4 ++-- .../00834_kill_mutation_replicated_zookeeper.sh | 4 ++-- .../00953_zookeeper_suetin_deduplication_bug.sh | 14 +++++++------- .../00975_indices_mutation_replicated_zookeeper.sh | 4 ++-- .../00992_system_parts_race_condition_zookeeper.sh | 4 ++-- ...3_system_parts_race_condition_drop_zookeeper.sh | 2 +- .../01013_sync_replica_timeout_zookeeper.sh | 5 ++--- ...ns_with_nondeterministic_functions_zookeeper.sh | 5 ++--- .../01034_move_partition_from_table_zookeeper.sh | 12 ++++++------ ...ncurrent_move_partition_from_table_zookeeper.sh | 4 ++-- ..._zookeeper_system_mutations_with_parts_names.sh | 2 +- .../01076_parallel_alter_replicated_zookeeper.sh | 2 +- .../01079_bad_alters_zookeeper.reference | 4 ++-- .../0_stateless/01079_bad_alters_zookeeper.sh | 2 +- ...079_parallel_alter_add_drop_column_zookeeper.sh | 2 +- .../01079_parallel_alter_detach_table_zookeeper.sh | 2 +- .../01079_parallel_alter_modify_zookeeper.sh | 2 +- .../01103_optimize_drop_race_zookeeper.sh | 2 +- ...8_restart_replicas_rename_deadlock_zookeeper.sh | 2 +- .../0_stateless/01192_rename_database_zookeeper.sh | 2 +- .../01213_alter_rename_column_zookeeper.reference | 4 ++-- .../01213_alter_rename_column_zookeeper.sh | 2 +- .../01305_replica_create_drop_zookeeper.sh | 4 ++-- .../01307_multiple_leaders_zookeeper.sh | 2 +- .../01318_long_unsuccessful_mutation_zookeeper.sh | 2 +- .../01320_create_sync_race_condition_zookeeper.sh | 2 +- ...01338_long_select_and_alter_zookeeper.reference | 2 +- .../01338_long_select_and_alter_zookeeper.sh | 2 +- ...396_inactive_replica_cleanup_nodes_zookeeper.sh | 12 ++++++------ .../01414_mutations_and_errors_zookeeper.sh | 2 +- .../01417_freeze_partition_verbose_zookeeper.sh | 2 +- .../0_stateless/01459_manual_write_to_replicas.sh | 2 +- .../01459_manual_write_to_replicas_quorum.sh | 2 +- .../01508_race_condition_rename_clear_zookeeper.sh | 2 +- .../01509_check_many_parallel_quorum_inserts.sh | 2 +- .../01509_check_parallel_quorum_inserts.sh | 2 +- .../0_stateless/01509_parallel_quorum_and_merge.sh | 4 ++-- ...01593_concurrent_alter_mutations_kill.reference | 2 +- .../01593_concurrent_alter_mutations_kill.sh | 2 +- ...nt_alter_mutations_kill_many_replicas.reference | 10 +++++----- ...oncurrent_alter_mutations_kill_many_replicas.sh | 6 +++--- .../queries/0_stateless/01671_ddl_hang_timeout.sh | 2 +- .../01753_system_zookeeper_query_param_path.sh | 4 ++-- 49 files changed, 92 insertions(+), 95 deletions(-) diff --git a/tests/queries/0_stateless/00029_test_zookeeper_optimize_exception.sh b/tests/queries/0_stateless/00029_test_zookeeper_optimize_exception.sh index 86f1d1f161c..3360b8da83d 100755 --- a/tests/queries/0_stateless/00029_test_zookeeper_optimize_exception.sh +++ b/tests/queries/0_stateless/00029_test_zookeeper_optimize_exception.sh @@ -4,12 +4,11 @@ CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) # shellcheck source=../shell_config.sh . "$CUR_DIR"/../shell_config.sh - ${CLICKHOUSE_CLIENT} --query="DROP TABLE IF EXISTS test_optimize_exception" ${CLICKHOUSE_CLIENT} --query="DROP TABLE IF EXISTS test_optimize_exception_replicated" ${CLICKHOUSE_CLIENT} --query="CREATE TABLE test_optimize_exception (date Date) ENGINE=MergeTree() PARTITION BY toYYYYMM(date) ORDER BY date" -${CLICKHOUSE_CLIENT} --query="CREATE TABLE test_optimize_exception_replicated (date Date) ENGINE=ReplicatedMergeTree('/clickhouse/tables/test_00029/optimize', 'r1') PARTITION BY toYYYYMM(date) ORDER BY date" +${CLICKHOUSE_CLIENT} --query="CREATE TABLE test_optimize_exception_replicated (date Date) ENGINE=ReplicatedMergeTree('/clickhouse/tables/$CLICKHOUSE_TEST_ZOOKEEPER_PREFIX/optimize', 'r1') PARTITION BY toYYYYMM(date) ORDER BY date" ${CLICKHOUSE_CLIENT} --query="INSERT INTO test_optimize_exception VALUES (toDate('2017-09-09')), (toDate('2017-09-10'))" ${CLICKHOUSE_CLIENT} --query="INSERT INTO test_optimize_exception VALUES (toDate('2017-09-09')), (toDate('2017-09-10'))" diff --git a/tests/queries/0_stateless/00446_clear_column_in_partition_concurrent_zookeeper.sh b/tests/queries/0_stateless/00446_clear_column_in_partition_concurrent_zookeeper.sh index 60de1822318..5c5ecd4564b 100755 --- a/tests/queries/0_stateless/00446_clear_column_in_partition_concurrent_zookeeper.sh +++ b/tests/queries/0_stateless/00446_clear_column_in_partition_concurrent_zookeeper.sh @@ -8,8 +8,8 @@ ch="$CLICKHOUSE_CLIENT --stacktrace -q" $ch "DROP TABLE IF EXISTS clear_column1" $ch "DROP TABLE IF EXISTS clear_column2" -$ch "CREATE TABLE clear_column1 (d Date, i Int64, s String) ENGINE = ReplicatedMergeTree('/clickhouse/test_00446/tables/clear_column_concurrent', '1', d, d, 8192)" -$ch "CREATE TABLE clear_column2 (d Date, i Int64, s String) ENGINE = ReplicatedMergeTree('/clickhouse/test_00446/tables/clear_column_concurrent', '2', d, d, 8192)" +$ch "CREATE TABLE clear_column1 (d Date, i Int64, s String) ENGINE = ReplicatedMergeTree('/clickhouse/$CLICKHOUSE_TEST_ZOOKEEPER_PREFIX/tables/clear_column_concurrent', '1', d, d, 8192)" +$ch "CREATE TABLE clear_column2 (d Date, i Int64, s String) ENGINE = ReplicatedMergeTree('/clickhouse/$CLICKHOUSE_TEST_ZOOKEEPER_PREFIX/tables/clear_column_concurrent', '2', d, d, 8192)" $ch "ALTER TABLE clear_column1 CLEAR COLUMN VasyaUnexistingColumn IN PARTITION '200001'" --replication_alter_partitions_sync=2 1>/dev/null 2>/dev/null rc=$? diff --git a/tests/queries/0_stateless/00626_replace_partition_from_table_zookeeper.sh b/tests/queries/0_stateless/00626_replace_partition_from_table_zookeeper.sh index f5f667d084b..443f2856c88 100755 --- a/tests/queries/0_stateless/00626_replace_partition_from_table_zookeeper.sh +++ b/tests/queries/0_stateless/00626_replace_partition_from_table_zookeeper.sh @@ -35,8 +35,8 @@ $CLICKHOUSE_CLIENT --query="DROP TABLE IF EXISTS dst_r1;" $CLICKHOUSE_CLIENT --query="DROP TABLE IF EXISTS dst_r2;" $CLICKHOUSE_CLIENT --query="CREATE TABLE src (p UInt64, k String, d UInt64) ENGINE = MergeTree PARTITION BY p ORDER BY k;" -$CLICKHOUSE_CLIENT --query="CREATE TABLE dst_r1 (p UInt64, k String, d UInt64) ENGINE = ReplicatedMergeTree('/clickhouse/test_00626/dst_1', '1') PARTITION BY p ORDER BY k SETTINGS old_parts_lifetime=1, cleanup_delay_period=1, cleanup_delay_period_random_add=0;" -$CLICKHOUSE_CLIENT --query="CREATE TABLE dst_r2 (p UInt64, k String, d UInt64) ENGINE = ReplicatedMergeTree('/clickhouse/test_00626/dst_1', '2') PARTITION BY p ORDER BY k SETTINGS old_parts_lifetime=1, cleanup_delay_period=1, cleanup_delay_period_random_add=0;" +$CLICKHOUSE_CLIENT --query="CREATE TABLE dst_r1 (p UInt64, k String, d UInt64) ENGINE = ReplicatedMergeTree('/clickhouse/$CLICKHOUSE_TEST_ZOOKEEPER_PREFIX/dst_1', '1') PARTITION BY p ORDER BY k SETTINGS old_parts_lifetime=1, cleanup_delay_period=1, cleanup_delay_period_random_add=0;" +$CLICKHOUSE_CLIENT --query="CREATE TABLE dst_r2 (p UInt64, k String, d UInt64) ENGINE = ReplicatedMergeTree('/clickhouse/$CLICKHOUSE_TEST_ZOOKEEPER_PREFIX/dst_1', '2') PARTITION BY p ORDER BY k SETTINGS old_parts_lifetime=1, cleanup_delay_period=1, cleanup_delay_period_random_add=0;" $CLICKHOUSE_CLIENT --query="INSERT INTO src VALUES (0, '0', 1);" $CLICKHOUSE_CLIENT --query="INSERT INTO src VALUES (1, '0', 1);" diff --git a/tests/queries/0_stateless/00633_materialized_view_and_too_many_parts_zookeeper.sh b/tests/queries/0_stateless/00633_materialized_view_and_too_many_parts_zookeeper.sh index 817da08bfa0..def8e8f4cfe 100755 --- a/tests/queries/0_stateless/00633_materialized_view_and_too_many_parts_zookeeper.sh +++ b/tests/queries/0_stateless/00633_materialized_view_and_too_many_parts_zookeeper.sh @@ -10,10 +10,10 @@ ${CLICKHOUSE_CLIENT} --query "DROP TABLE IF EXISTS a" ${CLICKHOUSE_CLIENT} --query "DROP TABLE IF EXISTS b" ${CLICKHOUSE_CLIENT} --query "DROP TABLE IF EXISTS c" -${CLICKHOUSE_CLIENT} --query "CREATE TABLE root (d UInt64) ENGINE = ReplicatedMergeTree('/clickhouse/test_00633/root', '1') ORDER BY d" -${CLICKHOUSE_CLIENT} --query "CREATE MATERIALIZED VIEW a (d UInt64) ENGINE = ReplicatedMergeTree('/clickhouse/test_00633/a', '1') ORDER BY d AS SELECT * FROM root" -${CLICKHOUSE_CLIENT} --query "CREATE MATERIALIZED VIEW b (d UInt64) ENGINE = ReplicatedMergeTree('/clickhouse/test_00633/b', '1') ORDER BY d SETTINGS parts_to_delay_insert=1, parts_to_throw_insert=1 AS SELECT * FROM root" -${CLICKHOUSE_CLIENT} --query "CREATE MATERIALIZED VIEW c (d UInt64) ENGINE = ReplicatedMergeTree('/clickhouse/test_00633/c', '1') ORDER BY d AS SELECT * FROM root" +${CLICKHOUSE_CLIENT} --query "CREATE TABLE root (d UInt64) ENGINE = ReplicatedMergeTree('/clickhouse/$CLICKHOUSE_TEST_ZOOKEEPER_PREFIX/root', '1') ORDER BY d" +${CLICKHOUSE_CLIENT} --query "CREATE MATERIALIZED VIEW a (d UInt64) ENGINE = ReplicatedMergeTree('/clickhouse/$CLICKHOUSE_TEST_ZOOKEEPER_PREFIX/a', '1') ORDER BY d AS SELECT * FROM root" +${CLICKHOUSE_CLIENT} --query "CREATE MATERIALIZED VIEW b (d UInt64) ENGINE = ReplicatedMergeTree('/clickhouse/$CLICKHOUSE_TEST_ZOOKEEPER_PREFIX/b', '1') ORDER BY d SETTINGS parts_to_delay_insert=1, parts_to_throw_insert=1 AS SELECT * FROM root" +${CLICKHOUSE_CLIENT} --query "CREATE MATERIALIZED VIEW c (d UInt64) ENGINE = ReplicatedMergeTree('/clickhouse/$CLICKHOUSE_TEST_ZOOKEEPER_PREFIX/c', '1') ORDER BY d AS SELECT * FROM root" ${CLICKHOUSE_CLIENT} --query "INSERT INTO root VALUES (1)"; ${CLICKHOUSE_CLIENT} --query "SELECT _table, d FROM merge('${CLICKHOUSE_DATABASE}', '^[abc]\$') ORDER BY _table" @@ -33,7 +33,7 @@ ${CLICKHOUSE_CLIENT} --query "DROP TABLE c" # Deduplication check for non-replicated root table echo ${CLICKHOUSE_CLIENT} --query "CREATE TABLE root (d UInt64) ENGINE = Null" -${CLICKHOUSE_CLIENT} --query "CREATE MATERIALIZED VIEW d (d UInt64) ENGINE = ReplicatedMergeTree('/clickhouse/test_00633/d', '1') ORDER BY d AS SELECT * FROM root" +${CLICKHOUSE_CLIENT} --query "CREATE MATERIALIZED VIEW d (d UInt64) ENGINE = ReplicatedMergeTree('/clickhouse/$CLICKHOUSE_TEST_ZOOKEEPER_PREFIX/d', '1') ORDER BY d AS SELECT * FROM root" ${CLICKHOUSE_CLIENT} --query "INSERT INTO root VALUES (1)"; ${CLICKHOUSE_CLIENT} --query "INSERT INTO root VALUES (1)"; ${CLICKHOUSE_CLIENT} --query "SELECT * FROM d"; diff --git a/tests/queries/0_stateless/00652_replicated_mutations_default_database_zookeeper.sh b/tests/queries/0_stateless/00652_replicated_mutations_default_database_zookeeper.sh index 02f552c250d..58295e17790 100755 --- a/tests/queries/0_stateless/00652_replicated_mutations_default_database_zookeeper.sh +++ b/tests/queries/0_stateless/00652_replicated_mutations_default_database_zookeeper.sh @@ -11,7 +11,7 @@ ${CLICKHOUSE_CLIENT} --multiquery << EOF DROP TABLE IF EXISTS mutations_r1; DROP TABLE IF EXISTS for_subquery; -CREATE TABLE mutations_r1(x UInt32, y UInt32) ENGINE ReplicatedMergeTree('/clickhouse/tables/${CLICKHOUSE_DATABASE}/mutations', 'r1') ORDER BY x; +CREATE TABLE mutations_r1(x UInt32, y UInt32) ENGINE ReplicatedMergeTree('/clickhouse/tables/$CLICKHOUSE_TEST_ZOOKEEPER_PREFIX/mutations', 'r1') ORDER BY x; INSERT INTO mutations_r1 VALUES (123, 1), (234, 2), (345, 3); CREATE TABLE for_subquery(x UInt32) ENGINE TinyLog; diff --git a/tests/queries/0_stateless/00652_replicated_mutations_zookeeper.sh b/tests/queries/0_stateless/00652_replicated_mutations_zookeeper.sh index 08a39c58c3e..3ec6e4e3e90 100755 --- a/tests/queries/0_stateless/00652_replicated_mutations_zookeeper.sh +++ b/tests/queries/0_stateless/00652_replicated_mutations_zookeeper.sh @@ -10,8 +10,8 @@ CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) ${CLICKHOUSE_CLIENT} --query="DROP TABLE IF EXISTS mutations_r1" ${CLICKHOUSE_CLIENT} --query="DROP TABLE IF EXISTS mutations_r2" -${CLICKHOUSE_CLIENT} --query="CREATE TABLE mutations_r1(d Date, x UInt32, s String, m MATERIALIZED x + 2) ENGINE ReplicatedMergeTree('/clickhouse/tables/test_00652/mutations', 'r1', d, intDiv(x, 10), 8192)" -${CLICKHOUSE_CLIENT} --query="CREATE TABLE mutations_r2(d Date, x UInt32, s String, m MATERIALIZED x + 2) ENGINE ReplicatedMergeTree('/clickhouse/tables/test_00652/mutations', 'r2', d, intDiv(x, 10), 8192)" +${CLICKHOUSE_CLIENT} --query="CREATE TABLE mutations_r1(d Date, x UInt32, s String, m MATERIALIZED x + 2) ENGINE ReplicatedMergeTree('/clickhouse/tables/$CLICKHOUSE_TEST_ZOOKEEPER_PREFIX/mutations', 'r1', d, intDiv(x, 10), 8192)" +${CLICKHOUSE_CLIENT} --query="CREATE TABLE mutations_r2(d Date, x UInt32, s String, m MATERIALIZED x + 2) ENGINE ReplicatedMergeTree('/clickhouse/tables/$CLICKHOUSE_TEST_ZOOKEEPER_PREFIX/mutations', 'r2', d, intDiv(x, 10), 8192)" # Test a mutation on empty table ${CLICKHOUSE_CLIENT} --query="ALTER TABLE mutations_r1 DELETE WHERE x = 1 SETTINGS mutations_sync = 2" @@ -51,11 +51,11 @@ ${CLICKHOUSE_CLIENT} --query="DROP TABLE IF EXISTS mutations_cleaner_r1" ${CLICKHOUSE_CLIENT} --query="DROP TABLE IF EXISTS mutations_cleaner_r2" # Create 2 replicas with finished_mutations_to_keep = 2 -${CLICKHOUSE_CLIENT} --query="CREATE TABLE mutations_cleaner_r1(x UInt32) ENGINE ReplicatedMergeTree('/clickhouse/tables/test_00652/mutations_cleaner', 'r1') ORDER BY x SETTINGS \ +${CLICKHOUSE_CLIENT} --query="CREATE TABLE mutations_cleaner_r1(x UInt32) ENGINE ReplicatedMergeTree('/clickhouse/tables/$CLICKHOUSE_TEST_ZOOKEEPER_PREFIX/mutations_cleaner', 'r1') ORDER BY x SETTINGS \ finished_mutations_to_keep = 2, cleanup_delay_period = 1, cleanup_delay_period_random_add = 0" -${CLICKHOUSE_CLIENT} --query="CREATE TABLE mutations_cleaner_r2(x UInt32) ENGINE ReplicatedMergeTree('/clickhouse/tables/test_00652/mutations_cleaner', 'r2') ORDER BY x SETTINGS \ +${CLICKHOUSE_CLIENT} --query="CREATE TABLE mutations_cleaner_r2(x UInt32) ENGINE ReplicatedMergeTree('/clickhouse/tables/$CLICKHOUSE_TEST_ZOOKEEPER_PREFIX/mutations_cleaner', 'r2') ORDER BY x SETTINGS \ finished_mutations_to_keep = 2, cleanup_delay_period = 1, cleanup_delay_period_random_add = 0" diff --git a/tests/queries/0_stateless/00715_fetch_merged_or_mutated_part_zookeeper.sh b/tests/queries/0_stateless/00715_fetch_merged_or_mutated_part_zookeeper.sh index 54b6c80f2ac..48833d2643c 100755 --- a/tests/queries/0_stateless/00715_fetch_merged_or_mutated_part_zookeeper.sh +++ b/tests/queries/0_stateless/00715_fetch_merged_or_mutated_part_zookeeper.sh @@ -11,8 +11,8 @@ ${CLICKHOUSE_CLIENT} -n --query=" DROP TABLE IF EXISTS fetches_r1; DROP TABLE IF EXISTS fetches_r2" -${CLICKHOUSE_CLIENT} --query="CREATE TABLE fetches_r1(x UInt32) ENGINE ReplicatedMergeTree('/clickhouse/tables/test_00715/fetches', 'r1') ORDER BY x" -${CLICKHOUSE_CLIENT} --query="CREATE TABLE fetches_r2(x UInt32) ENGINE ReplicatedMergeTree('/clickhouse/tables/test_00715/fetches', 'r2') ORDER BY x \ +${CLICKHOUSE_CLIENT} --query="CREATE TABLE fetches_r1(x UInt32) ENGINE ReplicatedMergeTree('/clickhouse/tables/$CLICKHOUSE_TEST_ZOOKEEPER_PREFIX/fetches', 'r1') ORDER BY x" +${CLICKHOUSE_CLIENT} --query="CREATE TABLE fetches_r2(x UInt32) ENGINE ReplicatedMergeTree('/clickhouse/tables/$CLICKHOUSE_TEST_ZOOKEEPER_PREFIX/fetches', 'r2') ORDER BY x \ SETTINGS prefer_fetch_merged_part_time_threshold=0, \ prefer_fetch_merged_part_size_threshold=0" diff --git a/tests/queries/0_stateless/00834_kill_mutation_replicated_zookeeper.sh b/tests/queries/0_stateless/00834_kill_mutation_replicated_zookeeper.sh index d1f938f73fe..92ab6814235 100755 --- a/tests/queries/0_stateless/00834_kill_mutation_replicated_zookeeper.sh +++ b/tests/queries/0_stateless/00834_kill_mutation_replicated_zookeeper.sh @@ -10,8 +10,8 @@ CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) ${CLICKHOUSE_CLIENT} --query="DROP TABLE IF EXISTS kill_mutation_r1" ${CLICKHOUSE_CLIENT} --query="DROP TABLE IF EXISTS kill_mutation_r2" -${CLICKHOUSE_CLIENT} --query="CREATE TABLE kill_mutation_r1(d Date, x UInt32, s String) ENGINE ReplicatedMergeTree('/clickhouse/tables/test_00834/kill_mutation', '1') ORDER BY x PARTITION BY d" -${CLICKHOUSE_CLIENT} --query="CREATE TABLE kill_mutation_r2(d Date, x UInt32, s String) ENGINE ReplicatedMergeTree('/clickhouse/tables/test_00834/kill_mutation', '2') ORDER BY x PARTITION BY d" +${CLICKHOUSE_CLIENT} --query="CREATE TABLE kill_mutation_r1(d Date, x UInt32, s String) ENGINE ReplicatedMergeTree('/clickhouse/tables/$CLICKHOUSE_TEST_ZOOKEEPER_PREFIX/kill_mutation', '1') ORDER BY x PARTITION BY d" +${CLICKHOUSE_CLIENT} --query="CREATE TABLE kill_mutation_r2(d Date, x UInt32, s String) ENGINE ReplicatedMergeTree('/clickhouse/tables/$CLICKHOUSE_TEST_ZOOKEEPER_PREFIX/kill_mutation', '2') ORDER BY x PARTITION BY d" ${CLICKHOUSE_CLIENT} --query="INSERT INTO kill_mutation_r1 VALUES ('2000-01-01', 1, 'a')" ${CLICKHOUSE_CLIENT} --query="INSERT INTO kill_mutation_r1 VALUES ('2001-01-01', 2, 'b')" diff --git a/tests/queries/0_stateless/00953_zookeeper_suetin_deduplication_bug.sh b/tests/queries/0_stateless/00953_zookeeper_suetin_deduplication_bug.sh index bbc2d957937..d7a27693e98 100755 --- a/tests/queries/0_stateless/00953_zookeeper_suetin_deduplication_bug.sh +++ b/tests/queries/0_stateless/00953_zookeeper_suetin_deduplication_bug.sh @@ -15,7 +15,7 @@ CREATE TABLE elog ( engine_id UInt32, referrer String ) -ENGINE = ReplicatedMergeTree('/clickhouse/tables/test_00953/elog', 'test') +ENGINE = ReplicatedMergeTree('/clickhouse/tables/$CLICKHOUSE_TEST_ZOOKEEPER_PREFIX/elog', 'test') PARTITION BY date ORDER BY (engine_id) SETTINGS replicated_deduplication_window = 2, cleanup_delay_period=4, cleanup_delay_period_random_add=0;" @@ -28,35 +28,35 @@ $CLICKHOUSE_CLIENT --query="INSERT INTO elog VALUES (toDate('2018-10-01'), 3, 'h $CLICKHOUSE_CLIENT --query="SELECT count(*) from elog" # 3 rows -count=$($CLICKHOUSE_CLIENT --query="SELECT COUNT(*) FROM system.zookeeper where path = '/clickhouse/tables/test_00953/elog/blocks'") +count=$($CLICKHOUSE_CLIENT --query="SELECT COUNT(*) FROM system.zookeeper where path = '/clickhouse/tables/$CLICKHOUSE_TEST_ZOOKEEPER_PREFIX/elog/blocks'") while [[ $count != 2 ]] do sleep 1 - count=$($CLICKHOUSE_CLIENT --query="SELECT COUNT(*) FROM system.zookeeper where path = '/clickhouse/tables/test_00953/elog/blocks'") + count=$($CLICKHOUSE_CLIENT --query="SELECT COUNT(*) FROM system.zookeeper where path = '/clickhouse/tables/$CLICKHOUSE_TEST_ZOOKEEPER_PREFIX/elog/blocks'") done $CLICKHOUSE_CLIENT --query="INSERT INTO elog VALUES (toDate('2018-10-01'), 1, 'hello')" $CLICKHOUSE_CLIENT --query="SELECT count(*) from elog" # 4 rows -count=$($CLICKHOUSE_CLIENT --query="SELECT COUNT(*) FROM system.zookeeper where path = '/clickhouse/tables/test_00953/elog/blocks'") +count=$($CLICKHOUSE_CLIENT --query="SELECT COUNT(*) FROM system.zookeeper where path = '/clickhouse/tables/$CLICKHOUSE_TEST_ZOOKEEPER_PREFIX/elog/blocks'") while [[ $count != 2 ]] do sleep 1 - count=$($CLICKHOUSE_CLIENT --query="SELECT COUNT(*) FROM system.zookeeper where path = '/clickhouse/tables/test_00953/elog/blocks'") + count=$($CLICKHOUSE_CLIENT --query="SELECT COUNT(*) FROM system.zookeeper where path = '/clickhouse/tables/$CLICKHOUSE_TEST_ZOOKEEPER_PREFIX/elog/blocks'") done $CLICKHOUSE_CLIENT --query="INSERT INTO elog VALUES (toDate('2018-10-01'), 2, 'hello')" $CLICKHOUSE_CLIENT --query="SELECT count(*) from elog" # 5 rows -count=$($CLICKHOUSE_CLIENT --query="SELECT COUNT(*) FROM system.zookeeper where path = '/clickhouse/tables/test_00953/elog/blocks'") +count=$($CLICKHOUSE_CLIENT --query="SELECT COUNT(*) FROM system.zookeeper where path = '/clickhouse/tables/$CLICKHOUSE_TEST_ZOOKEEPER_PREFIX/elog/blocks'") while [[ $count != 2 ]] do sleep 1 - count=$($CLICKHOUSE_CLIENT --query="SELECT COUNT(*) FROM system.zookeeper where path = '/clickhouse/tables/test_00953/elog/blocks'") + count=$($CLICKHOUSE_CLIENT --query="SELECT COUNT(*) FROM system.zookeeper where path = '/clickhouse/tables/$CLICKHOUSE_TEST_ZOOKEEPER_PREFIX/elog/blocks'") done $CLICKHOUSE_CLIENT --query="INSERT INTO elog VALUES (toDate('2018-10-01'), 2, 'hello')" diff --git a/tests/queries/0_stateless/00975_indices_mutation_replicated_zookeeper.sh b/tests/queries/0_stateless/00975_indices_mutation_replicated_zookeeper.sh index 81c0c563db1..a3ac5692caa 100755 --- a/tests/queries/0_stateless/00975_indices_mutation_replicated_zookeeper.sh +++ b/tests/queries/0_stateless/00975_indices_mutation_replicated_zookeeper.sh @@ -17,7 +17,7 @@ CREATE TABLE indices_mutaions1 i64 Int64, i32 Int32, INDEX idx (i64, u64 * i64) TYPE minmax GRANULARITY 1 -) ENGINE = ReplicatedMergeTree('/clickhouse/tables/test_00975/indices_mutaions', 'r1') +) ENGINE = ReplicatedMergeTree('/clickhouse/tables/$CLICKHOUSE_TEST_ZOOKEEPER_PREFIX/indices_mutaions', 'r1') PARTITION BY i32 ORDER BY u64 SETTINGS index_granularity = 2; @@ -27,7 +27,7 @@ CREATE TABLE indices_mutaions2 i64 Int64, i32 Int32, INDEX idx (i64, u64 * i64) TYPE minmax GRANULARITY 1 -) ENGINE = ReplicatedMergeTree('/clickhouse/tables/test_00975/indices_mutaions', 'r2') +) ENGINE = ReplicatedMergeTree('/clickhouse/tables/$CLICKHOUSE_TEST_ZOOKEEPER_PREFIX/indices_mutaions', 'r2') PARTITION BY i32 ORDER BY u64 SETTINGS index_granularity = 2;" diff --git a/tests/queries/0_stateless/00992_system_parts_race_condition_zookeeper.sh b/tests/queries/0_stateless/00992_system_parts_race_condition_zookeeper.sh index 613e032f42a..1e61c8d64f3 100755 --- a/tests/queries/0_stateless/00992_system_parts_race_condition_zookeeper.sh +++ b/tests/queries/0_stateless/00992_system_parts_race_condition_zookeeper.sh @@ -10,8 +10,8 @@ $CLICKHOUSE_CLIENT -n -q " DROP TABLE IF EXISTS alter_table; DROP TABLE IF EXISTS alter_table2; - CREATE TABLE alter_table (a UInt8, b Int16, c Float32, d String, e Array(UInt8), f Nullable(UUID), g Tuple(UInt8, UInt16)) ENGINE = ReplicatedMergeTree('/clickhouse/tables/$CLICKHOUSE_DATABASE.alter_table', 'r1') ORDER BY a PARTITION BY b % 10 SETTINGS old_parts_lifetime = 1, cleanup_delay_period = 1, cleanup_delay_period_random_add = 0; - CREATE TABLE alter_table2 (a UInt8, b Int16, c Float32, d String, e Array(UInt8), f Nullable(UUID), g Tuple(UInt8, UInt16)) ENGINE = ReplicatedMergeTree('/clickhouse/tables/$CLICKHOUSE_DATABASE.alter_table', 'r2') ORDER BY a PARTITION BY b % 10 SETTINGS old_parts_lifetime = 1, cleanup_delay_period = 1, cleanup_delay_period_random_add = 0 + CREATE TABLE alter_table (a UInt8, b Int16, c Float32, d String, e Array(UInt8), f Nullable(UUID), g Tuple(UInt8, UInt16)) ENGINE = ReplicatedMergeTree('/clickhouse/tables/$CLICKHOUSE_TEST_ZOOKEEPER_PREFIX/alter_table', 'r1') ORDER BY a PARTITION BY b % 10 SETTINGS old_parts_lifetime = 1, cleanup_delay_period = 1, cleanup_delay_period_random_add = 0; + CREATE TABLE alter_table2 (a UInt8, b Int16, c Float32, d String, e Array(UInt8), f Nullable(UUID), g Tuple(UInt8, UInt16)) ENGINE = ReplicatedMergeTree('/clickhouse/tables/$CLICKHOUSE_TEST_ZOOKEEPER_PREFIX/alter_table', 'r2') ORDER BY a PARTITION BY b % 10 SETTINGS old_parts_lifetime = 1, cleanup_delay_period = 1, cleanup_delay_period_random_add = 0 " function thread1() diff --git a/tests/queries/0_stateless/00993_system_parts_race_condition_drop_zookeeper.sh b/tests/queries/0_stateless/00993_system_parts_race_condition_drop_zookeeper.sh index 1731148f71f..d960d8ff91d 100755 --- a/tests/queries/0_stateless/00993_system_parts_race_condition_drop_zookeeper.sh +++ b/tests/queries/0_stateless/00993_system_parts_race_condition_drop_zookeeper.sh @@ -52,7 +52,7 @@ function thread6() while true; do REPLICA=$(($RANDOM % 10)) $CLICKHOUSE_CLIENT -n -q "DROP TABLE IF EXISTS alter_table_$REPLICA; - CREATE TABLE alter_table_$REPLICA (a UInt8, b Int16, c Float32, d String, e Array(UInt8), f Nullable(UUID), g Tuple(UInt8, UInt16)) ENGINE = ReplicatedMergeTree('/clickhouse/tables/test_00993/alter_table', 'r_$REPLICA') ORDER BY a PARTITION BY b % 10 SETTINGS old_parts_lifetime = 1, cleanup_delay_period = 0, cleanup_delay_period_random_add = 0;"; + CREATE TABLE alter_table_$REPLICA (a UInt8, b Int16, c Float32, d String, e Array(UInt8), f Nullable(UUID), g Tuple(UInt8, UInt16)) ENGINE = ReplicatedMergeTree('/clickhouse/tables/$CLICKHOUSE_TEST_ZOOKEEPER_PREFIX/alter_table', 'r_$REPLICA') ORDER BY a PARTITION BY b % 10 SETTINGS old_parts_lifetime = 1, cleanup_delay_period = 0, cleanup_delay_period_random_add = 0;"; sleep 0.$RANDOM; done } diff --git a/tests/queries/0_stateless/01013_sync_replica_timeout_zookeeper.sh b/tests/queries/0_stateless/01013_sync_replica_timeout_zookeeper.sh index 724caa7f414..89b178a38ea 100755 --- a/tests/queries/0_stateless/01013_sync_replica_timeout_zookeeper.sh +++ b/tests/queries/0_stateless/01013_sync_replica_timeout_zookeeper.sh @@ -4,7 +4,6 @@ CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) # shellcheck source=../shell_config.sh . "$CURDIR"/../shell_config.sh - R1=table_1013_1 R2=table_1013_2 @@ -12,8 +11,8 @@ ${CLICKHOUSE_CLIENT} -n -q " DROP TABLE IF EXISTS $R1; DROP TABLE IF EXISTS $R2; - CREATE TABLE $R1 (x UInt32) ENGINE ReplicatedMergeTree('/clickhouse/tables/${CLICKHOUSE_DATABASE}.table_1013', 'r1') ORDER BY x; - CREATE TABLE $R2 (x UInt32) ENGINE ReplicatedMergeTree('/clickhouse/tables/${CLICKHOUSE_DATABASE}.table_1013', 'r2') ORDER BY x; + CREATE TABLE $R1 (x UInt32) ENGINE ReplicatedMergeTree('/clickhouse/tables/$CLICKHOUSE_TEST_ZOOKEEPER_PREFIX/table_1013', 'r1') ORDER BY x; + CREATE TABLE $R2 (x UInt32) ENGINE ReplicatedMergeTree('/clickhouse/tables/$CLICKHOUSE_TEST_ZOOKEEPER_PREFIX/table_1013', 'r2') ORDER BY x; SYSTEM STOP FETCHES $R2; INSERT INTO $R1 VALUES (1) diff --git a/tests/queries/0_stateless/01017_mutations_with_nondeterministic_functions_zookeeper.sh b/tests/queries/0_stateless/01017_mutations_with_nondeterministic_functions_zookeeper.sh index d7d0dab71b9..a10e5fb2788 100755 --- a/tests/queries/0_stateless/01017_mutations_with_nondeterministic_functions_zookeeper.sh +++ b/tests/queries/0_stateless/01017_mutations_with_nondeterministic_functions_zookeeper.sh @@ -4,7 +4,6 @@ CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) # shellcheck source=../shell_config.sh . "$CURDIR"/../shell_config.sh - R1=table_1017_1 R2=table_1017_2 T1=table_1017_merge @@ -29,8 +28,8 @@ ${CLICKHOUSE_CLIENT} -n -q " CREATE TABLE lookup_table (y UInt32, y_new UInt32) ENGINE = Join(ANY, LEFT, y); INSERT INTO lookup_table VALUES(1,1001),(2,1002); - CREATE TABLE $R1 (x UInt32, y UInt32) ENGINE ReplicatedMergeTree('/clickhouse/tables/${CLICKHOUSE_DATABASE}.table_1017', 'r1') ORDER BY x; - CREATE TABLE $R2 (x UInt32, y UInt32) ENGINE ReplicatedMergeTree('/clickhouse/tables/${CLICKHOUSE_DATABASE}.table_1017', 'r2') ORDER BY x; + CREATE TABLE $R1 (x UInt32, y UInt32) ENGINE ReplicatedMergeTree('/clickhouse/tables/$CLICKHOUSE_TEST_ZOOKEEPER_PREFIX/table_1017', 'r1') ORDER BY x; + CREATE TABLE $R2 (x UInt32, y UInt32) ENGINE ReplicatedMergeTree('/clickhouse/tables/$CLICKHOUSE_TEST_ZOOKEEPER_PREFIX/table_1017', 'r2') ORDER BY x; CREATE TABLE $T1 (x UInt32, y UInt32) ENGINE MergeTree() ORDER BY x; INSERT INTO $R1 VALUES (0, 1)(1, 2)(2, 3)(3, 4); diff --git a/tests/queries/0_stateless/01034_move_partition_from_table_zookeeper.sh b/tests/queries/0_stateless/01034_move_partition_from_table_zookeeper.sh index 11c7296932c..ae3dd7851c8 100755 --- a/tests/queries/0_stateless/01034_move_partition_from_table_zookeeper.sh +++ b/tests/queries/0_stateless/01034_move_partition_from_table_zookeeper.sh @@ -26,8 +26,8 @@ function query_with_retry $CLICKHOUSE_CLIENT --query="DROP TABLE IF EXISTS src;" $CLICKHOUSE_CLIENT --query="DROP TABLE IF EXISTS dst;" -$CLICKHOUSE_CLIENT --query="CREATE TABLE src (p UInt64, k String, d UInt64) ENGINE = ReplicatedMergeTree('/clickhouse/$CLICKHOUSE_DATABASE/src1', '1') PARTITION BY p ORDER BY k;" -$CLICKHOUSE_CLIENT --query="CREATE TABLE dst (p UInt64, k String, d UInt64) ENGINE = ReplicatedMergeTree('/clickhouse/$CLICKHOUSE_DATABASE/dst1', '1') PARTITION BY p ORDER BY k SETTINGS old_parts_lifetime=1, cleanup_delay_period=1, cleanup_delay_period_random_add=0;" +$CLICKHOUSE_CLIENT --query="CREATE TABLE src (p UInt64, k String, d UInt64) ENGINE = ReplicatedMergeTree('/clickhouse/$CLICKHOUSE_TEST_ZOOKEEPER_PREFIX/src1', '1') PARTITION BY p ORDER BY k;" +$CLICKHOUSE_CLIENT --query="CREATE TABLE dst (p UInt64, k String, d UInt64) ENGINE = ReplicatedMergeTree('/clickhouse/$CLICKHOUSE_TEST_ZOOKEEPER_PREFIX/dst1', '1') PARTITION BY p ORDER BY k SETTINGS old_parts_lifetime=1, cleanup_delay_period=1, cleanup_delay_period_random_add=0;" $CLICKHOUSE_CLIENT --query="INSERT INTO src VALUES (0, '0', 1);" $CLICKHOUSE_CLIENT --query="INSERT INTO src VALUES (1, '0', 1);" @@ -56,8 +56,8 @@ $CLICKHOUSE_CLIENT --query="DROP TABLE dst;" $CLICKHOUSE_CLIENT --query="SELECT 'MOVE incompatible schema missing column';" -$CLICKHOUSE_CLIENT --query="CREATE TABLE src (p UInt64, k String, d UInt64) ENGINE = ReplicatedMergeTree('/clickhouse/$CLICKHOUSE_DATABASE/src2', '1') PARTITION BY p ORDER BY (d, p);" -$CLICKHOUSE_CLIENT --query="CREATE TABLE dst (p UInt64, d UInt64) ENGINE = ReplicatedMergeTree('/clickhouse/$CLICKHOUSE_DATABASE/dst2', '1') PARTITION BY p ORDER BY (d, p) SETTINGS old_parts_lifetime=1, cleanup_delay_period=1, cleanup_delay_period_random_add=0;" +$CLICKHOUSE_CLIENT --query="CREATE TABLE src (p UInt64, k String, d UInt64) ENGINE = ReplicatedMergeTree('/clickhouse/$CLICKHOUSE_TEST_ZOOKEEPER_PREFIX/src2', '1') PARTITION BY p ORDER BY (d, p);" +$CLICKHOUSE_CLIENT --query="CREATE TABLE dst (p UInt64, d UInt64) ENGINE = ReplicatedMergeTree('/clickhouse/$CLICKHOUSE_TEST_ZOOKEEPER_PREFIX/dst2', '1') PARTITION BY p ORDER BY (d, p) SETTINGS old_parts_lifetime=1, cleanup_delay_period=1, cleanup_delay_period_random_add=0;" $CLICKHOUSE_CLIENT --query="INSERT INTO src VALUES (0, '0', 1);" $CLICKHOUSE_CLIENT --query="INSERT INTO src VALUES (1, '0', 1);" @@ -75,8 +75,8 @@ $CLICKHOUSE_CLIENT --query="DROP TABLE dst;" $CLICKHOUSE_CLIENT --query="SELECT 'MOVE incompatible schema different order by';" -$CLICKHOUSE_CLIENT --query="CREATE TABLE src (p UInt64, k String, d UInt64) ENGINE = ReplicatedMergeTree('/clickhouse/$CLICKHOUSE_DATABASE/src3', '1') PARTITION BY p ORDER BY (p, k, d);" -$CLICKHOUSE_CLIENT --query="CREATE TABLE dst (p UInt64, k String, d UInt64) ENGINE = ReplicatedMergeTree('/clickhouse/$CLICKHOUSE_DATABASE/dst3', '1') PARTITION BY p ORDER BY (d, k, p);" +$CLICKHOUSE_CLIENT --query="CREATE TABLE src (p UInt64, k String, d UInt64) ENGINE = ReplicatedMergeTree('/clickhouse/$CURR_DATABASE/src3', '1') PARTITION BY p ORDER BY (p, k, d);" +$CLICKHOUSE_CLIENT --query="CREATE TABLE dst (p UInt64, k String, d UInt64) ENGINE = ReplicatedMergeTree('/clickhouse/$CURR_DATABASE/dst3', '1') PARTITION BY p ORDER BY (d, k, p);" $CLICKHOUSE_CLIENT --query="INSERT INTO src VALUES (0, '0', 1);" diff --git a/tests/queries/0_stateless/01035_concurrent_move_partition_from_table_zookeeper.sh b/tests/queries/0_stateless/01035_concurrent_move_partition_from_table_zookeeper.sh index 4eb3cb9a7bd..7c15b795c36 100755 --- a/tests/queries/0_stateless/01035_concurrent_move_partition_from_table_zookeeper.sh +++ b/tests/queries/0_stateless/01035_concurrent_move_partition_from_table_zookeeper.sh @@ -9,8 +9,8 @@ CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) $CLICKHOUSE_CLIENT --query="DROP TABLE IF EXISTS src;" $CLICKHOUSE_CLIENT --query="DROP TABLE IF EXISTS dst;" -$CLICKHOUSE_CLIENT --query="CREATE TABLE src (p UInt64, k String) ENGINE = ReplicatedMergeTree('/clickhouse/$CLICKHOUSE_DATABASE/src', '1') PARTITION BY p ORDER BY k;" -$CLICKHOUSE_CLIENT --query="CREATE TABLE dst (p UInt64, k String) ENGINE = ReplicatedMergeTree('/clickhouse/$CLICKHOUSE_DATABASE/dst', '1') PARTITION BY p ORDER BY k SETTINGS old_parts_lifetime=1, cleanup_delay_period=1, cleanup_delay_period_random_add=0;" +$CLICKHOUSE_CLIENT --query="CREATE TABLE src (p UInt64, k String) ENGINE = ReplicatedMergeTree('/clickhouse/$CLICKHOUSE_TEST_ZOOKEEPER_PREFIX/src', '1') PARTITION BY p ORDER BY k;" +$CLICKHOUSE_CLIENT --query="CREATE TABLE dst (p UInt64, k String) ENGINE = ReplicatedMergeTree('/clickhouse/$CLICKHOUSE_TEST_ZOOKEEPER_PREFIX/dst', '1') PARTITION BY p ORDER BY k SETTINGS old_parts_lifetime=1, cleanup_delay_period=1, cleanup_delay_period_random_add=0;" function thread1() { diff --git a/tests/queries/0_stateless/01045_zookeeper_system_mutations_with_parts_names.sh b/tests/queries/0_stateless/01045_zookeeper_system_mutations_with_parts_names.sh index c035a692d12..6510fcf408d 100755 --- a/tests/queries/0_stateless/01045_zookeeper_system_mutations_with_parts_names.sh +++ b/tests/queries/0_stateless/01045_zookeeper_system_mutations_with_parts_names.sh @@ -47,7 +47,7 @@ ${CLICKHOUSE_CLIENT} --query="DROP TABLE IF EXISTS table_for_mutations" ${CLICKHOUSE_CLIENT} --query="DROP TABLE IF EXISTS replicated_table_for_mutations" -${CLICKHOUSE_CLIENT} --query="CREATE TABLE replicated_table_for_mutations(k UInt32, v1 UInt64) ENGINE ReplicatedMergeTree('/clickhouse/tables/test_01045/replicated_table_for_mutations', '1') ORDER BY k PARTITION BY modulo(k, 2)" +${CLICKHOUSE_CLIENT} --query="CREATE TABLE replicated_table_for_mutations(k UInt32, v1 UInt64) ENGINE ReplicatedMergeTree('/clickhouse/tables/$CLICKHOUSE_TEST_ZOOKEEPER_PREFIX/replicated_table_for_mutations', '1') ORDER BY k PARTITION BY modulo(k, 2)" ${CLICKHOUSE_CLIENT} --query="SYSTEM STOP MERGES replicated_table_for_mutations" diff --git a/tests/queries/0_stateless/01076_parallel_alter_replicated_zookeeper.sh b/tests/queries/0_stateless/01076_parallel_alter_replicated_zookeeper.sh index ca453ee8f0d..efe518046a1 100755 --- a/tests/queries/0_stateless/01076_parallel_alter_replicated_zookeeper.sh +++ b/tests/queries/0_stateless/01076_parallel_alter_replicated_zookeeper.sh @@ -20,7 +20,7 @@ for i in $(seq $REPLICAS); do done for i in $(seq $REPLICAS); do - $CLICKHOUSE_CLIENT --query "CREATE TABLE concurrent_mutate_mt_$i (key UInt64, value1 UInt64, value2 String) ENGINE = ReplicatedMergeTree('/clickhouse/tables/test_01076/concurrent_mutate_mt', '$i') ORDER BY key SETTINGS max_replicated_mutations_in_queue=1000, number_of_free_entries_in_pool_to_execute_mutation=0,max_replicated_merges_in_queue=1000,temporary_directories_lifetime=10,cleanup_delay_period=3,cleanup_delay_period_random_add=0" + $CLICKHOUSE_CLIENT --query "CREATE TABLE concurrent_mutate_mt_$i (key UInt64, value1 UInt64, value2 String) ENGINE = ReplicatedMergeTree('/clickhouse/tables/$CLICKHOUSE_TEST_ZOOKEEPER_PREFIX/concurrent_mutate_mt', '$i') ORDER BY key SETTINGS max_replicated_mutations_in_queue=1000, number_of_free_entries_in_pool_to_execute_mutation=0,max_replicated_merges_in_queue=1000,temporary_directories_lifetime=10,cleanup_delay_period=3,cleanup_delay_period_random_add=0" done $CLICKHOUSE_CLIENT --query "INSERT INTO concurrent_mutate_mt_1 SELECT number, number + 10, toString(number) from numbers(10)" diff --git a/tests/queries/0_stateless/01079_bad_alters_zookeeper.reference b/tests/queries/0_stateless/01079_bad_alters_zookeeper.reference index ebefe4b2a29..67510b28a34 100644 --- a/tests/queries/0_stateless/01079_bad_alters_zookeeper.reference +++ b/tests/queries/0_stateless/01079_bad_alters_zookeeper.reference @@ -1,6 +1,6 @@ Wrong column name. -CREATE TABLE default.table_for_bad_alters\n(\n `key` UInt64,\n `value1` UInt8,\n `value2` String\n)\nENGINE = ReplicatedMergeTree(\'/clickhouse/tables/test_01079/table_for_bad_alters\', \'1\')\nORDER BY key\nSETTINGS index_granularity = 8192 -CREATE TABLE default.table_for_bad_alters\n(\n `key` UInt64,\n `value1` UInt8,\n `value2` UInt32\n)\nENGINE = ReplicatedMergeTree(\'/clickhouse/tables/test_01079/table_for_bad_alters\', \'1\')\nORDER BY key\nSETTINGS index_granularity = 8192 +CREATE TABLE default.table_for_bad_alters\n(\n `key` UInt64,\n `value1` UInt8,\n `value2` String\n)\nENGINE = ReplicatedMergeTree(\'/clickhouse/tables/01079_bad_alters_zookeeper_default/table_for_bad_alters\', \'1\')\nORDER BY key\nSETTINGS index_granularity = 8192 +CREATE TABLE default.table_for_bad_alters\n(\n `key` UInt64,\n `value1` UInt8,\n `value2` UInt32\n)\nENGINE = ReplicatedMergeTree(\'/clickhouse/tables/01079_bad_alters_zookeeper_default/table_for_bad_alters\', \'1\')\nORDER BY key\nSETTINGS index_granularity = 8192 syntax error at begin of string. 7 Hello diff --git a/tests/queries/0_stateless/01079_bad_alters_zookeeper.sh b/tests/queries/0_stateless/01079_bad_alters_zookeeper.sh index 1c0206453b7..6452b830f38 100755 --- a/tests/queries/0_stateless/01079_bad_alters_zookeeper.sh +++ b/tests/queries/0_stateless/01079_bad_alters_zookeeper.sh @@ -10,7 +10,7 @@ $CLICKHOUSE_CLIENT -n --query "CREATE TABLE table_for_bad_alters ( key UInt64, value1 UInt8, value2 String -) ENGINE = ReplicatedMergeTree('/clickhouse/tables/test_01079/table_for_bad_alters', '1') +) ENGINE = ReplicatedMergeTree('/clickhouse/tables/$CLICKHOUSE_TEST_ZOOKEEPER_PREFIX/table_for_bad_alters', '1') ORDER BY key;" $CLICKHOUSE_CLIENT --query "INSERT INTO table_for_bad_alters VALUES(1, 1, 'Hello');" diff --git a/tests/queries/0_stateless/01079_parallel_alter_add_drop_column_zookeeper.sh b/tests/queries/0_stateless/01079_parallel_alter_add_drop_column_zookeeper.sh index b3a5de8f9bc..7a3e3cf155f 100755 --- a/tests/queries/0_stateless/01079_parallel_alter_add_drop_column_zookeeper.sh +++ b/tests/queries/0_stateless/01079_parallel_alter_add_drop_column_zookeeper.sh @@ -12,7 +12,7 @@ done for i in $(seq $REPLICAS); do - $CLICKHOUSE_CLIENT --query "CREATE TABLE concurrent_alter_add_drop_$i (key UInt64, value0 UInt8) ENGINE = ReplicatedMergeTree('/clickhouse/tables/test_01079/concurrent_alter_add_drop_column', '$i') ORDER BY key SETTINGS max_replicated_mutations_in_queue=1000, number_of_free_entries_in_pool_to_execute_mutation=0,max_replicated_merges_in_queue=1000" + $CLICKHOUSE_CLIENT --query "CREATE TABLE concurrent_alter_add_drop_$i (key UInt64, value0 UInt8) ENGINE = ReplicatedMergeTree('/clickhouse/tables/$CLICKHOUSE_TEST_ZOOKEEPER_PREFIX/concurrent_alter_add_drop_column', '$i') ORDER BY key SETTINGS max_replicated_mutations_in_queue=1000, number_of_free_entries_in_pool_to_execute_mutation=0,max_replicated_merges_in_queue=1000" done $CLICKHOUSE_CLIENT --query "INSERT INTO concurrent_alter_add_drop_1 SELECT number, number + 10 from numbers(100000)" diff --git a/tests/queries/0_stateless/01079_parallel_alter_detach_table_zookeeper.sh b/tests/queries/0_stateless/01079_parallel_alter_detach_table_zookeeper.sh index d5f0c987e5d..83f3196253a 100755 --- a/tests/queries/0_stateless/01079_parallel_alter_detach_table_zookeeper.sh +++ b/tests/queries/0_stateless/01079_parallel_alter_detach_table_zookeeper.sh @@ -11,7 +11,7 @@ for i in $(seq $REPLICAS); do done for i in $(seq $REPLICAS); do - $CLICKHOUSE_CLIENT --query "CREATE TABLE concurrent_alter_detach_$i (key UInt64, value1 UInt8, value2 UInt8) ENGINE = ReplicatedMergeTree('/clickhouse/tables/test_01079/concurrent_alter_detach', '$i') ORDER BY key SETTINGS max_replicated_mutations_in_queue=1000, number_of_free_entries_in_pool_to_execute_mutation=0,max_replicated_merges_in_queue=1000,temporary_directories_lifetime=10,cleanup_delay_period=3,cleanup_delay_period_random_add=0" + $CLICKHOUSE_CLIENT --query "CREATE TABLE concurrent_alter_detach_$i (key UInt64, value1 UInt8, value2 UInt8) ENGINE = ReplicatedMergeTree('/clickhouse/tables/$CLICKHOUSE_TEST_ZOOKEEPER_PREFIX/concurrent_alter_detach', '$i') ORDER BY key SETTINGS max_replicated_mutations_in_queue=1000, number_of_free_entries_in_pool_to_execute_mutation=0,max_replicated_merges_in_queue=1000,temporary_directories_lifetime=10,cleanup_delay_period=3,cleanup_delay_period_random_add=0" done $CLICKHOUSE_CLIENT --query "INSERT INTO concurrent_alter_detach_1 SELECT number, number + 10, number from numbers(10)" diff --git a/tests/queries/0_stateless/01079_parallel_alter_modify_zookeeper.sh b/tests/queries/0_stateless/01079_parallel_alter_modify_zookeeper.sh index 5b14c5a8543..9cca73b5eef 100755 --- a/tests/queries/0_stateless/01079_parallel_alter_modify_zookeeper.sh +++ b/tests/queries/0_stateless/01079_parallel_alter_modify_zookeeper.sh @@ -11,7 +11,7 @@ for i in $(seq $REPLICAS); do done for i in $(seq $REPLICAS); do - $CLICKHOUSE_CLIENT --query "CREATE TABLE concurrent_alter_mt_$i (key UInt64, value1 UInt64, value2 Int32) ENGINE = ReplicatedMergeTree('/clickhouse/tables/test_01079/concurrent_alter_mt', '$i') ORDER BY key SETTINGS max_replicated_mutations_in_queue=1000, number_of_free_entries_in_pool_to_execute_mutation=0,max_replicated_merges_in_queue=1000" + $CLICKHOUSE_CLIENT --query "CREATE TABLE concurrent_alter_mt_$i (key UInt64, value1 UInt64, value2 Int32) ENGINE = ReplicatedMergeTree('/clickhouse/tables/$CLICKHOUSE_TEST_ZOOKEEPER_PREFIX/concurrent_alter_mt', '$i') ORDER BY key SETTINGS max_replicated_mutations_in_queue=1000, number_of_free_entries_in_pool_to_execute_mutation=0,max_replicated_merges_in_queue=1000" done $CLICKHOUSE_CLIENT --query "INSERT INTO concurrent_alter_mt_1 SELECT number, number + 10, number from numbers(10)" diff --git a/tests/queries/0_stateless/01103_optimize_drop_race_zookeeper.sh b/tests/queries/0_stateless/01103_optimize_drop_race_zookeeper.sh index 287a63f858b..72459e21b69 100755 --- a/tests/queries/0_stateless/01103_optimize_drop_race_zookeeper.sh +++ b/tests/queries/0_stateless/01103_optimize_drop_race_zookeeper.sh @@ -26,7 +26,7 @@ function thread3() { while true; do $CLICKHOUSE_CLIENT -n -q "DROP TABLE IF EXISTS concurrent_optimize_table; - CREATE TABLE concurrent_optimize_table (a UInt8, b Int16, c Float32, d String, e Array(UInt8), f Nullable(UUID), g Tuple(UInt8, UInt16)) ENGINE = ReplicatedMergeTree('/clickhouse/tables/test_01103/concurrent_optimize_table', '1') ORDER BY a PARTITION BY b % 10 SETTINGS old_parts_lifetime = 1, cleanup_delay_period = 0, cleanup_delay_period_random_add = 0;"; + CREATE TABLE concurrent_optimize_table (a UInt8, b Int16, c Float32, d String, e Array(UInt8), f Nullable(UUID), g Tuple(UInt8, UInt16)) ENGINE = ReplicatedMergeTree('/clickhouse/tables/$CLICKHOUSE_TEST_ZOOKEEPER_PREFIX/concurrent_optimize_table', '1') ORDER BY a PARTITION BY b % 10 SETTINGS old_parts_lifetime = 1, cleanup_delay_period = 0, cleanup_delay_period_random_add = 0;"; sleep 0.$RANDOM; sleep 0.$RANDOM; sleep 0.$RANDOM; diff --git a/tests/queries/0_stateless/01108_restart_replicas_rename_deadlock_zookeeper.sh b/tests/queries/0_stateless/01108_restart_replicas_rename_deadlock_zookeeper.sh index cddf1ebcda6..d564594291a 100755 --- a/tests/queries/0_stateless/01108_restart_replicas_rename_deadlock_zookeeper.sh +++ b/tests/queries/0_stateless/01108_restart_replicas_rename_deadlock_zookeeper.sh @@ -7,7 +7,7 @@ CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) for i in $(seq 4); do $CLICKHOUSE_CLIENT -q "DROP TABLE IF EXISTS replica_01108_$i" $CLICKHOUSE_CLIENT -q "DROP TABLE IF EXISTS replica_01108_${i}_tmp" - $CLICKHOUSE_CLIENT -q "CREATE TABLE replica_01108_$i (n int) ENGINE=ReplicatedMergeTree('/clickhouse/tables/test_01108/replica_01108_$i', 'replica') ORDER BY tuple()" + $CLICKHOUSE_CLIENT -q "CREATE TABLE replica_01108_$i (n int) ENGINE=ReplicatedMergeTree('/clickhouse/tables/$CLICKHOUSE_TEST_ZOOKEEPER_PREFIX/replica_01108_$i', 'replica') ORDER BY tuple()" $CLICKHOUSE_CLIENT -q "INSERT INTO replica_01108_$i SELECT * FROM system.numbers LIMIT $i * 10, 10" done diff --git a/tests/queries/0_stateless/01192_rename_database_zookeeper.sh b/tests/queries/0_stateless/01192_rename_database_zookeeper.sh index 90b9baf4ebf..58bdfbf71ad 100755 --- a/tests/queries/0_stateless/01192_rename_database_zookeeper.sh +++ b/tests/queries/0_stateless/01192_rename_database_zookeeper.sh @@ -36,7 +36,7 @@ $CLICKHOUSE_CLIENT -q "SELECT count(n), sum(n) FROM test_01192_renamed.mt" # 5. check moving tables from Ordinary to Atomic (can be used to "alter" database engine) $CLICKHOUSE_CLIENT --default_database_engine=Ordinary -q "CREATE DATABASE test_01192" $CLICKHOUSE_CLIENT -q "CREATE TABLE test_01192.mt AS test_01192_renamed.mt ENGINE=MergeTree ORDER BY n" -$CLICKHOUSE_CLIENT -q "CREATE TABLE test_01192.rmt AS test_01192_renamed.mt ENGINE=ReplicatedMergeTree('/test/01192/', '1') ORDER BY n" +$CLICKHOUSE_CLIENT -q "CREATE TABLE test_01192.rmt AS test_01192_renamed.mt ENGINE=ReplicatedMergeTree('/test/$CLICKHOUSE_TEST_ZOOKEEPER_PREFIX/', '1') ORDER BY n" $CLICKHOUSE_CLIENT -q "CREATE MATERIALIZED VIEW test_01192.mv TO test_01192.rmt AS SELECT * FROM test_01192.mt" $CLICKHOUSE_CLIENT -q "INSERT INTO test_01192.mt SELECT number FROM numbers(10)" && echo "inserted" diff --git a/tests/queries/0_stateless/01213_alter_rename_column_zookeeper.reference b/tests/queries/0_stateless/01213_alter_rename_column_zookeeper.reference index fc2a74d1a93..35385731ad3 100644 --- a/tests/queries/0_stateless/01213_alter_rename_column_zookeeper.reference +++ b/tests/queries/0_stateless/01213_alter_rename_column_zookeeper.reference @@ -1,7 +1,7 @@ 1 -CREATE TABLE default.table_for_rename_replicated\n(\n `date` Date,\n `key` UInt64,\n `value1` String,\n `value2` String,\n `value3` String\n)\nENGINE = ReplicatedMergeTree(\'/clickhouse/tables/test_01213/table_for_rename_replicated\', \'1\')\nPARTITION BY date\nORDER BY key\nSETTINGS index_granularity = 8192 +CREATE TABLE default.table_for_rename_replicated\n(\n `date` Date,\n `key` UInt64,\n `value1` String,\n `value2` String,\n `value3` String\n)\nENGINE = ReplicatedMergeTree(\'/clickhouse/tables/01213_alter_rename_column_zookeeper_default/table_for_rename_replicated\', \'1\')\nPARTITION BY date\nORDER BY key\nSETTINGS index_granularity = 8192 renamed_value1 -CREATE TABLE default.table_for_rename_replicated\n(\n `date` Date,\n `key` UInt64,\n `renamed_value1` String,\n `value2` String,\n `value3` String\n)\nENGINE = ReplicatedMergeTree(\'/clickhouse/tables/test_01213/table_for_rename_replicated\', \'1\')\nPARTITION BY date\nORDER BY key\nSETTINGS index_granularity = 8192 +CREATE TABLE default.table_for_rename_replicated\n(\n `date` Date,\n `key` UInt64,\n `renamed_value1` String,\n `value2` String,\n `value3` String\n)\nENGINE = ReplicatedMergeTree(\'/clickhouse/tables/01213_alter_rename_column_zookeeper_default/table_for_rename_replicated\', \'1\')\nPARTITION BY date\nORDER BY key\nSETTINGS index_granularity = 8192 1 date key renamed_value1 value2 value3 2019-10-02 1 1 1 1 diff --git a/tests/queries/0_stateless/01213_alter_rename_column_zookeeper.sh b/tests/queries/0_stateless/01213_alter_rename_column_zookeeper.sh index 5ab0e800d39..5da8de70c46 100755 --- a/tests/queries/0_stateless/01213_alter_rename_column_zookeeper.sh +++ b/tests/queries/0_stateless/01213_alter_rename_column_zookeeper.sh @@ -15,7 +15,7 @@ CREATE TABLE table_for_rename_replicated value2 String, value3 String ) -ENGINE = ReplicatedMergeTree('/clickhouse/tables/test_01213/table_for_rename_replicated', '1') +ENGINE = ReplicatedMergeTree('/clickhouse/tables/$CLICKHOUSE_TEST_ZOOKEEPER_PREFIX/table_for_rename_replicated', '1') PARTITION BY date ORDER BY key; " diff --git a/tests/queries/0_stateless/01305_replica_create_drop_zookeeper.sh b/tests/queries/0_stateless/01305_replica_create_drop_zookeeper.sh index 5dd3d2b38d6..01bb9af461c 100755 --- a/tests/queries/0_stateless/01305_replica_create_drop_zookeeper.sh +++ b/tests/queries/0_stateless/01305_replica_create_drop_zookeeper.sh @@ -13,13 +13,13 @@ function thread() # Ignore "Replica already exists" exception while true; do $CLICKHOUSE_CLIENT -n -q "DROP TABLE IF EXISTS test_table_$1 NO DELAY; - CREATE TABLE test_table_$1 (a UInt8) ENGINE = ReplicatedMergeTree('/clickhouse/tables/test_01305/alter_table', 'r_$1') ORDER BY tuple();" 2>&1 | + CREATE TABLE test_table_$1 (a UInt8) ENGINE = ReplicatedMergeTree('/clickhouse/tables/$CLICKHOUSE_TEST_ZOOKEEPER_PREFIX/alter_table', 'r_$1') ORDER BY tuple();" 2>&1 | grep -vP '(^$)|(^Received exception from server)|(^\d+\. )|because the last replica of the table was dropped right now|is already started to be removing by another replica right now|is already finished removing by another replica right now|Removing leftovers from table|Another replica was suddenly created|was successfully removed from ZooKeeper|was created by another server at the same moment|was suddenly removed|some other replicas were created at the same time|already exists' done else while true; do $CLICKHOUSE_CLIENT -n -q "DROP TABLE IF EXISTS test_table_$1; - CREATE TABLE test_table_$1 (a UInt8) ENGINE = ReplicatedMergeTree('/clickhouse/tables/test_01305/alter_table', 'r_$1') ORDER BY tuple();" 2>&1 | + CREATE TABLE test_table_$1 (a UInt8) ENGINE = ReplicatedMergeTree('/clickhouse/tables/$CLICKHOUSE_TEST_ZOOKEEPER_PREFIX/alter_table', 'r_$1') ORDER BY tuple();" 2>&1 | grep -vP '(^$)|(^Received exception from server)|(^\d+\. )|because the last replica of the table was dropped right now|is already started to be removing by another replica right now|is already finished removing by another replica right now|Removing leftovers from table|Another replica was suddenly created|was successfully removed from ZooKeeper|was created by another server at the same moment|was suddenly removed|some other replicas were created at the same time' done fi diff --git a/tests/queries/0_stateless/01307_multiple_leaders_zookeeper.sh b/tests/queries/0_stateless/01307_multiple_leaders_zookeeper.sh index 24c6199a94a..21fc88d7c2d 100755 --- a/tests/queries/0_stateless/01307_multiple_leaders_zookeeper.sh +++ b/tests/queries/0_stateless/01307_multiple_leaders_zookeeper.sh @@ -12,7 +12,7 @@ DATA_SIZE=200 SEQ=$(seq 0 $(($NUM_REPLICAS - 1))) for REPLICA in $SEQ; do $CLICKHOUSE_CLIENT -n --query "DROP TABLE IF EXISTS r$REPLICA"; done -for REPLICA in $SEQ; do $CLICKHOUSE_CLIENT -n --query "CREATE TABLE r$REPLICA (x UInt64) ENGINE = ReplicatedMergeTree('/test_01307/table', 'r$REPLICA') ORDER BY x SETTINGS min_bytes_for_wide_part = '10M';"; done +for REPLICA in $SEQ; do $CLICKHOUSE_CLIENT -n --query "CREATE TABLE r$REPLICA (x UInt64) ENGINE = ReplicatedMergeTree('/$CLICKHOUSE_TEST_ZOOKEEPER_PREFIX/table', 'r$REPLICA') ORDER BY x SETTINGS min_bytes_for_wide_part = '10M';"; done function thread() { diff --git a/tests/queries/0_stateless/01318_long_unsuccessful_mutation_zookeeper.sh b/tests/queries/0_stateless/01318_long_unsuccessful_mutation_zookeeper.sh index ced668e9849..a05304c670c 100755 --- a/tests/queries/0_stateless/01318_long_unsuccessful_mutation_zookeeper.sh +++ b/tests/queries/0_stateless/01318_long_unsuccessful_mutation_zookeeper.sh @@ -11,7 +11,7 @@ $CLICKHOUSE_CLIENT --query " key UInt64, value String ) - ENGINE = ReplicatedMergeTree('/clickhouse/tables/test_01318/mutation_table', '1') + ENGINE = ReplicatedMergeTree('/clickhouse/tables/$CLICKHOUSE_TEST_ZOOKEEPER_PREFIX/mutation_table', '1') ORDER BY key PARTITION BY key % 10 " diff --git a/tests/queries/0_stateless/01320_create_sync_race_condition_zookeeper.sh b/tests/queries/0_stateless/01320_create_sync_race_condition_zookeeper.sh index a15d8c8d2cd..97c200c651f 100755 --- a/tests/queries/0_stateless/01320_create_sync_race_condition_zookeeper.sh +++ b/tests/queries/0_stateless/01320_create_sync_race_condition_zookeeper.sh @@ -11,7 +11,7 @@ $CLICKHOUSE_CLIENT --query "CREATE DATABASE test_01320 ENGINE=Ordinary" # Diff function thread1() { - while true; do $CLICKHOUSE_CLIENT -n --query "CREATE TABLE test_01320.r (x UInt64) ENGINE = ReplicatedMergeTree('/test_01320/table', 'r') ORDER BY x; DROP TABLE test_01320.r;"; done + while true; do $CLICKHOUSE_CLIENT -n --query "CREATE TABLE test_01320.r (x UInt64) ENGINE = ReplicatedMergeTree('/$CLICKHOUSE_TEST_ZOOKEEPER_PREFIX/table', 'r') ORDER BY x; DROP TABLE test_01320.r;"; done } function thread2() diff --git a/tests/queries/0_stateless/01338_long_select_and_alter_zookeeper.reference b/tests/queries/0_stateless/01338_long_select_and_alter_zookeeper.reference index e7db2788824..b4ed8efab63 100644 --- a/tests/queries/0_stateless/01338_long_select_and_alter_zookeeper.reference +++ b/tests/queries/0_stateless/01338_long_select_and_alter_zookeeper.reference @@ -1,3 +1,3 @@ 10 5 -CREATE TABLE default.alter_mt\n(\n `key` UInt64,\n `value` UInt64\n)\nENGINE = ReplicatedMergeTree(\'/clickhouse/tables/test_01338/alter_mt\', \'1\')\nORDER BY key\nSETTINGS index_granularity = 8192 +CREATE TABLE default.alter_mt\n(\n `key` UInt64,\n `value` UInt64\n)\nENGINE = ReplicatedMergeTree(\'/clickhouse/tables/01338_long_select_and_alter_zookeeper_default/alter_mt\', \'1\')\nORDER BY key\nSETTINGS index_granularity = 8192 diff --git a/tests/queries/0_stateless/01338_long_select_and_alter_zookeeper.sh b/tests/queries/0_stateless/01338_long_select_and_alter_zookeeper.sh index d990a8a1c08..4aeecc7343d 100755 --- a/tests/queries/0_stateless/01338_long_select_and_alter_zookeeper.sh +++ b/tests/queries/0_stateless/01338_long_select_and_alter_zookeeper.sh @@ -6,7 +6,7 @@ CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) $CLICKHOUSE_CLIENT --query "DROP TABLE IF EXISTS alter_mt" -$CLICKHOUSE_CLIENT --query "CREATE TABLE alter_mt (key UInt64, value String) ENGINE=ReplicatedMergeTree('/clickhouse/tables/test_01338/alter_mt', '1') ORDER BY key" +$CLICKHOUSE_CLIENT --query "CREATE TABLE alter_mt (key UInt64, value String) ENGINE=ReplicatedMergeTree('/clickhouse/tables/$CLICKHOUSE_TEST_ZOOKEEPER_PREFIX/alter_mt', '1') ORDER BY key" $CLICKHOUSE_CLIENT --query "INSERT INTO alter_mt SELECT number, toString(number) FROM numbers(5)" diff --git a/tests/queries/0_stateless/01396_inactive_replica_cleanup_nodes_zookeeper.sh b/tests/queries/0_stateless/01396_inactive_replica_cleanup_nodes_zookeeper.sh index 30b2b665658..b604ace85cc 100755 --- a/tests/queries/0_stateless/01396_inactive_replica_cleanup_nodes_zookeeper.sh +++ b/tests/queries/0_stateless/01396_inactive_replica_cleanup_nodes_zookeeper.sh @@ -12,8 +12,8 @@ SCALE=5000 $CLICKHOUSE_CLIENT -n --query " DROP TABLE IF EXISTS r1; DROP TABLE IF EXISTS r2; - CREATE TABLE r1 (x UInt64) ENGINE = ReplicatedMergeTree('/clickhouse/tables/test_01396/r', '1') ORDER BY x SETTINGS old_parts_lifetime = 1, cleanup_delay_period = 0, cleanup_delay_period_random_add = 1, parts_to_throw_insert = 100000, max_replicated_logs_to_keep = 10; - CREATE TABLE r2 (x UInt64) ENGINE = ReplicatedMergeTree('/clickhouse/tables/test_01396/r', '2') ORDER BY x SETTINGS old_parts_lifetime = 1, cleanup_delay_period = 0, cleanup_delay_period_random_add = 1, parts_to_throw_insert = 100000, max_replicated_logs_to_keep = 10; + CREATE TABLE r1 (x UInt64) ENGINE = ReplicatedMergeTree('/clickhouse/tables/$CLICKHOUSE_TEST_ZOOKEEPER_PREFIX/r', '1') ORDER BY x SETTINGS old_parts_lifetime = 1, cleanup_delay_period = 0, cleanup_delay_period_random_add = 1, parts_to_throw_insert = 100000, max_replicated_logs_to_keep = 10; + CREATE TABLE r2 (x UInt64) ENGINE = ReplicatedMergeTree('/clickhouse/tables/$CLICKHOUSE_TEST_ZOOKEEPER_PREFIX/r', '2') ORDER BY x SETTINGS old_parts_lifetime = 1, cleanup_delay_period = 0, cleanup_delay_period_random_add = 1, parts_to_throw_insert = 100000, max_replicated_logs_to_keep = 10; DETACH TABLE r2; " @@ -29,16 +29,16 @@ for _ in {1..60}; do done -$CLICKHOUSE_CLIENT --query "SELECT numChildren < $((SCALE / 4)) FROM system.zookeeper WHERE path = '/clickhouse/tables/test_01396/r' AND name = 'log'"; +$CLICKHOUSE_CLIENT --query "SELECT numChildren < $((SCALE / 4)) FROM system.zookeeper WHERE path = '/clickhouse/tables/$CLICKHOUSE_TEST_ZOOKEEPER_PREFIX/r' AND name = 'log'"; echo -e '\n---\n'; -$CLICKHOUSE_CLIENT --query "SELECT value FROM system.zookeeper WHERE path = '/clickhouse/tables/test_01396/r/replicas/1' AND name = 'is_lost'"; -$CLICKHOUSE_CLIENT --query "SELECT value FROM system.zookeeper WHERE path = '/clickhouse/tables/test_01396/r/replicas/2' AND name = 'is_lost'"; +$CLICKHOUSE_CLIENT --query "SELECT value FROM system.zookeeper WHERE path = '/clickhouse/tables/$CLICKHOUSE_TEST_ZOOKEEPER_PREFIX/r/replicas/1' AND name = 'is_lost'"; +$CLICKHOUSE_CLIENT --query "SELECT value FROM system.zookeeper WHERE path = '/clickhouse/tables/$CLICKHOUSE_TEST_ZOOKEEPER_PREFIX/r/replicas/2' AND name = 'is_lost'"; echo -e '\n---\n'; $CLICKHOUSE_CLIENT --query "ATTACH TABLE r2" $CLICKHOUSE_CLIENT --receive_timeout 600 --query "SYSTEM SYNC REPLICA r2" # Need to increase timeout, otherwise it timed out in debug build -$CLICKHOUSE_CLIENT --query "SELECT value FROM system.zookeeper WHERE path = '/clickhouse/tables/test_01396/r/replicas/2' AND name = 'is_lost'"; +$CLICKHOUSE_CLIENT --query "SELECT value FROM system.zookeeper WHERE path = '/clickhouse/tables/$CLICKHOUSE_TEST_ZOOKEEPER_PREFIX/r/replicas/2' AND name = 'is_lost'"; $CLICKHOUSE_CLIENT -n --query " DROP TABLE IF EXISTS r1; diff --git a/tests/queries/0_stateless/01414_mutations_and_errors_zookeeper.sh b/tests/queries/0_stateless/01414_mutations_and_errors_zookeeper.sh index ceeeed41049..6e1a6e01757 100755 --- a/tests/queries/0_stateless/01414_mutations_and_errors_zookeeper.sh +++ b/tests/queries/0_stateless/01414_mutations_and_errors_zookeeper.sh @@ -12,7 +12,7 @@ $CLICKHOUSE_CLIENT --query " key UInt64, value String ) - ENGINE = ReplicatedMergeTree('/clickhouse/tables/test_01414/mutation_table', '1') + ENGINE = ReplicatedMergeTree('/clickhouse/tables/$CLICKHOUSE_TEST_ZOOKEEPER_PREFIX/mutation_table', '1') ORDER BY tuple() PARTITION BY date " diff --git a/tests/queries/0_stateless/01417_freeze_partition_verbose_zookeeper.sh b/tests/queries/0_stateless/01417_freeze_partition_verbose_zookeeper.sh index 480daeefa46..bb935a950ff 100755 --- a/tests/queries/0_stateless/01417_freeze_partition_verbose_zookeeper.sh +++ b/tests/queries/0_stateless/01417_freeze_partition_verbose_zookeeper.sh @@ -11,7 +11,7 @@ FREEZE_OUT_STRUCTURE='backup_name String, backup_path String , part_backup_path # setup ${CLICKHOUSE_CLIENT} --query "DROP TABLE IF EXISTS table_for_freeze_replicated;" -${CLICKHOUSE_CLIENT} --query "CREATE TABLE table_for_freeze_replicated (key UInt64, value String) ENGINE = ReplicatedMergeTree('/test_01417/table_for_freeze_replicated', '1') ORDER BY key PARTITION BY key % 10;" +${CLICKHOUSE_CLIENT} --query "CREATE TABLE table_for_freeze_replicated (key UInt64, value String) ENGINE = ReplicatedMergeTree('/$CLICKHOUSE_TEST_ZOOKEEPER_PREFIX/table_for_freeze_replicated', '1') ORDER BY key PARTITION BY key % 10;" ${CLICKHOUSE_CLIENT} --query "INSERT INTO table_for_freeze_replicated SELECT number, toString(number) from numbers(10);" ${CLICKHOUSE_CLIENT} --query "ALTER TABLE table_for_freeze_replicated FREEZE WITH NAME 'test_01417' FORMAT TSVWithNames SETTINGS alter_partition_verbose_result = 1;" \ diff --git a/tests/queries/0_stateless/01459_manual_write_to_replicas.sh b/tests/queries/0_stateless/01459_manual_write_to_replicas.sh index 467c29d3d33..cf239fd7032 100755 --- a/tests/queries/0_stateless/01459_manual_write_to_replicas.sh +++ b/tests/queries/0_stateless/01459_manual_write_to_replicas.sh @@ -11,7 +11,7 @@ NUM_REPLICAS=10 for i in $(seq 1 $NUM_REPLICAS); do $CLICKHOUSE_CLIENT -n -q " DROP TABLE IF EXISTS r$i; - CREATE TABLE r$i (x UInt64) ENGINE = ReplicatedMergeTree('/clickhouse/tables/01459_manual_write_ro_replicas/r', 'r$i') ORDER BY x; + CREATE TABLE r$i (x UInt64) ENGINE = ReplicatedMergeTree('/clickhouse/tables/$CLICKHOUSE_TEST_ZOOKEEPER_PREFIX/r', 'r$i') ORDER BY x; " done diff --git a/tests/queries/0_stateless/01459_manual_write_to_replicas_quorum.sh b/tests/queries/0_stateless/01459_manual_write_to_replicas_quorum.sh index 376ee58859e..8c322798173 100755 --- a/tests/queries/0_stateless/01459_manual_write_to_replicas_quorum.sh +++ b/tests/queries/0_stateless/01459_manual_write_to_replicas_quorum.sh @@ -11,7 +11,7 @@ NUM_REPLICAS=10 for i in $(seq 1 $NUM_REPLICAS); do $CLICKHOUSE_CLIENT -n -q " DROP TABLE IF EXISTS r$i; - CREATE TABLE r$i (x UInt64) ENGINE = ReplicatedMergeTree('/clickhouse/tables/01459_manual_write_ro_replicas_quorum/r', 'r$i') ORDER BY x; + CREATE TABLE r$i (x UInt64) ENGINE = ReplicatedMergeTree('/clickhouse/tables/$CLICKHOUSE_TEST_ZOOKEEPER_PREFIX/r', 'r$i') ORDER BY x; " done diff --git a/tests/queries/0_stateless/01508_race_condition_rename_clear_zookeeper.sh b/tests/queries/0_stateless/01508_race_condition_rename_clear_zookeeper.sh index 4cb4734b448..156deb60ff9 100755 --- a/tests/queries/0_stateless/01508_race_condition_rename_clear_zookeeper.sh +++ b/tests/queries/0_stateless/01508_race_condition_rename_clear_zookeeper.sh @@ -8,7 +8,7 @@ $CLICKHOUSE_CLIENT --query "DROP TABLE IF EXISTS table_for_renames0" $CLICKHOUSE_CLIENT --query "DROP TABLE IF EXISTS table_for_renames50" -$CLICKHOUSE_CLIENT --query "CREATE TABLE table_for_renames0 (value UInt64, data String) ENGINE ReplicatedMergeTree('/clickhouse/tables/test_01508/concurrent_rename', '1') ORDER BY tuple() SETTINGS cleanup_delay_period = 1, cleanup_delay_period_random_add = 0, min_rows_for_compact_part = 100000, min_rows_for_compact_part = 10000000, write_ahead_log_max_bytes = 1" +$CLICKHOUSE_CLIENT --query "CREATE TABLE table_for_renames0 (value UInt64, data String) ENGINE ReplicatedMergeTree('/clickhouse/tables/$CLICKHOUSE_TEST_ZOOKEEPER_PREFIX/concurrent_rename', '1') ORDER BY tuple() SETTINGS cleanup_delay_period = 1, cleanup_delay_period_random_add = 0, min_rows_for_compact_part = 100000, min_rows_for_compact_part = 10000000, write_ahead_log_max_bytes = 1" $CLICKHOUSE_CLIENT --query "INSERT INTO table_for_renames0 SELECT number, toString(number) FROM numbers(1000)" diff --git a/tests/queries/0_stateless/01509_check_many_parallel_quorum_inserts.sh b/tests/queries/0_stateless/01509_check_many_parallel_quorum_inserts.sh index c5ffad1c4ca..b71654e7e6c 100755 --- a/tests/queries/0_stateless/01509_check_many_parallel_quorum_inserts.sh +++ b/tests/queries/0_stateless/01509_check_many_parallel_quorum_inserts.sh @@ -11,7 +11,7 @@ NUM_REPLICAS=10 for i in $(seq 1 $NUM_REPLICAS); do $CLICKHOUSE_CLIENT -n -q " DROP TABLE IF EXISTS r$i; - CREATE TABLE r$i (x UInt64) ENGINE = ReplicatedMergeTree('/clickhouse/tables/test_01509/parallel_quorum_many', 'r$i') ORDER BY x; + CREATE TABLE r$i (x UInt64) ENGINE = ReplicatedMergeTree('/clickhouse/tables/$CLICKHOUSE_TEST_ZOOKEEPER_PREFIX/parallel_quorum_many', 'r$i') ORDER BY x; " done diff --git a/tests/queries/0_stateless/01509_check_parallel_quorum_inserts.sh b/tests/queries/0_stateless/01509_check_parallel_quorum_inserts.sh index 898a68d9c77..78336ea073b 100755 --- a/tests/queries/0_stateless/01509_check_parallel_quorum_inserts.sh +++ b/tests/queries/0_stateless/01509_check_parallel_quorum_inserts.sh @@ -12,7 +12,7 @@ NUM_INSERTS=5 for i in $(seq 1 $NUM_REPLICAS); do $CLICKHOUSE_CLIENT -n -q " DROP TABLE IF EXISTS r$i; - CREATE TABLE r$i (x UInt64) ENGINE = ReplicatedMergeTree('/clickhouse/tables/test_01509/parallel_quorum', 'r$i') ORDER BY x; + CREATE TABLE r$i (x UInt64) ENGINE = ReplicatedMergeTree('/clickhouse/tables/$CLICKHOUSE_TEST_ZOOKEEPER_PREFIX/parallel_quorum', 'r$i') ORDER BY x; " done diff --git a/tests/queries/0_stateless/01509_parallel_quorum_and_merge.sh b/tests/queries/0_stateless/01509_parallel_quorum_and_merge.sh index ca5f58512a3..fbeb65419ce 100755 --- a/tests/queries/0_stateless/01509_parallel_quorum_and_merge.sh +++ b/tests/queries/0_stateless/01509_parallel_quorum_and_merge.sh @@ -10,9 +10,9 @@ $CLICKHOUSE_CLIENT -q "DROP TABLE IF EXISTS parallel_q1" $CLICKHOUSE_CLIENT -q "DROP TABLE IF EXISTS parallel_q2" -$CLICKHOUSE_CLIENT -q "CREATE TABLE parallel_q1 (x UInt64) ENGINE=ReplicatedMergeTree('/clickhouse/tables/test_01509/parallel_q', 'r1') ORDER BY tuple() SETTINGS old_parts_lifetime = 1, cleanup_delay_period = 0, cleanup_delay_period_random_add = 0" +$CLICKHOUSE_CLIENT -q "CREATE TABLE parallel_q1 (x UInt64) ENGINE=ReplicatedMergeTree('/clickhouse/tables/$CLICKHOUSE_TEST_ZOOKEEPER_PREFIX/parallel_q', 'r1') ORDER BY tuple() SETTINGS old_parts_lifetime = 1, cleanup_delay_period = 0, cleanup_delay_period_random_add = 0" -$CLICKHOUSE_CLIENT -q "CREATE TABLE parallel_q2 (x UInt64) ENGINE=ReplicatedMergeTree('/clickhouse/tables/test_01509/parallel_q', 'r2') ORDER BY tuple() SETTINGS always_fetch_merged_part = 1" +$CLICKHOUSE_CLIENT -q "CREATE TABLE parallel_q2 (x UInt64) ENGINE=ReplicatedMergeTree('/clickhouse/tables/$CLICKHOUSE_TEST_ZOOKEEPER_PREFIX/parallel_q', 'r2') ORDER BY tuple() SETTINGS always_fetch_merged_part = 1" $CLICKHOUSE_CLIENT -q "SYSTEM STOP REPLICATION QUEUES parallel_q2" diff --git a/tests/queries/0_stateless/01593_concurrent_alter_mutations_kill.reference b/tests/queries/0_stateless/01593_concurrent_alter_mutations_kill.reference index 94e15c09768..4b07f533f5a 100644 --- a/tests/queries/0_stateless/01593_concurrent_alter_mutations_kill.reference +++ b/tests/queries/0_stateless/01593_concurrent_alter_mutations_kill.reference @@ -1,2 +1,2 @@ -CREATE TABLE default.concurrent_mutate_kill\n(\n `key` UInt64,\n `value` Int64\n)\nENGINE = ReplicatedMergeTree(\'/clickhouse/tables/test_01593/concurrent_mutate_kill\', \'1\')\nPARTITION BY key % 100\nORDER BY key\nSETTINGS max_replicated_mutations_in_queue = 1000, number_of_free_entries_in_pool_to_execute_mutation = 0, max_replicated_merges_in_queue = 1000, index_granularity = 8192 +CREATE TABLE default.concurrent_mutate_kill\n(\n `key` UInt64,\n `value` Int64\n)\nENGINE = ReplicatedMergeTree(\'/clickhouse/tables/01593_concurrent_alter_mutations_kill_default/concurrent_mutate_kill\', \'1\')\nPARTITION BY key % 100\nORDER BY key\nSETTINGS max_replicated_mutations_in_queue = 1000, number_of_free_entries_in_pool_to_execute_mutation = 0, max_replicated_merges_in_queue = 1000, index_granularity = 8192 499999500000 diff --git a/tests/queries/0_stateless/01593_concurrent_alter_mutations_kill.sh b/tests/queries/0_stateless/01593_concurrent_alter_mutations_kill.sh index 6ae103bdf6e..d40406222c2 100755 --- a/tests/queries/0_stateless/01593_concurrent_alter_mutations_kill.sh +++ b/tests/queries/0_stateless/01593_concurrent_alter_mutations_kill.sh @@ -6,7 +6,7 @@ CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) $CLICKHOUSE_CLIENT --query "DROP TABLE IF EXISTS concurrent_mutate_kill" -$CLICKHOUSE_CLIENT --query "CREATE TABLE concurrent_mutate_kill (key UInt64, value String) ENGINE = ReplicatedMergeTree('/clickhouse/tables/test_01593/concurrent_mutate_kill', '1') ORDER BY key PARTITION BY key % 100 SETTINGS max_replicated_mutations_in_queue=1000, number_of_free_entries_in_pool_to_execute_mutation=0,max_replicated_merges_in_queue=1000" +$CLICKHOUSE_CLIENT --query "CREATE TABLE concurrent_mutate_kill (key UInt64, value String) ENGINE = ReplicatedMergeTree('/clickhouse/tables/$CLICKHOUSE_TEST_ZOOKEEPER_PREFIX/concurrent_mutate_kill', '1') ORDER BY key PARTITION BY key % 100 SETTINGS max_replicated_mutations_in_queue=1000, number_of_free_entries_in_pool_to_execute_mutation=0,max_replicated_merges_in_queue=1000" $CLICKHOUSE_CLIENT --query "INSERT INTO concurrent_mutate_kill SELECT number, toString(number) FROM numbers(1000000)" diff --git a/tests/queries/0_stateless/01593_concurrent_alter_mutations_kill_many_replicas.reference b/tests/queries/0_stateless/01593_concurrent_alter_mutations_kill_many_replicas.reference index cb1eace24a2..80b9977aac7 100644 --- a/tests/queries/0_stateless/01593_concurrent_alter_mutations_kill_many_replicas.reference +++ b/tests/queries/0_stateless/01593_concurrent_alter_mutations_kill_many_replicas.reference @@ -4,13 +4,13 @@ 499999500000 499999500000 Metadata version on replica 1 equal with first replica, OK -CREATE TABLE default.concurrent_kill_1\n(\n `key` UInt64,\n `value` Int64\n)\nENGINE = ReplicatedMergeTree(\'/clickhouse/tables/test_01593_concurrent_kill\', \'1\')\nORDER BY key\nSETTINGS max_replicated_mutations_in_queue = 1000, number_of_free_entries_in_pool_to_execute_mutation = 0, max_replicated_merges_in_queue = 1000, index_granularity = 8192 +CREATE TABLE default.concurrent_kill_1\n(\n `key` UInt64,\n `value` Int64\n)\nENGINE = ReplicatedMergeTree(\'/clickhouse/tables/01593_concurrent_alter_mutations_kill_many_replicas_default\', \'1\')\nORDER BY key\nSETTINGS max_replicated_mutations_in_queue = 1000, number_of_free_entries_in_pool_to_execute_mutation = 0, max_replicated_merges_in_queue = 1000, index_granularity = 8192 Metadata version on replica 2 equal with first replica, OK -CREATE TABLE default.concurrent_kill_2\n(\n `key` UInt64,\n `value` Int64\n)\nENGINE = ReplicatedMergeTree(\'/clickhouse/tables/test_01593_concurrent_kill\', \'2\')\nORDER BY key\nSETTINGS max_replicated_mutations_in_queue = 1000, number_of_free_entries_in_pool_to_execute_mutation = 0, max_replicated_merges_in_queue = 1000, index_granularity = 8192 +CREATE TABLE default.concurrent_kill_2\n(\n `key` UInt64,\n `value` Int64\n)\nENGINE = ReplicatedMergeTree(\'/clickhouse/tables/01593_concurrent_alter_mutations_kill_many_replicas_default\', \'2\')\nORDER BY key\nSETTINGS max_replicated_mutations_in_queue = 1000, number_of_free_entries_in_pool_to_execute_mutation = 0, max_replicated_merges_in_queue = 1000, index_granularity = 8192 Metadata version on replica 3 equal with first replica, OK -CREATE TABLE default.concurrent_kill_3\n(\n `key` UInt64,\n `value` Int64\n)\nENGINE = ReplicatedMergeTree(\'/clickhouse/tables/test_01593_concurrent_kill\', \'3\')\nORDER BY key\nSETTINGS max_replicated_mutations_in_queue = 1000, number_of_free_entries_in_pool_to_execute_mutation = 0, max_replicated_merges_in_queue = 1000, index_granularity = 8192 +CREATE TABLE default.concurrent_kill_3\n(\n `key` UInt64,\n `value` Int64\n)\nENGINE = ReplicatedMergeTree(\'/clickhouse/tables/01593_concurrent_alter_mutations_kill_many_replicas_default\', \'3\')\nORDER BY key\nSETTINGS max_replicated_mutations_in_queue = 1000, number_of_free_entries_in_pool_to_execute_mutation = 0, max_replicated_merges_in_queue = 1000, index_granularity = 8192 Metadata version on replica 4 equal with first replica, OK -CREATE TABLE default.concurrent_kill_4\n(\n `key` UInt64,\n `value` Int64\n)\nENGINE = ReplicatedMergeTree(\'/clickhouse/tables/test_01593_concurrent_kill\', \'4\')\nORDER BY key\nSETTINGS max_replicated_mutations_in_queue = 1000, number_of_free_entries_in_pool_to_execute_mutation = 0, max_replicated_merges_in_queue = 1000, index_granularity = 8192 +CREATE TABLE default.concurrent_kill_4\n(\n `key` UInt64,\n `value` Int64\n)\nENGINE = ReplicatedMergeTree(\'/clickhouse/tables/01593_concurrent_alter_mutations_kill_many_replicas_default\', \'4\')\nORDER BY key\nSETTINGS max_replicated_mutations_in_queue = 1000, number_of_free_entries_in_pool_to_execute_mutation = 0, max_replicated_merges_in_queue = 1000, index_granularity = 8192 Metadata version on replica 5 equal with first replica, OK -CREATE TABLE default.concurrent_kill_5\n(\n `key` UInt64,\n `value` Int64\n)\nENGINE = ReplicatedMergeTree(\'/clickhouse/tables/test_01593_concurrent_kill\', \'5\')\nORDER BY key\nSETTINGS max_replicated_mutations_in_queue = 1000, number_of_free_entries_in_pool_to_execute_mutation = 0, max_replicated_merges_in_queue = 1000, index_granularity = 8192 +CREATE TABLE default.concurrent_kill_5\n(\n `key` UInt64,\n `value` Int64\n)\nENGINE = ReplicatedMergeTree(\'/clickhouse/tables/01593_concurrent_alter_mutations_kill_many_replicas_default\', \'5\')\nORDER BY key\nSETTINGS max_replicated_mutations_in_queue = 1000, number_of_free_entries_in_pool_to_execute_mutation = 0, max_replicated_merges_in_queue = 1000, index_granularity = 8192 499999500000 diff --git a/tests/queries/0_stateless/01593_concurrent_alter_mutations_kill_many_replicas.sh b/tests/queries/0_stateless/01593_concurrent_alter_mutations_kill_many_replicas.sh index bfa68328c06..4cb3fd35294 100755 --- a/tests/queries/0_stateless/01593_concurrent_alter_mutations_kill_many_replicas.sh +++ b/tests/queries/0_stateless/01593_concurrent_alter_mutations_kill_many_replicas.sh @@ -11,7 +11,7 @@ for i in $(seq $REPLICAS); do done for i in $(seq $REPLICAS); do - $CLICKHOUSE_CLIENT --query "CREATE TABLE concurrent_kill_$i (key UInt64, value String) ENGINE = ReplicatedMergeTree('/clickhouse/tables/test_01593_concurrent_kill', '$i') ORDER BY key SETTINGS max_replicated_mutations_in_queue=1000, number_of_free_entries_in_pool_to_execute_mutation=0,max_replicated_merges_in_queue=1000" + $CLICKHOUSE_CLIENT --query "CREATE TABLE concurrent_kill_$i (key UInt64, value String) ENGINE = ReplicatedMergeTree('/clickhouse/tables/$CLICKHOUSE_TEST_ZOOKEEPER_PREFIX', '$i') ORDER BY key SETTINGS max_replicated_mutations_in_queue=1000, number_of_free_entries_in_pool_to_execute_mutation=0,max_replicated_merges_in_queue=1000" done $CLICKHOUSE_CLIENT --query "INSERT INTO concurrent_kill_1 SELECT number, toString(number) FROM numbers(1000000)" @@ -77,9 +77,9 @@ while true; do done -metadata_version=$($CLICKHOUSE_CLIENT --query "SELECT value FROM system.zookeeper WHERE path = '/clickhouse/tables/test_01593_concurrent_kill/replicas/$i/' and name = 'metadata_version'") +metadata_version=$($CLICKHOUSE_CLIENT --query "SELECT value FROM system.zookeeper WHERE path = '/clickhouse/tables/$CLICKHOUSE_TEST_ZOOKEEPER_PREFIX/replicas/$i/' and name = 'metadata_version'") for i in $(seq $REPLICAS); do - replica_metadata_version=$($CLICKHOUSE_CLIENT --query "SELECT value FROM system.zookeeper WHERE path = '/clickhouse/tables/test_01593_concurrent_kill/replicas/$i/' and name = 'metadata_version'") + replica_metadata_version=$($CLICKHOUSE_CLIENT --query "SELECT value FROM system.zookeeper WHERE path = '/clickhouse/tables/$CLICKHOUSE_TEST_ZOOKEEPER_PREFIX/replicas/$i/' and name = 'metadata_version'") if [ "$metadata_version" != "$replica_metadata_version" ]; then echo "Metadata version on replica $i differs from the first replica, FAIL" else diff --git a/tests/queries/0_stateless/01671_ddl_hang_timeout.sh b/tests/queries/0_stateless/01671_ddl_hang_timeout.sh index 2ca97e3978b..641eba2d8fa 100755 --- a/tests/queries/0_stateless/01671_ddl_hang_timeout.sh +++ b/tests/queries/0_stateless/01671_ddl_hang_timeout.sh @@ -6,7 +6,7 @@ CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) function thread_create_drop_table { while true; do REPLICA=$(($RANDOM % 10)) - $CLICKHOUSE_CLIENT --query "CREATE TABLE IF NOT EXISTS t1 (x UInt64, s Array(Nullable(String))) ENGINE = ReplicatedMergeTree('/clickhouse/tables/test_01671/test_01671', 'r_$REPLICA') order by x" 2>/dev/null + $CLICKHOUSE_CLIENT --query "CREATE TABLE IF NOT EXISTS t1 (x UInt64, s Array(Nullable(String))) ENGINE = ReplicatedMergeTree('/clickhouse/tables/$CLICKHOUSE_TEST_ZOOKEEPER_PREFIX/test_01671', 'r_$REPLICA') order by x" 2>/dev/null sleep 0.0$RANDOM $CLICKHOUSE_CLIENT --query "DROP TABLE IF EXISTS t1" done diff --git a/tests/queries/0_stateless/01753_system_zookeeper_query_param_path.sh b/tests/queries/0_stateless/01753_system_zookeeper_query_param_path.sh index 1f4ba412a19..d3046e73b93 100755 --- a/tests/queries/0_stateless/01753_system_zookeeper_query_param_path.sh +++ b/tests/queries/0_stateless/01753_system_zookeeper_query_param_path.sh @@ -6,9 +6,9 @@ CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) ${CLICKHOUSE_CLIENT} --query="DROP TABLE IF EXISTS test_01753"; -${CLICKHOUSE_CLIENT} --query="CREATE TABLE test_01753 (n Int8) ENGINE=ReplicatedMergeTree('/$CLICKHOUSE_DATABASE/test_01753/test', '1') ORDER BY n" +${CLICKHOUSE_CLIENT} --query="CREATE TABLE test_01753 (n Int8) ENGINE=ReplicatedMergeTree('/$CLICKHOUSE_TEST_ZOOKEEPER_PREFIX/test_01753/test', '1') ORDER BY n" -${CLICKHOUSE_CLIENT} --query="SELECT name FROM system.zookeeper WHERE path = {path:String}" --param_path "$CLICKHOUSE_DATABASE/test_01753" +${CLICKHOUSE_CLIENT} --query="SELECT name FROM system.zookeeper WHERE path = {path:String}" --param_path "$CLICKHOUSE_TEST_ZOOKEEPER_PREFIX/test_01753" ${CLICKHOUSE_CLIENT} --query="DROP TABLE test_01753 SYNC"; From a03a9051e1882adbf43b8ea23c6ef7a38c92cd98 Mon Sep 17 00:00:00 2001 From: Azat Khuzhin Date: Mon, 15 Mar 2021 21:03:24 +0300 Subject: [PATCH 501/544] Mark some tests as long From [1] and [2]: [1]: https://clickhouse-test-reports.s3.yandex.net/21724/dacb6066237b78511ad5f07cc65045b8cf4e5e7b/functional_stateless_tests_flaky_check_(address).html#fail1 [2]: https://clickhouse-test-reports.s3.yandex.net/21724/ee8ece157987f3cd8b68ca4a5b7df709f4688208/functional_stateless_tests_flaky_check_(address)/test_run.txt.out.log [3]: https://clickhouse-test-reports.s3.yandex.net/21724/e4485ce0b1d1b21c4360bbdecacd2485d1634a91/functional_stateless_tests_flaky_check_(address)/test_run.txt.out.log --- ...75_indices_mutation_replicated_zookeeper_long.reference} | 0 ... => 00975_indices_mutation_replicated_zookeeper_long.sh} | 0 ...92_system_parts_race_condition_zookeeper_long.reference} | 0 ... => 00992_system_parts_race_condition_zookeeper_long.sh} | 0 ....reference => 01079_bad_alters_zookeeper_long.reference} | 4 ++-- ...ters_zookeeper.sh => 01079_bad_alters_zookeeper_long.sh} | 0 ...=> 01079_parallel_alter_modify_zookeeper_long.reference} | 0 ...per.sh => 01079_parallel_alter_modify_zookeeper_long.sh} | 0 ...08_race_condition_rename_clear_zookeeper_long.reference} | 0 ... => 01508_race_condition_rename_clear_zookeeper_long.sh} | 0 ...01509_check_many_parallel_quorum_inserts_long.reference} | 0 ....sh => 01509_check_many_parallel_quorum_inserts_long.sh} | 0 ...e => 01509_check_parallel_quorum_inserts_long.reference} | 0 ...serts.sh => 01509_check_parallel_quorum_inserts_long.sh} | 0 ...rence => 01509_parallel_quorum_and_merge_long.reference} | 0 ...and_merge.sh => 01509_parallel_quorum_and_merge_long.sh} | 0 ...eout.reference => 01671_ddl_hang_timeout_long.reference} | 0 ...1_ddl_hang_timeout.sh => 01671_ddl_hang_timeout_long.sh} | 0 ... 01753_system_zookeeper_query_param_path_long.reference} | 0 ...h.sh => 01753_system_zookeeper_query_param_path_long.sh} | 0 tests/queries/skip_list.json | 6 +++--- 21 files changed, 5 insertions(+), 5 deletions(-) rename tests/queries/0_stateless/{00975_indices_mutation_replicated_zookeeper.reference => 00975_indices_mutation_replicated_zookeeper_long.reference} (100%) rename tests/queries/0_stateless/{00975_indices_mutation_replicated_zookeeper.sh => 00975_indices_mutation_replicated_zookeeper_long.sh} (100%) rename tests/queries/0_stateless/{00992_system_parts_race_condition_zookeeper.reference => 00992_system_parts_race_condition_zookeeper_long.reference} (100%) rename tests/queries/0_stateless/{00992_system_parts_race_condition_zookeeper.sh => 00992_system_parts_race_condition_zookeeper_long.sh} (100%) rename tests/queries/0_stateless/{01079_bad_alters_zookeeper.reference => 01079_bad_alters_zookeeper_long.reference} (52%) rename tests/queries/0_stateless/{01079_bad_alters_zookeeper.sh => 01079_bad_alters_zookeeper_long.sh} (100%) rename tests/queries/0_stateless/{01079_parallel_alter_modify_zookeeper.reference => 01079_parallel_alter_modify_zookeeper_long.reference} (100%) rename tests/queries/0_stateless/{01079_parallel_alter_modify_zookeeper.sh => 01079_parallel_alter_modify_zookeeper_long.sh} (100%) rename tests/queries/0_stateless/{01508_race_condition_rename_clear_zookeeper.reference => 01508_race_condition_rename_clear_zookeeper_long.reference} (100%) rename tests/queries/0_stateless/{01508_race_condition_rename_clear_zookeeper.sh => 01508_race_condition_rename_clear_zookeeper_long.sh} (100%) rename tests/queries/0_stateless/{01509_check_many_parallel_quorum_inserts.reference => 01509_check_many_parallel_quorum_inserts_long.reference} (100%) rename tests/queries/0_stateless/{01509_check_many_parallel_quorum_inserts.sh => 01509_check_many_parallel_quorum_inserts_long.sh} (100%) rename tests/queries/0_stateless/{01509_check_parallel_quorum_inserts.reference => 01509_check_parallel_quorum_inserts_long.reference} (100%) rename tests/queries/0_stateless/{01509_check_parallel_quorum_inserts.sh => 01509_check_parallel_quorum_inserts_long.sh} (100%) rename tests/queries/0_stateless/{01509_parallel_quorum_and_merge.reference => 01509_parallel_quorum_and_merge_long.reference} (100%) rename tests/queries/0_stateless/{01509_parallel_quorum_and_merge.sh => 01509_parallel_quorum_and_merge_long.sh} (100%) rename tests/queries/0_stateless/{01671_ddl_hang_timeout.reference => 01671_ddl_hang_timeout_long.reference} (100%) rename tests/queries/0_stateless/{01671_ddl_hang_timeout.sh => 01671_ddl_hang_timeout_long.sh} (100%) rename tests/queries/0_stateless/{01753_system_zookeeper_query_param_path.reference => 01753_system_zookeeper_query_param_path_long.reference} (100%) rename tests/queries/0_stateless/{01753_system_zookeeper_query_param_path.sh => 01753_system_zookeeper_query_param_path_long.sh} (100%) diff --git a/tests/queries/0_stateless/00975_indices_mutation_replicated_zookeeper.reference b/tests/queries/0_stateless/00975_indices_mutation_replicated_zookeeper_long.reference similarity index 100% rename from tests/queries/0_stateless/00975_indices_mutation_replicated_zookeeper.reference rename to tests/queries/0_stateless/00975_indices_mutation_replicated_zookeeper_long.reference diff --git a/tests/queries/0_stateless/00975_indices_mutation_replicated_zookeeper.sh b/tests/queries/0_stateless/00975_indices_mutation_replicated_zookeeper_long.sh similarity index 100% rename from tests/queries/0_stateless/00975_indices_mutation_replicated_zookeeper.sh rename to tests/queries/0_stateless/00975_indices_mutation_replicated_zookeeper_long.sh diff --git a/tests/queries/0_stateless/00992_system_parts_race_condition_zookeeper.reference b/tests/queries/0_stateless/00992_system_parts_race_condition_zookeeper_long.reference similarity index 100% rename from tests/queries/0_stateless/00992_system_parts_race_condition_zookeeper.reference rename to tests/queries/0_stateless/00992_system_parts_race_condition_zookeeper_long.reference diff --git a/tests/queries/0_stateless/00992_system_parts_race_condition_zookeeper.sh b/tests/queries/0_stateless/00992_system_parts_race_condition_zookeeper_long.sh similarity index 100% rename from tests/queries/0_stateless/00992_system_parts_race_condition_zookeeper.sh rename to tests/queries/0_stateless/00992_system_parts_race_condition_zookeeper_long.sh diff --git a/tests/queries/0_stateless/01079_bad_alters_zookeeper.reference b/tests/queries/0_stateless/01079_bad_alters_zookeeper_long.reference similarity index 52% rename from tests/queries/0_stateless/01079_bad_alters_zookeeper.reference rename to tests/queries/0_stateless/01079_bad_alters_zookeeper_long.reference index 67510b28a34..731cd871b3b 100644 --- a/tests/queries/0_stateless/01079_bad_alters_zookeeper.reference +++ b/tests/queries/0_stateless/01079_bad_alters_zookeeper_long.reference @@ -1,6 +1,6 @@ Wrong column name. -CREATE TABLE default.table_for_bad_alters\n(\n `key` UInt64,\n `value1` UInt8,\n `value2` String\n)\nENGINE = ReplicatedMergeTree(\'/clickhouse/tables/01079_bad_alters_zookeeper_default/table_for_bad_alters\', \'1\')\nORDER BY key\nSETTINGS index_granularity = 8192 -CREATE TABLE default.table_for_bad_alters\n(\n `key` UInt64,\n `value1` UInt8,\n `value2` UInt32\n)\nENGINE = ReplicatedMergeTree(\'/clickhouse/tables/01079_bad_alters_zookeeper_default/table_for_bad_alters\', \'1\')\nORDER BY key\nSETTINGS index_granularity = 8192 +CREATE TABLE default.table_for_bad_alters\n(\n `key` UInt64,\n `value1` UInt8,\n `value2` String\n)\nENGINE = ReplicatedMergeTree(\'/clickhouse/tables/01079_bad_alters_zookeeper_long_default/table_for_bad_alters\', \'1\')\nORDER BY key\nSETTINGS index_granularity = 8192 +CREATE TABLE default.table_for_bad_alters\n(\n `key` UInt64,\n `value1` UInt8,\n `value2` UInt32\n)\nENGINE = ReplicatedMergeTree(\'/clickhouse/tables/01079_bad_alters_zookeeper_long_default/table_for_bad_alters\', \'1\')\nORDER BY key\nSETTINGS index_granularity = 8192 syntax error at begin of string. 7 Hello diff --git a/tests/queries/0_stateless/01079_bad_alters_zookeeper.sh b/tests/queries/0_stateless/01079_bad_alters_zookeeper_long.sh similarity index 100% rename from tests/queries/0_stateless/01079_bad_alters_zookeeper.sh rename to tests/queries/0_stateless/01079_bad_alters_zookeeper_long.sh diff --git a/tests/queries/0_stateless/01079_parallel_alter_modify_zookeeper.reference b/tests/queries/0_stateless/01079_parallel_alter_modify_zookeeper_long.reference similarity index 100% rename from tests/queries/0_stateless/01079_parallel_alter_modify_zookeeper.reference rename to tests/queries/0_stateless/01079_parallel_alter_modify_zookeeper_long.reference diff --git a/tests/queries/0_stateless/01079_parallel_alter_modify_zookeeper.sh b/tests/queries/0_stateless/01079_parallel_alter_modify_zookeeper_long.sh similarity index 100% rename from tests/queries/0_stateless/01079_parallel_alter_modify_zookeeper.sh rename to tests/queries/0_stateless/01079_parallel_alter_modify_zookeeper_long.sh diff --git a/tests/queries/0_stateless/01508_race_condition_rename_clear_zookeeper.reference b/tests/queries/0_stateless/01508_race_condition_rename_clear_zookeeper_long.reference similarity index 100% rename from tests/queries/0_stateless/01508_race_condition_rename_clear_zookeeper.reference rename to tests/queries/0_stateless/01508_race_condition_rename_clear_zookeeper_long.reference diff --git a/tests/queries/0_stateless/01508_race_condition_rename_clear_zookeeper.sh b/tests/queries/0_stateless/01508_race_condition_rename_clear_zookeeper_long.sh similarity index 100% rename from tests/queries/0_stateless/01508_race_condition_rename_clear_zookeeper.sh rename to tests/queries/0_stateless/01508_race_condition_rename_clear_zookeeper_long.sh diff --git a/tests/queries/0_stateless/01509_check_many_parallel_quorum_inserts.reference b/tests/queries/0_stateless/01509_check_many_parallel_quorum_inserts_long.reference similarity index 100% rename from tests/queries/0_stateless/01509_check_many_parallel_quorum_inserts.reference rename to tests/queries/0_stateless/01509_check_many_parallel_quorum_inserts_long.reference diff --git a/tests/queries/0_stateless/01509_check_many_parallel_quorum_inserts.sh b/tests/queries/0_stateless/01509_check_many_parallel_quorum_inserts_long.sh similarity index 100% rename from tests/queries/0_stateless/01509_check_many_parallel_quorum_inserts.sh rename to tests/queries/0_stateless/01509_check_many_parallel_quorum_inserts_long.sh diff --git a/tests/queries/0_stateless/01509_check_parallel_quorum_inserts.reference b/tests/queries/0_stateless/01509_check_parallel_quorum_inserts_long.reference similarity index 100% rename from tests/queries/0_stateless/01509_check_parallel_quorum_inserts.reference rename to tests/queries/0_stateless/01509_check_parallel_quorum_inserts_long.reference diff --git a/tests/queries/0_stateless/01509_check_parallel_quorum_inserts.sh b/tests/queries/0_stateless/01509_check_parallel_quorum_inserts_long.sh similarity index 100% rename from tests/queries/0_stateless/01509_check_parallel_quorum_inserts.sh rename to tests/queries/0_stateless/01509_check_parallel_quorum_inserts_long.sh diff --git a/tests/queries/0_stateless/01509_parallel_quorum_and_merge.reference b/tests/queries/0_stateless/01509_parallel_quorum_and_merge_long.reference similarity index 100% rename from tests/queries/0_stateless/01509_parallel_quorum_and_merge.reference rename to tests/queries/0_stateless/01509_parallel_quorum_and_merge_long.reference diff --git a/tests/queries/0_stateless/01509_parallel_quorum_and_merge.sh b/tests/queries/0_stateless/01509_parallel_quorum_and_merge_long.sh similarity index 100% rename from tests/queries/0_stateless/01509_parallel_quorum_and_merge.sh rename to tests/queries/0_stateless/01509_parallel_quorum_and_merge_long.sh diff --git a/tests/queries/0_stateless/01671_ddl_hang_timeout.reference b/tests/queries/0_stateless/01671_ddl_hang_timeout_long.reference similarity index 100% rename from tests/queries/0_stateless/01671_ddl_hang_timeout.reference rename to tests/queries/0_stateless/01671_ddl_hang_timeout_long.reference diff --git a/tests/queries/0_stateless/01671_ddl_hang_timeout.sh b/tests/queries/0_stateless/01671_ddl_hang_timeout_long.sh similarity index 100% rename from tests/queries/0_stateless/01671_ddl_hang_timeout.sh rename to tests/queries/0_stateless/01671_ddl_hang_timeout_long.sh diff --git a/tests/queries/0_stateless/01753_system_zookeeper_query_param_path.reference b/tests/queries/0_stateless/01753_system_zookeeper_query_param_path_long.reference similarity index 100% rename from tests/queries/0_stateless/01753_system_zookeeper_query_param_path.reference rename to tests/queries/0_stateless/01753_system_zookeeper_query_param_path_long.reference diff --git a/tests/queries/0_stateless/01753_system_zookeeper_query_param_path.sh b/tests/queries/0_stateless/01753_system_zookeeper_query_param_path_long.sh similarity index 100% rename from tests/queries/0_stateless/01753_system_zookeeper_query_param_path.sh rename to tests/queries/0_stateless/01753_system_zookeeper_query_param_path_long.sh diff --git a/tests/queries/skip_list.json b/tests/queries/skip_list.json index 974ef48ef3c..9501343754c 100644 --- a/tests/queries/skip_list.json +++ b/tests/queries/skip_list.json @@ -307,7 +307,7 @@ "00954_client_prepared_statements", "00956_sensitive_data_masking", "00969_columns_clause", - "00975_indices_mutation_replicated_zookeeper", + "00975_indices_mutation_replicated_zookeeper_long", "00975_values_list", "00976_system_stop_ttl_merges", "00977_int_div", @@ -442,8 +442,8 @@ "01504_compression_multiple_streams", "01508_explain_header", "01508_partition_pruning_long", - "01509_check_parallel_quorum_inserts", - "01509_parallel_quorum_and_merge", + "01509_check_parallel_quorum_inserts_long", + "01509_parallel_quorum_and_merge_long", "01515_mv_and_array_join_optimisation_bag", "01516_create_table_primary_key", "01517_drop_mv_with_inner_table", From 0bb665428d166d1e589886b62a3f645d6a5dc5d6 Mon Sep 17 00:00:00 2001 From: Vitaliy Fedorchenko Date: Tue, 16 Mar 2021 09:05:08 +0200 Subject: [PATCH 502/544] Update gui.md: add SeekTable --- docs/en/interfaces/third-party/gui.md | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/docs/en/interfaces/third-party/gui.md b/docs/en/interfaces/third-party/gui.md index fa123d8b23d..5d14b3aa3cc 100644 --- a/docs/en/interfaces/third-party/gui.md +++ b/docs/en/interfaces/third-party/gui.md @@ -167,4 +167,21 @@ Features: [How to configure ClickHouse in Looker.](https://docs.looker.com/setup-and-management/database-config/clickhouse) +### SeekTable {#seektable} + +[SeekTable](https://www.seektable.com) is a self-service BI tool for data exploration and operational reporting. SeekTable is available both as a cloud service and a self-hosted version. SeekTable reports may be embedded into any web-app. + +Features: + +- Business users-friendly reports builder. +- Powerful report parameters for SQL filtering and report-specific query customizations. +- Can connect to ClickHouse both with a native TCP/IP endpoint and a HTTP(S) interface (2 different drivers). +- It is possible to use all power of CH SQL dialect in dimensions/measures definitions +- [Web API](https://www.seektable.com/help/web-api-integration) for automated reports generation. +- Supports reports development flow with account data [backup/restore](https://www.seektable.com/help/self-hosted-backup-restore), data models (cubes) / reports configuration is a human-readable XML and can be stored under version control. + +SeekTable is [free](https://www.seektable.com/help/cloud-pricing) for personal/individual usage. + +[How to configure ClickHouse connection in SeekTable.](https://www.seektable.com/help/clickhouse-pivot-table) + [Original article](https://clickhouse.tech/docs/en/interfaces/third-party/gui/) From 27b8d10bd8edcd0778ae06c2f480bf5762db4cd6 Mon Sep 17 00:00:00 2001 From: Ali Demirci Date: Tue, 16 Mar 2021 13:30:05 +0300 Subject: [PATCH 503/544] docs(fix): typo --- docs/en/sql-reference/functions/machine-learning-functions.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/en/sql-reference/functions/machine-learning-functions.md b/docs/en/sql-reference/functions/machine-learning-functions.md index c9abd130ef3..60dabd73781 100644 --- a/docs/en/sql-reference/functions/machine-learning-functions.md +++ b/docs/en/sql-reference/functions/machine-learning-functions.md @@ -9,7 +9,7 @@ toc_title: Machine Learning Prediction using fitted regression models uses `evalMLMethod` function. See link in `linearRegression`. -## stochasticLinearRegressionn {#stochastic-linear-regression} +## stochasticLinearRegression {#stochastic-linear-regression} The [stochasticLinearRegression](../../sql-reference/aggregate-functions/reference/stochasticlinearregression.md#agg_functions-stochasticlinearregression) aggregate function implements stochastic gradient descent method using linear model and MSE loss function. Uses `evalMLMethod` to predict on new data. From eadf0248d0e9cf57be70f5707277e2cda3c5855e Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Tue, 16 Mar 2021 14:07:53 +0300 Subject: [PATCH 504/544] Build fixes --- base/common/LocalDateTime.h | 24 +++++- base/mysqlxx/tests/CMakeLists.txt | 3 - base/mysqlxx/tests/failover.xml | 21 ----- base/mysqlxx/tests/mysqlxx_test.cpp | 77 ------------------- programs/odbc-bridge/ODBCBlockInputStream.cpp | 4 +- .../odbc-bridge/ODBCBlockOutputStream.cpp | 2 +- src/Storages/tests/CMakeLists.txt | 3 - src/Storages/tests/part_name.cpp | 21 ----- utils/wikistat-loader/main.cpp | 2 +- 9 files changed, 27 insertions(+), 130 deletions(-) delete mode 100644 base/mysqlxx/tests/failover.xml delete mode 100644 base/mysqlxx/tests/mysqlxx_test.cpp delete mode 100644 src/Storages/tests/part_name.cpp diff --git a/base/common/LocalDateTime.h b/base/common/LocalDateTime.h index 4c2cf0e637d..dde283e5ebb 100644 --- a/base/common/LocalDateTime.h +++ b/base/common/LocalDateTime.h @@ -106,8 +106,30 @@ public: void second(unsigned char x) { m_second = x; } LocalDate toDate() const { return LocalDate(m_year, m_month, m_day); } + LocalDateTime toStartOfDate() const { return LocalDateTime(m_year, m_month, m_day, 0, 0, 0); } - LocalDateTime toStartOfDate() { return LocalDateTime(m_year, m_month, m_day, 0, 0, 0); } + std::string toString() const + { + std::string s{"0000-00-00 00:00:00"}; + + s[0] += m_year / 1000; + s[1] += (m_year / 100) % 10; + s[2] += (m_year / 10) % 10; + s[3] += m_year % 10; + s[5] += m_month / 10; + s[6] += m_month % 10; + s[8] += m_day / 10; + s[9] += m_day % 10; + + s[11] += m_hour / 10; + s[12] += m_hour % 10; + s[14] += m_minute / 10; + s[15] += m_minute % 10; + s[17] += m_second / 10; + s[18] += m_second % 10; + + return s; + } bool operator< (const LocalDateTime & other) const { diff --git a/base/mysqlxx/tests/CMakeLists.txt b/base/mysqlxx/tests/CMakeLists.txt index 2cf19d78418..6473a927308 100644 --- a/base/mysqlxx/tests/CMakeLists.txt +++ b/base/mysqlxx/tests/CMakeLists.txt @@ -1,5 +1,2 @@ -add_executable (mysqlxx_test mysqlxx_test.cpp) -target_link_libraries (mysqlxx_test PRIVATE mysqlxx) - add_executable (mysqlxx_pool_test mysqlxx_pool_test.cpp) target_link_libraries (mysqlxx_pool_test PRIVATE mysqlxx) diff --git a/base/mysqlxx/tests/failover.xml b/base/mysqlxx/tests/failover.xml deleted file mode 100644 index 73702eabb29..00000000000 --- a/base/mysqlxx/tests/failover.xml +++ /dev/null @@ -1,21 +0,0 @@ - - - - 3306 - root - Metrica - qwerty - - example02t - 0 - - - example02t - 3306 - root - qwerty - Metrica - 1 - - - diff --git a/base/mysqlxx/tests/mysqlxx_test.cpp b/base/mysqlxx/tests/mysqlxx_test.cpp deleted file mode 100644 index c505d34a58d..00000000000 --- a/base/mysqlxx/tests/mysqlxx_test.cpp +++ /dev/null @@ -1,77 +0,0 @@ -#include -#include - - -int main(int, char **) -{ - try - { - mysqlxx::Connection connection("test", "127.0.0.1", "root", "qwerty", 3306); - std::cerr << "Connected." << std::endl; - - { - mysqlxx::Query query = connection.query(); - query << "SELECT 1 x, '2010-01-01 01:01:01' d"; - mysqlxx::UseQueryResult result = query.use(); - std::cerr << "use() called." << std::endl; - - while (mysqlxx::Row row = result.fetch()) - { - std::cerr << "Fetched row." << std::endl; - std::cerr << row[0] << ", " << row["x"] << std::endl; - std::cerr << row[1] << ", " << row["d"] - << ", " << row[1].getDate() - << ", " << row[1].getDateTime() - << ", " << row[1].getDate() - << ", " << row[1].getDateTime() - << std::endl - << row[1].getDate() << ", " << row[1].getDateTime() << std::endl - << row[1].getDate() << ", " << row[1].getDateTime() << std::endl - << row[1].getDate() << ", " << row[1].getDateTime() << std::endl - << row[1].getDate() << ", " << row[1].getDateTime() << std::endl - ; - - time_t t1 = row[0]; - time_t t2 = row[1]; - std::cerr << t1 << ", " << LocalDateTime(t1) << std::endl; - std::cerr << t2 << ", " << LocalDateTime(t2) << std::endl; - } - } - - { - mysqlxx::UseQueryResult result = connection.query("SELECT 'abc\\\\def' x").use(); - mysqlxx::Row row = result.fetch(); - std::cerr << row << std::endl; - std::cerr << row << std::endl; - } - - { - /// Копирование Query - mysqlxx::Query query1 = connection.query("SELECT"); - mysqlxx::Query query2 = query1; - query2 << " 1"; - - std::cerr << query1.str() << ", " << query2.str() << std::endl; - } - - { - /// NULL - mysqlxx::Null x = mysqlxx::null; - std::cerr << (x == mysqlxx::null ? "Ok" : "Fail") << std::endl; - std::cerr << (x == 0 ? "Fail" : "Ok") << std::endl; - std::cerr << (x.isNull() ? "Ok" : "Fail") << std::endl; - x = 1; - std::cerr << (x == mysqlxx::null ? "Fail" : "Ok") << std::endl; - std::cerr << (x == 0 ? "Fail" : "Ok") << std::endl; - std::cerr << (x == 1 ? "Ok" : "Fail") << std::endl; - std::cerr << (x.isNull() ? "Fail" : "Ok") << std::endl; - } - } - catch (const mysqlxx::Exception & e) - { - std::cerr << e.code() << ", " << e.message() << std::endl; - throw; - } - - return 0; -} diff --git a/programs/odbc-bridge/ODBCBlockInputStream.cpp b/programs/odbc-bridge/ODBCBlockInputStream.cpp index 3e2a2d0c7d4..b8a4209ac94 100644 --- a/programs/odbc-bridge/ODBCBlockInputStream.cpp +++ b/programs/odbc-bridge/ODBCBlockInputStream.cpp @@ -87,8 +87,8 @@ namespace case ValueType::vtDateTime: { Poco::DateTime datetime = value.convert(); - assert_cast(column).insertValue(time_t{LocalDateTime( - datetime.year(), datetime.month(), datetime.day(), datetime.hour(), datetime.minute(), datetime.second())}); + assert_cast(column).insertValue(DateLUT::instance().makeDateTime( + datetime.year(), datetime.month(), datetime.day(), datetime.hour(), datetime.minute(), datetime.second())); break; } case ValueType::vtUUID: diff --git a/programs/odbc-bridge/ODBCBlockOutputStream.cpp b/programs/odbc-bridge/ODBCBlockOutputStream.cpp index 4d8b9fa6bdf..db3c9441419 100644 --- a/programs/odbc-bridge/ODBCBlockOutputStream.cpp +++ b/programs/odbc-bridge/ODBCBlockOutputStream.cpp @@ -81,7 +81,7 @@ namespace case ValueType::vtDate: return Poco::Dynamic::Var(LocalDate(DayNum(field.get())).toString()).convert(); case ValueType::vtDateTime: - return Poco::Dynamic::Var(std::to_string(LocalDateTime(time_t(field.get())))).convert(); + return Poco::Dynamic::Var(DateLUT::instance().timeToString(time_t(field.get()))).convert(); case ValueType::vtUUID: return Poco::Dynamic::Var(UUID(field.get()).toUnderType().toHexString()).convert(); default: diff --git a/src/Storages/tests/CMakeLists.txt b/src/Storages/tests/CMakeLists.txt index b58fed9edf5..59d44829363 100644 --- a/src/Storages/tests/CMakeLists.txt +++ b/src/Storages/tests/CMakeLists.txt @@ -1,6 +1,3 @@ -add_executable (part_name part_name.cpp) -target_link_libraries (part_name PRIVATE dbms) - add_executable (remove_symlink_directory remove_symlink_directory.cpp) target_link_libraries (remove_symlink_directory PRIVATE dbms) diff --git a/src/Storages/tests/part_name.cpp b/src/Storages/tests/part_name.cpp deleted file mode 100644 index 227e19cf17c..00000000000 --- a/src/Storages/tests/part_name.cpp +++ /dev/null @@ -1,21 +0,0 @@ -#include -#include -#include - - -int main(int, char **) -{ - const DayNum today(DateLUT::instance().toDayNum(time(nullptr)).toUnderType()); - - for (DayNum date = today; DayNum(date + 10) > today; --date) - { - DB::MergeTreePartInfo part_info("partition", 0, 0, 0); - std::string name = part_info.getPartNameV0(date, date); - std::cerr << name << '\n'; - - time_t time = DateLUT::instance().YYYYMMDDToDate(DB::parse(name)); - std::cerr << LocalDateTime(time) << '\n'; - } - - return 0; -} diff --git a/utils/wikistat-loader/main.cpp b/utils/wikistat-loader/main.cpp index f2adcc43a3a..31ade014c74 100644 --- a/utils/wikistat-loader/main.cpp +++ b/utils/wikistat-loader/main.cpp @@ -151,7 +151,7 @@ try std::string time_str = options.at("time").as(); LocalDateTime time(time_str); - LocalDate date(time); + LocalDate date(time_str); DB::ReadBufferFromFileDescriptor in(STDIN_FILENO); DB::WriteBufferFromFileDescriptor out(STDOUT_FILENO); From f8a99804c178487ab2e7ca3e391cc7b823877054 Mon Sep 17 00:00:00 2001 From: kssenii Date: Tue, 16 Mar 2021 10:20:31 +0000 Subject: [PATCH 505/544] Add double quotes --- src/Databases/PostgreSQL/DatabasePostgreSQL.cpp | 3 ++- src/TableFunctions/TableFunctionPostgreSQL.cpp | 4 +++- tests/integration/test_storage_postgresql/test.py | 11 +++++++++++ 3 files changed, 16 insertions(+), 2 deletions(-) diff --git a/src/Databases/PostgreSQL/DatabasePostgreSQL.cpp b/src/Databases/PostgreSQL/DatabasePostgreSQL.cpp index 722b9c64edb..d2eb6797c84 100644 --- a/src/Databases/PostgreSQL/DatabasePostgreSQL.cpp +++ b/src/Databases/PostgreSQL/DatabasePostgreSQL.cpp @@ -16,6 +16,7 @@ #include #include #include +#include namespace DB @@ -162,7 +163,7 @@ StoragePtr DatabasePostgreSQL::fetchTable(const String & table_name, const Conte return StoragePtr{}; auto use_nulls = context.getSettingsRef().external_table_functions_use_nulls; - auto columns = fetchPostgreSQLTableStructure(connection->conn(), table_name, use_nulls); + auto columns = fetchPostgreSQLTableStructure(connection->conn(), doubleQuoteString(table_name), use_nulls); if (!columns) return StoragePtr{}; diff --git a/src/TableFunctions/TableFunctionPostgreSQL.cpp b/src/TableFunctions/TableFunctionPostgreSQL.cpp index 0e3f1c5da24..ae34ce41e47 100644 --- a/src/TableFunctions/TableFunctionPostgreSQL.cpp +++ b/src/TableFunctions/TableFunctionPostgreSQL.cpp @@ -11,6 +11,7 @@ #include "registerTableFunctions.h" #include #include +#include namespace DB @@ -41,7 +42,8 @@ ColumnsDescription TableFunctionPostgreSQL::getActualTableStructure(const Contex const bool use_nulls = context.getSettingsRef().external_table_functions_use_nulls; auto columns = fetchPostgreSQLTableStructure( connection->conn(), - remote_table_schema.empty() ? remote_table_name : remote_table_schema + '.' + remote_table_name, + remote_table_schema.empty() ? doubleQuoteString(remote_table_name) + : doubleQuoteString(remote_table_schema) + '.' + doubleQuoteString(remote_table_name), use_nulls); return ColumnsDescription{*columns}; diff --git a/tests/integration/test_storage_postgresql/test.py b/tests/integration/test_storage_postgresql/test.py index 58f3233bacc..8be72f5a7c7 100644 --- a/tests/integration/test_storage_postgresql/test.py +++ b/tests/integration/test_storage_postgresql/test.py @@ -152,6 +152,17 @@ def test_non_default_scema(started_cluster): result = node1.query('SELECT * FROM {}'.format(table_function)) assert(result == expected) + cursor.execute('''CREATE SCHEMA "test.nice.schema"''') + cursor.execute('''CREATE TABLE "test.nice.schema"."test.nice.table" (a integer)''') + cursor.execute('INSERT INTO "test.nice.schema"."test.nice.table" SELECT i FROM generate_series(0, 99) as t(i)') + + node1.query(''' + CREATE TABLE test_pg_table_schema_with_dots (a UInt32) + ENGINE PostgreSQL('postgres1:5432', 'clickhouse', 'test.nice.table', 'postgres', 'mysecretpassword', 'test.nice.schema'); + ''') + result = node1.query('SELECT * FROM test_pg_table_schema_with_dots') + assert(result == expected) + if __name__ == '__main__': cluster.start() From de091114f3cee6ac5a775a9811855295ff29a34e Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Tue, 16 Mar 2021 15:22:52 +0300 Subject: [PATCH 506/544] Fix UBSan report --- base/common/DateLUTImpl.h | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/base/common/DateLUTImpl.h b/base/common/DateLUTImpl.h index 98b767ccbcc..43fc1b8befd 100644 --- a/base/common/DateLUTImpl.h +++ b/base/common/DateLUTImpl.h @@ -63,7 +63,7 @@ private: template friend inline LUTIndex operator+(const LUTIndex & index, const T v) { - return LUTIndex{(index.toUnderType() + v) & date_lut_mask}; + return LUTIndex{(index.toUnderType() + UInt32(v)) & date_lut_mask}; } template @@ -80,7 +80,7 @@ private: template friend inline LUTIndex operator-(const LUTIndex & index, const T v) { - return LUTIndex{(index.toUnderType() - v) & date_lut_mask}; + return LUTIndex{(index.toUnderType() - UInt32(v)) & date_lut_mask}; } template @@ -97,7 +97,7 @@ private: template friend inline LUTIndex operator*(const LUTIndex & index, const T v) { - return LUTIndex{(index.toUnderType() * v) & date_lut_mask}; + return LUTIndex{(index.toUnderType() * UInt32(v)) & date_lut_mask}; } template @@ -109,13 +109,13 @@ private: template friend inline LUTIndex operator/(const LUTIndex & index, const T v) { - return LUTIndex{(index.toUnderType() / v) & date_lut_mask}; + return LUTIndex{(index.toUnderType() / UInt32(v)) & date_lut_mask}; } template friend inline LUTIndex operator/(const T v, const LUTIndex & index) { - return LUTIndex{(v / index.toUnderType()) & date_lut_mask}; + return LUTIndex{(UInt32(v) / index.toUnderType()) & date_lut_mask}; } public: From 172f668fa3fe78e49fec00f0eaa2faf0a124eff6 Mon Sep 17 00:00:00 2001 From: Anton Popov Date: Tue, 16 Mar 2021 15:45:08 +0300 Subject: [PATCH 507/544] fix doc --- docs/en/engines/table-engines/mergetree-family/mergetree.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/en/engines/table-engines/mergetree-family/mergetree.md b/docs/en/engines/table-engines/mergetree-family/mergetree.md index 70cf7f2212e..0b551040ba1 100644 --- a/docs/en/engines/table-engines/mergetree-family/mergetree.md +++ b/docs/en/engines/table-engines/mergetree-family/mergetree.md @@ -529,7 +529,7 @@ CREATE TABLE table_for_aggregation y Int ) ENGINE = MergeTree -ORDER BY k1, k2 +ORDER BY (k1, k2) TTL d + INTERVAL 1 MONTH GROUP BY k1, k2 SET x = max(x), y = min(y); ``` From 1dd730d2d6a48fb03d8f9f692888bee9f5969743 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Tue, 16 Mar 2021 15:59:14 +0300 Subject: [PATCH 508/544] Fix perf test --- tests/performance/date_time_long.xml | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/performance/date_time_long.xml b/tests/performance/date_time_long.xml index 3a61a5992e5..ae41602e825 100644 --- a/tests/performance/date_time_long.xml +++ b/tests/performance/date_time_long.xml @@ -77,7 +77,6 @@ toYYYYMMDDhhmmss toRelativeQuarterNum - toUnixTimestamp From 637b683ae9afedbad922ba6a7216ac8095891d12 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Tue, 16 Mar 2021 18:35:14 +0300 Subject: [PATCH 509/544] Fix clang-tidy --- src/DataTypes/DataTypeDateTime.cpp | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/src/DataTypes/DataTypeDateTime.cpp b/src/DataTypes/DataTypeDateTime.cpp index 09dcb5f3e2e..510747f6ef9 100644 --- a/src/DataTypes/DataTypeDateTime.cpp +++ b/src/DataTypes/DataTypeDateTime.cpp @@ -91,7 +91,7 @@ void DataTypeDateTime::deserializeWholeText(IColumn & column, ReadBuffer & istr, void DataTypeDateTime::deserializeTextEscaped(IColumn & column, ReadBuffer & istr, const FormatSettings & settings) const { - time_t x; + time_t x = 0; readTextHelper(x, istr, settings, time_zone, utc_time_zone); if (x < 0) x = 0; @@ -107,7 +107,8 @@ void DataTypeDateTime::serializeTextQuoted(const IColumn & column, size_t row_nu void DataTypeDateTime::deserializeTextQuoted(IColumn & column, ReadBuffer & istr, const FormatSettings & settings) const { - time_t x; + time_t x = 0; + if (checkChar('\'', istr)) /// Cases: '2017-08-31 18:36:48' or '1504193808' { readTextHelper(x, istr, settings, time_zone, utc_time_zone); @@ -131,7 +132,8 @@ void DataTypeDateTime::serializeTextJSON(const IColumn & column, size_t row_num, void DataTypeDateTime::deserializeTextJSON(IColumn & column, ReadBuffer & istr, const FormatSettings & settings) const { - time_t x; + time_t x = 0; + if (checkChar('"', istr)) { readTextHelper(x, istr, settings, time_zone, utc_time_zone); @@ -157,7 +159,7 @@ void DataTypeDateTime::serializeTextCSV(const IColumn & column, size_t row_num, void DataTypeDateTime::deserializeTextCSV(IColumn & column, ReadBuffer & istr, const FormatSettings & settings) const { - time_t x; + time_t x = 0; if (istr.eof()) throwReadAfterEOF(); From 02a06eb448d63668c1973c361feb7276cc14388a Mon Sep 17 00:00:00 2001 From: alexey-milovidov Date: Tue, 16 Mar 2021 18:39:39 +0300 Subject: [PATCH 510/544] Update mergetree.md --- .../table-engines/mergetree-family/mergetree.md | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/docs/en/engines/table-engines/mergetree-family/mergetree.md b/docs/en/engines/table-engines/mergetree-family/mergetree.md index 0b551040ba1..a24b7229d17 100644 --- a/docs/en/engines/table-engines/mergetree-family/mergetree.md +++ b/docs/en/engines/table-engines/mergetree-family/mergetree.md @@ -353,7 +353,7 @@ The `set` index can be used with all functions. Function subsets for other index | Function (operator) / Index | primary key | minmax | ngrambf_v1 | tokenbf_v1 | bloom_filter | |------------------------------------------------------------------------------------------------------------|-------------|--------|-------------|-------------|---------------| | [equals (=, ==)](../../../sql-reference/functions/comparison-functions.md#function-equals) | ✔ | ✔ | ✔ | ✔ | ✔ | -| [notEquals(!=, \<\>)](../../../sql-reference/functions/comparison-functions.md#function-notequals) | ✔ | ✔ | ✔ | ✔ | ✔ | +| [notEquals(!=, <>)](../../../sql-reference/functions/comparison-functions.md#function-notequals) | ✔ | ✔ | ✔ | ✔ | ✔ | | [like](../../../sql-reference/functions/string-search-functions.md#function-like) | ✔ | ✔ | ✔ | ✔ | ✗ | | [notLike](../../../sql-reference/functions/string-search-functions.md#function-notlike) | ✔ | ✔ | ✔ | ✔ | ✗ | | [startsWith](../../../sql-reference/functions/string-functions.md#startswith) | ✔ | ✔ | ✔ | ✔ | ✗ | @@ -361,10 +361,10 @@ The `set` index can be used with all functions. Function subsets for other index | [multiSearchAny](../../../sql-reference/functions/string-search-functions.md#function-multisearchany) | ✗ | ✗ | ✔ | ✗ | ✗ | | [in](../../../sql-reference/functions/in-functions.md#in-functions) | ✔ | ✔ | ✔ | ✔ | ✔ | | [notIn](../../../sql-reference/functions/in-functions.md#in-functions) | ✔ | ✔ | ✔ | ✔ | ✔ | -| [less (\<)](../../../sql-reference/functions/comparison-functions.md#function-less) | ✔ | ✔ | ✗ | ✗ | ✗ | -| [greater (\>)](../../../sql-reference/functions/comparison-functions.md#function-greater) | ✔ | ✔ | ✗ | ✗ | ✗ | -| [lessOrEquals (\<=)](../../../sql-reference/functions/comparison-functions.md#function-lessorequals) | ✔ | ✔ | ✗ | ✗ | ✗ | -| [greaterOrEquals (\>=)](../../../sql-reference/functions/comparison-functions.md#function-greaterorequals) | ✔ | ✔ | ✗ | ✗ | ✗ | +| [less (<)](../../../sql-reference/functions/comparison-functions.md#function-less) | ✔ | ✔ | ✗ | ✗ | ✗ | +| [greater (>)](../../../sql-reference/functions/comparison-functions.md#function-greater) | ✔ | ✔ | ✗ | ✗ | ✗ | +| [lessOrEquals (<=)](../../../sql-reference/functions/comparison-functions.md#function-lessorequals) | ✔ | ✔ | ✗ | ✗ | ✗ | +| [greaterOrEquals (>=)](../../../sql-reference/functions/comparison-functions.md#function-greaterorequals) | ✔ | ✔ | ✗ | ✗ | ✗ | | [empty](../../../sql-reference/functions/array-functions.md#function-empty) | ✔ | ✔ | ✗ | ✗ | ✗ | | [notEmpty](../../../sql-reference/functions/array-functions.md#function-notempty) | ✔ | ✔ | ✗ | ✗ | ✗ | | hasToken | ✗ | ✗ | ✗ | ✔ | ✗ | From 9f05fc22d062391dec8b704987a3d269f4f13741 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Tue, 16 Mar 2021 19:31:25 +0300 Subject: [PATCH 511/544] Modern JavaScript --- programs/server/play.html | 64 +++++++++++++++++++-------------------- 1 file changed, 32 insertions(+), 32 deletions(-) diff --git a/programs/server/play.html b/programs/server/play.html index 81fc13f1f86..7d0ceeeaeb1 100644 --- a/programs/server/play.html +++ b/programs/server/play.html @@ -306,10 +306,10 @@ /// Incremental request number. When response is received, /// if it's request number does not equal to the current request number, response will be ignored. /// This is to avoid race conditions. - var request_num = 0; + let request_num = 0; /// Save query in history only if it is different. - var previous_query = ''; + let previous_query = ''; /// Substitute the address of the server where the page is served. if (location.protocol != 'file:') { @@ -317,7 +317,7 @@ } /// Substitute user name if it's specified in the query string - var user_from_url = (new URL(window.location)).searchParams.get('user'); + let user_from_url = (new URL(window.location)).searchParams.get('user'); if (user_from_url) { document.getElementById('user').value = user_from_url; } @@ -326,10 +326,10 @@ { /// TODO: Check if URL already contains query string (append parameters). - var user = document.getElementById('user').value; - var password = document.getElementById('password').value; + let user = document.getElementById('user').value; + let password = document.getElementById('password').value; - var url = document.getElementById('url').value + + let url = document.getElementById('url').value + /// Ask server to allow cross-domain requests. '?add_http_cors_header=1' + '&user=' + encodeURIComponent(user) + @@ -338,7 +338,7 @@ /// Safety settings to prevent results that browser cannot display. '&max_result_rows=1000&max_result_bytes=10000000&result_overflow_mode=break'; - var xhr = new XMLHttpRequest; + let xhr = new XMLHttpRequest; xhr.open('POST', url, true); @@ -352,13 +352,13 @@ /// The query is saved in browser history (in state JSON object) /// as well as in URL fragment identifier. if (query != previous_query) { - var state = { + let state = { query: query, status: this.status, response: this.response.length > 100000 ? null : this.response /// Lower than the browser's limit. }; - var title = "ClickHouse Query: " + query; - var url = window.location.pathname + '?user=' + encodeURIComponent(user) + '#' + window.btoa(query); + let title = "ClickHouse Query: " + query; + let url = window.location.pathname + '?user=' + encodeURIComponent(user) + '#' + window.btoa(query); if (previous_query == '') { history.replaceState(state, title, url); } else { @@ -382,7 +382,7 @@ document.getElementById('hourglass').style.display = 'none'; if (status === 200) { - var json; + let json; try { json = JSON.parse(response); } catch (e) {} if (json !== undefined && json.statistics !== undefined) { renderResult(json); @@ -415,7 +415,7 @@ function post() { ++request_num; - var query = document.getElementById('query').value; + let query = document.getElementById('query').value; postImpl(request_num, query); } @@ -434,7 +434,7 @@ function clear() { - var table = document.getElementById('data-table'); + let table = document.getElementById('data-table'); while (table.firstChild) { table.removeChild(table.lastChild); } @@ -456,30 +456,30 @@ //console.log(response); clear(); - var stats = document.getElementById('stats'); + let stats = document.getElementById('stats'); stats.innerText = 'Elapsed: ' + response.statistics.elapsed.toFixed(3) + " sec, read " + response.statistics.rows_read + " rows."; - var thead = document.createElement('thead'); - for (var idx in response.meta) { - var th = document.createElement('th'); - var name = document.createTextNode(response.meta[idx].name); + let thead = document.createElement('thead'); + for (let idx in response.meta) { + let th = document.createElement('th'); + let name = document.createTextNode(response.meta[idx].name); th.appendChild(name); thead.appendChild(th); } /// To prevent hanging the browser, limit the number of cells in a table. /// It's important to have the limit on number of cells, not just rows, because tables may be wide or narrow. - var max_rows = 10000 / response.meta.length; - var row_num = 0; + let max_rows = 10000 / response.meta.length; + let row_num = 0; - var tbody = document.createElement('tbody'); - for (var row_idx in response.data) { - var tr = document.createElement('tr'); - for (var col_idx in response.data[row_idx]) { - var td = document.createElement('td'); - var cell = response.data[row_idx][col_idx]; - var is_null = (cell === null); - var content = document.createTextNode(is_null ? 'ᴺᵁᴸᴸ' : cell); + let tbody = document.createElement('tbody'); + for (let row_idx in response.data) { + let tr = document.createElement('tr'); + for (let col_idx in response.data[row_idx]) { + let td = document.createElement('td'); + let cell = response.data[row_idx][col_idx]; + let is_null = (cell === null); + let content = document.createTextNode(is_null ? 'ᴺᵁᴸᴸ' : cell); td.appendChild(content); /// TODO: Execute regexp only once for each column. td.className = response.meta[col_idx].type.match(/^(U?Int|Decimal|Float)/) ? 'right' : 'left'; @@ -496,7 +496,7 @@ } } - var table = document.getElementById('data-table'); + let table = document.getElementById('data-table'); table.appendChild(thead); table.appendChild(tbody); } @@ -505,7 +505,7 @@ function renderUnparsedResult(response) { clear(); - var data = document.getElementById('data-unparsed') + let data = document.getElementById('data-unparsed') if (response === '') { /// TODO: Fade or remove previous result when new request will be performed. @@ -531,12 +531,12 @@ } /// The choice of color theme is saved in browser. - var theme = window.localStorage.getItem('theme'); + let theme = window.localStorage.getItem('theme'); if (theme) { setColorTheme(theme); } else { /// Obtain system-level user preference - var media_query_list = window.matchMedia('prefers-color-scheme: dark') + let media_query_list = window.matchMedia('prefers-color-scheme: dark') if (media_query_list.matches) { /// Set without saving to localstorage From bc25624b885bf74ad0898c16f18229fa32d09c48 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Tue, 16 Mar 2021 19:54:16 +0300 Subject: [PATCH 512/544] Better formatting for Array and Map in Web UI --- programs/server/play.html | 19 ++++++++++++++++--- 1 file changed, 16 insertions(+), 3 deletions(-) diff --git a/programs/server/play.html b/programs/server/play.html index 7d0ceeeaeb1..e9404d13acb 100644 --- a/programs/server/play.html +++ b/programs/server/play.html @@ -472,17 +472,30 @@ let max_rows = 10000 / response.meta.length; let row_num = 0; + let column_classes = response.meta.map(elem => elem.type.match(/^(U?Int|Decimal|Float)/) ? 'right' : 'left'); + let tbody = document.createElement('tbody'); for (let row_idx in response.data) { let tr = document.createElement('tr'); for (let col_idx in response.data[row_idx]) { let td = document.createElement('td'); let cell = response.data[row_idx][col_idx]; + let is_null = (cell === null); - let content = document.createTextNode(is_null ? 'ᴺᵁᴸᴸ' : cell); - td.appendChild(content); + + /// Test: SELECT number, toString(number) AS str, number % 2 ? number : NULL AS nullable, range(number) AS arr, CAST((['hello', 'world'], [number, number % 2]) AS Map(String, UInt64)) AS map FROM numbers(10) + let text; + if (is_null) { + text = 'ᴺᵁᴸᴸ'; + } else if (typeof(cell) === 'object') { + text = JSON.stringify(cell); + } else { + text = cell; + } + + td.appendChild(document.createTextNode(text)); /// TODO: Execute regexp only once for each column. - td.className = response.meta[col_idx].type.match(/^(U?Int|Decimal|Float)/) ? 'right' : 'left'; + td.className = column_classes[col_idx]; if (is_null) { td.className += ' null'; } From 174bce9ca4966b56ee60ee876a5f2337fddbfa1e Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Tue, 16 Mar 2021 19:55:56 +0300 Subject: [PATCH 513/544] TODO is resolved --- programs/server/play.html | 1 - 1 file changed, 1 deletion(-) diff --git a/programs/server/play.html b/programs/server/play.html index e9404d13acb..0c039097ce1 100644 --- a/programs/server/play.html +++ b/programs/server/play.html @@ -494,7 +494,6 @@ } td.appendChild(document.createTextNode(text)); - /// TODO: Execute regexp only once for each column. td.className = column_classes[col_idx]; if (is_null) { td.className += ' null'; From 94da7f422c3ca954d422741edb76ed4f9a735b4a Mon Sep 17 00:00:00 2001 From: George Date: Tue, 16 Mar 2021 20:57:17 +0300 Subject: [PATCH 514/544] updated translation --- docs/ru/sql-reference/statements/detach.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/ru/sql-reference/statements/detach.md b/docs/ru/sql-reference/statements/detach.md index bec8f4c5ff7..90e9663def9 100644 --- a/docs/ru/sql-reference/statements/detach.md +++ b/docs/ru/sql-reference/statements/detach.md @@ -15,13 +15,13 @@ DETACH TABLE|VIEW [IF EXISTS] [db.]name [PERMANENTLY] [ON CLUSTER cluster] Но ни данные, ни метаданные таблицы или материализованного представления не удаляются. При следующем запуске сервера, если не было использовано `PERMANENTLY`, сервер прочитает метаданные и снова узнает о таблице/представлении. Если таблица или представление были откреплено перманентно, сервер не прикрепит их обратно автоматически. -Независимо от того, каким способом таблица была откреплена, ее можно прикрепить обратно с помощью запроса [ATTACH](../../sql-reference/statements/attach.md). Системные log таблицы также могут быть прикреплены обратно (к примеру `query_log`, `text_log` и др.) Другие системные таблицы не могут быть прикреплены обратно, но на следующем запуске сервер снова вспомнит об этих таблицах. +Независимо от того, каким способом таблица была откреплена, ее можно прикрепить обратно с помощью запроса [ATTACH](../../sql-reference/statements/attach.md). Системные log таблицы также могут быть прикреплены обратно (к примеру `query_log`, `text_log` и др.) Другие системные таблицы не могут быть прикреплены обратно, но на следующем запуске сервер снова "вспомнит" об этих таблицах. `ATTACH MATERIALIZED VIEW` не может быть использован с кратким синтаксисом (без `SELECT`), но можно прикрепить представление с помощью запроса `ATTACH TABLE`. Обратите внимание, что нельзя перманентно открепить таблицу, которая уже временно откреплена. Для этого ее сначала надо прикрепить обратно, а затем снова открепить перманентно. -Также нельзя использовать [DROP](../../sql-reference/statements/drop.md#drop-table) с открепленной таблицей или создавать таблицу с помощью [CREATE TABLE](../../sql-reference/statements/create/table.md) с таким же именем, как уже открепленная таблица. Еще нельзя заменить открепленную таблицу другой с помощью запроса [RENAME TABLE](../../sql-reference/statements/rename.md). +Также нельзя использовать [DROP](../../sql-reference/statements/drop.md#drop-table) с открепленной таблицей или создавать таблицу с помощью [CREATE TABLE](../../sql-reference/statements/create/table.md) с таким же именем, как у уже открепленной таблицы. Еще нельзя заменить открепленную таблицу другой с помощью запроса [RENAME TABLE](../../sql-reference/statements/rename.md). **Пример** From b39f8cc6ac2e470b01826458f3f0f286367da64e Mon Sep 17 00:00:00 2001 From: Azat Khuzhin Date: Mon, 8 Mar 2021 22:05:51 +0300 Subject: [PATCH 515/544] Move ErrorCodes::increment() into module part --- src/Common/ErrorCodes.cpp | 11 +++++++++++ src/Common/ErrorCodes.h | 11 +---------- 2 files changed, 12 insertions(+), 10 deletions(-) diff --git a/src/Common/ErrorCodes.cpp b/src/Common/ErrorCodes.cpp index 40ce23fffb2..f6c15848553 100644 --- a/src/Common/ErrorCodes.cpp +++ b/src/Common/ErrorCodes.cpp @@ -584,6 +584,17 @@ namespace ErrorCodes } ErrorCode end() { return END + 1; } + + void increment(ErrorCode error_code) + { + if (error_code >= end()) + { + /// For everything outside the range, use END. + /// (end() is the pointer pass the end, while END is the last value that has an element in values array). + error_code = end() - 1; + } + values[error_code].fetch_add(1, std::memory_order_relaxed); + } } } diff --git a/src/Common/ErrorCodes.h b/src/Common/ErrorCodes.h index cc610c5d927..919a4afdabf 100644 --- a/src/Common/ErrorCodes.h +++ b/src/Common/ErrorCodes.h @@ -31,16 +31,7 @@ namespace ErrorCodes ErrorCode end(); /// Add value for specified error_code. - inline void increment(ErrorCode error_code) - { - if (error_code >= end()) - { - /// For everything outside the range, use END. - /// (end() is the pointer pass the end, while END is the last value that has an element in values array). - error_code = end() - 1; - } - values[error_code].fetch_add(1, std::memory_order_relaxed); - } + void increment(ErrorCode error_code); } } From 259e5ba88e35546279bb46511a5e6ad18b4457d6 Mon Sep 17 00:00:00 2001 From: Azat Khuzhin Date: Mon, 8 Mar 2021 22:05:51 +0300 Subject: [PATCH 516/544] Separate accounting of remote exceptions in system.errors --- docs/en/operations/system-tables/errors.md | 3 ++- src/Common/ErrorCodes.cpp | 9 ++++--- src/Common/ErrorCodes.h | 9 +++++-- src/Common/Exception.cpp | 12 +++++----- src/Storages/System/StorageSystemErrors.cpp | 24 ++++++++++++------- .../0_stateless/01545_system_errors.reference | 3 ++- .../0_stateless/01545_system_errors.sh | 13 +++++++--- 7 files changed, 49 insertions(+), 24 deletions(-) diff --git a/docs/en/operations/system-tables/errors.md b/docs/en/operations/system-tables/errors.md index ec874efd711..bf3e33f5275 100644 --- a/docs/en/operations/system-tables/errors.md +++ b/docs/en/operations/system-tables/errors.md @@ -7,11 +7,12 @@ Columns: - `name` ([String](../../sql-reference/data-types/string.md)) — name of the error (`errorCodeToName`). - `code` ([Int32](../../sql-reference/data-types/int-uint.md)) — code number of the error. - `value` ([UInt64](../../sql-reference/data-types/int-uint.md)) — the number of times this error has been happened. +- `remote` ([UInt8](../../sql-reference/data-types/int-uint.md)) — remote exception (i.e. received during one of the distributed query). **Example** ``` sql -SELECT * +SELECT name, code, value FROM system.errors WHERE value > 0 ORDER BY code ASC diff --git a/src/Common/ErrorCodes.cpp b/src/Common/ErrorCodes.cpp index f6c15848553..ec3bf9c8917 100644 --- a/src/Common/ErrorCodes.cpp +++ b/src/Common/ErrorCodes.cpp @@ -563,7 +563,7 @@ namespace ErrorCodes #undef M constexpr Value END = 3000; - std::atomic values[END + 1]{}; + ValuePair values[END + 1]{}; struct ErrorCodesNames { @@ -585,7 +585,7 @@ namespace ErrorCodes ErrorCode end() { return END + 1; } - void increment(ErrorCode error_code) + void increment(ErrorCode error_code, bool remote) { if (error_code >= end()) { @@ -593,7 +593,10 @@ namespace ErrorCodes /// (end() is the pointer pass the end, while END is the last value that has an element in values array). error_code = end() - 1; } - values[error_code].fetch_add(1, std::memory_order_relaxed); + if (remote) + values[error_code].remote.fetch_add(1, std::memory_order_relaxed); + else + values[error_code].local.fetch_add(1, std::memory_order_relaxed); } } diff --git a/src/Common/ErrorCodes.h b/src/Common/ErrorCodes.h index 919a4afdabf..c4a9ae2907b 100644 --- a/src/Common/ErrorCodes.h +++ b/src/Common/ErrorCodes.h @@ -25,13 +25,18 @@ namespace ErrorCodes std::string_view getName(ErrorCode error_code); /// ErrorCode identifier -> current value of error_code. - extern std::atomic values[]; + struct ValuePair + { + std::atomic local; + std::atomic remote; + }; + extern ValuePair values[]; /// Get index just after last error_code identifier. ErrorCode end(); /// Add value for specified error_code. - void increment(ErrorCode error_code); + void increment(ErrorCode error_code, bool remote); } } diff --git a/src/Common/Exception.cpp b/src/Common/Exception.cpp index 16f15d4e6f2..1963c1513b9 100644 --- a/src/Common/Exception.cpp +++ b/src/Common/Exception.cpp @@ -34,9 +34,9 @@ namespace ErrorCodes extern const int CANNOT_MREMAP; } -/// Aborts the process if error code is LOGICAL_ERROR. -/// Increments error codes statistics. -void handle_error_code([[maybe_unused]] const std::string & msg, int code) +/// - Aborts the process if error code is LOGICAL_ERROR. +/// - Increments error codes statistics. +void handle_error_code([[maybe_unused]] const std::string & msg, int code, bool remote) { // In debug builds and builds with sanitizers, treat LOGICAL_ERROR as an assertion failure. // Log the message before we fail. @@ -47,20 +47,20 @@ void handle_error_code([[maybe_unused]] const std::string & msg, int code) abort(); } #endif - ErrorCodes::increment(code); + ErrorCodes::increment(code, remote); } Exception::Exception(const std::string & msg, int code, bool remote_) : Poco::Exception(msg, code) , remote(remote_) { - handle_error_code(msg, code); + handle_error_code(msg, code, remote); } Exception::Exception(const std::string & msg, const Exception & nested, int code) : Poco::Exception(msg, nested, code) { - handle_error_code(msg, code); + handle_error_code(msg, code, remote); } Exception::Exception(CreateFromPocoTag, const Poco::Exception & exc) diff --git a/src/Storages/System/StorageSystemErrors.cpp b/src/Storages/System/StorageSystemErrors.cpp index 89df058900b..1a29484e169 100644 --- a/src/Storages/System/StorageSystemErrors.cpp +++ b/src/Storages/System/StorageSystemErrors.cpp @@ -13,27 +13,35 @@ NamesAndTypesList StorageSystemErrors::getNamesAndTypes() { "name", std::make_shared() }, { "code", std::make_shared() }, { "value", std::make_shared() }, + { "remote", std::make_shared() }, }; } void StorageSystemErrors::fillData(MutableColumns & res_columns, const Context & context, const SelectQueryInfo &) const { + auto add_row = [&](std::string_view name, size_t code, size_t value, bool remote) + { + if (value || context.getSettingsRef().system_events_show_zero_values) + { + size_t col_num = 0; + res_columns[col_num++]->insert(name); + res_columns[col_num++]->insert(code); + res_columns[col_num++]->insert(value); + res_columns[col_num++]->insert(remote); + } + }; + for (size_t i = 0, end = ErrorCodes::end(); i < end; ++i) { - UInt64 value = ErrorCodes::values[i]; + const auto & error = ErrorCodes::values[i]; std::string_view name = ErrorCodes::getName(i); if (name.empty()) continue; - if (value || context.getSettingsRef().system_events_show_zero_values) - { - size_t col_num = 0; - res_columns[col_num++]->insert(name); - res_columns[col_num++]->insert(i); - res_columns[col_num++]->insert(value); - } + add_row(name, i, error.local, 0 /* remote=0 */); + add_row(name, i, error.remote, 1 /* remote=1 */); } } diff --git a/tests/queries/0_stateless/01545_system_errors.reference b/tests/queries/0_stateless/01545_system_errors.reference index d00491fd7e5..0e7f2447090 100644 --- a/tests/queries/0_stateless/01545_system_errors.reference +++ b/tests/queries/0_stateless/01545_system_errors.reference @@ -1 +1,2 @@ -1 +local=1 +remote=1 diff --git a/tests/queries/0_stateless/01545_system_errors.sh b/tests/queries/0_stateless/01545_system_errors.sh index 63af6bb8d43..970fd403866 100755 --- a/tests/queries/0_stateless/01545_system_errors.sh +++ b/tests/queries/0_stateless/01545_system_errors.sh @@ -4,7 +4,14 @@ CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) # shellcheck source=../shell_config.sh . "$CURDIR"/../shell_config.sh -prev="$(${CLICKHOUSE_CLIENT} -q "SELECT value FROM system.errors WHERE name = 'FUNCTION_THROW_IF_VALUE_IS_NON_ZERO'")" +# local +prev="$(${CLICKHOUSE_CLIENT} -q "SELECT value FROM system.errors WHERE name = 'FUNCTION_THROW_IF_VALUE_IS_NON_ZERO' AND NOT remote")" $CLICKHOUSE_CLIENT -q 'SELECT throwIf(1)' >& /dev/null -cur="$(${CLICKHOUSE_CLIENT} -q "SELECT value FROM system.errors WHERE name = 'FUNCTION_THROW_IF_VALUE_IS_NON_ZERO'")" -echo $((cur - prev)) +cur="$(${CLICKHOUSE_CLIENT} -q "SELECT value FROM system.errors WHERE name = 'FUNCTION_THROW_IF_VALUE_IS_NON_ZERO' AND NOT remote")" +echo local=$((cur - prev)) + +# remote +prev="$(${CLICKHOUSE_CLIENT} -q "SELECT value FROM system.errors WHERE name = 'FUNCTION_THROW_IF_VALUE_IS_NON_ZERO' AND remote")" +${CLICKHOUSE_CLIENT} -q "SELECT * FROM remote('127.2', system.one) where throwIf(not dummy)" >& /dev/null +cur="$(${CLICKHOUSE_CLIENT} -q "SELECT value FROM system.errors WHERE name = 'FUNCTION_THROW_IF_VALUE_IS_NON_ZERO' AND remote")" +echo remote=$((cur - prev)) From 7f73ac2b7a94ed809f4a2eaa59c13d7585756a6c Mon Sep 17 00:00:00 2001 From: Azat Khuzhin Date: Mon, 8 Mar 2021 22:05:51 +0300 Subject: [PATCH 517/544] Fix ErrorCodes::Value/ErrorCode types (sigh) Note, that system.errors already uses correct types --- src/Common/ErrorCodes.cpp | 2 +- src/Common/ErrorCodes.h | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/Common/ErrorCodes.cpp b/src/Common/ErrorCodes.cpp index ec3bf9c8917..6f1ff8e4f3a 100644 --- a/src/Common/ErrorCodes.cpp +++ b/src/Common/ErrorCodes.cpp @@ -562,7 +562,7 @@ namespace ErrorCodes APPLY_FOR_ERROR_CODES(M) #undef M - constexpr Value END = 3000; + constexpr ErrorCode END = 3000; ValuePair values[END + 1]{}; struct ErrorCodesNames diff --git a/src/Common/ErrorCodes.h b/src/Common/ErrorCodes.h index c4a9ae2907b..6373ad6d0f9 100644 --- a/src/Common/ErrorCodes.h +++ b/src/Common/ErrorCodes.h @@ -17,8 +17,8 @@ namespace DB namespace ErrorCodes { /// ErrorCode identifier (index in array). - using ErrorCode = size_t; - using Value = int; + using ErrorCode = int; + using Value = size_t; /// Get name of error_code by identifier. /// Returns statically allocated string. From 0d01eaf94fd122213b3d31593248fe3f3e9c6a40 Mon Sep 17 00:00:00 2001 From: Azat Khuzhin Date: Mon, 8 Mar 2021 23:08:04 +0300 Subject: [PATCH 518/544] Guard ErrorCodes with mutex over atomic --- src/Common/ErrorCodes.cpp | 30 +++++++++++++++++---- src/Common/ErrorCodes.h | 25 +++++++++++++---- src/Storages/System/StorageSystemErrors.cpp | 2 +- 3 files changed, 46 insertions(+), 11 deletions(-) diff --git a/src/Common/ErrorCodes.cpp b/src/Common/ErrorCodes.cpp index 6f1ff8e4f3a..32c9c4a452b 100644 --- a/src/Common/ErrorCodes.cpp +++ b/src/Common/ErrorCodes.cpp @@ -563,7 +563,7 @@ namespace ErrorCodes #undef M constexpr ErrorCode END = 3000; - ValuePair values[END + 1]{}; + ValuePairHolder values[END + 1]{}; struct ErrorCodesNames { @@ -593,10 +593,30 @@ namespace ErrorCodes /// (end() is the pointer pass the end, while END is the last value that has an element in values array). error_code = end() - 1; } - if (remote) - values[error_code].remote.fetch_add(1, std::memory_order_relaxed); - else - values[error_code].local.fetch_add(1, std::memory_order_relaxed); + + ValuePair inc_value{ + !remote, /* local */ + remote, /* remote */ + }; + values[error_code].increment(inc_value); + } + + ValuePair & ValuePair::operator+=(const ValuePair & value) + { + local += value.local; + remote += value.remote; + return *this; + } + + void ValuePairHolder::increment(const ValuePair & value_) + { + std::lock_guard lock(mutex); + value += value_; + } + ValuePair ValuePairHolder::get() + { + std::lock_guard lock(mutex); + return value; } } diff --git a/src/Common/ErrorCodes.h b/src/Common/ErrorCodes.h index 6373ad6d0f9..0db877db205 100644 --- a/src/Common/ErrorCodes.h +++ b/src/Common/ErrorCodes.h @@ -3,7 +3,7 @@ #include #include #include -#include +#include #include #include @@ -24,13 +24,28 @@ namespace ErrorCodes /// Returns statically allocated string. std::string_view getName(ErrorCode error_code); - /// ErrorCode identifier -> current value of error_code. struct ValuePair { - std::atomic local; - std::atomic remote; + Value local = 0; + Value remote = 0; + + ValuePair & operator+=(const ValuePair & value); }; - extern ValuePair values[]; + + /// Thread-safe + struct ValuePairHolder + { + public: + void increment(const ValuePair & value_); + ValuePair get(); + + private: + ValuePair value; + std::mutex mutex; + }; + + /// ErrorCode identifier -> current value of error_code. + extern ValuePairHolder values[]; /// Get index just after last error_code identifier. ErrorCode end(); diff --git a/src/Storages/System/StorageSystemErrors.cpp b/src/Storages/System/StorageSystemErrors.cpp index 1a29484e169..d57e8a0a670 100644 --- a/src/Storages/System/StorageSystemErrors.cpp +++ b/src/Storages/System/StorageSystemErrors.cpp @@ -34,7 +34,7 @@ void StorageSystemErrors::fillData(MutableColumns & res_columns, const Context & for (size_t i = 0, end = ErrorCodes::end(); i < end; ++i) { - const auto & error = ErrorCodes::values[i]; + const auto & error = ErrorCodes::values[i].get(); std::string_view name = ErrorCodes::getName(i); if (name.empty()) From c8852331a2f3356069155d6ab45ba79f1ecada57 Mon Sep 17 00:00:00 2001 From: Azat Khuzhin Date: Mon, 8 Mar 2021 23:16:46 +0300 Subject: [PATCH 519/544] Add system.errors.last_error_time column --- docs/en/operations/system-tables/errors.md | 1 + src/Common/ErrorCodes.cpp | 5 +++++ src/Common/ErrorCodes.h | 1 + src/Storages/System/StorageSystemErrors.cpp | 9 ++++++--- 4 files changed, 13 insertions(+), 3 deletions(-) diff --git a/docs/en/operations/system-tables/errors.md b/docs/en/operations/system-tables/errors.md index bf3e33f5275..b440ae4d787 100644 --- a/docs/en/operations/system-tables/errors.md +++ b/docs/en/operations/system-tables/errors.md @@ -7,6 +7,7 @@ Columns: - `name` ([String](../../sql-reference/data-types/string.md)) — name of the error (`errorCodeToName`). - `code` ([Int32](../../sql-reference/data-types/int-uint.md)) — code number of the error. - `value` ([UInt64](../../sql-reference/data-types/int-uint.md)) — the number of times this error has been happened. +- `last_error_time` ([DateTime](../../sql-reference/data-types/datetime.md)) — time when the last error happened. - `remote` ([UInt8](../../sql-reference/data-types/int-uint.md)) — remote exception (i.e. received during one of the distributed query). **Example** diff --git a/src/Common/ErrorCodes.cpp b/src/Common/ErrorCodes.cpp index 32c9c4a452b..3532c063651 100644 --- a/src/Common/ErrorCodes.cpp +++ b/src/Common/ErrorCodes.cpp @@ -1,4 +1,5 @@ #include +#include /** Previously, these constants were located in one enum. * But in this case there is a problem: when you add a new constant, you need to recompile @@ -605,6 +606,10 @@ namespace ErrorCodes { local += value.local; remote += value.remote; + + const auto now = std::chrono::system_clock::now(); + last_error_time_ms = std::chrono::duration_cast(now.time_since_epoch()).count(); + return *this; } diff --git a/src/Common/ErrorCodes.h b/src/Common/ErrorCodes.h index 0db877db205..c8c454b51a7 100644 --- a/src/Common/ErrorCodes.h +++ b/src/Common/ErrorCodes.h @@ -28,6 +28,7 @@ namespace ErrorCodes { Value local = 0; Value remote = 0; + UInt64 last_error_time_ms = 0; ValuePair & operator+=(const ValuePair & value); }; diff --git a/src/Storages/System/StorageSystemErrors.cpp b/src/Storages/System/StorageSystemErrors.cpp index d57e8a0a670..a3d68ff5d86 100644 --- a/src/Storages/System/StorageSystemErrors.cpp +++ b/src/Storages/System/StorageSystemErrors.cpp @@ -1,5 +1,6 @@ #include #include +#include #include #include #include @@ -13,6 +14,7 @@ NamesAndTypesList StorageSystemErrors::getNamesAndTypes() { "name", std::make_shared() }, { "code", std::make_shared() }, { "value", std::make_shared() }, + { "last_error_time", std::make_shared() }, { "remote", std::make_shared() }, }; } @@ -20,7 +22,7 @@ NamesAndTypesList StorageSystemErrors::getNamesAndTypes() void StorageSystemErrors::fillData(MutableColumns & res_columns, const Context & context, const SelectQueryInfo &) const { - auto add_row = [&](std::string_view name, size_t code, size_t value, bool remote) + auto add_row = [&](std::string_view name, size_t code, size_t value, UInt64 last_error_time_ms, bool remote) { if (value || context.getSettingsRef().system_events_show_zero_values) { @@ -28,6 +30,7 @@ void StorageSystemErrors::fillData(MutableColumns & res_columns, const Context & res_columns[col_num++]->insert(name); res_columns[col_num++]->insert(code); res_columns[col_num++]->insert(value); + res_columns[col_num++]->insert(last_error_time_ms / 1000); res_columns[col_num++]->insert(remote); } }; @@ -40,8 +43,8 @@ void StorageSystemErrors::fillData(MutableColumns & res_columns, const Context & if (name.empty()) continue; - add_row(name, i, error.local, 0 /* remote=0 */); - add_row(name, i, error.remote, 1 /* remote=1 */); + add_row(name, i, error.local, error.last_error_time_ms, 0 /* remote=0 */); + add_row(name, i, error.remote, error.last_error_time_ms, 1 /* remote=1 */); } } From 775f8f76827378b98d0b679eaa564ac7697ee81f Mon Sep 17 00:00:00 2001 From: Azat Khuzhin Date: Mon, 8 Mar 2021 23:31:51 +0300 Subject: [PATCH 520/544] Add system.errors.last_error_message column --- docs/en/operations/system-tables/errors.md | 1 + src/Common/ErrorCodes.cpp | 5 ++++- src/Common/ErrorCodes.h | 3 ++- src/Common/Exception.cpp | 2 +- src/Storages/System/StorageSystemErrors.cpp | 8 +++++--- 5 files changed, 13 insertions(+), 6 deletions(-) diff --git a/docs/en/operations/system-tables/errors.md b/docs/en/operations/system-tables/errors.md index b440ae4d787..f8ac1de29a8 100644 --- a/docs/en/operations/system-tables/errors.md +++ b/docs/en/operations/system-tables/errors.md @@ -8,6 +8,7 @@ Columns: - `code` ([Int32](../../sql-reference/data-types/int-uint.md)) — code number of the error. - `value` ([UInt64](../../sql-reference/data-types/int-uint.md)) — the number of times this error has been happened. - `last_error_time` ([DateTime](../../sql-reference/data-types/datetime.md)) — time when the last error happened. +- `last_error_message` ([String](../../sql-reference/data-types/string.md)) — message for the last error. - `remote` ([UInt8](../../sql-reference/data-types/int-uint.md)) — remote exception (i.e. received during one of the distributed query). **Example** diff --git a/src/Common/ErrorCodes.cpp b/src/Common/ErrorCodes.cpp index 3532c063651..6c9de122a26 100644 --- a/src/Common/ErrorCodes.cpp +++ b/src/Common/ErrorCodes.cpp @@ -586,7 +586,7 @@ namespace ErrorCodes ErrorCode end() { return END + 1; } - void increment(ErrorCode error_code, bool remote) + void increment(ErrorCode error_code, bool remote, const std::string & message) { if (error_code >= end()) { @@ -598,6 +598,8 @@ namespace ErrorCodes ValuePair inc_value{ !remote, /* local */ remote, /* remote */ + 0, /* last_error_time_ms */ + message, /* message */ }; values[error_code].increment(inc_value); } @@ -606,6 +608,7 @@ namespace ErrorCodes { local += value.local; remote += value.remote; + message = value.message; const auto now = std::chrono::system_clock::now(); last_error_time_ms = std::chrono::duration_cast(now.time_since_epoch()).count(); diff --git a/src/Common/ErrorCodes.h b/src/Common/ErrorCodes.h index c8c454b51a7..962b1f8a20a 100644 --- a/src/Common/ErrorCodes.h +++ b/src/Common/ErrorCodes.h @@ -29,6 +29,7 @@ namespace ErrorCodes Value local = 0; Value remote = 0; UInt64 last_error_time_ms = 0; + std::string message; ValuePair & operator+=(const ValuePair & value); }; @@ -52,7 +53,7 @@ namespace ErrorCodes ErrorCode end(); /// Add value for specified error_code. - void increment(ErrorCode error_code, bool remote); + void increment(ErrorCode error_code, bool remote, const std::string & message); } } diff --git a/src/Common/Exception.cpp b/src/Common/Exception.cpp index 1963c1513b9..1fe224edc6e 100644 --- a/src/Common/Exception.cpp +++ b/src/Common/Exception.cpp @@ -47,7 +47,7 @@ void handle_error_code([[maybe_unused]] const std::string & msg, int code, bool abort(); } #endif - ErrorCodes::increment(code, remote); + ErrorCodes::increment(code, remote, msg); } Exception::Exception(const std::string & msg, int code, bool remote_) diff --git a/src/Storages/System/StorageSystemErrors.cpp b/src/Storages/System/StorageSystemErrors.cpp index a3d68ff5d86..c06bb13beb6 100644 --- a/src/Storages/System/StorageSystemErrors.cpp +++ b/src/Storages/System/StorageSystemErrors.cpp @@ -15,6 +15,7 @@ NamesAndTypesList StorageSystemErrors::getNamesAndTypes() { "code", std::make_shared() }, { "value", std::make_shared() }, { "last_error_time", std::make_shared() }, + { "last_error_message",std::make_shared() }, { "remote", std::make_shared() }, }; } @@ -22,7 +23,7 @@ NamesAndTypesList StorageSystemErrors::getNamesAndTypes() void StorageSystemErrors::fillData(MutableColumns & res_columns, const Context & context, const SelectQueryInfo &) const { - auto add_row = [&](std::string_view name, size_t code, size_t value, UInt64 last_error_time_ms, bool remote) + auto add_row = [&](std::string_view name, size_t code, size_t value, UInt64 last_error_time_ms, const std::string & message, bool remote) { if (value || context.getSettingsRef().system_events_show_zero_values) { @@ -31,6 +32,7 @@ void StorageSystemErrors::fillData(MutableColumns & res_columns, const Context & res_columns[col_num++]->insert(code); res_columns[col_num++]->insert(value); res_columns[col_num++]->insert(last_error_time_ms / 1000); + res_columns[col_num++]->insert(message); res_columns[col_num++]->insert(remote); } }; @@ -43,8 +45,8 @@ void StorageSystemErrors::fillData(MutableColumns & res_columns, const Context & if (name.empty()) continue; - add_row(name, i, error.local, error.last_error_time_ms, 0 /* remote=0 */); - add_row(name, i, error.remote, error.last_error_time_ms, 1 /* remote=1 */); + add_row(name, i, error.local, error.last_error_time_ms, error.message, 0 /* remote=0 */); + add_row(name, i, error.remote, error.last_error_time_ms, error.message, 1 /* remote=1 */); } } From 44c9dc753da4add80dd0a92ce14152790138e85a Mon Sep 17 00:00:00 2001 From: Azat Khuzhin Date: Mon, 8 Mar 2021 23:39:19 +0300 Subject: [PATCH 521/544] Add system.errors.last_error_stacktrace column --- docs/en/operations/system-tables/errors.md | 1 + src/Common/ErrorCodes.cpp | 12 +++++++----- src/Common/ErrorCodes.h | 3 ++- src/Common/Exception.cpp | 8 ++++---- src/Storages/System/StorageSystemErrors.cpp | 20 +++++++++++--------- 5 files changed, 25 insertions(+), 19 deletions(-) diff --git a/docs/en/operations/system-tables/errors.md b/docs/en/operations/system-tables/errors.md index f8ac1de29a8..72a537f15b9 100644 --- a/docs/en/operations/system-tables/errors.md +++ b/docs/en/operations/system-tables/errors.md @@ -9,6 +9,7 @@ Columns: - `value` ([UInt64](../../sql-reference/data-types/int-uint.md)) — the number of times this error has been happened. - `last_error_time` ([DateTime](../../sql-reference/data-types/datetime.md)) — time when the last error happened. - `last_error_message` ([String](../../sql-reference/data-types/string.md)) — message for the last error. +- `last_error_stacktrace` ([String](../../sql-reference/data-types/string.md)) — stacktrace for the last error. - `remote` ([UInt8](../../sql-reference/data-types/int-uint.md)) — remote exception (i.e. received during one of the distributed query). **Example** diff --git a/src/Common/ErrorCodes.cpp b/src/Common/ErrorCodes.cpp index 6c9de122a26..14182467351 100644 --- a/src/Common/ErrorCodes.cpp +++ b/src/Common/ErrorCodes.cpp @@ -586,7 +586,7 @@ namespace ErrorCodes ErrorCode end() { return END + 1; } - void increment(ErrorCode error_code, bool remote, const std::string & message) + void increment(ErrorCode error_code, bool remote, const std::string & message, const std::string & stacktrace) { if (error_code >= end()) { @@ -596,10 +596,11 @@ namespace ErrorCodes } ValuePair inc_value{ - !remote, /* local */ - remote, /* remote */ - 0, /* last_error_time_ms */ - message, /* message */ + !remote, /* local */ + remote, /* remote */ + 0, /* last_error_time_ms */ + message, /* message */ + stacktrace, /* stacktrace */ }; values[error_code].increment(inc_value); } @@ -609,6 +610,7 @@ namespace ErrorCodes local += value.local; remote += value.remote; message = value.message; + stacktrace = value.stacktrace; const auto now = std::chrono::system_clock::now(); last_error_time_ms = std::chrono::duration_cast(now.time_since_epoch()).count(); diff --git a/src/Common/ErrorCodes.h b/src/Common/ErrorCodes.h index 962b1f8a20a..4c79614d55d 100644 --- a/src/Common/ErrorCodes.h +++ b/src/Common/ErrorCodes.h @@ -30,6 +30,7 @@ namespace ErrorCodes Value remote = 0; UInt64 last_error_time_ms = 0; std::string message; + std::string stacktrace; ValuePair & operator+=(const ValuePair & value); }; @@ -53,7 +54,7 @@ namespace ErrorCodes ErrorCode end(); /// Add value for specified error_code. - void increment(ErrorCode error_code, bool remote, const std::string & message); + void increment(ErrorCode error_code, bool remote, const std::string & message, const std::string & stacktrace); } } diff --git a/src/Common/Exception.cpp b/src/Common/Exception.cpp index 1fe224edc6e..08afd0397f5 100644 --- a/src/Common/Exception.cpp +++ b/src/Common/Exception.cpp @@ -36,7 +36,7 @@ namespace ErrorCodes /// - Aborts the process if error code is LOGICAL_ERROR. /// - Increments error codes statistics. -void handle_error_code([[maybe_unused]] const std::string & msg, int code, bool remote) +void handle_error_code([[maybe_unused]] const std::string & msg, const std::string & stacktrace, int code, bool remote) { // In debug builds and builds with sanitizers, treat LOGICAL_ERROR as an assertion failure. // Log the message before we fail. @@ -47,20 +47,20 @@ void handle_error_code([[maybe_unused]] const std::string & msg, int code, bool abort(); } #endif - ErrorCodes::increment(code, remote, msg); + ErrorCodes::increment(code, remote, msg, stacktrace); } Exception::Exception(const std::string & msg, int code, bool remote_) : Poco::Exception(msg, code) , remote(remote_) { - handle_error_code(msg, code, remote); + handle_error_code(msg, getStackTraceString(), code, remote); } Exception::Exception(const std::string & msg, const Exception & nested, int code) : Poco::Exception(msg, nested, code) { - handle_error_code(msg, code, remote); + handle_error_code(msg, getStackTraceString(), code, remote); } Exception::Exception(CreateFromPocoTag, const Poco::Exception & exc) diff --git a/src/Storages/System/StorageSystemErrors.cpp b/src/Storages/System/StorageSystemErrors.cpp index c06bb13beb6..87cf3f2f603 100644 --- a/src/Storages/System/StorageSystemErrors.cpp +++ b/src/Storages/System/StorageSystemErrors.cpp @@ -11,19 +11,20 @@ namespace DB NamesAndTypesList StorageSystemErrors::getNamesAndTypes() { return { - { "name", std::make_shared() }, - { "code", std::make_shared() }, - { "value", std::make_shared() }, - { "last_error_time", std::make_shared() }, - { "last_error_message",std::make_shared() }, - { "remote", std::make_shared() }, + { "name", std::make_shared() }, + { "code", std::make_shared() }, + { "value", std::make_shared() }, + { "last_error_time", std::make_shared() }, + { "last_error_message", std::make_shared() }, + { "last_error_stacktrace", std::make_shared() }, + { "remote", std::make_shared() }, }; } void StorageSystemErrors::fillData(MutableColumns & res_columns, const Context & context, const SelectQueryInfo &) const { - auto add_row = [&](std::string_view name, size_t code, size_t value, UInt64 last_error_time_ms, const std::string & message, bool remote) + auto add_row = [&](std::string_view name, size_t code, size_t value, UInt64 last_error_time_ms, const std::string & message, const std::string & stacktrace, bool remote) { if (value || context.getSettingsRef().system_events_show_zero_values) { @@ -33,6 +34,7 @@ void StorageSystemErrors::fillData(MutableColumns & res_columns, const Context & res_columns[col_num++]->insert(value); res_columns[col_num++]->insert(last_error_time_ms / 1000); res_columns[col_num++]->insert(message); + res_columns[col_num++]->insert(stacktrace); res_columns[col_num++]->insert(remote); } }; @@ -45,8 +47,8 @@ void StorageSystemErrors::fillData(MutableColumns & res_columns, const Context & if (name.empty()) continue; - add_row(name, i, error.local, error.last_error_time_ms, error.message, 0 /* remote=0 */); - add_row(name, i, error.remote, error.last_error_time_ms, error.message, 1 /* remote=1 */); + add_row(name, i, error.local, error.last_error_time_ms, error.message, error.stacktrace, 0 /* remote=0 */); + add_row(name, i, error.remote, error.last_error_time_ms, error.message, error.stacktrace, 1 /* remote=1 */); } } From efdd04c958960233fad73f480366d3cfcbffba0d Mon Sep 17 00:00:00 2001 From: Azat Khuzhin Date: Mon, 8 Mar 2021 23:43:58 +0300 Subject: [PATCH 522/544] Drop last_ prefix for ErrorCodes::ValuePair::error_time_ms --- src/Common/ErrorCodes.cpp | 4 ++-- src/Common/ErrorCodes.h | 2 +- src/Storages/System/StorageSystemErrors.cpp | 8 ++++---- 3 files changed, 7 insertions(+), 7 deletions(-) diff --git a/src/Common/ErrorCodes.cpp b/src/Common/ErrorCodes.cpp index 14182467351..d7e0d5fb16a 100644 --- a/src/Common/ErrorCodes.cpp +++ b/src/Common/ErrorCodes.cpp @@ -598,7 +598,7 @@ namespace ErrorCodes ValuePair inc_value{ !remote, /* local */ remote, /* remote */ - 0, /* last_error_time_ms */ + 0, /* error_time_ms */ message, /* message */ stacktrace, /* stacktrace */ }; @@ -613,7 +613,7 @@ namespace ErrorCodes stacktrace = value.stacktrace; const auto now = std::chrono::system_clock::now(); - last_error_time_ms = std::chrono::duration_cast(now.time_since_epoch()).count(); + error_time_ms = std::chrono::duration_cast(now.time_since_epoch()).count(); return *this; } diff --git a/src/Common/ErrorCodes.h b/src/Common/ErrorCodes.h index 4c79614d55d..1c8f0a58884 100644 --- a/src/Common/ErrorCodes.h +++ b/src/Common/ErrorCodes.h @@ -28,7 +28,7 @@ namespace ErrorCodes { Value local = 0; Value remote = 0; - UInt64 last_error_time_ms = 0; + UInt64 error_time_ms = 0; std::string message; std::string stacktrace; diff --git a/src/Storages/System/StorageSystemErrors.cpp b/src/Storages/System/StorageSystemErrors.cpp index 87cf3f2f603..c16eba6754b 100644 --- a/src/Storages/System/StorageSystemErrors.cpp +++ b/src/Storages/System/StorageSystemErrors.cpp @@ -24,7 +24,7 @@ NamesAndTypesList StorageSystemErrors::getNamesAndTypes() void StorageSystemErrors::fillData(MutableColumns & res_columns, const Context & context, const SelectQueryInfo &) const { - auto add_row = [&](std::string_view name, size_t code, size_t value, UInt64 last_error_time_ms, const std::string & message, const std::string & stacktrace, bool remote) + auto add_row = [&](std::string_view name, size_t code, size_t value, UInt64 error_time_ms, const std::string & message, const std::string & stacktrace, bool remote) { if (value || context.getSettingsRef().system_events_show_zero_values) { @@ -32,7 +32,7 @@ void StorageSystemErrors::fillData(MutableColumns & res_columns, const Context & res_columns[col_num++]->insert(name); res_columns[col_num++]->insert(code); res_columns[col_num++]->insert(value); - res_columns[col_num++]->insert(last_error_time_ms / 1000); + res_columns[col_num++]->insert(error_time_ms / 1000); res_columns[col_num++]->insert(message); res_columns[col_num++]->insert(stacktrace); res_columns[col_num++]->insert(remote); @@ -47,8 +47,8 @@ void StorageSystemErrors::fillData(MutableColumns & res_columns, const Context & if (name.empty()) continue; - add_row(name, i, error.local, error.last_error_time_ms, error.message, error.stacktrace, 0 /* remote=0 */); - add_row(name, i, error.remote, error.last_error_time_ms, error.message, error.stacktrace, 1 /* remote=1 */); + add_row(name, i, error.local, error.error_time_ms, error.message, error.stacktrace, 0 /* remote=0 */); + add_row(name, i, error.remote, error.error_time_ms, error.message, error.stacktrace, 1 /* remote=1 */); } } From cc87bcfb63fbca0a9024364156d60dc703521c63 Mon Sep 17 00:00:00 2001 From: Azat Khuzhin Date: Tue, 9 Mar 2021 09:09:28 +0300 Subject: [PATCH 523/544] Fix errorCodeToName() for signed integers - https://clickhouse-test-reports.s3.yandex.net/21529/2ce2772d35eb3d81628f4d294d5799e9f05333fd/functional_stateless_tests_(address).html#fail1 - https://clickhouse-test-reports.s3.yandex.net/21529/2ce2772d35eb3d81628f4d294d5799e9f05333fd/functional_stateless_tests_(ubsan).html#fail1 - https://clickhouse-test-reports.s3.yandex.net/21529/2ce2772d35eb3d81628f4d294d5799e9f05333fd/stress_test_(address).html#fail1 --- src/Common/ErrorCodes.cpp | 2 +- tests/queries/0_stateless/01544_errorCodeToName.reference | 1 + tests/queries/0_stateless/01544_errorCodeToName.sql | 1 + 3 files changed, 3 insertions(+), 1 deletion(-) diff --git a/src/Common/ErrorCodes.cpp b/src/Common/ErrorCodes.cpp index d7e0d5fb16a..879784bb43a 100644 --- a/src/Common/ErrorCodes.cpp +++ b/src/Common/ErrorCodes.cpp @@ -579,7 +579,7 @@ namespace ErrorCodes std::string_view getName(ErrorCode error_code) { - if (error_code >= END) + if (error_code < 0 || error_code >= END) return std::string_view(); return error_codes_names.names[error_code]; } diff --git a/tests/queries/0_stateless/01544_errorCodeToName.reference b/tests/queries/0_stateless/01544_errorCodeToName.reference index ace588644e1..fefccf984be 100644 --- a/tests/queries/0_stateless/01544_errorCodeToName.reference +++ b/tests/queries/0_stateless/01544_errorCodeToName.reference @@ -1,4 +1,5 @@ + OK UNSUPPORTED_METHOD diff --git a/tests/queries/0_stateless/01544_errorCodeToName.sql b/tests/queries/0_stateless/01544_errorCodeToName.sql index 9e28ed1116c..aa32270f00b 100644 --- a/tests/queries/0_stateless/01544_errorCodeToName.sql +++ b/tests/queries/0_stateless/01544_errorCodeToName.sql @@ -1,4 +1,5 @@ SELECT errorCodeToName(toUInt32(-1)); +SELECT errorCodeToName(-1); SELECT errorCodeToName(600); /* gap in error codes */ SELECT errorCodeToName(0); SELECT errorCodeToName(1); From 9921e7ca284b7d274af540d13812881dd6a0e578 Mon Sep 17 00:00:00 2001 From: Azat Khuzhin Date: Tue, 9 Mar 2021 09:12:08 +0300 Subject: [PATCH 524/544] Add 01545_system_errors into skip_list.parallel https://clickhouse-test-reports.s3.yandex.net/21529/2ce2772d35eb3d81628f4d294d5799e9f05333fd/functional_stateless_tests_flaky_check_(address).html#fail1 --- tests/queries/skip_list.json | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/queries/skip_list.json b/tests/queries/skip_list.json index bded0807db9..caab92636b3 100644 --- a/tests/queries/skip_list.json +++ b/tests/queries/skip_list.json @@ -739,6 +739,7 @@ "01541_max_memory_usage_for_user_long", "01542_dictionary_load_exception_race", "01560_optimize_on_insert_zookeeper", + "01545_system_errors", // looks at the difference of values in system.errors "01575_disable_detach_table_of_dictionary", "01593_concurrent_alter_mutations_kill", "01593_concurrent_alter_mutations_kill_many_replicas", From a337691b060e9ad2c2cf53e9762691352f732837 Mon Sep 17 00:00:00 2001 From: Azat Khuzhin Date: Tue, 9 Mar 2021 10:05:56 +0300 Subject: [PATCH 525/544] Fix modernize-use-bool-literals clang-tidy warning in StorageSystemErrors --- src/Storages/System/StorageSystemErrors.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/Storages/System/StorageSystemErrors.cpp b/src/Storages/System/StorageSystemErrors.cpp index c16eba6754b..c9aac9ce007 100644 --- a/src/Storages/System/StorageSystemErrors.cpp +++ b/src/Storages/System/StorageSystemErrors.cpp @@ -47,8 +47,8 @@ void StorageSystemErrors::fillData(MutableColumns & res_columns, const Context & if (name.empty()) continue; - add_row(name, i, error.local, error.error_time_ms, error.message, error.stacktrace, 0 /* remote=0 */); - add_row(name, i, error.remote, error.error_time_ms, error.message, error.stacktrace, 1 /* remote=1 */); + add_row(name, i, error.local, error.error_time_ms, error.message, error.stacktrace, false /* remote=0 */); + add_row(name, i, error.remote, error.error_time_ms, error.message, error.stacktrace, true /* remote=1 */); } } From 9dee842b6082a3e90c65a8a45e5a357de30106a6 Mon Sep 17 00:00:00 2001 From: Azat Khuzhin Date: Tue, 16 Mar 2021 21:31:14 +0300 Subject: [PATCH 526/544] Distinguish remote and local error info --- src/Common/ErrorCodes.cpp | 34 +++++++-------------- src/Common/ErrorCodes.h | 26 ++++++++++------ src/Storages/System/StorageSystemErrors.cpp | 16 +++++----- 3 files changed, 35 insertions(+), 41 deletions(-) diff --git a/src/Common/ErrorCodes.cpp b/src/Common/ErrorCodes.cpp index 879784bb43a..a4e2f8742ca 100644 --- a/src/Common/ErrorCodes.cpp +++ b/src/Common/ErrorCodes.cpp @@ -564,7 +564,7 @@ namespace ErrorCodes #undef M constexpr ErrorCode END = 3000; - ValuePairHolder values[END + 1]{}; + ErrorPairHolder values[END + 1]{}; struct ErrorCodesNames { @@ -595,35 +595,23 @@ namespace ErrorCodes error_code = end() - 1; } - ValuePair inc_value{ - !remote, /* local */ - remote, /* remote */ - 0, /* error_time_ms */ - message, /* message */ - stacktrace, /* stacktrace */ - }; - values[error_code].increment(inc_value); + values[error_code].increment(remote, message, stacktrace); } - ValuePair & ValuePair::operator+=(const ValuePair & value) + void ErrorPairHolder::increment(bool remote, const std::string & message, const std::string & stacktrace) { - local += value.local; - remote += value.remote; - message = value.message; - stacktrace = value.stacktrace; - const auto now = std::chrono::system_clock::now(); - error_time_ms = std::chrono::duration_cast(now.time_since_epoch()).count(); - return *this; - } - - void ValuePairHolder::increment(const ValuePair & value_) - { std::lock_guard lock(mutex); - value += value_; + + auto & error = remote ? value.remote : value.local; + + ++error.count; + error.message = message; + error.stacktrace = stacktrace; + error.error_time_ms = std::chrono::duration_cast(now.time_since_epoch()).count(); } - ValuePair ValuePairHolder::get() + ErrorPair ErrorPairHolder::get() { std::lock_guard lock(mutex); return value; diff --git a/src/Common/ErrorCodes.h b/src/Common/ErrorCodes.h index 1c8f0a58884..edb9be9e0c0 100644 --- a/src/Common/ErrorCodes.h +++ b/src/Common/ErrorCodes.h @@ -24,31 +24,37 @@ namespace ErrorCodes /// Returns statically allocated string. std::string_view getName(ErrorCode error_code); - struct ValuePair + struct Error { - Value local = 0; - Value remote = 0; + /// Number of times Exception with this ErrorCode had been throw. + Value count; + /// Time of the last error. UInt64 error_time_ms = 0; + /// Message for the last error. std::string message; + /// Stacktrace for the last error. std::string stacktrace; - - ValuePair & operator+=(const ValuePair & value); + }; + struct ErrorPair + { + Error local; + Error remote; }; /// Thread-safe - struct ValuePairHolder + struct ErrorPairHolder { public: - void increment(const ValuePair & value_); - ValuePair get(); + ErrorPair get(); + void increment(bool remote, const std::string & message, const std::string & stacktrace); private: - ValuePair value; + ErrorPair value; std::mutex mutex; }; /// ErrorCode identifier -> current value of error_code. - extern ValuePairHolder values[]; + extern ErrorPairHolder values[]; /// Get index just after last error_code identifier. ErrorCode end(); diff --git a/src/Storages/System/StorageSystemErrors.cpp b/src/Storages/System/StorageSystemErrors.cpp index c9aac9ce007..5243cb11aa3 100644 --- a/src/Storages/System/StorageSystemErrors.cpp +++ b/src/Storages/System/StorageSystemErrors.cpp @@ -24,17 +24,17 @@ NamesAndTypesList StorageSystemErrors::getNamesAndTypes() void StorageSystemErrors::fillData(MutableColumns & res_columns, const Context & context, const SelectQueryInfo &) const { - auto add_row = [&](std::string_view name, size_t code, size_t value, UInt64 error_time_ms, const std::string & message, const std::string & stacktrace, bool remote) + auto add_row = [&](std::string_view name, size_t code, const auto & error, bool remote) { - if (value || context.getSettingsRef().system_events_show_zero_values) + if (error.count || context.getSettingsRef().system_events_show_zero_values) { size_t col_num = 0; res_columns[col_num++]->insert(name); res_columns[col_num++]->insert(code); - res_columns[col_num++]->insert(value); - res_columns[col_num++]->insert(error_time_ms / 1000); - res_columns[col_num++]->insert(message); - res_columns[col_num++]->insert(stacktrace); + res_columns[col_num++]->insert(error.count); + res_columns[col_num++]->insert(error.error_time_ms / 1000); + res_columns[col_num++]->insert(error.message); + res_columns[col_num++]->insert(error.stacktrace); res_columns[col_num++]->insert(remote); } }; @@ -47,8 +47,8 @@ void StorageSystemErrors::fillData(MutableColumns & res_columns, const Context & if (name.empty()) continue; - add_row(name, i, error.local, error.error_time_ms, error.message, error.stacktrace, false /* remote=0 */); - add_row(name, i, error.remote, error.error_time_ms, error.message, error.stacktrace, true /* remote=1 */); + add_row(name, i, error.local, /* remote= */ false); + add_row(name, i, error.remote, /* remote= */ true); } } From 37a17749ea57b9d1958b3f0dba70eaff0e61883a Mon Sep 17 00:00:00 2001 From: 3ldar-nasyrov <80788015+3ldar-nasyrov@users.noreply.github.com> Date: Tue, 16 Mar 2021 23:17:07 +0300 Subject: [PATCH 527/544] fixed exceeded amount of tries typo 'retires' -> 'retries' --- src/Interpreters/DDLWorker.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Interpreters/DDLWorker.cpp b/src/Interpreters/DDLWorker.cpp index 0ecb27ee3aa..eceb48ae773 100644 --- a/src/Interpreters/DDLWorker.cpp +++ b/src/Interpreters/DDLWorker.cpp @@ -872,7 +872,7 @@ bool DDLWorker::tryExecuteQueryOnLeaderReplica( else /// If we exceeded amount of tries { LOG_WARNING(log, "Task {} was not executed by anyone, maximum number of retries exceeded", task.entry_name); - task.execution_status = ExecutionStatus(ErrorCodes::UNFINISHED, "Cannot execute replicated DDL query, maximum retires exceeded"); + task.execution_status = ExecutionStatus(ErrorCodes::UNFINISHED, "Cannot execute replicated DDL query, maximum retries exceeded"); } return false; } From 9ca0566132016abeb25282f2a14127da1ee463f6 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Wed, 17 Mar 2021 02:08:23 +0300 Subject: [PATCH 528/544] Minor modification --- src/Compression/LZ4_decompress_faster.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Compression/LZ4_decompress_faster.h b/src/Compression/LZ4_decompress_faster.h index dd923279ebf..30a0d7acb22 100644 --- a/src/Compression/LZ4_decompress_faster.h +++ b/src/Compression/LZ4_decompress_faster.h @@ -95,7 +95,7 @@ struct PerformanceStatistics /// How to select method to run. /// -1 - automatically, based on statistics (default); - /// 0..3 - always choose specified method (for performance testing); + /// >= 0 - always choose specified method (for performance testing); /// -2 - choose methods in round robin fashion (for performance testing). ssize_t choose_method = -1; From 9969124cc43555369f7ba36a3b3e911f3d441f8a Mon Sep 17 00:00:00 2001 From: George Date: Wed, 17 Mar 2021 03:09:42 +0300 Subject: [PATCH 529/544] Change wording --- docs/en/sql-reference/statements/detach.md | 2 +- docs/ru/sql-reference/statements/attach.md | 4 ++-- docs/ru/sql-reference/statements/detach.md | 14 +++++++------- 3 files changed, 10 insertions(+), 10 deletions(-) diff --git a/docs/en/sql-reference/statements/detach.md b/docs/en/sql-reference/statements/detach.md index cb0d7cf7b66..e9c9ed3693c 100644 --- a/docs/en/sql-reference/statements/detach.md +++ b/docs/en/sql-reference/statements/detach.md @@ -5,7 +5,7 @@ toc_title: DETACH # DETACH Statement {#detach} -Deletes information about the table or materialized view from the server. The server stops knowing about their existence. +Makes the server "forget" about the existence of the table or materialized view. Syntax: diff --git a/docs/ru/sql-reference/statements/attach.md b/docs/ru/sql-reference/statements/attach.md index be5b0b6d44a..55d4db80099 100644 --- a/docs/ru/sql-reference/statements/attach.md +++ b/docs/ru/sql-reference/statements/attach.md @@ -10,7 +10,7 @@ toc_title: ATTACH - вместо слова `CREATE` используется слово `ATTACH`; - запрос не создаёт данные на диске, а предполагает, что данные уже лежат в соответствующих местах, и всего лишь добавляет информацию о таблице на сервер. После выполнения запроса `ATTACH` сервер будет знать о существовании таблицы. -Если таблица перед этим была откреплена ([DETACH](../../sql-reference/statements/detach.md)), т.е. её структура известна, можно использовать сокращенную форму записи без определения структуры. +Если таблица перед этим была отключена ([DETACH](../../sql-reference/statements/detach.md)), т.е. её структура известна, можно использовать сокращенную форму записи без определения структуры. ``` sql ATTACH TABLE [IF NOT EXISTS] [db.]name [ON CLUSTER cluster] @@ -18,7 +18,7 @@ ATTACH TABLE [IF NOT EXISTS] [db.]name [ON CLUSTER cluster] Этот запрос используется при старте сервера. Сервер хранит метаданные таблиц в виде файлов с запросами `ATTACH`, которые он просто исполняет при запуске (за исключением некоторых системных таблиц, которые явно создаются на сервере). -Если таблица была откреплена перманентно, она не будет прикреплена обратно во время старта сервера, так что нужно явно использовать запрос `ATTACH`, чтобы прикрепить ее. +Если таблица была отключена перманентно, она не будет подключена обратно во время старта сервера, так что нужно явно использовать запрос `ATTACH`, чтобы подключить ее. [Оригинальная статья](https://clickhouse.tech/docs/ru/sql-reference/statements/attach/) diff --git a/docs/ru/sql-reference/statements/detach.md b/docs/ru/sql-reference/statements/detach.md index 90e9663def9..1809d85b958 100644 --- a/docs/ru/sql-reference/statements/detach.md +++ b/docs/ru/sql-reference/statements/detach.md @@ -5,7 +5,7 @@ toc_title: DETACH # DETACH {#detach-statement} -Удаляет из сервера информацию о таблице или материализованном представлении. Сервер перестаёт знать о существовании таблицы. +Заставляет сервер "забыть" о существовании таблицы или материализованного представления. Синтаксис: @@ -13,15 +13,15 @@ toc_title: DETACH DETACH TABLE|VIEW [IF EXISTS] [db.]name [PERMANENTLY] [ON CLUSTER cluster] ``` -Но ни данные, ни метаданные таблицы или материализованного представления не удаляются. При следующем запуске сервера, если не было использовано `PERMANENTLY`, сервер прочитает метаданные и снова узнает о таблице/представлении. Если таблица или представление были откреплено перманентно, сервер не прикрепит их обратно автоматически. +Но ни данные, ни метаданные таблицы или материализованного представления не удаляются. При следующем запуске сервера, если не было использовано `PERMANENTLY`, сервер прочитает метаданные и снова узнает о таблице/представлении. Если таблица или представление были отключены перманентно, сервер не подключит их обратно автоматически. -Независимо от того, каким способом таблица была откреплена, ее можно прикрепить обратно с помощью запроса [ATTACH](../../sql-reference/statements/attach.md). Системные log таблицы также могут быть прикреплены обратно (к примеру `query_log`, `text_log` и др.) Другие системные таблицы не могут быть прикреплены обратно, но на следующем запуске сервер снова "вспомнит" об этих таблицах. +Независимо от того, каким способом таблица была отключена, ее можно подключить обратно с помощью запроса [ATTACH](../../sql-reference/statements/attach.md). Системные log таблицы также могут быть подключены обратно (к примеру `query_log`, `text_log` и др.) Другие системные таблицы не могут быть подключены обратно, но на следующем запуске сервер снова "вспомнит" об этих таблицах. -`ATTACH MATERIALIZED VIEW` не может быть использован с кратким синтаксисом (без `SELECT`), но можно прикрепить представление с помощью запроса `ATTACH TABLE`. +`ATTACH MATERIALIZED VIEW` не может быть использован с кратким синтаксисом (без `SELECT`), но можно подключить представление с помощью запроса `ATTACH TABLE`. -Обратите внимание, что нельзя перманентно открепить таблицу, которая уже временно откреплена. Для этого ее сначала надо прикрепить обратно, а затем снова открепить перманентно. +Обратите внимание, что нельзя перманентно отключить таблицу, которая уже временно отключена. Для этого ее сначала надо подключить обратно, а затем снова отключить перманентно. -Также нельзя использовать [DROP](../../sql-reference/statements/drop.md#drop-table) с открепленной таблицей или создавать таблицу с помощью [CREATE TABLE](../../sql-reference/statements/create/table.md) с таким же именем, как у уже открепленной таблицы. Еще нельзя заменить открепленную таблицу другой с помощью запроса [RENAME TABLE](../../sql-reference/statements/rename.md). +Также нельзя использовать [DROP](../../sql-reference/statements/drop.md#drop-table) с отключенной таблицей или создавать таблицу с помощью [CREATE TABLE](../../sql-reference/statements/create/table.md) с таким же именем, как у отключенной таблицы. Еще нельзя заменить отключенную таблицу другой с помощью запроса [RENAME TABLE](../../sql-reference/statements/rename.md). **Пример** @@ -51,7 +51,7 @@ SELECT * FROM test; └────────┘ ``` -Открепление таблицы: +Отключение таблицы: Запрос: From 0640bb4cbaadafed46c74fc1f528200a99851145 Mon Sep 17 00:00:00 2001 From: Amos Bird Date: Wed, 17 Mar 2021 14:33:57 +0800 Subject: [PATCH 530/544] Fix arena data race in two level merge --- src/Interpreters/Aggregator.cpp | 39 +++++++++++++++++++++------------ 1 file changed, 25 insertions(+), 14 deletions(-) diff --git a/src/Interpreters/Aggregator.cpp b/src/Interpreters/Aggregator.cpp index ea81155e26a..ee8132cd40c 100644 --- a/src/Interpreters/Aggregator.cpp +++ b/src/Interpreters/Aggregator.cpp @@ -1389,35 +1389,46 @@ BlocksList Aggregator::prepareBlocksAndFillTwoLevelImpl( for (size_t i = data_variants.aggregates_pools.size(); i < max_threads; ++i) data_variants.aggregates_pools.push_back(std::make_shared()); - auto converter = [&](size_t bucket, ThreadGroupStatusPtr thread_group) + std::atomic next_bucket_to_merge = 0; + + auto converter = [&](size_t thread_id, ThreadGroupStatusPtr thread_group) { if (thread_group) CurrentThread::attachToIfDetached(thread_group); - /// Select Arena to avoid race conditions - size_t thread_number = static_cast(bucket) % max_threads; - Arena * arena = data_variants.aggregates_pools.at(thread_number).get(); + BlocksList blocks; + while (true) + { + UInt32 bucket = next_bucket_to_merge.fetch_add(1); - return convertOneBucketToBlock(data_variants, method, arena, final, bucket); + if (bucket >= Method::Data::NUM_BUCKETS) + break; + + if (method.data.impls[bucket].empty()) + continue; + + /// Select Arena to avoid race conditions + Arena * arena = data_variants.aggregates_pools.at(thread_id).get(); + blocks.emplace_back(convertOneBucketToBlock(data_variants, method, arena, final, bucket)); + } + return blocks; }; /// packaged_task is used to ensure that exceptions are automatically thrown into the main stream. - std::vector> tasks(Method::Data::NUM_BUCKETS); + std::vector> tasks(max_threads); try { - for (size_t bucket = 0; bucket < Method::Data::NUM_BUCKETS; ++bucket) + for (size_t thread_id = 0; thread_id < max_threads; ++thread_id) { - if (method.data.impls[bucket].empty()) - continue; - - tasks[bucket] = std::packaged_task([group = CurrentThread::getGroup(), bucket, &converter]{ return converter(bucket, group); }); + tasks[thread_id] = std::packaged_task( + [group = CurrentThread::getGroup(), thread_id, &converter] { return converter(thread_id, group); }); if (thread_pool) - thread_pool->scheduleOrThrowOnError([bucket, &tasks] { tasks[bucket](); }); + thread_pool->scheduleOrThrowOnError([thread_id, &tasks] { tasks[thread_id](); }); else - tasks[bucket](); + tasks[thread_id](); } } catch (...) @@ -1439,7 +1450,7 @@ BlocksList Aggregator::prepareBlocksAndFillTwoLevelImpl( if (!task.valid()) continue; - blocks.emplace_back(task.get_future().get()); + blocks.splice(blocks.end(), task.get_future().get()); } return blocks; From f69b6ecf0e030716574fae31b69b06db6ff9eae1 Mon Sep 17 00:00:00 2001 From: Vasily Nemkov Date: Wed, 17 Mar 2021 13:16:41 +0200 Subject: [PATCH 531/544] Fixed cross-links to other pages --- docs/en/sql-reference/statements/optimize.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/en/sql-reference/statements/optimize.md b/docs/en/sql-reference/statements/optimize.md index ea1fac90466..49a7404d76e 100644 --- a/docs/en/sql-reference/statements/optimize.md +++ b/docs/en/sql-reference/statements/optimize.md @@ -23,7 +23,7 @@ When `OPTIMIZE` is used with the [ReplicatedMergeTree](../../engines/table-engin ### BY expression {#by-expression} -If you want to perform deduplication on custom set of columns rather than on all, you can specify list of columns explicitly or use any combination of [`*`](../../../sql-reference/statements/select/index.md#asterisk), [`COLUMNS`](../../../sql-reference/statements/select/index.md#columns-expression) or [`EXCEPT`](../../../sql-reference/statements/select/index.md#except-modifier) expressions. The explictly written or implicitly expanded list of columns must include all columns specified in row ordering expression (both primary and sorting keys) and partitioning expression (partitioning key). +If you want to perform deduplication on custom set of columns rather than on all, you can specify list of columns explicitly or use any combination of [`*`](../../sql-reference/statements/select/index.md#asterisk), [`COLUMNS`](../../sql-reference/statements/select/index.md#columns-expression) or [`EXCEPT`](../../sql-reference/statements/select/index.md#except-modifier) expressions. The explictly written or implicitly expanded list of columns must include all columns specified in row ordering expression (both primary and sorting keys) and partitioning expression (partitioning key). Note that `*` behaves just like in `SELECT`: `MATERIALIZED`, and `ALIAS` columns are not used for expansion. Also, it is an error to specify empty list of columns, or write an expression that results in an empty list of columns, or deduplicate by an ALIAS column. From e6158be4dac1f19dfad279035dc721618c75bb16 Mon Sep 17 00:00:00 2001 From: alexey-milovidov Date: Wed, 17 Mar 2021 14:54:14 +0300 Subject: [PATCH 532/544] Update CHANGELOG.md --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 928991dc937..43531b60267 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,4 +1,4 @@ -## ClickHouse release 21.3 +## ClickHouse release 21.3 (LTS) ### ClickHouse release v21.3, 2021-03-12 From f8fc4281f2c540128cdb689dd6c909b63b037ef3 Mon Sep 17 00:00:00 2001 From: Kseniia Sumarokova <54203879+kssenii@users.noreply.github.com> Date: Wed, 17 Mar 2021 15:56:47 +0300 Subject: [PATCH 533/544] Update PostgreSQLReplicaConnection.h --- src/Storages/PostgreSQL/PostgreSQLReplicaConnection.h | 1 + 1 file changed, 1 insertion(+) diff --git a/src/Storages/PostgreSQL/PostgreSQLReplicaConnection.h b/src/Storages/PostgreSQL/PostgreSQLReplicaConnection.h index 9465d4a119b..e58d4bc8100 100644 --- a/src/Storages/PostgreSQL/PostgreSQLReplicaConnection.h +++ b/src/Storages/PostgreSQL/PostgreSQLReplicaConnection.h @@ -25,6 +25,7 @@ public: private: + /// Highest priority is 0, the bigger the number in map, the less the priority using ReplicasByPriority = std::map; Poco::Logger * log; From 738cb1af62a9f65846bba21757c9032583c9d00f Mon Sep 17 00:00:00 2001 From: Nikolai Kochetov Date: Thu, 11 Mar 2021 13:34:15 +0300 Subject: [PATCH 534/544] Fix filter push down columns order. --- src/Interpreters/ActionsDAG.cpp | 30 ++++++++++++++----- src/Interpreters/ActionsDAG.h | 4 +-- .../Optimizations/filterPushDown.cpp | 3 +- 3 files changed, 27 insertions(+), 10 deletions(-) diff --git a/src/Interpreters/ActionsDAG.cpp b/src/Interpreters/ActionsDAG.cpp index 83844176f3b..31abbc89634 100644 --- a/src/Interpreters/ActionsDAG.cpp +++ b/src/Interpreters/ActionsDAG.cpp @@ -1360,7 +1360,8 @@ ColumnsWithTypeAndName prepareFunctionArguments(const std::vector conjunction) + +ActionsDAGPtr ActionsDAG::cloneActionsForConjunction(std::vector conjunction, const ColumnsWithTypeAndName & all_inputs) { if (conjunction.empty()) return nullptr; @@ -1374,6 +1375,7 @@ ActionsDAGPtr ActionsDAG::cloneActionsForConjunction(std::vector conjunc std::make_shared())); std::unordered_map nodes_mapping; + std::unordered_map> added_inputs; struct Frame { @@ -1416,16 +1418,30 @@ ActionsDAGPtr ActionsDAG::cloneActionsForConjunction(std::vector conjunc child = nodes_mapping[child]; if (node.type == ActionType::INPUT) - { - actions->inputs.emplace_back(&node); - actions->index.insert(&node); - } + added_inputs[node.result_name].push_back(&node); stack.pop(); } } } + + for (const auto & col : all_inputs) + { + Node * input; + auto & list = added_inputs[col.name]; + if (list.empty()) + input = &const_cast(actions->addInput(col)); + else + { + input = list.front(); + list.pop_front(); + actions->inputs.push_back(input); + } + + actions->index.insert(input); + } + Node * result_predicate = nodes_mapping[*conjunction.begin()]; if (conjunction.size() > 1) @@ -1442,7 +1458,7 @@ ActionsDAGPtr ActionsDAG::cloneActionsForConjunction(std::vector conjunc return actions; } -ActionsDAGPtr ActionsDAG::splitActionsForFilter(const std::string & filter_name, bool can_remove_filter, const Names & available_inputs) +ActionsDAGPtr ActionsDAG::splitActionsForFilter(const std::string & filter_name, bool can_remove_filter, const Names & available_inputs, const ColumnsWithTypeAndName & all_inputs) { Node * predicate; @@ -1480,7 +1496,7 @@ ActionsDAGPtr ActionsDAG::splitActionsForFilter(const std::string & filter_name, } auto conjunction = getConjunctionNodes(predicate, allowed_nodes); - auto actions = cloneActionsForConjunction(conjunction.allowed); + auto actions = cloneActionsForConjunction(conjunction.allowed, all_inputs); if (!actions) return nullptr; diff --git a/src/Interpreters/ActionsDAG.h b/src/Interpreters/ActionsDAG.h index 165f712a627..a3cfd9cb776 100644 --- a/src/Interpreters/ActionsDAG.h +++ b/src/Interpreters/ActionsDAG.h @@ -292,7 +292,7 @@ public: /// Otherwise, return actions which inputs are from available_inputs. /// Returned actions add single column which may be used for filter. /// Also, replace some nodes of current inputs to constant 1 in case they are filtered. - ActionsDAGPtr splitActionsForFilter(const std::string & filter_name, bool can_remove_filter, const Names & available_inputs); + ActionsDAGPtr splitActionsForFilter(const std::string & filter_name, bool can_remove_filter, const Names & available_inputs, const ColumnsWithTypeAndName & all_inputs); private: Node & addNode(Node node, bool can_replace = false, bool add_to_index = true); @@ -323,7 +323,7 @@ private: void compileFunctions(); - ActionsDAGPtr cloneActionsForConjunction(std::vector conjunction); + ActionsDAGPtr cloneActionsForConjunction(std::vector conjunction, const ColumnsWithTypeAndName & all_inputs); }; diff --git a/src/Processors/QueryPlan/Optimizations/filterPushDown.cpp b/src/Processors/QueryPlan/Optimizations/filterPushDown.cpp index d64f082b7ee..f6a4eecbad1 100644 --- a/src/Processors/QueryPlan/Optimizations/filterPushDown.cpp +++ b/src/Processors/QueryPlan/Optimizations/filterPushDown.cpp @@ -43,7 +43,8 @@ static size_t tryAddNewFilterStep( // std::cerr << "Filter: \n" << expression->dumpDAG() << std::endl; - auto split_filter = expression->splitActionsForFilter(filter_column_name, removes_filter, allowed_inputs); + const auto & all_inputs = child->getInputStreams().front().header.getColumnsWithTypeAndName(); + auto split_filter = expression->splitActionsForFilter(filter_column_name, removes_filter, allowed_inputs, all_inputs); if (!split_filter) return 0; From 3eba817a688f95744679779775358dc88310a56d Mon Sep 17 00:00:00 2001 From: Nikolai Kochetov Date: Wed, 17 Mar 2021 19:08:46 +0300 Subject: [PATCH 535/544] Add comments. --- src/Interpreters/ActionsDAG.cpp | 13 ++++++++----- src/Interpreters/ActionsDAG.h | 18 +++++++++++++++++- .../QueryPlan/Optimizations/filterPushDown.cpp | 2 +- 3 files changed, 26 insertions(+), 7 deletions(-) diff --git a/src/Interpreters/ActionsDAG.cpp b/src/Interpreters/ActionsDAG.cpp index 31abbc89634..e67ab82a5ca 100644 --- a/src/Interpreters/ActionsDAG.cpp +++ b/src/Interpreters/ActionsDAG.cpp @@ -1360,7 +1360,6 @@ ColumnsWithTypeAndName prepareFunctionArguments(const std::vector conjunction, const ColumnsWithTypeAndName & all_inputs) { if (conjunction.empty()) @@ -1375,7 +1374,7 @@ ActionsDAGPtr ActionsDAG::cloneActionsForConjunction(std::vector conjunc std::make_shared())); std::unordered_map nodes_mapping; - std::unordered_map> added_inputs; + std::unordered_map> required_inputs; struct Frame { @@ -1418,7 +1417,7 @@ ActionsDAGPtr ActionsDAG::cloneActionsForConjunction(std::vector conjunc child = nodes_mapping[child]; if (node.type == ActionType::INPUT) - added_inputs[node.result_name].push_back(&node); + required_inputs[node.result_name].push_back(&node); stack.pop(); } @@ -1429,7 +1428,7 @@ ActionsDAGPtr ActionsDAG::cloneActionsForConjunction(std::vector conjunc for (const auto & col : all_inputs) { Node * input; - auto & list = added_inputs[col.name]; + auto & list = required_inputs[col.name]; if (list.empty()) input = &const_cast(actions->addInput(col)); else @@ -1458,7 +1457,11 @@ ActionsDAGPtr ActionsDAG::cloneActionsForConjunction(std::vector conjunc return actions; } -ActionsDAGPtr ActionsDAG::splitActionsForFilter(const std::string & filter_name, bool can_remove_filter, const Names & available_inputs, const ColumnsWithTypeAndName & all_inputs) +ActionsDAGPtr ActionsDAG::cloneActionsForFilterPushDown( + const std::string & filter_name, + bool can_remove_filter, + const Names & available_inputs, + const ColumnsWithTypeAndName & all_inputs) { Node * predicate; diff --git a/src/Interpreters/ActionsDAG.h b/src/Interpreters/ActionsDAG.h index a3cfd9cb776..e0e0e9c8957 100644 --- a/src/Interpreters/ActionsDAG.h +++ b/src/Interpreters/ActionsDAG.h @@ -292,7 +292,23 @@ public: /// Otherwise, return actions which inputs are from available_inputs. /// Returned actions add single column which may be used for filter. /// Also, replace some nodes of current inputs to constant 1 in case they are filtered. - ActionsDAGPtr splitActionsForFilter(const std::string & filter_name, bool can_remove_filter, const Names & available_inputs, const ColumnsWithTypeAndName & all_inputs); + /// + /// @param all_inputs should containt inputs from previous step, which will be used for result actions. + /// It is expected that all_inputs contain columns from available_inputs. + /// This parameter is needed to enforce result actions save columns order in block. + /// Otherwise for some queries, e.g. with GROUP BY, columns colum be swapped. + /// Example: SELECT sum(x), y, z FROM tab WHERE z > 0 and sum(x) > 0 + /// Pushed condition: z > 0 + /// GROUP BY step will transform columns `x, y, z` -> `sum(x), y, z` + /// If we just add filter step with actions `z -> z > 0` before GROUP BY, + /// columns will be transformed like `x, y, z` -> `z, z > 0, x, y` -(remove filter)-> `z, z, y`. + /// To avoid it, add inputs from `all_inputs` list, + /// so actions `x, y, z -> x, y, z, z > 0` -(remove filter)-> `x, y, z` will not change columns order. + ActionsDAGPtr cloneActionsForFilterPushDown( + const std::string & filter_name, + bool can_remove_filter, + const Names & available_inputs, + const ColumnsWithTypeAndName & all_inputs); private: Node & addNode(Node node, bool can_replace = false, bool add_to_index = true); diff --git a/src/Processors/QueryPlan/Optimizations/filterPushDown.cpp b/src/Processors/QueryPlan/Optimizations/filterPushDown.cpp index f6a4eecbad1..0b988f9803f 100644 --- a/src/Processors/QueryPlan/Optimizations/filterPushDown.cpp +++ b/src/Processors/QueryPlan/Optimizations/filterPushDown.cpp @@ -44,7 +44,7 @@ static size_t tryAddNewFilterStep( // std::cerr << "Filter: \n" << expression->dumpDAG() << std::endl; const auto & all_inputs = child->getInputStreams().front().header.getColumnsWithTypeAndName(); - auto split_filter = expression->splitActionsForFilter(filter_column_name, removes_filter, allowed_inputs, all_inputs); + auto split_filter = expression->cloneActionsForFilterPushDown(filter_column_name, removes_filter, allowed_inputs, all_inputs); if (!split_filter) return 0; From 2c1539f641aaac5267700a26e410c30fd54c2dd1 Mon Sep 17 00:00:00 2001 From: Nikolai Kochetov Date: Wed, 17 Mar 2021 20:08:30 +0300 Subject: [PATCH 536/544] Fix added input. --- src/Interpreters/ActionsDAG.cpp | 6 +++--- src/Interpreters/ActionsDAG.h | 4 ++-- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/src/Interpreters/ActionsDAG.cpp b/src/Interpreters/ActionsDAG.cpp index e67ab82a5ca..e5ae2dcfcf9 100644 --- a/src/Interpreters/ActionsDAG.cpp +++ b/src/Interpreters/ActionsDAG.cpp @@ -91,7 +91,7 @@ const ActionsDAG::Node & ActionsDAG::addInput(std::string name, DataTypePtr type return addNode(std::move(node), can_replace, add_to_index); } -const ActionsDAG::Node & ActionsDAG::addInput(ColumnWithTypeAndName column, bool can_replace) +const ActionsDAG::Node & ActionsDAG::addInput(ColumnWithTypeAndName column, bool can_replace, bool add_to_index) { Node node; node.type = ActionType::INPUT; @@ -99,7 +99,7 @@ const ActionsDAG::Node & ActionsDAG::addInput(ColumnWithTypeAndName column, bool node.result_name = std::move(column.name); node.column = std::move(column.column); - return addNode(std::move(node), can_replace); + return addNode(std::move(node), can_replace, add_to_index); } const ActionsDAG::Node & ActionsDAG::addColumn(ColumnWithTypeAndName column, bool can_replace, bool materialize) @@ -1430,7 +1430,7 @@ ActionsDAGPtr ActionsDAG::cloneActionsForConjunction(std::vector conjunc Node * input; auto & list = required_inputs[col.name]; if (list.empty()) - input = &const_cast(actions->addInput(col)); + input = &const_cast(actions->addInput(col, true, false)); else { input = list.front(); diff --git a/src/Interpreters/ActionsDAG.h b/src/Interpreters/ActionsDAG.h index e0e0e9c8957..fc6a0545ebd 100644 --- a/src/Interpreters/ActionsDAG.h +++ b/src/Interpreters/ActionsDAG.h @@ -200,7 +200,7 @@ public: std::string dumpDAG() const; const Node & addInput(std::string name, DataTypePtr type, bool can_replace = false, bool add_to_index = true); - const Node & addInput(ColumnWithTypeAndName column, bool can_replace = false); + const Node & addInput(ColumnWithTypeAndName column, bool can_replace = false, bool add_to_index = true); const Node & addColumn(ColumnWithTypeAndName column, bool can_replace = false, bool materialize = false); const Node & addAlias(const std::string & name, std::string alias, bool can_replace = false); const Node & addArrayJoin(const std::string & source_name, std::string result_name); @@ -301,7 +301,7 @@ public: /// Pushed condition: z > 0 /// GROUP BY step will transform columns `x, y, z` -> `sum(x), y, z` /// If we just add filter step with actions `z -> z > 0` before GROUP BY, - /// columns will be transformed like `x, y, z` -> `z, z > 0, x, y` -(remove filter)-> `z, z, y`. + /// columns will be transformed like `x, y, z` -> `z, z > 0, x, y` -(remove filter)-> `z, x, y`. /// To avoid it, add inputs from `all_inputs` list, /// so actions `x, y, z -> x, y, z, z > 0` -(remove filter)-> `x, y, z` will not change columns order. ActionsDAGPtr cloneActionsForFilterPushDown( From 6397099d3cf51b6721f1d3df372fe77e6401b329 Mon Sep 17 00:00:00 2001 From: Nikolai Kochetov Date: Wed, 17 Mar 2021 20:08:49 +0300 Subject: [PATCH 537/544] Added test. --- .../01763_filter_push_down_bugs.reference | 6 +++ .../01763_filter_push_down_bugs.sql | 37 +++++++++++++++++++ 2 files changed, 43 insertions(+) create mode 100644 tests/queries/0_stateless/01763_filter_push_down_bugs.reference create mode 100644 tests/queries/0_stateless/01763_filter_push_down_bugs.sql diff --git a/tests/queries/0_stateless/01763_filter_push_down_bugs.reference b/tests/queries/0_stateless/01763_filter_push_down_bugs.reference new file mode 100644 index 00000000000..66ea84a07c1 --- /dev/null +++ b/tests/queries/0_stateless/01763_filter_push_down_bugs.reference @@ -0,0 +1,6 @@ +1 2 +1 2 +[1] 2 +[[1]] 2 +String1_0 String2_0 String3_0 String4_0 1 +String1_0 String2_0 String3_0 String4_0 1 diff --git a/tests/queries/0_stateless/01763_filter_push_down_bugs.sql b/tests/queries/0_stateless/01763_filter_push_down_bugs.sql new file mode 100644 index 00000000000..5000eb38878 --- /dev/null +++ b/tests/queries/0_stateless/01763_filter_push_down_bugs.sql @@ -0,0 +1,37 @@ +SELECT * FROM (SELECT col1, col2 FROM (select '1' as col1, '2' as col2) GROUP by col1, col2) AS expr_qry WHERE col2 != ''; +SELECT * FROM (SELECT materialize('1') AS s1, materialize('2') AS s2 GROUP BY s1, s2) WHERE s2 = '2'; +SELECT * FROM (SELECT materialize([1]) AS s1, materialize('2') AS s2 GROUP BY s1, s2) WHERE s2 = '2'; +SELECT * FROM (SELECT materialize([[1]]) AS s1, materialize('2') AS s2 GROUP BY s1, s2) WHERE s2 = '2'; + +DROP TABLE IF EXISTS Test; + +CREATE TABLE Test +ENGINE = MergeTree() +PRIMARY KEY (String1,String2) +ORDER BY (String1,String2) +AS +SELECT + 'String1_' || toString(number) as String1, + 'String2_' || toString(number) as String2, + 'String3_' || toString(number) as String3, + 'String4_' || toString(number%4) as String4 +FROM numbers(1); + +SELECT * +FROM + ( + SELECT String1,String2,String3,String4,COUNT(*) + FROM Test + GROUP by String1,String2,String3,String4 + ) AS expr_qry; + +SELECT * +FROM + ( + SELECT String1,String2,String3,String4,COUNT(*) + FROM Test + GROUP by String1,String2,String3,String4 + ) AS expr_qry +WHERE String4 ='String4_0'; + +DROP TABLE IF EXISTS Test; From 89e79185a0f764e1009a061ba01ea4cb93704c55 Mon Sep 17 00:00:00 2001 From: Alexander Kuzmenkov <36882414+akuzm@users.noreply.github.com> Date: Wed, 17 Mar 2021 20:09:29 +0300 Subject: [PATCH 538/544] Update tests.md --- docs/en/development/tests.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/en/development/tests.md b/docs/en/development/tests.md index fb453e55417..7547497b9af 100644 --- a/docs/en/development/tests.md +++ b/docs/en/development/tests.md @@ -233,7 +233,7 @@ Google OSS-Fuzz can be found at `docker/fuzz`. We also use simple fuzz test to generate random SQL queries and to check that the server doesn’t die executing them. You can find it in `00746_sql_fuzzy.pl`. This test should be run continuously (overnight and longer). -We also use sophisticated AST-based query fuzzer that is able to find huge amount of corner cases. It does random permutations and substitutions in queries AST. It remembers AST nodes from previous tests to use them for fuzzing of subsequent tests while processing them in random order. +We also use sophisticated AST-based query fuzzer that is able to find huge amount of corner cases. It does random permutations and substitutions in queries AST. It remembers AST nodes from previous tests to use them for fuzzing of subsequent tests while processing them in random order. You can learn more about this fuzzer in [this blog article](https://clickhouse.tech/blog/en/2021/fuzzing-clickhouse/). ## Stress test From 8cb19d63aef4777ee8ebfa3fe94f229dbbd6dc4b Mon Sep 17 00:00:00 2001 From: robert Date: Wed, 17 Mar 2021 20:38:49 +0300 Subject: [PATCH 539/544] zookeeper-dump-tree: added ctime option to dump node ctime --- utils/zookeeper-dump-tree/main.cpp | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/utils/zookeeper-dump-tree/main.cpp b/utils/zookeeper-dump-tree/main.cpp index 5ab7bc2d536..5dcc260762f 100644 --- a/utils/zookeeper-dump-tree/main.cpp +++ b/utils/zookeeper-dump-tree/main.cpp @@ -17,6 +17,7 @@ int main(int argc, char ** argv) "addresses of ZooKeeper instances, comma separated. Example: example01e.yandex.ru:2181") ("path,p", boost::program_options::value()->default_value("/"), "where to start") + ("ctime,c", "print node ctime") ; boost::program_options::variables_map options; @@ -79,7 +80,11 @@ int main(int argc, char ** argv) throw; } - std::cout << it->first << '\t' << response.stat.numChildren << '\t' << response.stat.dataLength << '\n'; + std::cout << it->first << '\t' << response.stat.numChildren << '\t' << response.stat.dataLength; + if (options.count("ctime")) { + std::cout << '\t' << response.stat.ctime; + } + std::cout << '\n'; for (const auto & name : response.names) { From e1b11c786bf23844b4132d5560e4fac0248f741c Mon Sep 17 00:00:00 2001 From: robert Date: Wed, 17 Mar 2021 21:10:51 +0300 Subject: [PATCH 540/544] zookeeper-dump-tree: move ctime option to bool --- utils/zookeeper-dump-tree/main.cpp | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/utils/zookeeper-dump-tree/main.cpp b/utils/zookeeper-dump-tree/main.cpp index 5dcc260762f..47e23c5af71 100644 --- a/utils/zookeeper-dump-tree/main.cpp +++ b/utils/zookeeper-dump-tree/main.cpp @@ -31,6 +31,8 @@ int main(int argc, char ** argv) return 1; } + bool dump_ctime = (options.count("ctime")) ? true : false; + zkutil::ZooKeeperPtr zookeeper = std::make_shared(options.at("address").as()); std::string initial_path = options.at("path").as(); @@ -81,7 +83,7 @@ int main(int argc, char ** argv) } std::cout << it->first << '\t' << response.stat.numChildren << '\t' << response.stat.dataLength; - if (options.count("ctime")) { + if (dump_ctime) { std::cout << '\t' << response.stat.ctime; } std::cout << '\n'; From 9fdb0e667fbeef9e13750afc4be90a29618c292e Mon Sep 17 00:00:00 2001 From: Nikolai Kochetov Date: Wed, 17 Mar 2021 21:40:26 +0300 Subject: [PATCH 541/544] Add comment. --- src/Interpreters/ActionsDAG.cpp | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/Interpreters/ActionsDAG.cpp b/src/Interpreters/ActionsDAG.cpp index e5ae2dcfcf9..94d9b72b8e9 100644 --- a/src/Interpreters/ActionsDAG.cpp +++ b/src/Interpreters/ActionsDAG.cpp @@ -1424,7 +1424,8 @@ ActionsDAGPtr ActionsDAG::cloneActionsForConjunction(std::vector conjunc } } - + /// Actions must have the same inputs as in all_inputs list. + /// See comment to cloneActionsForFilterPushDown. for (const auto & col : all_inputs) { Node * input; From 45fead90621dfd2cf19bbd350aca703f4604b77f Mon Sep 17 00:00:00 2001 From: Nikolai Kochetov Date: Wed, 17 Mar 2021 21:42:42 +0300 Subject: [PATCH 542/544] Fix typos --- src/Interpreters/ActionsDAG.h | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/Interpreters/ActionsDAG.h b/src/Interpreters/ActionsDAG.h index fc6a0545ebd..d5a15324280 100644 --- a/src/Interpreters/ActionsDAG.h +++ b/src/Interpreters/ActionsDAG.h @@ -293,10 +293,10 @@ public: /// Returned actions add single column which may be used for filter. /// Also, replace some nodes of current inputs to constant 1 in case they are filtered. /// - /// @param all_inputs should containt inputs from previous step, which will be used for result actions. + /// @param all_inputs should contain inputs from previous step, which will be used for result actions. /// It is expected that all_inputs contain columns from available_inputs. /// This parameter is needed to enforce result actions save columns order in block. - /// Otherwise for some queries, e.g. with GROUP BY, columns colum be swapped. + /// Otherwise for some queries, e.g. with GROUP BY, columns will be mixed. /// Example: SELECT sum(x), y, z FROM tab WHERE z > 0 and sum(x) > 0 /// Pushed condition: z > 0 /// GROUP BY step will transform columns `x, y, z` -> `sum(x), y, z` From 3a8d7e9a48d254eb9adb6f5c9623834b65658df6 Mon Sep 17 00:00:00 2001 From: alexey-milovidov Date: Wed, 17 Mar 2021 22:12:10 +0300 Subject: [PATCH 543/544] Update main.cpp --- utils/zookeeper-dump-tree/main.cpp | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/utils/zookeeper-dump-tree/main.cpp b/utils/zookeeper-dump-tree/main.cpp index 47e23c5af71..893056564bb 100644 --- a/utils/zookeeper-dump-tree/main.cpp +++ b/utils/zookeeper-dump-tree/main.cpp @@ -31,7 +31,7 @@ int main(int argc, char ** argv) return 1; } - bool dump_ctime = (options.count("ctime")) ? true : false; + bool dump_ctime = options.count("ctime"); zkutil::ZooKeeperPtr zookeeper = std::make_shared(options.at("address").as()); @@ -83,9 +83,8 @@ int main(int argc, char ** argv) } std::cout << it->first << '\t' << response.stat.numChildren << '\t' << response.stat.dataLength; - if (dump_ctime) { + if (dump_ctime) std::cout << '\t' << response.stat.ctime; - } std::cout << '\n'; for (const auto & name : response.names) From c4dfd97709972c271b3cc9e306f7be80cbd3af9f Mon Sep 17 00:00:00 2001 From: Nikolai Kochetov Date: Thu, 18 Mar 2021 10:59:48 +0300 Subject: [PATCH 544/544] Fix ActionsDAG::Index::insert --- src/Interpreters/ActionsDAG.h | 27 +++++++++++++++++++++++++-- 1 file changed, 25 insertions(+), 2 deletions(-) diff --git a/src/Interpreters/ActionsDAG.h b/src/Interpreters/ActionsDAG.h index d5a15324280..4e334bd1be8 100644 --- a/src/Interpreters/ActionsDAG.h +++ b/src/Interpreters/ActionsDAG.h @@ -120,8 +120,31 @@ public: /// Insert method doesn't check if map already have node with the same name. /// If node with the same name exists, it is removed from map, but not list. /// It is expected and used for project(), when result may have several columns with the same name. - void insert(Node * node) { map[node->result_name] = list.emplace(list.end(), node); } - void prepend(Node * node) { map[node->result_name] = list.emplace(list.begin(), node); } + void insert(Node * node) + { + auto it = list.emplace(list.end(), node); + if (auto handle = map.extract(node->result_name)) + { + handle.key() = node->result_name; /// Change string_view + handle.mapped() = it; + map.insert(std::move(handle)); + } + else + map[node->result_name] = it; + } + + void prepend(Node * node) + { + auto it = list.emplace(list.begin(), node); + if (auto handle = map.extract(node->result_name)) + { + handle.key() = node->result_name; /// Change string_view + handle.mapped() = it; + map.insert(std::move(handle)); + } + else + map[node->result_name] = it; + } /// If node with same name exists in index, replace it. Otherwise insert new node to index. void replace(Node * node)