From 7f34af467e955a363debb3575b45f30cd60b64f3 Mon Sep 17 00:00:00 2001 From: Smita Kulkarni Date: Tue, 13 Feb 2024 16:00:26 +0100 Subject: [PATCH 001/115] Add azure_cache as storage policy to tests --- .../queries/0_stateless/02240_system_filesystem_cache_table.sh | 2 +- .../0_stateless/02241_filesystem_cache_on_write_operations.sh | 2 +- .../0_stateless/02242_system_filesystem_cache_log_table.sh | 2 +- tests/queries/0_stateless/02286_drop_filesystem_cache.sh | 2 +- tests/queries/0_stateless/02313_filesystem_cache_seeks.sh | 2 +- 5 files changed, 5 insertions(+), 5 deletions(-) diff --git a/tests/queries/0_stateless/02240_system_filesystem_cache_table.sh b/tests/queries/0_stateless/02240_system_filesystem_cache_table.sh index 6a94cffea5a..2102b7efdb2 100755 --- a/tests/queries/0_stateless/02240_system_filesystem_cache_table.sh +++ b/tests/queries/0_stateless/02240_system_filesystem_cache_table.sh @@ -7,7 +7,7 @@ CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) # shellcheck source=../shell_config.sh . "$CUR_DIR"/../shell_config.sh -for STORAGE_POLICY in 's3_cache' 'local_cache'; do +for STORAGE_POLICY in 's3_cache' 'local_cache' 'azure_cache'; do echo "Using storage policy: $STORAGE_POLICY" ${CLICKHOUSE_CLIENT} --query "SYSTEM STOP MERGES" ${CLICKHOUSE_CLIENT} --query "SYSTEM DROP FILESYSTEM CACHE" diff --git a/tests/queries/0_stateless/02241_filesystem_cache_on_write_operations.sh b/tests/queries/0_stateless/02241_filesystem_cache_on_write_operations.sh index 96f61cf61e8..dafd9f7a640 100755 --- a/tests/queries/0_stateless/02241_filesystem_cache_on_write_operations.sh +++ b/tests/queries/0_stateless/02241_filesystem_cache_on_write_operations.sh @@ -7,7 +7,7 @@ CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) # shellcheck source=../shell_config.sh . "$CUR_DIR"/../shell_config.sh -for STORAGE_POLICY in 's3_cache' 'local_cache'; do +for STORAGE_POLICY in 's3_cache' 'local_cache' 'azure_cache'; do echo "Using storage policy: $STORAGE_POLICY" $CLICKHOUSE_CLIENT --query "DROP TABLE IF EXISTS test_02241" diff --git a/tests/queries/0_stateless/02242_system_filesystem_cache_log_table.sh b/tests/queries/0_stateless/02242_system_filesystem_cache_log_table.sh index 4c92d1d2954..7a665d81eab 100755 --- a/tests/queries/0_stateless/02242_system_filesystem_cache_log_table.sh +++ b/tests/queries/0_stateless/02242_system_filesystem_cache_log_table.sh @@ -7,7 +7,7 @@ CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) # shellcheck source=../shell_config.sh . "$CUR_DIR"/../shell_config.sh -for STORAGE_POLICY in 's3_cache' 'local_cache'; do +for STORAGE_POLICY in 's3_cache' 'local_cache' 'azure_cache'; do echo "Using storage policy: $STORAGE_POLICY" $CLICKHOUSE_CLIENT --query "SYSTEM DROP FILESYSTEM CACHE" diff --git a/tests/queries/0_stateless/02286_drop_filesystem_cache.sh b/tests/queries/0_stateless/02286_drop_filesystem_cache.sh index 1e1841862e9..a2c9352b7aa 100755 --- a/tests/queries/0_stateless/02286_drop_filesystem_cache.sh +++ b/tests/queries/0_stateless/02286_drop_filesystem_cache.sh @@ -7,7 +7,7 @@ CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) # shellcheck source=../shell_config.sh . "$CUR_DIR"/../shell_config.sh -for STORAGE_POLICY in 's3_cache' 'local_cache'; do +for STORAGE_POLICY in 's3_cache' 'local_cache' 'azure_cache'; do echo "Using storage policy: $STORAGE_POLICY" $CLICKHOUSE_CLIENT --query "DROP TABLE IF EXISTS test_02286" diff --git a/tests/queries/0_stateless/02313_filesystem_cache_seeks.sh b/tests/queries/0_stateless/02313_filesystem_cache_seeks.sh index f5de4346fd6..fbaec1ffaa7 100755 --- a/tests/queries/0_stateless/02313_filesystem_cache_seeks.sh +++ b/tests/queries/0_stateless/02313_filesystem_cache_seeks.sh @@ -8,7 +8,7 @@ CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) . "$CUR_DIR"/../shell_config.sh -for STORAGE_POLICY in 's3_cache' 'local_cache' 's3_cache_multi'; do +for STORAGE_POLICY in 's3_cache' 'local_cache' 's3_cache_multi' 'azure_cache'; do echo "Using storage policy: $STORAGE_POLICY" $CLICKHOUSE_CLIENT --query "SYSTEM DROP FILESYSTEM CACHE" From 0ca07617393636bff00fe107287bb841138d050e Mon Sep 17 00:00:00 2001 From: Smita Kulkarni Date: Wed, 14 Feb 2024 11:57:49 +0100 Subject: [PATCH 002/115] Updated output of tests --- ...40_system_filesystem_cache_table.reference | 18 +++++++++++ ...system_cache_on_write_operations.reference | 31 +++++++++++++++++++ ...ystem_filesystem_cache_log_table.reference | 4 +++ .../02286_drop_filesystem_cache.reference | 9 ++++++ .../02313_filesystem_cache_seeks.reference | 1 + 5 files changed, 63 insertions(+) diff --git a/tests/queries/0_stateless/02240_system_filesystem_cache_table.reference b/tests/queries/0_stateless/02240_system_filesystem_cache_table.reference index 93b6d4de94f..6b5dd182112 100644 --- a/tests/queries/0_stateless/02240_system_filesystem_cache_table.reference +++ b/tests/queries/0_stateless/02240_system_filesystem_cache_table.reference @@ -34,3 +34,21 @@ DOWNLOADED 0 79 80 DOWNLOADED 0 745 746 2 Expect no cache +Using storage policy: azure_cache +0 +Expect cache +DOWNLOADED 0 0 1 +DOWNLOADED 0 79 80 +DOWNLOADED 0 745 746 +3 +Expect cache +DOWNLOADED 0 0 1 +DOWNLOADED 0 79 80 +DOWNLOADED 0 745 746 +3 +Expect no cache +Expect cache +DOWNLOADED 0 79 80 +DOWNLOADED 0 745 746 +2 +Expect no cache diff --git a/tests/queries/0_stateless/02241_filesystem_cache_on_write_operations.reference b/tests/queries/0_stateless/02241_filesystem_cache_on_write_operations.reference index 157837983f7..d23a4435bdf 100644 --- a/tests/queries/0_stateless/02241_filesystem_cache_on_write_operations.reference +++ b/tests/queries/0_stateless/02241_filesystem_cache_on_write_operations.reference @@ -60,3 +60,34 @@ state: DOWNLOADED INSERT INTO test_02241 SELECT number, toString(number) FROM numbers(5000000) 0 5010500 18816 +Using storage policy: azure_cache +0 +0 0 +Row 1: +────── +file_segment_range_begin: 0 +file_segment_range_end: 745 +size: 746 +state: DOWNLOADED +8 +8 1100 +0 +2 +2 +8 1100 +Row 1: +────── +file_segment_range_begin: 0 +file_segment_range_end: 1659 +size: 1660 +state: DOWNLOADED +8 +8 2014 +8 2014 +8 2014 +24 84045 +32 167243 +41 250541 +INSERT INTO test_02241 SELECT number, toString(number) FROM numbers(5000000) 0 +5010500 +18816 diff --git a/tests/queries/0_stateless/02242_system_filesystem_cache_log_table.reference b/tests/queries/0_stateless/02242_system_filesystem_cache_log_table.reference index 99f31df7def..447e1a275fc 100644 --- a/tests/queries/0_stateless/02242_system_filesystem_cache_log_table.reference +++ b/tests/queries/0_stateless/02242_system_filesystem_cache_log_table.reference @@ -6,3 +6,7 @@ Using storage policy: local_cache (0,519) READ_FROM_FS_AND_DOWNLOADED_TO_CACHE (0,808110) READ_FROM_FS_AND_DOWNLOADED_TO_CACHE (0,808110) READ_FROM_CACHE +Using storage policy: azure_cache +(0,519) READ_FROM_FS_AND_DOWNLOADED_TO_CACHE +(0,808110) READ_FROM_FS_AND_DOWNLOADED_TO_CACHE +(0,808110) READ_FROM_CACHE diff --git a/tests/queries/0_stateless/02286_drop_filesystem_cache.reference b/tests/queries/0_stateless/02286_drop_filesystem_cache.reference index b4e5b6715de..e3875dbabe1 100644 --- a/tests/queries/0_stateless/02286_drop_filesystem_cache.reference +++ b/tests/queries/0_stateless/02286_drop_filesystem_cache.reference @@ -16,3 +16,12 @@ Using storage policy: local_cache 1 1 0 +Using storage policy: azure_cache +0 +2 +0 +1 +1 +1 +1 +0 diff --git a/tests/queries/0_stateless/02313_filesystem_cache_seeks.reference b/tests/queries/0_stateless/02313_filesystem_cache_seeks.reference index 062aac259a4..0a9e1c20b59 100644 --- a/tests/queries/0_stateless/02313_filesystem_cache_seeks.reference +++ b/tests/queries/0_stateless/02313_filesystem_cache_seeks.reference @@ -1,3 +1,4 @@ Using storage policy: s3_cache Using storage policy: local_cache Using storage policy: s3_cache_multi +Using storage policy: azure_cache From 7c1ac996e1900313b2eb56b7b61bf1dfd7e3cfb5 Mon Sep 17 00:00:00 2001 From: Smita Kulkarni Date: Fri, 16 Feb 2024 11:30:39 +0100 Subject: [PATCH 003/115] Updated to add azure config to ARM --- tests/config/install.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/config/install.sh b/tests/config/install.sh index cfe810cda84..8b40d353a92 100755 --- a/tests/config/install.sh +++ b/tests/config/install.sh @@ -168,7 +168,7 @@ ARM="aarch64" OS="$(uname -m)" if [[ -n "$EXPORT_S3_STORAGE_POLICIES" ]]; then echo "$OS" - if [[ "$USE_DATABASE_REPLICATED" -eq 1 ]] || [[ "$OS" == "$ARM" ]]; then + if [[ "$USE_DATABASE_REPLICATED" -eq 1 ]]; then echo "Azure configuration will not be added" else echo "Adding azure configuration" From cfc5bf272b7762299fba2bedd115a1228ed45219 Mon Sep 17 00:00:00 2001 From: kssenii Date: Fri, 3 May 2024 16:16:21 +0200 Subject: [PATCH 004/115] Small refactoring for temporary data in cache --- src/Interpreters/Cache/FileSegment.cpp | 124 ++++++++++++------ src/Interpreters/Cache/FileSegment.h | 6 +- src/Interpreters/Cache/Metadata.cpp | 9 +- .../Cache/WriteBufferToFileSegment.cpp | 5 +- .../Cache/WriteBufferToFileSegment.h | 1 + 5 files changed, 89 insertions(+), 56 deletions(-) diff --git a/src/Interpreters/Cache/FileSegment.cpp b/src/Interpreters/Cache/FileSegment.cpp index 20a3af60c8c..6c7337e48ef 100644 --- a/src/Interpreters/Cache/FileSegment.cpp +++ b/src/Interpreters/Cache/FileSegment.cpp @@ -187,13 +187,6 @@ size_t FileSegment::getDownloadedSize() const return downloaded_size; } -void FileSegment::setDownloadedSize(size_t delta) -{ - auto lk = lock(); - downloaded_size += delta; - assert(downloaded_size == std::filesystem::file_size(getPath())); -} - bool FileSegment::isDownloaded() const { auto lk = lock(); @@ -343,7 +336,38 @@ void FileSegment::setRemoteFileReader(RemoteFileReaderPtr remote_file_reader_) void FileSegment::write(const char * from, size_t size, size_t offset) { ProfileEventTimeIncrement watch(ProfileEvents::FileSegmentWriteMicroseconds); + assertWriteAllowed(size, offset); + writeImpl(size, offset, [&]() + { + if (!cache_writer) + cache_writer = std::make_unique(getPath()); + cache_writer->write(from, size); + cache_writer->next(); + }); +} + +void FileSegment::write(WriteBufferFromFile & wb, size_t offset) +{ + ProfileEventTimeIncrement watch(ProfileEvents::FileSegmentWriteMicroseconds); + + const size_t size = wb.offset(); + assertWriteAllowed(size, offset); + + const auto & current_filename = wb.getFileName(); + const auto & expected_filename = getPath(); + if (current_filename != expected_filename) + { + throw Exception(ErrorCodes::LOGICAL_ERROR, + "Write buffer has unexpected file path: {}, expected: {}", + current_filename, expected_filename); + } + + writeImpl(size, offset, [&]() { wb.next(); }, &wb); +} + +void FileSegment::assertWriteAllowed(size_t size, size_t offset) +{ if (!size) throw Exception(ErrorCodes::LOGICAL_ERROR, "Writing zero size is not allowed"); @@ -353,53 +377,49 @@ void FileSegment::write(const char * from, size_t size, size_t offset) assertNotDetachedUnlocked(lk); } - const auto file_segment_path = getPath(); + if (download_state != State::DOWNLOADING) + throw Exception( + ErrorCodes::LOGICAL_ERROR, + "Expected DOWNLOADING state, got {}", stateToString(download_state)); - { - if (download_state != State::DOWNLOADING) - throw Exception( - ErrorCodes::LOGICAL_ERROR, - "Expected DOWNLOADING state, got {}", stateToString(download_state)); + const size_t first_non_downloaded_offset = getCurrentWriteOffset(); + if (offset != first_non_downloaded_offset) + throw Exception( + ErrorCodes::LOGICAL_ERROR, + "Attempt to write {} bytes to offset: {}, but current write offset is {}", + size, offset, first_non_downloaded_offset); - const size_t first_non_downloaded_offset = getCurrentWriteOffset(); - if (offset != first_non_downloaded_offset) - throw Exception( - ErrorCodes::LOGICAL_ERROR, - "Attempt to write {} bytes to offset: {}, but current write offset is {}", - size, offset, first_non_downloaded_offset); + const size_t current_downloaded_size = getDownloadedSize(); + chassert(reserved_size >= current_downloaded_size); - const size_t current_downloaded_size = getDownloadedSize(); - chassert(reserved_size >= current_downloaded_size); + const size_t free_reserved_size = reserved_size - current_downloaded_size; + if (free_reserved_size < size) + throw Exception( + ErrorCodes::LOGICAL_ERROR, + "Not enough space is reserved. Available: {}, expected: {}", free_reserved_size, size); - const size_t free_reserved_size = reserved_size - current_downloaded_size; - if (free_reserved_size < size) - throw Exception( - ErrorCodes::LOGICAL_ERROR, - "Not enough space is reserved. Available: {}, expected: {}", free_reserved_size, size); + if (!is_unbound && current_downloaded_size == range().size()) + throw Exception(ErrorCodes::LOGICAL_ERROR, "File segment is already fully downloaded"); - if (!is_unbound && current_downloaded_size == range().size()) - throw Exception(ErrorCodes::LOGICAL_ERROR, "File segment is already fully downloaded"); - - if (!cache_writer && current_downloaded_size > 0) - throw Exception( - ErrorCodes::LOGICAL_ERROR, - "Cache writer was finalized (downloaded size: {}, state: {})", - current_downloaded_size, stateToString(download_state)); - } + if (!cache_writer && current_downloaded_size > 0) + throw Exception( + ErrorCodes::LOGICAL_ERROR, + "Cache writer was finalized (downloaded size: {}, state: {})", + current_downloaded_size, stateToString(download_state)); +} +void FileSegment::writeImpl(size_t size, [[maybe_unused]] size_t offset, std::function do_write, WriteBuffer * external_wb) +{ + auto file_segment_path = getPath(); try { - if (!cache_writer) - cache_writer = std::make_unique(file_segment_path); - #ifdef ABORT_ON_LOGICAL_ERROR /// This mutex is only needed to have a valid assertion in assertCacheCorrectness(), /// which is only executed in debug/sanitizer builds (under ABORT_ON_LOGICAL_ERROR). std::lock_guard lock(write_mutex); #endif - cache_writer->write(from, size); - cache_writer->next(); + do_write(); downloaded_size += size; chassert(std::filesystem::file_size(file_segment_path) == downloaded_size); @@ -414,6 +434,18 @@ void FileSegment::write(const char * from, size_t size, size_t offset) e.addMessage(fmt::format("{}, current cache state: {}", e.what(), getInfoForLogUnlocked(lk))); setDownloadFailedUnlocked(lk); + if (external_wb) + { + try + { + external_wb->finalize(); + } + catch (...) + { + tryLogCurrentException(__PRETTY_FUNCTION__); + } + } + if (downloaded_size == 0 && fs::exists(file_segment_path)) { fs::remove(file_segment_path); @@ -431,7 +463,6 @@ void FileSegment::write(const char * from, size_t size, size_t offset) } throw; - } catch (Exception & e) { @@ -597,7 +628,14 @@ void FileSegment::setDownloadFailedUnlocked(const FileSegmentGuard::Lock & lock) if (cache_writer) { - cache_writer->finalize(); + try + { + cache_writer->finalize(); + } + catch (...) + { + tryLogCurrentException(__PRETTY_FUNCTION__); + } cache_writer.reset(); } @@ -825,7 +863,7 @@ bool FileSegment::assertCorrectnessUnlocked(const FileSegmentGuard::Lock & lock) }; const auto file_path = getPath(); - if (segment_kind != FileSegmentKind::Temporary) + { std::lock_guard lk(write_mutex); if (downloaded_size == 0) diff --git a/src/Interpreters/Cache/FileSegment.h b/src/Interpreters/Cache/FileSegment.h index 7793c50d2d5..818dd6e31d6 100644 --- a/src/Interpreters/Cache/FileSegment.h +++ b/src/Interpreters/Cache/FileSegment.h @@ -205,6 +205,7 @@ public: /// Write data into reserved space. void write(const char * from, size_t size, size_t offset); + void write(WriteBufferFromFile & wb, size_t offset); // Invariant: if state() != DOWNLOADING and remote file reader is present, the reader's // available() == 0, and getFileOffsetOfBufferEnd() == our getCurrentWriteOffset(). @@ -219,8 +220,6 @@ public: void setRemoteFileReader(RemoteFileReaderPtr remote_file_reader_); - void setDownloadedSize(size_t delta); - void setDownloadFailed(); private: @@ -244,6 +243,9 @@ private: LockedKeyPtr lockKeyMetadata(bool assert_exists = true) const; + void assertWriteAllowed(size_t size, size_t offset); + void writeImpl(size_t size, size_t offset, std::function do_write, WriteBuffer * external_wb = nullptr); + String tryGetPath() const; Key file_key; diff --git a/src/Interpreters/Cache/Metadata.cpp b/src/Interpreters/Cache/Metadata.cpp index c832473c4cd..451c6c74518 100644 --- a/src/Interpreters/Cache/Metadata.cpp +++ b/src/Interpreters/Cache/Metadata.cpp @@ -944,14 +944,7 @@ KeyMetadata::iterator LockedKey::removeFileSegmentImpl( try { const auto path = key_metadata->getFileSegmentPath(*file_segment); - if (file_segment->segment_kind == FileSegmentKind::Temporary) - { - /// FIXME: For temporary file segment the requirement is not as strong because - /// the implementation of "temporary data in cache" creates files in advance. - if (fs::exists(path)) - fs::remove(path); - } - else if (file_segment->downloaded_size == 0) + if (file_segment->downloaded_size == 0) { chassert(!fs::exists(path)); } diff --git a/src/Interpreters/Cache/WriteBufferToFileSegment.cpp b/src/Interpreters/Cache/WriteBufferToFileSegment.cpp index 2ac38aeeca7..d4135c775ca 100644 --- a/src/Interpreters/Cache/WriteBufferToFileSegment.cpp +++ b/src/Interpreters/Cache/WriteBufferToFileSegment.cpp @@ -97,15 +97,14 @@ void WriteBufferToFileSegment::nextImpl() { SwapHelper swap(*this, *impl); /// Write data to the underlying buffer. - impl->next(); + file_segment->write(*dynamic_cast(impl.get()), written_bytes); + written_bytes += bytes_to_write; } catch (...) { LOG_WARNING(getLogger("WriteBufferToFileSegment"), "Failed to write to the underlying buffer ({})", file_segment->getInfoForLog()); throw; } - - file_segment->setDownloadedSize(bytes_to_write); } std::unique_ptr WriteBufferToFileSegment::getReadBufferImpl() diff --git a/src/Interpreters/Cache/WriteBufferToFileSegment.h b/src/Interpreters/Cache/WriteBufferToFileSegment.h index 822488ceb48..62692ed9071 100644 --- a/src/Interpreters/Cache/WriteBufferToFileSegment.h +++ b/src/Interpreters/Cache/WriteBufferToFileSegment.h @@ -30,6 +30,7 @@ private: FileSegmentsHolderPtr segment_holder; const size_t reserve_space_lock_wait_timeout_milliseconds; + size_t written_bytes = 0; }; From 4d7902120ca2621e6dccbf5afe173d39d2c6683f Mon Sep 17 00:00:00 2001 From: Kseniia Sumarokova <54203879+kssenii@users.noreply.github.com> Date: Fri, 3 May 2024 17:00:09 +0200 Subject: [PATCH 005/115] Update src/Interpreters/Cache/FileSegment.cpp Co-authored-by: vdimir --- src/Interpreters/Cache/FileSegment.cpp | 1 + 1 file changed, 1 insertion(+) diff --git a/src/Interpreters/Cache/FileSegment.cpp b/src/Interpreters/Cache/FileSegment.cpp index 6c7337e48ef..b40dc2248d3 100644 --- a/src/Interpreters/Cache/FileSegment.cpp +++ b/src/Interpreters/Cache/FileSegment.cpp @@ -363,6 +363,7 @@ void FileSegment::write(WriteBufferFromFile & wb, size_t offset) current_filename, expected_filename); } + /// Data should already be in buffer, so we only call `next` writeImpl(size, offset, [&]() { wb.next(); }, &wb); } From 57c44c2d99a6a03a1f366f3123c61c8ebdeb321e Mon Sep 17 00:00:00 2001 From: Kseniia Sumarokova <54203879+kssenii@users.noreply.github.com> Date: Fri, 3 May 2024 17:01:20 +0200 Subject: [PATCH 006/115] Update src/Interpreters/Cache/FileSegment.h Co-authored-by: vdimir --- src/Interpreters/Cache/FileSegment.h | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/src/Interpreters/Cache/FileSegment.h b/src/Interpreters/Cache/FileSegment.h index 818dd6e31d6..30d696d40f9 100644 --- a/src/Interpreters/Cache/FileSegment.h +++ b/src/Interpreters/Cache/FileSegment.h @@ -205,6 +205,12 @@ public: /// Write data into reserved space. void write(const char * from, size_t size, size_t offset); + + /** Use this overload if you need to handle file writing externally. + * It writes data already stored in the `wb` buffer, ensuring the file segment's invariants. + * The write buffer should refer to the same path that `FileSegment::getPath` returns. + * For each file segment instance, only one overload of `write` can be used. + */ void write(WriteBufferFromFile & wb, size_t offset); // Invariant: if state() != DOWNLOADING and remote file reader is present, the reader's From dcfa78583e321e7169e35db234e55480422f2c2c Mon Sep 17 00:00:00 2001 From: kssenii Date: Mon, 6 May 2024 12:46:55 +0200 Subject: [PATCH 007/115] More explicit writing, fix assertion "expected file not to exist" --- .../IO/CachedOnDiskWriteBufferFromFile.cpp | 2 +- .../IO/CachedOnDiskWriteBufferFromFile.h | 2 +- src/Interpreters/Cache/FileSegment.cpp | 53 ++++++------------- src/Interpreters/Cache/FileSegment.h | 12 ++--- .../Cache/WriteBufferToFileSegment.cpp | 43 +++++++++------ .../Cache/WriteBufferToFileSegment.h | 10 +++- 6 files changed, 55 insertions(+), 67 deletions(-) diff --git a/src/Disks/IO/CachedOnDiskWriteBufferFromFile.cpp b/src/Disks/IO/CachedOnDiskWriteBufferFromFile.cpp index f4e309f461e..382c4a80cc4 100644 --- a/src/Disks/IO/CachedOnDiskWriteBufferFromFile.cpp +++ b/src/Disks/IO/CachedOnDiskWriteBufferFromFile.cpp @@ -41,7 +41,7 @@ FileSegmentRangeWriter::FileSegmentRangeWriter( { } -bool FileSegmentRangeWriter::write(const char * data, size_t size, size_t offset, FileSegmentKind segment_kind) +bool FileSegmentRangeWriter::write(char * data, size_t size, size_t offset, FileSegmentKind segment_kind) { if (finalized) return false; diff --git a/src/Disks/IO/CachedOnDiskWriteBufferFromFile.h b/src/Disks/IO/CachedOnDiskWriteBufferFromFile.h index ad4f6b5916d..ba2952d9c56 100644 --- a/src/Disks/IO/CachedOnDiskWriteBufferFromFile.h +++ b/src/Disks/IO/CachedOnDiskWriteBufferFromFile.h @@ -39,7 +39,7 @@ public: * Write a range of file segments. Allocate file segment of `max_file_segment_size` and write to * it until it is full and then allocate next file segment. */ - bool write(const char * data, size_t size, size_t offset, FileSegmentKind segment_kind); + bool write(char * data, size_t size, size_t offset, FileSegmentKind segment_kind); void finalize(); diff --git a/src/Interpreters/Cache/FileSegment.cpp b/src/Interpreters/Cache/FileSegment.cpp index b40dc2248d3..4548f1d9d20 100644 --- a/src/Interpreters/Cache/FileSegment.cpp +++ b/src/Interpreters/Cache/FileSegment.cpp @@ -304,6 +304,11 @@ FileSegment::RemoteFileReaderPtr FileSegment::getRemoteFileReader() return remote_file_reader; } +FileSegment::LocalCacheWriterPtr FileSegment::getLocalCacheWriter() +{ + return cache_writer; +} + void FileSegment::resetRemoteFileReader() { auto lk = lock(); @@ -333,40 +338,24 @@ void FileSegment::setRemoteFileReader(RemoteFileReaderPtr remote_file_reader_) remote_file_reader = remote_file_reader_; } -void FileSegment::write(const char * from, size_t size, size_t offset) +void FileSegment::write(char * from, size_t size, size_t offset_in_file) { ProfileEventTimeIncrement watch(ProfileEvents::FileSegmentWriteMicroseconds); - assertWriteAllowed(size, offset); - writeImpl(size, offset, [&]() + assertWriteAllowed(size, offset_in_file); + writeImpl(size, offset_in_file, [&]() { if (!cache_writer) - cache_writer = std::make_unique(getPath()); + cache_writer = std::make_unique(getPath(), /* buf_size */0); - cache_writer->write(from, size); + /// Size is equal to offset as offset for write buffer points to data end. + cache_writer->set(from, /* size */size, /* offset */size); + /// Reset the buffer when finished. + SCOPE_EXIT({ cache_writer->set(nullptr, 0); }); + /// Flush the buffer. cache_writer->next(); }); } -void FileSegment::write(WriteBufferFromFile & wb, size_t offset) -{ - ProfileEventTimeIncrement watch(ProfileEvents::FileSegmentWriteMicroseconds); - - const size_t size = wb.offset(); - assertWriteAllowed(size, offset); - - const auto & current_filename = wb.getFileName(); - const auto & expected_filename = getPath(); - if (current_filename != expected_filename) - { - throw Exception(ErrorCodes::LOGICAL_ERROR, - "Write buffer has unexpected file path: {}, expected: {}", - current_filename, expected_filename); - } - - /// Data should already be in buffer, so we only call `next` - writeImpl(size, offset, [&]() { wb.next(); }, &wb); -} - void FileSegment::assertWriteAllowed(size_t size, size_t offset) { if (!size) @@ -409,7 +398,7 @@ void FileSegment::assertWriteAllowed(size_t size, size_t offset) current_downloaded_size, stateToString(download_state)); } -void FileSegment::writeImpl(size_t size, [[maybe_unused]] size_t offset, std::function do_write, WriteBuffer * external_wb) +void FileSegment::writeImpl(size_t size, [[maybe_unused]] size_t offset, std::function do_write) { auto file_segment_path = getPath(); try @@ -435,18 +424,6 @@ void FileSegment::writeImpl(size_t size, [[maybe_unused]] size_t offset, std::fu e.addMessage(fmt::format("{}, current cache state: {}", e.what(), getInfoForLogUnlocked(lk))); setDownloadFailedUnlocked(lk); - if (external_wb) - { - try - { - external_wb->finalize(); - } - catch (...) - { - tryLogCurrentException(__PRETTY_FUNCTION__); - } - } - if (downloaded_size == 0 && fs::exists(file_segment_path)) { fs::remove(file_segment_path); diff --git a/src/Interpreters/Cache/FileSegment.h b/src/Interpreters/Cache/FileSegment.h index 30d696d40f9..f98fb832553 100644 --- a/src/Interpreters/Cache/FileSegment.h +++ b/src/Interpreters/Cache/FileSegment.h @@ -48,7 +48,7 @@ friend class FileCache; /// Because of reserved_size in tryReserve(). public: using Key = FileCacheKey; using RemoteFileReaderPtr = std::shared_ptr; - using LocalCacheWriterPtr = std::unique_ptr; + using LocalCacheWriterPtr = std::shared_ptr; using Downloader = std::string; using DownloaderId = std::string; using Priority = IFileCachePriority; @@ -204,14 +204,7 @@ public: bool reserve(size_t size_to_reserve, size_t lock_wait_timeout_milliseconds, FileCacheReserveStat * reserve_stat = nullptr); /// Write data into reserved space. - void write(const char * from, size_t size, size_t offset); - - /** Use this overload if you need to handle file writing externally. - * It writes data already stored in the `wb` buffer, ensuring the file segment's invariants. - * The write buffer should refer to the same path that `FileSegment::getPath` returns. - * For each file segment instance, only one overload of `write` can be used. - */ - void write(WriteBufferFromFile & wb, size_t offset); + void write(char * from, size_t size, size_t offset_in_file); // Invariant: if state() != DOWNLOADING and remote file reader is present, the reader's // available() == 0, and getFileOffsetOfBufferEnd() == our getCurrentWriteOffset(). @@ -219,6 +212,7 @@ public: // The reader typically requires its internal_buffer to be assigned from the outside before // calling next(). RemoteFileReaderPtr getRemoteFileReader(); + LocalCacheWriterPtr getLocalCacheWriter(); RemoteFileReaderPtr extractRemoteFileReader(); diff --git a/src/Interpreters/Cache/WriteBufferToFileSegment.cpp b/src/Interpreters/Cache/WriteBufferToFileSegment.cpp index d4135c775ca..4a03d008ed7 100644 --- a/src/Interpreters/Cache/WriteBufferToFileSegment.cpp +++ b/src/Interpreters/Cache/WriteBufferToFileSegment.cpp @@ -33,21 +33,20 @@ namespace } WriteBufferToFileSegment::WriteBufferToFileSegment(FileSegment * file_segment_) - : WriteBufferFromFileDecorator(std::make_unique(file_segment_->getPath())) + : WriteBufferFromFileBase(DBMS_DEFAULT_BUFFER_SIZE, nullptr, 0) , file_segment(file_segment_) , reserve_space_lock_wait_timeout_milliseconds(getCacheLockWaitTimeout()) { } WriteBufferToFileSegment::WriteBufferToFileSegment(FileSegmentsHolderPtr segment_holder_) - : WriteBufferFromFileDecorator( - segment_holder_->size() == 1 - ? std::make_unique(segment_holder_->front().getPath()) - : throw Exception(ErrorCodes::LOGICAL_ERROR, "WriteBufferToFileSegment can be created only from single segment")) + : WriteBufferFromFileBase(DBMS_DEFAULT_BUFFER_SIZE, nullptr, 0) , file_segment(&segment_holder_->front()) , segment_holder(std::move(segment_holder_)) , reserve_space_lock_wait_timeout_milliseconds(getCacheLockWaitTimeout()) { + if (segment_holder->size() != 1) + throw Exception(ErrorCodes::LOGICAL_ERROR, "WriteBufferToFileSegment can be created only from single segment"); } /// If it throws an exception, the file segment will be incomplete, so you should not use it in the future. @@ -82,9 +81,6 @@ void WriteBufferToFileSegment::nextImpl() reserve_stat_msg += fmt::format("{} hold {}, can release {}; ", toString(kind), ReadableSize(stat.non_releasable_size), ReadableSize(stat.releasable_size)); - if (std::filesystem::exists(file_segment->getPath())) - std::filesystem::remove(file_segment->getPath()); - throw Exception(ErrorCodes::NOT_ENOUGH_SPACE, "Failed to reserve {} bytes for {}: {}(segment info: {})", bytes_to_write, file_segment->getKind() == FileSegmentKind::Temporary ? "temporary file" : "the file in cache", @@ -95,9 +91,8 @@ void WriteBufferToFileSegment::nextImpl() try { - SwapHelper swap(*this, *impl); /// Write data to the underlying buffer. - file_segment->write(*dynamic_cast(impl.get()), written_bytes); + file_segment->write(working_buffer.begin(), bytes_to_write, written_bytes); written_bytes += bytes_to_write; } catch (...) @@ -107,16 +102,32 @@ void WriteBufferToFileSegment::nextImpl() } } +void WriteBufferToFileSegment::finalizeImpl() +{ + next(); + auto cache_writer = file_segment->getLocalCacheWriter(); + if (cache_writer) + { + SwapHelper swap(*this, *cache_writer); + cache_writer->finalize(); + } +} + +void WriteBufferToFileSegment::sync() +{ + next(); + auto cache_writer = file_segment->getLocalCacheWriter(); + if (cache_writer) + { + SwapHelper swap(*this, *cache_writer); + cache_writer->sync(); + } +} + std::unique_ptr WriteBufferToFileSegment::getReadBufferImpl() { finalize(); return std::make_unique(file_segment->getPath()); } -WriteBufferToFileSegment::~WriteBufferToFileSegment() -{ - /// To be sure that file exists before destructor of segment_holder is called - WriteBufferFromFileDecorator::finalize(); -} - } diff --git a/src/Interpreters/Cache/WriteBufferToFileSegment.h b/src/Interpreters/Cache/WriteBufferToFileSegment.h index 62692ed9071..4719dd4be89 100644 --- a/src/Interpreters/Cache/WriteBufferToFileSegment.h +++ b/src/Interpreters/Cache/WriteBufferToFileSegment.h @@ -9,14 +9,20 @@ namespace DB class FileSegment; -class WriteBufferToFileSegment : public WriteBufferFromFileDecorator, public IReadableWriteBuffer +class WriteBufferToFileSegment : public WriteBufferFromFileBase, public IReadableWriteBuffer { public: explicit WriteBufferToFileSegment(FileSegment * file_segment_); explicit WriteBufferToFileSegment(FileSegmentsHolderPtr segment_holder); void nextImpl() override; - ~WriteBufferToFileSegment() override; + + std::string getFileName() const override { return file_segment->getPath(); } + + void sync() override; + +protected: + void finalizeImpl() override; private: From b604fa576dbc39dbe32511be0df24a09ed6a26e5 Mon Sep 17 00:00:00 2001 From: kssenii Date: Mon, 6 May 2024 20:10:02 +0200 Subject: [PATCH 008/115] Fix build --- src/Interpreters/Cache/FileSegment.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Interpreters/Cache/FileSegment.h b/src/Interpreters/Cache/FileSegment.h index f98fb832553..ddbe4fdf45c 100644 --- a/src/Interpreters/Cache/FileSegment.h +++ b/src/Interpreters/Cache/FileSegment.h @@ -244,7 +244,7 @@ private: LockedKeyPtr lockKeyMetadata(bool assert_exists = true) const; void assertWriteAllowed(size_t size, size_t offset); - void writeImpl(size_t size, size_t offset, std::function do_write, WriteBuffer * external_wb = nullptr); + void writeImpl(size_t size, size_t offset, std::function do_write); String tryGetPath() const; From 28bf09a381a2827d5624a29e6156117dad99f992 Mon Sep 17 00:00:00 2001 From: kssenii Date: Tue, 7 May 2024 12:20:27 +0200 Subject: [PATCH 009/115] Minor --- src/Interpreters/Cache/FileSegment.cpp | 119 +++++++++++-------------- src/Interpreters/Cache/FileSegment.h | 3 - 2 files changed, 51 insertions(+), 71 deletions(-) diff --git a/src/Interpreters/Cache/FileSegment.cpp b/src/Interpreters/Cache/FileSegment.cpp index 4548f1d9d20..21b810346c3 100644 --- a/src/Interpreters/Cache/FileSegment.cpp +++ b/src/Interpreters/Cache/FileSegment.cpp @@ -341,9 +341,56 @@ void FileSegment::setRemoteFileReader(RemoteFileReaderPtr remote_file_reader_) void FileSegment::write(char * from, size_t size, size_t offset_in_file) { ProfileEventTimeIncrement watch(ProfileEvents::FileSegmentWriteMicroseconds); - assertWriteAllowed(size, offset_in_file); - writeImpl(size, offset_in_file, [&]() + auto file_segment_path = getPath(); { + if (!size) + throw Exception(ErrorCodes::LOGICAL_ERROR, "Writing zero size is not allowed"); + + { + auto lk = lock(); + assertIsDownloaderUnlocked("write", lk); + assertNotDetachedUnlocked(lk); + } + + if (download_state != State::DOWNLOADING) + throw Exception( + ErrorCodes::LOGICAL_ERROR, + "Expected DOWNLOADING state, got {}", stateToString(download_state)); + + const size_t first_non_downloaded_offset = getCurrentWriteOffset(); + if (offset_in_file != first_non_downloaded_offset) + throw Exception( + ErrorCodes::LOGICAL_ERROR, + "Attempt to write {} bytes to offset: {}, but current write offset is {}", + size, offset_in_file, first_non_downloaded_offset); + + const size_t current_downloaded_size = getDownloadedSize(); + chassert(reserved_size >= current_downloaded_size); + + const size_t free_reserved_size = reserved_size - current_downloaded_size; + if (free_reserved_size < size) + throw Exception( + ErrorCodes::LOGICAL_ERROR, + "Not enough space is reserved. Available: {}, expected: {}", free_reserved_size, size); + + if (!is_unbound && current_downloaded_size == range().size()) + throw Exception(ErrorCodes::LOGICAL_ERROR, "File segment is already fully downloaded"); + + if (!cache_writer && current_downloaded_size > 0) + throw Exception( + ErrorCodes::LOGICAL_ERROR, + "Cache writer was finalized (downloaded size: {}, state: {})", + current_downloaded_size, stateToString(download_state)); + } + + try + { +#ifdef ABORT_ON_LOGICAL_ERROR + /// This mutex is only needed to have a valid assertion in assertCacheCorrectness(), + /// which is only executed in debug/sanitizer builds (under ABORT_ON_LOGICAL_ERROR). + std::lock_guard lock(write_mutex); +#endif + if (!cache_writer) cache_writer = std::make_unique(getPath(), /* buf_size */0); @@ -353,63 +400,6 @@ void FileSegment::write(char * from, size_t size, size_t offset_in_file) SCOPE_EXIT({ cache_writer->set(nullptr, 0); }); /// Flush the buffer. cache_writer->next(); - }); -} - -void FileSegment::assertWriteAllowed(size_t size, size_t offset) -{ - if (!size) - throw Exception(ErrorCodes::LOGICAL_ERROR, "Writing zero size is not allowed"); - - { - auto lk = lock(); - assertIsDownloaderUnlocked("write", lk); - assertNotDetachedUnlocked(lk); - } - - if (download_state != State::DOWNLOADING) - throw Exception( - ErrorCodes::LOGICAL_ERROR, - "Expected DOWNLOADING state, got {}", stateToString(download_state)); - - const size_t first_non_downloaded_offset = getCurrentWriteOffset(); - if (offset != first_non_downloaded_offset) - throw Exception( - ErrorCodes::LOGICAL_ERROR, - "Attempt to write {} bytes to offset: {}, but current write offset is {}", - size, offset, first_non_downloaded_offset); - - const size_t current_downloaded_size = getDownloadedSize(); - chassert(reserved_size >= current_downloaded_size); - - const size_t free_reserved_size = reserved_size - current_downloaded_size; - if (free_reserved_size < size) - throw Exception( - ErrorCodes::LOGICAL_ERROR, - "Not enough space is reserved. Available: {}, expected: {}", free_reserved_size, size); - - if (!is_unbound && current_downloaded_size == range().size()) - throw Exception(ErrorCodes::LOGICAL_ERROR, "File segment is already fully downloaded"); - - if (!cache_writer && current_downloaded_size > 0) - throw Exception( - ErrorCodes::LOGICAL_ERROR, - "Cache writer was finalized (downloaded size: {}, state: {})", - current_downloaded_size, stateToString(download_state)); -} - -void FileSegment::writeImpl(size_t size, [[maybe_unused]] size_t offset, std::function do_write) -{ - auto file_segment_path = getPath(); - try - { -#ifdef ABORT_ON_LOGICAL_ERROR - /// This mutex is only needed to have a valid assertion in assertCacheCorrectness(), - /// which is only executed in debug/sanitizer builds (under ABORT_ON_LOGICAL_ERROR). - std::lock_guard lock(write_mutex); -#endif - - do_write(); downloaded_size += size; chassert(std::filesystem::file_size(file_segment_path) == downloaded_size); @@ -450,7 +440,7 @@ void FileSegment::writeImpl(size_t size, [[maybe_unused]] size_t offset, std::fu throw; } - chassert(getCurrentWriteOffset() == offset + size); + chassert(getCurrentWriteOffset() == offset_in_file + size); } FileSegment::State FileSegment::wait(size_t offset) @@ -606,14 +596,7 @@ void FileSegment::setDownloadFailedUnlocked(const FileSegmentGuard::Lock & lock) if (cache_writer) { - try - { - cache_writer->finalize(); - } - catch (...) - { - tryLogCurrentException(__PRETTY_FUNCTION__); - } + cache_writer->finalize(); cache_writer.reset(); } diff --git a/src/Interpreters/Cache/FileSegment.h b/src/Interpreters/Cache/FileSegment.h index ddbe4fdf45c..d6b37b60dc1 100644 --- a/src/Interpreters/Cache/FileSegment.h +++ b/src/Interpreters/Cache/FileSegment.h @@ -243,9 +243,6 @@ private: LockedKeyPtr lockKeyMetadata(bool assert_exists = true) const; - void assertWriteAllowed(size_t size, size_t offset); - void writeImpl(size_t size, size_t offset, std::function do_write); - String tryGetPath() const; Key file_key; From 056a40aaec19b66ae1cbb8f6ddc9a3d12eff05f7 Mon Sep 17 00:00:00 2001 From: kssenii Date: Wed, 8 May 2024 15:04:11 +0200 Subject: [PATCH 010/115] Fix test --- src/Interpreters/TemporaryDataOnDisk.cpp | 37 ++++++++++++++++-------- 1 file changed, 25 insertions(+), 12 deletions(-) diff --git a/src/Interpreters/TemporaryDataOnDisk.cpp b/src/Interpreters/TemporaryDataOnDisk.cpp index 9a237738b3e..355eb2b6bdc 100644 --- a/src/Interpreters/TemporaryDataOnDisk.cpp +++ b/src/Interpreters/TemporaryDataOnDisk.cpp @@ -226,30 +226,43 @@ struct TemporaryFileStream::OutputWriter struct TemporaryFileStream::InputReader { - InputReader(const String & path, const Block & header_, size_t size = 0) - : in_file_buf(path, size ? std::min(DBMS_DEFAULT_BUFFER_SIZE, size) : DBMS_DEFAULT_BUFFER_SIZE) - , in_compressed_buf(in_file_buf) - , in_reader(in_compressed_buf, header_, DBMS_TCP_PROTOCOL_VERSION) + InputReader(const String & path_, const Block & header_, size_t size_ = DBMS_DEFAULT_BUFFER_SIZE) + : path(path_) + , size(std::min(size_, DBMS_DEFAULT_BUFFER_SIZE)) + , header(header_) { LOG_TEST(getLogger("TemporaryFileStream"), "Reading {} from {}", header_.dumpStructure(), path); } - explicit InputReader(const String & path, size_t size = 0) - : in_file_buf(path, size ? std::min(DBMS_DEFAULT_BUFFER_SIZE, size) : DBMS_DEFAULT_BUFFER_SIZE) - , in_compressed_buf(in_file_buf) - , in_reader(in_compressed_buf, DBMS_TCP_PROTOCOL_VERSION) + explicit InputReader(const String & path_, size_t size_ = DBMS_DEFAULT_BUFFER_SIZE) + : path(path_) + , size(std::min(size_, DBMS_DEFAULT_BUFFER_SIZE)) { LOG_TEST(getLogger("TemporaryFileStream"), "Reading from {}", path); } Block read() { - return in_reader.read(); + if (!in_reader) + { + in_file_buf = std::make_unique(path, size); + + in_compressed_buf = std::make_unique(*in_file_buf); + if (header.has_value()) + in_reader = std::make_unique(*in_compressed_buf, header.value(), DBMS_TCP_PROTOCOL_VERSION); + else + in_reader = std::make_unique(*in_compressed_buf, DBMS_TCP_PROTOCOL_VERSION); + } + return in_reader->read(); } - ReadBufferFromFile in_file_buf; - CompressedReadBuffer in_compressed_buf; - NativeReader in_reader; + const std::string path; + const size_t size; + const std::optional header; + + std::unique_ptr in_file_buf; + std::unique_ptr in_compressed_buf; + std::unique_ptr in_reader; }; TemporaryFileStream::TemporaryFileStream(TemporaryFileOnDiskHolder file_, const Block & header_, TemporaryDataOnDisk * parent_) From 46ecb662d1c7cb59eb7e0ed4562d56bd08021e98 Mon Sep 17 00:00:00 2001 From: kssenii Date: Wed, 8 May 2024 17:07:53 +0200 Subject: [PATCH 011/115] Fix --- src/Interpreters/TemporaryDataOnDisk.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/Interpreters/TemporaryDataOnDisk.cpp b/src/Interpreters/TemporaryDataOnDisk.cpp index 355eb2b6bdc..8e2844fc6e4 100644 --- a/src/Interpreters/TemporaryDataOnDisk.cpp +++ b/src/Interpreters/TemporaryDataOnDisk.cpp @@ -228,7 +228,7 @@ struct TemporaryFileStream::InputReader { InputReader(const String & path_, const Block & header_, size_t size_ = DBMS_DEFAULT_BUFFER_SIZE) : path(path_) - , size(std::min(size_, DBMS_DEFAULT_BUFFER_SIZE)) + , size(size_ ? std::min(size_, DBMS_DEFAULT_BUFFER_SIZE) : DBMS_DEFAULT_BUFFER_SIZE) , header(header_) { LOG_TEST(getLogger("TemporaryFileStream"), "Reading {} from {}", header_.dumpStructure(), path); @@ -236,7 +236,7 @@ struct TemporaryFileStream::InputReader explicit InputReader(const String & path_, size_t size_ = DBMS_DEFAULT_BUFFER_SIZE) : path(path_) - , size(std::min(size_, DBMS_DEFAULT_BUFFER_SIZE)) + , size(size_ ? std::min(size_, DBMS_DEFAULT_BUFFER_SIZE) : DBMS_DEFAULT_BUFFER_SIZE) { LOG_TEST(getLogger("TemporaryFileStream"), "Reading from {}", path); } From 0f9f776314abaabdcc239ac4894ccc165127d17d Mon Sep 17 00:00:00 2001 From: kssenii Date: Fri, 10 May 2024 12:09:14 +0200 Subject: [PATCH 012/115] Fxi --- src/Interpreters/TemporaryDataOnDisk.cpp | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/src/Interpreters/TemporaryDataOnDisk.cpp b/src/Interpreters/TemporaryDataOnDisk.cpp index 8e2844fc6e4..c1beeb6e15c 100644 --- a/src/Interpreters/TemporaryDataOnDisk.cpp +++ b/src/Interpreters/TemporaryDataOnDisk.cpp @@ -2,6 +2,7 @@ #include #include +#include #include #include #include @@ -245,7 +246,10 @@ struct TemporaryFileStream::InputReader { if (!in_reader) { - in_file_buf = std::make_unique(path, size); + if (fs::exists(path)) + in_file_buf = std::make_unique(path, size); + else + in_file_buf = std::make_unique(); in_compressed_buf = std::make_unique(*in_file_buf); if (header.has_value()) @@ -260,7 +264,7 @@ struct TemporaryFileStream::InputReader const size_t size; const std::optional header; - std::unique_ptr in_file_buf; + std::unique_ptr in_file_buf; std::unique_ptr in_compressed_buf; std::unique_ptr in_reader; }; From dd7b4ea4d662f4c81efd2f12bdadaefeee7cfa40 Mon Sep 17 00:00:00 2001 From: kssenii Date: Mon, 13 May 2024 20:40:49 +0200 Subject: [PATCH 013/115] Debug test --- tests/queries/0_stateless/00429_long_http_bufferization.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/queries/0_stateless/00429_long_http_bufferization.sh b/tests/queries/0_stateless/00429_long_http_bufferization.sh index 98dd300e6ab..78cc588a64d 100755 --- a/tests/queries/0_stateless/00429_long_http_bufferization.sh +++ b/tests/queries/0_stateless/00429_long_http_bufferization.sh @@ -78,6 +78,7 @@ function cmp_cli_and_http() { $CLICKHOUSE_CLIENT -q "$(query "$1")" > "${CLICKHOUSE_TMP}"/res1 ch_url "buffer_size=$2&wait_end_of_query=0" "$1" > "${CLICKHOUSE_TMP}"/res2 ch_url "buffer_size=$2&wait_end_of_query=1" "$1" > "${CLICKHOUSE_TMP}"/res3 + cat "${CLICKHOUSE_TMP}"/res3 cmp "${CLICKHOUSE_TMP}"/res1 "${CLICKHOUSE_TMP}"/res2 && cmp "${CLICKHOUSE_TMP}"/res1 "${CLICKHOUSE_TMP}"/res3 || echo FAIL 5 "$@" rm -rf "${CLICKHOUSE_TMP}"/res1 "${CLICKHOUSE_TMP}"/res2 "${CLICKHOUSE_TMP}"/res3 } From dddcb5b8c9392e9eafbad382950aa365e7872e0b Mon Sep 17 00:00:00 2001 From: Kseniia Sumarokova <54203879+kssenii@users.noreply.github.com> Date: Tue, 14 May 2024 16:36:01 +0200 Subject: [PATCH 014/115] Debug test --- tests/queries/0_stateless/00429_long_http_bufferization.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/queries/0_stateless/00429_long_http_bufferization.sh b/tests/queries/0_stateless/00429_long_http_bufferization.sh index 78cc588a64d..72789ddd66c 100755 --- a/tests/queries/0_stateless/00429_long_http_bufferization.sh +++ b/tests/queries/0_stateless/00429_long_http_bufferization.sh @@ -7,7 +7,7 @@ CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) # shellcheck source=../shell_config.sh . "$CURDIR"/../shell_config.sh -format="RowBinary" +format="Pretty" function query { # bash isn't able to store \0 bytes, so use [1; 255] random range From 138feb0eb3e503cfc3143e9241b18a88d6aa3e20 Mon Sep 17 00:00:00 2001 From: kssenii Date: Wed, 15 May 2024 13:34:25 +0200 Subject: [PATCH 015/115] Remove debug --- tests/queries/0_stateless/00429_long_http_bufferization.sh | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/tests/queries/0_stateless/00429_long_http_bufferization.sh b/tests/queries/0_stateless/00429_long_http_bufferization.sh index 72789ddd66c..98dd300e6ab 100755 --- a/tests/queries/0_stateless/00429_long_http_bufferization.sh +++ b/tests/queries/0_stateless/00429_long_http_bufferization.sh @@ -7,7 +7,7 @@ CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) # shellcheck source=../shell_config.sh . "$CURDIR"/../shell_config.sh -format="Pretty" +format="RowBinary" function query { # bash isn't able to store \0 bytes, so use [1; 255] random range @@ -78,7 +78,6 @@ function cmp_cli_and_http() { $CLICKHOUSE_CLIENT -q "$(query "$1")" > "${CLICKHOUSE_TMP}"/res1 ch_url "buffer_size=$2&wait_end_of_query=0" "$1" > "${CLICKHOUSE_TMP}"/res2 ch_url "buffer_size=$2&wait_end_of_query=1" "$1" > "${CLICKHOUSE_TMP}"/res3 - cat "${CLICKHOUSE_TMP}"/res3 cmp "${CLICKHOUSE_TMP}"/res1 "${CLICKHOUSE_TMP}"/res2 && cmp "${CLICKHOUSE_TMP}"/res1 "${CLICKHOUSE_TMP}"/res3 || echo FAIL 5 "$@" rm -rf "${CLICKHOUSE_TMP}"/res1 "${CLICKHOUSE_TMP}"/res2 "${CLICKHOUSE_TMP}"/res3 } From 3a7e338839013e46d30e27f0fe7f959d4deb66cf Mon Sep 17 00:00:00 2001 From: Smita Kulkarni Date: Wed, 22 May 2024 16:24:16 +0200 Subject: [PATCH 016/115] Remove extra print --- tests/config/install.sh | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/tests/config/install.sh b/tests/config/install.sh index 3eaa39fa8d3..f15990fb9c9 100755 --- a/tests/config/install.sh +++ b/tests/config/install.sh @@ -181,8 +181,7 @@ elif [[ "$USE_AZURE_STORAGE_FOR_MERGE_TREE" == "1" ]]; then ln -sf $SRC_PATH/config.d/azure_storage_policy_by_default.xml $DEST_SERVER_PATH/config.d/ fi -if [[ -n "$EXPORT_S3_STORAGE_POLICIES" ]]; then - echo "$OS" +if [[ -n "$EXPORT_S3_STORAGE_POLICIES" ]]; then\ if [[ "$USE_DATABASE_REPLICATED" -eq 1 ]]; then echo "Azure configuration will not be added" else From bb2f51c51fab0942291b50194333e0a587541ac3 Mon Sep 17 00:00:00 2001 From: Azat Khuzhin Date: Mon, 3 Jun 2024 16:56:17 +0200 Subject: [PATCH 017/115] Fix filling parts columns from metadata (when columns.txt does not exists) The getFileNameForColumn() inside IMergeTreeDataPart::loadColumns() requries serializations that will be filled only at the end of loadColumns() (setColumns()): (lldb) bt * thread 718, name = 'ActiveParts', stop reason = breakpoint 1.1 * frame 0: 0x00000000111e50dc clickhouse`DB::IMergeTreeDataPart::getSerialization(this=0x00007ffd18c39918, column_name="key") const + 124 at IMergeTreeDataPart.cpp:508 frame 1: 0x0000000011340f57 clickhouse`DB::MergeTreeDataPartWide::getFileNameForColumn(this=0x00007ffd18c39918, column=) const + 55 at MergeTreeDataPartWide.cpp:301 frame 2: 0x00000000111e9a2b clickhouse`DB::IMergeTreeDataPart::loadColumns(this=0x00007ffd18c39918, require=false) + 1067 at IMergeTreeDataPart.cpp:1577 frame 3: 0x00000000111e8d52 clickhouse`DB::IMergeTreeDataPart::loadColumnsChecksumsIndexes(this=0x00007ffd18c39918, require_columns_checksums=, check_consistency=true) + 82 at IMergeTreeDataPart.cpp:713 frame 4: 0x0000000011284c10 clickhouse`DB::MergeTreeData::loadDataPart(this=0x00007ffcf9242e40, part_info=0x00007ffcf937c298, part_name="2024", part_disk_ptr=std::__1::shared_ptr::element_type @ 0x00007ffcf925af18 strong=11 weak=2, to_state=Active, part_loading_mutex=0x00007ffef97efeb0) + 3152 at MergeTreeData.cpp:1435 So let's add fallback into the getFileNameForColumn() in case of serializations was not filled yet. Note, that it is OK to add this check as a generic, since after loadColumns() serializations cannot be empty, since loadColumns() does not allows this (there is only one more caller of getFileNameForColumn() - loadIndexGranularity()) Signed-off-by: Azat Khuzhin --- src/Storages/MergeTree/MergeTreeDataPartWide.cpp | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/src/Storages/MergeTree/MergeTreeDataPartWide.cpp b/src/Storages/MergeTree/MergeTreeDataPartWide.cpp index 149f86cef00..d91fc5dfbd8 100644 --- a/src/Storages/MergeTree/MergeTreeDataPartWide.cpp +++ b/src/Storages/MergeTree/MergeTreeDataPartWide.cpp @@ -298,6 +298,11 @@ std::optional MergeTreeDataPartWide::getColumnModificationTime(const Str std::optional MergeTreeDataPartWide::getFileNameForColumn(const NameAndTypePair & column) const { std::optional filename; + + /// Fallback for the case when serializations was not loaded yet (called from loadColumns()) + if (getSerializations().empty()) + return getStreamNameForColumn(column, {}, DATA_FILE_EXTENSION, getDataPartStorage()); + getSerialization(column.name)->enumerateStreams([&](const ISerialization::SubstreamPath & substream_path) { if (!filename.has_value()) @@ -309,6 +314,7 @@ std::optional MergeTreeDataPartWide::getFileNameForColumn(const NameAndT filename = getStreamNameForColumn(column, substream_path, DATA_FILE_EXTENSION, getDataPartStorage()); } }); + return filename; } From a1cb72a283da0b2f5034864e9118719f3e61727c Mon Sep 17 00:00:00 2001 From: Azat Khuzhin Date: Mon, 3 Jun 2024 17:19:27 +0200 Subject: [PATCH 018/115] Fix "No columns in part" check Signed-off-by: Azat Khuzhin --- src/Storages/MergeTree/IMergeTreeDataPart.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Storages/MergeTree/IMergeTreeDataPart.cpp b/src/Storages/MergeTree/IMergeTreeDataPart.cpp index c276361559c..85a1e6b27be 100644 --- a/src/Storages/MergeTree/IMergeTreeDataPart.cpp +++ b/src/Storages/MergeTree/IMergeTreeDataPart.cpp @@ -1577,7 +1577,7 @@ void IMergeTreeDataPart::loadColumns(bool require) if (getFileNameForColumn(column)) loaded_columns.push_back(column); - if (columns.empty()) + if (loaded_columns.empty()) throw Exception(ErrorCodes::NO_FILE_IN_DATA_PART, "No columns in part {}", name); if (!is_readonly_storage) From 8d46078911194d81524bb9aaadca858e23756947 Mon Sep 17 00:00:00 2001 From: Azat Khuzhin Date: Wed, 5 Jun 2024 10:59:22 +0200 Subject: [PATCH 019/115] Use primary.cidx instead of columns.txt to break parts After fixes for columns.txt the test_max_suspicious_broken_parts starts to fail Signed-off-by: Azat Khuzhin --- tests/integration/test_max_suspicious_broken_parts/test.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/integration/test_max_suspicious_broken_parts/test.py b/tests/integration/test_max_suspicious_broken_parts/test.py index c1f34adbb62..831531ed690 100644 --- a/tests/integration/test_max_suspicious_broken_parts/test.py +++ b/tests/integration/test_max_suspicious_broken_parts/test.py @@ -25,7 +25,7 @@ def break_part(table, part_name): [ "bash", "-c", - f"rm /var/lib/clickhouse/data/default/{table}/{part_name}/columns.txt", + f"rm /var/lib/clickhouse/data/default/{table}/{part_name}/primary.cidx", ] ) From b4e12156dd04436877f0d5c719927fc76cc6c015 Mon Sep 17 00:00:00 2001 From: Dale Mcdiarmid Date: Tue, 11 Jun 2024 18:55:49 +0100 Subject: [PATCH 020/115] stackoverflow dataset --- .../example-datasets/stackoverflow.md | 393 ++++++++++++++++++ 1 file changed, 393 insertions(+) create mode 100644 docs/en/getting-started/example-datasets/stackoverflow.md diff --git a/docs/en/getting-started/example-datasets/stackoverflow.md b/docs/en/getting-started/example-datasets/stackoverflow.md new file mode 100644 index 00000000000..a27eaf5b8a1 --- /dev/null +++ b/docs/en/getting-started/example-datasets/stackoverflow.md @@ -0,0 +1,393 @@ +--- +slug: /en/getting-started/example-datasets/stackoverflow +sidebar_label: Stack Overflow +sidebar_position: 1 +description: Analyzing Stack Overflow data with ClickHouse +--- + +# Analyzing Stack Overflow data with ClickHouse + +This dataset contains every Post, User, Vote, Comment, Badge, PostHistory, and PostLink that has occurred on Stack Overflow. + +Users can either download pre-prepared Parquet versions of the data, containing every post up to April 2024, or download the latest data in XML format and load this. Stack Overflow provide updates to this data periodically - historically every 3 months. + +The following diagram shows the schema for the available tables assuming Parquet format. + + +A description of the schema of this data can be found [here](https://meta.stackexchange.com/questions/2677/database-schema-documentation-for-the-public-data-dump-and-sede). + +## Pre-prepared data + +We provide a copy of this data in Parquet format, upto date as of April 2024. While small for ClickHouse with respect to the number of rows (60 million posts), this dataset contains significant volumes of text and large String columns. + +```sql +CREATE DATABASE stackoverflow +``` + +The following timings are for a 96 GiB, 24 vCPU ClickHouse Cloud cluster located in `eu-west-2`. The dataset is located in `eu-west-3`. + +### Posts + +```sql +CREATE TABLE stackoverflow.posts +( + `Id` Int32 CODEC(Delta(4), ZSTD(1)), + `PostTypeId` Enum8('Question' = 1, 'Answer' = 2, 'Wiki' = 3, 'TagWikiExcerpt' = 4, 'TagWiki' = 5, 'ModeratorNomination' = 6, 'WikiPlaceholder' = 7, 'PrivilegeWiki' = 8), + `AcceptedAnswerId` UInt32, + `CreationDate` DateTime64(3, 'UTC'), + `Score` Int32, + `ViewCount` UInt32 CODEC(Delta(4), ZSTD(1)), + `Body` String, + `OwnerUserId` Int32, + `OwnerDisplayName` String, + `LastEditorUserId` Int32, + `LastEditorDisplayName` String, + `LastEditDate` DateTime64(3, 'UTC') CODEC(Delta(8), ZSTD(1)), + `LastActivityDate` DateTime64(3, 'UTC'), + `Title` String, + `Tags` String, + `AnswerCount` UInt16 CODEC(Delta(2), ZSTD(1)), + `CommentCount` UInt8, + `FavoriteCount` UInt8, + `ContentLicense` LowCardinality(String), + `ParentId` String, + `CommunityOwnedDate` DateTime64(3, 'UTC'), + `ClosedDate` DateTime64(3, 'UTC') +) +ENGINE = MergeTree +PARTITION BY toYear(CreationDate) +ORDER BY (PostTypeId, toDate(CreationDate), CreationDate) + +INSERT INTO stackoverflow.posts SELECT * FROM s3('https://datasets-documentation.s3.eu-west-3.amazonaws.com/stackoverflow/parquet/posts/*.parquet') + +0 rows in set. Elapsed: 265.466 sec. Processed 59.82 million rows, 38.07 GB (225.34 thousand rows/s., 143.42 MB/s.) +``` + +Posts are also available by year e.g. [https://datasets-documentation.s3.eu-west-3.amazonaws.com/stackoverflow/parquet/posts/2020.parquet](https://datasets-documentation.s3.eu-west-3.amazonaws.com/stackoverflow/parquet/posts/2020.parquet) + + +### Votes + +```sql +CREATE TABLE stackoverflow.votes +( + `Id` UInt32, + `PostId` Int32, + `VoteTypeId` UInt8, + `CreationDate` DateTime64(3, 'UTC'), + `UserId` Int32, + `BountyAmount` UInt8 +) +ENGINE = MergeTree +ORDER BY (VoteTypeId, CreationDate, PostId, UserId) + +INSERT INTO stackoverflow.votes SELECT * FROM s3('https://datasets-documentation.s3.eu-west-3.amazonaws.com/stackoverflow/parquet/votes/*.parquet') + +0 rows in set. Elapsed: 21.605 sec. Processed 238.98 million rows, 2.13 GB (11.06 million rows/s., 98.46 MB/s.) +``` + +Votes are also available by year e.g. [https://datasets-documentation.s3.eu-west-3.amazonaws.com/stackoverflow/parquet/posts/2020.parquet](https://datasets-documentation.s3.eu-west-3.amazonaws.com/stackoverflow/parquet/votes/2020.parquet) + + +### Comments + +```sql +CREATE TABLE stackoverflow.comments +( + `Id` UInt32, + `PostId` UInt32, + `Score` UInt16, + `Text` String, + `CreationDate` DateTime64(3, 'UTC'), + `UserId` Int32, + `UserDisplayName` LowCardinality(String) +) +ENGINE = MergeTree +ORDER BY CreationDate + +INSERT INTO stackoverflow.comments SELECT * FROM s3('https://datasets-documentation.s3.eu-west-3.amazonaws.com/stackoverflow/parquet/comments/*.parquet') + +0 rows in set. Elapsed: 56.593 sec. Processed 90.38 million rows, 11.14 GB (1.60 million rows/s., 196.78 MB/s.) +``` + +Comments are also available by year e.g. [https://datasets-documentation.s3.eu-west-3.amazonaws.com/stackoverflow/parquet/posts/2020.parquet](https://datasets-documentation.s3.eu-west-3.amazonaws.com/stackoverflow/parquet/comments/2020.parquet) + +### Users + +```sql +CREATE TABLE stackoverflow.users +( + `Id` Int32, + `Reputation` LowCardinality(String), + `CreationDate` DateTime64(3, 'UTC') CODEC(Delta(8), ZSTD(1)), + `DisplayName` String, + `LastAccessDate` DateTime64(3, 'UTC'), + `AboutMe` String, + `Views` UInt32, + `UpVotes` UInt32, + `DownVotes` UInt32, + `WebsiteUrl` String, + `Location` LowCardinality(String), + `AccountId` Int32 +) +ENGINE = MergeTree +ORDER BY (Id, CreationDate) + +INSERT INTO stackoverflow.users SELECT * FROM s3('https://datasets-documentation.s3.eu-west-3.amazonaws.com/stackoverflow/parquet/users.parquet') + +0 rows in set. Elapsed: 10.988 sec. Processed 22.48 million rows, 1.36 GB (2.05 million rows/s., 124.10 MB/s.) +``` + +### Badges + +```sql +CREATE TABLE stackoverflow.badges +( + `Id` UInt32, + `UserId` Int32, + `Name` LowCardinality(String), + `Date` DateTime64(3, 'UTC'), + `Class` Enum8('Gold' = 1, 'Silver' = 2, 'Bronze' = 3), + `TagBased` Bool +) +ENGINE = MergeTree +ORDER BY UserId + +INSERT INTO stackoverflow.badges SELECT * FROM s3('https://datasets-documentation.s3.eu-west-3.amazonaws.com/stackoverflow/parquet/badges.parquet') + +0 rows in set. Elapsed: 6.635 sec. Processed 51.29 million rows, 797.05 MB (7.73 million rows/s., 120.13 MB/s.) +``` + +### Postlinks + +```sql +CREATE TABLE stackoverflow.postlinks +( + `Id` UInt64, + `CreationDate` DateTime64(3, 'UTC'), + `PostId` Int32, + `RelatedPostId` Int32, + `LinkTypeId` Enum8('Linked' = 1, 'Duplicate' = 3) +) +ENGINE = MergeTree +ORDER BY (PostId, RelatedPostId) + +INSERT INTO stackoverflow.postlinks SELECT * FROM s3('https://datasets-documentation.s3.eu-west-3.amazonaws.com/stackoverflow/parquet/postlinks.parquet') + +0 rows in set. Elapsed: 1.534 sec. Processed 6.55 million rows, 129.70 MB (4.27 million rows/s., 84.57 MB/s.) +``` + +### PostHistory + +```sql +CREATE TABLE stackoverflow.posthistory +( + `Id` UInt64, + `PostHistoryTypeId` UInt8, + `PostId` Int32, + `RevisionGUID` String, + `CreationDate` DateTime64(3, 'UTC'), + `UserId` Int32, + `Text` String, + `ContentLicense` LowCardinality(String), + `Comment` String, + `UserDisplayName` String +) +ENGINE = MergeTree +ORDER BY (CreationDate, PostId) + +INSERT INTO stackoverflow.posthistory SELECT * FROM s3('https://datasets-documentation.s3.eu-west-3.amazonaws.com/stackoverflow/parquet/posthistory/*.parquet') + +0 rows in set. Elapsed: 422.795 sec. Processed 160.79 million rows, 67.08 GB (380.30 thousand rows/s., 158.67 MB/s.) +``` + +## Original dataset + +The original dataset is available in compressed (7zip) XML format at [https://archive.org/download/stackexchange](https://archive.org/download/stackexchange) - files with prefix `stackoverflow.com*`. + +### Download + +```bash +wget https://archive.org/download/stackexchange/stackoverflow.com-Badges.7z +wget https://archive.org/download/stackexchange/stackoverflow.com-Comments.7z +wget https://archive.org/download/stackexchange/stackoverflow.com-PostHistory.7z +wget https://archive.org/download/stackexchange/stackoverflow.com-PostLinks.7z +wget https://archive.org/download/stackexchange/stackoverflow.com-Posts.7z +wget https://archive.org/download/stackexchange/stackoverflow.com-Users.7z +wget https://archive.org/download/stackexchange/stackoverflow.com-Votes.7z +``` + +These files are upto 35GB and can take around 30 mins to download depending on internet connection - the download server throttles at around 20MB/sec. + +### Convert to JSON + +At the time of writing, ClickHouse does not have native support for XML as an input format. To load the data into ClickHouse we first convert to NdJSON. + +To convert XML to JSON we recommend the [`xq`](https://github.com/kislyuk/yq) linux tool, a simple `jq` wrapper for XML documents. + +Install xq and jq: + +```bash +sudo apt install jq +pip install yq +``` + +The following steps apply to any of the above files. We use the `stackoverflow.com-Posts.7z` file as an example. Modify as required. + +Extract the file using [p7zip](https://p7zip.sourceforge.net/). This will produce a single xml file - in this case `Posts.xml`. + +> Files are compressed approximately 4.5x. At 22GB compressed, the posts file requires around 97G uncompressed. + +```bash +p7zip -d stackoverflow.com-Posts.7z +``` + +The following splits the xml file into files, each containing 10000 rows. + +```bash +mkdir posts +cd posts +# the following splits the input xml file into sub files of 10000 rows +tail +3 ../Posts.xml | head -n -1 | split -l 10000 --filter='{ printf "\n"; cat - ; printf "\n"; } > $FILE' - +``` + +After running the above users will have a set of files, each with 10000 lines. This ensures the memory overhead of the next command is not excessive (xml to JSON conversion is done in memory). + +```bash +find . -maxdepth 1 -type f -exec xq -c '.rows.row[]' {} \; | sed -e 's:"@:":g' > posts_v2.json +``` + +The above command will produce a single `posts.json` file. + +Load into ClickHouse with the following command. Note the schema is specified for the `posts.json` file. This will need to be adjusted per data type to align with the target table. + +```bash +clickhouse local --query "SELECT * FROM file('posts.json', JSONEachRow, 'Id Int32, PostTypeId UInt8, AcceptedAnswerId UInt32, CreationDate DateTime64(3, \'UTC\'), Score Int32, ViewCount UInt32, Body String, OwnerUserId Int32, OwnerDisplayName String, LastEditorUserId Int32, LastEditorDisplayName String, LastEditDate DateTime64(3, \'UTC\'), LastActivityDate DateTime64(3, \'UTC\'), Title String, Tags String, AnswerCount UInt16, CommentCount UInt8, FavoriteCount UInt8, ContentLicense String, ParentId String, CommunityOwnedDate DateTime64(3, \'UTC\'), ClosedDate DateTime64(3, \'UTC\')') FORMAT Native" | clickhouse client --host --secure --password --query "INSERT INTO stackoverflow.posts_v2 FORMAT Native" +``` + +## Example queries + +A few simple questions to you get started. + +### Most popular tags on Stack Overflow + +```sql + +SELECT + arrayJoin(arrayFilter(t -> (t != ''), splitByChar('|', Tags))) AS Tags, + count() AS c +FROM stackoverflow.posts +GROUP BY Tags +ORDER BY c DESC +LIMIT 10 + +┌─Tags───────┬───────c─┐ +│ javascript │ 2527130 │ +│ python │ 2189638 │ +│ java │ 1916156 │ +│ c# │ 1614236 │ +│ php │ 1463901 │ +│ android │ 1416442 │ +│ html │ 1186567 │ +│ jquery │ 1034621 │ +│ c++ │ 806202 │ +│ css │ 803755 │ +└────────────┴─────────┘ + +10 rows in set. Elapsed: 1.013 sec. Processed 59.82 million rows, 1.21 GB (59.07 million rows/s., 1.19 GB/s.) +Peak memory usage: 224.03 MiB. +``` + +### User with the most answers (active accounts) + +Account requires a UserId. + +```sql +SELECT + any(OwnerUserId) UserId, + OwnerDisplayName, + count() AS c +FROM stackoverflow.posts WHERE OwnerDisplayName != '' AND PostTypeId='Answer' AND OwnerUserId != 0 +GROUP BY OwnerDisplayName +ORDER BY c DESC +LIMIT 5 + +┌─UserId─┬─OwnerDisplayName─┬────c─┐ +│ 22656 │ Jon Skeet │ 2727 │ +│ 23354 │ Marc Gravell │ 2150 │ +│ 12950 │ tvanfosson │ 1530 │ +│ 3043 │ Joel Coehoorn │ 1438 │ +│ 10661 │ S.Lott │ 1087 │ +└────────┴──────────────────┴──────┘ + +5 rows in set. Elapsed: 0.154 sec. Processed 35.83 million rows, 193.39 MB (232.33 million rows/s., 1.25 GB/s.) +Peak memory usage: 206.45 MiB. +``` + +### ClickHouse related posts with the most views + +```sql +SELECT + Id, + Title, + ViewCount, + AnswerCount +FROM stackoverflow.posts +WHERE Title ILIKE '%ClickHouse%' +ORDER BY ViewCount DESC +LIMIT 10 + +┌───────Id─┬─Title────────────────────────────────────────────────────────────────────────────┬─ViewCount─┬─AnswerCount─┐ +│ 52355143 │ Is it possible to delete old records from clickhouse table? │ 41462 │ 3 │ +│ 37954203 │ Clickhouse Data Import │ 38735 │ 3 │ +│ 37901642 │ Updating data in Clickhouse │ 36236 │ 6 │ +│ 58422110 │ Pandas: How to insert dataframe into Clickhouse │ 29731 │ 4 │ +│ 63621318 │ DBeaver - Clickhouse - SQL Error [159] .. Read timed out │ 27350 │ 1 │ +│ 47591813 │ How to filter clickhouse table by array column contents? │ 27078 │ 2 │ +│ 58728436 │ How to search the string in query with case insensitive on Clickhouse database? │ 26567 │ 3 │ +│ 65316905 │ Clickhouse: DB::Exception: Memory limit (for query) exceeded │ 24899 │ 2 │ +│ 49944865 │ How to add a column in clickhouse │ 24424 │ 1 │ +│ 59712399 │ How to cast date Strings to DateTime format with extended parsing in ClickHouse? │ 22620 │ 1 │ +└──────────┴──────────────────────────────────────────────────────────────────────────────────┴───────────┴─────────────┘ + +10 rows in set. Elapsed: 0.472 sec. Processed 59.82 million rows, 1.91 GB (126.63 million rows/s., 4.03 GB/s.) +Peak memory usage: 240.01 MiB. +``` + +### Most controversial posts + +```sql +SELECT + Id, + Title, + UpVotes, + DownVotes, + abs(UpVotes - DownVotes) AS Controversial_ratio +FROM stackoverflow.posts +INNER JOIN +( + SELECT + PostId, + countIf(VoteTypeId = 2) AS UpVotes, + countIf(VoteTypeId = 3) AS DownVotes + FROM stackoverflow.votes + GROUP BY PostId + HAVING (UpVotes > 10) AND (DownVotes > 10) +) AS votes ON posts.Id = votes.PostId +WHERE Title != '' +ORDER BY Controversial_ratio ASC +LIMIT 3 + +┌───────Id─┬─Title─────────────────────────────────────────────┬─UpVotes─┬─DownVotes─┬─Controversial_ratio─┐ +│ 583177 │ VB.NET Infinite For Loop │ 12 │ 12 │ 0 │ +│ 9756797 │ Read console input as enumerable - one statement? │ 16 │ 16 │ 0 │ +│ 13329132 │ What's the point of ARGV in Ruby? │ 22 │ 22 │ 0 │ +└──────────┴───────────────────────────────────────────────────┴─────────┴───────────┴─────────────────────┘ + +3 rows in set. Elapsed: 4.779 sec. Processed 298.80 million rows, 3.16 GB (62.52 million rows/s., 661.05 MB/s.) +Peak memory usage: 6.05 GiB. +``` + +## Attribution + +We thank Stack Overflow for providing this data under the `cc-by-sa 4.0` license, acknowledging their efforts and the original source of the data at [https://archive.org/details/stackexchange](https://archive.org/details/stackexchange). From 20385161e45bbba014e41820cbf7d77386cf22d0 Mon Sep 17 00:00:00 2001 From: Dale Mcdiarmid Date: Wed, 12 Jun 2024 12:33:18 +0100 Subject: [PATCH 021/115] add diagram --- .../example-datasets/images/stackoverflow.png | Bin 0 -> 165368 bytes .../example-datasets/stackoverflow.md | 1 + 2 files changed, 1 insertion(+) create mode 100644 docs/en/getting-started/example-datasets/images/stackoverflow.png diff --git a/docs/en/getting-started/example-datasets/images/stackoverflow.png b/docs/en/getting-started/example-datasets/images/stackoverflow.png new file mode 100644 index 0000000000000000000000000000000000000000..f31acdc8cc3ba1b80c99215f554c54a7ea50f4be GIT binary patch literal 165368 zcmeFZ1yfzmvNsF_cS&$}5@6%*?wTOM-66O;fe;<3GwE22=EUg!~z2PU-u9Y)WC0Gnk>lw z`7{ga-=9KXWxe_L{Tq$ff$nN5+rT8679Ta7H00%YjO=U}4UFv!O&Hy5zPt{A;B(^v zZf#7Q3`pHHQs|2;SGRle7!JcK#e_e)K^~>Uq`mFG=ns1n(6VGt=+M~G0)v7dgYIqb2|5we zyAs#C@U;)b6_bMMA_{2XB|P!D?eJ;4Vrgk`89F|$y#2yEH9dWqo}6(2{LmBeUN_Rd z#UzFN%jI81ivA8e-1j0XGLbw=N!k}@XQ9N}+$g*!Irx z{>}pvxlo`k?g~A2{9k!6pV(ym%iI7>12I3?Hl}7x`uBg~5dvW%tKaM|>jT!^{S5-e z!@#LlaU!bzp{zq;RXA*U6vvZho8v8k>Csyr}`_KfFdCOvlaj} z|4$JB(fpqx{GTHH>-_znBK)5s{Qp&iXoe|t>&3eFQ;f;;vQWv9LJ9Pmc_Lpelm9ep z(3pH*!ez$!!D?npr^7>SSiz|cViWL*P2-M5ulQxwBu%P{F_6H}A||Om_I6$@ol13C zLKhmr_&t<4R*nsBu?Sk5RNG~%?Oz<5e-+@EbTw8vuv7lxd`VRR=ZRD7nlUf`&(56^ zJ>k{$3@`PG#Qw$gh`k5gBRP2NviWb`tvlw`1G_J2nE#DYDJp=`yj6Ey&c8T$Aq~s{ z?1X&N>{lA3-SBZA)D`~x`Li{VEn4$sqx-F%cLtw(o@T8nb!I3QO#zpn>Axb1Z1kc} zr3rDg$|4-rze%;~t0D0=zKHs~1Lxh4#4GBSRB> ztU6k0nYZ~SD_vcxR{s9wVeLf@n~2BhY^h3NklT&^m0dS1x4rC;yicWSEG|bHO*gBa zZLYGh)bhr?-!M#h4-3TH8vXx=$}%+9)?7pY(kq$ha|GkT7^+R27%IwFzf5)Zbh9?x z7eSB@JPvicEgi16U3uRhNfb*V87U-p$* z{tpNQ%47)st4Gqn3vQ=9rP`a0m#4}!E_)FV@B!7CSWwA>Bcdz^X z!kt^8yF0q~oB(k#Tz@jr;G|}lRgWTG>-4~EYn>LC_#E-Dp=4Gw?jAhl2Iu-zR!1sW!g-yG$baRwjGCl(daYF6p{Ki(f!ntJ{q7vGxGvWvgH zbz{xL;aT{KgcA;lflxLU!UQ+3>sihX>r503uL?wNA`}pcOB^)PP7 zqTul=Tt2(!^%3P!*I@0L;B>l%|`Ik5++FTc0j8qHc7H8a9k9{gtobKP>JaGC1;^x(9zV#zCia`0m8xJSw zzj z<>9nrxZY-I#N`1FjD0}KHZ%EcwZSfP;cMi#j^}&Xla3dkTGa8heqzrNRS~W#$Xp*M8`CnZD0W z$z@Qw?nPMm?3P;`JKk55k_DF#umW;*Z#Gln0VN;Zlb%ccZ7Ye7yGx4O*i&`2t0R39HwL+7EANDL)u@6+NC zT(3t_z@+;FhLvtk%Mk7e`E)sP}vw?4(M@x!XicY=-I92*ziaxF;=Zw zH@}2;qPw?N0)kBCu(7OUe%`uF`U2wN*l2r?>G$o(q|g0M9| z?iJJ32uEHu5eY+YTtGGIOReeX_fZM-*de;-3&(;D7QlKNV+rL>e@DTM+ORd}>Jf!y z&RjpXLUG_=@MtA_xH;wQP)bM-=;M5AM>~6F=}(4!XN1d`P_Ex(`0IOtI*~Iz{=+x! zuN$p<7LuAuu)Uo*uh|(IByJ~Zi_0-f$Amf){TO1aD;Y;U^}*ZTX#x)Gr1LmY(bjSo zmF}t?{_ruvJun>e5$WDCGUeG&t^Lm|J%d{YWc%mdgoOmGeI<3_i=9y`pJS54{cz_Q zgF?kj4v$3wyHM)u(hDp*L7_G3}BX8qDp5rlP6t(eN1=K&ck z%?rSi7%31MJuW-3$(3i#!y)EOU!p!t+=_RqAxl<&XYz2ypeGIpzR3~}%(Go--VyZp zo(_4i0kiqm_l|oSu)I>FH3(TZcvx4Yzw2z$x&e7IPV4_ zV2!*N!4o=a*4!Wgghq)aC75bq<;rzJTn?n_J~7-=^nSE-1YEG+tv4=%zu!H;v6HKHmaV8H zL-2U_Lw{yCE_3&ciHqQY03slpA$ZbuF6i`Nb0jib>Q=o;cZk3i6p{coxAAI{o}1W< zRfWsvv$Rv!0>^+wY16h>rjL~{hmZck5jmo-dSRm`RXn@Ob zA(jrGOL}c&h{85?d=zDGZ(qjd14ES3`a6Zp>X0GW#c#P`UTDy8)5GWyV@dYt>CkHhk z1kJ~QwpXq$+OFeiPKTu153YyEiOGp&ol6_-iW8Ya;z$V>srAw|18KWn)K9s?AYBBA z$XTE!J}Gsq-)tx9ahx)Bd$r!}q)LSNF{3sk@DO15z{kwoWOzsoonS;!1EyRLN|VRs zXydyVar=C9UQ7tjN;9h}ZM2fUH-dI+_WB}PXeFEZE$recA9dVg+J2K9)DJ0$w$`K^ zK2|A*14`|N4iAJSvgZT(qe}s7FlxB7QpqH`1O+yVlN*^RlW0YHHtJ!~wEn=QKuE_i zS8xnB{MJaa7L9o5AR$g{XeBw2D|+Nn1vz~+7t2-fJNKC>QVw0UGNUf~Q0AKE$h+$8 zM0~p#(z@{ur-XQGWS)@%4LSQ~&()F3L)a7gDbL`Vi)Y#mh=*$39Ujft?Vh_WyKUMb z9a@AcnnBCu7~WC72J?U!k6ovT2Z?&O+nkC^kkkodRs8g3S~s`DgNpld+uNOtPAZP~TNM!Vv>Js1%D zQOb5DjD7fxM~tBcA$pqjr&8gvVa1(35zsdlM#$6Vu9m!Kr#PWgfeYk>${Y9A`M~WB zkgxAj+hsq`A74WjG`G91*uBjt!Rz?&B_M*15hk5z*{jr*9`htAVw91nc87m;85~VO1sR6Yk^*6>^k6tp)-SmH*Zixyom%d+6_hD!vY!4j$@%c%V`GqZ;}<>&(DZr(AnviLUQS6&oVvoV>J|kdj{`p!*1j-|)i2l14f9Hz(bstdKiGdLGA2$l1 z_FAA&hv!q2{p*lHGGO%%Md!HxB%6OB`i&w0ofeUG_s@SFB8B?8RiY97z?t~xtMeUs z^f6*Or)pZg`6Rgtsjm;W+g=qRj}xt`HWnxVkf#?yZyX4uSt}v^7^&lM?<8`o3oi?t znP>y}t)X~XnZ);WW@8!g$4i^6a!nfOfU2HR3#C)Fo|4Ea0;8B;%Mj}V%iiDet+6*XjjiG-Mr%_cr2Y)N?SWEjmKSOu1u4L)ok=yq1zc# z$`$H(u|Cdvp=Rh2CT#m`BX=rGn21ihD2uK?JlR)Pq}!rayG%p5SgSCYkjG0bfkBJT z<}>Gw)*PLx*VI)zW zCMG7dum&Q^@9H!ntpCa|PhZSOC8t;rWov`-*`V_P0U}Ai^ zW8?9HmFMyE{fYP3SJlF+M$L-Nistt%GsUW^0MMcoOVxQE^?Rc zl=e0NA;M&_xcI2q;-W*2I-0_fkS$sn`{uKx;MLh?=(VSv`9xO5OsST-!}eJDU3t7_ z?M@M@^As=Myfo<`^i=bzn14|%&#bwr;!B1;a_=mw$>F3 z`9m2V=2dnOlkJXY-ehtO5YsLKI6?SG3b&Te(=l4ty-mZaF%O>46ZnQy($UotomRUs z+C79(SFGX3aF*Gc_?GY00JCPjS3LKp{O|Ab#6k^C@Aj%Ueb(9>cgF|=)zU_t(jNYOrO{Y&ZPro|(40P;!dNcS2fSy*>KiDAglC$n`}^ zq3Fi7-g;qK!oe-UPFeJ+D==hh;79J5bX+o)}j^MP+_Le{*R0gl+laOFI$Rrfe7X&s| zL_!T_@TbzxDr>p0;Sh=pS+t->lK-^MW5?!$c0$Buf-g&Ew-spt1BYj}Xw@8 zTx%t_!to@ddhqe|3T+qDp9PzuiOcO_tM?boMl_bWjEP6fO-ntc$fBMbV-dQzA0LYu z)Qcx`wQf!hs2pVaq6iWf7tFpr>bkopy`@w69hdL@P`|_Zn*{f4Vy}L+>uk&_EBnp5 zd>|79Zh&l{F4K351`=5N$9|?;7aOgCEjm_y1=t?0XVHnKAa%IYxn=jTk$PD0Vbaa@ z)jsQi4zDue+W`m}>?Thv;S_Sgy*ngA}xy zWuTKgZoAmZZk}4DQBBUHk|mPz6Cu_keIp?QwhjFh7a46X2@WmPW~pBKaz|dihfV`~ zJ9l{hOW*N{aTMQukY=CccM&MO_|9f$g8Y=egP~ZOsF^ZlcIjVYXp!;6>aHB|yl+n; zQtNSX4jwI!Nk5ATPY2i~|P>GkiJTm{w5*QbK!LrNV!HceTu zO-fV^*I7V>5@JvfKNs~Kzq)gxWf453q;<7NNwualPJ*yM6z75`+;zrOV{bG#_3mWaa0ZB1F4@F1*EtNdh$3QG%@h`XT16%EO{PQQh)@fm zwV^9-2-gNA3z5N-5=rTgsDkV>d*slqV9!f*y^lwQG{2A_Xfs18MbpS}AhF?-(yV&HMgAC8d-Rxvr7|~B> z%KtkP@(moUT)@Sq;OdILMH_3kBpp?(g4-^R`O{%#2V&jQsel|46nBX(o0@(QgGQB~ z4*2#fGn#Ndja*V5Ymo>3Rdki&X@()6wb#+*z_$WSyl)WqE@x{@N`=I~{q{CaVw7KPQ9 zbx~Lcd9g?W7EDMt4gKpS#CJKfR=N5E;!C;woa|0lH1CbsJ{$ZFbU+wPVJN-()eglI zw}Lnqb|}}3C#-}WhqFCaRP*BZ0p1CSXTv2?f|u`EcQ){C+qS8X=~9o?!Z*nHj0Z@u z^zofB`9dd23yqPXO4Tb_A-`s?lcMSt1!#Ubuh<#bNa3Pb%g>Hs%Gw>>#cxmjvDT(k z$x2Zmn*M{+*JnFT@x6KppQDTo0MM6PazSxa6uj?a=x>nT} zhnVE;{b_^();Ad*CVuQ=w&VKLzA{JIb3c)9 zMSi4=UiM(sal)ScFr~(5>F1FsOdQ-=#xBYQn{iyD)pcQSaCBIKO0Y6egWM-(R!rfrAFQ85uYDsETt&T9NO$DmkgYWtZ1nLN3}CxD=}-8(4|9#RvSTf-TW1)_EbcYDXpFZsez&10LofW>r&|6ZHC5|f&n~e8D}wP`sI%ojNYvm);WrOS+a$jCqjst2tXsDBBdkgl+7=cw zRT+4{z=w8579&XKeZdnpScVE1(a6Nk{Xfl!&@4r|=}?y4nhdrMFT$|t4F7DcFxOzr%1Yq%sxxSos&pFGCF0xd;aV>5ED->gh=9pMB-e>GumqS1q9aj znFgWW8qG+bVb%{hbOttFe#a0Bj{$&l82%1cRI1=bMx|WhRT{RL^@tFzFs6haijj8L z+tF{Qx)Wh2xzIEUY0-q-j>Tl&Elnh5GY<@eTbqjQ(YLom67yY-LDH^Jp|j5VBR%L^ zBYX&_wulJY2vCE?oXTTg<}|g585cPwBU8iQ4^C=kRjv#2kSe1on++DZCT^2M;y|i< z_U1PLb#c)i;%%lreqZp1DPFKjuTOozaohyF@&51X5VJ$Ry3KBm+3L+ilDIA1AZIcZ zCqBcWx~Ug639p;Lez^u0USDa_TC9O|-o)Y&)fWFmP-UT2J*OlttIAgPe12GS=xO{{ zBy2ac^peBn~2#xm)LI2Ff~MkapB5?gl>+@no>SFAWK?F*QlVpN9dh z0oHc#(cO`~;*;9`Y)ivP8ndWbFp_Pa{QliWHk^^R4CPxLxYc%CT`QPAzP`E|wp)To z=qV91|1Ji*j8==_H`UypH~Sy@6$ulWeII}1T?(nod=CLCvh)orF0rA9VPXEy=AyV% z`+t&>przKoqYJeKd|`$|CA^vtpUUz11p{k^rHcW-@1}x(HHE>UC=Or{9?(-Y7aXFt zJp);Z9v_pj5TL06a7ACh#jXw&$7-fAG#q~)-xN-95FcNItw03k&il$dm44JmwI?gl z^s7-HCAtALON}t7xPhD<*+ZJ2X8N@=Mu-&3Vu2djX*IkiO4OEhC|#fXzcC{g82`qS zSvh0{Pv)}(t)G_h`^AhvZG2@!??14be}K}2457XvzMJ~(u7!2nIkEs$92M)*3I2QH z*BjE}SAgj~a{LCAG`&dev<)nh@dBt7KL@kLx3`uRE~+9`cGmBFh3zxyYVY8chFw z>i>lznpK@KUGuhe+>j=X$HcT1+!RajAGjDP!Yiv#^wj^;gz7H@&4hI(o(|oU_}YBJ zv`O{fNbA2qHx(IxP#osg`QN}Z%IL_~NhN>tz58D)`%i!?FAf0M=%@xZ|AU+V4eF5+ zy#k$)RqjA#^MAznANNvp0D7h(6W#y1+y8h8@c@m)AR>dykAG)Y1pqqp<4B|a*Jk|( z6;gkv*FE9s-YNc%UH%Vd^Ium?Crk_=YJY(g_oV=_IR%ZX0!=}GVOHjK#VP@O8Gpex z0blw95ZgX{Y9aYwL_+Ex_qyVIXm`H9PysS%3P5cCb@YI?g-{{6yNeha#f%Xr=~nF5 zr4K|yJ8Tb8y)*h?JWx}u1gt0nx`@R6VjUkiREt3Sn~P?M@7TN^r^dzZxE?3=qu2pr~TsNoew~77gNKPCbu;;et3aG*&;Ccw(0mWAUy$Y&+9VlwRTap zGHon}4WK89YBZHSMp^KM42Mx0{*H?pIYMYMN8EVb&*x%xzGu5P6ze2VU#Z1@&pEWd z1uIW~V@}I9u6WvXpf4iTYv``BwJ%*0F5-X&E;SVl{Or{ypvP0=4}xw*1Hg zr|T+tRN&!UG2QN1dhANejZB-zPrmDgcq|&l!J~!C5laBz$TR3^R;iOR2>t$j>d|a) zGPmDa<&uC%N6m7s=zVPANOVxpH1R_tCO7owpO%b2E$7&9yiOju7pi=6N0PXg_(F=M zvt{CCrphd^RMojJ%K=XgfPGr+j|WhV2VtQMrJvLJc+08vOpNz9QR&R1W*CG(mGU~ z23sR?@>hJfM>2BY&o6ZeIH`|WOuvDg34*g&v$jB`K+ab;<%l@X(ES6Wll7)pFS44>Ipmq6NFAW2k`^SwGLCqG+S<(#0_ zt3QtIF8M?CNH`$G=S{vM*YkFL{tg$n{P^3=Shd|@>${wj@N!(NDUSaH5U289G zEOb*3!vWc@&K4Dw-9HT{_b!XdMmn;8dc)hk_7?LSG?a9)Ue{(yJ3rrQ2fb#^K$gFE zp70ZoO5+z{BlGcOQLrk=VzRaRV-Y$77YUUUl(jpY!B71S^1JQCN$T03fp9X3j6|Qh zKD=rf`_ekQYLkOFw>yilTKVVWlgY@i(wvO9o^{n9MFbhef0`XM((5#jc-P~zmkbA^ zq;6g9L*H;*DS@=wr*JW>=E|}WjnzV@f`5OHukbRcDphVviTk)YJQQDQd}*oRGRwX~>;UiG9U+H-914<^ne4w*??qXtLX6EfHZUKt( zb6%6>+u|K=5z}Aa-^F`BToa&WKf+uL7RXoZ&T2}1JaCg-=Y~Gbz1#&Q8KqgU9I^=5 zQpv{07#+?P98h(`6JV+88h+>*yX|kFYp@o8$L5bb7#d&0Uy-bh|126sBrd z4y{>dE}i4BaqHoHXig)E3N$>5=R7S(^FXg}EVgPIU5>eVwsbpQKb-n9(^yV5wlnws zcJ{nRJ=Qw>BYwBD^y9>1^`y3ArNw#1m1uQG^O*Wsg2|uX^Jdud)YlvyI2%GI?T4rq ztF6UyNlb&*A96A>W=qxMrxDDI@4m+q>(cF6``JQmo=2v)7#}Y>V-o2ow1ShA&boOY z6~VVF#dktpvu1?~R8r#^D|O5|eQwv;G;0T<{zC{vNcSJhKOcl17R(6(5 zhJ#^wkRe}gd^3~e%V*#r6}Mk}ZUrwHH$InFm^xF&RoCwzJe32@sgTZs#s`(e^(J4Q zyRAR~CXY(YXUsE__Rx5g34zQwp?p^MaRN2=2@Z8&qepTDZl+RQOLBcnl|TR6!SSxs z8U{lCs+go%yO%rGzVu0WQ}nma%8rNx@BtXf9Xy+uC!eau@~4GllxLNwPIhRo1$kV* z@J6@jwB1|%{GgO zljOh=H>&G;ov%6UroS-3#%e^wtH=;Mc0BTY-Wf{=k+i;$Tt*w+f`Vrh(s^|B!VwNM zw+~FLXKR}9#x4MSM!G-p_@hNRU6kO-1?}U!;OR7)7K723Aewh^iG>-&rubJ zs*JQnw|?nB%y-hhE8O5ewJFjVa|d6vfg1pe7w)8hH_?&E%$PX8;eH!sKn--w#z4Vb z96EZ$3xJdzwg#BlF)L#WMR#XJ8%zaN(NAgqhIa;_I3=;Oo?dqJ~k42;(5(+DLxmQ;>b$x#=6X5D0 zy?dW08tpQ032M#Z*WKRUFC^yFIQPjVcH54hH`y#0Eok5Sot*0QAki`Y2U^@lhbMcd@d8>o)H*3ayv$;4C1 zwL2*{EOd9QHaj#`PWv*BKZ;t&_$yARpB`@5C6HTh(D(2T=WHgLbKCIRAo7X8i0vpZ zK4AH#w*9a*4Kki9<2F@~?Tgp~x-owk z^yto=!TCFP2!X3E`84{Jx=lw@-0XHd;Gq}mY#GJBzh!CWZdbjH@_Yugo&XKQjjv}N z&W@K&!uNT5JPT|JgkW;tpn}MYiH&=Ke*>gQcBEW{)X2YX$Dk z=1Q-RIv|?)_EeF_)v^%wwx_~_fLJ~*8VH1Opx^PWDJiQ@yEy^fma2h0(hKzrt5-)r zYm@iA9J{_(Im#Z?#STH`z^uA!{0|D9RV=rOY~vh*l?Fx7emS!y0R7PWzVL(AHA&pB zr0*`mqD;1m^dct5sC@ts{zRAh3YXJzW}-LE9XM4bd2Sf{$D( zeUU_V0t%I_kFCpImnauqa4)+Gkq#8t*nD@oUK^T34w{NPBM91RH(lEBR!~Sd^s~Dy zwSDG?Hiq3JDg5&*qAXs$v=K|J8YnJ@+ow%Pqs}gei(OZBK&?pY8J}~(`Z9lgXKDi# zi&{a|E^Or<)oDKC_xHXmI1~H`@m_GiE^(aY)rj$1BVzn=rkPB2aY1(Ih)_h!&Ymyz zwCv3`%T>bM^lT_oGXrZo{z>NMN*y=Pl9th0)^UcoI%K%d^{NC_pw=_H=`#tUFFg}v z1sLC-Xg7mGW4A*{Bn;VqAUC=AF}LPS8w}6cu;_QPPyLiF6x(kk+lI!h+X&@Ss`vaXgyX0Y(=@m$OK!R{(;%1!%i3?5W3VZQp+CY zh3mhdjaDD;khZ4B+tu=)%J^=MVlO(MZie`81ZX2Zp>=)x+Rx}rOwP|&;l0mn^SOcL zEs4*qaMSuXaqT7tlT4?lS+j4NFqNmJEw0SFU0q#67K#wLW4TfMHuy(CQvy4fymeha zEvOnLo%HPNoMx%sh7*A4ml~W2uGZ~?etXQd&_TIwZPB>=q?jxMt3`<7Ebn~UxuAAWlZ4bpKRCd^}90FYl$s>=m zslONUWj!ygK^j$J{3@GJnQ5N<@L9DOtEDIe91||#hn^fC!Dy`wdk}u=pz9USaU!6B z-D*~};eN)%tL=f9=HhcQlR?dz>+&4$9G9L>TR6#NbFpuSK%c&E4@#iC+-Hj6t~}F(a0x` zr7xyLV(z^`cPi<+<-7_OUQCV=$E=HBS6&KvxM{hXP#{`L;uub<#>!V8XY_zZ@qpF- z=9?oThf_V6<=1|F5auu~&zS^w4r4RY9flI&D5c9TbG1LMy4hXFVY^hFSZ}n}?0mQk z%R#_KEV>c%;fvX?=)?^E=F%z6i7VFjRPh7IOp70gt>58c7cj7)I zTD@E!pC5F&Y8Vdn|6X|<-GaWh>UnOMG#ahS0Gw*dt(JTW8@$^rE~0sa`!nXrLhU#% z#o3tkYU}7qY#`YObgdt`>ThB5!zr2ew&jIeDYemB&+*yJNuTb)s&@Y5HRk7rc!IWR z3M|#kvk~t4AqL}n($D=ioYHIUZQ*cKZ}qR(aG?kN z?2(v>h0l#5Wd`;tn7YBUb`PrH^fCEeIBxi0y+x+d;6N5esz4bd*g@<*81u3b7gc59bTw1oyECe(wA8iw%s1r=TJ6aHZ=+Q&3VSe z?>x$gPE|z#>P+wZAAIMxJ@@jw45xYyltmwz=Yog>n9*wVyF{=Y@*TEiq&;rl#8dy~ zjWp}_SgiQ`^Yb5PF;S;G2Wr_kG*KjF5)rfR7F0T&=D0iO=WL&?ysm92vMTGl_gF&^ zV+!!z{*%G&y?qvad1040^w@Iu3Gxw-Ur_zCbR`GG5h3tFF7~3he>N|tE<)Y6! zpIixY$;M(2RdPqwbyt_xs~wM$hDB^|h$c6>)cm{Z-{9*(%jCvGN6-m}(nEcVfCCX< zF^VAF{rmp{uymcxFuxL_I6+v>^p>WLuV5{JLN=Z=Z*bWDJRLh-YNkF7Ax7WE?}P8$ zK9)YWq+gcCY8L5qJSWfNazKe-`F6NM`z$i6kUL2TVWg5-os_AF=pzMBc%bva%sJOp zV|!w3VIvcbGF*{h zSEs|dV-@6cxtXxmV*LGV4Kg2c50&GaB9(lNY0m>My}-;+IuLG4#54{n~}mp9Xj9ZnS~ zlW1+8;IrSs?|i{{S1fi2Y6@;U!Y&@aI#W2j3)+o{iT!4{w7Z;x^j;U=%@EVB&>&QNJg6l!M^;o|MVsDznCA#pAehtTwU+}POo0RU z96=BD+RvqT=1$~rI1cKm#{u5KWaCLDb*o4E+`p0x}T^Ean_WZ0;eq49oA!?hIkg6n=-9_Qv*il0R~A z`=F|6O7>Cvly0$&n|fa)9tC)|2URqLOjD(WNm}2MzyWW5w8ejdR+8E?F+}t86$wyav>GjxOu> z(@*G_R2ux&ijnu&0m?*4&?qX({K2%@SYLi$1Q5S1S{3tiUFO8SJ|hX(F#R^EUTMn! zusNaAZur$x1=(z>Z4tvSdrN_?SacLe7e9vJ;NSX0m@BR%+2se{1^tB=oQfn|gv2G* z`ZNY-x^maQlqLBzARRLNL!@)T;Ll+Sa~(ER5BI@@cH$0cXMGvZdEwnB8iYL7*rAzH zllp~7oS90tJv>U`7wx&=0P2V%=rf6~=#S7L6$PxsN&2%u|8TSmtGjQ zyvcN*JT(W>+g%YDr2UD=_ok+tYzO7~dx+5I57hE0hT5~o0*_-!#rzXIydcAda6I0z z5yQ|Vk9MdQo~TWSEp#C{hTa7cFDY71j_BfJcL{Or&t~cXhQpEX#H5WRgC6{VJ(qC; zycn6fp+Gmv$sX133`!Cd)YJ^XGPE*;LT2;Bz%tz|katjOGl->7vgNc}B`^Q{ldbO# zOu&(n9K_5${^yr$gfbe5Sntok4aOD^1k>RR`39rw^-EK1+Zj%Czp=~)q3>@Q%GwNI z*n-6k?QrzAwQ7u$izO>Me7-edBP-_S@k91zwLUjL_m#r+o$3cjz~j^CIMep*t-VZT zPi@yJt4xfr1TXwb3m`r9e$XzjR_$*I>EOL_Ooj&fL##}TTXbm5aRvT(9xo42&F_8j zaeCkRZB}iwzw1jl0`+%Yd|OAg%_Cs|I?0>5QqLsIe1q=_$66)Y&^7Q)fBVWzLktf{ z>g-InRto#R^bR12_eEst8t%U3?$^WzyeKr;?~xM4*NQV$qf@yeQmb9s;Aw>nmQii=V2E5Vrt9(ah6y)x z02Pu9Mtc#Mgi#fg2?`=0Px3pCE>UApkmPMzR067WESDJ8leZJN`$3V$T&`L27u)I? zr!(x{l?n8V7A%%CrxL1F+TV z0?%7Wu8|rcHDMcsxO&gqiEAlbv{GlsXQynP=N%)xZ_iXdR~zO_R&B1m;95bc3SN$2 z@o@KncqaQQFU~~Kl2+|#3vS)m%$3H|kVroChMF(rG~EeW{~mVuT)uIOCahf^#a?vb z%tJuvF`nH!lEf^#x!$Pz*6g#R-*Z=e&@_Ted`EJc5Z))l^&M-N;Idz`38@-dzw0e7 z)mXECjdJQ(1%Q{F#`F|B;N~-sA{ia%msZoJ4pzRJT>1PFgszz9+WqruSO+fTPEVV} zP~6}mOgs!5FIV@;1H2q-XsxoDv5j$%-+njOVtOlhK!!U5ys>P`XM{e37a;A$bZC6K zTCri3R^7D81|}sUva$jlU+h^*$6s3c-^}{ggrWo_oW4C z@TUOc$4rKg>1IF#>Tta7Um(Z1<{;VWu;|qm@_QpqwNReb{>)mGc&RMQwzs;GqtJTJ zj{FaX(atao5)QX3p3X$(2~H8Qi>bqBaAnlpjC05Pn=4r|5d(cj;#pQRg2y;%sg{aI9WBpQvpJyzQuqy5J#vo5L$z zNq*Iw))TGn(pzS^5@Jp8oprYLA^9xUC7&oL#DOwh_!=x$y5&T^J=P!c?#ccaP#5o4 zd7V+pzgeP>QHZBNK3k&35!`Z)RdA5!sp?!4R#7kpgVS2Lnm&>oukN9(o)AgodD{%y zw?Fyd5NWgspV>i6wob|LEvK8UtM*j;g>wXaAONgiXz zF5DB<91$IB`pyX*n~nTW2&B5i_}uOIT!Reqb46vu=jySkilzb;F2F0Tc$&Ix>yn<@ z)T=L3`LePA4!6CloXdBee?5eNG6^aA(Grmd%NwsjiH7w0eRu3_Oxz6T07%I@*gMI6 zz8L53Lt$l&fmWridTF60sOcdC?nyWBrn+(G#pkRZp$RYoYt619B@Y{ZN=-_3j+^pe z7&lOEq|*VPg|gkBm2poxM#B=lAom0ZsZ#;(?c2~zmYpcUsXuQ|_@+yUw!K1*2V?F} zmNr{`z1ueDbvQLF=SwV1_fW4R?@G*C$NinR-kdu#n8Lk83qfND>7>Yq%n=J5eQybq z4XN<68ROw(j}9_QWn!;b+7U+MmnDbkC~AE3X?1WVgq%CVT>~vaJ?p6}vfoE$3sWeU zy6=XGE6%kuzWJ5cIL4tdH^ZWMWf!bGvt{hQZ$R+Ph~A@$?S zm40WjQK$chUWfAU020=w4FsD~6tcX;F)Z7e=}))yog!S^%k(%QFb4&Q0M=&hsK#+8 zjH@$~5ouphaTSUitOkA=+^1nqJRd34Zu~u3Gbge$riXckeN@Tt#Za*ZjGgw`ui59- zC}JgHk6m47zxXJOY2s7o#kmtgeUNAe9?5)c`t=`3Q7+mCk6y@D62F@T!;pulNAC6! zig*Vis|rs`?iW#$i#jW_&(~tbB0n<+^kIf z^+Qw(6&Hqg;B(ePQTY4cN4`J&eBdpMtxfJnEk!uaondRXb4aZvlmU9{M=Q$u3R^3U zj+-DlHs6E=;Yf4@yfY;2Og1qwy`IPVPFCHZmfv%hK)BbC2K7q@_}zeG|;}f?E7hL@MZ1*eO6zVk%00s~uiqTjM%+@Y0 zXSo``6d^;en>gk|Cu9njmRmbpCrNq1cKeTn6;|}StD3Dn^Op#>hOVuxr1SoYZ=^At zuuvX-X2V0cw<&6Z5V5j^Unk$6J^V9ekp5Pmmvsp@{@(z^e{QXZx&nNF$P>Us49d8Ne0;mX1CT=aotW>t|MCdHKX!!y2U&NlS6BVd1jPUPMEpYEGGBlBA)x+& zav(*P1r7qA-`rF6PwBI7JOF+`{DMOG?w<}~`zEO4SfNe)CqW(I##@2-Jo?@H|C*!# zlILi^LAv&>D@*?Atf9yOXD!qR3iki_z{_a-0G)?q&-_P!J%G&ya*Cu7vGV-rw3^=} z<0&&@E2?RLmm@;%|00wcD*FpaTI_>MtN;G}CI$Aq-#W|guKGT}l)~tsn(DB>T3!w` z&O~(G?`uD27QcZ3mBJozV?p5<#=B9VD(`1#jr`ZKO#aM`rxd{pIORfZg~25HL*@XT z<{R-*ZO%N`%HK>sj=p7@H83*$E6HWuwhs+ti$Kz;JTsfWPg^$Gy!d(t0GR}DodX#U ztn&D&6cCVsy1fH}O2Cd-v^D9R2%`<48cIln{`3Tb{};EAHU{FSz?rpC#xtn{5T99Y zv!AosAuJY)-`XDB8^$~Vw+kiEgbL(&Tn;Z8@0 zU7Jl12a$dKD)~n?jSOJCFFD9fMsb1KC%i_hJIHbc+nx6~RwA%;+2QuM%5L+=y~T21 zMXSBfwGz(0@hy8XNY<}&$Wt0$`a9=IBX^K=!S+ob#}nnVlh{i zTikC}m0rC*QZr_6-5g0{x5sz^dh|7j{rkGNwYFpjuzK5Tb!VjV&8jkjAS5HvHwXdG z9n^ArqP18t6!#ncViqeaq{d>7T*U1{AN5R(kc~~$kh=G7>9`Vab;{yF_qejO;J`q* z?3hAj^D9=XR8^>Uo>Nu5_DXA9{H%>_W0i4>i`@#0pSj9ru0l@mWWL;3%n}Y?mBcxD z&AzVtdE@8ZO;9JRmq%KiLUUbdtBY~NHNagI3iiyn#jStrgD^IIV+-tvz)coFWb^xY z2j66tHy%~Vu`UPN_HZo81Z%x1SpBPVsHzSr`2?T!nVc^^P$}uxae3*cY>?x>Y&lx2 z$rZEdFeTUo~tQ+xZ0a$CX%2C(!{IQ@((UmQJ(<)OjCdEIZ(+aQ_2Kg6 zn^=;9O*o$Y$uegg%**_heAdKaqt_#%ggD1BcgarJ>iDB{>9%J6ZUhFfj-M= zaMfpQ>e0QQwC7Q$P_YD1p^0D@q9BirP38?Hc=@yK6{oseZWqnifAi=~?{Zm|AaP2f zK796t>F-x)rx`Mt>#X3REKwq z%h}cS7f0OB<1Vk`>m>FHt#z^a^%OsE8MVh+pl(c`U%`(AsqmJ|4SqoU6mPP{O$q3e z{8~``#+xWFmM+UdGNe{=b@q)4;D078^wq(Y4oT<4H+jr=zeAdenhw5=>i}t%HEOu} zSb<+~0!V%#D}_pMqH2piYreRAp3@+?XZP9fK z?~}f)wIE^7up2F85@_LPB{B^Rc*zEH1$(O|&Cd)D^^23fFSGE6XHJ8rmA*x!U zemtutznevyAbh~-~;=6x|;&We}Rg7811KcB{Km25ymlm@k z#88a>F;fk?k>Q|}We_H(_eO&*{HT{KAh#;o*0sFopsWNM)djJ z<*jg{4SxVCOfmraS~`Dix^PM+W0aqxO%|kCG>4(ut2Gl``ibWw92C)-W%$D+v)6u^ zE$Y9)g@>n;Yd7fWJ|J@IUPJFizSaBfjc5$~Tl*fJ=j(nfiO)Gh|H9TC$J=h~&qg0p ze6Z8u+@aXZ@0N$t-5GHOah!qt8q^W89PgKZwSf0i9@d^;lVQ*a=nx7;K5*ZZ!%a3P z73Wu|4DI|7AG9cMC&S~eQ!+A;aeF$?A2EBq9($}Y$xD1pmlw!zcTXA?M3+t@5yk{hb;?(1rjo)`7)iboP7X0N$t8o-qe(6>h= z0_DWS5CZoO%!i}eGIL;glY0Dpg#|W6 zm@wTV;dTFwgwI2o<3m8fuFc+GeghADo;cYQgb*w| zF2ckj?Vz2evizQFvR`5qkg3(mNDIXDw>Vu+O2A`Bf4n(t*Rm8Gtya$4qxlm!9kg!~ zPhE-s7Pb+kwFQ77gk5f}@kVxLOH)%;S4CgRR+^j{dgB3t13seJ)#TEG1NM{)0@?*r zoe*FolYlNg&ZZs!4CIWo{{#LNB^Y(04?}bJ9)9zDiM>!*slgS9SO0BR* zr1G5XST5|SUacmp*?2|vfe;RmFdZpsHhz(n#>~}c5Yj3#=!TxGRN_Vo00$r~lQ+N{_BHT)}|eR;ZQl{q>$sgF7Y;M?iZMewWZHe0%TCi^+?RC^YRDz zh{sE*h!fHL_d+M_r*vqL)7bc+?aHPEtEBa`$KJzo*tX5Ret|SQz#j;B_*fdRXK}Xu zUIIRk+LH9# zWtR{$Nceyhy3zT05|k%ug_VMyJ9*vxq>fY)of(1~uqonDFexPfc&nqLdiwnWKKPEv zFj!J+5mHIi6dZu(v!e6duj<&W&nDz~n4dbq&s(x?mhd>e80e1XG5ET#WsU#!xon>V zko7T3b~$wG#^1Rn&EXkhb$r5uu66<6mm}cZ0LTOZxnAq^NGdB^ZwrApE)}pIuqV7y z)AQ4_uX+Y}o^`%9+Mx2Knr`Lv9rj>k%pjTax^ypt`YS5{*Q43epM?FO#T;Fqm%-cq z9J#?HJB8}9#$)YVS0wFK7o_)KU z4EJMRY_9BBhW=8|N9N0w5EAaV7B7AgW`Hi)abBQ-J7u`ZTF*CSq!-F^8lg$mLbHYnoVpEd0x;rcMeAO%wL^8`lAfl%%6E17* z{R;&IJ!2KTWBIt6N=62sep6g8t@C3g9hRBr7G(dt*dU{hfY^y->Fs8Q-ri|aEYd$l*!~%I-P;Ne)>vV-ohizi+(Z0|9YLQ z@I%l2T*GWJH>{BTE=sG&?6z6bWBgoJ*GI?aSVoT?9)p&ZA|Td#BX_@Ot%mh9d&&c` zx9x-ZlP-{%mw+e9RFW6#*fblShx@Y*1jREH^PN9o@f#GeSjx{~UbfxeP+Wrb*r6i- zRI}`Gz85`X;^7X~<{n2@Z#M`yHxvr6Z*!mVd{~&<(QrFipxE|%JWcR8@f!EM&F1ED zF5E=Xm2*|~+D8C@jSE8Vtwljix#M4RtFB8t<)+Bp+(E!KE%=TEhe7ilUKp9==Qujh z64oiwbfv58Py)4;6*J-{Jk6w#b73Q2O6ot!a&c&R4DFCSAY%;8-TUi}L-4&sN z&-SPOU~NiT_WYb|l)|`f08^w*H`Ke7+t?)JZrMy}1~lumPqqW=FT*H=(zM%pgZt!) z>=+D&zF}!;d>lr{5<@)l5($q%jYWjKD2!@LS!{d~8YYx(T7W?-;T7f-qjJ z9F=PVtTDO1Ey>ZRJ>MBEk)AzyZLP!1xc!b^NNgA2V_kD*bV8e4$MR)FdO(f{rNro> z(eZLpGx9~q^ERLO-56D&p?^i~I^iM@x!fDgPU2C2XP(sAjR+#Bt45!T+(pMbidlSU z&0sXDV=>$K*ghf9dZa&ItWw;x(5`OOxN6(2-+n9;C;r;udhvi0{m7zd^c^$8ty}C# zpQn-$7sH7m2&ji3i;rF9HFw=b$?jycBQSB9`eRQ3@ufd((E&S-Ag($_rwI*EMz}kscQ$|0%H;j@EgIS>?`LIi#e^d7@%UODqbYC)Fh&f* zfC?_XI+|EPVEuNcz_ZnvYo90CJN_O>dq>^@UJeN|6@-#CVZIysxfj46QhU2ofmj_; z+3fX_!Dkhs*W}(Gr^x3DAt+gt#$ATaYZ3#n*Ox>5W?~(P6Dx(nofA_b1`g`Mg4XTs zT7n9G9ecsg7+0L4%l?4)MFz=OTAGuU=9Gzdy4Ayp+3t}67f2lWKt%InuA5s3D7ik( z8+5QeCzdWnO!|VYw;%%u-K&TW*)7??*le`kDAL&5yhW1Zlg89C8iNT=Sd$=t*-q{d zd%X<<&xLbc-8kRM*5)*QxP}ZKA*zEvr~pKDjkg8{%dT?_T$ssQ8u z4JVMRc1V_HRaM_2=av2pvz!WnxLB1Ju=ZSj;cC5e#Zx!RSlM|Fe|F7h)W+`)XYhpc zH;{96wj@CZZzyQe%M+))`;qj#arrtRM$kq*uJ=beg?uQfd^!p%De8PTb8|YJtdobm z{pBX){709|kLv^0Kf-6{efYDoDF?l4C?6$o;h%SHW{lQEAdpa5^r=W=G@-K4pSIGrBUDa{a<8E zJ7fIDN!~s{tdwr0$z^VW$u7+CtKK$3-N@`==Vad83$VvcaRHjwOn+WF+_)W0Eo7sK zviqV59B?ixP`2xxMqSK1s{ET^M0s7Swnr0|FjvM;eDif0tCG0G?V{K9-*uiXxL&)5 z&!T*71I)=Ee=zgR$EYaIops3jQkD5O_!fDUXiEL&cVykC9^FhIOqNuO>WnbL6ZjObOU?-W&wv3bc^ zny?&fS1rDzWrOHl$*CkL1dw?7hXHMd`HlXK!r}e~HB|(qY-b;||If+64KgVik>~~( zq-CG_d;N}WzB#fFU4P)O!ZUCwA!Ip!m#wS9b>2W8fVQ8KDfU9Bk_n?w5%}(WNozfW z=vk3L8EU|?%Y5GG4@!De5`8DiP2J#Q+gfk08)(!zZ#vA#-z3@&<_#c z{d#6RB*Ox8y|-YF0KpRM?+PmpDe@tMzbJ_uH6I2A0eUpq)nvwYm$olTj}Iw$lNkBy zpcI_!yAq!#vv{n7S%5Dy%B>aaZ2sf08?}?7-fZ29IME;5FMoL6QUHNR(B6$-c0ILi zAmG`zqCV;PzM5%+e?tkt4;2`+5x`|He|u}L>Is(S%2&6*$3^x(6H&78mSR%+`)w#XN0dFAFiD9kTiXtGQV+TEzhR(7l zT??dT6r6nWuGwirSrZ*7elYz(G_nH!jYRTLG=%(`}!=a z;0A~6X*o!ct<2E2+uw+Q~5ER=}2v6JzZhb_}+k^({ zHXWH6`yep({ucl_`2iC`$oAv;q4rv5h^myO1R;1PCkH7-e@(b~Zd+W=9_ z@d7o_-GT1bV1shkcOKU*nk#J>?wEtiVZ&X<#_x+^Pmng3mQFp_Ql#e|ZclTNj^G+^ z+d_uha6)uVHfTA}9xi8v*Bq|4IJb1LB~= z`GY+6i%hcZ;Q=Oxuzm_`ei&hW>>NZh8Rgh$S1^?(qUP@2|qDN^!Pyl4KWXkS;A!{7I_z>OFcLN~M4 zSL3+#`(1x&q|{O&Rvs4;`G7zOIf5y}(Kj2#>XDqHC6W$NbcSz)Qc9~$_NI{#4 zjXZKTvsJ#<;qRCXlU+*{F6AfWbdQEBNO05Onh*XcOPL)x$WC=I^u}$*hhKa`Y`NiF zkgk6w#Uy|V z#Kt}CBRb7@?dqWeG3ZtfRmA->by!1o=~R{;Jd?4LOh$)4y{CJo8)9pl3ZEm>cO`EO-(s9D; zZoq9y^qeYyaW~7-DtYJEt11rm)f7s-Yo$m=&IyqfV$`Cj5P-z5)SKOR{u?i7+#>U- z8s@4yYH)V&wc6#&nU(ecK_&xpHJ)flN(xq5jl@zj^`9$x?fZ{&HB`&BYE4YH_H%4z zaeoYCv46}j+W)T29Z>f)$+%A9Y2*P!j3?*}75q+X` zxh^uPLU(l{Mj$MNI8~z@JY(~B!H*b}r3=cP1*zWnoId*u9g^ss>(a%mw`5roYu8zW zxV|9P_nFjLo+6&poZ&~q$YZjxTg=KYP2;gSaPnNO$%L;W-;#W`hFyHTzr+YxH{DzN z%s!^=`W-Z-@yw8dD7wF?G~8uxt7<=mG7pqfWnAs%bbPALN-eo2)CfMI2#euV(-Fha zBM?5al&f?e>OLm1P>}ppGqKg<;VNvNu7>8U%kt3wb4-`I3TS)S{W~NgT?djGbG8YB z!S6R=7j?ES@wlyj@5`7*Ggfmb`mb##8VJKIe$#W2pV>b=?rA2S#xOv@`^ z_Sxb0n?!~&A-|7RFGNvcu51qW;@8&e4*taQbnUWvfcp*+wL3UtCaiU_|Lz=Z*bE}z z-KIB4f4vx%tmmlOl;EWGmVVZStN6hbf8;WJ!Omb_=!7>Hq+Y^#^YQa=dBCjOF!B!W zrr-Ll`k-%-InB4Goc&b3clO_WO!jK_q&ne|pn0!U80^$LAwF0db_2B~4Iomv5}b9c zv{YAml)~w>h)^E)A&SvzQ5x%a9F)P$l0L+yNPg7u2uF&9kB@YO)Ar9kDaXxT8C8ld zHw?Nno?DEA+OE6cR}@SE|7-RDyrqRY%lnUb{Wes9U0nN;ajc#}&s9p{EkgdZUnUlX z8wbd)4!r88BJL7Y=0rR~Jno0eMSHluXM%OeK!~Y8ZnQ!D@ScnTXm>$WSE86~OI(y4xIMqo%>v#YFMbVnqcB zW!=$f)d>k_r_RH}>CzX7;LyWf(0V=%>@s~bG3k1de+L*1DEMDD?yH-J77-tUopxDI z1GQr0zeoHsB_x+`)ouWTa?k;We2(aEljTaU~k~d8ttg#gT>_-iU*A+ zJUsbL0{4AHULJgv#ad%9H*egWpc{0%&hu-!2rJcWJgFy*r(2xZRzfdaC z{YOerCpNPMm@Bv4X(wx#&02Zv!Xq1XOVhFCAe}^S#vPTI0sAdn;oMniGHUpeD}UFB z?maBaeKNNSMbZqL6C8ohJo`)Fb(I9^z{Xu7q5TtE9CjU1I&h>}7&MBbFIE_*=lZ$7 z+CDW3e^sU2Ero5TZ`mP)4G2lHN*F{3B*uV)>snU?2c;q5%mu@J5ps;LA3o?bnE}S% zgFs>pNZ7Em9=Wyrv4mU|7l6~xGTu7089ZCREyNfIxLNJ|j&<5{>31%8LKzq=DDS*o z#G1M~HwAY+gC%<(IMr>89NB}v1{!IP`f)$ z2}RtlF>{z^E($uR@j5QRv`puq>V-|3Rua?-_q0m>oC8FP*{FJfT|pNYm8jwvqxPd( zv1(6*iPE3ZP$jYmPpicmMH7FdY2jlj5JyEl&;6X(u?CCtkt_3FE0bZa`^5?gQeh<= zDqiahf_tbpb{TXtTd7uxAi~3NJXNJn4sq%n&ZA&oI;%z5+7WZ@Qy>h>ixcueobRW+IL-l55l#NjR&-F$ZE$G3_br>;`1O5q zguo1pm8WOA^HaCDM2#tUxGk8y7daRac09eu!I-KHqR=UxS9MDBIs8NFu$o@_dz_ zbiGVniTYU~db;mV=rUurxlb`I!xR{c^SS?$5jkxQ?x)<55IyGptqQY>+h|U_JK4Ou zxm_9cSWXhZYa^jR7TR2=+BgLZmEmkPOBZ#1dJS8$J}w6nPB8g&qBg&)c`pooTdOUs zb_#!APq4f_)gaqGg%B~8l|RQeYc#}d64u9 z&D|@BPG`ge-)(U%au*?;tfow08wLS4w~_9402d^~&Pm4Um?&WNZWiVM?Y@rsEF!!} zVdckiueaYKnvYktejl8p^}2q9gWm*gp%;bETzt$Wd@nW8bp;Jv1Kiz|PByhbrc-_6 zi5F^fy5>V*SgW`DUXVBYcQnE(= zg^XwyQ3Z^oIah@l<9AGsDyAR|g7Qa!orcaTh5J4%?DgUZeQVdXqS|=4!vqIK;@gtf zhFW$rt3!WP_~~%2FBPh{mA8~0#vHPOZ@f93w*aRFWCK44~Whn{(U?}biYi|*AleU2sxFxzu|xq znb7aw=~S}BH`O=w`_Ugbs_7*WP^i)!?CE@+=(aHbL1>5zLP6wx5EsnvSNhkVRiKFz z++b@E{R8jv_8zO=Xq1KVD#L$(Jb(l8zabi9@vQ#@iu}J9`7h{}|37}PES;=B2ID0L z6C$P0yqEuv!At*%Ln1sz=}*3);-u5~m{sv^m!D1U zt-44T8XeS1uAhV;tRe0%wiAHpseme3{Kr^E@v(8uXBu?@=&Ms|J+Eg_LpeS0T*zd( zhkW{QbG$h3fbj5nNms9nYLnY3G5(zUuge!83LFb8+>$Awiob}wpgCR7Vx;FD&4HNb z`Elyj@w%nnIJt6xJa}|}a(k%CYsq<`fD7|cRpKQ@-7j3A=i_#+bjq(XR>tth8$6)J zFIQ2H9uO!s_m?M=;;eF_`Al(ep0^7b1MbW7 zc6+0m1jiCa4V)1gFK!_ypdAScr(eqK%o3&FbyFF&w%)b*X*w&x}u9^H-S~2>d z94OtprWZkACs-Qog*WuHI>RLy1Glj7FuOt$*)kcqsxShOFIywx-i( zkMhU|d>5=Med%KD#$SBj=myyES{GV9%2~na(;IDWmtO!KU1HROZsm4&Nc0^qR|KW& zS^|xRP;{l9YMHk2_V3?CLlZe-rLrbsjzUw1p-DL``BF);K=0Gq?WPd}L~O?w{+X(T zsLhy64$K3j35m=5i133`W|AF_7gbt=2c1h2sMU)7$m6D!$4Y^E`QX8f#c;>I@=u4J z>zx6RR-M%<-Q&%%3)}f6IWe+IcaFB#;|h{p%i&ojW!|d@>`QfD6sY1=;^At4`)I)d zH9MyaPzcq$Zs!g#Z*TO3tJcN1v*9Ujmu6w7+3yaJY1N-pT%WL0iU2HsY#fDL%7znw z5-s)V1P@ihJv|MnTkuzWxjniZ<0YeSJ4;9KSwpcA3_KG%fY0y<|3X_W=lYw~W>%S( z!C|ne_VD{-ln&T~AegpB!DK8$jehI8^2E+2uh7=u$wsL~jrFR|Z_yhs26!fQo69E2 zB*NgF&zJCiPli!-VOueY^pIy+9u)bez-pjW&7aX_i&AqXT7rc!xzNqPzMrY z0R7GM4tUYHJpq-jDmSafd=s-#7~5u51?w%D?-?R8ufC>XWSWcA%2 zS^%`?{9@zrl1qwgh&{kK$>Z5XUzfL_YwP)95I#8U`80m;Y71q`+nxsclxRjcxLg1> zm+x-+{_qxfud~j6-dftn_zcs{nVYh&^wSRZgBa8AojKwOg^qwo{K?dYJgZo9aelA9lK-1sYT+_4=8`H*#Y7&o>koTWbT`{Im5&eg^3>J@(y zW?1lkD~{fBW!j86PiIMWved%lUM0ayuM>7^N{-inonh2?&A08M+=aj9(sZt@wj&SM zUeo3~<$;8jww-)D*l;y8WvxRT(^Mv~$w(et<#X}!PLk93hG=v`o0zA(XJKVe_( zzVMKwh#7V8zgz7R-%WgZb}vw14~VDInbZZsl~u3C89a?6^Ra@hT(apLWWWfhU!su* z1Hg0v8$e|~+ldc*cS?9vD010bgZNXzbq2g_FQ3-fr?6h(0I~INeD-!%5A1z?=3s$;;i)qps z5R^YgI_@tkK$~&{BPu^!mhre<uybeK}n9dd6R8wNGY|E{-W#(aKou8+~JNu$+bi@k?WG}R0=y+k~i zsSf8uVp{Yru!zH^w~f9BTfol{dMuSbAJ=y>BG0y&O?^qZef$lT8Rzf`*kQ44W>5W* z1F><>3Md>rlDUb%5&6bwiANnR95XXjMesZh$JHI}F_MV3(6KtkRH}`g*uS2MwFLY$ z)iM{IiGd+-SH^cYZD_e1^{6tx)^yvvMw!JokcZ{!&fxU`nlhrM$rcfaA3z&3DD;cG zySOHR7oN#}i~nezGMo(o@O+^&qjs&`cEOj=b0H*)irL6~G0MQ}xZWv0cCy+rv_}JE z2U*PXc@Es%T#ktS^hOktIl)fHuOaWT4`_nP7FMy@t@X{q`JRu4IC^<-TZDox60KpM z;jY-T96GWvq=B^KDxrOmbE)PHj|fhn$@0_~55eqB4?)Ek->c+T)sn2fGBfB_0WTU% zVBF!JxeANT>^E$7b4n6DpCM@V=h{+sV%&w*Yg?T`ig&!D_2nNRaiD1lX;bx>VSd_g z6CZJd0UVjGdUv?~6n5Wg^6zFP;tE=DGlBOR2in+xQ104VrV+Ii!R97exY@y zR?6d0D*}&Q;t>naig1a_W2;}m!om*u>J`;qmeS|p3TrB|lHYNKbx}{d<@<*=MIl09 z&jlN^EOOSt>HXBex`swB?QFwjgpV((=Y3sT+9%E$FDv$iR<|g)e(V_f?of|lMs|^~ z9l8~X@2(%gTHX|g)ts>34@NGXO6iYQC>}m${|87ioS)}DqR(TTfxPmP1zk(UYA(lT zwBw^cq}%vEcX{uXbT=b2UDUQ8j;xZ-y6MMz4@?>ZQ!=?5jmO^&;oObYud;T7xoz61 z0JX0Qed50Tl2riB>Y`CJzp$y{G<6w ztygaNR$T$$n;y5!XBmTi#5UxAm?H|}uB{D$;SmSWQ|md^898xlwoh(;eQ1LQxo}9R zm{)^I(JV&bVU5cpU=je7sJH2`)ZOa$shyEJw0KF(tKR6$TwPY?k-aFkJ-yQwvPc~E zn!0>}m!}=PP1g;5?g+PPlQ9+Hqq)WP0i$eZpfdA|z$XF=Mfy%7ZwFU94p22IjHwfK!kt3%>K0+^Yk%FSMn;=k8h!?9CsdTsVpN1_(1 zb#os2bfnyV0$lLW>I?!C*|EFrYPD7>S$wlgr!DlriqqC| z*2R2Kde?NFQqj$P(6L@_l` zj|uNun97h4dlx@My2Gnj0@Fp8Xrn==kosMLVXF8f(?AEF-6(yta<>dt)jIG*NJE5! z8j6T~D6y1btdY%1bBT+xvB55pF4C&=a4hHE5{5)P;THjJ?{#D_4kWwMNZf)cfV0i<<(VlbB{|DrR5Gn zL^tMIp&swxepW`*q1~cz0rRV{Z=57rp?;d1>aTFITGgQT)&t{u3k(c0u{MC%rRBW> z5KP7DW1bzs!uOGP1|Hdq>>bFW7)gOAlLPGv+c4Ub=?Z99SME4|&ZYa58`yZ@0YCD- z{2*(+Rk2Z#48C1N3|H3ylVdo8wz*?G+8N^YP_YO`fWlXyIH-;_=7DP-2J~a`3aj!R z@ilDA-Rtirl#9#GPE_skK_B(>#(Kg5Ex+r+^+N-{LU${!%}hZOU(yY(O!yGFaE&@H%g35*a$MHL=^o|Aj*@@r^!V`5AE-MUMw)+EiCJ0nf_p;ODH&q#REyL_SI&B zoDw7ZSxiibmm;4hT_Q^?Q@Ok&p?G%DEpf8O5KuV-+hG=2uwsQZ2@Hh&*!@S*1mH2| zH&J;S9t9|9_pjEY4$BJqOfytRs|FF1JlTI%^&0>-q3virp*& z7jsYFu{0;REecOF4dN!v{aG;f9DOT52#d#K+|HJD7yKKz=&QS9Nfq|5Fyb?uF^pUGnig!{s zt=rwY8eym>8eKQa$0Pym>~z4NxIC!)u>ykc2g=bd^(whG<5X;7V%&Pims^EO2gbW_ z&pkg&=ax8W^cvO%$ikNS7?6oo#8a5^Dh9R)UOnfyMgm}NF{Gai?4O|r3n60|$VU@wi)GALSYQN(y2iTR5aOZDUFT_4leP zb!%Hp{7p|i-R&(1Fw=4!qYM48f10}3BSEyeB8ZU{O`M0IXm1d+a6TxN&s9$HcJIW` zZjU1T)?gL~{pHt5$)<@dL}jT&=a79ds(>pDS3eHB)z5CopKD|M(bX72DkxPCtFO;{ zv>`bCjL9PLR5^otK^!^LI?4+p*@MMsy&u`Fhx>Q>w+5!9iGpwOtVvg6ZS!$IZ~pBP8K-1#|XiF6e-j9up6=~a}Cq;XRLX$R`{_1JkAY z(~8{d*jSW^O{vJ zcFTw_Eyy=NQ>=hd>edDos`F0ewX-qvKu>H)P49Kidlx|8HJd)PCj}(T!7Q{DxDuAPWy4(Ao@Hj>_F^K>o!ku; zYEnagPo@CM93EKY<4K`m>KA4Aki4$WP2N-czVC|F=Yl(#=9ka@WBYp#{B(o1)6 z^dYu!2Oj8IKlS~Mc~GC9!#WbEQLU((@xrBx15mj+?OXYD`?4RlnZfl&y^A8@N&CL9 zbsQp$9gkCcdbH8H7&1YDXWBl{yjv7OrH0Gpah8)rdzC<5FG5AHV0Y^%UQOwR!+&6d zFw{FcP?XSi2xIZ3u zXJkJ}8#ehIbuc(ideQ7G+wpTdxPuE{T%uJdV$pB|+7)A44T+IQJY69J04h>%%N6sg zwi|`I+ynaorKT3x>s0tKB16&Ui#Q;nzRuPW;iX-2orB(Iw+65~T<(k$y}R~ioe zgs%xKuX`I6cxhZxUGl$u{44IrBXKmgU>3uS4pwkbP>6t6xz*BO0Fm-N?}Yxb6*MJE zr($p@vw5W~JcgIWe9or?Ar$x?NU5-pq2ki9*+eBy&OFm{c2&F1vwS86Ltha>K|o`i_1f|Ming=2$?TTKecEtj&Ef)p6`I zG+TP{0zdA5j1RUWOB2h(DRwqy-seLx7v_7oEW6SQrt6w!7xChyvj3qP9 zCfBOXV6MX^p7V$33%#f5?na{*2XQGl4{r~tExEMaQ}`|?>4&6H&Xs87!&&aSigjgo z{2nn~`jh9rz`}N#x1UgEO-hq%c`dpkr#^=Hv7_X<^7P|r*YoT?o00tl%MY& zN)}$m2(8K@?O=EJ@EM|F#RvJ}js=pf|v zoHko(iI;(toI8Xf-31u{HV2K&PK3xq)9=CW!4hTy)z}qF49uAJy9*pG=R^5%s*k4F zzCbv~9A{SY6p7JhQznhapy8xy9*YYhY8?TJ2x21vg@m@I-svzg5#qyk1s=4J7})a= zukkj6e~Ht`_WEjH30t#9fbaeaK0kDIOy8<}6|aG50rqcxwZ3na`J0$wkuel4Wh_d9bBRS)r z_4?jXFA3-Kh3ta8ZuxO3zO$=?POJ)6^;S-p;50<-)hOrIOPGoJOcrv{`U^79K78D& z4#w&=(d+C-_#x=D%Y9CZc5o{gz#{vW*V!R5zl{$s30h=sp~;NyoXE ztA7l`vF7a*uN?w^BN$34noPA$=G#KnkiUG3e!^bD@BNfJN4SnVv)UTW6B0GK6)&|% z5fO?h@%$8iI&ukjT+2NtNqKENzN_VYjhF{Tj*PCN)XFgQFO~w`|GchYr^RxvE^F9>}0=O zJ&gBf#X$%V_1T4^(;7+)$Yn@3ZQ$)aO-(-Bo!I!?Qs0ExwAcJ)NS`~Pw|w{d_?$ZS zH}kG}E(#Bxwmzx-;i)tr%iSe4cc{T%SFAqJSq?xJU11 zZU&Z9VlL`c?z4nAt)JtsyTxGnll{8`@qV`lYMhc6fjRU+w@2`d^w0svux58h;^m#`EunwdWgo zEk%meM)C^ZWfjNuaIqmpf{B`h(2i8xat_6?0Il0!3T33*&>C(1gnk}M8gl`suc{+0 zB>YBC0OSJ*c)f9rO42uef;6yW-VSXu@Q(buPa=A@;3@Po{4ew2ZZl~6sW!o+j_w$N zc{p5tZ{j1TDk0#nU!j0|W3`0RzuyX~tS_+Urr06<>vcst^fXiTw;O5c{I2ZO>^}WB ze1;?$P|uWnHg?thCqV}3S_ohR@y@dSZ>G2?zs*jW)(hVKw_)bLl7e^xf|XsEk81zT z*pvVFR{n3Pi@^UUC#YjYJWG2)eBc+^;fU)2kIDZ9@AyF>_b2;SnU(Ejp5T%L1EBv0 zdv6sLSCnu40zrdA@ZcUaAwX~q?(PyaxVr`m!6mo`3+@nHgS)!~hu~5;chTMFjQ-B~ z`o7+m`@&!(KvnJ9wf9>8Ip=T2fRO;rUG1vxs;)*Hx2~Mx9cZq|RVi1~%pXmn1o64AwGsLgB+xYsI1H-s6hX-SK)bwfZ~K_Z z69>Ar8Tw!5SWFK%>=1Wt+Opj1f;Nh{Tq7Wu{lkG`ODwLcK40^>xP4 zH@5j)R^;YL`k*Zw(5&MBP=8Z0JC&wAEGp*Te0Ro+SLu@63Mwk9^lCL@^Z zK0TvQGQ&!rYiLA2s-i1V7^aeVt??IXvb`CR1Dmc`!10hQ*IT9rz6txfrx=FzEVt87 zGaVr3SSkfka6cGU&(mAxCQru=NB8x;u_ga5TO4$@HOc_@a=RXn8{)>bij4b)44CO+ ztmdni6?j#P)s%7+tBE{hNEm%!K!7(EjLOMCKWuEZ*+_qfNC;f|IFYV^dh`$Iv_7`2DG%-YS4i?p8#edZi`{ym@hEH)$$Q|N$VHt6gJnA z#>!DSZp$3)!@{?zF{QrT?|U|2Sl`FiAqpP5?fu>{-W?;ifMU)P+jvnFW*lFp%R_^V zsIR*VP9cZ;Yb16J+GV>W-OTT(gi~>^xg7eYEoXDpN>jdUwA}O{(r5J?D&*=malA9` z%ToB3-~SGD5}ed=nYV$)h(8Y*w6->&hxAMz6Y|4w$P@6KF9}qoK;D2EI9$I2()J>h z^3sG3=#`HKJPJ-GZDxv1<-e0nsr<;FD1w30i1r1-MQo$9g_O@B~a6Q;)LWqD$~CU@V2ZLQ}`^b1-rF^G*as~u+WrfOtBp5&H;7pqlezXU<{O#Azd8IYA^07oz3s)gA_W6ydvi}sr_06O44S& zutd7D!zT{^BR?0|ThWZA2IncCroW5zt4&08h857GPXM_3+?-=Y6uD81ygvWd=iat* z;r%Y>+*9XG7*Nr*xC$^~Fm^dXO?K#K@=T#ky>~Bj`3R0;emEhm)BupeY$P^=)6HLc z5j|WBfE7aa|}Dvhvp>9r{YmI>yFer+iGLT=VQua)9D{uGH6%*8kItX2S?T}whLslL_9@Y(v4`%D@*ZiMph5MQH zznbqS98F7j?EmBvioRDltCyUpF!Vl;rYjaJ2lLqeq|9Cm^C_4?SJi4$)SJ`Zw}d3B z>I=2D#DiIo6A{QF;yKW-hVei6J__I5hn43!7NEVeGZ1JB1-FfE*nJB=clE(dDBlrCyr!D30V3)V{dy zIiUQj_fb01wMtRAWUZj+y{xpl?LN>^e85ZEc5e3+oi~?0kNC ztLT0>jaT{r-qN)FSMyeUoqJ^R0uO;`6yI@og~sU7UVotT^*+;dH@I%GcOvISes)zX zd5;Fn)Hxq4Xc`y?8WN4d_w~)LPMFEnyt&)jdLnUezhHq>TWybYGZ)j zXS^r!YloBF@bi2n#STLa9ML)AQQG=!#Lby>@yUjuZ2ww$BGLm#aOSGU7slPlZ2szm2NlWjW!2a_-95lhp%_0W`2BBW)Kh| zl_+3$9Xjl?nT~WJQT-w>(FK1woCF!Je~S4$X*$RJlPH^tb(=}6MwI{h#7O&?s#Dw? zNi0g=V^5WBB*Ja$V}tdsf%|%p`cTpiK1n6E*bjRq)QcrzlSR$rRLg!ehX+*3Ef7FY zd~Rk6M2d4Z_KwY1mEkd)x+!;Ed8V@s0GtuEYle=+t9`mCfzz79h(p%x;S9c6l(=7N zft^l=3;p3KM;mtY0?_!UYuO=+XMgtX=L?mI4yEA)gDz&Ur6dYmwMGa6&8q*%Up#T=m2PXt)%e#TWFNI-**>;BYC zCrV zb_M2#axv^pdivrQ@9`f3)%AYov;r z+ux3W{RZzay6np#&$SEF+6dN{GIA2Th~N#KkqD9(xYNo+NZhm?6<|W=`=E0~`d9&Z z4Hi6%S71J7CXTa{W&d)+S+1s_rs zj(zdr^n5oYJ@YH|V`v`bn)_afK^@+L{x0^>qgRe|dy554Q{hE4pri4*6jD0ms9(T71k@6quu3!T9qshosrM%52aE{@vzgfVxg$EQyGe(*^D``$!N|TqE5v z4rHW+a1J}qHuXuBbEH&{J#4ppaMy6Qmn=ST2#3BM06IAlYC5m&dXJBbGNnRpX-^;F zNV)R?uE!Bx?Z&?5u`Ov?m~=bR_;NA194kKQ7}{0_7D8nB`^zN`tsa=UIapd`;=1F6A6NYYb zu18BVGe(=QAhf*KbX0!+_!6vqaEt35JNV}(EnW>(9A5*_crv!|%>&7Hm|=T9?Gp4Q z2UX}u0BF=-wz<@tRRcj7Yy6Re9ep46a^;O!5J_9z2RIj(0~Z84CX?*XWK%L zAD@oy6jwOH9==UojjmrVF~X{q>J^~z(=rJo$7#S0C?AL#8DiLhc^k)`wwSZRe#yvd zjNE8p{uwU9-$DFTz}>N%ooHvey>~SRg2v=8SD5F^T(cB#OuN1JwlF8!^z|DyA{U2< z4kI`p3*9wby1B3%`d_Jv5N-m==iJ7&P^=z^AmdbBa8tir7MfNx1v3!)_9iRqJGx$e zi7u8Hj)DVDv?M|sm&JF&Cct?>aX9sL4z!P!hmdyqqQB<-+0#Fg(Cz{`;yXKA6;G+M z;<;_^kY)uS!YTuQGmekp8RBAM<|N;}pYwj5Fy1bwKF%Y=?vw9~Pv+t}Fm6~uQG(z2 z2qwOJw8ByhgN2nNH&fq>z%8ki*Q=QL3M`)kkX{g@R64&Gq8Gj!>wl7Mc{DIM$JrDS zB)GrAk5NoDcx{i;J!c12n;KlF%!S$G>T(T_ZHT=eDOA!}ehc^$Td)R=zfRZUw0z3@ z-K<4H2~hUCPy(m3X0;LfIV;uau!t9cx+6HD_v% z9CxrBkfFh@)GCC#n4?t)_nG}z8H*sCVMpg1x(%h4S2iq%?aDr1M#1sk1EFQd z+X*mGnSf8O#JqVrm|GL{%QAFz zBiF|ZhcDIUOw{j@v%{-2ks;4u*7dmAKX^(B(LYd=ZP9xvw{B_CjZC-q$Hw2k15;)I z+CP!}w*WK>uAtqq4^YV;k#+Tvvj^XEfmn*-i2=FC3*kIyTe!*3*Z;$zt5ne2* zOpKgycr#O4_s!XFFg*mfG4ww5Fd@DWw`ulmeaP^vWfBn=oce+tsVz=>*Ev*eV}Brg zbUD&ZNbOwbD&A(#YsdOE)Eov8K0+90R{_M?E6wnnq8}gb0_}V+-lBbnCH~ z4x^Z`Qcs90DoDryn50O&!c{<`dkHj zmfzmh5wp8%ejYxU$eFn%FZ9zYjwT{)cw0seb;}OXb)y&_T#ND8cxbjgXbL zK359~o?>D?8DqbB``J=$+XhIqvB&8TYVMZ_`l#IQCDoBsH%Y^1;UgA%Zl-HXYnK6$ zq`ZGw!-zLyw@d9|8=4GuyK<3H-zTDkCo1R5%^F!(B%dmjnUDXl2rT)r&VD#lboAF# z)Mx_$9F2QK+}77Skb~T^w#OMXvi!DcJL0cCaO~#c*3fE4YO zB&9m>b!Lv?A5V=8V5X0CF9ReRlwQ*HAx2$(}MUM z4*t>6td0Y+s%^)v$Ke9IEG?J!@?evcNqXMZANPl-%de_I+IZYfA9^O?gE)4(aSNQD znvQPc$V!+~Q=*eC>}|_ArgTI+F3kiNdIWEk_->9Wlh-TQgmn$a zy3Dyu9V{mszdJ7&=7K~v_qU(qS=oGomEk&;pyQE)k`jx@2n!i3-x<_PV}g+RILT(F zL2gd(mr(tFz;*Sg>0nyJc|||g(ld%-q3$z1U=wq&2nGJ-u2QK#Ty0xz{$}v1abu*( znPXVpDZlBC@sxdle1{}fTX{_+hs)=SAL2A^2LbzhVfx*7hFPfyLe-1`8n^tnIXaggdddE50KYnSvn)@Wg&M9~qWZBN zS$ziNoSSaRj2%B$Gv?@aXPBf{$b~{yk{MhaMvbw$_2E)v!_JCf?xx0q!$xE7=PkPm-!}gx ziO*Pue5y0qWx6YW#e+SGoMwSAkHD-S{0wAodGczx$u)B%t5q(O#gUQvW!-OA9{^x} z8?dqw#aBY|muRbYb|n_F@+5Iw@c7PY7emGHGjqOtrpcfevJ9C@DNy1Jf;kTon%&PW zB1_N`NC{^B=zh4axR1hL@+rbR-f?CxJ>eU;z&_w!F}x$H*mo(?PWV&-pN(FqD)eA)jUnw!sx(f}vMZxx_UN~nh@d}RRT z(AqReYkV`p1F!vQ3QY2PU!uxL5H~n@&^$`?@zYmYeOjbi8pQC0v^u7D97!+5DSY#W zpo6QM^V#_}xu0k^qO-*Q!+z^0nUW2e=5@W%EqoZ@8JK3P3J)jZ5pJc^L45a?Cp^>5kMHZXcVg zcY_uwx;=s%iuhH-!g#L}Y#n?_z&J}<>@$hLL>RrLJ9hHn3w?42ic!a_Il%(#T@4LV z^VGHtH|}Hm2;ZNSb6g1AEN0$Wj&^l-Jc$N7M}K=qPcZE`f6A5a?jA!VU5>1}%#k}U zmKq?hMaR}ynj~bM(T64-tj)a98XkdxFQzku}#(J z_F<9sOthC~w4@=osXJekPLe(H31mBH`;;mFyTajree;zZhz|I;Iuy`Yzl30;OIBBR z?@+mU(SeHbUwSS%X4VK677Z@=5g=b*zW^6l2!h}axg=RB_L^1Q=LgdE1KNN7VgJ+s zu>4KkaQ&nCoa+ayg7)aen(N~8lz&kV{;h7|pAxo>78v9%*9Z{Hr~TinFC@sAu)y?+ z>2bf=^nVGG|MjQ|K&$3Fl<&p<*Yo&aFH<7y8E75W#$BHLA0B2c9|+MavUiiG-T&#i zPEY`5!Ni~P5lfo?^#K3u_W1P~m;+oVADwvZ|I>4=kpRsT|NkNRx5MNAk3&LrzYkQw zL=efN%7#DsXNw7NoO95tbHtK8pDTiMBq#F!bxwklGaIuHU#qf4w@5syOsU2k^9C5M zjQoEcuXq|S-=dx!*01(zW$Ng^QDbB|T&Rl~PR%hHnA)>Pb3EqrO?dVNiiJsBs1nU4@;TzY2EiEh`{MC02FA-}neS_@&-%6L$nCORljdZw%M+7M4daZ-RFS4G^TBa7$nzBH zHi?y3k0i30jrJe6u~d~4u$jgKi>zb^%nh~o?Sy!0HVvR_aYdML-_Fv6pUU!R^Ui!ve|4)7gms$d-NbPfo&+j8S6e0>JlF6L z)q_Q$tatt;HIsjbsME!}uvyM9P86vsYqt5sf!x@lKKo~%^htfN89UwtM1xAVC6JM? z^0*`{R4q;boCX=Oh+FcRt6U%8BPa%OOCbshDQUTBvnWy?W(KvAST4sM>RO8@$ zf*%&t+rz#Uz4<+rAIqezR-0`VE1O!^ZMe{=$_B9afsFdC+#?)i|Ll!uaGQMwN!(r> z6$XD3Q)+agKfWcZJh>8Bt%$MM(fIuhHx3?k|GE8_3M>dlEbdA`e*L}GTzg&4xfr>B zAO@XU$ydO;l?3@^`s8{!}3o7(232V!<;vy6jO=>fbfBzetX~hixYyMi(DQh={hZFlHzh zrot$k&slx5)K2{OeR$MaY##~)wf)#$$5zv`&J_&)G`0GT@z=-JgPvo;-1 z1z=sot)Z#`X8r3c-COU)vb1X7-0Fuew$2vnYQIqpZ{ja>8=JSCsIgZnUVo&*2!mP} zsoJ9WREm5--Bg$3?co|x%N6f}&Es@9q4y1s;SK0D`C2Q`){XJOY4__MJ(Je^FwH_~ zoHy~+k5k_V)vf*ZyHBwz>u4{LFLF3n@yVAPo$HFwNIuYk2vqzwQwmk>e)JXOus)~; z8#$(3R*;PRkb${C{@A9-*Yr{kg8nnGX=`INK9IeWY(B+k^f!LZqIP# zElJ@Fb4N5#xdFwf-)hB!FZM2?)Wx3MOs1>1DoV6)7E^QtTPQ)sj}dP~KYrSBW&kK} zR!;zT`?0i7}!sg}A%Kz{ua+Mg+<2KoHU zvVffo0eX!JKR&_H3_g~FnbLZ>rZxwi#5yF=>7N!*tq(^bA6A@MVLyChXs!bixM2@8 ze&lG8xPquCeS_Wl4-h$&&7j9)(W`095?#;m3sE@YZ7@EVA+{&WQ4GOE$`Co>Z*at8Ak?pPJ=VK{vak^?J@HK=NgB;s0Um@WbZG75N-DJKBM{Q@qWYAMxkDQ`S0Pytc-KwN~b`-Z(Q)tpUUY zBGr1-TQwOT4nRtnSUrvvu)V8N*H?3rD}$*9f|i+C%HN;EaTBTaqmilDv7!Q^zB;&px)RryFc;^O79wu*^4cMB{cKnqO3JYnR2tm}o!c**$f(uaef+tC z?<6dU0b3FZo@bZtxi>(9n3cs~RpvvK-SsxKI4@tHlzqq(}c7MkG{Dtev1^!rm9u>bip5L zSkKzc&O9s6J^N(Y(7l5(hx_|aongN9<)COG;n^V_@RZ^)At=`Rn81(nBhOSN{(eus z>r+tD<>93C+4dl1ruS8?hqSvGnxTT#($DO}kOvk(6@iZeAQw+m4=#sIh2ugg(`$xu zbawGZrz5%-@E5h-cIW9+Av=?!&1>v$xVNq)VNTZk+fxz5xa%*DXGL&wQIy2G&QFAs zBp>f@R6++zgIk970i`!Ww7|G4Sa#y4A|KGUkF7xoBzIroWI-)0$7%RmBM#aTgX<;TmTkj7&Ncc7Om zRhz~K?fhwb5d;CG18t(4xg2y!PtHYGJfGt^bH_;L?$Vm&0e+Eoj zW&vU9Q)JCMvwVgv(cMc2<}A-Luk%_Z%EmzcaOy@H-eSN9iC+|zzev6qFru}p=-CrVXvZ0%8$@WF)9(XpU6WCGT*pl@|l$_xg0QY zkhux+ZIL-2OvU7JM}Msm1b7ER25UvK^!3NPBaV&bHi6rzeK!DTP}&KRae(llYb;Ix z4o#tcd70H}rO4Xh<3^Mbkh#GrP4xB;D*OBR#An>mWc8XCbPMiNb`0feY!2bfLHVi+ zrE((v>(-cdR5As{2pd!`6X5rz|gEORV+F zEr&AV9V<*{QoV+Fj)3Y5;%4>7tNY2bv4F0v$8~g12&CR_tsUC^?hvvSgqIPcF4j!K zkci{3DLdQD1_s}i;~(8PY83-0P`rV=Vs@M|N(h}Y>K{t3Ix3_7A|me67;xbjqx1Oo zZZ+0O8|=k?C;Txc1n5VfxeT`-TE_@YFm8_SGT*n&ld;cPC%_@|>197>)?*T$%`n}P zlc;F|aGa?gadOGr(h6m+<;+ToT}%dJ0q;;nNY6Yv9MPrN#Y{nJeaRPN;B-~bZT^55 z0lgL2;gW^QO;u12=Qb17^*xnALkm^?3bzIVtyaIls?V%vMFLYwF`lX9Wdh^A`)<)0 zpX1Ey#YkvG6xeoYj4yUexWlPQLg+oby>?%%&0X!7&0iP}-EB^?7-!T;qh9D7H61f| zNbvBI8w_bz*j0L+_3tQYw7@^zcITMdi4wnxIbS#xCHYyk&lEZ?@3s~6W~(eDZ5^A$ z3&YI>m@oQ$hyrJ4O0|`rkjM3+LKF^68{$8JuCX1^>F+%vY{Dk!pwZ%G%uygYBs}8v2qKB@o5Wb9h(6U?eUcOB>=?%bvO!1kIvDuNg z=Al>Wq%h88duulxNn7|cgGMR(KI_HW$lox;#%8cRlCJ?ninX_b!{?sLu~!V~13Kk@ zu>fcxKwqrI6y#NjrEq+2$hU7fjb*6%iR2419&=2!87~RD243L+A5{UNEQt7*nP~00 z>rA2jFl({iBtp`mwbKwG@FC#o=p-GCr~iCYPsmY16ptCr>_2s#6*$DQre6jBE zB65?=81B#e%>wz%O3VwvOFQp-*tMg`lgFdwMh!Gkx6kX)-$Ix-tI+rb%k+fI*@T&l zU{{*(4Ogno>VrC7DWDHIY<*gI88`|$lV{IDDHt0SvW4p$!6Hd0GELy=#s#^>nPll? zKj78$2=@YM;HfYpg1F}BSvOMLAuk4QwGFsOjc}s-Ti(8P+~!qI(5jQrD60g0UfR_2 z)7lQfGtg05Zj&xYf88?-H(%-fI$6tEN`p1%V^G(l?YCIf+n*{e4xXnx>w0=D2%wf8 zPD*^O{yoWanJcP4uHcq^+t5=JoPjhx!O2j(KTU_yEJ@sw7+<_NL2SFEJ*`bmf_-t5 zF*ctI;&a(Qzrva?+3BL50uO z!1q|qOJYQ)SH4(fKJztuY$f_3Pm_eG4NS^CPrd`ITz14rxim1vYUR}`@Mcqwev*uK zEVtOS@@zD^fbG#>Fi{^)s0q7+fw*9@8JCe0z;cq5Pn8A*0u!tvh}x8>&Ov0x-7yX( zl!xdCn+|rp?m6}$1+RrO&UPU8g^~~o(v<`@j);s3tLmfpF`(~*8yO|EK1^Q(0t-pFh3iC&dAJ~D zFXd9^gCt!<8BYPs5vr3YIj{=X#H5id^UQzcbl4;Q;MTWl_=_192LHxqx8rWeJnD{j z(K!drMYq8s;>~!$Wd83^&>A^ZjY>>gTjO zHo|)VRJsz%zi$oaPi_xZTEg-u&1Y&tBNJE@qqd`{pDshVO%x5o}_(gh_rj*hK!wtbax)f=J%Q_l;L9D0)=%9EDwplJ8Ysj zBvOR953dHMHM5Q+hJLL`tUhKZwn~2Q9!n6I?%cs2BR@%CY@gU=FK4*`EAi>bxuC2C z+CkIMGxBvt2QfI+n0$i6ppNMAFJA*Xd5SuM{R~%T$AauoAENCZpC@FG>TO2Hn!Ch}z&Zq`G_>u7o9UJFrJ+#|VKy2ZzZ*BXa@s@n3H3*BHx zfV(-viK<75ZOBk4=E!vm{bMgy`GWo$pWNfm>C>ba8v08IWrLal>#@aF?~rLON%5YT z?hIj=Y^;-C3i7J;yU-k;{;==DSri7S{7oHOrT=Eso{CH?hJOoe@eIH->%wu)<_`<* zn0e@3Z13*<#jQsTMM8F|aAWGi70=GO=|}woLQEd-gSb)ezhz}7uwoeZMJu*oR;Bnu z_cY^JoOs@)LrVDrm$<)FJ!ks#>QM-@$R!WoTiuG!=e3y}YOz~mk@s4T&LeOMkOL-} z)fpONB7-_E=_^v0lQ$gA zD+&Lh*#1;N@i-;UT*MZK>S+{^imqr34el@-_gs-YL0irV+-Dh7ccU4R9BLuSY3s&U z^(x17=Oed5_v%0KR6_b?<$kJ$Q?e}j#kD3C!2aRkZ~EOxXUGgwUfQTmekwku(|^k#jf3MEXPwF z`S-`vtpvNvK?8g=nzx5~6-rBDFq{I=5Wmp2Vj**;&1S;Z@nBPBzR4JrlvH9ET9|kd zw4;Su%Q^1en-_-+s2;&oE1ZD(O=eLLN?=Im7;aAo-$i8l-eUeo;cQkSr>)-2p#D+( z_x-oCmCBxXT@*KWLYEw!L(Hz9&mhOGq}228>AgrV-Gsa)?vIB^xQtH(BiJ*`)~!%; z6*|7?1*}wVkd)JHHm3rBTz#`+qn)O>IKj-{w*pblYMuf>?Q)^2VEMV;DIuy`q(bj` za^vU(0W_g$m>q;^WyCFEW)%k zXJN??&?s1&gQ=qqQ_o<*6diPXAz3$#PE-=Mfez$?&yzhV0@l@Ala7%pc@8G?DohKe z9l$$P_T91hdTk4e^0n}%=rKaJm81EM{sM#`kJ1qXci@#QugAmMVAiWraeU`8Bgsah zfOqS@PhIycNLJLLY$@(o*P|x%$K~&e3c53?%yHrQ^w9Xdj{k=QyUOa*OWRkE(3<`} z-th0Q(EU<{0h)Hdaro-61~|zTPssk8w#dxCsJ^bYka{m+N3xM5{^eDgvL?_qU6k*@ zf*pdrAh+pmG51q3%^Jx+#A84&6Y2$o(_zPYnUGB>vM~+wRAE1z^Vk(R%yWCM*Ba{N zXYics{#=KP?k~GaZ<=jAfr5tfKiG8$rC5`GVLYbf`GbmMkj z)O-C69641AvuoW2TF;{fT9c8#?p2Pn%(4%PQ$Jp*eSTu1e|UE+1jBfJP`|DzL+)QA z)=uyfIHO{@vD}=#f^E1Yz@fvD)Q@SiMx*SY1+M$vO#jsQ3Py5o_MFBx3o?6tYmKt_ zKPTiuP*DhR7*wK^jcoz#$Du`vx;%}wHQ3IYtstfgT%JIr zi(l_LW@F*jsN@tEALKdqGMHY^=8(qjqL3*CG|{W*h2o?8$hGWR8M7~)F3oyw>o|*> zM`OduJn|c(JmzJ2Nti6WyoO#KJ7Fv>6zIFBcb5m&cbytvM9W@bolq+Xcd@Q-|3!Qh z8DtOaNIrG>x1{tfNJ`858YQUfifn4Qh&g^c2xQafv1)J=TJwl70sy)NU8Y2DM4ey! z6I&$SkEgwvaBSU%G*UJC3O-=WWbG+7{nJ!TW3xH{4_BBOltj|!h(G;+&pe$`YuA7P zeyH|hC79)E3wR8%)IIIKHFB=X)SHPkbRb1RB%>vQ)#JlQ4Iz7I4-AbfWfHl&1kI?Q zf7$q^wj(TNd&y_=5Ayo)-$M${@P?Z>f%M46e>inDc1kBYU&D-;>;wy;;En(YA!jvr zfn`XdYaX*qDu+0@SQs|2R+8Mv{uy-LTge=xPX!}GrXuN@w^RVX^nY&@SHc+x*i4b4e`g*QD67VWUxp6iNF#*f`GNFXoU_lzJ6PQ{p;vu zSOrTi4(Pwmc@Q7ZHf|ztSx_TyJ~J3gEq;Mu)6o)z2kZSqko!d)i+1ahuoWKDmCR7E zGni0UPl$X44n*1w>0(w>!bOy+2@;Y3+Z z`!C=-=h|rl+=WA_D5ZHJ3v4&LB8Q9=l)N;Q?jl?MP8p zW`hHbwL@~&_to}@b*s4hS`i6sD0D7bW8B263uXDAuC{XSXN5Z=txCSmMrJj(dy{4- zZ?;=;KJj$x6XE`pTjjpyvxhZk4th&XzM=(C1p$vUaxgd71Nqa3ks&L;>?>P#mPtIG zWziBh1x^%CkTCnsP-J_5d+Xg zAI+={t^@m}+^fa3Mle$XG9gL5JcE*-V^$^7I5O!H8wPTF6FyhXn(P~XD6 z!8Q=>wvS1Kzrki*d_#Q-Xz4g7BcOXRX{%6VLQ}@iFO*)bOe%>sx2@4WKjVtYw0i;a zH=x$nfB&L1U=`favr*z}grIUZ#`kMOB<=S@tKg86iacc;++X(()J1sfD-!ZRBbGm@bHDR$bx`d@JVA>)Pre+dWw!$Ct5S@OSr`Jaj{_-ca) zgisqqQ<8olRB|8X8La#`G0<6Cjx|IR}t28vj6!%|1I)ikbtFdPza^v zKaZ=qWJN{?8@(^VFOmOvg#(S_Sx#38A(0KwPfF65@O+z3o4-SSEY0T5 z(l1@xv!eZ17aG~1H_{Jh(e=@+WsvzVGctn&SI*xR9@mA3!=t5Sww<)S>0+Mm%(^Kq zM@t_eQLRyWsTVY0Db?{NB7@HVUJT?~|GXNnL@m{=4;{&D6TCbKQu-~0V?JLstyZs; z6HIz@b2B0EWR*1n1dH(>b+fPuO!^|TWx8?A>+6Op(*-X9idR|XLT{mobEZhf)s=I< z&sqGuHUJ??Lw6FJItw=5537$0223}9yj)q~U`XOO!lkBqUGFYS`Mo`zMgfEaEatM9 z-QC^C0aAYNWId{D-_62uB)uRa`hk)tcXMht&U_o(0b1BAygvwL@|H zuJazQ5X<)W)5pr`#|zqEw~L~G;U>)-aJs+lN11-Qx|0b#zhIOF4K8`0BYW~uV-oTj zFRLR2db#;V+G<;z5T(7Pnb+C*HS7C-96*QUc2Zv%w%P07JEW5-><7D}ZTYDic2KgZ zjm}*fl&NkySd?l=Uy;{?MqBPa4XeFWbvMk{nXhuXn0sq6TNiP0sMwUF!LY+hO+%yg zy^vO_9S3IpvJ22c1)gFI?-d#xKE6>eL%F$C%Z`sf=hYLSG{V$)rPJ&d+B-k({x?Gk zlnms+Atw&b5e0$6pxAST!{DZzT&4F4&y`gKyYjd8yiczRNp>0p!~wC-6{A= z4akp0>YSo7#+~mp_xc^SQiQ;12|g7%G#Gtx+n~Q&Z@^!ZQf+zF=gynk=^Am__ld4f z<%4*6Gq+}-PkHMZg?uCWcyhIwX3`(ZFA1Qjva&eHJcawRjk*3KvI2}L3{3-?<$6PC z^vKtce7&fn7>9*cT?Y3|E=&Get8vkuS@3~1?vPg`87xqc8PWsFD!sSBC7$GLlRBOh zmv2ThFW(-GWhuqei0*ie`?*U4zndLt1N>mxpWm`6es&E;j5bmf-u4nu2}Fn(B@`F? z^Xb4@!Qu^#B$ZJ;=R4#etXg28ty z)oRJ8Raw?p{9%$%{2M~NdJqFa%Tt7)<`{O}R^wWxl;?x!M8B)=WWM#MjMS}{@1M^` zDM6%7I)9`M(mW$xTo2>Fus280fyZW|AoC#%gyTH* zuEX)HQ_V8G6@%QlZjo?CIyr6MA zP{~lw;|swk_}~i3^YTaX-(0`2*A8sJy2TtyCmUM)Wsg1TPEzm|J<8}~u+e+_&JXld z6h8tG&%x=-NyzSj;tfhZw>!~^xrdN@dI9Moa{B|FZW_0@R*<_Xg$KCbh_~xKeVfQy zgGMe(O7fm)D_b?F`Y`p#?pb21&Wy9mMb_XjE%1p1dD_N&U#K10aq)tjl~v&fS$6=p zyQrwA;tu+sYXH_Cs6Dy86eU7`Q^;CxJ(26*J!^JpXH@|`{RDU&<09yh6ey9E-^>eL zGFs*M)L|Ng*4q6=oYD~V*e&6|MiH7F1QIaAq^a62)qEdY$1`*M3e;<9UXBhwffkSj z9&Bj+*Bxku!Y@dMQ7qw%jv6^0e^sZ&BVyS4J~vxY@okOMoU`PLGaj|)q>2VpLA^G*r8P`(>cNAi2F4UJ83lOrd}gh za1O7J`K!T9v-vm{y(hpE|7lwKX%2E^Dr|HLw{-Cct2+z{W@$MDdzZT4s5e zj|wj`*~~~fNI_tRX-lZa`M|}!kMC}2uw;N+P{unu__0m@DeHF_F*<+lJ8wl8E(rtj zyCec12-LoVIs&)D`;NktaAKQ!z*gU%>)Y%R-tUR%`uLHkRX&nL+8|s^Jpq7j12X;Qol| zcbu*sQ6?S;^?f z(t=%bSU~Ow<6GTk-xWe~|IJ7_{A=Xl-n%9`VTlhJJ-pP^ntyoRFrksV=Dfz<33g2& zGH7&s!atr^ZcrAV$>(J;ofu3=tznCJe_ICDC|8Jc2l~J%WL7PBsBitLzg7aA#e%4- z_@HzIj}(25>7ahq`VN!FZY?Xm%3%G|3ci6zFjjqKg5HiCCgT_-$in1*_!4xr+!y02 zy^!AYW0RPFu=Vay{C00=b+@0kCu~ClXV;{?5VJuBD*Xa}rUKO4H2f7ikhx4`(Z~F5 zg`R;(m_5go$;QgmE9I{0!Wba{YhI@IQz4^P;{e>>pz@EwC5O(RM*`~6Yh2ydbU4XY z^N!iuPa94Hr#*_KM`ELwPez&lek!d*G&FYGh))5A>mwA=fY74vW{XJQ=xAc?l??XC zkB{zS>A(Gsn}AE5Rje2Cy0t2iSz9&)Ef1;AdTabwI4l4&zlmAH_q9CQR4*-Z-w^Ej zg9O8EuFqkP!F=>BYd*wJHv)tT$$Gyo)_g{;wO*7O$}#$MmBhIl1AF@eEP)4M|6Z0; zO!a-*TWB-I)WD4`7G%NJU{Eh3w=(vJir~|RhT-Gg%^p?<^+qd~)}#VXMPXv3GC;as z?#U0mCHh)))3li~qK&uqN z{4(WIEdKdU1lFhR#8gn&zkdES|5ELqgm1G{5&8yXq(F_{x6AgV5V5c}gX;%WQ|p6} zUS7I|chYkx5UW3$dRd?8ww(+9AIjb`EX#G>{wAfRyFnTW>245dkVZPCL%LHMrKC$j z0qK$k>F(|h0qN$w(7E=WbFKYf$MJp?AMS^{t~keUjB}-X&AtI3nw8yMDGIV~J9K$}wPcI$pPMj(5Cr#(I<4XFhl*)$`HD zkYujh1gUPSc&_?n`Y3Z>l;5-;dOvYb{n+keITGgPy&Wndwzlg5jF5Y@0+E#Ey=EWZ z^yJ!r%Y3oMbB)^fT$~nz^#qAI%uSay6CA)`@QTa>=Ea^K`KOcOx60itm7glqRkWCX z>Wyg_Z@q+dPg2yb@rI5hc!z4MM`~Mj9DvR1nN>C$C~~+LyRk<|^u^wD~Cb!jBIX15c$CdK}wGw@75-i2cr z$f6%c{6_eK_v=UmV9B+MHKK$=JhH^BRT=m8#1wMC)v-kH{-XY}ORL^F0RBE0g-{ll zXR-j@8Y+C4luw%fNx)-!0rV|lREv)Hm4W$srG|~Ek%Q*RT&8HgpN?ptXa3S!^QBI$ zdEJNp-MhSrq@DH7;xsk;U?U@{V9QHU=Sh?R&s(zJB=6UgRN#U{SjLux0BiBGmjqkztLN;dPKxJiz za@toK*^$vgv?uf~I8WMy>YhBV`)Fb`9|*dd`K<8R-NzS-EIhhRgJzp&e|$9hAVERt zF=Lo+guS@quQHgwZjPB53UeYNCcs}9-->V+NjO!JCRBK4;Z7AZ22@-nWKK0MA?I=XS!)f`)=pTOMCvSx49E!FC9#&x??-3Ot5cDfhZd;c)0^LGG^U|DmA^DxBIRI1#afBsp%7_@OPgK0AKaTjU0wqWVz_Df}i zlg@7+bNHU)Wqg0 zoG`3=ot(~|%76I~6Plu5I?@q);oqpc@Mw-r>G?=amFU%c&!c&@ERRO3+rTa1WF=-fWt|Ns zCB-}Utk-4Auv*ntRjQ6}$B!t5g!HbrGEm0rRQTw%#?O;m$~P>xfnxTNi=RK@>-B4L zR8*I+utgbJb$Mp^d?R%KF6JI-wkh+ZC&o>>)qZ8BC z%Co3`(Fs*nRyJR1(n#jB6-g(}hk={QR(q~J9Q|;`16^LmA0;Cz`&rNDj@p zH<4KipxIQ+%*^N?Nd-OQoPS2yU*~;QEqnzs{c@V~?&N?2+m1Jbx}N|NzJP$!N|>Cy zb-pAuKds(*3?wVUhZDN+0^B!*sdVxb1U(fek5*ZJPG#26C$^2JC8UYErpKDfmS7`z zwNTR^A?L4n58KTXpgNWBTva8zL6ur-gLhcVRaJVQ!mEna{be z&eYAnom)XT!*H;oJgFQQR9=wP zI#9*GEzew|7AgX@bP-@n$^+FdB^h1oR$*t*n)73`EIu>(=0NIIO`NO@&Y?)Ewdh0UZv(HcCpQ{8@ZNSlzuhiBRm7DY_FO({h zbk~lI7$M+SA`KvbrJ1oP1j8iS_|@g>O@zfBZX6^waLqa1A_sUG1Rs?m-YNhuWM7C{d$!P0T2lBRVjzp z>*5Zz^6i8WXNq4?$M7)P@$u>B1tldXP}y_*E{W1Ear51~k;P%HObS_>IhDsz?OV<~ zfGW#}KjUFrV||4Y2-QLKnX--fVWbb zLjx@?HC0r*+Q#vVK*8X^z?;fwdiAf}I`Y~JlsF9l=;~?0WLA0odbTm&GB4fqgRtuz zOV6*dAs=Hrq1WGZK6DM5xUMvYnFoU6uw*AiDQ&ImFAP|i*ITUa^u#<45jc!G@?XAu zQC6e6zXcH?1ESWp^eU6#tE*2<&9QN4yrx$H69yk%qSs9B_Tm|%FZyx_k3w$~R{p`$ zlMisDcye$WIcB4Kw#g^2H(=*$mfX`i1&`4{2_|v8)}C^<&N1=I<-H>#{iC1iI1^Rm z!Et-o;Qr^6(0D0enAir5TT^Xo`fK`wPW*DbVeb^~u8Ssib|e6ASl+*RG(9rkJ8_*^ zrJf^+!t24Qv>E%9FdH{sn1sw%0*$$3wCFDV*@^gR5CQ^C|MQ9okho`_vXfehp&dN& zjrfPxv8tn{`!W%jba=XYY`%7P!+&QekDYk;iLvQ;*O;WC(29RhM$o2mvg%2>iHW_4zRFy5hT2N=pm6c05Cw6S0(gttgcpZqCYDI z<5nH$M_c51RsOof=KZJcz@WwZ_+dsa4YuD4DhEGbWN)|f*)NV}THk-nQb{qr&d3Uj zDiXjX#xzp@1Q%O}tLrMsp2Sk^YdTP*>1aNCn&iN5zsl{HRLbu=yno1ZvOOe}KXz$v zBw@t&zOMjyu;emW!{2Zg<>Y51)A|b<5mi{aV96yNhqfP+Sygf{y->GVTW7L1ZoF+nn)QBuO($a;m1F8Pd#N@fOR@MQ`;B+COe zA{W%9%Uf2$O8=@$$I-~5SZDU?DKQdBwybglI!XHetHhgT(EzA(PhA~bQ2+s~92!dB zd*j5jI{&@IyY#2Ts~b3p7ft!k`Ywp0dxx$_dE6F+WDE+t*-+AUk`r;;#bD#_cI%k7U6!2=)TmetBv+M}`ksKf})5y)5KkvHd$Xw%?3c-;JnK;QI0zXe?(3=#bySxBA%_WOe>^`(oV)raiH6A?D$G^TaVh0@Z^In6pv5B6C z#P$n?w>ZqYtxMXmo03!C($;V4EXFt|>5F4?%Q?y(S5IWk7%fnXV2PpmY4{_>;rDPH zW-X^K8`wMcNH`!6;fS0ow~OiNRl6^NIOXmeU3ip z_;{`wjpx0SLEjcQ*$+Q$A1~2t;Rr}z)}yn~X1Aqo<2RZQEbGgOj>_U?gm@p$X5lu+ zFf>`)ZPPyX29MRSKtn*k_3UVcl7jz7ZGZeZyY=YHK%8j7^FVV%KN>;3h-R|>fZ*i| zm527W&*khJ0W~ubbt!hzQ|tl*6n+CK)hf(5Ev>u|O)gWn6}r?PL#-z3Tzg)n<2f7q zoe&kWvM@dYzB>66T*mlWtLM+$OOp;LNb)4rdki32bP6-1M}@Y^k?bf*=+AuV30Nwme?G9ZLH85I&|~R@YweH#EVY zBCn{u6B5FyO-!(|Q($paOIF3KylT2PrBC!q!-ZVTsxCT($k-gbjUw;%IIm009Xz!z zcd@axiBG%sx18cshqjiQKLpjLofO-q{NDZgQ;=l)QLs1lC2E{736-xPRPz$3BbRxu z5%f{GtAO%jrQL`_ikF?KV_Q81Uw4JmK|{xQzKeR3C8+If21wXjOClFGX;dUB`YjnGyf!G0YjVFfNeIk^cYBke3DPBtjhZd0 zhf!p{JIMY58BHuSL3`0v4cr!=M($gaE#O(GINB1PYY$9K*-q0YbK$kjA+oTF9jzcd z?_~A!yLTYL!Xi5aH`;AeHi<}h%&u?pCq=)Qa>N3fj>V7-$%#mcNID6036T<|`I@fF2$od7aCq>sZ#KeEk45Q8~&D>er#mNP}xJ8ZnjerT7w zvJ$`NLYS>^Xg)G{FWp>hxB6MSJLKMs59wmJe2yFjeh4g_sGD;@?^<=hYBP(ynps*qs4@U;|JtV{zXC5b8&*KgW*>Q8Sb1FvJ z0G~2x`AfNmEO0G)Dv)@-%ojm0 z4rviRXJxxRfNWS@@C%wikjI9MeGv$Jp?wEc+|EWSiJ)qaKxhf`3+AaH$=-bOv;i4{ z3VxCQ;Znw#A7$^|Q|XF~6*oF{4zQ?| zLnaBbyS_7pLN{N^vB~b%+yGjnPB*k-rCYN668B92zTWiH zi9I(?gchH%q8X$g5!Hm2w_;#FMw@-46e|?~d`ca@ag^)g zU!|6Yf@UM(w7YI~Y?aS2v}E7sZZ{M>yr`7LBGC!Oc>V0QX$1*GD>ssWOTMr3BvJf8 zMXgvX31@40dz=oF)Et7)EBM3ww8jyC3b(BZIbZ*Mncou_D7}hmp!H0DLNnbp56`Vq z^1Kzv!8^}MqmGo$<{kth6ChuA{2{irSYN; znZK!;bd6q$wb0uHA#Qw8YQLr*X7k;7EbPIiL*QJUA1*QbY{RV-_NL$-yjmm}-hATg zdTurm!C?ea;$mC*5B4=W%_uOkrh4Hsja_IX$G)i4Ftyj+Z(F8+s0kCJ(Tfa2i!6@G3~(+ z)d(_dh*Y3JGQGdQ8GGo572B)vs;_RK3zf!vv73;vCqNlov3yv5dDj_??Se|kILiy% z8mwC#t&xNlh}a3ZsaM39np+*gbj36K*w!yLO}7KS{%WuSO{6wgt$#lbb;5!z)Ohfa zMVVzOUhgT!i}0Tv0_o%!8ZttU2sQ`RnVlvu8*5bLXsJNSi*8WkWoxP;ta}uE{u>oP zU4n4rM-4zmz{831GN4ud)t3fC!8m}}zPa%Y5=O=EiuMQu1=20-3rrRBAzU_tLMo<`+DDBscWvh+C0W^?c&}a4YG^BJGM}S`iAj-rXqvmrN3LY zraVywugYQ)TPUC?WLf4ULWq^~6U;NL7OG?6URi8n((W~V5YzNnfhg9E(h7^gw>sz;f3_X)zCjKTEmT)AnK#vGzd5(&GjH*2Vx+}u3x7@6lSuP( z7|$+OfQ9ee$VboPd>`}q$wx+HUq?IcyATL?qt38~FAzjWAO-hLrbD7JCj(U2x0^3L)q@ZLst|A zotC978D>FB|BDL#=Do?P@YglzUUTHa9>bIK^n7R1el@zzOS8XjbkC)~$K1>~+@>Mc zo48vg$w$WvdsFJe7e2qy_iz-gWxibL(kr&H7k)^89f-^QaLpi>)l2V}ZVZZ@jmR%Y zDefEv-xN+dm(ROuqjH>H-#kMXXr=}R@b0JHTw`ezT|7=BpTy6zEp{b#P zlrc5WWwVXOi~QkQ`{?6AIm{oZeS*4Q9!|%YbJ6i&+7rd+IEy!IM?cpX+x_f7Z3Df$ zy2E%e+KO-?pXQ>3l+zM=(?!62d^U_r6$1;kqprI-r1z>l_`FZizbqSN47TsF#gDVq zY@}RE{VR6%yUhVT!zA(OBw?M@ud#c$A_(i)P>YMI(`GtB4Wp?#7I3MP?Xs1&OS%-a zgsy$O5dJjqr-Bcw5Ab|)f~U*oBl2w!p1mO%?^U6w=Ut!Z7@$y7F5?-YtWAcv9xzbb1TMSv-&~r_KF1(Qc_X^R?h-A$a7jt$iYuM+6ND3J+{9L4|B?|@CjhD zo~B+|+NB9|=yHcxB%mLqwTfnytv!bs$|A2aA-7-2-jIAz=%W3dD&fG%&MG26Kh4)j z18*4>l1bdy&!O-PDp!^2Y8&RlWwRguYOcW|wie2KF+0dJ(SRwPSjhdb@!ZyC~i1A_fXp$VW}@}HGP<#hkF$G_aCNGI zuucu>Ng7^JSIj!x^yZz1+w3CurgMCT_Qs?7@zPS@6`3PcjaViw$)c|u-<`!dNYGLL zBu1B+6~Zdv>|b04+iPzU5pY}Gsf2e!_L+3)<>vrgP>O=nEMm&i)qZDMBfx^|ubH>u z+Fvj{QBAP1UC#ljQP!dg$N0u7l6m<(u@Ne_wF?I_;bexgn+oQBs%2nn!1c?khntB? z=?HY&J>ps0^xQE|`Po-%Z;9AXF`{UUwZ(?i?#x|aCs~3fbt4HlGD(F~J*ZgDkU;NrWS}8}hu(I|V z8lS{;M78R-oU-TvQ5%U_LR_9+BewzF#`D$ZC$qfH><122#nroG+^euZE%#4GDu4Mu zCmDX$_$G@Msd^a>PgHyy>7f~I)o?7>pD6QPfE?XXjm4VeIcCzP3|rwv+QO2W+MdYu z{`dy1f~g~elv|obbD>uu+dFdtDxjEHowf4P^d~8Zz`8Rz3|Mh-;f7D`|4oJXt^drE zLxF18#sFq=Une1&gNP$avY+Et5~om|*g)^Z+pNYXY#P6}Nq|hNd6Y#uhSo|f*o@7% zal=Rc<7n%wP}5e>J{ztoaPTJ>m~tr<>t&{I9sfufnP?b_ev8(Z!*5ipno#71DgONU zZxRSO)O3TyUj!MgUxRq2x$fJDYKeb}E&rwoc0KA)pQ3gDo5toA26U+7@vf{$008{& z_x+my76l}|Ay(4H|E6diV}Wwt#+PU@!rzKAZ39fY=dm;>vF5f0z~Sn`yKnvH@oC_< znQG*PN1y%PZ8ivJiUuK$=PQsRA|d5G;=%_D^%fvBX|$5}-L9gs@iWlaT_(`*WNRoN zaB_WBA^}g76cy2mwW`8+o%de?%rxur()Id$&i0E`G|70j1QM_3B@1XFB)6zE1dYls z41Ssmep?0LIXkNz;gz6cQCN6*4mzbI+9()XWxLbQLaN%3eR`z^F z>BdAy7ins0s=U(XH zHWsvWN}t9I`d1c!(KmI}f4{YgP(hKxAG*7Bxsb->@>3P)WNU!$7Y30L5(xBA_O|x+ zO7aUK(b1@P;Jntiw#?*x);du@0h;)cUT(U-wsZhUxn8}~&PVu-$r3#Uq1&T?&un@P z?_yF@WlehHSfwaY1MScDrt*PF=hIqbz}{2|=BTT?JAu;agu31c=%ppz!sT`C1bFh~ zD}qs>oBayqgho!Q=~BOgrW+fr8XID8X?s#};S>bUup){1)B@uYa0MT39k1786&Q2C zU^tKSIQ)c1XnuIU#nN1b%*x7oI%`=n z5-oIZ9_4*^Hf6E;Bh(4t{GkCB-%Ku#+EH_v;HS_)&nQXIdZbb_phPl~07I|F=8fCM z@o_dFeZwcoC?cwRB)t6)eu|+Ah}$;bRf~&k;}Q`uZdReGQMAb7&b|kIBgo<5(e>&1 z9oAo>Q-7^1CL6*1cL5oZ6}-{dtEhUm03`Ig6@$3nK?7)X5kVv%+~p1w0r+Ma|D>%P zG^R=!Bh-3#6mAv>0-3tA$hl=~^ISlTOWjiyZ5Z=Yay(HE^8<2M505@RU9unTJiYA4 z__!#|-d$0|Y}rIj;E8Qt#Y77g6&2kECc;ebyKA2Q9NhmD;$hRkXoYx9Q*;5I3SmDo z-vKhTWG-79oKXWuMEFVMEPT!=;qBq9fCjxrcU8{R7)eR;aC1do|?6GS#eQvd|)p^Rlj72JJ&_x(Z8cbjx63HNdVmOpr|1=2A*j7+6y>R z&lr#>M>hq4Wsp_(TQ>g3&`ewOD3@ypxi#88zB$;C`-VuQZY*^{q1K@oRNX9kUMB-w z9dK{nzWw?`4Ab^tzV5V_p`=?tx50H}vr+5qzUJFOd;*{97x)BTJAFUC)uOSq#o?q_ zwlI@aULa#GobEjD1oEpZP$x>l0Q(~ z-@yVzVn2!x9~xz!#vt|-?>X-Utp`lbF&@Dn_Tp@SF-`Nn2)Wrc*lwU40<2MIHYQ&% z)kKd70=={%qp@f*`ZnR>iI5+fK8cVwf1$)9y+rkAh~bHC?qxKod`a}b!wgPYormkn zDlF*D%jTk;Sp@(FeYBR<1TzbTHFl}R5M*LWgf2|;i7;#)(e5#R==uD6-MU0Fda!ovrE-n^_kF$J|>M^H{#W zsXciYi-pVl#&GN4hK`;-7y}cNc>Q~0;xeB1X@Q4D&*(-a$e=E4^Elg;KjJu>v-Q#H z*bBwzriZjD2ZZ?lXc%y)iWS@4s?&!lf^7$nX;vvqaFQ$73A zmO}J@+m__&Q?;mu|7BdVhf1U1RkTaf|EgSvk^A}c(I=XO%|UHp08=RKbBT}5!KsKe=vH{fk=--K#!6WzWaxfrpG zUF@%BK);0W20N&2kQewF)G?F#26rJhb1%O=Y&AD)n1G@mL-c}*`NUVBqSZ5T1mi^C z^O6}V7qjD;^vI>8>h(G;LM|KerY6;R^scI*LJ;pO zH8a%L(JB&_dC7%|#7vlU%$ z0BIQV^+vTKV$$f(Ak+;Dg{L6veTPlJ;{c<&j$$4~DkRbohW$t_Y>NKm(Uy3oFfA2< zmrxOJzEdav+a>kn;2$oj?xi`y(FDjlkDgB1D1s;C;VIhEGaQRbyp0K7P{mptxFTvH zPk(M(t@3=gKY0RR_V=e;%Rc#Hc6QGZkqX7WfYL`QufDL5#a!dXytNGhPTf$N%&NpL zUvsISn?=4nF~VK42|U6VG2q70l-w!9=;rk6zn*&kz8Z^wV87J!rylWY-A`OqF=ouq zDK%EPJ7aQu6FrE1eM4hL(E_r}jkmmDHj#A@3uj_y4+M@A3zyeeX@XvotK=_F7~g~$ zuB~nm`A0)6UfGOoK6i-nlcLnEnkB}T<9PczQG`8^0xVkcK;nQhsJ2SifL?aHt%b~r z6HN{qFwWdG67@ccaltcCl1NVV(e0(pznH0l1YjAow{GGQ5Ehe!F4Kt zm-aKr6|Be=uVk#GXnJaiB?@6-x&JyZ0=LR?PBNVpSh==aTHEX18u*=CbmKo7xjrvv zc;8dDUuHRMgf*zV!`F=|p&&2-gZqavct%7f-3D<;$O$*bH2Sq!;NnsN6G)uu7i9^s z5NAiKYWyOr$s88W7TNmz5F$KQ(8?aHH7V5KK_XEMe}cv7w0Ne~O!?_Dbj}X2Ua;?+ zaSdp&G`rpuPg(3!qK0-71il6n-HV4j`>3T#)~>c&c))dR0w*B`Lp0D%he7f!&yvka zLw5nh31fdq6xB0DE2~uW=Xc}Z7Diq-3tGN&cOMCkOj(}|jm_TyLmo?h0FKVqAo`0$ ztxEGy&nCysevYYtI=o60hLej8(4w^n`|WOYgg43wJt``f&4Yd6^#JxfukWTI*@J;@ zjX^t(n^Zjr*lKqyr*(8VHF}hFbR)aT1K>tytM-aFtvfjy`fiyR<2*HvBFiJLo~^dB zFdaZ}Sk8WeO;WeGIZty+(iFu+*I}WNTdm7BDeW465lMne1Qm6yPHTmSh!^AXjGFql z^~~k`MBPsp=TdCoD$ON-SY6M%7kbM5xKt$Mwhd{h8bz7IIr(X8oWkdTAQs#CTB&UyFz4M!F181tl1js!;K1KJCE|Y>OWi2( zNVB91d{;`Dd~Uu|j{WgT;hSx{$dHLxg^2v4qw8Z~al+E%)6u?I@sv`f_F$)zO*sNV zPdvS5N#>^0ugH~p-aBq|edZT)w$03KT{I-5T^G1a{?g_)E08RkEB0@lDJv0&ke(Ff zYLcO}&Vzz*{03py1-`%VfAz1WQVf6S!W>~`bq;ci>g2(*l}+WA-)+@^(qgnLDRo&A z)YUI~zV;--Z1T1u>e2gzPiaCWfY|B38AA^3*5)D6snyfd!ylFOjM^irQyw^G6YQ~Z z@>%Xf!=m!Os66apVammIAz-mqp&Gr0--&>bYb05q+^F=bS34F2-~-6EL5P2@9ZZu> z<>vV&u4{;{;ixeYZ1Y|%Dcnhno*hgCDGz2OcDO|(eel`(W}8PxVEgL+93n)KolF+KrLtjIu`iL2q?q(qXcTYzDG~D zLZXQb2AX<4qHcveB}RvR(*I6bg>{Oa>4X3tebM7kOjBHV6`&z6L0qK?MQ7u{x)ks6sFN=u1m4B6~`@br|UA zlE`4v`CK3eQqOAYNCWHfYs*A58jN}FPGfg&Of(04A-y_|clT$+uXLkGI6ihNxP}oN z@jpdk#3c3UWPDA?X~n`}Gaf@icoB$IX1GD0j#{1FMdI}d*xo4n3^(xln=fxES6~L( z0W>joyuq*oNbHXukYbhSh=d*Ci7_5Lj+;fa-gqZN4>#A^ICwTO$NQ`Uie-{mM31(; z`Q>F1ug?zw9XR*POg(e;4d*2;mt*R&bBx{)EPetm2dPfy-2lDI0bU0>_uZYXoh;Vq zk6`TQl#PAE-=szG`T zoer}^7*>{Q;Z&c?mp5|>noVw=TddP0ZikA1lw<)peL@dR4FemX>4J3A1JFeyV;}Va0n;wRuQSre|D(q86NAMNQZM?oB-O&n_j|CE z&ll6=qFK=b-87~>FF*V`T~`^3&CjPvfp%fTHM8A^fiz^ajBGc9W zK&#TH@3*Jb*v0Ks{$ecZnk~k*O3~KTL&abipRnj=T(>{Cj{LC)wlAdNJlAa%?DfX= zX=^RQ4ujx&R@$c7WQd}76WO(qFV3bIBuwnBt}8Kk|K+nuQXuNufriDn5!+qASaL-fAj!3TLo;1^0v@ zg|e!P5e51Tdxr1Nf8aQ1co7GB^+2J|3S5mlzUcDT$GB(;+=|hkOgbSK=ANpQ#++Q9 zGfod_RbP)ts*uQedeVFfDOxv0-AH}`AB1t*JYX}x7g@Qj(7HV_wHS9#s^zx~wSh7( z)`ycf)QX=cnRKmd_vrGn4^cd7abc%6%{VRD)2zo5xlwPm>(T}6HOi^tM92|=e-!{G zk0qh@5DE$5eQtT)EE*8_4-!Foqhv;gJSE($mP=-jaV_v#Yc&o8E8tT4D{r1Vvtnrt z^+xx1JbFD*@>$2o*v`QDIBoc8Zt|O|jGFf5fHtkgFFT!PZ|ECdHWcxQ<4BiELeWlT zl3ZiO`4Tf^SoEA=q?axC<9+ zEf})enFpa`{~vb8156fnX%>V^>=%Svcy4M%8qY&$;f7EcJ==($8m#3Fym0GUpNAjJ zLq$f8V?dkI133c)_6o!JWQra*ODXe_nFlT)EN&06)u z0Ri;PBVRenBhA(-MOW81pZzbEuHY34ekXiiEGt2y-GD*T&Ky%W~>k)j)IHsR;#B6H3nW0nVCXTFGX~2J3!sPL5 zQ0uuwLG`;ri+FUm)m7AGxi&!ZZZw)Jjrkf-mE@6CT23WD%yuJ&mF>c+PQ_taW)Z3t zgZ&7V+h}u34LsE69Y4tXW0wP+?0w!K*H73($pRrRs7U!Pi|Mj}l@BmS35siJ3(CW8 zXL5Om7ScJW&*EIWP;OFRU?fK6G4%}(Ur|_s?>(Fr!aim0Q`8oMBVQC~qffp1*gAg| zrPrTAMri7%Zw}?zREwIf5m64!R8Wh+4VF~!OpW9qbW)+WBq`VsbEh;Jb|8L)9)dG! zz#t@Kh}F%Jl`^8CjBLW?hdNqdw>*cs-tZaID0PVn5`AZE1nuGSHbQ3qjTAo&xukDs z4SD5UqenEnEnqe}^FA(UXuJ-PFRjjaaswlu_Trb{6FX4tfkPjx2xZ$8L7NtvL#8f9 z(%7!{@gd@{U5vw~*bUF%mFa+LCnY-8`|b&Ai_yTtEQ&&6lwBfBJXdpw=40>Rrx#UL zSHhMBFyQ$R;(-_}n<(n$U)8ksEr`1n3PM;WNR%t!>53F{?laznwqAI!ENSXaVQi8C zeHN6j95ys)w@e4&M98k+T*CeAPS89WGppPC_TflR?>=MQV>?U9YV}?Fdz?h+CC@u% z0-*z?nb#xjojPbXNf8j_B)8RskV_tv-0kBp3lCw#!PfL@?}Q!yYO92EX=Ji35t4Ti zgijtrw6XgJ1?W&-UMH(@e2kS#+u)oW-J7TLhl3R#%NxMz`iv(WP2^T%JX6Oo@aszs zaolpg(Nfdhc%hk#E~}H-$t-(@sp1_f5)$Twl~FD{Dw_Z!wV+vmncUpOd;BqMbu; zGPDWIrqWFP8;y+{tff)rFF)<}vz?1Wwhx=0@HXQRiQD_v=XZ>8+ z*|J);7ZG?&)MY@Pk4IrR*u28&%wWNa<0eWmb&l%`Kqiu5`=4A2w)nR7BbyU)vL_bybAr*ZY6^*=6A1%f7QA!6{8Lz;` z3_Di6-3o6fC)-e5`*B&n7DXsx-A)Qe>;^l^Kz-z0nr=%#_H3uEN-5)hPe6L*kP;Dt>H z8AL`#JdAVUP!%sFeP>!aQBn$^Za5iK@{0I%o*-}cr8tY997u0<)N%B7TeU>66Cg7A zeG<5F3rN&IHhqp@){JB}`0SXPrPa_Ic3?7wx5}8Su6WM5TqrJ0t@MD9C(etQI1@B- z(0;6B{a!JxGy0KwUxw;toW5v0x@D^jl&e2KUM250GJxXDpovI|baW_xhqi6tAn4RB zyy#Ww&s`pZKv+Z?VgD)TdWbrGL)u%c?HNHb(++Gx)bVargABwsOO{Q+AEHF?mJui{ zjJPl>mhL~b#%gbz=Mg-!T7Tefn&Tm*tuH7U(0^&=^qa1AO?Uj!9aT0R0ps1#SFx!n zE)MsL0S<)?=9A~D3ke<`*?~V=js2SGce5WXO0TC44SE8>XeN8}SeOHO3c7*5aHu5N zM^Daj71|s6b%pidWWFNKHwOlfqYntvDheY|dinBE2{|iNhPa#sPufY}A81B});oo+ zziC7b&vZD-gdBA-@HV$_x1!a)yu2)ZAxZmgtuBdNm~B`4yAv!5(X>LpM`D?RqsPin zv+0qcEaopZ^Ur>hMqp5qXdfCXAz<#TVP`VAT0kE=6&+*5{f2+c2ud>Q@SC1)a3+DL zOAGspagVhyv)+8DMp@A46Z-xLEjm0p|FB8B2dc7iB73O$QfTJGP8OV0-&bN7W8fSY zCsCv&c$!p3iTQeHB?AJ|U!dTlt${%XI@xpu)`7tV`K>-99Max2^J0P0Md`I`!-U$# zV>V35%+<;JmG(;!PI+hbwlJL3JPIpQA1u(so5p;u^`d?IhBPl)=t|Ui`X|=N2U2Pm z={KN7NBn8A=Ja&Fivo}2+}41e`J%Nn#aXT*T8m4^i~fw}xL}l@X82THond__Nbx>8 z8kL{8-W!a?EjG#)hGPWUac3)=%;nhx;e{@K$EY=enSYg+B)Cu}x$e+` z=HPYI+V$vA{#D2qQx$_o#u6YrG{OxR&hdxTkb)e-EIOsMRvQ%y*a9#`=oOx`k9bRM z_wf53t9^E#dx}T2`g9Ge!VIggr%z#ZugPGF*(E0^FRPB2D`W1YM;pEu^Aroh!;)laI#YypN~YFpC}!l<56c zRx`5muq%%Kbi3NN&0T6zWu`A#>VqWbl6_>uze`I?Q&e^Z;|))X`I20 z^~ikotE2fo8k^xs1@<_+G}M?fd{+#rPbrBssi(%+K@Sb}QNu2Uwo=DQFddu4VCoe` zoS0UhJ9!`>;79r^?0#;K7;VL1TD){vm1H7rP3Q0l3xeH0d~ScmEe z4()pAwp1KG!{Uu+8#Ru#D)D7*M!^|eJ5wwwe|qoHB9>`BVu-(bczS1XcNGnxX1a1a zg=fC>jqk`%FAkP`{dLA`wCQaDT3~*J@FTfqR2Ox1g?1i47}7ens3-ldtm}NNs~^iC zOICroABjyA$y~0Ypmvh!ZT?+2ho-P~lAET`#QeHJ;xQv^^(!USM#I1_SkA4s>9zSf z<3vJ1D-EAl8v`1pj^EIqpzcy`JRGWznYv0g6uX$0xt81hOhpz8ermMz1EKP5M_6i6 za5AG#EG?lE-=6n~}Q)OmM54!l4o z(3loGv1a1<<5<^VW$R8Gw}Cv{z$p@I#0fRP3Id?1PrO4`w9jbgJ8evW3KfMY5 znw9v*Vdr>eH)f2gCtgMB=S${FHUeXY#Z$EY>&zKim8E!J5r}|vZKQ(#2|NI3yS}BY zZyqm!9&o^C-tSkpE(&Z&)>lQPl0y{xT*g%6>a#l5iz?xCx1zF8*A?L2`;d=IZ=!6o zb4!us2U=FY9oqRmYB`W=8^CrKSIo}x(9ubDvE-AX@y|9`cwvGb0OnURb;pAV%3pbHv-WBeV#{O3)b^MwfFeYqO7|-8m6R%Bcgi4&icmo9@2VdyH;mI~Z974k#zzSZ5#0}fqay*R5!rxP!OE_m)P8Ob z<9@L#O1Cu>o0w0t(BKw3TVpA~=OQt-IWT8&lqvYS!Iky6WQE2DtfF>6efc;ogU|NE z8?1%;cbn|u_P;_3oQuiocm83T24~+r1nyg6hLC{uBPAV_&;neeqh&@2r}uEA;nbs` z?l{BF`g!J)??(Yw_FGi_? zu-EpCS&sVeTjTao7{qM723^%~9IP@we|`oUd|?!8ohhVS6>`1;4wPk(GUar6q=Ym6 zCf{}Pt0RzlQ#y)`rAKCEUcJiekWm}`K-onb<8|DsM{->Cb|!8Wl_)z$3Ufap3-{k| z{v#~ve?+${@ObMB;9XtgKHdn}uLfQY|TM|0k{GcPkGr=EjAB;#>(}8?q=NFz0|xP7#pB z+x)h)c2~zHyCXT4S!m%`)yZL4ne85Ds1X6#r9t(rHBS!5UeY5iFHN-t}d+;4CvFY7Z;Z*3JrG- zkf`9EV5p#{{)={gWOe~AH5^b;Wpl;op!lF%=U`%ST2TK*EMzjSf9cA4&P6GbV8R^a z1<|s9uD=qUGU@8Gt;@`JKwcH-u}aW1rig6S9-QG_Gcx-AHC%z9B;RaT-v>s3~a5vr@N8|iTpEF>Z? z03oIC_GUwYx~j=jadB~0OwWOlJ>BD+PU~cdx2@5bX8v1QKlH_t)Y;xd?$4c0TL9_heryUqwu?BH}ykp1QPsr*RH6FsbX~n=SCr?qs+BHCf#?={i`c0BB+Ag$y2aMF1 z%|F1hhu`+bdCsv9nC*9pxoAc}eYf~}YJW2S3X$2b+FGl7x?5G92BgN0mx>5@=LKVM zitss7vqF!*kFx#(Jj=hsK&SR4u;O^K8gKd-bvaJ=(T;z8`Tq!e>!>QXwGUJg5mdUn zyIZLAtxUL8YX-yF)^{;ZF8G=X`tL`^Oz)4P65ri}k+mjAuT-`l#J3 zoLupf&;`in3Qb3BRQw_DV{R_n-)iScqPm}j&!m?3cZkATEH=sqp4VAS z<&sG=^_YqmA_fG$AS3m>bIjSFdMJeC%IxI_>eImOV|QU;&9|_y&c9y8k^>J_rPjvg zZAWNH`oBt+Qv}~;Cn3sQnV!csyBtv^@(+fsCDOelEHtHXc{y50dt zZNTz-Y-dCl0Y7-AeCcM$iy;CKJBdZA3C5Ye{8=}1eQ-~ZcTbgqWWuib+qxB3Gimo5 zsVQG6un8zM4mE*%m$!iW>`CaCISQTMP9hBX3IT4F{$!D3qq_5|t4kQb_4P#xx7D~N z^$v^;JKDgiA;cg!ZQHE&+1|*{4}zl5#AjkF@7TYpkkL_`&tkmMm-}k%2|I*%Pbg)2 zh?1caiaOzguIeq*+l{8-*2B;HYKOZMEd|glfATEdrCvbQ8Nc_+iVg~r1b5TCFUELV zfl8&6phO7dJ>~-^u#)&K1Ch`_iIW)}xdQ8}9zop}^n;p0f__Czi7vP2=JVau8fx)6 zP4;!~Eg1joF12v+VU)<{1g@PTghaBJ{p2W~SY+UO)A**;1f=^Q65>PgTTYkO#cJfd zqw@W8t8Z_~_i!&3n1)!@Y2@^{Oqj~+3D`ifu6a$6ARsjEKbZYqf4mZ>{U!$R_(~0% z(!6J@q&61OA*A`sT!5(@%aCbjLopyGR?+D76|i1qOIwlJ`Y*H0C5(Tnn3r3osMGU8 zd3u$Pz_Wb}<6E`9z^iWfr0J0DfnbCF23eo|`ldNx+v0~EK+-k>Rmtvk zaBMBGC3J%Wv95_3w27qQZHx-IXu^QRC74bOcH&LbKDT)g8q>6R{)U0?-6mgL%l~2T z$-?Fd2*&-cCo(X98$UgcqjR&13oL^!c(Rz2j(wJ=XF$^$0kR@%m9^gkYI4yl$F4kppSeP<0D=uyKlbcU5&bw>7C4@^`wPl8ve z(2?G@eYRGZn%(#3sxpASU2{82oSkYdQG#KLBX~SAk)>W)^bvFgigqsiTR5~`5CR!G z>MO_)gsQ>KpV_`xD`DssHQ7N5$vDW6S;lVk3ph9EAkyZ6y2bq@PXdG&lNP;(e}9j; z_8)$R+vu8ksS?TVS z`%7hV=sb=A^(31-17oCkR${Xco~Bm$wfm(R>gqL7zDMUOgdpbt1F-yh1Ki^|wz9T%W z)b}s4%LzQ6Z6yCBNv%?z4Wc*43?q3zKI`tg)Z6u{u0lunVuGPr;Prj2v0j(uYs~W(Lft_LA}0=A8!G(=6iFdm`kh^Y!41J0M|^V ztH?M_`h;{L&8FL~f7Vog3vJh@7)j&3yQg-3T^nz8!CKLJ3pLW`-jDlzNWLV2cDL<{ z&(T$^(jxhMUg{LANnt`+q()zWF^7&>p%}jRebZ3&Ky}p9bL8R;j#q=(&VS4;wpVHq zj|**?_nDWo_B=+u>JwRsdW99NxVeNtU*J3(G2(yu-nlFFl*}eH7!+=QvJB#--2eE> zt6Z78X-CF?KO{gH6l~ZY!X7w-+jt4dJp+7CsO;)*^|%Z41%Lkvs~^xty#LHg-qA!< z3YquA4U|YVom6aeI#4=mYZtk(Wr_uC53JT?@HEtNq1)2I`Hw;SQ)Sc;45ifutmr?v z%_coZW-4!)fX_g;@D;G$kfPmf)O9^k#zeT}oFl(3Gp;y!uFvrk$p%*?P5~_+YesMhG z59|U^e09n+e2D;I+#uZS=;FXG@x7ALH`T^=7xxBG9&f@!AZ%koP}O2=z(b!g1o|=@ z1MWz_fgUhv*&7q<{bgW=6Pbg+Fy{%J2+CMKZ3;G{(A=)KsED&%&L5nk&FpC{dv`12j>hgWroy#L^pfc>5OfxiFTn zWr-fbm_hLPd6>+m1C3?MCEKM%4L_>QB(COj*#|X3M?YBg&=8C=`w^N6-{5U!(rS-8yu`mC$&eH55v`wkVnSWNvVqlwC~krGKe9FHnrrGHZJ zysq@rX}mVVT>x`Uh{g5Ki<2euT;^rj)DQYK2fzY5U8edH-z&#-I5iJgXcX|$C_Ot} zToICaXAECji)y=y;k)^P;okCbHWd{ z&0?*3Eq2wbokN@jOegeO7u=f7Cs`D-dXS!-_448F^nL@1eqSXfrFv#&w#875M&HKx zJ}*@^I+r0SGCZ7|!fchD-F8{fL&WH7?=kjvo27aoyguGf{DwH@)%;)3&%?&ORjg(T z6@??+`^emR4aMY0D(z-Tthei1fLns`ItGNmImhp5+bEHD=awZZ53#&S>D0apqI$i< z6kZXeHBM6^cCVqRMKym+Ow`uY0=ly`FqxjNX$f6#t$e+O@C5`VtXnz&w+DlE%aZru z4j=hh&vE|zcmZ#M4ti8ka3FQvRgZt7T=p0*n~Vn#Ubrg(T&{pL`JnIj6#?Mo?R%SF zbZM-A%0qp0s51D9y9uCaHds8CsLEJ6mIvO|srXk(2TJkkX*0H3i2{@={wEB~eq1T; zm8VU1RL6kcShDHps5JDwofhs_+)r1$p1U`{k{gUOazSn%B(b@m`~RVp^-T5NwJJ5N-? z?fDFw$F}x6blUB^rL)rOV=-AiRz-ZI*2`O*>KzyFh_)iD;Mmx`WMZ_SK;g7D&v3^6 zN=V;UqEDwrU)>6^59}@1#%$;kQ8(tXF&Z;2i1J;dBaHhiMcOkS%Z>)u5X*JiiY3jf zPGdWUm~_w2EC=5nEL2%fyblRyfy^A0@+UqF?CL=dNbee4sP2xN)X>ij8g6#Wm1H$##;XtrGjh*^t5wOG>sVuRSTa}Yv8Lc zy>wgq`I=Q1RZ6hw2U{-V8&}jTL@5br>5|o&(oXjiB0Z)oodFf-oQvRK^(@ohl4&sM z=j9usigQ7Y7{9*Aqj>&ix7zAc{30Jt3hna(+UGki5rM=j&dT3KYIztE6kR;|qOh{_ zEw&kW0gHDu8nwc@8$A*Eyq*vrmiHhcjwI*mqdKe_8uh6p*V!QMP5-MtO zezvC1@hAWp?BL-hPM6}(#s(&;E~K8BnINxwbKaJl(jAE}$bjwtqi|cTuXi#;&VYQt z4kJrp5Z0zl{A)ZQ=Y7(9WUM5$OQ7|(t8<$TBSi;KMlsg6Y? zZjHePaGvd>Ya*YL&Uyb-6}uXx=s)3){*MCI8?$k!)H6X!onKlB!#S}unfPFauM!E2flPBSCrsI4QhuMaj^bFiKC z#R_0`(s^c-9z|PBmkJp6gykaROy{arynxjwx>89%!najmQz=oAX6C=#x+^^Q9+<9J z5hfOWKlKOztNMd<6VD&`p&xj4sOt;u*b_sQ^|mDAr+&&t6udecEHZMkO`Na`Rp=-^ zQs30VX0`ZU1t-0-()z(=v1T*OpdbQw4x-VosRje%D`}b5t1jb?cwmmtEWF(R!Sc5T zawpwB(JLt6gd-)~g}GEdoX{3yHyt%#6-*BQ;<8UG)O+V@nJe8;C`b{g2gArP6Q+B3 z9d{XeX^%_n&Phzt)lek|TiZNrvBh)of2F%6Hs1GklkeWkF-?b%IBfq}d^|F4r?TS& z!qY;PetW*9jE!<3{#6;|sHU{9TdY7Li1DbYo6)gTpcFh)DWDBoLr9p|{$S&kEQG^r z^+Mu9DBq)6Og;{FJq&-AX-fFC_yJm*8--%u~bIZGLO|hHJPf{k+ zO3(LqpO;kZn6ta>(?Ss;B@7rw=m z9lZME1$2s1r;b{+>#_K*Bm|cuvJT6&zw)@kWif4qT}vUBFk`%Vv_t7fH?%Z3Y4wFTxUlTXy3gK;t>8_uf zdzN8fVW$S8LF@xd&>O@7>%EMh$lCHh=bFW0?6!2I$of2I_r(luKHnWix-Xt?uCR!P zG>`PRY0AQ7jW;QJz-AEbm{Hp6lDy?v({G%?yqKXK7C5?Uzh1W-&b{N#`*cqi4H!x0 z^}1E{(Ln~;AI-Y^xP{-t&CD_JFh_R;8(1eyJs6o7q9ithK`H!lqEMeFMj{8&ynzht zeWIkt1I^X%T3X-Vq_#yqYBhdC9(+G#>2u$+#jaYFEwN|%TZ8yvi=Bq<=`@+=4i;Bo z@we&VH;eZb4gf7?PC>h`k}C^)c%qA!6K0kb=POEo3@?mfB$wDDG2rsqN zfmEIwq{xUBhq08??p0Vb=?+*dPh|>tO7{@=5_VhwE`BukhrWFm+hll?)_;(^Gy z^^>9kyqVwMtOkTIPN4mL_;rvfoex=tjz6l50UH)YV#EQm+U8Ye>4Ah6gS_I}eebms{_ zzR-x!AK@h>4(CSo_wiwch`H__Dr4$Cixq0ZC_t}*i$54jU=$f|sB-Z*nAMN#V)Q3@ zmqwT)3ODVu)iKC!9KhCZO{<-%ZKaG_e9C1fO{K*DsgmFmI?RUzyU7pPO&pMX9ow6S zhtdGT93En5+V==p;uDFE(?&FoJFjW&VCaeBd=nJ29TXxE01P6d<5H@N-`m37HRVcXADkoQtX_i1Qp9oe>8Sz zeeeKZqTP#DP`oTR$Q#P`Feu5Ur(nV+Sl<`z?;@-8GxsrA*itL5Ig z!-&gnpBG2!AwPn`6d|A296`w!Z`%ZKf2>ssWHKN}8DXCmD-~r^of#N>{1Nf)jAL!~ z@4IqD<2$qRE%rC=BhwRmxMpfn_s4P_@`CfrlV?0U?qMzM;tNMt7et z{|_>P9}|S&htIDlWC*t{4JczBnO*b_-bPbOuMCc1jOTCqmh)whpO?=E-oMsB@Rk0* zFW&`FakdOWSpMh3{{F~+{wcQ#HV4&JJ|Y}~=ZzeN2IA<*W^+xxP7fC)0_v|D^4GI{ zzB4Z%Ll9qfjZ%x^v&o=>8}$c?Lc5uT=X}+O$2a934;@`>q5e@?e{IdpOX0~~4+$L2 z0{r}iN+Woj4=AW~+i`}H&U~lbQT%*pAy_S3722BH?;qxEN`kewK<7H!WC+XLAT9T| zM6LbQ?fZv&&0G2I5V~6Zjr#Xa6#u$7-=8us(A)zakh@O&!FPTKflAdV2hPcgH4q{;SdyU0s<5)ht(%I>+<9)crVt9SmK*HCNZ{A<|62eU&(`0CWb3<|9D684! zSQy}|vJ=Z7whW7@k^aVYhUm?kH&oi38#NO0IQ3>@ajk7_=)Xfw8s6{z=1|mJD)M&bYoH*IXG<4EdcRU6F|6?zW|)Oa&Is86~dEQDZVbKX&g(W zMMknuU0!5;p@w+|?3Dx*mmlwjN-H0H4WK*u>Tec@CXyg6VA4e&zbH9WV9MLY$$XHeU{p@Pzcz^4I#`%nW zNf3c#I}jwuD<~j=sKB-b9CUPaN$EC;zCH<+UQxn-Ul9t#T+0M4g;@B;-#0vyI4Dvs z5^pK)OOBzs`|lxDwLO)jc*H)dI;G?YJg$73s_ZC_t2Fkxhe|&xQIJ>nF*-ULczB?x zY}REyd(dB>7&hs?9dchI#T{{tZMSWGWKu3s8KM}^3xj>>yNHCrQpw&;P7)PbQ{x5S z;%FoL1|ByLn95HSM*FK1s>ZZhPE7)OJ2?gE7F0ZiykA0vnxVtY!|mlJn=oNk!E9SY z{gCrC6%LN1LdT=$-oYHjTNt=}U_qj38TfZ`@ZVX{`1^S-{KkI?_t#La7UPez0?wOR z(`A|fxXpD=|6s=*ta$_)Wn#UN8KccbkvxJ+%}(+qobuQNcsq6cJ~F|ePf@9DSi?`qOnBQbDsr89;f6dHa{7fA~U5Ec){`hg&)LTW=E=O0V; zhqjU*TYDr}k4!~^(95)%MC0AIeVNoxjMgC79)*b=q-KB;b{NK|)h1my54V z1{-C*&|LBGNJZ7qOB{7XS``m~DAzt5dn?`S3^3jpe9%d!cmy;Z=`8(%SqN4tv9~;i zy#=yqL~a~4TY8{3^K2XH$VL@{+>HWP(ssJlk6qwr4BXY=)$cXy)(_V39QW2m=G%E( zBn;pFpxi+Cdv$%jDj%?Py1hUY`s?tTMpWr~!o=ydPSP?gI??3Nu=KJ!{E|cyNnKLw zGdG6=n}*#F&+WOV%6A)|u)ZICdG+QzM|b;4#&X7F@V&*H?sQt(fT`SS8lN|zC$p75 ze*}!MNE-7MNJMXLo=ko>UiQ@~y{kvF&fR}nSU5H7?g%%pzh%CCdA}apVULOR%2Lqg zsC-^@+F$4=uyB`b9%HgmIVP)K(TlFG_lLRsaX!Wa zoWKfxi89#mB%V^c$ASw5O&8)6m#!BkqN`hzDNw4k*R@^ec{TZ8}ZT;P}l@fx{$(i zs!J0>LFikv>EB;%0@q65_?Wy4^dqrw2p~VjnJei)&kuFPq*DOM+%Xf9oPnBuFU7a8e6ajW8UOPzcMD zN+6&VwaLon`|luo{=(@<6o?26m1=X>z2sdxl>G0ZAP7FWKsSD9Sxm#-Ing=WNG5qX z4Mfw?YvQ=L1J#!6uq7cdMN$$H_46Ybzor{Sc@hYs@FBbWY5{pNd9&O8vt8TJZ%A6@ z5?8s5c2D&_ToFQ!k3RZpyTJBm_ViN~-1Eq|5!E~YR)x|%Q%0M@wza@(w0hrY+Bfs6 zXBGUTNIJiK67u%Bzh8ikOkWoM^Y=xOgP4wUvoDFF@pkm*#cUc(89eodLUXc2E9`B&)>dg$Qjw@`g%6wv`j`su24|q z+)x4N$L)`{!#>UU}wq1@r6{dRXw$b}pxeONd>E0)}U7qTNP ziZmRXJ@*ZnY<8k)hYSe0V1%Px7@&-NOaa(~=*rU{hx$7#T%+`q)YMe{Fkmet2hp3l z0;FwYC<6wfFC2dUyzuQ5q>mjLSm_|WxnUj36c$FuWNG8_Hq*RQOj%@VyZT$bu zF}W`=anw7)hezf@1`-LXfLV4NL-7_C@pUrAFJU*6Rs%H@p944OVWPv{#+#aD^ae^ z5~ot=LRWXU;_9;JEsF3X0;~BVSePNUfX_QH>nJClF5VIW)Ixn@aS3}00V_-=C0)dO z$6f`i!wFEok|67W@@=X}F(so^YwRSPTrj~<4;OMA`H$taTBz6k-qrhQr_;96Vs)Y4 zIXUzSA3v6Q&EVkR?CwulZf<}>1zHRR(fWH3sVd*$Ki(*b>H=$Ye6%fzO4TeU{&laV z{MC}XgHxICRI%HxEr>q|@D?gk@n6xr{K5IpC18zobxl5gf3jQ}l|%C;0=MOQnW^V9 zQ+oQ;tWeR*yj+qqZpx3x^nlqR88g=k?9x$CP(A|P{`lq2ux=)O$Nl``jJ?RV7Of@= zd2e+Dpnny$$JsuftH<|(!r6YWLZu5CY^(mQ==E9qzP2i4)qrf z?vYw`#at`o!J%=nHgHr@-Eo{e9Dj))Twz?W_gb-=( zS9A_1pUlJyY?t^K+h1Hzy(66A0(2rO5qVCoS?ZT9^U(>bKE_yi(BGTZ>%8Ckd9_bX zUo0quV)^XeL*ITMa(OT(=D_#hgNoM|*M)iBwdeuzxz7*5B(5!rERM-_w$?p8KhF&a zMLX*r4N5*ij@e1Si^MZFxqA=2D`VhJM@U3INBceB14$!KXK{2ht}n zFz&>Tu^H*kdyDCCc%1zm{tuk@h4TJ&GU*B-M4G~j@Lmsx^xD1H{~RS_L<9*=HR;}> z_}l$B7K6Kp=b!;1r?;x({&`#JLIr6nX)(4xb-z1N$B;he<=ZTW0=UWcB_5WYv@7NS zqCmJBT`%9e#3^Tce2wg`vJ`E)Xt&tGD&G4)iY11sMk2FkBYb5#!Aiw%T#);JNLrVvcLuQWKIO`ym?zAwA6~&BTw4mweJ;Bk```s_4Mqu z+TENg*RDneu$Zs6DYd!Ru^6UDe9c7H{md$zWHr04o`#x3EC!c@$(*r$=NQ{NGh3DO ztHb!6yVd>%=P;-bu4m^b3>ERRT$UdQI%kQUj}wVXw4kKEl}{O)I?ql++UW8hj@tY%I{cw0^8!=XPN_Ezm3#ku_-U*Ku0 z67SaWSnmS_;$;Fr!@7Z*%zeX~%ylJ|%%C5}D+2gMs(a^q&~1MP0%SI~<|_7L$ZJu>{xF5NoaPWR$t3Hh_JN|4{}@0;)5)dG9dKe24BzI>`~r zGan$nWZF{QkeaSZ#&}>nxh|!4sWR)~2L3ay^6pgG?~?x3@Ra(Zt!1CbwrkNI*nV!( zVD|rOPf#R7%A`UokPas?lhNmL*mexVdvX;D$_m0`_Q2U+8pxBS%K$P2=1}?rMBB+W zmpRRrusJCdeBi-~chq#*3P@B99}nuTXuCuU2*}6={657M-nKB4?X4z9Ifjb|I5&V=!u_ zdHHWy10l+cJc=_m+vR@vrw-FjpOEwA7Ta-PA|kO}5#=WU_D#PN4x=3k1R;8^TAR8u z)do7g0C9Qabd=zv>*2JPLYp+s`D{4Nxz|Ta75&_l44|^XLv@nb;CYWVtGf{we37p8 zy(XsglJa3|Zz3eV#EC+lL{d);-xhzV2OHW*lpxtwm-QhYOkQQhLr)S!-*h)=4>@lWK>MmTbi;(suo8+D8);I26=d zYmWWV7rMq{jK|+s8FN5!nWxj{1mcb55j?3WrKH5UK63Jc!kde{tWehb1H{lcY*)Xh134WE)|u%tez;R;nJx7f_rDyQ^U1qGyy-xREk-;Esc zd_F$ToeS{v^sGy(sbPl2<@~zg;4&7)Q)e5Geb;$f2n}>>DGuTAd1Zzk zl1G8%%Y})w!JY&T%HKfT-*q7W7%79-qS^iu7k?+}v-Q>zi*1mH+wa3Gv(e1QRN8tj zr~O%9;EyPs;e8S>je-8*D7Efm1=xoQ9B?)k!Kr3)cG!v%JcD=W*+WBNAKSJDQt-s3 zx-lbAO(Q1@*NXZ%gQHE&^ML84U`6{SSvVGlU|VaK`yr?}jY5BZ!kuP%SB?=x@(BaG zGho)fBk4q|%>);SHj%MnD>P?hN(sV-5hi4MZV%3$#<43#wq3oiY68TIQHo}nUD_P` za?j)7*j;T9Ffony2ak~q7Z892t9E>s2U5A1@OCG#nVoTJMa2Qd7h3Ab&vfFZGOvaL z)_IDTRh%*Jn!ll8uMfz6d{3GIseW;J$y7Wz5KSCp|GAp_d^c|I9E4v@P_n(0rLQl& zsGctfi~T9vNI(#!2@#`ad=pZ~`7!yxh;*D+z1Ea3q5M_Cp)vltAmK&!u4acmHLb4A ze1S1n>GK)xzb_P62&NH4b3Olb^ZQCDrh zuVR&D+7HijO`R+Pmp+|^>f>JCc->SSbI7k#2pZ1J7+t>rl`D2Ur zjrr2e*;w4SZ{I=@$3O%6e~szPOoZLZD&ipZB0XN5zDcTS69WPAJ{*mkyOJpl5^SNb z;)4yHslJ!b__JAz6=7f}bg+19oznpm7NWj|ZW=O-WWKTx-It9hpO&$1(26 zS>vbu>9WuVCfbBHG<9Pj5zD!C# z2~*abM6wkqf-Ih3Q^Up*#Br{Gg}M_k%f&}ZwSTtrhYq#fSrrO#3rVW zjt)))RR}PkuQ7yVX+nl%Aa6wmS#J2C#qdLMeN8&!vao3?fp%fnd*!p3u3ddP%to&T zTfRb`WOSUAoP2b*RKi|>x&iPwgML`rSV5(NHg?s|(DP0&+KSH5|QbaHhy;d!*=A=hvMItl;7 zB;;IS9F5s!-7rtV#%h;8+nxGFpMt#R+wj{`Bjs(SQZt2HCo^yOAVWsuTGRoaU|5^R z0Vt>=MRqN_?zF!kPc6B>AkUGbbkp5{r@?8rxW;Cv^MiaRP_b_Z^~?+&((FhxdEHj= zrQPYKJw4+WprNHU^6aHgHPR&{brg$iJ2W=vzpR1i4=-Px#)B#Wy zY*i;k@P@^G)2F1S1<-hBC7z={I~;J_T0by2()^}+dYi2DO?`D%VATfs6U|1Uov?!g zj+fUE$3Y;w<{>>mNq65pKSF0@3?362k+kq^+LY23rFkAXG}yD#^zP};qT4cXaHM6! z0lAT{$gRZr8mgUU$8|_*nv%z6=4b08#&%=TrO3(2Ykm%391pSUvm`DcEQ$R8?P zj53IZ(XzY`g3K`3cfW?mr)TFdJFCmW64OS;s_CcCGQeT$X5<6?)r)s+TH}NJ@L%fq zKB!;xeM}+`N>P5&N=Ir&SA>Y!6f|I+M{-k0M(^jwT=CsFiKkIk$dK&!;^B}9UL5VR zv&a1R7D}dP$!nO*mB$*$ATExt`4GmIFmO;Wy~6uYzcuStm%eO4_t1EWCv@w)K23A- z2H#aLrf2Nk`^6M5vzCU5NNX1{Y*^#gR=kOxa=Fx3V5{1GF|U095wNWNB~sg;-;T8M zcSakqOVUhsT`_ZNi*NjFfN4guK~b*tY5D)=!wZP+6n>Q++{;nK8RKAL3cEkR)6USN zITQ97%LE^a$ARMBWcn2mEQi$+{4v(QN)+-!l8uPTdQc8Nq;zKfuT%Wjc7;L-QPIIj zKC^Q+aKFMrrmKSH)3l5T^}<&R;d?3r`X3vo073ez_lyvr;|B+$1-ks5>AU}&pP=a$ z$*>a2Vh<{#7efAjh8H=has%;-`OZ0P;ZEBn{`sCS?} zx&?vP{Lfz2|MuADH{a99d@$Tyl0RI&dHzitB%`?!hWq19uRxzW9yHhl1%43L8ui)t zm#9&5aML)xdf5U4Y88o#{oC;#>4n21!!rHnj+%OnXtEmlo;y;RRvoVp^HYMCQdRal z>)A$HA>6nBx2u5dC9@LQCqPI43DWK!{K9TOABwYdFAFj^H#d1RGmO0adwzQ;7Yquqf zgG%99r;>6f@)~+Gjz6C2$Z<`OrRmJ27D96(;LDekAS%Ut7PqO=tc9^L^qV`JlDFL6 z-h9oUIU>*=>J}HL?M31ifs}fEMG8QQ0Sh=%Y-(@*3$uYHDFr9wrvgn!Ua3?VFp3b# zKJlDz29w569D)w!uTS~RrrbSf5<}|hiN7SnVu0rt*ItU-$^-~@1OdVN7Go~JD)W#X zbOZLff(cY+r}D`0GDNGYsx}V}N<(<%cLDD|NHh{p0QeuP28e5E;ZP%)rl9&_I!8bt z{37%fc;%t^SrDS>&y{R@hBjw|Y2KMUPzUjW5>>5ZV7bmBrWM>70ZK~tLg8Sf+Rxos zDINzi4de}kZ((@4kJ{?2>xEwbUvUSX4~hvGnW)>c@V%fTEoTK?s#O;VY+Cs%+E0lN zms&E)ViQJ-6bmlykIfeVR>pUK{SKHa3k(2ptpq@(3z?2HyqzbW9*#X!0!bzST{atY zqF~%m(KLB3jmIN>+3kuWyqnxpIH|$yOhp+Qocp7fJ+7p&7&7U>p?!4m4K+cODrD*` zljHv{lz4P|1;2W1P6bk$Gy3Bl3`8!ZIdKssfcNCoEog_?he^0-w3!f3oieFkZB5x3 z;QZgQ{Q1F&Ln3@GBJjlefcE_JN&)^w(+u}pH@L@+K*JDLrS9bpd^b7G&;=_&!B+Jo z@_-nM9MIa-=&+ijs;`%ttQPU6%>(`AqH0;{o5O;P9m_PUyW3lfDz$^>gA?fC{`8D3 zyezXX14qYALJ;;rS2<;#>-7d8hvio4VktRX36wd1BZtmQCLv!x2({zH%C2uobX*>4 zD4NYuhJ=Mxb3jr3dy-NhfJrKUjbaD)_ayCVzq^=uZ9&NnXrdsj2f0kWCSvi!vbL6% zAQV)2dASCf(`9NC9WSrUOj>Gx(f~VLxh{dIeK)rAc+| zvOY{ce<4AqrONM8tBuBZc*=Dj7tvs0-o2Zd0(O|=pz|7LKqEk~0xjON{jpo{2Rc4J z5cev7oX|q?0jOc;v`6)FwK20p8_BPzL(AY`9CoHztq9bAKkS2N1mhrsAOasObZQ6~ ztQH@%Bbq+070B4xhcU4j{d&Q3JUS{TsYlwoD)4IeBuNo?p`-3AP}4PoSb>7;Ye@I& zHSZ$LGDUTAEMBWhs;hFXDw{Hrw{4A$u2{_OkfwM}ImqO)_C#>>At52_Rt>!&uPsxJ zeh?D$x9_Lpx$|e2E+D)5e4g#D7rPp+bsJcjY<7Aly=V_GV32G}GUOY2SJpLociD1d z-e2|dU%QKl?=xR!O|gsPd4jq@fUD2;^7M#R!wD7^)6I?KOE2kXm;e3%#6IR-fAr56 zYh;V#$Je_DC|Fohw_DHfq|-P9YUX5AUAkfg1Qw4SaD^7JMf78m-4}{wY3_BNtFInU z60UM_B29MBiwo}OCg&!r@1*8jqOq1bdo|;LGqiUQsYK7_U9Y)SYdChpSo=fJWS;Z7 zXHH#N*ML;TU~D_WzwdZIq$DKd`_@BC<+zda>85UnRaa1Q@UM}tt{*5|%lZKrVTMGk zw>r6Odc1BL=}Y5MNNxqac5^M&@pz)L zr{G8o)bumLp6P=eWmT6HOdFsr6KBtqPaRR1Iy5_FE6&a zlfDa}f?JhE`XW)2i?{iQ{~NhF3p^07vNAi?yvkx{Z=Wp@dz5$FeivP?-C}a_{U6xY zPYa&6p>KS_(MSHu26yimw}p~$?JcmD{<>JwI2H)2Iq?Dn9+QEzmo$(+Dgpxh*vQSq zC%&??eOkQ*j0f7wGs}6ysneiJP8n$ywEC+(5;+ODp`$(;Y%$5H4Y zE(ZndJw)3e+~t7z=)`*m;4X1{lejV-hl!owkRDAO2t1)LfNl3S2Ix~I7?PhoSgDy| zJhy5$&wg2F;L#Taqe8-?!J&U-VYSn2Ag0x4*mApwW4Ta3^VU&hnkKT8wW(*RHbhh;G_|zEV0Yw3o?rYa%wA9+6F88HqYy9@h0jxY9O|2@ zi|H|-{=NIngnJHSv;$rFs#H{m)zYX?=;c55Oa3*YD13*gNZ!wh_;6fF0+SWGiCFA5 zg>z%1Ld}0;a9v$pqg%5J67i*xOFl6%vFCWs*MG9k0UR>!c@T1pGIgetoyf|`n zX$B1u@77LoM&Sk`6^iH$HQ&<;)35ES1$H59$Vh;ooeHY+&@cI1Rq=6WB%=Uy1kKyZ?EA3PGcw4g^tMDs`}E$v zgJ+MN0rlkDw{HmtGOuVgug84-gOoAH0qvF3<1&>>hMT)sL0Fh1ATaq>wQyB1jmwDy z9o>}E{u(;c@&P6WM|A*gs4p7z!yEYIh^fyn)Ly=P2LS@;iwi*vC6)K_u;%1974K)b z#Lk&cHb)FL3zzFsPVQ4-Q<~bJKG>y@_}`_MiGiSU7(FgMc=u0CTJ9X?7k9qQ;k;6G zP5kj}RjBX}ldt4_o7S_HFFo7=>P8;6!)03NaaErD7wpEZ`j%~^VK}XWU)SrVZj5z^ zsfJ@Dt5b}U=rpXpfybHUVdcrw67NhG{$nhdyw)69r8SNu2uZk_PeO+5jGv<+1eHq8ObtUsB#kvP5*1;#$pOAk%4l}AM~t1*{Qv`0vQ>2oF1MF0_(SUWo}O*y z(`O@v3HSZ+)S#677%A3D5yIV4%qI5HP{?%OvuRPqk4+GFgo?9ZR*E zh*+)1qipRXz++8VX9!>$L*sNLo=`5Ll(y2`#_d z$8B{~fM01(s!n^;748BcM^4pLwhMXXh{^IpDWLiS>DmUMg2r!qn5O>xq?v?@HgkQS zI_qSQ2#6Zo6Al8T&=`>5SR(y5c=%8%0t`A-UZ+r;2{${o_i=`PDpu?jYuKXw@F9%N z_C^!XxEq9m*qV=4S2IIeR=nBwa7Du~gi&W1#2sNY&@;XhwjinB9AtC8Iw=H7#XvNA zbrJtJ7xM75Ifnqy*{$fNc3M2Zi>*!kB%4iug80ytB_y_&v2^E%ZL9t~29QNhzZ3#G z>G=)%lX3unp*;7}>#fc1R=8>tzbOo$-{mFhs+MrxEotLcT~ zv6Bt*SfZ0rx!-{6x&ocF&!6|EjU3mUvDq>L_l|9oU!Ba7TeyzDGzWs zSzL}1=XFs05}3ReDLfTO-u|roqsU@0`3bkP8g^RXI{`ueGOeadagx)H%%SVjc^mtF zjYhw8@?w~Dt8EzXoJ6Zv+I0Dn@Yq;ltz`x;&$)>WyKZGlB?mj?Keo?IGvp|a!BTXS z34olvYkEe`LcYQkk0OJJ4@xo=qh*qij!53yFjx$oOb|aEbWBX2{4zLT6gg5Bv@8Du zo?Q;R0Ud{_gVk9k8nprL?(vF(R9E9VfDwycXQ6z3t((GDrkr(k&Rny*;HEp~$A^}* z;|kB~%IA^SII1YiSF!Ww`(`kUpSBx_ax!K^tC~es5%^9AVGw@x0N+s*J$!A~De;r+ za2~Y~3OU;~_IigX+Xu~fIq%!ReVXO~S_P`vs?TyMiEV)-pZp@A2uq{-F_{Fs9WEVd zEM1H3Lf9J#Xn+m4gv867o&BKPb<=R3gGQTtnp#0aU5xCcdx;mzw#{xw{CHBZA|c3p zakvnhT=}DS*)ynA10Bn=vQ_#8C*4Q%DigKHUGv`~!FH~o_Wm#(Hr`7cJ8SE*e zP&HDyYIlA3Ss=7`p+>5fvIi2ZXhoL3+1-buQwyul_lcAt1ZDLBf~{R>6HwxRUcuNwz8J+ZuqBMN@QiRSL$rB}v%Z zhfwtj_mNjzLn_|gC!b-INAxfTYi4(yV6#+qER%1qpgEd_P%vVCgVgEgel(ef5;&$6P&_LPIp&onV^vmPINu47Q-X|y+H zsXjgVcDkqPq<%p(z<;T>JGV-rao}t5C=DX@rkzeK8^ku@dpRQR^u$X=7}Rp!OCgBM z>aq4(M3I9qAg3+hi?ml#W`m7s^Xk==<)w!Crds}czNzyKJ_6Pvww4mH^x$l$17|{_ zX`gI{xB^Lm``$PI-bA6p@Y$2yui!=-HDfrY`}*Wj5$*HWyIIWSykQ*xAoHf4Wqv4K zL3i%UCqmBTve=JDK?&ifmcztHHGtLJOkG8m%INw3(DjvJQGVOoil~4h-Hnu_bcdvL zNtY7RUD6`b-QC>`-QC^Yjg++XyYZLjJ^$my2b^nwnP;B8_gd>NI2=0?NQ>ZG_Q*1x zWX2%zO?To(o<(6KHKKi`Vp{9RihP9e(A1HOxw))k?a3(VBmyMDi9RF@Oo9<#lx7G; zqqKF2CLK-Zu1=f_*-e)ksU%Ho(u23(3zs?-h0Cibz+fvnkSePf_%w9sg5Ao+%g5iM z>y;^UXRWN4-X>_-%389~;^q4=CM={QKMxKLo~}H%W-3}tcDu{uSwzNZdH45op zN^AA7&obMvi2oX%qS8ZYH<0z#yA*U^ZLbZT?zL}BBF4U;`ImNddujVQlF>ECPdro)JOyJ6- zG`9v1ef+IiQgL+#^+kL#MUXKqma+XUih*sB?VU4ImyLR!<>{4mJ(uNBg@m2KUb^_! zIkt{Vzl;g?V&$hWtIDfs|@aSHC){*bFJq!USF&24l061g)bR<{DmGYOvmaYJoB zb9{9+HZi&>@D|Q0(%={c!EvJG?Be@iUNvsROicXE+1bTlp%$K{ugYwpi2jhoZUOL& zP7&f_KASI8YS<(7%tWb{64bWu)OGS~1MUK|o$1uETtBIzPx;5DlgA~fpurBa{qqov9$rC;puryut*cWyUTO4{^MsjQ~l zTdhz5=HE+vz! z752LedLa90t^wSl#2iO(`C%AUP>~Z(A8-Jku&YwXx4IG(j?E zVzg(Xt+)H zyCq$nzyv=`dQmyqo)oG{hqP%{?V>H4{?%~=IBu%sN-oJT1_P zSV@ESmktqVV|qUCE!bRxNP;-MHl z2ruVFR=uc-XUdj_8IH0_Uj~x+wLIts!)0}OwrU2GNRE+1j|u}d0?`7k0@8G0;Ndsc z?)*WjeQxiwovJ?(#JE_c&7d?vlZoDlw*o(+sWnfRYcss(ev406J<6677YvB7?c$lRSx7+ z787R6SzO%o%mDRScHhYdZBIMW{t$@Rhv*M@7AP)!-!B!-NJ&o?Q$< zH>kA|MT5;l#q8ewKeI^1(YK5e=54(A=~z#PF`$p5niFNV1MwN7(guY`q9o&w&zHvn ze+&CI->2bPTTmF<(!O>fk?>c(oOwMHd!lP^qH|w4*p^epLKn_J-`~BXK7@$_n9Q(` zd)`yg-_TjH(YAdRSdFbKTamT0Lhh=Ph=1MuVN2{m17s2IBKFKkwsFJG^`d$GD30%F zdnEir5ZY-N=Ic7S4wc3c%q>kGzl;NH9`1`SMdil zf+`7BGE5bhOf?AiuUhzDqa-*IcfgqWY;i;r@?7;R0vuce)XmEB@c%t_O36XuW0{+G zOS7mM;1&q@*G~EOmifmw_5onPX#bMg4*H+X^#6NG4f&(!xp~HRn14TRhiEWafaUPz zi_C`)sCYciG4=KJ-2(%Gzc%SSZF9 zslYZkk=FV;u?}kgz{BflbaS{gk7y#NeHQ4RM|Fa^v-%#8AYkPQf~8!$kDCE8_x(fO zFp#<=%~qKw>p3xjY1^M)?rCC=Xsd+I!aE$mb^}nuTvqe2`u5UH)454tJ@96sXn8d( z{{)1~z%iR2MCa2FV?moBVRGdl0Kd@CGHVA)FdqY=xg6Vk^x1mLwP)k$PULDD0~>&v zzW!mbr#>*URp8^@%_S->t^`P(E^`?E^yw2YmlT9RKtzLH2{NhJG%gC-KbKYp7p9rN zn9aGDP2@p+DY*_p#Gz!}^mTGqpwlc?e%Ptal^PM6HDyhesd~@Fd{k;hjwb>pRmK znt@u&R~WSFDGn!_vWJTemn|Y{R<^dJHLgX<5dH%>l zTyp&!D_Z&^Y&Cv%h}8iM5Xg$G1E^Rs`3_UshjloZcOaWg5_@b+dLV-r<4y=n%GQ0I z4f8bSqPe)a4Oh}S%3ZEsJ$mZSMPw09BPNtEoygGe;7LiTXhCH7dN-Q3V|bM`-(a|T zaZe0OQUM@MwjiH1OY^bDj3ZT06phvc=kFuwC%p<8G|$v}!J3Oc=kPg)wr94v8g-Mk zoasEkq5AxQJEy2|C-}*nAOhFVN0D?|N%#)yNQ|#!Gkh8Lt`D1(s;0@4zO<^d+HA|z z-rrpto^Fq2m9KVlY?PlxrG4$)wlYHvTYxbiIM5(SrCzlDX_mABBVY z6ErXlO$Wn}o!vjXbO_q`ruGohqvNEn6kA}(JKT-rL1A(D7!DS8D$&QvtbaSz2kl9OH1ptYjKw(>S^d(RNGo3+1wgDyk zbVVfDhMSzbs4qO+0Ojf+6(!Ws&t(VgIv&F)~ z$`pHN*@Wh)*%L-OV|(HEipG!Ikm?{lqyY8?1j+wErs{83eFzWFlRyy{2?b;y*|J_S zE-u-k%gf6VV7NK(L_8T7YlSOs_UQkd(sBz;9aaOrb;-rcOHoZ zIDnZPku3LG_6mZur5?+U`dmj_X+Y&JNNp`wUjzc4)Lnq__xMnKE^3E^S%2*gw6|q- z7f^uk_dIgriviHs_1l<7PYjUS98^l(z;s!g5B2+WG2Yd}a)a{-a17DOtY&(#M0n?El009=YClMOz7 z04|Z)Tqya6t^v&r_yyqV|Ckh?Hx&YdPN|BJW$(Yd005?wZoM^FHX&*IW2!hf3dg+U zBg6ZTvB^M=b^wf2q)bBTvywJk%|}i367^~zW~qIQe*Jyn?^5&j!oSm}n(UB6&Q@1Z z`p=Xhk|`h-Q*{QSBccJ~AcGc81P1V~PU$sR+n*}7TO@-}uNM_&3#tohnd;#E_H&3s zKD<L^;#r1BGd{Vb)pxVZLdFeW`e<@zE8veF8*{? zZc*@X1b*~v=B>L3fxdHd-emeQ(m*w_LW0re1)l|%V8d5n^0_}IFIGUm;b&>*G!DBq z1$j$O*6R+d$~nNeG$b%Y0K#!%!li6bgBWzGTFw<{%&DQMVxp&o^|#Kt&Mj~mP9!1S4>ThUSPTf zRaCfwFaZDxi!nREW4>+f7n?VhPI${L6T)vkCcuoTRdu>g9I zcNNpzQdhv1wFeU!RM5=sKwvEEab-JxYkPdV#w4&0iX_AR+5wJpfRoHfmO(FYeu^L) zcx(41T*H7yh`(B;c3O2Zc1rToVh$=*Nyk~4Z2HxBfy(PjiG2xRZOFvh=4T3ptR+9K zQD+xe26dQ;q50(WLcQXjUrN(qK(DT`g1rVmoEMhRbnQb#nxiQ!z$yg>*F2`R3kN;I6T zHln|#xaqET1XuiDX9~k_j9J2IOI=qAp8WR!>}fKeGrw~Ze#Q!qu|(89-uwuJn!Nu# zC#159Zq^+z11S~qQhK9aV9f`w%I^glAN@&ctzBYbeu_6ejZ0<9 zWujd5N}3KrWwY3#LIrp!)q|>Kb|`(EHJi%$;SfN~C{09do4wno0ZmJmrfA6H$L@8K zDIao2an~v}8}U20>2)q^F@)4uOnawp*Uz}c8}X^10m=~tI<%kD=$Kr`s(h#D185t_OI(bfxf6=4=v#(AIJc8qPqDY1R>mfdOGK;-0 zh|Ic5b6_--lbKW<>}P!C?jb_KaV6)@V~30T*E4_jG`i8CreHzxBCe)V; ze3Ijw1?InmhlRndTcT%p{8*@~suk3#h0fG}ccpGY{r3+%Oy?+0ONqZb{`Hx=^9%54 zABje(`M5wcSVvcwLzjaF;*onfG|*!jjAl6P)E&Bx>#`dfpV;*c6=RX}{Sec-yKrM) zoJtSo>4&K67~I_~nc^q3V}oGV=xxv(qymI=)zK5^oQ|+WDvO#Gz0&3-bF%3=-k=tJs=xwhl%xpX zv{h8XVl%Q9hgpOrhQEHrXZqAS-CrBC%a!jS2V6hiYzV`#b3~;ABy*%}h)1;SGq(V# zd50S_YbAsM;KP~;=2tUwHP{@Pk zjKU5#EG_9x&O+-7S}t8frXRzgqCf%4E0`-*IdjjENyf|Z^Ft-VAq-93PCE?|!!__U zJTd5tGPT)}lkn0x{+S~-FjHD&SVop$vV}}awm8_kzoQa3TW%<%smVMIxARO?F8yGk zF7Ml0W@d#*?5)AXDSEvG+1K6+suvGQZkaRD6n)@q6S<>_mq)X==Wx4&vfZC$Z&Oao zcme-n%K|>_eQrxnT-w0e2Y_A!=ZlSX-YpO@pgGrb8HmRi=j>eX(0`J!!Rd+3k|P2f zn=*H&%)K4K9R%@m%~2fiP(7xv|23LlZoS1@Znc5U9rZP7p{Hxg+iq_<_Zr1h?}eh& zRh3a{Y#ZH@9K(a273k-(;b;eY#bRJzAQ24aEf%QAZxAbbg|vzjGBEhP!sE!I(^^s$ zB=sp2#`s~UJZ*RvsFm8ysDI`}p|1j|W}BkX5R+F?=95{bdP)pUkdMIGhGVw=<9B&El+Ib2NOADuI(;{6YLka zbVV>`CMx48s~fKz@gJ!9l&^v0siKggOVedTNWsy^ia!FG_x>0MxQdXsDTk9S0-_1N z4yiB~JRXA&e*9H6JLd(kQc<{MbLk9c(&_wg^=qwWyjIdUPqQJ8Qas~s8=^CwWota! zbpNRC?k+h<3!*LL2!@Xj5=T^m=oGyu^WUOK71(Kr04ELl#@f6z@5iSf8AwUVN4k+7 z+Sh?onmQWk_^TXBHKVf9j!Nh?Ad9#YCgvaX5bm6y(=^3m+RE z>g!?_pfHbXjJ+sJnn-bPxO8%i6l|d$%{-9=tw%P8t+S2V;x~^rgOjKxTQzk@P8);t z-7q%lBm$_LyKC(Qe0uE=$^aqR0^1|lz1F1nXey~Ry-kmAopP;QkHQp~fV;Vsz5Uo> zQ#snx6D@zGA8Y{pH>7KlmPPdIZUpy`7(m^!=f3nk7$~VQ^1t|NU;Jm3p^nUEN$dk8 zm+^xM?CGxu9IkB@1#M(Jld1=?#i(;pciAAr*W2ks2IqSi+kvc0vO-CgFnFAHAN{Kq zW?7Ta5V@b!%Kua%^IZH6t23=yo2J^4$b*-PZUUmxZNe_hO{uZ0Z?6LLhIeN(Q9zp} zMpLPFS(O5q95{938=$fmFe@sGupo8c1fDB>a?@hP)>}cdBCD&mE4HCPPA_EU?52iajYMB`@y3$y(+3}B+k z`iY0^XYuyozJOOh{^{8M8sQxWe698ZXs92_-2!mIF%eP{>UsqS_Zg!}>U7!TqKs6P zSz&W#Hn+)5yrLE*GmZk%UQs2vtGHPLyUKxCZO`k=AT1c-pyEq`Af3P^s-fINMqJdS zEmUoG>s3%Ynm4gqZtDH&!T=o6oVgoGNY+EMkEV)oy4o8fsl$GySgv_78GLlhQ?Ch2 zO5R(F^T|wq&y0Y84f@5)E5}_RlH;juS34p^M0XWK$@e+l1rwkvZVbJo-+1a?ge{o! z7&&0HiPdhpN_NLDRCiQVh@r7^^!4gO`iuj@uoc$ zv@dE(pDutpH&DAS8f@}uZoJ1Sl&E#KyZw-p!MuqCnvOX7pFm7Y!F9t#j<_wHn)+!k zY+I`7_Rd?@)b@=pf?y~z9$sNl=YxB1_2-tBmUUp?JHaEIO4W6ZLQh?ytD){Hz~8Ei zCMN;ogFG8DJ|1tlIzw7^T}gzTE+wcfk4?PPkUv$lV{ZWjcSeO%m4&>Q*R1-{2O6lT zCZ%@Fvwd>ZxBoz*M|9#PyM;sm`ybI%cY{a5SOLoJuoqqa-={@DNp{Va3dn)~IW~Tf zFZjqwL^xcZ?`H#3atRTnXf=U$1zVuWy8E4AhT}HQGCHRL!s5k#cVVla`zGA zgJmG|df=4SXT6UI3fn@04;emm>6C4wQQr=d-Oh*CuMc&0TUy3bTwSj8?hdi8u%7}; z3VKD`8pi=u$L*G$gts~os&DsabU0tdRHIxRPDRteyq;(^TJKIRn6BrS&*sk6k&I(| zrvs&W(0m*to$QwqMH4p$sA{zG;=+A*%5;v#1aIaJ=6@YUVHW)N1+ zi_>iliHs4r(N3^A-m+^ZRd1=vZ`HiD~Co zV2CivrX7}}y;bC9L){m2hZ|h1wX}%4Yr@oUaWEHmvl=cf=_lDEC$XNgwG>W++A%sT zr>&!ttJ7tdw~nH>40_}oy@@r(Y>2`~B^rWGvG=*ah|&>Co13z!2_TEPcBbZq=&b4x zaQmq zj&TBl8Y9M5{4{u#x6%Hiwv~y1{a|-CjB}CZ9K^$6LwA$U%7c#_q@AqKW zsd4T#3g|fcnS2a*zmd3t6v+o9?#X35!>m~46dIM1_0L=7LLei`HmhFsa{|H)JgzT3 zXb4jQskSu^eRkNKcBoxj#*+Tg)MIQm$l=BIhe?i-s8l4Pn4j1u_h!sX;*HZ916l;> zDPU<|;d7bLV(9tQ7m<@1mH~XR#8dsnfUZ+OxLW<~`E{PpbeTRK)#7ZKm8A!Lf&)9K zmK6+=fZnZ#hGk?_kE(yt_60iiXhB8Tj#5OxwKCC6o`Sgb5~;XS4)M*e8r)YkZ@$g>1G?yjb^rB!4nk)&O5g$xo8zFb zI#?pEaEX(ily&LXm(2Pvkjzy}buI1cHdU^v_9{tP*8#%iUMJ0m>NvP@!iql*y3IE~ zOi)c{BAR*=>g)AGmDROyT za}CIh6pIgYoyBC{LHz$NvVn0n|3RuZ!I;c*6_x1?3k)DU6AEhvsTYjZztDS72D-w( z@zW8kXdC=@jh80*$gEiC`lMrK1L3i5*#Nha_dJ68gDCEQeSz0Vae9DixCcC${>QWa z!&&Z+O_EQRWclEra|S0Ly+gi@{SPSq_pzM%AWrkVT5D?odn+m{6h1yaz`T@>&yUi= zUXkwHT)rrGy%mOg1v_Am3Rrq?I{*p#=CWzaQR`2ZxNU;2lmY=dws4npLI#*{Z#kO0 z{75`pCZca;QF1qLwwAU3AS9cup$Y+{SLjl02|ibUKrk*3DtzC;`J$jX%FRX08|mXY>`Xv^sGfcT05DKJjb|9H zQD1+qa;JEu_29F)Ag6qU?!?X_sUpP+8q&V5#RrZzu$w`KC*X;AXn1N57cK1&E+71v z!o$NC_Wjia?Cq7T)}ob_9ULY;QsqAfe*l9JLpq=;H+dv@%4xW{w24U68I z*kcY6=sNfO*=-3-vf}wbf%c&;XlVbN#N-3tedT^=(V-wYnw}{ zg`Tosq}MUpA2e8R)z!82n@eG_5Q@A=pzB&>{f6xc-B%MziO9~OI5_f*>dDoT0MDZ4 zx>~VWlBsIypZSF#bxP(%!>8$1~7itH5U?rZwMv&7cjKU z0yV!ZcjKp)7yyx_{rdH~90YAi(YVPE=PN^qnRTh^YUdArrYeiES+#SSy?-wNlxyWa z@29%}&sF!O^%G4t7c$?*8!NAxAJPbc*_sWl!19i<9E2z3R%R0 zX@ivO5AhfVvojl^@*kL)ndFA@`TAyOG;%e9*dBAMXz6Q-5NYvD$+ecDmTIs1FFgob#{`qj?ChBMQ1-|0)e0FVPi7s z7L~P*|5N3|n%Y_)p?skV+P|`eLB{!q;X0jd+pgc4oY$~6d(%OMc?sKn_X8!AoA16m zUG0cETU$kz1mMJac)tpbR>E_|*gI@{NWZtFE4Qg_A{SL0m~PI%IFwEN8*40hMn zNUuF(o`gR7wlto5xal-}fBgf;2q*;(IVGSQZZ@CGT6cd8w-uVAP0Z2pK7YuzEqfm! zd?@MW&&9>198lZ38o@Y_#2AcDtsHU4(|6(p6qRF&b~OuK#OC*|NXHIsMhMAH7~bco z7}0#L)B*3<@?}!L`M6nc#VXexN$wA)-onxv86d2fEt>nU*FoDe!5BqtR9%Ye66EYo zPUymW)q250b;zOU#I)DtpSA^;>GlJIT8Ql@_{)P7f=Y7IM3f8ZHU^ZXo#fAhq{_i* zE(3A+4CMV7_hVyYhLd|wso^v`5LMv_zb@_6xjBRn;bbipjo>z!(uk!LAePwCHZQe< zp9IdXtrTo*e1qgAm2EXTgY%wNnlt4NPL0}0-rk%*{_}n?S`wR;nu@|E>5m*$i)a*? zcZan1U(bGmwOY*yGhVp+niczLM@*X}2wiQWjk04qmcF~yQ%QhIl~KzRI6GPwz`@|H!ybu~1bkvhdR7 zYKQk1rlyW@2ZG@GJ8y4-q~=MKqSIFY;`a*zC|x5kb{iVy28NBShEdnFhkX`p8YLg! zsFfvTJBhqZc@~EmTCu<#wQ~{f)uLrt0io(gyf~^;&;2l89EmtCZI?0w%cD6 z0WvlH$8UXTBu6Mhz;JMw>UWnN zd(tpsqh-%$jEvb$(@OMj9#AtA%u?-K<#UbPa_Yo^f*3iVajEukP`=X{`&@5HU6l}} zCTDmAbs^R#7jyYZsswYlRnduU@&SRB;o(_~68K@o11%>vCfMJ$IhH*nmJd=Q```idMRP_i|e+yp=7a}%BMJC#f6r(S&L_Hl4^J)`)p zqqMR)GdDLj54Yst<3REJReSICO50JXdG%7OB&)p>cx>BypP9Y60sR^V`q#B@N7%}7 zqfXZ#uG#r{U|L~Bb0YcI=VJL&CLxE6_@HRSIH;NgU~`tWl0r~Vt>b{O=$92x?;y}R zidrk>`8 zKJMnOm2#ao%8u9Bf1fXT-BwUmhSDWlM?44T=q~eR`{xSoK$yH-_l8%F2fXDG?B+fe z7Fo2&b8om!4&xV#i;E^`pL=Unr@=L)=!9rHfM8ggyPK!fMsL18TBX4z_4j1l} z%-ZFpMqNO?GWes@5>9|$g0*$~`GsOc%?Y}kVDW)A{1g8e0q<3VH3Bs~wF3<00ltv7 zM#Squi%90#7Q8`~77yLF~eC1K?O;l7jI z8HGZy5zJ6*TFVJ~5qo>PR<87uG)rYgaf6H{_?_%q-Jv=kaOim#7JBCT!l9nrtkEI%_)D~oXv^EDc=?oiHom)XB4A;<1;ZyC~q|Hr??t&yVpLDM-LU%(vv4bo>#ZW`7o>xVG?tODF2>29U;kfJNnIU?90^@~;YSBL?&p-_8lX3^~jCqn^0#=T$0Gfj?>+!9s zZGFU60(*^sa#E~4!j;;CwXFKb^t7A zDQiA?t5JqYmzz!at-YKGrLM)?QXh3#l-3KvL2Wq@c}`)!*9ebu7ZUU30k{Bu*gkA9 z(S7hZ_ubrq&QlPeh5qcU^C2AD`tvLp3YIWWeFDWct7XIYl#rzxT6HEpeNU>R-pG8? z&p7k0NZE%y0NWs_#sE76KUKjZl&e49C zJijXwI-*%aWVz}7)=w@>e&$;i7Wdt+`<|ng0eS!-F@&}HJu>r;dOB$oPR1?F-RAUO z8?IcX(K(VSmeHPvoEKAUh?_HZA)u)A7lwcj&UfS*orgR&zmG^T?s}^N!D4m`0WZVV zHo`LdRB#J=L|-||7AJ96!;f50ClG}`g`34Pg#;Z*o-m$?Bx1bqnPzP#!fi{aX#zYN zaj=Wpshxn79At9wX(0^``>^}&B!BjrQci;lv&3A}ITlR*)n7gQ?5>aPQLD2zX=hmb zm8U(Zy`5AK>^f_SDsNfb=;7C>tKLUFrb0t)Z}s>xS#IbS76(0BKTn5pg(K7wn zxX;=Kjas7@Bg)$K{D-@?2^#QEtBMV)x%Y+ig8Eypsw#{?r{U8EZu;;+k;U z7}z~v5P?poDR%Mm5Rm&iZQiiQ@=nFm4sLqYra<=+yW^1iZ*O~qO=}M~^Bmh7cCs>F zwhTM^`D}N>x(auPt{jerWo9)+>K)104*s#{%J9H^@bSU?GRB+lj9Z4Ei45}QQ^pbB zi%Py3Rez}@-$pZ-^e%*?O-`5bs#%iWeve)U=ZY_&&zy61=7)KJe1g5&nhs7usg-q! z{$fXL_&GDuTlJxklvajCjk6yItCbGt1wD}iVHm6B3V9zwbt*{3nYFnTSBFL{Y|Bj4 zVm4e&G@h_lAG*Z|`4Gg0g?&D8GB!0K7a@myt*9JWtQ{VHTlw>7EuiIOe_w8Of0Q=c zS4K9$-SqarkFn@n0V>n;QNhoji2Xijehw=)-#PkDR4ljrW1MIw)crUE2JX^@gHzk( zeal_L58&)Ex`O=SmN^`QjXRl=(`1Pb*3n%ud=g?#dvN%)?Rii{niZm>)8_MB+L}&m3tGFTT0zO73z+Mte-i%Qg%Zm(;e@xlZyZk{J&J30n`Xtk;0Gh9A0q_+NH{Hc1DY4H03%(f zr7GUegG5l|0yBil%o%G}Iy)|{I(yY1oV-3$_Pd{F!;yc!_J7_cspuoD@uMYx4uUZF zHs27+o?4t2&_41$Z22lGQ@YXN$&TL`6TPF>4{WKy_40e)JU18a_3}Wx(S%fO2Ey=e z-gqp_lcc#VWMxGo2aFF`M7$#l!^6{W4oq3b14;S$XimkTfy?9H*X^&rO|b+mmBb}_ ze|Lcam1;aSc$)3qImUZ;Nni~{1%|h()fS=3OB2P1;$W~F*}wG)&8P5dT45`o6p}F; zAhu)qj=iCzhwdXCFkz5>0Q+PT zwd$rOy89x=onJ@cqKsg*FoXkpCGT$U`!#t6w{8dKRo{Y9>V_BoM1~ z7HbHMR%CGPmlM-HMms%b7{UJIzWI+^gyItvJT^3-$Vwwe&t{Wx(|+Uo`0$6nZbm0=25F&J5fHeMYE`WA9YQRPXy!tT0=2i|c)R7_{jh&Pzetn2X zy;`Sr>-hbCWaN3SFg(KVsUqaH+7bJ3ruZ9F6zdQU8v_9?m)mq3ToAgEnUNKi@BczW zzi+`zgL8DzWds$gMy)a!mtDEC#i$BU@8qJRqJ#k`9dFomWfcCkc7C=!9p}Gq2(gq5 z*auz!!+OEY2~3~c4TQamc79%6s9&wh?8%F-(j(SG(nJ{OPoKTqj!#c40w1Rg12l0} zykI3@FUWierX{Y%d42zL%>***@5?U8*cdCSR(TL2sgTbBgs%EKyI8y*&(DdhoPx`POLvI4>0HmPH&ap?9;sbVs7R*nymFxCQD`}oAewf1da50u z+*ljM`Hqg3HqY6m)v;VG1r-JGR3f>!UBr)G+8X`G?kq$U5}HKx_yCEukD{d2ogyUC zRv{1lrZHyCCNPb4@r&o@fh4LTm+tSei7t|>rVT*seL9{&zP;$=E$TGCeShI!9Jq3- z0Tv!IiOf`&n)=5;q9e718slQ%cc^}Y#Lw^Vi*k_u>wCIz8SdefbwR9R=_RWvD)ky` z(CXSgXsjg)OE zT_esT$^DX4<{XxdokO%gW))WyIF3CJJ5PW5ViQ9@M*B>%E`f}-zXMy46C6x3#Lyo* zR^Dg`D`)7=8PTCQ`aoQ$RX5o?8J1rgIYlicWl__&M41i37uJ`T`IL&|zBh+) zQsRk%Lq`0!e@+}TH{h*LP_L_)+1nII92^|joSR7btM;*+(*|QP1~4!(uyY>pTStR9 z=q#}Hj7l?Z#qFBVRM%F|&N@$-Q`w_C+TW7rJ^$}zMF0i+a7LbX3)m&&RlpM9Gcx>qBKQDd+&&R`69IVJDf>|{6y9h=`hKEhh0WiC; z#10OM${hzfx#qyp=AN{&ko)Pqz;S_mJY#?6aH;?#LC@?BV87CsuOSy0j%FZnGg)#q zu%FSf51Imdf0{`ej#vQa7MuuCjduHjTFZ3Qc8#vpfbQ9rGf|} zIsx4hWXhJRYK`A2pBqo)k$QTulogEXk;+g*o&C5BX>xO^>w-ZzlR~K)Lw=qD=~> zMbe9%q%w?`x#Seo)Z<)oMn>2~n0AVaN1knY3W*!eKF{%HrixGO2?I)iGed$f81lV6 z9Gm196qE*v6Z*=yh+We!)gQ_f;V*-i%Brfe9TpmDaTIb_1qW}~=LbhKL}AX>tv%8I zx@|TB`4;n+kA(@7*IW*2wsL@_5tIHPu2iP8vMTrO>DWUm4wyxy0FwflsUi*7*2L^= za&T4r(P>l#0c8*cqZ~+H_^+7E=3_J+cSvU%O=cK!ap<+70b@N51Q(0VRhsI&h+fLX z$4I%oI!k9Uoqa#MSzXN*$7~!G@CN?$_ryZx(c}4q>+5UTk6(E+vocHd#F$9Qv)|Jx z7c-g8eiuSS_D5?QMc8^P($)JNXBRZH6x`g2L)LZ2d$6}^$MB^nOC`WJk8a4WPzK7A zR3@IrVK7Njn<)2L4TM=keIDrq@t4%qWIoldsOZ)<=p%f(jNa$JP(<_ZJ|X`tcnX4%VDDa*R4`9R_gCwTqq=KGM_ zN~;h%^)_K+s++JQ+5SRRGBBDXzO{S>_v)2yex0(>R3)>b0BmEmqCcs;tI$;Yp6@3JmtfCmtIrVC3F2G2xkZE+X3+s&ms&N9cm6-8@x=59Z1J zQ${M-iyL$xmztGerTV$Z$(CyzNINqw^2qtIy-l%O9b{))NY=FQ2>qDxvi73gS8&G5 zTORQ_3UW}~HTH|`ts(6jfUaEHJ5S_9**gm(B7uW))sjvOPtfi89sfj;M)>b}Z&Gw7 z{#G>braZAfe;;HSa1sQNh*oFtYh>;0Dii1cWG5~Xob=J1y{}#?-Gv67;S-+o-ZbnT zXY8lNq8ERissKI(#VrxQ#fDmV5FaAm1~33|6@N0Q zV@Kx9n?Rdk8zxCaMD+b0n5R&^8;+;f@q%#|ZhDUY&W%wJHV13EnflqZwS5QW@fZM8 zWK7h#gap6K3l>{g7WNG`N?oU?9m6 zc)+Tx(-90#)*0_tx>UwX-Vv%vR3clxB*x>(e`y!$jrg1%K6&YEH_@QYi^hhHMbpb8 z6OYoJr{Bdn zX||I4(1Y&e5ty9(z9+<%SrBF3(GhebE_uP@bh6eW5pToeb1AD57xjmcivKb^F;n5S zSbk&*La8(qIzyY9ntWW7osVHF%$GE2ZC<~AZ3rS)v^Spbaw?ytI-k$zTM596H`42L z4g+OMhjgMOXf2qH1#fmI@-bX47VJkgAIfwYsHaFdLut>=lJ3th+#rNi@&z(4#bK)x zTT0jWtFrxVfBuOIW#-B+MSda*qXyDF*E@%(iG_d$1XdBWokaVrvWZ?0wREU!&j0Fs{|(ep7->F*Z-c6EAa3hq z47||ri&t=j)$0P|OO>YC)dR25v#O%WPhN#Jy<2j<@aJ;Q8$4(zS@aNE+?EyCIGjzW zc`<-8Q*Owq7K9&J39O}7yj9P75Ko6;pk8;jE;T|f;J1u_KANqyeXbVRi8#A0p6|)_Cj9;yn(qU(jg)*!p1a2$JyhCaVbI2>d|35Mj^N3 z>GBI1`FaF(iCDQdimM&7h(5n@%x|Jp83zAs81v$ z3L=10s83t;~Z4Gp58#;B3FAFd07(>VWsNwMq z2W38_{`7cvFOMxjBlrqgeabcNGM{}Bk2(&Dtr5Ss!yn_FTw*bqym@OObrCGd_TSln zQ6ndD2|p8i*TMaH6t)STcUU+5bwF5{c5S28=5u7>cM?qL>X6q_DhoJM7Dq?0#Q12R zU75{D#!AYk`6Zg!?xYMM!H)zv)A>pSkWtgs*K{eohW(~JmAx_xWtY>OB86CCY)X*L zJNUH#3{2cgqn*zSV^Z^?z-Hvx)FhkM{I8SP^jIV{^vIIaiU42xNYBpsr>TzvpzMJH zV^*|4C>>maKc|@Uu%Nr1N#Sw3bGA$sy5T}qULd-BN{bjUnk`Lmb<{=BVT54&-p~D8 zJRGWS7E><>^f37KbLIHd5=g`M@;>=pe1pa+S#EOznt52Tqg>^yR^(M55x`Bq`MoXk?xT0?r!M@>F(}l zp~rjAbMOCo=bbs@9A+H+?cd&et?&9|u9tET7U{QD2%RKkTwFHFM+Js0{I!aX^rM|) z9(l6YfjoTPq6w&OhFQ9}%t}qr+sR;c>?qQkuSJ7UC}1Fmip>L*J>{k|Z{L)J7NvslWR}&=OC7Ej;vO zC#$hWtbxXfK#|0y&>)lP?w_VAzgmq8UInmhmq20Ud&ogRyxmBAkMW+HXGa{5*W3xN zaJutrlU89L%B3d+DP)OnL-$_PK_k z#olmm)tM@k^YI^xhz8S6$q zETL$}sj=?d|6toPS7D3@G~@z^+;5^azJ@-wB=2r# zheYyQLi39s*I1ssETOaN!WRrKNOC6m>V5|5xsWDh_1fmj$f>#tf4Ra*C1Wkd3VvFh z%Aq!ZU@~x)HJ5ft)ck%-3@UfyRtE4eFlRWOT~$iQ9FN2HW=S~i55yhAb$d*wv3n-u zV^rAT(+ON-C34wlvarjt1w;)vyF{2f2w82RcNRdx-#c{^Tr5PYU{i*KDPo@+zT@s> zVC^#uXCj|{HMsPQa34#hCZw`*0f)`3o#E%U0!!lpEaE{YaTw0s-HBUIO>xRqVv|J@ zR6RyGGVY|_?4|Mxgk7;A-V-!(gz4^$rgSvy33cV%X2Tr?irY#+r;`$O^2-Pwy&x~< z-)+c}7QE!bBBz_RS9Xnrny|ig?~&K@m16OB)$0^xhbb3w7Ey0%I-f(brE{N8>Eildk-);i zp)w$1#)+GF*gH@lJtJCgkuHFJaoZ^ZwY()A`=(`A5l#RJ6VL7T$j{%=a42aF=XBiv@9$>AgcK$J#c<{M4V7T|!jTl&0;R84i0qp2RXpDVJ{^xBpjS8LT zLJ&Q7+r-Bgn++GdSwjy;1W702q2|s_!B1m`KjckdEBw50hMv16+Zg>NT5%N5=eP?= zo;W+ikd0%l6WCaap3Da6q}%~YFyHYi`QP}vRPl^N>f0{ff9m;m*L+7Jf+QNu1%1sf zmeO3XSo|Xb*0qHD;FV3~0ueR>?oYM2kBi0=_u6?iW(xlx&=zXHInhrCi?g$gscH=o z)zt~Ihf+Fj0C8cin^|9qr*i$91%N-~pKG|+QyLQkDCnqBbpqWMbTT`aeM=h+m^paJ zA0MpOviB=>+2-N9KJUDeSB|8X6i9=c>sF|l3h7W9$mFKy$y=+hQTKj3&J;Wt~9M^|0HzbREn_zU&Bx@~%;+K`DPVtG#A- zZZZf-BWj|OE^WD3;$|$t9>`VrhwOc1irzyYi0tLu{WjK4Qhz%Ei4SKT%;3l9Xu^xUPY&E>z#zT@!CSjNPbFU_sewJ*0^v`>ejx(+X)kNzW24n$+w$sAy-!|j%VY7xa*|C z$q*Qx))bo{|3W4I`tSg!|0eS4frp0&B-OBbd%sq;)Aitg9Qrb2o-leMV$H=kyX?s6 z%Sb-AERv4L&?5~8tFmAS$4Yg9C#1ZLOz3f4)woPm)JvecqQ(m44E&#$Aeo|$%IeaZ==ch*s>3o8*Wy5`L?#USy|H95tu%A{e5FXEPVt>gpmL`;IKtH?8)^->hSpSA1YpRa~hO% z`|Af9&sQFisEC-DY&}*nh&1EL(t_8*Ofx$-(vSB@FIDP4e1{f6(9I(oCruAg=I2H7 z>*H^0?b_d{mKop;T&_EL#$GI(+T}RfJQ|=qp`XTzzJDLA$L_kZ$0pqD_47s8$KkY0 zX5;fYHD(1oMVLq-2=upI>Y0HN{$HXYMHjeY3-+nP0gZah$9YW%XNkYMzb_gZ8fwz9 z(M;?N+tH4jfq*vy(18bv`NxJ-%G})5?PbAk@r43ov6(vhIARjJzVk4W7k!oa`NoXA zRHGv!n;TlL5qp5}0VGD{)u!TKF!x`VM)4E^%p>V+*^MNM+^-NYoIx)!vH(3zImfsj zb<);>i!eyb$asPON(#Bb5#&DKCtjcJ(VDr*a$nJ?J1QhLI4oAsz+{xbdS4>aZf63}oM~q5+gBL72*c~NVI!GZi<<8%elR6+I|N44Xpq@R z=U1Rfu>U0ur3=QPLlWWa6JKHi%?9+aDZ;8lO+r8JE`cQK{xTEx7L3fK4o~ z`Wg0YhR37k>zxS%&|#07pI`?0m>{s zZV%4aKJFB{$Itr-0t9KLUBN{M{;q7~SkQ;Y<3#HtUKGjGC0f@IfFf>`V`OCPPkT%S zrJyONgvTFKTHYXvUf5zX49U7Sdja0$vS;C}a{qx$T}H0HSV+Z`q z+AVkDZjmW1JCu>$6*@|Vah&9)4eN5Uhq%5co^;gW=aw0hT`!|z(g6H<1ShN-;L20< zUu?c*6KEq(Ss z-C1Gi3@Zy~;uo+mpT!#-eSk>?uYZMD`?=0$YBTzyt9b-r2cCUh(caz;pp2=i6-Hqs zP)ZP1o`-x@Fq4=;qj#Izt1gvQuGTk|F)VrOQk>(0-qPcyPb{n}S+ zOXsXxQ4Bje>)?f08(gcj#nZa!{zMI49WxFO$*A6W%FSq|alAjU%kR@&3Nuo-_)s_M zj@d-M!y~3SS~m1HyxIZC?0^tlt@n#iJR5pgbKhvh56cu6%ornPFyA~1_-ZYVi0Aoi znGIW*HMo-4EyLIEDpQzovjzoV=2#%NOXIl!*9-B~e50^oJ-Eu!ge=G#1C@ME>rv*v)E95-L^shKgeF`rRtZDm;wbJq=nqNdPmKdx9bQ4fOb%0K&L_PJ)*AYUd_F<%GiJdYZ4l%;7951|$;t zWYr%V=33j@uII;pcesw@+Vli_Ne8j}P1%1F!3fYhJa%?=0oI>^KmnX3edP>9Uk`k* z`#I;b+Tf*9T_nyokYHR`R`F|l*~s`rqe*%|cmMjUGUB^`AhG|}5504QUd1?hQ&wKy zx4O0`Eg;}QL#l2t*~EW*Rv=b)F;_lT1Jd-iIG_^)P^#fz2z#JPN7j`XnTn$QWBrcs zE0b9<17b?S!5Cv0BND`NX{ zn4Zy!+~uf4smhMHmhJIjg7YYtwUy5f-TMT_@AfkcjDg~p%;A=E(GrnIiGZ~2=ClDnziU2ML4R=cCfCT0Z(Cp2Y(xHUNtg1dlcWnq!h8$~rcer{%}cfgUWSP%!cDK9j1ya*C;W_+sK#WvKFu^34DglAl~ zpJc(#n3XjU*ZnMbY>Zh_R+bI%;d15cI|~cKy;)5CLh40-Yj66~89B^3%s(7)=3fd+*I_A@{4Leis3 zvq=cc7I7w9mi=*Hkx39QualL-QT(AC5J*ko;OJ;+v9bN-@RvuLO#0LvlG`@8F58e% zkh%+U6q0WT)hc00?AX{iU6=I&9yAYeI$RIc$`y@{4R80gvLr?Oq+-Hv#Xc#B5Ep6M5bo6mrhWP7S^e|_--bXnQM?RlrkP$Fjcxvrl(-(2BE~hExaXg9s8CX}x74*aO z_^?1BJ9_*&e2R9^1NaxLR+bO6w7g5c^2N!Rs#Y-~CPzhy#7F|7$eiAM>CIM;m^Cyb z68CJCxfs%U`J5{RK-g0i3k27kf=>LBlj+^uRJ(=q6V3Aneojw+Psrm`o6WEx;|DV( zQ>&rJiT%fqgdsR~e0O_gUxW5M-uz|Pg>FUL+UnvbZ3JKn48AmJ=KC7`cw3*;JYqqkv5|eM3c;-3tvz{ zcD~xe*#TzD1nyK;At)G$@U;Qd5&_inO9dlUDj>$BjqZ%a9MlHpy``bpYF=NyMdt5? z+1c6t<;9K0bL_%2hmQyJS>R{M04WQ7jE!tyeey}t8{Vy$Tuh4~mmR6K!Ra0$ZjCqp zg3SV}O+Ao@Mv%IWU_k>cVk=eN^V`&Ipoel^SX+T&=+-yZ#b$Gb)Kip^9|ib^l8 z$`*>XrSM-)46*@LWLlYFL^s&E9ZHp5*jYU$@&Qgn86onl@*Ypxi*uo({Y*cYDZw_i3__h3DupeJoeug=(X7=Gc;T zyE!8XPsI8D?9gpdY`~JX(CF)sHIJcyh=4D@ngxDnEyjFwP|KAUS>~fG|2iZz5twa= zhhJ6V+<29Un143PN%k>_Utl9ZkydDp2(i$nP;(%>NgF&d<)24A0ec;&*PxmmO70|; zP|0FCpY;#DnDP@Sam!G#%ucf!udo+m6~pi?^jii30cji!9qV0Xa&qyCjw)L=ITM&- zx)luVDs@`zaAZ@4lAHKJRJSg5$dB^RC+E%`_3H!J3L)6zpW8ZVLEU^;srO1@Jau~} zB44cN{=OM;w7?vv8o&d>tTyVAii40`>GLOpIL4>v_v#`&mFH z!FdqG#pEMcvoIvnr_p_!SJWtc?|KHa{}#jvAKIP1Y`C6@KkX(#46433cyOlNBP6ZA zY-IwcC3O%9h#qDLj0Vm$Jj$QcWkCUVAJh5alF+Bal&8mc&$9ziJBJk_k})k&arXm2 zUHq@w)#YWuZ2Lv_gSunJ!K>H{paNa0(I5u)*VBd5DrM*6k0qP11ZDqaH8gBmD`~a3 z`ckNKUH7@?$wutlMm!j+M?j_!neY=D+2Mz|tDxKOUt$hgTR~)+QO#nz2BI)b2bvEF#+s~YLI#-hsh&drDcB%?MZ9Ji{hyYQw+Q;*3yrUO< zm$4dwh-Ew4vW~y*86v~OshZ4hH7!H!FJ)0zi3AAYHh|xV*)C}-)%~UYO6Kp`HaP;^ z`EW>+L~_+KH3tg|cj$X|#Fk!XU@e5w+_AT>Rw&B~QBDX=eSLdaR-6gL=WH~e#nRE) zDUbWm;2#QmqtxgVZ9|2XOkFkm^dx%?+)}yt_Sd%$0Xq8Zb&q5OCSw z0;Nw(<+arYG+`)_SHi_A2cEg>4+2Ih7Z8i(_D7|-rc0Wj%9=CoXy8H~kxfY|Z?IsNcq@p$5e4%whQnaN z3o*EaYV!^N^vKlQ(EcFzKfMdzCH7`44J?2PlZAD+ZS=ewZ!B9Da-PiqO+=*Q z@#w4J-7v&^=MSP4X!G%%Encb_SN~?TA*giDLSFGkV$d?)@dh9Do^UPOweCGEAbWl4 z8stI4v=g>a=30D}@ax*w?sv=kRzyc%W*)#-msA}mN z{PM}btwW)I??ng}U353q`Wk=i0j%(4$wf%`o*x1M2K8%y2cVY9rhm97P+n-f+ECis z>WSFwkr+BOPYr?I?=**RqEr@X{D(#_o#K@pgn0!X#X^pAu4vlNvZW%6F9=+TF&u$p zK7rO7XYEruwPZ>T&1F%TR)`{_BOoLM(->n; ze7!NO!tkYOI;7;Yr*pf4{X*0A9P%{kfMm^=DF8Y`d95h(W5u#rwJ)avCHFuN6f@Sg zj96Gx{A<)|1E#qFJzD4C;t&#>)uhec00pK2|L9t8qb9k_5V41@UaRt+En27Qr(Ah+CU|~3k;?xV_in?(# zdvH<1ddCBSa-hC;w#;5;q0}giVaVzG-t9Sumf ze$L(X&uFAyJ!1Vz4Zl*Bw_d~J=&9o(svGUh|H8mUqlnhgsQrSHB|2f40&XT7v)@A>aYY>DS8%bNF#j_30@``i3 zh|X)B*H73^i<-a@XL0$a-tkP;mVR+4`mMmob3t}juIl^m7qbUC1CU2_aS^A#s#w?7 zFx&5VF3cC|z0~10;dTxN)G7n3Lr=!8&)imxGV>%;HXoN0tFNj*D`dXnC*}r5L~wuV zp09edlS@fi?legk$7RU+K+DmCC><}?bbt3Kx#YD&AL@kgExf!b`g@l#J(lmzD8%wD z(w|q+r#H__0v0g{@)x!~8VMb`j=T!B3e9y&vku}uk$;U+yvEXxvOM7ZF6azhk9Licfi1o{1kfP=LXI4E)YSfHhMkU$DDbU_wJ5&O#1dq zU)n7JRVkcTnc>j-6v9rL`xCc&*ViLUZI(2)oNH5N--$2_<0z0ML&6=#(fZ(bkR;?@ zW+^Z9-1yYhM#n<}M6S%ePAz})8%&*fvZk}l)t!Nty41$>z=UGU1uol;s7HMDQl*1u zSsAfMACTUxm5JOLxcvH<(tCk7$p#Y5;DQcgrmUb<`lY|;8v{tx<=gZe1tn!xtk2-B z`;)@*?4bm0r7hRx`%wGngWEM1YqxaSeY;bU7V{Yja2^j4aZ!Tkc>QsQJIu*KZITn5 zUvVQQz9JDYdNZ-4|El+7Pqvunac1Yeuuzq?h<< zAI_5Jy4$)*ICE~VS#1xSH@_b&(nDci2Mr`5WMvhNzxIa^9}@Iakb>uSWfHzs#-HLy z>iY&ZT#&!N&Z33XvsrN6P0K|H1j&`P7vON}eN&$ORVL^eMBkL4{bk_D%wLIvsvtiY zi%H%){Mie{^=YaJt`m%(-1mq|AFA{3Z~LS%HLEqTJ-;_4Co9XVJfG}xzfA=|+J1_mmW4s4Vn(}6P%tyu!JA5)gNLgFrPIl;i`jfU%q=-6aF2K0fJs5! zgvhrQyQg$6GQeztSi)D|gAB5_M3=n1<0?XykL|-|o-~9-fGOIDdx5j5A+A9eL;Q;J zCkKZayybXfy^#((h9ufM+k*vix@re(BM}6M+yufGJ%(18Bt23MwF~PfGW_mas@+J4 z_m{d%OML44$YNzeHzeZNJ>;~ol12EcCwT*zen_`;+ zFZ-m+8)$9B2f0Hev=VN$%1>r7DddQ_!F;$ic1d--7moYq^_r;=$t33Gb+%{NLqySn z?5R=zsCy~g9*4~nro{^5uX72!ht0~BNF0J78lbE!n^gfq-|vZMU)y!>a0w$H{E4!)bjqq{%TcB{nH|h3p?A^fkXAsFr2?RH#y?8A>bWg+q-5 z>vsFu06~D>XFl;AsLUWOGtk1oh3_lGnTI{Aba5iB`xd%t@ACC01_s$Tfm;2(H`g9e zLbRloKjN&Au3}?_!bKPfQyS+WaWH~whS+=@roIs@BDtoxd+PCB?ziBfMR%wMiW0tF zcPFj4Uk&3MM&uhLHI%t!OqjnqdiPYNbg4GfkEhtapyRuRT%nb(F`7Li=#}t5ebOFw z#4pHt2WBRyF7>BdI^klGt&}+i3va?5aq9#s^w=8~eQ(mNiExM8o5{34w0+ zIR^HwuQ&tR_9&!Su{@~g7mFYAte6XiQk^|LJw;A=EeL&ergd=Lih@+%|D>kk*!zZo<^1X#v`Zu0UFkYQV8;pe{w?lgcn2iTkg5XjaaYrTv6G&HH60* zu5muj3rPYZvAAg4@6j{^4%(`|NReB)t^hDTz0^gN(t+lFFD@;O+!ssVc&i!;r$bXJ zgMh{OZemIE?kxB~4@*(+(YI@%F4k^g&C|uu4!JJvNUIeny@5a_d9rtb}`5{W$-@*4l zpS2F5;7?=vBRd^8ZyLpIWktf*N?@m$PR#Z2=VCW+ z!>dvn+8?Ws5|hw-iYbup{_Gy}J>!%JtbWhP+G;Od>%2CDL&OX4e`#D>c{8PU%Hf# zPT8-7DD{8O(QgZh0Xh9I8V*~_n04CU<+ijvX$zSzO{AmtKq0YM;r6g^7B^w|^pW67 zjeRBld)1n1mv|s@cXTe$8TqM_MWLm)zuMye17E(&`psg;p7hqdm2NtWkY2+1wT#yO zRj|b;4};8t4aL0!rSvTj(Mfn?uDl8x#t>L1>WqBff@-Dg9JwJsW{rZP@k&tD%ysIy zG25xAHzy15n3*xZ-f6uO*^WA-E#z=ib3 zt>aDCd^9%U^6MUb`F4+|k~{I>IBz8w}Sx}iqObt8UmZ&yW(o0|KZa=kqn7Q^DV zxBC1KPM-ifwEnWLveU}y3!|ZL<@n%4<)aDM;I~<6*2q?>DpNxSH8qXY&U29!A9A(v zzU2T0OT@Zv2+7na@n-aXe? zjM6$*C5X(VC;6Z9=k41{MWwcgK2pZ6B}EmOep35+KDUByFi%RXiiyGy+OYA?cn>s` zT&50HDEmS7uxgoB22%8Tm`4|wLXf~@X!R2*u$?s`sLMux(Tjv$p00Y4AcP6%o z=19y=$Amxk-ohA|gzSv3wl^Pt_OOyoA#Fu%=G?rySU|QkV5&PU&~h8ytD1YJ_(YI! zy;<9+UBa=}Zih75(W1&nx!lk^vh!^0z#m}Mc9HIZ4t8iw`|6Jvyu~C6$h0ZE`*ha) z8!gQAYHsoJgW0cyQrVu0qs{laKL{f)sJ)%u&YtbX^l&)AkSKQ2OA(%Ozts_nrl$hF zy1%INBQR`70?w}cow6UkKKjPU-t|JGiS;ufknzRv>DDz;KSknCkkzXh|Hr*+m-K|O zrK_`3ia3{xFW%D-3410Ol-)?!xNKWZ?7s!ICZu_oHMxf$xzlt}k2eyw*;)e;K)rvQaS5zkd^|3+K_&-A6Xd(P7 zqD0|PuA4sv^hld2XT+azjJox26D9^`zq}f0#KF2l+jdy zskP{skU$+PlgKN?!0@8Kr*V7yBsEV-o5;0u4*sOg^;;}VKm#rwwG@SaN(#RqMD$$( z4b-x4d_1+y-pF?Vl|jS2M+wDUDMrTZUjdX61?^*fK>OmT5z0jSWL2TP2vy(Ewd<$~^;I3j@xA1xC(ERO-mD@{rj#inN zffoaVzn)a)5a_?B@EGDu2HNVM0lv2CJHHUdJh0c3HEHSjlx)^qAx{UM5H$EK}8_ku9ks;o!FrF2qnSVrRc406~imhhL?C5=bd?htd zD;KP5mntVq^(jo}u_XrLSuz`L&VxE|+Ur61E6_%`QQzEcwtZ#>`LM{Z7@k1ZG`SwL zfx@%o;7O+X7abjBv9$XcBg>f}wUu}slrhq%*4sZeru}@hO$4A4WB@sIm59`A5E8qa zR(0&ht&71~H-nNgvPbIfeVHiydZcPjiOGRP6_tmY(354Z@K9RS`UH_J8f*q8b@vM8;aP7xQQtm|tJ8-bUdX5RLD!ixnevT2GG}<2#1D;RK_=k;T%6rtQGPb~`rXG_-N0o5pUsZ+$)z$AK zjFgm=W-49CO{Z(Us7h>%ni^l*-9CA^hO|37+hOZD=$m<^Y8Vf4E2h`e)&9ep+AITc zwsRAX31>x7^wyzeF3+io4!94^*Uv!e;QAW%7S{S&&B^*G&^1hjR1h=(=s4l?Ib_30 z%*FIIo#afp$+YL8+rW%XWRIZ6!@UbTlPUFdrSYWZHV{nHIN859>?4S{lLVp+AYglQ zu=1r)SyPPNOyk`*320`vQf2p9b4kfS5%^&;+m!+!EePf8Ll+Yh>-!LTXGiA;^^tQ( z<2UMofEUOldxKp!XSD#AM@HR88V(}L;@#X{WQL$V(5Y6yMwwJ;OtV~o3U-|jaMXOt z^TYw7V6(xPlWwxhduE*Z9SC&{(EiJ0^zZGjgUDUk(qEeGYHNdx4!6^2z82~?QIR1* zastS#$$`Ea%-=^xH)UlCo|}Hs?VpXb1sYYFG{?U1c#lPQnFH}v#eIt`*u2irhQqir zpbJh1J@(s26xbL#wHDLLb$rzckThUD*40IRh^r>QF^KmVZNyQj4g6eX+!wExu|f6= z#nXFNgE0CX)ynFD-si55q$z33n~9@P)=J5>UJfXJHB*~-l%@r7Ai9U|E z6A2uYzH(HGKPZCyNdj!e!$(W?t#iF%{YFYftkF-sgFk(dT~( zX%Vr|=)#53asMp?QPjG<8i#M6QRH;HC)#Ew#3=6okcXLiGX76J3Mb%|8uL`Lc4Hp3 z7XSg0qG?>|l9hBL&CUVIm9~4VA#ehS3>Tu-xD8buD${e zqu2hSK9n2u35Lk&KGm;CB8v1Y=URi)O{1C^!ioN4|S| zrxW|L!NNqpa_&Cm2E#c=7MJtX6Pdszeh`b?EmuT+1@Zvt-B@dX!)z9PFu>`)aO|>) z@OzT_XKv$y2BueKno;1@Of{w=Y zwiEPD*BNW)#)SqFJ*IGb8ue;#sJe)`2kP47vY)Lw#lRPg zdtpp6a5F&4ov&OHh^k(mhD_os*_1B!6&VsPU9UJ3HhH=HBBXXnfs(W;T(}%CrIvNpBB-=s0%xUP_Zmy zk6!JSx8$~KBc&1+ep!BAGY8*gaP~@G-nx^wct)HWEF{&mkQNN9>T6jr=cd!S`@+A+ zQaBiIUti0Qc6_SEAMa`Bg7#8J>9W7i&zV$>6@YJc&IrPS$U1l3O3IE&Z|WVV#V^7R z;rW?Zg{l)?*l1UO0Xwkc>y$9noN=rcW0V{6173-STYUieWN_1u0lrEtebAnM`#$)7 ziXKu75?GMEU72IK)f-s0kTZrh1+)j0-X^0D7R$n$K- z=f9PB|9XA?$1krwdyM!w3${W0zlQ7o`-hZvpnO--p|BzR4~oIRzuRKxacbJZ+QI*? z3-GWKOexjX?`=%dkO@^pRQR_N=a&pnqa4VDP2Ts9?^=VHd2= z;ediYar~@ucyW>rScRzQtxfKKeMsRHc@EAAL4xgP7hj3k*x0sW!omcAc|9Ys?i|fA z!IN_ibvU9FWUj3JU{Q>(G+&H<9qi`*`mJ;Owr#zQ6?+I)Sbs~+QUj91w;mv zyFDy0TIU0ZIcVx%2x)FiuCD6MRBNM>;d02&&981ffNY|Z#riWe9GnMVUgxu1lXjV#fxj1B-?yj#&WI2bUy&kR{(OePj->HhNr$4Po_i! z(l*utnb;_}D?g`hMBk-<``Bv#-U9>_3aenKLtbnye6O$H{uUIZN~K&ntPk#RFQNx5 z`Xyw~187)LH$cGl#2vgtK*zg05a9v5&l{K1C1%ev4T=oE^Q=Iw86uJ>wf!5I#PqThpcb*IMTI%FAWo*LEf}U41uv{IL;9&GVT@?Oa&BohCUQyCI z3gi$N9x*E#Ejk_0k2N?cU4zbYJ-jFPA)V|6fM!r?i}xtKWRZ&fV_PB>~=?C zq{jNo(el7fFc4N;&9r^^t*lr!joGc8@W}QA+Fbi_wDX+pv7-zoLmAA<_{!fYZeYIn z7y*VX)}z(AYNrcLtWHSRbz<9;^v-|`F-GC6jsQIO_o zk~s5Sq~m3vz6TDQ=|8tI%$5kfS4fJXGD9I9-lFsP-ftC-KAx1@t)HJ=*)(RqaMmA+72iog@O+Cvd`Stt{B* zZp-a3Njr`h+p67*%^)q>qKU@wtb(u4z7vk-63AELuFNU!H3L6n4f8KbL3x0{WjTnfDUW@*w-WHK=NS15z!JS`a5WRa8`%4fk6iifp*K=g9KAc`?f458H-XqhHhb zSDB5*X;kS_hCb*v{rO(N8;TYK?x!t^ci(b~$K4v@8=5_k5>KwPFx1qa25M@1pq9Yl zj#&9TN)~DcHV#?;tlA-)~v)_Uz@4OE7v>Hah24J2YCFYPF0Bd z=CWs1J!BO9OL|j~`@_FQ>V9IB+<5o-sx(u7kP}fj5f_h)`5E^!$BTW+Z+3eNBnOXK zcQUVEgKQYtm?p&P2?vWg<;G|SrAi!n`^}T}F}rr0r#AjPla;18H3+4)dnbo&s8cgN zQJVHg=r#C`pO)Fdft7+*wfucO7Gdc9;mXis?z-pf&T2M3n{Ctz9)yO41(F(C<+@Aq zH`5h+*XsSa`vtMvi>(xs_|w&N+?)J(Zg$U~7gbFUQaU8*O(X;lAQHZNU_OfA)9=J- zmM-4lC?HwT3(FyHI2rs>dC4XDm~ILJrWsgwoPeQah?!{VRe@SlfJUyd0&v)Zx?w_* zNBIH#Z}|aN1dE}-Bqw`5uZv@^4R>L3TxpP_c3<3hUXM@HLG=U{P#<*t;3l%fiWp3y z+E9%JxeFHQVj;&;nNgRtS#(=XTCdFGTPB#9t%D5&}c z1L7L!!z~8~TcZ}UR}>c6dlLn4k8BhMoqBj%h%ol*3CgQYa<%9~kVNBaZ2tZW5UorR zCF83WK*EYWfuRQ35he?`5-kExy$|M7g&dx_;!Naxm~Q=Y78kPGf!Sfvw>MdY`QoY8 z>%vNl0Us!WBu&f;K$%4!63C8Pmds{KDb==O;MfPc0HbdjLf)g_=+f zclit$3K3*V*7rzto={{ZRm$mdy0x9rCU0dYP8%gyr_Jr8hfQ>JP|`(pi%N`l1E-MC z(cmNx{16i8df8$4f5~+tz2@2Y)l>20Y7JNAQ^0y@@byLan+8v$}?c9{%k#Fk>17nmlT?MqF zK*_q+VQpsj9EiP<*GWnMZf3uS5D6Uta{o%R`9c~90z(1IGi)u@9t>~dbrt#BTkKM3 z)a)vmNFG8+9;xp6dtM!$8toMfvSO)H;BrO3m_6RLA%*cxETE;;tgjbj3vZb&Lfl;} z!iT>q8v~Uwm2WFnhQ=h$<=H!K?vK^>#`T^uk&)F1?zDaUUFG5NL+pvCCgs62h|%Gw zn|^J}SKqrlBhoONdRwbD=SdN@;{F-J|CWl|;v-++Ho64zk_ONS&Y!wHBx&@H>j~}T zT&u5#Z#1g8w@*cCXdO->e|y37Rr}9 ztl2N#KckxSQg*|{Jfzd8rO^nCB$J9BE?FU;YNZco6Pqjl5?%uu&r~wZv4bc>6Udaj z6F0Q%qeNf2xfdK=|8&21nMOZRQmwOf3A^u<5&ptlFGJNeEmXq=(|tpemZT^g$-0Jb zYF4zex7JgFakq(Em&UesIU%oykDT9ObFtc!R=6>97T6>zDvD9GeAqronS4PDG+nww zNfWoz{ps&XT5XOzjvok0(4ypSh$o8_|7S{% zTPY@3==T^;+u+SUTWjhBJ*Pz2m5H_$f-e6-w7h6j@tu~2<~@efkI@kF9$&Tihs!eI zp9(_ux+dk~cOkDU$k@h*J{Hg9tNk4M2#xSql+&yOK@3Yuo_+lF<##aq^GenpJ%|U9 zS4(kW6zhLe&b>S8g5AbOmO&+O_1DnWQYCZJS0kZOBs>%_AY*# zP5V~4a_oejvD4xJmB3A=qx$Bwj6Ucz40ReVfS=+{BsPp#qFW}Jqad^*TgVS_Cqn0w z&9URw2pM{5d+}h9P~2|KQaClyyT+??u50auA=0C7V3EwhJk8I4YaXJn;bAgfh%Vvk zYh{P>=I61bMoLYC49`8XLgO>y-5#O9TWBhV>sHmOQt_Wmb!?Q4hLo;mdS@&~hlQJ> zO~zM~kxC%2k#h)+WL^&HcdN_05SZA7iuWt=g~4jG_4+|5Tcnl!$LM*=W1oKLY)iZ` zqy%PmWqs+s)!omk5D-@)v>aq26+t__K#qJ`V(^}fPa=zDx;_+#%~!EVRjBY5E~UCC z#T~?+5DKJ|p_4nGR2+6~91v67){I$0k9|J@3WPKnTAl7;8j-bCTwP3bVK|C+|x*^h*$GQl#6DU|q z1=l^pg*&{zNsqiah%T75$9GzLO--yXjc`JkD#xtUMSW`?b9+P1*V@^Rt6Zd2NTRkf zkPR@!aOH6kk+X~zuNOI6tBbW9)=f%XPWEoLORZ2SL$Fs>R?8%^7&WV&ix_fu*9#yg z&+kVJyB7*EL}nn;FP?9av9T*uB^}LA8=}tU=ML5vqZ>k_(S%&fBQsw4A>bKMFUA)x z*N!h?JznhN^^(6Mp)ztmHYjrzAK(9;6sEbwMu;_14qvdeTAEr>TB<0){BmbaxzoR= zaa~frUSmhsL{ryA?gd1Y2*0`z_Y{QCTra6vk|JqV80pxX{Lz`-8zW<5S+D2t8TIw* z_col0D%_EFbivPlmzVxT?$(=BLSCS%6AyoRwKS~GP(dEfkaf<>Xf7boINT6fbJbrU zWX!~(S*X%|)D^tf)(@Oihz=e2(&_3fTEONq-Y6RG^m$Ej*G5l^PM&LZ zet$z`<@xN9BlUe~E#6Ki3%Ix+^Z*9R-gdIan8SkPfg?GJ6Z1cn5_8U0pIW@q7FIQFaOtHaF>8cN=iz2m(6v>a!h6%O6|Heq7R@Sl8&T6CXOsBw(bVjrMHVo zXGf;FWPh^&wtuXek;}A?SjilE*~fpl(&(5WY2^Z^+5j}WYZIT3-rkTARWaGUs&3wZ zg66|l(`oF+M~-X_C_mrhdY=Z5-U-tzIK+-Q_P;-Iq!&L>vuw`W8OYh&llS)aXGz6G zP_S2HReZuy51rmGBuaL<4wJM>E07F7sK4qo2CY!a#};-QCexT4FyG+M4i?pRtfaKa ztS_cY(#^M6-dr_E5fY{#LqW2ZzG{9Z^x|7$Vq#-cJ2DOPU$W>Wn{>*5E z-JedqUKnWnN7YIL^tAxNymfshXf%*4!R5>KZKpK_c0Ys zG#bLy1_N>7L1j#8_=#EV?Qq+nyL)?-6cqd_h@crfy42e0Qg!&+$zP9}8)PPL&tAB4 zIxt3qlESy3Uoz7d-eoJ^d}CyNU^X~BY^SGT!u(ybOzGOAQTFsswH$iAMrNys2wuBD zXp>M&sr_q~$r(9uYPO+<>llF7tp=_WEHO)CfsG6HPe!8gCqX!oyBVc0@+IWffP`NTO4Hu$-zD~3J`)z zYv-O#tL+J!?9Py96!i3qpSY-`QQcDvJPMMa-;aB@-&4=pT&CR{4hxB_Tz zYr3gHJa;!qY2u!kz42iI=RNS*_V)A`7!x~Pn=a|7)_&m#f)yY&M%7L=kjBA4FN%nY zLU%cAA^*a{;RA@wqKR>qG&oS^8^(Kih`K%8K;41MV)!}aZD25i@7z8Ja{4A zr>;A0_jG-_^9?EgG>l>c09?`0Y}wSR1xMK$a@ki8NN}W@Kcpdfkza@)C9v$^0rK11 z2o+zqGdPQcfdP5`Uy7kby|_xJqy&NvP-2h1L1Wkh zniT>psfLreYqiX${$I2w&s(OkN9CJ*{9ty&7P z-|Muu?k$N9e#90XUK*x)qx5A ze*DJg;x-eWey-$F`orZym|ln3sEC#@2%i=K6>?{f^d7<7zvwtJT9qgOxFwT*280Gf zLoC2nq}0p<&G82Q&`Nl$cUGSxeF5lc$rce20rP5c++=^e`QhTigR8f+%*zYy$T9#? zPo7eQe)R=zeQMnZDYuiB<7BftdLubCjk$oLe(*TpG0Ta*ODH#6p5H&(*xb}?BI+0X zT8qKqvWBENnp+`a87ryxNG0b}a8}k%BNg14ZsW;dzqWf<0K<#(HR#nk=#~J({caWmj8VQ+n&Q2_a_@Vk0y5e^-f^e@=D9fG7V9x*xbK= z|BUM+hfN|!d>fvdMi;P$-XAqrKMN@;!MPUo+IXd}FMWPY_A;(DsbaO7z;TF)8=|Tc z8oRY+v0S^^$$SXjp3V?7o4n7|<2t;?damqMR{K4A9hr`lG1cXjjm1#)_iXZC;1OYQ zZ9&+1hxkpO7-3ro*45&nyfp47r0E3x7IYCG-$oQ(^9Q1Msyg-cb$2UVmtoNH+m6CY zltUO?ioEuTvjnUJsr1%n;qv3Let&a2B5)NI4gAMs;HY%PeV&Ac1cdj!53K#6NhwB? z)QgE3pN)I>WTFfR@Psfk=r5O+z7)7PrFk9aEI5dSqo9QKxm1N9#cBj`dj^0&doih4 zMt#^!l;Aa!c7yQI#^`86kt&nSMaA1_O>xY-M;mh%CkKtDuK8TwquXhos)E(uu=IiD zL%9O@c7z$N!S)qmypJN0u+?k=jS8dN+QIF9FT6z4;AIS)F&# z={2;n;&hTF?jRm2o{Z#56CAC3rfmQL#j}2!>3Gpl7XjONmSviiuL7F28!U^G&%TzI zV{}Mxgt`t?)|jBQBzhGFez3vOIX#C*GX&I(UR@U)LZn2y$6DHf6OZa50H9OFBx+34CPEr zq!L-zZP{2Lkhh?gbFDo{EwQhQ_>-N=n6c3d`Ms^bXa2M0O63S1NxFLCcwI^L^D-c- zP3DjgIk8&ku9R7cByd!)vvj%@h<$m=RrCySL#vO51n|9 zpFrwt93AED*UHk8viRd_f<~ji$5h%a6V{rTNk2_X5)_Ld$vT<87M{mPd?=RyA3C|b zUVP*b>!_|iNh|;2z$grA0|R|x8dXjdXoM4QJ9g&GXchw+^_CmDwZ?dPRm1Z0Z-1JL zGee=`%`#~UFmunUW5&pbgQtuvZ=RLL-l8oxpvSy?&DR)dV#~V=#*DKi)Ydqj}7B0xC~|Wp-m%_I16f!G)>UcYK0K}C5eGd-k2N0@#TayR8=b@2kN-YaXotK+gVu z`zCv|4!r2-&ijYOs&7<6bN-on0YT?``^v!5(i6g{y<**a1J!=nAjuv70WZ*r6@oOR%Zmd)@*`YMkQcyF-&p@3MQ?DC3dD53 zH_;TK+YQ-}c=P7X247eoo7*8$iz?#TSATB;b5U*FhcET@h2HUgFg*{SHK6yb%Yi*S zs5T!@ggz{?st0!SLWhv4`YM6e-+s;Nyq^B1S3QmxhTk!7wbs+@|F~C}h8C-d4K2^;O9+Ot_B8wM@=!62{^lMNRpiN%i zhF$4su#YlkYAhUWZVcv1o14vW$a(L8?>R`kyneHOIwJ3CrDhN)iXeUdGz>=Fpqd;i zDvmmuqtMXDL!%!*-kk756WP$n6C%tofF!9(oNN+sz@#!dIzZ4?akwk??mY)6YX(yU zXf=JYjc^a8Ox{eA*3igEqp6mV?FU()p2+k(2giis+xy%3I?Zlo5&DLPeb>1YxSQIX zd3Gf-B-4uicln@C9~sqeBE!S8X8N>=;*}`!k)fk9G<3oY$|#~wcAd#iUEJSU`=8-y zqon+GLw{Al0#^XJL9^B>&&$(2`r>&?Qc{*15F-YBQ(X)5wi=7rKe7~%CPW;36B4(! zHmJW5XSDv*=B4hO_*h=~*rS-JL`hp);}HT(72)!p?(PfgF>l_LWbcGD)4tW9Y`~qL2Y7J5_`^S3OYmtm9mc!DEk%Fhy#AT-wKos~ zH#+Ai;jOa1-ZFK}U%aWY>Pz~v$!gSVQlsv23I$Xjhuya) zbD=9yYOFaqIfFfAy(}B|Ga2Y&R>}B09!sY&D{5x#xv62Qvf3;S@ymex$Pv|GTvyRp*^0jwrS58sLGO2@m_Z zux-30o7DHXI*qHgz7_;>+DB1je%YT}5i6}Z`uMN<5|3n%HNN!5ck{D8%R}e85N>s* z84RLJby;M-(KB7Hw(1rc;p5{cp3a~iW#tlwaF@S&H-K5zd__$KffSKx&3o^n6)NW4 zf9v|$==MOWmu6jjETgs+hcL<8D)tZh5qvK`(IZJYh>AcV1I?QFcFtv4#~u5rJwolx zX)8?GwU*Cm9)H$F_Np9zcXLl zo-+_6j+8X^U&_%R-PBP}*KW&Ea5?X!w>PDEs|!T%@3Hb8Y@syOSLXzQ%GtD}XTu=F zd9>{F(=_h{k;d^+CSUU6Ah{pk6cumIv7bwW!bbpeg-ti0EJe(3+qFGf^tLb(&7rZ@ z(sY~=Q|+E|O0nXK$9{on6EmN%{EVuXYs}l6@zBt%()K1-{;L=0vL1d{ryZ@1C~fvN z6rk*BXZGB*eLUu3u0s!7MjR1Qa? zBjo97o6cA9uf+gHWF}3eBZ4#2;)!@ogq3^Eo})Q#Giv=YK>F-T*OOOlTZ->uO_8br zmF$^t4N!|DmgFR_)h954RD|53%h7VgfVI^j*)~LdQsNwXwB^o0iRe80i>DWArl0e; zNXHA6LzXB~^>_VrO-DD-Hs1CEz`^9p+#TBE6eQQm@zYmxehMdCAM%`cQ@f*-j%6_d zik2XEPvCEMFy-i5@VrlVr*r$JM~sxcsPLqCPZ723VV1-=*M?(VH4L%D5hi>K0h{IWp5RtS)vA*nrS{94bY;3%b6D+ceaX_~!$p7sN zKP+Fjr-RMqDABQSZhSnh+fMrC$mBIPL(uBH%V%ht-->j<5bd^vn(dlRXhRwa_i_Fg zM**L-!E<{X8@Z$V#4kNQek!3wstyTxdbB+)H5$^9R<{z2K=$kQ6F8#A_?2#=#9klP z!9i0wKM+irxi(U8=lbO#|6C*SSfO-FsYbJmI-Ju2Z*0uY1x2CDT@%~$_HAdnw*v3T zUrA#1r(^|qHEe=6_!y-)Fp$wqmkPNj`^OKLBGqjPubCKlU2c~MY8|}FvZ>y4?wAc} zf%a82B3OQN31v^@m#DgVLb{yo8xE7?+JtEi_3^J2`JoLvrcX~vg0oQZx@d*6HVEmP z5axfOz7@2bio*`$ib>UB?J=fp9574rTO4+CS&PCFH)1=PFfbaB@P*9(2?^6wK=Nt~k+Z1JK3i?jo5oI2n1Fb_PiUm3GM5&8)i5TLt zUk2ub+OeX@D~v*+aN!ITA~W*e#w~*CRbi=!^_8Dze=?X9iLXzO!S;5I zeUl159QoIv{5E zH?&Ue4Tg5N{^GgEV!H(4Kx^5({)+=9cJ>4;XL=3}wR$I$iVg!s)5rVj>a*?cXNM0Q zbCiq5hr&r3d$no|hTl<$DK`;xl}{I|V{4QP#61z(q)~rSGgO|j6Srx9L07AWCRy<< z5sQg|l~ov-xe_^w9GUYc54We}i9}#!Bh^?0z*r@GM5kft#z*iPqZoU;4O6|r+;x;g zPw1o2V9yY<1JaGf5juZjP)Eo(I9gBLv9W5iygbRAWJ5DS>J`4knxWa(4L0-jy-w<( zHF*l9rS(_UKx2OVu;1t$mE}KInAhcgf^$fzQxVhVBXD1} zQj8>NKUXfyr4SDuwYo`Rw{u1(kMuf_#+?N7_}e29FL)?{=c^J;!2vobBiAoLd`1$E z9XC(Y6-vuusXJGt z_`C3cl$4Yzo;N9I^}1+BUd|=o+Z8USD=F5!hM+CgF>~SjNsBbhdwb)ga1xcxSc#!} z3uuc~l~x@lx}R(tvwz)T)jz|TYOv%sPBkxp)zR89Co7r5VX0q*3p2 zTG?$yC~*JIL`kGMDF)vJ>X6zRPEBcK`I9{frR0LQgD%k{b55m_o_*2#xIE zT^Sj?-bw3@6cs6@T7a`aDYw?UWCvvMHYokwfdn5!o9Yr3>U4BG<9p=W;E0kJ8j32D z{!R#V8!-+cLQe2}S&z#rDk|jFtqfcBtjg@azGUtCSpI^5Ke*XKrpVxBz!AsYU$}+u zwDgTep`B`_`y)t{%aiHAos6NTOwOCs^_0@*4B3AAG&eu&J!sW4-P@HK4Rt@>S?&9d z;AV3uPW3_BxgCHDKYd@mgF@pqv`WB6pj_J1!60#Q`bNn8@GZWU8; z**ba&G42$s{_X3~w<`DU?JGBK8D^*PRPXOW(uU7>n?XbU9@6E~Fq2Zv3Lo795o*Ym zDyU5niwbg@Z9of<-k1Popt%PTVf+5k?Wj79Dkr;Nx|Pj82Q4i*8fTPxo;wv_RMIph!hM_jdG{Z* zS>7Cs1lTBUc(=AAGbdRiYX~pJtcANPA(c=*bviB?)1*nabI4_jZKzJ+Cekq$iNv(UG1i5e~P3hP{_ zW+jqEPW_92km+lU9$;xx=<5sT_P}T?Cq7$2_<%Npqdrq?>wRUmSWpj z%4bI^7bMC#zAyK{^5UHA6P-0oqWdvL3qVD*zAg1(84G2TQ#skK^~@UluFzsj5D^7p znXr{!@JRPQ-O3_KN~)qCXJwCYVrWv)gc= zRL2a$L7k~rX)Ja1EXoWmK&m41v%8L?ac!eTG*nji-BYl{A3R>LWDb ze;^qpT)3R=SCZysv?oE1er?U70->1Hd_1M?mQ0uWoW{7%Qk!yxSMRBaTlD?sq%T)* zy!l`KE#GP3w)gK- zZ+dnZ85xy6S0vPkX=p4%Agqp3Z{CbQW}||#JauGF&&qU*qMWX%7RO3qpLYrS6hS6+ zvvh+xGMNE=ApmpMIt1T4N_ytV@+Kfa$>cujUEl797#IVAaBxKGJfr{6gK9Jpgq*LZ zYLTfVL@D*^Qc6l%w7HRXhkP$CiK4X7JM7>B{||bw+@}o?=d>ZUi`sYYTy&+8MGY8@ zmo{(Iyb{v2b8yH82kXOGKtX;6Ght>{L_U`U4gv8@sGa@(+Q5Q|r#N{9lEn;1BtSXj zr)ytHzL==9#b(f~_3KZQrA){(7v|vP+`HfzUw^tsJDL7sV_~86lab+azPWkqdq4+L zsSmTy_b(HqO&^-G@I-C`2sAX=Wp+?3Tux!AFWZhm1O6uLZlu{%|AWTcf%ig#k^_zXnX88bNFJuDOdGtkeVjClgps z*olV$rkLB54!@w1e}BcdG4cs~L0EHf)RlH~VlSRY|5hS*YOc&VQH|+vVgM?sxU8(J zsI@XXIPcy=GZl%Zmd(0-c^(yLf#rO_rFkiF`t_sigyk+J%5A~Jt*44KQ0*gOoERGs zP^g=?KAblIDq|$Jr(pWN@1??^oDuEq1Ie88K%&uX`t9>8`iA977b784Q`5tJx~h>M zq_d3*y@OfQ9Xyy+x^AYqgb?!d_6w~_g9i*+hm9K3in4s7o9q_ZA|bK)_Lvlgyw=S} zq$y#ta-D^~Y3+Iv^l;ZNZ`%P27OP`7Wg#e2)}%d+hQZi@qWU%Hfy!{~q(hovhLcuv z*c`|R!RWU{qOlH*_^w&{Ql2K^UDU?64`Bd8ff`OlMTLHDdGEvlO*0&Cpd9gT<@Y&1 zm9j~3^KnZ(+fqc7=^nL*#afG;GO-0%>JXJh>;Xn9P9J23UF&W-Uy)#qHXU}(URzZlbI4(D^eX7 zy4>>ybK%CPBL%Wa?g z<)w0!+9BElmPm4n#=RL=oVFu9D#{L}MU;AUK=Dtt=;zHzAFI|DN0XPf#>N7p@p|X( z%n?3Nc8kqcIE>G_%!b8wBRjLOK!9FFgJR!3`ZS*jK9 zPUpoHo&Y?GkoEg3sfq1w&+!T3{HLvqx_v0(JlP0f0h|K!9>HNnV~ zp8is`P%k zTue{Z8v;$cxE6>qd2J&)T5)W9R?F8bDYpSM~KAGQAvcQlUOs-R;~-Du&|l($L}6B(;Uahzz@@J zfsFFc!&RwfO!OliW5cp)Tj@r98fS^ad0+CE{yKXd9!&`n!(giu?tnX_unl9QS1sGn zLTGN{IU42Th0xryh?AJ9`2!gWj9$zB+>8yti-1*N; z@l%Q=kPZhwH0;P+1G0iRG%f8nhPfrxmrN9ck$|JvYGaB`D+t7d)r+u6+*Rv}KW)d3 zpoM9;bGuWW1=YnRBov^rpS!W0(uTm7NCp$)Bo55xg+qxIDj2tXH^c<&VZ(qNm~;UF6Z}UTlPL|N4k}#~ zSXlk0eH+SHMuF+hYjV}WaS$l^q38F7PFBNigMK4!X~TM6ThPH=e#y@!S~((lJpJK= z`G!`i!|I!up7-ZD#UQQ`*$r!WcdbJI@ajBmfXKS#;{!uT(N_JC*y3s5m0373ZBbJM z-&xFEbkcMzL)5A&#Ut|Lp!AcDxa%BKMabY%g!DJZm8d`ASY$>{>31xg7NWn611Q&%W6g4lth*~nTIN_=Q z$<#H(15&&0h3hq^$PHH)$-TKYi-r z$qAz?Y5t)RoR-vJwdciw=-U|NqgIT^W@7wmpI0fD%oJUHx^|2MY?WV;l% z1b2@sK0*H>6Mz38{R4pWl?fhE|GDP;^%>u3pxmF^)(@le{hPG>XUF;L1A#Zl0s}=) zp1=5QcmK~nVG%&;I^9XA`lpWN@1JuIyS(5#>Zk{trY85s_=(p|Hwh;fr@W-yz3qAL zNT6YH>)G}AQJygZKq0sD6m(jT5^8C`$Azl0Y_fnVbGVvu1sS`PB&%W~!v(4?wxb4q zV7Q2^s)`O>_jGD%mz@H zF2Z!YPF*W&Yimt5!1JJJViG9gx!WN_O1krHXCapqXkH>gCG^!y$EbQU?D#(W{QUI$ z$1E&<0G7AoW_)A*`-T4b3z(t4d5L%2h7=>B!@|%&q%RTRP&0(T0B+2IgDU`7Km}#O zWvVyJWl+&6bZhVO{L<c|D~e3)6Sn_F9V zrqys<^6aZS+ZC3aogO6a|G4CJW|Xh#aTF=%v9!_F_7>YodyeM08I6nb(fA9s9B2^n zk|F+|mtTnwtjW~z!7h)U_xK@%1S8pLoPY}1p~H1WabX6|{{H?~C!^T(Mp}6%^X~p2 z`9g@3bJD=rShkfx>jeORRiDqt>0bDYBzxdV?Tc%fv0KdugM};Oz5iKJL8GWcMt=&| z5QvR^E-(L%9;$#gBr;mmw4 zU50Gq_(ysLuZun7fVlU2XmCK#Cl8pGpzq@;B|N+niLzDUIJUCF?3mwJ>7-q+y%Jjk zg+J^Vm`FLLz_HS|s(&210fOfne1kdrWdI$U8K=$n$RN@NGfdE( zvY?E|8>QT#AHmzJ0tQYX4vY~v_}spCBMk&@JX%BPlM{98^^R~cvIGH8`bmM=%Xhw| z3v!6)4uL+f(6)mKWHumD-2~adX#lPT3w7Re6)lm&me})j+52!-#OFeP@}42{$yidE zVx!Y5u+1}iy6^b|?L)=)VJN6x92BWNds@?y@TK*I{c3M6LMwv`d3Nt`g^)Ydis#W0 zG3cS?cKqs|z;lXhHdT#`BHbCHdvCP^DG;N{c~@}C`|A@CVFv?W`pAw+`m^Qnd3DO# zj7R?G^5R7=ry6%3f=wUVE-CYxt0IwzBb^^R7Q6PgT1YytcNqEdmT42JH6E z8ufO(RND2=Da257zka2b92hMD)a#U?8)>5^>sgFT|o@KjKG?vX2wYH{lp(B}O>+Bh%`lw9>#X~?7ui6nUV>x+bP(JmT?IIwB z7#kOYsgm(g$}Fc}(=N!i!?+x^)NZvD&KSGTlS%Z;giH1+`oG`-|9Q+eBZDEP#$->J zl-oMc8wAX1jm7uIrrXiS=18cP#)@B5=&xEi=2Y#it}W(B5oZmKwL|66{y6o5t0ec} zMzrm>xc;)7f)PI)b|A>V*CaHcIXo>wPLYX}Dm6+m~8<6rE^|ZXi zwU@?zDa^`>b9zs*f||7{oH)c#rM1O#B+;a(x`_*Sc)+@AQdC=?6^J+2!Eo$;Nl5QUP{xw zkdl67Xm4x%q0`~Fneafd0Ajq3i&Bvxgb20iqVG8y80WZc`zE%y z8rZH^gz21UrK~o`7?f9I84_v6H96m*!WP|1a9zDUtm?MSVjU6a=Yly1K2#jA#@Up3 zyXsIqdP{lm{jgZd%d3T+rDv7+zZmg9W7by{OrZUFP)QyeRHQ#{g$eW^kDDEn4P1C# z9%}@tFV>o4+z_S4CI8e6I(^JqE8irsx}}*qV%@eNhvnR_na$ z6^yi-Nm^ypEL6_}r-(bQ0I(WSAV*hc`<9!*oDLUX%`z(fhCt-8^ zx*&W{DaMDO`$4d!h-O%tX@AL0D#_caW1KeId;V~j%EqT#U`&no^Se-X(n!!0=9Vb7 z{r<9g!=830RI-+NZvjy`V0zeiag;cUUZ|ZfsCD&QcF2K?;$7AjyppjA-cUkLVYbzk z6}#tQH5#f-f5JQ9&YFRPW6R9Et9Ism!BBUxoyNkOVAQ>qz9L&iSK+8-{a*qbyAxf**;ob2p#YS)!x&}XePsXbv|B_=QOWtcqCo0p%jBnEW$ir$t*=il z;!a7$B&FlR%gJ$7joSOl1@UNeDu?^ zXX%N_YVz&PPTLwq*~xa+@7`gKBx6$~(nZk^kJaKqaNcakbIL$R?{-5)unvB|?rpp> zZDV8M)Gr96K!rt~^W0@%DAfqIFgYQsDX}mb-CC^#nPXXEsiTv+Vb6$6T&7J&A2lMs`F1=CoXDXDeuRI_xW0B6YcutCG}k7E#^-w($UPp& z90BrR60Q_#w~qZzrg%=r)iD-pAv2SFov$`A+79GZQ`s!9t`2|8GcKpoh2ZUL41+*H z)86<&o)#Y+uVqJ2(UYaB>nb1iu7zc_an~!qoz4CL%-eoo9*c2#bU+dml)-7=qIy8! zZq^I;4V-b-fa#{deRF!%3ryu4np?Q}H3o);gY&8G6*HUfBA=0Y{$Q-HL8$^LrIV$Hx#z(iQ-#N^F@Kv>Rq76o7To5F#hU-rnD@666qo9ZJ9# zSh7>xRrc!6QD6|KjFpw+h;a>G$RrSKfd5(f}aeKYG3C;0ul)t4RCXv4yG93_yVZ)69!dcGNMl)>hkA@z2U4)UQK1oDq{5kL| zkHch{>}<;O4px~4x|KJTwaYQ%wA{J5HL8wfN3GW)y(|)l^5L7)A9_=*SYFr{RnBbA z@AB2I*}sif4LHGIKJ31QqW$L__W~PZlcR$5sWKrV0!_#>0PO~bZ1$%#K7RZ}jvq8( zsT7aY+*pJVbnmh4n`c|I^I%{~7mpEdk>mq&J}VHZ88WO+r~9^Os;>;A(=I=JEDoEy zL9lmeb~b5LY93n^IP=J-Gdd}^8S zK>wFBA6J}Sq#E7zrR;W?s||tU*2LIW^}hCaJC7m>P({6n(h{P#T3YO{bSx;@pe@}| zh>9vLR1K0>&-#S+R8ZMJZwRUFc^T_-6U09aErCAm?>2$oPQrtW;W%kQz;;+lcD&>D z@P^{>(8N%uAa@Q(JIATu>RFi3$_3q}Rc2UHkG#WqAyiaU3Hq*7`TDN3JsxL*ImDn% zsEG8zmW+uSZ6&EKzSq?oRnzwGi1{G5pEccE7wf&v_SBJNbUC7U6NZ2AfEA%i4`oou}d3K3yi zp?76I{)wH7@)|!h|GvL~5y=S- zi`#non1t5%O@HToVc(-#46@*o3gg{ksh#US?6^kR zDz1R#<`bL%4+$rh5D9m+>I2*B#m#zm8&PRve5%*byI;-rm7yj=EZ?Z9s0Ky)8|ZQj zd+>j4cNx0qqt?9up+U6~lm(lM|lVL{9*-AkmU ziE5Qb)CIOdB#USSso;&t2_TXmB~U6KgPaTP#?K#sa%!1!pw{v+q*h!pGqJ8;3J!$^ zBD(=^j=AE|#H*}rSpO-Qs;uruaATP!iE4ubP0(z={RrE4C9WH~;OTSpKYzC<$p5h- zQMq(IIi=ubWR$7j?OQRx(dR7My3sb(*(9-B=XA6(DOatoe2d~i4M|?L@!+BXLZ>{i zE=8){lC)?Fir^_I9s_xI!EU0*a(?kh=;mX^s+Uqp*2~>&k^@OV>dfV~MF3)&A!b@Y zs1p(%JyOBbMF=Z@NMnzW^NlXMF3Eoqa{+Zp;y`Ms(j)~cwcb_}QsGW(w`(e`& zQd8sjPR+Grx-Ug)A6Z)M=@#TDDmk+_r)lXMmmL`(e?@mIcn-Sb0ktB!8oepeSm7S` z3GtV)@2vKsmeJZ9o{y*PTw`F=X4I8oDP|ns16EL)h$VeSt?C#2UhUrn>Gw75 zw?2!UaoHmRTz4h0$dd?ol`b_yiGFu{`*I7l#Yg6q2!~|krNk|d#+Vovwp7>maLcSf z&!0FlEhoULnR9C5oXWt`rU*~j^`^a^Re67ZpH8GV`m}_DG>t99nVh`avALN?Vy1tX zL*jn0As9#0KOr~&53u~_xsKMit+e&nwz%&;?Os6Lf}|E-1!{}|@#Fv7xcV2R2@e-# zAw-JBt|SmhnR&EhC}%U&6gV zR_R;5?ohxtxF4l-zN+>c|2qr7^B*D6Z|jDV1eop%2c}mR7GzRWj|vBZE0q3xhJJp= zxBEuSfI8R}otmC2ofNaSH=&QNcwi(gHKJO*I4DLf_wuDrm+8%;4Lk~wT~4e%0=95b za2GB4JKxsS8eJ3<`VCWMQlxO0nm+@VUi*&#G$Ogy7I}aZYl6&yKj(Sy87Sak zQXjp*Y3mRA_vy3Y-*;pHw;H%5zoZ9 ziHhQ^1M!V05T=U)E8}{51EDaei=`hDgKs6B{n*u0r1PfZBRTeerCsCd*iDCb*aEtGy3&fP9U8eo|4K%MFnpZn6N}lnXa@t6m@i3yt=`%90^rM1B6S; zxuK$w5gtG{!=}qXpC5g?`FQC%NX7~@)OnRrtxlw}MDV{joUb>s9aX`6V9fZCsfk2I zr7b@uMLq@+*>Nf>3M6)0PwoPB!dpAL%KE;)Ef@WIIv6(~yQH+Vp*gi!cYiEH>d%}q0_pVd4j(ihO-@f2yuw@{SzqkvsRmou2AGuWUa3xPj+YfJ zf!V_ppw&u^wRbG%E2gvK$8QWKDSsJE%vUSv5k3FsE#55%1gMU!IEffdzt%v680!elQ#7)wqIXbZNhH}Bo{|j)n^uA`XWj)WGu69lpBhqm z9`t02!%JkM+BQoALGr90NBx_2+>pSv=RyIf26PZ@1-OqB?gw7U)=It6$QMMX6)4ue zSO?!ccIP~AS{mOzekk_{4*2&#kbne;L=^({ zZ@e)Liz~zr;kWI>Md8EwU(c-Xv=C5`7Gt?j#knKX*rVEP6cj# z{lVEIS6A1KiSF*YTHQ$9^r;R#S$LzrwEGQpa(UsF%5b4;YgssiL(g|V5#ivZ>m5>2 z>x}3kBL2S~Z}NN)e3y#Hq*J&%`g&Km|H+ zi-c~>-y6nT1?6&(dxYlL3$in=RIWgBEV&0PbfC3NrPBq@cONT~z%@2jie1?FzjAmk zl;P53q^wBFcLA^7S=7~#Y;hu^!t)9-{;h}p@3QLKO@6cNb%87;dpjRKpcEf=Sd(L2 zt|$kf=o3j98KYOykTu@#Z7+kv!xt>lAGo39fXp=FEDQ87QeMB*)C>6SnHn)eJTwq& z>w2(~RKM7l3nH4L+Xs`FJ@0jK0c+dH7w;;^tikUMPP>X_Rfj$9M7>A1Jw?#i6T>~#abnmZwGg!?NF>bF-7ix;?YOvj$j&n|Y%gs%@j zb&#Ozc4GkaI0&L+=6W%sS5jOoUE4F^=jYLkWa5DJ6p96t)pC~?`09iaN%?@3)XE|# zv2bz4bUdamJW5VTZ(UjCT{|mpY{>F@VZo^^i+@#ok!~F><1F;2=6erD-6r=sV z_7`H@4q^h?{Me(xRBG;okdx^KL+%tl`<2o<6tFKPK3RBlw$n)Vn47z?J#k~iHfLPN zJxJ&JT+`v$!XwB@$w$DE`eEAjSOx>g3YK_-7ZSlQ_BJa*!=vYD(_JyHe|sJF#yej) z9S>c&(+&XHI`!qRF#dQ`V7Kq@ml??r;=Zvw*H>^1l$dh>r{Fzs6gAi@g)MtbjdbA< z5@xo>`up?VJnTU4HaHH>K>|LXr>CPqx{FXt3tGMr2>f5vX>=+%8Y{d3*_vFjtMY{h zU^OKoQ5P7?W7xNrHNOz3{;u2bbq{V^oR1p_!mijzo%g=B+?Z>oyZkZCv@tVWMcP<% zH!`FMWqYC`Bq}=Edc<9U0T0xdX&$Ud(*XS?fE>!EKpt#??UF$EFK4Y4B{k&@CkZ;? z-vK6bKZ_J>{8@DzDQqu42DFR-QWys@)BqHmOk_{FckdoBXg_BY)kc*y0`ilmA&6mr-hjA| zY$(r=jeyw}v+ zYzo}$fC#r8Rdokp)Gs1<&z_9tzYhi`xS1Y!D-Q%<;D(AD2kL>5UXtL%e8T4&CboSt zAoD;h_fzjqv%Qo4V4yPt!saE=i_HzxsS##B+HRjHx`7BGPOhPwE)YG~QxYy>A@^fq zxorwjR0T@}gz=FJg@l9z3I6N-MzhXnSMYqWeL*bw^I|MNe7KmXHUPLL74`lu98fl{ zq&HK&?wtjUii04+_eaD9!9oeuF`>s!TW%Z3I=lU?DeNc`);{aa<5#Qo><;*@)K}; zSZ)lHF(Bg9djgd8a(Z=6x&W_|R^J9`ettj5H|eeF1&dHzrb0xCt_@CT3v?^{-6s z_?k&PT<=ys^Xvk#lY>DQ7ZX?!WX$wmw`b}*zg^@aiFday7snt#;Tp@Z9$ujD%7yE!;hmh6R2E)BX`(~ZF*hf8ROhmn z*4JjTTXh$omDR(< z#K^J+BJo&36W+2PQ^g@;Wepj~CIhF({(HQCj=S^^ZPvcO@>D=AX>9MsX12b*x^wuT zaai)y2iu4Mui^(YJ*Zum&jTq_%E+v!)7APxt%jwa9m8;gIv`X}9!5HC52-&wK-ieM z9d;x}f;2-1PbmnZF3n9Z>fX^(RO<}Ti4}J(&YOm1XbEAxny8Q}Me+nH_>3xJ>}sPX z5boudiV!*QCrz2I2j8BeLJFP|YKic{T4^F^!Z8qgQbX_rWv|xazCHNcUni3kAOCsM zN-Dw`Qd+&Rkg}$jt*Z;nrwql627O?`9&>olh3 z(a1{++eo~8wEV4$b!|a3@&4;QZN*pwei3O1eGmEuOY_tTgi=>NOq+RE-2o$!G(v zYqQR`7tRWwCCsbIJEPh~gCk&ZT(K7Q`;kMr(t!;tEuL>jI6-buJ|-^PS8DA8&ms|X zH&M5mI`kQMBqc;RKsXv$8|#u-A)Ne2QZkw-V0cYJ^}t4?m`>^)Gk{mRyUXa8vdcW*e{u7V@t>T_Zor8`#Nb0R4j!tI|VG`f>W0-r64Yru(Ok&3jPt!+k6T!=;^;c>;G z%i&r^OeeAx9-~Gs8H?VJhrmOU3vBg6;|bh0py+U1YkT@-hU)gb-ps?_xA_@u_-=~i zuK4}XXl{1MiiErCDzpL|T(ptIqfoje%?rORFRKQ}Lk=`;dHBrnc7=swHr?D^qnev} z8z(mPG!r%MY*=FfFIF@{_@6`1ug_R9`)Ye{XECO5=eH9r2GH4_wzJ4Z*`}+Bf_v96 zc>v}20qC!9l#YcYUWw)k!lxfIGjVd}U>P;Ta>0+XZl15x{9i{Tkm^8(l%1VDhr5{{ z98z8w9P%mg_xtScc?>KFR6ta!odwx0DPf6t$j84e|^OX?d|q~rw9G~ zXP@CKYig8v_c_BrB}1HPPpoB3j6+9nKKC@MiJbTTqQjReV{dn3e7~Z3TzEG8yYv0|ot{pp?1=k9{HJP;!a@j=a zMV(`JI_>b0>DPtz^QS6I0E!d~KADXT@~In{GHe7Z)v%YncJ$G-oj&13(pMM4x02y4nAz{OFsD*P8O~v{{wdzWMe~ z%@pimY6KQkO!2eo>gn}1H#8{98%jw^OTS9DRFd-MHHn;I>sH3K_6FUS1&odLz3CDz zLw^E_>Oy*TH#ec@Q92nQxh+=O)J)9v+)ZG7Z26ZdVkzXKs!hDz1~q~fZtS#bOlk4) z$`^LGe)s_?%%!-51WdgZurmkgFpT|0ln!dirq`=~V@&x;x3g9E_~6e3#Q!t8!Hz5Oum=_QTVSF#?x2(~N6C+*hbq@ytT*P+*U|5QFCtu}~N z7*verK7a4jQ{EJY+b`_zj2zc}w_sRMZhyOQ(4?sEk%>#tVFm`p4|TjHn0q{P%qLze zoZw>CB>PVW+&FP!LF?9i!3~MP7{sEa*JS%<2cAwK*1^?pc>L+ z&)UC0Lfj0GGg&1wF)<0R=S|H`Wd>HvSawDOaAYenC!zqqygt%W|R-YPFn$g)G5h!;ro4Wh7~VAfTx+q;+Ty~ujI@|XhU z5LYM--M|0iqF)^eWD4+JgssgEi0Lyq4TTPNSNfF(4+_nUX~ zJ=bLsuu*qAM{B#0aZKvLG#9^NVVR;dz8aWlF9W}SQ5DZX^@dA7depkiDa6pKr>FO= zYRa;LwxQTMJ-WihqzyZoO!IXwN8{$rM47E-C79n`{rlEY>bK=mRCNt5U!{di78#c+ zimSTCE|4y2T1gECr^E4SJoR;6(GYE&6uCC?;ughpkPsbhwG!m$HkvL@%oKsE5Eqe* zkEOlsol0bV?Q9ZQ@2WZ+qNQh6p!xTIo+qs!xO2vW?3$XIaj{IOZ{NQtc@EVcJCb|( z{CNl-e%4kQCA4c#=hz`iDi<@tDWTQCs_gWB}w z;9&<^%j=A#W0O#nM7-s@X2W?eeYfpo#~d9-M#kisD2E>D61&%M@8?Caq|@y4$vM)# zFabbobk*;**V zTa2>Co&Zj-p}G9;-?i6c+I#(ch6>TIrs$6#jfxWCIX`Cm)a_7u5E?1v1Mw!$mx-x} z6_X!yYHBTKC92%NJPg7}6DAD=b1cf9h*>XY5@(DFBL#-CxAG0pSH2*n1|e5LTCr=1 ztf{T$c)!mDpDll9XESXr&m;dnYky55S}!!zf`i7KG7=wC0+U|+rbd`NCFV5HZ2#tg z3t<8K>=_F0ebb2REl9LK>onbuF_OqauQcXO1$OM_haE>bld4H*uZ;rJgSWtOH{WL_ zOK6yDEa>I6454Zfv3sFK77fllOAF~u8W;n<7m#$@FKYkB3{{~0gSpEq zoS5nFMI<_6`*JO`ZYW9L`uzNqO?M{u1LQSiMZQ!2j8s51dhlJ%LYx<@&6T>PwI1l` zn7@k4j*>MZnT?9|3YNLLI_XeW2L;VP*y^-%NWx2&L{nV9eBc(gY352X$2J!hp}3cG z(%{zNe$R+re8W1Jev%u)dokvCS82TLk8J8LcN#t%Xr@IotZo;aO_oAR_uEXd%lHij zC%9E*V~r);UJ~zArIBdH2a8#jNvRCR0s7^Rwr6E>>xqN5=oStihT2ktu><=Ko}cH? z$)^e(D|3Imby#$}DFQP2T&{G=$f_(eU9?FIs!tf*AmeG^3M_q zctE-6Lt<@4f#Iu1u*6*D(lzhr82EQFm!6xfSInHF>~_nozFZLDF3F^qu;-fUL=C35 zPk9AkCL3DcV+Bq#rwsV+gq%Vjmn;SgNzHAL z;4bl`w*)`!{PzpwulwZt;X}+_+YDVRUTvla*Ar6QUd92LwWOJLL+v@h0m zy_t3BWCr;7VdmJJKa3P48Z!3ZYM;$BR3h1W07a$)vzdRHd(Q7~~DU{_B>jW-Nqjc;>9KP1Ly?|r=^)6iaFvs{rHo!X;+(wrKWVIfXd{8@I$`i7^ zXL`~^Z`=5QqF^52^~Ut|889713mrRBZ$wV|o&)Y-a+aLE!bTKkkDOI)a3YUQSD6mq z_~+J71?cVZz@z!B_Vt{$t!6w@AMb4hN1IFCuN(nM6X-o*Ehp{ks7 z{g>Z5k5m%rh1E5ND~i5-{VYnXl^kMBE+f?rU`(+URqL6blD4Hl@nLUNmPbrtVq7=* z(c#mVF1>wa{BbA3nOGv06$18Q%lQ>&lYrzSpDy3ItT9>_WV5kw2BEo)n2x5@a&&FS zUp@OArNh&zbnIvX$P;KTz)-sf`Yp@bMF|p(IM_uF#NFLx!m*nN9Fo#?o&1sV6T3{lsSTMrf@UcKu&>M6+m*$g`&yg0@PxmgtTsZm|CJuPB;78yNE)yp zuFUzoB|%)AX}lN7)CzV;aD`)kPvTP#lAvA3yQQ=bpl)+ucX}I?6R9PY*nTU(noaVQfL zYTj3N#e`9%&qb*CLiH!(-_f){oSji0(4hwUvD(+z30IV#1C z3G$phn8XDKoRiXH^VB~l%fHUI4Z7qq*pvDQE@=2xGmyrX-U~eua3N0OANP*JSV4QW8Ewj zU0+vTyS-pJVT4`Bx_ATj@(UlNlek(On2sEYyJnq^x8KI<$ntqCEtS}R(HkCk5|SS0 zp*{MkkwdH@kqQehPGWL25DPkce(%Q*k&GQQgu92Oe(knsN6&JFdofB`%c-d z=I|58yIbluz9qnCahoWn?tOebL^JzB2zLPj6^|apt=8g@6)5YyMyQ)VRn%E$2VJ4v3yZuYQ=1jIMXYzx z2RuB*wBnFShZ(YfIbi#7d)U>mUb_4I8IeaLfod;Li6eI)mF5xDH2X2v){{o>9sr_n zy}bJj+rGA)NqJ*a9oe@g)KATySzq*xWgkB)lGW}vdIj(aob}O3z0NnQa;r(Ke`V2Z^p|HzSnPhIbUM%;>C;DXDoa0s7MjTJiD}@+i?6#;j@T4 z&XXx=T{N#5XT0a%tH53efIkg*M(H@ z=No%N42+8O>lsSb){H~LDs!{-@8*Irb=U0-`QC8J!9{o+)CM}1GMCSv;x`($3;gog zsKmyE$msba+o>A-!d4ke7Q};rLS7M$&Sw4kppJ;~(g$W`ByNp_mekxT@r(jNw)WoT zPt#2ud@$LWB;Q*!ihH8R*r% z_8#D6wwkfnN>@H0!8RA;+~$fQ&`d2F>Gy%YZE$aLubkv*d-(KeOYK*A`gVicAdjT> z7V4BV_4|MkT>x3ia}~O8UU}hk$v)X(1taRQjgj*P?-%P*oC@q|oB(l4&-F`}#!uj- zK)3-(Eo=}JXahgeHqIuhk^DFUOUMjlN-o zt*m_goIYn@f?P(AV!ms$YrV%-QHNz5W+x1fa=t7lBJe3<6-jSQgs%6(Invf!976VX zmBiabukjHBH?V`j10ubBd}&NqCk8}#9MAt_3;k_GRURL7dw7IxEYGGSSoFth96h~3 z{M)yMO=6{msV!$b)hCe#$T6!)pBcfo7Vh~&gxY~l&*DvYl8-SVO4{S=&BkPZ6N%6C z#WDgKuY-joHlCGHMjkzSoNP70Haq~dD4kiVAE&~rZ3U}0|73+QqAlyYw;W(uaCt4j zt^8of@(uEAj!{l_KW;#We$CTo zg5w&LB=1Zm9;8d!s{?${rG@edjt{?TcZ62dsksse?Chcl0 zX@-WT*=hIP*l(u0UmSC;btiO5Ph_d3n;A+AE+iTYp|j0(v`ash*@yCYMGH|&3cQ1g z3XMbHe7K01gssyd>ap)>j6#*5U3rDo=M^S1eU|k^`MdeNrUavRsjjbHMakbkz?rOw zvK6+Yb<&N7B_5nLC7><)crlsgB{ABQ1A@T8u)JF5-(~Xz-q3_1+w-XrlpFSknP>4EoZX*VD2+Q?WJ44j+DP2ytp}ED(cI{;RT(XCVT}MYT?x9?6SSgr7 zWr?Q2Dkm)Q_Ub_3=!J$t9+WaFTJY|><^|URD998qZbMt5YNDl_m>TDZPT>K7(56ff z#EoL|VZ&O!t^Ll8^+DfXxoDuYW?^R`P{!fB<^cD#*>d34DY-CvBU{E+-WFPv8ZsW4 z=C^K&V1>{*PTHhY5Qc+YuSu`X1_iIA+!K~?Of$G99Z|94pX@eS;UoJA&@2ri4L-If z2Ucgz%dpfASNqhE-YHQA3XOZshHtoDv(AUUzCQ73DOOxJ-x}vKe3X*Wpl&{_BXQ%d zOWo-(m{*}*cB?qov{{o=!{NEn#UWaWD`hxH{_NygDagjd&7+r+t_(3YthK{W4o?&k z4=G7}bf+8HvJeqoE7gwDCWOGHs^L=%^ajN#-u7<2TCXjGUo2Fc;sT=0<#kF|M4LlK z0)2VYn0}ZkVD3M9p4Os!c;7j%z|~0?3k!0jV`zCG(RchJM+JcbOc2CIE86_p1GX6w ze4`f><`I3yMaNdxBt%?rEAXM*-Aeu`JmU$IWfRNN79b}pq@dH-ptQ0*PAl% znbGLwxo7bnZV^~PZCm7n0_l<6cY)cmZ#TLfimJ~)hR-=jhV0hjcE5hs(C7Ejl5SEa z>>~EuI9mW55{+2I4O?W@FbSDeenMAmd>NI0{e-i6G_7}xOsj8s;6c(0^e{@k`o%nl z$bLrJ)qwwLyGR2H9$7C)qY6n`X zLrB200<^ zE`w)W9(v65G-#GWsQ$gDkq>M9OPypt+q3-WFRVo@)P1z+f~8n3+)s&@@Y9t?n9Al* zHf+;8QxfDHF0zT9>#C1((HS)tB7%}%pPtFXuLP&EUrS|UxLp> zQ_Mn%L7Qa|@{Agu*GK$ovz}xWC&w8%O+$>aoA7ntB)nehbJvJhgs&7PQ@MmU2?U`- zYsdGat;;;_*%^1%xpa0HqU#G*$qYUrC;`&*b>FP`w|$E~CA8N>W>Emy}<#Z0=+C|zrpAk@vU@Qj8|?$Y}>Q%*K#j#K;g zN^_J#FluCS%7PrAwVsD1Z=G9(hGjAz5>CmSK52>XYID2S>JmIrd34{Wem-$wsK^@Y zQF>-~q4%y83y~xJykF3CQ@;$xzmVvR9{^k7m$O#X`30K(`=vV|km(dSe(JZ4ymEB( ztIs>{cCq~8vi<#XKWmtjPD}7j$P$ z{F&1Z>>s=gze9bwx0M3c=FUSNQq)>baC6J;CSEbCyxVG&%cT>^E)XTR{%5Luy51#Z zkxBXU+MVUY%dLx+Kk@ihtyj3WYO#&$8ho*oZ%I<_y>TFyutYUJ0lNzpzO- zy|hEYbU=;M88`VaK4I91)1!x*g;iqOaI zjAX{04wP9mzT32fzU8tJ`bv>WMKr*3o4;HvF0pRvh@b$%Zmk^ir>U|j7l4fx+t=3@ z;U(s1AmLK+t|1DDTiTrSC4c(NnGhwPHWwIokh4zU#>%aRZ#m|&_D zY`@2n|7b9efSs4h^zXZiZ+ckN87mb%o% zkbo62mc(4ghbN{qX?jLAqjpT-9jVB1>h-4z7WB#D4wt@sVV~&j+;tHG>?=`_cMex| zG&VrM)2=fZ%vweInvrI^oN@J>>b5az^})h9$I%4d+aDMgax|0f$sao_tb6%Rq17Ja zyyVKpg9WqaOqxa~sVTsQgGu@eE5h3&VoVQ`udv-BIMkbLe0mnUjE|74K-R9SQFwUL zi)nATC@h|0$7umlNvq-U1OzQzz(Qnh*uHhlH5R6Tlq#PTy$-T5#R>kZ&he_-OwXcL=(tz+l{zN3 z`t#K7(F*|mO}OVvHF=rnT!$qG6X2q<*8=`bx<5SPvM4tElN7olP5aaU%@@GsJ~y=# zvX#Y@?EU)LB2^2fy^~VAHLC$Tuq5D7vd0lH9h`T!I8%n&et3QGz{kb-^H@j>u{5jj z$#9B3ZJ5j-_TuEJy)6vH>{ty&++n2hO`%D-b|%&t8;0FlGk02cn9OBd(T1f8k2Y<> zYRY1cOpmHo`#^xU0c1qLVmTag$T21U@|~-+wfeB9vdxS=D6RDP?oyy5(lXQgOq`ie zXAZ6U@^}E|XFfn*Y)-isFvNpWM5^q3PSC%JIy>sA;pWhvUsX8DjI@66#tVW;pdXxA zk6~E+U74S;@G)x&`o}r1u{kQi`$8M9-)+-t8e9sn6n9 z3d2Xle;C1d6W}=G>TMZ&1CW_Rl-SMhhRgOKPs0ebpMaoVS{vnHh)ouDvw$Z>af!e)ygGN-}@ zV99v9G>(fvo%c0i?>bp~vsb-(!LTI;eJ(yR@eHxTB5y z*W`l1ql*0gK?;^G*!4cG_Hk5e=P;saXsOfK-IU(~pu|<$$ESKKT%c#A;c+dO{t$!G z%wgwM@JKg_g62S$kPt%h{qvDG+Ex&OuSoN)-%^qGY@Z3*g}9KGAj9rpYGA+XxE2+RjldPt9Ya*11nbnb0x&Ts*^57gHX;dW%BB^~Vh=mI%xOmK{Od?T1o`U)J zbK0b{4U5>;=Z}lWdfXbRJMlXcQHLHd5%IQ#n4L#n{D6F+ zMY|dtKY4~%+IL=B8lH7GrKi`Wo^BQZ89%6*toe20LDg^#*(ig*!sm)Yot zIZY_*z->@{FVLG4WR1EuyQ*B2m16B}KT6i7_9GGHmr^~o)cVA>S+Cr+5k45z3~*fA z=`LxTZ{tzT>MXZp5bK3NgUqmQXSpB(vCJ4|dBbWim~gc6-kW(Bl{DA1H@)sciHm+Q zo6?TVW}VI|#pWH!kypJ%j@ad+>j`8d`GXt2FTNHck`0-Wv7)*hkA0H*|)iPWx~{A<;21s+Qi-=m;k}ULGmrs z)I6z%u%KR`7*mIG- zGciP)xnL0U$Hr~SswwmnLsR&rX!VIdZHTrz`+!V+%Ba-aqpQMNMi#n~$ z$abbmI&F)&Y)J?^4V)XUFia8;dYoi&y?LM;FD0sbx5|?nQI5uAT}g~HL)oQN|FwP( zVIBZ~h*CDzu7-W;>Uvp0ZU}!e$6wFf{33LcWUKzJG-0-QjvK$j?r)Nr$*Q(UCTT_2 zzMyv3H$b3SE-K~+MeyQfSxzEnetNF2kd%}$5DZHg~=`t{T>^dn6z15Qz zlYChJnON~mMI};((u<06zh`z7^P$5(xPxE4Iv}&hO0(bO;~Mh*;9O>Ci%kYtMgspgG(;zcsR8o)dw-_Bxs3 zQ$>0)e&}RH=)x`224^;}uQm@M85HWy&dywQTwdPk_T*P+u`9^n$_|EGH$FH(-FwHY zmm*4!KhYPG#~eGt2`eBJN;p|A#CXEZToc3wuxYjlB%v5~boIHR@RFaNE_=5# z4&?AjbC%XtcOaWRudFnlHrD>O63%#Vg{8f1Y95`}Cg`H-sU^$$%h&xYmixY(PvJ4S zToNsF>OT@RxOA^Dz}r*3h$lzy6T9z^@X-Z)_H&GGefUBbn1DqCVOGk2{_ zbWag~*Ke&h=$8OF%l^x?^_SEA-4={>oy?Yu^zZs5FoAxn&u4x7-B{OM0eC}#YP#Kj zWVe9x?7I}`*FwU^d;j6~x1;|)f#J~f=JNE)-0%9e-~s(o9!DJA?_mD@)*ry#khSCp zN@L%5X#T5T%FnPgDZ2aK@8;K%6*#VKS)HZ7>(>tn`o%znr6PVe_-$bD1N~S1AN;Og z%uk2-zmxj^WKu&L*&AtS4*09xR5YOd?*#v~cEKt0-(3q%v$6l~+P{zZ|7%C_`@I7g XDGmM)V?-kw@TaD%eG`4d>f!$agDlNz literal 0 HcmV?d00001 diff --git a/docs/en/getting-started/example-datasets/stackoverflow.md b/docs/en/getting-started/example-datasets/stackoverflow.md index a27eaf5b8a1..def0ea306cb 100644 --- a/docs/en/getting-started/example-datasets/stackoverflow.md +++ b/docs/en/getting-started/example-datasets/stackoverflow.md @@ -13,6 +13,7 @@ Users can either download pre-prepared Parquet versions of the data, containing The following diagram shows the schema for the available tables assuming Parquet format. +![Stackoverflow schema](./images/stackoverflow.png) A description of the schema of this data can be found [here](https://meta.stackexchange.com/questions/2677/database-schema-documentation-for-the-public-data-dump-and-sede). From 422288db683de584c546ccec9ca29f02bc02a25b Mon Sep 17 00:00:00 2001 From: avogar Date: Tue, 18 Jun 2024 17:26:39 +0000 Subject: [PATCH 022/115] Check cyclic dependencies on CREATE/REPLACE/RENAME/EXCHANGE queries --- src/Interpreters/DatabaseCatalog.cpp | 108 ++++++++++++++++++ src/Interpreters/DatabaseCatalog.h | 3 + src/Interpreters/InterpreterCreateQuery.cpp | 36 +++++- src/Interpreters/InterpreterRenameQuery.cpp | 26 +++-- ...ependencies_on_create_and_rename.reference | 0 ...clic_dependencies_on_create_and_rename.sql | 75 ++++++++++++ 6 files changed, 233 insertions(+), 15 deletions(-) create mode 100644 tests/queries/0_stateless/03173_check_cyclic_dependencies_on_create_and_rename.reference create mode 100644 tests/queries/0_stateless/03173_check_cyclic_dependencies_on_create_and_rename.sql diff --git a/src/Interpreters/DatabaseCatalog.cpp b/src/Interpreters/DatabaseCatalog.cpp index 0f4c8cc26a6..ee96352c347 100644 --- a/src/Interpreters/DatabaseCatalog.cpp +++ b/src/Interpreters/DatabaseCatalog.cpp @@ -63,6 +63,7 @@ namespace ErrorCodes extern const int LOGICAL_ERROR; extern const int HAVE_DEPENDENT_OBJECTS; extern const int UNFINISHED; + extern const int INFINITE_LOOP; } class DatabaseNameHints : public IHints<> @@ -1473,6 +1474,113 @@ void DatabaseCatalog::checkTableCanBeRemovedOrRenamedUnlocked( removing_table, fmt::join(from_other_databases, ", ")); } +void DatabaseCatalog::checkTableCanBeAddedWithNoCyclicDependencies( + const QualifiedTableName & table_name, + const TableNamesSet & new_referential_dependencies, + const TableNamesSet & new_loading_dependencies) +{ + std::lock_guard lock{databases_mutex}; + + StorageID table_id = StorageID{table_name}; + + auto check = [&](TablesDependencyGraph & dependencies, const TableNamesSet & new_dependencies) + { + auto old_dependencies = dependencies.removeDependencies(table_id); + dependencies.addDependencies(table_name, new_dependencies); + + if (dependencies.hasCyclicDependencies()) + { + auto cyclic_dependencies_description = dependencies.describeCyclicDependencies(); + /// Restore previous dependencies before throwing an exception. + dependencies.removeDependencies(table_id); + dependencies.addDependencies(table_id, old_dependencies); + throw Exception( + ErrorCodes::INFINITE_LOOP, + "Cannot add dependencies for '{}', because it will lead to cyclic dependencies: {}", + table_name.getFullName(), + cyclic_dependencies_description); + } + + /// Restore previous dependencies. + dependencies.removeDependencies(table_id); + dependencies.addDependencies(table_id, old_dependencies); + }; + + check(referential_dependencies, new_referential_dependencies); + check(loading_dependencies, new_loading_dependencies); +} + +void DatabaseCatalog::checkTableCanBeRenamedWithNoCyclicDependencies(const StorageID & from_table_id, const StorageID & to_table_id) +{ + std::lock_guard lock{databases_mutex}; + + auto check = [&](TablesDependencyGraph & dependencies) + { + auto old_dependencies = dependencies.removeDependencies(from_table_id); + dependencies.addDependencies(to_table_id, old_dependencies); + + if (dependencies.hasCyclicDependencies()) + { + auto cyclic_dependencies_description = dependencies.describeCyclicDependencies(); + /// Restore previous dependencies before throwing an exception. + dependencies.removeDependencies(to_table_id); + dependencies.addDependencies(from_table_id, old_dependencies); + throw Exception( + ErrorCodes::INFINITE_LOOP, + "Cannot rename '{}' to '{}', because it will lead to cyclic dependencies: {}", + from_table_id.getFullTableName(), + to_table_id.getFullTableName(), + cyclic_dependencies_description); + } + + /// Restore previous dependencies. + dependencies.removeDependencies(to_table_id); + dependencies.addDependencies(from_table_id, old_dependencies); + }; + + check(referential_dependencies); + check(loading_dependencies); +} + +void DatabaseCatalog::checkTablesCanBeExchangedWithNoCyclicDependencies(const StorageID & table_id_1, const StorageID & table_id_2) +{ + std::lock_guard lock{databases_mutex}; + + auto check = [&](TablesDependencyGraph & dependencies) + { + auto old_dependencies_1 = dependencies.removeDependencies(table_id_1); + auto old_dependencies_2 = dependencies.removeDependencies(table_id_2); + dependencies.addDependencies(table_id_1, old_dependencies_2); + dependencies.addDependencies(table_id_2, old_dependencies_1); + auto restore_dependencies = [&]() + { + dependencies.removeDependencies(table_id_1); + dependencies.removeDependencies(table_id_2); + dependencies.addDependencies(table_id_1, old_dependencies_1); + dependencies.addDependencies(table_id_2, old_dependencies_2); + }; + + if (dependencies.hasCyclicDependencies()) + { + auto cyclic_dependencies_description = dependencies.describeCyclicDependencies(); + /// Restore previous dependencies before throwing an exception. + restore_dependencies(); + throw Exception( + ErrorCodes::INFINITE_LOOP, + "Cannot exchange '{}' and '{}', because it will lead to cyclic dependencies: {}", + table_id_1.getFullTableName(), + table_id_2.getFullTableName(), + cyclic_dependencies_description); + } + + /// Restore previous dependencies. + restore_dependencies(); + }; + + check(referential_dependencies); + check(loading_dependencies); +} + void DatabaseCatalog::cleanupStoreDirectoryTask() { for (const auto & [disk_name, disk] : getContext()->getDisksMap()) diff --git a/src/Interpreters/DatabaseCatalog.h b/src/Interpreters/DatabaseCatalog.h index 37125d9900c..a5f1f47d7a0 100644 --- a/src/Interpreters/DatabaseCatalog.h +++ b/src/Interpreters/DatabaseCatalog.h @@ -244,6 +244,9 @@ public: void checkTableCanBeRemovedOrRenamed(const StorageID & table_id, bool check_referential_dependencies, bool check_loading_dependencies, bool is_drop_database = false) const; + void checkTableCanBeAddedWithNoCyclicDependencies(const QualifiedTableName & table_name, const TableNamesSet & new_referential_dependencies, const TableNamesSet & new_loading_dependencies); + void checkTableCanBeRenamedWithNoCyclicDependencies(const StorageID & from_table_id, const StorageID & to_table_id); + void checkTablesCanBeExchangedWithNoCyclicDependencies(const StorageID & table_id_1, const StorageID & table_id_2); struct TableMarkedAsDropped { diff --git a/src/Interpreters/InterpreterCreateQuery.cpp b/src/Interpreters/InterpreterCreateQuery.cpp index a78f6cc39ef..e0d743b130e 100644 --- a/src/Interpreters/InterpreterCreateQuery.cpp +++ b/src/Interpreters/InterpreterCreateQuery.cpp @@ -1077,6 +1077,27 @@ void InterpreterCreateQuery::assertOrSetUUID(ASTCreateQuery & create, const Data } +namespace +{ + +void addTableDependencies(const ASTCreateQuery & create, const ASTPtr & query_ptr, const ContextPtr & context) +{ + QualifiedTableName qualified_name{create.getDatabase(), create.getTable()}; + auto ref_dependencies = getDependenciesFromCreateQuery(context->getGlobalContext(), qualified_name, query_ptr); + auto loading_dependencies = getLoadingDependenciesFromCreateQuery(context->getGlobalContext(), qualified_name, query_ptr); + DatabaseCatalog::instance().addDependencies(qualified_name, ref_dependencies, loading_dependencies); +} + +void checkTableCanBeAddedWithNoCyclicDependencies(const ASTCreateQuery & create, const ASTPtr & query_ptr, const ContextPtr & context) +{ + QualifiedTableName qualified_name{create.getDatabase(), create.getTable()}; + auto ref_dependencies = getDependenciesFromCreateQuery(context->getGlobalContext(), qualified_name, query_ptr); + auto loading_dependencies = getLoadingDependenciesFromCreateQuery(context->getGlobalContext(), qualified_name, query_ptr); + DatabaseCatalog::instance().checkTableCanBeAddedWithNoCyclicDependencies(qualified_name, ref_dependencies, loading_dependencies); +} + +} + BlockIO InterpreterCreateQuery::createTable(ASTCreateQuery & create) { /// Temporary tables are created out of databases. @@ -1322,11 +1343,7 @@ BlockIO InterpreterCreateQuery::createTable(ASTCreateQuery & create) return {}; /// If table has dependencies - add them to the graph - QualifiedTableName qualified_name{database_name, create.getTable()}; - auto ref_dependencies = getDependenciesFromCreateQuery(getContext()->getGlobalContext(), qualified_name, query_ptr); - auto loading_dependencies = getLoadingDependenciesFromCreateQuery(getContext()->getGlobalContext(), qualified_name, query_ptr); - DatabaseCatalog::instance().addDependencies(qualified_name, ref_dependencies, loading_dependencies); - + addTableDependencies(create, query_ptr, getContext()); return fillTableIfNeeded(create); } @@ -1478,6 +1495,9 @@ bool InterpreterCreateQuery::doCreateTable(ASTCreateQuery & create, throw Exception(ErrorCodes::LOGICAL_ERROR, "Cannot find UUID mapping for {}, it's a bug", create.uuid); } + /// Before actually creating the table, check if it will lead to cyclic dependencies. + checkTableCanBeAddedWithNoCyclicDependencies(create, query_ptr, getContext()); + StoragePtr res; /// NOTE: CREATE query may be rewritten by Storage creator or table function if (create.as_table_function) @@ -1578,6 +1598,9 @@ BlockIO InterpreterCreateQuery::doCreateOrReplaceTable(ASTCreateQuery & create, ContextMutablePtr create_context = Context::createCopy(current_context); create_context->setQueryContext(std::const_pointer_cast(current_context)); + /// Before actually creating/replacing the table, check if it will lead to cyclic dependencies. + checkTableCanBeAddedWithNoCyclicDependencies(create, query_ptr, create_context); + auto make_drop_context = [&]() -> ContextMutablePtr { ContextMutablePtr drop_context = Context::createCopy(current_context); @@ -1624,6 +1647,9 @@ BlockIO InterpreterCreateQuery::doCreateOrReplaceTable(ASTCreateQuery & create, assert(done); created = true; + /// If table has dependencies - add them to the graph + addTableDependencies(create, query_ptr, getContext()); + /// Try fill temporary table BlockIO fill_io = fillTableIfNeeded(create); executeTrivialBlockIO(fill_io, getContext()); diff --git a/src/Interpreters/InterpreterRenameQuery.cpp b/src/Interpreters/InterpreterRenameQuery.cpp index eeb762b4d7e..b34368fe4ab 100644 --- a/src/Interpreters/InterpreterRenameQuery.cpp +++ b/src/Interpreters/InterpreterRenameQuery.cpp @@ -127,15 +127,17 @@ BlockIO InterpreterRenameQuery::executeToTables(const ASTRenameQuery & rename, c { StorageID from_table_id{elem.from_database_name, elem.from_table_name}; StorageID to_table_id{elem.to_database_name, elem.to_table_name}; - std::vector ref_dependencies; - std::vector loading_dependencies; - if (!exchange_tables) - { - bool check_ref_deps = getContext()->getSettingsRef().check_referential_table_dependencies; - bool check_loading_deps = !check_ref_deps && getContext()->getSettingsRef().check_table_dependencies; - std::tie(ref_dependencies, loading_dependencies) = database_catalog.removeDependencies(from_table_id, check_ref_deps, check_loading_deps); - } + if (exchange_tables) + DatabaseCatalog::instance().checkTablesCanBeExchangedWithNoCyclicDependencies(from_table_id, to_table_id); + else + DatabaseCatalog::instance().checkTableCanBeRenamedWithNoCyclicDependencies(from_table_id, to_table_id); + + bool check_ref_deps = getContext()->getSettingsRef().check_referential_table_dependencies; + bool check_loading_deps = !check_ref_deps && getContext()->getSettingsRef().check_table_dependencies; + auto [from_ref_dependencies, from_loading_dependencies] = database_catalog.removeDependencies(from_table_id, check_ref_deps, check_loading_deps); + /// In case of just rename to_ref_dependencies and to_loading_dependencies will be empty. + auto [to_ref_dependencies, to_loading_dependencies] = database_catalog.removeDependencies(to_table_id, check_ref_deps, check_loading_deps); try { @@ -147,12 +149,16 @@ BlockIO InterpreterRenameQuery::executeToTables(const ASTRenameQuery & rename, c exchange_tables, rename.dictionary); - DatabaseCatalog::instance().addDependencies(to_table_id, ref_dependencies, loading_dependencies); + DatabaseCatalog::instance().addDependencies(to_table_id, from_ref_dependencies, from_loading_dependencies); + /// In case of just rename to_ref_dependencies and to_loading_dependencies will be empty and no dependencies will be added. + DatabaseCatalog::instance().addDependencies(from_table_id, to_ref_dependencies, to_loading_dependencies); + } catch (...) { /// Restore dependencies if RENAME fails - DatabaseCatalog::instance().addDependencies(from_table_id, ref_dependencies, loading_dependencies); + DatabaseCatalog::instance().addDependencies(from_table_id, from_ref_dependencies, from_loading_dependencies); + DatabaseCatalog::instance().addDependencies(to_table_id, to_ref_dependencies, to_loading_dependencies); throw; } } diff --git a/tests/queries/0_stateless/03173_check_cyclic_dependencies_on_create_and_rename.reference b/tests/queries/0_stateless/03173_check_cyclic_dependencies_on_create_and_rename.reference new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tests/queries/0_stateless/03173_check_cyclic_dependencies_on_create_and_rename.sql b/tests/queries/0_stateless/03173_check_cyclic_dependencies_on_create_and_rename.sql new file mode 100644 index 00000000000..39dda475d1e --- /dev/null +++ b/tests/queries/0_stateless/03173_check_cyclic_dependencies_on_create_and_rename.sql @@ -0,0 +1,75 @@ +DROP TABLE IF EXISTS test; +CREATE TABLE test (id UInt64, value String) ENGINE=MergeTree ORDER BY id; +INSERT INTO test SELECT number, 'str_' || toString(number) FROM numbers(10); +DROP DICTIONARY IF EXISTS test_dict; +CREATE DICTIONARY test_dict +( + id UInt64, + value String +) +PRIMARY KEY id +SOURCE(CLICKHOUSE(QUERY 'SELECT * FROM test')) +LAYOUT(FLAT()) +LIFETIME(MIN 0 MAX 1000); +DROP TABLE IF EXISTS view_source; +CREATE TABLE view_source (id UInt64) ENGINE=MergeTree ORDER BY id; +INSERT INTO view_source SELECT * FROM numbers(5); +DROP VIEW IF EXISTS view; +CREATE VIEW view AS SELECT id, dictGet('test_dict', 'value', id) FROM view_source; + +CREATE OR REPLACE DICTIONARY test_dict +( + id UInt64, + value String +) +PRIMARY KEY id +SOURCE(CLICKHOUSE(QUERY 'SELECT * FROM view')) +LAYOUT(FLAT()) +LIFETIME(MIN 0 MAX 1000); -- {serverError INFINITE_LOOP} + +REPLACE DICTIONARY test_dict +( + id UInt64, + value String +) +PRIMARY KEY id +SOURCE(CLICKHOUSE(QUERY 'SELECT * FROM view')) +LAYOUT(FLAT()) +LIFETIME(MIN 0 MAX 1000); -- {serverError INFINITE_LOOP} + + +DROP DICTIONARY IF EXISTS test_dict_2; +CREATE DICTIONARY test_dict_2 +( + id UInt64, + value String +) +PRIMARY KEY id +SOURCE(CLICKHOUSE(QUERY 'SELECT * FROM view')) +LAYOUT(FLAT()) +LIFETIME(MIN 0 MAX 1000); + +EXCHANGE DICTIONARIES test_dict AND test_dict_2; -- {serverError INFINITE_LOOP} + +DROP DICTIONARY test_dict_2; + +CREATE OR REPLACE DICTIONARY test_dict_2 +( + id UInt64, + value String +) +PRIMARY KEY id +SOURCE(CLICKHOUSE(QUERY 'SELECT * FROM view')) +LAYOUT(FLAT()) +LIFETIME(MIN 0 MAX 1000); + +EXCHANGE DICTIONARIES test_dict AND test_dict_2; -- {serverError INFINITE_LOOP} + +DROP DICTIONARY test_dict; +RENAME DICTIONARY test_dict_2 to test_dict; -- {serverError INFINITE_LOOP} + +DROP DICTIONARY test_dict_2; +DROP VIEW view; +DROP TABLE test; +DROP TABLE view_source; + From bfb68da3507f82a5d77cc7cbf1bbd5bd12cc3200 Mon Sep 17 00:00:00 2001 From: avogar Date: Tue, 18 Jun 2024 17:33:31 +0000 Subject: [PATCH 023/115] Make the code better --- src/Interpreters/DatabaseCatalog.cpp | 29 ++++++++++++++-------------- 1 file changed, 15 insertions(+), 14 deletions(-) diff --git a/src/Interpreters/DatabaseCatalog.cpp b/src/Interpreters/DatabaseCatalog.cpp index ee96352c347..aaec94a4fb0 100644 --- a/src/Interpreters/DatabaseCatalog.cpp +++ b/src/Interpreters/DatabaseCatalog.cpp @@ -1487,13 +1487,17 @@ void DatabaseCatalog::checkTableCanBeAddedWithNoCyclicDependencies( { auto old_dependencies = dependencies.removeDependencies(table_id); dependencies.addDependencies(table_name, new_dependencies); + auto restore_dependencies = [&]() + { + dependencies.removeDependencies(table_id); + if (!old_dependencies.empty()) + dependencies.addDependencies(table_id, old_dependencies); + }; if (dependencies.hasCyclicDependencies()) { auto cyclic_dependencies_description = dependencies.describeCyclicDependencies(); - /// Restore previous dependencies before throwing an exception. - dependencies.removeDependencies(table_id); - dependencies.addDependencies(table_id, old_dependencies); + restore_dependencies(); throw Exception( ErrorCodes::INFINITE_LOOP, "Cannot add dependencies for '{}', because it will lead to cyclic dependencies: {}", @@ -1501,9 +1505,7 @@ void DatabaseCatalog::checkTableCanBeAddedWithNoCyclicDependencies( cyclic_dependencies_description); } - /// Restore previous dependencies. - dependencies.removeDependencies(table_id); - dependencies.addDependencies(table_id, old_dependencies); + restore_dependencies(); }; check(referential_dependencies, new_referential_dependencies); @@ -1518,13 +1520,16 @@ void DatabaseCatalog::checkTableCanBeRenamedWithNoCyclicDependencies(const Stora { auto old_dependencies = dependencies.removeDependencies(from_table_id); dependencies.addDependencies(to_table_id, old_dependencies); + auto restore_dependencies = [&]() + { + dependencies.removeDependencies(to_table_id); + dependencies.addDependencies(from_table_id, old_dependencies); + }; if (dependencies.hasCyclicDependencies()) { auto cyclic_dependencies_description = dependencies.describeCyclicDependencies(); - /// Restore previous dependencies before throwing an exception. - dependencies.removeDependencies(to_table_id); - dependencies.addDependencies(from_table_id, old_dependencies); + restore_dependencies(); throw Exception( ErrorCodes::INFINITE_LOOP, "Cannot rename '{}' to '{}', because it will lead to cyclic dependencies: {}", @@ -1533,9 +1538,7 @@ void DatabaseCatalog::checkTableCanBeRenamedWithNoCyclicDependencies(const Stora cyclic_dependencies_description); } - /// Restore previous dependencies. - dependencies.removeDependencies(to_table_id); - dependencies.addDependencies(from_table_id, old_dependencies); + restore_dependencies(); }; check(referential_dependencies); @@ -1563,7 +1566,6 @@ void DatabaseCatalog::checkTablesCanBeExchangedWithNoCyclicDependencies(const St if (dependencies.hasCyclicDependencies()) { auto cyclic_dependencies_description = dependencies.describeCyclicDependencies(); - /// Restore previous dependencies before throwing an exception. restore_dependencies(); throw Exception( ErrorCodes::INFINITE_LOOP, @@ -1573,7 +1575,6 @@ void DatabaseCatalog::checkTablesCanBeExchangedWithNoCyclicDependencies(const St cyclic_dependencies_description); } - /// Restore previous dependencies. restore_dependencies(); }; From bc5db30669f90fc0b2b13811b5c6539a16cb2429 Mon Sep 17 00:00:00 2001 From: avogar Date: Tue, 18 Jun 2024 19:08:24 +0000 Subject: [PATCH 024/115] Fix usage of current_database --- src/Databases/DDLLoadingDependencyVisitor.cpp | 4 +-- src/Databases/DDLLoadingDependencyVisitor.h | 2 +- src/Databases/DatabaseMemory.cpp | 4 +-- src/Databases/DatabaseOrdinary.cpp | 4 +-- src/Databases/TablesLoader.cpp | 2 +- src/Interpreters/InterpreterCreateQuery.cpp | 12 ++++++--- src/Interpreters/InterpreterRenameQuery.cpp | 26 ++++++++++++------- .../0_stateless/01191_rename_dictionary.sql | 2 +- ...clic_dependencies_on_create_and_rename.sql | 12 ++++----- 9 files changed, 40 insertions(+), 28 deletions(-) diff --git a/src/Databases/DDLLoadingDependencyVisitor.cpp b/src/Databases/DDLLoadingDependencyVisitor.cpp index b8690125aaa..0253709fb6e 100644 --- a/src/Databases/DDLLoadingDependencyVisitor.cpp +++ b/src/Databases/DDLLoadingDependencyVisitor.cpp @@ -21,11 +21,11 @@ namespace DB using TableLoadingDependenciesVisitor = DDLLoadingDependencyVisitor::Visitor; -TableNamesSet getLoadingDependenciesFromCreateQuery(ContextPtr global_context, const QualifiedTableName & table, const ASTPtr & ast) +TableNamesSet getLoadingDependenciesFromCreateQuery(ContextPtr global_context, const QualifiedTableName & table, const ASTPtr & ast, const String & default_database) { assert(global_context == global_context->getGlobalContext()); TableLoadingDependenciesVisitor::Data data; - data.default_database = global_context->getCurrentDatabase(); + data.default_database = default_database; data.create_query = ast; data.global_context = global_context; data.table_name = table; diff --git a/src/Databases/DDLLoadingDependencyVisitor.h b/src/Databases/DDLLoadingDependencyVisitor.h index a9e9f4d7a53..099d9c1979b 100644 --- a/src/Databases/DDLLoadingDependencyVisitor.h +++ b/src/Databases/DDLLoadingDependencyVisitor.h @@ -16,7 +16,7 @@ using TableNamesSet = std::unordered_set; /// Returns a list of all tables which should be loaded before a specified table. /// For example, a local ClickHouse table should be loaded before a dictionary which uses that table as its source. /// Does not validate AST, works a best-effort way. -TableNamesSet getLoadingDependenciesFromCreateQuery(ContextPtr global_context, const QualifiedTableName & table, const ASTPtr & ast); +TableNamesSet getLoadingDependenciesFromCreateQuery(ContextPtr global_context, const QualifiedTableName & table, const ASTPtr & ast, const String & default_database); class DDLMatcherBase diff --git a/src/Databases/DatabaseMemory.cpp b/src/Databases/DatabaseMemory.cpp index b82cf885b4a..dce20e3ac6f 100644 --- a/src/Databases/DatabaseMemory.cpp +++ b/src/Databases/DatabaseMemory.cpp @@ -154,8 +154,8 @@ void DatabaseMemory::alterTable(ContextPtr local_context, const StorageID & tabl applyMetadataChangesToCreateQuery(it->second, metadata); /// The create query of the table has been just changed, we need to update dependencies too. - auto ref_dependencies = getDependenciesFromCreateQuery(local_context->getGlobalContext(), table_id.getQualifiedName(), it->second); - auto loading_dependencies = getLoadingDependenciesFromCreateQuery(local_context->getGlobalContext(), table_id.getQualifiedName(), it->second); + auto ref_dependencies = getDependenciesFromCreateQuery(local_context, table_id.getQualifiedName(), it->second); + auto loading_dependencies = getLoadingDependenciesFromCreateQuery(local_context->getGlobalContext(), table_id.getQualifiedName(), it->second, local_context->getCurrentDatabase()); DatabaseCatalog::instance().updateDependencies(table_id, ref_dependencies, loading_dependencies); } diff --git a/src/Databases/DatabaseOrdinary.cpp b/src/Databases/DatabaseOrdinary.cpp index 10a8e06e8f0..18c92e6bcbc 100644 --- a/src/Databases/DatabaseOrdinary.cpp +++ b/src/Databases/DatabaseOrdinary.cpp @@ -539,8 +539,8 @@ void DatabaseOrdinary::alterTable(ContextPtr local_context, const StorageID & ta } /// The create query of the table has been just changed, we need to update dependencies too. - auto ref_dependencies = getDependenciesFromCreateQuery(local_context->getGlobalContext(), table_id.getQualifiedName(), ast); - auto loading_dependencies = getLoadingDependenciesFromCreateQuery(local_context->getGlobalContext(), table_id.getQualifiedName(), ast); + auto ref_dependencies = getDependenciesFromCreateQuery(local_context, table_id.getQualifiedName(), ast); + auto loading_dependencies = getLoadingDependenciesFromCreateQuery(local_context->getGlobalContext(), table_id.getQualifiedName(), ast, local_context->getCurrentDatabase()); DatabaseCatalog::instance().updateDependencies(table_id, ref_dependencies, loading_dependencies); commitAlterTable(table_id, table_metadata_tmp_path, table_metadata_path, statement, local_context); diff --git a/src/Databases/TablesLoader.cpp b/src/Databases/TablesLoader.cpp index 6aa13b7b759..4bfe44ba72c 100644 --- a/src/Databases/TablesLoader.cpp +++ b/src/Databases/TablesLoader.cpp @@ -138,7 +138,7 @@ void TablesLoader::buildDependencyGraph() for (const auto & [table_name, table_metadata] : metadata.parsed_tables) { auto new_ref_dependencies = getDependenciesFromCreateQuery(global_context, table_name, table_metadata.ast); - auto new_loading_dependencies = getLoadingDependenciesFromCreateQuery(global_context, table_name, table_metadata.ast); + auto new_loading_dependencies = getLoadingDependenciesFromCreateQuery(global_context, table_name, table_metadata.ast, global_context->getCurrentDatabase()); if (!new_ref_dependencies.empty()) referential_dependencies.addDependencies(table_name, new_ref_dependencies); diff --git a/src/Interpreters/InterpreterCreateQuery.cpp b/src/Interpreters/InterpreterCreateQuery.cpp index e0d743b130e..f6b6b34413a 100644 --- a/src/Interpreters/InterpreterCreateQuery.cpp +++ b/src/Interpreters/InterpreterCreateQuery.cpp @@ -1083,16 +1083,20 @@ namespace void addTableDependencies(const ASTCreateQuery & create, const ASTPtr & query_ptr, const ContextPtr & context) { QualifiedTableName qualified_name{create.getDatabase(), create.getTable()}; - auto ref_dependencies = getDependenciesFromCreateQuery(context->getGlobalContext(), qualified_name, query_ptr); - auto loading_dependencies = getLoadingDependenciesFromCreateQuery(context->getGlobalContext(), qualified_name, query_ptr); + auto ref_dependencies = getDependenciesFromCreateQuery(context, qualified_name, query_ptr); + String ref_dependencies_str; + for (const auto & ref_dep : ref_dependencies) + ref_dependencies_str += ref_dep.getFullName() + ","; + LOG_DEBUG(getLogger("InterpreterCreateQuery"), "Add ref dependencies for table {}: {}", qualified_name.getFullName(), ref_dependencies_str); + auto loading_dependencies = getLoadingDependenciesFromCreateQuery(context->getGlobalContext(), qualified_name, query_ptr, context->getCurrentDatabase()); DatabaseCatalog::instance().addDependencies(qualified_name, ref_dependencies, loading_dependencies); } void checkTableCanBeAddedWithNoCyclicDependencies(const ASTCreateQuery & create, const ASTPtr & query_ptr, const ContextPtr & context) { QualifiedTableName qualified_name{create.getDatabase(), create.getTable()}; - auto ref_dependencies = getDependenciesFromCreateQuery(context->getGlobalContext(), qualified_name, query_ptr); - auto loading_dependencies = getLoadingDependenciesFromCreateQuery(context->getGlobalContext(), qualified_name, query_ptr); + auto ref_dependencies = getDependenciesFromCreateQuery(context, qualified_name, query_ptr); + auto loading_dependencies = getLoadingDependenciesFromCreateQuery(context->getGlobalContext(), qualified_name, query_ptr, context->getCurrentDatabase()); DatabaseCatalog::instance().checkTableCanBeAddedWithNoCyclicDependencies(qualified_name, ref_dependencies, loading_dependencies); } diff --git a/src/Interpreters/InterpreterRenameQuery.cpp b/src/Interpreters/InterpreterRenameQuery.cpp index b34368fe4ab..32c475d138f 100644 --- a/src/Interpreters/InterpreterRenameQuery.cpp +++ b/src/Interpreters/InterpreterRenameQuery.cpp @@ -127,17 +127,24 @@ BlockIO InterpreterRenameQuery::executeToTables(const ASTRenameQuery & rename, c { StorageID from_table_id{elem.from_database_name, elem.from_table_name}; StorageID to_table_id{elem.to_database_name, elem.to_table_name}; + std::vector from_ref_dependencies; + std::vector from_loading_dependencies; + std::vector to_ref_dependencies; + std::vector to_loading_dependencies; if (exchange_tables) + { DatabaseCatalog::instance().checkTablesCanBeExchangedWithNoCyclicDependencies(from_table_id, to_table_id); + std::tie(from_ref_dependencies, from_loading_dependencies) = database_catalog.removeDependencies(from_table_id, false, false); + std::tie(to_ref_dependencies, to_loading_dependencies) = database_catalog.removeDependencies(to_table_id, false, false); + } else + { DatabaseCatalog::instance().checkTableCanBeRenamedWithNoCyclicDependencies(from_table_id, to_table_id); - - bool check_ref_deps = getContext()->getSettingsRef().check_referential_table_dependencies; - bool check_loading_deps = !check_ref_deps && getContext()->getSettingsRef().check_table_dependencies; - auto [from_ref_dependencies, from_loading_dependencies] = database_catalog.removeDependencies(from_table_id, check_ref_deps, check_loading_deps); - /// In case of just rename to_ref_dependencies and to_loading_dependencies will be empty. - auto [to_ref_dependencies, to_loading_dependencies] = database_catalog.removeDependencies(to_table_id, check_ref_deps, check_loading_deps); + bool check_ref_deps = getContext()->getSettingsRef().check_referential_table_dependencies; + bool check_loading_deps = !check_ref_deps && getContext()->getSettingsRef().check_table_dependencies; + std::tie(from_ref_dependencies, from_loading_dependencies) = database_catalog.removeDependencies(from_table_id, check_ref_deps, check_loading_deps); + } try { @@ -150,15 +157,16 @@ BlockIO InterpreterRenameQuery::executeToTables(const ASTRenameQuery & rename, c rename.dictionary); DatabaseCatalog::instance().addDependencies(to_table_id, from_ref_dependencies, from_loading_dependencies); - /// In case of just rename to_ref_dependencies and to_loading_dependencies will be empty and no dependencies will be added. - DatabaseCatalog::instance().addDependencies(from_table_id, to_ref_dependencies, to_loading_dependencies); + if (!to_ref_dependencies.empty() || !to_loading_dependencies.empty()) + DatabaseCatalog::instance().addDependencies(from_table_id, to_ref_dependencies, to_loading_dependencies); } catch (...) { /// Restore dependencies if RENAME fails DatabaseCatalog::instance().addDependencies(from_table_id, from_ref_dependencies, from_loading_dependencies); - DatabaseCatalog::instance().addDependencies(to_table_id, to_ref_dependencies, to_loading_dependencies); + if (!to_ref_dependencies.empty() || !to_loading_dependencies.empty()) + DatabaseCatalog::instance().addDependencies(to_table_id, to_ref_dependencies, to_loading_dependencies); throw; } } diff --git a/tests/queries/0_stateless/01191_rename_dictionary.sql b/tests/queries/0_stateless/01191_rename_dictionary.sql index 6666c3308ca..c5012dabc81 100644 --- a/tests/queries/0_stateless/01191_rename_dictionary.sql +++ b/tests/queries/0_stateless/01191_rename_dictionary.sql @@ -17,7 +17,7 @@ SELECT name, status FROM system.dictionaries WHERE database='test_01191'; SELECT name, engine FROM system.tables WHERE database='test_01191' ORDER BY name; RENAME DICTIONARY test_01191.table TO test_01191.table1; -- {serverError UNKNOWN_TABLE} -EXCHANGE DICTIONARIES test_01191._ AND test_01191.dict; -- {serverError INCORRECT_QUERY} +EXCHANGE DICTIONARIES test_01191._ AND test_01191.dict; -- {serverError INFINITE_LOOP} EXCHANGE TABLES test_01191.t AND test_01191.dict; SELECT name, status FROM system.dictionaries WHERE database='test_01191'; SELECT name, engine FROM system.tables WHERE database='test_01191' ORDER BY name; diff --git a/tests/queries/0_stateless/03173_check_cyclic_dependencies_on_create_and_rename.sql b/tests/queries/0_stateless/03173_check_cyclic_dependencies_on_create_and_rename.sql index 39dda475d1e..e6215c3da1e 100644 --- a/tests/queries/0_stateless/03173_check_cyclic_dependencies_on_create_and_rename.sql +++ b/tests/queries/0_stateless/03173_check_cyclic_dependencies_on_create_and_rename.sql @@ -8,14 +8,14 @@ CREATE DICTIONARY test_dict value String ) PRIMARY KEY id -SOURCE(CLICKHOUSE(QUERY 'SELECT * FROM test')) +SOURCE(CLICKHOUSE(TABLE test)) LAYOUT(FLAT()) LIFETIME(MIN 0 MAX 1000); DROP TABLE IF EXISTS view_source; CREATE TABLE view_source (id UInt64) ENGINE=MergeTree ORDER BY id; INSERT INTO view_source SELECT * FROM numbers(5); DROP VIEW IF EXISTS view; -CREATE VIEW view AS SELECT id, dictGet('test_dict', 'value', id) FROM view_source; +CREATE VIEW view AS SELECT id, dictGet('test_dict', 'value', id) as value FROM view_source; CREATE OR REPLACE DICTIONARY test_dict ( @@ -23,7 +23,7 @@ CREATE OR REPLACE DICTIONARY test_dict value String ) PRIMARY KEY id -SOURCE(CLICKHOUSE(QUERY 'SELECT * FROM view')) +SOURCE(CLICKHOUSE(TABLE view)) LAYOUT(FLAT()) LIFETIME(MIN 0 MAX 1000); -- {serverError INFINITE_LOOP} @@ -33,7 +33,7 @@ REPLACE DICTIONARY test_dict value String ) PRIMARY KEY id -SOURCE(CLICKHOUSE(QUERY 'SELECT * FROM view')) +SOURCE(CLICKHOUSE(TABLE view)) LAYOUT(FLAT()) LIFETIME(MIN 0 MAX 1000); -- {serverError INFINITE_LOOP} @@ -45,7 +45,7 @@ CREATE DICTIONARY test_dict_2 value String ) PRIMARY KEY id -SOURCE(CLICKHOUSE(QUERY 'SELECT * FROM view')) +SOURCE(CLICKHOUSE(TABLE view)) LAYOUT(FLAT()) LIFETIME(MIN 0 MAX 1000); @@ -59,7 +59,7 @@ CREATE OR REPLACE DICTIONARY test_dict_2 value String ) PRIMARY KEY id -SOURCE(CLICKHOUSE(QUERY 'SELECT * FROM view')) +SOURCE(CLICKHOUSE(TABLE view)) LAYOUT(FLAT()) LIFETIME(MIN 0 MAX 1000); From 1c15ad54f4345a6c719d8c0929833b3fab73d5ad Mon Sep 17 00:00:00 2001 From: avogar Date: Tue, 18 Jun 2024 19:11:38 +0000 Subject: [PATCH 025/115] Remove debug logging --- src/Interpreters/InterpreterCreateQuery.cpp | 4 ---- 1 file changed, 4 deletions(-) diff --git a/src/Interpreters/InterpreterCreateQuery.cpp b/src/Interpreters/InterpreterCreateQuery.cpp index f6b6b34413a..35207bc9d39 100644 --- a/src/Interpreters/InterpreterCreateQuery.cpp +++ b/src/Interpreters/InterpreterCreateQuery.cpp @@ -1084,10 +1084,6 @@ void addTableDependencies(const ASTCreateQuery & create, const ASTPtr & query_pt { QualifiedTableName qualified_name{create.getDatabase(), create.getTable()}; auto ref_dependencies = getDependenciesFromCreateQuery(context, qualified_name, query_ptr); - String ref_dependencies_str; - for (const auto & ref_dep : ref_dependencies) - ref_dependencies_str += ref_dep.getFullName() + ","; - LOG_DEBUG(getLogger("InterpreterCreateQuery"), "Add ref dependencies for table {}: {}", qualified_name.getFullName(), ref_dependencies_str); auto loading_dependencies = getLoadingDependenciesFromCreateQuery(context->getGlobalContext(), qualified_name, query_ptr, context->getCurrentDatabase()); DatabaseCatalog::instance().addDependencies(qualified_name, ref_dependencies, loading_dependencies); } From a577c87cf1d1bdeae204371e82071de401c4f61c Mon Sep 17 00:00:00 2001 From: avogar Date: Tue, 18 Jun 2024 20:21:20 +0000 Subject: [PATCH 026/115] Fix tests --- src/Databases/DDLDependencyVisitor.cpp | 5 +++++ .../00987_distributed_stack_overflow.sql | 6 +----- .../0_stateless/01552_dict_fixedstring.sql | 2 +- ...0_ddl_dictionary_use_current_database_name.sql | 2 +- .../0_stateless/01763_max_distributed_depth.sql | 15 +-------------- .../01764_table_function_dictionary.sql | 2 +- .../01804_dictionary_decimal256_type.sql | 2 ++ .../01904_dictionary_default_nullable_type.sql | 2 ++ .../queries/0_stateless/01910_view_dictionary.sql | 2 +- .../02008_complex_key_range_hashed_dictionary.sql | 4 ++-- .../0_stateless/02155_dictionary_comment.sql | 2 +- .../0_stateless/02183_dictionary_date_types.sql | 2 +- .../02185_range_hashed_dictionary_open_ranges.sql | 1 + .../0_stateless/02391_recursive_buffer.sql | 6 +----- ...03169_cache_complex_dict_short_circuit_bug.sql | 2 +- 15 files changed, 22 insertions(+), 33 deletions(-) diff --git a/src/Databases/DDLDependencyVisitor.cpp b/src/Databases/DDLDependencyVisitor.cpp index 75a01a6190f..fdc51f4f43d 100644 --- a/src/Databases/DDLDependencyVisitor.cpp +++ b/src/Databases/DDLDependencyVisitor.cpp @@ -95,6 +95,11 @@ namespace as_table.database = current_database; dependencies.emplace(as_table); } + + /// Visit nested select query only for views, for other cases it's not + /// an actual dependency as it will be executed only once to fill the table. + if (create.select && !create.isView()) + skip_asts.insert(create.select); } /// The definition of a dictionary: SOURCE(CLICKHOUSE(...)) LAYOUT(...) LIFETIME(...) diff --git a/tests/queries/0_stateless/00987_distributed_stack_overflow.sql b/tests/queries/0_stateless/00987_distributed_stack_overflow.sql index 5a22ac56413..ba58713fe0e 100644 --- a/tests/queries/0_stateless/00987_distributed_stack_overflow.sql +++ b/tests/queries/0_stateless/00987_distributed_stack_overflow.sql @@ -9,10 +9,6 @@ CREATE TABLE distr (x UInt8) ENGINE = Distributed(test_shard_localhost, currentD CREATE TABLE distr0 (x UInt8) ENGINE = Distributed(test_shard_localhost, '', distr0); -- { serverError INFINITE_LOOP } CREATE TABLE distr1 (x UInt8) ENGINE = Distributed(test_shard_localhost, currentDatabase(), distr2); -CREATE TABLE distr2 (x UInt8) ENGINE = Distributed(test_shard_localhost, currentDatabase(), distr1); - -SELECT * FROM distr1; -- { serverError TOO_LARGE_DISTRIBUTED_DEPTH } -SELECT * FROM distr2; -- { serverError TOO_LARGE_DISTRIBUTED_DEPTH } +CREATE TABLE distr2 (x UInt8) ENGINE = Distributed(test_shard_localhost, currentDatabase(), distr1); -- { serverError INFINITE_LOOP } DROP TABLE distr1; -DROP TABLE distr2; diff --git a/tests/queries/0_stateless/01552_dict_fixedstring.sql b/tests/queries/0_stateless/01552_dict_fixedstring.sql index 01d55656e3c..0b19c9980a4 100644 --- a/tests/queries/0_stateless/01552_dict_fixedstring.sql +++ b/tests/queries/0_stateless/01552_dict_fixedstring.sql @@ -16,5 +16,5 @@ LIFETIME(MIN 10 MAX 10); SELECT dictGet(currentDatabase() || '.dict', 's', number) FROM numbers(2); -DROP TABLE src; DROP DICTIONARY dict; +DROP TABLE src; diff --git a/tests/queries/0_stateless/01760_ddl_dictionary_use_current_database_name.sql b/tests/queries/0_stateless/01760_ddl_dictionary_use_current_database_name.sql index 55c0d1e3678..a7f04921f1f 100644 --- a/tests/queries/0_stateless/01760_ddl_dictionary_use_current_database_name.sql +++ b/tests/queries/0_stateless/01760_ddl_dictionary_use_current_database_name.sql @@ -27,5 +27,5 @@ SELECT dictGet('ddl_dictionary_test', 'value', number) FROM system.numbers LIMIT SELECT 'dictHas'; SELECT dictHas('ddl_dictionary_test', number) FROM system.numbers LIMIT 3; -DROP TABLE ddl_dictonary_test_source; DROP DICTIONARY ddl_dictionary_test; +DROP TABLE ddl_dictonary_test_source; diff --git a/tests/queries/0_stateless/01763_max_distributed_depth.sql b/tests/queries/0_stateless/01763_max_distributed_depth.sql index 08dc533876d..f722a88226d 100644 --- a/tests/queries/0_stateless/01763_max_distributed_depth.sql +++ b/tests/queries/0_stateless/01763_max_distributed_depth.sql @@ -17,19 +17,6 @@ ENGINE = Distributed('test_shard_localhost', '', 'tt7', rand()); DROP TABLE IF EXISTS tt7; -CREATE TABLE tt7 as tt6 ENGINE = Distributed('test_shard_localhost', '', 'tt6', rand()); - -INSERT INTO tt6 VALUES (1, 1, 1, 1, 'ok'); -- { serverError TOO_LARGE_DISTRIBUTED_DEPTH } - -SELECT * FROM tt6; -- { serverError TOO_LARGE_DISTRIBUTED_DEPTH } - -SET max_distributed_depth = 0; - --- stack overflow -INSERT INTO tt6 VALUES (1, 1, 1, 1, 'ok'); -- { serverError TOO_DEEP_RECURSION} - --- stack overflow -SELECT * FROM tt6; -- { serverError TOO_DEEP_RECURSION } +CREATE TABLE tt7 as tt6 ENGINE = Distributed('test_shard_localhost', '', 'tt6', rand()); -- {serverError INFINITE_LOOP} DROP TABLE tt6; -DROP TABLE tt7; diff --git a/tests/queries/0_stateless/01764_table_function_dictionary.sql b/tests/queries/0_stateless/01764_table_function_dictionary.sql index b642fdd741e..76e7213b367 100644 --- a/tests/queries/0_stateless/01764_table_function_dictionary.sql +++ b/tests/queries/0_stateless/01764_table_function_dictionary.sql @@ -23,5 +23,5 @@ LAYOUT(DIRECT()); SELECT * FROM dictionary('table_function_dictionary_test_dictionary'); -DROP TABLE table_function_dictionary_source_table; DROP DICTIONARY table_function_dictionary_test_dictionary; +DROP TABLE table_function_dictionary_source_table; diff --git a/tests/queries/0_stateless/01804_dictionary_decimal256_type.sql b/tests/queries/0_stateless/01804_dictionary_decimal256_type.sql index 77e9abfb742..08a8d0feb27 100644 --- a/tests/queries/0_stateless/01804_dictionary_decimal256_type.sql +++ b/tests/queries/0_stateless/01804_dictionary_decimal256_type.sql @@ -25,6 +25,8 @@ LAYOUT(FLAT()); SELECT 'Flat dictionary'; SELECT dictGet('flat_dictionary', 'decimal_value', toUInt64(1)); +DROP DICTIONARY flat_dictionary; + DROP DICTIONARY IF EXISTS hashed_dictionary; CREATE DICTIONARY hashed_dictionary ( diff --git a/tests/queries/0_stateless/01904_dictionary_default_nullable_type.sql b/tests/queries/0_stateless/01904_dictionary_default_nullable_type.sql index 4c623941a19..d28f9e5c4e6 100644 --- a/tests/queries/0_stateless/01904_dictionary_default_nullable_type.sql +++ b/tests/queries/0_stateless/01904_dictionary_default_nullable_type.sql @@ -111,6 +111,8 @@ LAYOUT(IP_TRIE()); SELECT 'IPTrie dictionary'; SELECT dictGet('ip_trie_dictionary', 'value', tuple(IPv4StringToNum('127.0.0.0'))); --{serverError UNSUPPORTED_METHOD} +DROP DICTIONARY ip_trie_dictionary; + DROP TABLE dictionary_nullable_source_table; DROP TABLE dictionary_nullable_default_source_table; diff --git a/tests/queries/0_stateless/01910_view_dictionary.sql b/tests/queries/0_stateless/01910_view_dictionary.sql index 1f9928735b4..05a67889825 100644 --- a/tests/queries/0_stateless/01910_view_dictionary.sql +++ b/tests/queries/0_stateless/01910_view_dictionary.sql @@ -45,5 +45,5 @@ FROM numbers(3); DROP TABLE dictionary_source_en; DROP TABLE dictionary_source_ru; -DROP TABLE dictionary_source_view; DROP DICTIONARY flat_dictionary; +DROP TABLE dictionary_source_view; diff --git a/tests/queries/0_stateless/02008_complex_key_range_hashed_dictionary.sql b/tests/queries/0_stateless/02008_complex_key_range_hashed_dictionary.sql index 72cac481376..ea2dad5c732 100644 --- a/tests/queries/0_stateless/02008_complex_key_range_hashed_dictionary.sql +++ b/tests/queries/0_stateless/02008_complex_key_range_hashed_dictionary.sql @@ -53,8 +53,8 @@ SELECT CountryID, StartDate, Tax FROM range_dictionary ORDER BY CountryID, Start SELECT 'onlySpecificColumn'; SELECT Tax FROM range_dictionary ORDER BY CountryID, StartDate, EndDate; -DROP TABLE date_table; DROP DICTIONARY range_dictionary; +DROP TABLE date_table; CREATE TABLE date_table ( @@ -107,5 +107,5 @@ SELECT CountryID, StartDate, Tax FROM range_dictionary_nullable ORDER BY Country SELECT 'onlySpecificColumn'; SELECT Tax FROM range_dictionary_nullable ORDER BY CountryID, StartDate, EndDate; -DROP TABLE date_table; DROP DICTIONARY range_dictionary_nullable; +DROP TABLE date_table; diff --git a/tests/queries/0_stateless/02155_dictionary_comment.sql b/tests/queries/0_stateless/02155_dictionary_comment.sql index 30b85e16a7c..8ebc7b259fc 100644 --- a/tests/queries/0_stateless/02155_dictionary_comment.sql +++ b/tests/queries/0_stateless/02155_dictionary_comment.sql @@ -49,5 +49,5 @@ SELECT name, comment FROM system.tables WHERE name == '02155_test_dictionary_vie SELECT name, comment FROM system.tables WHERE name == '02155_test_dictionary_view' AND database == currentDatabase(); DROP TABLE 02155_test_dictionary_view; -DROP TABLE 02155_test_table; DROP DICTIONARY 02155_test_dictionary; +DROP TABLE 02155_test_table; diff --git a/tests/queries/0_stateless/02183_dictionary_date_types.sql b/tests/queries/0_stateless/02183_dictionary_date_types.sql index e06863d5e53..a6db502e7e8 100644 --- a/tests/queries/0_stateless/02183_dictionary_date_types.sql +++ b/tests/queries/0_stateless/02183_dictionary_date_types.sql @@ -170,8 +170,8 @@ LIFETIME(0); SELECT 'Polygon dictionary'; SELECT * FROM 02183_polygon_dictionary; +DROP DICTIONARY 02183_polygon_dictionr; DROP TABLE 02183_polygon_dictionary_source_table; -DROP DICTIONARY 02183_polygon_dictionary; DROP TABLE IF EXISTS 02183_range_dictionary_source_table; CREATE TABLE 02183_range_dictionary_source_table diff --git a/tests/queries/0_stateless/02185_range_hashed_dictionary_open_ranges.sql b/tests/queries/0_stateless/02185_range_hashed_dictionary_open_ranges.sql index e6edee2ea18..a36c72de0ac 100644 --- a/tests/queries/0_stateless/02185_range_hashed_dictionary_open_ranges.sql +++ b/tests/queries/0_stateless/02185_range_hashed_dictionary_open_ranges.sql @@ -60,4 +60,5 @@ SELECT dictHas('02185_range_dictionary', 0, 0); SELECT dictHas('02185_range_dictionary', 0, 5001); SELECT dictHas('02185_range_dictionary', 0, 10001); +DROP DICTIONARY 02185_range_dictionary; DROP TABLE 02185_range_dictionary_source_table; diff --git a/tests/queries/0_stateless/02391_recursive_buffer.sql b/tests/queries/0_stateless/02391_recursive_buffer.sql index 1a630722b5a..60a2f0d1af1 100644 --- a/tests/queries/0_stateless/02391_recursive_buffer.sql +++ b/tests/queries/0_stateless/02391_recursive_buffer.sql @@ -10,9 +10,5 @@ DROP TABLE test; DROP TABLE IF EXISTS test1; DROP TABLE IF EXISTS test2; CREATE TABLE test1 (key UInt32) Engine = Buffer(currentDatabase(), test2, 16, 10, 100, 10000, 1000000, 10000000, 100000000); -CREATE TABLE test2 (key UInt32) Engine = Buffer(currentDatabase(), test1, 16, 10, 100, 10000, 1000000, 10000000, 100000000); -SELECT * FROM test1; -- { serverError TOO_DEEP_RECURSION } -SELECT * FROM test2; -- { serverError TOO_DEEP_RECURSION } -SELECT * FROM system.tables WHERE table IN ('test1', 'test2') AND database = currentDatabase(); -- { serverError TOO_DEEP_RECURSION } +CREATE TABLE test2 (key UInt32) Engine = Buffer(currentDatabase(), test1, 16, 10, 100, 10000, 1000000, 10000000, 100000000); -- { serverError INFINITE_LOOP } DROP TABLE test1; -DROP TABLE test2; diff --git a/tests/queries/0_stateless/03169_cache_complex_dict_short_circuit_bug.sql b/tests/queries/0_stateless/03169_cache_complex_dict_short_circuit_bug.sql index 8463d13d251..f91aaf39081 100644 --- a/tests/queries/0_stateless/03169_cache_complex_dict_short_circuit_bug.sql +++ b/tests/queries/0_stateless/03169_cache_complex_dict_short_circuit_bug.sql @@ -27,5 +27,5 @@ LAYOUT(COMPLEX_KEY_CACHE(SIZE_IN_CELLS 10)); SELECT dictGetOrDefault('cache_dictionary_complex_key_simple_attributes_short_circuit', 'value_first', (number, concat(toString(number))), toString(materialize('default'))) AS value_first FROM system.numbers LIMIT 20 FORMAT Null; SELECT dictGetOrDefault('cache_dictionary_complex_key_simple_attributes_short_circuit', 'value_first', (number, concat(toString(number))), toString(materialize('default'))) AS value_first FROM system.numbers LIMIT 20 FORMAT Null; -DROP TABLE IF EXISTS complex_key_simple_attributes_source_short_circuit_table; DROP DICTIONARY IF EXISTS cache_dictionary_complex_key_simple_attributes_short_circuit; +DROP TABLE IF EXISTS complex_key_simple_attributes_source_short_circuit_table; From a9615f2e0bdf58800fbbfa2eb4a39fe48a421325 Mon Sep 17 00:00:00 2001 From: avogar Date: Wed, 19 Jun 2024 10:31:19 +0000 Subject: [PATCH 027/115] Fix tests --- tests/queries/0_stateless/02183_dictionary_date_types.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/queries/0_stateless/02183_dictionary_date_types.sql b/tests/queries/0_stateless/02183_dictionary_date_types.sql index a6db502e7e8..7a2d2585e9d 100644 --- a/tests/queries/0_stateless/02183_dictionary_date_types.sql +++ b/tests/queries/0_stateless/02183_dictionary_date_types.sql @@ -170,7 +170,7 @@ LIFETIME(0); SELECT 'Polygon dictionary'; SELECT * FROM 02183_polygon_dictionary; -DROP DICTIONARY 02183_polygon_dictionr; +DROP DICTIONARY 02183_polygon_dictionry; DROP TABLE 02183_polygon_dictionary_source_table; DROP TABLE IF EXISTS 02183_range_dictionary_source_table; From 20abea8f64ff3d367d01b66e81857ca74e514992 Mon Sep 17 00:00:00 2001 From: Kruglov Pavel <48961922+Avogar@users.noreply.github.com> Date: Wed, 19 Jun 2024 13:31:18 +0200 Subject: [PATCH 028/115] Fix typo in test --- tests/queries/0_stateless/02183_dictionary_date_types.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/queries/0_stateless/02183_dictionary_date_types.sql b/tests/queries/0_stateless/02183_dictionary_date_types.sql index 7a2d2585e9d..5671f47cdab 100644 --- a/tests/queries/0_stateless/02183_dictionary_date_types.sql +++ b/tests/queries/0_stateless/02183_dictionary_date_types.sql @@ -170,7 +170,7 @@ LIFETIME(0); SELECT 'Polygon dictionary'; SELECT * FROM 02183_polygon_dictionary; -DROP DICTIONARY 02183_polygon_dictionry; +DROP DICTIONARY 02183_polygon_dictionary; DROP TABLE 02183_polygon_dictionary_source_table; DROP TABLE IF EXISTS 02183_range_dictionary_source_table; From b8187c7864e680de649482b06bb3938242458840 Mon Sep 17 00:00:00 2001 From: kssenii Date: Tue, 18 Jun 2024 12:52:01 +0200 Subject: [PATCH 029/115] Fix conflicts --- src/Interpreters/TemporaryDataOnDisk.cpp | 56 +++++++++--------------- src/Interpreters/TemporaryDataOnDisk.h | 10 +++-- 2 files changed, 27 insertions(+), 39 deletions(-) diff --git a/src/Interpreters/TemporaryDataOnDisk.cpp b/src/Interpreters/TemporaryDataOnDisk.cpp index 2fe82e082b2..7f0fb8cd6ca 100644 --- a/src/Interpreters/TemporaryDataOnDisk.cpp +++ b/src/Interpreters/TemporaryDataOnDisk.cpp @@ -226,53 +226,37 @@ struct TemporaryFileStream::OutputWriter bool finalized = false; }; -TemporaryFileStream::Reader::Reader(const String & path, const Block & header_, size_t size) - : in_file_buf(path, size ? std::min(DBMS_DEFAULT_BUFFER_SIZE, size) : DBMS_DEFAULT_BUFFER_SIZE) - , in_compressed_buf(in_file_buf) - , in_reader(in_compressed_buf, header_, DBMS_TCP_PROTOCOL_VERSION) +TemporaryFileStream::Reader::Reader(const String & path_, const Block & header_, size_t size_) + : path(path_) + , size(size_ ? std::min(size_, DBMS_DEFAULT_BUFFER_SIZE) : DBMS_DEFAULT_BUFFER_SIZE) + , header(header_) { - - Block read() - { - return in_reader->read(); - } - - const std::string path; - const size_t size; - const std::optional header; - - std::unique_ptr in_file_buf; - std::unique_ptr in_compressed_buf; - std::unique_ptr in_reader; -}; -======= LOG_TEST(getLogger("TemporaryFileStream"), "Reading {} from {}", header_.dumpStructure(), path); } -TemporaryFileStream::Reader::Reader(const String & path, size_t size) - : in_file_buf(path, size ? std::min(DBMS_DEFAULT_BUFFER_SIZE, size) : DBMS_DEFAULT_BUFFER_SIZE) - , in_compressed_buf(in_file_buf) - , in_reader(in_compressed_buf, DBMS_TCP_PROTOCOL_VERSION) +TemporaryFileStream::Reader::Reader(const String & path_, size_t size_) + : path(path_) + , size(size_ ? std::min(size_, DBMS_DEFAULT_BUFFER_SIZE) : DBMS_DEFAULT_BUFFER_SIZE) { LOG_TEST(getLogger("TemporaryFileStream"), "Reading from {}", path); } Block TemporaryFileStream::Reader::read() { - if (!in_reader) - { - if (fs::exists(path)) - in_file_buf = std::make_unique(path, size); - else - in_file_buf = std::make_unique(); + if (!in_reader) + { + if (fs::exists(path)) + in_file_buf = std::make_unique(path, size); + else + in_file_buf = std::make_unique(); - in_compressed_buf = std::make_unique(*in_file_buf); - if (header.has_value()) - in_reader = std::make_unique(*in_compressed_buf, header.value(), DBMS_TCP_PROTOCOL_VERSION); - else - in_reader = std::make_unique(*in_compressed_buf, DBMS_TCP_PROTOCOL_VERSION); - } - return in_reader.read(); + in_compressed_buf = std::make_unique(*in_file_buf); + if (header.has_value()) + in_reader = std::make_unique(*in_compressed_buf, header.value(), DBMS_TCP_PROTOCOL_VERSION); + else + in_reader = std::make_unique(*in_compressed_buf, DBMS_TCP_PROTOCOL_VERSION); + } + return in_reader->read(); } TemporaryFileStream::TemporaryFileStream(TemporaryFileOnDiskHolder file_, const Block & header_, TemporaryDataOnDisk * parent_) diff --git a/src/Interpreters/TemporaryDataOnDisk.h b/src/Interpreters/TemporaryDataOnDisk.h index 488eed70da9..d541c93e031 100644 --- a/src/Interpreters/TemporaryDataOnDisk.h +++ b/src/Interpreters/TemporaryDataOnDisk.h @@ -151,9 +151,13 @@ public: Block read(); - ReadBufferFromFile in_file_buf; - CompressedReadBuffer in_compressed_buf; - NativeReader in_reader; + const std::string path; + const size_t size; + const std::optional header; + + std::unique_ptr in_file_buf; + std::unique_ptr in_compressed_buf; + std::unique_ptr in_reader; }; struct Stat From 78ccd03dd6ae344a8bb63898262298ba343683c6 Mon Sep 17 00:00:00 2001 From: kssenii Date: Tue, 18 Jun 2024 19:06:06 +0200 Subject: [PATCH 030/115] A few code renames in preparation for AzureQueue --- src/CMakeLists.txt | 2 +- src/Common/SystemLogBase.h | 2 +- src/Core/SettingsEnums.cpp | 12 +- src/Core/SettingsEnums.h | 8 +- src/Interpreters/Context.cpp | 2 +- src/Interpreters/Context.h | 4 +- src/Interpreters/S3QueueLog.cpp | 8 +- src/Interpreters/S3QueueLog.h | 12 +- src/Interpreters/SystemLog.cpp | 2 +- src/Interpreters/SystemLog.h | 4 +- .../StorageObjectStorageSource.h | 2 +- .../ObjectStorageQueueIFileMetadata.cpp} | 40 ++-- .../ObjectStorageQueueIFileMetadata.h} | 8 +- .../ObjectStorageQueueMetadata.cpp} | 131 +++++------ .../ObjectStorageQueueMetadata.h} | 38 +-- .../ObjectStorageQueueMetadataFactory.cpp} | 18 +- .../ObjectStorageQueueMetadataFactory.h | 37 +++ ...ObjectStorageQueueOrderedFileMetadata.cpp} | 38 +-- .../ObjectStorageQueueOrderedFileMetadata.h} | 10 +- .../ObjectStorageQueueSettings.cpp} | 11 +- .../ObjectStorageQueueSettings.h | 47 ++++ .../ObjectStorageQueueSource.cpp} | 61 +++-- .../ObjectStorageQueueSource.h} | 42 ++-- .../ObjectStorageQueueTableMetadata.cpp} | 78 +++--- .../ObjectStorageQueueTableMetadata.h} | 20 +- ...jectStorageQueueUnorderedFileMetadata.cpp} | 12 +- ...ObjectStorageQueueUnorderedFileMetadata.h} | 6 +- .../StorageObjectStorageQueue.cpp} | 222 ++++++------------ .../StorageObjectStorageQueue.h} | 27 +-- .../ObjectStorageQueue/registerS3Queue.cpp | 87 +++++++ src/Storages/S3Queue/S3QueueMetadataFactory.h | 37 --- src/Storages/S3Queue/S3QueueSettings.h | 47 ---- src/Storages/System/StorageSystemS3Queue.cpp | 8 +- .../integration/test_storage_s3_queue/test.py | 8 +- 34 files changed, 547 insertions(+), 544 deletions(-) rename src/Storages/{S3Queue/S3QueueIFileMetadata.cpp => ObjectStorageQueue/ObjectStorageQueueIFileMetadata.cpp} (88%) rename src/Storages/{S3Queue/S3QueueIFileMetadata.h => ObjectStorageQueue/ObjectStorageQueueIFileMetadata.h} (95%) rename src/Storages/{S3Queue/S3QueueMetadata.cpp => ObjectStorageQueue/ObjectStorageQueueMetadata.cpp} (74%) rename src/Storages/{S3Queue/S3QueueMetadata.h => ObjectStorageQueue/ObjectStorageQueueMetadata.h} (62%) rename src/Storages/{S3Queue/S3QueueMetadataFactory.cpp => ObjectStorageQueue/ObjectStorageQueueMetadataFactory.cpp} (62%) create mode 100644 src/Storages/ObjectStorageQueue/ObjectStorageQueueMetadataFactory.h rename src/Storages/{S3Queue/S3QueueOrderedFileMetadata.cpp => ObjectStorageQueue/ObjectStorageQueueOrderedFileMetadata.cpp} (89%) rename src/Storages/{S3Queue/S3QueueOrderedFileMetadata.h => ObjectStorageQueue/ObjectStorageQueueOrderedFileMetadata.h} (86%) rename src/Storages/{S3Queue/S3QueueSettings.cpp => ObjectStorageQueue/ObjectStorageQueueSettings.cpp} (59%) create mode 100644 src/Storages/ObjectStorageQueue/ObjectStorageQueueSettings.h rename src/Storages/{S3Queue/S3QueueSource.cpp => ObjectStorageQueue/ObjectStorageQueueSource.cpp} (89%) rename src/Storages/{S3Queue/S3QueueSource.h => ObjectStorageQueue/ObjectStorageQueueSource.h} (69%) rename src/Storages/{S3Queue/S3QueueTableMetadata.cpp => ObjectStorageQueue/ObjectStorageQueueTableMetadata.cpp} (72%) rename src/Storages/{S3Queue/S3QueueTableMetadata.h => ObjectStorageQueue/ObjectStorageQueueTableMetadata.h} (50%) rename src/Storages/{S3Queue/S3QueueUnorderedFileMetadata.cpp => ObjectStorageQueue/ObjectStorageQueueUnorderedFileMetadata.cpp} (92%) rename src/Storages/{S3Queue/S3QueueUnorderedFileMetadata.h => ObjectStorageQueue/ObjectStorageQueueUnorderedFileMetadata.h} (75%) rename src/Storages/{S3Queue/StorageS3Queue.cpp => ObjectStorageQueue/StorageObjectStorageQueue.cpp} (59%) rename src/Storages/{S3Queue/StorageS3Queue.h => ObjectStorageQueue/StorageObjectStorageQueue.h} (75%) create mode 100644 src/Storages/ObjectStorageQueue/registerS3Queue.cpp delete mode 100644 src/Storages/S3Queue/S3QueueMetadataFactory.h delete mode 100644 src/Storages/S3Queue/S3QueueSettings.h diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 84aaec17a5b..79352bc7258 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -222,7 +222,7 @@ add_object_library(clickhouse_storages_mergetree Storages/MergeTree) add_object_library(clickhouse_storages_statistics Storages/Statistics) add_object_library(clickhouse_storages_liveview Storages/LiveView) add_object_library(clickhouse_storages_windowview Storages/WindowView) -add_object_library(clickhouse_storages_s3queue Storages/S3Queue) +add_object_library(clickhouse_storages_s3queue Storages/ObjectStorageQueue) add_object_library(clickhouse_storages_materializedview Storages/MaterializedView) add_object_library(clickhouse_client Client) add_object_library(clickhouse_bridge BridgeHelper) diff --git a/src/Common/SystemLogBase.h b/src/Common/SystemLogBase.h index 95906c63349..c6cc2f8f8c9 100644 --- a/src/Common/SystemLogBase.h +++ b/src/Common/SystemLogBase.h @@ -27,7 +27,7 @@ M(ZooKeeperLogElement) \ M(ProcessorProfileLogElement) \ M(TextLogElement) \ - M(S3QueueLogElement) \ + M(ObjectStorageQueueLogElement) \ M(FilesystemCacheLogElement) \ M(FilesystemReadPrefetchesLogElement) \ M(AsynchronousInsertLogElement) \ diff --git a/src/Core/SettingsEnums.cpp b/src/Core/SettingsEnums.cpp index 05985316566..18034d846df 100644 --- a/src/Core/SettingsEnums.cpp +++ b/src/Core/SettingsEnums.cpp @@ -201,13 +201,13 @@ IMPLEMENT_SETTING_ENUM(ORCCompression, ErrorCodes::BAD_ARGUMENTS, {"zlib", FormatSettings::ORCCompression::ZLIB}, {"lz4", FormatSettings::ORCCompression::LZ4}}) -IMPLEMENT_SETTING_ENUM(S3QueueMode, ErrorCodes::BAD_ARGUMENTS, - {{"ordered", S3QueueMode::ORDERED}, - {"unordered", S3QueueMode::UNORDERED}}) +IMPLEMENT_SETTING_ENUM(ObjectStorageQueueMode, ErrorCodes::BAD_ARGUMENTS, + {{"ordered", ObjectStorageQueueMode::ORDERED}, + {"unordered", ObjectStorageQueueMode::UNORDERED}}) -IMPLEMENT_SETTING_ENUM(S3QueueAction, ErrorCodes::BAD_ARGUMENTS, - {{"keep", S3QueueAction::KEEP}, - {"delete", S3QueueAction::DELETE}}) +IMPLEMENT_SETTING_ENUM(ObjectStorageQueueAction, ErrorCodes::BAD_ARGUMENTS, + {{"keep", ObjectStorageQueueAction::KEEP}, + {"delete", ObjectStorageQueueAction::DELETE}}) IMPLEMENT_SETTING_ENUM(ExternalCommandStderrReaction, ErrorCodes::BAD_ARGUMENTS, {{"none", ExternalCommandStderrReaction::NONE}, diff --git a/src/Core/SettingsEnums.h b/src/Core/SettingsEnums.h index 575cd8700c8..2d65bfc7463 100644 --- a/src/Core/SettingsEnums.h +++ b/src/Core/SettingsEnums.h @@ -341,21 +341,21 @@ DECLARE_SETTING_ENUM(ParallelReplicasCustomKeyFilterType) DECLARE_SETTING_ENUM(LocalFSReadMethod) -enum class S3QueueMode : uint8_t +enum class ObjectStorageQueueMode : uint8_t { ORDERED, UNORDERED, }; -DECLARE_SETTING_ENUM(S3QueueMode) +DECLARE_SETTING_ENUM(ObjectStorageQueueMode) -enum class S3QueueAction : uint8_t +enum class ObjectStorageQueueAction : uint8_t { KEEP, DELETE, }; -DECLARE_SETTING_ENUM(S3QueueAction) +DECLARE_SETTING_ENUM(ObjectStorageQueueAction) DECLARE_SETTING_ENUM(ExternalCommandStderrReaction) diff --git a/src/Interpreters/Context.cpp b/src/Interpreters/Context.cpp index 4b0ebc008ea..0d9f01813b3 100644 --- a/src/Interpreters/Context.cpp +++ b/src/Interpreters/Context.cpp @@ -4118,7 +4118,7 @@ std::shared_ptr Context::getFilesystemCacheLog() const return shared->system_logs->filesystem_cache_log; } -std::shared_ptr Context::getS3QueueLog() const +std::shared_ptr Context::getS3QueueLog() const { SharedLockGuard lock(shared->mutex); if (!shared->system_logs) diff --git a/src/Interpreters/Context.h b/src/Interpreters/Context.h index f3073ccc09c..e20a76598ae 100644 --- a/src/Interpreters/Context.h +++ b/src/Interpreters/Context.h @@ -106,7 +106,7 @@ class TransactionsInfoLog; class ProcessorsProfileLog; class FilesystemCacheLog; class FilesystemReadPrefetchesLog; -class S3QueueLog; +class ObjectStorageQueueLog; class AsynchronousInsertLog; class BackupLog; class BlobStorageLog; @@ -1106,7 +1106,7 @@ public: std::shared_ptr getTransactionsInfoLog() const; std::shared_ptr getProcessorsProfileLog() const; std::shared_ptr getFilesystemCacheLog() const; - std::shared_ptr getS3QueueLog() const; + std::shared_ptr getS3QueueLog() const; std::shared_ptr getFilesystemReadPrefetchesLog() const; std::shared_ptr getAsynchronousInsertLog() const; std::shared_ptr getBackupLog() const; diff --git a/src/Interpreters/S3QueueLog.cpp b/src/Interpreters/S3QueueLog.cpp index ba990a8ac25..09aba5d909d 100644 --- a/src/Interpreters/S3QueueLog.cpp +++ b/src/Interpreters/S3QueueLog.cpp @@ -14,13 +14,13 @@ namespace DB { -ColumnsDescription S3QueueLogElement::getColumnsDescription() +ColumnsDescription ObjectStorageQueueLogElement::getColumnsDescription() { auto status_datatype = std::make_shared( DataTypeEnum8::Values { - {"Processed", static_cast(S3QueueLogElement::S3QueueStatus::Processed)}, - {"Failed", static_cast(S3QueueLogElement::S3QueueStatus::Failed)}, + {"Processed", static_cast(ObjectStorageQueueLogElement::ObjectStorageQueueStatus::Processed)}, + {"Failed", static_cast(ObjectStorageQueueLogElement::ObjectStorageQueueStatus::Failed)}, }); return ColumnsDescription @@ -41,7 +41,7 @@ ColumnsDescription S3QueueLogElement::getColumnsDescription() }; } -void S3QueueLogElement::appendToBlock(MutableColumns & columns) const +void ObjectStorageQueueLogElement::appendToBlock(MutableColumns & columns) const { size_t i = 0; columns[i++]->insert(getFQDNOrHostName()); diff --git a/src/Interpreters/S3QueueLog.h b/src/Interpreters/S3QueueLog.h index 19e69c39247..b0e843a0cc3 100644 --- a/src/Interpreters/S3QueueLog.h +++ b/src/Interpreters/S3QueueLog.h @@ -9,7 +9,7 @@ namespace DB { -struct S3QueueLogElement +struct ObjectStorageQueueLogElement { time_t event_time{}; @@ -20,18 +20,18 @@ struct S3QueueLogElement std::string file_name; size_t rows_processed = 0; - enum class S3QueueStatus : uint8_t + enum class ObjectStorageQueueStatus : uint8_t { Processed, Failed, }; - S3QueueStatus status; + ObjectStorageQueueStatus status; ProfileEvents::Counters::Snapshot counters_snapshot; time_t processing_start_time; time_t processing_end_time; std::string exception; - static std::string name() { return "S3QueueLog"; } + static std::string name() { return "ObjectStorageQueueLog"; } static ColumnsDescription getColumnsDescription(); static NamesAndAliases getNamesAndAliases() { return {}; } @@ -39,9 +39,9 @@ struct S3QueueLogElement void appendToBlock(MutableColumns & columns) const; }; -class S3QueueLog : public SystemLog +class ObjectStorageQueueLog : public SystemLog { - using SystemLog::SystemLog; + using SystemLog::SystemLog; }; } diff --git a/src/Interpreters/SystemLog.cpp b/src/Interpreters/SystemLog.cpp index 3b25deeb59d..d8139504c8e 100644 --- a/src/Interpreters/SystemLog.cpp +++ b/src/Interpreters/SystemLog.cpp @@ -303,7 +303,7 @@ SystemLogs::SystemLogs(ContextPtr global_context, const Poco::Util::AbstractConf processors_profile_log = createSystemLog(global_context, "system", "processors_profile_log", config, "processors_profile_log", "Contains profiling information on processors level (building blocks for a pipeline for query execution."); asynchronous_insert_log = createSystemLog(global_context, "system", "asynchronous_insert_log", config, "asynchronous_insert_log", "Contains a history for all asynchronous inserts executed on current server."); backup_log = createSystemLog(global_context, "system", "backup_log", config, "backup_log", "Contains logging entries with the information about BACKUP and RESTORE operations."); - s3_queue_log = createSystemLog(global_context, "system", "s3queue_log", config, "s3queue_log", "Contains logging entries with the information files processes by S3Queue engine."); + s3_queue_log = createSystemLog(global_context, "system", "s3queue_log", config, "s3queue_log", "Contains logging entries with the information files processes by S3Queue engine."); blob_storage_log = createSystemLog(global_context, "system", "blob_storage_log", config, "blob_storage_log", "Contains logging entries with information about various blob storage operations such as uploads and deletes."); if (query_log) diff --git a/src/Interpreters/SystemLog.h b/src/Interpreters/SystemLog.h index af635ca1bdb..6e8875ef9a6 100644 --- a/src/Interpreters/SystemLog.h +++ b/src/Interpreters/SystemLog.h @@ -52,7 +52,7 @@ class FilesystemCacheLog; class FilesystemReadPrefetchesLog; class AsynchronousInsertLog; class BackupLog; -class S3QueueLog; +class ObjectStorageQueueLog; class BlobStorageLog; /// System logs should be destroyed in destructor of the last Context and before tables, @@ -74,7 +74,7 @@ struct SystemLogs std::shared_ptr metric_log; /// Used to log all metrics. std::shared_ptr filesystem_cache_log; std::shared_ptr filesystem_read_prefetches_log; - std::shared_ptr s3_queue_log; + std::shared_ptr s3_queue_log; /// Metrics from system.asynchronous_metrics. std::shared_ptr asynchronous_metric_log; /// OpenTelemetry trace spans. diff --git a/src/Storages/ObjectStorage/StorageObjectStorageSource.h b/src/Storages/ObjectStorage/StorageObjectStorageSource.h index fd7c7aa7102..944dd84865b 100644 --- a/src/Storages/ObjectStorage/StorageObjectStorageSource.h +++ b/src/Storages/ObjectStorage/StorageObjectStorageSource.h @@ -15,7 +15,7 @@ class SchemaCache; class StorageObjectStorageSource : public SourceWithKeyCondition, WithContext { - friend class StorageS3QueueSource; + friend class ObjectStorageQueueSource; public: using Configuration = StorageObjectStorage::Configuration; using ConfigurationPtr = StorageObjectStorage::ConfigurationPtr; diff --git a/src/Storages/S3Queue/S3QueueIFileMetadata.cpp b/src/Storages/ObjectStorageQueue/ObjectStorageQueueIFileMetadata.cpp similarity index 88% rename from src/Storages/S3Queue/S3QueueIFileMetadata.cpp rename to src/Storages/ObjectStorageQueue/ObjectStorageQueueIFileMetadata.cpp index 6c4089115d4..8719d9f4635 100644 --- a/src/Storages/S3Queue/S3QueueIFileMetadata.cpp +++ b/src/Storages/ObjectStorageQueue/ObjectStorageQueueIFileMetadata.cpp @@ -1,4 +1,4 @@ -#include +#include #include #include #include @@ -35,19 +35,19 @@ namespace } } -void S3QueueIFileMetadata::FileStatus::onProcessing() +void ObjectStorageQueueIFileMetadata::FileStatus::onProcessing() { state = FileStatus::State::Processing; processing_start_time = now(); } -void S3QueueIFileMetadata::FileStatus::onProcessed() +void ObjectStorageQueueIFileMetadata::FileStatus::onProcessed() { state = FileStatus::State::Processed; processing_end_time = now(); } -void S3QueueIFileMetadata::FileStatus::onFailed(const std::string & exception) +void ObjectStorageQueueIFileMetadata::FileStatus::onFailed(const std::string & exception) { state = FileStatus::State::Failed; processing_end_time = now(); @@ -55,13 +55,13 @@ void S3QueueIFileMetadata::FileStatus::onFailed(const std::string & exception) last_exception = exception; } -std::string S3QueueIFileMetadata::FileStatus::getException() const +std::string ObjectStorageQueueIFileMetadata::FileStatus::getException() const { std::lock_guard lock(last_exception_mutex); return last_exception; } -std::string S3QueueIFileMetadata::NodeMetadata::toString() const +std::string ObjectStorageQueueIFileMetadata::NodeMetadata::toString() const { Poco::JSON::Object json; json.set("file_path", file_path); @@ -76,7 +76,7 @@ std::string S3QueueIFileMetadata::NodeMetadata::toString() const return oss.str(); } -S3QueueIFileMetadata::NodeMetadata S3QueueIFileMetadata::NodeMetadata::fromString(const std::string & metadata_str) +ObjectStorageQueueIFileMetadata::NodeMetadata ObjectStorageQueueIFileMetadata::NodeMetadata::fromString(const std::string & metadata_str) { Poco::JSON::Parser parser; auto json = parser.parse(metadata_str).extract(); @@ -91,7 +91,7 @@ S3QueueIFileMetadata::NodeMetadata S3QueueIFileMetadata::NodeMetadata::fromStrin return metadata; } -S3QueueIFileMetadata::S3QueueIFileMetadata( +ObjectStorageQueueIFileMetadata::ObjectStorageQueueIFileMetadata( const std::string & path_, const std::string & processing_node_path_, const std::string & processed_node_path_, @@ -116,7 +116,7 @@ S3QueueIFileMetadata::S3QueueIFileMetadata( processed_node_path, processing_node_path, failed_node_path); } -S3QueueIFileMetadata::~S3QueueIFileMetadata() +ObjectStorageQueueIFileMetadata::~ObjectStorageQueueIFileMetadata() { if (processing_id_version.has_value()) { @@ -148,9 +148,9 @@ S3QueueIFileMetadata::~S3QueueIFileMetadata() } } -std::string S3QueueIFileMetadata::getNodeName(const std::string & path) +std::string ObjectStorageQueueIFileMetadata::getNodeName(const std::string & path) { - /// Since with are dealing with paths in s3 which can have "/", + /// Since with are dealing with paths in object storage which can have "/", /// we cannot create a zookeeper node with the name equal to path. /// Therefore we use a hash of the path as a node name. @@ -159,7 +159,7 @@ std::string S3QueueIFileMetadata::getNodeName(const std::string & path) return toString(path_hash.get64()); } -S3QueueIFileMetadata::NodeMetadata S3QueueIFileMetadata::createNodeMetadata( +ObjectStorageQueueIFileMetadata::NodeMetadata ObjectStorageQueueIFileMetadata::createNodeMetadata( const std::string & path, const std::string & exception, size_t retries) @@ -168,9 +168,9 @@ S3QueueIFileMetadata::NodeMetadata S3QueueIFileMetadata::createNodeMetadata( /// Since node name is just a hash we want to know to which file it corresponds, /// so we keep "file_path" in nodes data. - /// "last_processed_timestamp" is needed for TTL metadata nodes enabled by s3queue_tracked_file_ttl_sec. + /// "last_processed_timestamp" is needed for TTL metadata nodes enabled by tracked_file_ttl_sec. /// "last_exception" is kept for introspection, should also be visible in system.s3queue_log if it is enabled. - /// "retries" is kept for retrying the processing enabled by s3queue_loading_retries. + /// "retries" is kept for retrying the processing enabled by loading_retries. NodeMetadata metadata; metadata.file_path = path; metadata.last_processed_timestamp = now(); @@ -179,7 +179,7 @@ S3QueueIFileMetadata::NodeMetadata S3QueueIFileMetadata::createNodeMetadata( return metadata; } -std::string S3QueueIFileMetadata::getProcessorInfo(const std::string & processor_id) +std::string ObjectStorageQueueIFileMetadata::getProcessorInfo(const std::string & processor_id) { /// Add information which will be useful for debugging just in case. Poco::JSON::Object json; @@ -192,7 +192,7 @@ std::string S3QueueIFileMetadata::getProcessorInfo(const std::string & processor return oss.str(); } -bool S3QueueIFileMetadata::setProcessing() +bool ObjectStorageQueueIFileMetadata::setProcessing() { auto state = file_status->state.load(); if (state == FileStatus::State::Processing @@ -221,7 +221,7 @@ bool S3QueueIFileMetadata::setProcessing() return success; } -void S3QueueIFileMetadata::setProcessed() +void ObjectStorageQueueIFileMetadata::setProcessed() { LOG_TRACE(log, "Setting file {} as processed (path: {})", path, processed_node_path); @@ -235,7 +235,7 @@ void S3QueueIFileMetadata::setProcessed() LOG_TRACE(log, "Set file {} as processed (rows: {})", path, file_status->processed_rows); } -void S3QueueIFileMetadata::setFailed(const std::string & exception) +void ObjectStorageQueueIFileMetadata::setFailed(const std::string & exception) { LOG_TRACE(log, "Setting file {} as failed (exception: {}, path: {})", path, exception, failed_node_path); @@ -254,7 +254,7 @@ void S3QueueIFileMetadata::setFailed(const std::string & exception) LOG_TRACE(log, "Set file {} as failed (rows: {})", path, file_status->processed_rows); } -void S3QueueIFileMetadata::setFailedNonRetriable() +void ObjectStorageQueueIFileMetadata::setFailedNonRetriable() { auto zk_client = getZooKeeper(); Coordination::Requests requests; @@ -285,7 +285,7 @@ void S3QueueIFileMetadata::setFailedNonRetriable() throw Exception(ErrorCodes::LOGICAL_ERROR, "Unexpected error while setting file as failed: {}", code); } -void S3QueueIFileMetadata::setFailedRetriable() +void ObjectStorageQueueIFileMetadata::setFailedRetriable() { /// Instead of creating a persistent /failed/node_hash node /// we create a persistent /failed/node_hash.retriable node. diff --git a/src/Storages/S3Queue/S3QueueIFileMetadata.h b/src/Storages/ObjectStorageQueue/ObjectStorageQueueIFileMetadata.h similarity index 95% rename from src/Storages/S3Queue/S3QueueIFileMetadata.h rename to src/Storages/ObjectStorageQueue/ObjectStorageQueueIFileMetadata.h index e0b0d16cbcc..60fb403cfe0 100644 --- a/src/Storages/S3Queue/S3QueueIFileMetadata.h +++ b/src/Storages/ObjectStorageQueue/ObjectStorageQueueIFileMetadata.h @@ -6,7 +6,7 @@ namespace DB { -class S3QueueIFileMetadata +class ObjectStorageQueueIFileMetadata { public: struct FileStatus @@ -41,7 +41,7 @@ public: }; using FileStatusPtr = std::shared_ptr; - explicit S3QueueIFileMetadata( + explicit ObjectStorageQueueIFileMetadata( const std::string & path_, const std::string & processing_node_path_, const std::string & processed_node_path_, @@ -50,7 +50,7 @@ public: size_t max_loading_retries_, LoggerPtr log_); - virtual ~S3QueueIFileMetadata(); + virtual ~ObjectStorageQueueIFileMetadata(); bool setProcessing(); void setProcessed(); @@ -92,7 +92,7 @@ protected: LoggerPtr log; /// processing node is ephemeral, so we cannot verify with it if - /// this node was created by a certain processor on a previous s3 queue processing stage, + /// this node was created by a certain processor on a previous processing stage, /// because we could get a session expired in between the stages /// and someone else could just create this processing node. /// Therefore we also create a persistent processing node diff --git a/src/Storages/S3Queue/S3QueueMetadata.cpp b/src/Storages/ObjectStorageQueue/ObjectStorageQueueMetadata.cpp similarity index 74% rename from src/Storages/S3Queue/S3QueueMetadata.cpp rename to src/Storages/ObjectStorageQueue/ObjectStorageQueueMetadata.cpp index e828e9f0716..5eb6cfeaf4a 100644 --- a/src/Storages/S3Queue/S3QueueMetadata.cpp +++ b/src/Storages/ObjectStorageQueue/ObjectStorageQueueMetadata.cpp @@ -4,13 +4,12 @@ #include #include #include -#include -#include -#include -#include -#include -#include -#include +#include +#include +#include +#include +#include +#include #include #include #include @@ -63,7 +62,7 @@ namespace } } -class S3QueueMetadata::LocalFileStatuses +class ObjectStorageQueueMetadata::LocalFileStatuses { public: LocalFileStatuses() = default; @@ -114,90 +113,81 @@ private: } }; -S3QueueMetadata::S3QueueMetadata(const fs::path & zookeeper_path_, const S3QueueSettings & settings_) +ObjectStorageQueueMetadata::ObjectStorageQueueMetadata(const fs::path & zookeeper_path_, const ObjectStorageQueueSettings & settings_) : settings(settings_) , zookeeper_path(zookeeper_path_) , buckets_num(getBucketsNum(settings_)) - , log(getLogger("StorageS3Queue(" + zookeeper_path_.string() + ")")) + , log(getLogger("StorageObjectStorageQueue(" + zookeeper_path_.string() + ")")) , local_file_statuses(std::make_shared()) { - if (settings.mode == S3QueueMode::UNORDERED - && (settings.s3queue_tracked_files_limit || settings.s3queue_tracked_file_ttl_sec)) + if (settings.mode == ObjectStorageQueueMode::UNORDERED + && (settings.tracked_files_limit || settings.tracked_file_ttl_sec)) { task = Context::getGlobalContextInstance()->getSchedulePool().createTask( - "S3QueueCleanupFunc", + "ObjectStorageQueueCleanupFunc", [this] { cleanupThreadFunc(); }); task->activate(); task->scheduleAfter( generateRescheduleInterval( - settings.s3queue_cleanup_interval_min_ms, settings.s3queue_cleanup_interval_max_ms)); + settings.cleanup_interval_min_ms, settings.cleanup_interval_max_ms)); } } -S3QueueMetadata::~S3QueueMetadata() +ObjectStorageQueueMetadata::~ObjectStorageQueueMetadata() { shutdown(); } -void S3QueueMetadata::shutdown() +void ObjectStorageQueueMetadata::shutdown() { shutdown_called = true; if (task) task->deactivate(); } -void S3QueueMetadata::checkSettings(const S3QueueSettings & settings_) const +void ObjectStorageQueueMetadata::checkSettings(const ObjectStorageQueueSettings & settings_) const { - S3QueueTableMetadata::checkEquals(settings, settings_); + ObjectStorageQueueTableMetadata::checkEquals(settings, settings_); } -S3QueueMetadata::FileStatusPtr S3QueueMetadata::getFileStatus(const std::string & path) +ObjectStorageQueueMetadata::FileStatusPtr ObjectStorageQueueMetadata::getFileStatus(const std::string & path) { return local_file_statuses->get(path, /* create */false); } -S3QueueMetadata::FileStatuses S3QueueMetadata::getFileStatuses() const +ObjectStorageQueueMetadata::FileStatuses ObjectStorageQueueMetadata::getFileStatuses() const { return local_file_statuses->getAll(); } -S3QueueMetadata::FileMetadataPtr S3QueueMetadata::getFileMetadata( +ObjectStorageQueueMetadata::FileMetadataPtr ObjectStorageQueueMetadata::getFileMetadata( const std::string & path, - S3QueueOrderedFileMetadata::BucketInfoPtr bucket_info) + ObjectStorageQueueOrderedFileMetadata::BucketInfoPtr bucket_info) { auto file_status = local_file_statuses->get(path, /* create */true); switch (settings.mode.value) { - case S3QueueMode::ORDERED: - return std::make_shared( + case ObjectStorageQueueMode::ORDERED: + return std::make_shared( zookeeper_path, path, file_status, bucket_info, buckets_num, - settings.s3queue_loading_retries, + settings.loading_retries, log); - case S3QueueMode::UNORDERED: - return std::make_shared( + case ObjectStorageQueueMode::UNORDERED: + return std::make_shared( zookeeper_path, path, file_status, - settings.s3queue_loading_retries, + settings.loading_retries, log); } } -size_t S3QueueMetadata::getBucketsNum(const S3QueueSettings & settings) -{ - if (settings.s3queue_buckets) - return settings.s3queue_buckets; - if (settings.s3queue_processing_threads_num) - return settings.s3queue_processing_threads_num; - return 0; -} - -size_t S3QueueMetadata::getBucketsNum(const S3QueueTableMetadata & settings) +size_t ObjectStorageQueueMetadata::getBucketsNum(const ObjectStorageQueueSettings & settings) { if (settings.buckets) return settings.buckets; @@ -206,32 +196,41 @@ size_t S3QueueMetadata::getBucketsNum(const S3QueueTableMetadata & settings) return 0; } -bool S3QueueMetadata::useBucketsForProcessing() const +size_t ObjectStorageQueueMetadata::getBucketsNum(const ObjectStorageQueueTableMetadata & settings) { - return settings.mode == S3QueueMode::ORDERED && (buckets_num > 1); + if (settings.buckets) + return settings.buckets; + if (settings.processing_threads_num) + return settings.processing_threads_num; + return 0; } -S3QueueMetadata::Bucket S3QueueMetadata::getBucketForPath(const std::string & path) const +bool ObjectStorageQueueMetadata::useBucketsForProcessing() const { - return S3QueueOrderedFileMetadata::getBucketForPath(path, buckets_num); + return settings.mode == ObjectStorageQueueMode::ORDERED && (buckets_num > 1); } -S3QueueOrderedFileMetadata::BucketHolderPtr -S3QueueMetadata::tryAcquireBucket(const Bucket & bucket, const Processor & processor) +ObjectStorageQueueMetadata::Bucket ObjectStorageQueueMetadata::getBucketForPath(const std::string & path) const { - return S3QueueOrderedFileMetadata::tryAcquireBucket(zookeeper_path, bucket, processor); + return ObjectStorageQueueOrderedFileMetadata::getBucketForPath(path, buckets_num); } -void S3QueueMetadata::initialize( +ObjectStorageQueueOrderedFileMetadata::BucketHolderPtr +ObjectStorageQueueMetadata::tryAcquireBucket(const Bucket & bucket, const Processor & processor) +{ + return ObjectStorageQueueOrderedFileMetadata::tryAcquireBucket(zookeeper_path, bucket, processor); +} + +void ObjectStorageQueueMetadata::initialize( const ConfigurationPtr & configuration, const StorageInMemoryMetadata & storage_metadata) { - const auto metadata_from_table = S3QueueTableMetadata(*configuration, settings, storage_metadata); + const auto metadata_from_table = ObjectStorageQueueTableMetadata(*configuration, settings, storage_metadata); const auto & columns_from_table = storage_metadata.getColumns(); const auto table_metadata_path = zookeeper_path / "metadata"; - const auto metadata_paths = settings.mode == S3QueueMode::ORDERED - ? S3QueueOrderedFileMetadata::getMetadataPaths(buckets_num) - : S3QueueUnorderedFileMetadata::getMetadataPaths(); + const auto metadata_paths = settings.mode == ObjectStorageQueueMode::ORDERED + ? ObjectStorageQueueOrderedFileMetadata::getMetadataPaths(buckets_num) + : ObjectStorageQueueUnorderedFileMetadata::getMetadataPaths(); auto zookeeper = getZooKeeper(); zookeeper->createAncestors(zookeeper_path); @@ -240,7 +239,7 @@ void S3QueueMetadata::initialize( { if (zookeeper->exists(table_metadata_path)) { - const auto metadata_from_zk = S3QueueTableMetadata::parse(zookeeper->get(fs::path(zookeeper_path) / "metadata")); + const auto metadata_from_zk = ObjectStorageQueueTableMetadata::parse(zookeeper->get(fs::path(zookeeper_path) / "metadata")); const auto columns_from_zk = ColumnsDescription::parse(metadata_from_zk.columns); metadata_from_table.checkEquals(metadata_from_zk); @@ -265,8 +264,8 @@ void S3QueueMetadata::initialize( requests.emplace_back(zkutil::makeCreateRequest(zk_path, "", zkutil::CreateMode::Persistent)); } - if (!settings.s3queue_last_processed_path.value.empty()) - getFileMetadata(settings.s3queue_last_processed_path)->setProcessedAtStartRequests(requests, zookeeper); + if (!settings.last_processed_path.value.empty()) + getFileMetadata(settings.last_processed_path)->setProcessedAtStartRequests(requests, zookeeper); Coordination::Responses responses; auto code = zookeeper->tryMulti(requests, responses); @@ -290,10 +289,10 @@ void S3QueueMetadata::initialize( "of wrong zookeeper path or because of logical error"); } -void S3QueueMetadata::cleanupThreadFunc() +void ObjectStorageQueueMetadata::cleanupThreadFunc() { /// A background task is responsible for maintaining - /// settings.s3queue_tracked_files_limit and max_set_age settings for `unordered` processing mode. + /// settings.tracked_files_limit and max_set_age settings for `unordered` processing mode. if (shutdown_called) return; @@ -312,10 +311,10 @@ void S3QueueMetadata::cleanupThreadFunc() task->scheduleAfter( generateRescheduleInterval( - settings.s3queue_cleanup_interval_min_ms, settings.s3queue_cleanup_interval_max_ms)); + settings.cleanup_interval_min_ms, settings.cleanup_interval_max_ms)); } -void S3QueueMetadata::cleanupThreadFuncImpl() +void ObjectStorageQueueMetadata::cleanupThreadFuncImpl() { auto timer = DB::CurrentThread::getProfileEvents().timer(ProfileEvents::S3QueueCleanupMaxSetSizeOrTTLMicroseconds); const auto zk_client = getZooKeeper(); @@ -355,11 +354,11 @@ void S3QueueMetadata::cleanupThreadFuncImpl() return; } - chassert(settings.s3queue_tracked_files_limit || settings.s3queue_tracked_file_ttl_sec); - const bool check_nodes_limit = settings.s3queue_tracked_files_limit > 0; - const bool check_nodes_ttl = settings.s3queue_tracked_file_ttl_sec > 0; + chassert(settings.tracked_files_limit || settings.tracked_file_ttl_sec); + const bool check_nodes_limit = settings.tracked_files_limit > 0; + const bool check_nodes_ttl = settings.tracked_file_ttl_sec > 0; - const bool nodes_limit_exceeded = nodes_num > settings.s3queue_tracked_files_limit; + const bool nodes_limit_exceeded = nodes_num > settings.tracked_files_limit; if ((!nodes_limit_exceeded || !check_nodes_limit) && !check_nodes_ttl) { LOG_TEST(log, "No limit exceeded"); @@ -381,7 +380,7 @@ void S3QueueMetadata::cleanupThreadFuncImpl() struct Node { std::string zk_path; - S3QueueIFileMetadata::NodeMetadata metadata; + ObjectStorageQueueIFileMetadata::NodeMetadata metadata; }; auto node_cmp = [](const Node & a, const Node & b) { @@ -402,7 +401,7 @@ void S3QueueMetadata::cleanupThreadFuncImpl() std::string metadata_str; if (zk_client->tryGet(path, metadata_str)) { - sorted_nodes.emplace(path, S3QueueIFileMetadata::NodeMetadata::fromString(metadata_str)); + sorted_nodes.emplace(path, ObjectStorageQueueIFileMetadata::NodeMetadata::fromString(metadata_str)); LOG_TEST(log, "Fetched metadata for node {}", path); } else @@ -432,9 +431,9 @@ void S3QueueMetadata::cleanupThreadFuncImpl() wb << fmt::format("Node: {}, path: {}, timestamp: {};\n", node, metadata.file_path, metadata.last_processed_timestamp); return wb.str(); }; - LOG_TEST(log, "Checking node limits (max size: {}, max age: {}) for {}", settings.s3queue_tracked_files_limit, settings.s3queue_tracked_file_ttl_sec, get_nodes_str()); + LOG_TEST(log, "Checking node limits (max size: {}, max age: {}) for {}", settings.tracked_files_limit, settings.tracked_file_ttl_sec, get_nodes_str()); - size_t nodes_to_remove = check_nodes_limit && nodes_limit_exceeded ? nodes_num - settings.s3queue_tracked_files_limit : 0; + size_t nodes_to_remove = check_nodes_limit && nodes_limit_exceeded ? nodes_num - settings.tracked_files_limit : 0; for (const auto & node : sorted_nodes) { if (nodes_to_remove) @@ -453,7 +452,7 @@ void S3QueueMetadata::cleanupThreadFuncImpl() else if (check_nodes_ttl) { UInt64 node_age = getCurrentTime() - node.metadata.last_processed_timestamp; - if (node_age >= settings.s3queue_tracked_file_ttl_sec) + if (node_age >= settings.tracked_file_ttl_sec) { LOG_TRACE(log, "Removing node at path {} ({}) because file ttl is reached", node.metadata.file_path, node.zk_path); diff --git a/src/Storages/S3Queue/S3QueueMetadata.h b/src/Storages/ObjectStorageQueue/ObjectStorageQueueMetadata.h similarity index 62% rename from src/Storages/S3Queue/S3QueueMetadata.h rename to src/Storages/ObjectStorageQueue/ObjectStorageQueueMetadata.h index 25d01fb52b9..05060931b5a 100644 --- a/src/Storages/S3Queue/S3QueueMetadata.h +++ b/src/Storages/ObjectStorageQueue/ObjectStorageQueueMetadata.h @@ -7,23 +7,23 @@ #include #include #include -#include -#include -#include +#include +#include +#include namespace fs = std::filesystem; namespace Poco { class Logger; } namespace DB { -struct S3QueueSettings; -class StorageS3Queue; -struct S3QueueTableMetadata; +struct ObjectStorageQueueSettings; +class StorageObjectStorageQueue; +struct ObjectStorageQueueTableMetadata; struct StorageInMemoryMetadata; using ConfigurationPtr = StorageObjectStorage::ConfigurationPtr; /** - * A class for managing S3Queue metadata in zookeeper, e.g. + * A class for managing ObjectStorageQueue metadata in zookeeper, e.g. * the following folders: * - /processed * - /processing @@ -35,7 +35,7 @@ using ConfigurationPtr = StorageObjectStorage::ConfigurationPtr; * - /processing * - /failed * - * Depending on S3Queue processing mode (ordered or unordered) + * Depending on ObjectStorageQueue processing mode (ordered or unordered) * we can differently store metadata in /processed node. * * Implements caching of zookeeper metadata for faster responses. @@ -44,24 +44,24 @@ using ConfigurationPtr = StorageObjectStorage::ConfigurationPtr; * In case of Unordered mode - if files TTL is enabled or maximum tracked files limit is set * starts a background cleanup thread which is responsible for maintaining them. */ -class S3QueueMetadata +class ObjectStorageQueueMetadata { public: - using FileStatus = S3QueueIFileMetadata::FileStatus; - using FileMetadataPtr = std::shared_ptr; + using FileStatus = ObjectStorageQueueIFileMetadata::FileStatus; + using FileMetadataPtr = std::shared_ptr; using FileStatusPtr = std::shared_ptr; using FileStatuses = std::unordered_map; using Bucket = size_t; using Processor = std::string; - S3QueueMetadata(const fs::path & zookeeper_path_, const S3QueueSettings & settings_); - ~S3QueueMetadata(); + ObjectStorageQueueMetadata(const fs::path & zookeeper_path_, const ObjectStorageQueueSettings & settings_); + ~ObjectStorageQueueMetadata(); void initialize(const ConfigurationPtr & configuration, const StorageInMemoryMetadata & storage_metadata); - void checkSettings(const S3QueueSettings & settings) const; + void checkSettings(const ObjectStorageQueueSettings & settings) const; void shutdown(); - FileMetadataPtr getFileMetadata(const std::string & path, S3QueueOrderedFileMetadata::BucketInfoPtr bucket_info = {}); + FileMetadataPtr getFileMetadata(const std::string & path, ObjectStorageQueueOrderedFileMetadata::BucketInfoPtr bucket_info = {}); FileStatusPtr getFileStatus(const std::string & path); FileStatuses getFileStatuses() const; @@ -69,16 +69,16 @@ public: /// Method of Ordered mode parallel processing. bool useBucketsForProcessing() const; Bucket getBucketForPath(const std::string & path) const; - S3QueueOrderedFileMetadata::BucketHolderPtr tryAcquireBucket(const Bucket & bucket, const Processor & processor); + ObjectStorageQueueOrderedFileMetadata::BucketHolderPtr tryAcquireBucket(const Bucket & bucket, const Processor & processor); - static size_t getBucketsNum(const S3QueueSettings & settings); - static size_t getBucketsNum(const S3QueueTableMetadata & settings); + static size_t getBucketsNum(const ObjectStorageQueueSettings & settings); + static size_t getBucketsNum(const ObjectStorageQueueTableMetadata & settings); private: void cleanupThreadFunc(); void cleanupThreadFuncImpl(); - const S3QueueSettings settings; + const ObjectStorageQueueSettings settings; const fs::path zookeeper_path; const size_t buckets_num; diff --git a/src/Storages/S3Queue/S3QueueMetadataFactory.cpp b/src/Storages/ObjectStorageQueue/ObjectStorageQueueMetadataFactory.cpp similarity index 62% rename from src/Storages/S3Queue/S3QueueMetadataFactory.cpp rename to src/Storages/ObjectStorageQueue/ObjectStorageQueueMetadataFactory.cpp index a319b21ca3e..ffae33d6f41 100644 --- a/src/Storages/S3Queue/S3QueueMetadataFactory.cpp +++ b/src/Storages/ObjectStorageQueue/ObjectStorageQueueMetadataFactory.cpp @@ -1,4 +1,4 @@ -#include +#include #include namespace DB @@ -8,20 +8,20 @@ namespace ErrorCodes extern const int BAD_ARGUMENTS; } -S3QueueMetadataFactory & S3QueueMetadataFactory::instance() +ObjectStorageQueueMetadataFactory & ObjectStorageQueueMetadataFactory::instance() { - static S3QueueMetadataFactory ret; + static ObjectStorageQueueMetadataFactory ret; return ret; } -S3QueueMetadataFactory::FilesMetadataPtr -S3QueueMetadataFactory::getOrCreate(const std::string & zookeeper_path, const S3QueueSettings & settings) +ObjectStorageQueueMetadataFactory::FilesMetadataPtr +ObjectStorageQueueMetadataFactory::getOrCreate(const std::string & zookeeper_path, const ObjectStorageQueueSettings & settings) { std::lock_guard lock(mutex); auto it = metadata_by_path.find(zookeeper_path); if (it == metadata_by_path.end()) { - auto files_metadata = std::make_shared(zookeeper_path, settings); + auto files_metadata = std::make_shared(zookeeper_path, settings); it = metadata_by_path.emplace(zookeeper_path, std::move(files_metadata)).first; } else @@ -32,7 +32,7 @@ S3QueueMetadataFactory::getOrCreate(const std::string & zookeeper_path, const S3 return it->second.metadata; } -void S3QueueMetadataFactory::remove(const std::string & zookeeper_path) +void ObjectStorageQueueMetadataFactory::remove(const std::string & zookeeper_path) { std::lock_guard lock(mutex); auto it = metadata_by_path.find(zookeeper_path); @@ -57,9 +57,9 @@ void S3QueueMetadataFactory::remove(const std::string & zookeeper_path) } } -std::unordered_map S3QueueMetadataFactory::getAll() +std::unordered_map ObjectStorageQueueMetadataFactory::getAll() { - std::unordered_map result; + std::unordered_map result; for (const auto & [zk_path, metadata_and_ref_count] : metadata_by_path) result.emplace(zk_path, metadata_and_ref_count.metadata); return result; diff --git a/src/Storages/ObjectStorageQueue/ObjectStorageQueueMetadataFactory.h b/src/Storages/ObjectStorageQueue/ObjectStorageQueueMetadataFactory.h new file mode 100644 index 00000000000..a93f5ee3d83 --- /dev/null +++ b/src/Storages/ObjectStorageQueue/ObjectStorageQueueMetadataFactory.h @@ -0,0 +1,37 @@ +#pragma once +#include +#include +#include + +namespace DB +{ + +class ObjectStorageQueueMetadataFactory final : private boost::noncopyable +{ +public: + using FilesMetadataPtr = std::shared_ptr; + + static ObjectStorageQueueMetadataFactory & instance(); + + FilesMetadataPtr getOrCreate(const std::string & zookeeper_path, const ObjectStorageQueueSettings & settings); + + void remove(const std::string & zookeeper_path); + + std::unordered_map getAll(); + +private: + struct Metadata + { + explicit Metadata(std::shared_ptr metadata_) : metadata(metadata_), ref_count(1) {} + + std::shared_ptr metadata; + /// TODO: the ref count should be kept in keeper, because of the case with distributed processing. + size_t ref_count = 0; + }; + using MetadataByPath = std::unordered_map; + + MetadataByPath metadata_by_path; + std::mutex mutex; +}; + +} diff --git a/src/Storages/S3Queue/S3QueueOrderedFileMetadata.cpp b/src/Storages/ObjectStorageQueue/ObjectStorageQueueOrderedFileMetadata.cpp similarity index 89% rename from src/Storages/S3Queue/S3QueueOrderedFileMetadata.cpp rename to src/Storages/ObjectStorageQueue/ObjectStorageQueueOrderedFileMetadata.cpp index bac87c95cc9..f729ef00a0a 100644 --- a/src/Storages/S3Queue/S3QueueOrderedFileMetadata.cpp +++ b/src/Storages/ObjectStorageQueue/ObjectStorageQueueOrderedFileMetadata.cpp @@ -1,4 +1,4 @@ -#include +#include #include #include #include @@ -16,7 +16,7 @@ namespace ErrorCodes namespace { - S3QueueOrderedFileMetadata::Bucket getBucketForPathImpl(const std::string & path, size_t buckets_num) + ObjectStorageQueueOrderedFileMetadata::Bucket getBucketForPathImpl(const std::string & path, size_t buckets_num) { return sipHash64(path) % buckets_num; } @@ -40,7 +40,7 @@ namespace } } -S3QueueOrderedFileMetadata::BucketHolder::BucketHolder( +ObjectStorageQueueOrderedFileMetadata::BucketHolder::BucketHolder( const Bucket & bucket_, int bucket_version_, const std::string & bucket_lock_path_, @@ -55,13 +55,13 @@ S3QueueOrderedFileMetadata::BucketHolder::BucketHolder( { } -void S3QueueOrderedFileMetadata::BucketHolder::release() +void ObjectStorageQueueOrderedFileMetadata::BucketHolder::release() { if (released) return; released = true; - LOG_TEST(getLogger("S3QueueBucketHolder"), "Releasing bucket {}", bucket_info->bucket); + LOG_TEST(getLogger("ObjectStorageQueueBucketHolder"), "Releasing bucket {}", bucket_info->bucket); Coordination::Requests requests; /// Check that bucket lock version has not changed @@ -75,7 +75,7 @@ void S3QueueOrderedFileMetadata::BucketHolder::release() zkutil::KeeperMultiException::check(code, requests, responses); } -S3QueueOrderedFileMetadata::BucketHolder::~BucketHolder() +ObjectStorageQueueOrderedFileMetadata::BucketHolder::~BucketHolder() { try { @@ -87,7 +87,7 @@ S3QueueOrderedFileMetadata::BucketHolder::~BucketHolder() } } -S3QueueOrderedFileMetadata::S3QueueOrderedFileMetadata( +ObjectStorageQueueOrderedFileMetadata::ObjectStorageQueueOrderedFileMetadata( const std::filesystem::path & zk_path_, const std::string & path_, FileStatusPtr file_status_, @@ -95,7 +95,7 @@ S3QueueOrderedFileMetadata::S3QueueOrderedFileMetadata( size_t buckets_num_, size_t max_loading_retries_, LoggerPtr log_) - : S3QueueIFileMetadata( + : ObjectStorageQueueIFileMetadata( path_, /* processing_node_path */zk_path_ / "processing" / getNodeName(path_), /* processed_node_path */getProcessedPath(zk_path_, path_, buckets_num_), @@ -109,7 +109,7 @@ S3QueueOrderedFileMetadata::S3QueueOrderedFileMetadata( { } -std::vector S3QueueOrderedFileMetadata::getMetadataPaths(size_t buckets_num) +std::vector ObjectStorageQueueOrderedFileMetadata::getMetadataPaths(size_t buckets_num) { if (buckets_num > 1) { @@ -122,7 +122,7 @@ std::vector S3QueueOrderedFileMetadata::getMetadataPaths(size_t buc return {"failed", "processing"}; } -bool S3QueueOrderedFileMetadata::getMaxProcessedFile( +bool ObjectStorageQueueOrderedFileMetadata::getMaxProcessedFile( NodeMetadata & result, Coordination::Stat * stat, const zkutil::ZooKeeperPtr & zk_client) @@ -130,7 +130,7 @@ bool S3QueueOrderedFileMetadata::getMaxProcessedFile( return getMaxProcessedFile(result, stat, processed_node_path, zk_client); } -bool S3QueueOrderedFileMetadata::getMaxProcessedFile( +bool ObjectStorageQueueOrderedFileMetadata::getMaxProcessedFile( NodeMetadata & result, Coordination::Stat * stat, const std::string & processed_node_path_, @@ -146,12 +146,12 @@ bool S3QueueOrderedFileMetadata::getMaxProcessedFile( return false; } -S3QueueOrderedFileMetadata::Bucket S3QueueOrderedFileMetadata::getBucketForPath(const std::string & path_, size_t buckets_num) +ObjectStorageQueueOrderedFileMetadata::Bucket ObjectStorageQueueOrderedFileMetadata::getBucketForPath(const std::string & path_, size_t buckets_num) { return getBucketForPathImpl(path_, buckets_num); } -S3QueueOrderedFileMetadata::BucketHolderPtr S3QueueOrderedFileMetadata::tryAcquireBucket( +ObjectStorageQueueOrderedFileMetadata::BucketHolderPtr ObjectStorageQueueOrderedFileMetadata::tryAcquireBucket( const std::filesystem::path & zk_path, const Bucket & bucket, const Processor & processor) @@ -172,7 +172,7 @@ S3QueueOrderedFileMetadata::BucketHolderPtr S3QueueOrderedFileMetadata::tryAcqui bucket_lock_id_path, processor_info, zkutil::CreateMode::Persistent, /* ignore_if_exists */true)); /// Update bucket lock id path. We use its version as a version of ephemeral bucket lock node. - /// (See comment near S3QueueIFileMetadata::processing_node_version). + /// (See comment near ObjectStorageQueueIFileMetadata::processing_node_version). requests.push_back(zkutil::makeSetRequest(bucket_lock_id_path, processor_info, -1)); Coordination::Responses responses; @@ -183,7 +183,7 @@ S3QueueOrderedFileMetadata::BucketHolderPtr S3QueueOrderedFileMetadata::tryAcqui const auto bucket_lock_version = set_response->stat.version; LOG_TEST( - getLogger("S3QueueOrderedFileMetadata"), + getLogger("ObjectStorageQueueOrderedFileMetadata"), "Processor {} acquired bucket {} for processing (bucket lock version: {})", processor, bucket, bucket_lock_version); @@ -204,7 +204,7 @@ S3QueueOrderedFileMetadata::BucketHolderPtr S3QueueOrderedFileMetadata::tryAcqui throw Exception(ErrorCodes::LOGICAL_ERROR, "Unexpected error: {}", code); } -std::pair S3QueueOrderedFileMetadata::setProcessingImpl() +std::pair ObjectStorageQueueOrderedFileMetadata::setProcessingImpl() { /// In one zookeeper transaction do the following: enum RequestType @@ -300,7 +300,7 @@ std::pair S3QueueOrderedFileMetad } } -void S3QueueOrderedFileMetadata::setProcessedAtStartRequests( +void ObjectStorageQueueOrderedFileMetadata::setProcessedAtStartRequests( Coordination::Requests & requests, const zkutil::ZooKeeperPtr & zk_client) { @@ -318,7 +318,7 @@ void S3QueueOrderedFileMetadata::setProcessedAtStartRequests( } } -void S3QueueOrderedFileMetadata::setProcessedRequests( +void ObjectStorageQueueOrderedFileMetadata::setProcessedRequests( Coordination::Requests & requests, const zkutil::ZooKeeperPtr & zk_client, const std::string & processed_node_path_, @@ -359,7 +359,7 @@ void S3QueueOrderedFileMetadata::setProcessedRequests( } } -void S3QueueOrderedFileMetadata::setProcessedImpl() +void ObjectStorageQueueOrderedFileMetadata::setProcessedImpl() { /// In one zookeeper transaction do the following: enum RequestType diff --git a/src/Storages/S3Queue/S3QueueOrderedFileMetadata.h b/src/Storages/ObjectStorageQueue/ObjectStorageQueueOrderedFileMetadata.h similarity index 86% rename from src/Storages/S3Queue/S3QueueOrderedFileMetadata.h rename to src/Storages/ObjectStorageQueue/ObjectStorageQueueOrderedFileMetadata.h index 698ec0f54cc..c5e9744b57c 100644 --- a/src/Storages/S3Queue/S3QueueOrderedFileMetadata.h +++ b/src/Storages/ObjectStorageQueue/ObjectStorageQueueOrderedFileMetadata.h @@ -1,5 +1,5 @@ #pragma once -#include +#include #include #include #include @@ -7,7 +7,7 @@ namespace DB { -class S3QueueOrderedFileMetadata : public S3QueueIFileMetadata +class ObjectStorageQueueOrderedFileMetadata : public ObjectStorageQueueIFileMetadata { public: using Processor = std::string; @@ -21,7 +21,7 @@ public: }; using BucketInfoPtr = std::shared_ptr; - explicit S3QueueOrderedFileMetadata( + explicit ObjectStorageQueueOrderedFileMetadata( const std::filesystem::path & zk_path_, const std::string & path_, FileStatusPtr file_status_, @@ -38,7 +38,7 @@ public: const Bucket & bucket, const Processor & processor); - static S3QueueOrderedFileMetadata::Bucket getBucketForPath(const std::string & path, size_t buckets_num); + static ObjectStorageQueueOrderedFileMetadata::Bucket getBucketForPath(const std::string & path, size_t buckets_num); static std::vector getMetadataPaths(size_t buckets_num); @@ -72,7 +72,7 @@ private: bool ignore_if_exists); }; -struct S3QueueOrderedFileMetadata::BucketHolder +struct ObjectStorageQueueOrderedFileMetadata::BucketHolder { BucketHolder( const Bucket & bucket_, diff --git a/src/Storages/S3Queue/S3QueueSettings.cpp b/src/Storages/ObjectStorageQueue/ObjectStorageQueueSettings.cpp similarity index 59% rename from src/Storages/S3Queue/S3QueueSettings.cpp rename to src/Storages/ObjectStorageQueue/ObjectStorageQueueSettings.cpp index cb312adc5d9..3458105de8c 100644 --- a/src/Storages/S3Queue/S3QueueSettings.cpp +++ b/src/Storages/ObjectStorageQueue/ObjectStorageQueueSettings.cpp @@ -1,4 +1,4 @@ -#include +#include #include #include #include @@ -13,14 +13,19 @@ namespace ErrorCodes extern const int UNKNOWN_SETTING; } -IMPLEMENT_SETTINGS_TRAITS(S3QueueSettingsTraits, LIST_OF_S3QUEUE_SETTINGS) +IMPLEMENT_SETTINGS_TRAITS(ObjectStorageQueueSettingsTraits, LIST_OF_OBJECT_STORAGE_QUEUE_SETTINGS) -void S3QueueSettings::loadFromQuery(ASTStorage & storage_def) +void ObjectStorageQueueSettings::loadFromQuery(ASTStorage & storage_def) { if (storage_def.settings) { try { + /// We support settings starting with s3_ for compatibility. + for (auto & change : storage_def.settings->changes) + if (change.name.starts_with("s3queue_")) + change.name = change.name.substr(std::strlen("s3queue_")); + applyChanges(storage_def.settings->changes); } catch (Exception & e) diff --git a/src/Storages/ObjectStorageQueue/ObjectStorageQueueSettings.h b/src/Storages/ObjectStorageQueue/ObjectStorageQueueSettings.h new file mode 100644 index 00000000000..afe2730662c --- /dev/null +++ b/src/Storages/ObjectStorageQueue/ObjectStorageQueueSettings.h @@ -0,0 +1,47 @@ +#pragma once + +#include +#include +#include + + +namespace DB +{ +class ASTStorage; + + +#define OBJECT_STORAGE_QUEUE_RELATED_SETTINGS(M, ALIAS) \ + M(ObjectStorageQueueMode, \ + mode, \ + ObjectStorageQueueMode::ORDERED, \ + "With unordered mode, the set of all already processed files is tracked with persistent nodes in ZooKepeer." \ + "With ordered mode, only the max name of the successfully consumed file stored.", \ + 0) \ + M(ObjectStorageQueueAction, after_processing, ObjectStorageQueueAction::KEEP, "Delete or keep file in after successful processing", 0) \ + M(String, keeper_path, "", "Zookeeper node path", 0) \ + M(UInt32, loading_retries, 0, "Retry loading up to specified number of times", 0) \ + M(UInt32, processing_threads_num, 1, "Number of processing threads", 0) \ + M(UInt32, enable_logging_to_s3queue_log, 1, "Enable logging to system table system.s3queue_log", 0) \ + M(String, last_processed_path, "", "For Ordered mode. Files that have lexicographically smaller file name are considered already processed", 0) \ + M(UInt32, tracked_file_ttl_sec, 0, "Maximum number of seconds to store processed files in ZooKeeper node (store forever by default)", 0) \ + M(UInt32, polling_min_timeout_ms, 1000, "Minimal timeout before next polling", 0) \ + M(UInt32, polling_max_timeout_ms, 10000, "Maximum timeout before next polling", 0) \ + M(UInt32, polling_backoff_ms, 1000, "Polling backoff", 0) \ + M(UInt32, tracked_files_limit, 1000, "For unordered mode. Max set size for tracking processed files in ZooKeeper", 0) \ + M(UInt32, cleanup_interval_min_ms, 60000, "For unordered mode. Polling backoff min for cleanup", 0) \ + M(UInt32, cleanup_interval_max_ms, 60000, "For unordered mode. Polling backoff max for cleanup", 0) \ + M(UInt32, buckets, 0, "Number of buckets for Ordered mode parallel processing", 0) \ + +#define LIST_OF_OBJECT_STORAGE_QUEUE_SETTINGS(M, ALIAS) \ + OBJECT_STORAGE_QUEUE_RELATED_SETTINGS(M, ALIAS) \ + LIST_OF_ALL_FORMAT_SETTINGS(M, ALIAS) + +DECLARE_SETTINGS_TRAITS(ObjectStorageQueueSettingsTraits, LIST_OF_OBJECT_STORAGE_QUEUE_SETTINGS) + + +struct ObjectStorageQueueSettings : public BaseSettings +{ + void loadFromQuery(ASTStorage & storage_def); +}; + +} diff --git a/src/Storages/S3Queue/S3QueueSource.cpp b/src/Storages/ObjectStorageQueue/ObjectStorageQueueSource.cpp similarity index 89% rename from src/Storages/S3Queue/S3QueueSource.cpp rename to src/Storages/ObjectStorageQueue/ObjectStorageQueueSource.cpp index b5b1a8dd992..a56dd6ea343 100644 --- a/src/Storages/S3Queue/S3QueueSource.cpp +++ b/src/Storages/ObjectStorageQueue/ObjectStorageQueueSource.cpp @@ -5,17 +5,11 @@ #include #include #include -#include +#include #include #include -namespace CurrentMetrics -{ - extern const Metric StorageS3Threads; - extern const Metric StorageS3ThreadsActive; -} - namespace ProfileEvents { extern const Event S3QueuePullMicroseconds; @@ -26,12 +20,11 @@ namespace DB namespace ErrorCodes { - extern const int S3_ERROR; extern const int NOT_IMPLEMENTED; extern const int LOGICAL_ERROR; } -StorageS3QueueSource::S3QueueObjectInfo::S3QueueObjectInfo( +ObjectStorageQueueSource::ObjectStorageQueueObjectInfo::ObjectStorageQueueObjectInfo( const ObjectInfo & object_info, Metadata::FileMetadataPtr processing_holder_) : ObjectInfo(object_info.relative_path, object_info.metadata) @@ -39,12 +32,12 @@ StorageS3QueueSource::S3QueueObjectInfo::S3QueueObjectInfo( { } -StorageS3QueueSource::FileIterator::FileIterator( - std::shared_ptr metadata_, +ObjectStorageQueueSource::FileIterator::FileIterator( + std::shared_ptr metadata_, std::unique_ptr glob_iterator_, std::atomic & shutdown_called_, LoggerPtr logger_) - : StorageObjectStorageSource::IIterator("S3QueueIterator") + : StorageObjectStorageSource::IIterator("ObjectStorageQueueIterator") , metadata(metadata_) , glob_iterator(std::move(glob_iterator_)) , shutdown_called(shutdown_called_) @@ -52,15 +45,15 @@ StorageS3QueueSource::FileIterator::FileIterator( { } -size_t StorageS3QueueSource::FileIterator::estimatedKeysCount() +size_t ObjectStorageQueueSource::FileIterator::estimatedKeysCount() { throw Exception(ErrorCodes::NOT_IMPLEMENTED, "Method estimateKeysCount is not implemented"); } -StorageS3QueueSource::ObjectInfoPtr StorageS3QueueSource::FileIterator::nextImpl(size_t processor) +ObjectStorageQueueSource::ObjectInfoPtr ObjectStorageQueueSource::FileIterator::nextImpl(size_t processor) { ObjectInfoPtr object_info; - S3QueueOrderedFileMetadata::BucketInfoPtr bucket_info; + ObjectStorageQueueOrderedFileMetadata::BucketInfoPtr bucket_info; while (!shutdown_called) { @@ -80,13 +73,13 @@ StorageS3QueueSource::ObjectInfoPtr StorageS3QueueSource::FileIterator::nextImpl auto file_metadata = metadata->getFileMetadata(object_info->relative_path, bucket_info); if (file_metadata->setProcessing()) - return std::make_shared(*object_info, file_metadata); + return std::make_shared(*object_info, file_metadata); } return {}; } -std::pair -StorageS3QueueSource::FileIterator::getNextKeyFromAcquiredBucket(size_t processor) +std::pair +ObjectStorageQueueSource::FileIterator::getNextKeyFromAcquiredBucket(size_t processor) { /// We need this lock to maintain consistency between listing s3 directory /// and getting/putting result into listed_keys_cache. @@ -287,19 +280,19 @@ StorageS3QueueSource::FileIterator::getNextKeyFromAcquiredBucket(size_t processo } } -StorageS3QueueSource::StorageS3QueueSource( +ObjectStorageQueueSource::ObjectStorageQueueSource( String name_, size_t processor_id_, const Block & header_, std::unique_ptr internal_source_, - std::shared_ptr files_metadata_, - const S3QueueAction & action_, + std::shared_ptr files_metadata_, + const ObjectStorageQueueAction & action_, RemoveFileFunc remove_file_func_, const NamesAndTypesList & requested_virtual_columns_, ContextPtr context_, const std::atomic & shutdown_called_, const std::atomic & table_is_being_dropped_, - std::shared_ptr s3_queue_log_, + std::shared_ptr s3_queue_log_, const StorageID & storage_id_, LoggerPtr log_) : ISource(header_) @@ -319,12 +312,12 @@ StorageS3QueueSource::StorageS3QueueSource( { } -String StorageS3QueueSource::getName() const +String ObjectStorageQueueSource::getName() const { return name; } -void StorageS3QueueSource::lazyInitialize(size_t processor) +void ObjectStorageQueueSource::lazyInitialize(size_t processor) { if (initialized) return; @@ -336,7 +329,7 @@ void StorageS3QueueSource::lazyInitialize(size_t processor) initialized = true; } -Chunk StorageS3QueueSource::generate() +Chunk ObjectStorageQueueSource::generate() { lazyInitialize(processor_id); @@ -345,7 +338,7 @@ Chunk StorageS3QueueSource::generate() if (!reader) break; - const auto * object_info = dynamic_cast(&reader.getObjectInfo()); + const auto * object_info = dynamic_cast(&reader.getObjectInfo()); auto file_metadata = object_info->processing_holder; auto file_status = file_metadata->getFileStatus(); @@ -473,33 +466,33 @@ Chunk StorageS3QueueSource::generate() return {}; } -void StorageS3QueueSource::applyActionAfterProcessing(const String & path) +void ObjectStorageQueueSource::applyActionAfterProcessing(const String & path) { switch (action) { - case S3QueueAction::DELETE: + case ObjectStorageQueueAction::DELETE: { assert(remove_file_func); remove_file_func(path); break; } - case S3QueueAction::KEEP: + case ObjectStorageQueueAction::KEEP: break; } } -void StorageS3QueueSource::appendLogElement( +void ObjectStorageQueueSource::appendLogElement( const std::string & filename, - S3QueueMetadata::FileStatus & file_status_, + ObjectStorageQueueMetadata::FileStatus & file_status_, size_t processed_rows, bool processed) { if (!s3_queue_log) return; - S3QueueLogElement elem{}; + ObjectStorageQueueLogElement elem{}; { - elem = S3QueueLogElement + elem = ObjectStorageQueueLogElement { .event_time = std::chrono::system_clock::to_time_t(std::chrono::system_clock::now()), .database = storage_id.database_name, @@ -507,7 +500,7 @@ void StorageS3QueueSource::appendLogElement( .uuid = toString(storage_id.uuid), .file_name = filename, .rows_processed = processed_rows, - .status = processed ? S3QueueLogElement::S3QueueStatus::Processed : S3QueueLogElement::S3QueueStatus::Failed, + .status = processed ? ObjectStorageQueueLogElement::ObjectStorageQueueStatus::Processed : ObjectStorageQueueLogElement::ObjectStorageQueueStatus::Failed, .counters_snapshot = file_status_.profile_counters.getPartiallyAtomicSnapshot(), .processing_start_time = file_status_.processing_start_time, .processing_end_time = file_status_.processing_end_time, diff --git a/src/Storages/S3Queue/S3QueueSource.h b/src/Storages/ObjectStorageQueue/ObjectStorageQueueSource.h similarity index 69% rename from src/Storages/S3Queue/S3QueueSource.h rename to src/Storages/ObjectStorageQueue/ObjectStorageQueueSource.h index 6e098f8cb63..a6ebaf06b5f 100644 --- a/src/Storages/S3Queue/S3QueueSource.h +++ b/src/Storages/ObjectStorageQueue/ObjectStorageQueueSource.h @@ -3,7 +3,7 @@ #include #include -#include +#include #include #include #include @@ -16,7 +16,7 @@ namespace DB struct ObjectMetadata; -class StorageS3QueueSource : public ISource, WithContext +class ObjectStorageQueueSource : public ISource, WithContext { public: using Storage = StorageObjectStorage; @@ -24,16 +24,16 @@ public: using GlobIterator = StorageObjectStorageSource::GlobIterator; using ZooKeeperGetter = std::function; using RemoveFileFunc = std::function; - using FileStatusPtr = S3QueueMetadata::FileStatusPtr; + using FileStatusPtr = ObjectStorageQueueMetadata::FileStatusPtr; using ReaderHolder = StorageObjectStorageSource::ReaderHolder; - using Metadata = S3QueueMetadata; + using Metadata = ObjectStorageQueueMetadata; using ObjectInfo = StorageObjectStorageSource::ObjectInfo; using ObjectInfoPtr = std::shared_ptr; using ObjectInfos = std::vector; - struct S3QueueObjectInfo : public ObjectInfo + struct ObjectStorageQueueObjectInfo : public ObjectInfo { - S3QueueObjectInfo( + ObjectStorageQueueObjectInfo( const ObjectInfo & object_info, Metadata::FileMetadataPtr processing_holder_); @@ -44,7 +44,7 @@ public: { public: FileIterator( - std::shared_ptr metadata_, + std::shared_ptr metadata_, std::unique_ptr glob_iterator_, std::atomic & shutdown_called_, LoggerPtr logger_); @@ -57,10 +57,10 @@ public: size_t estimatedKeysCount() override; private: - using Bucket = S3QueueMetadata::Bucket; - using Processor = S3QueueMetadata::Processor; + using Bucket = ObjectStorageQueueMetadata::Bucket; + using Processor = ObjectStorageQueueMetadata::Processor; - const std::shared_ptr metadata; + const std::shared_ptr metadata; const std::unique_ptr glob_iterator; std::atomic & shutdown_called; @@ -75,24 +75,24 @@ public: }; std::unordered_map listed_keys_cache; bool iterator_finished = false; - std::unordered_map bucket_holders; + std::unordered_map bucket_holders; - std::pair getNextKeyFromAcquiredBucket(size_t processor); + std::pair getNextKeyFromAcquiredBucket(size_t processor); }; - StorageS3QueueSource( + ObjectStorageQueueSource( String name_, size_t processor_id_, const Block & header_, std::unique_ptr internal_source_, - std::shared_ptr files_metadata_, - const S3QueueAction & action_, + std::shared_ptr files_metadata_, + const ObjectStorageQueueAction & action_, RemoveFileFunc remove_file_func_, const NamesAndTypesList & requested_virtual_columns_, ContextPtr context_, const std::atomic & shutdown_called_, const std::atomic & table_is_being_dropped_, - std::shared_ptr s3_queue_log_, + std::shared_ptr s3_queue_log_, const StorageID & storage_id_, LoggerPtr log_); @@ -105,13 +105,13 @@ public: private: const String name; const size_t processor_id; - const S3QueueAction action; - const std::shared_ptr files_metadata; + const ObjectStorageQueueAction action; + const std::shared_ptr files_metadata; const std::shared_ptr internal_source; const NamesAndTypesList requested_virtual_columns; const std::atomic & shutdown_called; const std::atomic & table_is_being_dropped; - const std::shared_ptr s3_queue_log; + const std::shared_ptr s3_queue_log; const StorageID storage_id; RemoveFileFunc remove_file_func; @@ -122,10 +122,10 @@ private: std::atomic initialized{false}; size_t processed_rows_from_file = 0; - S3QueueOrderedFileMetadata::BucketHolderPtr current_bucket_holder; + ObjectStorageQueueOrderedFileMetadata::BucketHolderPtr current_bucket_holder; void applyActionAfterProcessing(const String & path); - void appendLogElement(const std::string & filename, S3QueueMetadata::FileStatus & file_status_, size_t processed_rows, bool processed); + void appendLogElement(const std::string & filename, ObjectStorageQueueMetadata::FileStatus & file_status_, size_t processed_rows, bool processed); void lazyInitialize(size_t processor); }; diff --git a/src/Storages/S3Queue/S3QueueTableMetadata.cpp b/src/Storages/ObjectStorageQueue/ObjectStorageQueueTableMetadata.cpp similarity index 72% rename from src/Storages/S3Queue/S3QueueTableMetadata.cpp rename to src/Storages/ObjectStorageQueue/ObjectStorageQueueTableMetadata.cpp index ecaa7ad57cc..cb9cdf8e186 100644 --- a/src/Storages/S3Queue/S3QueueTableMetadata.cpp +++ b/src/Storages/ObjectStorageQueue/ObjectStorageQueueTableMetadata.cpp @@ -3,9 +3,9 @@ #include #include #include -#include -#include -#include +#include +#include +#include #include @@ -20,33 +20,33 @@ namespace ErrorCodes namespace { - S3QueueMode modeFromString(const std::string & mode) + ObjectStorageQueueMode modeFromString(const std::string & mode) { if (mode == "ordered") - return S3QueueMode::ORDERED; + return ObjectStorageQueueMode::ORDERED; if (mode == "unordered") - return S3QueueMode::UNORDERED; - throw Exception(ErrorCodes::BAD_ARGUMENTS, "Unexpected S3Queue mode: {}", mode); + return ObjectStorageQueueMode::UNORDERED; + throw Exception(ErrorCodes::BAD_ARGUMENTS, "Unexpected ObjectStorageQueue mode: {}", mode); } } -S3QueueTableMetadata::S3QueueTableMetadata( +ObjectStorageQueueTableMetadata::ObjectStorageQueueTableMetadata( const StorageObjectStorage::Configuration & configuration, - const S3QueueSettings & engine_settings, + const ObjectStorageQueueSettings & engine_settings, const StorageInMemoryMetadata & storage_metadata) { format_name = configuration.format; after_processing = engine_settings.after_processing.toString(); mode = engine_settings.mode.toString(); - tracked_files_limit = engine_settings.s3queue_tracked_files_limit; - tracked_file_ttl_sec = engine_settings.s3queue_tracked_file_ttl_sec; - buckets = engine_settings.s3queue_buckets; - processing_threads_num = engine_settings.s3queue_processing_threads_num; + tracked_files_limit = engine_settings.tracked_files_limit; + tracked_file_ttl_sec = engine_settings.tracked_file_ttl_sec; + buckets = engine_settings.buckets; + processing_threads_num = engine_settings.processing_threads_num; columns = storage_metadata.getColumns().toString(); } -String S3QueueTableMetadata::toString() const +String ObjectStorageQueueTableMetadata::toString() const { Poco::JSON::Object json; json.set("after_processing", after_processing); @@ -65,7 +65,7 @@ String S3QueueTableMetadata::toString() const return oss.str(); } -void S3QueueTableMetadata::read(const String & metadata_str) +void ObjectStorageQueueTableMetadata::read(const String & metadata_str) { Poco::JSON::Parser parser; auto json = parser.parse(metadata_str).extract(); @@ -102,19 +102,19 @@ void S3QueueTableMetadata::read(const String & metadata_str) buckets = json->getValue("buckets"); } -S3QueueTableMetadata S3QueueTableMetadata::parse(const String & metadata_str) +ObjectStorageQueueTableMetadata ObjectStorageQueueTableMetadata::parse(const String & metadata_str) { - S3QueueTableMetadata metadata; + ObjectStorageQueueTableMetadata metadata; metadata.read(metadata_str); return metadata; } -void S3QueueTableMetadata::checkEquals(const S3QueueTableMetadata & from_zk) const +void ObjectStorageQueueTableMetadata::checkEquals(const ObjectStorageQueueTableMetadata & from_zk) const { checkImmutableFieldsEquals(from_zk); } -void S3QueueTableMetadata::checkImmutableFieldsEquals(const S3QueueTableMetadata & from_zk) const +void ObjectStorageQueueTableMetadata::checkImmutableFieldsEquals(const ObjectStorageQueueTableMetadata & from_zk) const { if (after_processing != from_zk.after_processing) throw Exception( @@ -164,29 +164,29 @@ void S3QueueTableMetadata::checkImmutableFieldsEquals(const S3QueueTableMetadata from_zk.last_processed_path, last_processed_path); - if (modeFromString(mode) == S3QueueMode::ORDERED) + if (modeFromString(mode) == ObjectStorageQueueMode::ORDERED) { if (buckets != from_zk.buckets) { throw Exception( ErrorCodes::METADATA_MISMATCH, - "Existing table metadata in ZooKeeper differs in s3queue_buckets setting. " + "Existing table metadata in ZooKeeper differs in buckets setting. " "Stored in ZooKeeper: {}, local: {}", from_zk.buckets, buckets); } - if (S3QueueMetadata::getBucketsNum(*this) != S3QueueMetadata::getBucketsNum(from_zk)) + if (ObjectStorageQueueMetadata::getBucketsNum(*this) != ObjectStorageQueueMetadata::getBucketsNum(from_zk)) { throw Exception( ErrorCodes::METADATA_MISMATCH, "Existing table metadata in ZooKeeper differs in processing buckets. " "Stored in ZooKeeper: {}, local: {}", - S3QueueMetadata::getBucketsNum(*this), S3QueueMetadata::getBucketsNum(from_zk)); + ObjectStorageQueueMetadata::getBucketsNum(*this), ObjectStorageQueueMetadata::getBucketsNum(from_zk)); } } } -void S3QueueTableMetadata::checkEquals(const S3QueueSettings & current, const S3QueueSettings & expected) +void ObjectStorageQueueTableMetadata::checkEquals(const ObjectStorageQueueSettings & current, const ObjectStorageQueueSettings & expected) { if (current.after_processing != expected.after_processing) throw Exception( @@ -204,48 +204,48 @@ void S3QueueTableMetadata::checkEquals(const S3QueueSettings & current, const S3 expected.mode.toString(), current.mode.toString()); - if (current.s3queue_tracked_files_limit != expected.s3queue_tracked_files_limit) + if (current.tracked_files_limit != expected.tracked_files_limit) throw Exception( ErrorCodes::METADATA_MISMATCH, "Existing table metadata in ZooKeeper differs in max set size. " "Stored in ZooKeeper: {}, local: {}", - expected.s3queue_tracked_files_limit, - current.s3queue_tracked_files_limit); + expected.tracked_files_limit, + current.tracked_files_limit); - if (current.s3queue_tracked_file_ttl_sec != expected.s3queue_tracked_file_ttl_sec) + if (current.tracked_file_ttl_sec != expected.tracked_file_ttl_sec) throw Exception( ErrorCodes::METADATA_MISMATCH, "Existing table metadata in ZooKeeper differs in max set age. " "Stored in ZooKeeper: {}, local: {}", - expected.s3queue_tracked_file_ttl_sec, - current.s3queue_tracked_file_ttl_sec); + expected.tracked_file_ttl_sec, + current.tracked_file_ttl_sec); - if (current.s3queue_last_processed_path.value != expected.s3queue_last_processed_path.value) + if (current.last_processed_path.value != expected.last_processed_path.value) throw Exception( ErrorCodes::METADATA_MISMATCH, "Existing table metadata in ZooKeeper differs in last_processed_path. " "Stored in ZooKeeper: {}, local: {}", - expected.s3queue_last_processed_path.value, - current.s3queue_last_processed_path.value); + expected.last_processed_path.value, + current.last_processed_path.value); - if (current.mode == S3QueueMode::ORDERED) + if (current.mode == ObjectStorageQueueMode::ORDERED) { - if (current.s3queue_buckets != expected.s3queue_buckets) + if (current.buckets != expected.buckets) { throw Exception( ErrorCodes::METADATA_MISMATCH, - "Existing table metadata in ZooKeeper differs in s3queue_buckets setting. " + "Existing table metadata in ZooKeeper differs in buckets setting. " "Stored in ZooKeeper: {}, local: {}", - expected.s3queue_buckets, current.s3queue_buckets); + expected.buckets, current.buckets); } - if (S3QueueMetadata::getBucketsNum(current) != S3QueueMetadata::getBucketsNum(expected)) + if (ObjectStorageQueueMetadata::getBucketsNum(current) != ObjectStorageQueueMetadata::getBucketsNum(expected)) { throw Exception( ErrorCodes::METADATA_MISMATCH, "Existing table metadata in ZooKeeper differs in processing buckets. " "Stored in ZooKeeper: {}, local: {}", - S3QueueMetadata::getBucketsNum(current), S3QueueMetadata::getBucketsNum(expected)); + ObjectStorageQueueMetadata::getBucketsNum(current), ObjectStorageQueueMetadata::getBucketsNum(expected)); } } } diff --git a/src/Storages/S3Queue/S3QueueTableMetadata.h b/src/Storages/ObjectStorageQueue/ObjectStorageQueueTableMetadata.h similarity index 50% rename from src/Storages/S3Queue/S3QueueTableMetadata.h rename to src/Storages/ObjectStorageQueue/ObjectStorageQueueTableMetadata.h index d53b60570ae..bbae06b66c6 100644 --- a/src/Storages/S3Queue/S3QueueTableMetadata.h +++ b/src/Storages/ObjectStorageQueue/ObjectStorageQueueTableMetadata.h @@ -1,6 +1,6 @@ #pragma once -#include +#include #include #include #include @@ -11,10 +11,10 @@ namespace DB class WriteBuffer; class ReadBuffer; -/** The basic parameters of S3Queue table engine for saving in ZooKeeper. +/** The basic parameters of ObjectStorageQueue table engine for saving in ZooKeeper. * Lets you verify that they match local ones. */ -struct S3QueueTableMetadata +struct ObjectStorageQueueTableMetadata { String format_name; String columns; @@ -26,22 +26,22 @@ struct S3QueueTableMetadata UInt64 processing_threads_num = 1; String last_processed_path; - S3QueueTableMetadata() = default; - S3QueueTableMetadata( + ObjectStorageQueueTableMetadata() = default; + ObjectStorageQueueTableMetadata( const StorageObjectStorage::Configuration & configuration, - const S3QueueSettings & engine_settings, + const ObjectStorageQueueSettings & engine_settings, const StorageInMemoryMetadata & storage_metadata); void read(const String & metadata_str); - static S3QueueTableMetadata parse(const String & metadata_str); + static ObjectStorageQueueTableMetadata parse(const String & metadata_str); String toString() const; - void checkEquals(const S3QueueTableMetadata & from_zk) const; - static void checkEquals(const S3QueueSettings & current, const S3QueueSettings & expected); + void checkEquals(const ObjectStorageQueueTableMetadata & from_zk) const; + static void checkEquals(const ObjectStorageQueueSettings & current, const ObjectStorageQueueSettings & expected); private: - void checkImmutableFieldsEquals(const S3QueueTableMetadata & from_zk) const; + void checkImmutableFieldsEquals(const ObjectStorageQueueTableMetadata & from_zk) const; }; diff --git a/src/Storages/S3Queue/S3QueueUnorderedFileMetadata.cpp b/src/Storages/ObjectStorageQueue/ObjectStorageQueueUnorderedFileMetadata.cpp similarity index 92% rename from src/Storages/S3Queue/S3QueueUnorderedFileMetadata.cpp rename to src/Storages/ObjectStorageQueue/ObjectStorageQueueUnorderedFileMetadata.cpp index c61e9557fc2..3a276953529 100644 --- a/src/Storages/S3Queue/S3QueueUnorderedFileMetadata.cpp +++ b/src/Storages/ObjectStorageQueue/ObjectStorageQueueUnorderedFileMetadata.cpp @@ -1,4 +1,4 @@ -#include +#include #include #include #include @@ -18,13 +18,13 @@ namespace } } -S3QueueUnorderedFileMetadata::S3QueueUnorderedFileMetadata( +ObjectStorageQueueUnorderedFileMetadata::ObjectStorageQueueUnorderedFileMetadata( const std::filesystem::path & zk_path, const std::string & path_, FileStatusPtr file_status_, size_t max_loading_retries_, LoggerPtr log_) - : S3QueueIFileMetadata( + : ObjectStorageQueueIFileMetadata( path_, /* processing_node_path */zk_path / "processing" / getNodeName(path_), /* processed_node_path */zk_path / "processed" / getNodeName(path_), @@ -35,7 +35,7 @@ S3QueueUnorderedFileMetadata::S3QueueUnorderedFileMetadata( { } -std::pair S3QueueUnorderedFileMetadata::setProcessingImpl() +std::pair ObjectStorageQueueUnorderedFileMetadata::setProcessingImpl() { /// In one zookeeper transaction do the following: enum RequestType @@ -89,7 +89,7 @@ std::pair S3QueueUnorderedFileMet throw Exception(ErrorCodes::LOGICAL_ERROR, "Unexpected state of zookeeper transaction: {}", magic_enum::enum_name(code)); } -void S3QueueUnorderedFileMetadata::setProcessedAtStartRequests( +void ObjectStorageQueueUnorderedFileMetadata::setProcessedAtStartRequests( Coordination::Requests & requests, const zkutil::ZooKeeperPtr &) { @@ -98,7 +98,7 @@ void S3QueueUnorderedFileMetadata::setProcessedAtStartRequests( processed_node_path, node_metadata.toString(), zkutil::CreateMode::Persistent)); } -void S3QueueUnorderedFileMetadata::setProcessedImpl() +void ObjectStorageQueueUnorderedFileMetadata::setProcessedImpl() { /// In one zookeeper transaction do the following: enum RequestType diff --git a/src/Storages/S3Queue/S3QueueUnorderedFileMetadata.h b/src/Storages/ObjectStorageQueue/ObjectStorageQueueUnorderedFileMetadata.h similarity index 75% rename from src/Storages/S3Queue/S3QueueUnorderedFileMetadata.h rename to src/Storages/ObjectStorageQueue/ObjectStorageQueueUnorderedFileMetadata.h index 24c2765bf3a..cc5d8a09ec9 100644 --- a/src/Storages/S3Queue/S3QueueUnorderedFileMetadata.h +++ b/src/Storages/ObjectStorageQueue/ObjectStorageQueueUnorderedFileMetadata.h @@ -1,17 +1,17 @@ #pragma once -#include +#include #include #include namespace DB { -class S3QueueUnorderedFileMetadata : public S3QueueIFileMetadata +class ObjectStorageQueueUnorderedFileMetadata : public ObjectStorageQueueIFileMetadata { public: using Bucket = size_t; - explicit S3QueueUnorderedFileMetadata( + explicit ObjectStorageQueueUnorderedFileMetadata( const std::filesystem::path & zk_path, const std::string & path_, FileStatusPtr file_status_, diff --git a/src/Storages/S3Queue/StorageS3Queue.cpp b/src/Storages/ObjectStorageQueue/StorageObjectStorageQueue.cpp similarity index 59% rename from src/Storages/S3Queue/StorageS3Queue.cpp rename to src/Storages/ObjectStorageQueue/StorageObjectStorageQueue.cpp index afb75a21b21..489980aafa8 100644 --- a/src/Storages/S3Queue/StorageS3Queue.cpp +++ b/src/Storages/ObjectStorageQueue/StorageObjectStorageQueue.cpp @@ -1,10 +1,7 @@ #include -#include "config.h" #include -#include #include -#include #include #include #include @@ -15,16 +12,15 @@ #include #include #include -#include -#include -#include -#include -#include +#include +#include +#include +#include +#include #include #include #include #include -#include #include #include @@ -32,12 +28,6 @@ namespace fs = std::filesystem; -namespace ProfileEvents -{ - extern const Event S3DeleteObjects; - extern const Event S3ListObjects; -} - namespace DB { @@ -45,23 +35,22 @@ namespace ErrorCodes { extern const int LOGICAL_ERROR; extern const int BAD_ARGUMENTS; - extern const int S3_ERROR; extern const int QUERY_NOT_ALLOWED; } namespace { - std::string chooseZooKeeperPath(const StorageID & table_id, const Settings & settings, const S3QueueSettings & s3queue_settings) + std::string chooseZooKeeperPath(const StorageID & table_id, const Settings & settings, const ObjectStorageQueueSettings & queue_settings) { std::string zk_path_prefix = settings.s3queue_default_zookeeper_path.value; if (zk_path_prefix.empty()) zk_path_prefix = "/"; std::string result_zk_path; - if (s3queue_settings.keeper_path.changed) + if (queue_settings.keeper_path.changed) { /// We do not add table uuid here on purpose. - result_zk_path = fs::path(zk_path_prefix) / s3queue_settings.keeper_path.value; + result_zk_path = fs::path(zk_path_prefix) / queue_settings.keeper_path.value; } else { @@ -71,35 +60,35 @@ namespace return zkutil::extractZooKeeperPath(result_zk_path, true); } - void checkAndAdjustSettings(S3QueueSettings & s3queue_settings, const Settings & settings, bool is_attach) + void checkAndAdjustSettings(ObjectStorageQueueSettings & queue_settings, const Settings & settings, bool is_attach) { - if (!is_attach && !s3queue_settings.mode.changed) + if (!is_attach && !queue_settings.mode.changed) { throw Exception(ErrorCodes::BAD_ARGUMENTS, "Setting `mode` (Unordered/Ordered) is not specified, but is required."); } /// In case !is_attach, we leave Ordered mode as default for compatibility. - if (!s3queue_settings.s3queue_processing_threads_num) + if (!queue_settings.processing_threads_num) { - throw Exception(ErrorCodes::BAD_ARGUMENTS, "Setting `s3queue_processing_threads_num` cannot be set to zero"); + throw Exception(ErrorCodes::BAD_ARGUMENTS, "Setting `processing_threads_num` cannot be set to zero"); } - if (!s3queue_settings.s3queue_enable_logging_to_s3queue_log.changed) + if (!queue_settings.enable_logging_to_s3queue_log.changed) { - s3queue_settings.s3queue_enable_logging_to_s3queue_log = settings.s3queue_enable_logging_to_s3queue_log; + queue_settings.enable_logging_to_s3queue_log = settings.s3queue_enable_logging_to_s3queue_log; } - if (s3queue_settings.s3queue_cleanup_interval_min_ms > s3queue_settings.s3queue_cleanup_interval_max_ms) + if (queue_settings.cleanup_interval_min_ms > queue_settings.cleanup_interval_max_ms) { throw Exception(ErrorCodes::BAD_ARGUMENTS, - "Setting `s3queue_cleanup_interval_min_ms` ({}) must be less or equal to `s3queue_cleanup_interval_max_ms` ({})", - s3queue_settings.s3queue_cleanup_interval_min_ms, s3queue_settings.s3queue_cleanup_interval_max_ms); + "Setting `cleanup_interval_min_ms` ({}) must be less or equal to `cleanup_interval_max_ms` ({})", + queue_settings.cleanup_interval_min_ms, queue_settings.cleanup_interval_max_ms); } } } -StorageS3Queue::StorageS3Queue( - std::unique_ptr s3queue_settings_, +StorageObjectStorageQueue::StorageObjectStorageQueue( + std::unique_ptr queue_settings_, const ConfigurationPtr configuration_, const StorageID & table_id_, const ColumnsDescription & columns_, @@ -111,12 +100,12 @@ StorageS3Queue::StorageS3Queue( LoadingStrictnessLevel mode) : IStorage(table_id_) , WithContext(context_) - , s3queue_settings(std::move(s3queue_settings_)) - , zk_path(chooseZooKeeperPath(table_id_, context_->getSettingsRef(), *s3queue_settings)) + , queue_settings(std::move(queue_settings_)) + , zk_path(chooseZooKeeperPath(table_id_, context_->getSettingsRef(), *queue_settings)) , configuration{configuration_} , format_settings(format_settings_) - , reschedule_processing_interval_ms(s3queue_settings->s3queue_polling_min_timeout_ms) - , log(getLogger("StorageS3Queue (" + table_id_.getFullTableName() + ")")) + , reschedule_processing_interval_ms(queue_settings->polling_min_timeout_ms) + , log(getLogger("StorageObjectStorageQueue (" + table_id_.getFullTableName() + ")")) { if (configuration->getPath().empty()) { @@ -128,10 +117,10 @@ StorageS3Queue::StorageS3Queue( } else if (!configuration->isPathWithGlobs()) { - throw Exception(ErrorCodes::QUERY_NOT_ALLOWED, "S3Queue url must either end with '/' or contain globs"); + throw Exception(ErrorCodes::QUERY_NOT_ALLOWED, "ObjectStorageQueue url must either end with '/' or contain globs"); } - checkAndAdjustSettings(*s3queue_settings, context_->getSettingsRef(), mode > LoadingStrictnessLevel::CREATE); + checkAndAdjustSettings(*queue_settings, context_->getSettingsRef(), mode > LoadingStrictnessLevel::CREATE); object_storage = configuration->createObjectStorage(context_, /* is_readonly */true); FormatFactory::instance().checkFormatName(configuration->format); @@ -149,30 +138,30 @@ StorageS3Queue::StorageS3Queue( setInMemoryMetadata(storage_metadata); LOG_INFO(log, "Using zookeeper path: {}", zk_path.string()); - task = getContext()->getSchedulePool().createTask("S3QueueStreamingTask", [this] { threadFunc(); }); + task = getContext()->getSchedulePool().createTask("ObjectStorageQueueStreamingTask", [this] { threadFunc(); }); - /// Get metadata manager from S3QueueMetadataFactory, + /// Get metadata manager from ObjectStorageQueueMetadataFactory, /// it will increase the ref count for the metadata object. - /// The ref count is decreased when StorageS3Queue::drop() method is called. - files_metadata = S3QueueMetadataFactory::instance().getOrCreate(zk_path, *s3queue_settings); + /// The ref count is decreased when StorageObjectStorageQueue::drop() method is called. + files_metadata = ObjectStorageQueueMetadataFactory::instance().getOrCreate(zk_path, *queue_settings); try { files_metadata->initialize(configuration_, storage_metadata); } catch (...) { - S3QueueMetadataFactory::instance().remove(zk_path); + ObjectStorageQueueMetadataFactory::instance().remove(zk_path); throw; } } -void StorageS3Queue::startup() +void StorageObjectStorageQueue::startup() { if (task) task->activateAndSchedule(); } -void StorageS3Queue::shutdown(bool is_drop) +void StorageObjectStorageQueue::shutdown(bool is_drop) { table_is_being_dropped = is_drop; shutdown_called = true; @@ -191,31 +180,31 @@ void StorageS3Queue::shutdown(bool is_drop) LOG_TRACE(log, "Shut down storage"); } -void StorageS3Queue::drop() +void StorageObjectStorageQueue::drop() { - S3QueueMetadataFactory::instance().remove(zk_path); + ObjectStorageQueueMetadataFactory::instance().remove(zk_path); } -bool StorageS3Queue::supportsSubsetOfColumns(const ContextPtr & context_) const +bool StorageObjectStorageQueue::supportsSubsetOfColumns(const ContextPtr & context_) const { return FormatFactory::instance().checkIfFormatSupportsSubsetOfColumns(configuration->format, context_, format_settings); } -class ReadFromS3Queue : public SourceStepWithFilter +class ReadFromObjectStorageQueue : public SourceStepWithFilter { public: - std::string getName() const override { return "ReadFromS3Queue"; } + std::string getName() const override { return "ReadFromObjectStorageQueue"; } void initializePipeline(QueryPipelineBuilder & pipeline, const BuildQueryPipelineSettings &) override; void applyFilters(ActionDAGNodes added_filter_nodes) override; - ReadFromS3Queue( + ReadFromObjectStorageQueue( const Names & column_names_, const SelectQueryInfo & query_info_, const StorageSnapshotPtr & storage_snapshot_, const ContextPtr & context_, Block sample_block, ReadFromFormatInfo info_, - std::shared_ptr storage_, + std::shared_ptr storage_, size_t max_block_size_) : SourceStepWithFilter( DataStream{.header = std::move(sample_block)}, @@ -231,15 +220,15 @@ public: private: ReadFromFormatInfo info; - std::shared_ptr storage; + std::shared_ptr storage; size_t max_block_size; - std::shared_ptr iterator; + std::shared_ptr iterator; void createIterator(const ActionsDAG::Node * predicate); }; -void ReadFromS3Queue::createIterator(const ActionsDAG::Node * predicate) +void ReadFromObjectStorageQueue::createIterator(const ActionsDAG::Node * predicate) { if (iterator) return; @@ -248,7 +237,7 @@ void ReadFromS3Queue::createIterator(const ActionsDAG::Node * predicate) } -void ReadFromS3Queue::applyFilters(ActionDAGNodes added_filter_nodes) +void ReadFromObjectStorageQueue::applyFilters(ActionDAGNodes added_filter_nodes) { SourceStepWithFilter::applyFilters(std::move(added_filter_nodes)); @@ -259,7 +248,7 @@ void ReadFromS3Queue::applyFilters(ActionDAGNodes added_filter_nodes) createIterator(predicate); } -void StorageS3Queue::read( +void StorageObjectStorageQueue::read( QueryPlan & query_plan, const Names & column_names, const StorageSnapshotPtr & storage_snapshot, @@ -281,10 +270,10 @@ void StorageS3Queue::read( "Cannot read from {} with attached materialized views", getName()); } - auto this_ptr = std::static_pointer_cast(shared_from_this()); + auto this_ptr = std::static_pointer_cast(shared_from_this()); auto read_from_format_info = prepareReadingFromFormat(column_names, storage_snapshot, supportsSubsetOfColumns(local_context)); - auto reading = std::make_unique( + auto reading = std::make_unique( column_names, query_info, storage_snapshot, @@ -297,10 +286,10 @@ void StorageS3Queue::read( query_plan.addStep(std::move(reading)); } -void ReadFromS3Queue::initializePipeline(QueryPipelineBuilder & pipeline, const BuildQueryPipelineSettings &) +void ReadFromObjectStorageQueue::initializePipeline(QueryPipelineBuilder & pipeline, const BuildQueryPipelineSettings &) { Pipes pipes; - const size_t adjusted_num_streams = storage->s3queue_settings->s3queue_processing_threads_num; + const size_t adjusted_num_streams = storage->queue_settings->processing_threads_num; createIterator(nullptr); for (size_t i = 0; i < adjusted_num_streams; ++i) @@ -320,10 +309,10 @@ void ReadFromS3Queue::initializePipeline(QueryPipelineBuilder & pipeline, const pipeline.init(std::move(pipe)); } -std::shared_ptr StorageS3Queue::createSource( +std::shared_ptr StorageObjectStorageQueue::createSource( size_t processor_id, const ReadFromFormatInfo & info, - std::shared_ptr file_iterator, + std::shared_ptr file_iterator, size_t max_block_size, ContextPtr local_context) { @@ -343,14 +332,14 @@ std::shared_ptr StorageS3Queue::createSource( { object_storage->removeObject(StoredObject(path)); }; - auto s3_queue_log = s3queue_settings->s3queue_enable_logging_to_s3queue_log ? local_context->getS3QueueLog() : nullptr; - return std::make_shared( + auto s3_queue_log = queue_settings->enable_logging_to_s3queue_log ? local_context->getS3QueueLog() : nullptr; + return std::make_shared( getName(), processor_id, info.source_header, std::move(internal_source), files_metadata, - s3queue_settings->after_processing, + queue_settings->after_processing, file_deleter, info.requested_virtual_columns, local_context, @@ -361,7 +350,7 @@ std::shared_ptr StorageS3Queue::createSource( log); } -bool StorageS3Queue::hasDependencies(const StorageID & table_id) +bool StorageObjectStorageQueue::hasDependencies(const StorageID & table_id) { // Check if all dependencies are attached auto view_ids = DatabaseCatalog::instance().getDependentViews(table_id); @@ -386,7 +375,7 @@ bool StorageS3Queue::hasDependencies(const StorageID & table_id) return true; } -void StorageS3Queue::threadFunc() +void StorageObjectStorageQueue::threadFunc() { if (shutdown_called) return; @@ -404,12 +393,12 @@ void StorageS3Queue::threadFunc() if (streamToViews()) { /// Reset the reschedule interval. - reschedule_processing_interval_ms = s3queue_settings->s3queue_polling_min_timeout_ms; + reschedule_processing_interval_ms = queue_settings->polling_min_timeout_ms; } else { /// Increase the reschedule interval. - reschedule_processing_interval_ms += s3queue_settings->s3queue_polling_backoff_ms; + reschedule_processing_interval_ms += queue_settings->polling_backoff_ms; } LOG_DEBUG(log, "Stopped streaming to {} attached views", dependencies_count); @@ -426,12 +415,12 @@ void StorageS3Queue::threadFunc() if (!shutdown_called) { - LOG_TRACE(log, "Reschedule S3 Queue processing thread in {} ms", reschedule_processing_interval_ms); + LOG_TRACE(log, "Reschedule processing thread in {} ms", reschedule_processing_interval_ms); task->scheduleAfter(reschedule_processing_interval_ms); } } -bool StorageS3Queue::streamToViews() +bool StorageObjectStorageQueue::streamToViews() { auto table_id = getStorageID(); auto table = DatabaseCatalog::instance().getTable(table_id, getContext()); @@ -444,29 +433,29 @@ bool StorageS3Queue::streamToViews() auto insert = std::make_shared(); insert->table_id = table_id; - auto s3queue_context = Context::createCopy(getContext()); - s3queue_context->makeQueryContext(); + auto context = Context::createCopy(getContext()); + context->makeQueryContext(); // Create a stream for each consumer and join them in a union stream // Only insert into dependent views and expect that input blocks contain virtual columns - InterpreterInsertQuery interpreter(insert, s3queue_context, false, true, true); + InterpreterInsertQuery interpreter(insert, context, false, true, true); auto block_io = interpreter.execute(); - auto file_iterator = createFileIterator(s3queue_context, nullptr); + auto file_iterator = createFileIterator(context, nullptr); - auto read_from_format_info = prepareReadingFromFormat(block_io.pipeline.getHeader().getNames(), storage_snapshot, supportsSubsetOfColumns(s3queue_context)); + auto read_from_format_info = prepareReadingFromFormat(block_io.pipeline.getHeader().getNames(), storage_snapshot, supportsSubsetOfColumns(context)); Pipes pipes; - pipes.reserve(s3queue_settings->s3queue_processing_threads_num); - for (size_t i = 0; i < s3queue_settings->s3queue_processing_threads_num; ++i) + pipes.reserve(queue_settings->processing_threads_num); + for (size_t i = 0; i < queue_settings->processing_threads_num; ++i) { - auto source = createSource(i, read_from_format_info, file_iterator, DBMS_DEFAULT_BUFFER_SIZE, s3queue_context); + auto source = createSource(i, read_from_format_info, file_iterator, DBMS_DEFAULT_BUFFER_SIZE, context); pipes.emplace_back(std::move(source)); } auto pipe = Pipe::unitePipes(std::move(pipes)); block_io.pipeline.complete(std::move(pipe)); - block_io.pipeline.setNumThreads(s3queue_settings->s3queue_processing_threads_num); - block_io.pipeline.setConcurrencyControl(s3queue_context->getSettingsRef().use_concurrency_control); + block_io.pipeline.setNumThreads(queue_settings->processing_threads_num); + block_io.pipeline.setConcurrencyControl(context->getSettingsRef().use_concurrency_control); std::atomic_size_t rows = 0; block_io.pipeline.setProgressCallback([&](const Progress & progress) { rows += progress.read_rows.load(); }); @@ -477,12 +466,12 @@ bool StorageS3Queue::streamToViews() return rows > 0; } -zkutil::ZooKeeperPtr StorageS3Queue::getZooKeeper() const +zkutil::ZooKeeperPtr StorageObjectStorageQueue::getZooKeeper() const { return getContext()->getZooKeeper(); } -std::shared_ptr StorageS3Queue::createFileIterator(ContextPtr local_context, const ActionsDAG::Node * predicate) +std::shared_ptr StorageObjectStorageQueue::createFileIterator(ContextPtr local_context, const ActionsDAG::Node * predicate) { auto settings = configuration->getQuerySettings(local_context); auto glob_iterator = std::make_unique( @@ -491,73 +480,4 @@ std::shared_ptr StorageS3Queue::createFileIterator return std::make_shared(files_metadata, std::move(glob_iterator), shutdown_called, log); } -#if USE_AWS_S3 -void registerStorageS3Queue(StorageFactory & factory) -{ - factory.registerStorage( - "S3Queue", - [](const StorageFactory::Arguments & args) - { - auto & engine_args = args.engine_args; - if (engine_args.empty()) - throw Exception(ErrorCodes::BAD_ARGUMENTS, "External data source must have arguments"); - - auto configuration = std::make_shared(); - StorageObjectStorage::Configuration::initialize(*configuration, args.engine_args, args.getContext(), false); - - // Use format settings from global server context + settings from - // the SETTINGS clause of the create query. Settings from current - // session and user are ignored. - std::optional format_settings; - - auto s3queue_settings = std::make_unique(); - if (args.storage_def->settings) - { - s3queue_settings->loadFromQuery(*args.storage_def); - FormatFactorySettings user_format_settings; - - // Apply changed settings from global context, but ignore the - // unknown ones, because we only have the format settings here. - const auto & changes = args.getContext()->getSettingsRef().changes(); - for (const auto & change : changes) - { - if (user_format_settings.has(change.name)) - user_format_settings.set(change.name, change.value); - else - LOG_TRACE(getLogger("StorageS3"), "Remove: {}", change.name); - args.storage_def->settings->changes.removeSetting(change.name); - } - - for (const auto & change : args.storage_def->settings->changes) - { - if (user_format_settings.has(change.name)) - user_format_settings.applyChange(change); - } - format_settings = getFormatSettings(args.getContext(), user_format_settings); - } - else - { - format_settings = getFormatSettings(args.getContext()); - } - - return std::make_shared( - std::move(s3queue_settings), - std::move(configuration), - args.table_id, - args.columns, - args.constraints, - args.comment, - args.getContext(), - format_settings, - args.storage_def, - args.mode); - }, - { - .supports_settings = true, - .supports_schema_inference = true, - .source_access_type = AccessType::S3, - }); -} -#endif - } diff --git a/src/Storages/S3Queue/StorageS3Queue.h b/src/Storages/ObjectStorageQueue/StorageObjectStorageQueue.h similarity index 75% rename from src/Storages/S3Queue/StorageS3Queue.h rename to src/Storages/ObjectStorageQueue/StorageObjectStorageQueue.h index ef83a1ccc25..fb5b4d679b3 100644 --- a/src/Storages/S3Queue/StorageS3Queue.h +++ b/src/Storages/ObjectStorageQueue/StorageObjectStorageQueue.h @@ -5,25 +5,24 @@ #include #include #include -#include -#include +#include +#include #include #include -#include #include namespace DB { -class S3QueueMetadata; +class ObjectStorageQueueMetadata; -class StorageS3Queue : public IStorage, WithContext +class StorageObjectStorageQueue : public IStorage, WithContext { public: using ConfigurationPtr = StorageObjectStorage::ConfigurationPtr; - StorageS3Queue( - std::unique_ptr s3queue_settings_, + StorageObjectStorageQueue( + std::unique_ptr queue_settings_, ConfigurationPtr configuration_, const StorageID & table_id_, const ColumnsDescription & columns_, @@ -34,7 +33,7 @@ public: ASTStorage * engine_args, LoadingStrictnessLevel mode); - String getName() const override { return "S3Queue"; } + String getName() const override { return "ObjectStorageQueue"; } void read( QueryPlan & query_plan, @@ -53,13 +52,13 @@ public: zkutil::ZooKeeperPtr getZooKeeper() const; private: - friend class ReadFromS3Queue; - using FileIterator = StorageS3QueueSource::FileIterator; + friend class ReadFromObjectStorageQueue; + using FileIterator = ObjectStorageQueueSource::FileIterator; - const std::unique_ptr s3queue_settings; + const std::unique_ptr queue_settings; const fs::path zk_path; - std::shared_ptr files_metadata; + std::shared_ptr files_metadata; ConfigurationPtr configuration; ObjectStoragePtr object_storage; @@ -83,10 +82,10 @@ private: bool supportsDynamicSubcolumns() const override { return true; } std::shared_ptr createFileIterator(ContextPtr local_context, const ActionsDAG::Node * predicate); - std::shared_ptr createSource( + std::shared_ptr createSource( size_t processor_id, const ReadFromFormatInfo & info, - std::shared_ptr file_iterator, + std::shared_ptr file_iterator, size_t max_block_size, ContextPtr local_context); diff --git a/src/Storages/ObjectStorageQueue/registerS3Queue.cpp b/src/Storages/ObjectStorageQueue/registerS3Queue.cpp new file mode 100644 index 00000000000..08b8104ae49 --- /dev/null +++ b/src/Storages/ObjectStorageQueue/registerS3Queue.cpp @@ -0,0 +1,87 @@ +#include "config.h" + +#if USE_AWS_S3 + +#include +#include +#include +#include +#include +#include + +namespace DB +{ + +namespace ErrorCodes +{ + extern const int BAD_ARGUMENTS; +} + +void registerStorageS3Queue(StorageFactory & factory) +{ + factory.registerStorage( + "S3Queue", + [](const StorageFactory::Arguments & args) + { + auto & engine_args = args.engine_args; + if (engine_args.empty()) + throw Exception(ErrorCodes::BAD_ARGUMENTS, "External data source must have arguments"); + + auto configuration = std::make_shared(); + StorageObjectStorage::Configuration::initialize(*configuration, args.engine_args, args.getContext(), false); + + // Use format settings from global server context + settings from + // the SETTINGS clause of the create query. Settings from current + // session and user are ignored. + std::optional format_settings; + + auto queue_settings = std::make_unique(); + if (args.storage_def->settings) + { + queue_settings->loadFromQuery(*args.storage_def); + FormatFactorySettings user_format_settings; + + // Apply changed settings from global context, but ignore the + // unknown ones, because we only have the format settings here. + const auto & changes = args.getContext()->getSettingsRef().changes(); + for (const auto & change : changes) + { + if (user_format_settings.has(change.name)) + user_format_settings.set(change.name, change.value); + + args.storage_def->settings->changes.removeSetting(change.name); + } + + for (const auto & change : args.storage_def->settings->changes) + { + if (user_format_settings.has(change.name)) + user_format_settings.applyChange(change); + } + format_settings = getFormatSettings(args.getContext(), user_format_settings); + } + else + { + format_settings = getFormatSettings(args.getContext()); + } + + return std::make_shared( + std::move(queue_settings), + std::move(configuration), + args.table_id, + args.columns, + args.constraints, + args.comment, + args.getContext(), + format_settings, + args.storage_def, + args.mode); + }, + { + .supports_settings = true, + .supports_schema_inference = true, + .source_access_type = AccessType::S3, + }); +} + +} +#endif diff --git a/src/Storages/S3Queue/S3QueueMetadataFactory.h b/src/Storages/S3Queue/S3QueueMetadataFactory.h deleted file mode 100644 index 80e96f8aa7e..00000000000 --- a/src/Storages/S3Queue/S3QueueMetadataFactory.h +++ /dev/null @@ -1,37 +0,0 @@ -#pragma once -#include -#include -#include - -namespace DB -{ - -class S3QueueMetadataFactory final : private boost::noncopyable -{ -public: - using FilesMetadataPtr = std::shared_ptr; - - static S3QueueMetadataFactory & instance(); - - FilesMetadataPtr getOrCreate(const std::string & zookeeper_path, const S3QueueSettings & settings); - - void remove(const std::string & zookeeper_path); - - std::unordered_map getAll(); - -private: - struct Metadata - { - explicit Metadata(std::shared_ptr metadata_) : metadata(metadata_), ref_count(1) {} - - std::shared_ptr metadata; - /// TODO: the ref count should be kept in keeper, because of the case with distributed processing. - size_t ref_count = 0; - }; - using MetadataByPath = std::unordered_map; - - MetadataByPath metadata_by_path; - std::mutex mutex; -}; - -} diff --git a/src/Storages/S3Queue/S3QueueSettings.h b/src/Storages/S3Queue/S3QueueSettings.h deleted file mode 100644 index 4a92d99c411..00000000000 --- a/src/Storages/S3Queue/S3QueueSettings.h +++ /dev/null @@ -1,47 +0,0 @@ -#pragma once - -#include -#include -#include - - -namespace DB -{ -class ASTStorage; - - -#define S3QUEUE_RELATED_SETTINGS(M, ALIAS) \ - M(S3QueueMode, \ - mode, \ - S3QueueMode::ORDERED, \ - "With unordered mode, the set of all already processed files is tracked with persistent nodes in ZooKepeer." \ - "With ordered mode, only the max name of the successfully consumed file stored.", \ - 0) \ - M(S3QueueAction, after_processing, S3QueueAction::KEEP, "Delete or keep file in S3 after successful processing", 0) \ - M(String, keeper_path, "", "Zookeeper node path", 0) \ - M(UInt32, s3queue_loading_retries, 0, "Retry loading up to specified number of times", 0) \ - M(UInt32, s3queue_processing_threads_num, 1, "Number of processing threads", 0) \ - M(UInt32, s3queue_enable_logging_to_s3queue_log, 1, "Enable logging to system table system.s3queue_log", 0) \ - M(String, s3queue_last_processed_path, "", "For Ordered mode. Files that have lexicographically smaller file name are considered already processed", 0) \ - M(UInt32, s3queue_tracked_file_ttl_sec, 0, "Maximum number of seconds to store processed files in ZooKeeper node (store forever by default)", 0) \ - M(UInt32, s3queue_polling_min_timeout_ms, 1000, "Minimal timeout before next polling", 0) \ - M(UInt32, s3queue_polling_max_timeout_ms, 10000, "Maximum timeout before next polling", 0) \ - M(UInt32, s3queue_polling_backoff_ms, 1000, "Polling backoff", 0) \ - M(UInt32, s3queue_tracked_files_limit, 1000, "For unordered mode. Max set size for tracking processed files in ZooKeeper", 0) \ - M(UInt32, s3queue_cleanup_interval_min_ms, 60000, "For unordered mode. Polling backoff min for cleanup", 0) \ - M(UInt32, s3queue_cleanup_interval_max_ms, 60000, "For unordered mode. Polling backoff max for cleanup", 0) \ - M(UInt32, s3queue_buckets, 0, "Number of buckets for Ordered mode parallel processing", 0) \ - -#define LIST_OF_S3QUEUE_SETTINGS(M, ALIAS) \ - S3QUEUE_RELATED_SETTINGS(M, ALIAS) \ - LIST_OF_ALL_FORMAT_SETTINGS(M, ALIAS) - -DECLARE_SETTINGS_TRAITS(S3QueueSettingsTraits, LIST_OF_S3QUEUE_SETTINGS) - - -struct S3QueueSettings : public BaseSettings -{ - void loadFromQuery(ASTStorage & storage_def); -}; - -} diff --git a/src/Storages/System/StorageSystemS3Queue.cpp b/src/Storages/System/StorageSystemS3Queue.cpp index 637182067f2..0920a9b19e7 100644 --- a/src/Storages/System/StorageSystemS3Queue.cpp +++ b/src/Storages/System/StorageSystemS3Queue.cpp @@ -11,9 +11,9 @@ #include #include #include -#include -#include -#include +#include +#include +#include #include @@ -43,7 +43,7 @@ StorageSystemS3Queue::StorageSystemS3Queue(const StorageID & table_id_) void StorageSystemS3Queue::fillData(MutableColumns & res_columns, ContextPtr, const ActionsDAG::Node *, std::vector) const { - for (const auto & [zookeeper_path, metadata] : S3QueueMetadataFactory::instance().getAll()) + for (const auto & [zookeeper_path, metadata] : ObjectStorageQueueMetadataFactory::instance().getAll()) { for (const auto & [file_name, file_status] : metadata->getFileStatuses()) { diff --git a/tests/integration/test_storage_s3_queue/test.py b/tests/integration/test_storage_s3_queue/test.py index 66631c51b03..3c55f0ff7f7 100644 --- a/tests/integration/test_storage_s3_queue/test.py +++ b/tests/integration/test_storage_s3_queue/test.py @@ -778,10 +778,10 @@ def test_max_set_age(started_cluster): files_path, additional_settings={ "keeper_path": keeper_path, - "s3queue_tracked_file_ttl_sec": max_age, - "s3queue_cleanup_interval_min_ms": 0, - "s3queue_cleanup_interval_max_ms": 0, - "s3queue_loading_retries": 0, + "tracked_file_ttl_sec": max_age, + "cleanup_interval_min_ms": 0, + "cleanup_interval_max_ms": 0, + "loading_retries": 0, }, ) create_mv(node, table_name, dst_table_name) From c47c8d603eaee53f63d20633b5357ae6f56109f2 Mon Sep 17 00:00:00 2001 From: kssenii Date: Wed, 19 Jun 2024 16:16:37 +0200 Subject: [PATCH 031/115] Add AzureQueue --- src/Common/ProfileEvents.cpp | 10 +- src/Common/SystemLogBase.cpp | 2 +- src/Interpreters/Context.cpp | 9 ++ src/Interpreters/Context.h | 1 + ...QueueLog.cpp => ObjectStorageQueueLog.cpp} | 2 +- .../{S3QueueLog.h => ObjectStorageQueueLog.h} | 0 src/Interpreters/SystemLog.cpp | 3 +- src/Interpreters/SystemLog.h | 1 + .../ObjectStorageQueueIFileMetadata.cpp | 10 +- .../ObjectStorageQueueMetadata.cpp | 13 +- .../ObjectStorageQueueSettings.h | 1 + .../ObjectStorageQueueSource.cpp | 20 +-- .../ObjectStorageQueueSource.h | 9 +- .../StorageObjectStorageQueue.cpp | 9 +- .../ObjectStorageQueue/registerS3Queue.cpp | 140 +++++++++++------- src/Storages/registerStorages.cpp | 5 + .../integration/test_storage_s3_queue/test.py | 75 ++++++++-- 17 files changed, 200 insertions(+), 110 deletions(-) rename src/Interpreters/{S3QueueLog.cpp => ObjectStorageQueueLog.cpp} (98%) rename src/Interpreters/{S3QueueLog.h => ObjectStorageQueueLog.h} (100%) diff --git a/src/Common/ProfileEvents.cpp b/src/Common/ProfileEvents.cpp index fef1c4a2b75..dd0379b8496 100644 --- a/src/Common/ProfileEvents.cpp +++ b/src/Common/ProfileEvents.cpp @@ -635,11 +635,11 @@ The server successfully detected this situation and will download merged part fr M(S3QueueSetFileProcessingMicroseconds, "Time spent to set file as processing")\ M(S3QueueSetFileProcessedMicroseconds, "Time spent to set file as processed")\ M(S3QueueSetFileFailedMicroseconds, "Time spent to set file as failed")\ - M(S3QueueFailedFiles, "Number of files which failed to be processed")\ - M(S3QueueProcessedFiles, "Number of files which were processed")\ - M(S3QueueCleanupMaxSetSizeOrTTLMicroseconds, "Time spent to set file as failed")\ - M(S3QueuePullMicroseconds, "Time spent to read file data")\ - M(S3QueueLockLocalFileStatusesMicroseconds, "Time spent to lock local file statuses")\ + M(ObjectStorageQueueFailedFiles, "Number of files which failed to be processed")\ + M(ObjectStorageQueueProcessedFiles, "Number of files which were processed")\ + M(ObjectStorageQueueCleanupMaxSetSizeOrTTLMicroseconds, "Time spent to set file as failed")\ + M(ObjectStorageQueuePullMicroseconds, "Time spent to read file data")\ + M(ObjectStorageQueueLockLocalFileStatusesMicroseconds, "Time spent to lock local file statuses")\ \ M(ServerStartupMilliseconds, "Time elapsed from starting server to listening to sockets in milliseconds")\ M(IOUringSQEsSubmitted, "Total number of io_uring SQEs submitted") \ diff --git a/src/Common/SystemLogBase.cpp b/src/Common/SystemLogBase.cpp index 15803db4929..f6a78850622 100644 --- a/src/Common/SystemLogBase.cpp +++ b/src/Common/SystemLogBase.cpp @@ -10,7 +10,7 @@ #include #include #include -#include +#include #include #include #include diff --git a/src/Interpreters/Context.cpp b/src/Interpreters/Context.cpp index 0d9f01813b3..99af6043aa3 100644 --- a/src/Interpreters/Context.cpp +++ b/src/Interpreters/Context.cpp @@ -4127,6 +4127,15 @@ std::shared_ptr Context::getS3QueueLog() const return shared->system_logs->s3_queue_log; } +std::shared_ptr Context::getAzureQueueLog() const +{ + SharedLockGuard lock(shared->mutex); + if (!shared->system_logs) + return {}; + + return shared->system_logs->azure_queue_log; +} + std::shared_ptr Context::getFilesystemReadPrefetchesLog() const { SharedLockGuard lock(shared->mutex); diff --git a/src/Interpreters/Context.h b/src/Interpreters/Context.h index e20a76598ae..842e4096368 100644 --- a/src/Interpreters/Context.h +++ b/src/Interpreters/Context.h @@ -1107,6 +1107,7 @@ public: std::shared_ptr getProcessorsProfileLog() const; std::shared_ptr getFilesystemCacheLog() const; std::shared_ptr getS3QueueLog() const; + std::shared_ptr getAzureQueueLog() const; std::shared_ptr getFilesystemReadPrefetchesLog() const; std::shared_ptr getAsynchronousInsertLog() const; std::shared_ptr getBackupLog() const; diff --git a/src/Interpreters/S3QueueLog.cpp b/src/Interpreters/ObjectStorageQueueLog.cpp similarity index 98% rename from src/Interpreters/S3QueueLog.cpp rename to src/Interpreters/ObjectStorageQueueLog.cpp index 09aba5d909d..24261429434 100644 --- a/src/Interpreters/S3QueueLog.cpp +++ b/src/Interpreters/ObjectStorageQueueLog.cpp @@ -8,7 +8,7 @@ #include #include #include -#include +#include namespace DB diff --git a/src/Interpreters/S3QueueLog.h b/src/Interpreters/ObjectStorageQueueLog.h similarity index 100% rename from src/Interpreters/S3QueueLog.h rename to src/Interpreters/ObjectStorageQueueLog.h diff --git a/src/Interpreters/SystemLog.cpp b/src/Interpreters/SystemLog.cpp index d8139504c8e..15bf5a35fa7 100644 --- a/src/Interpreters/SystemLog.cpp +++ b/src/Interpreters/SystemLog.cpp @@ -24,7 +24,7 @@ #include #include #include -#include +#include #include #include #include @@ -304,6 +304,7 @@ SystemLogs::SystemLogs(ContextPtr global_context, const Poco::Util::AbstractConf asynchronous_insert_log = createSystemLog(global_context, "system", "asynchronous_insert_log", config, "asynchronous_insert_log", "Contains a history for all asynchronous inserts executed on current server."); backup_log = createSystemLog(global_context, "system", "backup_log", config, "backup_log", "Contains logging entries with the information about BACKUP and RESTORE operations."); s3_queue_log = createSystemLog(global_context, "system", "s3queue_log", config, "s3queue_log", "Contains logging entries with the information files processes by S3Queue engine."); + azure_queue_log = createSystemLog(global_context, "system", "azure_queue_log", config, "azure_queue_log", "Contains logging entries with the information files processes by S3Queue engine."); blob_storage_log = createSystemLog(global_context, "system", "blob_storage_log", config, "blob_storage_log", "Contains logging entries with information about various blob storage operations such as uploads and deletes."); if (query_log) diff --git a/src/Interpreters/SystemLog.h b/src/Interpreters/SystemLog.h index 6e8875ef9a6..a60b69fc8f4 100644 --- a/src/Interpreters/SystemLog.h +++ b/src/Interpreters/SystemLog.h @@ -75,6 +75,7 @@ struct SystemLogs std::shared_ptr filesystem_cache_log; std::shared_ptr filesystem_read_prefetches_log; std::shared_ptr s3_queue_log; + std::shared_ptr azure_queue_log; /// Metrics from system.asynchronous_metrics. std::shared_ptr asynchronous_metric_log; /// OpenTelemetry trace spans. diff --git a/src/Storages/ObjectStorageQueue/ObjectStorageQueueIFileMetadata.cpp b/src/Storages/ObjectStorageQueue/ObjectStorageQueueIFileMetadata.cpp index 8719d9f4635..816d6621247 100644 --- a/src/Storages/ObjectStorageQueue/ObjectStorageQueueIFileMetadata.cpp +++ b/src/Storages/ObjectStorageQueue/ObjectStorageQueueIFileMetadata.cpp @@ -11,8 +11,8 @@ namespace ProfileEvents { - extern const Event S3QueueProcessedFiles; - extern const Event S3QueueFailedFiles; + extern const Event ObjectStorageQueueProcessedFiles; + extern const Event ObjectStorageQueueFailedFiles; }; namespace DB @@ -169,7 +169,7 @@ ObjectStorageQueueIFileMetadata::NodeMetadata ObjectStorageQueueIFileMetadata::c /// Since node name is just a hash we want to know to which file it corresponds, /// so we keep "file_path" in nodes data. /// "last_processed_timestamp" is needed for TTL metadata nodes enabled by tracked_file_ttl_sec. - /// "last_exception" is kept for introspection, should also be visible in system.s3queue_log if it is enabled. + /// "last_exception" is kept for introspection, should also be visible in system.s3(azure)queue_log if it is enabled. /// "retries" is kept for retrying the processing enabled by loading_retries. NodeMetadata metadata; metadata.file_path = path; @@ -225,7 +225,7 @@ void ObjectStorageQueueIFileMetadata::setProcessed() { LOG_TRACE(log, "Setting file {} as processed (path: {})", path, processed_node_path); - ProfileEvents::increment(ProfileEvents::S3QueueProcessedFiles); + ProfileEvents::increment(ProfileEvents::ObjectStorageQueueProcessedFiles); file_status->onProcessed(); setProcessedImpl(); @@ -239,7 +239,7 @@ void ObjectStorageQueueIFileMetadata::setFailed(const std::string & exception) { LOG_TRACE(log, "Setting file {} as failed (exception: {}, path: {})", path, exception, failed_node_path); - ProfileEvents::increment(ProfileEvents::S3QueueFailedFiles); + ProfileEvents::increment(ProfileEvents::ObjectStorageQueueFailedFiles); file_status->onFailed(exception); node_metadata.last_exception = exception; diff --git a/src/Storages/ObjectStorageQueue/ObjectStorageQueueMetadata.cpp b/src/Storages/ObjectStorageQueue/ObjectStorageQueueMetadata.cpp index 5eb6cfeaf4a..152a3f74a64 100644 --- a/src/Storages/ObjectStorageQueue/ObjectStorageQueueMetadata.cpp +++ b/src/Storages/ObjectStorageQueue/ObjectStorageQueueMetadata.cpp @@ -21,13 +21,8 @@ namespace ProfileEvents { - extern const Event S3QueueSetFileProcessingMicroseconds; - extern const Event S3QueueSetFileProcessedMicroseconds; - extern const Event S3QueueSetFileFailedMicroseconds; - extern const Event S3QueueFailedFiles; - extern const Event S3QueueProcessedFiles; - extern const Event S3QueueCleanupMaxSetSizeOrTTLMicroseconds; - extern const Event S3QueueLockLocalFileStatusesMicroseconds; + extern const Event ObjectStorageQueueCleanupMaxSetSizeOrTTLMicroseconds; + extern const Event ObjectStorageQueueLockLocalFileStatusesMicroseconds; }; namespace DB @@ -108,7 +103,7 @@ private: std::unique_lock lock() const { - auto timer = DB::CurrentThread::getProfileEvents().timer(ProfileEvents::S3QueueLockLocalFileStatusesMicroseconds); + auto timer = DB::CurrentThread::getProfileEvents().timer(ProfileEvents::ObjectStorageQueueLockLocalFileStatusesMicroseconds); return std::unique_lock(mutex); } }; @@ -316,7 +311,7 @@ void ObjectStorageQueueMetadata::cleanupThreadFunc() void ObjectStorageQueueMetadata::cleanupThreadFuncImpl() { - auto timer = DB::CurrentThread::getProfileEvents().timer(ProfileEvents::S3QueueCleanupMaxSetSizeOrTTLMicroseconds); + auto timer = DB::CurrentThread::getProfileEvents().timer(ProfileEvents::ObjectStorageQueueCleanupMaxSetSizeOrTTLMicroseconds); const auto zk_client = getZooKeeper(); const fs::path zookeeper_processed_path = zookeeper_path / "processed"; const fs::path zookeeper_failed_path = zookeeper_path / "failed"; diff --git a/src/Storages/ObjectStorageQueue/ObjectStorageQueueSettings.h b/src/Storages/ObjectStorageQueue/ObjectStorageQueueSettings.h index afe2730662c..0db5bb92bde 100644 --- a/src/Storages/ObjectStorageQueue/ObjectStorageQueueSettings.h +++ b/src/Storages/ObjectStorageQueue/ObjectStorageQueueSettings.h @@ -22,6 +22,7 @@ class ASTStorage; M(UInt32, loading_retries, 0, "Retry loading up to specified number of times", 0) \ M(UInt32, processing_threads_num, 1, "Number of processing threads", 0) \ M(UInt32, enable_logging_to_s3queue_log, 1, "Enable logging to system table system.s3queue_log", 0) \ + M(UInt32, enable_logging_to_azure_queue_log, 1, "Enable logging to system table system.s3queue_log", 0) \ M(String, last_processed_path, "", "For Ordered mode. Files that have lexicographically smaller file name are considered already processed", 0) \ M(UInt32, tracked_file_ttl_sec, 0, "Maximum number of seconds to store processed files in ZooKeeper node (store forever by default)", 0) \ M(UInt32, polling_min_timeout_ms, 1000, "Minimal timeout before next polling", 0) \ diff --git a/src/Storages/ObjectStorageQueue/ObjectStorageQueueSource.cpp b/src/Storages/ObjectStorageQueue/ObjectStorageQueueSource.cpp index a56dd6ea343..05f925e2e2a 100644 --- a/src/Storages/ObjectStorageQueue/ObjectStorageQueueSource.cpp +++ b/src/Storages/ObjectStorageQueue/ObjectStorageQueueSource.cpp @@ -12,7 +12,7 @@ namespace ProfileEvents { - extern const Event S3QueuePullMicroseconds; + extern const Event ObjectStorageQueuePullMicroseconds; } namespace DB @@ -81,7 +81,7 @@ ObjectStorageQueueSource::ObjectInfoPtr ObjectStorageQueueSource::FileIterator:: std::pair ObjectStorageQueueSource::FileIterator::getNextKeyFromAcquiredBucket(size_t processor) { - /// We need this lock to maintain consistency between listing s3 directory + /// We need this lock to maintain consistency between listing object storage directory /// and getting/putting result into listed_keys_cache. std::lock_guard lock(buckets_mutex); @@ -98,7 +98,7 @@ ObjectStorageQueueSource::FileIterator::getNextKeyFromAcquiredBucket(size_t proc /// and checks if corresponding bucket is already acquired by someone. /// In case it is already acquired, they put the key into listed_keys_cache, /// so that the thread who acquired the bucket will be able to see - /// those keys without the need to list s3 directory once again. + /// those keys without the need to list object storage directory once again. if (bucket_holder_it->second) { const auto bucket = bucket_holder_it->second->getBucket(); @@ -155,7 +155,7 @@ ObjectStorageQueueSource::FileIterator::getNextKeyFromAcquiredBucket(size_t proc } } /// If processing thread has already acquired some bucket - /// and while listing s3 directory gets a key which is in a different bucket, + /// and while listing object storage directory gets a key which is in a different bucket, /// it puts the key into listed_keys_cache to allow others to process it, /// because one processing thread can acquire only one bucket at a time. /// Once a thread is finished with its acquired bucket, it checks listed_keys_cache @@ -292,7 +292,7 @@ ObjectStorageQueueSource::ObjectStorageQueueSource( ContextPtr context_, const std::atomic & shutdown_called_, const std::atomic & table_is_being_dropped_, - std::shared_ptr s3_queue_log_, + std::shared_ptr system_queue_log_, const StorageID & storage_id_, LoggerPtr log_) : ISource(header_) @@ -305,7 +305,7 @@ ObjectStorageQueueSource::ObjectStorageQueueSource( , requested_virtual_columns(requested_virtual_columns_) , shutdown_called(shutdown_called_) , table_is_being_dropped(table_is_being_dropped_) - , s3_queue_log(s3_queue_log_) + , system_queue_log(system_queue_log_) , storage_id(storage_id_) , remove_file_func(remove_file_func_) , log(log_) @@ -400,11 +400,11 @@ Chunk ObjectStorageQueueSource::generate() auto * prev_scope = CurrentThread::get().attachProfileCountersScope(&file_status->profile_counters); SCOPE_EXIT({ CurrentThread::get().attachProfileCountersScope(prev_scope); }); - /// FIXME: if files are compressed, profile counters update does not work fully (s3 related counters are not saved). Why? + /// FIXME: if files are compressed, profile counters update does not work fully (object storage related counters are not saved). Why? try { - auto timer = DB::CurrentThread::getProfileEvents().timer(ProfileEvents::S3QueuePullMicroseconds); + auto timer = DB::CurrentThread::getProfileEvents().timer(ProfileEvents::ObjectStorageQueuePullMicroseconds); Chunk chunk; if (reader->pull(chunk)) @@ -487,7 +487,7 @@ void ObjectStorageQueueSource::appendLogElement( size_t processed_rows, bool processed) { - if (!s3_queue_log) + if (!system_queue_log) return; ObjectStorageQueueLogElement elem{}; @@ -507,7 +507,7 @@ void ObjectStorageQueueSource::appendLogElement( .exception = file_status_.getException(), }; } - s3_queue_log->add(std::move(elem)); + system_queue_log->add(std::move(elem)); } } diff --git a/src/Storages/ObjectStorageQueue/ObjectStorageQueueSource.h b/src/Storages/ObjectStorageQueue/ObjectStorageQueueSource.h index a6ebaf06b5f..15aba3df283 100644 --- a/src/Storages/ObjectStorageQueue/ObjectStorageQueueSource.h +++ b/src/Storages/ObjectStorageQueue/ObjectStorageQueueSource.h @@ -6,7 +6,7 @@ #include #include #include -#include +#include namespace Poco { class Logger; } @@ -49,9 +49,6 @@ public: std::atomic & shutdown_called_, LoggerPtr logger_); - /// Note: - /// List results in s3 are always returned in UTF-8 binary order. - /// (https://docs.aws.amazon.com/AmazonS3/latest/userguide/ListingKeysUsingAPIs.html) ObjectInfoPtr nextImpl(size_t processor) override; size_t estimatedKeysCount() override; @@ -92,7 +89,7 @@ public: ContextPtr context_, const std::atomic & shutdown_called_, const std::atomic & table_is_being_dropped_, - std::shared_ptr s3_queue_log_, + std::shared_ptr system_queue_log_, const StorageID & storage_id_, LoggerPtr log_); @@ -111,7 +108,7 @@ private: const NamesAndTypesList requested_virtual_columns; const std::atomic & shutdown_called; const std::atomic & table_is_being_dropped; - const std::shared_ptr s3_queue_log; + const std::shared_ptr system_queue_log; const StorageID storage_id; RemoveFileFunc remove_file_func; diff --git a/src/Storages/ObjectStorageQueue/StorageObjectStorageQueue.cpp b/src/Storages/ObjectStorageQueue/StorageObjectStorageQueue.cpp index 489980aafa8..397087f566f 100644 --- a/src/Storages/ObjectStorageQueue/StorageObjectStorageQueue.cpp +++ b/src/Storages/ObjectStorageQueue/StorageObjectStorageQueue.cpp @@ -105,7 +105,7 @@ StorageObjectStorageQueue::StorageObjectStorageQueue( , configuration{configuration_} , format_settings(format_settings_) , reschedule_processing_interval_ms(queue_settings->polling_min_timeout_ms) - , log(getLogger("StorageObjectStorageQueue (" + table_id_.getFullTableName() + ")")) + , log(getLogger(fmt::format("{}Queue ({})", configuration->getEngineName(), table_id_.getFullTableName()))) { if (configuration->getPath().empty()) { @@ -332,7 +332,10 @@ std::shared_ptr StorageObjectStorageQueue::createSourc { object_storage->removeObject(StoredObject(path)); }; - auto s3_queue_log = queue_settings->enable_logging_to_s3queue_log ? local_context->getS3QueueLog() : nullptr; + auto system_queue_log = queue_settings->enable_logging_to_s3queue_log + ? local_context->getS3QueueLog() + : queue_settings->enable_logging_to_azure_queue_log ? local_context->getAzureQueueLog() : nullptr; + return std::make_shared( getName(), processor_id, @@ -345,7 +348,7 @@ std::shared_ptr StorageObjectStorageQueue::createSourc local_context, shutdown_called, table_is_being_dropped, - s3_queue_log, + system_queue_log, getStorageID(), log); } diff --git a/src/Storages/ObjectStorageQueue/registerS3Queue.cpp b/src/Storages/ObjectStorageQueue/registerS3Queue.cpp index 08b8104ae49..20968143627 100644 --- a/src/Storages/ObjectStorageQueue/registerS3Queue.cpp +++ b/src/Storages/ObjectStorageQueue/registerS3Queue.cpp @@ -1,14 +1,19 @@ #include "config.h" -#if USE_AWS_S3 - -#include #include -#include #include #include #include +#if USE_AWS_S3 +#include +#include +#endif + +#if USE_AZURE_BLOB_STORAGE +#include +#endif + namespace DB { @@ -17,64 +22,71 @@ namespace ErrorCodes extern const int BAD_ARGUMENTS; } +template +StoragePtr createQueueStorage(const StorageFactory::Arguments & args) +{ + auto & engine_args = args.engine_args; + if (engine_args.empty()) + throw Exception(ErrorCodes::BAD_ARGUMENTS, "External data source must have arguments"); + + auto configuration = std::make_shared(); + StorageObjectStorage::Configuration::initialize(*configuration, args.engine_args, args.getContext(), false); + + // Use format settings from global server context + settings from + // the SETTINGS clause of the create query. Settings from current + // session and user are ignored. + std::optional format_settings; + + auto queue_settings = std::make_unique(); + if (args.storage_def->settings) + { + queue_settings->loadFromQuery(*args.storage_def); + FormatFactorySettings user_format_settings; + + // Apply changed settings from global context, but ignore the + // unknown ones, because we only have the format settings here. + const auto & changes = args.getContext()->getSettingsRef().changes(); + for (const auto & change : changes) + { + if (user_format_settings.has(change.name)) + user_format_settings.set(change.name, change.value); + + args.storage_def->settings->changes.removeSetting(change.name); + } + + for (const auto & change : args.storage_def->settings->changes) + { + if (user_format_settings.has(change.name)) + user_format_settings.applyChange(change); + } + format_settings = getFormatSettings(args.getContext(), user_format_settings); + } + else + { + format_settings = getFormatSettings(args.getContext()); + } + + return std::make_shared( + std::move(queue_settings), + std::move(configuration), + args.table_id, + args.columns, + args.constraints, + args.comment, + args.getContext(), + format_settings, + args.storage_def, + args.mode); +} + +#if USE_AWS_S3 void registerStorageS3Queue(StorageFactory & factory) { factory.registerStorage( "S3Queue", [](const StorageFactory::Arguments & args) { - auto & engine_args = args.engine_args; - if (engine_args.empty()) - throw Exception(ErrorCodes::BAD_ARGUMENTS, "External data source must have arguments"); - - auto configuration = std::make_shared(); - StorageObjectStorage::Configuration::initialize(*configuration, args.engine_args, args.getContext(), false); - - // Use format settings from global server context + settings from - // the SETTINGS clause of the create query. Settings from current - // session and user are ignored. - std::optional format_settings; - - auto queue_settings = std::make_unique(); - if (args.storage_def->settings) - { - queue_settings->loadFromQuery(*args.storage_def); - FormatFactorySettings user_format_settings; - - // Apply changed settings from global context, but ignore the - // unknown ones, because we only have the format settings here. - const auto & changes = args.getContext()->getSettingsRef().changes(); - for (const auto & change : changes) - { - if (user_format_settings.has(change.name)) - user_format_settings.set(change.name, change.value); - - args.storage_def->settings->changes.removeSetting(change.name); - } - - for (const auto & change : args.storage_def->settings->changes) - { - if (user_format_settings.has(change.name)) - user_format_settings.applyChange(change); - } - format_settings = getFormatSettings(args.getContext(), user_format_settings); - } - else - { - format_settings = getFormatSettings(args.getContext()); - } - - return std::make_shared( - std::move(queue_settings), - std::move(configuration), - args.table_id, - args.columns, - args.constraints, - args.comment, - args.getContext(), - format_settings, - args.storage_def, - args.mode); + return createQueueStorage(args); }, { .supports_settings = true, @@ -82,6 +94,22 @@ void registerStorageS3Queue(StorageFactory & factory) .source_access_type = AccessType::S3, }); } +#endif +#if USE_AZURE_BLOB_STORAGE +void registerStorageAzureQueue(StorageFactory & factory) +{ + factory.registerStorage( + "AzureQueue", + [](const StorageFactory::Arguments & args) + { + return createQueueStorage(args); + }, + { + .supports_settings = true, + .supports_schema_inference = true, + .source_access_type = AccessType::AZURE, + }); } #endif +} diff --git a/src/Storages/registerStorages.cpp b/src/Storages/registerStorages.cpp index 47542b7b47e..628e5a85437 100644 --- a/src/Storages/registerStorages.cpp +++ b/src/Storages/registerStorages.cpp @@ -34,6 +34,7 @@ void registerStorageFuzzJSON(StorageFactory & factory); void registerStorageS3(StorageFactory & factory); void registerStorageHudi(StorageFactory & factory); void registerStorageS3Queue(StorageFactory & factory); +void registerStorageAzureQueue(StorageFactory & factory); #if USE_PARQUET void registerStorageDeltaLake(StorageFactory & factory); @@ -126,6 +127,10 @@ void registerStorages() registerStorageFuzzJSON(factory); #endif +#if USE_AZURE_BLOB_STORAGE + registerStorageAzureQueue(factory); +#endif + #if USE_AWS_S3 registerStorageHudi(factory); registerStorageS3Queue(factory); diff --git a/tests/integration/test_storage_s3_queue/test.py b/tests/integration/test_storage_s3_queue/test.py index 3c55f0ff7f7..f1c8c07d523 100644 --- a/tests/integration/test_storage_s3_queue/test.py +++ b/tests/integration/test_storage_s3_queue/test.py @@ -12,6 +12,7 @@ import json AVAILABLE_MODES = ["unordered", "ordered"] DEFAULT_AUTH = ["'minio'", "'minio123'"] NO_AUTH = ["NOSIGN"] +AZURE_CONTAINER_NAME = "cont" def prepare_public_s3_bucket(started_cluster): @@ -84,6 +85,7 @@ def started_cluster(): "instance", user_configs=["configs/users.xml"], with_minio=True, + with_azurite=True, with_zookeeper=True, main_configs=[ "configs/zookeeper.xml", @@ -115,6 +117,9 @@ def started_cluster(): cluster.start() logging.info("Cluster started") + container_client = cluster.blob_service_client.get_container_client(AZURE_CONTAINER_NAME) + container_client.create_container() + yield cluster finally: cluster.shutdown() @@ -134,6 +139,7 @@ def generate_random_files( started_cluster, files_path, count, + storage = "s3", column_num=3, row_num=10, start_ind=0, @@ -155,7 +161,10 @@ def generate_random_files( values_csv = ( "\n".join((",".join(map(str, row)) for row in rand_values)) + "\n" ).encode() - put_s3_file_content(started_cluster, filename, values_csv, bucket) + if storage == "s3": + put_s3_file_content(started_cluster, filename, values_csv, bucket) + else: + put_azure_file_content(started_cluster, filename, values_csv, bucket) return total_values @@ -164,6 +173,11 @@ def put_s3_file_content(started_cluster, filename, data, bucket=None): buf = io.BytesIO(data) started_cluster.minio_client.put_object(bucket, filename, buf, len(data)) +def put_azure_file_content(started_cluster, filename, data, bucket=None): + client = started_cluster.blob_service_client.get_blob_client(AZURE_CONTAINER_NAME, filename) + buf = io.BytesIO(data) + client.upload_blob(buf, "BlockBlob", len(data)) + def create_table( started_cluster, @@ -171,6 +185,7 @@ def create_table( table_name, mode, files_path, + engine_name = "S3Queue", format="column1 UInt32, column2 UInt32, column3 UInt32", additional_settings={}, file_format="CSV", @@ -189,11 +204,17 @@ def create_table( } settings.update(additional_settings) - url = f"http://{started_cluster.minio_host}:{started_cluster.minio_port}/{bucket}/{files_path}/" + engine_def = None + if engine_name == "S3Queue": + url = f"http://{started_cluster.minio_host}:{started_cluster.minio_port}/{bucket}/{files_path}/" + engine_def = f"{engine_name}('{url}', {auth_params}, {file_format})" + else: + engine_def = f"{engine_name}('{started_cluster.env_variables['AZURITE_CONNECTION_STRING']}', 'cont', '{files_path}/', 'CSV')" + node.query(f"DROP TABLE IF EXISTS {table_name}") create_query = f""" CREATE TABLE {table_name} ({format}) - ENGINE = S3Queue('{url}', {auth_params}, {file_format}) + ENGINE = {engine_def} SETTINGS {",".join((k+"="+repr(v) for k, v in settings.items()))} """ @@ -224,17 +245,29 @@ def create_mv( ) -@pytest.mark.parametrize("mode", AVAILABLE_MODES) -def test_delete_after_processing(started_cluster, mode): +@pytest.mark.parametrize( + "mode, engine_name", + [ + pytest.param("unordered", "S3Queue"), + pytest.param("unordered", "AzureQueue"), + pytest.param("ordered", "S3Queue"), + pytest.param("ordered", "AzureQueue"), + ], +) +def test_delete_after_processing(started_cluster, mode, engine_name): node = started_cluster.instances["instance"] - table_name = f"test.delete_after_processing_{mode}" + table_name = f"test.delete_after_processing_{mode}_{engine_name}" dst_table_name = f"{table_name}_dst" files_path = f"{table_name}_data" files_num = 5 row_num = 10 + if engine_name == "S3Queue": + storage = "s3" + else: + storage = "azure" total_values = generate_random_files( - started_cluster, files_path, files_num, row_num=row_num + started_cluster, files_path, files_num, row_num=row_num, storage = storage ) create_table( started_cluster, @@ -243,6 +276,7 @@ def test_delete_after_processing(started_cluster, mode): mode, files_path, additional_settings={"after_processing": "delete"}, + engine_name = engine_name, ) create_mv(node, table_name, dst_table_name) @@ -263,15 +297,29 @@ def test_delete_after_processing(started_cluster, mode): ).splitlines() ] == sorted(total_values, key=lambda x: (x[0], x[1], x[2])) - minio = started_cluster.minio_client - objects = list(minio.list_objects(started_cluster.minio_bucket, recursive=True)) - assert len(objects) == 0 + if engine_name == "S3Queue": + minio = started_cluster.minio_client + objects = list(minio.list_objects(started_cluster.minio_bucket, recursive=True)) + assert len(objects) == 0 + else: + client = started_cluster.blob_service_client.get_container_client(AZURE_CONTAINER_NAME) + objects_iterator = client.list_blobs(files_path) + for objects in objects_iterator: + assert False -@pytest.mark.parametrize("mode", AVAILABLE_MODES) -def test_failed_retry(started_cluster, mode): +@pytest.mark.parametrize( + "mode, engine_name", + [ + pytest.param("unordered", "S3Queue"), + pytest.param("unordered", "AzureQueue"), + pytest.param("ordered", "S3Queue"), + pytest.param("ordered", "AzureQueue"), + ], +) +def test_failed_retry(started_cluster, mode, engine_name): node = started_cluster.instances["instance"] - table_name = f"test.failed_retry_{mode}" + table_name = f"test.failed_retry_{mode}_{engine_name}" dst_table_name = f"{table_name}_dst" files_path = f"{table_name}_data" file_path = f"{files_path}/trash_test.csv" @@ -296,6 +344,7 @@ def test_failed_retry(started_cluster, mode): "s3queue_loading_retries": retries_num, "keeper_path": keeper_path, }, + engine_name = engine_name, ) create_mv(node, table_name, dst_table_name) From dc6fa85bc37f1239bc65d599dd5dfa0a7c1c1cc4 Mon Sep 17 00:00:00 2001 From: robot-clickhouse Date: Wed, 19 Jun 2024 14:37:46 +0000 Subject: [PATCH 032/115] Automatic style fix --- .../integration/test_storage_s3_queue/test.py | 23 ++++++++++++------- 1 file changed, 15 insertions(+), 8 deletions(-) diff --git a/tests/integration/test_storage_s3_queue/test.py b/tests/integration/test_storage_s3_queue/test.py index f1c8c07d523..463f457e920 100644 --- a/tests/integration/test_storage_s3_queue/test.py +++ b/tests/integration/test_storage_s3_queue/test.py @@ -117,7 +117,9 @@ def started_cluster(): cluster.start() logging.info("Cluster started") - container_client = cluster.blob_service_client.get_container_client(AZURE_CONTAINER_NAME) + container_client = cluster.blob_service_client.get_container_client( + AZURE_CONTAINER_NAME + ) container_client.create_container() yield cluster @@ -139,7 +141,7 @@ def generate_random_files( started_cluster, files_path, count, - storage = "s3", + storage="s3", column_num=3, row_num=10, start_ind=0, @@ -173,8 +175,11 @@ def put_s3_file_content(started_cluster, filename, data, bucket=None): buf = io.BytesIO(data) started_cluster.minio_client.put_object(bucket, filename, buf, len(data)) + def put_azure_file_content(started_cluster, filename, data, bucket=None): - client = started_cluster.blob_service_client.get_blob_client(AZURE_CONTAINER_NAME, filename) + client = started_cluster.blob_service_client.get_blob_client( + AZURE_CONTAINER_NAME, filename + ) buf = io.BytesIO(data) client.upload_blob(buf, "BlockBlob", len(data)) @@ -185,7 +190,7 @@ def create_table( table_name, mode, files_path, - engine_name = "S3Queue", + engine_name="S3Queue", format="column1 UInt32, column2 UInt32, column3 UInt32", additional_settings={}, file_format="CSV", @@ -267,7 +272,7 @@ def test_delete_after_processing(started_cluster, mode, engine_name): storage = "azure" total_values = generate_random_files( - started_cluster, files_path, files_num, row_num=row_num, storage = storage + started_cluster, files_path, files_num, row_num=row_num, storage=storage ) create_table( started_cluster, @@ -276,7 +281,7 @@ def test_delete_after_processing(started_cluster, mode, engine_name): mode, files_path, additional_settings={"after_processing": "delete"}, - engine_name = engine_name, + engine_name=engine_name, ) create_mv(node, table_name, dst_table_name) @@ -302,7 +307,9 @@ def test_delete_after_processing(started_cluster, mode, engine_name): objects = list(minio.list_objects(started_cluster.minio_bucket, recursive=True)) assert len(objects) == 0 else: - client = started_cluster.blob_service_client.get_container_client(AZURE_CONTAINER_NAME) + client = started_cluster.blob_service_client.get_container_client( + AZURE_CONTAINER_NAME + ) objects_iterator = client.list_blobs(files_path) for objects in objects_iterator: assert False @@ -344,7 +351,7 @@ def test_failed_retry(started_cluster, mode, engine_name): "s3queue_loading_retries": retries_num, "keeper_path": keeper_path, }, - engine_name = engine_name, + engine_name=engine_name, ) create_mv(node, table_name, dst_table_name) From 1eabdd81b6492e6109b27ff639d11f95845b47e2 Mon Sep 17 00:00:00 2001 From: Kseniia Sumarokova <54203879+kssenii@users.noreply.github.com> Date: Wed, 19 Jun 2024 19:12:08 +0200 Subject: [PATCH 033/115] Fix logger name --- src/Storages/ObjectStorageQueue/StorageObjectStorageQueue.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Storages/ObjectStorageQueue/StorageObjectStorageQueue.cpp b/src/Storages/ObjectStorageQueue/StorageObjectStorageQueue.cpp index 397087f566f..640dbf22421 100644 --- a/src/Storages/ObjectStorageQueue/StorageObjectStorageQueue.cpp +++ b/src/Storages/ObjectStorageQueue/StorageObjectStorageQueue.cpp @@ -105,7 +105,7 @@ StorageObjectStorageQueue::StorageObjectStorageQueue( , configuration{configuration_} , format_settings(format_settings_) , reschedule_processing_interval_ms(queue_settings->polling_min_timeout_ms) - , log(getLogger(fmt::format("{}Queue ({})", configuration->getEngineName(), table_id_.getFullTableName()))) + , log(getLogger(fmt::format("Storage{}Queue ({})", configuration->getEngineName(), table_id_.getFullTableName()))) { if (configuration->getPath().empty()) { From ede8630205e4c073c7f1f8db46ea1c025e5e2de2 Mon Sep 17 00:00:00 2001 From: avogar Date: Wed, 19 Jun 2024 19:25:16 +0000 Subject: [PATCH 034/115] Fix some tests, add debug logging for test 00693_max_block_size_system_tables_columns, use global context + current database in getDependenciesFromCreateQuery --- src/Backups/RestorerFromBackup.cpp | 2 +- src/Databases/DDLDependencyVisitor.cpp | 8 ++++---- src/Databases/DDLDependencyVisitor.h | 2 +- src/Databases/DDLLoadingDependencyVisitor.cpp | 2 +- src/Databases/DatabaseMemory.cpp | 2 +- src/Databases/DatabaseOrdinary.cpp | 2 +- src/Databases/DatabaseReplicated.cpp | 2 +- src/Databases/TablesLoader.cpp | 2 +- src/Interpreters/InterpreterCreateQuery.cpp | 4 ++-- src/Storages/System/StorageSystemColumns.cpp | 1 + .../0_stateless/01083_expressions_in_engine_arguments.sql | 2 +- .../0_stateless/01852_dictionary_found_rate_long.sql | 2 +- .../02152_http_external_tables_memory_tracking.sh | 2 +- 13 files changed, 17 insertions(+), 16 deletions(-) diff --git a/src/Backups/RestorerFromBackup.cpp b/src/Backups/RestorerFromBackup.cpp index 1a3fdf58cc4..454a0468e9f 100644 --- a/src/Backups/RestorerFromBackup.cpp +++ b/src/Backups/RestorerFromBackup.cpp @@ -438,7 +438,7 @@ void RestorerFromBackup::findTableInBackupImpl(const QualifiedTableName & table_ String create_table_query_str = serializeAST(*create_table_query); bool is_predefined_table = DatabaseCatalog::instance().isPredefinedTable(StorageID{table_name.database, table_name.table}); - auto table_dependencies = getDependenciesFromCreateQuery(context, table_name, create_table_query); + auto table_dependencies = getDependenciesFromCreateQuery(context, table_name, create_table_query, context->getCurrentDatabase()); bool table_has_data = backup->hasFiles(data_path_in_backup); std::lock_guard lock{mutex}; diff --git a/src/Databases/DDLDependencyVisitor.cpp b/src/Databases/DDLDependencyVisitor.cpp index fdc51f4f43d..06df7371b50 100644 --- a/src/Databases/DDLDependencyVisitor.cpp +++ b/src/Databases/DDLDependencyVisitor.cpp @@ -30,8 +30,8 @@ namespace { friend void tryVisitNestedSelect(const String & query, DDLDependencyVisitorData & data); public: - DDLDependencyVisitorData(const ContextPtr & context_, const QualifiedTableName & table_name_, const ASTPtr & ast_) - : create_query(ast_), table_name(table_name_), current_database(context_->getCurrentDatabase()), context(context_) + DDLDependencyVisitorData(const ContextPtr & context_, const QualifiedTableName & table_name_, const ASTPtr & ast_, const String & current_database_) + : create_query(ast_), table_name(table_name_), current_database(current_database_), context(context_) { } @@ -464,9 +464,9 @@ namespace } -TableNamesSet getDependenciesFromCreateQuery(const ContextPtr & context, const QualifiedTableName & table_name, const ASTPtr & ast) +TableNamesSet getDependenciesFromCreateQuery(const ContextPtr & global_context, const QualifiedTableName & table_name, const ASTPtr & ast, const String & current_database) { - DDLDependencyVisitor::Data data{context, table_name, ast}; + DDLDependencyVisitor::Data data{global_context, table_name, ast, current_database}; DDLDependencyVisitor::Visitor visitor{data}; visitor.visit(ast); return std::move(data).getDependencies(); diff --git a/src/Databases/DDLDependencyVisitor.h b/src/Databases/DDLDependencyVisitor.h index 29ea6298b04..a17640f7a14 100644 --- a/src/Databases/DDLDependencyVisitor.h +++ b/src/Databases/DDLDependencyVisitor.h @@ -13,6 +13,6 @@ using TableNamesSet = std::unordered_set; /// Returns a list of all tables explicitly referenced in the create query of a specified table. /// For example, a column default expression can use dictGet() and thus reference a dictionary. /// Does not validate AST, works a best-effort way. -TableNamesSet getDependenciesFromCreateQuery(const ContextPtr & context, const QualifiedTableName & table_name, const ASTPtr & ast); +TableNamesSet getDependenciesFromCreateQuery(const ContextPtr & global_context, const QualifiedTableName & table_name, const ASTPtr & ast, const String & current_database); } diff --git a/src/Databases/DDLLoadingDependencyVisitor.cpp b/src/Databases/DDLLoadingDependencyVisitor.cpp index 0253709fb6e..5a7e9e45cf8 100644 --- a/src/Databases/DDLLoadingDependencyVisitor.cpp +++ b/src/Databases/DDLLoadingDependencyVisitor.cpp @@ -122,7 +122,7 @@ void DDLLoadingDependencyVisitor::visit(const ASTStorage & storage, Data & data) { if (storage.ttl_table) { - auto ttl_dependensies = getDependenciesFromCreateQuery(data.global_context, data.table_name, storage.ttl_table->ptr()); + auto ttl_dependensies = getDependenciesFromCreateQuery(data.global_context, data.table_name, storage.ttl_table->ptr(), data.default_database); data.dependencies.merge(ttl_dependensies); } diff --git a/src/Databases/DatabaseMemory.cpp b/src/Databases/DatabaseMemory.cpp index dce20e3ac6f..508e57a4eaf 100644 --- a/src/Databases/DatabaseMemory.cpp +++ b/src/Databases/DatabaseMemory.cpp @@ -154,7 +154,7 @@ void DatabaseMemory::alterTable(ContextPtr local_context, const StorageID & tabl applyMetadataChangesToCreateQuery(it->second, metadata); /// The create query of the table has been just changed, we need to update dependencies too. - auto ref_dependencies = getDependenciesFromCreateQuery(local_context, table_id.getQualifiedName(), it->second); + auto ref_dependencies = getDependenciesFromCreateQuery(local_context->getGlobalContext(), table_id.getQualifiedName(), it->second, local_context->getCurrentDatabase()); auto loading_dependencies = getLoadingDependenciesFromCreateQuery(local_context->getGlobalContext(), table_id.getQualifiedName(), it->second, local_context->getCurrentDatabase()); DatabaseCatalog::instance().updateDependencies(table_id, ref_dependencies, loading_dependencies); } diff --git a/src/Databases/DatabaseOrdinary.cpp b/src/Databases/DatabaseOrdinary.cpp index 18c92e6bcbc..d6a5f39a09f 100644 --- a/src/Databases/DatabaseOrdinary.cpp +++ b/src/Databases/DatabaseOrdinary.cpp @@ -539,7 +539,7 @@ void DatabaseOrdinary::alterTable(ContextPtr local_context, const StorageID & ta } /// The create query of the table has been just changed, we need to update dependencies too. - auto ref_dependencies = getDependenciesFromCreateQuery(local_context, table_id.getQualifiedName(), ast); + auto ref_dependencies = getDependenciesFromCreateQuery(local_context->getGlobalContext(), table_id.getQualifiedName(), ast, local_context->getCurrentDatabase()); auto loading_dependencies = getLoadingDependenciesFromCreateQuery(local_context->getGlobalContext(), table_id.getQualifiedName(), ast, local_context->getCurrentDatabase()); DatabaseCatalog::instance().updateDependencies(table_id, ref_dependencies, loading_dependencies); diff --git a/src/Databases/DatabaseReplicated.cpp b/src/Databases/DatabaseReplicated.cpp index badfedeec9b..fde84592e17 100644 --- a/src/Databases/DatabaseReplicated.cpp +++ b/src/Databases/DatabaseReplicated.cpp @@ -1146,7 +1146,7 @@ void DatabaseReplicated::recoverLostReplica(const ZooKeeperPtr & current_zookeep /// And QualifiedTableName::parseFromString doesn't handle this. auto qualified_name = QualifiedTableName{.database = getDatabaseName(), .table = table_name}; auto query_ast = parseQueryFromMetadataInZooKeeper(table_name, create_table_query); - tables_dependencies.addDependencies(qualified_name, getDependenciesFromCreateQuery(getContext(), qualified_name, query_ast)); + tables_dependencies.addDependencies(qualified_name, getDependenciesFromCreateQuery(getContext(), qualified_name, query_ast, getContext()->getCurrentDatabase())); } tables_dependencies.checkNoCyclicDependencies(); diff --git a/src/Databases/TablesLoader.cpp b/src/Databases/TablesLoader.cpp index 4bfe44ba72c..33944277995 100644 --- a/src/Databases/TablesLoader.cpp +++ b/src/Databases/TablesLoader.cpp @@ -137,7 +137,7 @@ void TablesLoader::buildDependencyGraph() { for (const auto & [table_name, table_metadata] : metadata.parsed_tables) { - auto new_ref_dependencies = getDependenciesFromCreateQuery(global_context, table_name, table_metadata.ast); + auto new_ref_dependencies = getDependenciesFromCreateQuery(global_context, table_name, table_metadata.ast, global_context->getCurrentDatabase()); auto new_loading_dependencies = getLoadingDependenciesFromCreateQuery(global_context, table_name, table_metadata.ast, global_context->getCurrentDatabase()); if (!new_ref_dependencies.empty()) diff --git a/src/Interpreters/InterpreterCreateQuery.cpp b/src/Interpreters/InterpreterCreateQuery.cpp index 35207bc9d39..242b093c0ea 100644 --- a/src/Interpreters/InterpreterCreateQuery.cpp +++ b/src/Interpreters/InterpreterCreateQuery.cpp @@ -1083,7 +1083,7 @@ namespace void addTableDependencies(const ASTCreateQuery & create, const ASTPtr & query_ptr, const ContextPtr & context) { QualifiedTableName qualified_name{create.getDatabase(), create.getTable()}; - auto ref_dependencies = getDependenciesFromCreateQuery(context, qualified_name, query_ptr); + auto ref_dependencies = getDependenciesFromCreateQuery(context->getGlobalContext(), qualified_name, query_ptr, context->getCurrentDatabase()); auto loading_dependencies = getLoadingDependenciesFromCreateQuery(context->getGlobalContext(), qualified_name, query_ptr, context->getCurrentDatabase()); DatabaseCatalog::instance().addDependencies(qualified_name, ref_dependencies, loading_dependencies); } @@ -1091,7 +1091,7 @@ void addTableDependencies(const ASTCreateQuery & create, const ASTPtr & query_pt void checkTableCanBeAddedWithNoCyclicDependencies(const ASTCreateQuery & create, const ASTPtr & query_ptr, const ContextPtr & context) { QualifiedTableName qualified_name{create.getDatabase(), create.getTable()}; - auto ref_dependencies = getDependenciesFromCreateQuery(context, qualified_name, query_ptr); + auto ref_dependencies = getDependenciesFromCreateQuery(context->getGlobalContext(), qualified_name, query_ptr, context->getCurrentDatabase()); auto loading_dependencies = getLoadingDependenciesFromCreateQuery(context->getGlobalContext(), qualified_name, query_ptr, context->getCurrentDatabase()); DatabaseCatalog::instance().checkTableCanBeAddedWithNoCyclicDependencies(qualified_name, ref_dependencies, loading_dependencies); } diff --git a/src/Storages/System/StorageSystemColumns.cpp b/src/Storages/System/StorageSystemColumns.cpp index 49da1eba9ec..a6a791f75b5 100644 --- a/src/Storages/System/StorageSystemColumns.cpp +++ b/src/Storages/System/StorageSystemColumns.cpp @@ -134,6 +134,7 @@ protected: cols_required_for_sorting_key = metadata_snapshot->getColumnsRequiredForSortingKey(); cols_required_for_primary_key = metadata_snapshot->getColumnsRequiredForPrimaryKey(); cols_required_for_sampling = metadata_snapshot->getColumnsRequiredForSampling(); + LOG_DEBUG(getLogger("StorageSystemColumns"), "Get column sizes for table {}.{}", database_name, table_name); column_sizes = storage->getColumnSizes(); } diff --git a/tests/queries/0_stateless/01083_expressions_in_engine_arguments.sql b/tests/queries/0_stateless/01083_expressions_in_engine_arguments.sql index 6268765aa27..bdfbf2a47cf 100644 --- a/tests/queries/0_stateless/01083_expressions_in_engine_arguments.sql +++ b/tests/queries/0_stateless/01083_expressions_in_engine_arguments.sql @@ -88,6 +88,7 @@ SELECT sum(n) from rich_syntax; SYSTEM DROP DNS CACHE; DROP TABLE file; +DROP DICTIONARY dict; DROP TABLE url; DROP TABLE view; DROP TABLE buffer; @@ -96,4 +97,3 @@ DROP TABLE merge_tf; DROP TABLE distributed; DROP TABLE distributed_tf; DROP TABLE rich_syntax; -DROP DICTIONARY dict; diff --git a/tests/queries/0_stateless/01852_dictionary_found_rate_long.sql b/tests/queries/0_stateless/01852_dictionary_found_rate_long.sql index d5108e98510..da364403893 100644 --- a/tests/queries/0_stateless/01852_dictionary_found_rate_long.sql +++ b/tests/queries/0_stateless/01852_dictionary_found_rate_long.sql @@ -310,6 +310,6 @@ SELECT name, found_rate FROM system.dictionaries WHERE database = currentDatabas SELECT tuple(x, y) as key, dictGet('polygon_dictionary_01862', 'name', key) FROM points_01862 FORMAT Null; SELECT name, found_rate FROM system.dictionaries WHERE database = currentDatabase() AND name = 'polygon_dictionary_01862'; +DROP DICTIONARY polygon_dictionary_01862; DROP TABLE polygons_01862; DROP TABLE points_01862; -DROP DICTIONARY polygon_dictionary_01862; diff --git a/tests/queries/0_stateless/02152_http_external_tables_memory_tracking.sh b/tests/queries/0_stateless/02152_http_external_tables_memory_tracking.sh index 5494f7d59cb..8aba362d6af 100755 --- a/tests/queries/0_stateless/02152_http_external_tables_memory_tracking.sh +++ b/tests/queries/0_stateless/02152_http_external_tables_memory_tracking.sh @@ -1,5 +1,5 @@ #!/usr/bin/env bash -# Tags: no-tsan, no-cpu-aarch64, no-parallel, no-debug +# Tags: no-tsan, no-cpu-aarch64, no-parallel # TSan does not supports tracing. # trace_log doesn't work on aarch64 From 347d67cbe525972bcf2caadaae190f5ddf4217ef Mon Sep 17 00:00:00 2001 From: avogar Date: Thu, 20 Jun 2024 15:58:08 +0000 Subject: [PATCH 035/115] Process databases correctly --- src/Databases/DDLDependencyVisitor.cpp | 44 +++++++++++++------ src/Databases/DDLDependencyVisitor.h | 1 + src/Databases/DDLLoadingDependencyVisitor.cpp | 22 +++++++--- src/Databases/DDLLoadingDependencyVisitor.h | 2 +- src/Databases/DatabaseMemory.cpp | 2 +- src/Databases/DatabaseOrdinary.cpp | 2 +- src/Databases/DatabaseReplicated.cpp | 2 +- src/Databases/TablesLoader.cpp | 2 +- src/Interpreters/InterpreterCreateQuery.cpp | 4 +- 9 files changed, 55 insertions(+), 26 deletions(-) diff --git a/src/Databases/DDLDependencyVisitor.cpp b/src/Databases/DDLDependencyVisitor.cpp index 06df7371b50..c85e8f5688a 100644 --- a/src/Databases/DDLDependencyVisitor.cpp +++ b/src/Databases/DDLDependencyVisitor.cpp @@ -30,8 +30,8 @@ namespace { friend void tryVisitNestedSelect(const String & query, DDLDependencyVisitorData & data); public: - DDLDependencyVisitorData(const ContextPtr & context_, const QualifiedTableName & table_name_, const ASTPtr & ast_, const String & current_database_) - : create_query(ast_), table_name(table_name_), current_database(current_database_), context(context_) + DDLDependencyVisitorData(const ContextPtr & global_context_, const QualifiedTableName & table_name_, const ASTPtr & ast_, const String & current_database_) + : create_query(ast_), table_name(table_name_), default_database(global_context_->getCurrentDatabase()), current_database(current_database_), global_context(global_context_) { } @@ -71,8 +71,9 @@ namespace ASTPtr create_query; std::unordered_set skip_asts; QualifiedTableName table_name; + String default_database; String current_database; - ContextPtr context; + ContextPtr global_context; TableNamesSet dependencies; /// CREATE TABLE or CREATE DICTIONARY or CREATE VIEW or CREATE TEMPORARY TABLE or CREATE DATABASE query. @@ -108,8 +109,8 @@ namespace if (!dictionary.source || dictionary.source->name != "clickhouse" || !dictionary.source->elements) return; - auto config = getDictionaryConfigurationFromAST(create_query->as(), context); - auto info = getInfoIfClickHouseDictionarySource(config, context); + auto config = getDictionaryConfigurationFromAST(create_query->as(), global_context); + auto info = getInfoIfClickHouseDictionarySource(config, global_context); /// We consider only dependencies on local tables. if (!info || !info->is_local) @@ -117,14 +118,21 @@ namespace if (!info->table_name.table.empty()) { + /// If database is not specified in dictionary source, use database of the dictionary itself, not the current/default database. if (info->table_name.database.empty()) - info->table_name.database = current_database; + info->table_name.database = table_name.database; dependencies.emplace(std::move(info->table_name)); } else { - /// We don't have a table name, we have a select query instead + /// We don't have a table name, we have a select query instead. + /// All tables from select query in dictionary definition won't + /// use current database, as this query is executed with global context. + /// Use default database from global context while visiting select query. + String current_database_ = current_database; + current_database = default_database; tryVisitNestedSelect(info->query, *this); + current_database = current_database_; } } @@ -181,7 +189,7 @@ namespace if (auto cluster_name = tryGetClusterNameFromArgument(table_engine, 0)) { - auto cluster = context->tryGetCluster(*cluster_name); + auto cluster = global_context->tryGetCluster(*cluster_name); if (cluster && cluster->getLocalShardCount()) has_local_replicas = true; } @@ -236,7 +244,7 @@ namespace { if (auto cluster_name = tryGetClusterNameFromArgument(function, 0)) { - if (auto cluster = context->tryGetCluster(*cluster_name)) + if (auto cluster = global_context->tryGetCluster(*cluster_name)) { if (cluster->getLocalShardCount()) has_local_replicas = true; @@ -308,7 +316,10 @@ namespace try { /// We're just searching for dependencies here, it's not safe to execute subqueries now. - auto evaluated = evaluateConstantExpressionOrIdentifierAsLiteral(arg, context); + /// Use copy of the global_context and set current database, because expressions can contain currentDatabase() function. + ContextMutablePtr global_context_copy = Context::createCopy(global_context); + global_context_copy->setCurrentDatabase(current_database); + auto evaluated = evaluateConstantExpressionOrIdentifierAsLiteral(arg, global_context_copy); const auto * literal = evaluated->as(); if (!literal || (literal->value.getType() != Field::Types::String)) return {}; @@ -449,7 +460,7 @@ namespace ParserSelectWithUnionQuery parser; String description = fmt::format("Query for ClickHouse dictionary {}", data.table_name); String fixed_query = removeWhereConditionPlaceholder(query); - const Settings & settings = data.context->getSettingsRef(); + const Settings & settings = data.global_context->getSettingsRef(); ASTPtr select = parseQuery(parser, fixed_query, description, settings.max_query_size, settings.max_parser_depth, settings.max_parser_backtracks); @@ -464,12 +475,19 @@ namespace } -TableNamesSet getDependenciesFromCreateQuery(const ContextPtr & global_context, const QualifiedTableName & table_name, const ASTPtr & ast, const String & current_database) +TableNamesSet getDependenciesFromCreateQuery(const ContextPtr & global_global_context, const QualifiedTableName & table_name, const ASTPtr & ast, const String & current_database) { - DDLDependencyVisitor::Data data{global_context, table_name, ast, current_database}; + DDLDependencyVisitor::Data data{global_global_context, table_name, ast, current_database}; DDLDependencyVisitor::Visitor visitor{data}; visitor.visit(ast); return std::move(data).getDependencies(); } +TableNamesSet getDependenciesFromDictionaryNestedSelectQuery(const ContextPtr & global_context, const QualifiedTableName & table_name, const ASTPtr & ast, const String & select_query, const String & current_database) +{ + DDLDependencyVisitor::Data data{global_context, table_name, ast, current_database}; + tryVisitNestedSelect(select_query, data); + return std::move(data).getDependencies(); +} + } diff --git a/src/Databases/DDLDependencyVisitor.h b/src/Databases/DDLDependencyVisitor.h index a17640f7a14..fcbad83f440 100644 --- a/src/Databases/DDLDependencyVisitor.h +++ b/src/Databases/DDLDependencyVisitor.h @@ -14,5 +14,6 @@ using TableNamesSet = std::unordered_set; /// For example, a column default expression can use dictGet() and thus reference a dictionary. /// Does not validate AST, works a best-effort way. TableNamesSet getDependenciesFromCreateQuery(const ContextPtr & global_context, const QualifiedTableName & table_name, const ASTPtr & ast, const String & current_database); +TableNamesSet getDependenciesFromDictionaryNestedSelectQuery(const ContextPtr & global_context, const QualifiedTableName & table_name, const ASTPtr & ast, const String & select_query, const String & current_database); } diff --git a/src/Databases/DDLLoadingDependencyVisitor.cpp b/src/Databases/DDLLoadingDependencyVisitor.cpp index 5a7e9e45cf8..d3f41cecb86 100644 --- a/src/Databases/DDLLoadingDependencyVisitor.cpp +++ b/src/Databases/DDLLoadingDependencyVisitor.cpp @@ -21,11 +21,11 @@ namespace DB using TableLoadingDependenciesVisitor = DDLLoadingDependencyVisitor::Visitor; -TableNamesSet getLoadingDependenciesFromCreateQuery(ContextPtr global_context, const QualifiedTableName & table, const ASTPtr & ast, const String & default_database) +TableNamesSet getLoadingDependenciesFromCreateQuery(ContextPtr global_context, const QualifiedTableName & table, const ASTPtr & ast) { assert(global_context == global_context->getGlobalContext()); TableLoadingDependenciesVisitor::Data data; - data.default_database = default_database; + data.default_database = global_context->getCurrentDatabase(); data.create_query = ast; data.global_context = global_context; data.table_name = table; @@ -110,12 +110,22 @@ void DDLLoadingDependencyVisitor::visit(const ASTFunctionWithKeyValueArguments & auto config = getDictionaryConfigurationFromAST(data.create_query->as(), data.global_context); auto info = getInfoIfClickHouseDictionarySource(config, data.global_context); - if (!info || !info->is_local || info->table_name.table.empty()) + if (!info || !info->is_local) return; - if (info->table_name.database.empty()) - info->table_name.database = data.default_database; - data.dependencies.emplace(std::move(info->table_name)); + if (!info->table_name.table.empty()) + { + /// If database is not specified in dictionary source, use database of the dictionary itself, not the current/default database. + if (info->table_name.database.empty()) + info->table_name.database = data.table_name.database; + data.dependencies.emplace(std::move(info->table_name)); + } + else + { + /// We don't have a table name, we have a select query instead that will be executed during dictionary loading. + auto select_query_dependencies = getDependenciesFromDictionaryNestedSelectQuery(data.global_context, data.table_name, data.create_query, info->query, data.default_database); + data.dependencies.merge(select_query_dependencies); + } } void DDLLoadingDependencyVisitor::visit(const ASTStorage & storage, Data & data) diff --git a/src/Databases/DDLLoadingDependencyVisitor.h b/src/Databases/DDLLoadingDependencyVisitor.h index 099d9c1979b..a9e9f4d7a53 100644 --- a/src/Databases/DDLLoadingDependencyVisitor.h +++ b/src/Databases/DDLLoadingDependencyVisitor.h @@ -16,7 +16,7 @@ using TableNamesSet = std::unordered_set; /// Returns a list of all tables which should be loaded before a specified table. /// For example, a local ClickHouse table should be loaded before a dictionary which uses that table as its source. /// Does not validate AST, works a best-effort way. -TableNamesSet getLoadingDependenciesFromCreateQuery(ContextPtr global_context, const QualifiedTableName & table, const ASTPtr & ast, const String & default_database); +TableNamesSet getLoadingDependenciesFromCreateQuery(ContextPtr global_context, const QualifiedTableName & table, const ASTPtr & ast); class DDLMatcherBase diff --git a/src/Databases/DatabaseMemory.cpp b/src/Databases/DatabaseMemory.cpp index 508e57a4eaf..86bf0471b8f 100644 --- a/src/Databases/DatabaseMemory.cpp +++ b/src/Databases/DatabaseMemory.cpp @@ -155,7 +155,7 @@ void DatabaseMemory::alterTable(ContextPtr local_context, const StorageID & tabl /// The create query of the table has been just changed, we need to update dependencies too. auto ref_dependencies = getDependenciesFromCreateQuery(local_context->getGlobalContext(), table_id.getQualifiedName(), it->second, local_context->getCurrentDatabase()); - auto loading_dependencies = getLoadingDependenciesFromCreateQuery(local_context->getGlobalContext(), table_id.getQualifiedName(), it->second, local_context->getCurrentDatabase()); + auto loading_dependencies = getLoadingDependenciesFromCreateQuery(local_context->getGlobalContext(), table_id.getQualifiedName(), it->second); DatabaseCatalog::instance().updateDependencies(table_id, ref_dependencies, loading_dependencies); } diff --git a/src/Databases/DatabaseOrdinary.cpp b/src/Databases/DatabaseOrdinary.cpp index d6a5f39a09f..7d4bb07e8ef 100644 --- a/src/Databases/DatabaseOrdinary.cpp +++ b/src/Databases/DatabaseOrdinary.cpp @@ -540,7 +540,7 @@ void DatabaseOrdinary::alterTable(ContextPtr local_context, const StorageID & ta /// The create query of the table has been just changed, we need to update dependencies too. auto ref_dependencies = getDependenciesFromCreateQuery(local_context->getGlobalContext(), table_id.getQualifiedName(), ast, local_context->getCurrentDatabase()); - auto loading_dependencies = getLoadingDependenciesFromCreateQuery(local_context->getGlobalContext(), table_id.getQualifiedName(), ast, local_context->getCurrentDatabase()); + auto loading_dependencies = getLoadingDependenciesFromCreateQuery(local_context->getGlobalContext(), table_id.getQualifiedName(), ast); DatabaseCatalog::instance().updateDependencies(table_id, ref_dependencies, loading_dependencies); commitAlterTable(table_id, table_metadata_tmp_path, table_metadata_path, statement, local_context); diff --git a/src/Databases/DatabaseReplicated.cpp b/src/Databases/DatabaseReplicated.cpp index fde84592e17..ad1603f08bb 100644 --- a/src/Databases/DatabaseReplicated.cpp +++ b/src/Databases/DatabaseReplicated.cpp @@ -1146,7 +1146,7 @@ void DatabaseReplicated::recoverLostReplica(const ZooKeeperPtr & current_zookeep /// And QualifiedTableName::parseFromString doesn't handle this. auto qualified_name = QualifiedTableName{.database = getDatabaseName(), .table = table_name}; auto query_ast = parseQueryFromMetadataInZooKeeper(table_name, create_table_query); - tables_dependencies.addDependencies(qualified_name, getDependenciesFromCreateQuery(getContext(), qualified_name, query_ast, getContext()->getCurrentDatabase())); + tables_dependencies.addDependencies(qualified_name, getDependenciesFromCreateQuery(getContext()->getGlobalContext(), qualified_name, query_ast, getContext()->getCurrentDatabase())); } tables_dependencies.checkNoCyclicDependencies(); diff --git a/src/Databases/TablesLoader.cpp b/src/Databases/TablesLoader.cpp index 33944277995..733e5d53981 100644 --- a/src/Databases/TablesLoader.cpp +++ b/src/Databases/TablesLoader.cpp @@ -138,7 +138,7 @@ void TablesLoader::buildDependencyGraph() for (const auto & [table_name, table_metadata] : metadata.parsed_tables) { auto new_ref_dependencies = getDependenciesFromCreateQuery(global_context, table_name, table_metadata.ast, global_context->getCurrentDatabase()); - auto new_loading_dependencies = getLoadingDependenciesFromCreateQuery(global_context, table_name, table_metadata.ast, global_context->getCurrentDatabase()); + auto new_loading_dependencies = getLoadingDependenciesFromCreateQuery(global_context, table_name, table_metadata.ast); if (!new_ref_dependencies.empty()) referential_dependencies.addDependencies(table_name, new_ref_dependencies); diff --git a/src/Interpreters/InterpreterCreateQuery.cpp b/src/Interpreters/InterpreterCreateQuery.cpp index 242b093c0ea..fad7fd421a8 100644 --- a/src/Interpreters/InterpreterCreateQuery.cpp +++ b/src/Interpreters/InterpreterCreateQuery.cpp @@ -1084,7 +1084,7 @@ void addTableDependencies(const ASTCreateQuery & create, const ASTPtr & query_pt { QualifiedTableName qualified_name{create.getDatabase(), create.getTable()}; auto ref_dependencies = getDependenciesFromCreateQuery(context->getGlobalContext(), qualified_name, query_ptr, context->getCurrentDatabase()); - auto loading_dependencies = getLoadingDependenciesFromCreateQuery(context->getGlobalContext(), qualified_name, query_ptr, context->getCurrentDatabase()); + auto loading_dependencies = getLoadingDependenciesFromCreateQuery(context->getGlobalContext(), qualified_name, query_ptr); DatabaseCatalog::instance().addDependencies(qualified_name, ref_dependencies, loading_dependencies); } @@ -1092,7 +1092,7 @@ void checkTableCanBeAddedWithNoCyclicDependencies(const ASTCreateQuery & create, { QualifiedTableName qualified_name{create.getDatabase(), create.getTable()}; auto ref_dependencies = getDependenciesFromCreateQuery(context->getGlobalContext(), qualified_name, query_ptr, context->getCurrentDatabase()); - auto loading_dependencies = getLoadingDependenciesFromCreateQuery(context->getGlobalContext(), qualified_name, query_ptr, context->getCurrentDatabase()); + auto loading_dependencies = getLoadingDependenciesFromCreateQuery(context->getGlobalContext(), qualified_name, query_ptr); DatabaseCatalog::instance().checkTableCanBeAddedWithNoCyclicDependencies(qualified_name, ref_dependencies, loading_dependencies); } From b9e6d1c26bea4161c37f0e496d18354d64136693 Mon Sep 17 00:00:00 2001 From: avogar Date: Thu, 20 Jun 2024 16:02:03 +0000 Subject: [PATCH 036/115] Better comments --- src/Databases/DDLDependencyVisitor.h | 2 ++ src/Databases/DDLLoadingDependencyVisitor.cpp | 1 + 2 files changed, 3 insertions(+) diff --git a/src/Databases/DDLDependencyVisitor.h b/src/Databases/DDLDependencyVisitor.h index fcbad83f440..400e6b04108 100644 --- a/src/Databases/DDLDependencyVisitor.h +++ b/src/Databases/DDLDependencyVisitor.h @@ -14,6 +14,8 @@ using TableNamesSet = std::unordered_set; /// For example, a column default expression can use dictGet() and thus reference a dictionary. /// Does not validate AST, works a best-effort way. TableNamesSet getDependenciesFromCreateQuery(const ContextPtr & global_context, const QualifiedTableName & table_name, const ASTPtr & ast, const String & current_database); + +/// Returns a list of all tables explicitly referenced in the select query specified as a dictionary source. TableNamesSet getDependenciesFromDictionaryNestedSelectQuery(const ContextPtr & global_context, const QualifiedTableName & table_name, const ASTPtr & ast, const String & select_query, const String & current_database); } diff --git a/src/Databases/DDLLoadingDependencyVisitor.cpp b/src/Databases/DDLLoadingDependencyVisitor.cpp index d3f41cecb86..40234abb20f 100644 --- a/src/Databases/DDLLoadingDependencyVisitor.cpp +++ b/src/Databases/DDLLoadingDependencyVisitor.cpp @@ -123,6 +123,7 @@ void DDLLoadingDependencyVisitor::visit(const ASTFunctionWithKeyValueArguments & else { /// We don't have a table name, we have a select query instead that will be executed during dictionary loading. + /// We need to find all tables used in this select query and add them to dependencies. auto select_query_dependencies = getDependenciesFromDictionaryNestedSelectQuery(data.global_context, data.table_name, data.create_query, info->query, data.default_database); data.dependencies.merge(select_query_dependencies); } From 5904847316115cfdae3662e39746e5c07b1647dc Mon Sep 17 00:00:00 2001 From: kssenii Date: Thu, 20 Jun 2024 18:32:00 +0200 Subject: [PATCH 037/115] Fix tests --- .../engines/table-engines/integrations/s3queue.md | 2 ++ tests/integration/test_storage_s3_queue/test.py | 13 ++++++++----- 2 files changed, 10 insertions(+), 5 deletions(-) diff --git a/docs/en/engines/table-engines/integrations/s3queue.md b/docs/en/engines/table-engines/integrations/s3queue.md index 0958680dc56..e8b1628d03e 100644 --- a/docs/en/engines/table-engines/integrations/s3queue.md +++ b/docs/en/engines/table-engines/integrations/s3queue.md @@ -28,6 +28,8 @@ CREATE TABLE s3_queue_engine_table (name String, value UInt32) [s3queue_cleanup_interval_max_ms = 30000,] ``` +Starting with `24.7` settings without `s3queue_` prefix are also supported. + **Engine parameters** - `path` — Bucket url with path to file. Supports following wildcards in readonly mode: `*`, `**`, `?`, `{abc,def}` and `{N..M}` where `N`, `M` — numbers, `'abc'`, `'def'` — strings. For more information see [below](#wildcards-in-path). diff --git a/tests/integration/test_storage_s3_queue/test.py b/tests/integration/test_storage_s3_queue/test.py index 463f457e920..61b74db74bb 100644 --- a/tests/integration/test_storage_s3_queue/test.py +++ b/tests/integration/test_storage_s3_queue/test.py @@ -339,7 +339,10 @@ def test_failed_retry(started_cluster, mode, engine_name): values_csv = ( "\n".join((",".join(map(str, row)) for row in values)) + "\n" ).encode() - put_s3_file_content(started_cluster, file_path, values_csv) + if engine_name == "S3Queue": + put_s3_file_content(started_cluster, file_path, values_csv) + else: + put_azure_file_content(started_cluster, file_path, values_csv) create_table( started_cluster, @@ -886,7 +889,7 @@ def test_max_set_age(started_cluster): failed_count = int( node.query( - "SELECT value FROM system.events WHERE name = 'S3QueueFailedFiles' SETTINGS system_events_show_zero_values=1" + "SELECT value FROM system.events WHERE name = 'ObjectStorageQueueFailedFiles' SETTINGS system_events_show_zero_values=1" ) ) @@ -901,7 +904,7 @@ def test_max_set_age(started_cluster): for _ in range(30): if failed_count + 1 == int( node.query( - "SELECT value FROM system.events WHERE name = 'S3QueueFailedFiles' SETTINGS system_events_show_zero_values=1" + "SELECT value FROM system.events WHERE name = 'ObjectStorageQueueFailedFiles' SETTINGS system_events_show_zero_values=1" ) ): break @@ -909,7 +912,7 @@ def test_max_set_age(started_cluster): assert failed_count + 1 == int( node.query( - "SELECT value FROM system.events WHERE name = 'S3QueueFailedFiles' SETTINGS system_events_show_zero_values=1" + "SELECT value FROM system.events WHERE name = 'ObjectStorageQueueFailedFiles' SETTINGS system_events_show_zero_values=1" ) ) @@ -926,7 +929,7 @@ def test_max_set_age(started_cluster): time.sleep(max_age + 1) assert failed_count + 2 == int( - node.query("SELECT value FROM system.events WHERE name = 'S3QueueFailedFiles'") + node.query("SELECT value FROM system.events WHERE name = 'ObjectStorageQueueFailedFiles'") ) node.query("SYSTEM FLUSH LOGS") From da55a213ef1b5a210613c0946c01b06688ab40aa Mon Sep 17 00:00:00 2001 From: robot-clickhouse Date: Thu, 20 Jun 2024 16:38:03 +0000 Subject: [PATCH 038/115] Automatic style fix --- tests/integration/test_storage_s3_queue/test.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/tests/integration/test_storage_s3_queue/test.py b/tests/integration/test_storage_s3_queue/test.py index 61b74db74bb..71886bf9dcd 100644 --- a/tests/integration/test_storage_s3_queue/test.py +++ b/tests/integration/test_storage_s3_queue/test.py @@ -929,7 +929,9 @@ def test_max_set_age(started_cluster): time.sleep(max_age + 1) assert failed_count + 2 == int( - node.query("SELECT value FROM system.events WHERE name = 'ObjectStorageQueueFailedFiles'") + node.query( + "SELECT value FROM system.events WHERE name = 'ObjectStorageQueueFailedFiles'" + ) ) node.query("SYSTEM FLUSH LOGS") From c26e7b506766d8068576e733e9870b4d2a6a0482 Mon Sep 17 00:00:00 2001 From: avogar Date: Thu, 20 Jun 2024 16:56:30 +0000 Subject: [PATCH 039/115] Fix test --- tests/queries/0_stateless/01760_system_dictionaries.sql | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/queries/0_stateless/01760_system_dictionaries.sql b/tests/queries/0_stateless/01760_system_dictionaries.sql index a5609281e49..2e7d4184811 100644 --- a/tests/queries/0_stateless/01760_system_dictionaries.sql +++ b/tests/queries/0_stateless/01760_system_dictionaries.sql @@ -25,8 +25,8 @@ SELECT * FROM 01760_db.example_simple_key_dictionary; SELECT name, database, key.names, key.types, attribute.names, attribute.types, status FROM system.dictionaries WHERE database='01760_db'; -DROP TABLE 01760_db.example_simple_key_source; DROP DICTIONARY 01760_db.example_simple_key_dictionary; +DROP TABLE 01760_db.example_simple_key_source; SELECT name, database, key.names, key.types, attribute.names, attribute.types, status FROM system.dictionaries WHERE database='01760_db'; @@ -53,7 +53,7 @@ SELECT * FROM 01760_db.example_complex_key_dictionary; SELECT name, database, key.names, key.types, attribute.names, attribute.types, status FROM system.dictionaries WHERE database='01760_db'; -DROP TABLE 01760_db.example_complex_key_source; DROP DICTIONARY 01760_db.example_complex_key_dictionary; +DROP TABLE 01760_db.example_complex_key_source; DROP DATABASE 01760_db; From 5447145c7aa2be0333a17066a9591e0d6d5c88a6 Mon Sep 17 00:00:00 2001 From: kssenii Date: Fri, 21 Jun 2024 11:53:01 +0200 Subject: [PATCH 040/115] Fix test --- tests/integration/test_storage_s3_queue/test.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/integration/test_storage_s3_queue/test.py b/tests/integration/test_storage_s3_queue/test.py index 71886bf9dcd..a118c8da52f 100644 --- a/tests/integration/test_storage_s3_queue/test.py +++ b/tests/integration/test_storage_s3_queue/test.py @@ -1475,7 +1475,7 @@ def test_settings_check(started_cluster): ) assert ( - "Existing table metadata in ZooKeeper differs in s3queue_buckets setting. Stored in ZooKeeper: 2, local: 3" + "Existing table metadata in ZooKeeper differs in buckets setting. Stored in ZooKeeper: 2, local: 3" in create_table( started_cluster, node_2, From 0b9e6de6fbfe5ad13d96e7f417479a588f2fde7c Mon Sep 17 00:00:00 2001 From: avogar Date: Fri, 21 Jun 2024 11:47:33 +0000 Subject: [PATCH 041/115] Fix remaining tests --- src/Storages/System/StorageSystemColumns.cpp | 1 - .../test_dictionaries_replace/test.py | 2 + ..._external_tables_memory_tracking.reference | 16 ------ ...52_http_external_tables_memory_tracking.sh | 57 ------------------- ...clic_dependencies_on_create_and_rename.sql | 2 + 5 files changed, 4 insertions(+), 74 deletions(-) delete mode 100644 tests/queries/0_stateless/02152_http_external_tables_memory_tracking.reference delete mode 100755 tests/queries/0_stateless/02152_http_external_tables_memory_tracking.sh diff --git a/src/Storages/System/StorageSystemColumns.cpp b/src/Storages/System/StorageSystemColumns.cpp index a6a791f75b5..49da1eba9ec 100644 --- a/src/Storages/System/StorageSystemColumns.cpp +++ b/src/Storages/System/StorageSystemColumns.cpp @@ -134,7 +134,6 @@ protected: cols_required_for_sorting_key = metadata_snapshot->getColumnsRequiredForSortingKey(); cols_required_for_primary_key = metadata_snapshot->getColumnsRequiredForPrimaryKey(); cols_required_for_sampling = metadata_snapshot->getColumnsRequiredForSampling(); - LOG_DEBUG(getLogger("StorageSystemColumns"), "Get column sizes for table {}.{}", database_name, table_name); column_sizes = storage->getColumnSizes(); } diff --git a/tests/integration/test_dictionaries_replace/test.py b/tests/integration/test_dictionaries_replace/test.py index bf406f46cb1..efb65c15c49 100644 --- a/tests/integration/test_dictionaries_replace/test.py +++ b/tests/integration/test_dictionaries_replace/test.py @@ -134,3 +134,5 @@ def test_create_or_replace(database, instance_to_create_dictionary, instances_to expected_result = TSV([[0, 1], [5, 26], [7, 50], [11, 0]]) assert instance.query(select_query) == expected_result assert instance.query(select_query, user="dictget_user") == expected_result + + instance_to_create_dictionary.query(f"DROP DICTIONARY IF EXISTS {database}.dict") diff --git a/tests/queries/0_stateless/02152_http_external_tables_memory_tracking.reference b/tests/queries/0_stateless/02152_http_external_tables_memory_tracking.reference deleted file mode 100644 index 1fc09c8d154..00000000000 --- a/tests/queries/0_stateless/02152_http_external_tables_memory_tracking.reference +++ /dev/null @@ -1,16 +0,0 @@ -Checking input_format_parallel_parsing=false& -1 -Checking input_format_parallel_parsing=false&cancel_http_readonly_queries_on_client_close=1&readonly=1 -1 -Checking input_format_parallel_parsing=false&send_progress_in_http_headers=true -1 -Checking input_format_parallel_parsing=false&cancel_http_readonly_queries_on_client_close=1&readonly=1&send_progress_in_http_headers=true -1 -Checking input_format_parallel_parsing=true& -1 -Checking input_format_parallel_parsing=true&cancel_http_readonly_queries_on_client_close=1&readonly=1 -1 -Checking input_format_parallel_parsing=true&send_progress_in_http_headers=true -1 -Checking input_format_parallel_parsing=true&cancel_http_readonly_queries_on_client_close=1&readonly=1&send_progress_in_http_headers=true -1 diff --git a/tests/queries/0_stateless/02152_http_external_tables_memory_tracking.sh b/tests/queries/0_stateless/02152_http_external_tables_memory_tracking.sh deleted file mode 100755 index 8aba362d6af..00000000000 --- a/tests/queries/0_stateless/02152_http_external_tables_memory_tracking.sh +++ /dev/null @@ -1,57 +0,0 @@ -#!/usr/bin/env bash -# Tags: no-tsan, no-cpu-aarch64, no-parallel -# TSan does not supports tracing. -# trace_log doesn't work on aarch64 - -# Regression for proper release of Context, -# via tracking memory of external tables. - -CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) -# shellcheck source=../shell_config.sh -. "$CURDIR"/../shell_config.sh - -tmp_file=$(mktemp "$CURDIR/clickhouse.XXXXXX.csv") -trap 'rm $tmp_file' EXIT - -$CLICKHOUSE_CLIENT -q "SELECT toString(number) FROM numbers(1e6) FORMAT TSV" > "$tmp_file" - -function run_and_check() -{ - local query_id - query_id="$(${CLICKHOUSE_CURL} -sS "${CLICKHOUSE_URL}" --data-binary @- <<<'SELECT generateUUIDv4()')" - - echo "Checking $*" - - # Run query with external table (implicit StorageMemory user) - $CLICKHOUSE_CURL -sS -F "s=@$tmp_file;" "$CLICKHOUSE_URL&s_structure=key+Int&query=SELECT+count()+FROM+s&memory_profiler_sample_probability=1&max_untracked_memory=0&query_id=$query_id&$*" -o /dev/null - - ${CLICKHOUSE_CURL} -sS "${CLICKHOUSE_URL}" --data-binary @- <<<'SYSTEM FLUSH LOGS' - - # Check that temporary table had been destroyed. - ${CLICKHOUSE_CURL} -sS "${CLICKHOUSE_URL}&allow_introspection_functions=1" --data-binary @- <<<" - WITH arrayStringConcat(arrayMap(x -> demangle(addressToSymbol(x)), trace), '\n') AS sym - SELECT 1 FROM system.trace_log - PREWHERE - query_id = '$query_id' AND - trace_type = 'MemorySample' AND - /* only deallocations */ - size < 0 AND - event_date >= yesterday() - WHERE - sym LIKE '%DB::StorageMemory::drop%\n%TemporaryTableHolder::~TemporaryTableHolder%' - LIMIT 1 - " -} - -for input_format_parallel_parsing in false true; do - query_args_variants=( - "" - "cancel_http_readonly_queries_on_client_close=1&readonly=1" - "send_progress_in_http_headers=true" - # nested progress callback - "cancel_http_readonly_queries_on_client_close=1&readonly=1&send_progress_in_http_headers=true" - ) - for query_args in "${query_args_variants[@]}"; do - run_and_check "input_format_parallel_parsing=$input_format_parallel_parsing&$query_args" - done -done diff --git a/tests/queries/0_stateless/03173_check_cyclic_dependencies_on_create_and_rename.sql b/tests/queries/0_stateless/03173_check_cyclic_dependencies_on_create_and_rename.sql index e6215c3da1e..0cadd4f5cee 100644 --- a/tests/queries/0_stateless/03173_check_cyclic_dependencies_on_create_and_rename.sql +++ b/tests/queries/0_stateless/03173_check_cyclic_dependencies_on_create_and_rename.sql @@ -1,3 +1,5 @@ +-- Tags: atomic-database + DROP TABLE IF EXISTS test; CREATE TABLE test (id UInt64, value String) ENGINE=MergeTree ORDER BY id; INSERT INTO test SELECT number, 'str_' || toString(number) FROM numbers(10); From 5c541af9953cc1448d5ecab771d6f7884b8abe8a Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Fri, 21 Jun 2024 15:13:56 +0200 Subject: [PATCH 042/115] Do not print jemalloc message at startup --- programs/main.cpp | 9 +++++++++ tests/queries/0_stateless/01505_pipeline_executor_UAF.sh | 2 +- 2 files changed, 10 insertions(+), 1 deletion(-) diff --git a/programs/main.cpp b/programs/main.cpp index c270388f17f..95b75193e21 100644 --- a/programs/main.cpp +++ b/programs/main.cpp @@ -13,6 +13,7 @@ #include +#include "config.h" #include "config_tools.h" #include @@ -439,6 +440,14 @@ extern "C" } #endif +/// Prevent messages from JeMalloc in the release build. +/// Some of these messages are non-actionable for the users, such as: +/// : Number of CPUs detected is not deterministic. Per-CPU arena disabled. +#if USE_JEMALLOC && defined(NDEBUG) && !defined(SANITIZER) +extern "C" void (*malloc_message)(void *, const char *s); +__attribute__((constructor(0))) void init_je_malloc_message() { malloc_message = [](void *, const char *){}; } +#endif + /// This allows to implement assert to forbid initialization of a class in static constructors. /// Usage: /// diff --git a/tests/queries/0_stateless/01505_pipeline_executor_UAF.sh b/tests/queries/0_stateless/01505_pipeline_executor_UAF.sh index c2750ad31b2..35c2b796570 100755 --- a/tests/queries/0_stateless/01505_pipeline_executor_UAF.sh +++ b/tests/queries/0_stateless/01505_pipeline_executor_UAF.sh @@ -14,7 +14,7 @@ for _ in {1..10}; do ${CLICKHOUSE_LOCAL} -q 'select * from numbers_mt(100000000) settings max_threads=100 FORMAT Null' # Binding to specific CPU is not required, but this makes the test more reliable. taskset --cpu-list 0 ${CLICKHOUSE_LOCAL} -q 'select * from numbers_mt(100000000) settings max_threads=100 FORMAT Null' 2>&1 | { - # build with santiziers does not have jemalloc + # build with sanitiziers does not have jemalloc # and for jemalloc we have separate test # 01502_jemalloc_percpu_arena grep -v ': Number of CPUs detected is not deterministic. Per-CPU arena disabled.' From 54c0bdee09d369bfd0b0f92bed898f7161316e25 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Fri, 21 Jun 2024 16:00:20 +0200 Subject: [PATCH 043/115] Fix style --- programs/main.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/programs/main.cpp b/programs/main.cpp index 95b75193e21..61e2bc18ed7 100644 --- a/programs/main.cpp +++ b/programs/main.cpp @@ -444,7 +444,7 @@ extern "C" /// Some of these messages are non-actionable for the users, such as: /// : Number of CPUs detected is not deterministic. Per-CPU arena disabled. #if USE_JEMALLOC && defined(NDEBUG) && !defined(SANITIZER) -extern "C" void (*malloc_message)(void *, const char *s); +extern "C" void (*malloc_message)(void *, const char *s); __attribute__((constructor(0))) void init_je_malloc_message() { malloc_message = [](void *, const char *){}; } #endif From 1957409b4311dc58b2b3501f3d47fddffe3ae725 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Fri, 21 Jun 2024 18:55:49 +0200 Subject: [PATCH 044/115] Fix tests --- .../01103_check_cpu_instructions_at_startup.reference | 2 -- .../0_stateless/01103_check_cpu_instructions_at_startup.sh | 2 +- tests/queries/0_stateless/01502_jemalloc_percpu_arena.reference | 2 -- 3 files changed, 1 insertion(+), 5 deletions(-) diff --git a/tests/queries/0_stateless/01103_check_cpu_instructions_at_startup.reference b/tests/queries/0_stateless/01103_check_cpu_instructions_at_startup.reference index 8984d35930a..03ed07cf1a4 100644 --- a/tests/queries/0_stateless/01103_check_cpu_instructions_at_startup.reference +++ b/tests/queries/0_stateless/01103_check_cpu_instructions_at_startup.reference @@ -2,6 +2,4 @@ Instruction check fail. The CPU does not support SSSE3 instruction set. Instruction check fail. The CPU does not support SSE4.1 instruction set. Instruction check fail. The CPU does not support SSE4.2 instruction set. Instruction check fail. The CPU does not support POPCNT instruction set. -: MADV_DONTNEED does not work (memset will be used instead) -: (This is the expected behaviour if you are running under QEMU) 1 diff --git a/tests/queries/0_stateless/01103_check_cpu_instructions_at_startup.sh b/tests/queries/0_stateless/01103_check_cpu_instructions_at_startup.sh index 01047aeb9ab..c37f1f95374 100755 --- a/tests/queries/0_stateless/01103_check_cpu_instructions_at_startup.sh +++ b/tests/queries/0_stateless/01103_check_cpu_instructions_at_startup.sh @@ -1,6 +1,6 @@ #!/usr/bin/env bash # Tags: no-tsan, no-asan, no-ubsan, no-msan, no-debug, no-fasttest, no-cpu-aarch64 -# Tag no-fasttest: avoid dependency on qemu -- invonvenient when running locally +# Tag no-fasttest: avoid dependency on qemu -- inconvenient when running locally CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) # shellcheck source=../shell_config.sh diff --git a/tests/queries/0_stateless/01502_jemalloc_percpu_arena.reference b/tests/queries/0_stateless/01502_jemalloc_percpu_arena.reference index fe093e39a56..5accb577786 100644 --- a/tests/queries/0_stateless/01502_jemalloc_percpu_arena.reference +++ b/tests/queries/0_stateless/01502_jemalloc_percpu_arena.reference @@ -1,5 +1,3 @@ -: Number of CPUs detected is not deterministic. Per-CPU arena disabled. 1 -: Number of CPUs detected is not deterministic. Per-CPU arena disabled. 100000000 1 From ecabbc293c3d0a2bb1f9d32d1080524d7409d738 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Sat, 22 Jun 2024 00:21:42 +0200 Subject: [PATCH 045/115] Fix test --- tests/queries/0_stateless/01502_jemalloc_percpu_arena.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/queries/0_stateless/01502_jemalloc_percpu_arena.sh b/tests/queries/0_stateless/01502_jemalloc_percpu_arena.sh index b3ea6eca3f4..c1bd1e0e1fa 100755 --- a/tests/queries/0_stateless/01502_jemalloc_percpu_arena.sh +++ b/tests/queries/0_stateless/01502_jemalloc_percpu_arena.sh @@ -1,5 +1,5 @@ #!/usr/bin/env bash -# Tags: no-tsan, no-asan, no-msan, no-ubsan, no-fasttest +# Tags: no-tsan, no-asan, no-msan, no-ubsan, no-fasttest, no-debug # ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ # NOTE: jemalloc is disabled under sanitizers From 312ef9b3b95a00e82fd7330f0805c331e136031e Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Sat, 22 Jun 2024 20:29:01 +0200 Subject: [PATCH 046/115] Maybe fix test 00763_lock_buffer_long.sh --- tests/queries/0_stateless/00763_lock_buffer_long.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/queries/0_stateless/00763_lock_buffer_long.sh b/tests/queries/0_stateless/00763_lock_buffer_long.sh index 50680724149..79c35e99486 100755 --- a/tests/queries/0_stateless/00763_lock_buffer_long.sh +++ b/tests/queries/0_stateless/00763_lock_buffer_long.sh @@ -1,5 +1,5 @@ #!/usr/bin/env bash -# Tags: long +# Tags: long, no-s3-storage set -e @@ -15,7 +15,7 @@ ${CLICKHOUSE_CLIENT} --query="CREATE TABLE buffer_00763_2 (s String) ENGINE = Bu function thread1() { - seq 1 500 | sed -r -e 's/.+/DROP TABLE IF EXISTS mt_00763_2; CREATE TABLE mt_00763_2 (s String) ENGINE = MergeTree ORDER BY s; INSERT INTO mt_00763_2 SELECT toString(number) FROM numbers(10);/' | ${CLICKHOUSE_CLIENT} --multiquery --ignore-error ||: + seq 1 500 | sed -r -e 's/.+/DROP TABLE IF EXISTS mt_00763_2; CREATE TABLE mt_00763_2 (s String) ENGINE = MergeTree ORDER BY s; INSERT INTO mt_00763_2 SELECT toString(number) FROM numbers(10);/' | ${CLICKHOUSE_CLIENT} --fsync-metadata 0 --multiquery --ignore-error ||: } function thread2() From dbdf4e18804f6b430df051d8fd8b7ad84caeae93 Mon Sep 17 00:00:00 2001 From: Alexander Tokmakov Date: Sun, 23 Jun 2024 01:58:28 +0200 Subject: [PATCH 047/115] improve ZooKeeper load balancing --- src/Common/GetPriorityForLoadBalancing.cpp | 19 ++ src/Common/GetPriorityForLoadBalancing.h | 9 + src/Common/ZooKeeper/IKeeper.h | 6 +- src/Common/ZooKeeper/TestKeeper.h | 1 - src/Common/ZooKeeper/ZooKeeper.cpp | 229 ++++++++++++++---- src/Common/ZooKeeper/ZooKeeper.h | 31 ++- src/Common/ZooKeeper/ZooKeeperArgs.cpp | 13 + src/Common/ZooKeeper/ZooKeeperArgs.h | 3 + src/Common/ZooKeeper/ZooKeeperImpl.cpp | 97 +++----- src/Common/ZooKeeper/ZooKeeperImpl.h | 8 +- src/IO/S3/Credentials.cpp | 15 ++ src/IO/S3/Credentials.h | 1 + src/Interpreters/Context.cpp | 2 - .../StorageSystemZooKeeperConnection.cpp | 4 +- .../configs/zookeeper_load_balancing2.xml | 29 +++ .../test.py | 36 +++ 16 files changed, 380 insertions(+), 123 deletions(-) create mode 100644 tests/integration/test_zookeeper_config_load_balancing/configs/zookeeper_load_balancing2.xml diff --git a/src/Common/GetPriorityForLoadBalancing.cpp b/src/Common/GetPriorityForLoadBalancing.cpp index d4c6f89ff92..83197f2365c 100644 --- a/src/Common/GetPriorityForLoadBalancing.cpp +++ b/src/Common/GetPriorityForLoadBalancing.cpp @@ -60,4 +60,23 @@ GetPriorityForLoadBalancing::getPriorityFunc(LoadBalancing load_balance, size_t return get_priority; } +bool GetPriorityForLoadBalancing::hasOptimalNode() const +{ + switch (load_balancing) + { + case LoadBalancing::NEAREST_HOSTNAME: + return true; + case LoadBalancing::HOSTNAME_LEVENSHTEIN_DISTANCE: + return true; + case LoadBalancing::IN_ORDER: + return false; + case LoadBalancing::RANDOM: + return false; + case LoadBalancing::FIRST_OR_RANDOM: + return true; + case LoadBalancing::ROUND_ROBIN: + return false; + } +} + } diff --git a/src/Common/GetPriorityForLoadBalancing.h b/src/Common/GetPriorityForLoadBalancing.h index 0de99730977..b635dea6c93 100644 --- a/src/Common/GetPriorityForLoadBalancing.h +++ b/src/Common/GetPriorityForLoadBalancing.h @@ -1,6 +1,7 @@ #pragma once #include +#include namespace DB { @@ -13,6 +14,7 @@ public: explicit GetPriorityForLoadBalancing(LoadBalancing load_balancing_, size_t last_used_ = 0) : load_balancing(load_balancing_), last_used(last_used_) { + saved_offset = thread_local_rng(); } GetPriorityForLoadBalancing() = default; @@ -29,6 +31,12 @@ public: } Func getPriorityFunc(LoadBalancing load_balance, size_t offset, size_t pool_size) const; + Func getPriorityFunc(size_t pool_size) const + { + return getPriorityFunc(load_balancing, saved_offset % pool_size, pool_size); + } + + bool hasOptimalNode() const; std::vector hostname_prefix_distance; /// Prefix distances from name of this host to the names of hosts of pools. std::vector hostname_levenshtein_distance; /// Levenshtein Distances from name of this host to the names of hosts of pools. @@ -37,6 +45,7 @@ public: private: mutable size_t last_used = 0; /// Last used for round_robin policy. + size_t saved_offset; /// Default random offset for round_robin policy. }; } diff --git a/src/Common/ZooKeeper/IKeeper.h b/src/Common/ZooKeeper/IKeeper.h index 7d574247aa5..2c6cbc4a5d5 100644 --- a/src/Common/ZooKeeper/IKeeper.h +++ b/src/Common/ZooKeeper/IKeeper.h @@ -559,6 +559,8 @@ public: /// Useful to check owner of ephemeral node. virtual int64_t getSessionID() const = 0; + virtual String tryGetAvailabilityZone() { return ""; } + /// If the method will throw an exception, callbacks won't be called. /// /// After the method is executed successfully, you must wait for callbacks @@ -635,10 +637,6 @@ public: virtual const DB::KeeperFeatureFlags * getKeeperFeatureFlags() const { return nullptr; } - /// A ZooKeeper session can have an optional deadline set on it. - /// After it has been reached, the session needs to be finalized. - virtual bool hasReachedDeadline() const = 0; - /// Expire session and finish all pending requests virtual void finalize(const String & reason) = 0; }; diff --git a/src/Common/ZooKeeper/TestKeeper.h b/src/Common/ZooKeeper/TestKeeper.h index 2774055652c..2194ad015bf 100644 --- a/src/Common/ZooKeeper/TestKeeper.h +++ b/src/Common/ZooKeeper/TestKeeper.h @@ -39,7 +39,6 @@ public: ~TestKeeper() override; bool isExpired() const override { return expired; } - bool hasReachedDeadline() const override { return false; } Int8 getConnectedNodeIdx() const override { return 0; } String getConnectedHostPort() const override { return "TestKeeper:0000"; } int32_t getConnectionXid() const override { return 0; } diff --git a/src/Common/ZooKeeper/ZooKeeper.cpp b/src/Common/ZooKeeper/ZooKeeper.cpp index 4ec44a39136..bc3f29302e3 100644 --- a/src/Common/ZooKeeper/ZooKeeper.cpp +++ b/src/Common/ZooKeeper/ZooKeeper.cpp @@ -8,6 +8,7 @@ #include #include #include +#include #include #include @@ -16,10 +17,12 @@ #include #include #include +#include #include "Common/ZooKeeper/IKeeper.h" #include #include #include +#include #include #include @@ -55,70 +58,174 @@ static void check(Coordination::Error code, const std::string & path) throw KeeperException::fromPath(code, path); } +UInt64 getSecondsUntilReconnect(const ZooKeeperArgs & args) +{ + std::uniform_int_distribution fallback_session_lifetime_distribution + { + args.fallback_session_lifetime.min_sec, + args.fallback_session_lifetime.max_sec, + }; + UInt32 session_lifetime_seconds = fallback_session_lifetime_distribution(thread_local_rng); + return session_lifetime_seconds; +} -void ZooKeeper::init(ZooKeeperArgs args_) +Coordination::ZooKeeper::Node hostToNode(const LoggerPtr & log, const ShuffleHost & host) +{ + /// We want to resolve all hosts without DNS cache for keeper connection. + Coordination::DNSResolver::instance().removeHostFromCache(host.host); + const Poco::Net::SocketAddress host_socket_addr{host.host}; + LOG_TEST(log, "Adding ZooKeeper host {} ({})", host.host, host_socket_addr.toString()); + return Coordination::ZooKeeper::Node{host_socket_addr, host.original_index, host.secure}; +} + +/// TODO get rid of this, converting "host" to "node" is stupid +Coordination::ZooKeeper::Nodes hostsToNodes(const LoggerPtr & log, std::vector & shuffled_hosts) +{ + Coordination::ZooKeeper::Nodes nodes; + + bool dns_error = false; + for (auto & host : shuffled_hosts) + { + auto & host_string = host.host; + try + { + host.secure = startsWith(host_string, "secure://"); + + if (host.secure) + host_string.erase(0, strlen("secure://")); + + nodes.emplace_back(hostToNode(log, host)); + } + catch (const Poco::Net::HostNotFoundException & e) + { + /// Most likely it's misconfiguration and wrong hostname was specified + LOG_ERROR(log, "Cannot use ZooKeeper host {}, reason: {}", host_string, e.displayText()); + } + catch (const Poco::Net::DNSException & e) + { + /// Most likely DNS is not available now + dns_error = true; + LOG_ERROR(log, "Cannot use ZooKeeper host {} due to DNS error: {}", host_string, e.displayText()); + } + } + + if (nodes.empty()) + { + /// For DNS errors we throw exception with ZCONNECTIONLOSS code, so it will be considered as hardware error, not user error + if (dns_error) + throw KeeperException::fromMessage( + Coordination::Error::ZCONNECTIONLOSS, "Cannot resolve any of provided ZooKeeper hosts due to DNS error"); + else + throw KeeperException::fromMessage(Coordination::Error::ZCONNECTIONLOSS, "Cannot use any of provided ZooKeeper nodes"); + } + + return nodes; +} + + +void ZooKeeper::updateAvailabilityZones() +{ + std::vector shuffled_hosts = shuffleHosts(); + Coordination::ZooKeeper::Nodes nodes = hostsToNodes(log, shuffled_hosts); + + for (const auto & node : nodes) + { + try + { + Coordination::ZooKeeper::Nodes single_node{node}; + auto tmp_impl = std::make_unique(single_node, args, zk_log); + auto idx = node.original_index; + availability_zones[idx] = tmp_impl->tryGetAvailabilityZone(); + LOG_DEBUG(log, "Got availability zone for {}: {}", args.hosts[idx], availability_zones[idx]); + } + catch (...) + { + DB::tryLogCurrentException(log, "Failed to get availability zone for " + node.address.toString()); + } + } +} + +void ZooKeeper::init(ZooKeeperArgs args_, std::unique_ptr existing_impl) { args = std::move(args_); log = getLogger("ZooKeeper"); - if (args.implementation == "zookeeper") + if (existing_impl) + { + chassert(args.implementation == "zookeeper"); + impl = std::move(existing_impl); + LOG_INFO(log, "Switching to connection to a more optimal node {}", impl->getConnectedHostPort()); + } + else if (args.implementation == "zookeeper") { if (args.hosts.empty()) throw KeeperException::fromMessage(Coordination::Error::ZBADARGUMENTS, "No hosts passed to ZooKeeper constructor."); - Coordination::ZooKeeper::Nodes nodes; - nodes.reserve(args.hosts.size()); + chassert(args.availability_zones.size() == args.hosts.size()); + if (availability_zones.empty()) + { + /// availability_zones is empty on server startup or after config reloading + /// We will keep the az info when starting new sessions + availability_zones = args.availability_zones; + if (args.availability_zone_autodetect) + updateAvailabilityZones(); + } /// Shuffle the hosts to distribute the load among ZooKeeper nodes. std::vector shuffled_hosts = shuffleHosts(); - - bool dns_error = false; - for (auto & host : shuffled_hosts) - { - auto & host_string = host.host; - try - { - const bool secure = startsWith(host_string, "secure://"); - - if (secure) - host_string.erase(0, strlen("secure://")); - - /// We want to resolve all hosts without DNS cache for keeper connection. - Coordination::DNSResolver::instance().removeHostFromCache(host_string); - - const Poco::Net::SocketAddress host_socket_addr{host_string}; - LOG_TEST(log, "Adding ZooKeeper host {} ({})", host_string, host_socket_addr.toString()); - nodes.emplace_back(Coordination::ZooKeeper::Node{host_socket_addr, host.original_index, secure}); - } - catch (const Poco::Net::HostNotFoundException & e) - { - /// Most likely it's misconfiguration and wrong hostname was specified - LOG_ERROR(log, "Cannot use ZooKeeper host {}, reason: {}", host_string, e.displayText()); - } - catch (const Poco::Net::DNSException & e) - { - /// Most likely DNS is not available now - dns_error = true; - LOG_ERROR(log, "Cannot use ZooKeeper host {} due to DNS error: {}", host_string, e.displayText()); - } - } - - if (nodes.empty()) - { - /// For DNS errors we throw exception with ZCONNECTIONLOSS code, so it will be considered as hardware error, not user error - if (dns_error) - throw KeeperException::fromMessage(Coordination::Error::ZCONNECTIONLOSS, "Cannot resolve any of provided ZooKeeper hosts due to DNS error"); - else - throw KeeperException::fromMessage(Coordination::Error::ZCONNECTIONLOSS, "Cannot use any of provided ZooKeeper nodes"); - } + Coordination::ZooKeeper::Nodes nodes = hostsToNodes(log, shuffled_hosts); impl = std::make_unique(nodes, args, zk_log); + Int8 node_idx = impl->getConnectedNodeIdx(); if (args.chroot.empty()) LOG_TRACE(log, "Initialized, hosts: {}", fmt::join(args.hosts, ",")); else LOG_TRACE(log, "Initialized, hosts: {}, chroot: {}", fmt::join(args.hosts, ","), args.chroot); + + + /// If the balancing strategy has an optimal node then it will be the first in the list + bool connected_to_suboptimal_node = node_idx != shuffled_hosts[0].original_index; + bool respect_az = !args.client_availability_zone.empty(); + bool may_benefit_from_reconnecting = respect_az || args.get_priority_load_balancing.hasOptimalNode(); + if (connected_to_suboptimal_node && may_benefit_from_reconnecting) + { + auto reconnect_timeout_sec = getSecondsUntilReconnect(args); + LOG_DEBUG(log, "Connected to a suboptimal ZooKeeper host ({}, index {})." + " To preserve balance in ZooKeeper usage, this ZooKeeper session will expire in {} seconds", + impl->getConnectedHostPort(), node_idx, reconnect_timeout_sec); + + auto reconnect_task_holder = DB::Context::getGlobalContextInstance()->getSchedulePool().createTask("ZKReconnect", [this, optimal_host = shuffled_hosts[0]]() + { + try + { + LOG_DEBUG(log, "Trying to connect to a more optimal node {}", optimal_host.host); + Coordination::ZooKeeper::Nodes node; + node.emplace_back(hostToNode(log, optimal_host)); + std::unique_ptr new_impl = std::make_unique(node, args, zk_log); + Int8 new_node_idx = new_impl->getConnectedNodeIdx(); + + /// Maybe the node was unavailable when getting AZs first time, update just in case + if (args.availability_zone_autodetect) + { + availability_zones[new_node_idx] = new_impl->tryGetAvailabilityZone(); + LOG_DEBUG(log, "Got availability zone for {}: {}", optimal_host.host, availability_zones[new_node_idx]); + } + + optimal_impl = std::move(new_impl); + impl->finalize("Connected to a more optimal node"); + } + catch (...) + { + LOG_WARNING(log, "Failed to connect to a more optimal ZooKeeper, will try again later: {}", DB::getCurrentExceptionMessage(/*with_stacktrace*/ false)); + (*reconnect_task)->scheduleAfter(getSecondsUntilReconnect(args) * 1000); + } + }); + reconnect_task = std::make_unique(std::move(reconnect_task_holder)); + (*reconnect_task)->activate(); + (*reconnect_task)->scheduleAfter(reconnect_timeout_sec * 1000); + } } else if (args.implementation == "testkeeper") { @@ -152,29 +259,47 @@ void ZooKeeper::init(ZooKeeperArgs args_) } } +ZooKeeper::~ZooKeeper() +{ + if (reconnect_task) + (*reconnect_task)->deactivate(); +} ZooKeeper::ZooKeeper(const ZooKeeperArgs & args_, std::shared_ptr zk_log_) : zk_log(std::move(zk_log_)) { - init(args_); + init(args_, /*existing_impl*/ {}); +} + + +ZooKeeper::ZooKeeper(const ZooKeeperArgs & args_, std::shared_ptr zk_log_, Strings availability_zones_, std::unique_ptr existing_impl) + : availability_zones(std::move(availability_zones_)), zk_log(std::move(zk_log_)) +{ + if (availability_zones.size() != args_.hosts.size()) + throw DB::Exception(DB::ErrorCodes::LOGICAL_ERROR, "Argument sizes mismatch: {} and {}", availability_zones.size(), args_.hosts.size()); + init(args_, std::move(existing_impl)); } ZooKeeper::ZooKeeper(const Poco::Util::AbstractConfiguration & config, const std::string & config_name, std::shared_ptr zk_log_) : zk_log(std::move(zk_log_)) { - init(ZooKeeperArgs(config, config_name)); + init(ZooKeeperArgs(config, config_name), /*existing_impl*/ {}); } std::vector ZooKeeper::shuffleHosts() const { - std::function get_priority = args.get_priority_load_balancing.getPriorityFunc(args.get_priority_load_balancing.load_balancing, 0, args.hosts.size()); + std::function get_priority = args.get_priority_load_balancing.getPriorityFunc(args.hosts.size()); std::vector shuffle_hosts; for (size_t i = 0; i < args.hosts.size(); ++i) { ShuffleHost shuffle_host; shuffle_host.host = args.hosts[i]; shuffle_host.original_index = static_cast(i); + + if (!availability_zones[i].empty()) + shuffle_host.az_info = availability_zones[i] == args.client_availability_zone ? ShuffleHost::SAME : ShuffleHost::OTHER; + if (get_priority) shuffle_host.priority = get_priority(i); shuffle_host.randomize(); @@ -1023,7 +1148,10 @@ ZooKeeperPtr ZooKeeper::create(const Poco::Util::AbstractConfiguration & config, ZooKeeperPtr ZooKeeper::startNewSession() const { - auto res = std::shared_ptr(new ZooKeeper(args, zk_log)); + if (reconnect_task) + (*reconnect_task)->deactivate(); + + auto res = std::shared_ptr(new ZooKeeper(args, zk_log, availability_zones, std::move(optimal_impl))); res->initSession(); return res; } @@ -1456,6 +1584,11 @@ int32_t ZooKeeper::getConnectionXid() const return impl->getConnectionXid(); } +String ZooKeeper::getConnectedHostAvailabilityZone() const +{ + return availability_zones.at(impl->getConnectedNodeIdx()); +} + size_t getFailedOpIndex(Coordination::Error exception_code, const Coordination::Responses & responses) { if (responses.empty()) diff --git a/src/Common/ZooKeeper/ZooKeeper.h b/src/Common/ZooKeeper/ZooKeeper.h index 08ff60a80cf..3915d884351 100644 --- a/src/Common/ZooKeeper/ZooKeeper.h +++ b/src/Common/ZooKeeper/ZooKeeper.h @@ -32,6 +32,7 @@ namespace DB { class ZooKeeperLog; class ZooKeeperWithFaultInjection; +class BackgroundSchedulePoolTaskHolder; namespace ErrorCodes { @@ -48,8 +49,17 @@ constexpr size_t MULTI_BATCH_SIZE = 100; struct ShuffleHost { + enum AvailabilityZoneInfo + { + SAME = 0, + UNKNOWN = 1, + OTHER = 2, + }; + String host; + bool secure = false; UInt8 original_index = 0; + AvailabilityZoneInfo az_info = UNKNOWN; Priority priority; UInt64 random = 0; @@ -60,8 +70,8 @@ struct ShuffleHost static bool compare(const ShuffleHost & lhs, const ShuffleHost & rhs) { - return std::forward_as_tuple(lhs.priority, lhs.random) - < std::forward_as_tuple(rhs.priority, rhs.random); + return std::forward_as_tuple(lhs.az_info, lhs.priority, lhs.random) + < std::forward_as_tuple(lhs.az_info, rhs.priority, rhs.random); } }; @@ -197,6 +207,9 @@ class ZooKeeper explicit ZooKeeper(const ZooKeeperArgs & args_, std::shared_ptr zk_log_ = nullptr); + /// Allows to keep info about availability zones when starting a new session + ZooKeeper(const ZooKeeperArgs & args_, std::shared_ptr zk_log_, Strings availability_zones_, std::unique_ptr existing_impl); + /** Config of the form: @@ -228,6 +241,8 @@ public: using Ptr = std::shared_ptr; using ErrorsList = std::initializer_list; + ~ZooKeeper(); + std::vector shuffleHosts() const; static Ptr create(const Poco::Util::AbstractConfiguration & config, @@ -596,8 +611,6 @@ public: UInt32 getSessionUptime() const { return static_cast(session_uptime.elapsedSeconds()); } - bool hasReachedDeadline() const { return impl->hasReachedDeadline(); } - uint64_t getSessionTimeoutMS() const { return args.session_timeout_ms; } void setServerCompletelyStarted(); @@ -606,6 +619,8 @@ public: String getConnectedHostPort() const; int32_t getConnectionXid() const; + String getConnectedHostAvailabilityZone() const; + const DB::KeeperFeatureFlags * getKeeperFeatureFlags() const { return impl->getKeeperFeatureFlags(); } /// Checks that our session was not killed, and allows to avoid applying a request from an old lost session. @@ -625,7 +640,8 @@ public: void addCheckSessionOp(Coordination::Requests & requests) const; private: - void init(ZooKeeperArgs args_); + void init(ZooKeeperArgs args_, std::unique_ptr existing_impl); + void updateAvailabilityZones(); /// The following methods don't any throw exceptions but return error codes. Coordination::Error createImpl(const std::string & path, const std::string & data, int32_t mode, std::string & path_created); @@ -690,15 +706,20 @@ private: } std::unique_ptr impl; + mutable std::unique_ptr optimal_impl; ZooKeeperArgs args; + Strings availability_zones; + LoggerPtr log = nullptr; std::shared_ptr zk_log; AtomicStopwatch session_uptime; int32_t session_node_version; + + std::unique_ptr reconnect_task; }; diff --git a/src/Common/ZooKeeper/ZooKeeperArgs.cpp b/src/Common/ZooKeeper/ZooKeeperArgs.cpp index a581b6a7f38..8f8f7cc1b7f 100644 --- a/src/Common/ZooKeeper/ZooKeeperArgs.cpp +++ b/src/Common/ZooKeeper/ZooKeeperArgs.cpp @@ -5,6 +5,7 @@ #include #include #include +#include #include namespace DB @@ -144,6 +145,7 @@ void ZooKeeperArgs::initFromKeeperSection(const Poco::Util::AbstractConfiguratio hosts.push_back( (config.getBool(config_name + "." + key + ".secure", false) ? "secure://" : "") + config.getString(config_name + "." + key + ".host") + ":" + config.getString(config_name + "." + key + ".port", "2181")); + availability_zones.push_back(config.getString(config_name + "." + key + ".availability_zone", "")); } else if (key == "session_timeout_ms") { @@ -199,6 +201,10 @@ void ZooKeeperArgs::initFromKeeperSection(const Poco::Util::AbstractConfiguratio { sessions_path = config.getString(config_name + "." + key); } + else if (key == "client_availability_zone") + { + client_availability_zone = config.getString(config_name + "." + key); + } else if (key == "implementation") { implementation = config.getString(config_name + "." + key); @@ -224,9 +230,16 @@ void ZooKeeperArgs::initFromKeeperSection(const Poco::Util::AbstractConfiguratio { use_compression = config.getBool(config_name + "." + key); } + else if (key == "availability_zone_autodetect") + { + availability_zone_autodetect = config.getBool(config_name + "." + key); + } else throw KeeperException(Coordination::Error::ZBADARGUMENTS, "Unknown key {} in config file", key); } + + if (availability_zone_autodetect) + client_availability_zone = DB::S3::tryGetRunningAvailabilityZone(); } } diff --git a/src/Common/ZooKeeper/ZooKeeperArgs.h b/src/Common/ZooKeeper/ZooKeeperArgs.h index 27ba173c0c3..cd7a5ddb7ce 100644 --- a/src/Common/ZooKeeper/ZooKeeperArgs.h +++ b/src/Common/ZooKeeper/ZooKeeperArgs.h @@ -32,10 +32,12 @@ struct ZooKeeperArgs String zookeeper_name = "zookeeper"; String implementation = "zookeeper"; Strings hosts; + Strings availability_zones; String auth_scheme; String identity; String chroot; String sessions_path = "/clickhouse/sessions"; + String client_availability_zone; int32_t connection_timeout_ms = Coordination::DEFAULT_CONNECTION_TIMEOUT_MS; int32_t session_timeout_ms = Coordination::DEFAULT_SESSION_TIMEOUT_MS; int32_t operation_timeout_ms = Coordination::DEFAULT_OPERATION_TIMEOUT_MS; @@ -47,6 +49,7 @@ struct ZooKeeperArgs UInt64 send_sleep_ms = 0; UInt64 recv_sleep_ms = 0; bool use_compression = false; + bool availability_zone_autodetect = false; SessionLifetimeConfiguration fallback_session_lifetime = {}; DB::GetPriorityForLoadBalancing get_priority_load_balancing; diff --git a/src/Common/ZooKeeper/ZooKeeperImpl.cpp b/src/Common/ZooKeeper/ZooKeeperImpl.cpp index ed7498b1ac9..61f17148d72 100644 --- a/src/Common/ZooKeeper/ZooKeeperImpl.cpp +++ b/src/Common/ZooKeeper/ZooKeeperImpl.cpp @@ -498,22 +498,6 @@ void ZooKeeper::connect( } original_index = static_cast(node.original_index); - - if (i != 0) - { - std::uniform_int_distribution fallback_session_lifetime_distribution - { - args.fallback_session_lifetime.min_sec, - args.fallback_session_lifetime.max_sec, - }; - UInt32 session_lifetime_seconds = fallback_session_lifetime_distribution(thread_local_rng); - client_session_deadline = clock::now() + std::chrono::seconds(session_lifetime_seconds); - - LOG_DEBUG(log, "Connected to a suboptimal ZooKeeper host ({}, index {})." - " To preserve balance in ZooKeeper usage, this ZooKeeper session will expire in {} seconds", - node.address.toString(), i, session_lifetime_seconds); - } - break; } catch (...) @@ -1153,7 +1137,6 @@ void ZooKeeper::pushRequest(RequestInfo && info) { try { - checkSessionDeadline(); info.time = clock::now(); auto maybe_zk_log = std::atomic_load(&zk_log); if (maybe_zk_log) @@ -1201,44 +1184,44 @@ bool ZooKeeper::isFeatureEnabled(KeeperFeatureFlag feature_flag) const return keeper_feature_flags.isEnabled(feature_flag); } -void ZooKeeper::initFeatureFlags() +std::optional ZooKeeper::tryGetSystemZnode(const std::string & path, const std::string & description) { - const auto try_get = [&](const std::string & path, const std::string & description) -> std::optional + auto promise = std::make_shared>(); + auto future = promise->get_future(); + + auto callback = [promise](const Coordination::GetResponse & response) mutable { - auto promise = std::make_shared>(); - auto future = promise->get_future(); - - auto callback = [promise](const Coordination::GetResponse & response) mutable - { - promise->set_value(response); - }; - - get(path, std::move(callback), {}); - if (future.wait_for(std::chrono::milliseconds(args.operation_timeout_ms)) != std::future_status::ready) - throw Exception(Error::ZOPERATIONTIMEOUT, "Failed to get {}: timeout", description); - - auto response = future.get(); - - if (response.error == Coordination::Error::ZNONODE) - { - LOG_TRACE(log, "Failed to get {}", description); - return std::nullopt; - } - else if (response.error != Coordination::Error::ZOK) - { - throw Exception(response.error, "Failed to get {}", description); - } - - return std::move(response.data); + promise->set_value(response); }; - if (auto feature_flags = try_get(keeper_api_feature_flags_path, "feature flags"); feature_flags.has_value()) + get(path, std::move(callback), {}); + if (future.wait_for(std::chrono::milliseconds(args.operation_timeout_ms)) != std::future_status::ready) + throw Exception(Error::ZOPERATIONTIMEOUT, "Failed to get {}: timeout", description); + + auto response = future.get(); + + if (response.error == Coordination::Error::ZNONODE) + { + LOG_TRACE(log, "Failed to get {}", description); + return std::nullopt; + } + else if (response.error != Coordination::Error::ZOK) + { + throw Exception(response.error, "Failed to get {}", description); + } + + return std::move(response.data); +} + +void ZooKeeper::initFeatureFlags() +{ + if (auto feature_flags = tryGetSystemZnode(keeper_api_feature_flags_path, "feature flags"); feature_flags.has_value()) { keeper_feature_flags.setFeatureFlags(std::move(*feature_flags)); return; } - auto keeper_api_version_string = try_get(keeper_api_version_path, "API version"); + auto keeper_api_version_string = tryGetSystemZnode(keeper_api_version_path, "API version"); DB::KeeperApiVersion keeper_api_version{DB::KeeperApiVersion::ZOOKEEPER_COMPATIBLE}; @@ -1256,6 +1239,17 @@ void ZooKeeper::initFeatureFlags() keeper_feature_flags.fromApiVersion(keeper_api_version); } +String ZooKeeper::tryGetAvailabilityZone() +{ + auto res = tryGetSystemZnode(keeper_availability_zone_path, "availability zone"); + if (res) + { + LOG_TRACE(log, "Availability zone for ZooKeeper at {}: {}", getConnectedHostPort(), *res); + return *res; + } + return ""; +} + void ZooKeeper::executeGenericRequest( const ZooKeeperRequestPtr & request, @@ -1587,17 +1581,6 @@ void ZooKeeper::setupFaultDistributions() inject_setup.test_and_set(); } -void ZooKeeper::checkSessionDeadline() const -{ - if (unlikely(hasReachedDeadline())) - throw Exception::fromMessage(Error::ZSESSIONEXPIRED, "Session expired (force expiry client-side)"); -} - -bool ZooKeeper::hasReachedDeadline() const -{ - return client_session_deadline.has_value() && clock::now() >= client_session_deadline.value(); -} - void ZooKeeper::maybeInjectSendFault() { if (unlikely(inject_setup.test() && send_inject_fault && send_inject_fault.value()(thread_local_rng))) diff --git a/src/Common/ZooKeeper/ZooKeeperImpl.h b/src/Common/ZooKeeper/ZooKeeperImpl.h index 8fdf0f97d9d..b42223f0462 100644 --- a/src/Common/ZooKeeper/ZooKeeperImpl.h +++ b/src/Common/ZooKeeper/ZooKeeperImpl.h @@ -130,9 +130,7 @@ public: String getConnectedHostPort() const override { return (original_index == -1) ? "" : args.hosts[original_index]; } int32_t getConnectionXid() const override { return next_xid.load(); } - /// A ZooKeeper session can have an optional deadline set on it. - /// After it has been reached, the session needs to be finalized. - bool hasReachedDeadline() const override; + String tryGetAvailabilityZone() override; /// Useful to check owner of ephemeral node. int64_t getSessionID() const override { return session_id; } @@ -271,7 +269,6 @@ private: clock::time_point time; }; - std::optional client_session_deadline {}; using RequestsQueue = ConcurrentBoundedQueue; RequestsQueue requests_queue{1024}; @@ -346,9 +343,10 @@ private: void logOperationIfNeeded(const ZooKeeperRequestPtr & request, const ZooKeeperResponsePtr & response = nullptr, bool finalize = false, UInt64 elapsed_microseconds = 0); + std::optional tryGetSystemZnode(const std::string & path, const std::string & description); + void initFeatureFlags(); - void checkSessionDeadline() const; CurrentMetrics::Increment active_session_metric_increment{CurrentMetrics::ZooKeeperSession}; std::shared_ptr zk_log; diff --git a/src/IO/S3/Credentials.cpp b/src/IO/S3/Credentials.cpp index fa9d018eaa6..dfb7727fca4 100644 --- a/src/IO/S3/Credentials.cpp +++ b/src/IO/S3/Credentials.cpp @@ -9,6 +9,21 @@ namespace ErrorCodes extern const int UNSUPPORTED_METHOD; } +namespace S3 +{ + std::string tryGetRunningAvailabilityZone() + { + try + { + return getRunningAvailabilityZone(); + } + catch (...) + { + tryLogCurrentException("tryGetRunningAvailabilityZone"); + return ""; + } + } +} } #if USE_AWS_S3 diff --git a/src/IO/S3/Credentials.h b/src/IO/S3/Credentials.h index b8698d9b302..7e14193647f 100644 --- a/src/IO/S3/Credentials.h +++ b/src/IO/S3/Credentials.h @@ -24,6 +24,7 @@ static inline constexpr char GCP_METADATA_SERVICE_ENDPOINT[] = "http://metadata. /// getRunningAvailabilityZone returns the availability zone of the underlying compute resources where the current process runs. std::string getRunningAvailabilityZone(); +std::string tryGetRunningAvailabilityZone(); class AWSEC2MetadataClient : public Aws::Internal::AWSHttpResourceClient { diff --git a/src/Interpreters/Context.cpp b/src/Interpreters/Context.cpp index 2807807b294..779f95b57ac 100644 --- a/src/Interpreters/Context.cpp +++ b/src/Interpreters/Context.cpp @@ -3396,8 +3396,6 @@ zkutil::ZooKeeperPtr Context::getZooKeeper() const const auto & config = shared->zookeeper_config ? *shared->zookeeper_config : getConfigRef(); if (!shared->zookeeper) shared->zookeeper = zkutil::ZooKeeper::create(config, zkutil::getZooKeeperConfigName(config), getZooKeeperLog()); - else if (shared->zookeeper->hasReachedDeadline()) - shared->zookeeper->finalize("ZooKeeper session has reached its deadline"); if (shared->zookeeper->expired()) { diff --git a/src/Storages/System/StorageSystemZooKeeperConnection.cpp b/src/Storages/System/StorageSystemZooKeeperConnection.cpp index 950e20512c0..ec29b84dac3 100644 --- a/src/Storages/System/StorageSystemZooKeeperConnection.cpp +++ b/src/Storages/System/StorageSystemZooKeeperConnection.cpp @@ -36,7 +36,8 @@ ColumnsDescription StorageSystemZooKeeperConnection::getColumnsDescription() /* 9 */ {"xid", std::make_shared(), "XID of the current session."}, /* 10*/ {"enabled_feature_flags", std::make_shared(std::move(feature_flags_enum)), "Feature flags which are enabled. Only applicable to ClickHouse Keeper." - } + }, + /* 11*/ {"availability_zone", std::make_shared(), "Availability zone"}, }; } @@ -85,6 +86,7 @@ void StorageSystemZooKeeperConnection::fillData(MutableColumns & res_columns, Co columns[8]->insert(zookeeper->getClientID()); columns[9]->insert(zookeeper->getConnectionXid()); add_enabled_feature_flags(zookeeper); + columns[11]->insert(zookeeper->getConnectedHostAvailabilityZone()); } }; diff --git a/tests/integration/test_zookeeper_config_load_balancing/configs/zookeeper_load_balancing2.xml b/tests/integration/test_zookeeper_config_load_balancing/configs/zookeeper_load_balancing2.xml new file mode 100644 index 00000000000..eb6ba93c421 --- /dev/null +++ b/tests/integration/test_zookeeper_config_load_balancing/configs/zookeeper_load_balancing2.xml @@ -0,0 +1,29 @@ + + + + random + + az2 + + 0 + 1 + + + + zoo1 + 2181 + az1 + + + zoo2 + 2181 + az2 + + + zoo3 + 2181 + az3 + + 3000 + + diff --git a/tests/integration/test_zookeeper_config_load_balancing/test.py b/tests/integration/test_zookeeper_config_load_balancing/test.py index f17e0c3f03f..6f4c5b908c0 100644 --- a/tests/integration/test_zookeeper_config_load_balancing/test.py +++ b/tests/integration/test_zookeeper_config_load_balancing/test.py @@ -1,6 +1,8 @@ +import time import pytest from helpers.cluster import ClickHouseCluster from helpers.network import PartitionManager +from helpers.test_tools import assert_eq_with_retry cluster = ClickHouseCluster( __file__, zookeeper_config_path="configs/zookeeper_load_balancing.xml" @@ -17,6 +19,10 @@ node3 = cluster.add_instance( "nod3", with_zookeeper=True, main_configs=["configs/zookeeper_load_balancing.xml"] ) +node4 = cluster.add_instance( + "nod4", with_zookeeper=True, main_configs=["configs/zookeeper_load_balancing2.xml"] +) + def change_balancing(old, new, reload=True): line = "{}<" @@ -515,3 +521,33 @@ def test_round_robin(started_cluster): finally: pm.heal_all() change_balancing("round_robin", "random", reload=False) + + +def test_az(started_cluster): + pm = PartitionManager() + try: + # make sure it disconnects from the optimal node + pm._add_rule( + { + "source": node1.ip_address, + "destination": cluster.get_instance_ip("zoo2"), + "action": "REJECT --reject-with tcp-reset", + } + ) + + node4.query_with_retry("select * from system.zookeeper where path='/'") + assert "az2\n" != node4.query( + "select availability_zone from system.zookeeper_connection" + ) + + # fallback_session_lifetime.max is 1 second, but it shouldn't drop current session until the node becomes available + + time.sleep(5) # this is fine + assert 5 >= int(node4.query("select zookeeperSessionUptime()").strip()) + + pm.heal_all() + assert_eq_with_retry( + node4, "select availability_zone from system.zookeeper_connection", "az2\n" + ) + finally: + pm.heal_all() From 5673446d9c97cb98ae4f4c087fb0cdd72348a670 Mon Sep 17 00:00:00 2001 From: Alexander Tokmakov Date: Sun, 23 Jun 2024 02:47:44 +0200 Subject: [PATCH 048/115] fix build --- src/IO/S3/Credentials.h | 1 + 1 file changed, 1 insertion(+) diff --git a/src/IO/S3/Credentials.h b/src/IO/S3/Credentials.h index 7e14193647f..95297ab0538 100644 --- a/src/IO/S3/Credentials.h +++ b/src/IO/S3/Credentials.h @@ -196,6 +196,7 @@ namespace DB namespace S3 { std::string getRunningAvailabilityZone(); +std::string tryGetRunningAvailabilityZone(); } } From 6b994b81a10f49d27cbea1bf4f9d2088134e56be Mon Sep 17 00:00:00 2001 From: Alexander Tokmakov Date: Sun, 23 Jun 2024 03:35:58 +0200 Subject: [PATCH 049/115] fix --- programs/keeper-client/KeeperClient.cpp | 4 ++++ src/Common/ZooKeeper/ZooKeeperArgs.cpp | 1 + .../examples/zkutil_test_commands_new_lib.cpp | 12 ++++-------- 3 files changed, 9 insertions(+), 8 deletions(-) diff --git a/programs/keeper-client/KeeperClient.cpp b/programs/keeper-client/KeeperClient.cpp index ebec337060c..8ef94c4b523 100644 --- a/programs/keeper-client/KeeperClient.cpp +++ b/programs/keeper-client/KeeperClient.cpp @@ -383,6 +383,9 @@ int KeeperClient::main(const std::vector & /* args */) for (const auto & key : keys) { + if (key != "node") + continue; + String prefix = "zookeeper." + key; String host = clickhouse_config.configuration->getString(prefix + ".host"); String port = clickhouse_config.configuration->getString(prefix + ".port"); @@ -401,6 +404,7 @@ int KeeperClient::main(const std::vector & /* args */) zk_args.hosts.push_back(host + ":" + port); } + zk_args.availability_zones.resize(zk_args.hosts.size()); zk_args.connection_timeout_ms = config().getInt("connection-timeout", 10) * 1000; zk_args.session_timeout_ms = config().getInt("session-timeout", 10) * 1000; zk_args.operation_timeout_ms = config().getInt("operation-timeout", 10) * 1000; diff --git a/src/Common/ZooKeeper/ZooKeeperArgs.cpp b/src/Common/ZooKeeper/ZooKeeperArgs.cpp index 8f8f7cc1b7f..3e89f2261be 100644 --- a/src/Common/ZooKeeper/ZooKeeperArgs.cpp +++ b/src/Common/ZooKeeper/ZooKeeperArgs.cpp @@ -54,6 +54,7 @@ ZooKeeperArgs::ZooKeeperArgs(const Poco::Util::AbstractConfiguration & config, c ZooKeeperArgs::ZooKeeperArgs(const String & hosts_string) { splitInto<','>(hosts, hosts_string); + availability_zones.resize(hosts.size()); } void ZooKeeperArgs::initFromKeeperServerSection(const Poco::Util::AbstractConfiguration & config) diff --git a/src/Common/ZooKeeper/examples/zkutil_test_commands_new_lib.cpp b/src/Common/ZooKeeper/examples/zkutil_test_commands_new_lib.cpp index 25d66b94b46..a95e8ac5381 100644 --- a/src/Common/ZooKeeper/examples/zkutil_test_commands_new_lib.cpp +++ b/src/Common/ZooKeeper/examples/zkutil_test_commands_new_lib.cpp @@ -25,14 +25,12 @@ try Poco::Logger::root().setChannel(channel); Poco::Logger::root().setLevel("trace"); - std::string hosts_arg = argv[1]; - std::vector hosts_strings; - splitInto<','>(hosts_strings, hosts_arg); + zkutil::ZooKeeperArgs args{argv[1]}; ZooKeeper::Nodes nodes; - nodes.reserve(hosts_strings.size()); - for (size_t i = 0; i < hosts_strings.size(); ++i) + nodes.reserve(args.hosts.size()); + for (size_t i = 0; i < args.hosts.size(); ++i) { - std::string host_string = hosts_strings[i]; + std::string host_string = args.hosts[i]; bool secure = startsWith(host_string, "secure://"); if (secure) @@ -41,8 +39,6 @@ try nodes.emplace_back(ZooKeeper::Node{Poco::Net::SocketAddress{host_string}, static_cast(i) , secure}); } - - zkutil::ZooKeeperArgs args; ZooKeeper zk(nodes, args, nullptr); Poco::Event event(true); From 7576bb2519682bab16c0e503fcaccb4d04874952 Mon Sep 17 00:00:00 2001 From: Alexander Tokmakov Date: Sun, 23 Jun 2024 13:57:20 +0200 Subject: [PATCH 050/115] fix --- src/Common/ZooKeeper/ZooKeeper.cpp | 6 +++++- src/Common/ZooKeeper/ZooKeeperArgs.cpp | 6 ++++++ src/Common/ZooKeeper/ZooKeeperImpl.cpp | 3 +-- .../test_zookeeper_config_load_balancing/test.py | 2 +- 4 files changed, 13 insertions(+), 4 deletions(-) diff --git a/src/Common/ZooKeeper/ZooKeeper.cpp b/src/Common/ZooKeeper/ZooKeeper.cpp index bc3f29302e3..d5465e0d6c0 100644 --- a/src/Common/ZooKeeper/ZooKeeper.cpp +++ b/src/Common/ZooKeeper/ZooKeeper.cpp @@ -171,6 +171,7 @@ void ZooKeeper::init(ZooKeeperArgs args_, std::unique_ptr if (args.availability_zone_autodetect) updateAvailabilityZones(); } + chassert(availability_zones.size() == args.hosts.size()); /// Shuffle the hosts to distribute the load among ZooKeeper nodes. std::vector shuffled_hosts = shuffleHosts(); @@ -1586,7 +1587,10 @@ int32_t ZooKeeper::getConnectionXid() const String ZooKeeper::getConnectedHostAvailabilityZone() const { - return availability_zones.at(impl->getConnectedNodeIdx()); + auto idx = impl->getConnectedNodeIdx(); + if (idx < 0) + return ""; + return availability_zones.at(idx); } size_t getFailedOpIndex(Coordination::Error exception_code, const Coordination::Responses & responses) diff --git a/src/Common/ZooKeeper/ZooKeeperArgs.cpp b/src/Common/ZooKeeper/ZooKeeperArgs.cpp index 3e89f2261be..ceafe68a92d 100644 --- a/src/Common/ZooKeeper/ZooKeeperArgs.cpp +++ b/src/Common/ZooKeeper/ZooKeeperArgs.cpp @@ -105,8 +105,11 @@ void ZooKeeperArgs::initFromKeeperServerSection(const Poco::Util::AbstractConfig for (const auto & key : keys) { if (startsWith(key, "server")) + { hosts.push_back( (secure ? "secure://" : "") + config.getString(raft_configuration_key + "." + key + ".hostname") + ":" + tcp_port); + availability_zones.push_back(config.getString(raft_configuration_key + "." + key + ".availability_zone", "")); + } } static constexpr std::array load_balancing_keys @@ -130,6 +133,9 @@ void ZooKeeperArgs::initFromKeeperServerSection(const Poco::Util::AbstractConfig } } + availability_zone_autodetect = config.getBool(std::string{config_name} + ".availability_zone_autodetect", false); + if (availability_zone_autodetect) + client_availability_zone = DB::S3::tryGetRunningAvailabilityZone(); } void ZooKeeperArgs::initFromKeeperSection(const Poco::Util::AbstractConfiguration & config, const std::string & config_name) diff --git a/src/Common/ZooKeeper/ZooKeeperImpl.cpp b/src/Common/ZooKeeper/ZooKeeperImpl.cpp index 61f17148d72..28dc0d77b1a 100644 --- a/src/Common/ZooKeeper/ZooKeeperImpl.cpp +++ b/src/Common/ZooKeeper/ZooKeeperImpl.cpp @@ -438,9 +438,8 @@ void ZooKeeper::connect( WriteBufferFromOwnString fail_reasons; for (size_t try_no = 0; try_no < num_tries; ++try_no) { - for (size_t i = 0; i < nodes.size(); ++i) + for (const auto & node : nodes) { - const auto & node = nodes[i]; try { /// Reset the state of previous attempt. diff --git a/tests/integration/test_zookeeper_config_load_balancing/test.py b/tests/integration/test_zookeeper_config_load_balancing/test.py index 6f4c5b908c0..f59f6a65a64 100644 --- a/tests/integration/test_zookeeper_config_load_balancing/test.py +++ b/tests/integration/test_zookeeper_config_load_balancing/test.py @@ -529,7 +529,7 @@ def test_az(started_cluster): # make sure it disconnects from the optimal node pm._add_rule( { - "source": node1.ip_address, + "source": node4.ip_address, "destination": cluster.get_instance_ip("zoo2"), "action": "REJECT --reject-with tcp-reset", } From 6d7441a94a98f653553c24116d772d6397472a20 Mon Sep 17 00:00:00 2001 From: Igor Nikonov Date: Sun, 23 Jun 2024 16:24:22 +0000 Subject: [PATCH 051/115] Fix: sync replicas --- .../test_parallel_replicas_distributed_skip_shards/test.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/tests/integration/test_parallel_replicas_distributed_skip_shards/test.py b/tests/integration/test_parallel_replicas_distributed_skip_shards/test.py index af114ade2d7..0f54d053ce1 100644 --- a/tests/integration/test_parallel_replicas_distributed_skip_shards/test.py +++ b/tests/integration/test_parallel_replicas_distributed_skip_shards/test.py @@ -53,6 +53,9 @@ def create_tables(cluster, table_name): node1.query(f"INSERT INTO {table_name} SELECT number, number FROM numbers(1000)") node2.query(f"INSERT INTO {table_name} SELECT -number, -number FROM numbers(1000)") node1.query(f"INSERT INTO {table_name} SELECT number, number FROM numbers(3)") + # need to sync replicas to have consistent result + node1.query(f"SYSTEM SYNC REPLICA {table_name}") + node2.query(f"SYSTEM SYNC REPLICA {table_name}") @pytest.mark.parametrize( From 7bbd14cd5e85f19ab67febe44b1d4c4203d3ed3a Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Sun, 23 Jun 2024 18:27:25 +0200 Subject: [PATCH 052/115] Update 00763_lock_buffer_long.sh --- tests/queries/0_stateless/00763_lock_buffer_long.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/queries/0_stateless/00763_lock_buffer_long.sh b/tests/queries/0_stateless/00763_lock_buffer_long.sh index 79c35e99486..6871296b480 100755 --- a/tests/queries/0_stateless/00763_lock_buffer_long.sh +++ b/tests/queries/0_stateless/00763_lock_buffer_long.sh @@ -1,5 +1,5 @@ #!/usr/bin/env bash -# Tags: long, no-s3-storage +# Tags: long, no-s3-storage, no-msan, no-asan, no-tsan, no-debug set -e From cb11e5b26f2d74c2f0e88029b05dcff5850f5356 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Sun, 23 Jun 2024 23:34:23 +0200 Subject: [PATCH 053/115] Fix WTF in metadata_type --- .../0_stateless/02963_test_flexible_disk_configuration.sql | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/queries/0_stateless/02963_test_flexible_disk_configuration.sql b/tests/queries/0_stateless/02963_test_flexible_disk_configuration.sql index 8f67cd7e030..7ebef866360 100644 --- a/tests/queries/0_stateless/02963_test_flexible_disk_configuration.sql +++ b/tests/queries/0_stateless/02963_test_flexible_disk_configuration.sql @@ -22,7 +22,7 @@ create table test (a Int32) engine = MergeTree() order by tuple() settings disk=disk(name='test2', type = object_storage, object_storage_type = s3, - metadata_storage_type = local, + metadata_type = local, endpoint = 'http://localhost:11111/test/common/', access_key_id = clickhouse, secret_access_key = clickhouse); @@ -32,7 +32,7 @@ create table test (a Int32) engine = MergeTree() order by tuple() settings disk=disk(name='test3', type = object_storage, object_storage_type = s3, - metadata_storage_type = local, + metadata_type = local, metadata_keep_free_space_bytes = 1024, endpoint = 'http://localhost:11111/test/common/', access_key_id = clickhouse, @@ -43,7 +43,7 @@ create table test (a Int32) engine = MergeTree() order by tuple() settings disk=disk(name='test4', type = object_storage, object_storage_type = s3, - metadata_storage_type = local, + metadata_type = local, metadata_keep_free_space_bytes = 0, endpoint = 'http://localhost:11111/test/common/', access_key_id = clickhouse, From 3423a5571a35ad1a8a66388188c531eb206e1dde Mon Sep 17 00:00:00 2001 From: Alexander Tokmakov Date: Sun, 23 Jun 2024 23:55:45 +0200 Subject: [PATCH 054/115] fix tests --- .../test.py | 2 +- .../test_zookeeper_fallback_session/test.py | 22 +++++++++++++++++-- 2 files changed, 21 insertions(+), 3 deletions(-) diff --git a/tests/integration/test_zookeeper_config_load_balancing/test.py b/tests/integration/test_zookeeper_config_load_balancing/test.py index f59f6a65a64..8ef69f0c74f 100644 --- a/tests/integration/test_zookeeper_config_load_balancing/test.py +++ b/tests/integration/test_zookeeper_config_load_balancing/test.py @@ -543,7 +543,7 @@ def test_az(started_cluster): # fallback_session_lifetime.max is 1 second, but it shouldn't drop current session until the node becomes available time.sleep(5) # this is fine - assert 5 >= int(node4.query("select zookeeperSessionUptime()").strip()) + assert 5 <= int(node4.query("select zookeeperSessionUptime()").strip()) pm.heal_all() assert_eq_with_retry( diff --git a/tests/integration/test_zookeeper_fallback_session/test.py b/tests/integration/test_zookeeper_fallback_session/test.py index 9afabfa3da3..932bbe482d2 100644 --- a/tests/integration/test_zookeeper_fallback_session/test.py +++ b/tests/integration/test_zookeeper_fallback_session/test.py @@ -84,10 +84,28 @@ def test_fallback_session(started_cluster: ClickHouseCluster): ) # at this point network partitioning has been reverted. - # the nodes should switch to zoo1 automatically because of `in_order` load-balancing. + # the nodes should switch to zoo1 because of `in_order` load-balancing. # otherwise they would connect to a random replica + + # but there's no reason to reconnect because current session works + # and there's no "optimal" node with `in_order` load-balancing + # so we need to break the current session + for node in [node1, node2, node3]: - assert_uses_zk_node(node, "zoo1") + assert_uses_zk_node(node, "zoo3") + + with PartitionManager() as pm: + for node in started_cluster.instances.values(): + pm._add_rule( + { + "source": node.ip_address, + "destination": cluster.get_instance_ip("zoo3"), + "action": "REJECT --reject-with tcp-reset", + } + ) + + for node in [node1, node2, node3]: + assert_uses_zk_node(node, "zoo1") node1.query_with_retry("INSERT INTO simple VALUES ({0}, {0})".format(2)) for node in [node2, node3]: From 3491fad643464db1f28d1f8ea291d14f4448b238 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Mon, 24 Jun 2024 00:35:15 +0200 Subject: [PATCH 055/115] Fix slow test --- .../0_stateless/00731_long_merge_tree_select_opened_files.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/queries/0_stateless/00731_long_merge_tree_select_opened_files.sh b/tests/queries/0_stateless/00731_long_merge_tree_select_opened_files.sh index 1bb4dbd34de..af746c43da9 100755 --- a/tests/queries/0_stateless/00731_long_merge_tree_select_opened_files.sh +++ b/tests/queries/0_stateless/00731_long_merge_tree_select_opened_files.sh @@ -1,5 +1,5 @@ #!/usr/bin/env bash -# Tags: long, no-s3-storage +# Tags: long, no-s3-storage, no-tsan # no-s3 because read FileOpen metric set -e @@ -31,6 +31,6 @@ $CLICKHOUSE_CLIENT $settings -q "$touching_many_parts_query" &> /dev/null $CLICKHOUSE_CLIENT $settings -q "SYSTEM FLUSH LOGS" -$CLICKHOUSE_CLIENT $settings -q "SELECT ProfileEvents['FileOpen'] as opened_files FROM system.query_log WHERE query='$touching_many_parts_query' and current_database = currentDatabase() ORDER BY event_time DESC, opened_files DESC LIMIT 1;" +$CLICKHOUSE_CLIENT $settings -q "SELECT ProfileEvents['FileOpen'] as opened_files FROM system.query_log WHERE query = '$touching_many_parts_query' AND current_database = currentDatabase() AND event_date >= yesterday() ORDER BY event_time DESC, opened_files DESC LIMIT 1;" $CLICKHOUSE_CLIENT $settings -q "DROP TABLE IF EXISTS merge_tree_table;" From 614e985a3718b80625b994048c358c72a5f79fc2 Mon Sep 17 00:00:00 2001 From: Alexander Tokmakov Date: Mon, 24 Jun 2024 14:11:50 +0200 Subject: [PATCH 056/115] address review comments --- src/Common/GetPriorityForLoadBalancing.cpp | 3 +++ src/Common/GetPriorityForLoadBalancing.h | 6 ------ src/Common/ZooKeeper/ZooKeeper.cpp | 3 ++- src/Common/ZooKeeper/ZooKeeperArgs.cpp | 10 +++++++--- 4 files changed, 12 insertions(+), 10 deletions(-) diff --git a/src/Common/GetPriorityForLoadBalancing.cpp b/src/Common/GetPriorityForLoadBalancing.cpp index 83197f2365c..dc5704ef6b5 100644 --- a/src/Common/GetPriorityForLoadBalancing.cpp +++ b/src/Common/GetPriorityForLoadBalancing.cpp @@ -60,6 +60,9 @@ GetPriorityForLoadBalancing::getPriorityFunc(LoadBalancing load_balance, size_t return get_priority; } +/// Some load balancing strategies (such as "nearest hostname") have preferred nodes to connect to. +/// Usually it's a node in the same data center/availability zone. +/// For other strategies there's no difference between nodes. bool GetPriorityForLoadBalancing::hasOptimalNode() const { switch (load_balancing) diff --git a/src/Common/GetPriorityForLoadBalancing.h b/src/Common/GetPriorityForLoadBalancing.h index b635dea6c93..a2079b28dad 100644 --- a/src/Common/GetPriorityForLoadBalancing.h +++ b/src/Common/GetPriorityForLoadBalancing.h @@ -14,7 +14,6 @@ public: explicit GetPriorityForLoadBalancing(LoadBalancing load_balancing_, size_t last_used_ = 0) : load_balancing(load_balancing_), last_used(last_used_) { - saved_offset = thread_local_rng(); } GetPriorityForLoadBalancing() = default; @@ -31,10 +30,6 @@ public: } Func getPriorityFunc(LoadBalancing load_balance, size_t offset, size_t pool_size) const; - Func getPriorityFunc(size_t pool_size) const - { - return getPriorityFunc(load_balancing, saved_offset % pool_size, pool_size); - } bool hasOptimalNode() const; @@ -45,7 +40,6 @@ public: private: mutable size_t last_used = 0; /// Last used for round_robin policy. - size_t saved_offset; /// Default random offset for round_robin policy. }; } diff --git a/src/Common/ZooKeeper/ZooKeeper.cpp b/src/Common/ZooKeeper/ZooKeeper.cpp index d5465e0d6c0..3b4286e37d2 100644 --- a/src/Common/ZooKeeper/ZooKeeper.cpp +++ b/src/Common/ZooKeeper/ZooKeeper.cpp @@ -290,7 +290,8 @@ ZooKeeper::ZooKeeper(const Poco::Util::AbstractConfiguration & config, const std std::vector ZooKeeper::shuffleHosts() const { - std::function get_priority = args.get_priority_load_balancing.getPriorityFunc(args.hosts.size()); + std::function get_priority = args.get_priority_load_balancing.getPriorityFunc( + args.get_priority_load_balancing.load_balancing, /* offset for first_or_random */ 0, args.hosts.size()); std::vector shuffle_hosts; for (size_t i = 0; i < args.hosts.size(); ++i) { diff --git a/src/Common/ZooKeeper/ZooKeeperArgs.cpp b/src/Common/ZooKeeper/ZooKeeperArgs.cpp index ceafe68a92d..2154d9d9232 100644 --- a/src/Common/ZooKeeper/ZooKeeperArgs.cpp +++ b/src/Common/ZooKeeper/ZooKeeperArgs.cpp @@ -128,7 +128,7 @@ void ZooKeeperArgs::initFromKeeperServerSection(const Poco::Util::AbstractConfig auto load_balancing = magic_enum::enum_cast(Poco::toUpper(load_balancing_str)); if (!load_balancing) throw DB::Exception(DB::ErrorCodes::BAD_ARGUMENTS, "Unknown load balancing: {}", load_balancing_str); - get_priority_load_balancing.load_balancing = *load_balancing; + get_priority_load_balancing = DB::GetPriorityForLoadBalancing(*load_balancing, thread_local_rng() % hosts.size()); break; } } @@ -145,6 +145,8 @@ void ZooKeeperArgs::initFromKeeperSection(const Poco::Util::AbstractConfiguratio Poco::Util::AbstractConfiguration::Keys keys; config.keys(config_name, keys); + std::optional load_balancing; + for (const auto & key : keys) { if (key.starts_with("node")) @@ -220,10 +222,9 @@ void ZooKeeperArgs::initFromKeeperSection(const Poco::Util::AbstractConfiguratio { String load_balancing_str = config.getString(config_name + "." + key); /// Use magic_enum to avoid dependency from dbms (`SettingFieldLoadBalancingTraits::fromString(...)`) - auto load_balancing = magic_enum::enum_cast(Poco::toUpper(load_balancing_str)); + load_balancing = magic_enum::enum_cast(Poco::toUpper(load_balancing_str)); if (!load_balancing) throw DB::Exception(DB::ErrorCodes::BAD_ARGUMENTS, "Unknown load balancing: {}", load_balancing_str); - get_priority_load_balancing.load_balancing = *load_balancing; } else if (key == "fallback_session_lifetime") { @@ -245,6 +246,9 @@ void ZooKeeperArgs::initFromKeeperSection(const Poco::Util::AbstractConfiguratio throw KeeperException(Coordination::Error::ZBADARGUMENTS, "Unknown key {} in config file", key); } + if (load_balancing) + get_priority_load_balancing = DB::GetPriorityForLoadBalancing(*load_balancing, thread_local_rng() % hosts.size()); + if (availability_zone_autodetect) client_availability_zone = DB::S3::tryGetRunningAvailabilityZone(); } From eda9b9d874c9c7f6ad89a23155c05af70b6ba469 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Tue, 25 Jun 2024 01:37:14 +0200 Subject: [PATCH 057/115] Do not load inactive parts on readonly disks --- programs/local/LocalServer.cpp | 1 - src/Storages/MergeTree/MergeTreeData.cpp | 7 +++++-- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/programs/local/LocalServer.cpp b/programs/local/LocalServer.cpp index cb1c35743b2..00fb79e1af9 100644 --- a/programs/local/LocalServer.cpp +++ b/programs/local/LocalServer.cpp @@ -145,7 +145,6 @@ void LocalServer::initialize(Poco::Util::Application & self) config().getUInt("max_io_thread_pool_free_size", 0), config().getUInt("io_thread_pool_queue_size", 10000)); - const size_t active_parts_loading_threads = config().getUInt("max_active_parts_loading_thread_pool_size", 64); getActivePartsLoadingThreadPool().initialize( active_parts_loading_threads, diff --git a/src/Storages/MergeTree/MergeTreeData.cpp b/src/Storages/MergeTree/MergeTreeData.cpp index 2e0ea4cdbcd..1571479fd44 100644 --- a/src/Storages/MergeTree/MergeTreeData.cpp +++ b/src/Storages/MergeTree/MergeTreeData.cpp @@ -1759,11 +1759,14 @@ void MergeTreeData::loadDataParts(bool skip_sanity_checks, std::optional runner(getActivePartsLoadingThreadPool().get(), "ActiveParts"); + bool all_disks_are_readonly = true; for (size_t i = 0; i < disks.size(); ++i) { const auto & disk_ptr = disks[i]; if (disk_ptr->isBroken()) continue; + if (!disk_ptr->isReadOnly()) + all_disks_are_readonly = false; auto & disk_parts = parts_to_load_by_disk[i]; auto & unexpected_disk_parts = unexpected_parts_to_load_by_disk[i]; @@ -1916,7 +1919,6 @@ void MergeTreeData::loadDataParts(bool skip_sanity_checks, std::optionalrenameToDetached("broken-on-start"); /// detached parts must not have '_' in prefixes @@ -1961,7 +1963,8 @@ void MergeTreeData::loadDataParts(bool skip_sanity_checks, std::optional Date: Tue, 25 Jun 2024 12:40:09 +0200 Subject: [PATCH 058/115] Better system queue log management --- .../ObjectStorageQueueSettings.cpp | 4 +++ .../ObjectStorageQueueSettings.h | 3 +- .../StorageObjectStorageQueue.cpp | 36 +++++++++++++------ ...erS3Queue.cpp => registerQueueStorage.cpp} | 0 4 files changed, 30 insertions(+), 13 deletions(-) rename src/Storages/ObjectStorageQueue/{registerS3Queue.cpp => registerQueueStorage.cpp} (100%) diff --git a/src/Storages/ObjectStorageQueue/ObjectStorageQueueSettings.cpp b/src/Storages/ObjectStorageQueue/ObjectStorageQueueSettings.cpp index 3458105de8c..67743db6197 100644 --- a/src/Storages/ObjectStorageQueue/ObjectStorageQueueSettings.cpp +++ b/src/Storages/ObjectStorageQueue/ObjectStorageQueueSettings.cpp @@ -23,8 +23,12 @@ void ObjectStorageQueueSettings::loadFromQuery(ASTStorage & storage_def) { /// We support settings starting with s3_ for compatibility. for (auto & change : storage_def.settings->changes) + { if (change.name.starts_with("s3queue_")) change.name = change.name.substr(std::strlen("s3queue_")); + if (change.name == "enable_logging_to_s3queue_log") + change.name = "enable_logging_to_queue_log"; + } applyChanges(storage_def.settings->changes); } diff --git a/src/Storages/ObjectStorageQueue/ObjectStorageQueueSettings.h b/src/Storages/ObjectStorageQueue/ObjectStorageQueueSettings.h index 0db5bb92bde..b1025167b06 100644 --- a/src/Storages/ObjectStorageQueue/ObjectStorageQueueSettings.h +++ b/src/Storages/ObjectStorageQueue/ObjectStorageQueueSettings.h @@ -21,8 +21,7 @@ class ASTStorage; M(String, keeper_path, "", "Zookeeper node path", 0) \ M(UInt32, loading_retries, 0, "Retry loading up to specified number of times", 0) \ M(UInt32, processing_threads_num, 1, "Number of processing threads", 0) \ - M(UInt32, enable_logging_to_s3queue_log, 1, "Enable logging to system table system.s3queue_log", 0) \ - M(UInt32, enable_logging_to_azure_queue_log, 1, "Enable logging to system table system.s3queue_log", 0) \ + M(UInt32, enable_logging_to_queue_log, 1, "Enable logging to system table system.(s3/azure_)queue_log", 0) \ M(String, last_processed_path, "", "For Ordered mode. Files that have lexicographically smaller file name are considered already processed", 0) \ M(UInt32, tracked_file_ttl_sec, 0, "Maximum number of seconds to store processed files in ZooKeeper node (store forever by default)", 0) \ M(UInt32, polling_min_timeout_ms, 1000, "Minimal timeout before next polling", 0) \ diff --git a/src/Storages/ObjectStorageQueue/StorageObjectStorageQueue.cpp b/src/Storages/ObjectStorageQueue/StorageObjectStorageQueue.cpp index 640dbf22421..fba7dcd8b09 100644 --- a/src/Storages/ObjectStorageQueue/StorageObjectStorageQueue.cpp +++ b/src/Storages/ObjectStorageQueue/StorageObjectStorageQueue.cpp @@ -60,7 +60,7 @@ namespace return zkutil::extractZooKeeperPath(result_zk_path, true); } - void checkAndAdjustSettings(ObjectStorageQueueSettings & queue_settings, const Settings & settings, bool is_attach) + void checkAndAdjustSettings(ObjectStorageQueueSettings & queue_settings, bool is_attach) { if (!is_attach && !queue_settings.mode.changed) { @@ -73,11 +73,6 @@ namespace throw Exception(ErrorCodes::BAD_ARGUMENTS, "Setting `processing_threads_num` cannot be set to zero"); } - if (!queue_settings.enable_logging_to_s3queue_log.changed) - { - queue_settings.enable_logging_to_s3queue_log = settings.s3queue_enable_logging_to_s3queue_log; - } - if (queue_settings.cleanup_interval_min_ms > queue_settings.cleanup_interval_max_ms) { throw Exception(ErrorCodes::BAD_ARGUMENTS, @@ -85,6 +80,28 @@ namespace queue_settings.cleanup_interval_min_ms, queue_settings.cleanup_interval_max_ms); } } + + std::shared_ptr getQueueLog(const ObjectStoragePtr & storage, const ContextPtr & context, const ObjectStorageQueueSettings & table_settings) + { + const auto & settings = context->getSettingsRef(); + switch (storage->getType()) + { + case DB::ObjectStorageType::S3: + { + if (table_settings.enable_logging_to_queue_log || settings.s3queue_enable_logging_to_s3queue_log) + return context->getS3QueueLog(); + return nullptr; + } + case DB::ObjectStorageType::Azure: + { + if (table_settings.enable_logging_to_queue_log) + return context->getAzureQueueLog(); + return nullptr; + } + default: + throw Exception(ErrorCodes::LOGICAL_ERROR, "Unexpected object storage type: {}", storage->getType()); + } + } } StorageObjectStorageQueue::StorageObjectStorageQueue( @@ -120,7 +137,7 @@ StorageObjectStorageQueue::StorageObjectStorageQueue( throw Exception(ErrorCodes::QUERY_NOT_ALLOWED, "ObjectStorageQueue url must either end with '/' or contain globs"); } - checkAndAdjustSettings(*queue_settings, context_->getSettingsRef(), mode > LoadingStrictnessLevel::CREATE); + checkAndAdjustSettings(*queue_settings, mode > LoadingStrictnessLevel::CREATE); object_storage = configuration->createObjectStorage(context_, /* is_readonly */true); FormatFactory::instance().checkFormatName(configuration->format); @@ -332,9 +349,6 @@ std::shared_ptr StorageObjectStorageQueue::createSourc { object_storage->removeObject(StoredObject(path)); }; - auto system_queue_log = queue_settings->enable_logging_to_s3queue_log - ? local_context->getS3QueueLog() - : queue_settings->enable_logging_to_azure_queue_log ? local_context->getAzureQueueLog() : nullptr; return std::make_shared( getName(), @@ -348,7 +362,7 @@ std::shared_ptr StorageObjectStorageQueue::createSourc local_context, shutdown_called, table_is_being_dropped, - system_queue_log, + getQueueLog(object_storage, local_context, *queue_settings), getStorageID(), log); } diff --git a/src/Storages/ObjectStorageQueue/registerS3Queue.cpp b/src/Storages/ObjectStorageQueue/registerQueueStorage.cpp similarity index 100% rename from src/Storages/ObjectStorageQueue/registerS3Queue.cpp rename to src/Storages/ObjectStorageQueue/registerQueueStorage.cpp From 95b1651ba68a94213dec14a7771fcfa6822a5bcc Mon Sep 17 00:00:00 2001 From: kssenii Date: Tue, 25 Jun 2024 13:56:01 +0200 Subject: [PATCH 059/115] Better pytest --- .../integration/test_storage_s3_queue/test.py | 22 ++++--------------- 1 file changed, 4 insertions(+), 18 deletions(-) diff --git a/tests/integration/test_storage_s3_queue/test.py b/tests/integration/test_storage_s3_queue/test.py index a118c8da52f..b1e685af37a 100644 --- a/tests/integration/test_storage_s3_queue/test.py +++ b/tests/integration/test_storage_s3_queue/test.py @@ -250,15 +250,8 @@ def create_mv( ) -@pytest.mark.parametrize( - "mode, engine_name", - [ - pytest.param("unordered", "S3Queue"), - pytest.param("unordered", "AzureQueue"), - pytest.param("ordered", "S3Queue"), - pytest.param("ordered", "AzureQueue"), - ], -) +@pytest.mark.parametrize("mode", ["unordered", "ordered"]) +@pytest.mark.parametrize("engine_name", ["S3Queue", "AzureQueue"]) def test_delete_after_processing(started_cluster, mode, engine_name): node = started_cluster.instances["instance"] table_name = f"test.delete_after_processing_{mode}_{engine_name}" @@ -315,15 +308,8 @@ def test_delete_after_processing(started_cluster, mode, engine_name): assert False -@pytest.mark.parametrize( - "mode, engine_name", - [ - pytest.param("unordered", "S3Queue"), - pytest.param("unordered", "AzureQueue"), - pytest.param("ordered", "S3Queue"), - pytest.param("ordered", "AzureQueue"), - ], -) +@pytest.mark.parametrize("mode", ["unordered", "ordered"]) +@pytest.mark.parametrize("engine_name", ["S3Queue", "AzureQueue"]) def test_failed_retry(started_cluster, mode, engine_name): node = started_cluster.instances["instance"] table_name = f"test.failed_retry_{mode}_{engine_name}" From 782669a9fd99738aeef4814b2610a0736fe5ba18 Mon Sep 17 00:00:00 2001 From: Nikita Mikhaylov Date: Tue, 25 Jun 2024 14:23:37 +0000 Subject: [PATCH 060/115] Refactor --- programs/client/Client.cpp | 8 +- programs/client/Client.h | 3 + programs/local/LocalServer.cpp | 207 ++++++++++++++-------------- programs/local/LocalServer.h | 3 + src/Client/ClientApplicationBase.h | 16 +++ src/Client/ClientBase.cpp | 210 ++++++++++++++++------------- src/Client/ClientBase.h | 35 +++-- src/Client/LineReader.cpp | 35 +++-- src/Client/LineReader.h | 19 ++- src/Client/LocalConnection.cpp | 29 +++- src/Client/ReplxxLineReader.cpp | 17 ++- src/Client/ReplxxLineReader.h | 15 ++- src/Common/ProgressIndication.cpp | 10 +- src/Common/ProgressIndication.h | 17 +++ src/Common/TerminalSize.cpp | 10 +- src/Common/TerminalSize.h | 4 +- 16 files changed, 390 insertions(+), 248 deletions(-) create mode 100644 src/Client/ClientApplicationBase.h diff --git a/programs/client/Client.cpp b/programs/client/Client.cpp index efe23d57478..eecf3a90e87 100644 --- a/programs/client/Client.cpp +++ b/programs/client/Client.cpp @@ -241,6 +241,10 @@ std::vector Client::loadWarningMessages() } } +Poco::Util::LayeredConfiguration & Client::getClientConfiguration() +{ + return config(); +} void Client::initialize(Poco::Util::Application & self) { @@ -690,9 +694,7 @@ bool Client::processWithFuzzing(const String & full_query) const char * begin = full_query.data(); orig_ast = parseQuery(begin, begin + full_query.size(), global_context->getSettingsRef(), - /*allow_multi_statements=*/ true, - /*is_interactive=*/ is_interactive, - /*ignore_error=*/ ignore_error); + /*allow_multi_statements=*/ true); } catch (const Exception & e) { diff --git a/programs/client/Client.h b/programs/client/Client.h index bef948b3c1e..229608f787d 100644 --- a/programs/client/Client.h +++ b/programs/client/Client.h @@ -16,6 +16,9 @@ public: int main(const std::vector & /*args*/) override; protected: + + Poco::Util::LayeredConfiguration & getClientConfiguration() override; + bool processWithFuzzing(const String & full_query) override; std::optional processFuzzingStep(const String & query_to_execute, const ASTPtr & parsed_query); diff --git a/programs/local/LocalServer.cpp b/programs/local/LocalServer.cpp index cb1c35743b2..503cb0fb97d 100644 --- a/programs/local/LocalServer.cpp +++ b/programs/local/LocalServer.cpp @@ -82,6 +82,11 @@ void applySettingsOverridesForLocal(ContextMutablePtr context) context->setSettings(settings); } +Poco::Util::LayeredConfiguration & LocalServer::getClientConfiguration() +{ + return config(); +} + void LocalServer::processError(const String &) const { if (ignore_error) @@ -117,19 +122,19 @@ void LocalServer::initialize(Poco::Util::Application & self) Poco::Util::Application::initialize(self); /// Load config files if exists - if (config().has("config-file") || fs::exists("config.xml")) + if (getClientConfiguration().has("config-file") || fs::exists("config.xml")) { - const auto config_path = config().getString("config-file", "config.xml"); + const auto config_path = getClientConfiguration().getString("config-file", "config.xml"); ConfigProcessor config_processor(config_path, false, true); ConfigProcessor::setConfigPath(fs::path(config_path).parent_path()); auto loaded_config = config_processor.loadConfig(); - config().add(loaded_config.configuration.duplicate(), PRIO_DEFAULT, false); + getClientConfiguration().add(loaded_config.configuration.duplicate(), PRIO_DEFAULT, false); } GlobalThreadPool::initialize( - config().getUInt("max_thread_pool_size", 10000), - config().getUInt("max_thread_pool_free_size", 1000), - config().getUInt("thread_pool_queue_size", 10000) + getClientConfiguration().getUInt("max_thread_pool_size", 10000), + getClientConfiguration().getUInt("max_thread_pool_free_size", 1000), + getClientConfiguration().getUInt("thread_pool_queue_size", 10000) ); #if USE_AZURE_BLOB_STORAGE @@ -141,18 +146,18 @@ void LocalServer::initialize(Poco::Util::Application & self) #endif getIOThreadPool().initialize( - config().getUInt("max_io_thread_pool_size", 100), - config().getUInt("max_io_thread_pool_free_size", 0), - config().getUInt("io_thread_pool_queue_size", 10000)); + getClientConfiguration().getUInt("max_io_thread_pool_size", 100), + getClientConfiguration().getUInt("max_io_thread_pool_free_size", 0), + getClientConfiguration().getUInt("io_thread_pool_queue_size", 10000)); - const size_t active_parts_loading_threads = config().getUInt("max_active_parts_loading_thread_pool_size", 64); + const size_t active_parts_loading_threads = getClientConfiguration().getUInt("max_active_parts_loading_thread_pool_size", 64); getActivePartsLoadingThreadPool().initialize( active_parts_loading_threads, 0, // We don't need any threads one all the parts will be loaded active_parts_loading_threads); - const size_t outdated_parts_loading_threads = config().getUInt("max_outdated_parts_loading_thread_pool_size", 32); + const size_t outdated_parts_loading_threads = getClientConfiguration().getUInt("max_outdated_parts_loading_thread_pool_size", 32); getOutdatedPartsLoadingThreadPool().initialize( outdated_parts_loading_threads, 0, // We don't need any threads one all the parts will be loaded @@ -160,7 +165,7 @@ void LocalServer::initialize(Poco::Util::Application & self) getOutdatedPartsLoadingThreadPool().setMaxTurboThreads(active_parts_loading_threads); - const size_t unexpected_parts_loading_threads = config().getUInt("max_unexpected_parts_loading_thread_pool_size", 32); + const size_t unexpected_parts_loading_threads = getClientConfiguration().getUInt("max_unexpected_parts_loading_thread_pool_size", 32); getUnexpectedPartsLoadingThreadPool().initialize( unexpected_parts_loading_threads, 0, // We don't need any threads one all the parts will be loaded @@ -168,7 +173,7 @@ void LocalServer::initialize(Poco::Util::Application & self) getUnexpectedPartsLoadingThreadPool().setMaxTurboThreads(active_parts_loading_threads); - const size_t cleanup_threads = config().getUInt("max_parts_cleaning_thread_pool_size", 128); + const size_t cleanup_threads = getClientConfiguration().getUInt("max_parts_cleaning_thread_pool_size", 128); getPartsCleaningThreadPool().initialize( cleanup_threads, 0, // We don't need any threads one all the parts will be deleted @@ -201,10 +206,10 @@ void LocalServer::tryInitPath() { std::string path; - if (config().has("path")) + if (getClientConfiguration().has("path")) { // User-supplied path. - path = config().getString("path"); + path = getClientConfiguration().getString("path"); Poco::trimInPlace(path); if (path.empty()) @@ -263,13 +268,13 @@ void LocalServer::tryInitPath() global_context->setUserFilesPath(""); /// user's files are everywhere - std::string user_scripts_path = config().getString("user_scripts_path", fs::path(path) / "user_scripts/"); + std::string user_scripts_path = getClientConfiguration().getString("user_scripts_path", fs::path(path) / "user_scripts/"); global_context->setUserScriptsPath(user_scripts_path); /// top_level_domains_lists - const std::string & top_level_domains_path = config().getString("top_level_domains_path", fs::path(path) / "top_level_domains/"); + const std::string & top_level_domains_path = getClientConfiguration().getString("top_level_domains_path", fs::path(path) / "top_level_domains/"); if (!top_level_domains_path.empty()) - TLDListsHolder::getInstance().parseConfig(fs::path(top_level_domains_path) / "", config()); + TLDListsHolder::getInstance().parseConfig(fs::path(top_level_domains_path) / "", getClientConfiguration()); } @@ -311,14 +316,14 @@ void LocalServer::cleanup() std::string LocalServer::getInitialCreateTableQuery() { - if (!config().has("table-structure") && !config().has("table-file") && !config().has("table-data-format") && (!isRegularFile(STDIN_FILENO) || queries.empty())) + if (!getClientConfiguration().has("table-structure") && !getClientConfiguration().has("table-file") && !getClientConfiguration().has("table-data-format") && (!isRegularFile(STDIN_FILENO) || queries.empty())) return {}; - auto table_name = backQuoteIfNeed(config().getString("table-name", "table")); - auto table_structure = config().getString("table-structure", "auto"); + auto table_name = backQuoteIfNeed(getClientConfiguration().getString("table-name", "table")); + auto table_structure = getClientConfiguration().getString("table-structure", "auto"); String table_file; - if (!config().has("table-file") || config().getString("table-file") == "-") + if (!getClientConfiguration().has("table-file") || getClientConfiguration().getString("table-file") == "-") { /// Use Unix tools stdin naming convention table_file = "stdin"; @@ -326,7 +331,7 @@ std::string LocalServer::getInitialCreateTableQuery() else { /// Use regular file - auto file_name = config().getString("table-file"); + auto file_name = getClientConfiguration().getString("table-file"); table_file = quoteString(file_name); } @@ -374,18 +379,18 @@ void LocalServer::setupUsers() ConfigurationPtr users_config; auto & access_control = global_context->getAccessControl(); - access_control.setNoPasswordAllowed(config().getBool("allow_no_password", true)); - access_control.setPlaintextPasswordAllowed(config().getBool("allow_plaintext_password", true)); - if (config().has("config-file") || fs::exists("config.xml")) + access_control.setNoPasswordAllowed(getClientConfiguration().getBool("allow_no_password", true)); + access_control.setPlaintextPasswordAllowed(getClientConfiguration().getBool("allow_plaintext_password", true)); + if (getClientConfiguration().has("config-file") || fs::exists("config.xml")) { - String config_path = config().getString("config-file", ""); - bool has_user_directories = config().has("user_directories"); + String config_path = getClientConfiguration().getString("config-file", ""); + bool has_user_directories = getClientConfiguration().has("user_directories"); const auto config_dir = fs::path{config_path}.remove_filename().string(); - String users_config_path = config().getString("users_config", ""); + String users_config_path = getClientConfiguration().getString("users_config", ""); if (users_config_path.empty() && has_user_directories) { - users_config_path = config().getString("user_directories.users_xml.path"); + users_config_path = getClientConfiguration().getString("user_directories.users_xml.path"); if (fs::path(users_config_path).is_relative() && fs::exists(fs::path(config_dir) / users_config_path)) users_config_path = fs::path(config_dir) / users_config_path; } @@ -409,10 +414,10 @@ void LocalServer::setupUsers() void LocalServer::connect() { - connection_parameters = ConnectionParameters(config(), "localhost"); + connection_parameters = ConnectionParameters(getClientConfiguration(), "localhost"); ReadBuffer * in; - auto table_file = config().getString("table-file", "-"); + auto table_file = getClientConfiguration().getString("table-file", "-"); if (table_file == "-" || table_file == "stdin") { in = &std_in; @@ -433,7 +438,7 @@ try UseSSL use_ssl; thread_status.emplace(); - StackTrace::setShowAddresses(config().getBool("show_addresses_in_stack_traces", true)); + StackTrace::setShowAddresses(getClientConfiguration().getBool("show_addresses_in_stack_traces", true)); setupSignalHandler(); @@ -448,7 +453,7 @@ try if (rlim.rlim_cur < rlim.rlim_max) { - rlim.rlim_cur = config().getUInt("max_open_files", static_cast(rlim.rlim_max)); + rlim.rlim_cur = getClientConfiguration().getUInt("max_open_files", static_cast(rlim.rlim_max)); int rc = setrlimit(RLIMIT_NOFILE, &rlim); if (rc != 0) std::cerr << fmt::format("Cannot set max number of file descriptors to {}. Try to specify max_open_files according to your system limits. error: {}", rlim.rlim_cur, errnoToString()) << '\n'; @@ -456,8 +461,8 @@ try } is_interactive = stdin_is_a_tty - && (config().hasOption("interactive") - || (queries.empty() && !config().has("table-structure") && queries_files.empty() && !config().has("table-file"))); + && (getClientConfiguration().hasOption("interactive") + || (queries.empty() && !getClientConfiguration().has("table-structure") && queries_files.empty() && !getClientConfiguration().has("table-file"))); if (!is_interactive) { @@ -481,7 +486,7 @@ try SCOPE_EXIT({ cleanup(); }); - initTTYBuffer(toProgressOption(config().getString("progress", "default"))); + initTTYBuffer(toProgressOption(getClientConfiguration().getString("progress", "default"))); ASTAlterCommand::setFormatAlterCommandsWithParentheses(true); applyCmdSettings(global_context); @@ -489,7 +494,7 @@ try /// try to load user defined executable functions, throw on error and die try { - global_context->loadOrReloadUserDefinedExecutableFunctions(config()); + global_context->loadOrReloadUserDefinedExecutableFunctions(getClientConfiguration()); } catch (...) { @@ -530,7 +535,7 @@ try } catch (const DB::Exception & e) { - bool need_print_stack_trace = config().getBool("stacktrace", false); + bool need_print_stack_trace = getClientConfiguration().getBool("stacktrace", false); std::cerr << getExceptionMessage(e, need_print_stack_trace, true) << std::endl; return e.code() ? e.code() : -1; } @@ -542,42 +547,42 @@ catch (...) void LocalServer::updateLoggerLevel(const String & logs_level) { - config().setString("logger.level", logs_level); - updateLevels(config(), logger()); + getClientConfiguration().setString("logger.level", logs_level); + updateLevels(getClientConfiguration(), logger()); } void LocalServer::processConfig() { - if (!queries.empty() && config().has("queries-file")) + if (!queries.empty() && getClientConfiguration().has("queries-file")) throw Exception(ErrorCodes::BAD_ARGUMENTS, "Options '--query' and '--queries-file' cannot be specified at the same time"); - if (config().has("multiquery")) + if (getClientConfiguration().has("multiquery")) is_multiquery = true; - pager = config().getString("pager", ""); + pager = getClientConfiguration().getString("pager", ""); - delayed_interactive = config().has("interactive") && (!queries.empty() || config().has("queries-file")); + delayed_interactive = getClientConfiguration().has("interactive") && (!queries.empty() || getClientConfiguration().has("queries-file")); if (!is_interactive || delayed_interactive) { - echo_queries = config().hasOption("echo") || config().hasOption("verbose"); - ignore_error = config().getBool("ignore-error", false); + echo_queries = getClientConfiguration().hasOption("echo") || getClientConfiguration().hasOption("verbose"); + ignore_error = getClientConfiguration().getBool("ignore-error", false); } - print_stack_trace = config().getBool("stacktrace", false); + print_stack_trace = getClientConfiguration().getBool("stacktrace", false); const std::string clickhouse_dialect{"clickhouse"}; - load_suggestions = (is_interactive || delayed_interactive) && !config().getBool("disable_suggestion", false) - && config().getString("dialect", clickhouse_dialect) == clickhouse_dialect; - wait_for_suggestions_to_load = config().getBool("wait_for_suggestions_to_load", false); + load_suggestions = (is_interactive || delayed_interactive) && !getClientConfiguration().getBool("disable_suggestion", false) + && getClientConfiguration().getString("dialect", clickhouse_dialect) == clickhouse_dialect; + wait_for_suggestions_to_load = getClientConfiguration().getBool("wait_for_suggestions_to_load", false); - auto logging = (config().has("logger.console") - || config().has("logger.level") - || config().has("log-level") - || config().has("send_logs_level") - || config().has("logger.log")); + auto logging = (getClientConfiguration().has("logger.console") + || getClientConfiguration().has("logger.level") + || getClientConfiguration().has("log-level") + || getClientConfiguration().has("send_logs_level") + || getClientConfiguration().has("logger.log")); - auto level = config().getString("log-level", "trace"); + auto level = getClientConfiguration().getString("log-level", "trace"); - if (config().has("server_logs_file")) + if (getClientConfiguration().has("server_logs_file")) { auto poco_logs_level = Poco::Logger::parseLevel(level); Poco::Logger::root().setLevel(poco_logs_level); @@ -587,10 +592,10 @@ void LocalServer::processConfig() } else { - config().setString("logger", "logger"); + getClientConfiguration().setString("logger", "logger"); auto log_level_default = logging ? level : "fatal"; - config().setString("logger.level", config().getString("log-level", config().getString("send_logs_level", log_level_default))); - buildLoggers(config(), logger(), "clickhouse-local"); + getClientConfiguration().setString("logger.level", getClientConfiguration().getString("log-level", getClientConfiguration().getString("send_logs_level", log_level_default))); + buildLoggers(getClientConfiguration(), logger(), "clickhouse-local"); } shared_context = Context::createShared(); @@ -604,13 +609,13 @@ void LocalServer::processConfig() LoggerRawPtr log = &logger(); /// Maybe useless - if (config().has("macros")) - global_context->setMacros(std::make_unique(config(), "macros", log)); + if (getClientConfiguration().has("macros")) + global_context->setMacros(std::make_unique(getClientConfiguration(), "macros", log)); setDefaultFormatsAndCompressionFromConfiguration(); /// Sets external authenticators config (LDAP, Kerberos). - global_context->setExternalAuthenticatorsConfig(config()); + global_context->setExternalAuthenticatorsConfig(getClientConfiguration()); setupUsers(); @@ -619,12 +624,12 @@ void LocalServer::processConfig() global_context->getProcessList().setMaxSize(0); const size_t physical_server_memory = getMemoryAmount(); - const double cache_size_to_ram_max_ratio = config().getDouble("cache_size_to_ram_max_ratio", 0.5); + const double cache_size_to_ram_max_ratio = getClientConfiguration().getDouble("cache_size_to_ram_max_ratio", 0.5); const size_t max_cache_size = static_cast(physical_server_memory * cache_size_to_ram_max_ratio); - String uncompressed_cache_policy = config().getString("uncompressed_cache_policy", DEFAULT_UNCOMPRESSED_CACHE_POLICY); - size_t uncompressed_cache_size = config().getUInt64("uncompressed_cache_size", DEFAULT_UNCOMPRESSED_CACHE_MAX_SIZE); - double uncompressed_cache_size_ratio = config().getDouble("uncompressed_cache_size_ratio", DEFAULT_UNCOMPRESSED_CACHE_SIZE_RATIO); + String uncompressed_cache_policy = getClientConfiguration().getString("uncompressed_cache_policy", DEFAULT_UNCOMPRESSED_CACHE_POLICY); + size_t uncompressed_cache_size = getClientConfiguration().getUInt64("uncompressed_cache_size", DEFAULT_UNCOMPRESSED_CACHE_MAX_SIZE); + double uncompressed_cache_size_ratio = getClientConfiguration().getDouble("uncompressed_cache_size_ratio", DEFAULT_UNCOMPRESSED_CACHE_SIZE_RATIO); if (uncompressed_cache_size > max_cache_size) { uncompressed_cache_size = max_cache_size; @@ -632,9 +637,9 @@ void LocalServer::processConfig() } global_context->setUncompressedCache(uncompressed_cache_policy, uncompressed_cache_size, uncompressed_cache_size_ratio); - String mark_cache_policy = config().getString("mark_cache_policy", DEFAULT_MARK_CACHE_POLICY); - size_t mark_cache_size = config().getUInt64("mark_cache_size", DEFAULT_MARK_CACHE_MAX_SIZE); - double mark_cache_size_ratio = config().getDouble("mark_cache_size_ratio", DEFAULT_MARK_CACHE_SIZE_RATIO); + String mark_cache_policy = getClientConfiguration().getString("mark_cache_policy", DEFAULT_MARK_CACHE_POLICY); + size_t mark_cache_size = getClientConfiguration().getUInt64("mark_cache_size", DEFAULT_MARK_CACHE_MAX_SIZE); + double mark_cache_size_ratio = getClientConfiguration().getDouble("mark_cache_size_ratio", DEFAULT_MARK_CACHE_SIZE_RATIO); if (!mark_cache_size) LOG_ERROR(log, "Too low mark cache size will lead to severe performance degradation."); if (mark_cache_size > max_cache_size) @@ -644,9 +649,9 @@ void LocalServer::processConfig() } global_context->setMarkCache(mark_cache_policy, mark_cache_size, mark_cache_size_ratio); - String index_uncompressed_cache_policy = config().getString("index_uncompressed_cache_policy", DEFAULT_INDEX_UNCOMPRESSED_CACHE_POLICY); - size_t index_uncompressed_cache_size = config().getUInt64("index_uncompressed_cache_size", DEFAULT_INDEX_UNCOMPRESSED_CACHE_MAX_SIZE); - double index_uncompressed_cache_size_ratio = config().getDouble("index_uncompressed_cache_size_ratio", DEFAULT_INDEX_UNCOMPRESSED_CACHE_SIZE_RATIO); + String index_uncompressed_cache_policy = getClientConfiguration().getString("index_uncompressed_cache_policy", DEFAULT_INDEX_UNCOMPRESSED_CACHE_POLICY); + size_t index_uncompressed_cache_size = getClientConfiguration().getUInt64("index_uncompressed_cache_size", DEFAULT_INDEX_UNCOMPRESSED_CACHE_MAX_SIZE); + double index_uncompressed_cache_size_ratio = getClientConfiguration().getDouble("index_uncompressed_cache_size_ratio", DEFAULT_INDEX_UNCOMPRESSED_CACHE_SIZE_RATIO); if (index_uncompressed_cache_size > max_cache_size) { index_uncompressed_cache_size = max_cache_size; @@ -654,9 +659,9 @@ void LocalServer::processConfig() } global_context->setIndexUncompressedCache(index_uncompressed_cache_policy, index_uncompressed_cache_size, index_uncompressed_cache_size_ratio); - String index_mark_cache_policy = config().getString("index_mark_cache_policy", DEFAULT_INDEX_MARK_CACHE_POLICY); - size_t index_mark_cache_size = config().getUInt64("index_mark_cache_size", DEFAULT_INDEX_MARK_CACHE_MAX_SIZE); - double index_mark_cache_size_ratio = config().getDouble("index_mark_cache_size_ratio", DEFAULT_INDEX_MARK_CACHE_SIZE_RATIO); + String index_mark_cache_policy = getClientConfiguration().getString("index_mark_cache_policy", DEFAULT_INDEX_MARK_CACHE_POLICY); + size_t index_mark_cache_size = getClientConfiguration().getUInt64("index_mark_cache_size", DEFAULT_INDEX_MARK_CACHE_MAX_SIZE); + double index_mark_cache_size_ratio = getClientConfiguration().getDouble("index_mark_cache_size_ratio", DEFAULT_INDEX_MARK_CACHE_SIZE_RATIO); if (index_mark_cache_size > max_cache_size) { index_mark_cache_size = max_cache_size; @@ -664,7 +669,7 @@ void LocalServer::processConfig() } global_context->setIndexMarkCache(index_mark_cache_policy, index_mark_cache_size, index_mark_cache_size_ratio); - size_t mmap_cache_size = config().getUInt64("mmap_cache_size", DEFAULT_MMAP_CACHE_MAX_SIZE); + size_t mmap_cache_size = getClientConfiguration().getUInt64("mmap_cache_size", DEFAULT_MMAP_CACHE_MAX_SIZE); if (mmap_cache_size > max_cache_size) { mmap_cache_size = max_cache_size; @@ -676,8 +681,8 @@ void LocalServer::processConfig() global_context->setQueryCache(0, 0, 0, 0); #if USE_EMBEDDED_COMPILER - size_t compiled_expression_cache_max_size_in_bytes = config().getUInt64("compiled_expression_cache_size", DEFAULT_COMPILED_EXPRESSION_CACHE_MAX_SIZE); - size_t compiled_expression_cache_max_elements = config().getUInt64("compiled_expression_cache_elements_size", DEFAULT_COMPILED_EXPRESSION_CACHE_MAX_ENTRIES); + size_t compiled_expression_cache_max_size_in_bytes = getClientConfiguration().getUInt64("compiled_expression_cache_size", DEFAULT_COMPILED_EXPRESSION_CACHE_MAX_SIZE); + size_t compiled_expression_cache_max_elements = getClientConfiguration().getUInt64("compiled_expression_cache_elements_size", DEFAULT_COMPILED_EXPRESSION_CACHE_MAX_ENTRIES); CompiledExpressionCacheFactory::instance().init(compiled_expression_cache_max_size_in_bytes, compiled_expression_cache_max_elements); #endif @@ -689,16 +694,16 @@ void LocalServer::processConfig() applyCmdOptions(global_context); /// Load global settings from default_profile and system_profile. - global_context->setDefaultProfiles(config()); + global_context->setDefaultProfiles(getClientConfiguration()); /// We load temporary database first, because projections need it. DatabaseCatalog::instance().initializeAndLoadTemporaryDatabase(); - std::string default_database = config().getString("default_database", "default"); + std::string default_database = getClientConfiguration().getString("default_database", "default"); DatabaseCatalog::instance().attachDatabase(default_database, createClickHouseLocalDatabaseOverlay(default_database, global_context)); global_context->setCurrentDatabase(default_database); - if (config().has("path")) + if (getClientConfiguration().has("path")) { String path = global_context->getPath(); fs::create_directories(fs::path(path)); @@ -713,7 +718,7 @@ void LocalServer::processConfig() attachInformationSchema(global_context, *createMemoryDatabaseIfNotExists(global_context, DatabaseCatalog::INFORMATION_SCHEMA_UPPERCASE)); waitLoad(TablesLoaderForegroundPoolId, startup_system_tasks); - if (!config().has("only-system-tables")) + if (!getClientConfiguration().has("only-system-tables")) { DatabaseCatalog::instance().createBackgroundTasks(); waitLoad(loadMetadata(global_context)); @@ -725,15 +730,15 @@ void LocalServer::processConfig() LOG_DEBUG(log, "Loaded metadata."); } - else if (!config().has("no-system-tables")) + else if (!getClientConfiguration().has("no-system-tables")) { attachSystemTablesServer(global_context, *createMemoryDatabaseIfNotExists(global_context, DatabaseCatalog::SYSTEM_DATABASE), false); attachInformationSchema(global_context, *createMemoryDatabaseIfNotExists(global_context, DatabaseCatalog::INFORMATION_SCHEMA)); attachInformationSchema(global_context, *createMemoryDatabaseIfNotExists(global_context, DatabaseCatalog::INFORMATION_SCHEMA_UPPERCASE)); } - server_display_name = config().getString("display_name", ""); - prompt_by_server_display_name = config().getRawString("prompt_by_server_display_name.default", ":) "); + server_display_name = getClientConfiguration().getString("display_name", ""); + prompt_by_server_display_name = getClientConfiguration().getRawString("prompt_by_server_display_name.default", ":) "); global_context->setQueryKindInitial(); global_context->setQueryKind(query_kind); @@ -811,7 +816,7 @@ void LocalServer::applyCmdSettings(ContextMutablePtr context) void LocalServer::applyCmdOptions(ContextMutablePtr context) { - context->setDefaultFormat(config().getString("output-format", config().getString("format", is_interactive ? "PrettyCompact" : "TSV"))); + context->setDefaultFormat(getClientConfiguration().getString("output-format", getClientConfiguration().getString("format", is_interactive ? "PrettyCompact" : "TSV"))); applyCmdSettings(context); } @@ -819,33 +824,33 @@ void LocalServer::applyCmdOptions(ContextMutablePtr context) void LocalServer::processOptions(const OptionsDescription &, const CommandLineOptions & options, const std::vector &, const std::vector &) { if (options.count("table")) - config().setString("table-name", options["table"].as()); + getClientConfiguration().setString("table-name", options["table"].as()); if (options.count("file")) - config().setString("table-file", options["file"].as()); + getClientConfiguration().setString("table-file", options["file"].as()); if (options.count("structure")) - config().setString("table-structure", options["structure"].as()); + getClientConfiguration().setString("table-structure", options["structure"].as()); if (options.count("no-system-tables")) - config().setBool("no-system-tables", true); + getClientConfiguration().setBool("no-system-tables", true); if (options.count("only-system-tables")) - config().setBool("only-system-tables", true); + getClientConfiguration().setBool("only-system-tables", true); if (options.count("database")) - config().setString("default_database", options["database"].as()); + getClientConfiguration().setString("default_database", options["database"].as()); if (options.count("input-format")) - config().setString("table-data-format", options["input-format"].as()); + getClientConfiguration().setString("table-data-format", options["input-format"].as()); if (options.count("output-format")) - config().setString("output-format", options["output-format"].as()); + getClientConfiguration().setString("output-format", options["output-format"].as()); if (options.count("logger.console")) - config().setBool("logger.console", options["logger.console"].as()); + getClientConfiguration().setBool("logger.console", options["logger.console"].as()); if (options.count("logger.log")) - config().setString("logger.log", options["logger.log"].as()); + getClientConfiguration().setString("logger.log", options["logger.log"].as()); if (options.count("logger.level")) - config().setString("logger.level", options["logger.level"].as()); + getClientConfiguration().setString("logger.level", options["logger.level"].as()); if (options.count("send_logs_level")) - config().setString("send_logs_level", options["send_logs_level"].as()); + getClientConfiguration().setString("send_logs_level", options["send_logs_level"].as()); if (options.count("wait_for_suggestions_to_load")) - config().setBool("wait_for_suggestions_to_load", true); + getClientConfiguration().setBool("wait_for_suggestions_to_load", true); } void LocalServer::readArguments(int argc, char ** argv, Arguments & common_arguments, std::vector &, std::vector &) diff --git a/programs/local/LocalServer.h b/programs/local/LocalServer.h index 4856e68ff9b..4ab09ffc353 100644 --- a/programs/local/LocalServer.h +++ b/programs/local/LocalServer.h @@ -30,6 +30,9 @@ public: int main(const std::vector & /*args*/) override; protected: + + Poco::Util::LayeredConfiguration & getClientConfiguration() override; + void connect() override; void processError(const String & query) const override; diff --git a/src/Client/ClientApplicationBase.h b/src/Client/ClientApplicationBase.h new file mode 100644 index 00000000000..f85e79c734d --- /dev/null +++ b/src/Client/ClientApplicationBase.h @@ -0,0 +1,16 @@ +#pragma once + +#include +#include +#include + +namespace DB +{ + + +class ClientApplicationBase : public ClientBase, public Poco::Util::Application, public IHints<2> +{ + +} + +} diff --git a/src/Client/ClientBase.cpp b/src/Client/ClientBase.cpp index 15a4836ef7a..b0b9c9107db 100644 --- a/src/Client/ClientBase.cpp +++ b/src/Client/ClientBase.cpp @@ -301,8 +301,29 @@ public: ClientBase::~ClientBase() = default; -ClientBase::ClientBase() = default; - +ClientBase::ClientBase( + int in_fd_, + int out_fd_, + int err_fd_, + std::istream & input_stream_, + std::ostream & output_stream_, + std::ostream & error_stream_ +) + : std_in(in_fd_) + , std_out(out_fd_) + , progress_indication(output_stream_, in_fd_, err_fd_) + , in_fd(in_fd_) + , out_fd(out_fd_) + , err_fd(err_fd_) + , input_stream(input_stream_) + , output_stream(output_stream_) + , error_stream(error_stream_) +{ + stdin_is_a_tty = isatty(in_fd); + stdout_is_a_tty = isatty(out_fd); + stderr_is_a_tty = isatty(err_fd); + terminal_width = getTerminalWidth(in_fd, err_fd); +} void ClientBase::setupSignalHandler() { @@ -329,7 +350,7 @@ void ClientBase::setupSignalHandler() } -ASTPtr ClientBase::parseQuery(const char *& pos, const char * end, const Settings & settings, bool allow_multi_statements, bool is_interactive, bool ignore_error) +ASTPtr ClientBase::parseQuery(const char *& pos, const char * end, const Settings & settings, bool allow_multi_statements) { std::unique_ptr parser; ASTPtr res; @@ -358,7 +379,7 @@ ASTPtr ClientBase::parseQuery(const char *& pos, const char * end, const Setting if (!res) { - std::cerr << std::endl << message << std::endl << std::endl; + error_stream << std::endl << message << std::endl << std::endl; return nullptr; } } @@ -372,11 +393,11 @@ ASTPtr ClientBase::parseQuery(const char *& pos, const char * end, const Setting if (is_interactive) { - std::cout << std::endl; - WriteBufferFromOStream res_buf(std::cout, 4096); + output_stream << std::endl; + WriteBufferFromOStream res_buf(output_stream, 4096); formatAST(*res, res_buf); res_buf.finalize(); - std::cout << std::endl << std::endl; + output_stream << std::endl << std::endl; } return res; @@ -480,7 +501,7 @@ void ClientBase::onData(Block & block, ASTPtr parsed_query) if (need_render_progress && tty_buf) { if (select_into_file && !select_into_file_and_stdout) - std::cerr << "\r"; + error_stream << "\r"; progress_indication.writeProgress(*tty_buf); } } @@ -740,17 +761,17 @@ bool ClientBase::isRegularFile(int fd) void ClientBase::setDefaultFormatsAndCompressionFromConfiguration() { - if (config().has("output-format")) + if (getClientConfiguration().has("output-format")) { - default_output_format = config().getString("output-format"); + default_output_format = getClientConfiguration().getString("output-format"); is_default_format = false; } - else if (config().has("format")) + else if (getClientConfiguration().has("format")) { - default_output_format = config().getString("format"); + default_output_format = getClientConfiguration().getString("format"); is_default_format = false; } - else if (config().has("vertical")) + else if (getClientConfiguration().has("vertical")) { default_output_format = "Vertical"; is_default_format = false; @@ -776,17 +797,17 @@ void ClientBase::setDefaultFormatsAndCompressionFromConfiguration() default_output_format = "TSV"; } - if (config().has("input-format")) + if (getClientConfiguration().has("input-format")) { - default_input_format = config().getString("input-format"); + default_input_format = getClientConfiguration().getString("input-format"); } - else if (config().has("format")) + else if (getClientConfiguration().has("format")) { - default_input_format = config().getString("format"); + default_input_format = getClientConfiguration().getString("format"); } - else if (config().getString("table-file", "-") != "-") + else if (getClientConfiguration().getString("table-file", "-") != "-") { - auto file_name = config().getString("table-file"); + auto file_name = getClientConfiguration().getString("table-file"); std::optional format_from_file_name = FormatFactory::instance().tryGetFormatFromFileName(file_name); if (format_from_file_name) default_input_format = *format_from_file_name; @@ -802,7 +823,7 @@ void ClientBase::setDefaultFormatsAndCompressionFromConfiguration() default_input_format = "TSV"; } - format_max_block_size = config().getUInt64("format_max_block_size", + format_max_block_size = getClientConfiguration().getUInt64("format_max_block_size", global_context->getSettingsRef().max_block_size); /// Setting value from cmd arg overrides one from config @@ -812,7 +833,7 @@ void ClientBase::setDefaultFormatsAndCompressionFromConfiguration() } else { - insert_format_max_block_size = config().getUInt64("insert_format_max_block_size", + insert_format_max_block_size = getClientConfiguration().getUInt64("insert_format_max_block_size", global_context->getSettingsRef().max_insert_block_size); } } @@ -923,9 +944,7 @@ void ClientBase::processTextAsSingleQuery(const String & full_query) const char * begin = full_query.data(); auto parsed_query = parseQuery(begin, begin + full_query.size(), global_context->getSettingsRef(), - /*allow_multi_statements=*/ false, - is_interactive, - ignore_error); + /*allow_multi_statements=*/ false); if (!parsed_query) return; @@ -1099,7 +1118,7 @@ void ClientBase::processOrdinaryQuery(const String & query_to_execute, ASTPtr pa /// has been received yet. if (processed_rows == 0 && e.code() == ErrorCodes::DEADLOCK_AVOIDED && --retries_left) { - std::cerr << "Got a transient error from the server, will" + error_stream << "Got a transient error from the server, will" << " retry (" << retries_left << " retries left)"; } else @@ -1153,7 +1172,7 @@ void ClientBase::receiveResult(ASTPtr parsed_query, Int32 signals_before_stop, b double elapsed = receive_watch.elapsedSeconds(); if (break_on_timeout && elapsed > receive_timeout.totalSeconds()) { - std::cout << "Timeout exceeded while receiving data from server." + output_stream << "Timeout exceeded while receiving data from server." << " Waited for " << static_cast(elapsed) << " seconds," << " timeout is " << receive_timeout.totalSeconds() << " seconds." << std::endl; @@ -1188,7 +1207,7 @@ void ClientBase::receiveResult(ASTPtr parsed_query, Int32 signals_before_stop, b if (cancelled && is_interactive) { - std::cout << "Query was cancelled." << std::endl; + output_stream << "Query was cancelled." << std::endl; cancelled_printed = true; } } @@ -1307,9 +1326,9 @@ void ClientBase::onEndOfStream() if (is_interactive) { if (cancelled && !cancelled_printed) - std::cout << "Query was cancelled." << std::endl; + output_stream << "Query was cancelled." << std::endl; else if (!written_first_block) - std::cout << "Ok." << std::endl; + output_stream << "Ok." << std::endl; } } @@ -1862,7 +1881,7 @@ void ClientBase::cancelQuery() progress_indication.clearProgressOutput(*tty_buf); if (is_interactive) - std::cout << "Cancelling query." << std::endl; + output_stream << "Cancelling query." << std::endl; cancelled = true; } @@ -2025,7 +2044,7 @@ void ClientBase::processParsedSingleQuery(const String & full_query, const Strin { const String & new_database = use_query->getDatabase(); /// If the client initiates the reconnection, it takes the settings from the config. - config().setString("database", new_database); + getClientConfiguration().setString("database", new_database); /// If the connection initiates the reconnection, it uses its variable. connection->setDefaultDatabase(new_database); } @@ -2045,21 +2064,21 @@ void ClientBase::processParsedSingleQuery(const String & full_query, const Strin if (is_interactive) { - std::cout << std::endl; + output_stream << std::endl; if (!server_exception || processed_rows != 0) - std::cout << processed_rows << " row" << (processed_rows == 1 ? "" : "s") << " in set. "; - std::cout << "Elapsed: " << progress_indication.elapsedSeconds() << " sec. "; + output_stream << processed_rows << " row" << (processed_rows == 1 ? "" : "s") << " in set. "; + output_stream << "Elapsed: " << progress_indication.elapsedSeconds() << " sec. "; progress_indication.writeFinalProgress(); - std::cout << std::endl << std::endl; + output_stream << std::endl << std::endl; } - else if (print_time_to_stderr) + else if (getClientConfiguration().getBool("print-time-to-stderr", false)) { - std::cerr << progress_indication.elapsedSeconds() << "\n"; + error_stream << progress_indication.elapsedSeconds() << "\n"; } - if (!is_interactive && print_num_processed_rows) + if (!is_interactive && getClientConfiguration().getBool("print-num-processed-rows", false)) { - std::cout << "Processed rows: " << processed_rows << "\n"; + output_stream << "Processed rows: " << processed_rows << "\n"; } if (have_error && report_error) @@ -2109,9 +2128,7 @@ MultiQueryProcessingStage ClientBase::analyzeMultiQueryText( { parsed_query = parseQuery(this_query_end, all_queries_end, global_context->getSettingsRef(), - /*allow_multi_statements=*/ true, - is_interactive, - ignore_error); + /*allow_multi_statements=*/ true); } catch (const Exception & e) { @@ -2427,12 +2444,12 @@ void ClientBase::initQueryIdFormats() return; /// Initialize query_id_formats if any - if (config().has("query_id_formats")) + if (getClientConfiguration().has("query_id_formats")) { Poco::Util::AbstractConfiguration::Keys keys; - config().keys("query_id_formats", keys); + getClientConfiguration().keys("query_id_formats", keys); for (const auto & name : keys) - query_id_formats.emplace_back(name + ":", config().getString("query_id_formats." + name)); + query_id_formats.emplace_back(name + ":", getClientConfiguration().getString("query_id_formats." + name)); } if (query_id_formats.empty()) @@ -2477,9 +2494,9 @@ bool ClientBase::addMergeTreeSettings(ASTCreateQuery & ast_create) void ClientBase::runInteractive() { - if (config().has("query_id")) + if (getClientConfiguration().has("query_id")) throw Exception(ErrorCodes::BAD_ARGUMENTS, "query_id could be specified only in non-interactive mode"); - if (print_time_to_stderr) + if (getClientConfiguration().getBool("print-time-to-stderr", false)) throw Exception(ErrorCodes::BAD_ARGUMENTS, "time option could be specified only in non-interactive mode"); initQueryIdFormats(); @@ -2492,9 +2509,9 @@ void ClientBase::runInteractive() { /// Load suggestion data from the server. if (global_context->getApplicationType() == Context::ApplicationType::CLIENT) - suggest->load(global_context, connection_parameters, config().getInt("suggestion_limit"), wait_for_suggestions_to_load); + suggest->load(global_context, connection_parameters, getClientConfiguration().getInt("suggestion_limit"), wait_for_suggestions_to_load); else if (global_context->getApplicationType() == Context::ApplicationType::LOCAL) - suggest->load(global_context, connection_parameters, config().getInt("suggestion_limit"), wait_for_suggestions_to_load); + suggest->load(global_context, connection_parameters, getClientConfiguration().getInt("suggestion_limit"), wait_for_suggestions_to_load); } if (home_path.empty()) @@ -2505,8 +2522,8 @@ void ClientBase::runInteractive() } /// Load command history if present. - if (config().has("history_file")) - history_file = config().getString("history_file"); + if (getClientConfiguration().has("history_file")) + history_file = getClientConfiguration().getString("history_file"); else { auto * history_file_from_env = getenv("CLICKHOUSE_HISTORY_FILE"); // NOLINT(concurrency-mt-unsafe) @@ -2527,7 +2544,7 @@ void ClientBase::runInteractive() { if (e.getErrno() != EEXIST) { - std::cerr << getCurrentExceptionMessage(false) << '\n'; + error_stream << getCurrentExceptionMessage(false) << '\n'; } } } @@ -2538,13 +2555,13 @@ void ClientBase::runInteractive() #if USE_REPLXX replxx::Replxx::highlighter_callback_t highlight_callback{}; - if (config().getBool("highlight", true)) + if (getClientConfiguration().getBool("highlight", true)) highlight_callback = highlight; ReplxxLineReader lr( *suggest, history_file, - config().has("multiline"), + getClientConfiguration().has("multiline"), query_extenders, query_delimiters, word_break_characters, @@ -2552,7 +2569,7 @@ void ClientBase::runInteractive() #else LineReader lr( history_file, - config().has("multiline"), + getClientConfiguration().has("multiline"), query_extenders, query_delimiters, word_break_characters); @@ -2632,7 +2649,7 @@ void ClientBase::runInteractive() { // If a separate connection loading suggestions failed to open a new session, // use the main session to receive them. - suggest->load(*connection, connection_parameters.timeouts, config().getInt("suggestion_limit"), global_context->getClientInfo()); + suggest->load(*connection, connection_parameters.timeouts, getClientConfiguration().getInt("suggestion_limit"), global_context->getClientInfo()); } try @@ -2644,7 +2661,7 @@ void ClientBase::runInteractive() catch (const Exception & e) { /// We don't need to handle the test hints in the interactive mode. - std::cerr << "Exception on client:" << std::endl << getExceptionMessage(e, print_stack_trace, true) << std::endl << std::endl; + error_stream << "Exception on client:" << std::endl << getExceptionMessage(e, print_stack_trace, true) << std::endl << std::endl; client_exception.reset(e.clone()); } @@ -2661,11 +2678,11 @@ void ClientBase::runInteractive() while (true); if (isNewYearMode()) - std::cout << "Happy new year." << std::endl; + output_stream << "Happy new year." << std::endl; else if (isChineseNewYearMode(local_tz)) - std::cout << "Happy Chinese new year. 春节快乐!" << std::endl; + output_stream << "Happy Chinese new year. 春节快乐!" << std::endl; else - std::cout << "Bye." << std::endl; + output_stream << "Bye." << std::endl; } @@ -2676,7 +2693,7 @@ bool ClientBase::processMultiQueryFromFile(const String & file_name) ReadBufferFromFile in(file_name); readStringUntilEOF(queries_from_file, in); - if (!has_log_comment) + if (!getClientConfiguration().has("log_comment")) { Settings settings = global_context->getSettings(); /// NOTE: cannot use even weakly_canonical() since it fails for /dev/stdin due to resolving of "pipe:[X]" @@ -2785,13 +2802,13 @@ void ClientBase::clearTerminal() /// It is needed if garbage is left in terminal. /// Show cursor. It can be left hidden by invocation of previous programs. /// A test for this feature: perl -e 'print "x"x100000'; echo -ne '\033[0;0H\033[?25l'; clickhouse-client - std::cout << "\033[0J" "\033[?25h"; + output_stream << "\033[0J" "\033[?25h"; } void ClientBase::showClientVersion() { - std::cout << VERSION_NAME << " " + getName() + " version " << VERSION_STRING << VERSION_OFFICIAL << "." << std::endl; + output_stream << VERSION_NAME << " " + getName() + " version " << VERSION_STRING << VERSION_OFFICIAL << "." << std::endl; } namespace @@ -3076,18 +3093,18 @@ void ClientBase::init(int argc, char ** argv) if (options.count("version-clean")) { - std::cout << VERSION_STRING; + output_stream << VERSION_STRING; exit(0); // NOLINT(concurrency-mt-unsafe) } if (options.count("verbose")) - config().setBool("verbose", true); + getClientConfiguration().setBool("verbose", true); /// Output of help message. if (options.count("help") || (options.count("host") && options["host"].as() == "elp")) /// If user writes -help instead of --help. { - if (config().getBool("verbose", false)) + if (getClientConfiguration().getBool("verbose", false)) printHelpMessage(options_description, true); else printHelpMessage(options_description_non_verbose, false); @@ -3095,72 +3112,75 @@ void ClientBase::init(int argc, char ** argv) } /// Common options for clickhouse-client and clickhouse-local. + + /// Output execution time to stderr in batch mode. if (options.count("time")) - print_time_to_stderr = true; + getClientConfiguration().setBool("print-time-to-stderr", true); if (options.count("query")) queries = options["query"].as>(); if (options.count("query_id")) - config().setString("query_id", options["query_id"].as()); + getClientConfiguration().setString("query_id", options["query_id"].as()); if (options.count("database")) - config().setString("database", options["database"].as()); + getClientConfiguration().setString("database", options["database"].as()); if (options.count("config-file")) - config().setString("config-file", options["config-file"].as()); + getClientConfiguration().setString("config-file", options["config-file"].as()); if (options.count("queries-file")) queries_files = options["queries-file"].as>(); if (options.count("multiline")) - config().setBool("multiline", true); + getClientConfiguration().setBool("multiline", true); if (options.count("multiquery")) - config().setBool("multiquery", true); + getClientConfiguration().setBool("multiquery", true); if (options.count("ignore-error")) - config().setBool("ignore-error", true); + getClientConfiguration().setBool("ignore-error", true); if (options.count("format")) - config().setString("format", options["format"].as()); + getClientConfiguration().setString("format", options["format"].as()); if (options.count("output-format")) - config().setString("output-format", options["output-format"].as()); + getClientConfiguration().setString("output-format", options["output-format"].as()); if (options.count("vertical")) - config().setBool("vertical", true); + getClientConfiguration().setBool("vertical", true); if (options.count("stacktrace")) - config().setBool("stacktrace", true); + getClientConfiguration().setBool("stacktrace", true); if (options.count("print-profile-events")) - config().setBool("print-profile-events", true); + getClientConfiguration().setBool("print-profile-events", true); if (options.count("profile-events-delay-ms")) - config().setUInt64("profile-events-delay-ms", options["profile-events-delay-ms"].as()); + getClientConfiguration().setUInt64("profile-events-delay-ms", options["profile-events-delay-ms"].as()); + /// Whether to print the number of processed rows at if (options.count("processed-rows")) - print_num_processed_rows = true; + getClientConfiguration().setBool("print-num-processed-rows", true); if (options.count("progress")) { switch (options["progress"].as()) { case DEFAULT: - config().setString("progress", "default"); + getClientConfiguration().setString("progress", "default"); break; case OFF: - config().setString("progress", "off"); + getClientConfiguration().setString("progress", "off"); break; case TTY: - config().setString("progress", "tty"); + getClientConfiguration().setString("progress", "tty"); break; case ERR: - config().setString("progress", "err"); + getClientConfiguration().setString("progress", "err"); break; } } if (options.count("echo")) - config().setBool("echo", true); + getClientConfiguration().setBool("echo", true); if (options.count("disable_suggestion")) - config().setBool("disable_suggestion", true); + getClientConfiguration().setBool("disable_suggestion", true); if (options.count("wait_for_suggestions_to_load")) - config().setBool("wait_for_suggestions_to_load", true); + getClientConfiguration().setBool("wait_for_suggestions_to_load", true); if (options.count("suggestion_limit")) - config().setInt("suggestion_limit", options["suggestion_limit"].as()); + getClientConfiguration().setInt("suggestion_limit", options["suggestion_limit"].as()); if (options.count("highlight")) - config().setBool("highlight", options["highlight"].as()); + getClientConfiguration().setBool("highlight", options["highlight"].as()); if (options.count("history_file")) - config().setString("history_file", options["history_file"].as()); + getClientConfiguration().setString("history_file", options["history_file"].as()); if (options.count("interactive")) - config().setBool("interactive", true); + getClientConfiguration().setBool("interactive", true); if (options.count("pager")) - config().setString("pager", options["pager"].as()); + getClientConfiguration().setString("pager", options["pager"].as()); if (options.count("log-level")) Poco::Logger::root().setLevel(options["log-level"].as()); @@ -3178,13 +3198,13 @@ void ClientBase::init(int argc, char ** argv) alias_names.reserve(options_description.main_description->options().size()); for (const auto& option : options_description.main_description->options()) alias_names.insert(option->long_name()); - argsToConfig(common_arguments, config(), 100, &alias_names); + argsToConfig(common_arguments, getClientConfiguration(), 100, &alias_names); } clearPasswordFromCommandLine(argc, argv); /// Limit on total memory usage - std::string max_client_memory_usage = config().getString("max_memory_usage_in_client", "0" /*default value*/); + std::string max_client_memory_usage = getClientConfiguration().getString("max_memory_usage_in_client", "0" /*default value*/); if (max_client_memory_usage != "0") { UInt64 max_client_memory_usage_int = parseWithSizeSuffix(max_client_memory_usage.c_str(), max_client_memory_usage.length()); @@ -3193,8 +3213,6 @@ void ClientBase::init(int argc, char ** argv) total_memory_tracker.setDescription("(total)"); total_memory_tracker.setMetric(CurrentMetrics::MemoryTracking); } - - has_log_comment = config().has("log_comment"); } } diff --git a/src/Client/ClientBase.h b/src/Client/ClientBase.h index 220fcddc038..ed8768e7998 100644 --- a/src/Client/ClientBase.h +++ b/src/Client/ClientBase.h @@ -18,7 +18,6 @@ #include #include - namespace po = boost::program_options; @@ -67,13 +66,22 @@ class ClientBase : public Poco::Util::Application, public IHints<2> public: using Arguments = std::vector; - ClientBase(); + explicit ClientBase + ( + int in_fd_ = STDIN_FILENO, + int out_fd_ = STDOUT_FILENO, + int err_fd_ = STDERR_FILENO, + std::istream & input_stream_ = std::cin, + std::ostream & output_stream_ = std::cout, + std::ostream & error_stream_ = std::cerr + ); + ~ClientBase() override; void init(int argc, char ** argv); std::vector getAllRegisteredNames() const override { return cmd_options; } - static ASTPtr parseQuery(const char *& pos, const char * end, const Settings & settings, bool allow_multi_statements, bool is_interactive, bool ignore_error); + ASTPtr parseQuery(const char *& pos, const char * end, const Settings & settings, bool allow_multi_statements); protected: void runInteractive(); @@ -82,6 +90,9 @@ protected: char * argv0 = nullptr; void runLibFuzzer(); + /// This is the analogue of Poco::Application::config() + virtual Poco::Util::LayeredConfiguration & getClientConfiguration() = 0; + virtual bool processWithFuzzing(const String &) { throw Exception(ErrorCodes::NOT_IMPLEMENTED, "Query processing with fuzzing is not implemented"); @@ -107,7 +118,7 @@ protected: String & query_to_execute, ASTPtr & parsed_query, const String & all_queries_text, std::unique_ptr & current_exception); - static void clearTerminal(); + void clearTerminal(); void showClientVersion(); using ProgramOptionsDescription = boost::program_options::options_description; @@ -205,7 +216,6 @@ protected: bool echo_queries = false; /// Print queries before execution in batch mode. bool ignore_error = false; /// In case of errors, don't print error message, continue to next query. Only applicable for non-interactive mode. - bool print_time_to_stderr = false; /// Output execution time to stderr in batch mode. std::optional suggest; bool load_suggestions = false; @@ -250,9 +260,9 @@ protected: ConnectionParameters connection_parameters; /// Buffer that reads from stdin in batch mode. - ReadBufferFromFileDescriptor std_in{STDIN_FILENO}; + ReadBufferFromFileDescriptor std_in; /// Console output. - WriteBufferFromFileDescriptor std_out{STDOUT_FILENO}; + WriteBufferFromFileDescriptor std_out; std::unique_ptr pager_cmd; /// The user can specify to redirect query output to a file. @@ -283,7 +293,6 @@ protected: bool need_render_profile_events = true; bool written_first_block = false; size_t processed_rows = 0; /// How many rows have been read or written. - bool print_num_processed_rows = false; /// Whether to print the number of processed rows at bool print_stack_trace = false; /// The last exception that was received from the server. Is used for the @@ -331,8 +340,14 @@ protected: bool cancelled = false; bool cancelled_printed = false; - /// Does log_comment has specified by user? - bool has_log_comment = false; + /// Unpacked descriptors and streams for the ease of use. + int in_fd = STDIN_FILENO; + int out_fd = STDOUT_FILENO; + int err_fd = STDERR_FILENO; + std::istream & input_stream; + std::ostream & output_stream; + std::ostream & error_stream; + }; } diff --git a/src/Client/LineReader.cpp b/src/Client/LineReader.cpp index b3559657ced..487ef232fdc 100644 --- a/src/Client/LineReader.cpp +++ b/src/Client/LineReader.cpp @@ -23,14 +23,6 @@ void trim(String & s) s.erase(std::find_if(s.rbegin(), s.rend(), [](int ch) { return !std::isspace(ch); }).base(), s.end()); } -/// Check if multi-line query is inserted from the paste buffer. -/// Allows delaying the start of query execution until the entirety of query is inserted. -bool hasInputData() -{ - pollfd fd{STDIN_FILENO, POLLIN, 0}; - return poll(&fd, 1, 0) == 1; -} - struct NoCaseCompare { bool operator()(const std::string & str1, const std::string & str2) @@ -63,6 +55,14 @@ void addNewWords(Words & to, const Words & from, Compare comp) namespace DB { +/// Check if multi-line query is inserted from the paste buffer. +/// Allows delaying the start of query execution until the entirety of query is inserted. +bool LineReader::hasInputData() const +{ + pollfd fd{in_fd, POLLIN, 0}; + return poll(&fd, 1, 0) == 1; +} + replxx::Replxx::completions_t LineReader::Suggest::getCompletions(const String & prefix, size_t prefix_length, const char * word_break_characters) { std::string_view last_word; @@ -131,11 +131,22 @@ void LineReader::Suggest::addWords(Words && new_words) // NOLINT(cppcoreguidelin } } -LineReader::LineReader(const String & history_file_path_, bool multiline_, Patterns extenders_, Patterns delimiters_) +LineReader::LineReader( + const String & history_file_path_, + bool multiline_, + Patterns extenders_, + Patterns delimiters_, + std::istream & input_stream_, + std::ostream & output_stream_, + int in_fd_ +) : history_file_path(history_file_path_) , multiline(multiline_) , extenders(std::move(extenders_)) , delimiters(std::move(delimiters_)) + , input_stream(input_stream_) + , output_stream(output_stream_) + , in_fd(in_fd_) { /// FIXME: check extender != delimiter } @@ -212,9 +223,9 @@ LineReader::InputStatus LineReader::readOneLine(const String & prompt) input.clear(); { - std::cout << prompt; - std::getline(std::cin, input); - if (!std::cin.good()) + output_stream << prompt; + std::getline(input_stream, input); + if (!input_stream.good()) return ABORT; } diff --git a/src/Client/LineReader.h b/src/Client/LineReader.h index fc19eaa5667..0172bd7ec22 100644 --- a/src/Client/LineReader.h +++ b/src/Client/LineReader.h @@ -1,5 +1,7 @@ #pragma once +#include +#include #include #include #include @@ -37,7 +39,16 @@ public: using Patterns = std::vector; - LineReader(const String & history_file_path, bool multiline, Patterns extenders, Patterns delimiters); + LineReader( + const String & history_file_path, + bool multiline, + Patterns extenders, + Patterns delimiters, + std::istream & input_stream_ = std::cin, + std::ostream & output_stream_ = std::cout, + int in_fd_ = STDIN_FILENO + ); + virtual ~LineReader() = default; /// Reads the whole line until delimiter (in multiline mode) or until the last line without extender. @@ -56,6 +67,8 @@ public: virtual void enableBracketedPaste() {} virtual void disableBracketedPaste() {} + bool hasInputData() const; + protected: enum InputStatus { @@ -77,6 +90,10 @@ protected: virtual InputStatus readOneLine(const String & prompt); virtual void addToHistory(const String &) {} + + std::istream & input_stream; + std::ostream & output_stream; + int in_fd; }; } diff --git a/src/Client/LocalConnection.cpp b/src/Client/LocalConnection.cpp index c7494e31605..f5fda7ec0e7 100644 --- a/src/Client/LocalConnection.cpp +++ b/src/Client/LocalConnection.cpp @@ -16,6 +16,11 @@ #include #include #include +#include +#include +#include +#include + namespace DB @@ -151,12 +156,26 @@ void LocalConnection::sendQuery( state->block = sample; String current_format = "Values"; + + const auto & settings = context->getSettingsRef(); const char * begin = state->query.data(); - auto parsed_query = ClientBase::parseQuery(begin, begin + state->query.size(), - context->getSettingsRef(), - /*allow_multi_statements=*/ false, - /*is_interactive=*/ false, - /*ignore_error=*/ false); + const char * end = begin + state->query.size(); + const Dialect & dialect = settings.dialect; + + std::unique_ptr parser; + if (dialect == Dialect::kusto) + parser = std::make_unique(end, settings.allow_settings_after_format_in_insert); + else if (dialect == Dialect::prql) + parser = std::make_unique(settings.max_query_size, settings.max_parser_depth, settings.max_parser_backtracks); + else + parser = std::make_unique(end, settings.allow_settings_after_format_in_insert); + + ASTPtr parsed_query; + if (dialect == Dialect::kusto) + parsed_query = parseKQLQueryAndMovePosition(*parser, begin, end, "", /*allow_multi_statements*/false, settings.max_query_size, settings.max_parser_depth, settings.max_parser_backtracks); + else + parsed_query = parseQueryAndMovePosition(*parser, begin, end, "", /*allow_multi_statements*/false, settings.max_query_size, settings.max_parser_depth, settings.max_parser_backtracks); + if (const auto * insert = parsed_query->as()) { if (!insert->format.empty()) diff --git a/src/Client/ReplxxLineReader.cpp b/src/Client/ReplxxLineReader.cpp index 9e0f5946205..46600168695 100644 --- a/src/Client/ReplxxLineReader.cpp +++ b/src/Client/ReplxxLineReader.cpp @@ -297,8 +297,15 @@ ReplxxLineReader::ReplxxLineReader( Patterns extenders_, Patterns delimiters_, const char word_break_characters_[], - replxx::Replxx::highlighter_callback_t highlighter_) - : LineReader(history_file_path_, multiline_, std::move(extenders_), std::move(delimiters_)), highlighter(std::move(highlighter_)) + replxx::Replxx::highlighter_callback_t highlighter_, + [[ maybe_unused ]] std::istream & input_stream_, + [[ maybe_unused ]] std::ostream & output_stream_, + [[ maybe_unused ]] int in_fd_, + [[ maybe_unused ]] int out_fd_, + [[ maybe_unused ]] int err_fd_ +) + : LineReader(history_file_path_, multiline_, std::move(extenders_), std::move(delimiters_), input_stream_, output_stream_, in_fd_) + , highlighter(std::move(highlighter_)) , word_break_characters(word_break_characters_) , editor(getEditor()) { @@ -471,7 +478,7 @@ ReplxxLineReader::ReplxxLineReader( ReplxxLineReader::~ReplxxLineReader() { - if (close(history_file_fd)) + if (history_file_fd >= 0 && close(history_file_fd)) rx.print("Close of history file failed: %s\n", errnoToString().c_str()); } @@ -496,7 +503,7 @@ void ReplxxLineReader::addToHistory(const String & line) // but replxx::Replxx::history_load() does not // and that is why flock() is added here. bool locked = false; - if (flock(history_file_fd, LOCK_EX)) + if (history_file_fd >= 0 && flock(history_file_fd, LOCK_EX)) rx.print("Lock of history file failed: %s\n", errnoToString().c_str()); else locked = true; @@ -507,7 +514,7 @@ void ReplxxLineReader::addToHistory(const String & line) if (!rx.history_save(history_file_path)) rx.print("Saving history failed: %s\n", errnoToString().c_str()); - if (locked && 0 != flock(history_file_fd, LOCK_UN)) + if (history_file_fd >= 0 && locked && 0 != flock(history_file_fd, LOCK_UN)) rx.print("Unlock of history file failed: %s\n", errnoToString().c_str()); } diff --git a/src/Client/ReplxxLineReader.h b/src/Client/ReplxxLineReader.h index 6ad149e38f2..c46080420ef 100644 --- a/src/Client/ReplxxLineReader.h +++ b/src/Client/ReplxxLineReader.h @@ -1,6 +1,7 @@ #pragma once -#include "LineReader.h" +#include +#include #include namespace DB @@ -9,14 +10,22 @@ namespace DB class ReplxxLineReader : public LineReader { public: - ReplxxLineReader( + ReplxxLineReader + ( Suggest & suggest, const String & history_file_path, bool multiline, Patterns extenders_, Patterns delimiters_, const char word_break_characters_[], - replxx::Replxx::highlighter_callback_t highlighter_); + replxx::Replxx::highlighter_callback_t highlighter_, + std::istream & input_stream_ = std::cin, + std::ostream & output_stream_ = std::cout, + int in_fd_ = STDIN_FILENO, + int out_fd_ = STDOUT_FILENO, + int err_fd_ = STDERR_FILENO + ); + ~ReplxxLineReader() override; void enableBracketedPaste() override; diff --git a/src/Common/ProgressIndication.cpp b/src/Common/ProgressIndication.cpp index 7b07c72824a..0b482cb09be 100644 --- a/src/Common/ProgressIndication.cpp +++ b/src/Common/ProgressIndication.cpp @@ -92,19 +92,19 @@ void ProgressIndication::writeFinalProgress() if (progress.read_rows < 1000) return; - std::cout << "Processed " << formatReadableQuantity(progress.read_rows) << " rows, " + output_stream << "Processed " << formatReadableQuantity(progress.read_rows) << " rows, " << formatReadableSizeWithDecimalSuffix(progress.read_bytes); UInt64 elapsed_ns = getElapsedNanoseconds(); if (elapsed_ns) - std::cout << " (" << formatReadableQuantity(progress.read_rows * 1000000000.0 / elapsed_ns) << " rows/s., " + output_stream << " (" << formatReadableQuantity(progress.read_rows * 1000000000.0 / elapsed_ns) << " rows/s., " << formatReadableSizeWithDecimalSuffix(progress.read_bytes * 1000000000.0 / elapsed_ns) << "/s.)"; else - std::cout << ". "; + output_stream << ". "; auto peak_memory_usage = getMemoryUsage().peak; if (peak_memory_usage >= 0) - std::cout << "\nPeak memory usage: " << formatReadableSizeWithBinarySuffix(peak_memory_usage) << "."; + output_stream << "\nPeak memory usage: " << formatReadableSizeWithBinarySuffix(peak_memory_usage) << "."; } void ProgressIndication::writeProgress(WriteBufferFromFileDescriptor & message) @@ -125,7 +125,7 @@ void ProgressIndication::writeProgress(WriteBufferFromFileDescriptor & message) const char * indicator = indicators[increment % 8]; - size_t terminal_width = getTerminalWidth(); + size_t terminal_width = getTerminalWidth(in_fd, err_fd); if (!written_progress_chars) { diff --git a/src/Common/ProgressIndication.h b/src/Common/ProgressIndication.h index a9965785889..ae39fb49bcc 100644 --- a/src/Common/ProgressIndication.h +++ b/src/Common/ProgressIndication.h @@ -32,6 +32,19 @@ using HostToTimesMap = std::unordered_map; class ProgressIndication { public: + + explicit ProgressIndication + ( + std::ostream & output_stream_ = std::cout, + int in_fd_ = STDIN_FILENO, + int err_fd_ = STDERR_FILENO + ) + : output_stream(output_stream_), + in_fd(in_fd_), + err_fd(err_fd_) + { + } + /// Write progress bar. void writeProgress(WriteBufferFromFileDescriptor & message); void clearProgressOutput(WriteBufferFromFileDescriptor & message); @@ -103,6 +116,10 @@ private: /// - hosts_data/cpu_usage_meter (guarded with profile_events_mutex) mutable std::mutex profile_events_mutex; mutable std::mutex progress_mutex; + + std::ostream & output_stream; + int in_fd; + int err_fd; }; } diff --git a/src/Common/TerminalSize.cpp b/src/Common/TerminalSize.cpp index bc5b4474384..8139f4f7616 100644 --- a/src/Common/TerminalSize.cpp +++ b/src/Common/TerminalSize.cpp @@ -13,17 +13,17 @@ namespace DB::ErrorCodes extern const int SYSTEM_ERROR; } -uint16_t getTerminalWidth() +uint16_t getTerminalWidth(int in_fd, int err_fd) { struct winsize terminal_size {}; - if (isatty(STDIN_FILENO)) + if (isatty(in_fd)) { - if (ioctl(STDIN_FILENO, TIOCGWINSZ, &terminal_size)) + if (ioctl(in_fd, TIOCGWINSZ, &terminal_size)) throw DB::ErrnoException(DB::ErrorCodes::SYSTEM_ERROR, "Cannot obtain terminal window size (ioctl TIOCGWINSZ)"); } - else if (isatty(STDERR_FILENO)) + else if (isatty(err_fd)) { - if (ioctl(STDERR_FILENO, TIOCGWINSZ, &terminal_size)) + if (ioctl(err_fd, TIOCGWINSZ, &terminal_size)) throw DB::ErrnoException(DB::ErrorCodes::SYSTEM_ERROR, "Cannot obtain terminal window size (ioctl TIOCGWINSZ)"); } /// Default - 0. diff --git a/src/Common/TerminalSize.h b/src/Common/TerminalSize.h index b5fc6de7921..f1334f2bcb9 100644 --- a/src/Common/TerminalSize.h +++ b/src/Common/TerminalSize.h @@ -1,16 +1,16 @@ #pragma once #include +#include #include namespace po = boost::program_options; -uint16_t getTerminalWidth(); +uint16_t getTerminalWidth(int in_fd = STDIN_FILENO, int err_fd = STDERR_FILENO); /** Creates po::options_description with name and an appropriate size for option displaying * when program is called with option --help * */ po::options_description createOptionsDescription(const std::string &caption, unsigned short terminal_width); /// NOLINT - From 96866864a687c22bba4e3c7e932b32b6e3813b1e Mon Sep 17 00:00:00 2001 From: Nikita Mikhaylov Date: Tue, 25 Jun 2024 14:25:59 +0000 Subject: [PATCH 061/115] Remove unndeded file --- src/Client/ClientApplicationBase.h | 16 ---------------- 1 file changed, 16 deletions(-) delete mode 100644 src/Client/ClientApplicationBase.h diff --git a/src/Client/ClientApplicationBase.h b/src/Client/ClientApplicationBase.h deleted file mode 100644 index f85e79c734d..00000000000 --- a/src/Client/ClientApplicationBase.h +++ /dev/null @@ -1,16 +0,0 @@ -#pragma once - -#include -#include -#include - -namespace DB -{ - - -class ClientApplicationBase : public ClientBase, public Poco::Util::Application, public IHints<2> -{ - -} - -} From b4289dd8f5f67c0199565d38dfca1f4fa8894845 Mon Sep 17 00:00:00 2001 From: Nikita Mikhaylov Date: Tue, 25 Jun 2024 14:46:43 +0000 Subject: [PATCH 062/115] Style --- src/Client/LocalConnection.cpp | 2 -- utils/check-style/check-style | 4 ++++ 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/src/Client/LocalConnection.cpp b/src/Client/LocalConnection.cpp index f5fda7ec0e7..3b2c14ee4f9 100644 --- a/src/Client/LocalConnection.cpp +++ b/src/Client/LocalConnection.cpp @@ -21,8 +21,6 @@ #include #include - - namespace DB { diff --git a/utils/check-style/check-style b/utils/check-style/check-style index 722dfbcad16..380656cd1ca 100755 --- a/utils/check-style/check-style +++ b/utils/check-style/check-style @@ -322,10 +322,14 @@ std_cerr_cout_excludes=( src/Client/LineReader.cpp src/Client/QueryFuzzer.cpp src/Client/Suggest.cpp + src/Client/ClientBase.h + src/Client/LineReader.h + src/Client/ReplxxLineReader.h src/Bridge/IBridge.cpp src/Daemon/BaseDaemon.cpp src/Loggers/Loggers.cpp src/Common/GWPAsan.cpp + src/Common/ProgressIndication.h ) sources_with_std_cerr_cout=( $( find $ROOT_PATH/{src,base} -name '*.h' -or -name '*.cpp' | \ From 43cb2f6af7c4e5d7f2e799cf7bc149c7c0f8e84f Mon Sep 17 00:00:00 2001 From: Alexander Tokmakov Date: Wed, 26 Jun 2024 02:03:22 +0200 Subject: [PATCH 063/115] remove trash --- programs/keeper/Keeper.cpp | 5 +- programs/server/Server.cpp | 7 +- src/Common/GetPriorityForLoadBalancing.h | 1 - src/Common/ZooKeeper/ZooKeeper.cpp | 98 +++++-------------- src/Common/ZooKeeper/ZooKeeper.h | 7 +- src/Common/ZooKeeper/ZooKeeperArgs.cpp | 15 +-- src/Common/ZooKeeper/ZooKeeperArgs.h | 1 + src/Common/ZooKeeper/ZooKeeperImpl.cpp | 53 +++++++++- src/Common/ZooKeeper/ZooKeeperImpl.h | 14 +-- src/Server/CloudPlacementInfo.cpp | 24 ++++- .../configs/zookeeper_load_balancing2.xml | 8 +- 11 files changed, 122 insertions(+), 111 deletions(-) diff --git a/programs/keeper/Keeper.cpp b/programs/keeper/Keeper.cpp index 0d3c1f10894..0b0db31dd7c 100644 --- a/programs/keeper/Keeper.cpp +++ b/programs/keeper/Keeper.cpp @@ -355,10 +355,7 @@ try std::string include_from_path = config().getString("include_from", "/etc/metrika.xml"); - if (config().has(DB::PlacementInfo::PLACEMENT_CONFIG_PREFIX)) - { - PlacementInfo::PlacementInfo::instance().initialize(config()); - } + PlacementInfo::PlacementInfo::instance().initialize(config()); GlobalThreadPool::initialize( /// We need to have sufficient amount of threads for connections + nuraft workers + keeper workers, 1000 is an estimation diff --git a/programs/server/Server.cpp b/programs/server/Server.cpp index 7bc2be806f7..22fce7b141a 100644 --- a/programs/server/Server.cpp +++ b/programs/server/Server.cpp @@ -1003,6 +1003,8 @@ try ServerUUID::load(path / "uuid", log); + PlacementInfo::PlacementInfo::instance().initialize(config()); + zkutil::validateZooKeeperConfig(config()); bool has_zookeeper = zkutil::hasZooKeeperConfig(config()); @@ -1804,11 +1806,6 @@ try } - if (config().has(DB::PlacementInfo::PLACEMENT_CONFIG_PREFIX)) - { - PlacementInfo::PlacementInfo::instance().initialize(config()); - } - { std::lock_guard lock(servers_lock); /// We should start interserver communications before (and more important shutdown after) tables. diff --git a/src/Common/GetPriorityForLoadBalancing.h b/src/Common/GetPriorityForLoadBalancing.h index a2079b28dad..01dae9a1289 100644 --- a/src/Common/GetPriorityForLoadBalancing.h +++ b/src/Common/GetPriorityForLoadBalancing.h @@ -1,7 +1,6 @@ #pragma once #include -#include namespace DB { diff --git a/src/Common/ZooKeeper/ZooKeeper.cpp b/src/Common/ZooKeeper/ZooKeeper.cpp index 3b4286e37d2..56db9adb787 100644 --- a/src/Common/ZooKeeper/ZooKeeper.cpp +++ b/src/Common/ZooKeeper/ZooKeeper.cpp @@ -69,81 +69,27 @@ UInt64 getSecondsUntilReconnect(const ZooKeeperArgs & args) return session_lifetime_seconds; } -Coordination::ZooKeeper::Node hostToNode(const LoggerPtr & log, const ShuffleHost & host) -{ - /// We want to resolve all hosts without DNS cache for keeper connection. - Coordination::DNSResolver::instance().removeHostFromCache(host.host); - - const Poco::Net::SocketAddress host_socket_addr{host.host}; - LOG_TEST(log, "Adding ZooKeeper host {} ({})", host.host, host_socket_addr.toString()); - return Coordination::ZooKeeper::Node{host_socket_addr, host.original_index, host.secure}; -} - -/// TODO get rid of this, converting "host" to "node" is stupid -Coordination::ZooKeeper::Nodes hostsToNodes(const LoggerPtr & log, std::vector & shuffled_hosts) -{ - Coordination::ZooKeeper::Nodes nodes; - - bool dns_error = false; - for (auto & host : shuffled_hosts) - { - auto & host_string = host.host; - try - { - host.secure = startsWith(host_string, "secure://"); - - if (host.secure) - host_string.erase(0, strlen("secure://")); - - nodes.emplace_back(hostToNode(log, host)); - } - catch (const Poco::Net::HostNotFoundException & e) - { - /// Most likely it's misconfiguration and wrong hostname was specified - LOG_ERROR(log, "Cannot use ZooKeeper host {}, reason: {}", host_string, e.displayText()); - } - catch (const Poco::Net::DNSException & e) - { - /// Most likely DNS is not available now - dns_error = true; - LOG_ERROR(log, "Cannot use ZooKeeper host {} due to DNS error: {}", host_string, e.displayText()); - } - } - - if (nodes.empty()) - { - /// For DNS errors we throw exception with ZCONNECTIONLOSS code, so it will be considered as hardware error, not user error - if (dns_error) - throw KeeperException::fromMessage( - Coordination::Error::ZCONNECTIONLOSS, "Cannot resolve any of provided ZooKeeper hosts due to DNS error"); - else - throw KeeperException::fromMessage(Coordination::Error::ZCONNECTIONLOSS, "Cannot use any of provided ZooKeeper nodes"); - } - - return nodes; -} - void ZooKeeper::updateAvailabilityZones() { - std::vector shuffled_hosts = shuffleHosts(); - Coordination::ZooKeeper::Nodes nodes = hostsToNodes(log, shuffled_hosts); + ShuffleHosts shuffled_hosts = shuffleHosts(); - for (const auto & node : nodes) + for (const auto & node : shuffled_hosts) { try { - Coordination::ZooKeeper::Nodes single_node{node}; + ShuffleHosts single_node{node}; auto tmp_impl = std::make_unique(single_node, args, zk_log); auto idx = node.original_index; availability_zones[idx] = tmp_impl->tryGetAvailabilityZone(); - LOG_DEBUG(log, "Got availability zone for {}: {}", args.hosts[idx], availability_zones[idx]); + LOG_TEST(log, "Got availability zone for {}: {}", args.hosts[idx], availability_zones[idx]); } catch (...) { - DB::tryLogCurrentException(log, "Failed to get availability zone for " + node.address.toString()); + DB::tryLogCurrentException(log, "Failed to get availability zone for " + node.host); } } + LOG_DEBUG(log, "Updated availability zones: [{}]", fmt::join(availability_zones, ", ")); } void ZooKeeper::init(ZooKeeperArgs args_, std::unique_ptr existing_impl) @@ -168,16 +114,16 @@ void ZooKeeper::init(ZooKeeperArgs args_, std::unique_ptr /// availability_zones is empty on server startup or after config reloading /// We will keep the az info when starting new sessions availability_zones = args.availability_zones; + LOG_TEST(log, "Availability zones from config: [{}], client: {}", fmt::join(availability_zones, ", "), args.client_availability_zone); if (args.availability_zone_autodetect) updateAvailabilityZones(); } chassert(availability_zones.size() == args.hosts.size()); /// Shuffle the hosts to distribute the load among ZooKeeper nodes. - std::vector shuffled_hosts = shuffleHosts(); - Coordination::ZooKeeper::Nodes nodes = hostsToNodes(log, shuffled_hosts); + ShuffleHosts shuffled_hosts = shuffleHosts(); - impl = std::make_unique(nodes, args, zk_log); + impl = std::make_unique(shuffled_hosts, args, zk_log); Int8 node_idx = impl->getConnectedNodeIdx(); if (args.chroot.empty()) @@ -188,7 +134,7 @@ void ZooKeeper::init(ZooKeeperArgs args_, std::unique_ptr /// If the balancing strategy has an optimal node then it will be the first in the list bool connected_to_suboptimal_node = node_idx != shuffled_hosts[0].original_index; - bool respect_az = !args.client_availability_zone.empty(); + bool respect_az = args.prefer_local_availability_zone && !args.client_availability_zone.empty(); bool may_benefit_from_reconnecting = respect_az || args.get_priority_load_balancing.hasOptimalNode(); if (connected_to_suboptimal_node && may_benefit_from_reconnecting) { @@ -202,13 +148,12 @@ void ZooKeeper::init(ZooKeeperArgs args_, std::unique_ptr try { LOG_DEBUG(log, "Trying to connect to a more optimal node {}", optimal_host.host); - Coordination::ZooKeeper::Nodes node; - node.emplace_back(hostToNode(log, optimal_host)); + ShuffleHosts node{optimal_host}; std::unique_ptr new_impl = std::make_unique(node, args, zk_log); Int8 new_node_idx = new_impl->getConnectedNodeIdx(); /// Maybe the node was unavailable when getting AZs first time, update just in case - if (args.availability_zone_autodetect) + if (args.availability_zone_autodetect && availability_zones[new_node_idx].empty()) { availability_zones[new_node_idx] = new_impl->tryGetAvailabilityZone(); LOG_DEBUG(log, "Got availability zone for {}: {}", optimal_host.host, availability_zones[new_node_idx]); @@ -277,7 +222,8 @@ ZooKeeper::ZooKeeper(const ZooKeeperArgs & args_, std::shared_ptr ZooKeeper::shuffleHosts() const +ShuffleHosts ZooKeeper::shuffleHosts() const { std::function get_priority = args.get_priority_load_balancing.getPriorityFunc( args.get_priority_load_balancing.load_balancing, /* offset for first_or_random */ 0, args.hosts.size()); - std::vector shuffle_hosts; + ShuffleHosts shuffle_hosts; for (size_t i = 0; i < args.hosts.size(); ++i) { ShuffleHost shuffle_host; shuffle_host.host = args.hosts[i]; shuffle_host.original_index = static_cast(i); - if (!availability_zones[i].empty()) + shuffle_host.secure = startsWith(shuffle_host.host, "secure://"); + if (shuffle_host.secure) + shuffle_host.host.erase(0, strlen("secure://")); + + if (!args.client_availability_zone.empty() && !availability_zones[i].empty()) shuffle_host.az_info = availability_zones[i] == args.client_availability_zone ? ShuffleHost::SAME : ShuffleHost::OTHER; if (get_priority) @@ -1588,9 +1538,11 @@ int32_t ZooKeeper::getConnectionXid() const String ZooKeeper::getConnectedHostAvailabilityZone() const { - auto idx = impl->getConnectedNodeIdx(); - if (idx < 0) + if (args.implementation != "zookeeper" || !impl) return ""; + Int8 idx = impl->getConnectedNodeIdx(); + if (idx < 0) + return ""; /// session expired return availability_zones.at(idx); } diff --git a/src/Common/ZooKeeper/ZooKeeper.h b/src/Common/ZooKeeper/ZooKeeper.h index 3915d884351..9fa41130e76 100644 --- a/src/Common/ZooKeeper/ZooKeeper.h +++ b/src/Common/ZooKeeper/ZooKeeper.h @@ -63,6 +63,9 @@ struct ShuffleHost Priority priority; UInt64 random = 0; + /// We should resolve it each time without caching + mutable std::optional address; + void randomize() { random = thread_local_rng(); @@ -75,6 +78,8 @@ struct ShuffleHost } }; +using ShuffleHosts = std::vector; + struct RemoveException { explicit RemoveException(std::string_view path_ = "", bool remove_subtree_ = true) @@ -243,7 +248,7 @@ public: ~ZooKeeper(); - std::vector shuffleHosts() const; + ShuffleHosts shuffleHosts() const; static Ptr create(const Poco::Util::AbstractConfiguration & config, const std::string & config_name, diff --git a/src/Common/ZooKeeper/ZooKeeperArgs.cpp b/src/Common/ZooKeeper/ZooKeeperArgs.cpp index 2154d9d9232..18dff779a70 100644 --- a/src/Common/ZooKeeper/ZooKeeperArgs.cpp +++ b/src/Common/ZooKeeper/ZooKeeperArgs.cpp @@ -5,6 +5,8 @@ #include #include #include +#include +#include #include #include @@ -134,8 +136,9 @@ void ZooKeeperArgs::initFromKeeperServerSection(const Poco::Util::AbstractConfig } availability_zone_autodetect = config.getBool(std::string{config_name} + ".availability_zone_autodetect", false); - if (availability_zone_autodetect) - client_availability_zone = DB::S3::tryGetRunningAvailabilityZone(); + prefer_local_availability_zone = config.getBool(std::string{config_name} + ".prefer_local_availability_zone", false); + if (prefer_local_availability_zone) + client_availability_zone = DB::PlacementInfo::PlacementInfo::instance().getAvailabilityZone(); } void ZooKeeperArgs::initFromKeeperSection(const Poco::Util::AbstractConfiguration & config, const std::string & config_name) @@ -210,9 +213,9 @@ void ZooKeeperArgs::initFromKeeperSection(const Poco::Util::AbstractConfiguratio { sessions_path = config.getString(config_name + "." + key); } - else if (key == "client_availability_zone") + else if (key == "prefer_local_availability_zone") { - client_availability_zone = config.getString(config_name + "." + key); + prefer_local_availability_zone = config.getBool(config_name + "." + key); } else if (key == "implementation") { @@ -249,8 +252,8 @@ void ZooKeeperArgs::initFromKeeperSection(const Poco::Util::AbstractConfiguratio if (load_balancing) get_priority_load_balancing = DB::GetPriorityForLoadBalancing(*load_balancing, thread_local_rng() % hosts.size()); - if (availability_zone_autodetect) - client_availability_zone = DB::S3::tryGetRunningAvailabilityZone(); + if (prefer_local_availability_zone) + client_availability_zone = DB::PlacementInfo::PlacementInfo::instance().getAvailabilityZone(); } } diff --git a/src/Common/ZooKeeper/ZooKeeperArgs.h b/src/Common/ZooKeeper/ZooKeeperArgs.h index cd7a5ddb7ce..945b77bf9c1 100644 --- a/src/Common/ZooKeeper/ZooKeeperArgs.h +++ b/src/Common/ZooKeeper/ZooKeeperArgs.h @@ -49,6 +49,7 @@ struct ZooKeeperArgs UInt64 send_sleep_ms = 0; UInt64 recv_sleep_ms = 0; bool use_compression = false; + bool prefer_local_availability_zone = false; bool availability_zone_autodetect = false; SessionLifetimeConfiguration fallback_session_lifetime = {}; diff --git a/src/Common/ZooKeeper/ZooKeeperImpl.cpp b/src/Common/ZooKeeper/ZooKeeperImpl.cpp index 28dc0d77b1a..8653af51308 100644 --- a/src/Common/ZooKeeper/ZooKeeperImpl.cpp +++ b/src/Common/ZooKeeper/ZooKeeperImpl.cpp @@ -23,6 +23,9 @@ #include #include +#include +#include + #include "Coordination/KeeperConstants.h" #include "config.h" @@ -338,7 +341,7 @@ ZooKeeper::~ZooKeeper() ZooKeeper::ZooKeeper( - const Nodes & nodes, + const zkutil::ShuffleHosts & nodes, const zkutil::ZooKeeperArgs & args_, std::shared_ptr zk_log_) : args(args_) @@ -426,7 +429,7 @@ ZooKeeper::ZooKeeper( void ZooKeeper::connect( - const Nodes & nodes, + const zkutil::ShuffleHosts & nodes, Poco::Timespan connection_timeout) { if (nodes.empty()) @@ -434,6 +437,40 @@ void ZooKeeper::connect( static constexpr size_t num_tries = 3; bool connected = false; + bool dns_error = false; + + size_t resolved_count = 0; + for (const auto & node : nodes) + { + try + { + const Poco::Net::SocketAddress host_socket_addr{node.host}; + LOG_TRACE(log, "Adding ZooKeeper host {} ({}), az: {}, priority: {}", node.host, host_socket_addr.toString(), node.az_info, node.priority); + node.address = host_socket_addr; + ++resolved_count; + } + catch (const Poco::Net::HostNotFoundException & e) + { + /// Most likely it's misconfiguration and wrong hostname was specified + LOG_ERROR(log, "Cannot use ZooKeeper host {}, reason: {}", node.host, e.displayText()); + } + catch (const Poco::Net::DNSException & e) + { + /// Most likely DNS is not available now + dns_error = true; + LOG_ERROR(log, "Cannot use ZooKeeper host {} due to DNS error: {}", node.host, e.displayText()); + } + } + + if (resolved_count == 0) + { + /// For DNS errors we throw exception with ZCONNECTIONLOSS code, so it will be considered as hardware error, not user error + if (dns_error) + throw zkutil::KeeperException::fromMessage( + Coordination::Error::ZCONNECTIONLOSS, "Cannot resolve any of provided ZooKeeper hosts due to DNS error"); + else + throw zkutil::KeeperException::fromMessage(Coordination::Error::ZCONNECTIONLOSS, "Cannot use any of provided ZooKeeper nodes"); + } WriteBufferFromOwnString fail_reasons; for (size_t try_no = 0; try_no < num_tries; ++try_no) @@ -442,6 +479,9 @@ void ZooKeeper::connect( { try { + if (!node.address) + continue; + /// Reset the state of previous attempt. if (node.secure) { @@ -457,7 +497,7 @@ void ZooKeeper::connect( socket = Poco::Net::StreamSocket(); } - socket.connect(node.address, connection_timeout); + socket.connect(*node.address, connection_timeout); socket_address = socket.peerAddress(); socket.setReceiveTimeout(args.operation_timeout_ms * 1000); @@ -501,7 +541,7 @@ void ZooKeeper::connect( } catch (...) { - fail_reasons << "\n" << getCurrentExceptionMessage(false) << ", " << node.address.toString(); + fail_reasons << "\n" << getCurrentExceptionMessage(false) << ", " << node.address->toString(); } } @@ -515,6 +555,9 @@ void ZooKeeper::connect( bool first = true; for (const auto & node : nodes) { + if (!node.address) + continue; + if (first) first = false; else @@ -523,7 +566,7 @@ void ZooKeeper::connect( if (node.secure) message << "secure://"; - message << node.address.toString(); + message << node.address->toString(); } message << fail_reasons.str() << "\n"; diff --git a/src/Common/ZooKeeper/ZooKeeperImpl.h b/src/Common/ZooKeeper/ZooKeeperImpl.h index b42223f0462..0c88c35b381 100644 --- a/src/Common/ZooKeeper/ZooKeeperImpl.h +++ b/src/Common/ZooKeeper/ZooKeeperImpl.h @@ -8,6 +8,7 @@ #include #include #include +#include #include #include @@ -102,21 +103,12 @@ using namespace DB; class ZooKeeper final : public IKeeper { public: - struct Node - { - Poco::Net::SocketAddress address; - UInt8 original_index; - bool secure; - }; - - using Nodes = std::vector; - /** Connection to nodes is performed in order. If you want, shuffle them manually. * Operation timeout couldn't be greater than session timeout. * Operation timeout applies independently for network read, network write, waiting for events and synchronization. */ ZooKeeper( - const Nodes & nodes, + const zkutil::ShuffleHosts & nodes, const zkutil::ZooKeeperArgs & args_, std::shared_ptr zk_log_); @@ -313,7 +305,7 @@ private: LoggerPtr log; void connect( - const Nodes & node, + const zkutil::ShuffleHosts & node, Poco::Timespan connection_timeout); void sendHandshake(); diff --git a/src/Server/CloudPlacementInfo.cpp b/src/Server/CloudPlacementInfo.cpp index 0790f825a45..d8810bb30de 100644 --- a/src/Server/CloudPlacementInfo.cpp +++ b/src/Server/CloudPlacementInfo.cpp @@ -11,6 +11,11 @@ namespace DB { +namespace ErrorCodes +{ +extern const int LOGICAL_ERROR; +} + namespace PlacementInfo { @@ -46,7 +51,15 @@ PlacementInfo & PlacementInfo::instance() } void PlacementInfo::initialize(const Poco::Util::AbstractConfiguration & config) +try { + if (!config.has(DB::PlacementInfo::PLACEMENT_CONFIG_PREFIX)) + { + availability_zone = ""; + initialized = true; + return; + } + use_imds = config.getBool(getConfigPath("use_imds"), false); if (use_imds) @@ -67,14 +80,17 @@ void PlacementInfo::initialize(const Poco::Util::AbstractConfiguration & config) LOG_DEBUG(log, "Loaded info: availability_zone: {}", availability_zone); initialized = true; } +catch (...) +{ + tryLogCurrentException("Failed to get availability zone"); + availability_zone = ""; + initialized = true; +} std::string PlacementInfo::getAvailabilityZone() const { if (!initialized) - { - LOG_WARNING(log, "Placement info has not been loaded"); - return ""; - } + throw Exception(ErrorCodes::LOGICAL_ERROR, "Placement info has not been loaded"); return availability_zone; } diff --git a/tests/integration/test_zookeeper_config_load_balancing/configs/zookeeper_load_balancing2.xml b/tests/integration/test_zookeeper_config_load_balancing/configs/zookeeper_load_balancing2.xml index eb6ba93c421..fd416cad505 100644 --- a/tests/integration/test_zookeeper_config_load_balancing/configs/zookeeper_load_balancing2.xml +++ b/tests/integration/test_zookeeper_config_load_balancing/configs/zookeeper_load_balancing2.xml @@ -3,7 +3,8 @@ random - az2 + 1 + 0 1 @@ -26,4 +27,9 @@ 3000 + + + 0 + az2 + From 628359ddc9f914f9b27ca3a4749b025c79b7e305 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Wed, 26 Jun 2024 02:29:04 +0200 Subject: [PATCH 064/115] Add memory limit for clickhouse-local by default --- programs/local/LocalServer.cpp | 97 +++++++++++++++++++++++----------- programs/local/LocalServer.h | 2 + 2 files changed, 69 insertions(+), 30 deletions(-) diff --git a/programs/local/LocalServer.cpp b/programs/local/LocalServer.cpp index cb1c35743b2..806cec7d3e3 100644 --- a/programs/local/LocalServer.cpp +++ b/programs/local/LocalServer.cpp @@ -32,6 +32,7 @@ #include #include #include +#include #include #include #include @@ -59,8 +60,13 @@ # include #endif + namespace fs = std::filesystem; +namespace CurrentMetrics +{ + extern const Metric MemoryTracking; +} namespace DB { @@ -126,11 +132,12 @@ void LocalServer::initialize(Poco::Util::Application & self) config().add(loaded_config.configuration.duplicate(), PRIO_DEFAULT, false); } + server_settings.loadSettingsFromConfig(config()); + GlobalThreadPool::initialize( - config().getUInt("max_thread_pool_size", 10000), - config().getUInt("max_thread_pool_free_size", 1000), - config().getUInt("thread_pool_queue_size", 10000) - ); + server_settings.max_thread_pool_size, + server_settings.max_thread_pool_free_size, + server_settings.thread_pool_queue_size); #if USE_AZURE_BLOB_STORAGE /// See the explanation near the same line in Server.cpp @@ -141,18 +148,17 @@ void LocalServer::initialize(Poco::Util::Application & self) #endif getIOThreadPool().initialize( - config().getUInt("max_io_thread_pool_size", 100), - config().getUInt("max_io_thread_pool_free_size", 0), - config().getUInt("io_thread_pool_queue_size", 10000)); + server_settings.max_io_thread_pool_size, + server_settings.max_io_thread_pool_free_size, + server_settings.io_thread_pool_queue_size); - - const size_t active_parts_loading_threads = config().getUInt("max_active_parts_loading_thread_pool_size", 64); + const size_t active_parts_loading_threads = server_settings.max_active_parts_loading_thread_pool_size; getActivePartsLoadingThreadPool().initialize( active_parts_loading_threads, 0, // We don't need any threads one all the parts will be loaded active_parts_loading_threads); - const size_t outdated_parts_loading_threads = config().getUInt("max_outdated_parts_loading_thread_pool_size", 32); + const size_t outdated_parts_loading_threads = server_settings.max_outdated_parts_loading_thread_pool_size; getOutdatedPartsLoadingThreadPool().initialize( outdated_parts_loading_threads, 0, // We don't need any threads one all the parts will be loaded @@ -160,7 +166,7 @@ void LocalServer::initialize(Poco::Util::Application & self) getOutdatedPartsLoadingThreadPool().setMaxTurboThreads(active_parts_loading_threads); - const size_t unexpected_parts_loading_threads = config().getUInt("max_unexpected_parts_loading_thread_pool_size", 32); + const size_t unexpected_parts_loading_threads = server_settings.max_unexpected_parts_loading_thread_pool_size; getUnexpectedPartsLoadingThreadPool().initialize( unexpected_parts_loading_threads, 0, // We don't need any threads one all the parts will be loaded @@ -168,7 +174,7 @@ void LocalServer::initialize(Poco::Util::Application & self) getUnexpectedPartsLoadingThreadPool().setMaxTurboThreads(active_parts_loading_threads); - const size_t cleanup_threads = config().getUInt("max_parts_cleaning_thread_pool_size", 128); + const size_t cleanup_threads = server_settings.max_parts_cleaning_thread_pool_size; getPartsCleaningThreadPool().initialize( cleanup_threads, 0, // We don't need any threads one all the parts will be deleted @@ -433,7 +439,7 @@ try UseSSL use_ssl; thread_status.emplace(); - StackTrace::setShowAddresses(config().getBool("show_addresses_in_stack_traces", true)); + StackTrace::setShowAddresses(server_settings.show_addresses_in_stack_traces); setupSignalHandler(); @@ -619,12 +625,43 @@ void LocalServer::processConfig() global_context->getProcessList().setMaxSize(0); const size_t physical_server_memory = getMemoryAmount(); - const double cache_size_to_ram_max_ratio = config().getDouble("cache_size_to_ram_max_ratio", 0.5); + + size_t max_server_memory_usage = server_settings.max_server_memory_usage; + double max_server_memory_usage_to_ram_ratio = server_settings.max_server_memory_usage_to_ram_ratio; + + size_t default_max_server_memory_usage = static_cast(physical_server_memory * max_server_memory_usage_to_ram_ratio); + + if (max_server_memory_usage == 0) + { + max_server_memory_usage = default_max_server_memory_usage; + LOG_INFO(log, "Setting max_server_memory_usage was set to {}" + " ({} available * {:.2f} max_server_memory_usage_to_ram_ratio)", + formatReadableSizeWithBinarySuffix(max_server_memory_usage), + formatReadableSizeWithBinarySuffix(physical_server_memory), + max_server_memory_usage_to_ram_ratio); + } + else if (max_server_memory_usage > default_max_server_memory_usage) + { + max_server_memory_usage = default_max_server_memory_usage; + LOG_INFO(log, "Setting max_server_memory_usage was lowered to {}" + " because the system has low amount of memory. The amount was" + " calculated as {} available" + " * {:.2f} max_server_memory_usage_to_ram_ratio", + formatReadableSizeWithBinarySuffix(max_server_memory_usage), + formatReadableSizeWithBinarySuffix(physical_server_memory), + max_server_memory_usage_to_ram_ratio); + } + + total_memory_tracker.setHardLimit(max_server_memory_usage); + total_memory_tracker.setDescription("(total)"); + total_memory_tracker.setMetric(CurrentMetrics::MemoryTracking); + + const double cache_size_to_ram_max_ratio = server_settings.cache_size_to_ram_max_ratio; const size_t max_cache_size = static_cast(physical_server_memory * cache_size_to_ram_max_ratio); - String uncompressed_cache_policy = config().getString("uncompressed_cache_policy", DEFAULT_UNCOMPRESSED_CACHE_POLICY); - size_t uncompressed_cache_size = config().getUInt64("uncompressed_cache_size", DEFAULT_UNCOMPRESSED_CACHE_MAX_SIZE); - double uncompressed_cache_size_ratio = config().getDouble("uncompressed_cache_size_ratio", DEFAULT_UNCOMPRESSED_CACHE_SIZE_RATIO); + String uncompressed_cache_policy = server_settings.uncompressed_cache_policy; + size_t uncompressed_cache_size = server_settings.uncompressed_cache_size; + double uncompressed_cache_size_ratio = server_settings.uncompressed_cache_size_ratio; if (uncompressed_cache_size > max_cache_size) { uncompressed_cache_size = max_cache_size; @@ -632,9 +669,9 @@ void LocalServer::processConfig() } global_context->setUncompressedCache(uncompressed_cache_policy, uncompressed_cache_size, uncompressed_cache_size_ratio); - String mark_cache_policy = config().getString("mark_cache_policy", DEFAULT_MARK_CACHE_POLICY); - size_t mark_cache_size = config().getUInt64("mark_cache_size", DEFAULT_MARK_CACHE_MAX_SIZE); - double mark_cache_size_ratio = config().getDouble("mark_cache_size_ratio", DEFAULT_MARK_CACHE_SIZE_RATIO); + String mark_cache_policy = server_settings.mark_cache_policy; + size_t mark_cache_size = server_settings.mark_cache_size; + double mark_cache_size_ratio = server_settings.mark_cache_size_ratio; if (!mark_cache_size) LOG_ERROR(log, "Too low mark cache size will lead to severe performance degradation."); if (mark_cache_size > max_cache_size) @@ -644,9 +681,9 @@ void LocalServer::processConfig() } global_context->setMarkCache(mark_cache_policy, mark_cache_size, mark_cache_size_ratio); - String index_uncompressed_cache_policy = config().getString("index_uncompressed_cache_policy", DEFAULT_INDEX_UNCOMPRESSED_CACHE_POLICY); - size_t index_uncompressed_cache_size = config().getUInt64("index_uncompressed_cache_size", DEFAULT_INDEX_UNCOMPRESSED_CACHE_MAX_SIZE); - double index_uncompressed_cache_size_ratio = config().getDouble("index_uncompressed_cache_size_ratio", DEFAULT_INDEX_UNCOMPRESSED_CACHE_SIZE_RATIO); + String index_uncompressed_cache_policy = server_settings.index_uncompressed_cache_policy; + size_t index_uncompressed_cache_size = server_settings.index_uncompressed_cache_size; + double index_uncompressed_cache_size_ratio = server_settings.index_uncompressed_cache_size_ratio; if (index_uncompressed_cache_size > max_cache_size) { index_uncompressed_cache_size = max_cache_size; @@ -654,9 +691,9 @@ void LocalServer::processConfig() } global_context->setIndexUncompressedCache(index_uncompressed_cache_policy, index_uncompressed_cache_size, index_uncompressed_cache_size_ratio); - String index_mark_cache_policy = config().getString("index_mark_cache_policy", DEFAULT_INDEX_MARK_CACHE_POLICY); - size_t index_mark_cache_size = config().getUInt64("index_mark_cache_size", DEFAULT_INDEX_MARK_CACHE_MAX_SIZE); - double index_mark_cache_size_ratio = config().getDouble("index_mark_cache_size_ratio", DEFAULT_INDEX_MARK_CACHE_SIZE_RATIO); + String index_mark_cache_policy = server_settings.index_mark_cache_policy; + size_t index_mark_cache_size = server_settings.index_mark_cache_size; + double index_mark_cache_size_ratio = server_settings.index_mark_cache_size_ratio; if (index_mark_cache_size > max_cache_size) { index_mark_cache_size = max_cache_size; @@ -664,7 +701,7 @@ void LocalServer::processConfig() } global_context->setIndexMarkCache(index_mark_cache_policy, index_mark_cache_size, index_mark_cache_size_ratio); - size_t mmap_cache_size = config().getUInt64("mmap_cache_size", DEFAULT_MMAP_CACHE_MAX_SIZE); + size_t mmap_cache_size = server_settings.mmap_cache_size; if (mmap_cache_size > max_cache_size) { mmap_cache_size = max_cache_size; @@ -676,8 +713,8 @@ void LocalServer::processConfig() global_context->setQueryCache(0, 0, 0, 0); #if USE_EMBEDDED_COMPILER - size_t compiled_expression_cache_max_size_in_bytes = config().getUInt64("compiled_expression_cache_size", DEFAULT_COMPILED_EXPRESSION_CACHE_MAX_SIZE); - size_t compiled_expression_cache_max_elements = config().getUInt64("compiled_expression_cache_elements_size", DEFAULT_COMPILED_EXPRESSION_CACHE_MAX_ENTRIES); + size_t compiled_expression_cache_max_size_in_bytes = server_settings.compiled_expression_cache_size; + size_t compiled_expression_cache_max_elements = server_settings.compiled_expression_cache_elements_size; CompiledExpressionCacheFactory::instance().init(compiled_expression_cache_max_size_in_bytes, compiled_expression_cache_max_elements); #endif @@ -694,7 +731,7 @@ void LocalServer::processConfig() /// We load temporary database first, because projections need it. DatabaseCatalog::instance().initializeAndLoadTemporaryDatabase(); - std::string default_database = config().getString("default_database", "default"); + std::string default_database = server_settings.default_database; DatabaseCatalog::instance().attachDatabase(default_database, createClickHouseLocalDatabaseOverlay(default_database, global_context)); global_context->setCurrentDatabase(default_database); diff --git a/programs/local/LocalServer.h b/programs/local/LocalServer.h index 4856e68ff9b..bd174e41eac 100644 --- a/programs/local/LocalServer.h +++ b/programs/local/LocalServer.h @@ -63,6 +63,8 @@ private: void applyCmdOptions(ContextMutablePtr context); void applyCmdSettings(ContextMutablePtr context); + ServerSettings server_settings; + std::optional status; std::optional temporary_directory_to_delete; From 09bf7e92a9ac09f3e837a0625986aa33d21cf63d Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Wed, 26 Jun 2024 02:31:31 +0200 Subject: [PATCH 065/115] Add memory limit by default in clickhouse-local --- .../queries/0_stateless/03196_local_memory_limit.reference | 1 + tests/queries/0_stateless/03196_local_memory_limit.sh | 7 +++++++ 2 files changed, 8 insertions(+) create mode 100644 tests/queries/0_stateless/03196_local_memory_limit.reference create mode 100755 tests/queries/0_stateless/03196_local_memory_limit.sh diff --git a/tests/queries/0_stateless/03196_local_memory_limit.reference b/tests/queries/0_stateless/03196_local_memory_limit.reference new file mode 100644 index 00000000000..f2e22e8aa5b --- /dev/null +++ b/tests/queries/0_stateless/03196_local_memory_limit.reference @@ -0,0 +1 @@ +maximum: 95.37 MiB diff --git a/tests/queries/0_stateless/03196_local_memory_limit.sh b/tests/queries/0_stateless/03196_local_memory_limit.sh new file mode 100755 index 00000000000..346b37be006 --- /dev/null +++ b/tests/queries/0_stateless/03196_local_memory_limit.sh @@ -0,0 +1,7 @@ +#!/usr/bin/env bash + +CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh +. "$CUR_DIR"/../shell_config.sh + +${CLICKHOUSE_LOCAL} --config-file <(echo "100M") --query "SELECT number FROM system.numbers GROUP BY number HAVING count() > 1" 2>&1 | grep -o -P 'maximum: [\d\.]+ MiB' From 3b8adbb8def6ea1b3f3b90be46634255b989bca7 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Wed, 26 Jun 2024 03:15:25 +0200 Subject: [PATCH 066/115] Add "backup" tool --- utils/backup/backup | 47 +++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 47 insertions(+) create mode 100755 utils/backup/backup diff --git a/utils/backup/backup b/utils/backup/backup new file mode 100755 index 00000000000..6aa9c179033 --- /dev/null +++ b/utils/backup/backup @@ -0,0 +1,47 @@ +#!/bin/bash + +user="default" +path="." + +usage() { + echo + echo "A trivial script to upload your files into ClickHouse." + echo "You might want to use something like Dropbox instead, but..." + echo + echo "Usage: $0 --host [--user ] --password " + exit 1 +} + +while [[ "$#" -gt 0 ]]; do + case "$1" in + --host) + host="$2" + shift 2 + ;; + --user) + user="$2" + shift 2 + ;; + --password) + password="$2" + shift 2 + ;; + --help) + usage + ;; + *) + path="$1" + shift 1 + ;; + esac +done + +if [ -z "$host" ] || [ -z "$password" ]; then + echo "Error: --host and --password are mandatory." + usage +fi + +clickhouse-client --host "$host" --user "$user" --password "$password" --secure --query "CREATE TABLE IF NOT EXISTS default.files (time DEFAULT now(), path String, content String CODEC(ZSTD(6))) ENGINE = MergeTree ORDER BY (path, time)" && +find "$path" -type f | clickhouse-local --input-format LineAsString \ + --max-block-size 1 --min-insert-block-size-rows 0 --min-insert-block-size-bytes '100M' --max-insert-threads 1 \ + --query "INSERT INTO FUNCTION remoteSecure('$host', default.files, '$user', '$password') (path, content) SELECT line, file(line) FROM table" --progress From e2e8562d89d061ceb5b75da09c7196fe70f41935 Mon Sep 17 00:00:00 2001 From: Your Name Date: Thu, 13 Jun 2024 20:41:46 +0200 Subject: [PATCH 067/115] Fix --- src/Client/LocalConnection.cpp | 18 +++++++++++++++--- 1 file changed, 15 insertions(+), 3 deletions(-) diff --git a/src/Client/LocalConnection.cpp b/src/Client/LocalConnection.cpp index c7494e31605..1c2f01fb0ab 100644 --- a/src/Client/LocalConnection.cpp +++ b/src/Client/LocalConnection.cpp @@ -477,13 +477,25 @@ Packet LocalConnection::receivePacket() return packet; } + bool ret = false; if (!next_packet_type) - poll(0); + ret = poll(0); if (!next_packet_type) { - packet.type = Protocol::Server::EndOfStream; - return packet; + if (state && !ret) + { + /// We should retry. + /// Let's send a dummy packet and hope for the best. + /// TODO proper fix + packet.type = Protocol::Server::Progress; + return packet; + } + else + { + packet.type = Protocol::Server::EndOfStream; + return packet; + } } packet.type = next_packet_type.value(); From 8542221f2e7e472ec6ec938412fc1c5c6e92bbb4 Mon Sep 17 00:00:00 2001 From: Konstantin Bogdanov Date: Wed, 26 Jun 2024 00:08:18 +0200 Subject: [PATCH 068/115] Better fix --- src/Client/LocalConnection.cpp | 27 +++++++++++---------------- 1 file changed, 11 insertions(+), 16 deletions(-) diff --git a/src/Client/LocalConnection.cpp b/src/Client/LocalConnection.cpp index 1c2f01fb0ab..252c0a1b952 100644 --- a/src/Client/LocalConnection.cpp +++ b/src/Client/LocalConnection.cpp @@ -295,7 +295,14 @@ void LocalConnection::sendCancel() bool LocalConnection::pullBlock(Block & block) { if (state->executor) - return state->executor->pull(block, query_context->getSettingsRef().interactive_delay / 1000); + while (!block) + { + bool pull_result = state->executor->pull(block, query_context->getSettingsRef().interactive_delay / 1000); + + /// Pipeline is finished. + if (!pull_result) + return false; + } return false; } @@ -477,25 +484,13 @@ Packet LocalConnection::receivePacket() return packet; } - bool ret = false; if (!next_packet_type) - ret = poll(0); + poll(0); if (!next_packet_type) { - if (state && !ret) - { - /// We should retry. - /// Let's send a dummy packet and hope for the best. - /// TODO proper fix - packet.type = Protocol::Server::Progress; - return packet; - } - else - { - packet.type = Protocol::Server::EndOfStream; - return packet; - } + packet.type = Protocol::Server::EndOfStream; + return packet; } packet.type = next_packet_type.value(); From e85dda6a4bf95fed7894661545414f21b4578699 Mon Sep 17 00:00:00 2001 From: Konstantin Bogdanov Date: Wed, 26 Jun 2024 07:07:10 +0200 Subject: [PATCH 069/115] Another fix --- src/Client/LocalConnection.cpp | 11 ++--------- 1 file changed, 2 insertions(+), 9 deletions(-) diff --git a/src/Client/LocalConnection.cpp b/src/Client/LocalConnection.cpp index 252c0a1b952..47730edf695 100644 --- a/src/Client/LocalConnection.cpp +++ b/src/Client/LocalConnection.cpp @@ -295,14 +295,7 @@ void LocalConnection::sendCancel() bool LocalConnection::pullBlock(Block & block) { if (state->executor) - while (!block) - { - bool pull_result = state->executor->pull(block, query_context->getSettingsRef().interactive_delay / 1000); - - /// Pipeline is finished. - if (!pull_result) - return false; - } + return state->executor->pull(block, query_context->getSettingsRef().interactive_delay / 1000); return false; } @@ -485,7 +478,7 @@ Packet LocalConnection::receivePacket() } if (!next_packet_type) - poll(0); + while (state && !poll(0)); if (!next_packet_type) { From 35820eaa7baf67a02e7a875d413b53e306ba4ae9 Mon Sep 17 00:00:00 2001 From: Antonio Andelic Date: Wed, 26 Jun 2024 11:13:18 +0200 Subject: [PATCH 070/115] Build jemalloc with profiler --- contrib/jemalloc-cmake/CMakeLists.txt | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/contrib/jemalloc-cmake/CMakeLists.txt b/contrib/jemalloc-cmake/CMakeLists.txt index b633f0fda50..7b8feaa80d4 100644 --- a/contrib/jemalloc-cmake/CMakeLists.txt +++ b/contrib/jemalloc-cmake/CMakeLists.txt @@ -34,9 +34,9 @@ if (OS_LINUX) # avoid spurious latencies and additional work associated with # MADV_DONTNEED. See # https://github.com/ClickHouse/ClickHouse/issues/11121 for motivation. - set (JEMALLOC_CONFIG_MALLOC_CONF "percpu_arena:percpu,oversize_threshold:0,muzzy_decay_ms:0,dirty_decay_ms:5000") + set (JEMALLOC_CONFIG_MALLOC_CONF "percpu_arena:percpu,oversize_threshold:0,muzzy_decay_ms:0,dirty_decay_ms:5000,prof:true,prof_active:false") else() - set (JEMALLOC_CONFIG_MALLOC_CONF "oversize_threshold:0,muzzy_decay_ms:0,dirty_decay_ms:5000") + set (JEMALLOC_CONFIG_MALLOC_CONF "oversize_threshold:0,muzzy_decay_ms:0,dirty_decay_ms:5000,prof:true,prof_active:false") endif() # CACHE variable is empty to allow changing defaults without the necessity # to purge cache From 258e2f04215cd01b7bb7f203b53e79f7206d03e6 Mon Sep 17 00:00:00 2001 From: Kseniia Sumarokova <54203879+kssenii@users.noreply.github.com> Date: Wed, 26 Jun 2024 16:45:02 +0200 Subject: [PATCH 071/115] Update test.py --- tests/integration/test_storage_s3_queue/test.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/integration/test_storage_s3_queue/test.py b/tests/integration/test_storage_s3_queue/test.py index d7804af4666..b93e560d5b9 100644 --- a/tests/integration/test_storage_s3_queue/test.py +++ b/tests/integration/test_storage_s3_queue/test.py @@ -1784,7 +1784,7 @@ def test_commit_on_limit(started_cluster): assert 1 == int( node.query( - "SELECT value FROM system.events WHERE name = 'S3QueueFailedFiles' SETTINGS system_events_show_zero_values=1" + "SELECT value FROM system.events WHERE name = 'ObjectStorageQueueFailedFiles' SETTINGS system_events_show_zero_values=1" ) ) From 7d3cf887a843f5167512d222e7be2bec27676732 Mon Sep 17 00:00:00 2001 From: Alexander Tokmakov Date: Wed, 26 Jun 2024 22:34:21 +0200 Subject: [PATCH 072/115] Update ClientBase.cpp --- src/Client/ClientBase.cpp | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/Client/ClientBase.cpp b/src/Client/ClientBase.cpp index c97837b685d..30c8938d167 100644 --- a/src/Client/ClientBase.cpp +++ b/src/Client/ClientBase.cpp @@ -2862,7 +2862,10 @@ private: } - +/// Enable optimizations even in debug builds because otherwise options parsing becomes extremely slow affecting .sh tests +#if defined(__clang__) +#pragma clang optimize on +#endif void ClientBase::parseAndCheckOptions(OptionsDescription & options_description, po::variables_map & options, Arguments & arguments) { if (allow_repeated_settings) From fa108feab97b495a16a724480629af6ff046de8d Mon Sep 17 00:00:00 2001 From: Alexander Tokmakov Date: Wed, 26 Jun 2024 23:52:44 +0200 Subject: [PATCH 073/115] fix --- src/Common/ZooKeeper/ZooKeeper.cpp | 5 + src/Common/ZooKeeper/ZooKeeper.h | 6 +- .../examples/zkutil_test_commands_new_lib.cpp | 12 +- .../test.py | 116 +++--------------- 4 files changed, 29 insertions(+), 110 deletions(-) diff --git a/src/Common/ZooKeeper/ZooKeeper.cpp b/src/Common/ZooKeeper/ZooKeeper.cpp index 56db9adb787..24d8c4a3c2a 100644 --- a/src/Common/ZooKeeper/ZooKeeper.cpp +++ b/src/Common/ZooKeeper/ZooKeeper.cpp @@ -51,6 +51,11 @@ const int CreateMode::Ephemeral = 1; const int CreateMode::PersistentSequential = 2; const int CreateMode::EphemeralSequential = 3; +bool ShuffleHost::compare(const ShuffleHost & lhs, const ShuffleHost & rhs) +{ + return std::forward_as_tuple(lhs.az_info, lhs.priority, lhs.random) + < std::forward_as_tuple(lhs.az_info, rhs.priority, rhs.random); +} static void check(Coordination::Error code, const std::string & path) { diff --git a/src/Common/ZooKeeper/ZooKeeper.h b/src/Common/ZooKeeper/ZooKeeper.h index 9fa41130e76..98600ec2074 100644 --- a/src/Common/ZooKeeper/ZooKeeper.h +++ b/src/Common/ZooKeeper/ZooKeeper.h @@ -71,11 +71,7 @@ struct ShuffleHost random = thread_local_rng(); } - static bool compare(const ShuffleHost & lhs, const ShuffleHost & rhs) - { - return std::forward_as_tuple(lhs.az_info, lhs.priority, lhs.random) - < std::forward_as_tuple(lhs.az_info, rhs.priority, rhs.random); - } + static bool compare(const ShuffleHost & lhs, const ShuffleHost & rhs); }; using ShuffleHosts = std::vector; diff --git a/src/Common/ZooKeeper/examples/zkutil_test_commands_new_lib.cpp b/src/Common/ZooKeeper/examples/zkutil_test_commands_new_lib.cpp index a95e8ac5381..b3a1564b8ab 100644 --- a/src/Common/ZooKeeper/examples/zkutil_test_commands_new_lib.cpp +++ b/src/Common/ZooKeeper/examples/zkutil_test_commands_new_lib.cpp @@ -26,17 +26,21 @@ try Poco::Logger::root().setLevel("trace"); zkutil::ZooKeeperArgs args{argv[1]}; - ZooKeeper::Nodes nodes; + zkutil::ShuffleHosts nodes; nodes.reserve(args.hosts.size()); for (size_t i = 0; i < args.hosts.size(); ++i) { + zkutil::ShuffleHost node; std::string host_string = args.hosts[i]; - bool secure = startsWith(host_string, "secure://"); + node.secure = startsWith(host_string, "secure://"); - if (secure) + if (node.secure) host_string.erase(0, strlen("secure://")); - nodes.emplace_back(ZooKeeper::Node{Poco::Net::SocketAddress{host_string}, static_cast(i) , secure}); + node.host = host_string; + node.original_index = i; + + nodes.emplace_back(node); } ZooKeeper zk(nodes, args, nullptr); diff --git a/tests/integration/test_zookeeper_config_load_balancing/test.py b/tests/integration/test_zookeeper_config_load_balancing/test.py index 8ef69f0c74f..9cdf7db2b08 100644 --- a/tests/integration/test_zookeeper_config_load_balancing/test.py +++ b/tests/integration/test_zookeeper_config_load_balancing/test.py @@ -411,113 +411,27 @@ def test_hostname_levenshtein_distance(started_cluster): def test_round_robin(started_cluster): pm = PartitionManager() try: - pm._add_rule( - { - "source": node1.ip_address, - "destination": cluster.get_instance_ip("zoo1"), - "action": "REJECT --reject-with tcp-reset", - } - ) - pm._add_rule( - { - "source": node2.ip_address, - "destination": cluster.get_instance_ip("zoo1"), - "action": "REJECT --reject-with tcp-reset", - } - ) - pm._add_rule( - { - "source": node3.ip_address, - "destination": cluster.get_instance_ip("zoo1"), - "action": "REJECT --reject-with tcp-reset", - } - ) change_balancing("random", "round_robin") - - print( - str( - node1.exec_in_container( - [ - "bash", - "-c", - "lsof -a -i4 -i6 -itcp -w | grep ':2181' | grep ESTABLISHED", - ], - privileged=True, - user="root", - ) + for node in [node1, node2, node3]: + idx = int( + node.query("select index from system.zookeeper_connection").strip() ) - ) - assert ( - "1" - == str( - node1.exec_in_container( - [ - "bash", - "-c", - "lsof -a -i4 -i6 -itcp -w | grep 'testzookeeperconfigloadbalancing_zoo2_1.*testzookeeperconfigloadbalancing_default:2181' | grep ESTABLISHED | wc -l", - ], - privileged=True, - user="root", - ) - ).strip() - ) + new_idx = (idx + 1) % 3 - print( - str( - node2.exec_in_container( - [ - "bash", - "-c", - "lsof -a -i4 -i6 -itcp -w | grep ':2181' | grep ESTABLISHED", - ], - privileged=True, - user="root", - ) + pm._add_rule( + { + "source": node.ip_address, + "destination": cluster.get_instance_ip("zoo" + str(idx + 1)), + "action": "REJECT --reject-with tcp-reset", + } ) - ) - assert ( - "1" - == str( - node2.exec_in_container( - [ - "bash", - "-c", - "lsof -a -i4 -i6 -itcp -w | grep 'testzookeeperconfigloadbalancing_zoo2_1.*testzookeeperconfigloadbalancing_default:2181' | grep ESTABLISHED | wc -l", - ], - privileged=True, - user="root", - ) - ).strip() - ) - print( - str( - node3.exec_in_container( - [ - "bash", - "-c", - "lsof -a -i4 -i6 -itcp -w | grep ':2181' | grep ESTABLISHED", - ], - privileged=True, - user="root", - ) + assert_eq_with_retry( + node, + "select index from system.zookeeper_connection", + str(new_idx) + "\n", ) - ) - assert ( - "1" - == str( - node3.exec_in_container( - [ - "bash", - "-c", - "lsof -a -i4 -i6 -itcp -w | grep 'testzookeeperconfigloadbalancing_zoo2_1.*testzookeeperconfigloadbalancing_default:2181' | grep ESTABLISHED | wc -l", - ], - privileged=True, - user="root", - ) - ).strip() - ) - + pm.heal_all() finally: pm.heal_all() change_balancing("round_robin", "random", reload=False) From 443440ab87fe32f410d24fbdd219d48752fe9eca Mon Sep 17 00:00:00 2001 From: Konstantin Bogdanov Date: Thu, 27 Jun 2024 03:44:09 +0200 Subject: [PATCH 074/115] One more fix --- src/Client/LocalConnection.cpp | 12 ++++++++---- src/Client/LocalConnection.h | 1 + 2 files changed, 9 insertions(+), 4 deletions(-) diff --git a/src/Client/LocalConnection.cpp b/src/Client/LocalConnection.cpp index 47730edf695..44064414afa 100644 --- a/src/Client/LocalConnection.cpp +++ b/src/Client/LocalConnection.cpp @@ -356,7 +356,7 @@ bool LocalConnection::poll(size_t) try { - pollImpl(); + while (pollImpl()); } catch (const Exception & e) { @@ -456,7 +456,11 @@ bool LocalConnection::pollImpl() Block block; auto next_read = pullBlock(block); - if (block && !state->io.null_format) + if (!block && next_read) + { + return true; + } + else if (block && !state->io.null_format) { state->block.emplace(block); } @@ -465,7 +469,7 @@ bool LocalConnection::pollImpl() state->is_finished = true; } - return true; + return false; } Packet LocalConnection::receivePacket() @@ -478,7 +482,7 @@ Packet LocalConnection::receivePacket() } if (!next_packet_type) - while (state && !poll(0)); + poll(0); if (!next_packet_type) { diff --git a/src/Client/LocalConnection.h b/src/Client/LocalConnection.h index 899d134cce5..21b58c68bfd 100644 --- a/src/Client/LocalConnection.h +++ b/src/Client/LocalConnection.h @@ -151,6 +151,7 @@ private: void sendProfileEvents(); + /// Returns true on executor timeout, meaning a retryable error. bool pollImpl(); ContextMutablePtr query_context; From c9fa974472221cbb7163eca97c51d7db9c50f600 Mon Sep 17 00:00:00 2001 From: Antonio Andelic Date: Thu, 27 Jun 2024 08:46:10 +0200 Subject: [PATCH 075/115] Use background thread --- contrib/jemalloc-cmake/CMakeLists.txt | 4 +- .../test.py | 41 +++++++++---------- 2 files changed, 22 insertions(+), 23 deletions(-) diff --git a/contrib/jemalloc-cmake/CMakeLists.txt b/contrib/jemalloc-cmake/CMakeLists.txt index 7b8feaa80d4..6c874221a94 100644 --- a/contrib/jemalloc-cmake/CMakeLists.txt +++ b/contrib/jemalloc-cmake/CMakeLists.txt @@ -34,9 +34,9 @@ if (OS_LINUX) # avoid spurious latencies and additional work associated with # MADV_DONTNEED. See # https://github.com/ClickHouse/ClickHouse/issues/11121 for motivation. - set (JEMALLOC_CONFIG_MALLOC_CONF "percpu_arena:percpu,oversize_threshold:0,muzzy_decay_ms:0,dirty_decay_ms:5000,prof:true,prof_active:false") + set (JEMALLOC_CONFIG_MALLOC_CONF "percpu_arena:percpu,oversize_threshold:0,muzzy_decay_ms:0,dirty_decay_ms:5000,prof:true,prof_active:false,background_thread:true") else() - set (JEMALLOC_CONFIG_MALLOC_CONF "oversize_threshold:0,muzzy_decay_ms:0,dirty_decay_ms:5000,prof:true,prof_active:false") + set (JEMALLOC_CONFIG_MALLOC_CONF "oversize_threshold:0,muzzy_decay_ms:0,dirty_decay_ms:5000,prof:true,prof_active:false,background_thread:true") endif() # CACHE variable is empty to allow changing defaults without the necessity # to purge cache diff --git a/tests/integration/test_asynchronous_metric_jemalloc_profile_active/test.py b/tests/integration/test_asynchronous_metric_jemalloc_profile_active/test.py index a8f4ab05888..b3769a61b3f 100644 --- a/tests/integration/test_asynchronous_metric_jemalloc_profile_active/test.py +++ b/tests/integration/test_asynchronous_metric_jemalloc_profile_active/test.py @@ -7,7 +7,6 @@ cluster = ClickHouseCluster(__file__) node1 = cluster.add_instance( "node1", main_configs=["configs/asynchronous_metrics_update_period_s.xml"], - env_variables={"MALLOC_CONF": "background_thread:true,prof:true"}, ) @@ -29,26 +28,11 @@ def test_asynchronous_metric_jemalloc_profile_active(started_cluster): if node1.is_built_with_sanitizer(): pytest.skip("Disabled for sanitizers") - res_o = node1.query( + res = node1.query( "SELECT * FROM system.asynchronous_metrics WHERE metric ILIKE '%jemalloc.prof.active%' FORMAT Vertical;" ) assert ( - res_o - == """Row 1: -────── -metric: jemalloc.prof.active -value: 1 -description: An internal metric of the low-level memory allocator (jemalloc). See https://jemalloc.net/jemalloc.3.html -""" - ) - # disable - node1.query("SYSTEM JEMALLOC DISABLE PROFILE") - time.sleep(5) - res_t = node1.query( - "SELECT * FROM system.asynchronous_metrics WHERE metric ILIKE '%jemalloc.prof.active%' FORMAT Vertical;" - ) - assert ( - res_t + res == """Row 1: ────── metric: jemalloc.prof.active @@ -58,16 +42,31 @@ description: An internal metric of the low-level memory allocator (jemalloc). Se ) # enable node1.query("SYSTEM JEMALLOC ENABLE PROFILE") - time.sleep(5) - res_f = node1.query( + node1.query("SYSTEM RELOAD ASYNCHRONOUS METRICS") + res = node1.query( "SELECT * FROM system.asynchronous_metrics WHERE metric ILIKE '%jemalloc.prof.active%' FORMAT Vertical;" ) assert ( - res_f + res == """Row 1: ────── metric: jemalloc.prof.active value: 1 description: An internal metric of the low-level memory allocator (jemalloc). See https://jemalloc.net/jemalloc.3.html +""" + ) + # disable + node1.query("SYSTEM JEMALLOC DISABLE PROFILE") + node1.query("SYSTEM RELOAD ASYNCHRONOUS METRICS") + res = node1.query( + "SELECT * FROM system.asynchronous_metrics WHERE metric ILIKE '%jemalloc.prof.active%' FORMAT Vertical;" + ) + assert ( + res + == """Row 1: +────── +metric: jemalloc.prof.active +value: 0 +description: An internal metric of the low-level memory allocator (jemalloc). See https://jemalloc.net/jemalloc.3.html """ ) From d06d2a69db53007991d6e8387f0eff6447ac3fd8 Mon Sep 17 00:00:00 2001 From: Smita Kulkarni Date: Thu, 27 Jun 2024 11:17:36 +0200 Subject: [PATCH 076/115] Fixed tests --- tests/config/install.sh | 10 +-- ...system_cache_on_write_operations.reference | 79 +++++++++++++++++++ 2 files changed, 81 insertions(+), 8 deletions(-) diff --git a/tests/config/install.sh b/tests/config/install.sh index 4dd7b894e71..08ee11a7407 100755 --- a/tests/config/install.sh +++ b/tests/config/install.sh @@ -182,14 +182,8 @@ elif [[ "$USE_AZURE_STORAGE_FOR_MERGE_TREE" == "1" ]]; then ln -sf $SRC_PATH/config.d/azure_storage_policy_by_default.xml $DEST_SERVER_PATH/config.d/ fi -if [[ -n "$EXPORT_S3_STORAGE_POLICIES" ]]; then\ - if [[ "$USE_DATABASE_REPLICATED" -eq 1 ]]; then - echo "Azure configuration will not be added" - else - echo "Adding azure configuration" - ln -sf $SRC_PATH/config.d/azure_storage_conf.xml $DEST_SERVER_PATH/config.d/ - fi - +if [[ -n "$EXPORT_S3_STORAGE_POLICIES" ]]; then + ln -sf $SRC_PATH/config.d/azure_storage_conf.xml $DEST_SERVER_PATH/config.d/ ln -sf $SRC_PATH/config.d/storage_conf.xml $DEST_SERVER_PATH/config.d/ ln -sf $SRC_PATH/config.d/storage_conf_02944.xml $DEST_SERVER_PATH/config.d/ ln -sf $SRC_PATH/config.d/storage_conf_02963.xml $DEST_SERVER_PATH/config.d/ diff --git a/tests/queries/0_stateless/02241_filesystem_cache_on_write_operations.reference b/tests/queries/0_stateless/02241_filesystem_cache_on_write_operations.reference index 410e4c9ee5d..f53f00992e7 100644 --- a/tests/queries/0_stateless/02241_filesystem_cache_on_write_operations.reference +++ b/tests/queries/0_stateless/02241_filesystem_cache_on_write_operations.reference @@ -219,33 +219,112 @@ SELECT count() FROM test_02241 SELECT count() FROM test_02241 WHERE value LIKE '%010%' 18816 Using storage policy: azure_cache +DROP TABLE IF EXISTS test_02241 +CREATE TABLE test_02241 (key UInt32, value String) Engine=MergeTree() ORDER BY key SETTINGS storage_policy='azure_cache', min_bytes_for_wide_part = 10485760, compress_marks=false, compress_primary_key=false, ratio_of_defaults_for_sparse_serialization = 1 +SYSTEM STOP MERGES test_02241 +SYSTEM DROP FILESYSTEM CACHE +SELECT file_segment_range_begin, file_segment_range_end, size, state + FROM + ( + SELECT file_segment_range_begin, file_segment_range_end, size, state, local_path + FROM + ( + SELECT arrayJoin(cache_paths) AS cache_path, local_path, remote_path + FROM system.remote_data_paths + ) AS data_paths + INNER JOIN + system.filesystem_cache AS caches + ON data_paths.cache_path = caches.cache_path + ) + WHERE endsWith(local_path, 'data.bin') + FORMAT Vertical +SELECT count() FROM (SELECT arrayJoin(cache_paths) AS cache_path, local_path, remote_path FROM system.remote_data_paths ) AS data_paths INNER JOIN system.filesystem_cache AS caches ON data_paths.cache_path = caches.cache_path 0 +SELECT count(), sum(size) FROM system.filesystem_cache 0 0 +INSERT INTO test_02241 SELECT number, toString(number) FROM numbers(100) +SELECT file_segment_range_begin, file_segment_range_end, size, state + FROM + ( + SELECT file_segment_range_begin, file_segment_range_end, size, state, local_path + FROM + ( + SELECT arrayJoin(cache_paths) AS cache_path, local_path, remote_path + FROM system.remote_data_paths + ) AS data_paths + INNER JOIN + system.filesystem_cache AS caches + ON data_paths.cache_path = caches.cache_path + ) + WHERE endsWith(local_path, 'data.bin') + FORMAT Vertical Row 1: ────── file_segment_range_begin: 0 file_segment_range_end: 745 size: 746 state: DOWNLOADED +SELECT count() FROM (SELECT arrayJoin(cache_paths) AS cache_path, local_path, remote_path FROM system.remote_data_paths ) AS data_paths INNER JOIN system.filesystem_cache AS caches ON data_paths.cache_path = caches.cache_path 8 +SELECT count(), sum(size) FROM system.filesystem_cache 8 1100 +SELECT count() FROM system.filesystem_cache WHERE cache_hits > 0 0 +SELECT * FROM test_02241 FORMAT Null +SELECT count() FROM system.filesystem_cache WHERE cache_hits > 0 2 +SELECT * FROM test_02241 FORMAT Null +SELECT count() FROM system.filesystem_cache WHERE cache_hits > 0 2 +SELECT count(), sum(size) size FROM system.filesystem_cache 8 1100 +SYSTEM DROP FILESYSTEM CACHE +INSERT INTO test_02241 SELECT number, toString(number) FROM numbers(100, 200) +SELECT file_segment_range_begin, file_segment_range_end, size, state + FROM + ( + SELECT file_segment_range_begin, file_segment_range_end, size, state, local_path + FROM + ( + SELECT arrayJoin(cache_paths) AS cache_path, local_path, remote_path + FROM system.remote_data_paths + ) AS data_paths + INNER JOIN + system.filesystem_cache AS caches + ON data_paths.cache_path = caches.cache_path + ) + WHERE endsWith(local_path, 'data.bin') + FORMAT Vertical; Row 1: ────── file_segment_range_begin: 0 file_segment_range_end: 1659 size: 1660 state: DOWNLOADED +SELECT count() FROM (SELECT arrayJoin(cache_paths) AS cache_path, local_path, remote_path FROM system.remote_data_paths ) AS data_paths INNER JOIN system.filesystem_cache AS caches ON data_paths.cache_path = caches.cache_path 8 +SELECT count(), sum(size) FROM system.filesystem_cache 8 2014 +SELECT count(), sum(size) FROM system.filesystem_cache 8 2014 +INSERT INTO test_02241 SELECT number, toString(number) FROM numbers(100) SETTINGS enable_filesystem_cache_on_write_operations=0 +SELECT count(), sum(size) FROM system.filesystem_cache 8 2014 +INSERT INTO test_02241 SELECT number, toString(number) FROM numbers(100) +INSERT INTO test_02241 SELECT number, toString(number) FROM numbers(300, 10000) +SELECT count(), sum(size) FROM system.filesystem_cache 24 84045 +SYSTEM START MERGES test_02241 +OPTIMIZE TABLE test_02241 FINAL +SELECT count(), sum(size) FROM system.filesystem_cache 32 167243 +ALTER TABLE test_02241 UPDATE value = 'kek' WHERE key = 100 +SELECT count(), sum(size) FROM system.filesystem_cache 41 250541 +INSERT INTO test_02241 SELECT number, toString(number) FROM numbers(5000000) +SYSTEM FLUSH LOGS INSERT INTO test_02241 SELECT number, toString(number) FROM numbers(5000000) 0 +SELECT count() FROM test_02241 5010500 +SELECT count() FROM test_02241 WHERE value LIKE '%010%' 18816 From 86c3036bfb9eb1f307a79b35efacac3409e87a13 Mon Sep 17 00:00:00 2001 From: Antonio Andelic Date: Thu, 27 Jun 2024 14:04:44 +0200 Subject: [PATCH 077/115] Fix flaky test_replicated_database::test_alter_attach --- tests/integration/test_replicated_database/test.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tests/integration/test_replicated_database/test.py b/tests/integration/test_replicated_database/test.py index f23384b5c04..089c7f57d05 100644 --- a/tests/integration/test_replicated_database/test.py +++ b/tests/integration/test_replicated_database/test.py @@ -337,8 +337,10 @@ def test_alter_attach(started_cluster, attachable_part, engine): main_node.query(f"SELECT CounterID FROM {database}.alter_attach_test") == "123\n" ) + # On the other node, data is replicated only if using a Replicated table engine if engine == "ReplicatedMergeTree": + dummy_node.query(f"SYSTEM SYNC REPLICA {database}.alter_attach_test LIGHTWEIGHT") assert ( dummy_node.query(f"SELECT CounterID FROM {database}.alter_attach_test") == "123\n" From 988ba2e22dba550d5610a3658571d0bb3493b4c9 Mon Sep 17 00:00:00 2001 From: Antonio Andelic Date: Thu, 27 Jun 2024 14:17:37 +0200 Subject: [PATCH 078/115] Fix mysqlISO8601Date --- src/Functions/parseDateTime.cpp | 3 +-- .../0_stateless/03197_fix_parse_mysql_iso_date.reference | 1 + tests/queries/0_stateless/03197_fix_parse_mysql_iso_date.sql | 1 + 3 files changed, 3 insertions(+), 2 deletions(-) create mode 100644 tests/queries/0_stateless/03197_fix_parse_mysql_iso_date.reference create mode 100644 tests/queries/0_stateless/03197_fix_parse_mysql_iso_date.sql diff --git a/src/Functions/parseDateTime.cpp b/src/Functions/parseDateTime.cpp index 11e210d2cc2..7407eadf19c 100644 --- a/src/Functions/parseDateTime.cpp +++ b/src/Functions/parseDateTime.cpp @@ -1015,8 +1015,7 @@ namespace [[nodiscard]] static PosOrError mysqlISO8601Date(Pos cur, Pos end, const String & fragment, DateTime & date) { - if (auto status = checkSpace(cur, end, 10, "mysqlISO8601Date requires size >= 10", fragment)) - return tl::unexpected(status.error()); + RETURN_ERROR_IF_FAILED(checkSpace(cur, end, 10, "mysqlISO8601Date requires size >= 10", fragment)) Int32 year; Int32 month; diff --git a/tests/queries/0_stateless/03197_fix_parse_mysql_iso_date.reference b/tests/queries/0_stateless/03197_fix_parse_mysql_iso_date.reference new file mode 100644 index 00000000000..b3ac1c01adc --- /dev/null +++ b/tests/queries/0_stateless/03197_fix_parse_mysql_iso_date.reference @@ -0,0 +1 @@ +2024-06-20 20:00:00 diff --git a/tests/queries/0_stateless/03197_fix_parse_mysql_iso_date.sql b/tests/queries/0_stateless/03197_fix_parse_mysql_iso_date.sql new file mode 100644 index 00000000000..34796df6299 --- /dev/null +++ b/tests/queries/0_stateless/03197_fix_parse_mysql_iso_date.sql @@ -0,0 +1 @@ +SELECT parseDateTime(formatDateTime(toDateTime('2024-06-20 12:00:00'), '%F %T', 'Europe/Paris'), '%F %T') AS x From ade0906910bf5aa7bb5433f3bbef411d44d0ab17 Mon Sep 17 00:00:00 2001 From: Igor Nikonov Date: Thu, 27 Jun 2024 12:28:50 +0000 Subject: [PATCH 079/115] Fix: progress bar for read in order queries w/o limit --- src/Processors/QueryPlan/ReadFromMergeTree.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Processors/QueryPlan/ReadFromMergeTree.cpp b/src/Processors/QueryPlan/ReadFromMergeTree.cpp index aba3f6ff2da..c6fc58f9c62 100644 --- a/src/Processors/QueryPlan/ReadFromMergeTree.cpp +++ b/src/Processors/QueryPlan/ReadFromMergeTree.cpp @@ -565,7 +565,7 @@ Pipe ReadFromMergeTree::readInOrder( /// In this case we won't set approximate rows, because it will be accounted multiple times. /// Also do not count amount of read rows if we read in order of sorting key, /// because we don't know actual amount of read rows in case when limit is set. - bool set_rows_approx = !is_parallel_reading_from_replicas && !reader_settings.read_in_order; + bool set_rows_approx = !is_parallel_reading_from_replicas && !query_info.limit; Pipes pipes; for (size_t i = 0; i < parts_with_ranges.size(); ++i) From b972906dab69ddd46b4e66506d1a0868cc4ebf6b Mon Sep 17 00:00:00 2001 From: Antonio Andelic Date: Thu, 27 Jun 2024 14:42:21 +0200 Subject: [PATCH 080/115] Fix timezone --- .../0_stateless/03197_fix_parse_mysql_iso_date.reference | 2 +- tests/queries/0_stateless/03197_fix_parse_mysql_iso_date.sql | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/queries/0_stateless/03197_fix_parse_mysql_iso_date.reference b/tests/queries/0_stateless/03197_fix_parse_mysql_iso_date.reference index b3ac1c01adc..94d8605fd74 100644 --- a/tests/queries/0_stateless/03197_fix_parse_mysql_iso_date.reference +++ b/tests/queries/0_stateless/03197_fix_parse_mysql_iso_date.reference @@ -1 +1 @@ -2024-06-20 20:00:00 +2024-06-20 12:00:00 diff --git a/tests/queries/0_stateless/03197_fix_parse_mysql_iso_date.sql b/tests/queries/0_stateless/03197_fix_parse_mysql_iso_date.sql index 34796df6299..9a996803249 100644 --- a/tests/queries/0_stateless/03197_fix_parse_mysql_iso_date.sql +++ b/tests/queries/0_stateless/03197_fix_parse_mysql_iso_date.sql @@ -1 +1 @@ -SELECT parseDateTime(formatDateTime(toDateTime('2024-06-20 12:00:00'), '%F %T', 'Europe/Paris'), '%F %T') AS x +SELECT parseDateTime(formatDateTime(toDateTime('2024-06-20 12:00:00'), '%F %T', 'Europe/Paris'), '%F %T', 'UTC') AS x From 77e51dc693eb82dd1427e96413ddaee11f2a93a6 Mon Sep 17 00:00:00 2001 From: Igor Nikonov Date: Thu, 27 Jun 2024 12:54:40 +0000 Subject: [PATCH 081/115] Rename --- src/Processors/QueryPlan/ReadFromMergeTree.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/Processors/QueryPlan/ReadFromMergeTree.cpp b/src/Processors/QueryPlan/ReadFromMergeTree.cpp index c6fc58f9c62..bce007bcddb 100644 --- a/src/Processors/QueryPlan/ReadFromMergeTree.cpp +++ b/src/Processors/QueryPlan/ReadFromMergeTree.cpp @@ -565,7 +565,7 @@ Pipe ReadFromMergeTree::readInOrder( /// In this case we won't set approximate rows, because it will be accounted multiple times. /// Also do not count amount of read rows if we read in order of sorting key, /// because we don't know actual amount of read rows in case when limit is set. - bool set_rows_approx = !is_parallel_reading_from_replicas && !query_info.limit; + const bool set_total_rows_approx = !is_parallel_reading_from_replicas && !query_info.limit; Pipes pipes; for (size_t i = 0; i < parts_with_ranges.size(); ++i) @@ -595,7 +595,7 @@ Pipe ReadFromMergeTree::readInOrder( processor->addPartLevelToChunk(isQueryWithFinal()); auto source = std::make_shared(std::move(processor)); - if (set_rows_approx) + if (set_total_rows_approx) source->addTotalRowsApprox(total_rows); pipes.emplace_back(std::move(source)); From fcf8189b75e1239319fe38c36711be9a6d91bc2e Mon Sep 17 00:00:00 2001 From: Antonio Andelic Date: Thu, 27 Jun 2024 15:06:02 +0200 Subject: [PATCH 082/115] More fixes --- src/Functions/parseDateTime.cpp | 12 +++++------- .../03197_fix_parse_mysql_iso_date.reference | 3 ++- .../0_stateless/03197_fix_parse_mysql_iso_date.sql | 3 ++- 3 files changed, 9 insertions(+), 9 deletions(-) diff --git a/src/Functions/parseDateTime.cpp b/src/Functions/parseDateTime.cpp index 7407eadf19c..162b8c58873 100644 --- a/src/Functions/parseDateTime.cpp +++ b/src/Functions/parseDateTime.cpp @@ -978,8 +978,7 @@ namespace [[nodiscard]] static PosOrError mysqlAmericanDate(Pos cur, Pos end, const String & fragment, DateTime & date) { - if (auto status = checkSpace(cur, end, 8, "mysqlAmericanDate requires size >= 8", fragment)) - return tl::unexpected(status.error()); + RETURN_ERROR_IF_FAILED(checkSpace(cur, end, 8, "mysqlAmericanDate requires size >= 8", fragment)) Int32 month; ASSIGN_RESULT_OR_RETURN_ERROR(cur, (readNumber2(cur, end, fragment, month))) @@ -993,7 +992,7 @@ namespace Int32 year; ASSIGN_RESULT_OR_RETURN_ERROR(cur, (readNumber2(cur, end, fragment, year))) - RETURN_ERROR_IF_FAILED(date.setYear(year)) + RETURN_ERROR_IF_FAILED(date.setYear(year + 2000)) return cur; } @@ -1461,8 +1460,7 @@ namespace [[nodiscard]] static PosOrError jodaDayOfWeekText(size_t /*min_represent_digits*/, Pos cur, Pos end, const String & fragment, DateTime & date) { - if (auto result= checkSpace(cur, end, 3, "jodaDayOfWeekText requires size >= 3", fragment); !result.has_value()) - return tl::unexpected(result.error()); + RETURN_ERROR_IF_FAILED(checkSpace(cur, end, 3, "jodaDayOfWeekText requires size >= 3", fragment)) String text1(cur, 3); boost::to_lower(text1); @@ -1555,8 +1553,8 @@ namespace Int32 day_of_month; ASSIGN_RESULT_OR_RETURN_ERROR(cur, (readNumberWithVariableLength( cur, end, false, false, false, repetitions, std::max(repetitions, 2uz), fragment, day_of_month))) - if (auto res = date.setDayOfMonth(day_of_month); !res.has_value()) - return tl::unexpected(res.error()); + RETURN_ERROR_IF_FAILED(date.setDayOfMonth(day_of_month)) + return cur; } diff --git a/tests/queries/0_stateless/03197_fix_parse_mysql_iso_date.reference b/tests/queries/0_stateless/03197_fix_parse_mysql_iso_date.reference index 94d8605fd74..bd9ab3be3fa 100644 --- a/tests/queries/0_stateless/03197_fix_parse_mysql_iso_date.reference +++ b/tests/queries/0_stateless/03197_fix_parse_mysql_iso_date.reference @@ -1 +1,2 @@ -2024-06-20 12:00:00 +2024-06-20 00:00:00 +2024-06-20 00:00:00 diff --git a/tests/queries/0_stateless/03197_fix_parse_mysql_iso_date.sql b/tests/queries/0_stateless/03197_fix_parse_mysql_iso_date.sql index 9a996803249..e83738f7214 100644 --- a/tests/queries/0_stateless/03197_fix_parse_mysql_iso_date.sql +++ b/tests/queries/0_stateless/03197_fix_parse_mysql_iso_date.sql @@ -1 +1,2 @@ -SELECT parseDateTime(formatDateTime(toDateTime('2024-06-20 12:00:00'), '%F %T', 'Europe/Paris'), '%F %T', 'UTC') AS x +SELECT parseDateTime('2024-06-20', '%F', 'UTC') AS x; +SELECT parseDateTime('06/20/24', '%D', 'UTC') AS x; From 8c4c2b64c5aadeda927b9c2ba2a69e9123fb24b8 Mon Sep 17 00:00:00 2001 From: Alexander Tokmakov Date: Thu, 27 Jun 2024 16:39:25 +0200 Subject: [PATCH 083/115] fix --- src/Common/ZooKeeper/ZooKeeper.cpp | 5 ----- src/Common/ZooKeeper/ZooKeeper.h | 6 +++++- utils/keeper-bench/Runner.cpp | 10 +++++++--- 3 files changed, 12 insertions(+), 9 deletions(-) diff --git a/src/Common/ZooKeeper/ZooKeeper.cpp b/src/Common/ZooKeeper/ZooKeeper.cpp index 24d8c4a3c2a..56db9adb787 100644 --- a/src/Common/ZooKeeper/ZooKeeper.cpp +++ b/src/Common/ZooKeeper/ZooKeeper.cpp @@ -51,11 +51,6 @@ const int CreateMode::Ephemeral = 1; const int CreateMode::PersistentSequential = 2; const int CreateMode::EphemeralSequential = 3; -bool ShuffleHost::compare(const ShuffleHost & lhs, const ShuffleHost & rhs) -{ - return std::forward_as_tuple(lhs.az_info, lhs.priority, lhs.random) - < std::forward_as_tuple(lhs.az_info, rhs.priority, rhs.random); -} static void check(Coordination::Error code, const std::string & path) { diff --git a/src/Common/ZooKeeper/ZooKeeper.h b/src/Common/ZooKeeper/ZooKeeper.h index 98600ec2074..4ae2cfa6096 100644 --- a/src/Common/ZooKeeper/ZooKeeper.h +++ b/src/Common/ZooKeeper/ZooKeeper.h @@ -71,7 +71,11 @@ struct ShuffleHost random = thread_local_rng(); } - static bool compare(const ShuffleHost & lhs, const ShuffleHost & rhs); + static bool compare(const ShuffleHost & lhs, const ShuffleHost & rhs) + { + return std::forward_as_tuple(lhs.az_info, lhs.priority, lhs.random) + < std::forward_as_tuple(rhs.az_info, rhs.priority, rhs.random); + } }; using ShuffleHosts = std::vector; diff --git a/utils/keeper-bench/Runner.cpp b/utils/keeper-bench/Runner.cpp index ed7e09685f0..5ae4c7a0b1c 100644 --- a/utils/keeper-bench/Runner.cpp +++ b/utils/keeper-bench/Runner.cpp @@ -1238,9 +1238,13 @@ void Runner::createConnections() std::shared_ptr Runner::getConnection(const ConnectionInfo & connection_info, size_t connection_info_idx) { - Coordination::ZooKeeper::Node node{Poco::Net::SocketAddress{connection_info.host}, static_cast(connection_info_idx), connection_info.secure}; - std::vector nodes; - nodes.push_back(node); + zkutil::ShuffleHost host; + host.host = connection_info.host; + host.secure = connection_info.secure; + host.original_index = static_cast(connection_info_idx); + host.address = Poco::Net::SocketAddress{connection_info.host}; + + zkutil::ShuffleHosts nodes{host}; zkutil::ZooKeeperArgs args; args.session_timeout_ms = connection_info.session_timeout_ms; args.connection_timeout_ms = connection_info.connection_timeout_ms; From 3362b521d23e1031fa01181a49e6f6abb469bb29 Mon Sep 17 00:00:00 2001 From: robot-clickhouse Date: Thu, 27 Jun 2024 14:43:44 +0000 Subject: [PATCH 084/115] Automatic style fix --- tests/integration/test_replicated_database/test.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/tests/integration/test_replicated_database/test.py b/tests/integration/test_replicated_database/test.py index 089c7f57d05..60a6e099b22 100644 --- a/tests/integration/test_replicated_database/test.py +++ b/tests/integration/test_replicated_database/test.py @@ -340,7 +340,9 @@ def test_alter_attach(started_cluster, attachable_part, engine): # On the other node, data is replicated only if using a Replicated table engine if engine == "ReplicatedMergeTree": - dummy_node.query(f"SYSTEM SYNC REPLICA {database}.alter_attach_test LIGHTWEIGHT") + dummy_node.query( + f"SYSTEM SYNC REPLICA {database}.alter_attach_test LIGHTWEIGHT" + ) assert ( dummy_node.query(f"SELECT CounterID FROM {database}.alter_attach_test") == "123\n" From 7618ce12bb4ed11cdbc64b30b3cdaf0e8087a007 Mon Sep 17 00:00:00 2001 From: Igor Nikonov Date: Thu, 27 Jun 2024 18:54:32 +0000 Subject: [PATCH 085/115] Set total rows for in order queries with limit + renaming misleading query_info.limit -> trivial_limit --- src/Interpreters/InterpreterSelectQuery.cpp | 8 +++---- src/Planner/PlannerJoinTree.cpp | 10 ++++----- .../QueryPlan/ReadFromMergeTree.cpp | 21 ++++++++++--------- .../QueryPlan/ReadFromSystemNumbersStep.cpp | 2 +- src/QueryPipeline/SizeLimits.cpp | 1 - src/Storages/MergeTree/MergeTreeData.cpp | 4 ++-- src/Storages/SelectQueryInfo.h | 4 ++-- src/Storages/StorageGenerateRandom.cpp | 4 ++-- src/Storages/System/StorageSystemZeros.cpp | 4 ++-- 9 files changed, 29 insertions(+), 29 deletions(-) diff --git a/src/Interpreters/InterpreterSelectQuery.cpp b/src/Interpreters/InterpreterSelectQuery.cpp index c39c57d2169..90c484636ea 100644 --- a/src/Interpreters/InterpreterSelectQuery.cpp +++ b/src/Interpreters/InterpreterSelectQuery.cpp @@ -910,7 +910,7 @@ bool InterpreterSelectQuery::adjustParallelReplicasAfterAnalysis() UInt64 max_rows = maxBlockSizeByLimit(); if (settings.max_rows_to_read) max_rows = max_rows ? std::min(max_rows, settings.max_rows_to_read.value) : settings.max_rows_to_read; - query_info_copy.limit = max_rows; + query_info_copy.trivial_limit = max_rows; /// Apply filters to prewhere and add them to the query_info so we can filter out parts efficiently during row estimation applyFiltersToPrewhereInAnalysis(analysis_copy); @@ -2445,13 +2445,13 @@ void InterpreterSelectQuery::executeFetchColumns(QueryProcessingStage::Enum proc if (local_limits.local_limits.size_limits.max_rows != 0) { if (max_block_limited < local_limits.local_limits.size_limits.max_rows) - query_info.limit = max_block_limited; + query_info.trivial_limit = max_block_limited; else if (local_limits.local_limits.size_limits.max_rows < std::numeric_limits::max()) /// Ask to read just enough rows to make the max_rows limit effective (so it has a chance to be triggered). - query_info.limit = 1 + local_limits.local_limits.size_limits.max_rows; + query_info.trivial_limit = 1 + local_limits.local_limits.size_limits.max_rows; } else { - query_info.limit = max_block_limited; + query_info.trivial_limit = max_block_limited; } } diff --git a/src/Planner/PlannerJoinTree.cpp b/src/Planner/PlannerJoinTree.cpp index 43b223172e6..e163672966a 100644 --- a/src/Planner/PlannerJoinTree.cpp +++ b/src/Planner/PlannerJoinTree.cpp @@ -693,14 +693,14 @@ JoinTreeQueryPlan buildQueryPlanForTableExpression(QueryTreeNodePtr table_expres if (select_query_info.local_storage_limits.local_limits.size_limits.max_rows != 0) { if (max_block_size_limited < select_query_info.local_storage_limits.local_limits.size_limits.max_rows) - table_expression_query_info.limit = max_block_size_limited; + table_expression_query_info.trivial_limit = max_block_size_limited; /// Ask to read just enough rows to make the max_rows limit effective (so it has a chance to be triggered). else if (select_query_info.local_storage_limits.local_limits.size_limits.max_rows < std::numeric_limits::max()) - table_expression_query_info.limit = 1 + select_query_info.local_storage_limits.local_limits.size_limits.max_rows; + table_expression_query_info.trivial_limit = 1 + select_query_info.local_storage_limits.local_limits.size_limits.max_rows; } else { - table_expression_query_info.limit = max_block_size_limited; + table_expression_query_info.trivial_limit = max_block_size_limited; } } @@ -913,8 +913,8 @@ JoinTreeQueryPlan buildQueryPlanForTableExpression(QueryTreeNodePtr table_expres auto result_ptr = reading->selectRangesToRead(); UInt64 rows_to_read = result_ptr->selected_rows; - if (table_expression_query_info.limit > 0 && table_expression_query_info.limit < rows_to_read) - rows_to_read = table_expression_query_info.limit; + if (table_expression_query_info.trivial_limit > 0 && table_expression_query_info.trivial_limit < rows_to_read) + rows_to_read = table_expression_query_info.trivial_limit; if (max_block_size_limited && (max_block_size_limited < rows_to_read)) rows_to_read = max_block_size_limited; diff --git a/src/Processors/QueryPlan/ReadFromMergeTree.cpp b/src/Processors/QueryPlan/ReadFromMergeTree.cpp index bce007bcddb..457facc7d2f 100644 --- a/src/Processors/QueryPlan/ReadFromMergeTree.cpp +++ b/src/Processors/QueryPlan/ReadFromMergeTree.cpp @@ -250,9 +250,9 @@ void ReadFromMergeTree::AnalysisResult::checkLimits(const Settings & settings, c { /// Fail fast if estimated number of rows to read exceeds the limit size_t total_rows_estimate = selected_rows; - if (query_info_.limit > 0 && total_rows_estimate > query_info_.limit) + if (query_info_.trivial_limit > 0 && total_rows_estimate > query_info_.trivial_limit) { - total_rows_estimate = query_info_.limit; + total_rows_estimate = query_info_.trivial_limit; } limits.check(total_rows_estimate, 0, "rows (controlled by 'max_rows_to_read' setting)", ErrorCodes::TOO_MANY_ROWS); leaf_limits.check( @@ -398,8 +398,8 @@ Pipe ReadFromMergeTree::readFromPool( { size_t total_rows = parts_with_range.getRowsCountAllParts(); - if (query_info.limit > 0 && query_info.limit < total_rows) - total_rows = query_info.limit; + if (query_info.trivial_limit > 0 && query_info.trivial_limit < total_rows) + total_rows = query_info.trivial_limit; const auto & settings = context->getSettingsRef(); @@ -436,7 +436,7 @@ Pipe ReadFromMergeTree::readFromPool( * Because time spend during filling per thread tasks can be greater than whole query * execution for big tables with small limit. */ - bool use_prefetched_read_pool = query_info.limit == 0 && (allow_prefetched_remote || allow_prefetched_local); + bool use_prefetched_read_pool = query_info.trivial_limit == 0 && (allow_prefetched_remote || allow_prefetched_local); if (use_prefetched_read_pool) { @@ -563,9 +563,8 @@ Pipe ReadFromMergeTree::readInOrder( /// Actually it means that parallel reading from replicas enabled /// and we have to collaborate with initiator. /// In this case we won't set approximate rows, because it will be accounted multiple times. - /// Also do not count amount of read rows if we read in order of sorting key, - /// because we don't know actual amount of read rows in case when limit is set. - const bool set_total_rows_approx = !is_parallel_reading_from_replicas && !query_info.limit; + const auto in_order_limit = query_info.input_order_info->limit; + const bool set_total_rows_approx = !is_parallel_reading_from_replicas; Pipes pipes; for (size_t i = 0; i < parts_with_ranges.size(); ++i) @@ -573,8 +572,10 @@ Pipe ReadFromMergeTree::readInOrder( const auto & part_with_ranges = parts_with_ranges[i]; UInt64 total_rows = part_with_ranges.getRowsCount(); - if (query_info.limit > 0 && query_info.limit < total_rows) - total_rows = query_info.limit; + if (query_info.trivial_limit > 0 && query_info.trivial_limit < total_rows) + total_rows = query_info.trivial_limit; + else if (in_order_limit > 0 && in_order_limit < total_rows) + total_rows = in_order_limit; LOG_TRACE(log, "Reading {} ranges in{}order from part {}, approx. {} rows starting from {}", part_with_ranges.ranges.size(), diff --git a/src/Processors/QueryPlan/ReadFromSystemNumbersStep.cpp b/src/Processors/QueryPlan/ReadFromSystemNumbersStep.cpp index 5dbf6fa3318..eb974259c5e 100644 --- a/src/Processors/QueryPlan/ReadFromSystemNumbersStep.cpp +++ b/src/Processors/QueryPlan/ReadFromSystemNumbersStep.cpp @@ -393,7 +393,7 @@ ReadFromSystemNumbersStep::ReadFromSystemNumbersStep( , num_streams{num_streams_} , limit_length_and_offset(InterpreterSelectQuery::getLimitLengthAndOffset(query_info.query->as(), context)) , should_pushdown_limit(shouldPushdownLimit(query_info, limit_length_and_offset.first)) - , query_info_limit(query_info.limit) + , query_info_limit(query_info.trivial_limit) , storage_limits(query_info.storage_limits) { storage_snapshot->check(column_names); diff --git a/src/QueryPipeline/SizeLimits.cpp b/src/QueryPipeline/SizeLimits.cpp index 76832b1f951..4161f3f365f 100644 --- a/src/QueryPipeline/SizeLimits.cpp +++ b/src/QueryPipeline/SizeLimits.cpp @@ -2,7 +2,6 @@ #include #include #include -#include namespace ProfileEvents diff --git a/src/Storages/MergeTree/MergeTreeData.cpp b/src/Storages/MergeTree/MergeTreeData.cpp index f9cc65871fe..dbde5d3658b 100644 --- a/src/Storages/MergeTree/MergeTreeData.cpp +++ b/src/Storages/MergeTree/MergeTreeData.cpp @@ -7111,8 +7111,8 @@ UInt64 MergeTreeData::estimateNumberOfRowsToRead( query_context->getSettingsRef().max_threads); UInt64 total_rows = result_ptr->selected_rows; - if (query_info.limit > 0 && query_info.limit < total_rows) - total_rows = query_info.limit; + if (query_info.trivial_limit > 0 && query_info.trivial_limit < total_rows) + total_rows = query_info.trivial_limit; return total_rows; } diff --git a/src/Storages/SelectQueryInfo.h b/src/Storages/SelectQueryInfo.h index 65c90fc1e6c..bdf69b9be15 100644 --- a/src/Storages/SelectQueryInfo.h +++ b/src/Storages/SelectQueryInfo.h @@ -229,8 +229,8 @@ struct SelectQueryInfo bool is_parameterized_view = false; bool optimize_trivial_count = false; - // If limit is not 0, that means it's a trivial limit query. - UInt64 limit = 0; + // If not 0, that means it's a trivial limit query. + UInt64 trivial_limit = 0; /// For IStorageSystemOneBlock std::vector columns_mask; diff --git a/src/Storages/StorageGenerateRandom.cpp b/src/Storages/StorageGenerateRandom.cpp index 2f850c76465..754bc096958 100644 --- a/src/Storages/StorageGenerateRandom.cpp +++ b/src/Storages/StorageGenerateRandom.cpp @@ -705,7 +705,7 @@ Pipe StorageGenerateRandom::read( } } - UInt64 query_limit = query_info.limit; + UInt64 query_limit = query_info.trivial_limit; if (query_limit && num_streams * max_block_size > query_limit) { /// We want to avoid spawning more streams than necessary @@ -717,7 +717,7 @@ Pipe StorageGenerateRandom::read( /// Will create more seed values for each source from initial seed. pcg64 generate(random_seed); - auto shared_state = std::make_shared(query_info.limit); + auto shared_state = std::make_shared(query_info.trivial_limit); for (UInt64 i = 0; i < num_streams; ++i) { diff --git a/src/Storages/System/StorageSystemZeros.cpp b/src/Storages/System/StorageSystemZeros.cpp index 09a2bb5d963..0720a2f24d9 100644 --- a/src/Storages/System/StorageSystemZeros.cpp +++ b/src/Storages/System/StorageSystemZeros.cpp @@ -109,8 +109,8 @@ Pipe StorageSystemZeros::read( storage_snapshot->check(column_names); UInt64 query_limit = limit ? *limit : 0; - if (query_info.limit) - query_limit = query_limit ? std::min(query_limit, query_info.limit) : query_info.limit; + if (query_info.trivial_limit) + query_limit = query_limit ? std::min(query_limit, query_info.trivial_limit) : query_info.trivial_limit; if (query_limit && query_limit < max_block_size) max_block_size = query_limit; From bf1cc136db4b3ee1cad70f93c08039a31aafe1c0 Mon Sep 17 00:00:00 2001 From: Andrey Zvonov Date: Thu, 27 Jun 2024 19:30:45 +0000 Subject: [PATCH 086/115] make a better healthcheck --- tests/integration/compose/docker_compose_ldap.yml | 5 ++++- tests/integration/helpers/cluster.py | 4 +++- 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/tests/integration/compose/docker_compose_ldap.yml b/tests/integration/compose/docker_compose_ldap.yml index f199516f315..1f50b34735d 100644 --- a/tests/integration/compose/docker_compose_ldap.yml +++ b/tests/integration/compose/docker_compose_ldap.yml @@ -15,7 +15,10 @@ services: ports: - ${LDAP_EXTERNAL_PORT:-1389}:${LDAP_INTERNAL_PORT:-1389} healthcheck: - test: "ldapsearch -x -b dc=example,dc=org cn > /dev/null" + test: > + ldapsearch -x -H ldap://localhost:$$LDAP_PORT_NUMBER -D $$LDAP_ADMIN_DN -w $$LDAP_ADMIN_PASSWORD -b $$LDAP_ROOT + | grep -c -E "member: cn=j(ohn|ane)doe" + | grep 2 >> /dev/null interval: 10s retries: 10 timeout: 2s diff --git a/tests/integration/helpers/cluster.py b/tests/integration/helpers/cluster.py index 41c162217d2..803171c0dd0 100644 --- a/tests/integration/helpers/cluster.py +++ b/tests/integration/helpers/cluster.py @@ -2640,7 +2640,9 @@ class ClickHouseCluster: [ "bash", "-c", - f"/opt/bitnami/openldap/bin/ldapsearch -x -H ldap://{self.ldap_host}:{self.ldap_port} -D cn=admin,dc=example,dc=org -w clickhouse -b dc=example,dc=org", + f'/opt/bitnami/openldap/bin/ldapsearch -x -H ldap://{self.ldap_host}:{self.ldap_port} -D cn=admin,dc=example,dc=org -w clickhouse -b dc=example,dc=org' + f'| grep -c -E "member: cn=j(ohn|ane)doe"' + f'| grep 2 >> /dev/null', ], user="root", ) From 0fe0bc8c8d05f100a6db7bacb3bdc00ad8e4c1cf Mon Sep 17 00:00:00 2001 From: Max Kainov Date: Thu, 27 Jun 2024 19:34:41 +0000 Subject: [PATCH 087/115] AMI image with gh and jwt --- tests/ci/worker/prepare-ci-ami.sh | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/tests/ci/worker/prepare-ci-ami.sh b/tests/ci/worker/prepare-ci-ami.sh index 3e2f33c89d1..33b3889b196 100644 --- a/tests/ci/worker/prepare-ci-ami.sh +++ b/tests/ci/worker/prepare-ci-ami.sh @@ -9,7 +9,7 @@ set -xeuo pipefail echo "Running prepare script" export DEBIAN_FRONTEND=noninteractive -export RUNNER_VERSION=2.316.1 +export RUNNER_VERSION=2.317.0 export RUNNER_HOME=/home/ubuntu/actions-runner deb_arch() { @@ -54,7 +54,8 @@ apt-get install --yes --no-install-recommends \ python3-dev \ python3-pip \ qemu-user-static \ - unzip + unzip \ + gh # Install docker curl -fsSL https://download.docker.com/linux/ubuntu/gpg | gpg --dearmor -o /usr/share/keyrings/docker-archive-keyring.gpg @@ -101,7 +102,7 @@ sudo -u ubuntu docker buildx version sudo -u ubuntu docker buildx rm default-builder || : # if it's the second attempt sudo -u ubuntu docker buildx create --use --name default-builder -pip install boto3 pygithub requests urllib3 unidiff dohq-artifactory +pip install boto3 pygithub requests urllib3 unidiff dohq-artifactory jwt rm -rf $RUNNER_HOME # if it's the second attempt mkdir -p $RUNNER_HOME && cd $RUNNER_HOME From 9a9c36cc5fd04c3b5df42be07bb32a295163f89f Mon Sep 17 00:00:00 2001 From: Igor Nikonov Date: Thu, 27 Jun 2024 20:04:30 +0000 Subject: [PATCH 088/115] Fix --- src/Processors/QueryPlan/ReadFromMergeTree.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Processors/QueryPlan/ReadFromMergeTree.cpp b/src/Processors/QueryPlan/ReadFromMergeTree.cpp index 457facc7d2f..433dd4beee8 100644 --- a/src/Processors/QueryPlan/ReadFromMergeTree.cpp +++ b/src/Processors/QueryPlan/ReadFromMergeTree.cpp @@ -563,7 +563,7 @@ Pipe ReadFromMergeTree::readInOrder( /// Actually it means that parallel reading from replicas enabled /// and we have to collaborate with initiator. /// In this case we won't set approximate rows, because it will be accounted multiple times. - const auto in_order_limit = query_info.input_order_info->limit; + const auto in_order_limit = query_info.input_order_info ? query_info.input_order_info->limit : 0; const bool set_total_rows_approx = !is_parallel_reading_from_replicas; Pipes pipes; From 2dc571b759b40672047d831e184c1f63e2377868 Mon Sep 17 00:00:00 2001 From: Nikita Taranov Date: Fri, 28 Jun 2024 00:50:50 +0200 Subject: [PATCH 089/115] Forbid join algorithm randomisation for 03094_one_thousand_joins --- tests/queries/0_stateless/03094_one_thousand_joins.sql | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/queries/0_stateless/03094_one_thousand_joins.sql b/tests/queries/0_stateless/03094_one_thousand_joins.sql index ea159f0e4c0..1f6bd99df7f 100644 --- a/tests/queries/0_stateless/03094_one_thousand_joins.sql +++ b/tests/queries/0_stateless/03094_one_thousand_joins.sql @@ -1,6 +1,7 @@ -- Tags: no-fasttest, no-tsan, long -- (no-tsan because it has a small maximum stack size and the test would fail with TOO_DEEP_RECURSION) +SET join_algorithm = 'default'; -- for 'full_sorting_merge' the query is 10x slower SET allow_experimental_analyzer = 1; -- old analyzer returns TOO_DEEP_SUBQUERIES -- Bug 33446, marked as 'long' because it still runs around 10 sec From 3bc41192d8db143ccc8049b0acdfb3c8dfded340 Mon Sep 17 00:00:00 2001 From: Michael Kolupaev Date: Thu, 27 Jun 2024 23:27:14 +0000 Subject: [PATCH 090/115] Fix 02931_rewrite_sum_column_and_constant flakiness --- ..._rewrite_sum_column_and_constant.reference | 26 +++++++++---------- .../02931_rewrite_sum_column_and_constant.sql | 11 ++++---- 2 files changed, 19 insertions(+), 18 deletions(-) diff --git a/tests/queries/0_stateless/02931_rewrite_sum_column_and_constant.reference b/tests/queries/0_stateless/02931_rewrite_sum_column_and_constant.reference index 3124698d218..f9b72ba9c6a 100644 --- a/tests/queries/0_stateless/02931_rewrite_sum_column_and_constant.reference +++ b/tests/queries/0_stateless/02931_rewrite_sum_column_and_constant.reference @@ -245,21 +245,21 @@ EXPLAIN SYNTAX (SELECT 2 * count(uint64) - sum(uint64) From test_table); SELECT (2 * count(uint64)) - sum(uint64) FROM test_table SELECT sum(float64 + 2) From test_table; -26.5 +26.875 SELECT sum(2 + float64) From test_table; -26.5 +26.875 SELECT sum(float64 - 2) From test_table; -6.5 +6.875 SELECT sum(2 - float64) From test_table; --6.5 +-6.875 SELECT sum(float64) + 2 * count(float64) From test_table; -26.5 +26.875 SELECT 2 * count(float64) + sum(float64) From test_table; -26.5 +26.875 SELECT sum(float64) - 2 * count(float64) From test_table; -6.5 +6.875 SELECT 2 * count(float64) - sum(float64) From test_table; --6.5 +-6.875 EXPLAIN SYNTAX (SELECT sum(float64 + 2) From test_table); SELECT sum(float64) + (2 * count(float64)) FROM test_table @@ -375,25 +375,25 @@ EXPLAIN SYNTAX (SELECT (2 * count(uint64) - sum(uint64)) + (3 * count(uint64) - SELECT ((2 * count(uint64)) - sum(uint64)) + ((3 * count(uint64)) - sum(uint64)) FROM test_table SELECT sum(float64 + 2) + sum(float64 + 3) From test_table; -58 +58.75 SELECT sum(float64 + 2) - sum(float64 + 3) From test_table; -5 SELECT sum(float64 - 2) + sum(float64 - 3) From test_table; -8 +8.75 SELECT sum(float64 - 2) - sum(float64 - 3) From test_table; 5 SELECT sum(2 - float64) - sum(3 - float64) From test_table; -5 SELECT (sum(float64) + 2 * count(float64)) + (sum(float64) + 3 * count(float64)) From test_table; -58 +58.75 SELECT (sum(float64) + 2 * count(float64)) - (sum(float64) + 3 * count(float64)) From test_table; -5 SELECT (sum(float64) - 2 * count(float64)) + (sum(float64) - 3 * count(float64)) From test_table; -8 +8.75 SELECT (sum(float64) - 2 * count(float64)) - (sum(float64) - 3 * count(float64)) From test_table; 5 SELECT (2 * count(float64) - sum(float64)) + (3 * count(float64) - sum(float64)) From test_table; --8 +-8.75 EXPLAIN SYNTAX (SELECT sum(float64 + 2) + sum(float64 + 3) From test_table); SELECT (sum(float64) + (2 * count(float64))) + (sum(float64) + (3 * count(float64))) FROM test_table diff --git a/tests/queries/0_stateless/02931_rewrite_sum_column_and_constant.sql b/tests/queries/0_stateless/02931_rewrite_sum_column_and_constant.sql index c7b0ff82442..94baee6f1ba 100644 --- a/tests/queries/0_stateless/02931_rewrite_sum_column_and_constant.sql +++ b/tests/queries/0_stateless/02931_rewrite_sum_column_and_constant.sql @@ -23,11 +23,12 @@ CREATE TABLE test_table decimal32 Decimal32(5), ) ENGINE=MergeTree ORDER BY uint64; -INSERT INTO test_table VALUES (1, 1.1, 1.11); -INSERT INTO test_table VALUES (2, 2.2, 2.22); -INSERT INTO test_table VALUES (3, 3.3, 3.33); -INSERT INTO test_table VALUES (4, 4.4, 4.44); -INSERT INTO test_table VALUES (5, 5.5, 5.55); +-- Use Float64 numbers divisible by 1/16 (or some other small power of two), so that their sum doesn't depend on summation order. +INSERT INTO test_table VALUES (1, 1.125, 1.11); +INSERT INTO test_table VALUES (2, 2.250, 2.22); +INSERT INTO test_table VALUES (3, 3.375, 3.33); +INSERT INTO test_table VALUES (4, 4.500, 4.44); +INSERT INTO test_table VALUES (5, 5.625, 5.55); -- { echoOn } SELECT sum(uint64 + 1 AS i) from test_table where i > 0; From 8dbf159f3faf4da0e0bfa6dbb09518e38816978e Mon Sep 17 00:00:00 2001 From: jsc0218 Date: Fri, 28 Jun 2024 01:24:20 +0000 Subject: [PATCH 091/115] fix --- src/Interpreters/ExpressionActions.cpp | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/src/Interpreters/ExpressionActions.cpp b/src/Interpreters/ExpressionActions.cpp index 04f29f35c3c..87082497581 100644 --- a/src/Interpreters/ExpressionActions.cpp +++ b/src/Interpreters/ExpressionActions.cpp @@ -194,6 +194,10 @@ static void setLazyExecutionInfo( } lazy_execution_info.short_circuit_ancestors_info[parent].insert(indexes.begin(), indexes.end()); + /// After checking arguments_with_disabled_lazy_execution, if there is no relation with parent, + /// disable the current node. + if (indexes.empty()) + lazy_execution_info.can_be_lazy_executed = false; } else /// If lazy execution is disabled for one of parents, we should disable it for current node. @@ -291,9 +295,9 @@ static std::unordered_set processShortCircuitFunctions /// Firstly, find all short-circuit functions and get their settings. std::unordered_map short_circuit_nodes; - IFunctionBase::ShortCircuitSettings short_circuit_settings; for (const auto & node : nodes) { + IFunctionBase::ShortCircuitSettings short_circuit_settings; if (node.type == ActionsDAG::ActionType::FUNCTION && node.function_base->isShortCircuit(short_circuit_settings, node.children.size()) && !node.children.empty()) short_circuit_nodes[&node] = short_circuit_settings; } From f15755ecc6b14ff7cfb803fefc58d248b9dafb43 Mon Sep 17 00:00:00 2001 From: jsc0218 Date: Fri, 28 Jun 2024 01:38:06 +0000 Subject: [PATCH 092/115] add test --- .../03071_fix_short_circuit_logic.reference | 1 + .../03071_fix_short_circuit_logic.sql | 62 +++++++++++++++++++ 2 files changed, 63 insertions(+) create mode 100644 tests/queries/0_stateless/03071_fix_short_circuit_logic.reference create mode 100644 tests/queries/0_stateless/03071_fix_short_circuit_logic.sql diff --git a/tests/queries/0_stateless/03071_fix_short_circuit_logic.reference b/tests/queries/0_stateless/03071_fix_short_circuit_logic.reference new file mode 100644 index 00000000000..48aedfc3958 --- /dev/null +++ b/tests/queries/0_stateless/03071_fix_short_circuit_logic.reference @@ -0,0 +1 @@ +2024-01-02 16:54:59 diff --git a/tests/queries/0_stateless/03071_fix_short_circuit_logic.sql b/tests/queries/0_stateless/03071_fix_short_circuit_logic.sql new file mode 100644 index 00000000000..7745bceca0b --- /dev/null +++ b/tests/queries/0_stateless/03071_fix_short_circuit_logic.sql @@ -0,0 +1,62 @@ + + +CREATE FUNCTION IF NOT EXISTS unhexPrefixed AS value -> unhex(substring(value, 3)); +CREATE FUNCTION IF NOT EXISTS hex2bytes AS address -> CAST(unhexPrefixed(address), 'FixedString(20)'); +CREATE FUNCTION IF NOT EXISTS bytes2hex AS address -> concat('0x', lower(hex(address))); + +CREATE TABLE test +( + `transfer_id` String, + `address` FixedString(20), + `value` UInt256, + `block_timestamp` DateTime('UTC'), + `token_address` FixedString(20) +) +ENGINE = MergeTree +PARTITION BY toYYYYMM(block_timestamp) +PRIMARY KEY (address, block_timestamp) +ORDER BY (address, block_timestamp); + +INSERT INTO test SELECT 'token-transfer-0x758f1bbabb160683e1c80ed52dcd24a32b599d40edf1cec91b5f1199c0e392a2-56', hex2bytes('0xd387a6e4e84a6c86bd90c158c6028a58cc8ac459'), 3000000000000000000000, '2024-01-02 16:54:59', 'abc'; + +CREATE TABLE token_data +( + token_address_hex String, + chain String, + is_blacklisted Bool +) +ENGINE = TinyLog; + +INSERT INTO token_data SELECT bytes2hex('abc'), 'zksync', false; + +CREATE DICTIONARY token_data_map +( + token_address_hex String, + chain String, + is_blacklisted Bool +) +PRIMARY KEY token_address_hex, chain +SOURCE(Clickhouse(table token_data)) +LIFETIME(MIN 200 MAX 300) +LAYOUT(COMPLEX_KEY_HASHED_ARRAY()); + +SELECT block_timestamp +FROM +( + SELECT + block_timestamp, + bytes2hex(token_address) AS token_address_hex + FROM + ( + SELECT + transfer_id, + address, + value, + block_timestamp, + token_address, + 'zksync' AS chain + FROM test + ) + WHERE (address = hex2bytes('0xd387a6e4e84a6c86bd90c158c6028a58cc8ac459')) AND (transfer_id NOT LIKE 'gas%') AND (value > 0) AND (dictGetOrDefault(token_data_map, 'is_blacklisted', (token_address_hex, 'zksync'), true)) +) +SETTINGS max_threads = 1, short_circuit_function_evaluation = 'enable', allow_experimental_analyzer = 0; \ No newline at end of file From bbfdf1698f0296a254ff63f929822334a0788a3d Mon Sep 17 00:00:00 2001 From: Konstantin Bogdanov Date: Fri, 28 Jun 2024 05:13:56 +0200 Subject: [PATCH 093/115] Fix --- base/base/getFQDNOrHostName.cpp | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/base/base/getFQDNOrHostName.cpp b/base/base/getFQDNOrHostName.cpp index 2a4ba8e2e11..6b3da9699b9 100644 --- a/base/base/getFQDNOrHostName.cpp +++ b/base/base/getFQDNOrHostName.cpp @@ -6,6 +6,9 @@ namespace { std::string getFQDNOrHostNameImpl() { +#if defined(OS_DARWIN) + return Poco::Net::DNS::hostName(); +#else try { return Poco::Net::DNS::thisHost().name(); @@ -14,6 +17,7 @@ namespace { return Poco::Net::DNS::hostName(); } +#endif } } From f3028acc3dfa8551494681a5b40772dbafc12b6c Mon Sep 17 00:00:00 2001 From: Konstantin Bogdanov Date: Fri, 28 Jun 2024 08:03:55 +0200 Subject: [PATCH 094/115] Yet another fix --- src/Client/LocalConnection.cpp | 18 +++++++++++++++++- 1 file changed, 17 insertions(+), 1 deletion(-) diff --git a/src/Client/LocalConnection.cpp b/src/Client/LocalConnection.cpp index 44064414afa..6d2fc176f0b 100644 --- a/src/Client/LocalConnection.cpp +++ b/src/Client/LocalConnection.cpp @@ -356,7 +356,23 @@ bool LocalConnection::poll(size_t) try { - while (pollImpl()); + while (pollImpl()) + { + LOG_DEBUG(&Poco::Logger::get("LocalConnection"), "Executor timeout encountered"); + + if (send_progress && (state->after_send_progress.elapsedMicroseconds() >= query_context->getSettingsRef().interactive_delay)) + { + state->after_send_progress.restart(); + next_packet_type = Protocol::Server::Progress; + return true; + } + + if (send_profile_events && (state->after_send_profile_events.elapsedMicroseconds() >= query_context->getSettingsRef().interactive_delay)) + { + sendProfileEvents(); + return true; + } + } } catch (const Exception & e) { From 3ad17f5854fb591da725bf5cd51a8bc713e4dbea Mon Sep 17 00:00:00 2001 From: kssenii Date: Fri, 28 Jun 2024 13:11:35 +0200 Subject: [PATCH 095/115] Fix test 00429_http_long_bufferization.sh --- src/Interpreters/Cache/WriteBufferToFileSegment.cpp | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/src/Interpreters/Cache/WriteBufferToFileSegment.cpp b/src/Interpreters/Cache/WriteBufferToFileSegment.cpp index d3a2a967bec..e654d091561 100644 --- a/src/Interpreters/Cache/WriteBufferToFileSegment.cpp +++ b/src/Interpreters/Cache/WriteBufferToFileSegment.cpp @@ -4,6 +4,7 @@ #include #include #include +#include #include @@ -130,7 +131,10 @@ std::unique_ptr WriteBufferToFileSegment::getReadBufferImpl() * because in case destructor called without `getReadBufferImpl` called, data won't be read. */ finalize(); - return std::make_unique(file_segment->getPath()); + if (file_segment->getDownloadedSize() > 0) + return std::make_unique(file_segment->getPath()); + else + return std::make_unique(); } } From 8e034c499f58685216157d9351c4a2cc3296af81 Mon Sep 17 00:00:00 2001 From: vdimir Date: Fri, 28 Jun 2024 11:58:10 +0000 Subject: [PATCH 096/115] Propigate join_any_take_last_row to hash join in any query --- src/Core/Settings.h | 2 +- src/Planner/PlannerJoins.cpp | 11 +++++------ .../0_stateless/00830_join_overwrite.reference | 2 ++ tests/queries/0_stateless/00830_join_overwrite.sql | 9 +++++++++ 4 files changed, 17 insertions(+), 7 deletions(-) diff --git a/src/Core/Settings.h b/src/Core/Settings.h index 41878142bdc..a6a997deddd 100644 --- a/src/Core/Settings.h +++ b/src/Core/Settings.h @@ -470,7 +470,7 @@ class IColumn; M(UInt64, max_rows_in_join, 0, "Maximum size of the hash table for JOIN (in number of rows).", 0) \ M(UInt64, max_bytes_in_join, 0, "Maximum size of the hash table for JOIN (in number of bytes in memory).", 0) \ M(OverflowMode, join_overflow_mode, OverflowMode::THROW, "What to do when the limit is exceeded.", 0) \ - M(Bool, join_any_take_last_row, false, "When disabled (default) ANY JOIN will take the first found row for a key. When enabled, it will take the last row seen if there are multiple rows for the same key.", IMPORTANT) \ + M(Bool, join_any_take_last_row, false, "When disabled (default) ANY JOIN will take the first found row for a key. When enabled, it will take the last row seen if there are multiple rows for the same key. Can be applied only to hash join and storage join.", IMPORTANT) \ M(JoinAlgorithm, join_algorithm, JoinAlgorithm::DEFAULT, "Specify join algorithm.", 0) \ M(UInt64, cross_join_min_rows_to_compress, 10000000, "Minimal count of rows to compress block in CROSS JOIN. Zero value means - disable this threshold. This block is compressed when any of the two thresholds (by rows or by bytes) are reached.", 0) \ M(UInt64, cross_join_min_bytes_to_compress, 1_GiB, "Minimal size of block to compress in CROSS JOIN. Zero value means - disable this threshold. This block is compressed when any of the two thresholds (by rows or by bytes) are reached.", 0) \ diff --git a/src/Planner/PlannerJoins.cpp b/src/Planner/PlannerJoins.cpp index 84efdd21336..67fe45cad7e 100644 --- a/src/Planner/PlannerJoins.cpp +++ b/src/Planner/PlannerJoins.cpp @@ -802,13 +802,12 @@ static std::shared_ptr tryCreateJoin(JoinAlgorithm algorithm, algorithm == JoinAlgorithm::PARALLEL_HASH || algorithm == JoinAlgorithm::DEFAULT) { - if (table_join->allowParallelHashJoin()) - { - auto query_context = planner_context->getQueryContext(); - return std::make_shared(query_context, table_join, query_context->getSettings().max_threads, right_table_expression_header); - } + auto query_context = planner_context->getQueryContext(); - return std::make_shared(table_join, right_table_expression_header); + if (table_join->allowParallelHashJoin()) + return std::make_shared(query_context, table_join, query_context->getSettings().max_threads, right_table_expression_header); + + return std::make_shared(table_join, right_table_expression_header, query_context->getSettingsRef().join_any_take_last_row); } if (algorithm == JoinAlgorithm::FULL_SORTING_MERGE) diff --git a/tests/queries/0_stateless/00830_join_overwrite.reference b/tests/queries/0_stateless/00830_join_overwrite.reference index 4792e70f333..e7d6081b647 100644 --- a/tests/queries/0_stateless/00830_join_overwrite.reference +++ b/tests/queries/0_stateless/00830_join_overwrite.reference @@ -1,2 +1,4 @@ 2 3 +2 +3 diff --git a/tests/queries/0_stateless/00830_join_overwrite.sql b/tests/queries/0_stateless/00830_join_overwrite.sql index cb7e277906b..bc3662528db 100644 --- a/tests/queries/0_stateless/00830_join_overwrite.sql +++ b/tests/queries/0_stateless/00830_join_overwrite.sql @@ -9,5 +9,14 @@ INSERT INTO kv_overwrite VALUES (1, 2); INSERT INTO kv_overwrite VALUES (1, 3); SELECT joinGet('kv_overwrite', 'v', toUInt32(1)); + +CREATE TABLE t2 (k UInt32, v UInt32) ENGINE = Memory; +INSERT INTO t2 VALUES (1, 2), (1, 3); + +SET allow_experimental_analyzer = 1; + +SELECT v FROM (SELECT 1 as k) t1 ANY INNER JOIN t2 USING (k) SETTINGS join_any_take_last_row = 0; +SELECT v FROM (SELECT 1 as k) t1 ANY INNER JOIN t2 USING (k) SETTINGS join_any_take_last_row = 1; + DROP TABLE kv; DROP TABLE kv_overwrite; From c8497c81b7aa7fe19f61add5c906a3004aaa9a60 Mon Sep 17 00:00:00 2001 From: Konstantin Bogdanov Date: Fri, 28 Jun 2024 13:59:16 +0200 Subject: [PATCH 097/115] Lint --- src/Client/LocalConnection.cpp | 44 ++++++++++++++++------------------ src/Client/LocalConnection.h | 2 ++ 2 files changed, 23 insertions(+), 23 deletions(-) diff --git a/src/Client/LocalConnection.cpp b/src/Client/LocalConnection.cpp index 6d2fc176f0b..13f572a21ce 100644 --- a/src/Client/LocalConnection.cpp +++ b/src/Client/LocalConnection.cpp @@ -341,37 +341,17 @@ bool LocalConnection::poll(size_t) if (!state->is_finished) { - if (send_progress && (state->after_send_progress.elapsedMicroseconds() >= query_context->getSettingsRef().interactive_delay)) - { - state->after_send_progress.restart(); - next_packet_type = Protocol::Server::Progress; + if (needSendProgressOrMetrics()) return true; - } - - if (send_profile_events && (state->after_send_profile_events.elapsedMicroseconds() >= query_context->getSettingsRef().interactive_delay)) - { - sendProfileEvents(); - return true; - } try { while (pollImpl()) { - LOG_DEBUG(&Poco::Logger::get("LocalConnection"), "Executor timeout encountered"); + LOG_DEBUG(&Poco::Logger::get("LocalConnection"), "Executor timeout encountered, will retry"); - if (send_progress && (state->after_send_progress.elapsedMicroseconds() >= query_context->getSettingsRef().interactive_delay)) - { - state->after_send_progress.restart(); - next_packet_type = Protocol::Server::Progress; + if (needSendProgressOrMetrics()) return true; - } - - if (send_profile_events && (state->after_send_profile_events.elapsedMicroseconds() >= query_context->getSettingsRef().interactive_delay)) - { - sendProfileEvents(); - return true; - } } } catch (const Exception & e) @@ -467,6 +447,24 @@ bool LocalConnection::poll(size_t) return false; } +bool LocalConnection::needSendProgressOrMetrics() +{ + if (send_progress && (state->after_send_progress.elapsedMicroseconds() >= query_context->getSettingsRef().interactive_delay)) + { + state->after_send_progress.restart(); + next_packet_type = Protocol::Server::Progress; + return true; + } + + if (send_profile_events && (state->after_send_profile_events.elapsedMicroseconds() >= query_context->getSettingsRef().interactive_delay)) + { + sendProfileEvents(); + return true; + } + + return false; +} + bool LocalConnection::pollImpl() { Block block; diff --git a/src/Client/LocalConnection.h b/src/Client/LocalConnection.h index 21b58c68bfd..fb6fa1b55eb 100644 --- a/src/Client/LocalConnection.h +++ b/src/Client/LocalConnection.h @@ -154,6 +154,8 @@ private: /// Returns true on executor timeout, meaning a retryable error. bool pollImpl(); + bool needSendProgressOrMetrics(); + ContextMutablePtr query_context; Session session; From 8f9beffc6531e1adc6bab0387bf9df624822be6e Mon Sep 17 00:00:00 2001 From: Antonio Andelic Date: Fri, 28 Jun 2024 16:04:45 +0200 Subject: [PATCH 098/115] No jemalloc profiler for non-Linux --- contrib/jemalloc-cmake/CMakeLists.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/contrib/jemalloc-cmake/CMakeLists.txt b/contrib/jemalloc-cmake/CMakeLists.txt index 6c874221a94..023fdcf103a 100644 --- a/contrib/jemalloc-cmake/CMakeLists.txt +++ b/contrib/jemalloc-cmake/CMakeLists.txt @@ -36,7 +36,7 @@ if (OS_LINUX) # https://github.com/ClickHouse/ClickHouse/issues/11121 for motivation. set (JEMALLOC_CONFIG_MALLOC_CONF "percpu_arena:percpu,oversize_threshold:0,muzzy_decay_ms:0,dirty_decay_ms:5000,prof:true,prof_active:false,background_thread:true") else() - set (JEMALLOC_CONFIG_MALLOC_CONF "oversize_threshold:0,muzzy_decay_ms:0,dirty_decay_ms:5000,prof:true,prof_active:false,background_thread:true") + set (JEMALLOC_CONFIG_MALLOC_CONF "oversize_threshold:0,muzzy_decay_ms:0,dirty_decay_ms:5000") endif() # CACHE variable is empty to allow changing defaults without the necessity # to purge cache From 0d29df0fbaf3ea03987c5818df3c519fd1df8586 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Fri, 28 Jun 2024 16:56:34 +0200 Subject: [PATCH 099/115] Update prepare-ci-ami.sh --- tests/ci/worker/prepare-ci-ami.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/ci/worker/prepare-ci-ami.sh b/tests/ci/worker/prepare-ci-ami.sh index 33b3889b196..eb410ddcb00 100644 --- a/tests/ci/worker/prepare-ci-ami.sh +++ b/tests/ci/worker/prepare-ci-ami.sh @@ -213,9 +213,9 @@ chmod +x /usr/local/share/scripts/init-network.sh touch /var/tmp/clickhouse-ci-ami.success # END OF THE SCRIPT -# TOE description +# TOE (Task Orchestrator and Executor) description # name: CIInfrastructurePrepare -# description: instals the infrastructure for ClickHouse CI runners +# description: installs the infrastructure for ClickHouse CI runners # schemaVersion: 1.0 # # phases: From 1f768f4dd72ad21c86bd0d6250719b58d048b8c0 Mon Sep 17 00:00:00 2001 From: Smita Kulkarni Date: Fri, 28 Jun 2024 21:54:33 +0200 Subject: [PATCH 100/115] Basic docs for azure blob storage authentication --- .../table-engines/integrations/azureBlobStorage.md | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/docs/en/engines/table-engines/integrations/azureBlobStorage.md b/docs/en/engines/table-engines/integrations/azureBlobStorage.md index dfc27d6b8cf..bdf96832e9d 100644 --- a/docs/en/engines/table-engines/integrations/azureBlobStorage.md +++ b/docs/en/engines/table-engines/integrations/azureBlobStorage.md @@ -56,6 +56,15 @@ SELECT * FROM test_table; - `_size` — Size of the file in bytes. Type: `Nullable(UInt64)`. If the size is unknown, the value is `NULL`. - `_time` — Last modified time of the file. Type: `Nullable(DateTime)`. If the time is unknown, the value is `NULL`. +## Authentication + +Currently there are 3 ways to authenticate: +- `Managed Identity` - Can be used by providing an `endpoint`, `connection_string` or `storage_account_url`. +- `SAS Token` - Can be used by providing an `endpoint`, `connection_string` or `storage_account_url`. It is identified by presence of '?' in the url. +- `Workload Identity` - Can be used by providing an `endpoint` or `storage_account_url`. If `use_workload_identity` parameter is set in config, ([workload identity](https://github.com/Azure/azure-sdk-for-cpp/tree/main/sdk/identity/azure-identity#authenticate-azure-hosted-applications)) is used for authentication. + + + ## See also [Azure Blob Storage Table Function](/docs/en/sql-reference/table-functions/azureBlobStorage) From 0dd4735e07319295cf52043bf9fd1fd71f0fc1ea Mon Sep 17 00:00:00 2001 From: justindeguzman Date: Fri, 28 Jun 2024 19:32:51 -0700 Subject: [PATCH 101/115] [Docs] Fix style check errors --- .../example-datasets/stackoverflow.md | 16 ++++++++-------- .../check-style/aspell-ignore/en/aspell-dict.txt | 5 +++++ 2 files changed, 13 insertions(+), 8 deletions(-) diff --git a/docs/en/getting-started/example-datasets/stackoverflow.md b/docs/en/getting-started/example-datasets/stackoverflow.md index def0ea306cb..e982a3c3dfc 100644 --- a/docs/en/getting-started/example-datasets/stackoverflow.md +++ b/docs/en/getting-started/example-datasets/stackoverflow.md @@ -7,19 +7,19 @@ description: Analyzing Stack Overflow data with ClickHouse # Analyzing Stack Overflow data with ClickHouse -This dataset contains every Post, User, Vote, Comment, Badge, PostHistory, and PostLink that has occurred on Stack Overflow. +This dataset contains every `Post`, `User`, `Vote`, `Comment`, `Badge, `PostHistory`, and `PostLink` that has occurred on Stack Overflow. Users can either download pre-prepared Parquet versions of the data, containing every post up to April 2024, or download the latest data in XML format and load this. Stack Overflow provide updates to this data periodically - historically every 3 months. The following diagram shows the schema for the available tables assuming Parquet format. -![Stackoverflow schema](./images/stackoverflow.png) +![Stack Overflow schema](./images/stackoverflow.png) A description of the schema of this data can be found [here](https://meta.stackexchange.com/questions/2677/database-schema-documentation-for-the-public-data-dump-and-sede). ## Pre-prepared data -We provide a copy of this data in Parquet format, upto date as of April 2024. While small for ClickHouse with respect to the number of rows (60 million posts), this dataset contains significant volumes of text and large String columns. +We provide a copy of this data in Parquet format, up to date as of April 2024. While small for ClickHouse with respect to the number of rows (60 million posts), this dataset contains significant volumes of text and large String columns. ```sql CREATE DATABASE stackoverflow @@ -159,7 +159,7 @@ INSERT INTO stackoverflow.badges SELECT * FROM s3('https://datasets-documentatio 0 rows in set. Elapsed: 6.635 sec. Processed 51.29 million rows, 797.05 MB (7.73 million rows/s., 120.13 MB/s.) ``` -### Postlinks +### `PostLinks` ```sql CREATE TABLE stackoverflow.postlinks @@ -178,7 +178,7 @@ INSERT INTO stackoverflow.postlinks SELECT * FROM s3('https://datasets-documenta 0 rows in set. Elapsed: 1.534 sec. Processed 6.55 million rows, 129.70 MB (4.27 million rows/s., 84.57 MB/s.) ``` -### PostHistory +### `PostHistory` ```sql CREATE TABLE stackoverflow.posthistory @@ -218,11 +218,11 @@ wget https://archive.org/download/stackexchange/stackoverflow.com-Users.7z wget https://archive.org/download/stackexchange/stackoverflow.com-Votes.7z ``` -These files are upto 35GB and can take around 30 mins to download depending on internet connection - the download server throttles at around 20MB/sec. +These files are up to 35GB and can take around 30 mins to download depending on internet connection - the download server throttles at around 20MB/sec. ### Convert to JSON -At the time of writing, ClickHouse does not have native support for XML as an input format. To load the data into ClickHouse we first convert to NdJSON. +At the time of writing, ClickHouse does not have native support for XML as an input format. To load the data into ClickHouse we first convert to NDJSON. To convert XML to JSON we recommend the [`xq`](https://github.com/kislyuk/yq) linux tool, a simple `jq` wrapper for XML documents. @@ -301,7 +301,7 @@ Peak memory usage: 224.03 MiB. ### User with the most answers (active accounts) -Account requires a UserId. +Account requires a `UserId`. ```sql SELECT diff --git a/utils/check-style/aspell-ignore/en/aspell-dict.txt b/utils/check-style/aspell-ignore/en/aspell-dict.txt index 21e0ec3e40d..ec0d2843999 100644 --- a/utils/check-style/aspell-ignore/en/aspell-dict.txt +++ b/utils/check-style/aspell-ignore/en/aspell-dict.txt @@ -572,6 +572,7 @@ MySQLDump MySQLThreads NATS NCHAR +NDJSON NEKUDOTAYIM NEWDATE NEWDECIMAL @@ -714,6 +715,8 @@ PlantUML PointDistKm PointDistM PointDistRads +PostHistory +PostLink PostgreSQLConnection PostgreSQLThreads Postgres @@ -2496,6 +2499,7 @@ sqlite sqrt src srcReplicas +stackoverflow stacktrace stacktraces startsWith @@ -2834,6 +2838,7 @@ userver utils uuid uuidv +vCPU varPop varPopStable varSamp From af4e8618cbdeddc433ace8d2c78b85deb0ec80e3 Mon Sep 17 00:00:00 2001 From: Kevin Song <39701004+kevisong@users.noreply.github.com> Date: Sat, 29 Jun 2024 13:26:09 +0800 Subject: [PATCH 102/115] Update mergetree.md correct bullet points structure --- .../table-engines/mergetree-family/mergetree.md | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/docs/zh/engines/table-engines/mergetree-family/mergetree.md b/docs/zh/engines/table-engines/mergetree-family/mergetree.md index 67bd681269b..d5ece5b23a9 100644 --- a/docs/zh/engines/table-engines/mergetree-family/mergetree.md +++ b/docs/zh/engines/table-engines/mergetree-family/mergetree.md @@ -201,18 +201,18 @@ ClickHouse 不要求主键唯一，所以您可以插入多条具有相同主键 主键中列的数量并没有明确的限制。依据数据结构，您可以在主键包含多些或少些列。这样可以： - - 改善索引的性能。 +- 改善索引的性能。 - - 如果当前主键是 `(a, b)` ，在下列情况下添加另一个 `c` 列会提升性能： + 如果当前主键是 `(a, b)` ，在下列情况下添加另一个 `c` 列会提升性能： - - 查询会使用 `c` 列作为条件 - - 很长的数据范围（ `index_granularity` 的数倍）里 `(a, b)` 都是相同的值，并且这样的情况很普遍。换言之，就是加入另一列后，可以让您的查询略过很长的数据范围。 + - 查询会使用 `c` 列作为条件 + - 很长的数据范围（ `index_granularity` 的数倍）里 `(a, b)` 都是相同的值，并且这样的情况很普遍。换言之，就是加入另一列后，可以让您的查询略过很长的数据范围。 - - 改善数据压缩。 +- 改善数据压缩。 - ClickHouse 以主键排序片段数据，所以，数据的一致性越高，压缩越好。 + ClickHouse 以主键排序片段数据，所以，数据的一致性越高，压缩越好。 - - 在[CollapsingMergeTree](collapsingmergetree.md#table_engine-collapsingmergetree) 和 [SummingMergeTree](summingmergetree.md) 引擎里进行数据合并时会提供额外的处理逻辑。 +- 在[CollapsingMergeTree](collapsingmergetree.md#table_engine-collapsingmergetree) 和 [SummingMergeTree](summingmergetree.md) 引擎里进行数据合并时会提供额外的处理逻辑。 在这种情况下，指定与主键不同的 *排序键* 也是有意义的。 From 145b6ee94091b7e8028816c439292609e14d32b7 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Sat, 29 Jun 2024 15:48:44 +0200 Subject: [PATCH 103/115] Update 00763_lock_buffer_long.sh --- tests/queries/0_stateless/00763_lock_buffer_long.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/queries/0_stateless/00763_lock_buffer_long.sh b/tests/queries/0_stateless/00763_lock_buffer_long.sh index 6871296b480..046e4efaa85 100755 --- a/tests/queries/0_stateless/00763_lock_buffer_long.sh +++ b/tests/queries/0_stateless/00763_lock_buffer_long.sh @@ -1,5 +1,6 @@ #!/usr/bin/env bash # Tags: long, no-s3-storage, no-msan, no-asan, no-tsan, no-debug +# Some kind of stress test, it doesn't make sense to test in a non-release build set -e From 089bb6e0c91b9b06936f5f89b6417b41f864e7be Mon Sep 17 00:00:00 2001 From: Andrey Zvonov Date: Sat, 29 Jun 2024 17:00:33 +0000 Subject: [PATCH 104/115] make black even more happy --- tests/integration/helpers/cluster.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/integration/helpers/cluster.py b/tests/integration/helpers/cluster.py index 803171c0dd0..544b06cca1b 100644 --- a/tests/integration/helpers/cluster.py +++ b/tests/integration/helpers/cluster.py @@ -2640,9 +2640,9 @@ class ClickHouseCluster: [ "bash", "-c", - f'/opt/bitnami/openldap/bin/ldapsearch -x -H ldap://{self.ldap_host}:{self.ldap_port} -D cn=admin,dc=example,dc=org -w clickhouse -b dc=example,dc=org' + f"/opt/bitnami/openldap/bin/ldapsearch -x -H ldap://{self.ldap_host}:{self.ldap_port} -D cn=admin,dc=example,dc=org -w clickhouse -b dc=example,dc=org" f'| grep -c -E "member: cn=j(ohn|ane)doe"' - f'| grep 2 >> /dev/null', + f"| grep 2 >> /dev/null", ], user="root", ) From 4bfbfc21d24a2c8cf22006a0ea7582fd871f5f79 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Sat, 29 Jun 2024 21:50:34 +0200 Subject: [PATCH 105/115] Fix rocksdb --- tests/queries/0_stateless/02956_rocksdb_bulk_sink.sh | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/tests/queries/0_stateless/02956_rocksdb_bulk_sink.sh b/tests/queries/0_stateless/02956_rocksdb_bulk_sink.sh index 45e65b18e07..b1d1c483396 100755 --- a/tests/queries/0_stateless/02956_rocksdb_bulk_sink.sh +++ b/tests/queries/0_stateless/02956_rocksdb_bulk_sink.sh @@ -1,5 +1,5 @@ #!/usr/bin/env bash -# Tags: no-ordinary-database, use-rocksdb +# Tags: no-ordinary-database, use-rocksdb, no-random-settings CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) # shellcheck source=../shell_config.sh @@ -45,4 +45,3 @@ ${CLICKHOUSE_CLIENT} --query "INSERT INTO rocksdb_worm SELECT number, number+1 F ${CLICKHOUSE_CLIENT} --query "INSERT INTO rocksdb_worm SELECT number, number+1 FROM numbers_mt(1000000)" & wait ${CLICKHOUSE_CLIENT} --query "SELECT count() FROM rocksdb_worm;" - From b6500bcdf11e827322fce266f930626ebc5b0c99 Mon Sep 17 00:00:00 2001 From: cw5121 Date: Wed, 26 Jun 2024 22:48:53 +0800 Subject: [PATCH 106/115] Fix and prevent compatibility settings from becoming not effective --- src/Core/SettingsChangesHistory.h | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/Core/SettingsChangesHistory.h b/src/Core/SettingsChangesHistory.h index fbc414d4f2f..a9db10d3acd 100644 --- a/src/Core/SettingsChangesHistory.h +++ b/src/Core/SettingsChangesHistory.h @@ -261,7 +261,9 @@ static const std::map Date: Fri, 28 Jun 2024 12:03:07 +0000 Subject: [PATCH 107/115] Move parsedatetime_parse_without_leading_zeros to the right place --- src/Core/SettingsChangesHistory.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Core/SettingsChangesHistory.h b/src/Core/SettingsChangesHistory.h index a9db10d3acd..5eaef35f1ef 100644 --- a/src/Core/SettingsChangesHistory.h +++ b/src/Core/SettingsChangesHistory.h @@ -242,6 +242,7 @@ static const std::map Date: Thu, 27 Jun 2024 10:25:09 +0800 Subject: [PATCH 108/115] fix --- src/Core/SettingsChangesHistory.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Core/SettingsChangesHistory.h b/src/Core/SettingsChangesHistory.h index 5eaef35f1ef..a740e84074c 100644 --- a/src/Core/SettingsChangesHistory.h +++ b/src/Core/SettingsChangesHistory.h @@ -84,7 +84,7 @@ namespace SettingsChangesHistory /// For newly added setting choose the most appropriate previous_value (for example, if new setting /// controls new feature and it's 'true' by default, use 'false' as previous_value). /// It's used to implement `compatibility` setting (see https://github.com/ClickHouse/ClickHouse/issues/35972) -static const std::map settings_changes_history = +static const std::multimap settings_changes_history = { {"24.6", {{"materialize_skip_indexes_on_insert", true, true, "Added new setting to allow to disable materialization of skip indexes on insert"}, {"materialize_statistics_on_insert", true, true, "Added new setting to allow to disable materialization of statistics on insert"}, From b97158f5b68f877ace0471abe612f7213eefcec8 Mon Sep 17 00:00:00 2001 From: cw5121 Date: Thu, 27 Jun 2024 21:50:08 +0800 Subject: [PATCH 109/115] implement with std::initializer_list --- src/Core/Settings.cpp | 1 + src/Core/SettingsChangesHistory.h | 23 ++++++++++++++++++- .../System/StorageSystemSettingsChanges.cpp | 1 + 3 files changed, 24 insertions(+), 1 deletion(-) diff --git a/src/Core/Settings.cpp b/src/Core/Settings.cpp index 8257b94cd9f..8e5e5a94dc2 100644 --- a/src/Core/Settings.cpp +++ b/src/Core/Settings.cpp @@ -142,6 +142,7 @@ void Settings::applyCompatibilitySetting(const String & compatibility_value) return; ClickHouseVersion version(compatibility_value); + auto settings_changes_history = getSettingsChangesHistory(); /// Iterate through ClickHouse version in descending order and apply reversed /// changes for each version that is higher that version from compatibility setting for (auto it = settings_changes_history.rbegin(); it != settings_changes_history.rend(); ++it) diff --git a/src/Core/SettingsChangesHistory.h b/src/Core/SettingsChangesHistory.h index a740e84074c..3b74f9fe0e5 100644 --- a/src/Core/SettingsChangesHistory.h +++ b/src/Core/SettingsChangesHistory.h @@ -84,7 +84,8 @@ namespace SettingsChangesHistory /// For newly added setting choose the most appropriate previous_value (for example, if new setting /// controls new feature and it's 'true' by default, use 'false' as previous_value). /// It's used to implement `compatibility` setting (see https://github.com/ClickHouse/ClickHouse/issues/35972) -static const std::multimap settings_changes_history = +/// Note: please check if the key already exists to prevent duplicate entries. +static std::initializer_list> settings_changes_history_init = { {"24.6", {{"materialize_skip_indexes_on_insert", true, true, "Added new setting to allow to disable materialization of skip indexes on insert"}, {"materialize_statistics_on_insert", true, true, "Added new setting to allow to disable materialization of statistics on insert"}, @@ -322,4 +323,24 @@ static const std::multimap& getSettingsChangesHistory() +{ + static std::map settings_changes_history; + static bool initialized = false; + + if (!initialized) + { + for (auto it = settings_changes_history_init.begin(); it != settings_changes_history_init.end(); ++it) + { + if (settings_changes_history.contains(it->first)) + throw Exception{ErrorCodes::BAD_ARGUMENTS, "ClickHouse version {} already exists, please check for duplicates and merge them", it->first.toString()}; + + settings_changes_history[it->first] = it->second; + } + initialized = true; + } + + return settings_changes_history; +} + } diff --git a/src/Storages/System/StorageSystemSettingsChanges.cpp b/src/Storages/System/StorageSystemSettingsChanges.cpp index de47ec52031..f9b147628c8 100644 --- a/src/Storages/System/StorageSystemSettingsChanges.cpp +++ b/src/Storages/System/StorageSystemSettingsChanges.cpp @@ -26,6 +26,7 @@ ColumnsDescription StorageSystemSettingsChanges::getColumnsDescription() void StorageSystemSettingsChanges::fillData(MutableColumns & res_columns, ContextPtr, const ActionsDAG::Node *, std::vector) const { + auto settings_changes_history = getSettingsChangesHistory(); for (auto it = settings_changes_history.rbegin(); it != settings_changes_history.rend(); ++it) { res_columns[0]->insert(it->first.toString()); From cec2aba6bb48a7f0fe2b94ec47d8ab77b13d22e9 Mon Sep 17 00:00:00 2001 From: Robert Schulze Date: Fri, 28 Jun 2024 12:34:05 +0000 Subject: [PATCH 110/115] Some fixups --- src/Core/Settings.cpp | 2 +- src/Core/SettingsChangesHistory.h | 27 +++++++++++-------- .../System/StorageSystemSettingsChanges.cpp | 2 +- 3 files changed, 18 insertions(+), 13 deletions(-) diff --git a/src/Core/Settings.cpp b/src/Core/Settings.cpp index 8e5e5a94dc2..9c9c9c1db00 100644 --- a/src/Core/Settings.cpp +++ b/src/Core/Settings.cpp @@ -142,7 +142,7 @@ void Settings::applyCompatibilitySetting(const String & compatibility_value) return; ClickHouseVersion version(compatibility_value); - auto settings_changes_history = getSettingsChangesHistory(); + const auto & settings_changes_history = getSettingsChangesHistory(); /// Iterate through ClickHouse version in descending order and apply reversed /// changes for each version that is higher that version from compatibility setting for (auto it = settings_changes_history.rbegin(); it != settings_changes_history.rend(); ++it) diff --git a/src/Core/SettingsChangesHistory.h b/src/Core/SettingsChangesHistory.h index 3b74f9fe0e5..64432054cba 100644 --- a/src/Core/SettingsChangesHistory.h +++ b/src/Core/SettingsChangesHistory.h @@ -13,7 +13,7 @@ namespace DB namespace ErrorCodes { - extern const int BAD_ARGUMENTS; + extern const int LOGICAL_ERROR; } class ClickHouseVersion @@ -85,7 +85,7 @@ namespace SettingsChangesHistory /// controls new feature and it's 'true' by default, use 'false' as previous_value). /// It's used to implement `compatibility` setting (see https://github.com/ClickHouse/ClickHouse/issues/35972) /// Note: please check if the key already exists to prevent duplicate entries. -static std::initializer_list> settings_changes_history_init = +static std::initializer_list> settings_changes_history_initializer = { {"24.6", {{"materialize_skip_indexes_on_insert", true, true, "Added new setting to allow to disable materialization of skip indexes on insert"}, {"materialize_statistics_on_insert", true, true, "Added new setting to allow to disable materialization of statistics on insert"}, @@ -323,22 +323,27 @@ static std::initializer_list& getSettingsChangesHistory() +static +const std::map & getSettingsChangesHistory() { static std::map settings_changes_history; - static bool initialized = false; - if (!initialized) + static std::once_flag initialized_flag; + std::call_once(initialized_flag, []() { - for (auto it = settings_changes_history_init.begin(); it != settings_changes_history_init.end(); ++it) + for (const auto & setting_change : settings_changes_history_initializer) { - if (settings_changes_history.contains(it->first)) - throw Exception{ErrorCodes::BAD_ARGUMENTS, "ClickHouse version {} already exists, please check for duplicates and merge them", it->first.toString()}; + /// Disallow duplicate keys in the settings changes history. Example: + /// {"21.2", {{"some_setting_1", false, true, "[...]"}}}, + /// [...] + /// {"21.2", {{"some_setting_2", false, true, "[...]"}}}, + /// As std::set has unique keys, one of the entries would be overwritten. + if (settings_changes_history.contains(setting_change.first)) + throw Exception{ErrorCodes::LOGICAL_ERROR, "Detected duplicates version '{}'", setting_change.first.toString()}; - settings_changes_history[it->first] = it->second; + settings_changes_history[setting_change.first] = setting_change.second; } - initialized = true; - } + }); return settings_changes_history; } diff --git a/src/Storages/System/StorageSystemSettingsChanges.cpp b/src/Storages/System/StorageSystemSettingsChanges.cpp index f9b147628c8..d6c83870741 100644 --- a/src/Storages/System/StorageSystemSettingsChanges.cpp +++ b/src/Storages/System/StorageSystemSettingsChanges.cpp @@ -26,7 +26,7 @@ ColumnsDescription StorageSystemSettingsChanges::getColumnsDescription() void StorageSystemSettingsChanges::fillData(MutableColumns & res_columns, ContextPtr, const ActionsDAG::Node *, std::vector) const { - auto settings_changes_history = getSettingsChangesHistory(); + const auto & settings_changes_history = getSettingsChangesHistory(); for (auto it = settings_changes_history.rbegin(); it != settings_changes_history.rend(); ++it) { res_columns[0]->insert(it->first.toString()); From 8fdb3b83e0f6c158030adda820187a9066c583ad Mon Sep 17 00:00:00 2001 From: Robert Schulze Date: Fri, 28 Jun 2024 13:18:19 +0000 Subject: [PATCH 111/115] Fix style --- src/Core/SettingsChangesHistory.h | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/src/Core/SettingsChangesHistory.h b/src/Core/SettingsChangesHistory.h index 64432054cba..1311052b5e5 100644 --- a/src/Core/SettingsChangesHistory.h +++ b/src/Core/SettingsChangesHistory.h @@ -11,11 +11,6 @@ namespace DB { -namespace ErrorCodes -{ - extern const int LOGICAL_ERROR; -} - class ClickHouseVersion { public: @@ -339,7 +334,7 @@ const std::map & get /// {"21.2", {{"some_setting_2", false, true, "[...]"}}}, /// As std::set has unique keys, one of the entries would be overwritten. if (settings_changes_history.contains(setting_change.first)) - throw Exception{ErrorCodes::LOGICAL_ERROR, "Detected duplicates version '{}'", setting_change.first.toString()}; + throw Exception{ErrorCodes::LOGICAL_ERROR, "Detected duplicate version '{}'", setting_change.first.toString()}; settings_changes_history[setting_change.first] = setting_change.second; } From 235133aa5cdd61a36e28aeec783aeb9b4520268e Mon Sep 17 00:00:00 2001 From: Robert Schulze Date: Sun, 30 Jun 2024 08:47:42 +0000 Subject: [PATCH 112/115] Move code from header to source file --- src/Core/SettingsChangesHistory.cpp | 320 +++++++++++++++++++++++++++ src/Core/SettingsChangesHistory.h | 321 +--------------------------- 2 files changed, 329 insertions(+), 312 deletions(-) create mode 100644 src/Core/SettingsChangesHistory.cpp diff --git a/src/Core/SettingsChangesHistory.cpp b/src/Core/SettingsChangesHistory.cpp new file mode 100644 index 00000000000..81b1f63e2ba --- /dev/null +++ b/src/Core/SettingsChangesHistory.cpp @@ -0,0 +1,320 @@ +#include +#include +#include +#include +#include + +namespace DB +{ + +namespace ErrorCodes +{ + extern const int BAD_ARGUMENTS; + extern const int LOGICAL_ERROR; +} + +ClickHouseVersion::ClickHouseVersion(const String & version) +{ + Strings split; + boost::split(split, version, [](char c){ return c == '.'; }); + components.reserve(split.size()); + if (split.empty()) + throw Exception{ErrorCodes::BAD_ARGUMENTS, "Cannot parse ClickHouse version here: {}", version}; + + for (const auto & split_element : split) + { + size_t component; + ReadBufferFromString buf(split_element); + if (!tryReadIntText(component, buf) || !buf.eof()) + throw Exception{ErrorCodes::BAD_ARGUMENTS, "Cannot parse ClickHouse version here: {}", version}; + components.push_back(component); + } +} + +ClickHouseVersion::ClickHouseVersion(const char * version) + : ClickHouseVersion(String(version)) +{ +} + +String ClickHouseVersion::toString() const +{ + String version = std::to_string(components[0]); + for (size_t i = 1; i < components.size(); ++i) + version += "." + std::to_string(components[i]); + + return version; +} + +// clang-format off +/// History of settings changes that controls some backward incompatible changes +/// across all ClickHouse versions. It maps ClickHouse version to settings changes that were done +/// in this version. This history contains both changes to existing settings and newly added settings. +/// Settings changes is a vector of structs +/// {setting_name, previous_value, new_value, reason}. +/// For newly added setting choose the most appropriate previous_value (for example, if new setting +/// controls new feature and it's 'true' by default, use 'false' as previous_value). +/// It's used to implement `compatibility` setting (see https://github.com/ClickHouse/ClickHouse/issues/35972) +/// Note: please check if the key already exists to prevent duplicate entries. +static std::initializer_list> settings_changes_history_initializer = +{ + {"24.6", {{"materialize_skip_indexes_on_insert", true, true, "Added new setting to allow to disable materialization of skip indexes on insert"}, + {"materialize_statistics_on_insert", true, true, "Added new setting to allow to disable materialization of statistics on insert"}, + {"input_format_parquet_use_native_reader", false, false, "When reading Parquet files, to use native reader instead of arrow reader."}, + {"hdfs_throw_on_zero_files_match", false, false, "Allow to throw an error when ListObjects request cannot match any files in HDFS engine instead of empty query result"}, + {"azure_throw_on_zero_files_match", false, false, "Allow to throw an error when ListObjects request cannot match any files in AzureBlobStorage engine instead of empty query result"}, + {"s3_validate_request_settings", true, true, "Allow to disable S3 request settings validation"}, + {"allow_experimental_full_text_index", false, false, "Enable experimental full-text index"}, + {"azure_skip_empty_files", false, false, "Allow to skip empty files in azure table engine"}, + {"hdfs_ignore_file_doesnt_exist", false, false, "Allow to return 0 rows when the requested files don't exist instead of throwing an exception in HDFS table engine"}, + {"azure_ignore_file_doesnt_exist", false, false, "Allow to return 0 rows when the requested files don't exist instead of throwing an exception in AzureBlobStorage table engine"}, + {"s3_ignore_file_doesnt_exist", false, false, "Allow to return 0 rows when the requested files don't exist instead of throwing an exception in S3 table engine"}, + {"s3_max_part_number", 10000, 10000, "Maximum part number number for s3 upload part"}, + {"s3_max_single_operation_copy_size", 32 * 1024 * 1024, 32 * 1024 * 1024, "Maximum size for a single copy operation in s3"}, + {"input_format_parquet_max_block_size", 8192, DEFAULT_BLOCK_SIZE, "Increase block size for parquet reader."}, + {"input_format_parquet_prefer_block_bytes", 0, DEFAULT_BLOCK_SIZE * 256, "Average block bytes output by parquet reader."}, + {"enable_blob_storage_log", true, true, "Write information about blob storage operations to system.blob_storage_log table"}, + {"allow_deprecated_snowflake_conversion_functions", true, false, "Disabled deprecated functions snowflakeToDateTime[64] and dateTime[64]ToSnowflake."}, + {"allow_statistic_optimize", false, false, "Old setting which popped up here being renamed."}, + {"allow_experimental_statistic", false, false, "Old setting which popped up here being renamed."}, + {"allow_statistics_optimize", false, false, "The setting was renamed. The previous name is `allow_statistic_optimize`."}, + {"allow_experimental_statistics", false, false, "The setting was renamed. The previous name is `allow_experimental_statistic`."}, + {"enable_vertical_final", false, true, "Enable vertical final by default again after fixing bug"}, + {"parallel_replicas_custom_key_range_lower", 0, 0, "Add settings to control the range filter when using parallel replicas with dynamic shards"}, + {"parallel_replicas_custom_key_range_upper", 0, 0, "Add settings to control the range filter when using parallel replicas with dynamic shards. A value of 0 disables the upper limit"}, + {"output_format_pretty_display_footer_column_names", 0, 1, "Add a setting to display column names in the footer if there are many rows. Threshold value is controlled by output_format_pretty_display_footer_column_names_min_rows."}, + {"output_format_pretty_display_footer_column_names_min_rows", 0, 50, "Add a setting to control the threshold value for setting output_format_pretty_display_footer_column_names_min_rows. Default 50."}, + {"output_format_csv_serialize_tuple_into_separate_columns", true, true, "A new way of how interpret tuples in CSV format was added."}, + {"input_format_csv_deserialize_separate_columns_into_tuple", true, true, "A new way of how interpret tuples in CSV format was added."}, + {"input_format_csv_try_infer_strings_from_quoted_tuples", true, true, "A new way of how interpret tuples in CSV format was added."}, + }}, + {"24.5", {{"allow_deprecated_error_prone_window_functions", true, false, "Allow usage of deprecated error prone window functions (neighbor, runningAccumulate, runningDifferenceStartingWithFirstValue, runningDifference)"}, + {"allow_experimental_join_condition", false, false, "Support join with inequal conditions which involve columns from both left and right table. e.g. t1.y < t2.y."}, + {"input_format_tsv_crlf_end_of_line", false, false, "Enables reading of CRLF line endings with TSV formats"}, + {"output_format_parquet_use_custom_encoder", false, true, "Enable custom Parquet encoder."}, + {"cross_join_min_rows_to_compress", 0, 10000000, "Minimal count of rows to compress block in CROSS JOIN. Zero value means - disable this threshold. This block is compressed when any of the two thresholds (by rows or by bytes) are reached."}, + {"cross_join_min_bytes_to_compress", 0, 1_GiB, "Minimal size of block to compress in CROSS JOIN. Zero value means - disable this threshold. This block is compressed when any of the two thresholds (by rows or by bytes) are reached."}, + {"http_max_chunk_size", 0, 0, "Internal limitation"}, + {"prefer_external_sort_block_bytes", 0, DEFAULT_BLOCK_SIZE * 256, "Prefer maximum block bytes for external sort, reduce the memory usage during merging."}, + {"input_format_force_null_for_omitted_fields", false, false, "Disable type-defaults for omitted fields when needed"}, + {"cast_string_to_dynamic_use_inference", false, false, "Add setting to allow converting String to Dynamic through parsing"}, + {"allow_experimental_dynamic_type", false, false, "Add new experimental Dynamic type"}, + {"azure_max_blocks_in_multipart_upload", 50000, 50000, "Maximum number of blocks in multipart upload for Azure."}, + }}, + {"24.4", {{"input_format_json_throw_on_bad_escape_sequence", true, true, "Allow to save JSON strings with bad escape sequences"}, + {"max_parsing_threads", 0, 0, "Add a separate setting to control number of threads in parallel parsing from files"}, + {"ignore_drop_queries_probability", 0, 0, "Allow to ignore drop queries in server with specified probability for testing purposes"}, + {"lightweight_deletes_sync", 2, 2, "The same as 'mutation_sync', but controls only execution of lightweight deletes"}, + {"query_cache_system_table_handling", "save", "throw", "The query cache no longer caches results of queries against system tables"}, + {"input_format_json_ignore_unnecessary_fields", false, true, "Ignore unnecessary fields and not parse them. Enabling this may not throw exceptions on json strings of invalid format or with duplicated fields"}, + {"input_format_hive_text_allow_variable_number_of_columns", false, true, "Ignore extra columns in Hive Text input (if file has more columns than expected) and treat missing fields in Hive Text input as default values."}, + {"allow_experimental_database_replicated", false, true, "Database engine Replicated is now in Beta stage"}, + {"temporary_data_in_cache_reserve_space_wait_lock_timeout_milliseconds", (10 * 60 * 1000), (10 * 60 * 1000), "Wait time to lock cache for sapce reservation in temporary data in filesystem cache"}, + {"optimize_rewrite_sum_if_to_count_if", false, true, "Only available for the analyzer, where it works correctly"}, + {"azure_allow_parallel_part_upload", "true", "true", "Use multiple threads for azure multipart upload."}, + {"max_recursive_cte_evaluation_depth", DBMS_RECURSIVE_CTE_MAX_EVALUATION_DEPTH, DBMS_RECURSIVE_CTE_MAX_EVALUATION_DEPTH, "Maximum limit on recursive CTE evaluation depth"}, + {"query_plan_convert_outer_join_to_inner_join", false, true, "Allow to convert OUTER JOIN to INNER JOIN if filter after JOIN always filters default values"}, + }}, + {"24.3", {{"s3_connect_timeout_ms", 1000, 1000, "Introduce new dedicated setting for s3 connection timeout"}, + {"allow_experimental_shared_merge_tree", false, true, "The setting is obsolete"}, + {"use_page_cache_for_disks_without_file_cache", false, false, "Added userspace page cache"}, + {"read_from_page_cache_if_exists_otherwise_bypass_cache", false, false, "Added userspace page cache"}, + {"page_cache_inject_eviction", false, false, "Added userspace page cache"}, + {"default_table_engine", "None", "MergeTree", "Set default table engine to MergeTree for better usability"}, + {"input_format_json_use_string_type_for_ambiguous_paths_in_named_tuples_inference_from_objects", false, false, "Allow to use String type for ambiguous paths during named tuple inference from JSON objects"}, + {"traverse_shadow_remote_data_paths", false, false, "Traverse shadow directory when query system.remote_data_paths."}, + {"throw_if_deduplication_in_dependent_materialized_views_enabled_with_async_insert", false, true, "Deduplication is dependent materialized view cannot work together with async inserts."}, + {"parallel_replicas_allow_in_with_subquery", false, true, "If true, subquery for IN will be executed on every follower replica"}, + {"log_processors_profiles", false, true, "Enable by default"}, + {"function_locate_has_mysql_compatible_argument_order", false, true, "Increase compatibility with MySQL's locate function."}, + {"allow_suspicious_primary_key", true, false, "Forbid suspicious PRIMARY KEY/ORDER BY for MergeTree (i.e. SimpleAggregateFunction)"}, + {"filesystem_cache_reserve_space_wait_lock_timeout_milliseconds", 1000, 1000, "Wait time to lock cache for sapce reservation in filesystem cache"}, + {"max_parser_backtracks", 0, 1000000, "Limiting the complexity of parsing"}, + {"analyzer_compatibility_join_using_top_level_identifier", false, false, "Force to resolve identifier in JOIN USING from projection"}, + {"distributed_insert_skip_read_only_replicas", false, false, "If true, INSERT into Distributed will skip read-only replicas"}, + {"keeper_max_retries", 10, 10, "Max retries for general keeper operations"}, + {"keeper_retry_initial_backoff_ms", 100, 100, "Initial backoff timeout for general keeper operations"}, + {"keeper_retry_max_backoff_ms", 5000, 5000, "Max backoff timeout for general keeper operations"}, + {"s3queue_allow_experimental_sharded_mode", false, false, "Enable experimental sharded mode of S3Queue table engine. It is experimental because it will be rewritten"}, + {"allow_experimental_analyzer", false, true, "Enable analyzer and planner by default."}, + {"merge_tree_read_split_ranges_into_intersecting_and_non_intersecting_injection_probability", 0.0, 0.0, "For testing of `PartsSplitter` - split read ranges into intersecting and non intersecting every time you read from MergeTree with the specified probability."}, + {"allow_get_client_http_header", false, false, "Introduced a new function."}, + {"output_format_pretty_row_numbers", false, true, "It is better for usability."}, + {"output_format_pretty_max_value_width_apply_for_single_value", true, false, "Single values in Pretty formats won't be cut."}, + {"output_format_parquet_string_as_string", false, true, "ClickHouse allows arbitrary binary data in the String data type, which is typically UTF-8. Parquet/ORC/Arrow Strings only support UTF-8. That's why you can choose which Arrow's data type to use for the ClickHouse String data type - String or Binary. While Binary would be more correct and compatible, using String by default will correspond to user expectations in most cases."}, + {"output_format_orc_string_as_string", false, true, "ClickHouse allows arbitrary binary data in the String data type, which is typically UTF-8. Parquet/ORC/Arrow Strings only support UTF-8. That's why you can choose which Arrow's data type to use for the ClickHouse String data type - String or Binary. While Binary would be more correct and compatible, using String by default will correspond to user expectations in most cases."}, + {"output_format_arrow_string_as_string", false, true, "ClickHouse allows arbitrary binary data in the String data type, which is typically UTF-8. Parquet/ORC/Arrow Strings only support UTF-8. That's why you can choose which Arrow's data type to use for the ClickHouse String data type - String or Binary. While Binary would be more correct and compatible, using String by default will correspond to user expectations in most cases."}, + {"output_format_parquet_compression_method", "lz4", "zstd", "Parquet/ORC/Arrow support many compression methods, including lz4 and zstd. ClickHouse supports each and every compression method. Some inferior tools, such as 'duckdb', lack support for the faster `lz4` compression method, that's why we set zstd by default."}, + {"output_format_orc_compression_method", "lz4", "zstd", "Parquet/ORC/Arrow support many compression methods, including lz4 and zstd. ClickHouse supports each and every compression method. Some inferior tools, such as 'duckdb', lack support for the faster `lz4` compression method, that's why we set zstd by default."}, + {"output_format_pretty_highlight_digit_groups", false, true, "If enabled and if output is a terminal, highlight every digit corresponding to the number of thousands, millions, etc. with underline."}, + {"geo_distance_returns_float64_on_float64_arguments", false, true, "Increase the default precision."}, + {"azure_max_inflight_parts_for_one_file", 20, 20, "The maximum number of a concurrent loaded parts in multipart upload request. 0 means unlimited."}, + {"azure_strict_upload_part_size", 0, 0, "The exact size of part to upload during multipart upload to Azure blob storage."}, + {"azure_min_upload_part_size", 16*1024*1024, 16*1024*1024, "The minimum size of part to upload during multipart upload to Azure blob storage."}, + {"azure_max_upload_part_size", 5ull*1024*1024*1024, 5ull*1024*1024*1024, "The maximum size of part to upload during multipart upload to Azure blob storage."}, + {"azure_upload_part_size_multiply_factor", 2, 2, "Multiply azure_min_upload_part_size by this factor each time azure_multiply_parts_count_threshold parts were uploaded from a single write to Azure blob storage."}, + {"azure_upload_part_size_multiply_parts_count_threshold", 500, 500, "Each time this number of parts was uploaded to Azure blob storage, azure_min_upload_part_size is multiplied by azure_upload_part_size_multiply_factor."}, + {"output_format_csv_serialize_tuple_into_separate_columns", true, true, "A new way of how interpret tuples in CSV format was added."}, + {"input_format_csv_deserialize_separate_columns_into_tuple", true, true, "A new way of how interpret tuples in CSV format was added."}, + {"input_format_csv_try_infer_strings_from_quoted_tuples", true, true, "A new way of how interpret tuples in CSV format was added."}, + }}, + {"24.2", {{"allow_suspicious_variant_types", true, false, "Don't allow creating Variant type with suspicious variants by default"}, + {"validate_experimental_and_suspicious_types_inside_nested_types", false, true, "Validate usage of experimental and suspicious types inside nested types"}, + {"output_format_values_escape_quote_with_quote", false, false, "If true escape ' with '', otherwise quoted with \\'"}, + {"output_format_pretty_single_large_number_tip_threshold", 0, 1'000'000, "Print a readable number tip on the right side of the table if the block consists of a single number which exceeds this value (except 0)"}, + {"input_format_try_infer_exponent_floats", true, false, "Don't infer floats in exponential notation by default"}, + {"query_plan_optimize_prewhere", true, true, "Allow to push down filter to PREWHERE expression for supported storages"}, + {"async_insert_max_data_size", 1000000, 10485760, "The previous value appeared to be too small."}, + {"async_insert_poll_timeout_ms", 10, 10, "Timeout in milliseconds for polling data from asynchronous insert queue"}, + {"async_insert_use_adaptive_busy_timeout", false, true, "Use adaptive asynchronous insert timeout"}, + {"async_insert_busy_timeout_min_ms", 50, 50, "The minimum value of the asynchronous insert timeout in milliseconds; it also serves as the initial value, which may be increased later by the adaptive algorithm"}, + {"async_insert_busy_timeout_max_ms", 200, 200, "The minimum value of the asynchronous insert timeout in milliseconds; async_insert_busy_timeout_ms is aliased to async_insert_busy_timeout_max_ms"}, + {"async_insert_busy_timeout_increase_rate", 0.2, 0.2, "The exponential growth rate at which the adaptive asynchronous insert timeout increases"}, + {"async_insert_busy_timeout_decrease_rate", 0.2, 0.2, "The exponential growth rate at which the adaptive asynchronous insert timeout decreases"}, + {"format_template_row_format", "", "", "Template row format string can be set directly in query"}, + {"format_template_resultset_format", "", "", "Template result set format string can be set in query"}, + {"split_parts_ranges_into_intersecting_and_non_intersecting_final", true, true, "Allow to split parts ranges into intersecting and non intersecting during FINAL optimization"}, + {"split_intersecting_parts_ranges_into_layers_final", true, true, "Allow to split intersecting parts ranges into layers during FINAL optimization"}, + {"azure_max_single_part_copy_size", 256*1024*1024, 256*1024*1024, "The maximum size of object to copy using single part copy to Azure blob storage."}, + {"min_external_table_block_size_rows", DEFAULT_INSERT_BLOCK_SIZE, DEFAULT_INSERT_BLOCK_SIZE, "Squash blocks passed to external table to specified size in rows, if blocks are not big enough"}, + {"min_external_table_block_size_bytes", DEFAULT_INSERT_BLOCK_SIZE * 256, DEFAULT_INSERT_BLOCK_SIZE * 256, "Squash blocks passed to external table to specified size in bytes, if blocks are not big enough."}, + {"parallel_replicas_prefer_local_join", true, true, "If true, and JOIN can be executed with parallel replicas algorithm, and all storages of right JOIN part are *MergeTree, local JOIN will be used instead of GLOBAL JOIN."}, + {"optimize_time_filter_with_preimage", true, true, "Optimize Date and DateTime predicates by converting functions into equivalent comparisons without conversions (e.g. toYear(col) = 2023 -> col >= '2023-01-01' AND col <= '2023-12-31')"}, + {"extract_key_value_pairs_max_pairs_per_row", 0, 0, "Max number of pairs that can be produced by the `extractKeyValuePairs` function. Used as a safeguard against consuming too much memory."}, + {"default_view_definer", "CURRENT_USER", "CURRENT_USER", "Allows to set default `DEFINER` option while creating a view"}, + {"default_materialized_view_sql_security", "DEFINER", "DEFINER", "Allows to set a default value for SQL SECURITY option when creating a materialized view"}, + {"default_normal_view_sql_security", "INVOKER", "INVOKER", "Allows to set default `SQL SECURITY` option while creating a normal view"}, + {"mysql_map_string_to_text_in_show_columns", false, true, "Reduce the configuration effort to connect ClickHouse with BI tools."}, + {"mysql_map_fixed_string_to_text_in_show_columns", false, true, "Reduce the configuration effort to connect ClickHouse with BI tools."}, + }}, + {"24.1", {{"print_pretty_type_names", false, true, "Better user experience."}, + {"input_format_json_read_bools_as_strings", false, true, "Allow to read bools as strings in JSON formats by default"}, + {"output_format_arrow_use_signed_indexes_for_dictionary", false, true, "Use signed indexes type for Arrow dictionaries by default as it's recommended"}, + {"allow_experimental_variant_type", false, false, "Add new experimental Variant type"}, + {"use_variant_as_common_type", false, false, "Allow to use Variant in if/multiIf if there is no common type"}, + {"output_format_arrow_use_64_bit_indexes_for_dictionary", false, false, "Allow to use 64 bit indexes type in Arrow dictionaries"}, + {"parallel_replicas_mark_segment_size", 128, 128, "Add new setting to control segment size in new parallel replicas coordinator implementation"}, + {"ignore_materialized_views_with_dropped_target_table", false, false, "Add new setting to allow to ignore materialized views with dropped target table"}, + {"output_format_compression_level", 3, 3, "Allow to change compression level in the query output"}, + {"output_format_compression_zstd_window_log", 0, 0, "Allow to change zstd window log in the query output when zstd compression is used"}, + {"enable_zstd_qat_codec", false, false, "Add new ZSTD_QAT codec"}, + {"enable_vertical_final", false, true, "Use vertical final by default"}, + {"output_format_arrow_use_64_bit_indexes_for_dictionary", false, false, "Allow to use 64 bit indexes type in Arrow dictionaries"}, + {"max_rows_in_set_to_optimize_join", 100000, 0, "Disable join optimization as it prevents from read in order optimization"}, + {"output_format_pretty_color", true, "auto", "Setting is changed to allow also for auto value, disabling ANSI escapes if output is not a tty"}, + {"function_visible_width_behavior", 0, 1, "We changed the default behavior of `visibleWidth` to be more precise"}, + {"max_estimated_execution_time", 0, 0, "Separate max_execution_time and max_estimated_execution_time"}, + {"iceberg_engine_ignore_schema_evolution", false, false, "Allow to ignore schema evolution in Iceberg table engine"}, + {"optimize_injective_functions_in_group_by", false, true, "Replace injective functions by it's arguments in GROUP BY section in analyzer"}, + {"update_insert_deduplication_token_in_dependent_materialized_views", false, false, "Allow to update insert deduplication token with table identifier during insert in dependent materialized views"}, + {"azure_max_unexpected_write_error_retries", 4, 4, "The maximum number of retries in case of unexpected errors during Azure blob storage write"}, + {"split_parts_ranges_into_intersecting_and_non_intersecting_final", false, true, "Allow to split parts ranges into intersecting and non intersecting during FINAL optimization"}, + {"split_intersecting_parts_ranges_into_layers_final", true, true, "Allow to split intersecting parts ranges into layers during FINAL optimization"}}}, + {"23.12", {{"allow_suspicious_ttl_expressions", true, false, "It is a new setting, and in previous versions the behavior was equivalent to allowing."}, + {"input_format_parquet_allow_missing_columns", false, true, "Allow missing columns in Parquet files by default"}, + {"input_format_orc_allow_missing_columns", false, true, "Allow missing columns in ORC files by default"}, + {"input_format_arrow_allow_missing_columns", false, true, "Allow missing columns in Arrow files by default"}}}, + {"23.11", {{"parsedatetime_parse_without_leading_zeros", false, true, "Improved compatibility with MySQL DATE_FORMAT/STR_TO_DATE"}}}, + {"23.9", {{"optimize_group_by_constant_keys", false, true, "Optimize group by constant keys by default"}, + {"input_format_json_try_infer_named_tuples_from_objects", false, true, "Try to infer named Tuples from JSON objects by default"}, + {"input_format_json_read_numbers_as_strings", false, true, "Allow to read numbers as strings in JSON formats by default"}, + {"input_format_json_read_arrays_as_strings", false, true, "Allow to read arrays as strings in JSON formats by default"}, + {"input_format_json_infer_incomplete_types_as_strings", false, true, "Allow to infer incomplete types as Strings in JSON formats by default"}, + {"input_format_json_try_infer_numbers_from_strings", true, false, "Don't infer numbers from strings in JSON formats by default to prevent possible parsing errors"}, + {"http_write_exception_in_output_format", false, true, "Output valid JSON/XML on exception in HTTP streaming."}}}, + {"23.8", {{"rewrite_count_distinct_if_with_count_distinct_implementation", false, true, "Rewrite countDistinctIf with count_distinct_implementation configuration"}}}, + {"23.7", {{"function_sleep_max_microseconds_per_block", 0, 3000000, "In previous versions, the maximum sleep time of 3 seconds was applied only for `sleep`, but not for `sleepEachRow` function. In the new version, we introduce this setting. If you set compatibility with the previous versions, we will disable the limit altogether."}}}, + {"23.6", {{"http_send_timeout", 180, 30, "3 minutes seems crazy long. Note that this is timeout for a single network write call, not for the whole upload operation."}, + {"http_receive_timeout", 180, 30, "See http_send_timeout."}}}, + {"23.5", {{"input_format_parquet_preserve_order", true, false, "Allow Parquet reader to reorder rows for better parallelism."}, + {"parallelize_output_from_storages", false, true, "Allow parallelism when executing queries that read from file/url/s3/etc. This may reorder rows."}, + {"use_with_fill_by_sorting_prefix", false, true, "Columns preceding WITH FILL columns in ORDER BY clause form sorting prefix. Rows with different values in sorting prefix are filled independently"}, + {"output_format_parquet_compliant_nested_types", false, true, "Change an internal field name in output Parquet file schema."}}}, + {"23.4", {{"allow_suspicious_indices", true, false, "If true, index can defined with identical expressions"}, + {"allow_nonconst_timezone_arguments", true, false, "Allow non-const timezone arguments in certain time-related functions like toTimeZone(), fromUnixTimestamp*(), snowflakeToDateTime*()."}, + {"connect_timeout_with_failover_ms", 50, 1000, "Increase default connect timeout because of async connect"}, + {"connect_timeout_with_failover_secure_ms", 100, 1000, "Increase default secure connect timeout because of async connect"}, + {"hedged_connection_timeout_ms", 100, 50, "Start new connection in hedged requests after 50 ms instead of 100 to correspond with previous connect timeout"}, + {"formatdatetime_f_prints_single_zero", true, false, "Improved compatibility with MySQL DATE_FORMAT()/STR_TO_DATE()"}, + {"formatdatetime_parsedatetime_m_is_month_name", false, true, "Improved compatibility with MySQL DATE_FORMAT/STR_TO_DATE"}}}, + {"23.3", {{"output_format_parquet_version", "1.0", "2.latest", "Use latest Parquet format version for output format"}, + {"input_format_json_ignore_unknown_keys_in_named_tuple", false, true, "Improve parsing JSON objects as named tuples"}, + {"input_format_native_allow_types_conversion", false, true, "Allow types conversion in Native input forma"}, + {"output_format_arrow_compression_method", "none", "lz4_frame", "Use lz4 compression in Arrow output format by default"}, + {"output_format_parquet_compression_method", "snappy", "lz4", "Use lz4 compression in Parquet output format by default"}, + {"output_format_orc_compression_method", "none", "lz4_frame", "Use lz4 compression in ORC output format by default"}, + {"async_query_sending_for_remote", false, true, "Create connections and send query async across shards"}}}, + {"23.2", {{"output_format_parquet_fixed_string_as_fixed_byte_array", false, true, "Use Parquet FIXED_LENGTH_BYTE_ARRAY type for FixedString by default"}, + {"output_format_arrow_fixed_string_as_fixed_byte_array", false, true, "Use Arrow FIXED_SIZE_BINARY type for FixedString by default"}, + {"query_plan_remove_redundant_distinct", false, true, "Remove redundant Distinct step in query plan"}, + {"optimize_duplicate_order_by_and_distinct", true, false, "Remove duplicate ORDER BY and DISTINCT if it's possible"}, + {"insert_keeper_max_retries", 0, 20, "Enable reconnections to Keeper on INSERT, improve reliability"}}}, + {"23.1", {{"input_format_json_read_objects_as_strings", 0, 1, "Enable reading nested json objects as strings while object type is experimental"}, + {"input_format_json_defaults_for_missing_elements_in_named_tuple", false, true, "Allow missing elements in JSON objects while reading named tuples by default"}, + {"input_format_csv_detect_header", false, true, "Detect header in CSV format by default"}, + {"input_format_tsv_detect_header", false, true, "Detect header in TSV format by default"}, + {"input_format_custom_detect_header", false, true, "Detect header in CustomSeparated format by default"}, + {"query_plan_remove_redundant_sorting", false, true, "Remove redundant sorting in query plan. For example, sorting steps related to ORDER BY clauses in subqueries"}}}, + {"22.12", {{"max_size_to_preallocate_for_aggregation", 10'000'000, 100'000'000, "This optimizes performance"}, + {"query_plan_aggregation_in_order", 0, 1, "Enable some refactoring around query plan"}, + {"format_binary_max_string_size", 0, 1_GiB, "Prevent allocating large amount of memory"}}}, + {"22.11", {{"use_structure_from_insertion_table_in_table_functions", 0, 2, "Improve using structure from insertion table in table functions"}}}, + {"22.9", {{"force_grouping_standard_compatibility", false, true, "Make GROUPING function output the same as in SQL standard and other DBMS"}}}, + {"22.7", {{"cross_to_inner_join_rewrite", 1, 2, "Force rewrite comma join to inner"}, + {"enable_positional_arguments", false, true, "Enable positional arguments feature by default"}, + {"format_csv_allow_single_quotes", true, false, "Most tools don't treat single quote in CSV specially, don't do it by default too"}}}, + {"22.6", {{"output_format_json_named_tuples_as_objects", false, true, "Allow to serialize named tuples as JSON objects in JSON formats by default"}, + {"input_format_skip_unknown_fields", false, true, "Optimize reading subset of columns for some input formats"}}}, + {"22.5", {{"memory_overcommit_ratio_denominator", 0, 1073741824, "Enable memory overcommit feature by default"}, + {"memory_overcommit_ratio_denominator_for_user", 0, 1073741824, "Enable memory overcommit feature by default"}}}, + {"22.4", {{"allow_settings_after_format_in_insert", true, false, "Do not allow SETTINGS after FORMAT for INSERT queries because ClickHouse interpret SETTINGS as some values, which is misleading"}}}, + {"22.3", {{"cast_ipv4_ipv6_default_on_conversion_error", true, false, "Make functions cast(value, 'IPv4') and cast(value, 'IPv6') behave same as toIPv4 and toIPv6 functions"}}}, + {"21.12", {{"stream_like_engine_allow_direct_select", true, false, "Do not allow direct select for Kafka/RabbitMQ/FileLog by default"}}}, + {"21.9", {{"output_format_decimal_trailing_zeros", true, false, "Do not output trailing zeros in text representation of Decimal types by default for better looking output"}, + {"use_hedged_requests", false, true, "Enable Hedged Requests feature by default"}}}, + {"21.7", {{"legacy_column_name_of_tuple_literal", true, false, "Add this setting only for compatibility reasons. It makes sense to set to 'true', while doing rolling update of cluster from version lower than 21.7 to higher"}}}, + {"21.5", {{"async_socket_for_remote", false, true, "Fix all problems and turn on asynchronous reads from socket for remote queries by default again"}}}, + {"21.3", {{"async_socket_for_remote", true, false, "Turn off asynchronous reads from socket for remote queries because of some problems"}, + {"optimize_normalize_count_variants", false, true, "Rewrite aggregate functions that semantically equals to count() as count() by default"}, + {"normalize_function_names", false, true, "Normalize function names to their canonical names, this was needed for projection query routing"}}}, + {"21.2", {{"enable_global_with_statement", false, true, "Propagate WITH statements to UNION queries and all subqueries by default"}}}, + {"21.1", {{"insert_quorum_parallel", false, true, "Use parallel quorum inserts by default. It is significantly more convenient to use than sequential quorum inserts"}, + {"input_format_null_as_default", false, true, "Allow to insert NULL as default for input formats by default"}, + {"optimize_on_insert", false, true, "Enable data optimization on INSERT by default for better user experience"}, + {"use_compact_format_in_distributed_parts_names", false, true, "Use compact format for async INSERT into Distributed tables by default"}}}, + {"20.10", {{"format_regexp_escaping_rule", "Escaped", "Raw", "Use Raw as default escaping rule for Regexp format to male the behaviour more like to what users expect"}}}, + {"20.7", {{"show_table_uuid_in_table_create_query_if_not_nil", true, false, "Stop showing UID of the table in its CREATE query for Engine=Atomic"}}}, + {"20.5", {{"input_format_with_names_use_header", false, true, "Enable using header with names for formats with WithNames/WithNamesAndTypes suffixes"}, + {"allow_suspicious_codecs", true, false, "Don't allow to specify meaningless compression codecs"}}}, + {"20.4", {{"validate_polygons", false, true, "Throw exception if polygon is invalid in function pointInPolygon by default instead of returning possibly wrong results"}}}, + {"19.18", {{"enable_scalar_subquery_optimization", false, true, "Prevent scalar subqueries from (de)serializing large scalar values and possibly avoid running the same subquery more than once"}}}, + {"19.14", {{"any_join_distinct_right_table_keys", true, false, "Disable ANY RIGHT and ANY FULL JOINs by default to avoid inconsistency"}}}, + {"19.12", {{"input_format_defaults_for_omitted_fields", false, true, "Enable calculation of complex default expressions for omitted fields for some input formats, because it should be the expected behaviour"}}}, + {"19.5", {{"max_partitions_per_insert_block", 0, 100, "Add a limit for the number of partitions in one block"}}}, + {"18.12.17", {{"enable_optimize_predicate_expression", 0, 1, "Optimize predicates to subqueries by default"}}}, +}; + + +const std::map & getSettingsChangesHistory() +{ + static std::map settings_changes_history; + + static std::once_flag initialized_flag; + std::call_once(initialized_flag, []() + { + for (const auto & setting_change : settings_changes_history_initializer) + { + /// Disallow duplicate keys in the settings changes history. Example: + /// {"21.2", {{"some_setting_1", false, true, "[...]"}}}, + /// [...] + /// {"21.2", {{"some_setting_2", false, true, "[...]"}}}, + /// As std::set has unique keys, one of the entries would be overwritten. + if (settings_changes_history.contains(setting_change.first)) + throw Exception{ErrorCodes::LOGICAL_ERROR, "Detected duplicate version '{}'", setting_change.first.toString()}; + + settings_changes_history[setting_change.first] = setting_change.second; + } + }); + + return settings_changes_history; +} +} diff --git a/src/Core/SettingsChangesHistory.h b/src/Core/SettingsChangesHistory.h index 1311052b5e5..b1a69c3b6d6 100644 --- a/src/Core/SettingsChangesHistory.h +++ b/src/Core/SettingsChangesHistory.h @@ -1,11 +1,8 @@ #pragma once #include -#include -#include -#include -#include #include +#include namespace DB @@ -14,44 +11,15 @@ namespace DB class ClickHouseVersion { public: - ClickHouseVersion(const String & version) /// NOLINT(google-explicit-constructor) - { - Strings split; - boost::split(split, version, [](char c){ return c == '.'; }); - components.reserve(split.size()); - if (split.empty()) - throw Exception{ErrorCodes::BAD_ARGUMENTS, "Cannot parse ClickHouse version here: {}", version}; + /// NOLINTBEGIN(google-explicit-constructor) + ClickHouseVersion(const String & version); + ClickHouseVersion(const char * version); + /// NOLINTEND(google-explicit-constructor) - for (const auto & split_element : split) - { - size_t component; - ReadBufferFromString buf(split_element); - if (!tryReadIntText(component, buf) || !buf.eof()) - throw Exception{ErrorCodes::BAD_ARGUMENTS, "Cannot parse ClickHouse version here: {}", version}; - components.push_back(component); - } - } + String toString() const; - ClickHouseVersion(const char * version) : ClickHouseVersion(String(version)) {} /// NOLINT(google-explicit-constructor) - - String toString() const - { - String version = std::to_string(components[0]); - for (size_t i = 1; i < components.size(); ++i) - version += "." + std::to_string(components[i]); - - return version; - } - - bool operator<(const ClickHouseVersion & other) const - { - return components < other.components; - } - - bool operator>=(const ClickHouseVersion & other) const - { - return components >= other.components; - } + bool operator<(const ClickHouseVersion & other) const { return components < other.components; } + bool operator>=(const ClickHouseVersion & other) const { return components >= other.components; } private: std::vector components; @@ -70,277 +38,6 @@ namespace SettingsChangesHistory using SettingsChanges = std::vector; } -// clang-format off -/// History of settings changes that controls some backward incompatible changes -/// across all ClickHouse versions. It maps ClickHouse version to settings changes that were done -/// in this version. This history contains both changes to existing settings and newly added settings. -/// Settings changes is a vector of structs -/// {setting_name, previous_value, new_value, reason}. -/// For newly added setting choose the most appropriate previous_value (for example, if new setting -/// controls new feature and it's 'true' by default, use 'false' as previous_value). -/// It's used to implement `compatibility` setting (see https://github.com/ClickHouse/ClickHouse/issues/35972) -/// Note: please check if the key already exists to prevent duplicate entries. -static std::initializer_list> settings_changes_history_initializer = -{ - {"24.6", {{"materialize_skip_indexes_on_insert", true, true, "Added new setting to allow to disable materialization of skip indexes on insert"}, - {"materialize_statistics_on_insert", true, true, "Added new setting to allow to disable materialization of statistics on insert"}, - {"input_format_parquet_use_native_reader", false, false, "When reading Parquet files, to use native reader instead of arrow reader."}, - {"hdfs_throw_on_zero_files_match", false, false, "Allow to throw an error when ListObjects request cannot match any files in HDFS engine instead of empty query result"}, - {"azure_throw_on_zero_files_match", false, false, "Allow to throw an error when ListObjects request cannot match any files in AzureBlobStorage engine instead of empty query result"}, - {"s3_validate_request_settings", true, true, "Allow to disable S3 request settings validation"}, - {"allow_experimental_full_text_index", false, false, "Enable experimental full-text index"}, - {"azure_skip_empty_files", false, false, "Allow to skip empty files in azure table engine"}, - {"hdfs_ignore_file_doesnt_exist", false, false, "Allow to return 0 rows when the requested files don't exist instead of throwing an exception in HDFS table engine"}, - {"azure_ignore_file_doesnt_exist", false, false, "Allow to return 0 rows when the requested files don't exist instead of throwing an exception in AzureBlobStorage table engine"}, - {"s3_ignore_file_doesnt_exist", false, false, "Allow to return 0 rows when the requested files don't exist instead of throwing an exception in S3 table engine"}, - {"s3_max_part_number", 10000, 10000, "Maximum part number number for s3 upload part"}, - {"s3_max_single_operation_copy_size", 32 * 1024 * 1024, 32 * 1024 * 1024, "Maximum size for a single copy operation in s3"}, - {"input_format_parquet_max_block_size", 8192, DEFAULT_BLOCK_SIZE, "Increase block size for parquet reader."}, - {"input_format_parquet_prefer_block_bytes", 0, DEFAULT_BLOCK_SIZE * 256, "Average block bytes output by parquet reader."}, - {"enable_blob_storage_log", true, true, "Write information about blob storage operations to system.blob_storage_log table"}, - {"allow_deprecated_snowflake_conversion_functions", true, false, "Disabled deprecated functions snowflakeToDateTime[64] and dateTime[64]ToSnowflake."}, - {"allow_statistic_optimize", false, false, "Old setting which popped up here being renamed."}, - {"allow_experimental_statistic", false, false, "Old setting which popped up here being renamed."}, - {"allow_statistics_optimize", false, false, "The setting was renamed. The previous name is `allow_statistic_optimize`."}, - {"allow_experimental_statistics", false, false, "The setting was renamed. The previous name is `allow_experimental_statistic`."}, - {"enable_vertical_final", false, true, "Enable vertical final by default again after fixing bug"}, - {"parallel_replicas_custom_key_range_lower", 0, 0, "Add settings to control the range filter when using parallel replicas with dynamic shards"}, - {"parallel_replicas_custom_key_range_upper", 0, 0, "Add settings to control the range filter when using parallel replicas with dynamic shards. A value of 0 disables the upper limit"}, - {"output_format_pretty_display_footer_column_names", 0, 1, "Add a setting to display column names in the footer if there are many rows. Threshold value is controlled by output_format_pretty_display_footer_column_names_min_rows."}, - {"output_format_pretty_display_footer_column_names_min_rows", 0, 50, "Add a setting to control the threshold value for setting output_format_pretty_display_footer_column_names_min_rows. Default 50."}, - {"output_format_csv_serialize_tuple_into_separate_columns", true, true, "A new way of how interpret tuples in CSV format was added."}, - {"input_format_csv_deserialize_separate_columns_into_tuple", true, true, "A new way of how interpret tuples in CSV format was added."}, - {"input_format_csv_try_infer_strings_from_quoted_tuples", true, true, "A new way of how interpret tuples in CSV format was added."}, - }}, - {"24.5", {{"allow_deprecated_error_prone_window_functions", true, false, "Allow usage of deprecated error prone window functions (neighbor, runningAccumulate, runningDifferenceStartingWithFirstValue, runningDifference)"}, - {"allow_experimental_join_condition", false, false, "Support join with inequal conditions which involve columns from both left and right table. e.g. t1.y < t2.y."}, - {"input_format_tsv_crlf_end_of_line", false, false, "Enables reading of CRLF line endings with TSV formats"}, - {"output_format_parquet_use_custom_encoder", false, true, "Enable custom Parquet encoder."}, - {"cross_join_min_rows_to_compress", 0, 10000000, "Minimal count of rows to compress block in CROSS JOIN. Zero value means - disable this threshold. This block is compressed when any of the two thresholds (by rows or by bytes) are reached."}, - {"cross_join_min_bytes_to_compress", 0, 1_GiB, "Minimal size of block to compress in CROSS JOIN. Zero value means - disable this threshold. This block is compressed when any of the two thresholds (by rows or by bytes) are reached."}, - {"http_max_chunk_size", 0, 0, "Internal limitation"}, - {"prefer_external_sort_block_bytes", 0, DEFAULT_BLOCK_SIZE * 256, "Prefer maximum block bytes for external sort, reduce the memory usage during merging."}, - {"input_format_force_null_for_omitted_fields", false, false, "Disable type-defaults for omitted fields when needed"}, - {"cast_string_to_dynamic_use_inference", false, false, "Add setting to allow converting String to Dynamic through parsing"}, - {"allow_experimental_dynamic_type", false, false, "Add new experimental Dynamic type"}, - {"azure_max_blocks_in_multipart_upload", 50000, 50000, "Maximum number of blocks in multipart upload for Azure."}, - }}, - {"24.4", {{"input_format_json_throw_on_bad_escape_sequence", true, true, "Allow to save JSON strings with bad escape sequences"}, - {"max_parsing_threads", 0, 0, "Add a separate setting to control number of threads in parallel parsing from files"}, - {"ignore_drop_queries_probability", 0, 0, "Allow to ignore drop queries in server with specified probability for testing purposes"}, - {"lightweight_deletes_sync", 2, 2, "The same as 'mutation_sync', but controls only execution of lightweight deletes"}, - {"query_cache_system_table_handling", "save", "throw", "The query cache no longer caches results of queries against system tables"}, - {"input_format_json_ignore_unnecessary_fields", false, true, "Ignore unnecessary fields and not parse them. Enabling this may not throw exceptions on json strings of invalid format or with duplicated fields"}, - {"input_format_hive_text_allow_variable_number_of_columns", false, true, "Ignore extra columns in Hive Text input (if file has more columns than expected) and treat missing fields in Hive Text input as default values."}, - {"allow_experimental_database_replicated", false, true, "Database engine Replicated is now in Beta stage"}, - {"temporary_data_in_cache_reserve_space_wait_lock_timeout_milliseconds", (10 * 60 * 1000), (10 * 60 * 1000), "Wait time to lock cache for sapce reservation in temporary data in filesystem cache"}, - {"optimize_rewrite_sum_if_to_count_if", false, true, "Only available for the analyzer, where it works correctly"}, - {"azure_allow_parallel_part_upload", "true", "true", "Use multiple threads for azure multipart upload."}, - {"max_recursive_cte_evaluation_depth", DBMS_RECURSIVE_CTE_MAX_EVALUATION_DEPTH, DBMS_RECURSIVE_CTE_MAX_EVALUATION_DEPTH, "Maximum limit on recursive CTE evaluation depth"}, - {"query_plan_convert_outer_join_to_inner_join", false, true, "Allow to convert OUTER JOIN to INNER JOIN if filter after JOIN always filters default values"}, - }}, - {"24.3", {{"s3_connect_timeout_ms", 1000, 1000, "Introduce new dedicated setting for s3 connection timeout"}, - {"allow_experimental_shared_merge_tree", false, true, "The setting is obsolete"}, - {"use_page_cache_for_disks_without_file_cache", false, false, "Added userspace page cache"}, - {"read_from_page_cache_if_exists_otherwise_bypass_cache", false, false, "Added userspace page cache"}, - {"page_cache_inject_eviction", false, false, "Added userspace page cache"}, - {"default_table_engine", "None", "MergeTree", "Set default table engine to MergeTree for better usability"}, - {"input_format_json_use_string_type_for_ambiguous_paths_in_named_tuples_inference_from_objects", false, false, "Allow to use String type for ambiguous paths during named tuple inference from JSON objects"}, - {"traverse_shadow_remote_data_paths", false, false, "Traverse shadow directory when query system.remote_data_paths."}, - {"throw_if_deduplication_in_dependent_materialized_views_enabled_with_async_insert", false, true, "Deduplication is dependent materialized view cannot work together with async inserts."}, - {"parallel_replicas_allow_in_with_subquery", false, true, "If true, subquery for IN will be executed on every follower replica"}, - {"log_processors_profiles", false, true, "Enable by default"}, - {"function_locate_has_mysql_compatible_argument_order", false, true, "Increase compatibility with MySQL's locate function."}, - {"allow_suspicious_primary_key", true, false, "Forbid suspicious PRIMARY KEY/ORDER BY for MergeTree (i.e. SimpleAggregateFunction)"}, - {"filesystem_cache_reserve_space_wait_lock_timeout_milliseconds", 1000, 1000, "Wait time to lock cache for sapce reservation in filesystem cache"}, - {"max_parser_backtracks", 0, 1000000, "Limiting the complexity of parsing"}, - {"analyzer_compatibility_join_using_top_level_identifier", false, false, "Force to resolve identifier in JOIN USING from projection"}, - {"distributed_insert_skip_read_only_replicas", false, false, "If true, INSERT into Distributed will skip read-only replicas"}, - {"keeper_max_retries", 10, 10, "Max retries for general keeper operations"}, - {"keeper_retry_initial_backoff_ms", 100, 100, "Initial backoff timeout for general keeper operations"}, - {"keeper_retry_max_backoff_ms", 5000, 5000, "Max backoff timeout for general keeper operations"}, - {"s3queue_allow_experimental_sharded_mode", false, false, "Enable experimental sharded mode of S3Queue table engine. It is experimental because it will be rewritten"}, - {"allow_experimental_analyzer", false, true, "Enable analyzer and planner by default."}, - {"merge_tree_read_split_ranges_into_intersecting_and_non_intersecting_injection_probability", 0.0, 0.0, "For testing of `PartsSplitter` - split read ranges into intersecting and non intersecting every time you read from MergeTree with the specified probability."}, - {"allow_get_client_http_header", false, false, "Introduced a new function."}, - {"output_format_pretty_row_numbers", false, true, "It is better for usability."}, - {"output_format_pretty_max_value_width_apply_for_single_value", true, false, "Single values in Pretty formats won't be cut."}, - {"output_format_parquet_string_as_string", false, true, "ClickHouse allows arbitrary binary data in the String data type, which is typically UTF-8. Parquet/ORC/Arrow Strings only support UTF-8. That's why you can choose which Arrow's data type to use for the ClickHouse String data type - String or Binary. While Binary would be more correct and compatible, using String by default will correspond to user expectations in most cases."}, - {"output_format_orc_string_as_string", false, true, "ClickHouse allows arbitrary binary data in the String data type, which is typically UTF-8. Parquet/ORC/Arrow Strings only support UTF-8. That's why you can choose which Arrow's data type to use for the ClickHouse String data type - String or Binary. While Binary would be more correct and compatible, using String by default will correspond to user expectations in most cases."}, - {"output_format_arrow_string_as_string", false, true, "ClickHouse allows arbitrary binary data in the String data type, which is typically UTF-8. Parquet/ORC/Arrow Strings only support UTF-8. That's why you can choose which Arrow's data type to use for the ClickHouse String data type - String or Binary. While Binary would be more correct and compatible, using String by default will correspond to user expectations in most cases."}, - {"output_format_parquet_compression_method", "lz4", "zstd", "Parquet/ORC/Arrow support many compression methods, including lz4 and zstd. ClickHouse supports each and every compression method. Some inferior tools, such as 'duckdb', lack support for the faster `lz4` compression method, that's why we set zstd by default."}, - {"output_format_orc_compression_method", "lz4", "zstd", "Parquet/ORC/Arrow support many compression methods, including lz4 and zstd. ClickHouse supports each and every compression method. Some inferior tools, such as 'duckdb', lack support for the faster `lz4` compression method, that's why we set zstd by default."}, - {"output_format_pretty_highlight_digit_groups", false, true, "If enabled and if output is a terminal, highlight every digit corresponding to the number of thousands, millions, etc. with underline."}, - {"geo_distance_returns_float64_on_float64_arguments", false, true, "Increase the default precision."}, - {"azure_max_inflight_parts_for_one_file", 20, 20, "The maximum number of a concurrent loaded parts in multipart upload request. 0 means unlimited."}, - {"azure_strict_upload_part_size", 0, 0, "The exact size of part to upload during multipart upload to Azure blob storage."}, - {"azure_min_upload_part_size", 16*1024*1024, 16*1024*1024, "The minimum size of part to upload during multipart upload to Azure blob storage."}, - {"azure_max_upload_part_size", 5ull*1024*1024*1024, 5ull*1024*1024*1024, "The maximum size of part to upload during multipart upload to Azure blob storage."}, - {"azure_upload_part_size_multiply_factor", 2, 2, "Multiply azure_min_upload_part_size by this factor each time azure_multiply_parts_count_threshold parts were uploaded from a single write to Azure blob storage."}, - {"azure_upload_part_size_multiply_parts_count_threshold", 500, 500, "Each time this number of parts was uploaded to Azure blob storage, azure_min_upload_part_size is multiplied by azure_upload_part_size_multiply_factor."}, - {"output_format_csv_serialize_tuple_into_separate_columns", true, true, "A new way of how interpret tuples in CSV format was added."}, - {"input_format_csv_deserialize_separate_columns_into_tuple", true, true, "A new way of how interpret tuples in CSV format was added."}, - {"input_format_csv_try_infer_strings_from_quoted_tuples", true, true, "A new way of how interpret tuples in CSV format was added."}, - }}, - {"24.2", {{"allow_suspicious_variant_types", true, false, "Don't allow creating Variant type with suspicious variants by default"}, - {"validate_experimental_and_suspicious_types_inside_nested_types", false, true, "Validate usage of experimental and suspicious types inside nested types"}, - {"output_format_values_escape_quote_with_quote", false, false, "If true escape ' with '', otherwise quoted with \\'"}, - {"output_format_pretty_single_large_number_tip_threshold", 0, 1'000'000, "Print a readable number tip on the right side of the table if the block consists of a single number which exceeds this value (except 0)"}, - {"input_format_try_infer_exponent_floats", true, false, "Don't infer floats in exponential notation by default"}, - {"query_plan_optimize_prewhere", true, true, "Allow to push down filter to PREWHERE expression for supported storages"}, - {"async_insert_max_data_size", 1000000, 10485760, "The previous value appeared to be too small."}, - {"async_insert_poll_timeout_ms", 10, 10, "Timeout in milliseconds for polling data from asynchronous insert queue"}, - {"async_insert_use_adaptive_busy_timeout", false, true, "Use adaptive asynchronous insert timeout"}, - {"async_insert_busy_timeout_min_ms", 50, 50, "The minimum value of the asynchronous insert timeout in milliseconds; it also serves as the initial value, which may be increased later by the adaptive algorithm"}, - {"async_insert_busy_timeout_max_ms", 200, 200, "The minimum value of the asynchronous insert timeout in milliseconds; async_insert_busy_timeout_ms is aliased to async_insert_busy_timeout_max_ms"}, - {"async_insert_busy_timeout_increase_rate", 0.2, 0.2, "The exponential growth rate at which the adaptive asynchronous insert timeout increases"}, - {"async_insert_busy_timeout_decrease_rate", 0.2, 0.2, "The exponential growth rate at which the adaptive asynchronous insert timeout decreases"}, - {"format_template_row_format", "", "", "Template row format string can be set directly in query"}, - {"format_template_resultset_format", "", "", "Template result set format string can be set in query"}, - {"split_parts_ranges_into_intersecting_and_non_intersecting_final", true, true, "Allow to split parts ranges into intersecting and non intersecting during FINAL optimization"}, - {"split_intersecting_parts_ranges_into_layers_final", true, true, "Allow to split intersecting parts ranges into layers during FINAL optimization"}, - {"azure_max_single_part_copy_size", 256*1024*1024, 256*1024*1024, "The maximum size of object to copy using single part copy to Azure blob storage."}, - {"min_external_table_block_size_rows", DEFAULT_INSERT_BLOCK_SIZE, DEFAULT_INSERT_BLOCK_SIZE, "Squash blocks passed to external table to specified size in rows, if blocks are not big enough"}, - {"min_external_table_block_size_bytes", DEFAULT_INSERT_BLOCK_SIZE * 256, DEFAULT_INSERT_BLOCK_SIZE * 256, "Squash blocks passed to external table to specified size in bytes, if blocks are not big enough."}, - {"parallel_replicas_prefer_local_join", true, true, "If true, and JOIN can be executed with parallel replicas algorithm, and all storages of right JOIN part are *MergeTree, local JOIN will be used instead of GLOBAL JOIN."}, - {"optimize_time_filter_with_preimage", true, true, "Optimize Date and DateTime predicates by converting functions into equivalent comparisons without conversions (e.g. toYear(col) = 2023 -> col >= '2023-01-01' AND col <= '2023-12-31')"}, - {"extract_key_value_pairs_max_pairs_per_row", 0, 0, "Max number of pairs that can be produced by the `extractKeyValuePairs` function. Used as a safeguard against consuming too much memory."}, - {"default_view_definer", "CURRENT_USER", "CURRENT_USER", "Allows to set default `DEFINER` option while creating a view"}, - {"default_materialized_view_sql_security", "DEFINER", "DEFINER", "Allows to set a default value for SQL SECURITY option when creating a materialized view"}, - {"default_normal_view_sql_security", "INVOKER", "INVOKER", "Allows to set default `SQL SECURITY` option while creating a normal view"}, - {"mysql_map_string_to_text_in_show_columns", false, true, "Reduce the configuration effort to connect ClickHouse with BI tools."}, - {"mysql_map_fixed_string_to_text_in_show_columns", false, true, "Reduce the configuration effort to connect ClickHouse with BI tools."}, - }}, - {"24.1", {{"print_pretty_type_names", false, true, "Better user experience."}, - {"input_format_json_read_bools_as_strings", false, true, "Allow to read bools as strings in JSON formats by default"}, - {"output_format_arrow_use_signed_indexes_for_dictionary", false, true, "Use signed indexes type for Arrow dictionaries by default as it's recommended"}, - {"allow_experimental_variant_type", false, false, "Add new experimental Variant type"}, - {"use_variant_as_common_type", false, false, "Allow to use Variant in if/multiIf if there is no common type"}, - {"output_format_arrow_use_64_bit_indexes_for_dictionary", false, false, "Allow to use 64 bit indexes type in Arrow dictionaries"}, - {"parallel_replicas_mark_segment_size", 128, 128, "Add new setting to control segment size in new parallel replicas coordinator implementation"}, - {"ignore_materialized_views_with_dropped_target_table", false, false, "Add new setting to allow to ignore materialized views with dropped target table"}, - {"output_format_compression_level", 3, 3, "Allow to change compression level in the query output"}, - {"output_format_compression_zstd_window_log", 0, 0, "Allow to change zstd window log in the query output when zstd compression is used"}, - {"enable_zstd_qat_codec", false, false, "Add new ZSTD_QAT codec"}, - {"enable_vertical_final", false, true, "Use vertical final by default"}, - {"output_format_arrow_use_64_bit_indexes_for_dictionary", false, false, "Allow to use 64 bit indexes type in Arrow dictionaries"}, - {"max_rows_in_set_to_optimize_join", 100000, 0, "Disable join optimization as it prevents from read in order optimization"}, - {"output_format_pretty_color", true, "auto", "Setting is changed to allow also for auto value, disabling ANSI escapes if output is not a tty"}, - {"function_visible_width_behavior", 0, 1, "We changed the default behavior of `visibleWidth` to be more precise"}, - {"max_estimated_execution_time", 0, 0, "Separate max_execution_time and max_estimated_execution_time"}, - {"iceberg_engine_ignore_schema_evolution", false, false, "Allow to ignore schema evolution in Iceberg table engine"}, - {"optimize_injective_functions_in_group_by", false, true, "Replace injective functions by it's arguments in GROUP BY section in analyzer"}, - {"update_insert_deduplication_token_in_dependent_materialized_views", false, false, "Allow to update insert deduplication token with table identifier during insert in dependent materialized views"}, - {"azure_max_unexpected_write_error_retries", 4, 4, "The maximum number of retries in case of unexpected errors during Azure blob storage write"}, - {"split_parts_ranges_into_intersecting_and_non_intersecting_final", false, true, "Allow to split parts ranges into intersecting and non intersecting during FINAL optimization"}, - {"split_intersecting_parts_ranges_into_layers_final", true, true, "Allow to split intersecting parts ranges into layers during FINAL optimization"}}}, - {"23.12", {{"allow_suspicious_ttl_expressions", true, false, "It is a new setting, and in previous versions the behavior was equivalent to allowing."}, - {"input_format_parquet_allow_missing_columns", false, true, "Allow missing columns in Parquet files by default"}, - {"input_format_orc_allow_missing_columns", false, true, "Allow missing columns in ORC files by default"}, - {"input_format_arrow_allow_missing_columns", false, true, "Allow missing columns in Arrow files by default"}}}, - {"23.11", {{"parsedatetime_parse_without_leading_zeros", false, true, "Improved compatibility with MySQL DATE_FORMAT/STR_TO_DATE"}}}, - {"23.9", {{"optimize_group_by_constant_keys", false, true, "Optimize group by constant keys by default"}, - {"input_format_json_try_infer_named_tuples_from_objects", false, true, "Try to infer named Tuples from JSON objects by default"}, - {"input_format_json_read_numbers_as_strings", false, true, "Allow to read numbers as strings in JSON formats by default"}, - {"input_format_json_read_arrays_as_strings", false, true, "Allow to read arrays as strings in JSON formats by default"}, - {"input_format_json_infer_incomplete_types_as_strings", false, true, "Allow to infer incomplete types as Strings in JSON formats by default"}, - {"input_format_json_try_infer_numbers_from_strings", true, false, "Don't infer numbers from strings in JSON formats by default to prevent possible parsing errors"}, - {"http_write_exception_in_output_format", false, true, "Output valid JSON/XML on exception in HTTP streaming."}}}, - {"23.8", {{"rewrite_count_distinct_if_with_count_distinct_implementation", false, true, "Rewrite countDistinctIf with count_distinct_implementation configuration"}}}, - {"23.7", {{"function_sleep_max_microseconds_per_block", 0, 3000000, "In previous versions, the maximum sleep time of 3 seconds was applied only for `sleep`, but not for `sleepEachRow` function. In the new version, we introduce this setting. If you set compatibility with the previous versions, we will disable the limit altogether."}}}, - {"23.6", {{"http_send_timeout", 180, 30, "3 minutes seems crazy long. Note that this is timeout for a single network write call, not for the whole upload operation."}, - {"http_receive_timeout", 180, 30, "See http_send_timeout."}}}, - {"23.5", {{"input_format_parquet_preserve_order", true, false, "Allow Parquet reader to reorder rows for better parallelism."}, - {"parallelize_output_from_storages", false, true, "Allow parallelism when executing queries that read from file/url/s3/etc. This may reorder rows."}, - {"use_with_fill_by_sorting_prefix", false, true, "Columns preceding WITH FILL columns in ORDER BY clause form sorting prefix. Rows with different values in sorting prefix are filled independently"}, - {"output_format_parquet_compliant_nested_types", false, true, "Change an internal field name in output Parquet file schema."}}}, - {"23.4", {{"allow_suspicious_indices", true, false, "If true, index can defined with identical expressions"}, - {"allow_nonconst_timezone_arguments", true, false, "Allow non-const timezone arguments in certain time-related functions like toTimeZone(), fromUnixTimestamp*(), snowflakeToDateTime*()."}, - {"connect_timeout_with_failover_ms", 50, 1000, "Increase default connect timeout because of async connect"}, - {"connect_timeout_with_failover_secure_ms", 100, 1000, "Increase default secure connect timeout because of async connect"}, - {"hedged_connection_timeout_ms", 100, 50, "Start new connection in hedged requests after 50 ms instead of 100 to correspond with previous connect timeout"}, - {"formatdatetime_f_prints_single_zero", true, false, "Improved compatibility with MySQL DATE_FORMAT()/STR_TO_DATE()"}, - {"formatdatetime_parsedatetime_m_is_month_name", false, true, "Improved compatibility with MySQL DATE_FORMAT/STR_TO_DATE"}}}, - {"23.3", {{"output_format_parquet_version", "1.0", "2.latest", "Use latest Parquet format version for output format"}, - {"input_format_json_ignore_unknown_keys_in_named_tuple", false, true, "Improve parsing JSON objects as named tuples"}, - {"input_format_native_allow_types_conversion", false, true, "Allow types conversion in Native input forma"}, - {"output_format_arrow_compression_method", "none", "lz4_frame", "Use lz4 compression in Arrow output format by default"}, - {"output_format_parquet_compression_method", "snappy", "lz4", "Use lz4 compression in Parquet output format by default"}, - {"output_format_orc_compression_method", "none", "lz4_frame", "Use lz4 compression in ORC output format by default"}, - {"async_query_sending_for_remote", false, true, "Create connections and send query async across shards"}}}, - {"23.2", {{"output_format_parquet_fixed_string_as_fixed_byte_array", false, true, "Use Parquet FIXED_LENGTH_BYTE_ARRAY type for FixedString by default"}, - {"output_format_arrow_fixed_string_as_fixed_byte_array", false, true, "Use Arrow FIXED_SIZE_BINARY type for FixedString by default"}, - {"query_plan_remove_redundant_distinct", false, true, "Remove redundant Distinct step in query plan"}, - {"optimize_duplicate_order_by_and_distinct", true, false, "Remove duplicate ORDER BY and DISTINCT if it's possible"}, - {"insert_keeper_max_retries", 0, 20, "Enable reconnections to Keeper on INSERT, improve reliability"}}}, - {"23.1", {{"input_format_json_read_objects_as_strings", 0, 1, "Enable reading nested json objects as strings while object type is experimental"}, - {"input_format_json_defaults_for_missing_elements_in_named_tuple", false, true, "Allow missing elements in JSON objects while reading named tuples by default"}, - {"input_format_csv_detect_header", false, true, "Detect header in CSV format by default"}, - {"input_format_tsv_detect_header", false, true, "Detect header in TSV format by default"}, - {"input_format_custom_detect_header", false, true, "Detect header in CustomSeparated format by default"}, - {"query_plan_remove_redundant_sorting", false, true, "Remove redundant sorting in query plan. For example, sorting steps related to ORDER BY clauses in subqueries"}}}, - {"22.12", {{"max_size_to_preallocate_for_aggregation", 10'000'000, 100'000'000, "This optimizes performance"}, - {"query_plan_aggregation_in_order", 0, 1, "Enable some refactoring around query plan"}, - {"format_binary_max_string_size", 0, 1_GiB, "Prevent allocating large amount of memory"}}}, - {"22.11", {{"use_structure_from_insertion_table_in_table_functions", 0, 2, "Improve using structure from insertion table in table functions"}}}, - {"22.9", {{"force_grouping_standard_compatibility", false, true, "Make GROUPING function output the same as in SQL standard and other DBMS"}}}, - {"22.7", {{"cross_to_inner_join_rewrite", 1, 2, "Force rewrite comma join to inner"}, - {"enable_positional_arguments", false, true, "Enable positional arguments feature by default"}, - {"format_csv_allow_single_quotes", true, false, "Most tools don't treat single quote in CSV specially, don't do it by default too"}}}, - {"22.6", {{"output_format_json_named_tuples_as_objects", false, true, "Allow to serialize named tuples as JSON objects in JSON formats by default"}, - {"input_format_skip_unknown_fields", false, true, "Optimize reading subset of columns for some input formats"}}}, - {"22.5", {{"memory_overcommit_ratio_denominator", 0, 1073741824, "Enable memory overcommit feature by default"}, - {"memory_overcommit_ratio_denominator_for_user", 0, 1073741824, "Enable memory overcommit feature by default"}}}, - {"22.4", {{"allow_settings_after_format_in_insert", true, false, "Do not allow SETTINGS after FORMAT for INSERT queries because ClickHouse interpret SETTINGS as some values, which is misleading"}}}, - {"22.3", {{"cast_ipv4_ipv6_default_on_conversion_error", true, false, "Make functions cast(value, 'IPv4') and cast(value, 'IPv6') behave same as toIPv4 and toIPv6 functions"}}}, - {"21.12", {{"stream_like_engine_allow_direct_select", true, false, "Do not allow direct select for Kafka/RabbitMQ/FileLog by default"}}}, - {"21.9", {{"output_format_decimal_trailing_zeros", true, false, "Do not output trailing zeros in text representation of Decimal types by default for better looking output"}, - {"use_hedged_requests", false, true, "Enable Hedged Requests feature by default"}}}, - {"21.7", {{"legacy_column_name_of_tuple_literal", true, false, "Add this setting only for compatibility reasons. It makes sense to set to 'true', while doing rolling update of cluster from version lower than 21.7 to higher"}}}, - {"21.5", {{"async_socket_for_remote", false, true, "Fix all problems and turn on asynchronous reads from socket for remote queries by default again"}}}, - {"21.3", {{"async_socket_for_remote", true, false, "Turn off asynchronous reads from socket for remote queries because of some problems"}, - {"optimize_normalize_count_variants", false, true, "Rewrite aggregate functions that semantically equals to count() as count() by default"}, - {"normalize_function_names", false, true, "Normalize function names to their canonical names, this was needed for projection query routing"}}}, - {"21.2", {{"enable_global_with_statement", false, true, "Propagate WITH statements to UNION queries and all subqueries by default"}}}, - {"21.1", {{"insert_quorum_parallel", false, true, "Use parallel quorum inserts by default. It is significantly more convenient to use than sequential quorum inserts"}, - {"input_format_null_as_default", false, true, "Allow to insert NULL as default for input formats by default"}, - {"optimize_on_insert", false, true, "Enable data optimization on INSERT by default for better user experience"}, - {"use_compact_format_in_distributed_parts_names", false, true, "Use compact format for async INSERT into Distributed tables by default"}}}, - {"20.10", {{"format_regexp_escaping_rule", "Escaped", "Raw", "Use Raw as default escaping rule for Regexp format to male the behaviour more like to what users expect"}}}, - {"20.7", {{"show_table_uuid_in_table_create_query_if_not_nil", true, false, "Stop showing UID of the table in its CREATE query for Engine=Atomic"}}}, - {"20.5", {{"input_format_with_names_use_header", false, true, "Enable using header with names for formats with WithNames/WithNamesAndTypes suffixes"}, - {"allow_suspicious_codecs", true, false, "Don't allow to specify meaningless compression codecs"}}}, - {"20.4", {{"validate_polygons", false, true, "Throw exception if polygon is invalid in function pointInPolygon by default instead of returning possibly wrong results"}}}, - {"19.18", {{"enable_scalar_subquery_optimization", false, true, "Prevent scalar subqueries from (de)serializing large scalar values and possibly avoid running the same subquery more than once"}}}, - {"19.14", {{"any_join_distinct_right_table_keys", true, false, "Disable ANY RIGHT and ANY FULL JOINs by default to avoid inconsistency"}}}, - {"19.12", {{"input_format_defaults_for_omitted_fields", false, true, "Enable calculation of complex default expressions for omitted fields for some input formats, because it should be the expected behaviour"}}}, - {"19.5", {{"max_partitions_per_insert_block", 0, 100, "Add a limit for the number of partitions in one block"}}}, - {"18.12.17", {{"enable_optimize_predicate_expression", 0, 1, "Optimize predicates to subqueries by default"}}}, -}; - -static -const std::map & getSettingsChangesHistory() -{ - static std::map settings_changes_history; - - static std::once_flag initialized_flag; - std::call_once(initialized_flag, []() - { - for (const auto & setting_change : settings_changes_history_initializer) - { - /// Disallow duplicate keys in the settings changes history. Example: - /// {"21.2", {{"some_setting_1", false, true, "[...]"}}}, - /// [...] - /// {"21.2", {{"some_setting_2", false, true, "[...]"}}}, - /// As std::set has unique keys, one of the entries would be overwritten. - if (settings_changes_history.contains(setting_change.first)) - throw Exception{ErrorCodes::LOGICAL_ERROR, "Detected duplicate version '{}'", setting_change.first.toString()}; - - settings_changes_history[setting_change.first] = setting_change.second; - } - }); - - return settings_changes_history; -} +const std::map & getSettingsChangesHistory(); } From d7dae3328fbdfaa5ca99de76c9d03f6a4c9a289e Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Sun, 30 Jun 2024 18:03:51 +0200 Subject: [PATCH 113/115] Update the list of easy tasks --- tests/instructions/easy_tasks_sorted_ru.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/instructions/easy_tasks_sorted_ru.md b/tests/instructions/easy_tasks_sorted_ru.md index bc95e6b1c37..fbd86ebf08f 100644 --- a/tests/instructions/easy_tasks_sorted_ru.md +++ b/tests/instructions/easy_tasks_sorted_ru.md @@ -78,7 +78,7 @@ Upd: сделали по-другому: теперь всё безопасно. ## LEFT ONLY JOIN -## Функции makeDate, makeDateTime. +## + Функции makeDate, makeDateTime. `makeDate(year, month, day)` `makeDateTime(year, month, day, hour, minute, second, [timezone])` @@ -187,13 +187,13 @@ https://clickhouse.com/docs/en/operations/table_engines/external_data/ Не работает, если открыть clickhouse-client в интерактивном режиме и делать несколько запросов. -## + Настройка для возможности получить частичный результат при cancel-е. +## Настройка для возможности получить частичный результат при cancel-е. Хотим по Ctrl+C получить те данные, которые успели обработаться. ## Раскрытие кортежей в функциях высшего порядка. -## Табличная функция loop. +## + Табличная функция loop. `SELECT * FROM loop(database, table)` From 5a1125418b839a23db529f5068363b42922ecee2 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Sun, 30 Jun 2024 19:24:03 +0200 Subject: [PATCH 114/115] Update CHANGELOG.md --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index c4935f88245..4507b491493 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,7 +9,7 @@ # 2024 Changelog -### ClickHouse release 24.6, 2024-06-27 +### ClickHouse release 24.6, 2024-06-30 #### Backward Incompatible Change * Enable asynchronous load of databases and tables by default. See the `async_load_databases` in config.xml. While this change is fully compatible, it can introduce a difference in behavior. When `async_load_databases` is false, as in the previous versions, the server will not accept connections until all tables are loaded. When `async_load_databases` is true, as in the new version, the server can accept connections before all the tables are loaded. If a query is made to a table that is not yet loaded, it will wait for the table's loading, which can take considerable time. It can change the behavior of the server if it is part of a large distributed system under a load balancer. In the first case, the load balancer can get a connection refusal and quickly failover to another server. In the second case, the load balancer can connect to a server that is still loading the tables, and the query will have a higher latency. Moreover, if many queries accumulate in the waiting state, it can lead to a "thundering herd" problem when they start processing simultaneously. This can make a difference only for highly loaded distributed backends. You can set the value of `async_load_databases` to false to avoid this problem. [#57695](https://github.com/ClickHouse/ClickHouse/pull/57695) ([Alexey Milovidov](https://github.com/alexey-milovidov)). From 770e637f9b47b58f2846242368f32852f6d76092 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Mon, 1 Jul 2024 03:51:45 +0200 Subject: [PATCH 115/115] Add changelog for release 24.6 --- CHANGELOG.md | 150 ++++++++++++++++++++++++--------------------------- 1 file changed, 70 insertions(+), 80 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 4507b491493..19258f469c2 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,5 @@ ### Table of Contents -**[ClickHouse release v24.6, 2024-06-27](#246)**
+**[ClickHouse release v24.6, 2024-06-30](#246)**
**[ClickHouse release v24.5, 2024-05-30](#245)**
**[ClickHouse release v24.4, 2024-04-30](#244)**
**[ClickHouse release v24.3 LTS, 2024-03-26](#243)**
@@ -13,103 +13,101 @@ #### Backward Incompatible Change * Enable asynchronous load of databases and tables by default. See the `async_load_databases` in config.xml. While this change is fully compatible, it can introduce a difference in behavior. When `async_load_databases` is false, as in the previous versions, the server will not accept connections until all tables are loaded. When `async_load_databases` is true, as in the new version, the server can accept connections before all the tables are loaded. If a query is made to a table that is not yet loaded, it will wait for the table's loading, which can take considerable time. It can change the behavior of the server if it is part of a large distributed system under a load balancer. In the first case, the load balancer can get a connection refusal and quickly failover to another server. In the second case, the load balancer can connect to a server that is still loading the tables, and the query will have a higher latency. Moreover, if many queries accumulate in the waiting state, it can lead to a "thundering herd" problem when they start processing simultaneously. This can make a difference only for highly loaded distributed backends. You can set the value of `async_load_databases` to false to avoid this problem. [#57695](https://github.com/ClickHouse/ClickHouse/pull/57695) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Setting `replace_long_file_name_to_hash` is enabled by default for `MergeTree` tables. [#64457](https://github.com/ClickHouse/ClickHouse/pull/64457) ([Anton Popov](https://github.com/CurtizJ)). This setting is fully compatible, and no actions needed during upgrade. The new data format is supported from all versions starting from 23.9. After enabling this setting, you can no longer downgrade to a version 23.8 or older. * Some invalid queries will fail earlier during parsing. Note: disabled the support for inline KQL expressions (the experimental Kusto language) when they are put into a `kql` table function without a string literal, e.g. `kql(garbage | trash)` instead of `kql('garbage | trash')` or `kql($$garbage | trash$$)`. This feature was introduced unintentionally and should not exist. [#61500](https://github.com/ClickHouse/ClickHouse/pull/61500) ([Alexey Milovidov](https://github.com/alexey-milovidov)). * Rework parallel processing in `Ordered` mode of storage `S3Queue`. This PR is backward incompatible for Ordered mode if you used settings `s3queue_processing_threads_num` or `s3queue_total_shards_num`. Setting `s3queue_total_shards_num` is deleted, previously it was allowed to use only under `s3queue_allow_experimental_sharded_mode`, which is now deprecated. A new setting is added - `s3queue_buckets`. [#64349](https://github.com/ClickHouse/ClickHouse/pull/64349) ([Kseniia Sumarokova](https://github.com/kssenii)). * New functions `snowflakeIDToDateTime`, `snowflakeIDToDateTime64`, `dateTimeToSnowflakeID`, and `dateTime64ToSnowflakeID` were added. Unlike the existing functions `snowflakeToDateTime`, `snowflakeToDateTime64`, `dateTimeToSnowflake`, and `dateTime64ToSnowflake`, the new functions are compatible with function `generateSnowflakeID`, i.e. they accept the snowflake IDs generated by `generateSnowflakeID` and produce snowflake IDs of the same type as `generateSnowflakeID` (i.e. `UInt64`). Furthermore, the new functions default to the UNIX epoch (aka. 1970-01-01), just like `generateSnowflakeID`. If necessary, a different epoch, e.g. Twitter's/X's epoch 2010-11-04 aka. 1288834974657 msec since UNIX epoch, can be passed. The old conversion functions are deprecated and will be removed after a transition period: to use them regardless, enable setting `allow_deprecated_snowflake_conversion_functions`. [#64948](https://github.com/ClickHouse/ClickHouse/pull/64948) ([Robert Schulze](https://github.com/rschu1ze)). #### New Feature -* Introduce statistics of type "number of distinct values". [#59357](https://github.com/ClickHouse/ClickHouse/pull/59357) ([Han Fei](https://github.com/hanfei1991)). +* Allow to store named collections in ClickHouse Keeper. [#64574](https://github.com/ClickHouse/ClickHouse/pull/64574) ([Kseniia Sumarokova](https://github.com/kssenii)). +* Support empty tuples. [#55061](https://github.com/ClickHouse/ClickHouse/pull/55061) ([Amos Bird](https://github.com/amosbird)). * Add Hilbert Curve encode and decode functions. [#60156](https://github.com/ClickHouse/ClickHouse/pull/60156) ([Artem Mustafin](https://github.com/Artemmm91)). -* Added support for reading LINESTRING geometry in WKT format using function `readWKTLineString`. [#62519](https://github.com/ClickHouse/ClickHouse/pull/62519) ([Nikita Mikhaylov](https://github.com/nikitamikhaylov)). -* Allow to attach parts from a different disk. [#63087](https://github.com/ClickHouse/ClickHouse/pull/63087) ([Unalian](https://github.com/Unalian)). -* Allow proxy to be bypassed for hosts specified in `no_proxy` env variable and ClickHouse proxy configuration. [#63314](https://github.com/ClickHouse/ClickHouse/pull/63314) ([Arthur Passos](https://github.com/arthurpassos)). -* Added a new table function `loop` to support returning query results in an infinite loop. [#63452](https://github.com/ClickHouse/ClickHouse/pull/63452) ([Sariel](https://github.com/sarielwxm)). -* Added new SQL functions `generateSnowflakeID` for generating Twitter-style Snowflake IDs. [#63577](https://github.com/ClickHouse/ClickHouse/pull/63577) ([Danila Puzov](https://github.com/kazalika)). -* Add the ability to reshuffle rows during insert to optimize for size without violating the order set by `PRIMARY KEY`. It's controlled by the setting `optimize_row_order` (off by default). [#63578](https://github.com/ClickHouse/ClickHouse/pull/63578) ([Igor Markelov](https://github.com/ElderlyPassionFruit)). -* Added `merge_workload` and `mutation_workload` settings to regulate how resources are utilized and shared between merges, mutations and other workloads. [#64061](https://github.com/ClickHouse/ClickHouse/pull/64061) ([Sergei Trifonov](https://github.com/serxa)). -* Add support for comparing IPv4 and IPv6 types using the `=` operator. [#64292](https://github.com/ClickHouse/ClickHouse/pull/64292) ([Francisco J. Jurado Moreno](https://github.com/Beetelbrox)). -* Allow to store named collections in zookeeper. [#64574](https://github.com/ClickHouse/ClickHouse/pull/64574) ([Kseniia Sumarokova](https://github.com/kssenii)). -* Support decimal arguments in binary math functions (pow, atan2, max2, min2, hypot). [#64582](https://github.com/ClickHouse/ClickHouse/pull/64582) ([Mikhail Gorshkov](https://github.com/mgorshkov)). * Add support for index analysis over `hilbertEncode`. [#64662](https://github.com/ClickHouse/ClickHouse/pull/64662) ([Artem Mustafin](https://github.com/Artemmm91)). +* Added support for reading `LINESTRING` geometry in the WKT format using function `readWKTLineString`. [#62519](https://github.com/ClickHouse/ClickHouse/pull/62519) ([Nikita Mikhaylov](https://github.com/nikitamikhaylov)). +* Allow to attach parts from a different disk. [#63087](https://github.com/ClickHouse/ClickHouse/pull/63087) ([Unalian](https://github.com/Unalian)). +* Added new SQL functions `generateSnowflakeID` for generating Twitter-style Snowflake IDs. [#63577](https://github.com/ClickHouse/ClickHouse/pull/63577) ([Danila Puzov](https://github.com/kazalika)). +* Added `merge_workload` and `mutation_workload` settings to regulate how resources are utilized and shared between merges, mutations and other workloads. [#64061](https://github.com/ClickHouse/ClickHouse/pull/64061) ([Sergei Trifonov](https://github.com/serxa)). +* Add support for comparing `IPv4` and `IPv6` types using the `=` operator. [#64292](https://github.com/ClickHouse/ClickHouse/pull/64292) ([Francisco J. Jurado Moreno](https://github.com/Beetelbrox)). +* Support decimal arguments in binary math functions (pow, atan2, max2, min2, hypot). [#64582](https://github.com/ClickHouse/ClickHouse/pull/64582) ([Mikhail Gorshkov](https://github.com/mgorshkov)). * Added SQL functions `parseReadableSize` (along with `OrNull` and `OrZero` variants). [#64742](https://github.com/ClickHouse/ClickHouse/pull/64742) ([Francisco J. Jurado Moreno](https://github.com/Beetelbrox)). * Add server settings `max_table_num_to_throw` and `max_database_num_to_throw` to limit the number of databases or tables on `CREATE` queries. [#64781](https://github.com/ClickHouse/ClickHouse/pull/64781) ([Xu Jia](https://github.com/XuJia0210)). -* Add _time virtual column to file alike storages (s3/file/hdfs/url/azureBlobStorage). [#64947](https://github.com/ClickHouse/ClickHouse/pull/64947) ([Ilya Golshtein](https://github.com/ilejn)). +* Add `_time` virtual column to file alike storages (s3/file/hdfs/url/azureBlobStorage). [#64947](https://github.com/ClickHouse/ClickHouse/pull/64947) ([Ilya Golshtein](https://github.com/ilejn)). * Introduced new functions `base64URLEncode`, `base64URLDecode` and `tryBase64URLDecode`. [#64991](https://github.com/ClickHouse/ClickHouse/pull/64991) ([Mikhail Gorshkov](https://github.com/mgorshkov)). * Add new function `editDistanceUTF8`, which calculates the [edit distance](https://en.wikipedia.org/wiki/Edit_distance) between two UTF8 strings. [#65269](https://github.com/ClickHouse/ClickHouse/pull/65269) ([LiuNeng](https://github.com/liuneng1994)). +* Add `http_response_headers` setting to support custom response headers in custom HTTP handlers. [#63562](https://github.com/ClickHouse/ClickHouse/pull/63562) ([Grigorii](https://github.com/GSokol)). +* Added a new table function `loop` to support returning query results in an infinite loop. [#63452](https://github.com/ClickHouse/ClickHouse/pull/63452) ([Sariel](https://github.com/sarielwxm)). This is useful for testing. +* Introduced two additional columns in the `system.query_log`: `used_privileges` and `missing_privileges`. `used_privileges` is populated with the privileges that were checked during query execution, and `missing_privileges` contains required privileges that are missing. [#64597](https://github.com/ClickHouse/ClickHouse/pull/64597) ([Alexey Katsman](https://github.com/alexkats)). +* Added a setting `output_format_pretty_display_footer_column_names` which when enabled displays column names at the end of the table for long tables (50 rows by default), with the threshold value for minimum number of rows controlled by `output_format_pretty_display_footer_column_names_min_rows`. [#65144](https://github.com/ClickHouse/ClickHouse/pull/65144) ([Shaun Struwig](https://github.com/Blargian)). + +#### Experimental Feature +* Introduce statistics of type "number of distinct values". [#59357](https://github.com/ClickHouse/ClickHouse/pull/59357) ([Han Fei](https://github.com/hanfei1991)). +* Support statistics with ReplicatedMergeTree. [#64934](https://github.com/ClickHouse/ClickHouse/pull/64934) ([Han Fei](https://github.com/hanfei1991)). +* If "replica group" is configured for a `Replicated` database, automatically create a cluster that includes replicas from all groups. [#64312](https://github.com/ClickHouse/ClickHouse/pull/64312) ([Alexander Tokmakov](https://github.com/tavplubix)). +* Add settings `parallel_replicas_custom_key_range_lower` and `parallel_replicas_custom_key_range_upper` to control how parallel replicas with dynamic shards parallelizes queries when using a range filter. [#64604](https://github.com/ClickHouse/ClickHouse/pull/64604) ([josh-hildred](https://github.com/josh-hildred)). #### Performance Improvement +* Add the ability to reshuffle rows during insert to optimize for size without violating the order set by `PRIMARY KEY`. It's controlled by the setting `optimize_row_order` (off by default). [#63578](https://github.com/ClickHouse/ClickHouse/pull/63578) ([Igor Markelov](https://github.com/ElderlyPassionFruit)). * Add a native parquet reader, which can read parquet binary to ClickHouse Columns directly. It's controlled by the setting `input_format_parquet_use_native_reader` (disabled by default). [#60361](https://github.com/ClickHouse/ClickHouse/pull/60361) ([ZhiHong Zhang](https://github.com/copperybean)). -* Reduce the number of virtual function calls in ColumnNullable::size. [#60556](https://github.com/ClickHouse/ClickHouse/pull/60556) ([HappenLee](https://github.com/HappenLee)). -* Speedup `splitByRegexp` when the regular expression argument is a single-character. [#62696](https://github.com/ClickHouse/ClickHouse/pull/62696) ([Robert Schulze](https://github.com/rschu1ze)). -* Speed up FixedHashTable by keeping track of the min and max keys used. This allows to reduce the number of cells that need to be verified. [#62746](https://github.com/ClickHouse/ClickHouse/pull/62746) ([Jiebin Sun](https://github.com/jiebinn)). -* Optimize the resolution of in(LowCardinality, ConstantSet). [#64060](https://github.com/ClickHouse/ClickHouse/pull/64060) ([Zhiguo Zhou](https://github.com/ZhiguoZh)). -* Use a thread pool to initialize and destroy hash tables inside `ConcurrentHashJoin`. [#64241](https://github.com/ClickHouse/ClickHouse/pull/64241) ([Nikita Taranov](https://github.com/nickitat)). -* Optimized vertical merges in tables with sparse columns. [#64311](https://github.com/ClickHouse/ClickHouse/pull/64311) ([Anton Popov](https://github.com/CurtizJ)). -* Enabled prefetches of data from remote filesystem during vertical merges. It improves latency of vertical merges in tables with data stored on remote filesystem. [#64314](https://github.com/ClickHouse/ClickHouse/pull/64314) ([Anton Popov](https://github.com/CurtizJ)). -* Reduce redundant calls to `isDefault()` of `ColumnSparse::filter` to improve performance. [#64426](https://github.com/ClickHouse/ClickHouse/pull/64426) ([Jiebin Sun](https://github.com/jiebinn)). -* Speedup `find_super_nodes` and `find_big_family` keeper-client commands by making multiple asynchronous getChildren requests. [#64628](https://github.com/ClickHouse/ClickHouse/pull/64628) ([Alexander Gololobov](https://github.com/davenger)). -* Improve function least/greatest for nullable numberic type arguments. [#64668](https://github.com/ClickHouse/ClickHouse/pull/64668) ([KevinyhZou](https://github.com/KevinyhZou)). -* Allow merging two consequent `FilterSteps` of a query plan. This improves filter-push-down optimization if the filter condition can be pushed down from the parent step. [#64760](https://github.com/ClickHouse/ClickHouse/pull/64760) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). -* Remove bad optimization in vertical final implementation and re-enable vertical final algorithm by default. [#64783](https://github.com/ClickHouse/ClickHouse/pull/64783) ([Duc Canh Le](https://github.com/canhld94)). -* Remove ALIAS nodes from the filter expression. This slightly improves performance for queries with `PREWHERE` (with the new analyzer). [#64793](https://github.com/ClickHouse/ClickHouse/pull/64793) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). -* Fix performance regression in cross join introduced in [#60459](https://github.com/ClickHouse/ClickHouse/issues/60459) (24.5). [#65243](https://github.com/ClickHouse/ClickHouse/pull/65243) ([Nikita Taranov](https://github.com/nickitat)). - -#### Improvement -* Support empty tuples. [#55061](https://github.com/ClickHouse/ClickHouse/pull/55061) ([Amos Bird](https://github.com/amosbird)). -* Hot reload storage policy for distributed tables when adding a new disk. [#58285](https://github.com/ClickHouse/ClickHouse/pull/58285) ([Duc Canh Le](https://github.com/canhld94)). -* Avoid possible deadlock during MergeTree index analysis when scheduling threads in a saturated service. [#59427](https://github.com/ClickHouse/ClickHouse/pull/59427) ([Sean Haynes](https://github.com/seandhaynes)). * Support partial trivial count optimization when the query filter is able to select exact ranges from merge tree tables. [#60463](https://github.com/ClickHouse/ClickHouse/pull/60463) ([Amos Bird](https://github.com/amosbird)). * Reduce max memory usage of multithreaded `INSERT`s by collecting chunks of multiple threads in a single transform. [#61047](https://github.com/ClickHouse/ClickHouse/pull/61047) ([Yarik Briukhovetskyi](https://github.com/yariks5s)). * Reduce the memory usage when using Azure object storage by using fixed memory allocation, avoiding the allocation of an extra buffer. [#63160](https://github.com/ClickHouse/ClickHouse/pull/63160) ([SmitaRKulkarni](https://github.com/SmitaRKulkarni)). -* Several minor corner case fixes to proxy support & tunneling. [#63427](https://github.com/ClickHouse/ClickHouse/pull/63427) ([Arthur Passos](https://github.com/arthurpassos)). -* Add `http_response_headers` setting to support custom response headers in custom HTTP handlers. [#63562](https://github.com/ClickHouse/ClickHouse/pull/63562) ([Grigorii](https://github.com/GSokol)). -* Improve io_uring resubmit visibility. Rename profile event `IOUringSQEsResubmits` -> `IOUringSQEsResubmitsAsync` and add a new one `IOUringSQEsResubmitsSync`. [#63699](https://github.com/ClickHouse/ClickHouse/pull/63699) ([Tomer Shafir](https://github.com/tomershafir)). -* Introduce assertions to verify all functions are called with columns of the right size. [#63723](https://github.com/ClickHouse/ClickHouse/pull/63723) ([Raúl Marín](https://github.com/Algunenano)). +* Reduce the number of virtual function calls in `ColumnNullable::size`. [#60556](https://github.com/ClickHouse/ClickHouse/pull/60556) ([HappenLee](https://github.com/HappenLee)). +* Speedup `splitByRegexp` when the regular expression argument is a single-character. [#62696](https://github.com/ClickHouse/ClickHouse/pull/62696) ([Robert Schulze](https://github.com/rschu1ze)). +* Speed up aggregation by 8-bit and 16-bit keys by keeping track of the min and max keys used. This allows to reduce the number of cells that need to be verified. [#62746](https://github.com/ClickHouse/ClickHouse/pull/62746) ([Jiebin Sun](https://github.com/jiebinn)). +* Optimize operator IN when the left hand side is `LowCardinality` and the right is a set of constants. [#64060](https://github.com/ClickHouse/ClickHouse/pull/64060) ([Zhiguo Zhou](https://github.com/ZhiguoZh)). +* Use a thread pool to initialize and destroy hash tables inside `ConcurrentHashJoin`. [#64241](https://github.com/ClickHouse/ClickHouse/pull/64241) ([Nikita Taranov](https://github.com/nickitat)). +* Optimized vertical merges in tables with sparse columns. [#64311](https://github.com/ClickHouse/ClickHouse/pull/64311) ([Anton Popov](https://github.com/CurtizJ)). +* Enabled prefetches of data from remote filesystem during vertical merges. It improves latency of vertical merges in tables with data stored on remote filesystem. [#64314](https://github.com/ClickHouse/ClickHouse/pull/64314) ([Anton Popov](https://github.com/CurtizJ)). +* Reduce redundant calls to `isDefault` of `ColumnSparse::filter` to improve performance. [#64426](https://github.com/ClickHouse/ClickHouse/pull/64426) ([Jiebin Sun](https://github.com/jiebinn)). +* Speedup `find_super_nodes` and `find_big_family` keeper-client commands by making multiple asynchronous getChildren requests. [#64628](https://github.com/ClickHouse/ClickHouse/pull/64628) ([Alexander Gololobov](https://github.com/davenger)). +* Improve function `least`/`greatest` for nullable numberic type arguments. [#64668](https://github.com/ClickHouse/ClickHouse/pull/64668) ([KevinyhZou](https://github.com/KevinyhZou)). +* Allow merging two consequent filtering steps of a query plan. This improves filter-push-down optimization if the filter condition can be pushed down from the parent step. [#64760](https://github.com/ClickHouse/ClickHouse/pull/64760) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* Remove bad optimization in the vertical final implementation and re-enable vertical final algorithm by default. [#64783](https://github.com/ClickHouse/ClickHouse/pull/64783) ([Duc Canh Le](https://github.com/canhld94)). +* Remove ALIAS nodes from the filter expression. This slightly improves performance for queries with `PREWHERE` (with the new analyzer). [#64793](https://github.com/ClickHouse/ClickHouse/pull/64793) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* Re-enable OpenSSL session caching. [#65111](https://github.com/ClickHouse/ClickHouse/pull/65111) ([Robert Schulze](https://github.com/rschu1ze)). +* Added settings to disable materialization of skip indexes and statistics on inserts (`materialize_skip_indexes_on_insert` and `materialize_statistics_on_insert`). [#64391](https://github.com/ClickHouse/ClickHouse/pull/64391) ([Anton Popov](https://github.com/CurtizJ)). +* Use the allocated memory size to calculate the row group size and reduce the peak memory of the parquet writer in the single-threaded mode. [#64424](https://github.com/ClickHouse/ClickHouse/pull/64424) ([LiuNeng](https://github.com/liuneng1994)). +* Improve the iterator of sparse column to reduce call of `size`. [#64497](https://github.com/ClickHouse/ClickHouse/pull/64497) ([Jiebin Sun](https://github.com/jiebinn)). +* Update condition to use server-side copy for backups to Azure blob storage. [#64518](https://github.com/ClickHouse/ClickHouse/pull/64518) ([SmitaRKulkarni](https://github.com/SmitaRKulkarni)). +* Optimized memory usage of vertical merges for tables with high number of skip indexes. [#64580](https://github.com/ClickHouse/ClickHouse/pull/64580) ([Anton Popov](https://github.com/CurtizJ)). + +#### Improvement * `SHOW CREATE TABLE` executed on top of system tables will now show the super handy comment unique for each table which will explain why this table is needed. [#63788](https://github.com/ClickHouse/ClickHouse/pull/63788) ([Nikita Mikhaylov](https://github.com/nikitamikhaylov)). -* Added setting `metadata_storage_type` to keep free space on metadata storage disk. [#64128](https://github.com/ClickHouse/ClickHouse/pull/64128) ([MikhailBurdukov](https://github.com/MikhailBurdukov)). -* Add metrics to track the number of directories created and removed by the plain_rewritable metadata storage, and the number of entries in the local-to-remote in-memory map. [#64175](https://github.com/ClickHouse/ClickHouse/pull/64175) ([Julia Kartseva](https://github.com/jkartseva)). +* The second argument (scale) of functions `round()`, `roundBankers()`, `floor()`, `ceil()` and `trunc()` can now be non-const. [#64798](https://github.com/ClickHouse/ClickHouse/pull/64798) ([Mikhail Gorshkov](https://github.com/mgorshkov)). +* Hot reload storage policy for `Distributed` tables when adding a new disk. [#58285](https://github.com/ClickHouse/ClickHouse/pull/58285) ([Duc Canh Le](https://github.com/canhld94)). +* Avoid possible deadlock during MergeTree index analysis when scheduling threads in a saturated service. [#59427](https://github.com/ClickHouse/ClickHouse/pull/59427) ([Sean Haynes](https://github.com/seandhaynes)). +* Several minor corner case fixes to S3 proxy support & tunneling. [#63427](https://github.com/ClickHouse/ClickHouse/pull/63427) ([Arthur Passos](https://github.com/arthurpassos)). +* Improve io_uring resubmit visibility. Rename profile event `IOUringSQEsResubmits` -> `IOUringSQEsResubmitsAsync` and add a new one `IOUringSQEsResubmitsSync`. [#63699](https://github.com/ClickHouse/ClickHouse/pull/63699) ([Tomer Shafir](https://github.com/tomershafir)). +* Added a new setting, `metadata_keep_free_space_bytes` to keep free space on the metadata storage disk. [#64128](https://github.com/ClickHouse/ClickHouse/pull/64128) ([MikhailBurdukov](https://github.com/MikhailBurdukov)). +* Add metrics to track the number of directories created and removed by the `plain_rewritable` metadata storage, and the number of entries in the local-to-remote in-memory map. [#64175](https://github.com/ClickHouse/ClickHouse/pull/64175) ([Julia Kartseva](https://github.com/jkartseva)). * The query cache now considers identical queries with different settings as different. This increases robustness in cases where different settings (e.g. `limit` or `additional_table_filters`) would affect the query result. [#64205](https://github.com/ClickHouse/ClickHouse/pull/64205) ([Robert Schulze](https://github.com/rschu1ze)). -* Better Exception Message in Delete Table with Projection, users can understand the error and the steps should be taken. [#64212](https://github.com/ClickHouse/ClickHouse/pull/64212) ([jsc0218](https://github.com/jsc0218)). * Support the non standard error code `QpsLimitExceeded` in object storage as a retryable error. [#64225](https://github.com/ClickHouse/ClickHouse/pull/64225) ([Sema Checherinda](https://github.com/CheSema)). * Forbid converting a MergeTree table to replicated if the zookeeper path for this table already exists. [#64244](https://github.com/ClickHouse/ClickHouse/pull/64244) ([Kirill](https://github.com/kirillgarbar)). -* If "replica group" is configured for a `Replicated` database, automatically create a cluster that includes replicas from all groups. [#64312](https://github.com/ClickHouse/ClickHouse/pull/64312) ([Alexander Tokmakov](https://github.com/tavplubix)). -* Added settings to disable materialization of skip indexes and statistics on inserts (`materialize_skip_indexes_on_insert` and `materialize_statistics_on_insert`). [#64391](https://github.com/ClickHouse/ClickHouse/pull/64391) ([Anton Popov](https://github.com/CurtizJ)). -* Use the allocated memory size to calculate the row group size and reduce the peak memory of the parquet writer in single-threaded mode. [#64424](https://github.com/ClickHouse/ClickHouse/pull/64424) ([LiuNeng](https://github.com/liuneng1994)). -* Added new configuration input_format_parquet_prefer_block_bytes to control the average output block bytes, and modified the default value of input_format_parquet_max_block_size to 65409. [#64427](https://github.com/ClickHouse/ClickHouse/pull/64427) ([LiuNeng](https://github.com/liuneng1994)). +* Added a new setting `input_format_parquet_prefer_block_bytes` to control the average output block bytes, and modified the default value of `input_format_parquet_max_block_size` to 65409. [#64427](https://github.com/ClickHouse/ClickHouse/pull/64427) ([LiuNeng](https://github.com/liuneng1994)). +* Allow proxy to be bypassed for hosts specified in `no_proxy` env variable and ClickHouse proxy configuration. [#63314](https://github.com/ClickHouse/ClickHouse/pull/63314) ([Arthur Passos](https://github.com/arthurpassos)). * Always start Keeper with sufficient amount of threads in global thread pool. [#64444](https://github.com/ClickHouse/ClickHouse/pull/64444) ([Duc Canh Le](https://github.com/canhld94)). -* Settings from user config doesn't affect merges and mutations for MergeTree on top of object storage. [#64456](https://github.com/ClickHouse/ClickHouse/pull/64456) ([alesapin](https://github.com/alesapin)). -* Setting `replace_long_file_name_to_hash` is enabled by default for `MergeTree` tables. [#64457](https://github.com/ClickHouse/ClickHouse/pull/64457) ([Anton Popov](https://github.com/CurtizJ)). -* Improve the iterator of sparse column to reduce call of size(). [#64497](https://github.com/ClickHouse/ClickHouse/pull/64497) ([Jiebin Sun](https://github.com/jiebinn)). -* Update condition to use copy for azure blob storage. [#64518](https://github.com/ClickHouse/ClickHouse/pull/64518) ([SmitaRKulkarni](https://github.com/SmitaRKulkarni)). +* Settings from the user's config don't affect merges and mutations for `MergeTree` on top of object storage. [#64456](https://github.com/ClickHouse/ClickHouse/pull/64456) ([alesapin](https://github.com/alesapin)). * Support the non standard error code `TotalQpsLimitExceeded` in object storage as a retryable error. [#64520](https://github.com/ClickHouse/ClickHouse/pull/64520) ([Sema Checherinda](https://github.com/CheSema)). -* Optimized memory usage of vertical merges for tables with high number of skip indexes. [#64580](https://github.com/ClickHouse/ClickHouse/pull/64580) ([Anton Popov](https://github.com/CurtizJ)). -* Introduced two additional columns in the `system.query_log`: `used_privileges` and `missing_privileges`. `used_privileges` is populated with the privileges that were checked during query execution, and `missing_privileges` contains required privileges that are missing. [#64597](https://github.com/ClickHouse/ClickHouse/pull/64597) ([Alexey Katsman](https://github.com/alexkats)). -* Add settings `parallel_replicas_custom_key_range_lower` and `parallel_replicas_custom_key_range_upper` to control how parallel replicas with dynamic shards parallelizes queries when using a range filter. [#64604](https://github.com/ClickHouse/ClickHouse/pull/64604) ([josh-hildred](https://github.com/josh-hildred)). * Updated Advanced Dashboard for both open-source and ClickHouse Cloud versions to include a chart for 'Maximum concurrent network connections'. [#64610](https://github.com/ClickHouse/ClickHouse/pull/64610) ([Thom O'Connor](https://github.com/thomoco)). -* The second argument (scale) of functions `round()`, `roundBankers()`, `floor()`, `ceil()` and `trunc()` can now be non-const. [#64798](https://github.com/ClickHouse/ClickHouse/pull/64798) ([Mikhail Gorshkov](https://github.com/mgorshkov)). -* Improve progress report on zeros_mt and generateRandom. [#64804](https://github.com/ClickHouse/ClickHouse/pull/64804) ([Raúl Marín](https://github.com/Algunenano)). -* Add an asynchronous metric jemalloc.profile.active to show whether sampling is currently active. This is an activation mechanism in addition to prof.active; both must be active for the calling thread to sample. [#64842](https://github.com/ClickHouse/ClickHouse/pull/64842) ([Unalian](https://github.com/Unalian)). -* Support statistics with ReplicatedMergeTree. [#64934](https://github.com/ClickHouse/ClickHouse/pull/64934) ([Han Fei](https://github.com/hanfei1991)). +* Improve progress report on `zeros_mt` and `generateRandom`. [#64804](https://github.com/ClickHouse/ClickHouse/pull/64804) ([Raúl Marín](https://github.com/Algunenano)). +* Add an asynchronous metric `jemalloc.profile.active` to show whether sampling is currently active. This is an activation mechanism in addition to prof.active; both must be active for the calling thread to sample. [#64842](https://github.com/ClickHouse/ClickHouse/pull/64842) ([Unalian](https://github.com/Unalian)). * Remove mark of `allow_experimental_join_condition` as important. This mark may have prevented distributed queries in a mixed versions cluster from being executed successfully. [#65008](https://github.com/ClickHouse/ClickHouse/pull/65008) ([Nikita Mikhaylov](https://github.com/nikitamikhaylov)). * Added server Asynchronous metrics `DiskGetObjectThrottler*` and `DiskGetObjectThrottler*` reflecting request per second rate limit defined with `s3_max_get_rps` and `s3_max_put_rps` disk settings and currently available number of requests that could be sent without hitting throttling limit on the disk. Metrics are defined for every disk that has a configured limit. [#65050](https://github.com/ClickHouse/ClickHouse/pull/65050) ([Sergei Trifonov](https://github.com/serxa)). -* Added a setting `output_format_pretty_display_footer_column_names` which when enabled displays column names at the end of the table for long tables (50 rows by default), with the threshold value for minimum number of rows controlled by `output_format_pretty_display_footer_column_names_min_rows`. [#65144](https://github.com/ClickHouse/ClickHouse/pull/65144) ([Shaun Struwig](https://github.com/Blargian)). -* Returned back the behaviour of how ClickHouse works and interprets Tuples in CSV format. This change effectively reverts https://github.com/ClickHouse/ClickHouse/pull/60994 and makes it available only under a few settings: `output_format_csv_serialize_tuple_into_separate_columns`, `input_format_csv_deserialize_separate_columns_into_tuple` and `input_format_csv_try_infer_strings_from_quoted_tuples`. [#65170](https://github.com/ClickHouse/ClickHouse/pull/65170) ([Nikita Mikhaylov](https://github.com/nikitamikhaylov)). -* Initialize global trace collector for Poco::ThreadPool (needed for keeper, etc). [#65239](https://github.com/ClickHouse/ClickHouse/pull/65239) ([Kseniia Sumarokova](https://github.com/kssenii)). -* Add validation when creating a user with bcrypt_hash. [#65242](https://github.com/ClickHouse/ClickHouse/pull/65242) ([Raúl Marín](https://github.com/Algunenano)). -* Unite s3/hdfs/azure storage implementations into a single class working with IObjectStorage. Same for *Cluster, data lakes and Queue storages. [#59767](https://github.com/ClickHouse/ClickHouse/pull/59767) ([Kseniia Sumarokova](https://github.com/kssenii)). -* Refactor data part writer to remove dependencies on MergeTreeData and DataPart. [#63620](https://github.com/ClickHouse/ClickHouse/pull/63620) ([Alexander Gololobov](https://github.com/davenger)). -* Add profile events for number of rows read during/after prewhere. [#64198](https://github.com/ClickHouse/ClickHouse/pull/64198) ([Nikita Taranov](https://github.com/nickitat)). -* Print query in explain plan with parallel replicas. [#64298](https://github.com/ClickHouse/ClickHouse/pull/64298) ([vdimir](https://github.com/vdimir)). +* Initialize global trace collector for `Poco::ThreadPool` (needed for Keeper, etc). [#65239](https://github.com/ClickHouse/ClickHouse/pull/65239) ([Kseniia Sumarokova](https://github.com/kssenii)). +* Add a validation when creating a user with `bcrypt_hash`. [#65242](https://github.com/ClickHouse/ClickHouse/pull/65242) ([Raúl Marín](https://github.com/Algunenano)). +* Add profile events for number of rows read during/after `PREWHERE`. [#64198](https://github.com/ClickHouse/ClickHouse/pull/64198) ([Nikita Taranov](https://github.com/nickitat)). +* Print query in `EXPLAIN PLAN` with parallel replicas. [#64298](https://github.com/ClickHouse/ClickHouse/pull/64298) ([vdimir](https://github.com/vdimir)). * Rename `allow_deprecated_functions` to `allow_deprecated_error_prone_window_functions`. [#64358](https://github.com/ClickHouse/ClickHouse/pull/64358) ([Raúl Marín](https://github.com/Algunenano)). -* Respect `max_read_buffer_size` setting for file descriptors as well in file() table function. [#64532](https://github.com/ClickHouse/ClickHouse/pull/64532) ([Azat Khuzhin](https://github.com/azat)). +* Respect `max_read_buffer_size` setting for file descriptors as well in the `file` table function. [#64532](https://github.com/ClickHouse/ClickHouse/pull/64532) ([Azat Khuzhin](https://github.com/azat)). * Disable transactions for unsupported storages even for materialized views. [#64918](https://github.com/ClickHouse/ClickHouse/pull/64918) ([alesapin](https://github.com/alesapin)). -* Refactor `KeyCondition` and key analysis to improve PartitionPruner and trivial count optimization. This is separated from [#60463](https://github.com/ClickHouse/ClickHouse/issues/60463) . [#61459](https://github.com/ClickHouse/ClickHouse/pull/61459) ([Amos Bird](https://github.com/amosbird)). +* Forbid `QUALIFY` clause in the old analyzer. The old analyzer ignored `QUALIFY`, so it could lead to unexpected data removal in mutations. [#65356](https://github.com/ClickHouse/ClickHouse/pull/65356) ([Dmitry Novik](https://github.com/novikd)). #### Bug Fix (user-visible misbehavior in an official stable release) +* A bug in Apache ORC library was fixed: Fixed ORC statistics calculation, when writing, for unsigned types on all platforms and Int8 on ARM. [#64563](https://github.com/ClickHouse/ClickHouse/pull/64563) ([Michael Kolupaev](https://github.com/al13n321)). +* Returned back the behaviour of how ClickHouse works and interprets Tuples in CSV format. This change effectively reverts https://github.com/ClickHouse/ClickHouse/pull/60994 and makes it available only under a few settings: `output_format_csv_serialize_tuple_into_separate_columns`, `input_format_csv_deserialize_separate_columns_into_tuple` and `input_format_csv_try_infer_strings_from_quoted_tuples`. [#65170](https://github.com/ClickHouse/ClickHouse/pull/65170) ([Nikita Mikhaylov](https://github.com/nikitamikhaylov)). * Fix a permission error where a user in a specific situation can escalate their privileges on the default database without necessary grants. [#64769](https://github.com/ClickHouse/ClickHouse/pull/64769) ([pufit](https://github.com/pufit)). * Fix crash with UniqInjectiveFunctionsEliminationPass and uniqCombined. [#65188](https://github.com/ClickHouse/ClickHouse/pull/65188) ([Raúl Marín](https://github.com/Algunenano)). * Fix a bug in ClickHouse Keeper that causes digest mismatch during closing session. [#65198](https://github.com/ClickHouse/ClickHouse/pull/65198) ([Aleksei Filatov](https://github.com/aalexfvk)). -* Forbid `QUALIFY` clause in the old analyzer. The old analyzer ignored `QUALIFY`, so it could lead to unexpected data removal in mutations. [#65356](https://github.com/ClickHouse/ClickHouse/pull/65356) ([Dmitry Novik](https://github.com/novikd)). * Use correct memory alignment for Distinct combinator. Previously, crash could happen because of invalid memory allocation when the combinator was used. [#65379](https://github.com/ClickHouse/ClickHouse/pull/65379) ([Antonio Andelic](https://github.com/antonio2368)). * Fix crash with `DISTINCT` and window functions. [#64767](https://github.com/ClickHouse/ClickHouse/pull/64767) ([Igor Nikonov](https://github.com/devcrafter)). * Fixed 'set' skip index not working with IN and indexHint(). [#62083](https://github.com/ClickHouse/ClickHouse/pull/62083) ([Michael Kolupaev](https://github.com/al13n321)). @@ -132,7 +130,6 @@ * Fixed `optimize_read_in_order` behaviour for ORDER BY ... NULLS FIRST / LAST on tables with nullable keys. [#64483](https://github.com/ClickHouse/ClickHouse/pull/64483) ([Eduard Karacharov](https://github.com/korowa)). * Fix the `Expression nodes list expected 1 projection names` and `Unknown expression or identifier` errors for queries with aliases to `GLOBAL IN.`. [#64517](https://github.com/ClickHouse/ClickHouse/pull/64517) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). * Fix an error `Cannot find column` in distributed queries with constant CTE in the `GROUP BY` key. [#64519](https://github.com/ClickHouse/ClickHouse/pull/64519) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). -* Fixed ORC statistics calculation, when writing, for unsigned types on all platforms and Int8 on ARM. [#64563](https://github.com/ClickHouse/ClickHouse/pull/64563) ([Michael Kolupaev](https://github.com/al13n321)). * Fix the crash loop when restoring from backup is blocked by creating an MV with a definer that hasn't been restored yet. [#64595](https://github.com/ClickHouse/ClickHouse/pull/64595) ([pufit](https://github.com/pufit)). * Fix the output of function `formatDateTimeInJodaSyntax` when a formatter generates an uneven number of characters and the last character is `0`. For example, `SELECT formatDateTimeInJodaSyntax(toDate('2012-05-29'), 'D')` now correctly returns `150` instead of previously `15`. [#64614](https://github.com/ClickHouse/ClickHouse/pull/64614) ([LiuNeng](https://github.com/liuneng1994)). * Do not rewrite aggregation if `-If` combinator is already used. [#64638](https://github.com/ClickHouse/ClickHouse/pull/64638) ([Dmitry Novik](https://github.com/novikd)). @@ -166,21 +163,14 @@ * This PR ensures that the type of the constant(IN operator's second parameter) is always visible during the IN operator's type conversion process. Otherwise, losing type information may cause some conversions to fail, such as the conversion from DateTime to Date. This fixes ([#64487](https://github.com/ClickHouse/ClickHouse/issues/64487)). [#65315](https://github.com/ClickHouse/ClickHouse/pull/65315) ([pn](https://github.com/chloro-pn)). #### Build/Testing/Packaging Improvement -* Make `network` service be required when using the rc init script to start the ClickHouse server daemon. [#60650](https://github.com/ClickHouse/ClickHouse/pull/60650) ([Chun-Sheng, Li](https://github.com/peter279k)). -* Fix typo in test_hdfsCluster_unset_skip_unavailable_shards. The test writes data to unskip_unavailable_shards, but uses skip_unavailable_shards from the previous test. [#64243](https://github.com/ClickHouse/ClickHouse/pull/64243) ([Mikhail Artemenko](https://github.com/Michicosun)). -* Reduce the size of some slow tests. [#64387](https://github.com/ClickHouse/ClickHouse/pull/64387) ([Raúl Marín](https://github.com/Algunenano)). -* Reduce the size of some slow tests. [#64452](https://github.com/ClickHouse/ClickHouse/pull/64452) ([Raúl Marín](https://github.com/Algunenano)). -* Fix test_lost_part_other_replica. [#64512](https://github.com/ClickHouse/ClickHouse/pull/64512) ([Raúl Marín](https://github.com/Algunenano)). -* Add tests for experimental unequal joins and randomize new settings in clickhouse-test. [#64535](https://github.com/ClickHouse/ClickHouse/pull/64535) ([Nikita Fomichev](https://github.com/fm4v)). -* Upgrade tests: Update config and work with release candidates. [#64542](https://github.com/ClickHouse/ClickHouse/pull/64542) ([Raúl Marín](https://github.com/Algunenano)). -* Add support for LLVM XRay. [#64592](https://github.com/ClickHouse/ClickHouse/pull/64592) ([Tomer Shafir](https://github.com/tomershafir)). -* Speed up 02995_forget_partition. [#64761](https://github.com/ClickHouse/ClickHouse/pull/64761) ([Raúl Marín](https://github.com/Algunenano)). -* Fix 02790_async_queries_in_query_log. [#64764](https://github.com/ClickHouse/ClickHouse/pull/64764) ([Raúl Marín](https://github.com/Algunenano)). -* Support LLVM XRay on Linux amd64 only. [#64837](https://github.com/ClickHouse/ClickHouse/pull/64837) ([Tomer Shafir](https://github.com/tomershafir)). -* Get rid of custom code in `tests/ci/download_release_packages.py` and `tests/ci/get_previous_release_tag.py` to avoid issues after the https://github.com/ClickHouse/ClickHouse/pull/64759 is merged. [#64848](https://github.com/ClickHouse/ClickHouse/pull/64848) ([Mikhail f. Shiryaev](https://github.com/Felixoid)). -* Decrease the `unit-test` image a few times. [#65102](https://github.com/ClickHouse/ClickHouse/pull/65102) ([Mikhail f. Shiryaev](https://github.com/Felixoid)). +* Add support for LLVM XRay. [#64592](https://github.com/ClickHouse/ClickHouse/pull/64592) [#64837](https://github.com/ClickHouse/ClickHouse/pull/64837) ([Tomer Shafir](https://github.com/tomershafir)). +* Unite s3/hdfs/azure storage implementations into a single class working with IObjectStorage. Same for *Cluster, data lakes and Queue storages. [#59767](https://github.com/ClickHouse/ClickHouse/pull/59767) ([Kseniia Sumarokova](https://github.com/kssenii)). +* Refactor data part writer to remove dependencies on MergeTreeData and DataPart. [#63620](https://github.com/ClickHouse/ClickHouse/pull/63620) ([Alexander Gololobov](https://github.com/davenger)). +* Refactor `KeyCondition` and key analysis to improve PartitionPruner and trivial count optimization. This is separated from [#60463](https://github.com/ClickHouse/ClickHouse/issues/60463) . [#61459](https://github.com/ClickHouse/ClickHouse/pull/61459) ([Amos Bird](https://github.com/amosbird)). +* Introduce assertions to verify all functions are called with columns of the right size. [#63723](https://github.com/ClickHouse/ClickHouse/pull/63723) ([Raúl Marín](https://github.com/Algunenano)). +* Make `network` service be required when using the `rc` init script to start the ClickHouse server daemon. [#60650](https://github.com/ClickHouse/ClickHouse/pull/60650) ([Chun-Sheng, Li](https://github.com/peter279k)). +* Reduce the size of some slow tests. [#64387](https://github.com/ClickHouse/ClickHouse/pull/64387) [#64452](https://github.com/ClickHouse/ClickHouse/pull/64452) ([Raúl Marín](https://github.com/Algunenano)). * Replay ZooKeeper logs using keeper-bench. [#62481](https://github.com/ClickHouse/ClickHouse/pull/62481) ([Antonio Andelic](https://github.com/antonio2368)). -* Re-enable OpenSSL session caching. [#65111](https://github.com/ClickHouse/ClickHouse/pull/65111) ([Robert Schulze](https://github.com/rschu1ze)). ### ClickHouse release 24.5, 2024-05-30