From 00935c3bb33825ec0c993ac2df8f47974e2c4018 Mon Sep 17 00:00:00 2001 From: alesapin Date: Mon, 30 Aug 2021 16:07:58 +0300 Subject: [PATCH 01/27] Trying to debug keeper logs disappearance --- src/Coordination/Changelog.cpp | 4 ++++ tests/config/config.d/keeper_port.xml | 3 ++- 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/src/Coordination/Changelog.cpp b/src/Coordination/Changelog.cpp index df5d8792b25..f9e9fc3de9e 100644 --- a/src/Coordination/Changelog.cpp +++ b/src/Coordination/Changelog.cpp @@ -513,11 +513,15 @@ void Changelog::writeAt(uint64_t index, const LogEntryPtr & log_entry) void Changelog::compact(uint64_t up_to_log_index) { + LOG_INFO(log, "Compact logs up to log index {}", up_to_log_index); for (auto itr = existing_changelogs.begin(); itr != existing_changelogs.end();) { /// Remove all completely outdated changelog files if (itr->second.to_log_index <= up_to_log_index) { + if (current_writer && itr->second.from_log_index == current_writer->getStartIndex()) + throw Exception(ErrorCodes::LOGICAL_ERROR, "Trying to remove log {} which is current active log for write. It's a bug.", itr->second.path); + LOG_INFO(log, "Removing changelog {} because of compaction", itr->second.path); std::erase_if(index_to_start_pos, [right_index = itr->second.to_log_index] (const auto & item) { return item.first <= right_index; }); std::filesystem::remove(itr->second.path); diff --git a/tests/config/config.d/keeper_port.xml b/tests/config/config.d/keeper_port.xml index b21df47bc85..8a9b192de3f 100644 --- a/tests/config/config.d/keeper_port.xml +++ b/tests/config/config.d/keeper_port.xml @@ -9,7 +9,8 @@ false 60000 - 1000000000000000 + 10000 + 33333 From 2160a2f8f7d177a8f4ab1f1abbbd62065e3205ce Mon Sep 17 00:00:00 2001 From: alesapin Date: Mon, 30 Aug 2021 16:23:31 +0300 Subject: [PATCH 02/27] Followup --- src/Coordination/KeeperDispatcher.cpp | 37 +++++++++++++++------------ tests/config/config.d/keeper_port.xml | 2 +- 2 files changed, 22 insertions(+), 17 deletions(-) diff --git a/src/Coordination/KeeperDispatcher.cpp b/src/Coordination/KeeperDispatcher.cpp index 26db925b4c5..f9e92ca0fca 100644 --- a/src/Coordination/KeeperDispatcher.cpp +++ b/src/Coordination/KeeperDispatcher.cpp @@ -293,10 +293,12 @@ void KeeperDispatcher::shutdown() if (session_cleaner_thread.joinable()) session_cleaner_thread.join(); - /// FIXME not the best way to notify - requests_queue->push({}); - if (request_thread.joinable()) - request_thread.join(); + if (requests_queue) + { + requests_queue->push({}); + if (request_thread.joinable()) + request_thread.join(); + } responses_queue.push({}); if (responses_thread.joinable()) @@ -310,20 +312,23 @@ void KeeperDispatcher::shutdown() if (server) server->shutdown(); - KeeperStorage::RequestForSession request_for_session; - - /// Set session expired for all pending requests - while (requests_queue->tryPop(request_for_session)) + if (requests_queue) { - if (request_for_session.request) + KeeperStorage::RequestForSession request_for_session; + + /// Set session expired for all pending requests + while (requests_queue->tryPop(request_for_session)) { - auto response = request_for_session.request->makeResponse(); - response->error = Coordination::Error::ZSESSIONEXPIRED; - setResponse(request_for_session.session_id, response); - } - else - { - break; + if (request_for_session.request) + { + auto response = request_for_session.request->makeResponse(); + response->error = Coordination::Error::ZSESSIONEXPIRED; + setResponse(request_for_session.session_id, response); + } + else + { + break; + } } } diff --git a/tests/config/config.d/keeper_port.xml b/tests/config/config.d/keeper_port.xml index 8a9b192de3f..eff0555e5f1 100644 --- a/tests/config/config.d/keeper_port.xml +++ b/tests/config/config.d/keeper_port.xml @@ -10,7 +10,7 @@ 60000 10000 - 33333 + 33333 From a5400928abe73c7cdb64447d0d2b3a99a2d082a9 Mon Sep 17 00:00:00 2001 From: alesapin Date: Tue, 31 Aug 2021 12:12:11 +0300 Subject: [PATCH 03/27] Interm --- src/Coordination/Changelog.cpp | 68 ++++++++++++++-------- src/Coordination/Changelog.h | 7 ++- src/Coordination/tests/gtest_for_build.cpp | 1 + utils/CMakeLists.txt | 1 + 4 files changed, 50 insertions(+), 27 deletions(-) diff --git a/src/Coordination/Changelog.cpp b/src/Coordination/Changelog.cpp index f9e9fc3de9e..32117ef9f77 100644 --- a/src/Coordination/Changelog.cpp +++ b/src/Coordination/Changelog.cpp @@ -142,8 +142,12 @@ private: struct ChangelogReadResult { + /// Total entries read from log including skipped uint64_t entries_read; + /// First entry actually read log (not including skipped) uint64_t first_read_index; + /// Last entry read from log + uint64_t last_read_index; off_t last_position; bool error; }; @@ -224,6 +228,7 @@ public: /// Put it into in memory structure logs.emplace(record.header.index, log_entry); index_to_offset[record.header.index] = result.last_position; + result.last_read_index = record.header.index; if (result.entries_read % 50000 == 0) LOG_TRACE(log, "Reading changelog from path {}, entries {}", filepath, result.entries_read); @@ -282,14 +287,11 @@ void Changelog::readChangelogAndInitWriter(uint64_t last_commited_log_index, uin { uint64_t total_read = 0; - /// Amount of entries in last log index - uint64_t entries_in_last = 0; /// Log idx of the first incomplete log (key in existing_changelogs) + /// There must be no other logs after incomplete one. int64_t first_incomplete_log_start_index = -1; /// if -1 then no incomplete log exists - ChangelogReadResult result{}; - /// First log index which was read from all changelogs - uint64_t first_read_index = 0; + ChangelogReadResult last_log_read_result{}; /// We must start to read from this log index uint64_t start_to_read_from = last_commited_log_index; @@ -307,7 +309,7 @@ void Changelog::readChangelogAndInitWriter(uint64_t last_commited_log_index, uin for (const auto & [changelog_start_index, changelog_description] : existing_changelogs) { /// How many entries we have in the last changelog - entries_in_last = changelog_description.expectedEntriesCountInLog(); + uint64_t expected_entries_in_log = changelog_description.expectedEntriesCountInLog(); /// [from_log_index.>=.......start_to_read_from.....<=.to_log_index] if (changelog_description.to_log_index >= start_to_read_from) @@ -332,18 +334,26 @@ void Changelog::readChangelogAndInitWriter(uint64_t last_commited_log_index, uin ChangelogReader reader(changelog_description.path); - result = reader.readChangelog(logs, start_to_read_from, index_to_start_pos, log); + last_log_read_result = reader.readChangelog(logs, start_to_read_from, index_to_start_pos, log); started = true; /// Otherwise we have already initialized it - if (first_read_index == 0) - first_read_index = result.first_read_index; + if (min_log_id == 0) + min_log_id = last_log_read_result.first_read_index; + + total_read += last_log_read_result.entries_read; + + if (last_log_read_result.last_read_index != 0) + { + /// Entries in logs goes one by one so our max log entry is + /// first_read_index of the last log + total entries in this last log. + max_log_id = last_log_read_result.last_read_index; + } - total_read += result.entries_read; /// May happen after truncate, crash or simply unfinished log - if (result.entries_read < entries_in_last) + if (last_log_read_result.entries_read < expected_entries_in_log) { first_incomplete_log_start_index = changelog_start_index; break; @@ -351,10 +361,11 @@ void Changelog::readChangelogAndInitWriter(uint64_t last_commited_log_index, uin } } - if (first_read_index != 0) - start_index = first_read_index; - else /// We just may have no logs (only snapshot) - start_index = last_commited_log_index; + if (min_log_id == 0) /// We just may have no logs (only snapshot or nothing) + { + min_log_id = last_commited_log_index; + max_log_id = last_commited_log_index == 0 ? 0 : last_commited_log_index - 1; + } /// Found some broken or non finished logs /// We have to remove broken data and continue to write into incomplete log. @@ -381,20 +392,20 @@ void Changelog::readChangelogAndInitWriter(uint64_t last_commited_log_index, uin LOG_TRACE(log, "Continue to write into {}", description.path); current_writer = std::make_unique(description.path, WriteMode::Append, description.from_log_index); - current_writer->setEntriesWritten(result.entries_read); + current_writer->setEntriesWritten(last_log_read_result.entries_read); /// Truncate all broken entries from log - if (result.error) + if (last_log_read_result.error) { LOG_WARNING(log, "Read finished with error, truncating all broken log entries"); - current_writer->truncateToLength(result.last_position); + current_writer->truncateToLength(last_log_read_result.last_position); } } } /// Start new log if we don't initialize writer from previous log if (!current_writer) - rotate(start_index + total_read); + rotate(max_log_id + 1); } void Changelog::rotate(uint64_t new_start_log_index) @@ -439,7 +450,7 @@ void Changelog::appendEntry(uint64_t index, const LogEntryPtr & log_entry) throw Exception(ErrorCodes::LOGICAL_ERROR, "Changelog must be initialized before appending records"); if (logs.empty()) - start_index = index; + min_log_id = index; const auto & current_changelog_description = existing_changelogs[current_writer->getStartIndex()]; const bool log_is_complete = current_writer->getEntriesWritten() == current_changelog_description.expectedEntriesCountInLog(); @@ -452,6 +463,7 @@ void Changelog::appendEntry(uint64_t index, const LogEntryPtr & log_entry) throw Exception(ErrorCodes::LOGICAL_ERROR, "Record with index {} already exists", index); logs[index] = makeClone(log_entry); + max_log_id = index; } void Changelog::writeAt(uint64_t index, const LogEntryPtr & log_entry) @@ -513,11 +525,14 @@ void Changelog::writeAt(uint64_t index, const LogEntryPtr & log_entry) void Changelog::compact(uint64_t up_to_log_index) { + if (up_to_log_index > max_log_id) + throw Exception(ErrorCodes::LOGICAL_ERROR, "Trying to compact logs up to {}, but our max log is {}. It's a bug", up_to_log_index, max_log_id); + LOG_INFO(log, "Compact logs up to log index {}", up_to_log_index); for (auto itr = existing_changelogs.begin(); itr != existing_changelogs.end();) { /// Remove all completely outdated changelog files - if (itr->second.to_log_index <= up_to_log_index) + if (itr->second.to_log_index < up_to_log_index) { if (current_writer && itr->second.from_log_index == current_writer->getStartIndex()) throw Exception(ErrorCodes::LOGICAL_ERROR, "Trying to remove log {} which is current active log for write. It's a bug.", itr->second.path); @@ -530,8 +545,10 @@ void Changelog::compact(uint64_t up_to_log_index) else /// Files are ordered, so all subsequent should exist break; } - start_index = up_to_log_index + 1; + min_log_id = std::max(min_log_id, up_to_log_index + 1); std::erase_if(logs, [up_to_log_index] (const auto & item) { return item.first <= up_to_log_index; }); + + LOG_INFO(log, "Compaction up to {} finished new start index {}", up_to_log_index, min_log_id); } LogEntryPtr Changelog::getLastEntry() const @@ -539,10 +556,13 @@ LogEntryPtr Changelog::getLastEntry() const /// This entry treaded in special way by NuRaft static LogEntryPtr fake_entry = nuraft::cs_new(0, nuraft::buffer::alloc(sizeof(uint64_t))); - const uint64_t next_index = getNextEntryIndex() - 1; - auto entry = logs.find(next_index); + std::cerr << "MAX LOG ID:" << max_log_id << std::endl; + auto entry = logs.find(max_log_id); if (entry == logs.end()) + { + std::cerr << "MAX LOG ID NOT FOUND" << std::endl; return fake_entry; + } return entry->second; } diff --git a/src/Coordination/Changelog.h b/src/Coordination/Changelog.h index 119fc91c2c7..1d25b9dcf38 100644 --- a/src/Coordination/Changelog.h +++ b/src/Coordination/Changelog.h @@ -87,12 +87,12 @@ public: uint64_t getNextEntryIndex() const { - return start_index + logs.size(); + return max_log_id + 1; } uint64_t getStartIndex() const { - return start_index; + return min_log_id; } /// Last entry in log, or fake entry with term 0 if log is empty @@ -144,7 +144,8 @@ private: /// Mapping log_id -> log_entry IndexToLogEntry logs; /// Start log_id which exists in all "active" logs - uint64_t start_index = 0; + uint64_t min_log_id = 0; + uint64_t max_log_id = 0; }; } diff --git a/src/Coordination/tests/gtest_for_build.cpp b/src/Coordination/tests/gtest_for_build.cpp index 47eadbf9720..4b04bda88fb 100644 --- a/src/Coordination/tests/gtest_for_build.cpp +++ b/src/Coordination/tests/gtest_for_build.cpp @@ -404,6 +404,7 @@ TEST(CoordinationTest, ChangelogTestCompaction) /// And we able to read it DB::KeeperLogStore changelog_reader("./logs", 5, true); changelog_reader.init(7, 0); + EXPECT_EQ(changelog_reader.size(), 1); EXPECT_EQ(changelog_reader.start_index(), 7); EXPECT_EQ(changelog_reader.next_slot(), 8); diff --git a/utils/CMakeLists.txt b/utils/CMakeLists.txt index a6bf2843e9a..e6dc3c40b4c 100644 --- a/utils/CMakeLists.txt +++ b/utils/CMakeLists.txt @@ -20,6 +20,7 @@ if (NOT DEFINED ENABLE_UTILS OR ENABLE_UTILS) add_subdirectory (corrector_utf8) add_subdirectory (zookeeper-cli) add_subdirectory (zookeeper-test) + add_subdirectory (zookeeper-cleaner) add_subdirectory (zookeeper-dump-tree) add_subdirectory (zookeeper-remove-by-list) add_subdirectory (zookeeper-create-entry-to-download-part) From ff0ed1b290d91299a5b5d52889365eeade49bb27 Mon Sep 17 00:00:00 2001 From: alesapin Date: Tue, 31 Aug 2021 12:42:15 +0300 Subject: [PATCH 04/27] Fix unit test --- src/Coordination/tests/gtest_for_build.cpp | 30 +++++++++++----------- 1 file changed, 15 insertions(+), 15 deletions(-) diff --git a/src/Coordination/tests/gtest_for_build.cpp b/src/Coordination/tests/gtest_for_build.cpp index 4b04bda88fb..078d0743492 100644 --- a/src/Coordination/tests/gtest_for_build.cpp +++ b/src/Coordination/tests/gtest_for_build.cpp @@ -1318,7 +1318,7 @@ TEST(CoordinationTest, TestRotateIntervalChanges) } } - EXPECT_TRUE(fs::exists("./logs/changelog_0_99.bin")); + EXPECT_TRUE(fs::exists("./logs/changelog_1_100.bin")); DB::KeeperLogStore changelog_1("./logs", 10, true); changelog_1.init(0, 50); @@ -1331,8 +1331,8 @@ TEST(CoordinationTest, TestRotateIntervalChanges) changelog_1.end_of_append_batch(0, 0); } - EXPECT_TRUE(fs::exists("./logs/changelog_0_99.bin")); - EXPECT_TRUE(fs::exists("./logs/changelog_100_109.bin")); + EXPECT_TRUE(fs::exists("./logs/changelog_1_100.bin")); + EXPECT_TRUE(fs::exists("./logs/changelog_101_110.bin")); DB::KeeperLogStore changelog_2("./logs", 7, true); changelog_2.init(98, 55); @@ -1347,11 +1347,11 @@ TEST(CoordinationTest, TestRotateIntervalChanges) } changelog_2.compact(105); - EXPECT_FALSE(fs::exists("./logs/changelog_0_99.bin")); - EXPECT_TRUE(fs::exists("./logs/changelog_100_109.bin")); - EXPECT_TRUE(fs::exists("./logs/changelog_110_116.bin")); - EXPECT_TRUE(fs::exists("./logs/changelog_117_123.bin")); - EXPECT_TRUE(fs::exists("./logs/changelog_124_130.bin")); + EXPECT_FALSE(fs::exists("./logs/changelog_1_100.bin")); + EXPECT_TRUE(fs::exists("./logs/changelog_101_110.bin")); + EXPECT_TRUE(fs::exists("./logs/changelog_111_117.bin")); + EXPECT_TRUE(fs::exists("./logs/changelog_118_124.bin")); + EXPECT_TRUE(fs::exists("./logs/changelog_125_131.bin")); DB::KeeperLogStore changelog_3("./logs", 5, true); changelog_3.init(116, 3); @@ -1365,14 +1365,14 @@ TEST(CoordinationTest, TestRotateIntervalChanges) } changelog_3.compact(125); - EXPECT_FALSE(fs::exists("./logs/changelog_100_109.bin")); - EXPECT_FALSE(fs::exists("./logs/changelog_110_116.bin")); - EXPECT_FALSE(fs::exists("./logs/changelog_117_123.bin")); + EXPECT_FALSE(fs::exists("./logs/changelog_101_110.bin")); + EXPECT_FALSE(fs::exists("./logs/changelog_111_117.bin")); + EXPECT_FALSE(fs::exists("./logs/changelog_118_124.bin")); - EXPECT_TRUE(fs::exists("./logs/changelog_124_130.bin")); - EXPECT_TRUE(fs::exists("./logs/changelog_131_135.bin")); - EXPECT_TRUE(fs::exists("./logs/changelog_136_140.bin")); - EXPECT_TRUE(fs::exists("./logs/changelog_141_145.bin")); + EXPECT_TRUE(fs::exists("./logs/changelog_125_131.bin")); + EXPECT_TRUE(fs::exists("./logs/changelog_132_136.bin")); + EXPECT_TRUE(fs::exists("./logs/changelog_137_141.bin")); + EXPECT_TRUE(fs::exists("./logs/changelog_142_146.bin")); } From decbd993385be9c7e42c70dc02a25aeb1cd69878 Mon Sep 17 00:00:00 2001 From: alesapin Date: Tue, 31 Aug 2021 12:50:11 +0300 Subject: [PATCH 05/27] Remove debug logs --- src/Coordination/Changelog.cpp | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/Coordination/Changelog.cpp b/src/Coordination/Changelog.cpp index 32117ef9f77..635dedcec80 100644 --- a/src/Coordination/Changelog.cpp +++ b/src/Coordination/Changelog.cpp @@ -556,11 +556,9 @@ LogEntryPtr Changelog::getLastEntry() const /// This entry treaded in special way by NuRaft static LogEntryPtr fake_entry = nuraft::cs_new(0, nuraft::buffer::alloc(sizeof(uint64_t))); - std::cerr << "MAX LOG ID:" << max_log_id << std::endl; auto entry = logs.find(max_log_id); if (entry == logs.end()) { - std::cerr << "MAX LOG ID NOT FOUND" << std::endl; return fake_entry; } From 167372e48a78c34b1d986f324b308a16042db855 Mon Sep 17 00:00:00 2001 From: alesapin Date: Tue, 31 Aug 2021 12:57:22 +0300 Subject: [PATCH 06/27] Add comment --- src/Coordination/Changelog.cpp | 1 + 1 file changed, 1 insertion(+) diff --git a/src/Coordination/Changelog.cpp b/src/Coordination/Changelog.cpp index 635dedcec80..e5f2a52f2c7 100644 --- a/src/Coordination/Changelog.cpp +++ b/src/Coordination/Changelog.cpp @@ -545,6 +545,7 @@ void Changelog::compact(uint64_t up_to_log_index) else /// Files are ordered, so all subsequent should exist break; } + /// Compaction from the past is possible, so don't make our min_log_id smaller. min_log_id = std::max(min_log_id, up_to_log_index + 1); std::erase_if(logs, [up_to_log_index] (const auto & item) { return item.first <= up_to_log_index; }); From e8ab58bc877531cb3fc1778df06eeae78aa6b734 Mon Sep 17 00:00:00 2001 From: alesapin Date: Tue, 31 Aug 2021 13:07:05 +0300 Subject: [PATCH 07/27] Remove strange comment --- src/Coordination/Changelog.cpp | 5 ----- 1 file changed, 5 deletions(-) diff --git a/src/Coordination/Changelog.cpp b/src/Coordination/Changelog.cpp index e5f2a52f2c7..53631b440b1 100644 --- a/src/Coordination/Changelog.cpp +++ b/src/Coordination/Changelog.cpp @@ -345,12 +345,7 @@ void Changelog::readChangelogAndInitWriter(uint64_t last_commited_log_index, uin total_read += last_log_read_result.entries_read; if (last_log_read_result.last_read_index != 0) - { - /// Entries in logs goes one by one so our max log entry is - /// first_read_index of the last log + total entries in this last log. max_log_id = last_log_read_result.last_read_index; - } - /// May happen after truncate, crash or simply unfinished log if (last_log_read_result.entries_read < expected_entries_in_log) From 49d247ca40fdd5322e2cc59fdd61fedff857ebce Mon Sep 17 00:00:00 2001 From: alesapin Date: Tue, 31 Aug 2021 13:59:39 +0300 Subject: [PATCH 08/27] Fix compaction after snapshot --- src/Coordination/Changelog.cpp | 31 +++++++++++++++++++++++++------ 1 file changed, 25 insertions(+), 6 deletions(-) diff --git a/src/Coordination/Changelog.cpp b/src/Coordination/Changelog.cpp index 53631b440b1..b6996fb8931 100644 --- a/src/Coordination/Changelog.cpp +++ b/src/Coordination/Changelog.cpp @@ -520,17 +520,28 @@ void Changelog::writeAt(uint64_t index, const LogEntryPtr & log_entry) void Changelog::compact(uint64_t up_to_log_index) { - if (up_to_log_index > max_log_id) - throw Exception(ErrorCodes::LOGICAL_ERROR, "Trying to compact logs up to {}, but our max log is {}. It's a bug", up_to_log_index, max_log_id); + LOG_INFO(log, "Compact logs up to log index {}, our max log id is {}", up_to_log_index, max_log_id); + + bool remove_all_logs = false; + if (up_to_log_index > max_log_id) + { + LOG_INFO(log, "Seems like this node recovers from leaders snapshot, removing all logs"); + remove_all_logs = true; + } + + bool need_rotate = false; - LOG_INFO(log, "Compact logs up to log index {}", up_to_log_index); for (auto itr = existing_changelogs.begin(); itr != existing_changelogs.end();) { /// Remove all completely outdated changelog files - if (itr->second.to_log_index < up_to_log_index) + if (remove_all_logs || itr->second.to_log_index <= up_to_log_index) { if (current_writer && itr->second.from_log_index == current_writer->getStartIndex()) - throw Exception(ErrorCodes::LOGICAL_ERROR, "Trying to remove log {} which is current active log for write. It's a bug.", itr->second.path); + { + LOG_INFO(log, "Trying to remove log {} which is current active log for write. Possibly this node recovers from snapshot", itr->second.path); + need_rotate = true; + current_writer.reset(); + } LOG_INFO(log, "Removing changelog {} because of compaction", itr->second.path); std::erase_if(index_to_start_pos, [right_index = itr->second.to_log_index] (const auto & item) { return item.first <= right_index; }); @@ -542,9 +553,17 @@ void Changelog::compact(uint64_t up_to_log_index) } /// Compaction from the past is possible, so don't make our min_log_id smaller. min_log_id = std::max(min_log_id, up_to_log_index + 1); + + /// If we received snapshot from leader we may compact up to more fresh log + if (up_to_log_index > max_log_id) + max_log_id = up_to_log_index; + std::erase_if(logs, [up_to_log_index] (const auto & item) { return item.first <= up_to_log_index; }); - LOG_INFO(log, "Compaction up to {} finished new start index {}", up_to_log_index, min_log_id); + if (need_rotate) + rotate(up_to_log_index + 1); + + LOG_INFO(log, "Compaction up to {} finished new min index {}, new max index {}", up_to_log_index, min_log_id, max_log_id); } LogEntryPtr Changelog::getLastEntry() const From a1ac4fc7e0a342bee3181a77f0e6ef94e07470e3 Mon Sep 17 00:00:00 2001 From: alesapin Date: Tue, 31 Aug 2021 14:02:26 +0300 Subject: [PATCH 09/27] Comments --- src/Coordination/Changelog.cpp | 1 - src/Coordination/Changelog.h | 1 + 2 files changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Coordination/Changelog.cpp b/src/Coordination/Changelog.cpp index b6996fb8931..917cb75400e 100644 --- a/src/Coordination/Changelog.cpp +++ b/src/Coordination/Changelog.cpp @@ -530,7 +530,6 @@ void Changelog::compact(uint64_t up_to_log_index) } bool need_rotate = false; - for (auto itr = existing_changelogs.begin(); itr != existing_changelogs.end();) { /// Remove all completely outdated changelog files diff --git a/src/Coordination/Changelog.h b/src/Coordination/Changelog.h index 1d25b9dcf38..15ff4190c49 100644 --- a/src/Coordination/Changelog.h +++ b/src/Coordination/Changelog.h @@ -144,6 +144,7 @@ private: /// Mapping log_id -> log_entry IndexToLogEntry logs; /// Start log_id which exists in all "active" logs + /// min_log_id + 1 == max_log_id means empty log storage for NuRaft uint64_t min_log_id = 0; uint64_t max_log_id = 0; }; From 9a08b66becc62f5869009fb84f64b55ec980c1d6 Mon Sep 17 00:00:00 2001 From: alesapin Date: Tue, 31 Aug 2021 15:36:47 +0300 Subject: [PATCH 10/27] Remove some accident changes --- tests/config/config.d/keeper_port.xml | 3 +-- utils/CMakeLists.txt | 1 - 2 files changed, 1 insertion(+), 3 deletions(-) diff --git a/tests/config/config.d/keeper_port.xml b/tests/config/config.d/keeper_port.xml index eff0555e5f1..b21df47bc85 100644 --- a/tests/config/config.d/keeper_port.xml +++ b/tests/config/config.d/keeper_port.xml @@ -9,8 +9,7 @@ false 60000 - 10000 - 33333 + 1000000000000000 diff --git a/utils/CMakeLists.txt b/utils/CMakeLists.txt index e6dc3c40b4c..a6bf2843e9a 100644 --- a/utils/CMakeLists.txt +++ b/utils/CMakeLists.txt @@ -20,7 +20,6 @@ if (NOT DEFINED ENABLE_UTILS OR ENABLE_UTILS) add_subdirectory (corrector_utf8) add_subdirectory (zookeeper-cli) add_subdirectory (zookeeper-test) - add_subdirectory (zookeeper-cleaner) add_subdirectory (zookeeper-dump-tree) add_subdirectory (zookeeper-remove-by-list) add_subdirectory (zookeeper-create-entry-to-download-part) From 319f03d66f69e48dfd7661accf4ada93c916623c Mon Sep 17 00:00:00 2001 From: alesapin Date: Wed, 1 Sep 2021 10:51:08 +0300 Subject: [PATCH 11/27] Fix PVS --- src/Coordination/Changelog.cpp | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/src/Coordination/Changelog.cpp b/src/Coordination/Changelog.cpp index 917cb75400e..680b8f03bb3 100644 --- a/src/Coordination/Changelog.cpp +++ b/src/Coordination/Changelog.cpp @@ -526,6 +526,8 @@ void Changelog::compact(uint64_t up_to_log_index) if (up_to_log_index > max_log_id) { LOG_INFO(log, "Seems like this node recovers from leaders snapshot, removing all logs"); + /// If we received snapshot from leader we may compact up to more fresh log + max_log_id = up_to_log_index; remove_all_logs = true; } @@ -552,11 +554,6 @@ void Changelog::compact(uint64_t up_to_log_index) } /// Compaction from the past is possible, so don't make our min_log_id smaller. min_log_id = std::max(min_log_id, up_to_log_index + 1); - - /// If we received snapshot from leader we may compact up to more fresh log - if (up_to_log_index > max_log_id) - max_log_id = up_to_log_index; - std::erase_if(logs, [up_to_log_index] (const auto & item) { return item.first <= up_to_log_index; }); if (need_rotate) From a8003e444b17a364781ff18154ecc585e4c6259e Mon Sep 17 00:00:00 2001 From: alesapin Date: Thu, 2 Sep 2021 12:20:11 +0300 Subject: [PATCH 12/27] Review fixes --- src/Coordination/Changelog.cpp | 158 ++++++++++++++++---------- src/Coordination/Changelog.h | 6 + src/Coordination/KeeperDispatcher.cpp | 27 ++--- src/Server/KeeperTCPHandler.cpp | 2 +- 4 files changed, 115 insertions(+), 78 deletions(-) diff --git a/src/Coordination/Changelog.cpp b/src/Coordination/Changelog.cpp index 680b8f03bb3..06f692533d4 100644 --- a/src/Coordination/Changelog.cpp +++ b/src/Coordination/Changelog.cpp @@ -142,12 +142,22 @@ private: struct ChangelogReadResult { - /// Total entries read from log including skipped - uint64_t entries_read; + /// Total entries read from log including skipped. + /// Useful when we decide to continue to write in the same log and want to know + /// how many entries was already written in it. + uint64_t total_entries_read_from_log; + + /// First index in log + uint64_t log_start_index; + /// First entry actually read log (not including skipped) uint64_t first_read_index; - /// Last entry read from log + /// Last entry read from log (last entry in log) + /// When we don't skip anything last_read_index - first_read_index = total_entries_read_from_log. + /// But when some entries from the start of log can be skipped because they are not required. uint64_t last_read_index; + + /// last offset we were able to read from log off_t last_position; bool error; }; @@ -160,6 +170,7 @@ public: , read_buf(filepath) {} + /// start_log_index -- all entries with index < start_log_index will be skipped, but accounted into total_entries_read_from_log ChangelogReadResult readChangelog(IndexToLogEntry & logs, uint64_t start_log_index, IndexToOffset & index_to_offset, Poco::Logger * log) { uint64_t previous_index = 0; @@ -214,7 +225,7 @@ public: if (logs.count(record.header.index) != 0) throw Exception(ErrorCodes::CORRUPTED_DATA, "Duplicated index id {} in log {}", record.header.index, filepath); - result.entries_read += 1; + result.total_entries_read_from_log += 1; /// Read but skip this entry because our state is already more fresh if (record.header.index < start_log_index) @@ -230,8 +241,8 @@ public: index_to_offset[record.header.index] = result.last_position; result.last_read_index = record.header.index; - if (result.entries_read % 50000 == 0) - LOG_TRACE(log, "Reading changelog from path {}, entries {}", filepath, result.entries_read); + if (result.total_entries_read_from_log % 50000 == 0) + LOG_TRACE(log, "Reading changelog from path {}, entries {}", filepath, result.total_entries_read_from_log); } } catch (const Exception & ex) @@ -248,7 +259,7 @@ public: tryLogCurrentException(log); } - LOG_TRACE(log, "Totally read from changelog {} {} entries", filepath, result.entries_read); + LOG_TRACE(log, "Totally read from changelog {} {} entries", filepath, result.total_entries_read_from_log); return result; } @@ -285,13 +296,10 @@ Changelog::Changelog( void Changelog::readChangelogAndInitWriter(uint64_t last_commited_log_index, uint64_t logs_to_keep) { - uint64_t total_read = 0; + std::optional last_log_read_result; - /// Log idx of the first incomplete log (key in existing_changelogs) - /// There must be no other logs after incomplete one. - int64_t first_incomplete_log_start_index = -1; /// if -1 then no incomplete log exists - - ChangelogReadResult last_log_read_result{}; + /// Last log has some free space to write + bool last_log_is_not_complete = false; /// We must start to read from this log index uint64_t start_to_read_from = last_commited_log_index; @@ -302,19 +310,14 @@ void Changelog::readChangelogAndInitWriter(uint64_t last_commited_log_index, uin else start_to_read_from = 1; - /// At least we read something - bool started = false; - /// Got through changelog files in order of start_index for (const auto & [changelog_start_index, changelog_description] : existing_changelogs) { - /// How many entries we have in the last changelog - uint64_t expected_entries_in_log = changelog_description.expectedEntriesCountInLog(); /// [from_log_index.>=.......start_to_read_from.....<=.to_log_index] if (changelog_description.to_log_index >= start_to_read_from) { - if (!started) /// still nothing was read + if (!last_log_read_result) /// still nothing was read { /// Our first log starts from the more fresh log_id than we required to read and this changelog is not empty log. /// So we are missing something in our logs, but it's not dataloss, we will receive snapshot and required @@ -322,8 +325,12 @@ void Changelog::readChangelogAndInitWriter(uint64_t last_commited_log_index, uin if (changelog_description.from_log_index > last_commited_log_index && (changelog_description.from_log_index - last_commited_log_index) > 1) { LOG_ERROR(log, "Some records was lost, last committed log index {}, smallest available log index on disk {}. Hopefully will receive missing records from leader.", last_commited_log_index, changelog_description.from_log_index); - first_incomplete_log_start_index = changelog_start_index; - break; + /// Nothing to do with our more fresh log, leader will overwrite them, so remove everything and just start from last_commited_index + removeAllLogs(); + min_log_id = last_commited_log_index; + max_log_id = last_commited_log_index == 0 ? 0 : last_commited_log_index - 1; + rotate(max_log_id + 1); + return; } else if (changelog_description.from_log_index > start_to_read_from) { @@ -336,73 +343,100 @@ void Changelog::readChangelogAndInitWriter(uint64_t last_commited_log_index, uin ChangelogReader reader(changelog_description.path); last_log_read_result = reader.readChangelog(logs, start_to_read_from, index_to_start_pos, log); - started = true; - /// Otherwise we have already initialized it if (min_log_id == 0) - min_log_id = last_log_read_result.first_read_index; + min_log_id = last_log_read_result->first_read_index; - total_read += last_log_read_result.entries_read; + if (last_log_read_result->last_read_index != 0) + max_log_id = last_log_read_result->last_read_index; - if (last_log_read_result.last_read_index != 0) - max_log_id = last_log_read_result.last_read_index; + last_log_read_result->log_start_index = changelog_description.from_log_index; + + /// How many entries we have in the last changelog + uint64_t expected_entries_in_log = changelog_description.expectedEntriesCountInLog(); /// May happen after truncate, crash or simply unfinished log - if (last_log_read_result.entries_read < expected_entries_in_log) + if (last_log_read_result->total_entries_read_from_log < expected_entries_in_log) { - first_incomplete_log_start_index = changelog_start_index; + last_log_is_not_complete = true; break; } } } - if (min_log_id == 0) /// We just may have no logs (only snapshot or nothing) + /// we can have empty log (with zero entries) and last_log_read_result will be initialized + if (!last_log_read_result || min_log_id == 0) /// We just may have no logs (only snapshot or nothing) { + /// Just to be sure they don't exist + removeAllLogs(); + min_log_id = last_commited_log_index; max_log_id = last_commited_log_index == 0 ? 0 : last_commited_log_index - 1; } - - /// Found some broken or non finished logs - /// We have to remove broken data and continue to write into incomplete log. - if (first_incomplete_log_start_index != -1) /// otherwise all logs completed so just start a new one + else if (last_log_is_not_complete) /// if it's complete just start new one { - auto start_remove_from = existing_changelogs.begin(); - if (started) - start_remove_from = existing_changelogs.upper_bound(first_incomplete_log_start_index); + assert(last_log_read_result != std::nullopt); + /// Actually they shouldn't exist, but to be sure we remove them + removeAllLogsAfter(last_log_read_result->log_start_index); - /// All subsequent logs shouldn't exist. But they may exist if we crashed after writeAt started. Remove them. - for (auto itr = start_remove_from; itr != existing_changelogs.end();) - { - LOG_WARNING(log, "Removing changelog {}, because it's goes after broken changelog entry", itr->second.path); - std::filesystem::remove(itr->second.path); - itr = existing_changelogs.erase(itr); - } + assert(!existing_changelogs.empty()); + assert(existing_changelogs.find(last_log_read_result->log_start_index)->first == existing_changelogs.rbegin()->first); - /// Continue to write into existing log - if (!existing_changelogs.empty()) - { - auto description = existing_changelogs.rbegin()->second; - if (description.expectedEntriesCountInLog() != rotate_interval) - LOG_TRACE(log, "Looks like rotate_logs_interval was changed, current {}, expected entries in last log {}", rotate_interval, description.expectedEntriesCountInLog()); + /// Continue to write into incomplete existing log + auto description = existing_changelogs[last_log_read_result->log_start_index]; - LOG_TRACE(log, "Continue to write into {}", description.path); - current_writer = std::make_unique(description.path, WriteMode::Append, description.from_log_index); - current_writer->setEntriesWritten(last_log_read_result.entries_read); - - /// Truncate all broken entries from log - if (last_log_read_result.error) - { - LOG_WARNING(log, "Read finished with error, truncating all broken log entries"); - current_writer->truncateToLength(last_log_read_result.last_position); - } - } + if (last_log_read_result->error) + initWriter(description, last_log_read_result->total_entries_read_from_log, /* truncate_after = */ last_log_read_result->last_position); + else + initWriter(description, last_log_read_result->total_entries_read_from_log); } - /// Start new log if we don't initialize writer from previous log + /// Start new log if we don't initialize writer from previous log. All logs can be "complete". if (!current_writer) rotate(max_log_id + 1); } + +void Changelog::initWriter(const ChangelogFileDescription & description, uint64_t entries_already_written, std::optional truncate_to_offset) +{ + if (description.expectedEntriesCountInLog() != rotate_interval) + LOG_TRACE(log, "Looks like rotate_logs_interval was changed, current {}, expected entries in last log {}", rotate_interval, description.expectedEntriesCountInLog()); + + LOG_TRACE(log, "Continue to write into {}", description.path); + current_writer = std::make_unique(description.path, WriteMode::Append, description.from_log_index); + current_writer->setEntriesWritten(entries_already_written); + + if (truncate_to_offset) + { + LOG_WARNING(log, "Changelog {} contain broken enties, truncating all broken log entries", description.path); + current_writer->truncateToLength(*truncate_to_offset); + } +} + +void Changelog::removeAllLogsAfter(uint64_t start_to_remove_from_id) +{ + auto start_to_remove_from = existing_changelogs.upper_bound(start_to_remove_from_id); + + /// All subsequent logs shouldn't exist. But they may exist if we crashed after writeAt started. Remove them. + for (auto itr = start_to_remove_from; itr != existing_changelogs.end();) + { + LOG_WARNING(log, "Removing changelog {}, because it's goes after broken changelog entry", itr->second.path); + std::filesystem::remove(itr->second.path); + itr = existing_changelogs.erase(itr); + } +} + +void Changelog::removeAllLogs() +{ + LOG_WARNING(log, "Removing all changelogs"); + for (auto itr = existing_changelogs.begin(); itr != existing_changelogs.end();) + { + LOG_WARNING(log, "Removing changelog {}, because it's goes after broken changelog entry", itr->second.path); + std::filesystem::remove(itr->second.path); + itr = existing_changelogs.erase(itr); + } +} + void Changelog::rotate(uint64_t new_start_log_index) { /// Flush previous log diff --git a/src/Coordination/Changelog.h b/src/Coordination/Changelog.h index 15ff4190c49..fdf045cb8a1 100644 --- a/src/Coordination/Changelog.h +++ b/src/Coordination/Changelog.h @@ -2,6 +2,7 @@ #include // Y_IGNORE #include +#include #include #include #include @@ -128,6 +129,11 @@ private: /// Starts new file [new_start_log_index, new_start_log_index + rotate_interval] void rotate(uint64_t new_start_log_index); + void removeAllLogsAfter(uint64_t start_to_remove_from_id); + void removeAllLogs(); + + void initWriter(const ChangelogFileDescription & description, uint64_t entries_already_written, std::optional truncate_to_offset = {}); + private: const std::string changelogs_dir; const uint64_t rotate_interval; diff --git a/src/Coordination/KeeperDispatcher.cpp b/src/Coordination/KeeperDispatcher.cpp index f9e92ca0fca..b4dc367ff62 100644 --- a/src/Coordination/KeeperDispatcher.cpp +++ b/src/Coordination/KeeperDispatcher.cpp @@ -312,23 +312,20 @@ void KeeperDispatcher::shutdown() if (server) server->shutdown(); - if (requests_queue) - { - KeeperStorage::RequestForSession request_for_session; + KeeperStorage::RequestForSession request_for_session; - /// Set session expired for all pending requests - while (requests_queue->tryPop(request_for_session)) + /// Set session expired for all pending requests + while (requests_queue && requests_queue->tryPop(request_for_session)) + { + if (request_for_session.request) { - if (request_for_session.request) - { - auto response = request_for_session.request->makeResponse(); - response->error = Coordination::Error::ZSESSIONEXPIRED; - setResponse(request_for_session.session_id, response); - } - else - { - break; - } + auto response = request_for_session.request->makeResponse(); + response->error = Coordination::Error::ZSESSIONEXPIRED; + setResponse(request_for_session.session_id, response); + } + else + { + break; } } diff --git a/src/Server/KeeperTCPHandler.cpp b/src/Server/KeeperTCPHandler.cpp index 7ead4d0d419..689884598f2 100644 --- a/src/Server/KeeperTCPHandler.cpp +++ b/src/Server/KeeperTCPHandler.cpp @@ -192,7 +192,7 @@ struct SocketInterruptablePollWrapper KeeperTCPHandler::KeeperTCPHandler(IServer & server_, const Poco::Net::StreamSocket & socket_) : Poco::Net::TCPServerConnection(socket_) , server(server_) - , log(&Poco::Logger::get("NuKeeperTCPHandler")) + , log(&Poco::Logger::get("KeeperTCPHandler")) , global_context(Context::createCopy(server.context())) , keeper_dispatcher(global_context->getKeeperDispatcher()) , operation_timeout(0, global_context->getConfigRef().getUInt("keeper_server.operation_timeout_ms", Coordination::DEFAULT_OPERATION_TIMEOUT_MS) * 1000) From d681ffabf82fef9c898bc9625be703c0fa0e2c5d Mon Sep 17 00:00:00 2001 From: alesapin Date: Thu, 2 Sep 2021 12:22:51 +0300 Subject: [PATCH 13/27] Comment --- src/Coordination/Changelog.h | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/Coordination/Changelog.h b/src/Coordination/Changelog.h index fdf045cb8a1..09050fd2e83 100644 --- a/src/Coordination/Changelog.h +++ b/src/Coordination/Changelog.h @@ -129,9 +129,11 @@ private: /// Starts new file [new_start_log_index, new_start_log_index + rotate_interval] void rotate(uint64_t new_start_log_index); + /// Remove all changelogs from disk with start_index bigger than start_to_remove_from_id void removeAllLogsAfter(uint64_t start_to_remove_from_id); + /// Remove all logs from disk void removeAllLogs(); - + /// Init writer for exising log with some entries already written void initWriter(const ChangelogFileDescription & description, uint64_t entries_already_written, std::optional truncate_to_offset = {}); private: From 9b1b3ec916a3c32ac7c5953d7b08ac11343640c9 Mon Sep 17 00:00:00 2001 From: alesapin Date: Thu, 2 Sep 2021 14:02:37 +0300 Subject: [PATCH 14/27] Fix style check --- src/Coordination/Changelog.h | 2 +- .../tests/{gtest_for_build.cpp => gtest_coordination.cpp} | 0 2 files changed, 1 insertion(+), 1 deletion(-) rename src/Coordination/tests/{gtest_for_build.cpp => gtest_coordination.cpp} (100%) diff --git a/src/Coordination/Changelog.h b/src/Coordination/Changelog.h index 09050fd2e83..1dfec0edb46 100644 --- a/src/Coordination/Changelog.h +++ b/src/Coordination/Changelog.h @@ -133,7 +133,7 @@ private: void removeAllLogsAfter(uint64_t start_to_remove_from_id); /// Remove all logs from disk void removeAllLogs(); - /// Init writer for exising log with some entries already written + /// Init writer for existing log with some entries already written void initWriter(const ChangelogFileDescription & description, uint64_t entries_already_written, std::optional truncate_to_offset = {}); private: diff --git a/src/Coordination/tests/gtest_for_build.cpp b/src/Coordination/tests/gtest_coordination.cpp similarity index 100% rename from src/Coordination/tests/gtest_for_build.cpp rename to src/Coordination/tests/gtest_coordination.cpp From ba78a737dc6f96eefde9abab128914b532071c80 Mon Sep 17 00:00:00 2001 From: alesapin Date: Thu, 2 Sep 2021 17:47:00 +0300 Subject: [PATCH 15/27] Fix detach/attach for ReplicatedVersionedCollapsingMergeTree after alter --- .../MergeTree/ReplicatedMergeTreeTableMetadata.cpp | 10 +++++++++- .../01526_alter_add_and_modify_order_zookeeper.sql | 14 ++++++++++++++ 2 files changed, 23 insertions(+), 1 deletion(-) diff --git a/src/Storages/MergeTree/ReplicatedMergeTreeTableMetadata.cpp b/src/Storages/MergeTree/ReplicatedMergeTreeTableMetadata.cpp index db1c2bc89af..be8cf26dee6 100644 --- a/src/Storages/MergeTree/ReplicatedMergeTreeTableMetadata.cpp +++ b/src/Storages/MergeTree/ReplicatedMergeTreeTableMetadata.cpp @@ -46,7 +46,15 @@ ReplicatedMergeTreeTableMetadata::ReplicatedMergeTreeTableMetadata(const MergeTr primary_key = formattedAST(metadata_snapshot->getPrimaryKey().expression_list_ast); if (metadata_snapshot->isPrimaryKeyDefined()) - sorting_key = formattedAST(metadata_snapshot->getSortingKey().expression_list_ast); + { + /// We don't use preparsed AST `sorting_key.expression_list_ast` because + /// it contain version column for VersionedCollapsingMergeTree, which + /// shouldn't be defined as a part of key definition AST. So the best + /// compatible way is just to convert definition_ast to list and + /// serialize it. In all other places key.expression_list_ast should be + /// used. + sorting_key = formattedAST(extractKeyExpressionList(metadata_snapshot->getSortingKey().definition_ast)); + } data_format_version = data.format_version; diff --git a/tests/queries/0_stateless/01526_alter_add_and_modify_order_zookeeper.sql b/tests/queries/0_stateless/01526_alter_add_and_modify_order_zookeeper.sql index b718ba199c1..aaf003c53f4 100644 --- a/tests/queries/0_stateless/01526_alter_add_and_modify_order_zookeeper.sql +++ b/tests/queries/0_stateless/01526_alter_add_and_modify_order_zookeeper.sql @@ -19,10 +19,20 @@ SETTINGS index_granularity = 8192; INSERT INTO table_for_alter VALUES(toDate('2019-10-01'), 'a', 1, 'aa', 1, 1, 1); +DETACH TABLE table_for_alter; + +ATTACH TABLE table_for_alter; + + SELECT * FROM table_for_alter; ALTER TABLE table_for_alter ADD COLUMN order UInt32, MODIFY ORDER BY (d, order); + +DETACH TABLE table_for_alter; + +ATTACH TABLE table_for_alter; + SELECT * FROM table_for_alter; SHOW CREATE TABLE table_for_alter; @@ -35,4 +45,8 @@ SELECT * FROM table_for_alter ORDER BY d; SHOW CREATE TABLE table_for_alter; +DETACH TABLE table_for_alter; + +ATTACH TABLE table_for_alter; + DROP TABLE IF EXISTS table_for_alter; From 464b8167d1f5d167ff08df1c243ff806a6e68bb1 Mon Sep 17 00:00:00 2001 From: alesapin Date: Thu, 2 Sep 2021 18:29:26 +0300 Subject: [PATCH 16/27] Update comment --- src/Storages/MergeTree/ReplicatedMergeTreeTableMetadata.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Storages/MergeTree/ReplicatedMergeTreeTableMetadata.cpp b/src/Storages/MergeTree/ReplicatedMergeTreeTableMetadata.cpp index be8cf26dee6..0637a6bb027 100644 --- a/src/Storages/MergeTree/ReplicatedMergeTreeTableMetadata.cpp +++ b/src/Storages/MergeTree/ReplicatedMergeTreeTableMetadata.cpp @@ -49,7 +49,7 @@ ReplicatedMergeTreeTableMetadata::ReplicatedMergeTreeTableMetadata(const MergeTr { /// We don't use preparsed AST `sorting_key.expression_list_ast` because /// it contain version column for VersionedCollapsingMergeTree, which - /// shouldn't be defined as a part of key definition AST. So the best + /// is not stored in ZooKeeper for compatibility reasons. So the best /// compatible way is just to convert definition_ast to list and /// serialize it. In all other places key.expression_list_ast should be /// used. From 9bad77f806987b0a1547b6c8e879aaa0055188ca Mon Sep 17 00:00:00 2001 From: alesapin Date: Thu, 2 Sep 2021 19:16:29 +0300 Subject: [PATCH 17/27] Fix clang tidy --- src/Coordination/Changelog.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Coordination/Changelog.cpp b/src/Coordination/Changelog.cpp index 06f692533d4..3be4502d13c 100644 --- a/src/Coordination/Changelog.cpp +++ b/src/Coordination/Changelog.cpp @@ -386,7 +386,7 @@ void Changelog::readChangelogAndInitWriter(uint64_t last_commited_log_index, uin auto description = existing_changelogs[last_log_read_result->log_start_index]; if (last_log_read_result->error) - initWriter(description, last_log_read_result->total_entries_read_from_log, /* truncate_after = */ last_log_read_result->last_position); + initWriter(description, last_log_read_result->total_entries_read_from_log, /* truncate_to_offset = */ last_log_read_result->last_position); else initWriter(description, last_log_read_result->total_entries_read_from_log); } From d571c84a56855f645d0028a20d1f23fc2e264343 Mon Sep 17 00:00:00 2001 From: alesapin Date: Thu, 2 Sep 2021 23:40:52 +0300 Subject: [PATCH 18/27] Better test for flaky check --- .../01526_alter_add_and_modify_order_zookeeper.reference | 4 ++-- .../01526_alter_add_and_modify_order_zookeeper.sql | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/queries/0_stateless/01526_alter_add_and_modify_order_zookeeper.reference b/tests/queries/0_stateless/01526_alter_add_and_modify_order_zookeeper.reference index 4063d93d542..1dcd3543d4e 100644 --- a/tests/queries/0_stateless/01526_alter_add_and_modify_order_zookeeper.reference +++ b/tests/queries/0_stateless/01526_alter_add_and_modify_order_zookeeper.reference @@ -1,6 +1,6 @@ 2019-10-01 a 1 aa 1 1 1 2019-10-01 a 1 aa 1 1 1 0 -CREATE TABLE default.table_for_alter\n(\n `d` Date,\n `a` String,\n `b` UInt8,\n `x` String,\n `y` Int8,\n `version` UInt64,\n `sign` Int8 DEFAULT 1,\n `order` UInt32\n)\nENGINE = ReplicatedVersionedCollapsingMergeTree(\'/clickhouse/tables/01526_alter_add/t1\', \'1\', sign, version)\nPARTITION BY y\nPRIMARY KEY d\nORDER BY (d, order)\nSETTINGS index_granularity = 8192 +CREATE TABLE default.table_for_alter\n(\n `d` Date,\n `a` String,\n `b` UInt8,\n `x` String,\n `y` Int8,\n `version` UInt64,\n `sign` Int8 DEFAULT 1,\n `order` UInt32\n)\nENGINE = ReplicatedVersionedCollapsingMergeTree(\'/clickhouse/tables/default/01526_alter_add/t1\', \'1\', sign, version)\nPARTITION BY y\nPRIMARY KEY d\nORDER BY (d, order)\nSETTINGS index_granularity = 8192 2019-10-01 a 1 aa 1 1 1 0 0 2019-10-02 b 2 bb 2 2 2 1 2 -CREATE TABLE default.table_for_alter\n(\n `d` Date,\n `a` String,\n `b` UInt8,\n `x` String,\n `y` Int8,\n `version` UInt64,\n `sign` Int8 DEFAULT 1,\n `order` UInt32,\n `datum` UInt32\n)\nENGINE = ReplicatedVersionedCollapsingMergeTree(\'/clickhouse/tables/01526_alter_add/t1\', \'1\', sign, version)\nPARTITION BY y\nPRIMARY KEY d\nORDER BY (d, order, datum)\nSETTINGS index_granularity = 8192 +CREATE TABLE default.table_for_alter\n(\n `d` Date,\n `a` String,\n `b` UInt8,\n `x` String,\n `y` Int8,\n `version` UInt64,\n `sign` Int8 DEFAULT 1,\n `order` UInt32,\n `datum` UInt32\n)\nENGINE = ReplicatedVersionedCollapsingMergeTree(\'/clickhouse/tables/default/01526_alter_add/t1\', \'1\', sign, version)\nPARTITION BY y\nPRIMARY KEY d\nORDER BY (d, order, datum)\nSETTINGS index_granularity = 8192 diff --git a/tests/queries/0_stateless/01526_alter_add_and_modify_order_zookeeper.sql b/tests/queries/0_stateless/01526_alter_add_and_modify_order_zookeeper.sql index aaf003c53f4..db8c2de0950 100644 --- a/tests/queries/0_stateless/01526_alter_add_and_modify_order_zookeeper.sql +++ b/tests/queries/0_stateless/01526_alter_add_and_modify_order_zookeeper.sql @@ -12,7 +12,7 @@ CREATE TABLE table_for_alter `version` UInt64, `sign` Int8 DEFAULT 1 ) -ENGINE = ReplicatedVersionedCollapsingMergeTree('/clickhouse/tables/01526_alter_add/t1', '1', sign, version) +ENGINE = ReplicatedVersionedCollapsingMergeTree('/clickhouse/tables/{database}/01526_alter_add/t1', '1', sign, version) PARTITION BY y ORDER BY d SETTINGS index_granularity = 8192; From 0f284db1fc206a52e93398f5c288c2f43a8ae937 Mon Sep 17 00:00:00 2001 From: Nikita Mikhaylov Date: Thu, 2 Sep 2021 22:33:04 +0000 Subject: [PATCH 19/27] Done --- src/Functions/CRC.cpp | 1 + src/IO/ZlibDeflatingWriteBuffer.h | 3 +++ src/IO/ZlibInflatingReadBuffer.h | 1 + src/IO/examples/zlib_ng_bug.cpp | 2 ++ 4 files changed, 7 insertions(+) diff --git a/src/Functions/CRC.cpp b/src/Functions/CRC.cpp index 00aa631c85b..aa162a39f6c 100644 --- a/src/Functions/CRC.cpp +++ b/src/Functions/CRC.cpp @@ -1,3 +1,4 @@ +#define Z_TLS = __thread #include #include #include diff --git a/src/IO/ZlibDeflatingWriteBuffer.h b/src/IO/ZlibDeflatingWriteBuffer.h index 6f623f55f56..a9f87f5cd29 100644 --- a/src/IO/ZlibDeflatingWriteBuffer.h +++ b/src/IO/ZlibDeflatingWriteBuffer.h @@ -4,6 +4,9 @@ #include #include +/// https://github.com/zlib-ng/zlib-ng/pull/733 +/// This is disabed by default +#define Z_TLS = __thread #include diff --git a/src/IO/ZlibInflatingReadBuffer.h b/src/IO/ZlibInflatingReadBuffer.h index b8c141e9b9b..562a9b8c3a8 100644 --- a/src/IO/ZlibInflatingReadBuffer.h +++ b/src/IO/ZlibInflatingReadBuffer.h @@ -4,6 +4,7 @@ #include #include +#define Z_TLS = __thread #include diff --git a/src/IO/examples/zlib_ng_bug.cpp b/src/IO/examples/zlib_ng_bug.cpp index e9b3c448b88..68eb2ce2c77 100644 --- a/src/IO/examples/zlib_ng_bug.cpp +++ b/src/IO/examples/zlib_ng_bug.cpp @@ -1,6 +1,8 @@ #include #include #include + +#define Z_TLS = __thread #include #pragma GCC diagnostic ignored "-Wold-style-cast" From f8020a1f0705a2ee41b9291bd562e81e1d6ba67d Mon Sep 17 00:00:00 2001 From: Nikita Mikhaylov Date: Thu, 2 Sep 2021 22:47:51 +0000 Subject: [PATCH 20/27] Fix build --- src/Functions/CRC.cpp | 1 - 1 file changed, 1 deletion(-) diff --git a/src/Functions/CRC.cpp b/src/Functions/CRC.cpp index aa162a39f6c..00aa631c85b 100644 --- a/src/Functions/CRC.cpp +++ b/src/Functions/CRC.cpp @@ -1,4 +1,3 @@ -#define Z_TLS = __thread #include #include #include From f8e367757d904411c39587fe6dd4750880060227 Mon Sep 17 00:00:00 2001 From: benbiti Date: Fri, 3 Sep 2021 14:24:24 +0800 Subject: [PATCH 21/27] fix max_server_memory_usage_to_ram_ratio default value(0.9) in docs --- docs/en/operations/server-configuration-parameters/settings.md | 2 +- docs/ru/operations/server-configuration-parameters/settings.md | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/en/operations/server-configuration-parameters/settings.md b/docs/en/operations/server-configuration-parameters/settings.md index 66a2bde4b4b..f916126c4c6 100644 --- a/docs/en/operations/server-configuration-parameters/settings.md +++ b/docs/en/operations/server-configuration-parameters/settings.md @@ -535,7 +535,7 @@ Possible values: - Positive double. - 0 — The ClickHouse server can use all available RAM. -Default value: `0`. +Default value: `0.9`. **Usage** diff --git a/docs/ru/operations/server-configuration-parameters/settings.md b/docs/ru/operations/server-configuration-parameters/settings.md index 98c5748ba41..934c659ac2b 100644 --- a/docs/ru/operations/server-configuration-parameters/settings.md +++ b/docs/ru/operations/server-configuration-parameters/settings.md @@ -516,7 +516,7 @@ ClickHouse проверяет условия для `min_part_size` и `min_part - Положительное число с плавающей запятой. - 0 — сервер Clickhouse может использовать всю оперативную память. -Значение по умолчанию: `0`. +Значение по умолчанию: `0.9`. **Использование** From 5a4d554c151e7808e8821b8f782a8c482cdc11b9 Mon Sep 17 00:00:00 2001 From: Azat Khuzhin Date: Fri, 3 Sep 2021 10:34:08 +0300 Subject: [PATCH 22/27] FIx Bzip2WriteBuffer ifdef (USE_BZIP2 over USE_BROTLI) --- src/IO/Bzip2WriteBuffer.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/IO/Bzip2WriteBuffer.cpp b/src/IO/Bzip2WriteBuffer.cpp index 39c5356b792..6e63779d3cf 100644 --- a/src/IO/Bzip2WriteBuffer.cpp +++ b/src/IO/Bzip2WriteBuffer.cpp @@ -2,7 +2,7 @@ # include #endif -#if USE_BROTLI +#if USE_BZIP2 # include # include // Y_IGNORE From 4909d3f2e0160860b293023b2c2d5e257108231a Mon Sep 17 00:00:00 2001 From: Azat Khuzhin Date: Fri, 3 Sep 2021 10:35:11 +0300 Subject: [PATCH 23/27] Fix endless loop for truncated bzip2 Stress test (thread) found an issue [1]: elapsed: 2094.356407533 query: SELECT count(), max(x) FROM file thread_ids: [2927,11494,7502] current_database: test_19 Stacktrace for 7502: 0 __syscall () at ../base/glibc-compatibility/musl/x86_64/syscall.s:14 1 0x000000001ba033b8 in clock_nanosleep (clk=1, flags=1, req=0x7fbb8adaed68, rem=0x0) at ../base/glibc-compatibility/musl/clock_nanosleep.c:21 2 0x0000000018e25b84 in sleepForNanoseconds (nanoseconds=) at ../base/common/sleep.cpp:48 3 0x0000000009a47143 in DB::injection (yield_probability=0, migrate_probability=0, sleep_probability=0.10000000000000001, sleep_time_us=100000) at ../src/Common/ThreadFuzzer.cpp:193 4 DB::ThreadFuzzer::signalHandler () at ../src/Common/ThreadFuzzer.cpp:204 5 0x00000000098e1447 in __tsan::CallUserSignalHandler(__tsan::ThreadState*, bool, bool, bool, int, __sanitizer::__sanitizer_siginfo*, void*) () 6 0x00000000098d7f99 in __tsan::ProcessPendingSignals(__tsan::ThreadState*) () 7 0x00000000098e57de in pread () 8 0x00000000099fab84 in DB::ReadBufferFromFileDescriptor::nextImpl (this=0x7b38010ddb00) at ../src/IO/ReadBufferFromFileDescriptor.cpp:62 9 0x0000000009ab0cf3 in DB::ReadBuffer::next (this=0x7b38010ddb00) at ../src/IO/ReadBuffer.h:62 10 DB::ReadBuffer::nextIfAtEnd (this=0x7b38010ddb00) at ../src/IO/ReadBuffer.h:78 11 DB::Bzip2ReadBuffer::nextImpl (this=0x7b20032f4800) at ../src/IO/Bzip2ReadBuffer.cpp:59 It looks like bz2 is not capable to read/fail partially written archive: 2021.09.03 00:52:36.744818 [ 3553 ] {90419c0a-c410-42ea-ba76-7672ff5ae5ec} executeQuery: (from [::1]:54714) (comment: '/usr/share/clickhouse-test/queries/0_stateless/01059_storage_file_compression.sh') CREATE TABLE file (x UInt64) ENGINE = File(TSV, 'test_19/bz2.tsv.bz2') 2021.09.03 00:52:41.255409 [ 2927 ] {9c7b4e21-cc37-438d-9c13-f9dc01efd30c} executeQuery: (from [::1]:54750) (comment: '/usr/share/clickhouse-test/queries/0_stateless/01059_storage_file_compression.sh') TRUNCATE TABLE file 2021.09.03 00:52:43.762791 [ 2927 ] {82857e3d-f7cc-453d-82a4-e8b8fb82ae0e} executeQuery: (from [::1]:54758) (comment: '/usr/share/clickhouse-test/queries/0_stateless/01059_storage_file_compression.sh') INSERT INTO file SELECT * FROM numbers(1000000) 2021.09.03 00:52:54.939333 [ 2927 ] {82857e3d-f7cc-453d-82a4-e8b8fb82ae0e} executeQuery: Code: 241. DB::Exception: Memory limit (total) exceeded: would use 63.07 GiB (attempt to allocate chunk of 4194364 bytes), maximum: 62.94 GiB. (MEMORY_LIMIT_EXCEEDED) (version 21.10.1.7982) (from [::1]:54758) (comment: '/usr/share/clickhouse-test/queries/0_stateless/01059_storage_file_compression.sh') (in query: INSERT INTO file SELECT * FROM numbers(1000000)), Stack trace (when copying this message, always include the lines below): 2021.09.03 00:53:08.505015 [ 2927 ] {2bc7444e-c0d6-462e-9d6a-567f70b01697} executeQuery: (from [::1]:54842) (comment: '/usr/share/clickhouse-test/queries/0_stateless/01059_storage_file_compression.sh') SELECT count(), max(x) FROM file [1]: https://clickhouse-test-reports.s3.yandex.net/28474/6d5f01a56bb1715c47de8444bfc85b39228f3081/stress_test_(thread).html#fail1 --- src/IO/Bzip2ReadBuffer.cpp | 7 +++++++ .../0_stateless/02022_bzip2_truncated.reference | 0 .../queries/0_stateless/02022_bzip2_truncated.sh | 15 +++++++++++++++ 3 files changed, 22 insertions(+) create mode 100644 tests/queries/0_stateless/02022_bzip2_truncated.reference create mode 100755 tests/queries/0_stateless/02022_bzip2_truncated.sh diff --git a/src/IO/Bzip2ReadBuffer.cpp b/src/IO/Bzip2ReadBuffer.cpp index 99798bca325..098e829188f 100644 --- a/src/IO/Bzip2ReadBuffer.cpp +++ b/src/IO/Bzip2ReadBuffer.cpp @@ -12,6 +12,7 @@ namespace DB namespace ErrorCodes { extern const int BZIP2_STREAM_DECODER_FAILED; + extern const int UNEXPECTED_END_OF_FILE; } @@ -90,6 +91,12 @@ bool Bzip2ReadBuffer::nextImpl() "bzip2 stream decoder failed: error code: {}", ret); + if (in->eof()) + { + eof = true; + throw Exception(ErrorCodes::UNEXPECTED_END_OF_FILE, "Unexpected end of bzip2 archive"); + } + return true; } } diff --git a/tests/queries/0_stateless/02022_bzip2_truncated.reference b/tests/queries/0_stateless/02022_bzip2_truncated.reference new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tests/queries/0_stateless/02022_bzip2_truncated.sh b/tests/queries/0_stateless/02022_bzip2_truncated.sh new file mode 100755 index 00000000000..16d3e22feb9 --- /dev/null +++ b/tests/queries/0_stateless/02022_bzip2_truncated.sh @@ -0,0 +1,15 @@ +#!/usr/bin/env bash + +CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh +. "$CURDIR"/../shell_config.sh + +tmp_path=$(mktemp "$CURDIR/02022_bzip2_truncate.XXXXXX.bz2") +trap 'rm -f $tmp_path' EXIT + +${CLICKHOUSE_LOCAL} -q "SELECT * FROM numbers(1e6) FORMAT TSV" | bzip2 > "$tmp_path" +truncate -s10000 "$tmp_path" +# just ensure that it will exit eventually +${CLICKHOUSE_LOCAL} -q "SELECT count() FROM file('$tmp_path', 'TSV', 'n UInt64') FORMAT Null" >& /dev/null + +exit 0 From 36abf9f2f42947b2e0382a049c3684329df9621d Mon Sep 17 00:00:00 2001 From: alesapin Date: Fri, 3 Sep 2021 10:51:37 +0300 Subject: [PATCH 24/27] Fix broken kafka test --- tests/integration/test_storage_kafka/test.py | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/tests/integration/test_storage_kafka/test.py b/tests/integration/test_storage_kafka/test.py index 84f22d4d201..698d2032d32 100644 --- a/tests/integration/test_storage_kafka/test.py +++ b/tests/integration/test_storage_kafka/test.py @@ -7,6 +7,7 @@ import time import logging import io import string +import ast import avro.schema import avro.io @@ -2792,7 +2793,7 @@ def test_kafka_formats_with_broken_message(kafka_cluster): # broken message "(0,'BAD','AM',0.5,1)", ], - 'expected':r'''{"raw_message":"(0,'BAD','AM',0.5,1)","error":"Cannot parse string 'BAD' as UInt16: syntax error at begin of string. Note: there are toUInt16OrZero and toUInt16OrNull functions, which returns zero\/NULL instead of throwing exception.: while executing 'FUNCTION _CAST(assumeNotNull(_dummy_0) :: 2, 'UInt16' :: 1) -> _CAST(assumeNotNull(_dummy_0), 'UInt16') UInt16 : 4'"}''', + 'expected':r'''{"raw_message":"(0,'BAD','AM',0.5,1)","error":"Cannot parse string 'BAD' as UInt16: syntax error at begin of string. Note: there are toUInt16OrZero and toUInt16OrNull functions, which returns zero\/NULL instead of throwing exception"}''', 'supports_empty_value': True, 'printable':True, }, @@ -2934,11 +2935,13 @@ def test_kafka_formats_with_broken_message(kafka_cluster): '''.format(topic_name=topic_name, offset_0=offsets[0], offset_1=offsets[1], offset_2=offsets[2]) # print(('Checking result\n {result} \n expected \n {expected}\n'.format(result=str(result), expected=str(expected)))) assert TSV(result) == TSV(expected), 'Proper result for format: {}'.format(format_name) - errors_result = instance.query('SELECT raw_message, error FROM test.kafka_errors_{format_name}_mv format JSONEachRow'.format(format_name=format_name)) - errors_expected = format_opts['expected'] + errors_result = ast.literal_eval(instance.query('SELECT raw_message, error FROM test.kafka_errors_{format_name}_mv format JSONEachRow'.format(format_name=format_name))) + errors_expected = ast.literal_eval(format_opts['expected']) # print(errors_result.strip()) # print(errors_expected.strip()) - assert errors_result.strip() == errors_expected.strip(), 'Proper errors for format: {}'.format(format_name) + assert errors_result['raw_message'] == errors_expected['raw_message'], 'Proper raw_message for format: {}'.format(format_name) + # Errors text can change, just checking prefixes + assert errors_expected['error'] in errors_result['error'], 'Proper error for format: {}'.format(format_name) kafka_delete_topic(admin_client, topic_name) def wait_for_new_data(table_name, prev_count = 0, max_retries = 120): From a3f008c626aedfb0fa30e9d5f4a127c76c39931b Mon Sep 17 00:00:00 2001 From: Nikita Mikhaylov Date: Fri, 3 Sep 2021 08:30:03 +0000 Subject: [PATCH 25/27] Enable define from cmake --- cmake/find/zlib.cmake | 3 +++ src/IO/ZlibDeflatingWriteBuffer.h | 4 +--- src/IO/ZlibInflatingReadBuffer.h | 2 +- src/IO/examples/zlib_ng_bug.cpp | 1 - 4 files changed, 5 insertions(+), 5 deletions(-) diff --git a/cmake/find/zlib.cmake b/cmake/find/zlib.cmake index bd96424b60d..f66f9e6713d 100644 --- a/cmake/find/zlib.cmake +++ b/cmake/find/zlib.cmake @@ -29,6 +29,9 @@ if (NOT USE_INTERNAL_ZLIB_LIBRARY) endif () if (NOT ZLIB_FOUND AND NOT MISSING_INTERNAL_ZLIB_LIBRARY) + # https://github.com/zlib-ng/zlib-ng/pull/733 + # This is disabed by default + add_compile_definitions(Z_TLS=__thread) set (USE_INTERNAL_ZLIB_LIBRARY 1) set (ZLIB_INCLUDE_DIR "${ClickHouse_SOURCE_DIR}/contrib/${INTERNAL_ZLIB_NAME}" "${ClickHouse_BINARY_DIR}/contrib/${INTERNAL_ZLIB_NAME}" CACHE INTERNAL "") # generated zconf.h set (ZLIB_INCLUDE_DIRS ${ZLIB_INCLUDE_DIR}) # for poco diff --git a/src/IO/ZlibDeflatingWriteBuffer.h b/src/IO/ZlibDeflatingWriteBuffer.h index a9f87f5cd29..132f529cc54 100644 --- a/src/IO/ZlibDeflatingWriteBuffer.h +++ b/src/IO/ZlibDeflatingWriteBuffer.h @@ -4,9 +4,7 @@ #include #include -/// https://github.com/zlib-ng/zlib-ng/pull/733 -/// This is disabed by default -#define Z_TLS = __thread + #include diff --git a/src/IO/ZlibInflatingReadBuffer.h b/src/IO/ZlibInflatingReadBuffer.h index 562a9b8c3a8..e868dd6999d 100644 --- a/src/IO/ZlibInflatingReadBuffer.h +++ b/src/IO/ZlibInflatingReadBuffer.h @@ -4,7 +4,7 @@ #include #include -#define Z_TLS = __thread + #include diff --git a/src/IO/examples/zlib_ng_bug.cpp b/src/IO/examples/zlib_ng_bug.cpp index 68eb2ce2c77..9fe3c961913 100644 --- a/src/IO/examples/zlib_ng_bug.cpp +++ b/src/IO/examples/zlib_ng_bug.cpp @@ -2,7 +2,6 @@ #include #include -#define Z_TLS = __thread #include #pragma GCC diagnostic ignored "-Wold-style-cast" From fa5db1be76ded3b8563057eb3041aa4da4b73d31 Mon Sep 17 00:00:00 2001 From: benbiti Date: Fri, 3 Sep 2021 17:32:31 +0800 Subject: [PATCH 26/27] fix keep_alive_timeout default value in docs --- .../en/operations/server-configuration-parameters/settings.md | 4 ++-- .../ja/operations/server-configuration-parameters/settings.md | 4 ++-- .../ru/operations/server-configuration-parameters/settings.md | 4 ++-- .../zh/operations/server-configuration-parameters/settings.md | 4 ++-- 4 files changed, 8 insertions(+), 8 deletions(-) diff --git a/docs/en/operations/server-configuration-parameters/settings.md b/docs/en/operations/server-configuration-parameters/settings.md index f916126c4c6..68937c870a9 100644 --- a/docs/en/operations/server-configuration-parameters/settings.md +++ b/docs/en/operations/server-configuration-parameters/settings.md @@ -390,12 +390,12 @@ This section contains the following parameters: ## keep_alive_timeout {#keep-alive-timeout} -The number of seconds that ClickHouse waits for incoming requests before closing the connection. Defaults to 3 seconds. +The number of seconds that ClickHouse waits for incoming requests before closing the connection. Defaults to 10 seconds. **Example** ``` xml -3 +10 ``` ## listen_host {#server_configuration_parameters-listen_host} diff --git a/docs/ja/operations/server-configuration-parameters/settings.md b/docs/ja/operations/server-configuration-parameters/settings.md index f544a92e377..9f67c7e5f38 100644 --- a/docs/ja/operations/server-configuration-parameters/settings.md +++ b/docs/ja/operations/server-configuration-parameters/settings.md @@ -284,12 +284,12 @@ ClickHouseサーバー間でデータを交換するポート。 ## keep_alive_timeout {#keep-alive-timeout} -ClickHouseが接続を閉じる前に受信要求を待機する秒数。 既定値は3秒です。 +ClickHouseが接続を閉じる前に受信要求を待機する秒数。 既定値は10秒です。 **例** ``` xml -3 +10 ``` ## listen_host {#server_configuration_parameters-listen_host} diff --git a/docs/ru/operations/server-configuration-parameters/settings.md b/docs/ru/operations/server-configuration-parameters/settings.md index 934c659ac2b..8bd364e880c 100644 --- a/docs/ru/operations/server-configuration-parameters/settings.md +++ b/docs/ru/operations/server-configuration-parameters/settings.md @@ -371,12 +371,12 @@ ClickHouse проверяет условия для `min_part_size` и `min_part ## keep_alive_timeout {#keep-alive-timeout} -Время в секундах, в течение которого ClickHouse ожидает входящих запросов прежде, чем закрыть соединение. +Время в секундах, в течение которого ClickHouse ожидает входящих запросов прежде, чем 10акрыть соединение. **Пример** ``` xml -3 +10 ``` ## listen_host {#server_configuration_parameters-listen_host} diff --git a/docs/zh/operations/server-configuration-parameters/settings.md b/docs/zh/operations/server-configuration-parameters/settings.md index 0c6a5046877..84d4f601fb0 100644 --- a/docs/zh/operations/server-configuration-parameters/settings.md +++ b/docs/zh/operations/server-configuration-parameters/settings.md @@ -282,12 +282,12 @@ ClickHouse每x秒重新加载内置字典。 这使得编辑字典 “on the fly ## keep_alive_timeout {#keep-alive-timeout} -ClickHouse在关闭连接之前等待传入请求的秒数。 默认为3秒。 +ClickHouse在关闭连接之前等待传入请求的秒数。默认为10秒。 **示例** ``` xml -3 +10 ``` ## listen_host {#server_configuration_parameters-listen_host} From 1d9ad322ae758effb9f246e3c3d40e2dd460c5e2 Mon Sep 17 00:00:00 2001 From: Pavel Kruglov Date: Fri, 3 Sep 2021 12:43:56 +0300 Subject: [PATCH 27/27] Fix format names in docs --- docs/en/interfaces/formats.md | 20 ++++++++++---------- docs/ru/interfaces/formats.md | 30 +++++++++++++++--------------- docs/zh/interfaces/formats.md | 26 +++++++++++++------------- 3 files changed, 38 insertions(+), 38 deletions(-) diff --git a/docs/en/interfaces/formats.md b/docs/en/interfaces/formats.md index 015afd1cd24..bac67ecf140 100644 --- a/docs/en/interfaces/formats.md +++ b/docs/en/interfaces/formats.md @@ -26,17 +26,17 @@ The supported formats are: | [VerticalRaw](#verticalraw) | ✗ | ✔ | | [JSON](#json) | ✗ | ✔ | | [JSONAsString](#jsonasstring) | ✔ | ✗ | -| [JSONString](#jsonstring) | ✗ | ✔ | +| [JSONStrings](#jsonstrings) | ✗ | ✔ | | [JSONCompact](#jsoncompact) | ✗ | ✔ | -| [JSONCompactString](#jsoncompactstring) | ✗ | ✔ | +| [JSONCompactStrings](#jsoncompactstrings) | ✗ | ✔ | | [JSONEachRow](#jsoneachrow) | ✔ | ✔ | | [JSONEachRowWithProgress](#jsoneachrowwithprogress) | ✗ | ✔ | | [JSONStringsEachRow](#jsonstringseachrow) | ✔ | ✔ | | [JSONStringsEachRowWithProgress](#jsonstringseachrowwithprogress) | ✗ | ✔ | | [JSONCompactEachRow](#jsoncompacteachrow) | ✔ | ✔ | | [JSONCompactEachRowWithNamesAndTypes](#jsoncompacteachrowwithnamesandtypes) | ✔ | ✔ | -| [JSONCompactStringEachRow](#jsoncompactstringeachrow) | ✔ | ✔ | -| [JSONCompactStringEachRowWithNamesAndTypes](#jsoncompactstringeachrowwithnamesandtypes) | ✔ | ✔ | +| [JSONCompactStringsEachRow](#jsoncompactstringseachrow) | ✔ | ✔ | +| [JSONCompactStringsEachRowWithNamesAndTypes](#jsoncompactstringseachrowwithnamesandtypes) | ✔ | ✔ | | [TSKV](#tskv) | ✔ | ✔ | | [Pretty](#pretty) | ✗ | ✔ | | [PrettyCompact](#prettycompact) | ✗ | ✔ | @@ -464,7 +464,7 @@ ClickHouse supports [NULL](../sql-reference/syntax.md), which is displayed as `n - [JSONEachRow](#jsoneachrow) format - [output_format_json_array_of_rows](../operations/settings/settings.md#output-format-json-array-of-rows) setting -## JSONString {#jsonstring} +## JSONStrings {#jsonstrings} Differs from JSON only in that data fields are output in strings, not in typed JSON values. @@ -541,7 +541,7 @@ Result: ``` ## JSONCompact {#jsoncompact} -## JSONCompactString {#jsoncompactstring} +## JSONCompactStrings {#jsoncompactstrings} Differs from JSON only in that data rows are output in arrays, not in objects. @@ -580,7 +580,7 @@ Example: ``` ``` -// JSONCompactString +// JSONCompactStrings { "meta": [ @@ -614,7 +614,7 @@ Example: ## JSONEachRow {#jsoneachrow} ## JSONStringsEachRow {#jsonstringseachrow} ## JSONCompactEachRow {#jsoncompacteachrow} -## JSONCompactStringEachRow {#jsoncompactstringeachrow} +## JSONCompactStringsEachRow {#jsoncompactstringseachrow} When using these formats, ClickHouse outputs rows as separated, newline-delimited JSON values, but the data as a whole is not valid JSON. @@ -639,9 +639,9 @@ Differs from `JSONEachRow`/`JSONStringsEachRow` in that ClickHouse will also yie ``` ## JSONCompactEachRowWithNamesAndTypes {#jsoncompacteachrowwithnamesandtypes} -## JSONCompactStringEachRowWithNamesAndTypes {#jsoncompactstringeachrowwithnamesandtypes} +## JSONCompactStringsEachRowWithNamesAndTypes {#jsoncompactstringseachrowwithnamesandtypes} -Differs from `JSONCompactEachRow`/`JSONCompactStringEachRow` in that the column names and types are written as the first two rows. +Differs from `JSONCompactEachRow`/`JSONCompactStringsEachRow` in that the column names and types are written as the first two rows. ```json ["'hello'", "multiply(42, number)", "range(5)"] diff --git a/docs/ru/interfaces/formats.md b/docs/ru/interfaces/formats.md index 563a137ac17..1dd10e1f76d 100644 --- a/docs/ru/interfaces/formats.md +++ b/docs/ru/interfaces/formats.md @@ -25,17 +25,17 @@ ClickHouse может принимать (`INSERT`) и отдавать (`SELECT | [VerticalRaw](#verticalraw) | ✗ | ✔ | | [JSON](#json) | ✗ | ✔ | | [JSONAsString](#jsonasstring) | ✔ | ✗ | -| [JSONString](#jsonstring) | ✗ | ✔ | +| [JSONStrings](#jsonstrings) | ✗ | ✔ | | [JSONCompact](#jsoncompact) | ✗ | ✔ | -| [JSONCompactString](#jsoncompactstring) | ✗ | ✔ | +| [JSONCompactStrings](#jsoncompactstrings) | ✗ | ✔ | | [JSONEachRow](#jsoneachrow) | ✔ | ✔ | | [JSONEachRowWithProgress](#jsoneachrowwithprogress) | ✗ | ✔ | -| [JSONStringEachRow](#jsonstringeachrow) | ✔ | ✔ | -| [JSONStringEachRowWithProgress](#jsonstringeachrowwithprogress) | ✗ | ✔ | +| [JSONStringsEachRow](#jsonstringseachrow) | ✔ | ✔ | +| [JSONStringsEachRowWithProgress](#jsonstringseachrowwithprogress) | ✗ | ✔ | | [JSONCompactEachRow](#jsoncompacteachrow) | ✔ | ✔ | | [JSONCompactEachRowWithNamesAndTypes](#jsoncompacteachrowwithnamesandtypes) | ✔ | ✔ | -| [JSONCompactStringEachRow](#jsoncompactstringeachrow) | ✔ | ✔ | -| [JSONCompactStringEachRowWithNamesAndTypes](#jsoncompactstringeachrowwithnamesandtypes) | ✔ | ✔ | +| [JSONCompactStringsEachRow](#jsoncompactstringseachrow) | ✔ | ✔ | +| [JSONCompactStringsEachRowWithNamesAndTypes](#jsoncompactstringseachrowwithnamesandtypes) | ✔ | ✔ | | [TSKV](#tskv) | ✔ | ✔ | | [Pretty](#pretty) | ✗ | ✔ | | [PrettyCompact](#prettycompact) | ✗ | ✔ | @@ -442,7 +442,7 @@ ClickHouse поддерживает [NULL](../sql-reference/syntax.md), кото - Формат [JSONEachRow](#jsoneachrow) - Настройка [output_format_json_array_of_rows](../operations/settings/settings.md#output-format-json-array-of-rows) -## JSONString {#jsonstring} +## JSONStrings {#jsonstrings} Отличается от JSON только тем, что поля данных выводятся в строках, а не в типизированных значениях JSON. @@ -519,7 +519,7 @@ SELECT * FROM json_as_string; ``` ## JSONCompact {#jsoncompact} -## JSONCompactString {#jsoncompactstring} +## JSONCompactStrings {#jsoncompactstrings} Отличается от JSON только тем, что строчки данных выводятся в массивах, а не в object-ах. @@ -558,7 +558,7 @@ SELECT * FROM json_as_string; ``` ```json -// JSONCompactString +// JSONCompactStrings { "meta": [ @@ -590,9 +590,9 @@ SELECT * FROM json_as_string; ``` ## JSONEachRow {#jsoneachrow} -## JSONStringEachRow {#jsonstringeachrow} +## JSONStringsEachRow {#jsonstringseachrow} ## JSONCompactEachRow {#jsoncompacteachrow} -## JSONCompactStringEachRow {#jsoncompactstringeachrow} +## JSONCompactStringsEachRow {#jsoncompactstringseachrow} При использовании этих форматов ClickHouse выводит каждую запись как значения JSON (каждое значение отдельной строкой), при этом данные в целом — невалидный JSON. @@ -605,9 +605,9 @@ SELECT * FROM json_as_string; При вставке данных вы должны предоставить отдельное значение JSON для каждой строки. ## JSONEachRowWithProgress {#jsoneachrowwithprogress} -## JSONStringEachRowWithProgress {#jsonstringeachrowwithprogress} +## JSONStringsEachRowWithProgress {#jsonstringseachrowwithprogress} -Отличается от `JSONEachRow`/`JSONStringEachRow` тем, что ClickHouse будет выдавать информацию о ходе выполнения в виде значений JSON. +Отличается от `JSONEachRow`/`JSONStringsEachRow` тем, что ClickHouse будет выдавать информацию о ходе выполнения в виде значений JSON. ```json {"row":{"'hello'":"hello","multiply(42, number)":"0","range(5)":[0,1,2,3,4]}} @@ -617,9 +617,9 @@ SELECT * FROM json_as_string; ``` ## JSONCompactEachRowWithNamesAndTypes {#jsoncompacteachrowwithnamesandtypes} -## JSONCompactStringEachRowWithNamesAndTypes {#jsoncompactstringeachrowwithnamesandtypes} +## JSONCompactStringsEachRowWithNamesAndTypes {#jsoncompactstringseachrowwithnamesandtypes} -Отличается от `JSONCompactEachRow`/`JSONCompactStringEachRow` тем, что имена и типы столбцов записываются как первые две строки. +Отличается от `JSONCompactEachRow`/`JSONCompactStringsEachRow` тем, что имена и типы столбцов записываются как первые две строки. ```json ["'hello'", "multiply(42, number)", "range(5)"] diff --git a/docs/zh/interfaces/formats.md b/docs/zh/interfaces/formats.md index 1cd91690e57..6f95c287068 100644 --- a/docs/zh/interfaces/formats.md +++ b/docs/zh/interfaces/formats.md @@ -26,17 +26,17 @@ ClickHouse可以接受和返回各种格式的数据。受支持的输入格式 | [VerticalRaw](#verticalraw) | ✗ | ✔ | | [JSON](#json) | ✗ | ✔ | | [JSONAsString](#jsonasstring) | ✔ | ✗ | -| [JSONString](#jsonstring) | ✗ | ✔ | +| [JSONStrings](#jsonstrings) | ✗ | ✔ | | [JSONCompact](#jsoncompact) | ✗ | ✔ | -| [JSONCompactString](#jsoncompactstring) | ✗ | ✔ | +| [JSONCompactStrings](#jsoncompactstrings) | ✗ | ✔ | | [JSONEachRow](#jsoneachrow) | ✔ | ✔ | | [JSONEachRowWithProgress](#jsoneachrowwithprogress) | ✗ | ✔ | | [JSONStringsEachRow](#jsonstringseachrow) | ✔ | ✔ | | [JSONStringsEachRowWithProgress](#jsonstringseachrowwithprogress) | ✗ | ✔ | | [JSONCompactEachRow](#jsoncompacteachrow) | ✔ | ✔ | | [JSONCompactEachRowWithNamesAndTypes](#jsoncompacteachrowwithnamesandtypes) | ✔ | ✔ | -| [JSONCompactStringEachRow](#jsoncompactstringeachrow) | ✔ | ✔ | -| [JSONCompactStringEachRowWithNamesAndTypes](#jsoncompactstringeachrowwithnamesandtypes) | ✔ | ✔ | +| [JSONCompactStringsEachRow](#jsoncompactstringseachrow) | ✔ | ✔ | +| [JSONCompactStringsEachRowWithNamesAndTypes](#jsoncompactstringseachrowwithnamesandtypes) | ✔ | ✔ | | [TSKV](#tskv) | ✔ | ✔ | | [Pretty](#pretty) | ✗ | ✔ | | [PrettyCompact](#prettycompact) | ✗ | ✔ | @@ -465,7 +465,7 @@ ClickHouse支持[NULL](../sql-reference/syntax.md), 在JSON输出中显示为`nu - [JSONEachRow](#jsoneachrow)格式 - [output_format_json_array_of_rows](../operations/settings/settings.md#output-format-json-array-of-rows)设置 -## JSONString {#jsonstring} +## JSONStrings {#jsonstrings} 与JSON的不同之处在于数据字段以字符串输出，而不是以类型化JSON值输出。 @@ -543,7 +543,7 @@ SELECT * FROM json_as_string; ``` ## JSONCompact {#jsoncompact} -## JSONCompactString {#jsoncompactstring} +## JSONCompactStrings {#jsoncompactstrings} 与JSON格式不同的是它以数组的方式输出结果，而不是以结构体。 @@ -582,7 +582,7 @@ SELECT * FROM json_as_string; ``` ```json -// JSONCompactString +// JSONCompactStrings { "meta": [ @@ -614,9 +614,9 @@ SELECT * FROM json_as_string; ``` ## JSONEachRow {#jsoneachrow} -## JSONStringEachRow {#jsonstringeachrow} +## JSONStringsEachRow {#jsonstringseachrow} ## JSONCompactEachRow {#jsoncompacteachrow} -## JSONCompactStringEachRow {#jsoncompactstringeachrow} +## JSONCompactStringsEachRow {#jsoncompactstringseachrow} 使用这些格式时，ClickHouse会将行输出为用换行符分隔的JSON值，这些输出数据作为一个整体时，由于没有分隔符(,)因而不是有效的JSON文档。 @@ -629,9 +629,9 @@ SELECT * FROM json_as_string; 在插入数据时，应该为每一行提供一个单独的JSON值。 ## JSONEachRowWithProgress {#jsoneachrowwithprogress} -## JSONStringEachRowWithProgress {#jsonstringeachrowwithprogress} +## JSONStringsEachRowWithProgress {#jsonstringseachrowwithprogress} -与`JSONEachRow`/`JSONStringEachRow`不同的是，ClickHouse还将生成作为JSON值的进度信息。 +与`JSONEachRow`/`JSONStringsEachRow`不同的是，ClickHouse还将生成作为JSON值的进度信息。 ```json {"row":{"'hello'":"hello","multiply(42, number)":"0","range(5)":[0,1,2,3,4]}} @@ -641,9 +641,9 @@ SELECT * FROM json_as_string; ``` ## JSONCompactEachRowWithNamesAndTypes {#jsoncompacteachrowwithnamesandtypes} -## JSONCompactStringEachRowWithNamesAndTypes {#jsoncompactstringeachrowwithnamesandtypes} +## JSONCompactStringsEachRowWithNamesAndTypes {#jsoncompactstringseachrowwithnamesandtypes} -与`JSONCompactEachRow`/`JSONCompactStringEachRow`不同的是，列名和类型被写入前两行。 +与`JSONCompactEachRow`/`JSONCompactStringsEachRow`不同的是，列名和类型被写入前两行。 ```json ["'hello'", "multiply(42, number)", "range(5)"]