From bec92772d1541a0b3cff2cf41ebdcdfe78b01d08 Mon Sep 17 00:00:00 2001 From: helifu Date: Wed, 24 May 2023 12:00:36 +0800 Subject: [PATCH 1/2] Don't call tryRemove after session expired Too many worthless error logs after session expiration: 2023.05.23 17:43:27.230094 [ 15339 ] {} zkutil::EphemeralNodeHolder::~EphemeralNodeHolder(): Cannot remove /clickhouse/tables/700237742/c72b26392ff013a2_local/01/replicas/saas1/is_active: Code: 999. Coordination::Exception: Session expired, path: Session expired. (KEEPER_EXCEPTION), Stack trace (when copying this message, always include the lines below): 0. ./build/src/./src/Common/Exception.cpp:91: DB::Exception::Exception(DB::Exception::MessageMasked&&, int, bool) @ 0xe1db255 in /home/clickhouse/clickhouse-install/bin/clickhouse_23.3.2.37 1. ./build/src/Common/ZooKeeper/./contrib/llvm-project/libcxx/include/string:1499: Coordination::Exception::Exception(String const&, Coordination::Error, int) @ 0x14f7d190 in /home/clickhouse/clickhouse-install/bin/clickhouse_23.3.2.37 2. ./build/src/Common/ZooKeeper/./contrib/llvm-project/libcxx/include/string:1499: Coordination::Exception::Exception(Coordination::Error, String const&) @ 0x14f7d8d7 in /home/clickhouse/clickhouse-install/bin/clickhouse_23.3.2.37 3. ./build/src/Common/ZooKeeper/./src/Common/ZooKeeper/ZooKeeperImpl.cpp:0: Coordination::ZooKeeper::pushRequest(Coordination::ZooKeeper::RequestInfo&&) @ 0x14fd0e53 in /home/clickhouse/clickhouse-install/bin/clickhouse_23.3.2.37 4. ./build/src/Common/ZooKeeper/./src/Common/CurrentThread.cpp:50: Coordination::ZooKeeper::remove(String const&, int, std::function) @ 0x14fd220e in /home/clickhouse/clickhouse-install/bin/clickhouse_23.3.2.37 5. ./build/src/Common/ZooKeeper/./contrib/llvm-project/libcxx/include/__functional/function.h:818: zkutil::ZooKeeper::asyncTryRemoveNoThrow(String const&, int) @ 0x14f85c1b in /home/clickhouse/clickhouse-install/bin/clickhouse_23.3.2.37 6. ./build/src/Common/ZooKeeper/./contrib/llvm-project/libcxx/include/__chrono/duration.h:258: zkutil::ZooKeeper::removeImpl(String const&, int) @ 0x14f85825 in /home/clickhouse/clickhouse-install/bin/clickhouse_23.3.2.37 7. ./build/programs/copier/./src/Common/ZooKeeper/ZooKeeper.cpp:0: zkutil::EphemeralNodeHolder::~EphemeralNodeHolder() @ 0xe38bd03 in /home/clickhouse/clickhouse-install/bin/clickhouse_23.3.2.37 8. ./build/src/./contrib/llvm-project/libcxx/src/include/atomic_support.h:74: DB::StorageReplicatedMergeTree::partialShutdown() @ 0x140cd578 in /home/clickhouse/clickhouse-install/bin/clickhouse_23.3.2.37 9. ./build/src/./src/Storages/MergeTree/ReplicatedMergeTreeRestartingThread.cpp:119: DB::ReplicatedMergeTreeRestartingThread::runImpl() @ 0x147f5c18 in /home/clickhouse/clickhouse-install/bin/clickhouse_23.3.2.37 10. ./build/src/./src/Storages/MergeTree/ReplicatedMergeTreeRestartingThread.cpp:58: DB::ReplicatedMergeTreeRestartingThread::run() @ 0x147f4b1a in /home/clickhouse/clickhouse-install/bin/clickhouse_23.3.2.37 11. ./build/src/./base/glibc-compatibility/musl/clock_gettime.c:62: DB::BackgroundSchedulePoolTaskInfo::execute() @ 0x127b9ae6 in /home/clickhouse/clickhouse-install/bin/clickhouse_23.3.2.37 12. ./build/src/./contrib/llvm-project/libcxx/include/__memory/shared_ptr.h:701: DB::BackgroundSchedulePool::threadFunction() @ 0x127bc9ca in /home/clickhouse/clickhouse-install/bin/clickhouse_23.3.2.37 13. ./build/src/./src/Core/BackgroundSchedulePool.cpp:0: void std::__function::__policy_invoker::__call_impl::ThreadFromGlobalPoolImpl, StrongTypedef, char const*)::$_1>(DB::BackgroundSchedulePool::BackgroundSchedulePool(unsigned long, StrongTypedef, StrongTypedef, char const*)::$_1&&)::'lambda'(), void ()>>(std::__function::__policy_storage const*) @ 0x127bd80e in /home/clickhouse/clickhouse-install/bin/clickhouse_23.3.2.37 14. ./build/src/./base/base/../base/wide_integer_impl.h:789: ThreadPoolImpl::worker(std::__list_iterator) @ 0xe2aa693 in /home/clickhouse/clickhouse-install/bin/clickhouse_23.3.2.37 15. ./build/src/./contrib/llvm-project/libcxx/include/__memory/unique_ptr.h:302: void* std::__thread_proxy[abi:v15000]>, void ThreadPoolImpl::scheduleImpl(std::function, long, std::optional, bool)::'lambda0'()>>(void*) @ 0xe2b0301 in /home/clickhouse/clickhouse-install/bin/clickhouse_23.3.2.37 16. start_thread @ 0x7ea5 in /usr/lib64/libpthread-2.17.so 17. __clone @ 0xfeb0d in /usr/lib64/libc-2.17.so (version 23.3.2.37) --- src/Common/ZooKeeper/ZooKeeper.h | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/Common/ZooKeeper/ZooKeeper.h b/src/Common/ZooKeeper/ZooKeeper.h index d48ca0a4ef5..9ea7346a005 100644 --- a/src/Common/ZooKeeper/ZooKeeper.h +++ b/src/Common/ZooKeeper/ZooKeeper.h @@ -650,7 +650,8 @@ public: return; try { - zookeeper.tryRemove(path); + if (!zookeeper.expired()) + zookeeper.tryRemove(path); } catch (...) { From ebcc451afae56de7eef05c308fd8754f8b25cdfa Mon Sep 17 00:00:00 2001 From: helifu Date: Fri, 9 Jun 2023 17:24:20 +0800 Subject: [PATCH 2/2] Add a log message and increase profile event --- src/Common/ZooKeeper/ZooKeeper.cpp | 1 - src/Common/ZooKeeper/ZooKeeper.h | 6 ++++++ 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/src/Common/ZooKeeper/ZooKeeper.cpp b/src/Common/ZooKeeper/ZooKeeper.cpp index 62807fe2433..e87db2bec06 100644 --- a/src/Common/ZooKeeper/ZooKeeper.cpp +++ b/src/Common/ZooKeeper/ZooKeeper.cpp @@ -15,7 +15,6 @@ #include "Common/ZooKeeper/IKeeper.h" #include #include -#include #include #include diff --git a/src/Common/ZooKeeper/ZooKeeper.h b/src/Common/ZooKeeper/ZooKeeper.h index 9ea7346a005..e0538adeab8 100644 --- a/src/Common/ZooKeeper/ZooKeeper.h +++ b/src/Common/ZooKeeper/ZooKeeper.h @@ -7,6 +7,7 @@ #include #include #include +#include #include #include #include @@ -652,6 +653,11 @@ public: { if (!zookeeper.expired()) zookeeper.tryRemove(path); + else + { + ProfileEvents::increment(ProfileEvents::CannotRemoveEphemeralNode); + LOG_INFO(&Poco::Logger::get("EphemeralNodeHolder"), "Cannot remove {} since session has been expired", path); + } } catch (...) {