thevar1able/ClickHouse
1
0
Fork 0
mirror of https://github.com/ClickHouse/ClickHouse.git synced 2024-09-20 00:30:49 +00:00
Code Issues Packages Projects Releases Wiki Activity

Better Readonly metric

Browse Source
This commit is contained in:
Antonio Andelic 2023-11-27 14:25:45 +00:00
parent b10e46b2bc
commit 9077550bb1
4 changed files with 34 additions and 26 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

17
src/Storages/MergeTree/ReplicatedMergeTreeAttachThread.cpp
View File

@ -2,6 +2,11 @@
#include <Storages/StorageReplicatedMergeTree.h>
#include <Common/ZooKeeper/IKeeper.h>
namespace CurrentMetrics
{
extern const Metric ReadonlyReplica;
}
namespace DB
{
@ -67,6 +72,12 @@ void ReplicatedMergeTreeAttachThread::run()
LOG_ERROR(log, "Initialization failed, table will remain readonly. Error: {}", getCurrentExceptionMessage(/* with_stacktrace */ true));
storage.initialization_done = true;
}
if (!storage.is_readonly_metric_set)
{
storage.is_readonly_metric_set = true;
CurrentMetrics::add(CurrentMetrics::ReadonlyReplica);
}
}
if (!first_try_done.exchange(true))
@ -74,6 +85,12 @@ void ReplicatedMergeTreeAttachThread::run()
if (shutdown_called)
{
if (storage.is_readonly_metric_set)
{
storage.is_readonly_metric_set = false;
CurrentMetrics::sub(CurrentMetrics::ReadonlyReplica);
}
LOG_WARNING(log, "Shutdown called, cancelling initialization");
return;
}

27
src/Storages/MergeTree/ReplicatedMergeTreeRestartingThread.cpp
View File

@ -79,8 +79,9 @@ void ReplicatedMergeTreeRestartingThread::run()
if (first_time)
{
if (storage.is_readonly)
if (storage.is_readonly && !storage.is_readonly_metric_set)
{
storage.is_readonly_metric_set = true;
/// We failed to start replication, table is still readonly, so we should increment the metric. See also setNotReadonly().
CurrentMetrics::add(CurrentMetrics::ReadonlyReplica);
}
@ -360,21 +361,20 @@ void ReplicatedMergeTreeRestartingThread::setReadonly(bool on_shutdown)
return;
if (became_readonly)
CurrentMetrics::add(CurrentMetrics::ReadonlyReplica);
/// Replica was already readonly, but we should decrement the metric, because we are detaching/dropping table.
/// if first pass wasn't done we don't have to decrement because it wasn't incremented in the first place
/// the task should be deactivated if it's full shutdown so no race is present
if (!first_time && on_shutdown)
{
CurrentMetrics::sub(CurrentMetrics::ReadonlyReplica);
assert(CurrentMetrics::get(CurrentMetrics::ReadonlyReplica) >= 0);
storage.is_readonly_metric_set = true;
CurrentMetrics::add(CurrentMetrics::ReadonlyReplica);
return;
}
if (storage.since_metadata_err_incr_readonly_metric)
/// Replica was already readonly, but we should decrement the metric if it was set because we are detaching/dropping table.
/// the task should be deactivated if it's full shutdown so no race is present
chassert(on_shutdown);
if (storage.is_readonly_metric_set)
{
storage.is_readonly_metric_set = false;
CurrentMetrics::sub(CurrentMetrics::ReadonlyReplica);
assert(CurrentMetrics::get(CurrentMetrics::ReadonlyReplica) >= 0);
chassert(CurrentMetrics::get(CurrentMetrics::ReadonlyReplica) >= 0);
}
}
@ -384,10 +384,11 @@ void ReplicatedMergeTreeRestartingThread::setNotReadonly()
/// is_readonly is true on startup, but ReadonlyReplica metric is not incremented,
/// because we don't want to change this metric if replication is started successfully.
/// So we should not decrement it when replica stopped being readonly on startup.
if (storage.is_readonly.compare_exchange_strong(old_val, false) && !first_time)
if (storage.is_readonly.compare_exchange_strong(old_val, false) && storage.is_readonly_metric_set)
{
storage.is_readonly_metric_set = false;
CurrentMetrics::sub(CurrentMetrics::ReadonlyReplica);
assert(CurrentMetrics::get(CurrentMetrics::ReadonlyReplica) >= 0);
chassert(CurrentMetrics::get(CurrentMetrics::ReadonlyReplica) >= 0);
}
}

11
src/Storages/StorageReplicatedMergeTree.cpp
View File

@ -4991,9 +4991,9 @@ void StorageReplicatedMergeTree::startupImpl(bool from_attach_thread)
/// Do not start replication if ZooKeeper is not configured or there is no metadata in zookeeper
if (!has_metadata_in_zookeeper.has_value() || !*has_metadata_in_zookeeper)
{
if (!since_metadata_err_incr_readonly_metric)
if (!is_readonly_metric_set)
{
since_metadata_err_incr_readonly_metric = true;
is_readonly_metric_set = true;
CurrentMetrics::add(CurrentMetrics::ReadonlyReplica);
}
@ -5001,13 +5001,6 @@ void StorageReplicatedMergeTree::startupImpl(bool from_attach_thread)
return;
}
if (since_metadata_err_incr_readonly_metric)
{
since_metadata_err_incr_readonly_metric = false;
CurrentMetrics::sub(CurrentMetrics::ReadonlyReplica);
assert(CurrentMetrics::get(CurrentMetrics::ReadonlyReplica) >= 0);
}
try
{
auto zookeeper = getZooKeeper();

5
src/Storages/StorageReplicatedMergeTree.h
View File

@ -428,10 +428,7 @@ private:
/// If false - ZooKeeper is available, but there is no table metadata. It's safe to drop table in this case.
std::optional<bool> has_metadata_in_zookeeper;
/// during server restart or attach table process, set since_metadata_err_incr_readonly_metric = true and increase readonly metric if has_metadata_in_zookeeper = false.
/// during detach or drop table process, decrease readonly metric if since_metadata_err_incr_readonly_metric = true.
/// during restore replica process, set since_metadata_err_incr_readonly_metric = false and decrease readonly metric if since_metadata_err_incr_readonly_metric = true.
bool since_metadata_err_incr_readonly_metric = false;
bool is_readonly_metric_set = false;
static const String default_zookeeper_name;
const String zookeeper_name;