2021-01-19 14:22:28 +00:00
|
|
|
#pragma once
|
|
|
|
|
2023-03-24 14:37:40 +00:00
|
|
|
#include "Common/ZooKeeper/ZooKeeperCommon.h"
|
2022-09-28 13:29:29 +00:00
|
|
|
#include "config.h"
|
2021-02-01 13:18:17 +00:00
|
|
|
|
|
|
|
#if USE_NURAFT
|
|
|
|
|
2021-01-19 14:22:28 +00:00
|
|
|
#include <Common/ThreadPool.h>
|
|
|
|
#include <Common/ConcurrentBoundedQueue.h>
|
2021-01-22 16:04:57 +00:00
|
|
|
#include <Poco/Util/AbstractConfiguration.h>
|
2021-02-01 13:18:17 +00:00
|
|
|
#include <Common/Exception.h>
|
|
|
|
#include <functional>
|
2021-03-29 08:24:56 +00:00
|
|
|
#include <Coordination/KeeperServer.h>
|
2021-02-09 14:47:18 +00:00
|
|
|
#include <Coordination/CoordinationSettings.h>
|
2021-11-19 09:30:58 +00:00
|
|
|
#include <Coordination/Keeper4LWInfo.h>
|
2021-11-18 20:17:22 +00:00
|
|
|
#include <Coordination/KeeperConnectionStats.h>
|
2022-09-21 11:53:54 +00:00
|
|
|
#include <Coordination/KeeperSnapshotManagerS3.h>
|
2022-12-23 14:09:24 +00:00
|
|
|
#include <Common/MultiVersion.h>
|
|
|
|
#include <Common/Macros.h>
|
2021-01-19 14:22:28 +00:00
|
|
|
|
2021-01-21 15:09:48 +00:00
|
|
|
namespace DB
|
2021-01-19 14:22:28 +00:00
|
|
|
{
|
|
|
|
using ZooKeeperResponseCallback = std::function<void(const Coordination::ZooKeeperResponsePtr & response)>;
|
|
|
|
|
2021-08-24 12:30:31 +00:00
|
|
|
/// Highlevel wrapper for ClickHouse Keeper.
|
|
|
|
/// Process user requests via consensus and return responses.
|
2021-11-18 20:17:22 +00:00
|
|
|
class KeeperDispatcher
|
2021-01-19 14:22:28 +00:00
|
|
|
{
|
|
|
|
private:
|
2021-03-29 08:24:56 +00:00
|
|
|
using RequestsQueue = ConcurrentBoundedQueue<KeeperStorage::RequestForSession>;
|
2021-03-05 10:40:24 +00:00
|
|
|
using SessionToResponseCallback = std::unordered_map<int64_t, ZooKeeperResponseCallback>;
|
2023-04-20 13:26:02 +00:00
|
|
|
using ClusterUpdateQueue = ConcurrentBoundedQueue<ClusterUpdateAction>;
|
2021-03-05 10:40:24 +00:00
|
|
|
|
2021-04-16 13:50:09 +00:00
|
|
|
/// Size depends on coordination settings
|
|
|
|
std::unique_ptr<RequestsQueue> requests_queue;
|
2021-02-08 13:06:55 +00:00
|
|
|
ResponsesQueue responses_queue;
|
2021-03-05 10:40:24 +00:00
|
|
|
SnapshotsQueue snapshots_queue{1};
|
2021-10-19 13:11:29 +00:00
|
|
|
|
|
|
|
/// More than 1k updates is definitely misconfiguration.
|
2023-04-20 13:26:02 +00:00
|
|
|
ClusterUpdateQueue cluster_update_queue{1000};
|
2021-03-05 10:40:24 +00:00
|
|
|
|
2021-01-26 14:08:31 +00:00
|
|
|
std::atomic<bool> shutdown_called{false};
|
2021-01-19 14:22:28 +00:00
|
|
|
|
2021-10-27 12:26:42 +00:00
|
|
|
mutable std::mutex session_to_response_callback_mutex;
|
2021-04-16 13:50:09 +00:00
|
|
|
/// These two maps looks similar, but serves different purposes.
|
|
|
|
/// The first map is subscription map for normal responses like
|
|
|
|
/// (get, set, list, etc.). Dispatcher determines callback for each response
|
|
|
|
/// using session id from this map.
|
2021-01-19 14:22:28 +00:00
|
|
|
SessionToResponseCallback session_to_response_callback;
|
2021-08-24 12:30:31 +00:00
|
|
|
|
2021-04-16 13:50:09 +00:00
|
|
|
/// But when client connects to the server for the first time it doesn't
|
|
|
|
/// have session_id. It request it from server. We give temporary
|
|
|
|
/// internal id for such requests just to much client with its response.
|
|
|
|
SessionToResponseCallback new_session_id_response_callback;
|
2021-01-19 14:22:28 +00:00
|
|
|
|
2021-04-16 13:50:09 +00:00
|
|
|
/// Reading and batching new requests from client handlers
|
2021-02-08 13:06:55 +00:00
|
|
|
ThreadFromGlobalPool request_thread;
|
2021-04-16 13:50:09 +00:00
|
|
|
/// Pushing responses to clients client handlers
|
|
|
|
/// using session_id.
|
2021-02-08 13:06:55 +00:00
|
|
|
ThreadFromGlobalPool responses_thread;
|
2021-04-16 13:50:09 +00:00
|
|
|
/// Cleaning old dead sessions
|
2021-02-03 20:32:15 +00:00
|
|
|
ThreadFromGlobalPool session_cleaner_thread;
|
2021-04-16 13:50:09 +00:00
|
|
|
/// Dumping new snapshots to disk
|
2021-03-05 10:40:24 +00:00
|
|
|
ThreadFromGlobalPool snapshot_thread;
|
2021-10-19 13:11:29 +00:00
|
|
|
/// Apply or wait for configuration changes
|
2021-10-19 12:00:26 +00:00
|
|
|
ThreadFromGlobalPool update_configuration_thread;
|
|
|
|
|
2021-08-24 12:30:31 +00:00
|
|
|
/// RAFT wrapper.
|
2021-03-29 08:24:56 +00:00
|
|
|
std::unique_ptr<KeeperServer> server;
|
2021-02-01 11:27:26 +00:00
|
|
|
|
2021-11-18 20:17:22 +00:00
|
|
|
KeeperConnectionStats keeper_stats;
|
2021-10-27 12:26:42 +00:00
|
|
|
|
2021-11-18 20:17:22 +00:00
|
|
|
KeeperConfigurationAndSettingsPtr configuration_and_settings;
|
2021-10-27 12:26:42 +00:00
|
|
|
|
2021-02-01 11:27:26 +00:00
|
|
|
Poco::Logger * log;
|
2021-01-19 14:22:28 +00:00
|
|
|
|
2021-04-16 13:50:09 +00:00
|
|
|
/// Counter for new session_id requests.
|
|
|
|
std::atomic<int64_t> internal_session_id_counter{0};
|
|
|
|
|
2022-09-21 11:53:54 +00:00
|
|
|
KeeperSnapshotManagerS3 snapshot_s3;
|
2022-09-15 07:45:28 +00:00
|
|
|
|
2023-05-25 08:43:11 +00:00
|
|
|
KeeperContextPtr keeper_context;
|
|
|
|
|
2021-08-24 12:30:31 +00:00
|
|
|
/// Thread put requests to raft
|
2021-02-08 13:06:55 +00:00
|
|
|
void requestThread();
|
2021-08-24 12:30:31 +00:00
|
|
|
/// Thread put responses for subscribed sessions
|
2021-02-08 13:06:55 +00:00
|
|
|
void responseThread();
|
2021-08-24 12:30:31 +00:00
|
|
|
/// Thread clean disconnected sessions from memory
|
2021-02-03 20:32:15 +00:00
|
|
|
void sessionCleanerTask();
|
2021-08-24 12:30:31 +00:00
|
|
|
/// Thread create snapshots in the background
|
2021-03-05 10:40:24 +00:00
|
|
|
void snapshotThread();
|
2023-04-20 13:26:02 +00:00
|
|
|
|
|
|
|
// TODO (myrrc) this should be removed once "reconfig" is stabilized
|
|
|
|
void clusterUpdateWithReconfigDisabledThread();
|
|
|
|
void clusterUpdateThread();
|
2021-10-19 12:00:26 +00:00
|
|
|
|
2021-01-19 14:22:28 +00:00
|
|
|
void setResponse(int64_t session_id, const Coordination::ZooKeeperResponsePtr & response);
|
|
|
|
|
2021-04-16 13:50:09 +00:00
|
|
|
/// Add error responses for requests to responses queue.
|
|
|
|
/// Clears requests.
|
2021-04-17 14:06:49 +00:00
|
|
|
void addErrorResponses(const KeeperStorage::RequestsForSessions & requests_for_sessions, Coordination::Error error);
|
2021-04-16 13:50:09 +00:00
|
|
|
|
|
|
|
/// Forcefully wait for result and sets errors if something when wrong.
|
|
|
|
/// Clears both arguments
|
2021-04-17 14:06:49 +00:00
|
|
|
void forceWaitAndProcessResult(RaftAppendResult & result, KeeperStorage::RequestsForSessions & requests_for_sessions);
|
2021-04-16 13:50:09 +00:00
|
|
|
|
2021-01-19 14:22:28 +00:00
|
|
|
public:
|
2023-03-24 20:40:45 +00:00
|
|
|
std::mutex read_request_queue_mutex;
|
|
|
|
|
|
|
|
/// queue of read requests that can be processed after a request with specific session ID and XID is committed
|
|
|
|
std::unordered_map<int64_t, std::unordered_map<Coordination::XID, KeeperStorage::RequestsForSessions>> read_request_queue;
|
2023-03-24 14:37:40 +00:00
|
|
|
|
2021-08-24 12:30:31 +00:00
|
|
|
/// Just allocate some objects, real initialization is done by `intialize method`
|
|
|
|
KeeperDispatcher();
|
|
|
|
|
|
|
|
/// Call shutdown
|
2021-11-18 20:17:22 +00:00
|
|
|
~KeeperDispatcher();
|
2021-01-25 12:29:12 +00:00
|
|
|
|
2021-08-24 12:30:31 +00:00
|
|
|
/// Initialization from config.
|
|
|
|
/// standalone_keeper -- we are standalone keeper application (not inside clickhouse server)
|
2023-01-03 10:44:26 +00:00
|
|
|
/// 'macros' are used to substitute macros in endpoint of disks
|
2022-12-23 14:09:24 +00:00
|
|
|
void initialize(const Poco::Util::AbstractConfiguration & config, bool standalone_keeper, bool start_async, const MultiVersion<Macros>::Version & macros);
|
2021-10-14 10:21:41 +00:00
|
|
|
|
2022-04-13 14:08:13 +00:00
|
|
|
void startServer();
|
|
|
|
|
2021-10-14 10:21:41 +00:00
|
|
|
bool checkInit() const
|
|
|
|
{
|
|
|
|
return server && server->checkInit();
|
|
|
|
}
|
2021-01-22 16:04:57 +00:00
|
|
|
|
2022-04-26 07:32:02 +00:00
|
|
|
/// Is server accepting requests, i.e. connected to the cluster
|
|
|
|
/// and achieved quorum
|
2022-04-14 12:00:47 +00:00
|
|
|
bool isServerActive() const;
|
|
|
|
|
2022-12-23 14:09:24 +00:00
|
|
|
void updateConfiguration(const Poco::Util::AbstractConfiguration & config, const MultiVersion<Macros>::Version & macros);
|
2023-07-17 10:50:02 +00:00
|
|
|
void pushClusterUpdates(ClusterUpdateActions && actions);
|
2023-04-20 13:26:02 +00:00
|
|
|
bool reconfigEnabled() const;
|
2021-10-18 15:27:51 +00:00
|
|
|
|
2021-08-24 12:30:31 +00:00
|
|
|
/// Shutdown internal keeper parts (server, state machine, log storage, etc)
|
2021-01-26 14:08:31 +00:00
|
|
|
void shutdown();
|
|
|
|
|
2022-04-13 14:08:13 +00:00
|
|
|
void forceRecovery();
|
|
|
|
|
2021-08-24 12:30:31 +00:00
|
|
|
/// Put request to ClickHouse Keeper
|
2021-01-25 12:29:12 +00:00
|
|
|
bool putRequest(const Coordination::ZooKeeperRequestPtr & request, int64_t session_id);
|
2021-01-21 13:53:10 +00:00
|
|
|
|
2021-10-27 12:26:42 +00:00
|
|
|
/// Get new session ID
|
|
|
|
int64_t getSessionID(int64_t session_timeout_ms);
|
|
|
|
|
|
|
|
/// Register session and subscribe for responses with callback
|
|
|
|
void registerSession(int64_t session_id, ZooKeeperResponseCallback callback);
|
|
|
|
|
|
|
|
/// Call if we don't need any responses for this session no more (session was expired)
|
|
|
|
void finishSession(int64_t session_id);
|
|
|
|
|
|
|
|
/// Invoked when a request completes.
|
2021-11-18 20:17:22 +00:00
|
|
|
void updateKeeperStatLatency(uint64_t process_time_ms);
|
2021-10-27 12:26:42 +00:00
|
|
|
|
2021-08-24 12:30:31 +00:00
|
|
|
/// Are we leader
|
2021-11-18 20:17:22 +00:00
|
|
|
bool isLeader() const
|
2021-01-27 17:54:25 +00:00
|
|
|
{
|
|
|
|
return server->isLeader();
|
|
|
|
}
|
|
|
|
|
2022-06-30 22:07:22 +00:00
|
|
|
bool isFollower() const
|
|
|
|
{
|
|
|
|
return server->isFollower();
|
|
|
|
}
|
|
|
|
|
2021-11-18 20:17:22 +00:00
|
|
|
bool hasLeader() const
|
2021-02-01 07:51:10 +00:00
|
|
|
{
|
|
|
|
return server->isLeaderAlive();
|
|
|
|
}
|
|
|
|
|
2021-11-18 20:17:22 +00:00
|
|
|
bool isObserver() const
|
|
|
|
{
|
|
|
|
return server->isObserver();
|
|
|
|
}
|
2021-01-21 13:53:10 +00:00
|
|
|
|
2021-11-18 20:17:22 +00:00
|
|
|
uint64_t getLogDirSize() const;
|
2021-08-24 12:30:31 +00:00
|
|
|
|
2021-11-18 20:17:22 +00:00
|
|
|
uint64_t getSnapDirSize() const;
|
2021-11-05 10:21:34 +00:00
|
|
|
|
2021-10-27 12:26:42 +00:00
|
|
|
/// Request statistics such as qps, latency etc.
|
2022-03-03 06:30:22 +00:00
|
|
|
KeeperConnectionStats & getKeeperConnectionStats()
|
2021-10-27 12:26:42 +00:00
|
|
|
{
|
|
|
|
return keeper_stats;
|
|
|
|
}
|
|
|
|
|
2021-11-19 09:30:58 +00:00
|
|
|
Keeper4LWInfo getKeeper4LWInfo() const;
|
2021-10-27 12:26:42 +00:00
|
|
|
|
2021-11-18 20:17:22 +00:00
|
|
|
const KeeperStateMachine & getStateMachine() const
|
2021-10-27 12:26:42 +00:00
|
|
|
{
|
2021-11-18 20:17:22 +00:00
|
|
|
return *server->getKeeperStateMachine();
|
2021-10-27 12:26:42 +00:00
|
|
|
}
|
|
|
|
|
2021-11-18 20:17:22 +00:00
|
|
|
const KeeperConfigurationAndSettingsPtr & getKeeperConfigurationAndSettings() const
|
2021-10-27 12:26:42 +00:00
|
|
|
{
|
2021-11-18 20:17:22 +00:00
|
|
|
return configuration_and_settings;
|
2021-10-27 12:26:42 +00:00
|
|
|
}
|
|
|
|
|
2023-05-25 08:43:11 +00:00
|
|
|
const KeeperContextPtr & getKeeperContext() const
|
|
|
|
{
|
|
|
|
return keeper_context;
|
|
|
|
}
|
|
|
|
|
2021-11-18 20:17:22 +00:00
|
|
|
void incrementPacketsSent()
|
2021-10-28 14:22:56 +00:00
|
|
|
{
|
2021-11-18 20:17:22 +00:00
|
|
|
keeper_stats.incrementPacketsSent();
|
2021-10-28 14:22:56 +00:00
|
|
|
}
|
|
|
|
|
2021-11-18 20:17:22 +00:00
|
|
|
void incrementPacketsReceived()
|
2021-11-05 10:21:34 +00:00
|
|
|
{
|
2021-11-18 20:17:22 +00:00
|
|
|
keeper_stats.incrementPacketsReceived();
|
2021-11-05 10:21:34 +00:00
|
|
|
}
|
|
|
|
|
2021-11-18 20:17:22 +00:00
|
|
|
void resetConnectionStats()
|
2021-11-05 10:21:34 +00:00
|
|
|
{
|
2021-11-18 20:17:22 +00:00
|
|
|
keeper_stats.reset();
|
2021-11-05 10:21:34 +00:00
|
|
|
}
|
2022-09-26 10:29:15 +00:00
|
|
|
|
2022-10-24 12:08:58 +00:00
|
|
|
/// Create snapshot manually, return the last committed log index in the snapshot
|
|
|
|
uint64_t createSnapshot()
|
2022-09-26 10:29:15 +00:00
|
|
|
{
|
|
|
|
return server->createSnapshot();
|
|
|
|
}
|
2022-10-22 14:31:17 +00:00
|
|
|
|
2022-10-25 09:46:24 +00:00
|
|
|
/// Get Raft information
|
2022-10-24 12:08:58 +00:00
|
|
|
KeeperLogInfo getKeeperLogInfo()
|
2022-10-22 14:31:17 +00:00
|
|
|
{
|
2022-10-24 12:08:58 +00:00
|
|
|
return server->getKeeperLogInfo();
|
2022-10-22 14:31:17 +00:00
|
|
|
}
|
2022-11-08 04:01:53 +00:00
|
|
|
|
|
|
|
/// Request to be leader.
|
|
|
|
bool requestLeader()
|
|
|
|
{
|
|
|
|
return server->requestLeader();
|
|
|
|
}
|
2023-02-06 14:33:45 +00:00
|
|
|
|
|
|
|
void recalculateStorageStats()
|
|
|
|
{
|
|
|
|
return server->recalculateStorageStats();
|
|
|
|
}
|
2023-03-05 16:45:17 +00:00
|
|
|
|
|
|
|
static void cleanResources();
|
2021-01-19 14:22:28 +00:00
|
|
|
};
|
|
|
|
|
|
|
|
}
|
2021-02-01 13:18:17 +00:00
|
|
|
|
|
|
|
#endif
|