diff --git a/programs/keeper/CMakeLists.txt b/programs/keeper/CMakeLists.txt index 0c3d76e614f..b8a5d9c9c19 100644 --- a/programs/keeper/CMakeLists.txt +++ b/programs/keeper/CMakeLists.txt @@ -39,6 +39,7 @@ if (BUILD_STANDALONE_KEEPER) ${CMAKE_CURRENT_SOURCE_DIR}/../../src/Coordination/KeeperContext.cpp ${CMAKE_CURRENT_SOURCE_DIR}/../../src/Coordination/KeeperStateManager.cpp ${CMAKE_CURRENT_SOURCE_DIR}/../../src/Coordination/KeeperStorage.cpp + ${CMAKE_CURRENT_SOURCE_DIR}/../../src/Coordination/KeeperConstants.cpp ${CMAKE_CURRENT_SOURCE_DIR}/../../src/Coordination/KeeperAsynchronousMetrics.cpp ${CMAKE_CURRENT_SOURCE_DIR}/../../src/Coordination/pathUtils.cpp ${CMAKE_CURRENT_SOURCE_DIR}/../../src/Coordination/SessionExpiryQueue.cpp diff --git a/programs/keeper/Keeper.cpp b/programs/keeper/Keeper.cpp index b0c207347a3..5b844e7d650 100644 --- a/programs/keeper/Keeper.cpp +++ b/programs/keeper/Keeper.cpp @@ -32,9 +32,10 @@ #include #include -#include #include #include +#include +#include #include "Core/Defines.h" #include "config.h" @@ -488,19 +489,28 @@ try /// Prometheus (if defined and not setup yet with http_port) port_name = "prometheus.port"; - createServer(listen_host, port_name, listen_try, [&, my_http_context = std::move(http_context)](UInt16 port) mutable - { - Poco::Net::ServerSocket socket; - auto address = socketBindListen(socket, listen_host, port); - socket.setReceiveTimeout(my_http_context->getReceiveTimeout()); - socket.setSendTimeout(my_http_context->getSendTimeout()); - servers->emplace_back( - listen_host, - port_name, - "Prometheus: http://" + address.toString(), - std::make_unique( - std::move(my_http_context), createPrometheusMainHandlerFactory(*this, config_getter(), async_metrics, "PrometheusHandler-factory"), server_pool, socket, http_params)); - }); + createServer( + listen_host, + port_name, + listen_try, + [&, my_http_context = std::move(http_context)](UInt16 port) mutable + { + Poco::Net::ServerSocket socket; + auto address = socketBindListen(socket, listen_host, port); + socket.setReceiveTimeout(my_http_context->getReceiveTimeout()); + socket.setSendTimeout(my_http_context->getSendTimeout()); + auto metrics_writer = std::make_shared(config, "prometheus", async_metrics); + servers->emplace_back( + listen_host, + port_name, + "Prometheus: http://" + address.toString(), + std::make_unique( + std::move(my_http_context), + createPrometheusMainHandlerFactory(*this, config_getter(), metrics_writer, "PrometheusHandler-factory"), + server_pool, + socket, + http_params)); + }); /// HTTP control endpoints port_name = "keeper_server.http_control.port"; diff --git a/src/Common/CurrentMetrics.cpp b/src/Common/CurrentMetrics.cpp index f56149ed464..c6fbafa8dc3 100644 --- a/src/Common/CurrentMetrics.cpp +++ b/src/Common/CurrentMetrics.cpp @@ -2,6 +2,8 @@ /// Available metrics. Add something here as you wish. +/// If the metric is generic (i.e. not server specific) +/// it should be also added to src/Coordination/KeeperConstant.cpp #define APPLY_FOR_BUILTIN_METRICS(M) \ M(Query, "Number of executing queries") \ M(Merge, "Number of executing background merges") \ diff --git a/src/Common/ProfileEvents.cpp b/src/Common/ProfileEvents.cpp index bdc5d2d88a8..5a3aa401fab 100644 --- a/src/Common/ProfileEvents.cpp +++ b/src/Common/ProfileEvents.cpp @@ -4,6 +4,8 @@ /// Available events. Add something here as you wish. +/// If the event is generic (i.e. not server specific) +/// it should be also added to src/Coordination/KeeperConstant.cpp #define APPLY_FOR_BUILTIN_EVENTS(M) \ M(Query, "Number of queries to be interpreted and potentially executed. Does not include queries that failed to parse or were rejected due to AST size limits, quota limits or limits on the number of simultaneously running queries. May include internal queries initiated by ClickHouse itself. Does not count subqueries.") \ M(SelectQuery, "Same as Query, but only for SELECT queries.") \ diff --git a/src/Coordination/KeeperConstants.cpp b/src/Coordination/KeeperConstants.cpp new file mode 100644 index 00000000000..2aa84b691c4 --- /dev/null +++ b/src/Coordination/KeeperConstants.cpp @@ -0,0 +1,376 @@ +#include +#include + +/// Events which are useful for Keeper. +/// New events should be added manually. +#define APPLY_FOR_KEEPER_PROFILE_EVENTS(M) \ + M(FileOpen) \ + M(Seek) \ + M(ReadBufferFromFileDescriptorRead) \ + M(ReadBufferFromFileDescriptorReadFailed) \ + M(ReadBufferFromFileDescriptorReadBytes) \ + M(WriteBufferFromFileDescriptorWrite) \ + M(WriteBufferFromFileDescriptorWriteFailed) \ + M(WriteBufferFromFileDescriptorWriteBytes) \ + M(FileSync) \ + M(DirectorySync) \ + M(FileSyncElapsedMicroseconds) \ + M(DirectorySyncElapsedMicroseconds) \ + M(ReadCompressedBytes) \ + M(CompressedReadBufferBlocks) \ + M(CompressedReadBufferBytes) \ + M(AIOWrite) \ + M(AIOWriteBytes) \ + M(AIORead) \ + M(AIOReadBytes) \ + M(IOBufferAllocs) \ + M(IOBufferAllocBytes) \ + M(ArenaAllocChunks) \ + M(ArenaAllocBytes) \ + M(CreatedReadBufferOrdinary) \ + M(CreatedReadBufferDirectIO) \ + M(CreatedReadBufferDirectIOFailed) \ + M(CreatedReadBufferMMap) \ + M(CreatedReadBufferMMapFailed) \ + M(DiskReadElapsedMicroseconds) \ + M(DiskWriteElapsedMicroseconds) \ + M(NetworkReceiveElapsedMicroseconds) \ + M(NetworkSendElapsedMicroseconds) \ + M(NetworkReceiveBytes) \ + M(NetworkSendBytes) \ +\ + M(DiskS3GetRequestThrottlerCount) \ + M(DiskS3GetRequestThrottlerSleepMicroseconds) \ + M(DiskS3PutRequestThrottlerCount) \ + M(DiskS3PutRequestThrottlerSleepMicroseconds) \ + M(S3GetRequestThrottlerCount) \ + M(S3GetRequestThrottlerSleepMicroseconds) \ + M(S3PutRequestThrottlerCount) \ + M(S3PutRequestThrottlerSleepMicroseconds) \ + M(RemoteReadThrottlerBytes) \ + M(RemoteReadThrottlerSleepMicroseconds) \ + M(RemoteWriteThrottlerBytes) \ + M(RemoteWriteThrottlerSleepMicroseconds) \ + M(LocalReadThrottlerBytes) \ + M(LocalReadThrottlerSleepMicroseconds) \ + M(LocalWriteThrottlerBytes) \ + M(LocalWriteThrottlerSleepMicroseconds) \ + M(ThrottlerSleepMicroseconds) \ +\ + M(SlowRead) \ + M(ReadBackoff) \ +\ + M(ContextLock) \ + M(ContextLockWaitMicroseconds) \ +\ + M(RWLockAcquiredReadLocks) \ + M(RWLockAcquiredWriteLocks) \ + M(RWLockReadersWaitMilliseconds) \ + M(RWLockWritersWaitMilliseconds) \ + M(DNSError) \ + M(RealTimeMicroseconds) \ + M(UserTimeMicroseconds) \ + M(SystemTimeMicroseconds) \ + M(MemoryOvercommitWaitTimeMicroseconds) \ + M(MemoryAllocatorPurge) \ + M(MemoryAllocatorPurgeTimeMicroseconds) \ + M(SoftPageFaults) \ + M(HardPageFaults) \ +\ + M(OSIOWaitMicroseconds) \ + M(OSCPUWaitMicroseconds) \ + M(OSCPUVirtualTimeMicroseconds) \ + M(OSReadBytes) \ + M(OSWriteBytes) \ + M(OSReadChars) \ + M(OSWriteChars) \ +\ + M(PerfCpuCycles) \ + M(PerfInstructions) \ + M(PerfCacheReferences) \ + M(PerfCacheMisses) \ + M(PerfBranchInstructions) \ + M(PerfBranchMisses) \ + M(PerfBusCycles) \ + M(PerfStalledCyclesFrontend) \ + M(PerfStalledCyclesBackend) \ + M(PerfRefCpuCycles) \ +\ + M(PerfCpuClock) \ + M(PerfTaskClock) \ + M(PerfContextSwitches) \ + M(PerfCpuMigrations) \ + M(PerfAlignmentFaults) \ + M(PerfEmulationFaults) \ + M(PerfMinEnabledTime) \ + M(PerfMinEnabledRunningTime) \ + M(PerfDataTLBReferences) \ + M(PerfDataTLBMisses) \ + M(PerfInstructionTLBReferences) \ + M(PerfInstructionTLBMisses) \ + M(PerfLocalMemoryReferences) \ + M(PerfLocalMemoryMisses) \ +\ + M(CreatedHTTPConnections) \ + M(CannotWriteToWriteBufferDiscard) \ +\ + M(S3ReadMicroseconds) \ + M(S3ReadRequestsCount) \ + M(S3ReadRequestsErrors) \ + M(S3ReadRequestsThrottling) \ + M(S3ReadRequestsRedirects) \ +\ + M(S3WriteMicroseconds) \ + M(S3WriteRequestsCount) \ + M(S3WriteRequestsErrors) \ + M(S3WriteRequestsThrottling) \ + M(S3WriteRequestsRedirects) \ +\ + M(DiskS3ReadMicroseconds) \ + M(DiskS3ReadRequestsCount) \ + M(DiskS3ReadRequestsErrors) \ + M(DiskS3ReadRequestsThrottling) \ + M(DiskS3ReadRequestsRedirects) \ +\ + M(DiskS3WriteMicroseconds) \ + M(DiskS3WriteRequestsCount) \ + M(DiskS3WriteRequestsErrors) \ + M(DiskS3WriteRequestsThrottling) \ + M(DiskS3WriteRequestsRedirects) \ +\ + M(S3DeleteObjects) \ + M(S3CopyObject) \ + M(S3ListObjects) \ + M(S3HeadObject) \ + M(S3GetObjectAttributes) \ + M(S3CreateMultipartUpload) \ + M(S3UploadPartCopy) \ + M(S3UploadPart) \ + M(S3AbortMultipartUpload) \ + M(S3CompleteMultipartUpload) \ + M(S3PutObject) \ + M(S3GetObject) \ +\ + M(AzureUploadPart) \ + M(DiskAzureUploadPart) \ + M(AzureCopyObject) \ + M(DiskAzureCopyObject) \ + M(AzureDeleteObjects) \ + M(AzureListObjects) \ +\ + M(DiskS3DeleteObjects) \ + M(DiskS3CopyObject) \ + M(DiskS3ListObjects) \ + M(DiskS3HeadObject) \ + M(DiskS3GetObjectAttributes) \ + M(DiskS3CreateMultipartUpload) \ + M(DiskS3UploadPartCopy) \ + M(DiskS3UploadPart) \ + M(DiskS3AbortMultipartUpload) \ + M(DiskS3CompleteMultipartUpload) \ + M(DiskS3PutObject) \ + M(DiskS3GetObject) \ +\ + M(S3Clients) \ + M(TinyS3Clients) \ +\ + M(ReadBufferFromS3Microseconds) \ + M(ReadBufferFromS3InitMicroseconds) \ + M(ReadBufferFromS3Bytes) \ + M(ReadBufferFromS3RequestsErrors) \ + M(ReadBufferFromS3ResetSessions) \ + M(ReadBufferFromS3PreservedSessions) \ +\ + M(ReadWriteBufferFromHTTPPreservedSessions) \ +\ + M(WriteBufferFromS3Microseconds) \ + M(WriteBufferFromS3Bytes) \ + M(WriteBufferFromS3RequestsErrors) \ + M(WriteBufferFromS3WaitInflightLimitMicroseconds) \ + M(RemoteFSSeeks) \ + M(RemoteFSPrefetches) \ + M(RemoteFSCancelledPrefetches) \ + M(RemoteFSUnusedPrefetches) \ + M(RemoteFSPrefetchedReads) \ + M(RemoteFSPrefetchedBytes) \ + M(RemoteFSUnprefetchedReads) \ + M(RemoteFSUnprefetchedBytes) \ + M(RemoteFSLazySeeks) \ + M(RemoteFSSeeksWithReset) \ + M(RemoteFSBuffers) \ +\ + M(ThreadpoolReaderTaskMicroseconds) \ + M(ThreadpoolReaderPrepareMicroseconds) \ + M(ThreadpoolReaderReadBytes) \ + M(ThreadpoolReaderSubmit) \ + M(ThreadpoolReaderSubmitReadSynchronously) \ + M(ThreadpoolReaderSubmitReadSynchronouslyBytes) \ + M(ThreadpoolReaderSubmitReadSynchronouslyMicroseconds) \ + M(ThreadpoolReaderSubmitLookupInCacheMicroseconds) \ + M(AsynchronousReaderIgnoredBytes) \ +\ + M(FileSegmentWaitReadBufferMicroseconds) \ + M(FileSegmentReadMicroseconds) \ + M(FileSegmentCacheWriteMicroseconds) \ + M(FileSegmentPredownloadMicroseconds) \ + M(FileSegmentUsedBytes) \ +\ + M(ReadBufferSeekCancelConnection) \ +\ + M(SleepFunctionCalls) \ + M(SleepFunctionMicroseconds) \ + M(SleepFunctionElapsedMicroseconds) \ +\ + M(ThreadPoolReaderPageCacheHit) \ + M(ThreadPoolReaderPageCacheHitBytes) \ + M(ThreadPoolReaderPageCacheHitElapsedMicroseconds) \ + M(ThreadPoolReaderPageCacheMiss) \ + M(ThreadPoolReaderPageCacheMissBytes) \ + M(ThreadPoolReaderPageCacheMissElapsedMicroseconds) \ +\ + M(AsynchronousReadWaitMicroseconds) \ + M(SynchronousReadWaitMicroseconds) \ + M(AsynchronousRemoteReadWaitMicroseconds) \ + M(SynchronousRemoteReadWaitMicroseconds) \ +\ + M(ExternalDataSourceLocalCacheReadBytes) \ +\ + M(MainConfigLoads) \ +\ + M(KeeperPacketsSent) \ + M(KeeperPacketsReceived) \ + M(KeeperRequestTotal) \ + M(KeeperLatency) \ + M(KeeperCommits) \ + M(KeeperCommitsFailed) \ + M(KeeperSnapshotCreations) \ + M(KeeperSnapshotCreationsFailed) \ + M(KeeperSnapshotApplys) \ + M(KeeperSnapshotApplysFailed) \ + M(KeeperReadSnapshot) \ + M(KeeperSaveSnapshot) \ + M(KeeperCreateRequest) \ + M(KeeperRemoveRequest) \ + M(KeeperSetRequest) \ + M(KeeperReconfigRequest) \ + M(KeeperCheckRequest) \ + M(KeeperMultiRequest) \ + M(KeeperMultiReadRequest) \ + M(KeeperGetRequest) \ + M(KeeperListRequest) \ + M(KeeperExistsRequest) \ +\ + M(IOUringSQEsSubmitted) \ + M(IOUringSQEsResubmits) \ + M(IOUringCQEsCompleted) \ + M(IOUringCQEsFailed) \ +\ + M(LogTest) \ + M(LogTrace) \ + M(LogDebug) \ + M(LogInfo) \ + M(LogWarning) \ + M(LogError) \ + M(LogFatal) \ +\ + M(InterfaceHTTPSendBytes) \ + M(InterfaceHTTPReceiveBytes) \ + M(InterfaceNativeSendBytes) \ + M(InterfaceNativeReceiveBytes) \ + M(InterfacePrometheusSendBytes) \ + M(InterfacePrometheusReceiveBytes) \ + M(InterfaceInterserverSendBytes) \ + M(InterfaceInterserverReceiveBytes) \ + M(InterfaceMySQLSendBytes) \ + M(InterfaceMySQLReceiveBytes) \ + M(InterfacePostgreSQLSendBytes) \ + M(InterfacePostgreSQLReceiveBytes) + +namespace ProfileEvents +{ +#define M(NAME) extern const Event NAME; + APPLY_FOR_KEEPER_PROFILE_EVENTS(M) +#undef M + +#define M(NAME) NAME, +extern const std::vector keeper_profile_events +{ + APPLY_FOR_KEEPER_PROFILE_EVENTS(M) +}; +#undef M +} + +/// Metrics which are useful for Keeper. +/// New metrics should be added manually. +#define APPLY_FOR_KEEPER_METRICS(M) \ + M(BackgroundCommonPoolTask) \ + M(BackgroundCommonPoolSize) \ + M(TCPConnection) \ + M(HTTPConnection) \ + M(OpenFileForRead) \ + M(OpenFileForWrite) \ + M(Read) \ + M(RemoteRead) \ + M(Write) \ + M(NetworkReceive) \ + M(NetworkSend) \ + M(MemoryTracking) \ + M(ContextLockWait) \ + M(Revision) \ + M(VersionInteger) \ + M(RWLockWaitingReaders) \ + M(RWLockWaitingWriters) \ + M(RWLockActiveReaders) \ + M(RWLockActiveWriters) \ + M(GlobalThread) \ + M(GlobalThreadActive) \ + M(GlobalThreadScheduled) \ + M(LocalThread) \ + M(LocalThreadActive) \ + M(LocalThreadScheduled) \ + M(IOPrefetchThreads) \ + M(IOPrefetchThreadsActive) \ + M(IOPrefetchThreadsScheduled) \ + M(IOWriterThreads) \ + M(IOWriterThreadsActive) \ + M(IOWriterThreadsScheduled) \ + M(IOThreads) \ + M(IOThreadsActive) \ + M(IOThreadsScheduled) \ + M(ThreadPoolRemoteFSReaderThreads) \ + M(ThreadPoolRemoteFSReaderThreadsActive) \ + M(ThreadPoolRemoteFSReaderThreadsScheduled) \ + M(ThreadPoolFSReaderThreads) \ + M(ThreadPoolFSReaderThreadsActive) \ + M(ThreadPoolFSReaderThreadsScheduled) \ + M(DiskObjectStorageAsyncThreads) \ + M(DiskObjectStorageAsyncThreadsActive) \ + M(ObjectStorageS3Threads) \ + M(ObjectStorageS3ThreadsActive) \ + M(ObjectStorageS3ThreadsScheduled) \ + M(ObjectStorageAzureThreads) \ + M(ObjectStorageAzureThreadsActive) \ + M(ObjectStorageAzureThreadsScheduled) \ + M(MMappedFiles) \ + M(MMappedFileBytes) \ + M(AsynchronousReadWait) \ + M(S3Requests) \ + M(KeeperAliveConnections) \ + M(KeeperOutstandingRequets) \ + M(ThreadsInOvercommitTracker) \ + M(IOUringPendingEvents) \ + M(IOUringInFlightEvents) \ + +namespace CurrentMetrics +{ +#define M(NAME) extern const Metric NAME; + APPLY_FOR_KEEPER_METRICS(M) +#undef M + +#define M(NAME) NAME, +extern const std::vector keeper_metrics +{ + APPLY_FOR_KEEPER_METRICS(M) +}; +#undef M +} diff --git a/src/Server/HTTPHandlerFactory.cpp b/src/Server/HTTPHandlerFactory.cpp index 66b55f68217..06ca1182be5 100644 --- a/src/Server/HTTPHandlerFactory.cpp +++ b/src/Server/HTTPHandlerFactory.cpp @@ -1,3 +1,4 @@ +#include #include #include @@ -7,6 +8,7 @@ #include #include "HTTPHandler.h" +#include "Server/PrometheusMetricsWriter.h" #include "StaticRequestHandler.h" #include "ReplicasStatusHandler.h" #include "InterserverIOHTTPHandler.h" @@ -113,7 +115,10 @@ HTTPRequestHandlerFactoryPtr createHandlerFactory(IServer & server, const Poco:: else if (name == "InterserverIOHTTPHandler-factory" || name == "InterserverIOHTTPSHandler-factory") return createInterserverHTTPHandlerFactory(server, name); else if (name == "PrometheusHandler-factory") - return createPrometheusMainHandlerFactory(server, config, async_metrics, name); + { + auto metrics_writer = std::make_shared(config, "prometheus", async_metrics); + return createPrometheusMainHandlerFactory(server, config, metrics_writer, name); + } throw Exception(ErrorCodes::LOGICAL_ERROR, "LOGICAL ERROR: Unknown HTTP handler factory name."); } @@ -208,7 +213,7 @@ void addDefaultHandlersFactory( /// Otherwise it will be created separately, see createHandlerFactory(...). if (config.has("prometheus") && config.getInt("prometheus.port", 0) == 0) { - PrometheusMetricsWriter writer(config, "prometheus", async_metrics); + auto writer = std::make_shared(config, "prometheus", async_metrics); auto creator = [&server, writer] () -> std::unique_ptr { return std::make_unique(server, writer); diff --git a/src/Server/HTTPHandlerFactory.h b/src/Server/HTTPHandlerFactory.h index 94b02e52277..427d495f659 100644 --- a/src/Server/HTTPHandlerFactory.h +++ b/src/Server/HTTPHandlerFactory.h @@ -6,6 +6,7 @@ #include #include #include +#include #include @@ -130,10 +131,10 @@ createPrometheusHandlerFactory(IServer & server, AsynchronousMetrics & async_metrics, const std::string & config_prefix); -HTTPRequestHandlerFactoryPtr -createPrometheusMainHandlerFactory(IServer & server, +HTTPRequestHandlerFactoryPtr createPrometheusMainHandlerFactory( + IServer & server, const Poco::Util::AbstractConfiguration & config, - AsynchronousMetrics & async_metrics, + PrometheusMetricsWriterPtr metrics_writer, const std::string & name); /// @param server - used in handlers to check IServer::isCancelled() diff --git a/src/Server/PrometheusMetricsWriter.cpp b/src/Server/PrometheusMetricsWriter.cpp index 3d09c2165e5..d0fdcd61493 100644 --- a/src/Server/PrometheusMetricsWriter.cpp +++ b/src/Server/PrometheusMetricsWriter.cpp @@ -4,6 +4,8 @@ #include #include +#include "config.h" + namespace { @@ -38,8 +40,83 @@ void convertHelpToSingleLine(std::string & help) std::replace(help.begin(), help.end(), '\n', ' '); } +constexpr auto profile_events_prefix = "ClickHouseProfileEvents_"; +constexpr auto current_metrics_prefix = "ClickHouseMetrics_"; +constexpr auto asynchronous_metrics_prefix = "ClickHouseAsyncMetrics_"; +constexpr auto error_metrics_prefix = "ClickHouseErrorMetric_"; + +void writeEvent(DB::WriteBuffer & wb, ProfileEvents::Event event) +{ + const auto counter = ProfileEvents::global_counters[event].load(std::memory_order_relaxed); + + std::string metric_name{ProfileEvents::getName(static_cast(event))}; + std::string metric_doc{ProfileEvents::getDocumentation(static_cast(event))}; + + convertHelpToSingleLine(metric_doc); + + if (!replaceInvalidChars(metric_name)) + return; + + std::string key{profile_events_prefix + metric_name}; + + writeOutLine(wb, "# HELP", key, metric_doc); + writeOutLine(wb, "# TYPE", key, "counter"); + writeOutLine(wb, key, counter); } +void writeMetric(DB::WriteBuffer & wb, size_t metric) +{ + const auto value = CurrentMetrics::values[metric].load(std::memory_order_relaxed); + + std::string metric_name{CurrentMetrics::getName(static_cast(metric))}; + std::string metric_doc{CurrentMetrics::getDocumentation(static_cast(metric))}; + + convertHelpToSingleLine(metric_doc); + + if (!replaceInvalidChars(metric_name)) + return; + + std::string key{current_metrics_prefix + metric_name}; + + writeOutLine(wb, "# HELP", key, metric_doc); + writeOutLine(wb, "# TYPE", key, "gauge"); + writeOutLine(wb, key, value); +} + +void writeAsyncMetrics(DB::WriteBuffer & wb, const DB::AsynchronousMetricValues & values) +{ + for (const auto & name_value : values) + { + std::string key{asynchronous_metrics_prefix + name_value.first}; + + if (!replaceInvalidChars(key)) + continue; + + auto value = name_value.second; + + std::string metric_doc{value.documentation}; + convertHelpToSingleLine(metric_doc); + + writeOutLine(wb, "# HELP", key, metric_doc); + writeOutLine(wb, "# TYPE", key, "gauge"); + writeOutLine(wb, key, value.value); + } +} + +} + +#if USE_NURAFT +namespace ProfileEvents +{ + extern const std::vector keeper_profile_events; +} + +namespace CurrentMetrics +{ + extern const std::vector keeper_metrics; +} +#endif + namespace DB { @@ -60,65 +137,17 @@ void PrometheusMetricsWriter::write(WriteBuffer & wb) const if (send_events) { for (ProfileEvents::Event i = ProfileEvents::Event(0), end = ProfileEvents::end(); i < end; ++i) - { - const auto counter = ProfileEvents::global_counters[i].load(std::memory_order_relaxed); - - std::string metric_name{ProfileEvents::getName(static_cast(i))}; - std::string metric_doc{ProfileEvents::getDocumentation(static_cast(i))}; - - convertHelpToSingleLine(metric_doc); - - if (!replaceInvalidChars(metric_name)) - continue; - std::string key{profile_events_prefix + metric_name}; - - writeOutLine(wb, "# HELP", key, metric_doc); - writeOutLine(wb, "# TYPE", key, "counter"); - writeOutLine(wb, key, counter); - } + writeEvent(wb, i); } if (send_metrics) { for (size_t i = 0, end = CurrentMetrics::end(); i < end; ++i) - { - const auto value = CurrentMetrics::values[i].load(std::memory_order_relaxed); - - std::string metric_name{CurrentMetrics::getName(static_cast(i))}; - std::string metric_doc{CurrentMetrics::getDocumentation(static_cast(i))}; - - convertHelpToSingleLine(metric_doc); - - if (!replaceInvalidChars(metric_name)) - continue; - std::string key{current_metrics_prefix + metric_name}; - - writeOutLine(wb, "# HELP", key, metric_doc); - writeOutLine(wb, "# TYPE", key, "gauge"); - writeOutLine(wb, key, value); - } + writeMetric(wb, i); } if (send_asynchronous_metrics) - { - auto async_metrics_values = async_metrics.getValues(); - for (const auto & name_value : async_metrics_values) - { - std::string key{asynchronous_metrics_prefix + name_value.first}; - - if (!replaceInvalidChars(key)) - continue; - - auto value = name_value.second; - - std::string metric_doc{value.documentation}; - convertHelpToSingleLine(metric_doc); - - writeOutLine(wb, "# HELP", key, metric_doc); - writeOutLine(wb, "# TYPE", key, "gauge"); - writeOutLine(wb, key, value.value); - } - } + writeAsyncMetrics(wb, async_metrics.getValues()); if (send_errors) { @@ -152,4 +181,24 @@ void PrometheusMetricsWriter::write(WriteBuffer & wb) const } +void KeeperPrometheusMetricsWriter::write([[maybe_unused]] WriteBuffer & wb) const +{ +#if USE_NURAFT + if (send_events) + { + for (auto event : ProfileEvents::keeper_profile_events) + writeEvent(wb, event); + } + + if (send_metrics) + { + for (auto metric : CurrentMetrics::keeper_metrics) + writeMetric(wb, metric); + } + + if (send_asynchronous_metrics) + writeAsyncMetrics(wb, async_metrics.getValues()); +#endif +} + } diff --git a/src/Server/PrometheusMetricsWriter.h b/src/Server/PrometheusMetricsWriter.h index b909a0ddcf6..933ad909ee0 100644 --- a/src/Server/PrometheusMetricsWriter.h +++ b/src/Server/PrometheusMetricsWriter.h @@ -3,6 +3,7 @@ #include #include +#include #include #include @@ -19,20 +20,25 @@ public: const Poco::Util::AbstractConfiguration & config, const std::string & config_name, const AsynchronousMetrics & async_metrics_); - void write(WriteBuffer & wb) const; + virtual void write(WriteBuffer & wb) const; -private: + virtual ~PrometheusMetricsWriter() = default; + +protected: const AsynchronousMetrics & async_metrics; - const bool send_events; const bool send_metrics; const bool send_asynchronous_metrics; const bool send_errors; - - static inline constexpr auto profile_events_prefix = "ClickHouseProfileEvents_"; - static inline constexpr auto current_metrics_prefix = "ClickHouseMetrics_"; - static inline constexpr auto asynchronous_metrics_prefix = "ClickHouseAsyncMetrics_"; - static inline constexpr auto error_metrics_prefix = "ClickHouseErrorMetric_"; }; +class KeeperPrometheusMetricsWriter : public PrometheusMetricsWriter +{ + using PrometheusMetricsWriter::PrometheusMetricsWriter; + + void write(WriteBuffer & wb) const override; +}; + +using PrometheusMetricsWriterPtr = std::shared_ptr; + } diff --git a/src/Server/PrometheusRequestHandler.cpp b/src/Server/PrometheusRequestHandler.cpp index 8690ec9121e..dff960f7031 100644 --- a/src/Server/PrometheusRequestHandler.cpp +++ b/src/Server/PrometheusRequestHandler.cpp @@ -7,6 +7,7 @@ #include #include #include +#include "Server/PrometheusMetricsWriter.h" #include @@ -34,7 +35,7 @@ void PrometheusRequestHandler::handleRequest(HTTPServerRequest & request, HTTPSe WriteBufferFromHTTPServerResponse wb(response, request.getMethod() == Poco::Net::HTTPRequest::HTTP_HEAD, keep_alive_timeout, write_event); try { - metrics_writer.write(wb); + metrics_writer->write(wb); wb.finalize(); } catch (...) @@ -54,7 +55,7 @@ HTTPRequestHandlerFactoryPtr createPrometheusHandlerFactory( AsynchronousMetrics & async_metrics, const std::string & config_prefix) { - PrometheusMetricsWriter writer(config, config_prefix + ".handler", async_metrics); + auto writer = std::make_shared(config, config_prefix + ".handler", async_metrics); auto creator = [&server, writer]() -> std::unique_ptr { return std::make_unique(server, writer); @@ -66,13 +67,12 @@ HTTPRequestHandlerFactoryPtr createPrometheusHandlerFactory( } HTTPRequestHandlerFactoryPtr createPrometheusMainHandlerFactory( - IServer & server, const Poco::Util::AbstractConfiguration & config, AsynchronousMetrics & async_metrics, const std::string & name) + IServer & server, const Poco::Util::AbstractConfiguration & config, PrometheusMetricsWriterPtr metrics_writer, const std::string & name) { auto factory = std::make_shared(name); - PrometheusMetricsWriter writer(config, "prometheus", async_metrics); - auto creator = [&server, writer]() -> std::unique_ptr + auto creator = [&server, metrics_writer] { - return std::make_unique(server, writer); + return std::make_unique(server, metrics_writer); }; auto handler = std::make_shared>(std::move(creator)); diff --git a/src/Server/PrometheusRequestHandler.h b/src/Server/PrometheusRequestHandler.h index 9ec54cc2e4e..d120752c8c5 100644 --- a/src/Server/PrometheusRequestHandler.h +++ b/src/Server/PrometheusRequestHandler.h @@ -13,12 +13,12 @@ class PrometheusRequestHandler : public HTTPRequestHandler { private: IServer & server; - const PrometheusMetricsWriter & metrics_writer; + PrometheusMetricsWriterPtr metrics_writer; public: - PrometheusRequestHandler(IServer & server_, const PrometheusMetricsWriter & metrics_writer_) + PrometheusRequestHandler(IServer & server_, PrometheusMetricsWriterPtr metrics_writer_) : server(server_) - , metrics_writer(metrics_writer_) + , metrics_writer(std::move(metrics_writer_)) { } diff --git a/utils/check-style/check-style b/utils/check-style/check-style index badfd173172..f0745ab43f3 100755 --- a/utils/check-style/check-style +++ b/utils/check-style/check-style @@ -76,6 +76,7 @@ EXTERN_TYPES_EXCLUDES=( ProfileEvents::getProfileEvents ProfileEvents::ThreadIdToCountersSnapshot ProfileEvents::LOCAL_NAME + ProfileEvents::keeper_profile_events ProfileEvents::CountersIncrement CurrentMetrics::add @@ -87,6 +88,7 @@ EXTERN_TYPES_EXCLUDES=( CurrentMetrics::Metric CurrentMetrics::values CurrentMetrics::Value + CurrentMetrics::keeper_metrics ErrorCodes::ErrorCode ErrorCodes::getName @@ -106,7 +108,7 @@ for extern_type in ${!EXTERN_TYPES[@]}; do find $ROOT_PATH/{src,base,programs,utils} -name '*.h' -or -name '*.cpp' | { # NOTE: the check is pretty dumb and distinguish only by the type_of_extern, # and this matches with zkutil::CreateMode - grep -v 'src/Common/ZooKeeper/Types.h' + grep -v -e 'src/Common/ZooKeeper/Types.h' -e 'src/Coordination/KeeperConstants.cpp' } | { grep -vP $EXCLUDE_DIRS | xargs grep -l -P "extern const $type_of_extern $allowed_chars" } | while read file; do