2018-06-20 17:49:52 +00:00
|
|
|
#include "ProfileEventsExt.h"
|
|
|
|
#include <Common/typeid_cast.h>
|
2022-02-15 08:25:07 +00:00
|
|
|
#include <Common/MemoryTracker.h>
|
2022-03-01 07:54:23 +00:00
|
|
|
#include <Common/CurrentThread.h>
|
2023-01-21 10:24:57 +00:00
|
|
|
#include <Common/ConcurrentBoundedQueue.h>
|
2018-06-20 17:49:52 +00:00
|
|
|
#include <Columns/ColumnsNumber.h>
|
|
|
|
#include <Columns/ColumnString.h>
|
|
|
|
#include <Columns/ColumnArray.h>
|
2021-01-05 10:10:19 +00:00
|
|
|
#include <Columns/ColumnMap.h>
|
2018-06-20 17:49:52 +00:00
|
|
|
#include <DataTypes/DataTypesNumber.h>
|
|
|
|
#include <DataTypes/DataTypeString.h>
|
|
|
|
#include <DataTypes/DataTypeArray.h>
|
2023-03-24 02:44:52 +00:00
|
|
|
#include <DataTypes/DataTypeDateTime.h>
|
2018-06-20 17:49:52 +00:00
|
|
|
|
|
|
|
namespace ProfileEvents
|
|
|
|
{
|
|
|
|
|
2021-10-13 20:52:39 +00:00
|
|
|
std::shared_ptr<DB::DataTypeEnum8> TypeEnum = std::make_shared<DB::DataTypeEnum8>(DB::DataTypeEnum8::Values{
|
|
|
|
{ "increment", static_cast<Int8>(INCREMENT)},
|
|
|
|
{ "gauge", static_cast<Int8>(GAUGE)},
|
|
|
|
});
|
|
|
|
|
2018-06-20 17:49:52 +00:00
|
|
|
/// Put implementation here to avoid extra linking dependencies for clickhouse_common_io
|
2021-10-11 15:56:23 +00:00
|
|
|
void dumpToMapColumn(const Counters::Snapshot & counters, DB::IColumn * column, bool nonzero_only)
|
2018-06-20 17:49:52 +00:00
|
|
|
{
|
2021-01-05 10:10:19 +00:00
|
|
|
auto * column_map = column ? &typeid_cast<DB::ColumnMap &>(*column) : nullptr;
|
|
|
|
if (!column_map)
|
|
|
|
return;
|
2018-06-20 17:49:52 +00:00
|
|
|
|
2021-01-05 10:10:19 +00:00
|
|
|
auto & offsets = column_map->getNestedColumn().getOffsets();
|
|
|
|
auto & tuple_column = column_map->getNestedData();
|
|
|
|
auto & key_column = tuple_column.getColumn(0);
|
|
|
|
auto & value_column = tuple_column.getColumn(1);
|
2018-06-20 17:49:52 +00:00
|
|
|
|
2021-01-05 10:10:19 +00:00
|
|
|
size_t size = 0;
|
2023-04-01 09:52:23 +00:00
|
|
|
for (Event event = Event(0); event < Counters::num_counters; ++event)
|
2018-06-20 17:49:52 +00:00
|
|
|
{
|
2021-10-11 15:56:23 +00:00
|
|
|
UInt64 value = counters[event];
|
2018-06-20 17:49:52 +00:00
|
|
|
|
|
|
|
if (nonzero_only && 0 == value)
|
|
|
|
continue;
|
|
|
|
|
2022-02-15 12:26:53 +00:00
|
|
|
const char * desc = getName(event);
|
2021-01-05 10:10:19 +00:00
|
|
|
key_column.insertData(desc, strlen(desc));
|
|
|
|
value_column.insert(value);
|
|
|
|
size++;
|
2018-06-20 17:49:52 +00:00
|
|
|
}
|
2021-06-28 11:42:21 +00:00
|
|
|
|
2021-01-05 10:10:19 +00:00
|
|
|
offsets.push_back(offsets.back() + size);
|
2018-06-20 17:49:52 +00:00
|
|
|
}
|
|
|
|
|
2022-03-01 07:54:23 +00:00
|
|
|
/// Add records about provided non-zero ProfileEvents::Counters.
|
|
|
|
static void dumpProfileEvents(ProfileEventsSnapshot const & snapshot, DB::MutableColumns & columns, String const & host_name)
|
2022-02-15 08:25:07 +00:00
|
|
|
{
|
|
|
|
size_t rows = 0;
|
|
|
|
auto & name_column = columns[NAME_COLUMN_INDEX];
|
|
|
|
auto & value_column = columns[VALUE_COLUMN_INDEX];
|
2023-04-01 09:52:23 +00:00
|
|
|
for (Event event = Event(0); event < Counters::num_counters; ++event)
|
2022-02-15 08:25:07 +00:00
|
|
|
{
|
|
|
|
Int64 value = snapshot.counters[event];
|
|
|
|
|
|
|
|
if (value == 0)
|
|
|
|
continue;
|
|
|
|
|
2022-02-15 12:26:53 +00:00
|
|
|
const char * desc = getName(event);
|
2022-02-15 08:25:07 +00:00
|
|
|
name_column->insertData(desc, strlen(desc));
|
|
|
|
value_column->insert(value);
|
|
|
|
rows++;
|
|
|
|
}
|
|
|
|
|
|
|
|
// Fill the rest of the columns with data
|
|
|
|
for (size_t row = 0; row < rows; ++row)
|
|
|
|
{
|
|
|
|
size_t i = 0;
|
|
|
|
columns[i++]->insertData(host_name.data(), host_name.size());
|
2022-04-18 08:18:31 +00:00
|
|
|
columns[i++]->insert(static_cast<UInt64>(snapshot.current_time));
|
2022-02-15 08:25:07 +00:00
|
|
|
columns[i++]->insert(UInt64{snapshot.thread_id});
|
2022-02-15 12:26:53 +00:00
|
|
|
columns[i++]->insert(Type::INCREMENT);
|
2022-02-15 08:25:07 +00:00
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2022-03-01 07:54:23 +00:00
|
|
|
static void dumpMemoryTracker(ProfileEventsSnapshot const & snapshot, DB::MutableColumns & columns, String const & host_name)
|
2022-02-15 08:25:07 +00:00
|
|
|
{
|
2022-03-05 06:22:56 +00:00
|
|
|
size_t i = 0;
|
|
|
|
columns[i++]->insertData(host_name.data(), host_name.size());
|
2022-04-18 08:18:31 +00:00
|
|
|
columns[i++]->insert(static_cast<UInt64>(snapshot.current_time));
|
2022-04-20 11:33:15 +00:00
|
|
|
columns[i++]->insert(static_cast<UInt64>(snapshot.thread_id));
|
2022-03-05 06:22:56 +00:00
|
|
|
columns[i++]->insert(Type::GAUGE);
|
|
|
|
columns[i++]->insertData(MemoryTracker::USAGE_EVENT_NAME, strlen(MemoryTracker::USAGE_EVENT_NAME));
|
2023-06-28 09:31:44 +00:00
|
|
|
columns[i]->insert(snapshot.memory_usage);
|
|
|
|
|
|
|
|
i = 0;
|
|
|
|
columns[i++]->insertData(host_name.data(), host_name.size());
|
|
|
|
columns[i++]->insert(static_cast<UInt64>(snapshot.current_time));
|
|
|
|
columns[i++]->insert(static_cast<UInt64>(snapshot.thread_id));
|
|
|
|
columns[i++]->insert(Type::GAUGE);
|
|
|
|
columns[i++]->insertData(MemoryTracker::PEAK_USAGE_EVENT_NAME, strlen(MemoryTracker::PEAK_USAGE_EVENT_NAME));
|
|
|
|
columns[i]->insert(snapshot.peak_memory_usage);
|
2022-02-15 08:25:07 +00:00
|
|
|
}
|
|
|
|
|
2022-03-01 07:54:23 +00:00
|
|
|
void getProfileEvents(
|
2024-01-30 20:27:30 +00:00
|
|
|
const String & host_name,
|
2022-03-01 07:54:23 +00:00
|
|
|
DB::InternalProfileEventsQueuePtr profile_queue,
|
|
|
|
DB::Block & block,
|
|
|
|
ThreadIdToCountersSnapshot & last_sent_snapshots)
|
|
|
|
{
|
|
|
|
using namespace DB;
|
|
|
|
static const NamesAndTypesList column_names_and_types = {
|
|
|
|
{"host_name", std::make_shared<DataTypeString>()},
|
|
|
|
{"current_time", std::make_shared<DataTypeDateTime>()},
|
|
|
|
{"thread_id", std::make_shared<DataTypeUInt64>()},
|
|
|
|
{"type", TypeEnum},
|
|
|
|
{"name", std::make_shared<DataTypeString>()},
|
|
|
|
{"value", std::make_shared<DataTypeInt64>()},
|
|
|
|
};
|
|
|
|
|
2022-04-12 17:14:28 +00:00
|
|
|
ColumnsWithTypeAndName temp_columns;
|
2022-03-01 07:54:23 +00:00
|
|
|
for (auto const & name_and_type : column_names_and_types)
|
|
|
|
temp_columns.emplace_back(name_and_type.type, name_and_type.name);
|
|
|
|
|
|
|
|
block = std::move(temp_columns);
|
|
|
|
MutableColumns columns = block.mutateColumns();
|
|
|
|
auto thread_group = CurrentThread::getGroup();
|
|
|
|
ThreadIdToCountersSnapshot new_snapshots;
|
2023-03-14 11:36:26 +00:00
|
|
|
|
2022-03-01 07:54:23 +00:00
|
|
|
ProfileEventsSnapshot group_snapshot;
|
|
|
|
{
|
|
|
|
group_snapshot.thread_id = 0;
|
|
|
|
group_snapshot.current_time = time(nullptr);
|
|
|
|
group_snapshot.memory_usage = thread_group->memory_tracker.get();
|
2023-06-28 09:31:44 +00:00
|
|
|
group_snapshot.peak_memory_usage = thread_group->memory_tracker.getPeak();
|
2022-03-01 07:54:23 +00:00
|
|
|
auto group_counters = thread_group->performance_counters.getPartiallyAtomicSnapshot();
|
|
|
|
auto prev_group_snapshot = last_sent_snapshots.find(0);
|
|
|
|
group_snapshot.counters =
|
|
|
|
prev_group_snapshot != last_sent_snapshots.end()
|
|
|
|
? CountersIncrement(group_counters, prev_group_snapshot->second)
|
|
|
|
: CountersIncrement(group_counters);
|
|
|
|
new_snapshots[0] = std::move(group_counters);
|
|
|
|
}
|
|
|
|
last_sent_snapshots = std::move(new_snapshots);
|
|
|
|
|
2024-01-30 20:27:30 +00:00
|
|
|
dumpProfileEvents(group_snapshot, columns, host_name);
|
|
|
|
dumpMemoryTracker(group_snapshot, columns, host_name);
|
2022-03-01 07:54:23 +00:00
|
|
|
|
|
|
|
Block curr_block;
|
|
|
|
|
2022-08-01 16:00:54 +00:00
|
|
|
while (profile_queue->tryPop(curr_block))
|
2022-03-01 07:54:23 +00:00
|
|
|
{
|
|
|
|
auto curr_columns = curr_block.getColumns();
|
|
|
|
for (size_t j = 0; j < curr_columns.size(); ++j)
|
|
|
|
columns[j]->insertRangeFrom(*curr_columns[j], 0, curr_columns[j]->size());
|
|
|
|
}
|
|
|
|
|
|
|
|
bool empty = columns[0]->empty();
|
|
|
|
if (!empty)
|
|
|
|
block.setColumns(std::move(columns));
|
|
|
|
}
|
|
|
|
|
2018-06-20 17:49:52 +00:00
|
|
|
}
|