ClickHouse/dbms/Interpreters/PartLog.cpp

153 lines
5.5 KiB
C++
Raw Normal View History

#include <Columns/ColumnsNumber.h>
#include <Columns/ColumnArray.h>
#include <Columns/ColumnString.h>
#include <DataTypes/DataTypeArray.h>
#include <DataTypes/DataTypesNumber.h>
#include <DataTypes/DataTypeDateTime.h>
#include <DataTypes/DataTypeDate.h>
#include <DataTypes/DataTypeString.h>
#include <DataTypes/DataTypeEnum.h>
2019-10-10 16:30:30 +00:00
#include <Storages/MergeTree/IMergeTreeDataPart.h>
#include <Storages/MergeTree/MergeTreeData.h>
#include <Interpreters/PartLog.h>
namespace DB
{
Block PartLogElement::createBlock()
{
auto event_type_datatype = std::make_shared<DataTypeEnum8>(
2018-09-09 02:23:24 +00:00
DataTypeEnum8::Values
{
2019-08-01 16:37:50 +00:00
{"NewPart", static_cast<Int8>(NEW_PART)},
{"MergeParts", static_cast<Int8>(MERGE_PARTS)},
{"DownloadPart", static_cast<Int8>(DOWNLOAD_PART)},
{"RemovePart", static_cast<Int8>(REMOVE_PART)},
{"MutatePart", static_cast<Int8>(MUTATE_PART)},
2019-09-03 11:32:25 +00:00
{"MovePart", static_cast<Int8>(MOVE_PART)},
}
);
return
{
2019-09-06 12:18:56 +00:00
{ColumnInt8::create(), std::move(event_type_datatype), "event_type"},
2018-11-26 00:56:50 +00:00
{ColumnUInt16::create(), std::make_shared<DataTypeDate>(), "event_date"},
{ColumnUInt32::create(), std::make_shared<DataTypeDateTime>(), "event_time"},
{ColumnUInt64::create(), std::make_shared<DataTypeUInt64>(), "duration_ms"},
2018-11-26 00:56:50 +00:00
{ColumnString::create(), std::make_shared<DataTypeString>(), "database"},
{ColumnString::create(), std::make_shared<DataTypeString>(), "table"},
{ColumnString::create(), std::make_shared<DataTypeString>(), "part_name"},
{ColumnString::create(), std::make_shared<DataTypeString>(), "partition_id"},
2019-09-03 11:32:25 +00:00
{ColumnString::create(), std::make_shared<DataTypeString>(), "path_on_disk"},
2018-11-26 00:56:50 +00:00
{ColumnUInt64::create(), std::make_shared<DataTypeUInt64>(), "rows"},
{ColumnUInt64::create(), std::make_shared<DataTypeUInt64>(), "size_in_bytes"}, // On disk
/// Merge-specific info
{ColumnArray::create(ColumnString::create()), std::make_shared<DataTypeArray>(std::make_shared<DataTypeString>()), "merged_from"},
2018-11-26 00:56:50 +00:00
{ColumnUInt64::create(), std::make_shared<DataTypeUInt64>(), "bytes_uncompressed"}, // Result bytes
{ColumnUInt64::create(), std::make_shared<DataTypeUInt64>(), "read_rows"},
{ColumnUInt64::create(), std::make_shared<DataTypeUInt64>(), "read_bytes"},
2020-03-19 11:31:21 +00:00
{ColumnUInt64::create(), std::make_shared<DataTypeUInt64>(), "peak_memory_usage"},
/// Is there an error during the execution or commit
2018-11-26 00:56:50 +00:00
{ColumnUInt16::create(), std::make_shared<DataTypeUInt16>(), "error"},
{ColumnString::create(), std::make_shared<DataTypeString>(), "exception"},
2018-05-07 02:01:11 +00:00
};
}
void PartLogElement::appendToBlock(Block & block) const
{
MutableColumns columns = block.mutateColumns();
size_t i = 0;
columns[i++]->insert(event_type);
columns[i++]->insert(DateLUT::instance().toDayNum(event_time));
columns[i++]->insert(event_time);
columns[i++]->insert(duration_ms);
columns[i++]->insert(database_name);
columns[i++]->insert(table_name);
columns[i++]->insert(part_name);
2018-09-13 12:48:46 +00:00
columns[i++]->insert(partition_id);
2019-09-03 11:32:25 +00:00
columns[i++]->insert(path_on_disk);
columns[i++]->insert(rows);
columns[i++]->insert(bytes_compressed_on_disk);
Array source_part_names_array;
source_part_names_array.reserve(source_part_names.size());
for (const auto & name : source_part_names)
source_part_names_array.push_back(name);
columns[i++]->insert(source_part_names_array);
columns[i++]->insert(bytes_uncompressed);
columns[i++]->insert(rows_read);
columns[i++]->insert(bytes_read_uncompressed);
2020-03-19 11:31:21 +00:00
columns[i++]->insert(peak_memory_usage);
columns[i++]->insert(error);
columns[i++]->insert(exception);
block.setColumns(std::move(columns));
}
bool PartLog::addNewPart(Context & current_context, const MutableDataPartPtr & part, UInt64 elapsed_ns, const ExecutionStatus & execution_status)
{
return addNewParts(current_context, {part}, elapsed_ns, execution_status);
}
bool PartLog::addNewParts(Context & current_context, const PartLog::MutableDataPartsVector & parts, UInt64 elapsed_ns,
const ExecutionStatus & execution_status)
{
if (parts.empty())
return true;
2019-03-21 19:22:38 +00:00
std::shared_ptr<PartLog> part_log;
try
{
auto table_id = parts.front()->storage.getStorageID();
part_log = current_context.getPartLog(table_id.database_name); // assume parts belong to the same table
if (!part_log)
return false;
for (const auto & part : parts)
{
PartLogElement elem;
elem.event_type = PartLogElement::NEW_PART;
elem.event_time = time(nullptr);
elem.duration_ms = elapsed_ns / 1000000;
elem.database_name = table_id.database_name;
elem.table_name = table_id.table_name;
2018-09-13 12:48:46 +00:00
elem.partition_id = part->info.partition_id;
2019-01-31 17:30:56 +00:00
elem.part_name = part->name;
2019-09-03 11:32:25 +00:00
elem.path_on_disk = part->getFullPath();
2020-03-23 13:32:02 +00:00
elem.bytes_compressed_on_disk = part->getBytesOnDisk();
elem.rows = part->rows_count;
elem.error = static_cast<UInt16>(execution_status.code);
elem.exception = execution_status.message;
part_log->add(elem);
}
}
catch (...)
{
tryLogCurrentException(part_log ? part_log->log : &Logger::get("PartLog"), __PRETTY_FUNCTION__);
return false;
}
return true;
}
}