ClickHouse/src/Storages/System/StorageSystemParts.cpp

317 lines
16 KiB
C++
Raw Normal View History

#include "StorageSystemParts.h"
#include <Common/escapeForFileName.h>
#include <Columns/ColumnString.h>
#include <DataTypes/DataTypeString.h>
#include <DataTypes/DataTypeArray.h>
#include <DataTypes/DataTypesNumber.h>
#include <DataTypes/DataTypeDateTime.h>
#include <DataTypes/DataTypeDate.h>
#include <DataTypes/DataTypeUUID.h>
2017-11-20 05:22:54 +00:00
#include <Storages/VirtualColumnUtils.h>
#include <Databases/IDatabase.h>
#include <Parsers/queryToString.h>
2019-07-05 15:32:55 +00:00
#include <Common/hex.h>
2021-11-08 18:56:09 +00:00
#include <Interpreters/TransactionVersionMetadata.h>
2022-07-20 20:30:16 +00:00
#include <Interpreters/Context.h>
namespace DB
{
2020-08-12 20:40:13 +00:00
StorageSystemParts::StorageSystemParts(const StorageID & table_id_)
: StorageSystemPartsBase(table_id_,
{
{"partition", std::make_shared<DataTypeString>()},
{"name", std::make_shared<DataTypeString>()},
{"uuid", std::make_shared<DataTypeUUID>()},
{"part_type", std::make_shared<DataTypeString>()},
{"active", std::make_shared<DataTypeUInt8>()},
{"marks", std::make_shared<DataTypeUInt64>()},
{"rows", std::make_shared<DataTypeUInt64>()},
{"bytes_on_disk", std::make_shared<DataTypeUInt64>()},
{"data_compressed_bytes", std::make_shared<DataTypeUInt64>()},
{"data_uncompressed_bytes", std::make_shared<DataTypeUInt64>()},
{"marks_bytes", std::make_shared<DataTypeUInt64>()},
{"secondary_indices_compressed_bytes", std::make_shared<DataTypeUInt64>()},
{"secondary_indices_uncompressed_bytes", std::make_shared<DataTypeUInt64>()},
{"secondary_indices_marks_bytes", std::make_shared<DataTypeUInt64>()},
{"modification_time", std::make_shared<DataTypeDateTime>()},
{"remove_time", std::make_shared<DataTypeDateTime>()},
{"refcount", std::make_shared<DataTypeUInt32>()},
{"min_date", std::make_shared<DataTypeDate>()},
{"max_date", std::make_shared<DataTypeDate>()},
{"min_time", std::make_shared<DataTypeDateTime>()},
{"max_time", std::make_shared<DataTypeDateTime>()},
{"partition_id", std::make_shared<DataTypeString>()},
{"min_block_number", std::make_shared<DataTypeInt64>()},
{"max_block_number", std::make_shared<DataTypeInt64>()},
{"level", std::make_shared<DataTypeUInt32>()},
{"data_version", std::make_shared<DataTypeUInt64>()},
{"primary_key_bytes_in_memory", std::make_shared<DataTypeUInt64>()},
{"primary_key_bytes_in_memory_allocated", std::make_shared<DataTypeUInt64>()},
{"is_frozen", std::make_shared<DataTypeUInt8>()},
{"database", std::make_shared<DataTypeString>()},
{"table", std::make_shared<DataTypeString>()},
{"engine", std::make_shared<DataTypeString>()},
{"disk_name", std::make_shared<DataTypeString>()},
{"path", std::make_shared<DataTypeString>()},
{"hash_of_all_files", std::make_shared<DataTypeString>()},
{"hash_of_uncompressed_files", std::make_shared<DataTypeString>()},
{"uncompressed_hash_of_compressed_files", std::make_shared<DataTypeString>()},
2020-05-18 16:53:14 +00:00
{"delete_ttl_info_min", std::make_shared<DataTypeDateTime>()},
{"delete_ttl_info_max", std::make_shared<DataTypeDateTime>()},
{"move_ttl_info.expression", std::make_shared<DataTypeArray>(std::make_shared<DataTypeString>())},
{"move_ttl_info.min", std::make_shared<DataTypeArray>(std::make_shared<DataTypeDateTime>())},
{"move_ttl_info.max", std::make_shared<DataTypeArray>(std::make_shared<DataTypeDateTime>())},
{"default_compression_codec", std::make_shared<DataTypeString>()},
{"recompression_ttl_info.expression", std::make_shared<DataTypeArray>(std::make_shared<DataTypeString>())},
{"recompression_ttl_info.min", std::make_shared<DataTypeArray>(std::make_shared<DataTypeDateTime>())},
{"recompression_ttl_info.max", std::make_shared<DataTypeArray>(std::make_shared<DataTypeDateTime>())},
2020-12-25 14:52:46 +00:00
{"group_by_ttl_info.expression", std::make_shared<DataTypeArray>(std::make_shared<DataTypeString>())},
{"group_by_ttl_info.min", std::make_shared<DataTypeArray>(std::make_shared<DataTypeDateTime>())},
{"group_by_ttl_info.max", std::make_shared<DataTypeArray>(std::make_shared<DataTypeDateTime>())},
{"rows_where_ttl_info.expression", std::make_shared<DataTypeArray>(std::make_shared<DataTypeString>())},
{"rows_where_ttl_info.min", std::make_shared<DataTypeArray>(std::make_shared<DataTypeDateTime>())},
2021-08-26 11:01:15 +00:00
{"rows_where_ttl_info.max", std::make_shared<DataTypeArray>(std::make_shared<DataTypeDateTime>())},
{"projections", std::make_shared<DataTypeArray>(std::make_shared<DataTypeString>())},
2021-06-02 20:03:44 +00:00
{"visible", std::make_shared<DataTypeUInt8>()},
2022-02-14 19:47:17 +00:00
{"creation_tid", getTransactionIDDataType()},
2022-05-20 20:08:46 +00:00
{"removal_tid_lock", std::make_shared<DataTypeUInt64>()},
2022-02-14 19:47:17 +00:00
{"removal_tid", getTransactionIDDataType()},
{"creation_csn", std::make_shared<DataTypeUInt64>()},
{"removal_csn", std::make_shared<DataTypeUInt64>()},
}
)
{
}
void StorageSystemParts::processNextStorage(
2021-06-02 20:03:44 +00:00
ContextPtr context, MutableColumns & columns, std::vector<UInt8> & columns_mask, const StoragesInfo & info, bool has_state_column)
{
2022-08-12 11:03:57 +00:00
using State = MergeTreeDataPartState;
MergeTreeData::DataPartStateVector all_parts_state;
MergeTreeData::DataPartsVector all_parts;
all_parts = info.getParts(all_parts_state, has_state_column);
for (size_t part_number = 0; part_number < all_parts.size(); ++part_number)
{
const auto & part = all_parts[part_number];
auto part_state = all_parts_state[part_number];
ColumnSize columns_size = part->getTotalColumnsSize();
ColumnSize secondary_indexes_size = part->getTotalSeconaryIndicesSize();
size_t src_index = 0, res_index = 0;
if (columns_mask[src_index++])
{
WriteBufferFromOwnString out;
part->partition.serializeText(*info.data, out, format_settings);
columns[res_index++]->insert(out.str());
}
if (columns_mask[src_index++])
columns[res_index++]->insert(part->name);
if (columns_mask[src_index++])
columns[res_index++]->insert(part->uuid);
if (columns_mask[src_index++])
columns[res_index++]->insert(part->getTypeName());
if (columns_mask[src_index++])
2021-12-30 14:27:22 +00:00
columns[res_index++]->insert(part_state == State::Active);
if (columns_mask[src_index++])
columns[res_index++]->insert(part->getMarksCount());
if (columns_mask[src_index++])
columns[res_index++]->insert(part->rows_count);
if (columns_mask[src_index++])
columns[res_index++]->insert(part->getBytesOnDisk());
if (columns_mask[src_index++])
columns[res_index++]->insert(columns_size.data_compressed);
if (columns_mask[src_index++])
columns[res_index++]->insert(columns_size.data_uncompressed);
if (columns_mask[src_index++])
columns[res_index++]->insert(columns_size.marks);
if (columns_mask[src_index++])
columns[res_index++]->insert(secondary_indexes_size.data_compressed);
if (columns_mask[src_index++])
columns[res_index++]->insert(secondary_indexes_size.data_uncompressed);
if (columns_mask[src_index++])
columns[res_index++]->insert(secondary_indexes_size.marks);
if (columns_mask[src_index++])
columns[res_index++]->insert(static_cast<UInt64>(part->modification_time));
if (columns_mask[src_index++])
{
time_t remove_time = part->remove_time.load(std::memory_order_relaxed);
columns[res_index++]->insert(static_cast<UInt64>(remove_time == std::numeric_limits<time_t>::max() ? 0 : remove_time));
}
/// For convenience, in returned refcount, don't add references that was due to local variables in this method: all_parts, active_parts.
if (columns_mask[src_index++])
columns[res_index++]->insert(static_cast<UInt64>(part.use_count() - 1));
auto min_max_date = part->getMinMaxDate();
auto min_max_time = part->getMinMaxTime();
if (columns_mask[src_index++])
columns[res_index++]->insert(min_max_date.first);
if (columns_mask[src_index++])
columns[res_index++]->insert(min_max_date.second);
if (columns_mask[src_index++])
columns[res_index++]->insert(static_cast<UInt32>(min_max_time.first));
if (columns_mask[src_index++])
columns[res_index++]->insert(static_cast<UInt32>(min_max_time.second));
if (columns_mask[src_index++])
columns[res_index++]->insert(part->info.partition_id);
if (columns_mask[src_index++])
columns[res_index++]->insert(part->info.min_block);
if (columns_mask[src_index++])
columns[res_index++]->insert(part->info.max_block);
if (columns_mask[src_index++])
columns[res_index++]->insert(part->info.level);
if (columns_mask[src_index++])
columns[res_index++]->insert(static_cast<UInt64>(part->info.getDataVersion()));
if (columns_mask[src_index++])
columns[res_index++]->insert(part->getIndexSizeInBytes());
if (columns_mask[src_index++])
columns[res_index++]->insert(part->getIndexSizeInAllocatedBytes());
if (columns_mask[src_index++])
columns[res_index++]->insert(part->is_frozen.load(std::memory_order_relaxed));
if (columns_mask[src_index++])
columns[res_index++]->insert(info.database);
if (columns_mask[src_index++])
columns[res_index++]->insert(info.table);
if (columns_mask[src_index++])
columns[res_index++]->insert(info.engine);
2020-04-29 17:14:49 +00:00
if (part->isStoredOnDisk())
{
if (columns_mask[src_index++])
2022-06-20 18:18:17 +00:00
columns[res_index++]->insert(part->data_part_storage->getDiskName());
if (columns_mask[src_index++])
2022-04-22 16:58:09 +00:00
columns[res_index++]->insert(part->data_part_storage->getFullPath());
2020-04-29 17:14:49 +00:00
}
else
{
if (columns_mask[src_index++])
columns[res_index++]->insertDefault();
if (columns_mask[src_index++])
columns[res_index++]->insertDefault();
2020-04-29 17:14:49 +00:00
}
2019-07-05 15:32:55 +00:00
{
MinimalisticDataPartChecksums helper;
if (columns_mask[src_index] || columns_mask[src_index + 1] || columns_mask[src_index + 2])
helper.computeTotalChecksums(part->checksums);
2019-07-05 15:32:55 +00:00
if (columns_mask[src_index++])
{
auto checksum = helper.hash_of_all_files;
columns[res_index++]->insert(getHexUIntLowercase(checksum.first) + getHexUIntLowercase(checksum.second));
}
if (columns_mask[src_index++])
{
auto checksum = helper.hash_of_uncompressed_files;
columns[res_index++]->insert(getHexUIntLowercase(checksum.first) + getHexUIntLowercase(checksum.second));
}
if (columns_mask[src_index++])
{
auto checksum = helper.uncompressed_hash_of_compressed_files;
columns[res_index++]->insert(getHexUIntLowercase(checksum.first) + getHexUIntLowercase(checksum.second));
}
}
/// delete_ttl_info
if (columns_mask[src_index++])
columns[res_index++]->insert(static_cast<UInt32>(part->ttl_infos.table_ttl.min));
if (columns_mask[src_index++])
columns[res_index++]->insert(static_cast<UInt32>(part->ttl_infos.table_ttl.max));
auto add_ttl_info_map = [&](const TTLInfoMap & ttl_info_map)
{
2020-05-21 09:45:44 +00:00
Array expression_array;
Array min_array;
Array max_array;
if (columns_mask[src_index])
expression_array.reserve(ttl_info_map.size());
if (columns_mask[src_index + 1])
min_array.reserve(ttl_info_map.size());
if (columns_mask[src_index + 2])
max_array.reserve(ttl_info_map.size());
for (const auto & [expression, ttl_info] : ttl_info_map)
{
if (columns_mask[src_index])
expression_array.emplace_back(expression);
if (columns_mask[src_index + 1])
min_array.push_back(static_cast<UInt32>(ttl_info.min));
if (columns_mask[src_index + 2])
max_array.push_back(static_cast<UInt32>(ttl_info.max));
}
if (columns_mask[src_index++])
columns[res_index++]->insert(expression_array);
if (columns_mask[src_index++])
columns[res_index++]->insert(min_array);
if (columns_mask[src_index++])
columns[res_index++]->insert(max_array);
};
add_ttl_info_map(part->ttl_infos.moves_ttl);
if (columns_mask[src_index++])
columns[res_index++]->insert(queryToString(part->default_codec->getCodecDesc()));
add_ttl_info_map(part->ttl_infos.recompression_ttl);
2020-12-25 14:52:46 +00:00
add_ttl_info_map(part->ttl_infos.group_by_ttl);
add_ttl_info_map(part->ttl_infos.rows_where_ttl);
2021-08-26 11:01:15 +00:00
Array projections;
for (const auto & [name, _] : part->getProjectionParts())
projections.push_back(name);
if (columns_mask[src_index++])
columns[res_index++]->insert(projections);
2021-06-02 20:03:44 +00:00
if (columns_mask[src_index++])
{
auto txn = context->getCurrentTransaction();
if (txn)
columns[res_index++]->insert(part->version.isVisible(*txn));
2021-06-02 20:03:44 +00:00
else
columns[res_index++]->insert(part_state == State::Active);
2021-06-02 20:03:44 +00:00
}
2021-03-31 17:55:04 +00:00
auto get_tid_as_field = [](const TransactionID & tid) -> Field
{
return Tuple{tid.start_csn, tid.local_tid, tid.host_id};
};
if (columns_mask[src_index++])
columns[res_index++]->insert(get_tid_as_field(part->version.creation_tid));
2022-05-20 20:08:46 +00:00
if (columns_mask[src_index++])
columns[res_index++]->insert(part->version.removal_tid_lock.load(std::memory_order_relaxed));
2021-03-31 17:55:04 +00:00
if (columns_mask[src_index++])
columns[res_index++]->insert(get_tid_as_field(part->version.getRemovalTID()));
2021-03-31 17:55:04 +00:00
if (columns_mask[src_index++])
columns[res_index++]->insert(part->version.creation_csn.load(std::memory_order_relaxed));
2021-03-31 17:55:04 +00:00
if (columns_mask[src_index++])
columns[res_index++]->insert(part->version.removal_csn.load(std::memory_order_relaxed));
2021-05-18 19:34:49 +00:00
/// _state column should be the latest.
2021-02-09 08:51:17 +00:00
/// Do not use part->getState*, it can be changed from different thread
if (has_state_column)
columns[res_index++]->insert(IMergeTreeDataPart::stateString(part_state));
}
}
}