2019-05-31 04:03:46 +00:00
|
|
|
#include "StorageSystemParts.h"
|
|
|
|
|
2017-05-31 11:46:22 +00:00
|
|
|
#include <Common/escapeForFileName.h>
|
2017-04-01 09:19:00 +00:00
|
|
|
#include <Columns/ColumnString.h>
|
|
|
|
#include <DataTypes/DataTypeString.h>
|
|
|
|
#include <DataTypes/DataTypesNumber.h>
|
|
|
|
#include <DataTypes/DataTypeDateTime.h>
|
|
|
|
#include <DataTypes/DataTypeDate.h>
|
|
|
|
#include <DataStreams/OneBlockInputStream.h>
|
2017-11-20 05:22:54 +00:00
|
|
|
#include <Storages/VirtualColumnUtils.h>
|
2017-04-01 09:19:00 +00:00
|
|
|
#include <Databases/IDatabase.h>
|
2019-07-05 15:32:55 +00:00
|
|
|
#include <Common/hex.h>
|
2014-07-29 14:05:15 +00:00
|
|
|
|
|
|
|
namespace DB
|
|
|
|
{
|
|
|
|
|
2017-12-12 15:54:03 +00:00
|
|
|
StorageSystemParts::StorageSystemParts(const std::string & name)
|
|
|
|
: StorageSystemPartsBase(name,
|
2017-04-01 07:20:54 +00:00
|
|
|
{
|
2018-03-26 14:18:04 +00:00
|
|
|
{"partition", std::make_shared<DataTypeString>()},
|
|
|
|
{"name", std::make_shared<DataTypeString>()},
|
|
|
|
{"active", std::make_shared<DataTypeUInt8>()},
|
|
|
|
{"marks", std::make_shared<DataTypeUInt64>()},
|
|
|
|
{"rows", std::make_shared<DataTypeUInt64>()},
|
|
|
|
{"bytes_on_disk", std::make_shared<DataTypeUInt64>()},
|
|
|
|
{"data_compressed_bytes", std::make_shared<DataTypeUInt64>()},
|
|
|
|
{"data_uncompressed_bytes", std::make_shared<DataTypeUInt64>()},
|
|
|
|
{"marks_bytes", std::make_shared<DataTypeUInt64>()},
|
|
|
|
{"modification_time", std::make_shared<DataTypeDateTime>()},
|
|
|
|
{"remove_time", std::make_shared<DataTypeDateTime>()},
|
|
|
|
{"refcount", std::make_shared<DataTypeUInt32>()},
|
|
|
|
{"min_date", std::make_shared<DataTypeDate>()},
|
|
|
|
{"max_date", std::make_shared<DataTypeDate>()},
|
2018-12-10 10:20:19 +00:00
|
|
|
{"min_time", std::make_shared<DataTypeDateTime>()},
|
|
|
|
{"max_time", std::make_shared<DataTypeDateTime>()},
|
2018-06-13 15:47:40 +00:00
|
|
|
{"partition_id", std::make_shared<DataTypeString>()},
|
2018-03-26 14:18:04 +00:00
|
|
|
{"min_block_number", std::make_shared<DataTypeInt64>()},
|
|
|
|
{"max_block_number", std::make_shared<DataTypeInt64>()},
|
|
|
|
{"level", std::make_shared<DataTypeUInt32>()},
|
2018-06-13 15:47:40 +00:00
|
|
|
{"data_version", std::make_shared<DataTypeUInt64>()},
|
2018-03-26 14:18:04 +00:00
|
|
|
{"primary_key_bytes_in_memory", std::make_shared<DataTypeUInt64>()},
|
|
|
|
{"primary_key_bytes_in_memory_allocated", std::make_shared<DataTypeUInt64>()},
|
2019-05-31 04:03:46 +00:00
|
|
|
{"is_frozen", std::make_shared<DataTypeUInt8>()},
|
2017-04-01 07:20:54 +00:00
|
|
|
|
2018-03-26 14:18:04 +00:00
|
|
|
{"database", std::make_shared<DataTypeString>()},
|
|
|
|
{"table", std::make_shared<DataTypeString>()},
|
|
|
|
{"engine", std::make_shared<DataTypeString>()},
|
|
|
|
{"path", std::make_shared<DataTypeString>()},
|
2019-07-05 15:32:55 +00:00
|
|
|
|
|
|
|
{"hash_of_all_files", std::make_shared<DataTypeString>()},
|
|
|
|
{"hash_of_uncompressed_files", std::make_shared<DataTypeString>()},
|
|
|
|
{"uncompressed_hash_of_compressed_files", std::make_shared<DataTypeString>()}
|
2017-04-01 07:20:54 +00:00
|
|
|
}
|
2017-12-12 15:54:03 +00:00
|
|
|
)
|
2014-07-29 14:05:15 +00:00
|
|
|
{
|
|
|
|
}
|
|
|
|
|
2017-12-28 18:20:53 +00:00
|
|
|
void StorageSystemParts::processNextStorage(MutableColumns & columns, const StoragesInfo & info, bool has_state_column)
|
2014-07-29 14:05:15 +00:00
|
|
|
{
|
2017-12-12 15:54:03 +00:00
|
|
|
using State = MergeTreeDataPart::State;
|
2019-05-20 16:24:36 +00:00
|
|
|
MergeTreeData::DataPartStateVector all_parts_state;
|
|
|
|
MergeTreeData::DataPartsVector all_parts;
|
2017-04-01 07:20:54 +00:00
|
|
|
|
2019-05-20 16:24:36 +00:00
|
|
|
all_parts = info.getParts(all_parts_state, has_state_column);
|
|
|
|
|
|
|
|
for (size_t part_number = 0; part_number < all_parts.size(); ++part_number)
|
2017-04-01 07:20:54 +00:00
|
|
|
{
|
2019-05-20 16:24:36 +00:00
|
|
|
const auto & part = all_parts[part_number];
|
|
|
|
auto part_state = all_parts_state[part_number];
|
|
|
|
|
2019-07-16 17:13:12 +00:00
|
|
|
ColumnSize columns_size = part->getTotalColumnsSize();
|
2017-04-01 07:20:54 +00:00
|
|
|
|
2017-12-12 15:54:03 +00:00
|
|
|
size_t i = 0;
|
2017-04-01 07:20:54 +00:00
|
|
|
{
|
2017-12-12 15:54:03 +00:00
|
|
|
WriteBufferFromOwnString out;
|
2018-10-09 18:32:44 +00:00
|
|
|
part->partition.serializeText(*info.data, out, format_settings);
|
2017-12-28 18:20:53 +00:00
|
|
|
columns[i++]->insert(out.str());
|
2017-04-01 07:20:54 +00:00
|
|
|
}
|
2017-12-28 18:20:53 +00:00
|
|
|
columns[i++]->insert(part->name);
|
2018-10-22 08:54:54 +00:00
|
|
|
columns[i++]->insert(part_state == State::Committed);
|
2019-03-25 13:55:24 +00:00
|
|
|
columns[i++]->insert(part->getMarksCount());
|
2018-10-22 08:54:54 +00:00
|
|
|
columns[i++]->insert(part->rows_count);
|
|
|
|
columns[i++]->insert(part->bytes_on_disk.load(std::memory_order_relaxed));
|
|
|
|
columns[i++]->insert(columns_size.data_compressed);
|
|
|
|
columns[i++]->insert(columns_size.data_uncompressed);
|
|
|
|
columns[i++]->insert(columns_size.marks);
|
2017-12-28 18:20:53 +00:00
|
|
|
columns[i++]->insert(static_cast<UInt64>(part->modification_time));
|
2018-03-14 16:04:19 +00:00
|
|
|
|
|
|
|
time_t remove_time = part->remove_time.load(std::memory_order_relaxed);
|
|
|
|
columns[i++]->insert(static_cast<UInt64>(remove_time == std::numeric_limits<time_t>::max() ? 0 : remove_time));
|
2017-12-12 15:54:03 +00:00
|
|
|
|
|
|
|
/// For convenience, in returned refcount, don't add references that was due to local variables in this method: all_parts, active_parts.
|
2017-12-28 18:20:53 +00:00
|
|
|
columns[i++]->insert(static_cast<UInt64>(part.use_count() - 1));
|
2017-12-12 15:54:03 +00:00
|
|
|
|
2018-10-22 08:54:54 +00:00
|
|
|
columns[i++]->insert(part->getMinDate());
|
|
|
|
columns[i++]->insert(part->getMaxDate());
|
2018-12-10 10:20:19 +00:00
|
|
|
columns[i++]->insert(part->getMinTime());
|
|
|
|
columns[i++]->insert(part->getMaxTime());
|
2018-06-13 15:47:40 +00:00
|
|
|
columns[i++]->insert(part->info.partition_id);
|
2017-12-28 18:20:53 +00:00
|
|
|
columns[i++]->insert(part->info.min_block);
|
|
|
|
columns[i++]->insert(part->info.max_block);
|
2018-10-22 08:54:54 +00:00
|
|
|
columns[i++]->insert(part->info.level);
|
2018-06-13 15:47:40 +00:00
|
|
|
columns[i++]->insert(static_cast<UInt64>(part->info.getDataVersion()));
|
2018-10-22 08:54:54 +00:00
|
|
|
columns[i++]->insert(part->getIndexSizeInBytes());
|
|
|
|
columns[i++]->insert(part->getIndexSizeInAllocatedBytes());
|
2019-05-31 04:03:46 +00:00
|
|
|
columns[i++]->insert(part->is_frozen);
|
2017-12-12 15:54:03 +00:00
|
|
|
|
2017-12-28 18:20:53 +00:00
|
|
|
columns[i++]->insert(info.database);
|
|
|
|
columns[i++]->insert(info.table);
|
|
|
|
columns[i++]->insert(info.engine);
|
2018-02-22 09:09:14 +00:00
|
|
|
columns[i++]->insert(part->getFullPath());
|
2017-12-12 15:54:03 +00:00
|
|
|
|
|
|
|
if (has_state_column)
|
2017-12-28 18:20:53 +00:00
|
|
|
columns[i++]->insert(part->stateString());
|
2019-07-05 15:32:55 +00:00
|
|
|
|
|
|
|
MinimalisticDataPartChecksums helper;
|
|
|
|
helper.computeTotalChecksums(part->checksums);
|
|
|
|
|
|
|
|
auto checksum = helper.hash_of_all_files;
|
|
|
|
columns[i++]->insert(getHexUIntLowercase(checksum.first) + getHexUIntLowercase(checksum.second));
|
|
|
|
|
|
|
|
checksum = helper.hash_of_uncompressed_files;
|
|
|
|
columns[i++]->insert(getHexUIntLowercase(checksum.first) + getHexUIntLowercase(checksum.second));
|
|
|
|
|
|
|
|
checksum = helper.uncompressed_hash_of_compressed_files;
|
|
|
|
columns[i++]->insert(getHexUIntLowercase(checksum.first) + getHexUIntLowercase(checksum.second));
|
2017-04-01 07:20:54 +00:00
|
|
|
}
|
2014-07-29 14:05:15 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
}
|