2019-10-10 16:30:30 +00:00
|
|
|
#include "MergeTreeDataPartWide.h"
|
2016-02-14 04:58:47 +00:00
|
|
|
#include <Poco/File.h>
|
2019-10-10 16:30:30 +00:00
|
|
|
#include <Storages/MergeTree/MergeTreeReaderWide.h>
|
2019-10-21 17:23:06 +00:00
|
|
|
#include <Storages/MergeTree/MergeTreeDataPartWriterWide.h>
|
|
|
|
#include <Storages/MergeTree/IMergeTreeDataPartWriter.h>
|
2020-09-14 11:22:17 +00:00
|
|
|
#include <Core/NamesAndTypes.h>
|
2019-10-10 16:30:30 +00:00
|
|
|
|
|
|
|
|
2016-02-14 04:58:47 +00:00
|
|
|
namespace DB
|
|
|
|
{
|
|
|
|
|
|
|
|
namespace ErrorCodes
|
|
|
|
{
|
2020-02-25 18:02:41 +00:00
|
|
|
extern const int CANNOT_READ_ALL_DATA;
|
2017-04-01 07:20:54 +00:00
|
|
|
extern const int NO_FILE_IN_DATA_PART;
|
2018-03-21 20:21:34 +00:00
|
|
|
extern const int BAD_SIZE_OF_FILE_IN_DATA_PART;
|
2020-07-25 20:56:29 +00:00
|
|
|
extern const int LOGICAL_ERROR;
|
2016-02-14 04:58:47 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
|
2019-12-18 16:41:11 +00:00
|
|
|
MergeTreeDataPartWide::MergeTreeDataPartWide(
|
2019-10-10 16:30:30 +00:00
|
|
|
MergeTreeData & storage_,
|
|
|
|
const String & name_,
|
2020-05-09 21:24:15 +00:00
|
|
|
const VolumePtr & volume_,
|
2019-10-10 16:30:30 +00:00
|
|
|
const std::optional<String> & relative_path_)
|
2020-05-09 21:24:15 +00:00
|
|
|
: IMergeTreeDataPart(storage_, name_, volume_, relative_path_, Type::WIDE)
|
2017-08-16 19:24:50 +00:00
|
|
|
{
|
|
|
|
}
|
|
|
|
|
2019-10-10 16:30:30 +00:00
|
|
|
MergeTreeDataPartWide::MergeTreeDataPartWide(
|
|
|
|
const MergeTreeData & storage_,
|
|
|
|
const String & name_,
|
|
|
|
const MergeTreePartInfo & info_,
|
2020-05-09 21:24:15 +00:00
|
|
|
const VolumePtr & volume_,
|
2019-10-10 16:30:30 +00:00
|
|
|
const std::optional<String> & relative_path_)
|
2020-05-09 21:24:15 +00:00
|
|
|
: IMergeTreeDataPart(storage_, name_, info_, volume_, relative_path_, Type::WIDE)
|
2017-08-25 20:41:45 +00:00
|
|
|
{
|
|
|
|
}
|
|
|
|
|
2019-10-10 16:30:30 +00:00
|
|
|
IMergeTreeDataPart::MergeTreeReaderPtr MergeTreeDataPartWide::getReader(
|
|
|
|
const NamesAndTypesList & columns_to_read,
|
2020-06-17 16:39:58 +00:00
|
|
|
const StorageMetadataPtr & metadata_snapshot,
|
2019-10-10 16:30:30 +00:00
|
|
|
const MarkRanges & mark_ranges,
|
|
|
|
UncompressedCache * uncompressed_cache,
|
|
|
|
MarkCache * mark_cache,
|
2019-12-18 15:54:45 +00:00
|
|
|
const MergeTreeReaderSettings & reader_settings,
|
2019-10-10 16:30:30 +00:00
|
|
|
const ValueSizeMap & avg_value_size_hints,
|
|
|
|
const ReadBufferFromFileBase::ProfileCallback & profile_callback) const
|
2019-03-22 12:56:58 +00:00
|
|
|
{
|
2020-02-10 20:27:06 +00:00
|
|
|
auto ptr = std::static_pointer_cast<const MergeTreeDataPartWide>(shared_from_this());
|
|
|
|
return std::make_unique<MergeTreeReaderWide>(
|
2020-06-17 16:39:58 +00:00
|
|
|
ptr, columns_to_read, metadata_snapshot, uncompressed_cache,
|
2020-02-10 20:27:06 +00:00
|
|
|
mark_cache, mark_ranges, reader_settings,
|
|
|
|
avg_value_size_hints, profile_callback);
|
2019-03-22 12:56:58 +00:00
|
|
|
}
|
|
|
|
|
2019-10-21 15:33:59 +00:00
|
|
|
IMergeTreeDataPart::MergeTreeWriterPtr MergeTreeDataPartWide::getWriter(
|
|
|
|
const NamesAndTypesList & columns_list,
|
2020-06-17 12:39:20 +00:00
|
|
|
const StorageMetadataPtr & metadata_snapshot,
|
2019-11-07 11:11:38 +00:00
|
|
|
const std::vector<MergeTreeIndexPtr> & indices_to_recalc,
|
2020-08-26 15:29:46 +00:00
|
|
|
const CompressionCodecPtr & default_codec_,
|
2019-12-18 15:54:45 +00:00
|
|
|
const MergeTreeWriterSettings & writer_settings,
|
2019-11-07 11:11:38 +00:00
|
|
|
const MergeTreeIndexGranularity & computed_index_granularity) const
|
2019-10-21 15:33:59 +00:00
|
|
|
{
|
2019-10-21 17:23:06 +00:00
|
|
|
return std::make_unique<MergeTreeDataPartWriterWide>(
|
2020-06-17 12:39:20 +00:00
|
|
|
shared_from_this(), columns_list, metadata_snapshot, indices_to_recalc,
|
2019-10-21 17:23:06 +00:00
|
|
|
index_granularity_info.marks_file_extension,
|
2020-08-26 15:29:46 +00:00
|
|
|
default_codec_, writer_settings, computed_index_granularity);
|
2019-12-18 16:41:11 +00:00
|
|
|
}
|
2019-10-21 15:33:59 +00:00
|
|
|
|
2019-03-22 12:56:58 +00:00
|
|
|
|
2018-03-26 14:18:04 +00:00
|
|
|
/// Takes into account the fact that several columns can e.g. share their .size substreams.
|
|
|
|
/// When calculating totals these should be counted only once.
|
2019-10-10 16:30:30 +00:00
|
|
|
ColumnSize MergeTreeDataPartWide::getColumnSizeImpl(
|
2019-01-04 12:10:00 +00:00
|
|
|
const String & column_name, const IDataType & type, std::unordered_set<String> * processed_substreams) const
|
2017-03-24 13:52:50 +00:00
|
|
|
{
|
2018-03-26 14:18:04 +00:00
|
|
|
ColumnSize size;
|
|
|
|
if (checksums.empty())
|
|
|
|
return size;
|
|
|
|
|
2020-09-18 11:37:58 +00:00
|
|
|
type.enumerateStreams([&](const IDataType::SubstreamPath & substream_path, const IDataType & /* substream_type */)
|
2018-03-26 14:18:04 +00:00
|
|
|
{
|
2019-01-04 12:10:00 +00:00
|
|
|
String file_name = IDataType::getFileNameForStream(column_name, substream_path);
|
2018-03-26 14:18:04 +00:00
|
|
|
|
|
|
|
if (processed_substreams && !processed_substreams->insert(file_name).second)
|
|
|
|
return;
|
|
|
|
|
|
|
|
auto bin_checksum = checksums.files.find(file_name + ".bin");
|
|
|
|
if (bin_checksum != checksums.files.end())
|
|
|
|
{
|
|
|
|
size.data_compressed += bin_checksum->second.file_size;
|
|
|
|
size.data_uncompressed += bin_checksum->second.uncompressed_size;
|
|
|
|
}
|
|
|
|
|
2019-06-19 10:07:56 +00:00
|
|
|
auto mrk_checksum = checksums.files.find(file_name + index_granularity_info.marks_file_extension);
|
2018-03-26 14:18:04 +00:00
|
|
|
if (mrk_checksum != checksums.files.end())
|
|
|
|
size.marks += mrk_checksum->second.file_size;
|
|
|
|
}, {});
|
2016-02-14 04:58:47 +00:00
|
|
|
|
2018-03-26 14:18:04 +00:00
|
|
|
return size;
|
2017-03-24 13:52:50 +00:00
|
|
|
}
|
2016-02-14 04:58:47 +00:00
|
|
|
|
2019-10-10 16:30:30 +00:00
|
|
|
void MergeTreeDataPartWide::loadIndexGranularity()
|
2016-02-14 04:58:47 +00:00
|
|
|
{
|
2020-02-27 16:47:40 +00:00
|
|
|
String full_path = getFullRelativePath();
|
2020-05-09 21:24:15 +00:00
|
|
|
index_granularity_info.changeGranularityIfRequired(volume->getDisk(), full_path);
|
2019-06-19 10:07:56 +00:00
|
|
|
|
2019-12-09 21:21:17 +00:00
|
|
|
|
2018-11-15 14:06:54 +00:00
|
|
|
if (columns.empty())
|
|
|
|
throw Exception("No columns in part " + name, ErrorCodes::NO_FILE_IN_DATA_PART);
|
2017-04-01 07:20:54 +00:00
|
|
|
|
2018-11-15 14:06:54 +00:00
|
|
|
/// We can use any column, it doesn't matter
|
2019-12-19 14:05:26 +00:00
|
|
|
std::string marks_file_path = index_granularity_info.getMarksFilePath(full_path + getFileNameForColumn(columns.front()));
|
2020-05-09 21:24:15 +00:00
|
|
|
if (!volume->getDisk()->exists(marks_file_path))
|
|
|
|
throw Exception("Marks file '" + fullPath(volume->getDisk(), marks_file_path) + "' doesn't exist", ErrorCodes::NO_FILE_IN_DATA_PART);
|
2018-11-15 14:06:54 +00:00
|
|
|
|
2020-05-09 21:24:15 +00:00
|
|
|
size_t marks_file_size = volume->getDisk()->getFileSize(marks_file_path);
|
2018-11-12 17:44:43 +00:00
|
|
|
|
2019-06-19 10:07:56 +00:00
|
|
|
if (!index_granularity_info.is_adaptive)
|
2018-11-15 14:06:54 +00:00
|
|
|
{
|
2020-01-14 13:23:51 +00:00
|
|
|
size_t marks_count = marks_file_size / index_granularity_info.getMarkSizeInBytes();
|
2019-06-19 10:07:56 +00:00
|
|
|
index_granularity.resizeWithFixedGranularity(marks_count, index_granularity_info.fixed_index_granularity); /// all the same
|
2017-04-01 07:20:54 +00:00
|
|
|
}
|
2018-11-15 14:06:54 +00:00
|
|
|
else
|
|
|
|
{
|
2020-05-09 21:24:15 +00:00
|
|
|
auto buffer = volume->getDisk()->readFile(marks_file_path, marks_file_size);
|
2020-02-27 16:47:40 +00:00
|
|
|
while (!buffer->eof())
|
2018-11-15 14:06:54 +00:00
|
|
|
{
|
2020-02-27 16:47:40 +00:00
|
|
|
buffer->seek(sizeof(size_t) * 2, SEEK_CUR); /// skip offset_in_compressed file and offset_in_decompressed_block
|
2019-03-25 13:55:24 +00:00
|
|
|
size_t granularity;
|
2020-02-27 16:47:40 +00:00
|
|
|
readIntBinary(granularity, *buffer);
|
2019-03-25 13:55:24 +00:00
|
|
|
index_granularity.appendMark(granularity);
|
2018-11-15 14:06:54 +00:00
|
|
|
}
|
2019-10-31 14:44:17 +00:00
|
|
|
|
2020-01-14 13:23:51 +00:00
|
|
|
if (index_granularity.getMarksCount() * index_granularity_info.getMarkSizeInBytes() != marks_file_size)
|
2020-05-09 21:24:15 +00:00
|
|
|
throw Exception("Cannot read all marks from file " + fullPath(volume->getDisk(), marks_file_path), ErrorCodes::CANNOT_READ_ALL_DATA);
|
2018-11-15 14:06:54 +00:00
|
|
|
}
|
2016-02-14 04:58:47 +00:00
|
|
|
|
2019-10-31 14:44:17 +00:00
|
|
|
index_granularity.setInitialized();
|
2019-04-15 09:30:45 +00:00
|
|
|
}
|
|
|
|
|
2019-11-18 12:22:27 +00:00
|
|
|
MergeTreeDataPartWide::~MergeTreeDataPartWide()
|
|
|
|
{
|
|
|
|
removeIfNeeded();
|
|
|
|
}
|
|
|
|
|
2019-11-18 15:18:50 +00:00
|
|
|
void MergeTreeDataPartWide::checkConsistency(bool require_part_metadata) const
|
|
|
|
{
|
2020-01-15 19:16:56 +00:00
|
|
|
checkConsistencyBase();
|
2020-02-28 17:14:55 +00:00
|
|
|
String path = getFullRelativePath();
|
2019-11-18 15:18:50 +00:00
|
|
|
|
|
|
|
if (!checksums.empty())
|
|
|
|
{
|
|
|
|
if (require_part_metadata)
|
|
|
|
{
|
|
|
|
for (const NameAndTypePair & name_type : columns)
|
|
|
|
{
|
|
|
|
IDataType::SubstreamPath stream_path;
|
2020-09-18 11:37:58 +00:00
|
|
|
name_type.type->enumerateStreams([&](const IDataType::SubstreamPath & substream_path, const IDataType & /* substream_type */)
|
2019-11-18 15:18:50 +00:00
|
|
|
{
|
|
|
|
String file_name = IDataType::getFileNameForStream(name_type.name, substream_path);
|
|
|
|
String mrk_file_name = file_name + index_granularity_info.marks_file_extension;
|
|
|
|
String bin_file_name = file_name + ".bin";
|
|
|
|
if (!checksums.files.count(mrk_file_name))
|
2020-05-09 21:24:15 +00:00
|
|
|
throw Exception("No " + mrk_file_name + " file checksum for column " + name_type.name + " in part " + fullPath(volume->getDisk(), path),
|
2019-11-18 15:18:50 +00:00
|
|
|
ErrorCodes::NO_FILE_IN_DATA_PART);
|
|
|
|
if (!checksums.files.count(bin_file_name))
|
2020-05-09 21:24:15 +00:00
|
|
|
throw Exception("No " + bin_file_name + " file checksum for column " + name_type.name + " in part " + fullPath(volume->getDisk(), path),
|
2019-11-18 15:18:50 +00:00
|
|
|
ErrorCodes::NO_FILE_IN_DATA_PART);
|
|
|
|
}, stream_path);
|
|
|
|
}
|
|
|
|
}
|
2020-01-16 16:15:01 +00:00
|
|
|
|
2019-11-18 15:18:50 +00:00
|
|
|
}
|
|
|
|
else
|
|
|
|
{
|
|
|
|
/// Check that all marks are nonempty and have the same size.
|
|
|
|
std::optional<UInt64> marks_size;
|
|
|
|
for (const NameAndTypePair & name_type : columns)
|
|
|
|
{
|
2020-09-18 11:37:58 +00:00
|
|
|
name_type.type->enumerateStreams([&](const IDataType::SubstreamPath & substream_path, const IDataType & /* substream_type */)
|
2019-11-18 15:18:50 +00:00
|
|
|
{
|
2020-02-28 17:14:55 +00:00
|
|
|
auto file_path = path + IDataType::getFileNameForStream(name_type.name, substream_path) + index_granularity_info.marks_file_extension;
|
2019-11-18 15:18:50 +00:00
|
|
|
|
|
|
|
/// Missing file is Ok for case when new column was added.
|
2020-05-09 21:24:15 +00:00
|
|
|
if (volume->getDisk()->exists(file_path))
|
2019-11-18 15:18:50 +00:00
|
|
|
{
|
2020-05-09 21:24:15 +00:00
|
|
|
UInt64 file_size = volume->getDisk()->getFileSize(file_path);
|
2019-11-18 15:18:50 +00:00
|
|
|
|
|
|
|
if (!file_size)
|
2020-05-09 21:24:15 +00:00
|
|
|
throw Exception("Part " + path + " is broken: " + fullPath(volume->getDisk(), file_path) + " is empty.",
|
2019-11-18 15:18:50 +00:00
|
|
|
ErrorCodes::BAD_SIZE_OF_FILE_IN_DATA_PART);
|
|
|
|
|
|
|
|
if (!marks_size)
|
|
|
|
marks_size = file_size;
|
|
|
|
else if (file_size != *marks_size)
|
|
|
|
throw Exception("Part " + path + " is broken: marks have different sizes.",
|
|
|
|
ErrorCodes::BAD_SIZE_OF_FILE_IN_DATA_PART);
|
|
|
|
}
|
|
|
|
});
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
2017-12-02 22:12:27 +00:00
|
|
|
|
2020-09-14 11:22:17 +00:00
|
|
|
bool MergeTreeDataPartWide::hasColumnFiles(const NameAndTypePair & column) const
|
2019-12-09 21:21:17 +00:00
|
|
|
{
|
|
|
|
bool res = true;
|
2019-01-05 03:33:22 +00:00
|
|
|
|
2020-09-30 16:34:29 +00:00
|
|
|
column.type->enumerateStreams([&](const IDataType::SubstreamPath & substream_path, const IDataType & /* substream_type */)
|
2019-12-09 21:21:17 +00:00
|
|
|
{
|
2020-09-14 11:22:17 +00:00
|
|
|
String file_name = IDataType::getFileNameForStream(column, substream_path);
|
2017-09-11 22:40:51 +00:00
|
|
|
|
2019-12-09 21:21:17 +00:00
|
|
|
auto bin_checksum = checksums.files.find(file_name + ".bin");
|
|
|
|
auto mrk_checksum = checksums.files.find(file_name + index_granularity_info.marks_file_extension);
|
2017-09-11 22:40:51 +00:00
|
|
|
|
2019-12-09 21:21:17 +00:00
|
|
|
if (bin_checksum == checksums.files.end() || mrk_checksum == checksums.files.end())
|
|
|
|
res = false;
|
|
|
|
}, {});
|
2017-11-20 19:33:12 +00:00
|
|
|
|
2019-12-09 21:21:17 +00:00
|
|
|
return res;
|
|
|
|
}
|
2017-11-20 19:33:12 +00:00
|
|
|
|
2019-12-19 13:10:57 +00:00
|
|
|
String MergeTreeDataPartWide::getFileNameForColumn(const NameAndTypePair & column) const
|
|
|
|
{
|
|
|
|
String filename;
|
2020-09-18 11:37:58 +00:00
|
|
|
column.type->enumerateStreams([&](const IDataType::SubstreamPath & substream_path, const IDataType & /* substream_type */)
|
2019-12-19 14:05:26 +00:00
|
|
|
{
|
2019-12-19 13:10:57 +00:00
|
|
|
if (filename.empty())
|
|
|
|
filename = IDataType::getFileNameForStream(column.name, substream_path);
|
|
|
|
});
|
|
|
|
return filename;
|
|
|
|
}
|
|
|
|
|
2020-06-29 20:36:18 +00:00
|
|
|
void MergeTreeDataPartWide::calculateEachColumnSizes(ColumnSizeByName & each_columns_size, ColumnSize & total_size) const
|
2020-03-23 12:19:43 +00:00
|
|
|
{
|
|
|
|
std::unordered_set<String> processed_substreams;
|
|
|
|
for (const NameAndTypePair & column : columns)
|
|
|
|
{
|
|
|
|
ColumnSize size = getColumnSizeImpl(column.name, *column.type, &processed_substreams);
|
|
|
|
each_columns_size[column.name] = size;
|
|
|
|
total_size.add(size);
|
2020-07-25 14:42:20 +00:00
|
|
|
|
|
|
|
#ifndef NDEBUG
|
2020-07-27 09:42:37 +00:00
|
|
|
/// Most trivial types
|
|
|
|
if (rows_count != 0 && column.type->isValueRepresentedByNumber() && !column.type->haveSubtypes())
|
2020-07-25 14:42:20 +00:00
|
|
|
{
|
|
|
|
size_t rows_in_column = size.data_uncompressed / column.type->getSizeOfValueInMemory();
|
|
|
|
if (rows_in_column != rows_count)
|
|
|
|
{
|
|
|
|
throw Exception(
|
|
|
|
ErrorCodes::LOGICAL_ERROR,
|
2020-07-25 14:45:46 +00:00
|
|
|
"Column {} has rows count {} according to size in memory "
|
|
|
|
"and size of single value, but data part {} has {} rows", backQuote(column.name), rows_in_column, name, rows_count);
|
2020-07-25 14:42:20 +00:00
|
|
|
}
|
|
|
|
}
|
|
|
|
#endif
|
2020-07-25 20:54:33 +00:00
|
|
|
}
|
2020-03-23 12:19:43 +00:00
|
|
|
}
|
|
|
|
|
2016-02-14 04:58:47 +00:00
|
|
|
}
|