ClickHouse/src/Storages/MergeTree/MergedBlockOutputStream.cpp

318 lines
12 KiB
C++
Raw Normal View History

#include <Storages/MergeTree/MergedBlockOutputStream.h>
#include <Interpreters/Context.h>
2022-02-14 19:50:08 +00:00
#include <Interpreters/MergeTreeTransaction.h>
#include <Parsers/queryToString.h>
namespace DB
{
namespace ErrorCodes
{
2020-02-25 18:20:08 +00:00
extern const int LOGICAL_ERROR;
}
MergedBlockOutputStream::MergedBlockOutputStream(
2019-11-07 11:11:38 +00:00
const MergeTreeDataPartPtr & data_part,
2022-04-12 18:59:49 +00:00
DataPartStorageBuilderPtr data_part_storage_builder_,
const StorageMetadataPtr & metadata_snapshot_,
const NamesAndTypesList & columns_list_,
const MergeTreeIndices & skip_indices,
CompressionCodecPtr default_codec_,
2022-02-14 19:50:08 +00:00
const MergeTreeTransactionPtr & txn,
2021-10-29 17:21:02 +00:00
bool reset_columns_,
bool blocks_are_granules_size,
const WriteSettings & write_settings)
2022-04-12 18:59:49 +00:00
: IMergedBlockOutputStream(std::move(data_part_storage_builder_), data_part, metadata_snapshot_, columns_list_, reset_columns_)
2019-07-28 11:10:35 +00:00
, columns_list(columns_list_)
, default_codec(default_codec_)
{
2020-04-14 19:47:19 +00:00
MergeTreeWriterSettings writer_settings(
storage.getContext()->getSettings(),
write_settings,
storage.getSettings(),
data_part->index_granularity_info.is_adaptive,
2020-12-09 18:19:49 +00:00
/* rewrite_primary_key = */ true,
2020-04-14 19:47:19 +00:00
blocks_are_granules_size);
2019-11-05 11:53:22 +00:00
if (data_part->isStoredOnDisk())
data_part_storage_builder->createDirectories();
2022-02-15 15:00:45 +00:00
/// We should write version metadata on part creation to distinguish it from parts that were created without transaction.
TransactionID tid = txn ? txn->tid : Tx::PrehistoricTID;
/// NOTE do not pass context for writing to system.transactions_info_log,
/// because part may have temporary name (with temporary block numbers). Will write it later.
data_part->version.setCreationTID(tid, nullptr);
data_part->storeVersionMetadata();
2022-02-14 19:50:08 +00:00
2022-04-22 20:38:18 +00:00
writer = data_part->getWriter(data_part_storage_builder, columns_list, metadata_snapshot, skip_indices, default_codec, writer_settings, {});
}
2018-05-07 02:01:11 +00:00
/// If data is pre-sorted.
void MergedBlockOutputStream::write(const Block & block)
{
writeImpl(block, nullptr);
}
2017-03-12 19:18:07 +00:00
/** If the data is not sorted, but we pre-calculated the permutation, after which they will be sorted.
* This method is used to save RAM, since you do not need to keep two blocks at once - the source and the sorted.
*/
void MergedBlockOutputStream::writeWithPermutation(const Block & block, const IColumn::Permutation * permutation)
{
writeImpl(block, permutation);
}
struct MergedBlockOutputStream::Finalizer::Impl
{
IMergeTreeDataPartWriter & writer;
MergeTreeData::MutableDataPartPtr part;
2022-04-12 18:59:49 +00:00
DataPartStorageBuilderPtr data_part_storage_builder;
2022-02-08 19:21:16 +00:00
NameSet files_to_remove_after_finish;
std::vector<std::unique_ptr<WriteBufferFromFileBase>> written_files;
bool sync;
2022-04-12 18:59:49 +00:00
Impl(IMergeTreeDataPartWriter & writer_, MergeTreeData::MutableDataPartPtr part_, DataPartStorageBuilderPtr data_part_storage_builder_, const NameSet & files_to_remove_after_finish_, bool sync_)
2022-02-08 19:21:16 +00:00
: writer(writer_)
, part(std::move(part_))
2022-04-12 18:59:49 +00:00
, data_part_storage_builder(std::move(data_part_storage_builder_))
2022-02-08 19:21:16 +00:00
, files_to_remove_after_finish(files_to_remove_after_finish_)
, sync(sync_) {}
void finish();
};
void MergedBlockOutputStream::Finalizer::finish()
{
std::unique_ptr<Impl> to_finish = std::move(impl);
if (to_finish)
to_finish->finish();
}
void MergedBlockOutputStream::Finalizer::Impl::finish()
{
writer.finish(sync);
2022-02-08 19:21:16 +00:00
for (const auto & file_name: files_to_remove_after_finish)
2022-04-12 18:59:49 +00:00
data_part_storage_builder->removeFile(file_name);
2022-02-08 19:21:16 +00:00
for (auto & file : written_files)
{
file->finalize();
if (sync)
file->sync();
}
}
MergedBlockOutputStream::Finalizer::~Finalizer()
{
try
{
finish();
}
catch (...)
{
tryLogCurrentException("MergedBlockOutputStream");
}
}
MergedBlockOutputStream::Finalizer::Finalizer(Finalizer &&) noexcept = default;
MergedBlockOutputStream::Finalizer & MergedBlockOutputStream::Finalizer::operator=(Finalizer &&) noexcept = default;
MergedBlockOutputStream::Finalizer::Finalizer(std::unique_ptr<Impl> impl_) : impl(std::move(impl_)) {}
void MergedBlockOutputStream::finalizePart(
MergeTreeData::MutableDataPartPtr & new_part,
bool sync,
const NamesAndTypesList * total_columns_list,
MergeTreeData::DataPart::Checksums * additional_column_checksums)
{
finalizePartAsync(new_part, sync, total_columns_list, additional_column_checksums).finish();
}
MergedBlockOutputStream::Finalizer MergedBlockOutputStream::finalizePartAsync(
MergeTreeData::MutableDataPartPtr & new_part,
2022-01-14 19:53:55 +00:00
bool sync,
2020-03-09 02:55:28 +00:00
const NamesAndTypesList * total_columns_list,
MergeTreeData::DataPart::Checksums * additional_column_checksums,
const WriteSettings & write_settings)
{
2019-10-21 17:23:06 +00:00
/// Finish write and get checksums.
MergeTreeData::DataPart::Checksums checksums;
2019-10-28 11:00:29 +00:00
if (additional_column_checksums)
checksums = std::move(*additional_column_checksums);
/// Finish columns serialization.
writer->fillChecksums(checksums);
LOG_TRACE(&Poco::Logger::get("MergedBlockOutputStream"), "filled checksums {}", new_part->getNameWithState());
2022-01-14 19:53:55 +00:00
for (const auto & [projection_name, projection_part] : new_part->getProjectionParts())
checksums.addFile(
projection_name + ".proj",
projection_part->checksums.getTotalSizeOnDisk(),
projection_part->checksums.getTotalChecksumUInt128());
2022-02-08 19:21:16 +00:00
NameSet files_to_remove_after_sync;
2022-01-20 21:06:32 +00:00
if (reset_columns)
2022-01-21 00:20:41 +00:00
{
auto part_columns = total_columns_list ? *total_columns_list : columns_list;
auto serialization_infos = new_part->getSerializationInfos();
serialization_infos.replaceData(new_serialization_infos);
2022-02-08 19:21:16 +00:00
files_to_remove_after_sync = removeEmptyColumnsFromPart(new_part, part_columns, serialization_infos, checksums);
2022-01-21 00:20:41 +00:00
new_part->setColumns(part_columns);
new_part->setSerializationInfos(serialization_infos);
}
2021-04-15 21:47:11 +00:00
2022-04-12 18:59:49 +00:00
auto finalizer = std::make_unique<Finalizer::Impl>(*writer, new_part, data_part_storage_builder, files_to_remove_after_sync, sync);
2020-04-14 01:26:34 +00:00
if (new_part->isStoredOnDisk())
finalizer->written_files = finalizePartOnDisk(new_part, checksums, write_settings);
2020-04-14 01:26:34 +00:00
new_part->rows_count = rows_count;
new_part->modification_time = time(nullptr);
new_part->index = writer->releaseIndexColumns();
new_part->checksums = checksums;
new_part->setBytesOnDisk(checksums.getTotalSizeOnDisk());
new_part->index_granularity = writer->getIndexGranularity();
new_part->calculateColumnsAndSecondaryIndicesSizesOnDisk();
2022-01-14 19:53:55 +00:00
if (default_codec != nullptr)
new_part->default_codec = default_codec;
return Finalizer(std::move(finalizer));
2022-01-11 19:02:48 +00:00
}
MergedBlockOutputStream::WrittenFiles MergedBlockOutputStream::finalizePartOnDisk(
2022-01-20 23:22:16 +00:00
const MergeTreeData::DataPartPtr & new_part,
MergeTreeData::DataPart::Checksums & checksums,
2022-03-21 08:52:48 +00:00
const WriteSettings & settings)
2020-04-14 01:26:34 +00:00
{
WrittenFiles written_files;
if (new_part->isProjectionPart())
{
if (storage.format_version >= MERGE_TREE_DATA_MIN_FORMAT_VERSION_WITH_CUSTOM_PARTITIONING || isCompactPart(new_part))
{
2022-05-03 15:48:05 +00:00
auto count_out = data_part_storage_builder->writeFile("count.txt", 4096, settings);
HashingWriteBuffer count_out_hashing(*count_out);
writeIntText(rows_count, count_out_hashing);
count_out_hashing.next();
checksums.files["count.txt"].file_size = count_out_hashing.count();
checksums.files["count.txt"].file_hash = count_out_hashing.getHash();
count_out->preFinalize();
written_files.emplace_back(std::move(count_out));
}
}
else
{
if (new_part->uuid != UUIDHelpers::Nil)
{
2022-05-03 15:48:05 +00:00
auto out = data_part_storage_builder->writeFile(IMergeTreeDataPart::UUID_FILE_NAME, 4096, settings);
HashingWriteBuffer out_hashing(*out);
writeUUIDText(new_part->uuid, out_hashing);
checksums.files[IMergeTreeDataPart::UUID_FILE_NAME].file_size = out_hashing.count();
checksums.files[IMergeTreeDataPart::UUID_FILE_NAME].file_hash = out_hashing.getHash();
out->preFinalize();
written_files.emplace_back(std::move(out));
}
if (storage.format_version >= MERGE_TREE_DATA_MIN_FORMAT_VERSION_WITH_CUSTOM_PARTITIONING)
{
2022-04-12 18:59:49 +00:00
if (auto file = new_part->partition.store(storage, data_part_storage_builder, checksums))
written_files.emplace_back(std::move(file));
if (new_part->minmax_idx->initialized)
{
2022-04-12 18:59:49 +00:00
auto files = new_part->minmax_idx->store(storage, data_part_storage_builder, checksums);
for (auto & file : files)
written_files.emplace_back(std::move(file));
}
else if (rows_count)
throw Exception("MinMax index was not initialized for new non-empty part " + new_part->name
+ ". It is a bug.", ErrorCodes::LOGICAL_ERROR);
}
{
2022-05-03 15:48:05 +00:00
auto count_out = data_part_storage_builder->writeFile("count.txt", 4096, settings);
HashingWriteBuffer count_out_hashing(*count_out);
writeIntText(rows_count, count_out_hashing);
count_out_hashing.next();
checksums.files["count.txt"].file_size = count_out_hashing.count();
checksums.files["count.txt"].file_hash = count_out_hashing.getHash();
count_out->preFinalize();
written_files.emplace_back(std::move(count_out));
}
}
2019-10-17 18:55:07 +00:00
if (!new_part->ttl_infos.empty())
{
/// Write a file with ttl infos in json format.
2022-05-03 15:48:05 +00:00
auto out = data_part_storage_builder->writeFile("ttl.txt", 4096, settings);
HashingWriteBuffer out_hashing(*out);
new_part->ttl_infos.write(out_hashing);
checksums.files["ttl.txt"].file_size = out_hashing.count();
checksums.files["ttl.txt"].file_hash = out_hashing.getHash();
out->preFinalize();
written_files.emplace_back(std::move(out));
}
2022-01-20 23:22:16 +00:00
if (!new_part->getSerializationInfos().empty())
2021-03-12 16:33:41 +00:00
{
2022-05-03 15:48:05 +00:00
auto out = data_part_storage_builder->writeFile(IMergeTreeDataPart::SERIALIZATION_FILE_NAME, 4096, settings);
2021-03-12 16:33:41 +00:00
HashingWriteBuffer out_hashing(*out);
2022-01-20 23:22:16 +00:00
new_part->getSerializationInfos().writeJSON(out_hashing);
2021-03-12 16:33:41 +00:00
checksums.files[IMergeTreeDataPart::SERIALIZATION_FILE_NAME].file_size = out_hashing.count();
checksums.files[IMergeTreeDataPart::SERIALIZATION_FILE_NAME].file_hash = out_hashing.getHash();
out->preFinalize();
written_files.emplace_back(std::move(out));
2021-03-12 16:33:41 +00:00
}
{
/// Write a file with a description of columns.
2022-05-03 15:48:05 +00:00
auto out = data_part_storage_builder->writeFile("columns.txt", 4096, settings);
2022-01-20 23:22:16 +00:00
new_part->getColumns().writeText(*out);
out->preFinalize();
written_files.emplace_back(std::move(out));
}
2020-08-28 09:07:20 +00:00
if (default_codec != nullptr)
{
2022-05-03 15:48:05 +00:00
auto out = data_part_storage_builder->writeFile(IMergeTreeDataPart::DEFAULT_COMPRESSION_CODEC_FILE_NAME, 4096, settings);
DB::writeText(queryToString(default_codec->getFullCodecDesc()), *out);
out->preFinalize();
written_files.emplace_back(std::move(out));
}
2020-08-28 09:07:20 +00:00
else
{
throw Exception("Compression codec have to be specified for part on disk, empty for" + new_part->name
+ ". It is a bug.", ErrorCodes::LOGICAL_ERROR);
}
{
/// Write file with checksums.
2022-05-03 15:48:05 +00:00
auto out = data_part_storage_builder->writeFile("checksums.txt", 4096, settings);
checksums.write(*out);
out->preFinalize();
written_files.emplace_back(std::move(out));
}
return written_files;
}
void MergedBlockOutputStream::writeImpl(const Block & block, const IColumn::Permutation * permutation)
{
block.checkNumberOfRows();
size_t rows = block.rows();
2018-12-04 14:44:42 +00:00
if (!rows)
return;
2018-11-30 15:36:10 +00:00
2020-12-10 08:57:52 +00:00
writer->write(block, permutation);
2021-10-29 17:21:02 +00:00
if (reset_columns)
new_serialization_infos.add(block);
rows_count += rows;
}
}