From 1235c36f3e9120107658cda60e465d648e9fdeb5 Mon Sep 17 00:00:00 2001 From: Gleb Novikov Date: Sun, 10 May 2020 16:33:27 +0300 Subject: [PATCH] data_part instead of volume in IMergeTreeDataPartWriter (as it done in IMergeTreeReader) --- .../MergeTree/IMergeTreeDataPartWriter.cpp | 18 ++++++++---------- .../MergeTree/IMergeTreeDataPartWriter.h | 6 ++---- .../MergeTree/MergeTreeDataPartCompact.cpp | 2 +- .../MergeTree/MergeTreeDataPartWide.cpp | 2 +- .../MergeTreeDataPartWriterCompact.cpp | 13 +++++-------- .../MergeTree/MergeTreeDataPartWriterCompact.h | 4 +--- .../MergeTree/MergeTreeDataPartWriterWide.cpp | 12 +++++------- .../MergeTree/MergeTreeDataPartWriterWide.h | 4 +--- 8 files changed, 24 insertions(+), 37 deletions(-) diff --git a/src/Storages/MergeTree/IMergeTreeDataPartWriter.cpp b/src/Storages/MergeTree/IMergeTreeDataPartWriter.cpp index bb19a80a28a..239ccba1d89 100644 --- a/src/Storages/MergeTree/IMergeTreeDataPartWriter.cpp +++ b/src/Storages/MergeTree/IMergeTreeDataPartWriter.cpp @@ -63,18 +63,16 @@ void IMergeTreeDataPartWriter::Stream::addToChecksums(MergeTreeData::DataPart::C IMergeTreeDataPartWriter::IMergeTreeDataPartWriter( - const VolumePtr & volume_, - const String & part_path_, - const MergeTreeData & storage_, + const MergeTreeData::DataPartPtr & data_part_, const NamesAndTypesList & columns_list_, const std::vector & indices_to_recalc_, const String & marks_file_extension_, const CompressionCodecPtr & default_codec_, const MergeTreeWriterSettings & settings_, const MergeTreeIndexGranularity & index_granularity_) - : volume(volume_) - , part_path(part_path_) - , storage(storage_) + : data_part(data_part_) + , part_path(data_part_->getFullRelativePath()) + , storage(data_part_->storage) , columns_list(columns_list_) , marks_file_extension(marks_file_extension_) , index_granularity(index_granularity_) @@ -87,8 +85,8 @@ IMergeTreeDataPartWriter::IMergeTreeDataPartWriter( if (settings.blocks_are_granules_size && !index_granularity.empty()) throw Exception("Can't take information about index granularity from blocks, when non empty index_granularity array specified", ErrorCodes::LOGICAL_ERROR); - if (!volume->getDisk()->exists(part_path)) - volume->getDisk()->createDirectories(part_path); + if (!data_part->volume->getDisk()->exists(part_path)) + data_part->volume->getDisk()->createDirectories(part_path); } IMergeTreeDataPartWriter::~IMergeTreeDataPartWriter() = default; @@ -165,7 +163,7 @@ void IMergeTreeDataPartWriter::initPrimaryIndex() { if (storage.hasPrimaryKey()) { - index_file_stream = volume->getDisk()->writeFile(part_path + "primary.idx", DBMS_DEFAULT_BUFFER_SIZE, WriteMode::Rewrite); + index_file_stream = data_part->volume->getDisk()->writeFile(part_path + "primary.idx", DBMS_DEFAULT_BUFFER_SIZE, WriteMode::Rewrite); index_stream = std::make_unique(*index_file_stream); } @@ -180,7 +178,7 @@ void IMergeTreeDataPartWriter::initSkipIndices() skip_indices_streams.emplace_back( std::make_unique( stream_name, - volume->getDisk(), + data_part->volume->getDisk(), part_path + stream_name, INDEX_FILE_EXTENSION, part_path + stream_name, marks_file_extension, default_codec, settings.max_compress_block_size, diff --git a/src/Storages/MergeTree/IMergeTreeDataPartWriter.h b/src/Storages/MergeTree/IMergeTreeDataPartWriter.h index a4460ab861f..ffdba570544 100644 --- a/src/Storages/MergeTree/IMergeTreeDataPartWriter.h +++ b/src/Storages/MergeTree/IMergeTreeDataPartWriter.h @@ -61,9 +61,7 @@ public: using StreamPtr = std::unique_ptr; IMergeTreeDataPartWriter( - const VolumePtr & volume, - const String & part_path, - const MergeTreeData & storage, + const MergeTreeData::DataPartPtr & data_part, const NamesAndTypesList & columns_list, const std::vector & indices_to_recalc, const String & marks_file_extension, @@ -118,7 +116,7 @@ protected: using SerializationState = IDataType::SerializeBinaryBulkStatePtr; using SerializationStates = std::unordered_map; - VolumePtr volume; + MergeTreeData::DataPartPtr data_part; String part_path; const MergeTreeData & storage; NamesAndTypesList columns_list; diff --git a/src/Storages/MergeTree/MergeTreeDataPartCompact.cpp b/src/Storages/MergeTree/MergeTreeDataPartCompact.cpp index e417adf4eed..32acc266e42 100644 --- a/src/Storages/MergeTree/MergeTreeDataPartCompact.cpp +++ b/src/Storages/MergeTree/MergeTreeDataPartCompact.cpp @@ -68,7 +68,7 @@ IMergeTreeDataPart::MergeTreeWriterPtr MergeTreeDataPartCompact::getWriter( { return *getColumnPosition(lhs.name) < *getColumnPosition(rhs.name); }); return std::make_unique( - volume, getFullRelativePath(), storage, ordered_columns_list, indices_to_recalc, + shared_from_this(), ordered_columns_list, indices_to_recalc, index_granularity_info.marks_file_extension, default_codec, writer_settings, computed_index_granularity); } diff --git a/src/Storages/MergeTree/MergeTreeDataPartWide.cpp b/src/Storages/MergeTree/MergeTreeDataPartWide.cpp index 85903d0161e..d96b61b4bea 100644 --- a/src/Storages/MergeTree/MergeTreeDataPartWide.cpp +++ b/src/Storages/MergeTree/MergeTreeDataPartWide.cpp @@ -59,7 +59,7 @@ IMergeTreeDataPart::MergeTreeWriterPtr MergeTreeDataPartWide::getWriter( const MergeTreeIndexGranularity & computed_index_granularity) const { return std::make_unique( - volume, getFullRelativePath(), storage, columns_list, indices_to_recalc, + shared_from_this(), columns_list, indices_to_recalc, index_granularity_info.marks_file_extension, default_codec, writer_settings, computed_index_granularity); } diff --git a/src/Storages/MergeTree/MergeTreeDataPartWriterCompact.cpp b/src/Storages/MergeTree/MergeTreeDataPartWriterCompact.cpp index f107f18792e..1a7a757c149 100644 --- a/src/Storages/MergeTree/MergeTreeDataPartWriterCompact.cpp +++ b/src/Storages/MergeTree/MergeTreeDataPartWriterCompact.cpp @@ -6,26 +6,23 @@ namespace DB MergeTreeDataPartWriterCompact::MergeTreeDataPartWriterCompact( - const VolumePtr & volume_, - const String & part_path_, - const MergeTreeData & storage_, + const MergeTreeData::DataPartPtr & data_part_, const NamesAndTypesList & columns_list_, const std::vector & indices_to_recalc_, const String & marks_file_extension_, const CompressionCodecPtr & default_codec_, const MergeTreeWriterSettings & settings_, const MergeTreeIndexGranularity & index_granularity_) -: IMergeTreeDataPartWriter(volume_, part_path_, - storage_, columns_list_, - indices_to_recalc_, marks_file_extension_, - default_codec_, settings_, index_granularity_) + : IMergeTreeDataPartWriter(data_part_, columns_list_, + indices_to_recalc_, marks_file_extension_, + default_codec_, settings_, index_granularity_) { using DataPart = MergeTreeDataPartCompact; String data_file_name = DataPart::DATA_FILE_NAME; stream = std::make_unique( data_file_name, - volume->getDisk(), + data_part->volume->getDisk(), part_path + data_file_name, DataPart::DATA_FILE_EXTENSION, part_path + data_file_name, marks_file_extension, default_codec, diff --git a/src/Storages/MergeTree/MergeTreeDataPartWriterCompact.h b/src/Storages/MergeTree/MergeTreeDataPartWriterCompact.h index f786a09d93e..07caba94712 100644 --- a/src/Storages/MergeTree/MergeTreeDataPartWriterCompact.h +++ b/src/Storages/MergeTree/MergeTreeDataPartWriterCompact.h @@ -8,9 +8,7 @@ class MergeTreeDataPartWriterCompact : public IMergeTreeDataPartWriter { public: MergeTreeDataPartWriterCompact( - const VolumePtr & volume, - const String & part_path, - const MergeTreeData & storage, + const MergeTreeData::DataPartPtr & data_part, const NamesAndTypesList & columns_list, const std::vector & indices_to_recalc, const String & marks_file_extension, diff --git a/src/Storages/MergeTree/MergeTreeDataPartWriterWide.cpp b/src/Storages/MergeTree/MergeTreeDataPartWriterWide.cpp index 1b0e4441b9f..4cdf57a4700 100644 --- a/src/Storages/MergeTree/MergeTreeDataPartWriterWide.cpp +++ b/src/Storages/MergeTree/MergeTreeDataPartWriterWide.cpp @@ -13,18 +13,16 @@ namespace } MergeTreeDataPartWriterWide::MergeTreeDataPartWriterWide( - const VolumePtr & volume_, - const String & part_path_, - const MergeTreeData & storage_, + const MergeTreeData::DataPartPtr & data_part_, const NamesAndTypesList & columns_list_, const std::vector & indices_to_recalc_, const String & marks_file_extension_, const CompressionCodecPtr & default_codec_, const MergeTreeWriterSettings & settings_, const MergeTreeIndexGranularity & index_granularity_) - : IMergeTreeDataPartWriter(volume_, part_path_, - storage_, columns_list_, indices_to_recalc_, - marks_file_extension_, default_codec_, settings_, index_granularity_) + : IMergeTreeDataPartWriter(data_part_, columns_list_, + indices_to_recalc_, marks_file_extension_, + default_codec_, settings_, index_granularity_) { const auto & columns = storage.getColumns(); for (const auto & it : columns_list) @@ -46,7 +44,7 @@ void MergeTreeDataPartWriterWide::addStreams( column_streams[stream_name] = std::make_unique( stream_name, - volume->getDisk(), + data_part->volume->getDisk(), part_path + stream_name, DATA_FILE_EXTENSION, part_path + stream_name, marks_file_extension, effective_codec, diff --git a/src/Storages/MergeTree/MergeTreeDataPartWriterWide.h b/src/Storages/MergeTree/MergeTreeDataPartWriterWide.h index 42357af15a8..acd7f749d00 100644 --- a/src/Storages/MergeTree/MergeTreeDataPartWriterWide.h +++ b/src/Storages/MergeTree/MergeTreeDataPartWriterWide.h @@ -11,9 +11,7 @@ public: using ColumnToSize = std::map; MergeTreeDataPartWriterWide( - const VolumePtr & volume, - const String & part_path, - const MergeTreeData & storage, + const MergeTreeData::DataPartPtr & data_part, const NamesAndTypesList & columns_list, const std::vector & indices_to_recalc, const String & marks_file_extension,