diff --git a/src/Common/filesystemHelpers.cpp b/src/Common/filesystemHelpers.cpp index 765683138b5..b63f7406411 100644 --- a/src/Common/filesystemHelpers.cpp +++ b/src/Common/filesystemHelpers.cpp @@ -16,10 +16,17 @@ #include #include #include +#include #include namespace fs = std::filesystem; + +namespace ProfileEvents +{ + extern const Event ExternalProcessingFilesTotal; +} + namespace DB { @@ -35,7 +42,6 @@ namespace ErrorCodes extern const int CANNOT_CREATE_FILE; } - struct statvfs getStatVFS(const String & path) { struct statvfs fs; @@ -48,7 +54,6 @@ struct statvfs getStatVFS(const String & path) return fs; } - bool enoughSpaceInDirectory(const std::string & path, size_t data_size) { fs::path filepath(path); @@ -61,17 +66,11 @@ bool enoughSpaceInDirectory(const std::string & path, size_t data_size) std::unique_ptr createTemporaryFile(const std::string & path) { + ProfileEvents::increment(ProfileEvents::ExternalProcessingFilesTotal); fs::create_directories(path); return std::make_unique(path); } -std::unique_ptr createTemporaryFile(const DiskPtr & disk, std::unique_ptr metric_increment) -{ - /// is is possible to use with disk other than DickLocal ? - disk->createDirectories(disk->getPath()); - return std::make_unique(disk, disk->getPath(), std::move(metric_increment)); -} - #if !defined(OS_LINUX) [[noreturn]] #endif diff --git a/src/Common/filesystemHelpers.h b/src/Common/filesystemHelpers.h index 80b18fc2970..792b75dc33a 100644 --- a/src/Common/filesystemHelpers.h +++ b/src/Common/filesystemHelpers.h @@ -7,18 +7,17 @@ #include #include #include -#include +#include namespace fs = std::filesystem; namespace DB { -using TemporaryFile = TemporaryFileOnDisk; +using TemporaryFile = Poco::TemporaryFile; bool enoughSpaceInDirectory(const std::string & path, size_t data_size); std::unique_ptr createTemporaryFile(const std::string & path); -std::unique_ptr createTemporaryFile(const DiskPtr & disk, std::unique_ptr metric_increment = nullptr); // Determine what block device is responsible for specified path diff --git a/src/Common/TemporaryFile.cpp b/src/Disks/TemporaryFileOnDisk.cpp similarity index 75% rename from src/Common/TemporaryFile.cpp rename to src/Disks/TemporaryFileOnDisk.cpp index 6c293101028..90ecd602a38 100644 --- a/src/Common/TemporaryFile.cpp +++ b/src/Disks/TemporaryFileOnDisk.cpp @@ -1,5 +1,4 @@ -#include -#include +#include #include @@ -11,17 +10,17 @@ namespace ProfileEvents namespace DB { -TemporaryFileOnDisk::TemporaryFileOnDisk(const String & prefix_) - : tmp_file(std::make_unique(prefix_)) - , filepath(tmp_file->path()) -{ - ProfileEvents::increment(ProfileEvents::ExternalProcessingFilesTotal); -} +TemporaryFileOnDisk::TemporaryFileOnDisk(const DiskPtr & disk_, std::unique_ptr increment_) + : TemporaryFileOnDisk(disk_, disk_->getPath(), std::move(increment_)) +{} TemporaryFileOnDisk::TemporaryFileOnDisk(const DiskPtr & disk_, const String & prefix_, std::unique_ptr increment_) : disk(disk_) , sub_metric_increment(std::move(increment_)) { + /// is is possible to use with disk other than DickLocal ? + disk->createDirectories(prefix_); + ProfileEvents::increment(ProfileEvents::ExternalProcessingFilesTotal); /// Do not use default temporaty root path `/tmp/tmpXXXXXX`. diff --git a/src/Common/TemporaryFile.h b/src/Disks/TemporaryFileOnDisk.h similarity index 71% rename from src/Common/TemporaryFile.h rename to src/Disks/TemporaryFileOnDisk.h index 872ea9a28b5..354fae50a8d 100644 --- a/src/Common/TemporaryFile.h +++ b/src/Disks/TemporaryFileOnDisk.h @@ -2,7 +2,7 @@ #include #include -#include +#include #include @@ -13,18 +13,18 @@ namespace CurrentMetrics namespace DB { -class IDisk; using DiskPtr = std::shared_ptr; /// This class helps with the handling of temporary files or directories. /// A unique name for the temporary file or directory is automatically chosen based on a specified prefix. -/// Optionally can create a directory in the constructor. +/// Create a directory in the constructor. /// The destructor always removes the temporary file or directory with all contained files. class TemporaryFileOnDisk { public: - explicit TemporaryFileOnDisk(const DiskPtr & disk_, const String & prefix_ = "tmp", std::unique_ptr increment_ = nullptr); - explicit TemporaryFileOnDisk(const String & prefix_); + explicit TemporaryFileOnDisk(const DiskPtr & disk_, std::unique_ptr increment_); + explicit TemporaryFileOnDisk(const DiskPtr & disk_, const String & prefix_, std::unique_ptr increment_); + ~TemporaryFileOnDisk(); DiskPtr getDisk() const { return disk; } @@ -34,10 +34,6 @@ public: private: DiskPtr disk; - /// If disk is not provided, fallback to Poco::TemporaryFile - /// TODO: it's better to use DiskLocal for that case as well - std::unique_ptr tmp_file; - String filepath; CurrentMetrics::Increment metric_increment{CurrentMetrics::TotalTemporaryFiles}; @@ -45,4 +41,6 @@ private: std::unique_ptr sub_metric_increment; }; +using TemporaryFileOnDiskHolder = std::unique_ptr; + } diff --git a/src/Interpreters/Aggregator.cpp b/src/Interpreters/Aggregator.cpp index e42c774bdf1..f0716fb5c03 100644 --- a/src/Interpreters/Aggregator.cpp +++ b/src/Interpreters/Aggregator.cpp @@ -30,6 +30,7 @@ #include #include #include +#include #include @@ -1491,7 +1492,7 @@ void Aggregator::writeToTemporaryFile(AggregatedDataVariants & data_variants, si auto file = createTempFile(max_temp_file_size); - std::string path = file->getPath(); + const auto & path = file->getPath(); WriteBufferFromFile file_buf(path); CompressedWriteBuffer compressed_buf(file_buf); NativeWriter block_out(compressed_buf, DBMS_TCP_PROTOCOL_VERSION, getHeader(false)); @@ -1559,9 +1560,9 @@ void Aggregator::writeToTemporaryFile(AggregatedDataVariants & data_variants, si } -std::unique_ptr Aggregator::createTempFile(size_t max_temp_file_size) const +std::unique_ptr Aggregator::createTempFile(size_t max_temp_file_size) const { - auto file = createTemporaryFile(params.tmp_volume->getDisk(), + auto file = std::make_unique(params.tmp_volume->getDisk(), std::make_unique(CurrentMetrics::TemporaryFilesForAggregation)); // enoughSpaceInDirectory() is not enough to make it right, since diff --git a/src/Interpreters/SortedBlocksWriter.cpp b/src/Interpreters/SortedBlocksWriter.cpp index a9783630ad3..7608a7e7e45 100644 --- a/src/Interpreters/SortedBlocksWriter.cpp +++ b/src/Interpreters/SortedBlocksWriter.cpp @@ -7,6 +7,7 @@ #include #include #include +#include namespace ProfileEvents @@ -29,10 +30,9 @@ namespace DB namespace { - -std::unique_ptr flushToFile(const DiskPtr & disk, const Block & header, QueryPipelineBuilder pipeline, const String & codec) +TemporaryFileOnDiskHolder flushToFile(const DiskPtr & disk, const Block & header, QueryPipelineBuilder pipeline, const String & codec) { - auto tmp_file = createTemporaryFile(disk, std::make_unique(CurrentMetrics::TemporaryFilesForJoin)); + auto tmp_file = std::make_unsigned(disk, std::make_unique(CurrentMetrics::TemporaryFilesForJoin)); auto write_stat = TemporaryFileStream::write(tmp_file->getPath(), header, std::move(pipeline), codec); ProfileEvents::increment(ProfileEvents::ExternalProcessingCompressedBytesTotal, write_stat.compressed_bytes); @@ -48,7 +48,7 @@ std::unique_ptr flushToFile(const DiskPtr & disk, const Block & h SortedBlocksWriter::SortedFiles flushToManyFiles(const DiskPtr & disk, const Block & header, QueryPipelineBuilder builder, const String & codec, std::function callback = [](const Block &){}) { - std::vector> files; + std::vector files; auto pipeline = QueryPipelineBuilder::getPipeline(std::move(builder)); PullingPipelineExecutor executor(pipeline); diff --git a/src/Processors/Transforms/MergeSortingTransform.cpp b/src/Processors/Transforms/MergeSortingTransform.cpp index 001c33baa4a..e8e6a70f1fd 100644 --- a/src/Processors/Transforms/MergeSortingTransform.cpp +++ b/src/Processors/Transforms/MergeSortingTransform.cpp @@ -215,7 +215,7 @@ void MergeSortingTransform::consume(Chunk chunk) throw Exception("Not enough space for external sort in temporary storage", ErrorCodes::NOT_ENOUGH_SPACE); auto files_num_counter = std::make_unique(CurrentMetrics::TemporaryFilesForSort); - temporary_files.emplace_back(createTemporaryFile(reservation->getDisk(), std::move(files_num_counter))); + temporary_files.emplace_back(std::make_unique(reservation->getDisk(), std::move(files_num_counter))); const std::string & path = temporary_files.back()->path(); merge_sorter diff --git a/src/Processors/Transforms/MergeSortingTransform.h b/src/Processors/Transforms/MergeSortingTransform.h index e118a2a655b..465193548e7 100644 --- a/src/Processors/Transforms/MergeSortingTransform.h +++ b/src/Processors/Transforms/MergeSortingTransform.h @@ -3,6 +3,7 @@ #include #include #include +#include #include @@ -55,7 +56,7 @@ private: bool remerge_is_useful = true; /// Everything below is for external sorting. - std::vector> temporary_files; + std::vector temporary_files; /// Merge all accumulated blocks to keep no more than limit rows. void remerge(); diff --git a/src/Storages/MergeTree/DataPartStorageOnDisk.cpp b/src/Storages/MergeTree/DataPartStorageOnDisk.cpp index 30ec2b7a492..1cd6fd0305b 100644 --- a/src/Storages/MergeTree/DataPartStorageOnDisk.cpp +++ b/src/Storages/MergeTree/DataPartStorageOnDisk.cpp @@ -1,7 +1,7 @@ #include #include #include -#include +#include #include #include #include diff --git a/src/Storages/MergeTree/MergeTask.cpp b/src/Storages/MergeTree/MergeTask.cpp index 5361b691b67..81840f0cffb 100644 --- a/src/Storages/MergeTree/MergeTask.cpp +++ b/src/Storages/MergeTree/MergeTask.cpp @@ -241,7 +241,7 @@ bool MergeTask::ExecuteAndFinalizeHorizontalPart::prepare() } case MergeAlgorithm::Vertical : { - ctx->rows_sources_file = createTemporaryFile(ctx->tmp_disk); + ctx->rows_sources_file = createTemporaryFile(ctx->tmp_disk->getPath()); ctx->rows_sources_uncompressed_write_buf = ctx->tmp_disk->writeFile(fileName(ctx->rows_sources_file->path()), DBMS_DEFAULT_BUFFER_SIZE, WriteMode::Rewrite, global_ctx->context->getWriteSettings()); ctx->rows_sources_write_buf = std::make_unique(*ctx->rows_sources_uncompressed_write_buf); diff --git a/src/Storages/MergeTree/MergeTreeData.cpp b/src/Storages/MergeTree/MergeTreeData.cpp index 916df36dc20..54b8a900e87 100644 --- a/src/Storages/MergeTree/MergeTreeData.cpp +++ b/src/Storages/MergeTree/MergeTreeData.cpp @@ -63,7 +63,7 @@ #include #include #include -#include +#include #include #include #include diff --git a/src/Storages/StorageLog.cpp b/src/Storages/StorageLog.cpp index 0a1325f8fe5..c6bc55fd620 100644 --- a/src/Storages/StorageLog.cpp +++ b/src/Storages/StorageLog.cpp @@ -30,7 +30,7 @@ #include #include #include -#include +#include #include #include diff --git a/src/Storages/StorageMemory.cpp b/src/Storages/StorageMemory.cpp index 6ecbb53ee6f..f3f1162287f 100644 --- a/src/Storages/StorageMemory.cpp +++ b/src/Storages/StorageMemory.cpp @@ -30,7 +30,7 @@ #include #include #include -#include +#include #include diff --git a/src/Storages/StorageStripeLog.cpp b/src/Storages/StorageStripeLog.cpp index 7a469a7dceb..0ecbdb0db10 100644 --- a/src/Storages/StorageStripeLog.cpp +++ b/src/Storages/StorageStripeLog.cpp @@ -34,7 +34,7 @@ #include #include #include -#include +#include #include