diff --git a/dbms/src/Disks/DiskLocal.cpp b/dbms/src/Disks/DiskLocal.cpp index 8c8bffc0f5a..418ce966955 100644 --- a/dbms/src/Disks/DiskLocal.cpp +++ b/dbms/src/Disks/DiskLocal.cpp @@ -244,6 +244,16 @@ void DiskLocal::listFiles(const String & path, std::vector & file_names) Poco::File(disk_path + path).list(file_names); } +void DiskLocal::setLastModified(const String & path, const Poco::Timestamp & timestamp) +{ + Poco::File(disk_path + path).setLastModified(timestamp); +} + +Poco::Timestamp DiskLocal::getLastModified(const String & path) +{ + return Poco::File(disk_path + path).getLastModified(); +} + void DiskLocalReservation::update(UInt64 new_size) { diff --git a/dbms/src/Disks/DiskLocal.h b/dbms/src/Disks/DiskLocal.h index 03000f9e9a8..0bca5dc72d4 100644 --- a/dbms/src/Disks/DiskLocal.h +++ b/dbms/src/Disks/DiskLocal.h @@ -87,6 +87,10 @@ public: void removeRecursive(const String & path) override; + void setLastModified(const String & path, const Poco::Timestamp & timestamp) override; + + Poco::Timestamp getLastModified(const String & path) override; + private: bool tryReserve(UInt64 bytes); diff --git a/dbms/src/Disks/DiskMemory.h b/dbms/src/Disks/DiskMemory.h index 2b9ca65de70..f67a361a948 100644 --- a/dbms/src/Disks/DiskMemory.h +++ b/dbms/src/Disks/DiskMemory.h @@ -80,6 +80,10 @@ public: void removeRecursive(const String & path) override; + void setLastModified(const String &, const Poco::Timestamp &) override { } + + Poco::Timestamp getLastModified(const String &) override { return Poco::Timestamp(); } + private: void createDirectoriesImpl(const String & path); void replaceFileImpl(const String & from_path, const String & to_path); diff --git a/dbms/src/Disks/DiskS3.cpp b/dbms/src/Disks/DiskS3.cpp index 68e7beab9c6..abf40bccf83 100644 --- a/dbms/src/Disks/DiskS3.cpp +++ b/dbms/src/Disks/DiskS3.cpp @@ -614,6 +614,16 @@ void DiskS3::listFiles(const String & path, std::vector & file_names) file_names.push_back(it->name()); } +void DiskS3::setLastModified(const String & path, const Poco::Timestamp & timestamp) +{ + Poco::File(metadata_path + path).setLastModified(timestamp); +} + +Poco::Timestamp DiskS3::getLastModified(const String & path) +{ + return Poco::File(metadata_path + path).getLastModified(); +} + DiskS3Reservation::~DiskS3Reservation() { diff --git a/dbms/src/Disks/DiskS3.h b/dbms/src/Disks/DiskS3.h index 984544b018e..5b59089ffd1 100644 --- a/dbms/src/Disks/DiskS3.h +++ b/dbms/src/Disks/DiskS3.h @@ -87,6 +87,10 @@ public: void removeRecursive(const String & path) override; + void setLastModified(const String & path, const Poco::Timestamp & timestamp) override; + + Poco::Timestamp getLastModified(const String & path) override; + private: String getRandomName() const; diff --git a/dbms/src/Disks/IDisk.h b/dbms/src/Disks/IDisk.h index 0f0447a7169..877c6f84706 100644 --- a/dbms/src/Disks/IDisk.h +++ b/dbms/src/Disks/IDisk.h @@ -10,6 +10,7 @@ #include #include #include +#include namespace CurrentMetrics @@ -145,6 +146,12 @@ public: /// Remove file or directory with all children. Use with extra caution. Throws exception if file doesn't exists. virtual void removeRecursive(const String & path) = 0; + + /// Set last modified time to file or directory at `path`. + virtual void setLastModified(const String & path, const Poco::Timestamp & timestamp) = 0; + + /// Get last modified time of file or directory at `path`. + virtual Poco::Timestamp getLastModified(const String & path) = 0; }; using DiskPtr = std::shared_ptr; diff --git a/dbms/src/Storages/MergeTree/IMergeTreeDataPart.cpp b/dbms/src/Storages/MergeTree/IMergeTreeDataPart.cpp index 3cf4fede415..c7ac56821d7 100644 --- a/dbms/src/Storages/MergeTree/IMergeTreeDataPart.cpp +++ b/dbms/src/Storages/MergeTree/IMergeTreeDataPart.cpp @@ -673,7 +673,7 @@ void IMergeTreeDataPart::renameTo(const String & new_relative_path, bool remove_ } } - //from_file.setLastModified(Poco::Timestamp::fromEpochTime(time(nullptr))); + disk->setLastModified(from, Poco::Timestamp::fromEpochTime(time(nullptr))); disk->moveFile(from, to); relative_path = new_relative_path; } diff --git a/dbms/src/Storages/MergeTree/MergeTreeData.cpp b/dbms/src/Storages/MergeTree/MergeTreeData.cpp index 474f8512980..135f31d3c32 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeData.cpp +++ b/dbms/src/Storages/MergeTree/MergeTreeData.cpp @@ -1074,8 +1074,7 @@ void MergeTreeData::loadDataParts(bool skip_sanity_checks) else has_adaptive_parts.store(true, std::memory_order_relaxed); - /// TODO: Add modification time functionality to IDisk. - part->modification_time = Poco::File(getFullPathOnDisk(part_disk_ptr) + part_name).getLastModified().epochTime(); + part->modification_time = part_disk_ptr->getLastModified(relative_data_path + part_name).epochTime(); /// Assume that all parts are Committed, covered parts will be detected and marked as Outdated later part->state = DataPartState::Committed; @@ -1158,14 +1157,13 @@ void MergeTreeData::loadDataParts(bool skip_sanity_checks) /// Is the part directory old. /// True if its modification time and the modification time of all files inside it is less then threshold. /// (Only files on the first level of nesting are considered). -static bool isOldPartDirectory(Poco::File & directory, time_t threshold) +static bool isOldPartDirectory(const DiskPtr & disk, const String & directory_path, time_t threshold) { - if (directory.getLastModified().epochTime() >= threshold) + if (disk->getLastModified(directory_path).epochTime() >= threshold) return false; - Poco::DirectoryIterator end; - for (Poco::DirectoryIterator it(directory); it != end; ++it) - if (it->getLastModified().epochTime() >= threshold) + for (auto it = disk->iterateDirectory(directory_path); it->isValid(); it->next()) + if (disk->getLastModified(it->path()).epochTime() >= threshold) return false; return true; @@ -1192,15 +1190,12 @@ void MergeTreeData::clearOldTemporaryDirectories(ssize_t custom_directories_life { if (startsWith(it->name(), "tmp_")) { - /// TODO: Add modification time functionality to IDisk. - Poco::File tmp_dir(fullPath(disk, it->path())); - try { - if (tmp_dir.isDirectory() && isOldPartDirectory(tmp_dir, deadline)) + if (disk->isDirectory(it->path()) && isOldPartDirectory(disk, it->path(), deadline)) { LOG_WARNING(log, "Removing temporary directory " << fullPath(disk, it->path())); - tmp_dir.remove(true); + disk->removeRecursive(it->path()); } } catch (const Poco::FileNotFoundException &)