thevar1able/ClickHouse
1
0
Fork 0
mirror of https://github.com/ClickHouse/ClickHouse.git synced 2024-11-27 01:51:59 +00:00
Code Issues Packages Projects Releases Wiki Activity

IDisk get/set last modified time implementation.

Browse Source
This commit is contained in:
Pavel Kovalenko 2020-03-05 17:02:15 +03:00 committed by Pavel Kovalenko
parent 96bb607e66
commit caf26d8aff
8 changed files with 47 additions and 13 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

10
dbms/src/Disks/DiskLocal.cpp
View File

@ -244,6 +244,16 @@ void DiskLocal::listFiles(const String & path, std::vector<String> & file_names)
Poco::File(disk_path + path).list(file_names);
}
void DiskLocal::setLastModified(const String & path, const Poco::Timestamp & timestamp)
{
Poco::File(disk_path + path).setLastModified(timestamp);
}
Poco::Timestamp DiskLocal::getLastModified(const String & path)
{
return Poco::File(disk_path + path).getLastModified();
}
void DiskLocalReservation::update(UInt64 new_size)
{

4
dbms/src/Disks/DiskLocal.h
View File

@ -87,6 +87,10 @@ public:
void removeRecursive(const String & path) override;
void setLastModified(const String & path, const Poco::Timestamp & timestamp) override;
Poco::Timestamp getLastModified(const String & path) override;
private:
bool tryReserve(UInt64 bytes);

4
dbms/src/Disks/DiskMemory.h
View File

@ -80,6 +80,10 @@ public:
void removeRecursive(const String & path) override;
void setLastModified(const String &, const Poco::Timestamp &) override { }
Poco::Timestamp getLastModified(const String &) override { return Poco::Timestamp(); }
private:
void createDirectoriesImpl(const String & path);
void replaceFileImpl(const String & from_path, const String & to_path);

10
dbms/src/Disks/DiskS3.cpp
View File

@ -614,6 +614,16 @@ void DiskS3::listFiles(const String & path, std::vector<String> & file_names)
file_names.push_back(it->name());
}
void DiskS3::setLastModified(const String & path, const Poco::Timestamp & timestamp)
{
Poco::File(metadata_path + path).setLastModified(timestamp);
}
Poco::Timestamp DiskS3::getLastModified(const String & path)
{
return Poco::File(metadata_path + path).getLastModified();
}
DiskS3Reservation::~DiskS3Reservation()
{

4
dbms/src/Disks/DiskS3.h
View File

@ -87,6 +87,10 @@ public:
void removeRecursive(const String & path) override;
void setLastModified(const String & path, const Poco::Timestamp & timestamp) override;
Poco::Timestamp getLastModified(const String & path) override;
private:
String getRandomName() const;

7
dbms/src/Disks/IDisk.h
View File

@ -10,6 +10,7 @@
#include <utility>
#include <boost/noncopyable.hpp>
#include <Poco/Path.h>
#include <Poco/Timestamp.h>
namespace CurrentMetrics
@ -145,6 +146,12 @@ public:
/// Remove file or directory with all children. Use with extra caution. Throws exception if file doesn't exists.
virtual void removeRecursive(const String & path) = 0;
/// Set last modified time to file or directory at `path`.
virtual void setLastModified(const String & path, const Poco::Timestamp & timestamp) = 0;
/// Get last modified time of file or directory at `path`.
virtual Poco::Timestamp getLastModified(const String & path) = 0;
};
using DiskPtr = std::shared_ptr<IDisk>;

2
dbms/src/Storages/MergeTree/IMergeTreeDataPart.cpp
View File

@ -673,7 +673,7 @@ void IMergeTreeDataPart::renameTo(const String & new_relative_path, bool remove_
}
}
//from_file.setLastModified(Poco::Timestamp::fromEpochTime(time(nullptr)));
disk->setLastModified(from, Poco::Timestamp::fromEpochTime(time(nullptr)));
disk->moveFile(from, to);
relative_path = new_relative_path;
}

19
dbms/src/Storages/MergeTree/MergeTreeData.cpp
View File

@ -1074,8 +1074,7 @@ void MergeTreeData::loadDataParts(bool skip_sanity_checks)
else
has_adaptive_parts.store(true, std::memory_order_relaxed);
/// TODO: Add modification time functionality to IDisk.
part->modification_time = Poco::File(getFullPathOnDisk(part_disk_ptr) + part_name).getLastModified().epochTime();
part->modification_time = part_disk_ptr->getLastModified(relative_data_path + part_name).epochTime();
/// Assume that all parts are Committed, covered parts will be detected and marked as Outdated later
part->state = DataPartState::Committed;
@ -1158,14 +1157,13 @@ void MergeTreeData::loadDataParts(bool skip_sanity_checks)
/// Is the part directory old.
/// True if its modification time and the modification time of all files inside it is less then threshold.
/// (Only files on the first level of nesting are considered).
static bool isOldPartDirectory(Poco::File & directory, time_t threshold)
static bool isOldPartDirectory(const DiskPtr & disk, const String & directory_path, time_t threshold)
{
if (directory.getLastModified().epochTime() >= threshold)
if (disk->getLastModified(directory_path).epochTime() >= threshold)
return false;
Poco::DirectoryIterator end;
for (Poco::DirectoryIterator it(directory); it != end; ++it)
if (it->getLastModified().epochTime() >= threshold)
for (auto it = disk->iterateDirectory(directory_path); it->isValid(); it->next())
if (disk->getLastModified(it->path()).epochTime() >= threshold)
return false;
return true;
@ -1192,15 +1190,12 @@ void MergeTreeData::clearOldTemporaryDirectories(ssize_t custom_directories_life
{
if (startsWith(it->name(), "tmp_"))
{
/// TODO: Add modification time functionality to IDisk.
Poco::File tmp_dir(fullPath(disk, it->path()));
try
{
if (tmp_dir.isDirectory() && isOldPartDirectory(tmp_dir, deadline))
if (disk->isDirectory(it->path()) && isOldPartDirectory(disk, it->path(), deadline))
{
LOG_WARNING(log, "Removing temporary directory " << fullPath(disk, it->path()));
tmp_dir.remove(true);
disk->removeRecursive(it->path());
}
}
catch (const Poco::FileNotFoundException &)