thevar1able/ClickHouse
1
0
Fork 0
mirror of https://github.com/ClickHouse/ClickHouse.git synced 2024-12-14 02:12:21 +00:00
Code Issues Packages Projects Releases Wiki Activity

Update DiskLocal and DiskMemory

Browse Source
This commit is contained in:
kssenii 2021-04-30 09:52:40 +03:00
parent 654e58b77c
commit 7f2f138d56
3 changed files with 62 additions and 55 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

88
src/Disks/DiskLocal.cpp
View File

@ -60,28 +60,28 @@ private:
class DiskLocalDirectoryIterator : public IDiskDirectoryIterator
{
public:
explicit DiskLocalDirectoryIterator(const String & disk_path_, const String & dir_path_)
: dir_path(dir_path_), iter(disk_path_ + dir_path_)
explicit DiskLocalDirectoryIterator(const fs::path & disk_path_, const String & dir_path_)
: dir_path(dir_path_), entry(disk_path_ / dir_path_)
{
}
void next() override { ++iter; }
void next() override { ++entry; }
bool isValid() const override { return iter != Poco::DirectoryIterator(); }
bool isValid() const override { return entry != fs::directory_iterator(); }
String path() const override
{
if (iter->isDirectory())
return dir_path + iter.name() + '/';
if (entry->is_directory())
return dir_path / entry->path().filename() / "";
else
return dir_path + iter.name();
return dir_path / entry->path().filename();
}
String name() const override { return iter.name(); }
String name() const override { return entry->path().filename(); }
private:
String dir_path;
Poco::DirectoryIterator iter;
fs::path dir_path;
fs::directory_iterator entry;
};
@ -119,9 +119,9 @@ UInt64 DiskLocal::getTotalSpace() const
{
struct statvfs fs;
if (name == "default") /// for default disk we get space from path/data/
fs = getStatVFS(disk_path + "data/");
fs = getStatVFS(fs::path(disk_path / "data/").string());
else
fs = getStatVFS(disk_path);
fs = getStatVFS(disk_path.string());
UInt64 total_size = fs.f_blocks * fs.f_bsize;
if (total_size < keep_free_space_bytes)
return 0;
@ -134,9 +134,9 @@ UInt64 DiskLocal::getAvailableSpace() const
/// available for superuser only and for system purposes
struct statvfs fs;
if (name == "default") /// for default disk we get space from path/data/
fs = getStatVFS(disk_path + "data/");
fs = getStatVFS(fs::path(disk_path / "data/").string());
else
fs = getStatVFS(disk_path);
fs = getStatVFS(disk_path.string());
UInt64 total_size = fs.f_bavail * fs.f_bsize;
if (total_size < keep_free_space_bytes)
return 0;
@ -153,43 +153,43 @@ UInt64 DiskLocal::getUnreservedSpace() const
bool DiskLocal::exists(const String & path) const
{
return fs::exists(fs::path(disk_path) / path);
return fs::exists(disk_path / path);
}
bool DiskLocal::isFile(const String & path) const
{
return fs::is_regular_file(fs::path(disk_path) / path);
return fs::is_regular_file(disk_path / path);
}
bool DiskLocal::isDirectory(const String & path) const
{
return fs::is_directory(fs::path(disk_path) / path);
return fs::is_directory(disk_path / path);
}
size_t DiskLocal::getFileSize(const String & path) const
{
return fs::file_size(fs::path(disk_path) / path);
return fs::file_size(disk_path / path);
}
void DiskLocal::createDirectory(const String & path)
{
fs::create_directory(fs::path(disk_path) / path);
fs::create_directory(disk_path / path);
}
void DiskLocal::createDirectories(const String & path)
{
fs::create_directories(fs::path(disk_path) / path);
fs::create_directories(disk_path / path);
}
void DiskLocal::clearDirectory(const String & path)
{
for (auto & entry : fs::directory_iterator(fs::path(disk_path) / path))
for (auto & entry : fs::directory_iterator(disk_path / path))
fs::remove(entry.path());
}
void DiskLocal::moveDirectory(const String & from_path, const String & to_path)
{
fs::rename(fs::path(disk_path) / from_path, fs::path(disk_path) / to_path);
fs::rename(disk_path / from_path, disk_path / to_path);
}
DiskDirectoryIteratorPtr DiskLocal::iterateDirectory(const String & path)
@ -199,18 +199,18 @@ DiskDirectoryIteratorPtr DiskLocal::iterateDirectory(const String & path)
void DiskLocal::moveFile(const String & from_path, const String & to_path)
{
fs::rename(fs::path(disk_path) / from_path, fs::path(disk_path) / to_path);
fs::rename(disk_path / from_path, disk_path / to_path);
}
void DiskLocal::replaceFile(const String & from_path, const String & to_path)
{
fs::path from_file = fs::path(disk_path) / from_path;
fs::path to_file = fs::path(disk_path) / to_path;
fs::path from_file = disk_path / from_path;
fs::path to_file = disk_path / to_path;
if (fs::exists(to_file))
{
fs::path tmp_file(to_file.string() + ".old");
fs::rename(to_file, tmp_file);
fs::rename(from_file, fs::path(disk_path) / to_path);
fs::rename(from_file, disk_path / to_path);
fs::remove(tmp_file);
}
else
@ -223,35 +223,35 @@ std::unique_ptr<ReadBufferFromFileBase>
DiskLocal::readFile(
const String & path, size_t buf_size, size_t estimated_size, size_t aio_threshold, size_t mmap_threshold, MMappedFileCache * mmap_cache) const
{
return createReadBufferFromFileBase(disk_path + path, estimated_size, aio_threshold, mmap_threshold, mmap_cache, buf_size);
return createReadBufferFromFileBase(disk_path / path, estimated_size, aio_threshold, mmap_threshold, mmap_cache, buf_size);
}
std::unique_ptr<WriteBufferFromFileBase>
DiskLocal::writeFile(const String & path, size_t buf_size, WriteMode mode)
{
int flags = (mode == WriteMode::Append) ? (O_APPEND | O_CREAT | O_WRONLY) : -1;
return std::make_unique<WriteBufferFromFile>(disk_path + path, buf_size, flags);
return std::make_unique<WriteBufferFromFile>(disk_path / path, buf_size, flags);
}
void DiskLocal::removeFile(const String & path)
{
auto fs_path = disk_path + path;
auto fs_path = disk_path / path;
if (0 != unlink(fs_path.c_str()))
throwFromErrnoWithPath("Cannot unlink file " + fs_path, fs_path, ErrorCodes::CANNOT_UNLINK);
throwFromErrnoWithPath("Cannot unlink file " + fs_path.string(), fs_path, ErrorCodes::CANNOT_UNLINK);
}
void DiskLocal::removeFileIfExists(const String & path)
{
auto fs_path = disk_path + path;
auto fs_path = disk_path / path;
if (0 != unlink(fs_path.c_str()) && errno != ENOENT)
throwFromErrnoWithPath("Cannot unlink file " + fs_path, fs_path, ErrorCodes::CANNOT_UNLINK);
throwFromErrnoWithPath("Cannot unlink file " + fs_path.string(), fs_path, ErrorCodes::CANNOT_UNLINK);
}
void DiskLocal::removeDirectory(const String & path)
{
auto fs_path = disk_path + path;
auto fs_path = disk_path / path;
if (0 != rmdir(fs_path.c_str()))
throwFromErrnoWithPath("Cannot rmdir " + fs_path, fs_path, ErrorCodes::CANNOT_RMDIR);
throwFromErrnoWithPath("Cannot rmdir " + fs_path.string(), fs_path, ErrorCodes::CANNOT_RMDIR);
}
void DiskLocal::removeRecursive(const String & path)
@ -261,39 +261,41 @@ void DiskLocal::removeRecursive(const String & path)
void DiskLocal::listFiles(const String & path, std::vector<String> & file_names)
{
Poco::File(disk_path + path).list(file_names);
file_names.clear();
for (auto & entry : fs::directory_iterator(disk_path / path))
file_names.emplace_back(entry.path().filename());
}
void DiskLocal::setLastModified(const String & path, const Poco::Timestamp & timestamp)
{
Poco::File(disk_path + path).setLastModified(timestamp);
Poco::File(disk_path / path).setLastModified(timestamp);
}
Poco::Timestamp DiskLocal::getLastModified(const String & path)
{
return Poco::File(disk_path + path).getLastModified();
return Poco::File(disk_path / path).getLastModified();
}
void DiskLocal::createHardLink(const String & src_path, const String & dst_path)
{
DB::createHardLink(disk_path + src_path, disk_path + dst_path);
DB::createHardLink(disk_path / src_path, disk_path / dst_path);
}
void DiskLocal::truncateFile(const String & path, size_t size)
{
int res = truncate((disk_path + path).c_str(), size);
int res = truncate((disk_path / path).string().data(), size);
if (-1 == res)
throwFromErrnoWithPath("Cannot truncate file " + path, path, ErrorCodes::CANNOT_TRUNCATE_FILE);
}
void DiskLocal::createFile(const String & path)
{
Poco::File(disk_path + path).createFile();
Poco::File(disk_path / path).createFile();
}
void DiskLocal::setReadOnly(const String & path)
{
Poco::File(disk_path + path).setReadOnly(true);
Poco::File(disk_path / path).setReadOnly(true);
}
bool inline isSameDiskType(const IDisk & one, const IDisk & another)
@ -304,14 +306,14 @@ bool inline isSameDiskType(const IDisk & one, const IDisk & another)
void DiskLocal::copy(const String & from_path, const std::shared_ptr<IDisk> & to_disk, const String & to_path)
{
if (isSameDiskType(*this, *to_disk))
Poco::File(disk_path + from_path).copyTo(to_disk->getPath() + to_path); /// Use more optimal way.
Poco::File(disk_path / from_path).copyTo(to_disk->getPath() + to_path); /// Use more optimal way.
else
IDisk::copy(from_path, to_disk, to_path); /// Copy files through buffers.
}
SyncGuardPtr DiskLocal::getDirectorySyncGuard(const String & path) const
{
return std::make_unique<LocalDirectorySyncGuard>(disk_path + path);
return std::make_unique<LocalDirectorySyncGuard>(disk_path / path);
}
DiskPtr DiskLocalReservation::getDisk(size_t i) const

14
src/Disks/DiskLocal.h
View File

@ -8,6 +8,9 @@
#include <Poco/DirectoryIterator.h>
#include <Poco/File.h>
#include <filesystem>
namespace fs = std::filesystem;
namespace DB
{
@ -24,15 +27,15 @@ public:
friend class DiskLocalReservation;
DiskLocal(const String & name_, const String & path_, UInt64 keep_free_space_bytes_)
: name(name_), disk_path(path_), keep_free_space_bytes(keep_free_space_bytes_)
: name(name_), disk_path(path_), disk_path_str(path_), keep_free_space_bytes(keep_free_space_bytes_)
{
if (disk_path.back() != '/')
throw Exception("Disk path must ends with '/', but '" + disk_path + "' doesn't.", ErrorCodes::LOGICAL_ERROR);
if (disk_path_str.back() != '/')
throw Exception("Disk path must ends with '/', but '" + disk_path_str + "' doesn't.", ErrorCodes::LOGICAL_ERROR);
}
const String & getName() const override { return name; }
const String & getPath() const override { return disk_path; }
const String & getPath() const override { return disk_path_str; }
ReservationPtr reserve(UInt64 bytes) override;
@ -109,7 +112,8 @@ private:
private:
const String name;
const String disk_path;
const fs::path disk_path;
const String disk_path_str;
const UInt64 keep_free_space_bytes;
UInt64 reserved_bytes = 0;

15
src/Disks/DiskMemory.cpp
View File

@ -6,8 +6,9 @@
#include <IO/WriteBufferFromFileBase.h>
#include <IO/WriteBufferFromString.h>
#include <Interpreters/Context.h>
#include <Poco/Path.h>
#include <filesystem>
namespace fs = std::filesystem;
namespace DB
{
@ -24,7 +25,7 @@ namespace ErrorCodes
class DiskMemoryDirectoryIterator final : public IDiskDirectoryIterator
{
public:
explicit DiskMemoryDirectoryIterator(std::vector<Poco::Path> && dir_file_paths_)
explicit DiskMemoryDirectoryIterator(std::vector<fs::path> && dir_file_paths_)
: dir_file_paths(std::move(dir_file_paths_)), iter(dir_file_paths.begin())
{
}
@ -33,13 +34,13 @@ public:
bool isValid() const override { return iter != dir_file_paths.end(); }
String path() const override { return (*iter).toString(); }
String path() const override { return iter->string(); }
String name() const override { return (*iter).getFileName(); }
String name() const override { return iter->filename(); }
private:
std::vector<Poco::Path> dir_file_paths;
std::vector<Poco::Path>::iterator iter;
std::vector<fs::path> dir_file_paths;
std::vector<fs::path>::iterator iter;
};
@ -268,7 +269,7 @@ DiskDirectoryIteratorPtr DiskMemory::iterateDirectory(const String & path)
if (!path.empty() && files.find(path) == files.end())
throw Exception("Directory '" + path + "' does not exist", ErrorCodes::DIRECTORY_DOESNT_EXIST);
std::vector<Poco::Path> dir_file_paths;
std::vector<fs::path> dir_file_paths;
for (const auto & file : files)
if (parentPath(file.first) == path)
dir_file_paths.emplace_back(file.first);