ClickHouse/src/Common/FileChecker.cpp

Ignoring revisions in .git-blame-ignore-revs. Click here to bypass and see the normal blame view.

206 lines
5.9 KiB
C++
Raw Normal View History

#include <Common/FileChecker.h>
#include <Common/escapeForFileName.h>
#include <Common/logger_useful.h>
#include <Common/ErrorCodes.h>
#include <Disks/IDisk.h>
#include <IO/WriteBufferFromFile.h>
#include <IO/ReadBufferFromFile.h>
#include <IO/WriteBufferFromString.h>
#include <IO/WriteHelpers.h>
#include <IO/ReadHelpers.h>
#include <base/JSON.h>
namespace fs = std::filesystem;
namespace DB
{
namespace ErrorCodes
{
extern const int UNEXPECTED_END_OF_FILE;
extern const int LOGICAL_ERROR;
}
FileChecker::FileChecker(const String & file_info_path_) : FileChecker(nullptr, file_info_path_)
{
}
FileChecker::FileChecker(DiskPtr disk_, const String & file_info_path_)
: disk(std::move(disk_))
2024-01-23 17:04:50 +00:00
, log(getLogger("FileChecker"))
{
setPath(file_info_path_);
try
{
load();
}
catch (DB::Exception & e)
{
e.addMessage("Error loading file {}", files_info_path);
throw;
}
}
void FileChecker::setPath(const String & file_info_path_)
{
files_info_path = file_info_path_;
}
String FileChecker::getPath() const
{
return files_info_path;
}
void FileChecker::update(const String & full_file_path)
{
bool exists = fileReallyExists(full_file_path);
auto real_size = exists ? getRealFileSize(full_file_path) : 0; /// No race condition assuming no one else is working with these files.
map[fileName(full_file_path)] = real_size;
}
void FileChecker::setEmpty(const String & full_file_path)
{
map[fileName(full_file_path)] = 0;
}
size_t FileChecker::getFileSize(const String & full_file_path) const
{
auto it = map.find(fileName(full_file_path));
if (it == map.end())
throw Exception(ErrorCodes::LOGICAL_ERROR, "File {} is not added to the file checker", full_file_path);
return it->second;
}
size_t FileChecker::getTotalSize() const
{
size_t total_size = 0;
for (auto size : map | boost::adaptors::map_values)
total_size += size;
return total_size;
}
FileChecker::DataValidationTasksPtr FileChecker::getDataValidationTasks()
{
return std::make_unique<DataValidationTasks>(map);
}
std::optional<CheckResult> FileChecker::checkNextEntry(DataValidationTasksPtr & check_data_tasks) const
{
String name;
size_t expected_size;
bool is_finished = check_data_tasks->next(name, expected_size);
if (is_finished)
return {};
String path = parentPath(files_info_path) + name;
bool exists = fileReallyExists(path);
auto real_size = exists ? getRealFileSize(path) : 0; /// No race condition assuming no one else is working with these files.
if (real_size != expected_size)
{
String failure_message = exists
? ("Size of " + path + " is wrong. Size is " + toString(real_size) + " but should be " + toString(expected_size))
: ("File " + path + " doesn't exist");
return CheckResult(name, false, failure_message);
}
2023-08-01 10:57:59 +00:00
return CheckResult(name, true, "");
}
void FileChecker::repair()
{
for (const auto & name_size : map)
{
const String & name = name_size.first;
size_t expected_size = name_size.second;
String path = parentPath(files_info_path) + name;
bool exists = fileReallyExists(path);
auto real_size = exists ? getRealFileSize(path) : 0; /// No race condition assuming no one else is working with these files.
if (real_size < expected_size)
throw Exception(ErrorCodes::UNEXPECTED_END_OF_FILE, "Size of {} is less than expected. Size is {} but should be {}.",
path, real_size, expected_size);
if (real_size > expected_size)
{
LOG_WARNING(log, "Will truncate file {} that has size {} to size {}", path, real_size, expected_size);
disk->truncateFile(path, expected_size);
}
}
}
void FileChecker::save() const
{
std::string tmp_files_info_path = parentPath(files_info_path) + "tmp_" + fileName(files_info_path);
{
2023-05-27 09:47:44 +00:00
std::unique_ptr<WriteBufferFromFileBase> out = disk ? disk->writeFile(tmp_files_info_path) : std::make_unique<WriteBufferFromFile>(tmp_files_info_path);
2017-03-25 20:12:56 +00:00
/// So complex JSON structure - for compatibility with the old format.
2021-09-21 18:05:08 +00:00
writeCString("{\"clickhouse\":{", *out);
auto settings = FormatSettings();
for (auto it = map.begin(); it != map.end(); ++it)
{
if (it != map.begin())
writeString(",", *out);
/// `escapeForFileName` is not really needed. But it is left for compatibility with the old code.
writeJSONString(escapeForFileName(it->first), *out, settings);
2020-03-08 21:04:10 +00:00
writeString(R"(:{"size":")", *out);
writeIntText(it->second, *out);
writeString("\"}", *out);
}
writeCString("}}", *out);
2023-05-27 09:47:44 +00:00
out->sync();
out->finalize();
}
if (disk)
disk->replaceFile(tmp_files_info_path, files_info_path);
else
fs::rename(tmp_files_info_path, files_info_path);
}
void FileChecker::load()
{
map.clear();
if (!fileReallyExists(files_info_path))
return;
std::unique_ptr<ReadBuffer> in = disk ? disk->readFile(files_info_path) : std::make_unique<ReadBufferFromFile>(files_info_path);
2017-07-31 21:39:24 +00:00
WriteBufferFromOwnString out;
2017-07-31 21:39:24 +00:00
/// The JSON library does not support whitespace. We delete them. Inefficient.
while (!in->eof())
2017-07-31 21:39:24 +00:00
{
char c;
readChar(c, *in);
2017-07-31 21:39:24 +00:00
if (!isspace(c))
writeChar(c, out);
}
2017-07-31 21:39:24 +00:00
JSON json(out.str());
2021-09-21 18:05:08 +00:00
JSON files = json.has("clickhouse") ? json["clickhouse"] : json["yandex"];
2020-03-08 21:04:10 +00:00
for (const JSON file : files) // NOLINT
map[unescapeForFileName(file.getName())] = file.getValue()["size"].toUInt();
}
bool FileChecker::fileReallyExists(const String & path_) const
{
return disk ? disk->exists(path_) : fs::exists(path_);
}
size_t FileChecker::getRealFileSize(const String & path_) const
{
return disk ? disk->getFileSize(path_) : fs::file_size(path_);
}
}