ClickHouse/dbms/src/Common/FileChecker.cpp

159 lines
4.0 KiB
C++
Raw Normal View History

2015-09-29 19:19:54 +00:00
#include <common/JSON.h>
#include <Poco/Path.h>
#include <IO/WriteBufferFromFile.h>
#include <IO/ReadBufferFromFile.h>
#include <IO/WriteBufferFromString.h>
#include <IO/WriteHelpers.h>
#include <IO/ReadHelpers.h>
#include <Common/escapeForFileName.h>
#include <Common/FileChecker.h>
namespace DB
{
FileChecker::FileChecker(const std::string & file_info_path_)
{
setPath(file_info_path_);
}
void FileChecker::setPath(const std::string & file_info_path_)
{
files_info_path = file_info_path_;
Poco::Path path(files_info_path);
tmp_files_info_path = path.parent().toString() + "tmp_" + path.getFileName();
}
void FileChecker::update(const Poco::File & file)
{
initialize();
updateImpl(file);
save();
}
void FileChecker::update(const Files::const_iterator & begin, const Files::const_iterator & end)
{
initialize();
for (auto it = begin; it != end; ++it)
updateImpl(*it);
save();
}
2019-07-03 13:17:19 +00:00
CheckResults FileChecker::check() const
{
/** Read the files again every time you call `check` - so as not to violate the constancy.
* `check` method is rarely called.
*/
2019-07-03 13:17:19 +00:00
CheckResults results;
Map local_map;
2018-08-26 02:08:35 +00:00
load(local_map, files_info_path);
if (local_map.empty())
2019-07-03 13:17:19 +00:00
return {};
for (const auto & name_size : local_map)
{
2019-07-03 16:00:24 +00:00
Poco::Path path = Poco::Path(files_info_path).parent().toString() + "/" + name_size.first;
Poco::File file(path);
if (!file.exists())
{
2019-07-03 16:00:24 +00:00
results.emplace_back(path.getFileName(), false, "File " + file.path() + " doesn't exist");
2019-07-03 13:17:19 +00:00
break;
}
2019-07-03 13:17:19 +00:00
size_t real_size = file.getSize();
if (real_size != name_size.second)
{
2019-07-03 16:00:24 +00:00
results.emplace_back(path.getFileName(), false, "Size of " + file.path() + " is wrong. Size is " + toString(real_size) + " but should be " + toString(name_size.second));
2019-07-03 13:17:19 +00:00
break;
}
2019-07-03 16:00:24 +00:00
results.emplace_back(path.getFileName(), true, "");
}
2019-07-03 13:17:19 +00:00
return results;
}
void FileChecker::initialize()
{
if (initialized)
return;
2018-08-26 02:08:35 +00:00
load(map, files_info_path);
initialized = true;
}
void FileChecker::updateImpl(const Poco::File & file)
{
map[Poco::Path(file.path()).getFileName()] = file.getSize();
}
void FileChecker::save() const
{
{
WriteBufferFromFile out(tmp_files_info_path);
/// So complex JSON structure - for compatibility with the old format.
writeCString("{\"yandex\":{", out);
auto settings = FormatSettings();
for (auto it = map.begin(); it != map.end(); ++it)
{
if (it != map.begin())
writeString(",", out);
/// `escapeForFileName` is not really needed. But it is left for compatibility with the old code.
writeJSONString(escapeForFileName(it->first), out, settings);
writeString(":{\"size\":\"", out);
writeIntText(it->second, out);
writeString("\"}", out);
}
writeCString("}}", out);
out.next();
}
Poco::File current_file(files_info_path);
if (current_file.exists())
{
std::string old_file_name = files_info_path + ".old";
current_file.renameTo(old_file_name);
Poco::File(tmp_files_info_path).renameTo(files_info_path);
Poco::File(old_file_name).remove();
}
else
Poco::File(tmp_files_info_path).renameTo(files_info_path);
}
2018-08-26 02:08:35 +00:00
void FileChecker::load(Map & local_map, const std::string & path)
{
2018-08-26 02:08:35 +00:00
local_map.clear();
2018-08-26 02:08:35 +00:00
if (!Poco::File(path).exists())
return;
2018-08-26 02:08:35 +00:00
ReadBufferFromFile in(path);
2017-07-31 21:39:24 +00:00
WriteBufferFromOwnString out;
2017-07-31 21:39:24 +00:00
/// The JSON library does not support whitespace. We delete them. Inefficient.
while (!in.eof())
{
char c;
readChar(c, in);
if (!isspace(c))
writeChar(c, out);
}
2017-07-31 21:39:24 +00:00
JSON json(out.str());
JSON files = json["yandex"];
for (const JSON name_value : files)
2018-08-26 02:08:35 +00:00
local_map[unescapeForFileName(name_value.getName())] = name_value.getValue()["size"].toUInt();
}
}