From cf1afb1bd137443274dff37c722f5a22cadac2da Mon Sep 17 00:00:00 2001 From: Pavel Kartavyy Date: Fri, 1 Aug 2014 17:19:27 +0400 Subject: [PATCH] dbms: FileChecker moved to separate class [#METR-11709] --- dbms/include/DB/Common/FileChecker.h | 71 +++++++++++++++++++ dbms/include/DB/Storages/StorageTinyLog.h | 26 +++---- dbms/src/Storages/StorageTinyLog.cpp | 52 ++++---------- .../queries/0_stateless/00063_check_query.sql | 5 ++ 4 files changed, 99 insertions(+), 55 deletions(-) create mode 100644 dbms/include/DB/Common/FileChecker.h create mode 100644 dbms/tests/queries/0_stateless/00063_check_query.sql diff --git a/dbms/include/DB/Common/FileChecker.h b/dbms/include/DB/Common/FileChecker.h new file mode 100644 index 00000000000..a3d9f9c675b --- /dev/null +++ b/dbms/include/DB/Common/FileChecker.h @@ -0,0 +1,71 @@ +#pragma once + +#include +#include +#include +#include + +/// хранит размеры всех столбцов, и может проверять не побились ли столбцы +template +class FileChecker +{ +public: + FileChecker(const std::string &file_info_path_, Storage & storage_) : + files_info_path(file_info_path_), files_info(new Poco::Util::XMLConfiguration), storage(storage_), log(&Logger::get("FileChecker")) + { + try + { + files_info->load(files_info_path); + } + catch (Poco::FileNotFoundException & e) + { + files_info->loadEmpty("yandex"); + } + } + + void setPath(const std::string & file_info_path_) + { + files_info_path = file_info_path_; + } + + template + void update(const Iterator & begin, const Iterator & end) + { + for (auto it = begin; it != end; ++it) + { + auto & column_name = *it; + auto & file = storage.getFiles()[column_name].data_file; + files_info->setString(column_name + ".size", std::to_string(file.getSize())); + } + files_info->save(files_info_path); + } + + bool check() const + { + bool sizes_are_correct = true; + for (auto & pair : storage.getFiles()) + { + if (files_info->has(pair.first)) + { + auto & file = pair.second.data_file; + size_t expected_size = std::stoull(files_info->getString(pair.first + ".size")); + size_t real_size = file.getSize(); + if (real_size != expected_size) + { + LOG_ERROR(log, "Size of " << file.path() << "is wrong. Size is " << real_size << " but should be " << expected_size); + sizes_are_correct = false; + } + } + } + return sizes_are_correct; + } + +private: + std::string files_info_path; + + using FileInfo = Poco::AutoPtr; + FileInfo files_info; + + Storage & storage; + Logger * log; +}; diff --git a/dbms/include/DB/Storages/StorageTinyLog.h b/dbms/include/DB/Storages/StorageTinyLog.h index 9be09115b71..d91cc88d9a3 100644 --- a/dbms/include/DB/Storages/StorageTinyLog.h +++ b/dbms/include/DB/Storages/StorageTinyLog.h @@ -12,6 +12,7 @@ #include #include #include +#include #include @@ -56,7 +57,6 @@ private: void readData(const String & name, const IDataType & type, IColumn & column, size_t limit, size_t level = 0, bool read_offsets = true); }; - class TinyLogBlockOutputStream : public IBlockOutputStream { public: @@ -94,8 +94,6 @@ private: void addStream(const String & name, const IDataType & type, size_t level = 0); void writeData(const String & name, const IDataType & type, const IColumn & column, OffsetColumns & offset_columns, size_t level = 0); - - void updateFileSizes(const FileStreams::const_iterator & begin, const FileStreams::const_iterator & end); }; @@ -137,6 +135,15 @@ public: bool checkData() const override; + /// Данные столбца + struct ColumnData + { + Poco::File data_file; + }; + typedef std::map Files_t; + + Files_t & getFiles(); + private: String path; String name; @@ -144,20 +151,9 @@ private: size_t max_compress_block_size; - /// Данные столбца - struct ColumnData - { - Poco::File data_file; - }; - typedef std::map Files_t; Files_t files; - std::string size_file_path; - - /// хранит размеры всех столбцов, чтобы проверять не побились ли они - using SizeFile = Poco::AutoPtr; - SizeFile size_file; - SizeFile & sizeFile() { return size_file; } + FileChecker file_checker; Logger * log; diff --git a/dbms/src/Storages/StorageTinyLog.cpp b/dbms/src/Storages/StorageTinyLog.cpp index 367fd2b1ae0..c1dcdfef474 100644 --- a/dbms/src/Storages/StorageTinyLog.cpp +++ b/dbms/src/Storages/StorageTinyLog.cpp @@ -266,7 +266,12 @@ void TinyLogBlockOutputStream::writeSuffix() for (FileStreams::iterator it = streams.begin(); it != streams.end(); ++it) it->second->finalize(); - updateFileSizes(streams.begin(), streams.end()); + /// @TODO лишнее копирование. Можно б было использовать boost::transform_iterator, если б он работал с C++11 lambda + std::vector column_names; + for (auto & pair : streams) + column_names.push_back(pair.first); + + storage.file_checker.update(column_names.begin(), column_names.end()); streams.clear(); } @@ -289,7 +294,8 @@ void TinyLogBlockOutputStream::write(const Block & block) StorageTinyLog::StorageTinyLog(const std::string & path_, const std::string & name_, NamesAndTypesListPtr columns_, bool attach, size_t max_compress_block_size_) : path(path_), name(name_), columns(columns_), - max_compress_block_size(max_compress_block_size_), size_file(new Poco::Util::XMLConfiguration()), + max_compress_block_size(max_compress_block_size_), + file_checker(path + escapeForFileName(name) + '/' + "sizes.txt", *this), log(&Logger::get("StorageTinyLog")) { if (columns->empty()) @@ -305,17 +311,6 @@ StorageTinyLog::StorageTinyLog(const std::string & path_, const std::string & na for (NamesAndTypesList::const_iterator it = columns->begin(); it != columns->end(); ++it) addFile(it->name, *it->type); - - try - { - size_file_path = full_path + "sizes.txt"; - size_file->load(size_file_path); - } - catch (const Poco::FileNotFoundException & e) - { - /// нормальная ситуация, для старых таблиц файла не существует - size_file->loadEmpty("yandex"); - } } StoragePtr StorageTinyLog::create(const std::string & path_, const std::string & name_, NamesAndTypesListPtr columns_, bool attach, size_t max_compress_block_size_) @@ -375,7 +370,7 @@ void StorageTinyLog::rename(const String & new_path_to_db, const String & new_da path = new_path_to_db; name = new_table_name; - size_file_path = path + escapeForFileName(name) + "/" + "sizes.txt"; + file_checker.setPath(path + escapeForFileName(name) + "/" + "sizes.txt"); for (Files_t::iterator it = files.begin(); it != files.end(); ++it) it->second.data_file = Poco::File(path + escapeForFileName(name) + '/' + Poco::Path(it->second.data_file.path()).getFileName()); @@ -412,35 +407,12 @@ void StorageTinyLog::drop() bool StorageTinyLog::checkData() const { - bool sizes_are_correct = true; - for (auto & pair : files) - { - if (size_file->has(pair.first)) - { - auto & file = pair.second.data_file; - size_t expected_size = std::stoull(size_file->getString(pair.first + ".size")); - size_t real_size = file.getSize(); - if (real_size != expected_size) - { - LOG_ERROR(log, "Size of " << file.path() << "is wrong. Size is " << real_size << " but should be " << expected_size); - sizes_are_correct = false; - } - } - } - return sizes_are_correct; + return file_checker.check(); } -void TinyLogBlockOutputStream::updateFileSizes(const FileStreams::const_iterator & begin, - const FileStreams::const_iterator & end) +StorageTinyLog::Files_t & StorageTinyLog::getFiles() { - auto & size_file = storage.sizeFile(); - for (auto it = begin; it != end; ++it) - { - auto & column_name = it->first; - auto & file = storage.files[column_name].data_file; - size_file->setString(column_name + ".size", std::to_string(file.getSize())); - } - size_file->save(storage.size_file_path); + return files; } TinyLogBlockOutputStream::~TinyLogBlockOutputStream() diff --git a/dbms/tests/queries/0_stateless/00063_check_query.sql b/dbms/tests/queries/0_stateless/00063_check_query.sql new file mode 100644 index 00000000000..d2fa620fe39 --- /dev/null +++ b/dbms/tests/queries/0_stateless/00063_check_query.sql @@ -0,0 +1,5 @@ +CREATE TABLE check_query_tiny_log (UInt32 N, String S) Engine = TinyLog; + +INSERT INTO check_query_tiny_log VALUES (1, "A"), (2, "B"), (3, "C") + +DROP TABLE check_query_tiny_log;