From 61b9c7735bd1bc7d0eb857fa971f2948aa251cf7 Mon Sep 17 00:00:00 2001 From: Nikita Vasilev Date: Sat, 5 Jan 2019 21:33:30 +0300 Subject: [PATCH] add rw for MergeTreeIndexes --- .../Storages/MergeTree/MergeTreeIndexes.cpp | 47 +++++++++++++++++-- .../src/Storages/MergeTree/MergeTreeIndexes.h | 21 +++++---- dbms/src/Storages/MergeTree/checkDataPart.cpp | 10 ++-- 3 files changed, 60 insertions(+), 18 deletions(-) diff --git a/dbms/src/Storages/MergeTree/MergeTreeIndexes.cpp b/dbms/src/Storages/MergeTree/MergeTreeIndexes.cpp index c73999c2e3e..10256e76181 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeIndexes.cpp +++ b/dbms/src/Storages/MergeTree/MergeTreeIndexes.cpp @@ -1,4 +1,9 @@ #include +#include +#include +#include +#include + namespace DB { @@ -11,18 +16,45 @@ namespace ErrorCodes } -IndexType IndexCondition::indexType() const { - return index->indexType(); +void MergeTreeIndexes::writeText(DB::WriteBuffer &ostr) const +{ + writeString("indexes format version: 1\n", ostr); + DB::writeText(size(), ostr); + writeString(" indexes:\n", ostr); + for (auto index : *this) { + index->writeText(ostr); + writeChar('\n', ostr); + } } -void MergeTreeIndexFactory::registerIndex(const std::string &name, Creator creator) { +void MergeTreeIndexes::readText(DB::ReadBuffer &istr) +{ + const MergeTreeIndexFactory & factory = MergeTreeIndexFactory::instance(); + + assertString("indexes format version: 1\n", istr); + size_t count; + DB::readText(count, istr); + assertString(" indexes:\n", istr); + reserve(count); + for (size_t i = 0; i < count; ++i) { + String index_descr; + readString(index_descr, istr); + emplace_back(factory.get(index_descr)); + assertChar('\n', istr); + } +} + + +void MergeTreeIndexFactory::registerIndex(const std::string &name, Creator creator) +{ if (!indexes.emplace(name, std::move(creator)).second) throw Exception("MergeTreeIndexFactory: the Index creator name '" + name + "' is not unique", ErrorCodes::LOGICAL_ERROR); } -std::unique_ptr MergeTreeIndexFactory::get(std::shared_ptr node) const { +std::unique_ptr MergeTreeIndexFactory::get(std::shared_ptr node) const +{ if (!node->type) throw Exception( "for INDEX TYPE is required", @@ -35,4 +67,11 @@ std::unique_ptr MergeTreeIndexFactory::get(std::shared_ptrsecond(node); } +std::unique_ptr MergeTreeIndexFactory::get(const String & description) const +{ + ParserIndexDeclaration parser; + ASTPtr ast = parseQuery(parser, description.data(), description.data() + description.size(), "index factory", 0); + return get(std::dynamic_pointer_cast(ast)); +} + } \ No newline at end of file diff --git a/dbms/src/Storages/MergeTree/MergeTreeIndexes.h b/dbms/src/Storages/MergeTree/MergeTreeIndexes.h index 178c23ca373..bdd7bee17d4 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeIndexes.h +++ b/dbms/src/Storages/MergeTree/MergeTreeIndexes.h @@ -17,16 +17,11 @@ constexpr auto INDEX_FILE_PREFIX = "skp_idx_"; namespace DB { -enum class IndexType { - NONE = 0 -}; - class MergeTreeIndex; using MergeTreeIndexPtr = std::shared_ptr; using MutableMergeTreeIndexPtr = std::shared_ptr; -using MergeTreeIndexes = std::vector; /// Condition on the index. @@ -37,8 +32,6 @@ class IndexCondition { public: virtual ~IndexCondition() = default; - IndexType indexType() const; - /// Checks if this index is useful for query. virtual bool alwaysUnknownOrTrue() const = 0; @@ -82,18 +75,18 @@ public: virtual ~MergeTreeIndex() {}; - virtual IndexType indexType() const = 0; + virtual String indexType() const { return "UNKNOWN"; }; /// gets filename without extension virtual String getFileName() const = 0; - String getFileExt() const { return ".idx"; }; - virtual MergeTreeIndexGranulePtr createIndexGranule() const = 0; virtual IndexConditionPtr createIndexConditionOnPart( const SelectQueryInfo & query_info, const Context & context) const = 0; + virtual void writeText(WriteBuffer & ostr) const = 0; + String name; ExpressionActionsPtr expr; size_t granularity; @@ -103,6 +96,13 @@ public: }; +class MergeTreeIndexes : public std::vector +{ + void writeText(WriteBuffer & ostr) const; + void readText(ReadBuffer & istr); +}; + + class MergeTreeIndexFactory : public ext::singleton { friend class ext::singleton; @@ -111,6 +111,7 @@ public: using Creator = std::function(std::shared_ptr node)>; std::unique_ptr get(std::shared_ptr node) const; + std::unique_ptr get(const String & description) const; void registerIndex(const std::string & name, Creator creator); diff --git a/dbms/src/Storages/MergeTree/checkDataPart.cpp b/dbms/src/Storages/MergeTree/checkDataPart.cpp index eac9145692b..60ef117f0d7 100644 --- a/dbms/src/Storages/MergeTree/checkDataPart.cpp +++ b/dbms/src/Storages/MergeTree/checkDataPart.cpp @@ -30,12 +30,13 @@ namespace ErrorCodes namespace { -/** To read and checksum single stream (a pair of .bin, .mrk files) for a single column. +/** To read and checksum single stream (a pair of .bin, .mrk files) for a single column or secondary index. */ class Stream { public: String base_name; + String bin_file_ext; String bin_file_path; String mrk_file_path; private: @@ -50,10 +51,11 @@ private: public: HashingReadBuffer mrk_hashing_buf; - Stream(const String & path, const String & base_name) + Stream(const String & path, const String & base_name, const String & bin_file_ext = ".bin") : base_name(base_name), - bin_file_path(path + base_name + ".bin"), + bin_file_ext(bin_file_ext), + bin_file_path(path + base_name + bin_file_ext), mrk_file_path(path + base_name + ".mrk"), file_buf(bin_file_path), compressed_hashing_buf(file_buf), @@ -118,7 +120,7 @@ public: void saveChecksums(MergeTreeData::DataPart::Checksums & checksums) { - checksums.files[base_name + ".bin"] = MergeTreeData::DataPart::Checksums::Checksum( + checksums.files[base_name + bin_file_ext] = MergeTreeData::DataPart::Checksums::Checksum( compressed_hashing_buf.count(), compressed_hashing_buf.getHash(), uncompressed_hashing_buf.count(), uncompressed_hashing_buf.getHash());