diff --git a/dbms/src/Storages/MergeTree/MergeTreeData.cpp b/dbms/src/Storages/MergeTree/MergeTreeData.cpp index ea38f11454d..3063161852d 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeData.cpp +++ b/dbms/src/Storages/MergeTree/MergeTreeData.cpp @@ -355,7 +355,9 @@ void MergeTreeData::setSkipIndexes(const ASTs & indexes_asts, bool only_check) for (const auto &index_ast : indexes_asts) { indexes.push_back( std::move(MergeTreeIndexFactory::instance().get( - std::dynamic_pointer_cast(index_ast)))); + *this, + std::dynamic_pointer_cast(index_ast), + global_context))); } } } diff --git a/dbms/src/Storages/MergeTree/MergeTreeDataMergerMutator.cpp b/dbms/src/Storages/MergeTree/MergeTreeDataMergerMutator.cpp index 232146ea417..19b81833cbf 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeDataMergerMutator.cpp +++ b/dbms/src/Storages/MergeTree/MergeTreeDataMergerMutator.cpp @@ -553,18 +553,23 @@ MergeTreeData::MutableDataPartPtr MergeTreeDataMergerMutator::mergePartsToTempor Names all_column_names = data.getColumns().getNamesOfPhysical(); NamesAndTypesList all_columns = data.getColumns().getAllPhysical(); + LOG_DEBUG(log, "Before extract"); NamesAndTypesList gathering_columns, merging_columns; Names gathering_column_names, merging_column_names; extractMergingAndGatheringColumns( all_columns, data.sorting_key_expr, data.indexes, data.merging_params, gathering_columns, gathering_column_names, merging_columns, merging_column_names); + LOG_DEBUG(log, "After extract"); + MergeTreeData::MutableDataPartPtr new_data_part = std::make_shared( data, future_part.name, future_part.part_info); new_data_part->partition.assign(future_part.getPartition()); new_data_part->relative_path = TMP_PREFIX + future_part.name; new_data_part->is_temp = true; + LOG_DEBUG(log, "New Part"); + size_t sum_input_rows_upper_bound = merge_entry->total_size_marks * data.index_granularity; MergeAlgorithm merge_alg = chooseMergeAlgorithm(parts, sum_input_rows_upper_bound, gathering_columns, deduplicate); diff --git a/dbms/src/Storages/MergeTree/MergeTreeDataWriter.cpp b/dbms/src/Storages/MergeTree/MergeTreeDataWriter.cpp index 2b7ede696ad..5e5a7fecd21 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeDataWriter.cpp +++ b/dbms/src/Storages/MergeTree/MergeTreeDataWriter.cpp @@ -214,6 +214,17 @@ MergeTreeData::MutableDataPartPtr MergeTreeDataWriter::writeTempPart(BlockWithPa NamesAndTypesList columns = data.getColumns().getAllPhysical().filter(block.getNames()); MergedBlockOutputStream out(data, new_data_part->getFullPath(), columns, compression_codec); + for (auto index : data.indexes) + { + auto index_columns = index->expr->getRequiredColumnsWithTypes(); + for (const auto & column : index_columns) + { + if (!block.has(column.name)) + block.insert(ColumnWithTypeAndName(column.type, column.name)); + } + index->expr->execute(block); + } + out.writePrefix(); out.writeWithPermutation(block, perm_ptr); out.writeSuffixAndFinalizePart(new_data_part); diff --git a/dbms/src/Storages/MergeTree/MergeTreeIndexes.cpp b/dbms/src/Storages/MergeTree/MergeTreeIndexes.cpp index 2163fe151ef..2620dc6cc38 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeIndexes.cpp +++ b/dbms/src/Storages/MergeTree/MergeTreeIndexes.cpp @@ -18,36 +18,6 @@ namespace ErrorCodes } -void MergeTreeIndexes::writeText(DB::WriteBuffer &ostr) const -{ - writeString("indexes format version: 1\n", ostr); - DB::writeText(size(), ostr); - writeString(" indexes:\n", ostr); - for (auto index : *this) { - index->writeText(ostr); - writeChar('\n', ostr); - } -} - - -void MergeTreeIndexes::readText(DB::ReadBuffer &istr) -{ - const MergeTreeIndexFactory & factory = MergeTreeIndexFactory::instance(); - - assertString("indexes format version: 1\n", istr); - size_t count; - DB::readText(count, istr); - assertString(" indexes:\n", istr); - reserve(count); - for (size_t i = 0; i < count; ++i) { - String index_descr; - readString(index_descr, istr); - emplace_back(factory.get(index_descr)); - assertChar('\n', istr); - } -} - - void MergeTreeIndexFactory::registerIndex(const std::string &name, Creator creator) { if (!indexes.emplace(name, std::move(creator)).second) @@ -55,7 +25,10 @@ void MergeTreeIndexFactory::registerIndex(const std::string &name, Creator creat ErrorCodes::LOGICAL_ERROR); } -std::unique_ptr MergeTreeIndexFactory::get(std::shared_ptr node) const +std::unique_ptr MergeTreeIndexFactory::get( + const MergeTreeData & data, + std::shared_ptr node, + const Context & context) const { if (!node->type) throw Exception( @@ -74,14 +47,7 @@ std::unique_ptr MergeTreeIndexFactory::get(std::shared_ptrsecond(node); -} - -std::unique_ptr MergeTreeIndexFactory::get(const String & description) const -{ - ParserIndexDeclaration parser; - ASTPtr ast = parseQuery(parser, description.data(), description.data() + description.size(), "index factory", 0); - return get(std::dynamic_pointer_cast(ast)); + return it->second(data, node, context); } } \ No newline at end of file diff --git a/dbms/src/Storages/MergeTree/MergeTreeIndexes.h b/dbms/src/Storages/MergeTree/MergeTreeIndexes.h index 1a9efb8445f..cc97a82f3dc 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeIndexes.h +++ b/dbms/src/Storages/MergeTree/MergeTreeIndexes.h @@ -17,6 +17,7 @@ constexpr auto INDEX_FILE_PREFIX = "skp_idx_"; namespace DB { +class MergeTreeData; class MergeTreeIndex; using MergeTreeIndexPtr = std::shared_ptr; @@ -86,12 +87,7 @@ public: }; -class MergeTreeIndexes : public std::vector -{ -public: - void writeText(WriteBuffer & ostr) const; - void readText(ReadBuffer & istr); -}; +using MergeTreeIndexes = std::vector; class MergeTreeIndexFactory : public ext::singleton @@ -99,10 +95,16 @@ class MergeTreeIndexFactory : public ext::singleton friend class ext::singleton; public: - using Creator = std::function(std::shared_ptr node)>; + using Creator = std::function< + std::unique_ptr( + const MergeTreeData & data, + std::shared_ptr node, + const Context & context)>; - std::unique_ptr get(std::shared_ptr node) const; - std::unique_ptr get(const String & description) const; + std::unique_ptr get( + const MergeTreeData & data, + std::shared_ptr node, + const Context & context) const; void registerIndex(const std::string & name, Creator creator); diff --git a/dbms/src/Storages/MergeTree/MergeTreeTestIndex.h b/dbms/src/Storages/MergeTree/MergeTreeTestIndex.h index 538e45221e5..784c2b18f8c 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeTestIndex.h +++ b/dbms/src/Storages/MergeTree/MergeTreeTestIndex.h @@ -75,9 +75,10 @@ public: }; }; -std::unique_ptr MTItestCreator(std::shared_ptr node) { +std::unique_ptr MTItestCreator( + const MergeTreeData & data, std::shared_ptr node, const Context & ) { return std::make_unique( - node->name, nullptr, node->granularity.get(), Block{}); + node->name, data.primary_key_expr, node->granularity.get(), Block{}); } } \ No newline at end of file