From 71642b08809d8e6e29fc1e46d22a7e5829886116 Mon Sep 17 00:00:00 2001 From: Alexey Zatelepin Date: Tue, 6 Mar 2018 23:18:34 +0300 Subject: [PATCH 1/8] use ColumnsDescription everywhere [#CLICKHOUSE-3241] --- dbms/src/Analyzers/AnalyzeColumns.cpp | 4 +- dbms/src/Analyzers/ExecuteTableFunctions.cpp | 2 +- dbms/src/Common/ExternalTable.h | 4 +- dbms/src/Core/iostream_debug_helpers.cpp | 2 +- .../DataStreams/RemoteBlockInputStream.cpp | 2 +- dbms/src/Databases/DatabaseDictionary.cpp | 9 +- dbms/src/Databases/DatabaseDictionary.h | 5 +- dbms/src/Databases/DatabaseMemory.cpp | 5 +- dbms/src/Databases/DatabaseMemory.h | 5 +- dbms/src/Databases/DatabaseOrdinary.cpp | 7 +- dbms/src/Databases/DatabaseOrdinary.h | 5 +- dbms/src/Databases/DatabasesCommon.cpp | 4 +- dbms/src/Databases/IDatabase.h | 7 +- dbms/src/Interpreters/ExpressionAnalyzer.cpp | 12 +- .../Interpreters/InterpreterAlterQuery.cpp | 2 +- .../Interpreters/InterpreterCreateQuery.cpp | 54 ++++---- .../src/Interpreters/InterpreterCreateQuery.h | 20 +-- .../Interpreters/InterpreterDescribeQuery.cpp | 6 +- .../Interpreters/InterpreterInsertQuery.cpp | 6 +- .../Interpreters/InterpreterSelectQuery.cpp | 10 +- dbms/src/Server/TCPHandler.cpp | 3 +- dbms/src/Storages/AlterCommands.cpp | 99 +++++++-------- dbms/src/Storages/AlterCommands.h | 12 +- dbms/src/Storages/ColumnsDescription.cpp | 63 +++++++--- dbms/src/Storages/ColumnsDescription.h | 45 +++++-- .../Storages/Distributed/DirectoryMonitor.cpp | 2 +- dbms/src/Storages/IStorage.h | 4 + dbms/src/Storages/ITableDeclaration.cpp | 115 ++++-------------- dbms/src/Storages/ITableDeclaration.h | 65 +--------- .../Storages/MergeTree/DataPartsExchange.cpp | 1 + .../MergeTree/MergeTreeBlockInputStream.cpp | 4 +- .../MergeTree/MergeTreeBlockReadUtils.cpp | 4 +- dbms/src/Storages/MergeTree/MergeTreeData.cpp | 36 ++---- dbms/src/Storages/MergeTree/MergeTreeData.h | 21 ++-- .../MergeTree/MergeTreeDataMerger.cpp | 4 +- .../Storages/MergeTree/MergeTreeDataPart.cpp | 6 +- .../MergeTree/MergeTreeDataSelectExecutor.cpp | 4 +- .../MergeTree/MergeTreeDataSelectExecutor.h | 2 + .../MergeTree/MergeTreeDataWriter.cpp | 2 +- .../Storages/MergeTree/MergeTreeReadPool.cpp | 4 +- .../Storages/MergeTree/MergeTreeReader.cpp | 4 +- .../MergeTree/MergeTreeWhereOptimizer.cpp | 4 +- .../ReplicatedMergeTreeAlterThread.cpp | 44 +------ .../MergeTree/registerStorageMergeTree.cpp | 5 +- dbms/src/Storages/StorageBuffer.cpp | 15 +-- dbms/src/Storages/StorageBuffer.h | 5 +- dbms/src/Storages/StorageCatBoostPool.cpp | 11 +- dbms/src/Storages/StorageDictionary.cpp | 12 +- dbms/src/Storages/StorageDictionary.h | 6 +- dbms/src/Storages/StorageDistributed.cpp | 30 ++--- dbms/src/Storages/StorageDistributed.h | 16 +-- dbms/src/Storages/StorageFactory.cpp | 14 +-- dbms/src/Storages/StorageFactory.h | 12 +- dbms/src/Storages/StorageFile.cpp | 8 +- dbms/src/Storages/StorageFile.h | 12 +- dbms/src/Storages/StorageJoin.cpp | 22 +--- dbms/src/Storages/StorageJoin.h | 5 +- dbms/src/Storages/StorageKafka.cpp | 8 +- dbms/src/Storages/StorageKafka.h | 5 +- dbms/src/Storages/StorageLog.cpp | 18 ++- dbms/src/Storages/StorageLog.h | 5 +- dbms/src/Storages/StorageMaterializedView.cpp | 10 +- dbms/src/Storages/StorageMaterializedView.h | 6 +- dbms/src/Storages/StorageMemory.cpp | 12 +- dbms/src/Storages/StorageMemory.h | 13 +- dbms/src/Storages/StorageMerge.cpp | 16 +-- dbms/src/Storages/StorageMerge.h | 6 +- dbms/src/Storages/StorageMergeTree.cpp | 48 ++------ dbms/src/Storages/StorageMergeTree.h | 8 +- dbms/src/Storages/StorageMySQL.cpp | 6 +- dbms/src/Storages/StorageMySQL.h | 2 +- dbms/src/Storages/StorageNull.cpp | 9 +- dbms/src/Storages/StorageNull.h | 17 ++- dbms/src/Storages/StorageODBC.cpp | 6 +- dbms/src/Storages/StorageODBC.h | 2 +- .../Storages/StorageReplicatedMergeTree.cpp | 70 +++-------- .../src/Storages/StorageReplicatedMergeTree.h | 7 +- dbms/src/Storages/StorageSet.cpp | 25 ++-- dbms/src/Storages/StorageSet.h | 16 +-- dbms/src/Storages/StorageStripeLog.cpp | 8 +- dbms/src/Storages/StorageStripeLog.h | 5 +- dbms/src/Storages/StorageTinyLog.cpp | 12 +- dbms/src/Storages/StorageTinyLog.h | 5 +- dbms/src/Storages/StorageView.cpp | 4 +- dbms/src/Storages/StorageView.h | 2 +- .../StorageSystemAsynchronousMetrics.cpp | 2 +- .../System/StorageSystemBuildOptions.cpp | 2 +- .../Storages/System/StorageSystemClusters.cpp | 2 +- .../Storages/System/StorageSystemColumns.cpp | 8 +- .../System/StorageSystemDatabases.cpp | 2 +- .../System/StorageSystemDictionaries.cpp | 2 +- .../Storages/System/StorageSystemEvents.cpp | 2 +- .../System/StorageSystemFunctions.cpp | 2 +- .../Storages/System/StorageSystemGraphite.cpp | 2 +- .../Storages/System/StorageSystemMerges.cpp | 2 +- .../Storages/System/StorageSystemMetrics.cpp | 2 +- .../Storages/System/StorageSystemModels.cpp | 2 +- .../Storages/System/StorageSystemNumbers.cpp | 2 +- dbms/src/Storages/System/StorageSystemOne.cpp | 2 +- .../System/StorageSystemPartsBase.cpp | 10 +- .../System/StorageSystemPartsColumns.cpp | 6 +- .../System/StorageSystemProcesses.cpp | 2 +- .../Storages/System/StorageSystemReplicas.cpp | 2 +- .../System/StorageSystemReplicationQueue.cpp | 2 +- .../Storages/System/StorageSystemSettings.cpp | 2 +- .../Storages/System/StorageSystemTables.cpp | 2 +- .../System/StorageSystemZooKeeper.cpp | 2 +- dbms/src/Storages/tests/storage_log.cpp | 4 +- .../src/TableFunctions/TableFunctionMerge.cpp | 12 +- .../src/TableFunctions/TableFunctionMySQL.cpp | 2 +- dbms/src/TableFunctions/TableFunctionODBC.cpp | 3 +- .../getStructureOfRemoteTable.cpp | 2 +- 112 files changed, 478 insertions(+), 850 deletions(-) diff --git a/dbms/src/Analyzers/AnalyzeColumns.cpp b/dbms/src/Analyzers/AnalyzeColumns.cpp index 6367699ea8e..78568b080c6 100644 --- a/dbms/src/Analyzers/AnalyzeColumns.cpp +++ b/dbms/src/Analyzers/AnalyzeColumns.cpp @@ -173,7 +173,7 @@ ASTPtr createASTIdentifierForColumnInTable(const String & column, const CollectT void createASTsForAllColumnsInTable(const CollectTables::TableInfo & table, ASTs & res) { if (table.storage) - for (const auto & name : table.storage->getColumnNamesList()) + for (const auto & name : table.storage->columns.getNames()) res.emplace_back(createASTIdentifierForColumnInTable(name, table)); else for (size_t i = 0, size = table.structure_of_subquery.columns(); i < size; ++i) @@ -315,7 +315,7 @@ void processIdentifier( } else if (table->storage) { - info.data_type = table->storage->getDataTypeByName(column_name); + info.data_type = table->storage->getColumn(column_name).type; } else throw Exception("Logical error: no storage and no structure of subquery is specified for table", ErrorCodes::LOGICAL_ERROR); diff --git a/dbms/src/Analyzers/ExecuteTableFunctions.cpp b/dbms/src/Analyzers/ExecuteTableFunctions.cpp index fab9c6c3a3f..326c5df62ed 100644 --- a/dbms/src/Analyzers/ExecuteTableFunctions.cpp +++ b/dbms/src/Analyzers/ExecuteTableFunctions.cpp @@ -72,7 +72,7 @@ void ExecuteTableFunctions::dump(WriteBuffer & out) const { writeString(table.second->getName(), out); writeCString("\n\n", out); - writeString(table.second->getColumnsList().toString(), out); + writeString(table.second->columns.getList().toString(), out); writeCString("\n", out); } } diff --git a/dbms/src/Common/ExternalTable.h b/dbms/src/Common/ExternalTable.h index 624604d30a1..b5b0c522339 100644 --- a/dbms/src/Common/ExternalTable.h +++ b/dbms/src/Common/ExternalTable.h @@ -198,7 +198,9 @@ public: /// Create table NamesAndTypesList columns = sample_block.getNamesAndTypesList(); - StoragePtr storage = StorageMemory::create(data.second, columns, NamesAndTypesList{}, NamesAndTypesList{}, ColumnDefaults{}); + StoragePtr storage = StorageMemory::create( + data.second, + ColumnsDescription{columns, NamesAndTypesList{}, NamesAndTypesList{}, ColumnDefaults{}}); storage->startup(); context.addExternalTable(data.second, storage); BlockOutputStreamPtr output = storage->write(ASTPtr(), context.getSettingsRef()); diff --git a/dbms/src/Core/iostream_debug_helpers.cpp b/dbms/src/Core/iostream_debug_helpers.cpp index 3fbbb02a98f..822bbf58376 100644 --- a/dbms/src/Core/iostream_debug_helpers.cpp +++ b/dbms/src/Core/iostream_debug_helpers.cpp @@ -44,7 +44,7 @@ std::ostream & operator<<(std::ostream & stream, const IDataType & what) std::ostream & operator<<(std::ostream & stream, const IStorage & what) { stream << "IStorage(name = " << what.getName() << ", tableName = " << what.getTableName() << ") {" - << what.getColumnsList().toString() + << what.columns.getList().toString() << "}"; // isRemote supportsSampling supportsFinal supportsPrewhere return stream; diff --git a/dbms/src/DataStreams/RemoteBlockInputStream.cpp b/dbms/src/DataStreams/RemoteBlockInputStream.cpp index 7b63b36cadc..079cabe8dee 100644 --- a/dbms/src/DataStreams/RemoteBlockInputStream.cpp +++ b/dbms/src/DataStreams/RemoteBlockInputStream.cpp @@ -138,7 +138,7 @@ void RemoteBlockInputStream::sendExternalTables() { StoragePtr cur = table.second; QueryProcessingStage::Enum stage = QueryProcessingStage::Complete; - BlockInputStreams input = cur->read(cur->getColumnNamesList(), {}, context, + BlockInputStreams input = cur->read(cur->columns.getNames(), {}, context, stage, DEFAULT_BLOCK_SIZE, 1); if (input.size() == 0) res.push_back(std::make_pair(std::make_shared(cur->getSampleBlock()), table.first)); diff --git a/dbms/src/Databases/DatabaseDictionary.cpp b/dbms/src/Databases/DatabaseDictionary.cpp index 8a10535fd09..439fe8f4d79 100644 --- a/dbms/src/Databases/DatabaseDictionary.cpp +++ b/dbms/src/Databases/DatabaseDictionary.cpp @@ -42,7 +42,7 @@ Tables DatabaseDictionary::loadTables() const DictionaryStructure & dictionary_structure = dict_ptr->getStructure(); auto columns = StorageDictionary::getNamesAndTypes(dictionary_structure); tables[name] = StorageDictionary::create(name, - columns, NamesAndTypesList{}, NamesAndTypesList{}, ColumnDefaults{}, dictionary_structure, name); + ColumnsDescription{columns, NamesAndTypesList{}, NamesAndTypesList{}, ColumnDefaults{}}, dictionary_structure, name); } } @@ -77,7 +77,7 @@ StoragePtr DatabaseDictionary::tryGetTable( const DictionaryStructure & dictionary_structure = dict_ptr->getStructure(); auto columns = StorageDictionary::getNamesAndTypes(dictionary_structure); return StorageDictionary::create(table_name, - columns, NamesAndTypesList{}, NamesAndTypesList{}, ColumnDefaults{}, dictionary_structure, table_name); + ColumnsDescription{columns, NamesAndTypesList{}, NamesAndTypesList{}, ColumnDefaults{}}, dictionary_structure, table_name); } } } @@ -142,10 +142,7 @@ void DatabaseDictionary::renameTable( void DatabaseDictionary::alterTable( const Context &, const String &, - const NamesAndTypesList &, - const NamesAndTypesList &, - const NamesAndTypesList &, - const ColumnDefaults &, + const ColumnsDescription &, const ASTModifier &) { throw Exception("DatabaseDictionary: alterTable() is not supported", ErrorCodes::NOT_IMPLEMENTED); diff --git a/dbms/src/Databases/DatabaseDictionary.h b/dbms/src/Databases/DatabaseDictionary.h index 1308ea20a40..aba93faa5b1 100644 --- a/dbms/src/Databases/DatabaseDictionary.h +++ b/dbms/src/Databases/DatabaseDictionary.h @@ -79,10 +79,7 @@ public: void alterTable( const Context & context, const String & name, - const NamesAndTypesList & columns, - const NamesAndTypesList & materialized_columns, - const NamesAndTypesList & alias_columns, - const ColumnDefaults & column_defaults, + const ColumnsDescription & columns, const ASTModifier & engine_modifier) override; time_t getTableMetadataModificationTime( diff --git a/dbms/src/Databases/DatabaseMemory.cpp b/dbms/src/Databases/DatabaseMemory.cpp index 66b23f07ca6..57089862863 100644 --- a/dbms/src/Databases/DatabaseMemory.cpp +++ b/dbms/src/Databases/DatabaseMemory.cpp @@ -105,10 +105,7 @@ void DatabaseMemory::renameTable( void DatabaseMemory::alterTable( const Context &, const String &, - const NamesAndTypesList &, - const NamesAndTypesList &, - const NamesAndTypesList &, - const ColumnDefaults &, + const ColumnsDescription &, const ASTModifier &) { throw Exception("DatabaseMemory: alterTable() is not supported", ErrorCodes::NOT_IMPLEMENTED); diff --git a/dbms/src/Databases/DatabaseMemory.h b/dbms/src/Databases/DatabaseMemory.h index be095ad3755..586dcfbe4da 100644 --- a/dbms/src/Databases/DatabaseMemory.h +++ b/dbms/src/Databases/DatabaseMemory.h @@ -70,10 +70,7 @@ public: void alterTable( const Context & context, const String & name, - const NamesAndTypesList & columns, - const NamesAndTypesList & materialized_columns, - const NamesAndTypesList & alias_columns, - const ColumnDefaults & column_defaults, + const ColumnsDescription & columns, const ASTModifier & engine_modifier) override; time_t getTableMetadataModificationTime( diff --git a/dbms/src/Databases/DatabaseOrdinary.cpp b/dbms/src/Databases/DatabaseOrdinary.cpp index 0021f47c4c7..9dd705fbf9a 100644 --- a/dbms/src/Databases/DatabaseOrdinary.cpp +++ b/dbms/src/Databases/DatabaseOrdinary.cpp @@ -462,10 +462,7 @@ void DatabaseOrdinary::drop() void DatabaseOrdinary::alterTable( const Context & context, const String & name, - const NamesAndTypesList & columns, - const NamesAndTypesList & materialized_columns, - const NamesAndTypesList & alias_columns, - const ColumnDefaults & column_defaults, + const ColumnsDescription & columns, const ASTModifier & storage_modifier) { /// Read the definition of the table and replace the necessary parts with new ones. @@ -486,7 +483,7 @@ void DatabaseOrdinary::alterTable( ASTCreateQuery & ast_create_query = typeid_cast(*ast); - ASTPtr new_columns = InterpreterCreateQuery::formatColumns(columns, materialized_columns, alias_columns, column_defaults); + ASTPtr new_columns = InterpreterCreateQuery::formatColumns(columns); ast_create_query.replace(ast_create_query.columns, new_columns); if (storage_modifier) diff --git a/dbms/src/Databases/DatabaseOrdinary.h b/dbms/src/Databases/DatabaseOrdinary.h index dbd44586c74..25e99114430 100644 --- a/dbms/src/Databases/DatabaseOrdinary.h +++ b/dbms/src/Databases/DatabaseOrdinary.h @@ -45,10 +45,7 @@ public: void alterTable( const Context & context, const String & name, - const NamesAndTypesList & columns, - const NamesAndTypesList & materialized_columns, - const NamesAndTypesList & alias_columns, - const ColumnDefaults & column_defaults, + const ColumnsDescription & columns, const ASTModifier & engine_modifier) override; time_t getTableMetadataModificationTime( diff --git a/dbms/src/Databases/DatabasesCommon.cpp b/dbms/src/Databases/DatabasesCommon.cpp index 83a14145d58..66d05d11184 100644 --- a/dbms/src/Databases/DatabasesCommon.cpp +++ b/dbms/src/Databases/DatabasesCommon.cpp @@ -67,7 +67,7 @@ std::pair createTableFromDefinition( if (!ast_create_query.columns) throw Exception("Missing definition of columns.", ErrorCodes::EMPTY_LIST_OF_COLUMNS_PASSED); - InterpreterCreateQuery::ColumnsInfo columns_info = InterpreterCreateQuery::getColumnsInfo(*ast_create_query.columns, context); + ColumnsDescription columns = InterpreterCreateQuery::getColumnsDescription(*ast_create_query.columns, context); return { @@ -75,7 +75,7 @@ std::pair createTableFromDefinition( StorageFactory::instance().get( ast_create_query, database_data_path, ast_create_query.table, database_name, context, context.getGlobalContext(), - columns_info.columns, columns_info.materialized_columns, columns_info.alias_columns, columns_info.column_defaults, + columns, true, has_force_restore_data_flag) }; } diff --git a/dbms/src/Databases/IDatabase.h b/dbms/src/Databases/IDatabase.h index 74cbe0b2202..f7b8af632ca 100644 --- a/dbms/src/Databases/IDatabase.h +++ b/dbms/src/Databases/IDatabase.h @@ -2,7 +2,7 @@ #include #include -#include +#include #include #include #include @@ -113,10 +113,7 @@ public: virtual void alterTable( const Context & context, const String & name, - const NamesAndTypesList & columns, - const NamesAndTypesList & materialized_columns, - const NamesAndTypesList & alias_columns, - const ColumnDefaults & column_defaults, + const ColumnsDescription & columns, const ASTModifier & engine_modifier) = 0; /// Returns time of table's metadata change, 0 if there is no corresponding metadata file. diff --git a/dbms/src/Interpreters/ExpressionAnalyzer.cpp b/dbms/src/Interpreters/ExpressionAnalyzer.cpp index c3c663d59ad..fe85e154e92 100644 --- a/dbms/src/Interpreters/ExpressionAnalyzer.cpp +++ b/dbms/src/Interpreters/ExpressionAnalyzer.cpp @@ -190,7 +190,7 @@ ExpressionAnalyzer::ExpressionAnalyzer( } if (storage && source_columns.empty()) - source_columns = storage->getSampleBlock().getNamesAndTypesList(); + source_columns = storage->columns.getList(); else removeDuplicateColumns(source_columns); @@ -712,7 +712,7 @@ static std::shared_ptr interpretSubquery( /// get columns list for target table auto database_table = getDatabaseAndTableNameFromIdentifier(*table); const auto & storage = context.getTable(database_table.first, database_table.second); - const auto & columns = storage->getColumnsListNonMaterialized(); + const auto & columns = storage->columns.getListNonMaterialized(); select_expression_list->children.reserve(columns.size()); /// manually substitute column names in place of asterisk @@ -826,7 +826,9 @@ void ExpressionAnalyzer::addExternalStorage(ASTPtr & subquery_or_table_name_or_t Block sample = interpreter->getSampleBlock(); NamesAndTypesList columns = sample.getNamesAndTypesList(); - StoragePtr external_storage = StorageMemory::create(external_table_name, columns, NamesAndTypesList{}, NamesAndTypesList{}, ColumnDefaults{}); + StoragePtr external_storage = StorageMemory::create( + external_table_name, + ColumnsDescription{columns, NamesAndTypesList{}, NamesAndTypesList{}, ColumnDefaults{}}); external_storage->startup(); /** We replace the subquery with the name of the temporary table. @@ -1050,7 +1052,7 @@ void ExpressionAnalyzer::normalizeTreeImpl( if (storage) { /// If we select from a table, get only not MATERIALIZED, not ALIAS columns. - for (const auto & name_type : storage->getColumnsListNonMaterialized()) + for (const auto & name_type : storage->columns.getListNonMaterialized()) all_columns.emplace_back(std::make_shared(name_type.name)); } else @@ -1147,7 +1149,7 @@ void ExpressionAnalyzer::addAliasColumns() if (!storage) return; - source_columns.insert(std::end(source_columns), std::begin(storage->alias_columns), std::end(storage->alias_columns)); + source_columns.insert(std::end(source_columns), std::begin(storage->columns.aliases), std::end(storage->columns.aliases)); } diff --git a/dbms/src/Interpreters/InterpreterAlterQuery.cpp b/dbms/src/Interpreters/InterpreterAlterQuery.cpp index f990fd8570f..1fb3e1e80db 100644 --- a/dbms/src/Interpreters/InterpreterAlterQuery.cpp +++ b/dbms/src/Interpreters/InterpreterAlterQuery.cpp @@ -200,7 +200,7 @@ void InterpreterAlterQuery::PartitionCommands::validate(const IStorage * table) { String column_name = command.column_name.safeGet(); - if (!table->hasRealColumn(column_name)) + if (!table->columns.has(column_name)) { throw Exception("Wrong column name. Cannot find column " + column_name + " to clear it from partition", DB::ErrorCodes::ILLEGAL_COLUMN); diff --git a/dbms/src/Interpreters/InterpreterCreateQuery.cpp b/dbms/src/Interpreters/InterpreterCreateQuery.cpp index 2b6d333d1f6..6033947b474 100644 --- a/dbms/src/Interpreters/InterpreterCreateQuery.cpp +++ b/dbms/src/Interpreters/InterpreterCreateQuery.cpp @@ -301,15 +301,11 @@ ASTPtr InterpreterCreateQuery::formatColumns(const NamesAndTypesList & columns) return columns_list; } -ASTPtr InterpreterCreateQuery::formatColumns( - const NamesAndTypesList & columns, - const NamesAndTypesList & materialized_columns, - const NamesAndTypesList & alias_columns, - const ColumnDefaults & column_defaults) +ASTPtr InterpreterCreateQuery::formatColumns(const ColumnsDescription & columns) { auto columns_list = std::make_shared(); - for (const auto & column : boost::join(columns, boost::join(materialized_columns, alias_columns))) + for (const auto & column : boost::join(columns.ordinary, boost::join(columns.materialized, columns.aliases))) { const auto column_declaration = std::make_shared(); ASTPtr column_declaration_ptr{column_declaration}; @@ -324,8 +320,8 @@ ASTPtr InterpreterCreateQuery::formatColumns( column_declaration->type = parseQuery(storage_p, pos, end, "data type"); column_declaration->type->owned_string = type_name; - const auto it = column_defaults.find(column.name); - if (it != std::end(column_defaults)) + const auto it = columns.defaults.find(column.name); + if (it != std::end(columns.defaults)) { column_declaration->default_specifier = toString(it->second.type); column_declaration->default_expression = it->second.expression->clone(); @@ -338,49 +334,46 @@ ASTPtr InterpreterCreateQuery::formatColumns( } -InterpreterCreateQuery::ColumnsInfo InterpreterCreateQuery::getColumnsInfo(const ASTExpressionList & columns, const Context & context) +ColumnsDescription InterpreterCreateQuery::getColumnsDescription(const ASTExpressionList & columns, const Context & context) { - ColumnsInfo res; + ColumnsDescription res; auto && columns_and_defaults = parseColumns(columns, context); - res.materialized_columns = removeAndReturnColumns(columns_and_defaults, ColumnDefaultType::Materialized); - res.alias_columns = removeAndReturnColumns(columns_and_defaults, ColumnDefaultType::Alias); - res.columns = std::move(columns_and_defaults.first); - res.column_defaults = std::move(columns_and_defaults.second); + res.materialized = removeAndReturnColumns(columns_and_defaults, ColumnDefaultType::Materialized); + res.aliases = removeAndReturnColumns(columns_and_defaults, ColumnDefaultType::Alias); + res.ordinary = std::move(columns_and_defaults.first); + res.defaults = std::move(columns_and_defaults.second); - if (res.columns.size() + res.materialized_columns.size() == 0) + if (res.ordinary.size() + res.materialized.size() == 0) throw Exception{"Cannot CREATE table without physical columns", ErrorCodes::EMPTY_LIST_OF_COLUMNS_PASSED}; return res; } -InterpreterCreateQuery::ColumnsInfo InterpreterCreateQuery::setColumns( +ColumnsDescription InterpreterCreateQuery::setColumns( ASTCreateQuery & create, const Block & as_select_sample, const StoragePtr & as_storage) const { - ColumnsInfo res; + ColumnsDescription res; if (create.columns) { - res = getColumnsInfo(*create.columns, context); + res = getColumnsDescription(*create.columns, context); } else if (!create.as_table.empty()) { - res.columns = as_storage->getColumnsListNonMaterialized(); - res.materialized_columns = as_storage->materialized_columns; - res.alias_columns = as_storage->alias_columns; - res.column_defaults = as_storage->column_defaults; + res = as_storage->columns; } else if (create.select) { for (size_t i = 0; i < as_select_sample.columns(); ++i) - res.columns.emplace_back(as_select_sample.safeGetByPosition(i).name, as_select_sample.safeGetByPosition(i).type); + res.ordinary.emplace_back(as_select_sample.safeGetByPosition(i).name, as_select_sample.safeGetByPosition(i).type); } else throw Exception("Incorrect CREATE query: required list of column descriptions or AS section or SELECT.", ErrorCodes::INCORRECT_QUERY); /// Even if query has list of columns, canonicalize it (unfold Nested columns). - ASTPtr new_columns = formatColumns(res.columns, res.materialized_columns, res.alias_columns, res.column_defaults); + ASTPtr new_columns = formatColumns(res); if (create.columns) create.replace(create.columns, new_columns); else @@ -394,11 +387,11 @@ InterpreterCreateQuery::ColumnsInfo InterpreterCreateQuery::setColumns( throw Exception("Column " + backQuoteIfNeed(column_name_and_type.name) + " already exists", ErrorCodes::DUPLICATE_COLUMN); }; - for (const auto & elem : res.columns) + for (const auto & elem : res.ordinary) check_column_already_exists(elem); - for (const auto & elem : res.materialized_columns) + for (const auto & elem : res.materialized) check_column_already_exists(elem); - for (const auto & elem : res.alias_columns) + for (const auto & elem : res.aliases) check_column_already_exists(elem); return res; @@ -489,7 +482,7 @@ BlockIO InterpreterCreateQuery::createTable(ASTCreateQuery & create) } /// Set and retrieve list of columns. - ColumnsInfo columns = setColumns(create, as_select_sample, as_storage); + ColumnsDescription columns = setColumns(create, as_select_sample, as_storage); /// Set the table engine if it was not specified explicitly. setEngine(create); @@ -530,10 +523,7 @@ BlockIO InterpreterCreateQuery::createTable(ASTCreateQuery & create) database_name, context, context.getGlobalContext(), - columns.columns, - columns.materialized_columns, - columns.alias_columns, - columns.column_defaults, + columns, create.attach, false); diff --git a/dbms/src/Interpreters/InterpreterCreateQuery.h b/dbms/src/Interpreters/InterpreterCreateQuery.h index b72b7eb7d09..8bec93a0fdc 100644 --- a/dbms/src/Interpreters/InterpreterCreateQuery.h +++ b/dbms/src/Interpreters/InterpreterCreateQuery.h @@ -1,7 +1,7 @@ #pragma once #include -#include +#include class ThreadPool; @@ -27,11 +27,7 @@ public: /// List of columns and their types in AST. static ASTPtr formatColumns(const NamesAndTypesList & columns); - static ASTPtr formatColumns( - const NamesAndTypesList & columns, - const NamesAndTypesList & materialized_columns, - const NamesAndTypesList & alias_columns, - const ColumnDefaults & column_defaults); + static ASTPtr formatColumns(const ColumnsDescription & columns); void setDatabaseLoadingThreadpool(ThreadPool & thread_pool_) { @@ -48,23 +44,15 @@ public: internal = internal_; } - struct ColumnsInfo - { - NamesAndTypesList columns; - NamesAndTypesList materialized_columns; - NamesAndTypesList alias_columns; - ColumnDefaults column_defaults; - }; - /// Obtain information about columns, their types and default values, for case when columns in CREATE query is specified explicitly. - static ColumnsInfo getColumnsInfo(const ASTExpressionList & columns, const Context & context); + static ColumnsDescription getColumnsDescription(const ASTExpressionList & columns, const Context & context); private: BlockIO createDatabase(ASTCreateQuery & create); BlockIO createTable(ASTCreateQuery & create); /// Calculate list of columns of table and return it. - ColumnsInfo setColumns(ASTCreateQuery & create, const Block & as_select_sample, const StoragePtr & as_storage) const; + ColumnsDescription setColumns(ASTCreateQuery & create, const Block & as_select_sample, const StoragePtr & as_storage) const; void setEngine(ASTCreateQuery & create) const; void checkAccess(const ASTCreateQuery & create); diff --git a/dbms/src/Interpreters/InterpreterDescribeQuery.cpp b/dbms/src/Interpreters/InterpreterDescribeQuery.cpp index 65b0fea25f1..b0d7e4c0363 100644 --- a/dbms/src/Interpreters/InterpreterDescribeQuery.cpp +++ b/dbms/src/Interpreters/InterpreterDescribeQuery.cpp @@ -99,9 +99,9 @@ BlockInputStreamPtr InterpreterDescribeQuery::executeImpl() } auto table_lock = table->lockStructure(false, __PRETTY_FUNCTION__); - columns = table->getColumnsList(); - columns.insert(std::end(columns), std::begin(table->alias_columns), std::end(table->alias_columns)); - column_defaults = table->column_defaults; + columns = table->columns.getList(); + columns.insert(std::end(columns), std::begin(table->columns.aliases), std::end(table->columns.aliases)); + column_defaults = table->columns.defaults; } Block sample_block = getSampleBlock(); diff --git a/dbms/src/Interpreters/InterpreterInsertQuery.cpp b/dbms/src/Interpreters/InterpreterInsertQuery.cpp index 014c8d287ad..5f0b8201ac3 100644 --- a/dbms/src/Interpreters/InterpreterInsertQuery.cpp +++ b/dbms/src/Interpreters/InterpreterInsertQuery.cpp @@ -95,7 +95,7 @@ BlockIO InterpreterInsertQuery::execute() auto table_lock = table->lockStructure(true, __PRETTY_FUNCTION__); - NamesAndTypesList required_columns = table->getColumnsList(); + NamesAndTypesList required_columns = table->columns.getList(); /// We create a pipeline of several streams, into which we will write data. BlockOutputStreamPtr out; @@ -103,7 +103,7 @@ BlockIO InterpreterInsertQuery::execute() out = std::make_shared(query.database, query.table, table, context, query_ptr, query.no_destination); out = std::make_shared( - out, getSampleBlock(query, table), required_columns, table->column_defaults, context); + out, getSampleBlock(query, table), required_columns, table->columns.defaults, context); /// Do not squash blocks if it is a sync INSERT into Distributed if (!(context.getSettingsRef().insert_distributed_sync && table->getName() == "Distributed")) @@ -135,7 +135,7 @@ BlockIO InterpreterInsertQuery::execute() if (!allow_materialized) { Block in_header = res.in->getHeader(); - for (const auto & name_type : table->materialized_columns) + for (const auto & name_type : table->columns.materialized) if (in_header.has(name_type.name)) throw Exception("Cannot insert column " + name_type.name + ", because it is MATERIALIZED column.", ErrorCodes::ILLEGAL_COLUMN); } diff --git a/dbms/src/Interpreters/InterpreterSelectQuery.cpp b/dbms/src/Interpreters/InterpreterSelectQuery.cpp index 1d5a442bb62..04e05e975ac 100644 --- a/dbms/src/Interpreters/InterpreterSelectQuery.cpp +++ b/dbms/src/Interpreters/InterpreterSelectQuery.cpp @@ -516,12 +516,12 @@ QueryProcessingStage::Enum InterpreterSelectQuery::executeFetchColumns(Pipeline /// Are ALIAS columns required for query execution? auto alias_columns_required = false; - if (storage && !storage->alias_columns.empty()) + if (storage && !storage->columns.aliases.empty()) { for (const auto & column : required_columns) { - const auto default_it = storage->column_defaults.find(column); - if (default_it != std::end(storage->column_defaults) && default_it->second.type == ColumnDefaultType::Alias) + const auto default_it = storage->columns.defaults.find(column); + if (default_it != std::end(storage->columns.defaults) && default_it->second.type == ColumnDefaultType::Alias) { alias_columns_required = true; break; @@ -535,8 +535,8 @@ QueryProcessingStage::Enum InterpreterSelectQuery::executeFetchColumns(Pipeline for (const auto & column : required_columns) { - const auto default_it = storage->column_defaults.find(column); - if (default_it != std::end(storage->column_defaults) && default_it->second.type == ColumnDefaultType::Alias) + const auto default_it = storage->columns.defaults.find(column); + if (default_it != std::end(storage->columns.defaults) && default_it->second.type == ColumnDefaultType::Alias) required_columns_expr_list->children.emplace_back(setAlias(default_it->second.expression->clone(), column)); else required_columns_expr_list->children.emplace_back(std::make_shared(column)); diff --git a/dbms/src/Server/TCPHandler.cpp b/dbms/src/Server/TCPHandler.cpp index 55f85c662dc..76a79d09722 100644 --- a/dbms/src/Server/TCPHandler.cpp +++ b/dbms/src/Server/TCPHandler.cpp @@ -630,7 +630,8 @@ bool TCPHandler::receiveData() if (!(storage = query_context.tryGetExternalTable(external_table_name))) { NamesAndTypesList columns = block.getNamesAndTypesList(); - storage = StorageMemory::create(external_table_name, columns, NamesAndTypesList{}, NamesAndTypesList{}, ColumnDefaults{}); + storage = StorageMemory::create(external_table_name, + ColumnsDescription{columns, NamesAndTypesList{}, NamesAndTypesList{}, ColumnDefaults{}}); storage->startup(); query_context.addExternalTable(external_table_name, storage); } diff --git a/dbms/src/Storages/AlterCommands.cpp b/dbms/src/Storages/AlterCommands.cpp index 8411dfe1bf6..defad6f5cba 100644 --- a/dbms/src/Storages/AlterCommands.cpp +++ b/dbms/src/Storages/AlterCommands.cpp @@ -21,9 +21,7 @@ namespace ErrorCodes } -void AlterCommand::apply( - NamesAndTypesList & columns, NamesAndTypesList & materialized_columns, NamesAndTypesList & alias_columns, - ColumnDefaults & column_defaults) const +void AlterCommand::apply(ColumnsDescription & columns_description) const { if (type == ADD_COLUMN) { @@ -33,9 +31,9 @@ void AlterCommand::apply( std::bind(namesEqual, std::cref(column_name), std::placeholders::_1)); }; - if (exists_in(columns) || - exists_in(materialized_columns) || - exists_in(alias_columns)) + if (exists_in(columns_description.ordinary) || + exists_in(columns_description.materialized) || + exists_in(columns_description.aliases)) { throw Exception{ "Cannot add column " + column_name + ": column with this name already exists", @@ -68,24 +66,24 @@ void AlterCommand::apply( }; if (default_type == ColumnDefaultType::Default) - add_column(columns); + add_column(columns_description.ordinary); else if (default_type == ColumnDefaultType::Materialized) - add_column(materialized_columns); + add_column(columns_description.materialized); else if (default_type == ColumnDefaultType::Alias) - add_column(alias_columns); + add_column(columns_description.aliases); else throw Exception{"Unknown ColumnDefaultType value", ErrorCodes::LOGICAL_ERROR}; if (default_expression) - column_defaults.emplace(column_name, ColumnDefault{default_type, default_expression}); + columns_description.defaults.emplace(column_name, ColumnDefault{default_type, default_expression}); /// Slow, because each time a list is copied - columns = Nested::flatten(columns); + columns_description.ordinary = Nested::flatten(columns_description.ordinary); } else if (type == DROP_COLUMN) { /// look for a column in list and remove it if present, also removing corresponding entry from column_defaults - const auto remove_column = [&column_defaults, this] (NamesAndTypesList & columns) + const auto remove_column = [&columns_description, this] (NamesAndTypesList & columns) { auto removed = false; NamesAndTypesList::iterator column_it; @@ -95,15 +93,15 @@ void AlterCommand::apply( { removed = true; column_it = columns.erase(column_it); - column_defaults.erase(column_name); + columns_description.defaults.erase(column_name); } return removed; }; - if (!remove_column(columns) && - !remove_column(materialized_columns) && - !remove_column(alias_columns)) + if (!remove_column(columns_description.ordinary) && + !remove_column(columns_description.materialized) && + !remove_column(columns_description.aliases)) { throw Exception("Wrong column name. Cannot find column " + column_name + " to drop", ErrorCodes::ILLEGAL_COLUMN); @@ -111,14 +109,15 @@ void AlterCommand::apply( } else if (type == MODIFY_COLUMN) { - const auto default_it = column_defaults.find(column_name); - const auto had_default_expr = default_it != std::end(column_defaults); + const auto default_it = columns_description.defaults.find(column_name); + const auto had_default_expr = default_it != std::end(columns_description.defaults); const auto old_default_type = had_default_expr ? default_it->second.type : ColumnDefaultType{}; /// target column list - auto & new_columns = default_type == ColumnDefaultType::Default ? - columns : default_type == ColumnDefaultType::Materialized ? - materialized_columns : alias_columns; + auto & new_columns = + default_type == ColumnDefaultType::Default ? columns_description.ordinary + : default_type == ColumnDefaultType::Materialized ? columns_description.materialized + : columns_description.aliases; /// find column or throw exception const auto find_column = [this] (NamesAndTypesList & columns) @@ -136,9 +135,10 @@ void AlterCommand::apply( if (default_type != old_default_type) { /// source column list - auto & old_columns = old_default_type == ColumnDefaultType::Default ? - columns : old_default_type == ColumnDefaultType::Materialized ? - materialized_columns : alias_columns; + auto & old_columns = + old_default_type == ColumnDefaultType::Default ? columns_description.ordinary + : old_default_type == ColumnDefaultType::Materialized ? columns_description.materialized + : columns_description.aliases; const auto old_column_it = find_column(old_columns); new_columns.emplace_back(*old_column_it); @@ -146,7 +146,7 @@ void AlterCommand::apply( /// do not forget to change the default type of old column if (had_default_expr) - column_defaults[column_name].type = default_type; + columns_description.defaults[column_name].type = default_type; } /// find column in one of three column lists @@ -155,13 +155,13 @@ void AlterCommand::apply( if (!default_expression && had_default_expr) /// new column has no default expression, remove it from column_defaults along with it's type - column_defaults.erase(column_name); + columns_description.defaults.erase(column_name); else if (default_expression && !had_default_expr) /// new column has a default expression while the old one had not, add it it column_defaults - column_defaults.emplace(column_name, ColumnDefault{default_type, default_expression}); + columns_description.defaults.emplace(column_name, ColumnDefault{default_type, default_expression}); else if (had_default_expr) /// both old and new columns have default expression, update it - column_defaults[column_name].expression = default_expression; + columns_description.defaults[column_name].expression = default_expression; } else if (type == MODIFY_PRIMARY_KEY) { @@ -173,30 +173,21 @@ void AlterCommand::apply( } -void AlterCommands::apply(NamesAndTypesList & columns, - NamesAndTypesList & materialized_columns, - NamesAndTypesList & alias_columns, - ColumnDefaults & column_defaults) const +void AlterCommands::apply(ColumnsDescription & columns_description) const { - auto new_columns = columns; - auto new_materialized_columns = materialized_columns; - auto new_alias_columns = alias_columns; - auto new_column_defaults = column_defaults; + auto new_columns_description = columns_description; for (const AlterCommand & command : *this) - command.apply(new_columns, new_materialized_columns, new_alias_columns, new_column_defaults); + command.apply(new_columns_description); - columns = std::move(new_columns); - materialized_columns = std::move(new_materialized_columns); - alias_columns = std::move(new_alias_columns); - column_defaults = std::move(new_column_defaults); + columns_description = std::move(new_columns_description); } void AlterCommands::validate(IStorage * table, const Context & context) { - auto columns = table->getColumnsList(); - columns.insert(std::end(columns), std::begin(table->alias_columns), std::end(table->alias_columns)); - auto defaults = table->column_defaults; + auto all_columns = table->columns.getList(); + all_columns.insert(std::end(all_columns), std::begin(table->columns.aliases), std::end(table->columns.aliases)); + auto defaults = table->columns.defaults; std::vector> defaulted_columns{}; @@ -208,12 +199,12 @@ void AlterCommands::validate(IStorage * table, const Context & context) if (command.type == AlterCommand::ADD_COLUMN || command.type == AlterCommand::MODIFY_COLUMN) { const auto & column_name = command.column_name; - const auto column_it = std::find_if(std::begin(columns), std::end(columns), + const auto column_it = std::find_if(std::begin(all_columns), std::end(all_columns), std::bind(AlterCommand::namesEqual, std::cref(command.column_name), std::placeholders::_1)); if (command.type == AlterCommand::ADD_COLUMN) { - if (std::end(columns) != column_it) + if (std::end(all_columns) != column_it) throw Exception{ "Cannot add column " + column_name + ": column with this name already exists", ErrorCodes::ILLEGAL_COLUMN}; @@ -221,17 +212,17 @@ void AlterCommands::validate(IStorage * table, const Context & context) else if (command.type == AlterCommand::MODIFY_COLUMN) { - if (std::end(columns) == column_it) + if (std::end(all_columns) == column_it) throw Exception{ "Wrong column name. Cannot find column " + column_name + " to modify", ErrorCodes::ILLEGAL_COLUMN}; - columns.erase(column_it); + all_columns.erase(column_it); defaults.erase(column_name); } /// we're creating dummy DataTypeUInt8 in order to prevent the NullPointerException in ExpressionActions - columns.emplace_back(column_name, command.data_type ? command.data_type : std::make_shared()); + all_columns.emplace_back(column_name, command.data_type ? command.data_type : std::make_shared()); if (command.default_expression) { @@ -265,7 +256,7 @@ void AlterCommands::validate(IStorage * table, const Context & context) for (const auto & default_column : defaults) { const auto & default_expression = default_column.second.expression; - const auto actions = ExpressionAnalyzer{default_expression, context, {}, columns}.getActions(true); + const auto actions = ExpressionAnalyzer{default_expression, context, {}, all_columns}.getActions(true); const auto required_columns = actions->getRequiredColumns(); if (required_columns.end() != std::find(required_columns.begin(), required_columns.end(), command.column_name)) @@ -275,12 +266,12 @@ void AlterCommands::validate(IStorage * table, const Context & context) } auto found = false; - for (auto it = std::begin(columns); it != std::end(columns);) + for (auto it = std::begin(all_columns); it != std::end(all_columns);) { if (AlterCommand::namesEqual(command.column_name, *it)) { found = true; - it = columns.erase(it); + it = all_columns.erase(it); } else ++it; @@ -305,7 +296,7 @@ void AlterCommands::validate(IStorage * table, const Context & context) for (const auto & col_def : defaults) { const auto & column_name = col_def.first; - const auto column_it = std::find_if(columns.begin(), columns.end(), [&] (const NameAndTypePair & name_type) + const auto column_it = std::find_if(all_columns.begin(), all_columns.end(), [&] (const NameAndTypePair & name_type) { return AlterCommand::namesEqual(column_name, name_type); }); const auto tmp_column_name = column_name + "_tmp"; @@ -321,7 +312,7 @@ void AlterCommands::validate(IStorage * table, const Context & context) defaulted_columns.emplace_back(NameAndTypePair{column_name, column_type_ptr}, nullptr); } - const auto actions = ExpressionAnalyzer{default_expr_list, context, {}, columns}.getActions(true); + const auto actions = ExpressionAnalyzer{default_expr_list, context, {}, all_columns}.getActions(true); const auto block = actions->getSampleBlock(); /// set deduced types, modify default expression if necessary diff --git a/dbms/src/Storages/AlterCommands.h b/dbms/src/Storages/AlterCommands.h index 543106254e6..44c44e8bfd0 100644 --- a/dbms/src/Storages/AlterCommands.h +++ b/dbms/src/Storages/AlterCommands.h @@ -1,7 +1,7 @@ #pragma once #include -#include +#include namespace DB { @@ -43,10 +43,7 @@ struct AlterCommand return (name_with_dot == name_type.name.substr(0, name_without_dot.length() + 1) || name_without_dot == name_type.name); } - void apply(NamesAndTypesList & columns, - NamesAndTypesList & materialized_columns, - NamesAndTypesList & alias_columns, - ColumnDefaults & column_defaults) const; + void apply(ColumnsDescription & columns_description) const; AlterCommand() = default; AlterCommand(const Type type, const String & column_name, const DataTypePtr & data_type, @@ -63,10 +60,7 @@ class Context; class AlterCommands : public std::vector { public: - void apply(NamesAndTypesList & columns, - NamesAndTypesList & materialized_columns, - NamesAndTypesList & alias_columns, - ColumnDefaults & column_defaults) const; + void apply(ColumnsDescription & columns_description) const; void validate(IStorage * table, const Context & context); }; diff --git a/dbms/src/Storages/ColumnsDescription.cpp b/dbms/src/Storages/ColumnsDescription.cpp index c0bfabea727..b8fae53ed4e 100644 --- a/dbms/src/Storages/ColumnsDescription.cpp +++ b/dbms/src/Storages/ColumnsDescription.cpp @@ -1,3 +1,4 @@ +#include #include #include #include @@ -7,8 +8,13 @@ #include #include #include -#include #include +#include + +#include +#include + +#include namespace DB @@ -16,17 +22,47 @@ namespace DB namespace ErrorCodes { + extern const int NO_SUCH_COLUMN_IN_TABLE; extern const int CANNOT_PARSE_TEXT; } -template -String ColumnsDescription::toString() const +NamesAndTypesList ColumnsDescription::getList() const +{ + return ext::collection_cast(boost::join(ordinary, materialized)); +} + + +Names ColumnsDescription::getNames() const +{ + return ext::map(boost::join(ordinary, materialized), [] (const auto & it) { return it.name; }); +} + + +NameAndTypePair ColumnsDescription::get(const String & column_name) const +{ + for (auto & it : boost::join(ordinary, materialized)) + if (it.name == column_name) + return it; + throw Exception("There is no column " + column_name + " in table.", ErrorCodes::NO_SUCH_COLUMN_IN_TABLE); +} + + +bool ColumnsDescription::has(const String & column_name) const +{ + for (auto & it : boost::join(ordinary, materialized)) + if (it.name == column_name) + return true; + return false; +} + + +String ColumnsDescription::toString() const { WriteBufferFromOwnString buf; writeString("columns format version: 1\n", buf); - writeText(columns.size() + materialized.size() + alias.size(), buf); + writeText(ordinary.size() + materialized.size() + aliases.size(), buf); writeString(" columns:\n", buf); const auto write_columns = [this, &buf] (const NamesAndTypesList & columns) @@ -53,16 +89,15 @@ String ColumnsDescription::toString() const } }; - write_columns(columns); + write_columns(ordinary); write_columns(materialized); - write_columns(alias); + write_columns(aliases); return buf.str(); } -template <> -ColumnsDescription ColumnsDescription::parse(const String & str) +ColumnsDescription ColumnsDescription::parse(const String & str) { ReadBufferFromString buf{str}; @@ -74,7 +109,7 @@ ColumnsDescription ColumnsDescription::parse(const String & str) ParserExpression expr_parser; const DataTypeFactory & data_type_factory = DataTypeFactory::instance(); - ColumnsDescription result{}; + ColumnsDescription result; for (size_t i = 0; i < count; ++i) { String column_name; @@ -88,7 +123,7 @@ ColumnsDescription ColumnsDescription::parse(const String & str) { assertChar('\n', buf); - result.columns.emplace_back(column_name, std::move(type)); + result.ordinary.emplace_back(column_name, std::move(type)); continue; } assertChar('\t', buf); @@ -107,11 +142,11 @@ ColumnsDescription ColumnsDescription::parse(const String & str) ASTPtr default_expr = parseQuery(expr_parser, begin, end, "default expression"); if (ColumnDefaultType::Default == default_type) - result.columns.emplace_back(column_name, std::move(type)); + result.ordinary.emplace_back(column_name, std::move(type)); else if (ColumnDefaultType::Materialized == default_type) result.materialized.emplace_back(column_name, std::move(type)); else if (ColumnDefaultType::Alias == default_type) - result.alias.emplace_back(column_name, std::move(type)); + result.aliases.emplace_back(column_name, std::move(type)); result.defaults.emplace(column_name, ColumnDefault{default_type, default_expr}); } @@ -121,8 +156,4 @@ ColumnsDescription ColumnsDescription::parse(const String & str) return result; } - -template struct ColumnsDescription; -template struct ColumnsDescription; - } diff --git a/dbms/src/Storages/ColumnsDescription.h b/dbms/src/Storages/ColumnsDescription.h index cce6232d33b..888f0864701 100644 --- a/dbms/src/Storages/ColumnsDescription.h +++ b/dbms/src/Storages/ColumnsDescription.h @@ -1,28 +1,53 @@ #pragma once -#include #include +#include +#include +#include namespace DB { - -template struct ColumnsDescription { - template - using by_value_or_cref = std::conditional_t; + NamesAndTypesList ordinary; + NamesAndTypesList materialized; + NamesAndTypesList aliases; + ColumnDefaults defaults; + + + bool operator==(const ColumnsDescription & other) const + { + return ordinary == other.ordinary + && materialized == other.materialized + && aliases == other.aliases + && defaults == other.defaults; + } + + bool operator!=(const ColumnsDescription & other) const { return !(*this == other); } + + /** Get a list of names and table column types, only non-virtual. + */ + NamesAndTypesList getList() const; + const NamesAndTypesList & getListNonMaterialized() const { return ordinary; } + + /** Get a list of column names. + */ + Names getNames() const; + + /** Get a description of any column by its name. + */ + NameAndTypePair get(const String & column_name) const; + + /** Is there a column with that name. + */ + bool has(const String & column_name) const; - by_value_or_cref columns; - by_value_or_cref materialized; - by_value_or_cref alias; - by_value_or_cref defaults; String toString() const; static ColumnsDescription parse(const String & str); }; - } diff --git a/dbms/src/Storages/Distributed/DirectoryMonitor.cpp b/dbms/src/Storages/Distributed/DirectoryMonitor.cpp index 66a4fc84434..6347510f7b1 100644 --- a/dbms/src/Storages/Distributed/DirectoryMonitor.cpp +++ b/dbms/src/Storages/Distributed/DirectoryMonitor.cpp @@ -468,7 +468,7 @@ bool StorageDistributedDirectoryMonitor::maybeMarkAsBroken(const std::string & f std::string StorageDistributedDirectoryMonitor::getLoggerName() const { - return storage.name + '.' + storage.getName() + ".DirectoryMonitor"; + return storage.table_name + '.' + storage.getName() + ".DirectoryMonitor"; } } diff --git a/dbms/src/Storages/IStorage.h b/dbms/src/Storages/IStorage.h index a2970b0de96..ef9fd8004ce 100644 --- a/dbms/src/Storages/IStorage.h +++ b/dbms/src/Storages/IStorage.h @@ -83,6 +83,10 @@ public: /// The main name of the table type (for example, StorageMergeTree). virtual std::string getName() const = 0; + /** The name of the table. + */ + virtual std::string getTableName() const = 0; + /** Returns true if the storage receives data from a remote server or servers. */ virtual bool isRemote() const { return false; } diff --git a/dbms/src/Storages/ITableDeclaration.cpp b/dbms/src/Storages/ITableDeclaration.cpp index da4cc920608..d4bbb4b501e 100644 --- a/dbms/src/Storages/ITableDeclaration.cpp +++ b/dbms/src/Storages/ITableDeclaration.cpp @@ -1,13 +1,12 @@ -#include +#include +#include + +#include #include #include -#include -#include -#include -#include -#include -#include -#include + +#include +#include namespace DB @@ -25,77 +24,14 @@ namespace ErrorCodes } -NamesAndTypesList ITableDeclaration::getColumnsList() const -{ - return ext::collection_cast(getColumnsListRange()); -} - - -ITableDeclaration::ColumnsListRange ITableDeclaration::getColumnsListRange() const -{ - return boost::join(getColumnsListImpl(), materialized_columns); -} - - -bool ITableDeclaration::hasRealColumn(const String & column_name) const -{ - for (auto & it : getColumnsListRange()) - if (it.name == column_name) - return true; - return false; -} - - -Names ITableDeclaration::getColumnNamesList() const -{ - return ext::map(getColumnsListRange(), [] (const auto & it) { return it.name; }); -} - - -NameAndTypePair ITableDeclaration::getRealColumn(const String & column_name) const -{ - for (auto & it : getColumnsListRange()) - if (it.name == column_name) - return it; - throw Exception("There is no column " + column_name + " in table.", ErrorCodes::NO_SUCH_COLUMN_IN_TABLE); -} - -NameAndTypePair ITableDeclaration::getMaterializedColumn(const String & column_name) const -{ - for (auto & column : materialized_columns) - if (column.name == column_name) - return column; - - throw Exception("There is no column " + column_name + " in table.", ErrorCodes::NO_SUCH_COLUMN_IN_TABLE); -} - -bool ITableDeclaration::hasMaterializedColumn(const String & column_name) const -{ - for (auto & column : materialized_columns) - if (column.name == column_name) - return true; - - return false; -} - bool ITableDeclaration::hasColumn(const String & column_name) const { - return hasRealColumn(column_name); /// By default, we assume that there are no virtual columns in the storage. + return columns.has(column_name); /// By default, we assume that there are no virtual columns in the storage. } NameAndTypePair ITableDeclaration::getColumn(const String & column_name) const { - return getRealColumn(column_name); /// By default, we assume that there are no virtual columns in the storage. -} - - -const DataTypePtr ITableDeclaration::getDataTypeByName(const String & column_name) const -{ - for (const auto & column : getColumnsListRange()) - if (column.name == column_name) - return column.type; - - throw Exception("There is no column " + column_name + " in table.", ErrorCodes::NO_SUCH_COLUMN_IN_TABLE); + return columns.get(column_name); /// By default, we assume that there are no virtual columns in the storage. } @@ -103,7 +39,7 @@ Block ITableDeclaration::getSampleBlock() const { Block res; - for (const auto & col : getColumnsListRange()) + for (const auto & col : boost::join(columns.ordinary, columns.materialized)) res.insert({ col.type->createColumn(), col.type, col.name }); return res; @@ -114,7 +50,7 @@ Block ITableDeclaration::getSampleBlockNonMaterialized() const { Block res; - for (const auto & col : getColumnsListNonMaterialized()) + for (const auto & col : columns.ordinary) res.insert({ col.type->createColumn(), col.type, col.name }); return res; @@ -138,7 +74,7 @@ Block ITableDeclaration::getSampleBlockForColumns(const Names & column_names) co static std::string listOfColumns(const NamesAndTypesList & available_columns) { std::stringstream s; - for (NamesAndTypesList::const_iterator it = available_columns.begin(); it != available_columns.end(); ++it) + for (auto it = available_columns.begin(); it != available_columns.end(); ++it) { if (it != available_columns.begin()) s << ", "; @@ -175,7 +111,7 @@ static NamesAndTypesMap getColumnsMap(const Args &... args) void ITableDeclaration::check(const Names & column_names) const { - const NamesAndTypesList & available_columns = getColumnsList(); + const NamesAndTypesList & available_columns = columns.getList(); if (column_names.empty()) throw Exception("Empty list of columns queried. There are columns: " + listOfColumns(available_columns), @@ -201,16 +137,16 @@ void ITableDeclaration::check(const Names & column_names) const } -void ITableDeclaration::check(const NamesAndTypesList & columns) const +void ITableDeclaration::check(const NamesAndTypesList & provided_columns) const { - const NamesAndTypesList & available_columns = getColumnsList(); + const NamesAndTypesList & available_columns = columns.getList(); const auto columns_map = getColumnsMap(available_columns); using UniqueStrings = google::dense_hash_set; UniqueStrings unique_names; unique_names.set_empty_key(StringRef()); - for (const NameAndTypePair & column : columns) + for (const NameAndTypePair & column : provided_columns) { NamesAndTypesMap::const_iterator it = columns_map.find(column.name); if (columns_map.end() == it) @@ -229,11 +165,11 @@ void ITableDeclaration::check(const NamesAndTypesList & columns) const } -void ITableDeclaration::check(const NamesAndTypesList & columns, const Names & column_names) const +void ITableDeclaration::check(const NamesAndTypesList & provided_columns, const Names & column_names) const { - const NamesAndTypesList & available_columns = getColumnsList(); + const NamesAndTypesList & available_columns = columns.getList(); const auto available_columns_map = getColumnsMap(available_columns); - const NamesAndTypesMap & provided_columns_map = getColumnsMap(columns); + const NamesAndTypesMap & provided_columns_map = getColumnsMap(provided_columns); if (column_names.empty()) throw Exception("Empty list of columns queried. There are columns: " + listOfColumns(available_columns), @@ -268,7 +204,7 @@ void ITableDeclaration::check(const NamesAndTypesList & columns, const Names & c void ITableDeclaration::check(const Block & block, bool need_all) const { - const NamesAndTypesList & available_columns = getColumnsList(); + const NamesAndTypesList & available_columns = columns.getList(); const auto columns_map = getColumnsMap(available_columns); using NameSet = std::unordered_set; @@ -304,14 +240,11 @@ void ITableDeclaration::check(const Block & block, bool need_all) const } } -ITableDeclaration::ITableDeclaration(const NamesAndTypesList & columns, const NamesAndTypesList & materialized_columns, - const NamesAndTypesList & alias_columns, const ColumnDefaults & column_defaults) - : columns{columns}, - materialized_columns{materialized_columns}, - alias_columns{alias_columns}, - column_defaults{column_defaults} + +ITableDeclaration::ITableDeclaration(ColumnsDescription columns_) + : columns{std::move(columns_)} { - if (columns.empty()) + if (columns.ordinary.empty()) throw Exception("Empty list of columns passed to storage constructor", ErrorCodes::EMPTY_LIST_OF_COLUMNS_PASSED); } diff --git a/dbms/src/Storages/ITableDeclaration.h b/dbms/src/Storages/ITableDeclaration.h index 841ebc6c753..4c4ea59e4b5 100644 --- a/dbms/src/Storages/ITableDeclaration.h +++ b/dbms/src/Storages/ITableDeclaration.h @@ -1,50 +1,17 @@ #pragma once -#include -#include -#include -#include -#include - -#include -#include +#include namespace DB { -class Context; - /** Description of the table. - * Do not thread safe. See IStorage::lockStructure (). + * Is not thread safe. See IStorage::lockStructure (). */ class ITableDeclaration { public: - /** The name of the table. - */ - virtual std::string getTableName() const = 0; - - /** Get a list of names and table column types, only non-virtual. - */ - NamesAndTypesList getColumnsList() const; - const NamesAndTypesList & getColumnsListNonMaterialized() const { return getColumnsListImpl(); } - - /** Get a list of column table names, only non-virtual. - */ - virtual Names getColumnNamesList() const; - - /** Get a description of the real (non-virtual) column by its name. - */ - virtual NameAndTypePair getRealColumn(const String & column_name) const; - - /** Is there a real (non-virtual) column with that name. - */ - virtual bool hasRealColumn(const String & column_name) const; - - NameAndTypePair getMaterializedColumn(const String & column_name) const; - bool hasMaterializedColumn(const String & column_name) const; - /** Get a description of any column by its name. */ virtual NameAndTypePair getColumn(const String & column_name) const; @@ -53,10 +20,6 @@ public: */ virtual bool hasColumn(const String & column_name) const; - const DataTypePtr getDataTypeByName(const String & column_name) const; - - /** The same, but in the form of a block-sample. - */ Block getSampleBlock() const; Block getSampleBlockNonMaterialized() const; Block getSampleBlockForColumns(const Names & column_names) const; @@ -81,29 +44,11 @@ public: void check(const Block & block, bool need_all = false) const; + ITableDeclaration() = default; + explicit ITableDeclaration(ColumnsDescription columns_); virtual ~ITableDeclaration() = default; - ITableDeclaration() = default; - ITableDeclaration( - const NamesAndTypesList & columns, - const NamesAndTypesList & materialized_columns, - const NamesAndTypesList & alias_columns, - const ColumnDefaults & column_defaults); - - NamesAndTypesList columns; - NamesAndTypesList materialized_columns; - NamesAndTypesList alias_columns; - ColumnDefaults column_defaults; - -private: - virtual const NamesAndTypesList & getColumnsListImpl() const - { - return columns; - } - - using ColumnsListRange = boost::range::joined_range; - /// Returns a lazily joined range of table's ordinary and materialized columns, without unnecessary copying - ColumnsListRange getColumnsListRange() const; + ColumnsDescription columns; }; } diff --git a/dbms/src/Storages/MergeTree/DataPartsExchange.cpp b/dbms/src/Storages/MergeTree/DataPartsExchange.cpp index e2c884681d3..b2ff96e0fe3 100644 --- a/dbms/src/Storages/MergeTree/DataPartsExchange.cpp +++ b/dbms/src/Storages/MergeTree/DataPartsExchange.cpp @@ -1,4 +1,5 @@ #include +#include #include #include #include diff --git a/dbms/src/Storages/MergeTree/MergeTreeBlockInputStream.cpp b/dbms/src/Storages/MergeTree/MergeTreeBlockInputStream.cpp index e0a8dc94400..5a67714db64 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeBlockInputStream.cpp +++ b/dbms/src/Storages/MergeTree/MergeTreeBlockInputStream.cpp @@ -141,8 +141,8 @@ try if (!column_names.empty()) storage.check(data_part->columns, column_names); - pre_columns = storage.getColumnsList().addTypes(pre_column_names); - columns = storage.getColumnsList().addTypes(column_names); + pre_columns = storage.columns.getList().addTypes(pre_column_names); + columns = storage.columns.getList().addTypes(column_names); } else { diff --git a/dbms/src/Storages/MergeTree/MergeTreeBlockReadUtils.cpp b/dbms/src/Storages/MergeTree/MergeTreeBlockReadUtils.cpp index b0b09ae06eb..b6d8ff3cd5a 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeBlockReadUtils.cpp +++ b/dbms/src/Storages/MergeTree/MergeTreeBlockReadUtils.cpp @@ -26,9 +26,9 @@ NameSet injectRequiredColumns(const MergeTreeData & storage, const MergeTreeData continue; } - const auto default_it = storage.column_defaults.find(column_name); + const auto default_it = storage.columns.defaults.find(column_name); /// columns has no explicit default expression - if (default_it == std::end(storage.column_defaults)) + if (default_it == std::end(storage.columns.defaults)) continue; /// collect identifiers required for evaluation diff --git a/dbms/src/Storages/MergeTree/MergeTreeData.cpp b/dbms/src/Storages/MergeTree/MergeTreeData.cpp index 392c45a7cc7..7933a9454c7 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeData.cpp +++ b/dbms/src/Storages/MergeTree/MergeTreeData.cpp @@ -80,10 +80,7 @@ namespace ErrorCodes MergeTreeData::MergeTreeData( const String & database_, const String & table_, - const String & full_path_, const NamesAndTypesList & columns_, - const NamesAndTypesList & materialized_columns_, - const NamesAndTypesList & alias_columns_, - const ColumnDefaults & column_defaults_, + const String & full_path_, const ColumnsDescription & columns_, Context & context_, const ASTPtr & primary_expr_ast_, const ASTPtr & secondary_sort_expr_ast_, @@ -95,7 +92,7 @@ MergeTreeData::MergeTreeData( bool require_part_metadata_, bool attach, BrokenPartCallback broken_part_callback_) - : ITableDeclaration{columns_, materialized_columns_, alias_columns_, column_defaults_}, + : ITableDeclaration{columns_}, context(context_), sampling_expression(sampling_expression_), index_granularity(settings_.index_granularity), @@ -113,7 +110,7 @@ MergeTreeData::MergeTreeData( data_parts_by_state_and_info(data_parts_indexes.get()) { /// NOTE: using the same columns list as is read when performing actual merges. - merging_params.check(getColumnsList()); + merging_params.check(columns.getList()); if (!primary_expr_ast) throw Exception("Primary key cannot be empty", ErrorCodes::BAD_ARGUMENTS); @@ -225,11 +222,11 @@ void MergeTreeData::initPrimaryKey() primary_sort_descr.clear(); addSortDescription(primary_sort_descr, primary_expr_ast); - primary_expr = ExpressionAnalyzer(primary_expr_ast, context, nullptr, getColumnsList()).getActions(false); + primary_expr = ExpressionAnalyzer(primary_expr_ast, context, nullptr, columns.getList()).getActions(false); { ExpressionActionsPtr projected_expr = - ExpressionAnalyzer(primary_expr_ast, context, nullptr, getColumnsList()).getActions(true); + ExpressionAnalyzer(primary_expr_ast, context, nullptr, columns.getList()).getActions(true); primary_key_sample = projected_expr->getSampleBlock(); } @@ -244,10 +241,10 @@ void MergeTreeData::initPrimaryKey() if (secondary_sort_expr_ast) { addSortDescription(sort_descr, secondary_sort_expr_ast); - secondary_sort_expr = ExpressionAnalyzer(secondary_sort_expr_ast, context, nullptr, getColumnsList()).getActions(false); + secondary_sort_expr = ExpressionAnalyzer(secondary_sort_expr_ast, context, nullptr, columns.getList()).getActions(false); ExpressionActionsPtr projected_expr = - ExpressionAnalyzer(secondary_sort_expr_ast, context, nullptr, getColumnsList()).getActions(true); + ExpressionAnalyzer(secondary_sort_expr_ast, context, nullptr, columns.getList()).getActions(true); auto secondary_key_sample = projected_expr->getSampleBlock(); checkKeyExpression(*secondary_sort_expr, secondary_key_sample, "Secondary"); @@ -260,7 +257,7 @@ void MergeTreeData::initPartitionKey() if (!partition_expr_ast || partition_expr_ast->children.empty()) return; - partition_expr = ExpressionAnalyzer(partition_expr_ast, context, nullptr, getColumnsList()).getActions(false); + partition_expr = ExpressionAnalyzer(partition_expr_ast, context, nullptr, columns.getList()).getActions(false); for (const ASTPtr & ast : partition_expr_ast->children) { String col_name = ast->getColumnName(); @@ -865,10 +862,7 @@ void MergeTreeData::checkAlter(const AlterCommands & commands) { /// Check that needed transformations can be applied to the list of columns without considering type conversions. auto new_columns = columns; - auto new_materialized_columns = materialized_columns; - auto new_alias_columns = alias_columns; - auto new_column_defaults = column_defaults; - commands.apply(new_columns, new_materialized_columns, new_alias_columns, new_column_defaults); + commands.apply(new_columns); /// Set of columns that shouldn't be altered. NameSet columns_alter_forbidden; @@ -909,7 +903,7 @@ void MergeTreeData::checkAlter(const AlterCommands & commands) columns_alter_forbidden.insert(merging_params.sign_column); std::map old_types; - for (const auto & column : columns) + for (const auto & column : columns.ordinary) old_types.emplace(column.name, column.type.get()); for (const AlterCommand & command : commands) @@ -937,11 +931,7 @@ void MergeTreeData::checkAlter(const AlterCommands & commands) NameToNameMap unused_map; bool unused_bool; - /// augment plain columns with materialized columns for convert expression creation - new_columns.insert(std::end(new_columns), - std::begin(new_materialized_columns), std::end(new_materialized_columns)); - - createConvertExpression(nullptr, getColumnsList(), new_columns, unused_expression, unused_map, unused_bool); + createConvertExpression(nullptr, columns.getList(), new_columns.getList(), unused_expression, unused_map, unused_bool); } void MergeTreeData::createConvertExpression(const DataPartPtr & part, const NamesAndTypesList & old_columns, const NamesAndTypesList & new_columns, @@ -1845,7 +1835,7 @@ void MergeTreeData::addPartContributionToColumnSizes(const DataPartPtr & part) const auto & files = part->checksums.files; /// TODO This method doesn't take into account columns with multiple files. - for (const auto & column : getColumnsList()) + for (const auto & column : columns.getList()) { const auto escaped_name = escapeForFileName(column.name); const auto bin_file_name = escaped_name + ".bin"; @@ -1878,7 +1868,7 @@ void MergeTreeData::removePartContributionToColumnSizes(const DataPartPtr & part const auto & files = part->checksums.files; /// TODO This method doesn't take into account columns with multiple files. - for (const auto & column : columns) + for (const auto & column : columns.ordinary) { const auto escaped_name = escapeForFileName(column.name); const auto bin_file_name = escaped_name + ".bin"; diff --git a/dbms/src/Storages/MergeTree/MergeTreeData.h b/dbms/src/Storages/MergeTree/MergeTreeData.h index 519e7223d13..a32baf797ef 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeData.h +++ b/dbms/src/Storages/MergeTree/MergeTreeData.h @@ -4,7 +4,8 @@ #include #include #include -#include +#include +#include #include #include #include @@ -270,10 +271,8 @@ public: /// require_part_metadata - should checksums.txt and columns.txt exist in the part directory. /// attach - whether the existing table is attached or the new table is created. MergeTreeData(const String & database_, const String & table_, - const String & full_path_, const NamesAndTypesList & columns_, - const NamesAndTypesList & materialized_columns_, - const NamesAndTypesList & alias_columns_, - const ColumnDefaults & column_defaults_, + const String & full_path_, + const ColumnsDescription & columns_, Context & context_, const ASTPtr & primary_expr_ast_, const ASTPtr & secondary_sort_expr_ast_, @@ -305,8 +304,6 @@ public: Int64 getMaxDataPartIndex(); - const NamesAndTypesList & getColumnsListImpl() const override { return columns; } - NameAndTypePair getColumn(const String & column_name) const override { if (column_name == "_part") @@ -316,12 +313,12 @@ public: if (column_name == "_sample_factor") return NameAndTypePair("_sample_factor", std::make_shared()); - return ITableDeclaration::getColumn(column_name); + return columns.get(column_name); } bool hasColumn(const String & column_name) const override { - return ITableDeclaration::hasColumn(column_name) + return columns.has(column_name) || column_name == "_part" || column_name == "_part_index" || column_name == "_sample_factor"; @@ -329,7 +326,7 @@ public: String getDatabaseName() const { return database_name; } - String getTableName() const override { return table_name; } + String getTableName() const { return table_name; } String getFullPath() const { return full_path; } @@ -431,9 +428,6 @@ public: const ASTPtr & new_primary_key, bool skip_sanity_checks); - /// Must be called with locked lockStructureForAlter(). - void setColumnsList(const NamesAndTypesList & new_columns) { columns = new_columns; } - /// Should be called if part data is suspected to be corrupted. void reportBrokenPart(const String & name) { @@ -507,6 +501,7 @@ public: /// For ATTACH/DETACH/DROP PARTITION. String getPartitionIDFromQuery(const ASTPtr & partition, const Context & context); + MergeTreeDataFormatVersion format_version; Context & context; diff --git a/dbms/src/Storages/MergeTree/MergeTreeDataMerger.cpp b/dbms/src/Storages/MergeTree/MergeTreeDataMerger.cpp index cf9e6eef15c..4e3e9788aaa 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeDataMerger.cpp +++ b/dbms/src/Storages/MergeTree/MergeTreeDataMerger.cpp @@ -536,8 +536,8 @@ MergeTreeData::MutableDataPartPtr MergeTreeDataMerger::mergePartsToTemporaryPart for (const MergeTreeData::DataPartPtr & part : parts) part->accumulateColumnSizes(merged_column_to_size); - Names all_column_names = data.getColumnNamesList(); - NamesAndTypesList all_columns = data.getColumnsList(); + Names all_column_names = data.columns.getNames(); + NamesAndTypesList all_columns = data.columns.getList(); const SortDescription sort_desc = data.getSortDescription(); NamesAndTypesList gathering_columns, merging_columns; diff --git a/dbms/src/Storages/MergeTree/MergeTreeDataPart.cpp b/dbms/src/Storages/MergeTree/MergeTreeDataPart.cpp index df273f2d6b5..9c0269c7437 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeDataPart.cpp +++ b/dbms/src/Storages/MergeTree/MergeTreeDataPart.cpp @@ -418,7 +418,7 @@ UInt64 MergeTreeDataPart::getColumnMrkSize(const String & name) const */ String MergeTreeDataPart::getColumnNameWithMinumumCompressedSize() const { - const auto & columns = storage.getColumnsList(); + const auto & columns = storage.columns.getList(); const std::string * minimum_size_column = nullptr; UInt64 minimum_size = std::numeric_limits::max(); @@ -774,7 +774,7 @@ void MergeTreeDataPart::accumulateColumnSizes(ColumnToSize & column_to_size) con { std::shared_lock part_lock(columns_lock); - for (const NameAndTypePair & name_type : storage.columns) + for (const NameAndTypePair & name_type : storage.columns.getList()) { name_type.type->enumerateStreams([&](const IDataType::SubstreamPath & substream_path) { @@ -794,7 +794,7 @@ void MergeTreeDataPart::loadColumns(bool require) throw Exception("No columns.txt in part " + name, ErrorCodes::NO_FILE_IN_DATA_PART); /// If there is no file with a list of columns, write it down. - for (const NameAndTypePair & column : storage.getColumnsList()) + for (const NameAndTypePair & column : storage.columns.getList()) if (Poco::File(getFullPath() + escapeForFileName(column.name) + ".bin").exists()) columns.push_back(column); diff --git a/dbms/src/Storages/MergeTree/MergeTreeDataSelectExecutor.cpp b/dbms/src/Storages/MergeTree/MergeTreeDataSelectExecutor.cpp index fb7c1735ced..605091e6386 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeDataSelectExecutor.cpp +++ b/dbms/src/Storages/MergeTree/MergeTreeDataSelectExecutor.cpp @@ -175,7 +175,7 @@ BlockInputStreams MergeTreeDataSelectExecutor::read( } } - NamesAndTypesList available_real_columns = data.getColumnsList(); + NamesAndTypesList available_real_columns = data.columns.getList(); NamesAndTypesList available_real_and_virtual_columns = available_real_columns; for (const auto & name : virt_column_names) @@ -857,7 +857,7 @@ void MergeTreeDataSelectExecutor::createPositiveSignCondition( arguments->children.push_back(sign); arguments->children.push_back(one); - out_expression = ExpressionAnalyzer(function, context, {}, data.getColumnsList()).getActions(false); + out_expression = ExpressionAnalyzer(function, context, {}, data.columns.getList()).getActions(false); out_column = function->getColumnName(); } diff --git a/dbms/src/Storages/MergeTree/MergeTreeDataSelectExecutor.h b/dbms/src/Storages/MergeTree/MergeTreeDataSelectExecutor.h index fd0b0265ced..1b3caeeb259 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeDataSelectExecutor.h +++ b/dbms/src/Storages/MergeTree/MergeTreeDataSelectExecutor.h @@ -1,5 +1,7 @@ #pragma once +#include +#include #include #include diff --git a/dbms/src/Storages/MergeTree/MergeTreeDataWriter.cpp b/dbms/src/Storages/MergeTree/MergeTreeDataWriter.cpp index fabbca3a473..f4db6b934ba 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeDataWriter.cpp +++ b/dbms/src/Storages/MergeTree/MergeTreeDataWriter.cpp @@ -205,7 +205,7 @@ MergeTreeData::MutableDataPartPtr MergeTreeDataWriter::writeTempPart(BlockWithPa /// either default lz4 or compression method with zero thresholds on absolute and relative part size. auto compression_settings = data.context.chooseCompressionSettings(0, 0); - NamesAndTypesList columns = data.getColumnsList().filter(block.getNames()); + NamesAndTypesList columns = data.columns.getList().filter(block.getNames()); MergedBlockOutputStream out(data, new_data_part->getFullPath(), columns, compression_settings); out.writePrefix(); diff --git a/dbms/src/Storages/MergeTree/MergeTreeReadPool.cpp b/dbms/src/Storages/MergeTree/MergeTreeReadPool.cpp index e95c576274c..c32aa71a731 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeReadPool.cpp +++ b/dbms/src/Storages/MergeTree/MergeTreeReadPool.cpp @@ -238,8 +238,8 @@ std::vector MergeTreeReadPool::fillPerPartInfo( if (!required_column_names.empty()) data.check(part.data_part->columns, required_column_names); - per_part_pre_columns.push_back(data.getColumnsList().addTypes(required_pre_column_names)); - per_part_columns.push_back(data.getColumnsList().addTypes(required_column_names)); + per_part_pre_columns.push_back(data.columns.getList().addTypes(required_pre_column_names)); + per_part_columns.push_back(data.columns.getList().addTypes(required_column_names)); } else { diff --git a/dbms/src/Storages/MergeTree/MergeTreeReader.cpp b/dbms/src/Storages/MergeTree/MergeTreeReader.cpp index 63d763d5939..7bce149776a 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeReader.cpp +++ b/dbms/src/Storages/MergeTree/MergeTreeReader.cpp @@ -463,7 +463,7 @@ void MergeTreeReader::fillMissingColumns(Block & res, const Names & ordered_name if (!has_column) { should_sort = true; - if (storage.column_defaults.count(requested_column.name) != 0) + if (storage.columns.defaults.count(requested_column.name) != 0) { should_evaluate_defaults = true; continue; @@ -498,7 +498,7 @@ void MergeTreeReader::fillMissingColumns(Block & res, const Names & ordered_name /// evaluate defaulted columns if necessary if (should_evaluate_defaults) - evaluateMissingDefaults(res, columns, storage.column_defaults, storage.context); + evaluateMissingDefaults(res, columns, storage.columns.defaults, storage.context); /// sort columns to ensure consistent order among all blocks if (should_sort) diff --git a/dbms/src/Storages/MergeTree/MergeTreeWhereOptimizer.cpp b/dbms/src/Storages/MergeTree/MergeTreeWhereOptimizer.cpp index cf7bc0d33f7..005ca750226 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeWhereOptimizer.cpp +++ b/dbms/src/Storages/MergeTree/MergeTreeWhereOptimizer.cpp @@ -41,9 +41,9 @@ MergeTreeWhereOptimizer::MergeTreeWhereOptimizer( Logger * log) : primary_key_columns{ext::map(data.getPrimarySortDescription(), [] (const SortColumnDescription & col) { return col.column_name; })}, - table_columns{ext::map(data.getColumnsList(), + table_columns{ext::map(data.columns.getList(), [] (const NameAndTypePair & col) { return col.name; })}, - block_with_constants{PKCondition::getBlockWithConstants(query_info.query, context, data.getColumnsList())}, + block_with_constants{PKCondition::getBlockWithConstants(query_info.query, context, data.columns.getList())}, prepared_sets(query_info.sets), log{log} { diff --git a/dbms/src/Storages/MergeTree/ReplicatedMergeTreeAlterThread.cpp b/dbms/src/Storages/MergeTree/ReplicatedMergeTreeAlterThread.cpp index ef7582c497c..c359a889bfb 100644 --- a/dbms/src/Storages/MergeTree/ReplicatedMergeTreeAlterThread.cpp +++ b/dbms/src/Storages/MergeTree/ReplicatedMergeTreeAlterThread.cpp @@ -56,12 +56,7 @@ void ReplicatedMergeTreeAlterThread::run() zkutil::Stat stat; const String columns_str = zookeeper->get(storage.zookeeper_path + "/columns", &stat, wakeup_event); - auto columns_desc = ColumnsDescription::parse(columns_str); - - auto & columns = columns_desc.columns; - auto & materialized_columns = columns_desc.materialized; - auto & alias_columns = columns_desc.alias; - auto & column_defaults = columns_desc.defaults; + auto columns_in_zk = ColumnsDescription::parse(columns_str); bool changed_version = (stat.version != storage.columns_version); @@ -92,42 +87,15 @@ void ReplicatedMergeTreeAlterThread::run() auto table_lock = storage.lockStructureForAlter(__PRETTY_FUNCTION__); - const auto columns_changed = columns != storage.data.getColumnsListNonMaterialized(); - const auto materialized_columns_changed = materialized_columns != storage.data.materialized_columns; - const auto alias_columns_changed = alias_columns != storage.data.alias_columns; - const auto column_defaults_changed = column_defaults != storage.data.column_defaults; - - if (columns_changed || materialized_columns_changed || alias_columns_changed || - column_defaults_changed) + if (columns_in_zk != storage.data.columns) { LOG_INFO(log, "Columns list changed in ZooKeeper. Applying changes locally."); storage.context.getDatabase(storage.database_name)->alterTable( - storage.context, storage.table_name, - columns, materialized_columns, alias_columns, column_defaults, {}); + storage.context, storage.table_name, columns_in_zk, {}); - if (columns_changed) - { - storage.data.setColumnsList(columns); - } - - if (materialized_columns_changed) - { - storage.materialized_columns = materialized_columns; - storage.data.materialized_columns = std::move(materialized_columns); - } - - if (alias_columns_changed) - { - storage.alias_columns = alias_columns; - storage.data.alias_columns = std::move(alias_columns); - } - - if (column_defaults_changed) - { - storage.column_defaults = column_defaults; - storage.data.column_defaults = std::move(column_defaults); - } + storage.columns = columns_in_zk; + storage.data.columns = std::move(columns_in_zk); /// Reinitialize primary key because primary key column types might have changed. storage.data.initPrimaryKey(); @@ -158,7 +126,7 @@ void ReplicatedMergeTreeAlterThread::run() if (!changed_version) parts = storage.data.getDataParts(); - const auto columns_plus_materialized = storage.data.getColumnsList(); + const auto columns_plus_materialized = storage.data.columns.getList(); for (const MergeTreeData::DataPartPtr & part : parts) { diff --git a/dbms/src/Storages/MergeTree/registerStorageMergeTree.cpp b/dbms/src/Storages/MergeTree/registerStorageMergeTree.cpp index 89787fca4f0..4cdff05c2e3 100644 --- a/dbms/src/Storages/MergeTree/registerStorageMergeTree.cpp +++ b/dbms/src/Storages/MergeTree/registerStorageMergeTree.cpp @@ -628,14 +628,13 @@ static StoragePtr create(const StorageFactory::Arguments & args) if (replicated) return StorageReplicatedMergeTree::create( zookeeper_path, replica_name, args.attach, args.data_path, args.database_name, args.table_name, - args.columns, args.materialized_columns, args.alias_columns, args.column_defaults, + args.columns, args.context, primary_expr_list, secondary_sorting_expr_list, date_column_name, partition_expr_list, sampling_expression, merging_params, storage_settings, args.has_force_restore_data_flag); else return StorageMergeTree::create( - args.data_path, args.database_name, args.table_name, - args.columns, args.materialized_columns, args.alias_columns, args.column_defaults, args.attach, + args.data_path, args.database_name, args.table_name, args.columns, args.attach, args.context, primary_expr_list, secondary_sorting_expr_list, date_column_name, partition_expr_list, sampling_expression, merging_params, storage_settings, args.has_force_restore_data_flag); diff --git a/dbms/src/Storages/StorageBuffer.cpp b/dbms/src/Storages/StorageBuffer.cpp index 6b9ec92ce75..c76a1edc81b 100644 --- a/dbms/src/Storages/StorageBuffer.cpp +++ b/dbms/src/Storages/StorageBuffer.cpp @@ -48,14 +48,11 @@ namespace ErrorCodes } -StorageBuffer::StorageBuffer(const std::string & name_, const NamesAndTypesList & columns_, - const NamesAndTypesList & materialized_columns_, - const NamesAndTypesList & alias_columns_, - const ColumnDefaults & column_defaults_, +StorageBuffer::StorageBuffer(const std::string & name_, const ColumnsDescription & columns_, Context & context_, size_t num_shards_, const Thresholds & min_thresholds_, const Thresholds & max_thresholds_, const String & destination_database_, const String & destination_table_, bool allow_materialized_) - : IStorage{columns_, materialized_columns_, alias_columns_, column_defaults_}, + : IStorage{columns_}, name(name_), context(context_), num_shards(num_shards_), buffers(num_shards_), min_thresholds(min_thresholds_), max_thresholds(max_thresholds_), @@ -598,11 +595,8 @@ void StorageBuffer::alter(const AlterCommands & params, const String & database_ /// So that no blocks of the old structure remain. optimize({} /*query*/, {} /*partition_id*/, false /*final*/, false /*deduplicate*/, context); - params.apply(columns, materialized_columns, alias_columns, column_defaults); - - context.getDatabase(database_name)->alterTable( - context, table_name, - columns, materialized_columns, alias_columns, column_defaults, {}); + params.apply(columns); + context.getDatabase(database_name)->alterTable(context, table_name, columns, {}); } @@ -641,7 +635,6 @@ void registerStorageBuffer(StorageFactory & factory) return StorageBuffer::create( args.table_name, args.columns, - args.materialized_columns, args.alias_columns, args.column_defaults, args.context, num_buckets, StorageBuffer::Thresholds{min_time, min_rows, min_bytes}, diff --git a/dbms/src/Storages/StorageBuffer.h b/dbms/src/Storages/StorageBuffer.h index 9d9f72340c5..02c2ef9b09b 100644 --- a/dbms/src/Storages/StorageBuffer.h +++ b/dbms/src/Storages/StorageBuffer.h @@ -123,10 +123,7 @@ protected: /** num_shards - the level of internal parallelism (the number of independent buffers) * The buffer is flushed if all minimum thresholds or at least one of the maximum thresholds are exceeded. */ - StorageBuffer(const std::string & name_, const NamesAndTypesList & columns_, - const NamesAndTypesList & materialized_columns_, - const NamesAndTypesList & alias_columns_, - const ColumnDefaults & column_defaults_, + StorageBuffer(const std::string & name_, const ColumnsDescription & columns_, Context & context_, size_t num_shards_, const Thresholds & min_thresholds_, const Thresholds & max_thresholds_, const String & destination_database_, const String & destination_table_, bool allow_materialized_); diff --git a/dbms/src/Storages/StorageCatBoostPool.cpp b/dbms/src/Storages/StorageCatBoostPool.cpp index cbaf523e3bf..1a973f7dd75 100644 --- a/dbms/src/Storages/StorageCatBoostPool.cpp +++ b/dbms/src/Storages/StorageCatBoostPool.cpp @@ -220,7 +220,6 @@ void StorageCatBoostPool::parseColumnDescription() void StorageCatBoostPool::createSampleBlockAndColumns() { - columns.clear(); NamesAndTypesList cat_columns; NamesAndTypesList num_columns; sample_block.clear(); @@ -239,19 +238,19 @@ void StorageCatBoostPool::createSampleBlockAndColumns() else if (desc.column_type == DatasetColumnType::Num) num_columns.emplace_back(desc.column_name, type); else - materialized_columns.emplace_back(desc.column_name, type); + columns.materialized.emplace_back(desc.column_name, type); if (!desc.alias.empty()) { auto alias = std::make_shared(desc.column_name); - column_defaults[desc.alias] = {ColumnDefaultType::Alias, alias}; - alias_columns.emplace_back(desc.alias, type); + columns.defaults[desc.alias] = {ColumnDefaultType::Alias, alias}; + columns.aliases.emplace_back(desc.alias, type); } sample_block.insert(ColumnWithTypeAndName(type, desc.column_name)); } - columns.insert(columns.end(), num_columns.begin(), num_columns.end()); - columns.insert(columns.end(), cat_columns.begin(), cat_columns.end()); + columns.ordinary.insert(columns.ordinary.end(), num_columns.begin(), num_columns.end()); + columns.ordinary.insert(columns.ordinary.end(), cat_columns.begin(), cat_columns.end()); } BlockInputStreams StorageCatBoostPool::read(const Names & column_names, diff --git a/dbms/src/Storages/StorageDictionary.cpp b/dbms/src/Storages/StorageDictionary.cpp index 2bb69748c6d..954b8ae898b 100644 --- a/dbms/src/Storages/StorageDictionary.cpp +++ b/dbms/src/Storages/StorageDictionary.cpp @@ -24,13 +24,10 @@ namespace ErrorCodes StorageDictionary::StorageDictionary( const String & table_name_, - const NamesAndTypesList & columns_, - const NamesAndTypesList & materialized_columns_, - const NamesAndTypesList & alias_columns_, - const ColumnDefaults & column_defaults_, + const ColumnsDescription & columns_, const DictionaryStructure & dictionary_structure_, const String & dictionary_name_) - : IStorage{columns_, materialized_columns_, alias_columns_, column_defaults_}, table_name(table_name_), + : IStorage{columns_}, table_name(table_name_), dictionary_name(dictionary_name_), logger(&Poco::Logger::get("StorageDictionary")) { @@ -75,7 +72,7 @@ void StorageDictionary::checkNamesAndTypesCompatibleWithDictionary(const Diction auto dictionary_names_and_types = getNamesAndTypes(dictionary_structure); std::set namesAndTypesSet(dictionary_names_and_types.begin(), dictionary_names_and_types.end()); - for (auto & column : columns) + for (auto & column : columns.ordinary) { if (namesAndTypesSet.find(column) == namesAndTypesSet.end()) { @@ -105,8 +102,7 @@ void registerStorageDictionary(StorageFactory & factory) const DictionaryStructure & dictionary_structure = dictionary->getStructure(); return StorageDictionary::create( - args.table_name, args.columns, args.materialized_columns, args.alias_columns, - args.column_defaults, dictionary_structure, dictionary_name); + args.table_name, args.columns, dictionary_structure, dictionary_name); }); } diff --git a/dbms/src/Storages/StorageDictionary.h b/dbms/src/Storages/StorageDictionary.h index 0f21373f546..0320a514839 100644 --- a/dbms/src/Storages/StorageDictionary.h +++ b/dbms/src/Storages/StorageDictionary.h @@ -22,7 +22,6 @@ class StorageDictionary : public ext::shared_ptr_helper, publ public: std::string getName() const override { return "Dictionary"; } std::string getTableName() const override { return table_name; } - const NamesAndTypesList & getColumnsListImpl() const override { return columns; } BlockInputStreams read(const Names & column_names, const SelectQueryInfo & query_info, const Context & context, @@ -62,10 +61,7 @@ private: protected: StorageDictionary(const String & table_name_, - const NamesAndTypesList & columns_, - const NamesAndTypesList & materialized_columns_, - const NamesAndTypesList & alias_columns_, - const ColumnDefaults & column_defaults_, + const ColumnsDescription & columns_, const DictionaryStructure & dictionary_structure_, const String & dictionary_name_); }; diff --git a/dbms/src/Storages/StorageDistributed.cpp b/dbms/src/Storages/StorageDistributed.cpp index c1681e6fbea..1b7c5fdf69a 100644 --- a/dbms/src/Storages/StorageDistributed.cpp +++ b/dbms/src/Storages/StorageDistributed.cpp @@ -130,24 +130,21 @@ StorageDistributed::~StorageDistributed() = default; StorageDistributed::StorageDistributed( - const std::string & name_, - const NamesAndTypesList & columns_, - const NamesAndTypesList & materialized_columns_, - const NamesAndTypesList & alias_columns_, - const ColumnDefaults & column_defaults_, + const std::string & table_name_, + const ColumnsDescription & columns_, const String & remote_database_, const String & remote_table_, const String & cluster_name_, const Context & context_, const ASTPtr & sharding_key_, const String & data_path_) - : IStorage{columns_, materialized_columns_, alias_columns_, column_defaults_}, - name(name_), + : IStorage{columns_}, + table_name(table_name_), remote_database(remote_database_), remote_table(remote_table_), context(context_), cluster_name(context.getMacros().expand(cluster_name_)), has_sharding_key(sharding_key_), - sharding_key_expr(sharding_key_ ? ExpressionAnalyzer(sharding_key_, context, nullptr, columns).getActions(false) : nullptr), + sharding_key_expr(sharding_key_ ? ExpressionAnalyzer(sharding_key_, context, nullptr, columns.ordinary).getActions(false) : nullptr), sharding_key_column_name(sharding_key_ ? sharding_key_->getColumnName() : String{}), - path(data_path_.empty() ? "" : (data_path_ + escapeForFileName(name) + '/')) + path(data_path_.empty() ? "" : (data_path_ + escapeForFileName(table_name) + '/')) { } @@ -161,7 +158,7 @@ StoragePtr StorageDistributed::createWithOwnCluster( const Context & context_) { auto res = ext::shared_ptr_helper::create( - name_, columns_, NamesAndTypesList(), NamesAndTypesList(), ColumnDefaults(), + name_, ColumnsDescription{columns_, NamesAndTypesList(), NamesAndTypesList(), ColumnDefaults()}, remote_database_, remote_table_, String{}, context_, ASTPtr(), String()); res->owned_cluster = owned_cluster_; @@ -236,11 +233,9 @@ void StorageDistributed::alter(const AlterCommands & params, const String & data throw Exception("Storage engine " + getName() + " doesn't support primary key.", ErrorCodes::NOT_IMPLEMENTED); auto lock = lockStructureForAlter(__PRETTY_FUNCTION__); - params.apply(columns, materialized_columns, alias_columns, column_defaults); + params.apply(columns); - context.getDatabase(database_name)->alterTable( - context, table_name, - columns, materialized_columns, alias_columns, column_defaults, {}); + context.getDatabase(database_name)->alterTable(context, table_name, columns, {}); } @@ -291,13 +286,13 @@ NameAndTypePair StorageDistributed::getColumn(const String & column_name) const if (const auto & type = VirtualColumnFactory::tryGetType(column_name)) return { column_name, type }; - return getRealColumn(column_name); + return columns.get(column_name); } bool StorageDistributed::hasColumn(const String & column_name) const { - return VirtualColumnFactory::hasColumn(column_name) || IStorage::hasColumn(column_name); + return VirtualColumnFactory::hasColumn(column_name) || columns.has(column_name); } void StorageDistributed::createDirectoryMonitors() @@ -390,7 +385,7 @@ void registerStorageDistributed(StorageFactory & factory) /// Check that sharding_key exists in the table and has numeric type. if (sharding_key) { - auto sharding_expr = ExpressionAnalyzer(sharding_key, args.context, nullptr, args.columns).getActions(true); + auto sharding_expr = ExpressionAnalyzer(sharding_key, args.context, nullptr, args.columns.ordinary).getActions(true); const Block & block = sharding_expr->getSampleBlock(); if (block.columns() != 1) @@ -405,7 +400,6 @@ void registerStorageDistributed(StorageFactory & factory) return StorageDistributed::create( args.table_name, args.columns, - args.materialized_columns, args.alias_columns, args.column_defaults, remote_database, remote_table, cluster_name, args.context, sharding_key, args.data_path); }); diff --git a/dbms/src/Storages/StorageDistributed.h b/dbms/src/Storages/StorageDistributed.h index 6afb54ba007..22e24c3f9fb 100644 --- a/dbms/src/Storages/StorageDistributed.h +++ b/dbms/src/Storages/StorageDistributed.h @@ -34,20 +34,19 @@ public: ~StorageDistributed() override; static StoragePtr createWithOwnCluster( - const std::string & name_, /// The name of the table. - const NamesAndTypesList & columns_, /// List of columns. + const std::string & table_name_, + const NamesAndTypesList & columns_, const String & remote_database_, /// database on remote servers. const String & remote_table_, /// The name of the table on the remote servers. ClusterPtr & owned_cluster_, const Context & context_); std::string getName() const override { return "Distributed"; } - std::string getTableName() const override { return name; } + std::string getTableName() const override { return table_name; } bool supportsSampling() const override { return true; } bool supportsFinal() const override { return true; } bool supportsPrewhere() const override { return true; } - const NamesAndTypesList & getColumnsListImpl() const override { return columns; } NameAndTypePair getColumn(const String & column_name) const override; bool hasColumn(const String & column_name) const override; @@ -64,7 +63,7 @@ public: BlockOutputStreamPtr write(const ASTPtr & query, const Settings & settings) override; void drop() override {} - void rename(const String & /*new_path_to_db*/, const String & /*new_database_name*/, const String & new_table_name) override { name = new_table_name; } + void rename(const String & /*new_path_to_db*/, const String & /*new_database_name*/, const String & new_table_name) override { table_name = new_table_name; } /// in the sub-tables, you need to manually add and delete columns /// the structure of the sub-table is not checked void alter(const AlterCommands & params, const String & database_name, const String & table_name, const Context & context) override; @@ -95,7 +94,7 @@ public: ClusterPtr getCluster() const; - String name; + String table_name; String remote_database; String remote_table; @@ -132,10 +131,7 @@ public: protected: StorageDistributed( const std::string & name_, - const NamesAndTypesList & columns_, - const NamesAndTypesList & materialized_columns_, - const NamesAndTypesList & alias_columns_, - const ColumnDefaults & column_defaults_, + const ColumnsDescription & columns_, const String & remote_database_, const String & remote_table_, const String & cluster_name_, diff --git a/dbms/src/Storages/StorageFactory.cpp b/dbms/src/Storages/StorageFactory.cpp index 0bcc7166c1d..1b8970aba24 100644 --- a/dbms/src/Storages/StorageFactory.cpp +++ b/dbms/src/Storages/StorageFactory.cpp @@ -45,10 +45,7 @@ StoragePtr StorageFactory::get( const String & database_name, Context & local_context, Context & context, - const NamesAndTypesList & columns, - const NamesAndTypesList & materialized_columns, - const NamesAndTypesList & alias_columns, - const ColumnDefaults & column_defaults, + const ColumnsDescription & columns, bool attach, bool has_force_restore_data_flag) const { @@ -67,9 +64,9 @@ StoragePtr StorageFactory::get( { /// Check for some special types, that are not allowed to be stored in tables. Example: NULL data type. /// Exception: any type is allowed in View, because plain (non-materialized) View does not store anything itself. - checkAllTypesAreAllowedInTable(columns); - checkAllTypesAreAllowedInTable(materialized_columns); - checkAllTypesAreAllowedInTable(alias_columns); + checkAllTypesAreAllowedInTable(columns.ordinary); + checkAllTypesAreAllowedInTable(columns.materialized); + checkAllTypesAreAllowedInTable(columns.aliases); if (query.is_materialized_view) { @@ -130,9 +127,6 @@ StoragePtr StorageFactory::get( .local_context = local_context, .context = context, .columns = columns, - .materialized_columns = materialized_columns, - .alias_columns = alias_columns, - .column_defaults = column_defaults, .attach = attach, .has_force_restore_data_flag = has_force_restore_data_flag }; diff --git a/dbms/src/Storages/StorageFactory.h b/dbms/src/Storages/StorageFactory.h index 1af131943ba..2acb9fb7c00 100644 --- a/dbms/src/Storages/StorageFactory.h +++ b/dbms/src/Storages/StorageFactory.h @@ -14,7 +14,7 @@ class ASTStorage; /** Allows to create a table by the name and parameters of the engine. - * In 'columns', 'materialized_columns', etc., Nested data structures must be flattened. + * In 'columns' Nested data structures must be flattened. * You should subsequently call IStorage::startup method to work with table. */ class StorageFactory : public ext::singleton @@ -31,10 +31,7 @@ public: const String & database_name; Context & local_context; Context & context; - const NamesAndTypesList & columns; - const NamesAndTypesList & materialized_columns; - const NamesAndTypesList & alias_columns; - const ColumnDefaults & column_defaults; + const ColumnsDescription & columns; bool attach; bool has_force_restore_data_flag; }; @@ -48,10 +45,7 @@ public: const String & database_name, Context & local_context, Context & context, - const NamesAndTypesList & columns, - const NamesAndTypesList & materialized_columns, - const NamesAndTypesList & alias_columns, - const ColumnDefaults & column_defaults, + const ColumnsDescription & columns, bool attach, bool has_force_restore_data_flag) const; diff --git a/dbms/src/Storages/StorageFile.cpp b/dbms/src/Storages/StorageFile.cpp index 78fb3990978..2f606d5cbcf 100644 --- a/dbms/src/Storages/StorageFile.cpp +++ b/dbms/src/Storages/StorageFile.cpp @@ -54,12 +54,9 @@ StorageFile::StorageFile( const std::string & db_dir_path, const std::string & table_name_, const std::string & format_name_, - const NamesAndTypesList & columns_, - const NamesAndTypesList & materialized_columns_, - const NamesAndTypesList & alias_columns_, - const ColumnDefaults & column_defaults_, + const ColumnsDescription & columns_, Context & context_) - : IStorage(columns_, materialized_columns_, alias_columns_, column_defaults_), + : IStorage(columns_), table_name(table_name_), format_name(format_name_), context_global(context_), table_fd(table_fd_) { if (table_fd < 0) /// Will use file @@ -311,7 +308,6 @@ void registerStorageFile(StorageFactory & factory) return StorageFile::create( source_path, source_fd, args.data_path, args.table_name, format_name, args.columns, - args.materialized_columns, args.alias_columns, args.column_defaults, args.context); }); } diff --git a/dbms/src/Storages/StorageFile.h b/dbms/src/Storages/StorageFile.h index 8501415001c..4292ea6d7f9 100644 --- a/dbms/src/Storages/StorageFile.h +++ b/dbms/src/Storages/StorageFile.h @@ -31,11 +31,6 @@ public: return table_name; } - const NamesAndTypesList & getColumnsListImpl() const override - { - return columns; - } - BlockInputStreams read( const Names & column_names, const SelectQueryInfo & query_info, @@ -61,7 +56,7 @@ protected: /** there are three options (ordered by priority): - use specified file descriptor if (fd >= 0) - use specified table_path if it isn't empty - - create own tabale inside data/db/table/ + - create own table inside data/db/table/ */ StorageFile( const std::string & table_path_, @@ -69,10 +64,7 @@ protected: const std::string & db_dir_path, const std::string & table_name_, const std::string & format_name_, - const NamesAndTypesList & columns_, - const NamesAndTypesList & materialized_columns_, - const NamesAndTypesList & alias_columns_, - const ColumnDefaults & column_defaults_, + const ColumnsDescription & columns_, Context & context_); private: diff --git a/dbms/src/Storages/StorageJoin.cpp b/dbms/src/Storages/StorageJoin.cpp index 846c18d7055..c3dfc16c5d7 100644 --- a/dbms/src/Storages/StorageJoin.cpp +++ b/dbms/src/Storages/StorageJoin.cpp @@ -24,24 +24,12 @@ StorageJoin::StorageJoin( const String & name_, const Names & key_names_, ASTTableJoin::Kind kind_, ASTTableJoin::Strictness strictness_, - const NamesAndTypesList & columns_, - const NamesAndTypesList & materialized_columns_, - const NamesAndTypesList & alias_columns_, - const ColumnDefaults & column_defaults_) - : StorageSetOrJoinBase{path_, name_, columns_, materialized_columns_, alias_columns_, column_defaults_}, + const ColumnsDescription & columns_) + : StorageSetOrJoinBase{path_, name_, columns_}, key_names(key_names_), kind(kind_), strictness(strictness_) { - /// Check that key exists in table definition. - const auto check_key_exists = [] (const NamesAndTypesList & columns, const String & key) - { - for (const auto & column : columns) - if (column.name == key) - return true; - return false; - }; - for (const auto & key : key_names) - if (!check_key_exists(columns, key) && !check_key_exists(materialized_columns, key)) + if (!columns.has(key)) throw Exception{ "Key column (" + key + ") does not exist in table declaration.", ErrorCodes::NO_SUCH_COLUMN_IN_TABLE}; @@ -58,7 +46,7 @@ void StorageJoin::assertCompatible(ASTTableJoin::Kind kind_, ASTTableJoin::Stric { /// NOTE Could be more loose. if (!(kind == kind_ && strictness == strictness_)) - throw Exception("Table " + name + " has incompatible type of JOIN.", ErrorCodes::INCOMPATIBLE_TYPE_OF_JOIN); + throw Exception("Table " + table_name + " has incompatible type of JOIN.", ErrorCodes::INCOMPATIBLE_TYPE_OF_JOIN); } @@ -123,7 +111,7 @@ void registerStorageJoin(StorageFactory & factory) return StorageJoin::create( args.data_path, args.table_name, key_names, kind, strictness, - args.columns, args.materialized_columns, args.alias_columns, args.column_defaults); + args.columns); }); } diff --git a/dbms/src/Storages/StorageJoin.h b/dbms/src/Storages/StorageJoin.h index 135236beddb..c005450e772 100644 --- a/dbms/src/Storages/StorageJoin.h +++ b/dbms/src/Storages/StorageJoin.h @@ -47,10 +47,7 @@ protected: const String & name_, const Names & key_names_, ASTTableJoin::Kind kind_, ASTTableJoin::Strictness strictness_, - const NamesAndTypesList & columns_, - const NamesAndTypesList & materialized_columns_, - const NamesAndTypesList & alias_columns_, - const ColumnDefaults & column_defaults_); + const ColumnsDescription & columns_); }; } diff --git a/dbms/src/Storages/StorageKafka.cpp b/dbms/src/Storages/StorageKafka.cpp index c3b14a468ca..a822fe47a92 100644 --- a/dbms/src/Storages/StorageKafka.cpp +++ b/dbms/src/Storages/StorageKafka.cpp @@ -218,13 +218,10 @@ StorageKafka::StorageKafka( const std::string & table_name_, const std::string & database_name_, Context & context_, - const NamesAndTypesList & columns_, - const NamesAndTypesList & materialized_columns_, - const NamesAndTypesList & alias_columns_, - const ColumnDefaults & column_defaults_, + const ColumnsDescription & columns_, const String & brokers_, const String & group_, const Names & topics_, const String & format_name_, const String & schema_name_, size_t num_consumers_) - : IStorage{columns_, materialized_columns_, alias_columns_, column_defaults_}, + : IStorage{columns_}, table_name(table_name_), database_name(database_name_), context(context_), topics(topics_), brokers(brokers_), group(group_), format_name(format_name_), schema_name(schema_name_), num_consumers(num_consumers_), log(&Logger::get("StorageKafka (" + table_name_ + ")")), @@ -591,7 +588,6 @@ void registerStorageKafka(StorageFactory & factory) return StorageKafka::create( args.table_name, args.database_name, args.context, args.columns, - args.materialized_columns, args.alias_columns, args.column_defaults, brokers, group, topics, format, schema, num_consumers); }); } diff --git a/dbms/src/Storages/StorageKafka.h b/dbms/src/Storages/StorageKafka.h index 39df9c136a1..655b0bfe5e8 100644 --- a/dbms/src/Storages/StorageKafka.h +++ b/dbms/src/Storages/StorageKafka.h @@ -102,10 +102,7 @@ protected: const std::string & table_name_, const std::string & database_name_, Context & context_, - const NamesAndTypesList & columns_, - const NamesAndTypesList & materialized_columns_, - const NamesAndTypesList & alias_columns_, - const ColumnDefaults & column_defaults_, + const ColumnsDescription & columns_, const String & brokers_, const String & group_, const Names & topics_, const String & format_name_, const String & schema_name_, size_t num_consumers_); }; diff --git a/dbms/src/Storages/StorageLog.cpp b/dbms/src/Storages/StorageLog.cpp index 711d48621f0..27881d8b8fe 100644 --- a/dbms/src/Storages/StorageLog.cpp +++ b/dbms/src/Storages/StorageLog.cpp @@ -358,12 +358,9 @@ void LogBlockOutputStream::writeMarks(MarksForColumns && marks) StorageLog::StorageLog( const std::string & path_, const std::string & name_, - const NamesAndTypesList & columns_, - const NamesAndTypesList & materialized_columns_, - const NamesAndTypesList & alias_columns_, - const ColumnDefaults & column_defaults_, + const ColumnsDescription & columns_, size_t max_compress_block_size_) - : IStorage{columns_, materialized_columns_, alias_columns_, column_defaults_}, + : IStorage{columns_}, path(path_), name(name_), max_compress_block_size(max_compress_block_size_), file_checker(path + escapeForFileName(name) + '/' + "sizes.json") @@ -374,7 +371,7 @@ StorageLog::StorageLog( /// create files if they do not exist Poco::File(path + escapeForFileName(name) + '/').createDirectories(); - for (const auto & column : getColumnsList()) + for (const auto & column : columns.getList()) addFiles(column.name, *column.type); marks_file = Poco::File(path + escapeForFileName(name) + '/' + DBMS_STORAGE_LOG_MARKS_FILE_NAME); @@ -468,8 +465,8 @@ void StorageLog::rename(const String & new_path_to_db, const String & /*new_data const StorageLog::Marks & StorageLog::getMarksWithRealRowCount() const { - const String & column_name = columns.front().name; - const IDataType & column_type = *columns.front().type; + const String & column_name = columns.ordinary.front().name; + const IDataType & column_type = *columns.ordinary.front().type; String filename; /** We take marks from first column. @@ -502,7 +499,7 @@ BlockInputStreams StorageLog::read( processed_stage = QueryProcessingStage::FetchColumns; loadMarks(); - NamesAndTypesList columns = Nested::collect(getColumnsList().addTypes(column_names)); + NamesAndTypesList all_columns = Nested::collect(columns.getList().addTypes(column_names)); std::shared_lock lock(rwlock); @@ -526,7 +523,7 @@ BlockInputStreams StorageLog::read( res.emplace_back(std::make_shared( max_block_size, - columns, + all_columns, *this, mark_begin, rows_end - rows_begin, @@ -562,7 +559,6 @@ void registerStorageLog(StorageFactory & factory) return StorageLog::create( args.data_path, args.table_name, args.columns, - args.materialized_columns, args.alias_columns, args.column_defaults, args.context.getSettings().max_compress_block_size); }); } diff --git a/dbms/src/Storages/StorageLog.h b/dbms/src/Storages/StorageLog.h index 4f38f403138..b52a90ec563 100644 --- a/dbms/src/Storages/StorageLog.h +++ b/dbms/src/Storages/StorageLog.h @@ -52,10 +52,7 @@ protected: StorageLog( const std::string & path_, const std::string & name_, - const NamesAndTypesList & columns_, - const NamesAndTypesList & materialized_columns_, - const NamesAndTypesList & alias_columns_, - const ColumnDefaults & column_defaults_, + const ColumnsDescription & columns_, size_t max_compress_block_size_); private: diff --git a/dbms/src/Storages/StorageMaterializedView.cpp b/dbms/src/Storages/StorageMaterializedView.cpp index 33eec0ad8f7..1146028830b 100644 --- a/dbms/src/Storages/StorageMaterializedView.cpp +++ b/dbms/src/Storages/StorageMaterializedView.cpp @@ -64,12 +64,9 @@ StorageMaterializedView::StorageMaterializedView( const String & database_name_, Context & local_context, const ASTCreateQuery & query, - const NamesAndTypesList & columns_, - const NamesAndTypesList & materialized_columns_, - const NamesAndTypesList & alias_columns_, - const ColumnDefaults & column_defaults_, + const ColumnsDescription & columns_, bool attach_) - : IStorage{columns_, materialized_columns_, alias_columns_, column_defaults_}, table_name(table_name_), + : IStorage{columns_}, table_name(table_name_), database_name(database_name_), global_context(local_context.getGlobalContext()) { if (!query.select) @@ -213,8 +210,7 @@ void registerStorageMaterializedView(StorageFactory & factory) /// Pass local_context here to convey setting for inner table return StorageMaterializedView::create( args.table_name, args.database_name, args.local_context, args.query, - args.columns, args.materialized_columns, args.alias_columns, args.column_defaults, - args.attach); + args.columns, args.attach); }); } diff --git a/dbms/src/Storages/StorageMaterializedView.h b/dbms/src/Storages/StorageMaterializedView.h index 4dd0980e874..82a5b0822b4 100644 --- a/dbms/src/Storages/StorageMaterializedView.h +++ b/dbms/src/Storages/StorageMaterializedView.h @@ -17,7 +17,6 @@ class StorageMaterializedView : public ext::shared_ptr_helperclone(); }; StoragePtr getTargetTable() const; @@ -62,10 +61,7 @@ protected: const String & database_name_, Context & local_context, const ASTCreateQuery & query, - const NamesAndTypesList & columns_, - const NamesAndTypesList & materialized_columns_, - const NamesAndTypesList & alias_columns_, - const ColumnDefaults & column_defaults_, + const ColumnsDescription & columns_, bool attach_); }; diff --git a/dbms/src/Storages/StorageMemory.cpp b/dbms/src/Storages/StorageMemory.cpp index 96af6a9a138..7b36796c324 100644 --- a/dbms/src/Storages/StorageMemory.cpp +++ b/dbms/src/Storages/StorageMemory.cpp @@ -74,14 +74,8 @@ private: }; -StorageMemory::StorageMemory( - const std::string & name_, - const NamesAndTypesList & columns_, - const NamesAndTypesList & materialized_columns_, - const NamesAndTypesList & alias_columns_, - const ColumnDefaults & column_defaults_) - : IStorage{columns_, materialized_columns_, alias_columns_, column_defaults_}, - name(name_) +StorageMemory::StorageMemory(String table_name_, ColumnsDescription columns_description_) + : IStorage{std::move(columns_description_)}, table_name(std::move(table_name_)) { } @@ -144,7 +138,7 @@ void registerStorageMemory(StorageFactory & factory) "Engine " + args.engine_name + " doesn't support any arguments (" + toString(args.engine_args.size()) + " given)", ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH); - return StorageMemory::create(args.table_name, args.columns, args.materialized_columns, args.alias_columns, args.column_defaults); + return StorageMemory::create(args.table_name, args.columns); }); } diff --git a/dbms/src/Storages/StorageMemory.h b/dbms/src/Storages/StorageMemory.h index 956e4354727..454fdcc2823 100644 --- a/dbms/src/Storages/StorageMemory.h +++ b/dbms/src/Storages/StorageMemory.h @@ -24,7 +24,7 @@ friend class MemoryBlockOutputStream; public: std::string getName() const override { return "Memory"; } - std::string getTableName() const override { return name; } + std::string getTableName() const override { return table_name; } size_t getSize() const { return data.size(); } @@ -39,10 +39,10 @@ public: BlockOutputStreamPtr write(const ASTPtr & query, const Settings & settings) override; void drop() override; - void rename(const String & /*new_path_to_db*/, const String & /*new_database_name*/, const String & new_table_name) override { name = new_table_name; } + void rename(const String & /*new_path_to_db*/, const String & /*new_database_name*/, const String & new_table_name) override { table_name = new_table_name; } private: - String name; + String table_name; /// The data itself. `list` - so that when inserted to the end, the existing iterators are not invalidated. BlocksList data; @@ -50,12 +50,7 @@ private: std::mutex mutex; protected: - StorageMemory( - const std::string & name_, - const NamesAndTypesList & columns_, - const NamesAndTypesList & materialized_columns_, - const NamesAndTypesList & alias_columns_, - const ColumnDefaults & column_defaults_); + StorageMemory(String table_name_, ColumnsDescription columns_description_); }; } diff --git a/dbms/src/Storages/StorageMerge.cpp b/dbms/src/Storages/StorageMerge.cpp index 7ceecc62a43..39fed81e538 100644 --- a/dbms/src/Storages/StorageMerge.cpp +++ b/dbms/src/Storages/StorageMerge.cpp @@ -37,14 +37,11 @@ namespace ErrorCodes StorageMerge::StorageMerge( const std::string & name_, - const NamesAndTypesList & columns_, - const NamesAndTypesList & materialized_columns_, - const NamesAndTypesList & alias_columns_, - const ColumnDefaults & column_defaults_, + const ColumnsDescription & columns_, const String & source_database_, const String & table_name_regexp_, const Context & context_) - : IStorage{columns_, materialized_columns_, alias_columns_, column_defaults_}, + : IStorage{columns_}, name(name_), source_database(source_database_), table_name_regexp(table_name_regexp_), context(context_) { @@ -184,7 +181,7 @@ BlockInputStreams StorageMerge::read( /// If there are only virtual columns in query, you must request at least one other column. if (real_column_names.size() == 0) - real_column_names.push_back(ExpressionActions::getSmallestColumn(table->getColumnsList())); + real_column_names.push_back(ExpressionActions::getSmallestColumn(table->columns.getList())); /// Substitute virtual column for its value when querying tables. ASTPtr modified_query_ast = query->clone(); @@ -336,11 +333,9 @@ void StorageMerge::alter(const AlterCommands & params, const String & database_n throw Exception("Storage engine " + getName() + " doesn't support primary key.", ErrorCodes::NOT_IMPLEMENTED); auto lock = lockStructureForAlter(__PRETTY_FUNCTION__); - params.apply(columns, materialized_columns, alias_columns, column_defaults); - context.getDatabase(database_name)->alterTable( - context, table_name, - columns, materialized_columns, alias_columns, column_defaults, {}); + params.apply(columns); + context.getDatabase(database_name)->alterTable(context, table_name, columns, {}); } @@ -367,7 +362,6 @@ void registerStorageMerge(StorageFactory & factory) return StorageMerge::create( args.table_name, args.columns, - args.materialized_columns, args.alias_columns, args.column_defaults, source_database, table_name_regexp, args.context); }); } diff --git a/dbms/src/Storages/StorageMerge.h b/dbms/src/Storages/StorageMerge.h index 9bef645c0e0..0c37e4e7fb4 100644 --- a/dbms/src/Storages/StorageMerge.h +++ b/dbms/src/Storages/StorageMerge.h @@ -26,7 +26,6 @@ public: bool supportsFinal() const override { return true; } bool supportsIndexForIn() const override { return true; } - const NamesAndTypesList & getColumnsListImpl() const override { return columns; } NameAndTypePair getColumn(const String & column_name) const override; bool hasColumn(const String & column_name) const override; @@ -60,10 +59,7 @@ private: protected: StorageMerge( const std::string & name_, - const NamesAndTypesList & columns_, - const NamesAndTypesList & materialized_columns_, - const NamesAndTypesList & alias_columns_, - const ColumnDefaults & column_defaults_, + const ColumnsDescription & columns_, const String & source_database_, const String & table_name_regexp_, const Context & context_); diff --git a/dbms/src/Storages/StorageMergeTree.cpp b/dbms/src/Storages/StorageMergeTree.cpp index 8ee9f3c7c5b..71d8013e4db 100644 --- a/dbms/src/Storages/StorageMergeTree.cpp +++ b/dbms/src/Storages/StorageMergeTree.cpp @@ -35,10 +35,7 @@ StorageMergeTree::StorageMergeTree( const String & path_, const String & database_name_, const String & table_name_, - const NamesAndTypesList & columns_, - const NamesAndTypesList & materialized_columns_, - const NamesAndTypesList & alias_columns_, - const ColumnDefaults & column_defaults_, + const ColumnsDescription & columns_, bool attach, Context & context_, const ASTPtr & primary_expr_ast_, @@ -49,12 +46,11 @@ StorageMergeTree::StorageMergeTree( const MergeTreeData::MergingParams & merging_params_, const MergeTreeSettings & settings_, bool has_force_restore_data_flag) - : IStorage{columns_, materialized_columns_, alias_columns_, column_defaults_}, + : IStorage{columns_}, path(path_), database_name(database_name_), table_name(table_name_), full_path(path + escapeForFileName(table_name) + '/'), context(context_), background_pool(context_.getBackgroundPool()), data(database_name, table_name, full_path, columns_, - materialized_columns_, alias_columns_, column_defaults_, context_, primary_expr_ast_, secondary_sorting_expr_list_, date_column_name, partition_expr_ast_, sampling_expression_, merging_params_, settings_, false, attach), @@ -157,16 +153,9 @@ void StorageMergeTree::alter( data.checkAlter(params); - auto new_columns = data.getColumnsListNonMaterialized(); - auto new_materialized_columns = data.materialized_columns; - auto new_alias_columns = data.alias_columns; - auto new_column_defaults = data.column_defaults; + auto new_columns = data.columns; - params.apply(new_columns, new_materialized_columns, new_alias_columns, new_column_defaults); - - auto columns_for_parts = new_columns; - columns_for_parts.insert(std::end(columns_for_parts), - std::begin(new_materialized_columns), std::end(new_materialized_columns)); + params.apply(new_columns); std::vector transactions; @@ -187,6 +176,7 @@ void StorageMergeTree::alter( throw Exception("MODIFY PRIMARY KEY only supported for tables without sampling key", ErrorCodes::BAD_ARGUMENTS); auto parts = data.getDataParts({MergeTreeDataPartState::PreCommitted, MergeTreeDataPartState::Committed, MergeTreeDataPartState::Outdated}); + auto columns_for_parts = new_columns.getList(); for (const MergeTreeData::DataPartPtr & part : parts) { if (auto transaction = data.alterDataPart(part, columns_for_parts, new_primary_key_ast, false)) @@ -212,19 +202,10 @@ void StorageMergeTree::alter( }; } - context.getDatabase(database_name)->alterTable( - context, table_name, - new_columns, new_materialized_columns, new_alias_columns, new_column_defaults, - storage_modifier); + context.getDatabase(database_name)->alterTable(context, table_name, new_columns, storage_modifier); - materialized_columns = new_materialized_columns; - alias_columns = new_alias_columns; - column_defaults = new_column_defaults; - - data.setColumnsList(new_columns); - data.materialized_columns = std::move(new_materialized_columns); - data.alias_columns = std::move(new_alias_columns); - data.column_defaults = std::move(new_column_defaults); + columns = new_columns; + data.columns = std::move(new_columns); if (primary_key_is_modified) { @@ -438,17 +419,10 @@ void StorageMergeTree::clearColumnInPartition(const ASTPtr & partition, const Fi alter_command.type = AlterCommand::DROP_COLUMN; alter_command.column_name = get(column_name); - auto new_columns = data.getColumnsListNonMaterialized(); - auto new_materialized_columns = data.materialized_columns; - auto new_alias_columns = data.alias_columns; - auto new_column_defaults = data.column_defaults; - - alter_command.apply(new_columns, new_materialized_columns, new_alias_columns, new_column_defaults); - - auto columns_for_parts = new_columns; - columns_for_parts.insert(std::end(columns_for_parts), - std::begin(new_materialized_columns), std::end(new_materialized_columns)); + auto new_columns = data.columns; + alter_command.apply(new_columns); + auto columns_for_parts = new_columns.getList(); for (const auto & part : parts) { if (part->info.partition_id != partition_id) diff --git a/dbms/src/Storages/StorageMergeTree.h b/dbms/src/Storages/StorageMergeTree.h index e1121ec84d0..e2ad6177a82 100644 --- a/dbms/src/Storages/StorageMergeTree.h +++ b/dbms/src/Storages/StorageMergeTree.h @@ -2,6 +2,7 @@ #include +#include #include #include #include @@ -35,8 +36,6 @@ public: bool supportsFinal() const override { return data.supportsFinal(); } bool supportsPrewhere() const override { return data.supportsPrewhere(); } - const NamesAndTypesList & getColumnsListImpl() const override { return data.getColumnsListNonMaterialized(); } - NameAndTypePair getColumn(const String & column_name) const override { return data.getColumn(column_name); @@ -135,10 +134,7 @@ protected: const String & path_, const String & database_name_, const String & table_name_, - const NamesAndTypesList & columns_, - const NamesAndTypesList & materialized_columns_, - const NamesAndTypesList & alias_columns_, - const ColumnDefaults & column_defaults_, + const ColumnsDescription & columns_, bool attach, Context & context_, const ASTPtr & primary_expr_ast_, diff --git a/dbms/src/Storages/StorageMySQL.cpp b/dbms/src/Storages/StorageMySQL.cpp index f3f18518d83..28a367f17e0 100644 --- a/dbms/src/Storages/StorageMySQL.cpp +++ b/dbms/src/Storages/StorageMySQL.cpp @@ -23,8 +23,8 @@ StorageMySQL::StorageMySQL( mysqlxx::Pool && pool, const std::string & remote_database_name, const std::string & remote_table_name, - const NamesAndTypesList & columns_) - : IStorage{columns_, {}, {}, {}} + const ColumnsDescription & columns_) + : IStorage{columns_} , name(name) , remote_database_name(remote_database_name) , remote_table_name(remote_table_name) @@ -43,7 +43,7 @@ BlockInputStreams StorageMySQL::read( { check(column_names); processed_stage = QueryProcessingStage::FetchColumns; - String query = transformQueryForExternalDatabase(*query_info.query, columns, remote_database_name, remote_table_name, context); + String query = transformQueryForExternalDatabase(*query_info.query, columns.ordinary, remote_database_name, remote_table_name, context); Block sample_block; for (const String & name : column_names) diff --git a/dbms/src/Storages/StorageMySQL.h b/dbms/src/Storages/StorageMySQL.h index 9558605ff0e..9e2b233283e 100644 --- a/dbms/src/Storages/StorageMySQL.h +++ b/dbms/src/Storages/StorageMySQL.h @@ -24,7 +24,7 @@ public: mysqlxx::Pool && pool, const std::string & remote_database_name, const std::string & remote_table_name, - const NamesAndTypesList & columns_); + const ColumnsDescription & columns); std::string getName() const override { return "MySQL"; } std::string getTableName() const override { return name; } diff --git a/dbms/src/Storages/StorageNull.cpp b/dbms/src/Storages/StorageNull.cpp index 0cee00aa1b8..13232068ca9 100644 --- a/dbms/src/Storages/StorageNull.cpp +++ b/dbms/src/Storages/StorageNull.cpp @@ -25,8 +25,7 @@ void registerStorageNull(StorageFactory & factory) "Engine " + args.engine_name + " doesn't support any arguments (" + toString(args.engine_args.size()) + " given)", ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH); - return StorageNull::create(args.table_name, - args.columns, args.materialized_columns, args.alias_columns, args.column_defaults); + return StorageNull::create(args.table_name, args.columns); }); } @@ -34,11 +33,9 @@ void StorageNull::alter(const AlterCommands & params, const String & database_na { auto lock = lockStructureForAlter(__PRETTY_FUNCTION__); - params.apply(columns, materialized_columns, alias_columns, column_defaults); + params.apply(columns); - context.getDatabase(database_name)->alterTable( - context, table_name, - columns, materialized_columns, alias_columns, column_defaults, {}); + context.getDatabase(database_name)->alterTable(context, table_name, columns, {}); } } diff --git a/dbms/src/Storages/StorageNull.h b/dbms/src/Storages/StorageNull.h index ef3f0f6fcd1..cdde480a951 100644 --- a/dbms/src/Storages/StorageNull.h +++ b/dbms/src/Storages/StorageNull.h @@ -18,7 +18,7 @@ class StorageNull : public ext::shared_ptr_helper, public IStorage { public: std::string getName() const override { return "Null"; } - std::string getTableName() const override { return name; } + std::string getTableName() const override { return table_name; } BlockInputStreams read( const Names & column_names, @@ -38,22 +38,19 @@ public: void rename(const String & /*new_path_to_db*/, const String & /*new_database_name*/, const String & new_table_name) override { - name = new_table_name; + table_name = new_table_name; } void alter(const AlterCommands & params, const String & database_name, const String & table_name, const Context & context) override; private: - String name; + String table_name; protected: - StorageNull( - const std::string & name_, - const NamesAndTypesList & columns_, - const NamesAndTypesList & materialized_columns_, - const NamesAndTypesList & alias_columns_, - const ColumnDefaults & column_defaults_) - : IStorage{columns_, materialized_columns_, alias_columns_, column_defaults_}, name(name_) {} + StorageNull(String table_name_, ColumnsDescription columns_description_) + : IStorage{std::move(columns_description_)}, table_name(std::move(table_name_)) + { + } }; } diff --git a/dbms/src/Storages/StorageODBC.cpp b/dbms/src/Storages/StorageODBC.cpp index 67362bf2f90..7856ee9a231 100644 --- a/dbms/src/Storages/StorageODBC.cpp +++ b/dbms/src/Storages/StorageODBC.cpp @@ -21,8 +21,8 @@ StorageODBC::StorageODBC( const std::string & connection_string, const std::string & remote_database_name, const std::string & remote_table_name, - const NamesAndTypesList & columns_) - : IStorage{columns_, {}, {}, {}} + const ColumnsDescription & columns_) + : IStorage{columns_} , name(name) , remote_database_name(remote_database_name) , remote_table_name(remote_table_name) @@ -43,7 +43,7 @@ BlockInputStreams StorageODBC::read( { check(column_names); processed_stage = QueryProcessingStage::FetchColumns; - String query = transformQueryForExternalDatabase(*query_info.query, columns, remote_database_name, remote_table_name, context); + String query = transformQueryForExternalDatabase(*query_info.query, columns.ordinary, remote_database_name, remote_table_name, context); Block sample_block; for (const String & name : column_names) diff --git a/dbms/src/Storages/StorageODBC.h b/dbms/src/Storages/StorageODBC.h index f582c74a263..5632fe38282 100644 --- a/dbms/src/Storages/StorageODBC.h +++ b/dbms/src/Storages/StorageODBC.h @@ -30,7 +30,7 @@ public: const std::string & connection_string, const std::string & remote_database_name, const std::string & remote_table_name, - const NamesAndTypesList & columns_); + const ColumnsDescription & columns_); std::string getName() const override { return "ODBC"; } std::string getTableName() const override { return name; } diff --git a/dbms/src/Storages/StorageReplicatedMergeTree.cpp b/dbms/src/Storages/StorageReplicatedMergeTree.cpp index 9917b7e7a94..9f6553f6512 100644 --- a/dbms/src/Storages/StorageReplicatedMergeTree.cpp +++ b/dbms/src/Storages/StorageReplicatedMergeTree.cpp @@ -174,10 +174,7 @@ StorageReplicatedMergeTree::StorageReplicatedMergeTree( const String & replica_name_, bool attach, const String & path_, const String & database_name_, const String & name_, - const NamesAndTypesList & columns_, - const NamesAndTypesList & materialized_columns_, - const NamesAndTypesList & alias_columns_, - const ColumnDefaults & column_defaults_, + const ColumnsDescription & columns_, Context & context_, const ASTPtr & primary_expr_ast_, const ASTPtr & secondary_sorting_expr_list_, @@ -187,14 +184,13 @@ StorageReplicatedMergeTree::StorageReplicatedMergeTree( const MergeTreeData::MergingParams & merging_params_, const MergeTreeSettings & settings_, bool has_force_restore_data_flag) - : IStorage{columns_, materialized_columns_, alias_columns_, column_defaults_}, context(context_), + : IStorage{columns_}, context(context_), current_zookeeper(context.getZooKeeper()), database_name(database_name_), table_name(name_), full_path(path_ + escapeForFileName(table_name) + '/'), zookeeper_path(context.getMacros().expand(zookeeper_path_)), replica_name(context.getMacros().expand(replica_name_)), data(database_name, table_name, full_path, columns_, - materialized_columns_, alias_columns_, column_defaults_, context_, primary_expr_ast_, secondary_sorting_expr_list_, date_column_name, partition_expr_ast_, sampling_expression_, merging_params_, settings_, true, attach, @@ -488,9 +484,7 @@ void StorageReplicatedMergeTree::createTableIfNotExists() acl, zkutil::CreateMode::Persistent)); ops.emplace_back(std::make_unique(zookeeper_path + "/metadata", metadata, acl, zkutil::CreateMode::Persistent)); - ops.emplace_back(std::make_unique(zookeeper_path + "/columns", ColumnsDescription{ - data.getColumnsListNonMaterialized(), data.materialized_columns, - data.alias_columns, data.column_defaults}.toString(), + ops.emplace_back(std::make_unique(zookeeper_path + "/columns", data.columns.toString(), acl, zkutil::CreateMode::Persistent)); ops.emplace_back(std::make_unique(zookeeper_path + "/log", "", acl, zkutil::CreateMode::Persistent)); @@ -524,39 +518,28 @@ void StorageReplicatedMergeTree::checkTableStructure(bool skip_sanity_checks, bo TableMetadata(data).check(metadata_str); zkutil::Stat stat; - auto columns_desc = ColumnsDescription::parse(zookeeper->get(zookeeper_path + "/columns", &stat)); + auto columns_from_zk = ColumnsDescription::parse(zookeeper->get(zookeeper_path + "/columns", &stat)); - auto & columns = columns_desc.columns; - auto & materialized_columns = columns_desc.materialized; - auto & alias_columns = columns_desc.alias; - auto & column_defaults = columns_desc.defaults; columns_version = stat.version; - if (columns != data.getColumnsListNonMaterialized() || - materialized_columns != data.materialized_columns || - alias_columns != data.alias_columns || - column_defaults != data.column_defaults) + if (columns_from_zk != data.columns) { if (allow_alter && (skip_sanity_checks || - data.getColumnsListNonMaterialized().sizeOfDifference(columns) + - data.materialized_columns.sizeOfDifference(materialized_columns) <= 2)) + data.columns.ordinary.sizeOfDifference(columns_from_zk.ordinary) + + data.columns.materialized.sizeOfDifference(columns_from_zk.materialized) <= 2)) { LOG_WARNING(log, "Table structure in ZooKeeper is a little different from local table structure. Assuming ALTER."); /// Without any locks, because table has not been created yet. - context.getDatabase(database_name)->alterTable( - context, table_name, - columns, materialized_columns, alias_columns, column_defaults, {}); + context.getDatabase(database_name)->alterTable(context, table_name, columns_from_zk, {}); - data.setColumnsList(columns); - data.materialized_columns = std::move(materialized_columns); - data.alias_columns = std::move(alias_columns); - data.column_defaults = std::move(column_defaults); + columns = columns_from_zk; + data.columns = std::move(columns_from_zk); } else { - throw Exception("Table structure in ZooKeeper is too much different from local table structure.", + throw Exception("Table structure in ZooKeeper is too different from local table structure", ErrorCodes::INCOMPATIBLE_COLUMNS); } } @@ -714,12 +697,7 @@ void StorageReplicatedMergeTree::createReplica() LOG_DEBUG(log, "Copied " << source_queue.size() << " queue entries"); } - zookeeper->create(replica_path + "/columns", ColumnsDescription{ - data.getColumnsListNonMaterialized(), - data.materialized_columns, - data.alias_columns, - data.column_defaults - }.toString(), zkutil::CreateMode::Persistent); + zookeeper->create(replica_path + "/columns", data.columns.toString(), zkutil::CreateMode::Persistent); } @@ -1529,19 +1507,12 @@ void StorageReplicatedMergeTree::executeClearColumnInPartition(const LogEntry & alter_command.type = AlterCommand::DROP_COLUMN; alter_command.column_name = entry.column_name; - auto new_columns = data.getColumnsListNonMaterialized(); - auto new_materialized_columns = data.materialized_columns; - auto new_alias_columns = data.alias_columns; - auto new_column_defaults = data.column_defaults; - - alter_command.apply(new_columns, new_materialized_columns, new_alias_columns, new_column_defaults); - - auto columns_for_parts = new_columns; - columns_for_parts.insert(std::end(columns_for_parts), - std::begin(new_materialized_columns), std::end(new_materialized_columns)); + auto new_columns = data.columns; + alter_command.apply(new_columns); size_t modified_parts = 0; auto parts = data.getDataParts(); + auto columns_for_parts = new_columns.getList(); for (const auto & part : parts) { if (!entry_part_info.contains(part->info)) @@ -2573,15 +2544,10 @@ void StorageReplicatedMergeTree::alter(const AlterCommands & params, if (param.type == AlterCommand::MODIFY_PRIMARY_KEY) throw Exception("Modification of primary key is not supported for replicated tables", ErrorCodes::NOT_IMPLEMENTED); - NamesAndTypesList new_columns = data.getColumnsListNonMaterialized(); - NamesAndTypesList new_materialized_columns = data.materialized_columns; - NamesAndTypesList new_alias_columns = data.alias_columns; - ColumnDefaults new_column_defaults = data.column_defaults; - params.apply(new_columns, new_materialized_columns, new_alias_columns, new_column_defaults); + ColumnsDescription new_columns = data.columns; + params.apply(new_columns); - new_columns_str = ColumnsDescription{ - new_columns, new_materialized_columns, - new_alias_columns, new_column_defaults}.toString(); + new_columns_str = new_columns.toString(); /// Do ALTER. getZooKeeper()->set(zookeeper_path + "/columns", new_columns_str, -1, &stat); diff --git a/dbms/src/Storages/StorageReplicatedMergeTree.h b/dbms/src/Storages/StorageReplicatedMergeTree.h index b1cbf630b1b..468f097534d 100644 --- a/dbms/src/Storages/StorageReplicatedMergeTree.h +++ b/dbms/src/Storages/StorageReplicatedMergeTree.h @@ -84,8 +84,6 @@ public: bool supportsPrewhere() const override { return data.supportsPrewhere(); } bool supportsReplication() const override { return true; } - const NamesAndTypesList & getColumnsListImpl() const override { return data.getColumnsListNonMaterialized(); } - NameAndTypePair getColumn(const String & column_name) const override { return data.getColumn(column_name); @@ -443,10 +441,7 @@ protected: const String & replica_name_, bool attach, const String & path_, const String & database_name_, const String & name_, - const NamesAndTypesList & columns_, - const NamesAndTypesList & materialized_columns_, - const NamesAndTypesList & alias_columns_, - const ColumnDefaults & column_defaults_, + const ColumnsDescription & columns_, Context & context_, const ASTPtr & primary_expr_ast_, const ASTPtr & secondary_sorting_expr_list_, diff --git a/dbms/src/Storages/StorageSet.cpp b/dbms/src/Storages/StorageSet.cpp index 1303aa6271f..5ba5d737435 100644 --- a/dbms/src/Storages/StorageSet.cpp +++ b/dbms/src/Storages/StorageSet.cpp @@ -88,18 +88,14 @@ BlockOutputStreamPtr StorageSetOrJoinBase::write(const ASTPtr & /*query*/, const StorageSetOrJoinBase::StorageSetOrJoinBase( const String & path_, - const String & name_, - const NamesAndTypesList & columns_, - const NamesAndTypesList & materialized_columns_, - const NamesAndTypesList & alias_columns_, - const ColumnDefaults & column_defaults_) - : IStorage{columns_, materialized_columns_, alias_columns_, column_defaults_}, - name(name_) + const String & table_name_, + const ColumnsDescription & columns_) + : IStorage{columns_}, table_name(table_name_) { if (path_.empty()) throw Exception("Join and Set storages require data path", ErrorCodes::INCORRECT_FILE_NAME); - path = path_ + escapeForFileName(name_) + '/'; + path = path_ + escapeForFileName(table_name_) + '/'; } @@ -107,11 +103,8 @@ StorageSetOrJoinBase::StorageSetOrJoinBase( StorageSet::StorageSet( const String & path_, const String & name_, - const NamesAndTypesList & columns_, - const NamesAndTypesList & materialized_columns_, - const NamesAndTypesList & alias_columns_, - const ColumnDefaults & column_defaults_) - : StorageSetOrJoinBase{path_, name_, columns_, materialized_columns_, alias_columns_, column_defaults_}, + const ColumnsDescription & columns_) + : StorageSetOrJoinBase{path_, name_, columns_}, set(std::make_shared(SizeLimits())) { restore(); @@ -181,7 +174,7 @@ void StorageSetOrJoinBase::rename(const String & new_path_to_db, const String & Poco::File(path).renameTo(new_path); path = new_path + "/"; - name = new_table_name; + table_name = new_table_name; } @@ -194,9 +187,7 @@ void registerStorageSet(StorageFactory & factory) "Engine " + args.engine_name + " doesn't support any arguments (" + toString(args.engine_args.size()) + " given)", ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH); - return StorageSet::create( - args.data_path, args.table_name, args.columns, - args.materialized_columns, args.alias_columns, args.column_defaults); + return StorageSet::create(args.data_path, args.table_name, args.columns); }); } diff --git a/dbms/src/Storages/StorageSet.h b/dbms/src/Storages/StorageSet.h index 2546967b24a..d1e4ae89226 100644 --- a/dbms/src/Storages/StorageSet.h +++ b/dbms/src/Storages/StorageSet.h @@ -19,7 +19,7 @@ class StorageSetOrJoinBase : public IStorage friend class SetOrJoinBlockOutputStream; public: - String getTableName() const override { return name; } + String getTableName() const override { return table_name; } void rename(const String & new_path_to_db, const String & new_database_name, const String & new_table_name) override; @@ -30,14 +30,11 @@ public: protected: StorageSetOrJoinBase( const String & path_, - const String & name_, - const NamesAndTypesList & columns_, - const NamesAndTypesList & materialized_columns_, - const NamesAndTypesList & alias_columns_, - const ColumnDefaults & column_defaults_); + const String & table_name_, + const ColumnsDescription & columns_); String path; - String name; + String table_name; UInt64 increment = 0; /// For the backup file names. @@ -78,10 +75,7 @@ protected: StorageSet( const String & path_, const String & name_, - const NamesAndTypesList & columns_, - const NamesAndTypesList & materialized_columns_, - const NamesAndTypesList & alias_columns_, - const ColumnDefaults & column_defaults_); + const ColumnsDescription & columns_); }; } diff --git a/dbms/src/Storages/StorageStripeLog.cpp b/dbms/src/Storages/StorageStripeLog.cpp index 5f746dc246c..1bebca62322 100644 --- a/dbms/src/Storages/StorageStripeLog.cpp +++ b/dbms/src/Storages/StorageStripeLog.cpp @@ -193,13 +193,10 @@ private: StorageStripeLog::StorageStripeLog( const std::string & path_, const std::string & name_, - const NamesAndTypesList & columns_, - const NamesAndTypesList & materialized_columns_, - const NamesAndTypesList & alias_columns_, - const ColumnDefaults & column_defaults_, + const ColumnsDescription & columns_, bool attach, size_t max_compress_block_size_) - : IStorage{columns_, materialized_columns_, alias_columns_, column_defaults_}, + : IStorage{columns_}, path(path_), name(name_), max_compress_block_size(max_compress_block_size_), file_checker(path + escapeForFileName(name) + '/' + "sizes.json"), @@ -301,7 +298,6 @@ void registerStorageStripeLog(StorageFactory & factory) return StorageStripeLog::create( args.data_path, args.table_name, args.columns, - args.materialized_columns, args.alias_columns, args.column_defaults, args.attach, args.context.getSettings().max_compress_block_size); }); } diff --git a/dbms/src/Storages/StorageStripeLog.h b/dbms/src/Storages/StorageStripeLog.h index e1514f8329f..115f070f8d0 100644 --- a/dbms/src/Storages/StorageStripeLog.h +++ b/dbms/src/Storages/StorageStripeLog.h @@ -68,10 +68,7 @@ protected: StorageStripeLog( const std::string & path_, const std::string & name_, - const NamesAndTypesList & columns_, - const NamesAndTypesList & materialized_columns_, - const NamesAndTypesList & alias_columns_, - const ColumnDefaults & column_defaults_, + const ColumnsDescription & columns_, bool attach, size_t max_compress_block_size_); }; diff --git a/dbms/src/Storages/StorageTinyLog.cpp b/dbms/src/Storages/StorageTinyLog.cpp index 4ed7d6dfff2..039f470c366 100644 --- a/dbms/src/Storages/StorageTinyLog.cpp +++ b/dbms/src/Storages/StorageTinyLog.cpp @@ -276,13 +276,10 @@ void TinyLogBlockOutputStream::write(const Block & block) StorageTinyLog::StorageTinyLog( const std::string & path_, const std::string & name_, - const NamesAndTypesList & columns_, - const NamesAndTypesList & materialized_columns_, - const NamesAndTypesList & alias_columns_, - const ColumnDefaults & column_defaults_, + const ColumnsDescription & columns_, bool attach, size_t max_compress_block_size_) - : IStorage{columns_, materialized_columns_, alias_columns_, column_defaults_}, + : IStorage{columns_}, path(path_), name(name_), max_compress_block_size(max_compress_block_size_), file_checker(path + escapeForFileName(name) + '/' + "sizes.json"), @@ -299,7 +296,7 @@ StorageTinyLog::StorageTinyLog( throwFromErrno("Cannot create directory " + full_path, ErrorCodes::CANNOT_CREATE_DIRECTORY); } - for (const auto & col : getColumnsList()) + for (const auto & col : columns.getList()) addFiles(col.name, *col.type); } @@ -351,7 +348,7 @@ BlockInputStreams StorageTinyLog::read( check(column_names); processed_stage = QueryProcessingStage::FetchColumns; return BlockInputStreams(1, std::make_shared( - max_block_size, Nested::collect(getColumnsList().addTypes(column_names)), *this, context.getSettingsRef().max_read_buffer_size)); + max_block_size, Nested::collect(columns.getList().addTypes(column_names)), *this, context.getSettingsRef().max_read_buffer_size)); } @@ -379,7 +376,6 @@ void registerStorageTinyLog(StorageFactory & factory) return StorageTinyLog::create( args.data_path, args.table_name, args.columns, - args.materialized_columns, args.alias_columns, args.column_defaults, args.attach, args.context.getSettings().max_compress_block_size); }); } diff --git a/dbms/src/Storages/StorageTinyLog.h b/dbms/src/Storages/StorageTinyLog.h index e64301227ad..acaf7944f30 100644 --- a/dbms/src/Storages/StorageTinyLog.h +++ b/dbms/src/Storages/StorageTinyLog.h @@ -71,10 +71,7 @@ protected: StorageTinyLog( const std::string & path_, const std::string & name_, - const NamesAndTypesList & columns_, - const NamesAndTypesList & materialized_columns_, - const NamesAndTypesList & alias_columns_, - const ColumnDefaults & column_defaults_, + const ColumnsDescription & columns_, bool attach, size_t max_compress_block_size_); }; diff --git a/dbms/src/Storages/StorageView.cpp b/dbms/src/Storages/StorageView.cpp index 76cd481a855..06809ffb4e3 100644 --- a/dbms/src/Storages/StorageView.cpp +++ b/dbms/src/Storages/StorageView.cpp @@ -19,8 +19,8 @@ namespace ErrorCodes StorageView::StorageView( const String & table_name_, const ASTCreateQuery & query, - const NamesAndTypesList & columns_) - : IStorage{columns_, {}, {}, {}}, table_name(table_name_) + const ColumnsDescription & columns_) + : IStorage{columns_}, table_name(table_name_) { if (!query.select) throw Exception("SELECT query is not specified for " + getName(), ErrorCodes::INCORRECT_QUERY); diff --git a/dbms/src/Storages/StorageView.h b/dbms/src/Storages/StorageView.h index 6a16e0d16e6..20a8f76189c 100644 --- a/dbms/src/Storages/StorageView.h +++ b/dbms/src/Storages/StorageView.h @@ -42,7 +42,7 @@ protected: StorageView( const String & table_name_, const ASTCreateQuery & query, - const NamesAndTypesList & columns_); + const ColumnsDescription & columns_); }; } diff --git a/dbms/src/Storages/System/StorageSystemAsynchronousMetrics.cpp b/dbms/src/Storages/System/StorageSystemAsynchronousMetrics.cpp index 44f8d933c99..f08808f799a 100644 --- a/dbms/src/Storages/System/StorageSystemAsynchronousMetrics.cpp +++ b/dbms/src/Storages/System/StorageSystemAsynchronousMetrics.cpp @@ -16,7 +16,7 @@ StorageSystemAsynchronousMetrics::StorageSystemAsynchronousMetrics(const std::st : name(name_), async_metrics(async_metrics_) { - columns = NamesAndTypesList{ + columns.ordinary = NamesAndTypesList{ {"metric", std::make_shared()}, {"value", std::make_shared()}, }; diff --git a/dbms/src/Storages/System/StorageSystemBuildOptions.cpp b/dbms/src/Storages/System/StorageSystemBuildOptions.cpp index c8da9f12c40..7a864090e59 100644 --- a/dbms/src/Storages/System/StorageSystemBuildOptions.cpp +++ b/dbms/src/Storages/System/StorageSystemBuildOptions.cpp @@ -13,7 +13,7 @@ namespace DB StorageSystemBuildOptions::StorageSystemBuildOptions(const std::string & name_) : name(name_) { - columns = NamesAndTypesList{ + columns.ordinary = NamesAndTypesList{ { "name", std::make_shared() }, { "value", std::make_shared() }, }; diff --git a/dbms/src/Storages/System/StorageSystemClusters.cpp b/dbms/src/Storages/System/StorageSystemClusters.cpp index 084f295de74..8d772e5ebb8 100644 --- a/dbms/src/Storages/System/StorageSystemClusters.cpp +++ b/dbms/src/Storages/System/StorageSystemClusters.cpp @@ -14,7 +14,7 @@ namespace DB StorageSystemClusters::StorageSystemClusters(const std::string & name_) : name(name_) { - columns = NamesAndTypesList{ + columns.ordinary = NamesAndTypesList{ { "cluster", std::make_shared() }, { "shard_num", std::make_shared() }, { "shard_weight", std::make_shared() }, diff --git a/dbms/src/Storages/System/StorageSystemColumns.cpp b/dbms/src/Storages/System/StorageSystemColumns.cpp index 5f9116b08a5..adec6a70c58 100644 --- a/dbms/src/Storages/System/StorageSystemColumns.cpp +++ b/dbms/src/Storages/System/StorageSystemColumns.cpp @@ -18,7 +18,7 @@ namespace DB StorageSystemColumns::StorageSystemColumns(const std::string & name_) : name(name_) { - columns = NamesAndTypesList{ + columns.ordinary = NamesAndTypesList{ { "database", std::make_shared() }, { "table", std::make_shared() }, { "name", std::make_shared() }, @@ -137,9 +137,9 @@ BlockInputStreams StorageSystemColumns::read( throw; } - columns = storage->getColumnsList(); - columns.insert(std::end(columns), std::begin(storage->alias_columns), std::end(storage->alias_columns)); - column_defaults = storage->column_defaults; + columns = storage->columns.getList(); + columns.insert(std::end(columns), std::begin(storage->columns.aliases), std::end(storage->columns.aliases)); + column_defaults = storage->columns.defaults; /** Info about sizes of columns for tables of MergeTree family. * NOTE: It is possible to add getter for this info to IStorage interface. diff --git a/dbms/src/Storages/System/StorageSystemDatabases.cpp b/dbms/src/Storages/System/StorageSystemDatabases.cpp index 93917b81000..12c35c7ec9e 100644 --- a/dbms/src/Storages/System/StorageSystemDatabases.cpp +++ b/dbms/src/Storages/System/StorageSystemDatabases.cpp @@ -13,7 +13,7 @@ namespace DB StorageSystemDatabases::StorageSystemDatabases(const std::string & name_) : name(name_) { - columns = NamesAndTypesList{ + columns.ordinary = NamesAndTypesList{ {"name", std::make_shared()}, {"engine", std::make_shared()}, {"data_path", std::make_shared()}, diff --git a/dbms/src/Storages/System/StorageSystemDictionaries.cpp b/dbms/src/Storages/System/StorageSystemDictionaries.cpp index 93c742ef1a7..3cb4bd7962b 100644 --- a/dbms/src/Storages/System/StorageSystemDictionaries.cpp +++ b/dbms/src/Storages/System/StorageSystemDictionaries.cpp @@ -21,7 +21,7 @@ namespace DB StorageSystemDictionaries::StorageSystemDictionaries(const std::string & name) : name{name} { - columns = NamesAndTypesList{ + columns.ordinary = NamesAndTypesList{ { "name", std::make_shared() }, { "origin", std::make_shared() }, { "type", std::make_shared() }, diff --git a/dbms/src/Storages/System/StorageSystemEvents.cpp b/dbms/src/Storages/System/StorageSystemEvents.cpp index 96a02d72332..0838eb739a5 100644 --- a/dbms/src/Storages/System/StorageSystemEvents.cpp +++ b/dbms/src/Storages/System/StorageSystemEvents.cpp @@ -13,7 +13,7 @@ namespace DB StorageSystemEvents::StorageSystemEvents(const std::string & name_) : name(name_) { - columns = NamesAndTypesList + columns.ordinary = NamesAndTypesList { {"event", std::make_shared()}, {"value", std::make_shared()} diff --git a/dbms/src/Storages/System/StorageSystemFunctions.cpp b/dbms/src/Storages/System/StorageSystemFunctions.cpp index 74e58338d40..4d013eb2268 100644 --- a/dbms/src/Storages/System/StorageSystemFunctions.cpp +++ b/dbms/src/Storages/System/StorageSystemFunctions.cpp @@ -16,7 +16,7 @@ namespace DB StorageSystemFunctions::StorageSystemFunctions(const std::string & name_) : name(name_) { - columns = NamesAndTypesList{ + columns.ordinary = NamesAndTypesList{ { "name", std::make_shared() }, { "is_aggregate", std::make_shared() } }; diff --git a/dbms/src/Storages/System/StorageSystemGraphite.cpp b/dbms/src/Storages/System/StorageSystemGraphite.cpp index 76f3defe885..e99c037ea78 100644 --- a/dbms/src/Storages/System/StorageSystemGraphite.cpp +++ b/dbms/src/Storages/System/StorageSystemGraphite.cpp @@ -125,7 +125,7 @@ static Strings getAllGraphiteSections(const AbstractConfiguration & config) StorageSystemGraphite::StorageSystemGraphite(const std::string & name_) : name(name_) { - columns = NamesAndTypesList{ + columns.ordinary = NamesAndTypesList{ {"config_name", std::make_shared()}, {"regexp", std::make_shared()}, {"function", std::make_shared()}, diff --git a/dbms/src/Storages/System/StorageSystemMerges.cpp b/dbms/src/Storages/System/StorageSystemMerges.cpp index f6667c19908..fea37df35a1 100644 --- a/dbms/src/Storages/System/StorageSystemMerges.cpp +++ b/dbms/src/Storages/System/StorageSystemMerges.cpp @@ -14,7 +14,7 @@ namespace DB StorageSystemMerges::StorageSystemMerges(const std::string & name) : name{name} { - columns = NamesAndTypesList{ + columns.ordinary = NamesAndTypesList{ { "database", std::make_shared() }, { "table", std::make_shared() }, { "elapsed", std::make_shared() }, diff --git a/dbms/src/Storages/System/StorageSystemMetrics.cpp b/dbms/src/Storages/System/StorageSystemMetrics.cpp index c60d7f4b4f5..54002318ef6 100644 --- a/dbms/src/Storages/System/StorageSystemMetrics.cpp +++ b/dbms/src/Storages/System/StorageSystemMetrics.cpp @@ -14,7 +14,7 @@ namespace DB StorageSystemMetrics::StorageSystemMetrics(const std::string & name_) : name(name_) { - columns = NamesAndTypesList{ + columns.ordinary = NamesAndTypesList{ {"metric", std::make_shared()}, {"value", std::make_shared()}, }; diff --git a/dbms/src/Storages/System/StorageSystemModels.cpp b/dbms/src/Storages/System/StorageSystemModels.cpp index da0249b1705..9057c6de626 100644 --- a/dbms/src/Storages/System/StorageSystemModels.cpp +++ b/dbms/src/Storages/System/StorageSystemModels.cpp @@ -14,7 +14,7 @@ namespace DB StorageSystemModels::StorageSystemModels(const std::string & name) : name{name} { - columns = NamesAndTypesList{ + columns.ordinary = NamesAndTypesList{ { "name", std::make_shared() }, { "origin", std::make_shared() }, { "type", std::make_shared() }, diff --git a/dbms/src/Storages/System/StorageSystemNumbers.cpp b/dbms/src/Storages/System/StorageSystemNumbers.cpp index 30586bbbba3..803f651639d 100644 --- a/dbms/src/Storages/System/StorageSystemNumbers.cpp +++ b/dbms/src/Storages/System/StorageSystemNumbers.cpp @@ -47,7 +47,7 @@ private: StorageSystemNumbers::StorageSystemNumbers(const std::string & name_, bool multithreaded_, size_t limit_) : name(name_), multithreaded(multithreaded_), limit(limit_) { - columns = NamesAndTypesList{{"number", std::make_shared()}}; + columns.ordinary = NamesAndTypesList{{"number", std::make_shared()}}; } diff --git a/dbms/src/Storages/System/StorageSystemOne.cpp b/dbms/src/Storages/System/StorageSystemOne.cpp index 4ddc64031e3..f35580c3882 100644 --- a/dbms/src/Storages/System/StorageSystemOne.cpp +++ b/dbms/src/Storages/System/StorageSystemOne.cpp @@ -13,7 +13,7 @@ namespace DB StorageSystemOne::StorageSystemOne(const std::string & name_) : name(name_) { - columns = NamesAndTypesList{{"dummy", std::make_shared()}}; + columns.ordinary = NamesAndTypesList{{"dummy", std::make_shared()}}; } diff --git a/dbms/src/Storages/System/StorageSystemPartsBase.cpp b/dbms/src/Storages/System/StorageSystemPartsBase.cpp index f1f99c312f4..c6dc2a2c551 100644 --- a/dbms/src/Storages/System/StorageSystemPartsBase.cpp +++ b/dbms/src/Storages/System/StorageSystemPartsBase.cpp @@ -245,20 +245,20 @@ BlockInputStreams StorageSystemPartsBase::read( /// Create the result. - MutableColumns columns = getSampleBlock().cloneEmptyColumns(); + MutableColumns res_columns = getSampleBlock().cloneEmptyColumns(); if (has_state_column) - columns.push_back(ColumnString::create()); + res_columns.push_back(ColumnString::create()); while (StoragesInfo info = stream.next()) { - processNextStorage(columns, info, has_state_column); + processNextStorage(res_columns, info, has_state_column); } Block block = getSampleBlock(); if (has_state_column) block.insert(ColumnWithTypeAndName(std::make_shared(), "_state")); - return BlockInputStreams(1, std::make_shared(block.cloneWithColumns(std::move(columns)))); + return BlockInputStreams(1, std::make_shared(block.cloneWithColumns(std::move(res_columns)))); } NameAndTypePair StorageSystemPartsBase::getColumn(const String & column_name) const @@ -280,7 +280,7 @@ bool StorageSystemPartsBase::hasColumn(const String & column_name) const StorageSystemPartsBase::StorageSystemPartsBase(std::string name_, NamesAndTypesList && columns_) : name(std::move(name_)) { - columns = columns_; + columns.ordinary = columns_; } } diff --git a/dbms/src/Storages/System/StorageSystemPartsColumns.cpp b/dbms/src/Storages/System/StorageSystemPartsColumns.cpp index 9667399c534..5f0a584c1d6 100644 --- a/dbms/src/Storages/System/StorageSystemPartsColumns.cpp +++ b/dbms/src/Storages/System/StorageSystemPartsColumns.cpp @@ -61,9 +61,9 @@ void StorageSystemPartsColumns::processNextStorage(MutableColumns & columns, con String default_expression; }; - NamesAndTypesList columns_list = info.storage->getColumnsList(); - columns_list.insert(std::end(columns_list), std::begin(info.storage->alias_columns), std::end(info.storage->alias_columns)); - const auto & column_defaults = info.storage->column_defaults; + NamesAndTypesList columns_list = info.storage->columns.getList(); + columns_list.insert(std::end(columns_list), std::begin(info.storage->columns.aliases), std::end(info.storage->columns.aliases)); + const auto & column_defaults = info.storage->columns.defaults; std::unordered_map columns_info; for (const auto & column : columns_list) diff --git a/dbms/src/Storages/System/StorageSystemProcesses.cpp b/dbms/src/Storages/System/StorageSystemProcesses.cpp index 55f3bb120db..e8262723d89 100644 --- a/dbms/src/Storages/System/StorageSystemProcesses.cpp +++ b/dbms/src/Storages/System/StorageSystemProcesses.cpp @@ -14,7 +14,7 @@ namespace DB StorageSystemProcesses::StorageSystemProcesses(const std::string & name_) : name(name_) { - columns = NamesAndTypesList{ + columns.ordinary = NamesAndTypesList{ { "is_initial_query", std::make_shared() }, { "user", std::make_shared() }, diff --git a/dbms/src/Storages/System/StorageSystemReplicas.cpp b/dbms/src/Storages/System/StorageSystemReplicas.cpp index 6dc11214b7c..ca240a80568 100644 --- a/dbms/src/Storages/System/StorageSystemReplicas.cpp +++ b/dbms/src/Storages/System/StorageSystemReplicas.cpp @@ -17,7 +17,7 @@ namespace DB StorageSystemReplicas::StorageSystemReplicas(const std::string & name_) : name(name_) { - columns = NamesAndTypesList{ + columns.ordinary = NamesAndTypesList{ { "database", std::make_shared() }, { "table", std::make_shared() }, { "engine", std::make_shared() }, diff --git a/dbms/src/Storages/System/StorageSystemReplicationQueue.cpp b/dbms/src/Storages/System/StorageSystemReplicationQueue.cpp index a2432525044..363306437bb 100644 --- a/dbms/src/Storages/System/StorageSystemReplicationQueue.cpp +++ b/dbms/src/Storages/System/StorageSystemReplicationQueue.cpp @@ -20,7 +20,7 @@ namespace DB StorageSystemReplicationQueue::StorageSystemReplicationQueue(const std::string & name_) : name(name_) { - columns = NamesAndTypesList{ + columns.ordinary = NamesAndTypesList{ /// Table properties. { "database", std::make_shared() }, { "table", std::make_shared() }, diff --git a/dbms/src/Storages/System/StorageSystemSettings.cpp b/dbms/src/Storages/System/StorageSystemSettings.cpp index 6099883eeb6..3848583688b 100644 --- a/dbms/src/Storages/System/StorageSystemSettings.cpp +++ b/dbms/src/Storages/System/StorageSystemSettings.cpp @@ -14,7 +14,7 @@ namespace DB StorageSystemSettings::StorageSystemSettings(const std::string & name_) : name(name_) { - columns = NamesAndTypesList{ + columns.ordinary = NamesAndTypesList{ { "name", std::make_shared() }, { "value", std::make_shared() }, { "changed", std::make_shared() }, diff --git a/dbms/src/Storages/System/StorageSystemTables.cpp b/dbms/src/Storages/System/StorageSystemTables.cpp index 852392a63d1..91ab3581bc8 100644 --- a/dbms/src/Storages/System/StorageSystemTables.cpp +++ b/dbms/src/Storages/System/StorageSystemTables.cpp @@ -114,7 +114,7 @@ protected: StorageSystemTables::StorageSystemTables(const std::string & name_) : name(name_) { - columns = NamesAndTypesList + columns.ordinary = NamesAndTypesList { {"database", std::make_shared()}, {"name", std::make_shared()}, diff --git a/dbms/src/Storages/System/StorageSystemZooKeeper.cpp b/dbms/src/Storages/System/StorageSystemZooKeeper.cpp index 80ca3bc2f0e..9aae0ce5b3b 100644 --- a/dbms/src/Storages/System/StorageSystemZooKeeper.cpp +++ b/dbms/src/Storages/System/StorageSystemZooKeeper.cpp @@ -22,7 +22,7 @@ namespace DB StorageSystemZooKeeper::StorageSystemZooKeeper(const std::string & name_) : name(name_) { - columns = NamesAndTypesList{ + columns.ordinary = NamesAndTypesList{ { "name", std::make_shared() }, { "value", std::make_shared() }, { "czxid", std::make_shared() }, diff --git a/dbms/src/Storages/tests/storage_log.cpp b/dbms/src/Storages/tests/storage_log.cpp index 6d9cb5d0def..da53fcc356d 100644 --- a/dbms/src/Storages/tests/storage_log.cpp +++ b/dbms/src/Storages/tests/storage_log.cpp @@ -36,7 +36,7 @@ try { ColumnWithTypeAndName column; column.name = "a"; - column.type = table->getDataTypeByName("a"); + column.type = table->columns.getColumn("a").type; auto col = column.type->createColumn(); ColumnUInt64::Container & vec = typeid_cast(*col).getData(); @@ -51,7 +51,7 @@ try { ColumnWithTypeAndName column; column.name = "b"; - column.type = table->getDataTypeByName("b"); + column.type = table->columns.getColumn("b").type; auto col = column.type->createColumn(); ColumnUInt8::Container & vec = typeid_cast(*col).getData(); diff --git a/dbms/src/TableFunctions/TableFunctionMerge.cpp b/dbms/src/TableFunctions/TableFunctionMerge.cpp index d37d2002d5a..b471969b847 100644 --- a/dbms/src/TableFunctions/TableFunctionMerge.cpp +++ b/dbms/src/TableFunctions/TableFunctionMerge.cpp @@ -51,7 +51,7 @@ static NamesAndTypesList chooseColumns(const String & source_database, const Str throw Exception("Error while executing table function merge. In database " + source_database + " no one matches regular expression: " + table_name_regexp_, ErrorCodes::UNKNOWN_TABLE); - return any_table->getColumnsList(); + return any_table->columns.getList(); } @@ -79,10 +79,12 @@ StoragePtr TableFunctionMerge::executeImpl(const ASTPtr & ast_function, const Co auto res = StorageMerge::create( getName(), - chooseColumns(source_database, table_name_regexp, context), - NamesAndTypesList{}, - NamesAndTypesList{}, - ColumnDefaults{}, + ColumnsDescription{ + chooseColumns(source_database, table_name_regexp, context), + NamesAndTypesList{}, + NamesAndTypesList{}, + ColumnDefaults{}, + }, source_database, table_name_regexp, context); diff --git a/dbms/src/TableFunctions/TableFunctionMySQL.cpp b/dbms/src/TableFunctions/TableFunctionMySQL.cpp index f1b4e911385..1ced8800255 100644 --- a/dbms/src/TableFunctions/TableFunctionMySQL.cpp +++ b/dbms/src/TableFunctions/TableFunctionMySQL.cpp @@ -152,7 +152,7 @@ StoragePtr TableFunctionMySQL::executeImpl(const ASTPtr & ast_function, const Co std::move(pool), database_name, table_name, - columns); + ColumnsDescription{columns, NamesAndTypesList{}, NamesAndTypesList{}, ColumnDefaults{}}); res->startup(); return res; diff --git a/dbms/src/TableFunctions/TableFunctionODBC.cpp b/dbms/src/TableFunctions/TableFunctionODBC.cpp index cc4c430ae8f..cacdbe29670 100644 --- a/dbms/src/TableFunctions/TableFunctionODBC.cpp +++ b/dbms/src/TableFunctions/TableFunctionODBC.cpp @@ -113,7 +113,8 @@ StoragePtr TableFunctionODBC::executeImpl(const ASTPtr & ast_function, const Con columns.emplace_back(reinterpret_cast(column_name), getDataType(type)); } - auto result = StorageODBC::create(table_name, connection_string, "", table_name, columns); + auto result = StorageODBC::create(table_name, connection_string, "", table_name, + ColumnsDescription{columns, NamesAndTypesList{}, NamesAndTypesList{}, ColumnDefaults{}}); result->startup(); return result; } diff --git a/dbms/src/TableFunctions/getStructureOfRemoteTable.cpp b/dbms/src/TableFunctions/getStructureOfRemoteTable.cpp index 3cb68d25416..181448714ba 100644 --- a/dbms/src/TableFunctions/getStructureOfRemoteTable.cpp +++ b/dbms/src/TableFunctions/getStructureOfRemoteTable.cpp @@ -32,7 +32,7 @@ NamesAndTypesList getStructureOfRemoteTable( const auto & shard_info = cluster.getAnyShardInfo(); if (shard_info.isLocal()) - return context.getTable(database, table)->getColumnsList(); + return context.getTable(database, table)->columns.getList(); auto input = std::make_shared(shard_info.pool, query, InterpreterDescribeQuery::getSampleBlock(), context); input->setPoolMode(PoolMode::GET_ONE); From 972db7b0723db78f45554acd2cc5f7d195ae7c70 Mon Sep 17 00:00:00 2001 From: Alexey Zatelepin Date: Sat, 10 Mar 2018 23:37:26 +0300 Subject: [PATCH 2/8] iterate over materialized columns where it makes sense --- dbms/src/Storages/MergeTree/MergeTreeData.cpp | 4 ++-- dbms/src/Storages/StorageDistributed.cpp | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/dbms/src/Storages/MergeTree/MergeTreeData.cpp b/dbms/src/Storages/MergeTree/MergeTreeData.cpp index 7933a9454c7..ea36f75a96c 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeData.cpp +++ b/dbms/src/Storages/MergeTree/MergeTreeData.cpp @@ -903,7 +903,7 @@ void MergeTreeData::checkAlter(const AlterCommands & commands) columns_alter_forbidden.insert(merging_params.sign_column); std::map old_types; - for (const auto & column : columns.ordinary) + for (const auto & column : columns.getList()) old_types.emplace(column.name, column.type.get()); for (const AlterCommand & command : commands) @@ -1868,7 +1868,7 @@ void MergeTreeData::removePartContributionToColumnSizes(const DataPartPtr & part const auto & files = part->checksums.files; /// TODO This method doesn't take into account columns with multiple files. - for (const auto & column : columns.ordinary) + for (const auto & column : columns.getList()) { const auto escaped_name = escapeForFileName(column.name); const auto bin_file_name = escaped_name + ".bin"; diff --git a/dbms/src/Storages/StorageDistributed.cpp b/dbms/src/Storages/StorageDistributed.cpp index 1b7c5fdf69a..b543e802ef9 100644 --- a/dbms/src/Storages/StorageDistributed.cpp +++ b/dbms/src/Storages/StorageDistributed.cpp @@ -142,7 +142,7 @@ StorageDistributed::StorageDistributed( table_name(table_name_), remote_database(remote_database_), remote_table(remote_table_), context(context_), cluster_name(context.getMacros().expand(cluster_name_)), has_sharding_key(sharding_key_), - sharding_key_expr(sharding_key_ ? ExpressionAnalyzer(sharding_key_, context, nullptr, columns.ordinary).getActions(false) : nullptr), + sharding_key_expr(sharding_key_ ? ExpressionAnalyzer(sharding_key_, context, nullptr, columns.getList()).getActions(false) : nullptr), sharding_key_column_name(sharding_key_ ? sharding_key_->getColumnName() : String{}), path(data_path_.empty() ? "" : (data_path_ + escapeForFileName(table_name) + '/')) { @@ -385,7 +385,7 @@ void registerStorageDistributed(StorageFactory & factory) /// Check that sharding_key exists in the table and has numeric type. if (sharding_key) { - auto sharding_expr = ExpressionAnalyzer(sharding_key, args.context, nullptr, args.columns.ordinary).getActions(true); + auto sharding_expr = ExpressionAnalyzer(sharding_key, args.context, nullptr, args.columns.getList()).getActions(true); const Block & block = sharding_expr->getSampleBlock(); if (block.columns() != 1) From 78d92f67945db374033f13214ed3c62b27ac07d7 Mon Sep 17 00:00:00 2001 From: Alexey Zatelepin Date: Sat, 10 Mar 2018 20:03:57 +0300 Subject: [PATCH 3/8] add some useful methods, rename for clarity [#CLICKHOUSE-3241] --- dbms/src/Analyzers/ExecuteTableFunctions.cpp | 2 +- dbms/src/Common/ExternalTable.h | 4 +-- dbms/src/Core/NamesAndTypes.cpp | 10 ++++++ dbms/src/Core/NamesAndTypes.h | 2 ++ dbms/src/Core/iostream_debug_helpers.cpp | 2 +- .../tests/filter_stream_hitlog.cpp | 4 +-- dbms/src/DataStreams/tests/native_streams.cpp | 4 +-- dbms/src/DataStreams/tests/sorting_stream.cpp | 4 +-- dbms/src/Databases/DatabaseDictionary.cpp | 6 ++-- dbms/src/Interpreters/ExpressionAnalyzer.cpp | 10 +++--- .../Interpreters/InterpreterCreateQuery.cpp | 2 +- .../Interpreters/InterpreterDescribeQuery.cpp | 3 +- .../Interpreters/InterpreterInsertQuery.cpp | 2 +- .../tests/in_join_subqueries_preprocessor.cpp | 2 -- dbms/src/Storages/AlterCommands.cpp | 13 ++------ dbms/src/Storages/ColumnsDescription.cpp | 8 ++++- dbms/src/Storages/ColumnsDescription.h | 32 +++++++++++++------ dbms/src/Storages/ITableDeclaration.cpp | 8 ++--- .../MergeTree/MergeTreeBlockInputStream.cpp | 4 +-- dbms/src/Storages/MergeTree/MergeTreeData.cpp | 20 ++++++------ .../MergeTree/MergeTreeDataMerger.cpp | 2 +- .../Storages/MergeTree/MergeTreeDataPart.cpp | 6 ++-- .../MergeTree/MergeTreeDataSelectExecutor.cpp | 4 +-- .../MergeTree/MergeTreeDataWriter.cpp | 2 +- .../Storages/MergeTree/MergeTreeReadPool.cpp | 4 +-- .../MergeTree/MergeTreeWhereOptimizer.cpp | 4 +-- .../ReplicatedMergeTreeAlterThread.cpp | 4 +-- dbms/src/Storages/StorageDistributed.cpp | 6 ++-- dbms/src/Storages/StorageFactory.cpp | 4 +-- dbms/src/Storages/StorageLog.cpp | 4 +-- dbms/src/Storages/StorageMerge.cpp | 2 +- dbms/src/Storages/StorageMergeTree.cpp | 4 +-- .../Storages/StorageReplicatedMergeTree.cpp | 2 +- dbms/src/Storages/StorageTinyLog.cpp | 4 +-- .../StorageSystemAsynchronousMetrics.cpp | 4 +-- .../System/StorageSystemBuildOptions.cpp | 4 +-- .../Storages/System/StorageSystemClusters.cpp | 6 ++-- .../Storages/System/StorageSystemColumns.cpp | 7 ++-- .../System/StorageSystemDatabases.cpp | 4 +-- .../System/StorageSystemDictionaries.cpp | 6 ++-- .../Storages/System/StorageSystemEvents.cpp | 6 ++-- .../System/StorageSystemFunctions.cpp | 6 ++-- .../Storages/System/StorageSystemGraphite.cpp | 6 ++-- .../Storages/System/StorageSystemMerges.cpp | 4 +-- .../Storages/System/StorageSystemMetrics.cpp | 4 +-- .../Storages/System/StorageSystemModels.cpp | 6 ++-- .../Storages/System/StorageSystemNumbers.cpp | 2 +- dbms/src/Storages/System/StorageSystemOne.cpp | 2 +- .../System/StorageSystemPartsBase.cpp | 2 +- .../System/StorageSystemPartsColumns.cpp | 3 +- .../System/StorageSystemProcesses.cpp | 6 ++-- .../Storages/System/StorageSystemReplicas.cpp | 4 +-- .../System/StorageSystemReplicationQueue.cpp | 4 +-- .../Storages/System/StorageSystemSettings.cpp | 6 ++-- .../Storages/System/StorageSystemTables.cpp | 4 +-- .../System/StorageSystemZooKeeper.cpp | 4 +-- dbms/src/Storages/tests/hit_log.cpp | 4 +-- dbms/src/Storages/tests/storage_log.cpp | 8 ++--- .../src/TableFunctions/TableFunctionMerge.cpp | 9 ++---- .../src/TableFunctions/TableFunctionMySQL.cpp | 2 +- dbms/src/TableFunctions/TableFunctionODBC.cpp | 3 +- .../getStructureOfRemoteTable.cpp | 2 +- 62 files changed, 162 insertions(+), 160 deletions(-) diff --git a/dbms/src/Analyzers/ExecuteTableFunctions.cpp b/dbms/src/Analyzers/ExecuteTableFunctions.cpp index 326c5df62ed..3feabc8f490 100644 --- a/dbms/src/Analyzers/ExecuteTableFunctions.cpp +++ b/dbms/src/Analyzers/ExecuteTableFunctions.cpp @@ -72,7 +72,7 @@ void ExecuteTableFunctions::dump(WriteBuffer & out) const { writeString(table.second->getName(), out); writeCString("\n\n", out); - writeString(table.second->columns.getList().toString(), out); + writeString(table.second->columns.getPhysical().toString(), out); writeCString("\n", out); } } diff --git a/dbms/src/Common/ExternalTable.h b/dbms/src/Common/ExternalTable.h index b5b0c522339..7cc8069108b 100644 --- a/dbms/src/Common/ExternalTable.h +++ b/dbms/src/Common/ExternalTable.h @@ -198,9 +198,7 @@ public: /// Create table NamesAndTypesList columns = sample_block.getNamesAndTypesList(); - StoragePtr storage = StorageMemory::create( - data.second, - ColumnsDescription{columns, NamesAndTypesList{}, NamesAndTypesList{}, ColumnDefaults{}}); + StoragePtr storage = StorageMemory::create(data.second, ColumnsDescription{columns}); storage->startup(); context.addExternalTable(data.second, storage); BlockOutputStreamPtr output = storage->write(ASTPtr(), context.getSettingsRef()); diff --git a/dbms/src/Core/NamesAndTypes.cpp b/dbms/src/Core/NamesAndTypes.cpp index 14faaa133ea..a5240ba4a91 100644 --- a/dbms/src/Core/NamesAndTypes.cpp +++ b/dbms/src/Core/NamesAndTypes.cpp @@ -138,4 +138,14 @@ NamesAndTypesList NamesAndTypesList::addTypes(const Names & names) const return res; } +bool NamesAndTypesList::contains(const String & name) const +{ + for (const NameAndTypePair & column : *this) + { + if (column.name == name) + return true; + } + return false; +} + } diff --git a/dbms/src/Core/NamesAndTypes.h b/dbms/src/Core/NamesAndTypes.h index 7aa9f3b3ff4..849141e43f7 100644 --- a/dbms/src/Core/NamesAndTypes.h +++ b/dbms/src/Core/NamesAndTypes.h @@ -69,6 +69,8 @@ public: /// Unlike `filter`, returns columns in the order in which they go in `names`. NamesAndTypesList addTypes(const Names & names) const; + + bool contains(const String & name) const; }; } diff --git a/dbms/src/Core/iostream_debug_helpers.cpp b/dbms/src/Core/iostream_debug_helpers.cpp index 822bbf58376..723b3dc2e90 100644 --- a/dbms/src/Core/iostream_debug_helpers.cpp +++ b/dbms/src/Core/iostream_debug_helpers.cpp @@ -44,7 +44,7 @@ std::ostream & operator<<(std::ostream & stream, const IDataType & what) std::ostream & operator<<(std::ostream & stream, const IStorage & what) { stream << "IStorage(name = " << what.getName() << ", tableName = " << what.getTableName() << ") {" - << what.columns.getList().toString() + << what.columns.getPhysical().toString() << "}"; // isRemote supportsSampling supportsFinal supportsPrewhere return stream; diff --git a/dbms/src/DataStreams/tests/filter_stream_hitlog.cpp b/dbms/src/DataStreams/tests/filter_stream_hitlog.cpp index fe9ce592a43..2eb65c30df3 100644 --- a/dbms/src/DataStreams/tests/filter_stream_hitlog.cpp +++ b/dbms/src/DataStreams/tests/filter_stream_hitlog.cpp @@ -104,8 +104,8 @@ int main(int, char **) /// create an object of an existing hit log table - StoragePtr table = StorageLog::create("./", "HitLog", names_and_types_list, - NamesAndTypesList{}, NamesAndTypesList{}, ColumnDefaults{}, DEFAULT_MAX_COMPRESS_BLOCK_SIZE); + StoragePtr table = StorageLog::create( + "./", "HitLog", ColumnsDescription{names_and_types_list}, DEFAULT_MAX_COMPRESS_BLOCK_SIZE); table->startup(); /// read from it, apply the expression, filter, and write in tsv form to the console diff --git a/dbms/src/DataStreams/tests/native_streams.cpp b/dbms/src/DataStreams/tests/native_streams.cpp index bd0a998f88e..5d02d491ec4 100644 --- a/dbms/src/DataStreams/tests/native_streams.cpp +++ b/dbms/src/DataStreams/tests/native_streams.cpp @@ -95,8 +95,8 @@ try /// create an object of an existing hit log table - StoragePtr table = StorageLog::create("./", "HitLog", names_and_types_list, - NamesAndTypesList{}, NamesAndTypesList{}, ColumnDefaults{}, DEFAULT_MAX_COMPRESS_BLOCK_SIZE); + StoragePtr table = StorageLog::create( + "./", "HitLog", ColumnsDescription{names_and_types_list}, DEFAULT_MAX_COMPRESS_BLOCK_SIZE); table->startup(); /// read from it diff --git a/dbms/src/DataStreams/tests/sorting_stream.cpp b/dbms/src/DataStreams/tests/sorting_stream.cpp index cd6fe515a53..0cfffec295f 100644 --- a/dbms/src/DataStreams/tests/sorting_stream.cpp +++ b/dbms/src/DataStreams/tests/sorting_stream.cpp @@ -107,8 +107,8 @@ try /// create an object of an existing hit log table - StoragePtr table = StorageLog::create("./", "HitLog", names_and_types_list, - NamesAndTypesList{}, NamesAndTypesList{}, ColumnDefaults{}, DEFAULT_MAX_COMPRESS_BLOCK_SIZE); + StoragePtr table = StorageLog::create( + "./", "HitLog", ColumnsDescription{names_and_types_list}, DEFAULT_MAX_COMPRESS_BLOCK_SIZE); table->startup(); /// read from it, sort it, and write it in tsv form to the console diff --git a/dbms/src/Databases/DatabaseDictionary.cpp b/dbms/src/Databases/DatabaseDictionary.cpp index 439fe8f4d79..503d0195168 100644 --- a/dbms/src/Databases/DatabaseDictionary.cpp +++ b/dbms/src/Databases/DatabaseDictionary.cpp @@ -41,8 +41,7 @@ Tables DatabaseDictionary::loadTables() { const DictionaryStructure & dictionary_structure = dict_ptr->getStructure(); auto columns = StorageDictionary::getNamesAndTypes(dictionary_structure); - tables[name] = StorageDictionary::create(name, - ColumnsDescription{columns, NamesAndTypesList{}, NamesAndTypesList{}, ColumnDefaults{}}, dictionary_structure, name); + tables[name] = StorageDictionary::create(name, ColumnsDescription{columns}, dictionary_structure, name); } } @@ -76,8 +75,7 @@ StoragePtr DatabaseDictionary::tryGetTable( { const DictionaryStructure & dictionary_structure = dict_ptr->getStructure(); auto columns = StorageDictionary::getNamesAndTypes(dictionary_structure); - return StorageDictionary::create(table_name, - ColumnsDescription{columns, NamesAndTypesList{}, NamesAndTypesList{}, ColumnDefaults{}}, dictionary_structure, table_name); + return StorageDictionary::create(table_name, ColumnsDescription{columns}, dictionary_structure, table_name); } } } diff --git a/dbms/src/Interpreters/ExpressionAnalyzer.cpp b/dbms/src/Interpreters/ExpressionAnalyzer.cpp index fe85e154e92..e6190b8f1df 100644 --- a/dbms/src/Interpreters/ExpressionAnalyzer.cpp +++ b/dbms/src/Interpreters/ExpressionAnalyzer.cpp @@ -190,7 +190,7 @@ ExpressionAnalyzer::ExpressionAnalyzer( } if (storage && source_columns.empty()) - source_columns = storage->columns.getList(); + source_columns = storage->columns.getPhysical(); else removeDuplicateColumns(source_columns); @@ -712,7 +712,7 @@ static std::shared_ptr interpretSubquery( /// get columns list for target table auto database_table = getDatabaseAndTableNameFromIdentifier(*table); const auto & storage = context.getTable(database_table.first, database_table.second); - const auto & columns = storage->columns.getListNonMaterialized(); + const auto & columns = storage->columns.ordinary; select_expression_list->children.reserve(columns.size()); /// manually substitute column names in place of asterisk @@ -826,9 +826,7 @@ void ExpressionAnalyzer::addExternalStorage(ASTPtr & subquery_or_table_name_or_t Block sample = interpreter->getSampleBlock(); NamesAndTypesList columns = sample.getNamesAndTypesList(); - StoragePtr external_storage = StorageMemory::create( - external_table_name, - ColumnsDescription{columns, NamesAndTypesList{}, NamesAndTypesList{}, ColumnDefaults{}}); + StoragePtr external_storage = StorageMemory::create(external_table_name, ColumnsDescription{columns}); external_storage->startup(); /** We replace the subquery with the name of the temporary table. @@ -1052,7 +1050,7 @@ void ExpressionAnalyzer::normalizeTreeImpl( if (storage) { /// If we select from a table, get only not MATERIALIZED, not ALIAS columns. - for (const auto & name_type : storage->columns.getListNonMaterialized()) + for (const auto & name_type : storage->columns.ordinary) all_columns.emplace_back(std::make_shared(name_type.name)); } else diff --git a/dbms/src/Interpreters/InterpreterCreateQuery.cpp b/dbms/src/Interpreters/InterpreterCreateQuery.cpp index 6033947b474..e5d20648fbd 100644 --- a/dbms/src/Interpreters/InterpreterCreateQuery.cpp +++ b/dbms/src/Interpreters/InterpreterCreateQuery.cpp @@ -305,7 +305,7 @@ ASTPtr InterpreterCreateQuery::formatColumns(const ColumnsDescription & columns) { auto columns_list = std::make_shared(); - for (const auto & column : boost::join(columns.ordinary, boost::join(columns.materialized, columns.aliases))) + for (const auto & column : columns.getAll()) { const auto column_declaration = std::make_shared(); ASTPtr column_declaration_ptr{column_declaration}; diff --git a/dbms/src/Interpreters/InterpreterDescribeQuery.cpp b/dbms/src/Interpreters/InterpreterDescribeQuery.cpp index b0d7e4c0363..c03034af49a 100644 --- a/dbms/src/Interpreters/InterpreterDescribeQuery.cpp +++ b/dbms/src/Interpreters/InterpreterDescribeQuery.cpp @@ -99,8 +99,7 @@ BlockInputStreamPtr InterpreterDescribeQuery::executeImpl() } auto table_lock = table->lockStructure(false, __PRETTY_FUNCTION__); - columns = table->columns.getList(); - columns.insert(std::end(columns), std::begin(table->columns.aliases), std::end(table->columns.aliases)); + columns = table->columns.getAll(); column_defaults = table->columns.defaults; } diff --git a/dbms/src/Interpreters/InterpreterInsertQuery.cpp b/dbms/src/Interpreters/InterpreterInsertQuery.cpp index 5f0b8201ac3..973fdc31a27 100644 --- a/dbms/src/Interpreters/InterpreterInsertQuery.cpp +++ b/dbms/src/Interpreters/InterpreterInsertQuery.cpp @@ -95,7 +95,7 @@ BlockIO InterpreterInsertQuery::execute() auto table_lock = table->lockStructure(true, __PRETTY_FUNCTION__); - NamesAndTypesList required_columns = table->columns.getList(); + NamesAndTypesList required_columns = table->columns.getPhysical(); /// We create a pipeline of several streams, into which we will write data. BlockOutputStreamPtr out; diff --git a/dbms/src/Interpreters/tests/in_join_subqueries_preprocessor.cpp b/dbms/src/Interpreters/tests/in_join_subqueries_preprocessor.cpp index 0a67da3a5d8..e5c04025c59 100644 --- a/dbms/src/Interpreters/tests/in_join_subqueries_preprocessor.cpp +++ b/dbms/src/Interpreters/tests/in_join_subqueries_preprocessor.cpp @@ -38,7 +38,6 @@ public: std::string getRemoteTableName() const { return remote_table; } std::string getTableName() const override { return ""; } - const DB::NamesAndTypesList & getColumnsListImpl() const override { return names_and_types; } protected: StorageDistributedFake(const std::string & remote_database_, const std::string & remote_table_, size_t shard_count_) @@ -50,7 +49,6 @@ private: const std::string remote_database; const std::string remote_table; size_t shard_count; - DB::NamesAndTypesList names_and_types; }; diff --git a/dbms/src/Storages/AlterCommands.cpp b/dbms/src/Storages/AlterCommands.cpp index defad6f5cba..184f63c070f 100644 --- a/dbms/src/Storages/AlterCommands.cpp +++ b/dbms/src/Storages/AlterCommands.cpp @@ -25,15 +25,7 @@ void AlterCommand::apply(ColumnsDescription & columns_description) const { if (type == ADD_COLUMN) { - const auto exists_in = [this] (const NamesAndTypesList & columns) - { - return columns.end() != std::find_if(columns.begin(), columns.end(), - std::bind(namesEqual, std::cref(column_name), std::placeholders::_1)); - }; - - if (exists_in(columns_description.ordinary) || - exists_in(columns_description.materialized) || - exists_in(columns_description.aliases)) + if (columns_description.getAll().contains(column_name)) { throw Exception{ "Cannot add column " + column_name + ": column with this name already exists", @@ -185,8 +177,7 @@ void AlterCommands::apply(ColumnsDescription & columns_description) const void AlterCommands::validate(IStorage * table, const Context & context) { - auto all_columns = table->columns.getList(); - all_columns.insert(std::end(all_columns), std::begin(table->columns.aliases), std::end(table->columns.aliases)); + auto all_columns = table->columns.getAll(); auto defaults = table->columns.defaults; std::vector> defaulted_columns{}; diff --git a/dbms/src/Storages/ColumnsDescription.cpp b/dbms/src/Storages/ColumnsDescription.cpp index b8fae53ed4e..95b5001a1f9 100644 --- a/dbms/src/Storages/ColumnsDescription.cpp +++ b/dbms/src/Storages/ColumnsDescription.cpp @@ -27,12 +27,18 @@ namespace ErrorCodes } -NamesAndTypesList ColumnsDescription::getList() const +NamesAndTypesList ColumnsDescription::getPhysical() const { return ext::collection_cast(boost::join(ordinary, materialized)); } +NamesAndTypesList ColumnsDescription::getAll() const +{ + return ext::collection_cast(boost::join(ordinary, boost::join(materialized, aliases))); +} + + Names ColumnsDescription::getNames() const { return ext::map(boost::join(ordinary, materialized), [] (const auto & it) { return it.name; }); diff --git a/dbms/src/Storages/ColumnsDescription.h b/dbms/src/Storages/ColumnsDescription.h index 888f0864701..e4e45ff6fe2 100644 --- a/dbms/src/Storages/ColumnsDescription.h +++ b/dbms/src/Storages/ColumnsDescription.h @@ -16,6 +16,20 @@ struct ColumnsDescription NamesAndTypesList aliases; ColumnDefaults defaults; + ColumnsDescription() = default; + + ColumnsDescription( + NamesAndTypesList ordinary_, + NamesAndTypesList materialized_, + NamesAndTypesList aliases_, + ColumnDefaults defaults_) + : ordinary(std::move(ordinary_)) + , materialized(std::move(materialized_)) + , aliases(std::move(aliases_)) + , defaults(std::move(defaults_)) + {} + + explicit ColumnsDescription(NamesAndTypesList ordinary_) : ordinary(std::move(ordinary_)) {} bool operator==(const ColumnsDescription & other) const { @@ -27,21 +41,19 @@ struct ColumnsDescription bool operator!=(const ColumnsDescription & other) const { return !(*this == other); } - /** Get a list of names and table column types, only non-virtual. - */ - NamesAndTypesList getList() const; - const NamesAndTypesList & getListNonMaterialized() const { return ordinary; } + /// ordinary + materialized. + NamesAndTypesList getPhysical() const; - /** Get a list of column names. - */ + /// ordinary + materialized + aliases. + NamesAndTypesList getAll() const; + + /// Get names of physical columns. Names getNames() const; - /** Get a description of any column by its name. - */ + /// Get a physical column by name. NameAndTypePair get(const String & column_name) const; - /** Is there a column with that name. - */ + /// Is there a physical column with the given name. bool has(const String & column_name) const; diff --git a/dbms/src/Storages/ITableDeclaration.cpp b/dbms/src/Storages/ITableDeclaration.cpp index d4bbb4b501e..b5a6c035b96 100644 --- a/dbms/src/Storages/ITableDeclaration.cpp +++ b/dbms/src/Storages/ITableDeclaration.cpp @@ -111,7 +111,7 @@ static NamesAndTypesMap getColumnsMap(const Args &... args) void ITableDeclaration::check(const Names & column_names) const { - const NamesAndTypesList & available_columns = columns.getList(); + const NamesAndTypesList & available_columns = columns.getPhysical(); if (column_names.empty()) throw Exception("Empty list of columns queried. There are columns: " + listOfColumns(available_columns), @@ -139,7 +139,7 @@ void ITableDeclaration::check(const Names & column_names) const void ITableDeclaration::check(const NamesAndTypesList & provided_columns) const { - const NamesAndTypesList & available_columns = columns.getList(); + const NamesAndTypesList & available_columns = columns.getPhysical(); const auto columns_map = getColumnsMap(available_columns); using UniqueStrings = google::dense_hash_set; @@ -167,7 +167,7 @@ void ITableDeclaration::check(const NamesAndTypesList & provided_columns) const void ITableDeclaration::check(const NamesAndTypesList & provided_columns, const Names & column_names) const { - const NamesAndTypesList & available_columns = columns.getList(); + const NamesAndTypesList & available_columns = columns.getPhysical(); const auto available_columns_map = getColumnsMap(available_columns); const NamesAndTypesMap & provided_columns_map = getColumnsMap(provided_columns); @@ -204,7 +204,7 @@ void ITableDeclaration::check(const NamesAndTypesList & provided_columns, const void ITableDeclaration::check(const Block & block, bool need_all) const { - const NamesAndTypesList & available_columns = columns.getList(); + const NamesAndTypesList & available_columns = columns.getPhysical(); const auto columns_map = getColumnsMap(available_columns); using NameSet = std::unordered_set; diff --git a/dbms/src/Storages/MergeTree/MergeTreeBlockInputStream.cpp b/dbms/src/Storages/MergeTree/MergeTreeBlockInputStream.cpp index 5a67714db64..6fb7c8fb611 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeBlockInputStream.cpp +++ b/dbms/src/Storages/MergeTree/MergeTreeBlockInputStream.cpp @@ -141,8 +141,8 @@ try if (!column_names.empty()) storage.check(data_part->columns, column_names); - pre_columns = storage.columns.getList().addTypes(pre_column_names); - columns = storage.columns.getList().addTypes(column_names); + pre_columns = storage.columns.getPhysical().addTypes(pre_column_names); + columns = storage.columns.getPhysical().addTypes(column_names); } else { diff --git a/dbms/src/Storages/MergeTree/MergeTreeData.cpp b/dbms/src/Storages/MergeTree/MergeTreeData.cpp index ea36f75a96c..de16c22593d 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeData.cpp +++ b/dbms/src/Storages/MergeTree/MergeTreeData.cpp @@ -110,7 +110,7 @@ MergeTreeData::MergeTreeData( data_parts_by_state_and_info(data_parts_indexes.get()) { /// NOTE: using the same columns list as is read when performing actual merges. - merging_params.check(columns.getList()); + merging_params.check(columns.getPhysical()); if (!primary_expr_ast) throw Exception("Primary key cannot be empty", ErrorCodes::BAD_ARGUMENTS); @@ -222,11 +222,11 @@ void MergeTreeData::initPrimaryKey() primary_sort_descr.clear(); addSortDescription(primary_sort_descr, primary_expr_ast); - primary_expr = ExpressionAnalyzer(primary_expr_ast, context, nullptr, columns.getList()).getActions(false); + primary_expr = ExpressionAnalyzer(primary_expr_ast, context, nullptr, columns.getPhysical()).getActions(false); { ExpressionActionsPtr projected_expr = - ExpressionAnalyzer(primary_expr_ast, context, nullptr, columns.getList()).getActions(true); + ExpressionAnalyzer(primary_expr_ast, context, nullptr, columns.getPhysical()).getActions(true); primary_key_sample = projected_expr->getSampleBlock(); } @@ -241,10 +241,10 @@ void MergeTreeData::initPrimaryKey() if (secondary_sort_expr_ast) { addSortDescription(sort_descr, secondary_sort_expr_ast); - secondary_sort_expr = ExpressionAnalyzer(secondary_sort_expr_ast, context, nullptr, columns.getList()).getActions(false); + secondary_sort_expr = ExpressionAnalyzer(secondary_sort_expr_ast, context, nullptr, columns.getPhysical()).getActions(false); ExpressionActionsPtr projected_expr = - ExpressionAnalyzer(secondary_sort_expr_ast, context, nullptr, columns.getList()).getActions(true); + ExpressionAnalyzer(secondary_sort_expr_ast, context, nullptr, columns.getPhysical()).getActions(true); auto secondary_key_sample = projected_expr->getSampleBlock(); checkKeyExpression(*secondary_sort_expr, secondary_key_sample, "Secondary"); @@ -257,7 +257,7 @@ void MergeTreeData::initPartitionKey() if (!partition_expr_ast || partition_expr_ast->children.empty()) return; - partition_expr = ExpressionAnalyzer(partition_expr_ast, context, nullptr, columns.getList()).getActions(false); + partition_expr = ExpressionAnalyzer(partition_expr_ast, context, nullptr, columns.getPhysical()).getActions(false); for (const ASTPtr & ast : partition_expr_ast->children) { String col_name = ast->getColumnName(); @@ -903,7 +903,7 @@ void MergeTreeData::checkAlter(const AlterCommands & commands) columns_alter_forbidden.insert(merging_params.sign_column); std::map old_types; - for (const auto & column : columns.getList()) + for (const auto & column : columns.getPhysical()) old_types.emplace(column.name, column.type.get()); for (const AlterCommand & command : commands) @@ -931,7 +931,7 @@ void MergeTreeData::checkAlter(const AlterCommands & commands) NameToNameMap unused_map; bool unused_bool; - createConvertExpression(nullptr, columns.getList(), new_columns.getList(), unused_expression, unused_map, unused_bool); + createConvertExpression(nullptr, columns.getPhysical(), new_columns.getPhysical(), unused_expression, unused_map, unused_bool); } void MergeTreeData::createConvertExpression(const DataPartPtr & part, const NamesAndTypesList & old_columns, const NamesAndTypesList & new_columns, @@ -1835,7 +1835,7 @@ void MergeTreeData::addPartContributionToColumnSizes(const DataPartPtr & part) const auto & files = part->checksums.files; /// TODO This method doesn't take into account columns with multiple files. - for (const auto & column : columns.getList()) + for (const auto & column : columns.getPhysical()) { const auto escaped_name = escapeForFileName(column.name); const auto bin_file_name = escaped_name + ".bin"; @@ -1868,7 +1868,7 @@ void MergeTreeData::removePartContributionToColumnSizes(const DataPartPtr & part const auto & files = part->checksums.files; /// TODO This method doesn't take into account columns with multiple files. - for (const auto & column : columns.getList()) + for (const auto & column : columns.getPhysical()) { const auto escaped_name = escapeForFileName(column.name); const auto bin_file_name = escaped_name + ".bin"; diff --git a/dbms/src/Storages/MergeTree/MergeTreeDataMerger.cpp b/dbms/src/Storages/MergeTree/MergeTreeDataMerger.cpp index 4e3e9788aaa..8e30320aefb 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeDataMerger.cpp +++ b/dbms/src/Storages/MergeTree/MergeTreeDataMerger.cpp @@ -537,7 +537,7 @@ MergeTreeData::MutableDataPartPtr MergeTreeDataMerger::mergePartsToTemporaryPart part->accumulateColumnSizes(merged_column_to_size); Names all_column_names = data.columns.getNames(); - NamesAndTypesList all_columns = data.columns.getList(); + NamesAndTypesList all_columns = data.columns.getPhysical(); const SortDescription sort_desc = data.getSortDescription(); NamesAndTypesList gathering_columns, merging_columns; diff --git a/dbms/src/Storages/MergeTree/MergeTreeDataPart.cpp b/dbms/src/Storages/MergeTree/MergeTreeDataPart.cpp index 9c0269c7437..f37cd8e579c 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeDataPart.cpp +++ b/dbms/src/Storages/MergeTree/MergeTreeDataPart.cpp @@ -418,7 +418,7 @@ UInt64 MergeTreeDataPart::getColumnMrkSize(const String & name) const */ String MergeTreeDataPart::getColumnNameWithMinumumCompressedSize() const { - const auto & columns = storage.columns.getList(); + const auto & columns = storage.columns.getPhysical(); const std::string * minimum_size_column = nullptr; UInt64 minimum_size = std::numeric_limits::max(); @@ -774,7 +774,7 @@ void MergeTreeDataPart::accumulateColumnSizes(ColumnToSize & column_to_size) con { std::shared_lock part_lock(columns_lock); - for (const NameAndTypePair & name_type : storage.columns.getList()) + for (const NameAndTypePair & name_type : storage.columns.getPhysical()) { name_type.type->enumerateStreams([&](const IDataType::SubstreamPath & substream_path) { @@ -794,7 +794,7 @@ void MergeTreeDataPart::loadColumns(bool require) throw Exception("No columns.txt in part " + name, ErrorCodes::NO_FILE_IN_DATA_PART); /// If there is no file with a list of columns, write it down. - for (const NameAndTypePair & column : storage.columns.getList()) + for (const NameAndTypePair & column : storage.columns.getPhysical()) if (Poco::File(getFullPath() + escapeForFileName(column.name) + ".bin").exists()) columns.push_back(column); diff --git a/dbms/src/Storages/MergeTree/MergeTreeDataSelectExecutor.cpp b/dbms/src/Storages/MergeTree/MergeTreeDataSelectExecutor.cpp index 605091e6386..0890163078d 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeDataSelectExecutor.cpp +++ b/dbms/src/Storages/MergeTree/MergeTreeDataSelectExecutor.cpp @@ -175,7 +175,7 @@ BlockInputStreams MergeTreeDataSelectExecutor::read( } } - NamesAndTypesList available_real_columns = data.columns.getList(); + NamesAndTypesList available_real_columns = data.columns.getPhysical(); NamesAndTypesList available_real_and_virtual_columns = available_real_columns; for (const auto & name : virt_column_names) @@ -857,7 +857,7 @@ void MergeTreeDataSelectExecutor::createPositiveSignCondition( arguments->children.push_back(sign); arguments->children.push_back(one); - out_expression = ExpressionAnalyzer(function, context, {}, data.columns.getList()).getActions(false); + out_expression = ExpressionAnalyzer(function, context, {}, data.columns.getPhysical()).getActions(false); out_column = function->getColumnName(); } diff --git a/dbms/src/Storages/MergeTree/MergeTreeDataWriter.cpp b/dbms/src/Storages/MergeTree/MergeTreeDataWriter.cpp index f4db6b934ba..19eb2e8eecd 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeDataWriter.cpp +++ b/dbms/src/Storages/MergeTree/MergeTreeDataWriter.cpp @@ -205,7 +205,7 @@ MergeTreeData::MutableDataPartPtr MergeTreeDataWriter::writeTempPart(BlockWithPa /// either default lz4 or compression method with zero thresholds on absolute and relative part size. auto compression_settings = data.context.chooseCompressionSettings(0, 0); - NamesAndTypesList columns = data.columns.getList().filter(block.getNames()); + NamesAndTypesList columns = data.columns.getPhysical().filter(block.getNames()); MergedBlockOutputStream out(data, new_data_part->getFullPath(), columns, compression_settings); out.writePrefix(); diff --git a/dbms/src/Storages/MergeTree/MergeTreeReadPool.cpp b/dbms/src/Storages/MergeTree/MergeTreeReadPool.cpp index c32aa71a731..2c600dbd32d 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeReadPool.cpp +++ b/dbms/src/Storages/MergeTree/MergeTreeReadPool.cpp @@ -238,8 +238,8 @@ std::vector MergeTreeReadPool::fillPerPartInfo( if (!required_column_names.empty()) data.check(part.data_part->columns, required_column_names); - per_part_pre_columns.push_back(data.columns.getList().addTypes(required_pre_column_names)); - per_part_columns.push_back(data.columns.getList().addTypes(required_column_names)); + per_part_pre_columns.push_back(data.columns.getPhysical().addTypes(required_pre_column_names)); + per_part_columns.push_back(data.columns.getPhysical().addTypes(required_column_names)); } else { diff --git a/dbms/src/Storages/MergeTree/MergeTreeWhereOptimizer.cpp b/dbms/src/Storages/MergeTree/MergeTreeWhereOptimizer.cpp index 005ca750226..ce4047fc85f 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeWhereOptimizer.cpp +++ b/dbms/src/Storages/MergeTree/MergeTreeWhereOptimizer.cpp @@ -41,9 +41,9 @@ MergeTreeWhereOptimizer::MergeTreeWhereOptimizer( Logger * log) : primary_key_columns{ext::map(data.getPrimarySortDescription(), [] (const SortColumnDescription & col) { return col.column_name; })}, - table_columns{ext::map(data.columns.getList(), + table_columns{ext::map(data.columns.getPhysical(), [] (const NameAndTypePair & col) { return col.name; })}, - block_with_constants{PKCondition::getBlockWithConstants(query_info.query, context, data.columns.getList())}, + block_with_constants{PKCondition::getBlockWithConstants(query_info.query, context, data.columns.getPhysical())}, prepared_sets(query_info.sets), log{log} { diff --git a/dbms/src/Storages/MergeTree/ReplicatedMergeTreeAlterThread.cpp b/dbms/src/Storages/MergeTree/ReplicatedMergeTreeAlterThread.cpp index c359a889bfb..948902d8302 100644 --- a/dbms/src/Storages/MergeTree/ReplicatedMergeTreeAlterThread.cpp +++ b/dbms/src/Storages/MergeTree/ReplicatedMergeTreeAlterThread.cpp @@ -126,7 +126,7 @@ void ReplicatedMergeTreeAlterThread::run() if (!changed_version) parts = storage.data.getDataParts(); - const auto columns_plus_materialized = storage.data.columns.getList(); + const auto columns_for_parts = storage.data.columns.getPhysical(); for (const MergeTreeData::DataPartPtr & part : parts) { @@ -134,7 +134,7 @@ void ReplicatedMergeTreeAlterThread::run() /// TODO: You can skip checking for too large changes if ZooKeeper has, for example, /// node /flags/force_alter. auto transaction = storage.data.alterDataPart( - part, columns_plus_materialized, storage.data.primary_expr_ast, false); + part, columns_for_parts, storage.data.primary_expr_ast, false); if (!transaction) continue; diff --git a/dbms/src/Storages/StorageDistributed.cpp b/dbms/src/Storages/StorageDistributed.cpp index b543e802ef9..c010ca768d1 100644 --- a/dbms/src/Storages/StorageDistributed.cpp +++ b/dbms/src/Storages/StorageDistributed.cpp @@ -142,7 +142,7 @@ StorageDistributed::StorageDistributed( table_name(table_name_), remote_database(remote_database_), remote_table(remote_table_), context(context_), cluster_name(context.getMacros().expand(cluster_name_)), has_sharding_key(sharding_key_), - sharding_key_expr(sharding_key_ ? ExpressionAnalyzer(sharding_key_, context, nullptr, columns.getList()).getActions(false) : nullptr), + sharding_key_expr(sharding_key_ ? ExpressionAnalyzer(sharding_key_, context, nullptr, columns.getPhysical()).getActions(false) : nullptr), sharding_key_column_name(sharding_key_ ? sharding_key_->getColumnName() : String{}), path(data_path_.empty() ? "" : (data_path_ + escapeForFileName(table_name) + '/')) { @@ -158,7 +158,7 @@ StoragePtr StorageDistributed::createWithOwnCluster( const Context & context_) { auto res = ext::shared_ptr_helper::create( - name_, ColumnsDescription{columns_, NamesAndTypesList(), NamesAndTypesList(), ColumnDefaults()}, + name_, ColumnsDescription{columns_}, remote_database_, remote_table_, String{}, context_, ASTPtr(), String()); res->owned_cluster = owned_cluster_; @@ -385,7 +385,7 @@ void registerStorageDistributed(StorageFactory & factory) /// Check that sharding_key exists in the table and has numeric type. if (sharding_key) { - auto sharding_expr = ExpressionAnalyzer(sharding_key, args.context, nullptr, args.columns.getList()).getActions(true); + auto sharding_expr = ExpressionAnalyzer(sharding_key, args.context, nullptr, args.columns.getPhysical()).getActions(true); const Block & block = sharding_expr->getSampleBlock(); if (block.columns() != 1) diff --git a/dbms/src/Storages/StorageFactory.cpp b/dbms/src/Storages/StorageFactory.cpp index 1b8970aba24..e6e661571aa 100644 --- a/dbms/src/Storages/StorageFactory.cpp +++ b/dbms/src/Storages/StorageFactory.cpp @@ -64,9 +64,7 @@ StoragePtr StorageFactory::get( { /// Check for some special types, that are not allowed to be stored in tables. Example: NULL data type. /// Exception: any type is allowed in View, because plain (non-materialized) View does not store anything itself. - checkAllTypesAreAllowedInTable(columns.ordinary); - checkAllTypesAreAllowedInTable(columns.materialized); - checkAllTypesAreAllowedInTable(columns.aliases); + checkAllTypesAreAllowedInTable(columns.getAll()); if (query.is_materialized_view) { diff --git a/dbms/src/Storages/StorageLog.cpp b/dbms/src/Storages/StorageLog.cpp index 27881d8b8fe..137c1d55616 100644 --- a/dbms/src/Storages/StorageLog.cpp +++ b/dbms/src/Storages/StorageLog.cpp @@ -371,7 +371,7 @@ StorageLog::StorageLog( /// create files if they do not exist Poco::File(path + escapeForFileName(name) + '/').createDirectories(); - for (const auto & column : columns.getList()) + for (const auto & column : columns.getPhysical()) addFiles(column.name, *column.type); marks_file = Poco::File(path + escapeForFileName(name) + '/' + DBMS_STORAGE_LOG_MARKS_FILE_NAME); @@ -499,7 +499,7 @@ BlockInputStreams StorageLog::read( processed_stage = QueryProcessingStage::FetchColumns; loadMarks(); - NamesAndTypesList all_columns = Nested::collect(columns.getList().addTypes(column_names)); + NamesAndTypesList all_columns = Nested::collect(columns.getPhysical().addTypes(column_names)); std::shared_lock lock(rwlock); diff --git a/dbms/src/Storages/StorageMerge.cpp b/dbms/src/Storages/StorageMerge.cpp index 39fed81e538..f26544975cd 100644 --- a/dbms/src/Storages/StorageMerge.cpp +++ b/dbms/src/Storages/StorageMerge.cpp @@ -181,7 +181,7 @@ BlockInputStreams StorageMerge::read( /// If there are only virtual columns in query, you must request at least one other column. if (real_column_names.size() == 0) - real_column_names.push_back(ExpressionActions::getSmallestColumn(table->columns.getList())); + real_column_names.push_back(ExpressionActions::getSmallestColumn(table->columns.getPhysical())); /// Substitute virtual column for its value when querying tables. ASTPtr modified_query_ast = query->clone(); diff --git a/dbms/src/Storages/StorageMergeTree.cpp b/dbms/src/Storages/StorageMergeTree.cpp index 71d8013e4db..5e799c91909 100644 --- a/dbms/src/Storages/StorageMergeTree.cpp +++ b/dbms/src/Storages/StorageMergeTree.cpp @@ -176,7 +176,7 @@ void StorageMergeTree::alter( throw Exception("MODIFY PRIMARY KEY only supported for tables without sampling key", ErrorCodes::BAD_ARGUMENTS); auto parts = data.getDataParts({MergeTreeDataPartState::PreCommitted, MergeTreeDataPartState::Committed, MergeTreeDataPartState::Outdated}); - auto columns_for_parts = new_columns.getList(); + auto columns_for_parts = new_columns.getPhysical(); for (const MergeTreeData::DataPartPtr & part : parts) { if (auto transaction = data.alterDataPart(part, columns_for_parts, new_primary_key_ast, false)) @@ -422,7 +422,7 @@ void StorageMergeTree::clearColumnInPartition(const ASTPtr & partition, const Fi auto new_columns = data.columns; alter_command.apply(new_columns); - auto columns_for_parts = new_columns.getList(); + auto columns_for_parts = new_columns.getPhysical(); for (const auto & part : parts) { if (part->info.partition_id != partition_id) diff --git a/dbms/src/Storages/StorageReplicatedMergeTree.cpp b/dbms/src/Storages/StorageReplicatedMergeTree.cpp index 9f6553f6512..d69db2d388e 100644 --- a/dbms/src/Storages/StorageReplicatedMergeTree.cpp +++ b/dbms/src/Storages/StorageReplicatedMergeTree.cpp @@ -1512,7 +1512,7 @@ void StorageReplicatedMergeTree::executeClearColumnInPartition(const LogEntry & size_t modified_parts = 0; auto parts = data.getDataParts(); - auto columns_for_parts = new_columns.getList(); + auto columns_for_parts = new_columns.getPhysical(); for (const auto & part : parts) { if (!entry_part_info.contains(part->info)) diff --git a/dbms/src/Storages/StorageTinyLog.cpp b/dbms/src/Storages/StorageTinyLog.cpp index 039f470c366..8c7fe1963a8 100644 --- a/dbms/src/Storages/StorageTinyLog.cpp +++ b/dbms/src/Storages/StorageTinyLog.cpp @@ -296,7 +296,7 @@ StorageTinyLog::StorageTinyLog( throwFromErrno("Cannot create directory " + full_path, ErrorCodes::CANNOT_CREATE_DIRECTORY); } - for (const auto & col : columns.getList()) + for (const auto & col : columns.getPhysical()) addFiles(col.name, *col.type); } @@ -348,7 +348,7 @@ BlockInputStreams StorageTinyLog::read( check(column_names); processed_stage = QueryProcessingStage::FetchColumns; return BlockInputStreams(1, std::make_shared( - max_block_size, Nested::collect(columns.getList().addTypes(column_names)), *this, context.getSettingsRef().max_read_buffer_size)); + max_block_size, Nested::collect(columns.getPhysical().addTypes(column_names)), *this, context.getSettingsRef().max_read_buffer_size)); } diff --git a/dbms/src/Storages/System/StorageSystemAsynchronousMetrics.cpp b/dbms/src/Storages/System/StorageSystemAsynchronousMetrics.cpp index f08808f799a..3c77244c557 100644 --- a/dbms/src/Storages/System/StorageSystemAsynchronousMetrics.cpp +++ b/dbms/src/Storages/System/StorageSystemAsynchronousMetrics.cpp @@ -16,10 +16,10 @@ StorageSystemAsynchronousMetrics::StorageSystemAsynchronousMetrics(const std::st : name(name_), async_metrics(async_metrics_) { - columns.ordinary = NamesAndTypesList{ + columns = ColumnsDescription({ {"metric", std::make_shared()}, {"value", std::make_shared()}, - }; + }); } diff --git a/dbms/src/Storages/System/StorageSystemBuildOptions.cpp b/dbms/src/Storages/System/StorageSystemBuildOptions.cpp index 7a864090e59..f354da97944 100644 --- a/dbms/src/Storages/System/StorageSystemBuildOptions.cpp +++ b/dbms/src/Storages/System/StorageSystemBuildOptions.cpp @@ -13,10 +13,10 @@ namespace DB StorageSystemBuildOptions::StorageSystemBuildOptions(const std::string & name_) : name(name_) { - columns.ordinary = NamesAndTypesList{ + columns = ColumnsDescription({ { "name", std::make_shared() }, { "value", std::make_shared() }, - }; + }); } diff --git a/dbms/src/Storages/System/StorageSystemClusters.cpp b/dbms/src/Storages/System/StorageSystemClusters.cpp index 8d772e5ebb8..1e67e433d48 100644 --- a/dbms/src/Storages/System/StorageSystemClusters.cpp +++ b/dbms/src/Storages/System/StorageSystemClusters.cpp @@ -14,7 +14,7 @@ namespace DB StorageSystemClusters::StorageSystemClusters(const std::string & name_) : name(name_) { - columns.ordinary = NamesAndTypesList{ + columns = ColumnsDescription({ { "cluster", std::make_shared() }, { "shard_num", std::make_shared() }, { "shard_weight", std::make_shared() }, @@ -24,8 +24,8 @@ StorageSystemClusters::StorageSystemClusters(const std::string & name_) { "port", std::make_shared() }, { "is_local", std::make_shared() }, { "user", std::make_shared() }, - { "default_database", std::make_shared() } - }; + { "default_database", std::make_shared() }, + }); } diff --git a/dbms/src/Storages/System/StorageSystemColumns.cpp b/dbms/src/Storages/System/StorageSystemColumns.cpp index adec6a70c58..75c8afa1515 100644 --- a/dbms/src/Storages/System/StorageSystemColumns.cpp +++ b/dbms/src/Storages/System/StorageSystemColumns.cpp @@ -18,7 +18,7 @@ namespace DB StorageSystemColumns::StorageSystemColumns(const std::string & name_) : name(name_) { - columns.ordinary = NamesAndTypesList{ + columns = ColumnsDescription({ { "database", std::make_shared() }, { "table", std::make_shared() }, { "name", std::make_shared() }, @@ -28,7 +28,7 @@ StorageSystemColumns::StorageSystemColumns(const std::string & name_) { "data_compressed_bytes", std::make_shared() }, { "data_uncompressed_bytes", std::make_shared() }, { "marks_bytes", std::make_shared() }, - }; + }); } @@ -137,8 +137,7 @@ BlockInputStreams StorageSystemColumns::read( throw; } - columns = storage->columns.getList(); - columns.insert(std::end(columns), std::begin(storage->columns.aliases), std::end(storage->columns.aliases)); + columns = storage->columns.getAll(); column_defaults = storage->columns.defaults; /** Info about sizes of columns for tables of MergeTree family. diff --git a/dbms/src/Storages/System/StorageSystemDatabases.cpp b/dbms/src/Storages/System/StorageSystemDatabases.cpp index 12c35c7ec9e..79a3b3483a6 100644 --- a/dbms/src/Storages/System/StorageSystemDatabases.cpp +++ b/dbms/src/Storages/System/StorageSystemDatabases.cpp @@ -13,12 +13,12 @@ namespace DB StorageSystemDatabases::StorageSystemDatabases(const std::string & name_) : name(name_) { - columns.ordinary = NamesAndTypesList{ + columns = ColumnsDescription({ {"name", std::make_shared()}, {"engine", std::make_shared()}, {"data_path", std::make_shared()}, {"metadata_path", std::make_shared()}, - }; + }); } diff --git a/dbms/src/Storages/System/StorageSystemDictionaries.cpp b/dbms/src/Storages/System/StorageSystemDictionaries.cpp index 3cb4bd7962b..84bf4a2c98f 100644 --- a/dbms/src/Storages/System/StorageSystemDictionaries.cpp +++ b/dbms/src/Storages/System/StorageSystemDictionaries.cpp @@ -21,7 +21,7 @@ namespace DB StorageSystemDictionaries::StorageSystemDictionaries(const std::string & name) : name{name} { - columns.ordinary = NamesAndTypesList{ + columns = ColumnsDescription({ { "name", std::make_shared() }, { "origin", std::make_shared() }, { "type", std::make_shared() }, @@ -35,8 +35,8 @@ StorageSystemDictionaries::StorageSystemDictionaries(const std::string & name) { "load_factor", std::make_shared() }, { "creation_time", std::make_shared() }, { "source", std::make_shared() }, - { "last_exception", std::make_shared() } - }; + { "last_exception", std::make_shared() }, + }); } diff --git a/dbms/src/Storages/System/StorageSystemEvents.cpp b/dbms/src/Storages/System/StorageSystemEvents.cpp index 0838eb739a5..ac613a7e84e 100644 --- a/dbms/src/Storages/System/StorageSystemEvents.cpp +++ b/dbms/src/Storages/System/StorageSystemEvents.cpp @@ -13,11 +13,11 @@ namespace DB StorageSystemEvents::StorageSystemEvents(const std::string & name_) : name(name_) { - columns.ordinary = NamesAndTypesList + columns = ColumnsDescription( { {"event", std::make_shared()}, - {"value", std::make_shared()} - }; + {"value", std::make_shared()}, + }); } diff --git a/dbms/src/Storages/System/StorageSystemFunctions.cpp b/dbms/src/Storages/System/StorageSystemFunctions.cpp index 4d013eb2268..2d42bee9a46 100644 --- a/dbms/src/Storages/System/StorageSystemFunctions.cpp +++ b/dbms/src/Storages/System/StorageSystemFunctions.cpp @@ -16,10 +16,10 @@ namespace DB StorageSystemFunctions::StorageSystemFunctions(const std::string & name_) : name(name_) { - columns.ordinary = NamesAndTypesList{ + columns = ColumnsDescription({ { "name", std::make_shared() }, - { "is_aggregate", std::make_shared() } - }; + { "is_aggregate", std::make_shared() }, + }); } diff --git a/dbms/src/Storages/System/StorageSystemGraphite.cpp b/dbms/src/Storages/System/StorageSystemGraphite.cpp index e99c037ea78..30cc56c55e3 100644 --- a/dbms/src/Storages/System/StorageSystemGraphite.cpp +++ b/dbms/src/Storages/System/StorageSystemGraphite.cpp @@ -125,15 +125,15 @@ static Strings getAllGraphiteSections(const AbstractConfiguration & config) StorageSystemGraphite::StorageSystemGraphite(const std::string & name_) : name(name_) { - columns.ordinary = NamesAndTypesList{ + columns = ColumnsDescription({ {"config_name", std::make_shared()}, {"regexp", std::make_shared()}, {"function", std::make_shared()}, {"age", std::make_shared()}, {"precision", std::make_shared()}, {"priority", std::make_shared()}, - {"is_default", std::make_shared()} - }; + {"is_default", std::make_shared()}, + }); } diff --git a/dbms/src/Storages/System/StorageSystemMerges.cpp b/dbms/src/Storages/System/StorageSystemMerges.cpp index fea37df35a1..c32cad1265b 100644 --- a/dbms/src/Storages/System/StorageSystemMerges.cpp +++ b/dbms/src/Storages/System/StorageSystemMerges.cpp @@ -14,7 +14,7 @@ namespace DB StorageSystemMerges::StorageSystemMerges(const std::string & name) : name{name} { - columns.ordinary = NamesAndTypesList{ + columns = ColumnsDescription({ { "database", std::make_shared() }, { "table", std::make_shared() }, { "elapsed", std::make_shared() }, @@ -31,7 +31,7 @@ StorageSystemMerges::StorageSystemMerges(const std::string & name) { "columns_written", std::make_shared() }, { "memory_usage", std::make_shared() }, { "thread_number", std::make_shared() }, - }; + }); } diff --git a/dbms/src/Storages/System/StorageSystemMetrics.cpp b/dbms/src/Storages/System/StorageSystemMetrics.cpp index 54002318ef6..57935b13fea 100644 --- a/dbms/src/Storages/System/StorageSystemMetrics.cpp +++ b/dbms/src/Storages/System/StorageSystemMetrics.cpp @@ -14,10 +14,10 @@ namespace DB StorageSystemMetrics::StorageSystemMetrics(const std::string & name_) : name(name_) { - columns.ordinary = NamesAndTypesList{ + columns = ColumnsDescription({ {"metric", std::make_shared()}, {"value", std::make_shared()}, - }; + }); } diff --git a/dbms/src/Storages/System/StorageSystemModels.cpp b/dbms/src/Storages/System/StorageSystemModels.cpp index 9057c6de626..6ef9d157303 100644 --- a/dbms/src/Storages/System/StorageSystemModels.cpp +++ b/dbms/src/Storages/System/StorageSystemModels.cpp @@ -14,13 +14,13 @@ namespace DB StorageSystemModels::StorageSystemModels(const std::string & name) : name{name} { - columns.ordinary = NamesAndTypesList{ + columns = ColumnsDescription({ { "name", std::make_shared() }, { "origin", std::make_shared() }, { "type", std::make_shared() }, { "creation_time", std::make_shared() }, - { "last_exception", std::make_shared() } - }; + { "last_exception", std::make_shared() }, + }); } diff --git a/dbms/src/Storages/System/StorageSystemNumbers.cpp b/dbms/src/Storages/System/StorageSystemNumbers.cpp index 803f651639d..d5e72bcbdc3 100644 --- a/dbms/src/Storages/System/StorageSystemNumbers.cpp +++ b/dbms/src/Storages/System/StorageSystemNumbers.cpp @@ -47,7 +47,7 @@ private: StorageSystemNumbers::StorageSystemNumbers(const std::string & name_, bool multithreaded_, size_t limit_) : name(name_), multithreaded(multithreaded_), limit(limit_) { - columns.ordinary = NamesAndTypesList{{"number", std::make_shared()}}; + columns = ColumnsDescription({{"number", std::make_shared()}}); } diff --git a/dbms/src/Storages/System/StorageSystemOne.cpp b/dbms/src/Storages/System/StorageSystemOne.cpp index f35580c3882..9cd74a7525c 100644 --- a/dbms/src/Storages/System/StorageSystemOne.cpp +++ b/dbms/src/Storages/System/StorageSystemOne.cpp @@ -13,7 +13,7 @@ namespace DB StorageSystemOne::StorageSystemOne(const std::string & name_) : name(name_) { - columns.ordinary = NamesAndTypesList{{"dummy", std::make_shared()}}; + columns = ColumnsDescription({{"dummy", std::make_shared()}}); } diff --git a/dbms/src/Storages/System/StorageSystemPartsBase.cpp b/dbms/src/Storages/System/StorageSystemPartsBase.cpp index c6dc2a2c551..5143b73955f 100644 --- a/dbms/src/Storages/System/StorageSystemPartsBase.cpp +++ b/dbms/src/Storages/System/StorageSystemPartsBase.cpp @@ -280,7 +280,7 @@ bool StorageSystemPartsBase::hasColumn(const String & column_name) const StorageSystemPartsBase::StorageSystemPartsBase(std::string name_, NamesAndTypesList && columns_) : name(std::move(name_)) { - columns.ordinary = columns_; + columns = ColumnsDescription(std::move(columns_)); } } diff --git a/dbms/src/Storages/System/StorageSystemPartsColumns.cpp b/dbms/src/Storages/System/StorageSystemPartsColumns.cpp index 5f0a584c1d6..326dfbf30ad 100644 --- a/dbms/src/Storages/System/StorageSystemPartsColumns.cpp +++ b/dbms/src/Storages/System/StorageSystemPartsColumns.cpp @@ -61,8 +61,7 @@ void StorageSystemPartsColumns::processNextStorage(MutableColumns & columns, con String default_expression; }; - NamesAndTypesList columns_list = info.storage->columns.getList(); - columns_list.insert(std::end(columns_list), std::begin(info.storage->columns.aliases), std::end(info.storage->columns.aliases)); + NamesAndTypesList columns_list = info.storage->columns.getAll(); const auto & column_defaults = info.storage->columns.defaults; std::unordered_map columns_info; diff --git a/dbms/src/Storages/System/StorageSystemProcesses.cpp b/dbms/src/Storages/System/StorageSystemProcesses.cpp index e8262723d89..15ecc1a3d95 100644 --- a/dbms/src/Storages/System/StorageSystemProcesses.cpp +++ b/dbms/src/Storages/System/StorageSystemProcesses.cpp @@ -14,7 +14,7 @@ namespace DB StorageSystemProcesses::StorageSystemProcesses(const std::string & name_) : name(name_) { - columns.ordinary = NamesAndTypesList{ + columns = ColumnsDescription({ { "is_initial_query", std::make_shared() }, { "user", std::make_shared() }, @@ -50,8 +50,8 @@ StorageSystemProcesses::StorageSystemProcesses(const std::string & name_) { "written_bytes", std::make_shared() }, { "memory_usage", std::make_shared() }, { "peak_memory_usage", std::make_shared() }, - { "query", std::make_shared() } - }; + { "query", std::make_shared() }, + }); } diff --git a/dbms/src/Storages/System/StorageSystemReplicas.cpp b/dbms/src/Storages/System/StorageSystemReplicas.cpp index ca240a80568..fdcc6984861 100644 --- a/dbms/src/Storages/System/StorageSystemReplicas.cpp +++ b/dbms/src/Storages/System/StorageSystemReplicas.cpp @@ -17,7 +17,7 @@ namespace DB StorageSystemReplicas::StorageSystemReplicas(const std::string & name_) : name(name_) { - columns.ordinary = NamesAndTypesList{ + columns = ColumnsDescription({ { "database", std::make_shared() }, { "table", std::make_shared() }, { "engine", std::make_shared() }, @@ -44,7 +44,7 @@ StorageSystemReplicas::StorageSystemReplicas(const std::string & name_) { "absolute_delay", std::make_shared() }, { "total_replicas", std::make_shared() }, { "active_replicas", std::make_shared() }, - }; + }); } diff --git a/dbms/src/Storages/System/StorageSystemReplicationQueue.cpp b/dbms/src/Storages/System/StorageSystemReplicationQueue.cpp index 363306437bb..b0ba714fef4 100644 --- a/dbms/src/Storages/System/StorageSystemReplicationQueue.cpp +++ b/dbms/src/Storages/System/StorageSystemReplicationQueue.cpp @@ -20,7 +20,7 @@ namespace DB StorageSystemReplicationQueue::StorageSystemReplicationQueue(const std::string & name_) : name(name_) { - columns.ordinary = NamesAndTypesList{ + columns = ColumnsDescription({ /// Table properties. { "database", std::make_shared() }, { "table", std::make_shared() }, @@ -43,7 +43,7 @@ StorageSystemReplicationQueue::StorageSystemReplicationQueue(const std::string & { "num_postponed", std::make_shared() }, { "postpone_reason", std::make_shared() }, { "last_postpone_time", std::make_shared() }, - }; + }); } diff --git a/dbms/src/Storages/System/StorageSystemSettings.cpp b/dbms/src/Storages/System/StorageSystemSettings.cpp index 3848583688b..d8a758becc7 100644 --- a/dbms/src/Storages/System/StorageSystemSettings.cpp +++ b/dbms/src/Storages/System/StorageSystemSettings.cpp @@ -14,12 +14,12 @@ namespace DB StorageSystemSettings::StorageSystemSettings(const std::string & name_) : name(name_) { - columns.ordinary = NamesAndTypesList{ + columns = ColumnsDescription({ { "name", std::make_shared() }, { "value", std::make_shared() }, { "changed", std::make_shared() }, - { "description", std::make_shared() } - }; + { "description", std::make_shared() }, + }); } diff --git a/dbms/src/Storages/System/StorageSystemTables.cpp b/dbms/src/Storages/System/StorageSystemTables.cpp index 91ab3581bc8..2db2a8052cc 100644 --- a/dbms/src/Storages/System/StorageSystemTables.cpp +++ b/dbms/src/Storages/System/StorageSystemTables.cpp @@ -114,7 +114,7 @@ protected: StorageSystemTables::StorageSystemTables(const std::string & name_) : name(name_) { - columns.ordinary = NamesAndTypesList + columns = ColumnsDescription( { {"database", std::make_shared()}, {"name", std::make_shared()}, @@ -122,7 +122,7 @@ StorageSystemTables::StorageSystemTables(const std::string & name_) {"is_temporary", std::make_shared()}, {"data_path", std::make_shared()}, {"metadata_path", std::make_shared()}, - }; + }); virtual_columns = { diff --git a/dbms/src/Storages/System/StorageSystemZooKeeper.cpp b/dbms/src/Storages/System/StorageSystemZooKeeper.cpp index 9aae0ce5b3b..a5d5e2dab96 100644 --- a/dbms/src/Storages/System/StorageSystemZooKeeper.cpp +++ b/dbms/src/Storages/System/StorageSystemZooKeeper.cpp @@ -22,7 +22,7 @@ namespace DB StorageSystemZooKeeper::StorageSystemZooKeeper(const std::string & name_) : name(name_) { - columns.ordinary = NamesAndTypesList{ + columns = ColumnsDescription({ { "name", std::make_shared() }, { "value", std::make_shared() }, { "czxid", std::make_shared() }, @@ -37,7 +37,7 @@ StorageSystemZooKeeper::StorageSystemZooKeeper(const std::string & name_) { "numChildren", std::make_shared() }, { "pzxid", std::make_shared() }, { "path", std::make_shared() }, - }; + }); } diff --git a/dbms/src/Storages/tests/hit_log.cpp b/dbms/src/Storages/tests/hit_log.cpp index 3dd75206a2b..305974b22d9 100644 --- a/dbms/src/Storages/tests/hit_log.cpp +++ b/dbms/src/Storages/tests/hit_log.cpp @@ -98,8 +98,8 @@ try /// create a hit log table - StoragePtr table = StorageLog::create("./", "HitLog", names_and_types_list, - NamesAndTypesList{}, NamesAndTypesList{}, ColumnDefaults{}, DEFAULT_MAX_COMPRESS_BLOCK_SIZE); + StoragePtr table = StorageLog::create( + "./", "HitLog", ColumnsDescription{names_and_types_list}, DEFAULT_MAX_COMPRESS_BLOCK_SIZE); table->startup(); /// create a description of how to read data from the tab separated dump diff --git a/dbms/src/Storages/tests/storage_log.cpp b/dbms/src/Storages/tests/storage_log.cpp index da53fcc356d..edad84b5b8c 100644 --- a/dbms/src/Storages/tests/storage_log.cpp +++ b/dbms/src/Storages/tests/storage_log.cpp @@ -25,8 +25,8 @@ try names_and_types.emplace_back("a", std::make_shared()); names_and_types.emplace_back("b", std::make_shared()); - StoragePtr table = StorageLog::create("./", "test", names_and_types, - NamesAndTypesList{}, NamesAndTypesList{}, ColumnDefaults{}, DEFAULT_MAX_COMPRESS_BLOCK_SIZE); + StoragePtr table = StorageLog::create( + "./", "test", ColumnsDescription{names_and_types}, DEFAULT_MAX_COMPRESS_BLOCK_SIZE); table->startup(); /// write into it @@ -36,7 +36,7 @@ try { ColumnWithTypeAndName column; column.name = "a"; - column.type = table->columns.getColumn("a").type; + column.type = table->getColumn("a").type; auto col = column.type->createColumn(); ColumnUInt64::Container & vec = typeid_cast(*col).getData(); @@ -51,7 +51,7 @@ try { ColumnWithTypeAndName column; column.name = "b"; - column.type = table->columns.getColumn("b").type; + column.type = table->getColumn("b").type; auto col = column.type->createColumn(); ColumnUInt8::Container & vec = typeid_cast(*col).getData(); diff --git a/dbms/src/TableFunctions/TableFunctionMerge.cpp b/dbms/src/TableFunctions/TableFunctionMerge.cpp index b471969b847..2a12dbca275 100644 --- a/dbms/src/TableFunctions/TableFunctionMerge.cpp +++ b/dbms/src/TableFunctions/TableFunctionMerge.cpp @@ -51,7 +51,7 @@ static NamesAndTypesList chooseColumns(const String & source_database, const Str throw Exception("Error while executing table function merge. In database " + source_database + " no one matches regular expression: " + table_name_regexp_, ErrorCodes::UNKNOWN_TABLE); - return any_table->columns.getList(); + return any_table->columns.getPhysical(); } @@ -79,12 +79,7 @@ StoragePtr TableFunctionMerge::executeImpl(const ASTPtr & ast_function, const Co auto res = StorageMerge::create( getName(), - ColumnsDescription{ - chooseColumns(source_database, table_name_regexp, context), - NamesAndTypesList{}, - NamesAndTypesList{}, - ColumnDefaults{}, - }, + ColumnsDescription{chooseColumns(source_database, table_name_regexp, context)}, source_database, table_name_regexp, context); diff --git a/dbms/src/TableFunctions/TableFunctionMySQL.cpp b/dbms/src/TableFunctions/TableFunctionMySQL.cpp index 1ced8800255..1f7839ada86 100644 --- a/dbms/src/TableFunctions/TableFunctionMySQL.cpp +++ b/dbms/src/TableFunctions/TableFunctionMySQL.cpp @@ -152,7 +152,7 @@ StoragePtr TableFunctionMySQL::executeImpl(const ASTPtr & ast_function, const Co std::move(pool), database_name, table_name, - ColumnsDescription{columns, NamesAndTypesList{}, NamesAndTypesList{}, ColumnDefaults{}}); + ColumnsDescription{columns}); res->startup(); return res; diff --git a/dbms/src/TableFunctions/TableFunctionODBC.cpp b/dbms/src/TableFunctions/TableFunctionODBC.cpp index cacdbe29670..b75e48de054 100644 --- a/dbms/src/TableFunctions/TableFunctionODBC.cpp +++ b/dbms/src/TableFunctions/TableFunctionODBC.cpp @@ -113,8 +113,7 @@ StoragePtr TableFunctionODBC::executeImpl(const ASTPtr & ast_function, const Con columns.emplace_back(reinterpret_cast(column_name), getDataType(type)); } - auto result = StorageODBC::create(table_name, connection_string, "", table_name, - ColumnsDescription{columns, NamesAndTypesList{}, NamesAndTypesList{}, ColumnDefaults{}}); + auto result = StorageODBC::create(table_name, connection_string, "", table_name, ColumnsDescription{columns}); result->startup(); return result; } diff --git a/dbms/src/TableFunctions/getStructureOfRemoteTable.cpp b/dbms/src/TableFunctions/getStructureOfRemoteTable.cpp index 181448714ba..2c56688c522 100644 --- a/dbms/src/TableFunctions/getStructureOfRemoteTable.cpp +++ b/dbms/src/TableFunctions/getStructureOfRemoteTable.cpp @@ -32,7 +32,7 @@ NamesAndTypesList getStructureOfRemoteTable( const auto & shard_info = cluster.getAnyShardInfo(); if (shard_info.isLocal()) - return context.getTable(database, table)->columns.getList(); + return context.getTable(database, table)->columns.getPhysical(); auto input = std::make_shared(shard_info.pool, query, InterpreterDescribeQuery::getSampleBlock(), context); input->setPoolMode(PoolMode::GET_ONE); From de1db216a50ea83323d8c2cb9dae21e80d0fbee2 Mon Sep 17 00:00:00 2001 From: Alexey Zatelepin Date: Mon, 12 Mar 2018 16:47:01 +0300 Subject: [PATCH 4/8] getStructureOfRemoteTable returns info about column default kinds [#CLICKHOUSE-3241] --- .../Interpreters/InterpreterAlterQuery.cpp | 4 +- .../Interpreters/InterpreterCreateQuery.cpp | 12 +++--- .../Interpreters/InterpreterDescribeQuery.cpp | 2 +- .../Interpreters/InterpreterSelectQuery.cpp | 5 ++- dbms/src/Server/ClusterCopier.cpp | 4 +- dbms/src/Storages/AlterCommands.cpp | 28 ++++++------ dbms/src/Storages/AlterCommands.h | 6 +-- dbms/src/Storages/ColumnDefault.cpp | 26 +++++------ dbms/src/Storages/ColumnDefault.h | 8 ++-- dbms/src/Storages/ColumnsDescription.cpp | 16 +++---- dbms/src/Storages/StorageCatBoostPool.cpp | 2 +- dbms/src/Storages/StorageDistributed.cpp | 5 +-- dbms/src/Storages/StorageDistributed.h | 2 +- .../Storages/System/StorageSystemColumns.cpp | 2 +- .../System/StorageSystemPartsColumns.cpp | 2 +- .../getStructureOfRemoteTable.cpp | 43 ++++++++++++++----- .../getStructureOfRemoteTable.h | 4 +- 17 files changed, 97 insertions(+), 74 deletions(-) diff --git a/dbms/src/Interpreters/InterpreterAlterQuery.cpp b/dbms/src/Interpreters/InterpreterAlterQuery.cpp index 1fb3e1e80db..08254c8e8f8 100644 --- a/dbms/src/Interpreters/InterpreterAlterQuery.cpp +++ b/dbms/src/Interpreters/InterpreterAlterQuery.cpp @@ -110,7 +110,7 @@ void InterpreterAlterQuery::parseAlter( } if (ast_col_decl.default_expression) { - command.default_type = columnDefaultTypeFromString(ast_col_decl.default_specifier); + command.default_kind = columnDefaultKindFromString(ast_col_decl.default_specifier); command.default_expression = ast_col_decl.default_expression; } @@ -157,7 +157,7 @@ void InterpreterAlterQuery::parseAlter( if (ast_col_decl.default_expression) { - command.default_type = columnDefaultTypeFromString(ast_col_decl.default_specifier); + command.default_kind = columnDefaultKindFromString(ast_col_decl.default_specifier); command.default_expression = ast_col_decl.default_expression; } diff --git a/dbms/src/Interpreters/InterpreterCreateQuery.cpp b/dbms/src/Interpreters/InterpreterCreateQuery.cpp index e5d20648fbd..43f4336f896 100644 --- a/dbms/src/Interpreters/InterpreterCreateQuery.cpp +++ b/dbms/src/Interpreters/InterpreterCreateQuery.cpp @@ -246,7 +246,7 @@ static ColumnsAndDefaults parseColumns(const ASTExpressionList & column_list_ast explicit_type = block.getByName(column_name).type; defaults.emplace(column_name, ColumnDefault{ - columnDefaultTypeFromString(col_decl_ptr->default_specifier), + columnDefaultKindFromString(col_decl_ptr->default_specifier), col_decl_ptr->default_expression }); } @@ -256,7 +256,7 @@ static ColumnsAndDefaults parseColumns(const ASTExpressionList & column_list_ast } -static NamesAndTypesList removeAndReturnColumns(ColumnsAndDefaults & columns_and_defaults, const ColumnDefaultType type) +static NamesAndTypesList removeAndReturnColumns(ColumnsAndDefaults & columns_and_defaults, const ColumnDefaultKind kind) { auto & columns = columns_and_defaults.first; auto & defaults = columns_and_defaults.second; @@ -266,7 +266,7 @@ static NamesAndTypesList removeAndReturnColumns(ColumnsAndDefaults & columns_and for (auto it = std::begin(columns); it != std::end(columns);) { const auto jt = defaults.find(it->name); - if (jt != std::end(defaults) && jt->second.type == type) + if (jt != std::end(defaults) && jt->second.kind == kind) { removed.push_back(*it); it = columns.erase(it); @@ -323,7 +323,7 @@ ASTPtr InterpreterCreateQuery::formatColumns(const ColumnsDescription & columns) const auto it = columns.defaults.find(column.name); if (it != std::end(columns.defaults)) { - column_declaration->default_specifier = toString(it->second.type); + column_declaration->default_specifier = toString(it->second.kind); column_declaration->default_expression = it->second.expression->clone(); } @@ -339,8 +339,8 @@ ColumnsDescription InterpreterCreateQuery::getColumnsDescription(const ASTExpres ColumnsDescription res; auto && columns_and_defaults = parseColumns(columns, context); - res.materialized = removeAndReturnColumns(columns_and_defaults, ColumnDefaultType::Materialized); - res.aliases = removeAndReturnColumns(columns_and_defaults, ColumnDefaultType::Alias); + res.materialized = removeAndReturnColumns(columns_and_defaults, ColumnDefaultKind::Materialized); + res.aliases = removeAndReturnColumns(columns_and_defaults, ColumnDefaultKind::Alias); res.ordinary = std::move(columns_and_defaults.first); res.defaults = std::move(columns_and_defaults.second); diff --git a/dbms/src/Interpreters/InterpreterDescribeQuery.cpp b/dbms/src/Interpreters/InterpreterDescribeQuery.cpp index c03034af49a..07eefbd913b 100644 --- a/dbms/src/Interpreters/InterpreterDescribeQuery.cpp +++ b/dbms/src/Interpreters/InterpreterDescribeQuery.cpp @@ -119,7 +119,7 @@ BlockInputStreamPtr InterpreterDescribeQuery::executeImpl() } else { - res_columns[2]->insert(toString(it->second.type)); + res_columns[2]->insert(toString(it->second.kind)); res_columns[3]->insert(queryToString(it->second.expression)); } } diff --git a/dbms/src/Interpreters/InterpreterSelectQuery.cpp b/dbms/src/Interpreters/InterpreterSelectQuery.cpp index 04e05e975ac..ba93c737f08 100644 --- a/dbms/src/Interpreters/InterpreterSelectQuery.cpp +++ b/dbms/src/Interpreters/InterpreterSelectQuery.cpp @@ -521,7 +521,7 @@ QueryProcessingStage::Enum InterpreterSelectQuery::executeFetchColumns(Pipeline for (const auto & column : required_columns) { const auto default_it = storage->columns.defaults.find(column); - if (default_it != std::end(storage->columns.defaults) && default_it->second.type == ColumnDefaultType::Alias) + if (default_it != std::end(storage->columns.defaults) && default_it->second.kind == ColumnDefaultKind::Alias) { alias_columns_required = true; break; @@ -536,7 +536,7 @@ QueryProcessingStage::Enum InterpreterSelectQuery::executeFetchColumns(Pipeline for (const auto & column : required_columns) { const auto default_it = storage->columns.defaults.find(column); - if (default_it != std::end(storage->columns.defaults) && default_it->second.type == ColumnDefaultType::Alias) + if (default_it != std::end(storage->columns.defaults) && default_it->second.kind == ColumnDefaultKind::Alias) required_columns_expr_list->children.emplace_back(setAlias(default_it->second.expression->clone(), column)); else required_columns_expr_list->children.emplace_back(std::make_shared(column)); @@ -711,6 +711,7 @@ QueryProcessingStage::Enum InterpreterSelectQuery::executeFetchColumns(Pipeline else throw Exception("Logical error in InterpreterSelectQuery: nowhere to read", ErrorCodes::LOGICAL_ERROR); + StackTrace st; /// Aliases in table declaration. if (from_stage == QueryProcessingStage::FetchColumns && alias_actions) { diff --git a/dbms/src/Server/ClusterCopier.cpp b/dbms/src/Server/ClusterCopier.cpp index 77172a22b23..3ae6a9ed205 100644 --- a/dbms/src/Server/ClusterCopier.cpp +++ b/dbms/src/Server/ClusterCopier.cpp @@ -1120,8 +1120,8 @@ protected: if (!column.default_specifier.empty()) { - ColumnDefaultType type = columnDefaultTypeFromString(column.default_specifier); - if (type == ColumnDefaultType::Materialized || type == ColumnDefaultType::Alias) + ColumnDefaultKind kind = columnDefaultKindFromString(column.default_specifier); + if (kind == ColumnDefaultKind::Materialized || kind == ColumnDefaultKind::Alias) continue; } diff --git a/dbms/src/Storages/AlterCommands.cpp b/dbms/src/Storages/AlterCommands.cpp index 184f63c070f..6c16acf5300 100644 --- a/dbms/src/Storages/AlterCommands.cpp +++ b/dbms/src/Storages/AlterCommands.cpp @@ -57,17 +57,17 @@ void AlterCommand::apply(ColumnsDescription & columns_description) const columns.emplace(insert_it, column_name, data_type); }; - if (default_type == ColumnDefaultType::Default) + if (default_kind == ColumnDefaultKind::Default) add_column(columns_description.ordinary); - else if (default_type == ColumnDefaultType::Materialized) + else if (default_kind == ColumnDefaultKind::Materialized) add_column(columns_description.materialized); - else if (default_type == ColumnDefaultType::Alias) + else if (default_kind == ColumnDefaultKind::Alias) add_column(columns_description.aliases); else - throw Exception{"Unknown ColumnDefaultType value", ErrorCodes::LOGICAL_ERROR}; + throw Exception{"Unknown ColumnDefaultKind value", ErrorCodes::LOGICAL_ERROR}; if (default_expression) - columns_description.defaults.emplace(column_name, ColumnDefault{default_type, default_expression}); + columns_description.defaults.emplace(column_name, ColumnDefault{default_kind, default_expression}); /// Slow, because each time a list is copied columns_description.ordinary = Nested::flatten(columns_description.ordinary); @@ -103,12 +103,12 @@ void AlterCommand::apply(ColumnsDescription & columns_description) const { const auto default_it = columns_description.defaults.find(column_name); const auto had_default_expr = default_it != std::end(columns_description.defaults); - const auto old_default_type = had_default_expr ? default_it->second.type : ColumnDefaultType{}; + const auto old_default_kind = had_default_expr ? default_it->second.kind : ColumnDefaultKind{}; /// target column list auto & new_columns = - default_type == ColumnDefaultType::Default ? columns_description.ordinary - : default_type == ColumnDefaultType::Materialized ? columns_description.materialized + default_kind == ColumnDefaultKind::Default ? columns_description.ordinary + : default_kind == ColumnDefaultKind::Materialized ? columns_description.materialized : columns_description.aliases; /// find column or throw exception @@ -124,12 +124,12 @@ void AlterCommand::apply(ColumnsDescription & columns_description) const }; /// if default types differ, remove column from the old list, then add to the new list - if (default_type != old_default_type) + if (default_kind != old_default_kind) { /// source column list auto & old_columns = - old_default_type == ColumnDefaultType::Default ? columns_description.ordinary - : old_default_type == ColumnDefaultType::Materialized ? columns_description.materialized + old_default_kind == ColumnDefaultKind::Default ? columns_description.ordinary + : old_default_kind == ColumnDefaultKind::Materialized ? columns_description.materialized : columns_description.aliases; const auto old_column_it = find_column(old_columns); @@ -138,7 +138,7 @@ void AlterCommand::apply(ColumnsDescription & columns_description) const /// do not forget to change the default type of old column if (had_default_expr) - columns_description.defaults[column_name].type = default_type; + columns_description.defaults[column_name].kind = default_kind; } /// find column in one of three column lists @@ -150,7 +150,7 @@ void AlterCommand::apply(ColumnsDescription & columns_description) const columns_description.defaults.erase(column_name); else if (default_expression && !had_default_expr) /// new column has a default expression while the old one had not, add it it column_defaults - columns_description.defaults.emplace(column_name, ColumnDefault{default_type, default_expression}); + columns_description.defaults.emplace(column_name, ColumnDefault{default_kind, default_expression}); else if (had_default_expr) /// both old and new columns have default expression, update it columns_description.defaults[column_name].expression = default_expression; @@ -333,7 +333,7 @@ void AlterCommands::validate(IStorage * table, const Context & context) /// add a new alter command to modify existing column this->emplace_back(AlterCommand{ AlterCommand::MODIFY_COLUMN, column_name, explicit_type, - default_it->second.type, default_it->second.expression + default_it->second.kind, default_it->second.expression }); command_ptr = &this->back(); diff --git a/dbms/src/Storages/AlterCommands.h b/dbms/src/Storages/AlterCommands.h index 44c44e8bfd0..306d5cec54c 100644 --- a/dbms/src/Storages/AlterCommands.h +++ b/dbms/src/Storages/AlterCommands.h @@ -27,7 +27,7 @@ struct AlterCommand /// For ADD and MODIFY, a new column type. DataTypePtr data_type; - ColumnDefaultType default_type{}; + ColumnDefaultKind default_kind{}; ASTPtr default_expression{}; /// For ADD - after which column to add a new one. If an empty string, add to the end. To add to the beginning now it is impossible. @@ -47,9 +47,9 @@ struct AlterCommand AlterCommand() = default; AlterCommand(const Type type, const String & column_name, const DataTypePtr & data_type, - const ColumnDefaultType default_type, const ASTPtr & default_expression, + const ColumnDefaultKind default_kind, const ASTPtr & default_expression, const String & after_column = String{}) - : type{type}, column_name{column_name}, data_type{data_type}, default_type{default_type}, + : type{type}, column_name{column_name}, data_type{data_type}, default_kind{default_kind}, default_expression{default_expression}, after_column{after_column} {} }; diff --git a/dbms/src/Storages/ColumnDefault.cpp b/dbms/src/Storages/ColumnDefault.cpp index dcdfe2a6e51..0ff885ab1e5 100644 --- a/dbms/src/Storages/ColumnDefault.cpp +++ b/dbms/src/Storages/ColumnDefault.cpp @@ -6,12 +6,12 @@ namespace DB { -ColumnDefaultType columnDefaultTypeFromString(const std::string & str) +ColumnDefaultKind columnDefaultKindFromString(const std::string & str) { - static const std::unordered_map map{ - { "DEFAULT", ColumnDefaultType::Default }, - { "MATERIALIZED", ColumnDefaultType::Materialized }, - { "ALIAS", ColumnDefaultType::Alias } + static const std::unordered_map map{ + { "DEFAULT", ColumnDefaultKind::Default }, + { "MATERIALIZED", ColumnDefaultKind::Materialized }, + { "ALIAS", ColumnDefaultKind::Alias } }; const auto it = map.find(str); @@ -19,22 +19,22 @@ ColumnDefaultType columnDefaultTypeFromString(const std::string & str) } -std::string toString(const ColumnDefaultType type) +std::string toString(const ColumnDefaultKind kind) { - static const std::unordered_map map{ - { ColumnDefaultType::Default, "DEFAULT" }, - { ColumnDefaultType::Materialized, "MATERIALIZED" }, - { ColumnDefaultType::Alias, "ALIAS" } + static const std::unordered_map map{ + { ColumnDefaultKind::Default, "DEFAULT" }, + { ColumnDefaultKind::Materialized, "MATERIALIZED" }, + { ColumnDefaultKind::Alias, "ALIAS" } }; - const auto it = map.find(type); - return it != std::end(map) ? it->second : throw Exception{"Invalid ColumnDefaultType"}; + const auto it = map.find(kind); + return it != std::end(map) ? it->second : throw Exception{"Invalid ColumnDefaultKind"}; } bool operator==(const ColumnDefault & lhs, const ColumnDefault & rhs) { - return lhs.type == rhs.type && queryToString(lhs.expression) == queryToString(rhs.expression); + return lhs.kind == rhs.kind && queryToString(lhs.expression) == queryToString(rhs.expression); } } diff --git a/dbms/src/Storages/ColumnDefault.h b/dbms/src/Storages/ColumnDefault.h index 33f17952d08..95eb4d5b597 100644 --- a/dbms/src/Storages/ColumnDefault.h +++ b/dbms/src/Storages/ColumnDefault.h @@ -9,7 +9,7 @@ namespace DB { -enum class ColumnDefaultType +enum class ColumnDefaultKind { Default, Materialized, @@ -17,13 +17,13 @@ enum class ColumnDefaultType }; -ColumnDefaultType columnDefaultTypeFromString(const std::string & str); -std::string toString(const ColumnDefaultType type); +ColumnDefaultKind columnDefaultKindFromString(const std::string & str); +std::string toString(const ColumnDefaultKind type); struct ColumnDefault { - ColumnDefaultType type; + ColumnDefaultKind kind; ASTPtr expression; }; diff --git a/dbms/src/Storages/ColumnsDescription.cpp b/dbms/src/Storages/ColumnsDescription.cpp index 95b5001a1f9..683f10f324d 100644 --- a/dbms/src/Storages/ColumnsDescription.cpp +++ b/dbms/src/Storages/ColumnsDescription.cpp @@ -88,7 +88,7 @@ String ColumnsDescription::toString() const else writeChar('\t', buf); - writeString(DB::toString(it->second.type), buf); + writeString(DB::toString(it->second.kind), buf); writeChar('\t', buf); writeString(queryToString(it->second.expression), buf); writeChar('\n', buf); @@ -134,9 +134,9 @@ ColumnsDescription ColumnsDescription::parse(const String & str) } assertChar('\t', buf); - String default_type_str; - readString(default_type_str, buf); - const auto default_type = columnDefaultTypeFromString(default_type_str); + String default_kind_str; + readString(default_kind_str, buf); + const auto default_kind = columnDefaultKindFromString(default_kind_str); assertChar('\t', buf); String default_expr_str; @@ -147,14 +147,14 @@ ColumnsDescription ColumnsDescription::parse(const String & str) const auto end = begin + default_expr_str.size(); ASTPtr default_expr = parseQuery(expr_parser, begin, end, "default expression"); - if (ColumnDefaultType::Default == default_type) + if (ColumnDefaultKind::Default == default_kind) result.ordinary.emplace_back(column_name, std::move(type)); - else if (ColumnDefaultType::Materialized == default_type) + else if (ColumnDefaultKind::Materialized == default_kind) result.materialized.emplace_back(column_name, std::move(type)); - else if (ColumnDefaultType::Alias == default_type) + else if (ColumnDefaultKind::Alias == default_kind) result.aliases.emplace_back(column_name, std::move(type)); - result.defaults.emplace(column_name, ColumnDefault{default_type, default_expr}); + result.defaults.emplace(column_name, ColumnDefault{default_kind, default_expr}); } assertEOF(buf); diff --git a/dbms/src/Storages/StorageCatBoostPool.cpp b/dbms/src/Storages/StorageCatBoostPool.cpp index 1a973f7dd75..d20da89b170 100644 --- a/dbms/src/Storages/StorageCatBoostPool.cpp +++ b/dbms/src/Storages/StorageCatBoostPool.cpp @@ -243,7 +243,7 @@ void StorageCatBoostPool::createSampleBlockAndColumns() if (!desc.alias.empty()) { auto alias = std::make_shared(desc.column_name); - columns.defaults[desc.alias] = {ColumnDefaultType::Alias, alias}; + columns.defaults[desc.alias] = {ColumnDefaultKind::Alias, alias}; columns.aliases.emplace_back(desc.alias, type); } diff --git a/dbms/src/Storages/StorageDistributed.cpp b/dbms/src/Storages/StorageDistributed.cpp index c010ca768d1..e3ff7f30369 100644 --- a/dbms/src/Storages/StorageDistributed.cpp +++ b/dbms/src/Storages/StorageDistributed.cpp @@ -151,15 +151,14 @@ StorageDistributed::StorageDistributed( StoragePtr StorageDistributed::createWithOwnCluster( const std::string & name_, - const NamesAndTypesList & columns_, + const ColumnsDescription & columns_, const String & remote_database_, const String & remote_table_, ClusterPtr & owned_cluster_, const Context & context_) { auto res = ext::shared_ptr_helper::create( - name_, ColumnsDescription{columns_}, - remote_database_, remote_table_, String{}, context_, ASTPtr(), String()); + name_, columns_, remote_database_, remote_table_, String{}, context_, ASTPtr(), String()); res->owned_cluster = owned_cluster_; diff --git a/dbms/src/Storages/StorageDistributed.h b/dbms/src/Storages/StorageDistributed.h index 22e24c3f9fb..451c9f12d8a 100644 --- a/dbms/src/Storages/StorageDistributed.h +++ b/dbms/src/Storages/StorageDistributed.h @@ -35,7 +35,7 @@ public: static StoragePtr createWithOwnCluster( const std::string & table_name_, - const NamesAndTypesList & columns_, + const ColumnsDescription & columns_, const String & remote_database_, /// database on remote servers. const String & remote_table_, /// The name of the table on the remote servers. ClusterPtr & owned_cluster_, diff --git a/dbms/src/Storages/System/StorageSystemColumns.cpp b/dbms/src/Storages/System/StorageSystemColumns.cpp index 75c8afa1515..83c5dca9117 100644 --- a/dbms/src/Storages/System/StorageSystemColumns.cpp +++ b/dbms/src/Storages/System/StorageSystemColumns.cpp @@ -171,7 +171,7 @@ BlockInputStreams StorageSystemColumns::read( } else { - res_columns[i++]->insert(toString(it->second.type)); + res_columns[i++]->insert(toString(it->second.kind)); res_columns[i++]->insert(queryToString(it->second.expression)); } } diff --git a/dbms/src/Storages/System/StorageSystemPartsColumns.cpp b/dbms/src/Storages/System/StorageSystemPartsColumns.cpp index 326dfbf30ad..d804504f13a 100644 --- a/dbms/src/Storages/System/StorageSystemPartsColumns.cpp +++ b/dbms/src/Storages/System/StorageSystemPartsColumns.cpp @@ -72,7 +72,7 @@ void StorageSystemPartsColumns::processNextStorage(MutableColumns & columns, con const auto it = column_defaults.find(column.name); if (it != std::end(column_defaults)) { - column_info.default_kind = toString(it->second.type); + column_info.default_kind = toString(it->second.kind); column_info.default_expression = queryToString(it->second.expression); } diff --git a/dbms/src/TableFunctions/getStructureOfRemoteTable.cpp b/dbms/src/TableFunctions/getStructureOfRemoteTable.cpp index 2c56688c522..6eede709352 100644 --- a/dbms/src/TableFunctions/getStructureOfRemoteTable.cpp +++ b/dbms/src/TableFunctions/getStructureOfRemoteTable.cpp @@ -1,12 +1,12 @@ +#include #include #include #include #include #include #include -#include - -#include +#include +#include namespace DB @@ -18,21 +18,21 @@ namespace ErrorCodes } -NamesAndTypesList getStructureOfRemoteTable( +ColumnsDescription getStructureOfRemoteTable( const Cluster & cluster, const std::string & database, const std::string & table, const Context & context) { - /// Request for a table description - String query = "DESC TABLE " + backQuoteIfNeed(database) + "." + backQuoteIfNeed(table); - NamesAndTypesList res; - /// Send to the first any remote shard. const auto & shard_info = cluster.getAnyShardInfo(); if (shard_info.isLocal()) - return context.getTable(database, table)->columns.getPhysical(); + return context.getTable(database, table)->columns; + + /// Request for a table description + String query = "DESC TABLE " + backQuoteIfNeed(database) + "." + backQuoteIfNeed(table); + ColumnsDescription res; auto input = std::make_shared(shard_info.pool, query, InterpreterDescribeQuery::getSampleBlock(), context); input->setPoolMode(PoolMode::GET_ONE); @@ -41,18 +41,41 @@ NamesAndTypesList getStructureOfRemoteTable( const DataTypeFactory & data_type_factory = DataTypeFactory::instance(); + ParserExpression expr_parser; + while (Block current = input->read()) { ColumnPtr name = current.getByName("name").column; ColumnPtr type = current.getByName("type").column; + ColumnPtr default_kind = current.getByName("default_type").column; + ColumnPtr default_expr = current.getByName("default_expression").column; size_t size = name->size(); for (size_t i = 0; i < size; ++i) { String column_name = (*name)[i].get(); String data_type_name = (*type)[i].get(); + String kind_name = (*default_kind)[i].get(); - res.emplace_back(column_name, data_type_factory.get(data_type_name)); + auto data_type = data_type_factory.get(data_type_name); + + if (kind_name.empty()) + res.ordinary.emplace_back(column_name, std::move(data_type)); + else + { + auto kind = columnDefaultKindFromString(kind_name); + + String expr_str = (*default_expr)[i].get(); + ASTPtr expr = parseQuery(expr_parser, expr_str.data(), expr_str.data() + expr_str.size(), "default expression"); + res.defaults.emplace(column_name, ColumnDefault{kind, expr}); + + if (ColumnDefaultKind::Default == kind) + res.ordinary.emplace_back(column_name, std::move(data_type)); + else if (ColumnDefaultKind::Materialized == kind) + res.materialized.emplace_back(column_name, std::move(data_type)); + else if (ColumnDefaultKind::Alias == kind) + res.aliases.emplace_back(column_name, std::move(data_type)); + } } } diff --git a/dbms/src/TableFunctions/getStructureOfRemoteTable.h b/dbms/src/TableFunctions/getStructureOfRemoteTable.h index 70b3d83b8f1..20417ef50e1 100644 --- a/dbms/src/TableFunctions/getStructureOfRemoteTable.h +++ b/dbms/src/TableFunctions/getStructureOfRemoteTable.h @@ -1,6 +1,6 @@ #pragma once -#include +#include namespace DB @@ -11,7 +11,7 @@ class Context; /// Find the names and types of the table columns on any server in the cluster. /// Used to implement the `remote` table function and others. -NamesAndTypesList getStructureOfRemoteTable( +ColumnsDescription getStructureOfRemoteTable( const Cluster & cluster, const std::string & database, const std::string & table, From 2f573e4f7714e595d993a4c9eafbebe5245fdd18 Mon Sep 17 00:00:00 2001 From: Alexey Zatelepin Date: Mon, 12 Mar 2018 21:38:24 +0300 Subject: [PATCH 5/8] add tests [#CLICKHOUSE-3241] --- ...c_table_functions_and_subqueries.reference | 12 +++--- ...remote_and_columns_with_defaults.reference | 13 +++++++ ...shard_remote_and_columns_with_defaults.sql | 38 +++++++++++++++++++ 3 files changed, 57 insertions(+), 6 deletions(-) create mode 100644 dbms/tests/queries/0_stateless/00604_shard_remote_and_columns_with_defaults.reference create mode 100644 dbms/tests/queries/0_stateless/00604_shard_remote_and_columns_with_defaults.sql diff --git a/dbms/tests/queries/0_stateless/00515_shard_desc_table_functions_and_subqueries.reference b/dbms/tests/queries/0_stateless/00515_shard_desc_table_functions_and_subqueries.reference index 20d07b74593..b0615b7993f 100644 --- a/dbms/tests/queries/0_stateless/00515_shard_desc_table_functions_and_subqueries.reference +++ b/dbms/tests/queries/0_stateless/00515_shard_desc_table_functions_and_subqueries.reference @@ -12,15 +12,15 @@ val4 UInt64 ALIAS val - date Date val UInt64 -val2 UInt8 -val3 UInt8 -val4 UInt64 +val2 UInt8 DEFAULT 42 +val3 UInt8 DEFAULT CAST((val2 + 1) AS UInt8) +val4 UInt64 ALIAS val - date Date val UInt64 -val2 UInt8 -val3 UInt8 -val4 UInt64 +val2 UInt8 DEFAULT 42 +val3 UInt8 DEFAULT CAST((val2 + 1) AS UInt8) +val4 UInt64 ALIAS val - 1 UInt8 - diff --git a/dbms/tests/queries/0_stateless/00604_shard_remote_and_columns_with_defaults.reference b/dbms/tests/queries/0_stateless/00604_shard_remote_and_columns_with_defaults.reference new file mode 100644 index 00000000000..8a5c72b9000 --- /dev/null +++ b/dbms/tests/queries/0_stateless/00604_shard_remote_and_columns_with_defaults.reference @@ -0,0 +1,13 @@ +1 1 +2 2 +*** With a DEFAULT column *** +1 1 +2 2 +*** With a MATERIALIZED column *** +1 +1 2 +*** With an ALIAS column *** +1 +2 +1 2 +2 3 diff --git a/dbms/tests/queries/0_stateless/00604_shard_remote_and_columns_with_defaults.sql b/dbms/tests/queries/0_stateless/00604_shard_remote_and_columns_with_defaults.sql new file mode 100644 index 00000000000..ba955f7848b --- /dev/null +++ b/dbms/tests/queries/0_stateless/00604_shard_remote_and_columns_with_defaults.sql @@ -0,0 +1,38 @@ +DROP TABLE IF EXISTS test.t1; +DROP TABLE IF EXISTS test.t2; +DROP TABLE IF EXISTS test.t3; +DROP TABLE IF EXISTS test.t4; + +CREATE TABLE test.t1(x UInt32, y UInt32) ENGINE TinyLog; +CREATE TABLE test.t2(x UInt32, y UInt32 DEFAULT x + 1) ENGINE TinyLog; +CREATE TABLE test.t3(x UInt32, y UInt32 MATERIALIZED x + 1) ENGINE TinyLog; +CREATE TABLE test.t4(x UInt32, y UInt32 ALIAS x + 1) ENGINE TinyLog; + +INSERT INTO test.t1 VALUES (1, 1); +INSERT INTO test.t2 VALUES (1, 1); +INSERT INTO test.t3 VALUES (1); +INSERT INTO test.t4 VALUES (1); + +INSERT INTO FUNCTION remote('127.0.0.2', test.t1) VALUES (2, 2); +INSERT INTO FUNCTION remote('127.0.0.2', test.t2) VALUES (2, 2); +--TODO: INSERT into remote tables with MATERIALIZED columns. +--INSERT INTO FUNCTION remote('127.0.0.2', test.t3) VALUES (2); +INSERT INTO FUNCTION remote('127.0.0.2', test.t4) VALUES (2); + +SELECT * FROM remote('127.0.0.2', test.t1); + +SELECT '*** With a DEFAULT column ***'; +SELECT * FROM remote('127.0.0.2', test.t2); + +SELECT '*** With a MATERIALIZED column ***'; +SELECT * FROM remote('127.0.0.2', test.t3); +SELECT x, y FROM remote('127.0.0.2', test.t3); + +SELECT '*** With an ALIAS column ***'; +SELECT * FROM remote('127.0.0.2', test.t4); +SELECT x, y FROM remote('127.0.0.2', test.t4); + +DROP TABLE IF EXISTS test.t1; +DROP TABLE IF EXISTS test.t2; +DROP TABLE IF EXISTS test.t3; +DROP TABLE IF EXISTS test.t4; From fdffbe53ac010d2a54822d16c67944b1451b4218 Mon Sep 17 00:00:00 2001 From: Alexey Zatelepin Date: Tue, 13 Mar 2018 17:18:11 +0300 Subject: [PATCH 6/8] encapsulate ColumnsDeclaration [#CLICKHOUSE-3241] --- dbms/src/Analyzers/AnalyzeColumns.cpp | 2 +- dbms/src/Analyzers/ExecuteTableFunctions.cpp | 2 +- dbms/src/Core/iostream_debug_helpers.cpp | 2 +- .../DataStreams/RemoteBlockInputStream.cpp | 2 +- dbms/src/Interpreters/ExpressionAnalyzer.cpp | 9 +++--- .../Interpreters/InterpreterAlterQuery.cpp | 2 +- .../Interpreters/InterpreterCreateQuery.cpp | 2 +- .../Interpreters/InterpreterDescribeQuery.cpp | 4 +-- .../Interpreters/InterpreterInsertQuery.cpp | 6 ++-- .../Interpreters/InterpreterSelectQuery.cpp | 11 ++++---- dbms/src/Storages/AlterCommands.cpp | 4 +-- dbms/src/Storages/ITableDeclaration.cpp | 28 +++++++++++-------- dbms/src/Storages/ITableDeclaration.h | 11 ++++---- .../MergeTree/MergeTreeBlockInputStream.cpp | 5 ++-- .../MergeTree/MergeTreeBlockReadUtils.cpp | 4 +-- dbms/src/Storages/MergeTree/MergeTreeData.cpp | 22 +++++++-------- dbms/src/Storages/MergeTree/MergeTreeData.h | 4 +-- .../MergeTree/MergeTreeDataMerger.cpp | 4 +-- .../Storages/MergeTree/MergeTreeDataPart.cpp | 6 ++-- .../MergeTree/MergeTreeDataSelectExecutor.cpp | 4 +-- .../MergeTree/MergeTreeDataWriter.cpp | 2 +- .../Storages/MergeTree/MergeTreeReadPool.cpp | 5 ++-- .../Storages/MergeTree/MergeTreeReader.cpp | 4 +-- .../MergeTree/MergeTreeWhereOptimizer.cpp | 4 +-- .../ReplicatedMergeTreeAlterThread.cpp | 8 ++---- dbms/src/Storages/StorageBuffer.cpp | 6 ++-- dbms/src/Storages/StorageCatBoostPool.cpp | 3 ++ dbms/src/Storages/StorageDictionary.cpp | 2 +- dbms/src/Storages/StorageDistributed.cpp | 12 ++++---- dbms/src/Storages/StorageJoin.cpp | 2 +- dbms/src/Storages/StorageLog.cpp | 8 +++--- dbms/src/Storages/StorageMerge.cpp | 8 ++++-- dbms/src/Storages/StorageMergeTree.cpp | 12 +++----- dbms/src/Storages/StorageMergeTree.h | 3 ++ dbms/src/Storages/StorageMySQL.cpp | 2 +- dbms/src/Storages/StorageNull.cpp | 7 +++-- dbms/src/Storages/StorageODBC.cpp | 3 +- .../Storages/StorageReplicatedMergeTree.cpp | 21 +++++++------- .../src/Storages/StorageReplicatedMergeTree.h | 3 ++ dbms/src/Storages/StorageTinyLog.cpp | 4 +-- .../StorageSystemAsynchronousMetrics.cpp | 4 +-- .../System/StorageSystemBuildOptions.cpp | 4 +-- .../Storages/System/StorageSystemClusters.cpp | 4 +-- .../Storages/System/StorageSystemColumns.cpp | 8 +++--- .../System/StorageSystemDatabases.cpp | 4 +-- .../System/StorageSystemDictionaries.cpp | 4 +-- .../Storages/System/StorageSystemEvents.cpp | 4 +-- .../System/StorageSystemFunctions.cpp | 4 +-- .../Storages/System/StorageSystemGraphite.cpp | 4 +-- .../Storages/System/StorageSystemMerges.cpp | 4 +-- .../Storages/System/StorageSystemMetrics.cpp | 4 +-- .../Storages/System/StorageSystemModels.cpp | 4 +-- .../Storages/System/StorageSystemNumbers.cpp | 2 +- dbms/src/Storages/System/StorageSystemOne.cpp | 2 +- .../System/StorageSystemPartsBase.cpp | 2 +- .../System/StorageSystemPartsColumns.cpp | 4 +-- .../System/StorageSystemProcesses.cpp | 4 +-- .../Storages/System/StorageSystemReplicas.cpp | 4 +-- .../System/StorageSystemReplicationQueue.cpp | 4 +-- .../Storages/System/StorageSystemSettings.cpp | 4 +-- .../Storages/System/StorageSystemTables.cpp | 4 +-- .../System/StorageSystemZooKeeper.cpp | 4 +-- .../src/TableFunctions/TableFunctionMerge.cpp | 2 +- .../getStructureOfRemoteTable.cpp | 2 +- 64 files changed, 182 insertions(+), 161 deletions(-) diff --git a/dbms/src/Analyzers/AnalyzeColumns.cpp b/dbms/src/Analyzers/AnalyzeColumns.cpp index 78568b080c6..efcbaf66888 100644 --- a/dbms/src/Analyzers/AnalyzeColumns.cpp +++ b/dbms/src/Analyzers/AnalyzeColumns.cpp @@ -173,7 +173,7 @@ ASTPtr createASTIdentifierForColumnInTable(const String & column, const CollectT void createASTsForAllColumnsInTable(const CollectTables::TableInfo & table, ASTs & res) { if (table.storage) - for (const auto & name : table.storage->columns.getNames()) + for (const auto & name : table.storage->getColumns().getNames()) res.emplace_back(createASTIdentifierForColumnInTable(name, table)); else for (size_t i = 0, size = table.structure_of_subquery.columns(); i < size; ++i) diff --git a/dbms/src/Analyzers/ExecuteTableFunctions.cpp b/dbms/src/Analyzers/ExecuteTableFunctions.cpp index 3feabc8f490..37cc3d66eec 100644 --- a/dbms/src/Analyzers/ExecuteTableFunctions.cpp +++ b/dbms/src/Analyzers/ExecuteTableFunctions.cpp @@ -72,7 +72,7 @@ void ExecuteTableFunctions::dump(WriteBuffer & out) const { writeString(table.second->getName(), out); writeCString("\n\n", out); - writeString(table.second->columns.getPhysical().toString(), out); + writeString(table.second->getColumns().getPhysical().toString(), out); writeCString("\n", out); } } diff --git a/dbms/src/Core/iostream_debug_helpers.cpp b/dbms/src/Core/iostream_debug_helpers.cpp index 723b3dc2e90..bd19c453d8d 100644 --- a/dbms/src/Core/iostream_debug_helpers.cpp +++ b/dbms/src/Core/iostream_debug_helpers.cpp @@ -44,7 +44,7 @@ std::ostream & operator<<(std::ostream & stream, const IDataType & what) std::ostream & operator<<(std::ostream & stream, const IStorage & what) { stream << "IStorage(name = " << what.getName() << ", tableName = " << what.getTableName() << ") {" - << what.columns.getPhysical().toString() + << what.getColumns().getPhysical().toString() << "}"; // isRemote supportsSampling supportsFinal supportsPrewhere return stream; diff --git a/dbms/src/DataStreams/RemoteBlockInputStream.cpp b/dbms/src/DataStreams/RemoteBlockInputStream.cpp index 079cabe8dee..b5e20898b89 100644 --- a/dbms/src/DataStreams/RemoteBlockInputStream.cpp +++ b/dbms/src/DataStreams/RemoteBlockInputStream.cpp @@ -138,7 +138,7 @@ void RemoteBlockInputStream::sendExternalTables() { StoragePtr cur = table.second; QueryProcessingStage::Enum stage = QueryProcessingStage::Complete; - BlockInputStreams input = cur->read(cur->columns.getNames(), {}, context, + BlockInputStreams input = cur->read(cur->getColumns().getNames(), {}, context, stage, DEFAULT_BLOCK_SIZE, 1); if (input.size() == 0) res.push_back(std::make_pair(std::make_shared(cur->getSampleBlock()), table.first)); diff --git a/dbms/src/Interpreters/ExpressionAnalyzer.cpp b/dbms/src/Interpreters/ExpressionAnalyzer.cpp index e6190b8f1df..4f0fe52bcff 100644 --- a/dbms/src/Interpreters/ExpressionAnalyzer.cpp +++ b/dbms/src/Interpreters/ExpressionAnalyzer.cpp @@ -190,7 +190,7 @@ ExpressionAnalyzer::ExpressionAnalyzer( } if (storage && source_columns.empty()) - source_columns = storage->columns.getPhysical(); + source_columns = storage->getColumns().getPhysical(); else removeDuplicateColumns(source_columns); @@ -712,7 +712,7 @@ static std::shared_ptr interpretSubquery( /// get columns list for target table auto database_table = getDatabaseAndTableNameFromIdentifier(*table); const auto & storage = context.getTable(database_table.first, database_table.second); - const auto & columns = storage->columns.ordinary; + const auto & columns = storage->getColumns().ordinary; select_expression_list->children.reserve(columns.size()); /// manually substitute column names in place of asterisk @@ -1050,7 +1050,7 @@ void ExpressionAnalyzer::normalizeTreeImpl( if (storage) { /// If we select from a table, get only not MATERIALIZED, not ALIAS columns. - for (const auto & name_type : storage->columns.ordinary) + for (const auto & name_type : storage->getColumns().ordinary) all_columns.emplace_back(std::make_shared(name_type.name)); } else @@ -1147,7 +1147,8 @@ void ExpressionAnalyzer::addAliasColumns() if (!storage) return; - source_columns.insert(std::end(source_columns), std::begin(storage->columns.aliases), std::end(storage->columns.aliases)); + const auto & aliases = storage->getColumns().aliases; + source_columns.insert(std::end(source_columns), std::begin(aliases), std::end(aliases)); } diff --git a/dbms/src/Interpreters/InterpreterAlterQuery.cpp b/dbms/src/Interpreters/InterpreterAlterQuery.cpp index 08254c8e8f8..377f92d6c8d 100644 --- a/dbms/src/Interpreters/InterpreterAlterQuery.cpp +++ b/dbms/src/Interpreters/InterpreterAlterQuery.cpp @@ -200,7 +200,7 @@ void InterpreterAlterQuery::PartitionCommands::validate(const IStorage * table) { String column_name = command.column_name.safeGet(); - if (!table->columns.has(column_name)) + if (!table->getColumns().has(column_name)) { throw Exception("Wrong column name. Cannot find column " + column_name + " to clear it from partition", DB::ErrorCodes::ILLEGAL_COLUMN); diff --git a/dbms/src/Interpreters/InterpreterCreateQuery.cpp b/dbms/src/Interpreters/InterpreterCreateQuery.cpp index 43f4336f896..dd594d2993a 100644 --- a/dbms/src/Interpreters/InterpreterCreateQuery.cpp +++ b/dbms/src/Interpreters/InterpreterCreateQuery.cpp @@ -362,7 +362,7 @@ ColumnsDescription InterpreterCreateQuery::setColumns( } else if (!create.as_table.empty()) { - res = as_storage->columns; + res = as_storage->getColumns(); } else if (create.select) { diff --git a/dbms/src/Interpreters/InterpreterDescribeQuery.cpp b/dbms/src/Interpreters/InterpreterDescribeQuery.cpp index 07eefbd913b..536c554e996 100644 --- a/dbms/src/Interpreters/InterpreterDescribeQuery.cpp +++ b/dbms/src/Interpreters/InterpreterDescribeQuery.cpp @@ -99,8 +99,8 @@ BlockInputStreamPtr InterpreterDescribeQuery::executeImpl() } auto table_lock = table->lockStructure(false, __PRETTY_FUNCTION__); - columns = table->columns.getAll(); - column_defaults = table->columns.defaults; + columns = table->getColumns().getAll(); + column_defaults = table->getColumns().defaults; } Block sample_block = getSampleBlock(); diff --git a/dbms/src/Interpreters/InterpreterInsertQuery.cpp b/dbms/src/Interpreters/InterpreterInsertQuery.cpp index 973fdc31a27..3e054f576a4 100644 --- a/dbms/src/Interpreters/InterpreterInsertQuery.cpp +++ b/dbms/src/Interpreters/InterpreterInsertQuery.cpp @@ -95,7 +95,7 @@ BlockIO InterpreterInsertQuery::execute() auto table_lock = table->lockStructure(true, __PRETTY_FUNCTION__); - NamesAndTypesList required_columns = table->columns.getPhysical(); + NamesAndTypesList required_columns = table->getColumns().getPhysical(); /// We create a pipeline of several streams, into which we will write data. BlockOutputStreamPtr out; @@ -103,7 +103,7 @@ BlockIO InterpreterInsertQuery::execute() out = std::make_shared(query.database, query.table, table, context, query_ptr, query.no_destination); out = std::make_shared( - out, getSampleBlock(query, table), required_columns, table->columns.defaults, context); + out, getSampleBlock(query, table), required_columns, table->getColumns().defaults, context); /// Do not squash blocks if it is a sync INSERT into Distributed if (!(context.getSettingsRef().insert_distributed_sync && table->getName() == "Distributed")) @@ -135,7 +135,7 @@ BlockIO InterpreterInsertQuery::execute() if (!allow_materialized) { Block in_header = res.in->getHeader(); - for (const auto & name_type : table->columns.materialized) + for (const auto & name_type : table->getColumns().materialized) if (in_header.has(name_type.name)) throw Exception("Cannot insert column " + name_type.name + ", because it is MATERIALIZED column.", ErrorCodes::ILLEGAL_COLUMN); } diff --git a/dbms/src/Interpreters/InterpreterSelectQuery.cpp b/dbms/src/Interpreters/InterpreterSelectQuery.cpp index ba93c737f08..49a46abeb13 100644 --- a/dbms/src/Interpreters/InterpreterSelectQuery.cpp +++ b/dbms/src/Interpreters/InterpreterSelectQuery.cpp @@ -516,12 +516,13 @@ QueryProcessingStage::Enum InterpreterSelectQuery::executeFetchColumns(Pipeline /// Are ALIAS columns required for query execution? auto alias_columns_required = false; - if (storage && !storage->columns.aliases.empty()) + if (storage && !storage->getColumns().aliases.empty()) { + const auto & column_defaults = storage->getColumns().defaults; for (const auto & column : required_columns) { - const auto default_it = storage->columns.defaults.find(column); - if (default_it != std::end(storage->columns.defaults) && default_it->second.kind == ColumnDefaultKind::Alias) + const auto default_it = column_defaults.find(column); + if (default_it != std::end(column_defaults) && default_it->second.kind == ColumnDefaultKind::Alias) { alias_columns_required = true; break; @@ -535,8 +536,8 @@ QueryProcessingStage::Enum InterpreterSelectQuery::executeFetchColumns(Pipeline for (const auto & column : required_columns) { - const auto default_it = storage->columns.defaults.find(column); - if (default_it != std::end(storage->columns.defaults) && default_it->second.kind == ColumnDefaultKind::Alias) + const auto default_it = column_defaults.find(column); + if (default_it != std::end(column_defaults) && default_it->second.kind == ColumnDefaultKind::Alias) required_columns_expr_list->children.emplace_back(setAlias(default_it->second.expression->clone(), column)); else required_columns_expr_list->children.emplace_back(std::make_shared(column)); diff --git a/dbms/src/Storages/AlterCommands.cpp b/dbms/src/Storages/AlterCommands.cpp index 6c16acf5300..274f9315f5f 100644 --- a/dbms/src/Storages/AlterCommands.cpp +++ b/dbms/src/Storages/AlterCommands.cpp @@ -177,8 +177,8 @@ void AlterCommands::apply(ColumnsDescription & columns_description) const void AlterCommands::validate(IStorage * table, const Context & context) { - auto all_columns = table->columns.getAll(); - auto defaults = table->columns.defaults; + auto all_columns = table->getColumns().getAll(); + auto defaults = table->getColumns().defaults; std::vector> defaulted_columns{}; diff --git a/dbms/src/Storages/ITableDeclaration.cpp b/dbms/src/Storages/ITableDeclaration.cpp index b5a6c035b96..783b0b2a7ed 100644 --- a/dbms/src/Storages/ITableDeclaration.cpp +++ b/dbms/src/Storages/ITableDeclaration.cpp @@ -24,14 +24,22 @@ namespace ErrorCodes } +void ITableDeclaration::setColumns(ColumnsDescription columns_) +{ + if (columns_.ordinary.empty()) + throw Exception("Empty list of columns passed", ErrorCodes::EMPTY_LIST_OF_COLUMNS_PASSED); + columns = std::move(columns_); +} + + bool ITableDeclaration::hasColumn(const String & column_name) const { - return columns.has(column_name); /// By default, we assume that there are no virtual columns in the storage. + return getColumns().has(column_name); /// By default, we assume that there are no virtual columns in the storage. } NameAndTypePair ITableDeclaration::getColumn(const String & column_name) const { - return columns.get(column_name); /// By default, we assume that there are no virtual columns in the storage. + return getColumns().get(column_name); /// By default, we assume that there are no virtual columns in the storage. } @@ -39,7 +47,7 @@ Block ITableDeclaration::getSampleBlock() const { Block res; - for (const auto & col : boost::join(columns.ordinary, columns.materialized)) + for (const auto & col : boost::join(getColumns().ordinary, getColumns().materialized)) res.insert({ col.type->createColumn(), col.type, col.name }); return res; @@ -50,7 +58,7 @@ Block ITableDeclaration::getSampleBlockNonMaterialized() const { Block res; - for (const auto & col : columns.ordinary) + for (const auto & col : getColumns().ordinary) res.insert({ col.type->createColumn(), col.type, col.name }); return res; @@ -111,7 +119,7 @@ static NamesAndTypesMap getColumnsMap(const Args &... args) void ITableDeclaration::check(const Names & column_names) const { - const NamesAndTypesList & available_columns = columns.getPhysical(); + const NamesAndTypesList & available_columns = getColumns().getPhysical(); if (column_names.empty()) throw Exception("Empty list of columns queried. There are columns: " + listOfColumns(available_columns), @@ -139,7 +147,7 @@ void ITableDeclaration::check(const Names & column_names) const void ITableDeclaration::check(const NamesAndTypesList & provided_columns) const { - const NamesAndTypesList & available_columns = columns.getPhysical(); + const NamesAndTypesList & available_columns = getColumns().getPhysical(); const auto columns_map = getColumnsMap(available_columns); using UniqueStrings = google::dense_hash_set; @@ -167,7 +175,7 @@ void ITableDeclaration::check(const NamesAndTypesList & provided_columns) const void ITableDeclaration::check(const NamesAndTypesList & provided_columns, const Names & column_names) const { - const NamesAndTypesList & available_columns = columns.getPhysical(); + const NamesAndTypesList & available_columns = getColumns().getPhysical(); const auto available_columns_map = getColumnsMap(available_columns); const NamesAndTypesMap & provided_columns_map = getColumnsMap(provided_columns); @@ -204,7 +212,7 @@ void ITableDeclaration::check(const NamesAndTypesList & provided_columns, const void ITableDeclaration::check(const Block & block, bool need_all) const { - const NamesAndTypesList & available_columns = columns.getPhysical(); + const NamesAndTypesList & available_columns = getColumns().getPhysical(); const auto columns_map = getColumnsMap(available_columns); using NameSet = std::unordered_set; @@ -242,10 +250,8 @@ void ITableDeclaration::check(const Block & block, bool need_all) const ITableDeclaration::ITableDeclaration(ColumnsDescription columns_) - : columns{std::move(columns_)} { - if (columns.ordinary.empty()) - throw Exception("Empty list of columns passed to storage constructor", ErrorCodes::EMPTY_LIST_OF_COLUMNS_PASSED); + setColumns(std::move(columns_)); } } diff --git a/dbms/src/Storages/ITableDeclaration.h b/dbms/src/Storages/ITableDeclaration.h index 4c4ea59e4b5..270ab175576 100644 --- a/dbms/src/Storages/ITableDeclaration.h +++ b/dbms/src/Storages/ITableDeclaration.h @@ -12,12 +12,12 @@ namespace DB class ITableDeclaration { public: - /** Get a description of any column by its name. - */ - virtual NameAndTypePair getColumn(const String & column_name) const; + virtual const ColumnsDescription & getColumns() const { return columns; } + virtual void setColumns(ColumnsDescription columns_); - /** Is there a column with that name. - */ + /// NOTE: These methods should include virtual columns, but should NOT include ALIAS columns + /// (they are treated separately). + virtual NameAndTypePair getColumn(const String & column_name) const; virtual bool hasColumn(const String & column_name) const; Block getSampleBlock() const; @@ -48,6 +48,7 @@ public: explicit ITableDeclaration(ColumnsDescription columns_); virtual ~ITableDeclaration() = default; +private: ColumnsDescription columns; }; diff --git a/dbms/src/Storages/MergeTree/MergeTreeBlockInputStream.cpp b/dbms/src/Storages/MergeTree/MergeTreeBlockInputStream.cpp index 6fb7c8fb611..a6d2958a786 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeBlockInputStream.cpp +++ b/dbms/src/Storages/MergeTree/MergeTreeBlockInputStream.cpp @@ -141,8 +141,9 @@ try if (!column_names.empty()) storage.check(data_part->columns, column_names); - pre_columns = storage.columns.getPhysical().addTypes(pre_column_names); - columns = storage.columns.getPhysical().addTypes(column_names); + const NamesAndTypesList & physical_columns = storage.getColumns().getPhysical(); + pre_columns = physical_columns.addTypes(pre_column_names); + columns = physical_columns.addTypes(column_names); } else { diff --git a/dbms/src/Storages/MergeTree/MergeTreeBlockReadUtils.cpp b/dbms/src/Storages/MergeTree/MergeTreeBlockReadUtils.cpp index b6d8ff3cd5a..5d52eedfda1 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeBlockReadUtils.cpp +++ b/dbms/src/Storages/MergeTree/MergeTreeBlockReadUtils.cpp @@ -26,9 +26,9 @@ NameSet injectRequiredColumns(const MergeTreeData & storage, const MergeTreeData continue; } - const auto default_it = storage.columns.defaults.find(column_name); + const auto default_it = storage.getColumns().defaults.find(column_name); /// columns has no explicit default expression - if (default_it == std::end(storage.columns.defaults)) + if (default_it == std::end(storage.getColumns().defaults)) continue; /// collect identifiers required for evaluation diff --git a/dbms/src/Storages/MergeTree/MergeTreeData.cpp b/dbms/src/Storages/MergeTree/MergeTreeData.cpp index de16c22593d..ed16d1536de 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeData.cpp +++ b/dbms/src/Storages/MergeTree/MergeTreeData.cpp @@ -110,7 +110,7 @@ MergeTreeData::MergeTreeData( data_parts_by_state_and_info(data_parts_indexes.get()) { /// NOTE: using the same columns list as is read when performing actual merges. - merging_params.check(columns.getPhysical()); + merging_params.check(getColumns().getPhysical()); if (!primary_expr_ast) throw Exception("Primary key cannot be empty", ErrorCodes::BAD_ARGUMENTS); @@ -222,11 +222,11 @@ void MergeTreeData::initPrimaryKey() primary_sort_descr.clear(); addSortDescription(primary_sort_descr, primary_expr_ast); - primary_expr = ExpressionAnalyzer(primary_expr_ast, context, nullptr, columns.getPhysical()).getActions(false); + primary_expr = ExpressionAnalyzer(primary_expr_ast, context, nullptr, getColumns().getPhysical()).getActions(false); { ExpressionActionsPtr projected_expr = - ExpressionAnalyzer(primary_expr_ast, context, nullptr, columns.getPhysical()).getActions(true); + ExpressionAnalyzer(primary_expr_ast, context, nullptr, getColumns().getPhysical()).getActions(true); primary_key_sample = projected_expr->getSampleBlock(); } @@ -241,10 +241,10 @@ void MergeTreeData::initPrimaryKey() if (secondary_sort_expr_ast) { addSortDescription(sort_descr, secondary_sort_expr_ast); - secondary_sort_expr = ExpressionAnalyzer(secondary_sort_expr_ast, context, nullptr, columns.getPhysical()).getActions(false); + secondary_sort_expr = ExpressionAnalyzer(secondary_sort_expr_ast, context, nullptr, getColumns().getPhysical()).getActions(false); ExpressionActionsPtr projected_expr = - ExpressionAnalyzer(secondary_sort_expr_ast, context, nullptr, columns.getPhysical()).getActions(true); + ExpressionAnalyzer(secondary_sort_expr_ast, context, nullptr, getColumns().getPhysical()).getActions(true); auto secondary_key_sample = projected_expr->getSampleBlock(); checkKeyExpression(*secondary_sort_expr, secondary_key_sample, "Secondary"); @@ -257,7 +257,7 @@ void MergeTreeData::initPartitionKey() if (!partition_expr_ast || partition_expr_ast->children.empty()) return; - partition_expr = ExpressionAnalyzer(partition_expr_ast, context, nullptr, columns.getPhysical()).getActions(false); + partition_expr = ExpressionAnalyzer(partition_expr_ast, context, nullptr, getColumns().getPhysical()).getActions(false); for (const ASTPtr & ast : partition_expr_ast->children) { String col_name = ast->getColumnName(); @@ -861,7 +861,7 @@ bool isMetadataOnlyConversion(const IDataType * from, const IDataType * to) void MergeTreeData::checkAlter(const AlterCommands & commands) { /// Check that needed transformations can be applied to the list of columns without considering type conversions. - auto new_columns = columns; + auto new_columns = getColumns(); commands.apply(new_columns); /// Set of columns that shouldn't be altered. @@ -903,7 +903,7 @@ void MergeTreeData::checkAlter(const AlterCommands & commands) columns_alter_forbidden.insert(merging_params.sign_column); std::map old_types; - for (const auto & column : columns.getPhysical()) + for (const auto & column : getColumns().getPhysical()) old_types.emplace(column.name, column.type.get()); for (const AlterCommand & command : commands) @@ -931,7 +931,7 @@ void MergeTreeData::checkAlter(const AlterCommands & commands) NameToNameMap unused_map; bool unused_bool; - createConvertExpression(nullptr, columns.getPhysical(), new_columns.getPhysical(), unused_expression, unused_map, unused_bool); + createConvertExpression(nullptr, getColumns().getPhysical(), new_columns.getPhysical(), unused_expression, unused_map, unused_bool); } void MergeTreeData::createConvertExpression(const DataPartPtr & part, const NamesAndTypesList & old_columns, const NamesAndTypesList & new_columns, @@ -1835,7 +1835,7 @@ void MergeTreeData::addPartContributionToColumnSizes(const DataPartPtr & part) const auto & files = part->checksums.files; /// TODO This method doesn't take into account columns with multiple files. - for (const auto & column : columns.getPhysical()) + for (const auto & column : getColumns().getPhysical()) { const auto escaped_name = escapeForFileName(column.name); const auto bin_file_name = escaped_name + ".bin"; @@ -1868,7 +1868,7 @@ void MergeTreeData::removePartContributionToColumnSizes(const DataPartPtr & part const auto & files = part->checksums.files; /// TODO This method doesn't take into account columns with multiple files. - for (const auto & column : columns.getPhysical()) + for (const auto & column : getColumns().getPhysical()) { const auto escaped_name = escapeForFileName(column.name); const auto bin_file_name = escaped_name + ".bin"; diff --git a/dbms/src/Storages/MergeTree/MergeTreeData.h b/dbms/src/Storages/MergeTree/MergeTreeData.h index a32baf797ef..da6999ac965 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeData.h +++ b/dbms/src/Storages/MergeTree/MergeTreeData.h @@ -313,12 +313,12 @@ public: if (column_name == "_sample_factor") return NameAndTypePair("_sample_factor", std::make_shared()); - return columns.get(column_name); + return getColumns().get(column_name); } bool hasColumn(const String & column_name) const override { - return columns.has(column_name) + return getColumns().has(column_name) || column_name == "_part" || column_name == "_part_index" || column_name == "_sample_factor"; diff --git a/dbms/src/Storages/MergeTree/MergeTreeDataMerger.cpp b/dbms/src/Storages/MergeTree/MergeTreeDataMerger.cpp index 8e30320aefb..dafa27e2346 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeDataMerger.cpp +++ b/dbms/src/Storages/MergeTree/MergeTreeDataMerger.cpp @@ -536,8 +536,8 @@ MergeTreeData::MutableDataPartPtr MergeTreeDataMerger::mergePartsToTemporaryPart for (const MergeTreeData::DataPartPtr & part : parts) part->accumulateColumnSizes(merged_column_to_size); - Names all_column_names = data.columns.getNames(); - NamesAndTypesList all_columns = data.columns.getPhysical(); + Names all_column_names = data.getColumns().getNames(); + NamesAndTypesList all_columns = data.getColumns().getPhysical(); const SortDescription sort_desc = data.getSortDescription(); NamesAndTypesList gathering_columns, merging_columns; diff --git a/dbms/src/Storages/MergeTree/MergeTreeDataPart.cpp b/dbms/src/Storages/MergeTree/MergeTreeDataPart.cpp index f37cd8e579c..92936e2adc7 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeDataPart.cpp +++ b/dbms/src/Storages/MergeTree/MergeTreeDataPart.cpp @@ -418,7 +418,7 @@ UInt64 MergeTreeDataPart::getColumnMrkSize(const String & name) const */ String MergeTreeDataPart::getColumnNameWithMinumumCompressedSize() const { - const auto & columns = storage.columns.getPhysical(); + const auto & columns = storage.getColumns().getPhysical(); const std::string * minimum_size_column = nullptr; UInt64 minimum_size = std::numeric_limits::max(); @@ -774,7 +774,7 @@ void MergeTreeDataPart::accumulateColumnSizes(ColumnToSize & column_to_size) con { std::shared_lock part_lock(columns_lock); - for (const NameAndTypePair & name_type : storage.columns.getPhysical()) + for (const NameAndTypePair & name_type : storage.getColumns().getPhysical()) { name_type.type->enumerateStreams([&](const IDataType::SubstreamPath & substream_path) { @@ -794,7 +794,7 @@ void MergeTreeDataPart::loadColumns(bool require) throw Exception("No columns.txt in part " + name, ErrorCodes::NO_FILE_IN_DATA_PART); /// If there is no file with a list of columns, write it down. - for (const NameAndTypePair & column : storage.columns.getPhysical()) + for (const NameAndTypePair & column : storage.getColumns().getPhysical()) if (Poco::File(getFullPath() + escapeForFileName(column.name) + ".bin").exists()) columns.push_back(column); diff --git a/dbms/src/Storages/MergeTree/MergeTreeDataSelectExecutor.cpp b/dbms/src/Storages/MergeTree/MergeTreeDataSelectExecutor.cpp index 0890163078d..77ba844ee0b 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeDataSelectExecutor.cpp +++ b/dbms/src/Storages/MergeTree/MergeTreeDataSelectExecutor.cpp @@ -175,7 +175,7 @@ BlockInputStreams MergeTreeDataSelectExecutor::read( } } - NamesAndTypesList available_real_columns = data.columns.getPhysical(); + NamesAndTypesList available_real_columns = data.getColumns().getPhysical(); NamesAndTypesList available_real_and_virtual_columns = available_real_columns; for (const auto & name : virt_column_names) @@ -857,7 +857,7 @@ void MergeTreeDataSelectExecutor::createPositiveSignCondition( arguments->children.push_back(sign); arguments->children.push_back(one); - out_expression = ExpressionAnalyzer(function, context, {}, data.columns.getPhysical()).getActions(false); + out_expression = ExpressionAnalyzer(function, context, {}, data.getColumns().getPhysical()).getActions(false); out_column = function->getColumnName(); } diff --git a/dbms/src/Storages/MergeTree/MergeTreeDataWriter.cpp b/dbms/src/Storages/MergeTree/MergeTreeDataWriter.cpp index 19eb2e8eecd..c57d1d5bfdd 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeDataWriter.cpp +++ b/dbms/src/Storages/MergeTree/MergeTreeDataWriter.cpp @@ -205,7 +205,7 @@ MergeTreeData::MutableDataPartPtr MergeTreeDataWriter::writeTempPart(BlockWithPa /// either default lz4 or compression method with zero thresholds on absolute and relative part size. auto compression_settings = data.context.chooseCompressionSettings(0, 0); - NamesAndTypesList columns = data.columns.getPhysical().filter(block.getNames()); + NamesAndTypesList columns = data.getColumns().getPhysical().filter(block.getNames()); MergedBlockOutputStream out(data, new_data_part->getFullPath(), columns, compression_settings); out.writePrefix(); diff --git a/dbms/src/Storages/MergeTree/MergeTreeReadPool.cpp b/dbms/src/Storages/MergeTree/MergeTreeReadPool.cpp index 2c600dbd32d..452aee79bf3 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeReadPool.cpp +++ b/dbms/src/Storages/MergeTree/MergeTreeReadPool.cpp @@ -238,8 +238,9 @@ std::vector MergeTreeReadPool::fillPerPartInfo( if (!required_column_names.empty()) data.check(part.data_part->columns, required_column_names); - per_part_pre_columns.push_back(data.columns.getPhysical().addTypes(required_pre_column_names)); - per_part_columns.push_back(data.columns.getPhysical().addTypes(required_column_names)); + const NamesAndTypesList & physical_columns = data.getColumns().getPhysical(); + per_part_pre_columns.push_back(physical_columns.addTypes(required_pre_column_names)); + per_part_columns.push_back(physical_columns.addTypes(required_column_names)); } else { diff --git a/dbms/src/Storages/MergeTree/MergeTreeReader.cpp b/dbms/src/Storages/MergeTree/MergeTreeReader.cpp index 7bce149776a..e172760b17f 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeReader.cpp +++ b/dbms/src/Storages/MergeTree/MergeTreeReader.cpp @@ -463,7 +463,7 @@ void MergeTreeReader::fillMissingColumns(Block & res, const Names & ordered_name if (!has_column) { should_sort = true; - if (storage.columns.defaults.count(requested_column.name) != 0) + if (storage.getColumns().defaults.count(requested_column.name) != 0) { should_evaluate_defaults = true; continue; @@ -498,7 +498,7 @@ void MergeTreeReader::fillMissingColumns(Block & res, const Names & ordered_name /// evaluate defaulted columns if necessary if (should_evaluate_defaults) - evaluateMissingDefaults(res, columns, storage.columns.defaults, storage.context); + evaluateMissingDefaults(res, columns, storage.getColumns().defaults, storage.context); /// sort columns to ensure consistent order among all blocks if (should_sort) diff --git a/dbms/src/Storages/MergeTree/MergeTreeWhereOptimizer.cpp b/dbms/src/Storages/MergeTree/MergeTreeWhereOptimizer.cpp index ce4047fc85f..ac28d3e782d 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeWhereOptimizer.cpp +++ b/dbms/src/Storages/MergeTree/MergeTreeWhereOptimizer.cpp @@ -41,9 +41,9 @@ MergeTreeWhereOptimizer::MergeTreeWhereOptimizer( Logger * log) : primary_key_columns{ext::map(data.getPrimarySortDescription(), [] (const SortColumnDescription & col) { return col.column_name; })}, - table_columns{ext::map(data.columns.getPhysical(), + table_columns{ext::map(data.getColumns().getPhysical(), [] (const NameAndTypePair & col) { return col.name; })}, - block_with_constants{PKCondition::getBlockWithConstants(query_info.query, context, data.columns.getPhysical())}, + block_with_constants{PKCondition::getBlockWithConstants(query_info.query, context, data.getColumns().getPhysical())}, prepared_sets(query_info.sets), log{log} { diff --git a/dbms/src/Storages/MergeTree/ReplicatedMergeTreeAlterThread.cpp b/dbms/src/Storages/MergeTree/ReplicatedMergeTreeAlterThread.cpp index 948902d8302..1fcba112fed 100644 --- a/dbms/src/Storages/MergeTree/ReplicatedMergeTreeAlterThread.cpp +++ b/dbms/src/Storages/MergeTree/ReplicatedMergeTreeAlterThread.cpp @@ -87,15 +87,13 @@ void ReplicatedMergeTreeAlterThread::run() auto table_lock = storage.lockStructureForAlter(__PRETTY_FUNCTION__); - if (columns_in_zk != storage.data.columns) + if (columns_in_zk != storage.getColumns()) { LOG_INFO(log, "Columns list changed in ZooKeeper. Applying changes locally."); storage.context.getDatabase(storage.database_name)->alterTable( storage.context, storage.table_name, columns_in_zk, {}); - - storage.columns = columns_in_zk; - storage.data.columns = std::move(columns_in_zk); + storage.setColumns(std::move(columns_in_zk)); /// Reinitialize primary key because primary key column types might have changed. storage.data.initPrimaryKey(); @@ -126,7 +124,7 @@ void ReplicatedMergeTreeAlterThread::run() if (!changed_version) parts = storage.data.getDataParts(); - const auto columns_for_parts = storage.data.columns.getPhysical(); + const auto columns_for_parts = storage.getColumns().getPhysical(); for (const MergeTreeData::DataPartPtr & part : parts) { diff --git a/dbms/src/Storages/StorageBuffer.cpp b/dbms/src/Storages/StorageBuffer.cpp index c76a1edc81b..2072ac72c5c 100644 --- a/dbms/src/Storages/StorageBuffer.cpp +++ b/dbms/src/Storages/StorageBuffer.cpp @@ -595,8 +595,10 @@ void StorageBuffer::alter(const AlterCommands & params, const String & database_ /// So that no blocks of the old structure remain. optimize({} /*query*/, {} /*partition_id*/, false /*final*/, false /*deduplicate*/, context); - params.apply(columns); - context.getDatabase(database_name)->alterTable(context, table_name, columns, {}); + ColumnsDescription new_columns = getColumns(); + params.apply(new_columns); + context.getDatabase(database_name)->alterTable(context, table_name, new_columns, {}); + setColumns(std::move(new_columns)); } diff --git a/dbms/src/Storages/StorageCatBoostPool.cpp b/dbms/src/Storages/StorageCatBoostPool.cpp index d20da89b170..9fa8133617c 100644 --- a/dbms/src/Storages/StorageCatBoostPool.cpp +++ b/dbms/src/Storages/StorageCatBoostPool.cpp @@ -220,6 +220,7 @@ void StorageCatBoostPool::parseColumnDescription() void StorageCatBoostPool::createSampleBlockAndColumns() { + ColumnsDescription columns; NamesAndTypesList cat_columns; NamesAndTypesList num_columns; sample_block.clear(); @@ -251,6 +252,8 @@ void StorageCatBoostPool::createSampleBlockAndColumns() } columns.ordinary.insert(columns.ordinary.end(), num_columns.begin(), num_columns.end()); columns.ordinary.insert(columns.ordinary.end(), cat_columns.begin(), cat_columns.end()); + + setColumns(columns); } BlockInputStreams StorageCatBoostPool::read(const Names & column_names, diff --git a/dbms/src/Storages/StorageDictionary.cpp b/dbms/src/Storages/StorageDictionary.cpp index 954b8ae898b..b3dbf39e9c2 100644 --- a/dbms/src/Storages/StorageDictionary.cpp +++ b/dbms/src/Storages/StorageDictionary.cpp @@ -72,7 +72,7 @@ void StorageDictionary::checkNamesAndTypesCompatibleWithDictionary(const Diction auto dictionary_names_and_types = getNamesAndTypes(dictionary_structure); std::set namesAndTypesSet(dictionary_names_and_types.begin(), dictionary_names_and_types.end()); - for (auto & column : columns.ordinary) + for (auto & column : getColumns().ordinary) { if (namesAndTypesSet.find(column) == namesAndTypesSet.end()) { diff --git a/dbms/src/Storages/StorageDistributed.cpp b/dbms/src/Storages/StorageDistributed.cpp index e3ff7f30369..972a4817009 100644 --- a/dbms/src/Storages/StorageDistributed.cpp +++ b/dbms/src/Storages/StorageDistributed.cpp @@ -142,7 +142,7 @@ StorageDistributed::StorageDistributed( table_name(table_name_), remote_database(remote_database_), remote_table(remote_table_), context(context_), cluster_name(context.getMacros().expand(cluster_name_)), has_sharding_key(sharding_key_), - sharding_key_expr(sharding_key_ ? ExpressionAnalyzer(sharding_key_, context, nullptr, columns.getPhysical()).getActions(false) : nullptr), + sharding_key_expr(sharding_key_ ? ExpressionAnalyzer(sharding_key_, context, nullptr, getColumns().getPhysical()).getActions(false) : nullptr), sharding_key_column_name(sharding_key_ ? sharding_key_->getColumnName() : String{}), path(data_path_.empty() ? "" : (data_path_ + escapeForFileName(table_name) + '/')) { @@ -232,9 +232,11 @@ void StorageDistributed::alter(const AlterCommands & params, const String & data throw Exception("Storage engine " + getName() + " doesn't support primary key.", ErrorCodes::NOT_IMPLEMENTED); auto lock = lockStructureForAlter(__PRETTY_FUNCTION__); - params.apply(columns); - context.getDatabase(database_name)->alterTable(context, table_name, columns, {}); + ColumnsDescription new_columns = getColumns(); + params.apply(new_columns); + context.getDatabase(database_name)->alterTable(context, table_name, new_columns, {}); + setColumns(std::move(new_columns)); } @@ -285,13 +287,13 @@ NameAndTypePair StorageDistributed::getColumn(const String & column_name) const if (const auto & type = VirtualColumnFactory::tryGetType(column_name)) return { column_name, type }; - return columns.get(column_name); + return getColumns().get(column_name); } bool StorageDistributed::hasColumn(const String & column_name) const { - return VirtualColumnFactory::hasColumn(column_name) || columns.has(column_name); + return VirtualColumnFactory::hasColumn(column_name) || getColumns().has(column_name); } void StorageDistributed::createDirectoryMonitors() diff --git a/dbms/src/Storages/StorageJoin.cpp b/dbms/src/Storages/StorageJoin.cpp index c3dfc16c5d7..746f15bab50 100644 --- a/dbms/src/Storages/StorageJoin.cpp +++ b/dbms/src/Storages/StorageJoin.cpp @@ -29,7 +29,7 @@ StorageJoin::StorageJoin( key_names(key_names_), kind(kind_), strictness(strictness_) { for (const auto & key : key_names) - if (!columns.has(key)) + if (!getColumns().has(key)) throw Exception{ "Key column (" + key + ") does not exist in table declaration.", ErrorCodes::NO_SUCH_COLUMN_IN_TABLE}; diff --git a/dbms/src/Storages/StorageLog.cpp b/dbms/src/Storages/StorageLog.cpp index 137c1d55616..c23d92de8fa 100644 --- a/dbms/src/Storages/StorageLog.cpp +++ b/dbms/src/Storages/StorageLog.cpp @@ -371,7 +371,7 @@ StorageLog::StorageLog( /// create files if they do not exist Poco::File(path + escapeForFileName(name) + '/').createDirectories(); - for (const auto & column : columns.getPhysical()) + for (const auto & column : getColumns().getPhysical()) addFiles(column.name, *column.type); marks_file = Poco::File(path + escapeForFileName(name) + '/' + DBMS_STORAGE_LOG_MARKS_FILE_NAME); @@ -465,8 +465,8 @@ void StorageLog::rename(const String & new_path_to_db, const String & /*new_data const StorageLog::Marks & StorageLog::getMarksWithRealRowCount() const { - const String & column_name = columns.ordinary.front().name; - const IDataType & column_type = *columns.ordinary.front().type; + const String & column_name = getColumns().ordinary.front().name; + const IDataType & column_type = *getColumns().ordinary.front().type; String filename; /** We take marks from first column. @@ -499,7 +499,7 @@ BlockInputStreams StorageLog::read( processed_stage = QueryProcessingStage::FetchColumns; loadMarks(); - NamesAndTypesList all_columns = Nested::collect(columns.getPhysical().addTypes(column_names)); + NamesAndTypesList all_columns = Nested::collect(getColumns().getPhysical().addTypes(column_names)); std::shared_lock lock(rwlock); diff --git a/dbms/src/Storages/StorageMerge.cpp b/dbms/src/Storages/StorageMerge.cpp index f26544975cd..6f29850c8d1 100644 --- a/dbms/src/Storages/StorageMerge.cpp +++ b/dbms/src/Storages/StorageMerge.cpp @@ -181,7 +181,7 @@ BlockInputStreams StorageMerge::read( /// If there are only virtual columns in query, you must request at least one other column. if (real_column_names.size() == 0) - real_column_names.push_back(ExpressionActions::getSmallestColumn(table->columns.getPhysical())); + real_column_names.push_back(ExpressionActions::getSmallestColumn(table->getColumns().getPhysical())); /// Substitute virtual column for its value when querying tables. ASTPtr modified_query_ast = query->clone(); @@ -334,8 +334,10 @@ void StorageMerge::alter(const AlterCommands & params, const String & database_n auto lock = lockStructureForAlter(__PRETTY_FUNCTION__); - params.apply(columns); - context.getDatabase(database_name)->alterTable(context, table_name, columns, {}); + ColumnsDescription new_columns = getColumns(); + params.apply(new_columns); + context.getDatabase(database_name)->alterTable(context, table_name, new_columns, {}); + setColumns(new_columns); } diff --git a/dbms/src/Storages/StorageMergeTree.cpp b/dbms/src/Storages/StorageMergeTree.cpp index 5e799c91909..f22f24082fb 100644 --- a/dbms/src/Storages/StorageMergeTree.cpp +++ b/dbms/src/Storages/StorageMergeTree.cpp @@ -46,8 +46,7 @@ StorageMergeTree::StorageMergeTree( const MergeTreeData::MergingParams & merging_params_, const MergeTreeSettings & settings_, bool has_force_restore_data_flag) - : IStorage{columns_}, - path(path_), database_name(database_name_), table_name(table_name_), full_path(path + escapeForFileName(table_name) + '/'), + : path(path_), database_name(database_name_), table_name(table_name_), full_path(path + escapeForFileName(table_name) + '/'), context(context_), background_pool(context_.getBackgroundPool()), data(database_name, table_name, full_path, columns_, @@ -153,8 +152,7 @@ void StorageMergeTree::alter( data.checkAlter(params); - auto new_columns = data.columns; - + auto new_columns = data.getColumns(); params.apply(new_columns); std::vector transactions; @@ -203,9 +201,7 @@ void StorageMergeTree::alter( } context.getDatabase(database_name)->alterTable(context, table_name, new_columns, storage_modifier); - - columns = new_columns; - data.columns = std::move(new_columns); + setColumns(std::move(new_columns)); if (primary_key_is_modified) { @@ -419,7 +415,7 @@ void StorageMergeTree::clearColumnInPartition(const ASTPtr & partition, const Fi alter_command.type = AlterCommand::DROP_COLUMN; alter_command.column_name = get(column_name); - auto new_columns = data.columns; + auto new_columns = getColumns(); alter_command.apply(new_columns); auto columns_for_parts = new_columns.getPhysical(); diff --git a/dbms/src/Storages/StorageMergeTree.h b/dbms/src/Storages/StorageMergeTree.h index e2ad6177a82..acb0e03a60a 100644 --- a/dbms/src/Storages/StorageMergeTree.h +++ b/dbms/src/Storages/StorageMergeTree.h @@ -36,6 +36,9 @@ public: bool supportsFinal() const override { return data.supportsFinal(); } bool supportsPrewhere() const override { return data.supportsPrewhere(); } + const ColumnsDescription & getColumns() const override { return data.getColumns(); } + void setColumns(ColumnsDescription columns_) override { return data.setColumns(std::move(columns_)); } + NameAndTypePair getColumn(const String & column_name) const override { return data.getColumn(column_name); diff --git a/dbms/src/Storages/StorageMySQL.cpp b/dbms/src/Storages/StorageMySQL.cpp index 28a367f17e0..3c63ebe4ca5 100644 --- a/dbms/src/Storages/StorageMySQL.cpp +++ b/dbms/src/Storages/StorageMySQL.cpp @@ -43,7 +43,7 @@ BlockInputStreams StorageMySQL::read( { check(column_names); processed_stage = QueryProcessingStage::FetchColumns; - String query = transformQueryForExternalDatabase(*query_info.query, columns.ordinary, remote_database_name, remote_table_name, context); + String query = transformQueryForExternalDatabase(*query_info.query, getColumns().ordinary, remote_database_name, remote_table_name, context); Block sample_block; for (const String & name : column_names) diff --git a/dbms/src/Storages/StorageNull.cpp b/dbms/src/Storages/StorageNull.cpp index 13232068ca9..25ecfecf2c9 100644 --- a/dbms/src/Storages/StorageNull.cpp +++ b/dbms/src/Storages/StorageNull.cpp @@ -33,9 +33,10 @@ void StorageNull::alter(const AlterCommands & params, const String & database_na { auto lock = lockStructureForAlter(__PRETTY_FUNCTION__); - params.apply(columns); - - context.getDatabase(database_name)->alterTable(context, table_name, columns, {}); + ColumnsDescription new_columns = getColumns(); + params.apply(new_columns); + context.getDatabase(database_name)->alterTable(context, table_name, new_columns, {}); + setColumns(std::move(new_columns)); } } diff --git a/dbms/src/Storages/StorageODBC.cpp b/dbms/src/Storages/StorageODBC.cpp index 7856ee9a231..09791c0e314 100644 --- a/dbms/src/Storages/StorageODBC.cpp +++ b/dbms/src/Storages/StorageODBC.cpp @@ -43,7 +43,8 @@ BlockInputStreams StorageODBC::read( { check(column_names); processed_stage = QueryProcessingStage::FetchColumns; - String query = transformQueryForExternalDatabase(*query_info.query, columns.ordinary, remote_database_name, remote_table_name, context); + String query = transformQueryForExternalDatabase( + *query_info.query, getColumns().ordinary, remote_database_name, remote_table_name, context); Block sample_block; for (const String & name : column_names) diff --git a/dbms/src/Storages/StorageReplicatedMergeTree.cpp b/dbms/src/Storages/StorageReplicatedMergeTree.cpp index d69db2d388e..c2cf65e1c85 100644 --- a/dbms/src/Storages/StorageReplicatedMergeTree.cpp +++ b/dbms/src/Storages/StorageReplicatedMergeTree.cpp @@ -184,7 +184,7 @@ StorageReplicatedMergeTree::StorageReplicatedMergeTree( const MergeTreeData::MergingParams & merging_params_, const MergeTreeSettings & settings_, bool has_force_restore_data_flag) - : IStorage{columns_}, context(context_), + : context(context_), current_zookeeper(context.getZooKeeper()), database_name(database_name_), table_name(name_), full_path(path_ + escapeForFileName(table_name) + '/'), zookeeper_path(context.getMacros().expand(zookeeper_path_)), @@ -484,7 +484,7 @@ void StorageReplicatedMergeTree::createTableIfNotExists() acl, zkutil::CreateMode::Persistent)); ops.emplace_back(std::make_unique(zookeeper_path + "/metadata", metadata, acl, zkutil::CreateMode::Persistent)); - ops.emplace_back(std::make_unique(zookeeper_path + "/columns", data.columns.toString(), + ops.emplace_back(std::make_unique(zookeeper_path + "/columns", getColumns().toString(), acl, zkutil::CreateMode::Persistent)); ops.emplace_back(std::make_unique(zookeeper_path + "/log", "", acl, zkutil::CreateMode::Persistent)); @@ -519,23 +519,22 @@ void StorageReplicatedMergeTree::checkTableStructure(bool skip_sanity_checks, bo zkutil::Stat stat; auto columns_from_zk = ColumnsDescription::parse(zookeeper->get(zookeeper_path + "/columns", &stat)); - columns_version = stat.version; - if (columns_from_zk != data.columns) + const ColumnsDescription & old_columns = getColumns(); + if (columns_from_zk != old_columns) { if (allow_alter && (skip_sanity_checks || - data.columns.ordinary.sizeOfDifference(columns_from_zk.ordinary) + - data.columns.materialized.sizeOfDifference(columns_from_zk.materialized) <= 2)) + old_columns.ordinary.sizeOfDifference(columns_from_zk.ordinary) + + old_columns.materialized.sizeOfDifference(columns_from_zk.materialized) <= 2)) { LOG_WARNING(log, "Table structure in ZooKeeper is a little different from local table structure. Assuming ALTER."); /// Without any locks, because table has not been created yet. context.getDatabase(database_name)->alterTable(context, table_name, columns_from_zk, {}); - columns = columns_from_zk; - data.columns = std::move(columns_from_zk); + setColumns(std::move(columns_from_zk)); } else { @@ -697,7 +696,7 @@ void StorageReplicatedMergeTree::createReplica() LOG_DEBUG(log, "Copied " << source_queue.size() << " queue entries"); } - zookeeper->create(replica_path + "/columns", data.columns.toString(), zkutil::CreateMode::Persistent); + zookeeper->create(replica_path + "/columns", getColumns().toString(), zkutil::CreateMode::Persistent); } @@ -1507,7 +1506,7 @@ void StorageReplicatedMergeTree::executeClearColumnInPartition(const LogEntry & alter_command.type = AlterCommand::DROP_COLUMN; alter_command.column_name = entry.column_name; - auto new_columns = data.columns; + auto new_columns = getColumns(); alter_command.apply(new_columns); size_t modified_parts = 0; @@ -2544,7 +2543,7 @@ void StorageReplicatedMergeTree::alter(const AlterCommands & params, if (param.type == AlterCommand::MODIFY_PRIMARY_KEY) throw Exception("Modification of primary key is not supported for replicated tables", ErrorCodes::NOT_IMPLEMENTED); - ColumnsDescription new_columns = data.columns; + ColumnsDescription new_columns = data.getColumns(); params.apply(new_columns); new_columns_str = new_columns.toString(); diff --git a/dbms/src/Storages/StorageReplicatedMergeTree.h b/dbms/src/Storages/StorageReplicatedMergeTree.h index 468f097534d..4a2193b05db 100644 --- a/dbms/src/Storages/StorageReplicatedMergeTree.h +++ b/dbms/src/Storages/StorageReplicatedMergeTree.h @@ -84,6 +84,9 @@ public: bool supportsPrewhere() const override { return data.supportsPrewhere(); } bool supportsReplication() const override { return true; } + const ColumnsDescription & getColumns() const override { return data.getColumns(); } + void setColumns(ColumnsDescription columns_) override { return data.setColumns(std::move(columns_)); } + NameAndTypePair getColumn(const String & column_name) const override { return data.getColumn(column_name); diff --git a/dbms/src/Storages/StorageTinyLog.cpp b/dbms/src/Storages/StorageTinyLog.cpp index 8c7fe1963a8..bd4a8ca304f 100644 --- a/dbms/src/Storages/StorageTinyLog.cpp +++ b/dbms/src/Storages/StorageTinyLog.cpp @@ -296,7 +296,7 @@ StorageTinyLog::StorageTinyLog( throwFromErrno("Cannot create directory " + full_path, ErrorCodes::CANNOT_CREATE_DIRECTORY); } - for (const auto & col : columns.getPhysical()) + for (const auto & col : getColumns().getPhysical()) addFiles(col.name, *col.type); } @@ -348,7 +348,7 @@ BlockInputStreams StorageTinyLog::read( check(column_names); processed_stage = QueryProcessingStage::FetchColumns; return BlockInputStreams(1, std::make_shared( - max_block_size, Nested::collect(columns.getPhysical().addTypes(column_names)), *this, context.getSettingsRef().max_read_buffer_size)); + max_block_size, Nested::collect(getColumns().getPhysical().addTypes(column_names)), *this, context.getSettingsRef().max_read_buffer_size)); } diff --git a/dbms/src/Storages/System/StorageSystemAsynchronousMetrics.cpp b/dbms/src/Storages/System/StorageSystemAsynchronousMetrics.cpp index 3c77244c557..bc2f76379e9 100644 --- a/dbms/src/Storages/System/StorageSystemAsynchronousMetrics.cpp +++ b/dbms/src/Storages/System/StorageSystemAsynchronousMetrics.cpp @@ -16,10 +16,10 @@ StorageSystemAsynchronousMetrics::StorageSystemAsynchronousMetrics(const std::st : name(name_), async_metrics(async_metrics_) { - columns = ColumnsDescription({ + setColumns(ColumnsDescription({ {"metric", std::make_shared()}, {"value", std::make_shared()}, - }); + })); } diff --git a/dbms/src/Storages/System/StorageSystemBuildOptions.cpp b/dbms/src/Storages/System/StorageSystemBuildOptions.cpp index f354da97944..e62e6e9bbfd 100644 --- a/dbms/src/Storages/System/StorageSystemBuildOptions.cpp +++ b/dbms/src/Storages/System/StorageSystemBuildOptions.cpp @@ -13,10 +13,10 @@ namespace DB StorageSystemBuildOptions::StorageSystemBuildOptions(const std::string & name_) : name(name_) { - columns = ColumnsDescription({ + setColumns(ColumnsDescription({ { "name", std::make_shared() }, { "value", std::make_shared() }, - }); + })); } diff --git a/dbms/src/Storages/System/StorageSystemClusters.cpp b/dbms/src/Storages/System/StorageSystemClusters.cpp index 1e67e433d48..904d22d180e 100644 --- a/dbms/src/Storages/System/StorageSystemClusters.cpp +++ b/dbms/src/Storages/System/StorageSystemClusters.cpp @@ -14,7 +14,7 @@ namespace DB StorageSystemClusters::StorageSystemClusters(const std::string & name_) : name(name_) { - columns = ColumnsDescription({ + setColumns(ColumnsDescription({ { "cluster", std::make_shared() }, { "shard_num", std::make_shared() }, { "shard_weight", std::make_shared() }, @@ -25,7 +25,7 @@ StorageSystemClusters::StorageSystemClusters(const std::string & name_) { "is_local", std::make_shared() }, { "user", std::make_shared() }, { "default_database", std::make_shared() }, - }); + })); } diff --git a/dbms/src/Storages/System/StorageSystemColumns.cpp b/dbms/src/Storages/System/StorageSystemColumns.cpp index 83c5dca9117..18598b1c1dc 100644 --- a/dbms/src/Storages/System/StorageSystemColumns.cpp +++ b/dbms/src/Storages/System/StorageSystemColumns.cpp @@ -18,7 +18,7 @@ namespace DB StorageSystemColumns::StorageSystemColumns(const std::string & name_) : name(name_) { - columns = ColumnsDescription({ + setColumns(ColumnsDescription({ { "database", std::make_shared() }, { "table", std::make_shared() }, { "name", std::make_shared() }, @@ -28,7 +28,7 @@ StorageSystemColumns::StorageSystemColumns(const std::string & name_) { "data_compressed_bytes", std::make_shared() }, { "data_uncompressed_bytes", std::make_shared() }, { "marks_bytes", std::make_shared() }, - }); + })); } @@ -137,8 +137,8 @@ BlockInputStreams StorageSystemColumns::read( throw; } - columns = storage->columns.getAll(); - column_defaults = storage->columns.defaults; + columns = storage->getColumns().getAll(); + column_defaults = storage->getColumns().defaults; /** Info about sizes of columns for tables of MergeTree family. * NOTE: It is possible to add getter for this info to IStorage interface. diff --git a/dbms/src/Storages/System/StorageSystemDatabases.cpp b/dbms/src/Storages/System/StorageSystemDatabases.cpp index 79a3b3483a6..49c78688616 100644 --- a/dbms/src/Storages/System/StorageSystemDatabases.cpp +++ b/dbms/src/Storages/System/StorageSystemDatabases.cpp @@ -13,12 +13,12 @@ namespace DB StorageSystemDatabases::StorageSystemDatabases(const std::string & name_) : name(name_) { - columns = ColumnsDescription({ + setColumns(ColumnsDescription({ {"name", std::make_shared()}, {"engine", std::make_shared()}, {"data_path", std::make_shared()}, {"metadata_path", std::make_shared()}, - }); + })); } diff --git a/dbms/src/Storages/System/StorageSystemDictionaries.cpp b/dbms/src/Storages/System/StorageSystemDictionaries.cpp index 84bf4a2c98f..c57c1c7f459 100644 --- a/dbms/src/Storages/System/StorageSystemDictionaries.cpp +++ b/dbms/src/Storages/System/StorageSystemDictionaries.cpp @@ -21,7 +21,7 @@ namespace DB StorageSystemDictionaries::StorageSystemDictionaries(const std::string & name) : name{name} { - columns = ColumnsDescription({ + setColumns(ColumnsDescription({ { "name", std::make_shared() }, { "origin", std::make_shared() }, { "type", std::make_shared() }, @@ -36,7 +36,7 @@ StorageSystemDictionaries::StorageSystemDictionaries(const std::string & name) { "creation_time", std::make_shared() }, { "source", std::make_shared() }, { "last_exception", std::make_shared() }, - }); + })); } diff --git a/dbms/src/Storages/System/StorageSystemEvents.cpp b/dbms/src/Storages/System/StorageSystemEvents.cpp index ac613a7e84e..1dc49ad37b2 100644 --- a/dbms/src/Storages/System/StorageSystemEvents.cpp +++ b/dbms/src/Storages/System/StorageSystemEvents.cpp @@ -13,11 +13,11 @@ namespace DB StorageSystemEvents::StorageSystemEvents(const std::string & name_) : name(name_) { - columns = ColumnsDescription( + setColumns(ColumnsDescription( { {"event", std::make_shared()}, {"value", std::make_shared()}, - }); + })); } diff --git a/dbms/src/Storages/System/StorageSystemFunctions.cpp b/dbms/src/Storages/System/StorageSystemFunctions.cpp index 2d42bee9a46..909bf1d8089 100644 --- a/dbms/src/Storages/System/StorageSystemFunctions.cpp +++ b/dbms/src/Storages/System/StorageSystemFunctions.cpp @@ -16,10 +16,10 @@ namespace DB StorageSystemFunctions::StorageSystemFunctions(const std::string & name_) : name(name_) { - columns = ColumnsDescription({ + setColumns(ColumnsDescription({ { "name", std::make_shared() }, { "is_aggregate", std::make_shared() }, - }); + })); } diff --git a/dbms/src/Storages/System/StorageSystemGraphite.cpp b/dbms/src/Storages/System/StorageSystemGraphite.cpp index 30cc56c55e3..f558adc1d95 100644 --- a/dbms/src/Storages/System/StorageSystemGraphite.cpp +++ b/dbms/src/Storages/System/StorageSystemGraphite.cpp @@ -125,7 +125,7 @@ static Strings getAllGraphiteSections(const AbstractConfiguration & config) StorageSystemGraphite::StorageSystemGraphite(const std::string & name_) : name(name_) { - columns = ColumnsDescription({ + setColumns(ColumnsDescription({ {"config_name", std::make_shared()}, {"regexp", std::make_shared()}, {"function", std::make_shared()}, @@ -133,7 +133,7 @@ StorageSystemGraphite::StorageSystemGraphite(const std::string & name_) {"precision", std::make_shared()}, {"priority", std::make_shared()}, {"is_default", std::make_shared()}, - }); + })); } diff --git a/dbms/src/Storages/System/StorageSystemMerges.cpp b/dbms/src/Storages/System/StorageSystemMerges.cpp index c32cad1265b..d3af993e29e 100644 --- a/dbms/src/Storages/System/StorageSystemMerges.cpp +++ b/dbms/src/Storages/System/StorageSystemMerges.cpp @@ -14,7 +14,7 @@ namespace DB StorageSystemMerges::StorageSystemMerges(const std::string & name) : name{name} { - columns = ColumnsDescription({ + setColumns(ColumnsDescription({ { "database", std::make_shared() }, { "table", std::make_shared() }, { "elapsed", std::make_shared() }, @@ -31,7 +31,7 @@ StorageSystemMerges::StorageSystemMerges(const std::string & name) { "columns_written", std::make_shared() }, { "memory_usage", std::make_shared() }, { "thread_number", std::make_shared() }, - }); + })); } diff --git a/dbms/src/Storages/System/StorageSystemMetrics.cpp b/dbms/src/Storages/System/StorageSystemMetrics.cpp index 57935b13fea..9d3b1cc9fbc 100644 --- a/dbms/src/Storages/System/StorageSystemMetrics.cpp +++ b/dbms/src/Storages/System/StorageSystemMetrics.cpp @@ -14,10 +14,10 @@ namespace DB StorageSystemMetrics::StorageSystemMetrics(const std::string & name_) : name(name_) { - columns = ColumnsDescription({ + setColumns(ColumnsDescription({ {"metric", std::make_shared()}, {"value", std::make_shared()}, - }); + })); } diff --git a/dbms/src/Storages/System/StorageSystemModels.cpp b/dbms/src/Storages/System/StorageSystemModels.cpp index 6ef9d157303..5175989b861 100644 --- a/dbms/src/Storages/System/StorageSystemModels.cpp +++ b/dbms/src/Storages/System/StorageSystemModels.cpp @@ -14,13 +14,13 @@ namespace DB StorageSystemModels::StorageSystemModels(const std::string & name) : name{name} { - columns = ColumnsDescription({ + setColumns(ColumnsDescription({ { "name", std::make_shared() }, { "origin", std::make_shared() }, { "type", std::make_shared() }, { "creation_time", std::make_shared() }, { "last_exception", std::make_shared() }, - }); + })); } diff --git a/dbms/src/Storages/System/StorageSystemNumbers.cpp b/dbms/src/Storages/System/StorageSystemNumbers.cpp index d5e72bcbdc3..49a3c7ca513 100644 --- a/dbms/src/Storages/System/StorageSystemNumbers.cpp +++ b/dbms/src/Storages/System/StorageSystemNumbers.cpp @@ -47,7 +47,7 @@ private: StorageSystemNumbers::StorageSystemNumbers(const std::string & name_, bool multithreaded_, size_t limit_) : name(name_), multithreaded(multithreaded_), limit(limit_) { - columns = ColumnsDescription({{"number", std::make_shared()}}); + setColumns(ColumnsDescription({{"number", std::make_shared()}})); } diff --git a/dbms/src/Storages/System/StorageSystemOne.cpp b/dbms/src/Storages/System/StorageSystemOne.cpp index 9cd74a7525c..7714a0ea53c 100644 --- a/dbms/src/Storages/System/StorageSystemOne.cpp +++ b/dbms/src/Storages/System/StorageSystemOne.cpp @@ -13,7 +13,7 @@ namespace DB StorageSystemOne::StorageSystemOne(const std::string & name_) : name(name_) { - columns = ColumnsDescription({{"dummy", std::make_shared()}}); + setColumns(ColumnsDescription({{"dummy", std::make_shared()}})); } diff --git a/dbms/src/Storages/System/StorageSystemPartsBase.cpp b/dbms/src/Storages/System/StorageSystemPartsBase.cpp index 5143b73955f..9d94655fc18 100644 --- a/dbms/src/Storages/System/StorageSystemPartsBase.cpp +++ b/dbms/src/Storages/System/StorageSystemPartsBase.cpp @@ -280,7 +280,7 @@ bool StorageSystemPartsBase::hasColumn(const String & column_name) const StorageSystemPartsBase::StorageSystemPartsBase(std::string name_, NamesAndTypesList && columns_) : name(std::move(name_)) { - columns = ColumnsDescription(std::move(columns_)); + setColumns(ColumnsDescription(std::move(columns_))); } } diff --git a/dbms/src/Storages/System/StorageSystemPartsColumns.cpp b/dbms/src/Storages/System/StorageSystemPartsColumns.cpp index d804504f13a..8e55625c5cd 100644 --- a/dbms/src/Storages/System/StorageSystemPartsColumns.cpp +++ b/dbms/src/Storages/System/StorageSystemPartsColumns.cpp @@ -61,8 +61,8 @@ void StorageSystemPartsColumns::processNextStorage(MutableColumns & columns, con String default_expression; }; - NamesAndTypesList columns_list = info.storage->columns.getAll(); - const auto & column_defaults = info.storage->columns.defaults; + NamesAndTypesList columns_list = info.storage->getColumns().getAll(); + const auto & column_defaults = info.storage->getColumns().defaults; std::unordered_map columns_info; for (const auto & column : columns_list) diff --git a/dbms/src/Storages/System/StorageSystemProcesses.cpp b/dbms/src/Storages/System/StorageSystemProcesses.cpp index 15ecc1a3d95..793e3124a2a 100644 --- a/dbms/src/Storages/System/StorageSystemProcesses.cpp +++ b/dbms/src/Storages/System/StorageSystemProcesses.cpp @@ -14,7 +14,7 @@ namespace DB StorageSystemProcesses::StorageSystemProcesses(const std::string & name_) : name(name_) { - columns = ColumnsDescription({ + setColumns(ColumnsDescription({ { "is_initial_query", std::make_shared() }, { "user", std::make_shared() }, @@ -51,7 +51,7 @@ StorageSystemProcesses::StorageSystemProcesses(const std::string & name_) { "memory_usage", std::make_shared() }, { "peak_memory_usage", std::make_shared() }, { "query", std::make_shared() }, - }); + })); } diff --git a/dbms/src/Storages/System/StorageSystemReplicas.cpp b/dbms/src/Storages/System/StorageSystemReplicas.cpp index fdcc6984861..ea2c8b1cde2 100644 --- a/dbms/src/Storages/System/StorageSystemReplicas.cpp +++ b/dbms/src/Storages/System/StorageSystemReplicas.cpp @@ -17,7 +17,7 @@ namespace DB StorageSystemReplicas::StorageSystemReplicas(const std::string & name_) : name(name_) { - columns = ColumnsDescription({ + setColumns(ColumnsDescription({ { "database", std::make_shared() }, { "table", std::make_shared() }, { "engine", std::make_shared() }, @@ -44,7 +44,7 @@ StorageSystemReplicas::StorageSystemReplicas(const std::string & name_) { "absolute_delay", std::make_shared() }, { "total_replicas", std::make_shared() }, { "active_replicas", std::make_shared() }, - }); + })); } diff --git a/dbms/src/Storages/System/StorageSystemReplicationQueue.cpp b/dbms/src/Storages/System/StorageSystemReplicationQueue.cpp index b0ba714fef4..77b265ad065 100644 --- a/dbms/src/Storages/System/StorageSystemReplicationQueue.cpp +++ b/dbms/src/Storages/System/StorageSystemReplicationQueue.cpp @@ -20,7 +20,7 @@ namespace DB StorageSystemReplicationQueue::StorageSystemReplicationQueue(const std::string & name_) : name(name_) { - columns = ColumnsDescription({ + setColumns(ColumnsDescription({ /// Table properties. { "database", std::make_shared() }, { "table", std::make_shared() }, @@ -43,7 +43,7 @@ StorageSystemReplicationQueue::StorageSystemReplicationQueue(const std::string & { "num_postponed", std::make_shared() }, { "postpone_reason", std::make_shared() }, { "last_postpone_time", std::make_shared() }, - }); + })); } diff --git a/dbms/src/Storages/System/StorageSystemSettings.cpp b/dbms/src/Storages/System/StorageSystemSettings.cpp index d8a758becc7..efb50c559cc 100644 --- a/dbms/src/Storages/System/StorageSystemSettings.cpp +++ b/dbms/src/Storages/System/StorageSystemSettings.cpp @@ -14,12 +14,12 @@ namespace DB StorageSystemSettings::StorageSystemSettings(const std::string & name_) : name(name_) { - columns = ColumnsDescription({ + setColumns(ColumnsDescription({ { "name", std::make_shared() }, { "value", std::make_shared() }, { "changed", std::make_shared() }, { "description", std::make_shared() }, - }); + })); } diff --git a/dbms/src/Storages/System/StorageSystemTables.cpp b/dbms/src/Storages/System/StorageSystemTables.cpp index 2db2a8052cc..99ed89739f9 100644 --- a/dbms/src/Storages/System/StorageSystemTables.cpp +++ b/dbms/src/Storages/System/StorageSystemTables.cpp @@ -114,7 +114,7 @@ protected: StorageSystemTables::StorageSystemTables(const std::string & name_) : name(name_) { - columns = ColumnsDescription( + setColumns(ColumnsDescription( { {"database", std::make_shared()}, {"name", std::make_shared()}, @@ -122,7 +122,7 @@ StorageSystemTables::StorageSystemTables(const std::string & name_) {"is_temporary", std::make_shared()}, {"data_path", std::make_shared()}, {"metadata_path", std::make_shared()}, - }); + })); virtual_columns = { diff --git a/dbms/src/Storages/System/StorageSystemZooKeeper.cpp b/dbms/src/Storages/System/StorageSystemZooKeeper.cpp index a5d5e2dab96..67660ea3446 100644 --- a/dbms/src/Storages/System/StorageSystemZooKeeper.cpp +++ b/dbms/src/Storages/System/StorageSystemZooKeeper.cpp @@ -22,7 +22,7 @@ namespace DB StorageSystemZooKeeper::StorageSystemZooKeeper(const std::string & name_) : name(name_) { - columns = ColumnsDescription({ + setColumns(ColumnsDescription({ { "name", std::make_shared() }, { "value", std::make_shared() }, { "czxid", std::make_shared() }, @@ -37,7 +37,7 @@ StorageSystemZooKeeper::StorageSystemZooKeeper(const std::string & name_) { "numChildren", std::make_shared() }, { "pzxid", std::make_shared() }, { "path", std::make_shared() }, - }); + })); } diff --git a/dbms/src/TableFunctions/TableFunctionMerge.cpp b/dbms/src/TableFunctions/TableFunctionMerge.cpp index 2a12dbca275..6b0710d4ba0 100644 --- a/dbms/src/TableFunctions/TableFunctionMerge.cpp +++ b/dbms/src/TableFunctions/TableFunctionMerge.cpp @@ -51,7 +51,7 @@ static NamesAndTypesList chooseColumns(const String & source_database, const Str throw Exception("Error while executing table function merge. In database " + source_database + " no one matches regular expression: " + table_name_regexp_, ErrorCodes::UNKNOWN_TABLE); - return any_table->columns.getPhysical(); + return any_table->getColumns().getPhysical(); } diff --git a/dbms/src/TableFunctions/getStructureOfRemoteTable.cpp b/dbms/src/TableFunctions/getStructureOfRemoteTable.cpp index 6eede709352..56bb257f252 100644 --- a/dbms/src/TableFunctions/getStructureOfRemoteTable.cpp +++ b/dbms/src/TableFunctions/getStructureOfRemoteTable.cpp @@ -28,7 +28,7 @@ ColumnsDescription getStructureOfRemoteTable( const auto & shard_info = cluster.getAnyShardInfo(); if (shard_info.isLocal()) - return context.getTable(database, table)->columns; + return context.getTable(database, table)->getColumns(); /// Request for a table description String query = "DESC TABLE " + backQuoteIfNeed(database) + "." + backQuoteIfNeed(table); From 5906c9a9886ad74e4d4a95ebb454fadc4bb8987a Mon Sep 17 00:00:00 2001 From: Alexey Zatelepin Date: Tue, 13 Mar 2018 18:00:28 +0300 Subject: [PATCH 7/8] rename for clarity --- dbms/src/Analyzers/AnalyzeColumns.cpp | 2 +- dbms/src/Analyzers/ExecuteTableFunctions.cpp | 2 +- dbms/src/Core/iostream_debug_helpers.cpp | 2 +- .../DataStreams/RemoteBlockInputStream.cpp | 2 +- dbms/src/Functions/FunctionsMiscellaneous.cpp | 5 ++--- dbms/src/Interpreters/ExpressionAnalyzer.cpp | 2 +- .../Interpreters/InterpreterAlterQuery.cpp | 2 +- .../Interpreters/InterpreterInsertQuery.cpp | 2 +- dbms/src/Storages/ColumnsDescription.cpp | 8 ++++---- dbms/src/Storages/ColumnsDescription.h | 11 ++++------ dbms/src/Storages/ITableDeclaration.cpp | 12 +++++------ .../MergeTree/MergeTreeBlockInputStream.cpp | 2 +- dbms/src/Storages/MergeTree/MergeTreeData.cpp | 20 +++++++++---------- dbms/src/Storages/MergeTree/MergeTreeData.h | 4 ++-- .../MergeTree/MergeTreeDataMerger.cpp | 4 ++-- .../Storages/MergeTree/MergeTreeDataPart.cpp | 6 +++--- .../MergeTree/MergeTreeDataSelectExecutor.cpp | 4 ++-- .../MergeTree/MergeTreeDataWriter.cpp | 2 +- .../Storages/MergeTree/MergeTreeReadPool.cpp | 2 +- .../MergeTree/MergeTreeWhereOptimizer.cpp | 4 ++-- .../ReplicatedMergeTreeAlterThread.cpp | 2 +- dbms/src/Storages/StorageDistributed.cpp | 8 ++++---- dbms/src/Storages/StorageJoin.cpp | 2 +- dbms/src/Storages/StorageLog.cpp | 4 ++-- dbms/src/Storages/StorageMerge.cpp | 2 +- dbms/src/Storages/StorageMergeTree.cpp | 4 ++-- .../Storages/StorageReplicatedMergeTree.cpp | 2 +- dbms/src/Storages/StorageTinyLog.cpp | 4 ++-- .../src/TableFunctions/TableFunctionMerge.cpp | 2 +- 29 files changed, 62 insertions(+), 66 deletions(-) diff --git a/dbms/src/Analyzers/AnalyzeColumns.cpp b/dbms/src/Analyzers/AnalyzeColumns.cpp index efcbaf66888..73760c4ff01 100644 --- a/dbms/src/Analyzers/AnalyzeColumns.cpp +++ b/dbms/src/Analyzers/AnalyzeColumns.cpp @@ -173,7 +173,7 @@ ASTPtr createASTIdentifierForColumnInTable(const String & column, const CollectT void createASTsForAllColumnsInTable(const CollectTables::TableInfo & table, ASTs & res) { if (table.storage) - for (const auto & name : table.storage->getColumns().getNames()) + for (const auto & name : table.storage->getColumns().getNamesOfPhysical()) res.emplace_back(createASTIdentifierForColumnInTable(name, table)); else for (size_t i = 0, size = table.structure_of_subquery.columns(); i < size; ++i) diff --git a/dbms/src/Analyzers/ExecuteTableFunctions.cpp b/dbms/src/Analyzers/ExecuteTableFunctions.cpp index 37cc3d66eec..d79fa1481f0 100644 --- a/dbms/src/Analyzers/ExecuteTableFunctions.cpp +++ b/dbms/src/Analyzers/ExecuteTableFunctions.cpp @@ -72,7 +72,7 @@ void ExecuteTableFunctions::dump(WriteBuffer & out) const { writeString(table.second->getName(), out); writeCString("\n\n", out); - writeString(table.second->getColumns().getPhysical().toString(), out); + writeString(table.second->getColumns().getAllPhysical().toString(), out); writeCString("\n", out); } } diff --git a/dbms/src/Core/iostream_debug_helpers.cpp b/dbms/src/Core/iostream_debug_helpers.cpp index bd19c453d8d..2ad56c402b7 100644 --- a/dbms/src/Core/iostream_debug_helpers.cpp +++ b/dbms/src/Core/iostream_debug_helpers.cpp @@ -44,7 +44,7 @@ std::ostream & operator<<(std::ostream & stream, const IDataType & what) std::ostream & operator<<(std::ostream & stream, const IStorage & what) { stream << "IStorage(name = " << what.getName() << ", tableName = " << what.getTableName() << ") {" - << what.getColumns().getPhysical().toString() + << what.getColumns().getAllPhysical().toString() << "}"; // isRemote supportsSampling supportsFinal supportsPrewhere return stream; diff --git a/dbms/src/DataStreams/RemoteBlockInputStream.cpp b/dbms/src/DataStreams/RemoteBlockInputStream.cpp index b5e20898b89..7b851ec8469 100644 --- a/dbms/src/DataStreams/RemoteBlockInputStream.cpp +++ b/dbms/src/DataStreams/RemoteBlockInputStream.cpp @@ -138,7 +138,7 @@ void RemoteBlockInputStream::sendExternalTables() { StoragePtr cur = table.second; QueryProcessingStage::Enum stage = QueryProcessingStage::Complete; - BlockInputStreams input = cur->read(cur->getColumns().getNames(), {}, context, + BlockInputStreams input = cur->read(cur->getColumns().getNamesOfPhysical(), {}, context, stage, DEFAULT_BLOCK_SIZE, 1); if (input.size() == 0) res.push_back(std::make_pair(std::make_shared(cur->getSampleBlock()), table.first)); diff --git a/dbms/src/Functions/FunctionsMiscellaneous.cpp b/dbms/src/Functions/FunctionsMiscellaneous.cpp index 99ed8b2dda6..475fd47ce6f 100644 --- a/dbms/src/Functions/FunctionsMiscellaneous.cpp +++ b/dbms/src/Functions/FunctionsMiscellaneous.cpp @@ -1750,9 +1750,8 @@ void FunctionHasColumnInTable::executeImpl(Block & block, const ColumnNumbers & { std::vector> host_names = {{ host_name }}; auto cluster = std::make_shared(global_context.getSettings(), host_names, !user_name.empty() ? user_name : "default", password, global_context.getTCPPort(), false); - auto names_and_types_list = getStructureOfRemoteTable(*cluster, database_name, table_name, global_context); - const auto & names = names_and_types_list.getNames(); - has_column = std::find(names.begin(), names.end(), column_name) != names.end(); + auto remote_columns = getStructureOfRemoteTable(*cluster, database_name, table_name, global_context); + has_column = remote_columns.hasPhysical(column_name); } block.getByPosition(result).column = DataTypeUInt8().createColumnConst(block.rows(), UInt64(has_column)); diff --git a/dbms/src/Interpreters/ExpressionAnalyzer.cpp b/dbms/src/Interpreters/ExpressionAnalyzer.cpp index 4f0fe52bcff..f0bc8137029 100644 --- a/dbms/src/Interpreters/ExpressionAnalyzer.cpp +++ b/dbms/src/Interpreters/ExpressionAnalyzer.cpp @@ -190,7 +190,7 @@ ExpressionAnalyzer::ExpressionAnalyzer( } if (storage && source_columns.empty()) - source_columns = storage->getColumns().getPhysical(); + source_columns = storage->getColumns().getAllPhysical(); else removeDuplicateColumns(source_columns); diff --git a/dbms/src/Interpreters/InterpreterAlterQuery.cpp b/dbms/src/Interpreters/InterpreterAlterQuery.cpp index 377f92d6c8d..f4708a67c3d 100644 --- a/dbms/src/Interpreters/InterpreterAlterQuery.cpp +++ b/dbms/src/Interpreters/InterpreterAlterQuery.cpp @@ -200,7 +200,7 @@ void InterpreterAlterQuery::PartitionCommands::validate(const IStorage * table) { String column_name = command.column_name.safeGet(); - if (!table->getColumns().has(column_name)) + if (!table->getColumns().hasPhysical(column_name)) { throw Exception("Wrong column name. Cannot find column " + column_name + " to clear it from partition", DB::ErrorCodes::ILLEGAL_COLUMN); diff --git a/dbms/src/Interpreters/InterpreterInsertQuery.cpp b/dbms/src/Interpreters/InterpreterInsertQuery.cpp index 3e054f576a4..c21e3c26f93 100644 --- a/dbms/src/Interpreters/InterpreterInsertQuery.cpp +++ b/dbms/src/Interpreters/InterpreterInsertQuery.cpp @@ -95,7 +95,7 @@ BlockIO InterpreterInsertQuery::execute() auto table_lock = table->lockStructure(true, __PRETTY_FUNCTION__); - NamesAndTypesList required_columns = table->getColumns().getPhysical(); + NamesAndTypesList required_columns = table->getColumns().getAllPhysical(); /// We create a pipeline of several streams, into which we will write data. BlockOutputStreamPtr out; diff --git a/dbms/src/Storages/ColumnsDescription.cpp b/dbms/src/Storages/ColumnsDescription.cpp index 683f10f324d..d17f532b0ef 100644 --- a/dbms/src/Storages/ColumnsDescription.cpp +++ b/dbms/src/Storages/ColumnsDescription.cpp @@ -27,7 +27,7 @@ namespace ErrorCodes } -NamesAndTypesList ColumnsDescription::getPhysical() const +NamesAndTypesList ColumnsDescription::getAllPhysical() const { return ext::collection_cast(boost::join(ordinary, materialized)); } @@ -39,13 +39,13 @@ NamesAndTypesList ColumnsDescription::getAll() const } -Names ColumnsDescription::getNames() const +Names ColumnsDescription::getNamesOfPhysical() const { return ext::map(boost::join(ordinary, materialized), [] (const auto & it) { return it.name; }); } -NameAndTypePair ColumnsDescription::get(const String & column_name) const +NameAndTypePair ColumnsDescription::getPhysical(const String & column_name) const { for (auto & it : boost::join(ordinary, materialized)) if (it.name == column_name) @@ -54,7 +54,7 @@ NameAndTypePair ColumnsDescription::get(const String & column_name) const } -bool ColumnsDescription::has(const String & column_name) const +bool ColumnsDescription::hasPhysical(const String & column_name) const { for (auto & it : boost::join(ordinary, materialized)) if (it.name == column_name) diff --git a/dbms/src/Storages/ColumnsDescription.h b/dbms/src/Storages/ColumnsDescription.h index e4e45ff6fe2..288d2712b3b 100644 --- a/dbms/src/Storages/ColumnsDescription.h +++ b/dbms/src/Storages/ColumnsDescription.h @@ -42,19 +42,16 @@ struct ColumnsDescription bool operator!=(const ColumnsDescription & other) const { return !(*this == other); } /// ordinary + materialized. - NamesAndTypesList getPhysical() const; + NamesAndTypesList getAllPhysical() const; /// ordinary + materialized + aliases. NamesAndTypesList getAll() const; - /// Get names of physical columns. - Names getNames() const; + Names getNamesOfPhysical() const; - /// Get a physical column by name. - NameAndTypePair get(const String & column_name) const; + NameAndTypePair getPhysical(const String & column_name) const; - /// Is there a physical column with the given name. - bool has(const String & column_name) const; + bool hasPhysical(const String & column_name) const; String toString() const; diff --git a/dbms/src/Storages/ITableDeclaration.cpp b/dbms/src/Storages/ITableDeclaration.cpp index 783b0b2a7ed..9a6ac0e2538 100644 --- a/dbms/src/Storages/ITableDeclaration.cpp +++ b/dbms/src/Storages/ITableDeclaration.cpp @@ -34,12 +34,12 @@ void ITableDeclaration::setColumns(ColumnsDescription columns_) bool ITableDeclaration::hasColumn(const String & column_name) const { - return getColumns().has(column_name); /// By default, we assume that there are no virtual columns in the storage. + return getColumns().hasPhysical(column_name); /// By default, we assume that there are no virtual columns in the storage. } NameAndTypePair ITableDeclaration::getColumn(const String & column_name) const { - return getColumns().get(column_name); /// By default, we assume that there are no virtual columns in the storage. + return getColumns().getPhysical(column_name); /// By default, we assume that there are no virtual columns in the storage. } @@ -119,7 +119,7 @@ static NamesAndTypesMap getColumnsMap(const Args &... args) void ITableDeclaration::check(const Names & column_names) const { - const NamesAndTypesList & available_columns = getColumns().getPhysical(); + const NamesAndTypesList & available_columns = getColumns().getAllPhysical(); if (column_names.empty()) throw Exception("Empty list of columns queried. There are columns: " + listOfColumns(available_columns), @@ -147,7 +147,7 @@ void ITableDeclaration::check(const Names & column_names) const void ITableDeclaration::check(const NamesAndTypesList & provided_columns) const { - const NamesAndTypesList & available_columns = getColumns().getPhysical(); + const NamesAndTypesList & available_columns = getColumns().getAllPhysical(); const auto columns_map = getColumnsMap(available_columns); using UniqueStrings = google::dense_hash_set; @@ -175,7 +175,7 @@ void ITableDeclaration::check(const NamesAndTypesList & provided_columns) const void ITableDeclaration::check(const NamesAndTypesList & provided_columns, const Names & column_names) const { - const NamesAndTypesList & available_columns = getColumns().getPhysical(); + const NamesAndTypesList & available_columns = getColumns().getAllPhysical(); const auto available_columns_map = getColumnsMap(available_columns); const NamesAndTypesMap & provided_columns_map = getColumnsMap(provided_columns); @@ -212,7 +212,7 @@ void ITableDeclaration::check(const NamesAndTypesList & provided_columns, const void ITableDeclaration::check(const Block & block, bool need_all) const { - const NamesAndTypesList & available_columns = getColumns().getPhysical(); + const NamesAndTypesList & available_columns = getColumns().getAllPhysical(); const auto columns_map = getColumnsMap(available_columns); using NameSet = std::unordered_set; diff --git a/dbms/src/Storages/MergeTree/MergeTreeBlockInputStream.cpp b/dbms/src/Storages/MergeTree/MergeTreeBlockInputStream.cpp index a6d2958a786..1e9cea1887f 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeBlockInputStream.cpp +++ b/dbms/src/Storages/MergeTree/MergeTreeBlockInputStream.cpp @@ -141,7 +141,7 @@ try if (!column_names.empty()) storage.check(data_part->columns, column_names); - const NamesAndTypesList & physical_columns = storage.getColumns().getPhysical(); + const NamesAndTypesList & physical_columns = storage.getColumns().getAllPhysical(); pre_columns = physical_columns.addTypes(pre_column_names); columns = physical_columns.addTypes(column_names); } diff --git a/dbms/src/Storages/MergeTree/MergeTreeData.cpp b/dbms/src/Storages/MergeTree/MergeTreeData.cpp index ed16d1536de..b5200441383 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeData.cpp +++ b/dbms/src/Storages/MergeTree/MergeTreeData.cpp @@ -110,7 +110,7 @@ MergeTreeData::MergeTreeData( data_parts_by_state_and_info(data_parts_indexes.get()) { /// NOTE: using the same columns list as is read when performing actual merges. - merging_params.check(getColumns().getPhysical()); + merging_params.check(getColumns().getAllPhysical()); if (!primary_expr_ast) throw Exception("Primary key cannot be empty", ErrorCodes::BAD_ARGUMENTS); @@ -222,11 +222,11 @@ void MergeTreeData::initPrimaryKey() primary_sort_descr.clear(); addSortDescription(primary_sort_descr, primary_expr_ast); - primary_expr = ExpressionAnalyzer(primary_expr_ast, context, nullptr, getColumns().getPhysical()).getActions(false); + primary_expr = ExpressionAnalyzer(primary_expr_ast, context, nullptr, getColumns().getAllPhysical()).getActions(false); { ExpressionActionsPtr projected_expr = - ExpressionAnalyzer(primary_expr_ast, context, nullptr, getColumns().getPhysical()).getActions(true); + ExpressionAnalyzer(primary_expr_ast, context, nullptr, getColumns().getAllPhysical()).getActions(true); primary_key_sample = projected_expr->getSampleBlock(); } @@ -241,10 +241,10 @@ void MergeTreeData::initPrimaryKey() if (secondary_sort_expr_ast) { addSortDescription(sort_descr, secondary_sort_expr_ast); - secondary_sort_expr = ExpressionAnalyzer(secondary_sort_expr_ast, context, nullptr, getColumns().getPhysical()).getActions(false); + secondary_sort_expr = ExpressionAnalyzer(secondary_sort_expr_ast, context, nullptr, getColumns().getAllPhysical()).getActions(false); ExpressionActionsPtr projected_expr = - ExpressionAnalyzer(secondary_sort_expr_ast, context, nullptr, getColumns().getPhysical()).getActions(true); + ExpressionAnalyzer(secondary_sort_expr_ast, context, nullptr, getColumns().getAllPhysical()).getActions(true); auto secondary_key_sample = projected_expr->getSampleBlock(); checkKeyExpression(*secondary_sort_expr, secondary_key_sample, "Secondary"); @@ -257,7 +257,7 @@ void MergeTreeData::initPartitionKey() if (!partition_expr_ast || partition_expr_ast->children.empty()) return; - partition_expr = ExpressionAnalyzer(partition_expr_ast, context, nullptr, getColumns().getPhysical()).getActions(false); + partition_expr = ExpressionAnalyzer(partition_expr_ast, context, nullptr, getColumns().getAllPhysical()).getActions(false); for (const ASTPtr & ast : partition_expr_ast->children) { String col_name = ast->getColumnName(); @@ -903,7 +903,7 @@ void MergeTreeData::checkAlter(const AlterCommands & commands) columns_alter_forbidden.insert(merging_params.sign_column); std::map old_types; - for (const auto & column : getColumns().getPhysical()) + for (const auto & column : getColumns().getAllPhysical()) old_types.emplace(column.name, column.type.get()); for (const AlterCommand & command : commands) @@ -931,7 +931,7 @@ void MergeTreeData::checkAlter(const AlterCommands & commands) NameToNameMap unused_map; bool unused_bool; - createConvertExpression(nullptr, getColumns().getPhysical(), new_columns.getPhysical(), unused_expression, unused_map, unused_bool); + createConvertExpression(nullptr, getColumns().getAllPhysical(), new_columns.getAllPhysical(), unused_expression, unused_map, unused_bool); } void MergeTreeData::createConvertExpression(const DataPartPtr & part, const NamesAndTypesList & old_columns, const NamesAndTypesList & new_columns, @@ -1835,7 +1835,7 @@ void MergeTreeData::addPartContributionToColumnSizes(const DataPartPtr & part) const auto & files = part->checksums.files; /// TODO This method doesn't take into account columns with multiple files. - for (const auto & column : getColumns().getPhysical()) + for (const auto & column : getColumns().getAllPhysical()) { const auto escaped_name = escapeForFileName(column.name); const auto bin_file_name = escaped_name + ".bin"; @@ -1868,7 +1868,7 @@ void MergeTreeData::removePartContributionToColumnSizes(const DataPartPtr & part const auto & files = part->checksums.files; /// TODO This method doesn't take into account columns with multiple files. - for (const auto & column : getColumns().getPhysical()) + for (const auto & column : getColumns().getAllPhysical()) { const auto escaped_name = escapeForFileName(column.name); const auto bin_file_name = escaped_name + ".bin"; diff --git a/dbms/src/Storages/MergeTree/MergeTreeData.h b/dbms/src/Storages/MergeTree/MergeTreeData.h index da6999ac965..d440d86d62b 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeData.h +++ b/dbms/src/Storages/MergeTree/MergeTreeData.h @@ -313,12 +313,12 @@ public: if (column_name == "_sample_factor") return NameAndTypePair("_sample_factor", std::make_shared()); - return getColumns().get(column_name); + return getColumns().getPhysical(column_name); } bool hasColumn(const String & column_name) const override { - return getColumns().has(column_name) + return getColumns().hasPhysical(column_name) || column_name == "_part" || column_name == "_part_index" || column_name == "_sample_factor"; diff --git a/dbms/src/Storages/MergeTree/MergeTreeDataMerger.cpp b/dbms/src/Storages/MergeTree/MergeTreeDataMerger.cpp index dafa27e2346..229c40404bb 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeDataMerger.cpp +++ b/dbms/src/Storages/MergeTree/MergeTreeDataMerger.cpp @@ -536,8 +536,8 @@ MergeTreeData::MutableDataPartPtr MergeTreeDataMerger::mergePartsToTemporaryPart for (const MergeTreeData::DataPartPtr & part : parts) part->accumulateColumnSizes(merged_column_to_size); - Names all_column_names = data.getColumns().getNames(); - NamesAndTypesList all_columns = data.getColumns().getPhysical(); + Names all_column_names = data.getColumns().getNamesOfPhysical(); + NamesAndTypesList all_columns = data.getColumns().getAllPhysical(); const SortDescription sort_desc = data.getSortDescription(); NamesAndTypesList gathering_columns, merging_columns; diff --git a/dbms/src/Storages/MergeTree/MergeTreeDataPart.cpp b/dbms/src/Storages/MergeTree/MergeTreeDataPart.cpp index 92936e2adc7..38608f6ea32 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeDataPart.cpp +++ b/dbms/src/Storages/MergeTree/MergeTreeDataPart.cpp @@ -418,7 +418,7 @@ UInt64 MergeTreeDataPart::getColumnMrkSize(const String & name) const */ String MergeTreeDataPart::getColumnNameWithMinumumCompressedSize() const { - const auto & columns = storage.getColumns().getPhysical(); + const auto & columns = storage.getColumns().getAllPhysical(); const std::string * minimum_size_column = nullptr; UInt64 minimum_size = std::numeric_limits::max(); @@ -774,7 +774,7 @@ void MergeTreeDataPart::accumulateColumnSizes(ColumnToSize & column_to_size) con { std::shared_lock part_lock(columns_lock); - for (const NameAndTypePair & name_type : storage.getColumns().getPhysical()) + for (const NameAndTypePair & name_type : storage.getColumns().getAllPhysical()) { name_type.type->enumerateStreams([&](const IDataType::SubstreamPath & substream_path) { @@ -794,7 +794,7 @@ void MergeTreeDataPart::loadColumns(bool require) throw Exception("No columns.txt in part " + name, ErrorCodes::NO_FILE_IN_DATA_PART); /// If there is no file with a list of columns, write it down. - for (const NameAndTypePair & column : storage.getColumns().getPhysical()) + for (const NameAndTypePair & column : storage.getColumns().getAllPhysical()) if (Poco::File(getFullPath() + escapeForFileName(column.name) + ".bin").exists()) columns.push_back(column); diff --git a/dbms/src/Storages/MergeTree/MergeTreeDataSelectExecutor.cpp b/dbms/src/Storages/MergeTree/MergeTreeDataSelectExecutor.cpp index 77ba844ee0b..afc9700cf0b 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeDataSelectExecutor.cpp +++ b/dbms/src/Storages/MergeTree/MergeTreeDataSelectExecutor.cpp @@ -175,7 +175,7 @@ BlockInputStreams MergeTreeDataSelectExecutor::read( } } - NamesAndTypesList available_real_columns = data.getColumns().getPhysical(); + NamesAndTypesList available_real_columns = data.getColumns().getAllPhysical(); NamesAndTypesList available_real_and_virtual_columns = available_real_columns; for (const auto & name : virt_column_names) @@ -857,7 +857,7 @@ void MergeTreeDataSelectExecutor::createPositiveSignCondition( arguments->children.push_back(sign); arguments->children.push_back(one); - out_expression = ExpressionAnalyzer(function, context, {}, data.getColumns().getPhysical()).getActions(false); + out_expression = ExpressionAnalyzer(function, context, {}, data.getColumns().getAllPhysical()).getActions(false); out_column = function->getColumnName(); } diff --git a/dbms/src/Storages/MergeTree/MergeTreeDataWriter.cpp b/dbms/src/Storages/MergeTree/MergeTreeDataWriter.cpp index c57d1d5bfdd..e85efa435fd 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeDataWriter.cpp +++ b/dbms/src/Storages/MergeTree/MergeTreeDataWriter.cpp @@ -205,7 +205,7 @@ MergeTreeData::MutableDataPartPtr MergeTreeDataWriter::writeTempPart(BlockWithPa /// either default lz4 or compression method with zero thresholds on absolute and relative part size. auto compression_settings = data.context.chooseCompressionSettings(0, 0); - NamesAndTypesList columns = data.getColumns().getPhysical().filter(block.getNames()); + NamesAndTypesList columns = data.getColumns().getAllPhysical().filter(block.getNames()); MergedBlockOutputStream out(data, new_data_part->getFullPath(), columns, compression_settings); out.writePrefix(); diff --git a/dbms/src/Storages/MergeTree/MergeTreeReadPool.cpp b/dbms/src/Storages/MergeTree/MergeTreeReadPool.cpp index 452aee79bf3..370797968b3 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeReadPool.cpp +++ b/dbms/src/Storages/MergeTree/MergeTreeReadPool.cpp @@ -238,7 +238,7 @@ std::vector MergeTreeReadPool::fillPerPartInfo( if (!required_column_names.empty()) data.check(part.data_part->columns, required_column_names); - const NamesAndTypesList & physical_columns = data.getColumns().getPhysical(); + const NamesAndTypesList & physical_columns = data.getColumns().getAllPhysical(); per_part_pre_columns.push_back(physical_columns.addTypes(required_pre_column_names)); per_part_columns.push_back(physical_columns.addTypes(required_column_names)); } diff --git a/dbms/src/Storages/MergeTree/MergeTreeWhereOptimizer.cpp b/dbms/src/Storages/MergeTree/MergeTreeWhereOptimizer.cpp index ac28d3e782d..077ea9fed5d 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeWhereOptimizer.cpp +++ b/dbms/src/Storages/MergeTree/MergeTreeWhereOptimizer.cpp @@ -41,9 +41,9 @@ MergeTreeWhereOptimizer::MergeTreeWhereOptimizer( Logger * log) : primary_key_columns{ext::map(data.getPrimarySortDescription(), [] (const SortColumnDescription & col) { return col.column_name; })}, - table_columns{ext::map(data.getColumns().getPhysical(), + table_columns{ext::map(data.getColumns().getAllPhysical(), [] (const NameAndTypePair & col) { return col.name; })}, - block_with_constants{PKCondition::getBlockWithConstants(query_info.query, context, data.getColumns().getPhysical())}, + block_with_constants{PKCondition::getBlockWithConstants(query_info.query, context, data.getColumns().getAllPhysical())}, prepared_sets(query_info.sets), log{log} { diff --git a/dbms/src/Storages/MergeTree/ReplicatedMergeTreeAlterThread.cpp b/dbms/src/Storages/MergeTree/ReplicatedMergeTreeAlterThread.cpp index 1fcba112fed..35a2dfcc6b7 100644 --- a/dbms/src/Storages/MergeTree/ReplicatedMergeTreeAlterThread.cpp +++ b/dbms/src/Storages/MergeTree/ReplicatedMergeTreeAlterThread.cpp @@ -124,7 +124,7 @@ void ReplicatedMergeTreeAlterThread::run() if (!changed_version) parts = storage.data.getDataParts(); - const auto columns_for_parts = storage.getColumns().getPhysical(); + const auto columns_for_parts = storage.getColumns().getAllPhysical(); for (const MergeTreeData::DataPartPtr & part : parts) { diff --git a/dbms/src/Storages/StorageDistributed.cpp b/dbms/src/Storages/StorageDistributed.cpp index 972a4817009..690d513f50e 100644 --- a/dbms/src/Storages/StorageDistributed.cpp +++ b/dbms/src/Storages/StorageDistributed.cpp @@ -142,7 +142,7 @@ StorageDistributed::StorageDistributed( table_name(table_name_), remote_database(remote_database_), remote_table(remote_table_), context(context_), cluster_name(context.getMacros().expand(cluster_name_)), has_sharding_key(sharding_key_), - sharding_key_expr(sharding_key_ ? ExpressionAnalyzer(sharding_key_, context, nullptr, getColumns().getPhysical()).getActions(false) : nullptr), + sharding_key_expr(sharding_key_ ? ExpressionAnalyzer(sharding_key_, context, nullptr, getColumns().getAllPhysical()).getActions(false) : nullptr), sharding_key_column_name(sharding_key_ ? sharding_key_->getColumnName() : String{}), path(data_path_.empty() ? "" : (data_path_ + escapeForFileName(table_name) + '/')) { @@ -287,13 +287,13 @@ NameAndTypePair StorageDistributed::getColumn(const String & column_name) const if (const auto & type = VirtualColumnFactory::tryGetType(column_name)) return { column_name, type }; - return getColumns().get(column_name); + return getColumns().getPhysical(column_name); } bool StorageDistributed::hasColumn(const String & column_name) const { - return VirtualColumnFactory::hasColumn(column_name) || getColumns().has(column_name); + return VirtualColumnFactory::hasColumn(column_name) || getColumns().hasPhysical(column_name); } void StorageDistributed::createDirectoryMonitors() @@ -386,7 +386,7 @@ void registerStorageDistributed(StorageFactory & factory) /// Check that sharding_key exists in the table and has numeric type. if (sharding_key) { - auto sharding_expr = ExpressionAnalyzer(sharding_key, args.context, nullptr, args.columns.getPhysical()).getActions(true); + auto sharding_expr = ExpressionAnalyzer(sharding_key, args.context, nullptr, args.columns.getAllPhysical()).getActions(true); const Block & block = sharding_expr->getSampleBlock(); if (block.columns() != 1) diff --git a/dbms/src/Storages/StorageJoin.cpp b/dbms/src/Storages/StorageJoin.cpp index 746f15bab50..aab48c3ed2a 100644 --- a/dbms/src/Storages/StorageJoin.cpp +++ b/dbms/src/Storages/StorageJoin.cpp @@ -29,7 +29,7 @@ StorageJoin::StorageJoin( key_names(key_names_), kind(kind_), strictness(strictness_) { for (const auto & key : key_names) - if (!getColumns().has(key)) + if (!getColumns().hasPhysical(key)) throw Exception{ "Key column (" + key + ") does not exist in table declaration.", ErrorCodes::NO_SUCH_COLUMN_IN_TABLE}; diff --git a/dbms/src/Storages/StorageLog.cpp b/dbms/src/Storages/StorageLog.cpp index c23d92de8fa..04d9f18ffd1 100644 --- a/dbms/src/Storages/StorageLog.cpp +++ b/dbms/src/Storages/StorageLog.cpp @@ -371,7 +371,7 @@ StorageLog::StorageLog( /// create files if they do not exist Poco::File(path + escapeForFileName(name) + '/').createDirectories(); - for (const auto & column : getColumns().getPhysical()) + for (const auto & column : getColumns().getAllPhysical()) addFiles(column.name, *column.type); marks_file = Poco::File(path + escapeForFileName(name) + '/' + DBMS_STORAGE_LOG_MARKS_FILE_NAME); @@ -499,7 +499,7 @@ BlockInputStreams StorageLog::read( processed_stage = QueryProcessingStage::FetchColumns; loadMarks(); - NamesAndTypesList all_columns = Nested::collect(getColumns().getPhysical().addTypes(column_names)); + NamesAndTypesList all_columns = Nested::collect(getColumns().getAllPhysical().addTypes(column_names)); std::shared_lock lock(rwlock); diff --git a/dbms/src/Storages/StorageMerge.cpp b/dbms/src/Storages/StorageMerge.cpp index 6f29850c8d1..e7090ad66ed 100644 --- a/dbms/src/Storages/StorageMerge.cpp +++ b/dbms/src/Storages/StorageMerge.cpp @@ -181,7 +181,7 @@ BlockInputStreams StorageMerge::read( /// If there are only virtual columns in query, you must request at least one other column. if (real_column_names.size() == 0) - real_column_names.push_back(ExpressionActions::getSmallestColumn(table->getColumns().getPhysical())); + real_column_names.push_back(ExpressionActions::getSmallestColumn(table->getColumns().getAllPhysical())); /// Substitute virtual column for its value when querying tables. ASTPtr modified_query_ast = query->clone(); diff --git a/dbms/src/Storages/StorageMergeTree.cpp b/dbms/src/Storages/StorageMergeTree.cpp index f22f24082fb..293c3254a8f 100644 --- a/dbms/src/Storages/StorageMergeTree.cpp +++ b/dbms/src/Storages/StorageMergeTree.cpp @@ -174,7 +174,7 @@ void StorageMergeTree::alter( throw Exception("MODIFY PRIMARY KEY only supported for tables without sampling key", ErrorCodes::BAD_ARGUMENTS); auto parts = data.getDataParts({MergeTreeDataPartState::PreCommitted, MergeTreeDataPartState::Committed, MergeTreeDataPartState::Outdated}); - auto columns_for_parts = new_columns.getPhysical(); + auto columns_for_parts = new_columns.getAllPhysical(); for (const MergeTreeData::DataPartPtr & part : parts) { if (auto transaction = data.alterDataPart(part, columns_for_parts, new_primary_key_ast, false)) @@ -418,7 +418,7 @@ void StorageMergeTree::clearColumnInPartition(const ASTPtr & partition, const Fi auto new_columns = getColumns(); alter_command.apply(new_columns); - auto columns_for_parts = new_columns.getPhysical(); + auto columns_for_parts = new_columns.getAllPhysical(); for (const auto & part : parts) { if (part->info.partition_id != partition_id) diff --git a/dbms/src/Storages/StorageReplicatedMergeTree.cpp b/dbms/src/Storages/StorageReplicatedMergeTree.cpp index c2cf65e1c85..7ff7b1373e0 100644 --- a/dbms/src/Storages/StorageReplicatedMergeTree.cpp +++ b/dbms/src/Storages/StorageReplicatedMergeTree.cpp @@ -1511,7 +1511,7 @@ void StorageReplicatedMergeTree::executeClearColumnInPartition(const LogEntry & size_t modified_parts = 0; auto parts = data.getDataParts(); - auto columns_for_parts = new_columns.getPhysical(); + auto columns_for_parts = new_columns.getAllPhysical(); for (const auto & part : parts) { if (!entry_part_info.contains(part->info)) diff --git a/dbms/src/Storages/StorageTinyLog.cpp b/dbms/src/Storages/StorageTinyLog.cpp index bd4a8ca304f..cb62c911e63 100644 --- a/dbms/src/Storages/StorageTinyLog.cpp +++ b/dbms/src/Storages/StorageTinyLog.cpp @@ -296,7 +296,7 @@ StorageTinyLog::StorageTinyLog( throwFromErrno("Cannot create directory " + full_path, ErrorCodes::CANNOT_CREATE_DIRECTORY); } - for (const auto & col : getColumns().getPhysical()) + for (const auto & col : getColumns().getAllPhysical()) addFiles(col.name, *col.type); } @@ -348,7 +348,7 @@ BlockInputStreams StorageTinyLog::read( check(column_names); processed_stage = QueryProcessingStage::FetchColumns; return BlockInputStreams(1, std::make_shared( - max_block_size, Nested::collect(getColumns().getPhysical().addTypes(column_names)), *this, context.getSettingsRef().max_read_buffer_size)); + max_block_size, Nested::collect(getColumns().getAllPhysical().addTypes(column_names)), *this, context.getSettingsRef().max_read_buffer_size)); } diff --git a/dbms/src/TableFunctions/TableFunctionMerge.cpp b/dbms/src/TableFunctions/TableFunctionMerge.cpp index 6b0710d4ba0..45aa42fea32 100644 --- a/dbms/src/TableFunctions/TableFunctionMerge.cpp +++ b/dbms/src/TableFunctions/TableFunctionMerge.cpp @@ -51,7 +51,7 @@ static NamesAndTypesList chooseColumns(const String & source_database, const Str throw Exception("Error while executing table function merge. In database " + source_database + " no one matches regular expression: " + table_name_regexp_, ErrorCodes::UNKNOWN_TABLE); - return any_table->getColumns().getPhysical(); + return any_table->getColumns().getAllPhysical(); } From 8bb3672beac6b2c659f13ff09b1e95da38a2263f Mon Sep 17 00:00:00 2001 From: alexey-milovidov Date: Wed, 14 Mar 2018 05:42:48 +0300 Subject: [PATCH 8/8] Update InterpreterSelectQuery.cpp --- dbms/src/Interpreters/InterpreterSelectQuery.cpp | 1 - 1 file changed, 1 deletion(-) diff --git a/dbms/src/Interpreters/InterpreterSelectQuery.cpp b/dbms/src/Interpreters/InterpreterSelectQuery.cpp index 49a46abeb13..49b3ba27004 100644 --- a/dbms/src/Interpreters/InterpreterSelectQuery.cpp +++ b/dbms/src/Interpreters/InterpreterSelectQuery.cpp @@ -712,7 +712,6 @@ QueryProcessingStage::Enum InterpreterSelectQuery::executeFetchColumns(Pipeline else throw Exception("Logical error in InterpreterSelectQuery: nowhere to read", ErrorCodes::LOGICAL_ERROR); - StackTrace st; /// Aliases in table declaration. if (from_stage == QueryProcessingStage::FetchColumns && alias_actions) {