diff --git a/src/Client/LocalConnection.cpp b/src/Client/LocalConnection.cpp index c135ced7c76..d01c40a9c34 100644 --- a/src/Client/LocalConnection.cpp +++ b/src/Client/LocalConnection.cpp @@ -131,7 +131,7 @@ void LocalConnection::sendQuery( try { - state->io = executeQuery(state->query, query_context, false, state->stage).second; + state->io = executeQuery(state->query, query_context, QueryFlags{}, state->stage).second; if (state->io.pipeline.pushing()) { diff --git a/src/Databases/DatabaseOnDisk.cpp b/src/Databases/DatabaseOnDisk.cpp index 01afbdcaa57..96c084a261c 100644 --- a/src/Databases/DatabaseOnDisk.cpp +++ b/src/Databases/DatabaseOnDisk.cpp @@ -73,7 +73,7 @@ std::pair createTableFromAST( auto table_function = factory.get(table_function_ast, context); ColumnsDescription columns; if (ast_create_query.columns_list && ast_create_query.columns_list->columns) - columns = InterpreterCreateQuery::getColumnsDescription(*ast_create_query.columns_list->columns, context, true); + columns = InterpreterCreateQuery::getColumnsDescription(*ast_create_query.columns_list->columns, context, true, false); StoragePtr storage = table_function->execute(table_function_ast, context, ast_create_query.getTable(), std::move(columns)); storage->renameInMemory(ast_create_query); return {ast_create_query.getTable(), storage}; @@ -99,7 +99,7 @@ std::pair createTableFromAST( } else { - columns = InterpreterCreateQuery::getColumnsDescription(*ast_create_query.columns_list->columns, context, true); + columns = InterpreterCreateQuery::getColumnsDescription(*ast_create_query.columns_list->columns, context, true, false); constraints = InterpreterCreateQuery::getConstraintsDescription(ast_create_query.columns_list->constraints); } } diff --git a/src/Databases/DatabaseReplicated.cpp b/src/Databases/DatabaseReplicated.cpp index e21e65ec340..c9bcaf4bbf0 100644 --- a/src/Databases/DatabaseReplicated.cpp +++ b/src/Databases/DatabaseReplicated.cpp @@ -722,7 +722,7 @@ void DatabaseReplicated::checkQueryValid(const ASTPtr & query, ContextPtr query_ } } -BlockIO DatabaseReplicated::tryEnqueueReplicatedDDL(const ASTPtr & query, ContextPtr query_context, bool internal) +BlockIO DatabaseReplicated::tryEnqueueReplicatedDDL(const ASTPtr & query, ContextPtr query_context, QueryFlags flags) { if (query_context->getCurrentTransaction() && query_context->getSettingsRef().throw_on_unsupported_query_inside_transaction) @@ -731,7 +731,7 @@ BlockIO DatabaseReplicated::tryEnqueueReplicatedDDL(const ASTPtr & query, Contex if (is_readonly) throw Exception(ErrorCodes::NO_ZOOKEEPER, "Database is in readonly mode, because it cannot connect to ZooKeeper"); - if (!internal && (query_context->getClientInfo().query_kind != ClientInfo::QueryKind::INITIAL_QUERY)) + if (!flags.internal && (query_context->getClientInfo().query_kind != ClientInfo::QueryKind::INITIAL_QUERY)) throw Exception(ErrorCodes::INCORRECT_QUERY, "It's not initial query. ON CLUSTER is not allowed for Replicated database."); checkQueryValid(query, query_context); @@ -742,6 +742,7 @@ BlockIO DatabaseReplicated::tryEnqueueReplicatedDDL(const ASTPtr & query, Contex entry.initiator = ddl_worker->getCommonHostID(); entry.setSettingsIfRequired(query_context); entry.tracing_context = OpenTelemetry::CurrentContext(); + entry.is_backup_restore = flags.distributed_backup_restore; String node_path = ddl_worker->tryEnqueueAndExecuteEntry(entry, query_context); Strings hosts_to_wait; @@ -919,14 +920,14 @@ void DatabaseReplicated::recoverLostReplica(const ZooKeeperPtr & current_zookeep String query = fmt::format("CREATE DATABASE IF NOT EXISTS {} ENGINE=Ordinary", backQuoteIfNeed(to_db_name)); auto query_context = Context::createCopy(getContext()); query_context->setSetting("allow_deprecated_database_ordinary", 1); - executeQuery(query, query_context, true); + executeQuery(query, query_context, QueryFlags{ .internal = true }); /// But we want to avoid discarding UUID of ReplicatedMergeTree tables, because it will not work /// if zookeeper_path contains {uuid} macro. Replicated database do not recreate replicated tables on recovery, /// so it's ok to save UUID of replicated table. query = fmt::format("CREATE DATABASE IF NOT EXISTS {} ENGINE=Atomic", backQuoteIfNeed(to_db_name_replicated)); query_context = Context::createCopy(getContext()); - executeQuery(query, query_context, true); + executeQuery(query, query_context, QueryFlags{ .internal = true }); } size_t moved_tables = 0; diff --git a/src/Databases/DatabaseReplicated.h b/src/Databases/DatabaseReplicated.h index 005180624ed..1387ba1cb96 100644 --- a/src/Databases/DatabaseReplicated.h +++ b/src/Databases/DatabaseReplicated.h @@ -46,7 +46,7 @@ public: /// Try to execute DLL query on current host as initial query. If query is succeed, /// then it will be executed on all replicas. - BlockIO tryEnqueueReplicatedDDL(const ASTPtr & query, ContextPtr query_context, bool internal) override; + BlockIO tryEnqueueReplicatedDDL(const ASTPtr & query, ContextPtr query_context, QueryFlags flags) override; bool canExecuteReplicatedMetadataAlter() const override; diff --git a/src/Databases/DatabasesCommon.cpp b/src/Databases/DatabasesCommon.cpp index 3ffa08f8ec7..9b85e7194d3 100644 --- a/src/Databases/DatabasesCommon.cpp +++ b/src/Databases/DatabasesCommon.cpp @@ -372,6 +372,7 @@ void DatabaseWithOwnTablesBase::createTableRestoredFromBackup(const ASTPtr & cre /// Creates a table by executing a "CREATE TABLE" query. InterpreterCreateQuery interpreter{create_table_query, local_context}; interpreter.setInternal(true); + interpreter.setIsRestoreFromBackup(true); interpreter.execute(); } diff --git a/src/Databases/IDatabase.h b/src/Databases/IDatabase.h index 01d940b0429..e886f1adae3 100644 --- a/src/Databases/IDatabase.h +++ b/src/Databases/IDatabase.h @@ -3,6 +3,7 @@ #include #include #include +#include #include #include #include @@ -345,7 +346,7 @@ public: virtual bool shouldReplicateQuery(const ContextPtr & /*query_context*/, const ASTPtr & /*query_ptr*/) const { return false; } - virtual BlockIO tryEnqueueReplicatedDDL(const ASTPtr & /*query*/, ContextPtr /*query_context*/, [[maybe_unused]] bool internal = false) /// NOLINT + virtual BlockIO tryEnqueueReplicatedDDL(const ASTPtr & /*query*/, ContextPtr /*query_context*/, [[maybe_unused]] QueryFlags flags = {}) /// NOLINT { throw Exception(ErrorCodes::LOGICAL_ERROR, "Database engine {} does not have replicated DDL queue", getEngineName()); } diff --git a/src/Databases/MySQL/MaterializedMySQLSyncThread.cpp b/src/Databases/MySQL/MaterializedMySQLSyncThread.cpp index 3d10e66e964..ddb331f714e 100644 --- a/src/Databases/MySQL/MaterializedMySQLSyncThread.cpp +++ b/src/Databases/MySQL/MaterializedMySQLSyncThread.cpp @@ -75,7 +75,7 @@ static BlockIO tryToExecuteQuery(const String & query_to_execute, ContextMutable if (!database.empty()) query_context->setCurrentDatabase(database); - return executeQuery("/*" + comment + "*/ " + query_to_execute, query_context, true).second; + return executeQuery("/*" + comment + "*/ " + query_to_execute, query_context, QueryFlags{ .internal = true }).second; } catch (...) { diff --git a/src/Dictionaries/ClickHouseDictionarySource.cpp b/src/Dictionaries/ClickHouseDictionarySource.cpp index 92fae2bc495..37f94062ef9 100644 --- a/src/Dictionaries/ClickHouseDictionarySource.cpp +++ b/src/Dictionaries/ClickHouseDictionarySource.cpp @@ -168,7 +168,7 @@ QueryPipeline ClickHouseDictionarySource::createStreamForQuery(const String & qu if (configuration.is_local) { - pipeline = executeQuery(query, context_copy, true).second.pipeline; + pipeline = executeQuery(query, context_copy, QueryFlags{ .internal = true }).second.pipeline; pipeline.convertStructureTo(empty_sample_block.getColumnsWithTypeAndName()); } else @@ -190,7 +190,7 @@ std::string ClickHouseDictionarySource::doInvalidateQuery(const std::string & re if (configuration.is_local) { - return readInvalidateQuery(executeQuery(request, context_copy, true).second.pipeline); + return readInvalidateQuery(executeQuery(request, context_copy, QueryFlags{ .internal = true }).second.pipeline); } else { diff --git a/src/Interpreters/Access/InterpreterShowAccessEntitiesQuery.cpp b/src/Interpreters/Access/InterpreterShowAccessEntitiesQuery.cpp index b0937dc2f66..bffb47ac714 100644 --- a/src/Interpreters/Access/InterpreterShowAccessEntitiesQuery.cpp +++ b/src/Interpreters/Access/InterpreterShowAccessEntitiesQuery.cpp @@ -23,7 +23,7 @@ InterpreterShowAccessEntitiesQuery::InterpreterShowAccessEntitiesQuery(const AST BlockIO InterpreterShowAccessEntitiesQuery::execute() { - return executeQuery(getRewrittenQuery(), getContext(), true).second; + return executeQuery(getRewrittenQuery(), getContext(), QueryFlags{ .internal = true }).second; } diff --git a/src/Interpreters/Access/InterpreterShowPrivilegesQuery.cpp b/src/Interpreters/Access/InterpreterShowPrivilegesQuery.cpp index 213e3c813fa..1a0b441a06d 100644 --- a/src/Interpreters/Access/InterpreterShowPrivilegesQuery.cpp +++ b/src/Interpreters/Access/InterpreterShowPrivilegesQuery.cpp @@ -12,7 +12,7 @@ InterpreterShowPrivilegesQuery::InterpreterShowPrivilegesQuery(const ASTPtr & qu BlockIO InterpreterShowPrivilegesQuery::execute() { - return executeQuery("SELECT * FROM system.privileges", context, true).second; + return executeQuery("SELECT * FROM system.privileges", context, QueryFlags{ .internal = true }).second; } } diff --git a/src/Interpreters/DDLTask.cpp b/src/Interpreters/DDLTask.cpp index 4e684f5899f..632061a8ec5 100644 --- a/src/Interpreters/DDLTask.cpp +++ b/src/Interpreters/DDLTask.cpp @@ -113,6 +113,9 @@ String DDLLogEntry::toString() const writeChar('\n', wb); } + if (version >= BACKUP_RESTORE_FLAG_IN_ZK_VERSION) + wb << "is_backup_restore: " << is_backup_restore << "\n"; + return wb.str(); } @@ -165,6 +168,12 @@ void DDLLogEntry::parse(const String & data) checkChar('\n', rb); } + if (version >= BACKUP_RESTORE_FLAG_IN_ZK_VERSION) + { + checkString("is_backup_restore: ", rb); + readBoolText(is_backup_restore, rb); + checkChar('\n', rb); + } assertEOF(rb); diff --git a/src/Interpreters/DDLTask.h b/src/Interpreters/DDLTask.h index e92b1f9a885..db8b0628b4b 100644 --- a/src/Interpreters/DDLTask.h +++ b/src/Interpreters/DDLTask.h @@ -72,10 +72,11 @@ struct DDLLogEntry static constexpr const UInt64 NORMALIZE_CREATE_ON_INITIATOR_VERSION = 3; static constexpr const UInt64 OPENTELEMETRY_ENABLED_VERSION = 4; static constexpr const UInt64 PRESERVE_INITIAL_QUERY_ID_VERSION = 5; + static constexpr const UInt64 BACKUP_RESTORE_FLAG_IN_ZK_VERSION = 6; /// Add new version here /// Remember to update the value below once new version is added - static constexpr const UInt64 DDL_ENTRY_FORMAT_MAX_VERSION = 5; + static constexpr const UInt64 DDL_ENTRY_FORMAT_MAX_VERSION = 6; UInt64 version = 1; String query; @@ -84,6 +85,7 @@ struct DDLLogEntry std::optional settings; OpenTelemetry::TracingContext tracing_context; String initial_query_id; + bool is_backup_restore = false; void setSettingsIfRequired(ContextPtr context); String toString() const; diff --git a/src/Interpreters/DDLWorker.cpp b/src/Interpreters/DDLWorker.cpp index 39b0ee0b814..ed72f5594e5 100644 --- a/src/Interpreters/DDLWorker.cpp +++ b/src/Interpreters/DDLWorker.cpp @@ -489,7 +489,8 @@ bool DDLWorker::tryExecuteQuery(DDLTaskBase & task, const ZooKeeperPtr & zookeep if (!task.is_initial_query) query_scope.emplace(query_context); - executeQuery(istr, ostr, !task.is_initial_query, query_context, {}); + + executeQuery(istr, ostr, !task.is_initial_query, query_context, {}, QueryFlags{ .internal = false, .distributed_backup_restore = task.entry.is_backup_restore }); if (auto txn = query_context->getZooKeeperMetadataTransaction()) { diff --git a/src/Interpreters/InterpreterCreateQuery.cpp b/src/Interpreters/InterpreterCreateQuery.cpp index 252f45677ef..053cd08f044 100644 --- a/src/Interpreters/InterpreterCreateQuery.cpp +++ b/src/Interpreters/InterpreterCreateQuery.cpp @@ -480,7 +480,7 @@ ASTPtr InterpreterCreateQuery::formatProjections(const ProjectionsDescription & } ColumnsDescription InterpreterCreateQuery::getColumnsDescription( - const ASTExpressionList & columns_ast, ContextPtr context_, bool attach) + const ASTExpressionList & columns_ast, ContextPtr context_, bool attach, bool is_restore_from_backup) { /// First, deduce implicit types. @@ -489,7 +489,7 @@ ColumnsDescription InterpreterCreateQuery::getColumnsDescription( ASTPtr default_expr_list = std::make_shared(); NamesAndTypesList column_names_and_types; - bool make_columns_nullable = !attach && context_->getSettingsRef().data_type_default_nullable; + bool make_columns_nullable = !attach && !is_restore_from_backup && context_->getSettingsRef().data_type_default_nullable; for (const auto & ast : columns_ast.children) { @@ -645,7 +645,7 @@ ColumnsDescription InterpreterCreateQuery::getColumnsDescription( res.add(std::move(column)); } - if (!attach && context_->getSettingsRef().flatten_nested) + if (!attach && !is_restore_from_backup && context_->getSettingsRef().flatten_nested) res.flattenNested(); if (res.getAllPhysical().empty()) @@ -692,7 +692,7 @@ InterpreterCreateQuery::TableProperties InterpreterCreateQuery::getTableProperti if (create.columns_list->columns) { - properties.columns = getColumnsDescription(*create.columns_list->columns, getContext(), create.attach); + properties.columns = getColumnsDescription(*create.columns_list->columns, getContext(), create.attach, is_restore_from_backup); } if (create.columns_list->indices) @@ -752,7 +752,6 @@ InterpreterCreateQuery::TableProperties InterpreterCreateQuery::getTableProperti } else if (create.select) { - Block as_select_sample; if (getContext()->getSettingsRef().allow_experimental_analyzer) @@ -1077,7 +1076,7 @@ BlockIO InterpreterCreateQuery::createTable(ASTCreateQuery & create) auto guard = DatabaseCatalog::instance().getDDLGuard(database_name, create.getTable()); create.setDatabase(database_name); guard->releaseTableLock(); - return database->tryEnqueueReplicatedDDL(query_ptr, getContext(), internal); + return database->tryEnqueueReplicatedDDL(query_ptr, getContext(), QueryFlags{ .internal = internal, .distributed_backup_restore = is_restore_from_backup }); } if (!create.cluster.empty()) @@ -1233,7 +1232,7 @@ BlockIO InterpreterCreateQuery::createTable(ASTCreateQuery & create) auto guard = DatabaseCatalog::instance().getDDLGuard(create.getDatabase(), create.getTable()); assertOrSetUUID(create, database); guard->releaseTableLock(); - return database->tryEnqueueReplicatedDDL(query_ptr, getContext(), internal); + return database->tryEnqueueReplicatedDDL(query_ptr, getContext(), QueryFlags{ .internal = internal, .distributed_backup_restore = is_restore_from_backup }); } if (!create.cluster.empty()) diff --git a/src/Interpreters/InterpreterCreateQuery.h b/src/Interpreters/InterpreterCreateQuery.h index 67339dea928..0843a7ad15a 100644 --- a/src/Interpreters/InterpreterCreateQuery.h +++ b/src/Interpreters/InterpreterCreateQuery.h @@ -66,9 +66,14 @@ public: need_ddl_guard = false; } + void setIsRestoreFromBackup(bool is_restore_from_backup_) + { + is_restore_from_backup = is_restore_from_backup_; + } + /// Obtain information about columns, their types, default values and column comments, /// for case when columns in CREATE query is specified explicitly. - static ColumnsDescription getColumnsDescription(const ASTExpressionList & columns, ContextPtr context, bool attach); + static ColumnsDescription getColumnsDescription(const ASTExpressionList & columns, ContextPtr context, bool attach, bool is_restore_from_backup); static ConstraintsDescription getConstraintsDescription(const ASTExpressionList * constraints); static void prepareOnClusterQuery(ASTCreateQuery & create, ContextPtr context, const String & cluster_name); @@ -116,6 +121,7 @@ private: bool force_attach = false; bool load_database_without_tables = false; bool need_ddl_guard = true; + bool is_restore_from_backup = false; mutable String as_database_saved; mutable String as_table_saved; diff --git a/src/Interpreters/InterpreterKillQueryQuery.cpp b/src/Interpreters/InterpreterKillQueryQuery.cpp index 1c2e3ff6777..6e1422f2938 100644 --- a/src/Interpreters/InterpreterKillQueryQuery.cpp +++ b/src/Interpreters/InterpreterKillQueryQuery.cpp @@ -420,7 +420,7 @@ Block InterpreterKillQueryQuery::getSelectResult(const String & columns, const S if (where_expression) select_query += " WHERE " + queryToString(where_expression); - auto io = executeQuery(select_query, getContext(), true).second; + auto io = executeQuery(select_query, getContext(), QueryFlags{ .internal = true }).second; PullingPipelineExecutor executor(io.pipeline); Block res; while (!res && executor.pull(res)); diff --git a/src/Interpreters/InterpreterShowColumnsQuery.cpp b/src/Interpreters/InterpreterShowColumnsQuery.cpp index 9dea0b9a188..a5b22387448 100644 --- a/src/Interpreters/InterpreterShowColumnsQuery.cpp +++ b/src/Interpreters/InterpreterShowColumnsQuery.cpp @@ -161,7 +161,7 @@ WHERE BlockIO InterpreterShowColumnsQuery::execute() { - return executeQuery(getRewrittenQuery(), getContext(), true).second; + return executeQuery(getRewrittenQuery(), getContext(), QueryFlags{ .internal = true }).second; } diff --git a/src/Interpreters/InterpreterShowEngineQuery.cpp b/src/Interpreters/InterpreterShowEngineQuery.cpp index a2367e9bfdf..2927fbd0f2d 100644 --- a/src/Interpreters/InterpreterShowEngineQuery.cpp +++ b/src/Interpreters/InterpreterShowEngineQuery.cpp @@ -12,7 +12,7 @@ namespace DB BlockIO InterpreterShowEnginesQuery::execute() { - return executeQuery("SELECT * FROM system.table_engines ORDER BY name", getContext(), true).second; + return executeQuery("SELECT * FROM system.table_engines ORDER BY name", getContext(), QueryFlags{ .internal = true }).second; } } diff --git a/src/Interpreters/InterpreterShowFunctionsQuery.cpp b/src/Interpreters/InterpreterShowFunctionsQuery.cpp index ace22ca4bb6..a9da01b0988 100644 --- a/src/Interpreters/InterpreterShowFunctionsQuery.cpp +++ b/src/Interpreters/InterpreterShowFunctionsQuery.cpp @@ -15,7 +15,7 @@ InterpreterShowFunctionsQuery::InterpreterShowFunctionsQuery(const ASTPtr & quer BlockIO InterpreterShowFunctionsQuery::execute() { - return executeQuery(getRewrittenQuery(), getContext(), true).second; + return executeQuery(getRewrittenQuery(), getContext(), QueryFlags{ .internal = true }).second; } String InterpreterShowFunctionsQuery::getRewrittenQuery() diff --git a/src/Interpreters/InterpreterShowIndexesQuery.cpp b/src/Interpreters/InterpreterShowIndexesQuery.cpp index 549afd32506..09b70e951db 100644 --- a/src/Interpreters/InterpreterShowIndexesQuery.cpp +++ b/src/Interpreters/InterpreterShowIndexesQuery.cpp @@ -101,7 +101,7 @@ ORDER BY index_type, expression, column_name, seq_in_index;)", database, table, BlockIO InterpreterShowIndexesQuery::execute() { - return executeQuery(getRewrittenQuery(), getContext(), true).second; + return executeQuery(getRewrittenQuery(), getContext(), QueryFlags{ .internal = true }).second; } diff --git a/src/Interpreters/InterpreterShowProcesslistQuery.cpp b/src/Interpreters/InterpreterShowProcesslistQuery.cpp index 4ed5f4171c6..f711cc0dac9 100644 --- a/src/Interpreters/InterpreterShowProcesslistQuery.cpp +++ b/src/Interpreters/InterpreterShowProcesslistQuery.cpp @@ -12,7 +12,7 @@ namespace DB BlockIO InterpreterShowProcesslistQuery::execute() { - return executeQuery("SELECT * FROM system.processes ORDER BY elapsed DESC", getContext(), true).second; + return executeQuery("SELECT * FROM system.processes ORDER BY elapsed DESC", getContext(), QueryFlags{ .internal = true }).second; } } diff --git a/src/Interpreters/InterpreterShowSettingQuery.cpp b/src/Interpreters/InterpreterShowSettingQuery.cpp index 7567e77d28f..45e9b8a1f1c 100644 --- a/src/Interpreters/InterpreterShowSettingQuery.cpp +++ b/src/Interpreters/InterpreterShowSettingQuery.cpp @@ -26,9 +26,8 @@ String InterpreterShowSettingQuery::getRewrittenQuery() BlockIO InterpreterShowSettingQuery::execute() { - return executeQuery(getRewrittenQuery(), getContext(), true).second; + return executeQuery(getRewrittenQuery(), getContext(), QueryFlags{ .internal = true }).second; } } - diff --git a/src/Interpreters/InterpreterShowTablesQuery.cpp b/src/Interpreters/InterpreterShowTablesQuery.cpp index a8db55a317a..0ca6578128d 100644 --- a/src/Interpreters/InterpreterShowTablesQuery.cpp +++ b/src/Interpreters/InterpreterShowTablesQuery.cpp @@ -214,7 +214,7 @@ BlockIO InterpreterShowTablesQuery::execute() return res; } - return executeQuery(getRewrittenQuery(), getContext(), true).second; + return executeQuery(getRewrittenQuery(), getContext(), QueryFlags{ .internal = true }).second; } /// (*) Sorting is strictly speaking not necessary but 1. it is convenient for users, 2. SQL currently does not allow to diff --git a/src/Interpreters/InterpreterSystemQuery.cpp b/src/Interpreters/InterpreterSystemQuery.cpp index 8fac8deeca5..ea0e95c2b27 100644 --- a/src/Interpreters/InterpreterSystemQuery.cpp +++ b/src/Interpreters/InterpreterSystemQuery.cpp @@ -750,7 +750,7 @@ StoragePtr InterpreterSystemQuery::tryRestartReplica(const StorageID & replica, auto & create = create_ast->as(); create.attach = true; - auto columns = InterpreterCreateQuery::getColumnsDescription(*create.columns_list->columns, system_context, true); + auto columns = InterpreterCreateQuery::getColumnsDescription(*create.columns_list->columns, system_context, true, false); auto constraints = InterpreterCreateQuery::getConstraintsDescription(create.columns_list->constraints); auto data_path = database->getTableDataPath(create); diff --git a/src/Interpreters/MySQL/InterpretersMySQLDDLQuery.h b/src/Interpreters/MySQL/InterpretersMySQLDDLQuery.h index 27f467a12ae..484fd6a0207 100644 --- a/src/Interpreters/MySQL/InterpretersMySQLDDLQuery.h +++ b/src/Interpreters/MySQL/InterpretersMySQLDDLQuery.h @@ -75,7 +75,7 @@ public: ASTs rewritten_queries = InterpreterImpl::getRewrittenQueries(query, getContext(), mapped_to_database, mysql_database); for (const auto & rewritten_query : rewritten_queries) - executeQuery("/* Rewritten MySQL DDL Query */ " + queryToString(rewritten_query), getContext(), true); + executeQuery("/* Rewritten MySQL DDL Query */ " + queryToString(rewritten_query), getContext(), QueryFlags{ .internal = true }); return BlockIO{}; } diff --git a/src/Interpreters/executeQuery.cpp b/src/Interpreters/executeQuery.cpp index a5c0d9421d9..8cd3c8ab848 100644 --- a/src/Interpreters/executeQuery.cpp +++ b/src/Interpreters/executeQuery.cpp @@ -45,6 +45,7 @@ #include #include #include +#include #include #include #include @@ -646,10 +647,12 @@ static std::tuple executeQueryImpl( const char * begin, const char * end, ContextMutablePtr context, - bool internal, + QueryFlags flags, QueryProcessingStage::Enum stage, ReadBuffer * istr) { + const bool internal = flags.internal; + /// query_span is a special span, when this function exits, it's lifetime is not ended, but ends when the query finishes. /// Some internal queries might call this function recursively by setting 'internal' parameter to 'true', /// to make sure SpanHolders in current stack ends in correct order, we disable this span for these internal queries @@ -1085,6 +1088,9 @@ static std::tuple executeQueryImpl( insert_interpreter->addBuffer(std::move(insert_data_buffer_holder)); } + if (auto * create_interpreter = typeid_cast(&*interpreter)) + create_interpreter->setIsRestoreFromBackup(flags.distributed_backup_restore); + { std::unique_ptr span; if (OpenTelemetry::CurrentContext().isTraceEnabled()) @@ -1257,13 +1263,13 @@ static std::tuple executeQueryImpl( std::pair executeQuery( const String & query, ContextMutablePtr context, - bool internal, + QueryFlags flags, QueryProcessingStage::Enum stage) { ASTPtr ast; BlockIO res; - std::tie(ast, res) = executeQueryImpl(query.data(), query.data() + query.size(), context, internal, stage, nullptr); + std::tie(ast, res) = executeQueryImpl(query.data(), query.data() + query.size(), context, flags, stage, nullptr); if (const auto * ast_query_with_output = dynamic_cast(ast.get())) { @@ -1284,6 +1290,7 @@ void executeQuery( bool allow_into_outfile, ContextMutablePtr context, SetResultDetailsFunc set_result_details, + QueryFlags flags, const std::optional & output_format_settings, HandleExceptionInOutputFormatFunc handle_exception_in_output_format) { @@ -1372,7 +1379,7 @@ void executeQuery( try { - std::tie(ast, streams) = executeQueryImpl(begin, end, context, false, QueryProcessingStage::Complete, &istr); + std::tie(ast, streams) = executeQueryImpl(begin, end, context, flags, QueryProcessingStage::Complete, &istr); } catch (...) { diff --git a/src/Interpreters/executeQuery.h b/src/Interpreters/executeQuery.h index 6f14f54d7d6..0f599922668 100644 --- a/src/Interpreters/executeQuery.h +++ b/src/Interpreters/executeQuery.h @@ -29,6 +29,13 @@ struct QueryResultDetails using SetResultDetailsFunc = std::function; using HandleExceptionInOutputFormatFunc = std::function; +struct QueryFlags +{ + bool internal = false; /// If true, this query is caused by another query and thus needn't be registered in the ProcessList. + bool distributed_backup_restore = false; /// If true, this query is a part of backup restore. +}; + + /// Parse and execute a query. void executeQuery( ReadBuffer & istr, /// Where to read query from (and data for INSERT, if present). @@ -36,6 +43,7 @@ void executeQuery( bool allow_into_outfile, /// If true and the query contains INTO OUTFILE section, redirect output to that file. ContextMutablePtr context, /// DB, tables, data types, storage engines, functions, aggregate functions... SetResultDetailsFunc set_result_details, /// If a non-empty callback is passed, it will be called with the query id, the content-type, the format, and the timezone. + QueryFlags flags = {}, const std::optional & output_format_settings = std::nullopt, /// Format settings for output format, will be calculated from the context if not set. HandleExceptionInOutputFormatFunc handle_exception_in_output_format = {} /// If a non-empty callback is passed, it will be called on exception with created output format. ); @@ -58,7 +66,7 @@ void executeQuery( std::pair executeQuery( const String & query, /// Query text without INSERT data. The latter must be written to BlockIO::out. ContextMutablePtr context, /// DB, tables, data types, storage engines, functions, aggregate functions... - bool internal = false, /// If true, this query is caused by another query and thus needn't be registered in the ProcessList. + QueryFlags flags = {}, QueryProcessingStage::Enum stage = QueryProcessingStage::Complete /// To which stage the query must be executed. ); diff --git a/src/Interpreters/fuzzers/execute_query_fuzzer.cpp b/src/Interpreters/fuzzers/execute_query_fuzzer.cpp index 0f6bfc1ae58..40e2325e46e 100644 --- a/src/Interpreters/fuzzers/execute_query_fuzzer.cpp +++ b/src/Interpreters/fuzzers/execute_query_fuzzer.cpp @@ -42,7 +42,7 @@ extern "C" int LLVMFuzzerTestOneInput(const uint8_t * data, size_t size) static bool initialized = initialize(); (void) initialized; - auto io = DB::executeQuery(input, context, true, QueryProcessingStage::Complete).second; + auto io = DB::executeQuery(input, context, QueryFlags{ .internal = true }, QueryProcessingStage::Complete).second; PullingPipelineExecutor executor(io.pipeline); Block res; diff --git a/src/Interpreters/loadMetadata.cpp b/src/Interpreters/loadMetadata.cpp index faa1dcda2c0..3612dbfdc4e 100644 --- a/src/Interpreters/loadMetadata.cpp +++ b/src/Interpreters/loadMetadata.cpp @@ -282,7 +282,7 @@ static void convertOrdinaryDatabaseToAtomic(Poco::Logger * log, ContextMutablePt LOG_INFO(log, "Will convert database {} from Ordinary to Atomic", name_quoted); String create_database_query = fmt::format("CREATE DATABASE IF NOT EXISTS {}", tmp_name_quoted); - auto res = executeQuery(create_database_query, context, true).second; + auto res = executeQuery(create_database_query, context, QueryFlags{ .internal = true }).second; executeTrivialBlockIO(res, context); res = {}; auto tmp_database = DatabaseCatalog::instance().getDatabase(tmp_name); @@ -322,7 +322,7 @@ static void convertOrdinaryDatabaseToAtomic(Poco::Logger * log, ContextMutablePt String tmp_qualified_quoted_name = id.getFullTableName(); String move_table_query = fmt::format("RENAME TABLE {} TO {}", qualified_quoted_name, tmp_qualified_quoted_name); - res = executeQuery(move_table_query, context, true).second; + res = executeQuery(move_table_query, context, QueryFlags{ .internal = true }).second; executeTrivialBlockIO(res, context); res = {}; } @@ -334,12 +334,12 @@ static void convertOrdinaryDatabaseToAtomic(Poco::Logger * log, ContextMutablePt String drop_query = fmt::format("DROP DATABASE {}", name_quoted); context->setSetting("force_remove_data_recursively_on_drop", false); - res = executeQuery(drop_query, context, true).second; + res = executeQuery(drop_query, context, QueryFlags{ .internal = true }).second; executeTrivialBlockIO(res, context); res = {}; String rename_query = fmt::format("RENAME DATABASE {} TO {}", tmp_name_quoted, name_quoted); - res = executeQuery(rename_query, context, true).second; + res = executeQuery(rename_query, context, QueryFlags{ .internal = true }).second; executeTrivialBlockIO(res, context); LOG_INFO(log, "Finished database engine conversion of {}", name_quoted); @@ -409,7 +409,7 @@ static void maybeConvertOrdinaryDatabaseToAtomic(ContextMutablePtr context, cons /// Reload database just in case (and update logger name) String detach_query = fmt::format("DETACH DATABASE {}", backQuoteIfNeed(database_name)); - auto res = executeQuery(detach_query, context, true).second; + auto res = executeQuery(detach_query, context, QueryFlags{ .internal = true }).second; executeTrivialBlockIO(res, context); res = {}; diff --git a/src/Interpreters/parseColumnsListForTableFunction.cpp b/src/Interpreters/parseColumnsListForTableFunction.cpp index 196053fe509..87f76f7f824 100644 --- a/src/Interpreters/parseColumnsListForTableFunction.cpp +++ b/src/Interpreters/parseColumnsListForTableFunction.cpp @@ -73,7 +73,7 @@ ColumnsDescription parseColumnsListFromString(const std::string & structure, con if (!columns_list) throw Exception(ErrorCodes::LOGICAL_ERROR, "Could not cast AST to ASTExpressionList"); - auto columns = InterpreterCreateQuery::getColumnsDescription(*columns_list, context, false); + auto columns = InterpreterCreateQuery::getColumnsDescription(*columns_list, context, false, false); auto validation_settings = DataTypeValidationSettings(context->getSettingsRef()); for (const auto & [name, type] : columns.getAll()) validateDataType(type, validation_settings); @@ -100,7 +100,7 @@ bool tryParseColumnsListFromString(const std::string & structure, ColumnsDescrip try { - columns = InterpreterCreateQuery::getColumnsDescription(*columns_list, context, false); + columns = InterpreterCreateQuery::getColumnsDescription(*columns_list, context, false, false); auto validation_settings = DataTypeValidationSettings(context->getSettingsRef()); for (const auto & [name, type] : columns.getAll()) validateDataType(type, validation_settings); diff --git a/src/Server/HTTPHandler.cpp b/src/Server/HTTPHandler.cpp index a2d067af387..f9cd3b40f4a 100644 --- a/src/Server/HTTPHandler.cpp +++ b/src/Server/HTTPHandler.cpp @@ -886,6 +886,7 @@ void HTTPHandler::processQuery( /* allow_into_outfile = */ false, context, set_query_result, + QueryFlags{}, {}, handle_exception_in_output_format); diff --git a/src/Server/MySQLHandler.cpp b/src/Server/MySQLHandler.cpp index f9155a07e2b..21fa7f7227a 100644 --- a/src/Server/MySQLHandler.cpp +++ b/src/Server/MySQLHandler.cpp @@ -378,7 +378,7 @@ void MySQLHandler::comQuery(ReadBuffer & payload, bool binary_protocol) } }; - executeQuery(should_replace ? replacement : payload, *out, false, query_context, set_result_details, format_settings); + executeQuery(should_replace ? replacement : payload, *out, false, query_context, set_result_details, QueryFlags{}, format_settings); if (!with_output) packet_endpoint->sendPacket(OKPacket(0x00, client_capabilities, affected_rows, 0, 0), true); diff --git a/src/Server/TCPHandler.cpp b/src/Server/TCPHandler.cpp index 5082d8e4f3b..1da9806b4f5 100644 --- a/src/Server/TCPHandler.cpp +++ b/src/Server/TCPHandler.cpp @@ -496,7 +496,7 @@ void TCPHandler::runImpl() }); /// Processing Query - std::tie(state.parsed_query, state.io) = executeQuery(state.query, query_context, false, state.stage); + std::tie(state.parsed_query, state.io) = executeQuery(state.query, query_context, QueryFlags{}, state.stage); after_check_cancelled.restart(); after_send_progress.restart(); diff --git a/tests/queries/0_stateless/02907_backup_restore_default_nullable.reference b/tests/queries/0_stateless/02907_backup_restore_default_nullable.reference new file mode 100644 index 00000000000..0aed0444667 --- /dev/null +++ b/tests/queries/0_stateless/02907_backup_restore_default_nullable.reference @@ -0,0 +1,4 @@ +BACKUP_CREATED +CREATE TABLE default.test\n(\n `test` String\n)\nENGINE = MergeTree\nORDER BY tuple()\nSETTINGS index_granularity = 8192 +RESTORED +CREATE TABLE default.test\n(\n `test` String\n)\nENGINE = MergeTree\nORDER BY tuple()\nSETTINGS index_granularity = 8192 diff --git a/tests/queries/0_stateless/02907_backup_restore_default_nullable.sh b/tests/queries/0_stateless/02907_backup_restore_default_nullable.sh new file mode 100755 index 00000000000..8ed36a7edd7 --- /dev/null +++ b/tests/queries/0_stateless/02907_backup_restore_default_nullable.sh @@ -0,0 +1,22 @@ +#!/usr/bin/env bash + +CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh +. "$CUR_DIR"/../shell_config.sh + +${CLICKHOUSE_CLIENT} -nm --query " +drop table if exists test; +set data_type_default_nullable = 0; +create table test (test String) ENGINE = MergeTree() ORDER BY tuple(); +backup table ${CLICKHOUSE_DATABASE}.test on cluster test_shard_localhost to Disk('backups', '${CLICKHOUSE_TEST_UNIQUE_NAME}'); +" | grep -o "BACKUP_CREATED" + +${CLICKHOUSE_CLIENT} --query "show create table test" + +${CLICKHOUSE_CLIENT} -nm --query " +drop table test sync; +set data_type_default_nullable = 1; +restore table ${CLICKHOUSE_DATABASE}.test on cluster test_shard_localhost from Disk('backups', '${CLICKHOUSE_TEST_UNIQUE_NAME}'); +" | grep -o "RESTORED" + +${CLICKHOUSE_CLIENT} --query "show create table test" diff --git a/tests/queries/0_stateless/02907_backup_restore_flatten_nested.reference b/tests/queries/0_stateless/02907_backup_restore_flatten_nested.reference new file mode 100644 index 00000000000..aa8f22f590a --- /dev/null +++ b/tests/queries/0_stateless/02907_backup_restore_flatten_nested.reference @@ -0,0 +1,8 @@ +BACKUP_CREATED +CREATE TABLE default.test\n(\n `test` Array(Tuple(foo String, bar Float64))\n)\nENGINE = MergeTree\nORDER BY tuple()\nSETTINGS index_granularity = 8192 +BACKUP_CREATED +CREATE TABLE default.test2\n(\n `test` Nested(foo String, bar Float64)\n)\nENGINE = MergeTree\nORDER BY tuple()\nSETTINGS index_granularity = 8192 +RESTORED +CREATE TABLE default.test\n(\n `test` Array(Tuple(foo String, bar Float64))\n)\nENGINE = MergeTree\nORDER BY tuple()\nSETTINGS index_granularity = 8192 +RESTORED +CREATE TABLE default.test2\n(\n `test` Nested(foo String, bar Float64)\n)\nENGINE = MergeTree\nORDER BY tuple()\nSETTINGS index_granularity = 8192 diff --git a/tests/queries/0_stateless/02907_backup_restore_flatten_nested.sh b/tests/queries/0_stateless/02907_backup_restore_flatten_nested.sh new file mode 100755 index 00000000000..742d24a97eb --- /dev/null +++ b/tests/queries/0_stateless/02907_backup_restore_flatten_nested.sh @@ -0,0 +1,39 @@ +#!/usr/bin/env bash + +CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh +. "$CUR_DIR"/../shell_config.sh + +${CLICKHOUSE_CLIENT} -nm --query " +drop table if exists test; +set flatten_nested = 0; +create table test (test Array(Tuple(foo String, bar Float64))) ENGINE = MergeTree() ORDER BY tuple(); +backup table ${CLICKHOUSE_DATABASE}.test on cluster test_shard_localhost to Disk('backups', '${CLICKHOUSE_TEST_UNIQUE_NAME}'); +" | grep -o "BACKUP_CREATED" + +${CLICKHOUSE_CLIENT} --query "show create table test" + +${CLICKHOUSE_CLIENT} -nm --query " +drop table if exists test2; +set flatten_nested = 0; +create table test2 (test Nested(foo String, bar Float64)) ENGINE = MergeTree() ORDER BY tuple(); +backup table ${CLICKHOUSE_DATABASE}.test2 on cluster test_shard_localhost to Disk('backups', '${CLICKHOUSE_TEST_UNIQUE_NAME}2'); +" | grep -o "BACKUP_CREATED" + +${CLICKHOUSE_CLIENT} --query "show create table test2" + +${CLICKHOUSE_CLIENT} -nm --query " +drop table test sync; +set flatten_nested = 1; +restore table ${CLICKHOUSE_DATABASE}.test on cluster test_shard_localhost from Disk('backups', '${CLICKHOUSE_TEST_UNIQUE_NAME}'); +" | grep -o "RESTORED" + +${CLICKHOUSE_CLIENT} --query "show create table test" + +${CLICKHOUSE_CLIENT} -nm --query " +drop table test2 sync; +set flatten_nested = 1; +restore table ${CLICKHOUSE_DATABASE}.test2 on cluster test_shard_localhost from Disk('backups', '${CLICKHOUSE_TEST_UNIQUE_NAME}2'); +" | grep -o "RESTORED" + +${CLICKHOUSE_CLIENT} --query "show create table test2"