From ea5796fcb6860d7bb1854f8ad39ab7573941cb87 Mon Sep 17 00:00:00 2001 From: kssenii Date: Fri, 3 Nov 2023 16:46:07 +0100 Subject: [PATCH 1/9] Fix --- src/Databases/DatabaseOnDisk.cpp | 4 ++-- src/Databases/DatabasesCommon.cpp | 1 + src/Interpreters/InterpreterCreateQuery.cpp | 7 +++--- src/Interpreters/InterpreterCreateQuery.h | 8 ++++++- src/Interpreters/InterpreterSystemQuery.cpp | 2 +- .../parseColumnsListForTableFunction.cpp | 4 ++-- ...07_backup_restore_flatten_nested.reference | 4 ++++ .../02907_backup_restore_flatten_nested.sh | 22 +++++++++++++++++++ 8 files changed, 42 insertions(+), 10 deletions(-) create mode 100644 tests/queries/0_stateless/02907_backup_restore_flatten_nested.reference create mode 100755 tests/queries/0_stateless/02907_backup_restore_flatten_nested.sh diff --git a/src/Databases/DatabaseOnDisk.cpp b/src/Databases/DatabaseOnDisk.cpp index 01afbdcaa57..96c084a261c 100644 --- a/src/Databases/DatabaseOnDisk.cpp +++ b/src/Databases/DatabaseOnDisk.cpp @@ -73,7 +73,7 @@ std::pair createTableFromAST( auto table_function = factory.get(table_function_ast, context); ColumnsDescription columns; if (ast_create_query.columns_list && ast_create_query.columns_list->columns) - columns = InterpreterCreateQuery::getColumnsDescription(*ast_create_query.columns_list->columns, context, true); + columns = InterpreterCreateQuery::getColumnsDescription(*ast_create_query.columns_list->columns, context, true, false); StoragePtr storage = table_function->execute(table_function_ast, context, ast_create_query.getTable(), std::move(columns)); storage->renameInMemory(ast_create_query); return {ast_create_query.getTable(), storage}; @@ -99,7 +99,7 @@ std::pair createTableFromAST( } else { - columns = InterpreterCreateQuery::getColumnsDescription(*ast_create_query.columns_list->columns, context, true); + columns = InterpreterCreateQuery::getColumnsDescription(*ast_create_query.columns_list->columns, context, true, false); constraints = InterpreterCreateQuery::getConstraintsDescription(ast_create_query.columns_list->constraints); } } diff --git a/src/Databases/DatabasesCommon.cpp b/src/Databases/DatabasesCommon.cpp index 3ffa08f8ec7..9b85e7194d3 100644 --- a/src/Databases/DatabasesCommon.cpp +++ b/src/Databases/DatabasesCommon.cpp @@ -372,6 +372,7 @@ void DatabaseWithOwnTablesBase::createTableRestoredFromBackup(const ASTPtr & cre /// Creates a table by executing a "CREATE TABLE" query. InterpreterCreateQuery interpreter{create_table_query, local_context}; interpreter.setInternal(true); + interpreter.setIsRestoreFromBackup(true); interpreter.execute(); } diff --git a/src/Interpreters/InterpreterCreateQuery.cpp b/src/Interpreters/InterpreterCreateQuery.cpp index 252f45677ef..ca865264fcf 100644 --- a/src/Interpreters/InterpreterCreateQuery.cpp +++ b/src/Interpreters/InterpreterCreateQuery.cpp @@ -480,7 +480,7 @@ ASTPtr InterpreterCreateQuery::formatProjections(const ProjectionsDescription & } ColumnsDescription InterpreterCreateQuery::getColumnsDescription( - const ASTExpressionList & columns_ast, ContextPtr context_, bool attach) + const ASTExpressionList & columns_ast, ContextPtr context_, bool attach, bool is_restore_from_backup) { /// First, deduce implicit types. @@ -645,7 +645,7 @@ ColumnsDescription InterpreterCreateQuery::getColumnsDescription( res.add(std::move(column)); } - if (!attach && context_->getSettingsRef().flatten_nested) + if (!attach && !is_restore_from_backup && context_->getSettingsRef().flatten_nested) res.flattenNested(); if (res.getAllPhysical().empty()) @@ -692,7 +692,7 @@ InterpreterCreateQuery::TableProperties InterpreterCreateQuery::getTableProperti if (create.columns_list->columns) { - properties.columns = getColumnsDescription(*create.columns_list->columns, getContext(), create.attach); + properties.columns = getColumnsDescription(*create.columns_list->columns, getContext(), create.attach, is_restore_from_backup); } if (create.columns_list->indices) @@ -752,7 +752,6 @@ InterpreterCreateQuery::TableProperties InterpreterCreateQuery::getTableProperti } else if (create.select) { - Block as_select_sample; if (getContext()->getSettingsRef().allow_experimental_analyzer) diff --git a/src/Interpreters/InterpreterCreateQuery.h b/src/Interpreters/InterpreterCreateQuery.h index 67339dea928..0843a7ad15a 100644 --- a/src/Interpreters/InterpreterCreateQuery.h +++ b/src/Interpreters/InterpreterCreateQuery.h @@ -66,9 +66,14 @@ public: need_ddl_guard = false; } + void setIsRestoreFromBackup(bool is_restore_from_backup_) + { + is_restore_from_backup = is_restore_from_backup_; + } + /// Obtain information about columns, their types, default values and column comments, /// for case when columns in CREATE query is specified explicitly. - static ColumnsDescription getColumnsDescription(const ASTExpressionList & columns, ContextPtr context, bool attach); + static ColumnsDescription getColumnsDescription(const ASTExpressionList & columns, ContextPtr context, bool attach, bool is_restore_from_backup); static ConstraintsDescription getConstraintsDescription(const ASTExpressionList * constraints); static void prepareOnClusterQuery(ASTCreateQuery & create, ContextPtr context, const String & cluster_name); @@ -116,6 +121,7 @@ private: bool force_attach = false; bool load_database_without_tables = false; bool need_ddl_guard = true; + bool is_restore_from_backup = false; mutable String as_database_saved; mutable String as_table_saved; diff --git a/src/Interpreters/InterpreterSystemQuery.cpp b/src/Interpreters/InterpreterSystemQuery.cpp index 07a1ae7d170..ea4a5605256 100644 --- a/src/Interpreters/InterpreterSystemQuery.cpp +++ b/src/Interpreters/InterpreterSystemQuery.cpp @@ -745,7 +745,7 @@ StoragePtr InterpreterSystemQuery::tryRestartReplica(const StorageID & replica, auto & create = create_ast->as(); create.attach = true; - auto columns = InterpreterCreateQuery::getColumnsDescription(*create.columns_list->columns, system_context, true); + auto columns = InterpreterCreateQuery::getColumnsDescription(*create.columns_list->columns, system_context, true, false); auto constraints = InterpreterCreateQuery::getConstraintsDescription(create.columns_list->constraints); auto data_path = database->getTableDataPath(create); diff --git a/src/Interpreters/parseColumnsListForTableFunction.cpp b/src/Interpreters/parseColumnsListForTableFunction.cpp index 196053fe509..87f76f7f824 100644 --- a/src/Interpreters/parseColumnsListForTableFunction.cpp +++ b/src/Interpreters/parseColumnsListForTableFunction.cpp @@ -73,7 +73,7 @@ ColumnsDescription parseColumnsListFromString(const std::string & structure, con if (!columns_list) throw Exception(ErrorCodes::LOGICAL_ERROR, "Could not cast AST to ASTExpressionList"); - auto columns = InterpreterCreateQuery::getColumnsDescription(*columns_list, context, false); + auto columns = InterpreterCreateQuery::getColumnsDescription(*columns_list, context, false, false); auto validation_settings = DataTypeValidationSettings(context->getSettingsRef()); for (const auto & [name, type] : columns.getAll()) validateDataType(type, validation_settings); @@ -100,7 +100,7 @@ bool tryParseColumnsListFromString(const std::string & structure, ColumnsDescrip try { - columns = InterpreterCreateQuery::getColumnsDescription(*columns_list, context, false); + columns = InterpreterCreateQuery::getColumnsDescription(*columns_list, context, false, false); auto validation_settings = DataTypeValidationSettings(context->getSettingsRef()); for (const auto & [name, type] : columns.getAll()) validateDataType(type, validation_settings); diff --git a/tests/queries/0_stateless/02907_backup_restore_flatten_nested.reference b/tests/queries/0_stateless/02907_backup_restore_flatten_nested.reference new file mode 100644 index 00000000000..0375a7263a7 --- /dev/null +++ b/tests/queries/0_stateless/02907_backup_restore_flatten_nested.reference @@ -0,0 +1,4 @@ +BACKUP_CREATED +CREATE TABLE test.test\n(\n `test` Array(Tuple(foo String, bar Float64))\n)\nENGINE = MergeTree\nORDER BY tuple()\nSETTINGS index_granularity = 8192 +RESTORED +CREATE TABLE test.test\n(\n `test` Array(Tuple(foo String, bar Float64))\n)\nENGINE = MergeTree\nORDER BY tuple()\nSETTINGS index_granularity = 8192 diff --git a/tests/queries/0_stateless/02907_backup_restore_flatten_nested.sh b/tests/queries/0_stateless/02907_backup_restore_flatten_nested.sh new file mode 100755 index 00000000000..4ec269695a4 --- /dev/null +++ b/tests/queries/0_stateless/02907_backup_restore_flatten_nested.sh @@ -0,0 +1,22 @@ +#!/usr/bin/env bash + +CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh +. "$CUR_DIR"/../shell_config.sh + +${CLICKHOUSE_CLIENT} -nm --query " +drop table if exists test; +set flatten_nested = 0; +create table test (test Array(Tuple(foo String, bar Float64))) ENGINE = MergeTree() ORDER BY tuple(); +backup table test on cluster test_shard_localhost to Disk('default', '${CLICKHOUSE_TEST_UNIQUE_NAME}'); +" | grep -o "BACKUP_CREATED" + +${CLICKHOUSE_CLIENT} --query "show create table test" + +${CLICKHOUSE_CLIENT} -nm --query " +drop table test sync; +set flatten_nested = 1; +restore table test on cluster test_shard_localhost from Disk('default', '${CLICKHOUSE_TEST_UNIQUE_NAME}'); +" | grep -o "RESTORED" + +${CLICKHOUSE_CLIENT} --query "show create table test" From c5b1c200a9bd7b212c5f9642c1b0fd42c14ec412 Mon Sep 17 00:00:00 2001 From: Kseniia Sumarokova <54203879+kssenii@users.noreply.github.com> Date: Fri, 3 Nov 2023 17:19:29 +0100 Subject: [PATCH 2/9] Update 02907_backup_restore_flatten_nested.sh --- .../0_stateless/02907_backup_restore_flatten_nested.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/queries/0_stateless/02907_backup_restore_flatten_nested.sh b/tests/queries/0_stateless/02907_backup_restore_flatten_nested.sh index 4ec269695a4..a7d3ad23b1c 100755 --- a/tests/queries/0_stateless/02907_backup_restore_flatten_nested.sh +++ b/tests/queries/0_stateless/02907_backup_restore_flatten_nested.sh @@ -8,7 +8,7 @@ ${CLICKHOUSE_CLIENT} -nm --query " drop table if exists test; set flatten_nested = 0; create table test (test Array(Tuple(foo String, bar Float64))) ENGINE = MergeTree() ORDER BY tuple(); -backup table test on cluster test_shard_localhost to Disk('default', '${CLICKHOUSE_TEST_UNIQUE_NAME}'); +backup table test on cluster test_shard_localhost to Disk('backups', '${CLICKHOUSE_TEST_UNIQUE_NAME}'); " | grep -o "BACKUP_CREATED" ${CLICKHOUSE_CLIENT} --query "show create table test" @@ -16,7 +16,7 @@ ${CLICKHOUSE_CLIENT} --query "show create table test" ${CLICKHOUSE_CLIENT} -nm --query " drop table test sync; set flatten_nested = 1; -restore table test on cluster test_shard_localhost from Disk('default', '${CLICKHOUSE_TEST_UNIQUE_NAME}'); +restore table test on cluster test_shard_localhost from Disk('backups', '${CLICKHOUSE_TEST_UNIQUE_NAME}'); " | grep -o "RESTORED" ${CLICKHOUSE_CLIENT} --query "show create table test" From d4b1ae1c6099344d8e1d2bf3c85e02eaa1e30c51 Mon Sep 17 00:00:00 2001 From: Kseniia Sumarokova <54203879+kssenii@users.noreply.github.com> Date: Fri, 3 Nov 2023 17:52:18 +0100 Subject: [PATCH 3/9] Update 02907_backup_restore_flatten_nested.sh --- .../0_stateless/02907_backup_restore_flatten_nested.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/queries/0_stateless/02907_backup_restore_flatten_nested.sh b/tests/queries/0_stateless/02907_backup_restore_flatten_nested.sh index a7d3ad23b1c..d4c50502dc9 100755 --- a/tests/queries/0_stateless/02907_backup_restore_flatten_nested.sh +++ b/tests/queries/0_stateless/02907_backup_restore_flatten_nested.sh @@ -8,7 +8,7 @@ ${CLICKHOUSE_CLIENT} -nm --query " drop table if exists test; set flatten_nested = 0; create table test (test Array(Tuple(foo String, bar Float64))) ENGINE = MergeTree() ORDER BY tuple(); -backup table test on cluster test_shard_localhost to Disk('backups', '${CLICKHOUSE_TEST_UNIQUE_NAME}'); +backup table ${CLICKHOUSE_DATABASE}.test on cluster test_shard_localhost to Disk('backups', '${CLICKHOUSE_TEST_UNIQUE_NAME}'); " | grep -o "BACKUP_CREATED" ${CLICKHOUSE_CLIENT} --query "show create table test" @@ -16,7 +16,7 @@ ${CLICKHOUSE_CLIENT} --query "show create table test" ${CLICKHOUSE_CLIENT} -nm --query " drop table test sync; set flatten_nested = 1; -restore table test on cluster test_shard_localhost from Disk('backups', '${CLICKHOUSE_TEST_UNIQUE_NAME}'); +restore table ${CLICKHOUSE_DATABASE}.test on cluster test_shard_localhost from Disk('backups', '${CLICKHOUSE_TEST_UNIQUE_NAME}'); " | grep -o "RESTORED" ${CLICKHOUSE_CLIENT} --query "show create table test" From 804a7c7ca451236289677cc228d913504d7015a1 Mon Sep 17 00:00:00 2001 From: kssenii Date: Mon, 6 Nov 2023 12:12:55 +0100 Subject: [PATCH 4/9] Better test --- ...907_backup_restore_flatten_nested.reference | 8 ++++++-- .../02907_backup_restore_flatten_nested.sh | 18 ++++++++++++++++++ 2 files changed, 24 insertions(+), 2 deletions(-) diff --git a/tests/queries/0_stateless/02907_backup_restore_flatten_nested.reference b/tests/queries/0_stateless/02907_backup_restore_flatten_nested.reference index 0375a7263a7..aa8f22f590a 100644 --- a/tests/queries/0_stateless/02907_backup_restore_flatten_nested.reference +++ b/tests/queries/0_stateless/02907_backup_restore_flatten_nested.reference @@ -1,4 +1,8 @@ BACKUP_CREATED -CREATE TABLE test.test\n(\n `test` Array(Tuple(foo String, bar Float64))\n)\nENGINE = MergeTree\nORDER BY tuple()\nSETTINGS index_granularity = 8192 +CREATE TABLE default.test\n(\n `test` Array(Tuple(foo String, bar Float64))\n)\nENGINE = MergeTree\nORDER BY tuple()\nSETTINGS index_granularity = 8192 +BACKUP_CREATED +CREATE TABLE default.test2\n(\n `test` Nested(foo String, bar Float64)\n)\nENGINE = MergeTree\nORDER BY tuple()\nSETTINGS index_granularity = 8192 RESTORED -CREATE TABLE test.test\n(\n `test` Array(Tuple(foo String, bar Float64))\n)\nENGINE = MergeTree\nORDER BY tuple()\nSETTINGS index_granularity = 8192 +CREATE TABLE default.test\n(\n `test` Array(Tuple(foo String, bar Float64))\n)\nENGINE = MergeTree\nORDER BY tuple()\nSETTINGS index_granularity = 8192 +RESTORED +CREATE TABLE default.test2\n(\n `test` Nested(foo String, bar Float64)\n)\nENGINE = MergeTree\nORDER BY tuple()\nSETTINGS index_granularity = 8192 diff --git a/tests/queries/0_stateless/02907_backup_restore_flatten_nested.sh b/tests/queries/0_stateless/02907_backup_restore_flatten_nested.sh index d4c50502dc9..0470723a174 100755 --- a/tests/queries/0_stateless/02907_backup_restore_flatten_nested.sh +++ b/tests/queries/0_stateless/02907_backup_restore_flatten_nested.sh @@ -4,6 +4,7 @@ CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) # shellcheck source=../shell_config.sh . "$CUR_DIR"/../shell_config.sh +CLICKHOUSE_TEST_UNIQUE_NAME="kek1" ${CLICKHOUSE_CLIENT} -nm --query " drop table if exists test; set flatten_nested = 0; @@ -13,6 +14,15 @@ backup table ${CLICKHOUSE_DATABASE}.test on cluster test_shard_localhost to Disk ${CLICKHOUSE_CLIENT} --query "show create table test" +${CLICKHOUSE_CLIENT} -nm --query " +drop table if exists test2; +set flatten_nested = 0; +create table test2 (test Nested(foo String, bar Float64)) ENGINE = MergeTree() ORDER BY tuple(); +backup table ${CLICKHOUSE_DATABASE}.test2 on cluster test_shard_localhost to Disk('backups', '${CLICKHOUSE_TEST_UNIQUE_NAME}2'); +" | grep -o "BACKUP_CREATED" + +${CLICKHOUSE_CLIENT} --query "show create table test2" + ${CLICKHOUSE_CLIENT} -nm --query " drop table test sync; set flatten_nested = 1; @@ -20,3 +30,11 @@ restore table ${CLICKHOUSE_DATABASE}.test on cluster test_shard_localhost from D " | grep -o "RESTORED" ${CLICKHOUSE_CLIENT} --query "show create table test" + +${CLICKHOUSE_CLIENT} -nm --query " +drop table test2 sync; +set flatten_nested = 1; +restore table ${CLICKHOUSE_DATABASE}.test2 on cluster test_shard_localhost from Disk('backups', '${CLICKHOUSE_TEST_UNIQUE_NAME}2'); +" | grep -o "RESTORED" + +${CLICKHOUSE_CLIENT} --query "show create table test2" From fd60ec56d3b4e75eab940f647ff334dbe0d05699 Mon Sep 17 00:00:00 2001 From: Kseniia Sumarokova <54203879+kssenii@users.noreply.github.com> Date: Tue, 7 Nov 2023 12:41:47 +0100 Subject: [PATCH 5/9] Update 02907_backup_restore_flatten_nested.sh --- tests/queries/0_stateless/02907_backup_restore_flatten_nested.sh | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/queries/0_stateless/02907_backup_restore_flatten_nested.sh b/tests/queries/0_stateless/02907_backup_restore_flatten_nested.sh index 0470723a174..742d24a97eb 100755 --- a/tests/queries/0_stateless/02907_backup_restore_flatten_nested.sh +++ b/tests/queries/0_stateless/02907_backup_restore_flatten_nested.sh @@ -4,7 +4,6 @@ CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) # shellcheck source=../shell_config.sh . "$CUR_DIR"/../shell_config.sh -CLICKHOUSE_TEST_UNIQUE_NAME="kek1" ${CLICKHOUSE_CLIENT} -nm --query " drop table if exists test; set flatten_nested = 0; From 9178fd4ad118d31a57fce8338648721d1e8c4eef Mon Sep 17 00:00:00 2001 From: kssenii Date: Tue, 7 Nov 2023 15:58:25 +0100 Subject: [PATCH 6/9] Fix case with replicated database --- src/Client/LocalConnection.cpp | 2 +- src/Databases/DatabaseReplicated.cpp | 9 +++++---- src/Databases/DatabaseReplicated.h | 2 +- src/Databases/IDatabase.h | 3 ++- .../MySQL/MaterializedMySQLSyncThread.cpp | 2 +- src/Dictionaries/ClickHouseDictionarySource.cpp | 4 ++-- .../Access/InterpreterShowAccessEntitiesQuery.cpp | 2 +- .../Access/InterpreterShowPrivilegesQuery.cpp | 2 +- src/Interpreters/DDLTask.cpp | 9 +++++++++ src/Interpreters/DDLTask.h | 4 +++- src/Interpreters/DDLWorker.cpp | 3 ++- src/Interpreters/InterpreterCreateQuery.cpp | 4 ++-- src/Interpreters/InterpreterKillQueryQuery.cpp | 2 +- src/Interpreters/InterpreterShowColumnsQuery.cpp | 2 +- src/Interpreters/InterpreterShowEngineQuery.cpp | 2 +- .../InterpreterShowFunctionsQuery.cpp | 2 +- src/Interpreters/InterpreterShowIndexesQuery.cpp | 2 +- .../InterpreterShowProcesslistQuery.cpp | 2 +- src/Interpreters/InterpreterShowSettingQuery.cpp | 3 +-- src/Interpreters/InterpreterShowTablesQuery.cpp | 2 +- .../MySQL/InterpretersMySQLDDLQuery.h | 2 +- src/Interpreters/executeQuery.cpp | 15 +++++++++++---- src/Interpreters/executeQuery.h | 13 ++++++++++++- src/Interpreters/fuzzers/execute_query_fuzzer.cpp | 2 +- src/Interpreters/loadMetadata.cpp | 10 +++++----- src/Server/HTTPHandler.cpp | 1 + src/Server/MySQLHandler.cpp | 2 +- src/Server/TCPHandler.cpp | 2 +- 28 files changed, 71 insertions(+), 39 deletions(-) diff --git a/src/Client/LocalConnection.cpp b/src/Client/LocalConnection.cpp index c135ced7c76..d01c40a9c34 100644 --- a/src/Client/LocalConnection.cpp +++ b/src/Client/LocalConnection.cpp @@ -131,7 +131,7 @@ void LocalConnection::sendQuery( try { - state->io = executeQuery(state->query, query_context, false, state->stage).second; + state->io = executeQuery(state->query, query_context, QueryFlags{}, state->stage).second; if (state->io.pipeline.pushing()) { diff --git a/src/Databases/DatabaseReplicated.cpp b/src/Databases/DatabaseReplicated.cpp index 7075d277c17..b90d41b57a8 100644 --- a/src/Databases/DatabaseReplicated.cpp +++ b/src/Databases/DatabaseReplicated.cpp @@ -719,7 +719,7 @@ void DatabaseReplicated::checkQueryValid(const ASTPtr & query, ContextPtr query_ } } -BlockIO DatabaseReplicated::tryEnqueueReplicatedDDL(const ASTPtr & query, ContextPtr query_context, bool internal) +BlockIO DatabaseReplicated::tryEnqueueReplicatedDDL(const ASTPtr & query, ContextPtr query_context, QueryFlags flags) { if (query_context->getCurrentTransaction() && query_context->getSettingsRef().throw_on_unsupported_query_inside_transaction) @@ -728,7 +728,7 @@ BlockIO DatabaseReplicated::tryEnqueueReplicatedDDL(const ASTPtr & query, Contex if (is_readonly) throw Exception(ErrorCodes::NO_ZOOKEEPER, "Database is in readonly mode, because it cannot connect to ZooKeeper"); - if (!internal && (query_context->getClientInfo().query_kind != ClientInfo::QueryKind::INITIAL_QUERY)) + if (!flags.internal && (query_context->getClientInfo().query_kind != ClientInfo::QueryKind::INITIAL_QUERY)) throw Exception(ErrorCodes::INCORRECT_QUERY, "It's not initial query. ON CLUSTER is not allowed for Replicated database."); checkQueryValid(query, query_context); @@ -739,6 +739,7 @@ BlockIO DatabaseReplicated::tryEnqueueReplicatedDDL(const ASTPtr & query, Contex entry.initiator = ddl_worker->getCommonHostID(); entry.setSettingsIfRequired(query_context); entry.tracing_context = OpenTelemetry::CurrentContext(); + entry.is_backup_restore = flags.distributed_backup_restore; String node_path = ddl_worker->tryEnqueueAndExecuteEntry(entry, query_context); Strings hosts_to_wait; @@ -916,14 +917,14 @@ void DatabaseReplicated::recoverLostReplica(const ZooKeeperPtr & current_zookeep String query = fmt::format("CREATE DATABASE IF NOT EXISTS {} ENGINE=Ordinary", backQuoteIfNeed(to_db_name)); auto query_context = Context::createCopy(getContext()); query_context->setSetting("allow_deprecated_database_ordinary", 1); - executeQuery(query, query_context, true); + executeQuery(query, query_context, QueryFlags(true, false)); /// But we want to avoid discarding UUID of ReplicatedMergeTree tables, because it will not work /// if zookeeper_path contains {uuid} macro. Replicated database do not recreate replicated tables on recovery, /// so it's ok to save UUID of replicated table. query = fmt::format("CREATE DATABASE IF NOT EXISTS {} ENGINE=Atomic", backQuoteIfNeed(to_db_name_replicated)); query_context = Context::createCopy(getContext()); - executeQuery(query, query_context, true); + executeQuery(query, query_context, QueryFlags(true, false)); } size_t moved_tables = 0; diff --git a/src/Databases/DatabaseReplicated.h b/src/Databases/DatabaseReplicated.h index 005180624ed..1387ba1cb96 100644 --- a/src/Databases/DatabaseReplicated.h +++ b/src/Databases/DatabaseReplicated.h @@ -46,7 +46,7 @@ public: /// Try to execute DLL query on current host as initial query. If query is succeed, /// then it will be executed on all replicas. - BlockIO tryEnqueueReplicatedDDL(const ASTPtr & query, ContextPtr query_context, bool internal) override; + BlockIO tryEnqueueReplicatedDDL(const ASTPtr & query, ContextPtr query_context, QueryFlags flags) override; bool canExecuteReplicatedMetadataAlter() const override; diff --git a/src/Databases/IDatabase.h b/src/Databases/IDatabase.h index 01d940b0429..e886f1adae3 100644 --- a/src/Databases/IDatabase.h +++ b/src/Databases/IDatabase.h @@ -3,6 +3,7 @@ #include #include #include +#include #include #include #include @@ -345,7 +346,7 @@ public: virtual bool shouldReplicateQuery(const ContextPtr & /*query_context*/, const ASTPtr & /*query_ptr*/) const { return false; } - virtual BlockIO tryEnqueueReplicatedDDL(const ASTPtr & /*query*/, ContextPtr /*query_context*/, [[maybe_unused]] bool internal = false) /// NOLINT + virtual BlockIO tryEnqueueReplicatedDDL(const ASTPtr & /*query*/, ContextPtr /*query_context*/, [[maybe_unused]] QueryFlags flags = {}) /// NOLINT { throw Exception(ErrorCodes::LOGICAL_ERROR, "Database engine {} does not have replicated DDL queue", getEngineName()); } diff --git a/src/Databases/MySQL/MaterializedMySQLSyncThread.cpp b/src/Databases/MySQL/MaterializedMySQLSyncThread.cpp index 3d10e66e964..a59ad225a6e 100644 --- a/src/Databases/MySQL/MaterializedMySQLSyncThread.cpp +++ b/src/Databases/MySQL/MaterializedMySQLSyncThread.cpp @@ -75,7 +75,7 @@ static BlockIO tryToExecuteQuery(const String & query_to_execute, ContextMutable if (!database.empty()) query_context->setCurrentDatabase(database); - return executeQuery("/*" + comment + "*/ " + query_to_execute, query_context, true).second; + return executeQuery("/*" + comment + "*/ " + query_to_execute, query_context, QueryFlags(true, false)).second; } catch (...) { diff --git a/src/Dictionaries/ClickHouseDictionarySource.cpp b/src/Dictionaries/ClickHouseDictionarySource.cpp index 92fae2bc495..3312048c73d 100644 --- a/src/Dictionaries/ClickHouseDictionarySource.cpp +++ b/src/Dictionaries/ClickHouseDictionarySource.cpp @@ -168,7 +168,7 @@ QueryPipeline ClickHouseDictionarySource::createStreamForQuery(const String & qu if (configuration.is_local) { - pipeline = executeQuery(query, context_copy, true).second.pipeline; + pipeline = executeQuery(query, context_copy, QueryFlags(true, false)).second.pipeline; pipeline.convertStructureTo(empty_sample_block.getColumnsWithTypeAndName()); } else @@ -190,7 +190,7 @@ std::string ClickHouseDictionarySource::doInvalidateQuery(const std::string & re if (configuration.is_local) { - return readInvalidateQuery(executeQuery(request, context_copy, true).second.pipeline); + return readInvalidateQuery(executeQuery(request, context_copy, QueryFlags(true, false)).second.pipeline); } else { diff --git a/src/Interpreters/Access/InterpreterShowAccessEntitiesQuery.cpp b/src/Interpreters/Access/InterpreterShowAccessEntitiesQuery.cpp index b0937dc2f66..5d8e454fcde 100644 --- a/src/Interpreters/Access/InterpreterShowAccessEntitiesQuery.cpp +++ b/src/Interpreters/Access/InterpreterShowAccessEntitiesQuery.cpp @@ -23,7 +23,7 @@ InterpreterShowAccessEntitiesQuery::InterpreterShowAccessEntitiesQuery(const AST BlockIO InterpreterShowAccessEntitiesQuery::execute() { - return executeQuery(getRewrittenQuery(), getContext(), true).second; + return executeQuery(getRewrittenQuery(), getContext(), QueryFlags(true, false)).second; } diff --git a/src/Interpreters/Access/InterpreterShowPrivilegesQuery.cpp b/src/Interpreters/Access/InterpreterShowPrivilegesQuery.cpp index 213e3c813fa..227c6fea455 100644 --- a/src/Interpreters/Access/InterpreterShowPrivilegesQuery.cpp +++ b/src/Interpreters/Access/InterpreterShowPrivilegesQuery.cpp @@ -12,7 +12,7 @@ InterpreterShowPrivilegesQuery::InterpreterShowPrivilegesQuery(const ASTPtr & qu BlockIO InterpreterShowPrivilegesQuery::execute() { - return executeQuery("SELECT * FROM system.privileges", context, true).second; + return executeQuery("SELECT * FROM system.privileges", context, QueryFlags(true, false)).second; } } diff --git a/src/Interpreters/DDLTask.cpp b/src/Interpreters/DDLTask.cpp index 4e684f5899f..632061a8ec5 100644 --- a/src/Interpreters/DDLTask.cpp +++ b/src/Interpreters/DDLTask.cpp @@ -113,6 +113,9 @@ String DDLLogEntry::toString() const writeChar('\n', wb); } + if (version >= BACKUP_RESTORE_FLAG_IN_ZK_VERSION) + wb << "is_backup_restore: " << is_backup_restore << "\n"; + return wb.str(); } @@ -165,6 +168,12 @@ void DDLLogEntry::parse(const String & data) checkChar('\n', rb); } + if (version >= BACKUP_RESTORE_FLAG_IN_ZK_VERSION) + { + checkString("is_backup_restore: ", rb); + readBoolText(is_backup_restore, rb); + checkChar('\n', rb); + } assertEOF(rb); diff --git a/src/Interpreters/DDLTask.h b/src/Interpreters/DDLTask.h index e92b1f9a885..db8b0628b4b 100644 --- a/src/Interpreters/DDLTask.h +++ b/src/Interpreters/DDLTask.h @@ -72,10 +72,11 @@ struct DDLLogEntry static constexpr const UInt64 NORMALIZE_CREATE_ON_INITIATOR_VERSION = 3; static constexpr const UInt64 OPENTELEMETRY_ENABLED_VERSION = 4; static constexpr const UInt64 PRESERVE_INITIAL_QUERY_ID_VERSION = 5; + static constexpr const UInt64 BACKUP_RESTORE_FLAG_IN_ZK_VERSION = 6; /// Add new version here /// Remember to update the value below once new version is added - static constexpr const UInt64 DDL_ENTRY_FORMAT_MAX_VERSION = 5; + static constexpr const UInt64 DDL_ENTRY_FORMAT_MAX_VERSION = 6; UInt64 version = 1; String query; @@ -84,6 +85,7 @@ struct DDLLogEntry std::optional settings; OpenTelemetry::TracingContext tracing_context; String initial_query_id; + bool is_backup_restore = false; void setSettingsIfRequired(ContextPtr context); String toString() const; diff --git a/src/Interpreters/DDLWorker.cpp b/src/Interpreters/DDLWorker.cpp index da46aad0329..3b4608cc245 100644 --- a/src/Interpreters/DDLWorker.cpp +++ b/src/Interpreters/DDLWorker.cpp @@ -489,7 +489,8 @@ bool DDLWorker::tryExecuteQuery(DDLTaskBase & task, const ZooKeeperPtr & zookeep if (!task.is_initial_query) query_scope.emplace(query_context); - executeQuery(istr, ostr, !task.is_initial_query, query_context, {}); + + executeQuery(istr, ostr, !task.is_initial_query, query_context, {}, QueryFlags(false, task.entry.is_backup_restore)); if (auto txn = query_context->getZooKeeperMetadataTransaction()) { diff --git a/src/Interpreters/InterpreterCreateQuery.cpp b/src/Interpreters/InterpreterCreateQuery.cpp index ca865264fcf..843d8749f4b 100644 --- a/src/Interpreters/InterpreterCreateQuery.cpp +++ b/src/Interpreters/InterpreterCreateQuery.cpp @@ -1076,7 +1076,7 @@ BlockIO InterpreterCreateQuery::createTable(ASTCreateQuery & create) auto guard = DatabaseCatalog::instance().getDDLGuard(database_name, create.getTable()); create.setDatabase(database_name); guard->releaseTableLock(); - return database->tryEnqueueReplicatedDDL(query_ptr, getContext(), internal); + return database->tryEnqueueReplicatedDDL(query_ptr, getContext(), QueryFlags(internal, is_restore_from_backup)); } if (!create.cluster.empty()) @@ -1232,7 +1232,7 @@ BlockIO InterpreterCreateQuery::createTable(ASTCreateQuery & create) auto guard = DatabaseCatalog::instance().getDDLGuard(create.getDatabase(), create.getTable()); assertOrSetUUID(create, database); guard->releaseTableLock(); - return database->tryEnqueueReplicatedDDL(query_ptr, getContext(), internal); + return database->tryEnqueueReplicatedDDL(query_ptr, getContext(), QueryFlags(internal, is_restore_from_backup)); } if (!create.cluster.empty()) diff --git a/src/Interpreters/InterpreterKillQueryQuery.cpp b/src/Interpreters/InterpreterKillQueryQuery.cpp index 1c2e3ff6777..9aeebd9a213 100644 --- a/src/Interpreters/InterpreterKillQueryQuery.cpp +++ b/src/Interpreters/InterpreterKillQueryQuery.cpp @@ -420,7 +420,7 @@ Block InterpreterKillQueryQuery::getSelectResult(const String & columns, const S if (where_expression) select_query += " WHERE " + queryToString(where_expression); - auto io = executeQuery(select_query, getContext(), true).second; + auto io = executeQuery(select_query, getContext(), QueryFlags(true, false)).second; PullingPipelineExecutor executor(io.pipeline); Block res; while (!res && executor.pull(res)); diff --git a/src/Interpreters/InterpreterShowColumnsQuery.cpp b/src/Interpreters/InterpreterShowColumnsQuery.cpp index c8fb64e37f2..5210614a725 100644 --- a/src/Interpreters/InterpreterShowColumnsQuery.cpp +++ b/src/Interpreters/InterpreterShowColumnsQuery.cpp @@ -159,7 +159,7 @@ WHERE BlockIO InterpreterShowColumnsQuery::execute() { - return executeQuery(getRewrittenQuery(), getContext(), true).second; + return executeQuery(getRewrittenQuery(), getContext(), QueryFlags(true, false)).second; } diff --git a/src/Interpreters/InterpreterShowEngineQuery.cpp b/src/Interpreters/InterpreterShowEngineQuery.cpp index a2367e9bfdf..5d43b220d06 100644 --- a/src/Interpreters/InterpreterShowEngineQuery.cpp +++ b/src/Interpreters/InterpreterShowEngineQuery.cpp @@ -12,7 +12,7 @@ namespace DB BlockIO InterpreterShowEnginesQuery::execute() { - return executeQuery("SELECT * FROM system.table_engines ORDER BY name", getContext(), true).second; + return executeQuery("SELECT * FROM system.table_engines ORDER BY name", getContext(), QueryFlags(true, false)).second; } } diff --git a/src/Interpreters/InterpreterShowFunctionsQuery.cpp b/src/Interpreters/InterpreterShowFunctionsQuery.cpp index ace22ca4bb6..6755e05f75f 100644 --- a/src/Interpreters/InterpreterShowFunctionsQuery.cpp +++ b/src/Interpreters/InterpreterShowFunctionsQuery.cpp @@ -15,7 +15,7 @@ InterpreterShowFunctionsQuery::InterpreterShowFunctionsQuery(const ASTPtr & quer BlockIO InterpreterShowFunctionsQuery::execute() { - return executeQuery(getRewrittenQuery(), getContext(), true).second; + return executeQuery(getRewrittenQuery(), getContext(), QueryFlags(true, false)).second; } String InterpreterShowFunctionsQuery::getRewrittenQuery() diff --git a/src/Interpreters/InterpreterShowIndexesQuery.cpp b/src/Interpreters/InterpreterShowIndexesQuery.cpp index 549afd32506..7c8af6d52c5 100644 --- a/src/Interpreters/InterpreterShowIndexesQuery.cpp +++ b/src/Interpreters/InterpreterShowIndexesQuery.cpp @@ -101,7 +101,7 @@ ORDER BY index_type, expression, column_name, seq_in_index;)", database, table, BlockIO InterpreterShowIndexesQuery::execute() { - return executeQuery(getRewrittenQuery(), getContext(), true).second; + return executeQuery(getRewrittenQuery(), getContext(), QueryFlags(true, false)).second; } diff --git a/src/Interpreters/InterpreterShowProcesslistQuery.cpp b/src/Interpreters/InterpreterShowProcesslistQuery.cpp index 4ed5f4171c6..bb367ebdfa5 100644 --- a/src/Interpreters/InterpreterShowProcesslistQuery.cpp +++ b/src/Interpreters/InterpreterShowProcesslistQuery.cpp @@ -12,7 +12,7 @@ namespace DB BlockIO InterpreterShowProcesslistQuery::execute() { - return executeQuery("SELECT * FROM system.processes ORDER BY elapsed DESC", getContext(), true).second; + return executeQuery("SELECT * FROM system.processes ORDER BY elapsed DESC", getContext(), QueryFlags(true, false)).second; } } diff --git a/src/Interpreters/InterpreterShowSettingQuery.cpp b/src/Interpreters/InterpreterShowSettingQuery.cpp index 7567e77d28f..4d7200cb013 100644 --- a/src/Interpreters/InterpreterShowSettingQuery.cpp +++ b/src/Interpreters/InterpreterShowSettingQuery.cpp @@ -26,9 +26,8 @@ String InterpreterShowSettingQuery::getRewrittenQuery() BlockIO InterpreterShowSettingQuery::execute() { - return executeQuery(getRewrittenQuery(), getContext(), true).second; + return executeQuery(getRewrittenQuery(), getContext(), QueryFlags(true, false)).second; } } - diff --git a/src/Interpreters/InterpreterShowTablesQuery.cpp b/src/Interpreters/InterpreterShowTablesQuery.cpp index a8db55a317a..8ddb4eef074 100644 --- a/src/Interpreters/InterpreterShowTablesQuery.cpp +++ b/src/Interpreters/InterpreterShowTablesQuery.cpp @@ -214,7 +214,7 @@ BlockIO InterpreterShowTablesQuery::execute() return res; } - return executeQuery(getRewrittenQuery(), getContext(), true).second; + return executeQuery(getRewrittenQuery(), getContext(), QueryFlags(true, false)).second; } /// (*) Sorting is strictly speaking not necessary but 1. it is convenient for users, 2. SQL currently does not allow to diff --git a/src/Interpreters/MySQL/InterpretersMySQLDDLQuery.h b/src/Interpreters/MySQL/InterpretersMySQLDDLQuery.h index 27f467a12ae..36f2b0b9fc5 100644 --- a/src/Interpreters/MySQL/InterpretersMySQLDDLQuery.h +++ b/src/Interpreters/MySQL/InterpretersMySQLDDLQuery.h @@ -75,7 +75,7 @@ public: ASTs rewritten_queries = InterpreterImpl::getRewrittenQueries(query, getContext(), mapped_to_database, mysql_database); for (const auto & rewritten_query : rewritten_queries) - executeQuery("/* Rewritten MySQL DDL Query */ " + queryToString(rewritten_query), getContext(), true); + executeQuery("/* Rewritten MySQL DDL Query */ " + queryToString(rewritten_query), getContext(), QueryFlags(true, false)); return BlockIO{}; } diff --git a/src/Interpreters/executeQuery.cpp b/src/Interpreters/executeQuery.cpp index decda4c62f9..bafee07f72a 100644 --- a/src/Interpreters/executeQuery.cpp +++ b/src/Interpreters/executeQuery.cpp @@ -45,6 +45,7 @@ #include #include #include +#include #include #include #include @@ -645,10 +646,12 @@ static std::tuple executeQueryImpl( const char * begin, const char * end, ContextMutablePtr context, - bool internal, + QueryFlags flags, QueryProcessingStage::Enum stage, ReadBuffer * istr) { + const bool internal = flags.internal; + /// query_span is a special span, when this function exits, it's lifetime is not ended, but ends when the query finishes. /// Some internal queries might call this function recursively by setting 'internal' parameter to 'true', /// to make sure SpanHolders in current stack ends in correct order, we disable this span for these internal queries @@ -1084,6 +1087,9 @@ static std::tuple executeQueryImpl( insert_interpreter->addBuffer(std::move(insert_data_buffer_holder)); } + if (auto * create_interpreter = typeid_cast(&*interpreter)) + create_interpreter->setIsRestoreFromBackup(flags.distributed_backup_restore); + { std::unique_ptr span; if (OpenTelemetry::CurrentContext().isTraceEnabled()) @@ -1233,13 +1239,13 @@ static std::tuple executeQueryImpl( std::pair executeQuery( const String & query, ContextMutablePtr context, - bool internal, + QueryFlags flags, QueryProcessingStage::Enum stage) { ASTPtr ast; BlockIO res; - std::tie(ast, res) = executeQueryImpl(query.data(), query.data() + query.size(), context, internal, stage, nullptr); + std::tie(ast, res) = executeQueryImpl(query.data(), query.data() + query.size(), context, flags, stage, nullptr); if (const auto * ast_query_with_output = dynamic_cast(ast.get())) { @@ -1260,6 +1266,7 @@ void executeQuery( bool allow_into_outfile, ContextMutablePtr context, SetResultDetailsFunc set_result_details, + QueryFlags flags, const std::optional & output_format_settings, HandleExceptionInOutputFormatFunc handle_exception_in_output_format) { @@ -1348,7 +1355,7 @@ void executeQuery( try { - std::tie(ast, streams) = executeQueryImpl(begin, end, context, false, QueryProcessingStage::Complete, &istr); + std::tie(ast, streams) = executeQueryImpl(begin, end, context, flags, QueryProcessingStage::Complete, &istr); } catch (...) { diff --git a/src/Interpreters/executeQuery.h b/src/Interpreters/executeQuery.h index 6f14f54d7d6..79ede484592 100644 --- a/src/Interpreters/executeQuery.h +++ b/src/Interpreters/executeQuery.h @@ -29,6 +29,16 @@ struct QueryResultDetails using SetResultDetailsFunc = std::function; using HandleExceptionInOutputFormatFunc = std::function; +struct QueryFlags +{ + QueryFlags() = default; + explicit QueryFlags(bool internal_, bool distribted_backup_restore_) : internal(internal_), distributed_backup_restore(distribted_backup_restore_) {} + + bool internal = false; /// If true, this query is caused by another query and thus needn't be registered in the ProcessList. + bool distributed_backup_restore = false; /// If true, this query is a part of backup restore. +}; + + /// Parse and execute a query. void executeQuery( ReadBuffer & istr, /// Where to read query from (and data for INSERT, if present). @@ -36,6 +46,7 @@ void executeQuery( bool allow_into_outfile, /// If true and the query contains INTO OUTFILE section, redirect output to that file. ContextMutablePtr context, /// DB, tables, data types, storage engines, functions, aggregate functions... SetResultDetailsFunc set_result_details, /// If a non-empty callback is passed, it will be called with the query id, the content-type, the format, and the timezone. + QueryFlags flags = {}, const std::optional & output_format_settings = std::nullopt, /// Format settings for output format, will be calculated from the context if not set. HandleExceptionInOutputFormatFunc handle_exception_in_output_format = {} /// If a non-empty callback is passed, it will be called on exception with created output format. ); @@ -58,7 +69,7 @@ void executeQuery( std::pair executeQuery( const String & query, /// Query text without INSERT data. The latter must be written to BlockIO::out. ContextMutablePtr context, /// DB, tables, data types, storage engines, functions, aggregate functions... - bool internal = false, /// If true, this query is caused by another query and thus needn't be registered in the ProcessList. + QueryFlags flags = {}, QueryProcessingStage::Enum stage = QueryProcessingStage::Complete /// To which stage the query must be executed. ); diff --git a/src/Interpreters/fuzzers/execute_query_fuzzer.cpp b/src/Interpreters/fuzzers/execute_query_fuzzer.cpp index 0f6bfc1ae58..a5f2782c8ca 100644 --- a/src/Interpreters/fuzzers/execute_query_fuzzer.cpp +++ b/src/Interpreters/fuzzers/execute_query_fuzzer.cpp @@ -42,7 +42,7 @@ extern "C" int LLVMFuzzerTestOneInput(const uint8_t * data, size_t size) static bool initialized = initialize(); (void) initialized; - auto io = DB::executeQuery(input, context, true, QueryProcessingStage::Complete).second; + auto io = DB::executeQuery(input, context, QueryFlags(true, false), QueryProcessingStage::Complete).second; PullingPipelineExecutor executor(io.pipeline); Block res; diff --git a/src/Interpreters/loadMetadata.cpp b/src/Interpreters/loadMetadata.cpp index faa1dcda2c0..d7c26f4ba7b 100644 --- a/src/Interpreters/loadMetadata.cpp +++ b/src/Interpreters/loadMetadata.cpp @@ -282,7 +282,7 @@ static void convertOrdinaryDatabaseToAtomic(Poco::Logger * log, ContextMutablePt LOG_INFO(log, "Will convert database {} from Ordinary to Atomic", name_quoted); String create_database_query = fmt::format("CREATE DATABASE IF NOT EXISTS {}", tmp_name_quoted); - auto res = executeQuery(create_database_query, context, true).second; + auto res = executeQuery(create_database_query, context, QueryFlags(true, false)).second; executeTrivialBlockIO(res, context); res = {}; auto tmp_database = DatabaseCatalog::instance().getDatabase(tmp_name); @@ -322,7 +322,7 @@ static void convertOrdinaryDatabaseToAtomic(Poco::Logger * log, ContextMutablePt String tmp_qualified_quoted_name = id.getFullTableName(); String move_table_query = fmt::format("RENAME TABLE {} TO {}", qualified_quoted_name, tmp_qualified_quoted_name); - res = executeQuery(move_table_query, context, true).second; + res = executeQuery(move_table_query, context, QueryFlags(true, false)).second; executeTrivialBlockIO(res, context); res = {}; } @@ -334,12 +334,12 @@ static void convertOrdinaryDatabaseToAtomic(Poco::Logger * log, ContextMutablePt String drop_query = fmt::format("DROP DATABASE {}", name_quoted); context->setSetting("force_remove_data_recursively_on_drop", false); - res = executeQuery(drop_query, context, true).second; + res = executeQuery(drop_query, context, QueryFlags(true, false)).second; executeTrivialBlockIO(res, context); res = {}; String rename_query = fmt::format("RENAME DATABASE {} TO {}", tmp_name_quoted, name_quoted); - res = executeQuery(rename_query, context, true).second; + res = executeQuery(rename_query, context, QueryFlags(true, false)).second; executeTrivialBlockIO(res, context); LOG_INFO(log, "Finished database engine conversion of {}", name_quoted); @@ -409,7 +409,7 @@ static void maybeConvertOrdinaryDatabaseToAtomic(ContextMutablePtr context, cons /// Reload database just in case (and update logger name) String detach_query = fmt::format("DETACH DATABASE {}", backQuoteIfNeed(database_name)); - auto res = executeQuery(detach_query, context, true).second; + auto res = executeQuery(detach_query, context, QueryFlags(true, false)).second; executeTrivialBlockIO(res, context); res = {}; diff --git a/src/Server/HTTPHandler.cpp b/src/Server/HTTPHandler.cpp index a2d067af387..f9cd3b40f4a 100644 --- a/src/Server/HTTPHandler.cpp +++ b/src/Server/HTTPHandler.cpp @@ -886,6 +886,7 @@ void HTTPHandler::processQuery( /* allow_into_outfile = */ false, context, set_query_result, + QueryFlags{}, {}, handle_exception_in_output_format); diff --git a/src/Server/MySQLHandler.cpp b/src/Server/MySQLHandler.cpp index f9155a07e2b..21fa7f7227a 100644 --- a/src/Server/MySQLHandler.cpp +++ b/src/Server/MySQLHandler.cpp @@ -378,7 +378,7 @@ void MySQLHandler::comQuery(ReadBuffer & payload, bool binary_protocol) } }; - executeQuery(should_replace ? replacement : payload, *out, false, query_context, set_result_details, format_settings); + executeQuery(should_replace ? replacement : payload, *out, false, query_context, set_result_details, QueryFlags{}, format_settings); if (!with_output) packet_endpoint->sendPacket(OKPacket(0x00, client_capabilities, affected_rows, 0, 0), true); diff --git a/src/Server/TCPHandler.cpp b/src/Server/TCPHandler.cpp index 5082d8e4f3b..1da9806b4f5 100644 --- a/src/Server/TCPHandler.cpp +++ b/src/Server/TCPHandler.cpp @@ -496,7 +496,7 @@ void TCPHandler::runImpl() }); /// Processing Query - std::tie(state.parsed_query, state.io) = executeQuery(state.query, query_context, false, state.stage); + std::tie(state.parsed_query, state.io) = executeQuery(state.query, query_context, QueryFlags{}, state.stage); after_check_cancelled.restart(); after_send_progress.restart(); From 41a880e57cb02c9ec639f76fdc29411ae9100392 Mon Sep 17 00:00:00 2001 From: kssenii Date: Thu, 9 Nov 2023 16:03:51 +0100 Subject: [PATCH 7/9] Review fix --- src/Databases/DatabaseReplicated.cpp | 4 ++-- src/Databases/MySQL/MaterializedMySQLSyncThread.cpp | 2 +- src/Dictionaries/ClickHouseDictionarySource.cpp | 4 ++-- .../Access/InterpreterShowAccessEntitiesQuery.cpp | 2 +- .../Access/InterpreterShowPrivilegesQuery.cpp | 2 +- src/Interpreters/DDLWorker.cpp | 2 +- src/Interpreters/InterpreterCreateQuery.cpp | 4 ++-- src/Interpreters/InterpreterKillQueryQuery.cpp | 2 +- src/Interpreters/InterpreterShowColumnsQuery.cpp | 2 +- src/Interpreters/InterpreterShowEngineQuery.cpp | 2 +- src/Interpreters/InterpreterShowFunctionsQuery.cpp | 2 +- src/Interpreters/InterpreterShowIndexesQuery.cpp | 2 +- src/Interpreters/InterpreterShowProcesslistQuery.cpp | 2 +- src/Interpreters/InterpreterShowSettingQuery.cpp | 2 +- src/Interpreters/InterpreterShowTablesQuery.cpp | 2 +- src/Interpreters/MySQL/InterpretersMySQLDDLQuery.h | 2 +- src/Interpreters/executeQuery.h | 3 --- src/Interpreters/fuzzers/execute_query_fuzzer.cpp | 2 +- src/Interpreters/loadMetadata.cpp | 10 +++++----- 19 files changed, 25 insertions(+), 28 deletions(-) diff --git a/src/Databases/DatabaseReplicated.cpp b/src/Databases/DatabaseReplicated.cpp index b90d41b57a8..5db3999f96a 100644 --- a/src/Databases/DatabaseReplicated.cpp +++ b/src/Databases/DatabaseReplicated.cpp @@ -917,14 +917,14 @@ void DatabaseReplicated::recoverLostReplica(const ZooKeeperPtr & current_zookeep String query = fmt::format("CREATE DATABASE IF NOT EXISTS {} ENGINE=Ordinary", backQuoteIfNeed(to_db_name)); auto query_context = Context::createCopy(getContext()); query_context->setSetting("allow_deprecated_database_ordinary", 1); - executeQuery(query, query_context, QueryFlags(true, false)); + executeQuery(query, query_context, QueryFlags{ .internal = true }); /// But we want to avoid discarding UUID of ReplicatedMergeTree tables, because it will not work /// if zookeeper_path contains {uuid} macro. Replicated database do not recreate replicated tables on recovery, /// so it's ok to save UUID of replicated table. query = fmt::format("CREATE DATABASE IF NOT EXISTS {} ENGINE=Atomic", backQuoteIfNeed(to_db_name_replicated)); query_context = Context::createCopy(getContext()); - executeQuery(query, query_context, QueryFlags(true, false)); + executeQuery(query, query_context, QueryFlags{ .internal = true }); } size_t moved_tables = 0; diff --git a/src/Databases/MySQL/MaterializedMySQLSyncThread.cpp b/src/Databases/MySQL/MaterializedMySQLSyncThread.cpp index a59ad225a6e..ddb331f714e 100644 --- a/src/Databases/MySQL/MaterializedMySQLSyncThread.cpp +++ b/src/Databases/MySQL/MaterializedMySQLSyncThread.cpp @@ -75,7 +75,7 @@ static BlockIO tryToExecuteQuery(const String & query_to_execute, ContextMutable if (!database.empty()) query_context->setCurrentDatabase(database); - return executeQuery("/*" + comment + "*/ " + query_to_execute, query_context, QueryFlags(true, false)).second; + return executeQuery("/*" + comment + "*/ " + query_to_execute, query_context, QueryFlags{ .internal = true }).second; } catch (...) { diff --git a/src/Dictionaries/ClickHouseDictionarySource.cpp b/src/Dictionaries/ClickHouseDictionarySource.cpp index 3312048c73d..37f94062ef9 100644 --- a/src/Dictionaries/ClickHouseDictionarySource.cpp +++ b/src/Dictionaries/ClickHouseDictionarySource.cpp @@ -168,7 +168,7 @@ QueryPipeline ClickHouseDictionarySource::createStreamForQuery(const String & qu if (configuration.is_local) { - pipeline = executeQuery(query, context_copy, QueryFlags(true, false)).second.pipeline; + pipeline = executeQuery(query, context_copy, QueryFlags{ .internal = true }).second.pipeline; pipeline.convertStructureTo(empty_sample_block.getColumnsWithTypeAndName()); } else @@ -190,7 +190,7 @@ std::string ClickHouseDictionarySource::doInvalidateQuery(const std::string & re if (configuration.is_local) { - return readInvalidateQuery(executeQuery(request, context_copy, QueryFlags(true, false)).second.pipeline); + return readInvalidateQuery(executeQuery(request, context_copy, QueryFlags{ .internal = true }).second.pipeline); } else { diff --git a/src/Interpreters/Access/InterpreterShowAccessEntitiesQuery.cpp b/src/Interpreters/Access/InterpreterShowAccessEntitiesQuery.cpp index 5d8e454fcde..bffb47ac714 100644 --- a/src/Interpreters/Access/InterpreterShowAccessEntitiesQuery.cpp +++ b/src/Interpreters/Access/InterpreterShowAccessEntitiesQuery.cpp @@ -23,7 +23,7 @@ InterpreterShowAccessEntitiesQuery::InterpreterShowAccessEntitiesQuery(const AST BlockIO InterpreterShowAccessEntitiesQuery::execute() { - return executeQuery(getRewrittenQuery(), getContext(), QueryFlags(true, false)).second; + return executeQuery(getRewrittenQuery(), getContext(), QueryFlags{ .internal = true }).second; } diff --git a/src/Interpreters/Access/InterpreterShowPrivilegesQuery.cpp b/src/Interpreters/Access/InterpreterShowPrivilegesQuery.cpp index 227c6fea455..1a0b441a06d 100644 --- a/src/Interpreters/Access/InterpreterShowPrivilegesQuery.cpp +++ b/src/Interpreters/Access/InterpreterShowPrivilegesQuery.cpp @@ -12,7 +12,7 @@ InterpreterShowPrivilegesQuery::InterpreterShowPrivilegesQuery(const ASTPtr & qu BlockIO InterpreterShowPrivilegesQuery::execute() { - return executeQuery("SELECT * FROM system.privileges", context, QueryFlags(true, false)).second; + return executeQuery("SELECT * FROM system.privileges", context, QueryFlags{ .internal = true }).second; } } diff --git a/src/Interpreters/DDLWorker.cpp b/src/Interpreters/DDLWorker.cpp index 3b4608cc245..85099fed898 100644 --- a/src/Interpreters/DDLWorker.cpp +++ b/src/Interpreters/DDLWorker.cpp @@ -490,7 +490,7 @@ bool DDLWorker::tryExecuteQuery(DDLTaskBase & task, const ZooKeeperPtr & zookeep if (!task.is_initial_query) query_scope.emplace(query_context); - executeQuery(istr, ostr, !task.is_initial_query, query_context, {}, QueryFlags(false, task.entry.is_backup_restore)); + executeQuery(istr, ostr, !task.is_initial_query, query_context, {}, QueryFlags{ .internal = false, .distributed_backup_restore = task.entry.is_backup_restore }); if (auto txn = query_context->getZooKeeperMetadataTransaction()) { diff --git a/src/Interpreters/InterpreterCreateQuery.cpp b/src/Interpreters/InterpreterCreateQuery.cpp index 843d8749f4b..14bd223a362 100644 --- a/src/Interpreters/InterpreterCreateQuery.cpp +++ b/src/Interpreters/InterpreterCreateQuery.cpp @@ -1076,7 +1076,7 @@ BlockIO InterpreterCreateQuery::createTable(ASTCreateQuery & create) auto guard = DatabaseCatalog::instance().getDDLGuard(database_name, create.getTable()); create.setDatabase(database_name); guard->releaseTableLock(); - return database->tryEnqueueReplicatedDDL(query_ptr, getContext(), QueryFlags(internal, is_restore_from_backup)); + return database->tryEnqueueReplicatedDDL(query_ptr, getContext(), QueryFlags{ .internal = internal, .distributed_backup_restore = is_restore_from_backup }); } if (!create.cluster.empty()) @@ -1232,7 +1232,7 @@ BlockIO InterpreterCreateQuery::createTable(ASTCreateQuery & create) auto guard = DatabaseCatalog::instance().getDDLGuard(create.getDatabase(), create.getTable()); assertOrSetUUID(create, database); guard->releaseTableLock(); - return database->tryEnqueueReplicatedDDL(query_ptr, getContext(), QueryFlags(internal, is_restore_from_backup)); + return database->tryEnqueueReplicatedDDL(query_ptr, getContext(), QueryFlags{ .internal = internal, .distributed_backup_restore = is_restore_from_backup }); } if (!create.cluster.empty()) diff --git a/src/Interpreters/InterpreterKillQueryQuery.cpp b/src/Interpreters/InterpreterKillQueryQuery.cpp index 9aeebd9a213..6e1422f2938 100644 --- a/src/Interpreters/InterpreterKillQueryQuery.cpp +++ b/src/Interpreters/InterpreterKillQueryQuery.cpp @@ -420,7 +420,7 @@ Block InterpreterKillQueryQuery::getSelectResult(const String & columns, const S if (where_expression) select_query += " WHERE " + queryToString(where_expression); - auto io = executeQuery(select_query, getContext(), QueryFlags(true, false)).second; + auto io = executeQuery(select_query, getContext(), QueryFlags{ .internal = true }).second; PullingPipelineExecutor executor(io.pipeline); Block res; while (!res && executor.pull(res)); diff --git a/src/Interpreters/InterpreterShowColumnsQuery.cpp b/src/Interpreters/InterpreterShowColumnsQuery.cpp index 5210614a725..ee8b497f1de 100644 --- a/src/Interpreters/InterpreterShowColumnsQuery.cpp +++ b/src/Interpreters/InterpreterShowColumnsQuery.cpp @@ -159,7 +159,7 @@ WHERE BlockIO InterpreterShowColumnsQuery::execute() { - return executeQuery(getRewrittenQuery(), getContext(), QueryFlags(true, false)).second; + return executeQuery(getRewrittenQuery(), getContext(), QueryFlags{ .internal = true }).second; } diff --git a/src/Interpreters/InterpreterShowEngineQuery.cpp b/src/Interpreters/InterpreterShowEngineQuery.cpp index 5d43b220d06..2927fbd0f2d 100644 --- a/src/Interpreters/InterpreterShowEngineQuery.cpp +++ b/src/Interpreters/InterpreterShowEngineQuery.cpp @@ -12,7 +12,7 @@ namespace DB BlockIO InterpreterShowEnginesQuery::execute() { - return executeQuery("SELECT * FROM system.table_engines ORDER BY name", getContext(), QueryFlags(true, false)).second; + return executeQuery("SELECT * FROM system.table_engines ORDER BY name", getContext(), QueryFlags{ .internal = true }).second; } } diff --git a/src/Interpreters/InterpreterShowFunctionsQuery.cpp b/src/Interpreters/InterpreterShowFunctionsQuery.cpp index 6755e05f75f..a9da01b0988 100644 --- a/src/Interpreters/InterpreterShowFunctionsQuery.cpp +++ b/src/Interpreters/InterpreterShowFunctionsQuery.cpp @@ -15,7 +15,7 @@ InterpreterShowFunctionsQuery::InterpreterShowFunctionsQuery(const ASTPtr & quer BlockIO InterpreterShowFunctionsQuery::execute() { - return executeQuery(getRewrittenQuery(), getContext(), QueryFlags(true, false)).second; + return executeQuery(getRewrittenQuery(), getContext(), QueryFlags{ .internal = true }).second; } String InterpreterShowFunctionsQuery::getRewrittenQuery() diff --git a/src/Interpreters/InterpreterShowIndexesQuery.cpp b/src/Interpreters/InterpreterShowIndexesQuery.cpp index 7c8af6d52c5..09b70e951db 100644 --- a/src/Interpreters/InterpreterShowIndexesQuery.cpp +++ b/src/Interpreters/InterpreterShowIndexesQuery.cpp @@ -101,7 +101,7 @@ ORDER BY index_type, expression, column_name, seq_in_index;)", database, table, BlockIO InterpreterShowIndexesQuery::execute() { - return executeQuery(getRewrittenQuery(), getContext(), QueryFlags(true, false)).second; + return executeQuery(getRewrittenQuery(), getContext(), QueryFlags{ .internal = true }).second; } diff --git a/src/Interpreters/InterpreterShowProcesslistQuery.cpp b/src/Interpreters/InterpreterShowProcesslistQuery.cpp index bb367ebdfa5..f711cc0dac9 100644 --- a/src/Interpreters/InterpreterShowProcesslistQuery.cpp +++ b/src/Interpreters/InterpreterShowProcesslistQuery.cpp @@ -12,7 +12,7 @@ namespace DB BlockIO InterpreterShowProcesslistQuery::execute() { - return executeQuery("SELECT * FROM system.processes ORDER BY elapsed DESC", getContext(), QueryFlags(true, false)).second; + return executeQuery("SELECT * FROM system.processes ORDER BY elapsed DESC", getContext(), QueryFlags{ .internal = true }).second; } } diff --git a/src/Interpreters/InterpreterShowSettingQuery.cpp b/src/Interpreters/InterpreterShowSettingQuery.cpp index 4d7200cb013..45e9b8a1f1c 100644 --- a/src/Interpreters/InterpreterShowSettingQuery.cpp +++ b/src/Interpreters/InterpreterShowSettingQuery.cpp @@ -26,7 +26,7 @@ String InterpreterShowSettingQuery::getRewrittenQuery() BlockIO InterpreterShowSettingQuery::execute() { - return executeQuery(getRewrittenQuery(), getContext(), QueryFlags(true, false)).second; + return executeQuery(getRewrittenQuery(), getContext(), QueryFlags{ .internal = true }).second; } diff --git a/src/Interpreters/InterpreterShowTablesQuery.cpp b/src/Interpreters/InterpreterShowTablesQuery.cpp index 8ddb4eef074..0ca6578128d 100644 --- a/src/Interpreters/InterpreterShowTablesQuery.cpp +++ b/src/Interpreters/InterpreterShowTablesQuery.cpp @@ -214,7 +214,7 @@ BlockIO InterpreterShowTablesQuery::execute() return res; } - return executeQuery(getRewrittenQuery(), getContext(), QueryFlags(true, false)).second; + return executeQuery(getRewrittenQuery(), getContext(), QueryFlags{ .internal = true }).second; } /// (*) Sorting is strictly speaking not necessary but 1. it is convenient for users, 2. SQL currently does not allow to diff --git a/src/Interpreters/MySQL/InterpretersMySQLDDLQuery.h b/src/Interpreters/MySQL/InterpretersMySQLDDLQuery.h index 36f2b0b9fc5..484fd6a0207 100644 --- a/src/Interpreters/MySQL/InterpretersMySQLDDLQuery.h +++ b/src/Interpreters/MySQL/InterpretersMySQLDDLQuery.h @@ -75,7 +75,7 @@ public: ASTs rewritten_queries = InterpreterImpl::getRewrittenQueries(query, getContext(), mapped_to_database, mysql_database); for (const auto & rewritten_query : rewritten_queries) - executeQuery("/* Rewritten MySQL DDL Query */ " + queryToString(rewritten_query), getContext(), QueryFlags(true, false)); + executeQuery("/* Rewritten MySQL DDL Query */ " + queryToString(rewritten_query), getContext(), QueryFlags{ .internal = true }); return BlockIO{}; } diff --git a/src/Interpreters/executeQuery.h b/src/Interpreters/executeQuery.h index 79ede484592..0f599922668 100644 --- a/src/Interpreters/executeQuery.h +++ b/src/Interpreters/executeQuery.h @@ -31,9 +31,6 @@ using HandleExceptionInOutputFormatFunc = std::functionsetSetting("force_remove_data_recursively_on_drop", false); - res = executeQuery(drop_query, context, QueryFlags(true, false)).second; + res = executeQuery(drop_query, context, QueryFlags{ .internal = true }).second; executeTrivialBlockIO(res, context); res = {}; String rename_query = fmt::format("RENAME DATABASE {} TO {}", tmp_name_quoted, name_quoted); - res = executeQuery(rename_query, context, QueryFlags(true, false)).second; + res = executeQuery(rename_query, context, QueryFlags{ .internal = true }).second; executeTrivialBlockIO(res, context); LOG_INFO(log, "Finished database engine conversion of {}", name_quoted); @@ -409,7 +409,7 @@ static void maybeConvertOrdinaryDatabaseToAtomic(ContextMutablePtr context, cons /// Reload database just in case (and update logger name) String detach_query = fmt::format("DETACH DATABASE {}", backQuoteIfNeed(database_name)); - auto res = executeQuery(detach_query, context, QueryFlags(true, false)).second; + auto res = executeQuery(detach_query, context, QueryFlags{ .internal = true }).second; executeTrivialBlockIO(res, context); res = {}; From b141726ad796b828355364e3a49c2d9b2636fc37 Mon Sep 17 00:00:00 2001 From: kssenii Date: Thu, 9 Nov 2023 17:21:12 +0100 Subject: [PATCH 8/9] Fix for data_type_default_nullable --- src/Interpreters/InterpreterCreateQuery.cpp | 2 +- ..._backup_restore_default_nullable.reference | 4 ++++ .../02907_backup_restore_default_nullable.sh | 22 +++++++++++++++++++ 3 files changed, 27 insertions(+), 1 deletion(-) create mode 100644 tests/queries/0_stateless/02907_backup_restore_default_nullable.reference create mode 100644 tests/queries/0_stateless/02907_backup_restore_default_nullable.sh diff --git a/src/Interpreters/InterpreterCreateQuery.cpp b/src/Interpreters/InterpreterCreateQuery.cpp index 14bd223a362..053cd08f044 100644 --- a/src/Interpreters/InterpreterCreateQuery.cpp +++ b/src/Interpreters/InterpreterCreateQuery.cpp @@ -489,7 +489,7 @@ ColumnsDescription InterpreterCreateQuery::getColumnsDescription( ASTPtr default_expr_list = std::make_shared(); NamesAndTypesList column_names_and_types; - bool make_columns_nullable = !attach && context_->getSettingsRef().data_type_default_nullable; + bool make_columns_nullable = !attach && !is_restore_from_backup && context_->getSettingsRef().data_type_default_nullable; for (const auto & ast : columns_ast.children) { diff --git a/tests/queries/0_stateless/02907_backup_restore_default_nullable.reference b/tests/queries/0_stateless/02907_backup_restore_default_nullable.reference new file mode 100644 index 00000000000..0aed0444667 --- /dev/null +++ b/tests/queries/0_stateless/02907_backup_restore_default_nullable.reference @@ -0,0 +1,4 @@ +BACKUP_CREATED +CREATE TABLE default.test\n(\n `test` String\n)\nENGINE = MergeTree\nORDER BY tuple()\nSETTINGS index_granularity = 8192 +RESTORED +CREATE TABLE default.test\n(\n `test` String\n)\nENGINE = MergeTree\nORDER BY tuple()\nSETTINGS index_granularity = 8192 diff --git a/tests/queries/0_stateless/02907_backup_restore_default_nullable.sh b/tests/queries/0_stateless/02907_backup_restore_default_nullable.sh new file mode 100644 index 00000000000..8ed36a7edd7 --- /dev/null +++ b/tests/queries/0_stateless/02907_backup_restore_default_nullable.sh @@ -0,0 +1,22 @@ +#!/usr/bin/env bash + +CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh +. "$CUR_DIR"/../shell_config.sh + +${CLICKHOUSE_CLIENT} -nm --query " +drop table if exists test; +set data_type_default_nullable = 0; +create table test (test String) ENGINE = MergeTree() ORDER BY tuple(); +backup table ${CLICKHOUSE_DATABASE}.test on cluster test_shard_localhost to Disk('backups', '${CLICKHOUSE_TEST_UNIQUE_NAME}'); +" | grep -o "BACKUP_CREATED" + +${CLICKHOUSE_CLIENT} --query "show create table test" + +${CLICKHOUSE_CLIENT} -nm --query " +drop table test sync; +set data_type_default_nullable = 1; +restore table ${CLICKHOUSE_DATABASE}.test on cluster test_shard_localhost from Disk('backups', '${CLICKHOUSE_TEST_UNIQUE_NAME}'); +" | grep -o "RESTORED" + +${CLICKHOUSE_CLIENT} --query "show create table test" From 7ceadaa57620869688e2d179b7be234c0fffb517 Mon Sep 17 00:00:00 2001 From: kssenii Date: Thu, 9 Nov 2023 18:10:13 +0100 Subject: [PATCH 9/9] chmod --- .../queries/0_stateless/02907_backup_restore_default_nullable.sh | 0 1 file changed, 0 insertions(+), 0 deletions(-) mode change 100644 => 100755 tests/queries/0_stateless/02907_backup_restore_default_nullable.sh diff --git a/tests/queries/0_stateless/02907_backup_restore_default_nullable.sh b/tests/queries/0_stateless/02907_backup_restore_default_nullable.sh old mode 100644 new mode 100755