ClickHouse/dbms/Interpreters/InterpreterSystemQuery.cpp

552 lines
20 KiB
C++
Raw Normal View History

#include <Interpreters/InterpreterSystemQuery.h>
2018-04-19 13:56:14 +00:00
#include <Common/DNSResolver.h>
#include <Common/ActionLock.h>
#include "config_core.h"
#include <Common/typeid_cast.h>
2018-06-05 19:46:49 +00:00
#include <Common/getNumberOfPhysicalCPUCores.h>
#include <Common/ThreadPool.h>
2019-10-25 19:07:47 +00:00
#include <Common/escapeForFileName.h>
#include <Interpreters/Context.h>
#include <Interpreters/DatabaseCatalog.h>
#include <Interpreters/ExternalDictionariesLoader.h>
#include <Interpreters/EmbeddedDictionaries.h>
#include <Interpreters/ActionLocksManager.h>
#include <Interpreters/InterpreterDropQuery.h>
#include <Interpreters/InterpreterCreateQuery.h>
#include <Interpreters/InterpreterRenameQuery.h>
#include <Interpreters/QueryLog.h>
#include <Interpreters/DDLWorker.h>
#include <Interpreters/PartLog.h>
#include <Interpreters/QueryThreadLog.h>
2019-02-03 21:30:45 +00:00
#include <Interpreters/TraceLog.h>
#include <Interpreters/TextLog.h>
2019-08-13 14:31:46 +00:00
#include <Interpreters/MetricLog.h>
#include <Access/ContextAccess.h>
#include <Databases/IDatabase.h>
#include <Storages/StorageDistributed.h>
#include <Storages/StorageReplicatedMergeTree.h>
#include <Storages/StorageFactory.h>
#include <Parsers/ASTSystemQuery.h>
#include <Parsers/ASTDropQuery.h>
#include <Parsers/ASTCreateQuery.h>
#include <csignal>
2019-01-09 15:44:20 +00:00
#include <algorithm>
namespace DB
{
namespace ErrorCodes
{
2020-02-25 18:02:41 +00:00
extern const int LOGICAL_ERROR;
extern const int BAD_ARGUMENTS;
extern const int CANNOT_KILL;
extern const int NOT_IMPLEMENTED;
2019-09-19 11:04:57 +00:00
extern const int TIMEOUT_EXCEEDED;
}
namespace ActionLocks
{
extern StorageActionBlockType PartsMerge;
extern StorageActionBlockType PartsFetch;
extern StorageActionBlockType PartsSend;
extern StorageActionBlockType ReplicationQueue;
2019-04-22 15:11:16 +00:00
extern StorageActionBlockType DistributedSend;
2019-08-01 15:36:12 +00:00
extern StorageActionBlockType PartsTTLMerge;
2019-09-03 14:50:49 +00:00
extern StorageActionBlockType PartsMove;
}
namespace
{
ExecutionStatus getOverallExecutionStatusOfCommands()
{
return ExecutionStatus(0);
}
2019-01-22 19:56:53 +00:00
/// Consequently tries to execute all commands and generates final exception message for failed commands
template <typename Callable, typename ... Callables>
ExecutionStatus getOverallExecutionStatusOfCommands(Callable && command, Callables && ... commands)
{
ExecutionStatus status_head(0);
try
{
command();
}
catch (...)
{
status_head = ExecutionStatus::fromCurrentException();
}
ExecutionStatus status_tail = getOverallExecutionStatusOfCommands(std::forward<Callables>(commands)...);
auto res_status = status_head.code != 0 ? status_head.code : status_tail.code;
auto res_message = status_head.message + (status_tail.message.empty() ? "" : ("\n" + status_tail.message));
return ExecutionStatus(res_status, res_message);
}
/// Consequently tries to execute all commands and throws exception with info about failed commands
template <typename ... Callables>
void executeCommandsAndThrowIfError(Callables && ... commands)
{
auto status = getOverallExecutionStatusOfCommands(std::forward<Callables>(commands)...);
if (status.code != 0)
throw Exception(status.message, status.code);
}
2020-01-24 16:20:36 +00:00
AccessType getRequiredAccessType(StorageActionBlockType action_type)
{
if (action_type == ActionLocks::PartsMerge)
return AccessType::STOP_MERGES;
else if (action_type == ActionLocks::PartsFetch)
return AccessType::STOP_FETCHES;
else if (action_type == ActionLocks::PartsSend)
return AccessType::STOP_REPLICATED_SENDS;
else if (action_type == ActionLocks::ReplicationQueue)
return AccessType::STOP_REPLICATION_QUEUES;
else if (action_type == ActionLocks::DistributedSend)
return AccessType::STOP_DISTRIBUTED_SENDS;
else if (action_type == ActionLocks::PartsTTLMerge)
return AccessType::STOP_TTL_MERGES;
else if (action_type == ActionLocks::PartsMove)
return AccessType::STOP_MOVES;
else
throw Exception("Unknown action type: " + std::to_string(action_type), ErrorCodes::LOGICAL_ERROR);
}
2020-03-04 20:29:52 +00:00
}
2020-01-24 16:20:36 +00:00
/// Implements SYSTEM [START|STOP] <something action from ActionLocks>
2020-03-04 20:29:52 +00:00
void InterpreterSystemQuery::startStopAction(StorageActionBlockType action_type, bool start)
{
auto manager = context.getActionLocksManager();
manager->cleanExpired();
2020-03-04 20:29:52 +00:00
if (table_id)
{
2020-03-04 20:29:52 +00:00
context.checkAccess(getRequiredAccessType(action_type), table_id);
if (start)
2020-03-04 20:29:52 +00:00
manager->remove(table_id, action_type);
else
2020-03-04 20:29:52 +00:00
manager->add(table_id, action_type);
}
else
{
auto access = context.getAccess();
for (auto & elem : DatabaseCatalog::instance().getDatabases())
2020-01-24 16:20:36 +00:00
{
for (auto iterator = elem.second->getTablesIterator(context); iterator->isValid(); iterator->next())
{
if (!access->isGranted(log, getRequiredAccessType(action_type), elem.first, iterator->name()))
continue;
if (start)
manager->remove(iterator->table(), action_type);
else
manager->add(iterator->table(), action_type);
2020-01-24 16:20:36 +00:00
}
}
}
}
2020-01-24 16:20:36 +00:00
2020-03-04 20:29:52 +00:00
InterpreterSystemQuery::InterpreterSystemQuery(const ASTPtr & query_ptr_, Context & context_)
: query_ptr(query_ptr_->clone()), context(context_), log(&Poco::Logger::get("InterpreterSystemQuery"))
{
}
BlockIO InterpreterSystemQuery::execute()
{
auto & query = query_ptr->as<ASTSystemQuery &>();
if (!query.cluster.empty())
2020-01-24 16:20:36 +00:00
return executeDDLQueryOnCluster(query_ptr, context, getRequiredAccessForDDLOnCluster());
using Type = ASTSystemQuery::Type;
/// Use global context with fresh system profile settings
Context system_context = context.getGlobalContext();
system_context.setSetting("profile", context.getSystemProfileName());
/// Make canonical query for simpler processing
2020-03-04 20:29:52 +00:00
if (!query.table.empty())
table_id = context.resolveStorageID(StorageID(query.database, query.table), Context::ResolveOrdinary);
2019-12-23 03:48:34 +00:00
if (!query.target_dictionary.empty() && !query.database.empty())
query.target_dictionary = query.database + "." + query.target_dictionary;
switch (query.type)
{
case Type::SHUTDOWN:
2020-01-24 16:20:36 +00:00
context.checkAccess(AccessType::SHUTDOWN);
if (kill(0, SIGTERM))
throwFromErrno("System call kill(0, SIGTERM) failed", ErrorCodes::CANNOT_KILL);
break;
case Type::KILL:
2020-01-24 16:20:36 +00:00
context.checkAccess(AccessType::SHUTDOWN);
if (kill(0, SIGKILL))
throwFromErrno("System call kill(0, SIGKILL) failed", ErrorCodes::CANNOT_KILL);
break;
case Type::DROP_DNS_CACHE:
2020-01-24 16:20:36 +00:00
context.checkAccess(AccessType::DROP_CACHE);
2018-04-19 13:56:14 +00:00
DNSResolver::instance().dropCache();
/// Reinitialize clusters to update their resolved_addresses
system_context.reloadClusterConfig();
break;
case Type::DROP_MARK_CACHE:
2020-01-24 16:20:36 +00:00
context.checkAccess(AccessType::DROP_CACHE);
system_context.dropMarkCache();
break;
case Type::DROP_UNCOMPRESSED_CACHE:
2020-01-24 16:20:36 +00:00
context.checkAccess(AccessType::DROP_CACHE);
system_context.dropUncompressedCache();
break;
#if USE_EMBEDDED_COMPILER
case Type::DROP_COMPILED_EXPRESSION_CACHE:
2020-01-24 16:20:36 +00:00
context.checkAccess(AccessType::DROP_CACHE);
system_context.dropCompiledExpressionCache();
break;
#endif
case Type::RELOAD_DICTIONARY:
2020-01-24 16:20:36 +00:00
context.checkAccess(AccessType::RELOAD_DICTIONARY);
system_context.getExternalDictionariesLoader().loadOrReload(query.target_dictionary);
2020-03-03 08:32:58 +00:00
ExternalDictionariesLoader::resetAll();
break;
case Type::RELOAD_DICTIONARIES:
2020-01-24 16:20:36 +00:00
context.checkAccess(AccessType::RELOAD_DICTIONARY);
executeCommandsAndThrowIfError(
[&] () { system_context.getExternalDictionariesLoader().reloadAllTriedToLoad(); },
[&] () { system_context.getEmbeddedDictionaries().reload(); }
);
2020-03-03 08:32:58 +00:00
ExternalDictionariesLoader::resetAll();
break;
case Type::RELOAD_EMBEDDED_DICTIONARIES:
2020-01-24 16:20:36 +00:00
context.checkAccess(AccessType::RELOAD_DICTIONARY);
system_context.getEmbeddedDictionaries().reload();
break;
case Type::RELOAD_CONFIG:
2020-01-24 16:20:36 +00:00
context.checkAccess(AccessType::RELOAD_CONFIG);
system_context.reloadConfig();
break;
case Type::STOP_MERGES:
2020-03-04 20:29:52 +00:00
startStopAction(ActionLocks::PartsMerge, false);
break;
case Type::START_MERGES:
2020-03-04 20:29:52 +00:00
startStopAction(ActionLocks::PartsMerge, true);
break;
2019-08-01 15:36:12 +00:00
case Type::STOP_TTL_MERGES:
2020-03-04 20:29:52 +00:00
startStopAction(ActionLocks::PartsTTLMerge, false);
2019-08-01 15:36:12 +00:00
break;
case Type::START_TTL_MERGES:
2020-03-04 20:29:52 +00:00
startStopAction(ActionLocks::PartsTTLMerge, true);
2019-08-01 15:36:12 +00:00
break;
2019-09-03 14:50:49 +00:00
case Type::STOP_MOVES:
2020-03-04 20:29:52 +00:00
startStopAction(ActionLocks::PartsMove, false);
2019-09-03 14:50:49 +00:00
break;
case Type::START_MOVES:
2020-03-04 20:29:52 +00:00
startStopAction(ActionLocks::PartsMove, true);
2019-09-03 14:50:49 +00:00
break;
case Type::STOP_FETCHES:
2020-03-04 20:29:52 +00:00
startStopAction(ActionLocks::PartsFetch, false);
break;
case Type::START_FETCHES:
2020-03-04 20:29:52 +00:00
startStopAction(ActionLocks::PartsFetch, true);
break;
case Type::STOP_REPLICATED_SENDS:
2020-03-04 20:29:52 +00:00
startStopAction(ActionLocks::PartsSend, false);
break;
2019-02-02 11:28:43 +00:00
case Type::START_REPLICATED_SENDS:
2020-03-04 20:29:52 +00:00
startStopAction(ActionLocks::PartsSend, true);
break;
case Type::STOP_REPLICATION_QUEUES:
2020-03-04 20:29:52 +00:00
startStopAction(ActionLocks::ReplicationQueue, false);
break;
case Type::START_REPLICATION_QUEUES:
2020-03-04 20:29:52 +00:00
startStopAction(ActionLocks::ReplicationQueue, true);
break;
2019-04-22 15:11:16 +00:00
case Type::STOP_DISTRIBUTED_SENDS:
2020-03-04 20:29:52 +00:00
startStopAction(ActionLocks::DistributedSend, false);
2019-04-22 15:11:16 +00:00
break;
case Type::START_DISTRIBUTED_SENDS:
2020-03-04 20:29:52 +00:00
startStopAction(ActionLocks::DistributedSend, true);
2019-04-22 15:11:16 +00:00
break;
case Type::SYNC_REPLICA:
syncReplica(query);
break;
case Type::FLUSH_DISTRIBUTED:
flushDistributed(query);
2019-04-22 15:11:16 +00:00
break;
case Type::RESTART_REPLICAS:
restartReplicas(system_context);
break;
case Type::RESTART_REPLICA:
2020-03-04 20:29:52 +00:00
if (!tryRestartReplica(table_id, system_context))
throw Exception("There is no " + query.database + "." + query.table + " replicated table",
ErrorCodes::BAD_ARGUMENTS);
break;
case Type::FLUSH_LOGS:
2020-01-24 16:20:36 +00:00
context.checkAccess(AccessType::FLUSH_LOGS);
executeCommandsAndThrowIfError(
2018-12-14 15:39:48 +00:00
[&] () { if (auto query_log = context.getQueryLog()) query_log->flush(); },
[&] () { if (auto part_log = context.getPartLog("")) part_log->flush(); },
2019-02-03 21:30:45 +00:00
[&] () { if (auto query_thread_log = context.getQueryThreadLog()) query_thread_log->flush(); },
[&] () { if (auto trace_log = context.getTraceLog()) trace_log->flush(); },
2019-08-13 14:31:46 +00:00
[&] () { if (auto text_log = context.getTextLog()) text_log->flush(); },
[&] () { if (auto metric_log = context.getMetricLog()) metric_log->flush(); }
);
break;
case Type::STOP_LISTEN_QUERIES:
case Type::START_LISTEN_QUERIES:
throw Exception(String(ASTSystemQuery::typeToString(query.type)) + " is not supported yet", ErrorCodes::NOT_IMPLEMENTED);
default:
throw Exception("Unknown type of SYSTEM query", ErrorCodes::BAD_ARGUMENTS);
}
return BlockIO();
}
StoragePtr InterpreterSystemQuery::tryRestartReplica(const StorageID & replica, Context & system_context, bool need_ddl_guard)
{
2020-03-04 20:29:52 +00:00
context.checkAccess(AccessType::RESTART_REPLICA, replica);
2020-01-24 16:20:36 +00:00
auto table_ddl_guard = need_ddl_guard ? DatabaseCatalog::instance().getDDLGuard(replica.getDatabaseName(), replica.getTableName()) : nullptr;
2020-03-04 20:29:52 +00:00
auto [database, table] = DatabaseCatalog::instance().tryGetDatabaseAndTable(replica);
ASTPtr create_ast;
/// Detach actions
2020-03-04 20:29:52 +00:00
if (!table || !dynamic_cast<const StorageReplicatedMergeTree *>(table.get()))
return nullptr;
2020-03-04 20:29:52 +00:00
table->shutdown();
{
/// If table was already dropped by anyone, an exception will be thrown
2019-03-05 10:12:20 +00:00
auto table_lock = table->lockExclusively(context.getCurrentQueryId());
2020-03-04 20:29:52 +00:00
create_ast = database->getCreateTableQuery(system_context, replica.table_name);
2020-03-04 20:29:52 +00:00
database->detachTable(replica.table_name);
}
2020-03-04 20:29:52 +00:00
table.reset();
/// Attach actions
2020-03-04 20:29:52 +00:00
/// getCreateTableQuery must return canonical CREATE query representation, there are no need for AST postprocessing
auto & create = create_ast->as<ASTCreateQuery &>();
create.attach = true;
2020-03-04 20:29:52 +00:00
auto columns = InterpreterCreateQuery::getColumnsDescription(*create.columns_list->columns, system_context);
auto constraints = InterpreterCreateQuery::getConstraintsDescription(create.columns_list->constraints);
2020-03-04 20:29:52 +00:00
table = StorageFactory::instance().get(create,
database->getTableDataPath(create),
system_context,
system_context.getGlobalContext(),
columns,
constraints,
false);
2020-03-04 20:29:52 +00:00
database->createTable(system_context, replica.table_name, table, create_ast);
2020-03-04 20:29:52 +00:00
table->startup();
return table;
}
2018-08-27 18:05:28 +00:00
void InterpreterSystemQuery::restartReplicas(Context & system_context)
{
2020-03-04 20:29:52 +00:00
std::vector<StorageID> replica_names;
auto & catalog = DatabaseCatalog::instance();
for (auto & elem : catalog.getDatabases())
{
DatabasePtr & database = elem.second;
2019-10-10 17:33:01 +00:00
for (auto iterator = database->getTablesIterator(system_context); iterator->isValid(); iterator->next())
{
if (dynamic_cast<const StorageReplicatedMergeTree *>(iterator->table().get()))
2020-03-04 20:29:52 +00:00
replica_names.emplace_back(iterator->table()->getStorageID());
}
}
if (replica_names.empty())
return;
TableGuards guards;
for (const auto & name : replica_names)
guards.emplace(UniqueTableName{name.database_name, name.table_name}, nullptr);
for (auto & guard : guards)
guard.second = catalog.getDDLGuard(guard.first.database_name, guard.first.table_name);
2019-01-09 15:44:20 +00:00
ThreadPool pool(std::min(size_t(getNumberOfPhysicalCPUCores()), replica_names.size()));
for (auto & table : replica_names)
pool.scheduleOrThrowOnError([&]() { tryRestartReplica(table, system_context, false); });
pool.wait();
}
2020-03-04 20:29:52 +00:00
void InterpreterSystemQuery::syncReplica(ASTSystemQuery &)
{
2020-03-04 20:29:52 +00:00
context.checkAccess(AccessType::SYNC_REPLICA, table_id);
StoragePtr table = DatabaseCatalog::instance().getTable(table_id);
2019-04-22 15:11:16 +00:00
if (auto storage_replicated = dynamic_cast<StorageReplicatedMergeTree *>(table.get()))
2019-09-19 11:04:57 +00:00
{
LOG_TRACE(log, "Synchronizing entries in replica's queue with table's log and waiting for it to become empty");
if (!storage_replicated->waitForShrinkingQueueSize(0, context.getSettingsRef().receive_timeout.totalMilliseconds()))
{
2020-03-04 20:29:52 +00:00
LOG_ERROR(log, "SYNC REPLICA " + table_id.getNameForLogs() + ": Timed out!");
2019-09-19 11:04:57 +00:00
throw Exception(
2020-03-04 20:29:52 +00:00
"SYNC REPLICA " + table_id.getNameForLogs() + ": command timed out! "
2019-09-19 11:04:57 +00:00
"See the 'receive_timeout' setting", ErrorCodes::TIMEOUT_EXCEEDED);
}
2020-03-04 20:29:52 +00:00
LOG_TRACE(log, "SYNC REPLICA " + table_id.getNameForLogs() + ": OK");
2019-09-19 11:04:57 +00:00
}
else
2020-03-04 20:29:52 +00:00
throw Exception("Table " + table_id.getNameForLogs() + " is not replicated", ErrorCodes::BAD_ARGUMENTS);
}
2020-03-04 20:29:52 +00:00
void InterpreterSystemQuery::flushDistributed(ASTSystemQuery &)
2019-04-22 15:11:16 +00:00
{
2020-03-04 20:29:52 +00:00
context.checkAccess(AccessType::FLUSH_DISTRIBUTED, table_id);
2019-04-22 15:11:16 +00:00
2020-03-04 20:29:52 +00:00
if (auto storage_distributed = dynamic_cast<StorageDistributed *>(DatabaseCatalog::instance().getTable(table_id).get()))
storage_distributed->flushClusterNodesAllData();
2019-04-22 15:11:16 +00:00
else
2020-03-04 20:29:52 +00:00
throw Exception("Table " + table_id.getNameForLogs() + " is not distributed", ErrorCodes::BAD_ARGUMENTS);
}
2020-01-24 16:20:36 +00:00
AccessRightsElements InterpreterSystemQuery::getRequiredAccessForDDLOnCluster() const
{
const auto & query = query_ptr->as<const ASTSystemQuery &>();
using Type = ASTSystemQuery::Type;
AccessRightsElements required_access;
switch (query.type)
{
case Type::SHUTDOWN: [[fallthrough]];
case Type::KILL:
{
required_access.emplace_back(AccessType::SHUTDOWN);
break;
}
case Type::DROP_DNS_CACHE: [[fallthrough]];
case Type::DROP_MARK_CACHE: [[fallthrough]];
#if USE_EMBEDDED_COMPILER
case Type::DROP_COMPILED_EXPRESSION_CACHE: [[fallthrough]];
#endif
case Type::DROP_UNCOMPRESSED_CACHE:
{
required_access.emplace_back(AccessType::DROP_CACHE);
break;
}
case Type::RELOAD_DICTIONARY: [[fallthrough]];
case Type::RELOAD_DICTIONARIES: [[fallthrough]];
case Type::RELOAD_EMBEDDED_DICTIONARIES:
{
required_access.emplace_back(AccessType::RELOAD_DICTIONARY);
break;
}
case Type::RELOAD_CONFIG:
{
required_access.emplace_back(AccessType::RELOAD_CONFIG);
break;
}
case Type::STOP_MERGES: [[fallthrough]];
case Type::START_MERGES:
{
if (query.table.empty())
required_access.emplace_back(AccessType::STOP_MERGES);
else
required_access.emplace_back(AccessType::STOP_MERGES, query.database, query.table);
break;
}
case Type::STOP_TTL_MERGES: [[fallthrough]];
case Type::START_TTL_MERGES:
{
if (query.table.empty())
required_access.emplace_back(AccessType::STOP_TTL_MERGES);
else
required_access.emplace_back(AccessType::STOP_TTL_MERGES, query.database, query.table);
break;
}
case Type::STOP_MOVES: [[fallthrough]];
case Type::START_MOVES:
{
if (query.table.empty())
required_access.emplace_back(AccessType::STOP_MOVES);
else
required_access.emplace_back(AccessType::STOP_MOVES, query.database, query.table);
break;
}
case Type::STOP_FETCHES: [[fallthrough]];
case Type::START_FETCHES:
{
if (query.table.empty())
required_access.emplace_back(AccessType::STOP_FETCHES);
else
required_access.emplace_back(AccessType::STOP_FETCHES, query.database, query.table);
break;
}
case Type::STOP_DISTRIBUTED_SENDS: [[fallthrough]];
case Type::START_DISTRIBUTED_SENDS:
{
if (query.table.empty())
required_access.emplace_back(AccessType::STOP_DISTRIBUTED_SENDS);
else
required_access.emplace_back(AccessType::STOP_DISTRIBUTED_SENDS, query.database, query.table);
break;
}
case Type::STOP_REPLICATED_SENDS: [[fallthrough]];
case Type::START_REPLICATED_SENDS:
{
if (query.table.empty())
required_access.emplace_back(AccessType::STOP_REPLICATED_SENDS);
else
required_access.emplace_back(AccessType::STOP_REPLICATED_SENDS, query.database, query.table);
break;
}
case Type::STOP_REPLICATION_QUEUES: [[fallthrough]];
case Type::START_REPLICATION_QUEUES:
{
if (query.table.empty())
required_access.emplace_back(AccessType::STOP_REPLICATION_QUEUES);
else
required_access.emplace_back(AccessType::STOP_REPLICATION_QUEUES, query.database, query.table);
break;
}
case Type::SYNC_REPLICA:
{
required_access.emplace_back(AccessType::SYNC_REPLICA, query.database, query.table);
break;
}
case Type::RESTART_REPLICA:
{
required_access.emplace_back(AccessType::RESTART_REPLICA, query.database, query.table);
break;
}
case Type::RESTART_REPLICAS:
{
required_access.emplace_back(AccessType::RESTART_REPLICA);
break;
}
case Type::FLUSH_DISTRIBUTED:
{
required_access.emplace_back(AccessType::FLUSH_DISTRIBUTED, query.database, query.table);
break;
}
case Type::FLUSH_LOGS:
{
required_access.emplace_back(AccessType::FLUSH_LOGS);
break;
}
case Type::STOP_LISTEN_QUERIES: break;
case Type::START_LISTEN_QUERIES: break;
case Type::UNKNOWN: break;
case Type::END: break;
}
return required_access;
}
}