ClickHouse/src/Storages/IStorage.cpp

Ignoring revisions in .git-blame-ignore-revs. Click here to bypass and see the normal blame view.

315 lines
10 KiB
C++
Raw Normal View History

#include <Storages/IStorage.h>
#include <Common/StringUtils/StringUtils.h>
2020-12-18 17:13:28 +00:00
#include <IO/Operators.h>
#include <IO/WriteBufferFromString.h>
#include <Interpreters/Context.h>
2020-09-25 13:19:26 +00:00
#include <Interpreters/InterpreterSelectQuery.h>
2020-12-18 17:13:28 +00:00
#include <Parsers/ASTCreateQuery.h>
#include <Parsers/ASTFunction.h>
2020-12-18 17:13:28 +00:00
#include <Parsers/ASTSetQuery.h>
2021-10-16 14:03:50 +00:00
#include <QueryPipeline/Pipe.h>
2020-12-18 17:13:28 +00:00
#include <Processors/QueryPlan/ReadFromPreparedSource.h>
2021-09-08 18:29:38 +00:00
#include <Processors/QueryPlan/QueryPlan.h>
2020-12-18 17:13:28 +00:00
#include <Storages/AlterCommands.h>
#include <Backups/RestorerFromBackup.h>
#include <Backups/IBackup.h>
namespace DB
{
namespace ErrorCodes
{
2019-08-27 23:47:30 +00:00
extern const int TABLE_IS_DROPPED;
2019-12-26 18:17:05 +00:00
extern const int NOT_IMPLEMENTED;
extern const int DEADLOCK_AVOIDED;
extern const int CANNOT_RESTORE_TABLE;
}
2020-06-17 14:37:21 +00:00
bool IStorage::isVirtualColumn(const String & column_name, const StorageMetadataPtr & metadata_snapshot) const
2019-05-21 11:24:32 +00:00
{
2020-08-08 00:47:03 +00:00
/// Virtual column maybe overridden by real column
2020-06-17 14:37:21 +00:00
return !metadata_snapshot->getColumns().has(column_name) && getVirtuals().contains(column_name);
2019-05-21 11:24:32 +00:00
}
RWLockImpl::LockHolder IStorage::tryLockTimed(
const RWLock & rwlock, RWLockImpl::Type type, const String & query_id, const std::chrono::milliseconds & acquire_timeout) const
{
auto lock_holder = rwlock->getLock(type, query_id, acquire_timeout);
if (!lock_holder)
2020-04-07 11:34:35 +00:00
{
const String type_str = type == RWLockImpl::Type::Read ? "READ" : "WRITE";
throw Exception(
2020-12-07 09:30:47 +00:00
type_str + " locking attempt on \"" + getStorageID().getFullTableName() + "\" has timed out! ("
+ std::to_string(acquire_timeout.count())
+ "ms) "
"Possible deadlock avoided. Client should retry.",
ErrorCodes::DEADLOCK_AVOIDED);
2020-04-07 11:34:35 +00:00
}
2020-04-07 07:15:59 +00:00
return lock_holder;
}
TableLockHolder IStorage::lockForShare(const String & query_id, const std::chrono::milliseconds & acquire_timeout)
{
TableLockHolder result = tryLockTimed(drop_lock, RWLockImpl::Read, query_id, acquire_timeout);
if (is_dropped)
2022-09-28 16:59:07 +00:00
{
auto table_id = getStorageID();
throw Exception(ErrorCodes::TABLE_IS_DROPPED, "Table {}.{} is dropped", table_id.database_name, table_id.table_name);
}
return result;
}
TableLockHolder IStorage::tryLockForShare(const String & query_id, const std::chrono::milliseconds & acquire_timeout)
{
2020-06-18 16:10:47 +00:00
TableLockHolder result = tryLockTimed(drop_lock, RWLockImpl::Read, query_id, acquire_timeout);
if (is_dropped)
2021-05-03 09:52:13 +00:00
{
2022-09-28 15:41:51 +00:00
// Table was dropped while acquiring the lock
result = nullptr;
2021-05-03 09:52:13 +00:00
}
2020-06-18 16:10:47 +00:00
return result;
}
2021-10-25 17:49:49 +00:00
IStorage::AlterLockHolder IStorage::lockForAlter(const std::chrono::milliseconds & acquire_timeout)
{
2021-10-25 17:49:49 +00:00
AlterLockHolder lock{alter_lock, std::defer_lock};
if (!lock.try_lock_for(acquire_timeout))
throw Exception(ErrorCodes::DEADLOCK_AVOIDED,
"Locking attempt for ALTER on \"{}\" has timed out! ({} ms) "
"Possible deadlock avoided. Client should retry.",
getStorageID().getFullTableName(), std::to_string(acquire_timeout.count()));
if (is_dropped)
throw Exception("Table is dropped", ErrorCodes::TABLE_IS_DROPPED);
2020-06-18 16:10:47 +00:00
2021-10-25 17:49:49 +00:00
return lock;
}
TableExclusiveLockHolder IStorage::lockExclusively(const String & query_id, const std::chrono::milliseconds & acquire_timeout)
{
2020-06-18 16:10:47 +00:00
TableExclusiveLockHolder result;
2021-10-25 17:49:49 +00:00
result.drop_lock = tryLockTimed(drop_lock, RWLockImpl::Write, query_id, acquire_timeout);
if (is_dropped)
throw Exception("Table is dropped", ErrorCodes::TABLE_IS_DROPPED);
return result;
}
2022-05-20 19:49:31 +00:00
Pipe IStorage::watch(
const Names & /*column_names*/,
const SelectQueryInfo & /*query_info*/,
ContextPtr /*context*/,
QueryProcessingStage::Enum & /*processed_stage*/,
size_t /*max_block_size*/,
size_t /*num_streams*/)
2022-05-20 19:49:31 +00:00
{
throw Exception("Method watch is not supported by storage " + getName(), ErrorCodes::NOT_IMPLEMENTED);
}
2020-08-03 11:33:11 +00:00
Pipe IStorage::read(
2020-12-07 09:30:47 +00:00
const Names & /*column_names*/,
const StorageSnapshotPtr & /*storage_snapshot*/,
2020-12-07 09:30:47 +00:00
SelectQueryInfo & /*query_info*/,
ContextPtr /*context*/,
2020-12-07 09:30:47 +00:00
QueryProcessingStage::Enum /*processed_stage*/,
size_t /*max_block_size*/,
size_t /*num_streams*/)
2020-08-03 11:33:11 +00:00
{
throw Exception("Method read is not supported by storage " + getName(), ErrorCodes::NOT_IMPLEMENTED);
}
2020-09-14 14:13:58 +00:00
void IStorage::read(
2020-12-07 09:30:47 +00:00
QueryPlan & query_plan,
const Names & column_names,
const StorageSnapshotPtr & storage_snapshot,
2020-12-07 09:30:47 +00:00
SelectQueryInfo & query_info,
ContextPtr context,
2020-12-07 09:30:47 +00:00
QueryProcessingStage::Enum processed_stage,
size_t max_block_size,
size_t num_streams)
2020-09-14 14:13:58 +00:00
{
auto pipe = read(column_names, storage_snapshot, query_info, context, processed_stage, max_block_size, num_streams);
2022-05-20 19:49:31 +00:00
readFromPipe(query_plan, std::move(pipe), column_names, storage_snapshot, query_info, context, getName());
}
void IStorage::readFromPipe(
QueryPlan & query_plan,
Pipe pipe,
const Names & column_names,
const StorageSnapshotPtr & storage_snapshot,
SelectQueryInfo & query_info,
ContextPtr context,
std::string storage_name)
{
2020-09-25 13:19:26 +00:00
if (pipe.empty())
{
auto header = storage_snapshot->getSampleBlockForColumns(column_names);
InterpreterSelectQuery::addEmptySourceToQueryPlan(query_plan, header, query_info, context);
2020-09-25 13:19:26 +00:00
}
else
{
2022-05-31 14:43:38 +00:00
auto read_step = std::make_unique<ReadFromStorageStep>(std::move(pipe), storage_name, query_info.storage_limits);
2020-09-25 13:19:26 +00:00
query_plan.addStep(std::move(read_step));
}
2020-09-14 14:13:58 +00:00
}
2022-05-20 19:49:31 +00:00
std::optional<QueryPipeline> IStorage::distributedWrite(
const ASTInsertQuery & /*query*/,
ContextPtr /*context*/)
{
return {};
}
2020-08-03 11:33:11 +00:00
Pipe IStorage::alterPartition(
const StorageMetadataPtr & /* metadata_snapshot */, const PartitionCommands & /* commands */, ContextPtr /* context */)
2020-08-03 11:33:11 +00:00
{
throw Exception("Partition operations are not supported by storage " + getName(), ErrorCodes::NOT_IMPLEMENTED);
}
2021-10-25 17:49:49 +00:00
void IStorage::alter(const AlterCommands & params, ContextPtr context, AlterLockHolder &)
2018-12-25 23:11:36 +00:00
{
auto table_id = getStorageID();
2020-06-10 09:09:51 +00:00
StorageInMemoryMetadata new_metadata = getInMemoryMetadata();
params.apply(new_metadata, context);
DatabaseCatalog::instance().getDatabase(table_id.database_name)->alterTable(context, table_id, new_metadata);
setInMemoryMetadata(new_metadata);
2019-12-26 18:17:05 +00:00
}
void IStorage::checkAlterIsPossible(const AlterCommands & commands, ContextPtr /* context */) const
2019-12-26 18:17:05 +00:00
{
for (const auto & command : commands)
{
2019-12-27 14:36:59 +00:00
if (!command.isCommentAlter())
throw Exception(ErrorCodes::NOT_IMPLEMENTED, "Alter of type '{}' is not supported by storage {}",
command.type, getName());
2018-12-25 23:11:36 +00:00
}
}
void IStorage::checkMutationIsPossible(const MutationCommands & /*commands*/, const Settings & /*settings*/) const
{
throw Exception("Table engine " + getName() + " doesn't support mutations", ErrorCodes::NOT_IMPLEMENTED);
}
2020-12-07 09:30:47 +00:00
void IStorage::checkAlterPartitionIsPossible(
const PartitionCommands & /*commands*/, const StorageMetadataPtr & /*metadata_snapshot*/, const Settings & /*settings*/) const
2020-07-13 16:19:08 +00:00
{
2020-07-13 17:27:52 +00:00
throw Exception("Table engine " + getName() + " doesn't support partitioning", ErrorCodes::NOT_IMPLEMENTED);
2020-07-13 16:19:08 +00:00
}
2019-09-13 12:59:48 +00:00
2019-12-27 19:30:22 +00:00
StorageID IStorage::getStorageID() const
2019-12-03 16:25:32 +00:00
{
std::lock_guard lock(id_mutex);
2019-12-12 12:30:31 +00:00
return storage_id;
2019-12-03 16:25:32 +00:00
}
2020-04-07 14:05:51 +00:00
void IStorage::renameInMemory(const StorageID & new_table_id)
2019-12-03 16:25:32 +00:00
{
std::lock_guard lock(id_mutex);
2020-04-07 14:05:51 +00:00
storage_id = new_table_id;
2019-12-03 16:25:32 +00:00
}
NamesAndTypesList IStorage::getVirtuals() const
2020-04-27 13:55:30 +00:00
{
return {};
2020-04-27 13:55:30 +00:00
}
2020-12-07 09:30:47 +00:00
Names IStorage::getAllRegisteredNames() const
{
Names result;
auto getter = [](const auto & column) { return column.name; };
const NamesAndTypesList & available_columns = getInMemoryMetadata().getColumns().getAllPhysical();
std::transform(available_columns.begin(), available_columns.end(), std::back_inserter(result), getter);
return result;
}
NameDependencies IStorage::getDependentViewsByColumn(ContextPtr context) const
{
NameDependencies name_deps;
auto dependencies = DatabaseCatalog::instance().getDependencies(storage_id);
for (const auto & depend_id : dependencies)
{
auto depend_table = DatabaseCatalog::instance().getTable(depend_id, context);
2021-02-28 07:42:08 +00:00
if (depend_table->getInMemoryMetadataPtr()->select.inner_query)
{
const auto & select_query = depend_table->getInMemoryMetadataPtr()->select.inner_query;
auto required_columns = InterpreterSelectQuery(select_query, context, SelectQueryOptions{}.noModify()).getRequiredColumns();
for (const auto & col_name : required_columns)
name_deps[col_name].push_back(depend_id.table_name);
}
}
return name_deps;
}
2021-09-04 09:02:07 +00:00
bool IStorage::isStaticStorage() const
2021-08-23 11:26:54 +00:00
{
auto storage_policy = getStoragePolicy();
if (storage_policy)
{
2021-08-24 07:30:33 +00:00
for (const auto & disk : storage_policy->getDisks())
if (!(disk->isReadOnly() || disk->isWriteOnce()))
2021-08-23 11:26:54 +00:00
return false;
return true;
}
return false;
}
void IStorage::adjustCreateQueryForBackup(ASTPtr &) const
2021-08-18 22:19:14 +00:00
{
}
void IStorage::backupData(BackupEntriesCollector &, const String &, const std::optional<ASTs> &)
{
2021-08-18 22:19:14 +00:00
}
void IStorage::restoreDataFromBackup(RestorerFromBackup & restorer, const String & data_path_in_backup, const std::optional<ASTs> &)
2021-08-18 22:19:14 +00:00
{
/// If an inherited class doesn't override restoreDataFromBackup() that means it doesn't backup any data.
auto filenames = restorer.getBackup()->listFiles(data_path_in_backup);
if (!filenames.empty())
throw Exception(ErrorCodes::CANNOT_RESTORE_TABLE, "Cannot restore table {}: Folder {} in backup must be empty",
getStorageID().getFullTableName(), data_path_in_backup);
2021-08-18 22:19:14 +00:00
}
2021-06-25 14:49:28 +00:00
std::string PrewhereInfo::dump() const
2020-12-02 18:16:31 +00:00
{
2020-12-18 17:13:28 +00:00
WriteBufferFromOwnString ss;
2020-12-02 18:16:31 +00:00
ss << "PrewhereDagInfo\n";
if (row_level_filter)
{
ss << "row_level_filter " << row_level_filter->dumpDAG() << "\n";
}
2020-12-02 18:16:31 +00:00
if (prewhere_actions)
{
ss << "prewhere_actions " << prewhere_actions->dumpDAG() << "\n";
}
ss << "remove_prewhere_column " << remove_prewhere_column
<< ", need_filter " << need_filter << "\n";
return ss.str();
}
std::string FilterDAGInfo::dump() const
2020-12-02 18:16:31 +00:00
{
2020-12-18 17:13:28 +00:00
WriteBufferFromOwnString ss;
ss << "FilterDAGInfo for column '" << column_name <<"', do_remove_column "
2020-12-02 18:16:31 +00:00
<< do_remove_column << "\n";
if (actions)
2020-12-02 18:16:31 +00:00
{
ss << "actions " << actions->dumpDAG() << "\n";
2020-12-02 18:16:31 +00:00
}
return ss.str();
2020-12-23 06:44:44 +00:00
}
2020-12-02 18:16:31 +00:00
}