thevar1able/ClickHouse
1
0
Fork 0
mirror of https://github.com/ClickHouse/ClickHouse.git synced 2024-09-21 09:10:48 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'master' of github.com:yandex/ClickHouse into BayoNet-feature/randomASKII-function

Browse Source
This commit is contained in:
Alexey Milovidov 2019-12-29 04:15:23 +03:00
commit 88bb7b48bf
26 changed files with 664 additions and 1330 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

123
dbms/src/Databases/DatabaseDictionary.cpp
View File

@ -23,12 +23,8 @@ namespace ErrorCodes
}
DatabaseDictionary::DatabaseDictionary(const String & name_)
: name(name_),
log(&Logger::get("DatabaseDictionary(" + name + ")"))
{
}
void DatabaseDictionary::loadStoredObjects(Context &, bool)
: IDatabase(name_),
log(&Logger::get("DatabaseDictionary(" + database_name + ")"))
{
}
@ -69,65 +65,6 @@ bool DatabaseDictionary::isTableExist(
return context.getExternalDictionariesLoader().getCurrentStatus(table_name) != ExternalLoader::Status::NOT_EXIST;
}
bool DatabaseDictionary::isDictionaryExist(
const Context & /*context*/,
const String & /*table_name*/) const
{
return false;
}
DatabaseDictionariesIteratorPtr DatabaseDictionary::getDictionariesIterator(
const Context & /*context*/,
const FilterByNameFunction & /*filter_by_dictionary_name*/)
{
return std::make_unique<DatabaseDictionariesSnapshotIterator>();
}
void DatabaseDictionary::createDictionary(
const Context & /*context*/,
const String & /*dictionary_name*/,
const ASTPtr & /*query*/)
{
throw Exception("Dictionary engine doesn't support dictionaries.", ErrorCodes::UNSUPPORTED_METHOD);
}
void DatabaseDictionary::removeDictionary(
const Context & /*context*/,
const String & /*table_name*/)
{
throw Exception("Dictionary engine doesn't support dictionaries.", ErrorCodes::UNSUPPORTED_METHOD);
}
void DatabaseDictionary::attachDictionary(
const String & /*dictionary_name*/, const Context & /*context*/)
{
throw Exception("Dictionary engine doesn't support dictionaries.", ErrorCodes::UNSUPPORTED_METHOD);
}
void DatabaseDictionary::detachDictionary(const String & /*dictionary_name*/, const Context & /*context*/)
{
throw Exception("Dictionary engine doesn't support dictionaries.", ErrorCodes::UNSUPPORTED_METHOD);
}
ASTPtr DatabaseDictionary::tryGetCreateDictionaryQuery(
const Context & /*context*/,
const String & /*table_name*/) const
{
return nullptr;
}
ASTPtr DatabaseDictionary::getCreateDictionaryQuery(
const Context & /*context*/,
const String & /*table_name*/) const
{
throw Exception("Dictionary engine doesn't support dictionaries.", ErrorCodes::UNSUPPORTED_METHOD);
}
StoragePtr DatabaseDictionary::tryGetTable(
const Context & context,
const String & table_name) const
@ -153,39 +90,6 @@ bool DatabaseDictionary::empty(const Context & context) const
return !context.getExternalDictionariesLoader().hasCurrentlyLoadedObjects();
}
StoragePtr DatabaseDictionary::detachTable(const String & /*table_name*/)
{
throw Exception("DatabaseDictionary: detachTable() is not supported", ErrorCodes::NOT_IMPLEMENTED);
}
void DatabaseDictionary::attachTable(const String & /*table_name*/, const StoragePtr & /*table*/)
{
throw Exception("DatabaseDictionary: attachTable() is not supported", ErrorCodes::NOT_IMPLEMENTED);
}
void DatabaseDictionary::createTable(
const Context &,
const String &,
const StoragePtr &,
const ASTPtr &)
{
throw Exception("DatabaseDictionary: createTable() is not supported", ErrorCodes::NOT_IMPLEMENTED);
}
void DatabaseDictionary::removeTable(
const Context &,
const String &)
{
throw Exception("DatabaseDictionary: removeTable() is not supported", ErrorCodes::NOT_IMPLEMENTED);
}
time_t DatabaseDictionary::getObjectMetadataModificationTime(
const Context &,
const String &)
{
return static_cast<time_t>(0);
}
ASTPtr DatabaseDictionary::getCreateTableQueryImpl(const Context & context,
const String & table_name, bool throw_on_error) const
{
@ -196,9 +100,11 @@ ASTPtr DatabaseDictionary::getCreateTableQueryImpl(const Context & context,
const auto & dictionaries = context.getExternalDictionariesLoader();
auto dictionary = throw_on_error ? dictionaries.getDictionary(table_name)
: dictionaries.tryGetDictionary(table_name);
if (!dictionary)
return {};
auto names_and_types = StorageDictionary::getNamesAndTypes(dictionary->getStructure());
buffer << "CREATE TABLE " << backQuoteIfNeed(name) << '.' << backQuoteIfNeed(table_name) << " (";
buffer << "CREATE TABLE " << backQuoteIfNeed(database_name) << '.' << backQuoteIfNeed(table_name) << " (";
buffer << StorageDictionary::generateNamesAndTypesDescription(names_and_types.begin(), names_and_types.end());
buffer << ") Engine = Dictionary(" << backQuoteIfNeed(table_name) << ")";
}
@ -215,22 +121,12 @@ ASTPtr DatabaseDictionary::getCreateTableQueryImpl(const Context & context,
return ast;
}
ASTPtr DatabaseDictionary::getCreateTableQuery(const Context & context, const String & table_name) const
{
return getCreateTableQueryImpl(context, table_name, true);
}
ASTPtr DatabaseDictionary::tryGetCreateTableQuery(const Context & context, const String & table_name) const
{
return getCreateTableQueryImpl(context, table_name, false);
}
ASTPtr DatabaseDictionary::getCreateDatabaseQuery(const Context & /*context*/) const
ASTPtr DatabaseDictionary::getCreateDatabaseQuery() const
{
String query;
{
WriteBufferFromString buffer(query);
buffer << "CREATE DATABASE " << backQuoteIfNeed(name) << " ENGINE = Dictionary";
buffer << "CREATE DATABASE " << backQuoteIfNeed(database_name) << " ENGINE = Dictionary";
}
ParserCreateQuery parser;
return parseQuery(parser, query.data(), query.data() + query.size(), "", 0);
@ -240,9 +136,4 @@ void DatabaseDictionary::shutdown()
{
}
String DatabaseDictionary::getDatabaseName() const
{
return name;
}
}

57
dbms/src/Databases/DatabaseDictionary.h
View File

@ -24,85 +24,36 @@ class DatabaseDictionary : public IDatabase
public:
DatabaseDictionary(const String & name_);
String getDatabaseName() const override;
String getEngineName() const override
{
return "Dictionary";
}
void loadStoredObjects(
Context & context,
bool has_force_restore_data_flag) override;
bool isTableExist(
const Context & context,
const String & table_name) const override;
bool isDictionaryExist(const Context & context, const String & table_name) const override;
StoragePtr tryGetTable(
const Context & context,
const String & table_name) const override;
DatabaseTablesIteratorPtr getTablesIterator(const Context & context, const FilterByNameFunction & filter_by_table_name = {}) override;
DatabaseDictionariesIteratorPtr getDictionariesIterator(const Context & context, const FilterByNameFunction & filter_by_dictionary_name = {}) override;
bool empty(const Context & context) const override;
void createTable(
const Context & context,
const String & table_name,
const StoragePtr & table,
const ASTPtr & query) override;
void createDictionary(
const Context & context, const String & dictionary_name, const ASTPtr & query) override;
void removeTable(
const Context & context,
const String & table_name) override;
void removeDictionary(const Context & context, const String & table_name) override;
void attachTable(const String & table_name, const StoragePtr & table) override;
StoragePtr detachTable(const String & table_name) override;
time_t getObjectMetadataModificationTime(
const Context & context,
const String & table_name) override;
ASTPtr getCreateTableQuery(
const Context & context,
const String & table_name) const override;
ASTPtr tryGetCreateTableQuery(
const Context & context,
const String & table_name) const override;
ASTPtr getCreateDatabaseQuery(const Context & context) const override;
ASTPtr getCreateDictionaryQuery(const Context & context, const String & table_name) const override;
ASTPtr tryGetCreateDictionaryQuery(const Context & context, const String & table_name) const override;
void attachDictionary(const String & dictionary_name, const Context & context) override;
void detachDictionary(const String & dictionary_name, const Context & context) override;
ASTPtr getCreateDatabaseQuery() const override;
void shutdown() override;
protected:
ASTPtr getCreateTableQueryImpl(const Context & context, const String & table_name, bool throw_on_error) const override;
private:
const String name;
mutable std::mutex mutex;
Poco::Logger * log;
Tables listTables(const Context & context, const FilterByNameFunction & filter_by_name);
ASTPtr getCreateTableQueryImpl(const Context & context, const String & table_name, bool throw_on_error) const;
};
}

154
dbms/src/Databases/DatabaseLazy.cpp
View File

@ -3,7 +3,6 @@
#include <Databases/DatabaseOnDisk.h>
#include <Databases/DatabasesCommon.h>
#include <Interpreters/Context.h>
#include <IO/ReadBufferFromFile.h>
#include <IO/ReadHelpers.h>
#include <IO/WriteBufferFromFile.h>
#include <IO/WriteHelpers.h>
@ -31,11 +30,8 @@ namespace ErrorCodes
DatabaseLazy::DatabaseLazy(const String & name_, const String & metadata_path_, time_t expiration_time_, const Context & context_)
: name(name_)
, metadata_path(metadata_path_)
, data_path("data/" + escapeForFileName(name) + "/")
: DatabaseOnDisk(name_, metadata_path_, "DatabaseLazy (" + name_ + ")")
, expiration_time(expiration_time_)
, log(&Logger::get("DatabaseLazy (" + name + ")"))
{
Poco::File(context_.getPath() + getDataPath()).createDirectories();
}
@ -45,7 +41,7 @@ void DatabaseLazy::loadStoredObjects(
Context & context,
bool /* has_force_restore_data_flag */)
{
DatabaseOnDisk::iterateMetadataFiles(*this, log, context, [this](const String & file_name)
iterateMetadataFiles(context, [this](const String & file_name)
{
const std::string table_name = file_name.substr(0, file_name.size() - 4);
attachTable(table_name, nullptr);
@ -62,75 +58,21 @@ void DatabaseLazy::createTable(
SCOPE_EXIT({ clearExpiredTables(); });
if (!endsWith(table->getName(), "Log"))
throw Exception("Lazy engine can be used only with *Log tables.", ErrorCodes::UNSUPPORTED_METHOD);
DatabaseOnDisk::createTable(*this, context, table_name, table, query);
DatabaseOnDisk::createTable(context, table_name, table, query);
/// DatabaseOnDisk::createTable renames file, so we need to get new metadata_modification_time.
std::lock_guard lock(tables_mutex);
std::lock_guard lock(mutex);
auto it = tables_cache.find(table_name);
if (it != tables_cache.end())
it->second.metadata_modification_time = DatabaseOnDisk::getObjectMetadataModificationTime(*this, table_name);
it->second.metadata_modification_time = DatabaseOnDisk::getObjectMetadataModificationTime(table_name);
}
void DatabaseLazy::createDictionary(
const Context & /*context*/,
const String & /*dictionary_name*/,
const ASTPtr & /*query*/)
{
throw Exception("Lazy engine can be used only with *Log tables.", ErrorCodes::UNSUPPORTED_METHOD);
}
void DatabaseLazy::removeTable(
const Context & context,
const String & table_name)
{
SCOPE_EXIT({ clearExpiredTables(); });
DatabaseOnDisk::removeTable(*this, context, table_name, log);
}
void DatabaseLazy::removeDictionary(
const Context & /*context*/,
const String & /*table_name*/)
{
throw Exception("Lazy engine can be used only with *Log tables.", ErrorCodes::UNSUPPORTED_METHOD);
}
ASTPtr DatabaseLazy::getCreateDictionaryQuery(
const Context & /*context*/,
const String & /*table_name*/) const
{
throw Exception("Lazy engine can be used only with *Log tables.", ErrorCodes::UNSUPPORTED_METHOD);
}
ASTPtr DatabaseLazy::tryGetCreateDictionaryQuery(const Context & /*context*/, const String & /*table_name*/) const
{
return nullptr;
}
bool DatabaseLazy::isDictionaryExist(const Context & /*context*/, const String & /*table_name*/) const
{
return false;
}
DatabaseDictionariesIteratorPtr DatabaseLazy::getDictionariesIterator(
const Context & /*context*/,
const FilterByNameFunction & /*filter_by_dictionary_name*/)
{
return std::make_unique<DatabaseDictionariesSnapshotIterator>();
}
void DatabaseLazy::attachDictionary(
const String & /*dictionary_name*/,
const Context & /*context*/)
{
throw Exception("Lazy engine can be used only with *Log tables.", ErrorCodes::UNSUPPORTED_METHOD);
}
void DatabaseLazy::detachDictionary(const String & /*dictionary_name*/, const Context & /*context*/)
{
throw Exception("Lazy engine can be used only with *Log tables.", ErrorCodes::UNSUPPORTED_METHOD);
DatabaseOnDisk::removeTable(context, table_name);
}
void DatabaseLazy::renameTable(
@ -141,35 +83,17 @@ void DatabaseLazy::renameTable(
TableStructureWriteLockHolder & lock)
{
SCOPE_EXIT({ clearExpiredTables(); });
DatabaseOnDisk::renameTable<DatabaseLazy>(*this, context, table_name, to_database, to_table_name, lock);
DatabaseOnDisk::renameTable(context, table_name, to_database, to_table_name, lock);
}
time_t DatabaseLazy::getObjectMetadataModificationTime(
const Context & /* context */,
const String & table_name)
time_t DatabaseLazy::getObjectMetadataModificationTime(const String & table_name) const
{
std::lock_guard lock(tables_mutex);
std::lock_guard lock(mutex);
auto it = tables_cache.find(table_name);
if (it != tables_cache.end())
return it->second.metadata_modification_time;
else
throw Exception("Table " + backQuote(getDatabaseName()) + "." + backQuote(table_name) + " doesn't exist.", ErrorCodes::UNKNOWN_TABLE);
}
ASTPtr DatabaseLazy::getCreateTableQuery(const Context & context, const String & table_name) const
{
return DatabaseOnDisk::getCreateTableQuery(*this, context, table_name);
}
ASTPtr DatabaseLazy::tryGetCreateTableQuery(const Context & context, const String & table_name) const
{
return DatabaseOnDisk::tryGetCreateTableQuery(*this, context, table_name);
}
ASTPtr DatabaseLazy::getCreateDatabaseQuery(const Context & context) const
{
return DatabaseOnDisk::getCreateDatabaseQuery(*this, context);
throw Exception("Table " + backQuote(getDatabaseName()) + "." + backQuote(table_name) + " doesn't exist.", ErrorCodes::UNKNOWN_TABLE);
}
void DatabaseLazy::alterTable(
@ -177,22 +101,16 @@ void DatabaseLazy::alterTable(
const String & /* table_name */,
const StorageInMemoryMetadata & /* metadata */)
{
SCOPE_EXIT({ clearExpiredTables(); });
clearExpiredTables();
throw Exception("ALTER query is not supported for Lazy database.", ErrorCodes::UNSUPPORTED_METHOD);
}
void DatabaseLazy::drop(const Context & context)
{
DatabaseOnDisk::drop(*this, context);
}
bool DatabaseLazy::isTableExist(
const Context & /* context */,
const String & table_name) const
{
SCOPE_EXIT({ clearExpiredTables(); });
std::lock_guard lock(tables_mutex);
std::lock_guard lock(mutex);
return tables_cache.find(table_name) != tables_cache.end();
}
@ -202,7 +120,7 @@ StoragePtr DatabaseLazy::tryGetTable(
{
SCOPE_EXIT({ clearExpiredTables(); });
{
std::lock_guard lock(tables_mutex);
std::lock_guard lock(mutex);
auto it = tables_cache.find(table_name);
if (it == tables_cache.end())
throw Exception("Table " + backQuote(getDatabaseName()) + "." + backQuote(table_name) + " doesn't exist.", ErrorCodes::UNKNOWN_TABLE);
@ -222,7 +140,7 @@ StoragePtr DatabaseLazy::tryGetTable(
DatabaseTablesIteratorPtr DatabaseLazy::getTablesIterator(const Context & context, const FilterByNameFunction & filter_by_table_name)
{
std::lock_guard lock(tables_mutex);
std::lock_guard lock(mutex);
Strings filtered_tables;
for (const auto & [table_name, cached_table] : tables_cache)
{
@ -241,12 +159,12 @@ bool DatabaseLazy::empty(const Context & /* context */) const
void DatabaseLazy::attachTable(const String & table_name, const StoragePtr & table)
{
LOG_DEBUG(log, "Attach table " << backQuote(table_name) << ".");
std::lock_guard lock(tables_mutex);
std::lock_guard lock(mutex);
time_t current_time = std::chrono::system_clock::to_time_t(std::chrono::system_clock::now());
auto [it, inserted] = tables_cache.emplace(std::piecewise_construct,
std::forward_as_tuple(table_name),
std::forward_as_tuple(table, current_time, DatabaseOnDisk::getObjectMetadataModificationTime(*this, table_name)));
std::forward_as_tuple(table, current_time, DatabaseOnDisk::getObjectMetadataModificationTime(table_name)));
if (!inserted)
throw Exception("Table " + backQuote(getDatabaseName()) + "." + backQuote(table_name) + " already exists.", ErrorCodes::TABLE_ALREADY_EXISTS);
@ -258,7 +176,7 @@ StoragePtr DatabaseLazy::detachTable(const String & table_name)
StoragePtr res;
{
LOG_DEBUG(log, "Detach table " << backQuote(table_name) << ".");
std::lock_guard lock(tables_mutex);
std::lock_guard lock(mutex);
auto it = tables_cache.find(table_name);
if (it == tables_cache.end())
throw Exception("Table " + backQuote(getDatabaseName()) + "." + backQuote(table_name) + " doesn't exist.", ErrorCodes::UNKNOWN_TABLE);
@ -274,7 +192,7 @@ void DatabaseLazy::shutdown()
{
TablesCache tables_snapshot;
{
std::lock_guard lock(tables_mutex);
std::lock_guard lock(mutex);
tables_snapshot = tables_cache;
}
@ -284,7 +202,7 @@ void DatabaseLazy::shutdown()
kv.second.table->shutdown();
}
std::lock_guard lock(tables_mutex);
std::lock_guard lock(mutex);
tables_cache.clear();
}
@ -300,26 +218,6 @@ DatabaseLazy::~DatabaseLazy()
}
}
String DatabaseLazy::getDataPath() const
{
return data_path;
}
String DatabaseLazy::getMetadataPath() const
{
return metadata_path;
}
String DatabaseLazy::getDatabaseName() const
{
return name;
}
String DatabaseLazy::getObjectMetadataPath(const String & table_name) const
{
return DatabaseOnDisk::getObjectMetadataPath(*this, table_name);
}
StoragePtr DatabaseLazy::loadTable(const Context & context, const String & table_name) const
{
SCOPE_EXIT({ clearExpiredTables(); });
@ -330,19 +228,21 @@ StoragePtr DatabaseLazy::loadTable(const Context & context, const String & table
try
{
String table_name_;
StoragePtr table;
Context context_copy(context); /// some tables can change context, but not LogTables
auto ast = parseCreateQueryFromMetadataFile(table_metadata_path, log);
auto ast = parseQueryFromMetadata(table_metadata_path, /*throw_on_error*/ true, /*remove_empty*/false);
if (ast)
std::tie(table_name_, table) = createTableFromAST(
ast->as<const ASTCreateQuery &>(), name, getDataPath(), context_copy, false);
{
auto & ast_create = ast->as<const ASTCreateQuery &>();
String table_data_path_relative = getTableDataPath(ast_create);
table = createTableFromAST(ast_create, database_name, table_data_path_relative, context_copy, false).second;
}
if (!ast || !endsWith(table->getName(), "Log"))
throw Exception("Only *Log tables can be used with Lazy database engine.", ErrorCodes::LOGICAL_ERROR);
{
std::lock_guard lock(tables_mutex);
std::lock_guard lock(mutex);
auto it = tables_cache.find(table_name);
if (it == tables_cache.end())
throw Exception("Table " + backQuote(getDatabaseName()) + "." + backQuote(table_name) + " doesn't exist.", ErrorCodes::UNKNOWN_TABLE);
@ -364,7 +264,7 @@ StoragePtr DatabaseLazy::loadTable(const Context & context, const String & table
void DatabaseLazy::clearExpiredTables() const
{
std::lock_guard lock(tables_mutex);
std::lock_guard lock(mutex);
auto time_now = std::chrono::system_clock::to_time_t(std::chrono::system_clock::now());
CacheExpirationQueue expired_tables;

61
dbms/src/Databases/DatabaseLazy.h
View File

@ -1,6 +1,6 @@
#pragma once
#include <Databases/DatabasesCommon.h>
#include <Databases/DatabaseOnDisk.h>
#include <Interpreters/Context.h>
#include <Parsers/ASTCreateQuery.h>
@ -15,7 +15,7 @@ class DatabaseLazyIterator;
* Works like DatabaseOrdinary, but stores in memory only cache.
* Can be used only with *Log engines.
*/
class DatabaseLazy : public IDatabase
class DatabaseLazy : public DatabaseOnDisk
{
public:
DatabaseLazy(const String & name_, const String & metadata_path_, time_t expiration_time_, const Context & context_);
@ -32,19 +32,10 @@ public:
const StoragePtr & table,
const ASTPtr & query) override;
void createDictionary(
const Context & context,
const String & dictionary_name,
const ASTPtr & query) override;
void removeTable(
const Context & context,
const String & table_name) override;
void removeDictionary(
const Context & context,
const String & table_name) override;
void renameTable(
const Context & context,
const String & table_name,
@ -57,43 +48,12 @@ public:
const String & name,
const StorageInMemoryMetadata & metadata) override;
time_t getObjectMetadataModificationTime(
const Context & context,
const String & table_name) override;
ASTPtr getCreateTableQuery(
const Context & context,
const String & table_name) const override;
ASTPtr tryGetCreateTableQuery(
const Context & context,
const String & table_name) const override;
ASTPtr getCreateDictionaryQuery(
const Context & context,
const String & dictionary_name) const override;
ASTPtr tryGetCreateDictionaryQuery(
const Context & context,
const String & dictionary_name) const override;
ASTPtr getCreateDatabaseQuery(const Context & context) const override;
String getDataPath() const override;
String getDatabaseName() const override;
String getMetadataPath() const override;
String getObjectMetadataPath(const String & table_name) const override;
void drop(const Context & context) override;
time_t getObjectMetadataModificationTime(const String & table_name) const override;
bool isTableExist(
const Context & context,
const String & table_name) const override;
bool isDictionaryExist(
const Context & context,
const String & table_name) const override;
StoragePtr tryGetTable(
const Context & context,
const String & table_name) const override;
@ -102,16 +62,10 @@ public:
DatabaseTablesIteratorPtr getTablesIterator(const Context & context, const FilterByNameFunction & filter_by_table_name = {}) override;
DatabaseDictionariesIteratorPtr getDictionariesIterator(const Context & context, const FilterByNameFunction & filter_by_dictionary_name = {}) override;
void attachTable(const String & table_name, const StoragePtr & table) override;
StoragePtr detachTable(const String & table_name) override;
void attachDictionary(const String & dictionary_name, const Context & context) override;
void detachDictionary(const String & dictionary_name, const Context & context) override;
void shutdown() override;
~DatabaseLazy() override;
@ -143,19 +97,12 @@ private:
using TablesCache = std::unordered_map<String, CachedTable>;
String name;
const String metadata_path;
const String data_path;
const time_t expiration_time;
mutable std::mutex tables_mutex;
/// TODO use DatabaseWithOwnTablesBase::tables
mutable TablesCache tables_cache;
mutable CacheExpirationQueue cache_expiration_queue;
Poco::Logger * log;
StoragePtr loadTable(const Context & context, const String & table_name) const;
void clearExpiredTables() const;

86
dbms/src/Databases/DatabaseMemory.cpp
View File

@ -1,30 +1,16 @@
#include <common/logger_useful.h>
#include <Databases/DatabaseMemory.h>
#include <Databases/DatabasesCommon.h>
#include <Parsers/ASTCreateQuery.h>
namespace DB
{
namespace ErrorCodes
{
extern const int CANNOT_GET_CREATE_TABLE_QUERY;
extern const int CANNOT_GET_CREATE_DICTIONARY_QUERY;
extern const int UNSUPPORTED_METHOD;
}
DatabaseMemory::DatabaseMemory(String name_)
: DatabaseWithOwnTablesBase(std::move(name_))
, log(&Logger::get("DatabaseMemory(" + name + ")"))
DatabaseMemory::DatabaseMemory(const String & name_)
: DatabaseWithOwnTablesBase(name_, "DatabaseMemory(" + name_ + ")")
{}
void DatabaseMemory::loadStoredObjects(
Context & /*context*/,
bool /*has_force_restore_data_flag*/)
{
/// Nothing to load.
}
void DatabaseMemory::createTable(
const Context & /*context*/,
const String & table_name,
@ -34,21 +20,6 @@ void DatabaseMemory::createTable(
attachTable(table_name, table);
}
void DatabaseMemory::attachDictionary(const String & /*name*/, const Context & /*context*/)
{
throw Exception("There is no ATTACH DICTIONARY query for DatabaseMemory", ErrorCodes::UNSUPPORTED_METHOD);
}
void DatabaseMemory::createDictionary(
const Context & /*context*/,
const String & /*dictionary_name*/,
const ASTPtr & /*query*/)
{
throw Exception("There is no CREATE DICTIONARY query for DatabaseMemory", ErrorCodes::UNSUPPORTED_METHOD);
}
void DatabaseMemory::removeTable(
const Context & /*context*/,
const String & table_name)
@ -56,52 +27,13 @@ void DatabaseMemory::removeTable(
detachTable(table_name);
}
void DatabaseMemory::detachDictionary(const String & /*name*/, const Context & /*context*/)
ASTPtr DatabaseMemory::getCreateDatabaseQuery() const
{
throw Exception("There is no DETACH DICTIONARY query for DatabaseMemory", ErrorCodes::UNSUPPORTED_METHOD);
}
void DatabaseMemory::removeDictionary(
const Context & /*context*/,
const String & /*dictionary_name*/)
{
throw Exception("There is no DROP DICTIONARY query for DatabaseMemory", ErrorCodes::UNSUPPORTED_METHOD);
}
time_t DatabaseMemory::getObjectMetadataModificationTime(
const Context &, const String &)
{
return static_cast<time_t>(0);
}
ASTPtr DatabaseMemory::getCreateTableQuery(
const Context &,
const String &) const
{
throw Exception("There is no CREATE TABLE query for DatabaseMemory tables", ErrorCodes::CANNOT_GET_CREATE_TABLE_QUERY);
}
ASTPtr DatabaseMemory::getCreateDictionaryQuery(
const Context &,
const String &) const
{
throw Exception("There is no CREATE DICTIONARY query for DatabaseMemory dictionaries", ErrorCodes::CANNOT_GET_CREATE_DICTIONARY_QUERY);
}
ASTPtr DatabaseMemory::getCreateDatabaseQuery(
const Context &) const
{
throw Exception("There is no CREATE DATABASE query for DatabaseMemory", ErrorCodes::CANNOT_GET_CREATE_TABLE_QUERY);
}
String DatabaseMemory::getDatabaseName() const
{
return name;
auto create_query = std::make_shared<ASTCreateQuery>();
create_query->database = database_name;
create_query->set(create_query->storage, std::make_shared<ASTStorage>());
create_query->storage->set(create_query->storage->engine, makeASTFunction(getEngineName()));
return create_query;
}
}

37
dbms/src/Databases/DatabaseMemory.h
View File

@ -17,54 +17,21 @@ namespace DB
class DatabaseMemory : public DatabaseWithOwnTablesBase
{
public:
DatabaseMemory(String name_);
String getDatabaseName() const override;
DatabaseMemory(const String & name_);
String getEngineName() const override { return "Memory"; }
void loadStoredObjects(
Context & context,
bool has_force_restore_data_flag) override;
void createTable(
const Context & context,
const String & table_name,
const StoragePtr & table,
const ASTPtr & query) override;
void createDictionary(
const Context & context,
const String & dictionary_name,
const ASTPtr & query) override;
void attachDictionary(
const String & name,
const Context & context) override;
void removeTable(
const Context & context,
const String & table_name) override;
void removeDictionary(
const Context & context,
const String & dictionary_name) override;
void detachDictionary(
const String & name,
const Context & context) override;
time_t getObjectMetadataModificationTime(const Context & context, const String & table_name) override;
ASTPtr getCreateTableQuery(const Context & context, const String & table_name) const override;
ASTPtr getCreateDictionaryQuery(const Context & context, const String & table_name) const override;
ASTPtr tryGetCreateTableQuery(const Context &, const String &) const override { return nullptr; }
ASTPtr tryGetCreateDictionaryQuery(const Context &, const String &) const override { return nullptr; }
ASTPtr getCreateDatabaseQuery(const Context & context) const override;
private:
Poco::Logger * log;
ASTPtr getCreateDatabaseQuery() const override;
};
}

23
dbms/src/Databases/DatabaseMySQL.cpp
View File

@ -9,10 +9,8 @@
#include <Formats/MySQLBlockInputStream.h>
#include <DataTypes/DataTypeString.h>
#include <DataTypes/DataTypesNumber.h>
#include <DataTypes/DataTypeDate.h>
#include <DataTypes/DataTypeDateTime.h>
#include <DataTypes/DataTypeNullable.h>
#include <DataTypes/DataTypeFixedString.h>
#include <Storages/StorageMySQL.h>
#include <Parsers/ASTFunction.h>
#include <Parsers/ParserCreateQuery.h>
@ -63,8 +61,12 @@ static String toQueryStringWithQuote(const std::vector<String> & quote_list)
DatabaseMySQL::DatabaseMySQL(
const Context & global_context_, const String & database_name_, const String & metadata_path_,
const ASTStorage * database_engine_define_, const String & database_name_in_mysql_, mysqlxx::Pool && pool)
: global_context(global_context_), database_name(database_name_), metadata_path(metadata_path_),
database_engine_define(database_engine_define_->clone()), database_name_in_mysql(database_name_in_mysql_), mysql_pool(std::move(pool))
: IDatabase(database_name_)
, global_context(global_context_)
, metadata_path(metadata_path_)
, database_engine_define(database_engine_define_->clone())
, database_name_in_mysql(database_name_in_mysql_)
, mysql_pool(std::move(pool))
{
}
@ -150,19 +152,24 @@ static ASTPtr getCreateQueryFromStorage(const StoragePtr & storage, const ASTPtr
return create_table_query;
}
ASTPtr DatabaseMySQL::tryGetCreateTableQuery(const Context &, const String & table_name) const
ASTPtr DatabaseMySQL::getCreateTableQueryImpl(const Context &, const String & table_name, bool throw_on_error) const
{
std::lock_guard<std::mutex> lock(mutex);
fetchTablesIntoLocalCache();
if (local_tables_cache.find(table_name) == local_tables_cache.end())
throw Exception("MySQL table " + database_name_in_mysql + "." + table_name + " doesn't exist.", ErrorCodes::UNKNOWN_TABLE);
{
if (throw_on_error)
throw Exception("MySQL table " + database_name_in_mysql + "." + table_name + " doesn't exist..",
ErrorCodes::UNKNOWN_TABLE);
return nullptr;
}
return getCreateQueryFromStorage(local_tables_cache[table_name].second, database_engine_define);
}
time_t DatabaseMySQL::getObjectMetadataModificationTime(const Context &, const String & table_name)
time_t DatabaseMySQL::getObjectMetadataModificationTime(const String & table_name) const
{
std::lock_guard<std::mutex> lock(mutex);
@ -174,7 +181,7 @@ time_t DatabaseMySQL::getObjectMetadataModificationTime(const Context &, const S
return time_t(local_tables_cache[table_name].first);
}
ASTPtr DatabaseMySQL::getCreateDatabaseQuery(const Context &) const
ASTPtr DatabaseMySQL::getCreateDatabaseQuery() const
{
const auto & create_query = std::make_shared<ASTCreateQuery>();
create_query->database = database_name;

43
dbms/src/Databases/DatabaseMySQL.h
View File

@ -28,35 +28,17 @@ public:
String getEngineName() const override { return "MySQL"; }
String getDatabaseName() const override { return database_name; }
bool empty(const Context & context) const override;
DatabaseTablesIteratorPtr getTablesIterator(const Context & context, const FilterByNameFunction & filter_by_table_name = {}) override;
DatabaseDictionariesIteratorPtr getDictionariesIterator(const Context &, const FilterByNameFunction & = {}) override
{
return std::make_unique<DatabaseDictionariesSnapshotIterator>();
}
ASTPtr getCreateDatabaseQuery(const Context & context) const override;
ASTPtr getCreateDatabaseQuery() const override;
bool isTableExist(const Context & context, const String & name) const override;
bool isDictionaryExist(const Context &, const String &) const override { return false; }
StoragePtr tryGetTable(const Context & context, const String & name) const override;
ASTPtr tryGetCreateTableQuery(const Context & context, const String & name) const override;
ASTPtr getCreateDictionaryQuery(const Context &, const String &) const override
{
throw Exception("MySQL database engine does not support dictionaries.", ErrorCodes::NOT_IMPLEMENTED);
}
ASTPtr tryGetCreateDictionaryQuery(const Context &, const String &) const override { return nullptr; }
time_t getObjectMetadataModificationTime(const Context & context, const String & name) override;
time_t getObjectMetadataModificationTime(const String & name) const override;
void shutdown() override;
@ -74,29 +56,12 @@ public:
void attachTable(const String & table_name, const StoragePtr & storage) override;
void detachDictionary(const String &, const Context &) override
{
throw Exception("MySQL database engine does not support detach dictionary.", ErrorCodes::NOT_IMPLEMENTED);
}
void removeDictionary(const Context &, const String &) override
{
throw Exception("MySQL database engine does not support remove dictionary.", ErrorCodes::NOT_IMPLEMENTED);
}
void attachDictionary(const String &, const Context &) override
{
throw Exception("MySQL database engine does not support attach dictionary.", ErrorCodes::NOT_IMPLEMENTED);
}
void createDictionary(const Context &, const String &, const ASTPtr &) override
{
throw Exception("MySQL database engine does not support create dictionary.", ErrorCodes::NOT_IMPLEMENTED);
}
protected:
ASTPtr getCreateTableQueryImpl(const Context & context, const String & name, bool throw_on_error) const override;
private:
Context global_context;
String database_name;
String metadata_path;
ASTPtr database_engine_define;
String database_name_in_mysql;

416
dbms/src/Databases/DatabaseOnDisk.cpp
View File

@ -6,9 +6,6 @@
#include <IO/WriteHelpers.h>
#include <Interpreters/Context.h>
#include <Interpreters/InterpreterCreateQuery.h>
#include <Interpreters/ExternalDictionariesLoader.h>
#include <Interpreters/ExternalLoaderPresetConfigRepository.h>
#include <Dictionaries/getDictionaryConfigurationFromAST.h>
#include <Parsers/ASTCreateQuery.h>
#include <Parsers/ParserCreateQuery.h>
#include <Parsers/formatAST.h>
@ -19,7 +16,6 @@
#include <Common/escapeForFileName.h>
#include <common/logger_useful.h>
#include <ext/scope_guard.h>
#include <Poco/DirectoryIterator.h>
@ -31,8 +27,6 @@ static constexpr size_t METADATA_FILE_BUFFER_SIZE = 32768;
namespace ErrorCodes
{
extern const int CANNOT_GET_CREATE_TABLE_QUERY;
extern const int CANNOT_GET_CREATE_DICTIONARY_QUERY;
extern const int FILE_DOESNT_EXIST;
extern const int INCORRECT_FILE_NAME;
extern const int SYNTAX_ERROR;
@ -43,93 +37,10 @@ namespace ErrorCodes
}
namespace detail
{
String getObjectMetadataPath(const String & base_path, const String & table_name)
{
return base_path + (endsWith(base_path, "/") ? "" : "/") + escapeForFileName(table_name) + ".sql";
}
String getDatabaseMetadataPath(const String & base_path)
{
return (endsWith(base_path, "/") ? base_path.substr(0, base_path.size() - 1) : base_path) + ".sql";
}
ASTPtr getQueryFromMetadata(const String & metadata_path, bool throw_on_error)
{
String query;
try
{
ReadBufferFromFile in(metadata_path, 4096);
readStringUntilEOF(query, in);
}
catch (const Exception & e)
{
if (!throw_on_error && e.code() == ErrorCodes::FILE_DOESNT_EXIST)
return nullptr;
else
throw;
}
ParserCreateQuery parser;
const char * pos = query.data();
std::string error_message;
auto ast = tryParseQuery(parser, pos, pos + query.size(), error_message, /* hilite = */ false,
"in file " + metadata_path, /* allow_multi_statements = */ false, 0);
if (!ast && throw_on_error)
throw Exception(error_message, ErrorCodes::SYNTAX_ERROR);
return ast;
}
ASTPtr getCreateQueryFromMetadata(const String & metadata_path, const String & database, bool throw_on_error)
{
ASTPtr ast = getQueryFromMetadata(metadata_path, throw_on_error);
if (ast)
{
auto & ast_create_query = ast->as<ASTCreateQuery &>();
ast_create_query.attach = false;
ast_create_query.database = database;
}
return ast;
}
}
ASTPtr parseCreateQueryFromMetadataFile(const String & filepath, Poco::Logger * log)
{
String definition;
{
char in_buf[METADATA_FILE_BUFFER_SIZE];
ReadBufferFromFile in(filepath, METADATA_FILE_BUFFER_SIZE, -1, in_buf);
readStringUntilEOF(definition, in);
}
/** Empty files with metadata are generated after a rough restart of the server.
* Remove these files to slightly reduce the work of the admins on startup.
*/
if (definition.empty())
{
LOG_ERROR(log, "File " << filepath << " is empty. Removing.");
Poco::File(filepath).remove();
return nullptr;
}
ParserCreateQuery parser_create;
ASTPtr result = parseQuery(parser_create, definition, "in file " + filepath, 0);
return result;
}
std::pair<String, StoragePtr> createTableFromAST(
ASTCreateQuery ast_create_query,
const String & database_name,
const String & database_data_path_relative,
const String & table_data_path_relative,
Context & context,
bool has_force_restore_data_flag)
{
@ -144,7 +55,7 @@ std::pair<String, StoragePtr> createTableFromAST(
return {ast_create_query.table, storage};
}
/// We do not directly use `InterpreterCreateQuery::execute`, because
/// - the database has not been created yet;
/// - the database has not been loaded yet;
/// - the code is simpler, since the query is already brought to a suitable form.
if (!ast_create_query.columns_list || !ast_create_query.columns_list->columns)
throw Exception("Missing definition of columns.", ErrorCodes::EMPTY_LIST_OF_COLUMNS_PASSED);
@ -152,7 +63,6 @@ std::pair<String, StoragePtr> createTableFromAST(
ColumnsDescription columns = InterpreterCreateQuery::getColumnsDescription(*ast_create_query.columns_list->columns, context);
ConstraintsDescription constraints = InterpreterCreateQuery::getConstraintsDescription(ast_create_query.columns_list->constraints);
String table_data_path_relative = database_data_path_relative + escapeForFileName(ast_create_query.table) + '/';
return
{
ast_create_query.table,
@ -202,7 +112,6 @@ String getObjectDefinitionFromCreateQuery(const ASTPtr & query)
}
void DatabaseOnDisk::createTable(
IDatabase & database,
const Context & context,
const String & table_name,
const StoragePtr & table,
@ -222,14 +131,14 @@ void DatabaseOnDisk::createTable(
/// A race condition would be possible if a table with the same name is simultaneously created using CREATE and using ATTACH.
/// But there is protection from it - see using DDLGuard in InterpreterCreateQuery.
if (database.isDictionaryExist(context, table_name))
throw Exception("Dictionary " + backQuote(database.getDatabaseName()) + "." + backQuote(table_name) + " already exists.",
if (isDictionaryExist(context, table_name))
throw Exception("Dictionary " + backQuote(getDatabaseName()) + "." + backQuote(table_name) + " already exists.",
ErrorCodes::DICTIONARY_ALREADY_EXISTS);
if (database.isTableExist(context, table_name))
throw Exception("Table " + backQuote(database.getDatabaseName()) + "." + backQuote(table_name) + " already exists.", ErrorCodes::TABLE_ALREADY_EXISTS);
if (isTableExist(context, table_name))
throw Exception("Table " + backQuote(getDatabaseName()) + "." + backQuote(table_name) + " already exists.", ErrorCodes::TABLE_ALREADY_EXISTS);
String table_metadata_path = database.getObjectMetadataPath(table_name);
String table_metadata_path = getObjectMetadataPath(table_name);
String table_metadata_tmp_path = table_metadata_path + ".tmp";
String statement;
@ -248,7 +157,7 @@ void DatabaseOnDisk::createTable(
try
{
/// Add a table to the map of known tables.
database.attachTable(table_name, table);
attachTable(table_name, table);
/// If it was ATTACH query and file with table metadata already exist
/// (so, ATTACH is done after DETACH), then rename atomically replaces old file with new one.
@ -261,107 +170,11 @@ void DatabaseOnDisk::createTable(
}
}
void DatabaseOnDisk::createDictionary(
IDatabase & database,
const Context & context,
const String & dictionary_name,
const ASTPtr & query)
void DatabaseOnDisk::removeTable(const Context & /* context */, const String & table_name)
{
const auto & settings = context.getSettingsRef();
StoragePtr res = detachTable(table_name);
/** The code is based on the assumption that all threads share the same order of operations:
* - create the .sql.tmp file;
* - add the dictionary to ExternalDictionariesLoader;
* - load the dictionary in case dictionaries_lazy_load == false;
* - attach the dictionary;
* - rename .sql.tmp to .sql.
*/
/// A race condition would be possible if a dictionary with the same name is simultaneously created using CREATE and using ATTACH.
/// But there is protection from it - see using DDLGuard in InterpreterCreateQuery.
if (database.isDictionaryExist(context, dictionary_name))
throw Exception("Dictionary " + backQuote(database.getDatabaseName()) + "." + backQuote(dictionary_name) + " already exists.", ErrorCodes::DICTIONARY_ALREADY_EXISTS);
/// A dictionary with the same full name could be defined in *.xml config files.
String full_name = database.getDatabaseName() + "." + dictionary_name;
auto & external_loader = const_cast<ExternalDictionariesLoader &>(context.getExternalDictionariesLoader());
if (external_loader.getCurrentStatus(full_name) != ExternalLoader::Status::NOT_EXIST)
throw Exception(
"Dictionary " + backQuote(database.getDatabaseName()) + "." + backQuote(dictionary_name) + " already exists.",
ErrorCodes::DICTIONARY_ALREADY_EXISTS);
if (database.isTableExist(context, dictionary_name))
throw Exception("Table " + backQuote(database.getDatabaseName()) + "." + backQuote(dictionary_name) + " already exists.", ErrorCodes::TABLE_ALREADY_EXISTS);
String dictionary_metadata_path = database.getObjectMetadataPath(dictionary_name);
String dictionary_metadata_tmp_path = dictionary_metadata_path + ".tmp";
String statement = getObjectDefinitionFromCreateQuery(query);
{
/// Exclusive flags guarantees, that table is not created right now in another thread. Otherwise, exception will be thrown.
WriteBufferFromFile out(dictionary_metadata_tmp_path, statement.size(), O_WRONLY | O_CREAT | O_EXCL);
writeString(statement, out);
out.next();
if (settings.fsync_metadata)
out.sync();
out.close();
}
bool succeeded = false;
SCOPE_EXIT({
if (!succeeded)
Poco::File(dictionary_metadata_tmp_path).remove();
});
/// Add a temporary repository containing the dictionary.
/// We need this temp repository to try loading the dictionary before actually attaching it to the database.
static std::atomic<size_t> counter = 0;
String temp_repository_name = String(IExternalLoaderConfigRepository::INTERNAL_REPOSITORY_NAME_PREFIX) + " creating " + full_name + " "
+ std::to_string(++counter);
external_loader.addConfigRepository(
temp_repository_name,
std::make_unique<ExternalLoaderPresetConfigRepository>(
std::vector{std::pair{dictionary_metadata_tmp_path,
getDictionaryConfigurationFromAST(query->as<const ASTCreateQuery &>(), database.getDatabaseName())}}));
SCOPE_EXIT({ external_loader.removeConfigRepository(temp_repository_name); });
bool lazy_load = context.getConfigRef().getBool("dictionaries_lazy_load", true);
if (!lazy_load)
{
/// load() is called here to force loading the dictionary, wait until the loading is finished,
/// and throw an exception if the loading is failed.
external_loader.load(full_name);
}
database.attachDictionary(dictionary_name, context);
SCOPE_EXIT({
if (!succeeded)
database.detachDictionary(dictionary_name, context);
});
/// If it was ATTACH query and file with dictionary metadata already exist
/// (so, ATTACH is done after DETACH), then rename atomically replaces old file with new one.
Poco::File(dictionary_metadata_tmp_path).renameTo(dictionary_metadata_path);
/// ExternalDictionariesLoader doesn't know we renamed the metadata path.
/// So we have to manually call reloadConfig() here.
external_loader.reloadConfig(database.getDatabaseName(), full_name);
/// Everything's ok.
succeeded = true;
}
void DatabaseOnDisk::removeTable(
IDatabase & database,
const Context & /* context */,
const String & table_name,
Poco::Logger * log)
{
StoragePtr res = database.detachTable(table_name);
String table_metadata_path = database.getObjectMetadataPath(table_name);
String table_metadata_path = getObjectMetadataPath(table_name);
try
{
@ -378,51 +191,64 @@ void DatabaseOnDisk::removeTable(
{
LOG_WARNING(log, getCurrentExceptionMessage(__PRETTY_FUNCTION__));
}
database.attachTable(table_name, res);
attachTable(table_name, res);
throw;
}
}
void DatabaseOnDisk::removeDictionary(
IDatabase & database,
const Context & context,
const String & dictionary_name,
Poco::Logger * /*log*/)
void DatabaseOnDisk::renameTable(
const Context & context,
const String & table_name,
IDatabase & to_database,
const String & to_table_name,
TableStructureWriteLockHolder & lock)
{
database.detachDictionary(dictionary_name, context);
if (typeid(*this) != typeid(to_database))
throw Exception("Moving tables between databases of different engines is not supported", ErrorCodes::NOT_IMPLEMENTED);
String dictionary_metadata_path = database.getObjectMetadataPath(dictionary_name);
if (Poco::File(dictionary_metadata_path).exists())
StoragePtr table = tryGetTable(context, table_name);
if (!table)
throw Exception("Table " + backQuote(getDatabaseName()) + "." + backQuote(table_name) + " doesn't exist.", ErrorCodes::UNKNOWN_TABLE);
ASTPtr ast = parseQueryFromMetadata(getObjectMetadataPath(table_name));
if (!ast)
throw Exception("There is no metadata file for table " + backQuote(table_name) + ".", ErrorCodes::FILE_DOESNT_EXIST);
auto & create = ast->as<ASTCreateQuery &>();
create.table = to_table_name;
/// Notify the table that it is renamed. If the table does not support renaming, exception is thrown.
try
{
try
{
Poco::File(dictionary_metadata_path).remove();
}
catch (...)
{
/// If remove was not possible for some reason
database.attachDictionary(dictionary_name, context);
throw;
}
table->rename(to_database.getTableDataPath(create),
to_database.getDatabaseName(),
to_table_name, lock);
}
catch (const Exception &)
{
throw;
}
catch (const Poco::Exception & e)
{
/// Better diagnostics.
throw Exception{Exception::CreateFromPoco, e};
}
/// NOTE Non-atomic.
to_database.createTable(context, to_table_name, table, ast);
removeTable(context, table_name);
}
ASTPtr DatabaseOnDisk::getCreateTableQueryImpl(
const IDatabase & database,
const Context & context,
const String & table_name,
bool throw_on_error)
ASTPtr DatabaseOnDisk::getCreateTableQueryImpl(const Context & context, const String & table_name, bool throw_on_error) const
{
ASTPtr ast;
auto table_metadata_path = detail::getObjectMetadataPath(database.getMetadataPath(), table_name);
ast = detail::getCreateQueryFromMetadata(table_metadata_path, database.getDatabaseName(), throw_on_error);
auto table_metadata_path = getObjectMetadataPath(table_name);
ast = getCreateQueryFromMetadata(table_metadata_path, throw_on_error);
if (!ast && throw_on_error)
{
/// Handle system.* tables for which there are no table.sql files.
bool has_table = database.tryGetTable(context, table_name) != nullptr;
bool has_table = tryGetTable(context, table_name) != nullptr;
auto msg = has_table
? "There is no CREATE TABLE query for table "
@ -434,61 +260,18 @@ ASTPtr DatabaseOnDisk::getCreateTableQueryImpl(
return ast;
}
ASTPtr DatabaseOnDisk::getCreateDictionaryQueryImpl(
const IDatabase & database,
const Context & context,
const String & dictionary_name,
bool throw_on_error)
ASTPtr DatabaseOnDisk::getCreateDatabaseQuery() const
{
ASTPtr ast;
auto dictionary_metadata_path = detail::getObjectMetadataPath(database.getMetadataPath(), dictionary_name);
ast = detail::getCreateQueryFromMetadata(dictionary_metadata_path, database.getDatabaseName(), throw_on_error);
if (!ast && throw_on_error)
{
/// Handle system.* tables for which there are no table.sql files.
bool has_dictionary = database.isDictionaryExist(context, dictionary_name);
auto msg = has_dictionary ? "There is no CREATE DICTIONARY query for table " : "There is no metadata file for dictionary ";
throw Exception(msg + backQuote(dictionary_name), ErrorCodes::CANNOT_GET_CREATE_DICTIONARY_QUERY);
}
return ast;
}
ASTPtr DatabaseOnDisk::getCreateTableQuery(const IDatabase & database, const Context & context, const String & table_name)
{
return getCreateTableQueryImpl(database, context, table_name, true);
}
ASTPtr DatabaseOnDisk::tryGetCreateTableQuery(const IDatabase & database, const Context & context, const String & table_name)
{
return getCreateTableQueryImpl(database, context, table_name, false);
}
ASTPtr DatabaseOnDisk::getCreateDictionaryQuery(const IDatabase & database, const Context & context, const String & dictionary_name)
{
return getCreateDictionaryQueryImpl(database, context, dictionary_name, true);
}
ASTPtr DatabaseOnDisk::tryGetCreateDictionaryQuery(const IDatabase & database, const Context & context, const String & dictionary_name)
{
return getCreateDictionaryQueryImpl(database, context, dictionary_name, false);
}
ASTPtr DatabaseOnDisk::getCreateDatabaseQuery(const IDatabase & database, const Context & /*context*/)
{
ASTPtr ast;
auto database_metadata_path = detail::getDatabaseMetadataPath(database.getMetadataPath());
ast = detail::getCreateQueryFromMetadata(database_metadata_path, database.getDatabaseName(), true);
auto metadata_dir_path = getMetadataPath();
auto database_metadata_path = metadata_dir_path.substr(0, metadata_dir_path.size() - 1) + ".sql";
ast = getCreateQueryFromMetadata(database_metadata_path, true);
if (!ast)
{
/// Handle databases (such as default) for which there are no database.sql files.
String query = "CREATE DATABASE " + backQuoteIfNeed(database.getDatabaseName()) + " ENGINE = Lazy";
/// If database.sql doesn't exist, then engine is Ordinary
String query = "CREATE DATABASE " + backQuoteIfNeed(getDatabaseName()) + " ENGINE = Ordinary";
ParserCreateQuery parser;
ast = parseQuery(parser, query.data(), query.data() + query.size(), "", 0);
}
@ -496,22 +279,20 @@ ASTPtr DatabaseOnDisk::getCreateDatabaseQuery(const IDatabase & database, const
return ast;
}
void DatabaseOnDisk::drop(const IDatabase & database, const Context & context)
void DatabaseOnDisk::drop(const Context & context)
{
Poco::File(context.getPath() + database.getDataPath()).remove(false);
Poco::File(database.getMetadataPath()).remove(false);
Poco::File(context.getPath() + getDataPath()).remove(false);
Poco::File(getMetadataPath()).remove(false);
}
String DatabaseOnDisk::getObjectMetadataPath(const IDatabase & database, const String & table_name)
String DatabaseOnDisk::getObjectMetadataPath(const String & table_name) const
{
return detail::getObjectMetadataPath(database.getMetadataPath(), table_name);
return getMetadataPath() + escapeForFileName(table_name) + ".sql";
}
time_t DatabaseOnDisk::getObjectMetadataModificationTime(
const IDatabase & database,
const String & table_name)
time_t DatabaseOnDisk::getObjectMetadataModificationTime(const String & table_name) const
{
String table_metadata_path = getObjectMetadataPath(database, table_name);
String table_metadata_path = getObjectMetadataPath(table_name);
Poco::File meta_file(table_metadata_path);
if (meta_file.exists())
@ -520,10 +301,10 @@ time_t DatabaseOnDisk::getObjectMetadataModificationTime(
return static_cast<time_t>(0);
}
void DatabaseOnDisk::iterateMetadataFiles(const IDatabase & database, Poco::Logger * log, const Context & context, const IteratingFunction & iterating_function)
void DatabaseOnDisk::iterateMetadataFiles(const Context & context, const IteratingFunction & iterating_function) const
{
Poco::DirectoryIterator dir_end;
for (Poco::DirectoryIterator dir_it(database.getMetadataPath()); dir_it != dir_end; ++dir_it)
for (Poco::DirectoryIterator dir_it(getMetadataPath()); dir_it != dir_end; ++dir_it)
{
/// For '.svn', '.gitignore' directory and similar.
if (dir_it.name().at(0) == '.')
@ -538,10 +319,10 @@ void DatabaseOnDisk::iterateMetadataFiles(const IDatabase & database, Poco::Logg
if (endsWith(dir_it.name(), tmp_drop_ext))
{
const std::string object_name = dir_it.name().substr(0, dir_it.name().size() - strlen(tmp_drop_ext));
if (Poco::File(context.getPath() + database.getDataPath() + '/' + object_name).exists())
if (Poco::File(context.getPath() + getDataPath() + '/' + object_name).exists())
{
/// TODO maybe complete table drop and remove all table data (including data on other volumes and metadata in ZK)
Poco::File(dir_it->path()).renameTo(database.getMetadataPath() + object_name + ".sql");
Poco::File(dir_it->path()).renameTo(getMetadataPath() + object_name + ".sql");
LOG_WARNING(log, "Object " << backQuote(object_name) << " was not dropped previously and will be restored");
iterating_function(object_name + ".sql");
}
@ -567,9 +348,64 @@ void DatabaseOnDisk::iterateMetadataFiles(const IDatabase & database, Poco::Logg
iterating_function(dir_it.name());
}
else
throw Exception("Incorrect file extension: " + dir_it.name() + " in metadata directory " + database.getMetadataPath(),
throw Exception("Incorrect file extension: " + dir_it.name() + " in metadata directory " + getMetadataPath(),
ErrorCodes::INCORRECT_FILE_NAME);
}
}
ASTPtr DatabaseOnDisk::parseQueryFromMetadata(const String & metadata_file_path, bool throw_on_error /*= true*/, bool remove_empty /*= false*/) const
{
String query;
try
{
ReadBufferFromFile in(metadata_file_path, METADATA_FILE_BUFFER_SIZE);
readStringUntilEOF(query, in);
}
catch (const Exception & e)
{
if (!throw_on_error && e.code() == ErrorCodes::FILE_DOESNT_EXIST)
return nullptr;
else
throw;
}
/** Empty files with metadata are generated after a rough restart of the server.
* Remove these files to slightly reduce the work of the admins on startup.
*/
if (remove_empty && query.empty())
{
LOG_ERROR(log, "File " << metadata_file_path << " is empty. Removing.");
Poco::File(metadata_file_path).remove();
return nullptr;
}
ParserCreateQuery parser;
const char * pos = query.data();
std::string error_message;
auto ast = tryParseQuery(parser, pos, pos + query.size(), error_message, /* hilite = */ false,
"in file " + getMetadataPath(), /* allow_multi_statements = */ false, 0);
if (!ast && throw_on_error)
throw Exception(error_message, ErrorCodes::SYNTAX_ERROR);
else if (!ast)
return nullptr;
return ast;
}
ASTPtr DatabaseOnDisk::getCreateQueryFromMetadata(const String & database_metadata_path, bool throw_on_error) const
{
ASTPtr ast = parseQueryFromMetadata(database_metadata_path, throw_on_error);
if (ast)
{
auto & ast_create_query = ast->as<ASTCreateQuery &>();
ast_create_query.attach = false;
ast_create_query.database = database_name;
}
return ast;
}
}

162
dbms/src/Databases/DatabaseOnDisk.h
View File

@ -11,24 +11,14 @@
namespace DB
{
namespace detail
{
String getObjectMetadataPath(const String & base_path, const String & dictionary_name);
String getDatabaseMetadataPath(const String & base_path);
ASTPtr getQueryFromMetadata(const String & metadata_path, bool throw_on_error = true);
ASTPtr getCreateQueryFromMetadata(const String & metadata_path, const String & database, bool throw_on_error);
}
ASTPtr parseCreateQueryFromMetadataFile(const String & filepath, Poco::Logger * log);
std::pair<String, StoragePtr> createTableFromAST(
ASTCreateQuery ast_create_query,
const String & database_name,
const String & database_data_path_relative,
const String & table_data_path_relative,
Context & context,
bool has_force_restore_data_flag);
/** Get the row with the table definition based on the CREATE query.
/** Get the string with the table definition based on the CREATE query.
* It is an ATTACH query that you can execute to create a table from the correspondent database.
* See the implementation.
*/
@ -37,147 +27,59 @@ String getObjectDefinitionFromCreateQuery(const ASTPtr & query);
/* Class to provide basic operations with tables when metadata is stored on disk in .sql files.
*/
class DatabaseOnDisk
class DatabaseOnDisk : public DatabaseWithOwnTablesBase
{
public:
static void createTable(
IDatabase & database,
DatabaseOnDisk(const String & name, const String & metadata_path_, const String & logger)
: DatabaseWithOwnTablesBase(name, logger)
, metadata_path(metadata_path_)
, data_path("data/" + escapeForFileName(database_name) + "/") {}
void createTable(
const Context & context,
const String & table_name,
const StoragePtr & table,
const ASTPtr & query);
const ASTPtr & query) override;
static void createDictionary(
IDatabase & database,
void removeTable(
const Context & context,
const String & dictionary_name,
const ASTPtr & query);
const String & table_name) override;
static void removeTable(
IDatabase & database,
const Context & context,
const String & table_name,
Poco::Logger * log);
static void removeDictionary(
IDatabase & database,
const Context & context,
const String & dictionary_name,
Poco::Logger * log);
template <typename Database>
static void renameTable(
IDatabase & database,
void renameTable(
const Context & context,
const String & table_name,
IDatabase & to_database,
const String & to_table_name,
TableStructureWriteLockHolder & lock);
TableStructureWriteLockHolder & lock) override;
static ASTPtr getCreateTableQuery(
const IDatabase & database,
const Context & context,
const String & table_name);
ASTPtr getCreateDatabaseQuery() const override;
static ASTPtr tryGetCreateTableQuery(
const IDatabase & database,
const Context & context,
const String & table_name);
void drop(const Context & context) override;
static ASTPtr getCreateDictionaryQuery(
const IDatabase & database,
const Context & context,
const String & dictionary_name);
String getObjectMetadataPath(const String & object_name) const override;
static ASTPtr tryGetCreateDictionaryQuery(
const IDatabase & database,
const Context & context,
const String & dictionary_name);
static ASTPtr getCreateDatabaseQuery(
const IDatabase & database,
const Context & context);
static void drop(const IDatabase & database, const Context & context);
static String getObjectMetadataPath(
const IDatabase & database,
const String & object_name);
static time_t getObjectMetadataModificationTime(
const IDatabase & database,
const String & object_name);
time_t getObjectMetadataModificationTime(const String & object_name) const override;
String getDataPath() const override { return data_path; }
String getTableDataPath(const String & table_name) const override { return data_path + escapeForFileName(table_name) + "/"; }
String getTableDataPath(const ASTCreateQuery & query) const override { return getTableDataPath(query.table); }
String getMetadataPath() const override { return metadata_path; }
protected:
using IteratingFunction = std::function<void(const String &)>;
static void iterateMetadataFiles(const IDatabase & database, Poco::Logger * log, const Context & context, const IteratingFunction & iterating_function);
void iterateMetadataFiles(const Context & context, const IteratingFunction & iterating_function) const;
private:
static ASTPtr getCreateTableQueryImpl(
const IDatabase & database,
ASTPtr getCreateTableQueryImpl(
const Context & context,
const String & table_name,
bool throw_on_error);
bool throw_on_error) const override;
static ASTPtr getCreateDictionaryQueryImpl(
const IDatabase & database,
const Context & context,
const String & dictionary_name,
bool throw_on_error);
ASTPtr parseQueryFromMetadata(const String & metadata_file_path, bool throw_on_error = true, bool remove_empty = false) const;
ASTPtr getCreateQueryFromMetadata(const String & metadata_path, bool throw_on_error) const;
const String metadata_path;
const String data_path;
};
namespace ErrorCodes
{
extern const int NOT_IMPLEMENTED;
extern const int UNKNOWN_TABLE;
extern const int FILE_DOESNT_EXIST;
}
template <typename Database>
void DatabaseOnDisk::renameTable(
IDatabase & database,
const Context & context,
const String & table_name,
IDatabase & to_database,
const String & to_table_name,
TableStructureWriteLockHolder & lock)
{
Database * to_database_concrete = typeid_cast<Database *>(&to_database);
if (!to_database_concrete)
throw Exception("Moving tables between databases of different engines is not supported", ErrorCodes::NOT_IMPLEMENTED);
StoragePtr table = database.tryGetTable(context, table_name);
if (!table)
throw Exception("Table " + backQuote(database.getDatabaseName()) + "." + backQuote(table_name) + " doesn't exist.", ErrorCodes::UNKNOWN_TABLE);
/// Notify the table that it is renamed. If the table does not support renaming, exception is thrown.
try
{
table->rename("/data/" + escapeForFileName(to_database_concrete->getDatabaseName()) + "/" + escapeForFileName(to_table_name) + '/',
to_database_concrete->getDatabaseName(),
to_table_name, lock);
}
catch (const Exception &)
{
throw;
}
catch (const Poco::Exception & e)
{
/// Better diagnostics.
throw Exception{Exception::CreateFromPoco, e};
}
ASTPtr ast = detail::getQueryFromMetadata(detail::getObjectMetadataPath(database.getMetadataPath(), table_name));
if (!ast)
throw Exception("There is no metadata file for table " + backQuote(table_name) + ".", ErrorCodes::FILE_DOESNT_EXIST);
ast->as<ASTCreateQuery &>().table = to_table_name;
/// NOTE Non-atomic.
to_database_concrete->createTable(context, to_table_name, table, ast);
database.removeTable(context, table_name);
}
}

129
dbms/src/Databases/DatabaseOrdinary.cpp
View File

@ -1,7 +1,6 @@
#include <iomanip>
#include <Core/Settings.h>
#include <Databases/DatabaseMemory.h>
#include <Databases/DatabaseOnDisk.h>
#include <Databases/DatabaseOrdinary.h>
#include <Databases/DatabasesCommon.h>
@ -15,13 +14,10 @@
#include <Interpreters/ExternalDictionariesLoader.h>
#include <Parsers/ASTCreateQuery.h>
#include <Parsers/ParserCreateQuery.h>
#include <Parsers/ParserDictionary.h>
#include <Storages/StorageFactory.h>
#include <Dictionaries/DictionaryFactory.h>
#include <Parsers/parseQuery.h>
#include <Parsers/formatAST.h>
#include <Parsers/ASTSetQuery.h>
#include <Storages/IStorage.h>
#include <TableFunctions/TableFunctionFactory.h>
#include <Parsers/queryToString.h>
@ -29,7 +25,6 @@
#include <Poco/DirectoryIterator.h>
#include <Poco/Event.h>
#include <Common/Stopwatch.h>
#include <Common/StringUtils/StringUtils.h>
#include <Common/quoteString.h>
#include <Common/ThreadPool.h>
#include <Common/escapeForFileName.h>
@ -47,7 +42,6 @@ namespace ErrorCodes
extern const int CANNOT_CREATE_DICTIONARY_FROM_METADATA;
extern const int EMPTY_LIST_OF_COLUMNS_PASSED;
extern const int CANNOT_PARSE_TEXT;
extern const int EMPTY_LIST_OF_ATTRIBUTES_PASSED;
}
@ -71,7 +65,7 @@ namespace
String table_name;
StoragePtr table;
std::tie(table_name, table)
= createTableFromAST(query, database_name, database.getDataPath(), context, has_force_restore_data_flag);
= createTableFromAST(query, database_name, database.getTableDataPath(query), context, has_force_restore_data_flag);
database.attachTable(table_name, table);
}
catch (const Exception & e)
@ -117,11 +111,8 @@ namespace
}
DatabaseOrdinary::DatabaseOrdinary(String name_, const String & metadata_path_, const Context & context_)
: DatabaseWithOwnTablesBase(std::move(name_))
, metadata_path(metadata_path_)
, data_path("data/" + escapeForFileName(name) + "/")
, log(&Logger::get("DatabaseOrdinary (" + name + ")"))
DatabaseOrdinary::DatabaseOrdinary(const String & name_, const String & metadata_path_, const Context & context_)
: DatabaseWithDictionaries(name_, metadata_path_, "DatabaseOrdinary (" + name_ + ")")
{
Poco::File(context_.getPath() + getDataPath()).createDirectories();
}
@ -140,12 +131,12 @@ void DatabaseOrdinary::loadStoredObjects(
FileNames file_names;
size_t total_dictionaries = 0;
DatabaseOnDisk::iterateMetadataFiles(*this, log, context, [&file_names, &total_dictionaries, this](const String & file_name)
iterateMetadataFiles(context, [&file_names, &total_dictionaries, this](const String & file_name)
{
String full_path = metadata_path + "/" + file_name;
String full_path = getMetadataPath() + file_name;
try
{
auto ast = parseCreateQueryFromMetadataFile(full_path, log);
auto ast = parseQueryFromMetadata(full_path, /*throw_on_error*/ true, /*remove_empty*/false);
if (ast)
{
auto * create_query = ast->as<ASTCreateQuery>();
@ -240,91 +231,14 @@ void DatabaseOrdinary::startupTables(ThreadPool & thread_pool)
thread_pool.wait();
}
void DatabaseOrdinary::createTable(
const Context & context,
const String & table_name,
const StoragePtr & table,
const ASTPtr & query)
{
DatabaseOnDisk::createTable(*this, context, table_name, table, query);
}
void DatabaseOrdinary::createDictionary(
const Context & context,
const String & dictionary_name,
const ASTPtr & query)
{
DatabaseOnDisk::createDictionary(*this, context, dictionary_name, query);
}
void DatabaseOrdinary::removeTable(
const Context & context,
const String & table_name)
{
DatabaseOnDisk::removeTable(*this, context, table_name, log);
}
void DatabaseOrdinary::removeDictionary(
const Context & context,
const String & table_name)
{
DatabaseOnDisk::removeDictionary(*this, context, table_name, log);
}
void DatabaseOrdinary::renameTable(
const Context & context,
const String & table_name,
IDatabase & to_database,
const String & to_table_name,
TableStructureWriteLockHolder & lock)
{
DatabaseOnDisk::renameTable<DatabaseOrdinary>(*this, context, table_name, to_database, to_table_name, lock);
}
time_t DatabaseOrdinary::getObjectMetadataModificationTime(
const Context & /* context */,
const String & table_name)
{
return DatabaseOnDisk::getObjectMetadataModificationTime(*this, table_name);
}
ASTPtr DatabaseOrdinary::getCreateTableQuery(const Context & context, const String & table_name) const
{
return DatabaseOnDisk::getCreateTableQuery(*this, context, table_name);
}
ASTPtr DatabaseOrdinary::tryGetCreateTableQuery(const Context & context, const String & table_name) const
{
return DatabaseOnDisk::tryGetCreateTableQuery(*this, context, table_name);
}
ASTPtr DatabaseOrdinary::getCreateDictionaryQuery(const Context & context, const String & dictionary_name) const
{
return DatabaseOnDisk::getCreateDictionaryQuery(*this, context, dictionary_name);
}
ASTPtr DatabaseOrdinary::tryGetCreateDictionaryQuery(const Context & context, const String & dictionary_name) const
{
return DatabaseOnDisk::tryGetCreateTableQuery(*this, context, dictionary_name);
}
ASTPtr DatabaseOrdinary::getCreateDatabaseQuery(const Context & context) const
{
return DatabaseOnDisk::getCreateDatabaseQuery(*this, context);
}
void DatabaseOrdinary::alterTable(
const Context & context,
const String & table_name,
const StorageInMemoryMetadata & metadata)
{
/// Read the definition of the table and replace the necessary parts with new ones.
String table_name_escaped = escapeForFileName(table_name);
String table_metadata_tmp_path = getMetadataPath() + "/" + table_name_escaped + ".sql.tmp";
String table_metadata_path = getMetadataPath() + "/" + table_name_escaped + ".sql";
String table_metadata_path = getObjectMetadataPath(table_name);
String table_metadata_tmp_path = table_metadata_path + ".tmp";
String statement;
{
@ -383,31 +297,4 @@ void DatabaseOrdinary::alterTable(
}
}
void DatabaseOrdinary::drop(const Context & context)
{
DatabaseOnDisk::drop(*this, context);
}
String DatabaseOrdinary::getDataPath() const
{
return data_path;
}
String DatabaseOrdinary::getMetadataPath() const
{
return metadata_path;
}
String DatabaseOrdinary::getDatabaseName() const
{
return name;
}
String DatabaseOrdinary::getObjectMetadataPath(const String & table_name) const
{
return DatabaseOnDisk::getObjectMetadataPath(*this, table_name);
}
}

64
dbms/src/Databases/DatabaseOrdinary.h
View File

@ -1,6 +1,6 @@
#pragma once
#include <Databases/DatabasesCommon.h>
#include <Databases/DatabaseWithDictionaries.h>
#include <Common/ThreadPool.h>
@ -11,10 +11,10 @@ namespace DB
* It stores tables list in filesystem using list of .sql files,
* that contain declaration of table represented by SQL ATTACH TABLE query.
*/
class DatabaseOrdinary : public DatabaseWithOwnTablesBase
class DatabaseOrdinary : public DatabaseWithDictionaries //DatabaseWithOwnTablesBase
{
public:
DatabaseOrdinary(String name_, const String & metadata_path_, const Context & context);
DatabaseOrdinary(const String & name_, const String & metadata_path_, const Context & context);
String getEngineName() const override { return "Ordinary"; }
@ -22,70 +22,12 @@ public:
Context & context,
bool has_force_restore_data_flag) override;
void createTable(
const Context & context,
const String & table_name,
const StoragePtr & table,
const ASTPtr & query) override;
void createDictionary(
const Context & context,
const String & dictionary_name,
const ASTPtr & query) override;
void removeTable(
const Context & context,
const String & table_name) override;
void removeDictionary(
const Context & context,
const String & table_name) override;
void renameTable(
const Context & context,
const String & table_name,
IDatabase & to_database,
const String & to_table_name,
TableStructureWriteLockHolder &) override;
void alterTable(
const Context & context,
const String & name,
const StorageInMemoryMetadata & metadata) override;
time_t getObjectMetadataModificationTime(
const Context & context,
const String & table_name) override;
ASTPtr getCreateTableQuery(
const Context & context,
const String & table_name) const override;
ASTPtr tryGetCreateTableQuery(
const Context & context,
const String & table_name) const override;
ASTPtr tryGetCreateDictionaryQuery(
const Context & context,
const String & name) const override;
ASTPtr getCreateDictionaryQuery(
const Context & context,
const String & name) const override;
ASTPtr getCreateDatabaseQuery(const Context & context) const override;
String getDataPath() const override;
String getDatabaseName() const override;
String getMetadataPath() const override;
String getObjectMetadataPath(const String & table_name) const override;
void drop(const Context & context) override;
private:
const String metadata_path;
const String data_path;
Poco::Logger * log;
void startupTables(ThreadPool & thread_pool);
};

256
dbms/src/Databases/DatabaseWithDictionaries.cpp Normal file
View File

@ -0,0 +1,256 @@
#include <Databases/DatabaseWithDictionaries.h>
#include <Interpreters/ExternalDictionariesLoader.h>
#include <Interpreters/ExternalLoaderPresetConfigRepository.h>
#include <Dictionaries/getDictionaryConfigurationFromAST.h>
#include <Interpreters/Context.h>
#include <Storages/StorageDictionary.h>
#include <IO/WriteBufferFromFile.h>
#include <Poco/File.h>
#include <ext/scope_guard.h>
namespace DB
{
namespace ErrorCodes
{
extern const int EMPTY_LIST_OF_COLUMNS_PASSED;
extern const int TABLE_ALREADY_EXISTS;
extern const int UNKNOWN_TABLE;
extern const int LOGICAL_ERROR;
extern const int DICTIONARY_ALREADY_EXISTS;
}
void DatabaseWithDictionaries::attachDictionary(const String & dictionary_name, const Context & context)
{
String full_name = getDatabaseName() + "." + dictionary_name;
{
std::lock_guard lock(mutex);
if (!dictionaries.emplace(dictionary_name).second)
throw Exception("Dictionary " + full_name + " already exists.", ErrorCodes::DICTIONARY_ALREADY_EXISTS);
}
/// ExternalLoader::reloadConfig() will find out that the dictionary's config has been added
/// and in case `dictionaries_lazy_load == false` it will load the dictionary.
const auto & external_loader = context.getExternalDictionariesLoader();
external_loader.reloadConfig(getDatabaseName(), full_name);
}
void DatabaseWithDictionaries::detachDictionary(const String & dictionary_name, const Context & context)
{
String full_name = getDatabaseName() + "." + dictionary_name;
{
std::lock_guard lock(mutex);
auto it = dictionaries.find(dictionary_name);
if (it == dictionaries.end())
throw Exception("Dictionary " + full_name + " doesn't exist.", ErrorCodes::UNKNOWN_TABLE);
dictionaries.erase(it);
}
/// ExternalLoader::reloadConfig() will find out that the dictionary's config has been removed
/// and therefore it will unload the dictionary.
const auto & external_loader = context.getExternalDictionariesLoader();
external_loader.reloadConfig(getDatabaseName(), full_name);
}
void DatabaseWithDictionaries::createDictionary(const Context & context, const String & dictionary_name, const ASTPtr & query)
{
const auto & settings = context.getSettingsRef();
/** The code is based on the assumption that all threads share the same order of operations:
* - create the .sql.tmp file;
* - add the dictionary to ExternalDictionariesLoader;
* - load the dictionary in case dictionaries_lazy_load == false;
* - attach the dictionary;
* - rename .sql.tmp to .sql.
*/
/// A race condition would be possible if a dictionary with the same name is simultaneously created using CREATE and using ATTACH.
/// But there is protection from it - see using DDLGuard in InterpreterCreateQuery.
if (isDictionaryExist(context, dictionary_name))
throw Exception("Dictionary " + backQuote(getDatabaseName()) + "." + backQuote(dictionary_name) + " already exists.", ErrorCodes::DICTIONARY_ALREADY_EXISTS);
/// A dictionary with the same full name could be defined in *.xml config files.
String full_name = getDatabaseName() + "." + dictionary_name;
auto & external_loader = const_cast<ExternalDictionariesLoader &>(context.getExternalDictionariesLoader());
if (external_loader.getCurrentStatus(full_name) != ExternalLoader::Status::NOT_EXIST)
throw Exception(
"Dictionary " + backQuote(getDatabaseName()) + "." + backQuote(dictionary_name) + " already exists.",
ErrorCodes::DICTIONARY_ALREADY_EXISTS);
if (isTableExist(context, dictionary_name))
throw Exception("Table " + backQuote(getDatabaseName()) + "." + backQuote(dictionary_name) + " already exists.", ErrorCodes::TABLE_ALREADY_EXISTS);
String dictionary_metadata_path = getObjectMetadataPath(dictionary_name);
String dictionary_metadata_tmp_path = dictionary_metadata_path + ".tmp";
String statement = getObjectDefinitionFromCreateQuery(query);
{
/// Exclusive flags guarantees, that table is not created right now in another thread. Otherwise, exception will be thrown.
WriteBufferFromFile out(dictionary_metadata_tmp_path, statement.size(), O_WRONLY | O_CREAT | O_EXCL);
writeString(statement, out);
out.next();
if (settings.fsync_metadata)
out.sync();
out.close();
}
bool succeeded = false;
SCOPE_EXIT({
if (!succeeded)
Poco::File(dictionary_metadata_tmp_path).remove();
});
/// Add a temporary repository containing the dictionary.
/// We need this temp repository to try loading the dictionary before actually attaching it to the database.
static std::atomic<size_t> counter = 0;
String temp_repository_name = String(IExternalLoaderConfigRepository::INTERNAL_REPOSITORY_NAME_PREFIX) + " creating " + full_name + " "
+ std::to_string(++counter);
external_loader.addConfigRepository(
temp_repository_name,
std::make_unique<ExternalLoaderPresetConfigRepository>(
std::vector{std::pair{dictionary_metadata_tmp_path,
getDictionaryConfigurationFromAST(query->as<const ASTCreateQuery &>(), getDatabaseName())}}));
SCOPE_EXIT({ external_loader.removeConfigRepository(temp_repository_name); });
bool lazy_load = context.getConfigRef().getBool("dictionaries_lazy_load", true);
if (!lazy_load)
{
/// load() is called here to force loading the dictionary, wait until the loading is finished,
/// and throw an exception if the loading is failed.
external_loader.load(full_name);
}
attachDictionary(dictionary_name, context);
SCOPE_EXIT({
if (!succeeded)
detachDictionary(dictionary_name, context);
});
/// If it was ATTACH query and file with dictionary metadata already exist
/// (so, ATTACH is done after DETACH), then rename atomically replaces old file with new one.
Poco::File(dictionary_metadata_tmp_path).renameTo(dictionary_metadata_path);
/// ExternalDictionariesLoader doesn't know we renamed the metadata path.
/// So we have to manually call reloadConfig() here.
external_loader.reloadConfig(getDatabaseName(), full_name);
/// Everything's ok.
succeeded = true;
}
void DatabaseWithDictionaries::removeDictionary(const Context & context, const String & dictionary_name)
{
detachDictionary(dictionary_name, context);
String dictionary_metadata_path = getObjectMetadataPath(dictionary_name);
try
{
Poco::File(dictionary_metadata_path).remove();
}
catch (...)
{
/// If remove was not possible for some reason
attachDictionary(dictionary_name, context);
throw;
}
}
StoragePtr DatabaseWithDictionaries::tryGetTable(const Context & context, const String & table_name) const
{
if (auto table_ptr = DatabaseWithOwnTablesBase::tryGetTable(context, table_name))
return table_ptr;
if (isDictionaryExist(context, table_name))
/// We don't need lock database here, because database doesn't store dictionary itself
/// just metadata
return getDictionaryStorage(context, table_name);
return {};
}
DatabaseTablesIteratorPtr DatabaseWithDictionaries::getTablesWithDictionaryTablesIterator(const Context & context, const FilterByNameFunction & filter_by_name)
{
/// NOTE: it's not atomic
auto tables_it = getTablesIterator(context, filter_by_name);
auto dictionaries_it = getDictionariesIterator(context, filter_by_name);
Tables result;
while (tables_it && tables_it->isValid())
{
result.emplace(tables_it->name(), tables_it->table());
tables_it->next();
}
while (dictionaries_it && dictionaries_it->isValid())
{
auto table_name = dictionaries_it->name();
auto table_ptr = getDictionaryStorage(context, table_name);
if (table_ptr)
result.emplace(table_name, table_ptr);
dictionaries_it->next();
}
return std::make_unique<DatabaseTablesSnapshotIterator>(result);
}
DatabaseDictionariesIteratorPtr DatabaseWithDictionaries::getDictionariesIterator(const Context & /*context*/, const FilterByNameFunction & filter_by_dictionary_name)
{
std::lock_guard lock(mutex);
if (!filter_by_dictionary_name)
return std::make_unique<DatabaseDictionariesSnapshotIterator>(dictionaries);
Dictionaries filtered_dictionaries;
for (const auto & dictionary_name : dictionaries)
if (filter_by_dictionary_name(dictionary_name))
filtered_dictionaries.emplace(dictionary_name);
return std::make_unique<DatabaseDictionariesSnapshotIterator>(std::move(filtered_dictionaries));
}
bool DatabaseWithDictionaries::isDictionaryExist(const Context & /*context*/, const String & dictionary_name) const
{
std::lock_guard lock(mutex);
return dictionaries.find(dictionary_name) != dictionaries.end();
}
StoragePtr DatabaseWithDictionaries::getDictionaryStorage(const Context & context, const String & table_name) const
{
auto dict_name = database_name + "." + table_name;
const auto & external_loader = context.getExternalDictionariesLoader();
auto dict_ptr = external_loader.tryGetDictionary(dict_name);
if (dict_ptr)
{
const DictionaryStructure & dictionary_structure = dict_ptr->getStructure();
auto columns = StorageDictionary::getNamesAndTypes(dictionary_structure);
return StorageDictionary::create(database_name, table_name, ColumnsDescription{columns}, context, true, dict_name);
}
return nullptr;
}
ASTPtr DatabaseWithDictionaries::getCreateDictionaryQueryImpl(
const Context & context,
const String & dictionary_name,
bool throw_on_error) const
{
ASTPtr ast;
auto dictionary_metadata_path = getObjectMetadataPath(dictionary_name);
ast = getCreateQueryFromMetadata(dictionary_metadata_path, throw_on_error);
if (!ast && throw_on_error)
{
/// Handle system.* tables for which there are no table.sql files.
bool has_dictionary = isDictionaryExist(context, dictionary_name);
auto msg = has_dictionary ? "There is no CREATE DICTIONARY query for table " : "There is no metadata file for dictionary ";
throw Exception(msg + backQuote(dictionary_name), ErrorCodes::CANNOT_GET_CREATE_DICTIONARY_QUERY);
}
return ast;
}
}

39
dbms/src/Databases/DatabaseWithDictionaries.h Normal file
View File

@ -0,0 +1,39 @@
#include <Databases/DatabaseOnDisk.h>
namespace DB
{
class DatabaseWithDictionaries : public DatabaseOnDisk
{
public:
void attachDictionary(const String & name, const Context & context) override;
void detachDictionary(const String & name, const Context & context) override;
void createDictionary(const Context & context,
const String & dictionary_name,
const ASTPtr & query) override;
void removeDictionary(const Context & context, const String & dictionary_name) override;
StoragePtr tryGetTable(const Context & context, const String & table_name) const override;
DatabaseTablesIteratorPtr getTablesWithDictionaryTablesIterator(const Context & context, const FilterByNameFunction & filter_by_dictionary_name = {}) override;
DatabaseDictionariesIteratorPtr getDictionariesIterator(const Context & context, const FilterByNameFunction & filter_by_dictionary_name = {}) override;
bool isDictionaryExist(const Context & context, const String & dictionary_name) const override;
protected:
DatabaseWithDictionaries(const String & name, const String & metadata_path_, const String & logger)
: DatabaseOnDisk(name, metadata_path_, logger) {}
StoragePtr getDictionaryStorage(const Context & context, const String & table_name) const;
ASTPtr getCreateDictionaryQueryImpl(const Context & context,
const String & dictionary_name,
bool throw_on_error) const override;
};
}

125
dbms/src/Databases/DatabasesCommon.cpp
View File

@ -1,6 +1,4 @@
#include <Databases/DatabasesCommon.h>
#include <Interpreters/ExternalDictionariesLoader.h>
#include <Interpreters/Context.h>
#include <Interpreters/InterpreterCreateQuery.h>
#include <Parsers/ParserCreateQuery.h>
#include <Parsers/formatAST.h>
@ -22,23 +20,9 @@ namespace ErrorCodes
extern const int DICTIONARY_ALREADY_EXISTS;
}
namespace
DatabaseWithOwnTablesBase::DatabaseWithOwnTablesBase(const String & name_, const String & logger)
: IDatabase(name_), log(&Logger::get(logger))
{
StoragePtr getDictionaryStorage(const Context & context, const String & table_name, const String & db_name)
{
auto dict_name = db_name + "." + table_name;
const auto & external_loader = context.getExternalDictionariesLoader();
auto dict_ptr = external_loader.tryGetDictionary(dict_name);
if (dict_ptr)
{
const DictionaryStructure & dictionary_structure = dict_ptr->getStructure();
auto columns = StorageDictionary::getNamesAndTypes(dictionary_structure);
return StorageDictionary::create(db_name, table_name, ColumnsDescription{columns}, context, true, dict_name);
}
return nullptr;
}
}
bool DatabaseWithOwnTablesBase::isTableExist(
@ -49,57 +33,17 @@ bool DatabaseWithOwnTablesBase::isTableExist(
return tables.find(table_name) != tables.end() || dictionaries.find(table_name) != dictionaries.end();
}
bool DatabaseWithOwnTablesBase::isDictionaryExist(
const Context & /*context*/,
const String & dictionary_name) const
{
std::lock_guard lock(mutex);
return dictionaries.find(dictionary_name) != dictionaries.end();
}
StoragePtr DatabaseWithOwnTablesBase::tryGetTable(
const Context & context,
const Context & /*context*/,
const String & table_name) const
{
{
std::lock_guard lock(mutex);
auto it = tables.find(table_name);
if (it != tables.end())
return it->second;
}
if (isDictionaryExist(context, table_name))
/// We don't need lock database here, because database doesn't store dictionary itself
/// just metadata
return getDictionaryStorage(context, table_name, getDatabaseName());
std::lock_guard lock(mutex);
auto it = tables.find(table_name);
if (it != tables.end())
return it->second;
return {};
}
DatabaseTablesIteratorPtr DatabaseWithOwnTablesBase::getTablesWithDictionaryTablesIterator(const Context & context, const FilterByNameFunction & filter_by_name)
{
auto tables_it = getTablesIterator(context, filter_by_name);
auto dictionaries_it = getDictionariesIterator(context, filter_by_name);
Tables result;
while (tables_it && tables_it->isValid())
{
result.emplace(tables_it->name(), tables_it->table());
tables_it->next();
}
while (dictionaries_it && dictionaries_it->isValid())
{
auto table_name = dictionaries_it->name();
auto table_ptr = getDictionaryStorage(context, table_name, getDatabaseName());
if (table_ptr)
result.emplace(table_name, table_ptr);
dictionaries_it->next();
}
return std::make_unique<DatabaseTablesSnapshotIterator>(result);
}
DatabaseTablesIteratorPtr DatabaseWithOwnTablesBase::getTablesIterator(const Context & /*context*/, const FilterByNameFunction & filter_by_table_name)
{
std::lock_guard lock(mutex);
@ -114,20 +58,6 @@ DatabaseTablesIteratorPtr DatabaseWithOwnTablesBase::getTablesIterator(const Con
return std::make_unique<DatabaseTablesSnapshotIterator>(std::move(filtered_tables));
}
DatabaseDictionariesIteratorPtr DatabaseWithOwnTablesBase::getDictionariesIterator(const Context & /*context*/, const FilterByNameFunction & filter_by_dictionary_name)
{
std::lock_guard lock(mutex);
if (!filter_by_dictionary_name)
return std::make_unique<DatabaseDictionariesSnapshotIterator>(dictionaries);
Dictionaries filtered_dictionaries;
for (const auto & dictionary_name : dictionaries)
if (filter_by_dictionary_name(dictionary_name))
filtered_dictionaries.emplace(dictionary_name);
return std::make_unique<DatabaseDictionariesSnapshotIterator>(std::move(filtered_dictionaries));
}
bool DatabaseWithOwnTablesBase::empty(const Context & /*context*/) const
{
std::lock_guard lock(mutex);
@ -140,11 +70,11 @@ StoragePtr DatabaseWithOwnTablesBase::detachTable(const String & table_name)
{
std::lock_guard lock(mutex);
if (dictionaries.count(table_name))
throw Exception("Cannot detach dictionary " + name + "." + table_name + " as table, use DETACH DICTIONARY query.", ErrorCodes::UNKNOWN_TABLE);
throw Exception("Cannot detach dictionary " + database_name + "." + table_name + " as table, use DETACH DICTIONARY query.", ErrorCodes::UNKNOWN_TABLE);
auto it = tables.find(table_name);
if (it == tables.end())
throw Exception("Table " + backQuote(name) + "." + backQuote(table_name) + " doesn't exist.", ErrorCodes::UNKNOWN_TABLE);
throw Exception("Table " + backQuote(database_name) + "." + backQuote(table_name) + " doesn't exist.", ErrorCodes::UNKNOWN_TABLE);
res = it->second;
tables.erase(it);
}
@ -152,44 +82,11 @@ StoragePtr DatabaseWithOwnTablesBase::detachTable(const String & table_name)
return res;
}
void DatabaseWithOwnTablesBase::detachDictionary(const String & dictionary_name, const Context & context)
{
String full_name = getDatabaseName() + "." + dictionary_name;
{
std::lock_guard lock(mutex);
auto it = dictionaries.find(dictionary_name);
if (it == dictionaries.end())
throw Exception("Dictionary " + full_name + " doesn't exist.", ErrorCodes::UNKNOWN_TABLE);
dictionaries.erase(it);
}
/// ExternalLoader::reloadConfig() will find out that the dictionary's config has been removed
/// and therefore it will unload the dictionary.
const auto & external_loader = context.getExternalDictionariesLoader();
external_loader.reloadConfig(getDatabaseName(), full_name);
}
void DatabaseWithOwnTablesBase::attachTable(const String & table_name, const StoragePtr & table)
{
std::lock_guard lock(mutex);
if (!tables.emplace(table_name, table).second)
throw Exception("Table " + name + "." + table_name + " already exists.", ErrorCodes::TABLE_ALREADY_EXISTS);
}
void DatabaseWithOwnTablesBase::attachDictionary(const String & dictionary_name, const Context & context)
{
String full_name = getDatabaseName() + "." + dictionary_name;
{
std::lock_guard lock(mutex);
if (!dictionaries.emplace(dictionary_name).second)
throw Exception("Dictionary " + full_name + " already exists.", ErrorCodes::DICTIONARY_ALREADY_EXISTS);
}
/// ExternalLoader::reloadConfig() will find out that the dictionary's config has been added
/// and in case `dictionaries_lazy_load == false` it will load the dictionary.
const auto & external_loader = context.getExternalDictionariesLoader();
external_loader.reloadConfig(getDatabaseName(), full_name);
throw Exception("Table " + database_name + "." + table_name + " already exists.", ErrorCodes::TABLE_ALREADY_EXISTS);
}
void DatabaseWithOwnTablesBase::shutdown()
@ -217,7 +114,7 @@ DatabaseWithOwnTablesBase::~DatabaseWithOwnTablesBase()
{
try
{
shutdown();
DatabaseWithOwnTablesBase::shutdown();
}
catch (...)
{

15
dbms/src/Databases/DatabasesCommon.h
View File

@ -23,8 +23,6 @@ public:
const Context & context,
const String & table_name) const override;
bool isDictionaryExist(const Context & context, const String & dictionary_name) const override;
StoragePtr tryGetTable(
const Context & context,
const String & table_name) const override;
@ -33,30 +31,21 @@ public:
void attachTable(const String & table_name, const StoragePtr & table) override;
void attachDictionary(const String & name, const Context & context) override;
StoragePtr detachTable(const String & table_name) override;
void detachDictionary(const String & name, const Context & context) override;
DatabaseTablesIteratorPtr getTablesIterator(const Context & context, const FilterByNameFunction & filter_by_table_name = {}) override;
DatabaseDictionariesIteratorPtr getDictionariesIterator(const Context & context, const FilterByNameFunction & filter_by_dictionary_name = {}) override;
DatabaseTablesIteratorPtr getTablesWithDictionaryTablesIterator(const Context & context, const FilterByNameFunction & filter_by_dictionary_name = {}) override;
void shutdown() override;
virtual ~DatabaseWithOwnTablesBase() override;
protected:
String name;
mutable std::mutex mutex;
Tables tables;
Dictionaries dictionaries;
Poco::Logger * log;
DatabaseWithOwnTablesBase(String name_) : name(std::move(name_)) { }
DatabaseWithOwnTablesBase(const String & name_, const String & logger);
};
}

131
dbms/src/Databases/IDatabase.h
View File

@ -21,11 +21,15 @@ struct ConstraintsDescription;
class ColumnsDescription;
struct IndicesDescription;
struct TableStructureWriteLockHolder;
class ASTCreateQuery;
using Dictionaries = std::set<String>;
namespace ErrorCodes
{
extern const int NOT_IMPLEMENTED;
extern const int CANNOT_GET_CREATE_TABLE_QUERY;
extern const int CANNOT_GET_CREATE_TABLE_QUERY;
extern const int CANNOT_GET_CREATE_DICTIONARY_QUERY;
}
class IDatabaseTablesIterator
@ -97,14 +101,15 @@ using DatabaseDictionariesIteratorPtr = std::unique_ptr<DatabaseDictionariesSnap
class IDatabase : public std::enable_shared_from_this<IDatabase>
{
public:
IDatabase() = delete;
IDatabase(String database_name_) : database_name(std::move(database_name_)) {}
/// Get name of database engine.
virtual String getEngineName() const = 0;
/// Load a set of existing tables.
/// You can call only once, right after the object is created.
virtual void loadStoredObjects(
Context & context,
bool has_force_restore_data_flag) = 0;
virtual void loadStoredObjects(Context & /*context*/, bool /*has_force_restore_data_flag*/) {}
/// Check the existence of the table.
virtual bool isTableExist(
@ -113,8 +118,11 @@ public:
/// Check the existence of the dictionary
virtual bool isDictionaryExist(
const Context & context,
const String & name) const = 0;
const Context & /*context*/,
const String & /*name*/) const
{
return false;
}
/// Get the table for work. Return nullptr if there is no table.
virtual StoragePtr tryGetTable(
@ -128,7 +136,10 @@ public:
virtual DatabaseTablesIteratorPtr getTablesIterator(const Context & context, const FilterByNameFunction & filter_by_table_name = {}) = 0;
/// Get an iterator to pass through all the dictionaries.
virtual DatabaseDictionariesIteratorPtr getDictionariesIterator(const Context & context, const FilterByNameFunction & filter_by_dictionary_name = {}) = 0;
virtual DatabaseDictionariesIteratorPtr getDictionariesIterator(const Context & /*context*/, [[maybe_unused]] const FilterByNameFunction & filter_by_dictionary_name = {})
{
return std::make_unique<DatabaseDictionariesSnapshotIterator>();
}
/// Get an iterator to pass through all the tables and dictionary tables.
virtual DatabaseTablesIteratorPtr getTablesWithDictionaryTablesIterator(const Context & context, const FilterByNameFunction & filter_by_name = {})
@ -141,39 +152,63 @@ public:
/// Add the table to the database. Record its presence in the metadata.
virtual void createTable(
const Context & context,
const String & name,
const StoragePtr & table,
const ASTPtr & query) = 0;
const Context & /*context*/,
const String & /*name*/,
const StoragePtr & /*table*/,
const ASTPtr & /*query*/)
{
throw Exception("There is no CREATE TABLE query for Database" + getEngineName(), ErrorCodes::NOT_IMPLEMENTED);
}
/// Add the dictionary to the database. Record its presence in the metadata.
virtual void createDictionary(
const Context & context,
const String & dictionary_name,
const ASTPtr & query) = 0;
const Context & /*context*/,
const String & /*dictionary_name*/,
const ASTPtr & /*query*/)
{
throw Exception("There is no CREATE DICTIONARY query for Database" + getEngineName(), ErrorCodes::NOT_IMPLEMENTED);
}
/// Delete the table from the database. Delete the metadata.
virtual void removeTable(
const Context & context,
const String & name) = 0;
const Context & /*context*/,
const String & /*name*/)
{
throw Exception("There is no DROP TABLE query for Database" + getEngineName(), ErrorCodes::NOT_IMPLEMENTED);
}
/// Delete the dictionary from the database. Delete the metadata.
virtual void removeDictionary(
const Context & context,
const String & dictionary_name) = 0;
const Context & /*context*/,
const String & /*dictionary_name*/)
{
throw Exception("There is no DROP DICTIONARY query for Database" + getEngineName(), ErrorCodes::NOT_IMPLEMENTED);
}
/// Add a table to the database, but do not add it to the metadata. The database may not support this method.
virtual void attachTable(const String & name, const StoragePtr & table) = 0;
virtual void attachTable(const String & /*name*/, const StoragePtr & /*table*/)
{
throw Exception("There is no ATTACH TABLE query for Database" + getEngineName(), ErrorCodes::NOT_IMPLEMENTED);
}
/// Add dictionary to the database, but do not add it to the metadata. The database may not support this method.
/// If dictionaries_lazy_load is false it also starts loading the dictionary asynchronously.
virtual void attachDictionary(const String & name, const Context & context) = 0;
virtual void attachDictionary(const String & /*name*/, const Context & /*context*/)
{
throw Exception("There is no ATTACH DICTIONARY query for Database" + getEngineName(), ErrorCodes::NOT_IMPLEMENTED);
}
/// Forget about the table without deleting it, and return it. The database may not support this method.
virtual StoragePtr detachTable(const String & name) = 0;
virtual StoragePtr detachTable(const String & /*name*/)
{
throw Exception("There is no DETACH TABLE query for Database" + getEngineName(), ErrorCodes::NOT_IMPLEMENTED);
}
/// Forget about the dictionary without deleting it. The database may not support this method.
virtual void detachDictionary(const String & name, const Context & context) = 0;
virtual void detachDictionary(const String & /*name*/, const Context & /*context*/)
{
throw Exception("There is no DETACH DICTIONARY query for Database" + getEngineName(), ErrorCodes::NOT_IMPLEMENTED);
}
/// Rename the table and possibly move the table to another database.
virtual void renameTable(
@ -199,33 +234,44 @@ public:
}
/// Returns time of table's metadata change, 0 if there is no corresponding metadata file.
virtual time_t getObjectMetadataModificationTime(
const Context & context,
const String & name) = 0;
virtual time_t getObjectMetadataModificationTime(const String & /*name*/) const
{
return static_cast<time_t>(0);
}
/// Get the CREATE TABLE query for the table. It can also provide information for detached tables for which there is metadata.
virtual ASTPtr tryGetCreateTableQuery(const Context & context, const String & name) const = 0;
virtual ASTPtr getCreateTableQuery(const Context & context, const String & name) const
ASTPtr tryGetCreateTableQuery(const Context & context, const String & name) const noexcept
{
return tryGetCreateTableQuery(context, name);
return getCreateTableQueryImpl(context, name, false);
}
ASTPtr getCreateTableQuery(const Context & context, const String & name) const
{
return getCreateTableQueryImpl(context, name, true);
}
/// Get the CREATE DICTIONARY query for the dictionary. Returns nullptr if dictionary doesn't exists.
virtual ASTPtr tryGetCreateDictionaryQuery(const Context & context, const String & name) const = 0;
virtual ASTPtr getCreateDictionaryQuery(const Context & context, const String & name) const
ASTPtr tryGetCreateDictionaryQuery(const Context & context, const String & name) const noexcept
{
return tryGetCreateDictionaryQuery(context, name);
return getCreateDictionaryQueryImpl(context, name, false);
}
ASTPtr getCreateDictionaryQuery(const Context & context, const String & name) const
{
return getCreateDictionaryQueryImpl(context, name, true);
}
/// Get the CREATE DATABASE query for current database.
virtual ASTPtr getCreateDatabaseQuery(const Context & context) const = 0;
virtual ASTPtr getCreateDatabaseQuery() const = 0;
/// Get name of database.
virtual String getDatabaseName() const = 0;
String getDatabaseName() const { return database_name; }
/// Returns path for persistent data storage if the database supports it, empty string otherwise
virtual String getDataPath() const { return {}; }
/// Returns path for persistent data storage for table if the database supports it, empty string otherwise. Table must exist
virtual String getTableDataPath(const String & /*table_name*/) const { return {}; }
/// Returns path for persistent data storage for CREATE/ATTACH query if the database supports it, empty string otherwise
virtual String getTableDataPath(const ASTCreateQuery & /*query*/) const { return {}; }
/// Returns metadata path if the database supports it, empty string otherwise
virtual String getMetadataPath() const { return {}; }
/// Returns metadata path of a concrete table if the database supports it, empty string otherwise
@ -238,6 +284,23 @@ public:
virtual void drop(const Context & /*context*/) {}
virtual ~IDatabase() {}
protected:
virtual ASTPtr getCreateTableQueryImpl(const Context & /*context*/, const String & /*name*/, bool throw_on_error) const
{
if (throw_on_error)
throw Exception("There is no SHOW CREATE TABLE query for Database" + getEngineName(), ErrorCodes::CANNOT_GET_CREATE_TABLE_QUERY);
return nullptr;
}
virtual ASTPtr getCreateDictionaryQueryImpl(const Context & /*context*/, const String & /*name*/, bool throw_on_error) const
{
if (throw_on_error)
throw Exception("There is no SHOW CREATE DICTIONARY query for Database" + getEngineName(), ErrorCodes::CANNOT_GET_CREATE_DICTIONARY_QUERY);
return nullptr;
}
String database_name;
};
using DatabasePtr = std::shared_ptr<IDatabase>;

31
dbms/src/Interpreters/Context.cpp
View File

@ -1095,27 +1095,6 @@ DatabasePtr Context::detachDatabase(const String & database_name)
}
ASTPtr Context::getCreateTableQuery(const String & database_name, const String & table_name) const
{
auto lock = getLock();
String db = resolveDatabase(database_name, current_database);
assertDatabaseExists(db);
return shared->databases[db]->getCreateTableQuery(*this, table_name);
}
ASTPtr Context::getCreateDictionaryQuery(const String & database_name, const String & dictionary_name) const
{
auto lock = getLock();
String db = resolveDatabase(database_name, current_database);
assertDatabaseExists(db);
return shared->databases[db]->getCreateDictionaryQuery(*this, dictionary_name);
}
ASTPtr Context::getCreateExternalTableQuery(const String & table_name) const
{
TableAndCreateASTs::const_iterator jt = external_tables.find(table_name);
@ -1125,16 +1104,6 @@ ASTPtr Context::getCreateExternalTableQuery(const String & table_name) const
return jt->second.second;
}
ASTPtr Context::getCreateDatabaseQuery(const String & database_name) const
{
auto lock = getLock();
String db = resolveDatabase(database_name, current_database);
assertDatabaseExists(db);
return shared->databases[db]->getCreateDatabaseQuery(*this);
}
Settings Context::getSettings() const
{
return settings;

4
dbms/src/Interpreters/Context.h
View File

@ -153,6 +153,7 @@ private:
String default_format; /// Format, used when server formats data by itself and if query does not have FORMAT specification.
/// Thus, used in HTTP interface. If not specified - then some globally default format is used.
// TODO maybe replace with DatabaseMemory?
TableAndCreateASTs external_tables; /// Temporary tables.
Scalars scalars;
StoragePtr view_source; /// Temporary StorageValues used to generate alias columns for materialized views
@ -373,10 +374,7 @@ public:
std::optional<UInt16> getTCPPortSecure() const;
/// Get query for the CREATE table.
ASTPtr getCreateTableQuery(const String & database_name, const String & table_name) const;
ASTPtr getCreateExternalTableQuery(const String & table_name) const;
ASTPtr getCreateDatabaseQuery(const String & database_name) const;
ASTPtr getCreateDictionaryQuery(const String & database_name, const String & dictionary_name) const;
const DatabasePtr getDatabase(const String & database_name) const;
DatabasePtr getDatabase(const String & database_name);

2
dbms/src/Interpreters/ExternalLoaderDatabaseConfigRepository.cpp
View File

@ -37,7 +37,7 @@ bool ExternalLoaderDatabaseConfigRepository::exists(const std::string & loadable
Poco::Timestamp ExternalLoaderDatabaseConfigRepository::getUpdateTime(const std::string & loadable_definition_name)
{
return database->getObjectMetadataModificationTime(context, trimDatabaseName(loadable_definition_name, database));
return database->getObjectMetadataModificationTime(trimDatabaseName(loadable_definition_name, database));
}
std::set<std::string> ExternalLoaderDatabaseConfigRepository::getAllLoadablesDefinitionNames() const

10
dbms/src/Interpreters/InterpreterCreateQuery.cpp
View File

@ -511,7 +511,7 @@ void InterpreterCreateQuery::setEngine(ASTCreateQuery & create) const
String as_database_name = create.as_database.empty() ? context.getCurrentDatabase() : create.as_database;
String as_table_name = create.as_table;
ASTPtr as_create_ptr = context.getCreateTableQuery(as_database_name, as_table_name);
ASTPtr as_create_ptr = context.getDatabase(as_database_name)->getCreateTableQuery(context, as_table_name);
const auto & as_create = as_create_ptr->as<ASTCreateQuery &>();
if (as_create.is_view)
@ -549,7 +549,7 @@ BlockIO InterpreterCreateQuery::createTable(ASTCreateQuery & create)
if (create.attach && !create.storage && !create.columns_list)
{
// Table SQL definition is available even if the table is detached
auto query = context.getCreateTableQuery(create.database, create.table);
auto query = context.getDatabase(create.database)->getCreateTableQuery(context, create.table);
create = query->as<ASTCreateQuery &>(); // Copy the saved create query, but use ATTACH instead of CREATE
create.attach = true;
}
@ -583,7 +583,6 @@ bool InterpreterCreateQuery::doCreateTable(const ASTCreateQuery & create,
{
std::unique_ptr<DDLGuard> guard;
String data_path;
DatabasePtr database;
const String & table_name = create.table;
@ -591,7 +590,6 @@ bool InterpreterCreateQuery::doCreateTable(const ASTCreateQuery & create,
if (need_add_to_database)
{
database = context.getDatabase(create.database);
data_path = database->getDataPath();
/** If the request specifies IF NOT EXISTS, we allow concurrent CREATE queries (which do nothing).
* If table doesnt exist, one thread is creating table, while others wait in DDLGuard.
@ -632,7 +630,7 @@ bool InterpreterCreateQuery::doCreateTable(const ASTCreateQuery & create,
else
{
res = StorageFactory::instance().get(create,
data_path + escapeForFileName(table_name) + "/",
database ? database->getTableDataPath(create) : "",
table_name,
create.database,
context,
@ -710,7 +708,7 @@ BlockIO InterpreterCreateQuery::createDictionary(ASTCreateQuery & create)
if (create.attach)
{
auto query = context.getCreateDictionaryQuery(database_name, dictionary_name);
auto query = context.getDatabase(database_name)->getCreateDictionaryQuery(context, dictionary_name);
create = query->as<ASTCreateQuery &>();
create.attach = true;
}

9
dbms/src/Interpreters/InterpreterDropQuery.cpp
View File

@ -128,17 +128,16 @@ BlockIO InterpreterDropQuery::executeToTable(
throw;
}
String table_data_path_relative = database_and_table.first->getTableDataPath(table_name);
/// Delete table metadata and table itself from memory
database_and_table.first->removeTable(context, database_and_table.second->getTableName());
database_and_table.second->is_dropped = true;
String database_data_path = database_and_table.first->getDataPath();
/// If it is not virtual database like Dictionary then drop remaining data dir
if (!database_data_path.empty())
if (!table_data_path_relative.empty())
{
String table_data_path = context.getPath() + database_data_path + "/" + escapeForFileName(table_name);
String table_data_path = context.getPath() + table_data_path_relative;
if (Poco::File(table_data_path).exists())
Poco::File(table_data_path).remove(true);
}

6
dbms/src/Interpreters/InterpreterShowCreateQuery.cpp
View File

@ -49,19 +49,19 @@ BlockInputStreamPtr InterpreterShowCreateQuery::executeImpl()
if (show_query->temporary)
create_query = context.getCreateExternalTableQuery(show_query->table);
else
create_query = context.getCreateTableQuery(show_query->database, show_query->table);
create_query = context.getDatabase(show_query->database)->getCreateTableQuery(context, show_query->table);
}
else if ((show_query = query_ptr->as<ASTShowCreateDatabaseQuery>()))
{
if (show_query->temporary)
throw Exception("Temporary databases are not possible.", ErrorCodes::SYNTAX_ERROR);
create_query = context.getCreateDatabaseQuery(show_query->database);
create_query = context.getDatabase(show_query->database)->getCreateDatabaseQuery();
}
else if ((show_query = query_ptr->as<ASTShowCreateDictionaryQuery>()))
{
if (show_query->temporary)
throw Exception("Temporary dictionaries are not possible.", ErrorCodes::SYNTAX_ERROR);
create_query = context.getCreateDictionaryQuery(show_query->database, show_query->table);
create_query = context.getDatabase(show_query->database)->getCreateDictionaryQuery(context, show_query->table);
}
if (!create_query && show_query && show_query->temporary)

5
dbms/src/Interpreters/InterpreterSystemQuery.cpp
View File

@ -283,7 +283,7 @@ StoragePtr InterpreterSystemQuery::tryRestartReplica(const String & database_nam
/// If table was already dropped by anyone, an exception will be thrown
auto table_lock = table->lockExclusively(context.getCurrentQueryId());
create_ast = system_context.getCreateTableQuery(database_name, table_name);
create_ast = database->getCreateTableQuery(system_context, table_name);
database->detachTable(table_name);
}
@ -294,12 +294,11 @@ StoragePtr InterpreterSystemQuery::tryRestartReplica(const String & database_nam
auto & create = create_ast->as<ASTCreateQuery &>();
create.attach = true;
std::string data_path = database->getDataPath();
auto columns = InterpreterCreateQuery::getColumnsDescription(*create.columns_list->columns, system_context);
auto constraints = InterpreterCreateQuery::getConstraintsDescription(create.columns_list->constraints);
StoragePtr table = StorageFactory::instance().get(create,
data_path + escapeForFileName(table_name) + "/",
database->getTableDataPath(create),
table_name,
database_name,
system_context,

2
dbms/src/Storages/System/StorageSystemTables.cpp
View File

@ -258,7 +258,7 @@ protected:
res_columns[res_index++]->insert(database->getObjectMetadataPath(table_name));
if (columns_mask[src_index++])
res_columns[res_index++]->insert(static_cast<UInt64>(database->getObjectMetadataModificationTime(context, table_name)));
res_columns[res_index++]->insert(static_cast<UInt64>(database->getObjectMetadataModificationTime(table_name)));
{
Array dependencies_table_name_array;

4
dbms/tests/queries/0_stateless/01018_ddl_dictionaries_create.sql
View File

@ -89,7 +89,7 @@ CREATE DICTIONARY memory_db.dict2
PRIMARY KEY key_column
SOURCE(CLICKHOUSE(HOST 'localhost' PORT 9000 USER 'default' TABLE 'table_for_dict' PASSWORD '' DB 'database_for_dict'))
LIFETIME(MIN 1 MAX 10)
LAYOUT(FLAT()); -- {serverError 1}
LAYOUT(FLAT()); -- {serverError 48}
SHOW CREATE DICTIONARY memory_db.dict2; -- {serverError 487}
@ -114,7 +114,7 @@ CREATE DICTIONARY lazy_db.dict3
PRIMARY KEY key_column, second_column
SOURCE(CLICKHOUSE(HOST 'localhost' PORT 9000 USER 'default' TABLE 'table_for_dict' PASSWORD '' DB 'database_for_dict'))
LIFETIME(MIN 1 MAX 10)
LAYOUT(COMPLEX_KEY_HASHED()); -- {serverError 1}
LAYOUT(COMPLEX_KEY_HASHED()); -- {serverError 48}
DROP DATABASE IF EXISTS lazy_db;