thevar1able/ClickHouse
1
0
Fork 0
mirror of https://github.com/ClickHouse/ClickHouse.git synced 2024-11-11 18:14:03 +00:00
Code Issues Packages Projects Releases Wiki Activity

refactor databases: add DatabaseWithDictionaries

Browse Source
This commit is contained in:
Alexander Tokmakov 2019-11-05 23:26:14 +03:00
parent 73203a4b6b
commit 86e55e34d6
13 changed files with 165 additions and 152 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

7
dbms/src/Databases/DatabaseDictionary.cpp
View File

@ -65,13 +65,6 @@ bool DatabaseDictionary::isTableExist(
return context.getExternalDictionariesLoader().getCurrentStatus(table_name) != ExternalLoader::Status::NOT_EXIST;
}
DatabaseDictionariesIteratorPtr DatabaseDictionary::getDictionariesIterator(
const Context & /*context*/,
const FilterByNameFunction & /*filter_by_dictionary_name*/)
{
return std::make_unique<DatabaseDictionariesSnapshotIterator>();
}
StoragePtr DatabaseDictionary::tryGetTable(
const Context & context,
const String & table_name) const

2
dbms/src/Databases/DatabaseDictionary.h
View File

@ -39,8 +39,6 @@ public:
DatabaseTablesIteratorPtr getTablesIterator(const Context & context, const FilterByNameFunction & filter_by_table_name = {}) override;
DatabaseDictionariesIteratorPtr getDictionariesIterator(const Context & context, const FilterByNameFunction & filter_by_dictionary_name = {}) override;
bool empty(const Context & context) const override;
ASTPtr getCreateDatabaseQuery(const Context & context) const override;

7
dbms/src/Databases/DatabaseLazy.cpp
View File

@ -79,13 +79,6 @@ void DatabaseLazy::removeTable(
DatabaseOnDisk::removeTable(*this, context, table_name, log);
}
DatabaseDictionariesIteratorPtr DatabaseLazy::getDictionariesIterator(
const Context & /*context*/,
const FilterByNameFunction & /*filter_by_dictionary_name*/)
{
return std::make_unique<DatabaseDictionariesSnapshotIterator>();
}
void DatabaseLazy::renameTable(
const Context & context,
const String & table_name,

2
dbms/src/Databases/DatabaseLazy.h
View File

@ -75,8 +75,6 @@ public:
DatabaseTablesIteratorPtr getTablesIterator(const Context & context, const FilterByNameFunction & filter_by_table_name = {}) override;
DatabaseDictionariesIteratorPtr getDictionariesIterator(const Context & context, const FilterByNameFunction & filter_by_dictionary_name = {}) override;
void attachTable(const String & table_name, const StoragePtr & table) override;
StoragePtr detachTable(const String & table_name) override;

2
dbms/src/Databases/DatabaseMemory.h
View File

@ -33,8 +33,6 @@ public:
ASTPtr getCreateDatabaseQuery(const Context & context) const override;
//FIXME isDictionaryExist(...)
private:
Poco::Logger * log;
};

5
dbms/src/Databases/DatabaseMySQL.h
View File

@ -32,11 +32,6 @@ public:
DatabaseTablesIteratorPtr getTablesIterator(const Context & context, const FilterByNameFunction & filter_by_table_name = {}) override;
DatabaseDictionariesIteratorPtr getDictionariesIterator(const Context &, const FilterByNameFunction & = {}) override
{
return std::make_unique<DatabaseDictionariesSnapshotIterator>();
}
ASTPtr getCreateDatabaseQuery(const Context & context) const override;
bool isTableExist(const Context & context, const String & name) const override;

2
dbms/src/Databases/DatabaseOrdinary.cpp
View File

@ -115,7 +115,7 @@ namespace
DatabaseOrdinary::DatabaseOrdinary(String name_, const String & metadata_path_, const Context & context_)
: DatabaseWithOwnTablesBase(std::move(name_))
: DatabaseWithDictionaries(std::move(name_))
, metadata_path(metadata_path_)
, data_path("data/" + escapeForFileName(database_name) + "/")
, log(&Logger::get("DatabaseOrdinary (" + database_name + ")"))

4
dbms/src/Databases/DatabaseOrdinary.h
View File

@ -1,6 +1,6 @@
#pragma once
#include <Databases/DatabasesCommon.h>
#include <Databases/DatabaseWithDictionaries.h>
#include <Common/ThreadPool.h>
@ -11,7 +11,7 @@ namespace DB
* It stores tables list in filesystem using list of .sql files,
* that contain declaration of table represented by SQL ATTACH TABLE query.
*/
class DatabaseOrdinary : public DatabaseWithOwnTablesBase
class DatabaseOrdinary : public DatabaseWithDictionaries //DatabaseWithOwnTablesBase
{
public:
DatabaseOrdinary(String name_, const String & metadata_path_, const Context & context);

124
dbms/src/Databases/DatabaseWithDictionaries.cpp Normal file
View File

@ -0,0 +1,124 @@
#include <Databases/DatabaseWithDictionaries.h>
#include <Interpreters/ExternalDictionariesLoader.h>
#include <Interpreters/Context.h>
#include <Storages/StorageDictionary.h>
namespace DB
{
namespace ErrorCodes
{
extern const int EMPTY_LIST_OF_COLUMNS_PASSED;
extern const int TABLE_ALREADY_EXISTS;
extern const int UNKNOWN_TABLE;
extern const int LOGICAL_ERROR;
extern const int DICTIONARY_ALREADY_EXISTS;
}
void DatabaseWithDictionaries::attachDictionary(const String & dictionary_name, const Context & context)
{
String full_name = getDatabaseName() + "." + dictionary_name;
{
std::lock_guard lock(mutex);
if (!dictionaries.emplace(dictionary_name).second)
throw Exception("Dictionary " + full_name + " already exists.", ErrorCodes::DICTIONARY_ALREADY_EXISTS);
}
/// ExternalLoader::reloadConfig() will find out that the dictionary's config has been added
/// and in case `dictionaries_lazy_load == false` it will load the dictionary.
const auto & external_loader = context.getExternalDictionariesLoader();
external_loader.reloadConfig(getDatabaseName(), full_name);
}
void DatabaseWithDictionaries::detachDictionary(const String & dictionary_name, const Context & context)
{
String full_name = getDatabaseName() + "." + dictionary_name;
{
std::lock_guard lock(mutex);
auto it = dictionaries.find(dictionary_name);
if (it == dictionaries.end())
throw Exception("Dictionary " + full_name + " doesn't exist.", ErrorCodes::UNKNOWN_TABLE);
dictionaries.erase(it);
}
/// ExternalLoader::reloadConfig() will find out that the dictionary's config has been removed
/// and therefore it will unload the dictionary.
const auto & external_loader = context.getExternalDictionariesLoader();
external_loader.reloadConfig(getDatabaseName(), full_name);
}
StoragePtr DatabaseWithDictionaries::tryGetTable(const Context & context, const String & table_name) const
{
if (auto table_ptr = DatabaseWithOwnTablesBase::tryGetTable(context, table_name))
return table_ptr;
if (isDictionaryExist(context, table_name))
/// We don't need lock database here, because database doesn't store dictionary itself
/// just metadata
return getDictionaryStorage(context, table_name);
return {};
}
DatabaseTablesIteratorPtr DatabaseWithDictionaries::getTablesWithDictionaryTablesIterator(const Context & context, const FilterByNameFunction & filter_by_name)
{
/// NOTE: it's not atomic
auto tables_it = getTablesIterator(context, filter_by_name);
auto dictionaries_it = getDictionariesIterator(context, filter_by_name);
Tables result;
while (tables_it && tables_it->isValid())
{
result.emplace(tables_it->name(), tables_it->table());
tables_it->next();
}
while (dictionaries_it && dictionaries_it->isValid())
{
auto table_name = dictionaries_it->name();
auto table_ptr = getDictionaryStorage(context, table_name);
if (table_ptr)
result.emplace(table_name, table_ptr);
dictionaries_it->next();
}
return std::make_unique<DatabaseTablesSnapshotIterator>(result);
}
DatabaseDictionariesIteratorPtr DatabaseWithDictionaries::getDictionariesIterator(const Context & /*context*/, const FilterByNameFunction & filter_by_dictionary_name)
{
std::lock_guard lock(mutex);
if (!filter_by_dictionary_name)
return std::make_unique<DatabaseDictionariesSnapshotIterator>(dictionaries);
Dictionaries filtered_dictionaries;
for (const auto & dictionary_name : dictionaries)
if (filter_by_dictionary_name(dictionary_name))
filtered_dictionaries.emplace(dictionary_name);
return std::make_unique<DatabaseDictionariesSnapshotIterator>(std::move(filtered_dictionaries));
}
bool DatabaseWithDictionaries::isDictionaryExist(const Context & /*context*/, const String & dictionary_name) const
{
std::lock_guard lock(mutex);
return dictionaries.find(dictionary_name) != dictionaries.end();
}
StoragePtr DatabaseWithDictionaries::getDictionaryStorage(const Context & context, const String & table_name) const
{
auto dict_name = database_name + "." + table_name;
const auto & external_loader = context.getExternalDictionariesLoader();
auto dict_ptr = external_loader.tryGetDictionary(dict_name);
if (dict_ptr)
{
const DictionaryStructure & dictionary_structure = dict_ptr->getStructure();
auto columns = StorageDictionary::getNamesAndTypes(dictionary_structure);
return StorageDictionary::create(database_name, table_name, ColumnsDescription{columns}, context, true, dict_name);
}
return nullptr;
}
}

29
dbms/src/Databases/DatabaseWithDictionaries.h Normal file
View File

@ -0,0 +1,29 @@
#include <Databases/DatabasesCommon.h>
namespace DB
{
class DatabaseWithDictionaries : public DatabaseWithOwnTablesBase
{
public:
void attachDictionary(const String & name, const Context & context) override;
void detachDictionary(const String & name, const Context & context) override;
StoragePtr tryGetTable(const Context & context, const String & table_name) const override;
DatabaseTablesIteratorPtr getTablesWithDictionaryTablesIterator(const Context & context, const FilterByNameFunction & filter_by_dictionary_name = {}) override;
DatabaseDictionariesIteratorPtr getDictionariesIterator(const Context & context, const FilterByNameFunction & filter_by_dictionary_name = {}) override;
bool isDictionaryExist(const Context & context, const String & dictionary_name) const override;
protected:
DatabaseWithDictionaries(String name) : DatabaseWithOwnTablesBase(std::move(name)) {}
StoragePtr getDictionaryStorage(const Context & context, const String & table_name) const;
};
}

118
dbms/src/Databases/DatabasesCommon.cpp
View File

@ -1,6 +1,4 @@
#include <Databases/DatabasesCommon.h>
#include <Interpreters/ExternalDictionariesLoader.h>
#include <Interpreters/Context.h>
#include <Interpreters/InterpreterCreateQuery.h>
#include <Parsers/ParserCreateQuery.h>
#include <Parsers/formatAST.h>
@ -22,25 +20,6 @@ namespace ErrorCodes
extern const int DICTIONARY_ALREADY_EXISTS;
}
namespace
{
StoragePtr getDictionaryStorage(const Context & context, const String & table_name, const String & db_name)
{
auto dict_name = db_name + "." + table_name;
const auto & external_loader = context.getExternalDictionariesLoader();
auto dict_ptr = external_loader.tryGetDictionary(dict_name);
if (dict_ptr)
{
const DictionaryStructure & dictionary_structure = dict_ptr->getStructure();
auto columns = StorageDictionary::getNamesAndTypes(dictionary_structure);
return StorageDictionary::create(db_name, table_name, ColumnsDescription{columns}, context, true, dict_name);
}
return nullptr;
}
}
bool DatabaseWithOwnTablesBase::isTableExist(
const Context & /*context*/,
const String & table_name) const
@ -49,57 +28,17 @@ bool DatabaseWithOwnTablesBase::isTableExist(
return tables.find(table_name) != tables.end() || dictionaries.find(table_name) != dictionaries.end();
}
bool DatabaseWithOwnTablesBase::isDictionaryExist(
const Context & /*context*/,
const String & dictionary_name) const
{
std::lock_guard lock(mutex);
return dictionaries.find(dictionary_name) != dictionaries.end();
}
StoragePtr DatabaseWithOwnTablesBase::tryGetTable(
const Context & context,
const Context & /*context*/,
const String & table_name) const
{
{
std::lock_guard lock(mutex);
auto it = tables.find(table_name);
if (it != tables.end())
return it->second;
}
if (isDictionaryExist(context, table_name))
/// We don't need lock database here, because database doesn't store dictionary itself
/// just metadata
return getDictionaryStorage(context, table_name, getDatabaseName());
std::lock_guard lock(mutex);
auto it = tables.find(table_name);
if (it != tables.end())
return it->second;
return {};
}
DatabaseTablesIteratorPtr DatabaseWithOwnTablesBase::getTablesWithDictionaryTablesIterator(const Context & context, const FilterByNameFunction & filter_by_name)
{
auto tables_it = getTablesIterator(context, filter_by_name);
auto dictionaries_it = getDictionariesIterator(context, filter_by_name);
Tables result;
while (tables_it && tables_it->isValid())
{
result.emplace(tables_it->name(), tables_it->table());
tables_it->next();
}
while (dictionaries_it && dictionaries_it->isValid())
{
auto table_name = dictionaries_it->name();
auto table_ptr = getDictionaryStorage(context, table_name, getDatabaseName());
if (table_ptr)
result.emplace(table_name, table_ptr);
dictionaries_it->next();
}
return std::make_unique<DatabaseTablesSnapshotIterator>(result);
}
DatabaseTablesIteratorPtr DatabaseWithOwnTablesBase::getTablesIterator(const Context & /*context*/, const FilterByNameFunction & filter_by_table_name)
{
std::lock_guard lock(mutex);
@ -114,20 +53,6 @@ DatabaseTablesIteratorPtr DatabaseWithOwnTablesBase::getTablesIterator(const Con
return std::make_unique<DatabaseTablesSnapshotIterator>(std::move(filtered_tables));
}
DatabaseDictionariesIteratorPtr DatabaseWithOwnTablesBase::getDictionariesIterator(const Context & /*context*/, const FilterByNameFunction & filter_by_dictionary_name)
{
std::lock_guard lock(mutex);
if (!filter_by_dictionary_name)
return std::make_unique<DatabaseDictionariesSnapshotIterator>(dictionaries);
Dictionaries filtered_dictionaries;
for (const auto & dictionary_name : dictionaries)
if (filter_by_dictionary_name(dictionary_name))
filtered_dictionaries.emplace(dictionary_name);
return std::make_unique<DatabaseDictionariesSnapshotIterator>(std::move(filtered_dictionaries));
}
bool DatabaseWithOwnTablesBase::empty(const Context & /*context*/) const
{
std::lock_guard lock(mutex);
@ -152,23 +77,6 @@ StoragePtr DatabaseWithOwnTablesBase::detachTable(const String & table_name)
return res;
}
void DatabaseWithOwnTablesBase::detachDictionary(const String & dictionary_name, const Context & context)
{
String full_name = getDatabaseName() + "." + dictionary_name;
{
std::lock_guard lock(mutex);
auto it = dictionaries.find(dictionary_name);
if (it == dictionaries.end())
throw Exception("Dictionary " + full_name + " doesn't exist.", ErrorCodes::UNKNOWN_TABLE);
dictionaries.erase(it);
}
/// ExternalLoader::reloadConfig() will find out that the dictionary's config has been removed
/// and therefore it will unload the dictionary.
const auto & external_loader = context.getExternalDictionariesLoader();
external_loader.reloadConfig(getDatabaseName(), full_name);
}
void DatabaseWithOwnTablesBase::attachTable(const String & table_name, const StoragePtr & table)
{
std::lock_guard lock(mutex);
@ -176,22 +84,6 @@ void DatabaseWithOwnTablesBase::attachTable(const String & table_name, const Sto
throw Exception("Table " + database_name + "." + table_name + " already exists.", ErrorCodes::TABLE_ALREADY_EXISTS);
}
void DatabaseWithOwnTablesBase::attachDictionary(const String & dictionary_name, const Context & context)
{
String full_name = getDatabaseName() + "." + dictionary_name;
{
std::lock_guard lock(mutex);
if (!dictionaries.emplace(dictionary_name).second)
throw Exception("Dictionary " + full_name + " already exists.", ErrorCodes::DICTIONARY_ALREADY_EXISTS);
}
/// ExternalLoader::reloadConfig() will find out that the dictionary's config has been added
/// and in case `dictionaries_lazy_load == false` it will load the dictionary.
const auto & external_loader = context.getExternalDictionariesLoader();
external_loader.reloadConfig(getDatabaseName(), full_name);
}
void DatabaseWithOwnTablesBase::shutdown()
{
/// You can not hold a lock during shutdown.

10
dbms/src/Databases/DatabasesCommon.h
View File

@ -23,8 +23,6 @@ public:
const Context & context,
const String & table_name) const override;
bool isDictionaryExist(const Context & context, const String & dictionary_name) const override;
StoragePtr tryGetTable(
const Context & context,
const String & table_name) const override;
@ -33,18 +31,10 @@ public:
void attachTable(const String & table_name, const StoragePtr & table) override;
void attachDictionary(const String & name, const Context & context) override;
StoragePtr detachTable(const String & table_name) override;
void detachDictionary(const String & name, const Context & context) override;
DatabaseTablesIteratorPtr getTablesIterator(const Context & context, const FilterByNameFunction & filter_by_table_name = {}) override;
DatabaseDictionariesIteratorPtr getDictionariesIterator(const Context & context, const FilterByNameFunction & filter_by_dictionary_name = {}) override;
DatabaseTablesIteratorPtr getTablesWithDictionaryTablesIterator(const Context & context, const FilterByNameFunction & filter_by_dictionary_name = {}) override;
void shutdown() override;
virtual ~DatabaseWithOwnTablesBase() override;

5
dbms/src/Databases/IDatabase.h
View File

@ -134,7 +134,10 @@ public:
virtual DatabaseTablesIteratorPtr getTablesIterator(const Context & context, const FilterByNameFunction & filter_by_table_name = {}) = 0;
/// Get an iterator to pass through all the dictionaries.
virtual DatabaseDictionariesIteratorPtr getDictionariesIterator(const Context & context, const FilterByNameFunction & filter_by_dictionary_name = {}) = 0;
virtual DatabaseDictionariesIteratorPtr getDictionariesIterator(const Context & /*context*/, [[maybe_unused]] const FilterByNameFunction & filter_by_dictionary_name = {})
{
return std::make_unique<DatabaseDictionariesSnapshotIterator>();
}
/// Get an iterator to pass through all the tables and dictionary tables.
virtual DatabaseTablesIteratorPtr getTablesWithDictionaryTablesIterator(const Context & context, const FilterByNameFunction & filter_by_name = {})