thevar1able/ClickHouse
1
0
Fork 0
mirror of https://github.com/ClickHouse/ClickHouse.git synced 2024-11-24 08:32:02 +00:00
Code Issues Packages Projects Releases Wiki Activity

Better load of dictionaries

Browse Source
This commit is contained in:
alesapin 2019-10-16 20:06:52 +03:00
parent b4e0ded048
commit 01e23f0a22
16 changed files with 74 additions and 68 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
dbms/src/Databases/DatabaseDictionary.cpp
View File

@ -98,13 +98,13 @@ void DatabaseDictionary::removeDictionary(
}
void DatabaseDictionary::attachDictionary(
const String & /*dictionary_name*/)
const String & /*dictionary_name*/, const Context & /*context*/, bool /*load*/)
{
throw Exception("Dictionary engine doesn't support dictionaries.", ErrorCodes::UNSUPPORTED_METHOD);
}
void DatabaseDictionary::detachDictionary(
const String & /*dictionary_name*/)
const String & /*dictionary_name*/, const Context & /*context*/)
{
throw Exception("Dictionary engine doesn't support dictionaries.", ErrorCodes::UNSUPPORTED_METHOD);
}

4
dbms/src/Databases/DatabaseDictionary.h
View File

@ -89,9 +89,9 @@ public:
ASTPtr tryGetCreateDictionaryQuery(const Context & context, const String & table_name) const override;
void attachDictionary(const String & dictionary_name) override;
void attachDictionary(const String & dictionary_name, const Context & context, bool load) override;
void detachDictionary(const String & dictionary_name) override;
void detachDictionary(const String & dictionary_name, const Context & context) override;
void shutdown() override;

6
dbms/src/Databases/DatabaseLazy.cpp
View File

@ -122,12 +122,14 @@ DatabaseDictionariesIteratorPtr DatabaseLazy::getDictionariesIterator(
}
void DatabaseLazy::attachDictionary(
const String & /*dictionary_name*/)
const String & /*dictionary_name*/,
const Context & /*context*/,
bool /*load*/)
{
throw Exception("Lazy engine can be used only with *Log tables.", ErrorCodes::UNSUPPORTED_METHOD);
}
void DatabaseLazy::detachDictionary(const String & /*dictionary_name*/)
void DatabaseLazy::detachDictionary(const String & /*dictionary_name*/, const Context & /*context*/)
{
throw Exception("Lazy engine can be used only with *Log tables.", ErrorCodes::UNSUPPORTED_METHOD);
}

4
dbms/src/Databases/DatabaseLazy.h
View File

@ -111,9 +111,9 @@ public:
StoragePtr detachTable(const String & table_name) override;
void attachDictionary(const String & dictionary_name) override;
void attachDictionary(const String & dictionary_name, const Context & context, bool load) override;
void detachDictionary(const String & dictionary_name) override;
void detachDictionary(const String & dictionary_name, const Context & context) override;
void shutdown() override;

8
dbms/src/Databases/DatabaseMemory.cpp
View File

@ -35,11 +35,11 @@ void DatabaseMemory::createTable(
void DatabaseMemory::createDictionary(
const Context & /*context*/,
const Context & context,
const String & dictionary_name,
const ASTPtr & /*query*/)
{
attachDictionary(dictionary_name);
attachDictionary(dictionary_name, context, true);
}
@ -52,10 +52,10 @@ void DatabaseMemory::removeTable(
void DatabaseMemory::removeDictionary(
const Context &,
const Context & context,
const String & dictionary_name)
{
detachDictionary(dictionary_name);
detachDictionary(dictionary_name, context);
}

4
dbms/src/Databases/DatabaseMySQL.h
View File

@ -63,7 +63,7 @@ public:
throw Exception("MySQL database engine does not support detach table.", ErrorCodes::NOT_IMPLEMENTED);
}
void detachDictionary(const String &) override
void detachDictionary(const String &, const Context &) override
{
throw Exception("MySQL database engine does not support detach dictionary.", ErrorCodes::NOT_IMPLEMENTED);
}
@ -89,7 +89,7 @@ public:
throw Exception("MySQL database engine does not support attach table.", ErrorCodes::NOT_IMPLEMENTED);
}
void attachDictionary(const String &) override
void attachDictionary(const String &, const Context &, bool) override
{
throw Exception("MySQL database engine does not support attach dictionary.", ErrorCodes::NOT_IMPLEMENTED);
}

21
dbms/src/Databases/DatabaseOnDisk.cpp
View File

@ -296,21 +296,22 @@ void DatabaseOnDisk::createDictionary(
try
{
database.attachDictionary(dictionary_name);
/// Do not load it now
database.attachDictionary(dictionary_name, context, false);
/// If it was ATTACH query and file with table metadata already exist
/// (so, ATTACH is done after DETACH), then rename atomically replaces old file with new one.
Poco::File(dictionary_metadata_tmp_path).renameTo(dictionary_metadata_path);
/// Load dictionary
bool lazy_load = context.getConfigRef().getBool("dictionaries_lazy_load", true);
context.getExternalDictionariesLoader().reload(database.getDatabaseName() + "." + dictionary_name, !lazy_load);
}
catch (...)
{
Poco::File(dictionary_metadata_tmp_path).remove();
throw;
}
const auto & config = context.getConfigRef();
context.getExternalDictionariesLoader().reload(
database.getDatabaseName() + "." + dictionary_name, config.getBool("dictionaries_lazy_load", true));
}
@ -347,11 +348,11 @@ void DatabaseOnDisk::removeTable(
void DatabaseOnDisk::removeDictionary(
IDatabase & database,
const Context & /*context*/,
const Context & context,
const String & dictionary_name,
Poco::Logger * log)
{
database.detachDictionary(dictionary_name);
database.detachDictionary(dictionary_name, context);
String dictionary_metadata_path = database.getObjectMetadataPath(dictionary_name);
@ -370,7 +371,7 @@ void DatabaseOnDisk::removeDictionary(
{
LOG_WARNING(log, getCurrentExceptionMessage(__PRETTY_FUNCTION__));
}
database.attachDictionary(dictionary_name);
database.attachDictionary(dictionary_name, context);
throw;
}
}
@ -482,13 +483,9 @@ time_t DatabaseOnDisk::getObjectMetadataModificationTime(
Poco::File meta_file(table_metadata_path);
if (meta_file.exists())
{
return meta_file.getLastModified().epochTime();
}
else
{
return static_cast<time_t>(0);
}
}
void DatabaseOnDisk::iterateMetadataFiles(const IDatabase & database, Poco::Logger * log, const IteratingFunction & iterating_function)

18
dbms/src/Databases/DatabaseOrdinary.cpp
View File

@ -79,7 +79,7 @@ try
if (query.is_dictionary)
{
String dictionary_name = createDictionaryFromAST(query, database_name);
database.attachDictionary(dictionary_name);
database.attachDictionary(dictionary_name, context, false);
}
else
{
@ -125,6 +125,7 @@ void DatabaseOrdinary::loadStoredObjects(
Context & context,
bool has_force_restore_data_flag)
{
/** Tables load faster if they are loaded in sorted (by name) order.
* Otherwise (for the ext4 filesystem), `DirectoryIterator` iterates through them in some order,
* which does not correspond to order tables creation and does not correspond to order of their location on disk.
@ -183,7 +184,10 @@ void DatabaseOrdinary::loadStoredObjects(
/// After all tables was basically initialized, startup them.
startupTables(pool);
loadDictionaries(context);
/// Add database as repository
auto dictionaries_repository = std::make_unique<ExternalLoaderDatabaseConfigRepository>(shared_from_this(), context);
context.getExternalDictionariesLoader().addConfigRepository(
getDatabaseName(), std::move(dictionaries_repository), {"dictionary", "name"});
}
@ -210,16 +214,6 @@ void DatabaseOrdinary::startupTables(ThreadPool & thread_pool)
thread_pool.wait();
}
void DatabaseOrdinary::loadDictionaries(Context & context)
{
LOG_INFO(log, "Loading dictionaries.");
auto dictionaries_repository = std::make_unique<ExternalLoaderDatabaseConfigRepository>(shared_from_this(), context);
context.getExternalDictionariesLoader().addConfigRepository(
getDatabaseName(), std::move(dictionaries_repository), {"dictionary", "name"});
}
void DatabaseOrdinary::createTable(
const Context & context,
const String & table_name,

1
dbms/src/Databases/DatabaseOrdinary.h
View File

@ -91,7 +91,6 @@ private:
Poco::Logger * log;
void startupTables(ThreadPool & thread_pool);
void loadDictionaries(Context & context);
};
}

34
dbms/src/Databases/DatabasesCommon.cpp
View File

@ -1,5 +1,7 @@
#include <Databases/DatabasesCommon.h>
#include <Interpreters/ExternalDictionariesLoader.h>
#include <Interpreters/ExternalLoaderDatabaseConfigRepository.h>
#include <Interpreters/Context.h>
#include <Interpreters/InterpreterCreateQuery.h>
#include <Parsers/ASTCreateQuery.h>
@ -102,13 +104,17 @@ StoragePtr DatabaseWithOwnTablesBase::detachTable(const String & table_name)
return res;
}
void DatabaseWithOwnTablesBase::detachDictionary(const String & dictionary_name)
void DatabaseWithOwnTablesBase::detachDictionary(const String & dictionary_name, const Context & context)
{
std::lock_guard lock(mutex);
auto it = dictionaries.find(dictionary_name);
if (it == dictionaries.end())
throw Exception("Dictionary " + name + "." + dictionary_name + " doesn't exist.", ErrorCodes::UNKNOWN_TABLE);
dictionaries.erase(it);
{
std::lock_guard lock(mutex);
auto it = dictionaries.find(dictionary_name);
if (it == dictionaries.end())
throw Exception("Dictionary " + name + "." + dictionary_name + " doesn't exist.", ErrorCodes::UNKNOWN_TABLE);
dictionaries.erase(it);
}
context.getExternalDictionariesLoader().reload(getDatabaseName() + "." + dictionary_name, true);
}
@ -120,11 +126,19 @@ void DatabaseWithOwnTablesBase::attachTable(const String & table_name, const Sto
}
void DatabaseWithOwnTablesBase::attachDictionary(const String & dictionary_name)
void DatabaseWithOwnTablesBase::attachDictionary(const String & dictionary_name, const Context & context, bool load)
{
std::lock_guard lock(mutex);
if (!dictionaries.emplace(dictionary_name).second)
throw Exception("Dictionary " + name + "." + dictionary_name + " already exists.", ErrorCodes::DICTIONARY_ALREADY_EXISTS);
{
std::lock_guard lock(mutex);
if (!dictionaries.emplace(dictionary_name).second)
throw Exception("Dictionary " + name + "." + dictionary_name + " already exists.", ErrorCodes::DICTIONARY_ALREADY_EXISTS);
}
if (load)
{
bool lazy_load = context.getConfigRef().getBool("dictionaries_lazy_load", true);
context.getExternalDictionariesLoader().reload(getDatabaseName() + "." + dictionary_name, !lazy_load);
}
}
void DatabaseWithOwnTablesBase::shutdown()

4
dbms/src/Databases/DatabasesCommon.h
View File

@ -33,11 +33,11 @@ public:
void attachTable(const String & table_name, const StoragePtr & table) override;
void attachDictionary(const String & name) override;
void attachDictionary(const String & name, const Context & context, bool load) override;
StoragePtr detachTable(const String & table_name) override;
void detachDictionary(const String & name) override;
void detachDictionary(const String & name, const Context & context) override;
DatabaseTablesIteratorPtr getTablesIterator(const Context & context, const FilterByNameFunction & filter_by_table_name = {}) override;

4
dbms/src/Databases/IDatabase.h
View File

@ -159,13 +159,13 @@ public:
virtual void attachTable(const String & name, const StoragePtr & table) = 0;
/// Add dictionary to the database, but do not add it to the metadata. The database may not support this method.
virtual void attachDictionary(const String & name) = 0;
virtual void attachDictionary(const String & name, const Context & context, bool load=true) = 0;
/// Forget about the table without deleting it, and return it. The database may not support this method.
virtual StoragePtr detachTable(const String & name) = 0;
/// Forget about the dictionary without deleting it, and return it. The database may not support this method.
virtual void detachDictionary(const String & name) = 0;
virtual void detachDictionary(const String & name, const Context & context) = 0;
/// Rename the table and possibly move the table to another database.
virtual void renameTable(

14
dbms/src/Interpreters/ExternalLoader.cpp
View File

@ -112,8 +112,8 @@ private:
const auto names = repo_with_settings.first->getAllLoadablesDefinitionNames();
for (const auto & name : names)
{
auto it = loadables_infos.find(name);
if (it != loadables_infos.end())
auto it = loadables_infos.find(name);
if (it != loadables_infos.end())
{
LoadablesInfos & loadable_info = it->second;
if (readLoadablesInfo(*repo_with_settings.first, name, repo_with_settings.second, loadable_info))
@ -131,14 +131,14 @@ private:
}
}
std::vector<String> deleted_files;
std::vector<String> deleted_names;
for (auto & [path, loadable_info] : loadables_infos)
if (!loadable_info.in_use)
deleted_files.emplace_back(path);
if (!deleted_files.empty())
deleted_names.emplace_back(path);
if (!deleted_names.empty())
{
for (const String & deleted_file : deleted_files)
loadables_infos.erase(deleted_file);
for (const String & deleted_name : deleted_names)
loadables_infos.erase(deleted_name);
changed = true;
}
return changed;

2
dbms/src/Interpreters/InterpreterCreateQuery.cpp
View File

@ -729,7 +729,7 @@ BlockIO InterpreterCreateQuery::createDictionary(ASTCreateQuery & create)
}
if (create.attach)
database->attachDictionary(dictionary_name);
database->attachDictionary(dictionary_name, context);
else
database->createDictionary(context, dictionary_name, query_ptr);

2
dbms/src/Interpreters/InterpreterDropQuery.cpp
View File

@ -179,7 +179,7 @@ BlockIO InterpreterDropQuery::executeToDictionary(
if (kind == ASTDropQuery::Kind::Detach)
{
/// Drop dictionary from memory, don't touch data and metadata
database->detachDictionary(dictionary_name);
database->detachDictionary(dictionary_name, context);
}
else if (kind == ASTDropQuery::Kind::Truncate)
{

12
dbms/tests/queries/0_stateless/01018_ddl_dictionaries_create.sql
View File

@ -27,9 +27,9 @@ DROP DICTIONARY IF EXISTS ordinary_db.dict1;
CREATE DICTIONARY ordinary_db.dict1
(
key_column UInt64 DEFAULT 0,
second_column UInt8 DEFAULT 1,
third_column String DEFAULT 'qqq'
key_column UInt64 DEFAULT 0,
second_column UInt8 DEFAULT 1,
third_column String DEFAULT 'qqq'
)
PRIMARY KEY key_column
SOURCE(CLICKHOUSE(HOST 'localhost' PORT 9000 USER 'default' TABLE 'table_for_dict' PASSWORD '' DB 'database_for_dict'))
@ -138,9 +138,9 @@ SELECT '=DICTIONARY in Dictionary DB';
CREATE DICTIONARY dictionary_db.dict2
(
key_column UInt64 DEFAULT 0 INJECTIVE,
second_column UInt8 DEFAULT 1 EXPRESSION rand() % 222,
third_column String DEFAULT 'qqq'
key_column UInt64 DEFAULT 0 INJECTIVE,
second_column UInt8 DEFAULT 1 EXPRESSION rand() % 222,
third_column String DEFAULT 'qqq'
)
PRIMARY KEY key_column, second_column
SOURCE(CLICKHOUSE(HOST 'localhost' PORT 9000 USER 'default' TABLE 'table_for_dict' PASSWORD '' DB 'database_for_dict'))