Almost working

This commit is contained in:
alesapin 2019-10-16 17:59:52 +03:00
parent e690a3ca32
commit b4e0ded048
21 changed files with 63 additions and 131 deletions

View File

@ -73,13 +73,6 @@ bool DatabaseDictionary::isDictionaryExist(
return false;
}
DictionaryPtr DatabaseDictionary::tryGetDictionary(
const Context & /*context*/,
const String & /*dictionary_name*/) const
{
return nullptr;
}
DatabaseDictionariesIteratorPtr DatabaseDictionary::getDictionariesIterator(
const Context & /*context*/,
@ -92,7 +85,6 @@ DatabaseDictionariesIteratorPtr DatabaseDictionary::getDictionariesIterator(
void DatabaseDictionary::createDictionary(
const Context & /*context*/,
const String & /*dictionary_name*/,
const DictionaryPtr & /*dict_ptr*/,
const ASTPtr & /*query*/)
{
throw Exception("Dictionary engine doesn't support dictionaries.", ErrorCodes::UNSUPPORTED_METHOD);
@ -106,13 +98,12 @@ void DatabaseDictionary::removeDictionary(
}
void DatabaseDictionary::attachDictionary(
const String & /*dictionary_name*/,
const DictionaryPtr & /*table*/)
const String & /*dictionary_name*/)
{
throw Exception("Dictionary engine doesn't support dictionaries.", ErrorCodes::UNSUPPORTED_METHOD);
}
DictionaryPtr DatabaseDictionary::detachDictionary(
void DatabaseDictionary::detachDictionary(
const String & /*dictionary_name*/)
{
throw Exception("Dictionary engine doesn't support dictionaries.", ErrorCodes::UNSUPPORTED_METHOD);

View File

@ -41,13 +41,10 @@ public:
bool isDictionaryExist(const Context & context, const String & table_name) const override;
StoragePtr tryGetTable(
const Context & context,
const String & table_name) const override;
DictionaryPtr tryGetDictionary(const Context & context, const String & dictionary_name) const override;
DatabaseTablesIteratorPtr getTablesIterator(const Context & context, const FilterByNameFunction & filter_by_table_name = {}) override;
DatabaseDictionariesIteratorPtr getDictionariesIterator(const Context & context, const FilterByNameFunction & filter_by_dictionary_name = {}) override;
@ -61,7 +58,7 @@ public:
const ASTPtr & query) override;
void createDictionary(
const Context & context, const String & dictionary_name, const DictionaryPtr & dict_ptr, const ASTPtr & query) override;
const Context & context, const String & dictionary_name, const ASTPtr & query) override;
void removeTable(
const Context & context,
@ -92,10 +89,9 @@ public:
ASTPtr tryGetCreateDictionaryQuery(const Context & context, const String & table_name) const override;
void attachDictionary(const String & dictionary_name, const DictionaryPtr & table) override;
DictionaryPtr detachDictionary(const String & dictionary_name) override;
void attachDictionary(const String & dictionary_name) override;
void detachDictionary(const String & dictionary_name) override;
void shutdown() override;

View File

@ -75,7 +75,6 @@ void DatabaseLazy::createTable(
void DatabaseLazy::createDictionary(
const Context & /*context*/,
const String & /*dictionary_name*/,
const DictionaryPtr & /*dict_ptr*/,
const ASTPtr & /*query*/)
{
throw Exception("Lazy engine can be used only with *Log tables.", ErrorCodes::UNSUPPORTED_METHOD);
@ -114,10 +113,6 @@ bool DatabaseLazy::isDictionaryExist(const Context & /*context*/, const String &
return false;
}
DictionaryPtr DatabaseLazy::tryGetDictionary(const Context & /*context*/, const String & /*dictionary_name*/) const
{
return nullptr;
}
DatabaseDictionariesIteratorPtr DatabaseLazy::getDictionariesIterator(
const Context & /*context*/,
@ -127,13 +122,12 @@ DatabaseDictionariesIteratorPtr DatabaseLazy::getDictionariesIterator(
}
void DatabaseLazy::attachDictionary(
const String & /*dictionary_name*/,
const DictionaryPtr & /*table*/)
const String & /*dictionary_name*/)
{
throw Exception("Lazy engine can be used only with *Log tables.", ErrorCodes::UNSUPPORTED_METHOD);
}
DictionaryPtr DatabaseLazy::detachDictionary(const String & /*dictionary_name*/)
void DatabaseLazy::detachDictionary(const String & /*dictionary_name*/)
{
throw Exception("Lazy engine can be used only with *Log tables.", ErrorCodes::UNSUPPORTED_METHOD);
}

View File

@ -35,7 +35,6 @@ public:
void createDictionary(
const Context & context,
const String & dictionary_name,
const DictionaryPtr & dict_ptr,
const ASTPtr & query) override;
void removeTable(
@ -102,8 +101,6 @@ public:
const Context & context,
const String & table_name) const override;
DictionaryPtr tryGetDictionary(const Context & context, const String & dictionary_name) const override;
bool empty(const Context & context) const override;
DatabaseTablesIteratorPtr getTablesIterator(const Context & context, const FilterByNameFunction & filter_by_table_name = {}) override;
@ -114,9 +111,9 @@ public:
StoragePtr detachTable(const String & table_name) override;
void attachDictionary(const String & dictionary_name, const DictionaryPtr & table) override;
void attachDictionary(const String & dictionary_name) override;
DictionaryPtr detachDictionary(const String & dictionary_name) override;
void detachDictionary(const String & dictionary_name) override;
void shutdown() override;

View File

@ -37,10 +37,9 @@ void DatabaseMemory::createTable(
void DatabaseMemory::createDictionary(
const Context & /*context*/,
const String & dictionary_name,
const DictionaryPtr & dictionary,
const ASTPtr & /*query*/)
{
attachDictionary(dictionary_name, dictionary);
attachDictionary(dictionary_name);
}

View File

@ -36,7 +36,6 @@ public:
void createDictionary(
const Context & context,
const String & dictionary_name,
const DictionaryPtr & dictionary,
const ASTPtr & query) override;
void removeTable(

View File

@ -44,8 +44,6 @@ public:
StoragePtr tryGetTable(const Context & context, const String & name) const override;
DictionaryPtr tryGetDictionary(const Context &, const String &) const override { return {}; }
ASTPtr tryGetCreateTableQuery(const Context & context, const String & name) const override;
ASTPtr getCreateDictionaryQuery(const Context &, const String &) const override
@ -65,7 +63,7 @@ public:
throw Exception("MySQL database engine does not support detach table.", ErrorCodes::NOT_IMPLEMENTED);
}
DictionaryPtr detachDictionary(const String &) override
void detachDictionary(const String &) override
{
throw Exception("MySQL database engine does not support detach dictionary.", ErrorCodes::NOT_IMPLEMENTED);
}
@ -91,7 +89,7 @@ public:
throw Exception("MySQL database engine does not support attach table.", ErrorCodes::NOT_IMPLEMENTED);
}
void attachDictionary(const String &, const DictionaryPtr &) override
void attachDictionary(const String &) override
{
throw Exception("MySQL database engine does not support attach dictionary.", ErrorCodes::NOT_IMPLEMENTED);
}
@ -101,7 +99,7 @@ public:
throw Exception("MySQL database engine does not support create table.", ErrorCodes::NOT_IMPLEMENTED);
}
void createDictionary(const Context &, const String &, const DictionaryPtr &, const ASTPtr &) override
void createDictionary(const Context &, const String &, const ASTPtr &) override
{
throw Exception("MySQL database engine does not support create dictionary.", ErrorCodes::NOT_IMPLEMENTED);
}

View File

@ -255,7 +255,6 @@ void DatabaseOnDisk::createDictionary(
IDatabase & database,
const Context & context,
const String & dictionary_name,
const DictionaryPtr & dictionary,
const ASTPtr & query)
{
const auto & settings = context.getSettingsRef();
@ -297,8 +296,7 @@ void DatabaseOnDisk::createDictionary(
try
{
/// Add a table to the map of known tables.
database.attachDictionary(dictionary_name, dictionary);
database.attachDictionary(dictionary_name);
/// If it was ATTACH query and file with table metadata already exist
/// (so, ATTACH is done after DETACH), then rename atomically replaces old file with new one.
@ -353,7 +351,7 @@ void DatabaseOnDisk::removeDictionary(
const String & dictionary_name,
Poco::Logger * log)
{
DictionaryPtr res = database.detachDictionary(dictionary_name);
database.detachDictionary(dictionary_name);
String dictionary_metadata_path = database.getObjectMetadataPath(dictionary_name);
@ -372,7 +370,7 @@ void DatabaseOnDisk::removeDictionary(
{
LOG_WARNING(log, getCurrentExceptionMessage(__PRETTY_FUNCTION__));
}
database.attachDictionary(dictionary_name, res);
database.attachDictionary(dictionary_name);
throw;
}
}
@ -417,9 +415,9 @@ ASTPtr DatabaseOnDisk::getCreateDictionaryQueryImpl(
if (!ast && throw_on_error)
{
/// Handle system.* tables for which there are no table.sql files.
bool has_table = database.tryGetDictionary(context, dictionary_name) != nullptr;
bool has_dictionary = database.isDictionaryExist(context, dictionary_name);
auto msg = has_table ? "There is no CREATE DICTIONARY query for table " : "There is no metadata file for dictionary ";
auto msg = has_dictionary ? "There is no CREATE DICTIONARY query for table " : "There is no metadata file for dictionary ";
throw Exception(msg + backQuote(dictionary_name), ErrorCodes::CANNOT_GET_CREATE_DICTIONARY_QUERY);
}

View File

@ -51,7 +51,6 @@ public:
IDatabase & database,
const Context & context,
const String & dictionary_name,
const DictionaryPtr & dictionary,
const ASTPtr & query);
static void removeTable(

View File

@ -55,21 +55,16 @@ namespace
{
std::pair<String, DictionaryPtr> createDictionaryFromAST(
String createDictionaryFromAST(
ASTCreateQuery ast_create_query,
const String & database_name,
const Context & context)
const String & database_name)
{
ast_create_query.database = database_name;
if (!ast_create_query.dictionary_attributes_list)
throw Exception("Missing definition of dictionary attributes.", ErrorCodes::EMPTY_LIST_OF_ATTRIBUTES_PASSED);
return
{
ast_create_query.table,
DictionaryFactory::instance().create(ast_create_query.table, ast_create_query, context)
};
return ast_create_query.table;
}
void loadObject(
@ -83,10 +78,8 @@ try
{
if (query.is_dictionary)
{
String dictionary_name;
DictionaryPtr dictionary;
std::tie(dictionary_name, dictionary) = createDictionaryFromAST(query, database_name, context);
database.attachDictionary(dictionary_name, dictionary);
String dictionary_name = createDictionaryFromAST(query, database_name);
database.attachDictionary(dictionary_name);
}
else
{
@ -222,7 +215,8 @@ void DatabaseOrdinary::loadDictionaries(Context & context)
LOG_INFO(log, "Loading dictionaries.");
auto dictionaries_repository = std::make_unique<ExternalLoaderDatabaseConfigRepository>(shared_from_this(), context);
context.getExternalDictionariesLoader().addConfigRepository(getDatabaseName(), std::move(dictionaries_repository), {});
context.getExternalDictionariesLoader().addConfigRepository(
getDatabaseName(), std::move(dictionaries_repository), {"dictionary", "name"});
}
@ -238,10 +232,9 @@ void DatabaseOrdinary::createTable(
void DatabaseOrdinary::createDictionary(
const Context & context,
const String & dictionary_name,
const DictionaryPtr & dictionary,
const ASTPtr & query)
{
DatabaseOnDisk::createDictionary(*this, context, dictionary_name, dictionary, query);
DatabaseOnDisk::createDictionary(*this, context, dictionary_name, query);
}
void DatabaseOrdinary::removeTable(

View File

@ -31,7 +31,6 @@ public:
void createDictionary(
const Context & context,
const String & dictionary_name,
const DictionaryPtr & dict_ptr,
const ASTPtr & query) override;
void removeTable(

View File

@ -55,16 +55,6 @@ StoragePtr DatabaseWithOwnTablesBase::tryGetTable(
return it->second;
}
DictionaryPtr DatabaseWithOwnTablesBase::tryGetDictionary(const Context & /*context*/, const String & dictionary_name) const
{
std::lock_guard dict_lock{mutex};
auto it = dictionaries.find(dictionary_name);
if (it == dictionaries.end())
return {};
return it->second;
}
DatabaseTablesIteratorPtr DatabaseWithOwnTablesBase::getTablesIterator(const Context & /*context*/, const FilterByNameFunction & filter_by_table_name)
{
std::lock_guard lock(mutex);
@ -85,9 +75,9 @@ DatabaseDictionariesIteratorPtr DatabaseWithOwnTablesBase::getDictionariesIterat
return std::make_unique<DatabaseDictionariesSnapshotIterator>(dictionaries);
Dictionaries filtered_dictionaries;
for (const auto & [dictionary_name, dictionary] : dictionaries)
for (const auto & dictionary_name : dictionaries)
if (filter_by_dictionary_name(dictionary_name))
filtered_dictionaries.emplace(dictionary_name, dictionary);
filtered_dictionaries.emplace(dictionary_name);
return std::make_unique<DatabaseDictionariesSnapshotIterator>(std::move(filtered_dictionaries));
}
@ -112,19 +102,14 @@ StoragePtr DatabaseWithOwnTablesBase::detachTable(const String & table_name)
return res;
}
DictionaryPtr DatabaseWithOwnTablesBase::detachDictionary(const String & dictionary_name)
void DatabaseWithOwnTablesBase::detachDictionary(const String & dictionary_name)
{
DictionaryPtr res;
{
std::lock_guard lock(mutex);
auto it = dictionaries.find(dictionary_name);
if (it == dictionaries.end())
throw Exception("Dictionary " + name + "." + dictionary_name + " doesn't exist.", ErrorCodes::UNKNOWN_TABLE);
res = it->second;
dictionaries.erase(it);
}
std::lock_guard lock(mutex);
auto it = dictionaries.find(dictionary_name);
if (it == dictionaries.end())
throw Exception("Dictionary " + name + "." + dictionary_name + " doesn't exist.", ErrorCodes::UNKNOWN_TABLE);
dictionaries.erase(it);
return res;
}
void DatabaseWithOwnTablesBase::attachTable(const String & table_name, const StoragePtr & table)
@ -135,10 +120,10 @@ void DatabaseWithOwnTablesBase::attachTable(const String & table_name, const Sto
}
void DatabaseWithOwnTablesBase::attachDictionary(const String & dictionary_name, const DictionaryPtr & dictionary)
void DatabaseWithOwnTablesBase::attachDictionary(const String & dictionary_name)
{
std::lock_guard lock(mutex);
if (!dictionaries.emplace(dictionary_name, dictionary).second)
if (!dictionaries.emplace(dictionary_name).second)
throw Exception("Dictionary " + name + "." + dictionary_name + " already exists.", ErrorCodes::DICTIONARY_ALREADY_EXISTS);
}

View File

@ -29,17 +29,15 @@ public:
const Context & context,
const String & table_name) const override;
DictionaryPtr tryGetDictionary(const Context & context, const String & dictionary_name) const override;
bool empty(const Context & context) const override;
void attachTable(const String & table_name, const StoragePtr & table) override;
void attachDictionary(const String & name, const DictionaryPtr & dictionary) override;
void attachDictionary(const String & name) override;
StoragePtr detachTable(const String & table_name) override;
DictionaryPtr detachDictionary(const String & name) override;
void detachDictionary(const String & name) override;
DatabaseTablesIteratorPtr getTablesIterator(const Context & context, const FilterByNameFunction & filter_by_table_name = {}) override;

View File

@ -20,6 +20,7 @@ struct ConstraintsDescription;
class ColumnsDescription;
struct IndicesDescription;
struct TableStructureWriteLockHolder;
using Dictionaries = std::set<String>;
namespace ErrorCodes
{
@ -76,9 +77,7 @@ public:
bool isValid() const { return !dictionaries.empty() && it != dictionaries.end(); }
const String & name() const { return it->first; }
DictionaryPtr & dictionary() const { return it->second; }
const String & name() const { return *it; }
};
using DatabaseTablesIteratorPtr = std::unique_ptr<IDatabaseTablesIterator>;
@ -121,11 +120,6 @@ public:
const Context & context,
const String & name) const = 0;
/// Get the dictionary for work. Return nullptr if there is no table.
virtual DictionaryPtr tryGetDictionary(
const Context & context,
const String & name) const = 0;
using FilterByNameFunction = std::function<bool(const String &)>;
/// Get an iterator that allows you to pass through all the tables.
@ -149,7 +143,6 @@ public:
virtual void createDictionary(
const Context & context,
const String & dictionary_name,
const DictionaryPtr & dict_ptr,
const ASTPtr & query) = 0;
/// Delete the table from the database. Delete the metadata.
@ -166,13 +159,13 @@ public:
virtual void attachTable(const String & name, const StoragePtr & table) = 0;
/// Add dictionary to the database, but do not add it to the metadata. The database may not support this method.
virtual void attachDictionary(const String & name, const DictionaryPtr & dictionary) = 0;
virtual void attachDictionary(const String & name) = 0;
/// Forget about the table without deleting it, and return it. The database may not support this method.
virtual StoragePtr detachTable(const String & name) = 0;
/// Forget about the dictionary without deleting it, and return it. The database may not support this method.
virtual DictionaryPtr detachDictionary(const String & name) = 0;
virtual void detachDictionary(const String & name) = 0;
/// Rename the table and possibly move the table to another database.
virtual void renameTable(

View File

@ -35,7 +35,7 @@ DictionaryPtr DictionaryFactory::create(
const DictionaryStructure dict_struct{config, config_prefix + ".structure"};
auto source_ptr = DictionarySourceFactory::instance().create(name, config, config_prefix + ".source", dict_struct, context);
DictionarySourcePtr source_ptr = DictionarySourceFactory::instance().create(name, config, config_prefix + ".source", dict_struct, context);
const auto & layout_type = keys.front();
@ -43,8 +43,8 @@ DictionaryPtr DictionaryFactory::create(
const auto found = registered_layouts.find(layout_type);
if (found != registered_layouts.end())
{
const auto & create_layout = found->second;
return create_layout(name, dict_struct, config, config_prefix, std::move(source_ptr));
const auto & layout_creator = found->second;
return layout_creator(name, dict_struct, config, config_prefix, std::move(source_ptr));
}
}

View File

@ -2,7 +2,7 @@
#include <Core/Types.h>
#include <map>
#include <set>
#include <memory>
namespace DB
@ -10,6 +10,5 @@ namespace DB
struct IDictionaryBase;
using DictionaryPtr = std::shared_ptr<IDictionaryBase>;
using Dictionaries = std::map<String, DictionaryPtr>;
}

View File

@ -1,5 +1,6 @@
#include <map>
#include <set>
#include <Common/StackTrace.h>
#include <optional>
#include <memory>
#include <Poco/Mutex.h>

View File

@ -55,7 +55,7 @@ public:
using ObjectConfigsPtr = std::shared_ptr<const std::unordered_map<String /* object's name */, ObjectConfig>>;
/// Reads configuration files.
/// Reads configurations.
ObjectConfigsPtr read()
{
std::lock_guard lock{mutex};
@ -89,7 +89,7 @@ public:
}
private:
struct LoadablesInfos
struct LoadablesInfos
{
Poco::Timestamp last_update_time = 0;
std::vector<std::pair<String, ObjectConfig>> configs; // Parsed file's contents.
@ -146,19 +146,19 @@ private:
bool readLoadablesInfo(
IExternalLoaderConfigRepository & repository,
const String & path,
const String & name,
const ExternalLoaderConfigSettings & settings,
LoadablesInfos & loadable_info) const
{
try
{
if (path.empty() || !repository.exists(path))
if (name.empty() || !repository.exists(name))
{
LOG_WARNING(log, "Config file '" + path + "' does not exist");
LOG_WARNING(log, "Config file '" + name + "' does not exist");
return false;
}
auto update_time_from_repository = repository.getUpdateTime(path);
auto update_time_from_repository = repository.getUpdateTime(name);
/// Actually it can't be less, but for sure we check less or equal
if (update_time_from_repository <= loadable_info.last_update_time)
@ -167,7 +167,7 @@ private:
return false;
}
auto file_contents = repository.load(path);
auto file_contents = repository.load(name);
/// get all objects' definitions
Poco::Util::AbstractConfiguration::Keys keys;
@ -180,18 +180,18 @@ private:
if (!startsWith(key, settings.external_config))
{
if (!startsWith(key, "comment") && !startsWith(key, "include_from"))
LOG_WARNING(log, path << ": file contains unknown node '" << key << "', expected '" << settings.external_config << "'");
LOG_WARNING(log, name << ": file contains unknown node '" << key << "', expected '" << settings.external_config << "'");
continue;
}
String name = file_contents->getString(key + "." + settings.external_name);
if (name.empty())
String external_name = file_contents->getString(key + "." + settings.external_name);
if (external_name.empty())
{
LOG_WARNING(log, path << ": node '" << key << "' defines " << type_name << " with an empty name. It's not allowed");
LOG_WARNING(log, name << ": node '" << key << "' defines " << type_name << " with an empty name. It's not allowed");
continue;
}
configs_from_file.emplace_back(name, ObjectConfig{path, file_contents, key});
configs_from_file.emplace_back(name, ObjectConfig{external_name, file_contents, key});
}
loadable_info.configs = std::move(configs_from_file);
@ -201,7 +201,7 @@ private:
}
catch (...)
{
tryLogCurrentException(log, "Failed to read config file '" + path + "'");
tryLogCurrentException(log, "Failed to load config for dictionary '" + name + "'");
return false;
}
}

View File

@ -31,10 +31,6 @@ LoadablesConfigurationPtr ExternalLoaderDatabaseConfigRepository::load(const std
bool ExternalLoaderDatabaseConfigRepository::exists(const std::string & loadable_definition_name) const
{
std::cerr << "IS EXISTS:"
<< loadable_definition_name << std::endl;
std::cerr << "CUTTED:"
<< trimDatabaseName(loadable_definition_name, database) << std::endl;
return database->isDictionaryExist(
context, trimDatabaseName(loadable_definition_name, database));
}
@ -54,7 +50,6 @@ std::set<std::string> ExternalLoaderDatabaseConfigRepository::getAllLoadablesDef
result.insert(dbname + "." + itr->name());
itr->next();
}
std::cerr << "RESULTSIZE:" << result.size() << std::endl;
return result;
}

View File

@ -161,7 +161,6 @@ BlockIO InterpreterCreateQuery::createDatabase(ASTCreateQuery & create)
try
{
std::cerr << "ADDING DB NAME:" << database_name << std::endl;
context.addDatabase(database_name, database);
if (need_write_metadata)
@ -729,11 +728,10 @@ BlockIO InterpreterCreateQuery::createDictionary(ASTCreateQuery & create)
create.attach = true;
}
auto res = DictionaryFactory::instance().create(dictionary_name, create, context.getGlobalContext());
if (create.attach)
database->attachDictionary(dictionary_name, res);
database->attachDictionary(dictionary_name);
else
database->createDictionary(context, dictionary_name, res, query_ptr);
database->createDictionary(context, dictionary_name, query_ptr);
return {};
}

View File

@ -1,5 +1,5 @@
=DICTIONARY in Ordinary DB
CREATE DICTIONARY ordinary_db.dict1 (`key_column` UInt64 DEFAULT 0, `second_column` UInt8 DEFAULT 1, `third_column` String DEFAULT \'qqq\') PRIMARY KEY key_column SOURCE(CLICKHOUSE(HOST \'localhost\' PORT 9000 USER \'default\' TABLE \'table_for_dict\' PASSWORD \'\')) LIFETIME(MIN 1 MAX 10) LAYOUT(FLAT())
CREATE DICTIONARY ordinary_db.dict1 (`key_column` UInt64 DEFAULT 0, `second_column` UInt8 DEFAULT 1, `third_column` String DEFAULT \'qqq\') PRIMARY KEY key_column SOURCE(CLICKHOUSE(HOST \'localhost\' PORT 9000 USER \'default\' TABLE \'table_for_dict\' PASSWORD \'\' DB \'database_for_dict\')) LIFETIME(MIN 1 MAX 10) LAYOUT(FLAT())
dict1
1
ordinary_db dict1