thevar1able/ClickHouse
1
0
Fork 0
mirror of https://github.com/ClickHouse/ClickHouse.git synced 2024-11-21 15:12:02 +00:00
Code Issues Packages Projects Releases Wiki Activity

Add loading of external dictionaries

Browse Source
This commit is contained in:
alesapin 2019-10-15 17:09:57 +03:00
parent f850408ae4
commit 3e068b81de
8 changed files with 109 additions and 9 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
dbms/CMakeLists.txt
View File

@ -130,8 +130,8 @@ list (APPEND dbms_headers
list (APPEND dbms_sources src/TableFunctions/ITableFunction.cpp src/TableFunctions/TableFunctionFactory.cpp)
list (APPEND dbms_headers src/TableFunctions/ITableFunction.h src/TableFunctions/TableFunctionFactory.h)
list (APPEND dbms_sources src/Dictionaries/DictionaryFactory.cpp src/Dictionaries/DictionarySourceFactory.cpp src/Dictionaries/DictionaryStructure.cpp)
list (APPEND dbms_headers src/Dictionaries/DictionaryFactory.h src/Dictionaries/DictionarySourceFactory.h src/Dictionaries/DictionaryStructure.h)
list (APPEND dbms_sources src/Dictionaries/DictionaryFactory.cpp src/Dictionaries/DictionarySourceFactory.cpp src/Dictionaries/DictionaryStructure.cpp src/Dictionaries/getDictionaryConfigurationFromAST.cpp)
list (APPEND dbms_headers src/Dictionaries/DictionaryFactory.h src/Dictionaries/DictionarySourceFactory.h src/Dictionaries/DictionaryStructure.h src/Dictionaries/getDictionaryConfigurationFromAST.h)
if (NOT ENABLE_SSL)
list (REMOVE_ITEM clickhouse_common_io_sources src/Common/OpenSSLHelpers.cpp)

8
dbms/src/Dictionaries/DictionaryFactory.cpp
View File

@ -3,6 +3,7 @@
#include <memory>
#include "DictionarySourceFactory.h"
#include "DictionaryStructure.h"
#include "getDictionaryConfigurationFromAST.h"
namespace DB
{
@ -52,11 +53,8 @@ DictionaryPtr DictionaryFactory::create(
DictionaryPtr DictionaryFactory::create(const std::string & name, const ASTCreateQuery & ast, const Context & context) const
{
/// Temporary code for testing TODO(alesapin)
(void)(name);
(void)(ast);
(void)(context);
return nullptr;
auto configurationFromAST = getDictionaryConfigurationFromAST(ast);
return DictionaryFactory::create(name, *configurationFromAST, "dictionary", context);
}

1
dbms/src/Dictionaries/DictionaryFactory.h
View File

@ -34,7 +34,6 @@ public:
const std::string & config_prefix,
const Context & context) const;
/// Temporary method for testing TODO(alesapin)
DictionaryPtr create(const std::string & name,
const ASTCreateQuery & ast,
const Context & context) const;

3
dbms/src/Dictionaries/getDictionaryConfigurationFromAST.cpp
View File

@ -1,4 +1,5 @@
#include <Dictionaries/getDictionaryConfigurationFromAST.h>
#include <Poco/DOM/AutoPtr.h>
#include <Poco/DOM/Document.h>
#include <Poco/DOM/Element.h>
@ -15,6 +16,7 @@
namespace DB
{
namespace ErrorCodes
{
extern const int INCORRECT_DICTIONARY_DEFINITION;
@ -383,7 +385,6 @@ void checkAST(const ASTCreateQuery & query)
DictionaryConfigurationPtr getDictionaryConfigurationFromAST(const ASTCreateQuery & query)
{
checkAST(query);
AutoPtr<Poco::XML::Document> xml_document(new Poco::XML::Document());

14
dbms/src/Dictionaries/getDictionaryConfigurationFromAST.h Normal file
View File

@ -0,0 +1,14 @@
#pragma once
#include <Poco/Util/AbstractConfiguration.h>
#include <Parsers/ASTCreateQuery.h>
namespace DB
{
using DictionaryConfigurationPtr = Poco::AutoPtr<Poco::Util::AbstractConfiguration>;
/// Convert dictionary AST to Poco::AbstractConfiguration
/// This function is necessary because all loadable objects configuration are Poco::AbstractConfiguration
DictionaryConfigurationPtr getDictionaryConfigurationFromAST(const ASTCreateQuery & query);
}

3
dbms/src/Interpreters/Context.cpp
View File

@ -30,6 +30,7 @@
#include <Dictionaries/Embedded/GeoDictionariesLoader.h>
#include <Interpreters/EmbeddedDictionaries.h>
#include <Interpreters/ExternalLoaderXMLConfigRepository.h>
#include <Interpreters/ExternalLoaderDatabaseConfigRepository.h>
#include <Interpreters/ExternalDictionariesLoader.h>
#include <Interpreters/ExternalModelsLoader.h>
#include <Interpreters/ExpressionActions.h>
@ -1034,6 +1035,8 @@ void Context::addDatabase(const String & database_name, const DatabasePtr & data
assertDatabaseDoesntExist(database_name);
shared->databases[database_name] = database;
auto dictionaries_repository = std::make_unique<ExternalLoaderDatabaseConfigRepository>(database, *this);
getExternalDictionariesLoader().addConfigRepository(std::move(dictionaries_repository), {});
}

55
dbms/src/Interpreters/ExternalLoaderDatabaseConfigRepository.cpp Normal file
View File

@ -0,0 +1,55 @@
#include <Interpreters/ExternalLoaderDatabaseConfigRepository.h>
#include <Dictionaries/getDictionaryConfigurationFromAST.h>
namespace DB
{
namespace ErrorCodes
{
extern const int UNKNOWN_DICTIONARY;
}
namespace
{
String trimDatabaseName(const std::string & loadable_definition_name, const DatabasePtr database)
{
const auto & dbname = database->getDatabaseName();
if (!startsWith(loadable_definition_name, dbname))
throw Exception(
"Loadable '" + loadable_definition_name + "' is not from database '" + database->getDatabaseName(), ErrorCodes::UNKNOWN_DICTIONARY);
/// dbname.loadable_name
///--> remove <---
return loadable_definition_name.substr(dbname.length());
}
}
LoadablesConfigurationPtr ExternalLoaderDatabaseConfigRepository::load(const std::string & loadable_definition_name) const
{
String dictname = trimDatabaseName(loadable_definition_name, database);
return getDictionaryConfigurationFromAST(database->getCreateDictionaryQuery(context, dictname)->as<const ASTCreateQuery &>());
}
bool ExternalLoaderDatabaseConfigRepository::exists(const std::string & loadable_definition_name) const
{
return database->isDictionaryExist(context, trimDatabaseName(loadable_definition_name, database));
}
Poco::Timestamp ExternalLoaderDatabaseConfigRepository::getUpdateTime(const std::string & loadable_definition_name)
{
return database->getObjectMetadataModificationTime(context, trimDatabaseName(loadable_definition_name, database));
}
std::set<std::string> ExternalLoaderDatabaseConfigRepository::getAllLoadablesDefinitionNames() const
{
std::set<std::string> result;
const auto & dbname = database->getDatabaseName();
auto itr = database->getDictionariesIterator(context);
while (itr && itr->isValid())
{
result.insert(dbname + "." + itr->name());
itr->next();
}
return result;
}
}

30
dbms/src/Interpreters/ExternalLoaderDatabaseConfigRepository.h Normal file
View File

@ -0,0 +1,30 @@
#pragma once
#include <Interpreters/IExternalLoaderConfigRepository.h>
#include <Databases/IDatabase.h>
#include <Interpreters/Context.h>
namespace DB
{
class ExternalLoaderDatabaseConfigRepository : public IExternalLoaderConfigRepository
{
public:
ExternalLoaderDatabaseConfigRepository(const DatabasePtr & database_, const Context & context_)
: database(database_)
, context(context_)
{
}
std::set<std::string> getAllLoadablesDefinitionNames() const override;
bool exists(const std::string & loadable_definition_name) const override;
Poco::Timestamp getUpdateTime(const std::string & loadable_definition_name) override;
LoadablesConfigurationPtr load(const std::string & loadable_definition_name) const override;
private:
DatabasePtr database;
Context context;
};
}