2017-04-01 09:19:00 +00:00
|
|
|
#include <Dictionaries/DictionarySourceFactory.h>
|
|
|
|
|
|
|
|
#include <Core/Block.h>
|
|
|
|
#include <Dictionaries/DictionaryStructure.h>
|
|
|
|
#include <Dictionaries/FileDictionarySource.h>
|
|
|
|
#include <Dictionaries/ClickHouseDictionarySource.h>
|
|
|
|
#include <Dictionaries/ExecutableDictionarySource.h>
|
|
|
|
#include <Dictionaries/HTTPDictionarySource.h>
|
2017-09-05 01:08:26 +00:00
|
|
|
#include <Dictionaries/LibraryDictionarySource.h>
|
2018-09-28 02:46:33 +00:00
|
|
|
#include <Dictionaries/XDBCDictionarySource.h>
|
2017-04-01 09:19:00 +00:00
|
|
|
#include <DataTypes/DataTypesNumber.h>
|
|
|
|
#include <DataTypes/DataTypeDate.h>
|
2018-09-13 13:33:44 +00:00
|
|
|
#include <DataTypes/DataTypeNullable.h>
|
2017-11-24 13:55:31 +00:00
|
|
|
#include <Common/FieldVisitors.h>
|
2018-09-28 02:46:33 +00:00
|
|
|
#include <Common/XDBCBridgeHelper.h>
|
2017-04-01 09:19:00 +00:00
|
|
|
#include <Columns/ColumnsNumber.h>
|
|
|
|
#include <IO/HTTPCommon.h>
|
2016-12-08 02:49:04 +00:00
|
|
|
#include <memory>
|
2017-02-27 21:07:57 +00:00
|
|
|
#include <mutex>
|
2017-03-28 20:30:57 +00:00
|
|
|
|
2017-04-01 09:19:00 +00:00
|
|
|
#include <Common/config.h>
|
2018-05-14 18:36:01 +00:00
|
|
|
#if USE_POCO_MONGODB
|
2017-04-01 09:19:00 +00:00
|
|
|
#include <Dictionaries/MongoDBDictionarySource.h>
|
2017-03-28 20:30:57 +00:00
|
|
|
#endif
|
2018-05-14 18:36:01 +00:00
|
|
|
#if USE_POCO_SQLODBC || USE_POCO_DATAODBC
|
2017-04-01 07:20:54 +00:00
|
|
|
#include <Poco/Data/ODBC/Connector.h>
|
2017-03-28 20:30:57 +00:00
|
|
|
#endif
|
2017-04-19 00:25:57 +00:00
|
|
|
#if USE_MYSQL
|
|
|
|
#include <Dictionaries/MySQLDictionarySource.h>
|
|
|
|
#endif
|
2017-03-28 20:30:57 +00:00
|
|
|
|
2017-11-27 17:34:43 +00:00
|
|
|
#include <Poco/Logger.h>
|
2016-12-08 02:49:04 +00:00
|
|
|
|
2017-11-27 17:34:43 +00:00
|
|
|
#include <common/logger_useful.h>
|
2016-12-08 02:49:04 +00:00
|
|
|
|
|
|
|
namespace DB
|
|
|
|
{
|
|
|
|
|
|
|
|
namespace ErrorCodes
|
|
|
|
{
|
2017-04-01 07:20:54 +00:00
|
|
|
extern const int UNKNOWN_ELEMENT_IN_CONFIG;
|
|
|
|
extern const int EXCESSIVE_ELEMENT_IN_CONFIG;
|
|
|
|
extern const int LOGICAL_ERROR;
|
|
|
|
extern const int SUPPORT_IS_DISABLED;
|
2016-12-08 02:49:04 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
namespace
|
|
|
|
{
|
|
|
|
|
|
|
|
Block createSampleBlock(const DictionaryStructure & dict_struct)
|
|
|
|
{
|
2017-04-01 07:20:54 +00:00
|
|
|
Block block;
|
2016-12-08 02:49:04 +00:00
|
|
|
|
2017-04-01 07:20:54 +00:00
|
|
|
if (dict_struct.id)
|
2018-05-07 02:01:11 +00:00
|
|
|
block.insert(ColumnWithTypeAndName{ColumnUInt64::create(1, 0), std::make_shared<DataTypeUInt64>(), dict_struct.id->name});
|
2016-12-08 02:49:04 +00:00
|
|
|
|
2017-04-01 07:20:54 +00:00
|
|
|
if (dict_struct.key)
|
|
|
|
{
|
|
|
|
for (const auto & attribute : *dict_struct.key)
|
|
|
|
{
|
|
|
|
auto column = attribute.type->createColumn();
|
|
|
|
column->insertDefault();
|
2016-12-08 02:49:04 +00:00
|
|
|
|
2017-12-15 03:04:33 +00:00
|
|
|
block.insert(ColumnWithTypeAndName{std::move(column), attribute.type, attribute.name});
|
2017-04-01 07:20:54 +00:00
|
|
|
}
|
|
|
|
}
|
2016-12-08 02:49:04 +00:00
|
|
|
|
2017-04-01 07:20:54 +00:00
|
|
|
if (dict_struct.range_min)
|
2018-09-13 18:58:12 +00:00
|
|
|
{
|
2017-04-01 07:20:54 +00:00
|
|
|
for (const auto & attribute : { dict_struct.range_min, dict_struct.range_max })
|
2018-09-13 13:33:44 +00:00
|
|
|
{
|
|
|
|
const auto & type = std::make_shared<DataTypeNullable>(attribute->type);
|
|
|
|
auto column = type->createColumn();
|
|
|
|
column->insertDefault();
|
|
|
|
|
|
|
|
block.insert(ColumnWithTypeAndName{std::move(column), type, attribute->name});
|
|
|
|
}
|
2018-09-13 18:58:12 +00:00
|
|
|
}
|
2016-12-08 02:49:04 +00:00
|
|
|
|
2017-04-01 07:20:54 +00:00
|
|
|
for (const auto & attribute : dict_struct.attributes)
|
|
|
|
{
|
|
|
|
auto column = attribute.type->createColumn();
|
|
|
|
column->insert(attribute.null_value);
|
2016-12-08 02:49:04 +00:00
|
|
|
|
2017-12-15 03:04:33 +00:00
|
|
|
block.insert(ColumnWithTypeAndName{std::move(column), attribute.type, attribute.name});
|
2017-04-01 07:20:54 +00:00
|
|
|
}
|
2016-12-08 02:49:04 +00:00
|
|
|
|
2017-04-01 07:20:54 +00:00
|
|
|
return block;
|
2016-12-08 02:49:04 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
DictionarySourceFactory::DictionarySourceFactory()
|
2017-11-27 17:34:43 +00:00
|
|
|
: log(&Poco::Logger::get("DictionarySourceFactory"))
|
2016-12-08 02:49:04 +00:00
|
|
|
{
|
2018-05-14 18:36:01 +00:00
|
|
|
#if USE_POCO_SQLODBC || USE_POCO_DATAODBC
|
2017-04-01 07:20:54 +00:00
|
|
|
Poco::Data::ODBC::Connector::registerConnector();
|
2017-03-28 20:30:57 +00:00
|
|
|
#endif
|
2016-12-08 02:49:04 +00:00
|
|
|
}
|
|
|
|
|
2017-11-27 19:05:45 +00:00
|
|
|
void DictionarySourceFactory::registerSource(const std::string & source_type, Creator create_source)
|
2017-11-27 17:34:43 +00:00
|
|
|
{
|
|
|
|
LOG_DEBUG(log, "Register dictionary source type `" + source_type + "`");
|
2017-11-27 19:05:45 +00:00
|
|
|
if (!registered_sources.emplace(source_type, std::move(create_source)).second)
|
2017-11-27 19:12:18 +00:00
|
|
|
throw Exception("DictionarySourceFactory: the source name '" + source_type + "' is not unique",
|
2017-11-27 19:05:45 +00:00
|
|
|
ErrorCodes::LOGICAL_ERROR);
|
2017-11-27 17:34:43 +00:00
|
|
|
}
|
2016-12-08 02:49:04 +00:00
|
|
|
|
|
|
|
DictionarySourcePtr DictionarySourceFactory::create(
|
2017-10-06 10:31:06 +00:00
|
|
|
const std::string & name, const Poco::Util::AbstractConfiguration & config, const std::string & config_prefix,
|
2017-04-01 07:20:54 +00:00
|
|
|
const DictionaryStructure & dict_struct, Context & context) const
|
2016-12-08 02:49:04 +00:00
|
|
|
{
|
2017-04-01 07:20:54 +00:00
|
|
|
Poco::Util::AbstractConfiguration::Keys keys;
|
|
|
|
config.keys(config_prefix, keys);
|
|
|
|
if (keys.size() != 1)
|
2018-05-07 02:01:11 +00:00
|
|
|
throw Exception{name +": element dictionary.source should have exactly one child element", ErrorCodes::EXCESSIVE_ELEMENT_IN_CONFIG};
|
2017-04-01 07:20:54 +00:00
|
|
|
|
|
|
|
auto sample_block = createSampleBlock(dict_struct);
|
|
|
|
|
|
|
|
const auto & source_type = keys.front();
|
|
|
|
|
|
|
|
if ("file" == source_type)
|
|
|
|
{
|
|
|
|
if (dict_struct.has_expressions)
|
2018-05-07 02:01:11 +00:00
|
|
|
throw Exception{"Dictionary source of type `file` does not support attribute expressions", ErrorCodes::LOGICAL_ERROR};
|
2017-04-01 07:20:54 +00:00
|
|
|
|
|
|
|
const auto filename = config.getString(config_prefix + ".file.path");
|
|
|
|
const auto format = config.getString(config_prefix + ".file.format");
|
|
|
|
return std::make_unique<FileDictionarySource>(filename, format, sample_block, context);
|
|
|
|
}
|
|
|
|
else if ("mysql" == source_type)
|
|
|
|
{
|
2017-04-19 00:25:57 +00:00
|
|
|
#if USE_MYSQL
|
2017-04-01 07:20:54 +00:00
|
|
|
return std::make_unique<MySQLDictionarySource>(dict_struct, config, config_prefix + ".mysql", sample_block);
|
2017-04-19 00:25:57 +00:00
|
|
|
#else
|
2017-12-28 04:33:35 +00:00
|
|
|
throw Exception{"Dictionary source of type `mysql` is disabled because ClickHouse was built without mysql support.",
|
2017-04-19 00:25:57 +00:00
|
|
|
ErrorCodes::SUPPORT_IS_DISABLED};
|
|
|
|
#endif
|
2017-04-01 07:20:54 +00:00
|
|
|
}
|
|
|
|
else if ("clickhouse" == source_type)
|
|
|
|
{
|
|
|
|
return std::make_unique<ClickHouseDictionarySource>(dict_struct, config, config_prefix + ".clickhouse",
|
|
|
|
sample_block, context);
|
|
|
|
}
|
|
|
|
else if ("mongodb" == source_type)
|
|
|
|
{
|
2018-05-14 18:36:01 +00:00
|
|
|
#if USE_POCO_MONGODB
|
2017-04-01 07:20:54 +00:00
|
|
|
return std::make_unique<MongoDBDictionarySource>(dict_struct, config, config_prefix + ".mongodb", sample_block);
|
2017-03-28 20:30:57 +00:00
|
|
|
#else
|
2017-12-28 04:33:35 +00:00
|
|
|
throw Exception{"Dictionary source of type `mongodb` is disabled because poco library was built without mongodb support.",
|
2017-04-01 07:20:54 +00:00
|
|
|
ErrorCodes::SUPPORT_IS_DISABLED};
|
2017-03-28 20:30:57 +00:00
|
|
|
#endif
|
2017-04-01 07:20:54 +00:00
|
|
|
}
|
|
|
|
else if ("odbc" == source_type)
|
|
|
|
{
|
2018-05-14 18:36:01 +00:00
|
|
|
#if USE_POCO_SQLODBC || USE_POCO_DATAODBC
|
2018-10-09 15:03:41 +00:00
|
|
|
BridgeHelperPtr bridge = std::make_shared<XDBCBridgeHelper<ODBCBridgeMixin>>(config, context.getSettings().http_connection_timeout, config.getString(config_prefix + ".odbc.connection_string"));
|
2018-09-28 02:46:33 +00:00
|
|
|
return std::make_unique<XDBCDictionarySource>(dict_struct, config, config_prefix + ".odbc", sample_block, context, bridge);
|
2017-03-28 20:30:57 +00:00
|
|
|
#else
|
2017-12-28 04:33:35 +00:00
|
|
|
throw Exception{"Dictionary source of type `odbc` is disabled because poco library was built without ODBC support.",
|
2017-04-01 07:20:54 +00:00
|
|
|
ErrorCodes::SUPPORT_IS_DISABLED};
|
2017-03-28 20:30:57 +00:00
|
|
|
#endif
|
2018-09-28 02:46:33 +00:00
|
|
|
}
|
|
|
|
else if ("jdbc" == source_type)
|
|
|
|
{
|
|
|
|
throw Exception{"Dictionary source of type `jdbc` is disabled until consistend dealing with nullable fields.",
|
|
|
|
ErrorCodes::SUPPORT_IS_DISABLED};
|
|
|
|
// BridgeHelperPtr bridge = std::make_shared<XDBCBridgeHelper<JDBCBridgeMixin>>(config, context.getSettings().http_connection_timeout, config.getString(config_prefix + ".connection_string"));
|
|
|
|
// return std::make_unique<XDBCDictionarySource>(dict_struct, config, config_prefix + ".jdbc", sample_block, context, bridge);
|
2017-04-01 07:20:54 +00:00
|
|
|
}
|
|
|
|
else if ("executable" == source_type)
|
|
|
|
{
|
|
|
|
if (dict_struct.has_expressions)
|
2018-05-07 02:01:11 +00:00
|
|
|
throw Exception{"Dictionary source of type `executable` does not support attribute expressions", ErrorCodes::LOGICAL_ERROR};
|
2017-04-01 07:20:54 +00:00
|
|
|
|
|
|
|
return std::make_unique<ExecutableDictionarySource>(dict_struct, config, config_prefix + ".executable", sample_block, context);
|
|
|
|
}
|
|
|
|
else if ("http" == source_type)
|
|
|
|
{
|
|
|
|
|
|
|
|
if (dict_struct.has_expressions)
|
2018-05-07 02:01:11 +00:00
|
|
|
throw Exception{"Dictionary source of type `http` does not support attribute expressions", ErrorCodes::LOGICAL_ERROR};
|
2016-12-08 02:49:04 +00:00
|
|
|
|
2017-04-01 07:20:54 +00:00
|
|
|
return std::make_unique<HTTPDictionarySource>(dict_struct, config, config_prefix + ".http", sample_block, context);
|
|
|
|
}
|
2017-09-05 01:08:26 +00:00
|
|
|
else if ("library" == source_type)
|
|
|
|
{
|
|
|
|
return std::make_unique<LibraryDictionarySource>(dict_struct, config, config_prefix + ".library", sample_block, context);
|
|
|
|
}
|
2017-11-27 17:34:43 +00:00
|
|
|
else
|
|
|
|
{
|
|
|
|
const auto found = registered_sources.find(source_type);
|
|
|
|
if (found != registered_sources.end())
|
|
|
|
{
|
2017-11-27 19:07:07 +00:00
|
|
|
const auto & create_source = found->second;
|
2017-11-27 17:34:43 +00:00
|
|
|
return create_source(dict_struct, config, config_prefix, sample_block, context);
|
|
|
|
}
|
|
|
|
}
|
2016-12-08 02:49:04 +00:00
|
|
|
|
2018-05-07 02:01:11 +00:00
|
|
|
throw Exception{name + ": unknown dictionary source type: " + source_type, ErrorCodes::UNKNOWN_ELEMENT_IN_CONFIG};
|
2016-12-08 02:49:04 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
}
|