2017-05-22 16:38:24 +00:00
|
|
|
#include <DataTypes/DataTypeString.h>
|
|
|
|
#include <Columns/ColumnString.h>
|
2017-12-26 21:34:06 +00:00
|
|
|
#include <Common/PocoSessionPoolHelpers.h>
|
2016-12-08 02:49:04 +00:00
|
|
|
#include <Poco/Util/AbstractConfiguration.h>
|
2017-04-01 09:19:00 +00:00
|
|
|
#include <Dictionaries/ODBCDictionarySource.h>
|
|
|
|
#include <Dictionaries/ODBCBlockInputStream.h>
|
2017-01-21 04:24:28 +00:00
|
|
|
#include <common/logger_useful.h>
|
2017-05-22 16:38:24 +00:00
|
|
|
#include <Dictionaries/readInvalidateQuery.h>
|
2018-03-02 12:59:41 +00:00
|
|
|
#include <Interpreters/Context.h>
|
2016-05-29 01:12:10 +00:00
|
|
|
|
2016-06-05 15:21:35 +00:00
|
|
|
|
|
|
|
namespace DB
|
|
|
|
{
|
|
|
|
|
|
|
|
|
2016-12-08 02:49:04 +00:00
|
|
|
static const size_t max_block_size = 8192;
|
2016-06-05 15:21:35 +00:00
|
|
|
|
|
|
|
|
|
|
|
ODBCDictionarySource::ODBCDictionarySource(const DictionaryStructure & dict_struct_,
|
2017-04-01 07:20:54 +00:00
|
|
|
const Poco::Util::AbstractConfiguration & config, const std::string & config_prefix,
|
2018-03-02 12:59:41 +00:00
|
|
|
const Block & sample_block, const Context & context)
|
2017-04-01 07:20:54 +00:00
|
|
|
: log(&Logger::get("ODBCDictionarySource")),
|
2018-02-15 13:08:23 +00:00
|
|
|
update_time{std::chrono::system_clock::from_time_t(0)},
|
2017-04-01 07:20:54 +00:00
|
|
|
dict_struct{dict_struct_},
|
|
|
|
db{config.getString(config_prefix + ".db", "")},
|
|
|
|
table{config.getString(config_prefix + ".table")},
|
|
|
|
where{config.getString(config_prefix + ".where", "")},
|
2018-03-07 21:32:35 +00:00
|
|
|
update_field{config.getString(config_prefix + ".update_field", "")},
|
2017-04-01 07:20:54 +00:00
|
|
|
sample_block{sample_block},
|
|
|
|
query_builder{dict_struct, db, table, where, ExternalQueryBuilder::None}, /// NOTE Better to obtain quoting style via ODBC interface.
|
2017-05-15 14:16:10 +00:00
|
|
|
load_all_query{query_builder.composeLoadAllQuery()},
|
|
|
|
invalidate_query{config.getString(config_prefix + ".invalidate_query", "")}
|
2016-06-05 15:21:35 +00:00
|
|
|
{
|
2018-03-02 12:59:41 +00:00
|
|
|
std::size_t field_size = context.getSettingsRef().odbc_max_field_size;
|
|
|
|
|
2017-12-26 21:34:06 +00:00
|
|
|
pool = createAndCheckResizePocoSessionPool([&]
|
|
|
|
{
|
2018-03-02 12:59:41 +00:00
|
|
|
auto session = std::make_shared<Poco::Data::SessionPool>(
|
2017-12-26 21:34:06 +00:00
|
|
|
config.getString(config_prefix + ".connector", "ODBC"),
|
|
|
|
config.getString(config_prefix + ".connection_string"));
|
2018-03-02 12:59:41 +00:00
|
|
|
|
|
|
|
/// Default POCO value is 1024. Set property manually to make possible reading of longer strings.
|
|
|
|
session->setProperty("maxFieldSize", Poco::Any(field_size));
|
2018-03-02 19:40:44 +00:00
|
|
|
return session;
|
2017-05-03 18:18:14 +00:00
|
|
|
});
|
2016-06-05 15:21:35 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
/// copy-constructor is provided in order to support cloneability
|
|
|
|
ODBCDictionarySource::ODBCDictionarySource(const ODBCDictionarySource & other)
|
2017-04-01 07:20:54 +00:00
|
|
|
: log(&Logger::get("ODBCDictionarySource")),
|
2018-02-15 13:08:23 +00:00
|
|
|
update_time{other.update_time},
|
2017-04-01 07:20:54 +00:00
|
|
|
dict_struct{other.dict_struct},
|
|
|
|
db{other.db},
|
|
|
|
table{other.table},
|
|
|
|
where{other.where},
|
2018-02-15 13:08:23 +00:00
|
|
|
update_field{other.update_field},
|
2017-04-01 07:20:54 +00:00
|
|
|
sample_block{other.sample_block},
|
|
|
|
pool{other.pool},
|
|
|
|
query_builder{dict_struct, db, table, where, ExternalQueryBuilder::None},
|
2017-05-15 14:16:10 +00:00
|
|
|
load_all_query{other.load_all_query},
|
|
|
|
invalidate_query{other.invalidate_query}, invalidate_query_response{other.invalidate_query_response}
|
2016-06-05 15:21:35 +00:00
|
|
|
{
|
|
|
|
}
|
|
|
|
|
2018-02-15 13:08:23 +00:00
|
|
|
std::string ODBCDictionarySource::getUpdateFieldAndDate()
|
|
|
|
{
|
|
|
|
if (update_time != std::chrono::system_clock::from_time_t(0))
|
|
|
|
{
|
|
|
|
auto tmp_time = update_time;
|
|
|
|
update_time = std::chrono::system_clock::now();
|
|
|
|
time_t hr_time = std::chrono::system_clock::to_time_t(tmp_time) - 1;
|
|
|
|
std::string str_time = std::to_string(LocalDateTime(hr_time));
|
|
|
|
return query_builder.composeUpdateQuery(update_field, str_time);
|
|
|
|
}
|
|
|
|
else
|
|
|
|
{
|
|
|
|
update_time = std::chrono::system_clock::now();
|
|
|
|
std::string str_time("0000-00-00 00:00:00"); ///for initial load
|
|
|
|
return query_builder.composeUpdateQuery(update_field, str_time);
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2016-06-05 15:21:35 +00:00
|
|
|
BlockInputStreamPtr ODBCDictionarySource::loadAll()
|
|
|
|
{
|
2017-04-01 07:20:54 +00:00
|
|
|
LOG_TRACE(log, load_all_query);
|
|
|
|
return std::make_shared<ODBCBlockInputStream>(pool->get(), load_all_query, sample_block, max_block_size);
|
2016-06-05 15:21:35 +00:00
|
|
|
}
|
|
|
|
|
2018-02-15 13:08:23 +00:00
|
|
|
BlockInputStreamPtr ODBCDictionarySource::loadUpdatedAll()
|
|
|
|
{
|
|
|
|
std::string load_query_update = getUpdateFieldAndDate();
|
|
|
|
|
|
|
|
LOG_TRACE(log, load_query_update);
|
|
|
|
return std::make_shared<ODBCBlockInputStream>(pool->get(), load_query_update, sample_block, max_block_size);
|
|
|
|
}
|
|
|
|
|
2016-10-26 22:27:38 +00:00
|
|
|
BlockInputStreamPtr ODBCDictionarySource::loadIds(const std::vector<UInt64> & ids)
|
2016-06-05 15:21:35 +00:00
|
|
|
{
|
2017-04-01 07:20:54 +00:00
|
|
|
const auto query = query_builder.composeLoadIdsQuery(ids);
|
|
|
|
return std::make_shared<ODBCBlockInputStream>(pool->get(), query, sample_block, max_block_size);
|
2016-06-05 15:21:35 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
BlockInputStreamPtr ODBCDictionarySource::loadKeys(
|
2017-07-21 06:35:58 +00:00
|
|
|
const Columns & key_columns, const std::vector<size_t> & requested_rows)
|
2016-06-05 15:21:35 +00:00
|
|
|
{
|
2017-04-01 07:20:54 +00:00
|
|
|
const auto query = query_builder.composeLoadKeysQuery(key_columns, requested_rows, ExternalQueryBuilder::AND_OR_CHAIN);
|
|
|
|
return std::make_shared<ODBCBlockInputStream>(pool->get(), query, sample_block, max_block_size);
|
2016-06-05 15:21:35 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
bool ODBCDictionarySource::supportsSelectiveLoad() const
|
|
|
|
{
|
2017-04-01 07:20:54 +00:00
|
|
|
return true;
|
2016-06-05 15:21:35 +00:00
|
|
|
}
|
|
|
|
|
2018-02-15 13:08:23 +00:00
|
|
|
bool ODBCDictionarySource::hasUpdateField() const
|
|
|
|
{
|
|
|
|
return !update_field.empty();
|
|
|
|
}
|
|
|
|
|
2016-06-05 15:21:35 +00:00
|
|
|
DictionarySourcePtr ODBCDictionarySource::clone() const
|
|
|
|
{
|
2017-04-01 07:20:54 +00:00
|
|
|
return std::make_unique<ODBCDictionarySource>(*this);
|
2016-06-05 15:21:35 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
std::string ODBCDictionarySource::toString() const
|
|
|
|
{
|
2017-04-01 07:20:54 +00:00
|
|
|
return "ODBC: " + db + '.' + table + (where.empty() ? "" : ", where: " + where);
|
2016-06-05 15:21:35 +00:00
|
|
|
}
|
|
|
|
|
2017-05-15 14:16:10 +00:00
|
|
|
bool ODBCDictionarySource::isModified() const
|
|
|
|
{
|
|
|
|
if (!invalidate_query.empty())
|
|
|
|
{
|
2017-05-22 16:38:24 +00:00
|
|
|
auto response = doInvalidateQuery(invalidate_query);
|
|
|
|
if (invalidate_query_response == response)
|
2017-05-15 14:16:10 +00:00
|
|
|
return false;
|
|
|
|
invalidate_query_response = response;
|
|
|
|
}
|
|
|
|
return true;
|
|
|
|
}
|
|
|
|
|
|
|
|
|
2017-05-22 16:38:24 +00:00
|
|
|
std::string ODBCDictionarySource::doInvalidateQuery(const std::string & request) const
|
2017-05-15 14:16:10 +00:00
|
|
|
{
|
2017-05-22 16:38:24 +00:00
|
|
|
Block sample_block;
|
2017-12-14 01:43:19 +00:00
|
|
|
ColumnPtr column(ColumnString::create());
|
2017-05-22 16:38:24 +00:00
|
|
|
sample_block.insert(ColumnWithTypeAndName(column, std::make_shared<DataTypeString>(), "Sample Block"));
|
2017-05-23 16:36:07 +00:00
|
|
|
ODBCBlockInputStream block_input_stream(pool->get(), request, sample_block, 1);
|
|
|
|
return readInvalidateQuery(block_input_stream);
|
2017-05-15 14:16:10 +00:00
|
|
|
}
|
2016-06-05 15:21:35 +00:00
|
|
|
|
|
|
|
}
|