2017-09-05 01:08:26 +00:00
|
|
|
#include <DataStreams/OneBlockInputStream.h>
|
2018-11-28 11:37:12 +00:00
|
|
|
#include "LibraryDictionarySource.h"
|
|
|
|
#include "LibraryDictionarySourceExternal.h"
|
2017-09-05 01:08:26 +00:00
|
|
|
#include <Interpreters/Context.h>
|
|
|
|
#include <Poco/File.h>
|
|
|
|
#include <common/logger_useful.h>
|
2017-09-08 18:24:15 +00:00
|
|
|
#include <ext/bit_cast.h>
|
2017-12-09 06:32:22 +00:00
|
|
|
#include <ext/range.h>
|
2018-03-13 23:13:39 +00:00
|
|
|
#include <ext/scope_guard.h>
|
2018-11-28 11:37:12 +00:00
|
|
|
#include "DictionarySourceFactory.h"
|
|
|
|
#include "DictionaryStructure.h"
|
|
|
|
|
2017-12-09 06:32:22 +00:00
|
|
|
|
2017-09-05 01:08:26 +00:00
|
|
|
namespace DB
|
|
|
|
{
|
|
|
|
namespace ErrorCodes
|
|
|
|
{
|
|
|
|
extern const int NOT_IMPLEMENTED;
|
|
|
|
extern const int SIZES_OF_COLUMNS_DOESNT_MATCH;
|
|
|
|
extern const int FILE_DOESNT_EXIST;
|
2018-02-16 17:12:22 +00:00
|
|
|
extern const int EXTERNAL_LIBRARY_ERROR;
|
2017-09-05 01:08:26 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
class CStringsHolder
|
|
|
|
{
|
|
|
|
public:
|
2017-09-07 21:04:48 +00:00
|
|
|
using Container = std::vector<std::string>;
|
|
|
|
explicit CStringsHolder(const Container & strings_pass)
|
2017-09-05 01:08:26 +00:00
|
|
|
{
|
|
|
|
strings_holder = strings_pass;
|
|
|
|
strings.size = strings_holder.size();
|
|
|
|
ptr_holder = std::make_unique<ClickHouseLibrary::CString[]>(strings.size);
|
|
|
|
strings.data = ptr_holder.get();
|
|
|
|
size_t i = 0;
|
|
|
|
for (auto & str : strings_holder)
|
|
|
|
{
|
|
|
|
strings.data[i] = str.c_str();
|
|
|
|
++i;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
ClickHouseLibrary::CStrings strings; // will pass pointer to lib
|
|
|
|
|
|
|
|
private:
|
|
|
|
std::unique_ptr<ClickHouseLibrary::CString[]> ptr_holder = nullptr;
|
2017-09-07 21:04:48 +00:00
|
|
|
Container strings_holder;
|
2017-09-05 01:08:26 +00:00
|
|
|
};
|
|
|
|
|
2017-09-07 21:04:48 +00:00
|
|
|
|
|
|
|
namespace
|
|
|
|
{
|
2018-06-03 20:39:06 +00:00
|
|
|
constexpr auto lib_config_settings = ".settings";
|
2017-09-07 21:04:48 +00:00
|
|
|
|
|
|
|
|
2018-02-14 15:03:48 +00:00
|
|
|
CStringsHolder getLibSettings(const Poco::Util::AbstractConfiguration & config, const std::string & config_root)
|
2017-09-05 01:08:26 +00:00
|
|
|
{
|
2018-02-14 15:03:48 +00:00
|
|
|
Poco::Util::AbstractConfiguration::Keys config_keys;
|
|
|
|
config.keys(config_root, config_keys);
|
|
|
|
CStringsHolder::Container strings;
|
|
|
|
for (const auto & key : config_keys)
|
|
|
|
{
|
|
|
|
std::string key_name = key;
|
|
|
|
auto bracket_pos = key.find('[');
|
|
|
|
if (bracket_pos != std::string::npos && bracket_pos > 0)
|
|
|
|
key_name = key.substr(0, bracket_pos);
|
|
|
|
strings.emplace_back(key_name);
|
2018-02-26 16:57:14 +00:00
|
|
|
strings.emplace_back(config.getString(config_root + "." + key));
|
2018-02-14 15:03:48 +00:00
|
|
|
}
|
|
|
|
return CStringsHolder(strings);
|
2017-09-05 01:08:26 +00:00
|
|
|
}
|
|
|
|
|
2017-09-07 21:04:48 +00:00
|
|
|
|
2018-02-14 15:03:48 +00:00
|
|
|
Block dataToBlock(const Block & sample_block, const void * data)
|
|
|
|
{
|
|
|
|
if (!data)
|
2018-03-13 23:13:39 +00:00
|
|
|
throw Exception("LibraryDictionarySource: No data returned", ErrorCodes::EXTERNAL_LIBRARY_ERROR);
|
2017-09-07 21:04:48 +00:00
|
|
|
|
2018-02-16 17:12:22 +00:00
|
|
|
auto columns_received = static_cast<const ClickHouseLibrary::Table *>(data);
|
|
|
|
if (columns_received->error_code)
|
2018-03-13 23:13:39 +00:00
|
|
|
throw Exception("LibraryDictionarySource: Returned error: " + std::to_string(columns_received->error_code) + " "
|
2018-02-16 17:12:22 +00:00
|
|
|
+ (columns_received->error_string ? columns_received->error_string : ""),
|
|
|
|
ErrorCodes::EXTERNAL_LIBRARY_ERROR);
|
2017-12-15 03:04:33 +00:00
|
|
|
|
2018-02-14 15:03:48 +00:00
|
|
|
MutableColumns columns(sample_block.columns());
|
|
|
|
for (const auto i : ext::range(0, columns.size()))
|
|
|
|
columns[i] = sample_block.getByPosition(i).column->cloneEmpty();
|
2017-12-15 03:04:33 +00:00
|
|
|
|
2018-02-14 15:03:48 +00:00
|
|
|
for (size_t col_n = 0; col_n < columns_received->size; ++col_n)
|
|
|
|
{
|
|
|
|
if (columns.size() != columns_received->data[col_n].size)
|
2018-06-06 14:35:33 +00:00
|
|
|
throw Exception("LibraryDictionarySource: Returned unexpected number of columns: "
|
|
|
|
+ std::to_string(columns_received->data[col_n].size) + ", must be " + std::to_string(columns.size()),
|
2018-02-14 15:03:48 +00:00
|
|
|
ErrorCodes::SIZES_OF_COLUMNS_DOESNT_MATCH);
|
|
|
|
|
|
|
|
for (size_t row_n = 0; row_n < columns_received->data[col_n].size; ++row_n)
|
|
|
|
{
|
2018-02-16 17:12:22 +00:00
|
|
|
const auto & field = columns_received->data[col_n].data[row_n];
|
|
|
|
if (!field.data)
|
2018-10-11 15:21:12 +00:00
|
|
|
{
|
|
|
|
/// sample_block contains null_value (from config) inside corresponding column
|
|
|
|
const auto & col = sample_block.getByPosition(row_n);
|
|
|
|
columns[row_n]->insertFrom(*(col.column), 0);
|
|
|
|
}
|
|
|
|
else
|
|
|
|
{
|
|
|
|
const auto & size = field.size;
|
|
|
|
columns[row_n]->insertData(static_cast<const char *>(field.data), size);
|
|
|
|
}
|
2018-02-14 15:03:48 +00:00
|
|
|
}
|
|
|
|
}
|
2017-09-05 01:08:26 +00:00
|
|
|
|
2018-02-14 15:03:48 +00:00
|
|
|
return sample_block.cloneWithColumns(std::move(columns));
|
2017-09-05 01:08:26 +00:00
|
|
|
}
|
2017-09-07 21:04:48 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
|
2017-09-05 01:08:26 +00:00
|
|
|
LibraryDictionarySource::LibraryDictionarySource(const DictionaryStructure & dict_struct_,
|
|
|
|
const Poco::Util::AbstractConfiguration & config,
|
|
|
|
const std::string & config_prefix,
|
|
|
|
Block & sample_block,
|
|
|
|
const Context & context)
|
2018-02-14 15:03:48 +00:00
|
|
|
: log(&Logger::get("LibraryDictionarySource"))
|
|
|
|
, dict_struct{dict_struct_}
|
|
|
|
, config_prefix{config_prefix}
|
|
|
|
, path{config.getString(config_prefix + ".path", "")}
|
|
|
|
, sample_block{sample_block}
|
|
|
|
, context(context)
|
2017-09-05 01:08:26 +00:00
|
|
|
{
|
|
|
|
if (!Poco::File(path).exists())
|
|
|
|
throw Exception("LibraryDictionarySource: Can't load lib " + toString() + ": " + Poco::File(path).path() + " - File doesn't exist",
|
|
|
|
ErrorCodes::FILE_DOESNT_EXIST);
|
|
|
|
description.init(sample_block);
|
|
|
|
library = std::make_shared<SharedLibrary>(path);
|
|
|
|
settings = std::make_shared<CStringsHolder>(getLibSettings(config, config_prefix + lib_config_settings));
|
2018-06-06 20:21:16 +00:00
|
|
|
if (auto libNew = library->tryGet<decltype(lib_data) (*)(decltype(&settings->strings), decltype(&ClickHouseLibrary::log))>(
|
2018-06-07 17:41:14 +00:00
|
|
|
"ClickHouseDictionary_v3_libNew"))
|
2018-06-06 20:21:16 +00:00
|
|
|
lib_data = libNew(&settings->strings, ClickHouseLibrary::log);
|
2017-09-05 01:08:26 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
LibraryDictionarySource::LibraryDictionarySource(const LibraryDictionarySource & other)
|
2018-02-14 15:03:48 +00:00
|
|
|
: log(&Logger::get("LibraryDictionarySource"))
|
|
|
|
, dict_struct{other.dict_struct}
|
|
|
|
, config_prefix{other.config_prefix}
|
|
|
|
, path{other.path}
|
|
|
|
, sample_block{other.sample_block}
|
|
|
|
, context(other.context)
|
|
|
|
, library{other.library}
|
|
|
|
, description{other.description}
|
|
|
|
, settings{other.settings}
|
2018-02-26 16:57:14 +00:00
|
|
|
{
|
2018-06-07 17:41:14 +00:00
|
|
|
if (auto libClone = library->tryGet<decltype(lib_data) (*)(decltype(other.lib_data))>("ClickHouseDictionary_v3_libClone"))
|
2018-03-13 23:13:39 +00:00
|
|
|
lib_data = libClone(other.lib_data);
|
2018-06-06 20:21:16 +00:00
|
|
|
else if (auto libNew = library->tryGet<decltype(lib_data) (*)(decltype(&settings->strings), decltype(&ClickHouseLibrary::log))>(
|
2018-06-07 17:41:14 +00:00
|
|
|
"ClickHouseDictionary_v3_libNew"))
|
2018-06-06 20:21:16 +00:00
|
|
|
lib_data = libNew(&settings->strings, ClickHouseLibrary::log);
|
2018-02-26 16:57:14 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
LibraryDictionarySource::~LibraryDictionarySource()
|
2017-09-05 01:08:26 +00:00
|
|
|
{
|
2018-06-07 17:41:14 +00:00
|
|
|
if (auto libDelete = library->tryGet<void (*)(decltype(lib_data))>("ClickHouseDictionary_v3_libDelete"))
|
2018-03-13 23:13:39 +00:00
|
|
|
libDelete(lib_data);
|
2017-09-05 01:08:26 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
BlockInputStreamPtr LibraryDictionarySource::loadAll()
|
|
|
|
{
|
|
|
|
LOG_TRACE(log, "loadAll " + toString());
|
|
|
|
|
|
|
|
auto columns_holder = std::make_unique<ClickHouseLibrary::CString[]>(dict_struct.attributes.size());
|
|
|
|
ClickHouseLibrary::CStrings columns{
|
2017-09-06 18:40:49 +00:00
|
|
|
static_cast<decltype(ClickHouseLibrary::CStrings::data)>(columns_holder.get()), dict_struct.attributes.size()};
|
2017-09-05 01:08:26 +00:00
|
|
|
size_t i = 0;
|
|
|
|
for (auto & a : dict_struct.attributes)
|
|
|
|
{
|
|
|
|
columns.data[i] = a.name.c_str();
|
|
|
|
++i;
|
|
|
|
}
|
|
|
|
void * data_ptr = nullptr;
|
|
|
|
|
2018-02-26 16:57:14 +00:00
|
|
|
/// Get function pointer before dataNew call because library->get may throw.
|
2018-03-13 23:13:39 +00:00
|
|
|
auto func_loadAll
|
2018-06-07 17:41:14 +00:00
|
|
|
= library->get<void * (*)(decltype(data_ptr), decltype(&settings->strings), decltype(&columns))>("ClickHouseDictionary_v3_loadAll");
|
|
|
|
data_ptr = library->get<decltype(data_ptr) (*)(decltype(lib_data))>("ClickHouseDictionary_v3_dataNew")(lib_data);
|
2018-03-13 23:13:39 +00:00
|
|
|
auto data = func_loadAll(data_ptr, &settings->strings, &columns);
|
2017-12-15 03:04:33 +00:00
|
|
|
auto block = dataToBlock(description.sample_block, data);
|
2018-06-07 17:41:14 +00:00
|
|
|
SCOPE_EXIT(library->get<void (*)(decltype(lib_data), decltype(data_ptr))>("ClickHouseDictionary_v3_dataDelete")(lib_data, data_ptr));
|
2017-09-05 01:08:26 +00:00
|
|
|
return std::make_shared<OneBlockInputStream>(block);
|
|
|
|
}
|
|
|
|
|
|
|
|
BlockInputStreamPtr LibraryDictionarySource::loadIds(const std::vector<UInt64> & ids)
|
|
|
|
{
|
|
|
|
LOG_TRACE(log, "loadIds " << toString() << " size = " << ids.size());
|
|
|
|
|
2017-09-08 18:24:15 +00:00
|
|
|
const ClickHouseLibrary::VectorUInt64 ids_data{ext::bit_cast<decltype(ClickHouseLibrary::VectorUInt64::data)>(ids.data()), ids.size()};
|
2017-09-05 01:08:26 +00:00
|
|
|
auto columns_holder = std::make_unique<ClickHouseLibrary::CString[]>(dict_struct.attributes.size());
|
|
|
|
ClickHouseLibrary::CStrings columns_pass{
|
2017-09-06 18:40:49 +00:00
|
|
|
static_cast<decltype(ClickHouseLibrary::CStrings::data)>(columns_holder.get()), dict_struct.attributes.size()};
|
2017-09-05 01:08:26 +00:00
|
|
|
size_t i = 0;
|
|
|
|
for (auto & a : dict_struct.attributes)
|
|
|
|
{
|
|
|
|
columns_pass.data[i] = a.name.c_str();
|
|
|
|
++i;
|
|
|
|
}
|
|
|
|
void * data_ptr = nullptr;
|
|
|
|
|
2018-02-26 16:57:14 +00:00
|
|
|
/// Get function pointer before dataNew call because library->get may throw.
|
2018-03-13 23:13:39 +00:00
|
|
|
auto func_loadIds
|
|
|
|
= library->get<void * (*)(decltype(data_ptr), decltype(&settings->strings), decltype(&columns_pass), decltype(&ids_data))>(
|
2018-06-07 17:41:14 +00:00
|
|
|
"ClickHouseDictionary_v3_loadIds");
|
|
|
|
data_ptr = library->get<decltype(data_ptr) (*)(decltype(lib_data))>("ClickHouseDictionary_v3_dataNew")(lib_data);
|
2018-03-13 23:13:39 +00:00
|
|
|
auto data = func_loadIds(data_ptr, &settings->strings, &columns_pass, &ids_data);
|
2017-12-15 03:04:33 +00:00
|
|
|
auto block = dataToBlock(description.sample_block, data);
|
2018-06-07 17:41:14 +00:00
|
|
|
SCOPE_EXIT(library->get<void (*)(decltype(lib_data), decltype(data_ptr))>("ClickHouseDictionary_v3_dataDelete")(lib_data, data_ptr));
|
2017-09-05 01:08:26 +00:00
|
|
|
return std::make_shared<OneBlockInputStream>(block);
|
|
|
|
}
|
|
|
|
|
|
|
|
BlockInputStreamPtr LibraryDictionarySource::loadKeys(const Columns & key_columns, const std::vector<std::size_t> & requested_rows)
|
|
|
|
{
|
|
|
|
LOG_TRACE(log, "loadKeys " << toString() << " size = " << requested_rows.size());
|
|
|
|
|
2018-07-18 17:06:13 +00:00
|
|
|
auto holder = std::make_unique<ClickHouseLibrary::Row[]>(key_columns.size());
|
2018-07-19 08:47:35 +00:00
|
|
|
std::vector<std::unique_ptr<ClickHouseLibrary::Field[]>> column_data_holders;
|
2018-07-18 17:06:13 +00:00
|
|
|
for (size_t i = 0; i < key_columns.size(); ++i)
|
2017-09-05 01:08:26 +00:00
|
|
|
{
|
2018-07-19 08:47:35 +00:00
|
|
|
auto cell_holder = std::make_unique<ClickHouseLibrary::Field[]>(requested_rows.size());
|
2018-07-18 17:06:13 +00:00
|
|
|
for (size_t j = 0; j < requested_rows.size(); ++j)
|
|
|
|
{
|
2018-07-19 08:47:35 +00:00
|
|
|
auto data_ref = key_columns[i]->getDataAt(requested_rows[j]);
|
|
|
|
cell_holder[j] = ClickHouseLibrary::Field{.data = static_cast<const void *>(data_ref.data), .size = data_ref.size};
|
2018-07-18 17:06:13 +00:00
|
|
|
}
|
|
|
|
holder[i]
|
2018-07-19 08:47:35 +00:00
|
|
|
= ClickHouseLibrary::Row{.data = static_cast<ClickHouseLibrary::Field *>(cell_holder.get()), .size = requested_rows.size()};
|
2018-07-18 17:06:13 +00:00
|
|
|
|
2018-07-19 08:47:35 +00:00
|
|
|
column_data_holders.push_back(std::move(cell_holder));
|
2017-09-05 01:08:26 +00:00
|
|
|
}
|
|
|
|
|
2018-07-19 08:47:35 +00:00
|
|
|
ClickHouseLibrary::Table request_cols{.data = static_cast<ClickHouseLibrary::Row *>(holder.get()), .size = key_columns.size()};
|
2018-07-18 17:06:13 +00:00
|
|
|
|
|
|
|
void * data_ptr = nullptr;
|
2018-02-26 16:57:14 +00:00
|
|
|
/// Get function pointer before dataNew call because library->get may throw.
|
2018-07-19 08:47:35 +00:00
|
|
|
auto func_loadKeys = library->get<void * (*)(decltype(data_ptr), decltype(&settings->strings), decltype(&request_cols))>(
|
2018-07-18 17:06:13 +00:00
|
|
|
"ClickHouseDictionary_v3_loadKeys");
|
2018-06-07 17:41:14 +00:00
|
|
|
data_ptr = library->get<decltype(data_ptr) (*)(decltype(lib_data))>("ClickHouseDictionary_v3_dataNew")(lib_data);
|
2018-07-19 08:47:35 +00:00
|
|
|
auto data = func_loadKeys(data_ptr, &settings->strings, &request_cols);
|
2017-12-15 03:04:33 +00:00
|
|
|
auto block = dataToBlock(description.sample_block, data);
|
2018-06-07 17:41:14 +00:00
|
|
|
SCOPE_EXIT(library->get<void (*)(decltype(lib_data), decltype(data_ptr))>("ClickHouseDictionary_v3_dataDelete")(lib_data, data_ptr));
|
2017-09-05 01:08:26 +00:00
|
|
|
return std::make_shared<OneBlockInputStream>(block);
|
|
|
|
}
|
|
|
|
|
|
|
|
bool LibraryDictionarySource::isModified() const
|
|
|
|
{
|
2018-03-13 23:13:39 +00:00
|
|
|
if (auto func_isModified
|
2018-06-07 17:41:14 +00:00
|
|
|
= library->tryGet<bool (*)(decltype(lib_data), decltype(&settings->strings))>("ClickHouseDictionary_v3_isModified"))
|
2018-03-13 23:13:39 +00:00
|
|
|
return func_isModified(lib_data, &settings->strings);
|
2017-09-05 01:08:26 +00:00
|
|
|
return true;
|
|
|
|
}
|
|
|
|
|
|
|
|
bool LibraryDictionarySource::supportsSelectiveLoad() const
|
|
|
|
{
|
2018-03-13 23:13:39 +00:00
|
|
|
if (auto func_supportsSelectiveLoad
|
2018-06-07 17:41:14 +00:00
|
|
|
= library->tryGet<bool (*)(decltype(lib_data), decltype(&settings->strings))>("ClickHouseDictionary_v3_supportsSelectiveLoad"))
|
2018-03-13 23:13:39 +00:00
|
|
|
return func_supportsSelectiveLoad(lib_data, &settings->strings);
|
2017-09-05 01:08:26 +00:00
|
|
|
return true;
|
|
|
|
}
|
|
|
|
|
|
|
|
DictionarySourcePtr LibraryDictionarySource::clone() const
|
|
|
|
{
|
|
|
|
return std::make_unique<LibraryDictionarySource>(*this);
|
|
|
|
}
|
|
|
|
|
|
|
|
std::string LibraryDictionarySource::toString() const
|
|
|
|
{
|
|
|
|
return path;
|
|
|
|
}
|
2018-11-28 11:37:12 +00:00
|
|
|
|
|
|
|
void registerDictionarySourceLibrary(DictionarySourceFactory & factory)
|
|
|
|
{
|
|
|
|
auto createTableSource = [=](const DictionaryStructure & dict_struct,
|
|
|
|
const Poco::Util::AbstractConfiguration & config,
|
|
|
|
const std::string & config_prefix,
|
|
|
|
Block & sample_block,
|
|
|
|
const Context & context) -> DictionarySourcePtr {
|
|
|
|
return std::make_unique<LibraryDictionarySource>(dict_struct, config, config_prefix + ".library", sample_block, context);
|
|
|
|
};
|
|
|
|
factory.registerSource("library", createTableSource);
|
|
|
|
}
|
|
|
|
|
2017-09-05 01:08:26 +00:00
|
|
|
}
|