mirror of
https://github.com/ClickHouse/ClickHouse.git
synced 2024-11-25 17:12:03 +00:00
70d1adfe4b
* save format string for NetException * format exceptions * format exceptions 2 * format exceptions 3 * format exceptions 4 * format exceptions 5 * format exceptions 6 * fix * format exceptions 7 * format exceptions 8 * Update MergeTreeIndexGin.cpp * Update AggregateFunctionMap.cpp * Update AggregateFunctionMap.cpp * fix
213 lines
9.0 KiB
C++
213 lines
9.0 KiB
C++
#include "ExternalDictionaryLibraryHandler.h"
|
|
|
|
#include <base/scope_guard.h>
|
|
#include <base/find_symbols.h>
|
|
#include <IO/ReadHelpers.h>
|
|
|
|
|
|
namespace DB
|
|
{
|
|
|
|
namespace ErrorCodes
|
|
{
|
|
extern const int EXTERNAL_LIBRARY_ERROR;
|
|
extern const int SIZES_OF_COLUMNS_DOESNT_MATCH;
|
|
}
|
|
|
|
|
|
ExternalDictionaryLibraryHandler::ExternalDictionaryLibraryHandler(
|
|
const std::string & library_path_,
|
|
const std::vector<std::string> & library_settings,
|
|
const Block & sample_block_,
|
|
const std::vector<std::string> & attributes_names_)
|
|
: library_path(library_path_)
|
|
, sample_block(sample_block_)
|
|
, attributes_names(attributes_names_)
|
|
{
|
|
library = std::make_shared<SharedLibrary>(library_path);
|
|
settings_holder = std::make_shared<CStringsHolder>(CStringsHolder(library_settings));
|
|
|
|
auto lib_new = library->tryGet<ExternalDictionaryLibraryAPI::LibraryNewFunc>(ExternalDictionaryLibraryAPI::LIBRARY_CREATE_NEW_FUNC_NAME);
|
|
|
|
if (lib_new)
|
|
lib_data = lib_new(&settings_holder->strings, ExternalDictionaryLibraryAPI::log);
|
|
else
|
|
throw Exception(ErrorCodes::EXTERNAL_LIBRARY_ERROR, "Method extDict_libNew failed");
|
|
}
|
|
|
|
|
|
ExternalDictionaryLibraryHandler::ExternalDictionaryLibraryHandler(const ExternalDictionaryLibraryHandler & other)
|
|
: library_path{other.library_path}
|
|
, sample_block{other.sample_block}
|
|
, attributes_names{other.attributes_names}
|
|
, library{other.library}
|
|
, settings_holder{other.settings_holder}
|
|
{
|
|
|
|
auto lib_clone = library->tryGet<ExternalDictionaryLibraryAPI::LibraryCloneFunc>(ExternalDictionaryLibraryAPI::LIBRARY_CLONE_FUNC_NAME);
|
|
|
|
if (lib_clone)
|
|
{
|
|
lib_data = lib_clone(other.lib_data);
|
|
}
|
|
else
|
|
{
|
|
auto lib_new = library->tryGet<ExternalDictionaryLibraryAPI::LibraryNewFunc>(ExternalDictionaryLibraryAPI::LIBRARY_CREATE_NEW_FUNC_NAME);
|
|
|
|
if (lib_new)
|
|
lib_data = lib_new(&settings_holder->strings, ExternalDictionaryLibraryAPI::log);
|
|
}
|
|
}
|
|
|
|
|
|
ExternalDictionaryLibraryHandler::~ExternalDictionaryLibraryHandler()
|
|
{
|
|
auto lib_delete = library->tryGet<ExternalDictionaryLibraryAPI::LibraryDeleteFunc>(ExternalDictionaryLibraryAPI::LIBRARY_DELETE_FUNC_NAME);
|
|
|
|
if (lib_delete)
|
|
lib_delete(lib_data);
|
|
}
|
|
|
|
|
|
bool ExternalDictionaryLibraryHandler::isModified()
|
|
{
|
|
auto func_is_modified = library->tryGet<ExternalDictionaryLibraryAPI::LibraryIsModifiedFunc>(ExternalDictionaryLibraryAPI::LIBRARY_IS_MODIFIED_FUNC_NAME);
|
|
|
|
if (func_is_modified)
|
|
return func_is_modified(lib_data, &settings_holder->strings);
|
|
|
|
return true;
|
|
}
|
|
|
|
|
|
bool ExternalDictionaryLibraryHandler::supportsSelectiveLoad()
|
|
{
|
|
auto func_supports_selective_load = library->tryGet<ExternalDictionaryLibraryAPI::LibrarySupportsSelectiveLoadFunc>(ExternalDictionaryLibraryAPI::LIBRARY_SUPPORTS_SELECTIVE_LOAD_FUNC_NAME);
|
|
|
|
if (func_supports_selective_load)
|
|
return func_supports_selective_load(lib_data, &settings_holder->strings);
|
|
|
|
return true;
|
|
}
|
|
|
|
|
|
Block ExternalDictionaryLibraryHandler::loadAll()
|
|
{
|
|
auto columns_holder = std::make_unique<ExternalDictionaryLibraryAPI::CString[]>(attributes_names.size());
|
|
ExternalDictionaryLibraryAPI::CStrings columns{static_cast<decltype(ExternalDictionaryLibraryAPI::CStrings::data)>(columns_holder.get()), attributes_names.size()};
|
|
for (size_t i = 0; i < attributes_names.size(); ++i)
|
|
columns.data[i] = attributes_names[i].c_str();
|
|
|
|
auto load_all_func = library->get<ExternalDictionaryLibraryAPI::LibraryLoadAllFunc>(ExternalDictionaryLibraryAPI::LIBRARY_LOAD_ALL_FUNC_NAME);
|
|
auto data_new_func = library->get<ExternalDictionaryLibraryAPI::LibraryDataNewFunc>(ExternalDictionaryLibraryAPI::LIBRARY_DATA_NEW_FUNC_NAME);
|
|
auto data_delete_func = library->get<ExternalDictionaryLibraryAPI::LibraryDataDeleteFunc>(ExternalDictionaryLibraryAPI::LIBRARY_DATA_DELETE_FUNC_NAME);
|
|
|
|
ExternalDictionaryLibraryAPI::LibraryData data_ptr = data_new_func(lib_data);
|
|
SCOPE_EXIT(data_delete_func(lib_data, data_ptr));
|
|
|
|
ExternalDictionaryLibraryAPI::RawClickHouseLibraryTable data = load_all_func(data_ptr, &settings_holder->strings, &columns);
|
|
return dataToBlock(data);
|
|
}
|
|
|
|
|
|
Block ExternalDictionaryLibraryHandler::loadIds(const std::vector<uint64_t> & ids)
|
|
{
|
|
const ExternalDictionaryLibraryAPI::VectorUInt64 ids_data{std::bit_cast<decltype(ExternalDictionaryLibraryAPI::VectorUInt64::data)>(ids.data()), ids.size()};
|
|
|
|
auto columns_holder = std::make_unique<ExternalDictionaryLibraryAPI::CString[]>(attributes_names.size());
|
|
ExternalDictionaryLibraryAPI::CStrings columns_pass{static_cast<decltype(ExternalDictionaryLibraryAPI::CStrings::data)>(columns_holder.get()), attributes_names.size()};
|
|
|
|
auto load_ids_func = library->get<ExternalDictionaryLibraryAPI::LibraryLoadIdsFunc>(ExternalDictionaryLibraryAPI::LIBRARY_LOAD_IDS_FUNC_NAME);
|
|
auto data_new_func = library->get<ExternalDictionaryLibraryAPI::LibraryDataNewFunc>(ExternalDictionaryLibraryAPI::LIBRARY_DATA_NEW_FUNC_NAME);
|
|
auto data_delete_func = library->get<ExternalDictionaryLibraryAPI::LibraryDataDeleteFunc>(ExternalDictionaryLibraryAPI::LIBRARY_DATA_DELETE_FUNC_NAME);
|
|
|
|
ExternalDictionaryLibraryAPI::LibraryData data_ptr = data_new_func(lib_data);
|
|
SCOPE_EXIT(data_delete_func(lib_data, data_ptr));
|
|
|
|
ExternalDictionaryLibraryAPI::RawClickHouseLibraryTable data = load_ids_func(data_ptr, &settings_holder->strings, &columns_pass, &ids_data);
|
|
return dataToBlock(data);
|
|
}
|
|
|
|
|
|
Block ExternalDictionaryLibraryHandler::loadKeys(const Columns & key_columns)
|
|
{
|
|
auto holder = std::make_unique<ExternalDictionaryLibraryAPI::Row[]>(key_columns.size());
|
|
std::vector<std::unique_ptr<ExternalDictionaryLibraryAPI::Field[]>> column_data_holders;
|
|
|
|
for (size_t i = 0; i < key_columns.size(); ++i)
|
|
{
|
|
auto cell_holder = std::make_unique<ExternalDictionaryLibraryAPI::Field[]>(key_columns[i]->size());
|
|
|
|
for (size_t j = 0; j < key_columns[i]->size(); ++j)
|
|
{
|
|
auto data_ref = key_columns[i]->getDataAt(j);
|
|
|
|
cell_holder[j] = ExternalDictionaryLibraryAPI::Field{
|
|
.data = static_cast<const void *>(data_ref.data),
|
|
.size = data_ref.size};
|
|
}
|
|
|
|
holder[i] = ExternalDictionaryLibraryAPI::Row{
|
|
.data = static_cast<ExternalDictionaryLibraryAPI::Field *>(cell_holder.get()),
|
|
.size = key_columns[i]->size()};
|
|
|
|
column_data_holders.push_back(std::move(cell_holder));
|
|
}
|
|
|
|
ExternalDictionaryLibraryAPI::Table request_cols{
|
|
.data = static_cast<ExternalDictionaryLibraryAPI::Row *>(holder.get()),
|
|
.size = key_columns.size()};
|
|
|
|
auto load_keys_func = library->get<ExternalDictionaryLibraryAPI::LibraryLoadKeysFunc>(ExternalDictionaryLibraryAPI::LIBRARY_LOAD_KEYS_FUNC_NAME);
|
|
auto data_new_func = library->get<ExternalDictionaryLibraryAPI::LibraryDataNewFunc>(ExternalDictionaryLibraryAPI::LIBRARY_DATA_NEW_FUNC_NAME);
|
|
auto data_delete_func = library->get<ExternalDictionaryLibraryAPI::LibraryDataDeleteFunc>(ExternalDictionaryLibraryAPI::LIBRARY_DATA_DELETE_FUNC_NAME);
|
|
|
|
ExternalDictionaryLibraryAPI::LibraryData data_ptr = data_new_func(lib_data);
|
|
SCOPE_EXIT(data_delete_func(lib_data, data_ptr));
|
|
|
|
ExternalDictionaryLibraryAPI::RawClickHouseLibraryTable data = load_keys_func(data_ptr, &settings_holder->strings, &request_cols);
|
|
return dataToBlock(data);
|
|
}
|
|
|
|
|
|
Block ExternalDictionaryLibraryHandler::dataToBlock(ExternalDictionaryLibraryAPI::RawClickHouseLibraryTable data)
|
|
{
|
|
if (!data)
|
|
throw Exception(ErrorCodes::EXTERNAL_LIBRARY_ERROR, "LibraryDictionarySource: No data returned");
|
|
|
|
const auto * columns_received = static_cast<const ExternalDictionaryLibraryAPI::Table *>(data);
|
|
if (columns_received->error_code)
|
|
throw Exception(ErrorCodes::EXTERNAL_LIBRARY_ERROR, "LibraryDictionarySource: Returned error: {} {}",
|
|
std::to_string(columns_received->error_code), (columns_received->error_string ? columns_received->error_string : ""));
|
|
|
|
MutableColumns columns = sample_block.cloneEmptyColumns();
|
|
|
|
for (size_t col_n = 0; col_n < columns_received->size; ++col_n)
|
|
{
|
|
if (columns.size() != columns_received->data[col_n].size)
|
|
throw Exception(ErrorCodes::SIZES_OF_COLUMNS_DOESNT_MATCH, "LibraryDictionarySource: "
|
|
"Returned unexpected number of columns: {}, must be {}",
|
|
columns_received->data[col_n].size, columns.size());
|
|
|
|
for (size_t row_n = 0; row_n < columns_received->data[col_n].size; ++row_n)
|
|
{
|
|
const auto & field = columns_received->data[col_n].data[row_n];
|
|
if (!field.data)
|
|
{
|
|
/// sample_block contains null_value (from config) inside corresponding column
|
|
const auto & col = sample_block.getByPosition(row_n);
|
|
columns[row_n]->insertFrom(*(col.column), 0);
|
|
}
|
|
else
|
|
{
|
|
const auto & size = field.size;
|
|
columns[row_n]->insertData(static_cast<const char *>(field.data), size);
|
|
}
|
|
}
|
|
}
|
|
|
|
return sample_block.cloneWithColumns(std::move(columns));
|
|
}
|
|
|
|
}
|