From ea73b98fb9fc2ad962f8cb01dcc69ad85268f6f4 Mon Sep 17 00:00:00 2001 From: Robert Schulze Date: Wed, 3 Aug 2022 11:19:13 +0000 Subject: [PATCH] Prepare library-bridge for catboost integration - Rename generic file and identifier names in library-bridge to something more dictionary-specific. This is needed because later on, catboost will be integrated into library-bridge. - Also: Some smaller fixes like typos and un-inlining non-performance critical code. - The logic remains unchanged in this commit. --- programs/CMakeLists.txt | 6 +- programs/library-bridge/CMakeLists.txt | 12 +- ...e.cpp => ExternalDictionaryLibraryAPI.cpp} | 23 +- .../ExternalDictionaryLibraryAPI.h | 106 +++++++++ .../ExternalDictionaryLibraryHandler.cpp | 214 ++++++++++++++++++ ...r.h => ExternalDictionaryLibraryHandler.h} | 16 +- ...xternalDictionaryLibraryHandlerFactory.cpp | 62 +++++ ...ExternalDictionaryLibraryHandlerFactory.h} | 8 +- ...ils.h => ExternalDictionaryLibraryUtils.h} | 8 +- programs/library-bridge/HandlerFactory.cpp | 23 -- programs/library-bridge/HandlerFactory.h | 37 --- programs/library-bridge/LibraryBridge.cpp | 16 +- programs/library-bridge/LibraryBridge.h | 13 +- .../LibraryBridgeHandlerFactory.cpp | 34 +++ .../LibraryBridgeHandlerFactory.h | 27 +++ ...Handlers.cpp => LibraryBridgeHandlers.cpp} | 78 ++++--- .../{Handlers.h => LibraryBridgeHandlers.h} | 27 +-- programs/library-bridge/LibraryInterface.h | 110 --------- .../library-bridge/SharedLibraryHandler.cpp | 214 ------------------ .../SharedLibraryHandlerFactory.cpp | 62 ----- programs/library-bridge/library-bridge.cpp | 1 - programs/odbc-bridge/CMakeLists.txt | 6 +- programs/odbc-bridge/ODBCBridge.cpp | 16 +- programs/odbc-bridge/ODBCBridge.h | 13 +- ...dlerFactory.cpp => ODBCHandlerFactory.cpp} | 10 +- ...{HandlerFactory.h => ODBCHandlerFactory.h} | 8 +- src/BridgeHelper/IBridgeHelper.h | 1 - src/BridgeHelper/LibraryBridgeHelper.cpp | 26 +-- src/BridgeHelper/LibraryBridgeHelper.h | 24 +- src/Dictionaries/LibraryDictionarySource.cpp | 4 +- 30 files changed, 598 insertions(+), 607 deletions(-) rename programs/library-bridge/{LibraryInterface.cpp => ExternalDictionaryLibraryAPI.cpp} (55%) create mode 100644 programs/library-bridge/ExternalDictionaryLibraryAPI.h create mode 100644 programs/library-bridge/ExternalDictionaryLibraryHandler.cpp rename programs/library-bridge/{SharedLibraryHandler.h => ExternalDictionaryLibraryHandler.h} (64%) create mode 100644 programs/library-bridge/ExternalDictionaryLibraryHandlerFactory.cpp rename programs/library-bridge/{SharedLibraryHandlerFactory.h => ExternalDictionaryLibraryHandlerFactory.h} (77%) rename programs/library-bridge/{LibraryUtils.h => ExternalDictionaryLibraryUtils.h} (67%) delete mode 100644 programs/library-bridge/HandlerFactory.cpp delete mode 100644 programs/library-bridge/HandlerFactory.h create mode 100644 programs/library-bridge/LibraryBridgeHandlerFactory.cpp create mode 100644 programs/library-bridge/LibraryBridgeHandlerFactory.h rename programs/library-bridge/{Handlers.cpp => LibraryBridgeHandlers.cpp} (80%) rename programs/library-bridge/{Handlers.h => LibraryBridgeHandlers.h} (53%) delete mode 100644 programs/library-bridge/LibraryInterface.h delete mode 100644 programs/library-bridge/SharedLibraryHandler.cpp delete mode 100644 programs/library-bridge/SharedLibraryHandlerFactory.cpp rename programs/odbc-bridge/{HandlerFactory.cpp => ODBCHandlerFactory.cpp} (83%) rename programs/odbc-bridge/{HandlerFactory.h => ODBCHandlerFactory.h} (79%) diff --git a/programs/CMakeLists.txt b/programs/CMakeLists.txt index 31943ef7dae..873a508d94a 100644 --- a/programs/CMakeLists.txt +++ b/programs/CMakeLists.txt @@ -44,15 +44,13 @@ option (ENABLE_CLICKHOUSE_OBFUSCATOR "Table data obfuscator (convert real data t # https://clickhouse.com/docs/en/operations/utilities/odbc-bridge/ # TODO Also needs NANODBC. if (ENABLE_ODBC AND NOT USE_MUSL) - option (ENABLE_CLICKHOUSE_ODBC_BRIDGE "HTTP-server working like a proxy to ODBC driver" - ${ENABLE_CLICKHOUSE_ALL}) + option (ENABLE_CLICKHOUSE_ODBC_BRIDGE "HTTP-server working like a proxy to ODBC driver" ${ENABLE_CLICKHOUSE_ALL}) else () option (ENABLE_CLICKHOUSE_ODBC_BRIDGE "HTTP-server working like a proxy to ODBC driver" OFF) endif () if (NOT USE_MUSL) - option (ENABLE_CLICKHOUSE_LIBRARY_BRIDGE "HTTP-server working like a proxy to Library dictionary source" - ${ENABLE_CLICKHOUSE_ALL}) + option (ENABLE_CLICKHOUSE_LIBRARY_BRIDGE "HTTP-server working like a proxy to Library dictionary source" ${ENABLE_CLICKHOUSE_ALL}) endif () # https://presentations.clickhouse.com/matemarketing_2020/ diff --git a/programs/library-bridge/CMakeLists.txt b/programs/library-bridge/CMakeLists.txt index 2f52652c929..40cabacded4 100644 --- a/programs/library-bridge/CMakeLists.txt +++ b/programs/library-bridge/CMakeLists.txt @@ -1,13 +1,13 @@ include(${ClickHouse_SOURCE_DIR}/cmake/split_debug_symbols.cmake) set (CLICKHOUSE_LIBRARY_BRIDGE_SOURCES - library-bridge.cpp - LibraryInterface.cpp + ExternalDictionaryLibraryAPI.cpp + ExternalDictionaryLibraryHandler.cpp + ExternalDictionaryLibraryHandlerFactory.cpp LibraryBridge.cpp - Handlers.cpp - HandlerFactory.cpp - SharedLibraryHandler.cpp - SharedLibraryHandlerFactory.cpp + LibraryBridgeHandlerFactory.cpp + LibraryBridgeHandlers.cpp + library-bridge.cpp ) if (OS_LINUX) diff --git a/programs/library-bridge/LibraryInterface.cpp b/programs/library-bridge/ExternalDictionaryLibraryAPI.cpp similarity index 55% rename from programs/library-bridge/LibraryInterface.cpp rename to programs/library-bridge/ExternalDictionaryLibraryAPI.cpp index cae681e901a..70cd6fca375 100644 --- a/programs/library-bridge/LibraryInterface.cpp +++ b/programs/library-bridge/ExternalDictionaryLibraryAPI.cpp @@ -1,4 +1,4 @@ -#include "LibraryInterface.h" +#include "ExternalDictionaryLibraryAPI.h" #include @@ -7,24 +7,7 @@ namespace const char DICT_LOGGER_NAME[] = "LibraryDictionarySourceExternal"; } -namespace ClickHouseLibrary -{ - -std::string_view LIBRARY_CREATE_NEW_FUNC_NAME = "ClickHouseDictionary_v3_libNew"; -std::string_view LIBRARY_CLONE_FUNC_NAME = "ClickHouseDictionary_v3_libClone"; -std::string_view LIBRARY_DELETE_FUNC_NAME = "ClickHouseDictionary_v3_libDelete"; - -std::string_view LIBRARY_DATA_NEW_FUNC_NAME = "ClickHouseDictionary_v3_dataNew"; -std::string_view LIBRARY_DATA_DELETE_FUNC_NAME = "ClickHouseDictionary_v3_dataDelete"; - -std::string_view LIBRARY_LOAD_ALL_FUNC_NAME = "ClickHouseDictionary_v3_loadAll"; -std::string_view LIBRARY_LOAD_IDS_FUNC_NAME = "ClickHouseDictionary_v3_loadIds"; -std::string_view LIBRARY_LOAD_KEYS_FUNC_NAME = "ClickHouseDictionary_v3_loadKeys"; - -std::string_view LIBRARY_IS_MODIFIED_FUNC_NAME = "ClickHouseDictionary_v3_isModified"; -std::string_view LIBRARY_SUPPORTS_SELECTIVE_LOAD_FUNC_NAME = "ClickHouseDictionary_v3_supportsSelectiveLoad"; - -void log(LogLevel level, CString msg) +void ExternalDictionaryLibraryAPI::log(LogLevel level, CString msg) { auto & logger = Poco::Logger::get(DICT_LOGGER_NAME); switch (level) @@ -63,5 +46,3 @@ void log(LogLevel level, CString msg) break; } } - -} diff --git a/programs/library-bridge/ExternalDictionaryLibraryAPI.h b/programs/library-bridge/ExternalDictionaryLibraryAPI.h new file mode 100644 index 00000000000..04201710edb --- /dev/null +++ b/programs/library-bridge/ExternalDictionaryLibraryAPI.h @@ -0,0 +1,106 @@ +#pragma once + +#include +#include + +#define CLICKHOUSE_DICTIONARY_LIBRARY_API 1 + +struct ExternalDictionaryLibraryAPI +{ + using CString = const char *; + using ColumnName = CString; + using ColumnNames = ColumnName[]; + + struct CStrings + { + CString * data = nullptr; + uint64_t size = 0; + }; + + struct VectorUInt64 + { + const uint64_t * data = nullptr; + uint64_t size = 0; + }; + + struct ColumnsUInt64 + { + VectorUInt64 * data = nullptr; + uint64_t size = 0; + }; + + struct Field + { + const void * data = nullptr; + uint64_t size = 0; + }; + + struct Row + { + const Field * data = nullptr; + uint64_t size = 0; + }; + + struct Table + { + const Row * data = nullptr; + uint64_t size = 0; + uint64_t error_code = 0; // 0 = ok; !0 = error, with message in error_string + const char * error_string = nullptr; + }; + + enum LogLevel + { + FATAL = 1, + CRITICAL, + ERROR, + WARNING, + NOTICE, + INFORMATION, + DEBUG, + TRACE, + }; + + static void log(LogLevel level, CString msg); + + using LibraryContext = void *; + using LibraryLoggerFunc = void (*)(LogLevel, CString /* message */); + using LibrarySettings = CStrings *; + using LibraryData = void *; + using RawClickHouseLibraryTable = void *; + /// Can be safely casted into const Table * with static_cast + using RequestedColumnsNames = CStrings *; + using RequestedIds = const VectorUInt64 *; + using RequestedKeys = Table *; + + using LibraryNewFunc = LibraryContext (*)(LibrarySettings, LibraryLoggerFunc); + static constexpr const char * LIBRARY_CREATE_NEW_FUNC_NAME = "ClickHouseDictionary_v3_libNew"; + + using LibraryCloneFunc = LibraryContext (*)(LibraryContext); + static constexpr const char * LIBRARY_CLONE_FUNC_NAME = "ClickHouseDictionary_v3_libClone"; + + using LibraryDeleteFunc = void (*)(LibraryContext); + static constexpr const char * LIBRARY_DELETE_FUNC_NAME = "ClickHouseDictionary_v3_libDelete"; + + using LibraryDataNewFunc = LibraryData (*)(LibraryContext); + static constexpr const char * LIBRARY_DATA_NEW_FUNC_NAME = "ClickHouseDictionary_v3_dataNew"; + + using LibraryDataDeleteFunc = void (*)(LibraryContext, LibraryData); + static constexpr const char * LIBRARY_DATA_DELETE_FUNC_NAME = "ClickHouseDictionary_v3_dataDelete"; + + using LibraryLoadAllFunc = RawClickHouseLibraryTable (*)(LibraryData, LibrarySettings, RequestedColumnsNames); + static constexpr const char * LIBRARY_LOAD_ALL_FUNC_NAME = "ClickHouseDictionary_v3_loadAll"; + + using LibraryLoadIdsFunc = RawClickHouseLibraryTable (*)(LibraryData, LibrarySettings, RequestedColumnsNames, RequestedIds); + static constexpr const char * LIBRARY_LOAD_IDS_FUNC_NAME = "ClickHouseDictionary_v3_loadIds"; + + /// There are no requested column names for load keys func + using LibraryLoadKeysFunc = RawClickHouseLibraryTable (*)(LibraryData, LibrarySettings, RequestedKeys); + static constexpr const char * LIBRARY_LOAD_KEYS_FUNC_NAME = "ClickHouseDictionary_v3_loadKeys"; + + using LibraryIsModifiedFunc = bool (*)(LibraryContext, LibrarySettings); + static constexpr const char * LIBRARY_IS_MODIFIED_FUNC_NAME = "ClickHouseDictionary_v3_isModified"; + + using LibrarySupportsSelectiveLoadFunc = bool (*)(LibraryContext, LibrarySettings); + static constexpr const char * LIBRARY_SUPPORTS_SELECTIVE_LOAD_FUNC_NAME = "ClickHouseDictionary_v3_supportsSelectiveLoad"; +}; diff --git a/programs/library-bridge/ExternalDictionaryLibraryHandler.cpp b/programs/library-bridge/ExternalDictionaryLibraryHandler.cpp new file mode 100644 index 00000000000..14850da2ebf --- /dev/null +++ b/programs/library-bridge/ExternalDictionaryLibraryHandler.cpp @@ -0,0 +1,214 @@ +#include "ExternalDictionaryLibraryHandler.h" + +#include +#include +#include +#include + + +namespace DB +{ + +namespace ErrorCodes +{ + extern const int EXTERNAL_LIBRARY_ERROR; + extern const int SIZES_OF_COLUMNS_DOESNT_MATCH; +} + + +ExternalDictionaryLibraryHandler::ExternalDictionaryLibraryHandler( + const std::string & library_path_, + const std::vector & library_settings, + const Block & sample_block_, + const std::vector & attributes_names_) + : library_path(library_path_) + , sample_block(sample_block_) + , attributes_names(attributes_names_) +{ + library = std::make_shared(library_path); + settings_holder = std::make_shared(CStringsHolder(library_settings)); + + auto lib_new = library->tryGet(ExternalDictionaryLibraryAPI::LIBRARY_CREATE_NEW_FUNC_NAME); + + if (lib_new) + lib_data = lib_new(&settings_holder->strings, ExternalDictionaryLibraryAPI::log); + else + throw Exception("Method extDict_libNew failed", ErrorCodes::EXTERNAL_LIBRARY_ERROR); +} + + +ExternalDictionaryLibraryHandler::ExternalDictionaryLibraryHandler(const ExternalDictionaryLibraryHandler & other) + : library_path{other.library_path} + , sample_block{other.sample_block} + , attributes_names{other.attributes_names} + , library{other.library} + , settings_holder{other.settings_holder} +{ + + auto lib_clone = library->tryGet(ExternalDictionaryLibraryAPI::LIBRARY_CLONE_FUNC_NAME); + + if (lib_clone) + { + lib_data = lib_clone(other.lib_data); + } + else + { + auto lib_new = library->tryGet(ExternalDictionaryLibraryAPI::LIBRARY_CREATE_NEW_FUNC_NAME); + + if (lib_new) + lib_data = lib_new(&settings_holder->strings, ExternalDictionaryLibraryAPI::log); + } +} + + +ExternalDictionaryLibraryHandler::~ExternalDictionaryLibraryHandler() +{ + auto lib_delete = library->tryGet(ExternalDictionaryLibraryAPI::LIBRARY_DELETE_FUNC_NAME); + + if (lib_delete) + lib_delete(lib_data); +} + + +bool ExternalDictionaryLibraryHandler::isModified() +{ + auto func_is_modified = library->tryGet(ExternalDictionaryLibraryAPI::LIBRARY_IS_MODIFIED_FUNC_NAME); + + if (func_is_modified) + return func_is_modified(lib_data, &settings_holder->strings); + + return true; +} + + +bool ExternalDictionaryLibraryHandler::supportsSelectiveLoad() +{ + auto func_supports_selective_load = library->tryGet(ExternalDictionaryLibraryAPI::LIBRARY_SUPPORTS_SELECTIVE_LOAD_FUNC_NAME); + + if (func_supports_selective_load) + return func_supports_selective_load(lib_data, &settings_holder->strings); + + return true; +} + + +Block ExternalDictionaryLibraryHandler::loadAll() +{ + auto columns_holder = std::make_unique(attributes_names.size()); + ExternalDictionaryLibraryAPI::CStrings columns{static_cast(columns_holder.get()), attributes_names.size()}; + for (size_t i = 0; i < attributes_names.size(); ++i) + columns.data[i] = attributes_names[i].c_str(); + + auto load_all_func = library->get(ExternalDictionaryLibraryAPI::LIBRARY_LOAD_ALL_FUNC_NAME); + auto data_new_func = library->get(ExternalDictionaryLibraryAPI::LIBRARY_DATA_NEW_FUNC_NAME); + auto data_delete_func = library->get(ExternalDictionaryLibraryAPI::LIBRARY_DATA_DELETE_FUNC_NAME); + + ExternalDictionaryLibraryAPI::LibraryData data_ptr = data_new_func(lib_data); + SCOPE_EXIT(data_delete_func(lib_data, data_ptr)); + + ExternalDictionaryLibraryAPI::RawClickHouseLibraryTable data = load_all_func(data_ptr, &settings_holder->strings, &columns); + return dataToBlock(data); +} + + +Block ExternalDictionaryLibraryHandler::loadIds(const std::vector & ids) +{ + const ExternalDictionaryLibraryAPI::VectorUInt64 ids_data{bit_cast(ids.data()), ids.size()}; + + auto columns_holder = std::make_unique(attributes_names.size()); + ExternalDictionaryLibraryAPI::CStrings columns_pass{static_cast(columns_holder.get()), attributes_names.size()}; + + auto load_ids_func = library->get(ExternalDictionaryLibraryAPI::LIBRARY_LOAD_IDS_FUNC_NAME); + auto data_new_func = library->get(ExternalDictionaryLibraryAPI::LIBRARY_DATA_NEW_FUNC_NAME); + auto data_delete_func = library->get(ExternalDictionaryLibraryAPI::LIBRARY_DATA_DELETE_FUNC_NAME); + + ExternalDictionaryLibraryAPI::LibraryData data_ptr = data_new_func(lib_data); + SCOPE_EXIT(data_delete_func(lib_data, data_ptr)); + + ExternalDictionaryLibraryAPI::RawClickHouseLibraryTable data = load_ids_func(data_ptr, &settings_holder->strings, &columns_pass, &ids_data); + return dataToBlock(data); +} + + +Block ExternalDictionaryLibraryHandler::loadKeys(const Columns & key_columns) +{ + auto holder = std::make_unique(key_columns.size()); + std::vector> column_data_holders; + + for (size_t i = 0; i < key_columns.size(); ++i) + { + auto cell_holder = std::make_unique(key_columns[i]->size()); + + for (size_t j = 0; j < key_columns[i]->size(); ++j) + { + auto data_ref = key_columns[i]->getDataAt(j); + + cell_holder[j] = ExternalDictionaryLibraryAPI::Field{ + .data = static_cast(data_ref.data), + .size = data_ref.size}; + } + + holder[i] = ExternalDictionaryLibraryAPI::Row{ + .data = static_cast(cell_holder.get()), + .size = key_columns[i]->size()}; + + column_data_holders.push_back(std::move(cell_holder)); + } + + ExternalDictionaryLibraryAPI::Table request_cols{ + .data = static_cast(holder.get()), + .size = key_columns.size()}; + + auto load_keys_func = library->get(ExternalDictionaryLibraryAPI::LIBRARY_LOAD_KEYS_FUNC_NAME); + auto data_new_func = library->get(ExternalDictionaryLibraryAPI::LIBRARY_DATA_NEW_FUNC_NAME); + auto data_delete_func = library->get(ExternalDictionaryLibraryAPI::LIBRARY_DATA_DELETE_FUNC_NAME); + + ExternalDictionaryLibraryAPI::LibraryData data_ptr = data_new_func(lib_data); + SCOPE_EXIT(data_delete_func(lib_data, data_ptr)); + + ExternalDictionaryLibraryAPI::RawClickHouseLibraryTable data = load_keys_func(data_ptr, &settings_holder->strings, &request_cols); + return dataToBlock(data); +} + + +Block ExternalDictionaryLibraryHandler::dataToBlock(ExternalDictionaryLibraryAPI::RawClickHouseLibraryTable data) +{ + if (!data) + throw Exception("LibraryDictionarySource: No data returned", ErrorCodes::EXTERNAL_LIBRARY_ERROR); + + const auto * columns_received = static_cast(data); + if (columns_received->error_code) + throw Exception( + "LibraryDictionarySource: Returned error: " + std::to_string(columns_received->error_code) + " " + (columns_received->error_string ? columns_received->error_string : ""), + ErrorCodes::EXTERNAL_LIBRARY_ERROR); + + MutableColumns columns = sample_block.cloneEmptyColumns(); + + for (size_t col_n = 0; col_n < columns_received->size; ++col_n) + { + if (columns.size() != columns_received->data[col_n].size) + throw Exception( + "LibraryDictionarySource: Returned unexpected number of columns: " + std::to_string(columns_received->data[col_n].size) + ", must be " + std::to_string(columns.size()), + ErrorCodes::SIZES_OF_COLUMNS_DOESNT_MATCH); + + for (size_t row_n = 0; row_n < columns_received->data[col_n].size; ++row_n) + { + const auto & field = columns_received->data[col_n].data[row_n]; + if (!field.data) + { + /// sample_block contains null_value (from config) inside corresponding column + const auto & col = sample_block.getByPosition(row_n); + columns[row_n]->insertFrom(*(col.column), 0); + } + else + { + const auto & size = field.size; + columns[row_n]->insertData(static_cast(field.data), size); + } + } + } + + return sample_block.cloneWithColumns(std::move(columns)); +} + +} diff --git a/programs/library-bridge/SharedLibraryHandler.h b/programs/library-bridge/ExternalDictionaryLibraryHandler.h similarity index 64% rename from programs/library-bridge/SharedLibraryHandler.h rename to programs/library-bridge/ExternalDictionaryLibraryHandler.h index 633180257b7..7713e9a6830 100644 --- a/programs/library-bridge/SharedLibraryHandler.h +++ b/programs/library-bridge/ExternalDictionaryLibraryHandler.h @@ -2,7 +2,7 @@ #include #include -#include "LibraryUtils.h" +#include "ExternalDictionaryLibraryUtils.h" namespace DB @@ -10,21 +10,21 @@ namespace DB /// A class that manages all operations with library dictionary. /// Every library dictionary source has its own object of this class, accessed by UUID. -class SharedLibraryHandler +class ExternalDictionaryLibraryHandler { public: - SharedLibraryHandler( + ExternalDictionaryLibraryHandler( const std::string & library_path_, const std::vector & library_settings, const Block & sample_block_, const std::vector & attributes_names_); - SharedLibraryHandler(const SharedLibraryHandler & other); + ExternalDictionaryLibraryHandler(const ExternalDictionaryLibraryHandler & other); - SharedLibraryHandler & operator=(const SharedLibraryHandler & other) = delete; + ExternalDictionaryLibraryHandler & operator=(const ExternalDictionaryLibraryHandler & other) = delete; - ~SharedLibraryHandler(); + ~ExternalDictionaryLibraryHandler(); Block loadAll(); @@ -39,7 +39,7 @@ public: const Block & getSampleBlock() { return sample_block; } private: - Block dataToBlock(const ClickHouseLibrary::RawClickHouseLibraryTable data); + Block dataToBlock(ExternalDictionaryLibraryAPI::RawClickHouseLibraryTable data); std::string library_path; const Block sample_block; @@ -50,6 +50,6 @@ private: void * lib_data; }; -using SharedLibraryHandlerPtr = std::shared_ptr; +using SharedLibraryHandlerPtr = std::shared_ptr; } diff --git a/programs/library-bridge/ExternalDictionaryLibraryHandlerFactory.cpp b/programs/library-bridge/ExternalDictionaryLibraryHandlerFactory.cpp new file mode 100644 index 00000000000..ffa5ff6f493 --- /dev/null +++ b/programs/library-bridge/ExternalDictionaryLibraryHandlerFactory.cpp @@ -0,0 +1,62 @@ +#include "ExternalDictionaryLibraryHandlerFactory.h" + + +namespace DB +{ + +SharedLibraryHandlerPtr ExternalDictionaryLibraryHandlerFactory::get(const std::string & dictionary_id) +{ + std::lock_guard lock(mutex); + auto library_handler = library_handlers.find(dictionary_id); + + if (library_handler != library_handlers.end()) + return library_handler->second; + + return nullptr; +} + + +void ExternalDictionaryLibraryHandlerFactory::create( + const std::string & dictionary_id, + const std::string & library_path, + const std::vector & library_settings, + const Block & sample_block, + const std::vector & attributes_names) +{ + std::lock_guard lock(mutex); + if (!library_handlers.contains(dictionary_id)) + library_handlers.emplace(std::make_pair(dictionary_id, std::make_shared(library_path, library_settings, sample_block, attributes_names))); + else + LOG_WARNING(&Poco::Logger::get("ExternalDictionaryLibraryHandlerFactory"), "Library handler with dictionary id {} already exists", dictionary_id); +} + + +bool ExternalDictionaryLibraryHandlerFactory::clone(const std::string & from_dictionary_id, const std::string & to_dictionary_id) +{ + std::lock_guard lock(mutex); + auto from_library_handler = library_handlers.find(from_dictionary_id); + + if (from_library_handler == library_handlers.end()) + return false; + + /// extDict_libClone method will be called in copy constructor + library_handlers[to_dictionary_id] = std::make_shared(*from_library_handler->second); + return true; +} + + +bool ExternalDictionaryLibraryHandlerFactory::remove(const std::string & dictionary_id) +{ + std::lock_guard lock(mutex); + /// extDict_libDelete is called in destructor. + return library_handlers.erase(dictionary_id); +} + + +ExternalDictionaryLibraryHandlerFactory & ExternalDictionaryLibraryHandlerFactory::instance() +{ + static ExternalDictionaryLibraryHandlerFactory instance; + return instance; +} + +} diff --git a/programs/library-bridge/SharedLibraryHandlerFactory.h b/programs/library-bridge/ExternalDictionaryLibraryHandlerFactory.h similarity index 77% rename from programs/library-bridge/SharedLibraryHandlerFactory.h rename to programs/library-bridge/ExternalDictionaryLibraryHandlerFactory.h index 1a6dfb01e34..d821270c474 100644 --- a/programs/library-bridge/SharedLibraryHandlerFactory.h +++ b/programs/library-bridge/ExternalDictionaryLibraryHandlerFactory.h @@ -1,6 +1,6 @@ #pragma once -#include "SharedLibraryHandler.h" +#include "ExternalDictionaryLibraryHandler.h" #include #include @@ -11,11 +11,11 @@ namespace DB { /// Each library dictionary source has unique UUID. When clone() method is called, a new UUID is generated. -/// There is a unique mapping from diciotnary UUID to sharedLibraryHandler. -class SharedLibraryHandlerFactory final : private boost::noncopyable +/// There is a unique mapping from dictionary UUID to sharedLibraryHandler. +class ExternalDictionaryLibraryHandlerFactory final : private boost::noncopyable { public: - static SharedLibraryHandlerFactory & instance(); + static ExternalDictionaryLibraryHandlerFactory & instance(); SharedLibraryHandlerPtr get(const std::string & dictionary_id); diff --git a/programs/library-bridge/LibraryUtils.h b/programs/library-bridge/ExternalDictionaryLibraryUtils.h similarity index 67% rename from programs/library-bridge/LibraryUtils.h rename to programs/library-bridge/ExternalDictionaryLibraryUtils.h index b2498df0031..e813efab2a6 100644 --- a/programs/library-bridge/LibraryUtils.h +++ b/programs/library-bridge/ExternalDictionaryLibraryUtils.h @@ -5,7 +5,7 @@ #include #include -#include "LibraryInterface.h" +#include "ExternalDictionaryLibraryAPI.h" namespace DB @@ -22,7 +22,7 @@ public: strings_holder = strings_pass; strings.size = strings_holder.size(); - ptr_holder = std::make_unique(strings.size); + ptr_holder = std::make_unique(strings.size); strings.data = ptr_holder.get(); size_t i = 0; @@ -33,10 +33,10 @@ public: } } - ClickHouseLibrary::CStrings strings; // will pass pointer to lib + ExternalDictionaryLibraryAPI::CStrings strings; // will pass pointer to lib private: - std::unique_ptr ptr_holder = nullptr; + std::unique_ptr ptr_holder = nullptr; Container strings_holder; }; diff --git a/programs/library-bridge/HandlerFactory.cpp b/programs/library-bridge/HandlerFactory.cpp deleted file mode 100644 index 43087082c46..00000000000 --- a/programs/library-bridge/HandlerFactory.cpp +++ /dev/null @@ -1,23 +0,0 @@ -#include "HandlerFactory.h" - -#include -#include -#include "Handlers.h" - - -namespace DB -{ - std::unique_ptr LibraryBridgeHandlerFactory::createRequestHandler(const HTTPServerRequest & request) - { - Poco::URI uri{request.getURI()}; - LOG_DEBUG(log, "Request URI: {}", uri.toString()); - - if (request.getMethod() == Poco::Net::HTTPRequest::HTTP_GET) - return std::make_unique(keep_alive_timeout, getContext()); - - if (request.getMethod() == Poco::Net::HTTPRequest::HTTP_POST) - return std::make_unique(keep_alive_timeout, getContext()); - - return nullptr; - } -} diff --git a/programs/library-bridge/HandlerFactory.h b/programs/library-bridge/HandlerFactory.h deleted file mode 100644 index 381745a0e74..00000000000 --- a/programs/library-bridge/HandlerFactory.h +++ /dev/null @@ -1,37 +0,0 @@ -#pragma once - -#include -#include -#include - - -namespace DB -{ - -class SharedLibraryHandler; -using SharedLibraryHandlerPtr = std::shared_ptr; - -/// Factory for '/ping', '/' handlers. -class LibraryBridgeHandlerFactory : public HTTPRequestHandlerFactory, WithContext -{ -public: - LibraryBridgeHandlerFactory( - const std::string & name_, - size_t keep_alive_timeout_, - ContextPtr context_) - : WithContext(context_) - , log(&Poco::Logger::get(name_)) - , name(name_) - , keep_alive_timeout(keep_alive_timeout_) - { - } - - std::unique_ptr createRequestHandler(const HTTPServerRequest & request) override; - -private: - Poco::Logger * log; - std::string name; - size_t keep_alive_timeout; -}; - -} diff --git a/programs/library-bridge/LibraryBridge.cpp b/programs/library-bridge/LibraryBridge.cpp index 2e5d6041151..8a07ca57104 100644 --- a/programs/library-bridge/LibraryBridge.cpp +++ b/programs/library-bridge/LibraryBridge.cpp @@ -1,6 +1,5 @@ #include "LibraryBridge.h" -#pragma GCC diagnostic ignored "-Wmissing-declarations" int mainEntryClickHouseLibraryBridge(int argc, char ** argv) { DB::LibraryBridge app; @@ -15,3 +14,18 @@ int mainEntryClickHouseLibraryBridge(int argc, char ** argv) return code ? code : 1; } } + +namespace DB +{ + +std::string LibraryBridge::bridgeName() const +{ + return "LibraryBridge"; +} + +LibraryBridge::HandlerFactoryPtr LibraryBridge::getHandlerFactoryPtr(ContextPtr context) const +{ + return std::make_shared("LibraryRequestHandlerFactory", keep_alive_timeout, context); +} + +} diff --git a/programs/library-bridge/LibraryBridge.h b/programs/library-bridge/LibraryBridge.h index 9f0d0e148c1..04860a042a3 100644 --- a/programs/library-bridge/LibraryBridge.h +++ b/programs/library-bridge/LibraryBridge.h @@ -2,7 +2,7 @@ #include #include -#include "HandlerFactory.h" +#include "LibraryBridgeHandlerFactory.h" namespace DB @@ -12,15 +12,8 @@ class LibraryBridge : public IBridge { protected: - std::string bridgeName() const override - { - return "LibraryBridge"; - } - - HandlerFactoryPtr getHandlerFactoryPtr(ContextPtr context) const override - { - return std::make_shared("LibraryRequestHandlerFactory-factory", keep_alive_timeout, context); - } + std::string bridgeName() const override; + HandlerFactoryPtr getHandlerFactoryPtr(ContextPtr context) const override; }; } diff --git a/programs/library-bridge/LibraryBridgeHandlerFactory.cpp b/programs/library-bridge/LibraryBridgeHandlerFactory.cpp new file mode 100644 index 00000000000..e8e671db513 --- /dev/null +++ b/programs/library-bridge/LibraryBridgeHandlerFactory.cpp @@ -0,0 +1,34 @@ +#include "LibraryBridgeHandlerFactory.h" + +#include +#include +#include "LibraryBridgeHandlers.h" + + +namespace DB +{ +LibraryBridgeHandlerFactory::LibraryBridgeHandlerFactory( + const std::string & name_, + size_t keep_alive_timeout_, + ContextPtr context_) + : WithContext(context_) + , log(&Poco::Logger::get(name_)) + , name(name_) + , keep_alive_timeout(keep_alive_timeout_) +{ +} + +std::unique_ptr LibraryBridgeHandlerFactory::createRequestHandler(const HTTPServerRequest & request) +{ + Poco::URI uri{request.getURI()}; + LOG_DEBUG(log, "Request URI: {}", uri.toString()); + + if (request.getMethod() == Poco::Net::HTTPRequest::HTTP_GET) + return std::make_unique(keep_alive_timeout, getContext()); + + if (request.getMethod() == Poco::Net::HTTPRequest::HTTP_POST) + return std::make_unique(keep_alive_timeout, getContext()); + + return nullptr; +} +} diff --git a/programs/library-bridge/LibraryBridgeHandlerFactory.h b/programs/library-bridge/LibraryBridgeHandlerFactory.h new file mode 100644 index 00000000000..7565052c4cb --- /dev/null +++ b/programs/library-bridge/LibraryBridgeHandlerFactory.h @@ -0,0 +1,27 @@ +#pragma once + +#include +#include +#include + + +namespace DB +{ + +class LibraryBridgeHandlerFactory : public HTTPRequestHandlerFactory, WithContext +{ +public: + LibraryBridgeHandlerFactory( + const std::string & name_, + size_t keep_alive_timeout_, + ContextPtr context_); + + std::unique_ptr createRequestHandler(const HTTPServerRequest & request) override; + +private: + Poco::Logger * log; + const std::string name; + const size_t keep_alive_timeout; +}; + +} diff --git a/programs/library-bridge/Handlers.cpp b/programs/library-bridge/LibraryBridgeHandlers.cpp similarity index 80% rename from programs/library-bridge/Handlers.cpp rename to programs/library-bridge/LibraryBridgeHandlers.cpp index 58f9bd0a936..9537251954c 100644 --- a/programs/library-bridge/Handlers.cpp +++ b/programs/library-bridge/LibraryBridgeHandlers.cpp @@ -1,5 +1,5 @@ -#include "Handlers.h" -#include "SharedLibraryHandlerFactory.h" +#include "LibraryBridgeHandlers.h" +#include "ExternalDictionaryLibraryHandlerFactory.h" #include #include @@ -78,8 +78,14 @@ static void writeData(Block data, OutputFormatPtr format) executor.execute(); } +LibraryBridgeRequestHandler::LibraryBridgeRequestHandler(size_t keep_alive_timeout_, ContextPtr context_) + : WithContext(context_) + , log(&Poco::Logger::get("LibraryBridgeRequestHandler")) + , keep_alive_timeout(keep_alive_timeout_) +{ +} -void LibraryRequestHandler::handleRequest(HTTPServerRequest & request, HTTPServerResponse & response) +void LibraryBridgeRequestHandler::handleRequest(HTTPServerRequest & request, HTTPServerResponse & response) { LOG_TRACE(log, "Request URI: {}", request.getURI()); HTMLForm params(getContext()->getSettingsRef(), request); @@ -104,8 +110,8 @@ void LibraryRequestHandler::handleRequest(HTTPServerRequest & request, HTTPServe try { - bool lib_new = (method == "libNew"); - if (method == "libClone") + bool lib_new = (method == "extDict_libNew"); + if (method == "extDict_libClone") { if (!params.has("from_dictionary_id")) { @@ -115,7 +121,7 @@ void LibraryRequestHandler::handleRequest(HTTPServerRequest & request, HTTPServe std::string from_dictionary_id = params.get("from_dictionary_id"); bool cloned = false; - cloned = SharedLibraryHandlerFactory::instance().clone(from_dictionary_id, dictionary_id); + cloned = ExternalDictionaryLibraryHandlerFactory::instance().clone(from_dictionary_id, dictionary_id); if (cloned) { @@ -123,7 +129,7 @@ void LibraryRequestHandler::handleRequest(HTTPServerRequest & request, HTTPServe } else { - LOG_TRACE(log, "Cannot clone from dictionary with id: {}, will call libNew instead", from_dictionary_id); + LOG_TRACE(log, "Cannot clone from dictionary with id: {}, will call extDict_libNew instead", from_dictionary_id); lib_new = true; } } @@ -138,13 +144,14 @@ void LibraryRequestHandler::handleRequest(HTTPServerRequest & request, HTTPServe return; } + std::string library_path = params.get("library_path"); + if (!params.has("library_settings")) { processError(response, "No 'library_settings' in request URL"); return; } - std::string library_path = params.get("library_path"); const auto & settings_string = params.get("library_settings"); LOG_DEBUG(log, "Parsing library settings from binary string"); @@ -197,12 +204,12 @@ void LibraryRequestHandler::handleRequest(HTTPServerRequest & request, HTTPServe LOG_DEBUG(log, "Dictionary sample block with null values: {}", sample_block_with_nulls.dumpStructure()); - SharedLibraryHandlerFactory::instance().create(dictionary_id, library_path, library_settings, sample_block_with_nulls, attributes_names); + ExternalDictionaryLibraryHandlerFactory::instance().create(dictionary_id, library_path, library_settings, sample_block_with_nulls, attributes_names); writeStringBinary("1", out); } - else if (method == "libDelete") + else if (method == "extDict_libDelete") { - auto deleted = SharedLibraryHandlerFactory::instance().remove(dictionary_id); + bool deleted = ExternalDictionaryLibraryHandlerFactory::instance().remove(dictionary_id); /// Do not throw, a warning is ok. if (!deleted) @@ -210,57 +217,57 @@ void LibraryRequestHandler::handleRequest(HTTPServerRequest & request, HTTPServe writeStringBinary("1", out); } - else if (method == "isModified") + else if (method == "extDict_isModified") { - auto library_handler = SharedLibraryHandlerFactory::instance().get(dictionary_id); + auto library_handler = ExternalDictionaryLibraryHandlerFactory::instance().get(dictionary_id); if (!library_handler) throw Exception(ErrorCodes::BAD_REQUEST_PARAMETER, "Not found dictionary with id: {}", dictionary_id); bool res = library_handler->isModified(); writeStringBinary(std::to_string(res), out); } - else if (method == "supportsSelectiveLoad") + else if (method == "extDict_supportsSelectiveLoad") { - auto library_handler = SharedLibraryHandlerFactory::instance().get(dictionary_id); + auto library_handler = ExternalDictionaryLibraryHandlerFactory::instance().get(dictionary_id); if (!library_handler) throw Exception(ErrorCodes::BAD_REQUEST_PARAMETER, "Not found dictionary with id: {}", dictionary_id); bool res = library_handler->supportsSelectiveLoad(); writeStringBinary(std::to_string(res), out); } - else if (method == "loadAll") + else if (method == "extDict_loadAll") { - auto library_handler = SharedLibraryHandlerFactory::instance().get(dictionary_id); + auto library_handler = ExternalDictionaryLibraryHandlerFactory::instance().get(dictionary_id); if (!library_handler) throw Exception(ErrorCodes::BAD_REQUEST_PARAMETER, "Not found dictionary with id: {}", dictionary_id); const auto & sample_block = library_handler->getSampleBlock(); - LOG_DEBUG(log, "Calling loadAll() for dictionary id: {}", dictionary_id); + LOG_DEBUG(log, "Calling extDict_loadAll() for dictionary id: {}", dictionary_id); auto input = library_handler->loadAll(); LOG_DEBUG(log, "Started sending result data for dictionary id: {}", dictionary_id); auto output = FormatFactory::instance().getOutputFormat(FORMAT, out, sample_block, getContext()); writeData(std::move(input), std::move(output)); } - else if (method == "loadIds") + else if (method == "extDict_loadIds") { LOG_DEBUG(log, "Getting diciontary ids for dictionary with id: {}", dictionary_id); String ids_string; std::vector ids = parseIdsFromBinary(request.getStream()); - auto library_handler = SharedLibraryHandlerFactory::instance().get(dictionary_id); + auto library_handler = ExternalDictionaryLibraryHandlerFactory::instance().get(dictionary_id); if (!library_handler) throw Exception(ErrorCodes::BAD_REQUEST_PARAMETER, "Not found dictionary with id: {}", dictionary_id); const auto & sample_block = library_handler->getSampleBlock(); - LOG_DEBUG(log, "Calling loadIds() for dictionary id: {}", dictionary_id); + LOG_DEBUG(log, "Calling extDict_loadIds() for dictionary id: {}", dictionary_id); auto input = library_handler->loadIds(ids); LOG_DEBUG(log, "Started sending result data for dictionary id: {}", dictionary_id); auto output = FormatFactory::instance().getOutputFormat(FORMAT, out, sample_block, getContext()); writeData(std::move(input), std::move(output)); } - else if (method == "loadKeys") + else if (method == "extDict_loadKeys") { if (!params.has("requested_block_sample")) { @@ -289,18 +296,22 @@ void LibraryRequestHandler::handleRequest(HTTPServerRequest & request, HTTPServe Block block; executor.pull(block); - auto library_handler = SharedLibraryHandlerFactory::instance().get(dictionary_id); + auto library_handler = ExternalDictionaryLibraryHandlerFactory::instance().get(dictionary_id); if (!library_handler) throw Exception(ErrorCodes::BAD_REQUEST_PARAMETER, "Not found dictionary with id: {}", dictionary_id); const auto & sample_block = library_handler->getSampleBlock(); - LOG_DEBUG(log, "Calling loadKeys() for dictionary id: {}", dictionary_id); + LOG_DEBUG(log, "Calling extDict_loadKeys() for dictionary id: {}", dictionary_id); auto input = library_handler->loadKeys(block.getColumns()); LOG_DEBUG(log, "Started sending result data for dictionary id: {}", dictionary_id); auto output = FormatFactory::instance().getOutputFormat(FORMAT, out, sample_block, getContext()); writeData(std::move(input), std::move(output)); } + else + { + LOG_WARNING(log, "Unknown library method: '{}'", method); + } } catch (...) { @@ -329,8 +340,14 @@ void LibraryRequestHandler::handleRequest(HTTPServerRequest & request, HTTPServe } } +LibraryBridgeExistsHandler::LibraryBridgeExistsHandler(size_t keep_alive_timeout_, ContextPtr context_) + : WithContext(context_) + , keep_alive_timeout(keep_alive_timeout_) + , log(&Poco::Logger::get("LibraryBridgeExistsHandler")) +{ +} -void LibraryExistsHandler::handleRequest(HTTPServerRequest & request, HTTPServerResponse & response) +void LibraryBridgeExistsHandler::handleRequest(HTTPServerRequest & request, HTTPServerResponse & response) { try { @@ -344,15 +361,12 @@ void LibraryExistsHandler::handleRequest(HTTPServerRequest & request, HTTPServer } std::string dictionary_id = params.get("dictionary_id"); - auto library_handler = SharedLibraryHandlerFactory::instance().get(dictionary_id); - String res; - if (library_handler) - res = "1"; - else - res = "0"; + auto library_handler = ExternalDictionaryLibraryHandlerFactory::instance().get(dictionary_id); + + String res = library_handler ? "1" : "0"; setResponseDefaultHeaders(response, keep_alive_timeout); - LOG_TRACE(log, "Senging ping response: {} (dictionary id: {})", res, dictionary_id); + LOG_TRACE(log, "Sending ping response: {} (dictionary id: {})", res, dictionary_id); response.sendBuffer(res.data(), res.size()); } catch (...) diff --git a/programs/library-bridge/Handlers.h b/programs/library-bridge/LibraryBridgeHandlers.h similarity index 53% rename from programs/library-bridge/Handlers.h rename to programs/library-bridge/LibraryBridgeHandlers.h index 0a342a5d6c7..454bcc46acc 100644 --- a/programs/library-bridge/Handlers.h +++ b/programs/library-bridge/LibraryBridgeHandlers.h @@ -3,7 +3,7 @@ #include #include #include -#include "SharedLibraryHandler.h" +#include "ExternalDictionaryLibraryHandler.h" namespace DB @@ -11,23 +11,16 @@ namespace DB /// Handler for requests to Library Dictionary Source, returns response in RowBinary format. -/// When a library dictionary source is created, it sends libNew request to library bridge (which is started on first +/// When a library dictionary source is created, it sends 'extDict_libNew' request to library bridge (which is started on first /// request to it, if it was not yet started). On this request a new sharedLibrayHandler is added to a -/// sharedLibraryHandlerFactory by a dictionary uuid. With libNew request come: library_path, library_settings, +/// sharedLibraryHandlerFactory by a dictionary uuid. With 'extDict_libNew' request come: library_path, library_settings, /// names of dictionary attributes, sample block to parse block of null values, block of null values. Everything is /// passed in binary format and is urlencoded. When dictionary is cloned, a new handler is created. /// Each handler is unique to dictionary. -class LibraryRequestHandler : public HTTPRequestHandler, WithContext +class LibraryBridgeRequestHandler : public HTTPRequestHandler, WithContext { public: - - LibraryRequestHandler( - size_t keep_alive_timeout_, ContextPtr context_) - : WithContext(context_) - , log(&Poco::Logger::get("LibraryRequestHandler")) - , keep_alive_timeout(keep_alive_timeout_) - { - } + LibraryBridgeRequestHandler(size_t keep_alive_timeout_, ContextPtr context_); void handleRequest(HTTPServerRequest & request, HTTPServerResponse & response) override; @@ -39,22 +32,16 @@ private: }; -class LibraryExistsHandler : public HTTPRequestHandler, WithContext +class LibraryBridgeExistsHandler : public HTTPRequestHandler, WithContext { public: - explicit LibraryExistsHandler(size_t keep_alive_timeout_, ContextPtr context_) - : WithContext(context_) - , keep_alive_timeout(keep_alive_timeout_) - , log(&Poco::Logger::get("LibraryRequestHandler")) - { - } + LibraryBridgeExistsHandler(size_t keep_alive_timeout_, ContextPtr context_); void handleRequest(HTTPServerRequest & request, HTTPServerResponse & response) override; private: const size_t keep_alive_timeout; Poco::Logger * log; - }; } diff --git a/programs/library-bridge/LibraryInterface.h b/programs/library-bridge/LibraryInterface.h deleted file mode 100644 index d23de59bbb1..00000000000 --- a/programs/library-bridge/LibraryInterface.h +++ /dev/null @@ -1,110 +0,0 @@ -#pragma once - -#include -#include - -#define CLICKHOUSE_DICTIONARY_LIBRARY_API 1 - -namespace ClickHouseLibrary -{ -using CString = const char *; -using ColumnName = CString; -using ColumnNames = ColumnName[]; - -struct CStrings -{ - CString * data = nullptr; - uint64_t size = 0; -}; - -struct VectorUInt64 -{ - const uint64_t * data = nullptr; - uint64_t size = 0; -}; - -struct ColumnsUInt64 -{ - VectorUInt64 * data = nullptr; - uint64_t size = 0; -}; - -struct Field -{ - const void * data = nullptr; - uint64_t size = 0; -}; - -struct Row -{ - const Field * data = nullptr; - uint64_t size = 0; -}; - -struct Table -{ - const Row * data = nullptr; - uint64_t size = 0; - uint64_t error_code = 0; // 0 = ok; !0 = error, with message in error_string - const char * error_string = nullptr; -}; - -enum LogLevel -{ - FATAL = 1, - CRITICAL, - ERROR, - WARNING, - NOTICE, - INFORMATION, - DEBUG, - TRACE, -}; - -void log(LogLevel level, CString msg); - -extern std::string_view LIBRARY_CREATE_NEW_FUNC_NAME; -extern std::string_view LIBRARY_CLONE_FUNC_NAME; -extern std::string_view LIBRARY_DELETE_FUNC_NAME; - -extern std::string_view LIBRARY_DATA_NEW_FUNC_NAME; -extern std::string_view LIBRARY_DATA_DELETE_FUNC_NAME; - -extern std::string_view LIBRARY_LOAD_ALL_FUNC_NAME; -extern std::string_view LIBRARY_LOAD_IDS_FUNC_NAME; -extern std::string_view LIBRARY_LOAD_KEYS_FUNC_NAME; - -extern std::string_view LIBRARY_IS_MODIFIED_FUNC_NAME; -extern std::string_view LIBRARY_SUPPORTS_SELECTIVE_LOAD_FUNC_NAME; - -using LibraryContext = void *; - -using LibraryLoggerFunc = void (*)(LogLevel, CString /* message */); - -using LibrarySettings = CStrings *; - -using LibraryNewFunc = LibraryContext (*)(LibrarySettings, LibraryLoggerFunc); -using LibraryCloneFunc = LibraryContext (*)(LibraryContext); -using LibraryDeleteFunc = void (*)(LibraryContext); - -using LibraryData = void *; -using LibraryDataNewFunc = LibraryData (*)(LibraryContext); -using LibraryDataDeleteFunc = void (*)(LibraryContext, LibraryData); - -/// Can be safely casted into const Table * with static_cast -using RawClickHouseLibraryTable = void *; -using RequestedColumnsNames = CStrings *; - -using LibraryLoadAllFunc = RawClickHouseLibraryTable (*)(LibraryData, LibrarySettings, RequestedColumnsNames); - -using RequestedIds = const VectorUInt64 *; -using LibraryLoadIdsFunc = RawClickHouseLibraryTable (*)(LibraryData, LibrarySettings, RequestedColumnsNames, RequestedIds); - -using RequestedKeys = Table *; -/// There are no requested column names for load keys func -using LibraryLoadKeysFunc = RawClickHouseLibraryTable (*)(LibraryData, LibrarySettings, RequestedKeys); - -using LibraryIsModifiedFunc = bool (*)(LibraryContext, LibrarySettings); -using LibrarySupportsSelectiveLoadFunc = bool (*)(LibraryContext, LibrarySettings); - -} diff --git a/programs/library-bridge/SharedLibraryHandler.cpp b/programs/library-bridge/SharedLibraryHandler.cpp deleted file mode 100644 index cfdba2c9eb8..00000000000 --- a/programs/library-bridge/SharedLibraryHandler.cpp +++ /dev/null @@ -1,214 +0,0 @@ -#include "SharedLibraryHandler.h" - -#include -#include -#include -#include - - -namespace DB -{ - -namespace ErrorCodes -{ - extern const int EXTERNAL_LIBRARY_ERROR; - extern const int SIZES_OF_COLUMNS_DOESNT_MATCH; -} - - -SharedLibraryHandler::SharedLibraryHandler( - const std::string & library_path_, - const std::vector & library_settings, - const Block & sample_block_, - const std::vector & attributes_names_) - : library_path(library_path_) - , sample_block(sample_block_) - , attributes_names(attributes_names_) -{ - library = std::make_shared(library_path, RTLD_LAZY); - settings_holder = std::make_shared(CStringsHolder(library_settings)); - - auto lib_new = library->tryGet(ClickHouseLibrary::LIBRARY_CREATE_NEW_FUNC_NAME); - - if (lib_new) - lib_data = lib_new(&settings_holder->strings, ClickHouseLibrary::log); - else - throw Exception("Method libNew failed", ErrorCodes::EXTERNAL_LIBRARY_ERROR); -} - - -SharedLibraryHandler::SharedLibraryHandler(const SharedLibraryHandler & other) - : library_path{other.library_path} - , sample_block{other.sample_block} - , attributes_names{other.attributes_names} - , library{other.library} - , settings_holder{other.settings_holder} -{ - - auto lib_clone = library->tryGet(ClickHouseLibrary::LIBRARY_CLONE_FUNC_NAME); - - if (lib_clone) - { - lib_data = lib_clone(other.lib_data); - } - else - { - auto lib_new = library->tryGet(ClickHouseLibrary::LIBRARY_CREATE_NEW_FUNC_NAME); - - if (lib_new) - lib_data = lib_new(&settings_holder->strings, ClickHouseLibrary::log); - } -} - - -SharedLibraryHandler::~SharedLibraryHandler() -{ - auto lib_delete = library->tryGet(ClickHouseLibrary::LIBRARY_DELETE_FUNC_NAME); - - if (lib_delete) - lib_delete(lib_data); -} - - -bool SharedLibraryHandler::isModified() -{ - auto func_is_modified = library->tryGet(ClickHouseLibrary::LIBRARY_IS_MODIFIED_FUNC_NAME); - - if (func_is_modified) - return func_is_modified(lib_data, &settings_holder->strings); - - return true; -} - - -bool SharedLibraryHandler::supportsSelectiveLoad() -{ - auto func_supports_selective_load = library->tryGet(ClickHouseLibrary::LIBRARY_SUPPORTS_SELECTIVE_LOAD_FUNC_NAME); - - if (func_supports_selective_load) - return func_supports_selective_load(lib_data, &settings_holder->strings); - - return true; -} - - -Block SharedLibraryHandler::loadAll() -{ - auto columns_holder = std::make_unique(attributes_names.size()); - ClickHouseLibrary::CStrings columns{static_cast(columns_holder.get()), attributes_names.size()}; - for (size_t i = 0; i < attributes_names.size(); ++i) - columns.data[i] = attributes_names[i].c_str(); - - auto load_all_func = library->get(ClickHouseLibrary::LIBRARY_LOAD_ALL_FUNC_NAME); - auto data_new_func = library->get(ClickHouseLibrary::LIBRARY_DATA_NEW_FUNC_NAME); - auto data_delete_func = library->get(ClickHouseLibrary::LIBRARY_DATA_DELETE_FUNC_NAME); - - ClickHouseLibrary::LibraryData data_ptr = data_new_func(lib_data); - SCOPE_EXIT(data_delete_func(lib_data, data_ptr)); - - ClickHouseLibrary::RawClickHouseLibraryTable data = load_all_func(data_ptr, &settings_holder->strings, &columns); - return dataToBlock(data); -} - - -Block SharedLibraryHandler::loadIds(const std::vector & ids) -{ - const ClickHouseLibrary::VectorUInt64 ids_data{bit_cast(ids.data()), ids.size()}; - - auto columns_holder = std::make_unique(attributes_names.size()); - ClickHouseLibrary::CStrings columns_pass{static_cast(columns_holder.get()), attributes_names.size()}; - - auto load_ids_func = library->get(ClickHouseLibrary::LIBRARY_LOAD_IDS_FUNC_NAME); - auto data_new_func = library->get(ClickHouseLibrary::LIBRARY_DATA_NEW_FUNC_NAME); - auto data_delete_func = library->get(ClickHouseLibrary::LIBRARY_DATA_DELETE_FUNC_NAME); - - ClickHouseLibrary::LibraryData data_ptr = data_new_func(lib_data); - SCOPE_EXIT(data_delete_func(lib_data, data_ptr)); - - ClickHouseLibrary::RawClickHouseLibraryTable data = load_ids_func(data_ptr, &settings_holder->strings, &columns_pass, &ids_data); - return dataToBlock(data); -} - - -Block SharedLibraryHandler::loadKeys(const Columns & key_columns) -{ - auto holder = std::make_unique(key_columns.size()); - std::vector> column_data_holders; - - for (size_t i = 0; i < key_columns.size(); ++i) - { - auto cell_holder = std::make_unique(key_columns[i]->size()); - - for (size_t j = 0; j < key_columns[i]->size(); ++j) - { - auto data_ref = key_columns[i]->getDataAt(j); - - cell_holder[j] = ClickHouseLibrary::Field{ - .data = static_cast(data_ref.data), - .size = data_ref.size}; - } - - holder[i] = ClickHouseLibrary::Row{ - .data = static_cast(cell_holder.get()), - .size = key_columns[i]->size()}; - - column_data_holders.push_back(std::move(cell_holder)); - } - - ClickHouseLibrary::Table request_cols{ - .data = static_cast(holder.get()), - .size = key_columns.size()}; - - auto load_keys_func = library->get(ClickHouseLibrary::LIBRARY_LOAD_KEYS_FUNC_NAME); - auto data_new_func = library->get(ClickHouseLibrary::LIBRARY_DATA_NEW_FUNC_NAME); - auto data_delete_func = library->get(ClickHouseLibrary::LIBRARY_DATA_DELETE_FUNC_NAME); - - ClickHouseLibrary::LibraryData data_ptr = data_new_func(lib_data); - SCOPE_EXIT(data_delete_func(lib_data, data_ptr)); - - ClickHouseLibrary::RawClickHouseLibraryTable data = load_keys_func(data_ptr, &settings_holder->strings, &request_cols); - return dataToBlock(data); -} - - -Block SharedLibraryHandler::dataToBlock(const ClickHouseLibrary::RawClickHouseLibraryTable data) -{ - if (!data) - throw Exception("LibraryDictionarySource: No data returned", ErrorCodes::EXTERNAL_LIBRARY_ERROR); - - const auto * columns_received = static_cast(data); - if (columns_received->error_code) - throw Exception( - "LibraryDictionarySource: Returned error: " + std::to_string(columns_received->error_code) + " " + (columns_received->error_string ? columns_received->error_string : ""), - ErrorCodes::EXTERNAL_LIBRARY_ERROR); - - MutableColumns columns = sample_block.cloneEmptyColumns(); - - for (size_t col_n = 0; col_n < columns_received->size; ++col_n) - { - if (columns.size() != columns_received->data[col_n].size) - throw Exception( - "LibraryDictionarySource: Returned unexpected number of columns: " + std::to_string(columns_received->data[col_n].size) + ", must be " + std::to_string(columns.size()), - ErrorCodes::SIZES_OF_COLUMNS_DOESNT_MATCH); - - for (size_t row_n = 0; row_n < columns_received->data[col_n].size; ++row_n) - { - const auto & field = columns_received->data[col_n].data[row_n]; - if (!field.data) - { - /// sample_block contains null_value (from config) inside corresponding column - const auto & col = sample_block.getByPosition(row_n); - columns[row_n]->insertFrom(*(col.column), 0); - } - else - { - const auto & size = field.size; - columns[row_n]->insertData(static_cast(field.data), size); - } - } - } - - return sample_block.cloneWithColumns(std::move(columns)); -} - -} diff --git a/programs/library-bridge/SharedLibraryHandlerFactory.cpp b/programs/library-bridge/SharedLibraryHandlerFactory.cpp deleted file mode 100644 index 2abc208e502..00000000000 --- a/programs/library-bridge/SharedLibraryHandlerFactory.cpp +++ /dev/null @@ -1,62 +0,0 @@ -#include "SharedLibraryHandlerFactory.h" - - -namespace DB -{ - -SharedLibraryHandlerPtr SharedLibraryHandlerFactory::get(const std::string & dictionary_id) -{ - std::lock_guard lock(mutex); - auto library_handler = library_handlers.find(dictionary_id); - - if (library_handler != library_handlers.end()) - return library_handler->second; - - return nullptr; -} - - -void SharedLibraryHandlerFactory::create( - const std::string & dictionary_id, - const std::string & library_path, - const std::vector & library_settings, - const Block & sample_block, - const std::vector & attributes_names) -{ - std::lock_guard lock(mutex); - if (!library_handlers.contains(dictionary_id)) - library_handlers.emplace(std::make_pair(dictionary_id, std::make_shared(library_path, library_settings, sample_block, attributes_names))); - else - LOG_WARNING(&Poco::Logger::get("SharedLibraryHandlerFactory"), "Library handler with dictionary id {} already exists", dictionary_id); -} - - -bool SharedLibraryHandlerFactory::clone(const std::string & from_dictionary_id, const std::string & to_dictionary_id) -{ - std::lock_guard lock(mutex); - auto from_library_handler = library_handlers.find(from_dictionary_id); - - if (from_library_handler == library_handlers.end()) - return false; - - /// libClone method will be called in copy constructor - library_handlers[to_dictionary_id] = std::make_shared(*from_library_handler->second); - return true; -} - - -bool SharedLibraryHandlerFactory::remove(const std::string & dictionary_id) -{ - std::lock_guard lock(mutex); - /// libDelete is called in destructor. - return library_handlers.erase(dictionary_id); -} - - -SharedLibraryHandlerFactory & SharedLibraryHandlerFactory::instance() -{ - static SharedLibraryHandlerFactory ret; - return ret; -} - -} diff --git a/programs/library-bridge/library-bridge.cpp b/programs/library-bridge/library-bridge.cpp index 5fff2ffe525..9c44fdb4564 100644 --- a/programs/library-bridge/library-bridge.cpp +++ b/programs/library-bridge/library-bridge.cpp @@ -1,3 +1,2 @@ int mainEntryClickHouseLibraryBridge(int argc, char ** argv); int main(int argc_, char ** argv_) { return mainEntryClickHouseLibraryBridge(argc_, argv_); } - diff --git a/programs/odbc-bridge/CMakeLists.txt b/programs/odbc-bridge/CMakeLists.txt index a40df3cfb6e..f649e81c50a 100644 --- a/programs/odbc-bridge/CMakeLists.txt +++ b/programs/odbc-bridge/CMakeLists.txt @@ -2,17 +2,17 @@ include(${ClickHouse_SOURCE_DIR}/cmake/split_debug_symbols.cmake) set (CLICKHOUSE_ODBC_BRIDGE_SOURCES ColumnInfoHandler.cpp - getIdentifierQuote.cpp - HandlerFactory.cpp IdentifierQuoteHandler.cpp MainHandler.cpp ODBCBlockInputStream.cpp ODBCBlockOutputStream.cpp ODBCBridge.cpp + ODBCHandlerFactory.cpp PingHandler.cpp SchemaAllowedHandler.cpp - validateODBCConnectionString.cpp + getIdentifierQuote.cpp odbc-bridge.cpp + validateODBCConnectionString.cpp ) if (OS_LINUX) diff --git a/programs/odbc-bridge/ODBCBridge.cpp b/programs/odbc-bridge/ODBCBridge.cpp index 0deefe46014..e91cc3158df 100644 --- a/programs/odbc-bridge/ODBCBridge.cpp +++ b/programs/odbc-bridge/ODBCBridge.cpp @@ -1,6 +1,5 @@ #include "ODBCBridge.h" -#pragma GCC diagnostic ignored "-Wmissing-declarations" int mainEntryClickHouseODBCBridge(int argc, char ** argv) { DB::ODBCBridge app; @@ -15,3 +14,18 @@ int mainEntryClickHouseODBCBridge(int argc, char ** argv) return code ? code : 1; } } + +namespace DB +{ + +std::string ODBCBridge::bridgeName() const +{ + return "ODBCBridge"; +} + +ODBCBridge::HandlerFactoryPtr ODBCBridge::getHandlerFactoryPtr(ContextPtr context) const +{ + return std::make_shared("ODBCRequestHandlerFactory-factory", keep_alive_timeout, context); +} + +} diff --git a/programs/odbc-bridge/ODBCBridge.h b/programs/odbc-bridge/ODBCBridge.h index b07ae095d7c..5e56dce7c60 100644 --- a/programs/odbc-bridge/ODBCBridge.h +++ b/programs/odbc-bridge/ODBCBridge.h @@ -3,7 +3,7 @@ #include #include #include -#include "HandlerFactory.h" +#include "ODBCHandlerFactory.h" namespace DB @@ -13,14 +13,7 @@ class ODBCBridge : public IBridge { protected: - std::string bridgeName() const override - { - return "ODBCBridge"; - } - - HandlerFactoryPtr getHandlerFactoryPtr(ContextPtr context) const override - { - return std::make_shared("ODBCRequestHandlerFactory-factory", keep_alive_timeout, context); - } + std::string bridgeName() const override; + HandlerFactoryPtr getHandlerFactoryPtr(ContextPtr context) const override; }; } diff --git a/programs/odbc-bridge/HandlerFactory.cpp b/programs/odbc-bridge/ODBCHandlerFactory.cpp similarity index 83% rename from programs/odbc-bridge/HandlerFactory.cpp rename to programs/odbc-bridge/ODBCHandlerFactory.cpp index 4b6dacafe30..2ae533431d3 100644 --- a/programs/odbc-bridge/HandlerFactory.cpp +++ b/programs/odbc-bridge/ODBCHandlerFactory.cpp @@ -1,4 +1,4 @@ -#include "HandlerFactory.h" +#include "ODBCHandlerFactory.h" #include "PingHandler.h" #include "ColumnInfoHandler.h" #include @@ -9,6 +9,14 @@ namespace DB { +ODBCBridgeHandlerFactory::ODBCBridgeHandlerFactory(const std::string & name_, size_t keep_alive_timeout_, ContextPtr context_) + : WithContext(context_) + , log(&Poco::Logger::get(name_)) + , name(name_) + , keep_alive_timeout(keep_alive_timeout_) +{ +} + std::unique_ptr ODBCBridgeHandlerFactory::createRequestHandler(const HTTPServerRequest & request) { Poco::URI uri{request.getURI()}; diff --git a/programs/odbc-bridge/HandlerFactory.h b/programs/odbc-bridge/ODBCHandlerFactory.h similarity index 79% rename from programs/odbc-bridge/HandlerFactory.h rename to programs/odbc-bridge/ODBCHandlerFactory.h index ffbbe3670af..3e3da7c9f24 100644 --- a/programs/odbc-bridge/HandlerFactory.h +++ b/programs/odbc-bridge/ODBCHandlerFactory.h @@ -17,13 +17,7 @@ namespace DB class ODBCBridgeHandlerFactory : public HTTPRequestHandlerFactory, WithContext { public: - ODBCBridgeHandlerFactory(const std::string & name_, size_t keep_alive_timeout_, ContextPtr context_) - : WithContext(context_) - , log(&Poco::Logger::get(name_)) - , name(name_) - , keep_alive_timeout(keep_alive_timeout_) - { - } + ODBCBridgeHandlerFactory(const std::string & name_, size_t keep_alive_timeout_, ContextPtr context_); std::unique_ptr createRequestHandler(const HTTPServerRequest & request) override; diff --git a/src/BridgeHelper/IBridgeHelper.h b/src/BridgeHelper/IBridgeHelper.h index 7c9b16e9014..67ca5bf5cdd 100644 --- a/src/BridgeHelper/IBridgeHelper.h +++ b/src/BridgeHelper/IBridgeHelper.h @@ -41,7 +41,6 @@ protected: /// Check bridge is running. Can also check something else in the mean time. virtual bool bridgeHandShake() = 0; - /// clickhouse-odbc-bridge, clickhouse-library-bridge virtual String serviceAlias() const = 0; virtual String serviceFileName() const = 0; diff --git a/src/BridgeHelper/LibraryBridgeHelper.cpp b/src/BridgeHelper/LibraryBridgeHelper.cpp index 052ef3329b6..e24ce3d2434 100644 --- a/src/BridgeHelper/LibraryBridgeHelper.cpp +++ b/src/BridgeHelper/LibraryBridgeHelper.cpp @@ -38,10 +38,10 @@ LibraryBridgeHelper::LibraryBridgeHelper( , http_timeout(context_->getGlobalContext()->getSettingsRef().http_receive_timeout.value) , library_data(library_data_) , dictionary_id(dictionary_id_) + , bridge_host(config.getString("library_bridge.host", DEFAULT_HOST)) + , bridge_port(config.getUInt("library_bridge.port", DEFAULT_PORT)) , http_timeouts(ConnectionTimeouts::getHTTPTimeouts(context_)) { - bridge_port = config.getUInt("library_bridge.port", DEFAULT_PORT); - bridge_host = config.getString("library_bridge.host", DEFAULT_HOST); } @@ -91,12 +91,12 @@ bool LibraryBridgeHelper::bridgeHandShake() * 2. Bridge crashed or restarted for some reason while server did not. **/ if (result.size() != 1) - throw Exception(ErrorCodes::LOGICAL_ERROR, "Unexpected message from library bridge: {}. Check bridge and server have the same version.", result); + throw Exception(ErrorCodes::LOGICAL_ERROR, "Unexpected message from library bridge: {}. Check that bridge and server have the same version.", result); UInt8 dictionary_id_exists; auto parsed = tryParse(dictionary_id_exists, result); if (!parsed || (dictionary_id_exists != 0 && dictionary_id_exists != 1)) - throw Exception(ErrorCodes::LOGICAL_ERROR, "Unexpected message from library bridge: {} ({}). Check bridge and server have the same version.", + throw Exception(ErrorCodes::LOGICAL_ERROR, "Unexpected message from library bridge: {} ({}). Check that bridge and server have the same version.", result, parsed ? toString(dictionary_id_exists) : "failed to parse"); LOG_TRACE(log, "dictionary_id: {}, dictionary_id_exists on bridge side: {}, library confirmed to be initialized on server side: {}", @@ -113,7 +113,7 @@ bool LibraryBridgeHelper::bridgeHandShake() bool reinitialized = false; try { - auto uri = createRequestURI(LIB_NEW_METHOD); + auto uri = createRequestURI(EXT_DICT_LIB_NEW_METHOD); reinitialized = executeRequest(uri, getInitLibraryCallback()); } catch (...) @@ -153,7 +153,7 @@ ReadWriteBufferFromHTTP::OutStreamCallback LibraryBridgeHelper::getInitLibraryCa bool LibraryBridgeHelper::initLibrary() { startBridgeSync(); - auto uri = createRequestURI(LIB_NEW_METHOD); + auto uri = createRequestURI(EXT_DICT_LIB_NEW_METHOD); library_initialized = executeRequest(uri, getInitLibraryCallback()); return library_initialized; } @@ -162,7 +162,7 @@ bool LibraryBridgeHelper::initLibrary() bool LibraryBridgeHelper::cloneLibrary(const Field & other_dictionary_id) { startBridgeSync(); - auto uri = createRequestURI(LIB_CLONE_METHOD); + auto uri = createRequestURI(EXT_DICT_LIB_CLONE_METHOD); uri.addQueryParameter("from_dictionary_id", toString(other_dictionary_id)); /// We also pass initialization settings in order to create a library handler /// in case from_dictionary_id does not exist in bridge side (possible in case of bridge crash). @@ -177,7 +177,7 @@ bool LibraryBridgeHelper::removeLibrary() /// because in this case after restart it will not have this dictionaty id in memory anyway. if (bridgeHandShake()) { - auto uri = createRequestURI(LIB_DELETE_METHOD); + auto uri = createRequestURI(EXT_DICT_LIB_DELETE_METHOD); return executeRequest(uri); } return true; @@ -187,7 +187,7 @@ bool LibraryBridgeHelper::removeLibrary() bool LibraryBridgeHelper::isModified() { startBridgeSync(); - auto uri = createRequestURI(IS_MODIFIED_METHOD); + auto uri = createRequestURI(EXT_DICT_IS_MODIFIED_METHOD); return executeRequest(uri); } @@ -195,7 +195,7 @@ bool LibraryBridgeHelper::isModified() bool LibraryBridgeHelper::supportsSelectiveLoad() { startBridgeSync(); - auto uri = createRequestURI(SUPPORTS_SELECTIVE_LOAD_METHOD); + auto uri = createRequestURI(EXT_DICT_SUPPORTS_SELECTIVE_LOAD_METHOD); return executeRequest(uri); } @@ -203,7 +203,7 @@ bool LibraryBridgeHelper::supportsSelectiveLoad() QueryPipeline LibraryBridgeHelper::loadAll() { startBridgeSync(); - auto uri = createRequestURI(LOAD_ALL_METHOD); + auto uri = createRequestURI(EXT_DICT_LOAD_ALL_METHOD); return QueryPipeline(loadBase(uri)); } @@ -211,7 +211,7 @@ QueryPipeline LibraryBridgeHelper::loadAll() QueryPipeline LibraryBridgeHelper::loadIds(const std::vector & ids) { startBridgeSync(); - auto uri = createRequestURI(LOAD_IDS_METHOD); + auto uri = createRequestURI(EXT_DICT_LOAD_IDS_METHOD); uri.addQueryParameter("ids_num", toString(ids.size())); /// Not used parameter, but helpful auto ids_string = getDictIdsString(ids); return QueryPipeline(loadBase(uri, [ids_string](std::ostream & os) { os << ids_string; })); @@ -221,7 +221,7 @@ QueryPipeline LibraryBridgeHelper::loadIds(const std::vector & ids) QueryPipeline LibraryBridgeHelper::loadKeys(const Block & requested_block) { startBridgeSync(); - auto uri = createRequestURI(LOAD_KEYS_METHOD); + auto uri = createRequestURI(EXT_DICT_LOAD_KEYS_METHOD); /// Sample block to parse block from callback uri.addQueryParameter("requested_block_sample", requested_block.getNamesAndTypesList().toString()); ReadWriteBufferFromHTTP::OutStreamCallback out_stream_callback = [requested_block, this](std::ostream & os) diff --git a/src/BridgeHelper/LibraryBridgeHelper.h b/src/BridgeHelper/LibraryBridgeHelper.h index 598e473f07b..25cfd123679 100644 --- a/src/BridgeHelper/LibraryBridgeHelper.h +++ b/src/BridgeHelper/LibraryBridgeHelper.h @@ -45,10 +45,6 @@ public: QueryPipeline loadKeys(const Block & requested_block); - QueryPipeline loadBase(const Poco::URI & uri, ReadWriteBufferFromHTTP::OutStreamCallback out_stream_callback = {}); - - bool executeRequest(const Poco::URI & uri, ReadWriteBufferFromHTTP::OutStreamCallback out_stream_callback = {}) const; - LibraryInitData getLibraryData() const { return library_data; } protected: @@ -74,18 +70,22 @@ protected: Poco::URI createBaseURI() const override; + QueryPipeline loadBase(const Poco::URI & uri, ReadWriteBufferFromHTTP::OutStreamCallback out_stream_callback = {}); + + bool executeRequest(const Poco::URI & uri, ReadWriteBufferFromHTTP::OutStreamCallback out_stream_callback = {}) const; + ReadWriteBufferFromHTTP::OutStreamCallback getInitLibraryCallback() const; private: - static constexpr inline auto LIB_NEW_METHOD = "libNew"; - static constexpr inline auto LIB_CLONE_METHOD = "libClone"; - static constexpr inline auto LIB_DELETE_METHOD = "libDelete"; - static constexpr inline auto LOAD_ALL_METHOD = "loadAll"; - static constexpr inline auto LOAD_IDS_METHOD = "loadIds"; - static constexpr inline auto LOAD_KEYS_METHOD = "loadKeys"; - static constexpr inline auto IS_MODIFIED_METHOD = "isModified"; + static constexpr inline auto EXT_DICT_LIB_NEW_METHOD = "extDict_libNew"; + static constexpr inline auto EXT_DICT_LIB_CLONE_METHOD = "extDict_libClone"; + static constexpr inline auto EXT_DICT_LIB_DELETE_METHOD = "extDict_libDelete"; + static constexpr inline auto EXT_DICT_LOAD_ALL_METHOD = "extDict_loadAll"; + static constexpr inline auto EXT_DICT_LOAD_IDS_METHOD = "extDict_loadIds"; + static constexpr inline auto EXT_DICT_LOAD_KEYS_METHOD = "extDict_loadKeys"; + static constexpr inline auto EXT_DICT_IS_MODIFIED_METHOD = "extDict_isModified"; static constexpr inline auto PING = "ping"; - static constexpr inline auto SUPPORTS_SELECTIVE_LOAD_METHOD = "supportsSelectiveLoad"; + static constexpr inline auto EXT_DICT_SUPPORTS_SELECTIVE_LOAD_METHOD = "extDict_supportsSelectiveLoad"; Poco::URI createRequestURI(const String & method) const; diff --git a/src/Dictionaries/LibraryDictionarySource.cpp b/src/Dictionaries/LibraryDictionarySource.cpp index 936bbd72299..dc240cf20ad 100644 --- a/src/Dictionaries/LibraryDictionarySource.cpp +++ b/src/Dictionaries/LibraryDictionarySource.cpp @@ -12,8 +12,6 @@ #include #include -namespace fs = std::filesystem; - namespace DB { @@ -44,6 +42,8 @@ LibraryDictionarySource::LibraryDictionarySource( if (created_from_ddl && !fileOrSymlinkPathStartsWith(path, dictionaries_lib_path)) throw Exception(ErrorCodes::PATH_ACCESS_DENIED, "File path {} is not inside {}", path, dictionaries_lib_path); + namespace fs = std::filesystem; + if (!fs::exists(path)) throw Exception(ErrorCodes::FILE_DOESNT_EXIST, "LibraryDictionarySource: Can't load library {}: file doesn't exist", path);