Dictionary Library source: Persistent per-dictionary custom data store (#1917)

* LibraryDictionarySource: Support numeric types, initial support for strings

* Dictionary Library source with string types

* clang-format

* Dictionary Library source: Persistent per-dictionary custom data store

* Fix unused

* Allocate -> New

* Update examples

* fix

dbms/src/Dictionaries/LibraryDictionarySource.cpp

@@ -61,7 +61,7 @@ namespace
             if (bracket_pos != std::string::npos && bracket_pos > 0)
                 key_name = key.substr(0, bracket_pos);
             strings.emplace_back(key_name);
-            strings.emplace_back(config.getString(config_root + '.' + key));
+            strings.emplace_back(config.getString(config_root + "." + key));
         }
         return CStringsHolder(strings);
     }
@@ -118,13 +118,14 @@ LibraryDictionarySource::LibraryDictionarySource(const DictionaryStructure & dic
     , context(context)
 {
     if (!Poco::File(path).exists())
-    {
         throw Exception("LibraryDictionarySource: Can't load lib " + toString() + ": " + Poco::File(path).path() + " - File doesn't exist",
             ErrorCodes::FILE_DOESNT_EXIST);
-    }
     description.init(sample_block);
     library = std::make_shared<SharedLibrary>(path);
     settings = std::make_shared<CStringsHolder>(getLibSettings(config, config_prefix + lib_config_settings));
+    auto fptr = library->tryGet<void * (*)()>("ClickHouseDictionary_v2_libNew");
+    if (fptr)
+        lib_data = fptr();
 }
 
 LibraryDictionarySource::LibraryDictionarySource(const LibraryDictionarySource & other)
@@ -137,9 +138,17 @@ LibraryDictionarySource::LibraryDictionarySource(const LibraryDictionarySource &
     , library{other.library}
     , description{other.description}
     , settings{other.settings}
+    , lib_data{other.lib_data}
 {
 }
 
+LibraryDictionarySource::~LibraryDictionarySource()
+{
+    auto fptr = library->tryGet<void * (*)(void *)>("ClickHouseDictionary_v2_libDelete");
+    if (fptr)
+        fptr(lib_data);
+}
+
 BlockInputStreamPtr LibraryDictionarySource::loadAll()
 {
     LOG_TRACE(log, "loadAll " + toString());
@@ -155,13 +164,13 @@ BlockInputStreamPtr LibraryDictionarySource::loadAll()
     }
     void * data_ptr = nullptr;
 
-    /// Get function pointer before dataAllocate call because library->get may throw.
+    /// Get function pointer before dataNew call because library->get may throw.
     auto fptr
         = library->get<void * (*)(decltype(data_ptr), decltype(&settings->strings), decltype(&columns))>("ClickHouseDictionary_v2_loadAll");
-    data_ptr = library->get<void * (*)()>("ClickHouseDictionary_v2_dataAllocate")();
+    data_ptr = library->get<void * (*)(void *)>("ClickHouseDictionary_v2_dataNew")(lib_data);
     auto data = fptr(data_ptr, &settings->strings, &columns);
     auto block = dataToBlock(description.sample_block, data);
-    library->get<void (*)(void *)>("ClickHouseDictionary_v2_dataDelete")(data_ptr);
+    library->get<void (*)(void *, void *)>("ClickHouseDictionary_v2_dataDelete")(lib_data, data_ptr);
     return std::make_shared<OneBlockInputStream>(block);
 }
 
@@ -181,13 +190,13 @@ BlockInputStreamPtr LibraryDictionarySource::loadIds(const std::vector<UInt64> &
     }
     void * data_ptr = nullptr;
 
-    /// Get function pointer before dataAllocate call because library->get may throw.
+    /// Get function pointer before dataNew call because library->get may throw.
     auto fptr = library->get<void * (*)(decltype(data_ptr), decltype(&settings->strings), decltype(&columns_pass), decltype(&ids_data))>(
         "ClickHouseDictionary_v2_loadIds");
-    data_ptr = library->get<void * (*)()>("ClickHouseDictionary_v2_dataAllocate")();
+    data_ptr = library->get<void * (*)(void *)>("ClickHouseDictionary_v2_dataNew")(lib_data);
     auto data = fptr(data_ptr, &settings->strings, &columns_pass, &ids_data);
     auto block = dataToBlock(description.sample_block, data);
-    library->get<void (*)(void * data_ptr)>("ClickHouseDictionary_v2_dataDelete")(data_ptr);
+    library->get<void (*)(void *, void *)>("ClickHouseDictionary_v2_dataDelete")(lib_data, data_ptr);
     return std::make_shared<OneBlockInputStream>(block);
 }
 
@@ -208,30 +217,31 @@ BlockInputStreamPtr LibraryDictionarySource::loadKeys(const Columns & key_column
         ext::bit_cast<decltype(ClickHouseLibrary::VectorUInt64::data)>(requested_rows.data()), requested_rows.size()};
     void * data_ptr = nullptr;
 
-    /// Get function pointer before dataAllocate call because library->get may throw.
+    /// Get function pointer before dataNew call because library->get may throw.
     auto fptr
         = library->get<void * (*)(decltype(data_ptr), decltype(&settings->strings), decltype(&columns_pass), decltype(&requested_rows_c))>(
             "ClickHouseDictionary_v2_loadKeys");
-    data_ptr = library->get<void * (*)()>("ClickHouseDictionary_v2_dataAllocate")();
+    data_ptr = library->get<void * (*)(void *)>("ClickHouseDictionary_v2_dataNew")(lib_data);
     auto data = fptr(data_ptr, &settings->strings, &columns_pass, &requested_rows_c);
     auto block = dataToBlock(description.sample_block, data);
-    library->get<void (*)(void * data_ptr)>("ClickHouseDictionary_v2_dataDelete")(data_ptr);
+    library->get<void (*)(void *, void *)>("ClickHouseDictionary_v2_dataDelete")(lib_data, data_ptr);
     return std::make_shared<OneBlockInputStream>(block);
 }
 
 bool LibraryDictionarySource::isModified() const
 {
-    auto fptr = library->tryGet<void * (*)(decltype(&settings->strings))>("ClickHouseDictionary_v2_isModified");
+    auto fptr = library->tryGet<void * (*)(decltype(lib_data), decltype(&settings->strings))>("ClickHouseDictionary_v2_isModified");
     if (fptr)
-        return fptr(&settings->strings);
+        return fptr(lib_data, &settings->strings);
     return true;
 }
 
 bool LibraryDictionarySource::supportsSelectiveLoad() const
 {
-    auto fptr = library->tryGet<void * (*)(decltype(&settings->strings))>("ClickHouseDictionary_v2_supportsSelectiveLoad");
+    auto fptr
+        = library->tryGet<void * (*)(decltype(lib_data), decltype(&settings->strings))>("ClickHouseDictionary_v2_supportsSelectiveLoad");
     if (fptr)
-        return fptr(&settings->strings);
+        return fptr(lib_data, &settings->strings);
     return true;
 }
 

dbms/src/Dictionaries/LibraryDictionarySource.h

@@ -24,7 +24,7 @@ class CStringsHolder;
 
 /// Allows loading dictionaries from dynamic libraries (.so)
 /// Experimental version
-/// Now supports only uint64 types
+/// Example: dbms/tests/external_dictionaries/dictionary_library/dictionary_library.cpp
 class LibraryDictionarySource final : public IDictionarySource
 {
 public:
@@ -36,6 +36,8 @@ public:
 
     LibraryDictionarySource(const LibraryDictionarySource & other);
 
+    ~LibraryDictionarySource();
+
     BlockInputStreamPtr loadAll() override;
 
     BlockInputStreamPtr loadIds(const std::vector<UInt64> & ids) override;
@@ -63,5 +65,6 @@ private:
     SharedLibraryPtr library;
     ExternalResultDescription description;
     std::shared_ptr<CStringsHolder> settings;
+    void * lib_data = nullptr;
 };
 }

dbms/tests/external_dictionaries/dictionary_library/dictionary_library.cpp

@@ -1,5 +1,7 @@
 /// c++ sample dictionary library
 
+/// proller: TODO: describe
+
 #include <cstdint>
 #include <iostream>
 #include <memory>
@@ -10,13 +12,46 @@
 //#define DUMPS(VAR) #VAR " = " << VAR
 //#define DUMP(VAR) std::cerr << __FILE__ << ":" << __LINE__ << " " << DUMPS(VAR) << "\n";
 
+
+struct LibHolder
+{
+    //Some your data, maybe service connection
+};
+
 struct DataHolder
 {
-    std::vector<std::vector<uint64_t>> vector;
-    std::unique_ptr<ClickHouseLibrary::VectorUInt64[]> columnsHolder;
-    ClickHouseLibrary::ColumnsUInt64 columns;
+    std::vector<std::vector<uint64_t>> dataHolder; // Actual data storage
+    std::vector<std::vector<ClickHouseLibrary::Field>> fieldHolder; // Pointers and sizes of data
+    std::unique_ptr<ClickHouseLibrary::Row[]> rowHolder;
+    ClickHouseLibrary::Table ctable; // Result data prepared for transfer via c-style interface
+    LibHolder * lib = nullptr;
 };
 
+
+void MakeColumnsFromVector(DataHolder * ptr)
+{
+    for (const auto & row : ptr->dataHolder)
+    {
+        std::vector<ClickHouseLibrary::Field> fields;
+        for (const auto & field : row)
+            fields.push_back({&field, sizeof(field)});
+
+        ptr->fieldHolder.push_back(fields);
+    }
+
+    const auto rows_num = ptr->fieldHolder.size();
+    ptr->rowHolder = std::make_unique<ClickHouseLibrary::Row[]>(rows_num);
+    size_t i = 0;
+    for (auto & row : ptr->fieldHolder)
+    {
+        ptr->rowHolder[i].size = row.size();
+        ptr->rowHolder[i].data = row.data();
+        ++i;
+    }
+    ptr->ctable.size = rows_num;
+    ptr->ctable.data = ptr->rowHolder.get();
+}
+
 extern "C" {
 
 void * ClickHouseDictionary_v2_loadIds(void * data_ptr,
@@ -29,6 +64,9 @@ void * ClickHouseDictionary_v2_loadIds(void * data_ptr,
     if (ids)
         std::cerr << "loadIds lib call ptr=" << data_ptr << " => " << ptr << " size=" << ids->size << "\n";
 
+    if (!ptr)
+        return nullptr;
+
     if (settings)
     {
         std::cerr << "settings passed: " << settings->size << "\n";
@@ -42,42 +80,27 @@ void * ClickHouseDictionary_v2_loadIds(void * data_ptr,
         for (size_t i = 0; i < columns->size; ++i)
             std::cerr << "column " << i << " :" << columns->data[i] << "\n";
     }
-    if (ptr)
-    {
-        if (ids)
-        {
-            std::cerr << "ids passed: " << ids->size << "\n";
-            for (size_t i = 0; i < ids->size; ++i)
-            {
-                std::cerr << "id " << i << " :" << ids->data[i] << " replying.\n";
-                ptr->vector.emplace_back(std::vector<uint64_t>{ids->data[i], ids->data[i] + 1, (1 + ids->data[i]) * 10, 65});
-            }
-        }
 
-        ptr->columnsHolder = std::make_unique<ClickHouseLibrary::VectorUInt64[]>(ptr->vector.size());
-        size_t i = 0;
-        for (auto & col : ptr->vector)
+    if (ids)
+    {
+        std::cerr << "ids passed: " << ids->size << "\n";
+        for (size_t i = 0; i < ids->size; ++i)
         {
-            //DUMP(i);
-            //DUMP(col);
-            ptr->columnsHolder[i].size = col.size();
-            ptr->columnsHolder[i].data = col.data();
-            ++i;
+            std::cerr << "id " << i << " :" << ids->data[i] << " replying.\n";
+            ptr->dataHolder.emplace_back(std::vector<uint64_t>{ids->data[i], ids->data[i] + 1, (1 + ids->data[i]) * 10, 65});
         }
-        ptr->columns.size = ptr->vector.size();
-        //DUMP(ptr->columns.size);
-        ptr->columns.data = ptr->columnsHolder.get();
-        //DUMP(ptr->columns.columns);
-        return static_cast<void *>(&ptr->columns);
     }
 
-    return nullptr;
+    MakeColumnsFromVector(ptr);
+    return static_cast<void *>(&ptr->ctable);
 }
 
 void * ClickHouseDictionary_v2_loadAll(void * data_ptr, ClickHouseLibrary::CStrings * settings, ClickHouseLibrary::CStrings * /*columns*/)
 {
     auto ptr = static_cast<DataHolder *>(data_ptr);
     std::cerr << "loadAll lib call ptr=" << data_ptr << " => " << ptr << "\n";
+    if (!ptr)
+        return nullptr;
     if (settings)
     {
         std::cerr << "settings passed: " << settings->size << "\n";
@@ -87,31 +110,15 @@ void * ClickHouseDictionary_v2_loadAll(void * data_ptr, ClickHouseLibrary::CStri
         }
     }
 
-    if (ptr)
+    for (size_t i = 0; i < 7; ++i)
     {
-        for (size_t i = 0; i < 7; ++i)
-        {
-            std::cerr << "id " << i << " :"
-                      << " generating.\n";
-            ptr->vector.emplace_back(std::vector<uint64_t>{i, i + 1, (1 + i) * 10, 65});
-        }
-
-        ptr->columnsHolder = std::make_unique<ClickHouseLibrary::VectorUInt64[]>(ptr->vector.size());
-        size_t i = 0;
-        for (auto & col : ptr->vector)
-        {
-            ptr->columnsHolder[i].size = col.size();
-            ptr->columnsHolder[i].data = col.data();
-            ++i;
-        }
-        ptr->columns.size = ptr->vector.size();
-        //DUMP(ptr->columns.size);
-        ptr->columns.data = ptr->columnsHolder.get();
-        //DUMP(ptr->columns.columns);
-        return static_cast<void *>(&ptr->columns);
+        std::cerr << "id " << i << " :"
+                  << " generating.\n";
+        ptr->dataHolder.emplace_back(std::vector<uint64_t>{i, i + 1, (1 + i) * 10, 65});
     }
-    //return;
-    return nullptr;
+
+    MakeColumnsFromVector(ptr);
+    return static_cast<void *>(&ptr->ctable);
 }
 
 void * ClickHouseDictionary_v2_loadKeys(void * data_ptr,
@@ -146,16 +153,32 @@ void * ClickHouseDictionary_v2_loadKeys(void * data_ptr,
         }
     }
 
+    //MakeColumnsFromVector(ptr);
+
     return nullptr;
 }
 
-void * ClickHouseDictionary_v2_dataAllocate()
+void * ClickHouseDictionary_v2_libNew()
+{
+    auto lib_ptr = new LibHolder;
+    return lib_ptr;
+}
+
+void ClickHouseDictionary_v2_libDelete(void * lib_ptr)
+{
+    auto ptr = static_cast<LibHolder *>(lib_ptr);
+    delete ptr;
+    return;
+}
+
+void * ClickHouseDictionary_v2_dataNew(void * lib_ptr)
 {
     auto data_ptr = new DataHolder;
+    data_ptr->lib = static_cast<decltype(data_ptr->lib)>(lib_ptr);
     return data_ptr;
 }
 
-void ClickHouseDictionary_v2_dataDelete(void * data_ptr)
+void ClickHouseDictionary_v2_dataDelete(void * /*lib_ptr*/, void * data_ptr)
 {
     auto ptr = static_cast<DataHolder *>(data_ptr);
     delete ptr;

dbms/tests/external_dictionaries/dictionary_library/dictionary_library_c.c

@@ -19,39 +19,53 @@ typedef struct
     CString * data;
 } ClickHouseLibCStrings;
 
-void * ClickHouseDictionary_v1_loadIds(void * data_ptr, ClickHouseLibCStrings * settings, ClickHouseLibCStrings * columns, ClickHouseLibVectorUInt64 * ids)
+void * ClickHouseDictionary_v2_loadIds(
+    void * data_ptr, ClickHouseLibCStrings * settings, ClickHouseLibCStrings * columns, ClickHouseLibVectorUInt64 * ids)
 {
     printf("loadIds c lib call ptr=%p size=%" PRIu64 "\n", data_ptr, ids->size);
     return 0;
 }
 
-void * ClickHouseDictionary_v1_loadAll(void * data_ptr, ClickHouseLibCStrings * settings, ClickHouseLibCStrings * columns)
+void * ClickHouseDictionary_v2_loadAll(void * data_ptr, ClickHouseLibCStrings * settings, ClickHouseLibCStrings * columns)
 {
     printf("loadAll c lib call ptr=%p \n", data_ptr);
     return 0;
 }
 
-void * ClickHouseDictionary_v1_loadKeys(void * data_ptr,
-    ClickHouseLibCStrings * settings,
-    ClickHouseLibCStrings * columns,
-    const ClickHouseLibVectorUInt64 * requested_rows)
+void * ClickHouseDictionary_v2_loadKeys(
+    void * data_ptr, ClickHouseLibCStrings * settings, ClickHouseLibCStrings * columns, const ClickHouseLibVectorUInt64 * requested_rows)
 {
     printf("loadKeys c lib call ptr=%p size=%" PRIu64 "\n", data_ptr, requested_rows->size);
     return 0;
 }
 
+void * ClickHouseDictionary_v2_libNew()
+{
+    int size = 101;
+    void * lib_ptr = malloc(size);
+    printf("libNew c lib call lib_ptr=%p \n", lib_ptr);
+    return lib_ptr;
+}
 
-void * ClickHouseDictionary_v1_dataAllocate()
+void ClickHouseDictionary_v2_libDelete(void * lib_ptr)
+{
+    printf("libDelete c lib call lib_ptr=%p \n", lib_ptr);
+    free(lib_ptr);
+    return;
+}
+
+
+void * ClickHouseDictionary_v2_dataNew(void * lib_ptr)
 {
     int size = 100;
     void * data_ptr = malloc(size);
-    printf("dataAllocate c lib call ptr=%p \n", data_ptr);
+    printf("dataNew c lib call lib_ptr=%p data_ptr=%p \n", lib_ptr, data_ptr);
     return data_ptr;
 }
 
-void ClickHouseDictionary_v1_dataDelete(void * data_ptr)
+void ClickHouseDictionary_v2_dataDelete(void * lib_ptr, void * data_ptr)
 {
-    printf("dataDelete c lib call ptr=%p \n", data_ptr);
+    printf("dataDelete c lib call lib_ptr=%p data_ptr=%p \n", lib_ptr, data_ptr);
     free(data_ptr);
     return;
 }