thevar1able/ClickHouse
1
0
Fork 0
mirror of https://github.com/ClickHouse/ClickHouse.git synced 2024-09-20 08:40:50 +00:00
Code Issues Packages Projects Releases Wiki Activity

Encode passed data

Browse Source
This commit is contained in:
kssenii 2021-03-26 07:05:25 +00:00
parent bc36baef26
commit 97e6040cf1
5 changed files with 101 additions and 17 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

9
src/Bridge/LibraryBridgeHelper.cpp
View File

@ -13,6 +13,7 @@
#include <common/logger_useful.h>
#include <ext/range.h>
#include <Core/Field.h>
#include <Common/escapeForFileName.h>
namespace DB
@ -65,10 +66,10 @@ bool LibraryBridgeHelper::initLibrary(const std::string & library_path, const st
auto uri = createRequestURI(LIB_NEW_METHOD);
return executeRequest(uri, [library_path, library_settings, attributes_names, this](std::ostream & os)
{
os << "library_path=" << library_path << "&";
os << "library_settings=" << library_settings << "&";
os << "attributes_names=" << attributes_names << "&";
os << "sample_block=" << sample_block.getNamesAndTypesList().toString();
os << "library_path=" << escapeForFileName(library_path) << "&";
os << "library_settings=" << escapeForFileName(library_settings) << "&";
os << "attributes_names=" << escapeForFileName(attributes_names) << "&";
os << "sample_block=" << escapeForFileName(sample_block.getNamesAndTypesList().toString());
});
}

35
tests/integration/test_library_bridge/configs/dict_lib.cpp
View File

@ -7,8 +7,6 @@
#include <sstream>
#include <vector>
#define CLICKHOUSE_DICTIONARY_LIBRARY_API 1
namespace ClickHouseLibrary
{
using CString = const char *;
@ -81,7 +79,6 @@ void log(LogLevel level, CString msg);
struct LibHolder
{
std::function<void(ClickHouseLibrary::LogLevel, ClickHouseLibrary::CString)> log;
//Some your data, maybe service connection
};
@ -95,7 +92,8 @@ struct DataHolder
};
void MakeColumnsFromVector(DataHolder * ptr)
template <typename T>
void MakeColumnsFromVector(T * ptr)
{
for (const auto & row : ptr->dataHolder)
{
@ -159,7 +157,7 @@ void * ClickHouseDictionary_v3_loadIds(void * data_ptr,
LOG(ptr->lib->log, "ids passed: " << ids->size);
for (size_t i = 0; i < ids->size; ++i)
{
LOG(ptr->lib->log, "id " << i << " :" << ids->data[i] << " replying.");
LOG(ptr->lib->log, "id " << i << " :" << ids->data[i] << " generating.");
ptr->dataHolder.emplace_back(std::vector<uint64_t>{ids->data[i], ids->data[i] + 100, ids->data[i] + 200, ids->data[i] + 300});
}
}
@ -178,17 +176,34 @@ void * ClickHouseDictionary_v3_loadAll(void * data_ptr, ClickHouseLibrary::CStri
if (!ptr)
return nullptr;
size_t num_rows = 0;
size_t num_rows = 0, num_cols = 0;
std::string test_type;
std::vector<std::string> settings_values;
if (settings)
{
LOG(ptr->lib->log, "settings passed: " << settings->size);
LOG(ptr->lib->log, "settings size: " << settings->size);
for (size_t i = 0; i < settings->size; ++i)
{
num_rows = std::atoi(settings->data[i]);
std::string setting_name = settings->data[i];
std::string setting_value = settings->data[++i];
LOG(ptr->lib->log, "setting " + std::to_string(i) + " name " + setting_name + " value " + setting_value);
if (setting_name == "num_rows")
num_rows = std::atoi(setting_value.data());
else if (setting_name == "num_cols")
num_cols = std::atoi(setting_value.data());
else if (setting_name == "test_type")
test_type = setting_value;
else
{
LOG(ptr->lib->log, "Adding setting " + setting_name);
settings_values.push_back(setting_value);
}
}
}
if (!num_rows)
if (test_type == "test_simple")
{
for (size_t i = 0; i < 10; ++i)
{
@ -196,7 +211,7 @@ void * ClickHouseDictionary_v3_loadAll(void * data_ptr, ClickHouseLibrary::CStri
ptr->dataHolder.emplace_back(std::vector<uint64_t>{i, i + 10, i + 20, i + 30});
}
}
else
else if (test_type == "test_many_rows" && num_rows)
{
for (size_t i = 0; i < num_rows; ++i)
{

47
tests/integration/test_library_bridge/configs/dictionaries/dict1.xml Normal file
View File

@ -0,0 +1,47 @@
<?xml version="1.0"?>
<yandex>
<dictionary>
<name>dict1</name>
<source>
<library>
<path>/etc/clickhouse-server/config.d/dictionaries_lib/dict_lib.so</path>
<settings>
<test_type>test_simple</test_type>
<key>nice key</key>
<value>interesting, nice value</value>
<table>//home/interesting-path/to-/interesting_data</table>
<pass>11</pass>
<user>user-u -user</user>
</settings>
</library>
</source>
<layout>
<hashed/>
</layout>
<lifetime>
<max>1</max>
<min>1</min>
</lifetime>
<structure>
<id>
<name>key</name>
<type>UInt64</type>
</id>
<attribute>
<name>value1</name>
<null_value></null_value>
<type>UInt64</type>
</attribute>
<attribute>
<name>value2</name>
<null_value></null_value>
<type>UInt64</type>
</attribute>
<attribute>
<name>value3</name>
<null_value></null_value>
<type>UInt64</type>
</attribute>
</structure>
</dictionary>
</yandex>

4
tests/integration/test_library_bridge/configs/enable_dict.xml Normal file
View File

@ -0,0 +1,4 @@
<?xml version="1.0"?>
<yandex>
<dictionaries_config>/etc/clickhouse-server/config.d/dict1.xml</dictionaries_config>
</yandex>

23
tests/integration/test_library_bridge/test.py
View File

@ -7,7 +7,12 @@ from helpers.cluster import ClickHouseCluster, run_and_check
cluster = ClickHouseCluster(__file__)
instance = cluster.add_instance('instance', main_configs=['configs/config.d/config.xml', 'configs/log_conf.xml'])
instance = cluster.add_instance('instance',
main_configs=[
'configs/enable_dict.xml',
'configs/config.d/config.xml',
'configs/dictionaries/dict1.xml',
'configs/log_conf.xml'])
@pytest.fixture(scope="module")
def ch_cluster():
@ -41,7 +46,9 @@ def test_load_all(ch_cluster):
instance.query('''
CREATE DICTIONARY lib_dict (key UInt64, value1 UInt64, value2 UInt64, value3 UInt64)
PRIMARY KEY key
SOURCE(library(PATH '/etc/clickhouse-server/config.d/dictionaries_lib/dict_lib.so'))
SOURCE(library(
PATH '/etc/clickhouse-server/config.d/dictionaries_lib/dict_lib.so'
SETTINGS (test_type test_simple)))
LAYOUT(HASHED())
LIFETIME (MIN 0 MAX 10)
''')
@ -59,9 +66,19 @@ def test_load_all(ch_cluster):
"8\t18\t28\t38\n" +
"9\t19\t29\t39\n"
)
instance.query('SYSTEM RELOAD DICTIONARY dict1')
instance.query('DROP DICTIONARY lib_dict')
assert(result == expected)
instance.query("""
CREATE TABLE IF NOT EXISTS `dict1_table` (
key UInt64, value1 UInt64, value2 UInt64, value3 UInt64
) ENGINE = Dictionary(dict1)
""")
result = instance.query('SELECT * FROM dict1_table ORDER BY key')
assert(result == expected)
def test_load_ids(ch_cluster):
instance.query('''
@ -107,7 +124,7 @@ def test_load_all_many_rows(ch_cluster):
PRIMARY KEY key
SOURCE(library(
PATH '/etc/clickhouse-server/config.d/dictionaries_lib/dict_lib.so'
SETTINGS (num_rows {})))
SETTINGS (num_rows {} test_type test_many_rows)))
LAYOUT(HASHED())
LIFETIME (MIN 0 MAX 10)
'''.format(num))