thevar1able/ClickHouse
1
0
Fork 0
mirror of https://github.com/ClickHouse/ClickHouse.git synced 2024-11-22 07:31:57 +00:00
Code Issues Packages Projects Releases Wiki Activity

refactor parseTypeOfField & fix tests

Browse Source
This commit is contained in:
Mikhail Artemenko 2022-03-27 23:00:28 +03:00
parent 96cfc7f07a
commit 1886d9c2ea
7 changed files with 68 additions and 56 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
docker/test/integration/runner/compose/docker_compose_meili.yml
View File

@ -1,14 +1,14 @@
version: '2.3'
services:
meili1:
image: getmeili/meilisearch
image: getmeili/meilisearch:latest
restart: always
command: ./meilisearch
ports:
- ${MEILI_EXTERNAL_PORT}:${MEILI_INTERNAL_PORT}
meili_secure:
image: getmeili/meilisearch
image: getmeili/meilisearch:latest
restart: always
command: ./meilisearch --master-key="password"
ports:

36
src/Storages/MeiliSearch/MeiliSearchColumnDescriptionFetcher.cpp
View File

@ -1,11 +1,14 @@
#include <memory>
#include <string>
#include <Storages/MeiliSearch/MeiliSearchColumnDescriptionFetcher.h>
#include <base/JSON.h>
#include "DataTypes/DataTypeArray.h"
#include "DataTypes/DataTypeNullable.h"
#include "DataTypes/DataTypeString.h"
#include "DataTypes/DataTypesNumber.h"
#include "DataTypes/Serializations/ISerialization.h"
#include <DataTypes/DataTypeArray.h>
#include <DataTypes/DataTypeNullable.h>
#include <DataTypes/DataTypeString.h>
#include <DataTypes/DataTypesNumber.h>
#include <DataTypes/Serializations/ISerialization.h>
#include <IO/ReadHelpers.h>
#include <base/types.h>
namespace DB
{
@ -25,12 +28,9 @@ void MeiliSearchColumnDescriptionFetcher::addParam(const String & key, const Str
query_params[key] = val;
}
bool isDouble(const std::string & val)
bool checkIfInteger(const String & s)
{
std::istringstream iss(val);
double f;
iss >> std::noskipws >> f;
return iss.eof() && !iss.fail() && static_cast<Int64>(f) != f;
return s.find('.') == String::npos;
}
DataTypePtr parseTypeOfField(JSON ptr)
@ -53,19 +53,15 @@ DataTypePtr parseTypeOfField(JSON ptr)
DataTypePtr res = std::make_shared<DataTypeNullable>(res);
return res;
}
if (isDouble(ptr.toString()))
{
return std::make_shared<DataTypeFloat64>();
}
if (ptr.isNumber())
{
return std::make_shared<DataTypeInt64>();
if (checkIfInteger(ptr.toString()))
{
return std::make_shared<DataTypeInt64>();
}
return std::make_shared<DataTypeFloat64>();
}
if (ptr.isObject())
{
return std::make_shared<DataTypeString>();
}
throw Exception(ErrorCodes::UNSUPPORTED_MEILISEARCH_TYPE, "Can't recognize type of some fields");
return std::make_shared<DataTypeString>();
}
ColumnsDescription MeiliSearchColumnDescriptionFetcher::fetchColumnsDescription() const

9
src/Storages/MeiliSearch/MeiliSearchConnection.cpp
View File

@ -28,7 +28,14 @@ CURLcode MeiliSearchConnection::execQuery(std::string_view url, std::string_view
headers_list = nullptr;
headers_list = curl_slist_append(headers_list, "Content-Type: application/json");
headers_list = curl_slist_append(headers_list, config.key.c_str());
String full_key;
if (!config.key.empty())
{
full_key = config.key_prefix + config.key;
headers_list = curl_slist_append(headers_list, full_key.c_str());
}
handle = curl_easy_init();
curl_easy_setopt(handle, CURLOPT_URL, url.data());

5
src/Storages/MeiliSearch/MeiliSearchConnection.h
View File

@ -10,14 +10,15 @@ namespace DB
{
struct MeiliSearchConfiguration
{
inline const static String key_prefix = "Authorization: Bearer ";
String key;
String index;
String connection_string;
MeiliSearchConfiguration(const String & url_, const String & index_, const String & key_) : index{index_}
MeiliSearchConfiguration(const String & url_, const String & index_, const String & key_) : key{key_}, index{index_}
{
connection_string = url_ + "/indexes/" + index_ + "/";
key = "Authorization: Bearer " + key_;
}
};

35
src/Storages/MeiliSearch/StorageMeiliSearch.cpp
View File

@ -126,48 +126,33 @@ MeiliSearchConfiguration StorageMeiliSearch::getConfiguration(ASTs engine_args,
if (auto named_collection = getExternalDataSourceConfiguration(engine_args, context))
{
auto [common_configuration, storage_specific_args] = named_collection.value();
String url = common_configuration.addresses_expr;
String index = common_configuration.table;
String key = common_configuration.password;
for (const auto & [arg_name, arg_value] : storage_specific_args)
{
if (arg_name == "url")
{
if (!url.empty())
throw Exception(
"2 times given 'url' value, old = " + url + " new = " + arg_value->as<ASTLiteral>()->value.safeGet<String>(),
ErrorCodes::BAD_ARGUMENTS);
url = arg_value->as<ASTLiteral>()->value.safeGet<String>();
}
else if (arg_name == "key" || arg_name == "password")
{
if (!key.empty())
throw Exception(
"2 times given 'key' value, old = " + key + " new = " + arg_value->as<ASTLiteral>()->value.safeGet<String>(),
ErrorCodes::BAD_ARGUMENTS);
key = arg_value->as<ASTLiteral>()->value.safeGet<String>();
}
else
throw Exception("Unexpected key-value argument", ErrorCodes::BAD_ARGUMENTS);
}
if (url.empty() || index.empty())
{
throw Exception("Storage MeiliSearch requires 3 parameters: {url, index, [key]}", ErrorCodes::BAD_ARGUMENTS);
throw Exception(
"Storage MeiliSearch requires 3 parameters: MeiliSearch('url', 'index', 'key'= \"\")", ErrorCodes::BAD_ARGUMENTS);
}
return MeiliSearchConfiguration(url, index, key);
}
else
{
if (engine_args.size() != 3)
if (engine_args.size() < 2 || 3 < engine_args.size())
{
throw Exception(
"Storage MeiliSearch requires 3 parameters: MeiliSearch('url', 'index', 'key')",
"Storage MeiliSearch requires 3 parameters: MeiliSearch('url', 'index', 'key'= \"\")",
ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH);
}
String url = engine_args[0]->as<ASTLiteral &>().value.safeGet<String>();
String index = engine_args[1]->as<ASTLiteral &>().value.safeGet<String>();
String key = engine_args[2]->as<ASTLiteral &>().value.safeGet<String>();
String key;
if (engine_args.size() == 3)
key = engine_args[2]->as<ASTLiteral &>().value.safeGet<String>();
return MeiliSearchConfiguration(url, index, key);
}
}

8
tests/integration/test_storage_meilisearch/configs/named_collection.xml
View File

@ -11,6 +11,14 @@
<database>MeiliSearch</database>
<addresses_expr>meili_secure:7700</addresses_expr>
<table>new_table</table>
<password>password</password>
</named_collection_for_meili_secure>
<named_collection_for_meili_secure_no_password>
<database>MeiliSearch</database>
<addresses_expr>meili_secure:7700</addresses_expr>
<table>new_table</table>
</named_collection_for_meili_secure_no_password>
</named_collections>
</clickhouse>

27
tests/integration/test_storage_meilisearch/test.py
View File

@ -427,25 +427,37 @@ def test_named_collection(started_cluster):
@pytest.mark.parametrize('started_cluster', [False], indirect=['started_cluster'])
def test_named_collection_secure(started_cluster):
client = get_meili_secure_client(started_cluster)
table = client.index("new_table")
client_secure = get_meili_secure_client(started_cluster)
client_free = get_meili_client(started_cluster)
table_secure = client_secure.index("new_table")
table_free = client_free.index("new_table")
data = []
for i in range(0, 100):
data.append({'id': i, 'data': hex(i * i)})
push_data(client, table, data)
push_data(client_secure, table_secure, data)
push_data(client_free, table_free, data)
node = started_cluster.instances['meili']
node.query(
"CREATE TABLE simple_meili_table(id UInt64, data String) ENGINE = MeiliSearch( named_collection_for_meili_secure, key='password')")
"CREATE TABLE simple_meili_table(id UInt64, data String) ENGINE = MeiliSearch( named_collection_for_meili_secure )")
node.query(
"CREATE TABLE wrong_meili_table(id UInt64, data String) ENGINE = MeiliSearch( named_collection_for_meili_secure )")
"CREATE TABLE wrong_meili_table(id UInt64, data String) ENGINE = MeiliSearch( named_collection_for_meili_secure_no_password )")
node.query(
"CREATE TABLE combine_meili_table(id UInt64, data String) ENGINE = MeiliSearch( named_collection_for_meili_secure_no_password, password=\"password\" )")
assert node.query("SELECT COUNT() FROM simple_meili_table") == '100\n'
assert node.query("SELECT sum(id) FROM simple_meili_table") == str(sum(range(0, 100))) + '\n'
assert node.query("SELECT data FROM simple_meili_table WHERE id = 42") == hex(42 * 42) + '\n'
assert node.query("SELECT COUNT() FROM combine_meili_table") == '100\n'
assert node.query("SELECT sum(id) FROM combine_meili_table") == str(sum(range(0, 100))) + '\n'
assert node.query("SELECT data FROM combine_meili_table WHERE id = 42") == hex(42 * 42) + '\n'
error = node.query_and_get_error("SELECT COUNT() FROM wrong_meili_table")
assert("MEILISEARCH_EXCEPTION" in error)
@ -457,7 +469,10 @@ def test_named_collection_secure(started_cluster):
node.query("DROP TABLE simple_meili_table")
node.query("DROP TABLE wrong_meili_table")
table.delete()
node.query("DROP TABLE combine_meili_table")
table_secure.delete()
table_free.delete()
@pytest.mark.parametrize('started_cluster', [False], indirect=['started_cluster'])
def test_table_function(started_cluster):