thevar1able/ClickHouse
1
0
Fork 0
mirror of https://github.com/ClickHouse/ClickHouse.git synced 2024-11-10 01:25:21 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #34503 from Avogar/s3-signatures

Browse Source 
Add one more signature for s3 table function
This commit is contained in:
Kruglov Pavel 2022-02-16 12:30:11 +03:00 committed by GitHub
commit 8580d76bdd
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
2 changed files with 41 additions and 6 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

24
src/TableFunctions/TableFunctionS3.cpp
View File

@ -32,6 +32,7 @@ void TableFunctionS3::parseArguments(const ASTPtr & ast_function, ContextPtr con
" - url\n"
" - url, format\n" \
" - url, format, structure\n" \
" - url, access_key_id, secret_access_key\n" \
" - url, format, structure, compression_method\n" \
" - url, access_key_id, secret_access_key, format\n"
" - url, access_key_id, secret_access_key, format, structure\n" \
@ -75,7 +76,6 @@ void TableFunctionS3::parseArguments(const ASTPtr & ast_function, ContextPtr con
{
{1, {{}}},
{2, {{"format", 1}}},
{3, {{"format", 1}, {"structure", 2}}},
{5, {{"access_key_id", 1}, {"secret_access_key", 2}, {"format", 3}, {"structure", 4}}},
{6, {{"access_key_id", 1}, {"secret_access_key", 2}, {"format", 3}, {"structure", 4}, {"compression_method", 5}}}
};
@ -83,14 +83,26 @@ void TableFunctionS3::parseArguments(const ASTPtr & ast_function, ContextPtr con
std::map<String, size_t> args_to_idx;
/// For 4 arguments we support 2 possible variants:
/// s3(source, format, structure, compression_method) and s3(source, access_key_id, access_key_id, format)
/// We can distinguish them by looking at the 4-th argument: check if it's a format name or not.
/// We can distinguish them by looking at the 2-nd argument: check if it's a format name or not.
if (args.size() == 4)
{
auto last_arg = args[3]->as<ASTLiteral &>().value.safeGet<String>();
if (FormatFactory::instance().getAllFormats().contains(last_arg))
args_to_idx = {{"access_key_id", 1}, {"access_key_id", 2}, {"format", 3}};
else
auto second_arg = args[1]->as<ASTLiteral &>().value.safeGet<String>();
if (FormatFactory::instance().getAllFormats().contains(second_arg))
args_to_idx = {{"format", 1}, {"structure", 2}, {"compression_method", 3}};
else
args_to_idx = {{"access_key_id", 1}, {"secret_access_key", 2}, {"format", 3}};
}
/// For 3 arguments we support 2 possible variants:
/// s3(source, format, structure) and s3(source, access_key_id, access_key_id)
/// We can distinguish them by looking at the 2-nd argument: check if it's a format name or not.
else if (args.size() == 3)
{
auto second_arg = args[1]->as<ASTLiteral &>().value.safeGet<String>();
if (FormatFactory::instance().getAllFormats().contains(second_arg))
args_to_idx = {{"format", 1}, {"structure", 2}};
else
args_to_idx = {{"access_key_id", 1}, {"secret_access_key", 2}};
}
else
{

23
tests/integration/test_storage_s3/test.py
View File

@ -994,3 +994,26 @@ def test_format_detection(started_cluster):
assert(int(result) == 1)
def test_signatures(started_cluster):
bucket = started_cluster.minio_bucket
instance = started_cluster.instances["dummy"]
instance.query(f"create table test_signatures (x UInt64) engine=S3(s3_arrow)")
instance.query(f"truncate table test_signatures")
instance.query(f"insert into test_signatures select 1")
result = instance.query(f"select * from s3('http://{started_cluster.minio_host}:{started_cluster.minio_port}/{bucket}/test.arrow')")
assert(int(result) == 1)
result = instance.query(f"select * from s3('http://{started_cluster.minio_host}:{started_cluster.minio_port}/{bucket}/test.arrow', 'Arrow', 'x UInt64')")
assert(int(result) == 1)
result = instance.query(f"select * from s3('http://{started_cluster.minio_host}:{started_cluster.minio_port}/{bucket}/test.arrow', 'minio', 'minio123')")
assert(int(result) == 1)
result = instance.query(f"select * from s3('http://{started_cluster.minio_host}:{started_cluster.minio_port}/{bucket}/test.arrow', 'Arrow', 'x UInt64', 'auto')")
assert(int(result) == 1)
result = instance.query(f"select * from s3('http://{started_cluster.minio_host}:{started_cluster.minio_port}/{bucket}/test.arrow', 'minio', 'minio123', 'Arrow')")
assert(int(result) == 1)