thevar1able/ClickHouse
1
0
Fork 0
mirror of https://github.com/ClickHouse/ClickHouse.git synced 2024-11-24 08:32:02 +00:00
Code Issues Packages Projects Releases Wiki Activity

Backport #69147 to 24.8: Fix propogating structure argument in s3Cluster

Browse Source
This commit is contained in:
robot-clickhouse 2024-10-28 15:07:17 +00:00
parent 4541d4eef6
commit 2f6b49f294
17 changed files with 539 additions and 150 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

46
src/Storages/ObjectStorage/Azure/Configuration.cpp
View File

@ -24,6 +24,7 @@ namespace ErrorCodes
{
extern const int BAD_ARGUMENTS;
extern const int NUMBER_OF_ARGUMENTS_DOESNT_MATCH;
extern const int LOGICAL_ERROR;
}
const std::unordered_set<std::string_view> required_configuration_keys = {
@ -146,12 +147,13 @@ void StorageAzureConfiguration::fromNamedCollection(const NamedCollection & coll
void StorageAzureConfiguration::fromAST(ASTs & engine_args, ContextPtr context, bool with_structure)
{
if (engine_args.size() < 3 || engine_args.size() > (with_structure ? 8 : 7))
if (engine_args.size() < 3 || engine_args.size() > getMaxNumberOfArguments(with_structure))
{
throw Exception(ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH,
"Storage AzureBlobStorage requires 3 to 7 arguments: "
"AzureBlobStorage(connection_string|storage_account_url, container_name, blobpath, "
"[account_name, account_key, format, compression, structure)])");
throw Exception(
ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH,
"Storage AzureBlobStorage requires 3 to {} arguments. All supported signatures:\n{}",
getMaxNumberOfArguments(with_structure),
getSignatures(with_structure));
}
for (auto & engine_arg : engine_args)
@ -211,7 +213,7 @@ void StorageAzureConfiguration::fromAST(ASTs & engine_args, ContextPtr context,
if (with_structure)
{
format = fourth_arg;
compression_method = checkAndGetLiteralArgument<String>(engine_args[4], "compression");
compression_method = checkAn/dGetLiteralArgument<String>(engine_args[4], "compression");
structure = checkAndGetLiteralArgument<String>(engine_args[5], "structure");
}
else
@ -270,26 +272,30 @@ void StorageAzureConfiguration::fromAST(ASTs & engine_args, ContextPtr context,
connection_params = getConnectionParams(connection_url, container_name, account_name, account_key, context);
}
void StorageAzureConfiguration::addStructureAndFormatToArgs(
void StorageAzureConfiguration::addStructureAndFormatToArgsIfNeeded(
ASTs & args, const String & structure_, const String & format_, ContextPtr context)
{
if (tryGetNamedCollectionWithOverrides(args, context))
if (auto collection = tryGetNamedCollectionWithOverrides(args, context))
{
/// In case of named collection, just add key-value pair "structure='...'"
/// at the end of arguments to override existed structure.
ASTs equal_func_args = {std::make_shared<ASTIdentifier>("structure"), std::make_shared<ASTLiteral>(structure_)};
auto equal_func = makeASTFunction("equals", std::move(equal_func_args));
args.push_back(equal_func);
/// In case of named collection, just add key-value pairs "format='...', structure='...'"
/// at the end of arguments to override existed format and structure with "auto" values.
if (collection->getOrDefault<String>("format", "auto") == "auto")
{
ASTs format_equal_func_args = {std::make_shared<ASTIdentifier>("format"), std::make_shared<ASTLiteral>(format_)};
auto format_equal_func = makeASTFunction("equals", std::move(format_equal_func_args));
args.push_back(format_equal_func);
}
if (collection->getOrDefault<String>("structure", "auto") == "auto")
{
ASTs structure_equal_func_args = {std::make_shared<ASTIdentifier>("structure"), std::make_shared<ASTLiteral>(structure_)};
auto structure_equal_func = makeASTFunction("equals", std::move(structure_equal_func_args));
args.push_back(structure_equal_func);
}
}
else
{
if (args.size() < 3 || args.size() > 8)
{
throw Exception(ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH,
"Storage Azure requires 3 to 7 arguments: "
"StorageObjectStorage(connection_string|storage_account_url, container_name, "
"blobpath, [account_name, account_key, format, compression, structure])");
}
if (args.size() < 3 || args.size() > getMaxNumberOfArguments())
throw Exception(ErrorCodes::LOGICAL_ERROR, "Expected 3 to {} arguments in table function azureBlobStorage, got {}", getMaxNumberOfArguments(), args.size());
for (auto & arg : args)
arg = evaluateConstantExpressionOrIdentifierAsLiteral(arg, context);

28
src/Storages/ObjectStorage/Azure/Configuration.h
View File

@ -22,6 +22,29 @@ public:
static constexpr auto type_name = "azure";
static constexpr auto engine_name = "Azure";
/// All possible signatures for Azure engine with structure argument (for example for azureBlobStorage table function).
static constexpr auto max_number_of_arguments_with_structure = 8;
static constexpr auto signatures_with_structure =
" - connection_string, container_name, blobpath\n"
" - connection_string, container_name, blobpath, structure \n"
" - connection_string, container_name, blobpath, format \n"
" - connection_string, container_name, blobpath, format, compression \n"
" - connection_string, container_name, blobpath, format, compression, structure \n"
" - storage_account_url, container_name, blobpath, account_name, account_key\n"
" - storage_account_url, container_name, blobpath, account_name, account_key, structure\n"
" - storage_account_url, container_name, blobpath, account_name, account_key, format\n"
" - storage_account_url, container_name, blobpath, account_name, account_key, format, compression\n"
" - storage_account_url, container_name, blobpath, account_name, account_key, format, compression, structure\n";
/// All possible signatures for Azure engine without structure argument (for example for AzureBlobStorage table engine).
static constexpr auto max_number_of_arguments_without_structure = 7;
static constexpr auto signatures_without_structure =
" - connection_string, container_name, blobpath\n"
" - connection_string, container_name, blobpath, format \n"
" - connection_string, container_name, blobpath, format, compression \n"
" - storage_account_url, container_name, blobpath, account_name, account_key\n"
" - storage_account_url, container_name, blobpath, account_name, account_key, format\n"
" - storage_account_url, container_name, blobpath, account_name, account_key, format, compression\n";
StorageAzureConfiguration() = default;
StorageAzureConfiguration(const StorageAzureConfiguration & other);
@ -29,6 +52,9 @@ public:
std::string getTypeName() const override { return type_name; }
std::string getEngineName() const override { return engine_name; }
std::string getSignatures(bool with_structure = true) const { return with_structure ? signatures_with_structure : signatures_without_structure; }
size_t getMaxNumberOfArguments(bool with_structure = true) const { return with_structure ? max_number_of_arguments_with_structure : max_number_of_arguments_without_structure; }
Path getPath() const override { return blob_path; }
void setPath(const Path & path) override { blob_path = path; }
@ -44,7 +70,7 @@ public:
ObjectStoragePtr createObjectStorage(ContextPtr context, bool is_readonly) override;
void addStructureAndFormatToArgs(
void addStructureAndFormatToArgsIfNeeded(
ASTs & args,
const String & structure_,
const String & format_,

44
src/Storages/ObjectStorage/HDFS/Configuration.cpp
View File

@ -24,6 +24,7 @@ namespace ErrorCodes
{
extern const int BAD_ARGUMENTS;
extern const int NUMBER_OF_ARGUMENTS_DOESNT_MATCH;
extern const int LOGICAL_ERROR;
}
StorageHDFSConfiguration::StorageHDFSConfiguration(const StorageHDFSConfiguration & other)
@ -83,12 +84,13 @@ StorageObjectStorage::QuerySettings StorageHDFSConfiguration::getQuerySettings(c
void StorageHDFSConfiguration::fromAST(ASTs & args, ContextPtr context, bool with_structure)
{
const size_t max_args_num = with_structure ? 4 : 3;
if (args.empty() || args.size() > max_args_num)
{
throw Exception(ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH,
"Expected not more than {} arguments", max_args_num);
}
if (args.empty() || args.size() > getMaxNumberOfArguments(with_structure))
throw Exception(
ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH,
"Storage HDFS requires 1 to {} arguments. All supported signatures:\n{}",
getMaxNumberOfArguments(with_structure),
getSignatures(with_structure));
std::string url_str;
url_str = checkAndGetLiteralArgument<String>(args[0], "url");
@ -158,28 +160,34 @@ void StorageHDFSConfiguration::setURL(const std::string & url_)
LOG_TRACE(getLogger("StorageHDFSConfiguration"), "Using URL: {}, path: {}", url, path);
}
void StorageHDFSConfiguration::addStructureAndFormatToArgs(
void StorageHDFSConfiguration::addStructureAndFormatToArgsIfNeeded(
ASTs & args,
const String & structure_,
const String & format_,
ContextPtr context)
{
if (tryGetNamedCollectionWithOverrides(args, context))
if (auto collection = tryGetNamedCollectionWithOverrides(args, context))
{
/// In case of named collection, just add key-value pair "structure='...'"
/// at the end of arguments to override existed structure.
ASTs equal_func_args = {std::make_shared<ASTIdentifier>("structure"), std::make_shared<ASTLiteral>(structure_)};
auto equal_func = makeASTFunction("equals", std::move(equal_func_args));
args.push_back(equal_func);
/// In case of named collection, just add key-value pairs "format='...', structure='...'"
/// at the end of arguments to override existed format and structure with "auto" values.
if (collection->getOrDefault<String>("format", "auto") == "auto")
{
ASTs format_equal_func_args = {std::make_shared<ASTIdentifier>("format"), std::make_shared<ASTLiteral>(format_)};
auto format_equal_func = makeASTFunction("equals", std::move(format_equal_func_args));
args.push_back(format_equal_func);
}
if (collection->getOrDefault<String>("structure", "auto") == "auto")
{
ASTs structure_equal_func_args = {std::make_shared<ASTIdentifier>("structure"), std::make_shared<ASTLiteral>(structure_)};
auto structure_equal_func = makeASTFunction("equals", std::move(structure_equal_func_args));
args.push_back(structure_equal_func);
}
}
else
{
size_t count = args.size();
if (count == 0 || count > 4)
{
throw Exception(ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH,
"Expected 1 to 4 arguments in table function, got {}", count);
}
if (count == 0 || count > getMaxNumberOfArguments())
throw Exception(ErrorCodes::LOGICAL_ERROR, "Expected 1 to {} arguments in table function hdfs, got {}", getMaxNumberOfArguments(), count);
auto format_literal = std::make_shared<ASTLiteral>(format_);
auto structure_literal = std::make_shared<ASTLiteral>(structure_);

19
src/Storages/ObjectStorage/HDFS/Configuration.h
View File

@ -16,6 +16,20 @@ public:
static constexpr auto type_name = "hdfs";
static constexpr auto engine_name = "HDFS";
/// All possible signatures for HDFS engine with structure argument (for example for hdfs table function).
static constexpr auto max_number_of_arguments_with_structure = 4;
static constexpr auto signatures_with_structure =
" - uri\n"
" - uri, format\n"
" - uri, format, structure\n"
" - uri, format, structure, compression_method\n";
/// All possible signatures for HDFS engine without structure argument (for example for HS table engine).
static constexpr auto max_number_of_arguments_without_structure = 3;
static constexpr auto signatures_without_structure =
" - uri\n"
" - uri, format\n"
" - uri, format, compression_method\n";
StorageHDFSConfiguration() = default;
StorageHDFSConfiguration(const StorageHDFSConfiguration & other);
@ -23,6 +37,9 @@ public:
std::string getTypeName() const override { return type_name; }
std::string getEngineName() const override { return engine_name; }
std::string getSignatures(bool with_structure = true) const { return with_structure ? signatures_with_structure : signatures_without_structure; }
size_t getMaxNumberOfArguments(bool with_structure = true) const { return with_structure ? max_number_of_arguments_with_structure : max_number_of_arguments_without_structure; }
Path getPath() const override { return path; }
void setPath(const Path & path_) override { path = path_; }
@ -39,7 +56,7 @@ public:
ObjectStoragePtr createObjectStorage(ContextPtr context, bool is_readonly) override;
void addStructureAndFormatToArgs(
void addStructureAndFormatToArgsIfNeeded(
ASTs & args,
const String & structure_,
const String & format_,

160
src/Storages/ObjectStorage/S3/Configuration.cpp
View File

@ -170,21 +170,20 @@ void StorageS3Configuration::fromNamedCollection(const NamedCollection & collect
void StorageS3Configuration::fromAST(ASTs & args, ContextPtr context, bool with_structure)
{
/// Supported signatures: S3('url') S3('url', 'format') S3('url', 'format', 'compression') S3('url', NOSIGN) S3('url', NOSIGN, 'format') S3('url', NOSIGN, 'format', 'compression') S3('url', 'aws_access_key_id', 'aws_secret_access_key') S3('url', 'aws_access_key_id', 'aws_secret_access_key', 'session_token') S3('url', 'aws_access_key_id', 'aws_secret_access_key', 'format') S3('url', 'aws_access_key_id', 'aws_secret_access_key', 'session_token', 'format') S3('url', 'aws_access_key_id', 'aws_secret_access_key', 'format', 'compression')
/// S3('url', 'aws_access_key_id', 'aws_secret_access_key', 'session_token', 'format', 'compression')
/// with optional headers() function
size_t count = StorageURL::evalArgsAndCollectHeaders(args, headers_from_ast, context);
if (count == 0 || count > (with_structure ? 7 : 6))
if (count == 0 || count > getMaxNumberOfArguments(with_structure))
throw Exception(ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH,
"Storage S3 requires 1 to 5 arguments: "
"url, [NOSIGN | access_key_id, secret_access_key], name of used format and [compression_method]");
"Storage S3 requires 1 to {} arguments. All supported signatures:\n{}",
getMaxNumberOfArguments(with_structure),
getSignatures(with_structure));
std::unordered_map<std::string_view, size_t> engine_args_to_idx;
bool no_sign_request = false;
/// For 2 arguments we support 2 possible variants:
/// When adding new arguments in the signature don't forget to update addStructureAndFormatToArgsIfNeeded as well.
/// For 2 arguments we support:
/// - s3(source, format)
/// - s3(source, NOSIGN)
/// We can distinguish them by looking at the 2-nd argument: check if it's NOSIGN or not.
@ -196,10 +195,15 @@ void StorageS3Configuration::fromAST(ASTs & args, ContextPtr context, bool with_
else
engine_args_to_idx = {{"format", 1}};
}
/// For 3 arguments we support 2 possible variants:
/// For 3 arguments we support:
/// if with_structure == 0:
/// - s3(source, NOSIGN, format)
/// - s3(source, format, compression_method)
/// - s3(source, access_key_id, secret_access_key)
/// if with_structure == 1:
/// - s3(source, NOSIGN, format)
/// - s3(source, format, structure)
/// - s3(source, access_key_id, secret_access_key)
/// We can distinguish them by looking at the 2-nd argument: check if it's NOSIGN or format name.
else if (count == 3)
{
@ -219,7 +223,7 @@ void StorageS3Configuration::fromAST(ASTs & args, ContextPtr context, bool with_
else
engine_args_to_idx = {{"access_key_id", 1}, {"secret_access_key", 2}};
}
/// For 4 arguments we support 3 possible variants:
/// For 4 arguments we support:
/// if with_structure == 0:
/// - s3(source, access_key_id, secret_access_key, session_token)
/// - s3(source, access_key_id, secret_access_key, format)
@ -229,7 +233,7 @@ void StorageS3Configuration::fromAST(ASTs & args, ContextPtr context, bool with_
/// - s3(source, access_key_id, secret_access_key, format),
/// - s3(source, access_key_id, secret_access_key, session_token)
/// - s3(source, NOSIGN, format, structure)
/// We can distinguish them by looking at the 2-nd argument: check if it's a NOSIGN or not.
/// We can distinguish them by looking at the 2-nd argument: check if it's a NOSIGN, format name of something else.
else if (count == 4)
{
auto second_arg = checkAndGetLiteralArgument<String>(args[1], "access_key_id/NOSIGN");
@ -258,7 +262,7 @@ void StorageS3Configuration::fromAST(ASTs & args, ContextPtr context, bool with_
}
}
}
/// For 5 arguments we support 2 possible variants:
/// For 5 arguments we support:
/// if with_structure == 0:
/// - s3(source, access_key_id, secret_access_key, session_token, format)
/// - s3(source, access_key_id, secret_access_key, format, compression)
@ -302,13 +306,16 @@ void StorageS3Configuration::fromAST(ASTs & args, ContextPtr context, bool with_
}
}
}
/// For 6 arguments we support:
/// if with_structure == 0:
/// - s3(source, access_key_id, secret_access_key, session_token, format, compression_method)
/// if with_structure == 1:
/// - s3(source, access_key_id, secret_access_key, format, structure, compression_method)
/// - s3(source, access_key_id, secret_access_key, session_token, format, structure)
else if (count == 6)
{
if (with_structure)
{
/// - s3(source, access_key_id, secret_access_key, format, structure, compression_method)
/// - s3(source, access_key_id, secret_access_key, session_token, format, structure)
/// We can distinguish them by looking at the 4-th argument: check if it's a format name or not
auto fourth_arg = checkAndGetLiteralArgument<String>(args[3], "format/session_token");
if (fourth_arg == "auto" || FormatFactory::instance().exists(fourth_arg))
{
@ -324,6 +331,7 @@ void StorageS3Configuration::fromAST(ASTs & args, ContextPtr context, bool with_
engine_args_to_idx = {{"access_key_id", 1}, {"secret_access_key", 2}, {"session_token", 3}, {"format", 4}, {"compression_method", 5}};
}
}
/// s3(source, access_key_id, secret_access_key, session_token, format, structure, compression_method)
else if (with_structure && count == 7)
{
engine_args_to_idx = {{"access_key_id", 1}, {"secret_access_key", 2}, {"session_token", 3}, {"format", 4}, {"structure", 5}, {"compression_method", 6}};
@ -365,24 +373,33 @@ void StorageS3Configuration::fromAST(ASTs & args, ContextPtr context, bool with_
keys = {url.key};
}
void StorageS3Configuration::addStructureAndFormatToArgs(
void StorageS3Configuration::addStructureAndFormatToArgsIfNeeded(
ASTs & args, const String & structure_, const String & format_, ContextPtr context)
{
if (tryGetNamedCollectionWithOverrides(args, context))
if (auto collection = tryGetNamedCollectionWithOverrides(args, context))
{
/// In case of named collection, just add key-value pair "structure='...'"
/// at the end of arguments to override existed structure.
ASTs equal_func_args = {std::make_shared<ASTIdentifier>("structure"), std::make_shared<ASTLiteral>(structure_)};
auto equal_func = makeASTFunction("equals", std::move(equal_func_args));
args.push_back(equal_func);
/// In case of named collection, just add key-value pairs "format='...', structure='...'"
/// at the end of arguments to override existed format and structure with "auto" values.
if (collection->getOrDefault<String>("format", "auto") == "auto")
{
ASTs format_equal_func_args = {std::make_shared<ASTIdentifier>("format"), std::make_shared<ASTLiteral>(format_)};
auto format_equal_func = makeASTFunction("equals", std::move(format_equal_func_args));
args.push_back(format_equal_func);
}
if (collection->getOrDefault<String>("structure", "auto") == "auto")
{
ASTs structure_equal_func_args = {std::make_shared<ASTIdentifier>("structure"), std::make_shared<ASTLiteral>(structure_)};
auto structure_equal_func = makeASTFunction("equals", std::move(structure_equal_func_args));
args.push_back(structure_equal_func);
}
}
else
{
HTTPHeaderEntries tmp_headers;
size_t count = StorageURL::evalArgsAndCollectHeaders(args, tmp_headers, context);
if (count == 0 || count > 6)
throw Exception(ErrorCodes::LOGICAL_ERROR, "Expected 1 to 6 arguments in table function, got {}", count);
if (count == 0 || count > getMaxNumberOfArguments())
throw Exception(ErrorCodes::LOGICAL_ERROR, "Expected 1 to {} arguments in table function s3, got {}", getMaxNumberOfArguments(), count);
auto format_literal = std::make_shared<ASTLiteral>(format_);
auto structure_literal = std::make_shared<ASTLiteral>(structure_);
@ -394,14 +411,18 @@ void StorageS3Configuration::addStructureAndFormatToArgs(
args.push_back(std::make_shared<ASTLiteral>("auto"));
args.push_back(structure_literal);
}
/// s3(s3_url, format) or s3(s3_url, NOSIGN)
/// s3(s3_url, format) or
/// s3(s3_url, NOSIGN)
/// We can distinguish them by looking at the 2-nd argument: check if it's NOSIGN or not.
else if (count == 2)
{
auto second_arg = checkAndGetLiteralArgument<String>(args[1], "format/NOSIGN");
/// If there is NOSIGN, add format=auto before structure.
if (boost::iequals(second_arg, "NOSIGN"))
args.push_back(std::make_shared<ASTLiteral>("auto"));
args.push_back(format_literal);
else if (checkAndGetLiteralArgument<String>(args[1], "format") == "auto")
args[1] = format_literal;
args.push_back(structure_literal);
}
/// s3(source, format, structure) or
@ -413,21 +434,27 @@ void StorageS3Configuration::addStructureAndFormatToArgs(
auto second_arg = checkAndGetLiteralArgument<String>(args[1], "format/NOSIGN");
if (boost::iequals(second_arg, "NOSIGN"))
{
if (checkAndGetLiteralArgument<String>(args[2], "format") == "auto")
args[2] = format_literal;
args.push_back(structure_literal);
}
else if (second_arg == "auto" || FormatFactory::instance().exists(second_arg))
{
args[count - 1] = structure_literal;
if (second_arg == "auto")
args[1] = format_literal;
if (checkAndGetLiteralArgument<String>(args[2], "structure") == "auto")
args[2] = structure_literal;
}
else
{
/// Add format=auto before structure argument.
args.push_back(std::make_shared<ASTLiteral>("auto"));
/// Add format and structure arguments.
args.push_back(format_literal);
args.push_back(structure_literal);
}
}
/// s3(source, format, structure, compression_method) or
/// s3(source, access_key_id, secret_access_key, format) or
/// s3(source, access_key_id, secret_access_key, session_token) or
/// s3(source, NOSIGN, format, structure)
/// We can distinguish them by looking at the 2-nd argument: check if it's NOSIGN, format name or neither.
else if (count == 4)
@ -435,36 +462,93 @@ void StorageS3Configuration::addStructureAndFormatToArgs(
auto second_arg = checkAndGetLiteralArgument<String>(args[1], "format/NOSIGN");
if (boost::iequals(second_arg, "NOSIGN"))
{
args[count - 1] = structure_literal;
if (checkAndGetLiteralArgument<String>(args[2], "format") == "auto")
args[2] = format_literal;
if (checkAndGetLiteralArgument<String>(args[3], "structure") == "auto")
args[3] = structure_literal;
}
else if (second_arg == "auto" || FormatFactory::instance().exists(second_arg))
{
args[count - 2] = structure_literal;
if (second_arg == "auto")
args[1] = format_literal;
if (checkAndGetLiteralArgument<String>(args[2], "structure") == "auto")
args[2] = structure_literal;
}
else
{
args.push_back(structure_literal);
auto fourth_arg = checkAndGetLiteralArgument<String>(args[3], "format/session_token");
if (fourth_arg == "auto" || FormatFactory::instance().exists(fourth_arg))
{
if (checkAndGetLiteralArgument<String>(args[3], "format") == "auto")
args[3] = format_literal;
args.push_back(structure_literal);
}
else
{
args.push_back(format_literal);
args.push_back(structure_literal);
}
}
}
/// s3(source, access_key_id, secret_access_key, format, structure) or
/// s3(source, access_key_id, secret_access_key, session_token, format) or
/// s3(source, NOSIGN, format, structure, compression_method)
/// We can distinguish them by looking at the 2-nd argument: check if it's a NOSIGN keyword name or not.
else if (count == 5)
{
auto sedond_arg = checkAndGetLiteralArgument<String>(args[1], "format/NOSIGN");
if (boost::iequals(sedond_arg, "NOSIGN"))
auto second_arg = checkAndGetLiteralArgument<String>(args[1], "format/NOSIGN");
if (boost::iequals(second_arg, "NOSIGN"))
{
args[count - 2] = structure_literal;
if (checkAndGetLiteralArgument<String>(args[2], "format") == "auto")
args[2] = format_literal;
if (checkAndGetLiteralArgument<String>(args[2], "structure") == "auto")
args[3] = structure_literal;
}
else
{
args[count - 1] = structure_literal;
auto fourth_arg = checkAndGetLiteralArgument<String>(args[3], "format/session_token");
if (fourth_arg == "auto" || FormatFactory::instance().exists(fourth_arg))
{
if (checkAndGetLiteralArgument<String>(args[3], "format") == "auto")
args[3] = format_literal;
if (checkAndGetLiteralArgument<String>(args[4], "structure") == "auto")
args[4] = structure_literal;
}
else
{
if (checkAndGetLiteralArgument<String>(args[4], "format") == "auto")
args[4] = format_literal;
args.push_back(structure_literal);
}
}
}
/// s3(source, access_key_id, secret_access_key, format, structure, compression)
/// s3(source, access_key_id, secret_access_key, format, structure, compression) or
/// s3(source, access_key_id, secret_access_key, session_token, format, structure)
else if (count == 6)
{
args[count - 2] = structure_literal;
auto fourth_arg = checkAndGetLiteralArgument<String>(args[3], "format/session_token");
if (fourth_arg == "auto" || FormatFactory::instance().exists(fourth_arg))
{
if (checkAndGetLiteralArgument<String>(args[3], "format") == "auto")
args[3] = format_literal;
if (checkAndGetLiteralArgument<String>(args[4], "structure") == "auto")
args[4] = structure_literal;
}
else
{
if (checkAndGetLiteralArgument<String>(args[4], "format") == "auto")
args[4] = format_literal;
if (checkAndGetLiteralArgument<String>(args[5], "format") == "auto")
args[5] = structure_literal;
}
}
/// s3(source, access_key_id, secret_access_key, session_token, format, structure, compression_method)
else if (count == 7)
{
if (checkAndGetLiteralArgument<String>(args[4], "format") == "auto")
args[4] = format_literal;
if (checkAndGetLiteralArgument<String>(args[5], "format") == "auto")
args[5] = structure_literal;
}
}
}

42
src/Storages/ObjectStorage/S3/Configuration.h
View File

@ -16,6 +16,43 @@ public:
static constexpr auto type_name = "s3";
static constexpr auto namespace_name = "bucket";
/// All possible signatures for S3 storage with structure argument (for example for s3 table function).
static constexpr auto max_number_of_arguments_with_structure = 7;
static constexpr auto signatures_with_structure =
" - url\n"
" - url, NOSIGN\n"
" - url, format\n"
" - url, NOSIGN, format\n"
" - url, format, structure\n"
" - url, NOSIGN, format, structure\n"
" - url, format, structure, compression_method\n"
" - url, NOSIGN, format, structure, compression_method\n"
" - url, access_key_id, secret_access_key\n"
" - url, access_key_id, secret_access_key, session_token\n"
" - url, access_key_id, secret_access_key, format\n"
" - url, access_key_id, secret_access_key, session_token, format\n"
" - url, access_key_id, secret_access_key, format, structure\n"
" - url, access_key_id, secret_access_key, session_token, format, structure\n"
" - url, access_key_id, secret_access_key, format, structure, compression_method\n"
" - url, access_key_id, secret_access_key, session_token, format, structure, compression_method\n"
"All signatures supports optional headers (specified as `headers('name'='value', 'name2'='value2')`)";
/// All possible signatures for S3 storage without structure argument (for example for S3 table engine).
static constexpr auto max_number_of_arguments_without_structure = 6;
static constexpr auto signatures_without_structure =
" - url\n"
" - url, NOSIGN\n"
" - url, format\n"
" - url, NOSIGN, format\n"
" - url, format, compression_method\n"
" - url, NOSIGN, format, compression_method\n"
" - url, access_key_id, secret_access_key\n"
" - url, access_key_id, secret_access_key, session_token\n"
" - url, access_key_id, secret_access_key, format\n"
" - url, access_key_id, secret_access_key, session_token, format\n"
" - url, access_key_id, secret_access_key, format, compression_method\n"
" - url, access_key_id, secret_access_key, session_token, format, compression_method\n"
"All signatures supports optional headers (specified as `headers('name'='value', 'name2'='value2')`)";
StorageS3Configuration() = default;
StorageS3Configuration(const StorageS3Configuration & other);
@ -24,6 +61,9 @@ public:
std::string getEngineName() const override { return url.storage_name; }
std::string getNamespaceType() const override { return namespace_name; }
std::string getSignatures(bool with_structure = true) const { return with_structure ? signatures_with_structure : signatures_without_structure; }
size_t getMaxNumberOfArguments(bool with_structure = true) const { return with_structure ? max_number_of_arguments_with_structure : max_number_of_arguments_without_structure; }
Path getPath() const override { return url.key; }
void setPath(const Path & path) override { url.key = path; }
@ -44,7 +84,7 @@ public:
ObjectStoragePtr createObjectStorage(ContextPtr context, bool is_readonly) override;
void addStructureAndFormatToArgs(
void addStructureAndFormatToArgsIfNeeded(
ASTs & args,
const String & structure,
const String & format,

4
src/Storages/ObjectStorage/StorageObjectStorage.h
View File

@ -180,7 +180,9 @@ public:
virtual String getNamespace() const = 0;
virtual StorageObjectStorage::QuerySettings getQuerySettings(const ContextPtr &) const = 0;
virtual void addStructureAndFormatToArgs(
/// Add/replace structure and format arguments in the AST arguments if they have 'auto' values.
virtual void addStructureAndFormatToArgsIfNeeded(
ASTs & args, const String & structure_, const String & format_, ContextPtr context) = 0;
bool withPartitionWildcard() const;

2
src/Storages/ObjectStorage/StorageObjectStorageCluster.cpp
View File

@ -103,7 +103,7 @@ void StorageObjectStorageCluster::updateQueryToSendIfNeeded(
ASTPtr cluster_name_arg = args.front();
args.erase(args.begin());
configuration->addStructureAndFormatToArgs(args, structure, configuration->format, context);
configuration->addStructureAndFormatToArgsIfNeeded(args, structure, configuration->format, context);
args.insert(args.begin(), cluster_name_arg);
}

7
src/TableFunctions/ITableFunctionCluster.h
View File

@ -23,7 +23,6 @@ class ITableFunctionCluster : public Base
{
public:
String getName() const override = 0;
String getSignature() const override = 0;
static void updateStructureAndFormatArgumentsIfNeeded(ASTs & args, const String & structure_, const String & format_, const ContextPtr & context)
{
@ -46,7 +45,11 @@ protected:
void parseArgumentsImpl(ASTs & args, const ContextPtr & context) override
{
if (args.empty())
throw Exception(ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH, "The signature of table function {} shall be the following:\n{}", getName(), getSignature());
throw Exception(
ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH,
"The function {} should have arguments. The first argument must be the cluster name and the rest are the arguments of "
"corresponding table function",
getName());
/// Evaluate only first argument, everything else will be done Base class
args[0] = evaluateConstantExpressionOrIdentifierAsLiteral(args[0], context);

2
src/TableFunctions/ITableFunctionFileLike.cpp
View File

@ -57,7 +57,7 @@ void ITableFunctionFileLike::parseArguments(const ASTPtr & ast_function, Context
void ITableFunctionFileLike::parseArgumentsImpl(ASTs & args, const ContextPtr & context)
{
if (args.empty() || args.size() > 4)
if (args.empty() || args.size() > getMaxNumberOfArguments())
throw Exception(ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH, "The signature of table function {} shall be the following:\n{}", getName(), getSignature());
for (auto & arg : args)

3
src/TableFunctions/ITableFunctionFileLike.h
View File

@ -15,6 +15,7 @@ class Context;
class ITableFunctionFileLike : public ITableFunction
{
public:
static constexpr auto max_number_of_arguments = 4;
static constexpr auto signature = " - filename\n"
" - filename, format\n"
" - filename, format, structure\n"
@ -32,7 +33,7 @@ public:
NameSet getVirtualsToCheckBeforeUsingStructureHint() const override;
static size_t getMaxNumberOfArguments() { return 4; }
static size_t getMaxNumberOfArguments() { return max_number_of_arguments; }
static void updateStructureAndFormatArgumentsIfNeeded(ASTs & args, const String & structure, const String & format, const ContextPtr &);

43
src/TableFunctions/TableFunctionObjectStorage.h
View File

@ -22,72 +22,36 @@ struct AzureDefinition
{
static constexpr auto name = "azureBlobStorage";
static constexpr auto storage_type_name = "Azure";
static constexpr auto signature = " - connection_string, container_name, blobpath\n"
" - connection_string, container_name, blobpath, structure \n"
" - connection_string, container_name, blobpath, format \n"
" - connection_string, container_name, blobpath, format, compression \n"
" - connection_string, container_name, blobpath, format, compression, structure \n"
" - storage_account_url, container_name, blobpath, account_name, account_key\n"
" - storage_account_url, container_name, blobpath, account_name, account_key, structure\n"
" - storage_account_url, container_name, blobpath, account_name, account_key, format\n"
" - storage_account_url, container_name, blobpath, account_name, account_key, format, compression\n"
" - storage_account_url, container_name, blobpath, account_name, account_key, format, compression, structure\n";
static constexpr auto max_number_of_arguments = 8;
};
struct S3Definition
{
static constexpr auto name = "s3";
static constexpr auto storage_type_name = "S3";
static constexpr auto signature = " - url\n"
" - url, format\n"
" - url, format, structure\n"
" - url, format, structure, compression_method\n"
" - url, access_key_id, secret_access_key\n"
" - url, access_key_id, secret_access_key, session_token\n"
" - url, access_key_id, secret_access_key, format\n"
" - url, access_key_id, secret_access_key, session_token, format\n"
" - url, access_key_id, secret_access_key, format, structure\n"
" - url, access_key_id, secret_access_key, session_token, format, structure\n"
" - url, access_key_id, secret_access_key, format, structure, compression_method\n"
" - url, access_key_id, secret_access_key, session_token, format, structure, compression_method\n"
"All signatures supports optional headers (specified as `headers('name'='value', 'name2'='value2')`)";
static constexpr auto max_number_of_arguments = 8;
};
struct GCSDefinition
{
static constexpr auto name = "gcs";
static constexpr auto storage_type_name = "GCS";
static constexpr auto signature = S3Definition::signature;
static constexpr auto max_number_of_arguments = S3Definition::max_number_of_arguments;
};
struct COSNDefinition
{
static constexpr auto name = "cosn";
static constexpr auto storage_type_name = "COSN";
static constexpr auto signature = S3Definition::signature;
static constexpr auto max_number_of_arguments = S3Definition::max_number_of_arguments;
};
struct OSSDefinition
{
static constexpr auto name = "oss";
static constexpr auto storage_type_name = "OSS";
static constexpr auto signature = S3Definition::signature;
static constexpr auto max_number_of_arguments = S3Definition::max_number_of_arguments;
};
struct HDFSDefinition
{
static constexpr auto name = "hdfs";
static constexpr auto storage_type_name = "HDFS";
static constexpr auto signature = " - uri\n"
" - uri, format\n"
" - uri, format, structure\n"
" - uri, format, structure, compression_method\n";
static constexpr auto max_number_of_arguments = 4;
};
template <typename Definition, typename Configuration>
@ -95,14 +59,9 @@ class TableFunctionObjectStorage : public ITableFunction
{
public:
static constexpr auto name = Definition::name;
static constexpr auto signature = Definition::signature;
static size_t getMaxNumberOfArguments() { return Definition::max_number_of_arguments; }
String getName() const override { return name; }
virtual String getSignature() const { return signature; }
bool hasStaticStructure() const override { return configuration->structure != "auto"; }
bool needStructureHint() const override { return configuration->structure == "auto"; }
@ -130,7 +89,7 @@ public:
const String & format,
const ContextPtr & context)
{
Configuration().addStructureAndFormatToArgs(args, structure, format, context);
Configuration().addStructureAndFormatToArgsIfNeeded(args, structure, format, context);
}
protected:

22
src/TableFunctions/TableFunctionObjectStorageCluster.h
View File

@ -19,40 +19,22 @@ struct AzureClusterDefinition
{
static constexpr auto name = "azureBlobStorageCluster";
static constexpr auto storage_type_name = "AzureBlobStorageCluster";
static constexpr auto signature = " - cluster, connection_string|storage_account_url, container_name, blobpath, [account_name, account_key, format, compression, structure]";
static constexpr auto max_number_of_arguments = AzureDefinition::max_number_of_arguments + 1;
};
struct S3ClusterDefinition
{
static constexpr auto name = "s3Cluster";
static constexpr auto storage_type_name = "S3Cluster";
static constexpr auto signature = " - cluster, url\n"
" - cluster, url, format\n"
" - cluster, url, format, structure\n"
" - cluster, url, access_key_id, secret_access_key\n"
" - cluster, url, format, structure, compression_method\n"
" - cluster, url, access_key_id, secret_access_key, format\n"
" - cluster, url, access_key_id, secret_access_key, format, structure\n"
" - cluster, url, access_key_id, secret_access_key, format, structure, compression_method\n"
" - cluster, url, access_key_id, secret_access_key, session_token, format, structure, compression_method\n"
"All signatures supports optional headers (specified as `headers('name'='value', 'name2'='value2')`)";
static constexpr auto max_number_of_arguments = S3Definition::max_number_of_arguments + 1;
};
struct HDFSClusterDefinition
{
static constexpr auto name = "hdfsCluster";
static constexpr auto storage_type_name = "HDFSCluster";
static constexpr auto signature = " - cluster_name, uri\n"
" - cluster_name, uri, format\n"
" - cluster_name, uri, format, structure\n"
" - cluster_name, uri, format, structure, compression_method\n";
static constexpr auto max_number_of_arguments = HDFSDefinition::max_number_of_arguments + 1;
};
/**
* Class implementing s3/hdfs/azureBlobStorage)Cluster(...) table functions,
* Class implementing s3/hdfs/azureBlobStorageCluster(...) table functions,
* which allow to process many files from S3/HDFS/Azure blob storage on a specific cluster.
* On initiator it creates a connection to _all_ nodes in cluster, discloses asterisks
* in file path and dispatch each file dynamically.
@ -64,10 +46,8 @@ class TableFunctionObjectStorageCluster : public ITableFunctionCluster<TableFunc
{
public:
static constexpr auto name = Definition::name;
static constexpr auto signature = Definition::signature;
String getName() const override { return name; }
String getSignature() const override { return signature; }
protected:
using Base = TableFunctionObjectStorage<Definition, Configuration>;

7
tests/integration/test_s3_cluster/configs/named_collections.xml
View File

@ -6,5 +6,12 @@
<secret_access_key>minio123</secret_access_key>
<format>CSV</format>>
</test_s3>
<test_s3_with_default>
<url>http://minio1:9001/root/data/data{1,2,3}</url>
<access_key_id>minio</access_key_id>
<secret_access_key>minio123</secret_access_key>
<format>JSONEachRow</format>>
<structure>id UInt32, date Date DEFAULT 18262</structure>
</test_s3_with_default>
</named_collections>
</clickhouse>

48
tests/integration/test_s3_cluster/test.py
View File

@ -459,3 +459,51 @@ def test_cluster_format_detection(started_cluster):
)
assert result == expected_result
def test_cluster_default_expression(started_cluster):
node = started_cluster.instances["s0_0_0"]
node.query(
"insert into function s3('http://minio1:9001/root/data/data1', 'minio', 'minio123', JSONEachRow) select 1 as id settings s3_truncate_on_insert=1"
)
node.query(
"insert into function s3('http://minio1:9001/root/data/data2', 'minio', 'minio123', JSONEachRow) select * from numbers(0) settings s3_truncate_on_insert=1"
)
node.query(
"insert into function s3('http://minio1:9001/root/data/data3', 'minio', 'minio123', JSONEachRow) select 2 as id settings s3_truncate_on_insert=1"
)
expected_result = node.query(
"SELECT * FROM s3('http://minio1:9001/root/data/data{1,2,3}', 'minio', 'minio123', 'JSONEachRow', 'id UInt32, date Date DEFAULT 18262') order by id"
)
result = node.query(
"SELECT * FROM s3Cluster(cluster_simple, 'http://minio1:9001/root/data/data{1,2,3}', 'minio', 'minio123', 'JSONEachRow', 'id UInt32, date Date DEFAULT 18262') order by id"
)
assert result == expected_result
result = node.query(
"SELECT * FROM s3Cluster(cluster_simple, 'http://minio1:9001/root/data/data{1,2,3}', 'minio', 'minio123', 'auto', 'id UInt32, date Date DEFAULT 18262') order by id"
)
assert result == expected_result
result = node.query(
"SELECT * FROM s3Cluster(cluster_simple, 'http://minio1:9001/root/data/data{1,2,3}', 'minio', 'minio123', 'JSONEachRow', 'id UInt32, date Date DEFAULT 18262', 'auto') order by id"
)
assert result == expected_result
result = node.query(
"SELECT * FROM s3Cluster(cluster_simple, 'http://minio1:9001/root/data/data{1,2,3}', 'minio', 'minio123', 'auto', 'id UInt32, date Date DEFAULT 18262', 'auto') order by id"
)
assert result == expected_result
result = node.query(
"SELECT * FROM s3Cluster(cluster_simple, test_s3_with_default) order by id"
)
assert result == expected_result

192
tests/queries/0_stateless/01801_s3_cluster.reference
View File

@ -190,3 +190,195 @@
20 21 22
23 24 25
26 27 28
0 0 0
0 0 0
0 0 0
1 2 3
4 5 6
7 8 9
10 11 12
13 14 15
16 17 18
20 21 22
23 24 25
26 27 28
0 0 0
0 0 0
0 0 0
1 2 3
4 5 6
7 8 9
10 11 12
13 14 15
16 17 18
20 21 22
23 24 25
26 27 28
0 0 0
0 0 0
0 0 0
1 2 3
4 5 6
7 8 9
10 11 12
13 14 15
16 17 18
20 21 22
23 24 25
26 27 28
0 0 0
0 0 0
0 0 0
1 2 3
4 5 6
7 8 9
10 11 12
13 14 15
16 17 18
20 21 22
23 24 25
26 27 28
0 0 0
0 0 0
0 0 0
1 2 3
4 5 6
7 8 9
10 11 12
13 14 15
16 17 18
20 21 22
23 24 25
26 27 28
0 0 0
0 0 0
0 0 0
1 2 3
4 5 6
7 8 9
10 11 12
13 14 15
16 17 18
20 21 22
23 24 25
26 27 28
0 0 0
0 0 0
0 0 0
1 2 3
4 5 6
7 8 9
10 11 12
13 14 15
16 17 18
20 21 22
23 24 25
26 27 28
0 0 0
0 0 0
0 0 0
1 2 3
4 5 6
7 8 9
10 11 12
13 14 15
16 17 18
20 21 22
23 24 25
26 27 28
0 0 0
0 0 0
0 0 0
1 2 3
4 5 6
7 8 9
10 11 12
13 14 15
16 17 18
20 21 22
23 24 25
26 27 28
0 0 0
0 0 0
0 0 0
1 2 3
4 5 6
7 8 9
10 11 12
13 14 15
16 17 18
20 21 22
23 24 25
26 27 28
0 0 0
0 0 0
0 0 0
1 2 3
4 5 6
7 8 9
10 11 12
13 14 15
16 17 18
20 21 22
23 24 25
26 27 28
0 0 0
0 0 0
0 0 0
1 2 3
4 5 6
7 8 9
10 11 12
13 14 15
16 17 18
20 21 22
23 24 25
26 27 28
0 0 0
0 0 0
0 0 0
1 2 3
4 5 6
7 8 9
10 11 12
13 14 15
16 17 18
20 21 22
23 24 25
26 27 28
0 0 0
0 0 0
0 0 0
1 2 3
4 5 6
7 8 9
10 11 12
13 14 15
16 17 18
20 21 22
23 24 25
26 27 28
0 0 0
0 0 0
0 0 0
1 2 3
4 5 6
7 8 9
10 11 12
13 14 15
16 17 18
20 21 22
23 24 25
26 27 28
0 0 0
0 0 0
0 0 0
1 2 3
4 5 6
7 8 9
10 11 12
13 14 15
16 17 18
20 21 22
23 24 25
26 27 28

20
tests/queries/0_stateless/01801_s3_cluster.sql
View File

@ -2,21 +2,37 @@
-- Tag no-fasttest: Depends on AWS
select * from s3('http://localhost:11111/test/{a,b,c}.tsv') ORDER BY c1, c2, c3;
select * from s3('http://localhost:11111/test/{a,b,c}.tsv', NOSIGN) ORDER BY c1, c2, c3;
select * from s3('http://localhost:11111/test/{a,b,c}.tsv', 'TSV') ORDER BY c1, c2, c3;
select * from s3('http://localhost:11111/test/{a,b,c}.tsv', NOSIGN, 'TSV') ORDER BY c1, c2, c3;
select * from s3('http://localhost:11111/test/{a,b,c}.tsv', 'TSV', 'c1 UInt64, c2 UInt64, c3 UInt64') ORDER BY c1, c2, c3;
select * from s3('http://localhost:11111/test/{a,b,c}.tsv', 'test', 'testtest') ORDER BY c1, c2, c3;
select * from s3('http://localhost:11111/test/{a,b,c}.tsv', NOSIGN, 'TSV', 'c1 UInt64, c2 UInt64, c3 UInt64') ORDER BY c1, c2, c3;
select * from s3('http://localhost:11111/test/{a,b,c}.tsv', 'TSV', 'c1 UInt64, c2 UInt64, c3 UInt64', 'auto') ORDER BY c1, c2, c3;
select * from s3('http://localhost:11111/test/{a,b,c}.tsv', NOSIGN, 'TSV', 'c1 UInt64, c2 UInt64, c3 UInt64', 'auto') ORDER BY c1, c2, c3;
select * from s3('http://localhost:11111/test/{a,b,c}.tsv', 'test', 'testtest') ORDER BY c1, c2, c3;
select * from s3('http://localhost:11111/test/{a,b,c}.tsv', 'test', 'testtest', '') ORDER BY c1, c2, c3;
select * from s3('http://localhost:11111/test/{a,b,c}.tsv', 'test', 'testtest', 'TSV') ORDER BY c1, c2, c3;
select * from s3('http://localhost:11111/test/{a,b,c}.tsv', 'test', 'testtest', '', 'TSV') ORDER BY c1, c2, c3;
select * from s3('http://localhost:11111/test/{a,b,c}.tsv', 'test', 'testtest', 'TSV', 'c1 UInt64, c2 UInt64, c3 UInt64') ORDER BY c1, c2, c3;
select * from s3('http://localhost:11111/test/{a,b,c}.tsv', 'test', 'testtest', '', 'TSV', 'c1 UInt64, c2 UInt64, c3 UInt64') ORDER BY c1, c2, c3;
select * from s3('http://localhost:11111/test/{a,b,c}.tsv', 'test', 'testtest', 'TSV', 'c1 UInt64, c2 UInt64, c3 UInt64', 'auto') ORDER BY c1, c2, c3;
select * from s3('http://localhost:11111/test/{a,b,c}.tsv', 'test', 'testtest', '', 'TSV', 'c1 UInt64, c2 UInt64, c3 UInt64', 'auto') ORDER BY c1, c2, c3;
select * from s3Cluster('test_cluster_two_shards_localhost', 'http://localhost:11111/test/{a,b,c}.tsv') ORDER BY c1, c2, c3;
select * from s3Cluster('test_cluster_two_shards_localhost', 'http://localhost:11111/test/{a,b,c}.tsv', NOSIGN) ORDER BY c1, c2, c3;
select * from s3Cluster('test_cluster_two_shards_localhost', 'http://localhost:11111/test/{a,b,c}.tsv', 'TSV') ORDER BY c1, c2, c3;
select * from s3Cluster('test_cluster_two_shards_localhost', 'http://localhost:11111/test/{a,b,c}.tsv', NOSIGN, 'TSV') ORDER BY c1, c2, c3;
select * from s3Cluster('test_cluster_two_shards_localhost', 'http://localhost:11111/test/{a,b,c}.tsv', 'TSV', 'c1 UInt64, c2 UInt64, c3 UInt64') ORDER BY c1, c2, c3;
select * from s3Cluster('test_cluster_two_shards_localhost', 'http://localhost:11111/test/{a,b,c}.tsv', 'test', 'testtest') ORDER BY c1, c2, c3;
select * from s3Cluster('test_cluster_two_shards_localhost', 'http://localhost:11111/test/{a,b,c}.tsv', NOSIGN, 'TSV', 'c1 UInt64, c2 UInt64, c3 UInt64') ORDER BY c1, c2, c3;
select * from s3Cluster('test_cluster_two_shards_localhost', 'http://localhost:11111/test/{a,b,c}.tsv', 'TSV', 'c1 UInt64, c2 UInt64, c3 UInt64', 'auto') ORDER BY c1, c2, c3;
select * from s3Cluster('test_cluster_two_shards_localhost', 'http://localhost:11111/test/{a,b,c}.tsv', NOSIGN, 'TSV', 'c1 UInt64, c2 UInt64, c3 UInt64', 'auto') ORDER BY c1, c2, c3;
select * from s3Cluster('test_cluster_two_shards_localhost', 'http://localhost:11111/test/{a,b,c}.tsv', 'test', 'testtest') ORDER BY c1, c2, c3;
select * from s3Cluster('test_cluster_two_shards_localhost', 'http://localhost:11111/test/{a,b,c}.tsv', 'test', 'testtest', '') ORDER BY c1, c2, c3;
select * from s3Cluster('test_cluster_two_shards_localhost', 'http://localhost:11111/test/{a,b,c}.tsv', 'test', 'testtest', 'TSV') ORDER BY c1, c2, c3;
select * from s3Cluster('test_cluster_two_shards_localhost', 'http://localhost:11111/test/{a,b,c}.tsv', 'test', 'testtest', '', 'TSV') ORDER BY c1, c2, c3;
select * from s3Cluster('test_cluster_two_shards_localhost', 'http://localhost:11111/test/{a,b,c}.tsv', 'test', 'testtest', 'TSV', 'c1 UInt64, c2 UInt64, c3 UInt64') ORDER BY c1, c2, c3;
select * from s3Cluster('test_cluster_two_shards_localhost', 'http://localhost:11111/test/{a,b,c}.tsv', 'test', 'testtest', '', 'TSV', 'c1 UInt64, c2 UInt64, c3 UInt64') ORDER BY c1, c2, c3;
select * from s3Cluster('test_cluster_two_shards_localhost', 'http://localhost:11111/test/{a,b,c}.tsv', 'test', 'testtest', 'TSV', 'c1 UInt64, c2 UInt64, c3 UInt64', 'auto') ORDER BY c1, c2, c3;
select * from s3Cluster('test_cluster_two_shards_localhost', 'http://localhost:11111/test/{a,b,c}.tsv', 'test', 'testtest', '', 'TSV', 'c1 UInt64, c2 UInt64, c3 UInt64', 'auto') ORDER BY c1, c2, c3;