mirror of
https://github.com/ClickHouse/ClickHouse.git
synced 2024-09-19 16:20:50 +00:00
Merge pull request #69569 from ClickHouse/cherrypick/24.8/8610a01745a459459159b109b7193a5ab1a2d52e
Cherry pick #69147 to 24.8: Fix propogating structure argument in s3Cluster
This commit is contained in:
commit
fe91f72767
@ -24,6 +24,7 @@ namespace ErrorCodes
|
|||||||
{
|
{
|
||||||
extern const int BAD_ARGUMENTS;
|
extern const int BAD_ARGUMENTS;
|
||||||
extern const int NUMBER_OF_ARGUMENTS_DOESNT_MATCH;
|
extern const int NUMBER_OF_ARGUMENTS_DOESNT_MATCH;
|
||||||
|
extern const int LOGICAL_ERROR;
|
||||||
}
|
}
|
||||||
|
|
||||||
const std::unordered_set<std::string_view> required_configuration_keys = {
|
const std::unordered_set<std::string_view> required_configuration_keys = {
|
||||||
@ -146,12 +147,13 @@ void StorageAzureConfiguration::fromNamedCollection(const NamedCollection & coll
|
|||||||
|
|
||||||
void StorageAzureConfiguration::fromAST(ASTs & engine_args, ContextPtr context, bool with_structure)
|
void StorageAzureConfiguration::fromAST(ASTs & engine_args, ContextPtr context, bool with_structure)
|
||||||
{
|
{
|
||||||
if (engine_args.size() < 3 || engine_args.size() > (with_structure ? 8 : 7))
|
if (engine_args.size() < 3 || engine_args.size() > getMaxNumberOfArguments(with_structure))
|
||||||
{
|
{
|
||||||
throw Exception(ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH,
|
throw Exception(
|
||||||
"Storage AzureBlobStorage requires 3 to 7 arguments: "
|
ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH,
|
||||||
"AzureBlobStorage(connection_string|storage_account_url, container_name, blobpath, "
|
"Storage AzureBlobStorage requires 3 to {} arguments. All supported signatures:\n{}",
|
||||||
"[account_name, account_key, format, compression, structure)])");
|
getMaxNumberOfArguments(with_structure),
|
||||||
|
getSignatures(with_structure));
|
||||||
}
|
}
|
||||||
|
|
||||||
for (auto & engine_arg : engine_args)
|
for (auto & engine_arg : engine_args)
|
||||||
@ -211,7 +213,7 @@ void StorageAzureConfiguration::fromAST(ASTs & engine_args, ContextPtr context,
|
|||||||
if (with_structure)
|
if (with_structure)
|
||||||
{
|
{
|
||||||
format = fourth_arg;
|
format = fourth_arg;
|
||||||
compression_method = checkAndGetLiteralArgument<String>(engine_args[4], "compression");
|
compression_method = checkAn/dGetLiteralArgument<String>(engine_args[4], "compression");
|
||||||
structure = checkAndGetLiteralArgument<String>(engine_args[5], "structure");
|
structure = checkAndGetLiteralArgument<String>(engine_args[5], "structure");
|
||||||
}
|
}
|
||||||
else
|
else
|
||||||
@ -270,26 +272,30 @@ void StorageAzureConfiguration::fromAST(ASTs & engine_args, ContextPtr context,
|
|||||||
connection_params = getConnectionParams(connection_url, container_name, account_name, account_key, context);
|
connection_params = getConnectionParams(connection_url, container_name, account_name, account_key, context);
|
||||||
}
|
}
|
||||||
|
|
||||||
void StorageAzureConfiguration::addStructureAndFormatToArgs(
|
void StorageAzureConfiguration::addStructureAndFormatToArgsIfNeeded(
|
||||||
ASTs & args, const String & structure_, const String & format_, ContextPtr context)
|
ASTs & args, const String & structure_, const String & format_, ContextPtr context)
|
||||||
{
|
{
|
||||||
if (tryGetNamedCollectionWithOverrides(args, context))
|
if (auto collection = tryGetNamedCollectionWithOverrides(args, context))
|
||||||
{
|
{
|
||||||
/// In case of named collection, just add key-value pair "structure='...'"
|
/// In case of named collection, just add key-value pairs "format='...', structure='...'"
|
||||||
/// at the end of arguments to override existed structure.
|
/// at the end of arguments to override existed format and structure with "auto" values.
|
||||||
ASTs equal_func_args = {std::make_shared<ASTIdentifier>("structure"), std::make_shared<ASTLiteral>(structure_)};
|
if (collection->getOrDefault<String>("format", "auto") == "auto")
|
||||||
auto equal_func = makeASTFunction("equals", std::move(equal_func_args));
|
{
|
||||||
args.push_back(equal_func);
|
ASTs format_equal_func_args = {std::make_shared<ASTIdentifier>("format"), std::make_shared<ASTLiteral>(format_)};
|
||||||
|
auto format_equal_func = makeASTFunction("equals", std::move(format_equal_func_args));
|
||||||
|
args.push_back(format_equal_func);
|
||||||
|
}
|
||||||
|
if (collection->getOrDefault<String>("structure", "auto") == "auto")
|
||||||
|
{
|
||||||
|
ASTs structure_equal_func_args = {std::make_shared<ASTIdentifier>("structure"), std::make_shared<ASTLiteral>(structure_)};
|
||||||
|
auto structure_equal_func = makeASTFunction("equals", std::move(structure_equal_func_args));
|
||||||
|
args.push_back(structure_equal_func);
|
||||||
|
}
|
||||||
}
|
}
|
||||||
else
|
else
|
||||||
{
|
{
|
||||||
if (args.size() < 3 || args.size() > 8)
|
if (args.size() < 3 || args.size() > getMaxNumberOfArguments())
|
||||||
{
|
throw Exception(ErrorCodes::LOGICAL_ERROR, "Expected 3 to {} arguments in table function azureBlobStorage, got {}", getMaxNumberOfArguments(), args.size());
|
||||||
throw Exception(ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH,
|
|
||||||
"Storage Azure requires 3 to 7 arguments: "
|
|
||||||
"StorageObjectStorage(connection_string|storage_account_url, container_name, "
|
|
||||||
"blobpath, [account_name, account_key, format, compression, structure])");
|
|
||||||
}
|
|
||||||
|
|
||||||
for (auto & arg : args)
|
for (auto & arg : args)
|
||||||
arg = evaluateConstantExpressionOrIdentifierAsLiteral(arg, context);
|
arg = evaluateConstantExpressionOrIdentifierAsLiteral(arg, context);
|
||||||
|
@ -22,6 +22,29 @@ public:
|
|||||||
|
|
||||||
static constexpr auto type_name = "azure";
|
static constexpr auto type_name = "azure";
|
||||||
static constexpr auto engine_name = "Azure";
|
static constexpr auto engine_name = "Azure";
|
||||||
|
/// All possible signatures for Azure engine with structure argument (for example for azureBlobStorage table function).
|
||||||
|
static constexpr auto max_number_of_arguments_with_structure = 8;
|
||||||
|
static constexpr auto signatures_with_structure =
|
||||||
|
" - connection_string, container_name, blobpath\n"
|
||||||
|
" - connection_string, container_name, blobpath, structure \n"
|
||||||
|
" - connection_string, container_name, blobpath, format \n"
|
||||||
|
" - connection_string, container_name, blobpath, format, compression \n"
|
||||||
|
" - connection_string, container_name, blobpath, format, compression, structure \n"
|
||||||
|
" - storage_account_url, container_name, blobpath, account_name, account_key\n"
|
||||||
|
" - storage_account_url, container_name, blobpath, account_name, account_key, structure\n"
|
||||||
|
" - storage_account_url, container_name, blobpath, account_name, account_key, format\n"
|
||||||
|
" - storage_account_url, container_name, blobpath, account_name, account_key, format, compression\n"
|
||||||
|
" - storage_account_url, container_name, blobpath, account_name, account_key, format, compression, structure\n";
|
||||||
|
|
||||||
|
/// All possible signatures for Azure engine without structure argument (for example for AzureBlobStorage table engine).
|
||||||
|
static constexpr auto max_number_of_arguments_without_structure = 7;
|
||||||
|
static constexpr auto signatures_without_structure =
|
||||||
|
" - connection_string, container_name, blobpath\n"
|
||||||
|
" - connection_string, container_name, blobpath, format \n"
|
||||||
|
" - connection_string, container_name, blobpath, format, compression \n"
|
||||||
|
" - storage_account_url, container_name, blobpath, account_name, account_key\n"
|
||||||
|
" - storage_account_url, container_name, blobpath, account_name, account_key, format\n"
|
||||||
|
" - storage_account_url, container_name, blobpath, account_name, account_key, format, compression\n";
|
||||||
|
|
||||||
StorageAzureConfiguration() = default;
|
StorageAzureConfiguration() = default;
|
||||||
StorageAzureConfiguration(const StorageAzureConfiguration & other);
|
StorageAzureConfiguration(const StorageAzureConfiguration & other);
|
||||||
@ -29,6 +52,9 @@ public:
|
|||||||
std::string getTypeName() const override { return type_name; }
|
std::string getTypeName() const override { return type_name; }
|
||||||
std::string getEngineName() const override { return engine_name; }
|
std::string getEngineName() const override { return engine_name; }
|
||||||
|
|
||||||
|
std::string getSignatures(bool with_structure = true) const { return with_structure ? signatures_with_structure : signatures_without_structure; }
|
||||||
|
size_t getMaxNumberOfArguments(bool with_structure = true) const { return with_structure ? max_number_of_arguments_with_structure : max_number_of_arguments_without_structure; }
|
||||||
|
|
||||||
Path getPath() const override { return blob_path; }
|
Path getPath() const override { return blob_path; }
|
||||||
void setPath(const Path & path) override { blob_path = path; }
|
void setPath(const Path & path) override { blob_path = path; }
|
||||||
|
|
||||||
@ -44,7 +70,7 @@ public:
|
|||||||
|
|
||||||
ObjectStoragePtr createObjectStorage(ContextPtr context, bool is_readonly) override;
|
ObjectStoragePtr createObjectStorage(ContextPtr context, bool is_readonly) override;
|
||||||
|
|
||||||
void addStructureAndFormatToArgs(
|
void addStructureAndFormatToArgsIfNeeded(
|
||||||
ASTs & args,
|
ASTs & args,
|
||||||
const String & structure_,
|
const String & structure_,
|
||||||
const String & format_,
|
const String & format_,
|
||||||
|
@ -24,6 +24,7 @@ namespace ErrorCodes
|
|||||||
{
|
{
|
||||||
extern const int BAD_ARGUMENTS;
|
extern const int BAD_ARGUMENTS;
|
||||||
extern const int NUMBER_OF_ARGUMENTS_DOESNT_MATCH;
|
extern const int NUMBER_OF_ARGUMENTS_DOESNT_MATCH;
|
||||||
|
extern const int LOGICAL_ERROR;
|
||||||
}
|
}
|
||||||
|
|
||||||
StorageHDFSConfiguration::StorageHDFSConfiguration(const StorageHDFSConfiguration & other)
|
StorageHDFSConfiguration::StorageHDFSConfiguration(const StorageHDFSConfiguration & other)
|
||||||
@ -83,12 +84,13 @@ StorageObjectStorage::QuerySettings StorageHDFSConfiguration::getQuerySettings(c
|
|||||||
|
|
||||||
void StorageHDFSConfiguration::fromAST(ASTs & args, ContextPtr context, bool with_structure)
|
void StorageHDFSConfiguration::fromAST(ASTs & args, ContextPtr context, bool with_structure)
|
||||||
{
|
{
|
||||||
const size_t max_args_num = with_structure ? 4 : 3;
|
if (args.empty() || args.size() > getMaxNumberOfArguments(with_structure))
|
||||||
if (args.empty() || args.size() > max_args_num)
|
throw Exception(
|
||||||
{
|
ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH,
|
||||||
throw Exception(ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH,
|
"Storage HDFS requires 1 to {} arguments. All supported signatures:\n{}",
|
||||||
"Expected not more than {} arguments", max_args_num);
|
getMaxNumberOfArguments(with_structure),
|
||||||
}
|
getSignatures(with_structure));
|
||||||
|
|
||||||
|
|
||||||
std::string url_str;
|
std::string url_str;
|
||||||
url_str = checkAndGetLiteralArgument<String>(args[0], "url");
|
url_str = checkAndGetLiteralArgument<String>(args[0], "url");
|
||||||
@ -158,28 +160,34 @@ void StorageHDFSConfiguration::setURL(const std::string & url_)
|
|||||||
LOG_TRACE(getLogger("StorageHDFSConfiguration"), "Using URL: {}, path: {}", url, path);
|
LOG_TRACE(getLogger("StorageHDFSConfiguration"), "Using URL: {}, path: {}", url, path);
|
||||||
}
|
}
|
||||||
|
|
||||||
void StorageHDFSConfiguration::addStructureAndFormatToArgs(
|
void StorageHDFSConfiguration::addStructureAndFormatToArgsIfNeeded(
|
||||||
ASTs & args,
|
ASTs & args,
|
||||||
const String & structure_,
|
const String & structure_,
|
||||||
const String & format_,
|
const String & format_,
|
||||||
ContextPtr context)
|
ContextPtr context)
|
||||||
{
|
{
|
||||||
if (tryGetNamedCollectionWithOverrides(args, context))
|
if (auto collection = tryGetNamedCollectionWithOverrides(args, context))
|
||||||
{
|
{
|
||||||
/// In case of named collection, just add key-value pair "structure='...'"
|
/// In case of named collection, just add key-value pairs "format='...', structure='...'"
|
||||||
/// at the end of arguments to override existed structure.
|
/// at the end of arguments to override existed format and structure with "auto" values.
|
||||||
ASTs equal_func_args = {std::make_shared<ASTIdentifier>("structure"), std::make_shared<ASTLiteral>(structure_)};
|
if (collection->getOrDefault<String>("format", "auto") == "auto")
|
||||||
auto equal_func = makeASTFunction("equals", std::move(equal_func_args));
|
{
|
||||||
args.push_back(equal_func);
|
ASTs format_equal_func_args = {std::make_shared<ASTIdentifier>("format"), std::make_shared<ASTLiteral>(format_)};
|
||||||
|
auto format_equal_func = makeASTFunction("equals", std::move(format_equal_func_args));
|
||||||
|
args.push_back(format_equal_func);
|
||||||
|
}
|
||||||
|
if (collection->getOrDefault<String>("structure", "auto") == "auto")
|
||||||
|
{
|
||||||
|
ASTs structure_equal_func_args = {std::make_shared<ASTIdentifier>("structure"), std::make_shared<ASTLiteral>(structure_)};
|
||||||
|
auto structure_equal_func = makeASTFunction("equals", std::move(structure_equal_func_args));
|
||||||
|
args.push_back(structure_equal_func);
|
||||||
|
}
|
||||||
}
|
}
|
||||||
else
|
else
|
||||||
{
|
{
|
||||||
size_t count = args.size();
|
size_t count = args.size();
|
||||||
if (count == 0 || count > 4)
|
if (count == 0 || count > getMaxNumberOfArguments())
|
||||||
{
|
throw Exception(ErrorCodes::LOGICAL_ERROR, "Expected 1 to {} arguments in table function hdfs, got {}", getMaxNumberOfArguments(), count);
|
||||||
throw Exception(ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH,
|
|
||||||
"Expected 1 to 4 arguments in table function, got {}", count);
|
|
||||||
}
|
|
||||||
|
|
||||||
auto format_literal = std::make_shared<ASTLiteral>(format_);
|
auto format_literal = std::make_shared<ASTLiteral>(format_);
|
||||||
auto structure_literal = std::make_shared<ASTLiteral>(structure_);
|
auto structure_literal = std::make_shared<ASTLiteral>(structure_);
|
||||||
|
@ -16,6 +16,20 @@ public:
|
|||||||
|
|
||||||
static constexpr auto type_name = "hdfs";
|
static constexpr auto type_name = "hdfs";
|
||||||
static constexpr auto engine_name = "HDFS";
|
static constexpr auto engine_name = "HDFS";
|
||||||
|
/// All possible signatures for HDFS engine with structure argument (for example for hdfs table function).
|
||||||
|
static constexpr auto max_number_of_arguments_with_structure = 4;
|
||||||
|
static constexpr auto signatures_with_structure =
|
||||||
|
" - uri\n"
|
||||||
|
" - uri, format\n"
|
||||||
|
" - uri, format, structure\n"
|
||||||
|
" - uri, format, structure, compression_method\n";
|
||||||
|
|
||||||
|
/// All possible signatures for HDFS engine without structure argument (for example for HS table engine).
|
||||||
|
static constexpr auto max_number_of_arguments_without_structure = 3;
|
||||||
|
static constexpr auto signatures_without_structure =
|
||||||
|
" - uri\n"
|
||||||
|
" - uri, format\n"
|
||||||
|
" - uri, format, compression_method\n";
|
||||||
|
|
||||||
StorageHDFSConfiguration() = default;
|
StorageHDFSConfiguration() = default;
|
||||||
StorageHDFSConfiguration(const StorageHDFSConfiguration & other);
|
StorageHDFSConfiguration(const StorageHDFSConfiguration & other);
|
||||||
@ -23,6 +37,9 @@ public:
|
|||||||
std::string getTypeName() const override { return type_name; }
|
std::string getTypeName() const override { return type_name; }
|
||||||
std::string getEngineName() const override { return engine_name; }
|
std::string getEngineName() const override { return engine_name; }
|
||||||
|
|
||||||
|
std::string getSignatures(bool with_structure = true) const { return with_structure ? signatures_with_structure : signatures_without_structure; }
|
||||||
|
size_t getMaxNumberOfArguments(bool with_structure = true) const { return with_structure ? max_number_of_arguments_with_structure : max_number_of_arguments_without_structure; }
|
||||||
|
|
||||||
Path getPath() const override { return path; }
|
Path getPath() const override { return path; }
|
||||||
void setPath(const Path & path_) override { path = path_; }
|
void setPath(const Path & path_) override { path = path_; }
|
||||||
|
|
||||||
@ -39,7 +56,7 @@ public:
|
|||||||
|
|
||||||
ObjectStoragePtr createObjectStorage(ContextPtr context, bool is_readonly) override;
|
ObjectStoragePtr createObjectStorage(ContextPtr context, bool is_readonly) override;
|
||||||
|
|
||||||
void addStructureAndFormatToArgs(
|
void addStructureAndFormatToArgsIfNeeded(
|
||||||
ASTs & args,
|
ASTs & args,
|
||||||
const String & structure_,
|
const String & structure_,
|
||||||
const String & format_,
|
const String & format_,
|
||||||
|
@ -170,21 +170,20 @@ void StorageS3Configuration::fromNamedCollection(const NamedCollection & collect
|
|||||||
|
|
||||||
void StorageS3Configuration::fromAST(ASTs & args, ContextPtr context, bool with_structure)
|
void StorageS3Configuration::fromAST(ASTs & args, ContextPtr context, bool with_structure)
|
||||||
{
|
{
|
||||||
/// Supported signatures: S3('url') S3('url', 'format') S3('url', 'format', 'compression') S3('url', NOSIGN) S3('url', NOSIGN, 'format') S3('url', NOSIGN, 'format', 'compression') S3('url', 'aws_access_key_id', 'aws_secret_access_key') S3('url', 'aws_access_key_id', 'aws_secret_access_key', 'session_token') S3('url', 'aws_access_key_id', 'aws_secret_access_key', 'format') S3('url', 'aws_access_key_id', 'aws_secret_access_key', 'session_token', 'format') S3('url', 'aws_access_key_id', 'aws_secret_access_key', 'format', 'compression')
|
|
||||||
/// S3('url', 'aws_access_key_id', 'aws_secret_access_key', 'session_token', 'format', 'compression')
|
|
||||||
/// with optional headers() function
|
|
||||||
|
|
||||||
size_t count = StorageURL::evalArgsAndCollectHeaders(args, headers_from_ast, context);
|
size_t count = StorageURL::evalArgsAndCollectHeaders(args, headers_from_ast, context);
|
||||||
|
|
||||||
if (count == 0 || count > (with_structure ? 7 : 6))
|
if (count == 0 || count > getMaxNumberOfArguments(with_structure))
|
||||||
throw Exception(ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH,
|
throw Exception(ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH,
|
||||||
"Storage S3 requires 1 to 5 arguments: "
|
"Storage S3 requires 1 to {} arguments. All supported signatures:\n{}",
|
||||||
"url, [NOSIGN | access_key_id, secret_access_key], name of used format and [compression_method]");
|
getMaxNumberOfArguments(with_structure),
|
||||||
|
getSignatures(with_structure));
|
||||||
|
|
||||||
std::unordered_map<std::string_view, size_t> engine_args_to_idx;
|
std::unordered_map<std::string_view, size_t> engine_args_to_idx;
|
||||||
bool no_sign_request = false;
|
bool no_sign_request = false;
|
||||||
|
|
||||||
/// For 2 arguments we support 2 possible variants:
|
/// When adding new arguments in the signature don't forget to update addStructureAndFormatToArgsIfNeeded as well.
|
||||||
|
|
||||||
|
/// For 2 arguments we support:
|
||||||
/// - s3(source, format)
|
/// - s3(source, format)
|
||||||
/// - s3(source, NOSIGN)
|
/// - s3(source, NOSIGN)
|
||||||
/// We can distinguish them by looking at the 2-nd argument: check if it's NOSIGN or not.
|
/// We can distinguish them by looking at the 2-nd argument: check if it's NOSIGN or not.
|
||||||
@ -196,10 +195,15 @@ void StorageS3Configuration::fromAST(ASTs & args, ContextPtr context, bool with_
|
|||||||
else
|
else
|
||||||
engine_args_to_idx = {{"format", 1}};
|
engine_args_to_idx = {{"format", 1}};
|
||||||
}
|
}
|
||||||
/// For 3 arguments we support 2 possible variants:
|
/// For 3 arguments we support:
|
||||||
|
/// if with_structure == 0:
|
||||||
|
/// - s3(source, NOSIGN, format)
|
||||||
/// - s3(source, format, compression_method)
|
/// - s3(source, format, compression_method)
|
||||||
/// - s3(source, access_key_id, secret_access_key)
|
/// - s3(source, access_key_id, secret_access_key)
|
||||||
|
/// if with_structure == 1:
|
||||||
/// - s3(source, NOSIGN, format)
|
/// - s3(source, NOSIGN, format)
|
||||||
|
/// - s3(source, format, structure)
|
||||||
|
/// - s3(source, access_key_id, secret_access_key)
|
||||||
/// We can distinguish them by looking at the 2-nd argument: check if it's NOSIGN or format name.
|
/// We can distinguish them by looking at the 2-nd argument: check if it's NOSIGN or format name.
|
||||||
else if (count == 3)
|
else if (count == 3)
|
||||||
{
|
{
|
||||||
@ -219,7 +223,7 @@ void StorageS3Configuration::fromAST(ASTs & args, ContextPtr context, bool with_
|
|||||||
else
|
else
|
||||||
engine_args_to_idx = {{"access_key_id", 1}, {"secret_access_key", 2}};
|
engine_args_to_idx = {{"access_key_id", 1}, {"secret_access_key", 2}};
|
||||||
}
|
}
|
||||||
/// For 4 arguments we support 3 possible variants:
|
/// For 4 arguments we support:
|
||||||
/// if with_structure == 0:
|
/// if with_structure == 0:
|
||||||
/// - s3(source, access_key_id, secret_access_key, session_token)
|
/// - s3(source, access_key_id, secret_access_key, session_token)
|
||||||
/// - s3(source, access_key_id, secret_access_key, format)
|
/// - s3(source, access_key_id, secret_access_key, format)
|
||||||
@ -229,7 +233,7 @@ void StorageS3Configuration::fromAST(ASTs & args, ContextPtr context, bool with_
|
|||||||
/// - s3(source, access_key_id, secret_access_key, format),
|
/// - s3(source, access_key_id, secret_access_key, format),
|
||||||
/// - s3(source, access_key_id, secret_access_key, session_token)
|
/// - s3(source, access_key_id, secret_access_key, session_token)
|
||||||
/// - s3(source, NOSIGN, format, structure)
|
/// - s3(source, NOSIGN, format, structure)
|
||||||
/// We can distinguish them by looking at the 2-nd argument: check if it's a NOSIGN or not.
|
/// We can distinguish them by looking at the 2-nd argument: check if it's a NOSIGN, format name of something else.
|
||||||
else if (count == 4)
|
else if (count == 4)
|
||||||
{
|
{
|
||||||
auto second_arg = checkAndGetLiteralArgument<String>(args[1], "access_key_id/NOSIGN");
|
auto second_arg = checkAndGetLiteralArgument<String>(args[1], "access_key_id/NOSIGN");
|
||||||
@ -258,7 +262,7 @@ void StorageS3Configuration::fromAST(ASTs & args, ContextPtr context, bool with_
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
/// For 5 arguments we support 2 possible variants:
|
/// For 5 arguments we support:
|
||||||
/// if with_structure == 0:
|
/// if with_structure == 0:
|
||||||
/// - s3(source, access_key_id, secret_access_key, session_token, format)
|
/// - s3(source, access_key_id, secret_access_key, session_token, format)
|
||||||
/// - s3(source, access_key_id, secret_access_key, format, compression)
|
/// - s3(source, access_key_id, secret_access_key, format, compression)
|
||||||
@ -302,13 +306,16 @@ void StorageS3Configuration::fromAST(ASTs & args, ContextPtr context, bool with_
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
/// For 6 arguments we support:
|
||||||
|
/// if with_structure == 0:
|
||||||
|
/// - s3(source, access_key_id, secret_access_key, session_token, format, compression_method)
|
||||||
|
/// if with_structure == 1:
|
||||||
|
/// - s3(source, access_key_id, secret_access_key, format, structure, compression_method)
|
||||||
|
/// - s3(source, access_key_id, secret_access_key, session_token, format, structure)
|
||||||
else if (count == 6)
|
else if (count == 6)
|
||||||
{
|
{
|
||||||
if (with_structure)
|
if (with_structure)
|
||||||
{
|
{
|
||||||
/// - s3(source, access_key_id, secret_access_key, format, structure, compression_method)
|
|
||||||
/// - s3(source, access_key_id, secret_access_key, session_token, format, structure)
|
|
||||||
/// We can distinguish them by looking at the 4-th argument: check if it's a format name or not
|
|
||||||
auto fourth_arg = checkAndGetLiteralArgument<String>(args[3], "format/session_token");
|
auto fourth_arg = checkAndGetLiteralArgument<String>(args[3], "format/session_token");
|
||||||
if (fourth_arg == "auto" || FormatFactory::instance().exists(fourth_arg))
|
if (fourth_arg == "auto" || FormatFactory::instance().exists(fourth_arg))
|
||||||
{
|
{
|
||||||
@ -324,6 +331,7 @@ void StorageS3Configuration::fromAST(ASTs & args, ContextPtr context, bool with_
|
|||||||
engine_args_to_idx = {{"access_key_id", 1}, {"secret_access_key", 2}, {"session_token", 3}, {"format", 4}, {"compression_method", 5}};
|
engine_args_to_idx = {{"access_key_id", 1}, {"secret_access_key", 2}, {"session_token", 3}, {"format", 4}, {"compression_method", 5}};
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
/// s3(source, access_key_id, secret_access_key, session_token, format, structure, compression_method)
|
||||||
else if (with_structure && count == 7)
|
else if (with_structure && count == 7)
|
||||||
{
|
{
|
||||||
engine_args_to_idx = {{"access_key_id", 1}, {"secret_access_key", 2}, {"session_token", 3}, {"format", 4}, {"structure", 5}, {"compression_method", 6}};
|
engine_args_to_idx = {{"access_key_id", 1}, {"secret_access_key", 2}, {"session_token", 3}, {"format", 4}, {"structure", 5}, {"compression_method", 6}};
|
||||||
@ -365,24 +373,33 @@ void StorageS3Configuration::fromAST(ASTs & args, ContextPtr context, bool with_
|
|||||||
keys = {url.key};
|
keys = {url.key};
|
||||||
}
|
}
|
||||||
|
|
||||||
void StorageS3Configuration::addStructureAndFormatToArgs(
|
void StorageS3Configuration::addStructureAndFormatToArgsIfNeeded(
|
||||||
ASTs & args, const String & structure_, const String & format_, ContextPtr context)
|
ASTs & args, const String & structure_, const String & format_, ContextPtr context)
|
||||||
{
|
{
|
||||||
if (tryGetNamedCollectionWithOverrides(args, context))
|
if (auto collection = tryGetNamedCollectionWithOverrides(args, context))
|
||||||
{
|
{
|
||||||
/// In case of named collection, just add key-value pair "structure='...'"
|
/// In case of named collection, just add key-value pairs "format='...', structure='...'"
|
||||||
/// at the end of arguments to override existed structure.
|
/// at the end of arguments to override existed format and structure with "auto" values.
|
||||||
ASTs equal_func_args = {std::make_shared<ASTIdentifier>("structure"), std::make_shared<ASTLiteral>(structure_)};
|
if (collection->getOrDefault<String>("format", "auto") == "auto")
|
||||||
auto equal_func = makeASTFunction("equals", std::move(equal_func_args));
|
{
|
||||||
args.push_back(equal_func);
|
ASTs format_equal_func_args = {std::make_shared<ASTIdentifier>("format"), std::make_shared<ASTLiteral>(format_)};
|
||||||
|
auto format_equal_func = makeASTFunction("equals", std::move(format_equal_func_args));
|
||||||
|
args.push_back(format_equal_func);
|
||||||
|
}
|
||||||
|
if (collection->getOrDefault<String>("structure", "auto") == "auto")
|
||||||
|
{
|
||||||
|
ASTs structure_equal_func_args = {std::make_shared<ASTIdentifier>("structure"), std::make_shared<ASTLiteral>(structure_)};
|
||||||
|
auto structure_equal_func = makeASTFunction("equals", std::move(structure_equal_func_args));
|
||||||
|
args.push_back(structure_equal_func);
|
||||||
|
}
|
||||||
}
|
}
|
||||||
else
|
else
|
||||||
{
|
{
|
||||||
HTTPHeaderEntries tmp_headers;
|
HTTPHeaderEntries tmp_headers;
|
||||||
size_t count = StorageURL::evalArgsAndCollectHeaders(args, tmp_headers, context);
|
size_t count = StorageURL::evalArgsAndCollectHeaders(args, tmp_headers, context);
|
||||||
|
|
||||||
if (count == 0 || count > 6)
|
if (count == 0 || count > getMaxNumberOfArguments())
|
||||||
throw Exception(ErrorCodes::LOGICAL_ERROR, "Expected 1 to 6 arguments in table function, got {}", count);
|
throw Exception(ErrorCodes::LOGICAL_ERROR, "Expected 1 to {} arguments in table function s3, got {}", getMaxNumberOfArguments(), count);
|
||||||
|
|
||||||
auto format_literal = std::make_shared<ASTLiteral>(format_);
|
auto format_literal = std::make_shared<ASTLiteral>(format_);
|
||||||
auto structure_literal = std::make_shared<ASTLiteral>(structure_);
|
auto structure_literal = std::make_shared<ASTLiteral>(structure_);
|
||||||
@ -394,14 +411,18 @@ void StorageS3Configuration::addStructureAndFormatToArgs(
|
|||||||
args.push_back(std::make_shared<ASTLiteral>("auto"));
|
args.push_back(std::make_shared<ASTLiteral>("auto"));
|
||||||
args.push_back(structure_literal);
|
args.push_back(structure_literal);
|
||||||
}
|
}
|
||||||
/// s3(s3_url, format) or s3(s3_url, NOSIGN)
|
/// s3(s3_url, format) or
|
||||||
|
/// s3(s3_url, NOSIGN)
|
||||||
/// We can distinguish them by looking at the 2-nd argument: check if it's NOSIGN or not.
|
/// We can distinguish them by looking at the 2-nd argument: check if it's NOSIGN or not.
|
||||||
else if (count == 2)
|
else if (count == 2)
|
||||||
{
|
{
|
||||||
auto second_arg = checkAndGetLiteralArgument<String>(args[1], "format/NOSIGN");
|
auto second_arg = checkAndGetLiteralArgument<String>(args[1], "format/NOSIGN");
|
||||||
/// If there is NOSIGN, add format=auto before structure.
|
/// If there is NOSIGN, add format=auto before structure.
|
||||||
if (boost::iequals(second_arg, "NOSIGN"))
|
if (boost::iequals(second_arg, "NOSIGN"))
|
||||||
args.push_back(std::make_shared<ASTLiteral>("auto"));
|
args.push_back(format_literal);
|
||||||
|
else if (checkAndGetLiteralArgument<String>(args[1], "format") == "auto")
|
||||||
|
args[1] = format_literal;
|
||||||
|
|
||||||
args.push_back(structure_literal);
|
args.push_back(structure_literal);
|
||||||
}
|
}
|
||||||
/// s3(source, format, structure) or
|
/// s3(source, format, structure) or
|
||||||
@ -413,21 +434,27 @@ void StorageS3Configuration::addStructureAndFormatToArgs(
|
|||||||
auto second_arg = checkAndGetLiteralArgument<String>(args[1], "format/NOSIGN");
|
auto second_arg = checkAndGetLiteralArgument<String>(args[1], "format/NOSIGN");
|
||||||
if (boost::iequals(second_arg, "NOSIGN"))
|
if (boost::iequals(second_arg, "NOSIGN"))
|
||||||
{
|
{
|
||||||
|
if (checkAndGetLiteralArgument<String>(args[2], "format") == "auto")
|
||||||
|
args[2] = format_literal;
|
||||||
args.push_back(structure_literal);
|
args.push_back(structure_literal);
|
||||||
}
|
}
|
||||||
else if (second_arg == "auto" || FormatFactory::instance().exists(second_arg))
|
else if (second_arg == "auto" || FormatFactory::instance().exists(second_arg))
|
||||||
{
|
{
|
||||||
args[count - 1] = structure_literal;
|
if (second_arg == "auto")
|
||||||
|
args[1] = format_literal;
|
||||||
|
if (checkAndGetLiteralArgument<String>(args[2], "structure") == "auto")
|
||||||
|
args[2] = structure_literal;
|
||||||
}
|
}
|
||||||
else
|
else
|
||||||
{
|
{
|
||||||
/// Add format=auto before structure argument.
|
/// Add format and structure arguments.
|
||||||
args.push_back(std::make_shared<ASTLiteral>("auto"));
|
args.push_back(format_literal);
|
||||||
args.push_back(structure_literal);
|
args.push_back(structure_literal);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
/// s3(source, format, structure, compression_method) or
|
/// s3(source, format, structure, compression_method) or
|
||||||
/// s3(source, access_key_id, secret_access_key, format) or
|
/// s3(source, access_key_id, secret_access_key, format) or
|
||||||
|
/// s3(source, access_key_id, secret_access_key, session_token) or
|
||||||
/// s3(source, NOSIGN, format, structure)
|
/// s3(source, NOSIGN, format, structure)
|
||||||
/// We can distinguish them by looking at the 2-nd argument: check if it's NOSIGN, format name or neither.
|
/// We can distinguish them by looking at the 2-nd argument: check if it's NOSIGN, format name or neither.
|
||||||
else if (count == 4)
|
else if (count == 4)
|
||||||
@ -435,36 +462,93 @@ void StorageS3Configuration::addStructureAndFormatToArgs(
|
|||||||
auto second_arg = checkAndGetLiteralArgument<String>(args[1], "format/NOSIGN");
|
auto second_arg = checkAndGetLiteralArgument<String>(args[1], "format/NOSIGN");
|
||||||
if (boost::iequals(second_arg, "NOSIGN"))
|
if (boost::iequals(second_arg, "NOSIGN"))
|
||||||
{
|
{
|
||||||
args[count - 1] = structure_literal;
|
if (checkAndGetLiteralArgument<String>(args[2], "format") == "auto")
|
||||||
|
args[2] = format_literal;
|
||||||
|
if (checkAndGetLiteralArgument<String>(args[3], "structure") == "auto")
|
||||||
|
args[3] = structure_literal;
|
||||||
}
|
}
|
||||||
else if (second_arg == "auto" || FormatFactory::instance().exists(second_arg))
|
else if (second_arg == "auto" || FormatFactory::instance().exists(second_arg))
|
||||||
{
|
{
|
||||||
args[count - 2] = structure_literal;
|
if (second_arg == "auto")
|
||||||
|
args[1] = format_literal;
|
||||||
|
if (checkAndGetLiteralArgument<String>(args[2], "structure") == "auto")
|
||||||
|
args[2] = structure_literal;
|
||||||
}
|
}
|
||||||
else
|
else
|
||||||
{
|
{
|
||||||
args.push_back(structure_literal);
|
auto fourth_arg = checkAndGetLiteralArgument<String>(args[3], "format/session_token");
|
||||||
|
if (fourth_arg == "auto" || FormatFactory::instance().exists(fourth_arg))
|
||||||
|
{
|
||||||
|
if (checkAndGetLiteralArgument<String>(args[3], "format") == "auto")
|
||||||
|
args[3] = format_literal;
|
||||||
|
args.push_back(structure_literal);
|
||||||
|
}
|
||||||
|
else
|
||||||
|
{
|
||||||
|
args.push_back(format_literal);
|
||||||
|
args.push_back(structure_literal);
|
||||||
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
/// s3(source, access_key_id, secret_access_key, format, structure) or
|
/// s3(source, access_key_id, secret_access_key, format, structure) or
|
||||||
|
/// s3(source, access_key_id, secret_access_key, session_token, format) or
|
||||||
/// s3(source, NOSIGN, format, structure, compression_method)
|
/// s3(source, NOSIGN, format, structure, compression_method)
|
||||||
/// We can distinguish them by looking at the 2-nd argument: check if it's a NOSIGN keyword name or not.
|
/// We can distinguish them by looking at the 2-nd argument: check if it's a NOSIGN keyword name or not.
|
||||||
else if (count == 5)
|
else if (count == 5)
|
||||||
{
|
{
|
||||||
auto sedond_arg = checkAndGetLiteralArgument<String>(args[1], "format/NOSIGN");
|
auto second_arg = checkAndGetLiteralArgument<String>(args[1], "format/NOSIGN");
|
||||||
if (boost::iequals(sedond_arg, "NOSIGN"))
|
if (boost::iequals(second_arg, "NOSIGN"))
|
||||||
{
|
{
|
||||||
args[count - 2] = structure_literal;
|
if (checkAndGetLiteralArgument<String>(args[2], "format") == "auto")
|
||||||
|
args[2] = format_literal;
|
||||||
|
if (checkAndGetLiteralArgument<String>(args[2], "structure") == "auto")
|
||||||
|
args[3] = structure_literal;
|
||||||
}
|
}
|
||||||
else
|
else
|
||||||
{
|
{
|
||||||
args[count - 1] = structure_literal;
|
auto fourth_arg = checkAndGetLiteralArgument<String>(args[3], "format/session_token");
|
||||||
|
if (fourth_arg == "auto" || FormatFactory::instance().exists(fourth_arg))
|
||||||
|
{
|
||||||
|
if (checkAndGetLiteralArgument<String>(args[3], "format") == "auto")
|
||||||
|
args[3] = format_literal;
|
||||||
|
if (checkAndGetLiteralArgument<String>(args[4], "structure") == "auto")
|
||||||
|
args[4] = structure_literal;
|
||||||
|
}
|
||||||
|
else
|
||||||
|
{
|
||||||
|
if (checkAndGetLiteralArgument<String>(args[4], "format") == "auto")
|
||||||
|
args[4] = format_literal;
|
||||||
|
args.push_back(structure_literal);
|
||||||
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
/// s3(source, access_key_id, secret_access_key, format, structure, compression)
|
/// s3(source, access_key_id, secret_access_key, format, structure, compression) or
|
||||||
|
/// s3(source, access_key_id, secret_access_key, session_token, format, structure)
|
||||||
else if (count == 6)
|
else if (count == 6)
|
||||||
{
|
{
|
||||||
args[count - 2] = structure_literal;
|
auto fourth_arg = checkAndGetLiteralArgument<String>(args[3], "format/session_token");
|
||||||
|
if (fourth_arg == "auto" || FormatFactory::instance().exists(fourth_arg))
|
||||||
|
{
|
||||||
|
if (checkAndGetLiteralArgument<String>(args[3], "format") == "auto")
|
||||||
|
args[3] = format_literal;
|
||||||
|
if (checkAndGetLiteralArgument<String>(args[4], "structure") == "auto")
|
||||||
|
args[4] = structure_literal;
|
||||||
|
}
|
||||||
|
else
|
||||||
|
{
|
||||||
|
if (checkAndGetLiteralArgument<String>(args[4], "format") == "auto")
|
||||||
|
args[4] = format_literal;
|
||||||
|
if (checkAndGetLiteralArgument<String>(args[5], "format") == "auto")
|
||||||
|
args[5] = structure_literal;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
/// s3(source, access_key_id, secret_access_key, session_token, format, structure, compression_method)
|
||||||
|
else if (count == 7)
|
||||||
|
{
|
||||||
|
if (checkAndGetLiteralArgument<String>(args[4], "format") == "auto")
|
||||||
|
args[4] = format_literal;
|
||||||
|
if (checkAndGetLiteralArgument<String>(args[5], "format") == "auto")
|
||||||
|
args[5] = structure_literal;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
@ -16,6 +16,43 @@ public:
|
|||||||
|
|
||||||
static constexpr auto type_name = "s3";
|
static constexpr auto type_name = "s3";
|
||||||
static constexpr auto namespace_name = "bucket";
|
static constexpr auto namespace_name = "bucket";
|
||||||
|
/// All possible signatures for S3 storage with structure argument (for example for s3 table function).
|
||||||
|
static constexpr auto max_number_of_arguments_with_structure = 7;
|
||||||
|
static constexpr auto signatures_with_structure =
|
||||||
|
" - url\n"
|
||||||
|
" - url, NOSIGN\n"
|
||||||
|
" - url, format\n"
|
||||||
|
" - url, NOSIGN, format\n"
|
||||||
|
" - url, format, structure\n"
|
||||||
|
" - url, NOSIGN, format, structure\n"
|
||||||
|
" - url, format, structure, compression_method\n"
|
||||||
|
" - url, NOSIGN, format, structure, compression_method\n"
|
||||||
|
" - url, access_key_id, secret_access_key\n"
|
||||||
|
" - url, access_key_id, secret_access_key, session_token\n"
|
||||||
|
" - url, access_key_id, secret_access_key, format\n"
|
||||||
|
" - url, access_key_id, secret_access_key, session_token, format\n"
|
||||||
|
" - url, access_key_id, secret_access_key, format, structure\n"
|
||||||
|
" - url, access_key_id, secret_access_key, session_token, format, structure\n"
|
||||||
|
" - url, access_key_id, secret_access_key, format, structure, compression_method\n"
|
||||||
|
" - url, access_key_id, secret_access_key, session_token, format, structure, compression_method\n"
|
||||||
|
"All signatures supports optional headers (specified as `headers('name'='value', 'name2'='value2')`)";
|
||||||
|
|
||||||
|
/// All possible signatures for S3 storage without structure argument (for example for S3 table engine).
|
||||||
|
static constexpr auto max_number_of_arguments_without_structure = 6;
|
||||||
|
static constexpr auto signatures_without_structure =
|
||||||
|
" - url\n"
|
||||||
|
" - url, NOSIGN\n"
|
||||||
|
" - url, format\n"
|
||||||
|
" - url, NOSIGN, format\n"
|
||||||
|
" - url, format, compression_method\n"
|
||||||
|
" - url, NOSIGN, format, compression_method\n"
|
||||||
|
" - url, access_key_id, secret_access_key\n"
|
||||||
|
" - url, access_key_id, secret_access_key, session_token\n"
|
||||||
|
" - url, access_key_id, secret_access_key, format\n"
|
||||||
|
" - url, access_key_id, secret_access_key, session_token, format\n"
|
||||||
|
" - url, access_key_id, secret_access_key, format, compression_method\n"
|
||||||
|
" - url, access_key_id, secret_access_key, session_token, format, compression_method\n"
|
||||||
|
"All signatures supports optional headers (specified as `headers('name'='value', 'name2'='value2')`)";
|
||||||
|
|
||||||
StorageS3Configuration() = default;
|
StorageS3Configuration() = default;
|
||||||
StorageS3Configuration(const StorageS3Configuration & other);
|
StorageS3Configuration(const StorageS3Configuration & other);
|
||||||
@ -24,6 +61,9 @@ public:
|
|||||||
std::string getEngineName() const override { return url.storage_name; }
|
std::string getEngineName() const override { return url.storage_name; }
|
||||||
std::string getNamespaceType() const override { return namespace_name; }
|
std::string getNamespaceType() const override { return namespace_name; }
|
||||||
|
|
||||||
|
std::string getSignatures(bool with_structure = true) const { return with_structure ? signatures_with_structure : signatures_without_structure; }
|
||||||
|
size_t getMaxNumberOfArguments(bool with_structure = true) const { return with_structure ? max_number_of_arguments_with_structure : max_number_of_arguments_without_structure; }
|
||||||
|
|
||||||
Path getPath() const override { return url.key; }
|
Path getPath() const override { return url.key; }
|
||||||
void setPath(const Path & path) override { url.key = path; }
|
void setPath(const Path & path) override { url.key = path; }
|
||||||
|
|
||||||
@ -44,7 +84,7 @@ public:
|
|||||||
|
|
||||||
ObjectStoragePtr createObjectStorage(ContextPtr context, bool is_readonly) override;
|
ObjectStoragePtr createObjectStorage(ContextPtr context, bool is_readonly) override;
|
||||||
|
|
||||||
void addStructureAndFormatToArgs(
|
void addStructureAndFormatToArgsIfNeeded(
|
||||||
ASTs & args,
|
ASTs & args,
|
||||||
const String & structure,
|
const String & structure,
|
||||||
const String & format,
|
const String & format,
|
||||||
|
@ -180,7 +180,9 @@ public:
|
|||||||
virtual String getNamespace() const = 0;
|
virtual String getNamespace() const = 0;
|
||||||
|
|
||||||
virtual StorageObjectStorage::QuerySettings getQuerySettings(const ContextPtr &) const = 0;
|
virtual StorageObjectStorage::QuerySettings getQuerySettings(const ContextPtr &) const = 0;
|
||||||
virtual void addStructureAndFormatToArgs(
|
|
||||||
|
/// Add/replace structure and format arguments in the AST arguments if they have 'auto' values.
|
||||||
|
virtual void addStructureAndFormatToArgsIfNeeded(
|
||||||
ASTs & args, const String & structure_, const String & format_, ContextPtr context) = 0;
|
ASTs & args, const String & structure_, const String & format_, ContextPtr context) = 0;
|
||||||
|
|
||||||
bool withPartitionWildcard() const;
|
bool withPartitionWildcard() const;
|
||||||
|
@ -103,7 +103,7 @@ void StorageObjectStorageCluster::updateQueryToSendIfNeeded(
|
|||||||
|
|
||||||
ASTPtr cluster_name_arg = args.front();
|
ASTPtr cluster_name_arg = args.front();
|
||||||
args.erase(args.begin());
|
args.erase(args.begin());
|
||||||
configuration->addStructureAndFormatToArgs(args, structure, configuration->format, context);
|
configuration->addStructureAndFormatToArgsIfNeeded(args, structure, configuration->format, context);
|
||||||
args.insert(args.begin(), cluster_name_arg);
|
args.insert(args.begin(), cluster_name_arg);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -23,7 +23,6 @@ class ITableFunctionCluster : public Base
|
|||||||
{
|
{
|
||||||
public:
|
public:
|
||||||
String getName() const override = 0;
|
String getName() const override = 0;
|
||||||
String getSignature() const override = 0;
|
|
||||||
|
|
||||||
static void updateStructureAndFormatArgumentsIfNeeded(ASTs & args, const String & structure_, const String & format_, const ContextPtr & context)
|
static void updateStructureAndFormatArgumentsIfNeeded(ASTs & args, const String & structure_, const String & format_, const ContextPtr & context)
|
||||||
{
|
{
|
||||||
@ -46,7 +45,11 @@ protected:
|
|||||||
void parseArgumentsImpl(ASTs & args, const ContextPtr & context) override
|
void parseArgumentsImpl(ASTs & args, const ContextPtr & context) override
|
||||||
{
|
{
|
||||||
if (args.empty())
|
if (args.empty())
|
||||||
throw Exception(ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH, "The signature of table function {} shall be the following:\n{}", getName(), getSignature());
|
throw Exception(
|
||||||
|
ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH,
|
||||||
|
"The function {} should have arguments. The first argument must be the cluster name and the rest are the arguments of "
|
||||||
|
"corresponding table function",
|
||||||
|
getName());
|
||||||
|
|
||||||
/// Evaluate only first argument, everything else will be done Base class
|
/// Evaluate only first argument, everything else will be done Base class
|
||||||
args[0] = evaluateConstantExpressionOrIdentifierAsLiteral(args[0], context);
|
args[0] = evaluateConstantExpressionOrIdentifierAsLiteral(args[0], context);
|
||||||
|
@ -57,7 +57,7 @@ void ITableFunctionFileLike::parseArguments(const ASTPtr & ast_function, Context
|
|||||||
|
|
||||||
void ITableFunctionFileLike::parseArgumentsImpl(ASTs & args, const ContextPtr & context)
|
void ITableFunctionFileLike::parseArgumentsImpl(ASTs & args, const ContextPtr & context)
|
||||||
{
|
{
|
||||||
if (args.empty() || args.size() > 4)
|
if (args.empty() || args.size() > getMaxNumberOfArguments())
|
||||||
throw Exception(ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH, "The signature of table function {} shall be the following:\n{}", getName(), getSignature());
|
throw Exception(ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH, "The signature of table function {} shall be the following:\n{}", getName(), getSignature());
|
||||||
|
|
||||||
for (auto & arg : args)
|
for (auto & arg : args)
|
||||||
|
@ -15,6 +15,7 @@ class Context;
|
|||||||
class ITableFunctionFileLike : public ITableFunction
|
class ITableFunctionFileLike : public ITableFunction
|
||||||
{
|
{
|
||||||
public:
|
public:
|
||||||
|
static constexpr auto max_number_of_arguments = 4;
|
||||||
static constexpr auto signature = " - filename\n"
|
static constexpr auto signature = " - filename\n"
|
||||||
" - filename, format\n"
|
" - filename, format\n"
|
||||||
" - filename, format, structure\n"
|
" - filename, format, structure\n"
|
||||||
@ -32,7 +33,7 @@ public:
|
|||||||
|
|
||||||
NameSet getVirtualsToCheckBeforeUsingStructureHint() const override;
|
NameSet getVirtualsToCheckBeforeUsingStructureHint() const override;
|
||||||
|
|
||||||
static size_t getMaxNumberOfArguments() { return 4; }
|
static size_t getMaxNumberOfArguments() { return max_number_of_arguments; }
|
||||||
|
|
||||||
static void updateStructureAndFormatArgumentsIfNeeded(ASTs & args, const String & structure, const String & format, const ContextPtr &);
|
static void updateStructureAndFormatArgumentsIfNeeded(ASTs & args, const String & structure, const String & format, const ContextPtr &);
|
||||||
|
|
||||||
|
@ -22,72 +22,36 @@ struct AzureDefinition
|
|||||||
{
|
{
|
||||||
static constexpr auto name = "azureBlobStorage";
|
static constexpr auto name = "azureBlobStorage";
|
||||||
static constexpr auto storage_type_name = "Azure";
|
static constexpr auto storage_type_name = "Azure";
|
||||||
static constexpr auto signature = " - connection_string, container_name, blobpath\n"
|
|
||||||
" - connection_string, container_name, blobpath, structure \n"
|
|
||||||
" - connection_string, container_name, blobpath, format \n"
|
|
||||||
" - connection_string, container_name, blobpath, format, compression \n"
|
|
||||||
" - connection_string, container_name, blobpath, format, compression, structure \n"
|
|
||||||
" - storage_account_url, container_name, blobpath, account_name, account_key\n"
|
|
||||||
" - storage_account_url, container_name, blobpath, account_name, account_key, structure\n"
|
|
||||||
" - storage_account_url, container_name, blobpath, account_name, account_key, format\n"
|
|
||||||
" - storage_account_url, container_name, blobpath, account_name, account_key, format, compression\n"
|
|
||||||
" - storage_account_url, container_name, blobpath, account_name, account_key, format, compression, structure\n";
|
|
||||||
static constexpr auto max_number_of_arguments = 8;
|
|
||||||
};
|
};
|
||||||
|
|
||||||
struct S3Definition
|
struct S3Definition
|
||||||
{
|
{
|
||||||
static constexpr auto name = "s3";
|
static constexpr auto name = "s3";
|
||||||
static constexpr auto storage_type_name = "S3";
|
static constexpr auto storage_type_name = "S3";
|
||||||
static constexpr auto signature = " - url\n"
|
|
||||||
" - url, format\n"
|
|
||||||
" - url, format, structure\n"
|
|
||||||
" - url, format, structure, compression_method\n"
|
|
||||||
" - url, access_key_id, secret_access_key\n"
|
|
||||||
" - url, access_key_id, secret_access_key, session_token\n"
|
|
||||||
" - url, access_key_id, secret_access_key, format\n"
|
|
||||||
" - url, access_key_id, secret_access_key, session_token, format\n"
|
|
||||||
" - url, access_key_id, secret_access_key, format, structure\n"
|
|
||||||
" - url, access_key_id, secret_access_key, session_token, format, structure\n"
|
|
||||||
" - url, access_key_id, secret_access_key, format, structure, compression_method\n"
|
|
||||||
" - url, access_key_id, secret_access_key, session_token, format, structure, compression_method\n"
|
|
||||||
"All signatures supports optional headers (specified as `headers('name'='value', 'name2'='value2')`)";
|
|
||||||
static constexpr auto max_number_of_arguments = 8;
|
|
||||||
};
|
};
|
||||||
|
|
||||||
struct GCSDefinition
|
struct GCSDefinition
|
||||||
{
|
{
|
||||||
static constexpr auto name = "gcs";
|
static constexpr auto name = "gcs";
|
||||||
static constexpr auto storage_type_name = "GCS";
|
static constexpr auto storage_type_name = "GCS";
|
||||||
static constexpr auto signature = S3Definition::signature;
|
|
||||||
static constexpr auto max_number_of_arguments = S3Definition::max_number_of_arguments;
|
|
||||||
};
|
};
|
||||||
|
|
||||||
struct COSNDefinition
|
struct COSNDefinition
|
||||||
{
|
{
|
||||||
static constexpr auto name = "cosn";
|
static constexpr auto name = "cosn";
|
||||||
static constexpr auto storage_type_name = "COSN";
|
static constexpr auto storage_type_name = "COSN";
|
||||||
static constexpr auto signature = S3Definition::signature;
|
|
||||||
static constexpr auto max_number_of_arguments = S3Definition::max_number_of_arguments;
|
|
||||||
};
|
};
|
||||||
|
|
||||||
struct OSSDefinition
|
struct OSSDefinition
|
||||||
{
|
{
|
||||||
static constexpr auto name = "oss";
|
static constexpr auto name = "oss";
|
||||||
static constexpr auto storage_type_name = "OSS";
|
static constexpr auto storage_type_name = "OSS";
|
||||||
static constexpr auto signature = S3Definition::signature;
|
|
||||||
static constexpr auto max_number_of_arguments = S3Definition::max_number_of_arguments;
|
|
||||||
};
|
};
|
||||||
|
|
||||||
struct HDFSDefinition
|
struct HDFSDefinition
|
||||||
{
|
{
|
||||||
static constexpr auto name = "hdfs";
|
static constexpr auto name = "hdfs";
|
||||||
static constexpr auto storage_type_name = "HDFS";
|
static constexpr auto storage_type_name = "HDFS";
|
||||||
static constexpr auto signature = " - uri\n"
|
|
||||||
" - uri, format\n"
|
|
||||||
" - uri, format, structure\n"
|
|
||||||
" - uri, format, structure, compression_method\n";
|
|
||||||
static constexpr auto max_number_of_arguments = 4;
|
|
||||||
};
|
};
|
||||||
|
|
||||||
template <typename Definition, typename Configuration>
|
template <typename Definition, typename Configuration>
|
||||||
@ -95,14 +59,9 @@ class TableFunctionObjectStorage : public ITableFunction
|
|||||||
{
|
{
|
||||||
public:
|
public:
|
||||||
static constexpr auto name = Definition::name;
|
static constexpr auto name = Definition::name;
|
||||||
static constexpr auto signature = Definition::signature;
|
|
||||||
|
|
||||||
static size_t getMaxNumberOfArguments() { return Definition::max_number_of_arguments; }
|
|
||||||
|
|
||||||
String getName() const override { return name; }
|
String getName() const override { return name; }
|
||||||
|
|
||||||
virtual String getSignature() const { return signature; }
|
|
||||||
|
|
||||||
bool hasStaticStructure() const override { return configuration->structure != "auto"; }
|
bool hasStaticStructure() const override { return configuration->structure != "auto"; }
|
||||||
|
|
||||||
bool needStructureHint() const override { return configuration->structure == "auto"; }
|
bool needStructureHint() const override { return configuration->structure == "auto"; }
|
||||||
@ -130,7 +89,7 @@ public:
|
|||||||
const String & format,
|
const String & format,
|
||||||
const ContextPtr & context)
|
const ContextPtr & context)
|
||||||
{
|
{
|
||||||
Configuration().addStructureAndFormatToArgs(args, structure, format, context);
|
Configuration().addStructureAndFormatToArgsIfNeeded(args, structure, format, context);
|
||||||
}
|
}
|
||||||
|
|
||||||
protected:
|
protected:
|
||||||
|
@ -19,40 +19,22 @@ struct AzureClusterDefinition
|
|||||||
{
|
{
|
||||||
static constexpr auto name = "azureBlobStorageCluster";
|
static constexpr auto name = "azureBlobStorageCluster";
|
||||||
static constexpr auto storage_type_name = "AzureBlobStorageCluster";
|
static constexpr auto storage_type_name = "AzureBlobStorageCluster";
|
||||||
static constexpr auto signature = " - cluster, connection_string|storage_account_url, container_name, blobpath, [account_name, account_key, format, compression, structure]";
|
|
||||||
static constexpr auto max_number_of_arguments = AzureDefinition::max_number_of_arguments + 1;
|
|
||||||
};
|
};
|
||||||
|
|
||||||
struct S3ClusterDefinition
|
struct S3ClusterDefinition
|
||||||
{
|
{
|
||||||
static constexpr auto name = "s3Cluster";
|
static constexpr auto name = "s3Cluster";
|
||||||
static constexpr auto storage_type_name = "S3Cluster";
|
static constexpr auto storage_type_name = "S3Cluster";
|
||||||
static constexpr auto signature = " - cluster, url\n"
|
|
||||||
" - cluster, url, format\n"
|
|
||||||
" - cluster, url, format, structure\n"
|
|
||||||
" - cluster, url, access_key_id, secret_access_key\n"
|
|
||||||
" - cluster, url, format, structure, compression_method\n"
|
|
||||||
" - cluster, url, access_key_id, secret_access_key, format\n"
|
|
||||||
" - cluster, url, access_key_id, secret_access_key, format, structure\n"
|
|
||||||
" - cluster, url, access_key_id, secret_access_key, format, structure, compression_method\n"
|
|
||||||
" - cluster, url, access_key_id, secret_access_key, session_token, format, structure, compression_method\n"
|
|
||||||
"All signatures supports optional headers (specified as `headers('name'='value', 'name2'='value2')`)";
|
|
||||||
static constexpr auto max_number_of_arguments = S3Definition::max_number_of_arguments + 1;
|
|
||||||
};
|
};
|
||||||
|
|
||||||
struct HDFSClusterDefinition
|
struct HDFSClusterDefinition
|
||||||
{
|
{
|
||||||
static constexpr auto name = "hdfsCluster";
|
static constexpr auto name = "hdfsCluster";
|
||||||
static constexpr auto storage_type_name = "HDFSCluster";
|
static constexpr auto storage_type_name = "HDFSCluster";
|
||||||
static constexpr auto signature = " - cluster_name, uri\n"
|
|
||||||
" - cluster_name, uri, format\n"
|
|
||||||
" - cluster_name, uri, format, structure\n"
|
|
||||||
" - cluster_name, uri, format, structure, compression_method\n";
|
|
||||||
static constexpr auto max_number_of_arguments = HDFSDefinition::max_number_of_arguments + 1;
|
|
||||||
};
|
};
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Class implementing s3/hdfs/azureBlobStorage)Cluster(...) table functions,
|
* Class implementing s3/hdfs/azureBlobStorageCluster(...) table functions,
|
||||||
* which allow to process many files from S3/HDFS/Azure blob storage on a specific cluster.
|
* which allow to process many files from S3/HDFS/Azure blob storage on a specific cluster.
|
||||||
* On initiator it creates a connection to _all_ nodes in cluster, discloses asterisks
|
* On initiator it creates a connection to _all_ nodes in cluster, discloses asterisks
|
||||||
* in file path and dispatch each file dynamically.
|
* in file path and dispatch each file dynamically.
|
||||||
@ -64,10 +46,8 @@ class TableFunctionObjectStorageCluster : public ITableFunctionCluster<TableFunc
|
|||||||
{
|
{
|
||||||
public:
|
public:
|
||||||
static constexpr auto name = Definition::name;
|
static constexpr auto name = Definition::name;
|
||||||
static constexpr auto signature = Definition::signature;
|
|
||||||
|
|
||||||
String getName() const override { return name; }
|
String getName() const override { return name; }
|
||||||
String getSignature() const override { return signature; }
|
|
||||||
|
|
||||||
protected:
|
protected:
|
||||||
using Base = TableFunctionObjectStorage<Definition, Configuration>;
|
using Base = TableFunctionObjectStorage<Definition, Configuration>;
|
||||||
|
@ -6,5 +6,12 @@
|
|||||||
<secret_access_key>minio123</secret_access_key>
|
<secret_access_key>minio123</secret_access_key>
|
||||||
<format>CSV</format>>
|
<format>CSV</format>>
|
||||||
</test_s3>
|
</test_s3>
|
||||||
|
<test_s3_with_default>
|
||||||
|
<url>http://minio1:9001/root/data/data{1,2,3}</url>
|
||||||
|
<access_key_id>minio</access_key_id>
|
||||||
|
<secret_access_key>minio123</secret_access_key>
|
||||||
|
<format>JSONEachRow</format>>
|
||||||
|
<structure>id UInt32, date Date DEFAULT 18262</structure>
|
||||||
|
</test_s3_with_default>
|
||||||
</named_collections>
|
</named_collections>
|
||||||
</clickhouse>
|
</clickhouse>
|
||||||
|
@ -459,3 +459,51 @@ def test_cluster_format_detection(started_cluster):
|
|||||||
)
|
)
|
||||||
|
|
||||||
assert result == expected_result
|
assert result == expected_result
|
||||||
|
|
||||||
|
|
||||||
|
def test_cluster_default_expression(started_cluster):
|
||||||
|
node = started_cluster.instances["s0_0_0"]
|
||||||
|
|
||||||
|
node.query(
|
||||||
|
"insert into function s3('http://minio1:9001/root/data/data1', 'minio', 'minio123', JSONEachRow) select 1 as id settings s3_truncate_on_insert=1"
|
||||||
|
)
|
||||||
|
node.query(
|
||||||
|
"insert into function s3('http://minio1:9001/root/data/data2', 'minio', 'minio123', JSONEachRow) select * from numbers(0) settings s3_truncate_on_insert=1"
|
||||||
|
)
|
||||||
|
node.query(
|
||||||
|
"insert into function s3('http://minio1:9001/root/data/data3', 'minio', 'minio123', JSONEachRow) select 2 as id settings s3_truncate_on_insert=1"
|
||||||
|
)
|
||||||
|
|
||||||
|
expected_result = node.query(
|
||||||
|
"SELECT * FROM s3('http://minio1:9001/root/data/data{1,2,3}', 'minio', 'minio123', 'JSONEachRow', 'id UInt32, date Date DEFAULT 18262') order by id"
|
||||||
|
)
|
||||||
|
|
||||||
|
result = node.query(
|
||||||
|
"SELECT * FROM s3Cluster(cluster_simple, 'http://minio1:9001/root/data/data{1,2,3}', 'minio', 'minio123', 'JSONEachRow', 'id UInt32, date Date DEFAULT 18262') order by id"
|
||||||
|
)
|
||||||
|
|
||||||
|
assert result == expected_result
|
||||||
|
|
||||||
|
result = node.query(
|
||||||
|
"SELECT * FROM s3Cluster(cluster_simple, 'http://minio1:9001/root/data/data{1,2,3}', 'minio', 'minio123', 'auto', 'id UInt32, date Date DEFAULT 18262') order by id"
|
||||||
|
)
|
||||||
|
|
||||||
|
assert result == expected_result
|
||||||
|
|
||||||
|
result = node.query(
|
||||||
|
"SELECT * FROM s3Cluster(cluster_simple, 'http://minio1:9001/root/data/data{1,2,3}', 'minio', 'minio123', 'JSONEachRow', 'id UInt32, date Date DEFAULT 18262', 'auto') order by id"
|
||||||
|
)
|
||||||
|
|
||||||
|
assert result == expected_result
|
||||||
|
|
||||||
|
result = node.query(
|
||||||
|
"SELECT * FROM s3Cluster(cluster_simple, 'http://minio1:9001/root/data/data{1,2,3}', 'minio', 'minio123', 'auto', 'id UInt32, date Date DEFAULT 18262', 'auto') order by id"
|
||||||
|
)
|
||||||
|
|
||||||
|
assert result == expected_result
|
||||||
|
|
||||||
|
result = node.query(
|
||||||
|
"SELECT * FROM s3Cluster(cluster_simple, test_s3_with_default) order by id"
|
||||||
|
)
|
||||||
|
|
||||||
|
assert result == expected_result
|
||||||
|
@ -190,3 +190,195 @@
|
|||||||
20 21 22
|
20 21 22
|
||||||
23 24 25
|
23 24 25
|
||||||
26 27 28
|
26 27 28
|
||||||
|
0 0 0
|
||||||
|
0 0 0
|
||||||
|
0 0 0
|
||||||
|
1 2 3
|
||||||
|
4 5 6
|
||||||
|
7 8 9
|
||||||
|
10 11 12
|
||||||
|
13 14 15
|
||||||
|
16 17 18
|
||||||
|
20 21 22
|
||||||
|
23 24 25
|
||||||
|
26 27 28
|
||||||
|
0 0 0
|
||||||
|
0 0 0
|
||||||
|
0 0 0
|
||||||
|
1 2 3
|
||||||
|
4 5 6
|
||||||
|
7 8 9
|
||||||
|
10 11 12
|
||||||
|
13 14 15
|
||||||
|
16 17 18
|
||||||
|
20 21 22
|
||||||
|
23 24 25
|
||||||
|
26 27 28
|
||||||
|
0 0 0
|
||||||
|
0 0 0
|
||||||
|
0 0 0
|
||||||
|
1 2 3
|
||||||
|
4 5 6
|
||||||
|
7 8 9
|
||||||
|
10 11 12
|
||||||
|
13 14 15
|
||||||
|
16 17 18
|
||||||
|
20 21 22
|
||||||
|
23 24 25
|
||||||
|
26 27 28
|
||||||
|
0 0 0
|
||||||
|
0 0 0
|
||||||
|
0 0 0
|
||||||
|
1 2 3
|
||||||
|
4 5 6
|
||||||
|
7 8 9
|
||||||
|
10 11 12
|
||||||
|
13 14 15
|
||||||
|
16 17 18
|
||||||
|
20 21 22
|
||||||
|
23 24 25
|
||||||
|
26 27 28
|
||||||
|
0 0 0
|
||||||
|
0 0 0
|
||||||
|
0 0 0
|
||||||
|
1 2 3
|
||||||
|
4 5 6
|
||||||
|
7 8 9
|
||||||
|
10 11 12
|
||||||
|
13 14 15
|
||||||
|
16 17 18
|
||||||
|
20 21 22
|
||||||
|
23 24 25
|
||||||
|
26 27 28
|
||||||
|
0 0 0
|
||||||
|
0 0 0
|
||||||
|
0 0 0
|
||||||
|
1 2 3
|
||||||
|
4 5 6
|
||||||
|
7 8 9
|
||||||
|
10 11 12
|
||||||
|
13 14 15
|
||||||
|
16 17 18
|
||||||
|
20 21 22
|
||||||
|
23 24 25
|
||||||
|
26 27 28
|
||||||
|
0 0 0
|
||||||
|
0 0 0
|
||||||
|
0 0 0
|
||||||
|
1 2 3
|
||||||
|
4 5 6
|
||||||
|
7 8 9
|
||||||
|
10 11 12
|
||||||
|
13 14 15
|
||||||
|
16 17 18
|
||||||
|
20 21 22
|
||||||
|
23 24 25
|
||||||
|
26 27 28
|
||||||
|
0 0 0
|
||||||
|
0 0 0
|
||||||
|
0 0 0
|
||||||
|
1 2 3
|
||||||
|
4 5 6
|
||||||
|
7 8 9
|
||||||
|
10 11 12
|
||||||
|
13 14 15
|
||||||
|
16 17 18
|
||||||
|
20 21 22
|
||||||
|
23 24 25
|
||||||
|
26 27 28
|
||||||
|
0 0 0
|
||||||
|
0 0 0
|
||||||
|
0 0 0
|
||||||
|
1 2 3
|
||||||
|
4 5 6
|
||||||
|
7 8 9
|
||||||
|
10 11 12
|
||||||
|
13 14 15
|
||||||
|
16 17 18
|
||||||
|
20 21 22
|
||||||
|
23 24 25
|
||||||
|
26 27 28
|
||||||
|
0 0 0
|
||||||
|
0 0 0
|
||||||
|
0 0 0
|
||||||
|
1 2 3
|
||||||
|
4 5 6
|
||||||
|
7 8 9
|
||||||
|
10 11 12
|
||||||
|
13 14 15
|
||||||
|
16 17 18
|
||||||
|
20 21 22
|
||||||
|
23 24 25
|
||||||
|
26 27 28
|
||||||
|
0 0 0
|
||||||
|
0 0 0
|
||||||
|
0 0 0
|
||||||
|
1 2 3
|
||||||
|
4 5 6
|
||||||
|
7 8 9
|
||||||
|
10 11 12
|
||||||
|
13 14 15
|
||||||
|
16 17 18
|
||||||
|
20 21 22
|
||||||
|
23 24 25
|
||||||
|
26 27 28
|
||||||
|
0 0 0
|
||||||
|
0 0 0
|
||||||
|
0 0 0
|
||||||
|
1 2 3
|
||||||
|
4 5 6
|
||||||
|
7 8 9
|
||||||
|
10 11 12
|
||||||
|
13 14 15
|
||||||
|
16 17 18
|
||||||
|
20 21 22
|
||||||
|
23 24 25
|
||||||
|
26 27 28
|
||||||
|
0 0 0
|
||||||
|
0 0 0
|
||||||
|
0 0 0
|
||||||
|
1 2 3
|
||||||
|
4 5 6
|
||||||
|
7 8 9
|
||||||
|
10 11 12
|
||||||
|
13 14 15
|
||||||
|
16 17 18
|
||||||
|
20 21 22
|
||||||
|
23 24 25
|
||||||
|
26 27 28
|
||||||
|
0 0 0
|
||||||
|
0 0 0
|
||||||
|
0 0 0
|
||||||
|
1 2 3
|
||||||
|
4 5 6
|
||||||
|
7 8 9
|
||||||
|
10 11 12
|
||||||
|
13 14 15
|
||||||
|
16 17 18
|
||||||
|
20 21 22
|
||||||
|
23 24 25
|
||||||
|
26 27 28
|
||||||
|
0 0 0
|
||||||
|
0 0 0
|
||||||
|
0 0 0
|
||||||
|
1 2 3
|
||||||
|
4 5 6
|
||||||
|
7 8 9
|
||||||
|
10 11 12
|
||||||
|
13 14 15
|
||||||
|
16 17 18
|
||||||
|
20 21 22
|
||||||
|
23 24 25
|
||||||
|
26 27 28
|
||||||
|
0 0 0
|
||||||
|
0 0 0
|
||||||
|
0 0 0
|
||||||
|
1 2 3
|
||||||
|
4 5 6
|
||||||
|
7 8 9
|
||||||
|
10 11 12
|
||||||
|
13 14 15
|
||||||
|
16 17 18
|
||||||
|
20 21 22
|
||||||
|
23 24 25
|
||||||
|
26 27 28
|
||||||
|
@ -2,21 +2,37 @@
|
|||||||
-- Tag no-fasttest: Depends on AWS
|
-- Tag no-fasttest: Depends on AWS
|
||||||
|
|
||||||
select * from s3('http://localhost:11111/test/{a,b,c}.tsv') ORDER BY c1, c2, c3;
|
select * from s3('http://localhost:11111/test/{a,b,c}.tsv') ORDER BY c1, c2, c3;
|
||||||
|
select * from s3('http://localhost:11111/test/{a,b,c}.tsv', NOSIGN) ORDER BY c1, c2, c3;
|
||||||
select * from s3('http://localhost:11111/test/{a,b,c}.tsv', 'TSV') ORDER BY c1, c2, c3;
|
select * from s3('http://localhost:11111/test/{a,b,c}.tsv', 'TSV') ORDER BY c1, c2, c3;
|
||||||
|
select * from s3('http://localhost:11111/test/{a,b,c}.tsv', NOSIGN, 'TSV') ORDER BY c1, c2, c3;
|
||||||
select * from s3('http://localhost:11111/test/{a,b,c}.tsv', 'TSV', 'c1 UInt64, c2 UInt64, c3 UInt64') ORDER BY c1, c2, c3;
|
select * from s3('http://localhost:11111/test/{a,b,c}.tsv', 'TSV', 'c1 UInt64, c2 UInt64, c3 UInt64') ORDER BY c1, c2, c3;
|
||||||
select * from s3('http://localhost:11111/test/{a,b,c}.tsv', 'test', 'testtest') ORDER BY c1, c2, c3;
|
select * from s3('http://localhost:11111/test/{a,b,c}.tsv', NOSIGN, 'TSV', 'c1 UInt64, c2 UInt64, c3 UInt64') ORDER BY c1, c2, c3;
|
||||||
select * from s3('http://localhost:11111/test/{a,b,c}.tsv', 'TSV', 'c1 UInt64, c2 UInt64, c3 UInt64', 'auto') ORDER BY c1, c2, c3;
|
select * from s3('http://localhost:11111/test/{a,b,c}.tsv', 'TSV', 'c1 UInt64, c2 UInt64, c3 UInt64', 'auto') ORDER BY c1, c2, c3;
|
||||||
|
select * from s3('http://localhost:11111/test/{a,b,c}.tsv', NOSIGN, 'TSV', 'c1 UInt64, c2 UInt64, c3 UInt64', 'auto') ORDER BY c1, c2, c3;
|
||||||
|
select * from s3('http://localhost:11111/test/{a,b,c}.tsv', 'test', 'testtest') ORDER BY c1, c2, c3;
|
||||||
|
select * from s3('http://localhost:11111/test/{a,b,c}.tsv', 'test', 'testtest', '') ORDER BY c1, c2, c3;
|
||||||
select * from s3('http://localhost:11111/test/{a,b,c}.tsv', 'test', 'testtest', 'TSV') ORDER BY c1, c2, c3;
|
select * from s3('http://localhost:11111/test/{a,b,c}.tsv', 'test', 'testtest', 'TSV') ORDER BY c1, c2, c3;
|
||||||
|
select * from s3('http://localhost:11111/test/{a,b,c}.tsv', 'test', 'testtest', '', 'TSV') ORDER BY c1, c2, c3;
|
||||||
select * from s3('http://localhost:11111/test/{a,b,c}.tsv', 'test', 'testtest', 'TSV', 'c1 UInt64, c2 UInt64, c3 UInt64') ORDER BY c1, c2, c3;
|
select * from s3('http://localhost:11111/test/{a,b,c}.tsv', 'test', 'testtest', 'TSV', 'c1 UInt64, c2 UInt64, c3 UInt64') ORDER BY c1, c2, c3;
|
||||||
|
select * from s3('http://localhost:11111/test/{a,b,c}.tsv', 'test', 'testtest', '', 'TSV', 'c1 UInt64, c2 UInt64, c3 UInt64') ORDER BY c1, c2, c3;
|
||||||
select * from s3('http://localhost:11111/test/{a,b,c}.tsv', 'test', 'testtest', 'TSV', 'c1 UInt64, c2 UInt64, c3 UInt64', 'auto') ORDER BY c1, c2, c3;
|
select * from s3('http://localhost:11111/test/{a,b,c}.tsv', 'test', 'testtest', 'TSV', 'c1 UInt64, c2 UInt64, c3 UInt64', 'auto') ORDER BY c1, c2, c3;
|
||||||
|
select * from s3('http://localhost:11111/test/{a,b,c}.tsv', 'test', 'testtest', '', 'TSV', 'c1 UInt64, c2 UInt64, c3 UInt64', 'auto') ORDER BY c1, c2, c3;
|
||||||
|
|
||||||
|
|
||||||
select * from s3Cluster('test_cluster_two_shards_localhost', 'http://localhost:11111/test/{a,b,c}.tsv') ORDER BY c1, c2, c3;
|
select * from s3Cluster('test_cluster_two_shards_localhost', 'http://localhost:11111/test/{a,b,c}.tsv') ORDER BY c1, c2, c3;
|
||||||
|
select * from s3Cluster('test_cluster_two_shards_localhost', 'http://localhost:11111/test/{a,b,c}.tsv', NOSIGN) ORDER BY c1, c2, c3;
|
||||||
select * from s3Cluster('test_cluster_two_shards_localhost', 'http://localhost:11111/test/{a,b,c}.tsv', 'TSV') ORDER BY c1, c2, c3;
|
select * from s3Cluster('test_cluster_two_shards_localhost', 'http://localhost:11111/test/{a,b,c}.tsv', 'TSV') ORDER BY c1, c2, c3;
|
||||||
|
select * from s3Cluster('test_cluster_two_shards_localhost', 'http://localhost:11111/test/{a,b,c}.tsv', NOSIGN, 'TSV') ORDER BY c1, c2, c3;
|
||||||
select * from s3Cluster('test_cluster_two_shards_localhost', 'http://localhost:11111/test/{a,b,c}.tsv', 'TSV', 'c1 UInt64, c2 UInt64, c3 UInt64') ORDER BY c1, c2, c3;
|
select * from s3Cluster('test_cluster_two_shards_localhost', 'http://localhost:11111/test/{a,b,c}.tsv', 'TSV', 'c1 UInt64, c2 UInt64, c3 UInt64') ORDER BY c1, c2, c3;
|
||||||
select * from s3Cluster('test_cluster_two_shards_localhost', 'http://localhost:11111/test/{a,b,c}.tsv', 'test', 'testtest') ORDER BY c1, c2, c3;
|
select * from s3Cluster('test_cluster_two_shards_localhost', 'http://localhost:11111/test/{a,b,c}.tsv', NOSIGN, 'TSV', 'c1 UInt64, c2 UInt64, c3 UInt64') ORDER BY c1, c2, c3;
|
||||||
select * from s3Cluster('test_cluster_two_shards_localhost', 'http://localhost:11111/test/{a,b,c}.tsv', 'TSV', 'c1 UInt64, c2 UInt64, c3 UInt64', 'auto') ORDER BY c1, c2, c3;
|
select * from s3Cluster('test_cluster_two_shards_localhost', 'http://localhost:11111/test/{a,b,c}.tsv', 'TSV', 'c1 UInt64, c2 UInt64, c3 UInt64', 'auto') ORDER BY c1, c2, c3;
|
||||||
|
select * from s3Cluster('test_cluster_two_shards_localhost', 'http://localhost:11111/test/{a,b,c}.tsv', NOSIGN, 'TSV', 'c1 UInt64, c2 UInt64, c3 UInt64', 'auto') ORDER BY c1, c2, c3;
|
||||||
|
select * from s3Cluster('test_cluster_two_shards_localhost', 'http://localhost:11111/test/{a,b,c}.tsv', 'test', 'testtest') ORDER BY c1, c2, c3;
|
||||||
|
select * from s3Cluster('test_cluster_two_shards_localhost', 'http://localhost:11111/test/{a,b,c}.tsv', 'test', 'testtest', '') ORDER BY c1, c2, c3;
|
||||||
select * from s3Cluster('test_cluster_two_shards_localhost', 'http://localhost:11111/test/{a,b,c}.tsv', 'test', 'testtest', 'TSV') ORDER BY c1, c2, c3;
|
select * from s3Cluster('test_cluster_two_shards_localhost', 'http://localhost:11111/test/{a,b,c}.tsv', 'test', 'testtest', 'TSV') ORDER BY c1, c2, c3;
|
||||||
|
select * from s3Cluster('test_cluster_two_shards_localhost', 'http://localhost:11111/test/{a,b,c}.tsv', 'test', 'testtest', '', 'TSV') ORDER BY c1, c2, c3;
|
||||||
select * from s3Cluster('test_cluster_two_shards_localhost', 'http://localhost:11111/test/{a,b,c}.tsv', 'test', 'testtest', 'TSV', 'c1 UInt64, c2 UInt64, c3 UInt64') ORDER BY c1, c2, c3;
|
select * from s3Cluster('test_cluster_two_shards_localhost', 'http://localhost:11111/test/{a,b,c}.tsv', 'test', 'testtest', 'TSV', 'c1 UInt64, c2 UInt64, c3 UInt64') ORDER BY c1, c2, c3;
|
||||||
|
select * from s3Cluster('test_cluster_two_shards_localhost', 'http://localhost:11111/test/{a,b,c}.tsv', 'test', 'testtest', '', 'TSV', 'c1 UInt64, c2 UInt64, c3 UInt64') ORDER BY c1, c2, c3;
|
||||||
select * from s3Cluster('test_cluster_two_shards_localhost', 'http://localhost:11111/test/{a,b,c}.tsv', 'test', 'testtest', 'TSV', 'c1 UInt64, c2 UInt64, c3 UInt64', 'auto') ORDER BY c1, c2, c3;
|
select * from s3Cluster('test_cluster_two_shards_localhost', 'http://localhost:11111/test/{a,b,c}.tsv', 'test', 'testtest', 'TSV', 'c1 UInt64, c2 UInt64, c3 UInt64', 'auto') ORDER BY c1, c2, c3;
|
||||||
|
select * from s3Cluster('test_cluster_two_shards_localhost', 'http://localhost:11111/test/{a,b,c}.tsv', 'test', 'testtest', '', 'TSV', 'c1 UInt64, c2 UInt64, c3 UInt64', 'auto') ORDER BY c1, c2, c3;
|
||||||
|
|
||||||
|
Loading…
Reference in New Issue
Block a user