mirror of
https://github.com/ClickHouse/ClickHouse.git
synced 2024-09-19 16:20:50 +00:00
Fix propogating structure argument in s3Cluster
This commit is contained in:
parent
a7d0a5991e
commit
d7aaf053f9
@ -272,16 +272,25 @@ void StorageAzureConfiguration::fromAST(ASTs & engine_args, ContextPtr context,
|
||||
connection_params = getConnectionParams(connection_url, container_name, account_name, account_key, context);
|
||||
}
|
||||
|
||||
void StorageAzureConfiguration::addStructureAndFormatToArgs(
|
||||
void StorageAzureConfiguration::addStructureAndFormatToArgsIfNeeded(
|
||||
ASTs & args, const String & structure_, const String & format_, ContextPtr context)
|
||||
{
|
||||
if (tryGetNamedCollectionWithOverrides(args, context))
|
||||
if (auto collection = tryGetNamedCollectionWithOverrides(args, context))
|
||||
{
|
||||
/// In case of named collection, just add key-value pair "structure='...'"
|
||||
/// at the end of arguments to override existed structure.
|
||||
ASTs equal_func_args = {std::make_shared<ASTIdentifier>("structure"), std::make_shared<ASTLiteral>(structure_)};
|
||||
auto equal_func = makeASTFunction("equals", std::move(equal_func_args));
|
||||
args.push_back(equal_func);
|
||||
/// In case of named collection, just add key-value pairs "format='...', structure='...'"
|
||||
/// at the end of arguments to override existed format and structure with "auto" values.
|
||||
if (collection->getOrDefault<String>("format", "auto") == "auto")
|
||||
{
|
||||
ASTs format_equal_func_args = {std::make_shared<ASTIdentifier>("format"), std::make_shared<ASTLiteral>(format_)};
|
||||
auto format_equal_func = makeASTFunction("equals", std::move(format_equal_func_args));
|
||||
args.push_back(format_equal_func);
|
||||
}
|
||||
if (collection->getOrDefault<String>("structure", "auto") == "auto")
|
||||
{
|
||||
ASTs structure_equal_func_args = {std::make_shared<ASTIdentifier>("structure"), std::make_shared<ASTLiteral>(structure_)};
|
||||
auto structure_equal_func = makeASTFunction("equals", std::move(structure_equal_func_args));
|
||||
args.push_back(structure_equal_func);
|
||||
}
|
||||
}
|
||||
else
|
||||
{
|
||||
|
@ -44,7 +44,7 @@ public:
|
||||
|
||||
ObjectStoragePtr createObjectStorage(ContextPtr context, bool is_readonly) override;
|
||||
|
||||
void addStructureAndFormatToArgs(
|
||||
void addStructureAndFormatToArgsIfNeeded(
|
||||
ASTs & args,
|
||||
const String & structure_,
|
||||
const String & format_,
|
||||
|
@ -158,19 +158,28 @@ void StorageHDFSConfiguration::setURL(const std::string & url_)
|
||||
LOG_TRACE(getLogger("StorageHDFSConfiguration"), "Using URL: {}, path: {}", url, path);
|
||||
}
|
||||
|
||||
void StorageHDFSConfiguration::addStructureAndFormatToArgs(
|
||||
void StorageHDFSConfiguration::addStructureAndFormatToArgsIfNeeded(
|
||||
ASTs & args,
|
||||
const String & structure_,
|
||||
const String & format_,
|
||||
ContextPtr context)
|
||||
{
|
||||
if (tryGetNamedCollectionWithOverrides(args, context))
|
||||
if (auto collection = tryGetNamedCollectionWithOverrides(args, context))
|
||||
{
|
||||
/// In case of named collection, just add key-value pair "structure='...'"
|
||||
/// at the end of arguments to override existed structure.
|
||||
ASTs equal_func_args = {std::make_shared<ASTIdentifier>("structure"), std::make_shared<ASTLiteral>(structure_)};
|
||||
auto equal_func = makeASTFunction("equals", std::move(equal_func_args));
|
||||
args.push_back(equal_func);
|
||||
/// In case of named collection, just add key-value pairs "format='...', structure='...'"
|
||||
/// at the end of arguments to override existed format and structure with "auto" values.
|
||||
if (collection->getOrDefault<String>("format", "auto") == "auto")
|
||||
{
|
||||
ASTs format_equal_func_args = {std::make_shared<ASTIdentifier>("format"), std::make_shared<ASTLiteral>(format_)};
|
||||
auto format_equal_func = makeASTFunction("equals", std::move(format_equal_func_args));
|
||||
args.push_back(format_equal_func);
|
||||
}
|
||||
if (collection->getOrDefault<String>("structure", "auto") == "auto")
|
||||
{
|
||||
ASTs structure_equal_func_args = {std::make_shared<ASTIdentifier>("structure"), std::make_shared<ASTLiteral>(structure_)};
|
||||
auto structure_equal_func = makeASTFunction("equals", std::move(structure_equal_func_args));
|
||||
args.push_back(structure_equal_func);
|
||||
}
|
||||
}
|
||||
else
|
||||
{
|
||||
|
@ -39,7 +39,7 @@ public:
|
||||
|
||||
ObjectStoragePtr createObjectStorage(ContextPtr context, bool is_readonly) override;
|
||||
|
||||
void addStructureAndFormatToArgs(
|
||||
void addStructureAndFormatToArgsIfNeeded(
|
||||
ASTs & args,
|
||||
const String & structure_,
|
||||
const String & format_,
|
||||
|
@ -40,7 +40,7 @@ public:
|
||||
|
||||
ObjectStoragePtr createObjectStorage(ContextPtr, bool) override { return std::make_shared<LocalObjectStorage>("/"); }
|
||||
|
||||
void addStructureAndFormatToArgs(ASTs &, const String &, const String &, ContextPtr) override { }
|
||||
void addStructureAndFormatToArgsIfNeeded(ASTs &, const String &, const String &, ContextPtr) override { }
|
||||
|
||||
private:
|
||||
void fromNamedCollection(const NamedCollection & collection, ContextPtr context) override;
|
||||
|
@ -365,16 +365,25 @@ void StorageS3Configuration::fromAST(ASTs & args, ContextPtr context, bool with_
|
||||
keys = {url.key};
|
||||
}
|
||||
|
||||
void StorageS3Configuration::addStructureAndFormatToArgs(
|
||||
void StorageS3Configuration::addStructureAndFormatToArgsIfNeeded(
|
||||
ASTs & args, const String & structure_, const String & format_, ContextPtr context)
|
||||
{
|
||||
if (tryGetNamedCollectionWithOverrides(args, context))
|
||||
if (auto collection = tryGetNamedCollectionWithOverrides(args, context))
|
||||
{
|
||||
/// In case of named collection, just add key-value pair "structure='...'"
|
||||
/// at the end of arguments to override existed structure.
|
||||
ASTs equal_func_args = {std::make_shared<ASTIdentifier>("structure"), std::make_shared<ASTLiteral>(structure_)};
|
||||
auto equal_func = makeASTFunction("equals", std::move(equal_func_args));
|
||||
args.push_back(equal_func);
|
||||
/// In case of named collection, just add key-value pairs "format='...', structure='...'"
|
||||
/// at the end of arguments to override existed format and structure with "auto" values.
|
||||
if (collection->getOrDefault<String>("format", "auto") == "auto")
|
||||
{
|
||||
ASTs format_equal_func_args = {std::make_shared<ASTIdentifier>("format"), std::make_shared<ASTLiteral>(format_)};
|
||||
auto format_equal_func = makeASTFunction("equals", std::move(format_equal_func_args));
|
||||
args.push_back(format_equal_func);
|
||||
}
|
||||
if (collection->getOrDefault<String>("structure", "auto") == "auto")
|
||||
{
|
||||
ASTs structure_equal_func_args = {std::make_shared<ASTIdentifier>("structure"), std::make_shared<ASTLiteral>(structure_)};
|
||||
auto structure_equal_func = makeASTFunction("equals", std::move(structure_equal_func_args));
|
||||
args.push_back(structure_equal_func);
|
||||
}
|
||||
}
|
||||
else
|
||||
{
|
||||
@ -401,7 +410,10 @@ void StorageS3Configuration::addStructureAndFormatToArgs(
|
||||
auto second_arg = checkAndGetLiteralArgument<String>(args[1], "format/NOSIGN");
|
||||
/// If there is NOSIGN, add format=auto before structure.
|
||||
if (boost::iequals(second_arg, "NOSIGN"))
|
||||
args.push_back(std::make_shared<ASTLiteral>("auto"));
|
||||
args.push_back(format_literal);
|
||||
else if (checkAndGetLiteralArgument<String>(args[1], "format") == "auto")
|
||||
args[1] = format_literal;
|
||||
|
||||
args.push_back(structure_literal);
|
||||
}
|
||||
/// s3(source, format, structure) or
|
||||
@ -413,16 +425,21 @@ void StorageS3Configuration::addStructureAndFormatToArgs(
|
||||
auto second_arg = checkAndGetLiteralArgument<String>(args[1], "format/NOSIGN");
|
||||
if (boost::iequals(second_arg, "NOSIGN"))
|
||||
{
|
||||
if (checkAndGetLiteralArgument<String>(args[2], "format") == "auto")
|
||||
args[2] = format_literal;
|
||||
args.push_back(structure_literal);
|
||||
}
|
||||
else if (second_arg == "auto" || FormatFactory::instance().exists(second_arg))
|
||||
{
|
||||
args[count - 1] = structure_literal;
|
||||
if (second_arg == "auto")
|
||||
args[1] = format_literal;
|
||||
if (checkAndGetLiteralArgument<String>(args[2], "structure") == "auto")
|
||||
args[2] = structure_literal;
|
||||
}
|
||||
else
|
||||
{
|
||||
/// Add format=auto before structure argument.
|
||||
args.push_back(std::make_shared<ASTLiteral>("auto"));
|
||||
/// Add format and structure arguments.
|
||||
args.push_back(format_literal);
|
||||
args.push_back(structure_literal);
|
||||
}
|
||||
}
|
||||
@ -435,14 +452,22 @@ void StorageS3Configuration::addStructureAndFormatToArgs(
|
||||
auto second_arg = checkAndGetLiteralArgument<String>(args[1], "format/NOSIGN");
|
||||
if (boost::iequals(second_arg, "NOSIGN"))
|
||||
{
|
||||
args[count - 1] = structure_literal;
|
||||
if (checkAndGetLiteralArgument<String>(args[2], "format") == "auto")
|
||||
args[2] = format_literal;
|
||||
if (checkAndGetLiteralArgument<String>(args[3], "structure") == "auto")
|
||||
args[3] = structure_literal;
|
||||
}
|
||||
else if (second_arg == "auto" || FormatFactory::instance().exists(second_arg))
|
||||
{
|
||||
args[count - 2] = structure_literal;
|
||||
if (second_arg == "auto")
|
||||
args[1] = format_literal;
|
||||
if (checkAndGetLiteralArgument<String>(args[2], "structure") == "auto")
|
||||
args[2] = structure_literal;
|
||||
}
|
||||
else
|
||||
{
|
||||
if (checkAndGetLiteralArgument<String>(args[3], "format") == "auto")
|
||||
args[3] = format_literal;
|
||||
args.push_back(structure_literal);
|
||||
}
|
||||
}
|
||||
@ -454,17 +479,26 @@ void StorageS3Configuration::addStructureAndFormatToArgs(
|
||||
auto sedond_arg = checkAndGetLiteralArgument<String>(args[1], "format/NOSIGN");
|
||||
if (boost::iequals(sedond_arg, "NOSIGN"))
|
||||
{
|
||||
args[count - 2] = structure_literal;
|
||||
if (checkAndGetLiteralArgument<String>(args[2], "format") == "auto")
|
||||
args[2] = format_literal;
|
||||
if (checkAndGetLiteralArgument<String>(args[2], "structure") == "auto")
|
||||
args[3] = structure_literal;
|
||||
}
|
||||
else
|
||||
{
|
||||
args[count - 1] = structure_literal;
|
||||
if (checkAndGetLiteralArgument<String>(args[3], "format") == "auto")
|
||||
args[3] = format_literal;
|
||||
if (checkAndGetLiteralArgument<String>(args[4], "structure") == "auto")
|
||||
args[4] = structure_literal;
|
||||
}
|
||||
}
|
||||
/// s3(source, access_key_id, secret_access_key, format, structure, compression)
|
||||
else if (count == 6)
|
||||
{
|
||||
args[count - 2] = structure_literal;
|
||||
if (checkAndGetLiteralArgument<String>(args[3], "format") == "auto")
|
||||
args[3] = format_literal;
|
||||
if (checkAndGetLiteralArgument<String>(args[4], "structure") == "auto")
|
||||
args[4] = structure_literal;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
@ -44,7 +44,7 @@ public:
|
||||
|
||||
ObjectStoragePtr createObjectStorage(ContextPtr context, bool is_readonly) override;
|
||||
|
||||
void addStructureAndFormatToArgs(
|
||||
void addStructureAndFormatToArgsIfNeeded(
|
||||
ASTs & args,
|
||||
const String & structure,
|
||||
const String & format,
|
||||
|
@ -180,7 +180,9 @@ public:
|
||||
virtual String getNamespace() const = 0;
|
||||
|
||||
virtual StorageObjectStorage::QuerySettings getQuerySettings(const ContextPtr &) const = 0;
|
||||
virtual void addStructureAndFormatToArgs(
|
||||
|
||||
/// Add/replace structure and format arguments in the AST arguments if they have 'auto' values.
|
||||
virtual void addStructureAndFormatToArgsIfNeeded(
|
||||
ASTs & args, const String & structure_, const String & format_, ContextPtr context) = 0;
|
||||
|
||||
bool withPartitionWildcard() const;
|
||||
|
@ -103,7 +103,7 @@ void StorageObjectStorageCluster::updateQueryToSendIfNeeded(
|
||||
|
||||
ASTPtr cluster_name_arg = args.front();
|
||||
args.erase(args.begin());
|
||||
configuration->addStructureAndFormatToArgs(args, structure, configuration->format, context);
|
||||
configuration->addStructureAndFormatToArgsIfNeeded(args, structure, configuration->format, context);
|
||||
args.insert(args.begin(), cluster_name_arg);
|
||||
}
|
||||
|
||||
|
@ -142,7 +142,7 @@ public:
|
||||
const String & format,
|
||||
const ContextPtr & context)
|
||||
{
|
||||
Configuration().addStructureAndFormatToArgs(args, structure, format, context);
|
||||
Configuration().addStructureAndFormatToArgsIfNeeded(args, structure, format, context);
|
||||
}
|
||||
|
||||
protected:
|
||||
|
@ -6,5 +6,12 @@
|
||||
<secret_access_key>minio123</secret_access_key>
|
||||
<format>CSV</format>>
|
||||
</test_s3>
|
||||
<test_s3_with_default>
|
||||
<url>http://minio1:9001/root/data/data{1,2,3}</url>
|
||||
<access_key_id>minio</access_key_id>
|
||||
<secret_access_key>minio123</secret_access_key>
|
||||
<format>JSONEachRow</format>>
|
||||
<structure>id UInt32, date Date DEFAULT 18262</structure>
|
||||
</test_s3_with_default>
|
||||
</named_collections>
|
||||
</clickhouse>
|
||||
|
@ -459,3 +459,44 @@ def test_cluster_format_detection(started_cluster):
|
||||
)
|
||||
|
||||
assert result == expected_result
|
||||
|
||||
def test_cluster_default_expression(started_cluster):
|
||||
node = started_cluster.instances["s0_0_0"]
|
||||
|
||||
node.query("insert into function s3('http://minio1:9001/root/data/data1', 'minio', 'minio123', JSONEachRow) select 1 as id settings s3_truncate_on_insert=1")
|
||||
node.query("insert into function s3('http://minio1:9001/root/data/data2', 'minio', 'minio123', JSONEachRow) select * from numbers(0) settings s3_truncate_on_insert=1")
|
||||
node.query("insert into function s3('http://minio1:9001/root/data/data3', 'minio', 'minio123', JSONEachRow) select 2 as id settings s3_truncate_on_insert=1")
|
||||
|
||||
expected_result = node.query(
|
||||
"SELECT * FROM s3('http://minio1:9001/root/data/data{1,2,3}', 'minio', 'minio123', 'JSONEachRow', 'id UInt32, date Date DEFAULT 18262') order by id"
|
||||
)
|
||||
|
||||
result = node.query(
|
||||
"SELECT * FROM s3Cluster(cluster_simple, 'http://minio1:9001/root/data/data{1,2,3}', 'minio', 'minio123', 'JSONEachRow', 'id UInt32, date Date DEFAULT 18262') order by id"
|
||||
)
|
||||
|
||||
assert result == expected_result
|
||||
|
||||
result = node.query(
|
||||
"SELECT * FROM s3Cluster(cluster_simple, 'http://minio1:9001/root/data/data{1,2,3}', 'minio', 'minio123', 'auto', 'id UInt32, date Date DEFAULT 18262') order by id"
|
||||
)
|
||||
|
||||
assert result == expected_result
|
||||
|
||||
result = node.query(
|
||||
"SELECT * FROM s3Cluster(cluster_simple, 'http://minio1:9001/root/data/data{1,2,3}', 'minio', 'minio123', 'JSONEachRow', 'id UInt32, date Date DEFAULT 18262', 'auto') order by id"
|
||||
)
|
||||
|
||||
assert result == expected_result
|
||||
|
||||
result = node.query(
|
||||
"SELECT * FROM s3Cluster(cluster_simple, 'http://minio1:9001/root/data/data{1,2,3}', 'minio', 'minio123', 'auto', 'id UInt32, date Date DEFAULT 18262', 'auto') order by id"
|
||||
)
|
||||
|
||||
assert result == expected_result
|
||||
|
||||
result = node.query(
|
||||
"SELECT * FROM s3Cluster(cluster_simple, test_s3_with_default) order by id"
|
||||
)
|
||||
|
||||
assert result == expected_result
|
Loading…
Reference in New Issue
Block a user