thevar1able/ClickHouse
1
0
Fork 0
mirror of https://github.com/ClickHouse/ClickHouse.git synced 2024-09-19 16:20:50 +00:00
Code Issues Packages Projects Releases Wiki Activity

Fix propogating structure argument in s3Cluster

Browse Source
This commit is contained in:
avogar 2024-09-02 12:50:12 +00:00
parent a7d0a5991e
commit d7aaf053f9
12 changed files with 139 additions and 37 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

23
src/Storages/ObjectStorage/Azure/Configuration.cpp
View File

@ -272,16 +272,25 @@ void StorageAzureConfiguration::fromAST(ASTs & engine_args, ContextPtr context,
connection_params = getConnectionParams(connection_url, container_name, account_name, account_key, context);
}
void StorageAzureConfiguration::addStructureAndFormatToArgs(
void StorageAzureConfiguration::addStructureAndFormatToArgsIfNeeded(
ASTs & args, const String & structure_, const String & format_, ContextPtr context)
{
if (tryGetNamedCollectionWithOverrides(args, context))
if (auto collection = tryGetNamedCollectionWithOverrides(args, context))
{
/// In case of named collection, just add key-value pair "structure='...'"
/// at the end of arguments to override existed structure.
ASTs equal_func_args = {std::make_shared<ASTIdentifier>("structure"), std::make_shared<ASTLiteral>(structure_)};
auto equal_func = makeASTFunction("equals", std::move(equal_func_args));
args.push_back(equal_func);
/// In case of named collection, just add key-value pairs "format='...', structure='...'"
/// at the end of arguments to override existed format and structure with "auto" values.
if (collection->getOrDefault<String>("format", "auto") == "auto")
{
ASTs format_equal_func_args = {std::make_shared<ASTIdentifier>("format"), std::make_shared<ASTLiteral>(format_)};
auto format_equal_func = makeASTFunction("equals", std::move(format_equal_func_args));
args.push_back(format_equal_func);
}
if (collection->getOrDefault<String>("structure", "auto") == "auto")
{
ASTs structure_equal_func_args = {std::make_shared<ASTIdentifier>("structure"), std::make_shared<ASTLiteral>(structure_)};
auto structure_equal_func = makeASTFunction("equals", std::move(structure_equal_func_args));
args.push_back(structure_equal_func);
}
}
else
{

2
src/Storages/ObjectStorage/Azure/Configuration.h
View File

@ -44,7 +44,7 @@ public:
ObjectStoragePtr createObjectStorage(ContextPtr context, bool is_readonly) override;
void addStructureAndFormatToArgs(
void addStructureAndFormatToArgsIfNeeded(
ASTs & args,
const String & structure_,
const String & format_,

23
src/Storages/ObjectStorage/HDFS/Configuration.cpp
View File

@ -158,19 +158,28 @@ void StorageHDFSConfiguration::setURL(const std::string & url_)
LOG_TRACE(getLogger("StorageHDFSConfiguration"), "Using URL: {}, path: {}", url, path);
}
void StorageHDFSConfiguration::addStructureAndFormatToArgs(
void StorageHDFSConfiguration::addStructureAndFormatToArgsIfNeeded(
ASTs & args,
const String & structure_,
const String & format_,
ContextPtr context)
{
if (tryGetNamedCollectionWithOverrides(args, context))
if (auto collection = tryGetNamedCollectionWithOverrides(args, context))
{
/// In case of named collection, just add key-value pair "structure='...'"
/// at the end of arguments to override existed structure.
ASTs equal_func_args = {std::make_shared<ASTIdentifier>("structure"), std::make_shared<ASTLiteral>(structure_)};
auto equal_func = makeASTFunction("equals", std::move(equal_func_args));
args.push_back(equal_func);
/// In case of named collection, just add key-value pairs "format='...', structure='...'"
/// at the end of arguments to override existed format and structure with "auto" values.
if (collection->getOrDefault<String>("format", "auto") == "auto")
{
ASTs format_equal_func_args = {std::make_shared<ASTIdentifier>("format"), std::make_shared<ASTLiteral>(format_)};
auto format_equal_func = makeASTFunction("equals", std::move(format_equal_func_args));
args.push_back(format_equal_func);
}
if (collection->getOrDefault<String>("structure", "auto") == "auto")
{
ASTs structure_equal_func_args = {std::make_shared<ASTIdentifier>("structure"), std::make_shared<ASTLiteral>(structure_)};
auto structure_equal_func = makeASTFunction("equals", std::move(structure_equal_func_args));
args.push_back(structure_equal_func);
}
}
else
{

2
src/Storages/ObjectStorage/HDFS/Configuration.h
View File

@ -39,7 +39,7 @@ public:
ObjectStoragePtr createObjectStorage(ContextPtr context, bool is_readonly) override;
void addStructureAndFormatToArgs(
void addStructureAndFormatToArgsIfNeeded(
ASTs & args,
const String & structure_,
const String & format_,

2
src/Storages/ObjectStorage/Local/Configuration.h
View File

@ -40,7 +40,7 @@ public:
ObjectStoragePtr createObjectStorage(ContextPtr, bool) override { return std::make_shared<LocalObjectStorage>("/"); }
void addStructureAndFormatToArgs(ASTs &, const String &, const String &, ContextPtr) override { }
void addStructureAndFormatToArgsIfNeeded(ASTs &, const String &, const String &, ContextPtr) override { }
private:
void fromNamedCollection(const NamedCollection & collection, ContextPtr context) override;

66
src/Storages/ObjectStorage/S3/Configuration.cpp
View File

@ -365,16 +365,25 @@ void StorageS3Configuration::fromAST(ASTs & args, ContextPtr context, bool with_
keys = {url.key};
}
void StorageS3Configuration::addStructureAndFormatToArgs(
void StorageS3Configuration::addStructureAndFormatToArgsIfNeeded(
ASTs & args, const String & structure_, const String & format_, ContextPtr context)
{
if (tryGetNamedCollectionWithOverrides(args, context))
if (auto collection = tryGetNamedCollectionWithOverrides(args, context))
{
/// In case of named collection, just add key-value pair "structure='...'"
/// at the end of arguments to override existed structure.
ASTs equal_func_args = {std::make_shared<ASTIdentifier>("structure"), std::make_shared<ASTLiteral>(structure_)};
auto equal_func = makeASTFunction("equals", std::move(equal_func_args));
args.push_back(equal_func);
/// In case of named collection, just add key-value pairs "format='...', structure='...'"
/// at the end of arguments to override existed format and structure with "auto" values.
if (collection->getOrDefault<String>("format", "auto") == "auto")
{
ASTs format_equal_func_args = {std::make_shared<ASTIdentifier>("format"), std::make_shared<ASTLiteral>(format_)};
auto format_equal_func = makeASTFunction("equals", std::move(format_equal_func_args));
args.push_back(format_equal_func);
}
if (collection->getOrDefault<String>("structure", "auto") == "auto")
{
ASTs structure_equal_func_args = {std::make_shared<ASTIdentifier>("structure"), std::make_shared<ASTLiteral>(structure_)};
auto structure_equal_func = makeASTFunction("equals", std::move(structure_equal_func_args));
args.push_back(structure_equal_func);
}
}
else
{
@ -401,7 +410,10 @@ void StorageS3Configuration::addStructureAndFormatToArgs(
auto second_arg = checkAndGetLiteralArgument<String>(args[1], "format/NOSIGN");
/// If there is NOSIGN, add format=auto before structure.
if (boost::iequals(second_arg, "NOSIGN"))
args.push_back(std::make_shared<ASTLiteral>("auto"));
args.push_back(format_literal);
else if (checkAndGetLiteralArgument<String>(args[1], "format") == "auto")
args[1] = format_literal;
args.push_back(structure_literal);
}
/// s3(source, format, structure) or
@ -413,16 +425,21 @@ void StorageS3Configuration::addStructureAndFormatToArgs(
auto second_arg = checkAndGetLiteralArgument<String>(args[1], "format/NOSIGN");
if (boost::iequals(second_arg, "NOSIGN"))
{
if (checkAndGetLiteralArgument<String>(args[2], "format") == "auto")
args[2] = format_literal;
args.push_back(structure_literal);
}
else if (second_arg == "auto" || FormatFactory::instance().exists(second_arg))
{
args[count - 1] = structure_literal;
if (second_arg == "auto")
args[1] = format_literal;
if (checkAndGetLiteralArgument<String>(args[2], "structure") == "auto")
args[2] = structure_literal;
}
else
{
/// Add format=auto before structure argument.
args.push_back(std::make_shared<ASTLiteral>("auto"));
/// Add format and structure arguments.
args.push_back(format_literal);
args.push_back(structure_literal);
}
}
@ -435,14 +452,22 @@ void StorageS3Configuration::addStructureAndFormatToArgs(
auto second_arg = checkAndGetLiteralArgument<String>(args[1], "format/NOSIGN");
if (boost::iequals(second_arg, "NOSIGN"))
{
args[count - 1] = structure_literal;
if (checkAndGetLiteralArgument<String>(args[2], "format") == "auto")
args[2] = format_literal;
if (checkAndGetLiteralArgument<String>(args[3], "structure") == "auto")
args[3] = structure_literal;
}
else if (second_arg == "auto" || FormatFactory::instance().exists(second_arg))
{
args[count - 2] = structure_literal;
if (second_arg == "auto")
args[1] = format_literal;
if (checkAndGetLiteralArgument<String>(args[2], "structure") == "auto")
args[2] = structure_literal;
}
else
{
if (checkAndGetLiteralArgument<String>(args[3], "format") == "auto")
args[3] = format_literal;
args.push_back(structure_literal);
}
}
@ -454,17 +479,26 @@ void StorageS3Configuration::addStructureAndFormatToArgs(
auto sedond_arg = checkAndGetLiteralArgument<String>(args[1], "format/NOSIGN");
if (boost::iequals(sedond_arg, "NOSIGN"))
{
args[count - 2] = structure_literal;
if (checkAndGetLiteralArgument<String>(args[2], "format") == "auto")
args[2] = format_literal;
if (checkAndGetLiteralArgument<String>(args[2], "structure") == "auto")
args[3] = structure_literal;
}
else
{
args[count - 1] = structure_literal;
if (checkAndGetLiteralArgument<String>(args[3], "format") == "auto")
args[3] = format_literal;
if (checkAndGetLiteralArgument<String>(args[4], "structure") == "auto")
args[4] = structure_literal;
}
}
/// s3(source, access_key_id, secret_access_key, format, structure, compression)
else if (count == 6)
{
args[count - 2] = structure_literal;
if (checkAndGetLiteralArgument<String>(args[3], "format") == "auto")
args[3] = format_literal;
if (checkAndGetLiteralArgument<String>(args[4], "structure") == "auto")
args[4] = structure_literal;
}
}
}

2
src/Storages/ObjectStorage/S3/Configuration.h
View File

@ -44,7 +44,7 @@ public:
ObjectStoragePtr createObjectStorage(ContextPtr context, bool is_readonly) override;
void addStructureAndFormatToArgs(
void addStructureAndFormatToArgsIfNeeded(
ASTs & args,
const String & structure,
const String & format,

4
src/Storages/ObjectStorage/StorageObjectStorage.h
View File

@ -180,7 +180,9 @@ public:
virtual String getNamespace() const = 0;
virtual StorageObjectStorage::QuerySettings getQuerySettings(const ContextPtr &) const = 0;
virtual void addStructureAndFormatToArgs(
/// Add/replace structure and format arguments in the AST arguments if they have 'auto' values.
virtual void addStructureAndFormatToArgsIfNeeded(
ASTs & args, const String & structure_, const String & format_, ContextPtr context) = 0;
bool withPartitionWildcard() const;

2
src/Storages/ObjectStorage/StorageObjectStorageCluster.cpp
View File

@ -103,7 +103,7 @@ void StorageObjectStorageCluster::updateQueryToSendIfNeeded(
ASTPtr cluster_name_arg = args.front();
args.erase(args.begin());
configuration->addStructureAndFormatToArgs(args, structure, configuration->format, context);
configuration->addStructureAndFormatToArgsIfNeeded(args, structure, configuration->format, context);
args.insert(args.begin(), cluster_name_arg);
}

2
src/TableFunctions/TableFunctionObjectStorage.h
View File

@ -142,7 +142,7 @@ public:
const String & format,
const ContextPtr & context)
{
Configuration().addStructureAndFormatToArgs(args, structure, format, context);
Configuration().addStructureAndFormatToArgsIfNeeded(args, structure, format, context);
}
protected:

7
tests/integration/test_s3_cluster/configs/named_collections.xml
View File

@ -6,5 +6,12 @@
<secret_access_key>minio123</secret_access_key>
<format>CSV</format>>
</test_s3>
<test_s3_with_default>
<url>http://minio1:9001/root/data/data{1,2,3}</url>
<access_key_id>minio</access_key_id>
<secret_access_key>minio123</secret_access_key>
<format>JSONEachRow</format>>
<structure>id UInt32, date Date DEFAULT 18262</structure>
</test_s3_with_default>
</named_collections>
</clickhouse>

41
tests/integration/test_s3_cluster/test.py
View File

@ -459,3 +459,44 @@ def test_cluster_format_detection(started_cluster):
)
assert result == expected_result
def test_cluster_default_expression(started_cluster):
node = started_cluster.instances["s0_0_0"]
node.query("insert into function s3('http://minio1:9001/root/data/data1', 'minio', 'minio123', JSONEachRow) select 1 as id settings s3_truncate_on_insert=1")
node.query("insert into function s3('http://minio1:9001/root/data/data2', 'minio', 'minio123', JSONEachRow) select * from numbers(0) settings s3_truncate_on_insert=1")
node.query("insert into function s3('http://minio1:9001/root/data/data3', 'minio', 'minio123', JSONEachRow) select 2 as id settings s3_truncate_on_insert=1")
expected_result = node.query(
"SELECT * FROM s3('http://minio1:9001/root/data/data{1,2,3}', 'minio', 'minio123', 'JSONEachRow', 'id UInt32, date Date DEFAULT 18262') order by id"
)
result = node.query(
"SELECT * FROM s3Cluster(cluster_simple, 'http://minio1:9001/root/data/data{1,2,3}', 'minio', 'minio123', 'JSONEachRow', 'id UInt32, date Date DEFAULT 18262') order by id"
)
assert result == expected_result
result = node.query(
"SELECT * FROM s3Cluster(cluster_simple, 'http://minio1:9001/root/data/data{1,2,3}', 'minio', 'minio123', 'auto', 'id UInt32, date Date DEFAULT 18262') order by id"
)
assert result == expected_result
result = node.query(
"SELECT * FROM s3Cluster(cluster_simple, 'http://minio1:9001/root/data/data{1,2,3}', 'minio', 'minio123', 'JSONEachRow', 'id UInt32, date Date DEFAULT 18262', 'auto') order by id"
)
assert result == expected_result
result = node.query(
"SELECT * FROM s3Cluster(cluster_simple, 'http://minio1:9001/root/data/data{1,2,3}', 'minio', 'minio123', 'auto', 'id UInt32, date Date DEFAULT 18262', 'auto') order by id"
)
assert result == expected_result
result = node.query(
"SELECT * FROM s3Cluster(cluster_simple, test_s3_with_default) order by id"
)
assert result == expected_result