From d7aaf053f992d4d6b27e9f88133818bb0a48d011 Mon Sep 17 00:00:00 2001 From: avogar Date: Mon, 2 Sep 2024 12:50:12 +0000 Subject: [PATCH] Fix propogating structure argument in s3Cluster --- .../ObjectStorage/Azure/Configuration.cpp | 23 +++++-- .../ObjectStorage/Azure/Configuration.h | 2 +- .../ObjectStorage/HDFS/Configuration.cpp | 23 +++++-- .../ObjectStorage/HDFS/Configuration.h | 2 +- .../ObjectStorage/Local/Configuration.h | 2 +- .../ObjectStorage/S3/Configuration.cpp | 66 ++++++++++++++----- src/Storages/ObjectStorage/S3/Configuration.h | 2 +- .../ObjectStorage/StorageObjectStorage.h | 4 +- .../StorageObjectStorageCluster.cpp | 2 +- .../TableFunctionObjectStorage.h | 2 +- .../configs/named_collections.xml | 7 ++ tests/integration/test_s3_cluster/test.py | 41 ++++++++++++ 12 files changed, 139 insertions(+), 37 deletions(-) diff --git a/src/Storages/ObjectStorage/Azure/Configuration.cpp b/src/Storages/ObjectStorage/Azure/Configuration.cpp index 9730391d429..fa5346e9288 100644 --- a/src/Storages/ObjectStorage/Azure/Configuration.cpp +++ b/src/Storages/ObjectStorage/Azure/Configuration.cpp @@ -272,16 +272,25 @@ void StorageAzureConfiguration::fromAST(ASTs & engine_args, ContextPtr context, connection_params = getConnectionParams(connection_url, container_name, account_name, account_key, context); } -void StorageAzureConfiguration::addStructureAndFormatToArgs( +void StorageAzureConfiguration::addStructureAndFormatToArgsIfNeeded( ASTs & args, const String & structure_, const String & format_, ContextPtr context) { - if (tryGetNamedCollectionWithOverrides(args, context)) + if (auto collection = tryGetNamedCollectionWithOverrides(args, context)) { - /// In case of named collection, just add key-value pair "structure='...'" - /// at the end of arguments to override existed structure. - ASTs equal_func_args = {std::make_shared("structure"), std::make_shared(structure_)}; - auto equal_func = makeASTFunction("equals", std::move(equal_func_args)); - args.push_back(equal_func); + /// In case of named collection, just add key-value pairs "format='...', structure='...'" + /// at the end of arguments to override existed format and structure with "auto" values. + if (collection->getOrDefault("format", "auto") == "auto") + { + ASTs format_equal_func_args = {std::make_shared("format"), std::make_shared(format_)}; + auto format_equal_func = makeASTFunction("equals", std::move(format_equal_func_args)); + args.push_back(format_equal_func); + } + if (collection->getOrDefault("structure", "auto") == "auto") + { + ASTs structure_equal_func_args = {std::make_shared("structure"), std::make_shared(structure_)}; + auto structure_equal_func = makeASTFunction("equals", std::move(structure_equal_func_args)); + args.push_back(structure_equal_func); + } } else { diff --git a/src/Storages/ObjectStorage/Azure/Configuration.h b/src/Storages/ObjectStorage/Azure/Configuration.h index 4e6bfbc0745..8cf583c32db 100644 --- a/src/Storages/ObjectStorage/Azure/Configuration.h +++ b/src/Storages/ObjectStorage/Azure/Configuration.h @@ -44,7 +44,7 @@ public: ObjectStoragePtr createObjectStorage(ContextPtr context, bool is_readonly) override; - void addStructureAndFormatToArgs( + void addStructureAndFormatToArgsIfNeeded( ASTs & args, const String & structure_, const String & format_, diff --git a/src/Storages/ObjectStorage/HDFS/Configuration.cpp b/src/Storages/ObjectStorage/HDFS/Configuration.cpp index 85eb29a3868..8abc51a7526 100644 --- a/src/Storages/ObjectStorage/HDFS/Configuration.cpp +++ b/src/Storages/ObjectStorage/HDFS/Configuration.cpp @@ -158,19 +158,28 @@ void StorageHDFSConfiguration::setURL(const std::string & url_) LOG_TRACE(getLogger("StorageHDFSConfiguration"), "Using URL: {}, path: {}", url, path); } -void StorageHDFSConfiguration::addStructureAndFormatToArgs( +void StorageHDFSConfiguration::addStructureAndFormatToArgsIfNeeded( ASTs & args, const String & structure_, const String & format_, ContextPtr context) { - if (tryGetNamedCollectionWithOverrides(args, context)) + if (auto collection = tryGetNamedCollectionWithOverrides(args, context)) { - /// In case of named collection, just add key-value pair "structure='...'" - /// at the end of arguments to override existed structure. - ASTs equal_func_args = {std::make_shared("structure"), std::make_shared(structure_)}; - auto equal_func = makeASTFunction("equals", std::move(equal_func_args)); - args.push_back(equal_func); + /// In case of named collection, just add key-value pairs "format='...', structure='...'" + /// at the end of arguments to override existed format and structure with "auto" values. + if (collection->getOrDefault("format", "auto") == "auto") + { + ASTs format_equal_func_args = {std::make_shared("format"), std::make_shared(format_)}; + auto format_equal_func = makeASTFunction("equals", std::move(format_equal_func_args)); + args.push_back(format_equal_func); + } + if (collection->getOrDefault("structure", "auto") == "auto") + { + ASTs structure_equal_func_args = {std::make_shared("structure"), std::make_shared(structure_)}; + auto structure_equal_func = makeASTFunction("equals", std::move(structure_equal_func_args)); + args.push_back(structure_equal_func); + } } else { diff --git a/src/Storages/ObjectStorage/HDFS/Configuration.h b/src/Storages/ObjectStorage/HDFS/Configuration.h index 04884542908..85f11aa6a98 100644 --- a/src/Storages/ObjectStorage/HDFS/Configuration.h +++ b/src/Storages/ObjectStorage/HDFS/Configuration.h @@ -39,7 +39,7 @@ public: ObjectStoragePtr createObjectStorage(ContextPtr context, bool is_readonly) override; - void addStructureAndFormatToArgs( + void addStructureAndFormatToArgsIfNeeded( ASTs & args, const String & structure_, const String & format_, diff --git a/src/Storages/ObjectStorage/Local/Configuration.h b/src/Storages/ObjectStorage/Local/Configuration.h index ba4de63ac47..c713a81975b 100644 --- a/src/Storages/ObjectStorage/Local/Configuration.h +++ b/src/Storages/ObjectStorage/Local/Configuration.h @@ -40,7 +40,7 @@ public: ObjectStoragePtr createObjectStorage(ContextPtr, bool) override { return std::make_shared("/"); } - void addStructureAndFormatToArgs(ASTs &, const String &, const String &, ContextPtr) override { } + void addStructureAndFormatToArgsIfNeeded(ASTs &, const String &, const String &, ContextPtr) override { } private: void fromNamedCollection(const NamedCollection & collection, ContextPtr context) override; diff --git a/src/Storages/ObjectStorage/S3/Configuration.cpp b/src/Storages/ObjectStorage/S3/Configuration.cpp index 7542f59dcc4..73f25009422 100644 --- a/src/Storages/ObjectStorage/S3/Configuration.cpp +++ b/src/Storages/ObjectStorage/S3/Configuration.cpp @@ -365,16 +365,25 @@ void StorageS3Configuration::fromAST(ASTs & args, ContextPtr context, bool with_ keys = {url.key}; } -void StorageS3Configuration::addStructureAndFormatToArgs( +void StorageS3Configuration::addStructureAndFormatToArgsIfNeeded( ASTs & args, const String & structure_, const String & format_, ContextPtr context) { - if (tryGetNamedCollectionWithOverrides(args, context)) + if (auto collection = tryGetNamedCollectionWithOverrides(args, context)) { - /// In case of named collection, just add key-value pair "structure='...'" - /// at the end of arguments to override existed structure. - ASTs equal_func_args = {std::make_shared("structure"), std::make_shared(structure_)}; - auto equal_func = makeASTFunction("equals", std::move(equal_func_args)); - args.push_back(equal_func); + /// In case of named collection, just add key-value pairs "format='...', structure='...'" + /// at the end of arguments to override existed format and structure with "auto" values. + if (collection->getOrDefault("format", "auto") == "auto") + { + ASTs format_equal_func_args = {std::make_shared("format"), std::make_shared(format_)}; + auto format_equal_func = makeASTFunction("equals", std::move(format_equal_func_args)); + args.push_back(format_equal_func); + } + if (collection->getOrDefault("structure", "auto") == "auto") + { + ASTs structure_equal_func_args = {std::make_shared("structure"), std::make_shared(structure_)}; + auto structure_equal_func = makeASTFunction("equals", std::move(structure_equal_func_args)); + args.push_back(structure_equal_func); + } } else { @@ -401,7 +410,10 @@ void StorageS3Configuration::addStructureAndFormatToArgs( auto second_arg = checkAndGetLiteralArgument(args[1], "format/NOSIGN"); /// If there is NOSIGN, add format=auto before structure. if (boost::iequals(second_arg, "NOSIGN")) - args.push_back(std::make_shared("auto")); + args.push_back(format_literal); + else if (checkAndGetLiteralArgument(args[1], "format") == "auto") + args[1] = format_literal; + args.push_back(structure_literal); } /// s3(source, format, structure) or @@ -413,16 +425,21 @@ void StorageS3Configuration::addStructureAndFormatToArgs( auto second_arg = checkAndGetLiteralArgument(args[1], "format/NOSIGN"); if (boost::iequals(second_arg, "NOSIGN")) { + if (checkAndGetLiteralArgument(args[2], "format") == "auto") + args[2] = format_literal; args.push_back(structure_literal); } else if (second_arg == "auto" || FormatFactory::instance().exists(second_arg)) { - args[count - 1] = structure_literal; + if (second_arg == "auto") + args[1] = format_literal; + if (checkAndGetLiteralArgument(args[2], "structure") == "auto") + args[2] = structure_literal; } else { - /// Add format=auto before structure argument. - args.push_back(std::make_shared("auto")); + /// Add format and structure arguments. + args.push_back(format_literal); args.push_back(structure_literal); } } @@ -435,14 +452,22 @@ void StorageS3Configuration::addStructureAndFormatToArgs( auto second_arg = checkAndGetLiteralArgument(args[1], "format/NOSIGN"); if (boost::iequals(second_arg, "NOSIGN")) { - args[count - 1] = structure_literal; + if (checkAndGetLiteralArgument(args[2], "format") == "auto") + args[2] = format_literal; + if (checkAndGetLiteralArgument(args[3], "structure") == "auto") + args[3] = structure_literal; } else if (second_arg == "auto" || FormatFactory::instance().exists(second_arg)) { - args[count - 2] = structure_literal; + if (second_arg == "auto") + args[1] = format_literal; + if (checkAndGetLiteralArgument(args[2], "structure") == "auto") + args[2] = structure_literal; } else { + if (checkAndGetLiteralArgument(args[3], "format") == "auto") + args[3] = format_literal; args.push_back(structure_literal); } } @@ -454,17 +479,26 @@ void StorageS3Configuration::addStructureAndFormatToArgs( auto sedond_arg = checkAndGetLiteralArgument(args[1], "format/NOSIGN"); if (boost::iequals(sedond_arg, "NOSIGN")) { - args[count - 2] = structure_literal; + if (checkAndGetLiteralArgument(args[2], "format") == "auto") + args[2] = format_literal; + if (checkAndGetLiteralArgument(args[2], "structure") == "auto") + args[3] = structure_literal; } else { - args[count - 1] = structure_literal; + if (checkAndGetLiteralArgument(args[3], "format") == "auto") + args[3] = format_literal; + if (checkAndGetLiteralArgument(args[4], "structure") == "auto") + args[4] = structure_literal; } } /// s3(source, access_key_id, secret_access_key, format, structure, compression) else if (count == 6) { - args[count - 2] = structure_literal; + if (checkAndGetLiteralArgument(args[3], "format") == "auto") + args[3] = format_literal; + if (checkAndGetLiteralArgument(args[4], "structure") == "auto") + args[4] = structure_literal; } } } diff --git a/src/Storages/ObjectStorage/S3/Configuration.h b/src/Storages/ObjectStorage/S3/Configuration.h index 39a646c7df2..0aed0f366a6 100644 --- a/src/Storages/ObjectStorage/S3/Configuration.h +++ b/src/Storages/ObjectStorage/S3/Configuration.h @@ -44,7 +44,7 @@ public: ObjectStoragePtr createObjectStorage(ContextPtr context, bool is_readonly) override; - void addStructureAndFormatToArgs( + void addStructureAndFormatToArgsIfNeeded( ASTs & args, const String & structure, const String & format, diff --git a/src/Storages/ObjectStorage/StorageObjectStorage.h b/src/Storages/ObjectStorage/StorageObjectStorage.h index 562ca259089..f39586c23b4 100644 --- a/src/Storages/ObjectStorage/StorageObjectStorage.h +++ b/src/Storages/ObjectStorage/StorageObjectStorage.h @@ -180,7 +180,9 @@ public: virtual String getNamespace() const = 0; virtual StorageObjectStorage::QuerySettings getQuerySettings(const ContextPtr &) const = 0; - virtual void addStructureAndFormatToArgs( + + /// Add/replace structure and format arguments in the AST arguments if they have 'auto' values. + virtual void addStructureAndFormatToArgsIfNeeded( ASTs & args, const String & structure_, const String & format_, ContextPtr context) = 0; bool withPartitionWildcard() const; diff --git a/src/Storages/ObjectStorage/StorageObjectStorageCluster.cpp b/src/Storages/ObjectStorage/StorageObjectStorageCluster.cpp index 08a0739d929..d712e4eec20 100644 --- a/src/Storages/ObjectStorage/StorageObjectStorageCluster.cpp +++ b/src/Storages/ObjectStorage/StorageObjectStorageCluster.cpp @@ -103,7 +103,7 @@ void StorageObjectStorageCluster::updateQueryToSendIfNeeded( ASTPtr cluster_name_arg = args.front(); args.erase(args.begin()); - configuration->addStructureAndFormatToArgs(args, structure, configuration->format, context); + configuration->addStructureAndFormatToArgsIfNeeded(args, structure, configuration->format, context); args.insert(args.begin(), cluster_name_arg); } diff --git a/src/TableFunctions/TableFunctionObjectStorage.h b/src/TableFunctions/TableFunctionObjectStorage.h index 3468e5c5007..6fa10face82 100644 --- a/src/TableFunctions/TableFunctionObjectStorage.h +++ b/src/TableFunctions/TableFunctionObjectStorage.h @@ -142,7 +142,7 @@ public: const String & format, const ContextPtr & context) { - Configuration().addStructureAndFormatToArgs(args, structure, format, context); + Configuration().addStructureAndFormatToArgsIfNeeded(args, structure, format, context); } protected: diff --git a/tests/integration/test_s3_cluster/configs/named_collections.xml b/tests/integration/test_s3_cluster/configs/named_collections.xml index 64d1bd98df2..2d3a69a8c38 100644 --- a/tests/integration/test_s3_cluster/configs/named_collections.xml +++ b/tests/integration/test_s3_cluster/configs/named_collections.xml @@ -6,5 +6,12 @@ minio123 CSV> + + http://minio1:9001/root/data/data{1,2,3} + minio + minio123 + JSONEachRow> + id UInt32, date Date DEFAULT 18262 + diff --git a/tests/integration/test_s3_cluster/test.py b/tests/integration/test_s3_cluster/test.py index 03919ee6a4d..b2954276ca5 100644 --- a/tests/integration/test_s3_cluster/test.py +++ b/tests/integration/test_s3_cluster/test.py @@ -459,3 +459,44 @@ def test_cluster_format_detection(started_cluster): ) assert result == expected_result + +def test_cluster_default_expression(started_cluster): + node = started_cluster.instances["s0_0_0"] + + node.query("insert into function s3('http://minio1:9001/root/data/data1', 'minio', 'minio123', JSONEachRow) select 1 as id settings s3_truncate_on_insert=1") + node.query("insert into function s3('http://minio1:9001/root/data/data2', 'minio', 'minio123', JSONEachRow) select * from numbers(0) settings s3_truncate_on_insert=1") + node.query("insert into function s3('http://minio1:9001/root/data/data3', 'minio', 'minio123', JSONEachRow) select 2 as id settings s3_truncate_on_insert=1") + + expected_result = node.query( + "SELECT * FROM s3('http://minio1:9001/root/data/data{1,2,3}', 'minio', 'minio123', 'JSONEachRow', 'id UInt32, date Date DEFAULT 18262') order by id" + ) + + result = node.query( + "SELECT * FROM s3Cluster(cluster_simple, 'http://minio1:9001/root/data/data{1,2,3}', 'minio', 'minio123', 'JSONEachRow', 'id UInt32, date Date DEFAULT 18262') order by id" + ) + + assert result == expected_result + + result = node.query( + "SELECT * FROM s3Cluster(cluster_simple, 'http://minio1:9001/root/data/data{1,2,3}', 'minio', 'minio123', 'auto', 'id UInt32, date Date DEFAULT 18262') order by id" + ) + + assert result == expected_result + + result = node.query( + "SELECT * FROM s3Cluster(cluster_simple, 'http://minio1:9001/root/data/data{1,2,3}', 'minio', 'minio123', 'JSONEachRow', 'id UInt32, date Date DEFAULT 18262', 'auto') order by id" + ) + + assert result == expected_result + + result = node.query( + "SELECT * FROM s3Cluster(cluster_simple, 'http://minio1:9001/root/data/data{1,2,3}', 'minio', 'minio123', 'auto', 'id UInt32, date Date DEFAULT 18262', 'auto') order by id" + ) + + assert result == expected_result + + result = node.query( + "SELECT * FROM s3Cluster(cluster_simple, test_s3_with_default) order by id" + ) + + assert result == expected_result \ No newline at end of file