diff --git a/src/Disks/ObjectStorages/S3/S3ObjectStorage.cpp b/src/Disks/ObjectStorages/S3/S3ObjectStorage.cpp index 1868d558b5c..e5539bcf945 100644 --- a/src/Disks/ObjectStorages/S3/S3ObjectStorage.cpp +++ b/src/Disks/ObjectStorages/S3/S3ObjectStorage.cpp @@ -127,17 +127,17 @@ std::string S3ObjectStorage::generateBlobNameForPath(const std::string & /* path size_t S3ObjectStorage::getObjectSize(const std::string & bucket_from, const std::string & key) const { - return S3::getObjectSize(*client.get(), bucket_from, key, "", true); + return S3::getObjectSize(*client.get(), bucket_from, key, {}, /* for_disk_s3= */ true); } bool S3ObjectStorage::exists(const StoredObject & object) const { - return S3::objectExists(*client.get(), bucket, object.absolute_path, "", true); + return S3::objectExists(*client.get(), bucket, object.absolute_path, {}, /* for_disk_s3= */ true); } std::pair S3ObjectStorage::checkObjectExists(const std::string & bucket_from, const std::string & key) const { - return S3::checkObjectExists(*client.get(), bucket_from, key, "", true); + return S3::checkObjectExists(*client.get(), bucket_from, key, {}, /* for_disk_s3= */ true); } std::unique_ptr S3ObjectStorage::readObjects( /// NOLINT @@ -414,10 +414,10 @@ ObjectMetadata S3ObjectStorage::getObjectMetadata(const std::string & path) cons { ObjectMetadata result; - auto object_info = S3::getObjectInfo(*client.get(), bucket, path, "", true, true); + auto object_info = S3::getObjectInfo(*client.get(), bucket, path, {}, /* for_disk_s3= */ true); result.size_bytes = object_info.size; result.last_modified = object_info.last_modification_time; - result.attributes = S3::getObjectMetadata(*client.get(), bucket, path, "", true, true); + result.attributes = S3::getObjectMetadata(*client.get(), bucket, path, {}, /* for_disk_s3= */ true); return result; } diff --git a/src/IO/ReadBufferFromS3.cpp b/src/IO/ReadBufferFromS3.cpp index 2d274435a74..69d2a244097 100644 --- a/src/IO/ReadBufferFromS3.cpp +++ b/src/IO/ReadBufferFromS3.cpp @@ -250,7 +250,7 @@ size_t ReadBufferFromS3::getFileSize() if (file_size) return *file_size; - auto object_size = S3::getObjectSize(*client_ptr, bucket, key, version_id, true, read_settings.for_object_storage); + auto object_size = S3::getObjectSize(*client_ptr, bucket, key, version_id, /* for_disk_s3= */ read_settings.for_object_storage); file_size = object_size; return *file_size; diff --git a/src/IO/S3Common.cpp b/src/IO/S3Common.cpp index b6937e5c40d..e019086e2d1 100644 --- a/src/IO/S3Common.cpp +++ b/src/IO/S3Common.cpp @@ -919,7 +919,7 @@ namespace S3 return error == Aws::S3::S3Errors::RESOURCE_NOT_FOUND || error == Aws::S3::S3Errors::NO_SUCH_KEY; } - S3::ObjectInfo getObjectInfo(const Aws::S3::S3Client & client, const String & bucket, const String & key, const String & version_id, bool throw_on_error, bool for_disk_s3) + S3::ObjectInfo getObjectInfo(const Aws::S3::S3Client & client, const String & bucket, const String & key, const String & version_id, bool for_disk_s3, bool throw_on_error) { auto outcome = getObjectAttributes(client, bucket, key, version_id, for_disk_s3); if (outcome.IsSuccess()) @@ -937,19 +937,19 @@ namespace S3 return {}; } - size_t getObjectSize(const Aws::S3::S3Client & client, const String & bucket, const String & key, const String & version_id, bool throw_on_error, bool for_disk_s3) + size_t getObjectSize(const Aws::S3::S3Client & client, const String & bucket, const String & key, const String & version_id, bool for_disk_s3, bool throw_on_error) { - return getObjectInfo(client, bucket, key, version_id, throw_on_error, for_disk_s3).size; + return getObjectInfo(client, bucket, key, version_id, for_disk_s3, throw_on_error).size; } - bool objectExists(const Aws::S3::S3Client & client, const String & bucket, const String & key, const String & version_id, bool for_disk_s3) + bool objectExists(const Aws::S3::S3Client & client, const String & bucket, const String & key, const String & version_id, bool for_disk_s3, bool throw_on_error) { auto [exists, error] = checkObjectExists(client, bucket, key, version_id, for_disk_s3); if (exists) return true; - if (isNotFoundError(error.GetErrorType())) + if (!throw_on_error || isNotFoundError(error.GetErrorType())) return false; throw S3Exception(error.GetErrorType(), @@ -965,7 +965,7 @@ namespace S3 return {false, std::move(outcome.GetError())}; } - std::map getObjectMetadata(const Aws::S3::S3Client & client, const String & bucket, const String & key, const String & version_id, bool throw_on_error, bool for_disk_s3) + std::map getObjectMetadata(const Aws::S3::S3Client & client, const String & bucket, const String & key, const String & version_id, bool for_disk_s3, bool throw_on_error) { ProfileEvents::increment(ProfileEvents::S3GetObjectMetadata); if (for_disk_s3) diff --git a/src/IO/S3Common.h b/src/IO/S3Common.h index 081fbab777e..b1749b636d3 100644 --- a/src/IO/S3Common.h +++ b/src/IO/S3Common.h @@ -137,11 +137,11 @@ struct ObjectInfo time_t last_modification_time = 0; }; -S3::ObjectInfo getObjectInfo(const Aws::S3::S3Client & client, const String & bucket, const String & key, const String & version_id = "", bool throw_on_error = true, bool for_disk_s3 = false); +S3::ObjectInfo getObjectInfo(const Aws::S3::S3Client & client, const String & bucket, const String & key, const String & version_id = "", bool for_disk_s3 = false, bool throw_on_error = true); -size_t getObjectSize(const Aws::S3::S3Client & client, const String & bucket, const String & key, const String & version_id = "", bool throw_on_error = true, bool for_disk_s3 = false); +size_t getObjectSize(const Aws::S3::S3Client & client, const String & bucket, const String & key, const String & version_id = "", bool for_disk_s3 = false, bool throw_on_error = true); -bool objectExists(const Aws::S3::S3Client & client, const String & bucket, const String & key, const String & version_id = "", bool for_disk_s3 = false); +bool objectExists(const Aws::S3::S3Client & client, const String & bucket, const String & key, const String & version_id = "", bool for_disk_s3 = false, bool throw_on_error = true); /// Checks if the object exists. If it doesn't exists the function returns an error without throwing any exception. std::pair checkObjectExists(const Aws::S3::S3Client & client, const String & bucket, const String & key, const String & version_id = "", bool for_disk_s3 = false); @@ -149,7 +149,7 @@ std::pair checkObjectExists(const Aws::S3:: bool isNotFoundError(Aws::S3::S3Errors error); /// Returns the object's metadata. -std::map getObjectMetadata(const Aws::S3::S3Client & client, const String & bucket, const String & key, const String & version_id = "", bool throw_on_error = true, bool for_disk_s3 = false); +std::map getObjectMetadata(const Aws::S3::S3Client & client, const String & bucket, const String & key, const String & version_id = "", bool for_disk_s3 = false, bool throw_on_error = true); } #endif diff --git a/src/Storages/StorageS3.cpp b/src/Storages/StorageS3.cpp index 9cb992bd24f..3257ccb5022 100644 --- a/src/Storages/StorageS3.cpp +++ b/src/Storages/StorageS3.cpp @@ -444,7 +444,7 @@ public: /// (which means we eventually need this info anyway, so it should be ok to do it now) if (object_infos_) { - info = S3::getObjectInfo(client_, bucket, key, version_id_, true, false); + info = S3::getObjectInfo(client_, bucket, key, version_id_); total_size += info->size; String path = fs::path(bucket) / key; @@ -569,9 +569,7 @@ StorageS3Source::ReaderHolder StorageS3Source::createReader() if (current_key.empty()) return {}; - size_t object_size = info - ? info->size - : S3::getObjectSize(*client, bucket, current_key, version_id, true, false); + size_t object_size = info ? info->size : S3::getObjectSize(*client, bucket, current_key, version_id); int zstd_window_log_max = static_cast(getContext()->getSettingsRef().zstd_window_log_max); auto read_buf = wrapReadBufferWithCompressionMethod( @@ -1523,7 +1521,7 @@ std::optional StorageS3::tryGetColumnsFromCache( /// Note that in case of exception in getObjectInfo returned info will be empty, /// but schema cache will handle this case and won't return columns from cache /// because we can't say that it's valid without last modification time. - info = S3::getObjectInfo(*s3_configuration.client, s3_configuration.uri.bucket, *it, s3_configuration.uri.version_id, false, false); + info = S3::getObjectInfo(*s3_configuration.client, s3_configuration.uri.bucket, *it, s3_configuration.uri.version_id, {}, /* throw_on_error= */ false); if (object_infos) (*object_infos)[path] = info; }