diff --git a/src/Storages/ObjectStorage/DataLakes/DataLakeConfiguration.h b/src/Storages/ObjectStorage/DataLakes/DataLakeConfiguration.h index 52b5a553de1..29bd7b992d5 100644 --- a/src/Storages/ObjectStorage/DataLakes/DataLakeConfiguration.h +++ b/src/Storages/ObjectStorage/DataLakes/DataLakeConfiguration.h @@ -11,7 +11,6 @@ #include #include #include -#include "Common/DateLUT.h" #include #include "Storages/ColumnsDescription.h" @@ -47,9 +46,11 @@ public: { BaseStorageConfiguration::update(object_storage, local_context); + bool existed = current_metadata != nullptr; + if (updateMetadataObjectIfNeeded(object_storage, local_context)) { - if (hasExternalDynamicMetadata()) + if (hasExternalDynamicMetadata() && existed) { throw Exception( ErrorCodes::FORMAT_VERSION_TOO_OLD, @@ -103,6 +104,7 @@ public: BaseStorageConfiguration::setPaths(current_metadata->getDataFiles()); BaseStorageConfiguration::setPartitionColumns(current_metadata->getPartitionColumns()); } + return ColumnsDescription{current_metadata->getTableSchema()}; } diff --git a/src/Storages/ObjectStorage/DataLakes/Iceberg/IcebergMetadata.cpp b/src/Storages/ObjectStorage/DataLakes/Iceberg/IcebergMetadata.cpp index 1d5da6a1a09..94b646f30f5 100644 --- a/src/Storages/ObjectStorage/DataLakes/Iceberg/IcebergMetadata.cpp +++ b/src/Storages/ObjectStorage/DataLakes/Iceberg/IcebergMetadata.cpp @@ -1,45 +1,25 @@ -#include -#include -#include "Common/Config/ConfigProcessor.h" -#include "Common/DateLUT.h" -#include "Storages/ObjectStorage/DataLakes/Iceberg/ManifestFile.h" #include "config.h" #if USE_AVRO -# include -# include -# include -# include -# include -# include -# include -# include -# include -# include -# include -# include -# include -# include -# include -# include -# include -# include -# include -# include -# include -# include -# include -# include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include -# include -# include +#include "Storages/ObjectStorage/DataLakes/Iceberg/IcebergMetadata.h" +#include "Storages/ObjectStorage/DataLakes/Iceberg/Utils.h" -# include "Storages/ObjectStorage/DataLakes/Iceberg/IcebergMetadata.h" -# include "Storages/ObjectStorage/DataLakes/Iceberg/Utils.h" - -# include "Storages/ObjectStorage/DataLakes/Iceberg/ManifestFileImpl.h" -# include "Storages/ObjectStorage/DataLakes/Iceberg/Snapshot.h" +#include "Storages/ObjectStorage/DataLakes/Iceberg/ManifestFileImpl.h" +#include "Storages/ObjectStorage/DataLakes/Iceberg/Snapshot.h" namespace DB { @@ -255,12 +235,15 @@ bool IcebergMetadata::update(const ContextPtr & local_context) const auto [metadata_version, metadata_file_path] = getMetadataFileAndVersion(object_storage, *configuration_ptr); + if (metadata_version == current_metadata_version) + return false; + + current_metadata_version = metadata_version; + auto metadata_object = readJson(metadata_file_path, local_context); chassert(format_version == metadata_object->getValue("format-version")); - if (metadata_version == current_metadata_version) - return false; auto manifest_list_file = getRelevantManifestList(metadata_object); if (manifest_list_file && (!current_snapshot.has_value() || (manifest_list_file.value() != current_snapshot->getName()))) diff --git a/src/Storages/ObjectStorage/DataLakes/Iceberg/IcebergMetadata.h b/src/Storages/ObjectStorage/DataLakes/Iceberg/IcebergMetadata.h index b4113b6dc26..c407f22dea0 100644 --- a/src/Storages/ObjectStorage/DataLakes/Iceberg/IcebergMetadata.h +++ b/src/Storages/ObjectStorage/DataLakes/Iceberg/IcebergMetadata.h @@ -3,24 +3,21 @@ #if USE_AVRO /// StorageIceberg depending on Avro to parse metadata with Avro format. -# include -# include -# include -# include -# include +#include +#include +#include +#include +#include -# include -# include -# include +#include +#include +#include -# include "Storages/ObjectStorage/DataLakes/Iceberg/ManifestFile.h" -# include "Storages/ObjectStorage/DataLakes/Iceberg/SchemaProcessor.h" -# include "Storages/ObjectStorage/DataLakes/Iceberg/Snapshot.h" +#include "Storages/ObjectStorage/DataLakes/Iceberg/ManifestFile.h" +#include "Storages/ObjectStorage/DataLakes/Iceberg/SchemaProcessor.h" +#include "Storages/ObjectStorage/DataLakes/Iceberg/Snapshot.h" -# include -# include -# include -# include +#include namespace DB { diff --git a/src/Storages/ObjectStorage/DataLakes/Iceberg/ManifestFile.cpp b/src/Storages/ObjectStorage/DataLakes/Iceberg/ManifestFile.cpp index 1819d8bf599..a34422c72ca 100644 --- a/src/Storages/ObjectStorage/DataLakes/Iceberg/ManifestFile.cpp +++ b/src/Storages/ObjectStorage/DataLakes/Iceberg/ManifestFile.cpp @@ -3,26 +3,12 @@ #if USE_AVRO -# include "Storages/ObjectStorage/DataLakes/Iceberg/ManifestFileImpl.h" -# include "Storages/ObjectStorage/DataLakes/Iceberg/Utils.h" +#include "Storages/ObjectStorage/DataLakes/Iceberg/ManifestFileImpl.h" +#include "Storages/ObjectStorage/DataLakes/Iceberg/Utils.h" -# include -# include -# include -# include -# include -# include - -# include -# include -# include - -# include -# include "DataTypes/DataTypeTuple.h" -# include "Formats/FormatSettings.h" - -# include +#include +#include "DataTypes/DataTypeTuple.h" namespace DB diff --git a/src/Storages/ObjectStorage/DataLakes/Iceberg/ManifestFile.h b/src/Storages/ObjectStorage/DataLakes/Iceberg/ManifestFile.h index 9dd98799917..57f013c55e5 100644 --- a/src/Storages/ObjectStorage/DataLakes/Iceberg/ManifestFile.h +++ b/src/Storages/ObjectStorage/DataLakes/Iceberg/ManifestFile.h @@ -1,11 +1,11 @@ #pragma once -#include + #include "config.h" #if USE_AVRO -# include -# include +#include +#include namespace DB diff --git a/src/Storages/ObjectStorage/DataLakes/Iceberg/ManifestFileImpl.h b/src/Storages/ObjectStorage/DataLakes/Iceberg/ManifestFileImpl.h index 75618a6a44b..668f6604317 100644 --- a/src/Storages/ObjectStorage/DataLakes/Iceberg/ManifestFileImpl.h +++ b/src/Storages/ObjectStorage/DataLakes/Iceberg/ManifestFileImpl.h @@ -2,28 +2,8 @@ #if USE_AVRO - -# include "Storages/ObjectStorage/DataLakes/Iceberg/ManifestFile.h" -# include "Storages/ObjectStorage/DataLakes/Iceberg/Utils.h" - - -# include -# include -# include -# include -# include -# include - -# include -# include -# include - -# include -# include "DataTypes/DataTypeTuple.h" -# include "Formats/FormatSettings.h" - -# include - +#include "Storages/ObjectStorage/DataLakes/Iceberg/ManifestFile.h" +#include namespace DB { diff --git a/src/Storages/ObjectStorage/DataLakes/Iceberg/SchemaProcessor.cpp b/src/Storages/ObjectStorage/DataLakes/Iceberg/SchemaProcessor.cpp index 886626c6de0..1e2df7a4dbb 100644 --- a/src/Storages/ObjectStorage/DataLakes/Iceberg/SchemaProcessor.cpp +++ b/src/Storages/ObjectStorage/DataLakes/Iceberg/SchemaProcessor.cpp @@ -1,4 +1,4 @@ -#include "Storages/ObjectStorage/DataLakes/Iceberg/SchemaProcessor.h" +#include #include #include @@ -21,14 +21,18 @@ #include #include -namespace ErrorCodes -{ -extern const int LOGICAL_ERROR; -} +#include + namespace DB { +namespace ErrorCodes +{ +extern const int LOGICAL_ERROR; +extern const int BAD_ARGUMENTS; +extern const int UNSUPPORTED_METHOD; +} namespace Iceberg { diff --git a/src/Storages/ObjectStorage/DataLakes/Iceberg/SchemaProcessor.h b/src/Storages/ObjectStorage/DataLakes/Iceberg/SchemaProcessor.h index 755f8e940b4..657fd352da1 100644 --- a/src/Storages/ObjectStorage/DataLakes/Iceberg/SchemaProcessor.h +++ b/src/Storages/ObjectStorage/DataLakes/Iceberg/SchemaProcessor.h @@ -2,20 +2,19 @@ #include #include -#include #include "config.h" +#include #include -#include -#include -#include -#include +#include + #include #include #include +#include namespace DB { diff --git a/src/Storages/ObjectStorage/DataLakes/Iceberg/Snapshot.h b/src/Storages/ObjectStorage/DataLakes/Iceberg/Snapshot.h index d75ebd6b2ab..36a141095b4 100644 --- a/src/Storages/ObjectStorage/DataLakes/Iceberg/Snapshot.h +++ b/src/Storages/ObjectStorage/DataLakes/Iceberg/Snapshot.h @@ -2,7 +2,7 @@ #include "config.h" #if USE_AVRO -# include "Storages/ObjectStorage/DataLakes/Iceberg/ManifestFile.h" +#include "Storages/ObjectStorage/DataLakes/Iceberg/ManifestFile.h" namespace DB { diff --git a/src/Storages/ObjectStorage/DataLakes/Iceberg/Utils.cpp b/src/Storages/ObjectStorage/DataLakes/Iceberg/Utils.cpp index c5b1e5f65c8..22e0912e590 100644 --- a/src/Storages/ObjectStorage/DataLakes/Iceberg/Utils.cpp +++ b/src/Storages/ObjectStorage/DataLakes/Iceberg/Utils.cpp @@ -1,26 +1,10 @@ #include "config.h" -#if USE_AVRO /// StorageIceberg depending on Avro to parse metadata with Avro format. +#if USE_AVRO -# include "Storages/ObjectStorage/DataLakes/Iceberg/ManifestFile.h" - -# include -# include -# include -# include -# include -# include - -# include -# include -# include - -# include -# include "DataTypes/DataTypeTuple.h" -# include "Formats/FormatSettings.h" - -# include +#include +#include namespace DB { diff --git a/src/Storages/ObjectStorage/DataLakes/Iceberg/Utils.h b/src/Storages/ObjectStorage/DataLakes/Iceberg/Utils.h index 69515e50055..2e6161bd775 100644 --- a/src/Storages/ObjectStorage/DataLakes/Iceberg/Utils.h +++ b/src/Storages/ObjectStorage/DataLakes/Iceberg/Utils.h @@ -2,24 +2,9 @@ #include "config.h" -#if USE_AVRO /// StorageIceberg depending on Avro to parse metadata with Avro format. +#if USE_AVRO - -# include -# include -# include -# include -# include -# include - -# include -# include -# include - -# include -# include "Formats/FormatSettings.h" - -# include +#include namespace DB {