From bc93760c5ac9ddce2028fbdff7eff62114d22c98 Mon Sep 17 00:00:00 2001
From: divanik <dan.ivanik@clickhouse.com>
Date: Mon, 9 Dec 2024 12:04:31 +0000
Subject: [PATCH] Deal with includes

---
 .../DataLakes/DataLakeConfiguration.h         |  6 +-
 .../DataLakes/Iceberg/IcebergMetadata.cpp     | 59 +++++++------------
 .../DataLakes/Iceberg/IcebergMetadata.h       | 27 ++++-----
 .../DataLakes/Iceberg/ManifestFile.cpp        | 22 ++-----
 .../DataLakes/Iceberg/ManifestFile.h          |  6 +-
 .../DataLakes/Iceberg/ManifestFileImpl.h      | 24 +-------
 .../DataLakes/Iceberg/SchemaProcessor.cpp     | 14 +++--
 .../DataLakes/Iceberg/SchemaProcessor.h       |  9 ++-
 .../DataLakes/Iceberg/Snapshot.h              |  2 +-
 .../ObjectStorage/DataLakes/Iceberg/Utils.cpp | 22 +------
 .../ObjectStorage/DataLakes/Iceberg/Utils.h   | 19 +-----
 11 files changed, 65 insertions(+), 145 deletions(-)

diff --git a/src/Storages/ObjectStorage/DataLakes/DataLakeConfiguration.h b/src/Storages/ObjectStorage/DataLakes/DataLakeConfiguration.h
index 52b5a553de1..29bd7b992d5 100644
--- a/src/Storages/ObjectStorage/DataLakes/DataLakeConfiguration.h
+++ b/src/Storages/ObjectStorage/DataLakes/DataLakeConfiguration.h
@@ -11,7 +11,6 @@
 #include <Storages/ObjectStorage/S3/Configuration.h>
 #include <Storages/ObjectStorage/StorageObjectStorage.h>
 #include <Storages/StorageFactory.h>
-#include "Common/DateLUT.h"
 #include <Common/logger_useful.h>
 #include "Storages/ColumnsDescription.h"
 
@@ -47,9 +46,11 @@ public:
     {
         BaseStorageConfiguration::update(object_storage, local_context);
 
+        bool existed = current_metadata != nullptr;
+
         if (updateMetadataObjectIfNeeded(object_storage, local_context))
         {
-            if (hasExternalDynamicMetadata())
+            if (hasExternalDynamicMetadata() && existed)
             {
                 throw Exception(
                     ErrorCodes::FORMAT_VERSION_TOO_OLD,
@@ -103,6 +104,7 @@ public:
             BaseStorageConfiguration::setPaths(current_metadata->getDataFiles());
             BaseStorageConfiguration::setPartitionColumns(current_metadata->getPartitionColumns());
         }
+
         return ColumnsDescription{current_metadata->getTableSchema()};
     }
 
diff --git a/src/Storages/ObjectStorage/DataLakes/Iceberg/IcebergMetadata.cpp b/src/Storages/ObjectStorage/DataLakes/Iceberg/IcebergMetadata.cpp
index 1d5da6a1a09..94b646f30f5 100644
--- a/src/Storages/ObjectStorage/DataLakes/Iceberg/IcebergMetadata.cpp
+++ b/src/Storages/ObjectStorage/DataLakes/Iceberg/IcebergMetadata.cpp
@@ -1,45 +1,25 @@
-#include <memory>
-#include <Poco/Logger.h>
-#include "Common/Config/ConfigProcessor.h"
-#include "Common/DateLUT.h"
-#include "Storages/ObjectStorage/DataLakes/Iceberg/ManifestFile.h"
 #include "config.h"
 
 #if USE_AVRO
 
-#    include <Columns/ColumnString.h>
-#    include <Columns/ColumnTuple.h>
-#    include <Columns/IColumn.h>
-#    include <Core/Settings.h>
-#    include <DataTypes/DataTypeArray.h>
-#    include <DataTypes/DataTypeDate.h>
-#    include <DataTypes/DataTypeDateTime64.h>
-#    include <DataTypes/DataTypeFactory.h>
-#    include <DataTypes/DataTypeFixedString.h>
-#    include <DataTypes/DataTypeMap.h>
-#    include <DataTypes/DataTypeNullable.h>
-#    include <DataTypes/DataTypeString.h>
-#    include <DataTypes/DataTypeTuple.h>
-#    include <DataTypes/DataTypeUUID.h>
-#    include <DataTypes/DataTypesDecimal.h>
-#    include <DataTypes/DataTypesNumber.h>
-#    include <Formats/FormatFactory.h>
-#    include <IO/ReadBufferFromFileBase.h>
-#    include <IO/ReadBufferFromString.h>
-#    include <IO/ReadHelpers.h>
-#    include <Processors/Formats/Impl/AvroRowInputFormat.h>
-#    include <Storages/ObjectStorage/DataLakes/Common.h>
-#    include <Storages/ObjectStorage/StorageObjectStorageSource.h>
-#    include <Common/logger_useful.h>
+#include <Columns/ColumnString.h>
+#include <Columns/ColumnTuple.h>
+#include <Columns/IColumn.h>
+#include <Core/Settings.h>
+#include <Formats/FormatFactory.h>
+#include <IO/ReadBufferFromFileBase.h>
+#include <IO/ReadBufferFromString.h>
+#include <IO/ReadHelpers.h>
+#include <Processors/Formats/Impl/AvroRowInputFormat.h>
+#include <Storages/ObjectStorage/DataLakes/Common.h>
+#include <Storages/ObjectStorage/StorageObjectStorageSource.h>
+#include <Common/logger_useful.h>
 
-#    include <filesystem>
-#    include <sstream>
+#include "Storages/ObjectStorage/DataLakes/Iceberg/IcebergMetadata.h"
+#include "Storages/ObjectStorage/DataLakes/Iceberg/Utils.h"
 
-#    include "Storages/ObjectStorage/DataLakes/Iceberg/IcebergMetadata.h"
-#    include "Storages/ObjectStorage/DataLakes/Iceberg/Utils.h"
-
-#    include "Storages/ObjectStorage/DataLakes/Iceberg/ManifestFileImpl.h"
-#    include "Storages/ObjectStorage/DataLakes/Iceberg/Snapshot.h"
+#include "Storages/ObjectStorage/DataLakes/Iceberg/ManifestFileImpl.h"
+#include "Storages/ObjectStorage/DataLakes/Iceberg/Snapshot.h"
 
 namespace DB
 {
@@ -255,12 +235,15 @@ bool IcebergMetadata::update(const ContextPtr & local_context)
 
     const auto [metadata_version, metadata_file_path] = getMetadataFileAndVersion(object_storage, *configuration_ptr);
 
+    if (metadata_version == current_metadata_version)
+        return false;
+
+    current_metadata_version = metadata_version;
+
     auto metadata_object = readJson(metadata_file_path, local_context);
 
     chassert(format_version == metadata_object->getValue<int>("format-version"));
 
-    if (metadata_version == current_metadata_version)
-        return false;
 
     auto manifest_list_file = getRelevantManifestList(metadata_object);
     if (manifest_list_file && (!current_snapshot.has_value() || (manifest_list_file.value() != current_snapshot->getName())))
diff --git a/src/Storages/ObjectStorage/DataLakes/Iceberg/IcebergMetadata.h b/src/Storages/ObjectStorage/DataLakes/Iceberg/IcebergMetadata.h
index b4113b6dc26..c407f22dea0 100644
--- a/src/Storages/ObjectStorage/DataLakes/Iceberg/IcebergMetadata.h
+++ b/src/Storages/ObjectStorage/DataLakes/Iceberg/IcebergMetadata.h
@@ -3,24 +3,21 @@
 
 #if USE_AVRO /// StorageIceberg depending on Avro to parse metadata with Avro format.
 
-#    include <Core/Types.h>
-#    include <Disks/ObjectStorages/IObjectStorage.h>
-#    include <Interpreters/Context_fwd.h>
-#    include <Storages/ObjectStorage/DataLakes/IDataLakeMetadata.h>
-#    include <Storages/ObjectStorage/StorageObjectStorage.h>
+#include <Core/Types.h>
+#include <Disks/ObjectStorages/IObjectStorage.h>
+#include <Interpreters/Context_fwd.h>
+#include <Storages/ObjectStorage/DataLakes/IDataLakeMetadata.h>
+#include <Storages/ObjectStorage/StorageObjectStorage.h>
 
-#    include <Poco/JSON/Array.h>
-#    include <Poco/JSON/Object.h>
-#    include <Poco/JSON/Parser.h>
+#include <Poco/JSON/Array.h>
+#include <Poco/JSON/Object.h>
+#include <Poco/JSON/Parser.h>
 
-#    include "Storages/ObjectStorage/DataLakes/Iceberg/ManifestFile.h"
-#    include "Storages/ObjectStorage/DataLakes/Iceberg/SchemaProcessor.h"
-#    include "Storages/ObjectStorage/DataLakes/Iceberg/Snapshot.h"
+#include "Storages/ObjectStorage/DataLakes/Iceberg/ManifestFile.h"
+#include "Storages/ObjectStorage/DataLakes/Iceberg/SchemaProcessor.h"
+#include "Storages/ObjectStorage/DataLakes/Iceberg/Snapshot.h"
 
-#    include <memory>
-#    include <mutex>
-#    include <optional>
-#    include <unordered_map>
+#include <unordered_map>
 
 namespace DB
 {
diff --git a/src/Storages/ObjectStorage/DataLakes/Iceberg/ManifestFile.cpp b/src/Storages/ObjectStorage/DataLakes/Iceberg/ManifestFile.cpp
index 1819d8bf599..a34422c72ca 100644
--- a/src/Storages/ObjectStorage/DataLakes/Iceberg/ManifestFile.cpp
+++ b/src/Storages/ObjectStorage/DataLakes/Iceberg/ManifestFile.cpp
@@ -3,26 +3,12 @@
 #if USE_AVRO
 
 
-#    include "Storages/ObjectStorage/DataLakes/Iceberg/ManifestFileImpl.h"
-#    include "Storages/ObjectStorage/DataLakes/Iceberg/Utils.h"
+#include "Storages/ObjectStorage/DataLakes/Iceberg/ManifestFileImpl.h"
+#include "Storages/ObjectStorage/DataLakes/Iceberg/Utils.h"
 
 
-#    include <Core/Types.h>
-#    include <Disks/ObjectStorages/IObjectStorage.h>
-#    include <Interpreters/Context_fwd.h>
-#    include <Storages/ObjectStorage/DataLakes/IDataLakeMetadata.h>
-#    include <Storages/ObjectStorage/StorageObjectStorage.h>
-#    include <DataFile.hh>
-
-#    include <Poco/JSON/Array.h>
-#    include <Poco/JSON/Object.h>
-#    include <Poco/JSON/Parser.h>
-
-#    include <Common/Exception.h>
-#    include "DataTypes/DataTypeTuple.h"
-#    include "Formats/FormatSettings.h"
-
-#    include <Processors/Formats/Impl/AvroRowInputFormat.h>
+#include <Columns/ColumnTuple.h>
+#include "DataTypes/DataTypeTuple.h"
 
 
 namespace DB
diff --git a/src/Storages/ObjectStorage/DataLakes/Iceberg/ManifestFile.h b/src/Storages/ObjectStorage/DataLakes/Iceberg/ManifestFile.h
index 9dd98799917..57f013c55e5 100644
--- a/src/Storages/ObjectStorage/DataLakes/Iceberg/ManifestFile.h
+++ b/src/Storages/ObjectStorage/DataLakes/Iceberg/ManifestFile.h
@@ -1,11 +1,11 @@
 #pragma once
-#include <memory>
+
 #include "config.h"
 
 #if USE_AVRO
 
-#    include <cstdint>
-#    include <Common/Exception.h>
+#include <cstdint>
+#include <Common/Exception.h>
 
 
 namespace DB
diff --git a/src/Storages/ObjectStorage/DataLakes/Iceberg/ManifestFileImpl.h b/src/Storages/ObjectStorage/DataLakes/Iceberg/ManifestFileImpl.h
index 75618a6a44b..668f6604317 100644
--- a/src/Storages/ObjectStorage/DataLakes/Iceberg/ManifestFileImpl.h
+++ b/src/Storages/ObjectStorage/DataLakes/Iceberg/ManifestFileImpl.h
@@ -2,28 +2,8 @@
 
 #if USE_AVRO
 
-
-#    include "Storages/ObjectStorage/DataLakes/Iceberg/ManifestFile.h"
-#    include "Storages/ObjectStorage/DataLakes/Iceberg/Utils.h"
-
-
-#    include <Core/Types.h>
-#    include <Disks/ObjectStorages/IObjectStorage.h>
-#    include <Interpreters/Context_fwd.h>
-#    include <Storages/ObjectStorage/DataLakes/IDataLakeMetadata.h>
-#    include <Storages/ObjectStorage/StorageObjectStorage.h>
-#    include <DataFile.hh>
-
-#    include <Poco/JSON/Array.h>
-#    include <Poco/JSON/Object.h>
-#    include <Poco/JSON/Parser.h>
-
-#    include <Common/Exception.h>
-#    include "DataTypes/DataTypeTuple.h"
-#    include "Formats/FormatSettings.h"
-
-#    include <Processors/Formats/Impl/AvroRowInputFormat.h>
-
+#include "Storages/ObjectStorage/DataLakes/Iceberg/ManifestFile.h"
+#include <Processors/Formats/Impl/AvroRowInputFormat.h>
 
 namespace DB
 {
diff --git a/src/Storages/ObjectStorage/DataLakes/Iceberg/SchemaProcessor.cpp b/src/Storages/ObjectStorage/DataLakes/Iceberg/SchemaProcessor.cpp
index 886626c6de0..1e2df7a4dbb 100644
--- a/src/Storages/ObjectStorage/DataLakes/Iceberg/SchemaProcessor.cpp
+++ b/src/Storages/ObjectStorage/DataLakes/Iceberg/SchemaProcessor.cpp
@@ -1,4 +1,4 @@
-#include "Storages/ObjectStorage/DataLakes/Iceberg/SchemaProcessor.h"
+#include <Storages/ObjectStorage/DataLakes/Iceberg/SchemaProcessor.h>
 
 #include <Poco/JSON/Array.h>
 #include <Poco/JSON/Object.h>
@@ -21,14 +21,18 @@
 #include <DataTypes/DataTypesNumber.h>
 #include <Formats/FormatFactory.h>
 
-namespace ErrorCodes
-{
-extern const int LOGICAL_ERROR;
-}
+#include <IO/ReadHelpers.h>
+
 
 namespace DB
 {
 
+namespace ErrorCodes
+{
+extern const int LOGICAL_ERROR;
+extern const int BAD_ARGUMENTS;
+extern const int UNSUPPORTED_METHOD;
+}
 namespace Iceberg
 {
 
diff --git a/src/Storages/ObjectStorage/DataLakes/Iceberg/SchemaProcessor.h b/src/Storages/ObjectStorage/DataLakes/Iceberg/SchemaProcessor.h
index 755f8e940b4..657fd352da1 100644
--- a/src/Storages/ObjectStorage/DataLakes/Iceberg/SchemaProcessor.h
+++ b/src/Storages/ObjectStorage/DataLakes/Iceberg/SchemaProcessor.h
@@ -2,20 +2,19 @@
 
 #include <memory>
 #include <mutex>
-#include <unordered_map>
 #include "config.h"
 
 
+#include <Core/NamesAndTypes.h>
 #include <Core/Types.h>
-#include <Disks/ObjectStorages/IObjectStorage.h>
-#include <Interpreters/Context_fwd.h>
-#include <Storages/ObjectStorage/DataLakes/IDataLakeMetadata.h>
-#include <Storages/ObjectStorage/StorageObjectStorage.h>
+#include <Interpreters/ActionsDAG.h>
+
 
 #include <Poco/JSON/Array.h>
 #include <Poco/JSON/Object.h>
 #include <Poco/JSON/Parser.h>
 
+#include <unordered_map>
 namespace DB
 {
 
diff --git a/src/Storages/ObjectStorage/DataLakes/Iceberg/Snapshot.h b/src/Storages/ObjectStorage/DataLakes/Iceberg/Snapshot.h
index d75ebd6b2ab..36a141095b4 100644
--- a/src/Storages/ObjectStorage/DataLakes/Iceberg/Snapshot.h
+++ b/src/Storages/ObjectStorage/DataLakes/Iceberg/Snapshot.h
@@ -2,7 +2,7 @@
 #include "config.h"
 
 #if USE_AVRO
-#    include "Storages/ObjectStorage/DataLakes/Iceberg/ManifestFile.h"
+#include "Storages/ObjectStorage/DataLakes/Iceberg/ManifestFile.h"
 
 namespace DB
 {
diff --git a/src/Storages/ObjectStorage/DataLakes/Iceberg/Utils.cpp b/src/Storages/ObjectStorage/DataLakes/Iceberg/Utils.cpp
index c5b1e5f65c8..22e0912e590 100644
--- a/src/Storages/ObjectStorage/DataLakes/Iceberg/Utils.cpp
+++ b/src/Storages/ObjectStorage/DataLakes/Iceberg/Utils.cpp
@@ -1,26 +1,10 @@
 
 #include "config.h"
 
-#if USE_AVRO /// StorageIceberg depending on Avro to parse metadata with Avro format.
+#if USE_AVRO
 
-#    include "Storages/ObjectStorage/DataLakes/Iceberg/ManifestFile.h"
-
-#    include <Core/Types.h>
-#    include <Disks/ObjectStorages/IObjectStorage.h>
-#    include <Interpreters/Context_fwd.h>
-#    include <Storages/ObjectStorage/DataLakes/IDataLakeMetadata.h>
-#    include <Storages/ObjectStorage/StorageObjectStorage.h>
-#    include <DataFile.hh>
-
-#    include <Poco/JSON/Array.h>
-#    include <Poco/JSON/Object.h>
-#    include <Poco/JSON/Parser.h>
-
-#    include <Common/Exception.h>
-#    include "DataTypes/DataTypeTuple.h"
-#    include "Formats/FormatSettings.h"
-
-#    include <Processors/Formats/Impl/AvroRowInputFormat.h>
+#include <Processors/Formats/Impl/AvroRowInputFormat.h>
+#include <Storages/ObjectStorage/DataLakes/Iceberg/Utils.h>
 
 namespace DB
 {
diff --git a/src/Storages/ObjectStorage/DataLakes/Iceberg/Utils.h b/src/Storages/ObjectStorage/DataLakes/Iceberg/Utils.h
index 69515e50055..2e6161bd775 100644
--- a/src/Storages/ObjectStorage/DataLakes/Iceberg/Utils.h
+++ b/src/Storages/ObjectStorage/DataLakes/Iceberg/Utils.h
@@ -2,24 +2,9 @@
 
 #include "config.h"
 
-#if USE_AVRO /// StorageIceberg depending on Avro to parse metadata with Avro format.
+#if USE_AVRO
 
-
-#    include <Core/Types.h>
-#    include <Disks/ObjectStorages/IObjectStorage.h>
-#    include <Interpreters/Context_fwd.h>
-#    include <Storages/ObjectStorage/DataLakes/IDataLakeMetadata.h>
-#    include <Storages/ObjectStorage/StorageObjectStorage.h>
-#    include <DataFile.hh>
-
-#    include <Poco/JSON/Array.h>
-#    include <Poco/JSON/Object.h>
-#    include <Poco/JSON/Parser.h>
-
-#    include <Common/Exception.h>
-#    include "Formats/FormatSettings.h"
-
-#    include <Processors/Formats/Impl/AvroRowInputFormat.h>
+#include <Processors/Formats/Impl/AvroRowInputFormat.h>
 
 namespace DB
 {