From eb9d5705a9055c505cc5aad169118927e4d321d4 Mon Sep 17 00:00:00 2001 From: kssenii Date: Tue, 3 Dec 2024 12:22:22 +0100 Subject: [PATCH] Correct merge --- src/Databases/Iceberg/RestCatalog.cpp | 11 ++++++++++- src/Interpreters/Context.h | 3 +-- .../ObjectStorage/DataLakes/IcebergMetadata.cpp | 2 +- .../ObjectStorage/DataLakes/IcebergMetadata.h | 1 + 4 files changed, 13 insertions(+), 4 deletions(-) diff --git a/src/Databases/Iceberg/RestCatalog.cpp b/src/Databases/Iceberg/RestCatalog.cpp index a3d8a2a2d5a..262ac53e99f 100644 --- a/src/Databases/Iceberg/RestCatalog.cpp +++ b/src/Databases/Iceberg/RestCatalog.cpp @@ -2,6 +2,7 @@ #include #include +#include #include #include #include @@ -27,6 +28,11 @@ namespace DB::ErrorCodes extern const int NOT_IMPLEMENTED; } +namespace DB::Setting +{ + extern const SettingsBool iceberg_engine_ignore_schema_evolution; +} + namespace Iceberg { @@ -571,8 +577,11 @@ bool RestCatalog::getTableMetadataImpl( if (result.requiresSchema()) { + const auto & settings = getContext()->getSettingsRef(); int format_version = metadata_object->getValue("format-version"); - result.setSchema(DB::IcebergMetadata::parseTableSchema(metadata_object, format_version, true).first); + auto schema = DB::IcebergMetadata::parseTableSchema( + metadata_object, log, format_version, settings[DB::Setting::iceberg_engine_ignore_schema_evolution]).first; + result.setSchema(schema); } if (result.requiresCredentials() && object->has("config")) diff --git a/src/Interpreters/Context.h b/src/Interpreters/Context.h index 6ddb3c1c0f5..9286de76427 100644 --- a/src/Interpreters/Context.h +++ b/src/Interpreters/Context.h @@ -1077,8 +1077,6 @@ public: void clearMarkCache() const; ThreadPool & getLoadMarksThreadpool() const; - ThreadPool & getIcebergCatalogThreadpool() const; - void setPrimaryIndexCache(const String & cache_policy, size_t max_cache_size_in_bytes, double size_ratio); void updatePrimaryIndexCacheConfiguration(const Poco::Util::AbstractConfiguration & config); std::shared_ptr getPrimaryIndexCache() const; @@ -1125,6 +1123,7 @@ public: size_t getPrefetchThreadpoolSize() const; ThreadPool & getBuildVectorSimilarityIndexThreadPool() const; + ThreadPool & getIcebergCatalogThreadpool() const; /// Settings for MergeTree background tasks stored in config.xml BackgroundTaskSchedulingSettings getBackgroundProcessingTaskSchedulingSettings() const; diff --git a/src/Storages/ObjectStorage/DataLakes/IcebergMetadata.cpp b/src/Storages/ObjectStorage/DataLakes/IcebergMetadata.cpp index a93af4ee780..7b2ad165972 100644 --- a/src/Storages/ObjectStorage/DataLakes/IcebergMetadata.cpp +++ b/src/Storages/ObjectStorage/DataLakes/IcebergMetadata.cpp @@ -321,7 +321,7 @@ parseTableSchemaV1Method(const Poco::JSON::Object::Ptr & metadata_object, bool i return {schema, current_schema_id}; } -std::pair parseTableSchema( +std::pair IcebergMetadata::parseTableSchema( const Poco::JSON::Object::Ptr & metadata_object, LoggerPtr metadata_logger, int format_version, bool ignore_schema_evolution) { Poco::JSON::Object::Ptr schema; diff --git a/src/Storages/ObjectStorage/DataLakes/IcebergMetadata.h b/src/Storages/ObjectStorage/DataLakes/IcebergMetadata.h index b28de471b40..5f62dbfb2ef 100644 --- a/src/Storages/ObjectStorage/DataLakes/IcebergMetadata.h +++ b/src/Storages/ObjectStorage/DataLakes/IcebergMetadata.h @@ -98,6 +98,7 @@ public: static std::pair parseTableSchema( const Poco::JSON::Object::Ptr & metadata_object, + LoggerPtr metadata_logger, int format_version, bool ignore_schema_evolution);