From 9256d370c8370477d9dbe7683c0e2cf84f3a6b82 Mon Sep 17 00:00:00 2001 From: kssenii Date: Mon, 9 Dec 2024 12:48:12 +0100 Subject: [PATCH] Resolve conflicts --- src/Core/Settings.cpp | 7 ------- src/Databases/Iceberg/DatabaseIceberg.cpp | 3 ++- src/Databases/Iceberg/DatabaseIcebergStorageType.h | 1 + src/Databases/Iceberg/RestCatalog.cpp | 10 +++++----- .../ObjectStorage/DataLakes/IcebergMetadata.cpp | 4 ---- src/Storages/ObjectStorage/DataLakes/IcebergMetadata.h | 7 ++----- src/Storages/ObjectStorage/StorageObjectStorage.cpp | 10 ++++++---- 7 files changed, 16 insertions(+), 26 deletions(-) diff --git a/src/Core/Settings.cpp b/src/Core/Settings.cpp index e4b3a28c60e..284a5572bec 100644 --- a/src/Core/Settings.cpp +++ b/src/Core/Settings.cpp @@ -5560,13 +5560,6 @@ Only available in ClickHouse Cloud. Exclude new data parts from SELECT queries u )", 0) \ DECLARE(Int64, prefer_warmed_unmerged_parts_seconds, 0, R"( Only available in ClickHouse Cloud. If a merged part is less than this many seconds old and is not pre-warmed (see cache_populated_by_fetch), but all its source parts are available and pre-warmed, SELECT queries will read from those parts instead. Only for ReplicatedMergeTree. Note that this only checks whether CacheWarmer processed the part; if the part was fetched into cache by something else, it'll still be considered cold until CacheWarmer gets to it; if it was warmed, then evicted from cache, it'll still be considered warm. -)", 0) \ - DECLARE(Bool, iceberg_engine_ignore_schema_evolution, false, R"( -Allow to ignore schema evolution in Iceberg table engine and read all data using schema specified by the user on table creation or latest schema parsed from metadata on table creation. - -:::note -Enabling this setting can lead to incorrect result as in case of evolved schema all data files will be read using the same schema. -::: )", 0) \ DECLARE(Bool, allow_experimental_database_iceberg, false, R"( Allow experimental database engine Iceberg diff --git a/src/Databases/Iceberg/DatabaseIceberg.cpp b/src/Databases/Iceberg/DatabaseIceberg.cpp index 62af8288933..7a49d11b19e 100644 --- a/src/Databases/Iceberg/DatabaseIceberg.cpp +++ b/src/Databases/Iceberg/DatabaseIceberg.cpp @@ -229,10 +229,11 @@ StoragePtr DatabaseIceberg::tryGetTable(const String & name, ContextPtr context_ storage_type = table_metadata.getStorageType(); const auto configuration = getConfiguration(storage_type); + auto storage_settings = std::make_unique(); /// with_table_structure = false: because there will be /// no table structure in table definition AST. - StorageObjectStorage::Configuration::initialize(*configuration, args, context_, /* with_table_structure */false); + StorageObjectStorage::Configuration::initialize(*configuration, args, context_, /* with_table_structure */false, std::move(storage_settings)); return std::make_shared( configuration, diff --git a/src/Databases/Iceberg/DatabaseIcebergStorageType.h b/src/Databases/Iceberg/DatabaseIcebergStorageType.h index cc3c8f8cb1d..9b8dc3a0633 100644 --- a/src/Databases/Iceberg/DatabaseIcebergStorageType.h +++ b/src/Databases/Iceberg/DatabaseIcebergStorageType.h @@ -1,3 +1,4 @@ +#pragma once #include namespace DB diff --git a/src/Databases/Iceberg/RestCatalog.cpp b/src/Databases/Iceberg/RestCatalog.cpp index 460fce78696..ef947e05f7a 100644 --- a/src/Databases/Iceberg/RestCatalog.cpp +++ b/src/Databases/Iceberg/RestCatalog.cpp @@ -577,11 +577,11 @@ bool RestCatalog::getTableMetadataImpl( if (result.requiresSchema()) { - const auto & settings = getContext()->getSettingsRef(); - int format_version = metadata_object->getValue("format-version"); - auto schema = DB::IcebergMetadata::parseTableSchema( - metadata_object, log, format_version, settings[DB::Setting::iceberg_engine_ignore_schema_evolution]).first; - result.setSchema(schema); + // int format_version = metadata_object->getValue("format-version"); + auto schema_processor = DB::IcebergSchemaProcessor(); + auto id = DB::IcebergMetadata::parseTableSchema(metadata_object, schema_processor, log); + auto schema = schema_processor.getClickhouseTableSchemaById(id); + result.setSchema(*schema); } if (result.requiresCredentials() && object->has("config")) diff --git a/src/Storages/ObjectStorage/DataLakes/IcebergMetadata.cpp b/src/Storages/ObjectStorage/DataLakes/IcebergMetadata.cpp index 293969d1c7e..8550717d4c7 100644 --- a/src/Storages/ObjectStorage/DataLakes/IcebergMetadata.cpp +++ b/src/Storages/ObjectStorage/DataLakes/IcebergMetadata.cpp @@ -52,9 +52,6 @@ namespace ErrorCodes extern const int LOGICAL_ERROR; } -Int32 parseTableSchema( - const Poco::JSON::Object::Ptr & metadata_object, IcebergSchemaProcessor & schema_processor, LoggerPtr metadata_logger); - IcebergMetadata::IcebergMetadata( ObjectStoragePtr object_storage_, ConfigurationObserverPtr configuration_, @@ -125,7 +122,6 @@ bool operator!=(const Poco::JSON::Object & first, const Poco::JSON::Object & sec { return !(first == second); } -} DataTypePtr IcebergSchemaProcessor::getSimpleType(const String & type_name) diff --git a/src/Storages/ObjectStorage/DataLakes/IcebergMetadata.h b/src/Storages/ObjectStorage/DataLakes/IcebergMetadata.h index ac55d90badc..d4e70030012 100644 --- a/src/Storages/ObjectStorage/DataLakes/IcebergMetadata.h +++ b/src/Storages/ObjectStorage/DataLakes/IcebergMetadata.h @@ -207,11 +207,8 @@ public: bool supportsExternalMetadataChange() const override { return true; } - static std::pair parseTableSchema( - const Poco::JSON::Object::Ptr & metadata_object, - LoggerPtr metadata_logger, - int format_version, - bool ignore_schema_evolution); + static Int32 parseTableSchema( + const Poco::JSON::Object::Ptr & metadata_object, IcebergSchemaProcessor & schema_processor, LoggerPtr metadata_logger); private: mutable std::unordered_map schema_id_by_data_file; diff --git a/src/Storages/ObjectStorage/StorageObjectStorage.cpp b/src/Storages/ObjectStorage/StorageObjectStorage.cpp index ba2f9095887..2091e6185b4 100644 --- a/src/Storages/ObjectStorage/StorageObjectStorage.cpp +++ b/src/Storages/ObjectStorage/StorageObjectStorage.cpp @@ -104,10 +104,12 @@ StorageObjectStorage::StorageObjectStorage( try { if (!lazy_init) - if (configuration->hasExternalDynamicMetadata()) - configuration->updateAndGetCurrentSchema(object_storage, context); - else - configuration->update(object_storage, context); + { + if (configuration->hasExternalDynamicMetadata()) + configuration->updateAndGetCurrentSchema(object_storage, context); + else + configuration->update(object_storage, context); + } } catch (...) {