thevar1able/ClickHouse
1
0
Fork 0
mirror of https://github.com/ClickHouse/ClickHouse.git synced 2024-12-18 20:32:43 +00:00
Code Issues Packages Projects Releases Wiki Activity

Deal with includes

Browse Source
This commit is contained in:
divanik 2024-12-09 12:04:31 +00:00
parent c28ae71b3f
commit bc93760c5a
11 changed files with 65 additions and 145 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
src/Storages/ObjectStorage/DataLakes/DataLakeConfiguration.h
View File

@ -11,7 +11,6 @@
#include <Storages/ObjectStorage/S3/Configuration.h> #include <Storages/ObjectStorage/S3/Configuration.h>
#include <Storages/ObjectStorage/StorageObjectStorage.h> #include <Storages/ObjectStorage/StorageObjectStorage.h>
#include <Storages/StorageFactory.h> #include <Storages/StorageFactory.h>
#include "Common/DateLUT.h"
#include <Common/logger_useful.h> #include <Common/logger_useful.h>
#include "Storages/ColumnsDescription.h" #include "Storages/ColumnsDescription.h"
@ -47,9 +46,11 @@ public:
{ {
BaseStorageConfiguration::update(object_storage, local_context); BaseStorageConfiguration::update(object_storage, local_context);
bool existed = current_metadata != nullptr;
if (updateMetadataObjectIfNeeded(object_storage, local_context)) if (updateMetadataObjectIfNeeded(object_storage, local_context))
{ {
if (hasExternalDynamicMetadata()) if (hasExternalDynamicMetadata() && existed)
{ {
throw Exception( throw Exception(
ErrorCodes::FORMAT_VERSION_TOO_OLD, ErrorCodes::FORMAT_VERSION_TOO_OLD,
@ -103,6 +104,7 @@ public:
BaseStorageConfiguration::setPaths(current_metadata->getDataFiles()); BaseStorageConfiguration::setPaths(current_metadata->getDataFiles());
BaseStorageConfiguration::setPartitionColumns(current_metadata->getPartitionColumns()); BaseStorageConfiguration::setPartitionColumns(current_metadata->getPartitionColumns());
} }
return ColumnsDescription{current_metadata->getTableSchema()}; return ColumnsDescription{current_metadata->getTableSchema()};
} }

59
src/Storages/ObjectStorage/DataLakes/Iceberg/IcebergMetadata.cpp
View File

@ -1,45 +1,25 @@
#include <memory>
#include <Poco/Logger.h>
#include "Common/Config/ConfigProcessor.h"
#include "Common/DateLUT.h"
#include "Storages/ObjectStorage/DataLakes/Iceberg/ManifestFile.h"
#include "config.h" #include "config.h"
#if USE_AVRO #if USE_AVRO
# include <Columns/ColumnString.h> #include <Columns/ColumnString.h>
# include <Columns/ColumnTuple.h> #include <Columns/ColumnTuple.h>
# include <Columns/IColumn.h> #include <Columns/IColumn.h>
# include <Core/Settings.h> #include <Core/Settings.h>
# include <DataTypes/DataTypeArray.h> #include <Formats/FormatFactory.h>
# include <DataTypes/DataTypeDate.h> #include <IO/ReadBufferFromFileBase.h>
# include <DataTypes/DataTypeDateTime64.h> #include <IO/ReadBufferFromString.h>
# include <DataTypes/DataTypeFactory.h> #include <IO/ReadHelpers.h>
# include <DataTypes/DataTypeFixedString.h> #include <Processors/Formats/Impl/AvroRowInputFormat.h>
# include <DataTypes/DataTypeMap.h> #include <Storages/ObjectStorage/DataLakes/Common.h>
# include <DataTypes/DataTypeNullable.h> #include <Storages/ObjectStorage/StorageObjectStorageSource.h>
# include <DataTypes/DataTypeString.h> #include <Common/logger_useful.h>
# include <DataTypes/DataTypeTuple.h>
# include <DataTypes/DataTypeUUID.h>
# include <DataTypes/DataTypesDecimal.h>
# include <DataTypes/DataTypesNumber.h>
# include <Formats/FormatFactory.h>
# include <IO/ReadBufferFromFileBase.h>
# include <IO/ReadBufferFromString.h>
# include <IO/ReadHelpers.h>
# include <Processors/Formats/Impl/AvroRowInputFormat.h>
# include <Storages/ObjectStorage/DataLakes/Common.h>
# include <Storages/ObjectStorage/StorageObjectStorageSource.h>
# include <Common/logger_useful.h>
# include <filesystem> #include "Storages/ObjectStorage/DataLakes/Iceberg/IcebergMetadata.h"
# include <sstream> #include "Storages/ObjectStorage/DataLakes/Iceberg/Utils.h"
# include "Storages/ObjectStorage/DataLakes/Iceberg/IcebergMetadata.h" #include "Storages/ObjectStorage/DataLakes/Iceberg/ManifestFileImpl.h"
# include "Storages/ObjectStorage/DataLakes/Iceberg/Utils.h" #include "Storages/ObjectStorage/DataLakes/Iceberg/Snapshot.h"
# include "Storages/ObjectStorage/DataLakes/Iceberg/ManifestFileImpl.h"
# include "Storages/ObjectStorage/DataLakes/Iceberg/Snapshot.h"
namespace DB namespace DB
{ {
@ -255,12 +235,15 @@ bool IcebergMetadata::update(const ContextPtr & local_context)
const auto [metadata_version, metadata_file_path] = getMetadataFileAndVersion(object_storage, *configuration_ptr); const auto [metadata_version, metadata_file_path] = getMetadataFileAndVersion(object_storage, *configuration_ptr);
if (metadata_version == current_metadata_version)
return false;
current_metadata_version = metadata_version;
auto metadata_object = readJson(metadata_file_path, local_context); auto metadata_object = readJson(metadata_file_path, local_context);
chassert(format_version == metadata_object->getValue<int>("format-version")); chassert(format_version == metadata_object->getValue<int>("format-version"));
if (metadata_version == current_metadata_version)
return false;
auto manifest_list_file = getRelevantManifestList(metadata_object); auto manifest_list_file = getRelevantManifestList(metadata_object);
if (manifest_list_file && (!current_snapshot.has_value() || (manifest_list_file.value() != current_snapshot->getName()))) if (manifest_list_file && (!current_snapshot.has_value() || (manifest_list_file.value() != current_snapshot->getName())))

27
src/Storages/ObjectStorage/DataLakes/Iceberg/IcebergMetadata.h
View File

@ -3,24 +3,21 @@
#if USE_AVRO /// StorageIceberg depending on Avro to parse metadata with Avro format. #if USE_AVRO /// StorageIceberg depending on Avro to parse metadata with Avro format.
# include <Core/Types.h> #include <Core/Types.h>
# include <Disks/ObjectStorages/IObjectStorage.h> #include <Disks/ObjectStorages/IObjectStorage.h>
# include <Interpreters/Context_fwd.h> #include <Interpreters/Context_fwd.h>
# include <Storages/ObjectStorage/DataLakes/IDataLakeMetadata.h> #include <Storages/ObjectStorage/DataLakes/IDataLakeMetadata.h>
# include <Storages/ObjectStorage/StorageObjectStorage.h> #include <Storages/ObjectStorage/StorageObjectStorage.h>
# include <Poco/JSON/Array.h> #include <Poco/JSON/Array.h>
# include <Poco/JSON/Object.h> #include <Poco/JSON/Object.h>
# include <Poco/JSON/Parser.h> #include <Poco/JSON/Parser.h>
# include "Storages/ObjectStorage/DataLakes/Iceberg/ManifestFile.h" #include "Storages/ObjectStorage/DataLakes/Iceberg/ManifestFile.h"
# include "Storages/ObjectStorage/DataLakes/Iceberg/SchemaProcessor.h" #include "Storages/ObjectStorage/DataLakes/Iceberg/SchemaProcessor.h"
# include "Storages/ObjectStorage/DataLakes/Iceberg/Snapshot.h" #include "Storages/ObjectStorage/DataLakes/Iceberg/Snapshot.h"
# include <memory> #include <unordered_map>
# include <mutex>
# include <optional>
# include <unordered_map>
namespace DB namespace DB
{ {

22
src/Storages/ObjectStorage/DataLakes/Iceberg/ManifestFile.cpp
View File

@ -3,26 +3,12 @@
#if USE_AVRO #if USE_AVRO
# include "Storages/ObjectStorage/DataLakes/Iceberg/ManifestFileImpl.h" #include "Storages/ObjectStorage/DataLakes/Iceberg/ManifestFileImpl.h"
# include "Storages/ObjectStorage/DataLakes/Iceberg/Utils.h" #include "Storages/ObjectStorage/DataLakes/Iceberg/Utils.h"
# include <Core/Types.h> #include <Columns/ColumnTuple.h>
# include <Disks/ObjectStorages/IObjectStorage.h> #include "DataTypes/DataTypeTuple.h"
# include <Interpreters/Context_fwd.h>
# include <Storages/ObjectStorage/DataLakes/IDataLakeMetadata.h>
# include <Storages/ObjectStorage/StorageObjectStorage.h>
# include <DataFile.hh>
# include <Poco/JSON/Array.h>
# include <Poco/JSON/Object.h>
# include <Poco/JSON/Parser.h>
# include <Common/Exception.h>
# include "DataTypes/DataTypeTuple.h"
# include "Formats/FormatSettings.h"
# include <Processors/Formats/Impl/AvroRowInputFormat.h>
namespace DB namespace DB

6
src/Storages/ObjectStorage/DataLakes/Iceberg/ManifestFile.h
View File

@ -1,11 +1,11 @@
#pragma once #pragma once
#include <memory>
#include "config.h" #include "config.h"
#if USE_AVRO #if USE_AVRO
# include <cstdint> #include <cstdint>
# include <Common/Exception.h> #include <Common/Exception.h>
namespace DB namespace DB

24
src/Storages/ObjectStorage/DataLakes/Iceberg/ManifestFileImpl.h
View File

@ -2,28 +2,8 @@
#if USE_AVRO #if USE_AVRO
#include "Storages/ObjectStorage/DataLakes/Iceberg/ManifestFile.h"
# include "Storages/ObjectStorage/DataLakes/Iceberg/ManifestFile.h" #include <Processors/Formats/Impl/AvroRowInputFormat.h>
# include "Storages/ObjectStorage/DataLakes/Iceberg/Utils.h"
# include <Core/Types.h>
# include <Disks/ObjectStorages/IObjectStorage.h>
# include <Interpreters/Context_fwd.h>
# include <Storages/ObjectStorage/DataLakes/IDataLakeMetadata.h>
# include <Storages/ObjectStorage/StorageObjectStorage.h>
# include <DataFile.hh>
# include <Poco/JSON/Array.h>
# include <Poco/JSON/Object.h>
# include <Poco/JSON/Parser.h>
# include <Common/Exception.h>
# include "DataTypes/DataTypeTuple.h"
# include "Formats/FormatSettings.h"
# include <Processors/Formats/Impl/AvroRowInputFormat.h>
namespace DB namespace DB
{ {

14
src/Storages/ObjectStorage/DataLakes/Iceberg/SchemaProcessor.cpp
View File

@ -1,4 +1,4 @@
#include "Storages/ObjectStorage/DataLakes/Iceberg/SchemaProcessor.h" #include <Storages/ObjectStorage/DataLakes/Iceberg/SchemaProcessor.h>
#include <Poco/JSON/Array.h> #include <Poco/JSON/Array.h>
#include <Poco/JSON/Object.h> #include <Poco/JSON/Object.h>
@ -21,14 +21,18 @@
#include <DataTypes/DataTypesNumber.h> #include <DataTypes/DataTypesNumber.h>
#include <Formats/FormatFactory.h> #include <Formats/FormatFactory.h>
namespace ErrorCodes #include <IO/ReadHelpers.h>
{
extern const int LOGICAL_ERROR;
}
namespace DB namespace DB
{ {
namespace ErrorCodes
{
extern const int LOGICAL_ERROR;
extern const int BAD_ARGUMENTS;
extern const int UNSUPPORTED_METHOD;
}
namespace Iceberg namespace Iceberg
{ {

9
src/Storages/ObjectStorage/DataLakes/Iceberg/SchemaProcessor.h
View File

@ -2,20 +2,19 @@
#include <memory> #include <memory>
#include <mutex> #include <mutex>
#include <unordered_map>
#include "config.h" #include "config.h"
#include <Core/NamesAndTypes.h>
#include <Core/Types.h> #include <Core/Types.h>
#include <Disks/ObjectStorages/IObjectStorage.h> #include <Interpreters/ActionsDAG.h>
#include <Interpreters/Context_fwd.h>
#include <Storages/ObjectStorage/DataLakes/IDataLakeMetadata.h>
#include <Storages/ObjectStorage/StorageObjectStorage.h>
#include <Poco/JSON/Array.h> #include <Poco/JSON/Array.h>
#include <Poco/JSON/Object.h> #include <Poco/JSON/Object.h>
#include <Poco/JSON/Parser.h> #include <Poco/JSON/Parser.h>
#include <unordered_map>
namespace DB namespace DB
{ {

2
src/Storages/ObjectStorage/DataLakes/Iceberg/Snapshot.h
View File

@ -2,7 +2,7 @@
#include "config.h" #include "config.h"
#if USE_AVRO #if USE_AVRO
# include "Storages/ObjectStorage/DataLakes/Iceberg/ManifestFile.h" #include "Storages/ObjectStorage/DataLakes/Iceberg/ManifestFile.h"
namespace DB namespace DB
{ {

22
src/Storages/ObjectStorage/DataLakes/Iceberg/Utils.cpp
View File

@ -1,26 +1,10 @@
#include "config.h" #include "config.h"
#if USE_AVRO /// StorageIceberg depending on Avro to parse metadata with Avro format. #if USE_AVRO
# include "Storages/ObjectStorage/DataLakes/Iceberg/ManifestFile.h" #include <Processors/Formats/Impl/AvroRowInputFormat.h>
#include <Storages/ObjectStorage/DataLakes/Iceberg/Utils.h>
# include <Core/Types.h>
# include <Disks/ObjectStorages/IObjectStorage.h>
# include <Interpreters/Context_fwd.h>
# include <Storages/ObjectStorage/DataLakes/IDataLakeMetadata.h>
# include <Storages/ObjectStorage/StorageObjectStorage.h>
# include <DataFile.hh>
# include <Poco/JSON/Array.h>
# include <Poco/JSON/Object.h>
# include <Poco/JSON/Parser.h>
# include <Common/Exception.h>
# include "DataTypes/DataTypeTuple.h"
# include "Formats/FormatSettings.h"
# include <Processors/Formats/Impl/AvroRowInputFormat.h>
namespace DB namespace DB
{ {

19
src/Storages/ObjectStorage/DataLakes/Iceberg/Utils.h
View File

@ -2,24 +2,9 @@
#include "config.h" #include "config.h"
#if USE_AVRO /// StorageIceberg depending on Avro to parse metadata with Avro format. #if USE_AVRO
#include <Processors/Formats/Impl/AvroRowInputFormat.h>
# include <Core/Types.h>
# include <Disks/ObjectStorages/IObjectStorage.h>
# include <Interpreters/Context_fwd.h>
# include <Storages/ObjectStorage/DataLakes/IDataLakeMetadata.h>
# include <Storages/ObjectStorage/StorageObjectStorage.h>
# include <DataFile.hh>
# include <Poco/JSON/Array.h>
# include <Poco/JSON/Object.h>
# include <Poco/JSON/Parser.h>
# include <Common/Exception.h>
# include "Formats/FormatSettings.h"
# include <Processors/Formats/Impl/AvroRowInputFormat.h>
namespace DB namespace DB
{ {