From eceb54b0016bbe4fd444b8af25ff77e569d1d568 Mon Sep 17 00:00:00 2001 From: kssenii Date: Tue, 21 Mar 2023 12:51:14 +0100 Subject: [PATCH] Hudi tests --- tests/integration/test_storage_hudi/test.py | 236 ++++++------------ .../.hoodie/.20220830083647456.commit.crc | Bin 60 -> 0 bytes .../.20220830083647456.commit.requested.crc | Bin 8 -> 0 bytes .../.hoodie/.20220830083647456.inflight.crc | Bin 44 -> 0 bytes .../test_table/.hoodie/.hoodie.properties.crc | Bin 16 -> 0 bytes .../.hoodie/20220830083647456.commit | 165 ------------ .../20220830083647456.commit.requested | 0 .../.hoodie/20220830083647456.inflight | 161 ------------ .../test_table/.hoodie/hoodie.properties | 21 -- .../.hoodie/.00000000000000.deltacommit.crc | Bin 72 -> 0 bytes .../.00000000000000.deltacommit.inflight.crc | Bin 32 -> 0 bytes .../.00000000000000.deltacommit.requested.crc | Bin 8 -> 0 bytes .../.20220830083647456.deltacommit.crc | Bin 76 -> 0 bytes ...20220830083647456.deltacommit.inflight.crc | Bin 32 -> 0 bytes ...0220830083647456.deltacommit.requested.crc | Bin 8 -> 0 bytes .../metadata/.hoodie/.hoodie.properties.crc | Bin 16 -> 0 bytes .../.hoodie/00000000000000.deltacommit | 97 ------- .../00000000000000.deltacommit.inflight | 116 --------- .../00000000000000.deltacommit.requested | 0 .../.hoodie/20220830083647456.deltacommit | 97 ------- .../20220830083647456.deltacommit.inflight | 116 --------- .../20220830083647456.deltacommit.requested | 0 .../metadata/.hoodie/hoodie.properties | 14 -- ....files-0000_00000000000000.log.1_0-0-0.crc | Bin 12 -> 0 bytes ...iles-0000_00000000000000.log.1_0-52-57.crc | Bin 96 -> 0 bytes ...iles-0000_00000000000000.log.2_0-83-93.crc | Bin 96 -> 0 bytes .../files/..hoodie_partition_metadata.crc | Bin 12 -> 0 bytes .../.files-0000_00000000000000.log.1_0-0-0 | Bin 124 -> 0 bytes .../.files-0000_00000000000000.log.1_0-52-57 | Bin 10928 -> 0 bytes .../.files-0000_00000000000000.log.2_0-83-93 | Bin 11180 -> 0 bytes .../metadata/files/.hoodie_partition_metadata | 4 - .../sao_paulo/..hoodie_partition_metadata.crc | Bin 12 -> 0 bytes ...ad-0_0-73-83_20220830083647456.parquet.crc | Bin 3432 -> 0 bytes .../sao_paulo/.hoodie_partition_metadata | 4 - ...aebbad-0_0-73-83_20220830083647456.parquet | Bin 437831 -> 0 bytes .../..hoodie_partition_metadata.crc | Bin 12 -> 0 bytes ...55-0_1-73-84_20220830083647456.parquet.crc | Bin 3432 -> 0 bytes .../san_francisco/.hoodie_partition_metadata | 4 - ...268e55-0_1-73-84_20220830083647456.parquet | Bin 438186 -> 0 bytes .../chennai/..hoodie_partition_metadata.crc | Bin 12 -> 0 bytes ...e1-0_2-73-85_20220830083647456.parquet.crc | Bin 3428 -> 0 bytes .../india/chennai/.hoodie_partition_metadata | 4 - ...e5d6e1-0_2-73-85_20220830083647456.parquet | Bin 437623 -> 0 bytes utils/data-lakes-importer.py | 9 +- 44 files changed, 82 insertions(+), 966 deletions(-) delete mode 100644 tests/integration/test_storage_hudi/test_table/.hoodie/.20220830083647456.commit.crc delete mode 100644 tests/integration/test_storage_hudi/test_table/.hoodie/.20220830083647456.commit.requested.crc delete mode 100644 tests/integration/test_storage_hudi/test_table/.hoodie/.20220830083647456.inflight.crc delete mode 100644 tests/integration/test_storage_hudi/test_table/.hoodie/.hoodie.properties.crc delete mode 100644 tests/integration/test_storage_hudi/test_table/.hoodie/20220830083647456.commit delete mode 100644 tests/integration/test_storage_hudi/test_table/.hoodie/20220830083647456.commit.requested delete mode 100644 tests/integration/test_storage_hudi/test_table/.hoodie/20220830083647456.inflight delete mode 100644 tests/integration/test_storage_hudi/test_table/.hoodie/hoodie.properties delete mode 100644 tests/integration/test_storage_hudi/test_table/.hoodie/metadata/.hoodie/.00000000000000.deltacommit.crc delete mode 100644 tests/integration/test_storage_hudi/test_table/.hoodie/metadata/.hoodie/.00000000000000.deltacommit.inflight.crc delete mode 100644 tests/integration/test_storage_hudi/test_table/.hoodie/metadata/.hoodie/.00000000000000.deltacommit.requested.crc delete mode 100644 tests/integration/test_storage_hudi/test_table/.hoodie/metadata/.hoodie/.20220830083647456.deltacommit.crc delete mode 100644 tests/integration/test_storage_hudi/test_table/.hoodie/metadata/.hoodie/.20220830083647456.deltacommit.inflight.crc delete mode 100644 tests/integration/test_storage_hudi/test_table/.hoodie/metadata/.hoodie/.20220830083647456.deltacommit.requested.crc delete mode 100644 tests/integration/test_storage_hudi/test_table/.hoodie/metadata/.hoodie/.hoodie.properties.crc delete mode 100644 tests/integration/test_storage_hudi/test_table/.hoodie/metadata/.hoodie/00000000000000.deltacommit delete mode 100644 tests/integration/test_storage_hudi/test_table/.hoodie/metadata/.hoodie/00000000000000.deltacommit.inflight delete mode 100644 tests/integration/test_storage_hudi/test_table/.hoodie/metadata/.hoodie/00000000000000.deltacommit.requested delete mode 100644 tests/integration/test_storage_hudi/test_table/.hoodie/metadata/.hoodie/20220830083647456.deltacommit delete mode 100644 tests/integration/test_storage_hudi/test_table/.hoodie/metadata/.hoodie/20220830083647456.deltacommit.inflight delete mode 100644 tests/integration/test_storage_hudi/test_table/.hoodie/metadata/.hoodie/20220830083647456.deltacommit.requested delete mode 100644 tests/integration/test_storage_hudi/test_table/.hoodie/metadata/.hoodie/hoodie.properties delete mode 100644 tests/integration/test_storage_hudi/test_table/.hoodie/metadata/files/..files-0000_00000000000000.log.1_0-0-0.crc delete mode 100644 tests/integration/test_storage_hudi/test_table/.hoodie/metadata/files/..files-0000_00000000000000.log.1_0-52-57.crc delete mode 100644 tests/integration/test_storage_hudi/test_table/.hoodie/metadata/files/..files-0000_00000000000000.log.2_0-83-93.crc delete mode 100644 tests/integration/test_storage_hudi/test_table/.hoodie/metadata/files/..hoodie_partition_metadata.crc delete mode 100644 tests/integration/test_storage_hudi/test_table/.hoodie/metadata/files/.files-0000_00000000000000.log.1_0-0-0 delete mode 100644 tests/integration/test_storage_hudi/test_table/.hoodie/metadata/files/.files-0000_00000000000000.log.1_0-52-57 delete mode 100644 tests/integration/test_storage_hudi/test_table/.hoodie/metadata/files/.files-0000_00000000000000.log.2_0-83-93 delete mode 100644 tests/integration/test_storage_hudi/test_table/.hoodie/metadata/files/.hoodie_partition_metadata delete mode 100644 tests/integration/test_storage_hudi/test_table/americas/brazil/sao_paulo/..hoodie_partition_metadata.crc delete mode 100644 tests/integration/test_storage_hudi/test_table/americas/brazil/sao_paulo/.8a9a08bb-8cbc-4ec9-a2d4-8a6cdcaebbad-0_0-73-83_20220830083647456.parquet.crc delete mode 100644 tests/integration/test_storage_hudi/test_table/americas/brazil/sao_paulo/.hoodie_partition_metadata delete mode 100644 tests/integration/test_storage_hudi/test_table/americas/brazil/sao_paulo/8a9a08bb-8cbc-4ec9-a2d4-8a6cdcaebbad-0_0-73-83_20220830083647456.parquet delete mode 100644 tests/integration/test_storage_hudi/test_table/americas/united_states/san_francisco/..hoodie_partition_metadata.crc delete mode 100644 tests/integration/test_storage_hudi/test_table/americas/united_states/san_francisco/.34b1b177-f0af-467b-9214-473ead268e55-0_1-73-84_20220830083647456.parquet.crc delete mode 100644 tests/integration/test_storage_hudi/test_table/americas/united_states/san_francisco/.hoodie_partition_metadata delete mode 100644 tests/integration/test_storage_hudi/test_table/americas/united_states/san_francisco/34b1b177-f0af-467b-9214-473ead268e55-0_1-73-84_20220830083647456.parquet delete mode 100644 tests/integration/test_storage_hudi/test_table/asia/india/chennai/..hoodie_partition_metadata.crc delete mode 100644 tests/integration/test_storage_hudi/test_table/asia/india/chennai/.92aa634e-d83f-4057-a385-ea3b22e5d6e1-0_2-73-85_20220830083647456.parquet.crc delete mode 100644 tests/integration/test_storage_hudi/test_table/asia/india/chennai/.hoodie_partition_metadata delete mode 100644 tests/integration/test_storage_hudi/test_table/asia/india/chennai/92aa634e-d83f-4057-a385-ea3b22e5d6e1-0_2-73-85_20220830083647456.parquet diff --git a/tests/integration/test_storage_hudi/test.py b/tests/integration/test_storage_hudi/test.py index 3328f859406..6850d0c116c 100644 --- a/tests/integration/test_storage_hudi/test.py +++ b/tests/integration/test_storage_hudi/test.py @@ -1,75 +1,25 @@ import logging +import pytest import os import json import helpers.client -import pytest from helpers.cluster import ClickHouseCluster from helpers.test_tools import TSV +from helpers.s3_tools import prepare_s3_bucket, upload_directory, get_file_contents + +import pyspark SCRIPT_DIR = os.path.dirname(os.path.realpath(__file__)) - - -def prepare_s3_bucket(started_cluster): - bucket_read_write_policy = { - "Version": "2012-10-17", - "Statement": [ - { - "Sid": "", - "Effect": "Allow", - "Principal": {"AWS": "*"}, - "Action": "s3:GetBucketLocation", - "Resource": "arn:aws:s3:::root", - }, - { - "Sid": "", - "Effect": "Allow", - "Principal": {"AWS": "*"}, - "Action": "s3:ListBucket", - "Resource": "arn:aws:s3:::root", - }, - { - "Sid": "", - "Effect": "Allow", - "Principal": {"AWS": "*"}, - "Action": "s3:GetObject", - "Resource": "arn:aws:s3:::root/*", - }, - { - "Sid": "", - "Effect": "Allow", - "Principal": {"AWS": "*"}, - "Action": "s3:PutObject", - "Resource": "arn:aws:s3:::root/*", - }, - ], - } - - minio_client = started_cluster.minio_client - minio_client.set_bucket_policy( - started_cluster.minio_bucket, json.dumps(bucket_read_write_policy) - ) - - -def upload_test_table(started_cluster): - bucket = started_cluster.minio_bucket - - for address, dirs, files in os.walk(SCRIPT_DIR + "/test_table"): - address_without_prefix = address[len(SCRIPT_DIR) :] - - for name in files: - started_cluster.minio_client.fput_object( - bucket, - os.path.join(address_without_prefix, name), - os.path.join(address, name), - ) +TABLE_NAME = "test_hudi_table" +USER_FILES_PATH = "/ClickHouse/tests/integration/test_storage_hudi/_instances/node1/database/user_files" @pytest.fixture(scope="module") def started_cluster(): try: cluster = ClickHouseCluster(__file__) - cluster.add_instance("main_server", with_minio=True) + cluster.add_instance("node1", with_minio=True) logging.info("Starting cluster...") cluster.start() @@ -77,9 +27,6 @@ def started_cluster(): prepare_s3_bucket(cluster) logging.info("S3 bucket created") - upload_test_table(cluster) - logging.info("Test table uploaded") - yield cluster finally: @@ -96,108 +43,75 @@ def run_query(instance, query, stdin=None, settings=None): return result -def test_create_query(started_cluster): - instance = started_cluster.instances["main_server"] +def get_spark(): + builder = ( + pyspark.sql.SparkSession.builder.appName("spark_test") + .config( + "spark.jars.packages", + "org.apache.hudi:hudi-spark3.3-bundle_2.12:0.13.0", + ) + .config( + "org.apache.spark.sql.hudi.catalog.HoodieCatalog", + ) + .config("spark.serializer", "org.apache.spark.serializer.KryoSerializer") + .config( + "spark.sql.catalog.local", "org.apache.spark.sql.hudi.catalog.HoodieCatalog" + ) + .config("spark.driver.memory", "20g") + .master("local") + ) + return builder.master("local").getOrCreate() + + +def write_hudi(spark, path, result_path): + spark.conf.set("spark.sql.debug.maxToStringFields", 100000) + spark.read.load(f"file://{path}").write.mode("overwrite").option( + "compression", "none" + ).option("compression", "none").format("hudi").option( + "hoodie.table.name", TABLE_NAME + ).option( + "hoodie.datasource.write.partitionpath.field", "partitionpath" + ).option( + "hoodie.datasource.write.table.name", TABLE_NAME + ).option( + "hoodie.datasource.write.operation", "insert_overwrite" + ).option( + "hoodie.parquet.compression.codec", "snappy" + ).option( + "hoodie.hfile.compression.algorithm", "uncompressed" + ).option( + "hoodie.datasource.write.recordkey.field", "a" + ).option( + "hoodie.datasource.write.precombine.field", "a" + ).save( + result_path + ) + + +def test_basic(started_cluster): + instance = started_cluster.instances["node1"] + + data_path = f"/var/lib/clickhouse/user_files/{TABLE_NAME}.parquet" + inserted_data = "SELECT number, toString(number) FROM numbers(100)" + instance.query( + f"INSERT INTO TABLE FUNCTION file('{data_path}', 'Parquet', 'a Int32, b String') {inserted_data} FORMAT Parquet SETTINGS output_format_parquet_compression_method='snappy'" + ) + + data_path = f"{USER_FILES_PATH}/{TABLE_NAME}.parquet" + result_path = f"/{TABLE_NAME}_result" + + spark = get_spark() + write_hudi(spark, data_path, result_path) + + minio_client = started_cluster.minio_client bucket = started_cluster.minio_bucket + upload_directory(minio_client, bucket, result_path, "") - create_query = f"""CREATE TABLE hudi ENGINE=Hudi('http://{started_cluster.minio_ip}:{started_cluster.minio_port}/{bucket}/test_table/', 'minio', 'minio123')""" - - run_query(instance, create_query) - - -def test_select_query(started_cluster): - instance = started_cluster.instances["main_server"] - bucket = started_cluster.minio_bucket - columns = [ - "_hoodie_commit_time", - "_hoodie_commit_seqno", - "_hoodie_record_key", - "_hoodie_partition_path", - "_hoodie_file_name", - "begin_lat", - "begin_lon", - "driver", - "end_lat", - "end_lon", - "fare", - "partitionpath", - "rider", - "ts", - "uuid", - ] - - # create query in case table doesn't exist - create_query = f"""CREATE TABLE IF NOT EXISTS hudi ENGINE=Hudi('http://{started_cluster.minio_ip}:{started_cluster.minio_port}/{bucket}/test_table/', 'minio', 'minio123')""" - - run_query(instance, create_query) - - select_query = "SELECT {} FROM hudi FORMAT TSV" - - select_table_function_query = "SELECT {col} FROM hudi('http://{ip}:{port}/{bucket}/test_table/', 'minio', 'minio123') FORMAT TSV" - - for column_name in columns: - result = run_query(instance, select_query.format(column_name)).splitlines() - assert len(result) > 0 - - for column_name in columns: - result = run_query( - instance, - select_table_function_query.format( - col=column_name, - ip=started_cluster.minio_ip, - port=started_cluster.minio_port, - bucket=bucket, - ), - ).splitlines() - assert len(result) > 0 - - # test if all partition paths is presented in result - distinct_select_query = ( - "SELECT DISTINCT partitionpath FROM hudi ORDER BY partitionpath FORMAT TSV" + instance.query( + f""" + DROP TABLE IF EXISTS {TABLE_NAME}; + CREATE TABLE {TABLE_NAME} ENGINE=Hudi('http://{started_cluster.minio_ip}:{started_cluster.minio_port}/{bucket}/{TABLE_NAME}_result/', 'minio', 'minio123')""" ) - - distinct_select_table_function_query = "SELECT DISTINCT partitionpath FROM hudi('http://{ip}:{port}/{bucket}/test_table/', 'minio', 'minio123') ORDER BY partitionpath FORMAT TSV" - - result = run_query(instance, distinct_select_query) - result_table_function = run_query( - instance, - distinct_select_table_function_query.format( - ip=started_cluster.minio_ip, port=started_cluster.minio_port, bucket=bucket - ), - ) - expected = [ - "americas/brazil/sao_paulo", - "americas/united_states/san_francisco", - "asia/india/chennai", - ] - - assert TSV(result) == TSV(expected) - assert TSV(result_table_function) == TSV(expected) - - -def test_describe_query(started_cluster): - instance = started_cluster.instances["main_server"] - bucket = started_cluster.minio_bucket - result = instance.query( - f"DESCRIBE hudi('http://{started_cluster.minio_ip}:{started_cluster.minio_port}/{bucket}/test_table/', 'minio', 'minio123') FORMAT TSV", - ) - - assert result == TSV( - [ - ["_hoodie_commit_time", "Nullable(String)"], - ["_hoodie_commit_seqno", "Nullable(String)"], - ["_hoodie_record_key", "Nullable(String)"], - ["_hoodie_partition_path", "Nullable(String)"], - ["_hoodie_file_name", "Nullable(String)"], - ["begin_lat", "Nullable(Float64)"], - ["begin_lon", "Nullable(Float64)"], - ["driver", "Nullable(String)"], - ["end_lat", "Nullable(Float64)"], - ["end_lon", "Nullable(Float64)"], - ["fare", "Nullable(Float64)"], - ["partitionpath", "Nullable(String)"], - ["rider", "Nullable(String)"], - ["ts", "Nullable(Int64)"], - ["uuid", "Nullable(String)"], - ] + assert instance.query(f"SELECT * FROM {TABLE_NAME}") == instance.query( + inserted_data ) diff --git a/tests/integration/test_storage_hudi/test_table/.hoodie/.20220830083647456.commit.crc b/tests/integration/test_storage_hudi/test_table/.hoodie/.20220830083647456.commit.crc deleted file mode 100644 index 4bba97b9515ddcd9dd09f226e1cd81e89f1e9026..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 60 zcmV-C0K@-da$^7h00IEYSJ`4^`+RY$Y`hZ?T{ diff --git a/tests/integration/test_storage_hudi/test_table/.hoodie/.20220830083647456.commit.requested.crc b/tests/integration/test_storage_hudi/test_table/.hoodie/.20220830083647456.commit.requested.crc deleted file mode 100644 index 3b7b044936a890cd8d651d349a752d819d71d22c..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 8 PcmYc;N@ieSU}69O2$TUk diff --git a/tests/integration/test_storage_hudi/test_table/.hoodie/.20220830083647456.inflight.crc b/tests/integration/test_storage_hudi/test_table/.hoodie/.20220830083647456.inflight.crc deleted file mode 100644 index 21984c840bc3f9329c55d1d5824515745f9d0468..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 44 zcmYc;N@ieSU}Cskw&P5j)zKi6c-hD;M%E!O)z;;Tsn#4f+2CBxmvH@r-orPu0BZCS AO#lD@ diff --git a/tests/integration/test_storage_hudi/test_table/.hoodie/.hoodie.properties.crc b/tests/integration/test_storage_hudi/test_table/.hoodie/.hoodie.properties.crc deleted file mode 100644 index f67f76b7101c0cf7b1aba9818ac05738d10a419c..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 16 XcmYc;N@ieSU}6a3SsyCGs3rmc8^!|c diff --git a/tests/integration/test_storage_hudi/test_table/.hoodie/20220830083647456.commit b/tests/integration/test_storage_hudi/test_table/.hoodie/20220830083647456.commit deleted file mode 100644 index f8d6c248f49..00000000000 --- a/tests/integration/test_storage_hudi/test_table/.hoodie/20220830083647456.commit +++ /dev/null @@ -1,165 +0,0 @@ -{ - "partitionToWriteStats" : { - "americas/brazil/sao_paulo" : [ { - "fileId" : "8a9a08bb-8cbc-4ec9-a2d4-8a6cdcaebbad-0", - "path" : "americas/brazil/sao_paulo/8a9a08bb-8cbc-4ec9-a2d4-8a6cdcaebbad-0_0-73-83_20220830083647456.parquet", - "prevCommit" : "null", - "numWrites" : 3, - "numDeletes" : 0, - "numUpdateWrites" : 0, - "numInserts" : 3, - "totalWriteBytes" : 437831, - "totalWriteErrors" : 0, - "tempPath" : null, - "partitionPath" : "americas/brazil/sao_paulo", - "totalLogRecords" : 0, - "totalLogFilesCompacted" : 0, - "totalLogSizeCompacted" : 0, - "totalUpdatedRecordsCompacted" : 0, - "totalLogBlocks" : 0, - "totalCorruptLogBlock" : 0, - "totalRollbackBlocks" : 0, - "fileSizeInBytes" : 437831, - "minEventTime" : null, - "maxEventTime" : null - } ], - "americas/united_states/san_francisco" : [ { - "fileId" : "34b1b177-f0af-467b-9214-473ead268e55-0", - "path" : "americas/united_states/san_francisco/34b1b177-f0af-467b-9214-473ead268e55-0_1-73-84_20220830083647456.parquet", - "prevCommit" : "null", - "numWrites" : 5, - "numDeletes" : 0, - "numUpdateWrites" : 0, - "numInserts" : 5, - "totalWriteBytes" : 438186, - "totalWriteErrors" : 0, - "tempPath" : null, - "partitionPath" : "americas/united_states/san_francisco", - "totalLogRecords" : 0, - "totalLogFilesCompacted" : 0, - "totalLogSizeCompacted" : 0, - "totalUpdatedRecordsCompacted" : 0, - "totalLogBlocks" : 0, - "totalCorruptLogBlock" : 0, - "totalRollbackBlocks" : 0, - "fileSizeInBytes" : 438186, - "minEventTime" : null, - "maxEventTime" : null - } ], - "asia/india/chennai" : [ { - "fileId" : "92aa634e-d83f-4057-a385-ea3b22e5d6e1-0", - "path" : "asia/india/chennai/92aa634e-d83f-4057-a385-ea3b22e5d6e1-0_2-73-85_20220830083647456.parquet", - "prevCommit" : "null", - "numWrites" : 2, - "numDeletes" : 0, - "numUpdateWrites" : 0, - "numInserts" : 2, - "totalWriteBytes" : 437623, - "totalWriteErrors" : 0, - "tempPath" : null, - "partitionPath" : "asia/india/chennai", - "totalLogRecords" : 0, - "totalLogFilesCompacted" : 0, - "totalLogSizeCompacted" : 0, - "totalUpdatedRecordsCompacted" : 0, - "totalLogBlocks" : 0, - "totalCorruptLogBlock" : 0, - "totalRollbackBlocks" : 0, - "fileSizeInBytes" : 437623, - "minEventTime" : null, - "maxEventTime" : null - } ] - }, - "compacted" : false, - "extraMetadata" : { - "schema" : "{\"type\":\"record\",\"name\":\"test_table_record\",\"namespace\":\"hoodie.test_table\",\"fields\":[{\"name\":\"begin_lat\",\"type\":[\"null\",\"double\"],\"default\":null},{\"name\":\"begin_lon\",\"type\":[\"null\",\"double\"],\"default\":null},{\"name\":\"driver\",\"type\":[\"null\",\"string\"],\"default\":null},{\"name\":\"end_lat\",\"type\":[\"null\",\"double\"],\"default\":null},{\"name\":\"end_lon\",\"type\":[\"null\",\"double\"],\"default\":null},{\"name\":\"fare\",\"type\":[\"null\",\"double\"],\"default\":null},{\"name\":\"partitionpath\",\"type\":[\"null\",\"string\"],\"default\":null},{\"name\":\"rider\",\"type\":[\"null\",\"string\"],\"default\":null},{\"name\":\"ts\",\"type\":[\"null\",\"long\"],\"default\":null},{\"name\":\"uuid\",\"type\":[\"null\",\"string\"],\"default\":null}]}" - }, - "operationType" : "UPSERT", - "writeStats" : [ { - "fileId" : "8a9a08bb-8cbc-4ec9-a2d4-8a6cdcaebbad-0", - "path" : "americas/brazil/sao_paulo/8a9a08bb-8cbc-4ec9-a2d4-8a6cdcaebbad-0_0-73-83_20220830083647456.parquet", - "prevCommit" : "null", - "numWrites" : 3, - "numDeletes" : 0, - "numUpdateWrites" : 0, - "numInserts" : 3, - "totalWriteBytes" : 437831, - "totalWriteErrors" : 0, - "tempPath" : null, - "partitionPath" : "americas/brazil/sao_paulo", - "totalLogRecords" : 0, - "totalLogFilesCompacted" : 0, - "totalLogSizeCompacted" : 0, - "totalUpdatedRecordsCompacted" : 0, - "totalLogBlocks" : 0, - "totalCorruptLogBlock" : 0, - "totalRollbackBlocks" : 0, - "fileSizeInBytes" : 437831, - "minEventTime" : null, - "maxEventTime" : null - }, { - "fileId" : "34b1b177-f0af-467b-9214-473ead268e55-0", - "path" : "americas/united_states/san_francisco/34b1b177-f0af-467b-9214-473ead268e55-0_1-73-84_20220830083647456.parquet", - "prevCommit" : "null", - "numWrites" : 5, - "numDeletes" : 0, - "numUpdateWrites" : 0, - "numInserts" : 5, - "totalWriteBytes" : 438186, - "totalWriteErrors" : 0, - "tempPath" : null, - "partitionPath" : "americas/united_states/san_francisco", - "totalLogRecords" : 0, - "totalLogFilesCompacted" : 0, - "totalLogSizeCompacted" : 0, - "totalUpdatedRecordsCompacted" : 0, - "totalLogBlocks" : 0, - "totalCorruptLogBlock" : 0, - "totalRollbackBlocks" : 0, - "fileSizeInBytes" : 438186, - "minEventTime" : null, - "maxEventTime" : null - }, { - "fileId" : "92aa634e-d83f-4057-a385-ea3b22e5d6e1-0", - "path" : "asia/india/chennai/92aa634e-d83f-4057-a385-ea3b22e5d6e1-0_2-73-85_20220830083647456.parquet", - "prevCommit" : "null", - "numWrites" : 2, - "numDeletes" : 0, - "numUpdateWrites" : 0, - "numInserts" : 2, - "totalWriteBytes" : 437623, - "totalWriteErrors" : 0, - "tempPath" : null, - "partitionPath" : "asia/india/chennai", - "totalLogRecords" : 0, - "totalLogFilesCompacted" : 0, - "totalLogSizeCompacted" : 0, - "totalUpdatedRecordsCompacted" : 0, - "totalLogBlocks" : 0, - "totalCorruptLogBlock" : 0, - "totalRollbackBlocks" : 0, - "fileSizeInBytes" : 437623, - "minEventTime" : null, - "maxEventTime" : null - } ], - "totalRecordsDeleted" : 0, - "totalLogRecordsCompacted" : 0, - "totalLogFilesCompacted" : 0, - "totalCompactedRecordsUpdated" : 0, - "totalLogFilesSize" : 0, - "totalScanTime" : 0, - "totalCreateTime" : 563, - "totalUpsertTime" : 0, - "minAndMaxEventTime" : { - "Optional.empty" : { - "val" : null, - "present" : false - } - }, - "writePartitionPaths" : [ "americas/brazil/sao_paulo", "americas/united_states/san_francisco", "asia/india/chennai" ], - "fileIdAndRelativePaths" : { - "92aa634e-d83f-4057-a385-ea3b22e5d6e1-0" : "asia/india/chennai/92aa634e-d83f-4057-a385-ea3b22e5d6e1-0_2-73-85_20220830083647456.parquet", - "34b1b177-f0af-467b-9214-473ead268e55-0" : "americas/united_states/san_francisco/34b1b177-f0af-467b-9214-473ead268e55-0_1-73-84_20220830083647456.parquet", - "8a9a08bb-8cbc-4ec9-a2d4-8a6cdcaebbad-0" : "americas/brazil/sao_paulo/8a9a08bb-8cbc-4ec9-a2d4-8a6cdcaebbad-0_0-73-83_20220830083647456.parquet" - } -} \ No newline at end of file diff --git a/tests/integration/test_storage_hudi/test_table/.hoodie/20220830083647456.commit.requested b/tests/integration/test_storage_hudi/test_table/.hoodie/20220830083647456.commit.requested deleted file mode 100644 index e69de29bb2d..00000000000 diff --git a/tests/integration/test_storage_hudi/test_table/.hoodie/20220830083647456.inflight b/tests/integration/test_storage_hudi/test_table/.hoodie/20220830083647456.inflight deleted file mode 100644 index f5ef5c92c2b..00000000000 --- a/tests/integration/test_storage_hudi/test_table/.hoodie/20220830083647456.inflight +++ /dev/null @@ -1,161 +0,0 @@ -{ - "partitionToWriteStats" : { - "americas/brazil/sao_paulo" : [ { - "fileId" : "", - "path" : null, - "prevCommit" : "null", - "numWrites" : 0, - "numDeletes" : 0, - "numUpdateWrites" : 0, - "numInserts" : 3, - "totalWriteBytes" : 0, - "totalWriteErrors" : 0, - "tempPath" : null, - "partitionPath" : null, - "totalLogRecords" : 0, - "totalLogFilesCompacted" : 0, - "totalLogSizeCompacted" : 0, - "totalUpdatedRecordsCompacted" : 0, - "totalLogBlocks" : 0, - "totalCorruptLogBlock" : 0, - "totalRollbackBlocks" : 0, - "fileSizeInBytes" : 0, - "minEventTime" : null, - "maxEventTime" : null - } ], - "americas/united_states/san_francisco" : [ { - "fileId" : "", - "path" : null, - "prevCommit" : "null", - "numWrites" : 0, - "numDeletes" : 0, - "numUpdateWrites" : 0, - "numInserts" : 5, - "totalWriteBytes" : 0, - "totalWriteErrors" : 0, - "tempPath" : null, - "partitionPath" : null, - "totalLogRecords" : 0, - "totalLogFilesCompacted" : 0, - "totalLogSizeCompacted" : 0, - "totalUpdatedRecordsCompacted" : 0, - "totalLogBlocks" : 0, - "totalCorruptLogBlock" : 0, - "totalRollbackBlocks" : 0, - "fileSizeInBytes" : 0, - "minEventTime" : null, - "maxEventTime" : null - } ], - "asia/india/chennai" : [ { - "fileId" : "", - "path" : null, - "prevCommit" : "null", - "numWrites" : 0, - "numDeletes" : 0, - "numUpdateWrites" : 0, - "numInserts" : 2, - "totalWriteBytes" : 0, - "totalWriteErrors" : 0, - "tempPath" : null, - "partitionPath" : null, - "totalLogRecords" : 0, - "totalLogFilesCompacted" : 0, - "totalLogSizeCompacted" : 0, - "totalUpdatedRecordsCompacted" : 0, - "totalLogBlocks" : 0, - "totalCorruptLogBlock" : 0, - "totalRollbackBlocks" : 0, - "fileSizeInBytes" : 0, - "minEventTime" : null, - "maxEventTime" : null - } ] - }, - "compacted" : false, - "extraMetadata" : { }, - "operationType" : "UPSERT", - "writeStats" : [ { - "fileId" : "", - "path" : null, - "prevCommit" : "null", - "numWrites" : 0, - "numDeletes" : 0, - "numUpdateWrites" : 0, - "numInserts" : 3, - "totalWriteBytes" : 0, - "totalWriteErrors" : 0, - "tempPath" : null, - "partitionPath" : null, - "totalLogRecords" : 0, - "totalLogFilesCompacted" : 0, - "totalLogSizeCompacted" : 0, - "totalUpdatedRecordsCompacted" : 0, - "totalLogBlocks" : 0, - "totalCorruptLogBlock" : 0, - "totalRollbackBlocks" : 0, - "fileSizeInBytes" : 0, - "minEventTime" : null, - "maxEventTime" : null - }, { - "fileId" : "", - "path" : null, - "prevCommit" : "null", - "numWrites" : 0, - "numDeletes" : 0, - "numUpdateWrites" : 0, - "numInserts" : 5, - "totalWriteBytes" : 0, - "totalWriteErrors" : 0, - "tempPath" : null, - "partitionPath" : null, - "totalLogRecords" : 0, - "totalLogFilesCompacted" : 0, - "totalLogSizeCompacted" : 0, - "totalUpdatedRecordsCompacted" : 0, - "totalLogBlocks" : 0, - "totalCorruptLogBlock" : 0, - "totalRollbackBlocks" : 0, - "fileSizeInBytes" : 0, - "minEventTime" : null, - "maxEventTime" : null - }, { - "fileId" : "", - "path" : null, - "prevCommit" : "null", - "numWrites" : 0, - "numDeletes" : 0, - "numUpdateWrites" : 0, - "numInserts" : 2, - "totalWriteBytes" : 0, - "totalWriteErrors" : 0, - "tempPath" : null, - "partitionPath" : null, - "totalLogRecords" : 0, - "totalLogFilesCompacted" : 0, - "totalLogSizeCompacted" : 0, - "totalUpdatedRecordsCompacted" : 0, - "totalLogBlocks" : 0, - "totalCorruptLogBlock" : 0, - "totalRollbackBlocks" : 0, - "fileSizeInBytes" : 0, - "minEventTime" : null, - "maxEventTime" : null - } ], - "totalRecordsDeleted" : 0, - "totalLogRecordsCompacted" : 0, - "totalLogFilesCompacted" : 0, - "totalCompactedRecordsUpdated" : 0, - "totalLogFilesSize" : 0, - "totalScanTime" : 0, - "totalCreateTime" : 0, - "totalUpsertTime" : 0, - "minAndMaxEventTime" : { - "Optional.empty" : { - "val" : null, - "present" : false - } - }, - "writePartitionPaths" : [ "americas/brazil/sao_paulo", "americas/united_states/san_francisco", "asia/india/chennai" ], - "fileIdAndRelativePaths" : { - "" : null - } -} \ No newline at end of file diff --git a/tests/integration/test_storage_hudi/test_table/.hoodie/hoodie.properties b/tests/integration/test_storage_hudi/test_table/.hoodie/hoodie.properties deleted file mode 100644 index 9ae364baf33..00000000000 --- a/tests/integration/test_storage_hudi/test_table/.hoodie/hoodie.properties +++ /dev/null @@ -1,21 +0,0 @@ -#Updated at 2022-08-30T08:36:49.089844Z -#Tue Aug 30 08:36:49 UTC 2022 -hoodie.table.type=COPY_ON_WRITE -hoodie.table.metadata.partitions=files -hoodie.table.precombine.field=ts -hoodie.table.partition.fields=partitionpath -hoodie.archivelog.folder=archived -hoodie.timeline.layout.version=1 -hoodie.table.checksum=2702201862 -hoodie.datasource.write.drop.partition.columns=false -hoodie.table.timeline.timezone=LOCAL -hoodie.table.recordkey.fields=uuid -hoodie.table.name=test_table -hoodie.partition.metafile.use.base.format=false -hoodie.datasource.write.hive_style_partitioning=false -hoodie.populate.meta.fields=true -hoodie.table.keygenerator.class=org.apache.hudi.keygen.SimpleKeyGenerator -hoodie.table.base.file.format=PARQUET -hoodie.database.name= -hoodie.datasource.write.partitionpath.urlencode=false -hoodie.table.version=5 diff --git a/tests/integration/test_storage_hudi/test_table/.hoodie/metadata/.hoodie/.00000000000000.deltacommit.crc b/tests/integration/test_storage_hudi/test_table/.hoodie/metadata/.hoodie/.00000000000000.deltacommit.crc deleted file mode 100644 index 352b882ec5a697e6d4d0754866a1cdec0be1d3be..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 72 zcmV-O0Jr~Ra$^7h00ID+B#3JXBN@*inEmG*kvdVpwWAbT>N@Qjh0Y0p<^OR6NMYYe e)p}4$Xm-%78APNu2!RHTD1CAabbD4@y#KTXRUT6S diff --git a/tests/integration/test_storage_hudi/test_table/.hoodie/metadata/.hoodie/.00000000000000.deltacommit.inflight.crc b/tests/integration/test_storage_hudi/test_table/.hoodie/metadata/.hoodie/.00000000000000.deltacommit.inflight.crc deleted file mode 100644 index b6b8f7fc1a3a439cc0c8cf1a3c06491fcf54223f..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 32 ncmYc;N@ieSU}8AynOb;@FLqIaqtjZc*wA7%;rSPOY^;g^r9}$) diff --git a/tests/integration/test_storage_hudi/test_table/.hoodie/metadata/.hoodie/.00000000000000.deltacommit.requested.crc b/tests/integration/test_storage_hudi/test_table/.hoodie/metadata/.hoodie/.00000000000000.deltacommit.requested.crc deleted file mode 100644 index 3b7b044936a890cd8d651d349a752d819d71d22c..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 8 PcmYc;N@ieSU}69O2$TUk diff --git a/tests/integration/test_storage_hudi/test_table/.hoodie/metadata/.hoodie/.20220830083647456.deltacommit.crc b/tests/integration/test_storage_hudi/test_table/.hoodie/metadata/.hoodie/.20220830083647456.deltacommit.crc deleted file mode 100644 index 54abc5e9b72e87753d458af67e967cccaa242c8e..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 76 zcmV-S0JHyNa$^7h00IEQeq^6A>X*!Te*(sYWI5Hv06BTKu=gO04+eJ-1Ey1Rqt#G) ismNQFuvqj*Kj+@fF0)E(wf0GSx`G_6G3KKp3jiwT&ms^2 diff --git a/tests/integration/test_storage_hudi/test_table/.hoodie/metadata/.hoodie/.20220830083647456.deltacommit.inflight.crc b/tests/integration/test_storage_hudi/test_table/.hoodie/metadata/.hoodie/.20220830083647456.deltacommit.inflight.crc deleted file mode 100644 index ec7cb5faf561c70caf37bd118b025f9e36947c07..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 32 ocmYc;N@ieSU}8AynOf-f;LElIN2j$?v7yDY|IOy^v9T%w0J~ufc>n+a diff --git a/tests/integration/test_storage_hudi/test_table/.hoodie/metadata/.hoodie/.20220830083647456.deltacommit.requested.crc b/tests/integration/test_storage_hudi/test_table/.hoodie/metadata/.hoodie/.20220830083647456.deltacommit.requested.crc deleted file mode 100644 index 3b7b044936a890cd8d651d349a752d819d71d22c..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 8 PcmYc;N@ieSU}69O2$TUk diff --git a/tests/integration/test_storage_hudi/test_table/.hoodie/metadata/.hoodie/.hoodie.properties.crc b/tests/integration/test_storage_hudi/test_table/.hoodie/metadata/.hoodie/.hoodie.properties.crc deleted file mode 100644 index 509ae4501ee2922036326030efe3e5c74d1db059..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 16 XcmYc;N@ieSU}EsgOFNnOa!LyTBu)jD diff --git a/tests/integration/test_storage_hudi/test_table/.hoodie/metadata/.hoodie/00000000000000.deltacommit b/tests/integration/test_storage_hudi/test_table/.hoodie/metadata/.hoodie/00000000000000.deltacommit deleted file mode 100644 index 6d22af6dd2e..00000000000 --- a/tests/integration/test_storage_hudi/test_table/.hoodie/metadata/.hoodie/00000000000000.deltacommit +++ /dev/null @@ -1,97 +0,0 @@ -{ - "partitionToWriteStats" : { - "files" : [ { - "fileId" : "files-0000", - "path" : "files/.files-0000_00000000000000.log.1_0-52-57", - "prevCommit" : "00000000000000", - "numWrites" : 1, - "numDeletes" : 0, - "numUpdateWrites" : 1, - "numInserts" : 0, - "totalWriteBytes" : 10928, - "totalWriteErrors" : 0, - "tempPath" : null, - "partitionPath" : "files", - "totalLogRecords" : 0, - "totalLogFilesCompacted" : 0, - "totalLogSizeCompacted" : 0, - "totalUpdatedRecordsCompacted" : 0, - "totalLogBlocks" : 0, - "totalCorruptLogBlock" : 0, - "totalRollbackBlocks" : 0, - "fileSizeInBytes" : 10928, - "minEventTime" : null, - "maxEventTime" : null, - "logVersion" : 1, - "logOffset" : 0, - "baseFile" : "", - "logFiles" : [ ".files-0000_00000000000000.log.1_0-52-57" ], - "recordsStats" : { - "val" : null, - "present" : false - }, - "columnStats" : { - "val" : null, - "present" : false - } - } ] - }, - "compacted" : false, - "extraMetadata" : { - "schema" : "{\"type\":\"record\",\"name\":\"HoodieMetadataRecord\",\"namespace\":\"org.apache.hudi.avro.model\",\"doc\":\"A record saved within the Metadata Table\",\"fields\":[{\"name\":\"key\",\"type\":{\"type\":\"string\",\"avro.java.string\":\"String\"}},{\"name\":\"type\",\"type\":\"int\",\"doc\":\"Type of the metadata record\"},{\"name\":\"filesystemMetadata\",\"type\":[\"null\",{\"type\":\"map\",\"values\":{\"type\":\"record\",\"name\":\"HoodieMetadataFileInfo\",\"fields\":[{\"name\":\"size\",\"type\":\"long\",\"doc\":\"Size of the file\"},{\"name\":\"isDeleted\",\"type\":\"boolean\",\"doc\":\"True if this file has been deleted\"}]},\"avro.java.string\":\"String\"}],\"doc\":\"Contains information about partitions and files within the dataset\"},{\"name\":\"BloomFilterMetadata\",\"type\":[\"null\",{\"type\":\"record\",\"name\":\"HoodieMetadataBloomFilter\",\"doc\":\"Data file bloom filter details\",\"fields\":[{\"name\":\"type\",\"type\":{\"type\":\"string\",\"avro.java.string\":\"String\"},\"doc\":\"Bloom filter type code\"},{\"name\":\"timestamp\",\"type\":{\"type\":\"string\",\"avro.java.string\":\"String\"},\"doc\":\"Instant timestamp when this metadata was created/updated\"},{\"name\":\"bloomFilter\",\"type\":\"bytes\",\"doc\":\"Bloom filter binary byte array\"},{\"name\":\"isDeleted\",\"type\":\"boolean\",\"doc\":\"Bloom filter entry valid/deleted flag\"}]}],\"doc\":\"Metadata Index of bloom filters for all data files in the user table\",\"default\":null},{\"name\":\"ColumnStatsMetadata\",\"type\":[\"null\",{\"type\":\"record\",\"name\":\"HoodieMetadataColumnStats\",\"doc\":\"Data file column statistics\",\"fields\":[{\"name\":\"fileName\",\"type\":[\"null\",{\"type\":\"string\",\"avro.java.string\":\"String\"}],\"doc\":\"File name for which this column statistics applies\",\"default\":null},{\"name\":\"columnName\",\"type\":[\"null\",{\"type\":\"string\",\"avro.java.string\":\"String\"}],\"doc\":\"Column name for which this column statistics applies\",\"default\":null},{\"name\":\"minValue\",\"type\":[\"null\",{\"type\":\"record\",\"name\":\"BooleanWrapper\",\"doc\":\"A record wrapping boolean type to be able to be used it w/in Avro's Union\",\"fields\":[{\"name\":\"value\",\"type\":\"boolean\"}]},{\"type\":\"record\",\"name\":\"IntWrapper\",\"doc\":\"A record wrapping int type to be able to be used it w/in Avro's Union\",\"fields\":[{\"name\":\"value\",\"type\":\"int\"}]},{\"type\":\"record\",\"name\":\"LongWrapper\",\"doc\":\"A record wrapping long type to be able to be used it w/in Avro's Union\",\"fields\":[{\"name\":\"value\",\"type\":\"long\"}]},{\"type\":\"record\",\"name\":\"FloatWrapper\",\"doc\":\"A record wrapping float type to be able to be used it w/in Avro's Union\",\"fields\":[{\"name\":\"value\",\"type\":\"float\"}]},{\"type\":\"record\",\"name\":\"DoubleWrapper\",\"doc\":\"A record wrapping double type to be able to be used it w/in Avro's Union\",\"fields\":[{\"name\":\"value\",\"type\":\"double\"}]},{\"type\":\"record\",\"name\":\"BytesWrapper\",\"doc\":\"A record wrapping bytes type to be able to be used it w/in Avro's Union\",\"fields\":[{\"name\":\"value\",\"type\":\"bytes\"}]},{\"type\":\"record\",\"name\":\"StringWrapper\",\"doc\":\"A record wrapping string type to be able to be used it w/in Avro's Union\",\"fields\":[{\"name\":\"value\",\"type\":{\"type\":\"string\",\"avro.java.string\":\"String\"}}]},{\"type\":\"record\",\"name\":\"DateWrapper\",\"doc\":\"A record wrapping Date logical type to be able to be used it w/in Avro's Union\",\"fields\":[{\"name\":\"value\",\"type\":\"int\"}]},{\"type\":\"record\",\"name\":\"DecimalWrapper\",\"doc\":\"A record wrapping Decimal logical type to be able to be used it w/in Avro's Union\",\"fields\":[{\"name\":\"value\",\"type\":{\"type\":\"bytes\",\"logicalType\":\"decimal\",\"precision\":30,\"scale\":15}}]},{\"type\":\"record\",\"name\":\"TimeMicrosWrapper\",\"doc\":\"A record wrapping Time-micros logical type to be able to be used it w/in Avro's Union\",\"fields\":[{\"name\":\"value\",\"type\":{\"type\":\"long\",\"logicalType\":\"time-micros\"}}]},{\"type\":\"record\",\"name\":\"TimestampMicrosWrapper\",\"doc\":\"A record wrapping Timestamp-micros logical type to be able to be used it w/in Avro's Union\",\"fields\":[{\"name\":\"value\",\"type\":\"long\"}]}],\"doc\":\"Minimum value in the range. Based on user data table schema, we can convert this to appropriate type\",\"default\":null},{\"name\":\"maxValue\",\"type\":[\"null\",\"BooleanWrapper\",\"IntWrapper\",\"LongWrapper\",\"FloatWrapper\",\"DoubleWrapper\",\"BytesWrapper\",\"StringWrapper\",\"DateWrapper\",\"DecimalWrapper\",\"TimeMicrosWrapper\",\"TimestampMicrosWrapper\"],\"doc\":\"Maximum value in the range. Based on user data table schema, we can convert it to appropriate type\",\"default\":null},{\"name\":\"valueCount\",\"type\":[\"null\",\"long\"],\"doc\":\"Total count of values\",\"default\":null},{\"name\":\"nullCount\",\"type\":[\"null\",\"long\"],\"doc\":\"Total count of null values\",\"default\":null},{\"name\":\"totalSize\",\"type\":[\"null\",\"long\"],\"doc\":\"Total storage size on disk\",\"default\":null},{\"name\":\"totalUncompressedSize\",\"type\":[\"null\",\"long\"],\"doc\":\"Total uncompressed storage size on disk\",\"default\":null},{\"name\":\"isDeleted\",\"type\":\"boolean\",\"doc\":\"Column range entry valid/deleted flag\"}]}],\"doc\":\"Metadata Index of column statistics for all data files in the user table\",\"default\":null}]}" - }, - "operationType" : "UPSERT_PREPPED", - "writeStats" : [ { - "fileId" : "files-0000", - "path" : "files/.files-0000_00000000000000.log.1_0-52-57", - "prevCommit" : "00000000000000", - "numWrites" : 1, - "numDeletes" : 0, - "numUpdateWrites" : 1, - "numInserts" : 0, - "totalWriteBytes" : 10928, - "totalWriteErrors" : 0, - "tempPath" : null, - "partitionPath" : "files", - "totalLogRecords" : 0, - "totalLogFilesCompacted" : 0, - "totalLogSizeCompacted" : 0, - "totalUpdatedRecordsCompacted" : 0, - "totalLogBlocks" : 0, - "totalCorruptLogBlock" : 0, - "totalRollbackBlocks" : 0, - "fileSizeInBytes" : 10928, - "minEventTime" : null, - "maxEventTime" : null, - "logVersion" : 1, - "logOffset" : 0, - "baseFile" : "", - "logFiles" : [ ".files-0000_00000000000000.log.1_0-52-57" ], - "recordsStats" : { - "val" : null, - "present" : false - }, - "columnStats" : { - "val" : null, - "present" : false - } - } ], - "totalRecordsDeleted" : 0, - "totalLogRecordsCompacted" : 0, - "totalLogFilesCompacted" : 0, - "totalCompactedRecordsUpdated" : 0, - "totalLogFilesSize" : 0, - "totalScanTime" : 0, - "totalCreateTime" : 0, - "totalUpsertTime" : 67, - "minAndMaxEventTime" : { - "Optional.empty" : { - "val" : null, - "present" : false - } - }, - "writePartitionPaths" : [ "files" ], - "fileIdAndRelativePaths" : { - "files-0000" : "files/.files-0000_00000000000000.log.1_0-52-57" - } -} \ No newline at end of file diff --git a/tests/integration/test_storage_hudi/test_table/.hoodie/metadata/.hoodie/00000000000000.deltacommit.inflight b/tests/integration/test_storage_hudi/test_table/.hoodie/metadata/.hoodie/00000000000000.deltacommit.inflight deleted file mode 100644 index bb2542e0186..00000000000 --- a/tests/integration/test_storage_hudi/test_table/.hoodie/metadata/.hoodie/00000000000000.deltacommit.inflight +++ /dev/null @@ -1,116 +0,0 @@ -{ - "partitionToWriteStats" : { - "files" : [ { - "fileId" : "", - "path" : null, - "prevCommit" : "null", - "numWrites" : 0, - "numDeletes" : 0, - "numUpdateWrites" : 0, - "numInserts" : 0, - "totalWriteBytes" : 0, - "totalWriteErrors" : 0, - "tempPath" : null, - "partitionPath" : null, - "totalLogRecords" : 0, - "totalLogFilesCompacted" : 0, - "totalLogSizeCompacted" : 0, - "totalUpdatedRecordsCompacted" : 0, - "totalLogBlocks" : 0, - "totalCorruptLogBlock" : 0, - "totalRollbackBlocks" : 0, - "fileSizeInBytes" : 0, - "minEventTime" : null, - "maxEventTime" : null - }, { - "fileId" : "files-0000", - "path" : null, - "prevCommit" : "00000000000000", - "numWrites" : 0, - "numDeletes" : 0, - "numUpdateWrites" : 1, - "numInserts" : 0, - "totalWriteBytes" : 0, - "totalWriteErrors" : 0, - "tempPath" : null, - "partitionPath" : null, - "totalLogRecords" : 0, - "totalLogFilesCompacted" : 0, - "totalLogSizeCompacted" : 0, - "totalUpdatedRecordsCompacted" : 0, - "totalLogBlocks" : 0, - "totalCorruptLogBlock" : 0, - "totalRollbackBlocks" : 0, - "fileSizeInBytes" : 0, - "minEventTime" : null, - "maxEventTime" : null - } ] - }, - "compacted" : false, - "extraMetadata" : { }, - "operationType" : "UPSERT_PREPPED", - "writeStats" : [ { - "fileId" : "", - "path" : null, - "prevCommit" : "null", - "numWrites" : 0, - "numDeletes" : 0, - "numUpdateWrites" : 0, - "numInserts" : 0, - "totalWriteBytes" : 0, - "totalWriteErrors" : 0, - "tempPath" : null, - "partitionPath" : null, - "totalLogRecords" : 0, - "totalLogFilesCompacted" : 0, - "totalLogSizeCompacted" : 0, - "totalUpdatedRecordsCompacted" : 0, - "totalLogBlocks" : 0, - "totalCorruptLogBlock" : 0, - "totalRollbackBlocks" : 0, - "fileSizeInBytes" : 0, - "minEventTime" : null, - "maxEventTime" : null - }, { - "fileId" : "files-0000", - "path" : null, - "prevCommit" : "00000000000000", - "numWrites" : 0, - "numDeletes" : 0, - "numUpdateWrites" : 1, - "numInserts" : 0, - "totalWriteBytes" : 0, - "totalWriteErrors" : 0, - "tempPath" : null, - "partitionPath" : null, - "totalLogRecords" : 0, - "totalLogFilesCompacted" : 0, - "totalLogSizeCompacted" : 0, - "totalUpdatedRecordsCompacted" : 0, - "totalLogBlocks" : 0, - "totalCorruptLogBlock" : 0, - "totalRollbackBlocks" : 0, - "fileSizeInBytes" : 0, - "minEventTime" : null, - "maxEventTime" : null - } ], - "totalRecordsDeleted" : 0, - "totalLogRecordsCompacted" : 0, - "totalLogFilesCompacted" : 0, - "totalCompactedRecordsUpdated" : 0, - "totalLogFilesSize" : 0, - "totalScanTime" : 0, - "totalCreateTime" : 0, - "totalUpsertTime" : 0, - "minAndMaxEventTime" : { - "Optional.empty" : { - "val" : null, - "present" : false - } - }, - "writePartitionPaths" : [ "files" ], - "fileIdAndRelativePaths" : { - "" : null, - "files-0000" : null - } -} \ No newline at end of file diff --git a/tests/integration/test_storage_hudi/test_table/.hoodie/metadata/.hoodie/00000000000000.deltacommit.requested b/tests/integration/test_storage_hudi/test_table/.hoodie/metadata/.hoodie/00000000000000.deltacommit.requested deleted file mode 100644 index e69de29bb2d..00000000000 diff --git a/tests/integration/test_storage_hudi/test_table/.hoodie/metadata/.hoodie/20220830083647456.deltacommit b/tests/integration/test_storage_hudi/test_table/.hoodie/metadata/.hoodie/20220830083647456.deltacommit deleted file mode 100644 index 210201f7135..00000000000 --- a/tests/integration/test_storage_hudi/test_table/.hoodie/metadata/.hoodie/20220830083647456.deltacommit +++ /dev/null @@ -1,97 +0,0 @@ -{ - "partitionToWriteStats" : { - "files" : [ { - "fileId" : "files-0000", - "path" : "files/.files-0000_00000000000000.log.2_0-83-93", - "prevCommit" : "00000000000000", - "numWrites" : 4, - "numDeletes" : 0, - "numUpdateWrites" : 4, - "numInserts" : 0, - "totalWriteBytes" : 11180, - "totalWriteErrors" : 0, - "tempPath" : null, - "partitionPath" : "files", - "totalLogRecords" : 0, - "totalLogFilesCompacted" : 0, - "totalLogSizeCompacted" : 0, - "totalUpdatedRecordsCompacted" : 0, - "totalLogBlocks" : 0, - "totalCorruptLogBlock" : 0, - "totalRollbackBlocks" : 0, - "fileSizeInBytes" : 11180, - "minEventTime" : null, - "maxEventTime" : null, - "logVersion" : 2, - "logOffset" : 0, - "baseFile" : "", - "logFiles" : [ ".files-0000_00000000000000.log.1_0-52-57", ".files-0000_00000000000000.log.1_0-0-0", ".files-0000_00000000000000.log.2_0-83-93" ], - "recordsStats" : { - "val" : null, - "present" : false - }, - "columnStats" : { - "val" : null, - "present" : false - } - } ] - }, - "compacted" : false, - "extraMetadata" : { - "schema" : "{\"type\":\"record\",\"name\":\"HoodieMetadataRecord\",\"namespace\":\"org.apache.hudi.avro.model\",\"doc\":\"A record saved within the Metadata Table\",\"fields\":[{\"name\":\"key\",\"type\":{\"type\":\"string\",\"avro.java.string\":\"String\"}},{\"name\":\"type\",\"type\":\"int\",\"doc\":\"Type of the metadata record\"},{\"name\":\"filesystemMetadata\",\"type\":[\"null\",{\"type\":\"map\",\"values\":{\"type\":\"record\",\"name\":\"HoodieMetadataFileInfo\",\"fields\":[{\"name\":\"size\",\"type\":\"long\",\"doc\":\"Size of the file\"},{\"name\":\"isDeleted\",\"type\":\"boolean\",\"doc\":\"True if this file has been deleted\"}]},\"avro.java.string\":\"String\"}],\"doc\":\"Contains information about partitions and files within the dataset\"},{\"name\":\"BloomFilterMetadata\",\"type\":[\"null\",{\"type\":\"record\",\"name\":\"HoodieMetadataBloomFilter\",\"doc\":\"Data file bloom filter details\",\"fields\":[{\"name\":\"type\",\"type\":{\"type\":\"string\",\"avro.java.string\":\"String\"},\"doc\":\"Bloom filter type code\"},{\"name\":\"timestamp\",\"type\":{\"type\":\"string\",\"avro.java.string\":\"String\"},\"doc\":\"Instant timestamp when this metadata was created/updated\"},{\"name\":\"bloomFilter\",\"type\":\"bytes\",\"doc\":\"Bloom filter binary byte array\"},{\"name\":\"isDeleted\",\"type\":\"boolean\",\"doc\":\"Bloom filter entry valid/deleted flag\"}]}],\"doc\":\"Metadata Index of bloom filters for all data files in the user table\",\"default\":null},{\"name\":\"ColumnStatsMetadata\",\"type\":[\"null\",{\"type\":\"record\",\"name\":\"HoodieMetadataColumnStats\",\"doc\":\"Data file column statistics\",\"fields\":[{\"name\":\"fileName\",\"type\":[\"null\",{\"type\":\"string\",\"avro.java.string\":\"String\"}],\"doc\":\"File name for which this column statistics applies\",\"default\":null},{\"name\":\"columnName\",\"type\":[\"null\",{\"type\":\"string\",\"avro.java.string\":\"String\"}],\"doc\":\"Column name for which this column statistics applies\",\"default\":null},{\"name\":\"minValue\",\"type\":[\"null\",{\"type\":\"record\",\"name\":\"BooleanWrapper\",\"doc\":\"A record wrapping boolean type to be able to be used it w/in Avro's Union\",\"fields\":[{\"name\":\"value\",\"type\":\"boolean\"}]},{\"type\":\"record\",\"name\":\"IntWrapper\",\"doc\":\"A record wrapping int type to be able to be used it w/in Avro's Union\",\"fields\":[{\"name\":\"value\",\"type\":\"int\"}]},{\"type\":\"record\",\"name\":\"LongWrapper\",\"doc\":\"A record wrapping long type to be able to be used it w/in Avro's Union\",\"fields\":[{\"name\":\"value\",\"type\":\"long\"}]},{\"type\":\"record\",\"name\":\"FloatWrapper\",\"doc\":\"A record wrapping float type to be able to be used it w/in Avro's Union\",\"fields\":[{\"name\":\"value\",\"type\":\"float\"}]},{\"type\":\"record\",\"name\":\"DoubleWrapper\",\"doc\":\"A record wrapping double type to be able to be used it w/in Avro's Union\",\"fields\":[{\"name\":\"value\",\"type\":\"double\"}]},{\"type\":\"record\",\"name\":\"BytesWrapper\",\"doc\":\"A record wrapping bytes type to be able to be used it w/in Avro's Union\",\"fields\":[{\"name\":\"value\",\"type\":\"bytes\"}]},{\"type\":\"record\",\"name\":\"StringWrapper\",\"doc\":\"A record wrapping string type to be able to be used it w/in Avro's Union\",\"fields\":[{\"name\":\"value\",\"type\":{\"type\":\"string\",\"avro.java.string\":\"String\"}}]},{\"type\":\"record\",\"name\":\"DateWrapper\",\"doc\":\"A record wrapping Date logical type to be able to be used it w/in Avro's Union\",\"fields\":[{\"name\":\"value\",\"type\":\"int\"}]},{\"type\":\"record\",\"name\":\"DecimalWrapper\",\"doc\":\"A record wrapping Decimal logical type to be able to be used it w/in Avro's Union\",\"fields\":[{\"name\":\"value\",\"type\":{\"type\":\"bytes\",\"logicalType\":\"decimal\",\"precision\":30,\"scale\":15}}]},{\"type\":\"record\",\"name\":\"TimeMicrosWrapper\",\"doc\":\"A record wrapping Time-micros logical type to be able to be used it w/in Avro's Union\",\"fields\":[{\"name\":\"value\",\"type\":{\"type\":\"long\",\"logicalType\":\"time-micros\"}}]},{\"type\":\"record\",\"name\":\"TimestampMicrosWrapper\",\"doc\":\"A record wrapping Timestamp-micros logical type to be able to be used it w/in Avro's Union\",\"fields\":[{\"name\":\"value\",\"type\":\"long\"}]}],\"doc\":\"Minimum value in the range. Based on user data table schema, we can convert this to appropriate type\",\"default\":null},{\"name\":\"maxValue\",\"type\":[\"null\",\"BooleanWrapper\",\"IntWrapper\",\"LongWrapper\",\"FloatWrapper\",\"DoubleWrapper\",\"BytesWrapper\",\"StringWrapper\",\"DateWrapper\",\"DecimalWrapper\",\"TimeMicrosWrapper\",\"TimestampMicrosWrapper\"],\"doc\":\"Maximum value in the range. Based on user data table schema, we can convert it to appropriate type\",\"default\":null},{\"name\":\"valueCount\",\"type\":[\"null\",\"long\"],\"doc\":\"Total count of values\",\"default\":null},{\"name\":\"nullCount\",\"type\":[\"null\",\"long\"],\"doc\":\"Total count of null values\",\"default\":null},{\"name\":\"totalSize\",\"type\":[\"null\",\"long\"],\"doc\":\"Total storage size on disk\",\"default\":null},{\"name\":\"totalUncompressedSize\",\"type\":[\"null\",\"long\"],\"doc\":\"Total uncompressed storage size on disk\",\"default\":null},{\"name\":\"isDeleted\",\"type\":\"boolean\",\"doc\":\"Column range entry valid/deleted flag\"}]}],\"doc\":\"Metadata Index of column statistics for all data files in the user table\",\"default\":null}]}" - }, - "operationType" : "UPSERT_PREPPED", - "writeStats" : [ { - "fileId" : "files-0000", - "path" : "files/.files-0000_00000000000000.log.2_0-83-93", - "prevCommit" : "00000000000000", - "numWrites" : 4, - "numDeletes" : 0, - "numUpdateWrites" : 4, - "numInserts" : 0, - "totalWriteBytes" : 11180, - "totalWriteErrors" : 0, - "tempPath" : null, - "partitionPath" : "files", - "totalLogRecords" : 0, - "totalLogFilesCompacted" : 0, - "totalLogSizeCompacted" : 0, - "totalUpdatedRecordsCompacted" : 0, - "totalLogBlocks" : 0, - "totalCorruptLogBlock" : 0, - "totalRollbackBlocks" : 0, - "fileSizeInBytes" : 11180, - "minEventTime" : null, - "maxEventTime" : null, - "logVersion" : 2, - "logOffset" : 0, - "baseFile" : "", - "logFiles" : [ ".files-0000_00000000000000.log.1_0-52-57", ".files-0000_00000000000000.log.1_0-0-0", ".files-0000_00000000000000.log.2_0-83-93" ], - "recordsStats" : { - "val" : null, - "present" : false - }, - "columnStats" : { - "val" : null, - "present" : false - } - } ], - "totalRecordsDeleted" : 0, - "totalLogRecordsCompacted" : 0, - "totalLogFilesCompacted" : 0, - "totalCompactedRecordsUpdated" : 0, - "totalLogFilesSize" : 0, - "totalScanTime" : 0, - "totalCreateTime" : 0, - "totalUpsertTime" : 39, - "minAndMaxEventTime" : { - "Optional.empty" : { - "val" : null, - "present" : false - } - }, - "writePartitionPaths" : [ "files" ], - "fileIdAndRelativePaths" : { - "files-0000" : "files/.files-0000_00000000000000.log.2_0-83-93" - } -} \ No newline at end of file diff --git a/tests/integration/test_storage_hudi/test_table/.hoodie/metadata/.hoodie/20220830083647456.deltacommit.inflight b/tests/integration/test_storage_hudi/test_table/.hoodie/metadata/.hoodie/20220830083647456.deltacommit.inflight deleted file mode 100644 index ea1b6a10c13..00000000000 --- a/tests/integration/test_storage_hudi/test_table/.hoodie/metadata/.hoodie/20220830083647456.deltacommit.inflight +++ /dev/null @@ -1,116 +0,0 @@ -{ - "partitionToWriteStats" : { - "files" : [ { - "fileId" : "", - "path" : null, - "prevCommit" : "null", - "numWrites" : 0, - "numDeletes" : 0, - "numUpdateWrites" : 0, - "numInserts" : 0, - "totalWriteBytes" : 0, - "totalWriteErrors" : 0, - "tempPath" : null, - "partitionPath" : null, - "totalLogRecords" : 0, - "totalLogFilesCompacted" : 0, - "totalLogSizeCompacted" : 0, - "totalUpdatedRecordsCompacted" : 0, - "totalLogBlocks" : 0, - "totalCorruptLogBlock" : 0, - "totalRollbackBlocks" : 0, - "fileSizeInBytes" : 0, - "minEventTime" : null, - "maxEventTime" : null - }, { - "fileId" : "files-0000", - "path" : null, - "prevCommit" : "00000000000000", - "numWrites" : 0, - "numDeletes" : 0, - "numUpdateWrites" : 4, - "numInserts" : 0, - "totalWriteBytes" : 0, - "totalWriteErrors" : 0, - "tempPath" : null, - "partitionPath" : null, - "totalLogRecords" : 0, - "totalLogFilesCompacted" : 0, - "totalLogSizeCompacted" : 0, - "totalUpdatedRecordsCompacted" : 0, - "totalLogBlocks" : 0, - "totalCorruptLogBlock" : 0, - "totalRollbackBlocks" : 0, - "fileSizeInBytes" : 0, - "minEventTime" : null, - "maxEventTime" : null - } ] - }, - "compacted" : false, - "extraMetadata" : { }, - "operationType" : "UPSERT_PREPPED", - "writeStats" : [ { - "fileId" : "", - "path" : null, - "prevCommit" : "null", - "numWrites" : 0, - "numDeletes" : 0, - "numUpdateWrites" : 0, - "numInserts" : 0, - "totalWriteBytes" : 0, - "totalWriteErrors" : 0, - "tempPath" : null, - "partitionPath" : null, - "totalLogRecords" : 0, - "totalLogFilesCompacted" : 0, - "totalLogSizeCompacted" : 0, - "totalUpdatedRecordsCompacted" : 0, - "totalLogBlocks" : 0, - "totalCorruptLogBlock" : 0, - "totalRollbackBlocks" : 0, - "fileSizeInBytes" : 0, - "minEventTime" : null, - "maxEventTime" : null - }, { - "fileId" : "files-0000", - "path" : null, - "prevCommit" : "00000000000000", - "numWrites" : 0, - "numDeletes" : 0, - "numUpdateWrites" : 4, - "numInserts" : 0, - "totalWriteBytes" : 0, - "totalWriteErrors" : 0, - "tempPath" : null, - "partitionPath" : null, - "totalLogRecords" : 0, - "totalLogFilesCompacted" : 0, - "totalLogSizeCompacted" : 0, - "totalUpdatedRecordsCompacted" : 0, - "totalLogBlocks" : 0, - "totalCorruptLogBlock" : 0, - "totalRollbackBlocks" : 0, - "fileSizeInBytes" : 0, - "minEventTime" : null, - "maxEventTime" : null - } ], - "totalRecordsDeleted" : 0, - "totalLogRecordsCompacted" : 0, - "totalLogFilesCompacted" : 0, - "totalCompactedRecordsUpdated" : 0, - "totalLogFilesSize" : 0, - "totalScanTime" : 0, - "totalCreateTime" : 0, - "totalUpsertTime" : 0, - "minAndMaxEventTime" : { - "Optional.empty" : { - "val" : null, - "present" : false - } - }, - "writePartitionPaths" : [ "files" ], - "fileIdAndRelativePaths" : { - "" : null, - "files-0000" : null - } -} \ No newline at end of file diff --git a/tests/integration/test_storage_hudi/test_table/.hoodie/metadata/.hoodie/20220830083647456.deltacommit.requested b/tests/integration/test_storage_hudi/test_table/.hoodie/metadata/.hoodie/20220830083647456.deltacommit.requested deleted file mode 100644 index e69de29bb2d..00000000000 diff --git a/tests/integration/test_storage_hudi/test_table/.hoodie/metadata/.hoodie/hoodie.properties b/tests/integration/test_storage_hudi/test_table/.hoodie/metadata/.hoodie/hoodie.properties deleted file mode 100644 index 845df718f6d..00000000000 --- a/tests/integration/test_storage_hudi/test_table/.hoodie/metadata/.hoodie/hoodie.properties +++ /dev/null @@ -1,14 +0,0 @@ -#Properties saved on 2022-08-30T08:36:47.657528Z -#Tue Aug 30 08:36:47 UTC 2022 -hoodie.compaction.payload.class=org.apache.hudi.metadata.HoodieMetadataPayload -hoodie.table.type=MERGE_ON_READ -hoodie.archivelog.folder=archived -hoodie.timeline.layout.version=1 -hoodie.table.checksum=1983687495 -hoodie.datasource.write.drop.partition.columns=false -hoodie.table.recordkey.fields=key -hoodie.table.name=test_table_metadata -hoodie.populate.meta.fields=false -hoodie.table.keygenerator.class=org.apache.hudi.metadata.HoodieTableMetadataKeyGenerator -hoodie.table.base.file.format=HFILE -hoodie.table.version=5 diff --git a/tests/integration/test_storage_hudi/test_table/.hoodie/metadata/files/..files-0000_00000000000000.log.1_0-0-0.crc b/tests/integration/test_storage_hudi/test_table/.hoodie/metadata/files/..files-0000_00000000000000.log.1_0-0-0.crc deleted file mode 100644 index e016a7f52627a5b0d3862ad6a1f662fc3ad8d94a..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 12 TcmYc;N@ieSU}9ifcZC}O5mf@t diff --git a/tests/integration/test_storage_hudi/test_table/.hoodie/metadata/files/..files-0000_00000000000000.log.1_0-52-57.crc b/tests/integration/test_storage_hudi/test_table/.hoodie/metadata/files/..files-0000_00000000000000.log.1_0-52-57.crc deleted file mode 100644 index c1136be0c0e257f2aa3aeed6a5c32dd7e097379c..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 96 zcmYc;N@ieSU}De?RQxLt_>_I4%7)XcTl}8oT=)}k%-!ySA(_>TZG7lbpTeoRdX$3CKW0JTK069!9Qvd(} diff --git a/tests/integration/test_storage_hudi/test_table/.hoodie/metadata/files/..files-0000_00000000000000.log.2_0-83-93.crc b/tests/integration/test_storage_hudi/test_table/.hoodie/metadata/files/..files-0000_00000000000000.log.2_0-83-93.crc deleted file mode 100644 index 0aaf9e50256c454e3ecb14ec9a5c5fae645fa37d..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 96 zcmYc;N@ieSU}ES={`d8LX<`T0sRhSg8n$*$xud%G6-VxaBWxdUt-gH5P1~*4QmFm& gwYUbcDec;eKAvHn%Cvog%j`|7N-J<-`{&K~0Q~|n`~Uy| diff --git a/tests/integration/test_storage_hudi/test_table/.hoodie/metadata/files/..hoodie_partition_metadata.crc b/tests/integration/test_storage_hudi/test_table/.hoodie/metadata/files/..hoodie_partition_metadata.crc deleted file mode 100644 index 1b5364eed343b390fb97133b3c42ae8d9e1c6af5..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 12 TcmYc;N@ieSU}9K*=PV-t6l(*N diff --git a/tests/integration/test_storage_hudi/test_table/.hoodie/metadata/files/.files-0000_00000000000000.log.1_0-0-0 b/tests/integration/test_storage_hudi/test_table/.hoodie/metadata/files/.files-0000_00000000000000.log.1_0-0-0 deleted file mode 100644 index 9bf687c1a4b5fe36c79ae9759b132a53717475c4..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 124 zcmY%P2zBvPW`KY^AjOD;L85#HC;%h`1QtNd1jGi64ADOMMd^Bp1&PTSsd^cuDVci7 j`MJ6Id3w3|DXBSnE~z=GC8PcPz0)n2wJ|VGo!@#RMsVc`%MB zneYAtA;HizP6+rcN1fy<3Fd^vkc5+xke~`+MnZ_ekj0SN zE<_b)BmLVY$r9i<1DTXQ}6zuVKq85Tk&SDUW38fOFMKlJ0@ichf zhH;gGS%;!Y441(X+7+3lQ7NKQ(69$HB#GCTJqlfe6EDP|dXJ*jX#5K?7mbiKMq>mn zibiPp3o$9h6sU;x?0-X`{swiz(Kvtyv+iGo#2`+ZpFH3YwR4j6oWROW3j)S)nsFGb z6(JBJL`Ye6C^fp%$OqOlz-)zzr)X5El~6B+bl8+gG`JaqC{!p=mhR{NweAZR5a3cq z8of11VF_I#gcSsko)&)yxTVFX&BSA@C+}8o_W1!oWUACBldaLZS#24e7*1K?p8k7ozmf zC3D&-l-38&R}h2}utKBcz?ApTt)3~}#(ltw>J3LR3o#$TN?_Aq3knJh>+OYZBGS2a zFiO-qiv^-xiZKU?LEzHgH4|PUW${(_#`u zfIys8(ZB^jD0-4-c~Lt+2L_Icr-7)Sxt}NuN62`H837?AiNxz08-x_Qk%Qtium#Q# zCUMpX5t1qp84T2@6P#8TkN}sUam*>u_a)FXK|+W^0Ws>*Xik#{R21+vt;qp>MQ(3G zp;F=@G(uC_m2^aCGl9-K{VPz7Nmb?d+dav-y?7$=!Ae{bVW@dhGR;Vk|`XSzDC{jlL#(DoghBw zFeDIYN!a3#tqIu*5Td}d1SP_QVnI2M;GiVOV^9)=d!{7Z`Yu#gm=45pp!DlIz`Ry4#?qU^*dtt1C zTIla_%DU2Qk0Grd`z#hN<_9cw1m72d*W6Cc1)fvB#*7J{y3v$*2bVZ9 zG6TG>O2=+U52~s9$l0IgSjDTVspW1-m*sbkG2)GmEG;vJjb1MLcHCSr!7*!}y})Nh za8Qtt`8prGxIOI)_3bCWXS`rUPDe-C-n|_uWw~2xw$}W0@_I+y2ah_(mJZ{jq4hE@ zF!Cc6R;XupBKS#rMQU-+>#{psdy{j)E8fai5U@nB%ohMMzQy5K`Sbu}GNNl@`HD9Q z+j(OxE|}#kSyEnM2}O1C-e*&hf6VGEx&@7Yz`v}WC^h^)?s2ksZ#Z%uCzox0Op9(uVL7qxkO zDYEf8SikDtf^5&);&Y;tc=6>s-#G>ccNG6mTd z@Q6tt>|L^+oJ8Cu&EJu`$CYdn7kZr&A1zwjcmm#Bmwv%)f^EA)ds^YejJ(X4Npf=C zNsFMb8dHB6_37px5-X*N>$$f_G}0#~_*nv=guqWxM%+;(>3$$r(P?nAKF&U#R|`_D03avoH#PrchiRW4nehPWlQ+3 zCwBWcjGj2&?o8X2#$UqnzRAB(1PrrZ#7U+u}nYp>ai~WA@UT?wT;on`pC^CynKG!ZqB#m_{f7xGqE$Nhsc~iW(qF_O?)5D-rtnGJF2!|^vg$f+>zz&BEu?N zuBJG+q`aK!V3yS~qugY+gKOmJSFWEHIQY!nG^1MSbRu_8tWPQ4YPo&2S;7vp_1vm_ z?vLSl$AyT|5zokorUf6rX}xh@-b$~~_4fHz=6{}ZT+hb}UE9OXjtLYUT9xNHbHD!- zx5+7W4uL;rJkF`F9PXC7rE=VHRJ6^pSeE{q;=8sA=jQ9yk6YUwtbR-NbT22aKxz3| zx#fgpmnw-Oy=X_Br`c`C$L9P|rVWyWVb!boS4A^O`%lLe@>(V9#cTM7hkt2*XPXex5XC^6AjmBSMV?=m06!x+$sn-w2K z*m8ShVi1Nk6ynF@D$s4knU&(~HDmn#^7Fch-|aMU_`!9Xj}?d0d^;ph7|cX|*e<}d z)o5(wL@so3=?tUU9CxGFjQlxoa8}Ok`G*olfWH#htmvfvJIbG_9G5Ql2Dug?5pXM# z`J2md85l4yU|_(&fPn!60|o{R3>X+NFkoQ7z<_}P0|Wmb41i~%U5`aMj`{xqt$WqU diff --git a/tests/integration/test_storage_hudi/test_table/.hoodie/metadata/files/.files-0000_00000000000000.log.2_0-83-93 b/tests/integration/test_storage_hudi/test_table/.hoodie/metadata/files/.files-0000_00000000000000.log.2_0-83-93 deleted file mode 100644 index 13a41e55cb2603f59eaf78103707eed6f56d0e61..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 11180 zcmeHMdpuNWA0HB;RW?Zp$56SBYZ7(~lO+r?x?wqH4o+syndY2vNp^cn%XV2xS1YZq z>+Rm!WV=zQRVZb*(2X`JZJV}Ky1dUhGls}zd-HzYKg{PdA7jq*{GRXg{e7R`n6O#z2bqG?7Vo}V{_7$n6kw{n~Aem0okR*$q>=DR94F4zwrF#sEK@vWS zIT0)-Q4+BDz=8DI@dtuO?f5B>ok61n8xbHRLLD&Kfs4R#0Rm(4 z>0Ge{flzAuC;?IPp=g+ZA`k?Fz=@*gCWR*TO0)bBxZ@ZJqZk39U>{;3Oo<7CBX9`` z$;CxLFvh1BB08T->a-IG*%`c>0LO)Z6%rBmO?mI!b}}X3I2)`e-Eai85cLtP1U3z} zARxf7?p}CDL~^(GM~UsuqJgw8MVW(mAaHfwH5G0M5*AVllDh1J7V^S?5sZZ7=AgJ} z;8NPla+SvcPxHhG3&cw&5P6laxlz@1QLk)z59tkF<6`cQ6nH&EQS+0 z8|#M@-ysLVNMH+`A(U?=6CxBTfTNf|jdH=Y>jLt^_((i;3Of4|=ouG7us{G&>XKJM=t~cmyswzqP+LZ zk2;+kDowU4qZ@|)NKsB6@GJsR!I`EhE*j-UOZCwW4Fros0#xeu9+I4N&PPNiCG4Z& z6QbB`DjxUDU046PNdx5^G1xpJmOD*eTEtNgKn(!NA}T~5XA%do6U00~t@I-h8XrPQ zD9#Ef&lQv^lL#mX1A!4(>>gq0oFT`^Y!B8{1nyC2UKrUoNRaUc0w9%0dj;nMQdQsR zsDw5U@aRO@D?Sea4)=q;NNU(XAfyfVP9YnY06FzdBA*^N5Fn+Ndq?R;<%PaM(&=L$ zFiP`MufCR+g?(d`M$rKPDL=yfiPODq1*DFCD1sUV3GgVC2MY#ZJ9HV{J4}R!3SmLt zl(VG+2N0~Bd#O+?m@;0t(ms?g1pq>?2qXfOpaQg$oi%9m0C%A4w3(dL>rT0#Z1Y8V zV!R(`fEsHlqzC^mHZ1EF-rX-%7s@a&``#H{kp7eH(Gv!KkHCqb69SHr7d5^ph6*J@ zs&J(G8fnunhOsDw1-XF^gAak0gf9N*nvkvlAp+bb2w@8-4wU0C21;@)77>GRPqjoK zi-SBb#zkV3I!$y*4yxZC${-;e|Nb(A*{z&-CvE$@lP0?)rLGB}OQ`J{*4~W`vXj+u z1mCAr>bv*&xPw(V{-d!9YN3C|DeX#kTtfHk(w3;b@Rj;s4wH+MAkgusp;RwGPo#U5 z9=t_e)O?6xYF^*gNeT_soy+UFUSSg?E{3BJh-&pIk@8U@2F&b9X|(kq3|vxyfJFdd z_kSfxr&*xq_(#k3qm7eQ?)0I9KCq4J-hzBkD;Jt1y@}NRo9)VVb@TBy1DBL`;DU0} zTiFX-RZa(E6#NZVs!GQ)R$p}H46zxh9RAY^$LUU)+??_CPZRONvC(@U8=bDIU;cDg z&TZw?^Zpk_Y86GJ3u{bF{h5!w8zM_vrc6xUt7`Z!i+v?0^V1wT=uCIkZ)-PhP15Fb zE{E7IPUW@Cb-;=nui$DLPb>Bts1|(oEOx4|%~jjmTHboU#yUql%YE*2q;#z)L;U0J zQ)yFq%#@Jde=(ZVXz;>lzI}P^@*<7gg&M2k^x1#wLF$Ub%?w;T8m$>WYuT%uvE05Y z;0$UxAus*@u0LPpt-Smo#l`Vn^W3*FIf2HkkO%Yk!QKni-B1<1n4zlWM#{T<)*W73 zv)Q;JOC!w&lP5&=- zaokJ0Nq{sgX6#)r_S{*1eq39?nT`3 ze}?wOfO)!!fseMF&^`WON6w1pQLhYcB&2K(tzB4E>{X-GGIHvTYxt?nFJfaIrXAT| z@%Gi}7w2BwN_l2=L-M5l-w#9ctrEA)u}aIXUhs2Vmfx1f>vt=^%)kEdX;|oy6JJ#h zxxBq~!8Mh}Z`}0{Eqe0Od3+12Cic&$inA3@UZfALRQXzx`Xoos<%D2L8>6m?xoFsn z;)I*?L+lfiLjKOE(G1GU7HJmd8`v6}9C-aW?X?T>=BwMs_uP{Bq`g_R@X`II)Phf@ zz$bkaTd$6`FTHuwd#va;BvYAt`FzWd;{ptP`QqT+e~mx9Xp{1B&e?DGp0y3v)KRWd z$(VTPDEpF)`nlu>>FV3uJ;eN~-OEyok`1E*eyA&b(3Z7y%s7o~tAN>k&-ADf{_tyayk2(1sbK|Ura#!;y1}JNDp6j>FrllbgPFKbx0*V!_eR+N{mtP4lLvo?p(&t4f)})>!RSAELL} z!kck#MW$JKeTdtL#b!rKNTp@GaXWq1o6L!MJzo1>YEDS5SIO5_m(q4Tu4{QzUn zwZV&!lrneCv)<~QZX3b9aNTc~ha;!4?7}3&Ab#N{$kib()^5*mV$GRAQ~N*MG&R05 zwwYn0>6D&3arsfL9HzB;Zp+DHRg+!k?*y+|XTzTH6Q4M@hZnrmZ2Yt{F1jfyjw{lv zhjW&xubIDj`B2S=JBk-ynw?;p<+s=*1UYc1&cwIj(q--YstnCcEfHUF^tH$W{})QO z4=1g1TkBU9Fx~fXBUf2H;QUj&YM%o0*JDMl8K$o4ZW+m(xOMf4I-|xN3?K3MmhmX& zSP!i~9L9tMlshxSWpF1a>gRCpvwmWyWUr_k4 z1}Em%ndarMt6SVJZ``w@iKAw^G-YTClsD|M@tv=tpFUplNzV3wVqwki$z>JS zjGT23o_M^n;ArWh<4JGk&)za^0^>_F-5D`9MGCikY{%9<+&ke4XX!6x`mWImw=;cy zuwnX&UJ*0)D`}lxZ>n!4vWQiked}kovl~9~*F)OPTOs`&>+i5`{aABBV?>_&MLQK_ z!l@^oRhFlxuzzy5wlA#Ks^qJ!6yDX*Ip&_8cdptZIG!aqH1A?F6kL+!yK#cSR|y`g=WB^Q*!6b-ORUJ=|8gsggHNSafBaetPq3@sJG5>Plm5SYex{ zs6owD_vYfEty%{amK`{?kK<|ZaxbfOR6&XU^ja$(?^T#%)^%;|*VTlE?vCFoozI2V z*^Sb8#0$99sAo9Fg~N+zZi%$YsozlTwX%MeXo5}o)}~>;DxS?ZKRwG=+@dr7&Pgra zhUi)W!|N~eHQSRLu(JJ(nA+is@_#9hzsj)~U%T?`ovNI?#SDw}HGdpFGFMk$P!u#X zG@~Jb+jRZLHnrRvmb)S^1#ZrCZkxsReWRY4lsK zKkdF*oL$=(o$a+H?9mLy?e9XbaA#2|k7?|ydUwdkD~dYiP=%j~f;GcYX}W?Z!=EwV zw(C!6C=Px^q3Z#b^oKZ4s%G|{?GCC8SPX+l;nZ(@gRemb1{oM+V32`91_l`zWMGhi eK?VjH7-V3Ofk6fa8Th|o0NjgqT#Yi!cl-we?@3z# diff --git a/tests/integration/test_storage_hudi/test_table/.hoodie/metadata/files/.hoodie_partition_metadata b/tests/integration/test_storage_hudi/test_table/.hoodie/metadata/files/.hoodie_partition_metadata deleted file mode 100644 index abeb826f8a4..00000000000 --- a/tests/integration/test_storage_hudi/test_table/.hoodie/metadata/files/.hoodie_partition_metadata +++ /dev/null @@ -1,4 +0,0 @@ -#partition metadata -#Tue Aug 30 08:36:48 UTC 2022 -commitTime=00000000000000 -partitionDepth=1 diff --git a/tests/integration/test_storage_hudi/test_table/americas/brazil/sao_paulo/..hoodie_partition_metadata.crc b/tests/integration/test_storage_hudi/test_table/americas/brazil/sao_paulo/..hoodie_partition_metadata.crc deleted file mode 100644 index 901b972aff0c2c1daa875aaf7ffe23d1fd77e34f..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 12 TcmYc;N@ieSU}9)DlYRvN5nclQ diff --git a/tests/integration/test_storage_hudi/test_table/americas/brazil/sao_paulo/.8a9a08bb-8cbc-4ec9-a2d4-8a6cdcaebbad-0_0-73-83_20220830083647456.parquet.crc b/tests/integration/test_storage_hudi/test_table/americas/brazil/sao_paulo/.8a9a08bb-8cbc-4ec9-a2d4-8a6cdcaebbad-0_0-73-83_20220830083647456.parquet.crc deleted file mode 100644 index eaff8b6a2b1a4363260f136c3a81f30a515c23f3..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 3432 zcmYc;N@ieSU}Et4_Q^jnZDTXn?nK7z2iUh?SX!{-Ka;_`y8G6=OaC^Clrt}?{;_xW zu>-E_;srQLpE);}OiL4z|3WPstIk$HtXbZZ=dCc4RZPdGen;n#L~P<9v8=N;X}?x< zUcx1>)jzQpmmEIv-`7$O(Kt_amnw^9iwYLc`+G)$Cj-a~oH@!qj@PI#|Kj#L~OZruZYP zy*7(~Gctd$us5V$)xhVLMQ5Zxqstq59}7}JM}hgW%T97H#3;ivl?=wfkB?#bw4==|f=+9Bv- z`1q50EUsEKyA|P6Lkqd9aZ@Eh=Jqb{EJrp!*z3w;WIi(cp&|cUWIj4O_0~L1bTJU$ zIqdK@WbyXk+7-xrWOnP-JQVjJ%h7}F@X>*v9`=x8N2-WQ_o}l;ucPa+TBg1ST@0Cj zLg_>rh`-O>b|r}J2s59KEKe+(Gw&(EIFYv~b%vRD_tf4uFt*L2V0joD8y!3Cq8~PK zLSl7l-Sg1Z+_;~gg)Ro-i#}*I!zPYPOvfgl3zr;7+*_goXFCp?zHP2cvAPSJT4b@n zs85B+d~ECnGuEXYj@!X)E^P9+#Fl-&1J3`r)!o7IB1u!&Pk gtRcgX?Dn7bagPbO+z}tYD-K<*&gZnGueNqE0Lh$Gn*aa+ diff --git a/tests/integration/test_storage_hudi/test_table/americas/brazil/sao_paulo/.hoodie_partition_metadata b/tests/integration/test_storage_hudi/test_table/americas/brazil/sao_paulo/.hoodie_partition_metadata deleted file mode 100644 index a8c28b02044..00000000000 --- a/tests/integration/test_storage_hudi/test_table/americas/brazil/sao_paulo/.hoodie_partition_metadata +++ /dev/null @@ -1,4 +0,0 @@ -#partition metadata -#Tue Aug 30 08:36:50 UTC 2022 -commitTime=20220830083647456 -partitionDepth=3 diff --git a/tests/integration/test_storage_hudi/test_table/americas/brazil/sao_paulo/8a9a08bb-8cbc-4ec9-a2d4-8a6cdcaebbad-0_0-73-83_20220830083647456.parquet b/tests/integration/test_storage_hudi/test_table/americas/brazil/sao_paulo/8a9a08bb-8cbc-4ec9-a2d4-8a6cdcaebbad-0_0-73-83_20220830083647456.parquet deleted file mode 100644 index d929431ccd391e2d88189be66c42046c70a86252..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 437831 zcmeI*4Rl;podEDjOw-WqC+!<18ll7zK{_Q((tNPEkd`7dJBiDGIymYWX-W2iaZJeUoI8fqYO( zQZkcY543OIz4w3r_jljCcV8wmb4Kg?Rt9CkJA%`L9S?oxhWRH1r9oLFa>O+ik-t3lAtTNV&CI`D#^BqjNRt2S=&x2KjoD2=-uTfb*0;T z+CLi2EX-W~o;w%*?I#ySn$B6&uzoPwOC`ZmCBe2^?)+rMnBjumE9bvyWyP%Z#~$-S zb>|PBnl<-@^FDpW{~dAUckf?v^Y1&qe(7y{f7>>;FE{ z_L*mHdwa{fj#=9s`^uj;oUruR@=w42riZKVy6(G2Z94SgPd$9^4^R8>f9<{H$B*y& z=`+7Q=YiR~ufFPz(+)7ZNN_@s4t{*|m!3Uj%-I<{KrVRn*n1oQ`Ik3d+VgLpj{f4| ze|cfi(~n;EyBjjQW=B4J>^t9g_+aPRNWZ-1)_Z185NW}8HvIm3|M1)XKWu;E={@a- zX7-&}@$c#LZ#%ndLF5zbAKEq$sVt}pP7a>_!oHs@80Gb3c)*@|%dD*FOA_Z*99|!vi;5__>dNeedF}kwdOM`INj! z-@fOLo90Xq>9C(IKl90Dzxwra&-Pzb_UL6Fy6UQ}vu=F$uMdAB68UEL*5`*1-=L%Y z&B5y6n$KSQr`er)b?TJKVJef+jv^3Q< zrJGU>m9b^1`sVhI`gB9QF5l2;w_0}L%b=GPB$gPeXCkl<%O+Gbf9j(9_=qa~iG3p;F1GkZMAmrli*SMB8lz3_3sc+kq$s{{dEBaEK``Y>v?OpksZM~_EOm8wc zJQB=3sx-E^ZLo4%N2a^GGpyd(ojNLVT2=NOnJr)Pit^2=^LjG*6%Nf+7>wKYvD8KR z#nD{xxryGs&c4n}Px$QY+mv5rQLak5vn$os6F#AHrE|gr4rxzq?CfdlO7um7{KuJ| z!34_(5BdBivr|3Ekp_d0vemQA%hHM7)L_eb!zsNoqgew-VJ>7@*sQc~^I)U0En7O1 zLnYDj%Bsqj7Svo)70j(%RZ&)6IVNXN7A=o{wj}!VlA23uDyqgf=Z>)}8aaCOxp|Cw zk;>)aBC{+g%`T#||3spHw=7r^z3;G~JyP?la5bGxEfS_S=1dOcwx=Yzqa>_*LSy9U zvCrdUpWbW3r$pFm`1}id?OzteqW!U;H&XM*(ve=rI6MaW>@SU8R~pvL_PJl@{Mddy zwevH&JDbr>l|g?=q-ICBNXpI8IEUmwS~r$NpD77z*KD6vHNo{`f}(KZ#y<;A&~A*0 ze0*6DjsE1#L3oa<@vVDx+o8QM5vwiG5Jln^E z;hHsf{0yBk2M?N(=z}F;Y0ZE9SylduA%8pUC~qy1Uor;Xca942esg$mN~C7jypeI` z&dCFvZi;Rz2}{FKEeONs7D2I!+{!2$e)Qh_@R!$Qr)6n& z052{YnU-Mx4u@4>6(K)#nne^OF&_u`tz z53P!gEZ_1rL!2@9k8vRNXS>^XWUx9?vvWyRP&V`=46npPQ*d=jSP)J@ZW$OGUK?ho zVD#SnDfnS_3i_+FQ?M(%^Uqx)2d?e|;RmlzyMigaK0TJrVPAFj`V@Wc&}ey0Mf8V< zEeY3|U9rVOC*Ku`^LsNZHg{}FbthuCS8T2ByXf3h_3_n%Ypv?#)!Efj_;_TcH?+JP zD%yN*q9a>AYC#y5r#n+!$<5WrpS?8~CASXCTMCBt2D?1Fx@SvQSNL#qUvFp6#_DsH zS0^(aVd!vIcn25WaP?ImpRID~^6}ygEFC8$;K)*QQo;?d0|%C-lM-@xDf_yj4KIC1 zM@eS3gzNf&X&6U;sBxy}KpGD&szUcne6y?i1L<;j>a({1 zqXWG9a0l)b4y0k<=FzRXGDFX-acr~q9-~{0m*hE@Mq-OD=$E~d=5B#0ZR%p&{ zrdrpx_M|`1FtFQS1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBly zK!5;&LK0Y4NXg2T009C72oNAZfB*pk1PBlyK!5-N0;2@{lyek`9j`&a@$(w6R3$)w z009C72oNAJGX>Vn%wZcf0RjXF5FkK+009C7rl&yoA<_O27$vZF)E+ws5FkK+009C7 z2oNAZ;2;RBJ_vp2f&c*m1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009E~E8sQE{gqOW z009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBly zK!5-N0{bDben0A{LZFZYe78j*&6ZpV6o7ztHy6MpNs#~n0t5&UAV6TI3;0>UOg{pQ zp8x>@1PBlyK!5-N0t5&UAV7csftf3?uAqj#rl7KwECB)p2oNAZfB*pk1PDx>fIGyg zvlW>D0RjXF5FkK+009C72oNAZfB=E%Bj9JD(`PPZLVy4P0t5&UAV7cs0RjXF5FkK+ z009D1A>iZ7r$SXs0t5&UAV7csfpG*1>qAz?8AP$}HaW#Qv&Kb$009C72oNAZfB*pk z1tZ|*ykMqEq67#KAV7cs0RjXF5FkK+009C72oNAZfB=C)7x134LZ3nT6Cgl<009E8 zQNVBJyhbq92@oJafB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk z1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pkGfrURj2p256Cgl<009C7 z2oNAZfB*pk1jZ5Y%_if(*iC={0RjXF5FpU%MKuQj0RmGe;0!Z$wjvWCK!5-N0t5&U zC>nt^MKcluAwYlt0RjXF5FkK+z$6NIm1Pn!g_?!}>!)E(QX)Wr009C72oNAZfB*pk z1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+0D&009C72oNAZfB*pk z1PBlyK!5-N0t5&UATYxP*3R(p8$JO71PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C7 z2oNAZ;57(%YwByjQk4J!0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N z0t5&UAV7cs0RjXF5FkK+009C73PZrxo)*SD$&vs80t5&UAV7cs0RjXF5FkK+0D*!L z*jO-`N|XQr0t5&UAV7cs0RjXF5FkK+009C72uz;9n#l_#5&;4P2oNAZfB*pk1PBly zK!5-N0t5&UAV7cs0RjXF5FkK+009CAQ^02u9!zb05gARv;54 zO#%c65FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZ zfB*pk1PBlyK!5-N0t5&UIA{WE4_Z$;B0zuu0RjXF5FkK+009C72oNAZfB*pk1PBly zK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF z5Fl`X0{iu$S?dlkxMl z0t5&UAV7cs0RjXF5FkK+009C72+TAAzp*~kX54rQ5FkK+009C72oNAJ?FD@C>9n6B z2@)VcfB*pk1PBlyK!5-N0t5&UAV7csfte|=W@ZlCs0k1tK!5-N0t5&UAV7cs0RjXF z5FkK+009C72)r=@-evU0xa*Yw0RjXF5FkK+009C72oNYr0ly1fl+$WZ2S>n{1Rfk` zy$~QkfB*pk1PBlyK!5-N0t5&UAV7cs0RodR;AO(eHx-cp0RjXF5FkK+009C72oNAZ zfB*pk1PBlyP!IxMC@P3)k|Y5F1PBlyK%npiRu}#djDP?E0t5&UAV8p)1uR&Ld3KGB z009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+0D)H%@bzo2CZwDI0RjYO zrhw05n3?l#)C33+AV7cs0Rq!pz(ReRPmuHo5FkKcItzGL{dArkxe*{hfB*pk1PBnA zLIJ-XHifuC6Cgl<009C72oNAZfIwjhtS_u|r!ZT;f-^a|j;P6T6omi*0t5&UAV7cs z0RjXFyiC9i_scjuAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs z0RjXF5FkK+009C7idA53u@20*2oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5GZ5; zUxHo8vnO`~1PBlyK!5-N0t5(5Hv!+SKHX+SJ_HC5AV7cs0RsCi@Xkmq*p$g6J5z1l zojq;6sg6uSSrv)B``kqDd0SF_@$TNG z=cjr%cV>E)u3WLQZbjYFquaN1b|sfKHKponJDO`->g$^uQyr`7I_f*piN=olhW4gZ jx<1URzOJsLrQw*$mljk-BK!Ye~1JEqzTD?!G-NC?o7ZX2NLJ6O8t#bo?N+Ai>(i!>tzs|XoxH(&35_$ zG9Q`!yF4@&nUBnNRQEY_&QbdevKXc8pOYtvfXvWr*ZThIa2E?9^K5^7NXDgRrJ>yl znB4Z~KkviX*yzKTUdLe*hlz1$#I1+1(dp8+%Vl9=6WM;o!`R0*+cm@3)S~kXs>@(z zo!KJGkZ|-hINhz@W9$tR2hoxaR`#*|huheCf?nbY5L9vrPaz-^w9VN8>1oKvvM^mnS z_z?pyTP(aU&X1@H;DnhQ`ZsVgh#mE|u^T2<6Ib>Q#zv=kPjhdEiQQZ+fZ|`6IE>~` zc>Dy$rZ#=Nw@wt7nT;QmP|`X~-4r*~y)ZU58a=*{)ts6o+mFmAm7VwX>tkZ{wEe8X zs3%fIRN!#~Gh;&Nu^lk>K+!vR-Qnq*PbY9OvOA=>*Ia<{jdv`T3z7<%!eb)|FC%8j zc&fnUanTlOTNZ-kcHF9=xb9Qh>yaydJd6X`u4es7IO_&vbuhNjcPSni8$>^}K7v<& zZ>_RWbzmJUvL0%(Lj^ZUfz0bs*bB~woVKrOi823X%mQ9w)S;_+?EWAXT@1vJ;qsWQ zaAxBgm|WnisqDPcw;mMIy3V*IJJo$b4$DKRKrsA)9yn zbmnXrf1S|=eHa^=HV<74&R27t_Y+-zm>RwXw^1tZ-hrntlMBvMks5t z+N%v?qtjQ{iI~8|n)2(VQ#xC|!uZH^p2Si`WIjIj>-DKWLGrmi7TKySr+;wS2NK`s!MO_n1=?lb diff --git a/tests/integration/test_storage_hudi/test_table/americas/united_states/san_francisco/.hoodie_partition_metadata b/tests/integration/test_storage_hudi/test_table/americas/united_states/san_francisco/.hoodie_partition_metadata deleted file mode 100644 index a8c28b02044..00000000000 --- a/tests/integration/test_storage_hudi/test_table/americas/united_states/san_francisco/.hoodie_partition_metadata +++ /dev/null @@ -1,4 +0,0 @@ -#partition metadata -#Tue Aug 30 08:36:50 UTC 2022 -commitTime=20220830083647456 -partitionDepth=3 diff --git a/tests/integration/test_storage_hudi/test_table/americas/united_states/san_francisco/34b1b177-f0af-467b-9214-473ead268e55-0_1-73-84_20220830083647456.parquet b/tests/integration/test_storage_hudi/test_table/americas/united_states/san_francisco/34b1b177-f0af-467b-9214-473ead268e55-0_1-73-84_20220830083647456.parquet deleted file mode 100644 index 053841a0b91b8966c1f374a245825dc5ac0de548..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 438186 zcmeI*3v^s%odEDjnwGTE(%Rl(0#Ry(ShNXwH4oM`qy?7Bv*OC~@T8Mu(gxE6k`zG@ z0a+J|b`cTK!-}Bma(u8XdVmA#E)NA4N(9zLK{&GDE+`dOaVs2H-8)GpnUWNS<~5V~ zaTI3m{T~1C_rLefcPIDWIidLjRY6hk?%=rKnukC8#YOK83WK6Zw`$JIA{p2_}R{_`;-O?!iMGrVa3tfZLJ$N zY`AjchIx^Nci(kKvdyr{rwW6!gD?H&rmG8+bt2=}*>~>cN6tI)$a&Fw=N;D7(c0a5 zdbB6jbHUMfFTU#HB^C4APJ3%v+Zol#Q|9kn%hks$+QH3`*Uf8(wysv(8 z@BTXu`thUZpVo2eBPZXp{(=YJw*CGMosSLN_BY3!U%LL+XKlG5T6FKK%2VcFdhg%; z>bcW@e_L>2`G;=$(%*ih^s*guZlAaB|8D>L-+uJ5fB3-Mt{0y<{h}j|dFaf6Z?!-2 z)B%m>pIb4IjJYT{A!PW^KivJbf-&eP#y4y@YWuO}_aAfIsz(ob@PSq9j{Wo@73)^5 zD}VHmY1i8CZNKKz z-#j4}d@%8s759GPFP6V@)-A7m?AaF!Vz=G8`j|ZoFA|&06jVeW z{!?Mt`kVtw3%4#i>7M(VPJ4M>;a{Khbkh$9Ubv*V^LtH)|KQ-&4}SZBrhOj$&Z{4L z`T3@Y`q~ex7>XzJ#y@uS!S5KD`-p!`tlamxrqgdY_P)W!i-OYN@Zi_i?iko-4C^!D9&yyc_pds-{E^kwweuq9{_9UK zUo;IuXFv4Y_5El3wCU<|pS5^a6opEjT zPny<6F8#ttnPyK1zjsG+!vgvm_P8-Ib`Tc6YU`=r8w$J}*ud(S552q{d zKW5!<_P=t@OH~Uekp2GP@@JpC`1J^XwBh9c`TD0ncVT?vEB9|)e#+;6@Y{k*7M=Qs zr&e6@$20qWa^m8Jk%Nll4y3QMAq z$Y6zFOof)JmdcXYxXWwPDjrx_U$?rdvZ}7UE>YWBUR&GRTHctbY%H&hTWcWE}hInmqM7VkS` zOLu2~qP?ZBKi;3{JESk(-O|w;?{4etYwHP{&7@dbw1?HywpO)Pg<*75#yiSu>*~X3 ztE+0uYwK$g@%HMvhQ#XC<&`Z}VO-@6wJl>*N@5djqCPF0;?zO8Bvza{9;dHP9Ztj5 zsUvRM>ILn+oo6L_%d4wuN@8hm(ux$Pj%{HJsl!?N>eSIFT%9^lrLSIi_Idj}QC@Rc zxca}2-@EYfwv!G^Tb&X9o6phw-p=;as77C>^;`U*8=rdkqJnVCjTLVy`pS;v%e2)~ zutkY&euq;?>1Ry>nAfPkU#gBHU;C zTl(XzUFk1ddJ}Cuz3sumf3>9u^>)E`yGc4Y@IdM?rn9}4yIJth=Yu5KA z&g||fuVL=K9V>meRVXo`iyvQe`kMZPj|~1@%~NeMV6$BbaZwlTDrr%I<)a6yrCT-y~aoqoybieU{;LC-PvhOYk=kCLM+vs6y(EBq5(c25c z0`DFDtdEUM0K&nJ%m7B0rfwWAvXiYVlAT;0PN7qWwBZBn&}LH*eW)N@TXw_3()3wJ`s=WxwE0N-nsNBP ze{jh6!xh1ik+PSP)AnJ#)Db*wST8LKhn1Sf#75>e$zhFMnm(+@lf&Bf&g8IeUi{iF zc9FYzao9y_DikYD&5x2@j9r@E#qS2wGuXwBC8J#oe%7Rha7#hijwNCHskv0FI5oja zwm)`hdiz`7oy^RE$sydiY&0{wet->jG<5vBb6MH-OLOb^b^W`O8^VEu$FFVC(J>Bv z6c3K={OCYI*|uoe*Mrj7=-e>vWe7R$@^QAi+mhYwJSbQbDf@lcThZ`#7kOP29o)@tt^?ZD`P`rNqzzk&K(BZANusWGB$;ue*c|} zVP{zqH2RrXbU|5hbmxA{!dYxrZ0Ydfx*~p7Z%;*E+or_kc_^1~mG z#Fl)tH~fC4w_39O&4j`Tx-009C72oNAZfB*pk1PBlyK!5-N0t5&U zAV7cs0RjXF5FkK+009C72oNAZfIyxItjm*e8YTe(1PBlyFv|tj&hqhPHoWk0t^OfE zfB*pkvq`{1*=AEVjfDUK0t5&UAV7cs0RodC;2~U-fT#%q1PBlyK!5-N0t5);yMU9+ zsow%z$~E=IdLTf6009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C7 zvO>VK-?AbNg2+YzPmjvRtcdA#1su^|m&Zy11PBlyK!5-N0t5&UAV7e?lnVH6W=eTA zPJjRb0t5&UAV7csf!q*qwMlN|Nt6T#5FjwE0=_$)R&JdUAV7cs0RjXF5FkK+z!(8P zJu-&G0s;gG5FkK+009C72oNAZAb$m1nGfT1_%%!K!5-N0t5&UAV7csf!QJ8TY}k<$1EL%dkwNQF~TB1fB*pk1PBlyK!5-N z0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjYaRlsi< zATVnMoPy8V z)EXE80<%oO@%nWVy5@D(Sw&!2z-D4on6Q9ntWB6u0|W>VAV7e?ObTqA$CQ#Md z}~xF_N{9I1PBlykY@sZUp3DXZny*p5FkK+009C72oNAZ zfB=EX5%AZX$#K*Qf!q?vpTAh=R$fF)fB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+ z009C72xN(X=T2lvB62Z|OmFqHk>z4`#Y}(z0RjZ3T);)XQ%W=^c0t5&UAV7cs0Rq`1;L@UO z%7$175FkK+K$Z%aFl13;BqNKGArt}xa!0`519K-=A|*h8009C72oNAZfB*pk1PEk> zfN$foA`OBdK!5-N0t5&U$YX(Zc|3MQCqRGz0RjXF5FkK+009C72oNAZfIuz_xQQp1 zvnTe+5b!sSNl;uf32n@>CS1=p%Mxlh1PBlyKwuIDoP$jwrltrGAV7cs0RjXF5FkK+ z009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5Xc9CwfQheVYBmn{ha$CT}D7W(`dIAIp5XeFS z7Yt@$QiPN(0#mztcmzYXak-A90D&nK@D=+M z;%a&V0-gjh0X$W+R={?hwW-Paz?|7)K(dJ}Cuz3nX@Nt_cqu(G~xbya0mU3p!iwza&rwzajq zF;UrAURP7u*ic;;uWx8iq}7g}y-V%(){c(0n)>?k%8rJ{^4gBJ)#VNG`0DcdnyS{; z_SV{}`gm<5_Wm>Cy=QJo^p|h$U4B-gx39COdwEqwRdq%6@|CSyI=kAJ*ViX%D%%<= z8*6GB>Jn|MtJ`YYI^uP0HMOnviH@3@>Z+RR>bAz(gG*lBr!*4D{C|;WzBVs%Lh}cz F{vYhxb9w*( diff --git a/tests/integration/test_storage_hudi/test_table/asia/india/chennai/..hoodie_partition_metadata.crc b/tests/integration/test_storage_hudi/test_table/asia/india/chennai/..hoodie_partition_metadata.crc deleted file mode 100644 index 901b972aff0c2c1daa875aaf7ffe23d1fd77e34f..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 12 TcmYc;N@ieSU}9)DlYRvN5nclQ diff --git a/tests/integration/test_storage_hudi/test_table/asia/india/chennai/.92aa634e-d83f-4057-a385-ea3b22e5d6e1-0_2-73-85_20220830083647456.parquet.crc b/tests/integration/test_storage_hudi/test_table/asia/india/chennai/.92aa634e-d83f-4057-a385-ea3b22e5d6e1-0_2-73-85_20220830083647456.parquet.crc deleted file mode 100644 index 9fae4a530439b64300fa078deaf19eda992646cc..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 3428 zcmYc;N@ieSU}9LgovlRl$*&o!w7J$i+&5ESa<^O7g0GXplb&tertr)`@_*KkCYS!s zSp_9GrruK^z`?M=oej-IZNbM&{?$(3GN^?ooM@W50 zN|X#fHG3AANrU9gr}Lb{rw&=}yO~EOG9MdTP<+czY~u71o4t5qBDL*$VO%drZ8J$V zZ`zv^;5^-XD1e7ly)gBfQ!8y@Y(lhQO|2?SjlY1mGmMRmUc8AR9Gf`0nA`W2OVGtY z{FJ3riPDcvFSW&_EK2SYGE@rj|2x;g5>)IAQ5V}Y?@baJ(l0gR1I-$~D9L*^61 z{=O%omKb%|)Nov|slp~steB~Wv?D&XKhAtOj!%9t$vx>ln}^LksYhh|&xoZ~Tk|S39;q5B@qb@UJVd_ITCTGFeLzTABT;@Pe vx2w(Ar)TD${v2lK^_B8BVQg%)Ps42ciE@c=u*reM=EWYrk1Y3UV;L&|A4@|J diff --git a/tests/integration/test_storage_hudi/test_table/asia/india/chennai/.hoodie_partition_metadata b/tests/integration/test_storage_hudi/test_table/asia/india/chennai/.hoodie_partition_metadata deleted file mode 100644 index a8c28b02044..00000000000 --- a/tests/integration/test_storage_hudi/test_table/asia/india/chennai/.hoodie_partition_metadata +++ /dev/null @@ -1,4 +0,0 @@ -#partition metadata -#Tue Aug 30 08:36:50 UTC 2022 -commitTime=20220830083647456 -partitionDepth=3 diff --git a/tests/integration/test_storage_hudi/test_table/asia/india/chennai/92aa634e-d83f-4057-a385-ea3b22e5d6e1-0_2-73-85_20220830083647456.parquet b/tests/integration/test_storage_hudi/test_table/asia/india/chennai/92aa634e-d83f-4057-a385-ea3b22e5d6e1-0_2-73-85_20220830083647456.parquet deleted file mode 100644 index c1ba250ed09695e1f95880fc1bf3457620be17b2..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 437623 zcmeI*3wV{~eE{$i3?U#Og5UAPMnu#+UI->RNg&Qu0^%d;I#B1V)`fF&P9QTRm;^14 zm#Oukb6Q=&b+vVBr*^Gdt+VQGuCtSFEu#*nPMx$`M%iPlYqfoxb6)m+CppQ{1mqeL z^2-y#cfR-X|G&TYyS%xaa~3zATOAY!CkCGg?s)2Rw@*Gb7!wr7Vn^Ln8teI(mRY*A zs-~)@=E17^SUNeaYub6q*q?v%w@(hj@*oyW3FZdBdS>Hu2bYK46bB_i*s#BKb9(90 zr7bN>OJZeTeD{H<&#=kLpe6Y1OKZL~CTbEJyh(iQ+A~VdIHM%~PbKr(TGH+5-;H-1 z)^Y9G|2QR9JMFdC=l1mU7e&Ft;pZKjAIyax&{weGJCh!tF|oMuu#3L`+|=(Z{q9dT zr=I@3@kihF?yH}=@3xD6e%`(ReCfCE`&>)SygxYT=#!qEe$m!*-#_xoQ{&SfdSUhE z`l4-X&wt~w)9c@j9em?wFT5ZMSscs?76-4r{)3v1e}i{JrtrlqwFsSQLaq{L;-&y)u44yh%@7JbB){iDNgtu=u|Aqt5%a2M_-G zIaeR^>Rs#Zc;t~sXFR>|rZc|RxZ&SVIPT^Xv*R|bdG4IYuKU}?&zF?Uz2W@_F8RTq zC4c_2Z~WImZ$G|%!a2K{T`Z)yAh>_?#$S!w!8|^&?Zw9)-tyCF|CjyEw=+L^?}rbM zi+%FqI~MnnL_Iw6(pPRR-9Zl@|Jk3MRQ>cjfAsD>SCpN6_%Ba;Xk4s&`dO=UJroD! z!Rf(kcfI=LAp?B=Gn$VRHq1J^^55!eYD;2Q{=@TYC-1>n&kr`$?8#e{Z|`^@qMzzvjI^elzySv(Ls0hWgj{{m&CcI~Z!q(sjRl>+&o9{@sJVQTi{f z2QPc^pxF7pb<&c)9?lOQxbCI)9dz(sUBl7q{_2VE{`gN5Cq1$5%WEdbZu;>b{!K5{ zkRG{51oMNNZdv#KxPkK$UQmwTaL%IMX}Np#Th){HU|JRjufOrV8+W97P0Q3P{_@7p zU)Olm>vxTtaYU@_TeH5(D2w)wjndOzM-YEs{2wx3*`(ug%M|?d_@7 zvcw=e3!2Pps7a;jleO8(%-m#4Wo=d6oXS*kZe3+Il}y*vWa~2Z+3L!wrkct*$;!EP zO#?<;me|uS<`kq;`qizs969~H7fvfnlwLXcTlanX*{!D)7H|3Q<*(dx$d;)12WPKH z+&<^0g~byxovoK=J1c9dlVyp52ZcRMeECytFMei2)I---FIp43^_9XNN?$(l!iFz= z<;ziVtYrF`Z~pL`A1Hn*J~p=Xh3a`>@!RW`wodx})8`d*J_z+k`H$~x&Ezxg-!AC0 zbX)E3Jav6hc*G@255MP~m!r*s;(c%=mB+6NTB7r?tUSJ@D7dYtcl*sT!M6PNrm=lh zZZ7THe4{kDAz$Oo@%`Id5ALh7VPbH7zRJc){Z%$i4zA7bUq7X<`r4^|n{Q1GuE}RI ztv8bp;+-jH9<#ioBh#9l-JR{~Zt6~@+X^?EIIESPxQn8dWEp2|(l9V=F} zhSghFWRHuTQyyLDqVmH&RNj@nti7YK!qj|)o_?Dy&8{vij^~S4raHS@yIVWjn^vZ} zmlsw!G+(8qwJqD!p6X9vLYTn#baq*5dsADgI~Ei^?r85xu%zc0&vzJA8JBI(^f%~v z6jhJPid#~h*`Ah@`cnFEMq_gaVZO`au-TaIuAWB4t5&sUdQ0NPW#wh>98z&rc`&i8 zuC%zMY*5aiI9?pTr6~R%MHN?7l$H;2Ssi3oJa+uR%kLodVr8>(%h@r}dbKzh6N|;) zI662izBvfev5GC>?Iubs7N$1nOy+XCvnYOBQCRoX`q=S(uSL>M(Z{lchg zW(JA)x8p%)tm4gZy_}Ca$l;NT`JxrE@vkNo2Gs@_}!-EHkVilXl z_RrCF4#|RP+BCLeb8-1#cZ$LGg;O=ig>JB#gG|<=GlO{iM~R>%R&jH6$vf$1!v5GfB_X)ix zN&nR*=Y3K9hN7_i)c%`qqVJ9zje(#@2AjpFe(juZlefO;LPPQCM2>_Js1nRY2i(7^q-< zP*^eu-r8fLW41gvBUZ6tV*j}Em)+h`U0)QIhNH@_DiWpnrA9QWfqM%_^|fQdQQdoF zG^&l0J{+OYZR4acLVjVA=v%Es5eDupjPQ6AVM{K;rpf&ga<_-v04|JQTU4=Wa@c)- zVUsA$uXdvD2ktHG{`si;yQfD3SU;tIT0VHo$;IipF|40bvFVVZx#IjHH+$2gE6&=f z{i*fb^KvO&6MwQOEUUQx(DFq88mwS59RHyE2RT;O92=czM+Ng^6>o*nihIwq@P68R zm|PR?50UdrxkTUkEa1y4)=~*L&|xX4*cy$vkvyIMM58 zL^qlE9aG}P6{YcKg2TeKVq0Qb@8xxN>hjKx*&3m&UEg9Z zx4zsfUG}X<_o%nN_3OZ1G99bJ#d&TTwxi$MxTAe{8uzS>2evGj{q2P9T|f_P`q3HZ zn(kh{(b9Z(LiSC4w2mLx!AB1_x7^>IhTUBQTeWrco>|+mjlO{x*lK%8UUGFTap)DD zt=-wq*(*9S*|yoy)gnml!009C72oNAZfB*pk1PBlyK!5-N0t5&U7)k-Z%os{-u?P?# zK!5-N0t5&UAV7cs0RjXF5FkK+0D%!B;0>ep%~UPgH{I%$009C72oNAZfB*pk1PBly zK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfWW>M@Tai*T3G)C2oNAZfB*pk1PBly zFp2~gj-s&|DuGcX;M5*PlWnL32oNAZfB*pk1PBlyK!5-N0t5&U7>)vdP(2)HMNR|= z5FkK+009C72oNAZfB=CZ74U{bL&_}}0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N z0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5)`QvuJS->1AfCqRGz z0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF44uG|p$nCdAKZhDd+_0RjXF5FkKcL<#t%{)n0`$r2zyfB*pk1PBlyK!5-N z0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+ z009C72oNB!hXURhbq@hGCqRGz0RjXF5FkK+009C72oTr@0$wS2AE4@z009C72oNAZ zfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0tEJ( zz=Hi2SzrPL2oNAZfB*pk1PBlyK!5-N0s{pW4%}rA0RjXF5FkK+009C72oNAZfB*pk z1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs z0Rkggz>DCHxJFBW009C72oNAZfB*pk1PBlyK!5-N0tAM; zz{264p!^6BAV6S$3M|^6!Nn#(fB*pk1PB}u0`m`uQ5XaP0t5&UAV7csfgu&})RQ6Q z7K{J^0t5&UAV7cs0RjXF5FkK+009C72oNAZV3Y~?v$jz-;f70q009C72oNAZfB*pk z1PBlyK!5-N0t5&UAV7e?5DIwdwjsn8iU0uu1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+ z009C72oTtzfLD6nA&)`=1PBlyK!5-N0t5&UAh3S~ycFF2aTk>U0RjXF5FkK+009C7 z2oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N z0t5&UAV7cs0RjXF5FkK+009C72oNAZfB=C#5%9{Yd!nZ;0RjXF5FkK+009CcN?^%| z%2u)j2oNAZfB*pk1PBlyK!5-N0tAMGz_Q_xgd7MEAV7cs0RjXF5Ez*Peoa3z=S|iG z2oNAZfB*pk2fV5FkK+009C72oNAZfB=EvE3j<%rY1821a>Uo^xd(KQUU}B5FkK+009C72<%UR z`TH}t*aQd=AV6R*2>g015iIZM$h2mgR+VUO=TN0bL$&gDywR% z>MLtg>2zg7DwU~h&L-=t)AgBbbF!hJb?Va}Xq~RfRM%(fQk5;~nsnH@rM@zis%x&y zCe!J#ESt(!C1Z)RR;D^HTb1puT+un>@@!{UYe)Nx>eYJ0b={eb!WU{6@SyR*8P GtN$O{ABc|t diff --git a/utils/data-lakes-importer.py b/utils/data-lakes-importer.py index c7ab0b59232..aa03f15b1c0 100755 --- a/utils/data-lakes-importer.py +++ b/utils/data-lakes-importer.py @@ -56,8 +56,9 @@ def get_spark_for_hudi(): .config( "spark.sql.catalog.local", "org.apache.spark.sql.hudi.catalog.HoodieCatalog" ) - .config("spark.driver.memory", "20g") \ - # .config('spark.sql.extensions", "org.apache.spark.sql.hudi.HoodieSparkSessionExtension') + .config( + "spark.driver.memory", "20g" + ) # .config('spark.sql.extensions", "org.apache.spark.sql.hudi.HoodieSparkSessionExtension') .master("local") ) return builder.master("local").getOrCreate() @@ -98,6 +99,10 @@ def main(): "hoodie.datasource.write.partitionpath.field", "partitionpath" ).option( "hoodie.datasource.write.table.name", "hudi" + ).option( + "hoodie.datasource.write.recordkey.field", "ts" + ).option( + "hoodie.datasource.write.precombine.field", "ts" ).option( "hoodie.datasource.write.operation", "insert_overwrite" ).save(