diff --git a/src/Storages/IStorage.h b/src/Storages/IStorage.h index 622b2203877..f842e12ae88 100644 --- a/src/Storages/IStorage.h +++ b/src/Storages/IStorage.h @@ -727,6 +727,15 @@ public: /// Does not take underlying Storage (if any) into account virtual std::optional totalBytesUncompressed(const Settings &) const { return {}; } + /// If it is possible to quickly determine exact number of bytes for the table on storage: + /// - disk (compressed) + /// + /// Used for: + /// - For total_bytes_with_inactive column in system.tables + // + /// Does not takes underlying Storage (if any) into account + virtual std::optional totalBytesWithInactive(const Settings &) const { return {}; } + /// Number of rows INSERTed since server start. /// /// Does not take the underlying Storage (if any) into account. diff --git a/src/Storages/StorageMergeTree.cpp b/src/Storages/StorageMergeTree.cpp index a99be13ae24..baa2fd9256e 100644 --- a/src/Storages/StorageMergeTree.cpp +++ b/src/Storages/StorageMergeTree.cpp @@ -319,6 +319,16 @@ std::optional StorageMergeTree::totalBytesUncompressed(const Settings &) return res; } +std::optional StorageMergeTree::totalBytesWithInactive(const Settings &) const +{ + UInt64 res = 0; + auto outdated_parts = getDataPartsVectorForInternalUsage({DataPartState::Outdated}); + for (const auto & part : outdated_parts) + res += part->getBytesOnDisk(); + return res; +} + + SinkToStoragePtr StorageMergeTree::write(const ASTPtr & /*query*/, const StorageMetadataPtr & metadata_snapshot, ContextPtr local_context, bool /*async_insert*/) { diff --git a/src/Storages/StorageMergeTree.h b/src/Storages/StorageMergeTree.h index 7bc070b12b4..f61d5cb1e9a 100644 --- a/src/Storages/StorageMergeTree.h +++ b/src/Storages/StorageMergeTree.h @@ -68,6 +68,7 @@ public: std::optional totalRowsByPartitionPredicate(const ActionsDAG & filter_actions_dag, ContextPtr) const override; std::optional totalBytes(const Settings &) const override; std::optional totalBytesUncompressed(const Settings &) const override; + std::optional totalBytesWithInactive(const Settings &) const override; SinkToStoragePtr write(const ASTPtr & query, const StorageMetadataPtr & /*metadata_snapshot*/, ContextPtr context, bool async_insert) override; diff --git a/src/Storages/StorageReplicatedMergeTree.cpp b/src/Storages/StorageReplicatedMergeTree.cpp index 0941158289d..00e16d7ad29 100644 --- a/src/Storages/StorageReplicatedMergeTree.cpp +++ b/src/Storages/StorageReplicatedMergeTree.cpp @@ -5757,6 +5757,15 @@ std::optional StorageReplicatedMergeTree::totalBytesUncompressed(const S return res; } +std::optional StorageReplicatedMergeTree::totalBytesWithInactive(const Settings &) const +{ + UInt64 res = 0; + auto outdated_parts = getDataPartsStateRange(DataPartState::Outdated); + for (const auto & part : outdated_parts) + res += part->getBytesOnDisk(); + return res; +} + void StorageReplicatedMergeTree::assertNotReadonly() const { if (is_readonly) diff --git a/src/Storages/StorageReplicatedMergeTree.h b/src/Storages/StorageReplicatedMergeTree.h index 814ba1e1a4e..22320611635 100644 --- a/src/Storages/StorageReplicatedMergeTree.h +++ b/src/Storages/StorageReplicatedMergeTree.h @@ -162,6 +162,7 @@ public: std::optional totalRowsByPartitionPredicate(const ActionsDAG & filter_actions_dag, ContextPtr context) const override; std::optional totalBytes(const Settings & settings) const override; std::optional totalBytesUncompressed(const Settings & settings) const override; + std::optional totalBytesWithInactive(const Settings & settings) const override; SinkToStoragePtr write(const ASTPtr & query, const StorageMetadataPtr & /*metadata_snapshot*/, ContextPtr context, bool async_insert) override; diff --git a/src/Storages/System/StorageSystemTables.cpp b/src/Storages/System/StorageSystemTables.cpp index 9d5c68c261f..c414891ddb7 100644 --- a/src/Storages/System/StorageSystemTables.cpp +++ b/src/Storages/System/StorageSystemTables.cpp @@ -179,6 +179,10 @@ StorageSystemTables::StorageSystemTables(const StorageID & table_id_) "Total number of uncompressed bytes, if it's possible to quickly determine the exact number " "of bytes from the part checksums for the table on storage, otherwise NULL (does not take underlying storage (if any) into account)." }, + {"total_bytes_with_inactive", std::make_shared(std::make_shared()), + "Total number of bytes with inactive parts, if it is possible to quickly determine exact number " + "of bytes for the table on storage, otherwise NULL (does not includes any underlying storage). " + }, {"parts", std::make_shared(std::make_shared()), "The total number of parts in this table."}, {"active_parts", std::make_shared(std::make_shared()), "The number of active parts in this table."}, {"total_marks", std::make_shared(std::make_shared()), "The total number of marks in all parts in this table."}, @@ -597,6 +601,15 @@ protected: res_columns[res_index++]->insertDefault(); } + if (columns_mask[src_index++]) + { + auto total_bytes_with_inactive = table->totalBytesWithInactive(settings); + if (total_bytes_with_inactive) + res_columns[res_index++]->insert(*total_bytes_with_inactive); + else + res_columns[res_index++]->insertDefault(); + } + auto table_merge_tree = std::dynamic_pointer_cast(table); if (columns_mask[src_index++]) { diff --git a/tests/integration/test_analyzer_compatibility/test.py b/tests/integration/test_analyzer_compatibility/test.py index 42bd140280b..78f3428e02e 100644 --- a/tests/integration/test_analyzer_compatibility/test.py +++ b/tests/integration/test_analyzer_compatibility/test.py @@ -43,7 +43,7 @@ def test_two_new_versions(start_cluster): query_id = str(uuid.uuid4()) current.query( - "SELECT * FROM clusterAllReplicas('test_cluster_mixed', system.tables);", + "SELECT name FROM clusterAllReplicas('test_cluster_mixed', system.tables);", query_id=query_id, ) @@ -73,7 +73,7 @@ WHERE initial_query_id = '{query_id}';""" query_id = str(uuid.uuid4()) backward.query( - "SELECT * FROM clusterAllReplicas('test_cluster_mixed', system.tables)", + "SELECT name FROM clusterAllReplicas('test_cluster_mixed', system.tables)", query_id=query_id, ) @@ -108,7 +108,7 @@ WHERE initial_query_id = '{query_id}';""" # to the remote server. query_id = str(uuid.uuid4()) current.query( - "SELECT * FROM clusterAllReplicas('test_cluster_mixed', system.tables) SETTINGS enable_analyzer = 1;", + "SELECT name FROM clusterAllReplicas('test_cluster_mixed', system.tables) SETTINGS enable_analyzer = 1;", query_id=query_id, ) diff --git a/tests/queries/0_stateless/00753_system_columns_and_system_tables_long.reference b/tests/queries/0_stateless/00753_system_columns_and_system_tables_long.reference index dd5860ae491..26946cd55b4 100644 --- a/tests/queries/0_stateless/00753_system_columns_and_system_tables_long.reference +++ b/tests/queries/0_stateless/00753_system_columns_and_system_tables_long.reference @@ -1,12 +1,13 @@ -┌─name────────────────┬─partition_key─┬─sorting_key─┬─primary_key─┬─sampling_key─┬─storage_policy─┬─total_rows─┐ -│ check_system_tables │ name2 │ name1 │ name1 │ name1 │ default │ 0 │ -└─────────────────────┴───────────────┴─────────────┴─────────────┴──────────────┴────────────────┴────────────┘ +┌─name────────────────┬─partition_key─┬─sorting_key─┬─primary_key─┬─sampling_key─┬─storage_policy─┬─total_rows─┬─total_bytes_with_inactive─┐ +│ check_system_tables │ name2 │ name1 │ name1 │ name1 │ default │ 0 │ 0 │ +└─────────────────────┴───────────────┴─────────────┴─────────────┴──────────────┴────────────────┴────────────┴───────────────────────────┘ ┌─name──┬─is_in_partition_key─┬─is_in_sorting_key─┬─is_in_primary_key─┬─is_in_sampling_key─┐ │ name1 │ 0 │ 1 │ 1 │ 1 │ │ name2 │ 1 │ 0 │ 0 │ 0 │ │ name3 │ 0 │ 0 │ 0 │ 0 │ └───────┴─────────────────────┴───────────────────┴───────────────────┴────────────────────┘ -3 231 1 +1 1 3 231 1 0 +3 1 6 234 2 462 ┌─name────────────────┬─partition_key─┬─sorting_key───┬─primary_key─┬─sampling_key─┐ │ check_system_tables │ date │ date, version │ date │ │ └─────────────────────┴───────────────┴───────────────┴─────────────┴──────────────┘ diff --git a/tests/queries/0_stateless/00753_system_columns_and_system_tables_long.sql b/tests/queries/0_stateless/00753_system_columns_and_system_tables_long.sql index 009fc0bbb9f..4c6a7b63a31 100644 --- a/tests/queries/0_stateless/00753_system_columns_and_system_tables_long.sql +++ b/tests/queries/0_stateless/00753_system_columns_and_system_tables_long.sql @@ -15,7 +15,7 @@ CREATE TABLE check_system_tables SAMPLE BY name1 SETTINGS min_bytes_for_wide_part = 0, compress_marks = false, compress_primary_key = false, ratio_of_defaults_for_sparse_serialization = 1; -SELECT name, partition_key, sorting_key, primary_key, sampling_key, storage_policy, total_rows +SELECT name, partition_key, sorting_key, primary_key, sampling_key, storage_policy, total_rows, total_bytes_with_inactive FROM system.tables WHERE name = 'check_system_tables' AND database = currentDatabase() FORMAT PrettyCompactNoEscapes; @@ -24,7 +24,12 @@ FROM system.columns WHERE table = 'check_system_tables' AND database = currentDa FORMAT PrettyCompactNoEscapes; INSERT INTO check_system_tables VALUES (1, 1, 1); -SELECT total_bytes_uncompressed, total_bytes, total_rows FROM system.tables WHERE name = 'check_system_tables' AND database = currentDatabase(); +SELECT parts, active_parts, total_bytes_uncompressed, total_bytes, total_rows, total_bytes_with_inactive FROM system.tables WHERE name = 'check_system_tables' AND database = currentDatabase(); + +INSERT INTO check_system_tables VALUES (2, 1, 3); +OPTIMIZE TABLE check_system_tables; +SELECT parts, active_parts, total_bytes_uncompressed, total_bytes, total_rows, total_bytes_with_inactive FROM system.tables WHERE name = 'check_system_tables' AND database = currentDatabase(); + DROP TABLE IF EXISTS check_system_tables; diff --git a/tests/queries/0_stateless/02117_show_create_table_system.reference b/tests/queries/0_stateless/02117_show_create_table_system.reference index ef5a2c6665f..6880deed59e 100644 --- a/tests/queries/0_stateless/02117_show_create_table_system.reference +++ b/tests/queries/0_stateless/02117_show_create_table_system.reference @@ -1115,6 +1115,7 @@ CREATE TABLE system.tables `total_rows` Nullable(UInt64), `total_bytes` Nullable(UInt64), `total_bytes_uncompressed` Nullable(UInt64), + `total_bytes_with_inactive` Nullable(UInt64), `parts` Nullable(UInt64), `active_parts` Nullable(UInt64), `total_marks` Nullable(UInt64),