Merge pull request #70106 from jrdi/system-projections

Add projections size to system.projections

docs/en/operations/system-tables/projections.md

@@ -11,6 +11,9 @@ Columns:
 - `table` ([String](../../sql-reference/data-types/string.md)) — Table name.
 - `name` ([String](../../sql-reference/data-types/string.md)) — Projection name.
 - `type` ([Enum](../../sql-reference/data-types/enum.md)) — Projection type ('Normal' = 0, 'Aggregate' = 1).
+- `total_rows`, ([UInt64](../../sql-reference/data-types/int-uint.md)) — Total number of rows.
+- `data_compressed_bytes` ([UInt64](../../sql-reference/data-types/int-uint.md)) — The size of compressed data, in bytes.
+- `data_uncompressed_bytes` ([UInt64](../../sql-reference/data-types/int-uint.md)) — The size of decompressed data, in bytes.
 - `sorting_key` ([Array(String)](../../sql-reference/data-types/array.md)) — Projection sorting key.
 - `query` ([String](../../sql-reference/data-types/string.md)) — Projection query.
 
@@ -23,19 +26,25 @@ SELECT * FROM system.projections LIMIT 2 FORMAT Vertical;
 ```text
 Row 1:
 ──────
-database:    default
-table:       landing
-name:        improved_sorting_key
-type:        Normal
-sorting_key: ['user_id','date']
-query:       SELECT * ORDER BY user_id, date
+database:                default
+table:                   landing
+name:                    improved_sorting_key
+type:                    Normal
+total_rows:              1000
+data_compressed_bytes:   8081
+data_uncompressed_bytes: 12890
+sorting_key:             ['user_id','date']
+query:                   SELECT * ORDER BY user_id, date
 
 Row 2:
 ──────
-database:    default
-table:       landing
-name:        agg_no_key
-type:        Aggregate
-sorting_key: []
-query:       SELECT count()
+database:                default
+table:                   landing
+name:                    agg
+type:                    Aggregate
+total_rows:              2
+data_compressed_bytes:   82
+data_uncompressed_bytes: 32
+sorting_key:             ['user_id']
+query:                   SELECT user_id, max(date) AS max_date GROUP BY user_id
 ```

src/Storages/System/StorageSystemProjections.cpp

@@ -6,6 +6,7 @@
 #include <DataTypes/DataTypeString.h>
 #include <DataTypes/DataTypesNumber.h>
 #include <Databases/IDatabase.h>
+#include <Storages/MergeTree/MergeTreeData.h>
 #include <Storages/VirtualColumnUtils.h>
 #include <Storages/System/getQueriedColumnsMaskAndHeader.h>
 #include <Interpreters/Context.h>
@@ -41,6 +42,9 @@ StorageSystemProjections::StorageSystemProjections(const StorageID & table_id_)
             { "table", std::make_shared<DataTypeString>(), "Table name."},
             { "name", std::make_shared<DataTypeString>(), "Projection name."},
             { "type", std::move(projection_type_datatype), "Projection type."},
+            { "total_rows", std::make_shared<DataTypeUInt64>(), "Total number of rows."},
+            { "data_compressed_bytes", std::make_shared<DataTypeUInt64>(), "The size of compressed data, in bytes."},
+            { "data_uncompressed_bytes", std::make_shared<DataTypeUInt64>(), "The size of decompressed data, in bytes."},
             { "sorting_key", std::make_shared<DataTypeArray>(std::make_shared<DataTypeString>()), "Projection sorting key."},
             { "query", std::make_shared<DataTypeString>(), "Projection query."},
         }));
@@ -108,13 +112,15 @@ protected:
                     continue;
 
                 const auto table = tables_it->table();
-                if (!table)
+                if (!table || !table->isMergeTree())
                     continue;
                 StorageMetadataPtr metadata_snapshot = table->getInMemoryMetadataPtr();
                 if (!metadata_snapshot)
                     continue;
                 const auto & projections = metadata_snapshot->getProjections();
 
+                auto parts = static_cast<const MergeTreeData &>(*table).getDataPartsVectorForInternalUsage({MergeTreeData::DataPartState::Active}, nullptr);
+
                 for (const auto & projection : projections)
                 {
                     ++rows_count;
@@ -134,6 +140,31 @@ protected:
                     // 'type' column
                     if (column_mask[src_index++])
                         res_columns[res_index++]->insert(projection.type);
+
+                    size_t total_rows = 0;
+                    size_t data_compressed = 0;
+                    size_t data_uncompressed = 0;
+                    for (const auto & part : parts)
+                    {
+                        auto projection_parts = part->getProjectionParts();
+                        auto projection_part = projection_parts[projection.name];
+                        auto column_size = projection_part->getTotalColumnsSize();
+
+                        total_rows += projection_part->rows_count;
+                        data_compressed += column_size.data_compressed;
+                        data_uncompressed += column_size.data_uncompressed;
+                    }
+
+                    // 'total_rows' column
+                    if (column_mask[src_index++])
+                        res_columns[res_index++]->insert(total_rows);
+                    // 'data_compressed_bytes' column
+                    if (column_mask[src_index++])
+                        res_columns[res_index++]->insert(data_compressed);
+                    // 'data_uncompressed_bytes' column
+                    if (column_mask[src_index++])
+                        res_columns[res_index++]->insert(data_uncompressed);
+
                     // 'sorting_key' column
                     if (column_mask[src_index++])
                     {

tests/queries/0_stateless/03230_system_projections.reference

@@ -1,6 +1,6 @@
-default	projections	improved_sorting_key	Normal	['d1','key']	SELECT * ORDER BY d1, key
-default	projections_2	agg	Aggregate	['name']	SELECT name, max(frequency) AS max_frequency GROUP BY name
-default	projections_2	agg_no_key	Aggregate	[]	SELECT max(frequency) AS max_frequency
+default	projections	improved_sorting_key	Normal	1000	1	1	['d1','key']	SELECT * ORDER BY d1, key
+default	projections_2	agg	Aggregate	2	1	1	['name']	SELECT name, max(frequency) AS max_frequency GROUP BY name
+default	projections_2	agg_no_key	Aggregate	1	1	1	[]	SELECT max(frequency) AS max_frequency
 1
 2
 improved_sorting_key

tests/queries/0_stateless/03230_system_projections.sql

@@ -28,7 +28,20 @@ CREATE TABLE projections_2
 Engine=MergeTree()
 ORDER BY name;
 
-SELECT * FROM system.projections WHERE database = currentDatabase();
+INSERT INTO projections SELECT 'name_' || number AS key, number AS d1 FROM numbers(1000);
+INSERT INTO projections_2 SELECT 'name_' || number % 2 AS name, number AS frequency FROM numbers(1000);
+
+SELECT
+    database,
+    table,
+    name,
+    type,
+    total_rows,
+    data_compressed_bytes > 0,
+    data_uncompressed_bytes > 0,
+    sorting_key,
+    query
+FROM system.projections WHERE database = currentDatabase();
 
 SELECT count(*) FROM system.projections WHERE table = 'projections' AND database = currentDatabase();
 SELECT count(*) FROM system.projections WHERE table = 'projections_2' AND database = currentDatabase();
@@ -36,4 +49,4 @@ SELECT count(*) FROM system.projections WHERE table = 'projections_2' AND databa
 SELECT name FROM system.projections WHERE type = 'Normal' AND database = currentDatabase();
 
 DROP TABLE projections;
-DROP TABLE projections_2;
+DROP TABLE projections_2;