Merge pull request #41236 from Avogar/fix-totals-extremes

Don't output totals/extremes in all row formats, update docs

docs/en/sql-reference/statements/select/group-by.md

@@ -213,9 +213,10 @@ If the `WITH TOTALS` modifier is specified, another row will be calculated. This
 
 This extra row is only produced in `JSON*`, `TabSeparated*`, and `Pretty*` formats, separately from the other rows:
 
--   In `JSON*` formats, this row is output as a separate ‘totals’ field.
--   In `TabSeparated*` formats, the row comes after the main result, preceded by an empty row (after the other data).
+-   In `XML` and `JSON*` formats, this row is output as a separate ‘totals’ field.
+-   In `TabSeparated*`, `CSV*` and `Vertical` formats, the row comes after the main result, preceded by an empty row (after the other data).
 -   In `Pretty*` formats, the row is output as a separate table after the main result.
+-   In `Template` format, the row is output according to specified template.
 -   In the other formats it is not available.
 
 :::note    

docs/en/sql-reference/statements/select/index.md

@@ -135,9 +135,9 @@ In all other cases, we do not recommend using the asterisk, since it only gives
 
 In addition to results, you can also get minimum and maximum values for the results columns. To do this, set the **extremes** setting to 1. Minimums and maximums are calculated for numeric types, dates, and dates with times. For other columns, the default values are output.
 
-An extra two rows are calculated – the minimums and maximums, respectively. These extra two rows are output in `JSON*`, `TabSeparated*`, and `Pretty*` [formats](../../../interfaces/formats.md), separate from the other rows. They are not output for other formats.
+An extra two rows are calculated – the minimums and maximums, respectively. These extra two rows are output in `XML`, `JSON*`, `TabSeparated*`, `CSV*`, `Vertical`, `Template` and `Pretty*` [formats](../../../interfaces/formats.md), separate from the other rows. They are not output for other formats.
 
-In `JSON*` formats, the extreme values are output in a separate ‘extremes’ field. In `TabSeparated*` formats, the row comes after the main result, and after ‘totals’ if present. It is preceded by an empty row (after the other data). In `Pretty*` formats, the row is output as a separate table after the main result, and after `totals` if present.
+In `JSON*` and `XML` formats, the extreme values are output in a separate ‘extremes’ field. In `TabSeparated*`, `CSV*` and `Vertical` formats, the row comes after the main result, and after ‘totals’ if present. It is preceded by an empty row (after the other data). In `Pretty*` formats, the row is output as a separate table after the main result, and after `totals` if present. In `Template` format the extreme values are output according to specified template.
 
 Extreme values are calculated for rows before `LIMIT`, but after `LIMIT BY`. However, when using `LIMIT offset, size`, the rows before `offset` are included in `extremes`. In stream requests, the result may also include a small number of rows that passed through `LIMIT`.
 

src/Processors/Formats/IRowOutputFormat.cpp

@@ -40,6 +40,9 @@ void IRowOutputFormat::consume(DB::Chunk chunk)
 
 void IRowOutputFormat::consumeTotals(DB::Chunk chunk)
 {
+    if (!supportTotals())
+        return;
+
     auto num_rows = chunk.getNumRows();
     if (num_rows != 1)
         throw Exception("Got " + toString(num_rows) + " in totals chunk, expected 1", ErrorCodes::LOGICAL_ERROR);
@@ -53,6 +56,9 @@ void IRowOutputFormat::consumeTotals(DB::Chunk chunk)
 
 void IRowOutputFormat::consumeExtremes(DB::Chunk chunk)
 {
+    if (!supportExtremes())
+        return;
+
     auto num_rows = chunk.getNumRows();
     const auto & columns = chunk.getColumns();
     if (num_rows != 2)

src/Processors/Formats/IRowOutputFormat.h

@@ -32,6 +32,9 @@ protected:
     void consumeTotals(Chunk chunk) override;
     void consumeExtremes(Chunk chunk) override;
 
+    virtual bool supportTotals() const { return false; }
+    virtual bool supportExtremes() const { return false; }
+
     /** Write a row.
       * Default implementation calls methods to write single values and delimiters
       * (except delimiter between rows (writeRowBetweenDelimiter())).

src/Processors/Formats/Impl/CSVRowOutputFormat.h

@@ -34,6 +34,10 @@ private:
     void writeField(const IColumn & column, const ISerialization & serialization, size_t row_num) override;
     void writeFieldDelimiter() override;
     void writeRowEndDelimiter() override;
+
+    bool supportTotals() const override { return true; }
+    bool supportExtremes() const override { return true; }
+
     void writeBeforeTotals() override;
     void writeBeforeExtremes() override;
 

src/Processors/Formats/Impl/JSONCompactEachRowRowOutputFormat.h

@@ -36,9 +36,8 @@ private:
     void writeRowStartDelimiter() override;
     void writeRowEndDelimiter() override;
 
+    bool supportTotals() const override { return true; }
     void consumeTotals(Chunk) override;
-    /// No extremes.
-    void consumeExtremes(Chunk) override {}
 
     void writeLine(const std::vector<String> & values);
 

src/Processors/Formats/Impl/JSONCompactRowOutputFormat.h

@@ -31,6 +31,9 @@ private:
     void writeRowStartDelimiter() override;
     void writeRowEndDelimiter() override;
 
+    bool supportTotals() const override { return true; }
+    bool supportExtremes() const override { return true; }
+
     void writeBeforeTotals() override;
     void writeAfterTotals() override;
 

src/Processors/Formats/Impl/JSONEachRowRowOutputFormat.h

@@ -39,10 +39,6 @@ protected:
     void writePrefix() override;
     void writeSuffix() override;
 
-    /// No totals and extremes.
-    void consumeTotals(Chunk) override {}
-    void consumeExtremes(Chunk) override {}
-
     size_t field_number = 0;
 
 private:

src/Processors/Formats/Impl/JSONRowOutputFormat.h

@@ -56,6 +56,9 @@ protected:
     void writeMaxExtreme(const Columns & columns, size_t row_num) override;
     void writeTotals(const Columns & columns, size_t row_num) override;
 
+    bool supportTotals() const override { return true; }
+    bool supportExtremes() const override { return true; }
+
     void writeBeforeTotals() override;
     void writeAfterTotals() override;
     void writeBeforeExtremes() override;

src/Processors/Formats/Impl/TSKVRowOutputFormat.h

@@ -22,6 +22,10 @@ private:
     void writeField(const IColumn & column, const ISerialization & serialization, size_t row_num) override;
     void writeRowEndDelimiter() override;
 
+    /// Disable totals and extremes, because they are enabled in TSV.
+    bool supportTotals() const override { return false; }
+    bool supportExtremes() const override { return false; }
+
     NamesAndTypes fields;
     size_t field_number = 0;
 };

src/Processors/Formats/Impl/TabSeparatedRowOutputFormat.h

@@ -37,6 +37,10 @@ protected:
     void writeField(const IColumn & column, const ISerialization & serialization, size_t row_num) override;
     void writeFieldDelimiter() override final;
     void writeRowEndDelimiter() override;
+
+    bool supportTotals() const override { return true; }
+    bool supportExtremes() const override { return true; }
+
     void writeBeforeTotals() override final;
     void writeBeforeExtremes() override final;
 

src/Processors/Formats/Impl/VerticalRowOutputFormat.h

@@ -32,6 +32,9 @@ private:
     void writeMaxExtreme(const Columns & columns, size_t row_num) override;
     void writeTotals(const Columns & columns, size_t row_num) override;
 
+    bool supportTotals() const override { return true; }
+    bool supportExtremes() const override { return true; }
+
     void writeBeforeTotals() override;
     void writeBeforeExtremes() override;
 

src/Processors/Formats/Impl/XMLRowOutputFormat.h

@@ -32,6 +32,9 @@ private:
     void writeMaxExtreme(const Columns & columns, size_t row_num) override;
     void writeTotals(const Columns & columns, size_t row_num) override;
 
+    bool supportTotals() const override { return true; }
+    bool supportExtremes() const override { return true; }
+
     void writeBeforeTotals() override;
     void writeAfterTotals() override;
     void writeBeforeExtremes() override;

tests/queries/0_stateless/02421_formats_with_totals_and_extremes.sql.j2

@@ -0,0 +1,22 @@
+-- Tags: no-fasttest
+
+set output_format_write_statistics=0;
+
+{% for format in ['CSV', 'TSV', 'XML', 'Vertical', 'Pretty', 'JSON', 'JSONCompact'] -%}
+
+select '{{ format }}';
+select sum(number) from numbers(10) group by number % 2 with totals format {{ format }} settings extremes=1;
+select '';
+
+{% endfor -%}
+
+select 'Formats without totals and extremes:';
+
+{% for format in ['CustomSeparated', 'JSONEachRow', 'JSONCompactEachRow', 'RowBinary', 'MsgPack', 'Markdown', 'SQLInsert', 'Values', 'TSKV'] -%}
+
+select '{{ format }}';
+select sum(number) from numbers(10) group by number % 2 with totals format {{ format }} settings extremes=1;
+select '';
+
+{% endfor -%}
+