From 8ac2fc7b2624007843e612baeb4d21cea6a6e69c Mon Sep 17 00:00:00 2001 From: avogar Date: Mon, 12 Sep 2022 17:21:40 +0000 Subject: [PATCH] Don't outout totals/extremes in all row formats, update docs --- .../statements/select/group-by.md | 5 ++-- .../sql-reference/statements/select/index.md | 4 ++-- src/Processors/Formats/IRowOutputFormat.cpp | 6 +++++ src/Processors/Formats/IRowOutputFormat.h | 3 +++ .../Formats/Impl/CSVRowOutputFormat.h | 4 ++++ .../Impl/JSONCompactEachRowRowOutputFormat.h | 3 +-- .../Formats/Impl/JSONCompactRowOutputFormat.h | 3 +++ .../Formats/Impl/JSONEachRowRowOutputFormat.h | 4 ---- .../Formats/Impl/JSONRowOutputFormat.h | 3 +++ .../Formats/Impl/TSKVRowOutputFormat.h | 4 ++++ .../Impl/TabSeparatedRowOutputFormat.h | 4 ++++ .../Formats/Impl/VerticalRowOutputFormat.h | 3 +++ .../Formats/Impl/XMLRowOutputFormat.h | 3 +++ ...formats_with_totals_and_extremes.reference | Bin 0 -> 2416 bytes ...21_formats_with_totals_and_extremes.sql.j2 | 22 ++++++++++++++++++ 15 files changed, 61 insertions(+), 10 deletions(-) create mode 100644 tests/queries/0_stateless/02421_formats_with_totals_and_extremes.reference create mode 100644 tests/queries/0_stateless/02421_formats_with_totals_and_extremes.sql.j2 diff --git a/docs/en/sql-reference/statements/select/group-by.md b/docs/en/sql-reference/statements/select/group-by.md index b5e194343ca..ac02e9ab5a1 100644 --- a/docs/en/sql-reference/statements/select/group-by.md +++ b/docs/en/sql-reference/statements/select/group-by.md @@ -213,9 +213,10 @@ If the `WITH TOTALS` modifier is specified, another row will be calculated. This This extra row is only produced in `JSON*`, `TabSeparated*`, and `Pretty*` formats, separately from the other rows: -- In `JSON*` formats, this row is output as a separate ‘totals’ field. -- In `TabSeparated*` formats, the row comes after the main result, preceded by an empty row (after the other data). +- In `XML` and `JSON*` formats, this row is output as a separate ‘totals’ field. +- In `TabSeparated*`, `CSV*` and `Vertical` formats, the row comes after the main result, preceded by an empty row (after the other data). - In `Pretty*` formats, the row is output as a separate table after the main result. +- In `Template` format, the row is output according to specified template. - In the other formats it is not available. :::note diff --git a/docs/en/sql-reference/statements/select/index.md b/docs/en/sql-reference/statements/select/index.md index c1692bd9b29..de190aaf982 100644 --- a/docs/en/sql-reference/statements/select/index.md +++ b/docs/en/sql-reference/statements/select/index.md @@ -135,9 +135,9 @@ In all other cases, we do not recommend using the asterisk, since it only gives In addition to results, you can also get minimum and maximum values for the results columns. To do this, set the **extremes** setting to 1. Minimums and maximums are calculated for numeric types, dates, and dates with times. For other columns, the default values are output. -An extra two rows are calculated – the minimums and maximums, respectively. These extra two rows are output in `JSON*`, `TabSeparated*`, and `Pretty*` [formats](../../../interfaces/formats.md), separate from the other rows. They are not output for other formats. +An extra two rows are calculated – the minimums and maximums, respectively. These extra two rows are output in `XML`, `JSON*`, `TabSeparated*`, `CSV*`, `Vertical`, `Template` and `Pretty*` [formats](../../../interfaces/formats.md), separate from the other rows. They are not output for other formats. -In `JSON*` formats, the extreme values are output in a separate ‘extremes’ field. In `TabSeparated*` formats, the row comes after the main result, and after ‘totals’ if present. It is preceded by an empty row (after the other data). In `Pretty*` formats, the row is output as a separate table after the main result, and after `totals` if present. +In `JSON*` and `XML` formats, the extreme values are output in a separate ‘extremes’ field. In `TabSeparated*`, `CSV*` and `Vertical` formats, the row comes after the main result, and after ‘totals’ if present. It is preceded by an empty row (after the other data). In `Pretty*` formats, the row is output as a separate table after the main result, and after `totals` if present. In `Template` format the extreme values are output according to specified template. Extreme values are calculated for rows before `LIMIT`, but after `LIMIT BY`. However, when using `LIMIT offset, size`, the rows before `offset` are included in `extremes`. In stream requests, the result may also include a small number of rows that passed through `LIMIT`. diff --git a/src/Processors/Formats/IRowOutputFormat.cpp b/src/Processors/Formats/IRowOutputFormat.cpp index f2f6b49ed3f..52ce9c1b227 100644 --- a/src/Processors/Formats/IRowOutputFormat.cpp +++ b/src/Processors/Formats/IRowOutputFormat.cpp @@ -40,6 +40,9 @@ void IRowOutputFormat::consume(DB::Chunk chunk) void IRowOutputFormat::consumeTotals(DB::Chunk chunk) { + if (!supportTotals()) + return; + auto num_rows = chunk.getNumRows(); if (num_rows != 1) throw Exception("Got " + toString(num_rows) + " in totals chunk, expected 1", ErrorCodes::LOGICAL_ERROR); @@ -53,6 +56,9 @@ void IRowOutputFormat::consumeTotals(DB::Chunk chunk) void IRowOutputFormat::consumeExtremes(DB::Chunk chunk) { + if (!supportExtremes()) + return; + auto num_rows = chunk.getNumRows(); const auto & columns = chunk.getColumns(); if (num_rows != 2) diff --git a/src/Processors/Formats/IRowOutputFormat.h b/src/Processors/Formats/IRowOutputFormat.h index 7a57753d765..da2eb192e90 100644 --- a/src/Processors/Formats/IRowOutputFormat.h +++ b/src/Processors/Formats/IRowOutputFormat.h @@ -32,6 +32,9 @@ protected: void consumeTotals(Chunk chunk) override; void consumeExtremes(Chunk chunk) override; + virtual bool supportTotals() const { return false; } + virtual bool supportExtremes() const { return false; } + /** Write a row. * Default implementation calls methods to write single values and delimiters * (except delimiter between rows (writeRowBetweenDelimiter())). diff --git a/src/Processors/Formats/Impl/CSVRowOutputFormat.h b/src/Processors/Formats/Impl/CSVRowOutputFormat.h index a36c5ff47fb..0efc00378e5 100644 --- a/src/Processors/Formats/Impl/CSVRowOutputFormat.h +++ b/src/Processors/Formats/Impl/CSVRowOutputFormat.h @@ -34,6 +34,10 @@ private: void writeField(const IColumn & column, const ISerialization & serialization, size_t row_num) override; void writeFieldDelimiter() override; void writeRowEndDelimiter() override; + + bool supportTotals() const override { return true; } + bool supportExtremes() const override { return true; } + void writeBeforeTotals() override; void writeBeforeExtremes() override; diff --git a/src/Processors/Formats/Impl/JSONCompactEachRowRowOutputFormat.h b/src/Processors/Formats/Impl/JSONCompactEachRowRowOutputFormat.h index 63e9e9f1b76..2b4c596d4b5 100644 --- a/src/Processors/Formats/Impl/JSONCompactEachRowRowOutputFormat.h +++ b/src/Processors/Formats/Impl/JSONCompactEachRowRowOutputFormat.h @@ -36,9 +36,8 @@ private: void writeRowStartDelimiter() override; void writeRowEndDelimiter() override; + bool supportTotals() const override { return true; } void consumeTotals(Chunk) override; - /// No extremes. - void consumeExtremes(Chunk) override {} void writeLine(const std::vector & values); diff --git a/src/Processors/Formats/Impl/JSONCompactRowOutputFormat.h b/src/Processors/Formats/Impl/JSONCompactRowOutputFormat.h index d17a6acf019..38123833f10 100644 --- a/src/Processors/Formats/Impl/JSONCompactRowOutputFormat.h +++ b/src/Processors/Formats/Impl/JSONCompactRowOutputFormat.h @@ -31,6 +31,9 @@ private: void writeRowStartDelimiter() override; void writeRowEndDelimiter() override; + bool supportTotals() const override { return true; } + bool supportExtremes() const override { return true; } + void writeBeforeTotals() override; void writeAfterTotals() override; diff --git a/src/Processors/Formats/Impl/JSONEachRowRowOutputFormat.h b/src/Processors/Formats/Impl/JSONEachRowRowOutputFormat.h index ac03c2991bf..37a7029d050 100644 --- a/src/Processors/Formats/Impl/JSONEachRowRowOutputFormat.h +++ b/src/Processors/Formats/Impl/JSONEachRowRowOutputFormat.h @@ -39,10 +39,6 @@ protected: void writePrefix() override; void writeSuffix() override; - /// No totals and extremes. - void consumeTotals(Chunk) override {} - void consumeExtremes(Chunk) override {} - size_t field_number = 0; private: diff --git a/src/Processors/Formats/Impl/JSONRowOutputFormat.h b/src/Processors/Formats/Impl/JSONRowOutputFormat.h index 66b8d0f682e..07c526885f6 100644 --- a/src/Processors/Formats/Impl/JSONRowOutputFormat.h +++ b/src/Processors/Formats/Impl/JSONRowOutputFormat.h @@ -56,6 +56,9 @@ protected: void writeMaxExtreme(const Columns & columns, size_t row_num) override; void writeTotals(const Columns & columns, size_t row_num) override; + bool supportTotals() const override { return true; } + bool supportExtremes() const override { return true; } + void writeBeforeTotals() override; void writeAfterTotals() override; void writeBeforeExtremes() override; diff --git a/src/Processors/Formats/Impl/TSKVRowOutputFormat.h b/src/Processors/Formats/Impl/TSKVRowOutputFormat.h index e9f9071f906..9bc44bbb4d7 100644 --- a/src/Processors/Formats/Impl/TSKVRowOutputFormat.h +++ b/src/Processors/Formats/Impl/TSKVRowOutputFormat.h @@ -22,6 +22,10 @@ private: void writeField(const IColumn & column, const ISerialization & serialization, size_t row_num) override; void writeRowEndDelimiter() override; + /// Disable totals and extremes, because they are enabled in TSV. + bool supportTotals() const override { return false; } + bool supportExtremes() const override { return false; } + NamesAndTypes fields; size_t field_number = 0; }; diff --git a/src/Processors/Formats/Impl/TabSeparatedRowOutputFormat.h b/src/Processors/Formats/Impl/TabSeparatedRowOutputFormat.h index 8781b7be0b1..533e30a3ee1 100644 --- a/src/Processors/Formats/Impl/TabSeparatedRowOutputFormat.h +++ b/src/Processors/Formats/Impl/TabSeparatedRowOutputFormat.h @@ -37,6 +37,10 @@ protected: void writeField(const IColumn & column, const ISerialization & serialization, size_t row_num) override; void writeFieldDelimiter() override final; void writeRowEndDelimiter() override; + + bool supportTotals() const override { return true; } + bool supportExtremes() const override { return true; } + void writeBeforeTotals() override final; void writeBeforeExtremes() override final; diff --git a/src/Processors/Formats/Impl/VerticalRowOutputFormat.h b/src/Processors/Formats/Impl/VerticalRowOutputFormat.h index 796c60d1cb1..c3dbafd8b9b 100644 --- a/src/Processors/Formats/Impl/VerticalRowOutputFormat.h +++ b/src/Processors/Formats/Impl/VerticalRowOutputFormat.h @@ -32,6 +32,9 @@ private: void writeMaxExtreme(const Columns & columns, size_t row_num) override; void writeTotals(const Columns & columns, size_t row_num) override; + bool supportTotals() const override { return true; } + bool supportExtremes() const override { return true; } + void writeBeforeTotals() override; void writeBeforeExtremes() override; diff --git a/src/Processors/Formats/Impl/XMLRowOutputFormat.h b/src/Processors/Formats/Impl/XMLRowOutputFormat.h index abec25efbb9..4d2f34b0bfe 100644 --- a/src/Processors/Formats/Impl/XMLRowOutputFormat.h +++ b/src/Processors/Formats/Impl/XMLRowOutputFormat.h @@ -32,6 +32,9 @@ private: void writeMaxExtreme(const Columns & columns, size_t row_num) override; void writeTotals(const Columns & columns, size_t row_num) override; + bool supportTotals() const override { return true; } + bool supportExtremes() const override { return true; } + void writeBeforeTotals() override; void writeAfterTotals() override; void writeBeforeExtremes() override; diff --git a/tests/queries/0_stateless/02421_formats_with_totals_and_extremes.reference b/tests/queries/0_stateless/02421_formats_with_totals_and_extremes.reference new file mode 100644 index 0000000000000000000000000000000000000000..ee8e589089c1dc532aa0f799c214802e2e1085da GIT binary patch literal 2416 zcmd5;%}(1u5bo7;=7PAadkzJ81agS(zGYq6mzbPBABANllH!9v!g6ZWxSW znMI;->33AfBJKZ z!A=-%<}aBl+P7@tA3+kz4+^24{H(XDIj*rxg_lIJnQ6V9D>&xbJiFCD$8A& zQcS+4;mFnx(@Lb7BVC!MxH_|wgou%WJkW$gdq`#~`B;fiGTCAosD^$I@lgpy-o+4zFi+wF!Q;TZ>womPFn zO&UAxT_V`A%gO3dX8lo-bY5;B)LUfLGK*{Yw2LnQI?NsN2$W2Upxt`Y$(~77Jd!dr IRr!(i8>Aj`2><{9 literal 0 HcmV?d00001 diff --git a/tests/queries/0_stateless/02421_formats_with_totals_and_extremes.sql.j2 b/tests/queries/0_stateless/02421_formats_with_totals_and_extremes.sql.j2 new file mode 100644 index 00000000000..32738766199 --- /dev/null +++ b/tests/queries/0_stateless/02421_formats_with_totals_and_extremes.sql.j2 @@ -0,0 +1,22 @@ +-- Tags: no-fasttest + +set output_format_write_statistics=0; + +{% for format in ['CSV', 'TSV', 'XML', 'Vertical', 'Pretty', 'JSON', 'JSONCompact'] -%} + +select '{{ format }}'; +select sum(number) from numbers(10) group by number % 2 with totals format {{ format }} settings extremes=1; +select ''; + +{% endfor -%} + +select 'Formats without totals and extremes:'; + +{% for format in ['CustomSeparated', 'JSONEachRow', 'JSONCompactEachRow', 'RowBinary', 'MsgPack', 'Markdown', 'SQLInsert', 'Values', 'TSKV'] -%} + +select '{{ format }}'; +select sum(number) from numbers(10) group by number % 2 with totals format {{ format }} settings extremes=1; +select ''; + +{% endfor -%} +