From f9b70ea77a3b3059d9c784e2fe4b90000e75d3d4 Mon Sep 17 00:00:00 2001 From: Alexey Korepanov Date: Sat, 22 Jun 2024 09:30:16 +0200 Subject: [PATCH 01/27] Add JSONCompactWithProgressRowOutputFormat --- src/Formats/registerFormats.cpp | 2 + ...JSONCompactWithProgressRowOutputFormat.cpp | 125 ++++++++++++++++++ .../JSONCompactWithProgressRowOutputFormat.h | 53 ++++++++ 3 files changed, 180 insertions(+) create mode 100644 src/Processors/Formats/Impl/JSONCompactWithProgressRowOutputFormat.cpp create mode 100644 src/Processors/Formats/Impl/JSONCompactWithProgressRowOutputFormat.h diff --git a/src/Formats/registerFormats.cpp b/src/Formats/registerFormats.cpp index 57ca1bb49c8..770b747fafd 100644 --- a/src/Formats/registerFormats.cpp +++ b/src/Formats/registerFormats.cpp @@ -95,6 +95,7 @@ void registerOutputFormatMarkdown(FormatFactory & factory); void registerOutputFormatPostgreSQLWire(FormatFactory & factory); void registerOutputFormatPrometheus(FormatFactory & factory); void registerOutputFormatSQLInsert(FormatFactory & factory); +void registerOutputFormatJSONCompactWithProgress(FormatFactory & factory); /// Input only formats. @@ -242,6 +243,7 @@ void registerFormats() registerOutputFormatCapnProto(factory); registerOutputFormatPrometheus(factory); registerOutputFormatSQLInsert(factory); + registerOutputFormatJSONCompactWithProgress(factory); registerInputFormatRegexp(factory); registerInputFormatJSONAsString(factory); diff --git a/src/Processors/Formats/Impl/JSONCompactWithProgressRowOutputFormat.cpp b/src/Processors/Formats/Impl/JSONCompactWithProgressRowOutputFormat.cpp new file mode 100644 index 00000000000..78cf5b9a003 --- /dev/null +++ b/src/Processors/Formats/Impl/JSONCompactWithProgressRowOutputFormat.cpp @@ -0,0 +1,125 @@ +#include +#include +#include + +#include + +#include + + + +namespace DB +{ + +JSONCompactWithProgressRowOutputFormat::JSONCompactWithProgressRowOutputFormat( + WriteBuffer & out_, + const Block & header, + const FormatSettings & settings_, + bool yield_strings_) + : JSONRowOutputFormat(out_, header, settings_, yield_strings_) +{ +} + +void JSONCompactWithProgressRowOutputFormat::writeField(const IColumn & column, const ISerialization & serialization, size_t row_num) +{ + JSONUtils::writeFieldFromColumn(column, serialization, row_num, yield_strings, settings, *ostr); + ++field_number; + LOG_DEBUG(getLogger("JSONCompactWithProgressRowOutputFormat"), "Field number: {}", field_number); +} + +void JSONCompactWithProgressRowOutputFormat::writeFieldDelimiter() +{ + JSONUtils::writeFieldCompactDelimiter(*ostr); +} + +void JSONCompactWithProgressRowOutputFormat::writeRowStartDelimiter() +{ + if (has_progress) + writeProgress(); + JSONUtils::writeCompactArrayStart(*ostr, 2); +} + +void JSONCompactWithProgressRowOutputFormat::writeRowEndDelimiter() +{ + JSONUtils::writeCompactArrayEnd(*ostr); + field_number = 0; + ++row_count; +} + +void JSONCompactWithProgressRowOutputFormat::writeBeforeTotals() +{ + JSONUtils::writeFieldDelimiter(*ostr, 2); + JSONUtils::writeCompactArrayStart(*ostr, 1, "totals"); +} + +void JSONCompactWithProgressRowOutputFormat::writeTotals(const Columns & columns, size_t row_num) +{ + JSONUtils::writeCompactColumns(columns, serializations, row_num, yield_strings, settings, *ostr); +} + +void JSONCompactWithProgressRowOutputFormat::writeAfterTotals() +{ + JSONUtils::writeCompactArrayEnd(*ostr); +} + +void JSONCompactWithProgressRowOutputFormat::writeExtremesElement(const char * title, const Columns & columns, size_t row_num) +{ + JSONUtils::writeCompactArrayStart(*ostr, 2, title); + JSONUtils::writeCompactColumns(columns, serializations, row_num, yield_strings, settings, *ostr); + JSONUtils::writeCompactArrayEnd(*ostr); +} + +void JSONCompactWithProgressRowOutputFormat::onProgress(const Progress & value) +{ + LOG_DEBUG(getLogger("JSONCompactWithProgressRowOutputFormat"), "onProgress: {}", value.read_rows); + + progress.incrementPiecewiseAtomically(value); + String progress_line; + WriteBufferFromString buf(progress_line); + writeCString("{\"progress\":", buf); + progress.writeJSON(buf); + writeCString("}\n", buf); + buf.finalize(); + std::lock_guard lock(progress_lines_mutex); + progress_lines.emplace_back(std::move(progress_line)); + has_progress = true; +} + + +void JSONCompactWithProgressRowOutputFormat::flush() +{ + if (has_progress) + writeProgress(); + JSONRowOutputFormat::flush(); +} + +void JSONCompactWithProgressRowOutputFormat::writeSuffix() +{ + if (has_progress) + writeProgress(); + JSONRowOutputFormat::writeSuffix(); +} + +void JSONCompactWithProgressRowOutputFormat::writeProgress() +{ + std::lock_guard lock(progress_lines_mutex); + for (const auto & progress_line : progress_lines) + writeString(progress_line, *ostr); + progress_lines.clear(); + has_progress = false; +} + +void registerOutputFormatJSONCompactWithProgress(FormatFactory & factory) +{ + factory.registerOutputFormat("JSONCompactWithProgress", []( + WriteBuffer & buf, + const Block & sample, + const FormatSettings & format_settings) + { + return std::make_shared(buf, sample, format_settings, false); + }); + + factory.markOutputFormatSupportsParallelFormatting("JSONCompactWithProgress"); +} + +} diff --git a/src/Processors/Formats/Impl/JSONCompactWithProgressRowOutputFormat.h b/src/Processors/Formats/Impl/JSONCompactWithProgressRowOutputFormat.h new file mode 100644 index 00000000000..4bc10d41f19 --- /dev/null +++ b/src/Processors/Formats/Impl/JSONCompactWithProgressRowOutputFormat.h @@ -0,0 +1,53 @@ +#pragma once + +#include +#include +#include +#include + + +namespace DB +{ + +struct FormatSettings; + +/** The stream for outputting data in the JSONCompact- formats. + */ +class JSONCompactWithProgressRowOutputFormat final : public JSONRowOutputFormat +{ +public: + JSONCompactWithProgressRowOutputFormat( + WriteBuffer & out_, + const Block & header, + const FormatSettings & settings_, + bool yield_strings_); + + String getName() const override { return "JSONCompactWithProgressRowOutputFormat"; } + + void onProgress(const Progress & value) override; + void flush() override; + +private: + void writeField(const IColumn & column, const ISerialization & serialization, size_t row_num) override; + void writeFieldDelimiter() override; + void writeRowStartDelimiter() override; + void writeRowEndDelimiter() override; + bool supportTotals() const override { return true; } + bool supportExtremes() const override { return true; } + void writeBeforeTotals() override; + void writeAfterTotals() override; + void writeExtremesElement(const char * title, const Columns & columns, size_t row_num) override; + void writeTotals(const Columns & columns, size_t row_num) override; + + void writeProgress(); + void writeSuffix() override; + + Progress progress; + std::vector progress_lines; + std::mutex progress_lines_mutex; + /// To not lock mutex and check progress_lines every row, + /// we will use atomic flag that progress_lines is not empty. + std::atomic_bool has_progress = false; +}; + +} From 20860ed8b04f6588de12b4e59baf932f02f90d27 Mon Sep 17 00:00:00 2001 From: Alexey Korepanov Date: Sun, 23 Jun 2024 14:34:54 +0200 Subject: [PATCH 02/27] Enable onProgress calls to JSONCompactWithProgressRowOutputFormat --- .../Formats/Impl/JSONCompactWithProgressRowOutputFormat.cpp | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/Processors/Formats/Impl/JSONCompactWithProgressRowOutputFormat.cpp b/src/Processors/Formats/Impl/JSONCompactWithProgressRowOutputFormat.cpp index 78cf5b9a003..2cc7d99f8e7 100644 --- a/src/Processors/Formats/Impl/JSONCompactWithProgressRowOutputFormat.cpp +++ b/src/Processors/Formats/Impl/JSONCompactWithProgressRowOutputFormat.cpp @@ -118,8 +118,6 @@ void registerOutputFormatJSONCompactWithProgress(FormatFactory & factory) { return std::make_shared(buf, sample, format_settings, false); }); - - factory.markOutputFormatSupportsParallelFormatting("JSONCompactWithProgress"); } } From e1c60c4e40a03db459e71dd2b54b082d5205654d Mon Sep 17 00:00:00 2001 From: Alexey Korepanov Date: Mon, 24 Jun 2024 10:11:46 +0200 Subject: [PATCH 03/27] Remove debug output --- .../Formats/Impl/JSONCompactWithProgressRowOutputFormat.cpp | 6 ------ 1 file changed, 6 deletions(-) diff --git a/src/Processors/Formats/Impl/JSONCompactWithProgressRowOutputFormat.cpp b/src/Processors/Formats/Impl/JSONCompactWithProgressRowOutputFormat.cpp index 2cc7d99f8e7..39532fb76fb 100644 --- a/src/Processors/Formats/Impl/JSONCompactWithProgressRowOutputFormat.cpp +++ b/src/Processors/Formats/Impl/JSONCompactWithProgressRowOutputFormat.cpp @@ -4,9 +4,6 @@ #include -#include - - namespace DB { @@ -24,7 +21,6 @@ void JSONCompactWithProgressRowOutputFormat::writeField(const IColumn & column, { JSONUtils::writeFieldFromColumn(column, serialization, row_num, yield_strings, settings, *ostr); ++field_number; - LOG_DEBUG(getLogger("JSONCompactWithProgressRowOutputFormat"), "Field number: {}", field_number); } void JSONCompactWithProgressRowOutputFormat::writeFieldDelimiter() @@ -71,8 +67,6 @@ void JSONCompactWithProgressRowOutputFormat::writeExtremesElement(const char * t void JSONCompactWithProgressRowOutputFormat::onProgress(const Progress & value) { - LOG_DEBUG(getLogger("JSONCompactWithProgressRowOutputFormat"), "onProgress: {}", value.read_rows); - progress.incrementPiecewiseAtomically(value); String progress_line; WriteBufferFromString buf(progress_line); From 88736a0d74b068f271ee21c69760128f82adbb91 Mon Sep 17 00:00:00 2001 From: Alexey Korepanov Date: Sun, 7 Jul 2024 21:12:26 +0200 Subject: [PATCH 04/27] Update JSONCompactWithProgressRowOutputFormat to print JSON on each row --- src/Formats/JSONUtils.cpp | 64 +++++++++++++++++++ src/Formats/JSONUtils.h | 10 +++ ...JSONCompactWithProgressRowOutputFormat.cpp | 39 ++++++++++- .../JSONCompactWithProgressRowOutputFormat.h | 4 ++ 4 files changed, 115 insertions(+), 2 deletions(-) diff --git a/src/Formats/JSONUtils.cpp b/src/Formats/JSONUtils.cpp index f0985f4a6b7..16083441f3a 100644 --- a/src/Formats/JSONUtils.cpp +++ b/src/Formats/JSONUtils.cpp @@ -483,6 +483,33 @@ namespace JSONUtils writeArrayEnd(out, 1); } + + void writeCompactMetadata(const Names & names, const DataTypes & types, const FormatSettings & settings, WriteBuffer & out) + { + writeCompactArrayStart(out, 0, "meta"); + + for (size_t i = 0; i < names.size(); ++i) + { + writeCompactObjectStart(out); + writeTitle("name", out, 0, ""); + + /// The field names are pre-escaped to be put into JSON string literal. + writeChar('"', out); + writeString(names[i], out); + writeChar('"', out); + + writeFieldCompactDelimiter(out); + writeTitle("type", out, 0, ""); + writeJSONString(types[i]->getName(), out, settings); + writeCompactObjectEnd(out); + + if (i + 1 < names.size()) + writeFieldCompactDelimiter(out); + } + + writeCompactArrayEnd(out); + } + void writeAdditionalInfo( size_t rows, size_t rows_before_limit, @@ -523,6 +550,43 @@ namespace JSONUtils } } + void writeCompactAdditionalInfo( + size_t rows, + size_t rows_before_limit, + bool applied_limit, + const Stopwatch & watch, + const Progress & progress, + bool write_statistics, + WriteBuffer & out) + { + writeCompactObjectStart(out, 0, "statistics"); + writeTitle("rows", out, 0, ""); + writeIntText(rows, out); + writeFieldCompactDelimiter(out); + + if (applied_limit) + { + writeTitle("rows_before_limit_at_least", out, 0, ""); + writeIntText(rows_before_limit, out); + writeFieldCompactDelimiter(out); + } + + if (write_statistics) + { + writeTitle("elapsed", out, 0, ""); + writeText(watch.elapsedSeconds(), out); + writeFieldCompactDelimiter(out); + + writeTitle("rows_read", out, 0, ""); + writeText(progress.read_rows.load(), out); + writeFieldCompactDelimiter(out); + + writeTitle("bytes_read", out, 0, ""); + writeText(progress.read_bytes.load(), out); + } + writeCompactObjectEnd(out); + } + void writeException(const String & exception_message, WriteBuffer & out, const FormatSettings & settings, size_t indent) { writeTitle("exception", out, indent, " "); diff --git a/src/Formats/JSONUtils.h b/src/Formats/JSONUtils.h index 7ee111c1285..718b9b2f610 100644 --- a/src/Formats/JSONUtils.h +++ b/src/Formats/JSONUtils.h @@ -99,6 +99,7 @@ namespace JSONUtils WriteBuffer & out); void writeMetadata(const Names & names, const DataTypes & types, const FormatSettings & settings, WriteBuffer & out); + void writeCompactMetadata(const Names & names, const DataTypes & types, const FormatSettings & settings, WriteBuffer & out); void writeAdditionalInfo( size_t rows, @@ -109,6 +110,15 @@ namespace JSONUtils bool write_statistics, WriteBuffer & out); + void writeCompactAdditionalInfo( + size_t rows, + size_t rows_before_limit, + bool applied_limit, + const Stopwatch & watch, + const Progress & progress, + bool write_statistics, + WriteBuffer & out); + void writeException(const String & exception_message, WriteBuffer & out, const FormatSettings & settings, size_t indent = 0); void skipColon(ReadBuffer & in); diff --git a/src/Processors/Formats/Impl/JSONCompactWithProgressRowOutputFormat.cpp b/src/Processors/Formats/Impl/JSONCompactWithProgressRowOutputFormat.cpp index 39532fb76fb..7a19b56d0ca 100644 --- a/src/Processors/Formats/Impl/JSONCompactWithProgressRowOutputFormat.cpp +++ b/src/Processors/Formats/Impl/JSONCompactWithProgressRowOutputFormat.cpp @@ -17,6 +17,14 @@ JSONCompactWithProgressRowOutputFormat::JSONCompactWithProgressRowOutputFormat( { } +void JSONCompactWithProgressRowOutputFormat::writePrefix() +{ + JSONUtils::writeCompactObjectStart(*ostr); + JSONUtils::writeCompactMetadata(names, types, settings, *ostr); + JSONUtils::writeCompactObjectEnd(*ostr); + writeCString("}\n", *ostr); +} + void JSONCompactWithProgressRowOutputFormat::writeField(const IColumn & column, const ISerialization & serialization, size_t row_num) { JSONUtils::writeFieldFromColumn(column, serialization, row_num, yield_strings, settings, *ostr); @@ -32,16 +40,22 @@ void JSONCompactWithProgressRowOutputFormat::writeRowStartDelimiter() { if (has_progress) writeProgress(); - JSONUtils::writeCompactArrayStart(*ostr, 2); + writeCString("{\"data\":", *ostr); + JSONUtils::writeCompactArrayStart(*ostr); } void JSONCompactWithProgressRowOutputFormat::writeRowEndDelimiter() { JSONUtils::writeCompactArrayEnd(*ostr); + writeCString("}\n", *ostr); field_number = 0; ++row_count; } +void JSONCompactWithProgressRowOutputFormat::writeRowBetweenDelimiter() +{ +} + void JSONCompactWithProgressRowOutputFormat::writeBeforeTotals() { JSONUtils::writeFieldDelimiter(*ostr, 2); @@ -91,7 +105,6 @@ void JSONCompactWithProgressRowOutputFormat::writeSuffix() { if (has_progress) writeProgress(); - JSONRowOutputFormat::writeSuffix(); } void JSONCompactWithProgressRowOutputFormat::writeProgress() @@ -103,6 +116,28 @@ void JSONCompactWithProgressRowOutputFormat::writeProgress() has_progress = false; } +void JSONCompactWithProgressRowOutputFormat::finalizeImpl() +{ + JSONUtils::writeCompactAdditionalInfo( + row_count, + statistics.rows_before_limit, + statistics.applied_limit, + statistics.watch, + statistics.progress, + settings.write_statistics && exception_message.empty(), + *ostr); + + exception_message = "Test exception message."; + if (!exception_message.empty()) + { + writeCString("\n", *ostr); + JSONUtils::writeCompactObjectStart(*ostr); + JSONUtils::writeException(exception_message, *ostr, settings, 0); + JSONUtils::writeCompactObjectEnd(*ostr); + } + ostr->next(); +} + void registerOutputFormatJSONCompactWithProgress(FormatFactory & factory) { factory.registerOutputFormat("JSONCompactWithProgress", []( diff --git a/src/Processors/Formats/Impl/JSONCompactWithProgressRowOutputFormat.h b/src/Processors/Formats/Impl/JSONCompactWithProgressRowOutputFormat.h index 4bc10d41f19..0c7e6b295e8 100644 --- a/src/Processors/Formats/Impl/JSONCompactWithProgressRowOutputFormat.h +++ b/src/Processors/Formats/Impl/JSONCompactWithProgressRowOutputFormat.h @@ -32,6 +32,7 @@ private: void writeFieldDelimiter() override; void writeRowStartDelimiter() override; void writeRowEndDelimiter() override; + void writeRowBetweenDelimiter() override; bool supportTotals() const override { return true; } bool supportExtremes() const override { return true; } void writeBeforeTotals() override; @@ -40,7 +41,10 @@ private: void writeTotals(const Columns & columns, size_t row_num) override; void writeProgress(); + void writePrefix() override; void writeSuffix() override; + void finalizeImpl() override; + Progress progress; std::vector progress_lines; From 2ae9490a681d3d522dd30e96b9f36b2d638ffd10 Mon Sep 17 00:00:00 2001 From: Alexey Korepanov Date: Sun, 7 Jul 2024 21:22:02 +0200 Subject: [PATCH 05/27] Fix progress rows and bytes read --- .../Formats/Impl/JSONCompactWithProgressRowOutputFormat.cpp | 5 ++--- .../Formats/Impl/JSONCompactWithProgressRowOutputFormat.h | 1 - 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/src/Processors/Formats/Impl/JSONCompactWithProgressRowOutputFormat.cpp b/src/Processors/Formats/Impl/JSONCompactWithProgressRowOutputFormat.cpp index 7a19b56d0ca..d72a6556c2b 100644 --- a/src/Processors/Formats/Impl/JSONCompactWithProgressRowOutputFormat.cpp +++ b/src/Processors/Formats/Impl/JSONCompactWithProgressRowOutputFormat.cpp @@ -81,11 +81,11 @@ void JSONCompactWithProgressRowOutputFormat::writeExtremesElement(const char * t void JSONCompactWithProgressRowOutputFormat::onProgress(const Progress & value) { - progress.incrementPiecewiseAtomically(value); + statistics.progress.incrementPiecewiseAtomically(value); String progress_line; WriteBufferFromString buf(progress_line); writeCString("{\"progress\":", buf); - progress.writeJSON(buf); + statistics.progress.writeJSON(buf); writeCString("}\n", buf); buf.finalize(); std::lock_guard lock(progress_lines_mutex); @@ -127,7 +127,6 @@ void JSONCompactWithProgressRowOutputFormat::finalizeImpl() settings.write_statistics && exception_message.empty(), *ostr); - exception_message = "Test exception message."; if (!exception_message.empty()) { writeCString("\n", *ostr); diff --git a/src/Processors/Formats/Impl/JSONCompactWithProgressRowOutputFormat.h b/src/Processors/Formats/Impl/JSONCompactWithProgressRowOutputFormat.h index 0c7e6b295e8..669e342c583 100644 --- a/src/Processors/Formats/Impl/JSONCompactWithProgressRowOutputFormat.h +++ b/src/Processors/Formats/Impl/JSONCompactWithProgressRowOutputFormat.h @@ -46,7 +46,6 @@ private: void finalizeImpl() override; - Progress progress; std::vector progress_lines; std::mutex progress_lines_mutex; /// To not lock mutex and check progress_lines every row, From 2469647609c53ac968d7fba60867c78ecebe522c Mon Sep 17 00:00:00 2001 From: Alexey Korepanov Date: Sun, 7 Jul 2024 21:30:36 +0200 Subject: [PATCH 06/27] Fix statistics section --- src/Formats/JSONUtils.cpp | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/Formats/JSONUtils.cpp b/src/Formats/JSONUtils.cpp index 16083441f3a..0918658cf6f 100644 --- a/src/Formats/JSONUtils.cpp +++ b/src/Formats/JSONUtils.cpp @@ -559,6 +559,7 @@ namespace JSONUtils bool write_statistics, WriteBuffer & out) { + writeCompactObjectStart(out); writeCompactObjectStart(out, 0, "statistics"); writeTitle("rows", out, 0, ""); writeIntText(rows, out); @@ -585,6 +586,7 @@ namespace JSONUtils writeText(progress.read_bytes.load(), out); } writeCompactObjectEnd(out); + writeCompactObjectEnd(out); } void writeException(const String & exception_message, WriteBuffer & out, const FormatSettings & settings, size_t indent) From 00e47d64b0c12023d926b383c6dea327e9806fc7 Mon Sep 17 00:00:00 2001 From: Alexey Korepanov Date: Mon, 8 Jul 2024 09:02:07 +0200 Subject: [PATCH 07/27] Add JSONCompactWithProgressStrings format --- .../Impl/JSONCompactWithProgressRowOutputFormat.cpp | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/src/Processors/Formats/Impl/JSONCompactWithProgressRowOutputFormat.cpp b/src/Processors/Formats/Impl/JSONCompactWithProgressRowOutputFormat.cpp index d72a6556c2b..91608cd5050 100644 --- a/src/Processors/Formats/Impl/JSONCompactWithProgressRowOutputFormat.cpp +++ b/src/Processors/Formats/Impl/JSONCompactWithProgressRowOutputFormat.cpp @@ -146,6 +146,14 @@ void registerOutputFormatJSONCompactWithProgress(FormatFactory & factory) { return std::make_shared(buf, sample, format_settings, false); }); + + factory.registerOutputFormat("JSONCompactWithProgressStrings", []( + WriteBuffer & buf, + const Block & sample, + const FormatSettings & format_settings) + { + return std::make_shared(buf, sample, format_settings, true); + }); } } From 58d34e306cec5b41ac372934e43639c2df24e54e Mon Sep 17 00:00:00 2001 From: Alexey Korepanov Date: Mon, 8 Jul 2024 09:08:03 +0200 Subject: [PATCH 08/27] Add documentation on JSONCompactWithProgress --- docs/en/interfaces/formats.md | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/docs/en/interfaces/formats.md b/docs/en/interfaces/formats.md index ffdd7e2ca25..6161cddefdd 100644 --- a/docs/en/interfaces/formats.md +++ b/docs/en/interfaces/formats.md @@ -38,6 +38,7 @@ The supported formats are: | [JSONCompact](#jsoncompact) | ✔ | ✔ | | [JSONCompactStrings](#jsoncompactstrings) | ✗ | ✔ | | [JSONCompactColumns](#jsoncompactcolumns) | ✔ | ✔ | +| [JSONCompactWithProgress](#jsoncompactwithprogress) | ✗ | ✔ | | [JSONEachRow](#jsoneachrow) | ✔ | ✔ | | [PrettyJSONEachRow](#prettyjsoneachrow) | ✗ | ✔ | | [JSONEachRowWithProgress](#jsoneachrowwithprogress) | ✗ | ✔ | @@ -925,6 +926,23 @@ Example: Columns that are not present in the block will be filled with default values (you can use [input_format_defaults_for_omitted_fields](/docs/en/operations/settings/settings-formats.md/#input_format_defaults_for_omitted_fields) setting here) +## JSONCompactWithProgress (#jsoncompactwithprogress) + +In this format, ClickHouse outputs each row as a separated, newline-delimited JSON Object. + +Each row is either a metadata object, data object, progress information or statistics object. + +Example: + +```json +{"meta": [{"name":"id", "type":"UInt32"}, {"name":"name", "type":"String"}]}} +{"progress":{"read_rows":"8","read_bytes":"168","written_rows":"0","written_bytes":"0","total_rows_to_read":"2","result_rows":"0","result_bytes":"0","elapsed_ns":"0"}} +{"data":["1", "John Doe"]} +{"data":["2", "Joe Doe"]} +{"statistics": {"rows":2, "rows_before_limit_at_least":8, "elapsed":0.001995, "rows_read":8, "bytes_read":168}} +``` + + ## JSONEachRow {#jsoneachrow} In this format, ClickHouse outputs each row as a separated, newline-delimited JSON Object. From 78cb02503f05d1c1c84dbb17fd51700d74f697df Mon Sep 17 00:00:00 2001 From: Alexey Korepanov Date: Mon, 8 Jul 2024 09:22:07 +0200 Subject: [PATCH 09/27] Remove extra space --- .../Formats/Impl/JSONCompactWithProgressRowOutputFormat.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Processors/Formats/Impl/JSONCompactWithProgressRowOutputFormat.cpp b/src/Processors/Formats/Impl/JSONCompactWithProgressRowOutputFormat.cpp index 91608cd5050..6be2d092305 100644 --- a/src/Processors/Formats/Impl/JSONCompactWithProgressRowOutputFormat.cpp +++ b/src/Processors/Formats/Impl/JSONCompactWithProgressRowOutputFormat.cpp @@ -111,7 +111,7 @@ void JSONCompactWithProgressRowOutputFormat::writeProgress() { std::lock_guard lock(progress_lines_mutex); for (const auto & progress_line : progress_lines) - writeString(progress_line, *ostr); + writeString(progress_line, *ostr); progress_lines.clear(); has_progress = false; } From 7fbcaeaae2bddc07898fd4e2eb7aa9331e2d0df5 Mon Sep 17 00:00:00 2001 From: Alexey Korepanov Date: Mon, 8 Jul 2024 09:29:14 +0200 Subject: [PATCH 10/27] Remove wrong comment --- .../Formats/Impl/JSONCompactWithProgressRowOutputFormat.h | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/Processors/Formats/Impl/JSONCompactWithProgressRowOutputFormat.h b/src/Processors/Formats/Impl/JSONCompactWithProgressRowOutputFormat.h index 669e342c583..dc7a70229e7 100644 --- a/src/Processors/Formats/Impl/JSONCompactWithProgressRowOutputFormat.h +++ b/src/Processors/Formats/Impl/JSONCompactWithProgressRowOutputFormat.h @@ -11,8 +11,6 @@ namespace DB struct FormatSettings; -/** The stream for outputting data in the JSONCompact- formats. - */ class JSONCompactWithProgressRowOutputFormat final : public JSONRowOutputFormat { public: From a3360d0e63c08132ecd73423d2dd9fc576b54589 Mon Sep 17 00:00:00 2001 From: Alexey Korepanov Date: Mon, 8 Jul 2024 13:53:24 +0200 Subject: [PATCH 11/27] Fix code style --- ...JSONCompactWithProgressRowOutputFormat.cpp | 29 +++++++------------ .../JSONCompactWithProgressRowOutputFormat.h | 6 +--- 2 files changed, 11 insertions(+), 24 deletions(-) diff --git a/src/Processors/Formats/Impl/JSONCompactWithProgressRowOutputFormat.cpp b/src/Processors/Formats/Impl/JSONCompactWithProgressRowOutputFormat.cpp index 6be2d092305..9603ce0265d 100644 --- a/src/Processors/Formats/Impl/JSONCompactWithProgressRowOutputFormat.cpp +++ b/src/Processors/Formats/Impl/JSONCompactWithProgressRowOutputFormat.cpp @@ -1,6 +1,6 @@ -#include #include #include +#include #include @@ -9,10 +9,7 @@ namespace DB { JSONCompactWithProgressRowOutputFormat::JSONCompactWithProgressRowOutputFormat( - WriteBuffer & out_, - const Block & header, - const FormatSettings & settings_, - bool yield_strings_) + WriteBuffer & out_, const Block & header, const FormatSettings & settings_, bool yield_strings_) : JSONRowOutputFormat(out_, header, settings_, yield_strings_) { } @@ -139,21 +136,15 @@ void JSONCompactWithProgressRowOutputFormat::finalizeImpl() void registerOutputFormatJSONCompactWithProgress(FormatFactory & factory) { - factory.registerOutputFormat("JSONCompactWithProgress", []( - WriteBuffer & buf, - const Block & sample, - const FormatSettings & format_settings) - { - return std::make_shared(buf, sample, format_settings, false); - }); + factory.registerOutputFormat( + "JSONCompactWithProgress", + [](WriteBuffer & buf, const Block & sample, const FormatSettings & format_settings) + { return std::make_shared(buf, sample, format_settings, false); }); - factory.registerOutputFormat("JSONCompactWithProgressStrings", []( - WriteBuffer & buf, - const Block & sample, - const FormatSettings & format_settings) - { - return std::make_shared(buf, sample, format_settings, true); - }); + factory.registerOutputFormat( + "JSONCompactWithProgressStrings", + [](WriteBuffer & buf, const Block & sample, const FormatSettings & format_settings) + { return std::make_shared(buf, sample, format_settings, true); }); } } diff --git a/src/Processors/Formats/Impl/JSONCompactWithProgressRowOutputFormat.h b/src/Processors/Formats/Impl/JSONCompactWithProgressRowOutputFormat.h index dc7a70229e7..1c21914d8cb 100644 --- a/src/Processors/Formats/Impl/JSONCompactWithProgressRowOutputFormat.h +++ b/src/Processors/Formats/Impl/JSONCompactWithProgressRowOutputFormat.h @@ -14,11 +14,7 @@ struct FormatSettings; class JSONCompactWithProgressRowOutputFormat final : public JSONRowOutputFormat { public: - JSONCompactWithProgressRowOutputFormat( - WriteBuffer & out_, - const Block & header, - const FormatSettings & settings_, - bool yield_strings_); + JSONCompactWithProgressRowOutputFormat(WriteBuffer & out_, const Block & header, const FormatSettings & settings_, bool yield_strings_); String getName() const override { return "JSONCompactWithProgressRowOutputFormat"; } From 0631ff93778801768edde14c5fd5ed22701f6ec3 Mon Sep 17 00:00:00 2001 From: Alexey Korepanov Date: Tue, 23 Jul 2024 12:14:54 +0100 Subject: [PATCH 12/27] Add jsoncompactwithprogress to aspell-dict.txt --- utils/check-style/aspell-ignore/en/aspell-dict.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/utils/check-style/aspell-ignore/en/aspell-dict.txt b/utils/check-style/aspell-ignore/en/aspell-dict.txt index 3967179f078..2e403f44e4e 100644 --- a/utils/check-style/aspell-ignore/en/aspell-dict.txt +++ b/utils/check-style/aspell-ignore/en/aspell-dict.txt @@ -1855,6 +1855,7 @@ jsoncompactstrings jsoncompactstringseachrow jsoncompactstringseachrowwithnames jsoncompactstringseachrowwithnamesandtypes +jsoncompactwithprogress jsoneachrow jsoneachrowwithprogress jsonobjecteachrow From 53a55221845802e67f9586af615fcb5ed84b20eb Mon Sep 17 00:00:00 2001 From: Alexey Korepanov Date: Tue, 23 Jul 2024 12:19:06 +0100 Subject: [PATCH 13/27] Fix example output in the documentation --- docs/en/interfaces/formats.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/en/interfaces/formats.md b/docs/en/interfaces/formats.md index 6161cddefdd..8795b71e0ac 100644 --- a/docs/en/interfaces/formats.md +++ b/docs/en/interfaces/formats.md @@ -935,7 +935,7 @@ Each row is either a metadata object, data object, progress information or stati Example: ```json -{"meta": [{"name":"id", "type":"UInt32"}, {"name":"name", "type":"String"}]}} +{"meta": [{"name":"id", "type":"UInt32"}, {"name":"name", "type":"String"}]} {"progress":{"read_rows":"8","read_bytes":"168","written_rows":"0","written_bytes":"0","total_rows_to_read":"2","result_rows":"0","result_bytes":"0","elapsed_ns":"0"}} {"data":["1", "John Doe"]} {"data":["2", "Joe Doe"]} From a4a0611c1c0b40ea6b99d7e5cf3972fda201bb9d Mon Sep 17 00:00:00 2001 From: Alexey Korepanov Date: Tue, 23 Jul 2024 12:29:40 +0100 Subject: [PATCH 14/27] Fix docs spelling --- utils/check-style/aspell-ignore/en/aspell-dict.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/utils/check-style/aspell-ignore/en/aspell-dict.txt b/utils/check-style/aspell-ignore/en/aspell-dict.txt index 2e403f44e4e..c64cfcb0968 100644 --- a/utils/check-style/aspell-ignore/en/aspell-dict.txt +++ b/utils/check-style/aspell-ignore/en/aspell-dict.txt @@ -406,6 +406,7 @@ JSONCompactStrings JSONCompactStringsEachRow JSONCompactStringsEachRowWithNames JSONCompactStringsEachRowWithNamesAndTypes +JSONCompactWithProgress JSONEachRow JSONEachRowWithProgress JSONExtract From ef4ad01d7d57942a79b46ad05c10160d557ff75f Mon Sep 17 00:00:00 2001 From: Alexey Korepanov Date: Tue, 23 Jul 2024 15:03:01 +0100 Subject: [PATCH 15/27] Fix totals formatting --- .../Impl/JSONCompactWithProgressRowOutputFormat.cpp | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/src/Processors/Formats/Impl/JSONCompactWithProgressRowOutputFormat.cpp b/src/Processors/Formats/Impl/JSONCompactWithProgressRowOutputFormat.cpp index 9603ce0265d..ddf071324c6 100644 --- a/src/Processors/Formats/Impl/JSONCompactWithProgressRowOutputFormat.cpp +++ b/src/Processors/Formats/Impl/JSONCompactWithProgressRowOutputFormat.cpp @@ -55,8 +55,8 @@ void JSONCompactWithProgressRowOutputFormat::writeRowBetweenDelimiter() void JSONCompactWithProgressRowOutputFormat::writeBeforeTotals() { - JSONUtils::writeFieldDelimiter(*ostr, 2); - JSONUtils::writeCompactArrayStart(*ostr, 1, "totals"); + JSONUtils::writeCompactObjectStart(*ostr); + JSONUtils::writeCompactArrayStart(*ostr, 0, "totals"); } void JSONCompactWithProgressRowOutputFormat::writeTotals(const Columns & columns, size_t row_num) @@ -67,6 +67,8 @@ void JSONCompactWithProgressRowOutputFormat::writeTotals(const Columns & columns void JSONCompactWithProgressRowOutputFormat::writeAfterTotals() { JSONUtils::writeCompactArrayEnd(*ostr); + JSONUtils::writeCompactObjectEnd(*ostr); + writeCString("}\n", *ostr); } void JSONCompactWithProgressRowOutputFormat::writeExtremesElement(const char * title, const Columns & columns, size_t row_num) @@ -131,6 +133,7 @@ void JSONCompactWithProgressRowOutputFormat::finalizeImpl() JSONUtils::writeException(exception_message, *ostr, settings, 0); JSONUtils::writeCompactObjectEnd(*ostr); } + writeCString("\n", *ostr); ostr->next(); } From f3710e9a29e4b866bfd55886c18ebad954a48e42 Mon Sep 17 00:00:00 2001 From: Alexey Korepanov Date: Tue, 23 Jul 2024 15:04:05 +0100 Subject: [PATCH 16/27] Add stateless test for JSONCompactWithProgress output format --- ...03174_json_compact_with_progress.reference | 14 +++++++++++++ .../03174_json_compact_with_progress.sh | 20 +++++++++++++++++++ 2 files changed, 34 insertions(+) create mode 100644 tests/queries/0_stateless/03174_json_compact_with_progress.reference create mode 100755 tests/queries/0_stateless/03174_json_compact_with_progress.sh diff --git a/tests/queries/0_stateless/03174_json_compact_with_progress.reference b/tests/queries/0_stateless/03174_json_compact_with_progress.reference new file mode 100644 index 00000000000..6e83db33565 --- /dev/null +++ b/tests/queries/0_stateless/03174_json_compact_with_progress.reference @@ -0,0 +1,14 @@ +1 +{"meta": [{"name":"value", "type":"UInt8"}, {"name":"name", "type":"String"}]}} +{"data":[1, "a"]} +{"data":[2, "b"]} +{"data":[3, "c"]} +{"progress":{"read_rows":"3","read_bytes":"33","written_rows":"0","written_bytes":"0","total_rows_to_read":"3","result_rows":"0","result_bytes":"0","elapsed_ns":"ELAPSED_NS"}} +{"statistics": {"rows":3, "elapsed":ELAPSED, "rows_read":3, "bytes_read":33}} +2 +{"meta": [{"name":"name", "type":"String"}, {"name":"c", "type":"UInt64"}]}} +{"data":["a", "1"]} +{"data":["b", "1"]} +{"data":["c", "1"]} +{"totals": ["", "3"]}} +{"statistics": {"rows":3, "elapsed":ELAPSED, "rows_read":3, "bytes_read":30}} diff --git a/tests/queries/0_stateless/03174_json_compact_with_progress.sh b/tests/queries/0_stateless/03174_json_compact_with_progress.sh new file mode 100755 index 00000000000..c21f7228517 --- /dev/null +++ b/tests/queries/0_stateless/03174_json_compact_with_progress.sh @@ -0,0 +1,20 @@ +#!/usr/bin/env bash +# Tags: no-fasttest, no-parallel + +CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh +. "$CUR_DIR"/../shell_config.sh + +$CLICKHOUSE_CLIENT -q "DROP TABLE IF EXISTS test_table;" + +$CLICKHOUSE_CLIENT -q "SELECT 1;" +# Check JSONCompactWithProgress Output +$CLICKHOUSE_CLIENT -q "CREATE TABLE test_table (value UInt8, name String) ENGINE = MergeTree() ORDER BY value;" +$CLICKHOUSE_CLIENT -q "INSERT INTO test_table VALUES (1, 'a'), (2, 'b'), (3, 'c');" +$CLICKHOUSE_CLIENT -q "SELECT * FROM test_table FORMAT JSONCompactWithProgress settings max_block_size=2;" | sed -E 's/"elapsed_ns":"[0-9]+"/"elapsed_ns":"ELAPSED_NS"/g; s/"elapsed":[0-9]+\.[0-9]+/"elapsed":ELAPSED/g' + +$CLICKHOUSE_CLIENT -q "SELECT 2;" +# Check Totals +$CLICKHOUSE_CLIENT -q "SELECT name, count() AS c FROM test_table GROUP BY name WITH TOTALS ORDER BY name FORMAT JSONCompactWithProgress settings max_block_size=2;" | sed -E 's/"elapsed_ns":"[0-9]+"/"elapsed_ns":"ELAPSED_NS"/g; s/"elapsed":[0-9]+\.[0-9]+/"elapsed":ELAPSED/g' + +$CLICKHOUSE_CLIENT -q "DROP TABLE IF EXISTS test_table;" From 118e329e23b51673e7d62bb3ffa8a34a8ecdb330 Mon Sep 17 00:00:00 2001 From: Alexey Korepanov Date: Wed, 24 Jul 2024 13:09:47 +0100 Subject: [PATCH 17/27] Fix flaky test --- ...03174_json_compact_with_progress.reference | 19 ++++++++++--------- .../03174_json_compact_with_progress.sh | 4 ++-- 2 files changed, 12 insertions(+), 11 deletions(-) diff --git a/tests/queries/0_stateless/03174_json_compact_with_progress.reference b/tests/queries/0_stateless/03174_json_compact_with_progress.reference index 6e83db33565..b735c871fbd 100644 --- a/tests/queries/0_stateless/03174_json_compact_with_progress.reference +++ b/tests/queries/0_stateless/03174_json_compact_with_progress.reference @@ -1,14 +1,15 @@ 1 -{"meta": [{"name":"value", "type":"UInt8"}, {"name":"name", "type":"String"}]}} -{"data":[1, "a"]} -{"data":[2, "b"]} -{"data":[3, "c"]} +{"meta": [{"name":"value", "type":"UInt8"}, {"name":"name", "type":"String"}, {"name":"sleep(0.1)", "type":"UInt8"}]}} {"progress":{"read_rows":"3","read_bytes":"33","written_rows":"0","written_bytes":"0","total_rows_to_read":"3","result_rows":"0","result_bytes":"0","elapsed_ns":"ELAPSED_NS"}} +{"data":[1, "a", 0]} +{"data":[2, "b", 0]} +{"data":[3, "c", 0]} {"statistics": {"rows":3, "elapsed":ELAPSED, "rows_read":3, "bytes_read":33}} 2 -{"meta": [{"name":"name", "type":"String"}, {"name":"c", "type":"UInt64"}]}} -{"data":["a", "1"]} -{"data":["b", "1"]} -{"data":["c", "1"]} -{"totals": ["", "3"]}} +{"meta": [{"name":"name", "type":"String"}, {"name":"count()", "type":"UInt64"}, {"name":"c", "type":"UInt8"}]}} +{"progress":{"read_rows":"3","read_bytes":"30","written_rows":"0","written_bytes":"0","total_rows_to_read":"3","result_rows":"0","result_bytes":"0","elapsed_ns":"ELAPSED_NS"}} +{"data":["a", "1", 0]} +{"data":["b", "1", 0]} +{"data":["c", "1", 0]} +{"totals": ["", "3", 0]}} {"statistics": {"rows":3, "elapsed":ELAPSED, "rows_read":3, "bytes_read":30}} diff --git a/tests/queries/0_stateless/03174_json_compact_with_progress.sh b/tests/queries/0_stateless/03174_json_compact_with_progress.sh index c21f7228517..383668de858 100755 --- a/tests/queries/0_stateless/03174_json_compact_with_progress.sh +++ b/tests/queries/0_stateless/03174_json_compact_with_progress.sh @@ -11,10 +11,10 @@ $CLICKHOUSE_CLIENT -q "SELECT 1;" # Check JSONCompactWithProgress Output $CLICKHOUSE_CLIENT -q "CREATE TABLE test_table (value UInt8, name String) ENGINE = MergeTree() ORDER BY value;" $CLICKHOUSE_CLIENT -q "INSERT INTO test_table VALUES (1, 'a'), (2, 'b'), (3, 'c');" -$CLICKHOUSE_CLIENT -q "SELECT * FROM test_table FORMAT JSONCompactWithProgress settings max_block_size=2;" | sed -E 's/"elapsed_ns":"[0-9]+"/"elapsed_ns":"ELAPSED_NS"/g; s/"elapsed":[0-9]+\.[0-9]+/"elapsed":ELAPSED/g' +$CLICKHOUSE_CLIENT -q "SELECT *, sleep(0.1) FROM test_table FORMAT JSONCompactWithProgress settings max_block_size=2;" | sed -E 's/"elapsed_ns":"[0-9]+"/"elapsed_ns":"ELAPSED_NS"/g; s/"elapsed":[0-9]+\.[0-9]+/"elapsed":ELAPSED/g' $CLICKHOUSE_CLIENT -q "SELECT 2;" # Check Totals -$CLICKHOUSE_CLIENT -q "SELECT name, count() AS c FROM test_table GROUP BY name WITH TOTALS ORDER BY name FORMAT JSONCompactWithProgress settings max_block_size=2;" | sed -E 's/"elapsed_ns":"[0-9]+"/"elapsed_ns":"ELAPSED_NS"/g; s/"elapsed":[0-9]+\.[0-9]+/"elapsed":ELAPSED/g' +$CLICKHOUSE_CLIENT -q "SELECT name, count(), sleep(0.1) AS c FROM test_table GROUP BY name WITH TOTALS ORDER BY name FORMAT JSONCompactWithProgress settings max_block_size=2;" | sed -E 's/"elapsed_ns":"[0-9]+"/"elapsed_ns":"ELAPSED_NS"/g; s/"elapsed":[0-9]+\.[0-9]+/"elapsed":ELAPSED/g' $CLICKHOUSE_CLIENT -q "DROP TABLE IF EXISTS test_table;" From b0ec8c92fcc09857345acb4e4f627827556d17cd Mon Sep 17 00:00:00 2001 From: Alexey Korepanov Date: Wed, 24 Jul 2024 17:49:33 +0100 Subject: [PATCH 18/27] Fix totals json object --- .../Formats/Impl/JSONCompactWithProgressRowOutputFormat.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Processors/Formats/Impl/JSONCompactWithProgressRowOutputFormat.cpp b/src/Processors/Formats/Impl/JSONCompactWithProgressRowOutputFormat.cpp index ddf071324c6..0814e5f45d6 100644 --- a/src/Processors/Formats/Impl/JSONCompactWithProgressRowOutputFormat.cpp +++ b/src/Processors/Formats/Impl/JSONCompactWithProgressRowOutputFormat.cpp @@ -68,7 +68,7 @@ void JSONCompactWithProgressRowOutputFormat::writeAfterTotals() { JSONUtils::writeCompactArrayEnd(*ostr); JSONUtils::writeCompactObjectEnd(*ostr); - writeCString("}\n", *ostr); + writeCString("\n", *ostr); } void JSONCompactWithProgressRowOutputFormat::writeExtremesElement(const char * title, const Columns & columns, size_t row_num) From caaf14a0a91bd4779cc292dd674761a47148d24d Mon Sep 17 00:00:00 2001 From: Alexey Korepanov Date: Wed, 24 Jul 2024 17:56:14 +0100 Subject: [PATCH 19/27] Fix flaky test by ignoring progress lines when comparing output --- ...03174_json_compact_with_progress.reference | 20 +++++++++---------- .../03174_json_compact_with_progress.sh | 4 ++-- 2 files changed, 11 insertions(+), 13 deletions(-) diff --git a/tests/queries/0_stateless/03174_json_compact_with_progress.reference b/tests/queries/0_stateless/03174_json_compact_with_progress.reference index b735c871fbd..b45f296e5c0 100644 --- a/tests/queries/0_stateless/03174_json_compact_with_progress.reference +++ b/tests/queries/0_stateless/03174_json_compact_with_progress.reference @@ -1,15 +1,13 @@ 1 -{"meta": [{"name":"value", "type":"UInt8"}, {"name":"name", "type":"String"}, {"name":"sleep(0.1)", "type":"UInt8"}]}} -{"progress":{"read_rows":"3","read_bytes":"33","written_rows":"0","written_bytes":"0","total_rows_to_read":"3","result_rows":"0","result_bytes":"0","elapsed_ns":"ELAPSED_NS"}} -{"data":[1, "a", 0]} -{"data":[2, "b", 0]} -{"data":[3, "c", 0]} +{"meta": [{"name":"value", "type":"UInt8"}, {"name":"name", "type":"String"}]}} +{"data":[1, "a"]} +{"data":[2, "b"]} +{"data":[3, "c"]} {"statistics": {"rows":3, "elapsed":ELAPSED, "rows_read":3, "bytes_read":33}} 2 -{"meta": [{"name":"name", "type":"String"}, {"name":"count()", "type":"UInt64"}, {"name":"c", "type":"UInt8"}]}} -{"progress":{"read_rows":"3","read_bytes":"30","written_rows":"0","written_bytes":"0","total_rows_to_read":"3","result_rows":"0","result_bytes":"0","elapsed_ns":"ELAPSED_NS"}} -{"data":["a", "1", 0]} -{"data":["b", "1", 0]} -{"data":["c", "1", 0]} -{"totals": ["", "3", 0]}} +{"meta": [{"name":"name", "type":"String"}, {"name":"c", "type":"UInt64"}]}} +{"data":["a", "1"]} +{"data":["b", "1"]} +{"data":["c", "1"]} +{"totals": ["", "3"]} {"statistics": {"rows":3, "elapsed":ELAPSED, "rows_read":3, "bytes_read":30}} diff --git a/tests/queries/0_stateless/03174_json_compact_with_progress.sh b/tests/queries/0_stateless/03174_json_compact_with_progress.sh index 383668de858..b440dbe2891 100755 --- a/tests/queries/0_stateless/03174_json_compact_with_progress.sh +++ b/tests/queries/0_stateless/03174_json_compact_with_progress.sh @@ -11,10 +11,10 @@ $CLICKHOUSE_CLIENT -q "SELECT 1;" # Check JSONCompactWithProgress Output $CLICKHOUSE_CLIENT -q "CREATE TABLE test_table (value UInt8, name String) ENGINE = MergeTree() ORDER BY value;" $CLICKHOUSE_CLIENT -q "INSERT INTO test_table VALUES (1, 'a'), (2, 'b'), (3, 'c');" -$CLICKHOUSE_CLIENT -q "SELECT *, sleep(0.1) FROM test_table FORMAT JSONCompactWithProgress settings max_block_size=2;" | sed -E 's/"elapsed_ns":"[0-9]+"/"elapsed_ns":"ELAPSED_NS"/g; s/"elapsed":[0-9]+\.[0-9]+/"elapsed":ELAPSED/g' +$CLICKHOUSE_CLIENT -q "SELECT * FROM test_table FORMAT JSONCompactWithProgress settings max_block_size=2;" | grep -v --text "progress" | sed -E 's/"elapsed":[0-9]+\.[0-9]+/"elapsed":ELAPSED/g' $CLICKHOUSE_CLIENT -q "SELECT 2;" # Check Totals -$CLICKHOUSE_CLIENT -q "SELECT name, count(), sleep(0.1) AS c FROM test_table GROUP BY name WITH TOTALS ORDER BY name FORMAT JSONCompactWithProgress settings max_block_size=2;" | sed -E 's/"elapsed_ns":"[0-9]+"/"elapsed_ns":"ELAPSED_NS"/g; s/"elapsed":[0-9]+\.[0-9]+/"elapsed":ELAPSED/g' +$CLICKHOUSE_CLIENT -q "SELECT name, count() AS c FROM test_table GROUP BY name WITH TOTALS ORDER BY name FORMAT JSONCompactWithProgress settings max_block_size=2;" | grep -v --text "progress" | sed -E 's/"elapsed":[0-9]+\.[0-9]+/"elapsed":ELAPSED/g' $CLICKHOUSE_CLIENT -q "DROP TABLE IF EXISTS test_table;" From 63ccbcbdfc7aa012e1879b11f9f6f4a23f9a1a0c Mon Sep 17 00:00:00 2001 From: Alexey Korepanov Date: Thu, 25 Jul 2024 13:22:47 +0100 Subject: [PATCH 20/27] Remove test tags --- tests/queries/0_stateless/03174_json_compact_with_progress.sh | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/queries/0_stateless/03174_json_compact_with_progress.sh b/tests/queries/0_stateless/03174_json_compact_with_progress.sh index b440dbe2891..8f524cdff76 100755 --- a/tests/queries/0_stateless/03174_json_compact_with_progress.sh +++ b/tests/queries/0_stateless/03174_json_compact_with_progress.sh @@ -1,5 +1,4 @@ #!/usr/bin/env bash -# Tags: no-fasttest, no-parallel CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) # shellcheck source=../shell_config.sh From 83fa51f400fba43cd4bb961419f7dd572ec240a0 Mon Sep 17 00:00:00 2001 From: Alexey Korepanov Date: Fri, 23 Aug 2024 21:42:54 +0200 Subject: [PATCH 21/27] PR fixes: update documentation on JSONCompactWithProgress format --- docs/en/interfaces/formats.md | 38 ++++++++++++++++++++++++++++++++++- 1 file changed, 37 insertions(+), 1 deletion(-) diff --git a/docs/en/interfaces/formats.md b/docs/en/interfaces/formats.md index 8795b71e0ac..f2ca803453e 100644 --- a/docs/en/interfaces/formats.md +++ b/docs/en/interfaces/formats.md @@ -930,7 +930,43 @@ Columns that are not present in the block will be filled with default values (yo In this format, ClickHouse outputs each row as a separated, newline-delimited JSON Object. -Each row is either a metadata object, data object, progress information or statistics object. +Each row is either a metadata object, data object, progress information or statistics object: + +1. **Metadata Object (`meta`)** + - Describes the structure of the data rows. + - Fields: `name` (column name), `type` (data type, e.g., `UInt32`, `String`, etc.). + - Example: `{"meta": [{"name":"id", "type":"UInt32"}, {"name":"name", "type":"String"}]}` + - Appears before any data objects. + +2. **Data Object (`data`)** + - Represents a row of query results. + - Fields: An array with values corresponding to the columns defined in the metadata. + - Example: `{"data":["1", "John Doe"]}` + - Appears after the metadata object, one per row. + +3. **Progress Information Object (`progress`)** + - Provides real-time progress feedback during query execution. + - Fields: `read_rows`, `read_bytes`, `written_rows`, `written_bytes`, `total_rows_to_read`, `result_rows`, `result_bytes`, `elapsed_ns`. + - Example: `{"progress":{"read_rows":"8","read_bytes":"168"}}` + - May appear intermittently. + +4. **Statistics Object (`statistics`)** + - Summarizes query execution statistics. + - Fields: `rows`, `rows_before_limit_at_least`, `elapsed`, `rows_read`, `bytes_read`. + - Example: `{"statistics": {"rows":2, "elapsed":0.001995, "rows_read":8}}` + - Appears at the end. + +5. **Exception Object (`exception`)** + - Represents an error that occurred during query execution. + - Fields: A single text field containing the error message. + - Example: `{"exception": "Code: 395. DB::Exception: Value passed to 'throwIf' function is non-zero..."}` + - Appears when an error is encountered. + +6. **Totals Object (`totals`)** + - Provides the totals for each numeric column in the result set. + - Fields: An array with total values corresponding to the columns defined in the metadata. + - Example: `{"totals": ["", "3"]}` + - Appears at the end of the data rows, if applicable. Example: From b772140514aa47ada3556f1d24712eae734a5645 Mon Sep 17 00:00:00 2001 From: Alexey Korepanov Date: Fri, 23 Aug 2024 22:56:24 +0200 Subject: [PATCH 22/27] Fix PR comments: check exception in the test --- .../0_stateless/03174_json_compact_with_progress.reference | 2 ++ tests/queries/0_stateless/03174_json_compact_with_progress.sh | 4 ++++ 2 files changed, 6 insertions(+) diff --git a/tests/queries/0_stateless/03174_json_compact_with_progress.reference b/tests/queries/0_stateless/03174_json_compact_with_progress.reference index b45f296e5c0..fe51e484bc1 100644 --- a/tests/queries/0_stateless/03174_json_compact_with_progress.reference +++ b/tests/queries/0_stateless/03174_json_compact_with_progress.reference @@ -11,3 +11,5 @@ {"data":["c", "1"]} {"totals": ["", "3"]} {"statistics": {"rows":3, "elapsed":ELAPSED, "rows_read":3, "bytes_read":30}} +3 +Value passed to 'throwIf' function is non-zero: diff --git a/tests/queries/0_stateless/03174_json_compact_with_progress.sh b/tests/queries/0_stateless/03174_json_compact_with_progress.sh index 8f524cdff76..b15dc7cfdb2 100755 --- a/tests/queries/0_stateless/03174_json_compact_with_progress.sh +++ b/tests/queries/0_stateless/03174_json_compact_with_progress.sh @@ -16,4 +16,8 @@ $CLICKHOUSE_CLIENT -q "SELECT 2;" # Check Totals $CLICKHOUSE_CLIENT -q "SELECT name, count() AS c FROM test_table GROUP BY name WITH TOTALS ORDER BY name FORMAT JSONCompactWithProgress settings max_block_size=2;" | grep -v --text "progress" | sed -E 's/"elapsed":[0-9]+\.[0-9]+/"elapsed":ELAPSED/g' +$CLICKHOUSE_CLIENT -q "SELECT 3;" +# Check exceptions +${CLICKHOUSE_CURL} -sS "$CLICKHOUSE_URL" -d "SELECT throwIf(number = 15), 1::Int64 as a, '\"' from numbers(100) format JSONCompactWithProgress settings output_format_json_quote_64bit_integers=1, max_block_size=10" | grep "exception" | cut -c42-88 + $CLICKHOUSE_CLIENT -q "DROP TABLE IF EXISTS test_table;" From 06f0267ed142d76109b2d6d48d3c413bdaf0fc17 Mon Sep 17 00:00:00 2001 From: Alexey Korepanov Date: Sun, 25 Aug 2024 11:27:43 +0200 Subject: [PATCH 23/27] Do not show statistics if exception happens --- ...JSONCompactWithProgressRowOutputFormat.cpp | 22 +++++++++---------- 1 file changed, 10 insertions(+), 12 deletions(-) diff --git a/src/Processors/Formats/Impl/JSONCompactWithProgressRowOutputFormat.cpp b/src/Processors/Formats/Impl/JSONCompactWithProgressRowOutputFormat.cpp index 0814e5f45d6..cb3398196f4 100644 --- a/src/Processors/Formats/Impl/JSONCompactWithProgressRowOutputFormat.cpp +++ b/src/Processors/Formats/Impl/JSONCompactWithProgressRowOutputFormat.cpp @@ -117,18 +117,16 @@ void JSONCompactWithProgressRowOutputFormat::writeProgress() void JSONCompactWithProgressRowOutputFormat::finalizeImpl() { - JSONUtils::writeCompactAdditionalInfo( - row_count, - statistics.rows_before_limit, - statistics.applied_limit, - statistics.watch, - statistics.progress, - settings.write_statistics && exception_message.empty(), - *ostr); - - if (!exception_message.empty()) - { - writeCString("\n", *ostr); + if (exception_message.empty()) { + JSONUtils::writeCompactAdditionalInfo( + row_count, + statistics.rows_before_limit, + statistics.applied_limit, + statistics.watch, + statistics.progress, + settings.write_statistics && exception_message.empty(), + *ostr); + } else { JSONUtils::writeCompactObjectStart(*ostr); JSONUtils::writeException(exception_message, *ostr, settings, 0); JSONUtils::writeCompactObjectEnd(*ostr); From 58aab9b8c789090834dd1db7bb00568db7b66cac Mon Sep 17 00:00:00 2001 From: Alexey Korepanov Date: Sun, 25 Aug 2024 11:29:07 +0200 Subject: [PATCH 24/27] Remove wrong bracket in the meta section --- .../Formats/Impl/JSONCompactWithProgressRowOutputFormat.cpp | 2 +- .../0_stateless/03174_json_compact_with_progress.reference | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/Processors/Formats/Impl/JSONCompactWithProgressRowOutputFormat.cpp b/src/Processors/Formats/Impl/JSONCompactWithProgressRowOutputFormat.cpp index cb3398196f4..b5cbd6fb232 100644 --- a/src/Processors/Formats/Impl/JSONCompactWithProgressRowOutputFormat.cpp +++ b/src/Processors/Formats/Impl/JSONCompactWithProgressRowOutputFormat.cpp @@ -19,7 +19,7 @@ void JSONCompactWithProgressRowOutputFormat::writePrefix() JSONUtils::writeCompactObjectStart(*ostr); JSONUtils::writeCompactMetadata(names, types, settings, *ostr); JSONUtils::writeCompactObjectEnd(*ostr); - writeCString("}\n", *ostr); + writeCString("\n", *ostr); } void JSONCompactWithProgressRowOutputFormat::writeField(const IColumn & column, const ISerialization & serialization, size_t row_num) diff --git a/tests/queries/0_stateless/03174_json_compact_with_progress.reference b/tests/queries/0_stateless/03174_json_compact_with_progress.reference index fe51e484bc1..cdbe7cfcb3e 100644 --- a/tests/queries/0_stateless/03174_json_compact_with_progress.reference +++ b/tests/queries/0_stateless/03174_json_compact_with_progress.reference @@ -1,11 +1,11 @@ 1 -{"meta": [{"name":"value", "type":"UInt8"}, {"name":"name", "type":"String"}]}} +{"meta": [{"name":"value", "type":"UInt8"}, {"name":"name", "type":"String"}]} {"data":[1, "a"]} {"data":[2, "b"]} {"data":[3, "c"]} {"statistics": {"rows":3, "elapsed":ELAPSED, "rows_read":3, "bytes_read":33}} 2 -{"meta": [{"name":"name", "type":"String"}, {"name":"c", "type":"UInt64"}]}} +{"meta": [{"name":"name", "type":"String"}, {"name":"c", "type":"UInt64"}]} {"data":["a", "1"]} {"data":["b", "1"]} {"data":["c", "1"]} From af3f600b5cca7cadb304e0f69fc281b7015deca9 Mon Sep 17 00:00:00 2001 From: Alexey Korepanov Date: Sun, 25 Aug 2024 11:35:35 +0200 Subject: [PATCH 25/27] Fix commas in the statistics section --- src/Formats/JSONUtils.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/Formats/JSONUtils.cpp b/src/Formats/JSONUtils.cpp index 0918658cf6f..6c2c6fc3700 100644 --- a/src/Formats/JSONUtils.cpp +++ b/src/Formats/JSONUtils.cpp @@ -563,17 +563,17 @@ namespace JSONUtils writeCompactObjectStart(out, 0, "statistics"); writeTitle("rows", out, 0, ""); writeIntText(rows, out); - writeFieldCompactDelimiter(out); if (applied_limit) { + writeFieldCompactDelimiter(out); writeTitle("rows_before_limit_at_least", out, 0, ""); writeIntText(rows_before_limit, out); - writeFieldCompactDelimiter(out); } if (write_statistics) { + writeFieldCompactDelimiter(out); writeTitle("elapsed", out, 0, ""); writeText(watch.elapsedSeconds(), out); writeFieldCompactDelimiter(out); From 0e4f2b17cc75a1f09994ad583df69fd067e582f8 Mon Sep 17 00:00:00 2001 From: Alexey Korepanov Date: Sun, 25 Aug 2024 11:38:29 +0200 Subject: [PATCH 26/27] Remove unnecessary check --- .../Formats/Impl/JSONCompactWithProgressRowOutputFormat.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Processors/Formats/Impl/JSONCompactWithProgressRowOutputFormat.cpp b/src/Processors/Formats/Impl/JSONCompactWithProgressRowOutputFormat.cpp index b5cbd6fb232..d1d7355724f 100644 --- a/src/Processors/Formats/Impl/JSONCompactWithProgressRowOutputFormat.cpp +++ b/src/Processors/Formats/Impl/JSONCompactWithProgressRowOutputFormat.cpp @@ -124,7 +124,7 @@ void JSONCompactWithProgressRowOutputFormat::finalizeImpl() statistics.applied_limit, statistics.watch, statistics.progress, - settings.write_statistics && exception_message.empty(), + settings.write_statistics, *ostr); } else { JSONUtils::writeCompactObjectStart(*ostr); From 312d32aaf972e01a88fceb2fea60fce424bfc49f Mon Sep 17 00:00:00 2001 From: Alexey Korepanov Date: Sun, 25 Aug 2024 12:14:18 +0200 Subject: [PATCH 27/27] Fix formatting --- .../Impl/JSONCompactWithProgressRowOutputFormat.cpp | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/src/Processors/Formats/Impl/JSONCompactWithProgressRowOutputFormat.cpp b/src/Processors/Formats/Impl/JSONCompactWithProgressRowOutputFormat.cpp index d1d7355724f..e90864ecdf3 100644 --- a/src/Processors/Formats/Impl/JSONCompactWithProgressRowOutputFormat.cpp +++ b/src/Processors/Formats/Impl/JSONCompactWithProgressRowOutputFormat.cpp @@ -117,7 +117,8 @@ void JSONCompactWithProgressRowOutputFormat::writeProgress() void JSONCompactWithProgressRowOutputFormat::finalizeImpl() { - if (exception_message.empty()) { + if (exception_message.empty()) + { JSONUtils::writeCompactAdditionalInfo( row_count, statistics.rows_before_limit, @@ -126,7 +127,9 @@ void JSONCompactWithProgressRowOutputFormat::finalizeImpl() statistics.progress, settings.write_statistics, *ostr); - } else { + } + else + { JSONUtils::writeCompactObjectStart(*ostr); JSONUtils::writeException(exception_message, *ostr, settings, 0); JSONUtils::writeCompactObjectEnd(*ostr);