Merge pull request #44876 from Avogar/fix-perf-tests

Revert some changes from #42777 to fix performance tests

src/Processors/Formats/Impl/CSVRowOutputFormat.cpp

@@ -35,16 +35,10 @@ void CSVRowOutputFormat::writePrefix()
     const auto & sample = getPort(PortKind::Main).getHeader();
 
     if (with_names)
-    {
         writeLine(sample.getNames());
-        writeRowBetweenDelimiter();
-    }
 
     if (with_types)
-    {
         writeLine(sample.getDataTypeNames());
-        writeRowBetweenDelimiter();
-    }
 }
 
 
@@ -60,38 +54,21 @@ void CSVRowOutputFormat::writeFieldDelimiter()
 }
 
 
-void CSVRowOutputFormat::writeRowBetweenDelimiter()
+void CSVRowOutputFormat::writeRowEndDelimiter()
 {
     if (format_settings.csv.crlf_end_of_line)
         writeChar('\r', out);
     writeChar('\n', out);
 }
 
-void CSVRowOutputFormat::writeSuffix()
-{
-    /// Write '\n' after data if we had any data.
-    if (haveWrittenData())
-        writeRowBetweenDelimiter();
-}
-
 void CSVRowOutputFormat::writeBeforeTotals()
 {
-    writeRowBetweenDelimiter();
+    writeChar('\n', out);
 }
 
 void CSVRowOutputFormat::writeBeforeExtremes()
 {
-    writeRowBetweenDelimiter();
-}
-
-void CSVRowOutputFormat::writeAfterTotals()
-{
-    writeRowBetweenDelimiter();
-}
-
-void CSVRowOutputFormat::writeAfterExtremes()
-{
-    writeRowBetweenDelimiter();
+    writeChar('\n', out);
 }
 
 

src/Processors/Formats/Impl/CSVRowOutputFormat.h

@@ -33,18 +33,15 @@ public:
 private:
     void writeField(const IColumn & column, const ISerialization & serialization, size_t row_num) override;
     void writeFieldDelimiter() override;
-    void writeRowBetweenDelimiter() override;
+    void writeRowEndDelimiter() override;
 
     bool supportTotals() const override { return true; }
     bool supportExtremes() const override { return true; }
 
     void writeBeforeTotals() override;
-    void writeAfterTotals() override;
     void writeBeforeExtremes() override;
-    void writeAfterExtremes() override;
 
     void writePrefix() override;
-    void writeSuffix() override;
     void writeLine(const std::vector<String> & values);
 
     bool with_names;

src/Processors/Formats/Impl/JSONCompactEachRowRowOutputFormat.cpp

@@ -52,17 +52,12 @@ void JSONCompactEachRowRowOutputFormat::writeRowStartDelimiter()
 
 void JSONCompactEachRowRowOutputFormat::writeRowEndDelimiter()
 {
-    writeChar(']', *ostr);
-}
-
-void JSONCompactEachRowRowOutputFormat::writeRowBetweenDelimiter()
-{
-    writeChar('\n', *ostr);
+    writeCString("]\n", *ostr);
 }
 
 void JSONCompactEachRowRowOutputFormat::writeTotals(const Columns & columns, size_t row_num)
 {
-    writeRowBetweenDelimiter();
+    writeChar('\n', *ostr);
     size_t columns_size = columns.size();
     writeRowStartDelimiter();
     for (size_t i = 0; i < columns_size; ++i)
@@ -73,7 +68,6 @@ void JSONCompactEachRowRowOutputFormat::writeTotals(const Columns & columns, siz
         writeField(*columns[i], *serializations[i], row_num);
     }
     writeRowEndDelimiter();
-    writeRowBetweenDelimiter();
 }
 
 void JSONCompactEachRowRowOutputFormat::writeLine(const std::vector<String> & values)
@@ -96,22 +90,10 @@ void JSONCompactEachRowRowOutputFormat::writePrefix()
     const auto & header = getPort(PortKind::Main).getHeader();
 
     if (with_names)
-    {
         writeLine(JSONUtils::makeNamesValidJSONStrings(header.getNames(), settings, settings.json.validate_utf8));
-        writeRowBetweenDelimiter();
-    }
 
     if (with_types)
-    {
         writeLine(JSONUtils::makeNamesValidJSONStrings(header.getDataTypeNames(), settings, settings.json.validate_utf8));
-        writeRowBetweenDelimiter();
-    }
-}
-
-void JSONCompactEachRowRowOutputFormat::writeSuffix()
-{
-    if (haveWrittenData())
-        writeChar('\n', *ostr);
 }
 
 void JSONCompactEachRowRowOutputFormat::consumeTotals(DB::Chunk chunk)

src/Processors/Formats/Impl/JSONCompactEachRowRowOutputFormat.h

@@ -26,7 +26,6 @@ public:
 
 private:
     void writePrefix() override;
-    void writeSuffix() override;
 
     void writeTotals(const Columns & columns, size_t row_num) override;
 
@@ -34,7 +33,6 @@ private:
     void writeFieldDelimiter() override;
     void writeRowStartDelimiter() override;
     void writeRowEndDelimiter() override;
-    void writeRowBetweenDelimiter() override;
 
     bool supportTotals() const override { return true; }
     void consumeTotals(Chunk) override;

src/Processors/Formats/Impl/JSONEachRowRowOutputFormat.cpp

@@ -41,7 +41,10 @@ void JSONEachRowRowOutputFormat::writeRowStartDelimiter()
 
 void JSONEachRowRowOutputFormat::writeRowEndDelimiter()
 {
-    writeCString("}", *ostr);
+    if (settings.json.array_of_rows)
+        writeChar('}', *ostr);
+    else
+        writeCString("}\n", *ostr);
     field_number = 0;
 }
 
@@ -49,9 +52,7 @@ void JSONEachRowRowOutputFormat::writeRowEndDelimiter()
 void JSONEachRowRowOutputFormat::writeRowBetweenDelimiter()
 {
     if (settings.json.array_of_rows)
-        writeChar(',', *ostr);
-
-    writeChar('\n', *ostr);
+        writeCString(",\n", *ostr);
 }
 
 
@@ -68,8 +69,6 @@ void JSONEachRowRowOutputFormat::writeSuffix()
 {
     if (settings.json.array_of_rows)
         writeCString("\n]\n", *ostr);
-    else if (haveWrittenData())
-        writeChar('\n', *ostr);
 }
 
 

src/Processors/Formats/Impl/JSONEachRowWithProgressRowOutputFormat.cpp

@@ -10,16 +10,13 @@ namespace DB
 void JSONEachRowWithProgressRowOutputFormat::writeRowStartDelimiter()
 {
     if (has_progress)
-    {
         writeProgress();
-        writeRowBetweenDelimiter();
-    }
     writeCString("{\"row\":{", *ostr);
 }
 
 void JSONEachRowWithProgressRowOutputFormat::writeRowEndDelimiter()
 {
-    writeCString("}}", *ostr);
+    writeCString("}}\n", *ostr);
     field_number = 0;
 }
 
@@ -30,7 +27,7 @@ void JSONEachRowWithProgressRowOutputFormat::onProgress(const Progress & value)
     WriteBufferFromString buf(progress_line);
     writeCString("{\"progress\":", buf);
     progress.writeJSON(buf);
-    writeCString("}", buf);
+    writeCString("}\n", buf);
     buf.finalize();
     std::lock_guard lock(progress_lines_mutex);
     progress_lines.emplace_back(std::move(progress_line));
@@ -40,33 +37,22 @@ void JSONEachRowWithProgressRowOutputFormat::onProgress(const Progress & value)
 void JSONEachRowWithProgressRowOutputFormat::flush()
 {
     if (has_progress)
-    {
-        if (haveWrittenData())
-            writeRowBetweenDelimiter();
         writeProgress();
-    }
     JSONEachRowRowOutputFormat::flush();
 }
 
 void JSONEachRowWithProgressRowOutputFormat::writeSuffix()
 {
     if (has_progress)
-    {
-        writeRowBetweenDelimiter();
         writeProgress();
-    }
     JSONEachRowRowOutputFormat::writeSuffix();
 }
 
 void JSONEachRowWithProgressRowOutputFormat::writeProgress()
 {
     std::lock_guard lock(progress_lines_mutex);
-    for (size_t i = 0; i != progress_lines.size(); ++i)
-    {
-        if (i != 0)
-            writeRowBetweenDelimiter();
-        writeString(progress_lines[i], *ostr);
-    }
+    for (const auto & progress_line : progress_lines)
+        writeString(progress_line,  *ostr);
     progress_lines.clear();
     has_progress = false;
 }

src/Processors/Formats/Impl/MarkdownRowOutputFormat.cpp

@@ -47,18 +47,7 @@ void MarkdownRowOutputFormat::writeFieldDelimiter()
 
 void MarkdownRowOutputFormat::writeRowEndDelimiter()
 {
-    writeCString(" |", out);
-}
-
-void MarkdownRowOutputFormat::writeRowBetweenDelimiter()
-{
-    writeChar('\n', out);
-}
-
-void MarkdownRowOutputFormat::writeSuffix()
-{
-    if (haveWrittenData())
-        writeChar('\n', out);
+    writeCString(" |\n", out);
 }
 
 void MarkdownRowOutputFormat::writeField(const IColumn & column, const ISerialization & serialization, size_t row_num)

src/Processors/Formats/Impl/MarkdownRowOutputFormat.h

@@ -21,7 +21,6 @@ private:
     /// |columnName1|columnName2|...|columnNameN|
     /// |:-:|:-:|...|:-:|
     void writePrefix() override;
-    void writeSuffix() override;
 
     /// Write '|' before each row
     void writeRowStartDelimiter() override;
@@ -29,12 +28,9 @@ private:
     /// Write '|' between values
     void writeFieldDelimiter() override;
 
-    /// Write '|' at the end of each row
+    /// Write '|\n' at the end of each row
     void writeRowEndDelimiter() override;
 
-    /// Write '\n' after each row
-    void writeRowBetweenDelimiter() override;
-
     void writeField(const IColumn & column, const ISerialization & serialization, size_t row_num) override;
 
     const FormatSettings format_settings;

src/Processors/Formats/Impl/SQLInsertRowOutputFormat.cpp

@@ -65,27 +65,24 @@ void SQLInsertRowOutputFormat::writeRowEndDelimiter()
 {
     writeChar(')', out);
     ++rows_in_line;
-    if (rows_in_line >= format_settings.sql_insert.max_batch_size)
-    {
-        writeChar(';', out);
-        rows_in_line = 0;
-    }
 }
 
 void SQLInsertRowOutputFormat::writeRowBetweenDelimiter()
 {
-    if (rows_in_line == 0)
-        writeChar('\n', out);
+    if (rows_in_line >= format_settings.sql_insert.max_batch_size)
+    {
+        writeCString(";\n", out);
+        rows_in_line = 0;
+    }
     else
+    {
         writeCString(", ", out);
+    }
 }
 
 void SQLInsertRowOutputFormat::writeSuffix()
 {
-    if (rows_in_line != 0)
-        writeChar(';', out);
-    if (haveWrittenData())
-        writeChar('\n', out);
+    writeCString(";\n", out);
 }
 
 void SQLInsertRowOutputFormat::resetFormatterImpl()

src/Processors/Formats/Impl/TSKVRowOutputFormat.cpp

@@ -29,14 +29,9 @@ void TSKVRowOutputFormat::writeField(const IColumn & column, const ISerializatio
 
 
 void TSKVRowOutputFormat::writeRowEndDelimiter()
-{
-    field_number = 0;
-}
-
-
-void TSKVRowOutputFormat::writeRowBetweenDelimiter()
 {
     writeChar('\n', out);
+    field_number = 0;
 }
 
 

src/Processors/Formats/Impl/TSKVRowOutputFormat.h

@@ -21,7 +21,6 @@ public:
 private:
     void writeField(const IColumn & column, const ISerialization & serialization, size_t row_num) override;
     void writeRowEndDelimiter() override;
-    void writeRowBetweenDelimiter() override;
 
     /// Disable totals and extremes, because they are enabled in TSV.
     bool supportTotals() const override { return false; }

src/Processors/Formats/Impl/TabSeparatedRowOutputFormat.cpp

@@ -36,16 +36,10 @@ void TabSeparatedRowOutputFormat::writePrefix()
     const auto & header = getPort(PortKind::Main).getHeader();
 
     if (with_names)
-    {
         writeLine(header.getNames());
-        writeRowBetweenDelimiter();
-    }
 
     if (with_types)
-    {
         writeLine(header.getDataTypeNames());
-        writeRowBetweenDelimiter();
-    }
 }
 
 
@@ -64,38 +58,21 @@ void TabSeparatedRowOutputFormat::writeFieldDelimiter()
 }
 
 
-void TabSeparatedRowOutputFormat::writeRowBetweenDelimiter()
+void TabSeparatedRowOutputFormat::writeRowEndDelimiter()
 {
     if (format_settings.tsv.crlf_end_of_line)
         writeChar('\r', out);
     writeChar('\n', out);
 }
 
-void TabSeparatedRowOutputFormat::writeSuffix()
-{
-    /// Output '\n' an the end of data if we had any data.
-    if (haveWrittenData())
-        writeRowBetweenDelimiter();
-}
-
 void TabSeparatedRowOutputFormat::writeBeforeTotals()
 {
-    writeRowBetweenDelimiter();
+    writeChar('\n', out);
 }
 
 void TabSeparatedRowOutputFormat::writeBeforeExtremes()
 {
-    writeRowBetweenDelimiter();
-}
-
-void TabSeparatedRowOutputFormat::writeAfterTotals()
-{
-    writeRowBetweenDelimiter();
-}
-
-void TabSeparatedRowOutputFormat::writeAfterExtremes()
-{
-    writeRowBetweenDelimiter();
+    writeChar('\n', out);
 }
 
 void registerOutputFormatTabSeparated(FormatFactory & factory)

src/Processors/Formats/Impl/TabSeparatedRowOutputFormat.h

@@ -35,18 +35,15 @@ public:
 protected:
     void writeField(const IColumn & column, const ISerialization & serialization, size_t row_num) override;
     void writeFieldDelimiter() override final;
-    void writeRowBetweenDelimiter() override;
+    void writeRowEndDelimiter() override;
 
     bool supportTotals() const override { return true; }
     bool supportExtremes() const override { return true; }
 
     void writeBeforeTotals() override final;
-    void writeAfterTotals() override final;
     void writeBeforeExtremes() override final;
-    void writeAfterExtremes() override final;
 
     void writePrefix() override;
-    void writeSuffix() override;
     void writeLine(const std::vector<String> & values);
 
     bool with_names;

src/Processors/Formats/Impl/XMLRowOutputFormat.cpp

@@ -89,20 +89,14 @@ void XMLRowOutputFormat::writeRowStartDelimiter()
 
 void XMLRowOutputFormat::writeRowEndDelimiter()
 {
-    writeCString("\t\t</row>", *ostr);
+    writeCString("\t\t</row>\n", *ostr);
     field_number = 0;
     ++row_count;
 }
 
-void XMLRowOutputFormat::writeRowBetweenDelimiter()
-{
-    writeChar('\n', *ostr);
-}
-
-
 void XMLRowOutputFormat::writeSuffix()
 {
-    writeCString("\n\t</data>\n", *ostr);
+    writeCString("\t</data>\n", *ostr);
 }
 
 
@@ -152,7 +146,7 @@ void XMLRowOutputFormat::writeMaxExtreme(const Columns & columns, size_t row_num
 
 void XMLRowOutputFormat::writeAfterExtremes()
 {
-    writeCString("\n\t</extremes>\n", *ostr);
+    writeCString("\t</extremes>\n", *ostr);
 }
 
 void XMLRowOutputFormat::writeExtremesElement(const char * title, const Columns & columns, size_t row_num)
@@ -179,7 +173,7 @@ void XMLRowOutputFormat::writeExtremesElement(const char * title, const Columns
 
     writeCString("\t\t</", *ostr);
     writeCString(title, *ostr);
-    writeCString(">", *ostr);
+    writeCString(">\n", *ostr);
 }
 
 

src/Processors/Formats/Impl/XMLRowOutputFormat.h

@@ -24,7 +24,6 @@ private:
     void writeField(const IColumn & column, const ISerialization & serialization, size_t row_num) override;
     void writeRowStartDelimiter() override;
     void writeRowEndDelimiter() override;
-    void writeRowBetweenDelimiter() override;
     void writePrefix() override;
     void writeSuffix() override;
     void finalizeImpl() override;

tests/queries/0_stateless/00966_live_view_watch_events_http.py

@@ -38,10 +38,10 @@ with client(name="client1>", log=log) as client1:
             name="client2>",
             log=log,
         ) as client2:
-            client2.expect(".*1")
+            client2.expect(".*1\n")
             client1.send("INSERT INTO test.mt VALUES (1),(2),(3)")
             client1.expect(prompt)
-            client2.expect(".*2")
+            client2.expect(".*2\n")
     finally:
         client1.send("DROP TABLE test.lv")
         client1.expect(prompt)

tests/queries/0_stateless/00967_live_view_watch_http.py

@@ -38,10 +38,10 @@ with client(name="client1>", log=log) as client1:
             name="client2>",
             log=log,
         ) as client2:
-            client2.expect(".*0\t1")
+            client2.expect(".*0\t1\n")
             client1.send("INSERT INTO test.mt VALUES (1),(2),(3)")
             client1.expect(prompt)
-            client2.expect(".*6\t2")
+            client2.expect(".*6\t2\n")
     finally:
         client1.send("DROP TABLE test.lv")
         client1.expect(prompt)

tests/queries/0_stateless/01246_insert_into_watch_live_view.py

@@ -53,15 +53,15 @@ with client(name="client1>", log=log) as client1, client(
     client1.send("INSERT INTO test.sums WATCH test.lv")
     client1.expect(r"INSERT INTO")
 
-    client3.expect("0,1.*")
+    client3.expect("0,1.*\n")
 
     client2.send("INSERT INTO test.mt VALUES (1),(2),(3)")
     client2.expect(prompt)
-    client3.expect("6,2.*")
+    client3.expect("6,2.*\n")
 
     client2.send("INSERT INTO test.mt VALUES (4),(5),(6)")
     client2.expect(prompt)
-    client3.expect("21,3.*")
+    client3.expect("21,3.*\n")
 
     # send Ctrl-C
     client3.send("\x03", eol="")

tests/queries/0_stateless/01249_flush_interactive.sh

@@ -16,10 +16,10 @@ function test()
 {
     timeout 5 ${CLICKHOUSE_LOCAL} --max_execution_time 10 --query "
         SELECT DISTINCT number % 5 FROM system.numbers" ||:
-    echo -e '\n---'
+    echo -e '---'
     timeout 5 ${CLICKHOUSE_CURL} -sS --no-buffer "${CLICKHOUSE_URL}&max_execution_time=10" --data-binary "
         SELECT DISTINCT number % 5 FROM system.numbers" ||:
-    echo -e '\n---'
+    echo -e '---'
 }
 
 # The test depends on timeouts. And there is a chance that under high system load the query

tests/queries/0_stateless/01553_settings_early_apply.reference

@@ -31,7 +31,6 @@ UInt64
 		</columns>
 	</meta>
 	<data>
-
 	</data>
 	<rows>0</rows>
 </result>