From a50c733345ee38a58f5b6f61403eb3e9eb46f2f2 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Wed, 17 Aug 2016 23:03:22 +0300 Subject: [PATCH] Make tests deterministic [#METR-22400]. --- dbms/include/DB/DataStreams/JSONRowOutputStream.h | 4 +++- dbms/include/DB/DataStreams/XMLRowOutputStream.h | 4 +++- dbms/include/DB/Interpreters/Settings.h | 4 ++++ dbms/src/DataStreams/FormatFactory.cpp | 10 +++++++--- dbms/src/DataStreams/JSONRowOutputStream.cpp | 8 +++++--- dbms/src/DataStreams/XMLRowOutputStream.cpp | 8 +++++--- .../0_stateless/00006_extremes_and_subquery_from.sql | 1 + .../00017_in_subquery_with_empty_result.sql | 4 +++- .../tests/queries/0_stateless/00037_totals_limit.sql | 4 +++- .../0_stateless/00285_not_all_data_in_totals.sql | 1 + .../00295_global_in_one_shard_rows_before_limit.sql | 1 + dbms/tests/queries/0_stateless/00307_format_xml.sql | 1 + dbms/tests/queries/0_stateless/00309_formats.sql | 1 + .../0_stateless/00313_const_totals_extremes.sh | 12 ++++++------ 14 files changed, 44 insertions(+), 19 deletions(-) diff --git a/dbms/include/DB/DataStreams/JSONRowOutputStream.h b/dbms/include/DB/DataStreams/JSONRowOutputStream.h index b2e441a741d..e1da63a80ea 100644 --- a/dbms/include/DB/DataStreams/JSONRowOutputStream.h +++ b/dbms/include/DB/DataStreams/JSONRowOutputStream.h @@ -15,7 +15,8 @@ namespace DB class JSONRowOutputStream : public IRowOutputStream { public: - JSONRowOutputStream(WriteBuffer & ostr_, const Block & sample_); + JSONRowOutputStream(WriteBuffer & ostr_, const Block & sample_, + bool write_statistics_); void writeField(const IColumn & column, const IDataType & type, size_t row_num) override; void writeFieldDelimiter() override; @@ -66,6 +67,7 @@ protected: Progress progress; Stopwatch watch; + bool write_statistics; }; } diff --git a/dbms/include/DB/DataStreams/XMLRowOutputStream.h b/dbms/include/DB/DataStreams/XMLRowOutputStream.h index fc76eb9f899..942c68820ad 100644 --- a/dbms/include/DB/DataStreams/XMLRowOutputStream.h +++ b/dbms/include/DB/DataStreams/XMLRowOutputStream.h @@ -15,7 +15,8 @@ namespace DB class XMLRowOutputStream : public IRowOutputStream { public: - XMLRowOutputStream(WriteBuffer & ostr_, const Block & sample_); + XMLRowOutputStream(WriteBuffer & ostr_, const Block & sample_, + bool write_statistics_); void writeField(const IColumn & column, const IDataType & type, size_t row_num) override; void writeRowStartDelimiter() override; @@ -66,6 +67,7 @@ protected: Progress progress; Stopwatch watch; + bool write_statistics; }; } diff --git a/dbms/include/DB/Interpreters/Settings.h b/dbms/include/DB/Interpreters/Settings.h index 17cd528d19a..ea75a5492d9 100644 --- a/dbms/include/DB/Interpreters/Settings.h +++ b/dbms/include/DB/Interpreters/Settings.h @@ -217,6 +217,10 @@ struct Settings \ /** What aggregate function to use for implementation of count(DISTINCT ...) */ \ M(SettingString, count_distinct_implementation, "uniq") \ + \ + /** Write statistics about read rows, bytes, time elapsed in suitable output formats */ \ + M(SettingBool, output_format_write_statistics, true) \ + /// Всевозможные ограничения на выполнение запроса. Limits limits; diff --git a/dbms/src/DataStreams/FormatFactory.cpp b/dbms/src/DataStreams/FormatFactory.cpp index 28ff4c199d1..e7a8bdfc1c5 100644 --- a/dbms/src/DataStreams/FormatFactory.cpp +++ b/dbms/src/DataStreams/FormatFactory.cpp @@ -1,3 +1,4 @@ +#include #include #include #include @@ -127,13 +128,16 @@ static BlockOutputStreamPtr getOutputImpl(const String & name, WriteBuffer & buf else if (name == "Values") return std::make_shared(std::make_shared(buf)); else if (name == "JSON") - return std::make_shared(std::make_shared(buf, sample)); + return std::make_shared(std::make_shared(buf, sample, + context.getSettingsRef().output_format_write_statistics)); else if (name == "JSONCompact") - return std::make_shared(std::make_shared(buf, sample)); + return std::make_shared(std::make_shared(buf, sample, + context.getSettingsRef().output_format_write_statistics)); else if (name == "JSONEachRow") return std::make_shared(std::make_shared(buf, sample)); else if (name == "XML") - return std::make_shared(std::make_shared(buf, sample)); + return std::make_shared(std::make_shared(buf, sample, + context.getSettingsRef().output_format_write_statistics)); else if (name == "TSKV") return std::make_shared(std::make_shared(buf, sample)); else if (name == "ODBCDriver") diff --git a/dbms/src/DataStreams/JSONRowOutputStream.cpp b/dbms/src/DataStreams/JSONRowOutputStream.cpp index 4d226714f2e..f319978d605 100644 --- a/dbms/src/DataStreams/JSONRowOutputStream.cpp +++ b/dbms/src/DataStreams/JSONRowOutputStream.cpp @@ -7,8 +7,8 @@ namespace DB { -JSONRowOutputStream::JSONRowOutputStream(WriteBuffer & ostr_, const Block & sample_) - : dst_ostr(ostr_) +JSONRowOutputStream::JSONRowOutputStream(WriteBuffer & ostr_, const Block & sample_, bool write_statistics_) + : dst_ostr(ostr_), write_statistics(write_statistics_) { NamesAndTypesList columns(sample_.getColumnsList()); fields.assign(columns.begin(), columns.end()); @@ -113,7 +113,9 @@ void JSONRowOutputStream::writeSuffix() writeIntText(row_count, *ostr); writeRowsBeforeLimitAtLeast(); - writeStatistics(); + + if (write_statistics) + writeStatistics(); writeChar('\n', *ostr); writeCString("}\n", *ostr); diff --git a/dbms/src/DataStreams/XMLRowOutputStream.cpp b/dbms/src/DataStreams/XMLRowOutputStream.cpp index 0082ba1b0b4..bfa022aa544 100644 --- a/dbms/src/DataStreams/XMLRowOutputStream.cpp +++ b/dbms/src/DataStreams/XMLRowOutputStream.cpp @@ -6,8 +6,8 @@ namespace DB { -XMLRowOutputStream::XMLRowOutputStream(WriteBuffer & ostr_, const Block & sample_) - : dst_ostr(ostr_) +XMLRowOutputStream::XMLRowOutputStream(WriteBuffer & ostr_, const Block & sample_, bool write_statistics_) + : dst_ostr(ostr_), write_statistics(write_statistics_) { NamesAndTypesList columns(sample_.getColumnsList()); fields.assign(columns.begin(), columns.end()); @@ -119,7 +119,9 @@ void XMLRowOutputStream::writeSuffix() writeCString("\n", *ostr); writeRowsBeforeLimitAtLeast(); - writeStatistics(); + + if (write_statistics) + writeStatistics(); writeCString("\n", *ostr); ostr->next(); diff --git a/dbms/tests/queries/0_stateless/00006_extremes_and_subquery_from.sql b/dbms/tests/queries/0_stateless/00006_extremes_and_subquery_from.sql index 99bc0c63c22..21f26a5f5a2 100644 --- a/dbms/tests/queries/0_stateless/00006_extremes_and_subquery_from.sql +++ b/dbms/tests/queries/0_stateless/00006_extremes_and_subquery_from.sql @@ -1,3 +1,4 @@ +SET output_format_write_statistics = 0; SET extremes = 1; SELECT 'Hello, world' FROM (SELECT number FROM system.numbers LIMIT 10) WHERE number < 0 FORMAT JSONCompact; diff --git a/dbms/tests/queries/0_stateless/00017_in_subquery_with_empty_result.sql b/dbms/tests/queries/0_stateless/00017_in_subquery_with_empty_result.sql index b83d597159d..3f0bb10f6a1 100644 --- a/dbms/tests/queries/0_stateless/00017_in_subquery_with_empty_result.sql +++ b/dbms/tests/queries/0_stateless/00017_in_subquery_with_empty_result.sql @@ -1,2 +1,4 @@ +SET output_format_write_statistics = 0; + SELECT count() FROM (SELECT * FROM system.numbers LIMIT 1000) WHERE 1 IN (SELECT 0 WHERE 0) -FORMAT JSON +FORMAT JSON; diff --git a/dbms/tests/queries/0_stateless/00037_totals_limit.sql b/dbms/tests/queries/0_stateless/00037_totals_limit.sql index 25a3c0075ec..e6ddd3fa9e3 100644 --- a/dbms/tests/queries/0_stateless/00037_totals_limit.sql +++ b/dbms/tests/queries/0_stateless/00037_totals_limit.sql @@ -1 +1,3 @@ -SELECT count(), arrayJoin([1, 2, 3]) AS n GROUP BY n WITH TOTALS ORDER BY n LIMIT 1 FORMAT JSON +SET output_format_write_statistics = 0; + +SELECT count(), arrayJoin([1, 2, 3]) AS n GROUP BY n WITH TOTALS ORDER BY n LIMIT 1 FORMAT JSON; diff --git a/dbms/tests/queries/0_stateless/00285_not_all_data_in_totals.sql b/dbms/tests/queries/0_stateless/00285_not_all_data_in_totals.sql index 26d80546c51..ce7599e825d 100644 --- a/dbms/tests/queries/0_stateless/00285_not_all_data_in_totals.sql +++ b/dbms/tests/queries/0_stateless/00285_not_all_data_in_totals.sql @@ -1,3 +1,4 @@ +SET output_format_write_statistics = 0; SET group_by_two_level_threshold = 1; SELECT ignore(x), count() FROM (SELECT number AS x FROM system.numbers LIMIT 1000 UNION ALL SELECT number AS x FROM system.numbers LIMIT 1000) GROUP BY x WITH TOTALS LIMIT 10 FORMAT JSONCompact; SELECT ignore(x), count() FROM (SELECT number AS x FROM system.numbers LIMIT 1000 UNION ALL SELECT number AS x FROM system.numbers LIMIT 1000) GROUP BY x WITH TOTALS ORDER BY x LIMIT 10 FORMAT JSONCompact; diff --git a/dbms/tests/queries/0_stateless/00295_global_in_one_shard_rows_before_limit.sql b/dbms/tests/queries/0_stateless/00295_global_in_one_shard_rows_before_limit.sql index ebb69846e50..3556e8d74ec 100644 --- a/dbms/tests/queries/0_stateless/00295_global_in_one_shard_rows_before_limit.sql +++ b/dbms/tests/queries/0_stateless/00295_global_in_one_shard_rows_before_limit.sql @@ -1,2 +1,3 @@ +SET output_format_write_statistics = 0; SELECT arrayJoin(range(100)) AS x FROM remote('127.0.0.1', system.one) WHERE x GLOBAL IN (SELECT toUInt8(arrayJoin(range(100)) + 50)) GROUP BY x ORDER BY x LIMIT 10 FORMAT JSONCompact; SELECT arrayJoin(range(100)) AS x FROM remote('127.0.0.{1,2}', system.one) WHERE x GLOBAL IN (SELECT toUInt8(arrayJoin(range(100)) + 50)) GROUP BY x ORDER BY x LIMIT 10 FORMAT JSONCompact; diff --git a/dbms/tests/queries/0_stateless/00307_format_xml.sql b/dbms/tests/queries/0_stateless/00307_format_xml.sql index 3c8df788b3e..7fdca83b69f 100644 --- a/dbms/tests/queries/0_stateless/00307_format_xml.sql +++ b/dbms/tests/queries/0_stateless/00307_format_xml.sql @@ -1 +1,2 @@ +SET output_format_write_statistics = 0; SELECT 'Hello & world' AS s, 'Hello\n', toDateTime('2001-02-03 04:05:06') AS time, arrayMap(x -> toString(x), range(10)) AS arr, (s, time) AS tpl SETTINGS extremes = 1 FORMAT XML; diff --git a/dbms/tests/queries/0_stateless/00309_formats.sql b/dbms/tests/queries/0_stateless/00309_formats.sql index f6671808fb6..f6669291a6e 100644 --- a/dbms/tests/queries/0_stateless/00309_formats.sql +++ b/dbms/tests/queries/0_stateless/00309_formats.sql @@ -1,3 +1,4 @@ +SET output_format_write_statistics = 0; SELECT number * 246 + 10 AS n, toDate('2000-01-01') + n AS d, toDateTime(d) + n AS t, range(n) AS arr, arrayStringConcat(arrayMap(x -> reinterpretAsString(x), arr)) AS s, (n, d, t) AS tuple FROM system.numbers LIMIT 2 FORMAT RowBinary; SELECT number * 246 + 10 AS n, toDate('2000-01-01') + n AS d, toDateTime(d) + n AS t, range(n) AS arr, arrayStringConcat(arrayMap(x -> reinterpretAsString(x), arr)) AS s, (n, d, t) AS tuple FROM system.numbers LIMIT 2 FORMAT TabSeparatedWithNamesAndTypes; SELECT number * 246 + 10 AS n, toDate('2000-01-01') + n AS d, toDateTime(d) + n AS t, range(n) AS arr, arrayStringConcat(arrayMap(x -> reinterpretAsString(x), arr)) AS s, (n, d, t) AS tuple FROM system.numbers LIMIT 2 FORMAT TabSeparatedRaw; diff --git a/dbms/tests/queries/0_stateless/00313_const_totals_extremes.sh b/dbms/tests/queries/0_stateless/00313_const_totals_extremes.sh index d7864312727..a0a3c0dfd37 100755 --- a/dbms/tests/queries/0_stateless/00313_const_totals_extremes.sh +++ b/dbms/tests/queries/0_stateless/00313_const_totals_extremes.sh @@ -1,9 +1,9 @@ #!/bin/bash -curl -sS 'http://localhost:8123/?extremes=1' -d "SELECT 1 AS k, count() GROUP BY k WITH TOTALS"; -curl -sS 'http://localhost:8123/?extremes=1' -d "SELECT 1234567890123 AS k, count() GROUP BY k WITH TOTALS FORMAT JSON"; -curl -sS 'http://localhost:8123/?extremes=1' -d "SELECT toFloat32(1.23) AS k, count() GROUP BY k WITH TOTALS FORMAT JSONCompact"; +curl -sS 'http://localhost:8123/?extremes=1&output_format_write_statistics=0' -d "SELECT 1 AS k, count() GROUP BY k WITH TOTALS"; +curl -sS 'http://localhost:8123/?extremes=1&output_format_write_statistics=0' -d "SELECT 1234567890123 AS k, count() GROUP BY k WITH TOTALS FORMAT JSON"; +curl -sS 'http://localhost:8123/?extremes=1&output_format_write_statistics=0' -d "SELECT toFloat32(1.23) AS k, count() GROUP BY k WITH TOTALS FORMAT JSONCompact"; -curl -sS 'http://localhost:8123/?extremes=1' -d "SELECT toDate('2010-01-01') AS k, count() GROUP BY k WITH TOTALS"; -curl -sS 'http://localhost:8123/?extremes=1' -d "SELECT toDateTime('2010-01-01 01:02:03') AS k, count() GROUP BY k WITH TOTALS FORMAT JSON"; -curl -sS 'http://localhost:8123/?extremes=1' -d "SELECT 1.1 AS k, count() GROUP BY k WITH TOTALS FORMAT JSONCompact"; +curl -sS 'http://localhost:8123/?extremes=1&output_format_write_statistics=0' -d "SELECT toDate('2010-01-01') AS k, count() GROUP BY k WITH TOTALS"; +curl -sS 'http://localhost:8123/?extremes=1&output_format_write_statistics=0' -d "SELECT toDateTime('2010-01-01 01:02:03') AS k, count() GROUP BY k WITH TOTALS FORMAT JSON"; +curl -sS 'http://localhost:8123/?extremes=1&output_format_write_statistics=0' -d "SELECT 1.1 AS k, count() GROUP BY k WITH TOTALS FORMAT JSONCompact";