From 354b689800b2313f6d444278122d22b008f0fe44 Mon Sep 17 00:00:00 2001 From: Nikita Taranov Date: Wed, 20 Dec 2023 15:57:52 +0100 Subject: [PATCH 01/23] impl --- src/Disks/IO/createReadBufferFromFileBase.cpp | 6 ++---- src/Disks/IO/createReadBufferFromFileBase.h | 3 +-- src/IO/AsynchronousReadBufferFromFile.h | 6 ++---- ...AsynchronousReadBufferFromFileDescriptor.cpp | 17 +++++------------ .../AsynchronousReadBufferFromFileDescriptor.h | 4 +--- 5 files changed, 11 insertions(+), 25 deletions(-) diff --git a/src/Disks/IO/createReadBufferFromFileBase.cpp b/src/Disks/IO/createReadBufferFromFileBase.cpp index 236dd43e9ee..a9d451496ff 100644 --- a/src/Disks/IO/createReadBufferFromFileBase.cpp +++ b/src/Disks/IO/createReadBufferFromFileBase.cpp @@ -36,8 +36,7 @@ std::unique_ptr createReadBufferFromFileBase( std::optional file_size, int flags, char * existing_memory, - size_t alignment, - bool use_external_buffer) + size_t alignment) { if (file_size.has_value() && !*file_size) return std::make_unique(); @@ -149,8 +148,7 @@ std::unique_ptr createReadBufferFromFileBase( existing_memory, buffer_alignment, file_size, - settings.local_throttler, - use_external_buffer); + settings.local_throttler); } else throw Exception(ErrorCodes::LOGICAL_ERROR, "Unknown read method"); diff --git a/src/Disks/IO/createReadBufferFromFileBase.h b/src/Disks/IO/createReadBufferFromFileBase.h index a00456eda82..e93725a967d 100644 --- a/src/Disks/IO/createReadBufferFromFileBase.h +++ b/src/Disks/IO/createReadBufferFromFileBase.h @@ -21,6 +21,5 @@ std::unique_ptr createReadBufferFromFileBase( std::optional file_size = {}, int flags_ = -1, char * existing_memory = nullptr, - size_t alignment = 0, - bool use_external_buffer = false); + size_t alignment = 0); } diff --git a/src/IO/AsynchronousReadBufferFromFile.h b/src/IO/AsynchronousReadBufferFromFile.h index 79f933cbe48..5b39b707803 100644 --- a/src/IO/AsynchronousReadBufferFromFile.h +++ b/src/IO/AsynchronousReadBufferFromFile.h @@ -67,10 +67,8 @@ public: char * existing_memory = nullptr, size_t alignment = 0, std::optional file_size_ = std::nullopt, - ThrottlerPtr throttler_ = {}, - bool use_external_buffer_ = false) - : AsynchronousReadBufferFromFileDescriptor( - reader_, priority_, -1, buf_size, existing_memory, alignment, file_size_, throttler_, use_external_buffer_) + ThrottlerPtr throttler_ = {}) + : AsynchronousReadBufferFromFileDescriptor(reader_, priority_, -1, buf_size, existing_memory, alignment, file_size_, throttler_) , file_name(file_name_) { file = OpenedFileCache::instance().get(file_name, flags); diff --git a/src/IO/AsynchronousReadBufferFromFileDescriptor.cpp b/src/IO/AsynchronousReadBufferFromFileDescriptor.cpp index a7c2c7bb7e3..f8c00d62732 100644 --- a/src/IO/AsynchronousReadBufferFromFileDescriptor.cpp +++ b/src/IO/AsynchronousReadBufferFromFileDescriptor.cpp @@ -97,11 +97,7 @@ bool AsynchronousReadBufferFromFileDescriptor::nextImpl() /// No pending request. Do synchronous read. ProfileEventTimeIncrement watch(ProfileEvents::SynchronousReadWaitMicroseconds); - if (!use_external_buffer) - result = asyncReadInto(memory.data(), memory.size(), DEFAULT_PREFETCH_PRIORITY).get(); - else - /// External buffer will be substituted in place of internal_buffer (see CachedOnDiskReadBufferFromFile) - result = asyncReadInto(internal_buffer.begin(), internal_buffer.size(), DEFAULT_PREFETCH_PRIORITY).get(); + result = asyncReadInto(memory.data(), memory.size(), DEFAULT_PREFETCH_PRIORITY).get(); } chassert(result.size >= result.offset); @@ -114,9 +110,8 @@ bool AsynchronousReadBufferFromFileDescriptor::nextImpl() if (bytes_read) { /// Adjust the working buffer so that it ignores `offset` bytes. - if (!use_external_buffer) - internal_buffer = Buffer(memory.data(), memory.data() + memory.size()); - working_buffer = Buffer(internal_buffer.begin() + result.offset, internal_buffer.begin() + result.size); + internal_buffer = Buffer(memory.data(), memory.data() + memory.size()); + working_buffer = Buffer(memory.data() + result.offset, memory.data() + result.size); pos = working_buffer.begin(); } @@ -142,15 +137,13 @@ AsynchronousReadBufferFromFileDescriptor::AsynchronousReadBufferFromFileDescript char * existing_memory, size_t alignment, std::optional file_size_, - ThrottlerPtr throttler_, - bool use_external_buffer_) + ThrottlerPtr throttler_) : ReadBufferFromFileBase(buf_size, existing_memory, alignment, file_size_) , reader(reader_) , base_priority(priority_) , required_alignment(alignment) , fd(fd_) , throttler(throttler_) - , use_external_buffer(use_external_buffer_) { if (required_alignment > buf_size) throw Exception( @@ -228,7 +221,7 @@ off_t AsynchronousReadBufferFromFileDescriptor::seek(off_t offset, int whence) file_offset_of_buffer_end = seek_pos; bytes_to_ignore = new_pos - seek_pos; - if (bytes_to_ignore >= internal_buffer.size() && !(use_external_buffer && internal_buffer.empty())) + if (bytes_to_ignore >= internal_buffer.size()) throw Exception(ErrorCodes::LOGICAL_ERROR, "Logical error in AsynchronousReadBufferFromFileDescriptor, bytes_to_ignore ({}" ") >= internal_buffer.size() ({})", bytes_to_ignore, internal_buffer.size()); diff --git a/src/IO/AsynchronousReadBufferFromFileDescriptor.h b/src/IO/AsynchronousReadBufferFromFileDescriptor.h index dcc5819b039..82659b1aca7 100644 --- a/src/IO/AsynchronousReadBufferFromFileDescriptor.h +++ b/src/IO/AsynchronousReadBufferFromFileDescriptor.h @@ -29,7 +29,6 @@ protected: size_t bytes_to_ignore = 0; /// How many bytes should we ignore upon a new read request. int fd; ThrottlerPtr throttler; - bool use_external_buffer; bool nextImpl() override; @@ -47,8 +46,7 @@ public: char * existing_memory = nullptr, size_t alignment = 0, std::optional file_size_ = std::nullopt, - ThrottlerPtr throttler_ = {}, - bool use_external_buffer_ = false); + ThrottlerPtr throttler_ = {}); ~AsynchronousReadBufferFromFileDescriptor() override; From 459946035cbe20b43d5aed00e770d2d3bb647e19 Mon Sep 17 00:00:00 2001 From: Blargian Date: Thu, 4 Jan 2024 16:10:36 +0200 Subject: [PATCH 02/23] #58363 :construction: modified Pretty in FormatSettings.h to have PrettyColor which can be 0,1 or auto. modified output_format_pretty_color in FormatFactory.cpp to make use of this, added the default to Settings.h. Implemented the logic for enabling/disabling based on output_format_pretty_color in PrettyBlockOutputFormat.h --- src/Core/Settings.h | 2 +- src/Formats/FormatFactory.cpp | 6 +++++- src/Formats/FormatSettings.h | 7 +++++++ .../Formats/Impl/PrettyBlockOutputFormat.h | 18 +++++++++++++++--- 4 files changed, 28 insertions(+), 5 deletions(-) diff --git a/src/Core/Settings.h b/src/Core/Settings.h index 988c4f357e0..4c815f848ce 100644 --- a/src/Core/Settings.h +++ b/src/Core/Settings.h @@ -1041,7 +1041,7 @@ class IColumn; M(UInt64, output_format_pretty_max_rows, 10000, "Rows limit for Pretty formats.", 0) \ M(UInt64, output_format_pretty_max_column_pad_width, 250, "Maximum width to pad all values in a column in Pretty formats.", 0) \ M(UInt64, output_format_pretty_max_value_width, 10000, "Maximum width of value to display in Pretty formats. If greater - it will be cut.", 0) \ - M(Bool, output_format_pretty_color, true, "Use ANSI escape sequences to paint colors in Pretty formats", 0) \ + M(String, output_format_pretty_color, "AUTO", "Use ANSI escape sequences to paint colors in Pretty formats", 0) \ M(String, output_format_pretty_grid_charset, "UTF-8", "Charset for printing grid borders. Available charsets: ASCII, UTF-8 (default one).", 0) \ M(UInt64, output_format_parquet_row_group_size, 1000000, "Target row group size in rows.", 0) \ M(UInt64, output_format_parquet_row_group_size_bytes, 512 * 1024 * 1024, "Target row group size in bytes, before compression.", 0) \ diff --git a/src/Formats/FormatFactory.cpp b/src/Formats/FormatFactory.cpp index 15743365d7d..1dc48e9a214 100644 --- a/src/Formats/FormatFactory.cpp +++ b/src/Formats/FormatFactory.cpp @@ -143,7 +143,11 @@ FormatSettings getFormatSettings(ContextPtr context, const Settings & settings) format_settings.parquet.write_batch_size = settings.output_format_parquet_batch_size; format_settings.parquet.local_read_min_bytes_for_seek = settings.input_format_parquet_local_file_min_bytes_for_seek; format_settings.pretty.charset = settings.output_format_pretty_grid_charset.toString() == "ASCII" ? FormatSettings::Pretty::Charset::ASCII : FormatSettings::Pretty::Charset::UTF8; - format_settings.pretty.color = settings.output_format_pretty_color; + format_settings.pretty.output_format_pretty_color = settings.output_format_pretty_color.toString() == "AUTO" + ? FormatSettings::Pretty::PrettyColor::AUTO + : settings.output_format_pretty_color.toString() == "ON" + ? FormatSettings::Pretty::PrettyColor::ON + : FormatSettings::Pretty::PrettyColor::OFF; format_settings.pretty.max_column_pad_width = settings.output_format_pretty_max_column_pad_width; format_settings.pretty.max_rows = settings.output_format_pretty_max_rows; format_settings.pretty.max_value_width = settings.output_format_pretty_max_value_width; diff --git a/src/Formats/FormatSettings.h b/src/Formats/FormatSettings.h index 8d5c044a311..7efb6628f96 100644 --- a/src/Formats/FormatSettings.h +++ b/src/Formats/FormatSettings.h @@ -271,6 +271,13 @@ struct FormatSettings UInt64 max_value_width = 10000; bool color = true; + enum class PrettyColor { + ON, + OFF, + AUTO + }; + + PrettyColor output_format_pretty_color = PrettyColor::AUTO; bool output_format_pretty_row_numbers = false; enum class Charset diff --git a/src/Processors/Formats/Impl/PrettyBlockOutputFormat.h b/src/Processors/Formats/Impl/PrettyBlockOutputFormat.h index dfb23ac63f9..13b694cbff6 100644 --- a/src/Processors/Formats/Impl/PrettyBlockOutputFormat.h +++ b/src/Processors/Formats/Impl/PrettyBlockOutputFormat.h @@ -73,10 +73,22 @@ void registerPrettyFormatWithNoEscapesAndMonoBlock(FormatFactory & factory, cons const Block & sample, const FormatSettings & format_settings) { - if (no_escapes) + FormatSettings changed_settings = format_settings; + using PrettyColor = FormatSettings::Pretty::PrettyColor; + switch(format_settings.pretty.output_format_pretty_color) + { + case PrettyColor::OFF: + changed_settings.pretty.color = false; + break; + case PrettyColor::ON: + no_escapes ? changed_settings.pretty.color = true : changed_settings.pretty.color = false; + break; + case PrettyColor::AUTO: + isatty(STDOUT_FILENO) ? changed_settings.pretty.color = true : changed_settings.pretty.color = false; + break; + } + if (format_settings.pretty.color != changed_settings.pretty.color) { - FormatSettings changed_settings = format_settings; - changed_settings.pretty.color = false; return std::make_shared(buf, sample, changed_settings, mono_block); } return std::make_shared(buf, sample, format_settings, mono_block); From a15b5733151a33d515f7335ffe3620c7e7da2c21 Mon Sep 17 00:00:00 2001 From: Blargian Date: Mon, 8 Jan 2024 15:25:14 +0200 Subject: [PATCH 03/23] #58363 - fix formatting issues and change ON, OFF, AUTO to 0, 1, auto --- src/Core/Settings.h | 2 +- src/Formats/FormatFactory.cpp | 9 +++--- src/Formats/FormatSettings.h | 14 +++++----- .../Formats/Impl/PrettyBlockOutputFormat.h | 28 +++++++++---------- 4 files changed, 26 insertions(+), 27 deletions(-) diff --git a/src/Core/Settings.h b/src/Core/Settings.h index 4c815f848ce..bc5fd7cc00f 100644 --- a/src/Core/Settings.h +++ b/src/Core/Settings.h @@ -1041,7 +1041,7 @@ class IColumn; M(UInt64, output_format_pretty_max_rows, 10000, "Rows limit for Pretty formats.", 0) \ M(UInt64, output_format_pretty_max_column_pad_width, 250, "Maximum width to pad all values in a column in Pretty formats.", 0) \ M(UInt64, output_format_pretty_max_value_width, 10000, "Maximum width of value to display in Pretty formats. If greater - it will be cut.", 0) \ - M(String, output_format_pretty_color, "AUTO", "Use ANSI escape sequences to paint colors in Pretty formats", 0) \ + M(String, output_format_pretty_color, "auto", "Use ANSI escape sequences to paint colors in Pretty formats", 0) \ M(String, output_format_pretty_grid_charset, "UTF-8", "Charset for printing grid borders. Available charsets: ASCII, UTF-8 (default one).", 0) \ M(UInt64, output_format_parquet_row_group_size, 1000000, "Target row group size in rows.", 0) \ M(UInt64, output_format_parquet_row_group_size_bytes, 512 * 1024 * 1024, "Target row group size in bytes, before compression.", 0) \ diff --git a/src/Formats/FormatFactory.cpp b/src/Formats/FormatFactory.cpp index 1dc48e9a214..4c6add2b0d5 100644 --- a/src/Formats/FormatFactory.cpp +++ b/src/Formats/FormatFactory.cpp @@ -143,11 +143,10 @@ FormatSettings getFormatSettings(ContextPtr context, const Settings & settings) format_settings.parquet.write_batch_size = settings.output_format_parquet_batch_size; format_settings.parquet.local_read_min_bytes_for_seek = settings.input_format_parquet_local_file_min_bytes_for_seek; format_settings.pretty.charset = settings.output_format_pretty_grid_charset.toString() == "ASCII" ? FormatSettings::Pretty::Charset::ASCII : FormatSettings::Pretty::Charset::UTF8; - format_settings.pretty.output_format_pretty_color = settings.output_format_pretty_color.toString() == "AUTO" - ? FormatSettings::Pretty::PrettyColor::AUTO - : settings.output_format_pretty_color.toString() == "ON" - ? FormatSettings::Pretty::PrettyColor::ON - : FormatSettings::Pretty::PrettyColor::OFF; + format_settings.pretty.output_format_pretty_color = settings.output_format_pretty_color.toString() == "auto" ? FormatSettings::Pretty::PrettyColor::auto + : settings.output_format_pretty_color.toString() == "1" + ? FormatSettings::Pretty::PrettyColor::1 + : FormatSettings::Pretty::PrettyColor::0; format_settings.pretty.max_column_pad_width = settings.output_format_pretty_max_column_pad_width; format_settings.pretty.max_rows = settings.output_format_pretty_max_rows; format_settings.pretty.max_value_width = settings.output_format_pretty_max_value_width; diff --git a/src/Formats/FormatSettings.h b/src/Formats/FormatSettings.h index 7efb6628f96..e20c14034c7 100644 --- a/src/Formats/FormatSettings.h +++ b/src/Formats/FormatSettings.h @@ -271,13 +271,13 @@ struct FormatSettings UInt64 max_value_width = 10000; bool color = true; - enum class PrettyColor { - ON, - OFF, - AUTO - }; - - PrettyColor output_format_pretty_color = PrettyColor::AUTO; + enum class PrettyColor { + 0, + 1, + auto + }; + PrettyColor output_format_pretty_color = PrettyColor::auto; + bool output_format_pretty_row_numbers = false; enum class Charset diff --git a/src/Processors/Formats/Impl/PrettyBlockOutputFormat.h b/src/Processors/Formats/Impl/PrettyBlockOutputFormat.h index 13b694cbff6..c978a2ac03b 100644 --- a/src/Processors/Formats/Impl/PrettyBlockOutputFormat.h +++ b/src/Processors/Formats/Impl/PrettyBlockOutputFormat.h @@ -73,20 +73,20 @@ void registerPrettyFormatWithNoEscapesAndMonoBlock(FormatFactory & factory, cons const Block & sample, const FormatSettings & format_settings) { - FormatSettings changed_settings = format_settings; - using PrettyColor = FormatSettings::Pretty::PrettyColor; - switch(format_settings.pretty.output_format_pretty_color) - { - case PrettyColor::OFF: - changed_settings.pretty.color = false; - break; - case PrettyColor::ON: - no_escapes ? changed_settings.pretty.color = true : changed_settings.pretty.color = false; - break; - case PrettyColor::AUTO: - isatty(STDOUT_FILENO) ? changed_settings.pretty.color = true : changed_settings.pretty.color = false; - break; - } + FormatSettings changed_settings = format_settings; + using PrettyColor = FormatSettings::Pretty::PrettyColor; + switch(format_settings.pretty.output_format_pretty_color) + { + case PrettyColor::0: + changed_settings.pretty.color = false; + break; + case PrettyColor::1: + no_escapes ? changed_settings.pretty.color = true : changed_settings.pretty.color = false; + break; + case PrettyColor::auto: + isatty(STDOUT_FILENO) ? changed_settings.pretty.color = true : changed_settings.pretty.color = false; + break; + } if (format_settings.pretty.color != changed_settings.pretty.color) { return std::make_shared(buf, sample, changed_settings, mono_block); From b65adbecc195a4f4159604c5bc3e3bdc035ab6c5 Mon Sep 17 00:00:00 2001 From: Blargian Date: Mon, 8 Jan 2024 23:52:25 +0200 Subject: [PATCH 04/23] minor fixes. Doesnt seem to be using ANSI escapes anymore --- src/Core/Settings.h | 2 +- src/Formats/FormatFactory.cpp | 14 +++++++--- src/Formats/FormatSettings.h | 8 +++--- .../Formats/Impl/PrettyBlockOutputFormat.h | 28 +++++++++---------- 4 files changed, 29 insertions(+), 23 deletions(-) diff --git a/src/Core/Settings.h b/src/Core/Settings.h index bc5fd7cc00f..dd768156eb1 100644 --- a/src/Core/Settings.h +++ b/src/Core/Settings.h @@ -1041,7 +1041,7 @@ class IColumn; M(UInt64, output_format_pretty_max_rows, 10000, "Rows limit for Pretty formats.", 0) \ M(UInt64, output_format_pretty_max_column_pad_width, 250, "Maximum width to pad all values in a column in Pretty formats.", 0) \ M(UInt64, output_format_pretty_max_value_width, 10000, "Maximum width of value to display in Pretty formats. If greater - it will be cut.", 0) \ - M(String, output_format_pretty_color, "auto", "Use ANSI escape sequences to paint colors in Pretty formats", 0) \ + M(String, output_format_pretty_color, "Auto", "Use ANSI escape sequences to paint colors in Pretty formats", 0) \ M(String, output_format_pretty_grid_charset, "UTF-8", "Charset for printing grid borders. Available charsets: ASCII, UTF-8 (default one).", 0) \ M(UInt64, output_format_parquet_row_group_size, 1000000, "Target row group size in rows.", 0) \ M(UInt64, output_format_parquet_row_group_size_bytes, 512 * 1024 * 1024, "Target row group size in bytes, before compression.", 0) \ diff --git a/src/Formats/FormatFactory.cpp b/src/Formats/FormatFactory.cpp index 4c6add2b0d5..9ad779f7230 100644 --- a/src/Formats/FormatFactory.cpp +++ b/src/Formats/FormatFactory.cpp @@ -143,10 +143,16 @@ FormatSettings getFormatSettings(ContextPtr context, const Settings & settings) format_settings.parquet.write_batch_size = settings.output_format_parquet_batch_size; format_settings.parquet.local_read_min_bytes_for_seek = settings.input_format_parquet_local_file_min_bytes_for_seek; format_settings.pretty.charset = settings.output_format_pretty_grid_charset.toString() == "ASCII" ? FormatSettings::Pretty::Charset::ASCII : FormatSettings::Pretty::Charset::UTF8; - format_settings.pretty.output_format_pretty_color = settings.output_format_pretty_color.toString() == "auto" ? FormatSettings::Pretty::PrettyColor::auto - : settings.output_format_pretty_color.toString() == "1" - ? FormatSettings::Pretty::PrettyColor::1 - : FormatSettings::Pretty::PrettyColor::0; + if(settings.output_format_pretty_color.toString() == "Auto") + { + format_settings.pretty.output_format_pretty_color = FormatSettings::Pretty::PrettyColor::Auto; + } else if (settings.output_format_pretty_color.toString() == "On") + { + format_settings.pretty.output_format_pretty_color = FormatSettings::Pretty::PrettyColor::On; + } else if (settings.output_format_pretty_color.toString() == "Off") + { + format_settings.pretty.output_format_pretty_color = FormatSettings::Pretty::PrettyColor::Off; + } format_settings.pretty.max_column_pad_width = settings.output_format_pretty_max_column_pad_width; format_settings.pretty.max_rows = settings.output_format_pretty_max_rows; format_settings.pretty.max_value_width = settings.output_format_pretty_max_value_width; diff --git a/src/Formats/FormatSettings.h b/src/Formats/FormatSettings.h index e20c14034c7..306599c3b0e 100644 --- a/src/Formats/FormatSettings.h +++ b/src/Formats/FormatSettings.h @@ -272,11 +272,11 @@ struct FormatSettings bool color = true; enum class PrettyColor { - 0, - 1, - auto + Off, + On, + Auto, }; - PrettyColor output_format_pretty_color = PrettyColor::auto; + PrettyColor output_format_pretty_color = PrettyColor::Auto; bool output_format_pretty_row_numbers = false; diff --git a/src/Processors/Formats/Impl/PrettyBlockOutputFormat.h b/src/Processors/Formats/Impl/PrettyBlockOutputFormat.h index c978a2ac03b..8ebda142028 100644 --- a/src/Processors/Formats/Impl/PrettyBlockOutputFormat.h +++ b/src/Processors/Formats/Impl/PrettyBlockOutputFormat.h @@ -73,20 +73,20 @@ void registerPrettyFormatWithNoEscapesAndMonoBlock(FormatFactory & factory, cons const Block & sample, const FormatSettings & format_settings) { - FormatSettings changed_settings = format_settings; - using PrettyColor = FormatSettings::Pretty::PrettyColor; - switch(format_settings.pretty.output_format_pretty_color) - { - case PrettyColor::0: - changed_settings.pretty.color = false; - break; - case PrettyColor::1: - no_escapes ? changed_settings.pretty.color = true : changed_settings.pretty.color = false; - break; - case PrettyColor::auto: - isatty(STDOUT_FILENO) ? changed_settings.pretty.color = true : changed_settings.pretty.color = false; - break; - } + FormatSettings changed_settings = format_settings; + using PrettyColor = FormatSettings::Pretty::PrettyColor; + switch(format_settings.pretty.output_format_pretty_color) + { + case PrettyColor::Off: + changed_settings.pretty.color = false; + break; + case PrettyColor::On: + changed_settings.pretty.color = no_escapes ? false : true; + break; + case PrettyColor::Auto: + changed_settings.pretty.color = (isatty(STDOUT_FILENO) && no_escapes) ? false : true; + break; + } if (format_settings.pretty.color != changed_settings.pretty.color) { return std::make_shared(buf, sample, changed_settings, mono_block); From 76ba291c9d37ce0ab184552a337e2523efbabdb7 Mon Sep 17 00:00:00 2001 From: Blargian Date: Tue, 9 Jan 2024 06:44:13 +0200 Subject: [PATCH 05/23] #58363 - added stateless test with expected behaviour for Off and On settings but not Auto --- .../Formats/Impl/PrettyBlockOutputFormat.h | 30 +- ...00405_output_format_pretty_color.reference | 363 ++++++++++++++++++ .../00405_output_format_pretty_color.sql | 32 ++ 3 files changed, 410 insertions(+), 15 deletions(-) create mode 100644 tests/queries/0_stateless/00405_output_format_pretty_color.reference create mode 100644 tests/queries/0_stateless/00405_output_format_pretty_color.sql diff --git a/src/Processors/Formats/Impl/PrettyBlockOutputFormat.h b/src/Processors/Formats/Impl/PrettyBlockOutputFormat.h index 8ebda142028..73bd3621dc7 100644 --- a/src/Processors/Formats/Impl/PrettyBlockOutputFormat.h +++ b/src/Processors/Formats/Impl/PrettyBlockOutputFormat.h @@ -73,21 +73,21 @@ void registerPrettyFormatWithNoEscapesAndMonoBlock(FormatFactory & factory, cons const Block & sample, const FormatSettings & format_settings) { - FormatSettings changed_settings = format_settings; - using PrettyColor = FormatSettings::Pretty::PrettyColor; - switch(format_settings.pretty.output_format_pretty_color) - { - case PrettyColor::Off: - changed_settings.pretty.color = false; - break; - case PrettyColor::On: - changed_settings.pretty.color = no_escapes ? false : true; - break; - case PrettyColor::Auto: - changed_settings.pretty.color = (isatty(STDOUT_FILENO) && no_escapes) ? false : true; - break; - } - if (format_settings.pretty.color != changed_settings.pretty.color) + FormatSettings changed_settings = format_settings; + using PrettyColor = FormatSettings::Pretty::PrettyColor; + switch (format_settings.pretty.output_format_pretty_color) + { + case PrettyColor::Off: + changed_settings.pretty.color = false; + break; + case PrettyColor::On: + changed_settings.pretty.color = no_escapes ? false : true; + break; + case PrettyColor::Auto: + changed_settings.pretty.color = isatty(STDOUT_FILENO) && no_escapes ? false : true; + break; + } + if (!changed_settings.pretty.color) { return std::make_shared(buf, sample, changed_settings, mono_block); } diff --git a/tests/queries/0_stateless/00405_output_format_pretty_color.reference b/tests/queries/0_stateless/00405_output_format_pretty_color.reference new file mode 100644 index 00000000000..3d6f2e17839 --- /dev/null +++ b/tests/queries/0_stateless/00405_output_format_pretty_color.reference @@ -0,0 +1,363 @@ +Off +┏━━━━━━━┳━━━━━━━┳━━━━━━━━━┳━━━━━━━━━━━━━━━━━┓ +┃ hello ┃ world ┃ tuple ┃ sometimes_nulls ┃ +┡━━━━━━━╇━━━━━━━╇━━━━━━━━━╇━━━━━━━━━━━━━━━━━┩ +│ 0 │ 0 │ (0,'0') │ ᴺᵁᴸᴸ │ +├───────┼───────┼─────────┼─────────────────┤ +│ 1 │ 1 │ (1,'1') │ 1 │ +├───────┼───────┼─────────┼─────────────────┤ +│ 2 │ 2 │ (2,'2') │ 2 │ +├───────┼───────┼─────────┼─────────────────┤ +│ 3 │ 3 │ (3,'3') │ ᴺᵁᴸᴸ │ +├───────┼───────┼─────────┼─────────────────┤ +│ 4 │ 4 │ (4,'4') │ 1 │ +└───────┴───────┴─────────┴─────────────────┘ +┏━━━━━━━┳━━━━━━━┳━━━━━━━━━┳━━━━━━━━━━━━━━━━━┓ +┃ hello ┃ world ┃ tuple ┃ sometimes_nulls ┃ +┡━━━━━━━╇━━━━━━━╇━━━━━━━━━╇━━━━━━━━━━━━━━━━━┩ +│ 5 │ 5 │ (5,'5') │ 2 │ +├───────┼───────┼─────────┼─────────────────┤ +│ 6 │ 6 │ (6,'6') │ ᴺᵁᴸᴸ │ +├───────┼───────┼─────────┼─────────────────┤ +│ 7 │ 7 │ (7,'7') │ 1 │ +├───────┼───────┼─────────┼─────────────────┤ +│ 8 │ 8 │ (8,'8') │ 2 │ +├───────┼───────┼─────────┼─────────────────┤ +│ 9 │ 9 │ (9,'9') │ ᴺᵁᴸᴸ │ +└───────┴───────┴─────────┴─────────────────┘ +┌─hello─┬─world─┬─tuple───┬─sometimes_nulls─┐ +│ 0 │ 0 │ (0,'0') │ ᴺᵁᴸᴸ │ +│ 1 │ 1 │ (1,'1') │ 1 │ +│ 2 │ 2 │ (2,'2') │ 2 │ +│ 3 │ 3 │ (3,'3') │ ᴺᵁᴸᴸ │ +│ 4 │ 4 │ (4,'4') │ 1 │ +└───────┴───────┴─────────┴─────────────────┘ +┌─hello─┬─world─┬─tuple───┬─sometimes_nulls─┐ +│ 5 │ 5 │ (5,'5') │ 2 │ +│ 6 │ 6 │ (6,'6') │ ᴺᵁᴸᴸ │ +│ 7 │ 7 │ (7,'7') │ 1 │ +│ 8 │ 8 │ (8,'8') │ 2 │ +│ 9 │ 9 │ (9,'9') │ ᴺᵁᴸᴸ │ +└───────┴───────┴─────────┴─────────────────┘ + hello world tuple sometimes_nulls + + 0 0 (0,'0') ᴺᵁᴸᴸ + 1 1 (1,'1') 1 + 2 2 (2,'2') 2 + 3 3 (3,'3') ᴺᵁᴸᴸ + 4 4 (4,'4') 1 + hello world tuple sometimes_nulls + + 5 5 (5,'5') 2 + 6 6 (6,'6') ᴺᵁᴸᴸ + 7 7 (7,'7') 1 + 8 8 (8,'8') 2 + 9 9 (9,'9') ᴺᵁᴸᴸ +┌─hello─┬─world─┬─tuple───┬─sometimes_nulls─┐ +│ 0 │ 0 │ (0,'0') │ ᴺᵁᴸᴸ │ +│ 1 │ 1 │ (1,'1') │ 1 │ +│ 2 │ 2 │ (2,'2') │ 2 │ +│ 3 │ 3 │ (3,'3') │ ᴺᵁᴸᴸ │ +│ 4 │ 4 │ (4,'4') │ 1 │ +│ 5 │ 5 │ (5,'5') │ 2 │ +│ 6 │ 6 │ (6,'6') │ ᴺᵁᴸᴸ │ +│ 7 │ 7 │ (7,'7') │ 1 │ +│ 8 │ 8 │ (8,'8') │ 2 │ +│ 9 │ 9 │ (9,'9') │ ᴺᵁᴸᴸ │ +└───────┴───────┴─────────┴─────────────────┘ +┏━━━━━━━┳━━━━━━━┳━━━━━━━━━┳━━━━━━━━━━━━━━━━━┓ +┃ hello ┃ world ┃ tuple ┃ sometimes_nulls ┃ +┡━━━━━━━╇━━━━━━━╇━━━━━━━━━╇━━━━━━━━━━━━━━━━━┩ +│ 0 │ 0 │ (0,'0') │ ᴺᵁᴸᴸ │ +├───────┼───────┼─────────┼─────────────────┤ +│ 1 │ 1 │ (1,'1') │ 1 │ +├───────┼───────┼─────────┼─────────────────┤ +│ 2 │ 2 │ (2,'2') │ 2 │ +├───────┼───────┼─────────┼─────────────────┤ +│ 3 │ 3 │ (3,'3') │ ᴺᵁᴸᴸ │ +├───────┼───────┼─────────┼─────────────────┤ +│ 4 │ 4 │ (4,'4') │ 1 │ +└───────┴───────┴─────────┴─────────────────┘ +┏━━━━━━━┳━━━━━━━┳━━━━━━━━━┳━━━━━━━━━━━━━━━━━┓ +┃ hello ┃ world ┃ tuple ┃ sometimes_nulls ┃ +┡━━━━━━━╇━━━━━━━╇━━━━━━━━━╇━━━━━━━━━━━━━━━━━┩ +│ 5 │ 5 │ (5,'5') │ 2 │ +├───────┼───────┼─────────┼─────────────────┤ +│ 6 │ 6 │ (6,'6') │ ᴺᵁᴸᴸ │ +├───────┼───────┼─────────┼─────────────────┤ +│ 7 │ 7 │ (7,'7') │ 1 │ +├───────┼───────┼─────────┼─────────────────┤ +│ 8 │ 8 │ (8,'8') │ 2 │ +├───────┼───────┼─────────┼─────────────────┤ +│ 9 │ 9 │ (9,'9') │ ᴺᵁᴸᴸ │ +└───────┴───────┴─────────┴─────────────────┘ +┌─hello─┬─world─┬─tuple───┬─sometimes_nulls─┐ +│ 0 │ 0 │ (0,'0') │ ᴺᵁᴸᴸ │ +│ 1 │ 1 │ (1,'1') │ 1 │ +│ 2 │ 2 │ (2,'2') │ 2 │ +│ 3 │ 3 │ (3,'3') │ ᴺᵁᴸᴸ │ +│ 4 │ 4 │ (4,'4') │ 1 │ +└───────┴───────┴─────────┴─────────────────┘ +┌─hello─┬─world─┬─tuple───┬─sometimes_nulls─┐ +│ 5 │ 5 │ (5,'5') │ 2 │ +│ 6 │ 6 │ (6,'6') │ ᴺᵁᴸᴸ │ +│ 7 │ 7 │ (7,'7') │ 1 │ +│ 8 │ 8 │ (8,'8') │ 2 │ +│ 9 │ 9 │ (9,'9') │ ᴺᵁᴸᴸ │ +└───────┴───────┴─────────┴─────────────────┘ + hello world tuple sometimes_nulls + + 0 0 (0,'0') ᴺᵁᴸᴸ + 1 1 (1,'1') 1 + 2 2 (2,'2') 2 + 3 3 (3,'3') ᴺᵁᴸᴸ + 4 4 (4,'4') 1 + hello world tuple sometimes_nulls + + 5 5 (5,'5') 2 + 6 6 (6,'6') ᴺᵁᴸᴸ + 7 7 (7,'7') 1 + 8 8 (8,'8') 2 + 9 9 (9,'9') ᴺᵁᴸᴸ +On +┏━━━━━━━┳━━━━━━━┳━━━━━━━━━┳━━━━━━━━━━━━━━━━━┓ +┃ hello ┃ world ┃ tuple  ┃ sometimes_nulls ┃ +┡━━━━━━━╇━━━━━━━╇━━━━━━━━━╇━━━━━━━━━━━━━━━━━┩ +│ 0 │ 0 │ (0,'0') │ ᴺᵁᴸᴸ │ +├───────┼───────┼─────────┼─────────────────┤ +│ 1 │ 1 │ (1,'1') │ 1 │ +├───────┼───────┼─────────┼─────────────────┤ +│ 2 │ 2 │ (2,'2') │ 2 │ +├───────┼───────┼─────────┼─────────────────┤ +│ 3 │ 3 │ (3,'3') │ ᴺᵁᴸᴸ │ +├───────┼───────┼─────────┼─────────────────┤ +│ 4 │ 4 │ (4,'4') │ 1 │ +└───────┴───────┴─────────┴─────────────────┘ +┏━━━━━━━┳━━━━━━━┳━━━━━━━━━┳━━━━━━━━━━━━━━━━━┓ +┃ hello ┃ world ┃ tuple  ┃ sometimes_nulls ┃ +┡━━━━━━━╇━━━━━━━╇━━━━━━━━━╇━━━━━━━━━━━━━━━━━┩ +│ 5 │ 5 │ (5,'5') │ 2 │ +├───────┼───────┼─────────┼─────────────────┤ +│ 6 │ 6 │ (6,'6') │ ᴺᵁᴸᴸ │ +├───────┼───────┼─────────┼─────────────────┤ +│ 7 │ 7 │ (7,'7') │ 1 │ +├───────┼───────┼─────────┼─────────────────┤ +│ 8 │ 8 │ (8,'8') │ 2 │ +├───────┼───────┼─────────┼─────────────────┤ +│ 9 │ 9 │ (9,'9') │ ᴺᵁᴸᴸ │ +└───────┴───────┴─────────┴─────────────────┘ +┌─hello─┬─world─┬─tuple───┬─sometimes_nulls─┐ +│ 0 │ 0 │ (0,'0') │ ᴺᵁᴸᴸ │ +│ 1 │ 1 │ (1,'1') │ 1 │ +│ 2 │ 2 │ (2,'2') │ 2 │ +│ 3 │ 3 │ (3,'3') │ ᴺᵁᴸᴸ │ +│ 4 │ 4 │ (4,'4') │ 1 │ +└───────┴───────┴─────────┴─────────────────┘ +┌─hello─┬─world─┬─tuple───┬─sometimes_nulls─┐ +│ 5 │ 5 │ (5,'5') │ 2 │ +│ 6 │ 6 │ (6,'6') │ ᴺᵁᴸᴸ │ +│ 7 │ 7 │ (7,'7') │ 1 │ +│ 8 │ 8 │ (8,'8') │ 2 │ +│ 9 │ 9 │ (9,'9') │ ᴺᵁᴸᴸ │ +└───────┴───────┴─────────┴─────────────────┘ + hello world tuple sometimes_nulls + + 0 0 (0,'0') ᴺᵁᴸᴸ + 1 1 (1,'1') 1 + 2 2 (2,'2') 2 + 3 3 (3,'3') ᴺᵁᴸᴸ + 4 4 (4,'4') 1 + hello world tuple sometimes_nulls + + 5 5 (5,'5') 2 + 6 6 (6,'6') ᴺᵁᴸᴸ + 7 7 (7,'7') 1 + 8 8 (8,'8') 2 + 9 9 (9,'9') ᴺᵁᴸᴸ +┌─hello─┬─world─┬─tuple───┬─sometimes_nulls─┐ +│ 0 │ 0 │ (0,'0') │ ᴺᵁᴸᴸ │ +│ 1 │ 1 │ (1,'1') │ 1 │ +│ 2 │ 2 │ (2,'2') │ 2 │ +│ 3 │ 3 │ (3,'3') │ ᴺᵁᴸᴸ │ +│ 4 │ 4 │ (4,'4') │ 1 │ +│ 5 │ 5 │ (5,'5') │ 2 │ +│ 6 │ 6 │ (6,'6') │ ᴺᵁᴸᴸ │ +│ 7 │ 7 │ (7,'7') │ 1 │ +│ 8 │ 8 │ (8,'8') │ 2 │ +│ 9 │ 9 │ (9,'9') │ ᴺᵁᴸᴸ │ +└───────┴───────┴─────────┴─────────────────┘ +┏━━━━━━━┳━━━━━━━┳━━━━━━━━━┳━━━━━━━━━━━━━━━━━┓ +┃ hello ┃ world ┃ tuple ┃ sometimes_nulls ┃ +┡━━━━━━━╇━━━━━━━╇━━━━━━━━━╇━━━━━━━━━━━━━━━━━┩ +│ 0 │ 0 │ (0,'0') │ ᴺᵁᴸᴸ │ +├───────┼───────┼─────────┼─────────────────┤ +│ 1 │ 1 │ (1,'1') │ 1 │ +├───────┼───────┼─────────┼─────────────────┤ +│ 2 │ 2 │ (2,'2') │ 2 │ +├───────┼───────┼─────────┼─────────────────┤ +│ 3 │ 3 │ (3,'3') │ ᴺᵁᴸᴸ │ +├───────┼───────┼─────────┼─────────────────┤ +│ 4 │ 4 │ (4,'4') │ 1 │ +└───────┴───────┴─────────┴─────────────────┘ +┏━━━━━━━┳━━━━━━━┳━━━━━━━━━┳━━━━━━━━━━━━━━━━━┓ +┃ hello ┃ world ┃ tuple ┃ sometimes_nulls ┃ +┡━━━━━━━╇━━━━━━━╇━━━━━━━━━╇━━━━━━━━━━━━━━━━━┩ +│ 5 │ 5 │ (5,'5') │ 2 │ +├───────┼───────┼─────────┼─────────────────┤ +│ 6 │ 6 │ (6,'6') │ ᴺᵁᴸᴸ │ +├───────┼───────┼─────────┼─────────────────┤ +│ 7 │ 7 │ (7,'7') │ 1 │ +├───────┼───────┼─────────┼─────────────────┤ +│ 8 │ 8 │ (8,'8') │ 2 │ +├───────┼───────┼─────────┼─────────────────┤ +│ 9 │ 9 │ (9,'9') │ ᴺᵁᴸᴸ │ +└───────┴───────┴─────────┴─────────────────┘ +┌─hello─┬─world─┬─tuple───┬─sometimes_nulls─┐ +│ 0 │ 0 │ (0,'0') │ ᴺᵁᴸᴸ │ +│ 1 │ 1 │ (1,'1') │ 1 │ +│ 2 │ 2 │ (2,'2') │ 2 │ +│ 3 │ 3 │ (3,'3') │ ᴺᵁᴸᴸ │ +│ 4 │ 4 │ (4,'4') │ 1 │ +└───────┴───────┴─────────┴─────────────────┘ +┌─hello─┬─world─┬─tuple───┬─sometimes_nulls─┐ +│ 5 │ 5 │ (5,'5') │ 2 │ +│ 6 │ 6 │ (6,'6') │ ᴺᵁᴸᴸ │ +│ 7 │ 7 │ (7,'7') │ 1 │ +│ 8 │ 8 │ (8,'8') │ 2 │ +│ 9 │ 9 │ (9,'9') │ ᴺᵁᴸᴸ │ +└───────┴───────┴─────────┴─────────────────┘ + hello world tuple sometimes_nulls + + 0 0 (0,'0') ᴺᵁᴸᴸ + 1 1 (1,'1') 1 + 2 2 (2,'2') 2 + 3 3 (3,'3') ᴺᵁᴸᴸ + 4 4 (4,'4') 1 + hello world tuple sometimes_nulls + + 5 5 (5,'5') 2 + 6 6 (6,'6') ᴺᵁᴸᴸ + 7 7 (7,'7') 1 + 8 8 (8,'8') 2 + 9 9 (9,'9') ᴺᵁᴸᴸ +Auto +┏━━━━━━━┳━━━━━━━┳━━━━━━━━━┳━━━━━━━━━━━━━━━━━┓ +┃ hello ┃ world ┃ tuple  ┃ sometimes_nulls ┃ +┡━━━━━━━╇━━━━━━━╇━━━━━━━━━╇━━━━━━━━━━━━━━━━━┩ +│ 0 │ 0 │ (0,'0') │ ᴺᵁᴸᴸ │ +├───────┼───────┼─────────┼─────────────────┤ +│ 1 │ 1 │ (1,'1') │ 1 │ +├───────┼───────┼─────────┼─────────────────┤ +│ 2 │ 2 │ (2,'2') │ 2 │ +├───────┼───────┼─────────┼─────────────────┤ +│ 3 │ 3 │ (3,'3') │ ᴺᵁᴸᴸ │ +├───────┼───────┼─────────┼─────────────────┤ +│ 4 │ 4 │ (4,'4') │ 1 │ +└───────┴───────┴─────────┴─────────────────┘ +┏━━━━━━━┳━━━━━━━┳━━━━━━━━━┳━━━━━━━━━━━━━━━━━┓ +┃ hello ┃ world ┃ tuple  ┃ sometimes_nulls ┃ +┡━━━━━━━╇━━━━━━━╇━━━━━━━━━╇━━━━━━━━━━━━━━━━━┩ +│ 5 │ 5 │ (5,'5') │ 2 │ +├───────┼───────┼─────────┼─────────────────┤ +│ 6 │ 6 │ (6,'6') │ ᴺᵁᴸᴸ │ +├───────┼───────┼─────────┼─────────────────┤ +│ 7 │ 7 │ (7,'7') │ 1 │ +├───────┼───────┼─────────┼─────────────────┤ +│ 8 │ 8 │ (8,'8') │ 2 │ +├───────┼───────┼─────────┼─────────────────┤ +│ 9 │ 9 │ (9,'9') │ ᴺᵁᴸᴸ │ +└───────┴───────┴─────────┴─────────────────┘ +┌─hello─┬─world─┬─tuple───┬─sometimes_nulls─┐ +│ 0 │ 0 │ (0,'0') │ ᴺᵁᴸᴸ │ +│ 1 │ 1 │ (1,'1') │ 1 │ +│ 2 │ 2 │ (2,'2') │ 2 │ +│ 3 │ 3 │ (3,'3') │ ᴺᵁᴸᴸ │ +│ 4 │ 4 │ (4,'4') │ 1 │ +└───────┴───────┴─────────┴─────────────────┘ +┌─hello─┬─world─┬─tuple───┬─sometimes_nulls─┐ +│ 5 │ 5 │ (5,'5') │ 2 │ +│ 6 │ 6 │ (6,'6') │ ᴺᵁᴸᴸ │ +│ 7 │ 7 │ (7,'7') │ 1 │ +│ 8 │ 8 │ (8,'8') │ 2 │ +│ 9 │ 9 │ (9,'9') │ ᴺᵁᴸᴸ │ +└───────┴───────┴─────────┴─────────────────┘ + hello world tuple sometimes_nulls + + 0 0 (0,'0') ᴺᵁᴸᴸ + 1 1 (1,'1') 1 + 2 2 (2,'2') 2 + 3 3 (3,'3') ᴺᵁᴸᴸ + 4 4 (4,'4') 1 + hello world tuple sometimes_nulls + + 5 5 (5,'5') 2 + 6 6 (6,'6') ᴺᵁᴸᴸ + 7 7 (7,'7') 1 + 8 8 (8,'8') 2 + 9 9 (9,'9') ᴺᵁᴸᴸ +┌─hello─┬─world─┬─tuple───┬─sometimes_nulls─┐ +│ 0 │ 0 │ (0,'0') │ ᴺᵁᴸᴸ │ +│ 1 │ 1 │ (1,'1') │ 1 │ +│ 2 │ 2 │ (2,'2') │ 2 │ +│ 3 │ 3 │ (3,'3') │ ᴺᵁᴸᴸ │ +│ 4 │ 4 │ (4,'4') │ 1 │ +│ 5 │ 5 │ (5,'5') │ 2 │ +│ 6 │ 6 │ (6,'6') │ ᴺᵁᴸᴸ │ +│ 7 │ 7 │ (7,'7') │ 1 │ +│ 8 │ 8 │ (8,'8') │ 2 │ +│ 9 │ 9 │ (9,'9') │ ᴺᵁᴸᴸ │ +└───────┴───────┴─────────┴─────────────────┘ +┏━━━━━━━┳━━━━━━━┳━━━━━━━━━┳━━━━━━━━━━━━━━━━━┓ +┃ hello ┃ world ┃ tuple  ┃ sometimes_nulls ┃ +┡━━━━━━━╇━━━━━━━╇━━━━━━━━━╇━━━━━━━━━━━━━━━━━┩ +│ 0 │ 0 │ (0,'0') │ ᴺᵁᴸᴸ │ +├───────┼───────┼─────────┼─────────────────┤ +│ 1 │ 1 │ (1,'1') │ 1 │ +├───────┼───────┼─────────┼─────────────────┤ +│ 2 │ 2 │ (2,'2') │ 2 │ +├───────┼───────┼─────────┼─────────────────┤ +│ 3 │ 3 │ (3,'3') │ ᴺᵁᴸᴸ │ +├───────┼───────┼─────────┼─────────────────┤ +│ 4 │ 4 │ (4,'4') │ 1 │ +└───────┴───────┴─────────┴─────────────────┘ +┏━━━━━━━┳━━━━━━━┳━━━━━━━━━┳━━━━━━━━━━━━━━━━━┓ +┃ hello ┃ world ┃ tuple  ┃ sometimes_nulls ┃ +┡━━━━━━━╇━━━━━━━╇━━━━━━━━━╇━━━━━━━━━━━━━━━━━┩ +│ 5 │ 5 │ (5,'5') │ 2 │ +├───────┼───────┼─────────┼─────────────────┤ +│ 6 │ 6 │ (6,'6') │ ᴺᵁᴸᴸ │ +├───────┼───────┼─────────┼─────────────────┤ +│ 7 │ 7 │ (7,'7') │ 1 │ +├───────┼───────┼─────────┼─────────────────┤ +│ 8 │ 8 │ (8,'8') │ 2 │ +├───────┼───────┼─────────┼─────────────────┤ +│ 9 │ 9 │ (9,'9') │ ᴺᵁᴸᴸ │ +└───────┴───────┴─────────┴─────────────────┘ +┌─hello─┬─world─┬─tuple───┬─sometimes_nulls─┐ +│ 0 │ 0 │ (0,'0') │ ᴺᵁᴸᴸ │ +│ 1 │ 1 │ (1,'1') │ 1 │ +│ 2 │ 2 │ (2,'2') │ 2 │ +│ 3 │ 3 │ (3,'3') │ ᴺᵁᴸᴸ │ +│ 4 │ 4 │ (4,'4') │ 1 │ +└───────┴───────┴─────────┴─────────────────┘ +┌─hello─┬─world─┬─tuple───┬─sometimes_nulls─┐ +│ 5 │ 5 │ (5,'5') │ 2 │ +│ 6 │ 6 │ (6,'6') │ ᴺᵁᴸᴸ │ +│ 7 │ 7 │ (7,'7') │ 1 │ +│ 8 │ 8 │ (8,'8') │ 2 │ +│ 9 │ 9 │ (9,'9') │ ᴺᵁᴸᴸ │ +└───────┴───────┴─────────┴─────────────────┘ + hello world tuple sometimes_nulls + + 0 0 (0,'0') ᴺᵁᴸᴸ + 1 1 (1,'1') 1 + 2 2 (2,'2') 2 + 3 3 (3,'3') ᴺᵁᴸᴸ + 4 4 (4,'4') 1 + hello world tuple sometimes_nulls + + 5 5 (5,'5') 2 + 6 6 (6,'6') ᴺᵁᴸᴸ + 7 7 (7,'7') 1 + 8 8 (8,'8') 2 + 9 9 (9,'9') ᴺᵁᴸᴸ diff --git a/tests/queries/0_stateless/00405_output_format_pretty_color.sql b/tests/queries/0_stateless/00405_output_format_pretty_color.sql new file mode 100644 index 00000000000..fcbba96b45e --- /dev/null +++ b/tests/queries/0_stateless/00405_output_format_pretty_color.sql @@ -0,0 +1,32 @@ +SET output_format_pretty_color = 'Off'; +SHOW SETTING output_format_pretty_color; + +SELECT number AS hello, toString(number) AS world, (hello, world) AS tuple, nullIf(hello % 3, 0) AS sometimes_nulls FROM system.numbers LIMIT 10 SETTINGS max_block_size = 5 FORMAT Pretty; +SELECT number AS hello, toString(number) AS world, (hello, world) AS tuple, nullIf(hello % 3, 0) AS sometimes_nulls FROM system.numbers LIMIT 10 SETTINGS max_block_size = 5 FORMAT PrettyCompact; +SELECT number AS hello, toString(number) AS world, (hello, world) AS tuple, nullIf(hello % 3, 0) AS sometimes_nulls FROM system.numbers LIMIT 10 SETTINGS max_block_size = 5 FORMAT PrettySpace; +SELECT number AS hello, toString(number) AS world, (hello, world) AS tuple, nullIf(hello % 3, 0) AS sometimes_nulls FROM system.numbers LIMIT 10 SETTINGS max_block_size = 5 FORMAT PrettyCompactMonoBlock; +SELECT number AS hello, toString(number) AS world, (hello, world) AS tuple, nullIf(hello % 3, 0) AS sometimes_nulls FROM system.numbers LIMIT 10 SETTINGS max_block_size = 5 FORMAT PrettyNoEscapes; +SELECT number AS hello, toString(number) AS world, (hello, world) AS tuple, nullIf(hello % 3, 0) AS sometimes_nulls FROM system.numbers LIMIT 10 SETTINGS max_block_size = 5 FORMAT PrettyCompactNoEscapes; +SELECT number AS hello, toString(number) AS world, (hello, world) AS tuple, nullIf(hello % 3, 0) AS sometimes_nulls FROM system.numbers LIMIT 10 SETTINGS max_block_size = 5 FORMAT PrettySpaceNoEscapes; + +SET output_format_pretty_color = 'On'; +SHOW SETTING output_format_pretty_color; + +SELECT number AS hello, toString(number) AS world, (hello, world) AS tuple, nullIf(hello % 3, 0) AS sometimes_nulls FROM system.numbers LIMIT 10 SETTINGS max_block_size = 5 FORMAT Pretty; +SELECT number AS hello, toString(number) AS world, (hello, world) AS tuple, nullIf(hello % 3, 0) AS sometimes_nulls FROM system.numbers LIMIT 10 SETTINGS max_block_size = 5 FORMAT PrettyCompact; +SELECT number AS hello, toString(number) AS world, (hello, world) AS tuple, nullIf(hello % 3, 0) AS sometimes_nulls FROM system.numbers LIMIT 10 SETTINGS max_block_size = 5 FORMAT PrettySpace; +SELECT number AS hello, toString(number) AS world, (hello, world) AS tuple, nullIf(hello % 3, 0) AS sometimes_nulls FROM system.numbers LIMIT 10 SETTINGS max_block_size = 5 FORMAT PrettyCompactMonoBlock; +SELECT number AS hello, toString(number) AS world, (hello, world) AS tuple, nullIf(hello % 3, 0) AS sometimes_nulls FROM system.numbers LIMIT 10 SETTINGS max_block_size = 5 FORMAT PrettyNoEscapes; +SELECT number AS hello, toString(number) AS world, (hello, world) AS tuple, nullIf(hello % 3, 0) AS sometimes_nulls FROM system.numbers LIMIT 10 SETTINGS max_block_size = 5 FORMAT PrettyCompactNoEscapes; +SELECT number AS hello, toString(number) AS world, (hello, world) AS tuple, nullIf(hello % 3, 0) AS sometimes_nulls FROM system.numbers LIMIT 10 SETTINGS max_block_size = 5 FORMAT PrettySpaceNoEscapes; + +SET output_format_pretty_color = 'Auto'; +SHOW SETTING output_format_pretty_color; + +SELECT number AS hello, toString(number) AS world, (hello, world) AS tuple, nullIf(hello % 3, 0) AS sometimes_nulls FROM system.numbers LIMIT 10 SETTINGS max_block_size = 5 FORMAT Pretty; +SELECT number AS hello, toString(number) AS world, (hello, world) AS tuple, nullIf(hello % 3, 0) AS sometimes_nulls FROM system.numbers LIMIT 10 SETTINGS max_block_size = 5 FORMAT PrettyCompact; +SELECT number AS hello, toString(number) AS world, (hello, world) AS tuple, nullIf(hello % 3, 0) AS sometimes_nulls FROM system.numbers LIMIT 10 SETTINGS max_block_size = 5 FORMAT PrettySpace; +SELECT number AS hello, toString(number) AS world, (hello, world) AS tuple, nullIf(hello % 3, 0) AS sometimes_nulls FROM system.numbers LIMIT 10 SETTINGS max_block_size = 5 FORMAT PrettyCompactMonoBlock; +SELECT number AS hello, toString(number) AS world, (hello, world) AS tuple, nullIf(hello % 3, 0) AS sometimes_nulls FROM system.numbers LIMIT 10 SETTINGS max_block_size = 5 FORMAT PrettyNoEscapes; +SELECT number AS hello, toString(number) AS world, (hello, world) AS tuple, nullIf(hello % 3, 0) AS sometimes_nulls FROM system.numbers LIMIT 10 SETTINGS max_block_size = 5 FORMAT PrettyCompactNoEscapes; +SELECT number AS hello, toString(number) AS world, (hello, world) AS tuple, nullIf(hello % 3, 0) AS sometimes_nulls FROM system.numbers LIMIT 10 SETTINGS max_block_size = 5 FORMAT PrettySpaceNoEscapes; From d25832a29f53c532e52ad131a0efb64fad7f81dc Mon Sep 17 00:00:00 2001 From: Azat Khuzhin Date: Mon, 8 Jan 2024 19:25:50 +0300 Subject: [PATCH 06/23] Refactor stacktrace symbolizer to avoid copy-paste Note, that this patch slightly changes the behavior, since now inline frames will be processed for sentry as well. And also it properly uses offset argument for non ELF. Signed-off-by: Azat Khuzhin --- src/Common/StackTrace.cpp | 127 ++++++++++++++++++------------------ src/Common/StackTrace.h | 9 ++- src/Daemon/SentryWriter.cpp | 8 +-- 3 files changed, 77 insertions(+), 67 deletions(-) diff --git a/src/Common/StackTrace.cpp b/src/Common/StackTrace.cpp index fe513199ac2..4e5c9bd7893 100644 --- a/src/Common/StackTrace.cpp +++ b/src/Common/StackTrace.cpp @@ -32,6 +32,10 @@ std::atomic show_addresses = true; bool shouldShowAddress(const void * addr) { + /// Likely inline frame + if (!addr) + return false; + /// If the address is less than 4096, most likely it is a nullptr dereference with offset, /// and showing this offset is secure nevertheless. /// NOTE: 4096 is the page size on x86 and it can be different on other systems, @@ -203,20 +207,24 @@ static void * getCallerAddress(const ucontext_t & context) #endif } -// FIXME: looks like this is used only for Sentry but duplicates the whole algo, maybe replace? -void StackTrace::symbolize( - const StackTrace::FramePointers & frame_pointers, [[maybe_unused]] size_t offset, size_t size, StackTrace::Frames & frames) +void StackTrace::forEachFrame( + const StackTrace::FramePointers & frame_pointers, + size_t offset, + size_t size, + std::function callback, + bool fatal) { #if defined(__ELF__) && !defined(OS_FREEBSD) const DB::SymbolIndex & symbol_index = DB::SymbolIndex::instance(); std::unordered_map dwarfs; - for (size_t i = 0; i < offset; ++i) - frames[i].virtual_addr = frame_pointers[i]; + using enum DB::Dwarf::LocationInfoMode; + const auto mode = fatal ? FULL_WITH_INLINE : FAST; for (size_t i = offset; i < size; ++i) { - StackTrace::Frame & current_frame = frames[i]; + StackTrace::Frame current_frame; + std::vector inline_frames; current_frame.virtual_addr = frame_pointers[i]; const auto * object = symbol_index.findObject(current_frame.virtual_addr); uintptr_t virtual_offset = object ? uintptr_t(object->address_begin) : 0; @@ -230,26 +238,41 @@ void StackTrace::symbolize( auto dwarf_it = dwarfs.try_emplace(object->name, object->elf).first; DB::Dwarf::LocationInfo location; - std::vector inline_frames; if (dwarf_it->second.findAddress( - uintptr_t(current_frame.physical_addr), location, DB::Dwarf::LocationInfoMode::FAST, inline_frames)) + uintptr_t(current_frame.physical_addr), location, mode, inline_frames)) { current_frame.file = location.file.toString(); current_frame.line = location.line; } } } - else - current_frame.object = "?"; if (const auto * symbol = symbol_index.findSymbol(current_frame.virtual_addr)) current_frame.symbol = demangle(symbol->name); - else - current_frame.symbol = "?"; + + for (const auto & frame : inline_frames) + { + StackTrace::Frame current_inline_frame; + const String file_for_inline_frame = frame.location.file.toString(); + + current_inline_frame.file = "inlined from " + file_for_inline_frame; + current_inline_frame.line = frame.location.line; + current_inline_frame.symbol = frame.name; + + callback(current_inline_frame); + } + + callback(current_frame); } #else - for (size_t i = 0; i < size; ++i) - frames[i].virtual_addr = frame_pointers[i]; + UNUSED(fatal); + + for (size_t i = offset; i < size; ++i) + { + StackTrace::Frame current_frame; + current_frame.virtual_addr = frame_pointers[i]; + callback(current_frame); + } #endif } @@ -349,72 +372,52 @@ toStringEveryLineImpl([[maybe_unused]] bool fatal, const StackTraceRefTriple & s if (stack_trace.size == 0) return callback(""); + size_t frame_index = stack_trace.offset; #if defined(__ELF__) && !defined(OS_FREEBSD) - - using enum DB::Dwarf::LocationInfoMode; - const auto mode = fatal ? FULL_WITH_INLINE : FAST; - - const DB::SymbolIndex & symbol_index = DB::SymbolIndex::instance(); - std::unordered_map dwarfs; - - for (size_t i = stack_trace.offset; i < stack_trace.size; ++i) + size_t inline_frame_index = 0; + auto callback_wrapper = [&](const StackTrace::Frame & frame) { - std::vector inline_frames; - const void * virtual_addr = stack_trace.pointers[i]; - const auto * object = symbol_index.findObject(virtual_addr); - uintptr_t virtual_offset = object ? uintptr_t(object->address_begin) : 0; - const void * physical_addr = reinterpret_cast(uintptr_t(virtual_addr) - virtual_offset); - DB::WriteBufferFromOwnString out; - out << i << ". "; - String file; - if (std::error_code ec; object && std::filesystem::exists(object->name, ec) && !ec) + /// Inline frame + if (!frame.virtual_addr) { - auto dwarf_it = dwarfs.try_emplace(object->name, object->elf).first; - - DB::Dwarf::LocationInfo location; - - if (dwarf_it->second.findAddress(uintptr_t(physical_addr), location, mode, inline_frames)) - { - file = location.file.toString(); - out << file << ":" << location.line << ": "; - } + out << frame_index << "." << inline_frame_index++ << ". "; + } + else + { + out << frame_index++ << ". "; + inline_frame_index = 0; } - if (const auto * const symbol = symbol_index.findSymbol(virtual_addr)) - out << demangleAndCollapseNames(file, symbol->name); + if (frame.file.has_value() && frame.line.has_value()) + out << *frame.file << ':' << *frame.line << ": "; + + if (frame.symbol.has_value() && frame.file.has_value()) + out << demangleAndCollapseNames(*frame.file, frame.symbol->data()); else out << "?"; - if (shouldShowAddress(physical_addr)) + if (shouldShowAddress(frame.physical_addr)) { out << " @ "; - DB::writePointerHex(physical_addr, out); + DB::writePointerHex(frame.physical_addr, out); } - out << " in " << (object ? object->name : "?"); - - for (size_t j = 0; j < inline_frames.size(); ++j) - { - const auto & frame = inline_frames[j]; - const String file_for_inline_frame = frame.location.file.toString(); - callback(fmt::format( - "{}.{}. inlined from {}:{}: {}", - i, - j + 1, - file_for_inline_frame, - frame.location.line, - demangleAndCollapseNames(file_for_inline_frame, frame.name))); - } + if (frame.object.has_value()) + out << " in " << *frame.object; callback(out.str()); - } + }; #else - for (size_t i = stack_trace.offset; i < stack_trace.size; ++i) - if (const void * const addr = stack_trace.pointers[i]; shouldShowAddress(addr)) - callback(fmt::format("{}. {}", i, addr)); + auto callback_wrapper = [&](const StackTrace::Frame & frame) + { + if (frame.virtual_addr && shouldShowAddress(frame.virtual_addr)) + callback(fmt::format("{}. {}", frame_index++, frame.virtual_addr)); + }; #endif + + StackTrace::forEachFrame(stack_trace.pointers, stack_trace.offset, stack_trace.size, callback_wrapper, fatal); } void StackTrace::toStringEveryLine(std::function callback) const diff --git a/src/Common/StackTrace.h b/src/Common/StackTrace.h index 656f543d837..e5654162ecb 100644 --- a/src/Common/StackTrace.h +++ b/src/Common/StackTrace.h @@ -62,7 +62,14 @@ public: static std::string toString(void ** frame_pointers, size_t offset, size_t size); static void dropCache(); - static void symbolize(const FramePointers & frame_pointers, size_t offset, size_t size, StackTrace::Frames & frames); + + /// @param fatal - if true, will process inline frames (slower) + static void forEachFrame( + const FramePointers & frame_pointers, + size_t offset, + size_t size, + std::function callback, + bool fatal); void toStringEveryLine(std::function callback) const; static void toStringEveryLine(const FramePointers & frame_pointers, std::function callback); diff --git a/src/Daemon/SentryWriter.cpp b/src/Daemon/SentryWriter.cpp index d6e7144ca3b..2050d503879 100644 --- a/src/Daemon/SentryWriter.cpp +++ b/src/Daemon/SentryWriter.cpp @@ -169,11 +169,9 @@ void SentryWriter::onFault(int sig, const std::string & error_message, const Sta }; StackTrace::Frames frames; - StackTrace::symbolize(stack_trace.getFramePointers(), offset, stack_size, frames); - for (ssize_t i = stack_size - 1; i >= offset; --i) + auto sentry_add_stack_trace = [&](const StackTrace::Frame & current_frame) { - const StackTrace::Frame & current_frame = frames[i]; sentry_value_t sentry_frame = sentry_value_new_object(); UInt64 frame_ptr = reinterpret_cast(current_frame.virtual_addr); @@ -190,7 +188,9 @@ void SentryWriter::onFault(int sig, const std::string & error_message, const Sta sentry_value_set_by_key(sentry_frame, "lineno", sentry_value_new_int32(static_cast(current_frame.line.value()))); sentry_value_append(sentry_frames, sentry_frame); - } + }; + + StackTrace::forEachFrame(stack_trace.getFramePointers(), offset, stack_size, sentry_add_stack_trace, /* fatal= */ true); } /// Prepare data for https://develop.sentry.dev/sdk/event-payloads/threads/ From aa8876a611660ca6c44d79d611808f261e386fb2 Mon Sep 17 00:00:00 2001 From: Blargian Date: Fri, 12 Jan 2024 12:31:57 +0200 Subject: [PATCH 07/23] #58363 - Changes based on review of draft PR - changed output_format_pretty_color to use UInt64Auto. Added isWritingToTerminal function to IO/WriteHelpers.h and updated test --- src/Core/Settings.h | 2 +- src/Formats/FormatFactory.cpp | 11 +---------- src/Formats/FormatSettings.h | 11 +++-------- src/IO/WriteHelpers.h | 8 ++++++++ .../Formats/Impl/PrettyBlockOutputFormat.h | 14 +++++++------- .../00405_output_format_pretty_color.reference | 18 +++++++++--------- .../00405_output_format_pretty_color.sql | 6 +++--- 7 files changed, 32 insertions(+), 38 deletions(-) diff --git a/src/Core/Settings.h b/src/Core/Settings.h index dd768156eb1..b177b88ea27 100644 --- a/src/Core/Settings.h +++ b/src/Core/Settings.h @@ -1041,7 +1041,7 @@ class IColumn; M(UInt64, output_format_pretty_max_rows, 10000, "Rows limit for Pretty formats.", 0) \ M(UInt64, output_format_pretty_max_column_pad_width, 250, "Maximum width to pad all values in a column in Pretty formats.", 0) \ M(UInt64, output_format_pretty_max_value_width, 10000, "Maximum width of value to display in Pretty formats. If greater - it will be cut.", 0) \ - M(String, output_format_pretty_color, "Auto", "Use ANSI escape sequences to paint colors in Pretty formats", 0) \ + M(UInt64Auto, output_format_pretty_color, "auto", "Use ANSI escape sequences in Pretty formats. 0 - disabled, 1 - enabled, 'auto' - enabled if a terminal.", 0) \ M(String, output_format_pretty_grid_charset, "UTF-8", "Charset for printing grid borders. Available charsets: ASCII, UTF-8 (default one).", 0) \ M(UInt64, output_format_parquet_row_group_size, 1000000, "Target row group size in rows.", 0) \ M(UInt64, output_format_parquet_row_group_size_bytes, 512 * 1024 * 1024, "Target row group size in bytes, before compression.", 0) \ diff --git a/src/Formats/FormatFactory.cpp b/src/Formats/FormatFactory.cpp index 9ad779f7230..92ffce032cb 100644 --- a/src/Formats/FormatFactory.cpp +++ b/src/Formats/FormatFactory.cpp @@ -143,16 +143,7 @@ FormatSettings getFormatSettings(ContextPtr context, const Settings & settings) format_settings.parquet.write_batch_size = settings.output_format_parquet_batch_size; format_settings.parquet.local_read_min_bytes_for_seek = settings.input_format_parquet_local_file_min_bytes_for_seek; format_settings.pretty.charset = settings.output_format_pretty_grid_charset.toString() == "ASCII" ? FormatSettings::Pretty::Charset::ASCII : FormatSettings::Pretty::Charset::UTF8; - if(settings.output_format_pretty_color.toString() == "Auto") - { - format_settings.pretty.output_format_pretty_color = FormatSettings::Pretty::PrettyColor::Auto; - } else if (settings.output_format_pretty_color.toString() == "On") - { - format_settings.pretty.output_format_pretty_color = FormatSettings::Pretty::PrettyColor::On; - } else if (settings.output_format_pretty_color.toString() == "Off") - { - format_settings.pretty.output_format_pretty_color = FormatSettings::Pretty::PrettyColor::Off; - } + format_settings.pretty.output_format_pretty_color = settings.output_format_pretty_color; format_settings.pretty.max_column_pad_width = settings.output_format_pretty_max_column_pad_width; format_settings.pretty.max_rows = settings.output_format_pretty_max_rows; format_settings.pretty.max_value_width = settings.output_format_pretty_max_value_width; diff --git a/src/Formats/FormatSettings.h b/src/Formats/FormatSettings.h index 306599c3b0e..d68202ffab2 100644 --- a/src/Formats/FormatSettings.h +++ b/src/Formats/FormatSettings.h @@ -4,7 +4,7 @@ #include #include #include - +#include namespace DB { @@ -270,13 +270,8 @@ struct FormatSettings UInt64 max_column_pad_width = 250; UInt64 max_value_width = 10000; bool color = true; - - enum class PrettyColor { - Off, - On, - Auto, - }; - PrettyColor output_format_pretty_color = PrettyColor::Auto; + + SettingFieldUInt64Auto output_format_pretty_color{"auto"}; bool output_format_pretty_row_numbers = false; diff --git a/src/IO/WriteHelpers.h b/src/IO/WriteHelpers.h index b4f8b476b11..f438990fd1c 100644 --- a/src/IO/WriteHelpers.h +++ b/src/IO/WriteHelpers.h @@ -29,6 +29,7 @@ #include #include #include +#include #include #include @@ -36,6 +37,7 @@ #include #include #include +#include #ifdef __clang__ #pragma clang diagnostic push @@ -1405,6 +1407,12 @@ void writePointerHex(const void * ptr, WriteBuffer & buf); String fourSpaceIndent(size_t indent); +bool inline isWritingToTerminal(const WriteBuffer & buf) +{ + const auto * write_buffer_to_descriptor = typeid_cast(&buf); + return write_buffer_to_descriptor && write_buffer_to_descriptor->getFD() == STDOUT_FILENO && isatty(STDOUT_FILENO); +} + } template<> diff --git a/src/Processors/Formats/Impl/PrettyBlockOutputFormat.h b/src/Processors/Formats/Impl/PrettyBlockOutputFormat.h index 73bd3621dc7..c17c8a863c2 100644 --- a/src/Processors/Formats/Impl/PrettyBlockOutputFormat.h +++ b/src/Processors/Formats/Impl/PrettyBlockOutputFormat.h @@ -4,7 +4,7 @@ #include #include #include - +#include namespace DB { @@ -74,17 +74,17 @@ void registerPrettyFormatWithNoEscapesAndMonoBlock(FormatFactory & factory, cons const FormatSettings & format_settings) { FormatSettings changed_settings = format_settings; - using PrettyColor = FormatSettings::Pretty::PrettyColor; - switch (format_settings.pretty.output_format_pretty_color) + auto value = format_settings.pretty.output_format_pretty_color.valueOr(2); + switch (value) { - case PrettyColor::Off: + case 0: changed_settings.pretty.color = false; break; - case PrettyColor::On: + case 1: changed_settings.pretty.color = no_escapes ? false : true; break; - case PrettyColor::Auto: - changed_settings.pretty.color = isatty(STDOUT_FILENO) && no_escapes ? false : true; + case 2: + changed_settings.pretty.color = isWritingToTerminal(buf) || no_escapes ? false : true; break; } if (!changed_settings.pretty.color) diff --git a/tests/queries/0_stateless/00405_output_format_pretty_color.reference b/tests/queries/0_stateless/00405_output_format_pretty_color.reference index 3d6f2e17839..71495678467 100644 --- a/tests/queries/0_stateless/00405_output_format_pretty_color.reference +++ b/tests/queries/0_stateless/00405_output_format_pretty_color.reference @@ -1,4 +1,4 @@ -Off +0 ┏━━━━━━━┳━━━━━━━┳━━━━━━━━━┳━━━━━━━━━━━━━━━━━┓ ┃ hello ┃ world ┃ tuple ┃ sometimes_nulls ┃ ┡━━━━━━━╇━━━━━━━╇━━━━━━━━━╇━━━━━━━━━━━━━━━━━┩ @@ -119,7 +119,7 @@ Off 7 7 (7,'7') 1 8 8 (8,'8') 2 9 9 (9,'9') ᴺᵁᴸᴸ -On +1 ┏━━━━━━━┳━━━━━━━┳━━━━━━━━━┳━━━━━━━━━━━━━━━━━┓ ┃ hello ┃ world ┃ tuple  ┃ sometimes_nulls ┃ ┡━━━━━━━╇━━━━━━━╇━━━━━━━━━╇━━━━━━━━━━━━━━━━━┩ @@ -240,7 +240,7 @@ On 7 7 (7,'7') 1 8 8 (8,'8') 2 9 9 (9,'9') ᴺᵁᴸᴸ -Auto +auto ┏━━━━━━━┳━━━━━━━┳━━━━━━━━━┳━━━━━━━━━━━━━━━━━┓ ┃ hello ┃ world ┃ tuple  ┃ sometimes_nulls ┃ ┡━━━━━━━╇━━━━━━━╇━━━━━━━━━╇━━━━━━━━━━━━━━━━━┩ @@ -308,7 +308,7 @@ Auto │ 9 │ 9 │ (9,'9') │ ᴺᵁᴸᴸ │ └───────┴───────┴─────────┴─────────────────┘ ┏━━━━━━━┳━━━━━━━┳━━━━━━━━━┳━━━━━━━━━━━━━━━━━┓ -┃ hello ┃ world ┃ tuple  ┃ sometimes_nulls ┃ +┃ hello ┃ world ┃ tuple ┃ sometimes_nulls ┃ ┡━━━━━━━╇━━━━━━━╇━━━━━━━━━╇━━━━━━━━━━━━━━━━━┩ │ 0 │ 0 │ (0,'0') │ ᴺᵁᴸᴸ │ ├───────┼───────┼─────────┼─────────────────┤ @@ -321,7 +321,7 @@ Auto │ 4 │ 4 │ (4,'4') │ 1 │ └───────┴───────┴─────────┴─────────────────┘ ┏━━━━━━━┳━━━━━━━┳━━━━━━━━━┳━━━━━━━━━━━━━━━━━┓ -┃ hello ┃ world ┃ tuple  ┃ sometimes_nulls ┃ +┃ hello ┃ world ┃ tuple ┃ sometimes_nulls ┃ ┡━━━━━━━╇━━━━━━━╇━━━━━━━━━╇━━━━━━━━━━━━━━━━━┩ │ 5 │ 5 │ (5,'5') │ 2 │ ├───────┼───────┼─────────┼─────────────────┤ @@ -333,28 +333,28 @@ Auto ├───────┼───────┼─────────┼─────────────────┤ │ 9 │ 9 │ (9,'9') │ ᴺᵁᴸᴸ │ └───────┴───────┴─────────┴─────────────────┘ -┌─hello─┬─world─┬─tuple───┬─sometimes_nulls─┐ +┌─hello─┬─world─┬─tuple───┬─sometimes_nulls─┐ │ 0 │ 0 │ (0,'0') │ ᴺᵁᴸᴸ │ │ 1 │ 1 │ (1,'1') │ 1 │ │ 2 │ 2 │ (2,'2') │ 2 │ │ 3 │ 3 │ (3,'3') │ ᴺᵁᴸᴸ │ │ 4 │ 4 │ (4,'4') │ 1 │ └───────┴───────┴─────────┴─────────────────┘ -┌─hello─┬─world─┬─tuple───┬─sometimes_nulls─┐ +┌─hello─┬─world─┬─tuple───┬─sometimes_nulls─┐ │ 5 │ 5 │ (5,'5') │ 2 │ │ 6 │ 6 │ (6,'6') │ ᴺᵁᴸᴸ │ │ 7 │ 7 │ (7,'7') │ 1 │ │ 8 │ 8 │ (8,'8') │ 2 │ │ 9 │ 9 │ (9,'9') │ ᴺᵁᴸᴸ │ └───────┴───────┴─────────┴─────────────────┘ - hello world tuple sometimes_nulls + hello world tuple sometimes_nulls 0 0 (0,'0') ᴺᵁᴸᴸ 1 1 (1,'1') 1 2 2 (2,'2') 2 3 3 (3,'3') ᴺᵁᴸᴸ 4 4 (4,'4') 1 - hello world tuple sometimes_nulls + hello world tuple sometimes_nulls 5 5 (5,'5') 2 6 6 (6,'6') ᴺᵁᴸᴸ diff --git a/tests/queries/0_stateless/00405_output_format_pretty_color.sql b/tests/queries/0_stateless/00405_output_format_pretty_color.sql index fcbba96b45e..bc2d0c3adbf 100644 --- a/tests/queries/0_stateless/00405_output_format_pretty_color.sql +++ b/tests/queries/0_stateless/00405_output_format_pretty_color.sql @@ -1,4 +1,4 @@ -SET output_format_pretty_color = 'Off'; +SET output_format_pretty_color = 0; SHOW SETTING output_format_pretty_color; SELECT number AS hello, toString(number) AS world, (hello, world) AS tuple, nullIf(hello % 3, 0) AS sometimes_nulls FROM system.numbers LIMIT 10 SETTINGS max_block_size = 5 FORMAT Pretty; @@ -9,7 +9,7 @@ SELECT number AS hello, toString(number) AS world, (hello, world) AS tuple, null SELECT number AS hello, toString(number) AS world, (hello, world) AS tuple, nullIf(hello % 3, 0) AS sometimes_nulls FROM system.numbers LIMIT 10 SETTINGS max_block_size = 5 FORMAT PrettyCompactNoEscapes; SELECT number AS hello, toString(number) AS world, (hello, world) AS tuple, nullIf(hello % 3, 0) AS sometimes_nulls FROM system.numbers LIMIT 10 SETTINGS max_block_size = 5 FORMAT PrettySpaceNoEscapes; -SET output_format_pretty_color = 'On'; +SET output_format_pretty_color = 1; SHOW SETTING output_format_pretty_color; SELECT number AS hello, toString(number) AS world, (hello, world) AS tuple, nullIf(hello % 3, 0) AS sometimes_nulls FROM system.numbers LIMIT 10 SETTINGS max_block_size = 5 FORMAT Pretty; @@ -20,7 +20,7 @@ SELECT number AS hello, toString(number) AS world, (hello, world) AS tuple, null SELECT number AS hello, toString(number) AS world, (hello, world) AS tuple, nullIf(hello % 3, 0) AS sometimes_nulls FROM system.numbers LIMIT 10 SETTINGS max_block_size = 5 FORMAT PrettyCompactNoEscapes; SELECT number AS hello, toString(number) AS world, (hello, world) AS tuple, nullIf(hello % 3, 0) AS sometimes_nulls FROM system.numbers LIMIT 10 SETTINGS max_block_size = 5 FORMAT PrettySpaceNoEscapes; -SET output_format_pretty_color = 'Auto'; +SET output_format_pretty_color = 'auto'; SHOW SETTING output_format_pretty_color; SELECT number AS hello, toString(number) AS world, (hello, world) AS tuple, nullIf(hello % 3, 0) AS sometimes_nulls FROM system.numbers LIMIT 10 SETTINGS max_block_size = 5 FORMAT Pretty; From 72b5cf59932172618921bfcd4c6b48e177365fa3 Mon Sep 17 00:00:00 2001 From: Blargian Date: Fri, 12 Jan 2024 19:46:03 +0200 Subject: [PATCH 08/23] #58363 - removed switch from PrettyBlockOutputFormat and modified BlockOutputFormats to use color variable. Updated english and russian documentation. Updated test 00405 reference file. --- .../operations/settings/settings-formats.md | 8 ++++- docs/ru/operations/settings/settings.md | 11 +++++++ src/Formats/FormatFactory.cpp | 2 +- src/Formats/FormatSettings.h | 4 +-- .../Formats/Impl/PrettyBlockOutputFormat.cpp | 10 +++---- .../Formats/Impl/PrettyBlockOutputFormat.h | 29 +++++-------------- .../Impl/PrettyCompactBlockOutputFormat.cpp | 12 ++++---- .../Impl/PrettyCompactBlockOutputFormat.h | 2 +- .../Impl/PrettySpaceBlockOutputFormat.cpp | 8 ++--- .../Impl/PrettySpaceBlockOutputFormat.h | 4 +-- ...00405_output_format_pretty_color.reference | 14 ++++----- 11 files changed, 52 insertions(+), 52 deletions(-) diff --git a/docs/en/operations/settings/settings-formats.md b/docs/en/operations/settings/settings-formats.md index 3d76bd9df73..0d40c77f6cc 100644 --- a/docs/en/operations/settings/settings-formats.md +++ b/docs/en/operations/settings/settings-formats.md @@ -1569,7 +1569,13 @@ Result: Use ANSI escape sequences to paint colors in Pretty formats. -Enabled by default. +possible values: + +- `0` — Disabled. Pretty formats do not use ANSI escape sequences. +- `1` — Enabled. Pretty formats will use ANSI escape sequences except for `NoEscapes` formats. +- `auto` - Enabled if `stdout` is a terminal except for `NoEscapes` formats. + +Default value is `auto`. ### output_format_pretty_grid_charset {#output_format_pretty_grid_charset} diff --git a/docs/ru/operations/settings/settings.md b/docs/ru/operations/settings/settings.md index 2081dcc59b6..cd949e9e6b1 100644 --- a/docs/ru/operations/settings/settings.md +++ b/docs/ru/operations/settings/settings.md @@ -2796,6 +2796,17 @@ SELECT TOP 3 name, value FROM system.settings; 3. │ max_block_size │ 65505 │ └─────────────────────────┴─────────┘ ``` +### output_format_pretty_color {#output_format_pretty_color} + +Включает/выключает управляющие последовательности ANSI в форматах Pretty. + +Возможные значения: + +- `0` — выключена. Не исползует ANSI последовательности в форматах Pretty. +- `1` — включена. Исползует ANSI последовательности с исключением форматов `NoEscapes`. +- `auto` - включена если `stdout` является терминалом с исключением форматов `NoEscapes`. + +Значение по умолчанию: `auto` ## system_events_show_zero_values {#system_events_show_zero_values} diff --git a/src/Formats/FormatFactory.cpp b/src/Formats/FormatFactory.cpp index 92ffce032cb..da3f0c7c02d 100644 --- a/src/Formats/FormatFactory.cpp +++ b/src/Formats/FormatFactory.cpp @@ -143,7 +143,7 @@ FormatSettings getFormatSettings(ContextPtr context, const Settings & settings) format_settings.parquet.write_batch_size = settings.output_format_parquet_batch_size; format_settings.parquet.local_read_min_bytes_for_seek = settings.input_format_parquet_local_file_min_bytes_for_seek; format_settings.pretty.charset = settings.output_format_pretty_grid_charset.toString() == "ASCII" ? FormatSettings::Pretty::Charset::ASCII : FormatSettings::Pretty::Charset::UTF8; - format_settings.pretty.output_format_pretty_color = settings.output_format_pretty_color; + format_settings.pretty.color = settings.output_format_pretty_color; format_settings.pretty.max_column_pad_width = settings.output_format_pretty_max_column_pad_width; format_settings.pretty.max_rows = settings.output_format_pretty_max_rows; format_settings.pretty.max_value_width = settings.output_format_pretty_max_value_width; diff --git a/src/Formats/FormatSettings.h b/src/Formats/FormatSettings.h index d68202ffab2..873cfdfab09 100644 --- a/src/Formats/FormatSettings.h +++ b/src/Formats/FormatSettings.h @@ -269,9 +269,7 @@ struct FormatSettings UInt64 max_rows = 10000; UInt64 max_column_pad_width = 250; UInt64 max_value_width = 10000; - bool color = true; - - SettingFieldUInt64Auto output_format_pretty_color{"auto"}; + SettingFieldUInt64Auto color{"auto"}; bool output_format_pretty_row_numbers = false; diff --git a/src/Processors/Formats/Impl/PrettyBlockOutputFormat.cpp b/src/Processors/Formats/Impl/PrettyBlockOutputFormat.cpp index 14648e68f94..eee0b24b5ba 100644 --- a/src/Processors/Formats/Impl/PrettyBlockOutputFormat.cpp +++ b/src/Processors/Formats/Impl/PrettyBlockOutputFormat.cpp @@ -12,8 +12,8 @@ namespace DB { PrettyBlockOutputFormat::PrettyBlockOutputFormat( - WriteBuffer & out_, const Block & header_, const FormatSettings & format_settings_, bool mono_block_) - : IOutputFormat(header_, out_), format_settings(format_settings_), serializations(header_.getSerializations()), mono_block(mono_block_) + WriteBuffer & out_, const Block & header_, const FormatSettings & format_settings_, bool mono_block_, bool color_) + : IOutputFormat(header_, out_), format_settings(format_settings_), serializations(header_.getSerializations()), color(color_), mono_block(mono_block_) { } @@ -237,7 +237,7 @@ void PrettyBlockOutputFormat::writeChunk(const Chunk & chunk, PortKind port_kind const auto & col = header.getByPosition(i); - if (format_settings.pretty.color) + if (color) writeCString("\033[1m", out); if (col.type->shouldAlignRightInPrettyFormats()) @@ -255,7 +255,7 @@ void PrettyBlockOutputFormat::writeChunk(const Chunk & chunk, PortKind port_kind writeChar(' ', out); } - if (format_settings.pretty.color) + if (color) writeCString("\033[0m", out); } writeCString(" ", out); @@ -335,7 +335,7 @@ void PrettyBlockOutputFormat::writeValueWithPadding( reinterpret_cast(serialized_value.data()), serialized_value.size(), 0, 1 + format_settings.pretty.max_value_width)); const char * ellipsis = format_settings.pretty.charset == FormatSettings::Pretty::Charset::UTF8 ? "⋯" : "~"; - if (format_settings.pretty.color) + if (color) { serialized_value += "\033[31;1m"; serialized_value += ellipsis; diff --git a/src/Processors/Formats/Impl/PrettyBlockOutputFormat.h b/src/Processors/Formats/Impl/PrettyBlockOutputFormat.h index c17c8a863c2..9b2087a1399 100644 --- a/src/Processors/Formats/Impl/PrettyBlockOutputFormat.h +++ b/src/Processors/Formats/Impl/PrettyBlockOutputFormat.h @@ -19,10 +19,10 @@ class PrettyBlockOutputFormat : public IOutputFormat { public: /// no_escapes - do not use ANSI escape sequences - to display in the browser, not in the console. - PrettyBlockOutputFormat(WriteBuffer & out_, const Block & header_, const FormatSettings & format_settings_, bool mono_block_); + PrettyBlockOutputFormat(WriteBuffer & out_, const Block & header_, const FormatSettings & format_settings_, bool mono_block_, bool color_); String getName() const override { return "PrettyBlockOutputFormat"; } - + protected: void consume(Chunk) override; void consumeTotals(Chunk) override; @@ -57,7 +57,9 @@ protected: total_rows = 0; } -private: + bool color; + +private: bool mono_block; /// For mono_block == true only Chunk mono_chunk; @@ -73,25 +75,8 @@ void registerPrettyFormatWithNoEscapesAndMonoBlock(FormatFactory & factory, cons const Block & sample, const FormatSettings & format_settings) { - FormatSettings changed_settings = format_settings; - auto value = format_settings.pretty.output_format_pretty_color.valueOr(2); - switch (value) - { - case 0: - changed_settings.pretty.color = false; - break; - case 1: - changed_settings.pretty.color = no_escapes ? false : true; - break; - case 2: - changed_settings.pretty.color = isWritingToTerminal(buf) || no_escapes ? false : true; - break; - } - if (!changed_settings.pretty.color) - { - return std::make_shared(buf, sample, changed_settings, mono_block); - } - return std::make_shared(buf, sample, format_settings, mono_block); + bool color = !no_escapes && format_settings.pretty.color.valueOr(isWritingToTerminal(buf)); + return std::make_shared(buf, sample, format_settings, mono_block, color); }); if (!mono_block) factory.markOutputFormatSupportsParallelFormatting(name); diff --git a/src/Processors/Formats/Impl/PrettyCompactBlockOutputFormat.cpp b/src/Processors/Formats/Impl/PrettyCompactBlockOutputFormat.cpp index 2ba9ec725e2..b547ce9358a 100644 --- a/src/Processors/Formats/Impl/PrettyCompactBlockOutputFormat.cpp +++ b/src/Processors/Formats/Impl/PrettyCompactBlockOutputFormat.cpp @@ -48,8 +48,8 @@ GridSymbols ascii_grid_symbols { } -PrettyCompactBlockOutputFormat::PrettyCompactBlockOutputFormat(WriteBuffer & out_, const Block & header, const FormatSettings & format_settings_, bool mono_block_) - : PrettyBlockOutputFormat(out_, header, format_settings_, mono_block_) +PrettyCompactBlockOutputFormat::PrettyCompactBlockOutputFormat(WriteBuffer & out_, const Block & header, const FormatSettings & format_settings_, bool mono_block_, bool color_) + : PrettyBlockOutputFormat(out_, header, format_settings_, mono_block_, color_) { } @@ -87,18 +87,18 @@ void PrettyCompactBlockOutputFormat::writeHeader( for (size_t k = 0; k < max_widths[i] - name_widths[i]; ++k) writeCString(grid_symbols.dash, out); - if (format_settings.pretty.color) + if (color) writeCString("\033[1m", out); writeString(col.name, out); - if (format_settings.pretty.color) + if (color) writeCString("\033[0m", out); } else { - if (format_settings.pretty.color) + if (color) writeCString("\033[1m", out); writeString(col.name, out); - if (format_settings.pretty.color) + if (color) writeCString("\033[0m", out); for (size_t k = 0; k < max_widths[i] - name_widths[i]; ++k) diff --git a/src/Processors/Formats/Impl/PrettyCompactBlockOutputFormat.h b/src/Processors/Formats/Impl/PrettyCompactBlockOutputFormat.h index 432ff0ee120..20cb931f282 100644 --- a/src/Processors/Formats/Impl/PrettyCompactBlockOutputFormat.h +++ b/src/Processors/Formats/Impl/PrettyCompactBlockOutputFormat.h @@ -13,7 +13,7 @@ namespace DB class PrettyCompactBlockOutputFormat : public PrettyBlockOutputFormat { public: - PrettyCompactBlockOutputFormat(WriteBuffer & out_, const Block & header, const FormatSettings & format_settings_, bool mono_block_); + PrettyCompactBlockOutputFormat(WriteBuffer & out_, const Block & header, const FormatSettings & format_settings_, bool mono_block_, bool color); String getName() const override { return "PrettyCompactBlockOutputFormat"; } private: diff --git a/src/Processors/Formats/Impl/PrettySpaceBlockOutputFormat.cpp b/src/Processors/Formats/Impl/PrettySpaceBlockOutputFormat.cpp index 0fb1a413a6c..f8e2ede869f 100644 --- a/src/Processors/Formats/Impl/PrettySpaceBlockOutputFormat.cpp +++ b/src/Processors/Formats/Impl/PrettySpaceBlockOutputFormat.cpp @@ -48,18 +48,18 @@ void PrettySpaceBlockOutputFormat::writeChunk(const Chunk & chunk, PortKind port for (ssize_t k = 0; k < std::max(0z, static_cast(max_widths[i] - name_widths[i])); ++k) writeChar(' ', out); - if (format_settings.pretty.color) + if (color) writeCString("\033[1m", out); writeString(col.name, out); - if (format_settings.pretty.color) + if (color) writeCString("\033[0m", out); } else { - if (format_settings.pretty.color) + if (color) writeCString("\033[1m", out); writeString(col.name, out); - if (format_settings.pretty.color) + if (color) writeCString("\033[0m", out); for (ssize_t k = 0; k < std::max(0z, static_cast(max_widths[i] - name_widths[i])); ++k) diff --git a/src/Processors/Formats/Impl/PrettySpaceBlockOutputFormat.h b/src/Processors/Formats/Impl/PrettySpaceBlockOutputFormat.h index 1cccd5fb476..032bba3e827 100644 --- a/src/Processors/Formats/Impl/PrettySpaceBlockOutputFormat.h +++ b/src/Processors/Formats/Impl/PrettySpaceBlockOutputFormat.h @@ -11,8 +11,8 @@ namespace DB class PrettySpaceBlockOutputFormat : public PrettyBlockOutputFormat { public: - PrettySpaceBlockOutputFormat(WriteBuffer & out_, const Block & header, const FormatSettings & format_settings_, bool mono_block_) - : PrettyBlockOutputFormat(out_, header, format_settings_, mono_block_) {} + PrettySpaceBlockOutputFormat(WriteBuffer & out_, const Block & header, const FormatSettings & format_settings_, bool mono_block_, bool color_) + : PrettyBlockOutputFormat(out_, header, format_settings_, mono_block_, color_) {} String getName() const override { return "PrettySpaceBlockOutputFormat"; } diff --git a/tests/queries/0_stateless/00405_output_format_pretty_color.reference b/tests/queries/0_stateless/00405_output_format_pretty_color.reference index 71495678467..aebdb5f1343 100644 --- a/tests/queries/0_stateless/00405_output_format_pretty_color.reference +++ b/tests/queries/0_stateless/00405_output_format_pretty_color.reference @@ -242,7 +242,7 @@ 9 9 (9,'9') ᴺᵁᴸᴸ auto ┏━━━━━━━┳━━━━━━━┳━━━━━━━━━┳━━━━━━━━━━━━━━━━━┓ -┃ hello ┃ world ┃ tuple  ┃ sometimes_nulls ┃ +┃ hello ┃ world ┃ tuple ┃ sometimes_nulls ┃ ┡━━━━━━━╇━━━━━━━╇━━━━━━━━━╇━━━━━━━━━━━━━━━━━┩ │ 0 │ 0 │ (0,'0') │ ᴺᵁᴸᴸ │ ├───────┼───────┼─────────┼─────────────────┤ @@ -255,7 +255,7 @@ auto │ 4 │ 4 │ (4,'4') │ 1 │ └───────┴───────┴─────────┴─────────────────┘ ┏━━━━━━━┳━━━━━━━┳━━━━━━━━━┳━━━━━━━━━━━━━━━━━┓ -┃ hello ┃ world ┃ tuple  ┃ sometimes_nulls ┃ +┃ hello ┃ world ┃ tuple ┃ sometimes_nulls ┃ ┡━━━━━━━╇━━━━━━━╇━━━━━━━━━╇━━━━━━━━━━━━━━━━━┩ │ 5 │ 5 │ (5,'5') │ 2 │ ├───────┼───────┼─────────┼─────────────────┤ @@ -267,35 +267,35 @@ auto ├───────┼───────┼─────────┼─────────────────┤ │ 9 │ 9 │ (9,'9') │ ᴺᵁᴸᴸ │ └───────┴───────┴─────────┴─────────────────┘ -┌─hello─┬─world─┬─tuple───┬─sometimes_nulls─┐ +┌─hello─┬─world─┬─tuple───┬─sometimes_nulls─┐ │ 0 │ 0 │ (0,'0') │ ᴺᵁᴸᴸ │ │ 1 │ 1 │ (1,'1') │ 1 │ │ 2 │ 2 │ (2,'2') │ 2 │ │ 3 │ 3 │ (3,'3') │ ᴺᵁᴸᴸ │ │ 4 │ 4 │ (4,'4') │ 1 │ └───────┴───────┴─────────┴─────────────────┘ -┌─hello─┬─world─┬─tuple───┬─sometimes_nulls─┐ +┌─hello─┬─world─┬─tuple───┬─sometimes_nulls─┐ │ 5 │ 5 │ (5,'5') │ 2 │ │ 6 │ 6 │ (6,'6') │ ᴺᵁᴸᴸ │ │ 7 │ 7 │ (7,'7') │ 1 │ │ 8 │ 8 │ (8,'8') │ 2 │ │ 9 │ 9 │ (9,'9') │ ᴺᵁᴸᴸ │ └───────┴───────┴─────────┴─────────────────┘ - hello world tuple sometimes_nulls + hello world tuple sometimes_nulls 0 0 (0,'0') ᴺᵁᴸᴸ 1 1 (1,'1') 1 2 2 (2,'2') 2 3 3 (3,'3') ᴺᵁᴸᴸ 4 4 (4,'4') 1 - hello world tuple sometimes_nulls + hello world tuple sometimes_nulls 5 5 (5,'5') 2 6 6 (6,'6') ᴺᵁᴸᴸ 7 7 (7,'7') 1 8 8 (8,'8') 2 9 9 (9,'9') ᴺᵁᴸᴸ -┌─hello─┬─world─┬─tuple───┬─sometimes_nulls─┐ +┌─hello─┬─world─┬─tuple───┬─sometimes_nulls─┐ │ 0 │ 0 │ (0,'0') │ ᴺᵁᴸᴸ │ │ 1 │ 1 │ (1,'1') │ 1 │ │ 2 │ 2 │ (2,'2') │ 2 │ From 0fdba3b83df01e0c92a58b0563337166ee902cf6 Mon Sep 17 00:00:00 2001 From: Blargian Date: Sat, 13 Jan 2024 12:14:54 +0200 Subject: [PATCH 09/23] #58363 - fix failing style check --- src/Formats/FormatFactory.cpp | 2 +- src/Processors/Formats/Impl/PrettyBlockOutputFormat.h | 4 +--- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/src/Formats/FormatFactory.cpp b/src/Formats/FormatFactory.cpp index da3f0c7c02d..15743365d7d 100644 --- a/src/Formats/FormatFactory.cpp +++ b/src/Formats/FormatFactory.cpp @@ -143,7 +143,7 @@ FormatSettings getFormatSettings(ContextPtr context, const Settings & settings) format_settings.parquet.write_batch_size = settings.output_format_parquet_batch_size; format_settings.parquet.local_read_min_bytes_for_seek = settings.input_format_parquet_local_file_min_bytes_for_seek; format_settings.pretty.charset = settings.output_format_pretty_grid_charset.toString() == "ASCII" ? FormatSettings::Pretty::Charset::ASCII : FormatSettings::Pretty::Charset::UTF8; - format_settings.pretty.color = settings.output_format_pretty_color; + format_settings.pretty.color = settings.output_format_pretty_color; format_settings.pretty.max_column_pad_width = settings.output_format_pretty_max_column_pad_width; format_settings.pretty.max_rows = settings.output_format_pretty_max_rows; format_settings.pretty.max_value_width = settings.output_format_pretty_max_value_width; diff --git a/src/Processors/Formats/Impl/PrettyBlockOutputFormat.h b/src/Processors/Formats/Impl/PrettyBlockOutputFormat.h index 9b2087a1399..fcddb17f042 100644 --- a/src/Processors/Formats/Impl/PrettyBlockOutputFormat.h +++ b/src/Processors/Formats/Impl/PrettyBlockOutputFormat.h @@ -20,9 +20,7 @@ class PrettyBlockOutputFormat : public IOutputFormat public: /// no_escapes - do not use ANSI escape sequences - to display in the browser, not in the console. PrettyBlockOutputFormat(WriteBuffer & out_, const Block & header_, const FormatSettings & format_settings_, bool mono_block_, bool color_); - String getName() const override { return "PrettyBlockOutputFormat"; } - protected: void consume(Chunk) override; void consumeTotals(Chunk) override; @@ -59,7 +57,7 @@ protected: bool color; -private: +private: bool mono_block; /// For mono_block == true only Chunk mono_chunk; From ce012d217f27dcbddee141bf8843cfb0d1dedba1 Mon Sep 17 00:00:00 2001 From: Blargian Date: Sat, 13 Jan 2024 13:21:36 +0200 Subject: [PATCH 10/23] #58363 - fix style check failing attempt no.2 --- src/Formats/FormatSettings.h | 2 +- src/Processors/Formats/Impl/PrettyBlockOutputFormat.h | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/Formats/FormatSettings.h b/src/Formats/FormatSettings.h index 873cfdfab09..10da965f126 100644 --- a/src/Formats/FormatSettings.h +++ b/src/Formats/FormatSettings.h @@ -269,7 +269,7 @@ struct FormatSettings UInt64 max_rows = 10000; UInt64 max_column_pad_width = 250; UInt64 max_value_width = 10000; - SettingFieldUInt64Auto color{"auto"}; + SettingFieldUInt64Auto color{"auto"}; bool output_format_pretty_row_numbers = false; diff --git a/src/Processors/Formats/Impl/PrettyBlockOutputFormat.h b/src/Processors/Formats/Impl/PrettyBlockOutputFormat.h index fcddb17f042..73441ec08ee 100644 --- a/src/Processors/Formats/Impl/PrettyBlockOutputFormat.h +++ b/src/Processors/Formats/Impl/PrettyBlockOutputFormat.h @@ -55,7 +55,7 @@ protected: total_rows = 0; } - bool color; + bool color; private: bool mono_block; @@ -73,7 +73,7 @@ void registerPrettyFormatWithNoEscapesAndMonoBlock(FormatFactory & factory, cons const Block & sample, const FormatSettings & format_settings) { - bool color = !no_escapes && format_settings.pretty.color.valueOr(isWritingToTerminal(buf)); + bool color = !no_escapes && format_settings.pretty.color.valueOr(isWritingToTerminal(buf)); return std::make_shared(buf, sample, format_settings, mono_block, color); }); if (!mono_block) From 76c9bf42d744c729dcae4565702cea84b66356c0 Mon Sep 17 00:00:00 2001 From: Antonio Andelic Date: Mon, 15 Jan 2024 12:45:57 +0100 Subject: [PATCH 11/23] Add docs for allocation profiling --- docs/en/operations/allocation-profiling.md | 203 ++++++++++++++++++ .../aspell-ignore/en/aspell-dict.txt | 3 + 2 files changed, 206 insertions(+) create mode 100644 docs/en/operations/allocation-profiling.md diff --git a/docs/en/operations/allocation-profiling.md b/docs/en/operations/allocation-profiling.md new file mode 100644 index 00000000000..67762832edc --- /dev/null +++ b/docs/en/operations/allocation-profiling.md @@ -0,0 +1,203 @@ +--- +slug: /en/operations/allocation-profiling +sidebar_label: "Allocation profiling" +title: "Allocation profiling" +--- + +import Tabs from '@theme/Tabs'; +import TabItem from '@theme/TabItem'; + +# Allocation profiling + +ClickHouse uses [jemalloc](https://github.com/jemalloc/jemalloc) as its global allocator that comes with some tools for allocation sampling and profiling. +To make allocation profiling more convenient, `SYSTEM` commands are provided along 4LW commands in Keeper. + +## Sampling allocations and flushing heap profiles + +If we want to sample and profile allocations in `jemalloc`, we need to start ClickHouse/Keeper with profiling enabled using environment variable `MALLOC_CONF`. + +```sh +MALLOC_CONF=background_thread:true,prof:true +``` + +`jemalloc` will sample allocation and store the information internally. + +We can tell `jemalloc` to flush current profile by running: + + + + + SYSTEM JEMALLOC FLUSH PROFILE + + + + + echo jmfp | nc localhost 9181 + + + + +By default, heap profile file will be generated in `/tmp/jemalloc_clickhouse._pid_._seqnum_.heap` where `_pid_` is the PID of ClickHouse and `_seqnum_` is the global sequence number for the current heap profile. +For Keeper, the default file is `/tmp/jemalloc_keeper._pid_._seqnum_.heap` following the same rules. + +A different location can be defined by appending the `MALLOC_CONF` environment variable with `prof_prefix` option. +For example, if we want to generate profiles in `/data` folder where the prefix for filename will be `my_current_profile` we can run ClickHouse/Keeper with following environment variable: +```sh +MALLOC_CONF=background_thread:true,prof:true,prof_prefix:/data/my_current_profile +``` +Generated file will append to prefix PID and sequence number. + +## Analyzing heap profiles + +After we generated heap profiles, we need to analyze them. +For that, we need to use `jemalloc`'s tool called [jeprof](https://github.com/jemalloc/jemalloc/blob/dev/bin/jeprof.in) which can be installed in multiple ways: +- installing `jemalloc` using system's package manager +- cloning [jemalloc repo](https://github.com/jemalloc/jemalloc) and running autogen.sh from the root folder that will provide you with `jeprof` script inside the `bin` folder + +:::note +`jeprof` uses `addr2line` to generate stacktraces which can be really slow. +If that’s the case, we recommend installing an [alternative implementation](https://github.com/gimli-rs/addr2line) of the tool. + +``` +git clone https://github.com/gimli-rs/addr2line +cd addr2line +cargo b --examples -r +cp ./target/release/examples/addr2line path/to/current/addr2line +``` +::: + +There are many different formats to generate from the heap profile using `jeprof`. +We recommend to run `jeprof --help` to check usage and many different options the tool provides. + +In general, `jeprof` command will look like this: + +```sh +jeprof path/to/binary path/to/heap/profile --output_format [ > output_file] +``` + +If we want to compare which allocations happened between 2 profiles we can set the base argument: + +```sh +jeprof path/to/binary --base path/to/first/heap/profile path/to/second/heap/profile --output_format [ > output_file] +``` + +For example: + +- if we want to generate a text file with each procedure written per line: + +```sh +jeprof path/to/binary path/to/heap/profile --text > result.txt +``` + +- if we want to generate a pdf file with call-graph: + +```sh +jeprof path/to/binary path/to/heap/profile --pdf > result.pdf +``` + +### Generating flame graph + +`jeprof` allows us to generate collapsed stacks for building flame graphs. + +We need to use `--collapsed` argument: + +```sh +jeprof path/to/binary path/to/heap/profile --collapsed > result.collapsed +``` + +After that, we can use many different tools to visualize collapsed stacks. + +Most popular would be [FlameGraph](https://github.com/brendangregg/FlameGraph) which contains a script called `flamegraph.pl`: + +```sh +cat result.collapsed | /path/to/FlameGraph/flamegraph.pl --color=mem --title="Allocation Flame Graph" --width 2400 > result.svg +``` + +Another interesting tool is [speedscope](https://www.speedscope.app/) that allows you to analyze collected stacks in a more interactive way. + +## Controlling allocation profiler during runtime + +If ClickHouse/Keeper were started with enabled profiler, they support additional commands for disabling/enabling allocation profiling during runtime. +Using those commands, it's easier to profile only specific intervals. + +Disable profiler: + + + + + SYSTEM JEMALLOC DISABLE PROFILE + + + + + echo jmdp | nc localhost 9181 + + + + +Enable profiler: + + + + + SYSTEM JEMALLOC ENABLE PROFILE + + + + + echo jmep | nc localhost 9181 + + + + +It's also possible to control the initial state of the profiler by setting `prof_active` option which is enabled by default. +For example, if we don't want to sample allocations during startup but only after we enable the profiler, we can start ClickHouse/Keeper with following environment variable: +```sh +MALLOC_CONF=background_thread:true,prof:true,prof_active:false +``` + +and enable profiler at later point. + +## Additional options for profiler + +`jemalloc` has many different options available related to profiler which can be controlled by modifying `MALLOC_CONF` environment variable. +For example, interval between allocation samples can be controlled with `lg_prof_sample`. +If you want to dump heap profile every N bytes you can enable it using `lg_prof_interval`. + +We recommend to check `jemalloc`s [reference page](https://jemalloc.net/jemalloc.3.html) for such options. + +## Other resources + +ClickHouse/Keeper expose `jemalloc` related metrics in many different ways. + +### System table `asynchronous_metrics` + +```sql +SELECT * +FROM system.asynchronous_metrics +WHERE metric ILIKE '%jemalloc%' +FORMAT Vertical +``` + +[Reference](/en/operations/system-tables/asynchronous_metrics) + +### System table `jemalloc_bins` + +Contains information about memory allocations done via jemalloc allocator in different size classes (bins) aggregated from all arenas. + +[Reference](/en/operations/system-tables/jemalloc_bins) + +### Prometheus + +All `jemalloc` related metrics from `asynchronous_metrics` are also exposed using Prometheus endpoint in both ClickHouse and Keeper. + +[Reference](/en/operations/server-configuration-parameters/settings#prometheus) + +### `jmst` 4LW command in Keeper + +Keeper supports `jmst` 4LW command which returns [basic allocator statistics](https://github.com/jemalloc/jemalloc/wiki/Use-Case%3A-Basic-Allocator-Statistics). + +Example: +```sh +echo jmst | nc localhost 9181 +``` diff --git a/utils/check-style/aspell-ignore/en/aspell-dict.txt b/utils/check-style/aspell-ignore/en/aspell-dict.txt index c4089b21953..075967b3202 100644 --- a/utils/check-style/aspell-ignore/en/aspell-dict.txt +++ b/utils/check-style/aspell-ignore/en/aspell-dict.txt @@ -282,6 +282,7 @@ FilesystemMainPathTotalINodes FilesystemMainPathUsedBytes FilesystemMainPathUsedINodes FixedString +FlameGraph Flink ForEach FreeBSD @@ -1130,6 +1131,7 @@ authenticators autocompletion autodetect autodetected +autogen autogenerate autogenerated autogeneration @@ -1718,6 +1720,7 @@ javaHashUTF jbod jdbc jemalloc +jeprof joinGet json jsonMergePatch From 5f500522a4dc8585e36193c522c3b15d42e31e4d Mon Sep 17 00:00:00 2001 From: Blargian Date: Mon, 15 Jan 2024 16:32:51 +0200 Subject: [PATCH 12/23] #58363 - added setting is_writing_to_terminal to FormatSettings.h, modified PrettyBlockOutputFormat to use this, which is set in FormatFactory.cpp getOutputFormat and getOutputFormatParallelIfPossible --- src/Formats/FormatFactory.cpp | 3 +++ src/Formats/FormatSettings.h | 1 + src/Processors/Formats/Impl/PrettyBlockOutputFormat.h | 3 +-- 3 files changed, 5 insertions(+), 2 deletions(-) diff --git a/src/Formats/FormatFactory.cpp b/src/Formats/FormatFactory.cpp index 15743365d7d..5013cfb59f4 100644 --- a/src/Formats/FormatFactory.cpp +++ b/src/Formats/FormatFactory.cpp @@ -5,6 +5,7 @@ #include #include #include +#include #include #include #include @@ -448,6 +449,7 @@ OutputFormatPtr FormatFactory::getOutputFormatParallelIfPossible( throw Exception(ErrorCodes::FORMAT_IS_NOT_SUITABLE_FOR_OUTPUT, "Format {} is not suitable for output", name); auto format_settings = _format_settings ? *_format_settings : getFormatSettings(context); + format_settings.is_writing_to_terminal = isWritingToTerminal(buf); const Settings & settings = context->getSettingsRef(); @@ -489,6 +491,7 @@ OutputFormatPtr FormatFactory::getOutputFormat( auto format_settings = _format_settings ? *_format_settings : getFormatSettings(context); format_settings.max_threads = context->getSettingsRef().max_threads; + format_settings.is_writing_to_terminal = format_settings.is_writing_to_terminal = isWritingToTerminal(buf); /** TODO: Materialization is needed, because formats can use the functions `IDataType`, * which only work with full columns. diff --git a/src/Formats/FormatSettings.h b/src/Formats/FormatSettings.h index 10da965f126..a960be9edba 100644 --- a/src/Formats/FormatSettings.h +++ b/src/Formats/FormatSettings.h @@ -34,6 +34,7 @@ struct FormatSettings bool null_as_default = true; bool decimal_trailing_zeros = false; bool defaults_for_omitted_fields = true; + bool is_writing_to_terminal = false; bool seekable_read = true; UInt64 max_rows_to_read_for_schema_inference = 25000; diff --git a/src/Processors/Formats/Impl/PrettyBlockOutputFormat.h b/src/Processors/Formats/Impl/PrettyBlockOutputFormat.h index 73441ec08ee..253a6a958cc 100644 --- a/src/Processors/Formats/Impl/PrettyBlockOutputFormat.h +++ b/src/Processors/Formats/Impl/PrettyBlockOutputFormat.h @@ -4,7 +4,6 @@ #include #include #include -#include namespace DB { @@ -73,7 +72,7 @@ void registerPrettyFormatWithNoEscapesAndMonoBlock(FormatFactory & factory, cons const Block & sample, const FormatSettings & format_settings) { - bool color = !no_escapes && format_settings.pretty.color.valueOr(isWritingToTerminal(buf)); + bool color = !no_escapes && format_settings.pretty.color.valueOr(format_settings.is_writing_to_terminal); return std::make_shared(buf, sample, format_settings, mono_block, color); }); if (!mono_block) From b008a1ea80e6013cbfb7c3c2c6a80cd64274d10d Mon Sep 17 00:00:00 2001 From: Blargian Date: Mon, 15 Jan 2024 19:15:15 +0200 Subject: [PATCH 13/23] #58363 - add change to SettingsChangesHistory.h and fix failing tests which use pretty by setting output_format_pretty_color=1 --- src/Core/SettingsChangesHistory.h | 1 + .../0_stateless/00085_visible_width_of_tuple_of_dates.sql | 1 + tests/queries/0_stateless/00098_k_union_all.sql | 1 + tests/queries/0_stateless/00298_enum_width_and_cast.sql | 1 + tests/queries/0_stateless/00405_PrettyCompactMonoBlock.sh | 8 ++++---- tests/queries/0_stateless/00405_pretty_formats.sql | 2 ++ .../0_stateless/00476_pretty_formats_and_widths.sql | 1 + tests/queries/0_stateless/00818_inner_join_bug_3567.sql | 1 + tests/queries/0_stateless/01018_ambiguous_column.sql | 1 + tests/queries/0_stateless/01074_partial_revokes.sql | 4 ++-- .../queries/0_stateless/01293_pretty_max_value_width.sql | 1 + tests/queries/0_stateless/01472_many_rows_in_totals.sql | 2 ++ .../01509_output_format_pretty_row_numbers.sql | 1 + tests/queries/0_stateless/01670_neighbor_lc_bug.sql | 1 + .../0_stateless/01671_merge_join_and_constants.sql | 1 + tests/queries/0_stateless/02152_bool_type.sql | 2 ++ tests/queries/0_stateless/02375_pretty_formats.sql.j2 | 2 +- ...y_blocks_from_ConvertingAggregatedToChunksTransform.sh | 4 ++-- .../02798_explain_settings_not_applied_bug.sql | 1 + .../0_stateless/02896_union_distinct_http_format.sh | 4 ++-- 20 files changed, 29 insertions(+), 11 deletions(-) diff --git a/src/Core/SettingsChangesHistory.h b/src/Core/SettingsChangesHistory.h index aad57ffebb7..fb0a2127e58 100644 --- a/src/Core/SettingsChangesHistory.h +++ b/src/Core/SettingsChangesHistory.h @@ -85,6 +85,7 @@ static std::map sett {"input_format_parquet_allow_missing_columns", false, true, "Allow missing columns in Parquet files by default"}, {"input_format_orc_allow_missing_columns", false, true, "Allow missing columns in ORC files by default"}, {"input_format_arrow_allow_missing_columns", false, true, "Allow missing columns in Arrow files by default"}}}, + {"output_format_pretty_color",true,"auto","Setting is changed to allow also for auto value, disabling ANSI escapes if output is not a tty"} {"23.9", {{"optimize_group_by_constant_keys", false, true, "Optimize group by constant keys by default"}, {"input_format_json_try_infer_named_tuples_from_objects", false, true, "Try to infer named Tuples from JSON objects by default"}, {"input_format_json_read_numbers_as_strings", false, true, "Allow to read numbers as strings in JSON formats by default"}, diff --git a/tests/queries/0_stateless/00085_visible_width_of_tuple_of_dates.sql b/tests/queries/0_stateless/00085_visible_width_of_tuple_of_dates.sql index 56e93d24087..09208b9151b 100644 --- a/tests/queries/0_stateless/00085_visible_width_of_tuple_of_dates.sql +++ b/tests/queries/0_stateless/00085_visible_width_of_tuple_of_dates.sql @@ -1 +1,2 @@ +SET output_format_pretty_color=1; SELECT (toDate('2000-01-01'), toDate('2000-01-01')) AS x FORMAT PrettyCompact; diff --git a/tests/queries/0_stateless/00098_k_union_all.sql b/tests/queries/0_stateless/00098_k_union_all.sql index 311e5bb19c4..059d27075d7 100644 --- a/tests/queries/0_stateless/00098_k_union_all.sql +++ b/tests/queries/0_stateless/00098_k_union_all.sql @@ -1,3 +1,4 @@ +SET output_format_pretty_color=1; SELECT 1 FORMAT PrettySpace; SELECT 1 UNION ALL SELECT 1 FORMAT PrettySpace; SELECT 1 UNION ALL SELECT 1 UNION ALL SELECT 1 FORMAT PrettySpace; diff --git a/tests/queries/0_stateless/00298_enum_width_and_cast.sql b/tests/queries/0_stateless/00298_enum_width_and_cast.sql index 3dda3e41270..35241f5124d 100644 --- a/tests/queries/0_stateless/00298_enum_width_and_cast.sql +++ b/tests/queries/0_stateless/00298_enum_width_and_cast.sql @@ -1,5 +1,6 @@ DROP TABLE IF EXISTS enum; +SET output_format_pretty_color=1; CREATE TABLE enum (x Enum8('Hello' = -100, '\\' = 0, '\t\\t' = 111), y UInt8) ENGINE = TinyLog; INSERT INTO enum (y) VALUES (0); SELECT * FROM enum ORDER BY x, y FORMAT PrettyCompact; diff --git a/tests/queries/0_stateless/00405_PrettyCompactMonoBlock.sh b/tests/queries/0_stateless/00405_PrettyCompactMonoBlock.sh index a5eca3d987e..710774700e9 100755 --- a/tests/queries/0_stateless/00405_PrettyCompactMonoBlock.sh +++ b/tests/queries/0_stateless/00405_PrettyCompactMonoBlock.sh @@ -5,10 +5,10 @@ CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) . "$CURDIR"/../shell_config.sh echo 'one block' -${CLICKHOUSE_LOCAL} --query="SELECT * FROM numbers(2)" --format PrettyCompactMonoBlock +${CLICKHOUSE_LOCAL} --query="SELECT * FROM numbers(2) SETTINGS output_format_pretty_color=1" --format PrettyCompactMonoBlock echo 'two blocks' -${CLICKHOUSE_LOCAL} --query="SELECT * FROM numbers(1) UNION ALL SELECT * FROM numbers(1)" --format PrettyCompactMonoBlock +${CLICKHOUSE_LOCAL} --query="SELECT * FROM numbers(1) UNION ALL SELECT * FROM numbers(1) SETTINGS output_format_pretty_color=1" --format PrettyCompactMonoBlock echo 'extremes' -${CLICKHOUSE_LOCAL} --query="SELECT * FROM numbers(3)" --format PrettyCompactMonoBlock --extremes=1 +${CLICKHOUSE_LOCAL} --query="SELECT * FROM numbers(3) SETTINGS output_format_pretty_color=1" --format PrettyCompactMonoBlock --extremes=1 echo 'totals' -${CLICKHOUSE_LOCAL} --query="SELECT sum(number) FROM numbers(3) GROUP BY number%2 WITH TOTALS ORDER BY number%2" --format PrettyCompactMonoBlock +${CLICKHOUSE_LOCAL} --query="SELECT sum(number) FROM numbers(3) GROUP BY number%2 WITH TOTALS ORDER BY number%2 SETTINGS output_format_pretty_color=1" --format PrettyCompactMonoBlock diff --git a/tests/queries/0_stateless/00405_pretty_formats.sql b/tests/queries/0_stateless/00405_pretty_formats.sql index 3c8af776278..00bb09a1c30 100644 --- a/tests/queries/0_stateless/00405_pretty_formats.sql +++ b/tests/queries/0_stateless/00405_pretty_formats.sql @@ -1,3 +1,5 @@ +SET output_format_pretty_color = 1; + SELECT number AS hello, toString(number) AS world, (hello, world) AS tuple, nullIf(hello % 3, 0) AS sometimes_nulls FROM system.numbers LIMIT 10 SETTINGS max_block_size = 5 FORMAT Pretty; SELECT number AS hello, toString(number) AS world, (hello, world) AS tuple, nullIf(hello % 3, 0) AS sometimes_nulls FROM system.numbers LIMIT 10 SETTINGS max_block_size = 5 FORMAT PrettyCompact; SELECT number AS hello, toString(number) AS world, (hello, world) AS tuple, nullIf(hello % 3, 0) AS sometimes_nulls FROM system.numbers LIMIT 10 SETTINGS max_block_size = 5 FORMAT PrettySpace; diff --git a/tests/queries/0_stateless/00476_pretty_formats_and_widths.sql b/tests/queries/0_stateless/00476_pretty_formats_and_widths.sql index be98d9ab5cc..ece046b738e 100644 --- a/tests/queries/0_stateless/00476_pretty_formats_and_widths.sql +++ b/tests/queries/0_stateless/00476_pretty_formats_and_widths.sql @@ -1,3 +1,4 @@ +SET output_format_pretty_color=1; SELECT toUInt64(round(exp10(number))) AS x, toString(x) AS s FROM system.numbers LIMIT 10 FORMAT Pretty; SELECT toUInt64(round(exp10(number))) AS x, toString(x) AS s FROM system.numbers LIMIT 10 FORMAT PrettyCompact; SELECT toUInt64(round(exp10(number))) AS x, toString(x) AS s FROM system.numbers LIMIT 10 FORMAT PrettySpace; diff --git a/tests/queries/0_stateless/00818_inner_join_bug_3567.sql b/tests/queries/0_stateless/00818_inner_join_bug_3567.sql index cc0b63f9def..2dec5ce3221 100644 --- a/tests/queries/0_stateless/00818_inner_join_bug_3567.sql +++ b/tests/queries/0_stateless/00818_inner_join_bug_3567.sql @@ -1,3 +1,4 @@ +SET output_format_pretty_color = 1; SET allow_experimental_analyzer = 1; DROP TABLE IF EXISTS table1; diff --git a/tests/queries/0_stateless/01018_ambiguous_column.sql b/tests/queries/0_stateless/01018_ambiguous_column.sql index 620bdb6ba3f..a94e1cd4601 100644 --- a/tests/queries/0_stateless/01018_ambiguous_column.sql +++ b/tests/queries/0_stateless/01018_ambiguous_column.sql @@ -1,3 +1,4 @@ +SET output_format_pretty_color=1; SET allow_experimental_analyzer = 1; select * from system.one cross join system.one; diff --git a/tests/queries/0_stateless/01074_partial_revokes.sql b/tests/queries/0_stateless/01074_partial_revokes.sql index 8c92b9511c7..973d77b3f63 100644 --- a/tests/queries/0_stateless/01074_partial_revokes.sql +++ b/tests/queries/0_stateless/01074_partial_revokes.sql @@ -43,7 +43,7 @@ REVOKE SELECT ON db.* FROM test_user_01074; GRANT SELECT ON db.table TO test_user_01074; REVOKE SELECT(col1) ON db.table FROM test_user_01074; SHOW GRANTS FOR test_user_01074; -SELECT * FROM system.grants WHERE user_name = 'test_user_01074' format Pretty; +SELECT * FROM system.grants WHERE user_name = 'test_user_01074' SETTINGS output_format_pretty_color=1 FORMAT Pretty; SELECT '--cleanup'; REVOKE SELECT ON *.* FROM test_user_01074; @@ -73,7 +73,7 @@ SELECT '--grant option 1'; GRANT SELECT ON *.* TO test_user_01074 WITH GRANT OPTION; REVOKE GRANT OPTION FOR SELECT(col1) ON db.table FROM test_user_01074; SHOW GRANTS FOR test_user_01074; -SELECT * FROM system.grants WHERE user_name = 'test_user_01074' format Pretty; +SELECT * FROM system.grants WHERE user_name = 'test_user_01074' SETTINGS output_format_pretty_color=1 FORMAT Pretty; SELECT '--cleanup'; REVOKE SELECT ON *.* FROM test_user_01074; diff --git a/tests/queries/0_stateless/01293_pretty_max_value_width.sql b/tests/queries/0_stateless/01293_pretty_max_value_width.sql index 992aec06f0a..2c9c56d0076 100644 --- a/tests/queries/0_stateless/01293_pretty_max_value_width.sql +++ b/tests/queries/0_stateless/01293_pretty_max_value_width.sql @@ -1,3 +1,4 @@ +SET output_format_pretty_color=1; SELECT 'привет' AS x, 'мир' AS y FORMAT Pretty; SET output_format_pretty_max_value_width = 5; diff --git a/tests/queries/0_stateless/01472_many_rows_in_totals.sql b/tests/queries/0_stateless/01472_many_rows_in_totals.sql index bea8c255f21..f3d3a1fcca2 100644 --- a/tests/queries/0_stateless/01472_many_rows_in_totals.sql +++ b/tests/queries/0_stateless/01472_many_rows_in_totals.sql @@ -1,3 +1,5 @@ +set output_format_pretty_color=1; + -- Disable external aggregation because it may produce several blocks instead of one. set max_bytes_before_external_group_by = 0; set output_format_write_statistics = 0; diff --git a/tests/queries/0_stateless/01509_output_format_pretty_row_numbers.sql b/tests/queries/0_stateless/01509_output_format_pretty_row_numbers.sql index f8ec0be74d7..3536b628ef2 100644 --- a/tests/queries/0_stateless/01509_output_format_pretty_row_numbers.sql +++ b/tests/queries/0_stateless/01509_output_format_pretty_row_numbers.sql @@ -1,3 +1,4 @@ +SET output_format_pretty_color=1; SELECT * FROM numbers(10) FORMAT Pretty; SELECT * FROM numbers(10) FORMAT PrettyCompact; SELECT * FROM numbers(10) FORMAT PrettyCompactMonoBlock; diff --git a/tests/queries/0_stateless/01670_neighbor_lc_bug.sql b/tests/queries/0_stateless/01670_neighbor_lc_bug.sql index f216befbb06..3cb194ccc64 100644 --- a/tests/queries/0_stateless/01670_neighbor_lc_bug.sql +++ b/tests/queries/0_stateless/01670_neighbor_lc_bug.sql @@ -40,6 +40,7 @@ FROM ORDER BY val_string, rowNr ) ORDER BY rowNr, val_string, str_m1, str_p1, val_low, low_m1, low_p1 +SETTINGS output_format_pretty_color=1 format PrettyCompact; drop table if exists neighbor_test; diff --git a/tests/queries/0_stateless/01671_merge_join_and_constants.sql b/tests/queries/0_stateless/01671_merge_join_and_constants.sql index 5cabd6f7f06..7a84bd4e97a 100644 --- a/tests/queries/0_stateless/01671_merge_join_and_constants.sql +++ b/tests/queries/0_stateless/01671_merge_join_and_constants.sql @@ -1,3 +1,4 @@ +SET output_format_pretty_color=1; SET allow_experimental_analyzer = 1; DROP TABLE IF EXISTS table1; diff --git a/tests/queries/0_stateless/02152_bool_type.sql b/tests/queries/0_stateless/02152_bool_type.sql index e9efde0795f..1ed3620c149 100644 --- a/tests/queries/0_stateless/02152_bool_type.sql +++ b/tests/queries/0_stateless/02152_bool_type.sql @@ -1,3 +1,5 @@ +SET output_format_pretty_color=1; + SELECT CAST('True', 'Bool'); SELECT CAST('TrUe', 'Bool'); SELECT CAST('true', 'Bool'); diff --git a/tests/queries/0_stateless/02375_pretty_formats.sql.j2 b/tests/queries/0_stateless/02375_pretty_formats.sql.j2 index cc61346d267..55462ea6b61 100644 --- a/tests/queries/0_stateless/02375_pretty_formats.sql.j2 +++ b/tests/queries/0_stateless/02375_pretty_formats.sql.j2 @@ -3,6 +3,6 @@ 'PrettySpaceNoEscapesMonoBlock'] -%} select '{{ format }}'; -select number as x, number + 1 as y from numbers(4) settings max_block_size=2 format {{ format }}; +select number as x, number + 1 as y from numbers(4) settings max_block_size=2, output_format_pretty_color=1 format {{ format }}; {% endfor -%} diff --git a/tests/queries/0_stateless/02418_do_not_return_empty_blocks_from_ConvertingAggregatedToChunksTransform.sh b/tests/queries/0_stateless/02418_do_not_return_empty_blocks_from_ConvertingAggregatedToChunksTransform.sh index 32693adff24..847e9682bf5 100755 --- a/tests/queries/0_stateless/02418_do_not_return_empty_blocks_from_ConvertingAggregatedToChunksTransform.sh +++ b/tests/queries/0_stateless/02418_do_not_return_empty_blocks_from_ConvertingAggregatedToChunksTransform.sh @@ -13,7 +13,7 @@ ${CLICKHOUSE_CURL} \ from numbers_mt(1e6) where number = 42 group by number - settings max_threads = 10, max_bytes_before_external_group_by = 1, group_by_two_level_threshold = 1 + settings max_threads = 10, max_bytes_before_external_group_by = 1, group_by_two_level_threshold = 1, output_format_pretty_color=1 format PrettyCompact" ${CLICKHOUSE_CURL} \ @@ -24,5 +24,5 @@ ${CLICKHOUSE_CURL} \ from numbers_mt(1e6) where number = 42 group by number - settings max_threads = 10, max_bytes_before_external_group_by = 0, group_by_two_level_threshold = 1 + settings max_threads = 10, max_bytes_before_external_group_by = 0, group_by_two_level_threshold = 1, output_format_pretty_color=1 format PrettyCompact" diff --git a/tests/queries/0_stateless/02798_explain_settings_not_applied_bug.sql b/tests/queries/0_stateless/02798_explain_settings_not_applied_bug.sql index 76f2129abfa..5b9976714ea 100644 --- a/tests/queries/0_stateless/02798_explain_settings_not_applied_bug.sql +++ b/tests/queries/0_stateless/02798_explain_settings_not_applied_bug.sql @@ -1,3 +1,4 @@ +SET output_format_pretty_color=1; SET read_in_order_two_level_merge_threshold=1000000; DROP TABLE IF EXISTS t; diff --git a/tests/queries/0_stateless/02896_union_distinct_http_format.sh b/tests/queries/0_stateless/02896_union_distinct_http_format.sh index bb35800e39d..9426321e62f 100755 --- a/tests/queries/0_stateless/02896_union_distinct_http_format.sh +++ b/tests/queries/0_stateless/02896_union_distinct_http_format.sh @@ -4,5 +4,5 @@ CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) # shellcheck source=../shell_config.sh . "$CURDIR"/../shell_config.sh -curl -d@- -sS "${CLICKHOUSE_URL}" <<< 'SELECT 1 UNION DISTINCT SELECT 1 FORMAT PrettyCompactMonoBlock' -curl -d@- -sS "${CLICKHOUSE_URL}" <<< 'SELECT * FROM (SELECT 1 as a UNION DISTINCT SELECT 2 as a) ORDER BY a FORMAT PrettyCompactMonoBlock' +curl -d@- -sS "${CLICKHOUSE_URL}" <<< 'SELECT 1 UNION DISTINCT SELECT 1 SETTINGS output_format_pretty_color=1 FORMAT PrettyCompactMonoBlock' +curl -d@- -sS "${CLICKHOUSE_URL}" <<< 'SELECT * FROM (SELECT 1 as a UNION DISTINCT SELECT 2 as a) ORDER BY a SETTINGS output_format_pretty_color=1 FORMAT PrettyCompactMonoBlock' From d044b95901a9587de7712ad40273f85676f5ab3d Mon Sep 17 00:00:00 2001 From: Shaun Struwig <41984034+Blargian@users.noreply.github.com> Date: Mon, 15 Jan 2024 19:27:55 +0200 Subject: [PATCH 14/23] Update src/Core/SettingsChangesHistory.h Co-authored-by: Kruglov Pavel <48961922+Avogar@users.noreply.github.com> --- src/Core/SettingsChangesHistory.h | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/Core/SettingsChangesHistory.h b/src/Core/SettingsChangesHistory.h index fb0a2127e58..ec3c88e1bc7 100644 --- a/src/Core/SettingsChangesHistory.h +++ b/src/Core/SettingsChangesHistory.h @@ -84,8 +84,8 @@ static std::map sett {"23.12", {{"allow_suspicious_ttl_expressions", true, false, "It is a new setting, and in previous versions the behavior was equivalent to allowing."}, {"input_format_parquet_allow_missing_columns", false, true, "Allow missing columns in Parquet files by default"}, {"input_format_orc_allow_missing_columns", false, true, "Allow missing columns in ORC files by default"}, - {"input_format_arrow_allow_missing_columns", false, true, "Allow missing columns in Arrow files by default"}}}, - {"output_format_pretty_color",true,"auto","Setting is changed to allow also for auto value, disabling ANSI escapes if output is not a tty"} + {"input_format_arrow_allow_missing_columns", false, true, "Allow missing columns in Arrow files by default"}, + {"output_format_pretty_color", true, "auto", "Setting is changed to allow also for auto value, disabling ANSI escapes if output is not a tty"}}}, {"23.9", {{"optimize_group_by_constant_keys", false, true, "Optimize group by constant keys by default"}, {"input_format_json_try_infer_named_tuples_from_objects", false, true, "Try to infer named Tuples from JSON objects by default"}, {"input_format_json_read_numbers_as_strings", false, true, "Allow to read numbers as strings in JSON formats by default"}, From e708b6ed9413c346851324ceb8d6e213f6c0e9e2 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Tue, 16 Jan 2024 09:45:20 +0100 Subject: [PATCH 15/23] Fix bad log message --- src/Common/AsyncLoader.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Common/AsyncLoader.cpp b/src/Common/AsyncLoader.cpp index 0e0fa25e7a1..c467862f58f 100644 --- a/src/Common/AsyncLoader.cpp +++ b/src/Common/AsyncLoader.cpp @@ -36,7 +36,7 @@ static constexpr size_t PRINT_MESSAGE_EACH_N_SECONDS = 5; void logAboutProgress(Poco::Logger * log, size_t processed, size_t total, AtomicStopwatch & watch) { - if (processed % PRINT_MESSAGE_EACH_N_OBJECTS == 0 || watch.compareAndRestart(PRINT_MESSAGE_EACH_N_SECONDS)) + if (total && (processed % PRINT_MESSAGE_EACH_N_OBJECTS == 0 || watch.compareAndRestart(PRINT_MESSAGE_EACH_N_SECONDS))) { LOG_INFO(log, "Processed: {}%", processed * 100.0 / total); watch.restart(); From f33f374ac5ce6aa77922ce308afa55f6964638b9 Mon Sep 17 00:00:00 2001 From: Antonio Andelic Date: Tue, 16 Jan 2024 15:09:49 +0100 Subject: [PATCH 16/23] Make Kovi happy --- docs/en/operations/allocation-profiling.md | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/docs/en/operations/allocation-profiling.md b/docs/en/operations/allocation-profiling.md index 67762832edc..64b4106a7e1 100644 --- a/docs/en/operations/allocation-profiling.md +++ b/docs/en/operations/allocation-profiling.md @@ -89,7 +89,7 @@ For example: jeprof path/to/binary path/to/heap/profile --text > result.txt ``` -- if we want to generate a pdf file with call-graph: +- if we want to generate a PDF file with call-graph: ```sh jeprof path/to/binary path/to/heap/profile --pdf > result.pdf @@ -156,7 +156,7 @@ For example, if we don't want to sample allocations during startup but only afte MALLOC_CONF=background_thread:true,prof:true,prof_active:false ``` -and enable profiler at later point. +and enable profiler at a later point. ## Additional options for profiler @@ -170,6 +170,10 @@ We recommend to check `jemalloc`s [reference page](https://jemalloc.net/jemalloc ClickHouse/Keeper expose `jemalloc` related metrics in many different ways. +:::warning Warning +It's important to be aware that none of these metrics are synchronized with each other and values may drift. +::: + ### System table `asynchronous_metrics` ```sql From 4ce6b64d62a9c4205c12bc33bf154ee96f0b143d Mon Sep 17 00:00:00 2001 From: Antonio Andelic Date: Tue, 16 Jan 2024 15:16:13 +0000 Subject: [PATCH 17/23] Fix 00089_group_by_arrays_of_fixed with external aggregation --- src/Interpreters/Aggregator.cpp | 4 ++-- tests/queries/0_stateless/00089_group_by_arrays_of_fixed.sql | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/Interpreters/Aggregator.cpp b/src/Interpreters/Aggregator.cpp index f0e64b5cbcf..e4856c33988 100644 --- a/src/Interpreters/Aggregator.cpp +++ b/src/Interpreters/Aggregator.cpp @@ -1797,8 +1797,8 @@ void Aggregator::writeToTemporaryFile(AggregatedDataVariants & data_variants, si rows, ReadableSize(uncompressed_size), ReadableSize(compressed_size), - static_cast(uncompressed_size) / rows, - static_cast(compressed_size) / rows, + rows ? static_cast(uncompressed_size) / rows : 0.0, + rows ? static_cast(compressed_size) / rows : 0.0, static_cast(uncompressed_size) / compressed_size, static_cast(rows) / elapsed_seconds, ReadableSize(static_cast(uncompressed_size) / elapsed_seconds), diff --git a/tests/queries/0_stateless/00089_group_by_arrays_of_fixed.sql b/tests/queries/0_stateless/00089_group_by_arrays_of_fixed.sql index 60ec1cb3396..a068671b999 100644 --- a/tests/queries/0_stateless/00089_group_by_arrays_of_fixed.sql +++ b/tests/queries/0_stateless/00089_group_by_arrays_of_fixed.sql @@ -1 +1 @@ -SELECT arr, count() AS c FROM (SELECT arrayMap(x -> x % 2, groupArray(number)) AS arr FROM (SELECT number FROM system.numbers LIMIT 10000) GROUP BY number % ((number * 0xABCDEF0123456789 % 1234) + 1)) GROUP BY arr ORDER BY c DESC, arr ASC; +SELECT arr, count() AS c FROM (SELECT arrayMap(x -> x % 2, arraySort(groupArray(number))) AS arr FROM (SELECT number FROM system.numbers LIMIT 10000) GROUP BY number % ((number * 0xABCDEF0123456789 % 1234) + 1)) GROUP BY arr ORDER BY c DESC, arr ASC; \ No newline at end of file From c325f08e798579e61a13579213459da406738d59 Mon Sep 17 00:00:00 2001 From: Dmitry Novik Date: Tue, 16 Jan 2024 15:34:45 +0000 Subject: [PATCH 18/23] Fix RPN construction for indexHint --- src/Storages/MergeTree/RPNBuilder.cpp | 16 +++++++++++++++ ...02962_indexHint_rpn_construction.reference | 0 .../02962_indexHint_rpn_construction.sql | 20 +++++++++++++++++++ 3 files changed, 36 insertions(+) create mode 100644 tests/queries/0_stateless/02962_indexHint_rpn_construction.reference create mode 100644 tests/queries/0_stateless/02962_indexHint_rpn_construction.sql diff --git a/src/Storages/MergeTree/RPNBuilder.cpp b/src/Storages/MergeTree/RPNBuilder.cpp index 3ffef1de718..8c5606e9b8a 100644 --- a/src/Storages/MergeTree/RPNBuilder.cpp +++ b/src/Storages/MergeTree/RPNBuilder.cpp @@ -14,7 +14,9 @@ #include #include +#include #include +#include #include @@ -390,6 +392,13 @@ size_t RPNBuilderFunctionTreeNode::getArgumentsSize() const } else { + if (dag_node->function_base->getName() == "indexHint") + { + const auto * adaptor = typeid_cast(dag_node->function_base.get()); + const auto * index_hint = typeid_cast(adaptor->getFunction().get()); + return index_hint->getActions()->getOutputs().size(); + } + return dag_node->children.size(); } } @@ -409,6 +418,13 @@ RPNBuilderTreeNode RPNBuilderFunctionTreeNode::getArgumentAt(size_t index) const } else { + if (dag_node->function_base->getName() == "indexHint") + { + const auto * adaptor = typeid_cast(dag_node->function_base.get()); + const auto * index_hint = typeid_cast(adaptor->getFunction().get()); + return RPNBuilderTreeNode(index_hint->getActions()->getOutputs()[index], tree_context); + } + return RPNBuilderTreeNode(dag_node->children[index], tree_context); } } diff --git a/tests/queries/0_stateless/02962_indexHint_rpn_construction.reference b/tests/queries/0_stateless/02962_indexHint_rpn_construction.reference new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tests/queries/0_stateless/02962_indexHint_rpn_construction.sql b/tests/queries/0_stateless/02962_indexHint_rpn_construction.sql new file mode 100644 index 00000000000..3532bea57fa --- /dev/null +++ b/tests/queries/0_stateless/02962_indexHint_rpn_construction.sql @@ -0,0 +1,20 @@ +CREATE TABLE tab +( + `foo` Array(LowCardinality(String)), + INDEX idx foo TYPE bloom_filter GRANULARITY 1 +) +ENGINE = MergeTree +PRIMARY KEY tuple(); + +INSERT INTO tab SELECT if(number % 2, ['value'], []) +FROM system.numbers +LIMIT 10000; + +SELECT * +FROM tab +PREWHERE indexHint(indexHint(-1, 0.)) +WHERE has(foo, 'b'); + +SELECT * +FROM tab +PREWHERE indexHint(0); From f3017959ffb6ec0e9e58f7513f48379a8d3ab183 Mon Sep 17 00:00:00 2001 From: Dmitry Novik Date: Tue, 16 Jan 2024 15:41:02 +0000 Subject: [PATCH 19/23] Analyzer: add test with GROUP BY on shards --- ...lyzer_resolve_group_by_on_shards.reference | 0 ...62_analyzer_resolve_group_by_on_shards.sql | 20 +++++++++++++++++++ 2 files changed, 20 insertions(+) create mode 100644 tests/queries/0_stateless/02962_analyzer_resolve_group_by_on_shards.reference create mode 100644 tests/queries/0_stateless/02962_analyzer_resolve_group_by_on_shards.sql diff --git a/tests/queries/0_stateless/02962_analyzer_resolve_group_by_on_shards.reference b/tests/queries/0_stateless/02962_analyzer_resolve_group_by_on_shards.reference new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tests/queries/0_stateless/02962_analyzer_resolve_group_by_on_shards.sql b/tests/queries/0_stateless/02962_analyzer_resolve_group_by_on_shards.sql new file mode 100644 index 00000000000..00a80067994 --- /dev/null +++ b/tests/queries/0_stateless/02962_analyzer_resolve_group_by_on_shards.sql @@ -0,0 +1,20 @@ +SELECT NULL AND (toDate(-2147483647, NULL) AND NULL) +FROM remote('127.0.0.{1,2}', view( + SELECT + NULL AND NULL, + NULL, + toDate(toDate('0.0001048577', toDate(NULL, 10 AND (toDate(257, 9223372036854775807, NULL) AND NULL AND NULL) AND NULL, 7, NULL), NULL, NULL) AND NULL AND -2147483648, NULL, NULL) AND NULL + FROM system.one + WHERE toDate(toDate(NULL, NULL, NULL), NULL) + GROUP BY + GROUPING SETS ((NULL)) +)); + +SELECT NULL AND (toDate(-2147483647, NULL) AND NULL) +FROM remote('127.0.0.{1,2}', view( + SELECT NULL + FROM system.one + WHERE toDate(toDate(NULL, NULL, NULL), NULL) + GROUP BY + GROUPING SETS (('')) +)); From 80291ec3f2d435a1eb8d5b38752fc42337351d14 Mon Sep 17 00:00:00 2001 From: Dmitry Novik Date: Tue, 16 Jan 2024 16:59:04 +0100 Subject: [PATCH 20/23] Add comments --- src/Storages/MergeTree/RPNBuilder.cpp | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/Storages/MergeTree/RPNBuilder.cpp b/src/Storages/MergeTree/RPNBuilder.cpp index 8c5606e9b8a..dc8c6b0c230 100644 --- a/src/Storages/MergeTree/RPNBuilder.cpp +++ b/src/Storages/MergeTree/RPNBuilder.cpp @@ -392,6 +392,8 @@ size_t RPNBuilderFunctionTreeNode::getArgumentsSize() const } else { + // indexHint arguments are stored inside of `FunctionIndexHint` class, + // because they are used only for index analysis. if (dag_node->function_base->getName() == "indexHint") { const auto * adaptor = typeid_cast(dag_node->function_base.get()); @@ -418,6 +420,8 @@ RPNBuilderTreeNode RPNBuilderFunctionTreeNode::getArgumentAt(size_t index) const } else { + // indexHint arguments are stored inside of `FunctionIndexHint` class, + // because they are used only for index analysis. if (dag_node->function_base->getName() == "indexHint") { const auto * adaptor = typeid_cast(dag_node->function_base.get()); From 1b81efc33774e91a8dbd97e0f32f5bb73b72a9f8 Mon Sep 17 00:00:00 2001 From: Blargian Date: Tue, 16 Jan 2024 19:48:23 +0200 Subject: [PATCH 21/23] fix failing 02421 stateless and (hopefully) fix failing integration tests --- src/Core/SettingsChangesHistory.h | 4 ++-- tests/integration/test_storage_kafka/test.py | 2 +- tests/integration/test_storage_rabbitmq/test.py | 2 +- .../0_stateless/02421_formats_with_totals_and_extremes.sql.j2 | 1 + 4 files changed, 5 insertions(+), 4 deletions(-) diff --git a/src/Core/SettingsChangesHistory.h b/src/Core/SettingsChangesHistory.h index fb0a2127e58..ec3c88e1bc7 100644 --- a/src/Core/SettingsChangesHistory.h +++ b/src/Core/SettingsChangesHistory.h @@ -84,8 +84,8 @@ static std::map sett {"23.12", {{"allow_suspicious_ttl_expressions", true, false, "It is a new setting, and in previous versions the behavior was equivalent to allowing."}, {"input_format_parquet_allow_missing_columns", false, true, "Allow missing columns in Parquet files by default"}, {"input_format_orc_allow_missing_columns", false, true, "Allow missing columns in ORC files by default"}, - {"input_format_arrow_allow_missing_columns", false, true, "Allow missing columns in Arrow files by default"}}}, - {"output_format_pretty_color",true,"auto","Setting is changed to allow also for auto value, disabling ANSI escapes if output is not a tty"} + {"input_format_arrow_allow_missing_columns", false, true, "Allow missing columns in Arrow files by default"}, + {"output_format_pretty_color", true, "auto", "Setting is changed to allow also for auto value, disabling ANSI escapes if output is not a tty"}}}, {"23.9", {{"optimize_group_by_constant_keys", false, true, "Optimize group by constant keys by default"}, {"input_format_json_try_infer_named_tuples_from_objects", false, true, "Try to infer named Tuples from JSON objects by default"}, {"input_format_json_read_numbers_as_strings", false, true, "Allow to read numbers as strings in JSON formats by default"}, diff --git a/tests/integration/test_storage_kafka/test.py b/tests/integration/test_storage_kafka/test.py index 2176b0151ff..5a8b5da89f7 100644 --- a/tests/integration/test_storage_kafka/test.py +++ b/tests/integration/test_storage_kafka/test.py @@ -4446,7 +4446,7 @@ def test_block_based_formats_1(kafka_cluster): kafka_group_name = '{topic}', kafka_format = 'PrettySpace'; - INSERT INTO test.kafka SELECT number * 10 as key, number * 100 as value FROM numbers(5) settings max_block_size=2, optimize_trivial_insert_select=0; + INSERT INTO test.kafka SELECT number * 10 as key, number * 100 as value FROM numbers(5) settings max_block_size=2, optimize_trivial_insert_select=0, output_format_pretty_color=1; """ ) diff --git a/tests/integration/test_storage_rabbitmq/test.py b/tests/integration/test_storage_rabbitmq/test.py index f26a273fe5e..6924f2e1508 100644 --- a/tests/integration/test_storage_rabbitmq/test.py +++ b/tests/integration/test_storage_rabbitmq/test.py @@ -3156,7 +3156,7 @@ def test_block_based_formats_1(rabbitmq_cluster): ) instance.query( - "INSERT INTO test.rabbitmq SELECT number * 10 as key, number * 100 as value FROM numbers(5) settings max_block_size=2, optimize_trivial_insert_select=0;" + "INSERT INTO test.rabbitmq SELECT number * 10 as key, number * 100 as value FROM numbers(5) settings max_block_size=2, optimize_trivial_insert_select=0, output_format_pretty_color=1;" ) insert_messages = [] diff --git a/tests/queries/0_stateless/02421_formats_with_totals_and_extremes.sql.j2 b/tests/queries/0_stateless/02421_formats_with_totals_and_extremes.sql.j2 index 32738766199..f936501e72a 100644 --- a/tests/queries/0_stateless/02421_formats_with_totals_and_extremes.sql.j2 +++ b/tests/queries/0_stateless/02421_formats_with_totals_and_extremes.sql.j2 @@ -1,5 +1,6 @@ -- Tags: no-fasttest +set output_format_pretty_color=1; set output_format_write_statistics=0; {% for format in ['CSV', 'TSV', 'XML', 'Vertical', 'Pretty', 'JSON', 'JSONCompact'] -%} From e4fa000f6fc2f53461ba9355625b735673b2e48a Mon Sep 17 00:00:00 2001 From: Denny Crane Date: Tue, 16 Jan 2024 18:29:15 -0400 Subject: [PATCH 22/23] Update Settings.h --- src/Core/Settings.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Core/Settings.h b/src/Core/Settings.h index 94b46eeb492..017de92ccfb 100644 --- a/src/Core/Settings.h +++ b/src/Core/Settings.h @@ -810,7 +810,7 @@ class IColumn; M(Bool, parallelize_output_from_storages, true, "Parallelize output for reading step from storage. It allows parallelizing query processing right after reading from storage if possible", 0) \ M(String, insert_deduplication_token, "", "If not empty, used for duplicate detection instead of data digest", 0) \ M(Bool, count_distinct_optimization, false, "Rewrite count distinct to subquery of group by", 0) \ - M(Bool, throw_if_no_data_to_insert, true, "Enables or disables empty INSERTs, enabled by default", 0) \ + M(Bool, throw_if_no_data_to_insert, true, "Allows or forbids empty INSERTs, enabled by default (throws an error on an empty insert)", 0) \ M(Bool, compatibility_ignore_auto_increment_in_create_table, false, "Ignore AUTO_INCREMENT keyword in column declaration if true, otherwise return error. It simplifies migration from MySQL", 0) \ M(Bool, multiple_joins_try_to_keep_original_names, false, "Do not add aliases to top level expression list on multiple joins rewrite", 0) \ M(Bool, optimize_sorting_by_input_stream_properties, true, "Optimize sorting by sorting properties of input stream", 0) \ From 0b3ebcda08eff6571561e0d771b06301a3ac697c Mon Sep 17 00:00:00 2001 From: Kseniia Sumarokova <54203879+kssenii@users.noreply.github.com> Date: Wed, 17 Jan 2024 11:13:57 +0100 Subject: [PATCH 23/23] Update SLRUFileCachePriority.cpp --- src/Interpreters/Cache/SLRUFileCachePriority.cpp | 1 - 1 file changed, 1 deletion(-) diff --git a/src/Interpreters/Cache/SLRUFileCachePriority.cpp b/src/Interpreters/Cache/SLRUFileCachePriority.cpp index eb933a5daef..4291b96c1b5 100644 --- a/src/Interpreters/Cache/SLRUFileCachePriority.cpp +++ b/src/Interpreters/Cache/SLRUFileCachePriority.cpp @@ -264,7 +264,6 @@ SLRUFileCachePriority::SLRUIterator::SLRUIterator( SLRUFileCachePriority::EntryPtr SLRUFileCachePriority::SLRUIterator::getEntry() const { - chassert(entry == lru_iterator.getEntry()); return entry; }