Merge pull request #2484 from yandex/format-settings

Allowed to switch between 'basic' and 'best effort' methods of parsing DateTime values from text formats.

ci/build-clang-from-sources.sh

@@ -26,7 +26,7 @@ cd "${WORKSPACE}/llvm/build"
 #  -DLLVM_ENABLE_LIBCXX=1
 # to the line below.
 
-cmake -DCMAKE_BUILD_TYPE:STRING=Release ../llvm
+cmake -DCMAKE_BUILD_TYPE:STRING=Release -DLLVM_ENABLE_LIBCXX=1 -DLLVM_ENABLE_RTTI=1 ../llvm
 
 make -j $THREADS
 $SUDO make install

dbms/programs/copier/ClusterCopier.cpp

@@ -52,6 +52,7 @@
 #include <Parsers/ASTDropQuery.h>
 #include <Parsers/ASTLiteral.h>
 #include <Parsers/ASTExpressionList.h>
+#include <DataTypes/FormatSettings.h>
 #include <DataStreams/RemoteBlockInputStream.h>
 #include <DataStreams/SquashingBlockInputStream.h>
 #include <DataStreams/AsynchronousBlockInputStream.h>
@@ -817,7 +818,7 @@ public:
 
             try
             {
-                type->deserializeTextQuoted(*column_dummy, rb);
+                type->deserializeTextQuoted(*column_dummy, rb, FormatSettings());
             }
             catch (Exception & e)
             {
@@ -1877,7 +1878,7 @@ protected:
             for (size_t i = 0; i < column.column->size(); ++i)
             {
                 WriteBufferFromOwnString wb;
-                column.type->serializeTextQuoted(*column.column, i, wb);
+                column.type->serializeTextQuoted(*column.column, i, wb, FormatSettings());
                 res.emplace(wb.str());
             }
         }

dbms/src/DataStreams/CSVRowInputStream.cpp

@@ -15,8 +15,8 @@ namespace ErrorCodes
 }
 
 
-CSVRowInputStream::CSVRowInputStream(ReadBuffer & istr_, const Block & header_, const char delimiter_, bool with_names_, bool with_types_)
-    : istr(istr_), header(header_), delimiter(delimiter_), with_names(with_names_), with_types(with_types_)
+CSVRowInputStream::CSVRowInputStream(ReadBuffer & istr_, const Block & header_, bool with_names_, const FormatSettings & format_settings)
+    : istr(istr_), header(header_), with_names(with_names_), format_settings(format_settings)
 {
     size_t num_columns = header.columns();
     data_types.resize(num_columns);
@@ -87,7 +87,7 @@ static void skipRow(ReadBuffer & istr, const char delimiter, size_t num_columns)
     for (size_t i = 0; i < num_columns; ++i)
     {
         skipWhitespacesAndTabs(istr);
-        readCSVString(tmp, istr);
+        readCSVString(tmp, istr, delimiter);
         skipWhitespacesAndTabs(istr);
 
         skipDelimiter(istr, delimiter, i + 1 == num_columns);
@@ -105,10 +105,7 @@ void CSVRowInputStream::readPrefix()
     String tmp;
 
     if (with_names)
-        skipRow(istr, delimiter, num_columns);
-
-    if (with_types)
-        skipRow(istr, delimiter, num_columns);
+        skipRow(istr, format_settings.csv.delimiter, num_columns);
 }
 
 
@@ -124,10 +121,10 @@ bool CSVRowInputStream::read(MutableColumns & columns)
     for (size_t i = 0; i < size; ++i)
     {
         skipWhitespacesAndTabs(istr);
-        data_types[i]->deserializeTextCSV(*columns[i], istr, delimiter);
+        data_types[i]->deserializeTextCSV(*columns[i], istr, format_settings);
         skipWhitespacesAndTabs(istr);
 
-        skipDelimiter(istr, delimiter, i + 1 == size);
+        skipDelimiter(istr, format_settings.csv.delimiter, i + 1 == size);
     }
 
     return true;
@@ -193,6 +190,8 @@ String CSVRowInputStream::getDiagnosticInfo()
 bool CSVRowInputStream::parseRowAndPrintDiagnosticInfo(MutableColumns & columns,
     WriteBuffer & out, size_t max_length_of_column_name, size_t max_length_of_data_type_name)
 {
+    const char delimiter = format_settings.csv.delimiter;
+
     size_t size = data_types.size();
     for (size_t i = 0; i < size; ++i)
     {
@@ -214,7 +213,7 @@ bool CSVRowInputStream::parseRowAndPrintDiagnosticInfo(MutableColumns & columns,
         {
             skipWhitespacesAndTabs(istr);
             prev_position = istr.position();
-            data_types[i]->deserializeTextCSV(*columns[i], istr, delimiter);
+            data_types[i]->deserializeTextCSV(*columns[i], istr, format_settings);
             curr_position = istr.position();
             skipWhitespacesAndTabs(istr);
         }

dbms/src/DataStreams/CSVRowInputStream.h

@@ -2,6 +2,7 @@
 
 #include <Core/Block.h>
 #include <DataStreams/IRowInputStream.h>
+#include <DataTypes/FormatSettings.h>
 
 
 namespace DB
@@ -18,7 +19,7 @@ public:
     /** with_names - in the first line the header with column names
       * with_types - on the next line header with type names
       */
-    CSVRowInputStream(ReadBuffer & istr_, const Block & header_, const char delimiter_, bool with_names_ = false, bool with_types_ = false);
+    CSVRowInputStream(ReadBuffer & istr_, const Block & header_, bool with_names_, const FormatSettings & format_settings);
 
     bool read(MutableColumns & columns) override;
     void readPrefix() override;
@@ -30,11 +31,11 @@ public:
 private:
     ReadBuffer & istr;
     Block header;
-    const char delimiter;
     bool with_names;
-    bool with_types;
     DataTypes data_types;
 
+    const FormatSettings format_settings;
+
     /// For convenient diagnostics in case of an error.
 
     size_t row_num = 0;

dbms/src/DataStreams/CSVRowOutputStream.cpp

@@ -7,8 +7,8 @@ namespace DB
 {
 
 
-CSVRowOutputStream::CSVRowOutputStream(WriteBuffer & ostr_, const Block & sample_, const char delimiter_, bool with_names_, bool with_types_)
-    : ostr(ostr_), sample(sample_), delimiter(delimiter_), with_names(with_names_), with_types(with_types_)
+CSVRowOutputStream::CSVRowOutputStream(WriteBuffer & ostr_, const Block & sample_, bool with_names_, const FormatSettings & format_settings)
+    : ostr(ostr_), sample(sample_), with_names(with_names_), format_settings(format_settings)
 {
     size_t columns = sample.columns();
     data_types.resize(columns);
@@ -32,16 +32,7 @@ void CSVRowOutputStream::writePrefix()
         for (size_t i = 0; i < columns; ++i)
         {
             writeCSVString(sample.safeGetByPosition(i).name, ostr);
-            writeChar(i == columns - 1 ? '\n' : delimiter, ostr);
-        }
-    }
-
-    if (with_types)
-    {
-        for (size_t i = 0; i < columns; ++i)
-        {
-            writeCSVString(sample.safeGetByPosition(i).type->getName(), ostr);
-            writeChar(i == columns - 1 ? '\n' : delimiter, ostr);
+            writeChar(i == columns - 1 ? '\n' : format_settings.csv.delimiter, ostr);
         }
     }
 }
@@ -49,13 +40,13 @@ void CSVRowOutputStream::writePrefix()
 
 void CSVRowOutputStream::writeField(const IColumn & column, const IDataType & type, size_t row_num)
 {
-    type.serializeTextCSV(column, row_num, ostr);
+    type.serializeTextCSV(column, row_num, ostr, format_settings);
 }
 
 
 void CSVRowOutputStream::writeFieldDelimiter()
 {
-    writeChar(delimiter, ostr);
+    writeChar(format_settings.csv.delimiter, ostr);
 }
 
 

dbms/src/DataStreams/CSVRowOutputStream.h

@@ -2,6 +2,7 @@
 
 #include <Core/Block.h>
 #include <DataStreams/IRowOutputStream.h>
+#include <DataTypes/FormatSettings.h>
 
 
 namespace DB
@@ -19,7 +20,7 @@ public:
     /** with_names - output in the first line a header with column names
       * with_types - output in the next line header with the names of the types
       */
-    CSVRowOutputStream(WriteBuffer & ostr_, const Block & sample_, const char delimiter_, bool with_names_ = false, bool with_types_ = false);
+    CSVRowOutputStream(WriteBuffer & ostr_, const Block & sample_, bool with_names_, const FormatSettings & format_settings);
 
     void writeField(const IColumn & column, const IDataType & type, size_t row_num) override;
     void writeFieldDelimiter() override;
@@ -35,7 +36,7 @@ public:
     /// https://www.iana.org/assignments/media-types/text/csv
     String getContentType() const override
     {
-        return String("text/csv; charset=UTF-8; header=") + ((with_names || with_types) ? "present" : "absent");
+        return String("text/csv; charset=UTF-8; header=") + (with_names ? "present" : "absent");
     }
 
 protected:
@@ -44,9 +45,8 @@ protected:
 
     WriteBuffer & ostr;
     const Block sample;
-    const char delimiter;
     bool with_names;
-    bool with_types;
+    const FormatSettings format_settings;
     DataTypes data_types;
     Block totals;
     Block extremes;

dbms/src/DataStreams/FormatFactory.cpp

@@ -29,7 +29,7 @@
 #include <DataStreams/MaterializingBlockOutputStream.h>
 #include <DataStreams/FormatFactory.h>
 #include <DataStreams/SquashingBlockOutputStream.h>
-#include <DataTypes/FormatSettingsJSON.h>
+#include <DataTypes/FormatSettings.h>
 #if USE_CAPNP
 #include <DataStreams/CapnProtoRowInputStream.h>
 #endif
@@ -51,6 +51,12 @@ BlockInputStreamPtr FormatFactory::getInput(const String & name, ReadBuffer & bu
 {
     const Settings & settings = context.getSettingsRef();
 
+    FormatSettings format_settings;
+    format_settings.csv.delimiter = settings.format_csv_delimiter;
+    format_settings.values.interpret_expressions = settings.input_format_values_interpret_expressions;
+    format_settings.skip_unknown_fields = settings.input_format_skip_unknown_fields;
+    format_settings.date_time_input_format = settings.date_time_input_format;
+
     auto wrap_row_stream = [&](auto && row_stream)
     {
         return std::make_shared<BlockInputStreamFromRowInputStream>(std::move(row_stream), sample, max_block_size,
@@ -67,34 +73,32 @@ BlockInputStreamPtr FormatFactory::getInput(const String & name, ReadBuffer & bu
     }
     else if (name == "TabSeparated" || name == "TSV") /// TSV is a synonym/alias for the original TabSeparated format
     {
-        return wrap_row_stream(std::make_shared<TabSeparatedRowInputStream>(buf, sample));
+        return wrap_row_stream(std::make_shared<TabSeparatedRowInputStream>(buf, sample, false, false, format_settings));
     }
     else if (name == "TabSeparatedWithNames" || name == "TSVWithNames")
     {
-        return wrap_row_stream(std::make_shared<TabSeparatedRowInputStream>(buf, sample, true));
+        return wrap_row_stream(std::make_shared<TabSeparatedRowInputStream>(buf, sample, true, false, format_settings));
     }
     else if (name == "TabSeparatedWithNamesAndTypes" || name == "TSVWithNamesAndTypes")
     {
-        return wrap_row_stream(std::make_shared<TabSeparatedRowInputStream>(buf, sample, true, true));
+        return wrap_row_stream(std::make_shared<TabSeparatedRowInputStream>(buf, sample, true, true, format_settings));
     }
     else if (name == "Values")
     {
-        return wrap_row_stream(std::make_shared<ValuesRowInputStream>(buf, sample, context, settings.input_format_values_interpret_expressions));
+        return wrap_row_stream(std::make_shared<ValuesRowInputStream>(buf, sample, context, format_settings));
     }
     else if (name == "CSV" || name == "CSVWithNames")
     {
-        char csv_delimiter = settings.format_csv_delimiter;
         bool with_names = name == "CSVWithNames";
-
-        return wrap_row_stream(std::make_shared<CSVRowInputStream>(buf, sample, csv_delimiter, with_names));
+        return wrap_row_stream(std::make_shared<CSVRowInputStream>(buf, sample, with_names, format_settings));
     }
     else if (name == "TSKV")
     {
-        return wrap_row_stream(std::make_shared<TSKVRowInputStream>(buf, sample, settings.input_format_skip_unknown_fields));
+        return wrap_row_stream(std::make_shared<TSKVRowInputStream>(buf, sample, format_settings));
     }
     else if (name == "JSONEachRow")
     {
-        return wrap_row_stream(std::make_shared<JSONEachRowRowInputStream>(buf, sample, settings.input_format_skip_unknown_fields));
+        return wrap_row_stream(std::make_shared<JSONEachRowRowInputStream>(buf, sample, format_settings));
     }
 #if USE_CAPNP
     else if (name == "CapnProto")
@@ -136,67 +140,80 @@ static BlockOutputStreamPtr getOutputImpl(const String & name, WriteBuffer & buf
     const Block & sample, const Context & context)
 {
     const Settings & settings = context.getSettingsRef();
-    FormatSettingsJSON json_settings(settings.output_format_json_quote_64bit_integers, settings.output_format_json_quote_denormals);
+
+    FormatSettings format_settings;
+    format_settings.json.quote_64bit_integers = settings.output_format_json_quote_64bit_integers;
+    format_settings.json.quote_denormals = settings.output_format_json_quote_denormals;
+    format_settings.csv.delimiter = settings.format_csv_delimiter;
+    format_settings.pretty.max_rows = settings.output_format_pretty_max_rows;
+    format_settings.pretty.color = settings.output_format_pretty_color;
+    format_settings.write_statistics = settings.output_format_write_statistics;
 
     if (name == "Native")
         return std::make_shared<NativeBlockOutputStream>(buf, 0, sample);
     else if (name == "RowBinary")
         return std::make_shared<BlockOutputStreamFromRowOutputStream>(std::make_shared<BinaryRowOutputStream>(buf), sample);
     else if (name == "TabSeparated" || name == "TSV")
-        return std::make_shared<BlockOutputStreamFromRowOutputStream>(std::make_shared<TabSeparatedRowOutputStream>(buf, sample), sample);
+        return std::make_shared<BlockOutputStreamFromRowOutputStream>(
+            std::make_shared<TabSeparatedRowOutputStream>(buf, sample, false, false, format_settings), sample);
     else if (name == "TabSeparatedWithNames" || name == "TSVWithNames")
-        return std::make_shared<BlockOutputStreamFromRowOutputStream>(std::make_shared<TabSeparatedRowOutputStream>(buf, sample, true), sample);
+        return std::make_shared<BlockOutputStreamFromRowOutputStream>(
+            std::make_shared<TabSeparatedRowOutputStream>(buf, sample, true, false, format_settings), sample);
     else if (name == "TabSeparatedWithNamesAndTypes" || name == "TSVWithNamesAndTypes")
-        return std::make_shared<BlockOutputStreamFromRowOutputStream>(std::make_shared<TabSeparatedRowOutputStream>(buf, sample, true, true), sample);
+        return std::make_shared<BlockOutputStreamFromRowOutputStream>(
+            std::make_shared<TabSeparatedRowOutputStream>(buf, sample, true, true, format_settings), sample);
     else if (name == "TabSeparatedRaw" || name == "TSVRaw")
-        return std::make_shared<BlockOutputStreamFromRowOutputStream>(std::make_shared<TabSeparatedRawRowOutputStream>(buf, sample), sample);
+        return std::make_shared<BlockOutputStreamFromRowOutputStream>(
+            std::make_shared<TabSeparatedRawRowOutputStream>(buf, sample, false, false, format_settings), sample);
     else if (name == "CSV" || name == "CSVWithNames")
     {
-        char csv_delimiter = settings.format_csv_delimiter;
         bool with_names = name == "CSVWithNames";
-
-        return std::make_shared<BlockOutputStreamFromRowOutputStream>(std::make_shared<CSVRowOutputStream>(buf, sample, csv_delimiter, with_names), sample);
+        return std::make_shared<BlockOutputStreamFromRowOutputStream>(std::make_shared<CSVRowOutputStream>(buf, sample, with_names, format_settings), sample);
     }
     else if (name == "Pretty")
-        return std::make_shared<PrettyBlockOutputStream>(buf, sample, false, settings.output_format_pretty_max_rows, context);
+        return std::make_shared<PrettyBlockOutputStream>(buf, sample, format_settings);
     else if (name == "PrettyCompact")
-        return std::make_shared<PrettyCompactBlockOutputStream>(buf, sample, false, settings.output_format_pretty_max_rows, context);
+        return std::make_shared<PrettyCompactBlockOutputStream>(buf, sample, format_settings);
     else if (name == "PrettyCompactMonoBlock")
     {
-        BlockOutputStreamPtr dst = std::make_shared<PrettyCompactBlockOutputStream>(buf, sample, false, settings.output_format_pretty_max_rows, context);
-        auto res = std::make_shared<SquashingBlockOutputStream>(dst, settings.output_format_pretty_max_rows, 0);
+        BlockOutputStreamPtr dst = std::make_shared<PrettyCompactBlockOutputStream>(buf, sample, format_settings);
+        auto res = std::make_shared<SquashingBlockOutputStream>(dst, format_settings.pretty.max_rows, 0);
         res->disableFlush();
         return res;
     }
     else if (name == "PrettySpace")
-        return std::make_shared<PrettySpaceBlockOutputStream>(buf, sample, false, settings.output_format_pretty_max_rows, context);
+        return std::make_shared<PrettySpaceBlockOutputStream>(buf, sample, format_settings);
     else if (name == "PrettyNoEscapes")
-        return std::make_shared<PrettyBlockOutputStream>(buf, sample, true, settings.output_format_pretty_max_rows, context);
+    {
+        format_settings.pretty.color = false;
+        return std::make_shared<PrettyBlockOutputStream>(buf, sample, format_settings);
+    }
     else if (name == "PrettyCompactNoEscapes")
-        return std::make_shared<PrettyCompactBlockOutputStream>(buf, sample, true, settings.output_format_pretty_max_rows, context);
+    {
+        format_settings.pretty.color = false;
+        return std::make_shared<PrettyCompactBlockOutputStream>(buf, sample, format_settings);
+    }
     else if (name == "PrettySpaceNoEscapes")
-        return std::make_shared<PrettySpaceBlockOutputStream>(buf, sample, true, settings.output_format_pretty_max_rows, context);
+    {
+        format_settings.pretty.color = false;
+        return std::make_shared<PrettySpaceBlockOutputStream>(buf, sample, format_settings);
+    }
     else if (name == "Vertical")
-        return std::make_shared<BlockOutputStreamFromRowOutputStream>(std::make_shared<VerticalRowOutputStream>(
-            buf, sample, settings.output_format_pretty_max_rows), sample);
+        return std::make_shared<BlockOutputStreamFromRowOutputStream>(std::make_shared<VerticalRowOutputStream>(buf, sample, format_settings), sample);
     else if (name == "Values")
-        return std::make_shared<BlockOutputStreamFromRowOutputStream>(std::make_shared<ValuesRowOutputStream>(buf), sample);
+        return std::make_shared<BlockOutputStreamFromRowOutputStream>(std::make_shared<ValuesRowOutputStream>(buf, format_settings), sample);
     else if (name == "JSON")
-        return std::make_shared<BlockOutputStreamFromRowOutputStream>(std::make_shared<JSONRowOutputStream>(
-            buf, sample, settings.output_format_write_statistics, json_settings), sample);
+        return std::make_shared<BlockOutputStreamFromRowOutputStream>(std::make_shared<JSONRowOutputStream>(buf, sample, format_settings), sample);
     else if (name == "JSONCompact")
-        return std::make_shared<BlockOutputStreamFromRowOutputStream>(std::make_shared<JSONCompactRowOutputStream>(
-            buf, sample, settings.output_format_write_statistics, json_settings), sample);
+        return std::make_shared<BlockOutputStreamFromRowOutputStream>(std::make_shared<JSONCompactRowOutputStream>(buf, sample, format_settings), sample);
     else if (name == "JSONEachRow")
-        return std::make_shared<BlockOutputStreamFromRowOutputStream>(std::make_shared<JSONEachRowRowOutputStream>(
-            buf, sample, json_settings), sample);
+        return std::make_shared<BlockOutputStreamFromRowOutputStream>(std::make_shared<JSONEachRowRowOutputStream>(buf, sample, format_settings), sample);
     else if (name == "XML")
-        return std::make_shared<BlockOutputStreamFromRowOutputStream>(std::make_shared<XMLRowOutputStream>(buf, sample,
-            settings.output_format_write_statistics), sample);
+        return std::make_shared<BlockOutputStreamFromRowOutputStream>(std::make_shared<XMLRowOutputStream>(buf, sample, format_settings), sample);
     else if (name == "TSKV")
-        return std::make_shared<BlockOutputStreamFromRowOutputStream>(std::make_shared<TSKVRowOutputStream>(buf, sample), sample);
+        return std::make_shared<BlockOutputStreamFromRowOutputStream>(std::make_shared<TSKVRowOutputStream>(buf, sample, format_settings), sample);
     else if (name == "ODBCDriver")
-        return std::make_shared<ODBCDriverBlockOutputStream>(buf, sample);
+        return std::make_shared<ODBCDriverBlockOutputStream>(buf, sample, format_settings);
     else if (name == "Null")
         return std::make_shared<NullBlockOutputStream>(sample);
     else

dbms/src/DataStreams/JSONCompactRowOutputStream.cpp

@@ -6,9 +6,8 @@
 namespace DB
 {
 
-JSONCompactRowOutputStream::JSONCompactRowOutputStream(WriteBuffer & ostr_, const Block & sample_,
-                                                       bool write_statistics_, const FormatSettingsJSON & settings_)
-    : JSONRowOutputStream(ostr_, sample_, write_statistics_, settings_)
+JSONCompactRowOutputStream::JSONCompactRowOutputStream(WriteBuffer & ostr_, const Block & sample_, const FormatSettings & settings)
+    : JSONRowOutputStream(ostr_, sample_, settings)
 {
 }
 
@@ -65,7 +64,7 @@ void JSONCompactRowOutputStream::writeTotals()
 }
 
 
-static void writeExtremesElement(const char * title, const Block & extremes, size_t row_num, WriteBuffer & ostr, const FormatSettingsJSON & settings)
+static void writeExtremesElement(const char * title, const Block & extremes, size_t row_num, WriteBuffer & ostr, const FormatSettings & settings)
 {
     writeCString("\t\t\"", ostr);
     writeCString(title, ostr);

dbms/src/DataStreams/JSONCompactRowOutputStream.h

@@ -5,17 +5,18 @@
 #include <IO/WriteBufferValidUTF8.h>
 #include <DataStreams/JSONRowOutputStream.h>
 
+
 namespace DB
 {
 
-struct FormatSettingsJSON;
+struct FormatSettings;
 
 /** The stream for outputting data in the JSONCompact format.
   */
 class JSONCompactRowOutputStream : public JSONRowOutputStream
 {
 public:
-    JSONCompactRowOutputStream(WriteBuffer & ostr_, const Block & sample_, bool write_statistics_, const FormatSettingsJSON & settings);
+    JSONCompactRowOutputStream(WriteBuffer & ostr_, const Block & sample_, const FormatSettings & settings);
 
     void writeField(const IColumn & column, const IDataType & type, size_t row_num) override;
     void writeFieldDelimiter() override;

dbms/src/DataStreams/JSONEachRowRowInputStream.cpp

@@ -12,8 +12,8 @@ namespace ErrorCodes
 }
 
 
-JSONEachRowRowInputStream::JSONEachRowRowInputStream(ReadBuffer & istr_, const Block & header_, bool skip_unknown_)
-    : istr(istr_), header(header_), skip_unknown(skip_unknown_), name_map(header.columns())
+JSONEachRowRowInputStream::JSONEachRowRowInputStream(ReadBuffer & istr_, const Block & header_, const FormatSettings & format_settings)
+    : istr(istr_), header(header_), format_settings(format_settings), name_map(header.columns())
 {
     /// In this format, BOM at beginning of stream cannot be confused with value, so it is safe to skip it.
     skipBOMIfExists(istr);
@@ -113,7 +113,7 @@ bool JSONEachRowRowInputStream::read(MutableColumns & columns)
         auto it = name_map.find(name_ref);
         if (name_map.end() == it)
         {
-            if (!skip_unknown)
+            if (!format_settings.skip_unknown_fields)
                 throw Exception("Unknown field found while parsing JSONEachRow format: " + name_ref.toString(), ErrorCodes::INCORRECT_DATA);
 
             skipColonDelimeter(istr);
@@ -132,7 +132,7 @@ bool JSONEachRowRowInputStream::read(MutableColumns & columns)
 
         try
         {
-            header.getByPosition(index).type->deserializeTextJSON(*columns[index], istr);
+            header.getByPosition(index).type->deserializeTextJSON(*columns[index], istr, format_settings);
         }
         catch (Exception & e)
         {

dbms/src/DataStreams/JSONEachRowRowInputStream.h

@@ -2,8 +2,10 @@
 
 #include <Core/Block.h>
 #include <DataStreams/IRowInputStream.h>
+#include <DataTypes/FormatSettings.h>
 #include <Common/HashTable/HashMap.h>
 
+
 namespace DB
 {
 
@@ -18,7 +20,7 @@ class ReadBuffer;
 class JSONEachRowRowInputStream : public IRowInputStream
 {
 public:
-    JSONEachRowRowInputStream(ReadBuffer & istr_, const Block & header_, bool skip_unknown_);
+    JSONEachRowRowInputStream(ReadBuffer & istr_, const Block & header_, const FormatSettings & format_settings);
 
     bool read(MutableColumns & columns) override;
     bool allowSyncAfterError() const override { return true; }
@@ -27,7 +29,8 @@ public:
 private:
     ReadBuffer & istr;
     Block header;
-    bool skip_unknown;
+
+    const FormatSettings format_settings;
 
     /// Buffer for the read from the stream field name. Used when you have to copy it.
     String name_buf;

dbms/src/DataStreams/JSONEachRowRowOutputStream.cpp

@@ -7,8 +7,8 @@ namespace DB
 {
 
 
-JSONEachRowRowOutputStream::JSONEachRowRowOutputStream(WriteBuffer & ostr_, const Block & sample, const FormatSettingsJSON & settings_)
-    : ostr(ostr_), settings(settings_)
+JSONEachRowRowOutputStream::JSONEachRowRowOutputStream(WriteBuffer & ostr_, const Block & sample, const FormatSettings & settings)
+    : ostr(ostr_), settings(settings)
 {
     size_t columns = sample.columns();
     fields.resize(columns);

dbms/src/DataStreams/JSONEachRowRowOutputStream.h

@@ -3,7 +3,7 @@
 #include <Core/Block.h>
 #include <IO/WriteBuffer.h>
 #include <DataStreams/IRowOutputStream.h>
-#include <DataTypes/FormatSettingsJSON.h>
+#include <DataTypes/FormatSettings.h>
 
 
 namespace DB
@@ -15,7 +15,7 @@ namespace DB
 class JSONEachRowRowOutputStream : public IRowOutputStream
 {
 public:
-    JSONEachRowRowOutputStream(WriteBuffer & ostr_, const Block & sample, const FormatSettingsJSON & settings);
+    JSONEachRowRowOutputStream(WriteBuffer & ostr_, const Block & sample, const FormatSettings & settings);
 
     void writeField(const IColumn & column, const IDataType & type, size_t row_num) override;
     void writeFieldDelimiter() override;
@@ -32,7 +32,7 @@ private:
     size_t field_number = 0;
     Names fields;
 
-    FormatSettingsJSON settings;
+    FormatSettings settings;
 };
 
 }

dbms/src/DataStreams/JSONRowOutputStream.cpp

@@ -6,8 +6,8 @@
 namespace DB
 {
 
-JSONRowOutputStream::JSONRowOutputStream(WriteBuffer & ostr_, const Block & sample_, bool write_statistics_, const FormatSettingsJSON & settings_)
-    : dst_ostr(ostr_), write_statistics(write_statistics_), settings(settings_)
+JSONRowOutputStream::JSONRowOutputStream(WriteBuffer & ostr_, const Block & sample_, const FormatSettings & settings)
+    : dst_ostr(ostr_), settings(settings)
 {
     NamesAndTypesList columns(sample_.getNamesAndTypesList());
     fields.assign(columns.begin(), columns.end());
@@ -111,7 +111,7 @@ void JSONRowOutputStream::writeSuffix()
 
     writeRowsBeforeLimitAtLeast();
 
-    if (write_statistics)
+    if (settings.write_statistics)
         writeStatistics();
 
     writeChar('\n', *ostr);
@@ -158,7 +158,7 @@ void JSONRowOutputStream::writeTotals()
 }
 
 
-static void writeExtremesElement(const char * title, const Block & extremes, size_t row_num, WriteBuffer & ostr, const FormatSettingsJSON & settings)
+static void writeExtremesElement(const char * title, const Block & extremes, size_t row_num, WriteBuffer & ostr, const FormatSettings & settings)
 {
     writeCString("\t\t\"", ostr);
     writeCString(title, ostr);

dbms/src/DataStreams/JSONRowOutputStream.h

@@ -5,7 +5,8 @@
 #include <IO/WriteBuffer.h>
 #include <Common/Stopwatch.h>
 #include <DataStreams/IRowOutputStream.h>
-#include <DataTypes/FormatSettingsJSON.h>
+#include <DataTypes/FormatSettings.h>
+
 
 namespace DB
 {
@@ -15,8 +16,7 @@ namespace DB
 class JSONRowOutputStream : public IRowOutputStream
 {
 public:
-    JSONRowOutputStream(WriteBuffer & ostr_, const Block & sample_,
-                        bool write_statistics_, const FormatSettingsJSON & settings_);
+    JSONRowOutputStream(WriteBuffer & ostr_, const Block & sample_, const FormatSettings & settings);
 
     void writeField(const IColumn & column, const IDataType & type, size_t row_num) override;
     void writeFieldDelimiter() override;
@@ -67,8 +67,7 @@ protected:
 
     Progress progress;
     Stopwatch watch;
-    bool write_statistics;
-    FormatSettingsJSON settings;
+    FormatSettings settings;
 };
 
 }

dbms/src/DataStreams/ODBCDriverBlockOutputStream.cpp

@@ -7,8 +7,8 @@
 namespace DB
 {
 
-ODBCDriverBlockOutputStream::ODBCDriverBlockOutputStream(WriteBuffer & out_, const Block & header_)
-    : out(out_), header(header_)
+ODBCDriverBlockOutputStream::ODBCDriverBlockOutputStream(WriteBuffer & out_, const Block & header_, const FormatSettings & format_settings)
+    : out(out_), header(header_), format_settings(format_settings)
 {
 }
 
@@ -32,7 +32,7 @@ void ODBCDriverBlockOutputStream::write(const Block & block)
 
             {
                 WriteBufferFromString text_out(text_value);
-                col.type->serializeText(*col.column, i, text_out);
+                col.type->serializeText(*col.column, i, text_out, format_settings);
             }
 
             writeStringBinary(text_value, out);

dbms/src/DataStreams/ODBCDriverBlockOutputStream.h

@@ -2,8 +2,10 @@
 
 #include <string>
 #include <DataStreams/IBlockOutputStream.h>
+#include <DataTypes/FormatSettings.h>
 #include <Core/Block.h>
 
+
 namespace DB
 {
 
@@ -19,7 +21,7 @@ class WriteBuffer;
 class ODBCDriverBlockOutputStream : public IBlockOutputStream
 {
 public:
-    ODBCDriverBlockOutputStream(WriteBuffer & out_, const Block & header_);
+    ODBCDriverBlockOutputStream(WriteBuffer & out_, const Block & header_, const FormatSettings & format_settings);
 
     Block getHeader() const override { return header; }
     void write(const Block & block) override;
@@ -31,6 +33,7 @@ public:
 private:
     WriteBuffer & out;
     const Block header;
+    const FormatSettings format_settings;
 };
 
 }

dbms/src/DataStreams/PrettyBlockOutputStream.cpp

@@ -17,8 +17,8 @@ namespace ErrorCodes
 
 
 PrettyBlockOutputStream::PrettyBlockOutputStream(
-    WriteBuffer & ostr_, const Block & header_, bool no_escapes_, size_t max_rows_, const Context & context_)
-     : ostr(ostr_), header(header_), max_rows(max_rows_), no_escapes(no_escapes_), context(context_)
+    WriteBuffer & ostr_, const Block & header_, const FormatSettings & format_settings)
+     : ostr(ostr_), header(header_), format_settings(format_settings)
 {
     struct winsize w;
     if (0 == ioctl(STDOUT_FILENO, TIOCGWINSZ, &w))
@@ -34,7 +34,8 @@ void PrettyBlockOutputStream::flush()
 
 /// Evaluate the visible width of the values and column names.
 /// Note that number of code points is just a rough approximation of visible string width.
-void PrettyBlockOutputStream::calculateWidths(const Block & block, WidthsPerColumn & widths, Widths & max_widths, Widths & name_widths)
+void PrettyBlockOutputStream::calculateWidths(
+    const Block & block, WidthsPerColumn & widths, Widths & max_widths, Widths & name_widths, const FormatSettings & format_settings)
 {
     size_t rows = block.rows();
     size_t columns = block.columns();
@@ -55,7 +56,7 @@ void PrettyBlockOutputStream::calculateWidths(const Block & block, WidthsPerColu
         {
             {
                 WriteBufferFromString out(serialized_value);
-                elem.type->serializeText(*elem.column, j, out);
+                elem.type->serializeText(*elem.column, j, out, format_settings);
             }
 
             widths[i][j] = UTF8::countCodePoints(reinterpret_cast<const UInt8 *>(serialized_value.data()), serialized_value.size());
@@ -73,6 +74,8 @@ void PrettyBlockOutputStream::calculateWidths(const Block & block, WidthsPerColu
 
 void PrettyBlockOutputStream::write(const Block & block)
 {
+    UInt64 max_rows = format_settings.pretty.max_rows;
+
     if (total_rows >= max_rows)
     {
         total_rows += block.rows();
@@ -85,7 +88,7 @@ void PrettyBlockOutputStream::write(const Block & block)
     WidthsPerColumn widths;
     Widths max_widths;
     Widths name_widths;
-    calculateWidths(block, widths, max_widths, name_widths);
+    calculateWidths(block, widths, max_widths, name_widths, format_settings);
 
     /// Create separators
     std::stringstream top_separator;
@@ -137,7 +140,7 @@ void PrettyBlockOutputStream::write(const Block & block)
 
         const ColumnWithTypeAndName & col = block.getByPosition(i);
 
-        if (!no_escapes)
+        if (format_settings.pretty.color)
             writeCString("\033[1m", ostr);
 
         if (col.type->shouldAlignRightInPrettyFormats())
@@ -155,7 +158,7 @@ void PrettyBlockOutputStream::write(const Block & block)
                 writeChar(' ', ostr);
         }
 
-        if (!no_escapes)
+        if (format_settings.pretty.color)
             writeCString("\033[0m", ostr);
     }
     writeCString(" ┃\n", ostr);
@@ -197,11 +200,11 @@ void PrettyBlockOutputStream::writeValueWithPadding(const ColumnWithTypeAndName
     if (elem.type->shouldAlignRightInPrettyFormats())
     {
         writePadding();
-        elem.type->serializeText(*elem.column.get(), row_num, ostr);
+        elem.type->serializeText(*elem.column.get(), row_num, ostr, format_settings);
     }
     else
     {
-        elem.type->serializeText(*elem.column.get(), row_num, ostr);
+        elem.type->serializeText(*elem.column.get(), row_num, ostr, format_settings);
         writePadding();
     }
 }
@@ -209,10 +212,10 @@ void PrettyBlockOutputStream::writeValueWithPadding(const ColumnWithTypeAndName
 
 void PrettyBlockOutputStream::writeSuffix()
 {
-    if (total_rows >= max_rows)
+    if (total_rows >= format_settings.pretty.max_rows)
     {
         writeCString("  Showed first ", ostr);
-        writeIntText(max_rows, ostr);
+        writeIntText(format_settings.pretty.max_rows, ostr);
         writeCString(".\n", ostr);
     }
 

dbms/src/DataStreams/PrettyBlockOutputStream.h

@@ -2,6 +2,7 @@
 
 #include <Core/Block.h>
 #include <DataStreams/IBlockOutputStream.h>
+#include <DataTypes/FormatSettings.h>
 
 
 namespace DB
@@ -17,7 +18,7 @@ class PrettyBlockOutputStream : public IBlockOutputStream
 {
 public:
     /// no_escapes - do not use ANSI escape sequences - to display in the browser, not in the console.
-    PrettyBlockOutputStream(WriteBuffer & ostr_, const Block & header_, bool no_escapes_, size_t max_rows_, const Context & context_);
+    PrettyBlockOutputStream(WriteBuffer & ostr_, const Block & header_, const FormatSettings & format_settings);
 
     Block getHeader() const override { return header; }
     void write(const Block & block) override;
@@ -34,21 +35,20 @@ protected:
 
     WriteBuffer & ostr;
     const Block header;
-    size_t max_rows;
     size_t total_rows = 0;
     size_t terminal_width = 0;
 
-    bool no_escapes;
+    const FormatSettings format_settings;
 
     Block totals;
     Block extremes;
 
-    const Context & context;
-
     using Widths = PODArray<size_t>;
     using WidthsPerColumn = std::vector<Widths>;
 
-    static void calculateWidths(const Block & block, WidthsPerColumn & widths, Widths & max_widths, Widths & name_widths);
+    static void calculateWidths(
+        const Block & block, WidthsPerColumn & widths, Widths & max_widths, Widths & name_widths, const FormatSettings & format_settings);
+
     void writeValueWithPadding(const ColumnWithTypeAndName & elem, size_t row_num, size_t value_width, size_t pad_to_width);
 };
 

dbms/src/DataStreams/PrettyCompactBlockOutputStream.cpp

@@ -32,18 +32,18 @@ void PrettyCompactBlockOutputStream::writeHeader(
             for (size_t k = 0; k < max_widths[i] - name_widths[i]; ++k)
                 writeCString("─", ostr);
 
-            if (!no_escapes)
+            if (format_settings.pretty.color)
                 writeCString("\033[1m", ostr);
             writeString(col.name, ostr);
-            if (!no_escapes)
+            if (format_settings.pretty.color)
                 writeCString("\033[0m", ostr);
         }
         else
         {
-            if (!no_escapes)
+            if (format_settings.pretty.color)
                 writeCString("\033[1m", ostr);
             writeString(col.name, ostr);
-            if (!no_escapes)
+            if (format_settings.pretty.color)
                 writeCString("\033[0m", ostr);
 
             for (size_t k = 0; k < max_widths[i] - name_widths[i]; ++k)
@@ -95,6 +95,8 @@ void PrettyCompactBlockOutputStream::writeRow(
 
 void PrettyCompactBlockOutputStream::write(const Block & block)
 {
+    UInt64 max_rows = format_settings.pretty.max_rows;
+
     if (total_rows >= max_rows)
     {
         total_rows += block.rows();
@@ -106,7 +108,7 @@ void PrettyCompactBlockOutputStream::write(const Block & block)
     WidthsPerColumn widths;
     Widths max_widths;
     Widths name_widths;
-    calculateWidths(block, widths, max_widths, name_widths);
+    calculateWidths(block, widths, max_widths, name_widths, format_settings);
 
     writeHeader(block, max_widths, name_widths);
 

dbms/src/DataStreams/PrettyCompactBlockOutputStream.h

@@ -11,8 +11,8 @@ namespace DB
 class PrettyCompactBlockOutputStream : public PrettyBlockOutputStream
 {
 public:
-    PrettyCompactBlockOutputStream(WriteBuffer & ostr_, const Block & header_, bool no_escapes_, size_t max_rows_, const Context & context_)
-        : PrettyBlockOutputStream(ostr_, header_, no_escapes_, max_rows_, context_) {}
+    PrettyCompactBlockOutputStream(WriteBuffer & ostr_, const Block & header_, const FormatSettings & format_settings)
+        : PrettyBlockOutputStream(ostr_, header_, format_settings) {}
 
     void write(const Block & block) override;
 

dbms/src/DataStreams/PrettySpaceBlockOutputStream.cpp

@@ -9,6 +9,8 @@ namespace DB
 
 void PrettySpaceBlockOutputStream::write(const Block & block)
 {
+    UInt64 max_rows = format_settings.pretty.max_rows;
+
     if (total_rows >= max_rows)
     {
         total_rows += block.rows();
@@ -21,7 +23,7 @@ void PrettySpaceBlockOutputStream::write(const Block & block)
     WidthsPerColumn widths;
     Widths max_widths;
     Widths name_widths;
-    calculateWidths(block, widths, max_widths, name_widths);
+    calculateWidths(block, widths, max_widths, name_widths, format_settings);
 
     /// Do not align on too long values.
     if (terminal_width > 80)
@@ -42,18 +44,18 @@ void PrettySpaceBlockOutputStream::write(const Block & block)
             for (ssize_t k = 0; k < std::max(static_cast<ssize_t>(0), static_cast<ssize_t>(max_widths[i] - name_widths[i])); ++k)
                 writeChar(' ', ostr);
 
-            if (!no_escapes)
+            if (format_settings.pretty.color)
                 writeCString("\033[1m", ostr);
             writeString(col.name, ostr);
-            if (!no_escapes)
+            if (format_settings.pretty.color)
                 writeCString("\033[0m", ostr);
         }
         else
         {
-            if (!no_escapes)
+            if (format_settings.pretty.color)
                 writeCString("\033[1m", ostr);
             writeString(col.name, ostr);
-            if (!no_escapes)
+            if (format_settings.pretty.color)
                 writeCString("\033[0m", ostr);
 
             for (ssize_t k = 0; k < std::max(static_cast<ssize_t>(0), static_cast<ssize_t>(max_widths[i] - name_widths[i])); ++k)
@@ -81,10 +83,10 @@ void PrettySpaceBlockOutputStream::write(const Block & block)
 
 void PrettySpaceBlockOutputStream::writeSuffix()
 {
-    if (total_rows >= max_rows)
+    if (total_rows >= format_settings.pretty.max_rows)
     {
         writeCString("\nShowed first ", ostr);
-        writeIntText(max_rows, ostr);
+        writeIntText(format_settings.pretty.max_rows, ostr);
         writeCString(".\n", ostr);
     }
 

dbms/src/DataStreams/PrettySpaceBlockOutputStream.h

@@ -11,8 +11,8 @@ namespace DB
 class PrettySpaceBlockOutputStream : public PrettyBlockOutputStream
 {
 public:
-    PrettySpaceBlockOutputStream(WriteBuffer & ostr_, const Block & header_, bool no_escapes_, size_t max_rows_, const Context & context_)
-        : PrettyBlockOutputStream(ostr_, header_, no_escapes_, max_rows_, context_) {}
+    PrettySpaceBlockOutputStream(WriteBuffer & ostr_, const Block & header_, const FormatSettings & format_settings)
+        : PrettyBlockOutputStream(ostr_, header_, format_settings) {}
 
     void write(const Block & block) override;
     void writeSuffix() override;

dbms/src/DataStreams/TSKVRowInputStream.cpp

@@ -14,8 +14,8 @@ namespace ErrorCodes
 }
 
 
-TSKVRowInputStream::TSKVRowInputStream(ReadBuffer & istr_, const Block & header_, bool skip_unknown_)
-    : istr(istr_), header(header_), skip_unknown(skip_unknown_), name_map(header.columns())
+TSKVRowInputStream::TSKVRowInputStream(ReadBuffer & istr_, const Block & header_, const FormatSettings & format_settings)
+    : istr(istr_), header(header_), format_settings(format_settings), name_map(header.columns())
 {
     /// In this format, we assume that column name cannot contain BOM,
     ///  so BOM at beginning of stream cannot be confused with name of field, and it is safe to skip it.
@@ -119,7 +119,7 @@ bool TSKVRowInputStream::read(MutableColumns & columns)
                 auto it = name_map.find(name_ref);
                 if (name_map.end() == it)
                 {
-                    if (!skip_unknown)
+                    if (!format_settings.skip_unknown_fields)
                         throw Exception("Unknown field found while parsing TSKV format: " + name_ref.toString(), ErrorCodes::INCORRECT_DATA);
 
                     /// If the key is not found, skip the value.
@@ -135,7 +135,7 @@ bool TSKVRowInputStream::read(MutableColumns & columns)
 
                     read_columns[index] = true;
 
-                    header.getByPosition(index).type->deserializeTextEscaped(*columns[index], istr);
+                    header.getByPosition(index).type->deserializeTextEscaped(*columns[index], istr, format_settings);
                 }
             }
             else

dbms/src/DataStreams/TSKVRowInputStream.h

@@ -2,6 +2,7 @@
 
 #include <Core/Block.h>
 #include <DataStreams/IRowInputStream.h>
+#include <DataTypes/FormatSettings.h>
 #include <Common/HashTable/HashMap.h>
 
 
@@ -22,7 +23,7 @@ class ReadBuffer;
 class TSKVRowInputStream : public IRowInputStream
 {
 public:
-    TSKVRowInputStream(ReadBuffer & istr_, const Block & header_, bool skip_unknown_);
+    TSKVRowInputStream(ReadBuffer & istr_, const Block & header_, const FormatSettings & format_settings);
 
     bool read(MutableColumns & columns) override;
     bool allowSyncAfterError() const override { return true; }
@@ -31,8 +32,8 @@ public:
 private:
     ReadBuffer & istr;
     Block header;
-    /// Skip unknown fields.
-    bool skip_unknown;
+
+    const FormatSettings format_settings;
 
     /// Buffer for the read from the stream the field name. Used when you have to copy it.
     String name_buf;

dbms/src/DataStreams/TSKVRowOutputStream.cpp

@@ -6,8 +6,8 @@
 namespace DB
 {
 
-TSKVRowOutputStream::TSKVRowOutputStream(WriteBuffer & ostr_, const Block & sample_)
-    : TabSeparatedRowOutputStream(ostr_, sample_)
+TSKVRowOutputStream::TSKVRowOutputStream(WriteBuffer & ostr_, const Block & sample_, const FormatSettings & format_settings)
+    : TabSeparatedRowOutputStream(ostr_, sample_, false, false, format_settings)
 {
     NamesAndTypesList columns(sample_.getNamesAndTypesList());
     fields.assign(columns.begin(), columns.end());
@@ -25,7 +25,7 @@ TSKVRowOutputStream::TSKVRowOutputStream(WriteBuffer & ostr_, const Block & samp
 void TSKVRowOutputStream::writeField(const IColumn & column, const IDataType & type, size_t row_num)
 {
     writeString(fields[field_number].name, ostr);
-    type.serializeTextEscaped(column, row_num, ostr);
+    type.serializeTextEscaped(column, row_num, ostr, format_settings);
     ++field_number;
 }
 

dbms/src/DataStreams/TSKVRowOutputStream.h

@@ -1,5 +1,6 @@
 #pragma once
 
+#include <DataTypes/FormatSettings.h>
 #include <DataStreams/TabSeparatedRowOutputStream.h>
 
 
@@ -13,7 +14,7 @@ namespace DB
 class TSKVRowOutputStream : public TabSeparatedRowOutputStream
 {
 public:
-    TSKVRowOutputStream(WriteBuffer & ostr_, const Block & sample_);
+    TSKVRowOutputStream(WriteBuffer & ostr_, const Block & sample_, const FormatSettings & format_settings);
     void writeField(const IColumn & column, const IDataType & type, size_t row_num) override;
     void writeRowEndDelimiter() override;
 

dbms/src/DataStreams/TabSeparatedRawRowOutputStream.h

@@ -1,5 +1,6 @@
 #pragma once
 
+#include <DataTypes/FormatSettings.h>
 #include <DataStreams/TabSeparatedRowOutputStream.h>
 
 
@@ -12,12 +13,12 @@ namespace DB
 class TabSeparatedRawRowOutputStream : public TabSeparatedRowOutputStream
 {
 public:
-    TabSeparatedRawRowOutputStream(WriteBuffer & ostr_, const Block & sample_, bool with_names_ = false, bool with_types_ = false)
-        : TabSeparatedRowOutputStream(ostr_, sample_, with_names_, with_types_) {}
+    TabSeparatedRawRowOutputStream(WriteBuffer & ostr_, const Block & sample_, bool with_names_, bool with_types_, const FormatSettings & format_settings)
+        : TabSeparatedRowOutputStream(ostr_, sample_, with_names_, with_types_, format_settings) {}
 
     void writeField(const IColumn & column, const IDataType & type, size_t row_num) override
     {
-        type.serializeText(column, row_num, ostr);
+        type.serializeText(column, row_num, ostr, format_settings);
     }
 };
 

dbms/src/DataStreams/TabSeparatedRowInputStream.cpp

@@ -16,8 +16,9 @@ namespace ErrorCodes
 }
 
 
-TabSeparatedRowInputStream::TabSeparatedRowInputStream(ReadBuffer & istr_, const Block & header_, bool with_names_, bool with_types_)
-    : istr(istr_), header(header_), with_names(with_names_), with_types(with_types_)
+TabSeparatedRowInputStream::TabSeparatedRowInputStream(
+    ReadBuffer & istr_, const Block & header_, bool with_names_, bool with_types_, const FormatSettings & format_settings)
+    : istr(istr_), header(header_), with_names(with_names_), with_types(with_types_), format_settings(format_settings)
 {
     size_t num_columns = header.columns();
     data_types.resize(num_columns);
@@ -83,7 +84,7 @@ bool TabSeparatedRowInputStream::read(MutableColumns & columns)
 
     for (size_t i = 0; i < size; ++i)
     {
-        data_types[i]->deserializeTextEscaped(*columns[i], istr);
+        data_types[i]->deserializeTextEscaped(*columns[i], istr, format_settings);
 
         /// skip separators
         if (i + 1 == size)
@@ -180,7 +181,7 @@ bool TabSeparatedRowInputStream::parseRowAndPrintDiagnosticInfo(MutableColumns &
 
         try
         {
-            data_types[i]->deserializeTextEscaped(*columns[i], istr);
+            data_types[i]->deserializeTextEscaped(*columns[i], istr, format_settings);
         }
         catch (...)
         {

dbms/src/DataStreams/TabSeparatedRowInputStream.h

@@ -1,6 +1,7 @@
 #pragma once
 
 #include <Core/Block.h>
+#include <DataTypes/FormatSettings.h>
 #include <DataStreams/IRowInputStream.h>
 
 
@@ -18,7 +19,8 @@ public:
     /** with_names - the first line is the header with the names of the columns
       * with_types - on the next line header with type names
       */
-    TabSeparatedRowInputStream(ReadBuffer & istr_, const Block & header_, bool with_names_ = false, bool with_types_ = false);
+    TabSeparatedRowInputStream(
+        ReadBuffer & istr_, const Block & header_, bool with_names_, bool with_types_, const FormatSettings & format_settings);
 
     bool read(MutableColumns & columns) override;
     void readPrefix() override;
@@ -32,6 +34,7 @@ private:
     Block header;
     bool with_names;
     bool with_types;
+    const FormatSettings format_settings;
     DataTypes data_types;
 
     /// For convenient diagnostics in case of an error.

dbms/src/DataStreams/TabSeparatedRowOutputStream.cpp

@@ -6,8 +6,9 @@
 namespace DB
 {
 
-TabSeparatedRowOutputStream::TabSeparatedRowOutputStream(WriteBuffer & ostr_, const Block & sample_, bool with_names_, bool with_types_)
-    : ostr(ostr_), sample(sample_), with_names(with_names_), with_types(with_types_)
+TabSeparatedRowOutputStream::TabSeparatedRowOutputStream(
+    WriteBuffer & ostr_, const Block & sample_, bool with_names_, bool with_types_, const FormatSettings & format_settings)
+    : ostr(ostr_), sample(sample_), with_names(with_names_), with_types(with_types_), format_settings(format_settings)
 {
 }
 
@@ -44,7 +45,7 @@ void TabSeparatedRowOutputStream::writePrefix()
 
 void TabSeparatedRowOutputStream::writeField(const IColumn & column, const IDataType & type, size_t row_num)
 {
-    type.serializeTextEscaped(column, row_num, ostr);
+    type.serializeTextEscaped(column, row_num, ostr, format_settings);
 }
 
 

dbms/src/DataStreams/TabSeparatedRowOutputStream.h

@@ -1,6 +1,7 @@
 #pragma once
 
 #include <Core/Block.h>
+#include <DataTypes/FormatSettings.h>
 #include <DataStreams/IRowOutputStream.h>
 
 
@@ -17,7 +18,7 @@ public:
     /** with_names - output in the first line a header with column names
       * with_types - output the next line header with the names of the types
       */
-    TabSeparatedRowOutputStream(WriteBuffer & ostr_, const Block & sample_, bool with_names_ = false, bool with_types_ = false);
+    TabSeparatedRowOutputStream(WriteBuffer & ostr_, const Block & sample_, bool with_names_, bool with_types_, const FormatSettings & format_settings);
 
     void writeField(const IColumn & column, const IDataType & type, size_t row_num) override;
     void writeFieldDelimiter() override;
@@ -41,6 +42,7 @@ protected:
     const Block sample;
     bool with_names;
     bool with_types;
+    const FormatSettings format_settings;
     Block totals;
     Block extremes;
 };

dbms/src/DataStreams/ValuesRowInputStream.cpp

@@ -26,8 +26,8 @@ namespace ErrorCodes
 }
 
 
-ValuesRowInputStream::ValuesRowInputStream(ReadBuffer & istr_, const Block & header_, const Context & context_, bool interpret_expressions_)
-    : istr(istr_), header(header_), context(context_), interpret_expressions(interpret_expressions_)
+ValuesRowInputStream::ValuesRowInputStream(ReadBuffer & istr_, const Block & header_, const Context & context_, const FormatSettings & format_settings)
+    : istr(istr_), header(header_), context(context_), format_settings(format_settings)
 {
     /// In this format, BOM at beginning of stream cannot be confused with value, so it is safe to skip it.
     skipBOMIfExists(istr);
@@ -61,7 +61,7 @@ bool ValuesRowInputStream::read(MutableColumns & columns)
         bool rollback_on_exception = false;
         try
         {
-            header.getByPosition(i).type->deserializeTextQuoted(*columns[i], istr);
+            header.getByPosition(i).type->deserializeTextQuoted(*columns[i], istr, format_settings);
             rollback_on_exception = true;
             skipWhitespaceIfAny(istr);
 
@@ -72,7 +72,7 @@ bool ValuesRowInputStream::read(MutableColumns & columns)
         }
         catch (const Exception & e)
         {
-            if (!interpret_expressions)
+            if (!format_settings.values.interpret_expressions)
                 throw;
 
             /** The normal streaming parser could not parse the value.

dbms/src/DataStreams/ValuesRowInputStream.h

@@ -2,6 +2,7 @@
 
 #include <Core/Block.h>
 #include <DataStreams/IRowInputStream.h>
+#include <DataTypes/FormatSettings.h>
 
 
 namespace DB
@@ -20,7 +21,7 @@ public:
       * If interpret_expressions is true, it will, in addition, try to use SQL parser and interpreter
       *  in case when streaming parser could not parse field (this is very slow).
       */
-    ValuesRowInputStream(ReadBuffer & istr_, const Block & header_, const Context & context_, bool interpret_expressions_);
+    ValuesRowInputStream(ReadBuffer & istr_, const Block & header_, const Context & context_, const FormatSettings & format_settings);
 
     bool read(MutableColumns & columns) override;
 
@@ -28,7 +29,7 @@ private:
     ReadBuffer & istr;
     Block header;
     const Context & context;
-    bool interpret_expressions;
+    const FormatSettings format_settings;
 };
 
 }

dbms/src/DataStreams/ValuesRowOutputStream.cpp

@@ -9,8 +9,8 @@ namespace DB
 {
 
 
-ValuesRowOutputStream::ValuesRowOutputStream(WriteBuffer & ostr_)
-    : ostr(ostr_)
+ValuesRowOutputStream::ValuesRowOutputStream(WriteBuffer & ostr_, const FormatSettings & format_settings)
+    : ostr(ostr_), format_settings(format_settings)
 {
 }
 
@@ -21,7 +21,7 @@ void ValuesRowOutputStream::flush()
 
 void ValuesRowOutputStream::writeField(const IColumn & column, const IDataType & type, size_t row_num)
 {
-    type.serializeTextQuoted(column, row_num, ostr);
+    type.serializeTextQuoted(column, row_num, ostr, format_settings);
 }
 
 void ValuesRowOutputStream::writeFieldDelimiter()

dbms/src/DataStreams/ValuesRowOutputStream.h

@@ -1,5 +1,6 @@
 #pragma once
 
+#include <DataTypes/FormatSettings.h>
 #include <DataStreams/IRowOutputStream.h>
 
 
@@ -14,7 +15,7 @@ class WriteBuffer;
 class ValuesRowOutputStream : public IRowOutputStream
 {
 public:
-    ValuesRowOutputStream(WriteBuffer & ostr_);
+    ValuesRowOutputStream(WriteBuffer & ostr_, const FormatSettings & format_settings);
 
     void writeField(const IColumn & column, const IDataType & type, size_t row_num) override;
     void writeFieldDelimiter() override;
@@ -25,6 +26,7 @@ public:
 
 private:
     WriteBuffer & ostr;
+    const FormatSettings format_settings;
 };
 
 }

dbms/src/DataStreams/VerticalRowOutputStream.cpp

@@ -10,8 +10,8 @@ namespace DB
 {
 
 VerticalRowOutputStream::VerticalRowOutputStream(
-    WriteBuffer & ostr_, const Block & sample_, size_t max_rows_)
-    : ostr(ostr_), sample(sample_), max_rows(max_rows_)
+    WriteBuffer & ostr_, const Block & sample_, const FormatSettings & format_settings)
+    : ostr(ostr_), sample(sample_), format_settings(format_settings)
 {
     size_t columns = sample.columns();
 
@@ -53,7 +53,7 @@ void VerticalRowOutputStream::flush()
 
 void VerticalRowOutputStream::writeField(const IColumn & column, const IDataType & type, size_t row_num)
 {
-    if (row_number > max_rows)
+    if (row_number > format_settings.pretty.max_rows)
         return;
 
     writeString(names_and_paddings[field_number], ostr);
@@ -66,7 +66,7 @@ void VerticalRowOutputStream::writeField(const IColumn & column, const IDataType
 
 void VerticalRowOutputStream::writeValue(const IColumn & column, const IDataType & type, size_t row_num) const
 {
-    type.serializeText(column, row_num, ostr);
+    type.serializeText(column, row_num, ostr, format_settings);
 }
 
 
@@ -74,7 +74,7 @@ void VerticalRowOutputStream::writeRowStartDelimiter()
 {
     ++row_number;
 
-    if (row_number > max_rows)
+    if (row_number > format_settings.pretty.max_rows)
         return;
 
     writeCString("Row ", ostr);
@@ -90,7 +90,7 @@ void VerticalRowOutputStream::writeRowStartDelimiter()
 
 void VerticalRowOutputStream::writeRowBetweenDelimiter()
 {
-    if (row_number > max_rows)
+    if (row_number > format_settings.pretty.max_rows)
         return;
 
     writeCString("\n", ostr);
@@ -100,10 +100,10 @@ void VerticalRowOutputStream::writeRowBetweenDelimiter()
 
 void VerticalRowOutputStream::writeSuffix()
 {
-    if (row_number > max_rows)
+    if (row_number > format_settings.pretty.max_rows)
     {
         writeCString("Showed first ", ostr);
-        writeIntText(max_rows, ostr);
+        writeIntText(format_settings.pretty.max_rows, ostr);
         writeCString(".\n", ostr);
     }
 

dbms/src/DataStreams/VerticalRowOutputStream.h

@@ -1,7 +1,7 @@
 #pragma once
 
 #include <Core/Block.h>
-#include <Core/Names.h>
+#include <DataTypes/FormatSettings.h>
 #include <DataStreams/IRowOutputStream.h>
 
 
@@ -18,7 +18,7 @@ class Context;
 class VerticalRowOutputStream : public IRowOutputStream
 {
 public:
-    VerticalRowOutputStream(WriteBuffer & ostr_, const Block & sample_, size_t max_rows_);
+    VerticalRowOutputStream(WriteBuffer & ostr_, const Block & sample_, const FormatSettings & format_settings);
 
     void writeField(const IColumn & column, const IDataType & type, size_t row_num) override;
     void writeRowStartDelimiter() override;
@@ -40,7 +40,7 @@ protected:
 
     WriteBuffer & ostr;
     const Block sample;
-    size_t max_rows;
+    const FormatSettings format_settings;
     size_t field_number = 0;
     size_t row_number = 0;
 

dbms/src/DataStreams/XMLRowOutputStream.cpp

@@ -6,8 +6,8 @@
 namespace DB
 {
 
-XMLRowOutputStream::XMLRowOutputStream(WriteBuffer & ostr_, const Block & sample_, bool write_statistics_)
-    : dst_ostr(ostr_), write_statistics(write_statistics_)
+XMLRowOutputStream::XMLRowOutputStream(WriteBuffer & ostr_, const Block & sample_, const FormatSettings & format_settings)
+    : dst_ostr(ostr_), format_settings(format_settings)
 {
     NamesAndTypesList columns(sample_.getNamesAndTypesList());
     fields.assign(columns.begin(), columns.end());
@@ -85,7 +85,7 @@ void XMLRowOutputStream::writeField(const IColumn & column, const IDataType & ty
     writeCString("\t\t\t<", *ostr);
     writeString(field_tag_names[field_number], *ostr);
     writeCString(">", *ostr);
-    type.serializeTextXML(column, row_num, *ostr);
+    type.serializeTextXML(column, row_num, *ostr, format_settings);
     writeCString("</", *ostr);
     writeString(field_tag_names[field_number], *ostr);
     writeCString(">\n", *ostr);
@@ -120,7 +120,7 @@ void XMLRowOutputStream::writeSuffix()
 
     writeRowsBeforeLimitAtLeast();
 
-    if (write_statistics)
+    if (format_settings.write_statistics)
         writeStatistics();
 
     writeCString("</result>\n", *ostr);
@@ -151,7 +151,7 @@ void XMLRowOutputStream::writeTotals()
             writeCString("\t\t<", *ostr);
             writeString(field_tag_names[i], *ostr);
             writeCString(">", *ostr);
-            column.type->serializeTextXML(*column.column.get(), 0, *ostr);
+            column.type->serializeTextXML(*column.column.get(), 0, *ostr, format_settings);
             writeCString("</", *ostr);
             writeString(field_tag_names[i], *ostr);
             writeCString(">\n", *ostr);
@@ -162,7 +162,8 @@ void XMLRowOutputStream::writeTotals()
 }
 
 
-static void writeExtremesElement(const char * title, const Block & extremes, size_t row_num, const Names & field_tag_names, WriteBuffer & ostr)
+static void writeExtremesElement(
+    const char * title, const Block & extremes, size_t row_num, const Names & field_tag_names, WriteBuffer & ostr, const FormatSettings & format_settings)
 {
     writeCString("\t\t<", ostr);
     writeCString(title, ostr);
@@ -176,7 +177,7 @@ static void writeExtremesElement(const char * title, const Block & extremes, siz
         writeCString("\t\t\t<", ostr);
         writeString(field_tag_names[i], ostr);
         writeCString(">", ostr);
-        column.type->serializeTextXML(*column.column.get(), row_num, ostr);
+        column.type->serializeTextXML(*column.column.get(), row_num, ostr, format_settings);
         writeCString("</", ostr);
         writeString(field_tag_names[i], ostr);
         writeCString(">\n", ostr);
@@ -192,8 +193,8 @@ void XMLRowOutputStream::writeExtremes()
     if (extremes)
     {
         writeCString("\t<extremes>\n", *ostr);
-        writeExtremesElement("min", extremes, 0, field_tag_names, *ostr);
-        writeExtremesElement("max", extremes, 1, field_tag_names, *ostr);
+        writeExtremesElement("min", extremes, 0, field_tag_names, *ostr, format_settings);
+        writeExtremesElement("max", extremes, 1, field_tag_names, *ostr, format_settings);
         writeCString("\t</extremes>\n", *ostr);
     }
 }

dbms/src/DataStreams/XMLRowOutputStream.h

@@ -4,6 +4,7 @@
 #include <IO/Progress.h>
 #include <IO/WriteBuffer.h>
 #include <Common/Stopwatch.h>
+#include <DataTypes/FormatSettings.h>
 #include <DataStreams/IRowOutputStream.h>
 
 
@@ -15,8 +16,7 @@ namespace DB
 class XMLRowOutputStream : public IRowOutputStream
 {
 public:
-    XMLRowOutputStream(WriteBuffer & ostr_, const Block & sample_,
-        bool write_statistics_);
+    XMLRowOutputStream(WriteBuffer & ostr_, const Block & sample_, const FormatSettings & format_settings);
 
     void writeField(const IColumn & column, const IDataType & type, size_t row_num) override;
     void writeRowStartDelimiter() override;
@@ -67,7 +67,7 @@ protected:
 
     Progress progress;
     Stopwatch watch;
-    bool write_statistics;
+    const FormatSettings format_settings;
 };
 
 }

dbms/src/DataStreams/tests/block_row_transforms.cpp

@@ -41,9 +41,11 @@ try
     ReadBufferFromFile in_buf("test_in");
     WriteBufferFromFile out_buf("test_out");
 
-    RowInputStreamPtr row_input = std::make_shared<TabSeparatedRowInputStream>(in_buf, sample);
+    FormatSettings format_settings;
+
+    RowInputStreamPtr row_input = std::make_shared<TabSeparatedRowInputStream>(in_buf, sample, false, false, format_settings);
     BlockInputStreamFromRowInputStream block_input(row_input, sample, DEFAULT_INSERT_BLOCK_SIZE, 0, 0);
-    RowOutputStreamPtr row_output = std::make_shared<TabSeparatedRowOutputStream>(out_buf, sample);
+    RowOutputStreamPtr row_output = std::make_shared<TabSeparatedRowOutputStream>(out_buf, sample, false, false, format_settings);
     BlockOutputStreamFromRowOutputStream block_output(row_output, sample);
 
     copyData(block_input, block_output);

dbms/src/DataStreams/tests/expression_stream.cpp

@@ -55,8 +55,10 @@ try
     in = std::make_shared<ExpressionBlockInputStream>(in, expression);
     in = std::make_shared<LimitBlockInputStream>(in, 10, std::max(static_cast<Int64>(0), static_cast<Int64>(n) - 10));
 
+    FormatSettings format_settings;
+
     WriteBufferFromOStream out1(std::cout);
-    RowOutputStreamPtr out2 = std::make_shared<TabSeparatedRowOutputStream>(out1, expression->getSampleBlock());
+    RowOutputStreamPtr out2 = std::make_shared<TabSeparatedRowOutputStream>(out1, expression->getSampleBlock(), false, false, format_settings);
     BlockOutputStreamFromRowOutputStream out(out2, expression->getSampleBlock());
 
     {

dbms/src/DataStreams/tests/filter_stream.cpp

@@ -59,8 +59,10 @@ try
     in = std::make_shared<FilterBlockInputStream>(in, expression, "equals(modulo(number, 3), 1)");
     in = std::make_shared<LimitBlockInputStream>(in, 10, std::max(static_cast<Int64>(0), static_cast<Int64>(n) - 10));
 
+    FormatSettings format_settings;
+
     WriteBufferFromOStream ob(std::cout);
-    RowOutputStreamPtr out_ = std::make_shared<TabSeparatedRowOutputStream>(ob, expression->getSampleBlock());
+    RowOutputStreamPtr out_ = std::make_shared<TabSeparatedRowOutputStream>(ob, expression->getSampleBlock(), false, false, format_settings);
     BlockOutputStreamFromRowOutputStream out(out_, expression->getSampleBlock());
 
 

dbms/src/DataStreams/tests/filter_stream_hitlog.cpp

@@ -131,8 +131,10 @@ int main(int, char **)
         in = std::make_shared<FilterBlockInputStream>(in, expression, "equals(URL, 'http://mail.yandex.ru/neo2/#inbox')");
         //in = std::make_shared<LimitBlockInputStream>(in, 10, 0);
 
+        FormatSettings format_settings;
+
         WriteBufferFromOStream ob(std::cout);
-        RowOutputStreamPtr out_ = std::make_shared<TabSeparatedRowOutputStream>(ob, expression->getSampleBlock());
+        RowOutputStreamPtr out_ = std::make_shared<TabSeparatedRowOutputStream>(ob, expression->getSampleBlock(), false, false, format_settings);
         BlockOutputStreamFromRowOutputStream out(out_, in->getHeader());
 
         copyData(*in, out);

dbms/src/DataStreams/tests/sorting_stream.cpp

@@ -150,8 +150,10 @@ try
     in = std::make_shared<MergeSortingBlockInputStream>(in, sort_columns, DEFAULT_BLOCK_SIZE, 0, 0, "");
     //in = std::make_shared<LimitBlockInputStream>(in, 10, 0);
 
+    FormatSettings format_settings;
+
     WriteBufferFromOStream ob(std::cout);
-    RowOutputStreamPtr out_ = std::make_shared<TabSeparatedRowOutputStream>(ob, sample);
+    RowOutputStreamPtr out_ = std::make_shared<TabSeparatedRowOutputStream>(ob, sample, false, false, format_settings);
     BlockOutputStreamFromRowOutputStream out(out_, sample);
 
     copyData(*in, out);

dbms/src/DataStreams/tests/tab_separated_streams.cpp

@@ -34,8 +34,10 @@ try
     ReadBufferFromFile in_buf("test_in");
     WriteBufferFromFile out_buf("test_out");
 
-    RowInputStreamPtr row_input = std::make_shared<TabSeparatedRowInputStream>(in_buf, sample);
-    RowOutputStreamPtr row_output = std::make_shared<TabSeparatedRowOutputStream>(out_buf, sample);
+    FormatSettings format_settings;
+
+    RowInputStreamPtr row_input = std::make_shared<TabSeparatedRowInputStream>(in_buf, sample, false, false, format_settings);
+    RowOutputStreamPtr row_output = std::make_shared<TabSeparatedRowOutputStream>(out_buf, sample, false, false, format_settings);
 
     BlockInputStreamFromRowInputStream block_input(row_input, sample, DEFAULT_INSERT_BLOCK_SIZE, 0, 0);
     BlockOutputStreamFromRowOutputStream block_output(row_output, sample);

dbms/src/DataTypes/DataTypeAggregateFunction.cpp

@@ -7,8 +7,10 @@
 
 #include <Common/typeid_cast.h>
 
+#include <DataTypes/FormatSettings.h>
 #include <DataTypes/DataTypeAggregateFunction.h>
 #include <DataTypes/DataTypeFactory.h>
+
 #include <AggregateFunctions/AggregateFunctionFactory.h>
 #include <Parsers/ASTFunction.h>
 #include <Parsers/ASTLiteral.h>
@@ -176,19 +178,19 @@ static void deserializeFromString(const AggregateFunctionPtr & function, IColumn
     column_concrete.getData().push_back(place);
 }
 
-void DataTypeAggregateFunction::serializeText(const IColumn & column, size_t row_num, WriteBuffer & ostr) const
+void DataTypeAggregateFunction::serializeText(const IColumn & column, size_t row_num, WriteBuffer & ostr, const FormatSettings &) const
 {
     writeString(serializeToString(function, column, row_num), ostr);
 }
 
 
-void DataTypeAggregateFunction::serializeTextEscaped(const IColumn & column, size_t row_num, WriteBuffer & ostr) const
+void DataTypeAggregateFunction::serializeTextEscaped(const IColumn & column, size_t row_num, WriteBuffer & ostr, const FormatSettings &) const
 {
     writeEscapedString(serializeToString(function, column, row_num), ostr);
 }
 
 
-void DataTypeAggregateFunction::deserializeTextEscaped(IColumn & column, ReadBuffer & istr) const
+void DataTypeAggregateFunction::deserializeTextEscaped(IColumn & column, ReadBuffer & istr, const FormatSettings &) const
 {
     String s;
     readEscapedString(s, istr);
@@ -196,13 +198,13 @@ void DataTypeAggregateFunction::deserializeTextEscaped(IColumn & column, ReadBuf
 }
 
 
-void DataTypeAggregateFunction::serializeTextQuoted(const IColumn & column, size_t row_num, WriteBuffer & ostr) const
+void DataTypeAggregateFunction::serializeTextQuoted(const IColumn & column, size_t row_num, WriteBuffer & ostr, const FormatSettings &) const
 {
     writeQuotedString(serializeToString(function, column, row_num), ostr);
 }
 
 
-void DataTypeAggregateFunction::deserializeTextQuoted(IColumn & column, ReadBuffer & istr) const
+void DataTypeAggregateFunction::deserializeTextQuoted(IColumn & column, ReadBuffer & istr, const FormatSettings &) const
 {
     String s;
     readQuotedStringWithSQLStyle(s, istr);
@@ -210,13 +212,13 @@ void DataTypeAggregateFunction::deserializeTextQuoted(IColumn & column, ReadBuff
 }
 
 
-void DataTypeAggregateFunction::serializeTextJSON(const IColumn & column, size_t row_num, WriteBuffer & ostr, const FormatSettingsJSON &) const
+void DataTypeAggregateFunction::serializeTextJSON(const IColumn & column, size_t row_num, WriteBuffer & ostr, const FormatSettings &) const
 {
     writeJSONString(serializeToString(function, column, row_num), ostr);
 }
 
 
-void DataTypeAggregateFunction::deserializeTextJSON(IColumn & column, ReadBuffer & istr) const
+void DataTypeAggregateFunction::deserializeTextJSON(IColumn & column, ReadBuffer & istr, const FormatSettings &) const
 {
     String s;
     readJSONString(s, istr);
@@ -224,22 +226,22 @@ void DataTypeAggregateFunction::deserializeTextJSON(IColumn & column, ReadBuffer
 }
 
 
-void DataTypeAggregateFunction::serializeTextXML(const IColumn & column, size_t row_num, WriteBuffer & ostr) const
+void DataTypeAggregateFunction::serializeTextXML(const IColumn & column, size_t row_num, WriteBuffer & ostr, const FormatSettings &) const
 {
     writeXMLString(serializeToString(function, column, row_num), ostr);
 }
 
 
-void DataTypeAggregateFunction::serializeTextCSV(const IColumn & column, size_t row_num, WriteBuffer & ostr) const
+void DataTypeAggregateFunction::serializeTextCSV(const IColumn & column, size_t row_num, WriteBuffer & ostr, const FormatSettings &) const
 {
     writeCSV(serializeToString(function, column, row_num), ostr);
 }
 
 
-void DataTypeAggregateFunction::deserializeTextCSV(IColumn & column, ReadBuffer & istr, const char delimiter) const
+void DataTypeAggregateFunction::deserializeTextCSV(IColumn & column, ReadBuffer & istr, const FormatSettings & settings) const
 {
     String s;
-    readCSV(s, istr, delimiter);
+    readCSV(s, istr, settings.csv.delimiter);
     deserializeFromString(function, column, s);
 }
 

dbms/src/DataTypes/DataTypeAggregateFunction.h

@@ -46,16 +46,16 @@ public:
     void deserializeBinary(IColumn & column, ReadBuffer & istr) const override;
     void serializeBinaryBulk(const IColumn & column, WriteBuffer & ostr, size_t offset, size_t limit) const override;
     void deserializeBinaryBulk(IColumn & column, ReadBuffer & istr, size_t limit, double avg_value_size_hint) const override;
-    void serializeText(const IColumn & column, size_t row_num, WriteBuffer & ostr) const override;
-    void serializeTextEscaped(const IColumn & column, size_t row_num, WriteBuffer & ostr) const override;
-    void deserializeTextEscaped(IColumn & column, ReadBuffer & istr) const override;
-    void serializeTextQuoted(const IColumn & column, size_t row_num, WriteBuffer & ostr) const override;
-    void deserializeTextQuoted(IColumn & column, ReadBuffer & istr) const override;
-    void serializeTextJSON(const IColumn & column, size_t row_num, WriteBuffer & ostr, const FormatSettingsJSON &) const override;
-    void deserializeTextJSON(IColumn & column, ReadBuffer & istr) const override;
-    void serializeTextXML(const IColumn & column, size_t row_num, WriteBuffer & ostr) const override;
-    void serializeTextCSV(const IColumn & column, size_t row_num, WriteBuffer & ostr) const override;
-    void deserializeTextCSV(IColumn & column, ReadBuffer & istr, const char delimiter) const override;
+    void serializeText(const IColumn & column, size_t row_num, WriteBuffer & ostr, const FormatSettings &) const override;
+    void serializeTextEscaped(const IColumn & column, size_t row_num, WriteBuffer & ostr, const FormatSettings &) const override;
+    void deserializeTextEscaped(IColumn & column, ReadBuffer & istr, const FormatSettings &) const override;
+    void serializeTextQuoted(const IColumn & column, size_t row_num, WriteBuffer & ostr, const FormatSettings &) const override;
+    void deserializeTextQuoted(IColumn & column, ReadBuffer & istr, const FormatSettings &) const override;
+    void serializeTextJSON(const IColumn & column, size_t row_num, WriteBuffer & ostr, const FormatSettings &) const override;
+    void deserializeTextJSON(IColumn & column, ReadBuffer & istr, const FormatSettings &) const override;
+    void serializeTextXML(const IColumn & column, size_t row_num, WriteBuffer & ostr, const FormatSettings &) const override;
+    void serializeTextCSV(const IColumn & column, size_t row_num, WriteBuffer & ostr, const FormatSettings &) const override;
+    void deserializeTextCSV(IColumn & column, ReadBuffer & istr, const FormatSettings & settings) const override;
 
     MutableColumnPtr createColumn() const override;
 

dbms/src/DataTypes/DataTypeArray.cpp

@@ -5,6 +5,7 @@
 #include <IO/ReadBufferFromString.h>
 #include <IO/WriteBufferFromString.h>
 
+#include <DataTypes/FormatSettings.h>
 #include <DataTypes/DataTypesNumber.h>
 #include <DataTypes/DataTypeArray.h>
 #include <DataTypes/DataTypeFactory.h>
@@ -310,51 +311,51 @@ static void deserializeTextImpl(IColumn & column, ReadBuffer & istr, Reader && r
 }
 
 
-void DataTypeArray::serializeText(const IColumn & column, size_t row_num, WriteBuffer & ostr) const
+void DataTypeArray::serializeText(const IColumn & column, size_t row_num, WriteBuffer & ostr, const FormatSettings & settings) const
 {
     serializeTextImpl(column, row_num, ostr,
         [&](const IColumn & nested_column, size_t i)
         {
-                        nested->serializeTextQuoted(nested_column, i, ostr);
+            nested->serializeTextQuoted(nested_column, i, ostr, settings);
         });
 }
 
 
-void DataTypeArray::deserializeText(IColumn & column, ReadBuffer & istr) const
+void DataTypeArray::deserializeText(IColumn & column, ReadBuffer & istr, const FormatSettings & settings) const
 {
     deserializeTextImpl(column, istr,
         [&](IColumn & nested_column)
         {
-                        nested->deserializeTextQuoted(nested_column, istr);
+            nested->deserializeTextQuoted(nested_column, istr, settings);
         });
 }
 
 
-void DataTypeArray::serializeTextEscaped(const IColumn & column, size_t row_num, WriteBuffer & ostr) const
+void DataTypeArray::serializeTextEscaped(const IColumn & column, size_t row_num, WriteBuffer & ostr, const FormatSettings & settings) const
 {
-    serializeText(column, row_num, ostr);
+    serializeText(column, row_num, ostr, settings);
 }
 
 
-void DataTypeArray::deserializeTextEscaped(IColumn & column, ReadBuffer & istr) const
+void DataTypeArray::deserializeTextEscaped(IColumn & column, ReadBuffer & istr, const FormatSettings & settings) const
 {
-    deserializeText(column, istr);
+    deserializeText(column, istr, settings);
 }
 
 
-void DataTypeArray::serializeTextQuoted(const IColumn & column, size_t row_num, WriteBuffer & ostr) const
+void DataTypeArray::serializeTextQuoted(const IColumn & column, size_t row_num, WriteBuffer & ostr, const FormatSettings & settings) const
 {
-    serializeText(column, row_num, ostr);
+    serializeText(column, row_num, ostr, settings);
 }
 
 
-void DataTypeArray::deserializeTextQuoted(IColumn & column, ReadBuffer & istr) const
+void DataTypeArray::deserializeTextQuoted(IColumn & column, ReadBuffer & istr, const FormatSettings & settings) const
 {
-    deserializeText(column, istr);
+    deserializeText(column, istr, settings);
 }
 
 
-void DataTypeArray::serializeTextJSON(const IColumn & column, size_t row_num, WriteBuffer & ostr, const FormatSettingsJSON & settings) const
+void DataTypeArray::serializeTextJSON(const IColumn & column, size_t row_num, WriteBuffer & ostr, const FormatSettings & settings) const
 {
     const ColumnArray & column_array = static_cast<const ColumnArray &>(column);
     const ColumnArray::Offsets & offsets = column_array.getOffsets();
@@ -375,13 +376,13 @@ void DataTypeArray::serializeTextJSON(const IColumn & column, size_t row_num, Wr
 }
 
 
-void DataTypeArray::deserializeTextJSON(IColumn & column, ReadBuffer & istr) const
+void DataTypeArray::deserializeTextJSON(IColumn & column, ReadBuffer & istr, const FormatSettings & settings) const
 {
-    deserializeTextImpl(column, istr, [&](IColumn & nested_column) { nested->deserializeTextJSON(nested_column, istr); });
+    deserializeTextImpl(column, istr, [&](IColumn & nested_column) { nested->deserializeTextJSON(nested_column, istr, settings); });
 }
 
 
-void DataTypeArray::serializeTextXML(const IColumn & column, size_t row_num, WriteBuffer & ostr) const
+void DataTypeArray::serializeTextXML(const IColumn & column, size_t row_num, WriteBuffer & ostr, const FormatSettings & settings) const
 {
     const ColumnArray & column_array = static_cast<const ColumnArray &>(column);
     const ColumnArray::Offsets & offsets = column_array.getOffsets();
@@ -395,28 +396,28 @@ void DataTypeArray::serializeTextXML(const IColumn & column, size_t row_num, Wri
     for (size_t i = offset; i < next_offset; ++i)
     {
         writeCString("<elem>", ostr);
-        nested->serializeTextXML(nested_column, i, ostr);
+        nested->serializeTextXML(nested_column, i, ostr, settings);
         writeCString("</elem>", ostr);
     }
     writeCString("</array>", ostr);
 }
 
 
-void DataTypeArray::serializeTextCSV(const IColumn & column, size_t row_num, WriteBuffer & ostr) const
+void DataTypeArray::serializeTextCSV(const IColumn & column, size_t row_num, WriteBuffer & ostr, const FormatSettings & settings) const
 {
     /// There is no good way to serialize an array in CSV. Therefore, we serialize it into a string, and then write the resulting string in CSV.
     WriteBufferFromOwnString wb;
-    serializeText(column, row_num, wb);
+    serializeText(column, row_num, wb, settings);
     writeCSV(wb.str(), ostr);
 }
 
 
-void DataTypeArray::deserializeTextCSV(IColumn & column, ReadBuffer & istr, const char delimiter) const
+void DataTypeArray::deserializeTextCSV(IColumn & column, ReadBuffer & istr, const FormatSettings & settings) const
 {
     String s;
-    readCSV(s, istr, delimiter);
+    readCSV(s, istr, settings.csv.delimiter);
     ReadBufferFromString rb(s);
-    deserializeText(column, rb);
+    deserializeText(column, rb, settings);
 }
 
 

dbms/src/DataTypes/DataTypeArray.h

@@ -38,22 +38,22 @@ public:
     void serializeBinary(const IColumn & column, size_t row_num, WriteBuffer & ostr) const override;
     void deserializeBinary(IColumn & column, ReadBuffer & istr) const override;
 
-    void serializeText(const IColumn & column, size_t row_num, WriteBuffer & ostr) const override;
-    void deserializeText(IColumn & column, ReadBuffer & istr) const;
+    void serializeText(const IColumn & column, size_t row_num, WriteBuffer & ostr, const FormatSettings &) const override;
+    void deserializeText(IColumn & column, ReadBuffer & istr, const FormatSettings &) const;
 
-    void serializeTextEscaped(const IColumn & column, size_t row_num, WriteBuffer & ostr) const override;
-    void deserializeTextEscaped(IColumn & column, ReadBuffer & istr) const override;
+    void serializeTextEscaped(const IColumn & column, size_t row_num, WriteBuffer & ostr, const FormatSettings &) const override;
+    void deserializeTextEscaped(IColumn & column, ReadBuffer & istr, const FormatSettings &) const override;
 
-    void serializeTextQuoted(const IColumn & column, size_t row_num, WriteBuffer & ostr) const override;
-    void deserializeTextQuoted(IColumn & column, ReadBuffer & istr) const override;
+    void serializeTextQuoted(const IColumn & column, size_t row_num, WriteBuffer & ostr, const FormatSettings &) const override;
+    void deserializeTextQuoted(IColumn & column, ReadBuffer & istr, const FormatSettings &) const override;
 
-    void serializeTextJSON(const IColumn & column, size_t row_num, WriteBuffer & ostr, const FormatSettingsJSON & settings) const override;
-    void deserializeTextJSON(IColumn & column, ReadBuffer & istr) const override;
+    void serializeTextJSON(const IColumn & column, size_t row_num, WriteBuffer & ostr, const FormatSettings &) const override;
+    void deserializeTextJSON(IColumn & column, ReadBuffer & istr, const FormatSettings &) const override;
 
-    void serializeTextXML(const IColumn & column, size_t row_num, WriteBuffer & ostr) const override;
+    void serializeTextXML(const IColumn & column, size_t row_num, WriteBuffer & ostr, const FormatSettings &) const override;
 
-    void serializeTextCSV(const IColumn & column, size_t row_num, WriteBuffer & ostr) const override;
-    void deserializeTextCSV(IColumn & column, ReadBuffer & istr, const char delimiter) const override;
+    void serializeTextCSV(const IColumn & column, size_t row_num, WriteBuffer & ostr, const FormatSettings &) const override;
+    void deserializeTextCSV(IColumn & column, ReadBuffer & istr, const FormatSettings &) const override;
 
     /** Streaming serialization of arrays is arranged in a special way:
       * - elements placed in a row are written/read without array sizes;

dbms/src/DataTypes/DataTypeDate.cpp

@@ -9,7 +9,7 @@
 namespace DB
 {
 
-void DataTypeDate::serializeText(const IColumn & column, size_t row_num, WriteBuffer & ostr) const
+void DataTypeDate::serializeText(const IColumn & column, size_t row_num, WriteBuffer & ostr, const FormatSettings &) const
 {
     writeDateText(DayNum(static_cast<const ColumnUInt16 &>(column).getData()[row_num]), ostr);
 }
@@ -21,24 +21,24 @@ static void deserializeText(IColumn & column, ReadBuffer & istr)
     static_cast<ColumnUInt16 &>(column).getData().push_back(x);
 }
 
-void DataTypeDate::serializeTextEscaped(const IColumn & column, size_t row_num, WriteBuffer & ostr) const
+void DataTypeDate::serializeTextEscaped(const IColumn & column, size_t row_num, WriteBuffer & ostr, const FormatSettings & settings) const
 {
-    serializeText(column, row_num, ostr);
+    serializeText(column, row_num, ostr, settings);
 }
 
-void DataTypeDate::deserializeTextEscaped(IColumn & column, ReadBuffer & istr) const
+void DataTypeDate::deserializeTextEscaped(IColumn & column, ReadBuffer & istr, const FormatSettings &) const
 {
     deserializeText(column, istr);
 }
 
-void DataTypeDate::serializeTextQuoted(const IColumn & column, size_t row_num, WriteBuffer & ostr) const
+void DataTypeDate::serializeTextQuoted(const IColumn & column, size_t row_num, WriteBuffer & ostr, const FormatSettings & settings) const
 {
     writeChar('\'', ostr);
-    serializeText(column, row_num, ostr);
+    serializeText(column, row_num, ostr, settings);
     writeChar('\'', ostr);
 }
 
-void DataTypeDate::deserializeTextQuoted(IColumn & column, ReadBuffer & istr) const
+void DataTypeDate::deserializeTextQuoted(IColumn & column, ReadBuffer & istr, const FormatSettings &) const
 {
     DayNum x;
     assertChar('\'', istr);
@@ -47,14 +47,14 @@ void DataTypeDate::deserializeTextQuoted(IColumn & column, ReadBuffer & istr) co
     static_cast<ColumnUInt16 &>(column).getData().push_back(x);    /// It's important to do this at the end - for exception safety.
 }
 
-void DataTypeDate::serializeTextJSON(const IColumn & column, size_t row_num, WriteBuffer & ostr, const FormatSettingsJSON &) const
+void DataTypeDate::serializeTextJSON(const IColumn & column, size_t row_num, WriteBuffer & ostr, const FormatSettings & settings) const
 {
     writeChar('"', ostr);
-    serializeText(column, row_num, ostr);
+    serializeText(column, row_num, ostr, settings);
     writeChar('"', ostr);
 }
 
-void DataTypeDate::deserializeTextJSON(IColumn & column, ReadBuffer & istr) const
+void DataTypeDate::deserializeTextJSON(IColumn & column, ReadBuffer & istr, const FormatSettings &) const
 {
     DayNum x;
     assertChar('"', istr);
@@ -63,14 +63,14 @@ void DataTypeDate::deserializeTextJSON(IColumn & column, ReadBuffer & istr) cons
     static_cast<ColumnUInt16 &>(column).getData().push_back(x);
 }
 
-void DataTypeDate::serializeTextCSV(const IColumn & column, size_t row_num, WriteBuffer & ostr) const
+void DataTypeDate::serializeTextCSV(const IColumn & column, size_t row_num, WriteBuffer & ostr, const FormatSettings & settings) const
 {
     writeChar('"', ostr);
-    serializeText(column, row_num, ostr);
+    serializeText(column, row_num, ostr, settings);
     writeChar('"', ostr);
 }
 
-void DataTypeDate::deserializeTextCSV(IColumn & column, ReadBuffer & istr, const char /*delimiter*/) const
+void DataTypeDate::deserializeTextCSV(IColumn & column, ReadBuffer & istr, const FormatSettings &) const
 {
     LocalDate value;
     readCSV(value, istr);

dbms/src/DataTypes/DataTypeDate.h

@@ -11,15 +11,15 @@ class DataTypeDate final : public DataTypeNumberBase<UInt16>
 public:
     const char * getFamilyName() const override { return "Date"; }
 
-    void serializeText(const IColumn & column, size_t row_num, WriteBuffer & ostr) const override;
-    void serializeTextEscaped(const IColumn & column, size_t row_num, WriteBuffer & ostr) const override;
-    void deserializeTextEscaped(IColumn & column, ReadBuffer & istr) const override;
-    void serializeTextQuoted(const IColumn & column, size_t row_num, WriteBuffer & ostr) const override;
-    void deserializeTextQuoted(IColumn & column, ReadBuffer & istr) const override;
-    void serializeTextJSON(const IColumn & column, size_t row_num, WriteBuffer & ostr, const FormatSettingsJSON &) const override;
-    void deserializeTextJSON(IColumn & column, ReadBuffer & istr) const override;
-    void serializeTextCSV(const IColumn & column, size_t row_num, WriteBuffer & ostr) const override;
-    void deserializeTextCSV(IColumn & column, ReadBuffer & istr, const char delimiter) const override;
+    void serializeText(const IColumn & column, size_t row_num, WriteBuffer & ostr, const FormatSettings &) const override;
+    void serializeTextEscaped(const IColumn & column, size_t row_num, WriteBuffer & ostr, const FormatSettings &) const override;
+    void deserializeTextEscaped(IColumn & column, ReadBuffer & istr, const FormatSettings &) const override;
+    void serializeTextQuoted(const IColumn & column, size_t row_num, WriteBuffer & ostr, const FormatSettings &) const override;
+    void deserializeTextQuoted(IColumn & column, ReadBuffer & istr, const FormatSettings &) const override;
+    void serializeTextJSON(const IColumn & column, size_t row_num, WriteBuffer & ostr, const FormatSettings &) const override;
+    void deserializeTextJSON(IColumn & column, ReadBuffer & istr, const FormatSettings &) const override;
+    void serializeTextCSV(const IColumn & column, size_t row_num, WriteBuffer & ostr, const FormatSettings &) const override;
+    void deserializeTextCSV(IColumn & column, ReadBuffer & istr, const FormatSettings & settings) const override;
 
     bool canBeUsedAsVersion() const override { return true; }
     bool isDateOrDateTime() const override { return true; }

dbms/src/DataTypes/DataTypeDateTime.cpp

@@ -1,9 +1,11 @@
 #include <IO/ReadHelpers.h>
 #include <IO/WriteHelpers.h>
+#include <IO/parseDateTimeBestEffort.h>
 
 #include <common/DateLUT.h>
 #include <Common/typeid_cast.h>
 #include <Columns/ColumnsNumber.h>
+#include <DataTypes/FormatSettings.h>
 #include <DataTypes/DataTypeDateTime.h>
 #include <DataTypes/DataTypeFactory.h>
 
@@ -18,7 +20,8 @@ namespace DB
 
 DataTypeDateTime::DataTypeDateTime(const std::string & time_zone_name)
     : has_explicit_time_zone(!time_zone_name.empty()),
-    time_zone(DateLUT::instance(time_zone_name))
+    time_zone(DateLUT::instance(time_zone_name)),
+    utc_time_zone(DateLUT::instance("UTC"))
 {
 }
 
@@ -32,36 +35,53 @@ std::string DataTypeDateTime::getName() const
     return out.str();
 }
 
-void DataTypeDateTime::serializeText(const IColumn & column, size_t row_num, WriteBuffer & ostr) const
+void DataTypeDateTime::serializeText(const IColumn & column, size_t row_num, WriteBuffer & ostr, const FormatSettings &) const
 {
     writeDateTimeText(static_cast<const ColumnUInt32 &>(column).getData()[row_num], ostr, time_zone);
 }
 
-void DataTypeDateTime::serializeTextEscaped(const IColumn & column, size_t row_num, WriteBuffer & ostr) const
+void DataTypeDateTime::serializeTextEscaped(const IColumn & column, size_t row_num, WriteBuffer & ostr, const FormatSettings & settings) const
 {
-    serializeText(column, row_num, ostr);
+    serializeText(column, row_num, ostr, settings);
 }
 
-void DataTypeDateTime::deserializeTextEscaped(IColumn & column, ReadBuffer & istr) const
+
+static inline void readText(time_t & x, ReadBuffer & istr, const FormatSettings & settings, const DateLUTImpl & time_zone, const DateLUTImpl & utc_time_zone)
+{
+    switch (settings.date_time_input_format)
+    {
+        case FormatSettings::DateTimeInputFormat::Basic:
+            readDateTimeText(x, istr, time_zone);
+            return;
+        case FormatSettings::DateTimeInputFormat::BestEffort:
+            parseDateTimeBestEffort(x, istr, time_zone, utc_time_zone);
+            return;
+        default:
+            __builtin_unreachable();
+    }
+}
+
+
+void DataTypeDateTime::deserializeTextEscaped(IColumn & column, ReadBuffer & istr, const FormatSettings & settings) const
 {
     time_t x;
-    readDateTimeText(x, istr, time_zone);
+    readText(x, istr, settings, time_zone, utc_time_zone);
     static_cast<ColumnUInt32 &>(column).getData().push_back(x);
 }
 
-void DataTypeDateTime::serializeTextQuoted(const IColumn & column, size_t row_num, WriteBuffer & ostr) const
+void DataTypeDateTime::serializeTextQuoted(const IColumn & column, size_t row_num, WriteBuffer & ostr, const FormatSettings & settings) const
 {
     writeChar('\'', ostr);
-    serializeText(column, row_num, ostr);
+    serializeText(column, row_num, ostr, settings);
     writeChar('\'', ostr);
 }
 
-void DataTypeDateTime::deserializeTextQuoted(IColumn & column, ReadBuffer & istr) const
+void DataTypeDateTime::deserializeTextQuoted(IColumn & column, ReadBuffer & istr, const FormatSettings & settings) const
 {
     time_t x;
     if (checkChar('\'', istr)) /// Cases: '2017-08-31 18:36:48' or '1504193808'
     {
-        readDateTimeText(x, istr, time_zone);
+        readText(x, istr, settings, time_zone, utc_time_zone);
         assertChar('\'', istr);
     }
     else /// Just 1504193808 or 01504193808
@@ -71,19 +91,19 @@ void DataTypeDateTime::deserializeTextQuoted(IColumn & column, ReadBuffer & istr
     static_cast<ColumnUInt32 &>(column).getData().push_back(x);    /// It's important to do this at the end - for exception safety.
 }
 
-void DataTypeDateTime::serializeTextJSON(const IColumn & column, size_t row_num, WriteBuffer & ostr, const FormatSettingsJSON &) const
+void DataTypeDateTime::serializeTextJSON(const IColumn & column, size_t row_num, WriteBuffer & ostr, const FormatSettings & settings) const
 {
     writeChar('"', ostr);
-    serializeText(column, row_num, ostr);
+    serializeText(column, row_num, ostr, settings);
     writeChar('"', ostr);
 }
 
-void DataTypeDateTime::deserializeTextJSON(IColumn & column, ReadBuffer & istr) const
+void DataTypeDateTime::deserializeTextJSON(IColumn & column, ReadBuffer & istr, const FormatSettings & settings) const
 {
     time_t x;
     if (checkChar('"', istr))
     {
-        readDateTimeText(x, istr, time_zone);
+        readText(x, istr, settings, time_zone, utc_time_zone);
         assertChar('"', istr);
     }
     else
@@ -93,17 +113,30 @@ void DataTypeDateTime::deserializeTextJSON(IColumn & column, ReadBuffer & istr)
     static_cast<ColumnUInt32 &>(column).getData().push_back(x);
 }
 
-void DataTypeDateTime::serializeTextCSV(const IColumn & column, size_t row_num, WriteBuffer & ostr) const
+void DataTypeDateTime::serializeTextCSV(const IColumn & column, size_t row_num, WriteBuffer & ostr, const FormatSettings & settings) const
 {
     writeChar('"', ostr);
-    serializeText(column, row_num, ostr);
+    serializeText(column, row_num, ostr, settings);
     writeChar('"', ostr);
 }
 
-void DataTypeDateTime::deserializeTextCSV(IColumn & column, ReadBuffer & istr, const char /*delimiter*/) const
+void DataTypeDateTime::deserializeTextCSV(IColumn & column, ReadBuffer & istr, const FormatSettings & settings) const
 {
     time_t x;
-    readDateTimeCSV(x, istr, time_zone);
+
+    if (istr.eof())
+        throwReadAfterEOF();
+
+    char maybe_quote = *istr.position();
+
+    if (maybe_quote == '\'' || maybe_quote == '\"')
+        ++istr.position();
+
+    readText(x, istr, settings, time_zone, utc_time_zone);
+
+    if (maybe_quote == '\'' || maybe_quote == '\"')
+        assertChar(maybe_quote, istr);
+
     static_cast<ColumnUInt32 &>(column).getData().push_back(x);
 }
 

dbms/src/DataTypes/DataTypeDateTime.h

@@ -36,15 +36,15 @@ public:
     const char * getFamilyName() const override { return "DateTime"; }
     std::string getName() const override;
 
-    void serializeText(const IColumn & column, size_t row_num, WriteBuffer & ostr) const override;
-    void serializeTextEscaped(const IColumn & column, size_t row_num, WriteBuffer & ostr) const override;
-    void deserializeTextEscaped(IColumn & column, ReadBuffer & istr) const override;
-    void serializeTextQuoted(const IColumn & column, size_t row_num, WriteBuffer & ostr) const override;
-    void deserializeTextQuoted(IColumn & column, ReadBuffer & istr) const override;
-    void serializeTextJSON(const IColumn & column, size_t row_num, WriteBuffer & ostr, const FormatSettingsJSON &) const override;
-    void deserializeTextJSON(IColumn & column, ReadBuffer & istr) const override;
-    void serializeTextCSV(const IColumn & column, size_t row_num, WriteBuffer & ostr) const override;
-    void deserializeTextCSV(IColumn & column, ReadBuffer & istr, const char delimiter) const override;
+    void serializeText(const IColumn & column, size_t row_num, WriteBuffer & ostr, const FormatSettings &) const override;
+    void serializeTextEscaped(const IColumn & column, size_t row_num, WriteBuffer & ostr, const FormatSettings &) const override;
+    void deserializeTextEscaped(IColumn & column, ReadBuffer & istr, const FormatSettings &) const override;
+    void serializeTextQuoted(const IColumn & column, size_t row_num, WriteBuffer & ostr, const FormatSettings &) const override;
+    void deserializeTextQuoted(IColumn & column, ReadBuffer & istr, const FormatSettings &) const override;
+    void serializeTextJSON(const IColumn & column, size_t row_num, WriteBuffer & ostr, const FormatSettings &) const override;
+    void deserializeTextJSON(IColumn & column, ReadBuffer & istr, const FormatSettings &) const override;
+    void serializeTextCSV(const IColumn & column, size_t row_num, WriteBuffer & ostr, const FormatSettings &) const override;
+    void deserializeTextCSV(IColumn & column, ReadBuffer & istr, const FormatSettings & settings) const override;
 
     bool canBeUsedAsVersion() const override { return true; }
     bool isDateOrDateTime() const override { return true; }
@@ -57,6 +57,7 @@ public:
 private:
     bool has_explicit_time_zone;
     const DateLUTImpl & time_zone;
+    const DateLUTImpl & utc_time_zone;
 };
 
 }

dbms/src/DataTypes/DataTypeEnum.cpp

@@ -1,4 +1,5 @@
 #include <IO/WriteBufferFromString.h>
+#include <DataTypes/FormatSettings.h>
 #include <DataTypes/DataTypeEnum.h>
 #include <DataTypes/DataTypeFactory.h>
 #include <Parsers/IAST.h>
@@ -129,19 +130,19 @@ void DataTypeEnum<Type>::deserializeBinary(IColumn & column, ReadBuffer & istr)
 }
 
 template <typename Type>
-void DataTypeEnum<Type>::serializeText(const IColumn & column, size_t row_num, WriteBuffer & ostr) const
+void DataTypeEnum<Type>::serializeText(const IColumn & column, size_t row_num, WriteBuffer & ostr, const FormatSettings &) const
 {
     writeString(getNameForValue(static_cast<const ColumnType &>(column).getData()[row_num]), ostr);
 }
 
 template <typename Type>
-void DataTypeEnum<Type>::serializeTextEscaped(const IColumn & column, size_t row_num, WriteBuffer & ostr) const
+void DataTypeEnum<Type>::serializeTextEscaped(const IColumn & column, size_t row_num, WriteBuffer & ostr, const FormatSettings &) const
 {
     writeEscapedString(getNameForValue(static_cast<const ColumnType &>(column).getData()[row_num]), ostr);
 }
 
 template <typename Type>
-void DataTypeEnum<Type>::deserializeTextEscaped(IColumn & column, ReadBuffer & istr) const
+void DataTypeEnum<Type>::deserializeTextEscaped(IColumn & column, ReadBuffer & istr, const FormatSettings &) const
 {
     /// NOTE It would be nice to do without creating a temporary object - at least extract std::string out.
     std::string name;
@@ -150,13 +151,13 @@ void DataTypeEnum<Type>::deserializeTextEscaped(IColumn & column, ReadBuffer & i
 }
 
 template <typename Type>
-void DataTypeEnum<Type>::serializeTextQuoted(const IColumn & column, size_t row_num, WriteBuffer & ostr) const
+void DataTypeEnum<Type>::serializeTextQuoted(const IColumn & column, size_t row_num, WriteBuffer & ostr, const FormatSettings &) const
 {
     writeQuotedString(getNameForValue(static_cast<const ColumnType &>(column).getData()[row_num]), ostr);
 }
 
 template <typename Type>
-void DataTypeEnum<Type>::deserializeTextQuoted(IColumn & column, ReadBuffer & istr) const
+void DataTypeEnum<Type>::deserializeTextQuoted(IColumn & column, ReadBuffer & istr, const FormatSettings &) const
 {
     std::string name;
     readQuotedStringWithSQLStyle(name, istr);
@@ -164,19 +165,19 @@ void DataTypeEnum<Type>::deserializeTextQuoted(IColumn & column, ReadBuffer & is
 }
 
 template <typename Type>
-void DataTypeEnum<Type>::serializeTextJSON(const IColumn & column, size_t row_num, WriteBuffer & ostr, const FormatSettingsJSON &) const
+void DataTypeEnum<Type>::serializeTextJSON(const IColumn & column, size_t row_num, WriteBuffer & ostr, const FormatSettings &) const
 {
     writeJSONString(getNameForValue(static_cast<const ColumnType &>(column).getData()[row_num]), ostr);
 }
 
 template <typename Type>
-void DataTypeEnum<Type>::serializeTextXML(const IColumn & column, size_t row_num, WriteBuffer & ostr) const
+void DataTypeEnum<Type>::serializeTextXML(const IColumn & column, size_t row_num, WriteBuffer & ostr, const FormatSettings &) const
 {
     writeXMLString(getNameForValue(static_cast<const ColumnType &>(column).getData()[row_num]), ostr);
 }
 
 template <typename Type>
-void DataTypeEnum<Type>::deserializeTextJSON(IColumn & column, ReadBuffer & istr) const
+void DataTypeEnum<Type>::deserializeTextJSON(IColumn & column, ReadBuffer & istr, const FormatSettings &) const
 {
     std::string name;
     readJSONString(name, istr);
@@ -184,16 +185,16 @@ void DataTypeEnum<Type>::deserializeTextJSON(IColumn & column, ReadBuffer & istr
 }
 
 template <typename Type>
-void DataTypeEnum<Type>::serializeTextCSV(const IColumn & column, size_t row_num, WriteBuffer & ostr) const
+void DataTypeEnum<Type>::serializeTextCSV(const IColumn & column, size_t row_num, WriteBuffer & ostr, const FormatSettings &) const
 {
     writeCSVString(getNameForValue(static_cast<const ColumnType &>(column).getData()[row_num]), ostr);
 }
 
 template <typename Type>
-void DataTypeEnum<Type>::deserializeTextCSV(IColumn & column, ReadBuffer & istr, const char delimiter) const
+void DataTypeEnum<Type>::deserializeTextCSV(IColumn & column, ReadBuffer & istr, const FormatSettings & settings) const
 {
     std::string name;
-    readCSVString(name, istr, delimiter);
+    readCSVString(name, istr, settings.csv.delimiter);
     static_cast<ColumnType &>(column).getData().push_back(getValue(StringRef(name)));
 }
 

dbms/src/DataTypes/DataTypeEnum.h

@@ -90,16 +90,16 @@ public:
     void deserializeBinary(Field & field, ReadBuffer & istr) const override;
     void serializeBinary(const IColumn & column, size_t row_num, WriteBuffer & ostr) const override;
     void deserializeBinary(IColumn & column, ReadBuffer & istr) const override;
-    void serializeText(const IColumn & column, size_t row_num, WriteBuffer & ostr) const override;
-    void serializeTextEscaped(const IColumn & column, size_t row_num, WriteBuffer & ostr) const override;
-    void deserializeTextEscaped(IColumn & column, ReadBuffer & istr) const override;
-    void serializeTextQuoted(const IColumn & column, size_t row_num, WriteBuffer & ostr) const override;
-    void deserializeTextQuoted(IColumn & column, ReadBuffer & istr) const override;
-    void serializeTextJSON(const IColumn & column, size_t row_num, WriteBuffer & ostr, const FormatSettingsJSON &) const override;
-    void deserializeTextJSON(IColumn & column, ReadBuffer & istr) const override;
-    void serializeTextXML(const IColumn & column, size_t row_num, WriteBuffer & ostr) const override;
-    void serializeTextCSV(const IColumn & column, size_t row_num, WriteBuffer & ostr) const override;
-    void deserializeTextCSV(IColumn & column, ReadBuffer & istr, const char delimiter) const override;
+    void serializeText(const IColumn & column, size_t row_num, WriteBuffer & ostr, const FormatSettings &) const override;
+    void serializeTextEscaped(const IColumn & column, size_t row_num, WriteBuffer & ostr, const FormatSettings &) const override;
+    void deserializeTextEscaped(IColumn & column, ReadBuffer & istr, const FormatSettings &) const override;
+    void serializeTextQuoted(const IColumn & column, size_t row_num, WriteBuffer & ostr, const FormatSettings &) const override;
+    void deserializeTextQuoted(IColumn & column, ReadBuffer & istr, const FormatSettings &) const override;
+    void serializeTextJSON(const IColumn & column, size_t row_num, WriteBuffer & ostr, const FormatSettings &) const override;
+    void deserializeTextJSON(IColumn & column, ReadBuffer & istr, const FormatSettings &) const override;
+    void serializeTextXML(const IColumn & column, size_t row_num, WriteBuffer & ostr, const FormatSettings &) const override;
+    void serializeTextCSV(const IColumn & column, size_t row_num, WriteBuffer & ostr, const FormatSettings &) const override;
+    void deserializeTextCSV(IColumn & column, ReadBuffer & istr, const FormatSettings & settings) const override;
 
     void serializeBinaryBulk(const IColumn & column, WriteBuffer & ostr, const size_t offset, size_t limit) const override;
     void deserializeBinaryBulk(IColumn & column, ReadBuffer & istr, const size_t limit, const double avg_value_size_hint) const override;

dbms/src/DataTypes/DataTypeFixedString.cpp

@@ -5,6 +5,7 @@
 #include <Columns/ColumnsNumber.h>
 #include <Columns/ColumnConst.h>
 
+#include <DataTypes/FormatSettings.h>
 #include <DataTypes/DataTypeFixedString.h>
 #include <DataTypes/DataTypeFactory.h>
 
@@ -108,13 +109,13 @@ void DataTypeFixedString::deserializeBinaryBulk(IColumn & column, ReadBuffer & i
 }
 
 
-void DataTypeFixedString::serializeText(const IColumn & column, size_t row_num, WriteBuffer & ostr) const
+void DataTypeFixedString::serializeText(const IColumn & column, size_t row_num, WriteBuffer & ostr, const FormatSettings &) const
 {
     writeString(reinterpret_cast<const char *>(&static_cast<const ColumnFixedString &>(column).getChars()[n * row_num]), n, ostr);
 }
 
 
-void DataTypeFixedString::serializeTextEscaped(const IColumn & column, size_t row_num, WriteBuffer & ostr) const
+void DataTypeFixedString::serializeTextEscaped(const IColumn & column, size_t row_num, WriteBuffer & ostr, const FormatSettings &) const
 {
     const char * pos = reinterpret_cast<const char *>(&static_cast<const ColumnFixedString &>(column).getChars()[n * row_num]);
     writeAnyEscapedString<'\''>(pos, pos + n, ostr);
@@ -148,55 +149,55 @@ static inline void read(const DataTypeFixedString & self, IColumn & column, Read
 }
 
 
-void DataTypeFixedString::deserializeTextEscaped(IColumn & column, ReadBuffer & istr) const
+void DataTypeFixedString::deserializeTextEscaped(IColumn & column, ReadBuffer & istr, const FormatSettings &) const
 {
     read(*this, column, [&istr](ColumnFixedString::Chars_t & data) { readEscapedStringInto(data, istr); });
 }
 
 
-void DataTypeFixedString::serializeTextQuoted(const IColumn & column, size_t row_num, WriteBuffer & ostr) const
+void DataTypeFixedString::serializeTextQuoted(const IColumn & column, size_t row_num, WriteBuffer & ostr, const FormatSettings &) const
 {
     const char * pos = reinterpret_cast<const char *>(&static_cast<const ColumnFixedString &>(column).getChars()[n * row_num]);
     writeAnyQuotedString<'\''>(pos, pos + n, ostr);
 }
 
 
-void DataTypeFixedString::deserializeTextQuoted(IColumn & column, ReadBuffer & istr) const
+void DataTypeFixedString::deserializeTextQuoted(IColumn & column, ReadBuffer & istr, const FormatSettings &) const
 {
     read(*this, column, [&istr](ColumnFixedString::Chars_t & data) { readQuotedStringInto<true>(data, istr); });
 }
 
 
-void DataTypeFixedString::serializeTextJSON(const IColumn & column, size_t row_num, WriteBuffer & ostr, const FormatSettingsJSON &) const
+void DataTypeFixedString::serializeTextJSON(const IColumn & column, size_t row_num, WriteBuffer & ostr, const FormatSettings &) const
 {
     const char * pos = reinterpret_cast<const char *>(&static_cast<const ColumnFixedString &>(column).getChars()[n * row_num]);
     writeJSONString(pos, pos + n, ostr);
 }
 
 
-void DataTypeFixedString::deserializeTextJSON(IColumn & column, ReadBuffer & istr) const
+void DataTypeFixedString::deserializeTextJSON(IColumn & column, ReadBuffer & istr, const FormatSettings &) const
 {
     read(*this, column, [&istr](ColumnFixedString::Chars_t & data) { readJSONStringInto(data, istr); });
 }
 
 
-void DataTypeFixedString::serializeTextXML(const IColumn & column, size_t row_num, WriteBuffer & ostr) const
+void DataTypeFixedString::serializeTextXML(const IColumn & column, size_t row_num, WriteBuffer & ostr, const FormatSettings &) const
 {
     const char * pos = reinterpret_cast<const char *>(&static_cast<const ColumnFixedString &>(column).getChars()[n * row_num]);
     writeXMLString(pos, pos + n, ostr);
 }
 
 
-void DataTypeFixedString::serializeTextCSV(const IColumn & column, size_t row_num, WriteBuffer & ostr) const
+void DataTypeFixedString::serializeTextCSV(const IColumn & column, size_t row_num, WriteBuffer & ostr, const FormatSettings &) const
 {
     const char * pos = reinterpret_cast<const char *>(&static_cast<const ColumnFixedString &>(column).getChars()[n * row_num]);
     writeCSVString(pos, pos + n, ostr);
 }
 
 
-void DataTypeFixedString::deserializeTextCSV(IColumn & column, ReadBuffer & istr, const char delimiter) const
+void DataTypeFixedString::deserializeTextCSV(IColumn & column, ReadBuffer & istr, const FormatSettings & settings) const
 {
-    read(*this, column, [&istr, delimiter](ColumnFixedString::Chars_t & data) { readCSVStringInto(data, istr, delimiter); });
+    read(*this, column, [&istr, delimiter = settings.csv.delimiter](ColumnFixedString::Chars_t & data) { readCSVStringInto(data, istr, delimiter); });
 }
 
 

dbms/src/DataTypes/DataTypeFixedString.h

@@ -47,21 +47,21 @@ public:
     void serializeBinaryBulk(const IColumn & column, WriteBuffer & ostr, size_t offset, size_t limit) const override;
     void deserializeBinaryBulk(IColumn & column, ReadBuffer & istr, size_t limit, double avg_value_size_hint) const override;
 
-    void serializeText(const IColumn & column, size_t row_num, WriteBuffer & ostr) const override;
+    void serializeText(const IColumn & column, size_t row_num, WriteBuffer & ostr, const FormatSettings &) const override;
 
-    void serializeTextEscaped(const IColumn & column, size_t row_num, WriteBuffer & ostr) const override;
-    void deserializeTextEscaped(IColumn & column, ReadBuffer & istr) const override;
+    void serializeTextEscaped(const IColumn & column, size_t row_num, WriteBuffer & ostr, const FormatSettings &) const override;
+    void deserializeTextEscaped(IColumn & column, ReadBuffer & istr, const FormatSettings &) const override;
 
-    void serializeTextQuoted(const IColumn & column, size_t row_num, WriteBuffer & ostr) const override;
-    void deserializeTextQuoted(IColumn & column, ReadBuffer & istr) const override;
+    void serializeTextQuoted(const IColumn & column, size_t row_num, WriteBuffer & ostr, const FormatSettings &) const override;
+    void deserializeTextQuoted(IColumn & column, ReadBuffer & istr, const FormatSettings &) const override;
 
-    void serializeTextJSON(const IColumn & column, size_t row_num, WriteBuffer & ostr, const FormatSettingsJSON &) const override;
-    void deserializeTextJSON(IColumn & column, ReadBuffer & istr) const override;
+    void serializeTextJSON(const IColumn & column, size_t row_num, WriteBuffer & ostr, const FormatSettings &) const override;
+    void deserializeTextJSON(IColumn & column, ReadBuffer & istr, const FormatSettings &) const override;
 
-    void serializeTextXML(const IColumn & column, size_t row_num, WriteBuffer & ostr) const override;
+    void serializeTextXML(const IColumn & column, size_t row_num, WriteBuffer & ostr, const FormatSettings &) const override;
 
-    void serializeTextCSV(const IColumn & column, size_t row_num, WriteBuffer & ostr) const override;
-    void deserializeTextCSV(IColumn & column, ReadBuffer & istr, const char delimiter) const override;
+    void serializeTextCSV(const IColumn & column, size_t row_num, WriteBuffer & ostr, const FormatSettings &) const override;
+    void deserializeTextCSV(IColumn & column, ReadBuffer & istr, const FormatSettings &) const override;
 
     MutableColumnPtr createColumn() const override;
 

dbms/src/DataTypes/DataTypeNullable.cpp

@@ -135,18 +135,18 @@ void DataTypeNullable::deserializeBinary(IColumn & column, ReadBuffer & istr) co
 }
 
 
-void DataTypeNullable::serializeTextEscaped(const IColumn & column, size_t row_num, WriteBuffer & ostr) const
+void DataTypeNullable::serializeTextEscaped(const IColumn & column, size_t row_num, WriteBuffer & ostr, const FormatSettings & settings) const
 {
     const ColumnNullable & col = static_cast<const ColumnNullable &>(column);
 
     if (col.isNullAt(row_num))
         writeCString("\\N", ostr);
     else
-        nested_data_type->serializeTextEscaped(col.getNestedColumn(), row_num, ostr);
+        nested_data_type->serializeTextEscaped(col.getNestedColumn(), row_num, ostr, settings);
 }
 
 
-void DataTypeNullable::deserializeTextEscaped(IColumn & column, ReadBuffer & istr) const
+void DataTypeNullable::deserializeTextEscaped(IColumn & column, ReadBuffer & istr, const FormatSettings & settings) const
 {
     /// Little tricky, because we cannot discriminate null from first character.
 
@@ -158,7 +158,7 @@ void DataTypeNullable::deserializeTextEscaped(IColumn & column, ReadBuffer & ist
     {
         safeDeserialize(column,
             [] { return false; },
-            [this, &istr] (IColumn & nested) { nested_data_type->deserializeTextEscaped(nested, istr); } );
+            [this, &istr, &settings] (IColumn & nested) { nested_data_type->deserializeTextEscaped(nested, istr, settings); } );
     }
     else
     {
@@ -178,13 +178,13 @@ void DataTypeNullable::deserializeTextEscaped(IColumn & column, ReadBuffer & ist
                 }
                 return false;
             },
-            [this, &istr] (IColumn & nested)
+            [this, &istr, &settings] (IColumn & nested)
             {
                 if (istr.position() != istr.buffer().begin())
                 {
                     /// We could step back to consume backslash again.
                     --istr.position();
-                    nested_data_type->deserializeTextEscaped(nested, istr);
+                    nested_data_type->deserializeTextEscaped(nested, istr, settings);
                 }
                 else
                 {
@@ -192,7 +192,7 @@ void DataTypeNullable::deserializeTextEscaped(IColumn & column, ReadBuffer & ist
                     ReadBufferFromMemory prefix("\\", 1);
                     ConcatReadBuffer prepended_istr(prefix, istr);
 
-                    nested_data_type->deserializeTextEscaped(nested, prepended_istr);
+                    nested_data_type->deserializeTextEscaped(nested, prepended_istr, settings);
 
                     /// Synchronise cursor position in original buffer.
 
@@ -203,42 +203,42 @@ void DataTypeNullable::deserializeTextEscaped(IColumn & column, ReadBuffer & ist
     }
 }
 
-void DataTypeNullable::serializeTextQuoted(const IColumn & column, size_t row_num, WriteBuffer & ostr) const
+void DataTypeNullable::serializeTextQuoted(const IColumn & column, size_t row_num, WriteBuffer & ostr, const FormatSettings & settings) const
 {
     const ColumnNullable & col = static_cast<const ColumnNullable &>(column);
 
     if (col.isNullAt(row_num))
         writeCString("NULL", ostr);
     else
-        nested_data_type->serializeTextQuoted(col.getNestedColumn(), row_num, ostr);
+        nested_data_type->serializeTextQuoted(col.getNestedColumn(), row_num, ostr, settings);
 }
 
 
-void DataTypeNullable::deserializeTextQuoted(IColumn & column, ReadBuffer & istr) const
+void DataTypeNullable::deserializeTextQuoted(IColumn & column, ReadBuffer & istr, const FormatSettings & settings) const
 {
     safeDeserialize(column,
         [&istr] { return checkStringByFirstCharacterAndAssertTheRestCaseInsensitive("NULL", istr); },
-        [this, &istr] (IColumn & nested) { nested_data_type->deserializeTextQuoted(nested, istr); } );
+        [this, &istr, &settings] (IColumn & nested) { nested_data_type->deserializeTextQuoted(nested, istr, settings); } );
 }
 
-void DataTypeNullable::serializeTextCSV(const IColumn & column, size_t row_num, WriteBuffer & ostr) const
+void DataTypeNullable::serializeTextCSV(const IColumn & column, size_t row_num, WriteBuffer & ostr, const FormatSettings & settings) const
 {
     const ColumnNullable & col = static_cast<const ColumnNullable &>(column);
 
     if (col.isNullAt(row_num))
         writeCString("\\N", ostr);
     else
-        nested_data_type->serializeTextCSV(col.getNestedColumn(), row_num, ostr);
+        nested_data_type->serializeTextCSV(col.getNestedColumn(), row_num, ostr, settings);
 }
 
-void DataTypeNullable::deserializeTextCSV(IColumn & column, ReadBuffer & istr, const char delimiter) const
+void DataTypeNullable::deserializeTextCSV(IColumn & column, ReadBuffer & istr, const FormatSettings & settings) const
 {
     safeDeserialize(column,
         [&istr] { return checkStringByFirstCharacterAndAssertTheRest("\\N", istr); },
-        [this, delimiter, &istr] (IColumn & nested) { nested_data_type->deserializeTextCSV(nested, istr, delimiter); } );
+        [this, &settings, &istr] (IColumn & nested) { nested_data_type->deserializeTextCSV(nested, istr, settings); } );
 }
 
-void DataTypeNullable::serializeText(const IColumn & column, size_t row_num, WriteBuffer & ostr) const
+void DataTypeNullable::serializeText(const IColumn & column, size_t row_num, WriteBuffer & ostr, const FormatSettings & settings) const
 {
     const ColumnNullable & col = static_cast<const ColumnNullable &>(column);
 
@@ -251,10 +251,10 @@ void DataTypeNullable::serializeText(const IColumn & column, size_t row_num, Wri
     if (col.isNullAt(row_num))
         writeCString("ᴺᵁᴸᴸ", ostr);
     else
-        nested_data_type->serializeText(col.getNestedColumn(), row_num, ostr);
+        nested_data_type->serializeText(col.getNestedColumn(), row_num, ostr, settings);
 }
 
-void DataTypeNullable::serializeTextJSON(const IColumn & column, size_t row_num, WriteBuffer & ostr, const FormatSettingsJSON & settings) const
+void DataTypeNullable::serializeTextJSON(const IColumn & column, size_t row_num, WriteBuffer & ostr, const FormatSettings & settings) const
 {
     const ColumnNullable & col = static_cast<const ColumnNullable &>(column);
 
@@ -264,21 +264,21 @@ void DataTypeNullable::serializeTextJSON(const IColumn & column, size_t row_num,
         nested_data_type->serializeTextJSON(col.getNestedColumn(), row_num, ostr, settings);
 }
 
-void DataTypeNullable::deserializeTextJSON(IColumn & column, ReadBuffer & istr) const
+void DataTypeNullable::deserializeTextJSON(IColumn & column, ReadBuffer & istr, const FormatSettings & settings) const
 {
     safeDeserialize(column,
         [&istr] { return checkStringByFirstCharacterAndAssertTheRest("null", istr); },
-        [this, &istr] (IColumn & nested) { nested_data_type->deserializeTextJSON(nested, istr); } );
+        [this, &istr, &settings] (IColumn & nested) { nested_data_type->deserializeTextJSON(nested, istr, settings); } );
 }
 
-void DataTypeNullable::serializeTextXML(const IColumn & column, size_t row_num, WriteBuffer & ostr) const
+void DataTypeNullable::serializeTextXML(const IColumn & column, size_t row_num, WriteBuffer & ostr, const FormatSettings & settings) const
 {
     const ColumnNullable & col = static_cast<const ColumnNullable &>(column);
 
     if (col.isNullAt(row_num))
         writeCString("\\N", ostr);
     else
-        nested_data_type->serializeTextXML(col.getNestedColumn(), row_num, ostr);
+        nested_data_type->serializeTextXML(col.getNestedColumn(), row_num, ostr, settings);
 }
 
 MutableColumnPtr DataTypeNullable::createColumn() const

dbms/src/DataTypes/DataTypeNullable.h

@@ -39,12 +39,12 @@ public:
     void deserializeBinary(Field & field, ReadBuffer & istr) const override { nested_data_type->deserializeBinary(field, istr); }
     void serializeBinary(const IColumn & column, size_t row_num, WriteBuffer & ostr) const override;
     void deserializeBinary(IColumn & column, ReadBuffer & istr) const override;
-    void serializeTextEscaped(const IColumn & column, size_t row_num, WriteBuffer & ostr) const override;
-    void deserializeTextEscaped(IColumn & column, ReadBuffer & istr) const override;
-    void serializeTextQuoted(const IColumn & column, size_t row_num, WriteBuffer & ostr) const override;
-    void deserializeTextQuoted(IColumn & column, ReadBuffer & istr) const override;
+    void serializeTextEscaped(const IColumn & column, size_t row_num, WriteBuffer & ostr, const FormatSettings &) const override;
+    void deserializeTextEscaped(IColumn & column, ReadBuffer & istr, const FormatSettings &) const override;
+    void serializeTextQuoted(const IColumn & column, size_t row_num, WriteBuffer & ostr, const FormatSettings &) const override;
+    void deserializeTextQuoted(IColumn & column, ReadBuffer & istr, const FormatSettings &) const override;
 
-    void serializeTextCSV(const IColumn & column, size_t row_num, WriteBuffer & ostr) const override;
+    void serializeTextCSV(const IColumn & column, size_t row_num, WriteBuffer & ostr, const FormatSettings &) const override;
 
     /** It is questionable, how NULL values could be represented in CSV. There are three variants:
       * 1. \N
@@ -53,12 +53,12 @@ public:
       * Now we support only first.
       * In CSV, non-NULL string value, starting with \N characters, must be placed in quotes, to avoid ambiguity.
       */
-    void deserializeTextCSV(IColumn & column, ReadBuffer & istr, const char delimiter) const override;
+    void deserializeTextCSV(IColumn & column, ReadBuffer & istr, const FormatSettings & settings) const override;
 
-    void serializeTextJSON(const IColumn & column, size_t row_num, WriteBuffer & ostr, const FormatSettingsJSON &) const override;
-    void deserializeTextJSON(IColumn & column, ReadBuffer & istr) const override;
-    void serializeText(const IColumn & column, size_t row_num, WriteBuffer & ostr) const override;
-    void serializeTextXML(const IColumn & column, size_t row_num, WriteBuffer & ostr) const override;
+    void serializeTextJSON(const IColumn & column, size_t row_num, WriteBuffer & ostr, const FormatSettings &) const override;
+    void deserializeTextJSON(IColumn & column, ReadBuffer & istr, const FormatSettings &) const override;
+    void serializeText(const IColumn & column, size_t row_num, WriteBuffer & ostr, const FormatSettings &) const override;
+    void serializeTextXML(const IColumn & column, size_t row_num, WriteBuffer & ostr, const FormatSettings &) const override;
 
     MutableColumnPtr createColumn() const override;
 

dbms/src/DataTypes/DataTypeNumberBase.cpp

@@ -6,22 +6,22 @@
 #include <IO/WriteHelpers.h>
 #include <Common/NaNUtils.h>
 #include <Common/typeid_cast.h>
-#include <DataTypes/FormatSettingsJSON.h>
+#include <DataTypes/FormatSettings.h>
 
 
 namespace DB
 {
 
 template <typename T>
-void DataTypeNumberBase<T>::serializeText(const IColumn & column, size_t row_num, WriteBuffer & ostr) const
+void DataTypeNumberBase<T>::serializeText(const IColumn & column, size_t row_num, WriteBuffer & ostr, const FormatSettings &) const
 {
     writeText(static_cast<const ColumnVector<T> &>(column).getData()[row_num], ostr);
 }
 
 template <typename T>
-void DataTypeNumberBase<T>::serializeTextEscaped(const IColumn & column, size_t row_num, WriteBuffer & ostr) const
+void DataTypeNumberBase<T>::serializeTextEscaped(const IColumn & column, size_t row_num, WriteBuffer & ostr, const FormatSettings & settings) const
 {
-    serializeText(column, row_num, ostr);
+    serializeText(column, row_num, ostr, settings);
 }
 
 
@@ -40,19 +40,19 @@ static void deserializeText(IColumn & column, ReadBuffer & istr)
 
 
 template <typename T>
-void DataTypeNumberBase<T>::deserializeTextEscaped(IColumn & column, ReadBuffer & istr) const
+void DataTypeNumberBase<T>::deserializeTextEscaped(IColumn & column, ReadBuffer & istr, const FormatSettings &) const
 {
     deserializeText<T>(column, istr);
 }
 
 template <typename T>
-void DataTypeNumberBase<T>::serializeTextQuoted(const IColumn & column, size_t row_num, WriteBuffer & ostr) const
+void DataTypeNumberBase<T>::serializeTextQuoted(const IColumn & column, size_t row_num, WriteBuffer & ostr, const FormatSettings & settings) const
 {
-    serializeText(column, row_num, ostr);
+    serializeText(column, row_num, ostr, settings);
 }
 
 template <typename T>
-void DataTypeNumberBase<T>::deserializeTextQuoted(IColumn & column, ReadBuffer & istr) const
+void DataTypeNumberBase<T>::deserializeTextQuoted(IColumn & column, ReadBuffer & istr, const FormatSettings &) const
 {
     deserializeText<T>(column, istr);
 }
@@ -87,20 +87,20 @@ static inline void writeDenormalNumber(T x, WriteBuffer & ostr)
 
 
 template <typename T>
-void DataTypeNumberBase<T>::serializeTextJSON(const IColumn & column, size_t row_num, WriteBuffer & ostr, const FormatSettingsJSON & settings) const
+void DataTypeNumberBase<T>::serializeTextJSON(const IColumn & column, size_t row_num, WriteBuffer & ostr, const FormatSettings & settings) const
 {
     auto x = static_cast<const ColumnVector<T> &>(column).getData()[row_num];
     bool is_finite = isFinite(x);
 
-    const bool need_quote = (std::is_integral_v<T> && (sizeof(T) == 8) && settings.force_quoting_64bit_integers)
-        || (settings.output_format_json_quote_denormals && !is_finite);
+    const bool need_quote = (std::is_integral_v<T> && (sizeof(T) == 8) && settings.json.quote_64bit_integers)
+        || (settings.json.quote_denormals && !is_finite);
 
     if (need_quote)
         writeChar('"', ostr);
 
     if (is_finite)
         writeText(x, ostr);
-    else if (!settings.output_format_json_quote_denormals)
+    else if (!settings.json.quote_denormals)
         writeCString("null", ostr);
     else
         writeDenormalNumber(x, ostr);
@@ -110,7 +110,7 @@ void DataTypeNumberBase<T>::serializeTextJSON(const IColumn & column, size_t row
 }
 
 template <typename T>
-void DataTypeNumberBase<T>::deserializeTextJSON(IColumn & column, ReadBuffer & istr) const
+void DataTypeNumberBase<T>::deserializeTextJSON(IColumn & column, ReadBuffer & istr, const FormatSettings &) const
 {
     bool has_quote = false;
     if (!istr.eof() && *istr.position() == '"')        /// We understand the number both in quotes and without.
@@ -161,13 +161,13 @@ void DataTypeNumberBase<T>::deserializeTextJSON(IColumn & column, ReadBuffer & i
 }
 
 template <typename T>
-void DataTypeNumberBase<T>::serializeTextCSV(const IColumn & column, size_t row_num, WriteBuffer & ostr) const
+void DataTypeNumberBase<T>::serializeTextCSV(const IColumn & column, size_t row_num, WriteBuffer & ostr, const FormatSettings & settings) const
 {
-    serializeText(column, row_num, ostr);
+    serializeText(column, row_num, ostr, settings);
 }
 
 template <typename T>
-void DataTypeNumberBase<T>::deserializeTextCSV(IColumn & column, ReadBuffer & istr, const char /*delimiter*/) const
+void DataTypeNumberBase<T>::deserializeTextCSV(IColumn & column, ReadBuffer & istr, const FormatSettings &) const
 {
     FieldType x;
     readCSV(x, istr);

dbms/src/DataTypes/DataTypeNumberBase.h

@@ -17,15 +17,15 @@ public:
 
     const char * getFamilyName() const override { return TypeName<T>::get(); }
 
-    void serializeText(const IColumn & column, size_t row_num, WriteBuffer & ostr) const override;
-    void serializeTextEscaped(const IColumn & column, size_t row_num, WriteBuffer & ostr) const override;
-    void deserializeTextEscaped(IColumn & column, ReadBuffer & istr) const override;
-    void serializeTextQuoted(const IColumn & column, size_t row_num, WriteBuffer & ostr) const override;
-    void deserializeTextQuoted(IColumn & column, ReadBuffer & istr) const override;
-    void serializeTextJSON(const IColumn & column, size_t row_num, WriteBuffer & ostr, const FormatSettingsJSON & settings) const override;
-    void deserializeTextJSON(IColumn & column, ReadBuffer & istr) const override;
-    void serializeTextCSV(const IColumn & column, size_t row_num, WriteBuffer & ostr) const override;
-    void deserializeTextCSV(IColumn & column, ReadBuffer & istr, const char delimiter) const override;
+    void serializeText(const IColumn & column, size_t row_num, WriteBuffer & ostr, const FormatSettings &) const override;
+    void serializeTextEscaped(const IColumn & column, size_t row_num, WriteBuffer & ostr, const FormatSettings &) const override;
+    void deserializeTextEscaped(IColumn & column, ReadBuffer & istr, const FormatSettings &) const override;
+    void serializeTextQuoted(const IColumn & column, size_t row_num, WriteBuffer & ostr, const FormatSettings &) const override;
+    void deserializeTextQuoted(IColumn & column, ReadBuffer & istr, const FormatSettings &) const override;
+    void serializeTextJSON(const IColumn & column, size_t row_num, WriteBuffer & ostr, const FormatSettings & settings) const override;
+    void deserializeTextJSON(IColumn & column, ReadBuffer & istr, const FormatSettings &) const override;
+    void serializeTextCSV(const IColumn & column, size_t row_num, WriteBuffer & ostr, const FormatSettings &) const override;
+    void deserializeTextCSV(IColumn & column, ReadBuffer & istr, const FormatSettings & settings) const override;
     Field getDefault() const override;
 
     /** Format is platform-dependent. */

dbms/src/DataTypes/DataTypeString.cpp

@@ -6,6 +6,7 @@
 
 #include <Common/typeid_cast.h>
 
+#include <DataTypes/FormatSettings.h>
 #include <DataTypes/DataTypeString.h>
 #include <DataTypes/DataTypeFactory.h>
 
@@ -206,13 +207,13 @@ void DataTypeString::deserializeBinaryBulk(IColumn & column, ReadBuffer & istr,
 }
 
 
-void DataTypeString::serializeText(const IColumn & column, size_t row_num, WriteBuffer & ostr) const
+void DataTypeString::serializeText(const IColumn & column, size_t row_num, WriteBuffer & ostr, const FormatSettings &) const
 {
     writeString(static_cast<const ColumnString &>(column).getDataAt(row_num), ostr);
 }
 
 
-void DataTypeString::serializeTextEscaped(const IColumn & column, size_t row_num, WriteBuffer & ostr) const
+void DataTypeString::serializeTextEscaped(const IColumn & column, size_t row_num, WriteBuffer & ostr, const FormatSettings &) const
 {
     writeEscapedString(static_cast<const ColumnString &>(column).getDataAt(row_num), ostr);
 }
@@ -243,51 +244,51 @@ static inline void read(IColumn & column, Reader && reader)
 }
 
 
-void DataTypeString::deserializeTextEscaped(IColumn & column, ReadBuffer & istr) const
+void DataTypeString::deserializeTextEscaped(IColumn & column, ReadBuffer & istr, const FormatSettings &) const
 {
     read(column, [&](ColumnString::Chars_t & data) { readEscapedStringInto(data, istr); });
 }
 
 
-void DataTypeString::serializeTextQuoted(const IColumn & column, size_t row_num, WriteBuffer & ostr) const
+void DataTypeString::serializeTextQuoted(const IColumn & column, size_t row_num, WriteBuffer & ostr, const FormatSettings &) const
 {
     writeQuotedString(static_cast<const ColumnString &>(column).getDataAt(row_num), ostr);
 }
 
 
-void DataTypeString::deserializeTextQuoted(IColumn & column, ReadBuffer & istr) const
+void DataTypeString::deserializeTextQuoted(IColumn & column, ReadBuffer & istr, const FormatSettings &) const
 {
     read(column, [&](ColumnString::Chars_t & data) { readQuotedStringInto<true>(data, istr); });
 }
 
 
-void DataTypeString::serializeTextJSON(const IColumn & column, size_t row_num, WriteBuffer & ostr, const FormatSettingsJSON &) const
+void DataTypeString::serializeTextJSON(const IColumn & column, size_t row_num, WriteBuffer & ostr, const FormatSettings &) const
 {
     writeJSONString(static_cast<const ColumnString &>(column).getDataAt(row_num), ostr);
 }
 
 
-void DataTypeString::deserializeTextJSON(IColumn & column, ReadBuffer & istr) const
+void DataTypeString::deserializeTextJSON(IColumn & column, ReadBuffer & istr, const FormatSettings &) const
 {
     read(column, [&](ColumnString::Chars_t & data) { readJSONStringInto(data, istr); });
 }
 
 
-void DataTypeString::serializeTextXML(const IColumn & column, size_t row_num, WriteBuffer & ostr) const
+void DataTypeString::serializeTextXML(const IColumn & column, size_t row_num, WriteBuffer & ostr, const FormatSettings &) const
 {
     writeXMLString(static_cast<const ColumnString &>(column).getDataAt(row_num), ostr);
 }
 
 
-void DataTypeString::serializeTextCSV(const IColumn & column, size_t row_num, WriteBuffer & ostr) const
+void DataTypeString::serializeTextCSV(const IColumn & column, size_t row_num, WriteBuffer & ostr, const FormatSettings &) const
 {
     writeCSVString<>(static_cast<const ColumnString &>(column).getDataAt(row_num), ostr);
 }
 
 
-void DataTypeString::deserializeTextCSV(IColumn & column, ReadBuffer & istr, const char delimiter) const
+void DataTypeString::deserializeTextCSV(IColumn & column, ReadBuffer & istr, const FormatSettings & settings) const
 {
-    read(column, [&](ColumnString::Chars_t & data) { readCSVStringInto(data, istr, delimiter); });
+    read(column, [&](ColumnString::Chars_t & data) { readCSVStringInto(data, istr, settings.csv.delimiter); });
 }
 
 

dbms/src/DataTypes/DataTypeString.h

@@ -27,21 +27,21 @@ public:
     void serializeBinaryBulk(const IColumn & column, WriteBuffer & ostr, size_t offset, size_t limit) const override;
     void deserializeBinaryBulk(IColumn & column, ReadBuffer & istr, size_t limit, double avg_value_size_hint) const override;
 
-    void serializeText(const IColumn & column, size_t row_num, WriteBuffer & ostr) const override;
+    void serializeText(const IColumn & column, size_t row_num, WriteBuffer & ostr, const FormatSettings &) const override;
 
-    void serializeTextEscaped(const IColumn & column, size_t row_num, WriteBuffer & ostr) const override;
-    void deserializeTextEscaped(IColumn & column, ReadBuffer & istr) const override;
+    void serializeTextEscaped(const IColumn & column, size_t row_num, WriteBuffer & ostr, const FormatSettings &) const override;
+    void deserializeTextEscaped(IColumn & column, ReadBuffer & istr, const FormatSettings &) const override;
 
-    void serializeTextQuoted(const IColumn & column, size_t row_num, WriteBuffer & ostr) const override;
-    void deserializeTextQuoted(IColumn & column, ReadBuffer & istr) const override;
+    void serializeTextQuoted(const IColumn & column, size_t row_num, WriteBuffer & ostr, const FormatSettings &) const override;
+    void deserializeTextQuoted(IColumn & column, ReadBuffer & istr, const FormatSettings &) const override;
 
-    void serializeTextJSON(const IColumn & column, size_t row_num, WriteBuffer & ostr, const FormatSettingsJSON &) const override;
-    void deserializeTextJSON(IColumn & column, ReadBuffer & istr) const override;
+    void serializeTextJSON(const IColumn & column, size_t row_num, WriteBuffer & ostr, const FormatSettings &) const override;
+    void deserializeTextJSON(IColumn & column, ReadBuffer & istr, const FormatSettings &) const override;
 
-    void serializeTextXML(const IColumn & column, size_t row_num, WriteBuffer & ostr) const override;
+    void serializeTextXML(const IColumn & column, size_t row_num, WriteBuffer & ostr, const FormatSettings &) const override;
 
-    void serializeTextCSV(const IColumn & column, size_t row_num, WriteBuffer & ostr) const override;
-    void deserializeTextCSV(IColumn & column, ReadBuffer & istr, const char delimiter) const override;
+    void serializeTextCSV(const IColumn & column, size_t row_num, WriteBuffer & ostr, const FormatSettings &) const override;
+    void deserializeTextCSV(IColumn & column, ReadBuffer & istr, const FormatSettings &) const override;
 
     MutableColumnPtr createColumn() const override;
 

dbms/src/DataTypes/DataTypeTuple.cpp

@@ -1,5 +1,6 @@
 #include <Common/StringUtils/StringUtils.h>
 #include <Columns/ColumnTuple.h>
+#include <DataTypes/FormatSettings.h>
 #include <DataTypes/DataTypeTuple.h>
 #include <DataTypes/DataTypeArray.h>
 #include <DataTypes/DataTypeFactory.h>
@@ -152,19 +153,19 @@ void DataTypeTuple::deserializeBinary(IColumn & column, ReadBuffer & istr) const
     });
 }
 
-void DataTypeTuple::serializeText(const IColumn & column, size_t row_num, WriteBuffer & ostr) const
+void DataTypeTuple::serializeText(const IColumn & column, size_t row_num, WriteBuffer & ostr, const FormatSettings & settings) const
 {
     writeChar('(', ostr);
     for (const auto i : ext::range(0, ext::size(elems)))
     {
         if (i != 0)
             writeChar(',', ostr);
-        elems[i]->serializeTextQuoted(extractElementColumn(column, i), row_num, ostr);
+        elems[i]->serializeTextQuoted(extractElementColumn(column, i), row_num, ostr, settings);
     }
     writeChar(')', ostr);
 }
 
-void DataTypeTuple::deserializeText(IColumn & column, ReadBuffer & istr) const
+void DataTypeTuple::deserializeText(IColumn & column, ReadBuffer & istr, const FormatSettings & settings) const
 {
     const size_t size = elems.size();
     assertChar('(', istr);
@@ -179,7 +180,7 @@ void DataTypeTuple::deserializeText(IColumn & column, ReadBuffer & istr) const
                 assertChar(',', istr);
                 skipWhitespaceIfAny(istr);
             }
-            elems[i]->deserializeTextQuoted(extractElementColumn(column, i), istr);
+            elems[i]->deserializeTextQuoted(extractElementColumn(column, i), istr, settings);
         }
     });
 
@@ -187,27 +188,27 @@ void DataTypeTuple::deserializeText(IColumn & column, ReadBuffer & istr) const
     assertChar(')', istr);
 }
 
-void DataTypeTuple::serializeTextEscaped(const IColumn & column, size_t row_num, WriteBuffer & ostr) const
+void DataTypeTuple::serializeTextEscaped(const IColumn & column, size_t row_num, WriteBuffer & ostr, const FormatSettings & settings) const
 {
-    serializeText(column, row_num, ostr);
+    serializeText(column, row_num, ostr, settings);
 }
 
-void DataTypeTuple::deserializeTextEscaped(IColumn & column, ReadBuffer & istr) const
+void DataTypeTuple::deserializeTextEscaped(IColumn & column, ReadBuffer & istr, const FormatSettings & settings) const
 {
-    deserializeText(column, istr);
+    deserializeText(column, istr, settings);
 }
 
-void DataTypeTuple::serializeTextQuoted(const IColumn & column, size_t row_num, WriteBuffer & ostr) const
+void DataTypeTuple::serializeTextQuoted(const IColumn & column, size_t row_num, WriteBuffer & ostr, const FormatSettings & settings) const
 {
-    serializeText(column, row_num, ostr);
+    serializeText(column, row_num, ostr, settings);
 }
 
-void DataTypeTuple::deserializeTextQuoted(IColumn & column, ReadBuffer & istr) const
+void DataTypeTuple::deserializeTextQuoted(IColumn & column, ReadBuffer & istr, const FormatSettings & settings) const
 {
-    deserializeText(column, istr);
+    deserializeText(column, istr, settings);
 }
 
-void DataTypeTuple::serializeTextJSON(const IColumn & column, size_t row_num, WriteBuffer & ostr, const FormatSettingsJSON & settings) const
+void DataTypeTuple::serializeTextJSON(const IColumn & column, size_t row_num, WriteBuffer & ostr, const FormatSettings & settings) const
 {
     writeChar('[', ostr);
     for (const auto i : ext::range(0, ext::size(elems)))
@@ -219,7 +220,7 @@ void DataTypeTuple::serializeTextJSON(const IColumn & column, size_t row_num, Wr
     writeChar(']', ostr);
 }
 
-void DataTypeTuple::deserializeTextJSON(IColumn & column, ReadBuffer & istr) const
+void DataTypeTuple::deserializeTextJSON(IColumn & column, ReadBuffer & istr, const FormatSettings & settings) const
 {
     const size_t size = elems.size();
     assertChar('[', istr);
@@ -234,7 +235,7 @@ void DataTypeTuple::deserializeTextJSON(IColumn & column, ReadBuffer & istr) con
                 assertChar(',', istr);
                 skipWhitespaceIfAny(istr);
             }
-            elems[i]->deserializeTextJSON(extractElementColumn(column, i), istr);
+            elems[i]->deserializeTextJSON(extractElementColumn(column, i), istr, settings);
         }
     });
 
@@ -242,29 +243,29 @@ void DataTypeTuple::deserializeTextJSON(IColumn & column, ReadBuffer & istr) con
     assertChar(']', istr);
 }
 
-void DataTypeTuple::serializeTextXML(const IColumn & column, size_t row_num, WriteBuffer & ostr) const
+void DataTypeTuple::serializeTextXML(const IColumn & column, size_t row_num, WriteBuffer & ostr, const FormatSettings & settings) const
 {
     writeCString("<tuple>", ostr);
     for (const auto i : ext::range(0, ext::size(elems)))
     {
         writeCString("<elem>", ostr);
-        elems[i]->serializeTextXML(extractElementColumn(column, i), row_num, ostr);
+        elems[i]->serializeTextXML(extractElementColumn(column, i), row_num, ostr, settings);
         writeCString("</elem>", ostr);
     }
     writeCString("</tuple>", ostr);
 }
 
-void DataTypeTuple::serializeTextCSV(const IColumn & column, size_t row_num, WriteBuffer & ostr) const
+void DataTypeTuple::serializeTextCSV(const IColumn & column, size_t row_num, WriteBuffer & ostr, const FormatSettings & settings) const
 {
     for (const auto i : ext::range(0, ext::size(elems)))
     {
         if (i != 0)
             writeChar(',', ostr);
-        elems[i]->serializeTextCSV(extractElementColumn(column, i), row_num, ostr);
+        elems[i]->serializeTextCSV(extractElementColumn(column, i), row_num, ostr, settings);
     }
 }
 
-void DataTypeTuple::deserializeTextCSV(IColumn & column, ReadBuffer & istr, const char delimiter) const
+void DataTypeTuple::deserializeTextCSV(IColumn & column, ReadBuffer & istr, const FormatSettings & settings) const
 {
     addElementSafe(elems, column, [&]
     {
@@ -274,10 +275,10 @@ void DataTypeTuple::deserializeTextCSV(IColumn & column, ReadBuffer & istr, cons
             if (i != 0)
             {
                 skipWhitespaceIfAny(istr);
-                assertChar(delimiter, istr);
+                assertChar(settings.csv.delimiter, istr);
                 skipWhitespaceIfAny(istr);
             }
-            elems[i]->deserializeTextCSV(extractElementColumn(column, i), istr, delimiter);
+            elems[i]->deserializeTextCSV(extractElementColumn(column, i), istr, settings);
         }
     });
 }

dbms/src/DataTypes/DataTypeTuple.h

@@ -37,19 +37,19 @@ public:
     void deserializeBinary(Field & field, ReadBuffer & istr) const override;
     void serializeBinary(const IColumn & column, size_t row_num, WriteBuffer & ostr) const override;
     void deserializeBinary(IColumn & column, ReadBuffer & istr) const override;
-    void serializeText(const IColumn & column, size_t row_num, WriteBuffer & ostr) const override;
-    void deserializeText(IColumn & column, ReadBuffer & istr) const;
-    void serializeTextEscaped(const IColumn & column, size_t row_num, WriteBuffer & ostr) const override;
-    void deserializeTextEscaped(IColumn & column, ReadBuffer & istr) const override;
-    void serializeTextQuoted(const IColumn & column, size_t row_num, WriteBuffer & ostr) const override;
-    void deserializeTextQuoted(IColumn & column, ReadBuffer & istr) const override;
-    void serializeTextJSON(const IColumn & column, size_t row_num, WriteBuffer & ostr, const FormatSettingsJSON & settings) const override;
-    void deserializeTextJSON(IColumn & column, ReadBuffer & istr) const override;
-    void serializeTextXML(const IColumn & column, size_t row_num, WriteBuffer & ostr) const override;
+    void serializeText(const IColumn & column, size_t row_num, WriteBuffer & ostr, const FormatSettings &) const override;
+    void deserializeText(IColumn & column, ReadBuffer & istr, const FormatSettings &) const;
+    void serializeTextEscaped(const IColumn & column, size_t row_num, WriteBuffer & ostr, const FormatSettings &) const override;
+    void deserializeTextEscaped(IColumn & column, ReadBuffer & istr, const FormatSettings &) const override;
+    void serializeTextQuoted(const IColumn & column, size_t row_num, WriteBuffer & ostr, const FormatSettings &) const override;
+    void deserializeTextQuoted(IColumn & column, ReadBuffer & istr, const FormatSettings &) const override;
+    void serializeTextJSON(const IColumn & column, size_t row_num, WriteBuffer & ostr, const FormatSettings &) const override;
+    void deserializeTextJSON(IColumn & column, ReadBuffer & istr, const FormatSettings &) const override;
+    void serializeTextXML(const IColumn & column, size_t row_num, WriteBuffer & ostr, const FormatSettings &) const override;
 
     /// Tuples in CSV format will be serialized as separate columns (that is, losing their nesting in the tuple).
-    void serializeTextCSV(const IColumn & column, size_t row_num, WriteBuffer & ostr) const override;
-    void deserializeTextCSV(IColumn & column, ReadBuffer & istr, const char delimiter) const override;
+    void serializeTextCSV(const IColumn & column, size_t row_num, WriteBuffer & ostr, const FormatSettings &) const override;
+    void deserializeTextCSV(IColumn & column, ReadBuffer & istr, const FormatSettings &) const override;
 
     /** Each sub-column in a tuple is serialized in separate stream.
       */

dbms/src/DataTypes/DataTypeUUID.cpp

@@ -5,7 +5,7 @@
 namespace DB
 {
 
-void DataTypeUUID::serializeText(const IColumn & column, size_t row_num, WriteBuffer & ostr) const
+void DataTypeUUID::serializeText(const IColumn & column, size_t row_num, WriteBuffer & ostr, const FormatSettings &) const
 {
     writeText(UUID(static_cast<const ColumnUInt128 &>(column).getData()[row_num]), ostr);
 }
@@ -17,24 +17,24 @@ static void deserializeText(IColumn & column, ReadBuffer & istr)
     static_cast<ColumnUInt128 &>(column).getData().push_back(x);
 }
 
-void DataTypeUUID::serializeTextEscaped(const IColumn & column, size_t row_num, WriteBuffer & ostr) const
+void DataTypeUUID::serializeTextEscaped(const IColumn & column, size_t row_num, WriteBuffer & ostr, const FormatSettings & settings) const
 {
-    serializeText(column, row_num, ostr);
+    serializeText(column, row_num, ostr, settings);
 }
 
-void DataTypeUUID::deserializeTextEscaped(IColumn & column, ReadBuffer & istr) const
+void DataTypeUUID::deserializeTextEscaped(IColumn & column, ReadBuffer & istr, const FormatSettings &) const
 {
     deserializeText(column, istr);
 }
 
-void DataTypeUUID::serializeTextQuoted(const IColumn & column, size_t row_num, WriteBuffer & ostr) const
+void DataTypeUUID::serializeTextQuoted(const IColumn & column, size_t row_num, WriteBuffer & ostr, const FormatSettings & settings) const
 {
     writeChar('\'', ostr);
-    serializeText(column, row_num, ostr);
+    serializeText(column, row_num, ostr, settings);
     writeChar('\'', ostr);
 }
 
-void DataTypeUUID::deserializeTextQuoted(IColumn & column, ReadBuffer & istr) const
+void DataTypeUUID::deserializeTextQuoted(IColumn & column, ReadBuffer & istr, const FormatSettings &) const
 {
     UUID x;
     assertChar('\'', istr);
@@ -43,14 +43,14 @@ void DataTypeUUID::deserializeTextQuoted(IColumn & column, ReadBuffer & istr) co
     static_cast<ColumnUInt128 &>(column).getData().push_back(x);    /// It's important to do this at the end - for exception safety.
 }
 
-void DataTypeUUID::serializeTextJSON(const IColumn & column, size_t row_num, WriteBuffer & ostr, const FormatSettingsJSON &) const
+void DataTypeUUID::serializeTextJSON(const IColumn & column, size_t row_num, WriteBuffer & ostr, const FormatSettings & settings) const
 {
     writeChar('"', ostr);
-    serializeText(column, row_num, ostr);
+    serializeText(column, row_num, ostr, settings);
     writeChar('"', ostr);
 }
 
-void DataTypeUUID::deserializeTextJSON(IColumn & column, ReadBuffer & istr) const
+void DataTypeUUID::deserializeTextJSON(IColumn & column, ReadBuffer & istr, const FormatSettings &) const
 {
     UUID x;
     assertChar('"', istr);
@@ -59,14 +59,14 @@ void DataTypeUUID::deserializeTextJSON(IColumn & column, ReadBuffer & istr) cons
     static_cast<ColumnUInt128 &>(column).getData().push_back(x);
 }
 
-void DataTypeUUID::serializeTextCSV(const IColumn & column, size_t row_num, WriteBuffer & ostr) const
+void DataTypeUUID::serializeTextCSV(const IColumn & column, size_t row_num, WriteBuffer & ostr, const FormatSettings & settings) const
 {
     writeChar('"', ostr);
-    serializeText(column, row_num, ostr);
+    serializeText(column, row_num, ostr, settings);
     writeChar('"', ostr);
 }
 
-void DataTypeUUID::deserializeTextCSV(IColumn & column, ReadBuffer & istr, const char /*delimiter*/) const
+void DataTypeUUID::deserializeTextCSV(IColumn & column, ReadBuffer & istr, const FormatSettings &) const
 {
     UUID value;
     readCSV(value, istr);

dbms/src/DataTypes/DataTypeUUID.h

@@ -17,15 +17,15 @@ public:
 
     bool equals(const IDataType & rhs) const override;
 
-    void serializeText(const IColumn & column, size_t row_num, WriteBuffer & ostr) const override;
-    void serializeTextEscaped(const IColumn & column, size_t row_num, WriteBuffer & ostr) const override;
-    void deserializeTextEscaped(IColumn & column, ReadBuffer & istr) const override;
-    void serializeTextQuoted(const IColumn & column, size_t row_num, WriteBuffer & ostr) const override;
-    void deserializeTextQuoted(IColumn & column, ReadBuffer & istr) const override;
-    void serializeTextJSON(const IColumn & column, size_t row_num, WriteBuffer & ostr, const FormatSettingsJSON &) const override;
-    void deserializeTextJSON(IColumn & column, ReadBuffer & istr) const override;
-    void serializeTextCSV(const IColumn & column, size_t row_num, WriteBuffer & ostr) const override;
-    void deserializeTextCSV(IColumn & column, ReadBuffer & istr, const char delimiter) const override;
+    void serializeText(const IColumn & column, size_t row_num, WriteBuffer & ostr, const FormatSettings &) const override;
+    void serializeTextEscaped(const IColumn & column, size_t row_num, WriteBuffer & ostr, const FormatSettings &) const override;
+    void deserializeTextEscaped(IColumn & column, ReadBuffer & istr, const FormatSettings &) const override;
+    void serializeTextQuoted(const IColumn & column, size_t row_num, WriteBuffer & ostr, const FormatSettings &) const override;
+    void deserializeTextQuoted(IColumn & column, ReadBuffer & istr, const FormatSettings &) const override;
+    void serializeTextJSON(const IColumn & column, size_t row_num, WriteBuffer & ostr, const FormatSettings &) const override;
+    void deserializeTextJSON(IColumn & column, ReadBuffer & istr, const FormatSettings &) const override;
+    void serializeTextCSV(const IColumn & column, size_t row_num, WriteBuffer & ostr, const FormatSettings &) const override;
+    void deserializeTextCSV(IColumn & column, ReadBuffer & istr, const FormatSettings & settings) const override;
 
     bool canBeUsedInBitOperations() const override { return true; }
     bool canBeInsideNullable() const override { return true; }

dbms/src/DataTypes/FormatSettings.h

@@ -0,0 +1,59 @@
+#pragma once
+
+#include <Core/Types.h>
+
+
+namespace DB
+{
+
+/** Various tweaks for input/output formats.
+  * Text serialization/deserialization of data types also depend on some of these settings.
+  * NOTE Parameters for unrelated formats and unrelated data types
+  *  are collected in this struct - it prevents modularity, but they are difficult to separate.
+  */
+struct FormatSettings
+{
+    struct JSON
+    {
+        bool quote_64bit_integers = true;
+        bool quote_denormals = true;
+    };
+
+    JSON json;
+
+    struct CSV
+    {
+        char delimiter = ',';
+    };
+
+    CSV csv;
+
+    struct Pretty
+    {
+        UInt64 max_rows = 10000;
+        bool color = true;
+    };
+
+    Pretty pretty;
+
+    struct Values
+    {
+        bool interpret_expressions = true;
+    };
+
+    Values values;
+
+    bool skip_unknown_fields = false;
+    bool write_statistics = true;
+
+    enum class DateTimeInputFormat
+    {
+        Basic,      /// Default format for fast parsing: YYYY-MM-DD hh:mm:ss (ISO-8601 without fractional part and timezone) or NNNNNNNNNN unix timestamp.
+        BestEffort  /// Use sophisticated rules to parse whatever possible.
+    };
+
+    DateTimeInputFormat date_time_input_format = DateTimeInputFormat::Basic;
+};
+
+}
+

dbms/src/DataTypes/IDataType.h

@@ -13,7 +13,7 @@ class ReadBuffer;
 class WriteBuffer;
 
 class IDataType;
-struct FormatSettingsJSON;
+struct FormatSettings;
 
 class IColumn;
 using ColumnPtr = COWPtr<IColumn>::Ptr;
@@ -163,41 +163,41 @@ public:
 
     /** Text serialization with escaping but without quoting.
       */
-    virtual void serializeTextEscaped(const IColumn & column, size_t row_num, WriteBuffer & ostr) const = 0;
+    virtual void serializeTextEscaped(const IColumn & column, size_t row_num, WriteBuffer & ostr, const FormatSettings &) const = 0;
 
-    virtual void deserializeTextEscaped(IColumn & column, ReadBuffer & istr) const = 0;
+    virtual void deserializeTextEscaped(IColumn & column, ReadBuffer & istr, const FormatSettings &) const = 0;
 
     /** Text serialization as a literal that may be inserted into a query.
       */
-    virtual void serializeTextQuoted(const IColumn & column, size_t row_num, WriteBuffer & ostr) const = 0;
+    virtual void serializeTextQuoted(const IColumn & column, size_t row_num, WriteBuffer & ostr, const FormatSettings &) const = 0;
 
-    virtual void deserializeTextQuoted(IColumn & column, ReadBuffer & istr) const = 0;
+    virtual void deserializeTextQuoted(IColumn & column, ReadBuffer & istr, const FormatSettings &) const = 0;
 
     /** Text serialization for the CSV format.
       */
-    virtual void serializeTextCSV(const IColumn & column, size_t row_num, WriteBuffer & ostr) const = 0;
+    virtual void serializeTextCSV(const IColumn & column, size_t row_num, WriteBuffer & ostr, const FormatSettings &) const = 0;
 
     /** delimiter - the delimiter we expect when reading a string value that is not double-quoted
       * (the delimiter is not consumed).
       */
-    virtual void deserializeTextCSV(IColumn & column, ReadBuffer & istr, const char delimiter) const = 0;
+    virtual void deserializeTextCSV(IColumn & column, ReadBuffer & istr, const FormatSettings &) const = 0;
 
     /** Text serialization for displaying on a terminal or saving into a text file, and the like.
       * Without escaping or quoting.
       */
-    virtual void serializeText(const IColumn & column, size_t row_num, WriteBuffer & ostr) const = 0;
+    virtual void serializeText(const IColumn & column, size_t row_num, WriteBuffer & ostr, const FormatSettings &) const = 0;
 
     /** Text serialization intended for using in JSON format.
       * force_quoting_64bit_integers parameter forces to brace UInt64 and Int64 types into quotes.
       */
-    virtual void serializeTextJSON(const IColumn & column, size_t row_num, WriteBuffer & ostr, const FormatSettingsJSON & settings) const = 0;
-    virtual void deserializeTextJSON(IColumn & column, ReadBuffer & istr) const = 0;
+    virtual void serializeTextJSON(const IColumn & column, size_t row_num, WriteBuffer & ostr, const FormatSettings &) const = 0;
+    virtual void deserializeTextJSON(IColumn & column, ReadBuffer & istr, const FormatSettings &) const = 0;
 
     /** Text serialization for putting into the XML format.
       */
-    virtual void serializeTextXML(const IColumn & column, size_t row_num, WriteBuffer & ostr) const
+    virtual void serializeTextXML(const IColumn & column, size_t row_num, WriteBuffer & ostr, const FormatSettings & settings) const
     {
-        serializeText(column, row_num, ostr);
+        serializeText(column, row_num, ostr, settings);
     }
 
     /** Create empty column for corresponding type.

dbms/src/DataTypes/IDataTypeDummy.h

@@ -26,15 +26,15 @@ public:
     void deserializeBinary(IColumn &, ReadBuffer &) const override                          { throwNoSerialization(); }
     void serializeBinaryBulk(const IColumn &, WriteBuffer &, size_t, size_t) const override { throwNoSerialization(); }
     void deserializeBinaryBulk(IColumn &, ReadBuffer &, size_t, double) const override      { throwNoSerialization(); }
-    void serializeText(const IColumn &, size_t, WriteBuffer &) const override               { throwNoSerialization(); }
-    void serializeTextEscaped(const IColumn &, size_t, WriteBuffer &) const override        { throwNoSerialization(); }
-    void deserializeTextEscaped(IColumn &, ReadBuffer &) const override                     { throwNoSerialization(); }
-    void serializeTextQuoted(const IColumn &, size_t, WriteBuffer &) const override         { throwNoSerialization(); }
-    void deserializeTextQuoted(IColumn &, ReadBuffer &) const override                      { throwNoSerialization(); }
-    void serializeTextJSON(const IColumn &, size_t, WriteBuffer &, const FormatSettingsJSON &) const override { throwNoSerialization(); }
-    void deserializeTextJSON(IColumn &, ReadBuffer &) const override                        { throwNoSerialization(); }
-    void serializeTextCSV(const IColumn &, size_t, WriteBuffer &) const override            { throwNoSerialization(); }
-    void deserializeTextCSV(IColumn &, ReadBuffer &, const char) const override             { throwNoSerialization(); }
+    void serializeText(const IColumn &, size_t, WriteBuffer &, const FormatSettings &) const override        { throwNoSerialization(); }
+    void serializeTextEscaped(const IColumn &, size_t, WriteBuffer &, const FormatSettings &) const override { throwNoSerialization(); }
+    void deserializeTextEscaped(IColumn &, ReadBuffer &, const FormatSettings &) const override              { throwNoSerialization(); }
+    void serializeTextQuoted(const IColumn &, size_t, WriteBuffer &, const FormatSettings &) const override  { throwNoSerialization(); }
+    void deserializeTextQuoted(IColumn &, ReadBuffer &, const FormatSettings &) const override               { throwNoSerialization(); }
+    void serializeTextJSON(const IColumn &, size_t, WriteBuffer &, const FormatSettings &) const override    { throwNoSerialization(); }
+    void deserializeTextJSON(IColumn &, ReadBuffer &, const FormatSettings &) const override                 { throwNoSerialization(); }
+    void serializeTextCSV(const IColumn &, size_t, WriteBuffer &, const FormatSettings &) const override     { throwNoSerialization(); }
+    void deserializeTextCSV(IColumn &, ReadBuffer &, const FormatSettings &) const override                  { throwNoSerialization(); }
 
     MutableColumnPtr createColumn() const override
     {

dbms/src/Dictionaries/DictionaryStructure.cpp

@@ -1,4 +1,5 @@
 #include <Dictionaries/DictionaryStructure.h>
+#include <DataTypes/FormatSettings.h>
 #include <DataTypes/DataTypeFactory.h>
 #include <Columns/IColumn.h>
 #include <Common/StringUtils/StringUtils.h>
@@ -222,7 +223,7 @@ size_t DictionaryStructure::getKeySize() const
 }
 
 
-static void CheckAttributeKeys(const Poco::Util::AbstractConfiguration::Keys & keys)
+static void checkAttributeKeys(const Poco::Util::AbstractConfiguration::Keys & keys)
 {
     static const std::unordered_set<std::string> valid_keys =
         { "name", "type", "expression", "null_value", "hierarchical", "injective", "is_object_id" };
@@ -234,6 +235,7 @@ static void CheckAttributeKeys(const Poco::Util::AbstractConfiguration::Keys & k
     }
 }
 
+
 std::vector<DictionaryAttribute> DictionaryStructure::getAttributes(
     const Poco::Util::AbstractConfiguration & config, const std::string & config_prefix,
     const bool hierarchy_allowed, const bool allow_null_values)
@@ -244,6 +246,8 @@ std::vector<DictionaryAttribute> DictionaryStructure::getAttributes(
 
     std::vector<DictionaryAttribute> attributes;
 
+    const FormatSettings format_settings;
+
     for (const auto & key : keys)
     {
         if (!startsWith(key.data(), "attribute"))
@@ -253,7 +257,7 @@ std::vector<DictionaryAttribute> DictionaryStructure::getAttributes(
         Poco::Util::AbstractConfiguration::Keys attribute_keys;
         config.keys(config_prefix + '.' + key, attribute_keys);
 
-        CheckAttributeKeys(attribute_keys);
+        checkAttributeKeys(attribute_keys);
 
         const auto name = config.getString(prefix + "name");
         const auto type_string = config.getString(prefix + "type");
@@ -272,7 +276,7 @@ std::vector<DictionaryAttribute> DictionaryStructure::getAttributes(
             {
                 ReadBufferFromString null_value_buffer{null_value_string};
                 auto column_with_null_value = type->createColumn();
-                type->deserializeTextEscaped(*column_with_null_value, null_value_buffer);
+                type->deserializeTextEscaped(*column_with_null_value, null_value_buffer, format_settings);
                 null_value = (*column_with_null_value)[0];
             }
             catch (Exception & e)

dbms/src/Dictionaries/ExternalQueryBuilder.cpp

@@ -320,7 +320,7 @@ void ExternalQueryBuilder::composeKeyCondition(const Columns & key_columns, cons
         /// key_i=value_i
         writeString(key_description.name, out);
         writeString("=", out);
-        key_description.type->serializeTextQuoted(*key_columns[i], row, out);
+        key_description.type->serializeTextQuoted(*key_columns[i], row, out, format_settings);
     }
 
     writeString(")", out);
@@ -362,7 +362,7 @@ void ExternalQueryBuilder::composeKeyTuple(const Columns & key_columns, const si
             writeString(", ", out);
 
         first = false;
-        (*dict_struct.key)[i].type->serializeTextQuoted(*key_columns[i], row, out);
+        (*dict_struct.key)[i].type->serializeTextQuoted(*key_columns[i], row, out, format_settings);
     }
 
     writeString(")", out);

dbms/src/Dictionaries/ExternalQueryBuilder.h

@@ -1,6 +1,7 @@
 #pragma once
 
 #include <string>
+#include <DataTypes/FormatSettings.h>
 #include <Columns/IColumn.h>
 
 
@@ -66,6 +67,8 @@ struct ExternalQueryBuilder
 
 
 private:
+    const FormatSettings format_settings;
+
     /// Expression in form (x = c1 AND y = c2 ...)
     void composeKeyCondition(const Columns & key_columns, const size_t row, WriteBuffer & out) const;
 

dbms/src/Functions/FunctionsConversion.h

@@ -22,6 +22,7 @@
 #include <DataTypes/DataTypeNothing.h>
 #include <DataTypes/DataTypeUUID.h>
 #include <DataTypes/DataTypeInterval.h>
+#include <DataTypes/FormatSettings.h>
 #include <Columns/ColumnString.h>
 #include <Columns/ColumnFixedString.h>
 #include <Columns/ColumnConst.h>
@@ -285,9 +286,10 @@ struct ConvertImplGenericToString
 
         WriteBufferFromVector<ColumnString::Chars_t> write_buffer(data_to);
 
+        FormatSettings format_settings;
         for (size_t i = 0; i < size; ++i)
         {
-            type.serializeText(col_from, i, write_buffer);
+            type.serializeText(col_from, i, write_buffer, format_settings);
             writeChar(0, write_buffer);
             offsets_to[i] = write_buffer.count();
         }
@@ -525,11 +527,12 @@ struct ConvertImplGenericFromString
 
             size_t current_offset = 0;
 
+            FormatSettings format_settings;
             for (size_t i = 0; i < size; ++i)
             {
                 ReadBufferFromMemory read_buffer(&chars[current_offset], offsets[i] - current_offset - 1);
 
-                data_type_to.deserializeTextEscaped(column_to, read_buffer);
+                data_type_to.deserializeTextEscaped(column_to, read_buffer, format_settings);
 
                 if (!read_buffer.eof())
                     throwExceptionForIncompletelyParsedValue(read_buffer, block, result);

dbms/src/Functions/FunctionsMiscellaneous.cpp

@@ -24,6 +24,7 @@
 #include <DataTypes/DataTypesNumber.h>
 #include <DataTypes/DataTypeEnum.h>
 #include <DataTypes/NumberTraits.h>
+#include <DataTypes/FormatSettings.h>
 #include <Functions/FunctionFactory.h>
 #include <Interpreters/Cluster.h>
 #include <Interpreters/Context.h>
@@ -1684,11 +1685,12 @@ void FunctionVisibleWidth::executeImpl(Block & block, const ColumnNumbers & argu
     /// For simplicity reasons, function is implemented by serializing into temporary buffer.
 
     String tmp;
+    FormatSettings format_settings;
     for (size_t i = 0; i < size; ++i)
     {
         {
             WriteBufferFromString out(tmp);
-            src.type->serializeText(*src.column, i, out);
+            src.type->serializeText(*src.column, i, out, format_settings);
         }
 
         res_data[i] = UTF8::countCodePoints(reinterpret_cast<const UInt8 *>(tmp.data()), tmp.size());

dbms/src/IO/ReadHelpers.h

@@ -407,7 +407,7 @@ void readStringUntilEOF(String & s, ReadBuffer & buf);
   * - if string is in quotes, then it will be read until closing quote,
   *   but sequences of two consecutive quotes are parsed as single quote inside string;
   */
-void readCSVString(String & s, ReadBuffer & buf, const char delimiter = ',');
+void readCSVString(String & s, ReadBuffer & buf, const char delimiter);
 
 
 /// Read and append result to array of characters.
@@ -430,7 +430,7 @@ template <typename Vector>
 void readStringUntilEOFInto(Vector & s, ReadBuffer & buf);
 
 template <typename Vector>
-void readCSVStringInto(Vector & s, ReadBuffer & buf, const char delimiter = ',');
+void readCSVStringInto(Vector & s, ReadBuffer & buf, const char delimiter);
 
 /// ReturnType is either bool or void. If bool, the function will return false instead of throwing an exception.
 template <typename Vector, typename ReturnType = void>
@@ -684,22 +684,6 @@ inline void readCSVSimple(T & x, ReadBuffer & buf)
         assertChar(maybe_quote, buf);
 }
 
-inline void readDateTimeCSV(time_t & datetime, ReadBuffer & buf, const DateLUTImpl & date_lut)
-{
-    if (buf.eof())
-        throwReadAfterEOF();
-
-    char maybe_quote = *buf.position();
-
-    if (maybe_quote == '\'' || maybe_quote == '\"')
-        ++buf.position();
-
-    readDateTimeText(datetime, buf, date_lut);
-
-    if (maybe_quote == '\'' || maybe_quote == '\"')
-        assertChar(maybe_quote, buf);
-}
-
 template <typename T>
 inline std::enable_if_t<std::is_arithmetic_v<T>, void>
 readCSV(T & x, ReadBuffer & buf) { readCSVSimple(x, buf); }

dbms/src/Interpreters/Settings.h

@@ -155,6 +155,7 @@ struct Settings
     M(SettingBool, output_format_json_quote_denormals, false, "Enables '+nan', '-nan', '+inf', '-inf' outputs in JSON output format.") \
     \
     M(SettingUInt64, output_format_pretty_max_rows, 10000, "Rows limit for Pretty formats.") \
+    M(SettingBool, output_format_pretty_color, true, "Use ANSI escape sequences to paint colors in Pretty formats") \
     \
     M(SettingBool, use_client_time_zone, false, "Use client timezone for interpreting DateTime string values, instead of adopting server timezone.") \
     \
@@ -261,6 +262,8 @@ struct Settings
     M(SettingUInt64, max_network_bandwidth_for_all_users, 0, "The maximum speed of data exchange over the network in bytes per second for all concurrently running queries. Zero means unlimited.") \
     M(SettingChar, format_csv_delimiter, ',', "The character to be considered as a delimiter in CSV data. If setting with a string, a string has to have a length of 1.") \
     M(SettingUInt64, enable_conditional_computation, 0, "Enable conditional computations") \
+    \
+    M(SettingDateTimeInputFormat, date_time_input_format, FormatSettings::DateTimeInputFormat::Basic, "Method to read DateTime from text input formats. Possible values: 'basic' and 'best_effort'.") \
 
 #define DECLARE(TYPE, NAME, DEFAULT, DESCRIPTION) \
     TYPE NAME {DEFAULT};

dbms/src/Interpreters/SettingsCommon.cpp

@@ -23,6 +23,7 @@ namespace ErrorCodes
     extern const int UNKNOWN_DISTRIBUTED_PRODUCT_MODE;
     extern const int UNKNOWN_GLOBAL_SUBQUERIES_METHOD;
     extern const int SIZE_OF_FIXED_STRING_DOESNT_MATCH;
+    extern const int BAD_ARGUMENTS;
 }
 
 template <typename IntType>
@@ -580,4 +581,49 @@ void SettingChar::write(WriteBuffer & buf) const
     writeBinary(toString(), buf);
 }
 
+
+SettingDateTimeInputFormat::Value SettingDateTimeInputFormat::getValue(const String & s)
+{
+    if (s == "basic")  return Value::Basic;
+    if (s == "best_effort")  return Value::BestEffort;
+
+    throw Exception("Unknown DateTime input format: '" + s + "', must be one of 'basic', 'best_effort'", ErrorCodes::BAD_ARGUMENTS);
+}
+
+String SettingDateTimeInputFormat::toString() const
+{
+    const char * strings[] = {"basic", "best_effort"};
+    if (value < Value::Basic || value > Value::BestEffort)
+        throw Exception("Unknown DateTime input format", ErrorCodes::BAD_ARGUMENTS);
+    return strings[static_cast<size_t>(value)];
+}
+
+void SettingDateTimeInputFormat::set(Value x)
+{
+    value = x;
+    changed = true;
+}
+
+void SettingDateTimeInputFormat::set(const Field & x)
+{
+    set(safeGet<const String &>(x));
+}
+
+void SettingDateTimeInputFormat::set(const String & x)
+{
+    set(getValue(x));
+}
+
+void SettingDateTimeInputFormat::set(ReadBuffer & buf)
+{
+    String x;
+    readBinary(x, buf);
+    set(x);
+}
+
+void SettingDateTimeInputFormat::write(WriteBuffer & buf) const
+{
+    writeBinary(toString(), buf);
+}
+
 }

dbms/src/Interpreters/SettingsCommon.h

@@ -2,6 +2,7 @@
 
 #include <Poco/Timespan.h>
 #include <DataStreams/SizeLimits.h>
+#include <DataTypes/FormatSettings.h>
 #include <IO/CompressedStream.h>
 #include <Core/Types.h>
 
@@ -348,4 +349,28 @@ public:
     void write(WriteBuffer & buf) const;
 };
 
+
+struct SettingDateTimeInputFormat
+{
+    using Value = FormatSettings::DateTimeInputFormat;
+
+    Value value;
+    bool changed = false;
+
+    SettingDateTimeInputFormat(Value x) : value(x) {}
+
+    operator Value() const { return value; }
+    SettingDateTimeInputFormat & operator= (Value x) { set(x); return *this; }
+
+    static Value getValue(const String & s);
+
+    String toString() const;
+
+    void set(Value x);
+    void set(const Field & x);
+    void set(const String & x);
+    void set(ReadBuffer & buf);
+    void write(WriteBuffer & buf) const;
+};
+
 }

dbms/src/Interpreters/tests/expression.cpp

@@ -124,7 +124,7 @@ int main(int argc, char ** argv)
         auto is = std::make_shared<OneBlockInputStream>(block);
         LimitBlockInputStream lis(is, 20, std::max(0, static_cast<int>(n) - 20));
         WriteBufferFromOStream out_buf(std::cout);
-        RowOutputStreamPtr os_ = std::make_shared<TabSeparatedRowOutputStream>(out_buf, block);
+        RowOutputStreamPtr os_ = std::make_shared<TabSeparatedRowOutputStream>(out_buf, block, false, false, FormatSettings());
         BlockOutputStreamFromRowOutputStream os(os_, is->getHeader());
 
         copyData(lis, os);

dbms/src/Storages/MergeTree/MergeTreeData.cpp

@@ -2096,6 +2096,7 @@ String MergeTreeData::getPartitionIDFromQuery(const ASTPtr & ast, const Context
             ", must be: " + toString(fields_count),
             ErrorCodes::INVALID_PARTITION_VALUE);
 
+    const FormatSettings format_settings;
     Row partition_row(fields_count);
 
     if (fields_count)
@@ -2105,7 +2106,7 @@ String MergeTreeData::getPartitionIDFromQuery(const ASTPtr & ast, const Context
         ReadBufferFromMemory right_paren_buf(")", 1);
         ConcatReadBuffer buf({&left_paren_buf, &fields_buf, &right_paren_buf});
 
-        ValuesRowInputStream input_stream(buf, partition_key_sample, context, /* interpret_expressions = */true);
+        ValuesRowInputStream input_stream(buf, partition_key_sample, context, format_settings);
         MutableColumns columns = partition_key_sample.cloneEmptyColumns();
 
         if (!input_stream.read(columns))
@@ -2127,7 +2128,7 @@ String MergeTreeData::getPartitionIDFromQuery(const ASTPtr & ast, const Context
         {
             WriteBufferFromOwnString buf;
             writeCString("Parsed partition value: ", buf);
-            partition.serializeTextQuoted(*this, buf);
+            partition.serializeTextQuoted(*this, buf, format_settings);
             writeCString(" doesn't match partition value for an existing part with the same partition ID: ", buf);
             writeString(existing_part_in_partition->name, buf);
             throw Exception(buf.str(), ErrorCodes::INVALID_PARTITION_VALUE);

dbms/src/Storages/MergeTree/MergeTreePartition.cpp

@@ -77,7 +77,7 @@ String MergeTreePartition::getID(const MergeTreeData & storage) const
     return result;
 }
 
-void MergeTreePartition::serializeTextQuoted(const MergeTreeData & storage, WriteBuffer & out) const
+void MergeTreePartition::serializeTextQuoted(const MergeTreeData & storage, WriteBuffer & out, const FormatSettings & format_settings) const
 {
     size_t key_size = storage.partition_key_sample.columns();
 
@@ -98,7 +98,7 @@ void MergeTreePartition::serializeTextQuoted(const MergeTreeData & storage, Writ
         const DataTypePtr & type = storage.partition_key_sample.getByPosition(i).type;
         auto column = type->createColumn();
         column->insert(value[i]);
-        type->serializeTextQuoted(*column, 0, out);
+        type->serializeTextQuoted(*column, 0, out, format_settings);
     }
 
     if (key_size > 1)

dbms/src/Storages/MergeTree/MergeTreePartition.h

@@ -8,6 +8,7 @@ namespace DB
 {
 
 class MergeTreeData;
+struct FormatSettings;
 struct MergeTreeDataPartChecksums;
 
 /// This class represents a partition value of a single part and encapsulates its loading/storing logic.
@@ -25,7 +26,7 @@ public:
 
     String getID(const MergeTreeData & storage) const;
 
-    void serializeTextQuoted(const MergeTreeData & storage, WriteBuffer & out) const;
+    void serializeTextQuoted(const MergeTreeData & storage, WriteBuffer & out, const FormatSettings & format_settings) const;
 
     void load(const MergeTreeData & storage, const String & part_path);
     void store(const MergeTreeData & storage, const String & part_path, MergeTreeDataPartChecksums & checksums) const;
@@ -33,5 +34,4 @@ public:
     void assign(const MergeTreePartition & other) { value.assign(other.value); }
 };
 
-
 }

dbms/src/Storages/System/StorageSystemParts.cpp

@@ -59,7 +59,7 @@ void StorageSystemParts::processNextStorage(MutableColumns & columns, const Stor
         size_t i = 0;
         {
             WriteBufferFromOwnString out;
-            part->partition.serializeTextQuoted(*info.data, out);
+            part->partition.serializeTextQuoted(*info.data, out, format_settings);
             columns[i++]->insert(out.str());
         }
         columns[i++]->insert(part->name);

dbms/src/Storages/System/StorageSystemPartsBase.h

@@ -1,6 +1,7 @@
 #pragma once
 
 #include <ext/shared_ptr_helper.h>
+#include <DataTypes/FormatSettings.h>
 #include <Storages/IStorage.h>
 #include <Storages/MergeTree/MergeTreeData.h>
 
@@ -49,10 +50,11 @@ public:
 private:
     const std::string name;
 
-
     bool hasStateColumn(const Names & column_names);
 
 protected:
+    const FormatSettings format_settings;
+
     StorageSystemPartsBase(std::string name_, NamesAndTypesList && columns_);
 
     virtual void processNextStorage(MutableColumns & columns, const StoragesInfo & info, bool has_state_column) = 0;

dbms/src/Storages/System/StorageSystemPartsColumns.cpp

@@ -104,7 +104,7 @@ void StorageSystemPartsColumns::processNextStorage(MutableColumns & columns, con
             size_t j = 0;
             {
                 WriteBufferFromOwnString out;
-                part->partition.serializeTextQuoted(*info.data, out);
+                part->partition.serializeTextQuoted(*info.data, out, format_settings);
                 columns[j++]->insert(out.str());
             }
             columns[j++]->insert(part->name);

dbms/src/Storages/tests/hit_log.cpp

@@ -114,12 +114,14 @@ try
         sample.insert(std::move(elem));
     }
 
+    FormatSettings format_settings;
+
     /// read the data from tsv file and simultaneously write to table
     if (argc == 2 && 0 == strcmp(argv[1], "write"))
     {
         ReadBufferFromFileDescriptor in_buf(STDIN_FILENO);
 
-        RowInputStreamPtr in_ = std::make_shared<TabSeparatedRowInputStream>(in_buf, sample);
+        RowInputStreamPtr in_ = std::make_shared<TabSeparatedRowInputStream>(in_buf, sample, false, false, format_settings);
         BlockInputStreamFromRowInputStream in(in_, sample, DEFAULT_INSERT_BLOCK_SIZE, 0, 0);
         BlockOutputStreamPtr out = table->write({}, {});
         copyData(in, *out);
@@ -133,7 +135,7 @@ try
         QueryProcessingStage::Enum stage;
 
         BlockInputStreamPtr in = table->read(column_names, {}, Context::createGlobal(), stage, 8192, 1)[0];
-        RowOutputStreamPtr out_ = std::make_shared<TabSeparatedRowOutputStream>(out_buf, sample);
+        RowOutputStreamPtr out_ = std::make_shared<TabSeparatedRowOutputStream>(out_buf, sample, false, false, format_settings);
         BlockOutputStreamFromRowOutputStream out(out_, sample);
         copyData(*in, out);
     }

dbms/src/Storages/tests/storage_log.cpp

@@ -91,8 +91,10 @@ try
 
         WriteBufferFromOStream out_buf(std::cout);
 
+        FormatSettings format_settings;
+
         LimitBlockInputStream in_limit(in, 10, 0);
-        RowOutputStreamPtr output_ = std::make_shared<TabSeparatedRowOutputStream>(out_buf, sample);
+        RowOutputStreamPtr output_ = std::make_shared<TabSeparatedRowOutputStream>(out_buf, sample, false, false, format_settings);
         BlockOutputStreamFromRowOutputStream output(output_, sample);
 
         copyData(in_limit, output);

dbms/src/Storages/tests/system_numbers.cpp

@@ -30,7 +30,7 @@ try
     QueryProcessingStage::Enum stage;
 
     LimitBlockInputStream input(table->read(column_names, {}, Context::createGlobal(), stage, 10, 1)[0], 10, 96);
-    RowOutputStreamPtr output_ = std::make_shared<TabSeparatedRowOutputStream>(out_buf, sample);
+    RowOutputStreamPtr output_ = std::make_shared<TabSeparatedRowOutputStream>(out_buf, sample, false, false, FormatSettings());
     BlockOutputStreamFromRowOutputStream output(output_, sample);
 
     copyData(input, output);

dbms/tests/queries/0_stateless/00645_date_time_input_format.reference

@@ -0,0 +1 @@
+2018-06-08 01:02:03

dbms/tests/queries/0_stateless/00645_date_time_input_format.sql

@@ -0,0 +1,4 @@
+CREATE TEMPORARY TABLE test (d DateTime) ENGINE = Memory;
+SET date_time_input_format = 'best_effort';
+INSERT INTO test VALUES ('2018-06-08T01:02:03.000Z');
+SELECT toTimeZone(d, 'UTC') FROM test;