Merge branch 'master' into collect-core-dumps-stateless

base/base/demangle.h

@@ -1,5 +1,6 @@
 #pragma once
 
+#include <cstdlib>
 #include <memory>
 #include <string>
 

base/base/extended_types.h

@@ -108,6 +108,14 @@ struct make_unsigned // NOLINT(readability-identifier-naming)
     using type = std::make_unsigned_t<T>;
 };
 
+template <> struct make_unsigned<Int8> { using type = UInt8; };
+template <> struct make_unsigned<UInt8> { using type = UInt8; };
+template <> struct make_unsigned<Int16> { using type = UInt16; };
+template <> struct make_unsigned<UInt16> { using type = UInt16; };
+template <> struct make_unsigned<Int32> { using type = UInt32; };
+template <> struct make_unsigned<UInt32> { using type = UInt32; };
+template <> struct make_unsigned<Int64> { using type = UInt64; };
+template <> struct make_unsigned<UInt64> { using type = UInt64; };
 template <> struct make_unsigned<Int128> { using type = UInt128; };
 template <> struct make_unsigned<UInt128> { using type = UInt128; };
 template <> struct make_unsigned<Int256>  { using type = UInt256; };
@@ -121,6 +129,14 @@ struct make_signed // NOLINT(readability-identifier-naming)
     using type = std::make_signed_t<T>;
 };
 
+template <> struct make_signed<Int8> { using type = Int8; };
+template <> struct make_signed<UInt8> { using type = Int8; };
+template <> struct make_signed<Int16> { using type = Int16; };
+template <> struct make_signed<UInt16> { using type = Int16; };
+template <> struct make_signed<Int32> { using type = Int32; };
+template <> struct make_signed<UInt32> { using type = Int32; };
+template <> struct make_signed<Int64> { using type = Int64; };
+template <> struct make_signed<UInt64> { using type = Int64; };
 template <> struct make_signed<Int128>  { using type = Int128; };
 template <> struct make_signed<UInt128> { using type = Int128; };
 template <> struct make_signed<Int256>  { using type = Int256; };

base/base/isSharedPtrUnique.h

@@ -0,0 +1,9 @@
+#pragma once
+
+#include <memory>
+
+template <typename T>
+bool isSharedPtrUnique(const std::shared_ptr<T> & ptr)
+{
+    return ptr.use_count() == 1;
+}

base/poco/Foundation/include/Poco/Format.h

@@ -232,7 +232,7 @@ void Foundation_API format(
     const Any & value10);
 
 
-void Foundation_API format(std::string & result, const std::string & fmt, const std::vector<Any> & values);
+void Foundation_API formatVector(std::string & result, const std::string & fmt, const std::vector<Any> & values);
 /// Supports a variable number of arguments and is used by
 /// all other variants of format().
 

base/poco/Foundation/include/Poco/RefCountedObject.h

@@ -21,6 +21,8 @@
 #include "Poco/AtomicCounter.h"
 #include "Poco/Foundation.h"
 
+#include <atomic>
+
 
 namespace Poco
 {

base/poco/Foundation/src/Format.cpp

@@ -51,8 +51,8 @@ namespace
 		}
 		if (width != 0) str.width(width);
 	}
-	
-	
+
+
 	void parsePrec(std::ostream& str, std::string::const_iterator& itFmt, const std::string::const_iterator& endFmt)
 	{
 		if (itFmt != endFmt && *itFmt == '.')
@@ -67,7 +67,7 @@ namespace
 			if (prec >= 0) str.precision(prec);
 		}
 	}
-	
+
 	char parseMod(std::string::const_iterator& itFmt, const std::string::const_iterator& endFmt)
 	{
 		char mod = 0;
@@ -77,13 +77,13 @@ namespace
 			{
 			case 'l':
 			case 'h':
-			case 'L': 
+			case 'L':
 			case '?': mod = *itFmt++; break;
 			}
 		}
 		return mod;
 	}
-	
+
 	std::size_t parseIndex(std::string::const_iterator& itFmt, const std::string::const_iterator& endFmt)
 	{
 		int index = 0;
@@ -110,8 +110,8 @@ namespace
 		case 'f': str << std::fixed; break;
 		}
 	}
-	
-	
+
+
 	void writeAnyInt(std::ostream& str, const Any& any)
 	{
 		if (any.type() == typeid(char))
@@ -201,7 +201,7 @@ namespace
 					str << RefAnyCast<std::string>(*itVal++);
 					break;
 				case 'z':
-					str << AnyCast<std::size_t>(*itVal++); 
+					str << AnyCast<std::size_t>(*itVal++);
 					break;
 				case 'I':
 				case 'D':
@@ -303,7 +303,7 @@ void format(std::string& result, const std::string& fmt, const Any& value)
 {
 	std::vector<Any> args;
 	args.push_back(value);
-	format(result, fmt, args);
+	formatVector(result, fmt, args);
 }
 
 
@@ -312,7 +312,7 @@ void format(std::string& result, const std::string& fmt, const Any& value1, cons
 	std::vector<Any> args;
 	args.push_back(value1);
 	args.push_back(value2);
-	format(result, fmt, args);
+	formatVector(result, fmt, args);
 }
 
 
@@ -322,7 +322,7 @@ void format(std::string& result, const std::string& fmt, const Any& value1, cons
 	args.push_back(value1);
 	args.push_back(value2);
 	args.push_back(value3);
-	format(result, fmt, args);
+	formatVector(result, fmt, args);
 }
 
 
@@ -333,7 +333,7 @@ void format(std::string& result, const std::string& fmt, const Any& value1, cons
 	args.push_back(value2);
 	args.push_back(value3);
 	args.push_back(value4);
-	format(result, fmt, args);
+	formatVector(result, fmt, args);
 }
 
 
@@ -345,7 +345,7 @@ void format(std::string& result, const std::string& fmt, const Any& value1, cons
 	args.push_back(value3);
 	args.push_back(value4);
 	args.push_back(value5);
-	format(result, fmt, args);
+	formatVector(result, fmt, args);
 }
 
 
@@ -358,7 +358,7 @@ void format(std::string& result, const std::string& fmt, const Any& value1, cons
 	args.push_back(value4);
 	args.push_back(value5);
 	args.push_back(value6);
-	format(result, fmt, args);
+	formatVector(result, fmt, args);
 }
 
 
@@ -372,7 +372,7 @@ void format(std::string& result, const std::string& fmt, const Any& value1, cons
 	args.push_back(value5);
 	args.push_back(value6);
 	args.push_back(value7);
-	format(result, fmt, args);
+	formatVector(result, fmt, args);
 }
 
 
@@ -387,7 +387,7 @@ void format(std::string& result, const std::string& fmt, const Any& value1, cons
 	args.push_back(value6);
 	args.push_back(value7);
 	args.push_back(value8);
-	format(result, fmt, args);
+	formatVector(result, fmt, args);
 }
 
 
@@ -403,7 +403,7 @@ void format(std::string& result, const std::string& fmt, const Any& value1, cons
 	args.push_back(value7);
 	args.push_back(value8);
 	args.push_back(value9);
-	format(result, fmt, args);
+	formatVector(result, fmt, args);
 }
 
 
@@ -420,16 +420,16 @@ void format(std::string& result, const std::string& fmt, const Any& value1, cons
 	args.push_back(value8);
 	args.push_back(value9);
 	args.push_back(value10);
-	format(result, fmt, args);
+	formatVector(result, fmt, args);
 }
 
 
-void format(std::string& result, const std::string& fmt, const std::vector<Any>& values)
+void formatVector(std::string& result, const std::string& fmt, const std::vector<Any>& values)
 {
 	std::string::const_iterator itFmt  = fmt.begin();
 	std::string::const_iterator endFmt = fmt.end();
 	std::vector<Any>::const_iterator itVal  = values.begin();
-	std::vector<Any>::const_iterator endVal = values.end(); 
+	std::vector<Any>::const_iterator endVal = values.end();
 	while (itFmt != endFmt)
 	{
 		switch (*itFmt)

base/poco/MongoDB/src/ObjectId.cpp

@@ -57,7 +57,7 @@ std::string ObjectId::toString(const std::string& fmt) const
 
 	for (int i = 0; i < 12; ++i)
 	{
-		s += format(fmt, (unsigned int) _id[i]);
+		s += Poco::format(fmt, (unsigned int) _id[i]);
 	}
 	return s;
 }

base/poco/MongoDB/src/OpMsgCursor.cpp

@@ -43,9 +43,9 @@ namespace Poco {
 namespace MongoDB {
 
 
-static const std::string keyCursor		{"cursor"};
-static const std::string keyFirstBatch	{"firstBatch"};
-static const std::string keyNextBatch	{"nextBatch"};
+[[ maybe_unused ]] static const std::string keyCursor		{"cursor"};
+[[ maybe_unused ]] static const std::string keyFirstBatch	{"firstBatch"};
+[[ maybe_unused ]] static const std::string keyNextBatch	{"nextBatch"};
 
 static Poco::Int64 cursorIdFromResponse(const MongoDB::Document& doc);
 
@@ -131,7 +131,7 @@ OpMsgMessage& OpMsgCursor::next(Connection& connection)
 			connection.readResponse(_response);
 		}
 		else
-#endif		
+#endif
 		{
 			_response.clear();
 			_query.setCursor(_cursorID, _batchSize);

base/poco/Net/src/HTTPMessage.cpp

@@ -17,9 +17,9 @@
 #include "Poco/NumberFormatter.h"
 #include "Poco/NumberParser.h"
 #include "Poco/String.h"
+#include <charconv>
 #include <format>
 
-
 using Poco::NumberFormatter;
 using Poco::NumberParser;
 using Poco::icompare;
@@ -75,7 +75,7 @@ void HTTPMessage::setContentLength(std::streamsize length)
 		erase(CONTENT_LENGTH);
 }
 
-	
+
 std::streamsize HTTPMessage::getContentLength() const
 {
 	const std::string& contentLength = get(CONTENT_LENGTH, EMPTY);
@@ -98,7 +98,7 @@ void HTTPMessage::setContentLength64(Poco::Int64 length)
 		erase(CONTENT_LENGTH);
 }
 
-	
+
 Poco::Int64 HTTPMessage::getContentLength64() const
 {
 	const std::string& contentLength = get(CONTENT_LENGTH, EMPTY);
@@ -133,13 +133,13 @@ void HTTPMessage::setChunkedTransferEncoding(bool flag)
 		setTransferEncoding(IDENTITY_TRANSFER_ENCODING);
 }
 
-	
+
 bool HTTPMessage::getChunkedTransferEncoding() const
 {
 	return icompare(getTransferEncoding(), CHUNKED_TRANSFER_ENCODING) == 0;
 }
 
-	
+
 void HTTPMessage::setContentType(const std::string& mediaType)
 {
 	if (mediaType.empty())
@@ -154,7 +154,7 @@ void HTTPMessage::setContentType(const MediaType& mediaType)
 	setContentType(mediaType.toString());
 }
 
-	
+
 const std::string& HTTPMessage::getContentType() const
 {
 	return get(CONTENT_TYPE, UNKNOWN_CONTENT_TYPE);

contrib/grpc

@@ -1 +1 @@
-Subproject commit 77b2737a709d43d8c6895e3f03ca62b00bd9201c
+Subproject commit f5b7fdc2dff09ada06dbf6c75df298fb40f898df

docker/test/fasttest/run.sh

@@ -284,6 +284,11 @@ function run_tests
       NPROC=1
     fi
 
+    export CLICKHOUSE_CONFIG_DIR=$FASTTEST_DATA
+    export CLICKHOUSE_CONFIG="$FASTTEST_DATA/config.xml"
+    export CLICKHOUSE_USER_FILES="$FASTTEST_DATA/user_files"
+    export CLICKHOUSE_SCHEMA_FILES="$FASTTEST_DATA/format_schemas"
+
     local test_opts=(
         --hung-check
         --fast-tests-only

docker/test/stateless/run.sh

@@ -9,6 +9,16 @@ set -e -x -a
 MAX_RUN_TIME=${MAX_RUN_TIME:-10800}
 MAX_RUN_TIME=$((MAX_RUN_TIME == 0 ? 10800 : MAX_RUN_TIME))
 
+USE_DATABASE_REPLICATED=${USE_DATABASE_REPLICATED:=0}
+USE_SHARED_CATALOG=${USE_SHARED_CATALOG:=0}
+
+# disable for now
+RUN_SEQUENTIAL_TESTS_IN_PARALLEL=0
+
+if [[ "$USE_DATABASE_REPLICATED" -eq 1 ]] || [[ "$USE_SHARED_CATALOG" -eq 1 ]]; then
+  RUN_SEQUENTIAL_TESTS_IN_PARALLEL=0
+fi
+
 # Choose random timezone for this test run.
 #
 # NOTE: that clickhouse-test will randomize session_timezone by itself as well
@@ -89,10 +99,57 @@ if [ "$NUM_TRIES" -gt "1" ]; then
     mkdir -p /var/run/clickhouse-server
 fi
 
+# Run a CH instance to execute sequential tests on it in parallel with all other tests.
+if [[ "$RUN_SEQUENTIAL_TESTS_IN_PARALLEL" -eq 1 ]]; then
+  mkdir -p /var/run/clickhouse-server3 /etc/clickhouse-server3 /var/lib/clickhouse3
+  cp -r -L /etc/clickhouse-server/* /etc/clickhouse-server3/
+
+  sudo chown clickhouse:clickhouse /var/run/clickhouse-server3 /var/lib/clickhouse3 /etc/clickhouse-server3/
+  sudo chown -R clickhouse:clickhouse /etc/clickhouse-server3/*
+
+  function replace(){
+      sudo find /etc/clickhouse-server3/ -type f -name '*.xml' -exec sed -i "$1" {} \;
+  }
+
+  replace "s|<port>9000</port>|<port>19000</port>|g"
+  replace "s|<port>9440</port>|<port>19440</port>|g"
+  replace "s|<port>9988</port>|<port>19988</port>|g"
+  replace "s|<port>9234</port>|<port>19234</port>|g"
+  replace "s|<port>9181</port>|<port>19181</port>|g"
+  replace "s|<https_port>8443</https_port>|<https_port>18443</https_port>|g"
+  replace "s|<tcp_port>9000</tcp_port>|<tcp_port>19000</tcp_port>|g"
+  replace "s|<tcp_port>9181</tcp_port>|<tcp_port>19181</tcp_port>|g"
+  replace "s|<tcp_port_secure>9440</tcp_port_secure>|<tcp_port_secure>19440</tcp_port_secure>|g"
+  replace "s|<tcp_with_proxy_port>9010</tcp_with_proxy_port>|<tcp_with_proxy_port>19010</tcp_with_proxy_port>|g"
+  replace "s|<mysql_port>9004</mysql_port>|<mysql_port>19004</mysql_port>|g"
+  replace "s|<postgresql_port>9005</postgresql_port>|<postgresql_port>19005</postgresql_port>|g"
+  replace "s|<interserver_http_port>9009</interserver_http_port>|<interserver_http_port>19009</interserver_http_port>|g"
+  replace "s|8123|18123|g"
+  replace "s|/var/lib/clickhouse/|/var/lib/clickhouse3/|g"
+  replace "s|/etc/clickhouse-server/|/etc/clickhouse-server3/|g"
+  # distributed cache
+  replace "s|<tcp_port>10001</tcp_port>|<tcp_port>10003</tcp_port>|g"
+  replace "s|<tcp_port>10002</tcp_port>|<tcp_port>10004</tcp_port>|g"
+
+  sudo -E -u clickhouse /usr/bin/clickhouse server --daemon --config /etc/clickhouse-server3/config.xml \
+  --pid-file /var/run/clickhouse-server3/clickhouse-server.pid \
+  -- --path /var/lib/clickhouse3/ --logger.stderr /var/log/clickhouse-server/stderr3.log \
+  --logger.log /var/log/clickhouse-server/clickhouse-server3.log --logger.errorlog /var/log/clickhouse-server/clickhouse-server3.err.log \
+  --tcp_port 19000 --tcp_port_secure 19440 --http_port 18123 --https_port 18443 --interserver_http_port 19009 --tcp_with_proxy_port 19010 \
+  --prometheus.port 19988 --keeper_server.raft_configuration.server.port 19234 --keeper_server.tcp_port 19181 \
+  --mysql_port 19004 --postgresql_port 19005
+
+  for _ in {1..100}
+  do
+      clickhouse-client --port 19000 --query "SELECT 1" && break
+      sleep 1
+  done
+fi
+
 # simplest way to forward env variables to server
 sudo -E -u clickhouse /usr/bin/clickhouse-server --config /etc/clickhouse-server/config.xml --daemon --pid-file /var/run/clickhouse-server/clickhouse-server.pid
 
-if [[ -n "$USE_DATABASE_REPLICATED" ]] && [[ "$USE_DATABASE_REPLICATED" -eq 1 ]]; then
+if [[ "$USE_DATABASE_REPLICATED" -eq 1 ]]; then
     sudo sed -i "s|<filesystem_caches_path>/var/lib/clickhouse/filesystem_caches/</filesystem_caches_path>|<filesystem_caches_path>/var/lib/clickhouse/filesystem_caches_1/</filesystem_caches_path>|" /etc/clickhouse-server1/config.d/filesystem_caches_path.xml
 
     sudo sed -i "s|<filesystem_caches_path>/var/lib/clickhouse/filesystem_caches/</filesystem_caches_path>|<filesystem_caches_path>/var/lib/clickhouse/filesystem_caches_2/</filesystem_caches_path>|" /etc/clickhouse-server2/config.d/filesystem_caches_path.xml
@@ -129,7 +186,7 @@ if [[ -n "$USE_DATABASE_REPLICATED" ]] && [[ "$USE_DATABASE_REPLICATED" -eq 1 ]]
     MAX_RUN_TIME=$((MAX_RUN_TIME != 0 ? MAX_RUN_TIME : 9000))    # set to 2.5 hours if 0 (unlimited)
 fi
 
-if [[ -n "$USE_SHARED_CATALOG" ]] && [[ "$USE_SHARED_CATALOG" -eq 1 ]]; then
+if [[ "$USE_SHARED_CATALOG" -eq 1 ]]; then
     sudo cat /etc/clickhouse-server1/config.d/filesystem_caches_path.xml \
     | sed "s|<filesystem_caches_path>/var/lib/clickhouse/filesystem_caches/</filesystem_caches_path>|<filesystem_caches_path>/var/lib/clickhouse/filesystem_caches_1/</filesystem_caches_path>|" \
     > /etc/clickhouse-server1/config.d/filesystem_caches_path.xml.tmp
@@ -209,15 +266,15 @@ function run_tests()
         ADDITIONAL_OPTIONS+=('--no-random-merge-tree-settings')
     fi
 
-    if [[ -n "$USE_SHARED_CATALOG" ]] && [[ "$USE_SHARED_CATALOG" -eq 1 ]]; then
+    if [[ "$USE_SHARED_CATALOG" -eq 1 ]]; then
         ADDITIONAL_OPTIONS+=('--shared-catalog')
     fi
 
-    if [[ -n "$USE_DATABASE_REPLICATED" ]] && [[ "$USE_DATABASE_REPLICATED" -eq 1 ]]; then
+    if [[ "$USE_DATABASE_REPLICATED" -eq 1 ]]; then
         ADDITIONAL_OPTIONS+=('--replicated-database')
         # Too many tests fail for DatabaseReplicated in parallel.
         ADDITIONAL_OPTIONS+=('--jobs')
-        ADDITIONAL_OPTIONS+=('2')
+        ADDITIONAL_OPTIONS+=('3')
     elif [[ 1 == $(clickhouse-client --query "SELECT value LIKE '%SANITIZE_COVERAGE%' FROM system.build_options WHERE name = 'CXX_FLAGS'") ]]; then
         # Coverage on a per-test basis could only be collected sequentially.
         # Do not set the --jobs parameter.
@@ -225,7 +282,11 @@ function run_tests()
     else
         # All other configurations are OK.
         ADDITIONAL_OPTIONS+=('--jobs')
-        ADDITIONAL_OPTIONS+=('8')
+        ADDITIONAL_OPTIONS+=('5')
+    fi
+
+    if [[ "$RUN_SEQUENTIAL_TESTS_IN_PARALLEL" -eq 1 ]]; then
+        ADDITIONAL_OPTIONS+=('--run-sequential-tests-in-parallel')
     fi
 
     if [[ -n "$RUN_BY_HASH_NUM" ]] && [[ -n "$RUN_BY_HASH_TOTAL" ]]; then
@@ -289,7 +350,7 @@ do
     err=$(clickhouse-client -q "select * from system.$table into outfile '/test_output/$table.tsv.gz' format TSVWithNamesAndTypes")
     echo "$err"
     [[ "0" != "${#err}"  ]] && failed_to_save_logs=1
-    if [[ -n "$USE_DATABASE_REPLICATED" ]] && [[ "$USE_DATABASE_REPLICATED" -eq 1 ]]; then
+    if [[ "$USE_DATABASE_REPLICATED" -eq 1 ]]; then
         err=$( { clickhouse-client --port 19000 -q "select * from system.$table format TSVWithNamesAndTypes" | zstd --threads=0 > /test_output/$table.1.tsv.zst; } 2>&1 )
         echo "$err"
         [[ "0" != "${#err}"  ]] && failed_to_save_logs=1
@@ -298,7 +359,7 @@ do
         [[ "0" != "${#err}"  ]] && failed_to_save_logs=1
     fi
 
-    if [[ -n "$USE_SHARED_CATALOG" ]] && [[ "$USE_SHARED_CATALOG" -eq 1 ]]; then
+    if [[ "$USE_SHARED_CATALOG" -eq 1 ]]; then
         err=$( { clickhouse-client --port 19000 -q "select * from system.$table format TSVWithNamesAndTypes" | zstd --threads=0 > /test_output/$table.1.tsv.zst; } 2>&1 )
         echo "$err"
         [[ "0" != "${#err}"  ]] && failed_to_save_logs=1
@@ -309,12 +370,17 @@ done
 # Why do we read data with clickhouse-local?
 # Because it's the simplest way to read it when server has crashed.
 sudo clickhouse stop ||:
-if [[ -n "$USE_DATABASE_REPLICATED" ]] && [[ "$USE_DATABASE_REPLICATED" -eq 1 ]]; then
+
+if [[ "$RUN_SEQUENTIAL_TESTS_IN_PARALLEL" -eq 1 ]]; then
+    sudo clickhouse stop --pid-path /var/run/clickhouse-server3 ||:
+fi
+
+if [[ "$USE_DATABASE_REPLICATED" -eq 1 ]]; then
     sudo clickhouse stop --pid-path /var/run/clickhouse-server1 ||:
     sudo clickhouse stop --pid-path /var/run/clickhouse-server2 ||:
 fi
 
-if [[ -n "$USE_SHARED_CATALOG" ]] && [[ "$USE_SHARED_CATALOG" -eq 1 ]]; then
+if [[ "$USE_SHARED_CATALOG" -eq 1 ]]; then
     sudo clickhouse stop --pid-path /var/run/clickhouse-server1 ||:
 fi
 
@@ -322,6 +388,12 @@ rg -Fa "<Fatal>" /var/log/clickhouse-server/clickhouse-server.log ||:
 rg -A50 -Fa "============" /var/log/clickhouse-server/stderr.log ||:
 zstd --threads=0 < /var/log/clickhouse-server/clickhouse-server.log > /test_output/clickhouse-server.log.zst &
 
+if [[ "$RUN_SEQUENTIAL_TESTS_IN_PARALLEL" -eq 1 ]]; then
+  rg -Fa "<Fatal>" /var/log/clickhouse-server3/clickhouse-server.log ||:
+  rg -A50 -Fa "============" /var/log/clickhouse-server3/stderr.log ||:
+  zstd --threads=0 < /var/log/clickhouse-server3/clickhouse-server.log > /test_output/clickhouse-server3.log.zst &
+fi
+
 data_path_config="--path=/var/lib/clickhouse/"
 if [[ -n "$USE_S3_STORAGE_FOR_MERGE_TREE" ]] && [[ "$USE_S3_STORAGE_FOR_MERGE_TREE" -eq 1 ]]; then
     # We need s3 storage configuration (but it's more likely that clickhouse-local will fail for some reason)
@@ -341,12 +413,17 @@ if [ $failed_to_save_logs -ne 0 ]; then
     for table in query_log zookeeper_log trace_log transactions_info_log metric_log blob_storage_log error_log
     do
         clickhouse-local "$data_path_config" --only-system-tables --stacktrace -q "select * from system.$table format TSVWithNamesAndTypes" | zstd --threads=0 > /test_output/$table.tsv.zst ||:
-        if [[ -n "$USE_DATABASE_REPLICATED" ]] && [[ "$USE_DATABASE_REPLICATED" -eq 1 ]]; then
+
+        if [[ "$RUN_SEQUENTIAL_TESTS_IN_PARALLEL" -eq 1 ]]; then
+            clickhouse-local --path /var/lib/clickhouse3/ --only-system-tables --stacktrace -q "select * from system.$table format TSVWithNamesAndTypes" | zstd --threads=0 > /test_output/$table.3.tsv.zst ||:
+        fi
+
+        if [[ "$USE_DATABASE_REPLICATED" -eq 1 ]]; then
             clickhouse-local --path /var/lib/clickhouse1/ --only-system-tables --stacktrace -q "select * from system.$table format TSVWithNamesAndTypes" | zstd --threads=0 > /test_output/$table.1.tsv.zst ||:
             clickhouse-local --path /var/lib/clickhouse2/ --only-system-tables --stacktrace -q "select * from system.$table format TSVWithNamesAndTypes" | zstd --threads=0 > /test_output/$table.2.tsv.zst ||:
         fi
 
-        if [[ -n "$USE_SHARED_CATALOG" ]] && [[ "$USE_SHARED_CATALOG" -eq 1 ]]; then
+        if [[ "$USE_SHARED_CATALOG" -eq 1 ]]; then
             clickhouse-local --path /var/lib/clickhouse1/ --only-system-tables --stacktrace -q "select * from system.$table format TSVWithNamesAndTypes" | zstd --threads=0 > /test_output/$table.1.tsv.zst ||:
         fi
     done
@@ -382,7 +459,14 @@ rm -rf /var/lib/clickhouse/data/system/*/
 tar -chf /test_output/store.tar /var/lib/clickhouse/store ||:
 tar -chf /test_output/metadata.tar /var/lib/clickhouse/metadata/*.sql ||:
 
-if [[ -n "$USE_DATABASE_REPLICATED" ]] && [[ "$USE_DATABASE_REPLICATED" -eq 1 ]]; then
+if [[ "$RUN_SEQUENTIAL_TESTS_IN_PARALLEL" -eq 1 ]]; then
+    rm -rf /var/lib/clickhouse3/data/system/*/
+    tar -chf /test_output/store.tar /var/lib/clickhouse3/store ||:
+    tar -chf /test_output/metadata.tar /var/lib/clickhouse3/metadata/*.sql ||:
+fi
+
+
+if [[ "$USE_DATABASE_REPLICATED" -eq 1 ]]; then
     rg -Fa "<Fatal>" /var/log/clickhouse-server/clickhouse-server1.log ||:
     rg -Fa "<Fatal>" /var/log/clickhouse-server/clickhouse-server2.log ||:
     zstd --threads=0 < /var/log/clickhouse-server/clickhouse-server1.log > /test_output/clickhouse-server1.log.zst ||:
@@ -393,7 +477,7 @@ if [[ -n "$USE_DATABASE_REPLICATED" ]] && [[ "$USE_DATABASE_REPLICATED" -eq 1 ]]
     tar -chf /test_output/coordination2.tar /var/lib/clickhouse2/coordination ||:
 fi
 
-if [[ -n "$USE_SHARED_CATALOG" ]] && [[ "$USE_SHARED_CATALOG" -eq 1 ]]; then
+if [[ "$USE_SHARED_CATALOG" -eq 1 ]]; then
     rg -Fa "<Fatal>" /var/log/clickhouse-server/clickhouse-server1.log ||:
     zstd --threads=0 < /var/log/clickhouse-server/clickhouse-server1.log > /test_output/clickhouse-server1.log.zst ||:
     mv /var/log/clickhouse-server/stderr1.log /test_output/ ||:

docs/en/interfaces/formats.md

@@ -1535,6 +1535,10 @@ the columns from input data will be mapped to the columns from the table by thei
 Otherwise, the first row will be skipped.
 If setting [input_format_with_types_use_header](/docs/en/operations/settings/settings-formats.md/#input_format_with_types_use_header) is set to 1,
 the types from input data will be compared with the types of the corresponding columns from the table. Otherwise, the second row will be skipped.
+If setting [output_format_binary_encode_types_in_binary_format](/docs/en/operations/settings/settings-formats.md/#output_format_binary_encode_types_in_binary_format) is set to 1,
+the types in header will be written using [binary encoding](/docs/en/sql-reference/data-types/data-types-binary-encoding.md) instead of strings with type names in RowBinaryWithNamesAndTypes output format.
+If setting [input_format_binary_encode_types_in_binary_format](/docs/en/operations/settings/settings-formats.md/#input_format_binary_encode_types_in_binary_format) is set to 1,
+the types in header will be read using [binary encoding](/docs/en/sql-reference/data-types/data-types-binary-encoding.md) instead of strings with type names in RowBinaryWithNamesAndTypes input format.
 :::
 
 ## RowBinaryWithDefaults {#rowbinarywithdefaults}

docs/en/operations/opentelemetry.md

@@ -2,15 +2,11 @@
 slug: /en/operations/opentelemetry
 sidebar_position: 62
 sidebar_label: Tracing ClickHouse with OpenTelemetry
-title: "[experimental] Tracing ClickHouse with OpenTelemetry"
+title: "Tracing ClickHouse with OpenTelemetry"
 ---
 
 [OpenTelemetry](https://opentelemetry.io/) is an open standard for collecting traces and metrics from the distributed application. ClickHouse has some support for OpenTelemetry.
 
-:::note    
-This is an experimental feature that will change in backwards-incompatible ways in future releases.
-:::
-
 ## Supplying Trace Context to ClickHouse
 
 ClickHouse accepts trace context HTTP headers, as described by the [W3C recommendation](https://www.w3.org/TR/trace-context/). It also accepts trace context over a native protocol that is used for communication between ClickHouse servers or between the client and server. For manual testing, trace context headers conforming to the Trace Context recommendation can be supplied to `clickhouse-client` using `--opentelemetry-traceparent` and `--opentelemetry-tracestate` flags.

docs/en/operations/settings/settings-formats.md

@@ -1951,6 +1951,18 @@ The maximum allowed size for String in RowBinary format. It prevents allocating
 
 Default value: `1GiB`.
 
+### output_format_binary_encode_types_in_binary_format {#output_format_binary_encode_types_in_binary_format}
+
+Write data types in [binary format](../../sql-reference/data-types/data-types-binary-encoding.md) instead of type names in RowBinaryWithNamesAndTypes output format.
+
+Disabled by default.
+
+### input_format_binary_decode_types_in_binary_format {#input_format_binary_decode_types_in_binary_format}
+
+Read data types in [binary format](../../sql-reference/data-types/data-types-binary-encoding.md) instead of type names in RowBinaryWithNamesAndTypes input format.
+
+Disabled by default.
+
 ## Native format settings {#native-format-settings}
 
 ### input_format_native_allow_types_conversion {#input_format_native_allow_types_conversion}
@@ -1958,3 +1970,15 @@ Default value: `1GiB`.
 Allow types conversion in Native input format between columns from input data and requested columns.
 
 Enabled by default.
+
+### output_format_native_encode_types_in_binary_format {#output_format_native_encode_types_in_binary_format}
+
+Write data types in [binary format](../../sql-reference/data-types/data-types-binary-encoding.md) instead of type names in Native output format.
+
+Disabled by default.
+
+### input_format_native_decode_types_in_binary_format {#input_format_native_decode_types_in_binary_format}
+
+Read data types in [binary format](../../sql-reference/data-types/data-types-binary-encoding.md) instead of type names in Native input format.
+
+Disabled by default.

docs/en/operations/settings/settings.md

@@ -1358,12 +1358,25 @@ Connection pool size for PostgreSQL table engine and database engine.
 
 Default value: 16
 
+## postgresql_connection_attempt_timeout {#postgresql-connection-attempt-timeout}
+
+Connection timeout in seconds of a single attempt to connect PostgreSQL end-point.
+The value is passed as a `connect_timeout` parameter of the connection URL.
+
+Default value: `2`.
+
 ## postgresql_connection_pool_wait_timeout {#postgresql-connection-pool-wait-timeout}
 
 Connection pool push/pop timeout on empty pool for PostgreSQL table engine and database engine. By default it will block on empty pool.
 
 Default value: 5000
 
+## postgresql_connection_pool_retries {#postgresql-connection-pool-retries}
+
+The maximum number of retries to establish a connection with the PostgreSQL end-point.
+
+Default value: `2`.
+
 ## postgresql_connection_pool_auto_close_connection {#postgresql-connection-pool-auto-close-connection}
 
 Close connection before returning connection to the pool.

docs/en/sql-reference/data-types/data-types-binary-encoding.md

@@ -0,0 +1,115 @@
+---
+slug: /en/sql-reference/data-types/data-types-binary-encoding
+sidebar_position: 56
+sidebar_label: Data types binary encoding specification.
+---
+
+
+# Data types binary encoding specification
+
+This specification describes the binary format that can be used for binary encoding and decoding of ClickHouse data types. This format is used in `Dynamic` column [binary serialization](dynamic.md#binary-output-format) and can be used in input/output formats [RowBinaryWithNamesAndTypes](../../interfaces/formats.md#rowbinarywithnamesandtypes) and [Native](../../interfaces/formats.md#native) under corresponding settings.
+
+The table below describes how each data type is represented in binary format. Each data type encoding consist of 1 byte that indicates the type and some optional additional information.
+`var_uint` in the binary encoding means that the size is encoded using Variable-Length Quantity compression.
+
+| ClickHouse data type                                                                 | Binary encoding                                                                                                                                                                                                                                                                                                        |
+|--------------------------------------------------------------------------------------|------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|
+| `Nothing`                                                                            | `0x00`                                                                                                                                                                                                                                                                                                                 |
+| `UInt8`                                                                              | `0x01`                                                                                                                                                                                                                                                                                                                 |
+| `UInt16`                                                                             | `0x02`                                                                                                                                                                                                                                                                                                                 |
+| `UInt32`                                                                             | `0x03`                                                                                                                                                                                                                                                                                                                 |
+| `UInt64`                                                                             | `0x04`                                                                                                                                                                                                                                                                                                                 |
+| `UInt128`                                                                            | `0x05`                                                                                                                                                                                                                                                                                                                 |
+| `UInt256`                                                                            | `0x06`                                                                                                                                                                                                                                                                                                                 |
+| `Int8`                                                                               | `0x07`                                                                                                                                                                                                                                                                                                                 |
+| `Int16`                                                                              | `0x08`                                                                                                                                                                                                                                                                                                                 |
+| `Int32`                                                                              | `0x09`                                                                                                                                                                                                                                                                                                                 |
+| `Int64`                                                                              | `0x0A`                                                                                                                                                                                                                                                                                                                 |
+| `Int128`                                                                             | `0x0B`                                                                                                                                                                                                                                                                                                                 |
+| `Int256`                                                                             | `0x0C`                                                                                                                                                                                                                                                                                                                 |
+| `Float32`                                                                            | `0x0D`                                                                                                                                                                                                                                                                                                                 |
+| `Float64`                                                                            | `0x0E`                                                                                                                                                                                                                                                                                                                 |
+| `Date`                                                                               | `0x0F`                                                                                                                                                                                                                                                                                                                 |
+| `Date32`                                                                             | `0x10`                                                                                                                                                                                                                                                                                                                 |
+| `DateTime`                                                                           | `0x11`                                                                                                                                                                                                                                                                                                                 |
+| `DateTime(time_zone)`                                                                | `0x12<var_uint_time_zone_name_size><time_zone_name_data>`                                                                                                                                                                                                                                                              |
+| `DateTime64(P)`                                                                      | `0x13<uint8_precision>`                                                                                                                                                                                                                                                                                                |
+| `DateTime64(P, time_zone)`                                                           | `0x14<uint8_precision><var_uint_time_zone_name_size><time_zone_name_data>`                                                                                                                                                                                                                                             |
+| `String`                                                                             | `0x15`                                                                                                                                                                                                                                                                                                                 |
+| `FixedString(N)`                                                                     | `0x16<var_uint_size>`                                                                                                                                                                                                                                                                                                  |
+| `Enum8`                                                                              | `0x17<var_uint_number_of_elements><var_uint_name_size_1><name_data_1><int8_value_1>...<var_uint_name_size_N><name_data_N><int8_value_N>`                                                                                                                                                                               |
+| `Enum16`                                                                             | `0x18<var_uint_number_of_elements><var_uint_name_size_1><name_data_1><int16_little_endian_value_1>...><var_uint_name_size_N><name_data_N><int16_little_endian_value_N>`                                                                                                                                                |
+| `Decimal32(P, S)`                                                                    | `0x19<uint8_precision><uint8_scale>`                                                                                                                                                                                                                                                                                   |
+| `Decimal64(P, S)`                                                                    | `0x1A<uint8_precision><uint8_scale>`                                                                                                                                                                                                                                                                                   |
+| `Decimal128(P, S)`                                                                   | `0x1B<uint8_precision><uint8_scale>`                                                                                                                                                                                                                                                                                   |
+| `Decimal256(P, S)`                                                                   | `0x1C<uint8_precision><uint8_scale>`                                                                                                                                                                                                                                                                                   |
+| `UUID`                                                                               | `0x1D`                                                                                                                                                                                                                                                                                                                 |
+| `Array(T)`                                                                           | `0x1E<nested_type_encoding>`                                                                                                                                                                                                                                                                                           |
+| `Tuple(T1, ..., TN)`                                                                 | `0x1F<var_uint_number_of_elements><nested_type_encoding_1>...<nested_type_encoding_N>`                                                                                                                                                                                                                                 |
+| `Tuple(name1 T1, ..., nameN TN)`                                                     | `0x20<var_uint_number_of_elements><var_uint_name_size_1><name_data_1><nested_type_encoding_1>...<var_uint_name_size_N><name_data_N><nested_type_encoding_N>`                                                                                                                                                           |
+| `Set`                                                                                | `0x21`                                                                                                                                                                                                                                                                                                                 |
+| `Interval`                                                                           | `0x22<interval_kind>` (see [interval kind binary encoding](#interval-kind-binary-encoding))                                                                                                                                                                                                                            |
+| `Nullable(T)`                                                                        | `0x23<nested_type_encoding>`                                                                                                                                                                                                                                                                                           |
+| `Function`                                                                           | `0x24<var_uint_number_of_arguments><argument_type_encoding_1>...<argument_type_encoding_N><return_type_encoding>`                                                                                                                                                                                                      |
+| `AggregateFunction(function_name(param_1, ..., param_N), arg_T1, ..., arg_TN)`       | `0x25<var_uint_version><var_uint_function_name_size><function_name_data><var_uint_number_of_parameters><param_1>...<param_N><var_uint_number_of_arguments><argument_type_encoding_1>...<argument_type_encoding_N>` (see [aggregate function parameter binary encoding](#aggregate-function-parameter-binary-encoding)) |
+| `LowCardinality(T)`                                                                  | `0x26<nested_type_encoding>`                                                                                                                                                                                                                                                                                           |
+| `Map(K, V)`                                                                          | `0x27<key_type_encoding><value_type_encoding>`                                                                                                                                                                                                                                                                         |
+| `IPv4`                                                                               | `0x28`                                                                                                                                                                                                                                                                                                                 |
+| `IPv6`                                                                               | `0x29`                                                                                                                                                                                                                                                                                                                 |
+| `Variant(T1, ..., TN)`                                                               | `0x2A<var_uint_number_of_variants><variant_type_encoding_1>...<variant_type_encoding_N>`                                                                                                                                                                                                                               |
+| `Dynamic(max_types=N)`                                                               | `0x2B<uint8_max_types>`                                                                                                                                                                                                                                                                                                |
+| `Custom type` (`Ring`, `Polygon`, etc)                                               | `0x2C<var_uint_type_name_size><type_name_data>`                                                                                                                                                                                                                                                                        |
+| `Bool`                                                                               | `0x2D`                                                                                                                                                                                                                                                                                                                 |
+| `SimpleAggregateFunction(function_name(param_1, ..., param_N), arg_T1, ..., arg_TN)` | `0x2E<var_uint_function_name_size><function_name_data><var_uint_number_of_parameters><param_1>...<param_N><var_uint_number_of_arguments><argument_type_encoding_1>...<argument_type_encoding_N>` (see [aggregate function parameter binary encoding](#aggregate-function-parameter-binary-encoding))                   |
+| `Nested(name1 T1, ..., nameN TN)`                                                    | `0x2F<var_uint_number_of_elements><var_uint_name_size_1><name_data_1><nested_type_encoding_1>...<var_uint_name_size_N><name_data_N><nested_type_encoding_N>`                                                                                                                                                           |
+
+
+### Interval kind binary encoding
+
+The table below describes how different interval kinds of `Interval` data type are encoded.
+
+| Interval kind | Binary encoding |
+|---------------|-----------------|
+| `Nanosecond`  | `0x00`          |
+| `Microsecond` | `0x01`          |
+| `Millisecond` | `0x02`          |
+| `Second`      | `0x03`          |
+| `Minute`      | `0x04`          |
+| `Hour`        | `0x05`          |
+| `Day`         | `0x06`          |
+| `Week`        | `0x07`          |
+| `Month`       | `0x08`          |
+| `Quarter`     | `0x09`          |
+| `Year`        | `0x1A`          |
+
+### Aggregate function parameter binary encoding
+
+The table below describes how parameters of `AggragateFunction` and `SimpleAggregateFunction` are encoded.
+The encoding of a parameter consists of 1 byte indicating the type of the parameter and the value itself.
+
+| Parameter type           | Binary encoding                                                                                                                |
+|--------------------------|--------------------------------------------------------------------------------------------------------------------------------|
+| `Null`                   | `0x00`                                                                                                                         |
+| `UInt64`                 | `0x01<var_uint_value>`                                                                                                         |
+| `Int64`                  | `0x02<var_int_value>`                                                                                                          |
+| `UInt128`                | `0x03<uint128_little_endian_value>`                                                                                            |
+| `Int128`                 | `0x04<int128_little_endian_value>`                                                                                             |
+| `UInt128`                | `0x05<uint128_little_endian_value>`                                                                                            |
+| `Int128`                 | `0x06<int128_little_endian_value>`                                                                                             |
+| `Float64`                | `0x07<float64_little_endian_value>`                                                                                            |
+| `Decimal32`              | `0x08<var_uint_scale><int32_little_endian_value>`                                                                              |
+| `Decimal64`              | `0x09<var_uint_scale><int64_little_endian_value>`                                                                              |
+| `Decimal128`             | `0x0A<var_uint_scale><int128_little_endian_value>`                                                                             |
+| `Decimal256`             | `0x0B<var_uint_scale><int256_little_endian_value>`                                                                             |
+| `String`                 | `0x0C<var_uint_size><data>`                                                                                                    |
+| `Array`                  | `0x0D<var_uint_size><value_encoding_1>...<value_encoding_N>`                                                                   |
+| `Tuple`                  | `0x0E<var_uint_size><value_encoding_1>...<value_encoding_N>`                                                                   |
+| `Map`                    | `0x0F<var_uint_size><key_encoding_1><value_encoding_1>...<key_endoding_N><value_encoding_N>`                                   |
+| `IPv4`                   | `0x10<uint32_little_endian_value>`                                                                                             |
+| `IPv6`                   | `0x11<uint128_little_endian_value>`                                                                                            |
+| `UUID`                   | `0x12<uuid_value>`                                                                                                             |
+| `Bool`                   | `0x13<bool_value>`                                                                                                             |
+| `Object`                 | `0x14<var_uint_size><var_uint_key_size_1><key_data_1><value_encoding_1>...<var_uint_key_size_N><key_data_N><value_encoding_N>` |
+| `AggregateFunctionState` | `0x15<var_uint_name_size><name_data><var_uint_data_size><data>`                                                                |
+| `Negative infinity`      | `0xFE`                                                                                                                         |
+| `Positive infinity`      | `0xFF`                                                                                                                         |

docs/en/sql-reference/data-types/dynamic.md

@@ -493,3 +493,14 @@ SELECT count(), dynamicType(d), _part FROM test GROUP BY _part, dynamicType(d) O
 ```
 
 As we can see, ClickHouse kept the most frequent types `UInt64` and `Array(UInt64)` and casted all other types to `String`.
+
+
+### Binary output format
+
+In [RowBinary](../../interfaces/formats.md#rowbinary-rowbinary) format values of `Dynamic` type are serialized in the following format:
+
+```text
+<binary_encoded_data_type><value_in_binary_format_according_to_the_data_type>
+```
+
+See the [data types binary encoding specification](../../sql-reference/data-types/data-types-binary-encoding.md)

docs/en/sql-reference/table-functions/file.md

@@ -130,7 +130,9 @@ SELECT * FROM file('user_files/archives/archive{1..2}.zip :: table.csv');
 
 ## Globs in path 
 
-Paths may use globbing. Files must match the whole path pattern, not only the suffix or prefix.
+Paths may use globbing. Files must match the whole path pattern, not only the suffix or prefix. There is one exception that if the path refers to an existing
+directory and does not use globs, a `*` will be implicitly added to the path so
+all the files in the directory are selected.
 
 - `*` — Represents arbitrarily many characters except `/` but including the empty string.
 - `?` — Represents an arbitrary single character.
@@ -163,6 +165,12 @@ An alternative path expression which achieves the same:
 SELECT count(*) FROM file('{some,another}_dir/*', 'TSV', 'name String, value UInt32');
 ```
 
+Query the total number of rows in `some_dir` using the implicit `*`:
+
+```sql
+SELECT count(*) FROM file('some_dir', 'TSV', 'name String, value UInt32');
+```
+
 :::note
 If your listing of files contains number ranges with leading zeros, use the construction with braces for each digit separately or use `?`.
 :::

programs/client/Client.cpp

@@ -1117,6 +1117,7 @@ void Client::processOptions(const OptionsDescription & options_description,
         if (!options["user"].defaulted())
             throw Exception(ErrorCodes::BAD_ARGUMENTS, "User and JWT flags can't be specified together");
         config().setString("jwt", options["jwt"].as<std::string>());
+        config().setString("user", "");
     }
     if (options.count("accept-invalid-certificate"))
     {

src/Analyzer/Passes/LogicalExpressionOptimizerPass.cpp

@@ -10,6 +10,7 @@
 #include <Analyzer/Utils.h>
 
 #include <DataTypes/DataTypeLowCardinality.h>
+#include <DataTypes/DataTypesNumber.h>
 
 namespace DB
 {
@@ -26,12 +27,103 @@ static constexpr std::array boolean_functions{
     "like"sv,     "notLike"sv,     "ilike"sv,  "notILike"sv,        "empty"sv,        "notEmpty"sv,        "not"sv,    "and"sv,
     "or"sv};
 
-static bool isBooleanFunction(const String & func_name)
+
+bool isBooleanFunction(const String & func_name)
 {
     return std::any_of(
         boolean_functions.begin(), boolean_functions.end(), [&](const auto boolean_func) { return func_name == boolean_func; });
 }
 
+bool isNodeFunction(const QueryTreeNodePtr & node, const String & func_name)
+{
+    if (const auto * function_node = node->as<FunctionNode>())
+        return function_node->getFunctionName() == func_name;
+    return false;
+}
+
+QueryTreeNodePtr getFunctionArgument(const QueryTreeNodePtr & node, size_t idx)
+{
+    if (const auto * function_node = node->as<FunctionNode>())
+    {
+        const auto & args = function_node->getArguments().getNodes();
+        if (idx < args.size())
+            return args[idx];
+    }
+    throw Exception(ErrorCodes::LOGICAL_ERROR, "Expected '{}' to be a function with at least {} arguments", node->formatASTForErrorMessage(), idx + 1);
+}
+
+QueryTreeNodePtr findEqualsFunction(const QueryTreeNodes & nodes)
+{
+    for (const auto & node : nodes)
+    {
+        const auto * function_node = node->as<FunctionNode>();
+        if (function_node && function_node->getFunctionName() == "equals" &&
+            function_node->getArguments().getNodes().size() == 2)
+        {
+            return node;
+        }
+    }
+    return nullptr;
+}
+
+bool isBooleanConstant(const QueryTreeNodePtr & node, bool expected_value)
+{
+    const auto * constant_node = node->as<ConstantNode>();
+    if (!constant_node || !constant_node->getResultType()->equals(DataTypeUInt8()))
+        return false;
+
+    UInt64 constant_value;
+    return (constant_node->getValue().tryGet<UInt64>(constant_value) && constant_value == expected_value);
+}
+
+/// Returns true if expression consists of only conjunctions of functions with the specified name or true constants
+bool isOnlyConjunctionOfFunctions(
+    const QueryTreeNodePtr & node,
+    const String & func_name,
+    const QueryTreeNodePtrWithHashSet & allowed_arguments)
+{
+    if (isBooleanConstant(node, true))
+        return true;
+
+    const auto * node_function = node->as<FunctionNode>();
+    if (!node_function)
+        return false;
+
+    if (node_function->getFunctionName() == func_name
+        && allowed_arguments.contains(node_function->getArgumentsNode()))
+        return true;
+
+    if (node_function->getFunctionName() == "and")
+    {
+        for (const auto & and_argument : node_function->getArguments().getNodes())
+        {
+            if (!isOnlyConjunctionOfFunctions(and_argument, func_name, allowed_arguments))
+                return false;
+        }
+        return true;
+    }
+    return false;
+}
+
+/// We can rewrite to a <=> b only if we are joining on a and b,
+/// because the function is not yet implemented for other cases.
+bool isTwoArgumentsFromDifferentSides(const FunctionNode & node_function, const JoinNode & join_node)
+{
+    const auto & argument_nodes = node_function.getArguments().getNodes();
+    if (argument_nodes.size() != 2)
+        return false;
+
+    auto first_src = getExpressionSource(argument_nodes[0]);
+    auto second_src = getExpressionSource(argument_nodes[1]);
+    if (!first_src || !second_src)
+        return false;
+
+    const auto & lhs_join = *join_node.getLeftTableExpression();
+    const auto & rhs_join = *join_node.getRightTableExpression();
+    return (first_src->isEqual(lhs_join) && second_src->isEqual(rhs_join)) ||
+           (first_src->isEqual(rhs_join) && second_src->isEqual(lhs_join));
+}
+
 /// Visitor that optimizes logical expressions _only_ in JOIN ON section
 class JoinOnLogicalExpressionOptimizerVisitor : public InDepthQueryTreeVisitorWithContext<JoinOnLogicalExpressionOptimizerVisitor>
 {
@@ -47,15 +139,16 @@ public:
     {
         auto * function_node = node->as<FunctionNode>();
 
-        if (!function_node)
-            return;
+        QueryTreeNodePtr new_node = nullptr;
+        if (function_node && function_node->getFunctionName() == "or")
+            new_node = tryOptimizeJoinOnNulls(function_node->getArguments().getNodes(), getContext());
+        else
+            new_node = tryOptimizeJoinOnNulls({node}, getContext());
 
-        if (function_node->getFunctionName() == "or")
+        if (new_node)
         {
-            bool is_argument_type_changed = tryOptimizeIsNotDistinctOrIsNull(node, getContext());
-            if (is_argument_type_changed)
-                need_rerun_resolve = true;
-            return;
+            need_rerun_resolve |= !new_node->getResultType()->equals(*node->getResultType());
+            node = new_node;
         }
     }
 
@@ -72,15 +165,11 @@ private:
     const JoinNode * join_node;
     bool need_rerun_resolve = false;
 
-    /// Returns true if type of some operand is changed and parent function needs to be re-resolved
-    bool tryOptimizeIsNotDistinctOrIsNull(QueryTreeNodePtr & node, const ContextPtr & context)
+    /// Returns optimized node or nullptr if nothing have been changed
+    QueryTreeNodePtr tryOptimizeJoinOnNulls(const QueryTreeNodes & nodes, const ContextPtr & context)
     {
-        auto & function_node = node->as<FunctionNode &>();
-        chassert(function_node.getFunctionName() == "or");
-
-
         QueryTreeNodes or_operands;
-        or_operands.reserve(function_node.getArguments().getNodes().size());
+        or_operands.reserve(nodes.size());
 
         /// Indices of `equals` or `isNotDistinctFrom` functions in the vector above
         std::vector<size_t> equals_functions_indices;
@@ -93,47 +182,73 @@ private:
           *     b => [(a IS NULL AND b IS NULL)]
           *     c => [(a IS NULL AND c IS NULL)]
           * }
-          * Then for each a <=> b we can find all operands that contains both a IS NULL and b IS NULL
+          * Then for each equality a = b we can check if we have operand (a IS NULL AND b IS NULL)
           */
         QueryTreeNodePtrWithHashMap<std::vector<size_t>> is_null_argument_to_indices;
 
-        for (const auto & argument : function_node.getArguments())
-        {
-            or_operands.push_back(argument);
+        bool is_anything_changed = false;
 
-            auto * argument_function = argument->as<FunctionNode>();
+        for (const auto & node : nodes)
+        {
+            if (isBooleanConstant(node, false))
+            {
+                /// Remove false constants from OR
+                is_anything_changed = true;
+                continue;
+            }
+
+            or_operands.push_back(node);
+            auto * argument_function = node->as<FunctionNode>();
             if (!argument_function)
                 continue;
 
             const auto & func_name = argument_function->getFunctionName();
             if (func_name == "equals" || func_name == "isNotDistinctFrom")
             {
-                const auto & argument_nodes = argument_function->getArguments().getNodes();
-                if (argument_nodes.size() != 2)
-                    continue;
-                /// We can rewrite to a <=> b only if we are joining on a and b,
-                /// because the function is not yet implemented for other cases.
-                auto first_src = getExpressionSource(argument_nodes[0]);
-                auto second_src = getExpressionSource(argument_nodes[1]);
-                if (!first_src || !second_src)
-                    continue;
-                const auto & lhs_join = *join_node->getLeftTableExpression();
-                const auto & rhs_join = *join_node->getRightTableExpression();
-                bool arguments_from_both_sides = (first_src->isEqual(lhs_join) && second_src->isEqual(rhs_join)) ||
-                                                 (first_src->isEqual(rhs_join) && second_src->isEqual(lhs_join));
-                if (!arguments_from_both_sides)
-                    continue;
-                equals_functions_indices.push_back(or_operands.size() - 1);
+                if (isTwoArgumentsFromDifferentSides(*argument_function, *join_node))
+                    equals_functions_indices.push_back(or_operands.size() - 1);
             }
             else if (func_name == "and")
             {
-                for (const auto & and_argument : argument_function->getArguments().getNodes())
+                const auto & and_arguments = argument_function->getArguments().getNodes();
+                bool all_are_is_null = and_arguments.size() == 2 && isNodeFunction(and_arguments[0], "isNull") && isNodeFunction(and_arguments[1], "isNull");
+                if (all_are_is_null)
                 {
-                    auto * and_argument_function = and_argument->as<FunctionNode>();
-                    if (and_argument_function && and_argument_function->getFunctionName() == "isNull")
+                    is_null_argument_to_indices[getFunctionArgument(and_arguments.front(), 0)].push_back(or_operands.size() - 1);
+                    is_null_argument_to_indices[getFunctionArgument(and_arguments.back(), 0)].push_back(or_operands.size() - 1);
+                }
+
+                /// Expression `a = b AND (a IS NOT NULL) AND true AND (b IS NOT NULL)` we can be replaced with `a = b`
+                /// Even though this expression are not equivalent (first is NULL on NULLs, while second is FALSE),
+                /// it is still correct since for JOIN ON condition NULL is treated as FALSE
+                if (const auto & equals_function = findEqualsFunction(and_arguments))
+                {
+                    const auto & equals_arguments = equals_function->as<FunctionNode>()->getArguments().getNodes();
+                    /// Expected isNotNull arguments
+                    QueryTreeNodePtrWithHashSet allowed_arguments;
+                    allowed_arguments.insert(QueryTreeNodePtrWithHash(std::make_shared<ListNode>(QueryTreeNodes{equals_arguments[0]})));
+                    allowed_arguments.insert(QueryTreeNodePtrWithHash(std::make_shared<ListNode>(QueryTreeNodes{equals_arguments[1]})));
+
+                    bool can_be_optimized = true;
+                    for (const auto & and_argument : and_arguments)
                     {
-                        const auto & is_null_argument = and_argument_function->getArguments().getNodes()[0];
-                        is_null_argument_to_indices[is_null_argument].push_back(or_operands.size() - 1);
+                        if (and_argument.get() == equals_function.get())
+                            continue;
+
+                        if (isOnlyConjunctionOfFunctions(and_argument, "isNotNull", allowed_arguments))
+                            continue;
+
+                        can_be_optimized = false;
+                        break;
+                    }
+
+                    if (can_be_optimized)
+                    {
+                        is_anything_changed = true;
+                        or_operands.pop_back();
+                        or_operands.push_back(equals_function);
+                        if (isTwoArgumentsFromDifferentSides(equals_function->as<FunctionNode &>(), *join_node))
+                            equals_functions_indices.push_back(or_operands.size() - 1);
                     }
                 }
             }
@@ -144,9 +259,9 @@ private:
 
         for (size_t equals_function_idx : equals_functions_indices)
         {
-            auto * equals_function = or_operands[equals_function_idx]->as<FunctionNode>();
+            const auto * equals_function = or_operands[equals_function_idx]->as<FunctionNode>();
 
-            /// For a <=> b we are looking for expressions containing both `a IS NULL` and `b IS NULL` combined with AND
+            /// For a = b we are looking for all expressions `a IS NULL AND b IS NULL`
             const auto & argument_nodes = equals_function->getArguments().getNodes();
             const auto & lhs_is_null_parents = is_null_argument_to_indices[argument_nodes[0]];
             const auto & rhs_is_null_parents = is_null_argument_to_indices[argument_nodes[1]];
@@ -161,60 +276,40 @@ private:
 
             for (size_t to_optimize_idx : operands_to_optimize)
             {
-                /// We are looking for operand `a IS NULL AND b IS NULL AND ...`
-                auto * operand_to_optimize = or_operands[to_optimize_idx]->as<FunctionNode>();
-
-                /// Remove `a IS NULL` and `b IS NULL` arguments from AND
-                QueryTreeNodes new_arguments;
-                for (const auto & and_argument : operand_to_optimize->getArguments().getNodes())
-                {
-                    bool to_eliminate = false;
-
-                    const auto * and_argument_function = and_argument->as<FunctionNode>();
-                    if (and_argument_function && and_argument_function->getFunctionName() == "isNull")
-                    {
-                        const auto & is_null_argument = and_argument_function->getArguments().getNodes()[0];
-                        to_eliminate = (is_null_argument->isEqual(*argument_nodes[0]) || is_null_argument->isEqual(*argument_nodes[1]));
-                    }
-
-                    if (to_eliminate)
-                        arguments_to_reresolve.insert(to_optimize_idx);
-                    else
-                        new_arguments.emplace_back(and_argument);
-                }
-                /// If less than two arguments left, we will remove or replace the whole AND below
-                operand_to_optimize->getArguments().getNodes() = std::move(new_arguments);
+                /// Remove `a IS NULL AND b IS NULL`
+                or_operands[to_optimize_idx] = nullptr;
+                is_anything_changed = true;
             }
         }
 
-        if (arguments_to_reresolve.empty())
+        if (arguments_to_reresolve.empty() && !is_anything_changed)
             /// Nothing have been changed
-            return false;
+            return nullptr;
 
         auto and_function_resolver = FunctionFactory::instance().get("and", context);
         auto strict_equals_function_resolver = FunctionFactory::instance().get("isNotDistinctFrom", context);
 
-        bool need_reresolve = false;
         QueryTreeNodes new_or_operands;
         for (size_t i = 0; i < or_operands.size(); ++i)
         {
             if (arguments_to_reresolve.contains(i))
             {
-                auto * function = or_operands[i]->as<FunctionNode>();
+                const auto * function = or_operands[i]->as<FunctionNode>();
                 if (function->getFunctionName() == "equals")
                 {
                     /// We should replace `a = b` with `a <=> b` because we removed checks for IS NULL
-                    need_reresolve |= function->getResultType()->isNullable();
-                    function->resolveAsFunction(strict_equals_function_resolver);
-                    new_or_operands.emplace_back(std::move(or_operands[i]));
+                    auto new_function = or_operands[i]->clone();
+                    new_function->as<FunctionNode>()->resolveAsFunction(strict_equals_function_resolver);
+                    new_or_operands.emplace_back(std::move(new_function));
                 }
                 else if (function->getFunctionName() == "and")
                 {
                     const auto & and_arguments = function->getArguments().getNodes();
                     if (and_arguments.size() > 1)
                     {
-                        function->resolveAsFunction(and_function_resolver);
-                        new_or_operands.emplace_back(std::move(or_operands[i]));
+                        auto new_function = or_operands[i]->clone();
+                        new_function->as<FunctionNode>()->resolveAsFunction(and_function_resolver);
+                        new_or_operands.emplace_back(std::move(new_function));
                     }
                     else if (and_arguments.size() == 1)
                     {
@@ -223,25 +318,26 @@ private:
                     }
                 }
                 else
-                    throw Exception(ErrorCodes::LOGICAL_ERROR, "Unexpected function name: '{}'", function->getFunctionName());
+                    throw Exception(ErrorCodes::LOGICAL_ERROR, "Unexpected function '{}'", function->getFunctionName());
             }
-            else
+            else if (or_operands[i])
             {
                 new_or_operands.emplace_back(std::move(or_operands[i]));
             }
         }
 
+        if (new_or_operands.empty())
+            return nullptr;
+
         if (new_or_operands.size() == 1)
-        {
-            node = std::move(new_or_operands[0]);
-            return need_reresolve;
-        }
+            return new_or_operands[0];
 
         /// Rebuild OR function
         auto or_function_resolver = FunctionFactory::instance().get("or", context);
-        function_node.getArguments().getNodes() = std::move(new_or_operands);
-        function_node.resolveAsFunction(or_function_resolver);
-        return need_reresolve;
+        auto function_node = std::make_shared<FunctionNode>("or");
+        function_node->getArguments().getNodes() = std::move(new_or_operands);
+        function_node->resolveAsFunction(or_function_resolver);
+        return function_node;
     }
 };
 

src/Analyzer/QueryTreeBuilder.cpp

@@ -940,6 +940,7 @@ QueryTreeNodePtr QueryTreeBuilder::buildJoinTree(const ASTPtr & tables_in_select
                 table_join.locality,
                 result_join_strictness,
                 result_join_kind);
+            join_node->setOriginalAST(table_element.table_join);
 
             /** Original AST is not set because it will contain only join part and does
               * not include left table expression.

src/Backups/BackupOperationInfo.h

@@ -3,6 +3,8 @@
 #include <Backups/BackupStatus.h>
 #include <Common/ProfileEvents.h>
 
+#include <exception>
+
 namespace DB
 {
 

src/Columns/ColumnDynamic.cpp

@@ -4,7 +4,9 @@
 #include <DataTypes/DataTypeFactory.h>
 #include <DataTypes/DataTypeVariant.h>
 #include <DataTypes/DataTypeString.h>
+#include <DataTypes/DataTypeNothing.h>
 #include <DataTypes/FieldToDataType.h>
+#include <DataTypes/DataTypesBinaryEncoding.h>
 #include <Common/Arena.h>
 #include <Common/SipHash.h>
 #include <Processors/Transforms/ColumnGathererTransform.h>
@@ -481,7 +483,7 @@ StringRef ColumnDynamic::serializeValueIntoArena(size_t n, DB::Arena & arena, co
     /// We cannot use Variant serialization here as it serializes discriminator + value,
     /// but Dynamic doesn't have fixed mapping discriminator <-> variant type
     /// as different Dynamic column can have different Variants.
-    /// Instead, we serialize null bit + variant type name (size + bytes) + value.
+    /// Instead, we serialize null bit + variant type in binary format (size + bytes) + value.
     const auto & variant_col = assert_cast<const ColumnVariant &>(*variant_column);
     auto discr = variant_col.globalDiscriminatorAt(n);
     StringRef res;
@@ -495,14 +497,15 @@ StringRef ColumnDynamic::serializeValueIntoArena(size_t n, DB::Arena & arena, co
         return res;
     }
 
-    const auto & variant_name = variant_info.variant_names[discr];
-    size_t variant_name_size = variant_name.size();
-    char * pos = arena.allocContinue(sizeof(UInt8) + sizeof(size_t) + variant_name.size(), begin);
+    const auto & variant_type = assert_cast<const DataTypeVariant &>(*variant_info.variant_type).getVariant(discr);
+    String variant_type_binary_data = encodeDataType(variant_type);
+    size_t variant_type_binary_data_size = variant_type_binary_data.size();
+    char * pos = arena.allocContinue(sizeof(UInt8) + sizeof(size_t) + variant_type_binary_data.size(), begin);
     memcpy(pos, &null_bit, sizeof(UInt8));
-    memcpy(pos + sizeof(UInt8), &variant_name_size, sizeof(size_t));
-    memcpy(pos + sizeof(UInt8) + sizeof(size_t), variant_name.data(), variant_name.size());
+    memcpy(pos + sizeof(UInt8), &variant_type_binary_data_size, sizeof(size_t));
+    memcpy(pos + sizeof(UInt8) + sizeof(size_t), variant_type_binary_data.data(), variant_type_binary_data.size());
     res.data = pos;
-    res.size = sizeof(UInt8) + sizeof(size_t) + variant_name.size();
+    res.size = sizeof(UInt8) + sizeof(size_t) + variant_type_binary_data.size();
 
     auto value_ref = variant_col.getVariantByGlobalDiscriminator(discr).serializeValueIntoArena(variant_col.offsetAt(n), arena, begin);
     res.data = value_ref.data - res.size;
@@ -521,13 +524,15 @@ const char * ColumnDynamic::deserializeAndInsertFromArena(const char * pos)
         return pos;
     }
 
-    /// Read variant type name.
-    const size_t variant_name_size = unalignedLoad<size_t>(pos);
-    pos += sizeof(variant_name_size);
-    String variant_name;
-    variant_name.resize(variant_name_size);
-    memcpy(variant_name.data(), pos, variant_name_size);
-    pos += variant_name_size;
+    /// Read variant type in binary format.
+    const size_t variant_type_binary_data_size = unalignedLoad<size_t>(pos);
+    pos += sizeof(variant_type_binary_data_size);
+    String variant_type_binary_data;
+    variant_type_binary_data.resize(variant_type_binary_data_size);
+    memcpy(variant_type_binary_data.data(), pos, variant_type_binary_data_size);
+    pos += variant_type_binary_data_size;
+    auto variant_type = decodeDataType(variant_type_binary_data);
+    auto variant_name = variant_type->getName();
     /// If we already have such variant, just deserialize it into corresponding variant column.
     auto it = variant_info.variant_name_to_discriminator.find(variant_name);
     if (it != variant_info.variant_name_to_discriminator.end())
@@ -537,7 +542,6 @@ const char * ColumnDynamic::deserializeAndInsertFromArena(const char * pos)
     }
 
     /// If we don't have such variant, add it.
-    auto variant_type = DataTypeFactory::instance().get(variant_name);
     if (likely(addNewVariant(variant_type)))
     {
         auto discr = variant_info.variant_name_to_discriminator[variant_name];
@@ -563,13 +567,13 @@ const char * ColumnDynamic::skipSerializedInArena(const char * pos) const
     if (null_bit)
         return pos;
 
-    const size_t variant_name_size = unalignedLoad<size_t>(pos);
-    pos += sizeof(variant_name_size);
-    String variant_name;
-    variant_name.resize(variant_name_size);
-    memcpy(variant_name.data(), pos, variant_name_size);
-    pos += variant_name_size;
-    auto tmp_variant_column = DataTypeFactory::instance().get(variant_name)->createColumn();
+    const size_t variant_type_binary_data_size = unalignedLoad<size_t>(pos);
+    pos += sizeof(variant_type_binary_data_size);
+    String variant_type_binary_data;
+    variant_type_binary_data.resize(variant_type_binary_data_size);
+    memcpy(variant_type_binary_data.data(), pos, variant_type_binary_data_size);
+    pos += variant_type_binary_data_size;
+    auto tmp_variant_column = decodeDataType(variant_type_binary_data)->createColumn();
     return tmp_variant_column->skipSerializedInArena(pos);
 }
 

src/Common/AtomicLogger.h

@@ -1,5 +1,7 @@
 #pragma once
 
+#include <atomic>
+#include <mutex>
 #include <memory>
 
 #include <base/types.h>

src/Common/ConcurrencyControl.h

@@ -1,5 +1,6 @@
 #pragma once
 
+#include <atomic>
 #include <list>
 #include <memory>
 #include <mutex>

src/Common/FieldBinaryEncoding.cpp

@@ -0,0 +1,389 @@
+#include <Common/FieldBinaryEncoding.h>
+#include <IO/WriteHelpers.h>
+#include <IO/ReadHelpers.h>
+
+
+namespace DB
+{
+
+namespace ErrorCodes
+{
+    extern const int UNSUPPORTED_METHOD;
+    extern const int INCORRECT_DATA;
+}
+
+namespace
+{
+
+enum class FieldBinaryTypeIndex: uint8_t
+{
+    Null = 0x00,
+    UInt64 = 0x01,
+    Int64 = 0x02,
+    UInt128 = 0x03,
+    Int128 = 0x04,
+    UInt256 = 0x05,
+    Int256 = 0x06,
+    Float64 = 0x07,
+    Decimal32 = 0x08,
+    Decimal64 = 0x09,
+    Decimal128 = 0x0A,
+    Decimal256 = 0x0B,
+    String = 0x0C,
+    Array = 0x0D,
+    Tuple = 0x0E,
+    Map = 0x0F,
+    IPv4 = 0x10,
+    IPv6 = 0x11,
+    UUID = 0x12,
+    Bool = 0x13,
+    Object = 0x14,
+    AggregateFunctionState = 0x15,
+
+    NegativeInfinity = 0xFE,
+    PositiveInfinity = 0xFF,
+};
+
+class FieldVisitorEncodeBinary
+{
+public:
+    void operator() (const Null & x, WriteBuffer & buf) const;
+    void operator() (const UInt64 & x, WriteBuffer & buf) const;
+    void operator() (const UInt128 & x, WriteBuffer & buf) const;
+    void operator() (const UInt256 & x, WriteBuffer & buf) const;
+    void operator() (const Int64 & x, WriteBuffer & buf) const;
+    void operator() (const Int128 & x, WriteBuffer & buf) const;
+    void operator() (const Int256 & x, WriteBuffer & buf) const;
+    void operator() (const UUID & x, WriteBuffer & buf) const;
+    void operator() (const IPv4 & x, WriteBuffer & buf) const;
+    void operator() (const IPv6 & x, WriteBuffer & buf) const;
+    void operator() (const Float64 & x, WriteBuffer & buf) const;
+    void operator() (const String & x, WriteBuffer & buf) const;
+    void operator() (const Array & x, WriteBuffer & buf) const;
+    void operator() (const Tuple & x, WriteBuffer & buf) const;
+    void operator() (const Map & x, WriteBuffer & buf) const;
+    void operator() (const Object & x, WriteBuffer & buf) const;
+    void operator() (const DecimalField<Decimal32> & x, WriteBuffer & buf) const;
+    void operator() (const DecimalField<Decimal64> & x, WriteBuffer & buf) const;
+    void operator() (const DecimalField<Decimal128> & x, WriteBuffer & buf) const;
+    void operator() (const DecimalField<Decimal256> & x, WriteBuffer & buf) const;
+    void operator() (const AggregateFunctionStateData & x, WriteBuffer & buf) const;
+    [[noreturn]] void operator() (const CustomType & x, WriteBuffer & buf) const;
+    void operator() (const bool & x, WriteBuffer & buf) const;
+};
+
+void FieldVisitorEncodeBinary::operator() (const Null & x, WriteBuffer & buf) const
+{
+    if (x.isNull())
+        writeBinary(UInt8(FieldBinaryTypeIndex::Null), buf);
+    else if (x.isPositiveInfinity())
+        writeBinary(UInt8(FieldBinaryTypeIndex::PositiveInfinity), buf);
+    else if (x.isNegativeInfinity())
+        writeBinary(UInt8(FieldBinaryTypeIndex::NegativeInfinity), buf);
+}
+
+void FieldVisitorEncodeBinary::operator() (const UInt64 & x, WriteBuffer & buf) const
+{
+    writeBinary(UInt8(FieldBinaryTypeIndex::UInt64), buf);
+    writeVarUInt(x, buf);
+}
+
+void FieldVisitorEncodeBinary::operator() (const Int64 & x, WriteBuffer & buf) const
+{
+    writeBinary(UInt8(FieldBinaryTypeIndex::Int64), buf);
+    writeVarInt(x, buf);
+}
+
+void FieldVisitorEncodeBinary::operator() (const Float64 & x, WriteBuffer & buf) const
+{
+    writeBinary(UInt8(FieldBinaryTypeIndex::Float64), buf);
+    writeBinaryLittleEndian(x, buf);
+}
+
+void FieldVisitorEncodeBinary::operator() (const String & x, WriteBuffer & buf) const
+{
+    writeBinary(UInt8(FieldBinaryTypeIndex::String), buf);
+    writeStringBinary(x, buf);
+}
+
+void FieldVisitorEncodeBinary::operator() (const UInt128 & x, WriteBuffer & buf) const
+{
+    writeBinary(UInt8(FieldBinaryTypeIndex::UInt128), buf);
+    writeBinaryLittleEndian(x, buf);
+}
+
+void FieldVisitorEncodeBinary::operator() (const Int128 & x, WriteBuffer & buf) const
+{
+    writeBinary(UInt8(FieldBinaryTypeIndex::Int128), buf);
+    writeBinaryLittleEndian(x, buf);
+}
+
+void FieldVisitorEncodeBinary::operator() (const UInt256 & x, WriteBuffer & buf) const
+{
+    writeBinary(UInt8(FieldBinaryTypeIndex::UInt256), buf);
+    writeBinaryLittleEndian(x, buf);
+}
+
+void FieldVisitorEncodeBinary::operator() (const Int256 & x, WriteBuffer & buf) const
+{
+    writeBinary(UInt8(FieldBinaryTypeIndex::Int256), buf);
+    writeBinaryLittleEndian(x, buf);
+}
+
+void FieldVisitorEncodeBinary::operator() (const UUID & x, WriteBuffer & buf) const
+{
+    writeBinary(UInt8(FieldBinaryTypeIndex::UUID), buf);
+    writeBinaryLittleEndian(x, buf);
+}
+
+void FieldVisitorEncodeBinary::operator() (const IPv4 & x, WriteBuffer & buf) const
+{
+    writeBinary(UInt8(FieldBinaryTypeIndex::IPv4), buf);
+    writeBinaryLittleEndian(x, buf);
+}
+
+void FieldVisitorEncodeBinary::operator() (const IPv6 & x, WriteBuffer & buf) const
+{
+    writeBinary(UInt8(FieldBinaryTypeIndex::IPv6), buf);
+    writeBinaryLittleEndian(x, buf);
+}
+
+void FieldVisitorEncodeBinary::operator() (const DecimalField<Decimal32> & x, WriteBuffer & buf) const
+{
+    writeBinary(UInt8(FieldBinaryTypeIndex::Decimal32), buf);
+    writeVarUInt(x.getScale(), buf);
+    writeBinaryLittleEndian(x.getValue(), buf);
+}
+
+void FieldVisitorEncodeBinary::operator() (const DecimalField<Decimal64> & x, WriteBuffer & buf) const
+{
+    writeBinary(UInt8(FieldBinaryTypeIndex::Decimal64), buf);
+    writeVarUInt(x.getScale(), buf);
+    writeBinaryLittleEndian(x.getValue(), buf);
+}
+
+void FieldVisitorEncodeBinary::operator() (const DecimalField<Decimal128> & x, WriteBuffer & buf) const
+{
+    writeBinary(UInt8(FieldBinaryTypeIndex::Decimal128), buf);
+    writeVarUInt(x.getScale(), buf);
+    writeBinaryLittleEndian(x.getValue(), buf);
+}
+
+void FieldVisitorEncodeBinary::operator() (const DecimalField<Decimal256> & x, WriteBuffer & buf) const
+{
+    writeBinary(UInt8(FieldBinaryTypeIndex::Decimal256), buf);
+    writeVarUInt(x.getScale(), buf);
+    writeBinaryLittleEndian(x.getValue(), buf);
+}
+
+void FieldVisitorEncodeBinary::operator() (const AggregateFunctionStateData & x, WriteBuffer & buf) const
+{
+    writeBinary(UInt8(FieldBinaryTypeIndex::AggregateFunctionState), buf);
+    writeStringBinary(x.name, buf);
+    writeStringBinary(x.data, buf);
+}
+
+void FieldVisitorEncodeBinary::operator() (const Array & x, WriteBuffer & buf) const
+{
+    writeBinary(UInt8(FieldBinaryTypeIndex::Array), buf);
+    size_t size = x.size();
+    writeVarUInt(size, buf);
+    for (size_t i = 0; i < size; ++i)
+        Field::dispatch([&buf] (const auto & value) { FieldVisitorEncodeBinary()(value, buf); }, x[i]);
+}
+
+void FieldVisitorEncodeBinary::operator() (const Tuple & x, WriteBuffer & buf) const
+{
+    writeBinary(UInt8(FieldBinaryTypeIndex::Tuple), buf);
+    size_t size = x.size();
+    writeVarUInt(size, buf);
+    for (size_t i = 0; i < size; ++i)
+        Field::dispatch([&buf] (const auto & value) { FieldVisitorEncodeBinary()(value, buf); }, x[i]);
+}
+
+void FieldVisitorEncodeBinary::operator() (const Map & x, WriteBuffer & buf) const
+{
+    writeBinary(UInt8(FieldBinaryTypeIndex::Map), buf);
+    size_t size = x.size();
+    writeVarUInt(size, buf);
+    for (size_t i = 0; i < size; ++i)
+    {
+        const Tuple & key_and_value = x[i].get<Tuple>();
+        Field::dispatch([&buf] (const auto & value) { FieldVisitorEncodeBinary()(value, buf); }, key_and_value[0]);
+        Field::dispatch([&buf] (const auto & value) { FieldVisitorEncodeBinary()(value, buf); }, key_and_value[1]);
+    }
+}
+
+void FieldVisitorEncodeBinary::operator() (const Object & x, WriteBuffer & buf) const
+{
+    writeBinary(UInt8(FieldBinaryTypeIndex::Object), buf);
+
+    size_t size = x.size();
+    writeVarUInt(size, buf);
+    for (const auto & [key, value] : x)
+    {
+        writeStringBinary(key, buf);
+        Field::dispatch([&buf] (const auto & val) { FieldVisitorEncodeBinary()(val, buf); }, value);
+    }
+}
+
+void FieldVisitorEncodeBinary::operator()(const bool & x, WriteBuffer & buf) const
+{
+    writeBinary(UInt8(FieldBinaryTypeIndex::Bool), buf);
+    writeBinary(static_cast<UInt8>(x), buf);
+}
+
+[[noreturn]] void FieldVisitorEncodeBinary::operator()(const CustomType &, WriteBuffer &) const
+{
+    /// TODO: Support binary encoding/decoding for custom types somehow.
+    throw Exception(ErrorCodes::UNSUPPORTED_METHOD, "Binary encoding of Field with custom type is not supported");
+}
+
+template <typename T>
+Field decodeBigInteger(ReadBuffer & buf)
+{
+    T value;
+    readBinaryLittleEndian(value, buf);
+    return value;
+}
+
+template <typename T>
+DecimalField<T> decodeDecimal(ReadBuffer & buf)
+{
+    UInt32 scale;
+    readVarUInt(scale, buf);
+    T value;
+    readBinaryLittleEndian(value, buf);
+    return DecimalField<T>(value, scale);
+}
+
+template <typename T>
+T decodeValueLittleEndian(ReadBuffer & buf)
+{
+    T value;
+    readBinaryLittleEndian(value, buf);
+    return value;
+}
+
+template <typename T>
+T decodeArrayLikeField(ReadBuffer & buf)
+{
+    size_t size;
+    readVarUInt(size, buf);
+    T value;
+    for (size_t i = 0; i != size; ++i)
+        value.push_back(decodeField(buf));
+    return value;
+}
+
+}
+void encodeField(const Field & x, WriteBuffer & buf)
+{
+    Field::dispatch([&buf] (const auto & val) { FieldVisitorEncodeBinary()(val, buf); }, x);
+}
+
+Field decodeField(ReadBuffer & buf)
+{
+    UInt8 type;
+    readBinary(type, buf);
+    switch (FieldBinaryTypeIndex(type))
+    {
+        case FieldBinaryTypeIndex::Null:
+            return Null();
+        case FieldBinaryTypeIndex::PositiveInfinity:
+            return POSITIVE_INFINITY;
+        case FieldBinaryTypeIndex::NegativeInfinity:
+            return NEGATIVE_INFINITY;
+        case FieldBinaryTypeIndex::Int64:
+        {
+            Int64 value;
+            readVarInt(value, buf);
+            return value;
+        }
+        case FieldBinaryTypeIndex::UInt64:
+        {
+            UInt64 value;
+            readVarUInt(value, buf);
+            return value;
+        }
+        case FieldBinaryTypeIndex::Int128:
+            return decodeBigInteger<Int128>(buf);
+        case FieldBinaryTypeIndex::UInt128:
+            return decodeBigInteger<UInt128>(buf);
+        case FieldBinaryTypeIndex::Int256:
+            return decodeBigInteger<Int256>(buf);
+        case FieldBinaryTypeIndex::UInt256:
+            return decodeBigInteger<UInt256>(buf);
+        case FieldBinaryTypeIndex::Float64:
+            return decodeValueLittleEndian<Float64>(buf);
+        case FieldBinaryTypeIndex::Decimal32:
+            return decodeDecimal<Decimal32>(buf);
+        case FieldBinaryTypeIndex::Decimal64:
+            return decodeDecimal<Decimal64>(buf);
+        case FieldBinaryTypeIndex::Decimal128:
+            return decodeDecimal<Decimal128>(buf);
+        case FieldBinaryTypeIndex::Decimal256:
+            return decodeDecimal<Decimal256>(buf);
+        case FieldBinaryTypeIndex::String:
+        {
+            String value;
+            readStringBinary(value, buf);
+            return value;
+        }
+        case FieldBinaryTypeIndex::UUID:
+            return decodeValueLittleEndian<UUID>(buf);
+        case FieldBinaryTypeIndex::IPv4:
+            return decodeValueLittleEndian<IPv4>(buf);
+        case FieldBinaryTypeIndex::IPv6:
+            return decodeValueLittleEndian<IPv6>(buf);
+        case FieldBinaryTypeIndex::Bool:
+        {
+            bool value;
+            readBinary(value, buf);
+            return value;
+        }
+        case FieldBinaryTypeIndex::Array:
+            return decodeArrayLikeField<Array>(buf);
+        case FieldBinaryTypeIndex::Tuple:
+            return decodeArrayLikeField<Tuple>(buf);
+        case FieldBinaryTypeIndex::Map:
+        {
+            size_t size;
+            readVarUInt(size, buf);
+            Map map;
+            for (size_t i = 0; i != size; ++i)
+            {
+                Tuple key_and_value;
+                key_and_value.push_back(decodeField(buf));
+                key_and_value.push_back(decodeField(buf));
+                map.push_back(key_and_value);
+            }
+            return map;
+        }
+        case FieldBinaryTypeIndex::Object:
+        {
+            size_t size;
+            readVarUInt(size, buf);
+            Object value;
+            for (size_t i = 0; i != size; ++i)
+            {
+                String name;
+                readStringBinary(name, buf);
+                value[name] = decodeField(buf);
+            }
+            return value;
+        }
+        case FieldBinaryTypeIndex::AggregateFunctionState:
+        {
+            String name;
+            readStringBinary(name, buf);
+            String data;
+            readStringBinary(data, buf);
+            return AggregateFunctionStateData{.name = name, .data = data};
+        }
+    }
+
+    throw Exception(ErrorCodes::INCORRECT_DATA, "Unknown Field type: {0:#04x}", UInt64(type));
+}
+
+}

src/Common/FieldBinaryEncoding.h

@@ -0,0 +1,43 @@
+#pragma once
+
+#include <Core/Field.h>
+
+namespace DB
+{
+
+/**
+Binary encoding for Fields:
+|--------------------------|--------------------------------------------------------------------------------------------------------------------------------|
+| Field type               | Binary encoding                                                                                                                |
+|--------------------------|--------------------------------------------------------------------------------------------------------------------------------|
+| `Null`                   | `0x00`                                                                                                                         |
+| `UInt64`                 | `0x01<var_uint_value>`                                                                                                         |
+| `Int64`                  | `0x02<var_int_value>`                                                                                                          |
+| `UInt128`                | `0x03<uint128_little_endian_value>`                                                                                            |
+| `Int128`                 | `0x04<int128_little_endian_value>`                                                                                             |
+| `UInt128`                | `0x05<uint128_little_endian_value>`                                                                                            |
+| `Int128`                 | `0x06<int128_little_endian_value>`                                                                                             |
+| `Float64`                | `0x07<float64_little_endian_value>`                                                                                            |
+| `Decimal32`              | `0x08<var_uint_scale><int32_little_endian_value>`                                                                              |
+| `Decimal64`              | `0x09<var_uint_scale><int64_little_endian_value>`                                                                              |
+| `Decimal128`             | `0x0A<var_uint_scale><int128_little_endian_value>`                                                                             |
+| `Decimal256`             | `0x0B<var_uint_scale><int256_little_endian_value>`                                                                             |
+| `String`                 | `0x0C<var_uint_size><data>`                                                                                                    |
+| `Array`                  | `0x0D<var_uint_size><value_encoding_1>...<value_encoding_N>`                                                                   |
+| `Tuple`                  | `0x0E<var_uint_size><value_encoding_1>...<value_encoding_N>`                                                                   |
+| `Map`                    | `0x0F<var_uint_size><key_encoding_1><value_encoding_1>...<key_endoding_N><value_encoding_N>`                                   |
+| `IPv4`                   | `0x10<uint32_little_endian_value>`                                                                                             |
+| `IPv6`                   | `0x11<uint128_little_endian_value>`                                                                                            |
+| `UUID`                   | `0x12<uuid_value>`                                                                                                             |
+| `Bool`                   | `0x13<bool_value>`                                                                                                             |
+| `Object`                 | `0x14<var_uint_size><var_uint_key_size_1><key_data_1><value_encoding_1>...<var_uint_key_size_N><key_data_N><value_encoding_N>` |
+| `AggregateFunctionState` | `0x15<var_uint_name_size><name_data><var_uint_data_size><data>`                                                                |
+| `Negative infinity`      | `0xFE`                                                                                                                         |
+| `Positive infinity`      | `0xFF`                                                                                                                         |
+|--------------------------|--------------------------------------------------------------------------------------------------------------------------------|
+*/
+
+void encodeField(const Field &, WriteBuffer & buf);
+Field decodeField(ReadBuffer & buf);
+
+}

src/Common/IntervalKind.h

@@ -7,19 +7,20 @@ namespace DB
 /// Kind of a temporal interval.
 struct IntervalKind
 {
+    /// note: The order and numbers are important and used in binary encoding, append new interval kinds to the end of list.
     enum class Kind : uint8_t
     {
-        Nanosecond,
-        Microsecond,
-        Millisecond,
-        Second,
-        Minute,
-        Hour,
-        Day,
-        Week,
-        Month,
-        Quarter,
-        Year,
+        Nanosecond = 0x00,
+        Microsecond = 0x01,
+        Millisecond = 0x02,
+        Second = 0x03,
+        Minute = 0x04,
+        Hour = 0x05,
+        Day = 0x06,
+        Week = 0x07,
+        Month = 0x08,
+        Quarter = 0x09,
+        Year = 0x0A,
     };
     Kind kind = Kind::Second;
 

src/Common/RemoteHostFilter.h

@@ -1,5 +1,6 @@
 #pragma once
 
+#include <atomic>
 #include <string>
 #include <vector>
 #include <mutex>

src/Coordination/Changelog.h

@@ -5,6 +5,7 @@
 #include <Common/ConcurrentBoundedQueue.h>
 
 #include <map>
+#include <variant>
 #include <unordered_map>
 #include <unordered_set>
 #include <future>

src/Coordination/FourLetterCommand.h

@@ -2,9 +2,11 @@
 
 #include "config.h"
 
+#include <atomic>
 #include <memory>
 #include <unordered_map>
 #include <string>
+#include <vector>
 #include <boost/noncopyable.hpp>
 
 namespace DB

src/Core/PostgreSQL/PoolWithFailover.cpp

@@ -27,7 +27,8 @@ PoolWithFailover::PoolWithFailover(
     size_t pool_size,
     size_t pool_wait_timeout_,
     size_t max_tries_,
-    bool auto_close_connection_)
+    bool auto_close_connection_,
+    size_t connection_attempt_timeout_)
     : pool_wait_timeout(pool_wait_timeout_)
     , max_tries(max_tries_)
     , auto_close_connection(auto_close_connection_)
@@ -39,8 +40,13 @@ PoolWithFailover::PoolWithFailover(
     {
         for (const auto & replica_configuration : configurations)
         {
-            auto connection_info = formatConnectionString(replica_configuration.database,
-                replica_configuration.host, replica_configuration.port, replica_configuration.username, replica_configuration.password);
+            auto connection_info = formatConnectionString(
+                replica_configuration.database,
+                replica_configuration.host,
+                replica_configuration.port,
+                replica_configuration.username,
+                replica_configuration.password,
+                connection_attempt_timeout_);
             replicas_with_priority[priority].emplace_back(connection_info, pool_size);
         }
     }
@@ -51,7 +57,8 @@ PoolWithFailover::PoolWithFailover(
     size_t pool_size,
     size_t pool_wait_timeout_,
     size_t max_tries_,
-    bool auto_close_connection_)
+    bool auto_close_connection_,
+    size_t connection_attempt_timeout_)
     : pool_wait_timeout(pool_wait_timeout_)
     , max_tries(max_tries_)
     , auto_close_connection(auto_close_connection_)
@@ -63,7 +70,13 @@ PoolWithFailover::PoolWithFailover(
     for (const auto & [host, port] : configuration.addresses)
     {
         LOG_DEBUG(getLogger("PostgreSQLPoolWithFailover"), "Adding address host: {}, port: {} to connection pool", host, port);
-        auto connection_string = formatConnectionString(configuration.database, host, port, configuration.username, configuration.password);
+        auto connection_string = formatConnectionString(
+            configuration.database,
+            host,
+            port,
+            configuration.username,
+            configuration.password,
+            connection_attempt_timeout_);
         replicas_with_priority[0].emplace_back(connection_string, pool_size);
     }
 }

src/Core/PostgreSQL/PoolWithFailover.h

@@ -14,7 +14,6 @@
 
 static constexpr inline auto POSTGRESQL_POOL_DEFAULT_SIZE = 16;
 static constexpr inline auto POSTGRESQL_POOL_WAIT_TIMEOUT = 5000;
-static constexpr inline auto POSTGRESQL_POOL_WITH_FAILOVER_DEFAULT_MAX_TRIES = 2;
 
 namespace postgres
 {
@@ -30,14 +29,16 @@ public:
         size_t pool_size,
         size_t pool_wait_timeout,
         size_t max_tries_,
-        bool auto_close_connection_);
+        bool auto_close_connection_,
+        size_t connection_attempt_timeout_);
 
     explicit PoolWithFailover(
         const DB::StoragePostgreSQL::Configuration & configuration,
         size_t pool_size,
         size_t pool_wait_timeout,
         size_t max_tries_,
-        bool auto_close_connection_);
+        bool auto_close_connection_,
+        size_t connection_attempt_timeout_);
 
     PoolWithFailover(const PoolWithFailover & other) = delete;
 

src/Core/PostgreSQL/Utils.cpp

@@ -8,7 +8,7 @@
 namespace postgres
 {
 
-ConnectionInfo formatConnectionString(String dbname, String host, UInt16 port, String user, String password)
+ConnectionInfo formatConnectionString(String dbname, String host, UInt16 port, String user, String password, UInt64 timeout)
 {
     DB::WriteBufferFromOwnString out;
     out << "dbname=" << DB::quote << dbname
@@ -16,7 +16,7 @@ ConnectionInfo formatConnectionString(String dbname, String host, UInt16 port, S
         << " port=" << port
         << " user=" << DB::quote << user
         << " password=" << DB::quote << password
-        << " connect_timeout=2";
+        << " connect_timeout=" << timeout;
     return {out.str(), host + ':' + DB::toString(port)};
 }
 

src/Core/PostgreSQL/Utils.h

@@ -18,7 +18,7 @@ namespace pqxx
 namespace postgres
 {
 
-ConnectionInfo formatConnectionString(String dbname, String host, UInt16 port, String user, String password);
+ConnectionInfo formatConnectionString(String dbname, String host, UInt16 port, String user, String password, UInt64 timeout);
 
 String getConnectionForLog(const String & host, UInt16 port);
 

src/Core/Protocol.h

@@ -63,7 +63,7 @@ const char USER_INTERSERVER_MARKER[] = " INTERSERVER SECRET ";
 /// Marker for SSH-keys-based authentication (passed as the user name)
 const char SSH_KEY_AUTHENTICAION_MARKER[] = " SSH KEY AUTHENTICATION ";
 
-/// Market for JSON Web Token authentication
+/// Marker for JSON Web Token authentication
 const char JWT_AUTHENTICAION_MARKER[] = " JWT AUTHENTICATION ";
 
 };

src/Core/Settings.h

@@ -561,7 +561,9 @@ class IColumn;
     M(UInt64, max_partition_size_to_drop, 50000000000lu, "Same as max_table_size_to_drop, but for the partitions.", 0) \
     \
     M(UInt64, postgresql_connection_pool_size, 16, "Connection pool size for PostgreSQL table engine and database engine.", 0) \
+    M(UInt64, postgresql_connection_attempt_timeout, 2, "Connection timeout to PostgreSQL table engine and database engine in seconds.", 0) \
     M(UInt64, postgresql_connection_pool_wait_timeout, 5000, "Connection pool push/pop timeout on empty pool for PostgreSQL table engine and database engine. By default it will block on empty pool.", 0) \
+    M(UInt64, postgresql_connection_pool_retries, 2, "Connection pool push/pop retries number for PostgreSQL table engine and database engine.", 0) \
     M(Bool, postgresql_connection_pool_auto_close_connection, false, "Close connection before returning connection to the pool.", 0) \
     M(UInt64, glob_expansion_max_elements, 1000, "Maximum number of allowed addresses (For external storages, table functions, etc).", 0) \
     M(UInt64, odbc_bridge_connection_pool_size, 16, "Connection pool size for each connection settings string in ODBC bridge.", 0) \
@@ -1132,6 +1134,8 @@ class IColumn;
     M(Bool, input_format_tsv_crlf_end_of_line, false, "If it is set true, file function will read TSV format with \\r\\n instead of \\n.", 0) \
     \
     M(Bool, input_format_native_allow_types_conversion, true, "Allow data types conversion in Native input format", 0) \
+    M(Bool, input_format_native_decode_types_in_binary_format, false, "Read data types in binary format instead of type names in Native input format", 0) \
+    M(Bool, output_format_native_encode_types_in_binary_format, false, "Write data types in binary format instead of type names in Native output format", 0) \
     \
     M(DateTimeInputFormat, date_time_input_format, FormatSettings::DateTimeInputFormat::Basic, "Method to read DateTime from text input formats. Possible values: 'basic', 'best_effort' and 'best_effort_us'.", 0) \
     M(DateTimeOutputFormat, date_time_output_format, FormatSettings::DateTimeOutputFormat::Simple, "Method to write DateTime to text output. Possible values: 'simple', 'iso', 'unix_timestamp'.", 0) \
@@ -1151,6 +1155,8 @@ class IColumn;
     M(Bool, input_format_avro_null_as_default, false, "For Avro/AvroConfluent format: insert default in case of null and non Nullable column", 0) \
     M(UInt64, format_binary_max_string_size, 1_GiB, "The maximum allowed size for String in RowBinary format. It prevents allocating large amount of memory in case of corrupted data. 0 means there is no limit", 0) \
     M(UInt64, format_binary_max_array_size, 1_GiB, "The maximum allowed size for Array in RowBinary format. It prevents allocating large amount of memory in case of corrupted data. 0 means there is no limit", 0) \
+    M(Bool, input_format_binary_decode_types_in_binary_format, false, "Read data types in binary format instead of type names in RowBinaryWithNamesAndTypes input format", 0) \
+    M(Bool, output_format_binary_encode_types_in_binary_format, false, "Write data types in binary format instead of type names in RowBinaryWithNamesAndTypes output format ", 0) \
     M(URI, format_avro_schema_registry_url, "", "For AvroConfluent format: Confluent Schema Registry URL.", 0) \
     \
     M(Bool, output_format_json_quote_64bit_integers, true, "Controls quoting of 64-bit integers in JSON output format.", 0) \
@@ -1171,9 +1177,9 @@ class IColumn;
     M(UInt64, output_format_pretty_max_value_width, 10000, "Maximum width of value to display in Pretty formats. If greater - it will be cut.", 0) \
     M(UInt64, output_format_pretty_max_value_width_apply_for_single_value, false, "Only cut values (see the `output_format_pretty_max_value_width` setting) when it is not a single value in a block. Otherwise output it entirely, which is useful for the `SHOW CREATE TABLE` query.", 0) \
     M(UInt64Auto, output_format_pretty_color, "auto", "Use ANSI escape sequences in Pretty formats. 0 - disabled, 1 - enabled, 'auto' - enabled if a terminal.", 0) \
-    M(String, output_format_pretty_grid_charset, "UTF-8", "Charset for printing grid borders. Available charsets: ASCII, UTF-8 (default one).", 0)                    \
-    M(UInt64, output_format_pretty_display_footer_column_names, true, "Display column names in the footer if there are 999 or more rows.", 0)                    \
-    M(UInt64, output_format_pretty_display_footer_column_names_min_rows, 50, "Sets the minimum threshold value of rows for which to enable displaying column names in the footer. 50 (default)", 0)                    \
+    M(String, output_format_pretty_grid_charset, "UTF-8", "Charset for printing grid borders. Available charsets: ASCII, UTF-8 (default one).", 0) \
+    M(UInt64, output_format_pretty_display_footer_column_names, true, "Display column names in the footer if there are 999 or more rows.", 0) \
+    M(UInt64, output_format_pretty_display_footer_column_names_min_rows, 50, "Sets the minimum threshold value of rows for which to enable displaying column names in the footer. 50 (default)", 0) \
     M(UInt64, output_format_parquet_row_group_size, 1000000, "Target row group size in rows.", 0) \
     M(UInt64, output_format_parquet_row_group_size_bytes, 512 * 1024 * 1024, "Target row group size in bytes, before compression.", 0) \
     M(Bool, output_format_parquet_string_as_string, true, "Use Parquet String type instead of Binary for String columns.", 0) \

src/Core/SettingsChangesHistory.cpp

@@ -58,6 +58,10 @@ String ClickHouseVersion::toString() const
 static std::initializer_list<std::pair<ClickHouseVersion, SettingsChangesHistory::SettingsChanges>> settings_changes_history_initializer =
 {
     {"24.7", {{"output_format_parquet_write_page_index", false, true, "Add a possibility to write page index into parquet files."},
+              {"output_format_binary_encode_types_in_binary_format", false, false, "Added new setting to allow to write type names in binary format in RowBinaryWithNamesAndTypes output format"},
+              {"input_format_binary_decode_types_in_binary_format", false, false, "Added new setting to allow to read type names in binary format in RowBinaryWithNamesAndTypes input format"},
+              {"output_format_native_encode_types_in_binary_format", false, false, "Added new setting to allow to write type names in binary format in Native output format"},
+              {"input_format_native_decode_types_in_binary_format", false, false, "Added new setting to allow to read type names in binary format in Native output format"},
               {"read_in_order_use_buffering", false, true, "Use buffering before merging while reading in order of primary key"},
               {"optimize_functions_to_subcolumns", false, true, "Enable optimization by default"},
               {"enable_named_columns_in_function_tuple", false, true, "Generate named tuples in function tuple() when all names are unique and can be treated as unquoted identifiers."},
@@ -70,6 +74,8 @@ static std::initializer_list<std::pair<ClickHouseVersion, SettingsChangesHistory
               {"azure_sdk_max_retries", 10, 10, "Maximum number of retries in azure sdk"},
               {"azure_sdk_retry_initial_backoff_ms", 10, 10, "Minimal backoff between retries in azure sdk"},
               {"azure_sdk_retry_max_backoff_ms", 1000, 1000, "Maximal backoff between retries in azure sdk"},
+              {"postgresql_connection_attempt_timeout", 2, 2, "Allow to control 'connect_timeout' parameter of PostgreSQL connection."},
+              {"postgresql_connection_pool_retries", 2, 2, "Allow to control the number of retries in PostgreSQL connection pool."}
               }},
     {"24.6", {{"materialize_skip_indexes_on_insert", true, true, "Added new setting to allow to disable materialization of skip indexes on insert"},
               {"materialize_statistics_on_insert", true, true, "Added new setting to allow to disable materialization of statistics on insert"},

src/Core/tests/gtest_fields_binary_enciding.cpp

@@ -0,0 +1,65 @@
+#include <gtest/gtest.h>
+#include <Common/FieldBinaryEncoding.h>
+#include <IO/WriteBufferFromString.h>
+#include <IO/ReadBufferFromString.h>
+
+using namespace DB;
+
+namespace DB::ErrorCodes
+{
+    extern const int UNSUPPORTED_METHOD;
+}
+
+
+void check(const Field & field)
+{
+//    std::cerr << "Check " << toString(field) << "\n";
+    WriteBufferFromOwnString ostr;
+    encodeField(field, ostr);
+    ReadBufferFromString istr(ostr.str());
+    Field decoded_field = decodeField(istr);
+    ASSERT_TRUE(istr.eof());
+    ASSERT_EQ(field, decoded_field);
+}
+
+GTEST_TEST(FieldBinaryEncoding, EncodeAndDecode)
+{
+    check(Null());
+    check(POSITIVE_INFINITY);
+    check(NEGATIVE_INFINITY);
+    check(true);
+    check(UInt64(42));
+    check(Int64(-42));
+    check(UInt128(42));
+    check(Int128(-42));
+    check(UInt256(42));
+    check(Int256(-42));
+    check(UUID(42));
+    check(IPv4(42));
+    check(IPv6(42));
+    check(Float64(42.42));
+    check(String("Hello, World!"));
+    check(Array({Field(UInt64(42)), Field(UInt64(43))}));
+    check(Tuple({Field(UInt64(42)), Field(Null()), Field(UUID(42)), Field(String("Hello, World!"))}));
+    check(Map({Tuple{Field(UInt64(42)), Field(String("str_42"))}, Tuple{Field(UInt64(43)), Field(String("str_43"))}}));
+    check(Object({{String("key_1"), Field(UInt64(42))}, {String("key_2"), Field(UInt64(43))}}));
+    check(DecimalField<Decimal32>(4242, 3));
+    check(DecimalField<Decimal64>(4242, 3));
+    check(DecimalField<Decimal128>(Int128(4242), 3));
+    check(DecimalField<Decimal256>(Int256(4242), 3));
+    check(AggregateFunctionStateData{.name="some_name", .data="some_data"});
+    try
+    {
+        check(CustomType());
+    }
+    catch (const Exception & e)
+    {
+        ASSERT_EQ(e.code(), ErrorCodes::UNSUPPORTED_METHOD);
+    }
+
+    check(Array({
+        Tuple({Field(UInt64(42)), Map({Tuple{Field(UInt64(42)), Field(String("str_42"))}, Tuple{Field(UInt64(43)), Field(String("str_43"))}}), Field(UUID(42)), Field(String("Hello, World!"))}),
+        Tuple({Field(UInt64(43)), Map({Tuple{Field(UInt64(43)), Field(String("str_43"))}, Tuple{Field(UInt64(44)), Field(String("str_44"))}}), Field(UUID(43)), Field(String("Hello, World 2!"))})
+    }));
+}
+

src/DataTypes/DataTypeAggregateFunction.h

@@ -25,7 +25,6 @@ private:
     mutable std::optional<size_t> version;
 
     String getNameImpl(bool with_version) const;
-    size_t getVersion() const;
 
 public:
     static constexpr bool is_parametric = true;
@@ -39,6 +38,8 @@ public:
     {
     }
 
+    size_t getVersion() const;
+
     String getFunctionName() const;
     AggregateFunctionPtr getFunction() const { return function; }
 

src/DataTypes/DataTypeCustomSimpleAggregateFunction.cpp

@@ -165,6 +165,19 @@ static std::pair<DataTypePtr, DataTypeCustomDescPtr> create(const ASTPtr & argum
     return std::make_pair(storage_type, std::make_unique<DataTypeCustomDesc>(std::move(custom_name), nullptr));
 }
 
+String DataTypeCustomSimpleAggregateFunction::getFunctionName() const
+{
+    return function->getName();
+}
+
+DataTypePtr createSimpleAggregateFunctionType(const AggregateFunctionPtr & function, const DataTypes & argument_types, const Array & parameters)
+{
+    auto custom_desc = std::make_unique<DataTypeCustomDesc>(
+        std::make_unique<DataTypeCustomSimpleAggregateFunction>(function, argument_types, parameters));
+
+    return DataTypeFactory::instance().getCustom(std::move(custom_desc));
+}
+
 void registerDataTypeDomainSimpleAggregateFunction(DataTypeFactory & factory)
 {
     factory.registerDataTypeCustom("SimpleAggregateFunction", create);

src/DataTypes/DataTypeCustomSimpleAggregateFunction.h

@@ -40,8 +40,13 @@ public:
             : function(function_), argument_types(argument_types_), parameters(parameters_) {}
 
     AggregateFunctionPtr getFunction() const { return function; }
+    String getFunctionName() const;
+    const DataTypes & getArgumentsDataTypes() const { return argument_types; }
+    const Array & getParameters() const { return parameters; }
     String getName() const override;
     static void checkSupportedFunctions(const AggregateFunctionPtr & function);
 };
 
+DataTypePtr createSimpleAggregateFunctionType(const AggregateFunctionPtr & function, const DataTypes & argument_types, const Array & parameters);
+
 }

src/DataTypes/DataTypeDynamic.cpp

@@ -71,7 +71,7 @@ static DataTypePtr create(const ASTPtr & arguments)
 
     auto * literal = argument->arguments->children[1]->as<ASTLiteral>();
 
-    if (!literal || literal->value.getType() != Field::Types::UInt64 || literal->value.get<UInt64>() == 0 || literal->value.get<UInt64>() > 255)
+    if (!literal || literal->value.getType() != Field::Types::UInt64 || literal->value.get<UInt64>() == 0 || literal->value.get<UInt64>() > ColumnVariant::MAX_NESTED_COLUMNS)
         throw Exception(ErrorCodes::UNEXPECTED_AST_STRUCTURE, "'max_types' argument for Dynamic type should be a positive integer between 1 and 255");
 
     return std::make_shared<DataTypeDynamic>(literal->value.get<UInt64>());

src/DataTypes/DataTypeNested.h

@@ -19,6 +19,8 @@ public:
     }
 
     String getName() const override;
+    const DataTypes & getElements() const { return elems; }
+    const Names & getNames() const { return names; }
 };
 
 DataTypePtr createNested(const DataTypes & types, const Names & names);

src/DataTypes/DataTypesBinaryEncoding.cpp

@@ -0,0 +1,705 @@
+#include <DataTypes/DataTypesBinaryEncoding.h>
+#include <DataTypes/DataTypeDateTime64.h>
+#include <DataTypes/DataTypeFixedString.h>
+#include <DataTypes/DataTypeEnum.h>
+#include <DataTypes/DataTypesDecimal.h>
+#include <DataTypes/DataTypeArray.h>
+#include <DataTypes/DataTypeTuple.h>
+#include <DataTypes/DataTypeNullable.h>
+#include <DataTypes/DataTypeFunction.h>
+#include <DataTypes/DataTypeLowCardinality.h>
+#include <DataTypes/DataTypeMap.h>
+#include <DataTypes/DataTypeVariant.h>
+#include <DataTypes/DataTypeString.h>
+#include <DataTypes/DataTypeUUID.h>
+#include <DataTypes/DataTypeSet.h>
+#include <DataTypes/DataTypeInterval.h>
+#include <DataTypes/DataTypeIPv4andIPv6.h>
+#include <DataTypes/DataTypeAggregateFunction.h>
+#include <DataTypes/DataTypeCustomSimpleAggregateFunction.h>
+#include <DataTypes/DataTypeNothing.h>
+#include <DataTypes/DataTypeDynamic.h>
+#include <DataTypes/DataTypeNested.h>
+#include <DataTypes/DataTypeFactory.h>
+#include <AggregateFunctions/IAggregateFunction.h>
+#include <AggregateFunctions/AggregateFunctionFactory.h>
+#include <Parsers/NullsAction.h>
+#include <IO/WriteBuffer.h>
+#include <IO/ReadBuffer.h>
+#include <IO/ReadBufferFromString.h>
+#include <IO/WriteHelpers.h>
+#include <IO/ReadHelpers.h>
+#include <Common/FieldBinaryEncoding.h>
+#include <Common/assert_cast.h>
+
+namespace DB
+{
+
+namespace ErrorCodes
+{
+    extern const int UNSUPPORTED_METHOD;
+    extern const int INCORRECT_DATA;
+}
+
+namespace
+{
+
+enum class BinaryTypeIndex : uint8_t
+{
+    Nothing = 0x00,
+    UInt8 = 0x01,
+    UInt16 = 0x02,
+    UInt32 = 0x03,
+    UInt64 = 0x04,
+    UInt128 = 0x05,
+    UInt256 = 0x06,
+    Int8 = 0x07,
+    Int16 = 0x08,
+    Int32 = 0x09,
+    Int64 = 0x0A,
+    Int128 = 0x0B,
+    Int256 = 0x0C,
+    Float32 = 0x0D,
+    Float64 = 0x0E,
+    Date = 0x0F,
+    Date32 = 0x10,
+    DateTimeUTC = 0x11,
+    DateTimeWithTimezone = 0x12,
+    DateTime64UTC = 0x13,
+    DateTime64WithTimezone = 0x14,
+    String = 0x15,
+    FixedString = 0x16,
+    Enum8 = 0x17,
+    Enum16 = 0x18,
+    Decimal32 = 0x19,
+    Decimal64 = 0x1A,
+    Decimal128 = 0x1B,
+    Decimal256 = 0x1C,
+    UUID = 0x1D,
+    Array = 0x1E,
+    UnnamedTuple = 0x1F,
+    NamedTuple = 0x20,
+    Set = 0x21,
+    Interval = 0x22,
+    Nullable = 0x23,
+    Function = 0x24,
+    AggregateFunction = 0x25,
+    LowCardinality = 0x26,
+    Map = 0x27,
+    IPv4 = 0x28,
+    IPv6 = 0x29,
+    Variant = 0x2A,
+    Dynamic = 0x2B,
+    Custom = 0x2C,
+    Bool = 0x2D,
+    SimpleAggregateFunction = 0x2E,
+    Nested = 0x2F,
+};
+
+BinaryTypeIndex getBinaryTypeIndex(const DataTypePtr & type)
+{
+    /// By default custom types don't have their own BinaryTypeIndex.
+    if (type->hasCustomName())
+    {
+        /// Some widely used custom types have separate BinaryTypeIndex for better serialization.
+        /// Right now it's Bool, SimpleAggregateFunction and Nested types.
+        /// TODO: Consider adding BinaryTypeIndex for more custom types.
+
+        if (isBool(type))
+            return BinaryTypeIndex::Bool;
+
+        if (typeid_cast<const DataTypeCustomSimpleAggregateFunction *>(type->getCustomName()))
+            return BinaryTypeIndex::SimpleAggregateFunction;
+
+        if (isNested(type))
+            return BinaryTypeIndex::Nested;
+
+        return BinaryTypeIndex::Custom;
+    }
+
+    switch (type->getTypeId())
+    {
+        case TypeIndex::Nothing:
+            return BinaryTypeIndex::Nothing;
+        case TypeIndex::UInt8:
+            return BinaryTypeIndex::UInt8;
+        case TypeIndex::UInt16:
+            return BinaryTypeIndex::UInt16;
+        case TypeIndex::UInt32:
+            return BinaryTypeIndex::UInt32;
+        case TypeIndex::UInt64:
+            return BinaryTypeIndex::UInt64;
+        case TypeIndex::UInt128:
+            return BinaryTypeIndex::UInt128;
+        case TypeIndex::UInt256:
+            return BinaryTypeIndex::UInt256;
+        case TypeIndex::Int8:
+            return BinaryTypeIndex::Int8;
+        case TypeIndex::Int16:
+            return BinaryTypeIndex::Int16;
+        case TypeIndex::Int32:
+            return BinaryTypeIndex::Int32;
+        case TypeIndex::Int64:
+            return BinaryTypeIndex::Int64;
+        case TypeIndex::Int128:
+            return BinaryTypeIndex::Int128;
+        case TypeIndex::Int256:
+            return BinaryTypeIndex::Int256;
+        case TypeIndex::Float32:
+            return BinaryTypeIndex::Float32;
+        case TypeIndex::Float64:
+            return BinaryTypeIndex::Float64;
+        case TypeIndex::Date:
+            return BinaryTypeIndex::Date;
+        case TypeIndex::Date32:
+            return BinaryTypeIndex::Date32;
+        case TypeIndex::DateTime:
+            if (assert_cast<const DataTypeDateTime &>(*type).hasExplicitTimeZone())
+                return BinaryTypeIndex::DateTimeWithTimezone;
+            return BinaryTypeIndex::DateTimeUTC;
+        case TypeIndex::DateTime64:
+            if (assert_cast<const DataTypeDateTime64 &>(*type).hasExplicitTimeZone())
+                return BinaryTypeIndex::DateTime64WithTimezone;
+            return BinaryTypeIndex::DateTime64UTC;
+        case TypeIndex::String:
+            return BinaryTypeIndex::String;
+        case TypeIndex::FixedString:
+            return BinaryTypeIndex::FixedString;
+        case TypeIndex::Enum8:
+            return BinaryTypeIndex::Enum8;
+        case TypeIndex::Enum16:
+            return BinaryTypeIndex::Enum16;
+        case TypeIndex::Decimal32:
+            return BinaryTypeIndex::Decimal32;
+        case TypeIndex::Decimal64:
+            return BinaryTypeIndex::Decimal64;
+        case TypeIndex::Decimal128:
+            return BinaryTypeIndex::Decimal128;
+        case TypeIndex::Decimal256:
+            return BinaryTypeIndex::Decimal256;
+        case TypeIndex::UUID:
+            return BinaryTypeIndex::UUID;
+        case TypeIndex::Array:
+            return BinaryTypeIndex::Array;
+        case TypeIndex::Tuple:
+        {
+            const auto & tuple_type = assert_cast<const DataTypeTuple &>(*type);
+            if (tuple_type.haveExplicitNames())
+                return BinaryTypeIndex::NamedTuple;
+            return BinaryTypeIndex::UnnamedTuple;
+        }
+        case TypeIndex::Set:
+            return BinaryTypeIndex::Set;
+        case TypeIndex::Interval:
+            return BinaryTypeIndex::Interval;
+        case TypeIndex::Nullable:
+            return BinaryTypeIndex::Nullable;
+        case TypeIndex::Function:
+            return BinaryTypeIndex::Function;
+        case TypeIndex::AggregateFunction:
+            return BinaryTypeIndex::AggregateFunction;
+        case TypeIndex::LowCardinality:
+            return BinaryTypeIndex::LowCardinality;
+        case TypeIndex::Map:
+            return BinaryTypeIndex::Map;
+        case TypeIndex::Object:
+            /// Object type will be deprecated and replaced by new implementation. No need to support it here.
+            throw Exception(ErrorCodes::UNSUPPORTED_METHOD, "Binary encoding of type Object is not supported");
+        case TypeIndex::IPv4:
+            return BinaryTypeIndex::IPv4;
+        case TypeIndex::IPv6:
+            return BinaryTypeIndex::IPv6;
+        case TypeIndex::Variant:
+            return BinaryTypeIndex::Variant;
+        case TypeIndex::Dynamic:
+            return BinaryTypeIndex::Dynamic;
+        /// JSONPaths is used only during schema inference and cannot be used anywhere else.
+        case TypeIndex::JSONPaths:
+            throw Exception(ErrorCodes::UNSUPPORTED_METHOD, "Binary encoding of type JSONPaths is not supported");
+    }
+}
+
+template <typename T>
+void encodeEnumValues(const DataTypePtr & type, WriteBuffer & buf)
+{
+    const auto & enum_type = assert_cast<const DataTypeEnum<T> &>(*type);
+    const auto & values = enum_type.getValues();
+    writeVarUInt(values.size(), buf);
+    for (const auto & [name, value] : values)
+    {
+        writeStringBinary(name, buf);
+        writeBinaryLittleEndian(value, buf);
+    }
+}
+
+template <typename T>
+DataTypePtr decodeEnum(ReadBuffer & buf)
+{
+    typename DataTypeEnum<T>::Values values;
+    size_t size;
+    readVarUInt(size, buf);
+    for (size_t i = 0; i != size; ++i)
+    {
+        String name;
+        readStringBinary(name, buf);
+        T value;
+        readBinaryLittleEndian(value, buf);
+        values.emplace_back(name, value);
+    }
+
+    return std::make_shared<DataTypeEnum<T>>(values);
+}
+
+template <typename T>
+void encodeDecimal(const DataTypePtr & type, WriteBuffer & buf)
+{
+    const auto & decimal_type = assert_cast<const DataTypeDecimal<T> &>(*type);
+    /// Both precision and scale should be less than 76, so we can decode it in 1 byte.
+    writeBinary(UInt8(decimal_type.getPrecision()), buf);
+    writeBinary(UInt8(decimal_type.getScale()), buf);
+}
+
+template <typename T>
+DataTypePtr decodeDecimal(ReadBuffer & buf)
+{
+    UInt8 precision;
+    readBinary(precision, buf);
+    UInt8 scale;
+    readBinary(scale, buf);
+    return std::make_shared<DataTypeDecimal<T>>(precision, scale);
+}
+
+void encodeAggregateFunction(const String & function_name, const Array & parameters, const DataTypes & arguments_types, WriteBuffer & buf)
+{
+    writeStringBinary(function_name, buf);
+    writeVarUInt(parameters.size(), buf);
+    for (const auto & param : parameters)
+        encodeField(param, buf);
+    writeVarUInt(arguments_types.size(), buf);
+    for (const auto & argument_type : arguments_types)
+        encodeDataType(argument_type, buf);
+}
+
+std::tuple<AggregateFunctionPtr, Array, DataTypes> decodeAggregateFunction(ReadBuffer & buf)
+{
+    String function_name;
+    readStringBinary(function_name, buf);
+    size_t num_parameters;
+    readVarUInt(num_parameters, buf);
+    Array parameters;
+    parameters.reserve(num_parameters);
+    for (size_t i = 0; i != num_parameters; ++i)
+        parameters.push_back(decodeField(buf));
+    size_t num_arguments;
+    readVarUInt(num_arguments, buf);
+    DataTypes arguments_types;
+    arguments_types.reserve(num_arguments);
+    for (size_t i = 0; i != num_arguments; ++i)
+        arguments_types.push_back(decodeDataType(buf));
+    AggregateFunctionProperties properties;
+    auto action = NullsAction::EMPTY;
+    auto function = AggregateFunctionFactory::instance().get(function_name, action, arguments_types, parameters, properties);
+    return {function, parameters, arguments_types};
+}
+
+}
+
+void encodeDataType(const DataTypePtr & type, WriteBuffer & buf)
+{
+    /// First, write the BinaryTypeIndex byte.
+    auto binary_type_index = getBinaryTypeIndex(type);
+    buf.write(UInt8(binary_type_index));
+    /// Then, write additional information depending on the data type.
+    switch (binary_type_index)
+    {
+        case BinaryTypeIndex::DateTimeWithTimezone:
+        {
+            const auto & datetime_type = assert_cast<const DataTypeDateTime &>(*type);
+            writeStringBinary(datetime_type.getTimeZone().getTimeZone(), buf);
+            break;
+        }
+        case BinaryTypeIndex::DateTime64UTC:
+        {
+            const auto & datetime64_type = assert_cast<const DataTypeDateTime64 &>(*type);
+            /// Maximum scale for DateTime64 is 9, so we can write it as 1 byte.
+            buf.write(UInt8(datetime64_type.getScale()));
+            break;
+        }
+        case BinaryTypeIndex::DateTime64WithTimezone:
+        {
+            const auto & datetime64_type = assert_cast<const DataTypeDateTime64 &>(*type);
+            buf.write(UInt8(datetime64_type.getScale()));
+            writeStringBinary(datetime64_type.getTimeZone().getTimeZone(), buf);
+            break;
+        }
+        case BinaryTypeIndex::FixedString:
+        {
+            const auto & fixed_string_type = assert_cast<const DataTypeFixedString &>(*type);
+            writeVarUInt(fixed_string_type.getN(), buf);
+            break;
+        }
+        case BinaryTypeIndex::Enum8:
+        {
+            encodeEnumValues<Int8>(type, buf);
+            break;
+        }
+        case BinaryTypeIndex::Enum16:
+        {
+            encodeEnumValues<Int16>(type, buf);
+            break;
+        }
+        case BinaryTypeIndex::Decimal32:
+        {
+            encodeDecimal<Decimal32>(type, buf);
+            break;
+        }
+        case BinaryTypeIndex::Decimal64:
+        {
+            encodeDecimal<Decimal64>(type, buf);
+            break;
+        }
+        case BinaryTypeIndex::Decimal128:
+        {
+            encodeDecimal<Decimal128>(type, buf);
+            break;
+        }
+        case BinaryTypeIndex::Decimal256:
+        {
+            encodeDecimal<Decimal256>(type, buf);
+            break;
+        }
+        case BinaryTypeIndex::Array:
+        {
+            const auto & array_type = assert_cast<const DataTypeArray &>(*type);
+            encodeDataType(array_type.getNestedType(), buf);
+            break;
+        }
+        case BinaryTypeIndex::NamedTuple:
+        {
+            const auto & tuple_type = assert_cast<const DataTypeTuple &>(*type);
+            const auto & types = tuple_type.getElements();
+            const auto & names = tuple_type.getElementNames();
+            writeVarUInt(types.size(), buf);
+            for (size_t i = 0; i != types.size(); ++i)
+            {
+                writeStringBinary(names[i], buf);
+                encodeDataType(types[i], buf);
+            }
+            break;
+        }
+        case BinaryTypeIndex::UnnamedTuple:
+        {
+            const auto & tuple_type = assert_cast<const DataTypeTuple &>(*type);
+            const auto & element_types = tuple_type.getElements();
+            writeVarUInt(element_types.size(), buf);
+            for (const auto & element_type : element_types)
+                encodeDataType(element_type, buf);
+            break;
+        }
+        case BinaryTypeIndex::Interval:
+        {
+            const auto & interval_type = assert_cast<const DataTypeInterval &>(*type);
+            writeBinary(UInt8(interval_type.getKind().kind), buf);
+            break;
+        }
+        case BinaryTypeIndex::Nullable:
+        {
+            const auto & nullable_type = assert_cast<const DataTypeNullable &>(*type);
+            encodeDataType(nullable_type.getNestedType(), buf);
+            break;
+        }
+        case BinaryTypeIndex::Function:
+        {
+            const auto & function_type = assert_cast<const DataTypeFunction &>(*type);
+            const auto & arguments_types = function_type.getArgumentTypes();
+            const auto & return_type = function_type.getReturnType();
+            writeVarUInt(arguments_types.size(), buf);
+            for (const auto & argument_type : arguments_types)
+                encodeDataType(argument_type, buf);
+            encodeDataType(return_type, buf);
+            break;
+        }
+        case BinaryTypeIndex::LowCardinality:
+        {
+            const auto & low_cardinality_type = assert_cast<const DataTypeLowCardinality &>(*type);
+            encodeDataType(low_cardinality_type.getDictionaryType(), buf);
+            break;
+        }
+        case BinaryTypeIndex::Map:
+        {
+            const auto & map_type = assert_cast<const DataTypeMap &>(*type);
+            encodeDataType(map_type.getKeyType(), buf);
+            encodeDataType(map_type.getValueType(), buf);
+            break;
+        }
+        case BinaryTypeIndex::Variant:
+        {
+            const auto & variant_type = assert_cast<const DataTypeVariant &>(*type);
+            const auto & variants = variant_type.getVariants();
+            writeVarUInt(variants.size(), buf);
+            for (const auto & variant : variants)
+                encodeDataType(variant, buf);
+            break;
+        }
+        case BinaryTypeIndex::Dynamic:
+        {
+            const auto & dynamic_type = assert_cast<const DataTypeDynamic &>(*type);
+            /// Maximum number of dynamic types is 255, we can write it as 1 byte.
+            writeBinary(UInt8(dynamic_type.getMaxDynamicTypes()), buf);
+            break;
+        }
+        case BinaryTypeIndex::AggregateFunction:
+        {
+            const auto & aggregate_function_type = assert_cast<const DataTypeAggregateFunction &>(*type);
+            writeVarUInt(aggregate_function_type.getVersion(), buf);
+            encodeAggregateFunction(aggregate_function_type.getFunctionName(), aggregate_function_type.getParameters(), aggregate_function_type.getArgumentsDataTypes(), buf);
+            break;
+        }
+        case BinaryTypeIndex::SimpleAggregateFunction:
+        {
+            const auto & simple_aggregate_function_type = assert_cast<const DataTypeCustomSimpleAggregateFunction &>(*type->getCustomName());
+            encodeAggregateFunction(simple_aggregate_function_type.getFunctionName(), simple_aggregate_function_type.getParameters(), simple_aggregate_function_type.getArgumentsDataTypes(), buf);
+            break;
+        }
+        case BinaryTypeIndex::Nested:
+        {
+            const auto & nested_type = assert_cast<const DataTypeNestedCustomName &>(*type->getCustomName());
+            const auto & elements = nested_type.getElements();
+            const auto & names = nested_type.getNames();
+            writeVarUInt(elements.size(), buf);
+            for (size_t i = 0; i != elements.size(); ++i)
+            {
+                writeStringBinary(names[i], buf);
+                encodeDataType(elements[i], buf);
+            }
+            break;
+        }
+        case BinaryTypeIndex::Custom:
+        {
+            const auto & type_name = type->getName();
+            writeStringBinary(type_name, buf);
+            break;
+        }
+        default:
+            break;
+    }
+}
+
+String encodeDataType(const DataTypePtr & type)
+{
+    WriteBufferFromOwnString buf;
+    encodeDataType(type, buf);
+    return buf.str();
+}
+
+DataTypePtr decodeDataType(ReadBuffer & buf)
+{
+    UInt8 type;
+    readBinary(type, buf);
+    switch (BinaryTypeIndex(type))
+    {
+        case BinaryTypeIndex::Nothing:
+            return std::make_shared<DataTypeNothing>();
+        case BinaryTypeIndex::UInt8:
+            return std::make_shared<DataTypeUInt8>();
+        case BinaryTypeIndex::Bool:
+            return DataTypeFactory::instance().get("Bool");
+        case BinaryTypeIndex::UInt16:
+            return std::make_shared<DataTypeUInt16>();
+        case BinaryTypeIndex::UInt32:
+            return std::make_shared<DataTypeUInt32>();
+        case BinaryTypeIndex::UInt64:
+            return std::make_shared<DataTypeUInt64>();
+        case BinaryTypeIndex::UInt128:
+            return std::make_shared<DataTypeUInt128>();
+        case BinaryTypeIndex::UInt256:
+            return std::make_shared<DataTypeUInt256>();
+        case BinaryTypeIndex::Int8:
+            return std::make_shared<DataTypeInt8>();
+        case BinaryTypeIndex::Int16:
+            return std::make_shared<DataTypeInt16>();
+        case BinaryTypeIndex::Int32:
+            return std::make_shared<DataTypeInt32>();
+        case BinaryTypeIndex::Int64:
+            return std::make_shared<DataTypeInt64>();
+        case BinaryTypeIndex::Int128:
+            return std::make_shared<DataTypeInt128>();
+        case BinaryTypeIndex::Int256:
+            return std::make_shared<DataTypeInt256>();
+        case BinaryTypeIndex::Float32:
+            return std::make_shared<DataTypeFloat32>();
+        case BinaryTypeIndex::Float64:
+            return std::make_shared<DataTypeFloat64>();
+        case BinaryTypeIndex::Date:
+            return std::make_shared<DataTypeDate>();
+        case BinaryTypeIndex::Date32:
+            return std::make_shared<DataTypeDate32>();
+        case BinaryTypeIndex::DateTimeUTC:
+            return std::make_shared<DataTypeDateTime>();
+        case BinaryTypeIndex::DateTimeWithTimezone:
+        {
+            String time_zone;
+            readStringBinary(time_zone, buf);
+            return std::make_shared<DataTypeDateTime>(time_zone);
+        }
+        case BinaryTypeIndex::DateTime64UTC:
+        {
+            UInt8 scale;
+            readBinary(scale, buf);
+            return std::make_shared<DataTypeDateTime64>(scale);
+        }
+        case BinaryTypeIndex::DateTime64WithTimezone:
+        {
+            UInt8 scale;
+            readBinary(scale, buf);
+            String time_zone;
+            readStringBinary(time_zone, buf);
+            return std::make_shared<DataTypeDateTime64>(scale, time_zone);
+        }
+        case BinaryTypeIndex::String:
+            return std::make_shared<DataTypeString>();
+        case BinaryTypeIndex::FixedString:
+        {
+            UInt64 size;
+            readVarUInt(size, buf);
+            return std::make_shared<DataTypeFixedString>(size);
+        }
+        case BinaryTypeIndex::Enum8:
+            return decodeEnum<Int8>(buf);
+        case BinaryTypeIndex::Enum16:
+            return decodeEnum<Int16>(buf);
+        case BinaryTypeIndex::Decimal32:
+            return decodeDecimal<Decimal32>(buf);
+        case BinaryTypeIndex::Decimal64:
+            return decodeDecimal<Decimal64>(buf);
+        case BinaryTypeIndex::Decimal128:
+            return decodeDecimal<Decimal128>(buf);
+        case BinaryTypeIndex::Decimal256:
+            return decodeDecimal<Decimal256>(buf);
+        case BinaryTypeIndex::UUID:
+            return std::make_shared<DataTypeUUID>();
+        case BinaryTypeIndex::Array:
+            return std::make_shared<DataTypeArray>(decodeDataType(buf));
+        case BinaryTypeIndex::NamedTuple:
+        {
+            size_t size;
+            readVarUInt(size, buf);
+            DataTypes elements;
+            elements.reserve(size);
+            Names names;
+            names.reserve(size);
+            for (size_t i = 0; i != size; ++i)
+            {
+                names.emplace_back();
+                readStringBinary(names.back(), buf);
+                elements.push_back(decodeDataType(buf));
+            }
+
+            return std::make_shared<DataTypeTuple>(elements, names);
+        }
+        case BinaryTypeIndex::UnnamedTuple:
+        {
+            size_t size;
+            readVarUInt(size, buf);
+            DataTypes elements;
+            elements.reserve(size);
+            for (size_t i = 0; i != size; ++i)
+                elements.push_back(decodeDataType(buf));
+            return std::make_shared<DataTypeTuple>(elements);
+        }
+        case BinaryTypeIndex::Set:
+            return std::make_shared<DataTypeSet>();
+        case BinaryTypeIndex::Interval:
+        {
+            UInt8 kind;
+            readBinary(kind, buf);
+            return std::make_shared<DataTypeInterval>(IntervalKind(IntervalKind::Kind(kind)));
+        }
+        case BinaryTypeIndex::Nullable:
+            return std::make_shared<DataTypeNullable>(decodeDataType(buf));
+        case BinaryTypeIndex::Function:
+        {
+            size_t arguments_size;
+            readVarUInt(arguments_size, buf);
+            DataTypes arguments;
+            arguments.reserve(arguments_size);
+            for (size_t i = 0; i != arguments_size; ++i)
+                arguments.push_back(decodeDataType(buf));
+            auto return_type = decodeDataType(buf);
+            return std::make_shared<DataTypeFunction>(arguments, return_type);
+        }
+        case BinaryTypeIndex::LowCardinality:
+            return std::make_shared<DataTypeLowCardinality>(decodeDataType(buf));
+        case BinaryTypeIndex::Map:
+        {
+            auto key_type = decodeDataType(buf);
+            auto value_type = decodeDataType(buf);
+            return std::make_shared<DataTypeMap>(key_type, value_type);
+        }
+        case BinaryTypeIndex::IPv4:
+            return std::make_shared<DataTypeIPv4>();
+        case BinaryTypeIndex::IPv6:
+            return std::make_shared<DataTypeIPv6>();
+        case BinaryTypeIndex::Variant:
+        {
+            size_t size;
+            readVarUInt(size, buf);
+            DataTypes variants;
+            variants.reserve(size);
+            for (size_t i = 0; i != size; ++i)
+                variants.push_back(decodeDataType(buf));
+            return std::make_shared<DataTypeVariant>(variants);
+        }
+        case BinaryTypeIndex::Dynamic:
+        {
+            UInt8 max_dynamic_types;
+            readBinary(max_dynamic_types, buf);
+            return std::make_shared<DataTypeDynamic>(max_dynamic_types);
+        }
+        case BinaryTypeIndex::AggregateFunction:
+        {
+            size_t version;
+            readVarUInt(version, buf);
+            const auto & [function, parameters, arguments_types] = decodeAggregateFunction(buf);
+            return std::make_shared<DataTypeAggregateFunction>(function, arguments_types, parameters, version);
+        }
+        case BinaryTypeIndex::SimpleAggregateFunction:
+        {
+            const auto & [function, parameters, arguments_types] = decodeAggregateFunction(buf);
+            return createSimpleAggregateFunctionType(function, arguments_types, parameters);
+        }
+        case BinaryTypeIndex::Nested:
+        {
+            size_t size;
+            readVarUInt(size, buf);
+            Names names;
+            names.reserve(size);
+            DataTypes elements;
+            elements.reserve(size);
+            for (size_t i = 0; i != size; ++i)
+            {
+                names.emplace_back();
+                readStringBinary(names.back(), buf);
+                elements.push_back(decodeDataType(buf));
+            }
+
+            return createNested(elements, names);
+        }
+        case BinaryTypeIndex::Custom:
+        {
+            String type_name;
+            readStringBinary(type_name, buf);
+            return DataTypeFactory::instance().get(type_name);
+        }
+    }
+
+    throw Exception(ErrorCodes::INCORRECT_DATA, "Unknown type code: {0:#04x}", UInt64(type));
+}
+
+DataTypePtr decodeDataType(const String & data)
+{
+    ReadBufferFromString buf(data);
+    return decodeDataType(buf);
+}
+
+}

src/DataTypes/DataTypesBinaryEncoding.h

@@ -0,0 +1,118 @@
+#pragma once
+
+#include <DataTypes/IDataType.h>
+
+namespace DB
+{
+
+/**
+
+Binary encoding for ClickHouse data types:
+|------------------------------------------------------------------------------------|------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|
+| ClickHouse data type                                                               | Binary encoding                                                                                                                                                                                                  |
+|------------------------------------------------------------------------------------|------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|
+| Nothing                                                                            | 0x00                                                                                                                                                                                                             |
+| UInt8                                                                              | 0x01                                                                                                                                                                                                             |
+| UInt16                                                                             | 0x02                                                                                                                                                                                                             |
+| UInt32                                                                             | 0x03                                                                                                                                                                                                             |
+| UInt64                                                                             | 0x04                                                                                                                                                                                                             |
+| UInt128                                                                            | 0x05                                                                                                                                                                                                             |
+| UInt256                                                                            | 0x06                                                                                                                                                                                                             |
+| Int8                                                                               | 0x07                                                                                                                                                                                                             |
+| Int16                                                                              | 0x08                                                                                                                                                                                                             |
+| Int32                                                                              | 0x09                                                                                                                                                                                                             |
+| Int64                                                                              | 0x0A                                                                                                                                                                                                             |
+| Int128                                                                             | 0x0B                                                                                                                                                                                                             |
+| Int256                                                                             | 0x0C                                                                                                                                                                                                             |
+| Float32                                                                            | 0x0D                                                                                                                                                                                                             |
+| Float64                                                                            | 0x0E                                                                                                                                                                                                             |
+| Date                                                                               | 0x0F                                                                                                                                                                                                             |
+| Date32                                                                             | 0x10                                                                                                                                                                                                             |
+| DateTime                                                                           | 0x11                                                                                                                                                                                                             |
+| DateTime(time_zone)                                                                | 0x12<var_uint_time_zone_name_size><time_zone_name_data>                                                                                                                                                          |
+| DateTime64(P)                                                                      | 0x13<uint8_precision>                                                                                                                                                                                            |
+| DateTime64(P, time_zone)                                                           | 0x14<uint8_precision><var_uint_time_zone_name_size><time_zone_name_data>                                                                                                                                         |
+| String                                                                             | 0x15                                                                                                                                                                                                             |
+| FixedString(N)                                                                     | 0x16<var_uint_size>                                                                                                                                                                                              |
+| Enum8                                                                              | 0x17<var_uint_number_of_elements><var_uint_name_size_1><name_data_1><int8_value_1>...<var_uint_name_size_N><name_data_N><int8_value_N>                                                                           |
+| Enum16                                                                             | 0x18<var_uint_number_of_elements><var_uint_name_size_1><name_data_1><int16_little_endian_value_1>...><var_uint_name_size_N><name_data_N><int16_little_endian_value_N>                                            |
+| Decimal32(P, S)                                                                    | 0x19<uint8_precision><uint8_scale>                                                                                                                                                                               |
+| Decimal64(P, S)                                                                    | 0x1A<uint8_precision><uint8_scale>                                                                                                                                                                               |
+| Decimal128(P, S)                                                                   | 0x1B<uint8_precision><uint8_scale>                                                                                                                                                                               |
+| Decimal256(P, S)                                                                   | 0x1C<uint8_precision><uint8_scale>                                                                                                                                                                               |
+| UUID                                                                               | 0x1D                                                                                                                                                                                                             |
+| Array(T)                                                                           | 0x1E<nested_type_encoding>                                                                                                                                                                                       |
+| Tuple(T1, ..., TN)                                                                 | 0x1F<var_uint_number_of_elements><nested_type_encoding_1>...<nested_type_encoding_N>                                                                                                                             |
+| Tuple(name1 T1, ..., nameN TN)                                                     | 0x20<var_uint_number_of_elements><var_uint_name_size_1><name_data_1><nested_type_encoding_1>...<var_uint_name_size_N><name_data_N><nested_type_encoding_N>                                                       |
+| Set                                                                                | 0x21                                                                                                                                                                                                             |
+| Interval                                                                           | 0x22<interval_kind>                                                                                                                                                                                              |
+| Nullable(T)                                                                        | 0x23<nested_type_encoding>                                                                                                                                                                                       |
+| Function                                                                           | 0x24<var_uint_number_of_arguments><argument_type_encoding_1>...<argument_type_encoding_N><return_type_encoding>                                                                                                  |
+| AggregateFunction(function_name(param_1, ..., param_N), arg_T1, ..., arg_TN)       | 0x25<var_uint_version><var_uint_function_name_size><function_name_data><var_uint_number_of_parameters><param_1>...<param_N><var_uint_number_of_arguments><argument_type_encoding_1>...<argument_type_encoding_N> |
+| LowCardinality(T)                                                                  | 0x26<nested_type_encoding>                                                                                                                                                                                       |
+| Map(K, V)                                                                          | 0x27<key_type_encoding><value_type_encoding>                                                                                                                                                                     |
+| IPv4                                                                               | 0x28                                                                                                                                                                                                             |
+| IPv6                                                                               | 0x29                                                                                                                                                                                                             |
+| Variant(T1, ..., TN)                                                               | 0x2A<var_uint_number_of_variants><variant_type_encoding_1>...<variant_type_encoding_N>                                                                                                                           |
+| Dynamic(max_types=N)                                                               | 0x2B<uint8_max_types>                                                                                                                                                                                            |
+| Custom type (Ring, Polygon, etc)                                                   | 0x2C<var_uint_type_name_size><type_name_data>                                                                                                                                                                    |
+| Bool                                                                               | 0x2D                                                                                                                                                                                                             |
+| SimpleAggregateFunction(function_name(param_1, ..., param_N), arg_T1, ..., arg_TN) | 0x2E<var_uint_function_name_size><function_name_data><var_uint_number_of_parameters><param_1>...<param_N><var_uint_number_of_arguments><argument_type_encoding_1>...<argument_type_encoding_N>                   |
+| Nested(name1 T1, ..., nameN TN)                                                    | 0x2F<var_uint_number_of_elements><var_uint_name_size_1><name_data_1><nested_type_encoding_1>...<var_uint_name_size_N><name_data_N><nested_type_encoding_N>                                                       |
+|------------------------------------------------------------------------------------|------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|
+
+Interval kind binary encoding:
+|---------------|-----------------|
+| Interval kind | Binary encoding |
+|---------------|-----------------|
+| Nanosecond    | 0x00            |
+| Microsecond   | 0x01            |
+| Millisecond   | 0x02            |
+| Second        | 0x03            |
+| Minute        | 0x04            |
+| Hour          | 0x05            |
+| Day           | 0x06            |
+| Week          | 0x07            |
+| Month         | 0x08            |
+| Quarter       | 0x09            |
+| Year          | 0x1A            |
+|---------------|-----------------|
+
+Aggregate function parameter binary encoding (binary encoding of a Field, see src/Common/FieldBinaryEncoding.h):
+|------------------------|------------------------------------------------------------------------------------------------------------------------------|
+| Parameter type         | Binary encoding                                                                                                              |
+|------------------------|------------------------------------------------------------------------------------------------------------------------------|
+| Null                   | 0x00                                                                                                                         |
+| UInt64                 | 0x01<var_uint_value>                                                                                                         |
+| Int64                  | 0x02<var_int_value>                                                                                                          |
+| UInt128                | 0x03<uint128_little_endian_value>                                                                                            |
+| Int128                 | 0x04<int128_little_endian_value>                                                                                             |
+| UInt128                | 0x05<uint128_little_endian_value>                                                                                            |
+| Int128                 | 0x06<int128_little_endian_value>                                                                                             |
+| Float64                | 0x07<float64_little_endian_value>                                                                                            |
+| Decimal32              | 0x08<var_uint_scale><int32_little_endian_value>                                                                              |
+| Decimal64              | 0x09<var_uint_scale><int64_little_endian_value>                                                                              |
+| Decimal128             | 0x0A<var_uint_scale><int128_little_endian_value>                                                                             |
+| Decimal256             | 0x0B<var_uint_scale><int256_little_endian_value>                                                                             |
+| String                 | 0x0C<var_uint_size><data>                                                                                                    |
+| Array                  | 0x0D<var_uint_size><value_encoding_1>...<value_encoding_N>                                                                   |
+| Tuple                  | 0x0E<var_uint_size><value_encoding_1>...<value_encoding_N>                                                                   |
+| Map                    | 0x0F<var_uint_size><key_encoding_1><value_encoding_1>...<key_endoding_N><value_encoding_N>                                   |
+| IPv4                   | 0x10<uint32_little_endian_value>                                                                                             |
+| IPv6                   | 0x11<uint128_little_endian_value>                                                                                            |
+| UUID                   | 0x12<uuid_value>                                                                                                             |
+| Bool                   | 0x13<bool_value>                                                                                                             |
+| Object                 | 0x14<var_uint_size><var_uint_key_size_1><key_data_1><value_encoding_1>...<var_uint_key_size_N><key_data_N><value_encoding_N> |
+| AggregateFunctionState | 0x15<var_uint_name_size><name_data><var_uint_data_size><data>                                                                |
+| Negative infinity      | 0xFE                                                                                                                         |
+| Positive infinity      | 0xFF                                                                                                                         |
+|------------------------|------------------------------------------------------------------------------------------------------------------------------|
+*/
+
+String encodeDataType(const DataTypePtr & type);
+void encodeDataType(const DataTypePtr & type, WriteBuffer & buf);
+
+DataTypePtr decodeDataType(const String & data);
+DataTypePtr decodeDataType(ReadBuffer & buf);
+
+}

src/DataTypes/Serializations/ISerialization.h

@@ -257,6 +257,9 @@ public:
 
         bool position_independent_encoding = true;
 
+        /// True if data type names should be serialized in binary encoding.
+        bool data_types_binary_encoding = false;
+
         bool use_compact_variant_discriminators_serialization = false;
 
         enum class DynamicStatisticsMode
@@ -278,6 +281,9 @@ public:
 
         bool position_independent_encoding = true;
 
+        /// True if data type names should be deserialized in binary encoding.
+        bool data_types_binary_encoding = false;
+
         bool native_format = false;
 
         /// If not zero, may be used to avoid reallocations while reading column of String type.

src/DataTypes/Serializations/SerializationArray.cpp

@@ -42,13 +42,13 @@ void SerializationArray::deserializeBinary(Field & field, ReadBuffer & istr, con
 {
     size_t size;
     readVarUInt(size, istr);
-    if (settings.max_binary_array_size && size > settings.max_binary_array_size)
+    if (settings.binary.max_binary_string_size && size > settings.binary.max_binary_string_size)
         throw Exception(
             ErrorCodes::TOO_LARGE_ARRAY_SIZE,
             "Too large array size: {}. The maximum is: {}. To increase the maximum, use setting "
             "format_binary_max_array_size",
             size,
-            settings.max_binary_array_size);
+            settings.binary.max_binary_string_size);
 
     field = Array();
     Array & arr = field.get<Array &>();
@@ -82,13 +82,13 @@ void SerializationArray::deserializeBinary(IColumn & column, ReadBuffer & istr,
 
     size_t size;
     readVarUInt(size, istr);
-    if (settings.max_binary_array_size && size > settings.max_binary_array_size)
+    if (settings.binary.max_binary_string_size && size > settings.binary.max_binary_string_size)
         throw Exception(
             ErrorCodes::TOO_LARGE_ARRAY_SIZE,
             "Too large array size: {}. The maximum is: {}. To increase the maximum, use setting "
             "format_binary_max_array_size",
             size,
-            settings.max_binary_array_size);
+            settings.binary.max_binary_string_size);
 
     IColumn & nested_column = column_array.getData();
 

src/DataTypes/Serializations/SerializationDynamic.cpp

@@ -4,6 +4,8 @@
 #include <DataTypes/DataTypeFactory.h>
 #include <DataTypes/DataTypeVariant.h>
 #include <DataTypes/DataTypeString.h>
+#include <DataTypes/DataTypeNothing.h>
+#include <DataTypes/DataTypesBinaryEncoding.h>
 
 #include <Columns/ColumnDynamic.h>
 #include <Columns/ColumnString.h>
@@ -109,7 +111,10 @@ void SerializationDynamic::serializeBinaryBulkStatePrefix(
     const auto & variant_column = column_dynamic.getVariantColumn();
 
     /// Write internal Variant type name.
-    writeStringBinary(dynamic_state->variant_type->getName(), *stream);
+    if (settings.data_types_binary_encoding)
+        encodeDataType(dynamic_state->variant_type, *stream);
+    else
+        writeStringBinary(dynamic_state->variant_type->getName(), *stream);
 
     /// Write statistics in prefix if needed.
     if (settings.dynamic_write_statistics == SerializeBinaryBulkSettings::DynamicStatisticsMode::PREFIX)
@@ -178,9 +183,16 @@ ISerialization::DeserializeBinaryBulkStatePtr SerializationDynamic::deserializeD
         readBinaryLittleEndian(structure_version, *structure_stream);
         auto structure_state = std::make_shared<DeserializeBinaryBulkStateDynamicStructure>(structure_version);
         /// Read internal Variant type name.
-        String data_type_name;
-        readStringBinary(data_type_name, *structure_stream);
-        structure_state->variant_type = DataTypeFactory::instance().get(data_type_name);
+        if (settings.data_types_binary_encoding)
+        {
+            structure_state->variant_type = decodeDataType(*structure_stream);
+        }
+        else
+        {
+            String data_type_name;
+            readStringBinary(data_type_name, *structure_stream);
+            structure_state->variant_type = DataTypeFactory::instance().get(data_type_name);
+        }
         const auto * variant_type = typeid_cast<const DataTypeVariant *>(structure_state->variant_type.get());
         if (!variant_type)
             throw Exception(ErrorCodes::INCORRECT_DATA, "Incorrect type of Dynamic nested column, expected Variant, got {}", structure_state->variant_type->getName());
@@ -280,33 +292,27 @@ void SerializationDynamic::deserializeBinaryBulkWithMultipleStreams(
 
 void SerializationDynamic::serializeBinary(const Field & field, WriteBuffer & ostr, const FormatSettings & settings) const
 {
-    UInt8 null_bit = field.isNull();
-    writeBinary(null_bit, ostr);
-    if (null_bit)
+    /// Serialize NULL as Nothing type with no value.
+    if (field.isNull())
+    {
+        encodeDataType(std::make_shared<DataTypeNothing>(), ostr);
         return;
+    }
 
     auto field_type = applyVisitor(FieldToDataType(), field);
-    auto field_type_name = field_type->getName();
-    writeVarUInt(field_type_name.size(), ostr);
-    writeString(field_type_name, ostr);
+    encodeDataType(field_type, ostr);
     field_type->getDefaultSerialization()->serializeBinary(field, ostr, settings);
 }
 
 void SerializationDynamic::deserializeBinary(Field & field, ReadBuffer & istr, const FormatSettings & settings) const
 {
-    UInt8 null_bit;
-    readBinary(null_bit, istr);
-    if (null_bit)
+    auto field_type = decodeDataType(istr);
+    if (isNothing(field_type))
     {
         field = Null();
         return;
     }
 
-    size_t field_type_name_size;
-    readVarUInt(field_type_name_size, istr);
-    String field_type_name(field_type_name_size, 0);
-    istr.readStrict(field_type_name.data(), field_type_name_size);
-    auto field_type = DataTypeFactory::instance().get(field_type_name);
     field_type->getDefaultSerialization()->deserializeBinary(field, istr, settings);
 }
 
@@ -317,15 +323,15 @@ void SerializationDynamic::serializeBinary(const IColumn & column, size_t row_nu
     const auto & variant_column = dynamic_column.getVariantColumn();
     auto global_discr = variant_column.globalDiscriminatorAt(row_num);
 
-    UInt8 null_bit = global_discr == ColumnVariant::NULL_DISCRIMINATOR;
-    writeBinary(null_bit, ostr);
-    if (null_bit)
+    /// Serialize NULL as Nothing type with no value.
+    if (global_discr == ColumnVariant::NULL_DISCRIMINATOR)
+    {
+        encodeDataType(std::make_shared<DataTypeNothing>(), ostr);
         return;
+    }
 
     const auto & variant_type = assert_cast<const DataTypeVariant &>(*variant_info.variant_type).getVariant(global_discr);
-    const auto & variant_type_name = variant_info.variant_names[global_discr];
-    writeVarUInt(variant_type_name.size(), ostr);
-    writeString(variant_type_name, ostr);
+    encodeDataType(variant_type, ostr);
     variant_type->getDefaultSerialization()->serializeBinary(variant_column.getVariantByGlobalDiscriminator(global_discr), variant_column.offsetAt(row_num), ostr, settings);
 }
 
@@ -346,30 +352,23 @@ static void deserializeVariant(
 void SerializationDynamic::deserializeBinary(IColumn & column, ReadBuffer & istr, const FormatSettings & settings) const
 {
     auto & dynamic_column = assert_cast<ColumnDynamic &>(column);
-    UInt8 null_bit;
-    readBinary(null_bit, istr);
-    if (null_bit)
+    auto variant_type = decodeDataType(istr);
+    if (isNothing(variant_type))
     {
         dynamic_column.insertDefault();
         return;
     }
 
-    size_t variant_type_name_size;
-    readVarUInt(variant_type_name_size, istr);
-    String variant_type_name(variant_type_name_size, 0);
-    istr.readStrict(variant_type_name.data(), variant_type_name_size);
-
+    auto variant_type_name = variant_type->getName();
     const auto & variant_info = dynamic_column.getVariantInfo();
     auto it = variant_info.variant_name_to_discriminator.find(variant_type_name);
     if (it != variant_info.variant_name_to_discriminator.end())
     {
-        const auto & variant_type = assert_cast<const DataTypeVariant &>(*variant_info.variant_type).getVariant(it->second);
         deserializeVariant(dynamic_column.getVariantColumn(), variant_type, it->second, istr, [&settings](const ISerialization & serialization, IColumn & variant, ReadBuffer & buf){ serialization.deserializeBinary(variant, buf, settings); });
         return;
     }
 
     /// We don't have this variant yet. Let's try to add it.
-    auto variant_type = DataTypeFactory::instance().get(variant_type_name);
     if (dynamic_column.addNewVariant(variant_type))
     {
         auto discr = variant_info.variant_name_to_discriminator.at(variant_type_name);

src/DataTypes/Serializations/SerializationMap.cpp

@@ -55,13 +55,13 @@ void SerializationMap::deserializeBinary(Field & field, ReadBuffer & istr, const
 {
     size_t size;
     readVarUInt(size, istr);
-    if (settings.max_binary_array_size && size > settings.max_binary_array_size)
+    if (settings.binary.max_binary_string_size && size > settings.binary.max_binary_string_size)
         throw Exception(
             ErrorCodes::TOO_LARGE_ARRAY_SIZE,
             "Too large map size: {}. The maximum is: {}. To increase the maximum, use setting "
             "format_binary_max_array_size",
             size,
-            settings.max_binary_array_size);
+            settings.binary.max_binary_string_size);
     field = Map();
     Map & map = field.get<Map &>();
     map.reserve(size);

src/DataTypes/Serializations/SerializationString.cpp

@@ -33,13 +33,13 @@ namespace ErrorCodes
 void SerializationString::serializeBinary(const Field & field, WriteBuffer & ostr, const FormatSettings & settings) const
 {
     const String & s = field.get<const String &>();
-    if (settings.max_binary_string_size && s.size() > settings.max_binary_string_size)
+    if (settings.binary.max_binary_string_size && s.size() > settings.binary.max_binary_string_size)
         throw Exception(
             ErrorCodes::TOO_LARGE_STRING_SIZE,
             "Too large string size: {}. The maximum is: {}. To increase the maximum, use setting "
             "format_binary_max_string_size",
             s.size(),
-            settings.max_binary_string_size);
+            settings.binary.max_binary_string_size);
 
     writeVarUInt(s.size(), ostr);
     writeString(s, ostr);
@@ -50,13 +50,13 @@ void SerializationString::deserializeBinary(Field & field, ReadBuffer & istr, co
 {
     UInt64 size;
     readVarUInt(size, istr);
-    if (settings.max_binary_string_size && size > settings.max_binary_string_size)
+    if (settings.binary.max_binary_string_size && size > settings.binary.max_binary_string_size)
         throw Exception(
             ErrorCodes::TOO_LARGE_STRING_SIZE,
             "Too large string size: {}. The maximum is: {}. To increase the maximum, use setting "
             "format_binary_max_string_size",
             size,
-            settings.max_binary_string_size);
+            settings.binary.max_binary_string_size);
 
     field = String();
     String & s = field.get<String &>();
@@ -68,13 +68,13 @@ void SerializationString::deserializeBinary(Field & field, ReadBuffer & istr, co
 void SerializationString::serializeBinary(const IColumn & column, size_t row_num, WriteBuffer & ostr, const FormatSettings & settings) const
 {
     const StringRef & s = assert_cast<const ColumnString &>(column).getDataAt(row_num);
-    if (settings.max_binary_string_size && s.size > settings.max_binary_string_size)
+    if (settings.binary.max_binary_string_size && s.size > settings.binary.max_binary_string_size)
         throw Exception(
             ErrorCodes::TOO_LARGE_STRING_SIZE,
             "Too large string size: {}. The maximum is: {}. To increase the maximum, use setting "
             "format_binary_max_string_size",
             s.size,
-            settings.max_binary_string_size);
+            settings.binary.max_binary_string_size);
 
     writeVarUInt(s.size, ostr);
     writeString(s, ostr);
@@ -89,13 +89,13 @@ void SerializationString::deserializeBinary(IColumn & column, ReadBuffer & istr,
 
     UInt64 size;
     readVarUInt(size, istr);
-    if (settings.max_binary_string_size && size > settings.max_binary_string_size)
+    if (settings.binary.max_binary_string_size && size > settings.binary.max_binary_string_size)
         throw Exception(
             ErrorCodes::TOO_LARGE_STRING_SIZE,
             "Too large string size: {}. The maximum is: {}. To increase the maximum, use setting "
             "format_binary_max_string_size",
             size,
-            settings.max_binary_string_size);
+            settings.binary.max_binary_string_size);
 
     size_t old_chars_size = data.size();
     size_t offset = old_chars_size + size + 1;

src/DataTypes/fuzzers/data_type_deserialization_fuzzer.cpp

@@ -72,8 +72,8 @@ extern "C" int LLVMFuzzerTestOneInput(const uint8_t * data, size_t size)
         DataTypePtr type = DataTypeFactory::instance().get(data_type);
 
         FormatSettings settings;
-        settings.max_binary_string_size = 100;
-        settings.max_binary_array_size = 100;
+        settings.binary.max_binary_string_size = 100;
+        settings.binary.max_binary_string_size = 100;
 
         Field field;
         type->getDefaultSerialization()->deserializeBinary(field, in, settings);

src/DataTypes/tests/gtest_data_types_binary_encoding.cpp

@@ -0,0 +1,129 @@
+#include <gtest/gtest.h>
+#include <Core/Field.h>
+#include <DataTypes/DataTypesBinaryEncoding.h>
+#include <DataTypes/DataTypeDateTime64.h>
+#include <DataTypes/DataTypeFixedString.h>
+#include <DataTypes/DataTypeEnum.h>
+#include <DataTypes/DataTypesDecimal.h>
+#include <DataTypes/DataTypeFunction.h>
+#include <DataTypes/DataTypeString.h>
+#include <DataTypes/DataTypeUUID.h>
+#include <DataTypes/DataTypeSet.h>
+#include <DataTypes/DataTypeInterval.h>
+#include <DataTypes/DataTypeIPv4andIPv6.h>
+#include <DataTypes/DataTypeAggregateFunction.h>
+#include <DataTypes/DataTypeNothing.h>
+#include <DataTypes/DataTypeDynamic.h>
+#include <DataTypes/DataTypeFactory.h>
+#include <AggregateFunctions/IAggregateFunction.h>
+#include <AggregateFunctions/AggregateFunctionFactory.h>
+#include <AggregateFunctions/registerAggregateFunctions.h>
+#include <IO/WriteBufferFromString.h>
+#include <IO/ReadBufferFromString.h>
+#include <Common/tests/gtest_global_register.h>
+
+using namespace DB;
+
+namespace DB::ErrorCodes
+{
+extern const int UNSUPPORTED_METHOD;
+}
+
+
+void check(const DataTypePtr & type)
+{
+//    std::cerr << "Check " << type->getName() << "\n";
+    WriteBufferFromOwnString ostr;
+    encodeDataType(type, ostr);
+    ReadBufferFromString istr(ostr.str());
+    DataTypePtr decoded_type = decodeDataType(istr);
+    ASSERT_TRUE(istr.eof());
+    ASSERT_EQ(type->getName(), decoded_type->getName());
+    ASSERT_TRUE(type->equals(*decoded_type));
+}
+
+GTEST_TEST(DataTypesBinaryEncoding, EncodeAndDecode)
+{
+    tryRegisterAggregateFunctions();
+    check(std::make_shared<DataTypeNothing>());
+    check(std::make_shared<DataTypeInt8>());
+    check(std::make_shared<DataTypeUInt8>());
+    check(std::make_shared<DataTypeInt16>());
+    check(std::make_shared<DataTypeUInt16>());
+    check(std::make_shared<DataTypeInt32>());
+    check(std::make_shared<DataTypeUInt32>());
+    check(std::make_shared<DataTypeInt64>());
+    check(std::make_shared<DataTypeUInt64>());
+    check(std::make_shared<DataTypeInt128>());
+    check(std::make_shared<DataTypeUInt128>());
+    check(std::make_shared<DataTypeInt256>());
+    check(std::make_shared<DataTypeUInt256>());
+    check(std::make_shared<DataTypeFloat32>());
+    check(std::make_shared<DataTypeFloat64>());
+    check(std::make_shared<DataTypeDate>());
+    check(std::make_shared<DataTypeDate32>());
+    check(std::make_shared<DataTypeDateTime>());
+    check(std::make_shared<DataTypeDateTime>("EST"));
+    check(std::make_shared<DataTypeDateTime>("CET"));
+    check(std::make_shared<DataTypeDateTime64>(3));
+    check(std::make_shared<DataTypeDateTime64>(3, "EST"));
+    check(std::make_shared<DataTypeDateTime64>(3, "CET"));
+    check(std::make_shared<DataTypeString>());
+    check(std::make_shared<DataTypeFixedString>(10));
+    check(DataTypeFactory::instance().get("Enum8('a' = 1, 'b' = 2, 'c' = 3, 'd' = -128)"));
+    check(DataTypeFactory::instance().get("Enum16('a' = 1, 'b' = 2, 'c' = 3, 'd' = -1000)"));
+    check(std::make_shared<DataTypeDecimal32>(3, 6));
+    check(std::make_shared<DataTypeDecimal64>(3, 6));
+    check(std::make_shared<DataTypeDecimal128>(3, 6));
+    check(std::make_shared<DataTypeDecimal256>(3, 6));
+    check(std::make_shared<DataTypeUUID>());
+    check(DataTypeFactory::instance().get("Array(UInt32)"));
+    check(DataTypeFactory::instance().get("Array(Array(Array(UInt32)))"));
+    check(DataTypeFactory::instance().get("Tuple(UInt32, String, UUID)"));
+    check(DataTypeFactory::instance().get("Tuple(UInt32, String, Tuple(UUID, Date, IPv4))"));
+    check(DataTypeFactory::instance().get("Tuple(c1 UInt32, c2 String, c3 UUID)"));
+    check(DataTypeFactory::instance().get("Tuple(c1 UInt32, c2 String, c3 Tuple(c4 UUID, c5 Date, c6 IPv4))"));
+    check(std::make_shared<DataTypeSet>());
+    check(std::make_shared<DataTypeInterval>(IntervalKind::Kind::Nanosecond));
+    check(std::make_shared<DataTypeInterval>(IntervalKind::Kind::Microsecond));
+    check(DataTypeFactory::instance().get("Nullable(UInt32)"));
+    check(DataTypeFactory::instance().get("Nullable(Nothing)"));
+    check(DataTypeFactory::instance().get("Nullable(UUID)"));
+    check(std::make_shared<DataTypeFunction>(
+        DataTypes{
+            std::make_shared<DataTypeInt8>(),
+            std::make_shared<DataTypeDate>(),
+            DataTypeFactory::instance().get("Array(Array(Array(UInt32)))")},
+        DataTypeFactory::instance().get("Tuple(c1 UInt32, c2 String, c3 UUID)")));
+    DataTypes argument_types = {std::make_shared<DataTypeUInt64>()};
+    Array parameters = {Field(0.1), Field(0.2)};
+    AggregateFunctionProperties properties;
+    AggregateFunctionPtr function = AggregateFunctionFactory::instance().get("quantiles", NullsAction::EMPTY, argument_types, parameters, properties);
+    check(std::make_shared<DataTypeAggregateFunction>(function, argument_types, parameters));
+    check(std::make_shared<DataTypeAggregateFunction>(function, argument_types, parameters, 2));
+    check(DataTypeFactory::instance().get("AggregateFunction(sum, UInt64)"));
+    check(DataTypeFactory::instance().get("AggregateFunction(quantiles(0.5, 0.9), UInt64)"));
+    check(DataTypeFactory::instance().get("AggregateFunction(sequenceMatch('(?1)(?2)'), Date, UInt8, UInt8)"));
+    check(DataTypeFactory::instance().get("AggregateFunction(sumMapFiltered([1, 4, 8]), Array(UInt64), Array(UInt64))"));
+    check(DataTypeFactory::instance().get("LowCardinality(UInt32)"));
+    check(DataTypeFactory::instance().get("LowCardinality(Nullable(String))"));
+    check(DataTypeFactory::instance().get("Map(String, UInt32)"));
+    check(DataTypeFactory::instance().get("Map(String, Map(String, Map(String, UInt32)))"));
+    check(std::make_shared<DataTypeIPv4>());
+    check(std::make_shared<DataTypeIPv6>());
+    check(DataTypeFactory::instance().get("Variant(String, UInt32, Date32)"));
+    check(std::make_shared<DataTypeDynamic>());
+    check(std::make_shared<DataTypeDynamic>(10));
+    check(std::make_shared<DataTypeDynamic>(255));
+    check(DataTypeFactory::instance().get("Bool"));
+    check(DataTypeFactory::instance().get("SimpleAggregateFunction(sum, UInt64)"));
+    check(DataTypeFactory::instance().get("SimpleAggregateFunction(maxMap, Tuple(Array(UInt32), Array(UInt32)))"));
+    check(DataTypeFactory::instance().get("SimpleAggregateFunction(groupArrayArray(19), Array(UInt64))"));
+    check(DataTypeFactory::instance().get("Nested(a UInt32, b UInt32)"));
+    check(DataTypeFactory::instance().get("Nested(a UInt32, b Nested(c String, d Nested(e Date)))"));
+    check(DataTypeFactory::instance().get("Ring"));
+    check(DataTypeFactory::instance().get("Point"));
+    check(DataTypeFactory::instance().get("Polygon"));
+    check(DataTypeFactory::instance().get("MultiPolygon"));
+    check(DataTypeFactory::instance().get("Tuple(Map(LowCardinality(String), Array(AggregateFunction(2, quantiles(0.1, 0.2), Float32))), Array(Array(Tuple(UInt32, Tuple(a Map(String, String), b Nullable(Date), c Variant(Tuple(g String, d Array(UInt32)), Date, Map(String, String)))))))"));
+}

src/Databases/DatabaseAtomic.cpp

@@ -1,4 +1,5 @@
 #include <filesystem>
+#include <base/isSharedPtrUnique.h>
 #include <Databases/DatabaseAtomic.h>
 #include <Databases/DatabaseFactory.h>
 #include <Databases/DatabaseOnDisk.h>
@@ -12,7 +13,7 @@
 #include <Interpreters/ExternalDictionariesLoader.h>
 #include <Parsers/formatAST.h>
 #include <Storages/StorageMaterializedView.h>
-#include "Common/logger_useful.h"
+#include <Common/logger_useful.h>
 #include <Common/PoolId.h>
 #include <Common/atomicRename.h>
 #include <Common/filesystemHelpers.h>
@@ -397,7 +398,7 @@ DatabaseAtomic::DetachedTables DatabaseAtomic::cleanupDetachedTables()
     LOG_DEBUG(log, "There are {} detached tables. Start searching non used tables.", detached_tables.size());
     while (it != detached_tables.end())
     {
-        if (it->second.unique())
+        if (isSharedPtrUnique(it->second))
         {
             not_in_use.emplace(it->first, it->second);
             it = detached_tables.erase(it);

src/Databases/DatabaseLazy.cpp

@@ -1,6 +1,7 @@
 #include <Databases/DatabaseLazy.h>
 
 #include <base/sort.h>
+#include <base/isSharedPtrUnique.h>
 #include <iomanip>
 #include <filesystem>
 #include <Common/CurrentMetrics.h>
@@ -305,7 +306,7 @@ try
         String table_name = expired_tables.front().table_name;
         auto it = tables_cache.find(table_name);
 
-        if (!it->second.table || it->second.table.unique())
+        if (!it->second.table || isSharedPtrUnique(it->second.table))
         {
             LOG_DEBUG(log, "Drop table {} from cache.", backQuote(it->first));
             it->second.table.reset();

src/Databases/MySQL/DatabaseMySQL.cpp

@@ -2,6 +2,7 @@
 
 #if USE_MYSQL
 #    include <string>
+#    include <base/isSharedPtrUnique.h>
 #    include <Databases/DatabaseFactory.h>
 #    include <DataTypes/DataTypeDateTime.h>
 #    include <DataTypes/DataTypeNullable.h>
@@ -354,7 +355,7 @@ void DatabaseMySQL::cleanOutdatedTables()
     {
         for (auto iterator = outdated_tables.begin(); iterator != outdated_tables.end();)
         {
-            if (!iterator->unique())
+            if (!isSharedPtrUnique(*iterator))
                 ++iterator;
             else
             {

src/Databases/PostgreSQL/DatabaseMaterializedPostgreSQL.cpp

@@ -529,7 +529,12 @@ void registerDatabaseMaterializedPostgreSQL(DatabaseFactory & factory)
         }
 
         auto connection_info = postgres::formatConnectionString(
-            configuration.database, configuration.host, configuration.port, configuration.username, configuration.password);
+            configuration.database,
+            configuration.host,
+            configuration.port,
+            configuration.username,
+            configuration.password,
+            args.context->getSettingsRef().postgresql_connection_attempt_timeout);
 
         auto postgresql_replica_settings = std::make_unique<MaterializedPostgreSQLSettings>();
         if (engine_define->settings)

src/Databases/PostgreSQL/DatabasePostgreSQL.cpp

@@ -545,8 +545,9 @@ void registerDatabasePostgreSQL(DatabaseFactory & factory)
             configuration,
             settings.postgresql_connection_pool_size,
             settings.postgresql_connection_pool_wait_timeout,
-            POSTGRESQL_POOL_WITH_FAILOVER_DEFAULT_MAX_TRIES,
-            settings.postgresql_connection_pool_auto_close_connection);
+            settings.postgresql_connection_pool_retries,
+            settings.postgresql_connection_pool_auto_close_connection,
+            settings.postgresql_connection_attempt_timeout);
 
         return std::make_shared<DatabasePostgreSQL>(
             args.context,

src/Dictionaries/PostgreSQLDictionarySource.cpp

@@ -205,8 +205,9 @@ void registerDictionarySourcePostgreSQL(DictionarySourceFactory & factory)
             configuration.replicas_configurations,
             settings.postgresql_connection_pool_size,
             settings.postgresql_connection_pool_wait_timeout,
-            POSTGRESQL_POOL_WITH_FAILOVER_DEFAULT_MAX_TRIES,
-            settings.postgresql_connection_pool_auto_close_connection);
+            settings.postgresql_connection_pool_retries,
+            settings.postgresql_connection_pool_auto_close_connection,
+            settings.postgresql_connection_attempt_timeout);
 
         PostgreSQLDictionarySource::Configuration dictionary_configuration
         {

src/Formats/EscapingRuleUtils.cpp

@@ -439,13 +439,15 @@ String getAdditionalFormatInfoByEscapingRule(const FormatSettings & settings, Fo
         case FormatSettings::EscapingRule::CSV:
             result += fmt::format(
                 ", use_best_effort_in_schema_inference={}, bool_true_representation={}, bool_false_representation={},"
-                " null_representation={}, delimiter={}, tuple_delimiter={}",
+                " null_representation={}, delimiter={}, tuple_delimiter={}, try_infer_numbers_from_strings={}, try_infer_strings_from_quoted_tuples={}",
                 settings.csv.use_best_effort_in_schema_inference,
                 settings.bool_true_representation,
                 settings.bool_false_representation,
                 settings.csv.null_representation,
                 settings.csv.delimiter,
-                settings.csv.tuple_delimiter);
+                settings.csv.tuple_delimiter,
+                settings.csv.try_infer_numbers_from_strings,
+                settings.csv.try_infer_strings_from_quoted_tuples);
             break;
         case FormatSettings::EscapingRule::JSON:
             result += fmt::format(

src/Formats/FormatFactory.cpp

@@ -270,9 +270,13 @@ FormatSettings getFormatSettings(const ContextPtr & context, const Settings & se
     format_settings.markdown.escape_special_characters = settings.output_format_markdown_escape_special_characters;
     format_settings.bson.output_string_as_string = settings.output_format_bson_string_as_string;
     format_settings.bson.skip_fields_with_unsupported_types_in_schema_inference = settings.input_format_bson_skip_fields_with_unsupported_types_in_schema_inference;
-    format_settings.max_binary_string_size = settings.format_binary_max_string_size;
-    format_settings.max_binary_array_size = settings.format_binary_max_array_size;
+    format_settings.binary.max_binary_string_size = settings.format_binary_max_string_size;
+    format_settings.binary.max_binary_array_size = settings.format_binary_max_array_size;
+    format_settings.binary.encode_types_in_binary_format = settings.output_format_binary_encode_types_in_binary_format;
+    format_settings.binary.decode_types_in_binary_format = settings.input_format_binary_decode_types_in_binary_format;
     format_settings.native.allow_types_conversion = settings.input_format_native_allow_types_conversion;
+    format_settings.native.encode_types_in_binary_format = settings.output_format_native_encode_types_in_binary_format;
+    format_settings.native.decode_types_in_binary_format = settings.input_format_native_decode_types_in_binary_format;
     format_settings.max_parser_depth = context->getSettingsRef().max_parser_depth;
     format_settings.client_protocol_version = context->getClientProtocolVersion();
     format_settings.date_time_overflow_behavior = settings.date_time_overflow_behavior;

src/Formats/FormatSettings.h

@@ -106,8 +106,6 @@ struct FormatSettings
     UInt64 input_allow_errors_num = 0;
     Float32 input_allow_errors_ratio = 0;
 
-    UInt64 max_binary_string_size = 1_GiB;
-    UInt64 max_binary_array_size = 1_GiB;
     UInt64 client_protocol_version = 0;
 
     UInt64 max_parser_depth = DBMS_DEFAULT_MAX_PARSER_DEPTH;
@@ -121,6 +119,14 @@ struct FormatSettings
         ZSTD
     };
 
+    struct
+    {
+        UInt64 max_binary_string_size = 1_GiB;
+        UInt64 max_binary_array_size = 1_GiB;
+        bool encode_types_in_binary_format = false;
+        bool decode_types_in_binary_format = false;
+    } binary{};
+
     struct
     {
         UInt64 row_group_size = 1000000;
@@ -459,6 +465,8 @@ struct FormatSettings
     struct
     {
         bool allow_types_conversion = true;
+        bool encode_types_in_binary_format = false;
+        bool decode_types_in_binary_format = false;
     } native{};
 
     struct

src/Formats/NativeReader.cpp

@@ -6,6 +6,7 @@
 #include <Compression/CompressedReadBufferFromFile.h>
 
 #include <DataTypes/DataTypeFactory.h>
+#include <DataTypes/DataTypesBinaryEncoding.h>
 #include <Common/typeid_cast.h>
 #include <base/range.h>
 
@@ -31,8 +32,8 @@ namespace ErrorCodes
 }
 
 
-NativeReader::NativeReader(ReadBuffer & istr_, UInt64 server_revision_)
-    : istr(istr_), server_revision(server_revision_)
+NativeReader::NativeReader(ReadBuffer & istr_, UInt64 server_revision_, std::optional<FormatSettings> format_settings_)
+    : istr(istr_), server_revision(server_revision_), format_settings(format_settings_)
 {
 }
 
@@ -40,16 +41,12 @@ NativeReader::NativeReader(
     ReadBuffer & istr_,
     const Block & header_,
     UInt64 server_revision_,
-    bool skip_unknown_columns_,
-    bool null_as_default_,
-    bool allow_types_conversion_,
+    std::optional<FormatSettings> format_settings_,
     BlockMissingValues * block_missing_values_)
     : istr(istr_)
     , header(header_)
     , server_revision(server_revision_)
-    , skip_unknown_columns(skip_unknown_columns_)
-    , null_as_default(null_as_default_)
-    , allow_types_conversion(allow_types_conversion_)
+    , format_settings(std::move(format_settings_))
     , block_missing_values(block_missing_values_)
 {
 }
@@ -83,13 +80,14 @@ void NativeReader::resetParser()
     use_index = false;
 }
 
-void NativeReader::readData(const ISerialization & serialization, ColumnPtr & column, ReadBuffer & istr, size_t rows, double avg_value_size_hint)
+static void readData(const ISerialization & serialization, ColumnPtr & column, ReadBuffer & istr, const std::optional<FormatSettings> & format_settings, size_t rows, double avg_value_size_hint)
 {
     ISerialization::DeserializeBinaryBulkSettings settings;
     settings.getter = [&](ISerialization::SubstreamPath) -> ReadBuffer * { return &istr; };
     settings.avg_value_size_hint = avg_value_size_hint;
     settings.position_independent_encoding = false;
     settings.native_format = true;
+    settings.data_types_binary_encoding = format_settings && format_settings->native.decode_types_in_binary_format;
 
     ISerialization::DeserializeBinaryBulkStatePtr state;
 
@@ -167,8 +165,16 @@ Block NativeReader::read()
 
         /// Type
         String type_name;
-        readBinary(type_name, istr);
-        column.type = data_type_factory.get(type_name);
+        if (format_settings && format_settings->native.decode_types_in_binary_format)
+        {
+            column.type = decodeDataType(istr);
+            type_name = column.type->getName();
+        }
+        else
+        {
+            readBinary(type_name, istr);
+            column.type = data_type_factory.get(type_name);
+        }
 
         setVersionToAggregateFunctions(column.type, true, server_revision);
 
@@ -203,7 +209,7 @@ Block NativeReader::read()
 
         double avg_value_size_hint = avg_value_size_hints.empty() ? 0 : avg_value_size_hints[i];
         if (rows)    /// If no rows, nothing to read.
-            readData(*serialization, read_column, istr, rows, avg_value_size_hint);
+            readData(*serialization, read_column, istr, format_settings, rows, avg_value_size_hint);
 
         column.column = std::move(read_column);
 
@@ -214,12 +220,12 @@ Block NativeReader::read()
             {
                 auto & header_column = header.getByName(column.name);
 
-                if (null_as_default)
+                if (format_settings && format_settings->null_as_default)
                     insertNullAsDefaultIfNeeded(column, header_column, header.getPositionByName(column.name), block_missing_values);
 
                 if (!header_column.type->equals(*column.type))
                 {
-                    if (allow_types_conversion)
+                    if (format_settings && format_settings->native.allow_types_conversion)
                     {
                         try
                         {
@@ -246,7 +252,7 @@ Block NativeReader::read()
             }
             else
             {
-                if (!skip_unknown_columns)
+                if (format_settings && !format_settings->skip_unknown_fields)
                     throw Exception(ErrorCodes::INCORRECT_DATA, "Unknown column with name {} found while reading data in Native format", column.name);
                 use_in_result = false;
             }

src/Formats/NativeReader.h

@@ -20,7 +20,7 @@ class NativeReader
 {
 public:
     /// If a non-zero server_revision is specified, additional block information may be expected and read.
-    NativeReader(ReadBuffer & istr_, UInt64 server_revision_);
+    NativeReader(ReadBuffer & istr_, UInt64 server_revision_, std::optional<FormatSettings> format_settings_ = std::nullopt);
 
     /// For cases when data structure (header) is known in advance.
     /// NOTE We may use header for data validation and/or type conversions. It is not implemented.
@@ -28,9 +28,7 @@ public:
         ReadBuffer & istr_,
         const Block & header_,
         UInt64 server_revision_,
-        bool skip_unknown_columns_ = false,
-        bool null_as_default_ = false,
-        bool allow_types_conversion_ = false,
+        std::optional<FormatSettings> format_settings_ = std::nullopt,
         BlockMissingValues * block_missing_values_ = nullptr);
 
     /// For cases when we have an index. It allows to skip columns. Only columns specified in the index will be read.
@@ -38,8 +36,6 @@ public:
         IndexForNativeFormat::Blocks::const_iterator index_block_it_,
         IndexForNativeFormat::Blocks::const_iterator index_block_end_);
 
-    static void readData(const ISerialization & serialization, ColumnPtr & column, ReadBuffer & istr, size_t rows, double avg_value_size_hint);
-
     Block getHeader() const;
 
     void resetParser();
@@ -50,9 +46,7 @@ private:
     ReadBuffer & istr;
     Block header;
     UInt64 server_revision;
-    bool skip_unknown_columns = false;
-    bool null_as_default = false;
-    bool allow_types_conversion = false;
+    std::optional<FormatSettings> format_settings = std::nullopt;
     BlockMissingValues * block_missing_values = nullptr;
 
     bool use_index = false;

src/Formats/NativeWriter.cpp

@@ -14,6 +14,7 @@
 #include <Columns/ColumnSparse.h>
 #include <DataTypes/DataTypeLowCardinality.h>
 #include <DataTypes/DataTypeAggregateFunction.h>
+#include <DataTypes/DataTypesBinaryEncoding.h>
 
 namespace DB
 {
@@ -25,10 +26,20 @@ namespace ErrorCodes
 
 
 NativeWriter::NativeWriter(
-    WriteBuffer & ostr_, UInt64 client_revision_, const Block & header_, bool remove_low_cardinality_,
-    IndexForNativeFormat * index_, size_t initial_size_of_file_)
-    : ostr(ostr_), client_revision(client_revision_), header(header_),
-      index(index_), initial_size_of_file(initial_size_of_file_), remove_low_cardinality(remove_low_cardinality_)
+    WriteBuffer & ostr_,
+    UInt64 client_revision_,
+    const Block & header_,
+    std::optional<FormatSettings> format_settings_,
+    bool remove_low_cardinality_,
+    IndexForNativeFormat * index_,
+    size_t initial_size_of_file_)
+    : ostr(ostr_)
+    , client_revision(client_revision_)
+    , header(header_)
+    , index(index_)
+    , initial_size_of_file(initial_size_of_file_)
+    , remove_low_cardinality(remove_low_cardinality_)
+    , format_settings(std::move(format_settings_))
 {
     if (index)
     {
@@ -45,7 +56,7 @@ void NativeWriter::flush()
 }
 
 
-static void writeData(const ISerialization & serialization, const ColumnPtr & column, WriteBuffer & ostr, UInt64 offset, UInt64 limit)
+static void writeData(const ISerialization & serialization, const ColumnPtr & column, WriteBuffer & ostr, const std::optional<FormatSettings> & format_settings, UInt64 offset, UInt64 limit)
 {
     /** If there are columns-constants - then we materialize them.
       * (Since the data type does not know how to serialize / deserialize constants.)
@@ -57,6 +68,7 @@ static void writeData(const ISerialization & serialization, const ColumnPtr & co
     settings.getter = [&ostr](ISerialization::SubstreamPath) -> WriteBuffer * { return &ostr; };
     settings.position_independent_encoding = false;
     settings.low_cardinality_max_dictionary_size = 0;
+    settings.data_types_binary_encoding = format_settings && format_settings->native.encode_types_in_binary_format;
 
     ISerialization::SerializeBinaryBulkStatePtr state;
     serialization.serializeBinaryBulkStatePrefix(*full_column, settings, state);
@@ -121,15 +133,22 @@ size_t NativeWriter::write(const Block & block)
         setVersionToAggregateFunctions(column.type, include_version, include_version ? std::optional<size_t>(client_revision) : std::nullopt);
 
         /// Type
-        String type_name = column.type->getName();
+        if (format_settings && format_settings->native.encode_types_in_binary_format)
+        {
+            encodeDataType(column.type, ostr);
+        }
+        else
+        {
+            String type_name = column.type->getName();
 
-        /// For compatibility, we will not send explicit timezone parameter in DateTime data type
-        ///  to older clients, that cannot understand it.
-        if (client_revision < DBMS_MIN_REVISION_WITH_TIME_ZONE_PARAMETER_IN_DATETIME_DATA_TYPE
-            && startsWith(type_name, "DateTime("))
-            type_name = "DateTime";
+            /// For compatibility, we will not send explicit timezone parameter in DateTime data type
+            ///  to older clients, that cannot understand it.
+            if (client_revision < DBMS_MIN_REVISION_WITH_TIME_ZONE_PARAMETER_IN_DATETIME_DATA_TYPE
+                && startsWith(type_name, "DateTime("))
+                type_name = "DateTime";
 
-        writeStringBinary(type_name, ostr);
+            writeStringBinary(type_name, ostr);
+        }
 
         /// Serialization. Dynamic, if client supports it.
         SerializationPtr serialization;
@@ -161,7 +180,7 @@ size_t NativeWriter::write(const Block & block)
 
         /// Data
         if (rows)    /// Zero items of data is always represented as zero number of bytes.
-            writeData(*serialization, column.column, ostr, 0, 0);
+            writeData(*serialization, column.column, ostr, format_settings, 0, 0);
 
         if (index)
         {

src/Formats/NativeWriter.h

@@ -3,6 +3,7 @@
 #include <base/types.h>
 #include <DataTypes/IDataType.h>
 #include <Core/Block.h>
+#include <Formats/FormatSettings.h>
 
 namespace DB
 {
@@ -23,7 +24,7 @@ public:
     /** If non-zero client_revision is specified, additional block information can be written.
       */
     NativeWriter(
-        WriteBuffer & ostr_, UInt64 client_revision_, const Block & header_, bool remove_low_cardinality_ = false,
+        WriteBuffer & ostr_, UInt64 client_revision_, const Block & header_, std::optional<FormatSettings> format_settings_ = std::nullopt, bool remove_low_cardinality_ = false,
         IndexForNativeFormat * index_ = nullptr, size_t initial_size_of_file_ = 0);
 
     Block getHeader() const { return header; }
@@ -44,6 +45,7 @@ private:
     CompressedWriteBuffer * ostr_concrete = nullptr;
 
     bool remove_low_cardinality;
+    std::optional<FormatSettings> format_settings;
 };
 
 }

src/Functions/FunctionsBitToArray.cpp

@@ -284,7 +284,12 @@ public:
             {
                 while (x)
                 {
-                    result_array_values_data.push_back(std::countr_zero(x));
+                    /// С++20 char8_t is not an unsigned integral type anymore https://godbolt.org/z/Mqcb7qn58
+                    /// and thus you cannot use std::countr_zero on it.
+                    if constexpr (std::is_same_v<UnsignedType, UInt8>)
+                        result_array_values_data.push_back(std::countr_zero(static_cast<unsigned char>(x)));
+                    else
+                        result_array_values_data.push_back(std::countr_zero(x));
                     x &= (x - 1);
                 }
             }
@@ -336,4 +341,3 @@ REGISTER_FUNCTION(BitToArray)
 }
 
 }
-

src/Functions/array/arrayAUC.cpp

@@ -103,27 +103,40 @@ private:
             sorted_labels[i].label = label;
         }
 
-        /// Stable sort is required for for labels to apply in same order if score is equal
-        std::stable_sort(sorted_labels.begin(), sorted_labels.end(), [](const auto & lhs, const auto & rhs) { return lhs.score > rhs.score; });
+        /// Sorting scores in descending order to traverse the ROC curve from left to right
+        std::sort(sorted_labels.begin(), sorted_labels.end(), [](const auto & lhs, const auto & rhs) { return lhs.score > rhs.score; });
 
         /// We will first calculate non-normalized area.
 
-        size_t area = 0;
-        size_t count_positive = 0;
+        Float64 area = 0.0;
+        Float64 prev_score = sorted_labels[0].score;
+        size_t prev_fp = 0, prev_tp = 0;
+        size_t curr_fp = 0, curr_tp = 0;
         for (size_t i = 0; i < size; ++i)
         {
+            // Only increment the area when the score changes
+            if (sorted_labels[i].score != prev_score)
+            {
+                area += (curr_fp - prev_fp) * (curr_tp + prev_tp) / 2.0; // Trapezoidal area under curve (might degenerate to zero or to a rectangle)
+                prev_fp = curr_fp;
+                prev_tp = curr_tp;
+                prev_score = sorted_labels[i].score;
+            }
+
             if (sorted_labels[i].label)
-                ++count_positive; /// The curve moves one step up. No area increase.
+                curr_tp += 1; /// The curve moves one step up.
             else
-                area += count_positive; /// The curve moves one step right. Area is increased by 1 * height = count_positive.
+                curr_fp += 1; /// The curve moves one step right.
         }
 
-        /// Then divide the area to the area of rectangle.
+        area += (curr_fp - prev_fp) * (curr_tp + prev_tp) / 2.0;
 
-        if (count_positive == 0 || count_positive == size)
+        /// Then normalize it dividing by the area to the area of rectangle.
+
+        if (curr_tp == 0 || curr_tp == size)
             return std::numeric_limits<Float64>::quiet_NaN();
 
-        return static_cast<Float64>(area) / count_positive / (size - count_positive);
+        return area / curr_tp / (size - curr_tp);
     }
 
     static void vector(

src/Interpreters/ActionsVisitor.cpp

@@ -405,10 +405,6 @@ Block createBlockForSet(
 
 }
 
-ScopeStack::Level::Level() = default;
-ScopeStack::Level::~Level() = default;
-ScopeStack::Level::Level(Level &&) noexcept = default;
-
 FutureSetPtr makeExplicitSet(
     const ASTFunction * node, const ActionsDAG & actions, ContextPtr context, PreparedSets & prepared_sets)
 {
@@ -462,6 +458,7 @@ public:
         for (const auto * node : index)
             map.emplace(node->result_name, node);
     }
+    ~Index() = default;
 
     void addNode(const ActionsDAG::Node * node)
     {
@@ -502,6 +499,10 @@ public:
     }
 };
 
+ScopeStack::Level::Level() = default;
+ScopeStack::Level::~Level() = default;
+ScopeStack::Level::Level(Level &&) noexcept = default;
+
 ActionsMatcher::Data::Data(
     ContextPtr context_,
     SizeLimits set_size_limit_,

src/Interpreters/Cache/Metadata.h

@@ -1,11 +1,16 @@
 #pragma once
+
+
 #include <boost/noncopyable.hpp>
+#include <base/isSharedPtrUnique.h>
 #include <Interpreters/Cache/Guards.h>
 #include <Interpreters/Cache/IFileCachePriority.h>
 #include <Interpreters/Cache/FileCacheKey.h>
 #include <Interpreters/Cache/FileSegment.h>
 #include <Interpreters/Cache/FileCache_fwd_internal.h>
 #include <Common/ThreadPool.h>
+
+#include <memory>
 #include <shared_mutex>
 
 namespace DB
@@ -30,7 +35,7 @@ struct FileSegmentMetadata : private boost::noncopyable
 
     explicit FileSegmentMetadata(FileSegmentPtr && file_segment_);
 
-    bool releasable() const { return file_segment.unique(); }
+    bool releasable() const { return isSharedPtrUnique(file_segment); }
 
     size_t size() const;
 

src/Interpreters/Context.cpp

@@ -4051,7 +4051,6 @@ std::shared_ptr<QueryThreadLog> Context::getQueryThreadLog() const
 std::shared_ptr<QueryViewsLog> Context::getQueryViewsLog() const
 {
     SharedLockGuard lock(shared->mutex);
-
     if (!shared->system_logs)
         return {};
 

src/Interpreters/DatabaseCatalog.cpp

@@ -1,3 +1,4 @@
+#include <algorithm>
 #include <string>
 #include <mutex>
 #include <Interpreters/DatabaseCatalog.h>
@@ -27,6 +28,7 @@
 #include <Common/noexcept_scope.h>
 #include <Common/checkStackSize.h>
 
+#include <base/isSharedPtrUnique.h>
 #include <boost/range/adaptor/map.hpp>
 
 #include "config.h"
@@ -1197,7 +1199,7 @@ void DatabaseCatalog::dequeueDroppedTableCleanup(StorageID table_id)
 
     /// It's unsafe to create another instance while the old one exists
     /// We cannot wait on shared_ptr's refcount, so it's busy wait
-    while (!dropped_table.table.unique())
+    while (!isSharedPtrUnique(dropped_table.table))
         std::this_thread::sleep_for(std::chrono::milliseconds(100));
     dropped_table.table.reset();
 
@@ -1237,7 +1239,7 @@ void DatabaseCatalog::dropTableDataTask()
         size_t tables_in_use_count = 0;
         auto it = std::find_if(tables_marked_dropped.begin(), tables_marked_dropped.end(), [&](const auto & elem)
         {
-            bool not_in_use = !elem.table || elem.table.unique();
+            bool not_in_use = !elem.table || isSharedPtrUnique(elem.table);
             bool old_enough = elem.drop_time <= current_time;
             min_drop_time = std::min(min_drop_time, elem.drop_time);
             tables_in_use_count += !not_in_use;

src/Interpreters/ExternalLoader.h

@@ -2,6 +2,7 @@
 
 #include <chrono>
 #include <functional>
+#include <exception>
 #include <unordered_map>
 #include <base/types.h>
 #include <Interpreters/IExternalLoadable.h>

src/Interpreters/Session.cpp

@@ -1,5 +1,6 @@
 #include <Interpreters/Session.h>
 
+#include <base/isSharedPtrUnique.h>
 #include <Access/AccessControl.h>
 #include <Access/Credentials.h>
 #include <Access/ContextAccess.h>
@@ -130,7 +131,7 @@ public:
 
             LOG_TRACE(log, "Reuse session from storage with session_id: {}, user_id: {}", key.second, key.first);
 
-            if (!session.unique())
+            if (!isSharedPtrUnique(session))
                 throw Exception(ErrorCodes::SESSION_IS_LOCKED, "Session {} is locked by a concurrent client", session_id);
             return {session, false};
         }
@@ -156,7 +157,7 @@ public:
             return;
         }
 
-        if (!it->second.unique())
+        if (!isSharedPtrUnique(it->second))
             throw Exception(ErrorCodes::LOGICAL_ERROR, "Cannot close session {} with refcount {}", session_id, it->second.use_count());
 
         sessions.erase(it);

src/Parsers/ASTFunction.cpp

@@ -408,25 +408,26 @@ void ASTFunction::formatImplWithoutAlias(const FormatSettings & settings, Format
         {
             const char * operators[] =
             {
-                "multiply",        " * ",
-                "divide",          " / ",
-                "modulo",          " % ",
-                "plus",            " + ",
-                "minus",           " - ",
-                "notEquals",       " != ",
-                "lessOrEquals",    " <= ",
-                "greaterOrEquals", " >= ",
-                "less",            " < ",
-                "greater",         " > ",
-                "equals",          " = ",
-                "like",            " LIKE ",
-                "ilike",           " ILIKE ",
-                "notLike",         " NOT LIKE ",
-                "notILike",        " NOT ILIKE ",
-                "in",              " IN ",
-                "notIn",           " NOT IN ",
-                "globalIn",        " GLOBAL IN ",
-                "globalNotIn",     " GLOBAL NOT IN ",
+                "multiply",          " * ",
+                "divide",            " / ",
+                "modulo",            " % ",
+                "plus",              " + ",
+                "minus",             " - ",
+                "notEquals",         " != ",
+                "lessOrEquals",      " <= ",
+                "greaterOrEquals",   " >= ",
+                "less",              " < ",
+                "greater",           " > ",
+                "equals",            " = ",
+                "isNotDistinctFrom", " <=> ",
+                "like",              " LIKE ",
+                "ilike",             " ILIKE ",
+                "notLike",           " NOT LIKE ",
+                "notILike",          " NOT ILIKE ",
+                "in",                " IN ",
+                "notIn",             " NOT IN ",
+                "globalIn",          " GLOBAL IN ",
+                "globalNotIn",       " GLOBAL NOT IN ",
                 nullptr
             };
 

src/Parsers/ASTTablesInSelectQuery.cpp

@@ -243,7 +243,7 @@ void ASTTableJoin::formatImplAfterTable(const FormatSettings & settings, FormatS
 void ASTTableJoin::formatImpl(const FormatSettings & settings, FormatState & state, FormatStateStacked frame) const
 {
     formatImplBeforeTable(settings, state, frame);
-    settings.ostr << " ... ";
+    settings.ostr << " ...";
     formatImplAfterTable(settings, state, frame);
 }
 

src/Planner/PlannerJoinTree.cpp

@@ -647,7 +647,8 @@ JoinTreeQueryPlan buildQueryPlanForTableExpression(QueryTreeNodePtr table_expres
         auto table_expression_query_info = select_query_info;
         table_expression_query_info.table_expression = table_expression;
         table_expression_query_info.filter_actions_dag = table_expression_data.getFilterActions();
-        table_expression_query_info.analyzer_can_use_parallel_replicas_on_follower = table_node == planner_context->getGlobalPlannerContext()->parallel_replicas_table;
+        table_expression_query_info.current_table_chosen_for_reading_with_parallel_replicas
+            = table_node == planner_context->getGlobalPlannerContext()->parallel_replicas_table;
 
         size_t max_streams = settings.max_threads;
         size_t max_threads_execute_query = settings.max_threads;
@@ -862,6 +863,15 @@ JoinTreeQueryPlan buildQueryPlanForTableExpression(QueryTreeNodePtr table_expres
                 from_stage = storage->getQueryProcessingStage(
                     query_context, select_query_options.to_stage, storage_snapshot, table_expression_query_info);
 
+                /// It is just a safety check needed until we have a proper sending plan to replicas.
+                /// If we have a non-trivial storage like View it might create its own Planner inside read(), run findTableForParallelReplicas()
+                /// and find some other table that might be used for reading with parallel replicas. It will lead to errors.
+                const bool other_table_already_chosen_for_reading_with_parallel_replicas
+                    = planner_context->getGlobalPlannerContext()->parallel_replicas_table
+                    && !table_expression_query_info.current_table_chosen_for_reading_with_parallel_replicas;
+                if (other_table_already_chosen_for_reading_with_parallel_replicas)
+                    planner_context->getMutableQueryContext()->setSetting("allow_experimental_parallel_reading_from_replicas", Field(0));
+
                 storage->read(
                     query_plan,
                     columns_names,

src/Planner/PlannerJoins.cpp

@@ -528,7 +528,7 @@ JoinClausesAndActions buildJoinClausesAndActions(
         size_t join_clause_key_nodes_size = join_clause.getLeftKeyNodes().size();
 
         if (join_clause_key_nodes_size == 0)
-            throw Exception(ErrorCodes::INVALID_JOIN_ON_EXPRESSION, "JOIN {} cannot get JOIN keys",
+            throw Exception(ErrorCodes::INVALID_JOIN_ON_EXPRESSION, "Cannot determine join keys in {}",
                 join_node.formatASTForErrorMessage());
 
         for (size_t i = 0; i < join_clause_key_nodes_size; ++i)

src/Processors/Formats/Impl/BinaryRowInputFormat.cpp

@@ -4,6 +4,7 @@
 #include <Formats/FormatFactory.h>
 #include <Formats/registerWithNamesAndTypes.h>
 #include <DataTypes/DataTypeFactory.h>
+#include <DataTypes/DataTypesBinaryEncoding.h>
 
 namespace DB
 {
@@ -55,10 +56,25 @@ std::vector<String> BinaryFormatReader<with_defaults>::readNames()
 template <bool with_defaults>
 std::vector<String> BinaryFormatReader<with_defaults>::readTypes()
 {
-    auto types = readHeaderRow();
-    for (const auto & type_name : types)
-        read_data_types.push_back(DataTypeFactory::instance().get(type_name));
-    return types;
+    read_data_types.reserve(read_columns);
+    Names type_names;
+    if (format_settings.binary.decode_types_in_binary_format)
+    {
+        type_names.reserve(read_columns);
+        for (size_t i = 0; i < read_columns; ++i)
+        {
+            read_data_types.push_back(decodeDataType(*in));
+            type_names.push_back(read_data_types.back()->getName());
+        }
+    }
+    else
+    {
+        type_names = readHeaderRow();
+        for (const auto & type_name : type_names)
+            read_data_types.push_back(DataTypeFactory::instance().get(type_name));
+    }
+
+    return type_names;
 }
 
 template <bool with_defaults>

src/Processors/Formats/Impl/BinaryRowOutputFormat.cpp

@@ -2,6 +2,7 @@
 #include <IO/WriteHelpers.h>
 #include <Columns/IColumn.h>
 #include <DataTypes/IDataType.h>
+#include <DataTypes/DataTypesBinaryEncoding.h>
 #include <Processors/Formats/Impl/BinaryRowOutputFormat.h>
 #include <Formats/FormatFactory.h>
 #include <Formats/registerWithNamesAndTypes.h>
@@ -35,9 +36,15 @@ void BinaryRowOutputFormat::writePrefix()
 
     if (with_types)
     {
-        for (size_t i = 0; i < columns; ++i)
+        if (format_settings.binary.encode_types_in_binary_format)
         {
-            writeStringBinary(header.safeGetByPosition(i).type->getName(), out);
+            for (size_t i = 0; i < columns; ++i)
+                encodeDataType(header.safeGetByPosition(i).type, out);
+        }
+        else
+        {
+            for (size_t i = 0; i < columns; ++i)
+                writeStringBinary(header.safeGetByPosition(i).type->getName(), out);
         }
     }
 }

src/Processors/Formats/Impl/CSVRowInputFormat.cpp

@@ -616,7 +616,7 @@ void registerCSVSchemaReader(FormatFactory & factory)
             {
                 String result = getAdditionalFormatInfoByEscapingRule(settings, FormatSettings::EscapingRule::CSV);
                 if (!with_names)
-                    result += fmt::format(", column_names_for_schema_inference={}, try_detect_header={}", settings.column_names_for_schema_inference, settings.csv.try_detect_header);
+                    result += fmt::format(", column_names_for_schema_inference={}, try_detect_header={}, skip_first_lines={}", settings.column_names_for_schema_inference, settings.csv.try_detect_header, settings.csv.skip_first_lines);
                 return result;
             });
         }

src/Processors/Formats/Impl/NativeFormat.cpp

@@ -21,9 +21,7 @@ public:
               buf,
               header_,
               0,
-              settings.skip_unknown_fields,
-              settings.null_as_default,
-              settings.native.allow_types_conversion,
+              settings,
               settings.defaults_for_omitted_fields ? &block_missing_values : nullptr))
         , header(header_) {}
 
@@ -72,9 +70,9 @@ private:
 class NativeOutputFormat final : public IOutputFormat
 {
 public:
-    NativeOutputFormat(WriteBuffer & buf, const Block & header, UInt64 client_protocol_version = 0)
+    NativeOutputFormat(WriteBuffer & buf, const Block & header, const FormatSettings & settings, UInt64 client_protocol_version = 0)
         : IOutputFormat(header, buf)
-        , writer(buf, client_protocol_version, header)
+        , writer(buf, client_protocol_version, header, settings)
     {
     }
 
@@ -103,14 +101,17 @@ private:
 class NativeSchemaReader : public ISchemaReader
 {
 public:
-    explicit NativeSchemaReader(ReadBuffer & in_) : ISchemaReader(in_) {}
+    explicit NativeSchemaReader(ReadBuffer & in_, const FormatSettings & settings_) : ISchemaReader(in_), settings(settings_) {}
 
     NamesAndTypesList readSchema() override
     {
-        auto reader = NativeReader(in, 0);
+        auto reader = NativeReader(in, 0, settings);
         auto block = reader.read();
         return block.getNamesAndTypesList();
     }
+
+private:
+    const FormatSettings settings;
 };
 
 
@@ -134,16 +135,16 @@ void registerOutputFormatNative(FormatFactory & factory)
         const Block & sample,
         const FormatSettings & settings)
     {
-        return std::make_shared<NativeOutputFormat>(buf, sample, settings.client_protocol_version);
+        return std::make_shared<NativeOutputFormat>(buf, sample, settings, settings.client_protocol_version);
     });
 }
 
 
 void registerNativeSchemaReader(FormatFactory & factory)
 {
-    factory.registerSchemaReader("Native", [](ReadBuffer & buf, const FormatSettings &)
+    factory.registerSchemaReader("Native", [](ReadBuffer & buf, const FormatSettings & settings)
     {
-        return std::make_shared<NativeSchemaReader>(buf);
+        return std::make_shared<NativeSchemaReader>(buf, settings);
     });
 }
 

src/Processors/Formats/Impl/TabSeparatedRowInputFormat.cpp

@@ -440,9 +440,10 @@ void registerTSVSchemaReader(FormatFactory & factory)
                         settings, is_raw ? FormatSettings::EscapingRule::Raw : FormatSettings::EscapingRule::Escaped);
                     if (!with_names)
                         result += fmt::format(
-                            ", column_names_for_schema_inference={}, try_detect_header={}",
+                            ", column_names_for_schema_inference={}, try_detect_header={}, skip_first_lines={}",
                             settings.column_names_for_schema_inference,
-                            settings.tsv.try_detect_header);
+                            settings.tsv.try_detect_header,
+                            settings.tsv.skip_first_lines);
                     return result;
                 });
             }

src/Server/TCPHandler.cpp

@@ -2107,6 +2107,7 @@ void TCPHandler::initBlockOutput(const Block & block)
             *state.maybe_compressed_out,
             client_tcp_protocol_version,
             block.cloneEmpty(),
+            std::nullopt,
             !query_settings.low_cardinality_allow_in_native_format);
     }
 }
@@ -2121,6 +2122,7 @@ void TCPHandler::initLogsBlockOutput(const Block & block)
             *out,
             client_tcp_protocol_version,
             block.cloneEmpty(),
+            std::nullopt,
             !query_settings.low_cardinality_allow_in_native_format);
     }
 }
@@ -2135,6 +2137,7 @@ void TCPHandler::initProfileEventsBlockOutput(const Block & block)
             *out,
             client_tcp_protocol_version,
             block.cloneEmpty(),
+            std::nullopt,
             !query_settings.low_cardinality_allow_in_native_format);
     }
 }

src/Storages/MergeTree/MergeTreeData.cpp

@@ -87,6 +87,7 @@
 
 #include <base/insertAtEnd.h>
 #include <base/interpolate.h>
+#include <base/isSharedPtrUnique.h>
 
 #include <algorithm>
 #include <atomic>
@@ -2464,7 +2465,7 @@ MergeTreeData::DataPartsVector MergeTreeData::grabOldParts(bool force)
             }
 
             /// Grab only parts that are not used by anyone (SELECTs for example).
-            if (!part.unique())
+            if (!isSharedPtrUnique(part))
             {
                 part->removal_state.store(DataPartRemovalState::NON_UNIQUE_OWNERSHIP, std::memory_order_relaxed);
                 skipped_parts.push_back(part->info);
@@ -4360,13 +4361,13 @@ bool MergeTreeData::tryRemovePartImmediately(DataPartPtr && part)
 
             part.reset();
 
-            if (!((*it)->getState() == DataPartState::Outdated && it->unique()))
+            if (!((*it)->getState() == DataPartState::Outdated && isSharedPtrUnique(*it)))
             {
                 if ((*it)->getState() != DataPartState::Outdated)
                     LOG_WARNING(log, "Cannot immediately remove part {} because it's not in Outdated state "
                              "usage counter {}", part_name_with_state, it->use_count());
 
-                if (!it->unique())
+                if (!isSharedPtrUnique(*it))
                     LOG_WARNING(log, "Cannot immediately remove part {} because someone using it right now "
                              "usage counter {}", part_name_with_state, it->use_count());
                 return false;
@@ -4432,7 +4433,7 @@ size_t MergeTreeData::getNumberOfOutdatedPartsWithExpiredRemovalTime() const
     for (const auto & part : outdated_parts_range)
     {
         auto part_remove_time = part->remove_time.load(std::memory_order_relaxed);
-        if (part_remove_time <= time_now && time_now - part_remove_time >= getSettings()->old_parts_lifetime.totalSeconds() && part.unique())
+        if (part_remove_time <= time_now && time_now - part_remove_time >= getSettings()->old_parts_lifetime.totalSeconds() && isSharedPtrUnique(part))
             ++res;
     }
 
@@ -8640,7 +8641,7 @@ size_t MergeTreeData::unloadPrimaryKeysOfOutdatedParts()
             /// Outdated part may be hold by SELECT query and still needs the index.
             /// This check requires lock of index_mutex but if outdated part is unique then there is no
             /// contention on it, so it's relatively cheap and it's ok to check under a global parts lock.
-            if (part.unique() && part->isIndexLoaded())
+            if (isSharedPtrUnique(part) && part->isIndexLoaded())
                 parts_to_unload_index.push_back(part);
         }
     }

src/Storages/MergeTree/ParallelReplicasReadingCoordinator.cpp

@@ -444,6 +444,9 @@ void DefaultCoordinator::doHandleInitialAllRangesAnnouncement(InitialAllRangesAn
             ErrorCodes::LOGICAL_ERROR, "Replica number ({}) is bigger than total replicas count ({})", replica_num, stats.size());
 
     ++stats[replica_num].number_of_requests;
+
+    if (replica_status[replica_num].is_announcement_received)
+        throw Exception(ErrorCodes::LOGICAL_ERROR, "Duplicate announcement received for replica number {}", replica_num);
     replica_status[replica_num].is_announcement_received = true;
 
     LOG_DEBUG(log, "Sent initial requests: {} Replicas count: {}", sent_initial_requests, replicas_count);

src/Storages/PostgreSQL/StorageMaterializedPostgreSQL.cpp

@@ -592,7 +592,8 @@ void registerStorageMaterializedPostgreSQL(StorageFactory & factory)
         auto configuration = StoragePostgreSQL::getConfiguration(args.engine_args, args.getContext());
         auto connection_info = postgres::formatConnectionString(
             configuration.database, configuration.host, configuration.port,
-            configuration.username, configuration.password);
+            configuration.username, configuration.password,
+            args.getContext()->getSettingsRef().postgresql_connection_attempt_timeout);
 
         bool has_settings = args.storage_def->settings;
         auto postgresql_replication_settings = std::make_unique<MaterializedPostgreSQLSettings>();

src/Storages/SelectQueryInfo.h

@@ -166,7 +166,7 @@ struct SelectQueryInfo
     /// It's guaranteed to be present in JOIN TREE of `query_tree`
     QueryTreeNodePtr table_expression;
 
-    bool analyzer_can_use_parallel_replicas_on_follower = false;
+    bool current_table_chosen_for_reading_with_parallel_replicas = false;
 
     /// Table expression modifiers for storage
     std::optional<TableExpressionModifiers> table_expression_modifiers;

src/Storages/StorageExternalDistributed.cpp

@@ -167,8 +167,9 @@ void registerStorageExternalDistributed(StorageFactory & factory)
                     current_configuration,
                     settings.postgresql_connection_pool_size,
                     settings.postgresql_connection_pool_wait_timeout,
-                    POSTGRESQL_POOL_WITH_FAILOVER_DEFAULT_MAX_TRIES,
-                    settings.postgresql_connection_pool_auto_close_connection);
+                    settings.postgresql_connection_pool_retries,
+                    settings.postgresql_connection_pool_auto_close_connection,
+                    settings.postgresql_connection_attempt_timeout);
                 shards.insert(std::make_shared<StoragePostgreSQL>(
                     args.table_id, std::move(pool), configuration.table, args.columns, args.constraints, String{}, context));
             }

src/Storages/StorageFile.cpp

@@ -366,12 +366,21 @@ Strings StorageFile::getPathsList(const String & table_path, const String & user
     }
     else if (path.find_first_of("*?{") == std::string::npos)
     {
-        std::error_code error;
-        size_t size = fs::file_size(path, error);
-        if (!error)
-            total_bytes_to_read += size;
+        if (!fs::is_directory(path))
+        {
+            std::error_code error;
+            size_t size = fs::file_size(path, error);
+            if (!error)
+                total_bytes_to_read += size;
 
-        paths.push_back(path);
+            paths.push_back(path);
+        }
+        else
+        {
+            /// We list non-directory files under that directory.
+            paths = listFilesWithRegexpMatching(path / fs::path("*"), total_bytes_to_read);
+            can_be_directory = false;
+        }
     }
     else
     {

src/Storages/StorageMemory.cpp

@@ -408,7 +408,7 @@ namespace
                 auto data_file_path = temp_dir / fs::path{file_paths[data_bin_pos]}.filename();
                 auto data_out_compressed = temp_disk->writeFile(data_file_path);
                 auto data_out = std::make_unique<CompressedWriteBuffer>(*data_out_compressed, CompressionCodecFactory::instance().getDefaultCodec(), max_compress_block_size);
-                NativeWriter block_out{*data_out, 0, metadata_snapshot->getSampleBlock(), false, &index};
+                NativeWriter block_out{*data_out, 0, metadata_snapshot->getSampleBlock(), std::nullopt, false, &index};
                 for (const auto & block : *blocks)
                     block_out.write(block);
                 data_out->finalize();

src/Storages/StorageMergeTree.cpp

@@ -252,7 +252,7 @@ void StorageMergeTree::read(
 
     const bool enable_parallel_reading = local_context->canUseParallelReplicasOnFollower()
         && local_context->getSettingsRef().parallel_replicas_for_non_replicated_merge_tree
-        && (!local_context->getSettingsRef().allow_experimental_analyzer || query_info.analyzer_can_use_parallel_replicas_on_follower);
+        && (!local_context->getSettingsRef().allow_experimental_analyzer || query_info.current_table_chosen_for_reading_with_parallel_replicas);
 
     if (auto plan = reader.read(
             column_names,

src/Storages/StoragePostgreSQL.cpp

@@ -613,8 +613,9 @@ void registerStoragePostgreSQL(StorageFactory & factory)
         auto pool = std::make_shared<postgres::PoolWithFailover>(configuration,
             settings.postgresql_connection_pool_size,
             settings.postgresql_connection_pool_wait_timeout,
-            POSTGRESQL_POOL_WITH_FAILOVER_DEFAULT_MAX_TRIES,
-            settings.postgresql_connection_pool_auto_close_connection);
+            settings.postgresql_connection_pool_retries,
+            settings.postgresql_connection_pool_auto_close_connection,
+            settings.postgresql_connection_attempt_timeout);
 
         return std::make_shared<StoragePostgreSQL>(
             args.table_id,

src/Storages/StorageReplicatedMergeTree.cpp

@@ -5540,7 +5540,8 @@ void StorageReplicatedMergeTree::readLocalImpl(
     const size_t num_streams)
 {
     const bool enable_parallel_reading = local_context->canUseParallelReplicasOnFollower()
-        && (!local_context->getSettingsRef().allow_experimental_analyzer || query_info.analyzer_can_use_parallel_replicas_on_follower);
+        && (!local_context->getSettingsRef().allow_experimental_analyzer
+            || query_info.current_table_chosen_for_reading_with_parallel_replicas);
 
     auto plan = reader.read(
         column_names, storage_snapshot, query_info,

src/Storages/StorageStripeLog.cpp

@@ -193,7 +193,7 @@ public:
         storage.saveFileSizes(lock);
 
         size_t initial_data_size = storage.file_checker.getFileSize(storage.data_file_path);
-        block_out = std::make_unique<NativeWriter>(*data_out, 0, metadata_snapshot->getSampleBlock(), false, &storage.indices, initial_data_size);
+        block_out = std::make_unique<NativeWriter>(*data_out, 0, metadata_snapshot->getSampleBlock(), std::nullopt, false, &storage.indices, initial_data_size);
     }
 
     String getName() const override { return "StripeLogSink"; }

src/TableFunctions/TableFunctionPostgreSQL.cpp

@@ -80,8 +80,9 @@ void TableFunctionPostgreSQL::parseArguments(const ASTPtr & ast_function, Contex
         *configuration,
         settings.postgresql_connection_pool_size,
         settings.postgresql_connection_pool_wait_timeout,
-        POSTGRESQL_POOL_WITH_FAILOVER_DEFAULT_MAX_TRIES,
-        settings.postgresql_connection_pool_auto_close_connection);
+        settings.postgresql_connection_pool_retries,
+        settings.postgresql_connection_pool_auto_close_connection,
+        settings.postgresql_connection_attempt_timeout);
 }
 
 }

tests/ci/ci_config.py

@@ -311,42 +311,42 @@ class CI:
             random_bucket="parrepl_with_sanitizer",
         ),
         JobNames.STATELESS_TEST_ASAN: CommonJobConfigs.STATELESS_TEST.with_properties(
-            required_builds=[BuildNames.PACKAGE_ASAN], num_batches=4
+            required_builds=[BuildNames.PACKAGE_ASAN], num_batches=2
         ),
         JobNames.STATELESS_TEST_TSAN: CommonJobConfigs.STATELESS_TEST.with_properties(
-            required_builds=[BuildNames.PACKAGE_TSAN], num_batches=5
+            required_builds=[BuildNames.PACKAGE_TSAN], num_batches=2
         ),
         JobNames.STATELESS_TEST_MSAN: CommonJobConfigs.STATELESS_TEST.with_properties(
-            required_builds=[BuildNames.PACKAGE_MSAN], num_batches=6
+            required_builds=[BuildNames.PACKAGE_MSAN], num_batches=3
         ),
         JobNames.STATELESS_TEST_UBSAN: CommonJobConfigs.STATELESS_TEST.with_properties(
-            required_builds=[BuildNames.PACKAGE_UBSAN], num_batches=2
+            required_builds=[BuildNames.PACKAGE_UBSAN], num_batches=1
         ),
         JobNames.STATELESS_TEST_DEBUG: CommonJobConfigs.STATELESS_TEST.with_properties(
-            required_builds=[BuildNames.PACKAGE_DEBUG], num_batches=5
+            required_builds=[BuildNames.PACKAGE_DEBUG], num_batches=2
         ),
         JobNames.STATELESS_TEST_RELEASE: CommonJobConfigs.STATELESS_TEST.with_properties(
             required_builds=[BuildNames.PACKAGE_RELEASE],
         ),
         JobNames.STATELESS_TEST_RELEASE_COVERAGE: CommonJobConfigs.STATELESS_TEST.with_properties(
-            required_builds=[BuildNames.PACKAGE_RELEASE_COVERAGE], num_batches=6
+            required_builds=[BuildNames.PACKAGE_RELEASE_COVERAGE], num_batches=5
         ),
         JobNames.STATELESS_TEST_AARCH64: CommonJobConfigs.STATELESS_TEST.with_properties(
             required_builds=[BuildNames.PACKAGE_AARCH64],
             runner_type=Runners.FUNC_TESTER_ARM,
         ),
         JobNames.STATELESS_TEST_OLD_ANALYZER_S3_REPLICATED_RELEASE: CommonJobConfigs.STATELESS_TEST.with_properties(
-            required_builds=[BuildNames.PACKAGE_RELEASE], num_batches=4
+            required_builds=[BuildNames.PACKAGE_RELEASE], num_batches=3
         ),
         JobNames.STATELESS_TEST_S3_DEBUG: CommonJobConfigs.STATELESS_TEST.with_properties(
-            required_builds=[BuildNames.PACKAGE_DEBUG], num_batches=6
+            required_builds=[BuildNames.PACKAGE_DEBUG], num_batches=2
         ),
         JobNames.STATELESS_TEST_AZURE_ASAN: CommonJobConfigs.STATELESS_TEST.with_properties(
-            required_builds=[BuildNames.PACKAGE_ASAN], num_batches=4, release_only=True
+            required_builds=[BuildNames.PACKAGE_ASAN], num_batches=2, release_only=True
         ),
         JobNames.STATELESS_TEST_S3_TSAN: CommonJobConfigs.STATELESS_TEST.with_properties(
             required_builds=[BuildNames.PACKAGE_TSAN],
-            num_batches=5,
+            num_batches=3,
         ),
         JobNames.STRESS_TEST_DEBUG: CommonJobConfigs.STRESS_TEST.with_properties(
             required_builds=[BuildNames.PACKAGE_DEBUG],

tests/clickhouse-test

@@ -709,9 +709,9 @@ def get_localzone():
 
 class SettingsRandomizer:
     settings = {
-        "max_insert_threads": lambda: (
-            0 if random.random() < 0.5 else random.randint(1, 16)
-        ),
+        "max_insert_threads": lambda: 32
+        if random.random() < 0.03
+        else random.randint(1, 3),
         "group_by_two_level_threshold": threshold_generator(0.2, 0.2, 1, 1000000),
         "group_by_two_level_threshold_bytes": threshold_generator(
             0.2, 0.2, 1, 50000000
@@ -727,7 +727,7 @@ class SettingsRandomizer:
         "prefer_localhost_replica": lambda: random.randint(0, 1),
         "max_block_size": lambda: random.randint(8000, 100000),
         "max_joined_block_size_rows": lambda: random.randint(8000, 100000),
-        "max_threads": lambda: random.randint(1, 64),
+        "max_threads": lambda: 64 if random.random() < 0.03 else random.randint(1, 3),
         "optimize_append_index": lambda: random.randint(0, 1),
         "optimize_if_chain_to_multiif": lambda: random.randint(0, 1),
         "optimize_if_transform_strings_to_enum": lambda: random.randint(0, 1),
@@ -1217,6 +1217,11 @@ class TestCase:
         ):
             return FailureReason.OBJECT_STORAGE
 
+        elif "no-batch" in tags and (
+            args.run_by_hash_num is not None or args.run_by_hash_total is not None
+        ):
+            return FailureReason.SKIP
+
         elif tags:
             for build_flag in args.build_flags:
                 if "no-" + build_flag in tags:
@@ -1447,8 +1452,7 @@ class TestCase:
         description_full = messages[result.status]
         description_full += self.print_test_time(result.total_time)
         if result.reason is not None:
-            description_full += " - "
-            description_full += result.reason.value
+            description_full += f"\nReason: {result.reason.value} "
 
         description_full += result.description
 
@@ -1575,10 +1579,11 @@ class TestCase:
         # pylint:disable-next=consider-using-with; TODO: fix
         proc = Popen(command, shell=True, env=os.environ, start_new_session=True)
 
-        while (
-            datetime.now() - start_time
-        ).total_seconds() < args.timeout and proc.poll() is None:
-            sleep(0.01)
+        try:
+            proc.wait(args.timeout)
+        except subprocess.TimeoutExpired:
+            # Whether the test timed out will be decided later
+            pass
 
         debug_log = ""
         if os.path.exists(self.testcase_args.debug_log_file):
@@ -1600,6 +1605,44 @@ class TestCase:
         # Normalize hostname in stdout file.
         replace_in_file(self.stdout_file, socket.gethostname(), "localhost")
 
+        if os.environ.get("CLICKHOUSE_PORT_TCP"):
+            replace_in_file(
+                self.stdout_file,
+                f"PORT {os.environ['CLICKHOUSE_PORT_TCP']}",
+                "PORT 9000",
+            )
+            replace_in_file(
+                self.stdout_file,
+                f"localhost	{os.environ['CLICKHOUSE_PORT_TCP']}",
+                "localhost	9000",
+            )
+
+        if os.environ.get("CLICKHOUSE_PORT_TCP_SECURE"):
+            replace_in_file(
+                self.stdout_file,
+                f"PORT {os.environ['CLICKHOUSE_PORT_TCP_SECURE']}",
+                "PORT 9440",
+            )
+            replace_in_file(
+                self.stdout_file,
+                f"localhost	{os.environ['CLICKHOUSE_PORT_TCP_SECURE']}",
+                "localhost	9440",
+            )
+
+        if os.environ.get("CLICKHOUSE_PATH"):
+            replace_in_file(
+                self.stdout_file,
+                os.environ["CLICKHOUSE_PATH"],
+                "/var/lib/clickhouse",
+            )
+
+        if os.environ.get("CLICKHOUSE_PORT_HTTPS"):
+            replace_in_file(
+                self.stdout_file,
+                f"https://localhost:{os.environ['CLICKHOUSE_PORT_HTTPS']}/",
+                "https://localhost:8443/",
+            )
+
         stdout = ""
         if os.path.exists(self.stdout_file):
             with open(self.stdout_file, "rb") as stdfd:
@@ -2056,8 +2099,13 @@ class GlobalTimeout(Exception):
     pass
 
 
-def run_tests_array(all_tests_with_params: Tuple[List[str], int, TestSuite]):
-    all_tests, num_tests, test_suite = all_tests_with_params
+def run_tests_array(all_tests_with_params: Tuple[List[str], int, TestSuite, bool]):
+    (
+        all_tests,
+        num_tests,
+        test_suite,
+        is_concurrent,
+    ) = all_tests_with_params
     global stop_time
     global exit_code
     global server_died
@@ -2100,14 +2148,12 @@ def run_tests_array(all_tests_with_params: Tuple[List[str], int, TestSuite]):
     failures_chain = 0
     start_time = datetime.now()
 
-    is_concurrent = multiprocessing.current_process().name != "MainProcess"
-
     client_options = get_additional_client_options(args)
 
     if num_tests > 0:
         about = "about " if is_concurrent else ""
         proc_name = multiprocessing.current_process().name
-        print(f"\nRunning {about}{num_tests} {test_suite.suite} tests ({proc_name}).\n")
+        print(f"Running {about}{num_tests} {test_suite.suite} tests ({proc_name}).")
 
     while True:
         if all_tests:
@@ -2128,16 +2174,17 @@ def run_tests_array(all_tests_with_params: Tuple[List[str], int, TestSuite]):
 
         try:
             description = ""
-            test_cace_name = removesuffix(test_case.name, ".gen", ".sql") + ": "
-            if not is_concurrent:
+            test_case_name = removesuffix(test_case.name, ".gen", ".sql") + ": "
+
+            if is_concurrent or args.run_sequential_tests_in_parallel:
+                description = f"{test_case_name:72}"
+            else:
                 sys.stdout.flush()
-                sys.stdout.write(f"{test_cace_name:72}")
+                sys.stdout.write(f"{test_case_name:72}")
                 # This flush is needed so you can see the test name of the long
                 # running test before it will finish. But don't do it in parallel
                 # mode, so that the lines don't mix.
                 sys.stdout.flush()
-            else:
-                description = f"{test_cace_name:72}"
 
             while True:
                 test_result = test_case.run(
@@ -2372,6 +2419,35 @@ def extract_key(key: str) -> str:
     )[1]
 
 
+def override_envs(*args_, **kwargs):
+    global args
+    args.client += " --port 19000"
+    args.http_port = 18123
+    args.https_port = 18443
+
+    updated_env = {
+        "CLICKHOUSE_CONFIG": "/etc/clickhouse-server3/config.xml",
+        "CLICKHOUSE_CONFIG_DIR": "/etc/clickhouse-server3",
+        "CLICKHOUSE_CONFIG_GREP": "/etc/clickhouse-server3/preprocessed/config.xml",
+        "CLICKHOUSE_USER_FILES": "/var/lib/clickhouse3/user_files",
+        "CLICKHOUSE_SCHEMA_FILES": "/var/lib/clickhouse3/format_schemas",
+        "CLICKHOUSE_PATH": "/var/lib/clickhouse3",
+        "CLICKHOUSE_PORT_TCP": "19000",
+        "CLICKHOUSE_PORT_TCP_SECURE": "19440",
+        "CLICKHOUSE_PORT_TCP_WITH_PROXY": "19010",
+        "CLICKHOUSE_PORT_HTTP": "18123",
+        "CLICKHOUSE_PORT_HTTPS": "18443",
+        "CLICKHOUSE_PORT_INTERSERVER": "19009",
+        "CLICKHOUSE_PORT_KEEPER": "19181",
+        "CLICKHOUSE_PORT_PROMTHEUS_PORT": "19988",
+        "CLICKHOUSE_PORT_MYSQL": "19004",
+        "CLICKHOUSE_PORT_POSTGRESQL": "19005",
+    }
+    os.environ.update(updated_env)
+
+    run_tests_array(*args_, **kwargs)
+
+
 def do_run_tests(jobs, test_suite: TestSuite):
     if jobs > 1 and len(test_suite.parallel_tests) > 0:
         print(
@@ -2400,24 +2476,55 @@ def do_run_tests(jobs, test_suite: TestSuite):
         for job in range(jobs):
             range_ = job * batch_size, job * batch_size + batch_size
             batch = test_suite.parallel_tests[range_[0] : range_[1]]
-            parallel_tests_array.append((batch, batch_size, test_suite))
+            parallel_tests_array.append((batch, batch_size, test_suite, True))
 
         try:
-            with multiprocessing.Pool(processes=jobs) as pool:
+            with multiprocessing.Pool(processes=jobs + 1) as pool:
                 future = pool.map_async(run_tests_array, parallel_tests_array)
+
+                if args.run_sequential_tests_in_parallel:
+                    # Run parallel tests and sequential tests at the same time
+                    # Sequential tests will use different ClickHouse instance
+                    # In this process we can safely override values in `args` and `os.environ`
+                    future_seq = pool.map_async(
+                        override_envs,
+                        [
+                            (
+                                test_suite.sequential_tests,
+                                len(test_suite.sequential_tests),
+                                test_suite,
+                                False,
+                            )
+                        ],
+                    )
+                    future_seq.wait()
+
                 future.wait()
         finally:
             pool.terminate()
             pool.close()
             pool.join()
 
-        run_tests_array(
-            (test_suite.sequential_tests, len(test_suite.sequential_tests), test_suite)
-        )
+        if not args.run_sequential_tests_in_parallel:
+            run_tests_array(
+                (
+                    test_suite.sequential_tests,
+                    len(test_suite.sequential_tests),
+                    test_suite,
+                    False,
+                )
+            )
         return len(test_suite.sequential_tests) + len(test_suite.parallel_tests)
     else:
         num_tests = len(test_suite.all_tests)
-        run_tests_array((test_suite.all_tests, num_tests, test_suite))
+        run_tests_array(
+            (
+                test_suite.all_tests,
+                num_tests,
+                test_suite,
+                False,
+            )
+        )
         return num_tests
 
 
@@ -2722,6 +2829,7 @@ def main(args):
                     f"{get_db_engine(args, db_name)}",
                     settings=get_create_database_settings(args, None),
                 )
+                break
             except HTTPError as e:
                 total_time = (datetime.now() - start_time).total_seconds()
                 if not need_retry(args, e.message, e.message, total_time):
@@ -3234,6 +3342,15 @@ def parse_args():
         help="Replace ordinary MergeTree engine with SharedMergeTree",
     )
 
+    parser.add_argument(
+        "--run-sequential-tests-in-parallel",
+        action="store_true",
+        default=False,
+        help="If `true`, tests with the tag `no-parallel` will run on a "
+        "separate ClickHouse instance in parallel with other tests. "
+        "This is used in CI to make test jobs run faster.",
+    )
+
     return parser.parse_args()
 
 

tests/config/install.sh

@@ -57,7 +57,6 @@ ln -sf $SRC_PATH/config.d/forbidden_headers.xml $DEST_SERVER_PATH/config.d/
 ln -sf $SRC_PATH/config.d/enable_keeper_map.xml $DEST_SERVER_PATH/config.d/
 ln -sf $SRC_PATH/config.d/custom_disks_base_path.xml $DEST_SERVER_PATH/config.d/
 ln -sf $SRC_PATH/config.d/display_name.xml $DEST_SERVER_PATH/config.d/
-ln -sf $SRC_PATH/config.d/reverse_dns_query_function.xml $DEST_SERVER_PATH/config.d/
 ln -sf $SRC_PATH/config.d/compressed_marks_and_index.xml $DEST_SERVER_PATH/config.d/
 ln -sf $SRC_PATH/config.d/disable_s3_env_credentials.xml $DEST_SERVER_PATH/config.d/
 ln -sf $SRC_PATH/config.d/enable_wait_for_shutdown_replicated_tables.xml $DEST_SERVER_PATH/config.d/